diff --git a/documentation/dev/testing.md b/documentation/dev/testing.md
index 45c3893fa88ea26b2006ced2ac56bb560bbe110a..a5a2718be1a78c80dc8ebd3ccca71ea017dbd3a9 100644
--- a/documentation/dev/testing.md
+++ b/documentation/dev/testing.md
@@ -186,7 +186,9 @@ Override the blacklist and attempt to install the `Rcpp` package. N.B. The regul
#### The mx pkgtest command
-The `mx pkgtest` command is a wrapper on `mx installpkgs` that forces the `--run-tests` option and also executes the same tests under GnuR and compares the results. In order to run the tests under GnuR, the `gnur` suite must be installed as a sibling to `fastr`.
+The `mx pkgtest` command is a wrapper on `mx installpkgs` that forces the `--run-tests` option and also executes the same tests under GnuR and compares the results. The packages are installed into `lib.install.packages.fastr` and `lib.install.packages.gnur`, respectively and the test results are stored in `test.fastr` and `test.gnur`, respectively. The differences between the results, computed using `diff -r`, are stored per package in the `test.diffs` directory. All these directories are cleaned and re-created at the start of the run.
+
+By default the local build of FastR and the internal GNU R that is built as part of the FastR build are used to run the tests. However, when `FASTR_GRAALVM` is set to the location of a `GraalVM` binary installation, that is used for FastR and the `gnur` suite must be installed and built as a sibling to `fastr`.
#### Running/Debugging Tests Locally
diff --git a/mx.fastr/mx_fastr_pkgs.py b/mx.fastr/mx_fastr_pkgs.py
index 9a489fd72b27198b597783accf6f8175d418a315..0e7fe3319fc8c82c75c152cd0defda535e9c73e4 100644
--- a/mx.fastr/mx_fastr_pkgs.py
+++ b/mx.fastr/mx_fastr_pkgs.py
@@ -91,10 +91,14 @@ def _create_libinstall(rvm, test_installed):
# install_tmp = join(_fastr_suite_dir(), "install.tmp")
shutil.rmtree(install_tmp, ignore_errors=True)
os.mkdir(install_tmp)
+ _create_testdot(rvm)
+ return libinstall, install_tmp
+
+def _create_testdot(rvm):
testdir = join(_fastr_suite_dir(), "test." + rvm)
shutil.rmtree(testdir, ignore_errors=True)
os.mkdir(testdir)
- return libinstall, install_tmp, testdir
+ return testdir
def _log_step(state, step, rvariant):
if not quiet:
@@ -149,8 +153,8 @@ def pkgtest(args):
'''
test_installed = '--no-install' in args
- fastr_libinstall, fastr_install_tmp, fastr_testdir = _create_libinstall('fastr', test_installed)
- gnur_libinstall, gnur_install_tmp, gnur_testdir = _create_libinstall('gnur', test_installed)
+ fastr_libinstall, fastr_install_tmp = _create_libinstall('fastr', test_installed)
+ gnur_libinstall, gnur_install_tmp = _create_libinstall('gnur', test_installed)
if "--quiet" in args:
global quiet
@@ -227,7 +231,7 @@ def pkgtest(args):
install_args += ['--print-install-status']
_log_step('BEGIN', 'install/test', 'FastR')
- # Currently installpkgs does not set a return code (in install.packages.packages.R)
+ # Currently installpkgs does not set a return code (in install.packages.R)
rc = _installpkgs(install_args, nonZeroIsFatal=False, env=env, out=out, err=out)
if rc == 100:
# fatal error connecting to package repo
@@ -253,36 +257,14 @@ def pkgtest(args):
rc = rc | 2
print '{0}: {1}'.format(pkg, test_status.status)
- # tar up the test results
- tar_tests(fastr_testdir)
- tar_tests(gnur_testdir)
+ diffdir = _create_testdot('diffs')
+ for pkg, _ in out.test_info.iteritems():
+ diff_file = join(diffdir, pkg)
+ subprocess.call(['diff', '-r', _pkg_testdir('fastr', pkg), _pkg_testdir('gnur', pkg)], stdout=open(diff_file, 'w'))
shutil.rmtree(fastr_install_tmp, ignore_errors=True)
return rc
-def tar_tests(testdir):
- if os.environ.has_key('FASTR_TEST_GZIP'):
- test_tar = testdir + '.tar'
- subprocess.call(['tar', 'cf', test_tar, os.path.basename(testdir)])
- if os.path.exists(test_tar + '.gz'):
- os.remove(test_tar + '.gz')
- subprocess.call(['gzip', test_tar])
- else:
- # workaround for lack of support for accessing gz files
- with open(testdir + '.agg', 'w') as o:
- for root, _, files in os.walk(testdir):
- for f in files:
- ext = os.path.splitext(f)[1]
- if f == 'test_time' or f == 'testfile_status' or ext == '.pdf' or ext == '.prev' or ext == '.save':
- continue
- absfile = join(root, f)
- relfile = relpath(absfile, _fastr_suite_dir())
- o.write('#### ' + relfile + '\n')
- with open(absfile) as inp:
- text = inp.read()
- o.write(text)
-
-
class TestFileStatus:
'''
Records the status of a test file. status is either "OK" or "FAILED".
@@ -341,7 +323,7 @@ def _args_to_forward_to_gnur(args):
def _gnur_install_test(forwarded_args, pkgs, gnur_libinstall, gnur_install_tmp):
'''
- Install/test with GNU R exactly those packages that installe3d correctly with FastR.
+ Install/test with GNU R exactly those packages that installed correctly with FastR.
N.B. That means that regardless of how the packages were specified to pkgtest
we always use a --pkg-filelist' arg to GNU R
'''