package testing: run tests in gnur in sibling repos for comparison

ci.hocon

@@ -49,31 +49,17 @@ gateStyle : ${common} {
   ]
 }
 
-benchRun: ${common} ${java8Downloads} {
-# workaround mx bug
-  environment: ${common.environment} {
-    MX_IDE : "eclipse"
-  }
-  run : [
-    ["git", "clone", "ssh://git@ol-bitbucket.us.oracle.com:7999/g/r-apptests.git",  "../r-apptests"]
-    ["git", "clone", "ssh://git@ol-bitbucket.us.oracle.com:7999/g/r-benchmarks.git",  "../r-benchmarks"]
-    ["git", "clone", "ssh://git@ol-bitbucket.us.oracle.com:7999/g/graal-core.git",  "../graal-core"]
-    [mx, --vm, server, --vmbuild, product, --dynamicimport, graal-core, --strict-compliance, build, -p, --force-javac]
-    ["mx", "--vm", "server", "--dynamicimport", "r-benchmarks,r-apptests,graal-core", "benchmark", "--results-file", "results.json"]
-    ["bench-uploader.py", "--url", "$BENCH_SERVER_URL", "results.json"]
-  ]
-  timelimit : "3:00:00"
-}
-
-
 # currently disabled gate commands: FindBugs,Checkheaders,Distribution Overlap Check,BuildJavaWithEcj
 
 # tests a (default) number of packages known to install correctly
 pkgtest: ${common} ${java8Downloads} {
   timelimit : "3:00:00"
+  setup : [
+    ["git", "clone", "--depth", "1", ${repoBase}"gnur.git",  "../gnur"]
+  ]
   run : [
-    ["mx", "build"]
-    ["mx", "pkgtest", "--print-ok-installs", "--ok-only"]
+    ["mx", "--dynamicimport", "gnur", "build"]
+    ["mx", "--dynamicimport", "gnur", "pkgtest", "--print-ok-installs", "--ok-only"]
   ]
 }
 
@@ -82,7 +68,7 @@ pkginstall: ${common} ${java8Downloads} {
   timelimit : "6:00:00"
   run : [
     ["mx", "build"]
-    ["mx", "pkgtest", "--invert-pkgset", "--ok-only", "--print-ok-installs", "--install-dependents-first", "--pkg-count", "1000"]
+    ["mx", "pkgtest", "--invert-pkgset", "--ok-only", "--install-only", "--print-ok-installs", "--install-dependents-first", "--pkg-count", "1000"]
   ]
 }
 
@@ -90,7 +76,6 @@ builds = [
   ${gateTest}       {capabilities : [linux, amd64, gate, post-merge],   name: "gate-test-linux-amd64"}
   ${gateStyle}      {capabilities : [linux, amd64, gate, post-merge],   name: "gate-style-linux-amd64"}
 #  ${gateTest}       {capabilities : [linux, sparcv9, gate, post-merge], name: "gate-test-linux-sparcv9"}
-  ${benchRun}       {capabilities : [linux, amd64, gate],   name: "bench-test-linux-amd64"}
-#  ${pkgtest}        {capabilities : [linux, amd64, gate], name: "pkgtest-test-linux-amd64"}
+  ${pkgtest}        {capabilities : [linux, amd64, gate], name: "pkgtest-test-linux-amd64"}
 #  ${pkginstall}        {capabilities : [linux, amd64, gate], name: "pkginstall-test-linux-amd64"}
 ]

com.oracle.truffle.r.test.cran/r/install.cran.packages.R

@@ -231,7 +231,7 @@ set.package.blacklist <- function() {
 	    # not set on command line
 		blacklist.file <<- Sys.getenv("PACKAGE_BLACKLIST", unset="package.blacklist")
 	}
-	if (!create.blacklist.file) {
+	if (!create.blacklist.file && !ignore.blacklist) {
 		if (!file.exists(blacklist.file)) {
 			cat(paste("blacklist file", blacklist.file, "does not exist, creating\n"))
 			create.blacklist.file <<- T
@@ -448,8 +448,8 @@ install.pkgs <- function(pkgnames, dependents.install=F) {
 
 
 get.blacklist <- function() {
-	get.initial.package.blacklist()
 	if (create.blacklist.file) {
+		get.initial.package.blacklist()
 		blacklist <- create.blacklist()
 		writeLines(sort(blacklist), blacklist.file)
 	} else {
@@ -603,7 +603,7 @@ test.package <- function(pkgname) {
 }
 
 is.fastr <- function() {
-	"package:fastr" %in% search()
+	exists(".fastr.context.get", baseenv())
 }
 
 system.test <- function(pkgname) {

mx.fastr/mx_fastr.py

@@ -576,14 +576,20 @@ def rcmplib(args):
     mx.run_java(['-cp', cp, 'com.oracle.truffle.r.test.tools.cmpr.CompareLibR'] + cmpArgs)
 
 def _cran_test_project():
-    return mx.project('com.oracle.truffle.r.test.cran').dir
+    return 'com.oracle.truffle.r.test.cran'
+
+def _cran_test_project_dir():
+    return mx.project(_cran_test_project()).dir
 
 def installpkgs(args):
     _installpkgs(args)
 
+def _installpkgs_script():
+    cran_test = _cran_test_project_dir()
+    return join(cran_test, 'r', 'install.cran.packages.R')
+
 def _installpkgs(args, out=None, err=None):
-    cran_test = _cran_test_project()
-    script = join(cran_test, 'r', 'install.cran.packages.R')
+    script = _installpkgs_script()
     return rscript([script] + args, out=out, err=err)
 
 _commands = {

mx.fastr/mx_fastr_pkgs.py

@@ -22,11 +22,26 @@
 #
 
 from argparse import ArgumentParser
-from os.path import join, abspath
+from os.path import join, abspath, exists, relpath
 import shutil, os, re
 import mx
 import mx_fastr
 
+def _mx_gnur():
+    return mx.suite('gnur')
+
+def _create_libinstall(s):
+    '''Create lib.install.cran/install.tmp for suite s
+    '''
+    libinstall = join(s.dir, "lib.install.cran")
+    # make sure its empty
+    shutil.rmtree(libinstall, ignore_errors=True)
+    os.mkdir(libinstall)
+    install_tmp = join(s.dir, "install.tmp")
+    shutil.rmtree(install_tmp, ignore_errors=True)
+    os.mkdir(install_tmp)
+    return libinstall, install_tmp
+
 def pkgtest(args):
     '''used for package installation/testing'''
     parser = ArgumentParser(prog='r test')
@@ -34,27 +49,26 @@ def pkgtest(args):
     parser.add_argument('--install-only', action='store_true', help='just install packages, do not test')
     # sundry options understood by installpkgs R code
     parser.add_argument('--pkg-count', action='store', help='number of packages to install/test', default='100')
+    parser.add_argument('--pkg-filelist', action='store', help='pass --pkg-filelist')
     parser.add_argument('--ignore-blacklist', action='store_true', help='pass --ignore-blacklist')
     parser.add_argument('--install-dependents-first', action='store_true', help='pass -install-dependents-first')
     parser.add_argument('--print-ok-installs', action='store_true', help='pass --print-ok-installs')
     parser.add_argument('--invert-pkgset', action='store_true', help='pass --invert-pkgset')
     args = parser.parse_args(args)
 
-    libinstall = abspath("lib.install.cran")
-    # make sure its empty
-    shutil.rmtree(libinstall, ignore_errors=True)
-    os.mkdir(libinstall)
-    install_tmp = "install.tmp"
-    shutil.rmtree(install_tmp, ignore_errors=True)
-    os.mkdir(install_tmp)
+    libinstall, install_tmp = _create_libinstall(mx.suite('fastr'))
     os.environ["TMPDIR"] = install_tmp
     os.environ['R_LIBS_USER'] = libinstall
     stacktrace_args = ['--J', '@-DR:-PrintErrorStacktracesToFile -DR:+PrintErrorStacktraces']
 
-    install_args = ['--pkg-count', args.pkg_count]
+    install_args = []
+    if args.pkg_count and not args.pkg_filelist:
+        install_args += ['--pkg-count', args.pkg_count]
     if args.ok_only:
         # only install/test packages that have been successfully installed
-        install_args += ['--pkg-filelist', join(mx_fastr._cran_test_project(), 'ok.packages')]
+        install_args += ['--pkg-filelist', join(mx_fastr._cran_test_project_dir(), 'ok.packages')]
+    if args.pkg_filelist:
+        install_args += ['--pkg-filelist', args.pkg_filelist]
     if not args.install_only:
         install_args += ['--run-tests']
     if args.ignore_blacklist:
@@ -66,11 +80,6 @@ def pkgtest(args):
     if args.invert_pkgset:
         install_args += ['--invert-pkgset']
 
-    class TestStatus:
-        def __init__(self):
-            self.status = "unknown"
-            self.filelist = []
-
     class OutputCapture:
         def __init__(self):
             self.install_data = None
@@ -116,30 +125,140 @@ def pkgtest(args):
                     begin_end = test_match.group(1)
                     pkg_name = test_match.group(2)
                     if begin_end == "END":
-                        pkg_testdir = join(mx.suite('fastr').dir, 'test', pkg_name)
-                        for root, _, files in os.walk(pkg_testdir):
-                            if not self.test_info.has_key(pkg_name):
-                                self.test_info[pkg_name] = TestStatus()
-                            for f in files:
-                                self.test_info[pkg_name].filelist.append(join(root, f))
+                        _get_test_outputs('fastr', pkg_name, self.test_info)
 
     out = OutputCapture()
 
-    rc = mx_fastr.installpkgs(stacktrace_args + install_args, out=out, err=out)
-    _set_test_status(out.test_info)
+    # install and (optionally) test the packages
+    rc = mx_fastr._installpkgs(stacktrace_args + install_args, out=out, err=out)
+    if not args.install_only:
+        # in order to compare the test output with GnuR we have to install/test the same
+        # set of packages with GnuR, which must be present as a sibling suite
+        ok_pkgs = [k for k, v in out.install_status.iteritems() if v]
+        _gnur_install_test(ok_pkgs)
+        _set_test_status(out.test_info)
+        print 'Test Status'
+        for pkg, test_status in out.test_info.iteritems():
+            print '{0}: {1}'.format(pkg, test_status.status)
 
     shutil.rmtree(install_tmp, ignore_errors=True)
     return rc
 
-def _set_test_status(test_info_dict):
-    for _, test_info in test_info_dict.iteritems():
-        for test_file in test_info.filelist:
-            with open(test_file, 'r') as f:
-                fastr_content = f.read()
-            # where to get GnuR test output?
-            gnur_content = ""
+class TestFileStatus:
+    '''
+    Records the status of a test file. status is either "OK" or "FAILED".
+    The latter means that the file had a .fail extension.
+    '''
+    def __init__(self, status, abspath):
+        self.status = status
+        self.abspath = abspath
+
+class TestStatus:
+    '''Records the test status of a package. status ends up as either "OK" or "FAILED",
+    unless GnuR also failed in which case it stays as UNKNOWN.
+    The testfile_outputs dict is keyed by the relative path of the output file to
+    the 'test/pkgname' directory. The value is an instance of TestFileStatus.
+    '''
+    def __init__(self):
+        self.status = "UNKNOWN"
+        self.testfile_outputs = dict()
+
+def _pkg_testdir(suite, pkg_name):
+    return join(mx.suite(suite).dir, 'test', pkg_name)
+def _get_test_outputs(suite, pkg_name, test_info):
+    pkg_testdir = _pkg_testdir(suite, pkg_name)
+    for root, _, files in os.walk(pkg_testdir):
+        if not test_info.has_key(pkg_name):
+            test_info[pkg_name] = TestStatus()
+        for f in files:
+            ext = os.path.splitext(f)[1]
+            if ext == '.R' or ext == '.prev':
+                continue
+            # suppress .pdf's for now (we can't compare them)
+            if ext == '.pdf':
+                continue
+            status = "OK"
+            if ext == '.fail':
+                # some fatal error during the test
+                status = "FAILED"
+                f = os.path.splitext(f)[0]
+
+            absfile = join(root, f)
+            relfile = relpath(absfile, pkg_testdir)
+            test_info[pkg_name].testfile_outputs[relfile] = TestFileStatus(status, absfile)
+
+def _gnur_install_test(pkgs):
+        gnur = _mx_gnur().extensions
+        gnur_packages = join(_mx_gnur().dir, 'gnur.packages')
+        with open(gnur_packages, 'w') as f:
+            for pkg in pkgs:
+                f.write(pkg)
+                f.write('\n')
+        # clone the cran test project into gnur
+        gnur_cran_test_project_dir = join(_mx_gnur().dir, mx_fastr._cran_test_project())
+        if not exists(gnur_cran_test_project_dir):
+            shutil.copytree(mx_fastr._cran_test_project_dir(), gnur_cran_test_project_dir)
+        gnur_libinstall, gnur_install_tmp = _create_libinstall(_mx_gnur())
+        gnur_cmd = [gnur._gnur_rscript_path(), mx_fastr._installpkgs_script()]
+        gnur_cmd += ['--pkg-filelist', gnur_packages]
+        gnur_cmd += ['--run-tests']
+        gnur_cmd += ['--ignore-blacklist']
+        env = os.environ.copy()
+        env["TMPDIR"] = gnur_install_tmp
+        env['R_LIBS_USER'] = gnur_libinstall
+        del env['R_HOME']
+        mx.run(gnur_cmd, nonZeroIsFatal=False, cwd=_mx_gnur().dir, env=env)
+
+def _set_test_status(fastr_test_info):
+    def _failed_outputs(outputs):
+        '''
+        return True iff outputs has any .fail files
+        '''
+        for _, testfile_status in outputs.iteritems():
+            if testfile_status.status == "FAILED":
+                return True
+        return False
+
+    gnur_test_info = dict()
+    for pkg, _ in fastr_test_info.iteritems():
+        _get_test_outputs('gnur', pkg, gnur_test_info)
+
+    # gnur is definitive so drive off that
+    for pkg in gnur_test_info.keys():
+        gnur_test_status = gnur_test_info[pkg]
+        fastr_test_status = fastr_test_info[pkg]
+        gnur_outputs = gnur_test_status.testfile_outputs
+        fastr_outputs = fastr_test_status.testfile_outputs
+        if _failed_outputs(gnur_outputs):
+            print "{0}: GnuR test had .fail outputs".format(pkg)
+            continue
+
+        if _failed_outputs(fastr_outputs):
+            print "{0}: FastR test had .fail outputs".format(pkg)
+            fastr_test_status.status = "FAILED"
+            continue
+
+
+        fastr_test_status.status = "OK" # optimistic
+        for gnur_test_output_relpath, gnur_testfile_status in gnur_outputs.iteritems():
+            if not gnur_test_output_relpath in fastr_outputs:
+                fastr_test_status.status = "FAILED"
+                print "{0}: FastR is missing output file: {1}".format(pkg, gnur_test_output_relpath)
+                break
+
+            gnur_content = None
+            with open(gnur_testfile_status.abspath) as f:
+                gnur_content = f.readlines()
+            fastr_content = None
+            fastr_testfile_status = fastr_outputs[gnur_test_output_relpath]
+            with open(fastr_testfile_status.abspath) as f:
+                fastr_content = f.readlines()
+
             result = _fuzzy_compare(gnur_content, fastr_content)
-            test_info.status = "OK" if result == 0 else "FAILED"
+            if result != 0:
+                fastr_test_status.status = "FAILED"
+                print "{0}: FastR output mismatch: {1}".format(pkg, gnur_test_output_relpath)
+                break
 
 def _find_start(content):
     marker = "Type 'q()' to quit R."