Merge pull request #451 in G/fastr from ~MICK.JORDAN_ORACLE.COM/fastr:bugfix/unbuilt to master

* commit '6031acb5':
  mx.fastr: remove nocompile in _set_graal_options
  remove redundant mx rcmplib
  support jars and dirs in DefaultResourceHandlerfactory
  add explicit method for accessing directory "resources"
  mx_fastr_dists: add doc comment
  properly ensure that recommended packages can be built in all situations and are included in the release

.gitignore

@@ -28,7 +28,7 @@
 /com.oracle.truffle.r.native/include/linked
 /com.oracle.truffle.r.native/fficall/jni.done
 /com.oracle.truffle.r.native/fficall/jniboot.done
-
+/com.oracle.truffle.r.native.recommended/install.recommended
 /com.oracle.truffle.r.test.native/packages/copy_recommended
 /com.oracle.truffle.r.test.native/packages/recommended
 /com.oracle.truffle.r.test.native/packages/*/lib/*

com.oracle.truffle.r.nodes.builtin/src/com/oracle/truffle/r/nodes/builtin/RBuiltinPackage.java

@@ -22,10 +22,6 @@
  */
 package com.oracle.truffle.r.nodes.builtin;
 
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -36,8 +32,8 @@ import com.oracle.truffle.api.frame.MaterializedFrame;
 import com.oracle.truffle.api.source.Source;
 import com.oracle.truffle.r.runtime.ArgumentsSignature;
 import com.oracle.truffle.r.runtime.RInternalError;
+import com.oracle.truffle.r.runtime.RSource;
 import com.oracle.truffle.r.runtime.ResourceHandlerFactory;
-import com.oracle.truffle.r.runtime.Utils;
 import com.oracle.truffle.r.runtime.builtins.RBuiltin;
 import com.oracle.truffle.r.runtime.builtins.RSpecialFactory;
 import com.oracle.truffle.r.runtime.context.Engine.ParseException;
@@ -93,7 +89,6 @@ public abstract class RBuiltinPackage {
 
     protected RBuiltinPackage(String name) {
         this.name = name;
-
         // Check for overriding R code
         ArrayList<Source> componentList = getRFiles(getName());
         if (componentList.size() > 0) {
@@ -111,22 +106,10 @@ public abstract class RBuiltinPackage {
      */
     public static ArrayList<Source> getRFiles(String pkgName) {
         ArrayList<Source> componentList = new ArrayList<>();
-        try {
-            InputStream is = ResourceHandlerFactory.getHandler().getResourceAsStream(RBuiltinPackage.class, pkgName + "/R");
-            if (is != null) {
-                try (BufferedReader r = new BufferedReader(new InputStreamReader(is))) {
-                    String line;
-                    while ((line = r.readLine()) != null) {
-                        if (line.endsWith(".r") || line.endsWith(".R")) {
-                            final String rResource = pkgName + "/R/" + line.trim();
-                            Source content = Utils.getResourceAsSource(RBuiltinPackage.class, rResource);
-                            componentList.add(content);
-                        }
-                    }
-                }
-            }
-        } catch (IOException ex) {
-            Utils.rSuicide("error loading R code from " + pkgName + " : " + ex);
+        String[] rFileContents = ResourceHandlerFactory.getHandler().getRFiles(RBuiltinPackage.class, pkgName);
+        for (String rFileContent : rFileContents) {
+            Source content = RSource.fromTextInternal(rFileContent, RSource.Internal.R_IMPL);
+            componentList.add(content);
         }
         return componentList;
     }

com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/DefaultResourceHandlerFactory.java

@@ -22,8 +22,19 @@
  */
 package com.oracle.truffle.r.runtime;
 
+import java.io.BufferedReader;
+import java.io.File;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URI;
 import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.CodeSource;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
 
 import com.oracle.truffle.r.runtime.ResourceHandlerFactory.Handler;
 
@@ -46,4 +57,59 @@ class DefaultResourceHandlerFactory extends ResourceHandlerFactory implements Ha
     protected Handler newHandler() {
         return this;
     }
+
+    @Override
+    public String[] getRFiles(Class<?> accessor, String pkgName) {
+        CodeSource source = accessor.getProtectionDomain().getCodeSource();
+        ArrayList<String> list = new ArrayList<>();
+        try {
+            URL url = source.getLocation();
+            Path sourcePath = Paths.get(url.toURI().getPath());
+            File sourceFile = sourcePath.toFile();
+            if (sourceFile.isDirectory()) {
+                InputStream is = accessor.getResourceAsStream(pkgName + "/R");
+                if (is != null) {
+                    try (BufferedReader r = new BufferedReader(new InputStreamReader(is))) {
+                        String line;
+                        while ((line = r.readLine()) != null) {
+                            if (line.endsWith(".r") || line.endsWith(".R")) {
+                                final String rResource = pkgName + "/R/" + line.trim();
+                                list.add(Utils.getResourceAsString(accessor, rResource, true));
+                            }
+                        }
+                    }
+                }
+            } else {
+                JarFile fastrJar = new JarFile(sourceFile);
+                Enumeration<JarEntry> iter = fastrJar.entries();
+                while (iter.hasMoreElements()) {
+                    JarEntry entry = iter.nextElement();
+                    String name = entry.getName();
+                    if (name.endsWith(".R") || name.endsWith(".r")) {
+                        Path p = Paths.get(name);
+                        String entryPkg = p.getName(p.getNameCount() - 3).getFileName().toString();
+                        String entryParent = p.getName(p.getNameCount() - 2).getFileName().toString();
+                        if (entryParent.equals("R") && entryPkg.equals(pkgName)) {
+                            int size = (int) entry.getSize();
+                            byte[] buf = new byte[size];
+                            InputStream is = fastrJar.getInputStream(entry);
+                            int totalRead = 0;
+                            int n;
+                            while ((n = is.read(buf, totalRead, buf.length - totalRead)) > 0) {
+                                totalRead += n;
+                            }
+                            list.add(new String(buf));
+                        }
+                    }
+                }
+            }
+            String[] result = new String[list.size()];
+            list.toArray(result);
+            return result;
+        } catch (Exception ex) {
+            Utils.rSuicide(ex.getMessage());
+            return null;
+        }
+    }
+
 }

com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/ResourceHandlerFactory.java

@@ -44,6 +44,12 @@ public abstract class ResourceHandlerFactory {
          * See {@link java.lang.Class#getResourceAsStream(String)}.
          */
         InputStream getResourceAsStream(Class<?> accessor, String name);
+
+        /**
+         * Return the contents of all "R" files (ending with ".r" or ".R") relative to
+         * {@code accessor} and {@code pkgname/R}. I.e. essentially a directory search.
+         */
+        String[] getRFiles(Class<?> accessor, String pkgName);
     }
 
     static {

com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/tools/cmpr/CompareLibR.java

-/*
- * Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-package com.oracle.truffle.r.test.tools.cmpr;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.nio.file.DirectoryStream;
-import java.nio.file.FileSystem;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.oracle.truffle.r.runtime.ResourceHandlerFactory;
-import com.oracle.truffle.r.runtime.Utils;
-
-/**
- * Compare the FastR versions of .R files in the standard packages against GnuR. Removes all
- * formatting to perform the check, replacing all whitespace (including newlines) with exactly one
- * space.
- * <p>
- * Usage:
- *
- * <pre>
- * --gnurhome path --package pkg | --files path1 path2
- * </pre>
- *
- * {@code gnurhome} is the path to the GnuR distribution. {@cpde package} gibes the package to
- * compare,e.g. {@code base}. The second form just compares the two files.
- */
-public class CompareLibR {
-
-    private static class FileContent {
-        Path path;
-        String content;
-        String flattened;
-
-        FileContent(Path path, String content) {
-            this.path = path;
-            this.content = content;
-        }
-
-        @Override
-        public String toString() {
-            return path.toString();
-        }
-    }
-
-    public static void main(String[] args) throws Exception {
-        // Checkstyle: stop system print check
-        String gnurHome = null;
-        String pkg = null;
-        String path1 = null;
-        String path2 = null;
-        boolean printPaths = false;
-        String diffApp = "diff";
-        int i = 0;
-        while (i < args.length) {
-            String arg = args[i];
-            switch (arg) {
-                case "--gnurhome":
-                    i++;
-                    gnurHome = args[i];
-                    break;
-                case "--package":
-                    i++;
-                    pkg = args[i];
-                    break;
-                case "--files":
-                    if (args.length == 3) {
-                        i++;
-                        path1 = args[i];
-                        i++;
-                        path2 = args[i];
-                    } else {
-                        usage();
-                    }
-                    break;
-                case "--paths":
-                    printPaths = true;
-                    break;
-
-                case "--diffapp":
-                    i++;
-                    diffApp = args[i];
-                    break;
-                default:
-                    usage();
-            }
-            i++;
-        }
-
-        if (gnurHome == null && path1 == null) {
-            usage();
-        }
-
-        if (path1 != null) {
-            compareFiles(path1, path2);
-        } else {
-
-            Map<String, FileContent> fastRFiles = getFastR(pkg);
-            Map<String, FileContent> gnuRFiles = getGnuR(gnurHome, pkg, fastRFiles);
-            deformat(gnuRFiles);
-            deformat(fastRFiles);
-            for (Map.Entry<String, FileContent> entry : fastRFiles.entrySet()) {
-                FileContent fastR = entry.getValue();
-                String fileName = entry.getKey();
-                FileContent gnuR = gnuRFiles.get(fileName);
-                if (gnuR == null) {
-                    System.out.println("FastR has file: " + fileName + " not found in GnuR");
-                } else {
-                    if (!fastR.flattened.equals(gnuR.flattened)) {
-                        if (printPaths) {
-                            System.out.printf("%s %s %s%n", diffApp, gnuR.toString(), replaceBin(fastR.toString()));
-                        } else {
-                            System.out.println(fileName + " differs");
-                        }
-                    } else {
-                        System.out.println(fileName + " is identical (modulo formatting)");
-                    }
-                }
-            }
-        }
-    }
-
-    private static String deformat(String s) {
-        return s.replaceAll("\\s+", " ");
-    }
-
-    private static void deformat(Map<String, FileContent> map) {
-        for (Map.Entry<String, FileContent> entry : map.entrySet()) {
-            FileContent fc = entry.getValue();
-            fc.flattened = deformat(fc.content);
-        }
-    }
-
-    private static Map<String, FileContent> getGnuR(String gnurHome, String lib, Map<String, FileContent> filter) throws IOException {
-        FileSystem fs = FileSystems.getDefault();
-        Path baseR = fs.getPath(lib, "R");
-        Path library = fs.getPath(gnurHome, "src", "library");
-        baseR = library.resolve(baseR);
-        Map<String, FileContent> result = new HashMap<>();
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(baseR)) {
-            for (Path entry : stream) {
-                String entryName = entry.getFileName().toString();
-                if (entryName.endsWith(".R") && (filter.get(entryName) != null)) {
-                    File file = entry.toFile();
-                    result.put(entryName, new FileContent(entry, readFileContent(file)));
-                }
-            }
-        }
-        return result;
-    }
-
-    private static String toFirstUpper(String input) {
-        return input.substring(0, 1).toUpperCase() + input.substring(1);
-    }
-
-    private static Map<String, FileContent> getFastR(String lib) throws Exception {
-        Class<?> klass = Class.forName("com.oracle.truffle.r.nodes.builtin." + lib + "." + toFirstUpper(lib) + "Package");
-        InputStream is = ResourceHandlerFactory.getHandler().getResourceAsStream(klass, "R");
-        Map<String, FileContent> result = new HashMap<>();
-        if (is == null) {
-            return result;
-        }
-        try (BufferedReader r = new BufferedReader(new InputStreamReader(is))) {
-            String line;
-            while ((line = r.readLine()) != null) {
-                if (line.endsWith(".r") || line.endsWith(".R")) {
-                    String fileName = line.trim();
-                    final String rResource = "R/" + fileName;
-                    URL url = klass.getResource(rResource);
-                    String content = Utils.getResourceAsString(klass, rResource, true);
-                    result.put(fileName, new FileContent(FileSystems.getDefault().getPath(url.getPath()), content));
-                }
-            }
-        }
-        return result;
-    }
-
-    private static String replaceBin(String s) {
-        return s.replace("/bin/", "/src/");
-    }
-
-    private static String readFileContent(File file) throws IOException {
-        byte[] buf = new byte[(int) file.length()];
-        try (BufferedInputStream bs = new BufferedInputStream(new FileInputStream(file))) {
-            bs.read(buf);
-        }
-        return new String(buf);
-    }
-
-    private static void writeFile(File file, String s) throws IOException {
-        try (BufferedOutputStream bs = new BufferedOutputStream(new FileOutputStream(file))) {
-            bs.write(s.getBytes());
-        }
-    }
-
-    private static void compareFiles(String path1, String path2) throws IOException {
-        String c1 = deformat(readFileContent(new File(path1)));
-        String c2 = deformat(readFileContent(new File(path2)));
-        if (c1.equals(c2)) {
-            System.out.println("files are identical (modulo formatting)");
-        } else {
-            System.out.println("files differ");
-            writeFile(new File(path1 + ".deformat"), c1);
-            writeFile(new File(path2 + ".deformat"), c2);
-        }
-    }
-
-    private static void usage() {
-        // Checkstyle: stop system print check
-        System.err.println("usage: --gnurhome path --package pkg | --files path1 path2");
-        System.exit(1);
-    }
-}

mx.fastr/mx_fastr.py

@@ -27,7 +27,7 @@ import mx
 import mx_gate
 import mx_fastr_pkgs
 import mx_fastr_dists
-from mx_fastr_dists import FastRNativeProject, FastRTestNativeProject, FastRReleaseProject #pylint: disable=unused-import
+from mx_fastr_dists import FastRNativeProject, FastRTestNativeProject, FastRReleaseProject, FastRNativeRecommendedProject #pylint: disable=unused-import
 import mx_copylib
 import mx_fastr_mkgramrd
 
@@ -90,15 +90,9 @@ def do_run_r(args, command, extraVmArgs=None, jdk=None, **kwargs):
     if not jdk:
         jdk = get_default_jdk()
 
-    vmArgs = ['-cp', mx.classpath(jdk=jdk)]
+    vmArgs = ['-cp', mx.classpath('FASTR', jdk=jdk)]
 
-    if 'nocompile' in kwargs:
-        nocompile = True
-        del kwargs['nocompile']
-    else:
-        nocompile = False
-
-    vmArgs += set_graal_options(nocompile)
+    vmArgs += set_graal_options()
 
     if extraVmArgs is None or not '-da' in extraVmArgs:
         # unless explicitly disabled we enable assertion checking
@@ -113,7 +107,7 @@ def do_run_r(args, command, extraVmArgs=None, jdk=None, **kwargs):
     return mx.run_java(vmArgs + args, jdk=jdk, **kwargs)
 
 def r_classpath(args):
-    print mx.classpath(jdk=mx.get_jdk())
+    print mx.classpath('FASTR', jdk=mx.get_jdk())
 
 def _sanitize_vmArgs(jdk, vmArgs):
     '''
@@ -137,11 +131,12 @@ def _sanitize_vmArgs(jdk, vmArgs):
         i = i + 1
     return xargs
 
-def set_graal_options(nocompile=False):
+def set_graal_options():
+    '''
+    If Graal is enabled, set some options specific to FastR
+    '''
     if _mx_graal:
         result = ['-Dgraal.InliningDepthError=500', '-Dgraal.EscapeAnalysisIterations=3', '-XX:JVMCINMethodSizeLimit=1000000']
-        if nocompile:
-            result += ['-Dgraal.TruffleCompilationThreshold=100000']
         return result
     else:
         return []
@@ -345,8 +340,6 @@ def _junit_r_harness(args, vmArgs, jdk, junitArgs):
     # no point in printing errors to file when running tests (that contain errors on purpose)
     vmArgs += ['-DR:-PrintErrorStacktracesToFile']
 
-    vmArgs += set_graal_options(nocompile=True)
-
     setREnvironment()
 
     return mx.run_java(vmArgs + junitArgs, nonZeroIsFatal=False, jdk=jdk)
@@ -511,27 +504,6 @@ def rbdiag(args):
 
     mx.run_java(['-cp', cp, 'com.oracle.truffle.r.nodes.test.RBuiltinDiagnostics'] + args)
 
-def rcmplib(args):
-    '''compare FastR library R sources against GnuR'''
-    parser = ArgumentParser(prog='mx rcmplib')
-    parser.add_argument('--gnurhome', action='store', help='path to GnuR sources', required=True)
-    parser.add_argument('--package', action='store', help='package to check', default="base")
-    parser.add_argument('--paths', action='store_true', help='print full paths of files that differ')
-    parser.add_argument('--diffapp', action='store', help='diff application', default="diff")
-    args = parser.parse_args(args)
-    cmpArgs = []
-    cmpArgs.append("--gnurhome")
-    cmpArgs.append(args.gnurhome)
-    cmpArgs.append("--package")
-    cmpArgs.append(args.package)
-    if args.paths:
-        cmpArgs.append("--paths")
-        cmpArgs.append("--diffapp")
-        cmpArgs.append(args.diffapp)
-
-    cp = mx.classpath([pcp.name for pcp in mx.projects_opt_limit_to_suites()])
-    mx.run_java(['-cp', cp, 'com.oracle.truffle.r.test.tools.cmpr.CompareLibR'] + cmpArgs)
-
 def _gnur_path():
     np = mx.project('com.oracle.truffle.r.native')
     return join(np.dir, 'gnur', r_version(), 'bin')
@@ -570,7 +542,6 @@ _commands = {
     'unittest' : [unittest, ['options']],
     'rbcheck' : [rbcheck, '--filter [gnur-only,fastr-only,both,both-diff]'],
     'rbdiag' : [rbdiag, '(builtin)* [-v] [-n] [-m] [--sweep | --sweep=lite | --sweep=total] [--mnonly] [--noSelfTest] [--matchLevel=same | --matchLevel=error] [--maxSweeps=N] [--outMaxLev=N]'],
-    'rcmplib' : [rcmplib, ['options']],
     'rrepl' : [rrepl, '[options]'],
     'rembed' : [rembed, '[options]'],
     'r-cp' : [r_classpath, '[options]'],

mx.fastr/mx_fastr_dists.py

@@ -264,6 +264,30 @@ class ReleaseBuildTask(mx.NativeBuildTask):
         rscript_launcher = join(self.subject.dir, 'src', 'Rscript_launcher')
         self._template(rscript_launcher, join(bin_dir, 'Rscript'), template_dict)
 
+class FastRNativeRecommendedProject(mx.NativeProject):
+    '''
+    This finesses an ordering problem on installing the recommended R packages.
+    These must be installed by FastR using bin/R CMD INSTALL. That will invoke a
+    nested 'mx R' invocation which requires the FASTR distribution to be available.
+    However, this dependency cannt be specified in the suite.py file so we achieve
+    it here by ensuring that it is built prior to the native.recommended project.
+    '''
+    def __init__(self, suite, name, deps, workingSets, theLicense, **args):
+        mx.NativeProject.__init__(self, suite, name, None, [], deps, workingSets, None, None, join(suite.dir, name), theLicense)
+
+    def getBuildTask(self, args):
+        return NativeRecommendedBuildTask(self, args)
+
+class NativeRecommendedBuildTask(mx.NativeBuildTask):
+    def __init__(self, project, args):
+        mx.NativeBuildTask.__init__(self, args, project)
+
+    def build(self):
+        # must archive FASTR before build so that nested mx R CMD INSTALL can execute
+        mx.archive(['@FASTR'])
+        mx.NativeBuildTask.build(self)
+
+
 class FastRArchiveParticipant:
     def __init__(self, dist):
         self.dist = dist
@@ -293,7 +317,6 @@ class FastRArchiveParticipant:
             include_dir = join(self.release_project.dir, 'include')
             shutil.rmtree(include_dir)
 
-
 def mx_post_parse_cmd_line(opts):
     for dist in mx_fastr._fastr_suite.dists:
         dist.set_archiveparticipant(FastRArchiveParticipant(dist))

mx.fastr/suite.py

@@ -261,22 +261,19 @@ suite = {
 
     "com.oracle.truffle.r.release" : {
       "sourceDirs" : ["src"],
-      "dependencies" : ["com.oracle.truffle.r.engine", "com.oracle.truffle.r.runtime.ffi", "com.oracle.truffle.r.native"],
+      "dependencies" : ["com.oracle.truffle.r.native.recommended"],
       "class" : "FastRReleaseProject",
       "output" : "com.oracle.truffle.r.release"
     },
 
     "com.oracle.truffle.r.native.recommended" : {
-      "sourceDirs" : [],
-      # these dependencies ensure that all distributions are built
-      # before the nested mx that does the CMD INSTALL runs
       "dependencies" : [
-        "com.oracle.truffle.r.release",
-        "com.oracle.truffle.r.test",
-        "com.oracle.truffle.r.test.native"
+        "com.oracle.truffle.r.native",
+        "com.oracle.truffle.r.engine",
+        "com.oracle.truffle.r.runtime.ffi"
       ],
+      "class" : "FastRNativeRecommendedProject",
       "native" : "true",
-      "output" : "com.oracle.truffle.r.native.recommended",
       "workingSets" : "FastR",
     },