diff --git a/.gitignore b/.gitignore index 5d9988fc555a0ffe21e17fc12ad5b733bf6df86d..3eed6ca4779f8143dad9407775f4be957a28d9d6 100644 --- a/.gitignore +++ b/.gitignore @@ -28,7 +28,7 @@ /com.oracle.truffle.r.native/include/linked /com.oracle.truffle.r.native/fficall/jni.done /com.oracle.truffle.r.native/fficall/jniboot.done - +/com.oracle.truffle.r.native.recommended/install.recommended /com.oracle.truffle.r.test.native/packages/copy_recommended /com.oracle.truffle.r.test.native/packages/recommended /com.oracle.truffle.r.test.native/packages/*/lib/* diff --git a/com.oracle.truffle.r.nodes.builtin/src/com/oracle/truffle/r/nodes/builtin/RBuiltinPackage.java b/com.oracle.truffle.r.nodes.builtin/src/com/oracle/truffle/r/nodes/builtin/RBuiltinPackage.java index 06b2e17a7b9cd91938240e5a16400b54c68ef8d5..bc9a8097e0afd1b4b9d4f1119f307885cf0144fa 100644 --- a/com.oracle.truffle.r.nodes.builtin/src/com/oracle/truffle/r/nodes/builtin/RBuiltinPackage.java +++ b/com.oracle.truffle.r.nodes.builtin/src/com/oracle/truffle/r/nodes/builtin/RBuiltinPackage.java @@ -22,10 +22,6 @@ */ package com.oracle.truffle.r.nodes.builtin; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -36,8 +32,8 @@ import com.oracle.truffle.api.frame.MaterializedFrame; import com.oracle.truffle.api.source.Source; import com.oracle.truffle.r.runtime.ArgumentsSignature; import com.oracle.truffle.r.runtime.RInternalError; +import com.oracle.truffle.r.runtime.RSource; import com.oracle.truffle.r.runtime.ResourceHandlerFactory; -import com.oracle.truffle.r.runtime.Utils; import com.oracle.truffle.r.runtime.builtins.RBuiltin; import com.oracle.truffle.r.runtime.builtins.RSpecialFactory; import com.oracle.truffle.r.runtime.context.Engine.ParseException; @@ -93,7 +89,6 @@ public abstract class RBuiltinPackage { protected RBuiltinPackage(String name) { this.name = name; - // Check for overriding R code ArrayList<Source> componentList = getRFiles(getName()); if (componentList.size() > 0) { @@ -111,22 +106,10 @@ public abstract class RBuiltinPackage { */ public static ArrayList<Source> getRFiles(String pkgName) { ArrayList<Source> componentList = new ArrayList<>(); - try { - InputStream is = ResourceHandlerFactory.getHandler().getResourceAsStream(RBuiltinPackage.class, pkgName + "/R"); - if (is != null) { - try (BufferedReader r = new BufferedReader(new InputStreamReader(is))) { - String line; - while ((line = r.readLine()) != null) { - if (line.endsWith(".r") || line.endsWith(".R")) { - final String rResource = pkgName + "/R/" + line.trim(); - Source content = Utils.getResourceAsSource(RBuiltinPackage.class, rResource); - componentList.add(content); - } - } - } - } - } catch (IOException ex) { - Utils.rSuicide("error loading R code from " + pkgName + " : " + ex); + String[] rFileContents = ResourceHandlerFactory.getHandler().getRFiles(RBuiltinPackage.class, pkgName); + for (String rFileContent : rFileContents) { + Source content = RSource.fromTextInternal(rFileContent, RSource.Internal.R_IMPL); + componentList.add(content); } return componentList; } diff --git a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/DefaultResourceHandlerFactory.java b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/DefaultResourceHandlerFactory.java index 98185edaafddc48b1194ac3b0b60e38cbe8eb7ea..0f90524e98b782cbc2fed6760582fa4c96e4a0e7 100644 --- a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/DefaultResourceHandlerFactory.java +++ b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/DefaultResourceHandlerFactory.java @@ -22,8 +22,19 @@ */ package com.oracle.truffle.r.runtime; +import java.io.BufferedReader; +import java.io.File; import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.CodeSource; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; import com.oracle.truffle.r.runtime.ResourceHandlerFactory.Handler; @@ -46,4 +57,59 @@ class DefaultResourceHandlerFactory extends ResourceHandlerFactory implements Ha protected Handler newHandler() { return this; } + + @Override + public String[] getRFiles(Class<?> accessor, String pkgName) { + CodeSource source = accessor.getProtectionDomain().getCodeSource(); + ArrayList<String> list = new ArrayList<>(); + try { + URL url = source.getLocation(); + Path sourcePath = Paths.get(url.toURI().getPath()); + File sourceFile = sourcePath.toFile(); + if (sourceFile.isDirectory()) { + InputStream is = accessor.getResourceAsStream(pkgName + "/R"); + if (is != null) { + try (BufferedReader r = new BufferedReader(new InputStreamReader(is))) { + String line; + while ((line = r.readLine()) != null) { + if (line.endsWith(".r") || line.endsWith(".R")) { + final String rResource = pkgName + "/R/" + line.trim(); + list.add(Utils.getResourceAsString(accessor, rResource, true)); + } + } + } + } + } else { + JarFile fastrJar = new JarFile(sourceFile); + Enumeration<JarEntry> iter = fastrJar.entries(); + while (iter.hasMoreElements()) { + JarEntry entry = iter.nextElement(); + String name = entry.getName(); + if (name.endsWith(".R") || name.endsWith(".r")) { + Path p = Paths.get(name); + String entryPkg = p.getName(p.getNameCount() - 3).getFileName().toString(); + String entryParent = p.getName(p.getNameCount() - 2).getFileName().toString(); + if (entryParent.equals("R") && entryPkg.equals(pkgName)) { + int size = (int) entry.getSize(); + byte[] buf = new byte[size]; + InputStream is = fastrJar.getInputStream(entry); + int totalRead = 0; + int n; + while ((n = is.read(buf, totalRead, buf.length - totalRead)) > 0) { + totalRead += n; + } + list.add(new String(buf)); + } + } + } + } + String[] result = new String[list.size()]; + list.toArray(result); + return result; + } catch (Exception ex) { + Utils.rSuicide(ex.getMessage()); + return null; + } + } + } diff --git a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/ResourceHandlerFactory.java b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/ResourceHandlerFactory.java index 0fddff7ddb5511d204ab70cce66397c540d59db1..b20589fd8459804332cbe644af1209bdca750a7c 100644 --- a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/ResourceHandlerFactory.java +++ b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/ResourceHandlerFactory.java @@ -44,6 +44,12 @@ public abstract class ResourceHandlerFactory { * See {@link java.lang.Class#getResourceAsStream(String)}. */ InputStream getResourceAsStream(Class<?> accessor, String name); + + /** + * Return the contents of all "R" files (ending with ".r" or ".R") relative to + * {@code accessor} and {@code pkgname/R}. I.e. essentially a directory search. + */ + String[] getRFiles(Class<?> accessor, String pkgName); } static { diff --git a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/tools/cmpr/CompareLibR.java b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/tools/cmpr/CompareLibR.java deleted file mode 100644 index 8d03d05af1ed778b56a9e4a412d0252eef883ae2..0000000000000000000000000000000000000000 --- a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/tools/cmpr/CompareLibR.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.truffle.r.test.tools.cmpr; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.URL; -import java.nio.file.DirectoryStream; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; - -import com.oracle.truffle.r.runtime.ResourceHandlerFactory; -import com.oracle.truffle.r.runtime.Utils; - -/** - * Compare the FastR versions of .R files in the standard packages against GnuR. Removes all - * formatting to perform the check, replacing all whitespace (including newlines) with exactly one - * space. - * <p> - * Usage: - * - * <pre> - * --gnurhome path --package pkg | --files path1 path2 - * </pre> - * - * {@code gnurhome} is the path to the GnuR distribution. {@cpde package} gibes the package to - * compare,e.g. {@code base}. The second form just compares the two files. - */ -public class CompareLibR { - - private static class FileContent { - Path path; - String content; - String flattened; - - FileContent(Path path, String content) { - this.path = path; - this.content = content; - } - - @Override - public String toString() { - return path.toString(); - } - } - - public static void main(String[] args) throws Exception { - // Checkstyle: stop system print check - String gnurHome = null; - String pkg = null; - String path1 = null; - String path2 = null; - boolean printPaths = false; - String diffApp = "diff"; - int i = 0; - while (i < args.length) { - String arg = args[i]; - switch (arg) { - case "--gnurhome": - i++; - gnurHome = args[i]; - break; - case "--package": - i++; - pkg = args[i]; - break; - case "--files": - if (args.length == 3) { - i++; - path1 = args[i]; - i++; - path2 = args[i]; - } else { - usage(); - } - break; - case "--paths": - printPaths = true; - break; - - case "--diffapp": - i++; - diffApp = args[i]; - break; - default: - usage(); - } - i++; - } - - if (gnurHome == null && path1 == null) { - usage(); - } - - if (path1 != null) { - compareFiles(path1, path2); - } else { - - Map<String, FileContent> fastRFiles = getFastR(pkg); - Map<String, FileContent> gnuRFiles = getGnuR(gnurHome, pkg, fastRFiles); - deformat(gnuRFiles); - deformat(fastRFiles); - for (Map.Entry<String, FileContent> entry : fastRFiles.entrySet()) { - FileContent fastR = entry.getValue(); - String fileName = entry.getKey(); - FileContent gnuR = gnuRFiles.get(fileName); - if (gnuR == null) { - System.out.println("FastR has file: " + fileName + " not found in GnuR"); - } else { - if (!fastR.flattened.equals(gnuR.flattened)) { - if (printPaths) { - System.out.printf("%s %s %s%n", diffApp, gnuR.toString(), replaceBin(fastR.toString())); - } else { - System.out.println(fileName + " differs"); - } - } else { - System.out.println(fileName + " is identical (modulo formatting)"); - } - } - } - } - } - - private static String deformat(String s) { - return s.replaceAll("\\s+", " "); - } - - private static void deformat(Map<String, FileContent> map) { - for (Map.Entry<String, FileContent> entry : map.entrySet()) { - FileContent fc = entry.getValue(); - fc.flattened = deformat(fc.content); - } - } - - private static Map<String, FileContent> getGnuR(String gnurHome, String lib, Map<String, FileContent> filter) throws IOException { - FileSystem fs = FileSystems.getDefault(); - Path baseR = fs.getPath(lib, "R"); - Path library = fs.getPath(gnurHome, "src", "library"); - baseR = library.resolve(baseR); - Map<String, FileContent> result = new HashMap<>(); - try (DirectoryStream<Path> stream = Files.newDirectoryStream(baseR)) { - for (Path entry : stream) { - String entryName = entry.getFileName().toString(); - if (entryName.endsWith(".R") && (filter.get(entryName) != null)) { - File file = entry.toFile(); - result.put(entryName, new FileContent(entry, readFileContent(file))); - } - } - } - return result; - } - - private static String toFirstUpper(String input) { - return input.substring(0, 1).toUpperCase() + input.substring(1); - } - - private static Map<String, FileContent> getFastR(String lib) throws Exception { - Class<?> klass = Class.forName("com.oracle.truffle.r.nodes.builtin." + lib + "." + toFirstUpper(lib) + "Package"); - InputStream is = ResourceHandlerFactory.getHandler().getResourceAsStream(klass, "R"); - Map<String, FileContent> result = new HashMap<>(); - if (is == null) { - return result; - } - try (BufferedReader r = new BufferedReader(new InputStreamReader(is))) { - String line; - while ((line = r.readLine()) != null) { - if (line.endsWith(".r") || line.endsWith(".R")) { - String fileName = line.trim(); - final String rResource = "R/" + fileName; - URL url = klass.getResource(rResource); - String content = Utils.getResourceAsString(klass, rResource, true); - result.put(fileName, new FileContent(FileSystems.getDefault().getPath(url.getPath()), content)); - } - } - } - return result; - } - - private static String replaceBin(String s) { - return s.replace("/bin/", "/src/"); - } - - private static String readFileContent(File file) throws IOException { - byte[] buf = new byte[(int) file.length()]; - try (BufferedInputStream bs = new BufferedInputStream(new FileInputStream(file))) { - bs.read(buf); - } - return new String(buf); - } - - private static void writeFile(File file, String s) throws IOException { - try (BufferedOutputStream bs = new BufferedOutputStream(new FileOutputStream(file))) { - bs.write(s.getBytes()); - } - } - - private static void compareFiles(String path1, String path2) throws IOException { - String c1 = deformat(readFileContent(new File(path1))); - String c2 = deformat(readFileContent(new File(path2))); - if (c1.equals(c2)) { - System.out.println("files are identical (modulo formatting)"); - } else { - System.out.println("files differ"); - writeFile(new File(path1 + ".deformat"), c1); - writeFile(new File(path2 + ".deformat"), c2); - } - } - - private static void usage() { - // Checkstyle: stop system print check - System.err.println("usage: --gnurhome path --package pkg | --files path1 path2"); - System.exit(1); - } -} diff --git a/mx.fastr/mx_fastr.py b/mx.fastr/mx_fastr.py index 6ce83b3ed57ea55ca38e6c2579552e5d1d602a4e..a54f65a4a94929156ec1f51d691179948f41a829 100644 --- a/mx.fastr/mx_fastr.py +++ b/mx.fastr/mx_fastr.py @@ -27,7 +27,7 @@ import mx import mx_gate import mx_fastr_pkgs import mx_fastr_dists -from mx_fastr_dists import FastRNativeProject, FastRTestNativeProject, FastRReleaseProject #pylint: disable=unused-import +from mx_fastr_dists import FastRNativeProject, FastRTestNativeProject, FastRReleaseProject, FastRNativeRecommendedProject #pylint: disable=unused-import import mx_copylib import mx_fastr_mkgramrd @@ -90,15 +90,9 @@ def do_run_r(args, command, extraVmArgs=None, jdk=None, **kwargs): if not jdk: jdk = get_default_jdk() - vmArgs = ['-cp', mx.classpath(jdk=jdk)] + vmArgs = ['-cp', mx.classpath('FASTR', jdk=jdk)] - if 'nocompile' in kwargs: - nocompile = True - del kwargs['nocompile'] - else: - nocompile = False - - vmArgs += set_graal_options(nocompile) + vmArgs += set_graal_options() if extraVmArgs is None or not '-da' in extraVmArgs: # unless explicitly disabled we enable assertion checking @@ -113,7 +107,7 @@ def do_run_r(args, command, extraVmArgs=None, jdk=None, **kwargs): return mx.run_java(vmArgs + args, jdk=jdk, **kwargs) def r_classpath(args): - print mx.classpath(jdk=mx.get_jdk()) + print mx.classpath('FASTR', jdk=mx.get_jdk()) def _sanitize_vmArgs(jdk, vmArgs): ''' @@ -137,11 +131,12 @@ def _sanitize_vmArgs(jdk, vmArgs): i = i + 1 return xargs -def set_graal_options(nocompile=False): +def set_graal_options(): + ''' + If Graal is enabled, set some options specific to FastR + ''' if _mx_graal: result = ['-Dgraal.InliningDepthError=500', '-Dgraal.EscapeAnalysisIterations=3', '-XX:JVMCINMethodSizeLimit=1000000'] - if nocompile: - result += ['-Dgraal.TruffleCompilationThreshold=100000'] return result else: return [] @@ -345,8 +340,6 @@ def _junit_r_harness(args, vmArgs, jdk, junitArgs): # no point in printing errors to file when running tests (that contain errors on purpose) vmArgs += ['-DR:-PrintErrorStacktracesToFile'] - vmArgs += set_graal_options(nocompile=True) - setREnvironment() return mx.run_java(vmArgs + junitArgs, nonZeroIsFatal=False, jdk=jdk) @@ -511,27 +504,6 @@ def rbdiag(args): mx.run_java(['-cp', cp, 'com.oracle.truffle.r.nodes.test.RBuiltinDiagnostics'] + args) -def rcmplib(args): - '''compare FastR library R sources against GnuR''' - parser = ArgumentParser(prog='mx rcmplib') - parser.add_argument('--gnurhome', action='store', help='path to GnuR sources', required=True) - parser.add_argument('--package', action='store', help='package to check', default="base") - parser.add_argument('--paths', action='store_true', help='print full paths of files that differ') - parser.add_argument('--diffapp', action='store', help='diff application', default="diff") - args = parser.parse_args(args) - cmpArgs = [] - cmpArgs.append("--gnurhome") - cmpArgs.append(args.gnurhome) - cmpArgs.append("--package") - cmpArgs.append(args.package) - if args.paths: - cmpArgs.append("--paths") - cmpArgs.append("--diffapp") - cmpArgs.append(args.diffapp) - - cp = mx.classpath([pcp.name for pcp in mx.projects_opt_limit_to_suites()]) - mx.run_java(['-cp', cp, 'com.oracle.truffle.r.test.tools.cmpr.CompareLibR'] + cmpArgs) - def _gnur_path(): np = mx.project('com.oracle.truffle.r.native') return join(np.dir, 'gnur', r_version(), 'bin') @@ -570,7 +542,6 @@ _commands = { 'unittest' : [unittest, ['options']], 'rbcheck' : [rbcheck, '--filter [gnur-only,fastr-only,both,both-diff]'], 'rbdiag' : [rbdiag, '(builtin)* [-v] [-n] [-m] [--sweep | --sweep=lite | --sweep=total] [--mnonly] [--noSelfTest] [--matchLevel=same | --matchLevel=error] [--maxSweeps=N] [--outMaxLev=N]'], - 'rcmplib' : [rcmplib, ['options']], 'rrepl' : [rrepl, '[options]'], 'rembed' : [rembed, '[options]'], 'r-cp' : [r_classpath, '[options]'], diff --git a/mx.fastr/mx_fastr_dists.py b/mx.fastr/mx_fastr_dists.py index 9acd6286367bc69f50269b21f174ae654c7b1a8b..3892b38573639e87efb45c1d060868e6b15934f2 100644 --- a/mx.fastr/mx_fastr_dists.py +++ b/mx.fastr/mx_fastr_dists.py @@ -264,6 +264,30 @@ class ReleaseBuildTask(mx.NativeBuildTask): rscript_launcher = join(self.subject.dir, 'src', 'Rscript_launcher') self._template(rscript_launcher, join(bin_dir, 'Rscript'), template_dict) +class FastRNativeRecommendedProject(mx.NativeProject): + ''' + This finesses an ordering problem on installing the recommended R packages. + These must be installed by FastR using bin/R CMD INSTALL. That will invoke a + nested 'mx R' invocation which requires the FASTR distribution to be available. + However, this dependency cannt be specified in the suite.py file so we achieve + it here by ensuring that it is built prior to the native.recommended project. + ''' + def __init__(self, suite, name, deps, workingSets, theLicense, **args): + mx.NativeProject.__init__(self, suite, name, None, [], deps, workingSets, None, None, join(suite.dir, name), theLicense) + + def getBuildTask(self, args): + return NativeRecommendedBuildTask(self, args) + +class NativeRecommendedBuildTask(mx.NativeBuildTask): + def __init__(self, project, args): + mx.NativeBuildTask.__init__(self, args, project) + + def build(self): + # must archive FASTR before build so that nested mx R CMD INSTALL can execute + mx.archive(['@FASTR']) + mx.NativeBuildTask.build(self) + + class FastRArchiveParticipant: def __init__(self, dist): self.dist = dist @@ -293,7 +317,6 @@ class FastRArchiveParticipant: include_dir = join(self.release_project.dir, 'include') shutil.rmtree(include_dir) - def mx_post_parse_cmd_line(opts): for dist in mx_fastr._fastr_suite.dists: dist.set_archiveparticipant(FastRArchiveParticipant(dist)) diff --git a/mx.fastr/suite.py b/mx.fastr/suite.py index 5d1379f0555a68c8ba42ceda0d8e440b547b8e63..da713a08244b552f29416732cf7676b27770bb3f 100644 --- a/mx.fastr/suite.py +++ b/mx.fastr/suite.py @@ -261,22 +261,19 @@ suite = { "com.oracle.truffle.r.release" : { "sourceDirs" : ["src"], - "dependencies" : ["com.oracle.truffle.r.engine", "com.oracle.truffle.r.runtime.ffi", "com.oracle.truffle.r.native"], + "dependencies" : ["com.oracle.truffle.r.native.recommended"], "class" : "FastRReleaseProject", "output" : "com.oracle.truffle.r.release" }, "com.oracle.truffle.r.native.recommended" : { - "sourceDirs" : [], - # these dependencies ensure that all distributions are built - # before the nested mx that does the CMD INSTALL runs "dependencies" : [ - "com.oracle.truffle.r.release", - "com.oracle.truffle.r.test", - "com.oracle.truffle.r.test.native" + "com.oracle.truffle.r.native", + "com.oracle.truffle.r.engine", + "com.oracle.truffle.r.runtime.ffi" ], + "class" : "FastRNativeRecommendedProject", "native" : "true", - "output" : "com.oracle.truffle.r.native.recommended", "workingSets" : "FastR", },