another attempt to capture packages in a distribution - introduce FASTR_UNIT_TESTS_NATIVE

.gitignore

@@ -25,7 +25,8 @@
 /com.oracle.truffle.r.native/include/R_ext/*.h
 /com.oracle.truffle.r.native/include/linked
 /com.oracle.truffle.r.test.native/packages/copy_recommended
-/com.oracle.truffle.r.test/rpackages/testrlibs_user
+/com.oracle.truffle.r.test.native/packages/recommended
+/com.oracle.truffle.r.test.native/packages/*/lib/*
 /com.oracle.truffle.r.test.native/urand/lib/liburand.so
 /tmptest/
 /com.oracle.truffle.r.release/lib/

com.oracle.truffle.r.test.native/packages/Makefile

@@ -29,6 +29,13 @@ R_VERSION := $(notdir $(wildcard $(NATIVE_PROJECT)/gnur/R-*))
 GNUR_HOME := $(NATIVE_PROJECT)/gnur/$(R_VERSION)
 GNUR_RECOMMENDED := $(wildcard $(GNUR_HOME)/src/library/Recommended/*.tgz)
 
+# We have to copy the GNU R recommended packages into this project
+# to keep the mx archiver happy, which asserts that every path
+# is inside the project.
+
+# WARNING: If you add/delete anything in this project you must update mx_fastr_dists.py
+# to reflect the changes, e.g., adding a new SUBDIR
+
 all: make_subdirs copy_recommended
 
 make_subdirs:
@@ -37,7 +44,8 @@ make_subdirs:
 	done
 
 copy_recommended:
-	cp $(GNUR_RECOMMENDED) $(MX_OUTPUT_DIR)/packages
+	mkdir -p recommended
+	cp $(GNUR_RECOMMENDED) recommended
 	touch copy_recommended
 
 clean: clean_subdirs clean_recommended
@@ -46,13 +54,6 @@ clean_subdirs:
 	for dir in $(SUBDIRS); do \
 		$(MAKE) PACKAGE=$$dir -C $$dir clean || exit 1; \
 	done
-
+	
 clean_recommended:
-	for f in $(notdir $(GNUR_RECOMMENDED)); do \
-		rm -f $(MX_OUTPUT_DIR)/packages/$$f; \
-	done
-	rm -f copy_recommended
-
-
-
-	
\ No newline at end of file
+	rm -f recommended clean_recommended

com.oracle.truffle.r.test.native/packages/package.mk

@@ -23,19 +23,17 @@
 
 # This "builds" a test package, resulting in a tar file,
 # which is then loaded by the unit tests in TestRPackages.
-# To facilitate location relative to the mx "output" directory
-# of this project, the tar file is created in that location
 
 .PHONY: all
 
 PKG_FILES = $(shell find $(PACKAGE)/ -type f -name '*')
 
-PKG_TAR = $(MX_OUTPUT_DIR)/packages/$(PACKAGE).tar
+PKG_TAR = lib/$(PACKAGE).tar
 
 all: $(PKG_TAR)
 
 $(PKG_TAR): $(PKG_FILES)
-	mkdir -p $(MX_OUTPUT_DIR)/packages
+	mkdir -p lib
 	tar cf $(PKG_TAR) $(PACKAGE)
 
 clean:

com.oracle.truffle.r.test.native/urand/Makefile

@@ -33,7 +33,7 @@ endif
 
 .PHONY: all clean
 
-OBJ = $(MX_OUTPUT_DIR)/urand
+OBJ = lib
 SRC = src
 C_SOURCES := $(wildcard $(SRC)/*.c)
 # Since this library is loaded explicitly we keep a consistent

com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/ExpectedTestOutput.test

@@ -111769,7 +111769,7 @@ Error: unexpected '*' in:
 [1] TRUE
 ##com.oracle.truffle.r.test.rffi.TestUserRNG.testUserRNG
-#{ dyn.load("mxbuild/com.oracle.truffle.r.test/bin/com/oracle/truffle/r/test/urand/liburand.so"); RNGkind("user"); print(RNGkind()); set.seed(4567); runif(10) }
+#{ dyn.load("com.oracle.truffle.r.test.native/urand/lib/liburand.so"); RNGkind("user"); print(RNGkind()); set.seed(4567); runif(10) }
 [1] "user-supplied" "Inversion"
  [1] 0.45336386 0.38848030 0.94576608 0.11726267 0.21542351 0.08672997
  [7] 0.35201276 0.16919220 0.93579263 0.26084486

com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/TestBase.java

@@ -462,6 +462,7 @@ public class TestBase {
     }
 
     private static final String TEST_PROJECT = "com.oracle.truffle.r.test";
+    private static final String TEST_NATIVE_PROJECT = "com.oracle.truffle.r.test.native";
 
     /**
      * Returns a path to {@code baseName}, assumed to be nested in {@link #testProjectOutputDir}.
@@ -473,6 +474,11 @@ public class TestBase {
         return result;
     }
 
+    public static Path getNativeProjectFile(Path baseName) {
+        Path path = Paths.get(TEST_NATIVE_PROJECT, baseName.toString());
+        return path;
+    }
+
     private static void microTestFailed() {
         if (!ProcessFailedTests) {
             System.err.printf("%nMicro-test failure: %s%n", getTestContext());

com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/rffi/TestUserRNG.java

@@ -38,7 +38,7 @@ import com.oracle.truffle.r.test.TestBase;
 public class TestUserRNG extends TestBase {
     @Test
     public void testUserRNG() {
-        Path libPath = TestBase.getProjectFile(Paths.get("urand", "liburand.so"));
+        Path libPath = TestBase.getNativeProjectFile(Paths.get("urand", "lib", "liburand.so"));
         assertEval(TestBase.template("{ dyn.load(\"%0\"); RNGkind(\"user\"); print(RNGkind()); set.seed(4567); runif(10) }", new String[]{libPath.toString()}));
     }
 }

com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/rpackages/TestRPackages.java

@@ -161,12 +161,12 @@ public abstract class TestRPackages extends TestBase {
      */
     protected static class Resolver {
         Path getPath(String p) {
-            return testNativePath().resolve(p + ".tar");
+            return testNativePath().resolve(p).resolve("lib").resolve(p + ".tar");
         }
     }
 
     private static Path testNativePath() {
-        Path p = TestBase.getProjectFile(Paths.get("packages"));
+        Path p = TestBase.getNativeProjectFile(Paths.get("packages"));
         return p;
     }
 

com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/rpackages/TestRecommendedPackages.java

@@ -50,7 +50,7 @@ public class TestRecommendedPackages extends TestRPackages {
         setupInstallTestPackages(OK_PACKAGES, new Resolver() {
             @Override
             Path getPath(String p) {
-                return TestBase.getProjectFile(Paths.get("packages")).resolve(p + ".tgz");
+                return TestBase.getNativeProjectFile(Paths.get("packages")).resolve("recommended").resolve(p + ".tgz");
             }
         });
     }

mx.fastr/mx_fastr.py

@@ -494,6 +494,14 @@ class FastRNativeProject(mx_fastr_dists.DelFastRNativeProject):
     def __init__(self, suite, name, deps, workingSets, theLicense, **args):
         mx_fastr_dists.DelFastRNativeProject.__init__(self, suite, name, deps, workingSets, theLicense)
 
+class FastRTestNativeProject(mx_fastr_dists.DelFastRTestNativeProject):
+    '''
+    Custom class for building the com.oracle.truffle.r.test.native project.
+    Delegates to mx_fastr_dists.DelFastRTestNativeProject to keep this file uncluttered
+    '''
+    def __init__(self, suite, name, deps, workingSets, theLicense, **args):
+        mx_fastr_dists.DelFastRTestNativeProject.__init__(self, suite, name, deps, workingSets, theLicense)
+
 class FastRReleaseProject(mx_fastr_dists.DelFastRReleaseProject):
     '''
     Custom class for creating the FastR release project, which supports the

mx.fastr/mx_fastr_dists.py

@@ -111,6 +111,42 @@ class DelFastRNativeProject(FastRProjectAdapter):
 
         return results
 
+class DelFastRTestNativeProject(FastRProjectAdapter):
+    '''
+    Custom class for building the com.oracle.truffle.r.native project.
+    The customization is to support the creation of an exact FASTR_NATIVE_DEV distribution.
+    '''
+    def __init__(self, suite, name, deps, workingSets, theLicense, **args):
+        FastRProjectAdapter.__init__(self, suite, name, deps, workingSets, theLicense)
+
+    def getBuildTask(self, args):
+        return mx.NativeBuildTask(args, self)
+
+    def getResults(self):
+        '''
+        Capture all the files from the com.oracle.truffle.r.test.native project that are needed
+        for running unit tests in an alternate implementation.
+        '''
+        # plain files
+        results = []
+
+        self._get_files(join('packages', 'recommended'), results)
+
+        fastr_packages = []
+        fastr_packages_dir = join(self.dir, 'packages')
+        for root, dirs, _ in os.walk(fastr_packages_dir):
+            for d in dirs:
+                if d == 'recommended':
+                    continue
+                if os.path.isdir(join(root, d)):
+                    fastr_packages.append(d)
+            break
+        for p in fastr_packages:
+            results.append(join(fastr_packages_dir, p, 'lib', p + '.tar'))
+
+        results.append(join(self.dir, 'urand', 'lib', 'liburand.so'))
+        return results
+
 class DelFastRReleaseProject(FastRProjectAdapter):
     '''
     Custom class for creating the FastR release project, which supports the

mx.fastr/suite.py

@@ -279,6 +279,7 @@ suite = {
     "com.oracle.truffle.r.test.native" : {
       "sourceDirs" : [],
       "dependencies" : ["com.oracle.truffle.r.native"],
+      "class" : "FastRTestNativeProject",
       "native" : "true",
       "workingSets" : "FastR",
     },
@@ -447,6 +448,14 @@ suite = {
         "truffle:TRUFFLE_TCK",
       ],
 
+
+    },
+
+    "FASTR_UNIT_TESTS_NATIVE" : {
+      "description" : "unit tests support (from test.native project)",
+      "dependencies" : ["com.oracle.truffle.r.test.native"],
+      "distDependencies" : ["FASTR_NATIVE_DEV"],
+      "exclude" : ["GNUR", "GNU_ICONV"],
     },
 
     "FASTR_NATIVE_DEV": {