use FastR-unique file when validating R_HOME

com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/REnvVars.java

@@ -68,8 +68,7 @@ public final class REnvVars implements RContext.ContextState {
         envVars.put("R_SHARE_DIR", fileSystem.getPath(rHome, "share").toString());
         String rLibsUserProperty = envVars.get("R_LIBS_USER");
         if (rLibsUserProperty == null) {
-            String os = System.getProperty("os.name");
-            if (os.contains("Mac OS")) {
+            if (isMacOS()) {
                 rLibsUserProperty = "~/Library/R/%v/library";
             } else {
                 rLibsUserProperty = "~/R/%p-library/%v";
@@ -140,6 +139,11 @@ public final class REnvVars implements RContext.ContextState {
         return val != null ? val : envVars.get(var.toUpperCase());
     }
 
+    private static boolean isMacOS() {
+        String os = System.getProperty("os.name");
+        return os.contains("Mac OS");
+    }
+
     private static final String R_HOME = "R_HOME";
 
     /**
@@ -147,6 +151,13 @@ public final class REnvVars implements RContext.ContextState {
      */
     private static String rHome;
 
+    /**
+     * Returns a file that only exists in a FastR {@code R_HOME}.
+     */
+    private static String markerFile() {
+        return "libjniboot." + (isMacOS() ? "dylib" : "so");
+    }
+
     /**
      * Returns the value of the {@code R_HOME} environment variable (setting it in the unusual case
      * where it it is not set by the initiating shell scripts. This is called very early in the
@@ -162,7 +173,7 @@ public final class REnvVars implements RContext.ContextState {
             } else {
                 rHomePath = Paths.get(rHome);
             }
-            if (!validateRHome(rHomePath)) {
+            if (!validateRHome(rHomePath, markerFile())) {
                 Utils.rSuicide("R_HOME is not set correctly");
             }
             rHome = rHomePath.toString();
@@ -181,8 +192,9 @@ public final class REnvVars implements RContext.ContextState {
      */
     private static Path getRHomePath() {
         Path path = Paths.get(REnvVars.class.getProtectionDomain().getCodeSource().getLocation().getPath()).getParent();
+        String markerFile = markerFile();
         while (path != null) {
-            if (validateRHome(path)) {
+            if (validateRHome(path, markerFile)) {
                 return path;
             }
             path = path.getParent();
@@ -193,12 +205,12 @@ public final class REnvVars implements RContext.ContextState {
     /**
      * Sanity check on the expected structure of an {@code R_HOME}.
      */
-    private static boolean validateRHome(Path path) {
+    private static boolean validateRHome(Path path, String markerFile) {
         if (path == null) {
             return false;
         }
-        Path bin = path.resolve("bin");
-        return Files.exists(bin) && Files.isDirectory(bin) && Files.exists(bin.resolve("R"));
+        Path lib = path.resolve("lib");
+        return Files.exists(lib) && Files.isDirectory(lib) && Files.exists(lib.resolve(markerFile));
     }
 
     private void checkRHome() {