From f5ae0da8db8ca48125cc2dc1d9433cec885a8936 Mon Sep 17 00:00:00 2001
From: Mick Jordan <mick.jordan@oracle.com>
Date: Fri, 30 Sep 2016 12:00:31 -0700
Subject: [PATCH] various minor changes for clean gate build
Makefile typo
---
ci.hocon | 11 ++-
.../truffle/r/engine/TruffleRLanguage.java | 1 +
com.oracle.truffle.r.native/fficall/Makefile | 6 +-
.../fficall/src/jniboot/jniboot.c | 13 +--
.../gnur/Makefile.libs | 18 ++--
.../truffle/r/runtime/ffi/jni/JNI_C.java | 9 +-
.../truffle/r/runtime/ffi/jni/JNI_PCRE.java | 5 -
.../truffle/r/runtime/ffi/jni/JNI_Zip.java | 5 -
.../oracle/truffle/r/runtime/ffi/ZipRFFI.java | 2 +-
mx.fastr/mx_copylib.py | 93 +++++++++++++++++++
mx.fastr/mx_fastr.py | 14 ++-
mx.fastr/mx_findlib.py | 52 -----------
12 files changed, 140 insertions(+), 89 deletions(-)
create mode 100644 mx.fastr/mx_copylib.py
delete mode 100644 mx.fastr/mx_findlib.py
diff --git a/ci.hocon b/ci.hocon
index 4e44a4cf6f..bd89ac5666 100644
--- a/ci.hocon
+++ b/ci.hocon
@@ -1,6 +1,6 @@
java7 : {name : oraclejdk, version : "7", platformspecific: true}
#java8 : {name : oraclejdk, version : "8u66", platformspecific: true}
-java8 : {name : labsjdk, version : "8u92-jvmci-0.20", platformspecific: true}
+java8 : {name : labsjdk, version : "8u92-jvmci-0.21", platformspecific: true}
logfiles : [
"fastr_errors.log"
"com.oracle.truffle.r.native/gnur/R-*/gnur_configure.log"
@@ -20,7 +20,10 @@ common : {
gcc-build-essentials : ">=4.9.1" # GCC 4.9.0 fails on cluster
readline : "==6.3"
}
+ # TODO this will eventually relocate elsewhere
environment : {
+ PKG_INCLUDE_FLAGS_OVERRIDE : """"-I/cm/shared/apps/zlib/1.2.8/include -I/cm/shared/apps/bzip2/1.0.6/include -I/cm/shared/apps/xz/5.2.2/include -I/cm/shared/apps/pcre/8.38/include -I/cm/shared/apps/curl/7.50.1/include""""
+ PKG_LDFLAGS_OVERRIDE : """"-L/cm/shared/apps/zlib/1.2.8/lib -L/cm/shared/apps/bzip2/1.0.6/lib -L/cm/shared/apps/xz/5.2.2/lib -L/cm/shared/apps/pcre/8.38/lib -L/cm/shared/apps/curl/7.50.1//lib""""
}
logs: ${logfiles}
timelimit : "1:00:00"
@@ -39,7 +42,7 @@ java8Downloads : {
gateTest : ${common} ${java8Downloads} {
run : [
- ${gateCmd} ["Versions,JDKReleaseInfo,Pylint,Canonicalization Check,BuildJavaWithJavac,UnitTests: ExpectedTestOutput file check,UnitTests"]
+ ${gateCmd} ["Versions,JDKReleaseInfo,Pylint,Canonicalization Check,BuildJavaWithJavac,LDD, UnitTests: ExpectedTestOutput file check,UnitTests"]
]
}
@@ -49,7 +52,7 @@ gateTestDarwin : ${java8Downloads} {
"pip:pylint" : "==1.1.0"
"xz" : ""
}
- environment : ${common.environment} {
+ environment : {
PATH : "/usr/local/bin:$JAVA_HOME/bin:$PATH"
F77: "/usr/local/bin/gfortran-4.9"
}
@@ -57,7 +60,7 @@ gateTestDarwin : ${java8Downloads} {
timelimit : "1:00:00"
run : [
- ${gateCmd} ["Versions,JDKReleaseInfo,Pylint,Canonicalization Check,BuildJavaWithJavac,UnitTests: ExpectedTestOutput file check,UnitTests"]
+ ${gateCmd} ["Versions,JDKReleaseInfo,Pylint,Canonicalization Check,BuildJavaWithJavac,LDD, UnitTests: ExpectedTestOutput file check,UnitTests"]
]
}
diff --git a/com.oracle.truffle.r.engine/src/com/oracle/truffle/r/engine/TruffleRLanguage.java b/com.oracle.truffle.r.engine/src/com/oracle/truffle/r/engine/TruffleRLanguage.java
index 4a5c2c8f91..2c70874945 100644
--- a/com.oracle.truffle.r.engine/src/com/oracle/truffle/r/engine/TruffleRLanguage.java
+++ b/com.oracle.truffle.r.engine/src/com/oracle/truffle/r/engine/TruffleRLanguage.java
@@ -79,6 +79,7 @@ public final class TruffleRLanguage extends TruffleLanguage<RContext> {
RPackageSource.initialize();
} catch (Throwable t) {
+ t.printStackTrace();
Utils.rSuicide("error during R language initialization");
}
}
diff --git a/com.oracle.truffle.r.native/fficall/Makefile b/com.oracle.truffle.r.native/fficall/Makefile
index 6e549b23ed..8b63c07966 100644
--- a/com.oracle.truffle.r.native/fficall/Makefile
+++ b/com.oracle.truffle.r.native/fficall/Makefile
@@ -43,13 +43,17 @@ VERSION_FLAGS := -current_version $(R_VERSION) -compatibility_version $(R_VERSIO
endif
BLAS_TARGET := $(FASTR_LIB_DIR)/libRblas$(DYLIB_EXT)
+LAPACK_TARGET := $(FASTR_LIB_DIR)/libRlapack$(DYLIB_EXT)
all: $(R_LIB) $(BOOTJNI_LIB)
$(R_LIB): objs
- $(DYLIB_LD) $(DYLIB_LDFLAGS) -o $(R_LIB) $(wildcard lib/*.o) -L $(GNUR_HOME)/lib -lRblas $(VERSION_FLAGS)
ifeq ($(OS_NAME),Darwin)
+ $(DYLIB_LD) $(DYLIB_LDFLAGS) -o $(R_LIB) $(wildcard lib/*.o) -L$(FASTR_LIB_DIR) -lRblas -lRlapack -lpcre -lz $(VERSION_FLAGS)
install_name_tool -change libRblas.dylib $(BLAS_TARGET) $(R_LIB)
+ install_name_tool -change libRlapack.dylib $(LAPACK_TARGET) $(R_LIB)
+else
+ $(DYLIB_LD) $(DYLIB_LDFLAGS) -Wl,-rpath,$(FASTR_LIB_DIR) -o $(R_LIB) $(wildcard lib/*.o) -L$(FASTR_LIB_DIR) -lRblas -lRlapack -lpcre -lz
endif
objs:
diff --git a/com.oracle.truffle.r.native/fficall/src/jniboot/jniboot.c b/com.oracle.truffle.r.native/fficall/src/jniboot/jniboot.c
index 47efbbee27..270dd2c82f 100644
--- a/com.oracle.truffle.r.native/fficall/src/jniboot/jniboot.c
+++ b/com.oracle.truffle.r.native/fficall/src/jniboot/jniboot.c
@@ -27,21 +27,22 @@
#include <dlfcn.h>
#include <jni.h>
+// It seems that an internal (JVM) dlsym call can occur between a call to these functions and dlerror
+// (probably resolving the JNI dlerror symbol, so we capture it here (N.B. depends on single
+// threaded limitation).
+
+static char *last_dlerror;
+
JNIEXPORT jlong JNICALL
Java_com_oracle_truffle_r_runtime_ffi_jni_JNI_1Base_native_1dlopen(JNIEnv *env, jclass c, jstring jpath, jboolean local, jboolean now) {
const char *path = (*env)->GetStringUTFChars(env, jpath, NULL);
int flags = (local ? RTLD_LOCAL : RTLD_GLOBAL) | (now ? RTLD_NOW : RTLD_LAZY);
void *handle = dlopen(path, flags);
+ last_dlerror = dlerror();
(*env)->ReleaseStringUTFChars(env, jpath, path);
return (jlong) handle;
}
-// It seems that an internal (JVM) dlsym call can occur between a call to this dlsym and dlerror
-// (probably resolving the JNI dlerror symbol, so we capture it here (N.B. depends on single
-// threaded limitation).
-
-static char *last_dlerror;
-
JNIEXPORT jlong JNICALL
Java_com_oracle_truffle_r_runtime_ffi_jni_JNI_1Base_native_1dlsym(JNIEnv *env, jclass c, jlong handle, jstring jsymbol) {
const char *symbol = (*env)->GetStringUTFChars(env, jsymbol, NULL);
diff --git a/com.oracle.truffle.r.native/gnur/Makefile.libs b/com.oracle.truffle.r.native/gnur/Makefile.libs
index 89f5cb31d9..4f7f92f0eb 100644
--- a/com.oracle.truffle.r.native/gnur/Makefile.libs
+++ b/com.oracle.truffle.r.native/gnur/Makefile.libs
@@ -34,11 +34,13 @@ endif
BLAS_TARGET := $(FASTR_LIB_DIR)/libRblas$(DYLIB_EXT)
LAPACK_TARGET := $(FASTR_LIB_DIR)/libRlapack$(DYLIB_EXT)
# at a minimum we need to know where libpcre/libz are located,
-# to keep the Java side simpler, we copy them to $(FASTR_LIB_DIR)
-PCRE_TARGET := $(FASTR_LIB_DIR)/libpcre$(DYLIB_EXT)
-Z_TARGET := $(FASTR_LIB_DIR)/libz$(DYLIB_EXT)
+# to keep the Java side simpler, we (may) copy them to $(FASTR_LIB_DIR)
+# unless they are found in system dirs
+#PCRE_TARGET := $(FASTR_LIB_DIR)/libpcre$(DYLIB_EXT)
+#Z_TARGET := $(FASTR_LIB_DIR)/libz$(DYLIB_EXT)
+.PHONY: all pcre_target z_target
-all: $(FASTR_LIB_DIR) $(BLAS_TARGET) $(LAPACK_TARGET) $(PCRE_TARGET) $(Z_TARGET)
+all: $(FASTR_LIB_DIR) $(BLAS_TARGET) $(LAPACK_TARGET) pcre_target z_target
$(FASTR_LIB_DIR):
mkdir -p $(FASTR_LIB_DIR)
@@ -55,11 +57,11 @@ ifeq ($(OS_NAME),Darwin)
install_name_tool -id $(LAPACK_TARGET) $(LAPACK_TARGET)
endif
-$(PCRE_TARGET):
- cp $(shell mx rfindlib pcre) $(PCRE_TARGET)
+pcre_target:
+ mx rcopylib pcre $(FASTR_LIB_DIR)
-$(Z_TARGET):
- cp $(shell mx rfindlib z) $(Z_TARGET)
+z_target:
+ mx rcopylib z $(FASTR_LIB_DIR)
clean:
rm -f $(BLAS_TARGET) $(LAPACK_TARGET) $(PCRE_TARGET) $(Z_TARGET)
diff --git a/com.oracle.truffle.r.runtime.ffi/src/com/oracle/truffle/r/runtime/ffi/jni/JNI_C.java b/com.oracle.truffle.r.runtime.ffi/src/com/oracle/truffle/r/runtime/ffi/jni/JNI_C.java
index e3bf27cfa8..160d050b37 100644
--- a/com.oracle.truffle.r.runtime.ffi/src/com/oracle/truffle/r/runtime/ffi/jni/JNI_C.java
+++ b/com.oracle.truffle.r.runtime.ffi/src/com/oracle/truffle/r/runtime/ffi/jni/JNI_C.java
@@ -24,15 +24,14 @@ package com.oracle.truffle.r.runtime.ffi.jni;
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
import com.oracle.truffle.r.runtime.ffi.CRFFI;
-import com.oracle.truffle.r.runtime.ffi.RFFIFactory;
public class JNI_C implements CRFFI {
/**
- * This is rather similar to {@link JNI_Call}, except the objects are guaranteed to be
- * native array types, no upcalls are possible, and no result is returned. However, the
- * receiving function expects actual native arrays (not SEXPs), so these have to be handled on
- * the JNI side.
+ * This is rather similar to {@link JNI_Call}, except the objects are guaranteed to be native
+ * array types, no upcalls are possible, and no result is returned. However, the receiving
+ * function expects actual native arrays (not SEXPs), so these have to be handled on the JNI
+ * side.
*/
@Override
@TruffleBoundary
diff --git a/com.oracle.truffle.r.runtime.ffi/src/com/oracle/truffle/r/runtime/ffi/jni/JNI_PCRE.java b/com.oracle.truffle.r.runtime.ffi/src/com/oracle/truffle/r/runtime/ffi/jni/JNI_PCRE.java
index b9b97036b2..7a73c6d80b 100644
--- a/com.oracle.truffle.r.runtime.ffi/src/com/oracle/truffle/r/runtime/ffi/jni/JNI_PCRE.java
+++ b/com.oracle.truffle.r.runtime.ffi/src/com/oracle/truffle/r/runtime/ffi/jni/JNI_PCRE.java
@@ -23,14 +23,9 @@
package com.oracle.truffle.r.runtime.ffi.jni;
import com.oracle.truffle.r.runtime.RInternalError;
-import com.oracle.truffle.r.runtime.ffi.LibPaths;
import com.oracle.truffle.r.runtime.ffi.PCRERFFI;
public class JNI_PCRE implements PCRERFFI {
- JNI_PCRE() {
- System.load(LibPaths.getBuiltinLibPath("pcre"));
- }
-
@Override
public long maketables() {
return nativeMaketables();
diff --git a/com.oracle.truffle.r.runtime.ffi/src/com/oracle/truffle/r/runtime/ffi/jni/JNI_Zip.java b/com.oracle.truffle.r.runtime.ffi/src/com/oracle/truffle/r/runtime/ffi/jni/JNI_Zip.java
index 883158f77b..452f10e678 100644
--- a/com.oracle.truffle.r.runtime.ffi/src/com/oracle/truffle/r/runtime/ffi/jni/JNI_Zip.java
+++ b/com.oracle.truffle.r.runtime.ffi/src/com/oracle/truffle/r/runtime/ffi/jni/JNI_Zip.java
@@ -23,7 +23,6 @@
package com.oracle.truffle.r.runtime.ffi.jni;
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
-import com.oracle.truffle.r.runtime.ffi.LibPaths;
import com.oracle.truffle.r.runtime.ffi.ZipRFFI;
/**
@@ -31,10 +30,6 @@ import com.oracle.truffle.r.runtime.ffi.ZipRFFI;
*/
public class JNI_Zip implements ZipRFFI {
- JNI_Zip() {
- System.load(LibPaths.getBuiltinLibPath("z"));
- }
-
@Override
@TruffleBoundary
public int compress(byte[] dest, byte[] source) {
diff --git a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/ffi/ZipRFFI.java b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/ffi/ZipRFFI.java
index a829c8c927..dc99a9ddb7 100644
--- a/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/ffi/ZipRFFI.java
+++ b/com.oracle.truffle.r.runtime/src/com/oracle/truffle/r/runtime/ffi/ZipRFFI.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
diff --git a/mx.fastr/mx_copylib.py b/mx.fastr/mx_copylib.py
new file mode 100644
index 0000000000..b52609cebf
--- /dev/null
+++ b/mx.fastr/mx_copylib.py
@@ -0,0 +1,93 @@
+#
+# Copyright (c) 2016, 2016, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+import os
+import platform
+import subprocess
+import shutil
+import mx
+
+def _copylib(lib, libpath, target):
+ '''
+ Just copying libxxx.so/dylib isn't sufficient as versioning is involved.
+ The path is likely a symlink to libxxx.so.n, for example, but what is really
+ important is the version that is encoded in the shared library itself.
+ Unfortunately getting that info is is OS specific.
+ '''
+ if platform.system() == 'Darwin':
+ try:
+ output = subprocess.check_output(['otool', '-L', libpath])
+ lines = output.split('\n')
+ for line in lines[1:]:
+ if lib in line:
+ parts = line.split(' ')
+ real_libpath = parts[0].strip()
+ except subprocess.CalledProcessError:
+ mx.abort('copylib: otool failed')
+ else:
+ try:
+ output = subprocess.check_output(['objdump', '-p', libpath])
+ lines = output.split('\n')
+ for line in lines:
+ if 'SONAME' in line:
+ ix = line.find('lib' + lib)
+ real_libpath = os.path.join(os.path.dirname(libpath), line[ix:])
+ except subprocess.CalledProcessError:
+ mx.abort('copylib: otool failed')
+ # copy both files
+ shutil.copy(real_libpath, target)
+ os.chdir(target)
+ if os.path.exists(os.path.basename(libpath)):
+ os.remove(os.path.basename(libpath))
+ os.symlink(os.path.basename(real_libpath), os.path.basename(libpath))
+ mx.log('copied ' + lib + ' library from ' + libpath + ' to ' + target)
+
+def copylib(args):
+ '''
+ This supports a configuration where no explicit setting (e.g. LD_LIBRARY_PATH) is
+ required at runtime for the libraries that are required by FastR, e.g. pcre.
+ The easy case is when the libraries are already installed with the correct versions
+ in one of the directories, e.g./usr/lib, that is searched by default by the system linker -
+ in which case no configuration is required.
+
+ Otherwise, since systems vary considerably in where such libraries are located, the general solution
+ is to copy libraries located in non-system locations into the FastR 'lib' directory. N.B. GNU R is
+ even more picky than FastR about library versions and depends on a larger set, so the local
+ copy embedded in FastR is built using PKG_LDFLAGS_OVERRIDE to specify the location of necessary external
+ libraries. However, the result of this analysis isn't captured anywhere, so we re-analyze here.
+ If PKG_LDFLAGS_OVERRIDE is unset, we assume the libraries are located in the system directories
+ and do nothing.
+ '''
+ if os.environ.has_key('PKG_LDFLAGS_OVERRIDE'):
+ parts = os.environ['PKG_LDFLAGS_OVERRIDE'].split(' ')
+ ext = '.dylib' if platform.system() == 'Darwin' else '.so'
+ name = 'lib' + args[0] + ext
+ for part in parts:
+ path = part.strip('"').lstrip('-L')
+ for f in os.listdir(path):
+ if name == f:
+ target_dir = args[1]
+ if not os.path.exists(os.path.join(target_dir, name)):
+ _copylib(args[0], os.path.join(path, f), args[1])
+ return 0
+
+ mx.log(args[0] + ' not found in PKG_LDFLAGS_OVERRIDE, assuming system location')
diff --git a/mx.fastr/mx_fastr.py b/mx.fastr/mx_fastr.py
index d7d3804ec6..bbb01999d9 100644
--- a/mx.fastr/mx_fastr.py
+++ b/mx.fastr/mx_fastr.py
@@ -28,7 +28,7 @@ import mx_gate
import mx_fastr_pkgs
import mx_fastr_dists
from mx_fastr_dists import FastRNativeProject, FastRTestNativeProject, FastRReleaseProject #pylint: disable=unused-import
-import mx_findlib
+import mx_copylib
import os
'''
@@ -244,6 +244,16 @@ def _fastr_gate_runner(args, tasks):
if mx.checkcopyrights(['--primary']) != 0:
t.abort('copyright errors')
+ with mx_gate.Task('LDD', tasks) as t:
+ if t:
+ libpath = join(_fastr_suite.dir, 'lib')
+ if platform.system() == 'Darwin':
+ rc = subprocess.call(['otool', '-L', join(libpath, 'libR.dylib')])
+ else:
+ rc = subprocess.call(['ldd', join(libpath, 'libR.so')])
+ if rc != 0:
+ t.abort('LDD failed')
+
# check that the expected test output file is up to date
with mx_gate.Task('UnitTests: ExpectedTestOutput file check', tasks) as t:
if t:
@@ -534,7 +544,7 @@ _commands = {
'pkgtest' : [mx_fastr_pkgs.pkgtest, ['options']],
'installpkgs' : [mx_fastr_pkgs.installpkgs, '[options]'],
'mkgramrd': [mx_fastr_mkgramrd.mkgramrd, '[options]'],
- 'rfindlib' : [mx_findlib.findlib, '[]'],
+ 'rcopylib' : [mx_copylib.copylib, '[]'],
}
mx.update_commands(_fastr_suite, _commands)
diff --git a/mx.fastr/mx_findlib.py b/mx.fastr/mx_findlib.py
deleted file mode 100644
index c5b07cc67c..0000000000
--- a/mx.fastr/mx_findlib.py
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Copyright (c) 2016, 2016, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-import os
-import platform
-import mx
-
-def findlib(args):
- '''
- GNU R is built using GNUR_LDFLAGS_OVERRIDE to specify the location of necessary external libraries,
- but this isn't captured anywhere, so we re-analyze here
- '''
- if not os.environ.has_key('GNUR_LDFLAGS_OVERRIDE'):
- mx.abort('GNUR_LDFLAGS_OVERRIDE is not set')
- parts = os.environ['GNUR_LDFLAGS_OVERRIDE'].split(' ')
- ext = '.dylib' if platform.system() == 'Darwin' else '.so'
- name = 'lib' + args[0] + ext
- for part in parts:
- path = part.lstrip('-I')
- for f in os.listdir(path):
- if name == f:
- mx.log(os.path.join(path, f))
- return 0
- # if Linux try LD_LBRARY_PATH
- if platform.system() == 'Linux':
- if os.environ.has_key('LD_LIBRARY_PATH'):
- ldpaths = os.environ['LD_LIBRARY_PATH'].split(":")
- for ldpath in ldpaths:
- for f in os.listdir(ldpath):
- if name == f:
- mx.log(os.path.join(ldpath, f))
- return 0
- mx.abort(name + ' not found')
--
GitLab