diff --git a/ci.hocon b/ci.hocon
index bd89ac5666dd23484b533f73722fc22bca4da813..57e9286e736a9b58b55ba8ff9aed73debcdaae2c 100644
--- a/ci.hocon
+++ b/ci.hocon
@@ -1,6 +1,21 @@
+# This is part of the framework used by FastR for continuous integration testing.
+# It is not intended that this "code" is directly usable by third-party developers
+# but it provide details on the commands that are used and these should be
+# executable in isolation; see further comments below.
+
java7 : {name : oraclejdk, version : "7", platformspecific: true}
#java8 : {name : oraclejdk, version : "8u66", platformspecific: true}
java8 : {name : labsjdk, version : "8u92-jvmci-0.21", platformspecific: true}
+
+java8Downloads : {
+ downloads : {
+ EXTRA_JAVA_HOMES : { pathlist :[
+ ${java7}
+ ]}
+ JAVA_HOME : ${java8}
+ }
+}
+
logfiles : [
"fastr_errors.log"
"com.oracle.truffle.r.native/gnur/R-*/gnur_configure.log"
@@ -10,60 +25,80 @@ logfiles : [
"com.oracle.truffle.r.native/gnur/libiconv-*/iconv_make.log"
]
-common : {
+# This is needed by all (Linux) builds but is specific to the module system employed
+# on the CI cluster. Not all of the modules are needed by FastR but all are needed by the
+# embedded GNU R that is built within FastR.
+
+pkgEnvironment: {
+ environment : {
+ PKG_INCLUDE_FLAGS_OVERRIDE : """"-I/cm/shared/apps/zlib/1.2.8/include -I/cm/shared/apps/bzip2/1.0.6/include -I/cm/shared/apps/xz/5.2.2/include -I/cm/shared/apps/pcre/8.38/include -I/cm/shared/apps/curl/7.50.1/include""""
+ PKG_LDFLAGS_OVERRIDE : """"-L/cm/shared/apps/zlib/1.2.8/lib -L/cm/shared/apps/bzip2/1.0.6/lib -L/cm/shared/apps/xz/5.2.2/lib -L/cm/shared/apps/pcre/8.38/lib -L/cm/shared/apps/curl/7.50.1//lib -L/cm/shared/apps/gcc/4.9.1/lib64""""
+ }
+}
+
+packagesLinux : ${pkgEnvironment} {
packages : {
git : ">=1.8.3"
mercurial : ">=2.2"
"pip:astroid" : "==1.1.0"
"pip:pylint" : "==1.1.0"
make : ">=3.83"
- gcc-build-essentials : ">=4.9.1" # GCC 4.9.0 fails on cluster
+ gcc-build-essentials : "==4.9.1" # GCC 4.9.0 fails on cluster
readline : "==6.3"
+ pcre : ">=8.38"
+ z : ">=1.2.8"
}
- # TODO this will eventually relocate elsewhere
- environment : {
- PKG_INCLUDE_FLAGS_OVERRIDE : """"-I/cm/shared/apps/zlib/1.2.8/include -I/cm/shared/apps/bzip2/1.0.6/include -I/cm/shared/apps/xz/5.2.2/include -I/cm/shared/apps/pcre/8.38/include -I/cm/shared/apps/curl/7.50.1/include""""
- PKG_LDFLAGS_OVERRIDE : """"-L/cm/shared/apps/zlib/1.2.8/lib -L/cm/shared/apps/bzip2/1.0.6/lib -L/cm/shared/apps/xz/5.2.2/lib -L/cm/shared/apps/pcre/8.38/lib -L/cm/shared/apps/curl/7.50.1//lib""""
- }
+}
+
+packagesDarwin : {
+ "pip:astroid" : "==1.1.0"
+ "pip:pylint" : "==1.1.0"
+ "xz" : ""
+ # assume pcre/z system installed
+}
+
+# Common settings for all builds but note that it uses the Linux package settings,
+# so these must be overridden in any darwin builds
+
+common : ${java8Downloads} ${packagesLinux} {
logs: ${logfiles}
timelimit : "1:00:00"
}
-gateCmd : ["mx", "--strict-compliance", "originalgate", "--strict-mode", "-t"]
+# Every "pull request" (PR) is subject to a series of "gate" commands that must pass
+# for the PR to be accepted into the master branch. The gate commands are run under
+# the "mx" tool. This defines a common prefix for all gate commands. The "-t"
+# arg indicates the exact set of gate "tasks" that will be run.
-java8Downloads : {
- downloads : {
- EXTRA_JAVA_HOMES : { pathlist :[
- ${java7}
- ]}
- JAVA_HOME : ${java8}
- }
-}
+gateCmd : ["mx", "--strict-compliance", "rgate", "--strict-mode", "-t"]
+
+# currently disabled gate commands: FindBugs,Checkheaders,Distribution Overlap Check,BuildJavaWithEcj
-gateTest : ${common} ${java8Downloads} {
+# The standard set of gate tasks: the actual executable tests are in the "UnitTests" task.
+
+gateTestCommon : ${common} {
run : [
- ${gateCmd} ["Versions,JDKReleaseInfo,Pylint,Canonicalization Check,BuildJavaWithJavac,LDD, UnitTests: ExpectedTestOutput file check,UnitTests"]
+ ${gateCmd} ["Versions,JDKReleaseInfo,BuildJavaWithJavac,UnitTests: ExpectedTestOutput file check,UnitTests"]
]
}
-gateTestDarwin : ${java8Downloads} {
- packages : {
- "pip:astroid" : "==1.1.0"
- "pip:pylint" : "==1.1.0"
- "xz" : ""
- }
+gateTestLinux : ${gateTestCommon} {
+}
+
+darwinEnvironment : {
environment : {
PATH : "/usr/local/bin:$JAVA_HOME/bin:$PATH"
F77: "/usr/local/bin/gfortran-4.9"
}
- logs: ${logfiles}
- timelimit : "1:00:00"
+}
- run : [
- ${gateCmd} ["Versions,JDKReleaseInfo,Pylint,Canonicalization Check,BuildJavaWithJavac,LDD, UnitTests: ExpectedTestOutput file check,UnitTests"]
- ]
+gateTestDarwin : ${gateTestCommon} ${darwinEnvironment} {
+ packages : ${packagesDarwin} {
+ }
}
+# This performs a number of "style" checks on the code to ensure it confirms to the project standards.
+
gateStyle : ${common} {
# need pyhocon fix
downloads : {
@@ -72,7 +107,7 @@ gateStyle : ${common} {
JDT : {name: ecj, version: "4.5.1", platformspecific: false}
ECLIPSE : {name: eclipse, version: "4.5.2", platformspecific: true}
}
- environment : ${common.environment} {
+ environment : {
ECLIPSE_EXE : "$ECLIPSE/eclipse"
}
run : [
@@ -80,20 +115,21 @@ gateStyle : ${common} {
]
}
-rbcheck : ${common} ${java8Downloads} {
+# This check runs diagnostics on the implementation of the R "builtins" in FastR, e.g., that the argument processing is sound.
+
+rbcheck : ${common} {
run : [
${gateCmd} ["Versions,JDKReleaseInfo,BuildJavaWithJavac"]
["mx", "rbcheck"]
["mx", "rbdiag", "-n", "-m"]
- ]
+ ]
}
-# currently disabled gate commands: FindBugs,Checkheaders,Distribution Overlap Check,BuildJavaWithEcj
+# The standard set of gate builds. N.B. the style/builtin checks are only run on Linux as they are not OS-dependent.
builds = [
- ${gateTest} {capabilities : [linux, amd64], targets : [gate, post-merge], name: "gate-test-linux-amd64"}
- ${gateStyle} {capabilities : [linux, amd64], targets : [gate, post-merge], name: "gate-style-linux-amd64"}
+ ${gateTestLinux} {capabilities : [linux, amd64], targets : [gate, post-merge], name: "gate-test-linux-amd64"}
${gateTestDarwin} {capabilities : [darwin, amd64], targets : [gate, post-merge], name: "gate-test-darwin-amd64"}
+ ${gateStyle} {capabilities : [linux, amd64], targets : [gate, post-merge], name: "gate-style-linux-amd64"}
${rbcheck} {capabilities : [linux, amd64], targets : [gate, post-merge], name: "gate-rbcheck-linux-amd64"}
-# ${gateTest} {capabilities : [linux, sparcv9], targets : [gate, post-merge], name: "gate-test-linux-sparcv9"}
]
diff --git a/mx.fastr/mx_fastr.py b/mx.fastr/mx_fastr.py
index 90d2c52562912f039c7ce610f01008a7aa8fa93b..b3e5006c3f4e6877eb5f68204574b201264943b6 100644
--- a/mx.fastr/mx_fastr.py
+++ b/mx.fastr/mx_fastr.py
@@ -29,6 +29,8 @@ import mx_fastr_pkgs
import mx_fastr_dists
from mx_fastr_dists import FastRNativeProject, FastRTestNativeProject, FastRReleaseProject #pylint: disable=unused-import
import mx_copylib
+import mx_fastr_mkgramrd
+
import os
'''
@@ -232,28 +234,18 @@ def rembed(args, nonZeroIsFatal=True, extraVmArgs=None):
run_r(args, 'rembed')
def _fastr_gate_runner(args, tasks):
- # Until fixed, we call Checkstyle here and limit to primary
- with mx_gate.Task('Checkstyle check', tasks) as t:
- if t:
- if mx.checkstyle(['--primary']) != 0:
- t.abort('Checkstyle warnings were found')
-
+ '''
+ The specific additional gates tasks provided by FastR:
+ 1. Copyright check
+ 2. Check that ExpectedTestOutput file is in sync with unit tests
+ 3. Unit tests
+ '''
# FastR has custom copyright check
with mx_gate.Task('Copyright check', tasks) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort('copyright errors')
- with mx_gate.Task('LDD', tasks) as t:
- if t:
- libpath = join(_fastr_suite.dir, 'lib')
- if platform.system() == 'Darwin':
- rc = subprocess.call(['otool', '-L', join(libpath, 'libR.dylib')])
- else:
- rc = subprocess.call(['ldd', join(libpath, 'libR.so')])
- if rc != 0:
- t.abort('LDD failed')
-
# check that the expected test output file is up to date
with mx_gate.Task('UnitTests: ExpectedTestOutput file check', tasks) as t:
if t:
@@ -267,15 +259,18 @@ def _fastr_gate_runner(args, tasks):
mx_gate.add_gate_runner(_fastr_suite, _fastr_gate_runner)
+def rgate(args):
+ '''
+ Run 'mx.gate' with given args (used in CI system).
+ N.B. This will fail if run without certain exclusions; use the local
+ 'gate' command for that.
+ '''
+ mx_gate.gate(args)
+
def gate(args):
- '''Run the R gate'''
- # exclude findbugs until compliant
+ '''Run 'mx.gate' with some standard tasks excluded as they currently fail'''
mx_gate.gate(args + ['-x', '-t', 'FindBugs,Checkheaders,Distribution Overlap Check,BuildJavaWithEcj'])
-def original_gate(args):
- '''Run the R gate (without filtering gate tasks)'''
- mx_gate.gate(args)
-
def _test_srcdir():
tp = 'com.oracle.truffle.r.test'
return join(mx.project(tp).dir, 'src', tp.replace('.', sep))
@@ -524,15 +519,13 @@ def rcmplib(args):
def mx_post_parse_cmd_line(opts):
mx_fastr_dists.mx_post_parse_cmd_line(opts)
-import mx_fastr_mkgramrd
-
_commands = {
'r' : [rshell, '[options]'],
'R' : [rshell, '[options]'],
'rscript' : [rscript, '[options]'],
'Rscript' : [rscript, '[options]'],
'rtestgen' : [testgen, ''],
- 'originalgate' : [original_gate, '[options]'],
+ 'rgate' : [rgate, ''],
'gate' : [gate, ''],
'junit' : [junit, ['options']],
'junitsimple' : [junit_simple, ['options']],