diff --git a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/ExpectedTestOutput.test b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/ExpectedTestOutput.test
index 71f1784301243b7a76d79cad999efb22a031ffe7..e58fcfcdd6a8e546f5e28219ae429b6960f49214 100644
--- a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/ExpectedTestOutput.test
+++ b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/ExpectedTestOutput.test
@@ -4180,7 +4180,7 @@ build FALSE FALSE FALSE FALSE
install FALSE FALSE FALSE FALSE
render FALSE FALSE FALSE FALSE
-##com.oracle.truffle.r.test.builtins.TestBuiltin_array.testarray11#Ignored.Unknown#
+##com.oracle.truffle.r.test.builtins.TestBuiltin_array.testarray11#
#argv <- list(list(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL), 8L, list(c('1', '2', '3', '4', '5', '6', '7', '8'))); .Internal(array(argv[[1]], argv[[2]], argv[[3]]))
$`1`
NULL
@@ -22571,7 +22571,7 @@ attr(,"Rd_tag")
#argv <- list(structure('BunchKaufman', class = structure('signature', package = 'methods'), .Names = 'x', package = 'methods'), structure('Matrix', .Names = 'x', package = 'Matrix', class = structure('signature', package = 'methods')), TRUE, TRUE, TRUE, TRUE, FALSE); .Internal(identical(argv[[1]], argv[[2]], argv[[3]], argv[[4]], argv[[5]], argv[[6]], argv[[7]]))
[1] FALSE
-##com.oracle.truffle.r.test.builtins.TestBuiltin_identical.testidentical28#Ignored.Unknown#
+##com.oracle.truffle.r.test.builtins.TestBuiltin_identical.testidentical28#Ignored.Unstable#
#argv <- list(structure(list(x = c(1, 1, 1, 1, 1, 1, 1, 1, 1, 1), y = c(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), fac = structure(c(1L, 3L, 2L, 3L, 3L, 1L, 2L, 3L, 2L, 2L), .Label = c('A', 'B', 'C'), class = 'factor')), .Names = c('x', 'y', 'fac'), row.names = c(NA, -10L), class = 'data.frame'), structure(list(x = c(1, 1, 1, 1, 1, 1, 1, 1, 1, 1), y = c(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), fac = structure(c(1L, 3L, 2L, 3L, 3L, 1L, 2L, 3L, 2L, 2L), .Label = c('A', 'B', 'C'), class = 'factor')), .Names = c('x', 'y', 'fac'), row.names = c(NA, 10L), class = 'data.frame'), TRUE, TRUE, TRUE, TRUE, FALSE); .Internal(identical(argv[[1]], argv[[2]], argv[[3]], argv[[4]], argv[[5]], argv[[6]], argv[[7]]))
[1] TRUE
@@ -48366,7 +48366,7 @@ Error: unexpected symbol in "argv <- list(c('* Edit the help file skeletons in '
[20] "," " "
-##com.oracle.truffle.r.test.builtins.TestBuiltin_strsplit.teststrsplit3#Ignored.Unknown#
+##com.oracle.truffle.r.test.builtins.TestBuiltin_strsplit.teststrsplit3#
#argv <- list(' \036 isSeekable() now returns FALSE on connections which have non-default encoding. Although documented to record if ‘in principle’ the connection supports seeking, it seems safer to report FALSE when it may not work.', '[ \t\n]', FALSE, TRUE, FALSE); .Internal(strsplit(argv[[1]], argv[[2]], argv[[3]], argv[[4]], argv[[5]]))
[[1]]
[1] "" "" "\036" "" "isSeekable()"
@@ -52680,7 +52680,7 @@ $lrow
attr(,"row.names")
[1] 1
-##com.oracle.truffle.r.test.builtins.TestBuiltin_unclass.testunclass26#Ignored.Unknown#
+##com.oracle.truffle.r.test.builtins.TestBuiltin_unclass.testunclass26#
#argv <- list(structure(list(a = 1), .Dim = 1L, .Dimnames = list('a')));unclass(argv[[1]]);
$a
[1] 1
@@ -57556,6 +57556,33 @@ NULL
#n <- 100; { e <- parent.frame(2); e$n}
[1] 100
+##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreErrorContext#
+#if (exists('.fastr.identity')) { cat('Error in .Internal(foo(44)): IgnoreErrorContext') } else { cat('Error in foo(42): IgnoreErrorContext') }
+Error in foo(42): IgnoreErrorContext
+##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreErrorContext#
+#if (exists('.fastr.identity')) { cat('Error in foo(42) : IgnoreErrorContext extra spaces') } else { cat('Error in foo(42): IgnoreErrorContext extra spaces') }
+Error in foo(42): IgnoreErrorContext extra spaces
+##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreWhitespace#
+#if (exists('.fastr.identity')) { cat('Error in foo(42) : IgnoreWhitespace extra spaces') } else { cat('Error in foo(42): \nIgnoreWhitespace extra spaces') }
+Error in foo(42):
+IgnoreWhitespace extra spaces
+##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreErrorMessage#
+#if (exists('.fastr.identity')) { cat('Error in foo(42): FastR error message') } else { cat('Error in foo(42): GnuR error message is different') }
+Error in foo(42): GnuR error message is different
+##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreErrorMessage#
+#if (exists('.fastr.identity')) { cat('Error in foo(42): IgnoreErrorMessage starts with newline') } else { cat('Error in foo(42):\nIgnoreErrorMessage starts with newline') }
+Error in foo(42):
+IgnoreErrorMessage starts with newline
+##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreErrorMessage#
+#if (exists('.fastr.identity')) { cat('Error in foo(44): IgnoreErrorMessage with different ctx') } else { cat('Error in foo(42): IgnoreErrorMessage with different ctx') }
+Error in foo(42): IgnoreErrorMessage with different ctx
+##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreWarningContext#
+#if (exists('.fastr.identity')) { cat('Warning message: In .Internal(foo(42)) : IgnoreWarningContext extra newline') } else { cat('Warning message: In foo(42) : \nIgnoreWarningContext extra newline') }
+Warning message: In foo(42) :
+IgnoreWarningContext extra newline
+##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreWarningContext#
+#if (exists('.fastr.identity')) { cat('Warning message: In .Internal(foo(42)) : IgnoreWarningContext') } else { cat('Warning message: In foo(42) : IgnoreWarningContext') }
+Warning message: In foo(42) : IgnoreWarningContext
##com.oracle.truffle.r.test.functions.TestFunctions.testArgEvaluationOrder#
#v <- 1; class(v) <- 'foo'; `-.foo` <- function(x,y,...) { cat('[-.foo]'); 1234 }; g <- function(...) { cat('[ing]'); ({cat('[-]'); `-`})(...) }; ({cat('[g]'); g})({cat('[x]'); v},{cat('[y]'); 3},{cat('[z]'); 5})
[g][ing][-][x][y][z][-.foo][1] 1234
@@ -111390,7 +111417,7 @@ In rbinom("aa", 10, 0.5) : NAs introduced by coercion
#set.seed(42); rbinom(c(1,2), 11:12, c(0.1, 0.5, 0.9))
[1] 3 9
-##com.oracle.truffle.r.test.library.stats.TestExternal_rnbinom.testRbinomWithMu#Ignored.Unimplemented#
+##com.oracle.truffle.r.test.library.stats.TestExternal_rnbinom.testRbinomWithMu#Ignored.Unstable#
#set.seed(42); rnbinom(100, c(-1, 0, 1, 0.8, 10, NA, NaN, 1/0, -1/0), mu=c(-1, 0, 1, 0.8, 3, 10, NA, NaN, 1/0, -1/0))
[1] NaN NaN 1 1 4 NaN NaN NaN NaN NaN NaN 0 0 0 NaN NaN NaN NaN
[19] NaN NaN NaN 0 0 NaN NaN 5 NaN NaN NaN NaN NaN 0 NaN NaN 5 NaN
diff --git a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/TestBase.java b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/TestBase.java
index f656624b7f3cf7bf6b8c14429c58b02229d9dd74..173cc5b2b3ddf1a23bf6aabb63ebdbaf60f3b7a0 100644
--- a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/TestBase.java
+++ b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/TestBase.java
@@ -22,6 +22,8 @@ import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map.Entry;
import java.util.SortedMap;
@@ -63,14 +65,17 @@ public class TestBase {
public static final boolean ProcessFailedTests = Boolean.getBoolean("ProcessFailedTests");
+ /**
+ * See {@link com.oracle.truffle.r.test.builtins.TestTestBase} for examples.
+ */
public enum Output implements TestTrait {
IgnoreErrorContext, // the error context is ignored (e.g., "a+b" vs. "a + b")
IgnoreErrorMessage, // the actual error message is ignored
IgnoreWarningContext, // the warning context is ignored
- MayIgnoreErrorContext,
+ MayIgnoreErrorContext, // like IgnoreErrorContext, but no warning if the messages match
MayIgnoreWarningContext,
ContainsReferences, // replaces references in form of 0xbcdef1 for numbers
- IgnoreWhitespace;
+ IgnoreWhitespace; // removes all whitespace from the whole output
@Override
public String getName() {
@@ -572,37 +577,51 @@ public class TestBase {
}
}
- private void evalAndCompare(String[] inputs, TestTrait... traits) {
- WhiteList[] whiteLists = TestTrait.collect(traits, WhiteList.class);
-
- boolean ignored = TestTrait.contains(traits, Ignored.class) ^ (ProcessFailedTests && !(TestTrait.contains(traits, Ignored.Unstable) || TestTrait.contains(traits, Ignored.SideEffects)));
+ /**
+ * Wraps the traits, there are some meta-traits like {@link #isIgnored}, other traits can be
+ * accessed through the corresponding enum-set.
+ */
+ private static class TestTraitsSet {
+ EnumSet<Ignored> ignored = EnumSet.noneOf(Ignored.class);
+ EnumSet<Output> output = EnumSet.noneOf(Output.class);
+ EnumSet<Context> context = EnumSet.noneOf(Context.class);
+ boolean isIgnored;
+ boolean containsError;
+
+ TestTraitsSet(TestTrait[] traits) {
+ ignored.addAll(Arrays.asList(TestTrait.collect(traits, Ignored.class)));
+ output.addAll(Arrays.asList(TestTrait.collect(traits, Output.class)));
+ context.addAll(Arrays.asList(TestTrait.collect(traits, Context.class)));
+ containsError = (!FULL_COMPARE_ERRORS && (output.contains(Output.IgnoreErrorContext) || output.contains(Output.IgnoreErrorMessage)));
+ isIgnored = ignored.size() > 0 ^ (ProcessFailedTests && !(ignored.contains(Ignored.Unstable) || ignored.contains(Ignored.SideEffects)));
+ assert !output.contains(Output.IgnoreWhitespace) || output.size() == 1 : "IgnoreWhitespace trait does not work with any other Output trait";
- boolean containsWarning = TestTrait.contains(traits, Output.IgnoreWarningContext);
- boolean containsError = (!FULL_COMPARE_ERRORS && (TestTrait.contains(traits, Output.IgnoreErrorContext) || TestTrait.contains(traits, Output.IgnoreErrorMessage)));
- boolean mayContainWarning = TestTrait.contains(traits, Output.MayIgnoreWarningContext);
- boolean mayContainError = TestTrait.contains(traits, Output.MayIgnoreErrorContext);
- boolean ambiguousError = TestTrait.contains(traits, Output.IgnoreErrorMessage);
- boolean ignoreWhitespace = TestTrait.contains(traits, Output.IgnoreWhitespace);
- boolean containsReferences = TestTrait.contains(traits, Output.ContainsReferences);
- boolean nonSharedContext = TestTrait.contains(traits, Context.NonShared);
- boolean longTimeout = TestTrait.contains(traits, Context.LongTimeout);
+ }
- ContextInfo contextInfo = nonSharedContext ? fastROutputManager.fastRSession.createContextInfo(ContextKind.SHARE_NOTHING) : null;
+ String preprocessOutput(String out) {
+ if (output.contains(Output.IgnoreWhitespace)) {
+ return out.replaceAll("\\s+", "");
+ }
+ if (output.contains(Output.ContainsReferences)) {
+ return convertReferencesInOutput(out);
+ }
+ return out;
+ }
+ }
+ private void evalAndCompare(String[] inputs, TestTrait... traitsList) {
+ WhiteList[] whiteLists = TestTrait.collect(traitsList, WhiteList.class);
+ TestTraitsSet traits = new TestTraitsSet(traitsList);
+ ContextInfo contextInfo = traits.context.contains(Context.NonShared) ? fastROutputManager.fastRSession.createContextInfo(ContextKind.SHARE_NOTHING) : null;
int index = 1;
boolean allOk = true;
for (String input : inputs) {
- String expected = expectedEval(input, traits);
- if (ignored || generatingExpected()) {
+ String expected = expectedEval(input, traitsList);
+ if (traits.isIgnored || generatingExpected()) {
ignoredInputCount++;
} else {
- String result = fastREval(input, contextInfo, longTimeout);
- if (ignoreWhitespace) {
- expected = expected.replaceAll("\\s+", "");
- result = result.replaceAll("\\s+", "");
- }
-
- CheckResult checkResult = checkResult(whiteLists, input, expected, result, containsWarning, mayContainWarning, containsError, mayContainError, ambiguousError, containsReferences);
+ String result = fastREval(input, contextInfo, traits.context.contains(Context.LongTimeout));
+ CheckResult checkResult = checkResult(whiteLists, input, traits.preprocessOutput(expected), traits.preprocessOutput(result), traits);
result = checkResult.result;
expected = checkResult.expected;
@@ -638,7 +657,7 @@ public class TestBase {
}
index++;
}
- if (ignored) {
+ if (traits.isIgnored) {
ignoredTestCount++;
} else if (allOk) {
successfulTestCount++;
@@ -660,25 +679,20 @@ public class TestBase {
}
}
- private CheckResult checkResult(WhiteList[] whiteLists, String input, String originalExpected, String originalResult, boolean containsWarning, boolean mayContainWarning, boolean containsError,
- boolean mayContainError, boolean ambiguousError, boolean convertReferences) {
+ private CheckResult checkResult(WhiteList[] whiteLists, String input, String originalExpected, String originalResult, TestTraitsSet traits) {
boolean ok;
String result = originalResult;
String expected = originalExpected;
- if (convertReferences) {
- result = convertReferencesInOutput(result);
- expected = convertReferencesInOutput(expected);
- }
- if (expected.equals(result) || searchWhiteLists(whiteLists, input, expected, result, containsWarning, mayContainWarning, containsError, mayContainError, ambiguousError, convertReferences)) {
+ if (expected.equals(result) || searchWhiteLists(whiteLists, input, expected, result, traits)) {
ok = true;
- if (containsError && !ambiguousError) {
+ if (traits.containsError && !traits.output.contains(Output.IgnoreErrorMessage)) {
System.out.println("unexpected correct error message: " + getTestContext());
}
- if (containsWarning) {
+ if (traits.output.contains(Output.IgnoreWarningContext)) {
System.out.println("unexpected correct warning message: " + getTestContext());
}
} else {
- if (containsWarning || (mayContainWarning && expected.contains(WARNING))) {
+ if (traits.output.contains(Output.IgnoreWarningContext) || (traits.output.contains(Output.MayIgnoreWarningContext) && expected.contains(WARNING))) {
String resultWarning = getWarningMessage(result);
String expectedWarning = getWarningMessage(expected);
ok = resultWarning.equals(expectedWarning);
@@ -688,8 +702,8 @@ public class TestBase {
ok = true;
}
if (ok) {
- if (containsError || (mayContainError && expected.startsWith(ERROR))) {
- ok = result.startsWith(ERROR) && (ambiguousError || checkMessageStripped(expected, result) || checkMessageVectorInIndex(expected, result));
+ if (traits.containsError || (traits.output.contains(Output.MayIgnoreErrorContext) && expected.startsWith(ERROR))) {
+ ok = result.startsWith(ERROR) && (traits.output.contains(Output.IgnoreErrorMessage) || checkMessageStripped(expected, result) || checkMessageVectorInIndex(expected, result));
} else {
ok = expected.equals(result);
}
@@ -714,8 +728,7 @@ public class TestBase {
return result;
}
- private boolean searchWhiteLists(WhiteList[] whiteLists, String input, String expected, String result, boolean containsWarning, boolean mayContainWarning, boolean containsError,
- boolean mayContainError, boolean ambiguousError, boolean convertReferences) {
+ private boolean searchWhiteLists(WhiteList[] whiteLists, String input, String expected, String result, TestTraitsSet testTraits) {
if (whiteLists == null) {
return false;
}
@@ -723,13 +736,13 @@ public class TestBase {
WhiteList.Results wlr = list.get(input);
if (wlr != null) {
// Sanity check that "expected" matches the entry in the WhiteList
- CheckResult checkedResult = checkResult(null, input, wlr.expected, expected, containsWarning, mayContainWarning, containsError, mayContainError, ambiguousError, convertReferences);
+ CheckResult checkedResult = checkResult(null, input, wlr.expected, expected, testTraits);
if (!checkedResult.ok) {
System.out.println("expected output does not match: " + wlr.expected + " vs. " + expected);
return false;
}
// Substitute the FastR output and try to match that
- CheckResult fastRResult = checkResult(null, input, wlr.fastR, result, containsWarning, mayContainWarning, containsError, mayContainError, ambiguousError, convertReferences);
+ CheckResult fastRResult = checkResult(null, input, wlr.fastR, result, testTraits);
if (fastRResult.ok) {
list.markUsed(input);
return true;
@@ -745,7 +758,7 @@ public class TestBase {
private static final Pattern warningPattern4 = Pattern.compile("^(?<pre>.*)Warning messages:\n1:(?<msg0>.*)\n2:(?<msg1>.*)$", Pattern.DOTALL);
private static final Pattern warningPattern5 = Pattern.compile("^(?<pre>.*)Warning message:(?<msg0>.*)$", Pattern.DOTALL);
- private static final Pattern warningMessagePattern = Pattern.compile("^\n? ? ?(?:In .* :[ \n])?(?<m>[^\n]*)\n?$", Pattern.DOTALL);
+ private static final Pattern warningMessagePattern = Pattern.compile("^\n? ? ?(?:In .* :[ \n])?[ \n]*(?<m>[^\n]*)\n?$", Pattern.DOTALL);
private static final Pattern[] warningPatterns = new Pattern[]{warningPattern1, warningPattern2, warningPattern3, warningPattern4, warningPattern5};
diff --git a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/builtins/TestTestBase.java b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/builtins/TestTestBase.java
new file mode 100644
index 0000000000000000000000000000000000000000..235264d8223d504e607cfffac0f69e535ecf3017
--- /dev/null
+++ b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/builtins/TestTestBase.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.truffle.r.test.builtins;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.oracle.truffle.r.test.TestBase;
+
+/**
+ * Documents and tests the testing API.
+ */
+public class TestTestBase extends TestBase {
+ @Test
+ public void testTraits() {
+ assertEval(Output.IgnoreErrorMessage, getCode("cat('Error in foo(42): FastR error message')", "cat('Error in foo(42): GnuR error message is different')"));
+ assertEval(Output.IgnoreErrorMessage, getCode("cat('Error in foo(44): IgnoreErrorMessage with different ctx')", "cat('Error in foo(42): IgnoreErrorMessage with different ctx')"));
+ assertEval(Output.IgnoreErrorMessage, getCode("cat('Error in foo(42): IgnoreErrorMessage starts with newline')", "cat('Error in foo(42):\\nIgnoreErrorMessage starts with newline')"));
+
+ assertEval(Output.IgnoreErrorContext, getCode("cat('Error in .Internal(foo(44)): IgnoreErrorContext')", "cat('Error in foo(42): IgnoreErrorContext')"));
+ assertEval(Output.IgnoreErrorContext, getCode("cat('Error in foo(42) : IgnoreErrorContext extra spaces')", "cat('Error in foo(42): IgnoreErrorContext extra spaces')"));
+
+ assertEval(Output.IgnoreWarningContext, getCode("cat('Warning message: In .Internal(foo(42)) : IgnoreWarningContext')", "cat('Warning message: In foo(42) : IgnoreWarningContext')"));
+ assertEval(Output.IgnoreWarningContext,
+ getCode("cat('Warning message: In .Internal(foo(42)) : IgnoreWarningContext extra newline')", "cat('Warning message: In foo(42) : \\nIgnoreWarningContext extra newline')"));
+
+ assertEval(Output.IgnoreWhitespace, getCode("cat('Error in foo(42) : IgnoreWhitespace extra spaces')", "cat('Error in foo(42): \\nIgnoreWhitespace extra spaces')"));
+ }
+
+ @Test
+ @Ignore // these tests should fail
+ public void negativeTestTraits() {
+ assertEval(Output.IgnoreErrorContext, getCode("cat('Error in foo(44): IgnoreErrorContext diff message')", "cat('Error in foo(42): IgnoreErrorContext should fail')"));
+ assertEval(Output.IgnoreWarningContext,
+ getCode("cat('Warning message: In .Internal(foo(42)): IgnoreWarningContext diff message')", "cat('Warning message in foo(42): IgnoreWarningContext should fail')"));
+ }
+
+ private String getCode(String fastr, String gnur) {
+ return "if (exists('.fastr.identity')) { " + fastr + " } else { " + gnur + " }";
+ }
+}