From c9bc2ffb16cfc20ebe3ec0074ed06340ddefd3ba Mon Sep 17 00:00:00 2001 From: stepan <stepan.sindelar@oracle.com> Date: Tue, 20 Dec 2016 17:08:59 +0100 Subject: [PATCH] Tests traits: documentation, small refactoring and fix --- .../truffle/r/test/ExpectedTestOutput.test | 37 ++++++- .../com/oracle/truffle/r/test/TestBase.java | 99 +++++++++++-------- .../truffle/r/test/builtins/TestTestBase.java | 61 ++++++++++++ 3 files changed, 149 insertions(+), 48 deletions(-) create mode 100644 com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/builtins/TestTestBase.java diff --git a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/ExpectedTestOutput.test b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/ExpectedTestOutput.test index 71f1784301..e58fcfcdd6 100644 --- a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/ExpectedTestOutput.test +++ b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/ExpectedTestOutput.test @@ -4180,7 +4180,7 @@ build FALSE FALSE FALSE FALSE install FALSE FALSE FALSE FALSE render FALSE FALSE FALSE FALSE -##com.oracle.truffle.r.test.builtins.TestBuiltin_array.testarray11#Ignored.Unknown# +##com.oracle.truffle.r.test.builtins.TestBuiltin_array.testarray11# #argv <- list(list(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL), 8L, list(c('1', '2', '3', '4', '5', '6', '7', '8'))); .Internal(array(argv[[1]], argv[[2]], argv[[3]])) $`1` NULL @@ -22571,7 +22571,7 @@ attr(,"Rd_tag") #argv <- list(structure('BunchKaufman', class = structure('signature', package = 'methods'), .Names = 'x', package = 'methods'), structure('Matrix', .Names = 'x', package = 'Matrix', class = structure('signature', package = 'methods')), TRUE, TRUE, TRUE, TRUE, FALSE); .Internal(identical(argv[[1]], argv[[2]], argv[[3]], argv[[4]], argv[[5]], argv[[6]], argv[[7]])) [1] FALSE -##com.oracle.truffle.r.test.builtins.TestBuiltin_identical.testidentical28#Ignored.Unknown# +##com.oracle.truffle.r.test.builtins.TestBuiltin_identical.testidentical28#Ignored.Unstable# #argv <- list(structure(list(x = c(1, 1, 1, 1, 1, 1, 1, 1, 1, 1), y = c(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), fac = structure(c(1L, 3L, 2L, 3L, 3L, 1L, 2L, 3L, 2L, 2L), .Label = c('A', 'B', 'C'), class = 'factor')), .Names = c('x', 'y', 'fac'), row.names = c(NA, -10L), class = 'data.frame'), structure(list(x = c(1, 1, 1, 1, 1, 1, 1, 1, 1, 1), y = c(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), fac = structure(c(1L, 3L, 2L, 3L, 3L, 1L, 2L, 3L, 2L, 2L), .Label = c('A', 'B', 'C'), class = 'factor')), .Names = c('x', 'y', 'fac'), row.names = c(NA, 10L), class = 'data.frame'), TRUE, TRUE, TRUE, TRUE, FALSE); .Internal(identical(argv[[1]], argv[[2]], argv[[3]], argv[[4]], argv[[5]], argv[[6]], argv[[7]])) [1] TRUE @@ -48366,7 +48366,7 @@ Error: unexpected symbol in "argv <- list(c('* Edit the help file skeletons in ' [20] "," " " -##com.oracle.truffle.r.test.builtins.TestBuiltin_strsplit.teststrsplit3#Ignored.Unknown# +##com.oracle.truffle.r.test.builtins.TestBuiltin_strsplit.teststrsplit3# #argv <- list(' \036 isSeekable() now returns FALSE on connections which have non-default encoding. Although documented to record if ‘in principle’ the connection supports seeking, it seems safer to report FALSE when it may not work.', '[ \t\n]', FALSE, TRUE, FALSE); .Internal(strsplit(argv[[1]], argv[[2]], argv[[3]], argv[[4]], argv[[5]])) [[1]] [1] "" "" "\036" "" "isSeekable()" @@ -52680,7 +52680,7 @@ $lrow attr(,"row.names") [1] 1 -##com.oracle.truffle.r.test.builtins.TestBuiltin_unclass.testunclass26#Ignored.Unknown# +##com.oracle.truffle.r.test.builtins.TestBuiltin_unclass.testunclass26# #argv <- list(structure(list(a = 1), .Dim = 1L, .Dimnames = list('a')));unclass(argv[[1]]); $a [1] 1 @@ -57556,6 +57556,33 @@ NULL #n <- 100; { e <- parent.frame(2); e$n} [1] 100 +##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreErrorContext# +#if (exists('.fastr.identity')) { cat('Error in .Internal(foo(44)): IgnoreErrorContext') } else { cat('Error in foo(42): IgnoreErrorContext') } +Error in foo(42): IgnoreErrorContext +##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreErrorContext# +#if (exists('.fastr.identity')) { cat('Error in foo(42) : IgnoreErrorContext extra spaces') } else { cat('Error in foo(42): IgnoreErrorContext extra spaces') } +Error in foo(42): IgnoreErrorContext extra spaces +##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreWhitespace# +#if (exists('.fastr.identity')) { cat('Error in foo(42) : IgnoreWhitespace extra spaces') } else { cat('Error in foo(42): \nIgnoreWhitespace extra spaces') } +Error in foo(42): +IgnoreWhitespace extra spaces +##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreErrorMessage# +#if (exists('.fastr.identity')) { cat('Error in foo(42): FastR error message') } else { cat('Error in foo(42): GnuR error message is different') } +Error in foo(42): GnuR error message is different +##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreErrorMessage# +#if (exists('.fastr.identity')) { cat('Error in foo(42): IgnoreErrorMessage starts with newline') } else { cat('Error in foo(42):\nIgnoreErrorMessage starts with newline') } +Error in foo(42): +IgnoreErrorMessage starts with newline +##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreErrorMessage# +#if (exists('.fastr.identity')) { cat('Error in foo(44): IgnoreErrorMessage with different ctx') } else { cat('Error in foo(42): IgnoreErrorMessage with different ctx') } +Error in foo(42): IgnoreErrorMessage with different ctx +##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreWarningContext# +#if (exists('.fastr.identity')) { cat('Warning message: In .Internal(foo(42)) : IgnoreWarningContext extra newline') } else { cat('Warning message: In foo(42) : \nIgnoreWarningContext extra newline') } +Warning message: In foo(42) : +IgnoreWarningContext extra newline +##com.oracle.truffle.r.test.builtins.TestTestBase.testTraits#Output.IgnoreWarningContext# +#if (exists('.fastr.identity')) { cat('Warning message: In .Internal(foo(42)) : IgnoreWarningContext') } else { cat('Warning message: In foo(42) : IgnoreWarningContext') } +Warning message: In foo(42) : IgnoreWarningContext ##com.oracle.truffle.r.test.functions.TestFunctions.testArgEvaluationOrder# #v <- 1; class(v) <- 'foo'; `-.foo` <- function(x,y,...) { cat('[-.foo]'); 1234 }; g <- function(...) { cat('[ing]'); ({cat('[-]'); `-`})(...) }; ({cat('[g]'); g})({cat('[x]'); v},{cat('[y]'); 3},{cat('[z]'); 5}) [g][ing][-][x][y][z][-.foo][1] 1234 @@ -111390,7 +111417,7 @@ In rbinom("aa", 10, 0.5) : NAs introduced by coercion #set.seed(42); rbinom(c(1,2), 11:12, c(0.1, 0.5, 0.9)) [1] 3 9 -##com.oracle.truffle.r.test.library.stats.TestExternal_rnbinom.testRbinomWithMu#Ignored.Unimplemented# +##com.oracle.truffle.r.test.library.stats.TestExternal_rnbinom.testRbinomWithMu#Ignored.Unstable# #set.seed(42); rnbinom(100, c(-1, 0, 1, 0.8, 10, NA, NaN, 1/0, -1/0), mu=c(-1, 0, 1, 0.8, 3, 10, NA, NaN, 1/0, -1/0)) [1] NaN NaN 1 1 4 NaN NaN NaN NaN NaN NaN 0 0 0 NaN NaN NaN NaN [19] NaN NaN NaN 0 0 NaN NaN 5 NaN NaN NaN NaN NaN 0 NaN NaN 5 NaN diff --git a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/TestBase.java b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/TestBase.java index f656624b7f..173cc5b2b3 100644 --- a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/TestBase.java +++ b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/TestBase.java @@ -22,6 +22,8 @@ import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; import java.util.HashMap; import java.util.Map.Entry; import java.util.SortedMap; @@ -63,14 +65,17 @@ public class TestBase { public static final boolean ProcessFailedTests = Boolean.getBoolean("ProcessFailedTests"); + /** + * See {@link com.oracle.truffle.r.test.builtins.TestTestBase} for examples. + */ public enum Output implements TestTrait { IgnoreErrorContext, // the error context is ignored (e.g., "a+b" vs. "a + b") IgnoreErrorMessage, // the actual error message is ignored IgnoreWarningContext, // the warning context is ignored - MayIgnoreErrorContext, + MayIgnoreErrorContext, // like IgnoreErrorContext, but no warning if the messages match MayIgnoreWarningContext, ContainsReferences, // replaces references in form of 0xbcdef1 for numbers - IgnoreWhitespace; + IgnoreWhitespace; // removes all whitespace from the whole output @Override public String getName() { @@ -572,37 +577,51 @@ public class TestBase { } } - private void evalAndCompare(String[] inputs, TestTrait... traits) { - WhiteList[] whiteLists = TestTrait.collect(traits, WhiteList.class); - - boolean ignored = TestTrait.contains(traits, Ignored.class) ^ (ProcessFailedTests && !(TestTrait.contains(traits, Ignored.Unstable) || TestTrait.contains(traits, Ignored.SideEffects))); + /** + * Wraps the traits, there are some meta-traits like {@link #isIgnored}, other traits can be + * accessed through the corresponding enum-set. + */ + private static class TestTraitsSet { + EnumSet<Ignored> ignored = EnumSet.noneOf(Ignored.class); + EnumSet<Output> output = EnumSet.noneOf(Output.class); + EnumSet<Context> context = EnumSet.noneOf(Context.class); + boolean isIgnored; + boolean containsError; + + TestTraitsSet(TestTrait[] traits) { + ignored.addAll(Arrays.asList(TestTrait.collect(traits, Ignored.class))); + output.addAll(Arrays.asList(TestTrait.collect(traits, Output.class))); + context.addAll(Arrays.asList(TestTrait.collect(traits, Context.class))); + containsError = (!FULL_COMPARE_ERRORS && (output.contains(Output.IgnoreErrorContext) || output.contains(Output.IgnoreErrorMessage))); + isIgnored = ignored.size() > 0 ^ (ProcessFailedTests && !(ignored.contains(Ignored.Unstable) || ignored.contains(Ignored.SideEffects))); + assert !output.contains(Output.IgnoreWhitespace) || output.size() == 1 : "IgnoreWhitespace trait does not work with any other Output trait"; - boolean containsWarning = TestTrait.contains(traits, Output.IgnoreWarningContext); - boolean containsError = (!FULL_COMPARE_ERRORS && (TestTrait.contains(traits, Output.IgnoreErrorContext) || TestTrait.contains(traits, Output.IgnoreErrorMessage))); - boolean mayContainWarning = TestTrait.contains(traits, Output.MayIgnoreWarningContext); - boolean mayContainError = TestTrait.contains(traits, Output.MayIgnoreErrorContext); - boolean ambiguousError = TestTrait.contains(traits, Output.IgnoreErrorMessage); - boolean ignoreWhitespace = TestTrait.contains(traits, Output.IgnoreWhitespace); - boolean containsReferences = TestTrait.contains(traits, Output.ContainsReferences); - boolean nonSharedContext = TestTrait.contains(traits, Context.NonShared); - boolean longTimeout = TestTrait.contains(traits, Context.LongTimeout); + } - ContextInfo contextInfo = nonSharedContext ? fastROutputManager.fastRSession.createContextInfo(ContextKind.SHARE_NOTHING) : null; + String preprocessOutput(String out) { + if (output.contains(Output.IgnoreWhitespace)) { + return out.replaceAll("\\s+", ""); + } + if (output.contains(Output.ContainsReferences)) { + return convertReferencesInOutput(out); + } + return out; + } + } + private void evalAndCompare(String[] inputs, TestTrait... traitsList) { + WhiteList[] whiteLists = TestTrait.collect(traitsList, WhiteList.class); + TestTraitsSet traits = new TestTraitsSet(traitsList); + ContextInfo contextInfo = traits.context.contains(Context.NonShared) ? fastROutputManager.fastRSession.createContextInfo(ContextKind.SHARE_NOTHING) : null; int index = 1; boolean allOk = true; for (String input : inputs) { - String expected = expectedEval(input, traits); - if (ignored || generatingExpected()) { + String expected = expectedEval(input, traitsList); + if (traits.isIgnored || generatingExpected()) { ignoredInputCount++; } else { - String result = fastREval(input, contextInfo, longTimeout); - if (ignoreWhitespace) { - expected = expected.replaceAll("\\s+", ""); - result = result.replaceAll("\\s+", ""); - } - - CheckResult checkResult = checkResult(whiteLists, input, expected, result, containsWarning, mayContainWarning, containsError, mayContainError, ambiguousError, containsReferences); + String result = fastREval(input, contextInfo, traits.context.contains(Context.LongTimeout)); + CheckResult checkResult = checkResult(whiteLists, input, traits.preprocessOutput(expected), traits.preprocessOutput(result), traits); result = checkResult.result; expected = checkResult.expected; @@ -638,7 +657,7 @@ public class TestBase { } index++; } - if (ignored) { + if (traits.isIgnored) { ignoredTestCount++; } else if (allOk) { successfulTestCount++; @@ -660,25 +679,20 @@ public class TestBase { } } - private CheckResult checkResult(WhiteList[] whiteLists, String input, String originalExpected, String originalResult, boolean containsWarning, boolean mayContainWarning, boolean containsError, - boolean mayContainError, boolean ambiguousError, boolean convertReferences) { + private CheckResult checkResult(WhiteList[] whiteLists, String input, String originalExpected, String originalResult, TestTraitsSet traits) { boolean ok; String result = originalResult; String expected = originalExpected; - if (convertReferences) { - result = convertReferencesInOutput(result); - expected = convertReferencesInOutput(expected); - } - if (expected.equals(result) || searchWhiteLists(whiteLists, input, expected, result, containsWarning, mayContainWarning, containsError, mayContainError, ambiguousError, convertReferences)) { + if (expected.equals(result) || searchWhiteLists(whiteLists, input, expected, result, traits)) { ok = true; - if (containsError && !ambiguousError) { + if (traits.containsError && !traits.output.contains(Output.IgnoreErrorMessage)) { System.out.println("unexpected correct error message: " + getTestContext()); } - if (containsWarning) { + if (traits.output.contains(Output.IgnoreWarningContext)) { System.out.println("unexpected correct warning message: " + getTestContext()); } } else { - if (containsWarning || (mayContainWarning && expected.contains(WARNING))) { + if (traits.output.contains(Output.IgnoreWarningContext) || (traits.output.contains(Output.MayIgnoreWarningContext) && expected.contains(WARNING))) { String resultWarning = getWarningMessage(result); String expectedWarning = getWarningMessage(expected); ok = resultWarning.equals(expectedWarning); @@ -688,8 +702,8 @@ public class TestBase { ok = true; } if (ok) { - if (containsError || (mayContainError && expected.startsWith(ERROR))) { - ok = result.startsWith(ERROR) && (ambiguousError || checkMessageStripped(expected, result) || checkMessageVectorInIndex(expected, result)); + if (traits.containsError || (traits.output.contains(Output.MayIgnoreErrorContext) && expected.startsWith(ERROR))) { + ok = result.startsWith(ERROR) && (traits.output.contains(Output.IgnoreErrorMessage) || checkMessageStripped(expected, result) || checkMessageVectorInIndex(expected, result)); } else { ok = expected.equals(result); } @@ -714,8 +728,7 @@ public class TestBase { return result; } - private boolean searchWhiteLists(WhiteList[] whiteLists, String input, String expected, String result, boolean containsWarning, boolean mayContainWarning, boolean containsError, - boolean mayContainError, boolean ambiguousError, boolean convertReferences) { + private boolean searchWhiteLists(WhiteList[] whiteLists, String input, String expected, String result, TestTraitsSet testTraits) { if (whiteLists == null) { return false; } @@ -723,13 +736,13 @@ public class TestBase { WhiteList.Results wlr = list.get(input); if (wlr != null) { // Sanity check that "expected" matches the entry in the WhiteList - CheckResult checkedResult = checkResult(null, input, wlr.expected, expected, containsWarning, mayContainWarning, containsError, mayContainError, ambiguousError, convertReferences); + CheckResult checkedResult = checkResult(null, input, wlr.expected, expected, testTraits); if (!checkedResult.ok) { System.out.println("expected output does not match: " + wlr.expected + " vs. " + expected); return false; } // Substitute the FastR output and try to match that - CheckResult fastRResult = checkResult(null, input, wlr.fastR, result, containsWarning, mayContainWarning, containsError, mayContainError, ambiguousError, convertReferences); + CheckResult fastRResult = checkResult(null, input, wlr.fastR, result, testTraits); if (fastRResult.ok) { list.markUsed(input); return true; @@ -745,7 +758,7 @@ public class TestBase { private static final Pattern warningPattern4 = Pattern.compile("^(?<pre>.*)Warning messages:\n1:(?<msg0>.*)\n2:(?<msg1>.*)$", Pattern.DOTALL); private static final Pattern warningPattern5 = Pattern.compile("^(?<pre>.*)Warning message:(?<msg0>.*)$", Pattern.DOTALL); - private static final Pattern warningMessagePattern = Pattern.compile("^\n? ? ?(?:In .* :[ \n])?(?<m>[^\n]*)\n?$", Pattern.DOTALL); + private static final Pattern warningMessagePattern = Pattern.compile("^\n? ? ?(?:In .* :[ \n])?[ \n]*(?<m>[^\n]*)\n?$", Pattern.DOTALL); private static final Pattern[] warningPatterns = new Pattern[]{warningPattern1, warningPattern2, warningPattern3, warningPattern4, warningPattern5}; diff --git a/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/builtins/TestTestBase.java b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/builtins/TestTestBase.java new file mode 100644 index 0000000000..235264d822 --- /dev/null +++ b/com.oracle.truffle.r.test/src/com/oracle/truffle/r/test/builtins/TestTestBase.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.truffle.r.test.builtins; + +import org.junit.Ignore; +import org.junit.Test; + +import com.oracle.truffle.r.test.TestBase; + +/** + * Documents and tests the testing API. + */ +public class TestTestBase extends TestBase { + @Test + public void testTraits() { + assertEval(Output.IgnoreErrorMessage, getCode("cat('Error in foo(42): FastR error message')", "cat('Error in foo(42): GnuR error message is different')")); + assertEval(Output.IgnoreErrorMessage, getCode("cat('Error in foo(44): IgnoreErrorMessage with different ctx')", "cat('Error in foo(42): IgnoreErrorMessage with different ctx')")); + assertEval(Output.IgnoreErrorMessage, getCode("cat('Error in foo(42): IgnoreErrorMessage starts with newline')", "cat('Error in foo(42):\\nIgnoreErrorMessage starts with newline')")); + + assertEval(Output.IgnoreErrorContext, getCode("cat('Error in .Internal(foo(44)): IgnoreErrorContext')", "cat('Error in foo(42): IgnoreErrorContext')")); + assertEval(Output.IgnoreErrorContext, getCode("cat('Error in foo(42) : IgnoreErrorContext extra spaces')", "cat('Error in foo(42): IgnoreErrorContext extra spaces')")); + + assertEval(Output.IgnoreWarningContext, getCode("cat('Warning message: In .Internal(foo(42)) : IgnoreWarningContext')", "cat('Warning message: In foo(42) : IgnoreWarningContext')")); + assertEval(Output.IgnoreWarningContext, + getCode("cat('Warning message: In .Internal(foo(42)) : IgnoreWarningContext extra newline')", "cat('Warning message: In foo(42) : \\nIgnoreWarningContext extra newline')")); + + assertEval(Output.IgnoreWhitespace, getCode("cat('Error in foo(42) : IgnoreWhitespace extra spaces')", "cat('Error in foo(42): \\nIgnoreWhitespace extra spaces')")); + } + + @Test + @Ignore // these tests should fail + public void negativeTestTraits() { + assertEval(Output.IgnoreErrorContext, getCode("cat('Error in foo(44): IgnoreErrorContext diff message')", "cat('Error in foo(42): IgnoreErrorContext should fail')")); + assertEval(Output.IgnoreWarningContext, + getCode("cat('Warning message: In .Internal(foo(42)): IgnoreWarningContext diff message')", "cat('Warning message in foo(42): IgnoreWarningContext should fail')")); + } + + private String getCode(String fastr, String gnur) { + return "if (exists('.fastr.identity')) { " + fastr + " } else { " + gnur + " }"; + } +} -- GitLab