diff --git a/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/PTAMain.java b/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/PTAMain.java
index a915581347a5476d9b47a1441cb7610631c45770..1febab1b4e9f77000910b2c08ecaa12005e05ea7 100644
--- a/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/PTAMain.java
+++ b/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/PTAMain.java
@@ -85,6 +85,7 @@ public class PTAMain {
parser.registerOption("glob", "*");
parser.registerOption("since", "last2weeks");
parser.registerOption("console");
+ parser.registerOption("verbose");
String[] remainingArgs = parser.parseOptions(args);
if (parser.has("help")) {
@@ -151,13 +152,20 @@ public class PTAMain {
LogManager.getLogManager().reset();
Logger rootLogger = Logger.getLogger("");
ConsoleHandler consoleHandler = new ConsoleHandler();
+
+ Level defaultLogLevel = Level.INFO;
+ if (parser.has("verbose")) {
+ defaultLogLevel = Level.ALL;
+ }
+ rootLogger.setLevel(defaultLogLevel);
+
if (parser.has("console")) {
- consoleHandler.setLevel(Level.INFO);
+ consoleHandler.setLevel(defaultLogLevel);
} else {
// set log level of console handlers to SEVERE
consoleHandler.setLevel(Level.SEVERE);
FileHandler fileHandler = new FileHandler(LOG_FILE_NAME);
- fileHandler.setLevel(Level.INFO);
+ fileHandler.setLevel(defaultLogLevel);
fileHandler.setFormatter(new SimpleFormatter());
rootLogger.addHandler(fileHandler);
}
@@ -281,8 +289,12 @@ public class PTAMain {
LogFile parseLogFile = lfParser.parseLogFile();
Collection<Problem> problems = parseLogFile.collectProblems();
+ pkgTestRun.setSuccess(parseLogFile.isSuccess());
+
+ // log problems
+ LOGGER.fine("Overall test result: " + (pkgTestRun.isSuccess() ? "OK" : "FAILED"));
for (Problem problem : problems) {
- LOGGER.info(problem.toString());
+ LOGGER.fine(problem.toString());
}
return problems;
diff --git a/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/RPackageTestRun.java b/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/RPackageTestRun.java
index 72526ca7183a89c95cc8a32dceea96bb40073a66..2bc9a132de822b6a93f650ca25b24d57b0920ade 100644
--- a/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/RPackageTestRun.java
+++ b/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/RPackageTestRun.java
@@ -31,6 +31,9 @@ public class RPackageTestRun {
private final int nr;
private Collection<Problem> problems;
+ /** The overall outcome of the package test run as reported. */
+ private boolean success;
+
protected RPackageTestRun(RPackage pkg, int nr) {
this.pkg = pkg;
this.nr = nr;
@@ -60,4 +63,46 @@ public class RPackageTestRun {
return pkg + "/" + nr;
}
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + nr;
+ result = prime * result + ((pkg == null) ? 0 : pkg.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ RPackageTestRun other = (RPackageTestRun) obj;
+ if (nr != other.nr) {
+ return false;
+ }
+ if (pkg == null) {
+ if (other.pkg != null) {
+ return false;
+ }
+ } else if (!pkg.equals(other.pkg)) {
+ return false;
+ }
+ return true;
+ }
+
+ public void setSuccess(boolean success) {
+ this.success = success;
+ }
+
+ public boolean isSuccess() {
+ return success;
+ }
+
}
diff --git a/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/AbstractDumper.java b/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/AbstractDumper.java
index 76cae7fb6eae58bb6f60218259b03d5e650470ce..b4912482bea6939ca516250791a1f1e0acfd9f78 100644
--- a/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/AbstractDumper.java
+++ b/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/AbstractDumper.java
@@ -29,6 +29,7 @@ import java.util.stream.Collectors;
import com.oracle.truffle.r.test.packages.analyzer.Problem;
import com.oracle.truffle.r.test.packages.analyzer.RPackage;
+import com.oracle.truffle.r.test.packages.analyzer.RPackageTestRun;
public abstract class AbstractDumper {
@@ -42,4 +43,8 @@ public abstract class AbstractDumper {
return problems.stream().collect(Collectors.groupingBy(p -> p.getPackageTestRun().getPackage()));
}
+ protected Map<RPackageTestRun, List<Problem>> groupByTestRuns(Collection<Problem> problems) {
+ return problems.stream().collect(Collectors.groupingBy(p -> p.getPackageTestRun()));
+ }
+
}
diff --git a/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/HtmlDumper.java b/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/HtmlDumper.java
index 7763747e920f7f4074cd691ff0143d09c0096564..62af7b87822dea3717c04213ac07a190a555c081 100644
--- a/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/HtmlDumper.java
+++ b/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/HtmlDumper.java
@@ -38,6 +38,7 @@ import java.util.stream.Collectors;
import com.oracle.truffle.r.test.packages.analyzer.Location;
import com.oracle.truffle.r.test.packages.analyzer.Problem;
import com.oracle.truffle.r.test.packages.analyzer.RPackage;
+import com.oracle.truffle.r.test.packages.analyzer.RPackageTestRun;
import com.oracle.truffle.r.test.packages.analyzer.dump.html.HTMLBuilder;
import com.oracle.truffle.r.test.packages.analyzer.dump.html.HTMLBuilder.Tag;
@@ -77,8 +78,8 @@ public class HtmlDumper extends AbstractDumper {
try (BufferedWriter bw = Files.newBufferedWriter(indexFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
HTMLBuilder builder = new HTMLBuilder(new PrintWriter(bw));
- Tag errorDistributionTable = generateDistributionTable(builder, groupByType(problems));
- Tag pkgDistributionTable = generatePkgDistributionTable(builder, groupByPkg(problems));
+ Tag errorDistributionTable = generateTypeDistributionTable(builder, groupByType(problems));
+ Tag pkgDistributionTable = generateTestRunDistributionTable(builder, groupByTestRuns(problems));
builder.html(builder.head(builder.title(TITLE)),
builder.body(builder.h1(TITLE), builder.h2("Distribution by Problem Type"), errorDistributionTable, builder.h2("Distribution by Package"), pkgDistributionTable));
@@ -88,19 +89,32 @@ public class HtmlDumper extends AbstractDumper {
}
}
- private Tag generatePkgDistributionTable(HTMLBuilder builder, Map<RPackage, List<Problem>> groupByPkg) {
- List<RPackage> collect = groupByPkg.keySet().stream().sorted((a, b) -> Integer.compare(groupByPkg.get(b).size(), groupByPkg.get(a).size())).collect(Collectors.toList());
-
- Tag table = builder.table();
- for (RPackage pkg : collect) {
- String pkgFileName = dumpRPackage(pkg, groupByPkg);
- table.addChild(builder.tr(builder.td(builder.a(pkgFileName, pkg.toString())),
- builder.td(Integer.toString(groupByPkg.get(pkg).size()))));
+ private Tag generateTestRunDistributionTable(HTMLBuilder builder, Map<RPackageTestRun, List<Problem>> groupByPkg) {
+ List<RPackageTestRun> collect = groupByPkg.keySet().stream().sorted((a, b) -> Integer.compare(groupByPkg.get(b).size(), groupByPkg.get(a).size())).collect(Collectors.toList());
+
+ Tag table = builder.table(builder.tr(
+ builder.th("Package"),
+ builder.th("Test Run"),
+ builder.th("Result"),
+ builder.th("Problem Count")));
+
+ for (RPackageTestRun testRun : collect) {
+ String pkgFileName = dumpRPackage(testRun, groupByPkg);
+ int n = groupByPkg.get(testRun).size();
+ Tag tableRow = builder.tr(
+ builder.td(builder.a(pkgFileName, testRun.getPackage().toString())),
+ builder.td(Integer.toString(testRun.getNr())),
+ builder.td(testRun.isSuccess() ? "OK" : "FAILED"),
+ builder.td(Integer.toString(n)));
+ if (testRun.isSuccess() && n > 0) {
+ tableRow.addAttribute("bgcolor", "#fdae61");
+ }
+ table.addChild(tableRow);
}
return table;
}
- private Tag generateDistributionTable(HTMLBuilder builder, Map<Class<? extends Problem>, List<Problem>> groupByType) {
+ private Tag generateTypeDistributionTable(HTMLBuilder builder, Map<Class<? extends Problem>, List<Problem>> groupByType) {
List<Class<? extends Problem>> collect = groupByType.keySet().stream().sorted((a, b) -> Integer.compare(groupByType.get(b).size(), groupByType.get(a).size())).collect(Collectors.toList());
Tag table = builder.table();
@@ -111,9 +125,9 @@ public class HtmlDumper extends AbstractDumper {
return table;
}
- private String dumpRPackage(RPackage pkg, Map<RPackage, List<Problem>> groupByPkg) {
+ private String dumpRPackage(RPackageTestRun pkg, Map<RPackageTestRun, List<Problem>> groupByPkg) {
- Path problemClassFile = destDir.resolve(pkg.toString() + ".html");
+ Path problemClassFile = destDir.resolve(pkg.getPackage().toString() + "_" + pkg.getNr() + ".html");
try (BufferedWriter bw = Files.newBufferedWriter(problemClassFile, StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
HTMLBuilder builder = new HTMLBuilder(new PrintWriter(bw));
diff --git a/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/html/HTMLBuilder.java b/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/html/HTMLBuilder.java
index 2dc24da79960e589373937e7e8c7daec97104f19..b870ecfd9eec8f47984118f3f4344d4222a73c70 100644
--- a/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/html/HTMLBuilder.java
+++ b/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/html/HTMLBuilder.java
@@ -74,6 +74,14 @@ public class HTMLBuilder {
return generic("tr", children);
}
+ public Tag th(String content) {
+ return generic("th", content);
+ }
+
+ public Tag th(Tag... children) {
+ return generic("th", children);
+ }
+
public Tag td(String content) {
return generic("td", content);
}
diff --git a/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/parser/LogFileParser.java b/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/parser/LogFileParser.java
index e3d6a9713002cf363e50e0181315e54e3dbfca69..d828fc464bf82ad7111e0e9ca31fe20e89a04e36 100644
--- a/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/parser/LogFileParser.java
+++ b/com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/parser/LogFileParser.java
@@ -33,6 +33,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.logging.Logger;
import java.util.stream.Collectors;
import com.oracle.truffle.r.test.packages.analyzer.Location;
@@ -44,6 +45,8 @@ import com.oracle.truffle.r.test.packages.analyzer.parser.DiffParser.DiffChunk;
public class LogFileParser {
+ private static final Logger LOGGER = Logger.getLogger(LogFileParser.class.getName());
+
private RPackageTestRun pkg;
private BufferedReader reader;
private Line curLine;
@@ -83,12 +86,12 @@ public class LogFileParser {
this.reader = r;
consumeLine();
Section installTest = parseInstallTest();
- if (!installTest.success()) {
+ if (!installTest.isSuccess()) {
return logFile;
}
- logFile.sections.add(parseInstallTest());
- logFile.sections.add(parseCheckResults());
- logFile.success = parseOverallStatus();
+ logFile.addSection(parseInstallTest());
+ logFile.addSection(parseCheckResults());
+ logFile.setSuccess(parseOverallStatus());
expectEOF();
} finally {
this.reader = null;
@@ -118,6 +121,7 @@ public class LogFileParser {
consumeLine();
Section checkResults = new Section(logFile, Token.BEGIN_CHECKING.linePrefix, curLine.lineNr);
+ checkResults.problems = new LinkedList<>();
// TODO depending on the result, parse other files
if (curLine.text.contains(Token.FAIL_OUTPUT_GNUR.linePrefix)) {
@@ -133,7 +137,20 @@ public class LogFileParser {
// format: <pkg name>: FastR output mismatch: <out file name>
int idx = curLine.text.indexOf(Token.OUTPUT_MISMATCH_FASTR.linePrefix);
String fileNameStr = curLine.text.substring(idx + Token.OUTPUT_MISMATCH_FASTR.linePrefix.length()).trim();
- checkResults.problems = parseOutputFile(logFile.path.resolveSibling(fileNameStr));
+ Path outputFile = logFile.path.resolveSibling(fileNameStr);
+
+ // report the problem
+ checkResults.problems.add(new OutputMismatchProblem(pkg, getCurrentLocation(), fileNameStr));
+
+ if (!Files.isReadable(outputFile)) {
+ LOGGER.warning("Cannot read output file " + outputFile);
+
+ // consume any lines to be able to continue
+ collectBody(Token.END_CHECKING);
+ } else {
+ checkResults.problems.addAll(applyDetectors(Token.OUTPUT_MISMATCH_FASTR, outputFile, 0, Files.readAllLines(outputFile)));
+ }
+ checkResults.setSuccess(false);
} else {
throw new LogFileParseException("Unexpected checking message: " + curLine.text);
}
@@ -142,12 +159,8 @@ public class LogFileParser {
return checkResults;
}
- private Collection<Problem> parseOutputFile(Path path) throws IOException {
- if (!Files.isReadable(path)) {
- throw new LogFileParseException("Cannot read output file " + path);
- }
-
- return applyDetectors(Token.OUTPUT_MISMATCH_FASTR, path, 0, Files.readAllLines(path));
+ private Location getCurrentLocation() {
+ return new Location(logFile.path, curLine.lineNr);
}
private Section parseInstallTest() throws IOException {
@@ -161,13 +174,13 @@ public class LogFileParser {
}
Section installationTask = parseInstallationTask();
- installTest.subsections.add(installationTask);
+ installTest.addSection(installationTask);
if ("FastR".equals(mode)) {
- installTest.success = parseInstallStatus() && success;
+ installTest.setSuccess(parseInstallStatus() && success);
}
parseInstallSuggests();
- installTest.success = parseTesting() && success;
+ installTest.setSuccess(parseTesting() && success);
expect(Token.END_INSTALL_TEST);
return installTest;
@@ -254,7 +267,7 @@ public class LogFileParser {
expect(Token.BEGIN_INSTALLATION);
Section installation = new Section(logFile, Token.BEGIN_INSTALLATION.linePrefix, curLine.lineNr);
Section processing = parseProcessingTask();
- installation.subsections.add(processing);
+ installation.addSection(processing);
expect(Token.END_INSTALLATION);
return installation;
}
@@ -511,22 +524,44 @@ public class LogFileParser {
}
- public static class LogFile {
+ public abstract static class AbstractSection {
+ private List<Section> sections = new LinkedList<>();
+ private boolean success;
+
+ public List<Section> getSections() {
+ return sections;
+ }
+
+ public void addSection(Section sub) {
+ sections.add(sub);
+ }
+
+ public abstract Collection<Problem> collectProblems();
+
+ public boolean isSuccess() {
+ return success;
+ }
+
+ public void setSuccess(boolean success) {
+ this.success = success;
+ }
+
+ }
+
+ public static class LogFile extends AbstractSection {
public LogFile(Path path) {
this.path = path;
}
private Path path;
- private List<Section> sections = new LinkedList<>();
- private boolean success;
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("LogFile ").append(path);
- if (!sections.isEmpty()) {
+ if (!getSections().isEmpty()) {
sb.append("(");
- for (Section section : sections) {
+ for (Section section : getSections()) {
sb.append(section).append(", ");
}
sb.append(")");
@@ -534,44 +569,31 @@ public class LogFileParser {
return sb.toString();
}
- public Path getPath() {
- return path;
- }
-
- public List<Section> getSections() {
- return sections;
+ @Override
+ public Collection<Problem> collectProblems() {
+ return getSections().stream().flatMap(s -> s.collectProblems().stream()).collect(Collectors.toList());
}
- public boolean isSuccess() {
- return success;
+ public Path getPath() {
+ return path;
}
- public Collection<Problem> collectProblems() {
- List<Problem> problems = new LinkedList<>();
- for (Section sec : sections) {
- problems.addAll(Section.collectProblems(sec));
- }
- return problems;
- }
}
- public static class Section {
+ public static class Section extends AbstractSection {
private String name;
- private LogFile logFile;
private int startLine;
+ private AbstractSection parent;
Collection<Problem> problems;
- Collection<Section> subsections = new LinkedList<>();
- private boolean success;
- protected Section(LogFile logFile, String name, int startLine) {
- this.logFile = logFile;
+ protected Section(AbstractSection parent, String name, int startLine) {
+ this.parent = parent;
this.startLine = startLine;
this.name = name;
- logFile.sections.add(this);
}
- public boolean success() {
- return success;
+ public AbstractSection getParent() {
+ return parent;
}
@Override
@@ -579,19 +601,43 @@ public class LogFileParser {
return String.format("Section %s (start: %d, problems: %d)", name, startLine, problems != null ? problems.size() : 0);
}
- public static Collection<Problem> collectProblems(Section s) {
- List<Problem> problems = new LinkedList<>();
- if (s.problems != null) {
- problems.addAll(s.problems);
+ @Override
+ public Collection<Problem> collectProblems() {
+ Collection<Problem> collected = new ArrayList<>();
+ if (problems != null) {
+ collected.addAll(problems);
}
- for (Section sec : s.subsections) {
- if (sec.problems != null) {
- problems.addAll(sec.problems);
- }
+ for (Section child : getSections()) {
+ collected.addAll(child.collectProblems());
}
- return problems;
+ return collected;
}
}
+ public static class OutputMismatchProblem extends Problem {
+
+ private final String details;
+
+ protected OutputMismatchProblem(RPackageTestRun pkg, Location location, String details) {
+ super(pkg, location);
+ this.details = details;
+ }
+
+ @Override
+ public String getSummary() {
+ return Token.OUTPUT_MISMATCH_FASTR.linePrefix;
+ }
+
+ @Override
+ public String getDetails() {
+ return details;
+ }
+
+ @Override
+ public String toString() {
+ return getSummary() + details;
+ }
+ }
+
}