Skip to content
Snippets Groups Projects
Commit 50a871e0 authored by Florian Angerer's avatar Florian Angerer
Browse files

Added verbose argument; restructuring of Section hierarchy; storing and 

dumping overall success of packge test.
parent 06b1e551
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 199 additions and 69 deletions
com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/PTAMain.java
+ 15
− 3
View file @ 50a871e0
......@@ -85,6 +85,7 @@ public class PTAMain {
parser.registerOption("glob", "*");
parser.registerOption("since", "last2weeks");
parser.registerOption("console");
parser.registerOption("verbose");
String[] remainingArgs = parser.parseOptions(args);
if (parser.has("help")) {
......@@ -151,13 +152,20 @@ public class PTAMain {
LogManager.getLogManager().reset();
Logger rootLogger = Logger.getLogger("");
ConsoleHandler consoleHandler = new ConsoleHandler();
Level defaultLogLevel = Level.INFO;
if (parser.has("verbose")) {
defaultLogLevel = Level.ALL;
}
rootLogger.setLevel(defaultLogLevel);
if (parser.has("console")) {
consoleHandler.setLevel(Level.INFO);
consoleHandler.setLevel(defaultLogLevel);
} else {
// set log level of console handlers to SEVERE
consoleHandler.setLevel(Level.SEVERE);
FileHandler fileHandler = new FileHandler(LOG_FILE_NAME);
fileHandler.setLevel(Level.INFO);
fileHandler.setLevel(defaultLogLevel);
fileHandler.setFormatter(new SimpleFormatter());
rootLogger.addHandler(fileHandler);
}
......@@ -281,8 +289,12 @@ public class PTAMain {
LogFile parseLogFile = lfParser.parseLogFile();
Collection<Problem> problems = parseLogFile.collectProblems();
pkgTestRun.setSuccess(parseLogFile.isSuccess());
// log problems
LOGGER.fine("Overall test result: " + (pkgTestRun.isSuccess() ? "OK" : "FAILED"));
for (Problem problem : problems) {
LOGGER.info(problem.toString());
LOGGER.fine(problem.toString());
}
return problems;
......
com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/RPackageTestRun.java
+ 45
− 0
View file @ 50a871e0
......@@ -31,6 +31,9 @@ public class RPackageTestRun {
private final int nr;
private Collection<Problem> problems;
/** The overall outcome of the package test run as reported. */
private boolean success;
protected RPackageTestRun(RPackage pkg, int nr) {
this.pkg = pkg;
this.nr = nr;
......@@ -60,4 +63,46 @@ public class RPackageTestRun {
return pkg + "/" + nr;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + nr;
result = prime * result + ((pkg == null) ? 0 : pkg.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
RPackageTestRun other = (RPackageTestRun) obj;
if (nr != other.nr) {
return false;
}
if (pkg == null) {
if (other.pkg != null) {
return false;
}
} else if (!pkg.equals(other.pkg)) {
return false;
}
return true;
}
public void setSuccess(boolean success) {
this.success = success;
}
public boolean isSuccess() {
return success;
}
}
com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/AbstractDumper.java
+ 5
− 0
View file @ 50a871e0
......@@ -29,6 +29,7 @@ import java.util.stream.Collectors;
import com.oracle.truffle.r.test.packages.analyzer.Problem;
import com.oracle.truffle.r.test.packages.analyzer.RPackage;
import com.oracle.truffle.r.test.packages.analyzer.RPackageTestRun;
public abstract class AbstractDumper {
......@@ -42,4 +43,8 @@ public abstract class AbstractDumper {
return problems.stream().collect(Collectors.groupingBy(p -> p.getPackageTestRun().getPackage()));
}
protected Map<RPackageTestRun, List<Problem>> groupByTestRuns(Collection<Problem> problems) {
return problems.stream().collect(Collectors.groupingBy(p -> p.getPackageTestRun()));
}
}
com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/HtmlDumper.java
+ 27
− 13
View file @ 50a871e0
......@@ -38,6 +38,7 @@ import java.util.stream.Collectors;
import com.oracle.truffle.r.test.packages.analyzer.Location;
import com.oracle.truffle.r.test.packages.analyzer.Problem;
import com.oracle.truffle.r.test.packages.analyzer.RPackage;
import com.oracle.truffle.r.test.packages.analyzer.RPackageTestRun;
import com.oracle.truffle.r.test.packages.analyzer.dump.html.HTMLBuilder;
import com.oracle.truffle.r.test.packages.analyzer.dump.html.HTMLBuilder.Tag;
......@@ -77,8 +78,8 @@ public class HtmlDumper extends AbstractDumper {
try (BufferedWriter bw = Files.newBufferedWriter(indexFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
HTMLBuilder builder = new HTMLBuilder(new PrintWriter(bw));
Tag errorDistributionTable = generateDistributionTable(builder, groupByType(problems));
Tag pkgDistributionTable = generatePkgDistributionTable(builder, groupByPkg(problems));
Tag errorDistributionTable = generateTypeDistributionTable(builder, groupByType(problems));
Tag pkgDistributionTable = generateTestRunDistributionTable(builder, groupByTestRuns(problems));
builder.html(builder.head(builder.title(TITLE)),
builder.body(builder.h1(TITLE), builder.h2("Distribution by Problem Type"), errorDistributionTable, builder.h2("Distribution by Package"), pkgDistributionTable));
......@@ -88,19 +89,32 @@ public class HtmlDumper extends AbstractDumper {
}
}
private Tag generatePkgDistributionTable(HTMLBuilder builder, Map<RPackage, List<Problem>> groupByPkg) {
List<RPackage> collect = groupByPkg.keySet().stream().sorted((a, b) -> Integer.compare(groupByPkg.get(b).size(), groupByPkg.get(a).size())).collect(Collectors.toList());
Tag table = builder.table();
for (RPackage pkg : collect) {
String pkgFileName = dumpRPackage(pkg, groupByPkg);
table.addChild(builder.tr(builder.td(builder.a(pkgFileName, pkg.toString())),
builder.td(Integer.toString(groupByPkg.get(pkg).size()))));
private Tag generateTestRunDistributionTable(HTMLBuilder builder, Map<RPackageTestRun, List<Problem>> groupByPkg) {
List<RPackageTestRun> collect = groupByPkg.keySet().stream().sorted((a, b) -> Integer.compare(groupByPkg.get(b).size(), groupByPkg.get(a).size())).collect(Collectors.toList());
Tag table = builder.table(builder.tr(
builder.th("Package"),
builder.th("Test Run"),
builder.th("Result"),
builder.th("Problem Count")));
for (RPackageTestRun testRun : collect) {
String pkgFileName = dumpRPackage(testRun, groupByPkg);
int n = groupByPkg.get(testRun).size();
Tag tableRow = builder.tr(
builder.td(builder.a(pkgFileName, testRun.getPackage().toString())),
builder.td(Integer.toString(testRun.getNr())),
builder.td(testRun.isSuccess() ? "OK" : "FAILED"),
builder.td(Integer.toString(n)));
if (testRun.isSuccess() && n > 0) {
tableRow.addAttribute("bgcolor", "#fdae61");
}
table.addChild(tableRow);
}
return table;
}
private Tag generateDistributionTable(HTMLBuilder builder, Map<Class<? extends Problem>, List<Problem>> groupByType) {
private Tag generateTypeDistributionTable(HTMLBuilder builder, Map<Class<? extends Problem>, List<Problem>> groupByType) {
List<Class<? extends Problem>> collect = groupByType.keySet().stream().sorted((a, b) -> Integer.compare(groupByType.get(b).size(), groupByType.get(a).size())).collect(Collectors.toList());
Tag table = builder.table();
......@@ -111,9 +125,9 @@ public class HtmlDumper extends AbstractDumper {
return table;
}
private String dumpRPackage(RPackage pkg, Map<RPackage, List<Problem>> groupByPkg) {
private String dumpRPackage(RPackageTestRun pkg, Map<RPackageTestRun, List<Problem>> groupByPkg) {
Path problemClassFile = destDir.resolve(pkg.toString() + ".html");
Path problemClassFile = destDir.resolve(pkg.getPackage().toString() + "_" + pkg.getNr() + ".html");
try (BufferedWriter bw = Files.newBufferedWriter(problemClassFile, StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
HTMLBuilder builder = new HTMLBuilder(new PrintWriter(bw));
......
com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/dump/html/HTMLBuilder.java
+ 8
− 0
View file @ 50a871e0
......@@ -74,6 +74,14 @@ public class HTMLBuilder {
return generic("tr", children);
}
public Tag th(String content) {
return generic("th", content);
}
public Tag th(Tag... children) {
return generic("th", children);
}
public Tag td(String content) {
return generic("td", content);
}
......
com.oracle.truffle.r.test.packages.analyzer/src/com/oracle/truffle/r/test/packages/analyzer/parser/LogFileParser.java
+ 99
− 53
View file @ 50a871e0
......@@ -33,6 +33,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import com.oracle.truffle.r.test.packages.analyzer.Location;
......@@ -44,6 +45,8 @@ import com.oracle.truffle.r.test.packages.analyzer.parser.DiffParser.DiffChunk;
public class LogFileParser {
private static final Logger LOGGER = Logger.getLogger(LogFileParser.class.getName());
private RPackageTestRun pkg;
private BufferedReader reader;
private Line curLine;
......@@ -83,12 +86,12 @@ public class LogFileParser {
this.reader = r;
consumeLine();
Section installTest = parseInstallTest();
if (!installTest.success()) {
if (!installTest.isSuccess()) {
return logFile;
}
logFile.sections.add(parseInstallTest());
logFile.sections.add(parseCheckResults());
logFile.success = parseOverallStatus();
logFile.addSection(parseInstallTest());
logFile.addSection(parseCheckResults());
logFile.setSuccess(parseOverallStatus());
expectEOF();
} finally {
this.reader = null;
......@@ -118,6 +121,7 @@ public class LogFileParser {
consumeLine();
Section checkResults = new Section(logFile, Token.BEGIN_CHECKING.linePrefix, curLine.lineNr);
checkResults.problems = new LinkedList<>();
// TODO depending on the result, parse other files
if (curLine.text.contains(Token.FAIL_OUTPUT_GNUR.linePrefix)) {
......@@ -133,7 +137,20 @@ public class LogFileParser {
// format: <pkg name>: FastR output mismatch: <out file name>
int idx = curLine.text.indexOf(Token.OUTPUT_MISMATCH_FASTR.linePrefix);
String fileNameStr = curLine.text.substring(idx + Token.OUTPUT_MISMATCH_FASTR.linePrefix.length()).trim();
checkResults.problems = parseOutputFile(logFile.path.resolveSibling(fileNameStr));
Path outputFile = logFile.path.resolveSibling(fileNameStr);
// report the problem
checkResults.problems.add(new OutputMismatchProblem(pkg, getCurrentLocation(), fileNameStr));
if (!Files.isReadable(outputFile)) {
LOGGER.warning("Cannot read output file " + outputFile);
// consume any lines to be able to continue
collectBody(Token.END_CHECKING);
} else {
checkResults.problems.addAll(applyDetectors(Token.OUTPUT_MISMATCH_FASTR, outputFile, 0, Files.readAllLines(outputFile)));
}
checkResults.setSuccess(false);
} else {
throw new LogFileParseException("Unexpected checking message: " + curLine.text);
}
......@@ -142,12 +159,8 @@ public class LogFileParser {
return checkResults;
}
private Collection<Problem> parseOutputFile(Path path) throws IOException {
if (!Files.isReadable(path)) {
throw new LogFileParseException("Cannot read output file " + path);
}
return applyDetectors(Token.OUTPUT_MISMATCH_FASTR, path, 0, Files.readAllLines(path));
private Location getCurrentLocation() {
return new Location(logFile.path, curLine.lineNr);
}
private Section parseInstallTest() throws IOException {
......@@ -161,13 +174,13 @@ public class LogFileParser {
}
Section installationTask = parseInstallationTask();
installTest.subsections.add(installationTask);
installTest.addSection(installationTask);
if ("FastR".equals(mode)) {
installTest.success = parseInstallStatus() && success;
installTest.setSuccess(parseInstallStatus() && success);
}
parseInstallSuggests();
installTest.success = parseTesting() && success;
installTest.setSuccess(parseTesting() && success);
expect(Token.END_INSTALL_TEST);
return installTest;
......@@ -254,7 +267,7 @@ public class LogFileParser {
expect(Token.BEGIN_INSTALLATION);
Section installation = new Section(logFile, Token.BEGIN_INSTALLATION.linePrefix, curLine.lineNr);
Section processing = parseProcessingTask();
installation.subsections.add(processing);
installation.addSection(processing);
expect(Token.END_INSTALLATION);
return installation;
}
......@@ -511,22 +524,44 @@ public class LogFileParser {
}
public static class LogFile {
public abstract static class AbstractSection {
private List<Section> sections = new LinkedList<>();
private boolean success;
public List<Section> getSections() {
return sections;
}
public void addSection(Section sub) {
sections.add(sub);
}
public abstract Collection<Problem> collectProblems();
public boolean isSuccess() {
return success;
}
public void setSuccess(boolean success) {
this.success = success;
}
}
public static class LogFile extends AbstractSection {
public LogFile(Path path) {
this.path = path;
}
private Path path;
private List<Section> sections = new LinkedList<>();
private boolean success;
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("LogFile ").append(path);
if (!sections.isEmpty()) {
if (!getSections().isEmpty()) {
sb.append("(");
for (Section section : sections) {
for (Section section : getSections()) {
sb.append(section).append(", ");
}
sb.append(")");
......@@ -534,44 +569,31 @@ public class LogFileParser {
return sb.toString();
}
public Path getPath() {
return path;
}
public List<Section> getSections() {
return sections;
@Override
public Collection<Problem> collectProblems() {
return getSections().stream().flatMap(s -> s.collectProblems().stream()).collect(Collectors.toList());
}
public boolean isSuccess() {
return success;
public Path getPath() {
return path;
}
public Collection<Problem> collectProblems() {
List<Problem> problems = new LinkedList<>();
for (Section sec : sections) {
problems.addAll(Section.collectProblems(sec));
}
return problems;
}
}
public static class Section {
public static class Section extends AbstractSection {
private String name;
private LogFile logFile;
private int startLine;
private AbstractSection parent;
Collection<Problem> problems;
Collection<Section> subsections = new LinkedList<>();
private boolean success;
protected Section(LogFile logFile, String name, int startLine) {
this.logFile = logFile;
protected Section(AbstractSection parent, String name, int startLine) {
this.parent = parent;
this.startLine = startLine;
this.name = name;
logFile.sections.add(this);
}
public boolean success() {
return success;
public AbstractSection getParent() {
return parent;
}
@Override
......@@ -579,19 +601,43 @@ public class LogFileParser {
return String.format("Section %s (start: %d, problems: %d)", name, startLine, problems != null ? problems.size() : 0);
}
public static Collection<Problem> collectProblems(Section s) {
List<Problem> problems = new LinkedList<>();
if (s.problems != null) {
problems.addAll(s.problems);
@Override
public Collection<Problem> collectProblems() {
Collection<Problem> collected = new ArrayList<>();
if (problems != null) {
collected.addAll(problems);
}
for (Section sec : s.subsections) {
if (sec.problems != null) {
problems.addAll(sec.problems);
}
for (Section child : getSections()) {
collected.addAll(child.collectProblems());
}
return problems;
return collected;
}
}
public static class OutputMismatchProblem extends Problem {
private final String details;
protected OutputMismatchProblem(RPackageTestRun pkg, Location location, String details) {
super(pkg, location);
this.details = details;
}
@Override
public String getSummary() {
return Token.OUTPUT_MISMATCH_FASTR.linePrefix;
}
@Override
public String getDetails() {
return details;
}
@Override
public String toString() {
return getSummary() + details;
}
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment