diff --git a/README.md b/README.md index 0f3ccd3..0ad120a 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,17 @@ command: go get -u github.com/jstemmer/go-junit-report ``` +## Contribution + +Create an Issue and discuss the fix or feature, then fork the package. +Clone to github.com/jstemmer/go-junit-report. This is necessary because go import uses this path. +Fix or implement feature. Test and then commit change. +Specify #Issue and describe change in the commit message. +Create Pull Request. It can be merged by owner or administrator then. + +## Run Tests +go test + ## Usage go-junit-report reads the `go test` verbose output from standard in and writes @@ -24,6 +35,12 @@ junit compatible XML to standard out. go test -v 2>&1 | go-junit-report > report.xml ``` +Note that it also can parse benchmark output with `-bench` flag: +```bash + go test -bench . -benchmem -count 100 + ``` +will return the average mean benchmark time as the test case time. + [travis-badge]: https://travis-ci.org/jstemmer/go-junit-report.svg [travis-link]: https://travis-ci.org/jstemmer/go-junit-report [report-badge]: https://goreportcard.com/badge/github.com/jstemmer/go-junit-report diff --git a/formatter/formatter.go b/formatter/formatter.go index eca451a..51b4593 100644 --- a/formatter/formatter.go +++ b/formatter/formatter.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "runtime" + "strconv" "strings" "time" @@ -38,6 +39,9 @@ type JUnitTestCase struct { Time string `xml:"time,attr"` SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"` Failure *JUnitFailure `xml:"failure,omitempty"` + // for benchmarks + Bytes string `xml:"bytes,attr,omitempty"` + Allocs string `xml:"allocs,attr,omitempty"` } // JUnitSkipMessage contains the reason why a testcase was skipped. @@ -65,8 +69,17 @@ func JUnitReportXML(report *parser.Report, noXMLHeader bool, goVersion string, w // convert Report to JUnit test suites for _, pkg := range report.Packages { + var tests int + if len(pkg.Tests) >= 1 && len(pkg.Benchmarks) >= 1 { + tests = len(pkg.Tests) + len(pkg.Benchmarks) + } else if len(pkg.Benchmarks) >= 1 { + tests = len(pkg.Benchmarks) + } else { + tests = len(pkg.Tests) + } + ts := JUnitTestSuite{ - Tests: len(pkg.Tests), + Tests: tests, Failures: 0, Time: formatTime(pkg.Duration), Name: pkg.Name, @@ -114,6 +127,25 @@ func JUnitReportXML(report *parser.Report, noXMLHeader bool, goVersion string, w ts.TestCases = append(ts.TestCases, testCase) } + // individual benchmarks + for _, benchmark := range pkg.Benchmarks { + benchmarkCase := JUnitTestCase{ + Classname: classname, + Name: benchmark.Name, + Time: formatBenchmarkTime(benchmark.Duration), + Failure: nil, + } + + if benchmark.Bytes != 0 { + benchmarkCase.Bytes = strconv.Itoa(benchmark.Bytes) + } + if benchmark.Allocs != 0 { + benchmarkCase.Allocs = strconv.Itoa(benchmark.Allocs) + } + + ts.TestCases = append(ts.TestCases, benchmarkCase) + } + suites.Suites = append(suites.Suites, ts) } @@ -139,3 +171,7 @@ func JUnitReportXML(report *parser.Report, noXMLHeader bool, goVersion string, w func formatTime(d time.Duration) string { return fmt.Sprintf("%.3f", d.Seconds()) } + +func formatBenchmarkTime(d time.Duration) string { + return fmt.Sprintf("%.9f", d.Seconds()) +} diff --git a/go-junit-report_test.go b/go-junit-report_test.go index 7752810..5b32ca9 100644 --- a/go-junit-report_test.go +++ b/go-junit-report_test.go @@ -919,6 +919,246 @@ var testCases = []TestCase{ }, }, }, + { + name: "22-bench.txt", + reportName: "22-report.xml", + report: &parser.Report{ + Packages: []parser.Package{ + { + Name: "package/basic", + Duration: 3212 * time.Millisecond, + Time: 3212, + Benchmarks: []*parser.Benchmark{ + { + Name: "BenchmarkParse", + Duration: 604 * time.Nanosecond, + }, + { + Name: "BenchmarkReadingList", + Duration: 1425 * time.Nanosecond, + }, + }, + }, + }, + }, + }, + { + name: "23-benchmem.txt", + reportName: "23-report.xml", + report: &parser.Report{ + Packages: []parser.Package{ + { + Name: "package/one", + Duration: 9415 * time.Millisecond, + Time: 9415, + Benchmarks: []*parser.Benchmark{ + { + Name: "BenchmarkIpsHistoryInsert", + Duration: 52568 * time.Nanosecond, + Bytes: 24879, + Allocs: 494, + Output: []string{}, + }, + { + Name: "BenchmarkIpsHistoryLookup", + Duration: 15208 * time.Nanosecond, + Bytes: 7369, + Allocs: 143, + Output: []string{}, + }, + }, + }, + }, + }, + }, + { + name: "24-benchtests.txt", + reportName: "24-report.xml", + report: &parser.Report{ + Packages: []parser.Package{ + { + Name: "package3/baz", + Duration: 1382 * time.Millisecond, + Time: 1382, + Tests: []*parser.Test{ + { + Name: "TestNew", + Duration: 0, + Time: 0, + Result: parser.PASS, + Output: []string{}, + }, + { + Name: "TestNew/no", + Duration: 0, + Time: 0, + Result: parser.PASS, + Output: []string{}, + }, + { + Name: "TestNew/normal", + Duration: 0, + Time: 0, + Result: parser.PASS, + Output: []string{}, + }, + { + Name: "TestWriteThis", + Duration: 0, + Time: 0, + Result: parser.PASS, + Output: []string{}, + }, + }, + Benchmarks: []*parser.Benchmark{ + { + Name: "BenchmarkDeepMerge", + Duration: 2611 * time.Nanosecond, + Bytes: 1110, + Allocs: 16, + Output: []string{}, + }, + { + Name: "BenchmarkNext", + Duration: 100 * time.Nanosecond, + Bytes: 100, + Allocs: 1, + Output: []string{}, + }, + }, + }, + }, + }, + }, + { + name: "25-benchcount.txt", + reportName: "25-report.xml", + report: &parser.Report{ + Packages: []parser.Package{ + { + Name: "pkg/count", + Duration: 14211 * time.Millisecond, + Time: 14211, + Benchmarks: []*parser.Benchmark{ + { + Name: "BenchmarkNew", + Duration: 352 * time.Nanosecond, + Bytes: 80, + Allocs: 3, + Count: 5, + Output: []string{}, + }, + { + Name: "BenchmarkFew", + Duration: 102 * time.Nanosecond, + Bytes: 20, + Allocs: 1, + Count: 5, + Output: []string{}, + }, + }, + }, + }, + }, + }, + { + name: "26-testbenchmultiple.txt", + reportName: "26-report.xml", + report: &parser.Report{ + Packages: []parser.Package{ + { + Name: "multiple/repeating", + Duration: 14211 * time.Millisecond, + Time: 14211, + Tests: []*parser.Test{ + { + Name: "TestRepeat", + Duration: 0, + Time: 0, + Result: parser.PASS, + Output: []string{}, + }, + { + Name: "TestRepeat", + Duration: 0, + Time: 0, + Result: parser.PASS, + Output: []string{}, + }, + { + Name: "TestRepeat", + Duration: 0, + Time: 0, + Result: parser.PASS, + Output: []string{}, + }, + { + Name: "TestRepeat", + Duration: 0, + Time: 0, + Result: parser.PASS, + Output: []string{}, + }, + { + Name: "TestRepeat", + Duration: 0, + Time: 0, + Result: parser.PASS, + Output: []string{}, + }, + }, + Benchmarks: []*parser.Benchmark{ + { + Name: "BenchmarkNew", + Duration: 352 * time.Nanosecond, + Bytes: 80, + Allocs: 3, + Count: 5, + Output: []string{}, + }, + { + Name: "BenchmarkFew", + Duration: 102 * time.Nanosecond, + Bytes: 20, + Allocs: 1, + Count: 5, + Output: []string{}, + }, + }, + }, + }, + }, + }, + { + name: "27-benchdecimal.txt", + reportName: "27-report.xml", + report: &parser.Report{ + Packages: []parser.Package{ + { + Name: "really/small", + Duration: 4344 * time.Millisecond, + Time: 4344, + Benchmarks: []*parser.Benchmark{ + { + Name: "BenchmarkItsy", + Duration: 45 * time.Nanosecond, + Output: []string{}, + }, + { + Name: "BenchmarkTeeny", + Duration: 2 * time.Nanosecond, + Output: []string{}, + }, + { + Name: "BenchmarkWeeny", + Duration: 0 * time.Second, + Output: []string{}, + }, + }, + }, + }, + }, + }, } func TestParser(t *testing.T) { @@ -994,6 +1234,37 @@ func TestParser(t *testing.T) { t.Errorf("Test.Output (%s) ==\n%s\n, want\n%s", test.Name, testOutput, expTestOutput) } } + + if len(pkg.Benchmarks) != len(expPkg.Benchmarks) { + t.Fatalf("Package Benchmarks == %d, want %d", len(pkg.Benchmarks), len(expPkg.Benchmarks)) + } + + for j, benchmark := range pkg.Benchmarks { + expBenchmark := expPkg.Benchmarks[j] + + if benchmark.Name != expBenchmark.Name { + t.Errorf("Test.Name == %s, want %s", benchmark.Name, expBenchmark.Name) + } + + if benchmark.Duration != expBenchmark.Duration { + t.Errorf("benchmark.Duration == %s, want %s", benchmark.Duration, expBenchmark.Duration) + } + + if benchmark.Bytes != expBenchmark.Bytes { + t.Errorf("benchmark.Bytes == %d, want %d", benchmark.Bytes, expBenchmark.Bytes) + } + + if benchmark.Allocs != expBenchmark.Allocs { + t.Errorf("benchmark.Allocs == %d, want %d", benchmark.Allocs, expBenchmark.Allocs) + } + + benchmarkOutput := strings.Join(benchmark.Output, "\n") + expBenchmarkOutput := strings.Join(expBenchmark.Output, "\n") + if benchmarkOutput != expBenchmarkOutput { + t.Errorf("Benchmark.Output (%s) ==\n%s\n, want\n%s", benchmark.Name, benchmarkOutput, expBenchmarkOutput) + } + } + if pkg.CoveragePct != expPkg.CoveragePct { t.Errorf("Package.CoveragePct == %s, want %s", pkg.CoveragePct, expPkg.CoveragePct) } diff --git a/parser/parser.go b/parser/parser.go index d28b03b..45ceb72 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -4,6 +4,7 @@ import ( "bufio" "io" "regexp" + "strconv" "strings" "time" ) @@ -28,6 +29,7 @@ type Package struct { Name string Duration time.Duration Tests []*Test + Benchmarks []*Benchmark CoveragePct string // Time is deprecated, use Duration instead. @@ -45,12 +47,27 @@ type Test struct { Time int // in milliseconds } +// Benchmark contains the results of a single benchmark. +type Benchmark struct { + Name string + Duration time.Duration + // number of B/op + Bytes int + // number of allocs/op + Allocs int + // number of times this benchmark has been seen (for averaging). + Count int + Output []string +} + var ( regexStatus = regexp.MustCompile(`--- (PASS|FAIL|SKIP): (.+) \((\d+\.\d+)(?: seconds|s)\)`) regexCoverage = regexp.MustCompile(`^coverage:\s+(\d+\.\d+)%\s+of\s+statements(?:\sin\s.+)?$`) regexResult = regexp.MustCompile(`^(ok|FAIL)\s+([^ ]+)\s+(?:(\d+\.\d+)s|\(cached\)|(\[\w+ failed]))(?:\s+coverage:\s+(\d+\.\d+)%\sof\sstatements(?:\sin\s.+)?)?$`) - regexOutput = regexp.MustCompile(`( )*\t(.*)`) - regexSummary = regexp.MustCompile(`^(PASS|FAIL|SKIP)$`) + // regexBenchmark captures 3-5 groups: benchmark name, number of times ran, ns/op (with or without decimal), B/op (optional), and allocs/op (optional). + regexBenchmark = regexp.MustCompile(`^(Benchmark\w+)-\d\s+(\d+)\s+((\d+|\d+\.\d+)\sns/op)(\s+\d+\sB/op)?(\s+\d+\sallocs/op)?`) + regexOutput = regexp.MustCompile(`( )*\t(.*)`) + regexSummary = regexp.MustCompile(`^(PASS|FAIL|SKIP)$`) ) // Parse parses go test output from reader r and returns a report with the @@ -64,10 +81,13 @@ func Parse(r io.Reader, pkgName string) (*Report, error) { // keep track of tests we find var tests []*Test + // keep track of benchmarks we find + var benchmarks []*Benchmark + // sum of tests' time, use this if current test has no result line (when it is compiled test) var testsTime time.Duration - // current test + // current test or benchmark var cur string // keep track if we've already seen a summary for the current test @@ -108,6 +128,60 @@ func Parse(r io.Reader, pkgName string) (*Report, error) { // clear the current build package, so output lines won't be added to that build capturedPackage = "" seenSummary = false + } else if strings.HasPrefix(line, "Benchmark") { + // parse benchmarking info + matches := regexBenchmark.FindStringSubmatch(line) + if len(matches) < 1 { + continue + } + var name string + var duration time.Duration + var bytes int + var allocs int + + for _, field := range matches[1:] { + field = strings.TrimSpace(field) + if strings.HasPrefix(field, "Benchmark") { + name = field + } + if strings.HasSuffix(field, " ns/op") { + durString := strings.TrimSuffix(field, " ns/op") + duration = parseNanoseconds(durString) + } + if strings.HasSuffix(field, " B/op") { + b, _ := strconv.Atoi(strings.TrimSuffix(field, " B/op")) + bytes = b + } + if strings.HasSuffix(field, " allocs/op") { + a, _ := strconv.Atoi(strings.TrimSuffix(field, " allocs/op")) + allocs = a + } + } + + var duplicate bool + // check if duplicate benchmark + for _, bench := range benchmarks { + if bench.Name == name { + duplicate = true + bench.Count++ + bench.Duration += duration + bench.Bytes += bytes + bench.Allocs += allocs + } + } + + if len(benchmarks) < 1 || duplicate == false { + // the first time this benchmark has been seen + benchmarks = append(benchmarks, &Benchmark{ + Name: name, + Duration: duration, + Bytes: bytes, + Allocs: allocs, + Count: 1, + }, + ) + } + } else if strings.HasPrefix(line, "=== PAUSE ") { continue } else if strings.HasPrefix(line, "=== CONT ") { @@ -137,10 +211,21 @@ func Parse(r io.Reader, pkgName string) (*Report, error) { } // all tests in this package are finished + + for _, bench := range benchmarks { + if bench.Count > 1 { + bench.Allocs = bench.Allocs / bench.Count + bench.Bytes = bench.Bytes / bench.Count + newDuration := bench.Duration / time.Duration(bench.Count) + bench.Duration = newDuration + } + } + report.Packages = append(report.Packages, Package{ Name: matches[2], Duration: parseSeconds(matches[3]), Tests: tests, + Benchmarks: benchmarks, CoveragePct: coveragePct, Time: int(parseSeconds(matches[3]) / time.Millisecond), // deprecated @@ -206,6 +291,7 @@ func Parse(r io.Reader, pkgName string) (*Report, error) { Duration: testsTime, Time: int(testsTime / time.Millisecond), Tests: tests, + Benchmarks: benchmarks, CoveragePct: coveragePct, }) } @@ -222,6 +308,16 @@ func parseSeconds(t string) time.Duration { return d } +func parseNanoseconds(t string) time.Duration { + // note: if input < 1 ns precision, result will be 0s. + if t == "" { + return time.Duration(0) + } + // ignore error + d, _ := time.ParseDuration(t + "ns") + return d +} + func findTest(tests []*Test, name string) *Test { for i := len(tests) - 1; i >= 0; i-- { if tests[i].Name == name { diff --git a/testdata/22-bench.txt b/testdata/22-bench.txt new file mode 100644 index 0000000..06847e8 --- /dev/null +++ b/testdata/22-bench.txt @@ -0,0 +1,7 @@ +goos: darwin +goarch: amd64 +pkg: code.internal/state +BenchmarkParse-8 2000000 604 ns/op +BenchmarkReadingList-8 1000000 1425 ns/op +PASS +ok package/basic 3.212s diff --git a/testdata/22-report.xml b/testdata/22-report.xml new file mode 100644 index 0000000..5c74ad7 --- /dev/null +++ b/testdata/22-report.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/testdata/23-benchmem.txt b/testdata/23-benchmem.txt new file mode 100644 index 0000000..355dd71 --- /dev/null +++ b/testdata/23-benchmem.txt @@ -0,0 +1,7 @@ +goos: darwin +goarch: amd64 +pkg: code.internal/state +BenchmarkIpsHistoryInsert-8 30000 52568 ns/op 24879 B/op 494 allocs/op +BenchmarkIpsHistoryLookup-8 100000 15208 ns/op 7369 B/op 143 allocs/op +PASS +ok package/one 9.415s diff --git a/testdata/23-report.xml b/testdata/23-report.xml new file mode 100644 index 0000000..31a04b3 --- /dev/null +++ b/testdata/23-report.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/testdata/24-benchtests.txt b/testdata/24-benchtests.txt new file mode 100644 index 0000000..f20b669 --- /dev/null +++ b/testdata/24-benchtests.txt @@ -0,0 +1,15 @@ +=== RUN TestNew +=== RUN TestNew/no +=== RUN TestNew/normal +--- PASS: TestNew (0.00s) + --- PASS: TestNew/no (0.00s) + --- PASS: TestNew/normal (0.00s) +=== RUN TestWriteThis +--- PASS: TestWriteThis (0.00s) +goos: darwin +goarch: amd64 +pkg: package3/baz +BenchmarkDeepMerge-8 500000 2611 ns/op 1110 B/op 16 allocs/op +BenchmarkNext-8 500000 100 ns/op 100 B/op 1 allocs/op +PASS +ok package3/baz 1.382s \ No newline at end of file diff --git a/testdata/24-report.xml b/testdata/24-report.xml new file mode 100644 index 0000000..199fa1e --- /dev/null +++ b/testdata/24-report.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/testdata/25-benchcount.txt b/testdata/25-benchcount.txt new file mode 100644 index 0000000..82c3f75 --- /dev/null +++ b/testdata/25-benchcount.txt @@ -0,0 +1,12 @@ +BenchmarkNew-8 5000000 350 ns/op 80 B/op 3 allocs/op +BenchmarkNew-8 5000000 357 ns/op 80 B/op 3 allocs/op +BenchmarkNew-8 5000000 354 ns/op 80 B/op 3 allocs/op +BenchmarkNew-8 5000000 358 ns/op 80 B/op 3 allocs/op +BenchmarkNew-8 5000000 345 ns/op 80 B/op 3 allocs/op +BenchmarkFew-8 5000000 100 ns/op 20 B/op 1 allocs/op +BenchmarkFew-8 5000000 105 ns/op 20 B/op 1 allocs/op +BenchmarkFew-8 5000000 102 ns/op 20 B/op 1 allocs/op +BenchmarkFew-8 5000000 102 ns/op 20 B/op 1 allocs/op +BenchmarkFew-8 5000000 102 ns/op 20 B/op 1 allocs/op +PASS +ok pkg/count 14.211s \ No newline at end of file diff --git a/testdata/25-report.xml b/testdata/25-report.xml new file mode 100644 index 0000000..a8399ae --- /dev/null +++ b/testdata/25-report.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/testdata/26-report.xml b/testdata/26-report.xml new file mode 100644 index 0000000..293d2ab --- /dev/null +++ b/testdata/26-report.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/testdata/26-testbenchmultiple.txt b/testdata/26-testbenchmultiple.txt new file mode 100644 index 0000000..1657d2d --- /dev/null +++ b/testdata/26-testbenchmultiple.txt @@ -0,0 +1,23 @@ +=== RUN TestRepeat +--- PASS: TestRepeat (0.00s) +=== RUN TestRepeat +--- PASS: TestRepeat (0.00s) +=== RUN TestRepeat +--- PASS: TestRepeat (0.00s) +=== RUN TestRepeat +--- PASS: TestRepeat (0.00s) +=== RUN TestRepeat +--- PASS: TestRepeat (0.00s) +pkg: multiple/repeating +BenchmarkNew-8 5000000 350 ns/op 80 B/op 3 allocs/op +BenchmarkNew-8 5000000 357 ns/op 80 B/op 3 allocs/op +BenchmarkNew-8 5000000 354 ns/op 80 B/op 3 allocs/op +BenchmarkNew-8 5000000 358 ns/op 80 B/op 3 allocs/op +BenchmarkNew-8 5000000 345 ns/op 80 B/op 3 allocs/op +BenchmarkFew-8 5000000 100 ns/op 20 B/op 1 allocs/op +BenchmarkFew-8 5000000 105 ns/op 20 B/op 1 allocs/op +BenchmarkFew-8 5000000 102 ns/op 20 B/op 1 allocs/op +BenchmarkFew-8 5000000 102 ns/op 20 B/op 1 allocs/op +BenchmarkFew-8 5000000 102 ns/op 20 B/op 1 allocs/op +PASS +ok multiple/repeating 14.211s \ No newline at end of file diff --git a/testdata/27-benchdecimal.txt b/testdata/27-benchdecimal.txt new file mode 100644 index 0000000..4728bdc --- /dev/null +++ b/testdata/27-benchdecimal.txt @@ -0,0 +1,8 @@ +goos: darwin +goarch: amd64 +pkg: really/small +BenchmarkItsy-8 30000000 45.7 ns/op +BenchmarkTeeny-8 1000000000 2.12 ns/op +BenchmarkWeeny-8 2000000000 0.26 ns/op +PASS +ok really/small 4.344s diff --git a/testdata/27-report.xml b/testdata/27-report.xml new file mode 100644 index 0000000..42ecd3f --- /dev/null +++ b/testdata/27-report.xml @@ -0,0 +1,11 @@ + + + + + + + + + + +