Revision control
++ // Copyright (C) 2016 Robin Burchell <robin.burchell@viroteck.net>
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++
++ package main
++
++ import (
++ "archive/zip"
++ "encoding/xml"
++ "flag"
++ "fmt"
++ "io"
++ "io/ioutil"
++ "log"
++ "os"
++ "sort"
++ "strconv"
++
++ "code.qt.io/qt/qtqa.git/src/goqtestlib"
++ "github.com/olekukonko/tablewriter"
++ )
++
++ func loadTestResult(path string) *goqtestlib.ParsedTestResult {
++ tr := goqtestlib.TestResult{PathToResultsXML: path}
++ r, err := tr.Parse()
++ if err != nil {
++ log.Fatalf("Can't open result at path %s: %s", path, err)
++ }
++
++ return r
++ }
++
++ type MergedTestResult struct {
++ Name string
++ OldDuration *float64
++ NewDuration *float64
++ OldInstructionReads *float64
++ NewInstructionReads *float64
++ }
++
++ type ByName []MergedTestResult
++
++ func (s ByName) Len() int { return len(s) }
++ func (s ByName) Swap(i int, j int) {
++ s[i], s[j] = s[j], s[i]
++ }
++ func (s ByName) Less(i int, j int) bool {
++ return s[i].Name < s[j].Name
++ }
++
++ func describeChange(pChange float64) string {
++ pStr := strconv.FormatFloat(pChange, 'f', 2, 64)
++ if pChange > 0 {
++ return fmt.Sprintf("+%s%%", pStr)
++ } else if pChange < 0 {
++ return fmt.Sprintf("%s%% FASTER! :)", pStr)
++ } else {
++ return "more or less the same"
++ }
++ }
++
++ type MergedTestResults map[string]MergedTestResult
++
++ func (results *MergedTestResults) addOldTestCase(prefix string, testCase *goqtestlib.ParsedTestResult) {
++ // XXX: add a way to specify what type of benchmarkresult to look for.
++ for _, fn := range testCase.Functions {
++ qualifiedName := prefix + fn.Name
++ for _, br := range fn.BenchmarkResults {
++ nameWithTag := qualifiedName
++ if br.Tag != "" {
++ nameWithTag += ":" + br.Tag
++ }
++ res := (*results)[nameWithTag]
++ res.Name = nameWithTag
++ val := br.Value
++ if br.Metric == "WalltimeMilliseconds" {
++ res.OldDuration = &val
++ } else if br.Metric == "InstructionReads" {
++ res.OldInstructionReads = &val
++ }
++ (*results)[nameWithTag] = res
++ }
++ }
++ }
++
++ func (results *MergedTestResults) addNewTestCase(prefix string, testCase *goqtestlib.ParsedTestResult) {
++ for _, fn := range testCase.Functions {
++ qualifiedName := prefix + fn.Name
++ for _, br := range fn.BenchmarkResults {
++ nameWithTag := qualifiedName
++ if br.Tag != "" {
++ nameWithTag += ":" + br.Tag
++ }
++ res := (*results)[nameWithTag]
++ res.Name = nameWithTag
++ val := br.Value
++ if br.Metric == "WalltimeMilliseconds" {
++ res.NewDuration = &val
++ } else if br.Metric == "InstructionReads" {
++ res.NewInstructionReads = &val
++ }
++ (*results)[nameWithTag] = res
++ }
++ }
++ }
++
++ func (results *MergedTestResults) compare(output io.Writer) {
++ // convert mergedResults to a slice, and sort it for stable results.
++ sortedResults := []MergedTestResult{}
++
++ for _, mr := range *results {
++ sortedResults = append(sortedResults, mr)
++ }
++
++ sort.Sort(ByName(sortedResults))
++
++ table := tablewriter.NewWriter(output)
++ table.SetAutoFormatHeaders(false)
++ table.SetHeader([]string{"Test", "From", "To", "Details"})
++ table.SetBorder(false)
++
++ totalPChange := 0.0
++
++ for _, mr := range sortedResults {
++ row := []string{}
++ row = append(row, mr.Name)
++
++ if mr.OldInstructionReads != nil && mr.NewInstructionReads != nil {
++ pChange := (*mr.NewInstructionReads - *mr.OldInstructionReads) / *mr.OldInstructionReads * 100
++ totalPChange += pChange
++ row = append(row, strconv.FormatFloat(*mr.OldInstructionReads, 'f', 2, 64)+" instr")
++ row = append(row, strconv.FormatFloat(*mr.NewInstructionReads, 'f', 2, 64)+" instr")
++ row = append(row, describeChange(pChange))
++
++ } else if mr.OldDuration != nil && mr.NewDuration != nil {
++ pChange := (*mr.NewDuration - *mr.OldDuration) / *mr.OldDuration * 100
++ totalPChange += pChange
++ row = append(row, strconv.FormatFloat(*mr.OldDuration, 'f', 2, 64)+" ms")
++ row = append(row, strconv.FormatFloat(*mr.NewDuration, 'f', 2, 64)+" ms")
++ row = append(row, describeChange(pChange))
++ } else {
++ // the comparison can't be made because either the data types are
++ // differing between the two runs, or we're missing a test in one
++ // run.
++ //
++ // try find something to display for old and new. fall back to "-"
++ // if we can't.
++ ostr := "-"
++ nstr := "-"
++
++ if mr.OldDuration != nil {
++ ostr = strconv.FormatFloat(*mr.OldDuration, 'f', 2, 64) + " ms"
++ } else if mr.OldInstructionReads != nil {
++ ostr = strconv.FormatFloat(*mr.OldInstructionReads, 'f', 2, 64) + " instr"
++ }
++
++ if mr.NewDuration != nil {
++ nstr = strconv.FormatFloat(*mr.NewDuration, 'f', 2, 64) + " ms"
++ } else if mr.NewInstructionReads != nil {
++ nstr = strconv.FormatFloat(*mr.NewInstructionReads, 'f', 2, 64) + " instr"
++ }
++
++ row = append(row, ostr)
++ row = append(row, nstr)
++ row = append(row, "-")
++ }
++
++ table.Append(row)
++ }
++
++ verdict := ""
++ totalPStr := strconv.FormatFloat(totalPChange, 'f', 2, 64)
++ if totalPChange > 0 {
++ verdict = fmt.Sprintf("+%s%% :(", totalPStr)
++ } else if totalPChange < 0 {
++ verdict = fmt.Sprintf("%s%% :)", totalPStr)
++ } else {
++ verdict = "more or less the same"
++ }
++
++ table.SetFooter([]string{"Overall result", "", "", verdict})
++ table.Render()
++
++ }
++
++ func compareSingleTestRuns(oxml string, nxml string) {
++ oldTest := loadTestResult(oxml)
++ newTest := loadTestResult(nxml)
++
++ if oldTest.Name != newTest.Name {
++ log.Fatalf("I can't compare two totally different things (old: %s, new: %s)", oldTest.Name, newTest.Name)
++ return
++ }
++
++ // merge the test functions into a singular representation.
++ mergedResults := MergedTestResults{}
++ prefix := ""
++ mergedResults.addOldTestCase(prefix, oldTest)
++ mergedResults.addNewTestCase(prefix, newTest)
++
++ mergedResults.compare(os.Stdout)
++ }
++
++ func unmarshalTestResult(reader io.ReadCloser) (*goqtestlib.ParsedTestResult, error) {
++ defer reader.Close()
++ xmlContents, err := ioutil.ReadAll(reader)
++ if err != nil {
++ return nil, fmt.Errorf("Error reading results xml file: %s", err)
++ }
++
++ testCase := &goqtestlib.ParsedTestResult{}
++ if err = xml.Unmarshal(xmlContents, &testCase); err != nil {
++ return nil, fmt.Errorf("Error unmarshalling testlib xml output: %s", err)
++ }
++ return testCase, nil
++ }
++
++ type testArchive struct {
++ reader *zip.ReadCloser
++ }
++
++ func openTestArchive(path string) (*testArchive, error) {
++ reader, err := zip.OpenReader(path)
++ return &testArchive{reader}, err
++ }
++
++ func (archive *testArchive) forEachTestCase(callback func(path string, testCase *goqtestlib.ParsedTestResult) error) error {
++ for _, f := range archive.reader.File {
++ reader, err := f.Open()
++ if err != nil {
++ log.Fatalf("Error opening entry in zip archive %s: %s", f.Name, err)
++ }
++ result, err := unmarshalTestResult(reader)
++ if err != nil {
++ log.Fatalf("Error unpacking test result for %s from zip archive: %s", f.Name, err)
++ }
++ if err := callback(f.Name, result); err != nil {
++ return err
++ }
++ }
++ return nil
++ }
++
++ func (archive *testArchive) Close() error {
++ return archive.reader.Close()
++ }
++
++ func compareArchivedTestRuns(oarch string, narch string) {
++ fmt.Printf("Comparing zipped runs %s vs %s\n", oarch, narch)
++
++ or, err := openTestArchive(oarch)
++ if err != nil {
++ log.Fatalf("Can't open old archive: %s\n", err)
++ }
++ defer or.Close()
++
++ nr, err := openTestArchive(narch)
++ if err != nil {
++ log.Fatalf("Can't open new archive: %s\n", err)
++ }
++ defer nr.Close()
++
++ // this is a map of xml to result, so e.g:
++ // qml/binding.xml -> result.
++ mergedResults := MergedTestResults{}
++
++ or.forEachTestCase(func(path string, testCase *goqtestlib.ParsedTestResult) error {
++ mergedResults.addOldTestCase(path+"/", testCase)
++ return nil
++ })
++ nr.forEachTestCase(func(path string, testCase *goqtestlib.ParsedTestResult) error {
++ mergedResults.addNewTestCase(path+"/", testCase)
++ return nil
++ })
++
++ mergedResults.compare(os.Stdout)
++ }
++
++ func main() {
++ var nf = flag.String("new", "", "the changed XML result to compare against")
++ var of = flag.String("old", "", "the baseline XML result to compare against")
++
++ var na = flag.String("newarchive", "", "the changed archived results to compare against")
++ var oa = flag.String("oldarchive", "", "the baseline archived results to compare against")
++ flag.Parse()
++
++ nxml := *nf
++ oxml := *of
++
++ narch := *na
++ oarch := *oa
++
++ hasNewFile := len(nxml) > 0
++ hasOldFile := len(oxml) > 0
++
++ hasNewArch := len(narch) > 0
++ hasOldArch := len(oarch) > 0
++
++ if (!hasNewFile || !hasOldFile) && (!hasNewArch || !hasOldArch) {
++ log.Fatalf("You need to provide either -new & -old, or -newarchive & -oldarchive.")
++ return
++ }
++
++ if hasNewFile && hasOldFile {
++ compareSingleTestRuns(oxml, nxml)
++ } else {
++ compareArchivedTestRuns(oarch, narch)
++ }
++ }