From 753dc64dff899db0032e67524e302bae050474fa Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Wed, 9 Jun 2021 20:53:43 +0900 Subject: [PATCH] Add benchmark data --- internal/cmd/benchgraph/go.mod | 1 + internal/cmd/benchgraph/go.sum | 1 + internal/cmd/benchgraph/main.go | 311 +++++++++++++++++++++++++++++--- 3 files changed, 289 insertions(+), 24 deletions(-) diff --git a/internal/cmd/benchgraph/go.mod b/internal/cmd/benchgraph/go.mod index d689bf3..01fbc7f 100644 --- a/internal/cmd/benchgraph/go.mod +++ b/internal/cmd/benchgraph/go.mod @@ -7,6 +7,7 @@ replace github.com/goccy/go-json => ../../../ require ( github.com/goccy/go-json v0.0.0-00010101000000-000000000000 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c + golang.org/x/tools v0.1.1 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 google.golang.org/api v0.47.0 ) diff --git a/internal/cmd/benchgraph/go.sum b/internal/cmd/benchgraph/go.sum index 4d8cbbe..26fe22e 100644 --- a/internal/cmd/benchgraph/go.sum +++ b/internal/cmd/benchgraph/go.sum @@ -351,6 +351,7 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/cmd/benchgraph/main.go b/internal/cmd/benchgraph/main.go index 54323eb..fb6a5e9 100644 --- a/internal/cmd/benchgraph/main.go +++ b/internal/cmd/benchgraph/main.go @@ -9,10 +9,12 @@ import ( "log" "net/http" "os" + "strings" "github.com/goccy/go-json" "golang.org/x/oauth2" "golang.org/x/oauth2/google" + "golang.org/x/tools/benchmark/parse" "golang.org/x/xerrors" "google.golang.org/api/option" "google.golang.org/api/sheets/v4" @@ -23,6 +25,81 @@ const ( benchgraphToken = "benchgraph-token.json" ) +type BenchmarkCodec string + +const ( + UnknownCodec BenchmarkCodec = "Unknown" + Encoder BenchmarkCodec = "Encode" + Decoder BenchmarkCodec = "Decode" +) + +func (c BenchmarkCodec) String() string { + return string(c) +} + +type BenchmarkKind string + +const ( + UnknownKind BenchmarkKind = "Unknown" + SmallStruct BenchmarkKind = "SmallStruct" + SmallStructCached BenchmarkKind = "SmallStructCached" + MediumStruct BenchmarkKind = "MediumStruct" + MediumStructCached BenchmarkKind = "MediumStructCached" + LargeStruct BenchmarkKind = "LargeStruct" + LargeStructCached BenchmarkKind = "LargeStructCached" +) + +func (k BenchmarkKind) String() string { + return string(k) +} + +type BenchmarkTarget string + +const ( + UnknownTarget BenchmarkTarget = "Unknown" + EncodingJson BenchmarkTarget = "EncodingJson" + GoJson BenchmarkTarget = "GoJson" + GoJsonNoEscape BenchmarkTarget = "GoJsonNoEscape" + GoJsonColored BenchmarkTarget = "GoJsonColored" + FFJson BenchmarkTarget = "FFJson" + JsonIter BenchmarkTarget = "JsonIter" + EasyJson BenchmarkTarget = "EasyJson" + Jettison BenchmarkTarget = "Jettison" + GoJay BenchmarkTarget = "GoJay" + SegmentioJson BenchmarkTarget = "SegmentioJson" +) + +func (t BenchmarkTarget) String() string { + return string(t) +} + +func (t BenchmarkTarget) DisplayName() string { + switch t { + case EncodingJson: + return "encoding/json" + case GoJson: + return "goccy/go-json" + case GoJsonNoEscape: + return "goccy/go-json (noescape)" + case GoJsonColored: + return "goccy/go-json (colored)" + case FFJson: + return "pquerna/ffjson" + case JsonIter: + return "json-iterator/go" + case EasyJson: + return "mailru/easyjson" + case Jettison: + return "wl2L/jettison" + case GoJay: + return "francoispqt/gojay" + case SegmentioJson: + return "segmentio/encoding/json" + default: + return "" + } +} + var credFile string func init() { @@ -45,7 +122,7 @@ func createClient(ctx context.Context) (*http.Client, error) { } // If modifying these scopes, delete your previously saved token.json. - config, err := google.ConfigFromJSON(b, sheets.SpreadsheetsScope, slides.PresentationsScope) //"https://spreadsheets.google.com/feeds" + config, err := google.ConfigFromJSON(b, sheets.SpreadsheetsScope, slides.PresentationsScope) if err != nil { return nil, xerrors.Errorf("failed to create config from scope: %w", err) } @@ -158,7 +235,7 @@ func createSpreadsheet(svc *sheets.Service, title string, headers []string, targ return spreadSheet, nil } -func generateChart(svc *sheets.Service, spreadSheet *sheets.Spreadsheet, title string) (int64, error) { +func generateChart(svc *sheets.Service, spreadSheet *sheets.Spreadsheet, title string) (*sheets.EmbeddedChart, error) { sheet := spreadSheet.Sheets[0] rows := sheet.Data[0].RowData rowSize := int64(len(rows)) @@ -194,7 +271,7 @@ func generateChart(svc *sheets.Service, spreadSheet *sheets.Spreadsheet, title s Title: title, BasicChart: &sheets.BasicChartSpec{ ChartType: "BAR", - LegendPosition: "RIGHT_LEGEND", + LegendPosition: "TOP_LEGEND", Domains: []*sheets.BasicChartDomain{ { Domain: &sheets.ChartData{ @@ -222,16 +299,16 @@ func generateChart(svc *sheets.Service, spreadSheet *sheets.Spreadsheet, title s }, }).Do() if err != nil { - return 0, xerrors.Errorf("failed to generate chart: %w", err) + return nil, xerrors.Errorf("failed to generate chart: %w", err) } for _, rep := range res.Replies { if rep.AddChart == nil { continue } - return rep.AddChart.Chart.ChartId, nil + return rep.AddChart.Chart, nil } - return 0, xerrors.Errorf("failed to find chartID") + return nil, xerrors.Errorf("failed to find chartID") } func createPresentationWithEmptySlide(presentationService *slides.PresentationsService) (*slides.Presentation, string, error) { @@ -260,19 +337,19 @@ func createPresentationWithEmptySlide(presentationService *slides.PresentationsS return nil, "", xerrors.Errorf("failed to find slide objectID") } -func addChartToPresentation(presentationService *slides.PresentationsService, presentation *slides.Presentation, slideID string, spreadSheetID string, chartID int64) error { +func addChartToPresentation(presentationService *slides.PresentationsService, presentation *slides.Presentation, slideID string, spreadSheetID string, chart *sheets.EmbeddedChart) error { if _, err := presentationService.BatchUpdate(presentation.PresentationId, &slides.BatchUpdatePresentationRequest{ Requests: []*slides.Request{ { CreateSheetsChart: &slides.CreateSheetsChartRequest{ LinkingMode: "LINKED", SpreadsheetId: spreadSheetID, - ChartId: chartID, + ChartId: chart.ChartId, ElementProperties: &slides.PageElementProperties{ PageObjectId: slideID, Size: &slides.Size{ Width: &slides.Dimension{ - Magnitude: 1024, + Magnitude: 1200, Unit: "PT", }, Height: &slides.Dimension{ @@ -325,32 +402,164 @@ func downloadImage(url, path string) error { return nil } -func _main(args []string) error { - ctx := context.Background() - client, err := createClient(ctx) +type BenchmarkData struct { + *parse.Benchmark + Codec BenchmarkCodec + Kind BenchmarkKind + Target BenchmarkTarget +} + +func toBenchmarkCodec(name string) (BenchmarkCodec, error) { + switch { + case strings.Contains(name, "_Encode_"): + return Encoder, nil + case strings.Contains(name, "_Decoder_"): + return Decoder, nil + default: + return UnknownCodec, xerrors.Errorf("cannot find codec from %s", name) + } +} + +func toBenchmarkKind(name string) (BenchmarkKind, error) { + switch { + case strings.Contains(name, "_SmallStruct_"): + return SmallStruct, nil + case strings.Contains(name, "_MediumStruct_"): + return MediumStruct, nil + case strings.Contains(name, "_LargeStruct_"): + return LargeStruct, nil + case strings.Contains(name, "_SmallStructCached_"): + return SmallStructCached, nil + case strings.Contains(name, "_MediumStructCached_"): + return MediumStructCached, nil + case strings.Contains(name, "_LargeStructCached_"): + return LargeStructCached, nil + default: + return UnknownKind, xerrors.Errorf("cannot find kind from %s", name) + } +} + +func toBenchmarkTarget(name string) (BenchmarkTarget, error) { + v := strings.ToLower(name) + switch { + case strings.Contains(v, strings.ToLower("EncodingJson")): + return EncodingJson, nil + case strings.Contains(v, strings.ToLower("GoJson")): + switch { + case strings.Contains(v, strings.ToLower("NoEscape")): + return GoJsonNoEscape, nil + case strings.Contains(v, strings.ToLower("Colored")): + return GoJsonColored, nil + } + return GoJson, nil + case strings.Contains(v, strings.ToLower("FFJson")): + return FFJson, nil + case strings.Contains(v, strings.ToLower("JsonIter")): + return JsonIter, nil + case strings.Contains(v, strings.ToLower("EasyJson")): + return EasyJson, nil + case strings.Contains(v, strings.ToLower("Jettison")): + return Jettison, nil + case strings.Contains(v, strings.ToLower("GoJay")): + return GoJay, nil + case strings.Contains(v, strings.ToLower("SegmentioJson")): + return SegmentioJson, nil + default: + return UnknownTarget, xerrors.Errorf("cannot find target from %s", name) + } +} + +func createBenchmarkData(bench *parse.Benchmark) (*BenchmarkData, error) { + codec, err := toBenchmarkCodec(bench.Name) if err != nil { - return xerrors.Errorf("failed to create client: %w", err) + return nil, xerrors.Errorf("failed to convert benchmark codec: %w", err) + } + kind, err := toBenchmarkKind(bench.Name) + if err != nil { + return nil, xerrors.Errorf("failed to convert benchmark kind: %w", err) + } + target, err := toBenchmarkTarget(bench.Name) + if err != nil { + return nil, xerrors.Errorf("failed to convert benchmark target: %w", err) + } + return &BenchmarkData{ + Benchmark: bench, + Codec: codec, + Kind: kind, + Target: target, + }, nil +} + +func createAllBenchmarkData(data string) ([]*BenchmarkData, error) { + allBenchData := []*BenchmarkData{} + for _, line := range strings.Split(data, "\n") { + if !strings.HasPrefix(line, "Benchmark") { + continue + } + bench, err := parse.ParseLine(line) + if err != nil { + return nil, xerrors.Errorf("failed to parse line: %w", err) + } + benchData, err := createBenchmarkData(bench) + if err != nil { + return nil, xerrors.Errorf("failed to create benchmark data: %w", err) + } + allBenchData = append(allBenchData, benchData) + } + return allBenchData, nil +} + +type Graph struct { + Title string + Codec BenchmarkCodec + Kinds []BenchmarkKind +} + +func (g *Graph) existsKind(target BenchmarkKind) bool { + for _, kind := range g.Kinds { + if kind == target { + return true + } + } + return false +} + +func generateBenchmarkGraph(ctx context.Context, client *http.Client, g *Graph, data []*BenchmarkData) error { + headers := []string{} + for _, kind := range g.Kinds { + headers = append(headers, kind.String()) + } + targetMap := map[string][]*BenchmarkData{} + targetToData := map[string][]float64{} + for _, v := range data { + if g.Codec != v.Codec { + continue + } + if !g.existsKind(v.Kind) { + continue + } + name := v.Target.DisplayName() + targetMap[name] = append(targetMap[name], v) + targetToData[name] = append(targetToData[name], v.NsPerOp) + } + targets := []string{} + for k := range targetMap { + targets = append(targets, k) } sheetService, err := sheets.NewService(ctx, option.WithHTTPClient(client)) if err != nil { return xerrors.Errorf("failed to create service for sheet: %w", err) } - headers := []string{"TypeA", "TypeB", "TypeC"} - targets := []string{"targetA", "targetB"} - data := map[string][]float64{ - targets[0]: []float64{10, 100, 1000}, - targets[1]: []float64{20, 200, 2000}, - } - spreadSheet, err := createSpreadsheet(sheetService, "go-json benchmark results", headers, targets, data) + spreadSheet, err := createSpreadsheet(sheetService, g.Title, headers, targets, targetToData) if err != nil { return xerrors.Errorf("failed to create spreadsheet: %w", err) } - chartID, err := generateChart(sheetService, spreadSheet, "Benchmark Result") + chart, err := generateChart(sheetService, spreadSheet, g.Title) if err != nil { return xerrors.Errorf("failed to generate chart: %w", err) } - log.Println("spreadSheetID = ", spreadSheet.SpreadsheetId, "chartID = ", chartID) + log.Println("spreadSheetID = ", spreadSheet.SpreadsheetId, "chartID = ", chart.ChartId) slideService, err := slides.NewService(ctx, option.WithHTTPClient(client)) if err != nil { @@ -361,7 +570,7 @@ func _main(args []string) error { if err != nil { return xerrors.Errorf("failed to create presentation with slide: %w", err) } - if err := addChartToPresentation(presentationService, presentation, slideID, spreadSheet.SpreadsheetId, chartID); err != nil { + if err := addChartToPresentation(presentationService, presentation, slideID, spreadSheet.SpreadsheetId, chart); err != nil { return xerrors.Errorf("failed to add chart to presentation: %w", err) } if err := downloadChartImage(presentationService, presentation, "bench.png"); err != nil { @@ -370,8 +579,62 @@ func _main(args []string) error { return nil } +func run(args []string) error { + benchData, err := createAllBenchmarkData(` +goos: darwin +goarch: amd64 +pkg: benchmark +Benchmark_Encode_SmallStruct_EncodingJson-16 2135164 555 ns/op 256 B/op 2 allocs/op +Benchmark_Encode_SmallStruct_FFJson-16 1348426 935 ns/op 584 B/op 9 allocs/op +Benchmark_Encode_SmallStruct_JsonIter-16 1970002 598 ns/op 264 B/op 3 allocs/op +Benchmark_Encode_SmallStruct_EasyJson-16 2202872 547 ns/op 720 B/op 4 allocs/op +Benchmark_Encode_SmallStruct_Jettison-16 2610375 461 ns/op 256 B/op 2 allocs/op +Benchmark_Encode_SmallStruct_GoJay-16 2763138 428 ns/op 624 B/op 2 allocs/op +Benchmark_Encode_SmallStruct_SegmentioJson-16 4124536 291 ns/op 256 B/op 2 allocs/op +Benchmark_Encode_SmallStruct_GoJsonColored-16 2530636 475 ns/op 432 B/op 2 allocs/op +Benchmark_Encode_SmallStruct_GoJson-16 4308301 282 ns/op 256 B/op 2 allocs/op +Benchmark_Encode_SmallStruct_GoJsonNoEscape-16 5406490 215 ns/op 144 B/op 1 allocs/op +Benchmark_Encode_SmallStructCached_EncodingJson-16 2386401 510 ns/op 144 B/op 1 allocs/op +Benchmark_Encode_SmallStructCached_FFJson-16 1450132 829 ns/op 472 B/op 8 allocs/op +Benchmark_Encode_SmallStructCached_JsonIter-16 2279529 526 ns/op 152 B/op 2 allocs/op +Benchmark_Encode_SmallStructCached_EasyJson-16 2225763 543 ns/op 720 B/op 4 allocs/op +Benchmark_Encode_SmallStructCached_Jettison-16 3059923 387 ns/op 144 B/op 1 allocs/op +Benchmark_Encode_SmallStructCached_GoJay-16 3187108 372 ns/op 512 B/op 1 allocs/op +Benchmark_Encode_SmallStructCached_SegmentioJson-16 5128329 229 ns/op 144 B/op 1 allocs/op +Benchmark_Encode_SmallStructCached_GoJsonColored-16 3028646 403 ns/op 320 B/op 1 allocs/op +Benchmark_Encode_SmallStructCached_GoJson-16 5458942 215 ns/op 144 B/op 1 allocs/op +Benchmark_Encode_SmallStructCached_GoJsonNoEscape-16 5725311 210 ns/op 144 B/op 1 allocs/op +PASS +ok benchmark 33.928s +`) + if err != nil { + return xerrors.Errorf("failed to parse benchmark data: %w", err) + } + if benchData == nil { + return nil + } + ctx := context.Background() + client, err := createClient(ctx) + if err != nil { + return xerrors.Errorf("failed to create client: %w", err) + } + graphs := []*Graph{ + { + Title: "Encoding Benchmark ( Small / Medium Struct )", + Codec: Encoder, + Kinds: []BenchmarkKind{SmallStruct, MediumStruct}, + }, + } + for _, graph := range graphs { + if err := generateBenchmarkGraph(ctx, client, graph, benchData); err != nil { + return xerrors.Errorf("failed to generate benchmark graph: %w", err) + } + } + return nil +} + func main() { - if err := _main(os.Args); err != nil { + if err := run(os.Args); err != nil { log.Fatalf("%+v", err) } }