[gold] Remove unused executables
Change-Id: Ic404de8f5110e82dcf531894c710e0c37342dc40
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/302657
Reviewed-by: Leandro Lovisolo <lovisolo@google.com>
Commit-Queue: Kevin Lubick <kjlubick@google.com>
diff --git a/golden/cmd/gen_whitelist/main.go b/golden/cmd/gen_whitelist/main.go
deleted file mode 100644
index 08d067f..0000000
--- a/golden/cmd/gen_whitelist/main.go
+++ /dev/null
@@ -1,61 +0,0 @@
-package main
-
-import (
- "encoding/json"
- "flag"
- "os"
- "strings"
-
- "go.skia.org/infra/go/common"
- "go.skia.org/infra/go/paramtools"
- "go.skia.org/infra/go/sklog"
- "go.skia.org/infra/go/util"
-)
-
-var (
- jobsFile = flag.String("jobs_file", "", "File that contains the job definitions.")
- outputFile = flag.String("out_file", "", "Output file where to write the expectations.")
-)
-
-func main() {
- common.Init()
-
- if (*jobsFile == "") || (*outputFile == "") {
- sklog.Fatalf("Jobs file and output file must be provided.")
- }
-
- // Load the jobs file and parse it.
- jobs := []string{}
- f, err := os.Open(*jobsFile)
- if err != nil {
- sklog.Fatalf("Unable to open jobs file: %s", err)
- }
- defer f.Close()
-
- if err := json.NewDecoder(f).Decode(&jobs); err != nil {
- sklog.Fatalf("Error parsing jobs file: %s", err)
- }
-
- // Split the builder names.
- models := util.StringSet{}
- for _, job := range jobs {
- if strings.HasPrefix(job, "Test") {
- parts := strings.Split(job, "-")
- if len(parts) > 3 {
- sklog.Infof("entry: %s", job)
- models[parts[3]] = true
- }
- }
- }
-
- // Write the output file.
- output := paramtools.ParamSet{"model": models.Keys()}
- if f, err = os.Create(*outputFile); err != nil {
- sklog.Fatalf("Error creating output file: %s", err)
- }
- defer f.Close()
-
- if err := json.NewEncoder(f).Encode(output); err != nil {
- sklog.Fatalf("Error encoding output file: %s", err)
- }
-}
diff --git a/golden/cmd/imagediff/main.go b/golden/cmd/imagediff/main.go
deleted file mode 100644
index 1537fca..0000000
--- a/golden/cmd/imagediff/main.go
+++ /dev/null
@@ -1,69 +0,0 @@
-// Simple command line app the applies our image diff library to two PNGs.
-package main
-
-import (
- "flag"
- "fmt"
- "image"
- "image/png"
- "io"
- "log"
- "os"
-
- "go.skia.org/infra/go/common"
- "go.skia.org/infra/go/skerr"
- "go.skia.org/infra/go/util"
- "go.skia.org/infra/golden/go/diff"
-)
-
-var (
- out = flag.String("out", "", "Filename to write the diff image to.")
-)
-
-func main() {
- common.Init()
- if flag.NArg() != 2 {
- log.Fatal("Usage: imagediff [--out filename] imagepath1.png imagepath2.png\n")
- }
- a, err := openNRGBAFromFile(flag.Arg(0))
- if err != nil {
- log.Fatal(err)
- }
- b, err := openNRGBAFromFile(flag.Arg(1))
- if err != nil {
- log.Fatal(err)
- }
- metrics, d := diff.PixelDiff(a, b)
- fmt.Printf("Dimensions are different: %v\n", metrics.DimDiffer)
- fmt.Printf("Number of pixels different: %v\n", metrics.NumDiffPixels)
- fmt.Printf("Pixel diff percent: %v\n", metrics.PixelDiffPercent)
- fmt.Printf("Max RGBA: %v\n", metrics.MaxRGBADiffs)
- if *out == "" {
- return
- } else {
- fmt.Println("Writing image diff.")
- }
- f, err := os.Create(*out)
- if err != nil {
- log.Fatal(err)
- }
- if err := png.Encode(f, d); err != nil {
- log.Fatal(err)
- }
-}
-
-func openNRGBAFromFile(fileName string) (*image.NRGBA, error) {
- var img *image.NRGBA
- err := util.WithReadFile(fileName, func(r io.Reader) error {
- im, err := png.Decode(r)
- if err != nil {
- return err
- }
- img = diff.GetNRGBA(im)
- return nil
- })
- if err != nil {
- return nil, skerr.Wrap(err)
- }
- return img, nil
-}
diff --git a/golden/cmd/logextractor/main.go b/golden/cmd/logextractor/main.go
deleted file mode 100644
index 0ab9737..0000000
--- a/golden/cmd/logextractor/main.go
+++ /dev/null
@@ -1,104 +0,0 @@
-// Simple command line tool that parses logs extracted from cloud-logging, extracts
-// the query strings and prints them to stdout.
-package main
-
-import (
- "bufio"
- "encoding/json"
- "flag"
- "fmt"
- "os"
- "path/filepath"
- "strings"
-
- "go.skia.org/infra/go/common"
- "go.skia.org/infra/go/fileutil"
- "go.skia.org/infra/go/sklog"
- "go.skia.org/infra/go/util"
-)
-
-const (
- // RAW_QUERY_STR contains the string that indicates the beginning of the search
- // query logged by skiacorrectness.
- RAW_QUERY_STR = "RawQuery:\""
-)
-
-var (
- logInputDir = flag.String("input_dir", "", "Directory with the input files. Can be nested.")
-)
-
-func main() {
- common.Init()
-
- if (*logInputDir == "") || (!fileutil.FileExists(*logInputDir)) {
- sklog.Error("Flag 'input_dir' must be set and the directory must exist.")
- os.Exit(1)
- }
-
- var result []string
- err := filepath.Walk(*logInputDir, func(path string, info os.FileInfo, err error) error {
- if err != nil {
- return err
- }
- queries, err := extractQueries(path)
- if err != nil {
- return err
- }
-
- result = append(result, queries...)
- return nil
- })
- if err != nil {
- sklog.Fatalf("Err: %s", err)
- }
-
- for _, q := range result {
- fmt.Println(q)
- }
-}
-
-type LogEntry struct {
- TextPayload string
-}
-
-func extractQueries(path string) ([]string, error) {
- if strings.HasSuffix(path, ".json") {
- fmt.Println("Path: " + path)
- file, err := os.Open(path)
- if err != nil {
- return nil, err
- }
- defer util.Close(file)
-
- scanner := bufio.NewScanner(file)
- for scanner.Scan() {
- rec := LogEntry{}
- recText := scanner.Text()
- if err := json.Unmarshal([]byte(recText), &rec); err != nil {
- return nil, err
- }
-
- s := rec.TextPayload
- start := strings.Index(s, RAW_QUERY_STR)
- if start != -1 {
- start += len(RAW_QUERY_STR)
- end := start
- for (end < len(s)) && (s[end] != '"') {
- end++
- }
- if s[end] == '"' {
- target := s[start:end]
- if target != "" {
- fmt.Println(target)
- }
- }
- }
- }
-
- if err := scanner.Err(); err != nil {
- return nil, err
- }
- }
-
- return nil, nil
-}
diff --git a/golden/cmd/netdiff/main.go b/golden/cmd/netdiff/main.go
deleted file mode 100644
index 4e89a83..0000000
--- a/golden/cmd/netdiff/main.go
+++ /dev/null
@@ -1,60 +0,0 @@
-// Simple command line app that compares the images based on their digests.
-// This is a simple standalone client to the skia_image_server.
-// Primarily used for debugging.
-package main
-
-import (
- "context"
- "flag"
- "fmt"
- "os"
-
- "go.skia.org/infra/go/common"
- "go.skia.org/infra/go/sklog"
- "go.skia.org/infra/golden/go/diffstore"
- "go.skia.org/infra/golden/go/types"
- "google.golang.org/grpc"
-)
-
-var (
- grpcAddr = flag.String("grpc_address", "localhost:9000", "gRPC service address (e.g., ':9000')")
-)
-
-func main() {
- common.Init()
- if flag.NArg() < 2 {
- sklog.Fatalf("Usage: %s digest1 digest2 [digest3 ... digestN]\n", os.Args[0])
- }
-
- args := flag.Args()
- mainDigest := types.Digest(args[0])
- rightDigests := make(types.DigestSlice, 0, len(args)-1)
- for _, d := range args[1:] {
- rightDigests = append(rightDigests, types.Digest(d))
- }
-
- // Create the client connection and connect to the server.
- conn, err := grpc.Dial(*grpcAddr, grpc.WithInsecure())
- if err != nil {
- sklog.Fatalf("Unable to connect to grpc service: %s", err)
- }
-
- diffStore, err := diffstore.NewNetDiffStore(context.Background(), conn, "")
- if err != nil {
- sklog.Fatalf("Unable to initialize NetDiffStore: %s", err)
- }
-
- diffResult, err := diffStore.Get(context.Background(), mainDigest, rightDigests)
- if err != nil {
- sklog.Fatalf("Unable to compare digests: %s", err)
- }
-
- for _, rDigest := range rightDigests {
- fmt.Printf("%s <-> %s\n", mainDigest, rDigest)
- metrics := diffResult[rDigest]
- fmt.Printf(" Dimensions are different: %v\n", metrics.DimDiffer)
- fmt.Printf(" Number of pixels different: %v\n", metrics.NumDiffPixels)
- fmt.Printf(" Pixel diff percent: %v\n", metrics.PixelDiffPercent)
- fmt.Printf(" Max RGBA: %v\n", metrics.MaxRGBADiffs)
- }
-}
diff --git a/golden/cmd/skia_knowledge/main.go b/golden/cmd/skia_knowledge/main.go
deleted file mode 100644
index 678d020..0000000
--- a/golden/cmd/skia_knowledge/main.go
+++ /dev/null
@@ -1,137 +0,0 @@
-// Queries a Gold instance and downloads all positive images for each tests.
-// Stores metadata in the meta.json output file.
-package main
-
-import (
- "flag"
- "fmt"
- "io"
- "net/http"
- "os"
- "path/filepath"
- "strings"
-
- "go.skia.org/infra/go/common"
- "go.skia.org/infra/go/fileutil"
- "go.skia.org/infra/go/httputils"
- "go.skia.org/infra/go/sklog"
- "go.skia.org/infra/go/util"
- "go.skia.org/infra/golden/go/search/export"
-)
-
-const (
- // Default query that will get all tests.
- DEFAULT_QUERY = "fdiffmax=-1&fref=false&frgbamax=255&frgbamin=0&head=true&include=false&limit=50&match=name&metric=combined&neg=false&offset=0&pos=true&query=source_type%3Dgm&sort=desc&unt=false&nodiff=true"
-
- // Default URL for the Gold instance we want to query.
- GOLD_URL = "https://gold.skia.org"
-
- // Search endpoint of Gold.
- SEARCH_PATH = "/json/export"
-
- // Template for image file names. Generally the digest is filled in.
- IMG_FILENAME_TMPL = "%s.png"
-
- // Name of the output metameta data file.
- META_DATA_FILE = "meta.json"
-)
-
-var (
- baseURL = flag.String("base_url", GOLD_URL, "Query URL to retrieve meta data.")
- indexFile = flag.String("index_file", "./"+META_DATA_FILE, "Path of the index file.")
- outputDir = flag.String("output_dir", "", "Directory where images should be written. If empty no images will be written.")
-)
-
-func main() {
- common.Init()
-
- // We need at least the index file or an output directory.
- if *outputDir == "" && *indexFile == "" {
- sklog.Fatal("No index file or output directory specified.")
- }
-
- // If the index file is empty write it to output directory.
- useIndexPath := *indexFile
- if useIndexPath == "" {
- useIndexPath = filepath.Join(*outputDir, META_DATA_FILE)
- }
-
- // Set up the http client.
- client := httputils.DefaultClientConfig().Client()
-
- // load the test meta data from Gold.
- testRecords, err := loadMetaData(client, *baseURL, DEFAULT_QUERY, META_DATA_FILE)
- if err != nil {
- sklog.Fatalf("Error loading meta data: %s", err)
- }
- sklog.Info("Meta data loaded from Gold.")
-
- // Write the index file to disk.
- if err := export.WriteTestRecordsFile(testRecords, useIndexPath); err != nil {
- sklog.Fatalf("Error writing index file: %s", err)
- }
- sklog.Info("Index file written to disk.")
-
- // If an output directory was given, download the images referenced in the index file.
- if *outputDir != "" {
- if err := downloadImages(*outputDir, client, testRecords); err != nil {
- sklog.Fatalf("Error downloading images: %s", err)
- }
- }
-
- sklog.Infof("Success. Knowledge data written to %s", *outputDir)
-}
-
-// loadMetaData makes a query to a Gold instance and parses the JSON response.
-func loadMetaData(client *http.Client, baseURL, query, metaDataFileName string) ([]*export.TestRecord, error) {
- url := strings.TrimRight(baseURL, "/") + SEARCH_PATH + "?" + query
- sklog.Infof("Requesting url: %s", url)
- resp, err := client.Get(url)
- if err != nil {
- return nil, err
- }
- defer util.Close(resp.Body)
- return export.ReadTestRecords(resp.Body)
-}
-
-// downloadImages downloads all images referenced in the meta data to disk.
-// One directory is created for each test.
-func downloadImages(baseDir string, client *http.Client, testRecs []*export.TestRecord) error {
- for _, testRec := range testRecs {
- testDir := filepath.Join(baseDir, string(testRec.TestName))
- absDirPath, err := fileutil.EnsureDirExists(testDir)
- if err != nil {
- sklog.Errorf("Error creating directory '%s'. Skipping. Got error: %s", testDir, err)
- continue
- }
-
- for _, digestRec := range testRec.Digests {
- // Download the image and then write it to disk.
- resp, err := client.Get(digestRec.URL)
- if err != nil {
- sklog.Errorf("Error retrieving file '%s'. Got error: %s", digestRec.URL, err)
- continue
- }
- defer util.Close(resp.Body)
-
- // Write the image to disk.
- imgFileName := fmt.Sprintf(IMG_FILENAME_TMPL, digestRec.Digest)
- fileName := filepath.Join(absDirPath, imgFileName)
- func(outputFileName string, reader io.Reader) {
- f, err := os.Create(outputFileName)
- if err != nil {
- sklog.Errorf("Error opening output file '%s'. Got error: %s", outputFileName, err)
- return
- }
- defer util.Close(f)
-
- if _, err := io.Copy(f, reader); err != nil {
- sklog.Errorf("Error saving file '%s'. Got error: %s", outputFileName, err)
- return
- }
- sklog.Infof("Downloaded %s sucessfully.", outputFileName)
- }(fileName, resp.Body)
- }
- }
- return nil
-}
diff --git a/golden/go/search/export/export.go b/golden/go/search/export/export.go
index db91e94..1c6aba8 100644
--- a/golden/go/search/export/export.go
+++ b/golden/go/search/export/export.go
@@ -1,15 +1,13 @@
-// package export has the functionality needed to export results from search
-// to JSON. It is primarily used by the skia_knowledge executable.
+// Package export has the functionality needed to export results from search
+// to JSON. It was at one point used by skqp. That dependency is unclear at present.
package export
import (
"encoding/json"
"fmt"
"io"
- "os"
"strings"
- "go.skia.org/infra/go/skerr"
"go.skia.org/infra/go/sklog"
"go.skia.org/infra/golden/go/search/frontend"
"go.skia.org/infra/golden/go/types"
@@ -44,7 +42,7 @@
digestInfo := &DigestInfo{
SearchResult: oneDigest,
- URL: DigestUrl(imgBaseURL, oneDigest.Digest),
+ URL: digestURL(imgBaseURL, oneDigest.Digest),
}
testName := types.TestName(oneDigest.ParamSet[types.PrimaryKeyField][0])
@@ -67,37 +65,22 @@
return ret
}
-// WriteTestRecordsFile writes the retrieved information about tests to a file as JSON.
-func WriteTestRecordsFile(testRecs []*TestRecord, outputPath string) error {
- f, err := os.Create(outputPath)
- if err != nil {
- return err
- }
- if err := WriteTestRecords(testRecs, f); err != nil {
- return skerr.Wrapf(err, "writing test records to %s", outputPath)
- }
- if err := f.Close(); err != nil {
- return skerr.Wrapf(err, "closing %s", outputPath)
- }
- return nil
-}
-
// WriteTestRecords writes the retrieved information about tests to the given writer JSON.
func WriteTestRecords(testRecs []*TestRecord, writer io.Writer) error {
return json.NewEncoder(writer).Encode(testRecs)
}
-// ReadTestRecords loads a file with test records.
-func ReadTestRecords(reader io.Reader) ([]*TestRecord, error) {
- ret := []*TestRecord{}
+// readTestRecords loads a file with test records.
+func readTestRecords(reader io.Reader) ([]*TestRecord, error) {
+ var ret []*TestRecord
if err := json.NewDecoder(reader).Decode(&ret); err != nil {
return nil, err
}
return ret, nil
}
-// GetURL returns the URL given a base URL and the digest.
-func DigestUrl(baseURL string, digest types.Digest) string {
+// digestURL returns the URL given a base URL and the digest.
+func digestURL(baseURL string, digest types.Digest) string {
baseURL = strings.TrimRight(baseURL, "/")
return fmt.Sprintf(urlTemplate, baseURL, digest)
}
diff --git a/golden/go/search/export/export_test.go b/golden/go/search/export/export_test.go
index ec4ca44..4f4632f 100644
--- a/golden/go/search/export/export_test.go
+++ b/golden/go/search/export/export_test.go
@@ -28,7 +28,7 @@
var buf bytes.Buffer
require.NoError(t, WriteTestRecords(testRecs, &buf))
- found, err := ReadTestRecords(&buf)
+ found, err := readTestRecords(&buf)
require.NoError(t, err)
require.Equal(t, testRecs, found)
}
