perf-ingest - Cache the results from git.
Bug: skia:
Change-Id: I629ac402bc84795086f3d3745e8485b61611a601
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/201377
Commit-Queue: Joe Gregorio <jcgregorio@google.com>
Reviewed-by: Joe Gregorio <jcgregorio@google.com>
diff --git a/perf/go/perf-ingest/main.go b/perf/go/perf-ingest/main.go
index 1db3f79..395aba2 100644
--- a/perf/go/perf-ingest/main.go
+++ b/perf/go/perf-ingest/main.go
@@ -14,6 +14,7 @@
_ "net/http/pprof"
"os"
"strings"
+ "sync"
"time"
"cloud.google.com/go/bigtable"
@@ -49,6 +50,13 @@
)
var (
+ // mutex protects hashCache.
+ mutex = sync.Mutex{}
+ // hashCache is a cache of results from calling vcs.IndexOf().
+ hashCache = map[string]int{}
+)
+
+var (
// NonRecoverableError is returned if the error is non-recoverable and we
// should Ack the PubSub message. This might happen if, for example, a
// non-JSON file gets dropped in the bucket.
@@ -107,6 +115,21 @@
return params, values, ps
}
+func indexFromCache(hash string) (int, bool) {
+ mutex.Lock()
+ defer mutex.Unlock()
+
+ index, ok := hashCache[hash]
+ return index, ok
+}
+
+func indexToCache(hash string, index int) {
+ mutex.Lock()
+ defer mutex.Unlock()
+
+ hashCache[hash] = index
+}
+
// processSingleFile parses the contents of a single JSON file and writes the values into BigTable.
func processSingleFile(ctx context.Context, store *btts.BigTableTraceStore, vcs vcsinfo.VCS, name string, r io.Reader, timestamp time.Time) error {
benchData, err := ingestcommon.ParseBenchDataFromReader(r)
@@ -116,16 +139,21 @@
}
params, values, paramset := getParamsAndValues(benchData)
- index, err := vcs.IndexOf(ctx, benchData.Hash)
- if err != nil {
- if err := vcs.Update(context.Background(), true, false); err != nil {
- return fmt.Errorf("Could not ingest, failed to pull: %s", err)
- }
+ index, ok := indexFromCache(benchData.Hash)
+ if !ok {
+ var err error
index, err = vcs.IndexOf(ctx, benchData.Hash)
if err != nil {
- sklog.Errorf("Could not ingest, hash not found even after pulling %q: %s", benchData.Hash, err)
- return NonRecoverableError
+ if err := vcs.Update(context.Background(), true, false); err != nil {
+ return fmt.Errorf("Could not ingest, failed to pull: %s", err)
+ }
+ index, err = vcs.IndexOf(ctx, benchData.Hash)
+ if err != nil {
+ sklog.Errorf("Could not ingest, hash not found even after pulling %q: %s", benchData.Hash, err)
+ return NonRecoverableError
+ }
}
+ indexToCache(benchData.Hash, index)
}
tileKey := store.TileKey(int32(index))
ops, err := store.UpdateOrderedParamSet(tileKey, paramset)
