Remove unused code in Gold search now that cache is enabled. - Make cache on by default in search. - Delete the code/queries that are no longer used. Bug: b/377757813 Change-Id: I44b60b606f8def7374bdcb1f7743ccea94599da2 Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/922569 Commit-Queue: Ashwin Verleker <ashwinpv@google.com> Reviewed-by: Wenbin Zhang <wenbinzhang@google.com>
diff --git a/golden/cmd/gold_frontend/gold_frontend.go b/golden/cmd/gold_frontend/gold_frontend.go index 4c5503b..c98b24d 100644 --- a/golden/cmd/gold_frontend/gold_frontend.go +++ b/golden/cmd/gold_frontend/gold_frontend.go
@@ -168,17 +168,16 @@ templates[crs.ID] = crs.URLTemplate } - s2a := search.New(sqlDB, fsc.WindowSize) cacheClient, err := fsc.GetCacheClient(ctx) if err != nil { - // TODO(ashwinpv): Once we are fully onboarded, this error should cause a failure. - sklog.Warningf("Error while trying to create a new cache client: %v", err) + sklog.Fatalf("Error while trying to create a new cache client: %v", err) } - if cacheClient != nil { - sklog.Debugf("Enabling cache for search.") - s2a.EnableCache(cacheClient, fsc.CachingCorpora) + if cacheClient == nil { + sklog.Fatalf("Cache is not configured correctly for this instance.") } + s2a := search.New(sqlDB, fsc.WindowSize, cacheClient, fsc.CachingCorpora) + s2a.SetReviewSystemTemplates(templates) sklog.Infof("SQL Search loaded with CRS templates %s", templates) err = s2a.StartCacheProcess(ctx, 5*time.Minute, fsc.WindowSize)
diff --git a/golden/go/search/BUILD.bazel b/golden/go/search/BUILD.bazel index 334f382..f84cccc 100644 --- a/golden/go/search/BUILD.bazel +++ b/golden/go/search/BUILD.bazel
@@ -37,6 +37,7 @@ srcs = ["search_test.go"], embed = [":search"], deps = [ + "//go/cache/local", "//go/paramtools", "//golden/go/expectations", "//golden/go/publicparams",
diff --git a/golden/go/search/search.go b/golden/go/search/search.go index 039b80d..21551f9 100644 --- a/golden/go/search/search.go +++ b/golden/go/search/search.go
@@ -211,7 +211,7 @@ } // New returns an implementation of API. -func New(sqlDB *pgxpool.Pool, windowLength int) *Impl { +func New(sqlDB *pgxpool.Pool, windowLength int, cacheClient cache.Cache, cache_corpora []string) *Impl { cc, err := lru.New(commitCacheSize) if err != nil { panic(err) // should only happen if commitCacheSize is negative. @@ -234,14 +234,10 @@ traceCache: tc, paramsetCache: pc, reviewSystemMapping: map[string]string{}, + cacheManager: caching.New(cacheClient, sqlDB, cache_corpora, windowLength), } } -// EnableCache enables reading data from cache for search. -func (s *Impl) EnableCache(cacheClient cache.Cache, cache_corpora []string) { - s.cacheManager = caching.New(cacheClient, s.db, cache_corpora, s.windowLength) -} - // SetReviewSystemTemplates sets the URL templates that are used to link to the code review system. // The Changelist ID will be substituted in using fmt.Sprintf and a %s placeholder. func (s *Impl) SetReviewSystemTemplates(m map[string]string) { @@ -2739,82 +2735,23 @@ ctx, span := trace.StartSpan(ctx, "getTracesWithUntriagedDigestsAtHead") defer span.End() - // Caching is enabled. - if s.cacheManager != nil { - sklog.Debugf("Search cache is enabled.") - byBlameData, err := s.cacheManager.GetByBlameData(ctx, string(common.GetFirstCommitID(ctx)), corpus) - if err != nil { - sklog.Errorf("Error encountered when retrieving ByBlame data from cache: %v", err) - return nil, err - } - - sklog.Debugf("Retrieved %d items from search cache for corpus %s", len(byBlameData), corpus) - rv := map[groupingDigestKey][]schema.TraceID{} - var key groupingDigestKey - groupingKey := key.groupingID[:] - digestKey := key.digest[:] - for _, data := range byBlameData { - copy(groupingKey, data.GroupingID) - copy(digestKey, data.Digest) - rv[key] = append(rv[key], data.TraceID) - } - - return rv, nil - } - - sklog.Debugf("Search cache is not enabled. Proceeding with regular search.") - statement := `WITH -UntriagedDigests AS ( - SELECT grouping_id, digest FROM Expectations - WHERE label = 'u' -), -UnignoredDataAtHead AS ( - SELECT trace_id, grouping_id, digest FROM ValuesAtHead@corpus_commit_ignore_idx - WHERE matches_any_ignore_rule = FALSE AND most_recent_commit_id >= $1 AND corpus = $2 -) -SELECT UnignoredDataAtHead.trace_id, UnignoredDataAtHead.grouping_id, UnignoredDataAtHead.digest FROM -UntriagedDigests -JOIN UnignoredDataAtHead ON UntriagedDigests.grouping_id = UnignoredDataAtHead.grouping_id AND - UntriagedDigests.digest = UnignoredDataAtHead.digest -` - arguments := []interface{}{common.GetFirstCommitID(ctx), corpus} - if mv := s.getMaterializedView(byBlameView, corpus); mv != "" { - // While using the by blame view, it's important that we filter out digests that have since - // been triaged, or the user might notice a delay of several minutes between the moment they - // perform a triage action and the moment their action seemingly takes effect. - statement = `WITH - ByBlameMaterializedView AS ( - SELECT * FROM ` + mv + ` - ) - SELECT ByBlameMaterializedView.trace_id, - ByBlameMaterializedView.grouping_id, - ByBlameMaterializedView.digest - FROM ByBlameMaterializedView - JOIN Expectations USING (grouping_id, digest) - WHERE Expectations.label = '` + string(schema.LabelUntriaged) + `'` - arguments = nil - } - - rows, err := s.db.Query(ctx, statement, arguments...) + byBlameData, err := s.cacheManager.GetByBlameData(ctx, string(common.GetFirstCommitID(ctx)), corpus) if err != nil { - return nil, skerr.Wrap(err) + sklog.Errorf("Error encountered when retrieving ByBlame data from cache: %v", err) + return nil, err } - defer rows.Close() + + sklog.Debugf("Retrieved %d items from search cache for corpus %s", len(byBlameData), corpus) rv := map[groupingDigestKey][]schema.TraceID{} var key groupingDigestKey groupingKey := key.groupingID[:] digestKey := key.digest[:] - for rows.Next() { - var traceID schema.TraceID - var groupingID schema.GroupingID - var digest schema.DigestBytes - if err := rows.Scan(&traceID, &groupingID, &digest); err != nil { - return nil, skerr.Wrap(err) - } - copy(groupingKey, groupingID) - copy(digestKey, digest) - rv[key] = append(rv[key], traceID) + for _, data := range byBlameData { + copy(groupingKey, data.GroupingID) + copy(digestKey, data.Digest) + rv[key] = append(rv[key], data.TraceID) } + return rv, nil }
diff --git a/golden/go/search/search_test.go b/golden/go/search/search_test.go index b6b8746..299c5f1 100644 --- a/golden/go/search/search_test.go +++ b/golden/go/search/search_test.go
@@ -15,6 +15,7 @@ "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.skia.org/infra/go/cache/local" "go.skia.org/infra/go/paramtools" "go.skia.org/infra/golden/go/expectations" "go.skia.org/infra/golden/go/publicparams" @@ -41,7 +42,9 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartCacheProcess(ctx, time.Minute, 100)) rv, err := s.NewAndUntriagedSummaryForCL(ctx, sql.Qualify(dks.GerritCRS, dks.ChangelistIDThatAttemptsToFixIOS)) require.NoError(t, err) @@ -68,7 +71,9 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartCacheProcess(ctx, time.Minute, 100)) rv, err := s.NewAndUntriagedSummaryForCL(ctx, sql.Qualify(dks.GerritCRS, dks.ChangelistIDWithMultipleDatapointsPerTrace)) require.NoError(t, err) @@ -93,7 +98,9 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartCacheProcess(ctx, time.Minute, 100)) rv, err := s.NewAndUntriagedSummaryForCL(ctx, sql.Qualify(dks.GerritInternalCRS, dks.ChangelistIDThatAddsNewTests)) require.NoError(t, err) @@ -162,7 +169,9 @@ require.NoError(t, sqltest.BulkInsertDataTables(ctx, db, b.Build())) waitForSystemTime() - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartCacheProcess(ctx, time.Minute, 100)) rv, err := s.NewAndUntriagedSummaryForCL(ctx, sql.Qualify(dks.GerritCRS, clID)) require.NoError(t, err) @@ -193,9 +202,11 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartCacheProcess(ctx, time.Minute, 100)) - _, err := s.NewAndUntriagedSummaryForCL(ctx, sql.Qualify(dks.GerritInternalCRS, "does not exist")) + _, err = s.NewAndUntriagedSummaryForCL(ctx, sql.Qualify(dks.GerritInternalCRS, "does not exist")) require.Error(t, err) assert.Contains(t, err.Error(), "not found") } @@ -263,7 +274,9 @@ require.NoError(t, sqltest.BulkInsertDataTables(ctx, db, b.Build())) waitForSystemTime() - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartCacheProcess(ctx, time.Minute, 100)) rv, err := s.NewAndUntriagedSummaryForCL(ctx, sql.Qualify(dks.GerritCRS, clID)) require.NoError(t, err) @@ -351,7 +364,9 @@ require.NoError(t, sqltest.BulkInsertDataTables(ctx, db, b.Build())) waitForSystemTime() - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartCacheProcess(ctx, time.Minute, 100)) rv, err := s.NewAndUntriagedSummaryForCL(ctx, sql.Qualify(dks.GerritCRS, clID)) require.NoError(t, err) @@ -434,7 +449,9 @@ require.NoError(t, sqltest.BulkInsertDataTables(ctx, db, b.Build())) waitForSystemTime() - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartCacheProcess(ctx, time.Minute, 100)) rv, err := s.NewAndUntriagedSummaryForCL(ctx, sql.Qualify(dks.GerritCRS, clID)) require.NoError(t, err) @@ -471,7 +488,9 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) // Update the caches aggressively to be writing to the shared cache while reading from it. require.NoError(t, s.StartCacheProcess(ctx, 100*time.Millisecond, 100)) @@ -507,7 +526,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) ts, err := s.ChangelistLastUpdated(ctx, sql.Qualify(dks.GerritInternalCRS, dks.ChangelistIDThatAddsNewTests)) require.NoError(t, err) assert.Equal(t, changelistTSForNewTests, ts) @@ -518,7 +539,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) ts, err := s.ChangelistLastUpdated(ctx, sql.Qualify(dks.GerritInternalCRS, "does not exist")) require.NoError(t, err) assert.True(t, ts.IsZero()) @@ -529,7 +552,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: true, IncludePositiveDigests: false, @@ -553,7 +578,9 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 10) // Otherwise there's no commit for the materialized views + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 10, cache, nil) // Otherwise there's no commit for the materialized views require.NoError(t, s.StartMaterializedViews(ctx, []string{dks.CornersCorpus, dks.RoundCorpus}, time.Minute)) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: true, @@ -799,7 +826,9 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 10) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 10, cache, nil) require.NoError(t, s.StartMaterializedViews(ctx, []string{dks.CornersCorpus, dks.RoundCorpus}, time.Minute)) assertNumRows(t, db, "mv_corners_traces", 21) @@ -811,7 +840,9 @@ ctx, cancel := context.WithCancel(context.Background()) defer cancel() db := sqltest.NewCockroachDBForTestsWithProductionSchema(ctx, t) - s := New(db, 10) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 10, cache, nil) require.NoError(t, s.StartMaterializedViews(ctx, []string{dks.CornersCorpus, dks.RoundCorpus}, time.Second)) // no data yet assertNumRows(t, db, "mv_corners_traces", 0) @@ -836,7 +867,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: true, IncludePositiveDigests: false, @@ -951,7 +984,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: true, IncludePositiveDigests: false, @@ -1041,7 +1076,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: true, IncludePositiveDigests: true, @@ -1486,7 +1523,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: true, IncludePositiveDigests: true, @@ -1512,7 +1551,9 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 10) // Otherwise there's no commit for the materialized views + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 10, cache, nil) // Otherwise there's no commit for the materialized views require.NoError(t, s.StartMaterializedViews(ctx, []string{dks.CornersCorpus, dks.RoundCorpus}, time.Minute)) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: true, @@ -1754,7 +1795,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: true, IncludePositiveDigests: true, @@ -1881,7 +1924,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: false, IncludePositiveDigests: false, @@ -1905,7 +1950,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 10) // Otherwise there's no commit for the materialized views + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 10, cache, nil) // Otherwise there's no commit for the materialized views require.NoError(t, s.StartMaterializedViews(ctx, []string{dks.CornersCorpus, dks.RoundCorpus}, time.Minute)) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: false, @@ -2307,7 +2354,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: false, IncludePositiveDigests: false, @@ -2334,7 +2383,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 10) // Otherwise there's no commit for the materialized views + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 10, cache, nil) // Otherwise there's no commit for the materialized views require.NoError(t, s.StartMaterializedViews(ctx, []string{dks.CornersCorpus, dks.RoundCorpus}, time.Minute)) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: false, @@ -2474,7 +2525,9 @@ require.NoError(t, sqltest.BulkInsertDataTables(ctx, db, b.Build())) waitForSystemTime() - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: true, IncludePositiveDigests: false, @@ -2637,7 +2690,9 @@ }) require.NoError(t, err) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartApplyingPublicParams(ctx, matcher, time.Minute)) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: true, @@ -2669,7 +2724,9 @@ }) require.NoError(t, err) - s := New(db, 10) // Otherwise there's no commit for the materialized views + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 10, cache, nil) // Otherwise there's no commit for the materialized views require.NoError(t, s.StartApplyingPublicParams(ctx, matcher, time.Minute)) require.NoError(t, s.StartMaterializedViews(ctx, []string{dks.CornersCorpus, dks.RoundCorpus}, time.Minute)) res, err := s.Search(ctx, &query.Search{ @@ -2823,7 +2880,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ OnlyIncludeDigestsProducedAtHead: true, IncludePositiveDigests: false, @@ -3032,7 +3091,9 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) s.SetReviewSystemTemplates(map[string]string{ dks.GerritCRS: "http://example.com/public/%s", dks.GerritInternalCRS: "http://example.com/internal/%s", @@ -3205,7 +3266,9 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartCacheProcess(ctx, time.Minute, 100)) res, err := s.Search(ctx, &query.Search{ IncludePositiveDigests: true, @@ -3462,7 +3525,9 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) s.SetReviewSystemTemplates(map[string]string{ dks.GerritCRS: "http://example.com/public/%s", dks.GerritInternalCRS: "http://example.com/internal/%s", @@ -3584,7 +3649,9 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) s.SetReviewSystemTemplates(map[string]string{ dks.GerritCRS: "http://example.com/public/%s", dks.GerritInternalCRS: "http://example.com/internal/%s", @@ -3936,7 +4003,9 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) s.SetReviewSystemTemplates(map[string]string{ dks.GerritCRS: "http://example.com/public/%s", dks.GerritInternalCRS: "http://example.com/internal/%s", @@ -4175,7 +4244,9 @@ defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) s.SetReviewSystemTemplates(map[string]string{ dks.GerritCRS: "http://example.com/public/%s", dks.GerritInternalCRS: "http://example.com/internal/%s", @@ -4305,7 +4376,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) ps, err := s.GetPrimaryBranchParamset(ctx) require.NoError(t, err) assert.Equal(t, paramtools.ReadOnlyParamSet{ @@ -4333,7 +4406,9 @@ }) require.NoError(t, err) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartApplyingPublicParams(ctx, matcher, time.Minute)) ps, err := s.GetPrimaryBranchParamset(ctx) @@ -4352,7 +4427,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) ps, err := s.GetChangelistParamset(ctx, dks.GerritCRS, dks.ChangelistIDThatAttemptsToFixIOS) require.NoError(t, err) assert.Equal(t, paramtools.ReadOnlyParamSet{ @@ -4389,8 +4466,10 @@ ctx := context.Background() db := sqltest.NewCockroachDBForTestsWithProductionSchema(ctx, t) - s := New(db, 100) - _, err := s.GetChangelistParamset(ctx, "does not", "exist") + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) + _, err = s.GetChangelistParamset(ctx, "does not", "exist") require.Error(t, err) assert.Contains(t, err.Error(), "Could not find") } @@ -4414,7 +4493,9 @@ }) require.NoError(t, err) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartApplyingPublicParams(ctx, matcher, time.Minute)) ps, err := s.GetChangelistParamset(ctx, dks.GerritCRS, dks.ChangelistIDThatAttemptsToFixIOS) require.NoError(t, err) @@ -4441,7 +4522,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) blames, err := s.GetBlamesForUntriagedDigests(ctx, dks.RoundCorpus) require.NoError(t, err) @@ -4453,7 +4536,9 @@ ctx, cancel := context.WithCancel(context.Background()) defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 9) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 9, cache, nil) require.NoError(t, s.StartMaterializedViews(ctx, []string{dks.CornersCorpus, dks.RoundCorpus}, time.Minute)) blames, err := s.GetBlamesForUntriagedDigests(ctx, dks.RoundCorpus) @@ -4510,7 +4595,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ BlameGroupID: string(dks.WindowsDriverUpdateCommitID), @@ -4539,7 +4626,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 10) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 10, cache, nil) if withMaterializedView { // Ensure the materialized view does not get updated for the duration of this test. @@ -4684,7 +4773,9 @@ ctx, cancel := context.WithCancel(context.Background()) defer cancel() db := useKitchenSinkData(ctx, t) - s := New(db, 10) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 10, cache, nil) require.NoError(t, s.StartMaterializedViews(ctx, []string{dks.CornersCorpus, dks.RoundCorpus}, time.Minute)) res, err := s.Search(ctx, &query.Search{ @@ -4839,7 +4930,9 @@ require.NoError(t, sqltest.BulkInsertDataTables(ctx, db, existingData)) waitForSystemTime() - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) // We want to assert that searching with the given blameID, we return exactly one result // corresponding to the given digest. This makes sure we can take the results of @@ -4878,7 +4971,9 @@ betaGrouping := paramtools.Params{types.PrimaryKeyField: "beta", types.CorpusField: "test_corpus"} gammaGrouping := paramtools.Params{types.PrimaryKeyField: "gamma", types.CorpusField: "test_corpus"} - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.GetBlamesForUntriagedDigests(ctx, "test_corpus") require.NoError(t, err) @@ -4970,7 +5065,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ BlameGroupID: "0000000106:0000000108", @@ -5099,7 +5196,9 @@ }) require.NoError(t, err) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartApplyingPublicParams(ctx, matcher, time.Minute)) res, err := s.Search(ctx, &query.Search{ BlameGroupID: "0000000106:0000000109", @@ -5198,7 +5297,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ BlameGroupID: string(dks.WindowsDriverUpdateCommitID), @@ -5226,7 +5327,9 @@ _, err := db.Exec(ctx, "DELETE FROM DiffMetrics") require.NoError(t, err) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.Search(ctx, &query.Search{ IncludePositiveDigests: true, MustIncludeReferenceFilter: true, @@ -5266,7 +5369,9 @@ ChangelistURL: "http://example.com/public/CL_fix_ios", }) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) s.SetReviewSystemTemplates(map[string]string{ dks.GerritCRS: "http://example.com/public/%s", dks.GerritInternalCRS: "http://example.com/internal/%s", @@ -5307,7 +5412,9 @@ }) require.NoError(t, err) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartApplyingPublicParams(ctx, matcher, time.Minute)) blames, err := s.GetBlamesForUntriagedDigests(ctx, dks.RoundCorpus) @@ -5346,7 +5453,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) // None of the traces for the corner tests have unignored, untriaged digests at head. // As a result, the blame returned should be empty. @@ -6218,7 +6327,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.GetCluster(ctx, ClusterOptions{ Grouping: paramtools.Params{ types.CorpusField: dks.CornersCorpus, @@ -6298,7 +6409,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.GetCluster(ctx, ClusterOptions{ Grouping: paramtools.Params{ types.CorpusField: dks.CornersCorpus, @@ -6319,7 +6432,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.GetCluster(ctx, ClusterOptions{ Grouping: paramtools.Params{ types.CorpusField: dks.RoundCorpus, @@ -6384,7 +6499,9 @@ }, }) require.NoError(t, err) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartApplyingPublicParams(ctx, matcher, time.Minute)) res, err := s.GetCluster(ctx, ClusterOptions{ Grouping: paramtools.Params{ @@ -6524,7 +6641,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) commits, err := s.GetCommitsInWindow(ctx) require.NoError(t, err) assert.Equal(t, makeKitchenSinkCommits(), commits) @@ -6538,7 +6657,9 @@ allCommits := makeKitchenSinkCommits() mostRecentCommits := allCommits[len(allCommits)-3:] - s := New(db, 3) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 3, cache, nil) commits, err := s.GetCommitsInWindow(ctx) require.NoError(t, err) assert.Equal(t, mostRecentCommits, commits) @@ -6549,7 +6670,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) digests, err := s.GetDigestsForGrouping(ctx, paramtools.Params{ types.PrimaryKeyField: dks.CircleTest, types.CorpusField: dks.RoundCorpus, @@ -6567,7 +6690,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) digests, err := s.GetDigestsForGrouping(ctx, paramtools.Params{ types.PrimaryKeyField: "not a real test", types.CorpusField: "not a real corpus", @@ -6585,7 +6710,9 @@ types.CorpusField: dks.RoundCorpus, } - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) details, err := s.GetDigestDetails(ctx, inputGrouping, dks.DigestC02Pos, "", "") require.NoError(t, err) assert.Equal(t, frontend.DigestDetails{ @@ -6703,7 +6830,9 @@ }) require.NoError(t, err) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartApplyingPublicParams(ctx, matcher, time.Minute)) details, err := s.GetDigestDetails(ctx, inputGrouping, dks.DigestC02Pos, "", "") require.NoError(t, err) @@ -6805,7 +6934,9 @@ }) require.NoError(t, err) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartApplyingPublicParams(ctx, matcher, time.Minute)) _, err = s.GetDigestDetails(ctx, inputGrouping, dks.DigestC02Pos, "", "") require.Error(t, err) @@ -6822,7 +6953,9 @@ types.CorpusField: dks.RoundCorpus, } - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) // This digest is not seen in that grouping on the primary branch. details, err := s.GetDigestDetails(ctx, inputGrouping, dks.DigestE03Unt_CL, "", "") require.NoError(t, err) @@ -6871,7 +7004,9 @@ Subject: "Fix iOS", ChangelistURL: "http://example.com/public/CL_fix_ios", }) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) s.SetReviewSystemTemplates(map[string]string{ dks.GerritCRS: "http://example.com/public/%s", dks.GerritInternalCRS: "http://example.com/internal/%s", @@ -6975,7 +7110,9 @@ }) require.NoError(t, err) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartApplyingPublicParams(ctx, matcher, time.Minute)) s.SetReviewSystemTemplates(map[string]string{ dks.GerritCRS: "http://example.com/public/%s", @@ -7064,7 +7201,9 @@ }) require.NoError(t, err) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartApplyingPublicParams(ctx, matcher, time.Minute)) s.SetReviewSystemTemplates(map[string]string{ dks.GerritCRS: "http://example.com/public/%s", @@ -7095,7 +7234,9 @@ Subject: "multiple datapoints", ChangelistURL: "http://example.com/public/CLmultipledatapoints", }) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) s.SetReviewSystemTemplates(map[string]string{ dks.GerritCRS: "http://example.com/public/%s", dks.GerritInternalCRS: "http://example.com/internal/%s", @@ -7190,13 +7331,15 @@ types.CorpusField: dks.RoundCorpus, } - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) s.SetReviewSystemTemplates(map[string]string{ dks.GerritCRS: "http://example.com/public/%s", dks.GerritInternalCRS: "http://example.com/internal/%s", }) - _, err := s.GetDigestDetails(ctx, inputGrouping, dks.DigestE03Unt_CL, dks.ChangelistIDThatAttemptsToFixIOS, dks.GerritCRS) + _, err = s.GetDigestDetails(ctx, inputGrouping, dks.DigestE03Unt_CL, dks.ChangelistIDThatAttemptsToFixIOS, dks.GerritCRS) require.Error(t, err) assert.Contains(t, err.Error(), "No results found") } @@ -7211,7 +7354,9 @@ types.CorpusField: dks.RoundCorpus, } - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) rv, err := s.GetDigestsDiff(ctx, inputGrouping, dks.DigestC01Pos, dks.DigestC03Unt, "", "") require.NoError(t, err) assert.Equal(t, frontend.DigestComparison{ @@ -7257,8 +7402,10 @@ } const notARealDigest = `ffffffffffffffffffffffffffffffff` - s := New(db, 100) - _, err := s.GetDigestsDiff(ctx, inputGrouping, dks.DigestC01Pos, notARealDigest, "", "") + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) + _, err = s.GetDigestsDiff(ctx, inputGrouping, dks.DigestC01Pos, notARealDigest, "", "") require.Error(t, err) assert.Contains(t, err.Error(), "missing") @@ -7281,7 +7428,9 @@ types.CorpusField: dks.RoundCorpus, } - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) rv, err := s.GetDigestsDiff(ctx, inputGrouping, dks.DigestC01Pos, dks.DigestC06Pos_CL, dks.ChangelistIDThatAttemptsToFixIOS, dks.GerritCRS) require.NoError(t, err) assert.Equal(t, frontend.DigestComparison{ @@ -7325,7 +7474,9 @@ types.CorpusField: dks.RoundCorpus, } - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) rv, err := s.GetDigestsDiff(ctx, inputGrouping, dks.DigestC01Pos, dks.DigestC07Unt_CL, dks.ChangelistIDThatAttemptsToFixIOS, dks.GerritCRS) require.NoError(t, err) assert.Equal(t, frontend.DigestComparison{ @@ -7369,7 +7520,9 @@ types.CorpusField: dks.CornersCorpus, } - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) // In this CL a tryjob was executed multiple times at the last patchset, generating multiple // datapoints for the same trace at the last patchset. DigestC01Pos was drawn on the last two // tryjob runs. @@ -7419,7 +7572,9 @@ types.CorpusField: dks.RoundCorpus, } - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) rv, err := s.GetDigestsDiff(ctx, inputGrouping, dks.DigestC01Pos, dks.DigestC03Unt, "not a real CL", dks.GerritCRS) require.NoError(t, err) assert.Equal(t, frontend.DigestComparison{ @@ -7464,7 +7619,9 @@ types.CorpusField: dks.RoundCorpus, } - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) rv, err := s.GetDigestsDiff(ctx, inputGrouping, dks.DigestC01Pos, dks.DigestC06Pos_CL, "not a real CL", dks.GerritCRS) require.NoError(t, err) assert.Equal(t, frontend.DigestComparison{ @@ -7506,7 +7663,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) resp, err := s.CountDigestsByTest(ctx, frontend.ListTestsQuery{ Corpus: dks.CornersCorpus, IgnoreState: types.ExcludeIgnoredTraces, @@ -7562,7 +7721,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) resp, err := s.CountDigestsByTest(ctx, frontend.ListTestsQuery{ Corpus: dks.CornersCorpus, IgnoreState: types.IncludeIgnoredTraces, @@ -7621,7 +7782,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) resp, err := s.CountDigestsByTest(ctx, frontend.ListTestsQuery{ Corpus: dks.CornersCorpus, IgnoreState: types.ExcludeIgnoredTraces, @@ -7661,8 +7824,10 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) - _, err := s.CountDigestsByTest(ctx, frontend.ListTestsQuery{ + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) + _, err = s.CountDigestsByTest(ctx, frontend.ListTestsQuery{ Corpus: dks.CornersCorpus, IgnoreState: types.ExcludeIgnoredTraces, TraceValues: paramtools.ParamSet{ @@ -7680,7 +7845,9 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.ComputeGUIStatus(ctx) require.NoError(t, err) @@ -7717,7 +7884,9 @@ require.NoError(t, sqltest.BulkInsertDataTables(ctx, db, data)) waitForSystemTime() - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) res, err := s.ComputeGUIStatus(ctx) require.NoError(t, err) @@ -7753,7 +7922,9 @@ }) require.NoError(t, err) - s := New(db, 100) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) require.NoError(t, s.StartApplyingPublicParams(ctx, matcher, time.Minute)) res, err := s.ComputeGUIStatus(ctx) @@ -7827,8 +7998,10 @@ require.NoError(t, sqltest.BulkInsertDataTables(ctx, db, existingData)) waitForSystemTime() - s := New(db, 100) - ctx, err := s.addCommitsData(ctx) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) + ctx, err = s.addCommitsData(ctx) require.NoError(t, err) fec, err := s.getCommits(ctx) require.NoError(t, err) @@ -7880,8 +8053,10 @@ require.NoError(t, sqltest.BulkInsertDataTables(ctx, db, existingData)) waitForSystemTime() - s := New(db, 100) - ctx, err := s.addCommitsData(ctx) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) + ctx, err = s.addCommitsData(ctx) require.NoError(t, err) fec, err := s.getCommits(ctx) require.NoError(t, err) @@ -7902,8 +8077,10 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) - ctx, err := s.addCommitsData(ctx) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) + ctx, err = s.addCommitsData(ctx) require.NoError(t, err) test := func(name string, grouping schema.GroupingID, expectedDigests ...types.Digest) { @@ -7932,8 +8109,10 @@ ctx := context.Background() db := useKitchenSinkData(ctx, t) - s := New(db, 100) - ctx, err := s.addCommitsData(ctx) + cache, err := local.New(100) + require.NoError(t, err) + s := New(db, 100, cache, nil) + ctx, err = s.addCommitsData(ctx) require.NoError(t, err) test := func(name string, rightTraceKeys paramtools.ParamSet, expectedDigests ...types.Digest) {
diff --git a/golden/go/web/BUILD.bazel b/golden/go/web/BUILD.bazel index bdf2850..be51b90 100644 --- a/golden/go/web/BUILD.bazel +++ b/golden/go/web/BUILD.bazel
@@ -54,6 +54,7 @@ "//go/alogin", "//go/alogin/mocks", "//go/alogin/proxylogin", + "//go/cache/local", "//go/now", "//go/paramtools", "//go/roles",
diff --git a/golden/go/web/web_test.go b/golden/go/web/web_test.go index 9020749..c870d7c 100644 --- a/golden/go/web/web_test.go +++ b/golden/go/web/web_test.go
@@ -17,6 +17,7 @@ "github.com/go-chi/chi/v5" ttlcache "github.com/patrickmn/go-cache" + "go.skia.org/infra/go/cache/local" "go.skia.org/infra/go/roles" "github.com/google/uuid" @@ -1210,9 +1211,12 @@ db := sqltest.NewCockroachDBForTestsWithProductionSchema(ctx, t) require.NoError(t, sqltest.BulkInsertDataTables(ctx, db, dks.Build())) + cache, err := local.New(100) + require.NoError(t, err) + wh := initCaches(&Handlers{ HandlersConfig: HandlersConfig{ - Search2API: search.New(db, 10), + Search2API: search.New(db, 10, cache, nil), DB: db, }, })