[gold] Add more logging to backup ingestion code
This also expands the window on Chrome as a speculative fix
for the linked bug.
Bug: skia:12100
Change-Id: I51ff4d73f78610f032a4ee3e37ae00c9263c8276
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/418963
Reviewed-by: Leandro Lovisolo <lovisolo@google.com>
Commit-Queue: Kevin Lubick <kjlubick@google.com>
diff --git a/golden/cmd/gold_ingestion/gold_ingestion.go b/golden/cmd/gold_ingestion/gold_ingestion.go
index 75c3354..a556c8e 100644
--- a/golden/cmd/gold_ingestion/gold_ingestion.go
+++ b/golden/cmd/gold_ingestion/gold_ingestion.go
@@ -438,17 +438,13 @@
})
go util.RepeatCtx(ctx, isc.BackupPollInterval.Duration, func(ctx context.Context) {
- ctx, span := trace.StartSpan(ctx, "ingestion_backupPollingCycle")
+ ctx, span := trace.StartSpan(ctx, "ingestion_backupPollingCycle", trace.WithSampler(trace.AlwaysSample()))
defer span.End()
startTime, endTime := getTimesToPoll(ctx, isc.BackupPollScope.Duration)
- processed := int64(0)
- ignored := int64(0)
-
+ totalIgnored, totalProcessed := 0, 0
+ sklog.Infof("Starting backup polling for %d sources in time range [%s,%s]", len(sourcesToScan), startTime, endTime)
for _, src := range sourcesToScan {
- // Failure to do this can cause a race condition in tests.
- if stringer, ok := src.(fmt.Stringer); ok {
- sklog.Infof("Performing backup scan of %s", stringer.String())
- }
+ ignored, processed := 0, 0
files := src.SearchForFiles(ctx, startTime, endTime)
for _, f := range files {
ok, err := pss.IngestionStore.WasIngested(ctx, f)
@@ -462,9 +458,17 @@
processed++
pss.ingestFile(ctx, f)
}
+ srcName := "<unknown>"
+ // Failure to do this can cause a race condition in tests.
+ if stringer, ok := src.(fmt.Stringer); ok {
+ srcName = stringer.String()
+ }
+ sklog.Infof("backup polling for %s processed/ignored: %d/%d", srcName, processed, ignored)
+ totalIgnored += ignored
+ totalProcessed += processed
}
pollingLiveness.Reset()
- sklog.Infof("Backup polling received/processed/ignored: %d/%d/%d", ignored+processed, processed, ignored)
+ sklog.Infof("Total backup polling [%s,%s] processed/ignored: %d/%d/%d", startTime, endTime, totalProcessed, totalIgnored)
})
}
diff --git a/golden/k8s-instances/chrome/chrome-ingestion-bt.json5 b/golden/k8s-instances/chrome/chrome-ingestion-bt.json5
index b1ef3c6..d655457 100644
--- a/golden/k8s-instances/chrome/chrome-ingestion-bt.json5
+++ b/golden/k8s-instances/chrome/chrome-ingestion-bt.json5
@@ -1,6 +1,6 @@
{
backup_poll_interval: "1h",
- backup_poll_scope: "2h",
+ backup_poll_scope: "4h",
ingestion_files_topic: "gold-chrome-eventbus-bt",
ingestion_subscription: "gold-ingestion+gold-chrome-eventbus-bt",
files_processed_in_parallel: 20, // chrome has lots of small files.
