[gold] Remove ingestion-bt
We will just have one ingestion service that ingests from
the primary branch and optionally tryjob data.
Change-Id: I26ed12f1b25aa9feb81061bcc58cece3ff0d0c98
Bug: skia:10582
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/448264
Reviewed-by: Leandro Lovisolo <lovisolo@google.com>
diff --git a/golden/cmd/gold_ingestion/gold_ingestion.go b/golden/cmd/gold_ingestion/gold_ingestion.go
index e2ac556..a269e8f 100644
--- a/golden/cmd/gold_ingestion/gold_ingestion.go
+++ b/golden/cmd/gold_ingestion/gold_ingestion.go
@@ -248,14 +248,7 @@
}
var primaryBranchProcessor ingestion.Processor
- var err error
- if conf.Type == ingestion_processors.BigTableTraceStore {
- primaryBranchProcessor, err = ingestion_processors.PrimaryBranchBigTable(ctx, src, conf.ExtraParams, vcs)
- if err != nil {
- return nil, nil, skerr.Wrap(err)
- }
- sklog.Infof("Configured BT-backed primary branch ingestion")
- } else if conf.Type == ingestion_processors.SQLPrimaryBranch {
+ if conf.Type == ingestion_processors.SQLPrimaryBranch {
sqlProcessor := ingestion_processors.PrimaryBranchSQL(src, conf.ExtraParams, db)
sqlProcessor.MonitorCacheMetrics(ctx)
primaryBranchProcessor = sqlProcessor
diff --git a/golden/cmd/goldpushk/goldpushk/services_map.go b/golden/cmd/goldpushk/goldpushk/services_map.go
index 11408a5..6735555 100644
--- a/golden/cmd/goldpushk/goldpushk/services_map.go
+++ b/golden/cmd/goldpushk/goldpushk/services_map.go
@@ -22,15 +22,8 @@
DiffCalculator Service = "diffcalculator"
Frontend Service = "frontend"
GitilesFollower Service = "gitilesfollower"
- Ingestion Service = "ingestion" // New, SQL based ingestion
- IngestionBT Service = "ingestion-bt" // Deprecated, BigTable based ingestion
+ Ingestion Service = "ingestion"
PeriodicTasks Service = "periodictasks"
-
- // Testing Gold instances.
- TestInstance1 Instance = "goldpushk-test1"
- TestInstance2 Instance = "goldpushk-test2"
- TestCorpInstance1 Instance = "goldpushk-corp-test1"
- TestCorpInstance2 Instance = "goldpushk-corp-test2"
)
var (
@@ -67,7 +60,6 @@
Frontend,
GitilesFollower,
Ingestion,
- IngestionBT,
PeriodicTasks,
},
}
@@ -82,7 +74,6 @@
s.add(instance, DiffCalculator)
s.add(instance, Frontend)
s.add(instance, Ingestion)
- s.add(instance, IngestionBT)
s.add(instance, PeriodicTasks)
s.add(instance, GitilesFollower)
}
diff --git a/golden/cmd/goldpushk/main_test.go b/golden/cmd/goldpushk/main_test.go
index e3591df..7243333 100644
--- a/golden/cmd/goldpushk/main_test.go
+++ b/golden/cmd/goldpushk/main_test.go
@@ -129,7 +129,6 @@
chromeDiffCalculator := makeID(goldpushk.Chrome, goldpushk.DiffCalculator)
chromeFrontend := makeID(goldpushk.Chrome, goldpushk.Frontend)
chromeIngestion := makeID(goldpushk.Chrome, goldpushk.Ingestion)
- chromeIngestionBT := makeID(goldpushk.Chrome, goldpushk.IngestionBT)
chromeGitilesFollower := makeID(goldpushk.Chrome, goldpushk.GitilesFollower)
chromePeriodicTasks := makeID(goldpushk.Chrome, goldpushk.PeriodicTasks)
chromePublicFrontend := makeID(goldpushk.ChromePublic, goldpushk.Frontend)
@@ -149,7 +148,6 @@
skiaInfraBaselineServer := makeID(goldpushk.SkiaInfra, goldpushk.BaselineServer)
skiaInfraFrontend := makeID(goldpushk.SkiaInfra, goldpushk.Frontend)
skiaIngestion := makeID(goldpushk.Skia, goldpushk.Ingestion)
- skiaIngestionBT := makeID(goldpushk.Skia, goldpushk.IngestionBT)
skiaPublicFrontend := makeID(goldpushk.SkiaPublic, goldpushk.Frontend)
test := func(name string, flagInstances, flagServices, flagCanaries []string, expectedDeployableUnitIDs, expectedCanariedDeployableUnitIDs []goldpushk.DeployableUnitID) {
@@ -211,28 +209,28 @@
////////////////////////////////////////////////////////////////////////////////////////////////
test("Single instance, all services, no canary",
[]string{"chrome"}, []string{"all"}, nil,
- []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeIngestionBT, chromeIngestion, chromeFrontend, chromeGitilesFollower, chromePeriodicTasks},
+ []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeIngestion, chromeFrontend, chromeGitilesFollower, chromePeriodicTasks},
nil)
test("Single instance, all services, one canary",
[]string{"chrome"}, []string{"all"}, []string{"chrome:frontend"},
- []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeIngestionBT, chromeIngestion, chromeGitilesFollower, chromePeriodicTasks},
+ []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeIngestion, chromeGitilesFollower, chromePeriodicTasks},
[]goldpushk.DeployableUnitID{chromeFrontend})
test("Single instance, all services, multiple canaries",
- []string{"chrome"}, []string{"all"}, []string{"chrome:ingestion-bt", "chrome:frontend"},
- []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeIngestion, chromeGitilesFollower, chromePeriodicTasks},
- []goldpushk.DeployableUnitID{chromeIngestionBT, chromeFrontend})
+ []string{"chrome"}, []string{"all"}, []string{"chrome:ingestion", "chrome:frontend"},
+ []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeGitilesFollower, chromePeriodicTasks},
+ []goldpushk.DeployableUnitID{chromeIngestion, chromeFrontend})
test("Multiple instances, all services, no canary",
[]string{"chrome", "skia"}, []string{"all"}, nil,
- []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeIngestionBT, chromeIngestion, chromeFrontend, chromeGitilesFollower, chromePeriodicTasks, skiaDiffCalculator, skiaIngestionBT, skiaIngestion, skiaFrontend, skiaGitilesFollower, skiaPeriodicTasks},
+ []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeIngestion, chromeFrontend, chromeGitilesFollower, chromePeriodicTasks, skiaDiffCalculator, skiaIngestion, skiaFrontend, skiaGitilesFollower, skiaPeriodicTasks},
nil)
test("Multiple instances, all services, one canary",
[]string{"chrome", "skia"}, []string{"all"}, []string{"skia:frontend"},
- []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeIngestionBT, chromeIngestion, chromeFrontend, chromeGitilesFollower, chromePeriodicTasks, skiaDiffCalculator, skiaIngestionBT, skiaIngestion, skiaGitilesFollower, skiaPeriodicTasks},
+ []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeIngestion, chromeFrontend, chromeGitilesFollower, chromePeriodicTasks, skiaDiffCalculator, skiaIngestion, skiaGitilesFollower, skiaPeriodicTasks},
[]goldpushk.DeployableUnitID{skiaFrontend})
test("Multiple instances, all services, multiple canaries",
- []string{"chrome", "skia"}, []string{"all"}, []string{"skia:ingestion-bt", "skia:frontend"},
- []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeIngestionBT, chromeIngestion, chromeFrontend, chromeGitilesFollower, chromePeriodicTasks, skiaDiffCalculator, skiaIngestion, skiaGitilesFollower, skiaPeriodicTasks},
- []goldpushk.DeployableUnitID{skiaIngestionBT, skiaFrontend})
+ []string{"chrome", "skia"}, []string{"all"}, []string{"skia:ingestion", "skia:frontend"},
+ []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeIngestion, chromeFrontend, chromeGitilesFollower, chromePeriodicTasks, skiaDiffCalculator, skiaGitilesFollower, skiaPeriodicTasks},
+ []goldpushk.DeployableUnitID{skiaIngestion, skiaFrontend})
////////////////////////////////////////////////////////////////////////////////////////////////
// Wildcard: --instance all //
diff --git a/golden/go/ingestion_processors/BUILD.bazel b/golden/go/ingestion_processors/BUILD.bazel
index 51b90ac..a7c7d92 100644
--- a/golden/go/ingestion_processors/BUILD.bazel
+++ b/golden/go/ingestion_processors/BUILD.bazel
@@ -7,7 +7,6 @@
"buildbucketlookup.go",
"common.go",
"primarysql.go",
- "tracestore_impl.go",
"tryjob_ingestion.go",
],
importpath = "go.skia.org/infra/golden/go/ingestion_processors",
@@ -34,8 +33,6 @@
"//golden/go/jsonio",
"//golden/go/sql",
"//golden/go/sql/schema",
- "//golden/go/tracestore",
- "//golden/go/tracestore/bt_tracestore",
"//golden/go/types",
"@com_github_cockroachdb_cockroach_go_v2//crdb/crdbpgx",
"@com_github_hashicorp_golang_lru//:golang-lru",
@@ -54,8 +51,6 @@
"buildbucketlookup_manual_test.go",
"common_test.go",
"primarysql_test.go",
- "tracestore_impl_manual_test.go",
- "tracestore_impl_test.go",
"tryjob_ingestion_test.go",
],
data = glob(["testdata/**"]),
@@ -81,17 +76,14 @@
"//golden/go/ingestion",
"//golden/go/ingestion_processors/mocks",
"//golden/go/jsonio",
- "//golden/go/mocks",
"//golden/go/sql/databuilder",
"//golden/go/sql/datakitchensink",
"//golden/go/sql/schema",
"//golden/go/sql/sqltest",
- "//golden/go/tracestore",
"//golden/go/types",
"@com_github_hashicorp_golang_lru//:golang-lru",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//mock",
"@com_github_stretchr_testify//require",
- "@com_google_cloud_go_storage//:storage",
],
)
diff --git a/golden/go/ingestion_processors/primarysql.go b/golden/go/ingestion_processors/primarysql.go
index cfc706c..ab59574 100644
--- a/golden/go/ingestion_processors/primarysql.go
+++ b/golden/go/ingestion_processors/primarysql.go
@@ -3,6 +3,7 @@
import (
"context"
"crypto/md5"
+ "errors"
"strconv"
"time"
@@ -857,5 +858,44 @@
}
}
+// paramsAndOptions creates the params and options maps from a given file and entry.
+func paramsAndOptions(gr *jsonio.GoldResults, r jsonio.Result) (map[string]string, map[string]string) {
+ params := make(map[string]string, len(gr.Key)+len(r.Key))
+ for k, v := range gr.Key {
+ params[k] = v
+ }
+ for k, v := range r.Key {
+ params[k] = v
+ }
+ return params, r.Options
+}
+
+// shouldIngest returns a descriptive error if we should ignore an entry
+// with these params/options.
+func shouldIngest(params, options map[string]string) error {
+ // Ignore anything that is not a png. In the early days (pre-2015), ext was omitted
+ // but implied to be "png". Thus if ext is not provided, it will be ingested.
+ // New entries (created by goldctl) will always have ext set.
+ if ext, ok := options["ext"]; ok && (ext != "png") {
+ return errors.New("ignoring non-png entry")
+ }
+
+ // Make sure the test name meets basic requirements.
+ testName := params[types.PrimaryKeyField]
+
+ // Ignore results that don't have a test given and log an error since that
+ // should not happen. But we want to keep other results in the same input file.
+ if testName == "" {
+ return errors.New("missing test name")
+ }
+
+ // Make sure the test name does not exceed the allowed length.
+ if len(testName) > types.MaximumNameLength {
+ return skerr.Fmt("received test name which is longer than the allowed %d bytes: %s", types.MaximumNameLength, testName)
+ }
+
+ return nil
+}
+
// Make sure sqlPrimaryIngester implements the ingestion.Processor interface.
var _ ingestion.Processor = (*sqlPrimaryIngester)(nil)
diff --git a/golden/go/ingestion_processors/primarysql_test.go b/golden/go/ingestion_processors/primarysql_test.go
index c17406d..3bf8491 100644
--- a/golden/go/ingestion_processors/primarysql_test.go
+++ b/golden/go/ingestion_processors/primarysql_test.go
@@ -1,9 +1,13 @@
package ingestion_processors
import (
+ "bytes"
"context"
"crypto/md5"
"encoding/hex"
+ "io"
+ "io/ioutil"
+ "path/filepath"
"sync"
"testing"
"time"
@@ -14,6 +18,7 @@
"go.skia.org/infra/go/metrics2"
"go.skia.org/infra/go/now"
"go.skia.org/infra/go/paramtools"
+ "go.skia.org/infra/go/testutils"
"go.skia.org/infra/go/testutils/unittest"
"go.skia.org/infra/golden/go/sql/databuilder"
dks "go.skia.org/infra/golden/go/sql/datakitchensink"
@@ -1546,3 +1551,22 @@
tileID: tileID,
})
}
+
+type fakeGCSSource struct {
+ content []byte
+}
+
+func fakeGCSSourceFromFile(t *testing.T, file string) *fakeGCSSource {
+ fp := filepath.Join(testutils.TestDataDir(t), file)
+ b, err := ioutil.ReadFile(fp)
+ require.NoError(t, err)
+ return &fakeGCSSource{content: b}
+}
+
+func (f *fakeGCSSource) GetReader(_ context.Context, _ string) (io.ReadCloser, error) {
+ return ioutil.NopCloser(bytes.NewReader(f.content)), nil
+}
+
+func (f *fakeGCSSource) HandlesFile(_ string) bool {
+ return true
+}
diff --git a/golden/go/ingestion_processors/tracestore_impl.go b/golden/go/ingestion_processors/tracestore_impl.go
deleted file mode 100644
index 7ae4d8e..0000000
--- a/golden/go/ingestion_processors/tracestore_impl.go
+++ /dev/null
@@ -1,234 +0,0 @@
-package ingestion_processors
-
-import (
- "context"
- "errors"
- "io/ioutil"
- "regexp"
- "time"
-
- "go.opencensus.io/trace"
-
- "go.skia.org/infra/go/skerr"
- "go.skia.org/infra/go/sklog"
- "go.skia.org/infra/go/vcsinfo"
- "go.skia.org/infra/golden/go/ingestion"
- "go.skia.org/infra/golden/go/jsonio"
- "go.skia.org/infra/golden/go/tracestore"
- "go.skia.org/infra/golden/go/tracestore/bt_tracestore"
- "go.skia.org/infra/golden/go/types"
-)
-
-const (
- // BigTableTraceStore identifies a primary-branch ingester backed by BigTable.
- BigTableTraceStore = "big_table_tracestore"
-
- btProjectConfig = "BTProjectID"
- btInstanceConfig = "BTInstance"
- btTableConfig = "BTTable"
-)
-
-// PrimaryBranchBigTable creates a Processor that uses a BigTable-backed tracestore.
-func PrimaryBranchBigTable(ctx context.Context, src ingestion.Source, configParams map[string]string, vcs vcsinfo.VCS) (ingestion.Processor, error) {
- btc := bt_tracestore.BTConfig{
- ProjectID: configParams[btProjectConfig],
- InstanceID: configParams[btInstanceConfig],
- TableID: configParams[btTableConfig],
- VCS: vcs,
- }
-
- bts, err := bt_tracestore.New(ctx, btc, true)
- if err != nil {
- return nil, skerr.Fmt("could not instantiate BT tracestore: %s", err)
- }
- return &btProcessor{
- ts: bts,
- vcs: btc.VCS,
- source: src,
- }, nil
-}
-
-// btProcessor implements the ingestion.Processor interface for gold using
-// the BigTable TraceStore
-type btProcessor struct {
- ts tracestore.TraceStore
- vcs vcsinfo.VCS
- source ingestion.Source
-}
-
-// HandlesFile returns true if the configured source handles this file.
-func (b *btProcessor) HandlesFile(name string) bool {
- return b.source.HandlesFile(name)
-}
-
-// Process implements the ingestion.Processor interface.
-func (b *btProcessor) Process(ctx context.Context, fileName string) error {
- ctx, span := trace.StartSpan(ctx, "ingestion_BigTableProcess")
- defer span.End()
- r, err := b.source.GetReader(ctx, fileName)
- if err != nil {
- return skerr.Wrap(err)
- }
- gr, err := processGoldResults(ctx, r)
- if err != nil {
- return skerr.Wrapf(err, "could not process file %s from source %s", fileName, b.source)
- }
-
- if len(gr.Results) == 0 {
- sklog.Infof("file %s had no results", fileName)
- return nil
- }
- span.AddAttributes(trace.Int64Attribute("num_results", int64(len(gr.Results))))
-
- // If the target commit is not in the primary repository we look it up
- // in the secondary that has the primary as a dependency.
- var targetHash string
- if gr.CommitID != "" && gr.CommitMetadata != "" {
- targetHash, err = b.lookupChromeOSMetadata(ctx, gr.CommitMetadata)
- if err != nil {
- return skerr.Wrapf(err, "Looking up githash from Chromeos metadata: %q", gr.CommitMetadata)
- }
- } else {
- targetHash, err = getCanonicalCommitHash(ctx, b.vcs, gr.GitHash)
- if err != nil {
- return skerr.Wrapf(err, "could not identify canonical commit from %q", gr.GitHash)
- }
-
- if ok, err := b.isOnMaster(ctx, targetHash); err != nil {
- return skerr.Wrapf(err, "could not determine branch for %s", targetHash)
- } else if !ok {
- return skerr.Fmt("Commit %s is not in primary branch", targetHash)
- }
- }
-
- // Get the entries that should be added to the tracestore.
- entries, err := extractTraceStoreEntries(gr, fileName)
- if err != nil {
- // Probably invalid, don't retry
- return skerr.Wrapf(err, "could not create entries")
- }
-
- sklog.Debugf("Ingested %d entries to commit %s from file %s", len(entries), targetHash, fileName)
- // Write the result to the tracestore.
- err = b.ts.Put(ctx, targetHash, entries, time.Now())
- if err != nil {
- sklog.Errorf("Could not add entries to tracestore for file %s: %s", fileName, err)
- return ingestion.ErrRetryable
- }
- return nil
-}
-
-// isOnMaster returns true if the given commit hash is on the master branch.
-func (b *btProcessor) isOnMaster(ctx context.Context, hash string) (bool, error) {
- // BT_VCS is configured to only look at master, so if we just look up the index of the hash,
- // we will know if it is on the master branch.
- // We can ignore the error, because it would be a "commit not found" error.
- if i, _ := b.vcs.IndexOf(ctx, hash); i >= 0 {
- return true, nil
- }
-
- if err := b.vcs.Update(ctx, true /*=pull*/, false /*=all branches*/); err != nil {
- return false, skerr.Wrapf(err, "could not update VCS")
- }
- if i, _ := b.vcs.IndexOf(ctx, hash); i >= 0 {
- return true, nil
- }
- return false, nil
-}
-
-// extractTraceStoreEntries creates a slice of tracestore.Entry for the given
-// file. It will omit any entries that should be ignored. It returns an
-// error if there were no un-ignored entries in the file.
-func extractTraceStoreEntries(gr *jsonio.GoldResults, name string) ([]*tracestore.Entry, error) {
- ret := make([]*tracestore.Entry, 0, len(gr.Results))
- for _, result := range gr.Results {
- params, options := paramsAndOptions(gr, result)
- if err := shouldIngest(params, options); err != nil {
- sklog.Infof("Not ingesting %s : %s", name, err)
- continue
- }
-
- ret = append(ret, &tracestore.Entry{
- Params: params,
- Options: options,
- Digest: result.Digest,
- })
- }
-
- // If all results were ignored then we return an error.
- if len(ret) == 0 {
- return nil, skerr.Fmt("no valid results in file")
- }
-
- return ret, nil
-}
-
-// paramsAndOptions creates the params and options maps from a given file and entry.
-func paramsAndOptions(gr *jsonio.GoldResults, r jsonio.Result) (map[string]string, map[string]string) {
- params := make(map[string]string, len(gr.Key)+len(r.Key))
- for k, v := range gr.Key {
- params[k] = v
- }
- for k, v := range r.Key {
- params[k] = v
- }
- return params, r.Options
-}
-
-// shouldIngest returns a descriptive error if we should ignore an entry
-// with these params/options.
-func shouldIngest(params, options map[string]string) error {
- // Ignore anything that is not a png. In the early days (pre-2015), ext was omitted
- // but implied to be "png". Thus if ext is not provided, it will be ingested.
- // New entries (created by goldctl) will always have ext set.
- if ext, ok := options["ext"]; ok && (ext != "png") {
- return errors.New("ignoring non-png entry")
- }
-
- // Make sure the test name meets basic requirements.
- testName := params[types.PrimaryKeyField]
-
- // Ignore results that don't have a test given and log an error since that
- // should not happen. But we want to keep other results in the same input file.
- if testName == "" {
- return errors.New("missing test name")
- }
-
- // Make sure the test name does not exceed the allowed length.
- if len(testName) > types.MaximumNameLength {
- return skerr.Fmt("received test name which is longer than the allowed %d bytes: %s", types.MaximumNameLength, testName)
- }
-
- return nil
-}
-
-// Example input is gs://chromeos-image-archive/sentry-release/R92-13944.0.0/manifest.xml
-var metadataURLRegex = regexp.MustCompile(`gs://(?P<bucket>[a-z\-]+)/(?P<object>.+)`)
-
-// Instead of parsing XML, we use a simple regex to extract the one field we need.
-var revisionRegex = regexp.MustCompile(`name="chromiumos/platform/tast-tests" path="src/platform/tast-tests" revision="(?P<githash>[a-f0-9]+)"`)
-
-// lookupChromeOSMetadata extracts the git hash (of the tast-tests repo) from the provided
-// manifest.xml file. This manifest file contains the revisions of all ChromeOS repos at the
-// time of the build, but we only are tracking the tast-tests one.
-func (b *btProcessor) lookupChromeOSMetadata(ctx context.Context, gcsURL string) (string, error) {
- gcsClient := b.source.(*ingestion.GCSSource).Client
- matches := metadataURLRegex.FindStringSubmatch(gcsURL)
- if matches == nil {
- return "", skerr.Fmt("Invalid metadata string %q", gcsURL)
- }
- // match[1] is the bucket, match[2] is the object.
- r, err := gcsClient.Bucket(matches[1]).Object(matches[2]).NewReader(ctx)
- if err != nil {
- return "", skerr.Wrapf(err, "getting reader for %q", gcsURL)
- }
- xmlBytes, err := ioutil.ReadAll(r)
- if err != nil {
- return "", skerr.Wrapf(err, "reading from %q", gcsURL)
- }
- match := revisionRegex.FindStringSubmatch(string(xmlBytes))
- if match == nil {
- return "", skerr.Fmt("Could not find tast-tests revision in XML from %q", gcsURL)
- }
- return match[1], nil
-}
diff --git a/golden/go/ingestion_processors/tracestore_impl_manual_test.go b/golden/go/ingestion_processors/tracestore_impl_manual_test.go
deleted file mode 100644
index 3e6662c..0000000
--- a/golden/go/ingestion_processors/tracestore_impl_manual_test.go
+++ /dev/null
@@ -1,30 +0,0 @@
-package ingestion_processors
-
-import (
- "context"
- "testing"
-
- "cloud.google.com/go/storage"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- "go.skia.org/infra/go/testutils/unittest"
- "go.skia.org/infra/golden/go/ingestion"
-)
-
-func TestLookupChromeOSMetadata_Success(t *testing.T) {
- unittest.ManualTest(t)
-
- ctx := context.Background()
- client, err := storage.NewClient(ctx)
- require.NoError(t, err)
- b := &btProcessor{
- source: &ingestion.GCSSource{
- Client: client,
- },
- }
- const testURL = "gs://chromeos-image-archive/sentry-release/R92-13944.0.0/manifest.xml"
- gitHash, err := b.lookupChromeOSMetadata(ctx, testURL)
- require.NoError(t, err)
- // This is the commit that the tast-tests repo was at in the given URL.
- assert.Equal(t, "f8eb6fa2e313974fe9b66d665793ba15ebe520a7", gitHash)
-}
diff --git a/golden/go/ingestion_processors/tracestore_impl_test.go b/golden/go/ingestion_processors/tracestore_impl_test.go
deleted file mode 100644
index c5b633a..0000000
--- a/golden/go/ingestion_processors/tracestore_impl_test.go
+++ /dev/null
@@ -1,121 +0,0 @@
-package ingestion_processors
-
-import (
- "bytes"
- "context"
- "io"
- "io/ioutil"
- "path/filepath"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
- "github.com/stretchr/testify/require"
-
- "go.skia.org/infra/go/testutils"
- "go.skia.org/infra/go/testutils/unittest"
- "go.skia.org/infra/go/vcsinfo"
- mock_vcs "go.skia.org/infra/go/vcsinfo/mocks"
- "go.skia.org/infra/golden/go/mocks"
- "go.skia.org/infra/golden/go/tracestore"
- "go.skia.org/infra/golden/go/types"
-)
-
-// TestTraceStoreProcessorSunnyDay tests that the ingestion makes the right calls
-// to the VCS and the Tracestore for the sample file in testdata/.
-func TestTraceStoreProcessorSunnyDay(t *testing.T) {
- unittest.MediumTest(t)
-
- mts := &mocks.TraceStore{}
- mvs := &mock_vcs.VCS{}
- defer mts.AssertExpectations(t)
- defer mvs.AssertExpectations(t)
-
- commit := &vcsinfo.LongCommit{
- ShortCommit: &vcsinfo.ShortCommit{
- Hash: testCommitHash,
- Subject: "Really big code change",
- },
- // arbitrary time
- Timestamp: time.Date(2019, time.June, 16, 3, 23, 17, 0, time.UTC),
- }
- mvs.On("Details", testutils.AnyContext, testCommitHash, false).Return(commit, nil)
- // arbitrary result
- mvs.On("IndexOf", testutils.AnyContext, testCommitHash).Return(12, nil)
-
- // There are 3 entries in the file, but one of them is pdf, which should
- // be ignored by this ingester.
- expectedEntries := []*tracestore.Entry{
- {
- Params: map[string]string{
- "arch": "x86_64",
- "compiler": "MSVC",
- "config": "pipe-8888",
- "configuration": "Debug",
- "cpu_or_gpu": "CPU",
- "cpu_or_gpu_value": "AVX2",
- "model": "ShuttleB",
- types.PrimaryKeyField: "aaclip",
- "os": "Win8",
- types.CorpusField: "gm",
- },
- Options: map[string]string{
- "ext": "png",
- },
- Digest: "fa3c371d201d6f88f7a47b41862e2e85",
- },
- {
- Params: map[string]string{
- "arch": "x86_64",
- "compiler": "MSVC",
- "config": "pipe-8888",
- "configuration": "Debug",
- "cpu_or_gpu": "CPU",
- "cpu_or_gpu_value": "AVX2",
- "model": "ShuttleB",
- types.PrimaryKeyField: "clipcubic",
- "os": "Win8",
- types.CorpusField: "gm",
- },
- Options: map[string]string{
- "ext": "png",
- },
- Digest: "64e446d96bebba035887dd7dda6db6c4",
- },
- }
-
- mts.On("Put", testutils.AnyContext, testCommitHash, expectedEntries, mock.AnythingOfType("time.Time")).Return(nil)
-
- src := fakeGCSSourceFromFile(t, "dm.json")
- p := &btProcessor{
- ts: mts,
- vcs: mvs,
- source: src,
- }
- err := p.Process(context.Background(), "whatever")
- assert.NoError(t, err)
-}
-
-const (
- testCommitHash = "02cb37309c01506e2552e931efa9c04a569ed266"
-)
-
-type fakeGCSSource struct {
- content []byte
-}
-
-func fakeGCSSourceFromFile(t *testing.T, file string) *fakeGCSSource {
- fp := filepath.Join(testutils.TestDataDir(t), file)
- b, err := ioutil.ReadFile(fp)
- require.NoError(t, err)
- return &fakeGCSSource{content: b}
-}
-
-func (f *fakeGCSSource) GetReader(_ context.Context, _ string) (io.ReadCloser, error) {
- return ioutil.NopCloser(bytes.NewReader(f.content)), nil
-}
-
-func (f *fakeGCSSource) HandlesFile(_ string) bool {
- return true
-}
diff --git a/golden/k8s-instances/angle/angle-ingestion-bt.json5 b/golden/k8s-instances/angle/angle-ingestion-bt.json5
deleted file mode 100644
index d532af1..0000000
--- a/golden/k8s-instances/angle/angle-ingestion-bt.json5
+++ /dev/null
@@ -1,38 +0,0 @@
-{
- backup_poll_interval: "1h",
- backup_poll_scope: "2h",
- ingestion_files_topic: "gold-angle-eventbus-bt",
- ingestion_subscription: "gold-ingestion+gold-angle-eventbus-bt",
- prom_port: ":20000",
- ready_port: ":9091",
-
- primary_branch_config: {
- type: "big_table_tracestore",
- gcs_source: {
- bucket: "skia-gold-angle",
- prefix: "dm-json-v1"
- },
- extra_configuration: {
- BTProjectID: "skia-public",
- BTInstance: "production",
- BTTable: "gold-angle"
- }
- },
- secondary_branch_config: {
- type: "sql_secondary",
- gcs_source: {
- bucket: "skia-gold-angle",
- prefix: "trybot/dm-json-v1"
- },
- extra_configuration: {
- CodeReviewSystems: "gerrit",
- GerritURL: "https://chromium-review.googlesource.com",
- ContinuousIntegrationSystems: "buildbucket",
- },
- },
-
- // These values affect the k8s deployment; they are not read in by the binary.
- K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
- K8S_NODE_SELECTOR_VALUE1: "n1-standard-16",
- K8S_NODE_SELECTOR_VALUE2: "e2-standard-16"
-}
diff --git a/golden/k8s-instances/angle/angle-ingestion.json5 b/golden/k8s-instances/angle/angle-ingestion.json5
index bf58a30..8b54c5c 100644
--- a/golden/k8s-instances/angle/angle-ingestion.json5
+++ b/golden/k8s-instances/angle/angle-ingestion.json5
@@ -16,19 +16,18 @@
TileWidth: "100"
}
},
-// TODO(kjlubick) enable the below once ingestion-bt is removed
-// secondary_branch_config: {
-// type: "sql_secondary",
-// gcs_source: {
-// bucket: "skia-gold-angle",
-// prefix: "trybot/dm-json-v1"
-// },
-// extra_configuration: {
-// CodeReviewSystems: "gerrit",
-// GerritURL: "https://chromium-review.googlesource.com",
-// ContinuousIntegrationSystems: "buildbucket",
-// },
-// },
+ secondary_branch_config: {
+ type: "sql_secondary",
+ gcs_source: {
+ bucket: "skia-gold-angle",
+ prefix: "trybot/dm-json-v1"
+ },
+ extra_configuration: {
+ CodeReviewSystems: "gerrit",
+ GerritURL: "https://chromium-review.googlesource.com",
+ ContinuousIntegrationSystems: "buildbucket",
+ },
+ },
// These values affect the k8s deployment; they are not read in by the binary.
K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
diff --git a/golden/k8s-instances/chrome/chrome-ingestion-bt.json5 b/golden/k8s-instances/chrome/chrome-ingestion-bt.json5
deleted file mode 100644
index d655457..0000000
--- a/golden/k8s-instances/chrome/chrome-ingestion-bt.json5
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- backup_poll_interval: "1h",
- backup_poll_scope: "4h",
- ingestion_files_topic: "gold-chrome-eventbus-bt",
- ingestion_subscription: "gold-ingestion+gold-chrome-eventbus-bt",
- files_processed_in_parallel: 20, // chrome has lots of small files.
- prom_port: ":20000",
- pubsub_fetch_size: 100,
- ready_port: ":9091",
-
- primary_branch_config: {
- type: "big_table_tracestore",
- gcs_source: {
- bucket: "skia-gold-chrome",
- prefix: "dm-json-v1"
- },
- extra_configuration: {
- BTProjectID: "skia-public",
- BTInstance: "production",
- BTTable: "gold-chrome"
- }
- },
- secondary_branch_config: {
- type: "sql_secondary",
- gcs_source: {
- bucket: "skia-gold-chrome",
- prefix: "trybot/dm-json-v1"
- },
- extra_configuration: {
- // TODO(kjlubick) this is clunky; it would be preferable to use CodeReviewSystems
- CodeReviewSystems: "gerrit,gerrit-internal",
- GerritURL: "https://chromium-review.googlesource.com",
- GerritInternalURL: "https://chrome-internal-review.googlesource.com",
-
- ContinuousIntegrationSystems: "buildbucket,buildbucket-internal",
- }
- },
-
- // These values affect the k8s deployment; they are not read in by the binary.
- K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
- K8S_NODE_SELECTOR_VALUE1: "n1-standard-16",
- K8S_NODE_SELECTOR_VALUE2: "e2-standard-16"
-}
diff --git a/golden/k8s-instances/chrome/chrome-ingestion.json5 b/golden/k8s-instances/chrome/chrome-ingestion.json5
index 843bf85..be0fe2f 100644
--- a/golden/k8s-instances/chrome/chrome-ingestion.json5
+++ b/golden/k8s-instances/chrome/chrome-ingestion.json5
@@ -18,22 +18,21 @@
TileWidth: "100"
}
},
-// TODO(kjlubick) enable the below once ingestion-bt is removed
-// secondary_branch_config: {
-// type: "sql_secondary",
-// gcs_source: {
-// bucket: "skia-gold-chrome",
-// prefix: "trybot/dm-json-v1"
-// },
-// extra_configuration: {
-// // TODO(kjlubick) this is clunky; it would be preferable to use CodeReviewSystems
-// CodeReviewSystems: "gerrit,gerrit-internal",
-// GerritURL: "https://chromium-review.googlesource.com",
-// GerritInternalURL: "https://chrome-internal-review.googlesource.com",
-//
-// ContinuousIntegrationSystems: "buildbucket",
-// }
-// },
+ secondary_branch_config: {
+ type: "sql_secondary",
+ gcs_source: {
+ bucket: "skia-gold-chrome",
+ prefix: "trybot/dm-json-v1"
+ },
+ extra_configuration: {
+ // TODO(kjlubick) this is clunky; it would be preferable to use CodeReviewSystems
+ CodeReviewSystems: "gerrit,gerrit-internal",
+ GerritURL: "https://chromium-review.googlesource.com",
+ GerritInternalURL: "https://chrome-internal-review.googlesource.com",
+
+ ContinuousIntegrationSystems: "buildbucket",
+ }
+ },
// These values affect the k8s deployment; they are not read in by the binary.
K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
diff --git a/golden/k8s-instances/cros-tast-dev/cros-tast-dev-ingestion-bt.json5 b/golden/k8s-instances/cros-tast-dev/cros-tast-dev-ingestion-bt.json5
deleted file mode 100644
index 3d9b7ce..0000000
--- a/golden/k8s-instances/cros-tast-dev/cros-tast-dev-ingestion-bt.json5
+++ /dev/null
@@ -1,41 +0,0 @@
-{
- backup_poll_interval: "1h",
- backup_poll_scope: "2h",
- ingestion_files_topic: "gold-cros-tast-dev-eventbus-bt",
- ingestion_subscription: "gold-ingestion+gold-cros-tast-dev-eventbus-bt",
- prom_port: ":20000",
- ready_port: ":9091",
-
- primary_branch_config: {
- type: "big_table_tracestore",
- gcs_source: {
- bucket: "skia-gold-cros-tast-dev",
- prefix: "dm-json-v1"
- },
- extra_configuration: {
- BTProjectID: "skia-public",
- BTInstance: "production",
- BTTable: "gold-cros-tast-dev"
- }
- },
- secondary_branch_config: {
- type: "sql_secondary",
- gcs_source: {
- bucket: "skia-gold-cros-tast-dev",
- prefix: "trybot/dm-json-v1"
- },
- extra_configuration: {
- CodeReviewSystems: "gerrit",
- GerritURL: "https://chromium-review.googlesource.com",
-
- ContinuousIntegrationSystems: "buildbucket",
-
- LookupCLsIn: "buildbucket",
- }
- },
-
- // These values affect the k8s deployment; they are not read in by the binary.
- K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
- K8S_NODE_SELECTOR_VALUE1: "n1-standard-16",
- K8S_NODE_SELECTOR_VALUE2: "e2-standard-16"
-}
diff --git a/golden/k8s-instances/cros-tast-dev/cros-tast-dev-ingestion.json5 b/golden/k8s-instances/cros-tast-dev/cros-tast-dev-ingestion.json5
index 7ef882c..fd694aa 100644
--- a/golden/k8s-instances/cros-tast-dev/cros-tast-dev-ingestion.json5
+++ b/golden/k8s-instances/cros-tast-dev/cros-tast-dev-ingestion.json5
@@ -16,20 +16,19 @@
TileWidth: "100"
}
},
-// TODO(kjlubick) enable the below once ingestion-bt is removed
-// secondary_branch_config: {
-// type: "sql_secondary",
-// gcs_source: {
-// bucket: "skia-gold-cros-tast-dev",
-// prefix: "trybot/dm-json-v1"
-// },
-// extra_configuration: {
-// CodeReviewSystems: "gerrit",
-// GerritURL: "https://chromium-review.googlesource.com",
-//
-// ContinuousIntegrationSystems: "buildbucket",
-// }
-// },
+ secondary_branch_config: {
+ type: "sql_secondary",
+ gcs_source: {
+ bucket: "skia-gold-cros-tast-dev",
+ prefix: "trybot/dm-json-v1"
+ },
+ extra_configuration: {
+ CodeReviewSystems: "gerrit",
+ GerritURL: "https://chromium-review.googlesource.com",
+
+ ContinuousIntegrationSystems: "buildbucket",
+ }
+ },
// These values affect the k8s deployment; they are not read in by the binary.
K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
diff --git a/golden/k8s-instances/flutter-engine/flutter-engine-ingestion-bt.json5 b/golden/k8s-instances/flutter-engine/flutter-engine-ingestion-bt.json5
deleted file mode 100644
index 2f27690..0000000
--- a/golden/k8s-instances/flutter-engine/flutter-engine-ingestion-bt.json5
+++ /dev/null
@@ -1,40 +0,0 @@
-{
- backup_poll_interval: "1h",
- backup_poll_scope: "2h",
- ingestion_files_topic: "gold-flutter-engine-eventbus-bt",
- ingestion_subscription: "gold-ingestion+gold-flutter-engine-eventbus-bt",
- prom_port: ":20000",
- ready_port: ":9091",
-
- primary_branch_config: {
- type: "big_table_tracestore",
- gcs_source: {
- bucket: "skia-gold-flutter-engine",
- prefix: "dm-json-v1"
- },
- extra_configuration: {
- BTProjectID: "skia-public",
- BTInstance: "production",
- BTTable: "gold-flutter-engine"
- }
- },
- secondary_branch_config: {
- type: "sql_secondary",
- gcs_source: {
- bucket: "skia-gold-flutter-engine",
- prefix: "trybot/dm-json-v1"
- },
- extra_configuration: {
- CodeReviewSystems: "github",
- GitHubCredentialsPath: "/var/secrets/github/github_token",
- GitHubRepo: "flutter/engine",
-
- ContinuousIntegrationSystems: "cirrus,buildbucket",
- }
- },
-
- // These values affect the k8s deployment; they are not read in by the binary.
- K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
- K8S_NODE_SELECTOR_VALUE1: "n1-standard-16",
- K8S_NODE_SELECTOR_VALUE2: "e2-standard-16"
-}
diff --git a/golden/k8s-instances/flutter-engine/flutter-engine-ingestion.json5 b/golden/k8s-instances/flutter-engine/flutter-engine-ingestion.json5
index 7fb527e..31b1054 100644
--- a/golden/k8s-instances/flutter-engine/flutter-engine-ingestion.json5
+++ b/golden/k8s-instances/flutter-engine/flutter-engine-ingestion.json5
@@ -16,21 +16,20 @@
TileWidth: "100"
}
},
-// TODO(kjlubick) enable the below once ingestion-bt is removed
-// secondary_branch_config: {
-// type: "sql_secondary",
-// gcs_source: {
-// bucket: "skia-gold-flutter-engine",
-// prefix: "trybot/dm-json-v1"
-// },
-// extra_configuration: {
-// CodeReviewSystems: "github",
-// GitHubCredentialsPath: "/var/secrets/github/github_token",
-// GitHubRepo: "flutter/engine",
-//
-// ContinuousIntegrationSystems: "cirrus,buildbucket",
-// }
-// },
+ secondary_branch_config: {
+ type: "sql_secondary",
+ gcs_source: {
+ bucket: "skia-gold-flutter-engine",
+ prefix: "trybot/dm-json-v1"
+ },
+ extra_configuration: {
+ CodeReviewSystems: "github",
+ GitHubCredentialsPath: "/var/secrets/github/github_token",
+ GitHubRepo: "flutter/engine",
+
+ ContinuousIntegrationSystems: "cirrus,buildbucket",
+ }
+ },
// These values affect the k8s deployment; they are not read in by the binary.
K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
diff --git a/golden/k8s-instances/flutter/flutter-ingestion-bt.json5 b/golden/k8s-instances/flutter/flutter-ingestion-bt.json5
deleted file mode 100644
index 1ae8a00..0000000
--- a/golden/k8s-instances/flutter/flutter-ingestion-bt.json5
+++ /dev/null
@@ -1,40 +0,0 @@
-{
- backup_poll_interval: "1h",
- backup_poll_scope: "2h",
- ingestion_files_topic: "gold-flutter-eventbus-bt",
- ingestion_subscription: "gold-ingestion+gold-flutter-eventbus-bt",
- prom_port: ":20000",
- ready_port: ":9091",
-
- primary_branch_config: {
- type: "big_table_tracestore",
- gcs_source: {
- bucket: "skia-gold-flutter",
- prefix: "dm-json-v1"
- },
- extra_configuration: {
- BTProjectID: "skia-public",
- BTInstance: "production",
- BTTable: "gold-flutter"
- }
- },
- secondary_branch_config: {
- type: "sql_secondary",
- gcs_source: {
- bucket: "skia-gold-flutter",
- prefix: "trybot/dm-json-v1"
- },
- extra_configuration: {
- CodeReviewSystems: "github",
- GitHubCredentialsPath: "/var/secrets/github/github_token",
- GitHubRepo: "flutter/flutter",
-
- ContinuousIntegrationSystems: "cirrus,buildbucket",
- }
- },
-
- // These values affect the k8s deployment; they are not read in by the binary.
- K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
- K8S_NODE_SELECTOR_VALUE1: "n1-standard-16",
- K8S_NODE_SELECTOR_VALUE2: "e2-standard-16"
-}
diff --git a/golden/k8s-instances/flutter/flutter-ingestion.json5 b/golden/k8s-instances/flutter/flutter-ingestion.json5
index da72762..45e66bf 100644
--- a/golden/k8s-instances/flutter/flutter-ingestion.json5
+++ b/golden/k8s-instances/flutter/flutter-ingestion.json5
@@ -16,21 +16,21 @@
TileWidth: "100"
}
},
-// TODO(kjlubick) enable the below once ingestion-bt is removed
-// secondary_branch_config: {
-// type: "sql_secondary",
-// gcs_source: {
-// bucket: "skia-gold-flutter",
-// prefix: "trybot/dm-json-v1"
-// },
-// extra_configuration: {
-// CodeReviewSystems: "github",
-// GitHubCredentialsPath: "/var/secrets/github/github_token",
-// GitHubRepo: "flutter/flutter",
-//
-// ContinuousIntegrationSystems: "cirrus,buildbucket",
-// }
-// },
+
+ secondary_branch_config: {
+ type: "sql_secondary",
+ gcs_source: {
+ bucket: "skia-gold-flutter",
+ prefix: "trybot/dm-json-v1"
+ },
+ extra_configuration: {
+ CodeReviewSystems: "github",
+ GitHubCredentialsPath: "/var/secrets/github/github_token",
+ GitHubRepo: "flutter/flutter",
+
+ ContinuousIntegrationSystems: "cirrus,buildbucket",
+ }
+ },
// These values affect the k8s deployment; they are not read in by the binary.
K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
diff --git a/golden/k8s-instances/lottie/lottie-ingestion-bt.json5 b/golden/k8s-instances/lottie/lottie-ingestion-bt.json5
deleted file mode 100644
index 0bbacdf..0000000
--- a/golden/k8s-instances/lottie/lottie-ingestion-bt.json5
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- backup_poll_interval: "1h",
- backup_poll_scope: "2h",
- ingestion_files_topic: "gold-lottie-eventbus-bt",
- ingestion_subscription: "gold-ingestion+gold-lottie-eventbus-bt",
- prom_port: ":20000",
- ready_port: ":9091",
-
- primary_branch_config: {
- type: "big_table_tracestore",
- gcs_source: {
- bucket: "skia-gold-lottie",
- prefix: "dm-json-v1"
- },
- extra_configuration: {
- BTProjectID: "skia-public",
- BTInstance: "production",
- BTTable: "gold-lottie"
- }
- },
-
- // These values affect the k8s deployment; they are not read in by the binary.
- K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
- K8S_NODE_SELECTOR_VALUE1: "n1-standard-16",
- K8S_NODE_SELECTOR_VALUE2: "e2-standard-16"
-}
diff --git a/golden/k8s-instances/pdfium/pdfium-ingestion-bt.json5 b/golden/k8s-instances/pdfium/pdfium-ingestion-bt.json5
deleted file mode 100644
index 7b30888..0000000
--- a/golden/k8s-instances/pdfium/pdfium-ingestion-bt.json5
+++ /dev/null
@@ -1,39 +0,0 @@
-{
- backup_poll_interval: "1h",
- backup_poll_scope: "2h",
- ingestion_files_topic: "gold-pdfium-eventbus-bt",
- ingestion_subscription: "gold-ingestion+gold-pdfium-eventbus-bt",
- prom_port: ":20000",
- ready_port: ":9091",
-
- primary_branch_config: {
- type: "big_table_tracestore",
- gcs_source: {
- bucket: "skia-pdfium-gm",
- prefix: "dm-json-v1"
- },
- extra_configuration: {
- BTProjectID: "skia-public",
- BTInstance: "production",
- BTTable: "gold-pdfium"
- }
- },
- secondary_branch_config: {
- type: "sql_secondary",
- gcs_source: {
- bucket: "skia-pdfium-gm",
- prefix: "trybot/dm-json-v1"
- },
- extra_configuration: {
- CodeReviewSystems: "gerrit",
- GerritURL: "https://pdfium-review.googlesource.com",
-
- ContinuousIntegrationSystems: "buildbucket",
- }
- },
-
- // These values affect the k8s deployment; they are not read in by the binary.
- K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
- K8S_NODE_SELECTOR_VALUE1: "n1-standard-16",
- K8S_NODE_SELECTOR_VALUE2: "e2-standard-16"
-}
diff --git a/golden/k8s-instances/pdfium/pdfium-ingestion.json5 b/golden/k8s-instances/pdfium/pdfium-ingestion.json5
index cde3d27..728ae55 100644
--- a/golden/k8s-instances/pdfium/pdfium-ingestion.json5
+++ b/golden/k8s-instances/pdfium/pdfium-ingestion.json5
@@ -16,20 +16,19 @@
TileWidth: "100"
}
},
- // TODO(kjlubick) enable the below once ingestion-bt is removed
-// secondary_branch_config: {
-// type: "sql_secondary",
-// gcs_source: {
-// bucket: "skia-pdfium-gm",
-// prefix: "trybot/dm-json-v1"
-// },
-// extra_configuration: {
-// CodeReviewSystems: "gerrit",
-// GerritURL: "https://pdfium-review.googlesource.com",
-//
-// ContinuousIntegrationSystems: "buildbucket",
-// }
-// },
+ secondary_branch_config: {
+ type: "sql_secondary",
+ gcs_source: {
+ bucket: "skia-pdfium-gm",
+ prefix: "trybot/dm-json-v1"
+ },
+ extra_configuration: {
+ CodeReviewSystems: "gerrit",
+ GerritURL: "https://pdfium-review.googlesource.com",
+
+ ContinuousIntegrationSystems: "buildbucket",
+ }
+ },
// These values affect the k8s deployment; they are not read in by the binary.
K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
diff --git a/golden/k8s-instances/skia-infra/skia-infra-ingestion-bt.json5 b/golden/k8s-instances/skia-infra/skia-infra-ingestion-bt.json5
deleted file mode 100644
index 80b9074..0000000
--- a/golden/k8s-instances/skia-infra/skia-infra-ingestion-bt.json5
+++ /dev/null
@@ -1,39 +0,0 @@
-{
- backup_poll_interval: "1h",
- backup_poll_scope: "2h",
- ingestion_files_topic: "gold-skia-infra-eventbus-bt",
- ingestion_subscription: "gold-ingestion+gold-skia-infra-eventbus-bt",
- prom_port: ":20000",
- ready_port: ":9091",
-
- primary_branch_config: {
- type: "big_table_tracestore",
- gcs_source: {
- bucket: "skia-gold-skia-infra",
- prefix: "dm-json-v1"
- },
- extra_configuration: {
- BTProjectID: "skia-public",
- BTInstance: "production",
- BTTable: "gold-skia-infra"
- }
- },
- secondary_branch_config: {
- type: "sql_secondary",
- gcs_source: {
- bucket: "skia-gold-skia-infra",
- prefix: "trybot/dm-json-v1"
- },
- extra_configuration: {
- CodeReviewSystems: "gerrit",
- GerritURL: "https://skia-review.googlesource.com",
-
- ContinuousIntegrationSystems: "buildbucket",
- }
- },
-
- // These values affect the k8s deployment; they are not read in by the binary.
- K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
- K8S_NODE_SELECTOR_VALUE1: "n1-standard-16",
- K8S_NODE_SELECTOR_VALUE2: "e2-standard-16"
-}
diff --git a/golden/k8s-instances/skia-infra/skia-infra-ingestion.json5 b/golden/k8s-instances/skia-infra/skia-infra-ingestion.json5
index 125dc1b..7ab727c 100644
--- a/golden/k8s-instances/skia-infra/skia-infra-ingestion.json5
+++ b/golden/k8s-instances/skia-infra/skia-infra-ingestion.json5
@@ -16,20 +16,20 @@
TileWidth: "100",
}
},
-// TODO(kjlubick) enable the below once ingestion-bt is removed
-// secondary_branch_config: {
-// type: "sql_secondary",
-// gcs_source: {
-// bucket: "skia-gold-skia-infra",
-// prefix: "trybot/dm-json-v1"
-// },
-// extra_configuration: {
-// CodeReviewSystems: "gerrit",
-// GerritURL: "https://skia-review.googlesource.com",
-//
-// ContinuousIntegrationSystems: "buildbucket",
-// }
-// },
+
+ secondary_branch_config: {
+ type: "sql_secondary",
+ gcs_source: {
+ bucket: "skia-gold-skia-infra",
+ prefix: "trybot/dm-json-v1"
+ },
+ extra_configuration: {
+ CodeReviewSystems: "gerrit",
+ GerritURL: "https://skia-review.googlesource.com",
+
+ ContinuousIntegrationSystems: "buildbucket",
+ }
+ },
// These values affect the k8s deployment; they are not read in by the binary.
K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
diff --git a/golden/k8s-instances/skia/skia-ingestion-bt.json5 b/golden/k8s-instances/skia/skia-ingestion-bt.json5
deleted file mode 100644
index 093dbc1..0000000
--- a/golden/k8s-instances/skia/skia-ingestion-bt.json5
+++ /dev/null
@@ -1,39 +0,0 @@
-{
- backup_poll_interval: "1h",
- backup_poll_scope: "2h",
- ingestion_files_topic: "gold-skia-eventbus-bt",
- ingestion_subscription: "gold-ingestion+gold-skia-eventbus-bt",
- prom_port: ":20000",
- ready_port: ":9091",
-
- primary_branch_config: {
- type: "big_table_tracestore",
- gcs_source: {
- bucket: "skia-infra-gm",
- prefix: "dm-json-v1"
- },
- extra_configuration: {
- BTProjectID: "skia-public",
- BTInstance: "production",
- BTTable: "gold-skia"
- }
- },
- secondary_branch_config: {
- type: "sql_secondary",
- gcs_source: {
- bucket: "skia-infra-gm",
- prefix: "trybot/dm-json-v1"
- },
- extra_configuration: {
- CodeReviewSystems: "gerrit",
- GerritURL: "https://skia-review.googlesource.com",
-
- ContinuousIntegrationSystems: "buildbucket",
- }
- },
-
- // These values affect the k8s deployment; they are not read in by the binary.
- K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
- K8S_NODE_SELECTOR_VALUE1: "n1-standard-16",
- K8S_NODE_SELECTOR_VALUE2: "e2-standard-16"
-}
diff --git a/golden/k8s-instances/skia/skia-ingestion.json5 b/golden/k8s-instances/skia/skia-ingestion.json5
index 99bbaeb..95d8fb8 100644
--- a/golden/k8s-instances/skia/skia-ingestion.json5
+++ b/golden/k8s-instances/skia/skia-ingestion.json5
@@ -16,20 +16,19 @@
TileWidth: "100"
}
},
-// TODO(kjlubick) enable the below once ingestion-bt is removed
-// secondary_branch_config: {
-// type: "sql_secondary",
-// gcs_source: {
-// bucket: "skia-infra-gm",
-// prefix: "trybot/dm-json-v1"
-// },
-// extra_configuration: {
-// CodeReviewSystems: "gerrit",
-// GerritURL: "https://skia-review.googlesource.com",
-//
-// ContinuousIntegrationSystems: "buildbucket",
-// }
-// },
+ secondary_branch_config: {
+ type: "sql_secondary",
+ gcs_source: {
+ bucket: "skia-infra-gm",
+ prefix: "trybot/dm-json-v1"
+ },
+ extra_configuration: {
+ CodeReviewSystems: "gerrit",
+ GerritURL: "https://skia-review.googlesource.com",
+
+ ContinuousIntegrationSystems: "buildbucket",
+ }
+ },
// These values affect the k8s deployment; they are not read in by the binary.
K8S_NODE_SELECTOR_KEY: "beta.kubernetes.io/instance-type",
