[perf] Have all instances of perf use CockroachDB to back perfgit.
I discovered that falling back to SQLite isn't really an option because
the Android repos have over a million commits and startup takes close
to an hour.
Change-Id: Ie1f984c010d627d020379d3a9554bf49923c2aef
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/282260
Reviewed-by: Kevin Lubick <kjlubick@google.com>
Commit-Queue: Joe Gregorio <jcgregorio@google.com>
diff --git a/perf/configs/android-prod.json b/perf/configs/android-prod.json
index 2bebabb..dbc0099 100644
--- a/perf/configs/android-prod.json
+++ b/perf/configs/android-prod.json
@@ -2,7 +2,7 @@
"URL": "https://android-master-perf.skia.org",
"data_store_config": {
"datastore_type": "gcp",
- "connection_string": "/tmp/git-sqlite.db",
+ "connection_string": "postgresql://root@perf-android-cockroachdb-public:26257/android?sslmode=disable",
"tile_size": 8192,
"project": "skia-public",
"instance": "production",
diff --git a/perf/configs/android-x.json b/perf/configs/android-x.json
index 8c26ac9..2f5d1e8 100644
--- a/perf/configs/android-x.json
+++ b/perf/configs/android-x.json
@@ -2,7 +2,7 @@
"URL": "https://androidx-perf.skia.org/",
"data_store_config": {
"datastore_type": "gcp",
- "connection_string": "/tmp/git-sqlite.db",
+ "connection_string": "postgresql://root@perf-androidx-cockroachdb-public:26257/androidx?sslmode=disable",
"tile_size": 512,
"project": "skia-public",
"instance": "production",
diff --git a/perf/configs/ct-prod.json b/perf/configs/ct-prod.json
index ae01781..ddeefd5 100644
--- a/perf/configs/ct-prod.json
+++ b/perf/configs/ct-prod.json
@@ -2,7 +2,7 @@
"URL": "https://ct-perf.skia.org",
"data_store_config": {
"datastore_type": "gcp",
- "connection_string": "/tmp/git-sqlite.db",
+ "connection_string": "postgresql://root@perf-ct-cockroachdb-public:26257/ct?sslmode=disable",
"tile_size": 256,
"project": "skia-public",
"instance": "production",
diff --git a/perf/configs/flutter.json b/perf/configs/flutter.json
index 9ce5751..d3f4b54 100644
--- a/perf/configs/flutter.json
+++ b/perf/configs/flutter.json
@@ -2,7 +2,7 @@
"URL": "https://flutter-perf.skia.org/",
"data_store_config": {
"datastore_type": "gcp",
- "connection_string": "/tmp/git-sqlite.db",
+ "connection_string": "postgresql://root@perf-flutter-cockroachdb-public:26257/flutter?sslmode=disable",
"tile_size": 256,
"project": "skia-public",
"instance": "production",
diff --git a/perf/configs/nano.json b/perf/configs/nano.json
index 0eb9562..35383f1 100644
--- a/perf/configs/nano.json
+++ b/perf/configs/nano.json
@@ -2,7 +2,7 @@
"URL": "https://perf.skia.org",
"data_store_config": {
"datastore_type": "gcp",
- "connection_string": "/tmp/git-sqlite.db",
+ "connection_string": "postgresql://root@perf-skia-cockroachdb-public:26257/skia?sslmode=disable",
"tile_size": 256,
"project": "skia-public",
"instance": "production",
diff --git a/perf/go/builders/builders.go b/perf/go/builders/builders.go
index 11e6464..d3b6c42 100644
--- a/perf/go/builders/builders.go
+++ b/perf/go/builders/builders.go
@@ -102,9 +102,17 @@
var dialect perfsql.Dialect
switch instanceConfig.DataStoreConfig.DataStoreType {
case config.GCPDataStoreType:
- // Even for BigTable backed datastores we still stand up an sqlite
- // instance to hold the perfgit info.
- dialect = perfsql.SQLiteDialect
+ if strings.HasPrefix(instanceConfig.DataStoreConfig.ConnectionString, "postgresql://") {
+ // This is a temporary path as we migrate away from BigTable to
+ // CockroachDB. The first small step in that migration is to host the
+ // perfgit Commits table on CockroachDB, which has no analog in the
+ // "gcs" world.
+ dialect = perfsql.CockroachDBDialect
+ } else {
+ // Even for BigTable backed datastores we still stand up an sqlite
+ // instance to hold the perfgit info.
+ dialect = perfsql.SQLiteDialect
+ }
case config.CockroachDBDataStoreType:
dialect = perfsql.CockroachDBDialect
case config.SQLite3DataStoreType:
diff --git a/perf/go/builders/builders_test.go b/perf/go/builders/builders_test.go
index b998e70..cb1dad4 100644
--- a/perf/go/builders/builders_test.go
+++ b/perf/go/builders/builders_test.go
@@ -317,6 +317,22 @@
assert.Equal(t, hashes[2], gitHash)
}
+func TestNewPerfGitFromConfig_GCP_SuccessIfConnectionStringIsCockroachDB(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, _, _, hashes, _, instanceConfig, cleanup := gittest.NewForTest(t, perfsql.CockroachDBDialect)
+ defer cleanup()
+
+ instanceConfig.DataStoreConfig.DataStoreType = config.GCPDataStoreType
+ instanceConfig.DataStoreConfig.ConnectionString = fmt.Sprintf("postgresql://root@%s/%s?sslmode=disable", perfsql.GetCockroachDBEmulatorHost(), gittest.CockroachDatabaseName)
+
+ g, err := NewPerfGitFromConfig(ctx, false, instanceConfig)
+ require.NoError(t, err)
+
+ gitHash, err := g.GitHashFromCommitNumber(ctx, types.CommitNumber(2))
+ require.NoError(t, err)
+ assert.Equal(t, hashes[2], gitHash)
+}
+
func TestNewPerfGitFromConfig_SQLite3_Success(t *testing.T) {
unittest.LargeTest(t)
ctx, _, _, hashes, _, instanceConfig, cleanup := gittest.NewForTest(t, perfsql.SQLiteDialect)
diff --git a/perf/go/config/config.go b/perf/go/config/config.go
index 2df3859..250c3ef 100644
--- a/perf/go/config/config.go
+++ b/perf/go/config/config.go
@@ -52,8 +52,8 @@
// https://www.cockroachlabs.com/docs/stable/connection-parameters.html for
// more details.
//
- // If the datastore type is 'gcs' then this value is a filename where
- // the sqlite database that caches git information should be stored.
+ // If the datastore type is 'gcs' then this value is used for just the SQL
+ // database that caches git information.
//
// In addition, for 'cockroachdb' databases, the database name given in the
// connection string must exist and the user given in the connection string
diff --git a/perf/go/git/gittest/gittest.go b/perf/go/git/gittest/gittest.go
index 48402bd..5d2fe5e 100644
--- a/perf/go/git/gittest/gittest.go
+++ b/perf/go/git/gittest/gittest.go
@@ -18,6 +18,12 @@
"go.skia.org/infra/perf/go/sql/sqltest"
)
+const (
+ // CockroachDatabaseName is the name of the database in CockroachDB that
+ // NewForTest will create.
+ CockroachDatabaseName = "git"
+)
+
// CleanupFunc is the type of clean up function that NewForTest returns.
type CleanupFunc func()
@@ -52,7 +58,7 @@
if dialect == perfsql.SQLiteDialect {
db, sqlCleanup = sqltest.NewSQLite3DBForTests(t)
} else {
- db, sqlCleanup = sqltest.NewCockroachDBForTests(t, "git", sqltest.ApplyMigrations)
+ db, sqlCleanup = sqltest.NewCockroachDBForTests(t, CockroachDatabaseName, sqltest.ApplyMigrations)
}
// Get tmp dir to use for repo checkout.
