[perf] Add ability to specify auth for Git.
Change-Id: I59d7d03923c75135e5e2a6f54b40879543cd9ff4
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/279146
Reviewed-by: Joe Gregorio <jcgregorio@google.com>
Commit-Queue: Joe Gregorio <jcgregorio@google.com>
diff --git a/perf/configs/android-prod.json b/perf/configs/android-prod.json
index e7851c0..74d0ff9 100644
--- a/perf/configs/android-prod.json
+++ b/perf/configs/android-prod.json
@@ -21,6 +21,7 @@
"file_ingestion_pubsub_topic_name": "perf-ingestion-complete-android-production"
},
"git_repo_config": {
+ "git_auth_type": "gerrit",
"url": "https://skia.googlesource.com/perf-buildid/android-master",
"dir": "/tmp/repo",
"debounce_commit_url": true
diff --git a/perf/configs/android-x.json b/perf/configs/android-x.json
index e5bd942..f806b0e 100644
--- a/perf/configs/android-x.json
+++ b/perf/configs/android-x.json
@@ -21,6 +21,7 @@
"file_ingestion_pubsub_topic_name": ""
},
"git_repo_config": {
+ "git_auth_type": "gerrit",
"url": "https://skia.googlesource.com/perf-buildid/android-master",
"dir": "/tmp/repo",
"debounce_commit_url": true
diff --git a/perf/go/config/config.go b/perf/go/config/config.go
index f3e459f..f86af86 100644
--- a/perf/go/config/config.go
+++ b/perf/go/config/config.go
@@ -143,8 +143,27 @@
FileIngestionTopicName string `json:"file_ingestion_pubsub_topic_name"`
}
+// GitAuthType is the type of authentication Git should use, if any.
+type GitAuthType string
+
+const (
+ // GitAuthNone implies no authentication is needed when cloning/pulling a
+ // Git repo, i.e. it is public. The value is the empty string so that the
+ // default is no authentication.
+ GitAuthNone GitAuthType = ""
+
+ // GitAuthGerrit is for repos that are hosted by Gerrit and require
+ // authentication. This setting implies that a
+ // GOOGLE_APPLICATION_CREDENTIALS environment variable will be set and the
+ // associated service account has read access to the Gerrit repo.
+ GitAuthGerrit GitAuthType = "gerrit"
+)
+
// GitRepoConfig is the config for the git repo.
type GitRepoConfig struct {
+ // The type of authentication the repo requires.
+ GitAuthType GitAuthType `json:"git_auth_type"`
+
// URL that the Git repo is fetched from.
URL string `json:"url"`
diff --git a/perf/go/git/git.go b/perf/go/git/git.go
index 590b727..35fb66b 100644
--- a/perf/go/git/git.go
+++ b/perf/go/git/git.go
@@ -6,8 +6,11 @@
"context"
lru "github.com/hashicorp/golang-lru"
+ "go.skia.org/infra/go/auth"
"go.skia.org/infra/go/git/gitinfo"
+ "go.skia.org/infra/go/gitauth"
"go.skia.org/infra/go/skerr"
+ "go.skia.org/infra/go/sklog"
"go.skia.org/infra/perf/go/config"
"go.skia.org/infra/perf/go/types"
)
@@ -21,7 +24,17 @@
}
// New creates a new *Git from the given instance configuration.
-func New(ctx context.Context, instanceConfig *config.InstanceConfig) (*Git, error) {
+func New(ctx context.Context, local bool, instanceConfig *config.InstanceConfig) (*Git, error) {
+ if instanceConfig.GitRepoConfig.GitAuthType == config.GitAuthGerrit {
+ sklog.Info("Authenticating to Gerrit.")
+ ts, err := auth.NewDefaultTokenSource(local, auth.SCOPE_GERRIT)
+ if err != nil {
+ return nil, skerr.Wrap(err)
+ }
+ if _, err := gitauth.New(ts, "/tmp/git-cookie", true, ""); err != nil {
+ return nil, skerr.Wrap(err)
+ }
+ }
repo, err := gitinfo.CloneOrUpdate(ctx, instanceConfig.GitRepoConfig.URL, instanceConfig.GitRepoConfig.Dir, false)
if err != nil {
return nil, skerr.Wrap(err)
diff --git a/perf/go/git/git_test.go b/perf/go/git/git_test.go
index 93e56aa..f7e5fba 100644
--- a/perf/go/git/git_test.go
+++ b/perf/go/git/git_test.go
@@ -35,7 +35,7 @@
Dir: tmpDir,
},
}
- g, err := New(ctx, instanceConfig)
+ g, err := New(ctx, true, instanceConfig)
require.NoError(t, err)
return ctx, g, clean
}
@@ -81,6 +81,6 @@
Dir: tmpDir,
},
}
- _, err = New(ctx, instanceConfig)
+ _, err = New(ctx, true, instanceConfig)
require.Error(t, err)
}
diff --git a/perf/go/ingest/process/process.go b/perf/go/ingest/process/process.go
index 3ba871c..a7f5a08 100644
--- a/perf/go/ingest/process/process.go
+++ b/perf/go/ingest/process/process.go
@@ -20,7 +20,7 @@
//
// Except for file.Sources of type "dir" this function should never return
// except on error.
-func Start(ctx context.Context, instanceConfig *config.InstanceConfig) error {
+func Start(ctx context.Context, local bool, instanceConfig *config.InstanceConfig) error {
// Metrics.
filesReceived := metrics2.GetCounter("perfserver_ingest_files_received")
failedToParse := metrics2.GetCounter("perfserver_ingest_failed_to_parse")
@@ -46,7 +46,7 @@
// New gitinfo.GitInfo.
sklog.Infof("Cloning repo %q into %q", instanceConfig.GitRepoConfig.URL, instanceConfig.GitRepoConfig.Dir)
- g, err := perfgit.New(ctx, instanceConfig)
+ g, err := perfgit.New(ctx, local, instanceConfig)
if err != nil {
return skerr.Wrap(err)
}
diff --git a/perf/go/ingest/process/process_test.go b/perf/go/ingest/process/process_test.go
index ef2bd60..94e62fc 100644
--- a/perf/go/ingest/process/process_test.go
+++ b/perf/go/ingest/process/process_test.go
@@ -51,7 +51,7 @@
},
}
- err = Start(context.Background(), &instanceConfig)
+ err = Start(context.Background(), true, &instanceConfig)
require.NoError(t, err)
// The integration data set has 9 good files, 1 file with a bad commit, and
// 1 malformed JSON file.
diff --git a/perf/go/perfserver/cmd/ingest.go b/perf/go/perfserver/cmd/ingest.go
index 7d52259..5fa7ed4 100644
--- a/perf/go/perfserver/cmd/ingest.go
+++ b/perf/go/perfserver/cmd/ingest.go
@@ -16,7 +16,7 @@
with that data.
`,
RunE: func(cmd *cobra.Command, args []string) error {
- return process.Start(context.Background(), instanceConfig)
+ return process.Start(context.Background(), local, instanceConfig)
},
}
diff --git a/perf/go/perfserver/cmd/root.go b/perf/go/perfserver/cmd/root.go
index f8f66c4..9c250a6 100644
--- a/perf/go/perfserver/cmd/root.go
+++ b/perf/go/perfserver/cmd/root.go
@@ -13,6 +13,7 @@
var instanceConfigFile string
var instanceConfig *config.InstanceConfig
var promPort string
+var local bool
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
@@ -65,6 +66,7 @@
}
rootCmd.PersistentFlags().StringVar(&promPort, "prom_port", ":20000", "Metrics service address (e.g., ':20000')")
+ rootCmd.PersistentFlags().BoolVar(&local, "local", false, "True if running locally and not in production.")
alertInit()
ingestInit()
