[gold] Remove old auth method
Add some debugging to auth to make it more verbose.
Pipe local into gcs client to avoid messing with prod data.
Allow CPU profiling via internal port.
Change-Id: I0954b2c1b626530fd620978992f45da4b1509eb3
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/234117
Reviewed-by: Ben Wagner aka dogben <benjaminwagner@google.com>
Commit-Queue: Kevin Lubick <kjlubick@google.com>
diff --git a/go/auth/auth.go b/go/auth/auth.go
index aa0e946..968fb7e 100644
--- a/go/auth/auth.go
+++ b/go/auth/auth.go
@@ -16,6 +16,7 @@
"go.skia.org/infra/go/exec"
"go.skia.org/infra/go/httputils"
"go.skia.org/infra/go/metadata"
+ "go.skia.org/infra/go/skerr"
"go.skia.org/infra/go/sklog"
"go.skia.org/infra/go/util"
"golang.org/x/oauth2"
@@ -358,7 +359,7 @@
if err != nil {
body, err = ioutil.ReadFile(filename)
if err != nil {
- return nil, fmt.Errorf("Couldn't find JWT via metadata or in a local file.")
+ return nil, skerr.Fmt("Couldn't find JWT via metadata %q or in a local file %q.", metadataname, filename)
}
sklog.Infof("Read from file %s", filename)
} else {
@@ -369,7 +370,8 @@
ctx := context.WithValue(context.Background(), oauth2.HTTPClient, tokenClient)
jwtConfig, err := google.JWTConfigFromJSON(body, scopes...)
if err != nil {
- return nil, err
+ sklog.Errorf("Invalid JWT/JSON for token source: %s", body)
+ return nil, skerr.Wrapf(err, "failed to load JWT from JSON. See logs for full detail")
}
return jwtConfig.TokenSource(ctx), nil
}
diff --git a/golden/cmd/skiacorrectness/main.go b/golden/cmd/skiacorrectness/main.go
index ef13542..86f8b0f 100644
--- a/golden/cmd/skiacorrectness/main.go
+++ b/golden/cmd/skiacorrectness/main.go
@@ -20,7 +20,6 @@
"cloud.google.com/go/datastore"
"github.com/flynn/json5"
"github.com/gorilla/mux"
- "golang.org/x/oauth2"
"google.golang.org/api/option"
gstorage "google.golang.org/api/storage/v1"
"google.golang.org/grpc"
@@ -119,7 +118,6 @@
pubsubProjectID = flag.String("pubsub_project_id", "", "Project ID that houses the pubsub topics (e.g. for ingestion).")
redirectURL = flag.String("redirect_url", "https://gold.skia.org/oauth2callback/", "OAuth2 redirect url. Only used when local=false.")
resourcesDir = flag.String("resources_dir", "", "The directory to find templates, JS, and CSS files. If blank the directory relative to the source code files will be used.")
- serviceAccountFile = flag.String("service_account_file", "", "Credentials file for service account.")
showBotProgress = flag.Bool("show_bot_progress", true, "Query status.skia.org for the progress of bot results.")
siteURL = flag.String("site_url", "https://gold.skia.org", "URL where this app is hosted.")
tileFreshness = flag.Duration("tile_freshness", time.Minute, "How often to re-fetch the tile")
@@ -201,6 +199,7 @@
// Register pprof handlers
internalRouter.HandleFunc("/debug/pprof/", netpprof.Index)
internalRouter.HandleFunc("/debug/pprof/symbol", netpprof.Symbol)
+ internalRouter.HandleFunc("/debug/pprof/profile", netpprof.Profile)
internalRouter.HandleFunc("/debug/pprof/{profile}", netpprof.Index)
go func() {
@@ -228,17 +227,12 @@
// Get the token source for the service account with access to GCS, the Monorail issue tracker,
// cloud pubsub, and datastore.
- var deprecatedTS oauth2.TokenSource
- if *local {
- deprecatedTS = auth.NewGCloudTokenSource("")
- } else {
- deprecatedTS, err = auth.NewJWTServiceAccountTokenSource("", *serviceAccountFile, datastore.ScopeDatastore, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email")
- if err != nil {
- sklog.Fatalf("Failed to authenticate service account: %s", err)
- }
+ tokenSource, err := auth.NewDefaultTokenSource(*local, datastore.ScopeDatastore, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email")
+ if err != nil {
+ sklog.Fatalf("Failed to authenticate service account: %s", err)
}
// TODO(dogben): Ok to add request/dial timeouts?
- client := httputils.DefaultClientConfig().WithTokenSource(deprecatedTS).WithoutRetries().Client()
+ client := httputils.DefaultClientConfig().WithTokenSource(tokenSource).WithoutRetries().Client()
// serviceName uniquely identifies this host and app and is used as ID for other services.
nodeName, err := gevent.GetNodeName(appName, *local)
@@ -282,7 +276,7 @@
// depending whether an PubSub topic was defined.
var evt eventbus.EventBus = nil
if *eventTopic != "" {
- evt, err = gevent.New(*pubsubProjectID, *eventTopic, nodeName, option.WithTokenSource(deprecatedTS))
+ evt, err = gevent.New(*pubsubProjectID, *eventTopic, nodeName, option.WithTokenSource(tokenSource))
if err != nil {
sklog.Fatalf("Unable to create global event client. Got error: %s", err)
}
@@ -362,6 +356,7 @@
gsClientOpt := storage.GCSClientOptions{
HashesGSPath: *hashesGSPath,
BaselineGSPath: *baselineGSPath,
+ Dryrun: *local,
}
gsClient, err := storage.NewGCSClient(client, gsClientOpt)
@@ -369,7 +364,7 @@
sklog.Fatalf("Unable to create GCSClient: %s", err)
}
- if err := ds.InitWithOpt(*dsProjectID, *dsNamespace, option.WithTokenSource(deprecatedTS)); err != nil {
+ if err := ds.InitWithOpt(*dsProjectID, *dsNamespace, option.WithTokenSource(tokenSource)); err != nil {
sklog.Fatalf("Unable to configure cloud datastore: %s", err)
}
@@ -379,7 +374,7 @@
var fsClient *firestore.Client
if *fsLegacyAuth {
- fsClient, err = firestore.NewClient(context.Background(), *fsProjectID, "gold", *fsNamespace, deprecatedTS)
+ fsClient, err = firestore.NewClient(context.Background(), *fsProjectID, "gold", *fsNamespace, tokenSource)
if err != nil {
sklog.Fatalf("Unable to configure Firestore: %s", err)
}
diff --git a/golden/go/storage/gcsclient.go b/golden/go/storage/gcsclient.go
index 90aa576..f71cd14 100644
--- a/golden/go/storage/gcsclient.go
+++ b/golden/go/storage/gcsclient.go
@@ -25,6 +25,9 @@
// BaselineGSPath is the bucket and path for storing the baseline information.
// This is considered to be a directory and will be used as such.
BaselineGSPath string
+
+ // If DryRun is true, don't actually write the files (e.g. running locally)
+ Dryrun bool
}
// GCSClient provides an abstraction around read/writes to Google storage.
@@ -80,6 +83,10 @@
// WriteKnownDigests fulfills the GCSClient interface.
func (g *ClientImpl) WriteKnownDigests(digests types.DigestSlice) error {
+ if g.options.Dryrun {
+ sklog.Infof("dryrun: Writing %d digests", len(digests))
+ return nil
+ }
writeFn := func(w *gstorage.Writer) error {
for _, digest := range digests {
if _, err := w.Write([]byte(digest + "\n")); err != nil {
@@ -94,6 +101,11 @@
// ReadBaseline fulfills the GCSClient interface.
func (g *ClientImpl) WriteBaseline(b *baseline.Baseline, commitHash string) (string, error) {
+ if g.options.Dryrun {
+ sklog.Infof("dryrun: Writing baseline")
+ outPath := g.getBaselinePath(commitHash, b.Issue)
+ return "gs://" + outPath, nil
+ }
writeFn := func(w *gstorage.Writer) error {
if err := json.NewEncoder(w).Encode(b); err != nil {
return fmt.Errorf("Error encoding baseline to JSON: %s", err)
