[bugs-central] Poll the various issue frameworks for Skia's clients
Bug: skia:10783
Change-Id: I980f8217a2372e846944ea729fa79e1bad174615
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/329077
Reviewed-by: Joe Gregorio <jcgregorio@google.com>
diff --git a/bugs-central/go/poller/poller.go b/bugs-central/go/poller/poller.go
new file mode 100644
index 0000000..e025ece
--- /dev/null
+++ b/bugs-central/go/poller/poller.go
@@ -0,0 +1,203 @@
+package poller
+
+// Initializes and polls the various issue frameworks.
+
+import (
+ "context"
+ "os/user"
+ "path/filepath"
+ "time"
+
+ "cloud.google.com/go/storage"
+ "golang.org/x/oauth2"
+ "google.golang.org/api/option"
+
+ "go.skia.org/infra/bugs-central/go/bugs"
+ "go.skia.org/infra/bugs-central/go/bugs/github"
+ "go.skia.org/infra/bugs-central/go/bugs/issuetracker"
+ "go.skia.org/infra/bugs-central/go/bugs/monorail"
+ "go.skia.org/infra/bugs-central/go/db"
+ "go.skia.org/infra/bugs-central/go/types"
+ "go.skia.org/infra/go/baseapp"
+ "go.skia.org/infra/go/cleanup"
+ github_lib "go.skia.org/infra/go/github"
+ "go.skia.org/infra/go/httputils"
+ "go.skia.org/infra/go/skerr"
+ "go.skia.org/infra/go/sklog"
+)
+
+const (
+ // All recognized clients.
+ AndroidClient types.RecognizedClient = "Android"
+ ChromiumClient types.RecognizedClient = "Chromium"
+ FlutterNativeClient types.RecognizedClient = "Flutter-native"
+ FlutterOnWebClient types.RecognizedClient = "Flutter-on-web"
+ SkiaClient types.RecognizedClient = "Skia"
+)
+
+// IssuesPoller will be used to poll the different issue frameworks.
+type IssuesPoller struct {
+ storageClient *storage.Client
+ pathToGithubToken string
+ pathToServiceAccountFile string
+
+ dbClient *db.FirestoreDB
+}
+
+// New returns an instance of IssuesPoller.
+func New(ctx context.Context, ts oauth2.TokenSource, pathToServiceAccountFile string, dbClient *db.FirestoreDB) (*IssuesPoller, error) {
+ httpClient := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()
+ storageClient, err := storage.NewClient(ctx, option.WithHTTPClient(httpClient))
+ if err != nil {
+ return nil, skerr.Wrapf(err, "failed to init storage client")
+ }
+
+ pathToGithubToken := filepath.Join(github_lib.GITHUB_TOKEN_SERVER_PATH, github_lib.GITHUB_TOKEN_FILENAME)
+ if *baseapp.Local {
+ usr, err := user.Current()
+ if err != nil {
+ return nil, err
+ }
+ pathToGithubToken = filepath.Join(usr.HomeDir, github_lib.GITHUB_TOKEN_FILENAME)
+ }
+
+ return &IssuesPoller{
+ storageClient: storageClient,
+ pathToGithubToken: pathToGithubToken,
+ pathToServiceAccountFile: pathToServiceAccountFile,
+ dbClient: dbClient,
+ }, nil
+}
+
+// Start polls the different issue frameworks and populates DB and an in-memory object with that data.
+// It hardcodes information about Skia's various clients. It may be possible to extract some/all of these into
+// flags or YAML config files in the future.
+func (p *IssuesPoller) Start(ctx context.Context, pollInterval time.Duration) error {
+ // Instantiate the in-memory open issues object that will be passed to the different frameworks to
+ // populate.
+ openIssues := bugs.InitOpenIssues()
+
+ // Instantiate the bug frameworks with the different client configurations and then poll them.
+ bugFrameworks := []bugs.BugFramework{}
+
+ //////////////////// Android - IssueTracker ////////////////////
+ androidQueryConfig := &issuetracker.IssueTrackerQueryConfig{
+ Query: "componentid:1346 status:open",
+ Client: AndroidClient,
+ UntriagedPriorities: []string{"P4"},
+ UntriagedAliases: []string{"skia-android-triage@google.com"},
+ }
+ androidIssueTracker, err := issuetracker.New(p.storageClient, openIssues, androidQueryConfig)
+ if err != nil {
+ return skerr.Wrapf(err, "failed to init issuetracker for android")
+ }
+ bugFrameworks = append(bugFrameworks, androidIssueTracker)
+
+ //////////////////// Flutter_on_web - Github ////////////////////
+ flutterOnWebQueryConfig := &github.GithubQueryConfig{
+ Labels: []string{"e: web_canvaskit"},
+ Open: true,
+ PriorityRequired: true,
+ Client: FlutterOnWebClient,
+ }
+ flutterOnWebGithub, err := github.New(ctx, "flutter", "flutter", p.pathToGithubToken, openIssues, flutterOnWebQueryConfig)
+ if err != nil {
+ return skerr.Wrapf(err, "failed to init github for flutter-on-web")
+ }
+ bugFrameworks = append(bugFrameworks, flutterOnWebGithub)
+
+ //////////////////// Flutter_native - Github ////////////////////
+ flutterNativeQueryConfig := &github.GithubQueryConfig{
+ Labels: []string{"dependency: skia"},
+ ExcludeLabels: []string{"e: web_canvaskit"}, // These issues are already covered by flutter-on-web
+ Open: true,
+ PriorityRequired: false,
+ Client: FlutterNativeClient,
+ }
+ flutterNativeGithub, err := github.New(ctx, "flutter", "flutter", p.pathToGithubToken, openIssues, flutterNativeQueryConfig)
+ if err != nil {
+ return skerr.Wrapf(err, "failed to init github for flutter-on-web")
+ }
+ bugFrameworks = append(bugFrameworks, flutterNativeGithub)
+
+ //////////////////// Chromium:Internals>Skia - Monorail ////////////////////
+ crQueryConfig1 := &monorail.MonorailQueryConfig{
+ Instance: "chromium",
+ Query: "is:open component=Internals>Skia",
+ Client: ChromiumClient,
+ UntriagedStatuses: []string{"Untriaged", "Unconfirmed"},
+ }
+ crMonorail1, err := monorail.New(ctx, p.pathToServiceAccountFile, openIssues, crQueryConfig1)
+ if err != nil {
+ return skerr.Wrapf(err, "failed to init monorail for chromium")
+ }
+ bugFrameworks = append(bugFrameworks, crMonorail1)
+
+ //////////////////// Chromium:Internals>Skia>Compositing - Monorail ////////////////////
+ crQueryConfig2 := &monorail.MonorailQueryConfig{
+ Instance: "chromium",
+ Query: "is:open component=Internals>Skia>Compositing",
+ Client: ChromiumClient,
+ UntriagedStatuses: []string{"Untriaged", "Unconfirmed"},
+ }
+ crMonorail2, err := monorail.New(ctx, p.pathToServiceAccountFile, openIssues, crQueryConfig2)
+ if err != nil {
+ return skerr.Wrapf(err, "failed to init monorail for chromium")
+ }
+ bugFrameworks = append(bugFrameworks, crMonorail2)
+
+ //////////////////// Chromium:Internals>Skia>PDF - Monorail ////////////////////
+ crQueryConfig3 := &monorail.MonorailQueryConfig{
+ Instance: "chromium",
+ Query: "is:open component=Internals>Skia>PDF",
+ Client: ChromiumClient,
+ UntriagedStatuses: []string{"Untriaged", "Unconfirmed"},
+ }
+ crMonorail3, err := monorail.New(ctx, p.pathToServiceAccountFile, openIssues, crQueryConfig3)
+ if err != nil {
+ return skerr.Wrapf(err, "failed to init monorail for chromium")
+ }
+ bugFrameworks = append(bugFrameworks, crMonorail3)
+
+ //////////////////// Skia - Monorail ////////////////////
+ skQueryConfig := &monorail.MonorailQueryConfig{
+ Instance: "skia",
+ Query: "is:open",
+ Client: SkiaClient,
+ UntriagedStatuses: []string{"New"},
+ }
+ skMonorail, err := monorail.New(ctx, p.pathToServiceAccountFile, openIssues, skQueryConfig)
+ if err != nil {
+ return skerr.Wrapf(err, "failed to init monorail for skia")
+ }
+ bugFrameworks = append(bugFrameworks, skMonorail)
+
+ cleanup.Repeat(pollInterval, func(ctx context.Context) {
+ if !*baseapp.Local {
+ // Ignore the passed-in context; this allows us to continue running even if the
+ // context is canceled due to transient errors.
+ ctx = context.Background()
+ }
+
+ // Create a runID timestamp to associate all found issues with this poll iteration.
+ runId := p.dbClient.GenerateRunId(time.Now())
+
+ // Search all bug frameworks.
+ for _, b := range bugFrameworks {
+ if err := b.SearchClientAndPersist(ctx, p.dbClient, runId); err != nil {
+ sklog.Errorf("Error when searching and saving issues: %s", err)
+ return
+ }
+ }
+
+ // We are done with this iteration. Add the runId timestamp to the DB.
+ if err := p.dbClient.StoreRunId(context.Background(), runId); err != nil {
+ sklog.Errorf("Could not store runId in DB: %s", err)
+ return
+ }
+
+ openIssues.PrettyPrintOpenIssues()
+ }, nil)
+
+ return nil
+}
