[autoroll] Add Fuchsia SDK RepoManager Does not roll commits; instead, finds versions of the Fuchsia SDK in Google Cloud Storage to roll, writing the version ID to a file. Bug: skia:7734 Change-Id: I237b5698f28f74d9e237ea2681b346aa5e1e6275 Reviewed-on: https://skia-review.googlesource.com/115321 Reviewed-by: Ravi Mistry <rmistry@google.com> Commit-Queue: Eric Boren <borenet@google.com>
diff --git a/autoroll/go/repo_manager/chromium_afdo_repo_manager.go b/autoroll/go/repo_manager/chromium_afdo_repo_manager.go index 3f3c4c1..0156ac7 100644 --- a/autoroll/go/repo_manager/chromium_afdo_repo_manager.go +++ b/autoroll/go/repo_manager/chromium_afdo_repo_manager.go
@@ -63,7 +63,7 @@ NewAFDORepoManager func(context.Context, string, string, string, string, *gerrit.Gerrit, string, *http.Client) (RepoManager, error) = newAfdoRepoManager // Error used to indicate that a version number is invalid. - errInvalidVersion = errors.New("Invalid AFDO version.") + errInvalidAFDOVersion = errors.New("Invalid AFDO version.") ) // Parse the AFDO version. @@ -80,7 +80,7 @@ } return matchInts, nil } else { - return matchInts, errInvalidVersion + return matchInts, errInvalidAFDOVersion } } @@ -146,7 +146,7 @@ name := strings.TrimPrefix(item.Name, AFDO_GS_PATH) if _, err := parseAFDOVersion(name); err == nil { available = append(available, name) - } else if err == errInvalidVersion { + } else if err == errInvalidAFDOVersion { sklog.Warningf("Found AFDO file with improperly formatted name: %s", name) } else { sklog.Error(err)
diff --git a/autoroll/go/repo_manager/chromium_afdo_repo_manager_test.go b/autoroll/go/repo_manager/chromium_afdo_repo_manager_test.go index f1c25b7..513d6e8 100644 --- a/autoroll/go/repo_manager/chromium_afdo_repo_manager_test.go +++ b/autoroll/go/repo_manager/chromium_afdo_repo_manager_test.go
@@ -23,6 +23,10 @@ afdoRevPrev = "chromeos-chrome-amd64-66.0.3336.3_rc-r0.afdo.bz2" afdoRevBase = "chromeos-chrome-amd64-66.0.3336.3_rc-r1.afdo.bz2" afdoRevNext = "chromeos-chrome-amd64-66.0.3337.3_rc-r1.afdo.bz2" + + afdoTimePrev = "2009-11-10T23:00:00Z" + afdoTimeBase = "2009-11-10T23:01:00Z" + afdoTimeNext = "2009-11-10T23:02:00Z" ) func setupAfdo(t *testing.T) (context.Context, string, *git_testutils.GitBuilder, *exec.CommandCollector, *mockhttpclient.URLMock, func()) { @@ -155,13 +159,13 @@ t2([AFDO_VERSION_LENGTH]int{0, 5, 5, 5, 5}, [AFDO_VERSION_LENGTH]int{5, 5, 5, 5, 5}, false) } -func mockGSList(t *testing.T, urlmock *mockhttpclient.URLMock, bucket, path string, items []string) { +func mockGSList(t *testing.T, urlmock *mockhttpclient.URLMock, bucket, path string, items map[string]string) { fakeUrl := fmt.Sprintf("https://www.googleapis.com/storage/v1/b/%s/o?alt=json&delimiter=&pageToken=&prefix=%s&projection=full&versions=false", bucket, url.PathEscape(path)) resp := gsObjectList{ Kind: "storage#objects", Items: []gsObject{}, } - for _, item := range items { + for item, timestamp := range items { resp.Items = append(resp.Items, gsObject{ Kind: "storage#object", Id: bucket + path + item, @@ -171,10 +175,10 @@ Generation: "1", Metageneration: "1", ContentType: "application/octet-stream", - TimeCreated: "???", - Updated: "???", + TimeCreated: timestamp, + Updated: timestamp, StorageClass: "MULTI_REGIONAL", - TimeStorageClassUpdated: "???", + TimeStorageClassUpdated: timestamp, Size: "12345", Md5Hash: "dsafkldkldsaf", MediaLink: fakeUrl + item, @@ -195,7 +199,9 @@ g := setupFakeGerrit(t, wd) // Initial update, everything up-to-date. - mockGSList(t, urlmock, AFDO_GS_BUCKET, AFDO_GS_PATH, []string{afdoRevBase}) + mockGSList(t, urlmock, AFDO_GS_BUCKET, AFDO_GS_PATH, map[string]string{ + afdoRevBase: afdoTimeBase, + }) rm, err := NewAFDORepoManager(ctx, wd, gb.RepoUrl(), "master", depot_tools.GetDepotTools(t, ctx), g, "fake.server.com", urlmock.Client()) assert.NoError(t, err) assert.Equal(t, mockUser, rm.User()) @@ -217,7 +223,10 @@ assert.Equal(t, 0, rm.CommitsNotRolled()) // There's a new version. - mockGSList(t, urlmock, AFDO_GS_BUCKET, AFDO_GS_PATH, []string{afdoRevBase, afdoRevNext}) + mockGSList(t, urlmock, AFDO_GS_BUCKET, AFDO_GS_PATH, map[string]string{ + afdoRevBase: afdoTimeBase, + afdoRevNext: afdoTimeNext, + }) assert.NoError(t, rm.Update(ctx)) assert.Equal(t, afdoRevBase, rm.LastRollRev()) assert.Equal(t, afdoRevNext, rm.NextRollRev())
diff --git a/autoroll/go/repo_manager/fuchsia_sdk_repo_manager.go b/autoroll/go/repo_manager/fuchsia_sdk_repo_manager.go new file mode 100644 index 0000000..ed6998a --- /dev/null +++ b/autoroll/go/repo_manager/fuchsia_sdk_repo_manager.go
@@ -0,0 +1,324 @@ +package repo_manager + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "path" + "sort" + "strings" + "time" + + "cloud.google.com/go/storage" + "go.skia.org/infra/go/exec" + "go.skia.org/infra/go/gcs" + "go.skia.org/infra/go/gerrit" + "go.skia.org/infra/go/sklog" + "go.skia.org/infra/go/util" + "google.golang.org/api/option" +) + +const ( + FUCHSIA_SDK_GS_BUCKET = "fuchsia" + FUCHSIA_SDK_GS_PATH = "sdk" + + FUCHSIA_SDK_VERSION_FILE_PATH = "build/fuchsia/update_sdk.py" + + FUCHSIA_SDK_COMMIT_MSG_TMPL = `Roll Fuchsia SDK from %s to %s + +` + COMMIT_MSG_FOOTER_TMPL +) + +var ( + NewFuchsiaSDKRepoManager func(context.Context, string, string, string, string, *gerrit.Gerrit, string, *http.Client) (RepoManager, error) = newFuchsiaSDKRepoManager +) + +// fuchsiaSDKVersion corresponds to one version of the Fuchsia SDK. +type fuchsiaSDKVersion struct { + Timestamp time.Time + Version string +} + +// Return true iff this fuchsiaSDKVersion is newer than the other. +func (a *fuchsiaSDKVersion) Greater(b *fuchsiaSDKVersion) bool { + return a.Timestamp.After(b.Timestamp) +} + +type fuchsiaSDKVersionSlice []*fuchsiaSDKVersion + +func (s fuchsiaSDKVersionSlice) Len() int { + return len(s) +} + +func (s fuchsiaSDKVersionSlice) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +// We sort newest to oldest. +func (s fuchsiaSDKVersionSlice) Less(i, j int) bool { + return s[i].Greater(s[j]) +} + +// fuchsiaSDKRepoManager is a RepoManager which rolls the Fuchsia SDK version +// into Chromium. Unlike other rollers, there is no child repo to sync; the +// version number is obtained from Google Cloud Storage. +type fuchsiaSDKRepoManager struct { + *depotToolsRepoManager + commitsNotRolled int // Protected by infoMtx. + gcs gcs.GCSClient + gsPath string + lastRollRev *fuchsiaSDKVersion // Protected by infoMtx. + nextRollRev *fuchsiaSDKVersion // Protected by infoMtx. + versionFile string + versions []*fuchsiaSDKVersion // Protected by infoMtx. +} + +// Return a fuchsiaSDKRepoManager instance. +func newFuchsiaSDKRepoManager(ctx context.Context, workdir, parentRepo, parentBranch, depotTools string, g *gerrit.Gerrit, serverURL string, authClient *http.Client) (RepoManager, error) { + storageClient, err := storage.NewClient(ctx, option.WithHTTPClient(authClient)) + if err != nil { + return nil, err + } + + user, err := g.GetUserEmail() + if err != nil { + return nil, fmt.Errorf("Failed to determine Gerrit user: %s", err) + } + sklog.Infof("Repo Manager user: %s", user) + + wd := path.Join(workdir, "repo_manager") + if err := os.MkdirAll(wd, os.ModePerm); err != nil { + return nil, err + } + + parentBase := strings.TrimSuffix(path.Base(parentRepo), ".git") + parentDir := path.Join(wd, parentBase) + + rv := &fuchsiaSDKRepoManager{ + depotToolsRepoManager: &depotToolsRepoManager{ + commonRepoManager: &commonRepoManager{ + parentBranch: parentBranch, + g: g, + serverURL: serverURL, + user: user, + workdir: wd, + }, + depot_tools: depotTools, + gclient: path.Join(depotTools, GCLIENT), + parentDir: parentDir, + parentRepo: parentRepo, + }, + gcs: gcs.NewGCSClient(storageClient, FUCHSIA_SDK_GS_BUCKET), + gsPath: FUCHSIA_SDK_GS_PATH, + versionFile: path.Join(parentDir, FUCHSIA_SDK_VERSION_FILE_PATH), + } + return rv, rv.Update(ctx) +} + +// See documentation for RepoManager interface. +func (rm *fuchsiaSDKRepoManager) CreateNewRoll(ctx context.Context, from, to string, emails []string, cqExtraTrybots string, dryRun bool) (int64, error) { + rm.repoMtx.Lock() + defer rm.repoMtx.Unlock() + + // Clean the checkout, get onto a fresh branch. + if err := rm.cleanParent(ctx); err != nil { + return 0, err + } + if _, err := exec.RunCwd(ctx, rm.parentDir, "git", "checkout", "-b", ROLL_BRANCH, "-t", fmt.Sprintf("origin/%s", rm.parentBranch), "-f"); err != nil { + return 0, err + } + + // Defer some more cleanup. + defer func() { + util.LogErr(rm.cleanParent(ctx)) + }() + + // Create the roll CL. + if _, err := exec.RunCwd(ctx, rm.parentDir, "git", "config", "user.name", rm.user); err != nil { + return 0, err + } + if _, err := exec.RunCwd(ctx, rm.parentDir, "git", "config", "user.email", rm.user); err != nil { + return 0, err + } + + // Write the file. + if err := ioutil.WriteFile(rm.versionFile, []byte(to), os.ModePerm); err != nil { + return 0, err + } + + // Commit. + commitMsg := fmt.Sprintf(AFDO_COMMIT_MSG_TMPL, afdoShortVersion(from), afdoShortVersion(to), rm.serverURL) + if _, err := exec.RunCommand(ctx, &exec.Command{ + Dir: rm.parentDir, + Env: rm.GetEnvForDepotTools(), + Name: "git", + Args: []string{"commit", "-a", "-m", commitMsg}, + }); err != nil { + return 0, err + } + + // Upload the CL. + uploadCmd := &exec.Command{ + Dir: rm.parentDir, + Env: rm.GetEnvForDepotTools(), + Name: "git", + Args: []string{"cl", "upload", "--bypass-hooks", "-f", "-v", "-v"}, + Timeout: 2 * time.Minute, + } + if dryRun { + uploadCmd.Args = append(uploadCmd.Args, "--cq-dry-run") + } else { + uploadCmd.Args = append(uploadCmd.Args, "--use-commit-queue") + } + uploadCmd.Args = append(uploadCmd.Args, "--gerrit") + tbr := "\nTBR=" + if emails != nil && len(emails) > 0 { + emailStr := strings.Join(emails, ",") + tbr += emailStr + uploadCmd.Args = append(uploadCmd.Args, "--send-mail", "--cc", emailStr) + } + commitMsg += tbr + uploadCmd.Args = append(uploadCmd.Args, "-m", commitMsg) + + // Upload the CL. + sklog.Infof("Running command: git %s", strings.Join(uploadCmd.Args, " ")) + if _, err := exec.RunCommand(ctx, uploadCmd); err != nil { + return 0, err + } + + // Obtain the issue number. + tmp, err := ioutil.TempDir("", "") + if err != nil { + return 0, err + } + defer util.RemoveAll(tmp) + jsonFile := path.Join(tmp, "issue.json") + if _, err := exec.RunCommand(ctx, &exec.Command{ + Dir: rm.parentDir, + Env: rm.GetEnvForDepotTools(), + Name: "git", + Args: []string{"cl", "issue", fmt.Sprintf("--json=%s", jsonFile)}, + }); err != nil { + return 0, err + } + f, err := os.Open(jsonFile) + if err != nil { + return 0, err + } + var issue issueJson + if err := json.NewDecoder(f).Decode(&issue); err != nil { + return 0, err + } + return issue.Issue, nil +} + +// See documentation for RepoManager interface. +func (rm *fuchsiaSDKRepoManager) Update(ctx context.Context) error { + // Sync the projects. + rm.repoMtx.Lock() + defer rm.repoMtx.Unlock() + + if err := rm.createAndSyncParent(ctx); err != nil { + return fmt.Errorf("Could not create and sync parent repo: %s", err) + } + + // Read the file to determine the last roll rev. + lastRollRevBytes, err := ioutil.ReadFile(rm.versionFile) + if err != nil { + return err + } + lastRollRevStr := string(lastRollRevBytes) + + // Get the available SDK versions. + availableVersions := []*fuchsiaSDKVersion{} + if err := rm.gcs.AllFilesInDirectory(ctx, rm.gsPath, func(item *storage.ObjectAttrs) { + availableVersions = append(availableVersions, &fuchsiaSDKVersion{ + Timestamp: item.Updated, + Version: item.Name, + }) + }); err != nil { + return err + } + if len(availableVersions) == 0 { + return fmt.Errorf("No matching items found.") + } + sort.Sort(fuchsiaSDKVersionSlice(availableVersions)) + + // Get the next roll rev. + nextRollRev := availableVersions[0] + + // Find the last roll rev in the list of available versions. + lastIdx := -1 + for idx, v := range availableVersions { + if v.Version == lastRollRevStr { + lastIdx = idx + } + } + if lastIdx == -1 { + return fmt.Errorf("Last roll rev %q not found in available versions. Not-rolled count will be wrong. Versions: %v", lastRollRevStr, availableVersions) + } + + rm.infoMtx.Lock() + defer rm.infoMtx.Unlock() + rm.lastRollRev = availableVersions[lastIdx] + rm.nextRollRev = nextRollRev + // Versions are in reverse chronological order, so the next roll rev is + // the first in the list. Therefore the index of the last roll rev is + // the same as the number of revs we have not yet rolled. + rm.commitsNotRolled = lastIdx + rm.versions = availableVersions + return nil +} + +// See documentation for RepoManager interface. +func (rm *fuchsiaSDKRepoManager) FullChildHash(ctx context.Context, ver string) (string, error) { + rm.infoMtx.RLock() + defer rm.infoMtx.RUnlock() + for _, v := range rm.versions { + if strings.HasPrefix(v.Version, ver) { + return v.Version, nil + } + } + return "", fmt.Errorf("Unable to find version: %s", ver) +} + +// See documentation for RepoManager interface. +func (r *fuchsiaSDKRepoManager) LastRollRev() string { + r.infoMtx.RLock() + defer r.infoMtx.RUnlock() + return r.lastRollRev.Version +} + +// See documentation for RepoManager interface. +func (r *fuchsiaSDKRepoManager) NextRollRev() string { + r.infoMtx.RLock() + defer r.infoMtx.RUnlock() + return r.nextRollRev.Version +} + +// See documentation for RepoManager interface. +func (rm *fuchsiaSDKRepoManager) RolledPast(ctx context.Context, ver string) (bool, error) { + // TODO(borenet): Use a map? + var testVer *fuchsiaSDKVersion + for _, v := range rm.versions { + if v.Version == ver { + testVer = v + } + } + if testVer == nil { + return false, fmt.Errorf("Unknown version: %s", ver) + } + rm.infoMtx.RLock() + defer rm.infoMtx.RUnlock() + return !testVer.Greater(rm.lastRollRev), nil +} + +// See documentation for RepoManager interface. +func (rm *fuchsiaSDKRepoManager) CommitsNotRolled() int { + rm.infoMtx.RLock() + defer rm.infoMtx.RUnlock() + return rm.commitsNotRolled +}
diff --git a/autoroll/go/repo_manager/fuchsia_sdk_repo_manager_test.go b/autoroll/go/repo_manager/fuchsia_sdk_repo_manager_test.go new file mode 100644 index 0000000..07ace4a --- /dev/null +++ b/autoroll/go/repo_manager/fuchsia_sdk_repo_manager_test.go
@@ -0,0 +1,127 @@ +package repo_manager + +import ( + "context" + "io/ioutil" + "path" + "strings" + "testing" + + assert "github.com/stretchr/testify/require" + "go.skia.org/infra/go/autoroll" + depot_tools "go.skia.org/infra/go/depot_tools/testutils" + "go.skia.org/infra/go/exec" + git_testutils "go.skia.org/infra/go/git/testutils" + "go.skia.org/infra/go/mockhttpclient" + "go.skia.org/infra/go/testutils" +) + +const ( + fuchsiaSDKRevPrev = "def456" + fuchsiaSDKRevBase = "abc123" + fuchsiaSDKRevNext = "ace999" + + fuchsiaSDKTimePrev = "2009-11-10T23:00:01Z" + fuchsiaSDKTimeBase = "2009-11-10T23:00:02Z" + fuchsiaSDKTimeNext = "2009-11-10T23:00:03Z" +) + +func setupFuchsiaSDK(t *testing.T) (context.Context, string, *git_testutils.GitBuilder, *exec.CommandCollector, *mockhttpclient.URLMock, func()) { + wd, err := ioutil.TempDir("", "") + assert.NoError(t, err) + + // Create child and parent repos. + parent := git_testutils.GitInit(t, context.Background()) + parent.Add(context.Background(), FUCHSIA_SDK_VERSION_FILE_PATH, fuchsiaSDKRevBase) + parent.Commit(context.Background()) + + mockRun := &exec.CommandCollector{} + mockRun.SetDelegateRun(func(cmd *exec.Command) error { + if cmd.Name == "git" && cmd.Args[0] == "cl" { + if cmd.Args[1] == "upload" { + return nil + } else if cmd.Args[1] == "issue" { + json := testutils.MarshalJSON(t, &issueJson{ + Issue: issueNum, + IssueUrl: "???", + }) + f := strings.Split(cmd.Args[2], "=")[1] + testutils.WriteFile(t, f, json) + return nil + } + } + return exec.DefaultRun(cmd) + }) + ctx := exec.NewContext(context.Background(), mockRun.Run) + urlmock := mockhttpclient.NewURLMock() + + cleanup := func() { + testutils.RemoveAll(t, wd) + parent.Cleanup() + } + + return ctx, wd, parent, mockRun, urlmock, cleanup +} + +func TestFuchsiaSDKRepoManager(t *testing.T) { + testutils.LargeTest(t) + + ctx, wd, gb, _, urlmock, cleanup := setupFuchsiaSDK(t) + defer cleanup() + g := setupFakeGerrit(t, wd) + + // Initial update, everything up-to-date. + mockGSList(t, urlmock, FUCHSIA_SDK_GS_BUCKET, FUCHSIA_SDK_GS_PATH, map[string]string{ + fuchsiaSDKRevBase: fuchsiaSDKTimeBase, + fuchsiaSDKRevPrev: fuchsiaSDKTimePrev, + }) + rm, err := NewFuchsiaSDKRepoManager(ctx, wd, gb.RepoUrl(), "master", depot_tools.GetDepotTools(t, ctx), g, "fake.server.com", urlmock.Client()) + assert.NoError(t, err) + assert.Equal(t, mockUser, rm.User()) + assert.Equal(t, fuchsiaSDKRevBase, rm.LastRollRev()) + assert.Equal(t, fuchsiaSDKRevBase, rm.NextRollRev()) + fch, err := rm.FullChildHash(ctx, rm.LastRollRev()) + assert.NoError(t, err) + assert.Equal(t, fch, rm.LastRollRev()) + rolledPast, err := rm.RolledPast(ctx, fuchsiaSDKRevPrev) + assert.NoError(t, err) + assert.True(t, rolledPast) + rolledPast, err = rm.RolledPast(ctx, fuchsiaSDKRevBase) + assert.NoError(t, err) + assert.True(t, rolledPast) + assert.Nil(t, rm.PreUploadSteps()) + assert.Equal(t, 0, rm.CommitsNotRolled()) + + // There's a new version. + mockGSList(t, urlmock, FUCHSIA_SDK_GS_BUCKET, FUCHSIA_SDK_GS_PATH, map[string]string{ + fuchsiaSDKRevPrev: fuchsiaSDKTimePrev, + fuchsiaSDKRevBase: fuchsiaSDKTimeBase, + fuchsiaSDKRevNext: fuchsiaSDKTimeNext, + }) + assert.NoError(t, rm.Update(ctx)) + assert.Equal(t, fuchsiaSDKRevBase, rm.LastRollRev()) + assert.Equal(t, fuchsiaSDKRevNext, rm.NextRollRev()) + rolledPast, err = rm.RolledPast(ctx, fuchsiaSDKRevPrev) + assert.NoError(t, err) + assert.True(t, rolledPast) + rolledPast, err = rm.RolledPast(ctx, fuchsiaSDKRevBase) + assert.NoError(t, err) + assert.True(t, rolledPast) + rolledPast, err = rm.RolledPast(ctx, fuchsiaSDKRevNext) + assert.NoError(t, err) + assert.False(t, rolledPast) + assert.Equal(t, 1, rm.CommitsNotRolled()) + + // Upload a CL. + issue, err := rm.CreateNewRoll(ctx, rm.LastRollRev(), rm.NextRollRev(), emails, cqExtraTrybots, false) + assert.NoError(t, err) + assert.Equal(t, issueNum, issue) + msg, err := ioutil.ReadFile(path.Join(rm.(*fuchsiaSDKRepoManager).parentDir, ".git", "COMMIT_EDITMSG")) + assert.NoError(t, err) + from, to, err := autoroll.RollRev(strings.Split(string(msg), "\n")[0], func(h string) (string, error) { + return rm.FullChildHash(ctx, h) + }) + assert.NoError(t, err) + assert.Equal(t, fuchsiaSDKRevBase, from) + assert.Equal(t, fuchsiaSDKRevNext, to) +}