[Autoroller] Add new Github CIPD DEPS repo manager
Highlights:
* Added new repo manager github_cipd_deps_repo_manager. It does not require a separate github user for each roller and instead uses a different github branch based on the rollerName.
* Refactored cleanParentWithRemote and createAndSyncParentWithRemote to handle branch name parameter.
* Refactored license scripts pre-upload step and made a new FlutterLicenseScriptsForTools.
* Added CIPD related methods from https://skia-review.googlesource.com/c/buildbot/+/215085
* Added new interface in go/cipd to create mocks from.
* Added a config file for a potential new roller- fuchsia-mac-sdk-cipd-flutter-engine.json
Bug: skia:9078
Change-Id: Ic82b6e4f81d46952af9786b6ddbd63f310d30526
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/214688
Commit-Queue: Ravi Mistry <rmistry@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
diff --git a/autoroll/config/fuchsia-mac-sdk-cipd-flutter-engine.json b/autoroll/config/fuchsia-mac-sdk-cipd-flutter-engine.json
new file mode 100644
index 0000000..32b71d1
--- /dev/null
+++ b/autoroll/config/fuchsia-mac-sdk-cipd-flutter-engine.json
@@ -0,0 +1,66 @@
+// See https://skia.googlesource.com/buildbot.git/+/master/autoroll/go/roller/config.go#130
+// for documentation of the autoroller config.
+{
+ "childName": "Skia",
+ "contacts": [
+ "rmistry@google.com"
+ ],
+ "isInternal": false,
+ "parentName": "Flutter",
+ "parentWaterfall": "https://build.chromium.org/p/client.flutter/console",
+ "rollerName": "cipd-flutter-engine-autoroll",
+ "serviceAccount": "flutter-engine-autoroll@skia-public.iam.gserviceaccount.com",
+ "sheriff": [
+ "rmistry@google.com"
+ ],
+ "github": {
+ "repoOwner": "flutter",
+ "repoName": "engine",
+ "checksNum": 4,
+ "checksWaitFor": [
+ "luci-engine"
+ ]
+ },
+ "githubCipdDEPSRepoManager": {
+ "childBranch": "master",
+ "childPath": "src/fuchsia/sdk/mac",
+ "parentBranch": "master",
+ "preUploadSteps": [
+ "FlutterLicenseScriptsForTools"
+ ],
+ "parentRepo": "git@github.com:flutter/engine.git",
+ "gclientSpec": "solutions=[{\"name\":\"src/flutter\",\"url\":\"git@github.com:rmistry/engine.git\",\"deps_file\":\"DEPS\",\"managed\":False,\"custom_deps\":{},\"custom_vars\":{\"host_os\":\"mac\"},\"safesync_url\":\"\"}]",
+ "githubParentPath": "src/flutter",
+ "cipdAssetName": "fuchsia/sdk/core/mac-amd64",
+ "cipdAssetTag": "latest"
+ },
+ "kubernetes": {
+ "cpu": "1",
+ "memory": "8Gi",
+ "disk": "50Gi",
+ "readinessInitialDelaySeconds": "600",
+ "readinessPeriodSeconds": "60",
+ "readinessFailureThreshold": "10",
+ "secrets": [
+ {
+ "name": "flutter-engine-github-token",
+ "mountPath": "/var/secrets/github-token"
+ },
+ {
+ "name": "flutter-engine-ssh-key",
+ "mountPath": "/var/secrets/ssh-key"
+ }
+ ]
+ },
+ "maxRollFrequency": "3h",
+ "notifiers": [
+ {
+ "filter": "warning",
+ "email": {
+ "emails": [
+ "$SHERIFF"
+ ]
+ }
+ }
+ ]
+}
diff --git a/autoroll/go/repo_manager/github_cipd_deps_repo_manager.go b/autoroll/go/repo_manager/github_cipd_deps_repo_manager.go
new file mode 100644
index 0000000..08ae168
--- /dev/null
+++ b/autoroll/go/repo_manager/github_cipd_deps_repo_manager.go
@@ -0,0 +1,358 @@
+package repo_manager
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "os"
+ "path"
+ "strings"
+ "time"
+
+ "go.skia.org/infra/autoroll/go/codereview"
+ "go.skia.org/infra/autoroll/go/revision"
+ "go.skia.org/infra/autoroll/go/strategy"
+ "go.skia.org/infra/go/cipd"
+ "go.skia.org/infra/go/exec"
+ "go.skia.org/infra/go/git"
+ "go.skia.org/infra/go/github"
+ "go.skia.org/infra/go/sklog"
+ "go.skia.org/infra/go/util"
+)
+
+const (
+ cipdPackageUrlTmpl = "%s/p/%s/+/%s"
+
+ cipdGithubTitleTmpl = "Roll %s from %s to %s"
+ cipdCommitMsgTmpl = cipdGithubTitleTmpl + `
+
+` + COMMIT_MSG_FOOTER_TMPL
+)
+
+var (
+ // Use this function to instantiate a NewGithubCipdDEPSRepoManager. This is able to be
+ // overridden for testing.
+ NewGithubCipdDEPSRepoManager func(context.Context, *GithubCipdDEPSRepoManagerConfig, string, string, *github.GitHub, string, string, *http.Client, codereview.CodeReview, bool) (RepoManager, error) = newGithubCipdDEPSRepoManager
+)
+
+// GithubCipdDEPSRepoManagerConfig provides configuration for the Github RepoManager.
+type GithubCipdDEPSRepoManagerConfig struct {
+ GithubDEPSRepoManagerConfig
+ CipdAssetName string `json:"cipdAssetName"`
+ CipdAssetTag string `json:"cipdAssetTag"`
+}
+
+// Validate the config.
+func (c *GithubCipdDEPSRepoManagerConfig) Validate() error {
+ if c.CipdAssetName == "" {
+ return fmt.Errorf("CipdAssetName is required.")
+ }
+ if c.CipdAssetTag == "" {
+ return fmt.Errorf("CipdAssetTag is required.")
+ }
+ return c.GithubDEPSRepoManagerConfig.Validate()
+}
+
+// githubCipdDEPSRepoManager is a struct used by the autoroller for managing checkouts.
+type githubCipdDEPSRepoManager struct {
+ *githubDEPSRepoManager
+ rollBranchName string
+ cipdAssetName string
+ cipdAssetTag string
+ CipdClient cipd.CIPDClient
+}
+
+// newGithubCipdDEPSRepoManager returns a RepoManager instance which operates in the given
+// working directory and updates at the given frequency.
+func newGithubCipdDEPSRepoManager(ctx context.Context, c *GithubCipdDEPSRepoManagerConfig, workdir, rollerName string, githubClient *github.GitHub, recipeCfgFile, serverURL string, httpClient *http.Client, cr codereview.CodeReview, local bool) (RepoManager, error) {
+ if err := c.Validate(); err != nil {
+ return nil, err
+ }
+ wd := path.Join(workdir, strings.TrimSuffix(path.Base(c.DepotToolsRepoManagerConfig.ParentRepo), ".git"))
+ drm, err := newDepotToolsRepoManager(ctx, c.DepotToolsRepoManagerConfig, wd, recipeCfgFile, serverURL, nil, httpClient, cr, local)
+ if err != nil {
+ return nil, err
+ }
+ dr := &depsRepoManager{
+ depotToolsRepoManager: drm,
+ }
+ if c.GithubParentPath != "" {
+ dr.parentDir = path.Join(wd, c.GithubParentPath)
+ }
+ gr := &githubDEPSRepoManager{
+ depsRepoManager: dr,
+ githubClient: githubClient,
+ }
+ sklog.Infof("Roller name is: %s\n", rollerName)
+ cipdClient, err := cipd.NewClient(httpClient, path.Join(workdir, "cipd"))
+ if err != nil {
+ return nil, err
+ }
+ gcr := &githubCipdDEPSRepoManager{
+ githubDEPSRepoManager: gr,
+ rollBranchName: rollerName,
+ cipdAssetName: c.CipdAssetName,
+ cipdAssetTag: c.CipdAssetTag,
+ CipdClient: cipdClient,
+ }
+
+ return gcr, nil
+}
+
+// See documentation for RepoManager interface.
+func (rm *githubCipdDEPSRepoManager) Update(ctx context.Context) error {
+ // Sync the projects.
+ rm.repoMtx.Lock()
+ defer rm.repoMtx.Unlock()
+
+ sklog.Info("Updating github repository")
+
+ // If parentDir does not exist yet then create the directory structure and
+ // populate it.
+ if _, err := os.Stat(rm.parentDir); err != nil {
+ if os.IsNotExist(err) {
+ if err := rm.createAndSyncParent(ctx); err != nil {
+ return fmt.Errorf("Could not create and sync %s: %s", rm.parentDir, err)
+ }
+ // Run gclient hooks to bring in any required binaries.
+ if _, err := exec.RunCommand(ctx, &exec.Command{
+ Dir: rm.parentDir,
+ Env: rm.depotToolsEnv,
+ Name: rm.gclient,
+ Args: []string{"runhooks"},
+ }); err != nil {
+ return fmt.Errorf("Error when running gclient runhooks on %s: %s", rm.parentDir, err)
+ }
+ } else {
+ return fmt.Errorf("Error when running os.Stat on %s: %s", rm.parentDir, err)
+ }
+ }
+
+ // Check to see whether there is an upstream yet.
+ remoteOutput, err := git.GitDir(rm.parentDir).Git(ctx, "remote", "show")
+ if err != nil {
+ return err
+ }
+ remoteFound := false
+ remoteLines := strings.Split(remoteOutput, "\n")
+ for _, remoteLine := range remoteLines {
+ if remoteLine == GITHUB_UPSTREAM_REMOTE_NAME {
+ remoteFound = true
+ break
+ }
+ }
+ if !remoteFound {
+ if _, err := git.GitDir(rm.parentDir).Git(ctx, "remote", "add", GITHUB_UPSTREAM_REMOTE_NAME, rm.parentRepo); err != nil {
+ return err
+ }
+ }
+ // Pull from upstream.
+ if _, err := git.GitDir(rm.parentDir).Git(ctx, "pull", GITHUB_UPSTREAM_REMOTE_NAME, rm.parentBranch); err != nil {
+ return err
+ }
+
+ // Get the last roll revision.
+ lastRollRev, err := rm.getLastRollRev(ctx)
+ if err != nil {
+ return err
+ }
+
+ // Find the not-rolled child repo commits.
+ notRolledRevs, err := getNotRolledRevs(ctx, rm.CipdClient, rm.lastRollRev, rm.cipdAssetName, rm.cipdAssetTag)
+ if err != nil {
+ return err
+ }
+
+ // Get the next roll revision.
+ nextRollRev, err := rm.getNextRollRev(ctx, notRolledRevs, lastRollRev)
+ if err != nil {
+ return err
+ }
+
+ rm.infoMtx.Lock()
+ defer rm.infoMtx.Unlock()
+ if rm.childRepoUrl == "" {
+ childRepo, err := exec.RunCwd(ctx, rm.parentDir, "git", "remote", "get-url", "origin")
+ if err != nil {
+ return err
+ }
+ rm.childRepoUrl = childRepo
+ }
+
+ rm.lastRollRev = lastRollRev
+ rm.nextRollRev = nextRollRev
+ rm.notRolledRevs = notRolledRevs
+
+ sklog.Infof("lastRollRev is: %s", rm.lastRollRev)
+ sklog.Infof("nextRollRev is: %s", nextRollRev)
+ sklog.Infof("notRolledRevs: %v", rm.notRolledRevs)
+ return nil
+}
+
+// getNotRolledRevs is a utility function that uses CIPD to find the not-yet-rolled versions of
+// the specified package.
+// Note: that this just finds all versions of the package between the last rolled version and the
+// version currently pointed to by cipdAssetTag; we can't know whether the ref we're tracking was
+// ever actually applied to any of the package instances in between.
+func getNotRolledRevs(ctx context.Context, cipdClient cipd.CIPDClient, lastRollRev, cipdAssetName, cipdAssetTag string) ([]*revision.Revision, error) {
+ head, err := cipdClient.ResolveVersion(ctx, cipdAssetName, cipdAssetTag)
+ if err != nil {
+ return nil, err
+ }
+ iter, err := cipdClient.ListInstances(ctx, cipdAssetName)
+ if err != nil {
+ return nil, err
+ }
+ notRolledRevs := []*revision.Revision{}
+ foundHead := false
+ for {
+ instances, err := iter.Next(ctx, 100)
+ if err != nil {
+ return nil, err
+ }
+ if len(instances) == 0 {
+ break
+ }
+ for _, instance := range instances {
+ id := instance.Pin.InstanceID
+ if id == head.InstanceID {
+ foundHead = true
+ }
+ if id == lastRollRev {
+ break
+ }
+ if foundHead {
+ notRolledRevs = append(notRolledRevs, &revision.Revision{
+ Id: id,
+ Display: instance.Pin.String(),
+ Description: instance.Pin.String(),
+ Timestamp: time.Time(instance.RegisteredTs),
+ URL: fmt.Sprintf(cipdPackageUrlTmpl, cipd.SERVICE_URL, cipdAssetName, id),
+ })
+ }
+ }
+ }
+ return notRolledRevs, nil
+}
+
+// See documentation for RepoManager interface.
+func (rm *githubCipdDEPSRepoManager) getLastRollRev(ctx context.Context) (string, error) {
+ output, err := exec.RunCwd(ctx, rm.parentDir, "python", rm.gclient, "getdep", "-r", fmt.Sprintf("%s:%s", rm.childPath, rm.cipdAssetName))
+ if err != nil {
+ return "", err
+ }
+ commit := strings.TrimSpace(output)
+ if commit == "" {
+ return "", fmt.Errorf("Got invalid output for `gclient getdep`: %s", output)
+ }
+ return commit, nil
+}
+
+// See documentation for RepoManager interface.
+func (rm *githubCipdDEPSRepoManager) CreateNewRoll(ctx context.Context, from, to string, emails []string, cqExtraTrybots string, dryRun bool) (int64, error) {
+ rm.repoMtx.Lock()
+ defer rm.repoMtx.Unlock()
+
+ sklog.Info("Creating a new Github Roll")
+
+ // Clean the checkout, get onto a fresh branch.
+ if err := rm.cleanParentWithRemoteAndBranch(ctx, GITHUB_UPSTREAM_REMOTE_NAME, rm.rollBranchName); err != nil {
+ return 0, err
+ }
+ if _, err := git.GitDir(rm.parentDir).Git(ctx, "checkout", fmt.Sprintf("%s/%s", GITHUB_UPSTREAM_REMOTE_NAME, rm.parentBranch), "-b", rm.rollBranchName); err != nil {
+ return 0, err
+ }
+ // Defer cleanup.
+ defer func() {
+ util.LogErr(rm.cleanParentWithRemoteAndBranch(ctx, GITHUB_UPSTREAM_REMOTE_NAME, rm.rollBranchName))
+ }()
+
+ // Make sure the forked repo is at the same hash as the target repo before
+ // creating the pull request on the rm.rollBranchName.
+ if _, err := git.GitDir(rm.parentDir).Git(ctx, "push", "origin", rm.rollBranchName, "-f"); err != nil {
+ return 0, err
+ }
+
+ // Make sure the right name and email are set.
+ if !rm.local {
+ if _, err := git.GitDir(rm.parentDir).Git(ctx, "config", "user.name", rm.codereview.UserName()); err != nil {
+ return 0, err
+ }
+ if _, err := git.GitDir(rm.parentDir).Git(ctx, "config", "user.email", rm.codereview.UserEmail()); err != nil {
+ return 0, err
+ }
+ }
+
+ // Run "gclient setdep".
+ args := []string{"setdep", "-r", fmt.Sprintf("%s:%s@%s", rm.childPath, rm.cipdAssetName, to)}
+ if _, err := exec.RunCommand(ctx, &exec.Command{
+ Dir: rm.parentDir,
+ Env: rm.depotToolsEnv,
+ Name: rm.gclient,
+ Args: args,
+ }); err != nil {
+ return 0, err
+ }
+
+ // Make the checkout match the new DEPS.
+ sklog.Info("Running gclient sync on the checkout")
+ if _, err := exec.RunCommand(ctx, &exec.Command{
+ Dir: rm.depsRepoManager.parentDir,
+ Env: rm.depotToolsEnv,
+ Name: rm.gclient,
+ Args: []string{"sync", "-D", "--process-all-deps", "-f"},
+ }); err != nil {
+ return 0, fmt.Errorf("Error when running gclient sync to make checkout match the new DEPS: %s", err)
+ }
+
+ // Run the pre-upload steps.
+ for _, s := range rm.PreUploadSteps() {
+ if err := s(ctx, rm.depotToolsEnv, rm.httpClient, rm.parentDir); err != nil {
+ return 0, fmt.Errorf("Error when running pre-upload step: %s", err)
+ }
+ }
+
+ // Build the commitMsg.
+ commitMsg := fmt.Sprintf(cipdCommitMsgTmpl, rm.cipdAssetName, from, to, rm.serverURL)
+
+ // Commit.
+ if _, err := git.GitDir(rm.parentDir).Git(ctx, "commit", "-a", "-m", commitMsg); err != nil {
+ return 0, err
+ }
+
+ // Push to the forked repository.
+ if _, err := git.GitDir(rm.parentDir).Git(ctx, "push", "origin", rm.rollBranchName, "-f"); err != nil {
+ return 0, err
+ }
+
+ // Create a pull request.
+ title := fmt.Sprintf(cipdGithubTitleTmpl, rm.cipdAssetName, from[:5]+"...", to[:5]+"...")
+ headBranch := fmt.Sprintf("%s:%s", rm.codereview.UserName(), rm.rollBranchName)
+ pr, err := rm.githubClient.CreatePullRequest(title, rm.parentBranch, headBranch, commitMsg)
+ if err != nil {
+ return 0, err
+ }
+
+ // Add appropriate label to the pull request.
+ label := github.COMMIT_LABEL
+ if dryRun {
+ label = github.DRYRUN_LABEL
+ }
+ if err := rm.githubClient.AddLabel(pr.GetNumber(), label); err != nil {
+ return 0, err
+ }
+
+ return int64(pr.GetNumber()), nil
+}
+
+// See documentation for RepoManager interface.
+func (r *githubCipdDEPSRepoManager) DefaultStrategy() string {
+ return strategy.ROLL_STRATEGY_BATCH
+}
+
+// See documentation for RepoManager interface.
+func (r *githubCipdDEPSRepoManager) ValidStrategies() []string {
+ return []string{
+ strategy.ROLL_STRATEGY_BATCH,
+ }
+}
diff --git a/autoroll/go/repo_manager/github_cipd_deps_repo_manager_test.go b/autoroll/go/repo_manager/github_cipd_deps_repo_manager_test.go
new file mode 100644
index 0000000..df57329
--- /dev/null
+++ b/autoroll/go/repo_manager/github_cipd_deps_repo_manager_test.go
@@ -0,0 +1,255 @@
+package repo_manager
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "path/filepath"
+ "testing"
+
+ assert "github.com/stretchr/testify/require"
+ "go.chromium.org/luci/cipd/client/cipd"
+ "go.chromium.org/luci/cipd/common"
+ "go.skia.org/infra/autoroll/go/strategy"
+ "go.skia.org/infra/go/cipd/mocks"
+ "go.skia.org/infra/go/exec"
+ git_testutils "go.skia.org/infra/go/git/testutils"
+ "go.skia.org/infra/go/recipe_cfg"
+ "go.skia.org/infra/go/testutils"
+ "go.skia.org/infra/go/testutils/unittest"
+)
+
+const (
+ GITHUB_CIPD_DEPS_CHILD_PATH = "path/to/child"
+ GITHUB_CIPD_ASSET_NAME = "test/cipd/name"
+ GITHUB_CIPD_ASSET_TAG = "latest"
+
+ LAST_ROLLED = "xyz12345"
+ NOT_ROLLED_1 = "abc12345"
+ NOT_ROLLED_2 = "def12345"
+)
+
+func githubCipdDEPSRmCfg() *GithubCipdDEPSRepoManagerConfig {
+ return &GithubCipdDEPSRepoManagerConfig{
+ GithubDEPSRepoManagerConfig: GithubDEPSRepoManagerConfig{
+ DepotToolsRepoManagerConfig: DepotToolsRepoManagerConfig{
+ CommonRepoManagerConfig: CommonRepoManagerConfig{
+ ChildBranch: "master",
+ ChildPath: GITHUB_CIPD_DEPS_CHILD_PATH,
+ ParentBranch: "master",
+ },
+ },
+ },
+ CipdAssetName: GITHUB_CIPD_ASSET_NAME,
+ CipdAssetTag: "latest",
+ }
+}
+
+func setupGithubCipdDEPS(t *testing.T) (context.Context, string, *git_testutils.GitBuilder, *exec.CommandCollector, func()) {
+ wd, err := ioutil.TempDir("", "")
+ assert.NoError(t, err)
+
+ // Create child and parent repos.
+ childPath := filepath.Join(wd, "github_repos", "earth")
+ assert.NoError(t, os.MkdirAll(childPath, 0755))
+
+ parent := git_testutils.GitInit(t, context.Background())
+ parent.Add(context.Background(), "DEPS", fmt.Sprintf(`
+deps = {
+ "%s": {
+ "packages": [
+ {
+ "package": "%s",
+ "version": "%s"
+ }
+ ],
+ },
+}`, GITHUB_CIPD_DEPS_CHILD_PATH, GITHUB_CIPD_ASSET_NAME, LAST_ROLLED))
+ parent.Commit(context.Background())
+
+ mockRun := &exec.CommandCollector{}
+ mockRun.SetDelegateRun(func(cmd *exec.Command) error {
+ if cmd.Name == "git" {
+ if cmd.Args[0] == "clone" || cmd.Args[0] == "fetch" {
+ return nil
+ }
+ if cmd.Args[0] == "checkout" && cmd.Args[1] == "remote/master" {
+ // Pretend origin is the remote branch for testing ease.
+ cmd.Args[1] = "origin/master"
+ }
+ }
+ return exec.DefaultRun(cmd)
+ })
+ ctx := exec.NewContext(context.Background(), mockRun.Run)
+
+ cleanup := func() {
+ testutils.RemoveAll(t, wd)
+ parent.Cleanup()
+ }
+
+ return ctx, wd, parent, mockRun, cleanup
+}
+
+type instanceEnumeratorImpl struct {
+ done bool
+}
+
+func (e *instanceEnumeratorImpl) Next(ctx context.Context, limit int) ([]cipd.InstanceInfo, error) {
+ if e.done {
+ return nil, nil
+ }
+ instances := []cipd.InstanceInfo{}
+ instance0 := cipd.InstanceInfo{
+ Pin: common.Pin{
+ PackageName: GITHUB_CIPD_ASSET_NAME,
+ InstanceID: LAST_ROLLED,
+ },
+ RegisteredBy: "aquaman@ocean.com",
+ }
+ instance1 := cipd.InstanceInfo{
+ Pin: common.Pin{
+ PackageName: GITHUB_CIPD_ASSET_NAME,
+ InstanceID: NOT_ROLLED_1,
+ },
+ RegisteredBy: "superman@krypton.com",
+ }
+ instance2 := cipd.InstanceInfo{
+ Pin: common.Pin{
+ PackageName: GITHUB_CIPD_ASSET_NAME,
+ InstanceID: NOT_ROLLED_2,
+ },
+ RegisteredBy: "batman@gotham.com",
+ }
+ instances = append(instances, instance0, instance1, instance2)
+ e.done = true
+ return instances, nil
+}
+
+func getCipdMock(ctx context.Context) *mocks.CIPDClient {
+ cipdClient := &mocks.CIPDClient{}
+ head := common.Pin{
+ PackageName: "test/cipd/name",
+ InstanceID: NOT_ROLLED_1,
+ }
+ cipdClient.On("ResolveVersion", ctx, GITHUB_CIPD_ASSET_NAME, GITHUB_CIPD_ASSET_TAG).Return(head, nil).Once()
+ cipdClient.On("ListInstances", ctx, GITHUB_CIPD_ASSET_NAME).Return(&instanceEnumeratorImpl{}, nil).Once()
+ return cipdClient
+}
+
+// TestGithubRepoManager tests all aspects of the GithubRepoManager except for CreateNewRoll.
+func TestGithubCipdDEPSRepoManager(t *testing.T) {
+ unittest.LargeTest(t)
+
+ ctx, wd, parent, _, cleanup := setupGithubCipdDEPS(t)
+ defer cleanup()
+ recipesCfg := filepath.Join(testutils.GetRepoRoot(t), recipe_cfg.RECIPE_CFG_PATH)
+
+ g, _ := setupFakeGithub(t, nil)
+ cfg := githubCipdDEPSRmCfg()
+ cfg.ParentRepo = parent.RepoUrl()
+ rm, err := NewGithubCipdDEPSRepoManager(ctx, cfg, wd, "test_roller_name", g, recipesCfg, "fake.server.com", nil, githubCR(t, g), false)
+ assert.NoError(t, err)
+ rm.(*githubCipdDEPSRepoManager).CipdClient = getCipdMock(ctx)
+ assert.NoError(t, SetStrategy(ctx, rm, strategy.ROLL_STRATEGY_BATCH))
+ assert.NoError(t, rm.Update(ctx))
+
+ // Assert last roll, next roll and not rolled yet.
+ assert.Equal(t, LAST_ROLLED, rm.LastRollRev())
+ assert.Equal(t, NOT_ROLLED_1, rm.NextRollRev())
+ assert.Equal(t, 2, len(rm.NotRolledRevisions()))
+ assert.Equal(t, NOT_ROLLED_1, rm.NotRolledRevisions()[0].Id)
+ assert.Equal(t, fmt.Sprintf("%s:%s", GITHUB_CIPD_ASSET_NAME, NOT_ROLLED_1), rm.NotRolledRevisions()[0].Display)
+ assert.Equal(t, NOT_ROLLED_2, rm.NotRolledRevisions()[1].Id)
+ assert.Equal(t, fmt.Sprintf("%s:%s", GITHUB_CIPD_ASSET_NAME, NOT_ROLLED_2), rm.NotRolledRevisions()[1].Display)
+}
+
+func TestCreateNewGithubCipdDEPSRoll(t *testing.T) {
+ unittest.LargeTest(t)
+
+ ctx, wd, parent, _, cleanup := setupGithubCipdDEPS(t)
+ defer cleanup()
+ recipesCfg := filepath.Join(testutils.GetRepoRoot(t), recipe_cfg.RECIPE_CFG_PATH)
+
+ g, urlMock := setupFakeGithub(t, nil)
+ cfg := githubCipdDEPSRmCfg()
+ cfg.ParentRepo = parent.RepoUrl()
+ rm, err := NewGithubCipdDEPSRepoManager(ctx, cfg, wd, "test_roller_name", g, recipesCfg, "fake.server.com", nil, githubCR(t, g), false)
+ assert.NoError(t, err)
+ rm.(*githubCipdDEPSRepoManager).CipdClient = getCipdMock(ctx)
+ assert.NoError(t, SetStrategy(ctx, rm, strategy.ROLL_STRATEGY_BATCH))
+ assert.NoError(t, rm.Update(ctx))
+
+ // Create a roll.
+ mockGithubRequests(t, urlMock, rm.LastRollRev(), rm.NextRollRev(), len(rm.NotRolledRevisions()))
+ issue, err := rm.CreateNewRoll(ctx, rm.LastRollRev(), rm.NextRollRev(), githubEmails, cqExtraTrybots, false)
+ assert.NoError(t, err)
+ assert.Equal(t, issueNum, issue)
+}
+
+// Verify that we ran the PreUploadSteps.
+func TestRanPreUploadStepsGithubCipdDEPS(t *testing.T) {
+ unittest.LargeTest(t)
+
+ ctx, wd, parent, _, cleanup := setupGithubCipdDEPS(t)
+ defer cleanup()
+ recipesCfg := filepath.Join(testutils.GetRepoRoot(t), recipe_cfg.RECIPE_CFG_PATH)
+
+ g, urlMock := setupFakeGithub(t, nil)
+ cfg := githubCipdDEPSRmCfg()
+ cfg.ParentRepo = parent.RepoUrl()
+ rm, err := NewGithubCipdDEPSRepoManager(ctx, cfg, wd, "test_roller_name", g, recipesCfg, "fake.server.com", nil, githubCR(t, g), false)
+ assert.NoError(t, err)
+ rm.(*githubCipdDEPSRepoManager).CipdClient = getCipdMock(ctx)
+ assert.NoError(t, SetStrategy(ctx, rm, strategy.ROLL_STRATEGY_BATCH))
+ assert.NoError(t, rm.Update(ctx))
+
+ ran := false
+ rm.(*githubCipdDEPSRepoManager).preUploadSteps = []PreUploadStep{
+ func(context.Context, []string, *http.Client, string) error {
+ ran = true
+ return nil
+ },
+ }
+
+ // Create a roll, assert that we ran the PreUploadSteps.
+ mockGithubRequests(t, urlMock, rm.LastRollRev(), rm.NextRollRev(), len(rm.NotRolledRevisions()))
+ _, createErr := rm.CreateNewRoll(ctx, rm.LastRollRev(), rm.NextRollRev(), githubEmails, cqExtraTrybots, false)
+ assert.NoError(t, createErr)
+ assert.True(t, ran)
+}
+
+// Verify that we fail when a PreUploadStep fails.
+func TestErrorPreUploadStepsGithubCipdDEPS(t *testing.T) {
+ unittest.LargeTest(t)
+
+ ctx, wd, parent, _, cleanup := setupGithubCipdDEPS(t)
+ defer cleanup()
+ recipesCfg := filepath.Join(testutils.GetRepoRoot(t), recipe_cfg.RECIPE_CFG_PATH)
+
+ g, urlMock := setupFakeGithub(t, nil)
+ cfg := githubCipdDEPSRmCfg()
+ cfg.ParentRepo = parent.RepoUrl()
+ rm, err := NewGithubCipdDEPSRepoManager(ctx, cfg, wd, "test_roller_name", g, recipesCfg, "fake.server.com", nil, githubCR(t, g), false)
+ assert.NoError(t, err)
+ rm.(*githubCipdDEPSRepoManager).CipdClient = getCipdMock(ctx)
+ assert.NoError(t, SetStrategy(ctx, rm, strategy.ROLL_STRATEGY_BATCH))
+ assert.NoError(t, rm.Update(ctx))
+
+ ran := false
+ expectedErr := errors.New("Expected error")
+ rm.(*githubCipdDEPSRepoManager).preUploadSteps = []PreUploadStep{
+ func(context.Context, []string, *http.Client, string) error {
+ ran = true
+ return expectedErr
+ },
+ }
+
+ // Create a roll, assert that we ran the PreUploadSteps.
+ mockGithubRequests(t, urlMock, rm.LastRollRev(), rm.NextRollRev(), len(rm.NotRolledRevisions()))
+ _, createErr := rm.CreateNewRoll(ctx, rm.LastRollRev(), rm.NextRollRev(), githubEmails, cqExtraTrybots, false)
+ assert.Error(t, expectedErr, createErr)
+ assert.True(t, ran)
+}
diff --git a/autoroll/go/repo_manager/github_deps_repo_manager.go b/autoroll/go/repo_manager/github_deps_repo_manager.go
index 3b02f55..d901839 100644
--- a/autoroll/go/repo_manager/github_deps_repo_manager.go
+++ b/autoroll/go/repo_manager/github_deps_repo_manager.go
@@ -124,7 +124,7 @@
}
// gclient sync to get latest version of child repo to find the next roll
// rev from.
- if err := rm.createAndSyncParentWithRemote(ctx, GITHUB_UPSTREAM_REMOTE_NAME); err != nil {
+ if err := rm.createAndSyncParentWithRemoteAndBranch(ctx, GITHUB_UPSTREAM_REMOTE_NAME, ROLL_BRANCH); err != nil {
return fmt.Errorf("Could not create and sync parent repo: %s", err)
}
@@ -174,7 +174,7 @@
sklog.Info("Creating a new Github Roll")
// Clean the checkout, get onto a fresh branch.
- if err := rm.cleanParentWithRemote(ctx, GITHUB_UPSTREAM_REMOTE_NAME); err != nil {
+ if err := rm.cleanParentWithRemoteAndBranch(ctx, GITHUB_UPSTREAM_REMOTE_NAME, ROLL_BRANCH); err != nil {
return 0, err
}
if _, err := git.GitDir(rm.parentDir).Git(ctx, "checkout", fmt.Sprintf("%s/%s", GITHUB_UPSTREAM_REMOTE_NAME, rm.parentBranch), "-b", ROLL_BRANCH); err != nil {
@@ -182,7 +182,7 @@
}
// Defer cleanup.
defer func() {
- util.LogErr(rm.cleanParentWithRemote(ctx, GITHUB_UPSTREAM_REMOTE_NAME))
+ util.LogErr(rm.cleanParentWithRemoteAndBranch(ctx, GITHUB_UPSTREAM_REMOTE_NAME, ROLL_BRANCH))
}()
// Make sure the forked repo is at the same hash as the target repo before
diff --git a/autoroll/go/repo_manager/github_repo_manager_test.go b/autoroll/go/repo_manager/github_repo_manager_test.go
index fadebad..2eea5d3 100644
--- a/autoroll/go/repo_manager/github_repo_manager_test.go
+++ b/autoroll/go/repo_manager/github_repo_manager_test.go
@@ -115,8 +115,10 @@
assert.NoError(t, err)
urlMock.MockOnce(githubApiUrl+"/user", mockhttpclient.MockGetDialogue(serializedUser))
- // Mock getRawFile.
- urlMock.MockOnce("https://raw.githubusercontent.com/superman/krypton/master/dummy-file.txt", mockhttpclient.MockGetDialogue([]byte(childCommits[0])))
+ if childCommits != nil && len(childCommits) > 0 {
+ // Mock getRawFile.
+ urlMock.MockOnce("https://raw.githubusercontent.com/superman/krypton/master/dummy-file.txt", mockhttpclient.MockGetDialogue([]byte(childCommits[0])))
+ }
// Mock /issues endpoint for get and patch requests.
serializedIssue, err := json.Marshal(&github_api.Issue{
diff --git a/autoroll/go/repo_manager/pre_upload_steps.go b/autoroll/go/repo_manager/pre_upload_steps.go
index 6a50d73..545fb8b 100644
--- a/autoroll/go/repo_manager/pre_upload_steps.go
+++ b/autoroll/go/repo_manager/pre_upload_steps.go
@@ -34,9 +34,10 @@
// Return the PreUploadStep with the given name.
func GetPreUploadStep(s string) (PreUploadStep, error) {
rv, ok := map[string]PreUploadStep{
- "GoGenerateCipd": GoGenerateCipd,
- "TrainInfra": TrainInfra,
- "FlutterLicenseScripts": FlutterLicenseScripts,
+ "GoGenerateCipd": GoGenerateCipd,
+ "TrainInfra": TrainInfra,
+ "FlutterLicenseScripts": FlutterLicenseScripts,
+ "FlutterLicenseScriptsForTools": FlutterLicenseScriptsForTools,
}[s]
if !ok {
return nil, fmt.Errorf("No such pre-upload step: %s", s)
@@ -61,7 +62,7 @@
func TrainInfra(ctx context.Context, env []string, client *http.Client, parentRepoDir string) error {
// TODO(borenet): Should we plumb through --local and --workdir?
sklog.Info("Installing Go...")
- _, goEnv, err := go_install.EnsureGo(client, cipdRoot)
+ _, goEnv, err := go_install.EnsureGo(ctx, client, cipdRoot)
if err != nil {
return err
}
@@ -102,12 +103,32 @@
// https://bugs.chromium.org/p/skia/issues/detail?id=7730#c6 and in
// https://github.com/flutter/engine/blob/master/tools/licenses/README.md
func FlutterLicenseScripts(ctx context.Context, _ []string, _ *http.Client, parentRepoDir string) error {
- sklog.Info("Running flutter license scripts.")
licenseScriptFailure := int64(1)
defer func() {
metrics2.GetInt64Metric("flutter_license_script_failure", nil).Update(licenseScriptFailure)
}()
+ if err := flutterLicenseScripts(ctx, parentRepoDir, "licenses_skia"); err != nil {
+ return err
+ }
+ licenseScriptFailure = 0
+ return nil
+}
+// Run the flutter license scripts for tools.
+func FlutterLicenseScriptsForTools(ctx context.Context, _ []string, _ *http.Client, parentRepoDir string) error {
+ licenseScriptFailure := int64(1)
+ defer func() {
+ metrics2.GetInt64Metric("flutter_license_script_failure", nil).Update(licenseScriptFailure)
+ }()
+ if err := flutterLicenseScripts(ctx, parentRepoDir, "tools_signature"); err != nil {
+ return err
+ }
+ licenseScriptFailure = 0
+ return nil
+}
+
+func flutterLicenseScripts(ctx context.Context, parentRepoDir, licenseFileName string) error {
+ sklog.Info("Running flutter license scripts.")
binariesPath := filepath.Join(parentRepoDir, "..", "third_party", "dart", "tools", "sdks", "dart-sdk", "bin")
// Step1: Run pub get.
@@ -138,21 +159,20 @@
return fmt.Errorf("Error when running dart license script: %s", err)
}
- licensesThirdPartyFileName := "licenses_skia"
- // Step4: Check to see if licenses_third_party was created in the out dir.
- // It will be created if the third_party hash changes.
- if _, err := os.Stat(filepath.Join(licensesOutDir, licensesThirdPartyFileName)); err == nil {
- sklog.Infof("Found %s", licensesThirdPartyFileName)
+ // Step4: Check to see if the target license file was created in the out dir.
+ // It will be created if the hash changes.
+ if _, err := os.Stat(filepath.Join(licensesOutDir, licenseFileName)); err == nil {
+ sklog.Infof("Found %s", licenseFileName)
// Step5: Copy from out dir to goldens dir. This is required for updating
// the release file in sky_engine/LICENSE.
- if _, err := exec.RunCwd(ctx, licenseToolsDir, "cp", filepath.Join(licensesOutDir, licensesThirdPartyFileName), filepath.Join(licensesGoldenDir, licensesThirdPartyFileName)); err != nil {
- return fmt.Errorf("Error when copying licenses_third_party from out to golden dir: %s", err)
+ if _, err := exec.RunCwd(ctx, licenseToolsDir, "cp", filepath.Join(licensesOutDir, licenseFileName), filepath.Join(licensesGoldenDir, licenseFileName)); err != nil {
+ return fmt.Errorf("Error when copying %s from out to golden dir: %s", licenseFileName, err)
}
- // Step6: Capture diff of licenses_golden/licenses_third_party.
- licensesDiffOutput, err := git.GitDir(licenseToolsDir).Git(ctx, "diff", "--no-ext-diff", filepath.Join(licensesGoldenDir, licensesThirdPartyFileName))
+ // Step6: Capture diff of licenses_golden/${licenseFileName}.
+ licensesDiffOutput, err := git.GitDir(licenseToolsDir).Git(ctx, "diff", "--no-ext-diff", filepath.Join(licensesGoldenDir, licenseFileName))
if err != nil {
- return fmt.Errorf("Error when seeing diff of golden licenses_third_party: %s", err)
+ return fmt.Errorf("Error when seeing diff of golden %s: %s", licenseFileName, err)
}
sklog.Infof("The licenses diff output is:\n%s", licensesDiffOutput)
@@ -177,7 +197,6 @@
}
sklog.Info("Done running flutter license scripts.")
- licenseScriptFailure = 0
return nil
}
@@ -185,7 +204,7 @@
func GoGenerateCipd(ctx context.Context, _ []string, client *http.Client, parentRepoDir string) error {
// TODO(borenet): Should we plumb through --local and --workdir?
sklog.Info("Installing Go...")
- goExc, goEnv, err := go_install.EnsureGo(client, cipdRoot)
+ goExc, goEnv, err := go_install.EnsureGo(ctx, client, cipdRoot)
if err != nil {
return err
}
@@ -193,7 +212,7 @@
// Also install the protoc asset. Use a different CIPD root dir to
// prevent conflicts with the Go packages.
protocRoot := path.Join(os.TempDir(), "cipd_protoc")
- if err := cipd.Ensure(client, protocRoot, cipd.PkgProtoc); err != nil {
+ if err := cipd.Ensure(ctx, client, protocRoot, cipd.PkgProtoc); err != nil {
return err
}
diff --git a/autoroll/go/repo_manager/repo_manager.go b/autoroll/go/repo_manager/repo_manager.go
index 258c83b..402f5d6 100644
--- a/autoroll/go/repo_manager/repo_manager.go
+++ b/autoroll/go/repo_manager/repo_manager.go
@@ -394,10 +394,10 @@
// cleanParent forces the parent checkout into a clean state.
func (r *depotToolsRepoManager) cleanParent(ctx context.Context) error {
- return r.cleanParentWithRemote(ctx, "origin")
+ return r.cleanParentWithRemoteAndBranch(ctx, "origin", ROLL_BRANCH)
}
-func (r *depotToolsRepoManager) cleanParentWithRemote(ctx context.Context, remote string) error {
+func (r *depotToolsRepoManager) cleanParentWithRemoteAndBranch(ctx context.Context, remote, branch string) error {
if _, err := exec.RunCwd(ctx, r.parentDir, "git", "clean", "-d", "-f", "-f"); err != nil {
return err
}
@@ -405,7 +405,7 @@
if _, err := exec.RunCwd(ctx, r.parentDir, "git", "checkout", fmt.Sprintf("%s/%s", remote, r.parentBranch), "-f"); err != nil {
return err
}
- _, _ = exec.RunCwd(ctx, r.parentDir, "git", "branch", "-D", ROLL_BRANCH)
+ _, _ = exec.RunCwd(ctx, r.parentDir, "git", "branch", "-D", branch)
if _, err := exec.RunCommand(ctx, &exec.Command{
Dir: r.workdir,
Env: r.depotToolsEnv,
@@ -418,10 +418,10 @@
}
func (r *depotToolsRepoManager) createAndSyncParent(ctx context.Context) error {
- return r.createAndSyncParentWithRemote(ctx, "origin")
+ return r.createAndSyncParentWithRemoteAndBranch(ctx, "origin", ROLL_BRANCH)
}
-func (r *depotToolsRepoManager) createAndSyncParentWithRemote(ctx context.Context, remote string) error {
+func (r *depotToolsRepoManager) createAndSyncParentWithRemoteAndBranch(ctx context.Context, remote, branch string) error {
// Create the working directory if needed.
if _, err := os.Stat(r.workdir); err != nil {
if err := os.MkdirAll(r.workdir, 0755); err != nil {
@@ -430,7 +430,7 @@
}
if _, err := os.Stat(path.Join(r.parentDir, ".git")); err == nil {
- if err := r.cleanParentWithRemote(ctx, remote); err != nil {
+ if err := r.cleanParentWithRemoteAndBranch(ctx, remote, branch); err != nil {
return err
}
// Update the repo.
diff --git a/autoroll/go/roller/autoroller.go b/autoroll/go/roller/autoroller.go
index ba91766..afd1163 100644
--- a/autoroll/go/roller/autoroller.go
+++ b/autoroll/go/roller/autoroller.go
@@ -110,6 +110,8 @@
rm, err = repo_manager.NewFuchsiaSDKRepoManager(ctx, c.FuchsiaSDKRepoManager, workdir, g, serverURL, gitcookiesPath, nil, cr, local)
} else if c.GithubRepoManager != nil {
rm, err = repo_manager.NewGithubRepoManager(ctx, c.GithubRepoManager, workdir, githubClient, recipesCfgFile, serverURL, client, cr, local)
+ } else if c.GithubCipdDEPSRepoManager != nil {
+ rm, err = repo_manager.NewGithubCipdDEPSRepoManager(ctx, c.GithubCipdDEPSRepoManager, workdir, rollerName, githubClient, recipesCfgFile, serverURL, client, cr, local)
} else if c.GithubDEPSRepoManager != nil {
rm, err = repo_manager.NewGithubDEPSRepoManager(ctx, c.GithubDEPSRepoManager, workdir, githubClient, recipesCfgFile, serverURL, client, cr, local)
} else if c.ManifestRepoManager != nil {
diff --git a/autoroll/go/roller/config.go b/autoroll/go/roller/config.go
index 6d98cb7..2dcc480 100644
--- a/autoroll/go/roller/config.go
+++ b/autoroll/go/roller/config.go
@@ -175,6 +175,7 @@
FuchsiaSDKAndroidRepoManager *repo_manager.FuchsiaSDKAndroidRepoManagerConfig `json:"fuchsiaSDKAndroidRepoManager,omitempty"`
FuchsiaSDKRepoManager *repo_manager.FuchsiaSDKRepoManagerConfig `json:"fuchsiaSDKRepoManager,omitempty"`
GithubRepoManager *repo_manager.GithubRepoManagerConfig `json:"githubRepoManager,omitempty"`
+ GithubCipdDEPSRepoManager *repo_manager.GithubCipdDEPSRepoManagerConfig `json:"githubCipdDEPSRepoManager,omitempty"`
GithubDEPSRepoManager *repo_manager.GithubDEPSRepoManagerConfig `json:"githubDEPSRepoManager,omitempty"`
Google3RepoManager *Google3FakeRepoManagerConfig `json:"google3,omitempty"`
ManifestRepoManager *repo_manager.ManifestRepoManagerConfig `json:"manifestRepoManager,omitempty"`
@@ -275,6 +276,9 @@
if c.GithubRepoManager != nil {
rm = append(rm, c.GithubRepoManager)
}
+ if c.GithubCipdDEPSRepoManager != nil {
+ rm = append(rm, c.GithubCipdDEPSRepoManager)
+ }
if c.GithubDEPSRepoManager != nil {
rm = append(rm, c.GithubDEPSRepoManager)
}
diff --git a/go/cipd/cipd.go b/go/cipd/cipd.go
index f53d8ed..569ce24 100644
--- a/go/cipd/cipd.go
+++ b/go/cipd/cipd.go
@@ -56,19 +56,48 @@
}
// Run "cipd ensure" to get the correct packages in the given location. Note
-// that any previously-installed packages in the given rootdir will be removed
+// that any previously-installed packages in the given rootDir will be removed
// if not specified again.
-func Ensure(client *http.Client, rootdir string, packages ...*Package) error {
- c, err := cipd.NewClient(cipd.ClientOptions{
- ServiceURL: SERVICE_URL,
- Root: rootdir,
- AuthenticatedClient: client,
- })
+func Ensure(ctx context.Context, c *http.Client, rootDir string, packages ...*Package) error {
+ cipdClient, err := NewClient(c, rootDir)
if err != nil {
return fmt.Errorf("Failed to create CIPD client: %s", err)
}
+ return cipdClient.Ensure(ctx, packages...)
+}
- ctx := context.Background()
+// CIPDClient is the interface for interactions with the CIPD API.
+type CIPDClient interface {
+ cipd.Client
+
+ // Ensure runs "cipd ensure" to get the correct packages in the given location. Note
+ // that any previously-installed packages in the given rootDir will be removed
+ // if not specified again.
+ Ensure(ctx context.Context, packages ...*Package) error
+
+ // Describe is a convenience wrapper around cipd.Client.DescribeInstance.
+ Describe(ctx context.Context, pkg, instance string) (*cipd.InstanceDescription, error)
+}
+
+// Client is a struct used for interacting with the CIPD API.
+type Client struct {
+ cipd.Client
+}
+
+// NewClient returns a CIPD client.
+func NewClient(c *http.Client, rootDir string) (*Client, error) {
+ cipdClient, err := cipd.NewClient(cipd.ClientOptions{
+ ServiceURL: SERVICE_URL,
+ Root: rootDir,
+ AuthenticatedClient: c,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("Failed to create CIPD client: %s", err)
+ }
+ return &Client{cipdClient}, nil
+}
+
+func (c *Client) Ensure(ctx context.Context, packages ...*Package) error {
pkgs := common.PinSliceBySubdir{}
for _, pkg := range packages {
pin, err := c.ResolveVersion(ctx, pkg.Name, pkg.Version)
@@ -78,8 +107,20 @@
sklog.Infof("Installing version %s (from %s) of %s", pin.InstanceID, pkg.Version, pkg.Name)
pkgs[pkg.Dest] = common.PinSlice{pin}
}
- if _, err := c.EnsurePackages(context.Background(), pkgs, cipd.CheckPresence, false); err != nil {
+ if _, err := c.EnsurePackages(ctx, pkgs, cipd.CheckPresence, false); err != nil {
return fmt.Errorf("Failed to ensure packages: %s", err)
}
return nil
}
+
+func (c *Client) Describe(ctx context.Context, pkg, instance string) (*cipd.InstanceDescription, error) {
+ pin := common.Pin{
+ PackageName: pkg,
+ InstanceID: instance,
+ }
+ opts := &cipd.DescribeInstanceOpts{
+ DescribeRefs: true,
+ DescribeTags: true,
+ }
+ return c.DescribeInstance(ctx, pin, opts)
+}
diff --git a/go/cipd/mocks/CIPDClient.go b/go/cipd/mocks/CIPDClient.go
new file mode 100644
index 0000000..2b05b65
--- /dev/null
+++ b/go/cipd/mocks/CIPDClient.go
@@ -0,0 +1,453 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+package mocks
+
+import cipd "go.skia.org/infra/go/cipd"
+import clientcipd "go.chromium.org/luci/cipd/client/cipd"
+import common "go.chromium.org/luci/cipd/common"
+import context "context"
+import deployer "go.chromium.org/luci/cipd/client/cipd/deployer"
+import io "io"
+import mock "github.com/stretchr/testify/mock"
+import pkg "go.chromium.org/luci/cipd/client/cipd/pkg"
+import time "time"
+
+// CIPDClient is an autogenerated mock type for the CIPDClient type
+type CIPDClient struct {
+ mock.Mock
+}
+
+// AttachTagsWhenReady provides a mock function with given fields: ctx, pin, tags
+func (_m *CIPDClient) AttachTagsWhenReady(ctx context.Context, pin common.Pin, tags []string) error {
+ ret := _m.Called(ctx, pin, tags)
+
+ var r0 error
+ if rf, ok := ret.Get(0).(func(context.Context, common.Pin, []string) error); ok {
+ r0 = rf(ctx, pin, tags)
+ } else {
+ r0 = ret.Error(0)
+ }
+
+ return r0
+}
+
+// BeginBatch provides a mock function with given fields: ctx
+func (_m *CIPDClient) BeginBatch(ctx context.Context) {
+ _m.Called(ctx)
+}
+
+// CheckDeployment provides a mock function with given fields: ctx, paranoia
+func (_m *CIPDClient) CheckDeployment(ctx context.Context, paranoia deployer.ParanoidMode) (clientcipd.ActionMap, error) {
+ ret := _m.Called(ctx, paranoia)
+
+ var r0 clientcipd.ActionMap
+ if rf, ok := ret.Get(0).(func(context.Context, deployer.ParanoidMode) clientcipd.ActionMap); ok {
+ r0 = rf(ctx, paranoia)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).(clientcipd.ActionMap)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, deployer.ParanoidMode) error); ok {
+ r1 = rf(ctx, paranoia)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// Describe provides a mock function with given fields: ctx, _a1, instance
+func (_m *CIPDClient) Describe(ctx context.Context, _a1 string, instance string) (*clientcipd.InstanceDescription, error) {
+ ret := _m.Called(ctx, _a1, instance)
+
+ var r0 *clientcipd.InstanceDescription
+ if rf, ok := ret.Get(0).(func(context.Context, string, string) *clientcipd.InstanceDescription); ok {
+ r0 = rf(ctx, _a1, instance)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).(*clientcipd.InstanceDescription)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok {
+ r1 = rf(ctx, _a1, instance)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// DescribeClient provides a mock function with given fields: ctx, pin
+func (_m *CIPDClient) DescribeClient(ctx context.Context, pin common.Pin) (*clientcipd.ClientDescription, error) {
+ ret := _m.Called(ctx, pin)
+
+ var r0 *clientcipd.ClientDescription
+ if rf, ok := ret.Get(0).(func(context.Context, common.Pin) *clientcipd.ClientDescription); ok {
+ r0 = rf(ctx, pin)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).(*clientcipd.ClientDescription)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, common.Pin) error); ok {
+ r1 = rf(ctx, pin)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// DescribeInstance provides a mock function with given fields: ctx, pin, opts
+func (_m *CIPDClient) DescribeInstance(ctx context.Context, pin common.Pin, opts *clientcipd.DescribeInstanceOpts) (*clientcipd.InstanceDescription, error) {
+ ret := _m.Called(ctx, pin, opts)
+
+ var r0 *clientcipd.InstanceDescription
+ if rf, ok := ret.Get(0).(func(context.Context, common.Pin, *clientcipd.DescribeInstanceOpts) *clientcipd.InstanceDescription); ok {
+ r0 = rf(ctx, pin, opts)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).(*clientcipd.InstanceDescription)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, common.Pin, *clientcipd.DescribeInstanceOpts) error); ok {
+ r1 = rf(ctx, pin, opts)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// EndBatch provides a mock function with given fields: ctx
+func (_m *CIPDClient) EndBatch(ctx context.Context) {
+ _m.Called(ctx)
+}
+
+// Ensure provides a mock function with given fields: ctx, packages
+func (_m *CIPDClient) Ensure(ctx context.Context, packages ...*cipd.Package) error {
+ _va := make([]interface{}, len(packages))
+ for _i := range packages {
+ _va[_i] = packages[_i]
+ }
+ var _ca []interface{}
+ _ca = append(_ca, ctx)
+ _ca = append(_ca, _va...)
+ ret := _m.Called(_ca...)
+
+ var r0 error
+ if rf, ok := ret.Get(0).(func(context.Context, ...*cipd.Package) error); ok {
+ r0 = rf(ctx, packages...)
+ } else {
+ r0 = ret.Error(0)
+ }
+
+ return r0
+}
+
+// EnsurePackages provides a mock function with given fields: ctx, pkgs, paranoia, dryRun
+func (_m *CIPDClient) EnsurePackages(ctx context.Context, pkgs common.PinSliceBySubdir, paranoia deployer.ParanoidMode, dryRun bool) (clientcipd.ActionMap, error) {
+ ret := _m.Called(ctx, pkgs, paranoia, dryRun)
+
+ var r0 clientcipd.ActionMap
+ if rf, ok := ret.Get(0).(func(context.Context, common.PinSliceBySubdir, deployer.ParanoidMode, bool) clientcipd.ActionMap); ok {
+ r0 = rf(ctx, pkgs, paranoia, dryRun)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).(clientcipd.ActionMap)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, common.PinSliceBySubdir, deployer.ParanoidMode, bool) error); ok {
+ r1 = rf(ctx, pkgs, paranoia, dryRun)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// FetchACL provides a mock function with given fields: ctx, prefix
+func (_m *CIPDClient) FetchACL(ctx context.Context, prefix string) ([]clientcipd.PackageACL, error) {
+ ret := _m.Called(ctx, prefix)
+
+ var r0 []clientcipd.PackageACL
+ if rf, ok := ret.Get(0).(func(context.Context, string) []clientcipd.PackageACL); ok {
+ r0 = rf(ctx, prefix)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).([]clientcipd.PackageACL)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
+ r1 = rf(ctx, prefix)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// FetchAndDeployInstance provides a mock function with given fields: ctx, subdir, pin
+func (_m *CIPDClient) FetchAndDeployInstance(ctx context.Context, subdir string, pin common.Pin) error {
+ ret := _m.Called(ctx, subdir, pin)
+
+ var r0 error
+ if rf, ok := ret.Get(0).(func(context.Context, string, common.Pin) error); ok {
+ r0 = rf(ctx, subdir, pin)
+ } else {
+ r0 = ret.Error(0)
+ }
+
+ return r0
+}
+
+// FetchInstance provides a mock function with given fields: ctx, pin
+func (_m *CIPDClient) FetchInstance(ctx context.Context, pin common.Pin) (pkg.Source, error) {
+ ret := _m.Called(ctx, pin)
+
+ var r0 pkg.Source
+ if rf, ok := ret.Get(0).(func(context.Context, common.Pin) pkg.Source); ok {
+ r0 = rf(ctx, pin)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).(pkg.Source)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, common.Pin) error); ok {
+ r1 = rf(ctx, pin)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// FetchInstanceTo provides a mock function with given fields: ctx, pin, output
+func (_m *CIPDClient) FetchInstanceTo(ctx context.Context, pin common.Pin, output io.WriteSeeker) error {
+ ret := _m.Called(ctx, pin, output)
+
+ var r0 error
+ if rf, ok := ret.Get(0).(func(context.Context, common.Pin, io.WriteSeeker) error); ok {
+ r0 = rf(ctx, pin, output)
+ } else {
+ r0 = ret.Error(0)
+ }
+
+ return r0
+}
+
+// FetchPackageRefs provides a mock function with given fields: ctx, packageName
+func (_m *CIPDClient) FetchPackageRefs(ctx context.Context, packageName string) ([]clientcipd.RefInfo, error) {
+ ret := _m.Called(ctx, packageName)
+
+ var r0 []clientcipd.RefInfo
+ if rf, ok := ret.Get(0).(func(context.Context, string) []clientcipd.RefInfo); ok {
+ r0 = rf(ctx, packageName)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).([]clientcipd.RefInfo)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
+ r1 = rf(ctx, packageName)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// FetchRoles provides a mock function with given fields: ctx, prefix
+func (_m *CIPDClient) FetchRoles(ctx context.Context, prefix string) ([]string, error) {
+ ret := _m.Called(ctx, prefix)
+
+ var r0 []string
+ if rf, ok := ret.Get(0).(func(context.Context, string) []string); ok {
+ r0 = rf(ctx, prefix)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).([]string)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
+ r1 = rf(ctx, prefix)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// ListInstances provides a mock function with given fields: ctx, packageName
+func (_m *CIPDClient) ListInstances(ctx context.Context, packageName string) (clientcipd.InstanceEnumerator, error) {
+ ret := _m.Called(ctx, packageName)
+
+ var r0 clientcipd.InstanceEnumerator
+ if rf, ok := ret.Get(0).(func(context.Context, string) clientcipd.InstanceEnumerator); ok {
+ r0 = rf(ctx, packageName)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).(clientcipd.InstanceEnumerator)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
+ r1 = rf(ctx, packageName)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// ListPackages provides a mock function with given fields: ctx, prefix, recursive, includeHidden
+func (_m *CIPDClient) ListPackages(ctx context.Context, prefix string, recursive bool, includeHidden bool) ([]string, error) {
+ ret := _m.Called(ctx, prefix, recursive, includeHidden)
+
+ var r0 []string
+ if rf, ok := ret.Get(0).(func(context.Context, string, bool, bool) []string); ok {
+ r0 = rf(ctx, prefix, recursive, includeHidden)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).([]string)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, string, bool, bool) error); ok {
+ r1 = rf(ctx, prefix, recursive, includeHidden)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// ModifyACL provides a mock function with given fields: ctx, prefix, changes
+func (_m *CIPDClient) ModifyACL(ctx context.Context, prefix string, changes []clientcipd.PackageACLChange) error {
+ ret := _m.Called(ctx, prefix, changes)
+
+ var r0 error
+ if rf, ok := ret.Get(0).(func(context.Context, string, []clientcipd.PackageACLChange) error); ok {
+ r0 = rf(ctx, prefix, changes)
+ } else {
+ r0 = ret.Error(0)
+ }
+
+ return r0
+}
+
+// RegisterInstance provides a mock function with given fields: ctx, pin, body, timeout
+func (_m *CIPDClient) RegisterInstance(ctx context.Context, pin common.Pin, body io.ReadSeeker, timeout time.Duration) error {
+ ret := _m.Called(ctx, pin, body, timeout)
+
+ var r0 error
+ if rf, ok := ret.Get(0).(func(context.Context, common.Pin, io.ReadSeeker, time.Duration) error); ok {
+ r0 = rf(ctx, pin, body, timeout)
+ } else {
+ r0 = ret.Error(0)
+ }
+
+ return r0
+}
+
+// RepairDeployment provides a mock function with given fields: ctx, paranoia
+func (_m *CIPDClient) RepairDeployment(ctx context.Context, paranoia deployer.ParanoidMode) (clientcipd.ActionMap, error) {
+ ret := _m.Called(ctx, paranoia)
+
+ var r0 clientcipd.ActionMap
+ if rf, ok := ret.Get(0).(func(context.Context, deployer.ParanoidMode) clientcipd.ActionMap); ok {
+ r0 = rf(ctx, paranoia)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).(clientcipd.ActionMap)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, deployer.ParanoidMode) error); ok {
+ r1 = rf(ctx, paranoia)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// ResolveVersion provides a mock function with given fields: ctx, packageName, version
+func (_m *CIPDClient) ResolveVersion(ctx context.Context, packageName string, version string) (common.Pin, error) {
+ ret := _m.Called(ctx, packageName, version)
+
+ var r0 common.Pin
+ if rf, ok := ret.Get(0).(func(context.Context, string, string) common.Pin); ok {
+ r0 = rf(ctx, packageName, version)
+ } else {
+ r0 = ret.Get(0).(common.Pin)
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok {
+ r1 = rf(ctx, packageName, version)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// SearchInstances provides a mock function with given fields: ctx, packageName, tags
+func (_m *CIPDClient) SearchInstances(ctx context.Context, packageName string, tags []string) (common.PinSlice, error) {
+ ret := _m.Called(ctx, packageName, tags)
+
+ var r0 common.PinSlice
+ if rf, ok := ret.Get(0).(func(context.Context, string, []string) common.PinSlice); ok {
+ r0 = rf(ctx, packageName, tags)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).(common.PinSlice)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, string, []string) error); ok {
+ r1 = rf(ctx, packageName, tags)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// SetRefWhenReady provides a mock function with given fields: ctx, ref, pin
+func (_m *CIPDClient) SetRefWhenReady(ctx context.Context, ref string, pin common.Pin) error {
+ ret := _m.Called(ctx, ref, pin)
+
+ var r0 error
+ if rf, ok := ret.Get(0).(func(context.Context, string, common.Pin) error); ok {
+ r0 = rf(ctx, ref, pin)
+ } else {
+ r0 = ret.Error(0)
+ }
+
+ return r0
+}
diff --git a/go/cipd/mocks/generate.go b/go/cipd/mocks/generate.go
new file mode 100644
index 0000000..59240cd
--- /dev/null
+++ b/go/cipd/mocks/generate.go
@@ -0,0 +1,3 @@
+package mocks
+
+//go:generate mockery -name CIPDClient -dir ../ -output .
diff --git a/go/go_install/go_install.go b/go/go_install/go_install.go
index 2589655..4fc3718 100644
--- a/go/go_install/go_install.go
+++ b/go/go_install/go_install.go
@@ -5,6 +5,7 @@
*/
import (
+ "context"
"fmt"
"net/http"
"path"
@@ -17,9 +18,9 @@
// environment variables or any errors which occurred. If includeDeps is true,
// also installs the go_deps CIPD package which contains all dependencies for
// the go.skia.org/infra repository as of the last update of that package.
-func EnsureGo(client *http.Client, cipdRoot string) (string, map[string]string, error) {
+func EnsureGo(ctx context.Context, client *http.Client, cipdRoot string) (string, map[string]string, error) {
pkgs := []*cipd.Package{cipd.PkgGo}
- if err := cipd.Ensure(client, cipdRoot, pkgs...); err != nil {
+ if err := cipd.Ensure(ctx, client, cipdRoot, pkgs...); err != nil {
return "", nil, fmt.Errorf("Failed to ensure Go CIPD package: %s", err)
}
goRoot := path.Join(cipdRoot, cipd.PkgGo.Dest, "go")
