blob: 4805353e81a0ab362533eff0d024660f202f88f8 [file] [log] [blame]
package repo_manager
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"html/template"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"strconv"
"strings"
"time"
"go.skia.org/infra/autoroll/go/codereview"
"go.skia.org/infra/autoroll/go/strategy"
"go.skia.org/infra/go/cipd"
"go.skia.org/infra/go/exec"
"go.skia.org/infra/go/gerrit"
"go.skia.org/infra/go/git"
"go.skia.org/infra/go/sklog"
"go.skia.org/infra/go/util"
)
/*
Repo Manager which rolls infra asset versions.
*/
const (
// Path to asset VERSION files within a given repo.
ASSET_VERSION_TMPL = "infra/bots/assets/%s/VERSION"
// Commit message for AssetRepoManager.
// TODO(borenet): autoroll.FromGerritChangeInfo wants to parse the
// revisions from the roll subject. In this case, it makes no sense
// bit it's still required for the roller to operate. We should find
// a way around it so that we can be more free with our commit message.
TMPL_COMMIT_MESSAGE_ASSETS = `Roll {{.Asset}} {{.From}}..{{.To}}
The AutoRoll server is located here: {{.ServerURL}}
Documentation for the AutoRoller is here:
https://skia.googlesource.com/buildbot/+/master/autoroll/README.md
If the roll is causing failures, please contact the current sheriff, who should
be CC'd on the roll, and stop the roller if necessary.
TBR={{.Emails}}
`
)
var (
commitMsgTmplAssets = template.Must(template.New("commitMsgAssets").Parse(TMPL_COMMIT_MESSAGE_ASSETS))
)
// AssetRepoManagerConfig provides configuration for the AssetRepoManager.
type AssetRepoManagerConfig struct {
DepotToolsRepoManagerConfig
// Which assets to roll.
Asset string `json:"asset"`
// URL of the child repo.
ChildRepo string `json:"childRepo"`
}
// Validate the AssetRepoManagerConfig.
func (c *AssetRepoManagerConfig) Validate() error {
if err := c.DepotToolsRepoManagerConfig.Validate(); err != nil {
return err
}
if c.Asset == "" {
return errors.New("Asset is required.")
}
if c.ChildRepo == "" {
return errors.New("ChildRepo is required.")
}
return nil
}
// assetRepoManager is a RepoManager which rolls infra asset versions so that
// the "parent" repo's versions match the "child" repo's versions.
type assetRepoManager struct {
*depotToolsRepoManager
asset string
}
// NewAssetRepoManager returns a RepoManager instance which rolls infra asset
// versions so that the "parent" repo's versions match the "child" repo's
// versions.
func NewAssetRepoManager(ctx context.Context, c *AssetRepoManagerConfig, workdir string, g gerrit.GerritInterface, recipeCfgFile, serverURL string, client *http.Client, cr codereview.CodeReview, local bool) (RepoManager, error) {
if err := c.Validate(); err != nil {
return nil, err
}
drm, err := newDepotToolsRepoManager(ctx, c.DepotToolsRepoManagerConfig, workdir, recipeCfgFile, serverURL, g, client, cr, local)
if err != nil {
return nil, err
}
rv := &assetRepoManager{
depotToolsRepoManager: drm,
asset: c.Asset,
}
// By default, depotToolsRepoManager assumes that the child repo is
// nested inside the parent repo and is obtained via "gclient sync".
// This is not the case for AssetRepoManager, so we have to sync it
// ourselves.
childRepo, err := git.NewCheckout(ctx, c.ChildRepo, workdir)
if err != nil {
return nil, err
}
rv.childRepo = childRepo
rv.childDir = childRepo.Dir()
return rv, nil
}
// See documentation for noCheckoutRepoManagerBuildCommitMessageFunc.
func (rm *assetRepoManager) buildCommitMessage(from, to, serverURL, cqExtraTrybots string, emails []string) (string, error) {
rm.infoMtx.RLock()
defer rm.infoMtx.RUnlock()
// Build the commit message.
data := struct {
Asset string
Emails string
From string
ServerURL string
To string
}{
Asset: rm.asset,
Emails: strings.Join(emails, ","),
From: from,
ServerURL: serverURL,
To: to,
}
var buf bytes.Buffer
if err := commitMsgTmplAssets.Execute(&buf, data); err != nil {
return "", err
}
msg := buf.String()
if cqExtraTrybots != "" {
msg += fmt.Sprintf("\nCQ_INCLUDE_TRYBOTS=%s", cqExtraTrybots)
}
return buf.String(), nil
}
// Read the version of the given asset from the given repo at the given commit.
func readAssetVersion(ctx context.Context, repo git.GitDir, commit, asset string) (string, error) {
return repo.GetFile(ctx, fmt.Sprintf(ASSET_VERSION_TMPL, asset), commit)
}
// See documentation for RepoManager interface.
func (rm *assetRepoManager) Update(ctx context.Context) error {
rm.repoMtx.Lock()
defer rm.repoMtx.Unlock()
if err := rm.createAndSyncParent(ctx); err != nil {
return fmt.Errorf("Could not create and sync parent repo: %s", err)
}
// By default, depotToolsRepoManager assumes that the child repo is
// nested inside the parent repo and is obtained via "gclient sync".
// This is not the case for AssetRepoManager, so we have to sync it
// ourselves.
if err := rm.childRepo.Update(ctx); err != nil {
return err
}
// Obtain the hash of the current parent and child HEADs.
parentHead, err := git.GitDir(rm.parentDir).RevParse(ctx, "origin/"+rm.parentBranch)
if err != nil {
return err
}
childHead, err := rm.childRepo.RevParse(ctx, "origin/"+rm.childBranch)
if err != nil {
return err
}
// Read the asset versions from both repos.
lastRollRev, err := readAssetVersion(ctx, git.GitDir(rm.parentDir), parentHead, rm.asset)
if err != nil {
return err
}
lastRollRev = strings.TrimSpace(lastRollRev)
nextRollRev, err := readAssetVersion(ctx, git.GitDir(rm.childDir), childHead, rm.asset)
if err != nil {
return err
}
nextRollRev = strings.TrimSpace(nextRollRev)
// Subtract the last-rolled version number from the next version number
// to obtain commitsNotRolled.
lastInt, err := strconv.Atoi(lastRollRev)
if err != nil {
return err
}
nextInt, err := strconv.Atoi(nextRollRev)
if err != nil {
return err
}
commitsNotRolled := nextInt - lastInt
rm.infoMtx.Lock()
defer rm.infoMtx.Unlock()
rm.lastRollRev = lastRollRev
rm.nextRollRev = nextRollRev
rm.commitsNotRolled = commitsNotRolled
return nil
}
// See documentation for RepoManager interface.
func (rm *assetRepoManager) CreateNewRoll(ctx context.Context, from, to string, emails []string, cqExtraTrybots string, dryRun bool) (int64, error) {
rm.repoMtx.Lock()
defer rm.repoMtx.Unlock()
// Clean the checkout, get onto a fresh branch.
if err := rm.cleanParent(ctx); err != nil {
return 0, err
}
if _, err := exec.RunCwd(ctx, rm.parentDir, "git", "checkout", "-b", ROLL_BRANCH, "-t", fmt.Sprintf("origin/%s", rm.parentBranch), "-f"); err != nil {
return 0, err
}
// Defer some more cleanup.
defer func() {
util.LogErr(rm.cleanParent(ctx))
}()
// Create the roll CL.
if !rm.local {
if _, err := exec.RunCwd(ctx, rm.parentDir, "git", "config", "user.name", rm.codereview.UserName()); err != nil {
return 0, err
}
if _, err := exec.RunCwd(ctx, rm.parentDir, "git", "config", "user.email", rm.codereview.UserEmail()); err != nil {
return 0, err
}
}
assetFile := path.Join(rm.parentDir, fmt.Sprintf(ASSET_VERSION_TMPL, rm.asset))
if err := util.WithWriteFile(assetFile, func(w io.Writer) error {
_, err := w.Write([]byte(to))
return err
}); err != nil {
return 0, err
}
sklog.Infof("Rolling package %s from %s to %s.", rm.asset, from, to)
// Before running the pre-upload steps, which install Go and the infra
// repo's dependencies via CIPD, update the version of the Go DEPS CIPD
// package to install the one requested by the roll.
// TODO(borenet): Rather than modifying this global variable, which
// could affect other callers (though there aren't any at the time of
// writing), find a way to plumb the desired version through the
// pre-upload step.
cipd.PkgGoDEPS.Version = cipd.VersionTag(to)
// Run the pre-upload steps.
sklog.Infof("Running pre-upload steps.")
for _, s := range rm.PreUploadSteps() {
if err := s(ctx, rm.httpClient, rm.parentDir); err != nil {
return 0, fmt.Errorf("Failed pre-upload step: %s", err)
}
}
// Commit.
commitMsg, err := rm.buildCommitMessage(from, to, rm.serverURL, cqExtraTrybots, emails)
if err != nil {
return 0, err
}
if _, err := exec.RunCwd(ctx, rm.parentDir, "git", "commit", "-a", "-m", commitMsg); err != nil {
return 0, err
}
// Upload the CL.
uploadCmd := &exec.Command{
Dir: rm.parentDir,
Env: rm.depotToolsEnv,
InheritEnv: true,
Name: "git",
Args: []string{"cl", "upload", "--bypass-hooks", "-f", "-v", "-v"},
Timeout: 2 * time.Minute,
}
if dryRun {
uploadCmd.Args = append(uploadCmd.Args, "--cq-dry-run")
} else {
uploadCmd.Args = append(uploadCmd.Args, "--use-commit-queue")
}
uploadCmd.Args = append(uploadCmd.Args, "--gerrit")
uploadCmd.Args = append(uploadCmd.Args, "-m", commitMsg)
// Upload the CL.
sklog.Infof("Running command: git %s", strings.Join(uploadCmd.Args, " "))
if _, err := exec.RunCommand(ctx, uploadCmd); err != nil {
return 0, err
}
// Obtain the issue number.
sklog.Infof("Retrieving issue number of uploaded CL.")
tmp, err := ioutil.TempDir("", "")
if err != nil {
return 0, err
}
defer util.RemoveAll(tmp)
jsonFile := path.Join(tmp, "issue.json")
if _, err := exec.RunCommand(ctx, &exec.Command{
Dir: rm.parentDir,
Env: rm.depotToolsEnv,
InheritEnv: true,
Name: "git",
Args: []string{"cl", "issue", fmt.Sprintf("--json=%s", jsonFile)},
}); err != nil {
return 0, err
}
f, err := os.Open(jsonFile)
if err != nil {
return 0, err
}
var issue issueJson
if err := json.NewDecoder(f).Decode(&issue); err != nil {
return 0, err
}
return issue.Issue, nil
}
// See documentation for RepoManager interface.
func (rm *assetRepoManager) FullChildHash(ctx context.Context, version string) (string, error) {
// We're rolling asset version numbers, not commit hashes, and we don't
// shorten them. Return the given version.
return version, nil
}
// See documentation for RepoManager interface.
func (r *assetRepoManager) CreateNextRollStrategy(ctx context.Context, s string) (strategy.NextRollStrategy, error) {
return strategy.GetNextRollStrategy(ctx, s, r.childBranch, DEFAULT_REMOTE, "", []string{}, nil, nil)
}
// See documentation for RepoManager interface.
func (r *assetRepoManager) SetStrategy(s strategy.NextRollStrategy) {
r.strategyMtx.Lock()
defer r.strategyMtx.Unlock()
r.strategy = s
}
// See documentation for RepoManager interface.
func (r *assetRepoManager) DefaultStrategy() string {
return strategy.ROLL_STRATEGY_BATCH
}
// See documentation for RepoManager interface.
func (r *assetRepoManager) ValidStrategies() []string {
return []string{
strategy.ROLL_STRATEGY_BATCH,
}
}
// See documentation for RepoManager interface.
func (rm *assetRepoManager) RolledPast(ctx context.Context, version string) (bool, error) {
lastInt, err := strconv.Atoi(rm.lastRollRev)
if err != nil {
return false, err
}
versionInt, err := strconv.Atoi(version)
if err != nil {
return false, err
}
return lastInt >= versionInt, nil
}