blob: b61eb650af14601ef628b323db32e01dba1dbf54 [file] [log] [blame]
package repo_manager
import (
"bytes"
"context"
"errors"
"fmt"
"io/ioutil"
"net/http"
"os"
"path"
"regexp"
"strings"
"go.skia.org/infra/autoroll/go/codereview"
"go.skia.org/infra/autoroll/go/revision"
"go.skia.org/infra/go/depot_tools"
"go.skia.org/infra/go/exec"
"go.skia.org/infra/go/gerrit"
"go.skia.org/infra/go/gitiles"
"go.skia.org/infra/go/issues"
"go.skia.org/infra/go/skerr"
"go.skia.org/infra/go/sklog"
"go.skia.org/infra/go/util"
"go.skia.org/infra/go/vcsinfo"
)
var (
// Use this function to instantiate a RepoManager. This is able to be
// overridden for testing.
NewNoCheckoutDEPSRepoManager func(context.Context, *NoCheckoutDEPSRepoManagerConfig, string, gerrit.GerritInterface, string, string, *http.Client, codereview.CodeReview, bool) (RepoManager, error) = newNoCheckoutDEPSRepoManager
getDepRegex = regexp.MustCompile("[a-f0-9]+")
)
// NoCheckoutDEPSRepoManagerConfig provides configuration for RepoManagers which
// don't use a local checkout.
type NoCheckoutDEPSRepoManagerConfig struct {
NoCheckoutRepoManagerConfig
// URL of the child repo.
ChildRepo string `json:"childRepo"` // TODO(borenet): Can we just get this from DEPS?
// If false, roll CLs do not link to bugs from the commits in the child
// repo.
IncludeBugs bool `json:"includeBugs"`
// If false, roll CLs do not include a git log.
IncludeLog bool `json:"includeLog"`
// Optional; transitive dependencies to roll. This is a mapping of
// dependencies of the child repo which are also dependencies of the
// parent repo and should be rolled at the same time. Keys are paths
// to transitive dependencies within the child repo (as specified in
// DEPS), and values are paths to those dependencies within the parent
// repo.
TransitiveDeps map[string]string `json:"transitiveDeps"`
}
func (c *NoCheckoutDEPSRepoManagerConfig) Validate() error {
if err := c.NoCheckoutRepoManagerConfig.Validate(); err != nil {
return err
}
if c.ChildRepo == "" {
return errors.New("ChildRepo is required.")
}
if c.ParentBranch == "" {
return errors.New("ParentBranch is required.")
}
if c.ParentRepo == "" {
return errors.New("ParentRepo is required.")
}
for _, s := range c.PreUploadSteps {
if _, err := GetPreUploadStep(s); err != nil {
return err
}
}
return nil
}
type noCheckoutDEPSRepoManager struct {
*noCheckoutRepoManager
childRepo *gitiles.Repo
childRepoUrl string
depotTools string
gclient string
includeBugs bool
includeLog bool
parentRepoUrl string
transitiveDeps map[string]string
}
// newNoCheckoutDEPSRepoManager returns a RepoManager instance which does not use
// a local checkout.
func newNoCheckoutDEPSRepoManager(ctx context.Context, c *NoCheckoutDEPSRepoManagerConfig, workdir string, g gerrit.GerritInterface, recipeCfgFile, serverURL string, client *http.Client, cr codereview.CodeReview, local bool) (RepoManager, error) {
if err := c.Validate(); err != nil {
return nil, err
}
if err := os.MkdirAll(workdir, os.ModePerm); err != nil {
return nil, err
}
depotTools, err := depot_tools.GetDepotTools(ctx, workdir, recipeCfgFile)
if err != nil {
return nil, err
}
rv := &noCheckoutDEPSRepoManager{
childRepo: gitiles.NewRepo(c.ChildRepo, client),
childRepoUrl: c.ChildRepo,
depotTools: depotTools,
gclient: path.Join(depotTools, GCLIENT),
includeBugs: c.IncludeBugs,
includeLog: c.IncludeLog,
parentRepoUrl: c.ParentRepo,
transitiveDeps: c.TransitiveDeps,
}
ncrm, err := newNoCheckoutRepoManager(ctx, c.NoCheckoutRepoManagerConfig, workdir, g, serverURL, client, cr, rv.createRoll, rv.updateHelper, local)
if err != nil {
return nil, err
}
rv.noCheckoutRepoManager = ncrm
return rv, nil
}
// getDEPSFile downloads and returns the path to the DEPS file, and a cleanup
// function to run when finished with it. Requires that the caller holds
// rm.infoMtx.
func (rm *noCheckoutDEPSRepoManager) getDEPSFile(ctx context.Context, repo *gitiles.Repo, baseCommit string) (rv string, cleanup func(), rvErr error) {
wd, err := ioutil.TempDir("", "")
if err != nil {
return "", nil, err
}
defer func() {
if rvErr != nil {
util.RemoveAll(wd)
}
}()
// Download the DEPS file from the parent repo.
buf := bytes.NewBuffer([]byte{})
if err := repo.ReadFileAtRef(ctx, "DEPS", baseCommit, buf); err != nil {
return "", nil, err
}
// Use "gclient getdep" to retrieve the last roll revision.
// "gclient getdep" requires a .gclient file.
if _, err := exec.RunCwd(ctx, wd, "python", rm.gclient, "config", repo.URL); err != nil {
return "", nil, err
}
splitRepo := strings.Split(repo.URL, "/")
fakeCheckoutDir := path.Join(wd, strings.TrimSuffix(splitRepo[len(splitRepo)-1], ".git"))
if err := os.Mkdir(fakeCheckoutDir, os.ModePerm); err != nil {
return "", nil, err
}
depsFile := path.Join(fakeCheckoutDir, "DEPS")
if err := ioutil.WriteFile(depsFile, buf.Bytes(), os.ModePerm); err != nil {
return "", nil, err
}
return depsFile, func() { util.RemoveAll(wd) }, nil
}
// See documentation for noCheckoutRepoManagerCreateRollHelperFunc.
func (rm *noCheckoutDEPSRepoManager) createRoll(ctx context.Context, from, to *revision.Revision, rolling []*revision.Revision, serverURL, cqExtraTrybots string, emails []string) (string, map[string]string, error) {
rm.infoMtx.RLock()
defer rm.infoMtx.RUnlock()
// Download the DEPS file from the parent repo.
depsFile, cleanup, err := rm.getDEPSFile(ctx, rm.parentRepo, rm.baseCommit)
if err != nil {
return "", nil, err
}
defer cleanup()
// Write the new DEPS content.
if err := rm.setdep(ctx, depsFile, rm.childPath, to.Id); err != nil {
return "", nil, err
}
// Update any transitive DEPS.
transitiveDeps := []*TransitiveDep{}
if len(rm.transitiveDeps) > 0 {
childDepsFile, childCleanup, err := rm.getDEPSFile(ctx, rm.childRepo, to.Id)
if err != nil {
return "", nil, err
}
defer childCleanup()
for childPath, parentPath := range rm.transitiveDeps {
newRev, err := rm.getdep(ctx, childDepsFile, childPath)
if err != nil {
return "", nil, err
}
oldRev, err := rm.getdep(ctx, depsFile, parentPath)
if err != nil {
return "", nil, err
}
if oldRev != newRev {
if err := rm.setdep(ctx, depsFile, parentPath, newRev); err != nil {
return "", nil, err
}
transitiveDeps = append(transitiveDeps, &TransitiveDep{
ParentPath: parentPath,
RollingFrom: oldRev,
RollingTo: newRev,
})
}
}
}
// Read the updated DEPS content.
newDEPSContent, err := ioutil.ReadFile(depsFile)
if err != nil {
return "", nil, err
}
// Build the commit message.
bugs := []string{}
monorailProject := issues.REPO_PROJECT_MAPPING[rm.parentRepoUrl]
if monorailProject == "" {
sklog.Warningf("Found no entry in issues.REPO_PROJECT_MAPPING for %q", rm.parentRepoUrl)
}
nextRollCommits := make([]*vcsinfo.LongCommit, 0, len(rolling))
for _, rev := range rolling {
c, err := rm.childRepo.Details(ctx, rev.Id)
if err != nil {
return "", nil, err
}
nextRollCommits = append(nextRollCommits, c)
}
for _, c := range nextRollCommits {
if rm.includeBugs && monorailProject != "" {
b := util.BugsFromCommitMsg(c.Body)
for _, bug := range b[monorailProject] {
bugs = append(bugs, fmt.Sprintf("%s:%s", monorailProject, bug))
}
}
}
commitMsg, err := rm.buildCommitMsg(&CommitMsgVars{
Bugs: bugs,
ChildPath: rm.childPath,
ChildRepo: rm.childRepoUrl,
CqExtraTrybots: cqExtraTrybots,
IncludeLog: rm.includeLog,
Reviewers: emails,
Revisions: rolling,
RollingFrom: from,
RollingTo: to,
ServerURL: serverURL,
TransitiveDeps: transitiveDeps,
})
if err != nil {
return "", nil, fmt.Errorf("Failed to build commit msg: %s", err)
}
return commitMsg, map[string]string{"DEPS": string(newDEPSContent)}, nil
}
func (rm *noCheckoutDEPSRepoManager) getdep(ctx context.Context, depsFile, depPath string) (string, error) {
output, err := exec.RunCwd(ctx, path.Dir(depsFile), "python", rm.gclient, "getdep", "-r", depPath)
if err != nil {
return "", err
}
splitGetdep := strings.Split(strings.TrimSpace(output), "\n")
rev := strings.TrimSpace(splitGetdep[len(splitGetdep)-1])
if getDepRegex.MatchString(rev) {
if len(rev) == 40 {
return rev, nil
}
// The DEPS entry may be a shortened commit hash. Try to resolve
// the full hash.
rev, err = rm.childRepo.ResolveRef(ctx, rev)
if err != nil {
return "", skerr.Wrapf(err, "`gclient getdep` produced what appears to be a shortened commit hash, but failed to resolve it as a commit via gitiles. Output of `gclient getdep`:\n%s", output)
}
return rev, nil
}
return "", fmt.Errorf("Got invalid output for `gclient getdep`: %s", output)
}
func (rm *noCheckoutDEPSRepoManager) setdep(ctx context.Context, depsFile, depPath, rev string) error {
args := []string{"setdep", "-r", fmt.Sprintf("%s@%s", depPath, rev)}
_, err := exec.RunCommand(ctx, &exec.Command{
Dir: path.Dir(depsFile),
Env: depot_tools.Env(rm.depotTools),
Name: rm.gclient,
Args: args,
})
return err
}
// See documentation for noCheckoutRepoManagerUpdateHelperFunc.
func (rm *noCheckoutDEPSRepoManager) updateHelper(ctx context.Context, parentRepo *gitiles.Repo, baseCommit string) (*revision.Revision, *revision.Revision, []*revision.Revision, error) {
rm.infoMtx.Lock()
defer rm.infoMtx.Unlock()
// Find the last roll rev.
depsFile, cleanup, err := rm.getDEPSFile(ctx, rm.parentRepo, baseCommit)
if err != nil {
return nil, nil, nil, err
}
defer cleanup()
lastRollHash, err := rm.getdep(ctx, depsFile, rm.childPath)
if err != nil {
return nil, nil, nil, err
}
lastRollDetails, err := rm.childRepo.Details(ctx, lastRollHash)
if err != nil {
return nil, nil, nil, err
}
lastRollRev := revision.FromLongCommit(rm.childRevLinkTmpl, lastRollDetails)
// Get the tip-of-tree revision.
tipRevDetails, err := rm.childRepo.Details(ctx, rm.childBranch)
if err != nil {
return nil, nil, nil, err
}
tipRev := revision.FromLongCommit(rm.childRevLinkTmpl, tipRevDetails)
// Find the not-yet-rolled child repo commits.
// Only consider commits on the "main" branch as roll candidates.
notRolled, err := rm.childRepo.LogFirstParent(ctx, lastRollRev.Id, tipRev.Id)
if err != nil {
return nil, nil, nil, err
}
notRolledRevs := revision.FromLongCommits(rm.childRevLinkTmpl, notRolled)
return lastRollRev, tipRev, notRolledRevs, nil
}
// See documentation for RepoManager interface.
func (r *noCheckoutDEPSRepoManager) GetRevision(ctx context.Context, id string) (*revision.Revision, error) {
details, err := r.childRepo.Details(ctx, id)
if err != nil {
return nil, err
}
return revision.FromLongCommit(r.childRevLinkTmpl, details), nil
}