scripts/roll_recipe_deps/roll_recipe_deps.go

package main

/*
	Program used for updating recipe DEPS.

	Follows a dependency graph of repos and creates roll CLs for those which
	are not up-to-date. If any dependency of a repo is not up-to-date, that
	repo is skipped. Therefore, run this script repeatedly as CLs are
	created and land, until all repos are up-to-date.

	For example:

        // Upload recipe roll CLs for infra and skia-recipes repos:
	$ go run scripts/roll_recipe_deps/roll_recipe_deps.go --alsologtostderr

	// After the skia-recipes roll above lands, the following will upload
	// a roll CL for the skia repo:
	$ go run scripts/roll_recipe_deps/roll_recipe_deps.go --alsologtostderr

	// After the skia roll above lands, the following is a no-op:
        $ go run scripts/roll_recipe_deps/roll_recipe_deps.go --alsologtostderr

	Note that if you run this script again before an uploaded roll lands,
	the script will upload another roll.
*/

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"sort"
	"strings"

	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
)

var (
	// RECIPES_PY_PATH indicates where in the given repo the recipes.py
	// file lives.
	RECIPES_PY_PATH = map[string]string{
		common.REPO_SKIA:       "infra/bots/recipes.py",
		common.REPO_SKIA_INFRA: "infra/bots/recipes.py",
	}

	// REPOS maps out the recipe dependency relationship between
	// repositories.
	REPOS = map[string][]string{
		common.REPO_SKIA_INFRA: {},
		common.REPO_SKIA:       {},
	}
)

// issueJson is a struct which matches the format of the JSON output of
// "git cl issue".
type issueJson struct {
	Issue    int    `json:"issue"`
	IssueUrl string `json:"issue_url"`
}

// rollJson is a struct which matches the format of the JSON output of
// "recipes.py autoroll".
type rollJson struct {
	PickedRollDetails struct {
		CommitInfos map[string][]struct {
			Author   string `json:"author"`
			Message  string `json:"message"`
			RepoId   string `json:"repo_id"`
			Revision string `json:"revision"`
		} `json:"commit_infos"`
	} `json:"picked_roll_details"`
}

// rollOnce performs a single recipe roll and returns the commits in the roll.
// The result will be empty if the repo is up-to-date.
func rollOnce(ctx context.Context, repoUrl, cwd string) (map[string][]string, error) {
	tmpDir, err := ioutil.TempDir("", "recipe_roll_")
	if err != nil {
		return nil, err
	}
	defer util.RemoveAll(tmpDir)
	outJson := path.Join(tmpDir, "roll.json")
	if _, err := exec.RunCwd(ctx, cwd, "python", RECIPES_PY_PATH[repoUrl], "autoroll", "--output-json", outJson); err != nil {
		return nil, err
	}
	f, err := os.Open(outJson)
	if err != nil {
		return nil, err
	}
	defer util.Close(f)
	var js rollJson
	if err := json.NewDecoder(f).Decode(&js); err != nil {
		return nil, err
	}
	rv := make(map[string][]string, len(js.PickedRollDetails.CommitInfos))
	for repo, commits := range js.PickedRollDetails.CommitInfos {
		shortCommits := make([]string, 0, len(commits))
		for _, c := range commits {
			msg := strings.Split(c.Message, "\n")[0]
			if len(msg) > 64 {
				msg = msg[:64]
			}
			shortCommits = append(shortCommits, fmt.Sprintf("%s %s", c.Revision[:7], msg))
		}
		rv[repo] = shortCommits
	}

	return rv, nil
}

// rollRepo performs a DEPS roll and uploads a CL if the repo is not up-to-date.
// Returns the URL of the uploaded CL, if any.
func rollRepo(ctx context.Context, repoUrl string) (string, error) {
	sklog.Infof("  Creating checkout...")
	tmpDir, err := ioutil.TempDir("", "recipe_roll_")
	if err != nil {
		return "", err
	}
	defer util.RemoveAll(tmpDir)
	repo, err := git.NewCheckout(ctx, repoUrl, tmpDir)
	if err != nil {
		return "", err
	}
	sklog.Infof("  Rolling recipe DEPS...")
	details, err := rollOnce(ctx, repoUrl, repo.Dir())
	if err != nil {
		return "", err
	}
	if len(details) == 0 {
		return "", nil
	}
	if _, err := repo.Git(ctx, "commit", "-a", "-m", "Roll Recipe DEPS"); err != nil {
		return "", err
	}

	commitMsg := "Roll Recipe DEPS\n\n"
	repoNames := make([]string, 0, len(details))
	for repo := range details {
		repoNames = append(repoNames, repo)
	}
	sort.Strings(repoNames)
	for _, repo := range repoNames {
		commitMsg += fmt.Sprintf("%s:\n", repo)
		for _, c := range details[repo] {
			commitMsg += fmt.Sprintf("\t%s\n", c)
		}
		commitMsg += "\n"
	}

	sklog.Infof("  Uploading roll CL...")
	if _, err := repo.Git(ctx, "cl", "upload", "--gerrit", "--bypass-hooks", "--cq-dry-run", "-f", "-m", commitMsg); err != nil {
		return "", err
	}
	issueFile := path.Join(tmpDir, "issue.json")
	if _, err := repo.Git(ctx, "cl", "issue", fmt.Sprintf("--json=%s", issueFile)); err != nil {
		return "", err
	}
	f, err := os.Open(issueFile)
	if err != nil {
		return "", err
	}
	defer util.Close(f)
	var issue issueJson
	if err := json.NewDecoder(f).Decode(&issue); err != nil {
		return "", err
	}
	return issue.IssueUrl, nil
}

func main() {
	common.Init()

	uploaded := []string{}
	ctx := context.Background()

	// Traverse the dependency graph of repos. If any repo is not
	// up-to-date, create a recipe roll for that repo. If any of a repo's
	// dependencies is not up-to-date, do not roll that repo.
	cachedResult := map[string]bool{}
	var recurse func(string) bool
	recurse = func(repoUrl string) bool {
		if result, ok := cachedResult[repoUrl]; ok {
			return result
		}
		for _, dep := range REPOS[repoUrl] {
			if !recurse(dep) {
				sklog.Infof("Not rolling %s; dependency %s is not up to date.", repoUrl, dep)
				return false
			}
		}
		sklog.Infof("Rolling %s...", repoUrl)
		rollIssue, err := rollRepo(ctx, repoUrl)
		if err != nil {
			sklog.Fatal(err)
		}
		if rollIssue == "" {
			cachedResult[repoUrl] = true
		} else {
			uploaded = append(uploaded, rollIssue)
			cachedResult[repoUrl] = false
		}
		return cachedResult[repoUrl]
	}

	for repo := range REPOS {
		recurse(repo)
	}

	if len(uploaded) > 0 {
		sklog.Infof("Uploaded CLs:")
		for _, cl := range uploaded {
			sklog.Infof("\t%s", cl)
		}
	}
}