blob: 80741ec7f73cf448200750c55ca047235927e60f [file] [log] [blame]
// Copyright 2021 Google Inc.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package main
import (
"context"
"flag"
"fmt"
"os"
"path/filepath"
"strings"
"go.skia.org/infra/go/depot_tools"
"go.skia.org/infra/go/exec"
"go.skia.org/infra/go/gerrit"
"go.skia.org/infra/go/git/git_common"
"go.skia.org/infra/go/sklog"
"go.skia.org/infra/promk/go/pushgateway"
"go.skia.org/infra/task_driver/go/lib/auth_steps"
"go.skia.org/infra/task_driver/go/lib/checkout"
"go.skia.org/infra/task_driver/go/lib/gerrit_steps"
"go.skia.org/infra/task_driver/go/lib/os_steps"
"go.skia.org/infra/task_driver/go/td"
"go.skia.org/infra/task_scheduler/go/types"
)
const (
// Metric constants for pushgateway.
jobName = "recreate-skps"
buildFailureMetricName = "recreate_skps_build_failure"
creatingSKPsFailureMetricName = "recreate_skps_creation_failure"
metricValue_NoFailure = "0"
metricValue_Failure = "1"
)
func botUpdate(ctx context.Context, checkoutRoot, gitCacheDir, skiaRev, patchRef, depotToolsDir string, local bool) error {
return td.Do(ctx, td.Props("bot_update").Infra(), func(ctx context.Context) error {
tmp, err := os_steps.TempDir(ctx, "", "")
if err != nil {
return err
}
defer func() {
_ = os_steps.RemoveAll(ctx, tmp)
}()
ctx = td.WithEnv(ctx, []string{
"DEPOT_TOOLS_COLLECT_METRICS",
"DEPOT_TOOLS_UPDATE=0",
"GIT_CONFIG_NOSYSTEM: 1",
"GIT_HTTP_LOW_SPEED_LIMIT: 102400",
"GIT_HTTP_LOW_SPEED_TIME: 1800",
"GIT_TERMINAL_PROMPT: 0",
"GIT_USER_AGENT=",
})
if _, err := exec.RunCwd(ctx, ".", "which", "git"); err != nil {
return err
}
if _, err := exec.RunCwd(ctx, ".", "git", "--version"); err != nil {
return err
}
if !local {
if err := git_common.EnsureGitIsFromCIPD(ctx); err != nil {
return err
}
}
spec := `
solutions = [
{ "name" : 'src',
"url" : 'https://chromium.googlesource.com/chromium/src.git',
"deps_file" : 'DEPS',
"managed" : False,
"custom_deps" : {
},
"custom_vars": {},
},
]
`
specPath := filepath.Join(checkoutRoot, ".gclient")
if err := os_steps.WriteFile(ctx, specPath, []byte(spec), os.ModePerm); err != nil {
return err
}
skiaRepoURL := "https://skia.googlesource.com/skia.git"
botUpdateScript := filepath.Join(depotToolsDir, "recipes", "recipe_modules", "bot_update", "resources", "bot_update.py")
mainRepoName := "src"
patchRoot := "src/third_party/skia"
revision := "origin/main"
// These are required for some reason, despite our not using them.
outputJson := filepath.Join(tmp, "bot_update.json")
revMapFile := filepath.Join(tmp, "revmap")
revMap := `{"got_revision": "src"}`
if err := os_steps.WriteFile(ctx, revMapFile, []byte(revMap), os.ModePerm); err != nil {
return err
}
cleanupDir := filepath.Join(tmp, "cleanup")
if err := os_steps.MkdirAll(ctx, cleanupDir); err != nil {
return err
}
cmd := []string{
"vpython", "-u", botUpdateScript,
"--spec-path", specPath,
"--patch_root", patchRoot,
"--revision_mapping_file", revMapFile,
"--cleanup-dir", cleanupDir,
"--output_json", outputJson,
"--revision", fmt.Sprintf("%s@%s", mainRepoName, revision),
"--revision", fmt.Sprintf("%s@%s", "https://skia.googlesource.com/skia.git", skiaRev),
}
if gitCacheDir != "" {
cmd = append(cmd, "--git-cache-dir", gitCacheDir)
}
if patchRef != "" {
patchRepoURL := skiaRepoURL
patchBaseRev := skiaRev
cmd = append(cmd, "--patch_ref", fmt.Sprintf("%s@%s:%s", patchRepoURL, patchBaseRev, patchRef))
}
if _, err := exec.RunCwd(ctx, checkoutRoot, cmd...); err != nil {
return err
}
if _, err := exec.RunCwd(ctx, checkoutRoot, "vpython", "-u", filepath.Join(depotToolsDir, "gclient.py"), "runhooks"); err != nil {
return err
}
return nil
})
}
func main() {
var (
projectId = flag.String("project_id", "", "ID of the Google Cloud project.")
taskId = flag.String("task_id", "", "ID of this task.")
taskName = flag.String("task_name", "", "Name of the task.")
output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.")
local = flag.Bool("local", true, "True if running locally (as opposed to on the bots)")
dryRun = flag.Bool("dry_run", false, "If set, generate SKPs but do not upload or commit them.")
skiaRev = flag.String("skia_revision", "origin/main", "Revision of Skia at which this task is running.")
patchRef = flag.String("patch_ref", "", "Patch ref, if any, associated with this task.")
skipSync = flag.Bool("skip-sync", false, "Skip sync. Helpful for running locally.")
skipBuild = flag.Bool("skip-build", false, "skip build. Helpful for running locally.")
gitCacheDirFlag = flag.String("git_cache", "", "Git cache directory.")
checkoutRootFlag = flag.String("checkout_root", "", "Directory to use for checkouts.")
dmPathFlag = flag.String("dm_path", "", "Path to the DM binary.")
)
ctx := td.StartRun(projectId, taskId, taskName, output, local)
defer td.EndRun(ctx)
// Setup.
client, err := auth_steps.InitHttpClient(ctx, *local, gerrit.AuthScope)
if err != nil {
td.Fatal(ctx, err)
}
var g gerrit.GerritInterface
if !*dryRun {
g, err = gerrit.NewGerrit("https://skia-review.googlesource.com", client)
if err != nil {
td.Fatal(ctx, err)
}
}
cwd, err := os.Getwd()
if err != nil {
td.Fatal(ctx, err)
}
skiaDir := filepath.Join(cwd, "skia")
gitCacheDir := ""
if *gitCacheDirFlag != "" {
gitCacheDir = filepath.Join(cwd, *gitCacheDirFlag)
}
checkoutRoot := cwd
if *checkoutRootFlag != "" {
checkoutRoot = filepath.Join(cwd, *checkoutRootFlag)
}
if *dmPathFlag == "" {
td.Fatal(ctx, fmt.Errorf("Must specify --dm_path"))
}
dmPath := filepath.Join(cwd, *dmPathFlag)
// Fetch `sk`
if _, err := exec.RunCwd(ctx, skiaDir, "python3", filepath.Join("bin", "fetch-sk")); err != nil {
td.Fatal(ctx, err)
}
// Create a temporary directory.
tmp, err := os_steps.TempDir(ctx, "", "")
if err != nil {
td.Fatal(ctx, err)
}
defer func() {
_ = os_steps.RemoveAll(ctx, tmp)
}()
recipesCfgFile := filepath.Join(skiaDir, "infra", "config", "recipes.cfg")
// Check out depot_tools at the exact revision expected by tests (defined in recipes.cfg), and
// make it available to tests by by adding it to the PATH.
var depotToolsDir string
if err := td.Do(ctx, td.Props("Check out depot_tools"), func(ctx context.Context) error {
var err error
depotToolsDir, err = depot_tools.Sync(ctx, tmp, recipesCfgFile)
return err
}); err != nil {
td.Fatal(ctx, err)
}
ctx = td.WithEnv(ctx, []string{"PATH=%(PATH)s:" + depotToolsDir})
// Sync Chrome.
if !*skipSync {
if err := botUpdate(ctx, checkoutRoot, gitCacheDir, *skiaRev, *patchRef, depotToolsDir, *local); err != nil {
td.Fatal(ctx, err)
}
}
// For updating metrics.
pg := pushgateway.New(client, jobName, pushgateway.DefaultPushgatewayURL)
chromiumDir := filepath.Join(checkoutRoot, "src")
outDir := filepath.Join(chromiumDir, "out", "Release")
// Build Chrome.
if !*skipBuild {
if err := td.Do(ctx, td.Props("Build Chrome"), func(ctx context.Context) error {
// Run "gn gen". This task driver only runs on Linux.
gn := filepath.Join(chromiumDir, "buildtools", "linux64", "gn")
if _, err := exec.RunCommand(ctx, &exec.Command{
Name: gn,
Args: []string{"gen", outDir},
Dir: chromiumDir,
Env: []string{
"CPPFLAGS=-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1",
"GYP_GENERATORS=ninja",
},
InheritEnv: true,
InheritPath: true,
}); err != nil {
return err
}
// Perform the build.
ninja := filepath.Join(depotToolsDir, "ninja")
if _, err := exec.RunCwd(ctx, chromiumDir, ninja, "-C", outDir, "chrome"); err != nil {
return err
}
return nil
}); err != nil {
// Report that the build failed.
pg.Push(ctx, buildFailureMetricName, metricValue_Failure)
td.Fatal(ctx, err)
}
// Report that the build was successful.
pg.Push(ctx, buildFailureMetricName, metricValue_NoFailure)
}
// Capture and upload the SKPs.
script := filepath.Join(skiaDir, "infra", "bots", "assets", "skp", "create_and_upload.py")
cmd := []string{
"vpython3", "-u", script,
"--chrome_src_path", chromiumDir,
"--browser_executable", filepath.Join(outDir, "chrome"),
"--dm_path", dmPath,
}
if *dryRun {
cmd = append(cmd, "--dry_run")
}
if *local {
cmd = append(cmd, "--local")
}
command := &exec.Command{
Name: filepath.Join(depotToolsDir, cmd[0]),
Args: cmd[1:],
Dir: skiaDir,
Env: []string{
fmt.Sprintf("PATH=%s:%s", os.Getenv("PATH"), depotToolsDir),
},
}
sklog.Infof("Running command: %s %s", command.Name, strings.Join(command.Args, " "))
if err := exec.Run(ctx, command); err != nil {
// Creating SKP asset in RecreateSKPs failed.
pg.Push(ctx, creatingSKPsFailureMetricName, metricValue_Failure)
td.Fatal(ctx, err)
}
// Report that the asset creation was successful.
pg.Push(ctx, creatingSKPsFailureMetricName, metricValue_NoFailure)
if *dryRun {
return
}
// Retrieve the new SKP version.
versionFileSubPath := filepath.Join("infra", "bots", "assets", "skp", "VERSION")
skpVersion, err := os_steps.ReadFile(ctx, filepath.Join(skiaDir, versionFileSubPath))
if err != nil {
td.Fatal(ctx, err)
}
// Sync a new checkout of Skia to create the CL.
tmpSkia := filepath.Join(tmp, "skia")
co, err := checkout.EnsureGitCheckout(ctx, tmpSkia, types.RepoState{
Repo: "https://skia.googlesource.com/skia.git",
Revision: "origin/main",
})
if err != nil {
td.Fatal(ctx, err)
}
baseRev, err := co.FullHash(ctx, "HEAD")
if err != nil {
td.Fatal(ctx, err)
}
// Write the new SKP version.
if err := os_steps.WriteFile(ctx, filepath.Join(co.Dir(), versionFileSubPath), skpVersion, os.ModePerm); err != nil {
td.Fatal(ctx, err)
}
// Regenerate tasks.json.
if _, err := exec.RunCwd(ctx, tmpSkia, "go", "run", "./infra/bots/gen_tasks.go"); err != nil {
td.Fatal(ctx, err)
}
// Upload a CL.
commitMsg := `Update SKP version
Automatic commit by the RecreateSKPs bot.
`
gerrit_steps.UploadCL(ctx, g, co, "skia", "main", baseRev, commitMsg, []string{"rmistry@google.com"}, false)
}