[bazel] Add Infra-PerCommit-Test-Bazel-{RBE,Local} tryjobs.
The RBE variant runs "bazel test --config=remote //..." and will upload Puppeteer test screenshots to Gold (TO DO).
The local variant requires a bit more setup (start the emulators, ensure there is a depot_tools checkout, etc.) and does not upload screenshots to Gold. Some of the extra setup code is borrowed from //infra/bots/task_drivers/infra_tests/infra_tests.go.
Note that currently the RBE variant is commented out until we figure out the correct way to grant RBE access to Swarming bots. Both the local and RBE variants of this tryjob will be equivalent until said code is uncommented.
See go/skia-infra-bazel for reference.
Bug: skia:11111
Change-Id: I3bf611ac3f3eb2bb935d74ba874582bfede188e7
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/373131
Commit-Queue: Leandro Lovisolo <lovisolo@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go
index 797e0dc..f9f16c7 100644
--- a/infra/bots/gen_tasks.go
+++ b/infra/bots/gen_tasks.go
@@ -74,6 +74,8 @@
"Infra-PerCommit-ValidateAutorollConfigs",
"Infra-PerCommit-Build-Bazel-Local",
"Infra-PerCommit-Build-Bazel-RBE",
+ "Infra-PerCommit-Test-Bazel-Local",
+ "Infra-PerCommit-Test-Bazel-RBE",
"Infra-Experimental-Small-Linux",
"Infra-Experimental-Small-Win",
}
@@ -595,6 +597,47 @@
return name
}
+func bazelTest(b *specs.TasksCfgBuilder, name string, rbe bool) string {
+ cipd := append([]*specs.CipdPackage{}, specs.CIPD_PKGS_GIT_LINUX_AMD64...)
+ cipd = append(cipd, specs.CIPD_PKGS_PYTHON_LINUX_AMD64...)
+ cipd = append(cipd, specs.CIPD_PKGS_GSUTIL...)
+ cipd = append(cipd, specs.CIPD_PKGS_ISOLATE...)
+ cipd = append(cipd, b.MustGetCipdPackageFromAsset("bazel"))
+ cipd = append(cipd, b.MustGetCipdPackageFromAsset("go"))
+ cipd = append(cipd, b.MustGetCipdPackageFromAsset("cockroachdb"))
+ cipd = append(cipd, b.MustGetCipdPackageFromAsset("gcloud_linux"))
+
+ t := &specs.TaskSpec{
+ Caches: CACHES_GO,
+ CasSpec: CAS_WHOLE_REPO,
+ CipdPackages: cipd,
+ Command: []string{
+ "./bazel_test_all",
+ "--project_id", "skia-swarming-bots",
+ "--task_id", specs.PLACEHOLDER_TASK_ID,
+ "--task_name", name,
+ "--workdir", ".",
+ fmt.Sprintf("--rbe=%t", rbe),
+ "--alsologtostderr",
+ },
+ Dependencies: []string{buildTaskDrivers(b, "Linux", "x86_64")},
+ Dimensions: linuxGceDimensions(MACHINE_TYPE_LARGE),
+ EnvPrefixes: map[string][]string{
+ "PATH": {
+ "cipd_bin_packages",
+ "cipd_bin_packages/bin",
+ "go/go/bin",
+ "bazel/bin",
+ "cockroachdb",
+ "gcloud_linux/bin",
+ },
+ },
+ ServiceAccount: SERVICE_ACCOUNT_COMPILE,
+ }
+ b.MustAddTask(name, t)
+ return name
+}
+
// process generates Tasks and Jobs for the given Job name.
func process(b *specs.TasksCfgBuilder, name string) {
var priority float64 // Leave as default for most jobs.
@@ -620,6 +663,10 @@
deps = append(deps, bazelBuild(b, name, false /* =rbe */))
} else if strings.Contains(name, "Build-Bazel-RBE") {
deps = append(deps, bazelBuild(b, name, true /* =rbe */))
+ } else if strings.Contains(name, "Test-Bazel-Local") {
+ deps = append(deps, bazelTest(b, name, false /* =rbe */))
+ } else if strings.Contains(name, "Test-Bazel-RBE") {
+ deps = append(deps, bazelTest(b, name, true /* =rbe */))
} else {
// Infra tests.
if strings.Contains(name, "Infra-PerCommit") {
diff --git a/infra/bots/task_drivers/bazel_test_all/BUILD.bazel b/infra/bots/task_drivers/bazel_test_all/BUILD.bazel
new file mode 100644
index 0000000..583c596
--- /dev/null
+++ b/infra/bots/task_drivers/bazel_test_all/BUILD.bazel
@@ -0,0 +1,25 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+ name = "bazel_test_all_lib",
+ srcs = ["bazel_test_all.go"],
+ importpath = "go.skia.org/infra/infra/bots/task_drivers/bazel_test_all",
+ visibility = ["//visibility:private"],
+ deps = [
+ "//go/depot_tools",
+ "//go/emulators",
+ "//go/exec",
+ "//go/git",
+ "//go/recipe_cfg",
+ "//go/sklog",
+ "//task_driver/go/lib/golang",
+ "//task_driver/go/lib/os_steps",
+ "//task_driver/go/td",
+ ],
+)
+
+go_binary(
+ name = "bazel_test_all",
+ embed = [":bazel_test_all_lib"],
+ visibility = ["//visibility:public"],
+)
diff --git a/infra/bots/task_drivers/bazel_test_all/bazel_test_all.go b/infra/bots/task_drivers/bazel_test_all/bazel_test_all.go
new file mode 100644
index 0000000..749e1cd
--- /dev/null
+++ b/infra/bots/task_drivers/bazel_test_all/bazel_test_all.go
@@ -0,0 +1,195 @@
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "path/filepath"
+ "time"
+
+ "go.skia.org/infra/go/depot_tools"
+ "go.skia.org/infra/go/emulators"
+ "go.skia.org/infra/go/exec"
+ "go.skia.org/infra/go/git"
+ "go.skia.org/infra/go/recipe_cfg"
+ "go.skia.org/infra/go/sklog"
+ "go.skia.org/infra/task_driver/go/lib/golang"
+ "go.skia.org/infra/task_driver/go/lib/os_steps"
+ "go.skia.org/infra/task_driver/go/td"
+)
+
+var (
+ // Required properties for this task.
+ projectID = flag.String("project_id", "", "ID of the Google Cloud project.")
+ taskID = flag.String("task_id", "", "ID of this task.")
+ taskName = flag.String("task_name", "", "Name of the task.")
+ workDirFlag = flag.String("workdir", ".", "Working directory.")
+ rbe = flag.Bool("rbe", false, "Whether to run Bazel on RBE or locally.")
+
+ // Optional flags.
+ local = flag.Bool("local", false, "True if running locally (as opposed to on the bots)")
+ output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.")
+
+ // Various directory paths.
+ workDir string
+ repoDir string
+ bazelCacheDir string
+)
+
+func main() {
+ // Setup.
+ ctx := td.StartRun(projectID, taskID, taskName, output, local)
+ defer td.EndRun(ctx)
+
+ // Compute various directory paths.
+ var err error
+ workDir, err = os_steps.Abs(ctx, *workDirFlag)
+ if err != nil {
+ td.Fatal(ctx, err)
+ }
+ repoDir = filepath.Join(workDir, "buildbot") // Repository checkout.
+
+ // Temporary directory for the Bazel cache.
+ //
+ // We cannot use the default Bazel cache location ($HOME/.cache/bazel) because:
+ //
+ // - The cache can be large (>10G).
+ // - Swarming bots have limited storage space on the root partition (15G).
+ // - Because the above, the Bazel build fails with a "no space left on device" error.
+ // - The Bazel cache under $HOME/.cache/bazel lingers after the tryjob completes, causing the
+ // Swarming bot to be quarantined due to low disk space.
+ // - Generally, it's considered poor hygiene to leave a bot in a different state.
+ //
+ // The temporary directory created by the below function call lives under /mnt/pd0, which has
+ // significantly more storage space, and will be wiped after the tryjob completes.
+ //
+ // Reference: https://docs.bazel.build/versions/master/output_directories.html#current-layout.
+ bazelCacheDir, err = os_steps.TempDir(ctx, "", "bazel-user-cache-*")
+ if err != nil {
+ td.Fatal(ctx, err)
+ }
+
+ // Clean up the temporary Bazel cache directory when running locally, because during development,
+ // we do not want to leave behind a ~10GB Bazel cache directory under /tmp after each run.
+ //
+ // This is not necessary under Swarming because the temporary directory will be cleaned up
+ // automatically.
+ if *local {
+ if err := os_steps.RemoveAll(ctx, bazelCacheDir); err != nil {
+ td.Fatal(ctx, err)
+ }
+ }
+
+ // Print out the Bazel version for debugging purposes.
+ bazel(ctx, "version")
+
+ // Run the tests.
+ if *rbe {
+ // TODO(lovisolo): Uncomment once we figure out how to authenticate against RBE.
+ // testOnRBE(ctx)
+ testLocally(ctx) // TODO(lovisolo): Remove.
+ } else {
+ testLocally(ctx)
+ }
+}
+
+// By invoking Bazel via this function, we ensure that we will always use the temporary cache.
+func bazel(ctx context.Context, args ...string) {
+ command := []string{"bazel", "--output_user_root=" + bazelCacheDir}
+ command = append(command, args...)
+ if _, err := exec.RunCwd(ctx, repoDir, command...); err != nil {
+ td.Fatal(ctx, err)
+ }
+}
+
+func testOnRBE(ctx context.Context) {
+ // Run all tests in the repository. The tryjob will fail upon any failing tests.
+ bazel(ctx, "test", "--config=remote", "//...")
+
+ // TODO(lovisolo): Upload Puppeteer test screenshots to Gold.
+}
+
+func testLocally(ctx context.Context) {
+ // We skip the following steps when running on a developer's workstation because we assume that
+ // the environment already has everything we need to run this task driver (the repository checkout
+ // has a .git directory, the Go environment variables are properly set, etc.).
+ if !*local {
+ // Initialize a fake Git repository. Some tests require this. We receive the code via Isolate,
+ // but it doesn't include the .git dir.
+ gitDir := git.GitDir(repoDir)
+ err := td.Do(ctx, td.Props("Initialize fake Git repository"), func(ctx context.Context) error {
+ if gitVer, err := gitDir.Git(ctx, "version"); err != nil {
+ td.Fatal(ctx, err)
+ } else {
+ sklog.Infof("Git version %s", gitVer)
+ }
+ if _, err := gitDir.Git(ctx, "init"); err != nil {
+ td.Fatal(ctx, err)
+ }
+ if _, err := gitDir.Git(ctx, "config", "--local", "user.name", "Skia bots"); err != nil {
+ td.Fatal(ctx, err)
+ }
+ if _, err := gitDir.Git(ctx, "config", "--local", "user.email", "fake@skia.bots"); err != nil {
+ td.Fatal(ctx, err)
+ }
+ if _, err := gitDir.Git(ctx, "add", "."); err != nil {
+ td.Fatal(ctx, err)
+ }
+ if _, err := gitDir.Git(ctx, "commit", "--no-verify", "-m", "Fake commit to detect diffs"); err != nil {
+ td.Fatal(ctx, err)
+ }
+ return nil
+ })
+ if err != nil {
+ td.Fatal(ctx, err)
+ }
+
+ // Set up go.
+ ctx = golang.WithEnv(ctx, workDir)
+
+ // Check out depot_tools at the exact revision expected by tests (defined in recipes.cfg), and
+ // make it available to tests by by adding it to the PATH.
+ var depotToolsDir string
+ err = td.Do(ctx, td.Props("Check out depot_tools"), func(ctx context.Context) error {
+ var err error
+ depotToolsDir, err = depot_tools.Sync(ctx, workDir, filepath.Join(repoDir, recipe_cfg.RECIPE_CFG_PATH))
+ if err != nil {
+ td.Fatal(ctx, err)
+ }
+ return nil
+ })
+ if err != nil {
+ td.Fatal(ctx, err)
+ }
+ ctx = td.WithEnv(ctx, []string{"PATH=%(PATH)s:" + depotToolsDir})
+ }
+
+ // Start the emulators. When running this task driver locally (e.g. with --local), this will kill
+ // any existing emulator instances prior to launching all emulators.
+ if err := emulators.StartAllEmulators(); err != nil {
+ td.Fatal(ctx, err)
+ }
+ defer func() {
+ if err := emulators.StopAllEmulators(); err != nil {
+ td.Fatal(ctx, err)
+ }
+ }()
+ time.Sleep(5 * time.Second) // Give emulators time to boot.
+
+ // Set *_EMULATOR_HOST environment variables.
+ emulatorHostEnvVars := []string{}
+ for _, emulator := range emulators.AllEmulators {
+ // We need to set the *_EMULATOR_HOST variable for the current emulator before we can retrieve
+ // its value via emulators.GetEmulatorHostEnvVar().
+ if err := emulators.SetEmulatorHostEnvVar(emulator); err != nil {
+ td.Fatal(ctx, err)
+ }
+ name := emulators.GetEmulatorHostEnvVarName(emulator)
+ value := emulators.GetEmulatorHostEnvVar(emulator)
+ emulatorHostEnvVars = append(emulatorHostEnvVars, fmt.Sprintf("%s=%s", name, value))
+ }
+ ctx = td.WithEnv(ctx, emulatorHostEnvVars)
+
+ // Run all tests in the repository. The tryjob will fail upon any failing tests.
+ bazel(ctx, "test", "//...", "--test_output=errors")
+}
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index 513ee64..2375bfb 100755
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -79,6 +79,16 @@
"Infra-PerCommit-Small"
]
},
+ "Infra-PerCommit-Test-Bazel-Local": {
+ "tasks": [
+ "Infra-PerCommit-Test-Bazel-Local"
+ ]
+ },
+ "Infra-PerCommit-Test-Bazel-RBE": {
+ "tasks": [
+ "Infra-PerCommit-Test-Bazel-RBE"
+ ]
+ },
"Infra-PerCommit-ValidateAutorollConfigs": {
"tasks": [
"Infra-PerCommit-ValidateAutorollConfigs"
@@ -1727,6 +1737,226 @@
"max_attempts": 2,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
+ "Infra-PerCommit-Test-Bazel-Local": {
+ "caches": [
+ {
+ "name": "go_cache",
+ "path": "cache/go_cache"
+ },
+ {
+ "name": "gopath",
+ "path": "cache/gopath"
+ }
+ ],
+ "casSpec": "whole-repo",
+ "cipd_packages": [
+ {
+ "name": "infra/3pp/tools/cpython/linux-amd64",
+ "path": "cipd_bin_packages",
+ "version": "version:2.7.18.chromium.30"
+ },
+ {
+ "name": "infra/3pp/tools/git/linux-amd64",
+ "path": "cipd_bin_packages",
+ "version": "version:2.29.2.chromium.6"
+ },
+ {
+ "name": "infra/gsutil",
+ "path": "cipd_bin_packages",
+ "version": "version:4.46"
+ },
+ {
+ "name": "infra/tools/git/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:14be8b751c0fb567535f520f8a7bc60c3f40b378"
+ },
+ {
+ "name": "infra/tools/luci/git-credential-luci/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:14be8b751c0fb567535f520f8a7bc60c3f40b378"
+ },
+ {
+ "name": "infra/tools/luci/isolate/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:14be8b751c0fb567535f520f8a7bc60c3f40b378"
+ },
+ {
+ "name": "infra/tools/luci/isolated/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:14be8b751c0fb567535f520f8a7bc60c3f40b378"
+ },
+ {
+ "name": "infra/tools/luci/vpython/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:14be8b751c0fb567535f520f8a7bc60c3f40b378"
+ },
+ {
+ "name": "skia/bots/bazel",
+ "path": "bazel",
+ "version": "version:2"
+ },
+ {
+ "name": "skia/bots/cockroachdb",
+ "path": "cockroachdb",
+ "version": "version:3"
+ },
+ {
+ "name": "skia/bots/gcloud_linux",
+ "path": "gcloud_linux",
+ "version": "version:14"
+ },
+ {
+ "name": "skia/bots/go",
+ "path": "go",
+ "version": "version:9"
+ }
+ ],
+ "command": [
+ "./bazel_test_all",
+ "--project_id",
+ "skia-swarming-bots",
+ "--task_id",
+ "<(TASK_ID)",
+ "--task_name",
+ "Infra-PerCommit-Test-Bazel-Local",
+ "--workdir",
+ ".",
+ "--rbe=false",
+ "--alsologtostderr"
+ ],
+ "dependencies": [
+ "Housekeeper-PerCommit-BuildTaskDrivers-Linux-x86_64"
+ ],
+ "dimensions": [
+ "pool:Skia",
+ "os:Debian-10.3",
+ "gpu:none",
+ "cpu:x86-64-Haswell_GCE",
+ "machine_type:n1-highcpu-64",
+ "docker_installed:true"
+ ],
+ "env_prefixes": {
+ "PATH": [
+ "cipd_bin_packages",
+ "cipd_bin_packages/bin",
+ "go/go/bin",
+ "bazel/bin",
+ "cockroachdb",
+ "gcloud_linux/bin"
+ ]
+ },
+ "service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
+ },
+ "Infra-PerCommit-Test-Bazel-RBE": {
+ "caches": [
+ {
+ "name": "go_cache",
+ "path": "cache/go_cache"
+ },
+ {
+ "name": "gopath",
+ "path": "cache/gopath"
+ }
+ ],
+ "casSpec": "whole-repo",
+ "cipd_packages": [
+ {
+ "name": "infra/3pp/tools/cpython/linux-amd64",
+ "path": "cipd_bin_packages",
+ "version": "version:2.7.18.chromium.30"
+ },
+ {
+ "name": "infra/3pp/tools/git/linux-amd64",
+ "path": "cipd_bin_packages",
+ "version": "version:2.29.2.chromium.6"
+ },
+ {
+ "name": "infra/gsutil",
+ "path": "cipd_bin_packages",
+ "version": "version:4.46"
+ },
+ {
+ "name": "infra/tools/git/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:14be8b751c0fb567535f520f8a7bc60c3f40b378"
+ },
+ {
+ "name": "infra/tools/luci/git-credential-luci/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:14be8b751c0fb567535f520f8a7bc60c3f40b378"
+ },
+ {
+ "name": "infra/tools/luci/isolate/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:14be8b751c0fb567535f520f8a7bc60c3f40b378"
+ },
+ {
+ "name": "infra/tools/luci/isolated/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:14be8b751c0fb567535f520f8a7bc60c3f40b378"
+ },
+ {
+ "name": "infra/tools/luci/vpython/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:14be8b751c0fb567535f520f8a7bc60c3f40b378"
+ },
+ {
+ "name": "skia/bots/bazel",
+ "path": "bazel",
+ "version": "version:2"
+ },
+ {
+ "name": "skia/bots/cockroachdb",
+ "path": "cockroachdb",
+ "version": "version:3"
+ },
+ {
+ "name": "skia/bots/gcloud_linux",
+ "path": "gcloud_linux",
+ "version": "version:14"
+ },
+ {
+ "name": "skia/bots/go",
+ "path": "go",
+ "version": "version:9"
+ }
+ ],
+ "command": [
+ "./bazel_test_all",
+ "--project_id",
+ "skia-swarming-bots",
+ "--task_id",
+ "<(TASK_ID)",
+ "--task_name",
+ "Infra-PerCommit-Test-Bazel-RBE",
+ "--workdir",
+ ".",
+ "--rbe=true",
+ "--alsologtostderr"
+ ],
+ "dependencies": [
+ "Housekeeper-PerCommit-BuildTaskDrivers-Linux-x86_64"
+ ],
+ "dimensions": [
+ "pool:Skia",
+ "os:Debian-10.3",
+ "gpu:none",
+ "cpu:x86-64-Haswell_GCE",
+ "machine_type:n1-highcpu-64",
+ "docker_installed:true"
+ ],
+ "env_prefixes": {
+ "PATH": [
+ "cipd_bin_packages",
+ "cipd_bin_packages/bin",
+ "go/go/bin",
+ "bazel/bin",
+ "cockroachdb",
+ "gcloud_linux/bin"
+ ]
+ },
+ "service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
+ },
"Infra-PerCommit-ValidateAutorollConfigs": {
"casSpec": "autoroll-configs",
"command": [
