[infra] Use command_wrapper task driver to pull CIPD packages
Change-Id: I98f01aa4bbf024656a54d712bd867c487d6186cb
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/611424
Commit-Queue: Eric Boren <borenet@google.com>
Reviewed-by: Joe Gregorio <jcgregorio@google.com>
diff --git a/cipd.ensure b/cipd.ensure
index f710bc4..1d61f7f 100644
--- a/cipd.ensure
+++ b/cipd.ensure
@@ -49,6 +49,7 @@
@subdir task_drivers
skia/tools/bazel_build_all/${platform} git_revision:2195169c3c8754d261f092b32704003ee21bd9bf
skia/tools/bazel_test_all/${platform} git_revision:2195169c3c8754d261f092b32704003ee21bd9bf
+skia/tools/command_wrapper/${platform} git_revision:b65a23129f1d7028291ccb0cead545d60b84d296
skia/tools/presubmit/${platform} git_revision:2195169c3c8754d261f092b32704003ee21bd9bf
@Subdir
diff --git a/go/cipd/asset_versions_gen.go b/go/cipd/asset_versions_gen.go
index 7fda13d..c1ce4ba 100644
--- a/go/cipd/asset_versions_gen.go
+++ b/go/cipd/asset_versions_gen.go
@@ -203,6 +203,11 @@
Name: "skia/tools/bazel_test_all/${platform}",
Version: "git_revision:2195169c3c8754d261f092b32704003ee21bd9bf",
},
+ "skia/tools/command_wrapper/${platform}": {
+ Path: "task_drivers",
+ Name: "skia/tools/command_wrapper/${platform}",
+ Version: "git_revision:b65a23129f1d7028291ccb0cead545d60b84d296",
+ },
"skia/tools/goldctl/${platform}": {
Path: "cipd_bin_packages",
Name: "skia/tools/goldctl/${platform}",
diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go
index 7e78c59..20e12c0 100644
--- a/infra/bots/gen_tasks.go
+++ b/infra/bots/gen_tasks.go
@@ -137,9 +137,11 @@
Dimensions: linuxGceDimensions(machineTypeMedium),
// This task is idempotent but unlikely to ever be deduped
// because it depends on the entire repo...
- Idempotent: true,
+ Idempotent: true,
+ ServiceAccount: compileServiceAccount,
}
usesBazelisk(b, t)
+ usesWrapperTaskDriver(b, name, false, t)
b.MustAddTask(name, t)
return name
}
@@ -179,6 +181,7 @@
// call to usePreBuiltTaskDrivers.
usesPreBuiltTaskDrivers(b, t)
usesBazelisk(b, t)
+ usesWrapperTaskDriver(b, name, true, t)
b.MustAddTask(name, t)
return name
}
@@ -216,6 +219,7 @@
ServiceAccount: recreateSKPsServiceAccount,
}
usesBazelisk(b, t)
+ usesWrapperTaskDriver(b, name, true, t)
b.MustAddTask(name, t)
return name
}
@@ -243,6 +247,30 @@
t.Dependencies = newDeps
}
+func usesWrapperTaskDriver(b *specs.TasksCfgBuilder, name string, isTaskDriver bool, t *specs.TaskSpec) {
+ newCmd := []string{
+ "./task_drivers/command_wrapper",
+ "--project_id", "skia-swarming-bots",
+ "--task_id", specs.PLACEHOLDER_TASK_ID,
+ "--task_name", name,
+ "--workdir", ".",
+ }
+ for _, pkg := range t.CipdPackages {
+ flag := fmt.Sprintf("%s:%s@%s", pkg.Path, pkg.Name, pkg.Version)
+ newCmd = append(newCmd, "--cipd", flag)
+ }
+ if isTaskDriver {
+ newCmd = append(newCmd, "--command-is-task-driver")
+ }
+ newCmd = append(newCmd, "--")
+ newCmd = append(newCmd, t.Command...)
+ t.Command = newCmd
+
+ t.CipdPackages = []*specs.CipdPackage{
+ cipd.MustGetPackage("skia/tools/command_wrapper/${platform}"),
+ }
+}
+
func bazelBuild(b *specs.TasksCfgBuilder, name string, rbe bool) string {
pkgs := append([]*specs.CipdPackage{}, specs.CIPD_PKGS_GIT_LINUX_AMD64...)
@@ -279,6 +307,7 @@
// call to usePreBuiltTaskDrivers.
usesPreBuiltTaskDrivers(b, t)
usesBazelisk(b, t)
+ usesWrapperTaskDriver(b, name, true, t)
b.MustAddTask(name, t)
return name
}
@@ -323,6 +352,7 @@
// call to usePreBuiltTaskDrivers.
usesPreBuiltTaskDrivers(b, t)
usesBazelisk(b, t)
+ usesWrapperTaskDriver(b, name, true, t)
b.MustAddTask(name, t)
return name
}
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index 91f141b..7d28fa4 100755
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -39,32 +39,33 @@
"casSpec": "empty",
"cipd_packages": [
{
- "name": "infra/3pp/tools/git/linux-amd64",
- "path": "cipd_bin_packages",
- "version": "version:2@2.38.1.chromium.9"
- },
- {
- "name": "infra/tools/git/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "infra/tools/luci/git-credential-luci/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "skia/bots/bazelisk",
- "path": "bazelisk",
- "version": "version:0"
- },
- {
- "name": "skia/tools/presubmit/${platform}",
+ "name": "skia/tools/command_wrapper/${platform}",
"path": "task_drivers",
- "version": "git_revision:2195169c3c8754d261f092b32704003ee21bd9bf"
+ "version": "git_revision:b65a23129f1d7028291ccb0cead545d60b84d296"
}
],
"command": [
+ "./task_drivers/command_wrapper",
+ "--project_id",
+ "skia-swarming-bots",
+ "--task_id",
+ "<(TASK_ID)",
+ "--task_name",
+ "Housekeeper-OnDemand-Presubmit",
+ "--workdir",
+ ".",
+ "--cipd",
+ "cipd_bin_packages:infra/3pp/tools/git/linux-amd64@version:2@2.38.1.chromium.9",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/git/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/luci/git-credential-luci/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "task_drivers:skia/tools/presubmit/${platform}@git_revision:2195169c3c8754d261f092b32704003ee21bd9bf",
+ "--cipd",
+ "bazelisk:skia/bots/bazelisk@version:0",
+ "--command-is-task-driver",
+ "--",
"./task_drivers/presubmit",
"--project_id",
"skia-swarming-bots",
@@ -110,12 +111,24 @@
"casSpec": "whole-repo",
"cipd_packages": [
{
- "name": "skia/bots/bazelisk",
- "path": "bazelisk",
- "version": "version:0"
+ "name": "skia/tools/command_wrapper/${platform}",
+ "path": "task_drivers",
+ "version": "git_revision:b65a23129f1d7028291ccb0cead545d60b84d296"
}
],
"command": [
+ "./task_drivers/command_wrapper",
+ "--project_id",
+ "skia-swarming-bots",
+ "--task_id",
+ "<(TASK_ID)",
+ "--task_name",
+ "Housekeeper-PerCommit-BuildTaskDrivers-Linux-x86_64",
+ "--workdir",
+ ".",
+ "--cipd",
+ "bazelisk:skia/bots/bazelisk@version:0",
+ "--",
"/bin/bash",
"buildbot/infra/bots/build_task_drivers.sh",
"${ISOLATED_OUTDIR}",
@@ -134,7 +147,8 @@
"bazelisk"
]
},
- "idempotent": true
+ "idempotent": true,
+ "service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Housekeeper-Weekly-UpdateCIPDPackages": {
"caches": [
@@ -150,37 +164,35 @@
"casSpec": "empty",
"cipd_packages": [
{
- "name": "infra/3pp/tools/git/linux-amd64",
- "path": "cipd_bin_packages",
- "version": "version:2@2.38.1.chromium.9"
- },
- {
- "name": "infra/tools/git/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "infra/tools/luci/git-credential-luci/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "skia/bots/bazelisk",
- "path": "bazelisk",
- "version": "version:0"
- },
- {
- "name": "skia/bots/go",
- "path": "go",
- "version": "version:16"
- },
- {
- "name": "skia/bots/protoc",
- "path": "protoc",
- "version": "version:0"
+ "name": "skia/tools/command_wrapper/${platform}",
+ "path": "task_drivers",
+ "version": "git_revision:b65a23129f1d7028291ccb0cead545d60b84d296"
}
],
"command": [
+ "./task_drivers/command_wrapper",
+ "--project_id",
+ "skia-swarming-bots",
+ "--task_id",
+ "<(TASK_ID)",
+ "--task_name",
+ "Housekeeper-Weekly-UpdateCIPDPackages",
+ "--workdir",
+ ".",
+ "--cipd",
+ "cipd_bin_packages:infra/3pp/tools/git/linux-amd64@version:2@2.38.1.chromium.9",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/git/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/luci/git-credential-luci/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "go:skia/bots/go@version:16",
+ "--cipd",
+ "protoc:skia/bots/protoc@version:0",
+ "--cipd",
+ "bazelisk:skia/bots/bazelisk@version:0",
+ "--command-is-task-driver",
+ "--",
"./roll_cipd_packages",
"--project_id",
"skia-swarming-bots",
@@ -232,32 +244,33 @@
"casSpec": "empty",
"cipd_packages": [
{
- "name": "infra/3pp/tools/git/linux-amd64",
- "path": "cipd_bin_packages",
- "version": "version:2@2.38.1.chromium.9"
- },
- {
- "name": "infra/tools/git/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "infra/tools/luci/git-credential-luci/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "skia/bots/bazelisk",
- "path": "bazelisk",
- "version": "version:0"
- },
- {
- "name": "skia/tools/bazel_build_all/${platform}",
+ "name": "skia/tools/command_wrapper/${platform}",
"path": "task_drivers",
- "version": "git_revision:2195169c3c8754d261f092b32704003ee21bd9bf"
+ "version": "git_revision:b65a23129f1d7028291ccb0cead545d60b84d296"
}
],
"command": [
+ "./task_drivers/command_wrapper",
+ "--project_id",
+ "skia-swarming-bots",
+ "--task_id",
+ "<(TASK_ID)",
+ "--task_name",
+ "Infra-PerCommit-Build-Bazel-Local",
+ "--workdir",
+ ".",
+ "--cipd",
+ "cipd_bin_packages:infra/3pp/tools/git/linux-amd64@version:2@2.38.1.chromium.9",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/git/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/luci/git-credential-luci/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "task_drivers:skia/tools/bazel_build_all/${platform}@git_revision:2195169c3c8754d261f092b32704003ee21bd9bf",
+ "--cipd",
+ "bazelisk:skia/bots/bazelisk@version:0",
+ "--command-is-task-driver",
+ "--",
"./task_drivers/bazel_build_all",
"--project_id",
"skia-swarming-bots",
@@ -302,32 +315,33 @@
"casSpec": "empty",
"cipd_packages": [
{
- "name": "infra/3pp/tools/git/linux-amd64",
- "path": "cipd_bin_packages",
- "version": "version:2@2.38.1.chromium.9"
- },
- {
- "name": "infra/tools/git/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "infra/tools/luci/git-credential-luci/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "skia/bots/bazelisk",
- "path": "bazelisk",
- "version": "version:0"
- },
- {
- "name": "skia/tools/bazel_build_all/${platform}",
+ "name": "skia/tools/command_wrapper/${platform}",
"path": "task_drivers",
- "version": "git_revision:2195169c3c8754d261f092b32704003ee21bd9bf"
+ "version": "git_revision:b65a23129f1d7028291ccb0cead545d60b84d296"
}
],
"command": [
+ "./task_drivers/command_wrapper",
+ "--project_id",
+ "skia-swarming-bots",
+ "--task_id",
+ "<(TASK_ID)",
+ "--task_name",
+ "Infra-PerCommit-Build-Bazel-RBE",
+ "--workdir",
+ ".",
+ "--cipd",
+ "cipd_bin_packages:infra/3pp/tools/git/linux-amd64@version:2@2.38.1.chromium.9",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/git/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/luci/git-credential-luci/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "task_drivers:skia/tools/bazel_build_all/${platform}@git_revision:2195169c3c8754d261f092b32704003ee21bd9bf",
+ "--cipd",
+ "bazelisk:skia/bots/bazelisk@version:0",
+ "--command-is-task-driver",
+ "--",
"./task_drivers/bazel_build_all",
"--project_id",
"skia-swarming-bots",
@@ -373,42 +387,37 @@
"casSpec": "empty",
"cipd_packages": [
{
- "name": "infra/3pp/tools/git/linux-amd64",
- "path": "cipd_bin_packages",
- "version": "version:2@2.38.1.chromium.9"
- },
- {
- "name": "infra/tools/git/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "infra/tools/luci/git-credential-luci/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "infra/tools/luci/isolate/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "infra/tools/luci/isolated/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:dc3a3dc4272aeef30698752d137ccd4f09526d69"
- },
- {
- "name": "skia/bots/bazelisk",
- "path": "bazelisk",
- "version": "version:0"
- },
- {
- "name": "skia/tools/bazel_test_all/${platform}",
+ "name": "skia/tools/command_wrapper/${platform}",
"path": "task_drivers",
- "version": "git_revision:2195169c3c8754d261f092b32704003ee21bd9bf"
+ "version": "git_revision:b65a23129f1d7028291ccb0cead545d60b84d296"
}
],
"command": [
+ "./task_drivers/command_wrapper",
+ "--project_id",
+ "skia-swarming-bots",
+ "--task_id",
+ "<(TASK_ID)",
+ "--task_name",
+ "Infra-PerCommit-Test-Bazel-Local",
+ "--workdir",
+ ".",
+ "--cipd",
+ "cipd_bin_packages:infra/3pp/tools/git/linux-amd64@version:2@2.38.1.chromium.9",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/git/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/luci/git-credential-luci/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/luci/isolate/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/luci/isolated/${platform}@git_revision:dc3a3dc4272aeef30698752d137ccd4f09526d69",
+ "--cipd",
+ "task_drivers:skia/tools/bazel_test_all/${platform}@git_revision:2195169c3c8754d261f092b32704003ee21bd9bf",
+ "--cipd",
+ "bazelisk:skia/bots/bazelisk@version:0",
+ "--command-is-task-driver",
+ "--",
"./task_drivers/bazel_test_all",
"--project_id",
"skia-swarming-bots",
@@ -455,42 +464,37 @@
"casSpec": "empty",
"cipd_packages": [
{
- "name": "infra/3pp/tools/git/linux-amd64",
- "path": "cipd_bin_packages",
- "version": "version:2@2.38.1.chromium.9"
- },
- {
- "name": "infra/tools/git/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "infra/tools/luci/git-credential-luci/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "infra/tools/luci/isolate/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024"
- },
- {
- "name": "infra/tools/luci/isolated/${platform}",
- "path": "cipd_bin_packages",
- "version": "git_revision:dc3a3dc4272aeef30698752d137ccd4f09526d69"
- },
- {
- "name": "skia/bots/bazelisk",
- "path": "bazelisk",
- "version": "version:0"
- },
- {
- "name": "skia/tools/bazel_test_all/${platform}",
+ "name": "skia/tools/command_wrapper/${platform}",
"path": "task_drivers",
- "version": "git_revision:2195169c3c8754d261f092b32704003ee21bd9bf"
+ "version": "git_revision:b65a23129f1d7028291ccb0cead545d60b84d296"
}
],
"command": [
+ "./task_drivers/command_wrapper",
+ "--project_id",
+ "skia-swarming-bots",
+ "--task_id",
+ "<(TASK_ID)",
+ "--task_name",
+ "Infra-PerCommit-Test-Bazel-RBE",
+ "--workdir",
+ ".",
+ "--cipd",
+ "cipd_bin_packages:infra/3pp/tools/git/linux-amd64@version:2@2.38.1.chromium.9",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/git/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/luci/git-credential-luci/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/luci/isolate/${platform}@git_revision:4f585e471d9b3ba86b2ac0ec0cdefaa8fdb67024",
+ "--cipd",
+ "cipd_bin_packages:infra/tools/luci/isolated/${platform}@git_revision:dc3a3dc4272aeef30698752d137ccd4f09526d69",
+ "--cipd",
+ "task_drivers:skia/tools/bazel_test_all/${platform}@git_revision:2195169c3c8754d261f092b32704003ee21bd9bf",
+ "--cipd",
+ "bazelisk:skia/bots/bazelisk@version:0",
+ "--command-is-task-driver",
+ "--",
"./task_drivers/bazel_test_all",
"--project_id",
"skia-swarming-bots",
