[BCID] Add build-images Go program
This will be packaged inside the "cd" Docker image to be used in Louhi
CD pipelines.
Change-Id: I2de2e672dd7ae1ced0629692f6a5f2d881423ec5
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/583457
Reviewed-by: Ravi Mistry <rmistry@google.com>
Commit-Queue: Eric Boren <borenet@google.com>
diff --git a/cd/Dockerfile b/cd/Dockerfile
index a67976e..75e7cd3 100644
--- a/cd/Dockerfile
+++ b/cd/Dockerfile
@@ -55,6 +55,7 @@
ARG CIPD_ROOT
ENV CIPD_ROOT=$CIPD_ROOT
COPY --from=install_pkgs ${CIPD_ROOT} ${CIPD_ROOT}
+COPY ./build-images /bin/build-images
ENV GOPATH=/go
RUN mkdir -p ${GOPATH}
ENV PATH="${CIPD_ROOT}/bin:${CIPD_ROOT}/go/bin:${CIPD_ROOT}:${GOPATH}/bin:${PATH}"
diff --git a/cd/Makefile b/cd/Makefile
new file mode 100644
index 0000000..a97b23f
--- /dev/null
+++ b/cd/Makefile
@@ -0,0 +1,9 @@
+include ../make/bazel.mk
+
+.PHONY: build-images
+build-images:
+ $(BAZEL) build //cd/go/build-images:build-images
+
+.PHONY: release
+release: build-images
+ ./build_release
\ No newline at end of file
diff --git a/cd/build_release b/cd/build_release
index 05fb846..9636a96 100755
--- a/cd/build_release
+++ b/cd/build_release
@@ -12,6 +12,7 @@
INSTALL_DIR="install -d --verbose --backup=none"
${INSTALL} --mode=644 -T Dockerfile ${ROOT}/Dockerfile
${INSTALL} --mode=644 -T ../infra/config/recipes.cfg ${ROOT}/recipes.cfg
+${INSTALL} --mode=755 -T ../_bazel_bin/cd/go/build-images/build-images_/build-images ${ROOT}/build-images
}
source ../bash/docker_build.sh
diff --git a/cd/go/build-images/BUILD.bazel b/cd/go/build-images/BUILD.bazel
new file mode 100644
index 0000000..c03ef80
--- /dev/null
+++ b/cd/go/build-images/BUILD.bazel
@@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+load("//bazel/go:go_test.bzl", "go_test")
+
+go_library(
+ name = "build-images_lib",
+ srcs = ["main.go"],
+ importpath = "go.skia.org/infra/cd/go/build-images",
+ visibility = ["//visibility:private"],
+ deps = [
+ "//go/common",
+ "//go/exec",
+ "//go/git",
+ "//task_driver/go/td",
+ "@org_golang_x_sync//errgroup",
+ ],
+)
+
+go_binary(
+ name = "build-images",
+ embed = [":build-images_lib"],
+ visibility = ["//visibility:public"],
+)
+
+go_test(
+ name = "build-images_test",
+ srcs = ["main_test.go"],
+ embed = [":build-images_lib"],
+ deps = ["@com_github_stretchr_testify//require"],
+)
diff --git a/cd/go/build-images/main.go b/cd/go/build-images/main.go
new file mode 100644
index 0000000..caa7b40
--- /dev/null
+++ b/cd/go/build-images/main.go
@@ -0,0 +1,153 @@
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "os/user"
+ "path"
+ "strings"
+ "time"
+
+ "go.skia.org/infra/go/common"
+ "go.skia.org/infra/go/exec"
+ "go.skia.org/infra/go/git"
+ "go.skia.org/infra/task_driver/go/td"
+ "golang.org/x/sync/errgroup"
+)
+
+/*
+ build-images is used for building Skia Infrastructure Docker images using
+ Bazel. It is intended to run inside of Louhi as part of a CD pipeline.
+*/
+
+func main() {
+ // Setup.
+ commit := flag.String("commit", "", "Commit at which to build the image.")
+ repo := flag.String("repo", "", "Repository URL.")
+ workspace := flag.String("workspace", "", "Path to Louhi workspace.")
+ targets := common.NewMultiStringFlag("target", nil, "Bazel target + image path pairs in the form \"//bazel-package:bazel-target:gcr.io/image/path\". ")
+ rbe := flag.Bool("rbe", false, "Whether or not to use Bazel RBE.")
+ username := flag.String("user", "", "User name to attribute the build. If not specified, attempt to determine automatically.")
+
+ // We're using the task driver framework because it provides logging and
+ // helpful insight into what's occurring as the program runs.
+ fakeProjectId := ""
+ fakeTaskId := ""
+ fakeTaskName := ""
+ output := "-"
+ local := true
+ ctx := td.StartRun(&fakeProjectId, &fakeTaskId, &fakeTaskName, &output, &local)
+ defer td.EndRun(ctx)
+
+ if *commit == "" {
+ td.Fatalf(ctx, "--commit is required.")
+ }
+ if *repo == "" {
+ td.Fatalf(ctx, "--repo is required.")
+ }
+ if *workspace == "" {
+ td.Fatalf(ctx, "--workspace is required.")
+ }
+ if len(*targets) == 0 {
+ td.Fatalf(ctx, "At least one --target is required.")
+ }
+ bazelTargetToImagePath := make(map[string]string, len(*targets))
+ for _, target := range *targets {
+ targetSplit := strings.Split(target, ":")
+ if len(targetSplit) != 3 {
+ td.Fatalf(ctx, "Invalid target specification %q; expected \"//bazel-target:bazel-target:gcr.io/image/path\"", target)
+ }
+ bazelTarget := strings.Join(targetSplit[:2], ":")
+ imagePath := targetSplit[2]
+ bazelTargetToImagePath[bazelTarget] = imagePath
+ }
+
+ // Create a shallow clone of the repo.
+ checkoutDir, err := shallowClone(ctx, *repo, *commit)
+ if err != nil {
+ td.Fatal(ctx, err)
+ }
+
+ // Create the timestamped Docker image tag.
+ // 2022-09-21T13_13_46Z-louhi-02b6ac9-clean
+ ts := time.Now().UTC().Format("2006-01-02T15_04_05Z")
+ if *username == "" {
+ userObj, err := user.Current() // TODO(borenet): Will this work in Louhi?
+ if err != nil {
+ td.Fatal(ctx, err)
+ }
+ *username = userObj.Username
+ }
+ imageTag := fmt.Sprintf("%s-%s-%s-%s", ts, *username, (*commit)[:7], "clean")
+
+ // Perform the builds concurrently.
+ eg, ctx := errgroup.WithContext(ctx)
+ for bazelTarget, imagePath := range bazelTargetToImagePath {
+ // https://golang.org/doc/faq#closures_and_goroutines
+ bazelTarget := bazelTarget
+ louhiImageTag := fmt.Sprintf("louhi_ws/%s:%s", imagePath, imageTag)
+ eg.Go(func() error {
+ return bazelRun(ctx, checkoutDir, bazelTarget, louhiImageTag, *rbe)
+ })
+ }
+ if err := eg.Wait(); err != nil {
+ td.Fatal(ctx, err)
+ }
+}
+
+// shallowClone creates a shallow clone of the given repo at the given commit.
+// Returns the location of the checkout or any error which occurred.
+func shallowClone(ctx context.Context, repoURL, commit string) (string, error) {
+ ctx = td.StartStep(ctx, td.Props("Clone"))
+ defer td.EndStep(ctx)
+
+ checkoutDir, err := ioutil.TempDir("", "")
+ if err != nil {
+ return "", td.FailStep(ctx, err)
+ }
+ git, err := git.Executable(ctx)
+ if err != nil {
+ return "", td.FailStep(ctx, err)
+ }
+ if _, err := exec.RunCwd(ctx, checkoutDir, git, "init"); err != nil {
+ return "", td.FailStep(ctx, err)
+ }
+ if _, err := exec.RunCwd(ctx, checkoutDir, git, "remote", "add", "origin", repoURL); err != nil {
+ return "", td.FailStep(ctx, err)
+ }
+ if _, err := exec.RunCwd(ctx, checkoutDir, git, "fetch", "--depth=1", "origin", commit); err != nil {
+ return "", td.FailStep(ctx, err)
+ }
+ if _, err := exec.RunCwd(ctx, checkoutDir, git, "checkout", "FETCH_HEAD"); err != nil {
+ return "", td.FailStep(ctx, err)
+ }
+ return checkoutDir, nil
+}
+
+// bazelTargetToDockerTag converts a Bazel target specification to a Docker
+// image tag which is applied to the image during the Bazel build.
+func bazelTargetToDockerTag(target string) string {
+ return path.Join("bazel", target)
+}
+
+// bazelRun executes `bazel run` for the given target and applies the given tag
+// to the resulting image.
+func bazelRun(ctx context.Context, cwd, target, louhiImageTag string, rbe bool) error {
+ ctx = td.StartStep(ctx, td.Props(fmt.Sprintf("Build %s", target)))
+ defer td.EndStep(ctx)
+
+ cmd := []string{"bazelisk", "run"}
+ if rbe {
+ cmd = append(cmd, "--config=remote", "--google_default_credentials")
+ }
+ cmd = append(cmd, target)
+ if _, err := exec.RunCwd(ctx, cwd, cmd...); err != nil {
+ return td.FailStep(ctx, err)
+ }
+ if _, err := exec.RunCwd(ctx, cwd, "docker", "tag", bazelTargetToDockerTag(target), louhiImageTag); err != nil {
+ return td.FailStep(ctx, err)
+ }
+ return nil
+}
diff --git a/cd/go/build-images/main_test.go b/cd/go/build-images/main_test.go
new file mode 100644
index 0000000..925014a
--- /dev/null
+++ b/cd/go/build-images/main_test.go
@@ -0,0 +1,16 @@
+package main
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestBazelTargetToDockerTag(t *testing.T) {
+ tc := map[string]string{
+ "//task_scheduler:task_scheduler_jc_container": "bazel/task_scheduler:task_scheduler_jc_container",
+ }
+ for input, expect := range tc {
+ require.Equal(t, expect, bazelTargetToDockerTag(input))
+ }
+}
