Setup autodeployment for api's bazel build
* Using docker.New instead of docker.Login which needed configDir passed around.
* Also cleaned up unneeded references to infraCommonEnv and infraCommonBuildArgs.
Bug: skia:12812
Change-Id: Ie60ad77d75a5a17f9350e0028afb8c54bf4c63cb
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/494581
Reviewed-by: Kevin Lubick <kjlubick@google.com>
Commit-Queue: Ravi Mistry <rmistry@google.com>
diff --git a/infra/bots/task_drivers/push_apps_from_skia_image/push_apps_from_skia_image.go b/infra/bots/task_drivers/push_apps_from_skia_image/push_apps_from_skia_image.go
index ffb58e5..9086edd 100644
--- a/infra/bots/task_drivers/push_apps_from_skia_image/push_apps_from_skia_image.go
+++ b/infra/bots/task_drivers/push_apps_from_skia_image/push_apps_from_skia_image.go
@@ -11,10 +11,8 @@
import (
"context"
- "errors"
"flag"
"fmt"
- "io/ioutil"
"os"
"path"
"path/filepath"
@@ -26,7 +24,6 @@
"go.skia.org/infra/go/common"
docker_pubsub "go.skia.org/infra/go/docker/build/pubsub"
sk_exec "go.skia.org/infra/go/exec"
- "go.skia.org/infra/go/util"
"go.skia.org/infra/task_driver/go/lib/auth_steps"
"go.skia.org/infra/task_driver/go/lib/bazel"
"go.skia.org/infra/task_driver/go/lib/checkout"
@@ -57,31 +54,7 @@
apiImageName = "api"
)
-var (
- infraCommonEnv = []string{
- "SKIP_BUILD=1",
- "ROOT=/OUT",
- }
- infraCommonBuildArgs = map[string]string{
- "SKIA_IMAGE_NAME": "skia-release",
- }
-)
-
-func buildPushFiddlerImage(ctx context.Context, tag, repo, configDir string, topic *pubsub.Topic) error {
- // Checkout out the Skia infra repo at the specified commit.
- if *infraRevision == "" {
- return errors.New("Must specify --infra_revision")
- }
- // Check out the Skia infra repo at the specified commit.
- rs := types.RepoState{
- Repo: common.REPO_SKIA_INFRA,
- Revision: *infraRevision,
- }
- infraCheckoutDir := filepath.Join("infra_repo")
- if _, err := checkout.EnsureGitCheckout(ctx, infraCheckoutDir, rs); err != nil {
- return err
- }
-
+func buildPushFiddlerImage(ctx context.Context, dkr *docker.Docker, tag, infraCheckoutDir string, topic *pubsub.Topic) error {
// Run skia-release image and extract products out of /tmp/skia/skia. See
// https://skia.googlesource.com/skia/+/0e845dc8b05cb2d40d1c880184e33dd76081283a/docker/skia-release/Dockerfile#33
productsDir, err := os_steps.TempDir(ctx, "", "")
@@ -93,15 +66,7 @@
}
skiaCopyCmd := []string{"/bin/sh", "-c", "cd /tmp; tar cvzf skia.tar.gz --directory=/tmp/skia skia; cp /tmp/skia.tar.gz /OUT/"}
releaseImg := fmt.Sprintf("gcr.io/skia-public/skia-release:%s", tag)
- if err := docker.Run(ctx, releaseImg, configDir, skiaCopyCmd, volumes, nil); err != nil {
- return err
- }
-
- // Ensure that the bazel cache is setup.
- opts := bazel.BazelOptions{
- CachePath: "/mnt/pd0/bazel_cache",
- }
- if err := bazel.EnsureBazelRCFile(ctx, opts); err != nil {
+ if err := dkr.Run(ctx, releaseImg, skiaCopyCmd, volumes, nil); err != nil {
return err
}
@@ -131,26 +96,17 @@
return err
}
- // Remove all temporary files from the host machine. Swarming gets upset if there are root-owned
- // files it cannot clean up.
- // TODO(rmistry): Move to cleanupTempFiles after api is migrated to bazel.
- cleanupCmd := []string{"/bin/sh", "-c", "rm -rf /OUT/*"}
- if err := docker.Run(ctx, releaseImg, configDir, cleanupCmd, volumes, nil); err != nil {
- return err
- }
-
- return nil
+ return cleanupTempFiles(ctx, dkr, releaseImg, volumes)
}
-func cleanupTempFiles(ctx context.Context, configDir string, volumes []string) error {
+func cleanupTempFiles(ctx context.Context, dkr *docker.Docker, image string, volumes []string) error {
// Remove all temporary files from the host machine. Swarming gets upset if there are root-owned
// files it cannot clean up.
- const infraImageWithTag = "gcr.io/skia-public/infra:prod"
cleanupCmd := []string{"/bin/sh", "-c", "rm -rf /OUT/*"}
- return docker.Run(ctx, infraImageWithTag, configDir, cleanupCmd, volumes, nil)
+ return dkr.Run(ctx, image, cleanupCmd, volumes, nil)
}
-func buildPushApiImage(ctx context.Context, tag, repo, configDir, checkoutDir string, topic *pubsub.Topic) error {
+func buildPushApiImage(ctx context.Context, dkr *docker.Docker, tag, checkoutDir, infraCheckoutDir string, topic *pubsub.Topic) error {
tempDir, err := os_steps.TempDir(ctx, "", "")
if err != nil {
return err
@@ -168,20 +124,38 @@
"OUTPUT_DIRECTORY=/OUT",
}
doxygenCmd := []string{"/bin/sh", "-c", "cd /CHECKOUT/tools/doxygen && doxygen ProdDoxyfile"}
- if err := docker.Run(ctx, "gcr.io/skia-public/doxygen:testing-slim", configDir, doxygenCmd, volumes, env); err != nil {
+ doxygenImg := "gcr.io/skia-public/doxygen:testing-slim"
+ if err := dkr.Run(ctx, doxygenImg, doxygenCmd, volumes, env); err != nil {
return err
}
- image := fmt.Sprintf("gcr.io/skia-public/%s", apiImageName)
- cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/api && make release_ci"}
- infraEnv := util.CopyStringSlice(infraCommonEnv)
- infraEnv = append(infraEnv, "DOXYGEN_HTML=/OUT/html")
- infraVolumes := []string{fmt.Sprintf("%s:/OUT", tempDir)}
- err = docker.BuildPushImageFromInfraImage(ctx, "Api", image, tag, repo, configDir, tempDir, "prod", topic, cmd, infraVolumes, infraEnv, infraCommonBuildArgs)
+ err = td.Do(ctx, td.Props("Build "+apiImageName+" image").Infra(), func(ctx context.Context) error {
+ runCmd := &sk_exec.Command{
+ Name: "make",
+ Args: []string{"release-api-ci"},
+ InheritEnv: true,
+ Env: []string{
+ "COPY_FROM_DIR=" + filepath.Join(tempDir, "html"),
+ "STABLE_DOCKER_TAG=" + tag,
+ },
+ Dir: filepath.Join(infraCheckoutDir, "api"),
+ LogStdout: true,
+ LogStderr: true,
+ }
+ _, err := sk_exec.RunCommand(ctx, runCmd)
+ if err != nil {
+ return err
+ }
+ return nil
+ })
if err != nil {
return err
}
- return cleanupTempFiles(ctx, configDir, volumes)
+ if err := docker.PublishToTopic(ctx, "gcr.io/skia-public/"+apiImageName, tag, common.REPO_SKIA, topic); err != nil {
+ return err
+ }
+
+ return cleanupTempFiles(ctx, dkr, doxygenImg, volumes)
}
func main() {
@@ -189,25 +163,46 @@
ctx := td.StartRun(projectId, taskId, taskName, output, local)
defer td.EndRun(ctx)
+ if *infraRevision == "" {
+ td.Fatalf(ctx, "Must specify --infra_revision")
+ }
+
rs, err := checkout.GetRepoState(checkoutFlags)
if err != nil {
td.Fatal(ctx, err)
}
-
wd, err := os_steps.Abs(ctx, *workdir)
if err != nil {
td.Fatal(ctx, err)
}
-
- // Check out the code.
+ // Check out the Skia repo code.
co, err := checkout.EnsureGitCheckout(ctx, path.Join(wd, "repo"), rs)
if err != nil {
td.Fatal(ctx, err)
}
+ skiaCheckoutDir := co.Dir()
+
+ // Checkout out the Skia infra repo at the specified commit.
+ infraRS := types.RepoState{
+ Repo: common.REPO_SKIA_INFRA,
+ Revision: *infraRevision,
+ }
+ infraCheckoutDir := filepath.Join("infra_repo")
+ if _, err := checkout.EnsureGitCheckout(ctx, infraCheckoutDir, infraRS); err != nil {
+ td.Fatal(ctx, err)
+ }
// Setup go.
ctx = golang.WithEnv(ctx, wd)
+ // Ensure that the bazel cache is setup.
+ opts := bazel.BazelOptions{
+ CachePath: "/mnt/pd0/bazel_cache",
+ }
+ if err := bazel.EnsureBazelRCFile(ctx, opts); err != nil {
+ td.Fatal(ctx, err)
+ }
+
// Create token source with scope for cloud registry (storage) and pubsub.
ts, err := auth_steps.Init(ctx, *local, auth.ScopeUserinfoEmail, auth.ScopeFullControl, pubsub.ScopePubSub)
if err != nil {
@@ -226,36 +221,18 @@
if rs.Issue != "" && rs.Patchset != "" {
tag = fmt.Sprintf("%s_%s", rs.Issue, rs.Patchset)
}
- // Add the tag to infraCommonBuildArgs.
- infraCommonBuildArgs["SKIA_IMAGE_TAG"] = tag
- // Create a temporary config dir for Docker.
- configDir, err := ioutil.TempDir("", "")
+ // Instantiate docker.
+ dkr, err := docker.New(ctx, ts)
if err != nil {
td.Fatal(ctx, err)
}
- defer util.RemoveAll(configDir)
-
- // Login to docker (required to push to docker).
- token, err := ts.Token()
- if err != nil {
- td.Fatal(ctx, err)
- }
- if err := docker.Login(ctx, token.AccessToken, "gcr.io/skia-public/", configDir); err != nil {
- td.Fatal(ctx, err)
- }
- // TODO(rmistry): After api is migrated to bazel use this way to instantiate
- // docker instead:
- // dkr, err := docker.New(ctx, ts)
- // if err != nil {
- // td.Fatal(ctx, err)
- // }
// Build and push all apps of interest below.
- if err := buildPushApiImage(ctx, tag, rs.Repo, configDir, co.Dir(), topic); err != nil {
+ if err := buildPushApiImage(ctx, dkr, tag, skiaCheckoutDir, infraCheckoutDir, topic); err != nil {
td.Fatal(ctx, err)
}
- if err := buildPushFiddlerImage(ctx, tag, rs.Repo, configDir, topic); err != nil {
+ if err := buildPushFiddlerImage(ctx, dkr, tag, infraCheckoutDir, topic); err != nil {
td.Fatal(ctx, err)
}
}
