[canvaskit] Add job to compile GMTests js/wasm
Next step is to add the following task:
Test-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-WasmGMTests_WebGL2
Bug: skia:10812
Change-Id: Ibe45b7205cebd30f0e7904ea6d93a01ea3df87fe
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/324617
Commit-Queue: Kevin Lubick <kjlubick@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
diff --git a/infra/bots/gen_tasks_logic/gen_tasks_logic.go b/infra/bots/gen_tasks_logic/gen_tasks_logic.go
index 60c0dcf..82e07f4 100644
--- a/infra/bots/gen_tasks_logic/gen_tasks_logic.go
+++ b/infra/bots/gen_tasks_logic/gen_tasks_logic.go
@@ -442,7 +442,7 @@
"NoGPUThreads", "ProcDump", "DDL1", "DDL3", "OOPRDDL", "T8888",
"DDLTotal", "DDLRecord", "9x9", "BonusConfigs", "SkottieTracing", "SkottieWASM",
"GpuTess", "NonNVPR", "Mskp", "Docker", "PDF", "SkVM", "Puppeteer",
- "SkottieFrames", "RenderSKP", "CanvasPerf", "AllPathsVolatile"}
+ "SkottieFrames", "RenderSKP", "CanvasPerf", "AllPathsVolatile", "WebGL2"}
keep := make([]string, 0, len(ec))
for _, part := range ec {
if !In(part, ignore) {
@@ -1015,84 +1015,88 @@
// compile generates a compile task. Returns the name of the compile task.
func (b *jobBuilder) compile() string {
name := b.deriveCompileTaskName()
- b.addTask(name, func(b *taskBuilder) {
- recipe := "compile"
- isolate := "compile.isolate"
- if b.extraConfig("NoDEPS", "CMake", "CommandBuffer", "Flutter", "SKQP") {
- recipe = "sync_and_compile"
- isolate = "swarm_recipe.isolate"
- b.recipeProps(EXTRA_PROPS)
- b.usesGit()
- if !b.extraConfig("NoDEPS") {
- b.cache(CACHES_WORKDIR...)
+ if b.extraConfig("WasmGMTests") {
+ b.compileWasmGMTests(name)
+ } else {
+ b.addTask(name, func(b *taskBuilder) {
+ recipe := "compile"
+ isolate := "compile.isolate"
+ if b.extraConfig("NoDEPS", "CMake", "CommandBuffer", "Flutter", "SKQP") {
+ recipe = "sync_and_compile"
+ isolate = "swarm_recipe.isolate"
+ b.recipeProps(EXTRA_PROPS)
+ b.usesGit()
+ if !b.extraConfig("NoDEPS") {
+ b.cache(CACHES_WORKDIR...)
+ }
+ } else {
+ b.idempotent()
}
- } else {
- b.idempotent()
- }
- b.kitchenTask(recipe, OUTPUT_BUILD)
- b.isolate(isolate)
- b.serviceAccount(b.cfg.ServiceAccountCompile)
- b.swarmDimensions()
- if b.extraConfig("Docker", "LottieWeb", "SKQP", "CMake") || b.compiler("EMCC") {
- b.usesDocker()
- b.cache(CACHES_DOCKER...)
- }
+ b.kitchenTask(recipe, OUTPUT_BUILD)
+ b.isolate(isolate)
+ b.serviceAccount(b.cfg.ServiceAccountCompile)
+ b.swarmDimensions()
+ if b.extraConfig("Docker", "LottieWeb", "SKQP", "CMake") || b.compiler("EMCC") {
+ b.usesDocker()
+ b.cache(CACHES_DOCKER...)
+ }
- // Android bots require a toolchain.
- if b.extraConfig("Android") {
- if b.matchOs("Mac") {
- b.asset("android_ndk_darwin")
- } else if b.matchOs("Win") {
- pkg := b.MustGetCipdPackageFromAsset("android_ndk_windows")
- pkg.Path = "n"
- b.cipd(pkg)
- } else if !b.extraConfig("SKQP") {
- b.asset("android_ndk_linux")
- }
- } else if b.extraConfig("Chromebook") {
- b.asset("clang_linux")
- if b.arch("x86_64") {
- b.asset("chromebook_x86_64_gles")
- } else if b.arch("arm") {
- b.asset("armhf_sysroot")
- b.asset("chromebook_arm_gles")
- }
- } else if b.isLinux() {
- if b.compiler("Clang") {
+ // Android bots require a toolchain.
+ if b.extraConfig("Android") {
+ if b.matchOs("Mac") {
+ b.asset("android_ndk_darwin")
+ } else if b.matchOs("Win") {
+ pkg := b.MustGetCipdPackageFromAsset("android_ndk_windows")
+ pkg.Path = "n"
+ b.cipd(pkg)
+ } else if !b.extraConfig("SKQP") {
+ b.asset("android_ndk_linux")
+ }
+ } else if b.extraConfig("Chromebook") {
b.asset("clang_linux")
+ if b.arch("x86_64") {
+ b.asset("chromebook_x86_64_gles")
+ } else if b.arch("arm") {
+ b.asset("armhf_sysroot")
+ b.asset("chromebook_arm_gles")
+ }
+ } else if b.isLinux() {
+ if b.compiler("Clang") {
+ b.asset("clang_linux")
+ }
+ if b.extraConfig("SwiftShader") {
+ b.asset("cmake_linux")
+ }
+ if b.extraConfig("OpenCL") {
+ b.asset("opencl_headers", "opencl_ocl_icd_linux")
+ }
+ b.asset("ccache_linux")
+ b.usesCCache()
+ } else if b.matchOs("Win") {
+ b.asset("win_toolchain")
+ if b.compiler("Clang") {
+ b.asset("clang_win")
+ }
+ if b.extraConfig("OpenCL") {
+ b.asset("opencl_headers")
+ }
+ } else if b.matchOs("Mac") {
+ b.cipd(CIPD_PKGS_XCODE...)
+ b.Spec.Caches = append(b.Spec.Caches, &specs.Cache{
+ Name: "xcode",
+ Path: "cache/Xcode.app",
+ })
+ b.asset("ccache_mac")
+ b.usesCCache()
+ if b.extraConfig("CommandBuffer") {
+ b.timeout(2 * time.Hour)
+ }
+ if b.extraConfig("iOS") {
+ b.asset("provisioning_profile_ios")
+ }
}
- if b.extraConfig("SwiftShader") {
- b.asset("cmake_linux")
- }
- if b.extraConfig("OpenCL") {
- b.asset("opencl_headers", "opencl_ocl_icd_linux")
- }
- b.asset("ccache_linux")
- b.usesCCache()
- } else if b.matchOs("Win") {
- b.asset("win_toolchain")
- if b.compiler("Clang") {
- b.asset("clang_win")
- }
- if b.extraConfig("OpenCL") {
- b.asset("opencl_headers")
- }
- } else if b.matchOs("Mac") {
- b.cipd(CIPD_PKGS_XCODE...)
- b.Spec.Caches = append(b.Spec.Caches, &specs.Cache{
- Name: "xcode",
- Path: "cache/Xcode.app",
- })
- b.asset("ccache_mac")
- b.usesCCache()
- if b.extraConfig("CommandBuffer") {
- b.timeout(2 * time.Hour)
- }
- if b.extraConfig("iOS") {
- b.asset("provisioning_profile_ios")
- }
- }
- })
+ })
+ }
// All compile tasks are runnable as their own Job. Assert that the Job
// is listed in jobs.
@@ -1714,3 +1718,34 @@
})
})
}
+
+// compileWasmGMTests uses a task driver to compile the GMs and unit tests for Web Assembly (WASM)
+// using WebGL if necessary.
+func (b *jobBuilder) compileWasmGMTests(compileName string) {
+ b.addTask(compileName, func(b *taskBuilder) {
+ b.attempts(1)
+ b.usesDocker()
+ b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
+ b.cipd(CIPD_PKG_LUCI_AUTH)
+ b.dep(b.buildTaskDrivers())
+ b.output("wasm_out")
+ b.timeout(20 * time.Minute)
+ b.isolate("compile.isolate")
+ b.serviceAccount(b.cfg.ServiceAccountCompile)
+ b.cache(CACHES_DOCKER...)
+ // For now, we only have one compile mode - a GPU release mode. This should be sufficient to
+ // run CPU, WebGL1, and WebGL2 tests. Debug mode is not needed for the waterfall because
+ // when using puppeteer, stacktraces from exceptions are hard to get access to, so we do not
+ // even bother.
+ b.cmd(
+ "./compile_wasm_gm_tests",
+ "--project_id", "skia-swarming-bots",
+ "--task_id", specs.PLACEHOLDER_TASK_ID,
+ "--task_name", compileName,
+ "--out_path", "./wasm_out",
+ "--skia_path", "./skia",
+ "--work_path", "./cache/docker/wasm_gm",
+ "--alsologtostderr",
+ )
+ })
+}
diff --git a/infra/bots/jobs.json b/infra/bots/jobs.json
index 3d4e356..8d70099 100644
--- a/infra/bots/jobs.json
+++ b/infra/bots/jobs.json
@@ -75,6 +75,7 @@
"Build-Debian10-EMCC-wasm-Release-CanvasKit",
"Build-Debian10-EMCC-wasm-Release-CanvasKit_CPU",
"Build-Debian10-EMCC-wasm-Release-PathKit",
+ "Build-Debian10-EMCC-wasm-Release-WasmGMTests",
"Build-Mac10.15.5-Clang-arm64-Debug-iOS",
"Build-Mac10.15.5-Clang-arm64-Debug-iOS_Metal",
"Build-Mac10.15.5-Clang-arm64-Release-iOS",
diff --git a/infra/bots/task_drivers/cifuzz/cifuzz.go b/infra/bots/task_drivers/cifuzz/cifuzz.go
index fec7254..4f380bd 100644
--- a/infra/bots/task_drivers/cifuzz/cifuzz.go
+++ b/infra/bots/task_drivers/cifuzz/cifuzz.go
@@ -110,8 +110,6 @@
cifuzzDockerImage = "gcr.io/oss-fuzz-base/cifuzz-base:latest"
buildFuzzersDockerImage = "local_build_fuzzers"
runFuzzersDockerImage = "local_run_fuzzers"
-
- pinnedSwiftshaderRevision = "45510ad8a77862c1ce2e33f0efed41544f5f048b"
)
func setupCIFuzzRepoAndDocker(ctx context.Context, workdir, gitAbsPath string) error {
diff --git a/infra/bots/task_drivers/compile_wasm_gm_tests/compile_wasm_gm_tests.go b/infra/bots/task_drivers/compile_wasm_gm_tests/compile_wasm_gm_tests.go
new file mode 100644
index 0000000..9b49264
--- /dev/null
+++ b/infra/bots/task_drivers/compile_wasm_gm_tests/compile_wasm_gm_tests.go
@@ -0,0 +1,117 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package main
+
+import (
+ "context"
+ "flag"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "cloud.google.com/go/storage"
+ "go.skia.org/infra/go/auth"
+ "go.skia.org/infra/go/skerr"
+ "go.skia.org/infra/task_driver/go/lib/auth_steps"
+ "go.skia.org/infra/task_driver/go/lib/docker"
+ "go.skia.org/infra/task_driver/go/lib/os_steps"
+ "go.skia.org/infra/task_driver/go/td"
+)
+
+const dockerImage = "gcr.io/skia-public/canvaskit-emsdk:2.0.6_v1"
+const innerBuildScript = "/SRC/infra/canvaskit/build_gmtests.sh"
+
+func main() {
+ var (
+ // Required properties for this task.
+ outPath = flag.String("out_path", "", "The directory to put the built wasm/js code.")
+ projectID = flag.String("project_id", "", "ID of the Google Cloud project.")
+ skiaPath = flag.String("skia_path", "", "Path to skia repo root.")
+ taskID = flag.String("task_id", "", "task id this data was generated on")
+ taskName = flag.String("task_name", "", "Name of the task.")
+ workPath = flag.String("work_path", "", "The directory to use to store temporary files (e.g. docker build)")
+
+ // Debugging flags.
+ local = flag.Bool("local", false, "True if running locally (as opposed to on the bots)")
+ outputSteps = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.")
+ )
+
+ // Setup.
+ ctx := td.StartRun(projectID, taskID, taskName, outputSteps, local)
+ defer td.EndRun(ctx)
+
+ outAbsPath := getAbsoluteOfRequiredFlag(ctx, *outPath, "out_path")
+ skiaAbsPath := getAbsoluteOfRequiredFlag(ctx, *skiaPath, "skia_path")
+ workAbsPath := getAbsoluteOfRequiredFlag(ctx, *workPath, "work_path")
+
+ if err := os_steps.MkdirAll(ctx, workAbsPath); err != nil {
+ td.Fatal(ctx, err)
+ }
+ if err := os_steps.MkdirAll(ctx, outAbsPath); err != nil {
+ td.Fatal(ctx, err)
+ }
+
+ doc, err := setupDocker(ctx, *local)
+ if err != nil {
+ td.Fatal(ctx, err)
+ }
+ defer doc.Cleanup(ctx)
+
+ command := []string{innerBuildScript}
+ volumes := []string{skiaAbsPath + ":/SRC", workAbsPath + ":/OUT"}
+
+ if err := doc.Run(ctx, dockerImage, command, volumes, nil); err != nil {
+ td.Fatal(ctx, err)
+ }
+
+ if err := extractOutput(ctx, workAbsPath, outAbsPath); err != nil {
+ td.Fatal(ctx, err)
+ }
+}
+
+func setupDocker(ctx context.Context, isLocal bool) (*docker.Docker, error) {
+ ctx = td.StartStep(ctx, td.Props("setup docker").Infra())
+ defer td.EndStep(ctx)
+ // Create token source with scope for cloud registry (storage).
+ ts, err := auth_steps.Init(ctx, isLocal, auth.SCOPE_USERINFO_EMAIL, storage.ScopeReadOnly)
+ if err != nil {
+ return nil, td.FailStep(ctx, err)
+ }
+
+ return docker.New(ctx, ts)
+}
+
+func getAbsoluteOfRequiredFlag(ctx context.Context, nonEmptyPath, flag string) string {
+ if nonEmptyPath == "" {
+ td.Fatalf(ctx, "--%s must be specified", flag)
+ }
+ absPath, err := filepath.Abs(nonEmptyPath)
+ if err != nil {
+ td.Fatalf(ctx, "error with path %s - %s", nonEmptyPath, err)
+ }
+ return absPath
+}
+
+func extractOutput(ctx context.Context, workDir, outAbsPath string) error {
+ ctx = td.StartStep(ctx, td.Props("copy compiled JS and wasm into output"))
+ defer td.EndStep(ctx)
+
+ files, err := os_steps.ReadDir(ctx, workDir)
+ if err != nil {
+ return td.FailStep(ctx, skerr.Wrapf(err, "getting output from %s", workDir))
+ }
+
+ for _, f := range files {
+ name := f.Name()
+ if strings.Contains(name, "wasm_gm_tests") {
+ oldFile := filepath.Join(workDir, name)
+ newFile := filepath.Join(outAbsPath, name)
+ if err := os.Rename(oldFile, newFile); err != nil {
+ return td.FailStep(ctx, skerr.Wrapf(err, "copying %s to %s", oldFile, newFile))
+ }
+ }
+ }
+ return nil
+}
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index e33be2f..b868a14 100755
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -330,6 +330,11 @@
"Build-Debian10-EMCC-wasm-Release-PathKit"
]
},
+ "Build-Debian10-EMCC-wasm-Release-WasmGMTests": {
+ "tasks": [
+ "Build-Debian10-EMCC-wasm-Release-WasmGMTests"
+ ]
+ },
"Build-Debian10-GCC-x86-Debug-Docker": {
"tasks": [
"Build-Debian10-GCC-x86-Debug-Docker"
@@ -8477,6 +8482,56 @@
],
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
+ "Build-Debian10-EMCC-wasm-Release-WasmGMTests": {
+ "caches": [
+ {
+ "name": "docker",
+ "path": "cache/docker"
+ }
+ ],
+ "cipd_packages": [
+ {
+ "name": "infra/tools/luci-auth/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:08768c6d238082f3c552dcabef6aaf4c6792d91a"
+ }
+ ],
+ "command": [
+ "./compile_wasm_gm_tests",
+ "--project_id",
+ "skia-swarming-bots",
+ "--task_id",
+ "<(TASK_ID)",
+ "--task_name",
+ "Build-Debian10-EMCC-wasm-Release-WasmGMTests",
+ "--out_path",
+ "./wasm_out",
+ "--skia_path",
+ "./skia",
+ "--work_path",
+ "./cache/docker/wasm_gm",
+ "--alsologtostderr"
+ ],
+ "dependencies": [
+ "Housekeeper-PerCommit-BuildTaskDrivers"
+ ],
+ "dimensions": [
+ "docker_installed:true",
+ "cpu:x86-64-Haswell_GCE",
+ "gpu:none",
+ "machine_type:n1-standard-16",
+ "os:Debian-10.3",
+ "pool:Skia"
+ ],
+ "execution_timeout_ns": 1200000000000,
+ "io_timeout_ns": 1200000000000,
+ "isolate": "compile.isolate",
+ "max_attempts": 1,
+ "outputs": [
+ "wasm_out"
+ ],
+ "service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
+ },
"Build-Debian10-GCC-x86-Debug-Docker": {
"caches": [
{
diff --git a/infra/canvaskit/build_gmtests.sh b/infra/canvaskit/build_gmtests.sh
new file mode 100755
index 0000000..ba7e8bd
--- /dev/null
+++ b/infra/canvaskit/build_gmtests.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This assumes it is being run inside a docker container of emsdk-base
+# and a Skia checkout has been mounted at /SRC and the output directory
+# is mounted at /OUT
+
+set +e
+set -x
+# Clean out previous builds (ignoring any errors for things like folders)
+# (e.g. we don't want to delete /OUT/depot_tools/)
+rm -f /OUT/*
+set -e
+
+# BASE_DIR is the dir this script is in ($SKIA_ROOT/infra/canvaskit)
+BASE_DIR=`cd $(dirname ${BASH_SOURCE[0]}) && pwd`
+CANVASKIT_DIR=$BASE_DIR/../../modules/canvaskit
+
+BUILD_DIR=/OUT $CANVASKIT_DIR/compile_gm.sh $@
+
+# Make sure everybody can read and write the contents of /OUT
+chmod -R a+rw /OUT/*
+
diff --git a/modules/canvaskit/README.md b/modules/canvaskit/README.md
index fed7bd5..8294c61 100644
--- a/modules/canvaskit/README.md
+++ b/modules/canvaskit/README.md
@@ -133,6 +133,8 @@
7. In `$SKIA_ROOT/infra/bots/recipe_modules/build/`, update `canvaskit.py`
and `pathkit.py` to have `DOCKER_IMAGE` point to the desired tagged Docker
containers from steps 2 and 5 (which should be the same).
+ 8. In `$SKIA_ROOT/infra/bots/task_drivers/compile_wasm_gm_tests.go`, update dockerImage
+ to refer to the desired Docker containers from steps 2 and 5.
9. In `$SKIA_ROOT/infra/bots/`, run `make train` to re-train the recipes.
10. Optional: Run something like `git grep 1\\.38\\.` in `$SKIA_ROOT` to see if
there are any other references that need updating.
