| // Copyright 2023 Google LLC |
| // |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| package main |
| |
| import ( |
| "context" |
| "strings" |
| "testing" |
| "time" |
| |
| "github.com/stretchr/testify/assert" |
| "github.com/stretchr/testify/require" |
| "go.skia.org/infra/go/exec" |
| exec_testutils "go.skia.org/infra/go/exec/testutils" |
| "go.skia.org/infra/go/gcs" |
| "go.skia.org/infra/go/gcs/mocks" |
| "go.skia.org/infra/go/now" |
| infra_testutils "go.skia.org/infra/go/testutils" |
| "go.skia.org/infra/task_driver/go/lib/os_steps" |
| "go.skia.org/infra/task_driver/go/td" |
| "go.skia.org/skia/infra/bots/task_drivers/common" |
| "go.skia.org/skia/infra/bots/task_drivers/testutils" |
| ) |
| |
| func TestRun_UnitTest_Success(t *testing.T) { |
| commandCollector := exec.CommandCollector{} |
| res := td.RunTestSteps(t, false, func(ctx context.Context) error { |
| ctx = td.WithExecRunFn(ctx, commandCollector.Run) |
| |
| err := run(ctx, taskDriverArgs{ |
| commandPath: "/path/to/command", |
| commandWorkDir: "/path/to/workdir", |
| testKind: unitTest, |
| }) |
| |
| assert.NoError(t, err) |
| return err |
| }) |
| |
| require.Empty(t, res.Errors) |
| require.Empty(t, res.Exceptions) |
| |
| testutils.AssertStepNames(t, res, "/path/to/command") |
| |
| assert.Equal(t, "/path/to/workdir", commandCollector.Commands()[0].Dir) |
| exec_testutils.AssertCommandsMatch(t, [][]string{{"/path/to/command"}}, commandCollector.Commands()) |
| } |
| |
| func TestRun_GMTest_Success(t *testing.T) { |
| // Given that we have tests for common.UploadToGold(), it suffices to test a couple of |
| // "happy" cases here. |
| |
| test := func(name string, tdArgs taskDriverArgs, goldctlWorkDir string, goldctlImgtestInitStepName string, goldctlImgtestInitArgs []string) { |
| t.Run(name, func(t *testing.T) { |
| // Create directory with fake undeclared test outputs. |
| testutils.PopulateDir(t, tdArgs.undeclaredOutputsDir, map[string]string{ |
| // The contents of PNG files does not matter for this test. |
| "alfa.png": "fake PNG", |
| "alfa.json": `{ |
| "md5": "a01a01a01a01a01a01a01a01a01a01a0", |
| "keys": { |
| "build_system": "bazel", |
| "name": "alfa", |
| "source_type": "gm" |
| } |
| }`, |
| "beta.png": "fake PNG", |
| "beta.json": `{ |
| "md5": "b02b02b02b02b02b02b02b02b02b02b0", |
| "keys": { |
| "build_system": "bazel", |
| "name": "beta", |
| "source_type": "gm" |
| } |
| }`, |
| }) |
| |
| commandCollector := exec.CommandCollector{} |
| res := td.RunTestSteps(t, false, func(ctx context.Context) error { |
| ctx = td.WithExecRunFn(ctx, commandCollector.Run) |
| |
| // We don't need to assert the exact number of times that os_steps.TempDir() is called |
| // because said function produces a "Creating TempDir" task driver step, and we check the |
| // exact set of steps produced. |
| ctx = context.WithValue(ctx, os_steps.TempDirContextKey, testutils.MakeTempDirMockFn(t, goldctlWorkDir)) |
| |
| err := run(ctx, tdArgs) |
| |
| assert.NoError(t, err) |
| return err |
| }) |
| |
| require.Empty(t, res.Errors) |
| require.Empty(t, res.Exceptions) |
| |
| testutils.AssertStepNames(t, res, |
| "/path/to/command --device-specific-bazel-config Pixel5 --key arch arm64 model Pixel5 os Android --gpuName Adreno620", |
| "Gather JSON and PNG files produced by GMs", |
| "Gather \"alfa.png\"", |
| "Gather \"beta.png\"", |
| "Upload GM outputs to Gold", |
| "Creating TempDir", |
| "/path/to/goldctl auth --work-dir "+goldctlWorkDir+" --luci", |
| goldctlImgtestInitStepName, |
| "/path/to/goldctl imgtest add --work-dir "+goldctlWorkDir+" --test-name alfa --png-file "+tdArgs.undeclaredOutputsDir+"/alfa.png --png-digest a01a01a01a01a01a01a01a01a01a01a0 --add-test-key build_system:bazel --add-test-key name:alfa --add-test-key source_type:gm", |
| "/path/to/goldctl imgtest add --work-dir "+goldctlWorkDir+" --test-name beta --png-file "+tdArgs.undeclaredOutputsDir+"/beta.png --png-digest b02b02b02b02b02b02b02b02b02b02b0 --add-test-key build_system:bazel --add-test-key name:beta --add-test-key source_type:gm", |
| "/path/to/goldctl imgtest finalize --work-dir "+goldctlWorkDir, |
| ) |
| |
| assert.Equal(t, "/path/to/workdir", commandCollector.Commands()[0].Dir) |
| exec_testutils.AssertCommandsMatch(t, [][]string{ |
| { |
| "/path/to/command", |
| "--device-specific-bazel-config", "Pixel5", |
| "--key", |
| "arch", "arm64", |
| "model", "Pixel5", |
| "os", "Android", |
| "--gpuName", "Adreno620", |
| }, |
| { |
| "/path/to/goldctl", |
| "auth", |
| "--work-dir", goldctlWorkDir, |
| "--luci", |
| }, |
| append([]string{"/path/to/goldctl"}, goldctlImgtestInitArgs...), |
| { |
| "/path/to/goldctl", |
| "imgtest", |
| "add", |
| "--work-dir", goldctlWorkDir, |
| "--test-name", "alfa", |
| "--png-file", tdArgs.undeclaredOutputsDir + "/alfa.png", |
| "--png-digest", "a01a01a01a01a01a01a01a01a01a01a0", |
| "--add-test-key", "build_system:bazel", |
| "--add-test-key", "name:alfa", |
| "--add-test-key", "source_type:gm", |
| }, |
| { |
| "/path/to/goldctl", |
| "imgtest", |
| "add", |
| "--work-dir", goldctlWorkDir, |
| "--test-name", "beta", |
| "--png-file", tdArgs.undeclaredOutputsDir + "/beta.png", |
| "--png-digest", "b02b02b02b02b02b02b02b02b02b02b0", |
| "--add-test-key", "build_system:bazel", |
| "--add-test-key", "name:beta", |
| "--add-test-key", "source_type:gm", |
| }, |
| { |
| "/path/to/goldctl", |
| "imgtest", |
| "finalize", |
| "--work-dir", goldctlWorkDir, |
| }, |
| }, commandCollector.Commands()) |
| }) |
| } |
| |
| goldctlWorkDir := t.TempDir() |
| test( |
| "post-submit task", |
| taskDriverArgs{ |
| UploadToGoldArgs: common.UploadToGoldArgs{ |
| TestOnlyAllowAnyBazelLabel: true, |
| BazelLabel: "//some/test:target", |
| DeviceSpecificBazelConfig: "Pixel5", |
| GoldctlPath: "/path/to/goldctl", |
| GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", |
| }, |
| commandPath: "/path/to/command", |
| commandWorkDir: "/path/to/workdir", |
| testKind: gmTest, |
| deviceSpecificBazelConfigName: "Pixel5", |
| undeclaredOutputsDir: t.TempDir(), |
| }, |
| goldctlWorkDir, |
| "/path/to/goldctl imgtest init --work-dir "+goldctlWorkDir+" --instance skia --url https://gold.skia.org --bucket skia-infra-gm --git_hash ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99 --key arch:arm64 --key model:Pixel5 --key os:Android", |
| []string{ |
| "imgtest", |
| "init", |
| "--work-dir", goldctlWorkDir, |
| "--instance", "skia", |
| "--url", "https://gold.skia.org", |
| "--bucket", "skia-infra-gm", |
| "--git_hash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", |
| "--key", "arch:arm64", |
| "--key", "model:Pixel5", |
| "--key", "os:Android", |
| }, |
| ) |
| |
| goldctlWorkDir = t.TempDir() |
| test( |
| "CL task", |
| taskDriverArgs{ |
| UploadToGoldArgs: common.UploadToGoldArgs{ |
| TestOnlyAllowAnyBazelLabel: true, |
| BazelLabel: "//some/test:target", |
| DeviceSpecificBazelConfig: "Pixel5", |
| GoldctlPath: "/path/to/goldctl", |
| GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", |
| ChangelistID: "changelist-id", |
| PatchsetOrder: "1", |
| TryjobID: "tryjob-id", |
| }, |
| commandPath: "/path/to/command", |
| commandWorkDir: "/path/to/workdir", |
| testKind: gmTest, |
| deviceSpecificBazelConfigName: "Pixel5", |
| undeclaredOutputsDir: t.TempDir(), |
| }, |
| goldctlWorkDir, |
| "/path/to/goldctl imgtest init --work-dir "+goldctlWorkDir+" --instance skia --url https://gold.skia.org --bucket skia-infra-gm --git_hash ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99 --crs gerrit --cis buildbucket --changelist changelist-id --patchset 1 --jobid tryjob-id --key arch:arm64 --key model:Pixel5 --key os:Android", |
| []string{ |
| "imgtest", |
| "init", |
| "--work-dir", goldctlWorkDir, |
| "--instance", "skia", |
| "--url", "https://gold.skia.org", |
| "--bucket", "skia-infra-gm", |
| "--git_hash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", |
| "--crs", "gerrit", |
| "--cis", "buildbucket", |
| "--changelist", "changelist-id", |
| "--patchset", "1", |
| "--jobid", "tryjob-id", |
| "--key", "arch:arm64", |
| "--key", "model:Pixel5", |
| "--key", "os:Android", |
| }, |
| ) |
| } |
| |
| func TestRun_BenchmarkTest_Success(t *testing.T) { |
| // Given that we have tests for common.UploadToPerf(), it suffices to test a couple of "happy" |
| // cases here. |
| |
| test := func(name string, tdArgs taskDriverArgs, benchmarkInvocation []string) { |
| t.Run(name, func(t *testing.T) { |
| // Create directory with fake undeclared test outputs. |
| resultsJSONFileContents := `{"foo": "this test requires that this file exists; its contents do not matter"}` |
| testutils.PopulateDir(t, tdArgs.undeclaredOutputsDir, map[string]string{ |
| "results.json": resultsJSONFileContents, |
| "some-image.png": "fake PNG", |
| "some-plaintext.txt": "fake TXT", |
| }) |
| |
| commandCollector := exec.CommandCollector{} |
| |
| gcsClient := mocks.NewGCSClient(t) |
| gcsClient.On("Bucket").Return("skia-perf") |
| gcsClient.On( |
| "SetFileContents", |
| infra_testutils.AnyContext, |
| "nano-json-v1/2022/01/31/01/ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99/BazelTest-Foo-Bar/results_1234567890.json", |
| gcs.FILE_WRITE_OPTS_TEXT, |
| []byte(resultsJSONFileContents)). |
| Return(nil).Once() |
| tdArgs.gcsClient = gcsClient |
| |
| res := td.RunTestSteps(t, false, func(ctx context.Context) error { |
| // Make sure we use UTC instead of the system timezone. The GCS path reflects the fact that |
| // we convert from UTC+1 to UTC. |
| fakeNow := time.Date(2022, time.January, 31, 2, 2, 3, 0, time.FixedZone("UTC+1", 60*60)) |
| ctx = now.TimeTravelingContext(fakeNow).WithContext(ctx) |
| ctx = td.WithExecRunFn(ctx, commandCollector.Run) |
| |
| err := run(ctx, tdArgs) |
| |
| assert.NoError(t, err) |
| return err |
| }) |
| |
| require.Empty(t, res.Errors) |
| require.Empty(t, res.Exceptions) |
| |
| testutils.AssertStepNames(t, res, |
| strings.Join(benchmarkInvocation, " "), |
| "Stat "+tdArgs.undeclaredOutputsDir+"/results.json", |
| "Read "+tdArgs.undeclaredOutputsDir+"/results.json", |
| "Upload gs://skia-perf/nano-json-v1/2022/01/31/01/ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99/BazelTest-Foo-Bar/results_1234567890.json", |
| ) |
| |
| assert.Equal(t, "/path/to/workdir", commandCollector.Commands()[0].Dir) |
| exec_testutils.AssertCommandsMatch(t, [][]string{benchmarkInvocation}, commandCollector.Commands()) |
| }) |
| } |
| |
| test( |
| "post-submit task", |
| taskDriverArgs{ |
| BenchmarkInfo: common.BenchmarkInfo{ |
| GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", |
| TaskName: "BazelTest-Foo-Bar", |
| TaskID: "1234567890", |
| }, |
| commandPath: "/path/to/command", |
| commandWorkDir: "/path/to/workdir", |
| testKind: benchmarkTest, |
| deviceSpecificBazelConfigName: "Pixel5", |
| undeclaredOutputsDir: t.TempDir(), |
| }, |
| []string{ |
| "/path/to/command", |
| "--gitHash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", |
| "--links", |
| "task", "https://task-scheduler.skia.org/task/1234567890", |
| "--device-specific-bazel-config", "Pixel5", |
| "--key", |
| "arch", "arm64", |
| "model", "Pixel5", |
| "os", "Android", |
| "--gpuName", "Adreno620", |
| }, |
| ) |
| |
| test( |
| "CL task", |
| taskDriverArgs{ |
| BenchmarkInfo: common.BenchmarkInfo{ |
| GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", |
| TaskName: "BazelTest-Foo-Bar", |
| TaskID: "1234567890", |
| ChangelistID: "12345", |
| PatchsetOrder: "3", |
| }, |
| commandPath: "/path/to/command", |
| commandWorkDir: "/path/to/workdir", |
| testKind: benchmarkTest, |
| deviceSpecificBazelConfigName: "Pixel5", |
| undeclaredOutputsDir: t.TempDir(), |
| }, |
| []string{ |
| "/path/to/command", |
| "--gitHash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", |
| "--issue", "12345", |
| "--patchset", "3", |
| "--links", |
| "task", "https://task-scheduler.skia.org/task/1234567890", |
| "changelist", "https://skia-review.googlesource.com/c/skia/+/12345/3", |
| "--device-specific-bazel-config", "Pixel5", |
| "--key", |
| "arch", "arm64", |
| "model", "Pixel5", |
| "os", "Android", |
| "--gpuName", "Adreno620", |
| }, |
| ) |
| } |
