infra/bots/task_drivers/common/perf_steps_test.go

// Copyright 2023 Google LLC
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package common

import (
	"context"
	"path/filepath"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"go.skia.org/infra/go/gcs"
	"go.skia.org/infra/go/gcs/mocks"
	"go.skia.org/infra/go/now"

	infra_testutils "go.skia.org/infra/go/testutils"
	"go.skia.org/infra/task_driver/go/lib/os_steps"
	"go.skia.org/infra/task_driver/go/td"
	"go.skia.org/skia/infra/bots/task_drivers/testutils"
)

func TestComputeBenchmarkTestRunnerCLIFlags_Success(t *testing.T) {
	test := func(name string, benchmarkInfo BenchmarkInfo, expectedFlags []string) {
		t.Run(name, func(t *testing.T) {
			actualFlags := ComputeBenchmarkTestRunnerCLIFlags(benchmarkInfo)
			assert.Equal(t, expectedFlags, actualFlags)
		})
	}

	test("post-submit task", BenchmarkInfo{
		GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		TaskName:  "BazelTest-Foo-Bar",
		TaskID:    "1234567890",
	}, []string{
		"--gitHash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		"--links",
		"task", "https://task-scheduler.skia.org/task/1234567890",
	})

	test("CL task", BenchmarkInfo{
		GitCommit:     "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		TaskName:      "BazelTest-Foo-Bar",
		TaskID:        "1234567890",
		ChangelistID:  "12345",
		PatchsetOrder: "3",
	}, []string{
		"--gitHash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		"--issue", "12345",
		"--patchset", "3",
		"--links",
		"task", "https://task-scheduler.skia.org/task/1234567890",
		"changelist", "https://skia-review.googlesource.com/c/skia/+/12345/3",
	})
}

func TestUploadToPerf_NoOutputsZIPOrDir_Error(t *testing.T) {
	test := func(name string, benchmarkInfo BenchmarkInfo) {
		t.Run(name, func(t *testing.T) {
			gcsClient := mocks.NewGCSClient(t)
			res := td.RunTestSteps(t, false, func(ctx context.Context) error {
				err := UploadToPerf(ctx, gcsClient, benchmarkInfo, "/no/such/outputs.zip")
				assert.Error(t, err)
				assert.Contains(t, err.Error(), "stat /no/such/outputs.zip: no such file or directory")
				return err
			})

			require.Empty(t, res.Errors)
			require.Empty(t, res.Exceptions)
			testutils.AssertStepNames(t, res) // No steps.

			gcsClient.AssertNotCalled(t, "Bucket")
			gcsClient.AssertNotCalled(t, "SetFileContents")
			gcsClient.AssertExpectations(t)
		})
	}

	test("post-submit task", BenchmarkInfo{
		GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		TaskName:  "BazelTest-Foo-Bar",
		TaskID:    "1234567890",
	})

	test("CL task", BenchmarkInfo{
		GitCommit:     "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		TaskName:      "BazelTest-Foo-Bar",
		TaskID:        "1234567890",
		ChangelistID:  "12345",
		PatchsetOrder: "3",
	})
}

func TestUploadToPerf_OutputsZip_NoResultsJSONFile_Error(t *testing.T) {
	test := func(name string, benchmarkInfo BenchmarkInfo) {
		t.Run(name, func(t *testing.T) {
			undeclaredTestOutputs := map[string]string{
				// Does not contain a results.json file. File contents do not matter for this test.
				"some-image.png":     "fake PNG",
				"some-plaintext.txt": "fake TXT",
			}

			// Write undeclared test outputs to disk.
			outputsZIP := filepath.Join(t.TempDir(), "outputs.zip")
			testutils.MakeZIP(t, outputsZIP, undeclaredTestOutputs)

			// Will be returned by the mocked os_steps.TempDir() when the task driver tries to create a
			// directory in which to extract the undeclared outputs ZIP archive.
			outputsZIPExtractionDir := t.TempDir()

			gcsClient := mocks.NewGCSClient(t)
			res := td.RunTestSteps(t, false, func(ctx context.Context) error {
				// We don't need to assert the exact number of times that os_steps.TempDir() is called
				// because said function produces a "Creating TempDir" task driver step, and we check the
				// exact set of steps produced.
				ctx = context.WithValue(ctx, os_steps.TempDirContextKey, testutils.MakeTempDirMockFn(t, outputsZIPExtractionDir))

				err := UploadToPerf(ctx, gcsClient, benchmarkInfo, outputsZIP)
				assert.Error(t, err)
				assert.Contains(t, err.Error(), "stat "+outputsZIPExtractionDir+"/results.json: no such file or directory")
				return err
			})

			require.Empty(t, res.Errors)
			require.Empty(t, res.Exceptions)

			testutils.AssertStepNames(t, res,
				"Creating TempDir",
				"Extract undeclared outputs archive "+outputsZIP+" into "+outputsZIPExtractionDir,
				"Extracting file: some-image.png",
				"Not extracting non-PNG / non-JSON file: some-plaintext.txt",
				"Stat "+outputsZIPExtractionDir+"/results.json",
			)

			gcsClient.AssertNotCalled(t, "Bucket")
			gcsClient.AssertNotCalled(t, "SetFileContents")
			gcsClient.AssertExpectations(t)
		})
	}

	test("post-submit task", BenchmarkInfo{
		GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		TaskName:  "BazelTest-Foo-Bar",
		TaskID:    "1234567890",
	})

	test("CL task", BenchmarkInfo{
		GitCommit:     "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		TaskName:      "BazelTest-Foo-Bar",
		TaskID:        "1234567890",
		ChangelistID:  "12345",
		PatchsetOrder: "3",
	})
}

func TestUploadToPerf_OutputsDirectory_NoResultsJSONFile_Error(t *testing.T) {
	test := func(name string, benchmarkInfo BenchmarkInfo) {
		t.Run(name, func(t *testing.T) {
			undeclaredTestOutputs := map[string]string{
				// Does not contain a results.json file. File contents do not matter for this test.
				"some-image.png":     "fake PNG",
				"some-plaintext.txt": "fake TXT",
			}

			// Write undeclared test outputs to disk.
			outputsDir := t.TempDir()
			testutils.PopulateDir(t, outputsDir, undeclaredTestOutputs)

			gcsClient := mocks.NewGCSClient(t)
			res := td.RunTestSteps(t, false, func(ctx context.Context) error {
				err := UploadToPerf(ctx, gcsClient, benchmarkInfo, outputsDir)
				assert.Error(t, err)
				assert.Contains(t, err.Error(), "stat "+outputsDir+"/results.json: no such file or directory")
				return err
			})

			require.Empty(t, res.Errors)
			require.Empty(t, res.Exceptions)

			testutils.AssertStepNames(t, res, "Stat "+outputsDir+"/results.json")

			gcsClient.AssertNotCalled(t, "Bucket")
			gcsClient.AssertNotCalled(t, "SetFileContents")
			gcsClient.AssertExpectations(t)
		})
	}

	test("post-submit task", BenchmarkInfo{
		GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		TaskName:  "BazelTest-Foo-Bar",
		TaskID:    "1234567890",
	})

	test("CL task", BenchmarkInfo{
		GitCommit:     "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		TaskName:      "BazelTest-Foo-Bar",
		TaskID:        "1234567890",
		ChangelistID:  "12345",
		PatchsetOrder: "3",
	})
}

func TestUploadToPerf_OutputsZip_Success(t *testing.T) {
	test := func(name string, benchmarkInfo BenchmarkInfo) {
		t.Run(name, func(t *testing.T) {
			resultsJSONFileContents := `{"foo": "this test requires that this file exists; its contents do not matter"}`
			undeclaredTestOutputs := map[string]string{
				"results.json":       resultsJSONFileContents,
				"some-image.png":     "fake PNG",
				"some-plaintext.txt": "fake TXT",
			}

			// Write undeclared test outputs to disk.
			outputsZIP := filepath.Join(t.TempDir(), "outputs.zip")
			testutils.MakeZIP(t, outputsZIP, undeclaredTestOutputs)

			// Will be returned by the mocked os_steps.TempDir() when the task driver tries to create a
			// directory in which to extract the undeclared outputs ZIP archive.
			outputsZIPExtractionDir := t.TempDir()

			gcsClient := mocks.NewGCSClient(t)
			gcsClient.On("Bucket").Return("skia-perf")
			gcsClient.On(
				"SetFileContents",
				infra_testutils.AnyContext,
				"nano-json-v1/2022/01/31/01/ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99/BazelTest-Foo-Bar/results_1234567890.json",
				gcs.FILE_WRITE_OPTS_TEXT,
				[]byte(resultsJSONFileContents)).
				Return(nil).Once()

			res := td.RunTestSteps(t, false, func(ctx context.Context) error {
				// Make sure we use UTC instead of the system timezone. The GCS path reflects the fact that
				// we convert from UTC+1 to UTC.
				fakeNow := time.Date(2022, time.January, 31, 2, 2, 3, 0, time.FixedZone("UTC+1", 60*60))
				ctx = now.TimeTravelingContext(fakeNow).WithContext(ctx)

				// We don't need to assert the exact number of times that os_steps.TempDir() is called
				// because said function produces a "Creating TempDir" task driver step, and we check the
				// exact set of steps produced.
				ctx = context.WithValue(ctx, os_steps.TempDirContextKey, testutils.MakeTempDirMockFn(t, outputsZIPExtractionDir))

				err := UploadToPerf(ctx, gcsClient, benchmarkInfo, outputsZIP)
				assert.NoError(t, err)
				return err
			})

			require.Empty(t, res.Errors)
			require.Empty(t, res.Exceptions)

			testutils.AssertStepNames(t, res,
				"Creating TempDir",
				"Extract undeclared outputs archive "+outputsZIP+" into "+outputsZIPExtractionDir,
				"Extracting file: results.json",
				"Extracting file: some-image.png",
				"Not extracting non-PNG / non-JSON file: some-plaintext.txt",
				"Stat "+outputsZIPExtractionDir+"/results.json",
				"Read "+outputsZIPExtractionDir+"/results.json",
				"Upload gs://skia-perf/nano-json-v1/2022/01/31/01/ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99/BazelTest-Foo-Bar/results_1234567890.json",
			)

			gcsClient.AssertExpectations(t)
		})
	}

	test("post-submit task", BenchmarkInfo{
		GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		TaskName:  "BazelTest-Foo-Bar",
		TaskID:    "1234567890",
	})

	test("CL task", BenchmarkInfo{
		GitCommit:     "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		TaskName:      "BazelTest-Foo-Bar",
		TaskID:        "1234567890",
		ChangelistID:  "12345",
		PatchsetOrder: "3",
	})
}

func TestUploadToPerf_OutputsDirectory_Success(t *testing.T) {
	test := func(name string, benchmarkInfo BenchmarkInfo) {
		t.Run(name, func(t *testing.T) {
			resultsJSONFileContents := `{"foo": "this test requires that this file exists; its contents do not matter"}`
			undeclaredTestOutputs := map[string]string{
				"results.json":       resultsJSONFileContents,
				"some-image.png":     "fake PNG",
				"some-plaintext.txt": "fake TXT",
			}

			// Write undeclared test outputs to disk.
			outputsDir := t.TempDir()
			testutils.PopulateDir(t, outputsDir, undeclaredTestOutputs)

			gcsClient := mocks.NewGCSClient(t)
			gcsClient.On("Bucket").Return("skia-perf")
			gcsClient.On(
				"SetFileContents",
				infra_testutils.AnyContext,
				"nano-json-v1/2022/01/31/01/ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99/BazelTest-Foo-Bar/results_1234567890.json",
				gcs.FILE_WRITE_OPTS_TEXT,
				[]byte(resultsJSONFileContents)).
				Return(nil).Once()

			res := td.RunTestSteps(t, false, func(ctx context.Context) error {
				// Make sure we use UTC instead of the system timezone. The GCS path reflects the fact that
				// we convert from UTC+1 to UTC.
				fakeNow := time.Date(2022, time.January, 31, 2, 2, 3, 0, time.FixedZone("UTC+1", 60*60))
				ctx = now.TimeTravelingContext(fakeNow).WithContext(ctx)

				err := UploadToPerf(ctx, gcsClient, benchmarkInfo, outputsDir)
				assert.NoError(t, err)
				return err
			})

			require.Empty(t, res.Errors)
			require.Empty(t, res.Exceptions)

			testutils.AssertStepNames(t, res,
				"Stat "+outputsDir+"/results.json",
				"Read "+outputsDir+"/results.json",
				"Upload gs://skia-perf/nano-json-v1/2022/01/31/01/ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99/BazelTest-Foo-Bar/results_1234567890.json",
			)

			gcsClient.AssertExpectations(t)
		})
	}

	test("post-submit task, directory", BenchmarkInfo{
		GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		TaskName:  "BazelTest-Foo-Bar",
		TaskID:    "1234567890",
	})

	test("CL task, directory", BenchmarkInfo{
		GitCommit:     "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99",
		TaskName:      "BazelTest-Foo-Bar",
		TaskID:        "1234567890",
		ChangelistID:  "12345",
		PatchsetOrder: "3",
	})
}