gold-client/cmd/goldctl/cmd_imgtest_test.go

package main

import (
	"bytes"
	"context"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/require"

	"go.skia.org/infra/go/now"
	"go.skia.org/infra/go/paramtools"
	"go.skia.org/infra/go/testutils"
	"go.skia.org/infra/gold-client/go/gcsuploader"
	"go.skia.org/infra/gold-client/go/goldclient"
	"go.skia.org/infra/gold-client/go/httpclient"
	"go.skia.org/infra/gold-client/go/imagedownloader"
	"go.skia.org/infra/gold-client/go/imgmatching"
	"go.skia.org/infra/gold-client/go/mocks"
	"go.skia.org/infra/golden/go/expectations"
	"go.skia.org/infra/golden/go/jsonio"
	"go.skia.org/infra/golden/go/sql"
	"go.skia.org/infra/golden/go/tiling"
	"go.skia.org/infra/golden/go/types"
	"go.skia.org/infra/golden/go/web/frontend"
)

const (
	// These images are copied from the datakitchensink set
	blankDigest = "00000000000000000000000000000000" // all pixels blank
	a01Digest   = "a01a01a01a01a01a01a01a01a01a01a0" // has a square drawn
	a05Digest   = "a05a05a05a05a05a05a05a05a05a05a0" // small difference from a01
	a09Digest   = "a09a09a09a09a09a09a09a09a09a09a0" // large difference from a01
)

var (
	timeOne = time.Date(2021, time.January, 23, 22, 21, 20, 19, time.UTC)
	timeTwo = time.Date(2021, time.January, 23, 22, 22, 0, 0, time.UTC)
)

func TestImgTest_Init_LoadKeysFromDisk_WritesProperResultState(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)

	keysFile := filepath.Join(workDir, "keys.json")
	require.NoError(t, os.WriteFile(keysFile, []byte(`{"os": "Android"}`), 0644))

	mh := mockRPCResponses("https://my-instance-gold.skia.org").Positive("pixel-tests", blankDigest).
		Negative("other-test", blankDigest).
		Known("11111111111111111111111111111111").Build()

	// Call imgtest init with the following flags. We expect it to load the baseline expectations
	// (with two images triaged) and the known hashes (one entry).
	ctx, output, exit := testContext(nil, mh, nil, nil)
	env := imgTest{
		gitHash:      "1234567890123456789012345678901234567890",
		corpus:       "my_corpus",
		instanceID:   "my-instance",
		keysFile:     keysFile,
		passFailStep: true,
		workDir:      workDir,
	}
	runUntilExit(t, func() {
		env.Init(ctx)
	})
	exit.AssertWasCalledWithCode(t, 0, output.String())

	b, err := os.ReadFile(filepath.Join(workDir, "result-state.json"))
	require.NoError(t, err)
	resultState := string(b)
	assert.Contains(t, resultState, `"key":{"os":"Android","source_type":"my_corpus"}`)
	assert.Contains(t, resultState, `"KnownHashes":{"00000000000000000000000000000000":true,"11111111111111111111111111111111":true}`)
	assert.Contains(t, resultState, `"Expectations":{"other-test":{"00000000000000000000000000000000":"negative"},"pixel-tests":{"00000000000000000000000000000000":"positive"}}`)
	assert.Contains(t, resultState, `"gitHash":"1234567890123456789012345678901234567890"`)
}

func TestImgTest_Init_CommitIDAndMetadataSet_WritesProperResultState(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)

	keysFile := filepath.Join(workDir, "keys.json")
	require.NoError(t, os.WriteFile(keysFile, []byte(`{"os": "Android"}`), 0644))

	mh := mockRPCResponses("https://my-instance-gold.skia.org").Positive("pixel-tests", blankDigest).
		Negative("other-test", blankDigest).
		Known("11111111111111111111111111111111").Build()

	// Call imgtest init with the following flags. We expect it to load the baseline expectations
	// (with two images triaged) and the known hashes (one entry).
	ctx, output, exit := testContext(nil, mh, nil, nil)
	env := imgTest{
		commitID:       "92.103234.1.123456",
		commitMetadata: "http://example.com/92.103234.1.123456.xml",
		corpus:         "my_corpus",
		instanceID:     "my-instance",
		keysFile:       keysFile,
		passFailStep:   true,
		workDir:        workDir,
	}
	runUntilExit(t, func() {
		env.Init(ctx)
	})
	exit.AssertWasCalledWithCode(t, 0, output.String())

	b, err := os.ReadFile(filepath.Join(workDir, "result-state.json"))
	require.NoError(t, err)
	resultState := string(b)
	assert.Contains(t, resultState, `"key":{"os":"Android","source_type":"my_corpus"}`)
	assert.Contains(t, resultState, `"KnownHashes":{"00000000000000000000000000000000":true,"11111111111111111111111111111111":true}`)
	assert.Contains(t, resultState, `"Expectations":{"other-test":{"00000000000000000000000000000000":"negative"},"pixel-tests":{"00000000000000000000000000000000":"positive"}}`)
	assert.Contains(t, resultState, `"commit_id":"92.103234.1.123456","commit_metadata":"http://example.com/92.103234.1.123456.xml"`)
}

func TestImgTest_Init_ChangeListWithoutCommitHash_WritesProperResultState(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)

	keysFile := filepath.Join(workDir, "keys.json")
	require.NoError(t, os.WriteFile(keysFile, []byte(`{"os": "Android"}`), 0644))

	mh := mockRPCResponses("https://my-instance-gold.skia.org").Positive("pixel-tests", blankDigest).
		Negative("other-test", blankDigest).
		Known("11111111111111111111111111111111").BuildForCL("my_CRS", "my_CL")

	// Call imgtest init with the following flags. We expect it to load the baseline expectations
	// and the known hashes (both empty).
	ctx, output, exit := testContext(nil, mh, nil, nil)
	env := imgTest{
		corpus:                      "my_corpus",
		instanceID:                  "my-instance",
		keysFile:                    keysFile,
		passFailStep:                true,
		workDir:                     workDir,
		codeReviewSystem:            "my_CRS",
		changelistID:                "my_CL",
		patchsetID:                  "some_patchset",
		continuousIntegrationSystem: "my_CIS",
		tryJobID:                    "some_tryjob",
	}
	runUntilExit(t, func() {
		env.Init(ctx)
	})
	exit.AssertWasCalledWithCode(t, 0, output.String())

	b, err := os.ReadFile(filepath.Join(workDir, "result-state.json"))
	require.NoError(t, err)
	resultState := string(b)
	assert.Contains(t, resultState, `"key":{"os":"Android","source_type":"my_corpus"}`)
	assert.Contains(t, resultState, `"KnownHashes":{"00000000000000000000000000000000":true,"11111111111111111111111111111111":true}`)
	assert.Contains(t, resultState, `"Expectations":{"other-test":{"00000000000000000000000000000000":"negative"},"pixel-tests":{"00000000000000000000000000000000":"positive"}}`)
	assert.Contains(t, resultState, `"change_list_id":"my_CL","patch_set_order":0,"patch_set_id":"some_patchset","crs":"my_CRS","try_job_id":"some_tryjob","cis":"my_CIS"`)
}

func TestImgTest_Init_NoChangeListNorCommitHash_NonzeroExitCode(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)

	keysFile := filepath.Join(workDir, "keys.json")
	require.NoError(t, os.WriteFile(keysFile, []byte(`{"os": "Android"}`), 0644))

	// Call imgtest init with the following flags. We expect it to fail because we need to provide
	// a commit or CL info
	ctx, output, exit := testContext(nil, nil, nil, nil)
	env := imgTest{
		corpus:       "my_corpus",
		instanceID:   "my-instance",
		keysFile:     keysFile,
		passFailStep: true,
		workDir:      workDir,
	}
	runUntilExit(t, func() {
		env.Init(ctx)
	})
	outStr := output.String()
	exit.AssertWasCalledWithCode(t, 1, outStr)
	assert.Contains(t, outStr, `invalid configuration: field "gitHash", "commit_id", or "change_list_id" must be set`)
}

func TestImgTest_Init_EmptyExpectationsReturned_EmitsWarning(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)

	keysFile := filepath.Join(workDir, "keys.json")
	require.NoError(t, os.WriteFile(keysFile, []byte(`{"os": "Android"}`), 0644))

	mh := mockRPCResponses("https://my-instance-gold.skia.org").
		Known("11111111111111111111111111111111").Build()

	ctx, output, exit := testContext(nil, mh, nil, nil)
	env := imgTest{
		gitHash:      "1234567890123456789012345678901234567890",
		corpus:       "my_corpus",
		instanceID:   "my-instance",
		keysFile:     keysFile,
		passFailStep: true,
		workDir:      workDir,
	}
	runUntilExit(t, func() {
		env.Init(ctx)
	})
	logs := output.String()
	exit.AssertWasCalledWithCode(t, 0, logs)
	assert.Contains(t, logs, "warning: got empty expectations when querying https://my-instance-gold.skia.org/json/v2/expectations\n")
}

func TestImgTest_InitCheck_EmptyExpectationsReturned_ReturnsNonzeroExitCode(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)

	keysFile := filepath.Join(workDir, "keys.json")
	require.NoError(t, os.WriteFile(keysFile, []byte(`{"os": "Android"}`), 0644))

	mh := mockRPCResponses("https://my-instance-gold.skia.org").
		Known("11111111111111111111111111111111").Build()

	ctx, output, exit := testContext(nil, mh, nil, nil)
	env := imgTest{
		gitHash:      "1234567890123456789012345678901234567890",
		corpus:       "my_corpus",
		instanceID:   "my-instance",
		keysFile:     keysFile,
		passFailStep: true,
		workDir:      workDir,
	}
	runUntilExit(t, func() {
		env.Init(ctx)
	})
	exit.AssertWasCalledWithCode(t, 0, output.String())

	output.buf.Reset()
	runUntilExit(t, func() {
		env.Check(ctx)
	})
	logs := output.String()
	exit.AssertWasCalledWithCode(t, 1, logs)
	assert.Contains(t, logs, "Expectations are empty, despite re-loading them from Gold")
	assert.Contains(t, logs, `raw expectation response "{}"`)
}

func TestImgTest_InitAdd_StreamingPassFail_DoesNotMatchExpectations_NonzeroExitCode(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)
	td := testutils.TestDataDir(t)

	mh := mockRPCResponses("https://my-instance-gold.skia.org").Build()

	// Call imgtest init with the following flags. We expect it to load the baseline expectations
	// and the known hashes (both empty).
	ctx, output, exit := testContext(nil, mh, nil, nil)
	env := imgTest{
		gitHash:         "1234567890123456789012345678901234567890",
		corpus:          "my_corpus",
		instanceID:      "my-instance",
		passFailStep:    true,
		failureFile:     filepath.Join(workDir, "failures.txt"),
		workDir:         workDir,
		testKeysStrings: []string{"os:Android"},
	}
	runUntilExit(t, func() {
		env.Init(ctx)
	})
	exit.AssertWasCalledWithCode(t, 0, output.String())

	mg := &mocks.GCSUploader{}
	resultsMatcher := mock.MatchedBy(func(results jsonio.GoldResults) bool {
		assert.Equal(t, jsonio.GoldResults{
			GitHash: "1234567890123456789012345678901234567890",
			Key: map[string]string{
				"os":          "Android",
				"source_type": "my_corpus",
			},
			Results: []jsonio.Result{{
				Key:     map[string]string{"name": "pixel-tests", "device": "angler"},
				Options: map[string]string{"some_option": "is optional", "ext": "png"},
				Digest:  blankDigest,
			}},
		}, results)
		return true
	})
	mg.On("UploadJSON", testutils.AnyContext, resultsMatcher, mock.Anything,
		`skia-gold-my-instance/dm-json-v1/2021/01/23/22/1234567890123456789012345678901234567890/waterfall/dm-1611440480000000019.json`).Return(nil)
	bytesMatcher := mock.MatchedBy(func(b []byte) bool {
		assert.Len(t, b, 78) // spot check length
		return true
	})
	mg.On("UploadBytes", testutils.AnyContext, bytesMatcher, mock.Anything,
		`gs://skia-gold-my-instance/dm-images-v1/00000000000000000000000000000000.png`).Return(nil)

	// Now call imgtest add with the following flags. This is simulating a test uploading a single
	// result for a test called pixel-tests.
	ctx, output, exit = testContext(mg, mh, nil, &timeOne)
	env = imgTest{
		workDir:                 workDir,
		testName:                "pixel-tests",
		pngFile:                 filepath.Join(td, "00000000000000000000000000000000.png"),
		pngDigest:               blankDigest,
		testKeysStrings:         []string{"device:angler"},
		testOptionalKeysStrings: []string{"some_option:is optional"},
	}
	runUntilExit(t, func() {
		env.Add(ctx)
	})
	logs := output.String()
	exit.AssertWasCalledWithCode(t, 1, logs)
	mg.AssertExpectations(t)
	assert.Contains(t, logs, `Untriaged or negative image: https://my-instance-gold.skia.org/detail?grouping=name%3Dpixel-tests%26source_type%3Dmy_corpus&digest=00000000000000000000000000000000`)
	assert.Contains(t, logs, `Test: pixel-tests FAIL`)

	fb, err := os.ReadFile(filepath.Join(workDir, "failures.txt"))
	require.NoError(t, err)
	assert.Contains(t, string(fb), "https://my-instance-gold.skia.org/detail?grouping=name%3Dpixel-tests%26source_type%3Dmy_corpus&digest=00000000000000000000000000000000")
}

func TestImgTest_InitAdd_StreamingPassFail_NoCorpusSpecified_NonzeroExitCodeAndNothingUploadedToGCS(t *testing.T) {
	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)
	td := testutils.TestDataDir(t)

	mh := mockRPCResponses("https://my-instance-gold.skia.org").Build()

	// Call imgtest init with the following flags. We expect it to load the baseline expectations
	// and the known hashes (both empty).
	ctx, output, exit := testContext(nil, mh, nil, nil)
	env := imgTest{
		// No corpus specified.
		gitHash:         "1234567890123456789012345678901234567890",
		instanceID:      "my-instance",
		passFailStep:    true,
		failureFile:     filepath.Join(workDir, "failures.txt"),
		workDir:         workDir,
		testKeysStrings: []string{"os:Android"},
	}
	runUntilExit(t, func() {
		env.Init(ctx)
	})
	exit.AssertWasCalledWithCode(t, 0, output.String())

	// Note that there is no mock GCSUploader: this test expects that nothing gets uploaded to GCS.
	ctx, output, exit = testContext(nil, mh, nil, &timeOne)

	// Now call imgtest add with the following flags. This is simulating a test uploading a single
	// result for a test called pixel-tests.
	env = imgTest{
		workDir:                 workDir,
		testName:                "pixel-tests",
		pngFile:                 filepath.Join(td, "00000000000000000000000000000000.png"),
		pngDigest:               blankDigest,
		testKeysStrings:         []string{"device:angler"},
		testOptionalKeysStrings: []string{"some_option:is optional"},
	}
	runUntilExit(t, func() {
		env.Add(ctx)
	})
	logs := output.String()
	exit.AssertWasCalledWithCode(t, 1, logs)

	// We did not specify a corpus, so goldctl defaulted to the instance's name as the corpus, i.e.
	// "my-instance". However, the /json/v1/groupings RPC does not return an entry for said corpus.
	assert.Contains(t, logs, `grouping params for corpus "my-instance" are unknown`)

	// No failures file is generated.
	_, err := os.Stat(filepath.Join(workDir, "failures.txt"))
	require.Error(t, err)
	assert.True(t, errors.Is(err, os.ErrNotExist))
}

func TestImgTest_InitAdd_OverwriteBucketAndURL_ProperLinks(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)
	td := testutils.TestDataDir(t)

	mh := mockRPCResponses("https://my-custom-gold-url.example.com").Build()

	// Call imgtest init with the following flags. We expect it to load the baseline expectations
	// and the known hashes (both empty).
	ctx, output, exit := testContext(nil, mh, nil, nil)
	env := imgTest{
		bucketOverride:  "my-custom-bucket",
		gitHash:         "1234567890123456789012345678901234567890",
		corpus:          "my_corpus",
		instanceID:      "my-instance",
		passFailStep:    true,
		failureFile:     filepath.Join(workDir, "failures.txt"),
		workDir:         workDir,
		testKeysStrings: []string{"os:Android"},
		urlOverride:     "https://my-custom-gold-url.example.com",
	}
	runUntilExit(t, func() {
		env.Init(ctx)
	})
	exit.AssertWasCalledWithCode(t, 0, output.String())

	mg := &mocks.GCSUploader{}
	resultsMatcher := mock.MatchedBy(func(results jsonio.GoldResults) bool {
		assert.Equal(t, jsonio.GoldResults{
			GitHash: "1234567890123456789012345678901234567890",
			Key: map[string]string{
				"os":          "Android",
				"source_type": "my_corpus",
			},
			Results: []jsonio.Result{{
				Key:     map[string]string{"name": "pixel-tests", "device": "angler"},
				Options: map[string]string{"some_option": "is optional", "ext": "png"},
				Digest:  blankDigest,
			}},
		}, results)
		return true
	})
	mg.On("UploadJSON", testutils.AnyContext, resultsMatcher, mock.Anything,
		`my-custom-bucket/dm-json-v1/2021/01/23/22/1234567890123456789012345678901234567890/waterfall/dm-1611440480000000019.json`).Return(nil)
	bytesMatcher := mock.MatchedBy(func(b []byte) bool {
		assert.Len(t, b, 78) // spot check length
		return true
	})
	mg.On("UploadBytes", testutils.AnyContext, bytesMatcher, mock.Anything,
		`gs://my-custom-bucket/dm-images-v1/00000000000000000000000000000000.png`).Return(nil)

	// Now call imgtest add with the following flags. This is simulating a test uploading a single
	// result for a test called pixel-tests.
	ctx, output, exit = testContext(mg, mh, nil, &timeOne)
	env = imgTest{
		workDir:                 workDir,
		testName:                "pixel-tests",
		pngFile:                 filepath.Join(td, "00000000000000000000000000000000.png"),
		pngDigest:               blankDigest,
		testKeysStrings:         []string{"device:angler"},
		testOptionalKeysStrings: []string{"some_option:is optional"},
	}
	runUntilExit(t, func() {
		env.Add(ctx)
	})
	logs := output.String()
	exit.AssertWasCalledWithCode(t, 1, logs)
	mg.AssertExpectations(t)
	assert.Contains(t, logs, `Untriaged or negative image: https://my-custom-gold-url.example.com/detail?grouping=name%3Dpixel-tests%26source_type%3Dmy_corpus&digest=00000000000000000000000000000000`)
	assert.Contains(t, logs, `Test: pixel-tests FAIL`)

	fb, err := os.ReadFile(filepath.Join(workDir, "failures.txt"))
	require.NoError(t, err)
	assert.Contains(t, string(fb), "https://my-custom-gold-url.example.com/detail?grouping=name%3Dpixel-tests%26source_type%3Dmy_corpus&digest=00000000000000000000000000000000")
}

func TestImgTest_InitAdd_StreamingPassFail_MatchesExpectations_ZeroExitCode(t *testing.T) {
	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)
	td := testutils.TestDataDir(t)

	mh := mockRPCResponses("https://my-instance-gold.skia.org").Positive("pixel-tests", blankDigest).Build()

	// Call imgtest init with the following flags. We expect it to load the baseline expectations
	// and the known hashes.
	ctx, output, exit := testContext(nil, mh, nil, nil)
	env := imgTest{
		gitHash:         "1234567890123456789012345678901234567890",
		corpus:          "my_corpus",
		instanceID:      "my-instance",
		passFailStep:    true,
		failureFile:     filepath.Join(workDir, "failures.txt"),
		workDir:         workDir,
		testKeysStrings: []string{"os:Android"},
	}
	runUntilExit(t, func() {
		env.Init(ctx)
	})
	exit.AssertWasCalledWithCode(t, 0, output.String())

	mg := &mocks.GCSUploader{}
	resultsMatcher := mock.MatchedBy(func(results jsonio.GoldResults) bool {
		assert.Equal(t, jsonio.GoldResults{
			GitHash: "1234567890123456789012345678901234567890",
			Key: map[string]string{
				"os":          "Android",
				"source_type": "my_corpus",
			},
			Results: []jsonio.Result{{
				Key:     map[string]string{"name": "pixel-tests", "device": "angler"},
				Options: map[string]string{"some_option": "is optional", "ext": "png"},
				Digest:  blankDigest,
			}},
		}, results)
		return true
	})
	mg.On("UploadJSON", testutils.AnyContext, resultsMatcher, mock.Anything,
		`skia-gold-my-instance/dm-json-v1/2021/01/23/22/1234567890123456789012345678901234567890/waterfall/dm-1611440480000000019.json`).Return(nil)

	// Now call imgtest add with the following flags. This is simulating a test uploading a single
	// result for a test called pixel-tests. The digest has already been triaged positive.
	ctx, output, exit = testContext(mg, mh, nil, &timeOne)
	env = imgTest{
		workDir:                 workDir,
		testName:                "pixel-tests",
		pngFile:                 filepath.Join(td, "00000000000000000000000000000000.png"),
		pngDigest:               blankDigest,
		testKeysStrings:         []string{"device:angler"},
		testOptionalKeysStrings: []string{"some_option:is optional"},
	}
	runUntilExit(t, func() {
		env.Add(ctx)
	})
	logs := output.String()
	exit.AssertWasCalledWithCode(t, 0, logs)
	mg.AssertExpectations(t)
}

func TestImgTest_InitAdd_StreamingPassFail_SuccessiveCalls_ProperJSONUploaded(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)
	td := testutils.TestDataDir(t)

	mh := mockRPCResponses("https://my-instance-gold.skia.org").Positive("pixel-tests", blankDigest).Build()

	// Call imgtest init with the following flags. We expect it to load the baseline expectations
	// and the known hashes.
	ctx, output, exit := testContext(nil, mh, nil, nil)
	env := imgTest{
		gitHash:         "1234567890123456789012345678901234567890",
		corpus:          "my_corpus",
		instanceID:      "my-instance",
		passFailStep:    true,
		workDir:         workDir,
		testKeysStrings: []string{"os:Android"},
	}
	runUntilExit(t, func() {
		env.Init(ctx)
	})
	exit.AssertWasCalledWithCode(t, 0, output.String())

	mg := &mocks.GCSUploader{}
	resultsMatcher := mock.MatchedBy(func(results jsonio.GoldResults) bool {
		assert.Equal(t, jsonio.GoldResults{
			GitHash: "1234567890123456789012345678901234567890",
			Key: map[string]string{
				"os":          "Android",
				"source_type": "my_corpus",
			},
			Results: []jsonio.Result{{
				Key:     map[string]string{"name": "pixel-tests", "device": "angler"},
				Options: map[string]string{"some_option": "is optional", "ext": "png"},
				Digest:  blankDigest,
			}},
		}, results)
		return true
	})
	mg.On("UploadJSON", testutils.AnyContext, resultsMatcher, mock.Anything,
		`skia-gold-my-instance/dm-json-v1/2021/01/23/22/1234567890123456789012345678901234567890/waterfall/dm-1611440480000000019.json`).Return(nil)

	// Now call imgtest add with the following flags. This is simulating a test uploading a single
	// result for a test called pixel-tests. The digest has already been triaged positive.
	ctx, output, exit = testContext(mg, mh, nil, &timeOne)
	env = imgTest{
		workDir:                 workDir,
		testName:                "pixel-tests",
		pngFile:                 filepath.Join(td, "00000000000000000000000000000000.png"),
		pngDigest:               blankDigest,
		testKeysStrings:         []string{"device:angler"},
		testOptionalKeysStrings: []string{"some_option:is optional"},
	}
	runUntilExit(t, func() {
		env.Add(ctx)
	})
	logs := output.String()
	exit.AssertWasCalledWithCode(t, 0, logs)
	mg.AssertExpectations(t)

	mg = &mocks.GCSUploader{}
	resultsMatcher = mock.MatchedBy(func(results jsonio.GoldResults) bool {
		assert.Equal(t, jsonio.GoldResults{
			GitHash: "1234567890123456789012345678901234567890",
			Key: map[string]string{
				"os":          "Android",
				"source_type": "my_corpus",
			},
			Results: []jsonio.Result{{
				Key:     map[string]string{"name": "pixel-tests", "device": "bullhead"},
				Options: map[string]string{"some_option": "is VERY DIFFERENT", "ext": "png"},
				Digest:  blankDigest,
			}},
		}, results)
		return true
	})
	mg.On("UploadJSON", testutils.AnyContext, resultsMatcher, mock.Anything,
		`skia-gold-my-instance/dm-json-v1/2021/01/23/22/1234567890123456789012345678901234567890/waterfall/dm-1611440520000000000.json`).Return(nil)

	// Call imgtest add for a second device running the same test as above.
	ctx, output, exit = testContext(mg, mh, nil, &timeTwo)
	env = imgTest{
		workDir:                 workDir,
		testName:                "pixel-tests",
		pngFile:                 filepath.Join(td, "00000000000000000000000000000000.png"),
		pngDigest:               blankDigest,
		testKeysStrings:         []string{"device:bullhead"},
		testOptionalKeysStrings: []string{"some_option:is VERY DIFFERENT"},
	}
	runUntilExit(t, func() {
		env.Add(ctx)
	})
	logs = output.String()
	exit.AssertWasCalledWithCode(t, 0, logs)
	mg.AssertExpectations(t)
}

// This tests calling imgtest add without calling imgtest init first.
func TestImgTest_Add_StreamingPassFail_MatchesExpectations_ZeroExitCode(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)
	td := testutils.TestDataDir(t)

	keysFile := filepath.Join(workDir, "keys.json")
	require.NoError(t, os.WriteFile(keysFile, []byte(`{"os": "Android"}`), 0644))

	mh := mockRPCResponses("https://my-instance-gold.skia.org").Positive("pixel-tests", blankDigest).Build()

	mg := &mocks.GCSUploader{}
	resultsMatcher := mock.MatchedBy(func(results jsonio.GoldResults) bool {
		assert.Equal(t, jsonio.GoldResults{
			GitHash: "1234567890123456789012345678901234567890",
			Key: map[string]string{
				"os": "Android",
			},
			Results: []jsonio.Result{{
				Key:     map[string]string{"name": "pixel-tests", "device": "angler", "source_type": "my_corpus"},
				Options: map[string]string{"some_option": "is optional", "ext": "png"},
				Digest:  blankDigest,
			}},
		}, results)
		return true
	})
	mg.On("UploadJSON", testutils.AnyContext, resultsMatcher, mock.Anything,
		`skia-gold-my-instance/dm-json-v1/2021/01/23/22/1234567890123456789012345678901234567890/waterfall/dm-1611440480000000019.json`).Return(nil)

	// Now call imgtest add with the following flags. This is simulating a test uploading a single
	// result for a test called pixel-tests. The digest has already been triaged positive.
	ctx, output, exit := testContext(mg, mh, nil, &timeOne)
	env := imgTest{
		gitHash:                 "1234567890123456789012345678901234567890",
		corpus:                  "my_corpus",
		failureFile:             filepath.Join(workDir, "failures.txt"),
		instanceID:              "my-instance",
		keysFile:                keysFile,
		passFailStep:            true,
		pngDigest:               blankDigest,
		pngFile:                 filepath.Join(td, "00000000000000000000000000000000.png"),
		testKeysStrings:         []string{"device:angler"},
		testName:                "pixel-tests",
		testOptionalKeysStrings: []string{"some_option:is optional"},
		workDir:                 workDir,
	}
	runUntilExit(t, func() {
		env.Add(ctx)
	})
	logs := output.String()
	exit.AssertWasCalledWithCode(t, 0, logs)
	mg.AssertExpectations(t)
}

func TestImgTest_InitAddFinalize_BatchMode_ExpectationsMatch_ProperJSONUploaded(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)
	td := testutils.TestDataDir(t)

	mh := mockRPCResponses("https://my-instance-gold.skia.org").Positive("pixel-tests", blankDigest).Build()

	// Call imgtest init with the following flags. We expect it to load the baseline expectations
	// and the known hashes.
	ctx, output, exit := testContext(nil, mh, nil, nil)
	env := imgTest{
		gitHash:         "1234567890123456789012345678901234567890",
		corpus:          "my_corpus",
		instanceID:      "my-instance",
		workDir:         workDir,
		testKeysStrings: []string{"os:Android"},
	}
	runUntilExit(t, func() {
		env.Init(ctx)
	})
	exit.AssertWasCalledWithCode(t, 0, output.String())

	// Now call imgtest add with the following flags. This is simulating adding a result for
	// a test called pixel-tests. The digest has been triaged positive for this test.
	ctx, output, exit = testContext(nil, mh, nil, nil)
	env = imgTest{
		workDir:                 workDir,
		testName:                "pixel-tests",
		pngFile:                 filepath.Join(td, "00000000000000000000000000000000.png"),
		pngDigest:               blankDigest,
		testKeysStrings:         []string{"device:angler"},
		testOptionalKeysStrings: []string{"some_option:is optional"},
	}
	runUntilExit(t, func() {
		env.Add(ctx)
	})
	logs := output.String()
	exit.AssertWasCalledWithCode(t, 0, logs)

	// Call imgtest add for a second device running the same test as above.
	ctx, output, exit = testContext(nil, mh, nil, nil)
	env = imgTest{
		workDir:                 workDir,
		testName:                "pixel-tests",
		pngFile:                 filepath.Join(td, "00000000000000000000000000000000.png"),
		pngDigest:               blankDigest,
		testKeysStrings:         []string{"device:bullhead"},
		testOptionalKeysStrings: []string{"some_option:is VERY DIFFERENT"},
	}
	runUntilExit(t, func() {
		env.Add(ctx)
	})
	logs = output.String()
	exit.AssertWasCalledWithCode(t, 0, logs)

	mg := &mocks.GCSUploader{}
	resultsMatcher := mock.MatchedBy(func(results jsonio.GoldResults) bool {
		assert.Equal(t, jsonio.GoldResults{
			GitHash: "1234567890123456789012345678901234567890",
			Key: map[string]string{
				"os":          "Android",
				"source_type": "my_corpus",
			},
			Results: []jsonio.Result{{
				Key:     map[string]string{"name": "pixel-tests", "device": "angler"},
				Options: map[string]string{"some_option": "is optional", "ext": "png"},
				Digest:  blankDigest,
			}, {
				Key:     map[string]string{"name": "pixel-tests", "device": "bullhead"},
				Options: map[string]string{"some_option": "is VERY DIFFERENT", "ext": "png"},
				Digest:  blankDigest,
			}},
		}, results)
		return true
	})
	mg.On("UploadJSON", testutils.AnyContext, resultsMatcher, mock.Anything,
		`skia-gold-my-instance/dm-json-v1/2021/01/23/22/1234567890123456789012345678901234567890/waterfall/dm-1611440480000000019.json`).Return(nil)

	// Call imgtest finalize, expecting to see all data before uploaded.
	ctx, output, exit = testContext(mg, nil, nil, &timeOne)
	env = imgTest{
		workDir: workDir,
	}
	runUntilExit(t, func() {
		env.Finalize(ctx)
	})
	logs = output.String()
	exit.AssertWasCalledWithCode(t, 0, logs)
	mg.AssertExpectations(t)
}

func TestImgTest_InitAddFinalize_BatchMode_ExpectationsDoNotMatch_ProperJSONAndImageUploaded(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)
	td := testutils.TestDataDir(t)

	mh := mockRPCResponses("https://my-instance-gold.skia.org").Build()

	// Call imgtest init with the following flags. We expect it to load the baseline expectations
	// and the known hashes.
	ctx, output, exit := testContext(nil, mh, nil, nil)
	env := imgTest{
		gitHash:         "1234567890123456789012345678901234567890",
		corpus:          "my_corpus",
		instanceID:      "my-instance",
		workDir:         workDir,
		testKeysStrings: []string{"os:Android"},
	}
	runUntilExit(t, func() {
		env.Init(ctx)
	})
	exit.AssertWasCalledWithCode(t, 0, output.String())

	mg := &mocks.GCSUploader{}
	bytesMatcher := mock.MatchedBy(func(b []byte) bool {
		assert.Len(t, b, 78) // spot check length
		return true
	})
	mg.On("UploadBytes", testutils.AnyContext, bytesMatcher, mock.Anything,
		`gs://skia-gold-my-instance/dm-images-v1/00000000000000000000000000000000.png`).Return(nil)

	// Now call imgtest add with the following flags. This is simulating adding a result for
	// a test called pixel-tests. The digest has not been seen before.
	ctx, output, exit = testContext(mg, mh, nil, nil)
	env = imgTest{
		workDir:                 workDir,
		testName:                "pixel-tests",
		pngFile:                 filepath.Join(td, "00000000000000000000000000000000.png"),
		pngDigest:               blankDigest,
		testKeysStrings:         []string{"device:angler"},
		testOptionalKeysStrings: []string{"some_option:is optional"},
	}
	runUntilExit(t, func() {
		env.Add(ctx)
	})
	logs := output.String()
	exit.AssertWasCalledWithCode(t, 0, logs)

	// Call imgtest add for a second device running the same test as above.
	// TODO(kjlubick) Append to the known digests to prevent a duplicate upload.
	ctx, output, exit = testContext(mg, mh, nil, nil)
	env = imgTest{
		workDir:                 workDir,
		testName:                "pixel-tests",
		pngFile:                 filepath.Join(td, "00000000000000000000000000000000.png"),
		pngDigest:               blankDigest,
		testKeysStrings:         []string{"device:bullhead"},
		testOptionalKeysStrings: []string{"some_option:is VERY DIFFERENT"},
	}
	runUntilExit(t, func() {
		env.Add(ctx)
	})
	logs = output.String()
	exit.AssertWasCalledWithCode(t, 0, logs)

	mg = &mocks.GCSUploader{}
	resultsMatcher := mock.MatchedBy(func(results jsonio.GoldResults) bool {
		assert.Equal(t, jsonio.GoldResults{
			GitHash: "1234567890123456789012345678901234567890",
			Key: map[string]string{
				"os":          "Android",
				"source_type": "my_corpus",
			},
			Results: []jsonio.Result{{
				Key:     map[string]string{"name": "pixel-tests", "device": "angler"},
				Options: map[string]string{"some_option": "is optional", "ext": "png"},
				Digest:  blankDigest,
			}, {
				Key:     map[string]string{"name": "pixel-tests", "device": "bullhead"},
				Options: map[string]string{"some_option": "is VERY DIFFERENT", "ext": "png"},
				Digest:  blankDigest,
			}},
		}, results)
		return true
	})
	mg.On("UploadJSON", testutils.AnyContext, resultsMatcher, mock.Anything,
		`skia-gold-my-instance/dm-json-v1/2021/01/23/22/1234567890123456789012345678901234567890/waterfall/dm-1611440480000000019.json`).Return(nil)

	// Call imgtest finalize, expecting to see all data before uploaded.
	ctx, output, exit = testContext(mg, nil, nil, &timeOne)
	env = imgTest{
		workDir: workDir,
	}
	runUntilExit(t, func() {
		env.Finalize(ctx)
	})
	logs = output.String()
	// In Batch mode, even though the images were untriaged, we return 0 (not failing).
	exit.AssertWasCalledWithCode(t, 0, logs)
	mg.AssertExpectations(t)
}

// This test compares image a01 and a05. These images have 2 pixels different, with a maximum
// delta of 7, so the settings are close enough to let those match.
func TestImgTest_Check_CloseEnoughForFuzzyMatch_ExitCodeZero(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)
	td := testutils.TestDataDir(t)

	mh := mockRPCResponses("https://my-instance-gold.skia.org").Positive("pixel-tests", a01Digest).
		LatestPositive(a01Digest, paramtools.Params{
			"device": "bullhead", "name": "pixel-tests", "source_type": "my-instance",
		}).Build()

	a01Bytes, err := os.ReadFile(filepath.Join(td, a01Digest+".png"))
	require.NoError(t, err)
	mi := &mocks.ImageDownloader{}
	mi.On("DownloadImage", testutils.AnyContext, "https://my-instance-gold.skia.org", types.Digest(a01Digest)).Return(a01Bytes, nil)

	ctx, output, exit := testContext(nil, mh, mi, nil)
	env := imgTest{
		workDir:         workDir,
		instanceID:      "my-instance",
		pngFile:         filepath.Join(td, a05Digest+".png"),
		testName:        "pixel-tests",
		testKeysStrings: []string{"device:bullhead"},
		testOptionalKeysStrings: []string{
			string(imgmatching.AlgorithmNameOptKey + ":" + imgmatching.FuzzyMatching),
			string(imgmatching.MaxDifferentPixels + ":2"),
			string(imgmatching.PixelDeltaThreshold + ":10"),
		},
	}
	runUntilExit(t, func() {
		env.Check(ctx)
	})
	logs := output.String()
	exit.AssertWasCalledWithCode(t, 0, logs)
	assert.Contains(t, logs, `Non-exact image comparison using algorithm "fuzzy" against most recent positive digest "a01a01a01a01a01a01a01a01a01a01a0".`)
	assert.Contains(t, logs, `Test: pixel-tests PASS`)
}

func TestImgTest_Check_TooDifferentOnChangelist_ExitCodeOne(t *testing.T) {

	workDir := t.TempDir()
	setupAuthWithGSUtil(t, workDir)
	td := testutils.TestDataDir(t)

	mh := mockRPCResponses("https://my-instance-gold.skia.org").Positive("pixel-tests", a01Digest).
		LatestPositive(a01Digest, paramtools.Params{
			"device": "bullhead", "name": "pixel-tests", "source_type": "my-instance",
		}).BuildForCL("gerritHub", "cl_1234")

	a01Bytes, err := os.ReadFile(filepath.Join(td, a01Digest+".png"))
	require.NoError(t, err)
	mi := &mocks.ImageDownloader{}
	mi.On("DownloadImage", testutils.AnyContext, "https://my-instance-gold.skia.org", types.Digest(a01Digest)).Return(a01Bytes, nil)

	ctx, output, exit := testContext(nil, mh, mi, nil)
	env := imgTest{
		workDir:          workDir,
		changelistID:     "cl_1234",
		codeReviewSystem: "gerritHub",
		instanceID:       "my-instance",
		pngFile:          filepath.Join(td, a09Digest+".png"),
		testName:         "pixel-tests",
		testKeysStrings:  []string{"device:bullhead"},
		testOptionalKeysStrings: []string{
			string(imgmatching.AlgorithmNameOptKey + ":" + imgmatching.FuzzyMatching),
			string(imgmatching.MaxDifferentPixels + ":2"),
			string(imgmatching.PixelDeltaThreshold + ":10"),
		},
	}
	runUntilExit(t, func() {
		env.Check(ctx)
	})
	logs := output.String()
	exit.AssertWasCalledWithCode(t, 1, logs)
	assert.Contains(t, logs, `Non-exact image comparison using algorithm "fuzzy" against most recent positive digest "a01a01a01a01a01a01a01a01a01a01a0".`)
	assert.Contains(t, logs, `Test: pixel-tests FAIL`)
}

func testContext(g gcsuploader.GCSUploader, h httpclient.HTTPClient, i imagedownloader.ImageDownloader, ts *time.Time) (context.Context, *threadSafeBuffer, *exitCodeRecorder) {
	output := &threadSafeBuffer{}
	exit := &exitCodeRecorder{}

	ctx := executionContext(context.Background(), output, output, exit.ExitWithCode)
	if ts != nil {
		ctx = context.WithValue(ctx, now.ContextKey, *ts)
	}
	return goldclient.WithContext(ctx, g, h, i), output, exit
}

type threadSafeBuffer struct {
	buf   bytes.Buffer
	mutex sync.Mutex
}

func (t *threadSafeBuffer) Write(p []byte) (n int, err error) {
	t.mutex.Lock()
	defer t.mutex.Unlock()
	return t.buf.Write(p)
}

func (t *threadSafeBuffer) String() string {
	t.mutex.Lock()
	defer t.mutex.Unlock()
	return t.buf.String()
}

type rpcResponsesBuilder struct {
	knownDigests    []string
	exp             *expectations.Expectations
	latestPositives map[tiling.TraceIDV2]types.Digest
	urlBase         string
}

func mockRPCResponses(instanceURL string) *rpcResponsesBuilder {
	return &rpcResponsesBuilder{
		exp:     &expectations.Expectations{},
		urlBase: instanceURL,
	}
}

func (r *rpcResponsesBuilder) Positive(name string, digest types.Digest) *rpcResponsesBuilder {
	r.exp.Set(types.TestName(name), digest, expectations.Positive)
	r.knownDigests = append(r.knownDigests, string(digest))
	return r
}

func (r *rpcResponsesBuilder) Negative(name string, digest types.Digest) *rpcResponsesBuilder {
	r.exp.Set(types.TestName(name), digest, expectations.Negative)
	r.knownDigests = append(r.knownDigests, string(digest))
	return r
}

func (r *rpcResponsesBuilder) Known(digest types.Digest) *rpcResponsesBuilder {
	r.knownDigests = append(r.knownDigests, string(digest))
	return r
}

func (r *rpcResponsesBuilder) LatestPositive(digest types.Digest, traceKeys paramtools.Params) *rpcResponsesBuilder {
	if len(r.latestPositives) == 0 {
		r.latestPositives = map[tiling.TraceIDV2]types.Digest{}
	}
	_, traceIDBytes := sql.SerializeMap(traceKeys)
	traceID := tiling.TraceIDV2(hex.EncodeToString(traceIDBytes))
	r.latestPositives[traceID] = digest
	return r
}

func (r *rpcResponsesBuilder) Build() *mocks.HTTPClient {
	mh := &mocks.HTTPClient{}
	knownResp := strings.Join(r.knownDigests, "\n")
	mh.On("Get", r.urlBase+"/json/v1/hashes").Return(httpResponse(knownResp, "200 OK", http.StatusOK), nil)

	exp, err := json.Marshal(frontend.BaselineV2Response{
		Expectations: r.exp.AsBaseline(),
	})
	if err != nil {
		panic(err)
	}
	mh.On("Get", r.urlBase+"/json/v2/expectations").Return(
		httpResponse(string(exp), "200 OK", http.StatusOK), nil)

	for traceID, digest := range r.latestPositives {
		j, err := json.Marshal(frontend.MostRecentPositiveDigestResponse{Digest: digest})
		if err != nil {
			panic(err)
		}
		url := r.urlBase + "/json/v2/latestpositivedigest/" + string(traceID)
		mh.On("Get", url).Return(
			httpResponse(string(j), "200 OK", http.StatusOK), nil)
	}

	groupingsResp := httpResponse(`{"grouping_param_keys_by_corpus": {"my_corpus": ["name", "source_type"]}}`, "200 OK", http.StatusOK)
	mh.On("Get", r.urlBase+"/json/v1/groupings").Return(groupingsResp, nil)

	return mh
}

func (r *rpcResponsesBuilder) BuildForCL(crs, clID string) *mocks.HTTPClient {
	mh := &mocks.HTTPClient{}
	knownResp := strings.Join(r.knownDigests, "\n")
	mh.On("Get", r.urlBase+"/json/v1/hashes").Return(httpResponse(knownResp, "200 OK", http.StatusOK), nil)

	exp, err := json.Marshal(frontend.BaselineV2Response{
		Expectations:     r.exp.AsBaseline(),
		ChangelistID:     clID,
		CodeReviewSystem: crs,
	})
	if err != nil {
		panic(err)
	}
	url := fmt.Sprintf("%s/json/v2/expectations?issue=%s&crs=%s", r.urlBase, clID, crs)
	mh.On("Get", url).Return(
		httpResponse(string(exp), "200 OK", http.StatusOK), nil)

	for traceID, digest := range r.latestPositives {
		j, err := json.Marshal(frontend.MostRecentPositiveDigestResponse{Digest: digest})
		if err != nil {
			panic(err)
		}
		url := r.urlBase + "/json/v2/latestpositivedigest/" + string(traceID)
		mh.On("Get", url).Return(
			httpResponse(string(j), "200 OK", http.StatusOK), nil)
	}

	groupingsResp := httpResponse(`{"grouping_param_keys_by_corpus": {"my_corpus": ["name", "source_type"]}}`, "200 OK", http.StatusOK)
	mh.On("Get", r.urlBase+"/json/v1/groupings").Return(groupingsResp, nil)

	return mh
}

func TestRPCResponsesBuilder_Default_ReturnsBlankValues(t *testing.T) {

	mh := mockRPCResponses("https://my-instance-gold.skia.org").Build()
	resp, err := mh.Get("https://my-instance-gold.skia.org/json/v1/hashes")
	require.NoError(t, err)
	b, err := io.ReadAll(resp.Body)
	require.NoError(t, err)
	assert.Equal(t, "", string(b))

	resp, err = mh.Get("https://my-instance-gold.skia.org/json/v2/expectations")
	require.NoError(t, err)
	b, err = io.ReadAll(resp.Body)
	require.NoError(t, err)
	assert.Equal(t, `{}`, string(b))
}

func TestRPCResponsesBuilder_WithValues_ReturnsValidListsAndJSON(t *testing.T) {

	mh := mockRPCResponses("http://my-custom-url.example.com").
		Known("first_digest").
		Positive("alpha test", "second_digest").
		Positive("beta test", "third_digest").
		Negative("alpha test", "fourth_digest").
		LatestPositive("third_digest", paramtools.Params{"alpha": "beta", "gamma": "delta epsilon"}).
		Known("fifth_digest").
		Build()
	resp, err := mh.Get("http://my-custom-url.example.com/json/v1/hashes")
	require.NoError(t, err)
	b, err := io.ReadAll(resp.Body)
	require.NoError(t, err)
	assert.Equal(t, `first_digest
second_digest
third_digest
fourth_digest
fifth_digest`, string(b))

	resp, err = mh.Get("http://my-custom-url.example.com/json/v2/expectations")
	require.NoError(t, err)
	b, err = io.ReadAll(resp.Body)
	require.NoError(t, err)
	assert.Equal(t, `{"primary":{"alpha test":{"fourth_digest":"negative","second_digest":"positive"},"beta test":{"third_digest":"positive"}}}`, string(b))

	const expectedTraceID = "fad8dda3d6600fde059cb81f4ec64059"
	_, tb := sql.SerializeMap(map[string]string{"alpha": "beta", "gamma": "delta epsilon"})
	require.Equal(t, expectedTraceID, hex.EncodeToString(tb))
	resp, err = mh.Get("http://my-custom-url.example.com/json/v2/latestpositivedigest/" + expectedTraceID)
	require.NoError(t, err)
	b, err = io.ReadAll(resp.Body)
	require.NoError(t, err)
	assert.Equal(t, `{"digest":"third_digest"}`, string(b))
}