golden/go/diffstore/mem_diffstore_test.go

package diffstore

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"

	"cloud.google.com/go/storage"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/require"
	"google.golang.org/api/option"

	"go.skia.org/infra/go/firestore"
	"go.skia.org/infra/go/gcs/gcsclient"
	"go.skia.org/infra/go/gcs/test_gcsclient"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/testutils"
	"go.skia.org/infra/go/testutils/unittest"
	"go.skia.org/infra/golden/go/diff"
	"go.skia.org/infra/golden/go/diffstore/common"
	"go.skia.org/infra/golden/go/diffstore/metricsstore/fs_metricsstore"
	diffstore_mocks "go.skia.org/infra/golden/go/diffstore/mocks"
	"go.skia.org/infra/golden/go/image/text"
	one_by_five "go.skia.org/infra/golden/go/testutils/data_one_by_five"
	"go.skia.org/infra/golden/go/types"
)

const (
	// Missing digest (valid, but arbitrary).
	missingDigest       = types.Digest("ffffffffffffffffffffffffffffffff")
	missingDigestGsPath = gcsImageBaseDir + "/" + string(missingDigest) + ".png"

	// Digest for the 16-bit image stored in the testdata directory. This is the same image used to
	// test preservation of color space information in skbug.com/9483.
	digest16BitImage       = types.Digest("8a90a2f1245dc87d96bfb74bdc4ab97e")
	digest16BitImageGsPath = gcsImageBaseDir + "/" + string(digest16BitImage) + ".png"

	// MD5 hash of the 16-bit PNG image above. Needed for the storage.ObjectAttrs instance returned by
	// the mock GCS client.
	md5Hash16BitImage = "22ea2cb4e3eabd2bb3faba7a07e18b7a" // = md5sum(8a90a2f1245dc87d96bfb74bdc4ab97e.png)

	invalidDigest1 = types.Digest("invaliddigest1")
	invalidDigest2 = types.Digest("invaliddigest2")

	gcsTestBucket = "skia-infra-testdata"
)

func TestMD5Hash16BitImageIsCorrect(t *testing.T) {
	unittest.SmallTest(t)
	b := testutils.ReadFileBytes(t, fmt.Sprintf("%s.png", digest16BitImage))
	require.Equal(t, md5Hash16BitImage, bytesToMD5HashString(b))
}

// TestMemDiffStoreGetSunnyDay tests the case where we are getting metrics for two digests
// that are both cache misses.
func TestMemDiffStoreGetSunnyDay(t *testing.T) {
	unittest.SmallTest(t)

	mms := &diffstore_mocks.MetricsStore{}
	mgc := test_gcsclient.NewMockClient()
	defer mms.AssertExpectations(t)
	defer mgc.AssertExpectations(t)

	// These diffs are the actual diffs between the respective 2 images.
	// These values were computed by using the default algorithm and manual inspection.
	dm1_2 := &diff.DiffMetrics{
		NumDiffPixels:    5,
		PixelDiffPercent: 100,
		MaxRGBADiffs:     [4]int{1, 1, 1, 1},
		CombinedMetric:   0.6262243,
	}
	dm1_3 := &diff.DiffMetrics{
		NumDiffPixels:    4,
		PixelDiffPercent: 80,
		MaxRGBADiffs:     [4]int{2, 0, 1, 2},
		CombinedMetric:   0.6859943,
	}

	// Assume everything is a cache miss
	expectedDiffIDs := []string{common.DiffID(digest1, digest2), common.DiffID(digest1, digest3)}
	mms.On("LoadDiffMetrics", testutils.AnyContext, expectedDiffIDs).Return([]*diff.DiffMetrics{nil, nil}, nil)

	expectImageWillBeRead(mgc, image1GCSPath, image1MD5Hash, image1)
	expectImageWillBeRead(mgc, image2GCSPath, image2MD5Hash, image2)
	expectImageWillBeRead(mgc, image3GCSPath, image3MD5Hash, image3)

	mms.On("SaveDiffMetrics", testutils.AnyContext, common.DiffID(digest1, digest2), dm1_2).Return(nil)
	mms.On("SaveDiffMetrics", testutils.AnyContext, common.DiffID(digest1, digest3), dm1_3).Return(nil)

	diffStore, err := NewMemDiffStore(mgc, gcsImageBaseDir, 1, mms)
	require.NoError(t, err)

	diffDigests := []types.Digest{digest2, digest3}

	diffs, err := diffStore.Get(context.Background(), digest1, diffDigests)
	require.NoError(t, err)
	assert.Len(t, diffs, 2)
	assert.Equal(t, dm1_2, diffs[digest2])
	assert.Equal(t, dm1_3, diffs[digest3])
}

// TestMemDiffStoreGetIntegration performs the Get operation backed by real GCS and a firestore
// emulator.
func TestMemDiffStoreGetIntegration(t *testing.T) {
	unittest.LargeTest(t)

	// The test bucket is a public bucket, so we don't need to worry about authentication.
	unauthedClient := httputils.DefaultClientConfig().Client()

	storageClient, err := storage.NewClient(context.Background(), option.WithHTTPClient(unauthedClient))
	require.NoError(t, err)
	gcsClient := gcsclient.New(storageClient, gcsTestBucket)

	// create a client against the firestore emulator.
	c, cleanup := firestore.NewClientForTesting(context.Background(), t)
	defer cleanup()
	fsMetrics := fs_metricsstore.New(c)

	// These are two nearly identical images in the skia-infra-testdata bucket.
	// The names are arbitrary (they don't actually correspond with the hash of the pixels).
	original := types.Digest("000da2ce46164b5027ee964b8c040335")
	cross := types.Digest("cccd4f34d847bd8a540c7c9cf1602107")

	// There are 5 pixels in the cross image that are black instead of white.
	// These values were computed by using the default algorithm and manual inspection.
	dm := &diff.DiffMetrics{
		NumDiffPixels:    5,
		PixelDiffPercent: 0.0010146104,
		MaxRGBADiffs:     [4]int{255, 255, 255, 0},
		CombinedMetric:   0.02964251,
	}

	diffStore, err := NewMemDiffStore(gcsClient, gcsImageBaseDir, 1, fsMetrics)
	require.NoError(t, err)

	diffDigests := []types.Digest{cross}

	diffs, err := diffStore.Get(context.Background(), original, diffDigests)
	require.NoError(t, err)
	assert.Len(t, diffs, 1)
	assert.Equal(t, dm, diffs[cross])

	// make sure they are actually stored

	actual, err := fsMetrics.LoadDiffMetrics(context.Background(), []string{common.DiffID(original, cross)})
	require.NoError(t, err)
	assert.Equal(t, dm, actual[0])
}

// TestMemDiffStoreGetPartialCacheMatch tests the case where we are getting metrics for two digests
// and one is a cache hit and the other is a cache miss.
func TestMemDiffStoreGetPartialCacheMatch(t *testing.T) {
	unittest.SmallTest(t)

	mms := &diffstore_mocks.MetricsStore{}
	mgc := test_gcsclient.NewMockClient()
	defer mms.AssertExpectations(t)
	defer mgc.AssertExpectations(t)

	// These diffs are the actual diffs between the respective 2 images.
	// These values were computed by using the default algorithm and manual inspection.
	dm1_2 := &diff.DiffMetrics{
		NumDiffPixels:    5,
		PixelDiffPercent: 100,
		MaxRGBADiffs:     [4]int{1, 1, 1, 1},
		CombinedMetric:   0.6262243,
	}
	dm1_3 := &diff.DiffMetrics{
		NumDiffPixels:    4,
		PixelDiffPercent: 80,
		MaxRGBADiffs:     [4]int{2, 0, 1, 2},
		CombinedMetric:   0.6859943,
	}

	// One cache hit, one cache miss.
	expectedDiffIDs := []string{common.DiffID(digest1, digest2), common.DiffID(digest1, digest3)}
	mms.On("LoadDiffMetrics", testutils.AnyContext, expectedDiffIDs).Return([]*diff.DiffMetrics{dm1_2, nil}, nil)

	expectImageWillBeRead(mgc, image1GCSPath, image1MD5Hash, image1)
	expectImageWillBeRead(mgc, image3GCSPath, image3MD5Hash, image3)

	mms.On("SaveDiffMetrics", testutils.AnyContext, common.DiffID(digest1, digest3), dm1_3).Return(nil)

	diffStore, err := NewMemDiffStore(mgc, gcsImageBaseDir, 1, mms)
	require.NoError(t, err)

	diffDigests := []types.Digest{digest2, digest3}

	diffs, err := diffStore.Get(context.Background(), digest1, diffDigests)
	require.NoError(t, err)
	assert.Len(t, diffs, 2)
	assert.Equal(t, dm1_2, diffs[digest2])
	assert.Equal(t, dm1_3, diffs[digest3])
}

// TestMemDiffStoreGetIdentity tests the case where the mainDigest is in the list of things
// to diff against.
func TestMemDiffStoreGetIdentity(t *testing.T) {
	unittest.SmallTest(t)

	mms := &diffstore_mocks.MetricsStore{}
	mgc := test_gcsclient.NewMockClient()
	defer mms.AssertExpectations(t)
	defer mgc.AssertExpectations(t)

	// These diffs are the actual diffs between the respective 2 images.
	// These values were computed by using the default algorithm and manual inspection.
	dm1_2 := &diff.DiffMetrics{
		NumDiffPixels:    5,
		PixelDiffPercent: 100,
		MaxRGBADiffs:     [4]int{1, 1, 1, 1},
		CombinedMetric:   0.6262243,
	}
	dm1_3 := &diff.DiffMetrics{
		NumDiffPixels:    4,
		PixelDiffPercent: 80,
		MaxRGBADiffs:     [4]int{2, 0, 1, 2},
		CombinedMetric:   0.6859943,
	}

	// Assume everything is a cache hit
	expectedDiffIDs := []string{common.DiffID(digest1, digest2), common.DiffID(digest1, digest3)}
	mms.On("LoadDiffMetrics", testutils.AnyContext, expectedDiffIDs).Return([]*diff.DiffMetrics{dm1_2, dm1_3}, nil)

	diffStore, err := NewMemDiffStore(mgc, gcsImageBaseDir, 1, mms)
	require.NoError(t, err)

	diffDigests := []types.Digest{digest2, digest1, digest3}

	diffs, err := diffStore.Get(context.Background(), digest1, diffDigests)
	require.NoError(t, err)
	assert.Len(t, diffs, 2)
	assert.Equal(t, dm1_2, diffs[digest2])
	assert.Equal(t, dm1_3, diffs[digest3])
}

// TestFailureHandlingGet tests a case where two digests are not found in the GCS bucket.
func TestFailureHandlingGet(t *testing.T) {
	unittest.SmallTest(t)

	mms := &diffstore_mocks.MetricsStore{}
	mgc := test_gcsclient.NewMockClient()
	defer mms.AssertExpectations(t)
	defer mgc.AssertExpectations(t)

	dm := &diff.DiffMetrics{
		// This data is arbitrary - just to make sure we get the right object
		MaxRGBADiffs: [4]int{1, 2, 3, 4},
	}

	expectedDiffIDs := []string{common.DiffID(digest1, digest2), common.DiffID(digest1, invalidDigest1), common.DiffID(digest1, invalidDigest2)}
	mms.On("LoadDiffMetrics", testutils.AnyContext, expectedDiffIDs).Return([]*diff.DiffMetrics{dm, nil, nil}, nil)

	// mgc succeeds for digest1 (which is loaded anyway in an attempt to compare against the two
	// invalid digests).
	expectImageWillBeRead(mgc, image1GCSPath, image1MD5Hash, image1)

	// mgc should fail to return the invalid digest (the first of which should cause the rest to fail.
	img := fmt.Sprintf("%s/%s.png", gcsImageBaseDir, invalidDigest1)
	mgc.On("GetFileObjectAttrs", testutils.AnyContext, img).Return(nil, errors.New("not found"))
	mgc.On("Bucket").Return("whatever")

	diffStore, err := NewMemDiffStore(mgc, gcsImageBaseDir, 1, mms)
	require.NoError(t, err)

	diffDigests := []types.Digest{digest2, invalidDigest1, invalidDigest2}

	_, err = diffStore.Get(context.Background(), digest1, diffDigests)
	require.Error(t, err)
}

// TestMetricsStoreFlakiness tests that we still can compute diffs if the metricStore is down.
func TestMetricsStoreFlakiness(t *testing.T) {
	unittest.SmallTest(t)

	mms := &diffstore_mocks.MetricsStore{}
	mgc := test_gcsclient.NewMockClient()
	defer mms.AssertExpectations(t)
	defer mgc.AssertExpectations(t)

	// These diffs are the actual diffs between the respective 2 images.
	// These values were computed by using the default algorithm and manual inspection.
	dm1_2 := &diff.DiffMetrics{
		NumDiffPixels:    5,
		PixelDiffPercent: 100,
		MaxRGBADiffs:     [4]int{1, 1, 1, 1},
		CombinedMetric:   0.6262243,
	}

	// Assume we can't read or write to metricsstore
	mms.On("LoadDiffMetrics", testutils.AnyContext, mock.Anything).Return(nil, errors.New("out of quota"))
	mms.On("SaveDiffMetrics", testutils.AnyContext, mock.Anything, mock.Anything).Return(errors.New("out of quota"))

	expectImageWillBeRead(mgc, image1GCSPath, image1MD5Hash, image1)
	expectImageWillBeRead(mgc, image2GCSPath, image2MD5Hash, image2)

	diffStore, err := NewMemDiffStore(mgc, gcsImageBaseDir, 1, mms)
	require.NoError(t, err)

	diffDigests := []types.Digest{digest2}

	diffs, err := diffStore.Get(context.Background(), digest1, diffDigests)
	require.NoError(t, err)
	assert.Len(t, diffs, 1)
	assert.Equal(t, dm1_2, diffs[digest2])
}

// TestPurgeDigests makes we correctly purge digests from both metrics and GCS
func TestPurgeDigests(t *testing.T) {
	unittest.SmallTest(t)

	mms := &diffstore_mocks.MetricsStore{}
	mgc := test_gcsclient.NewMockClient()
	defer mms.AssertExpectations(t)
	defer mgc.AssertExpectations(t)

	oa := &storage.ObjectAttrs{}
	img := fmt.Sprintf("%s/%s.png", gcsImageBaseDir, invalidDigest2)
	mgc.On("GetFileObjectAttrs", testutils.AnyContext, img).Return(oa, nil)
	mgc.On("DeleteFile", testutils.AnyContext, img).Return(nil)

	mms.On("PurgeDiffMetrics", testutils.AnyContext, types.DigestSlice{invalidDigest1}).Return(nil)
	mms.On("PurgeDiffMetrics", testutils.AnyContext, types.DigestSlice{invalidDigest2}).Return(nil)

	diffStore, err := NewMemDiffStore(mgc, gcsImageBaseDir, 1, mms)
	require.NoError(t, err)

	require.NoError(t, diffStore.PurgeDigests(context.Background(), types.DigestSlice{invalidDigest1}, false))
	require.NoError(t, diffStore.PurgeDigests(context.Background(), types.DigestSlice{invalidDigest2}, true))
}

func TestMemDiffStoreImageHandler(t *testing.T) {
	unittest.SmallTest(t)

	// This is a white-box test. The mock GCS client below return only what is
	// needed for this test to pass, and nothing more (e.g. field "MD5" in *storage.ObjectAttrs).

	// Build mock GCSClient.
	mockBucketClient := test_gcsclient.NewMockClient()
	defer mockBucketClient.AssertExpectations(t)

	// Only used for logging errors, which only some tests produce.
	mockBucketClient.On("Bucket").Return("test-bucket")

	// missingDigest is not present in the GCS bucket.
	var oa *storage.ObjectAttrs
	mockBucketClient.On("GetFileObjectAttrs", testutils.AnyContext, missingDigestGsPath).Return(oa, errors.New("not found"))

	// digest1 is present in the GCS bucket.
	oa1 := &storage.ObjectAttrs{MD5: md5HashToBytes(image1MD5Hash)}
	mockBucketClient.On("GetFileObjectAttrs", testutils.AnyContext, image1GCSPath).Return(oa1, nil)

	// digest1 is read.
	pngImage1Bytes := imageToPng(image1).Bytes()
	reader1 := ioutil.NopCloser(bytes.NewBuffer(pngImage1Bytes))
	mockBucketClient.On("FileReader", testutils.AnyContext, image1GCSPath).Return(reader1, nil)

	// digest2 is present in the GCS bucket.
	oa2 := &storage.ObjectAttrs{MD5: md5HashToBytes(image2MD5Hash)}
	mockBucketClient.On("GetFileObjectAttrs", testutils.AnyContext, image2GCSPath).Return(oa2, nil)

	// digest2 is read.
	reader2 := ioutil.NopCloser(imageToPng(image2))
	mockBucketClient.On("FileReader", testutils.AnyContext, image2GCSPath).Return(reader2, nil)

	// digest16BitImage is present in the GCS bucket.
	oa3 := &storage.ObjectAttrs{MD5: md5HashToBytes(md5Hash16BitImage)}
	mockBucketClient.On("GetFileObjectAttrs", testutils.AnyContext, digest16BitImageGsPath).Return(oa3, nil)

	// digest16BitImage is read.
	bytes16BitImage := testutils.ReadFileBytes(t, fmt.Sprintf("%s.png", digest16BitImage))
	reader3 := ioutil.NopCloser(bytes.NewReader(bytes16BitImage))
	mockBucketClient.On("FileReader", testutils.AnyContext, digest16BitImageGsPath).Return(reader3, nil)

	// Metrics store.
	mStore := &diffstore_mocks.MetricsStore{}

	// Build MemDiffStore instance under test.
	diffStore, err := NewMemDiffStore(mockBucketClient, gcsImageBaseDir, 10, mStore)
	require.NoError(t, err)

	// Get the HTTP handler function under test.
	handlerFn, err := diffStore.ImageHandler("/img/")
	require.NoError(t, err)

	// Executes GET requests.
	get := func(urlFmt string, elem ...interface{}) *httptest.ResponseRecorder {
		url := fmt.Sprintf(urlFmt, elem...)

		// Create request.
		req, err := http.NewRequest("GET", url, nil)
		require.NoError(t, err)

		// Create the ResponseRecorder that will be returned after the request is served.
		rr := httptest.NewRecorder()

		// Serve request.
		handlerFn.ServeHTTP(rr, req)

		return rr
	}

	// Invalid digest.
	rr := get("/img/images/foo.png")
	require.Equal(t, http.StatusNotFound, rr.Code)
	require.Equal(t, "no-cache, no-store, must-revalidate", rr.Header().Get("Cache-Control"))

	// Missing digest.
	rr = get("/img/images/%s.png", missingDigest)
	require.Equal(t, http.StatusNotFound, rr.Code)
	require.Equal(t, "no-cache, no-store, must-revalidate", rr.Header().Get("Cache-Control"))

	// Image 1.
	rr = get("/img/images/%s.png", digest1)
	require.Equal(t, http.StatusOK, rr.Code)
	require.Equal(t, pngImage1Bytes, rr.Body.Bytes())
	require.Equal(t, "public, max-age=43200", rr.Header().Get("Cache-Control"))

	// Diff between images 1 and 2.
	rr = get("/img/diffs/%s-%s.png", digest1, digest2)
	require.Equal(t, http.StatusOK, rr.Code)
	d := text.MustToNRGBA(one_by_five.DiffImageOneAndTwo)
	require.Equal(t, imageToPng(d).Bytes(), rr.Body.Bytes())
	require.Equal(t, "public, max-age=43200", rr.Header().Get("Cache-Control"))

	// 16-bit image is returned verbatim as found in GCS. See skbug.com/9483 for more context.
	rr = get("/img/images/%s.png", digest16BitImage)
	require.Equal(t, http.StatusOK, rr.Code)
	require.Equal(t, bytes16BitImage, rr.Body.Bytes())
	require.Equal(t, "public, max-age=43200", rr.Header().Get("Cache-Control"))
}

func TestDecodeImageSuccess(t *testing.T) {
	unittest.SmallTest(t)

	// Inputs.
	b := imageToPng(image1).Bytes()

	actual, err := common.DecodeImg(bytes.NewReader(b))
	require.NoError(t, err)
	require.Equal(t, image1, actual)
}

func TestDecodeImagesInvalid(t *testing.T) {
	unittest.SmallTest(t)

	b := []byte("I'm not a PNG image")

	_, err := common.DecodeImg(bytes.NewReader(b))
	require.Error(t, err)
}