golden/go/search/benchmarks_test.go

package search

import (
	"context"
	"crypto/md5"
	"encoding/hex"
	"math/rand"
	"testing"

	"github.com/stretchr/testify/require"

	"go.skia.org/infra/go/paramtools"
	"go.skia.org/infra/go/testutils"
	"go.skia.org/infra/golden/go/clstore"
	mock_clstore "go.skia.org/infra/golden/go/clstore/mocks"
	"go.skia.org/infra/golden/go/code_review"
	"go.skia.org/infra/golden/go/expectations"
	mock_index "go.skia.org/infra/golden/go/indexer/mocks"
	"go.skia.org/infra/golden/go/search/query"
	"go.skia.org/infra/golden/go/tiling"
	"go.skia.org/infra/golden/go/tjstore"
	mock_tjstore "go.skia.org/infra/golden/go/tjstore/mocks"
	"go.skia.org/infra/golden/go/types"
)

const (
	// These consts were arbitrarily picked to approximate a representative Skia Changelist
	numResultParams  = 6
	numOptionsParams = 8
	numGroupParams   = 12
	numIgnoreRules   = 100
	// numIgnorableValues can be tuned higher to ignore fewer values and lower to ignore more.
	// Right now, this value ignores 10% on average in BenchmarkExtractChangelistDigests
	numIgnorableValues = 500
)

// BenchmarkExtractChangelistDigests benchmarks extractChangelistDigests, specifically
// focusing on the filtering logic after the TryJobResults are returned.
func BenchmarkExtractChangelistDigests(b *testing.B) {
	const gerritCRS = "gerrit"
	mis := &mock_index.IndexSearcher{}
	mcls := &mock_clstore.Store{}
	mtjs := &mock_tjstore.Store{}

	clID := "123"
	psOrder := 1
	// 500k was observed as a typical number of results for Skia, as were 15 unique options and
	// 15 unique groups.
	xtr := genTryJobResults(500000, 15, 15)

	mcls.On("GetPatchsets", testutils.AnyContext, clID).Return([]code_review.Patchset{
		{
			SystemID:     "first_one",
			ChangelistID: clID,
			Order:        psOrder,
			// All the rest are ignored
		},
	}, nil)

	mis.On("GetIgnoreMatcher").Return(makeIgnoreRules())
	combinedID := tjstore.CombinedPSID{CL: "123", CRS: "gerrit", PS: "first_one"}
	// return a copy of the slice so we can mess around with the data however we like.
	mtjs.On("GetResults", testutils.AnyContext, combinedID).Return(func(context.Context, tjstore.CombinedPSID) []tjstore.TryJobResult {
		c := make([]tjstore.TryJobResult, len(xtr))
		copy(c, xtr)
		return c
	}, nil)

	reviewSystems := []clstore.ReviewSystem{
		{
			ID:    gerritCRS,
			Store: mcls,
			// Client and URLTemplate are unused here
		},
	}

	s := &SearchImpl{
		reviewSystems: reviewSystems,
		tryJobStore:   mtjs,
	}

	fn := func(_ types.TestName, _ types.Digest, _ paramtools.Params, _ tiling.TracePair) {}

	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		err := s.extractChangelistDigests(context.Background(), &query.Search{
			Patchsets:               []int64{int64(psOrder)},
			TraceValues:             map[string][]string{},
			IncludeUntriagedDigests: true,
			CodeReviewSystemID:      "gerrit",
			ChangelistID:            clID,
		}, mis, expectations.EmptyClassifier(), fn)
		require.NoError(b, err)
	}
}

// ignorableFields and ignorableValues allow us to have somewhat random inputs that can be
// re-used and thus matched upon by ignores.
var ignorableFields = []string{types.PrimaryKeyField, "config", "gpu", "os", "flavor", "smell", "weight"}
var ignorableValues []string = nil

func init() {
	for i := 0; i < numIgnorableValues; i++ {
		ignorableValues = append(ignorableValues, randValue())
	}
}

// makeIgnoreRules makes a set of synthetic ignore rules that approximately represents
// those found in Skia. It uses ignorableFields and ignorableValues to have a mix of things
// that could possibly line up with the values in genTryJobResults.
func makeIgnoreRules() paramtools.ParamMatcher {
	var pm paramtools.ParamMatcher
	for i := 0; i < numIgnoreRules; i++ {
		// 1-2 fields
		numFields := rand.Intn(2) + 1
		p := paramtools.ParamSet{}
		fieldPerm := rand.Perm(len(ignorableFields))
		for _, r := range fieldPerm[:numFields] {
			f := ignorableFields[r]

			// 1-4 values
			numValues := rand.Intn(4) + 1
			var v []string
			fieldPerm := rand.Perm(len(ignorableFields))
			for _, r := range fieldPerm[:numValues] {
				v = append(v, ignorableValues[r])
			}
			p[f] = v
		}
		pm = append(pm, p)
	}
	return pm
}

// genTryJobResults makes TryJobResults with synthetic data that approximately represents
// the data created by Skia. The data is structured such that some (~10%) of the results will
// be ignored by makeIgnoreRules(), but the vast majority will not. Additionally, the number of
// fields in the various Params is generally representative.
func genTryJobResults(results, uniqueOptions, uniqueGroups int) []tjstore.TryJobResult {
	opts := make([]paramtools.Params, 0, uniqueOptions)
	for i := 0; i < uniqueOptions; i++ {
		opts = append(opts, makeParams(numOptionsParams))
	}
	groups := make([]paramtools.Params, 0, uniqueGroups)
	for i := 0; i < uniqueGroups; i++ {
		groups = append(groups, makeParams(numGroupParams))
	}
	xtr := make([]tjstore.TryJobResult, 0, results)
	for i := 0; i < results; i++ {
		o := rand.Intn(uniqueOptions)
		g := rand.Intn(uniqueGroups)
		xtr = append(xtr, tjstore.TryJobResult{
			GroupParams:  groups[g],
			Options:      opts[o],
			ResultParams: makeResults(),
			Digest:       types.Digest(randValue()),
		})
	}

	return xtr
}

// makeParams makes a Params that is a blend of ignorable values, and non-ignorable values.
func makeParams(n int) paramtools.Params {
	p := paramtools.Params{}
	numIgnorables := 3
	for i := 0; len(p) < n && i < numIgnorables; i++ {
		r := rand.Intn(len(ignorableFields))
		f := ignorableFields[r]
		// don't have name for these - that should be in result params
		if f == types.PrimaryKeyField {
			f = ignorableFields[1]
		}
		r = rand.Intn(len(ignorableValues))
		v := ignorableValues[r]
		p[f] = v
	}

	numMaybes := 2
	for i := 0; len(p) < n && i < numMaybes; i++ {
		r := rand.Intn(len(ignorableFields))
		f := ignorableFields[r]
		// don't have name for these - that should be in result params
		if f == types.PrimaryKeyField {
			f = ignorableFields[1]
		}
		p[f] = randValue()
	}

	for len(p) < n {
		p[randValue()] = randValue()
	}
	return p
}

// makeResults returns a Params similar to those in Skia results. They always have a name.
// The rest of the fields are possibly ignorable.
func makeResults() paramtools.Params {
	p := paramtools.Params{}
	if rand.Float32() < 0.5 {
		p[types.PrimaryKeyField] = randValue()
	} else {
		r := rand.Intn(len(ignorableValues))
		p[types.PrimaryKeyField] = ignorableValues[r]
	}
	numIgnorables := 2
	for i := 1; len(p) < numResultParams && i < numIgnorables; i++ {
		r := rand.Intn(len(ignorableFields))
		f := ignorableFields[r]
		// don't have multiple names
		if f == types.PrimaryKeyField {
			f = ignorableFields[2]
		}
		r = rand.Intn(len(ignorableValues))
		v := ignorableValues[r]
		p[f] = v
	}

	for len(p) < numResultParams {
		r := rand.Intn(len(ignorableFields))
		f := ignorableFields[r]
		// don't have multiple names
		if f == types.PrimaryKeyField {
			f = ignorableFields[2]
		}
		p[f] = randValue()
	}

	return p
}

// randValue returns a random string. It happens to be a hex encoded Digest, but can be used
// any place a random medium-length string is needed.
func randValue() string {
	b := make([]byte, md5.Size)
	_, _ = rand.Read(b)
	return hex.EncodeToString(b)
}