[gitstore] Bring back old implementation under go/gitstore_deprecated
Bug: skia:9084
Change-Id: Iee71e00dbbda0d4adb69ba29e6af428627a2245b
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/236345
Reviewed-by: Kevin Lubick <kjlubick@google.com>
Commit-Queue: Eric Boren <borenet@google.com>
diff --git a/datahopper/go/datahopper/main.go b/datahopper/go/datahopper/main.go
index 515bee7..87f3755 100644
--- a/datahopper/go/datahopper/main.go
+++ b/datahopper/go/datahopper/main.go
@@ -22,7 +22,7 @@
"go.skia.org/infra/go/gcs/gcsclient"
"go.skia.org/infra/go/git"
"go.skia.org/infra/go/gitauth"
- "go.skia.org/infra/go/gitstore/bt_gitstore"
+ "go.skia.org/infra/go/gitstore_deprecated/bt_gitstore"
"go.skia.org/infra/go/httputils"
"go.skia.org/infra/go/metrics2"
"go.skia.org/infra/go/sklog"
diff --git a/go/gitstore/bt_gitstore/bt_gitstore.go b/go/gitstore/bt_gitstore/bt_gitstore.go
index 35acd3b..cec4777 100644
--- a/go/gitstore/bt_gitstore/bt_gitstore.go
+++ b/go/gitstore/bt_gitstore/bt_gitstore.go
@@ -18,6 +18,7 @@
"cloud.google.com/go/bigtable"
"go.skia.org/infra/go/git"
"go.skia.org/infra/go/gitstore"
+ deprecated "go.skia.org/infra/go/gitstore_deprecated/bt_gitstore"
"go.skia.org/infra/go/skerr"
"go.skia.org/infra/go/sklog"
"go.skia.org/infra/go/util"
@@ -38,6 +39,9 @@
// The given repoURL serves to identify the repository. Internally it is stored normalized
// via a call to git.NormalizeURL.
func New(ctx context.Context, config *BTConfig, repoURL string) (*BigTableGitStore, error) {
+ if config.TableID == deprecated.DEPRECATED_TABLE_ID {
+ return nil, skerr.Fmt("This implementation of BigTableGitStore cannot be used with deprecated table %q", deprecated.DEPRECATED_TABLE_ID)
+ }
// Create the client.
client, err := bigtable.NewClient(ctx, config.ProjectID, config.InstanceID)
if err != nil {
diff --git a/go/gitstore_deprecated/bt_gitstore/bt_gitstore.go b/go/gitstore_deprecated/bt_gitstore/bt_gitstore.go
new file mode 100644
index 0000000..0637eae
--- /dev/null
+++ b/go/gitstore_deprecated/bt_gitstore/bt_gitstore.go
@@ -0,0 +1,750 @@
+package bt_gitstore
+
+// The bt_gitstore package implements a way to store Git metadata in BigTable for faster retrieval
+// than requiring a local checkout.
+
+import (
+ "bytes"
+ "context"
+ "encoding/binary"
+ "fmt"
+ "hash/crc32"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "time"
+
+ "cloud.google.com/go/bigtable"
+ "go.skia.org/infra/go/git"
+ "go.skia.org/infra/go/gitstore_deprecated"
+ "go.skia.org/infra/go/skerr"
+ "go.skia.org/infra/go/util"
+ "go.skia.org/infra/go/vcsinfo"
+ "golang.org/x/sync/errgroup"
+)
+
+const (
+ DEPRECATED_TABLE_ID = "git-repos"
+ TESTING_TABLE_ID = "test-git-repos"
+)
+
+// BigTableGitStore implements the GitStore interface based on BigTable.
+type BigTableGitStore struct {
+ RepoID int64
+ RepoURL string
+ shards uint32
+ table *bigtable.Table
+}
+
+// New returns an instance of GitStore that uses BigTable as its backend storage.
+// The given repoURL serves to identify the repository. Internally it is stored normalized
+// via a call to git.NormalizeURL.
+func New(ctx context.Context, config *BTConfig, repoURL string) (*BigTableGitStore, error) {
+ if config.TableID != DEPRECATED_TABLE_ID && config.TableID != TESTING_TABLE_ID {
+ return nil, skerr.Fmt("This deprecated implementation of BigTableGitStore only works with table %q, not %q", DEPRECATED_TABLE_ID, config.TableID)
+ }
+ // Create the client.
+ client, err := bigtable.NewClient(ctx, config.ProjectID, config.InstanceID)
+ if err != nil {
+ return nil, skerr.Wrapf(err, "creating bigtable client (project: %s; instance: %s)", config.ProjectID, config.InstanceID)
+ }
+
+ repoURL, err = git.NormalizeURL(repoURL)
+ if err != nil {
+ return nil, skerr.Wrapf(err, "normalizing URL %q", repoURL)
+ }
+
+ shards := config.Shards
+ if shards <= 0 {
+ shards = DefaultShards
+ }
+
+ ret := &BigTableGitStore{
+ table: client.Open(config.TableID),
+ shards: uint32(shards),
+ RepoURL: repoURL,
+ }
+
+ repoInfo, err := ret.loadRepoInfo(ctx, true)
+ if err != nil {
+ return nil, skerr.Wrapf(err, "getting initial repo info for %s (project %s; instance: %s)", repoURL, config.ProjectID, config.InstanceID)
+ }
+ ret.RepoID = repoInfo.ID
+ return ret, nil
+}
+
+const (
+ // Column families.
+ cfBranches = "B"
+ cfCommit = "C"
+ cfTsCommit = "T"
+ cfMeta = "M"
+
+ // meta data columns.
+ colMetaID = "metaID"
+ colMetaIDCounter = "metaIDCounter"
+
+ // Keys of meta data rows.
+ metaVarRepo = "repo"
+ metaVarIDCounter = "idcounter"
+
+ // index commit
+ colHash = "h"
+ colTimestamp = "t"
+
+ // shortcommit
+ colAuthor = "a"
+ colSubject = "s"
+ colParents = "p"
+ colBody = "b"
+ colBranches = "br"
+
+ // Define the row types.
+ typIndex = "i"
+ typTimeStamp = "z"
+ typCommit = "k"
+ typMeta = "!"
+
+ // allCommitsBranch is a pseudo branch name to index all commits in a repo.
+ allCommitsBranch = "@all-commits"
+
+ // getBatchSize is the batchsize for the Get operation. Each call to bigtable is made with maximally
+ // this number of git hashes. This is a conservative number to stay within the 1M request
+ // size limit. Since requests are sharded this will not limit throughput in practice.
+ getBatchSize = 5000
+
+ // writeBatchSize is the number of mutations to write at once. This could be fine tuned for different
+ // row types.
+ writeBatchSize = 1000
+)
+
+var (
+ // Default number of shards used, if not shards provided in BTConfig.
+ DefaultShards = 32
+)
+
+// Put implements the GitStore interface.
+func (b *BigTableGitStore) Put(ctx context.Context, commits []*vcsinfo.LongCommit) error {
+ branch := ""
+
+ if err := b.writeLongCommits(ctx, commits); err != nil {
+ return skerr.Fmt("Error writing long commits: %s", err)
+ }
+
+ // Retrieve the commits in time chronological order and set the index.
+ indexCommits, err := b.RangeByTime(ctx, vcsinfo.MinTime, vcsinfo.MaxTime, branch)
+ if err != err {
+ return skerr.Fmt("Error retrieving commits in order: %s", err)
+ }
+
+ for idx, idxCommit := range indexCommits {
+ idxCommit.Index = idx
+ }
+ return b.writeIndexCommits(ctx, indexCommits, branch)
+}
+
+// Get implements the GitStore interface.
+func (b *BigTableGitStore) Get(ctx context.Context, hashes []string) ([]*vcsinfo.LongCommit, error) {
+ rowNames := make(bigtable.RowList, len(hashes))
+ hashOrder := make(map[string]int, len(hashes))
+ for idx, h := range hashes {
+ rowNames[idx] = b.rowName("", typCommit, h)
+ hashOrder[h] = idx
+ }
+
+ var egroup errgroup.Group
+ tempRet := make([]*vcsinfo.LongCommit, len(hashes))
+ prefix := cfCommit + ":"
+
+ for batchStart := 0; batchStart < len(rowNames); batchStart += getBatchSize {
+ func(bStart, bEnd int) {
+ egroup.Go(func() error {
+ bRowNames := rowNames[bStart:bEnd]
+ batchIdx := int64(bStart - 1)
+
+ err := b.table.ReadRows(ctx, bRowNames, func(row bigtable.Row) bool {
+ longCommit := vcsinfo.NewLongCommit()
+ longCommit.Hash = keyFromRowName(row.Key())
+
+ for _, col := range row[cfCommit] {
+ switch strings.TrimPrefix(col.Column, prefix) {
+ case colHash:
+ longCommit.Timestamp = col.Timestamp.Time().UTC()
+ case colAuthor:
+ longCommit.Author = string(col.Value)
+ case colSubject:
+ longCommit.Subject = string(col.Value)
+ case colParents:
+ if len(col.Value) > 0 {
+ longCommit.Parents = strings.Split(string(col.Value), ":")
+ }
+ case colBody:
+ longCommit.Body = string(col.Value)
+ }
+ }
+ targetIdx := atomic.AddInt64(&batchIdx, 1)
+ tempRet[targetIdx] = longCommit
+ return true
+ })
+ if err != nil {
+ return skerr.Fmt("Error running ReadRows: %s", err)
+ }
+ return nil
+ })
+ }(batchStart, util.MinInt(batchStart+getBatchSize, len(rowNames)))
+ }
+
+ if err := egroup.Wait(); err != nil {
+ return nil, err
+ }
+
+ // Put the results into their places based of the order of the input hashes.
+ ret := make([]*vcsinfo.LongCommit, len(hashes))
+ for _, commit := range tempRet {
+ if commit != nil {
+ targetIdx := hashOrder[commit.Hash]
+ ret[targetIdx] = commit
+ }
+ }
+ return ret, nil
+}
+
+// PutBranches implements the GitStore interface.
+func (b *BigTableGitStore) PutBranches(ctx context.Context, branches map[string]string) error {
+ repoInfo, err := b.loadRepoInfo(ctx, false)
+ if err != nil {
+ return err
+ }
+
+ // Load the commit graph.
+ graph, err := b.GetGraph(ctx)
+ if err != nil {
+ return skerr.Fmt("Error loading graph: %s", err)
+ }
+
+ // updateFromm maps branchName -> branch_pointer_to_old_head to capture the branches we need to update
+ // and whether the branch existed before this update (the value of the map is not nil).
+ updateFrom := make(map[string]*gitstore_deprecated.BranchPointer, len(branches))
+ for branchName, head := range branches {
+ // Assume we start out with a completely fresh branch
+ var oldHeadPtr *gitstore_deprecated.BranchPointer = nil
+ if foundHeadPtr, ok := repoInfo.Branches[branchName]; ok {
+ // We are already done and do not need to update this branch.
+ if foundHeadPtr.Head == head {
+ continue
+ }
+
+ oldHeadNode := graph.GetNode(foundHeadPtr.Head)
+ if oldHeadNode == nil {
+ return skerr.Fmt("Unable to find previous head commit %s in graph", foundHeadPtr.Head)
+ }
+ oldHeadPtr = foundHeadPtr
+ }
+ updateFrom[branchName] = oldHeadPtr
+ }
+
+ var egroup errgroup.Group
+ for branchName, oldHeadPtr := range updateFrom {
+ func(branchName string, oldHeadPtr *gitstore_deprecated.BranchPointer) {
+ egroup.Go(func() error {
+ if branches[branchName] == gitstore_deprecated.DELETE_BRANCH {
+ return b.deleteBranchPointer(ctx, branchName)
+ } else {
+ return b.updateBranch(ctx, branchName, branches[branchName], oldHeadPtr, graph)
+ }
+ })
+ }(branchName, oldHeadPtr)
+ }
+ if err := egroup.Wait(); err != nil {
+ return skerr.Fmt("Error updating branches: %s", err)
+ }
+ return nil
+}
+
+// GetBranches implements the GitStore interface.
+func (b *BigTableGitStore) GetBranches(ctx context.Context) (map[string]*gitstore_deprecated.BranchPointer, error) {
+ repoInfo, err := b.loadRepoInfo(ctx, false)
+ if err != nil {
+ return nil, err
+ }
+
+ // Replace the pseudo branch for all commits with an empty branch name.
+ if found, ok := repoInfo.Branches[allCommitsBranch]; ok {
+ repoInfo.Branches[""] = found
+ delete(repoInfo.Branches, allCommitsBranch)
+ }
+
+ return repoInfo.Branches, nil
+}
+
+// RangeByTime implements the GitStore interface.
+func (b *BigTableGitStore) RangeByTime(ctx context.Context, start, end time.Time, branch string) ([]*vcsinfo.IndexCommit, error) {
+ startTS := sortableTimestamp(start)
+ endTS := sortableTimestamp(end)
+
+ result := newSRTimestampCommits(b.shards)
+ filters := []bigtable.Filter{bigtable.FamilyFilter(cfTsCommit)}
+ err := b.iterShardedRange(ctx, branch, typTimeStamp, startTS, endTS, filters, result)
+ if err != nil {
+ return nil, err
+ }
+
+ return result.Sorted(), nil
+}
+
+// RangeByTime implements the GitStore interface.
+func (b *BigTableGitStore) RangeN(ctx context.Context, startIndex, endIndex int, branch string) ([]*vcsinfo.IndexCommit, error) {
+ startIdx := sortableIndex(startIndex)
+ endIdx := sortableIndex(endIndex)
+
+ result := newSRIndexCommits(b.shards)
+ filters := []bigtable.Filter{bigtable.FamilyFilter(cfCommit)}
+ err := b.iterShardedRange(ctx, branch, typIndex, startIdx, endIdx, filters, result)
+ if err != nil {
+ return nil, err
+ }
+ return result.Sorted(), nil
+}
+
+func (b *BigTableGitStore) loadRepoInfo(ctx context.Context, create bool) (*gitstore_deprecated.RepoInfo, error) {
+ // load repo info
+ rowName := getRepoInfoRowName(b.RepoURL)
+ row, err := b.table.ReadRow(ctx, rowName, bigtable.RowFilter(bigtable.LatestNFilter(1)))
+ if err != nil {
+ return nil, err
+ }
+
+ if row != nil {
+ return extractRepoInfo(row)
+ }
+
+ // If we are not create new repo information, return an error.
+ if !create {
+ return nil, skerr.Fmt("Repo information for %s not found.", b.RepoURL)
+ }
+
+ // Get a new ID from the DB
+ rmw := bigtable.NewReadModifyWrite()
+ rmw.Increment(cfMeta, colMetaIDCounter, 1)
+ row, err = b.table.ApplyReadModifyWrite(ctx, unshardedRowName(typMeta, metaVarIDCounter), rmw)
+ if err != nil {
+ return nil, err
+ }
+
+ // encID contains the big-endian encoded ID
+ encID := []byte(rowMap(row).GetStr(cfMeta, colMetaIDCounter))
+ id := int64(binary.BigEndian.Uint64(encID))
+ mut := bigtable.NewMutation()
+ mut.Set(cfMeta, colMetaID, bigtable.ServerTime, encID)
+ if err := b.table.Apply(ctx, rowName, mut); err != nil {
+ return nil, err
+ }
+
+ b.RepoID = id
+ return &gitstore_deprecated.RepoInfo{
+ RepoURL: b.RepoURL,
+ ID: id,
+ Branches: map[string]*gitstore_deprecated.BranchPointer{},
+ }, nil
+}
+
+// graphColFilter defines a filter (regex) that only keeps columns we need to build the commit graph.
+// Used by GetGraph.
+var graphColFilter = fmt.Sprintf("(%s)", strings.Join([]string{colHash, colParents}, "|"))
+
+// GetGraph implements the GitStore interface.
+func (b *BigTableGitStore) GetGraph(ctx context.Context) (*gitstore_deprecated.CommitGraph, error) {
+ result := newRawNodesResult(b.shards)
+ filters := []bigtable.Filter{
+ bigtable.FamilyFilter(cfCommit),
+ bigtable.ColumnFilter(graphColFilter),
+ }
+ if err := b.iterShardedRange(ctx, "", typCommit, "", "", filters, result); err != nil {
+ return nil, skerr.Fmt("Error getting sharded commits: %s", err)
+ }
+ rawGraph, timeStamps := result.Merge()
+ return gitstore_deprecated.BuildGraph(rawGraph, timeStamps), nil
+}
+
+func (b *BigTableGitStore) getAsIndexCommits(ctx context.Context, ancestors []*gitstore_deprecated.Node, startIdx int) ([]*vcsinfo.IndexCommit, error) {
+ ret := make([]*vcsinfo.IndexCommit, len(ancestors))
+ for idx, commitNode := range ancestors {
+ ret[idx] = &vcsinfo.IndexCommit{
+ Index: startIdx + idx,
+ Hash: commitNode.Hash,
+ Timestamp: commitNode.Timestamp,
+ }
+ }
+ return ret, nil
+}
+
+// updateBranch updates the indices for the named branch and stores the branch pointer. It
+// calculates the branch based on the given commit graph.
+// If there is no previous branch then oldBranchPtr should be nil.
+func (b *BigTableGitStore) updateBranch(ctx context.Context, branchName, newBranchHead string, oldBranchPtr *gitstore_deprecated.BranchPointer, graph *gitstore_deprecated.CommitGraph) error {
+ // Make sure the new head node is in branch.
+ headNode := graph.GetNode(newBranchHead)
+ if headNode == nil {
+ return skerr.Fmt("Head commit %s not found in commit graph", newBranchHead)
+ }
+
+ // If we have not previous branch we set the corresponding values so the logic below still works.
+ if oldBranchPtr == nil {
+ oldBranchPtr = &gitstore_deprecated.BranchPointer{Head: "", Index: 0}
+ }
+
+ branchNodes := graph.DecendantChain(oldBranchPtr.Head, newBranchHead)
+ startIndex := 0
+
+ // If the hash of the first Node matches the hash of the old branchpointer we need to adjust
+ // the initial value of index.
+ if branchNodes[0].Hash == oldBranchPtr.Head {
+ startIndex = oldBranchPtr.Index
+ }
+ indexCommits, err := b.getAsIndexCommits(ctx, branchNodes, startIndex)
+ if err != nil {
+ return skerr.Fmt("Error getting index commits for branch %s: %s", branchName, err)
+ }
+
+ // Write the index commits.
+ if err := b.writeIndexCommits(ctx, indexCommits, branchName); err != nil {
+ return err
+ }
+
+ // Write the index commits of the branch sorted by timestamps.
+ return b.writeTimestampIndex(ctx, indexCommits, branchName)
+}
+
+// putBranchPointer writes the branch pointer (the HEAD of a branch) to the row that stores
+// the repo information. idxCommit is the index commit of the HEAD of the branch.
+func (b *BigTableGitStore) putBranchPointer(ctx context.Context, repoInfoRowName, branchName string, idxCommit *vcsinfo.IndexCommit) error {
+ if branchName == "" {
+ branchName = allCommitsBranch
+ }
+
+ mut := bigtable.NewMutation()
+ now := bigtable.Now()
+ mut.Set(cfBranches, branchName, now, encBranchPointer(idxCommit.Hash, idxCommit.Index))
+ mut.DeleteTimestampRange(cfBranches, branchName, 0, now)
+ return b.table.Apply(ctx, repoInfoRowName, mut)
+}
+
+// deleteBranchPointer deletes the row containing the branch pointer.
+func (b *BigTableGitStore) deleteBranchPointer(ctx context.Context, branchName string) error {
+ mut := bigtable.NewMutation()
+ mut.DeleteCellsInColumn(cfBranches, branchName)
+ return b.table.Apply(ctx, getRepoInfoRowName(b.RepoURL), mut)
+}
+
+// writeLongCommits writes the LongCommits to the store idempotently.
+func (b *BigTableGitStore) writeLongCommits(ctx context.Context, commits []*vcsinfo.LongCommit) error {
+ branch := ""
+
+ // Assemble the mutations.
+ nMutations := len(commits)
+ rowNames := make([]string, 0, nMutations)
+ mutations := make([]*bigtable.Mutation, 0, nMutations)
+
+ // Assemble the records for the Timestamp index.
+ tsIdxCommits := make([]*vcsinfo.IndexCommit, 0, nMutations)
+
+ for _, commit := range commits {
+ // Add the long commits
+ rowNames = append(rowNames, b.rowName(branch, typCommit, commit.Hash))
+ mutations = append(mutations, b.getCommitMutation(commit))
+
+ tsIdxCommits = append(tsIdxCommits, &vcsinfo.IndexCommit{
+ Hash: commit.Hash,
+ Timestamp: commit.Timestamp,
+ })
+ }
+
+ if err := b.applyBulkBatched(ctx, rowNames, mutations, writeBatchSize); err != nil {
+ return skerr.Fmt("Error writing commits: %s", err)
+ }
+ return b.writeTimestampIndex(ctx, tsIdxCommits, branch)
+}
+
+// applyBulkBatched writes the given rowNames/mutation pairs to bigtable in batches that are
+// maximally of size 'batchSize'. The batches are written in parallel.
+func (b *BigTableGitStore) applyBulkBatched(ctx context.Context, rowNames []string, mutations []*bigtable.Mutation, batchSize int) error {
+ var egroup errgroup.Group
+ err := util.ChunkIter(len(rowNames), batchSize, func(chunkStart, chunkEnd int) error {
+ egroup.Go(func() error {
+ rowNames := rowNames[chunkStart:chunkEnd]
+ mutations := mutations[chunkStart:chunkEnd]
+ errs, err := b.table.ApplyBulk(ctx, rowNames, mutations)
+ if err != nil {
+ return skerr.Fmt("Error writing batch [%d:%d]: %s", chunkStart, chunkEnd, err)
+ }
+ if errs != nil {
+ return skerr.Fmt("Error writing some portions of batch [%d:%d]: %s", chunkStart, chunkEnd, errs)
+ }
+ return nil
+ })
+ return nil
+ })
+ if err != nil {
+ return skerr.Fmt("Error running ChunkIter: %s", err)
+ }
+ return egroup.Wait()
+}
+
+// writeIndexCommits writes the given index commits keyed by their indices for the given branch.
+func (b *BigTableGitStore) writeIndexCommits(ctx context.Context, indexCommits []*vcsinfo.IndexCommit, branch string) error {
+ idxRowNames := make([]string, 0, len(indexCommits))
+ idxMutations := make([]*bigtable.Mutation, 0, len(indexCommits))
+
+ for idx, commit := range indexCommits {
+ sIndex := sortableIndex(indexCommits[idx].Index)
+ idxRowNames = append(idxRowNames, b.rowName(branch, typIndex, sIndex))
+ idxMutations = append(idxMutations, b.simpleMutation(cfCommit, commit.Timestamp, [2]string{colHash, commit.Hash}))
+ }
+
+ if err := b.applyBulkBatched(ctx, idxRowNames, idxMutations, writeBatchSize); err != nil {
+ return skerr.Fmt("Error writing indices: %s", err)
+ }
+ return b.putBranchPointer(ctx, getRepoInfoRowName(b.RepoURL), branch, indexCommits[len(indexCommits)-1])
+}
+
+// writeTimestampIndexCommits writes the given index commits keyed by their timestamp for the
+// given branch.
+func (b *BigTableGitStore) writeTimestampIndex(ctx context.Context, indexCommits []*vcsinfo.IndexCommit, branch string) error {
+ nMutations := len(indexCommits)
+ tsRowNames := make([]string, 0, nMutations)
+ tsMutations := make([]*bigtable.Mutation, 0, nMutations)
+
+ for _, commit := range indexCommits {
+ tsRowName := b.rowName(branch, typTimeStamp, sortableTimestamp(commit.Timestamp))
+ tsRowNames = append(tsRowNames, tsRowName)
+ tsMutations = append(tsMutations, b.simpleMutation(cfTsCommit, commit.Timestamp, [][2]string{
+ {commit.Hash, sortableIndex(commit.Index)},
+ }...))
+ }
+
+ // Write the timestamped index.
+ if err := b.applyBulkBatched(ctx, tsRowNames, tsMutations, writeBatchSize); err != nil {
+ return skerr.Fmt("Error writing timestamps: %s", err)
+ }
+ return nil
+}
+
+// iterShardedRange iterates the keys in the half open interval [startKey, endKey) across all
+// shards triggering as many queries as there are shards. If endKey is empty, then startKey is
+// used to generate a prefix and a Prefix scan is performed.
+// The results of the query are added to the instance of shardedResults.
+func (b *BigTableGitStore) iterShardedRange(ctx context.Context, branch, rowType, startKey, endKey string, filters []bigtable.Filter, result shardedResults) error {
+ var egroup errgroup.Group
+
+ // Query all shards in parallel.
+ for shard := uint32(0); shard < b.shards; shard++ {
+ func(shard uint32) {
+ egroup.Go(func() error {
+ defer result.Finish(shard)
+
+ var rr bigtable.RowRange
+ // Treat the startKey as part of a prefix and do a prefix scan.
+ if endKey == "" {
+ rowPrefix := b.shardedRowName(shard, branch, rowType, startKey)
+ rr = bigtable.PrefixRange(rowPrefix)
+ } else {
+ // Derive the start and end row names.
+ rStart := b.shardedRowName(shard, branch, rowType, startKey)
+ rEnd := b.shardedRowName(shard, branch, rowType, endKey)
+ rr = bigtable.NewRange(rStart, rEnd)
+ }
+
+ var addErr error
+ err := b.table.ReadRows(ctx, rr, func(row bigtable.Row) bool {
+ addErr = result.Add(shard, row)
+ return addErr == nil
+ }, filtersToReadOptions(filters)...)
+ if err != nil {
+ return err
+ }
+ return addErr
+ })
+ }(shard)
+ }
+
+ if err := egroup.Wait(); err != nil {
+ return err
+ }
+ return nil
+}
+
+// simpleMutation assembles a simple mutation consisting of a column family, a timestamp and a
+// set of column/value pairs. The timestamp is applied to all column/pairs.
+func (b *BigTableGitStore) simpleMutation(cfFam string, timeStamp time.Time, colValPairs ...[2]string) *bigtable.Mutation {
+ ts := bigtable.Time(timeStamp.UTC())
+ ret := bigtable.NewMutation()
+ for _, pair := range colValPairs {
+ ret.Set(cfFam, pair[0], ts, []byte(pair[1]))
+ }
+ return ret
+}
+
+// getCommitMutation gets the mutation to write a long commit. Since the timestamp is set to the
+// timestamp of the commit this is idempotent.
+func (b *BigTableGitStore) getCommitMutation(commit *vcsinfo.LongCommit) *bigtable.Mutation {
+ ts := bigtable.Time(commit.Timestamp.UTC())
+ ret := bigtable.NewMutation()
+ ret.Set(cfCommit, colHash, ts, []byte(commit.Hash))
+ ret.Set(cfCommit, colAuthor, ts, []byte(commit.Author))
+ ret.Set(cfCommit, colSubject, ts, []byte(commit.Subject))
+ ret.Set(cfCommit, colParents, ts, []byte(strings.Join(commit.Parents, ":")))
+ ret.Set(cfCommit, colBody, ts, []byte(commit.Body))
+ return ret
+}
+
+// rowName returns that BT rowName based on the tuple: (branch,rowType,Key).
+// It also derives a unique shard for the given tuple and generates the complete rowName.
+func (b *BigTableGitStore) rowName(branch string, rowType string, key string) string {
+ return b.shardedRowName(crc32.ChecksumIEEE([]byte(key))%b.shards, branch, rowType, key)
+}
+
+// shardedRowName returns the row name from (shard, branch, rowType, key) this is useful
+// when we want to generate a specific row name with a defined shard.
+func (b *BigTableGitStore) shardedRowName(shard uint32, branch, rowType, key string) string {
+ return fmt.Sprintf("%02d:%04d:%s:%s:%s", shard, b.RepoID, branch, rowType, key)
+}
+
+// unshardedRowName concatenates parts without prefixing any sharding information. This is for
+// row types that are not sharded.
+func unshardedRowName(parts ...string) string {
+ return strings.Join(parts, ":")
+}
+
+// getRepoInfoRowName returns the name of the row where the repo meta data is stored based on the repoURL.
+func getRepoInfoRowName(repoURL string) string {
+ return unshardedRowName(typMeta, metaVarRepo, repoURL)
+}
+
+// getRepoInfoRowNamePrefix returns the row Name prefix to scan all rows containing repo meta data.
+func getRepoInfoRowNamePrefix() string {
+ return unshardedRowName(typMeta, metaVarRepo)
+}
+
+// extractRepoInfo extract the repo meta data information from a read row.
+func extractRepoInfo(row bigtable.Row) (*gitstore_deprecated.RepoInfo, error) {
+ rm := rowMap(row)
+
+ // Extract the branch info.
+ branchInfo := rm.GetStrMap(cfBranches)
+ branches := make(map[string]*gitstore_deprecated.BranchPointer, len(branchInfo))
+ var err error
+ for name, b := range branchInfo {
+ branches[name], err = decBranchPointer([]byte(b))
+ if err != nil {
+ return nil, skerr.Fmt("Error decoding branch pointer: %s", err)
+ }
+ }
+
+ // Extract the repo ID.
+ idBytes := []byte(rm.GetStr(cfMeta, colMetaID))
+ if len(idBytes) != 8 {
+ return nil, skerr.Fmt("Error: Got id that's not exactly 8 bytes: '%x': %s", idBytes, err)
+ }
+
+ ret := &gitstore_deprecated.RepoInfo{
+ RepoURL: keyFromRowName(row.Key()),
+ ID: int64(binary.BigEndian.Uint64(idBytes)),
+ Branches: branches,
+ }
+ return ret, nil
+}
+
+// filtersToReadOptions converts a list of filters to []bigtable.ReadOption. It will inject
+// a ChainFilters(...) instance if multiple filters are defined. By returning a slice we are
+// able to pass 0 - n filter to a ReadRows call.
+func filtersToReadOptions(filters []bigtable.Filter) []bigtable.ReadOption {
+ if len(filters) == 0 {
+ return []bigtable.ReadOption{}
+ }
+
+ // If there is more than one filter then chain them.
+ if len(filters) > 1 {
+ filters = []bigtable.Filter{bigtable.ChainFilters(filters...)}
+ }
+
+ return []bigtable.ReadOption{bigtable.RowFilter(filters[0])}
+}
+
+// keyFromRowName assumes that key segments are separated by ':' and the last segment is the
+// actual key we are interested in.
+func keyFromRowName(rowName string) string {
+ parts := strings.Split(rowName, ":")
+ return parts[len(parts)-1]
+}
+
+// sortableTimestamp returns a timestamp as a string (in seconds) that can used as a key in BT.
+func sortableTimestamp(ts time.Time) string {
+ // Convert the timestamp in seconds to a string that is sortable and limit it to 10 digits.
+ // That is the equivalent of valid timestamps up to November 2286.
+ return fmt.Sprintf("%010d", util.MinInt64(9999999999, ts.Unix()))
+}
+
+// sortableIndex returns an index as a string that can be used as a key in BT.
+func sortableIndex(index int) string {
+ return fmt.Sprintf("%08d", util.MinInt(99999999, index))
+}
+
+// parseIndex parses an index previously generated with sortableIndex.
+func parseIndex(indexStr string) int {
+ ret, err := strconv.ParseInt(indexStr, 10, 64)
+ if err != nil {
+ return -1
+ }
+ return int(ret)
+}
+
+// encBranchPointer converts a hash and an index into a string where the parts are separated by ':'
+func encBranchPointer(hash string, index int) []byte {
+ idxBuf := make([]byte, 8)
+ binary.BigEndian.PutUint64(idxBuf, uint64(index))
+ return []byte(hash + ":" + string(idxBuf))
+}
+
+// decBranchPointer a string previously generated by encBranchPointer into a BranchPointer,
+// containing a head and index.
+func decBranchPointer(encPointer []byte) (*gitstore_deprecated.BranchPointer, error) {
+ parts := bytes.SplitN(encPointer, []byte(":"), 2)
+ if len(parts) != 2 || len(parts[1]) != 8 {
+ return nil, skerr.Fmt("Received wrong branch pointer. Expected format <commit>:<big_endian_64_bit>")
+ }
+ return &gitstore_deprecated.BranchPointer{
+ Head: string(parts[0]),
+ Index: int(binary.BigEndian.Uint64([]byte(parts[1]))),
+ }, nil
+}
+
+// rowMap is a helper type that wraps around a bigtable.Row and allows to extract columns and their
+// values.
+type rowMap bigtable.Row
+
+// GetStr extracts that value of colFam:colName as a string from the row. If it doesn't exist it
+// returns ""
+func (r rowMap) GetStr(colFamName, colName string) string {
+ prefix := colFamName + ":"
+ for _, col := range r[colFamName] {
+ if strings.TrimPrefix(col.Column, prefix) == colName {
+ return string(col.Value)
+ }
+ }
+ return ""
+}
+
+// GetStrMap extracts a map[string]string from the row that maps columns -> values for the given
+// column family.
+func (r rowMap) GetStrMap(colFamName string) map[string]string {
+ prefix := colFamName + ":"
+ ret := make(map[string]string, len(r[colFamName]))
+ for _, col := range r[colFamName] {
+ trimmed := strings.TrimPrefix(col.Column, prefix)
+ ret[trimmed] = string(col.Value)
+ }
+ return ret
+}
+
+// Make sure BigTableGitStore fulfills the GitStore interface
+var _ gitstore_deprecated.GitStore = (*BigTableGitStore)(nil)
diff --git a/go/gitstore_deprecated/bt_gitstore/bt_gitstore_test.go b/go/gitstore_deprecated/bt_gitstore/bt_gitstore_test.go
new file mode 100644
index 0000000..1ffd438
--- /dev/null
+++ b/go/gitstore_deprecated/bt_gitstore/bt_gitstore_test.go
@@ -0,0 +1,144 @@
+package bt_gitstore_test
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "sort"
+ "testing"
+
+ assert "github.com/stretchr/testify/require"
+ "go.skia.org/infra/go/common"
+ "go.skia.org/infra/go/git"
+ "go.skia.org/infra/go/git/gitinfo"
+ "go.skia.org/infra/go/gitstore_deprecated"
+ gitstore_testutils "go.skia.org/infra/go/gitstore_deprecated/testutils"
+ "go.skia.org/infra/go/testutils/unittest"
+ "go.skia.org/infra/go/timer"
+ "go.skia.org/infra/go/vcsinfo"
+ vcs_testutils "go.skia.org/infra/go/vcsinfo/testutils"
+)
+
+const (
+ localRepoURL = "https://example.com/local.git"
+)
+
+// This test uses a checkout of a repo (can be really any repo) in /tmp/skia.
+// If not present, syncs skia.git to /tmp/skia first.
+func TestGitStoreSkiaRepo(t *testing.T) {
+ unittest.ManualTest(t)
+ t.Skip()
+ skiaRepoDir := filepath.Join(os.TempDir(), "skia")
+ if _, err := os.Stat(skiaRepoDir); os.IsNotExist(err) {
+ _, err = git.NewRepo(context.Background(), common.REPO_SKIA, os.TempDir())
+ assert.NoError(t, err)
+ } else if err != nil {
+ assert.FailNow(t, err.Error())
+ }
+ testGitStore(t, common.REPO_SKIA, skiaRepoDir, true)
+}
+
+func TestGitStoreLocalRepo(t *testing.T) {
+ unittest.LargeTest(t)
+
+ repoDir, cleanup := vcs_testutils.InitTempRepo()
+ defer cleanup()
+ testGitStore(t, localRepoURL, repoDir, true)
+}
+
+func testGitStore(t *testing.T, repoURL, repoDir string, freshLoad bool) {
+ // Get all commits that have been added to the gitstore.
+ _, longCommits, gitStore := gitstore_testutils.SetupAndLoadBTGitStore(t, repoURL, repoDir, freshLoad)
+
+ // Sort long commits they way they are sorted by BigTable (by timestamp/hash)
+ sort.Slice(longCommits, func(i, j int) bool {
+ tsI := longCommits[i].Timestamp.Unix()
+ tsJ := longCommits[j].Timestamp.Unix()
+ return (tsI < tsJ) || ((tsI == tsJ) && (longCommits[i].Hash < longCommits[j].Hash))
+ })
+ indexCommits := make([]*vcsinfo.IndexCommit, len(longCommits))
+ for idx, commit := range longCommits {
+ indexCommits[idx] = &vcsinfo.IndexCommit{
+ Index: idx,
+ Hash: commit.Hash,
+ Timestamp: commit.Timestamp,
+ }
+ }
+
+ // Find all the commits in the repository independent of branches.
+ foundIndexCommits, foundLongCommits := getFromRange(t, gitStore, 0, len(longCommits), "")
+ assert.Equal(t, len(indexCommits), len(foundIndexCommits))
+ assert.Equal(t, len(longCommits), len(foundLongCommits))
+
+ // Make sure they match what we found.
+ for idx, expected := range longCommits {
+ foundLongCommits[idx].Branches = expected.Branches
+ assert.Equal(t, expected, foundLongCommits[idx])
+ }
+
+ // Verify that the branches from the GitStore match what's in the checkout.
+ branchNames, branchCommits := getBranchCommits(t, repoDir)
+ for branchIdx, branchName := range branchNames {
+ expHashes := branchCommits[branchIdx]
+ foundIndexCommits, foundLongCommits := getFromRange(t, gitStore, 0, len(longCommits), branchName)
+ assert.Equal(t, len(expHashes), len(foundIndexCommits))
+ assert.Equal(t, len(expHashes), len(foundLongCommits))
+ expIdx := len(expHashes) - 1
+ for idx := len(foundIndexCommits) - 1; idx >= 0; idx-- {
+ expHash := expHashes[expIdx]
+ assert.Equal(t, foundIndexCommits[idx].Hash, foundLongCommits[idx].Hash)
+ assert.Equal(t, expHash, foundIndexCommits[idx].Hash)
+ expIdx--
+ }
+ }
+}
+
+func getBranchCommits(t *testing.T, repoDir string) ([]string, [][]string) {
+ ctx := context.TODO()
+ vcs, err := gitinfo.NewGitInfo(ctx, repoDir, false, true)
+ assert.NoError(t, err)
+
+ branches, err := vcs.GetBranches(ctx)
+ assert.NoError(t, err)
+
+ branchNames := make([]string, 0, len(branches))
+ branchCommits := make([][]string, 0, len(branches))
+ for _, branch := range branches {
+ // if strings.Contains(branch.Name, "m62") {
+ // continue
+ // }
+ branchNames = append(branchNames, branch.Name)
+ indexCommits, err := gitinfo.GetBranchCommits(ctx, repoDir, branch.Name)
+ assert.NoError(t, err)
+ commitHashes := make([]string, len(indexCommits))
+ for idx, idxCommit := range indexCommits {
+ commitHashes[idx] = idxCommit.Hash
+ }
+ branchCommits = append(branchCommits, commitHashes)
+ }
+
+ return branchNames, branchCommits
+}
+
+func getFromRange(t *testing.T, gitStore gitstore_deprecated.GitStore, startIdx, endIdx int, branchName string) ([]*vcsinfo.IndexCommit, []*vcsinfo.LongCommit) {
+ ctx := context.TODO()
+
+ tQuery := timer.New(fmt.Sprintf("RangeN %d - %d commits from branch %q", startIdx, endIdx, branchName))
+ foundIndexCommits, err := gitStore.RangeN(ctx, startIdx, endIdx, branchName)
+ assert.NoError(t, err)
+ tQuery.Stop()
+
+ hashes := make([]string, 0, len(foundIndexCommits))
+ for _, commit := range foundIndexCommits {
+ hashes = append(hashes, commit.Hash)
+ }
+
+ tLongCommits := timer.New(fmt.Sprintf("Get %d LongCommits from branch %q", len(hashes), branchName))
+ foundLongCommits, err := gitStore.Get(ctx, hashes)
+ assert.NoError(t, err)
+ assert.Equal(t, len(foundIndexCommits), len(foundLongCommits))
+ tLongCommits.Stop()
+
+ return foundIndexCommits, foundLongCommits
+}
diff --git a/go/gitstore_deprecated/bt_gitstore/helpers.go b/go/gitstore_deprecated/bt_gitstore/helpers.go
new file mode 100644
index 0000000..6166650
--- /dev/null
+++ b/go/gitstore_deprecated/bt_gitstore/helpers.go
@@ -0,0 +1,104 @@
+package bt_gitstore
+
+// This file has some helper functions for working with BigTable.
+
+import (
+ "context"
+ "strconv"
+
+ "cloud.google.com/go/bigtable"
+ "go.skia.org/infra/go/bt"
+ "go.skia.org/infra/go/git/repograph"
+ "go.skia.org/infra/go/gitstore_deprecated"
+ "go.skia.org/infra/go/skerr"
+)
+
+// InitBT initializes the BT instance for the given configuration. It uses the default way
+// to get auth information from the environment and must be called with an account that has
+// admin rights.
+func InitBT(conf *BTConfig) error {
+ return bt.InitBigtable(conf.ProjectID, conf.InstanceID, conf.TableID, []string{
+ cfCommit,
+ cfMeta,
+ cfBranches,
+ cfTsCommit,
+ })
+}
+
+// AllRepos returns a map of all repos contained in given BigTable project/instance/table.
+// It returns map[normalized_URL]RepoInfo.
+func AllRepos(ctx context.Context, conf *BTConfig) (map[string]*gitstore_deprecated.RepoInfo, error) {
+ // Create the client.
+ client, err := bigtable.NewClient(ctx, conf.ProjectID, conf.InstanceID)
+ if err != nil {
+ return nil, skerr.Fmt("Error creating bigtable client: %s", err)
+ }
+
+ table := client.Open(conf.TableID)
+ rowNamesPrefix := getRepoInfoRowNamePrefix()
+ ret := map[string]*gitstore_deprecated.RepoInfo{}
+ var readRowErr error = nil
+ err = table.ReadRows(ctx, bigtable.PrefixRange(rowNamesPrefix), func(row bigtable.Row) bool {
+ if readRowErr != nil {
+ return false
+ }
+
+ var repoInfo *gitstore_deprecated.RepoInfo
+ repoInfo, readRowErr = extractRepoInfo(row)
+ if readRowErr != nil {
+ return false
+ }
+ // save the repo info and set the all-commits branch.
+ ret[repoInfo.RepoURL] = repoInfo
+ if found, ok := repoInfo.Branches[allCommitsBranch]; ok {
+ repoInfo.Branches[""] = found
+ delete(repoInfo.Branches, allCommitsBranch)
+ }
+
+ return true
+ }, bigtable.RowFilter(bigtable.LatestNFilter(1)))
+
+ if err != nil {
+ return nil, skerr.Fmt("Error reading repo info: %s", err)
+ }
+ return ret, nil
+}
+
+// RepoURLFromID retrieves the URL of a repository by its corresponding numeric ID.
+// If a repository with the given ID can be found it will be returned and the second return value
+// is true. In any other case "" and false will be returned.
+func RepoURLFromID(ctx context.Context, conf *BTConfig, repoIDStr string) (string, bool) {
+ id, err := strconv.ParseInt(repoIDStr, 10, 64)
+ if err != nil {
+ return "", false
+ }
+
+ repoInfos, err := AllRepos(ctx, conf)
+ if err != nil {
+ return "", false
+ }
+
+ for repoURL, info := range repoInfos {
+ if info.ID == id {
+ return repoURL, false
+ }
+ }
+ return "", false
+}
+
+// NewGitStoreMap returns a Map instance with Graphs for the given GitStores.
+func NewBTGitStoreMap(ctx context.Context, repoUrls []string, btConf *BTConfig) (repograph.Map, error) {
+ rv := make(map[string]*repograph.Graph, len(repoUrls))
+ for _, repoUrl := range repoUrls {
+ gs, err := New(ctx, btConf, repoUrl)
+ if err != nil {
+ return nil, skerr.Wrapf(err, "Failed to create GitStore for %s", repoUrl)
+ }
+ graph, err := gitstore_deprecated.GetRepoGraph(ctx, gs)
+ if err != nil {
+ return nil, skerr.Wrapf(err, "Failed to create Graph from GitStore for %s", repoUrl)
+ }
+ rv[repoUrl] = graph
+ }
+ return rv, nil
+}
diff --git a/go/gitstore_deprecated/bt_gitstore/repo_impl_test.go b/go/gitstore_deprecated/bt_gitstore/repo_impl_test.go
new file mode 100644
index 0000000..cd060fd
--- /dev/null
+++ b/go/gitstore_deprecated/bt_gitstore/repo_impl_test.go
@@ -0,0 +1,155 @@
+package bt_gitstore
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ assert "github.com/stretchr/testify/require"
+ "go.skia.org/infra/go/git"
+ "go.skia.org/infra/go/git/repograph"
+ repograph_shared_tests "go.skia.org/infra/go/git/repograph/shared_tests"
+ git_testutils "go.skia.org/infra/go/git/testutils"
+ "go.skia.org/infra/go/gitstore_deprecated"
+ "go.skia.org/infra/go/testutils/unittest"
+ "go.skia.org/infra/go/vcsinfo"
+)
+
+// gitstoreRefresher is an implementation of repograph_shared_tests.RepoImplRefresher
+// used for testing a GitStore.
+type gitstoreRefresher struct {
+ gs gitstore_deprecated.GitStore
+ repo git.GitDir
+ t *testing.T
+}
+
+func newGitstoreUpdater(t *testing.T, gs gitstore_deprecated.GitStore, gb *git_testutils.GitBuilder) repograph_shared_tests.RepoImplRefresher {
+ return &gitstoreRefresher{
+ gs: gs,
+ repo: git.GitDir(gb.Dir()),
+ t: t,
+ }
+}
+
+func (u *gitstoreRefresher) Refresh(commits ...*vcsinfo.LongCommit) {
+ ctx := context.Background()
+ // Add the commits.
+ assert.NoError(u.t, u.gs.Put(ctx, commits))
+ branches, err := u.repo.Branches(ctx)
+ assert.NoError(u.t, err)
+ putBranches := make(map[string]string, len(branches))
+ for _, branch := range branches {
+ putBranches[branch.Name] = branch.Head
+ }
+ oldBranches, err := u.gs.GetBranches(ctx)
+ assert.NoError(u.t, err)
+ for name := range oldBranches {
+ if name == "" {
+ continue
+ }
+ if _, ok := putBranches[name]; !ok {
+ putBranches[name] = gitstore_deprecated.DELETE_BRANCH
+ }
+ }
+ assert.NoError(u.t, u.gs.PutBranches(ctx, putBranches))
+
+ // Wait for GitStore to be up to date.
+ for {
+ time.Sleep(10 * time.Millisecond)
+ actual, err := u.gs.GetBranches(ctx)
+ assert.NoError(u.t, err)
+ allMatch := true
+ for _, expectBranch := range branches {
+ actualBranch, ok := actual[expectBranch.Name]
+ if !ok || actualBranch.Head != expectBranch.Head {
+ allMatch = false
+ break
+ }
+ }
+ for name := range actual {
+ if _, ok := putBranches[name]; name != "" && !ok {
+ allMatch = false
+ break
+ }
+ }
+ if allMatch {
+ break
+ }
+ }
+}
+
+// setupGitStore performs common setup for GitStore based Graphs.
+func setupGitStore(t *testing.T) (context.Context, *git_testutils.GitBuilder, *repograph.Graph, repograph_shared_tests.RepoImplRefresher, func()) {
+ ctx, g, cleanup := repograph_shared_tests.CommonSetup(t)
+
+ btConf := &BTConfig{
+ ProjectID: "fake-project",
+ InstanceID: fmt.Sprintf("fake-instance-%s", uuid.New()),
+ TableID: TESTING_TABLE_ID,
+ }
+ assert.NoError(t, InitBT(btConf))
+ gs, err := New(context.Background(), btConf, g.RepoUrl())
+ assert.NoError(t, err)
+ ud := newGitstoreUpdater(t, gs, g)
+ repo, err := gitstore_deprecated.GetRepoGraph(ctx, gs)
+ assert.NoError(t, err)
+ return ctx, g, repo, ud, cleanup
+}
+
+func TestGraphWellFormedBTGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestGraphWellFormed(t, ctx, g, repo, ud)
+}
+
+func TestRecurseBTGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestRecurse(t, ctx, g, repo, ud)
+}
+
+func TestRecurseAllBranchesBTGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestRecurseAllBranches(t, ctx, g, repo, ud)
+}
+
+func TestLogLinearBTGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestLogLinear(t, ctx, g, repo, ud)
+}
+
+func TestUpdateHistoryChangedBTGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestUpdateHistoryChanged(t, ctx, g, repo, ud)
+}
+
+func TestUpdateAndReturnCommitDiffsBTGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestUpdateAndReturnCommitDiffs(t, ctx, g, repo, ud)
+}
+
+func TestRevListBTGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestRevList(t, ctx, g, repo, ud)
+}
+
+func TestBranchMembershipBTGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestBranchMembership(t, ctx, g, repo, ud)
+}
diff --git a/go/gitstore_deprecated/bt_gitstore/sharded_result.go b/go/gitstore_deprecated/bt_gitstore/sharded_result.go
new file mode 100644
index 0000000..e7acbe5
--- /dev/null
+++ b/go/gitstore_deprecated/bt_gitstore/sharded_result.go
@@ -0,0 +1,183 @@
+package bt_gitstore
+
+import (
+ "sort"
+ "strings"
+ "sync/atomic"
+ "time"
+
+ "cloud.google.com/go/bigtable"
+ multierror "github.com/hashicorp/go-multierror"
+ "go.skia.org/infra/go/skerr"
+ "go.skia.org/infra/go/vcsinfo"
+)
+
+// shardedResults collects the results of call to iterShardedRange.
+type shardedResults interface {
+ // Add adds the result of a shard.
+ Add(shard uint32, row bigtable.Row) error
+
+ // Finish indicates that the shard is done processing its results.
+ Finish(shard uint32)
+}
+
+// srIndexCommits implements the shardedResults interface for collecting results that are vcsinfo.IndexCommits
+type srIndexCommits struct {
+ errs []*multierror.Error
+ results [][]*vcsinfo.IndexCommit
+ retSize int64
+}
+
+// create a new instance to collect IndexCommits.
+func newSRIndexCommits(shards uint32) *srIndexCommits {
+ return &srIndexCommits{
+ results: make([][]*vcsinfo.IndexCommit, shards),
+ errs: make([]*multierror.Error, shards),
+ }
+}
+
+// Add implements the shardedResults interface.
+func (s *srIndexCommits) Add(shard uint32, row bigtable.Row) error {
+ idx := parseIndex(keyFromRowName(row.Key()))
+ if idx < 0 {
+ return skerr.Fmt("Unable to parse index key %q. Invalid index", row.Key())
+ }
+
+ var hash string
+ var timeStamp bigtable.Timestamp
+ prefix := cfCommit + ":"
+ for _, col := range row[cfCommit] {
+ if strings.TrimPrefix(col.Column, prefix) == colHash {
+ hash = string(col.Value)
+ timeStamp = col.Timestamp
+ }
+ }
+
+ s.results[shard] = append(s.results[shard], &vcsinfo.IndexCommit{
+ Index: idx,
+ Hash: hash,
+ Timestamp: timeStamp.Time().UTC(),
+ })
+ return nil
+}
+
+// Finish implements the shardedResults interface.
+func (s *srIndexCommits) Finish(shard uint32) {
+ atomic.AddInt64(&s.retSize, int64(len(s.results)))
+}
+
+// Sorted returns the resulting IndexCommits by Index->TimeStamp->Hash.
+// Using the hash ensures results with identical timestamps are sorted stably.
+func (s *srIndexCommits) Sorted() []*vcsinfo.IndexCommit {
+ // Concatenate the shard results into a single output and sort it.
+ ret := make([]*vcsinfo.IndexCommit, 0, s.retSize)
+ for _, sr := range s.results {
+ ret = append(ret, sr...)
+ }
+ sort.Slice(ret, func(i, j int) bool {
+ return ret[i].Index < ret[j].Index ||
+ ((ret[i].Index == ret[j].Index) && ret[i].Timestamp.Before(ret[j].Timestamp)) ||
+ ((ret[i].Index == ret[j].Index) && ret[i].Timestamp.Equal(ret[j].Timestamp) &&
+ (ret[i].Hash < ret[j].Hash))
+ })
+ return ret
+}
+
+// srTimestampCommits is an adaptation of srIndexCommits that extracts a different
+// column family from a timestamp-based index. Otherwise it is identical.
+type srTimestampCommits struct {
+ *srIndexCommits
+}
+
+// newSRTimestampCommits creates a new instance to receive timestamp based IndexCommits.
+func newSRTimestampCommits(shards uint32) *srTimestampCommits {
+ return &srTimestampCommits{
+ srIndexCommits: newSRIndexCommits(shards),
+ }
+}
+
+// Add implements the shardedResults interface and overrides the Add function in srIndexCommits.
+func (s *srTimestampCommits) Add(shard uint32, row bigtable.Row) error {
+ prefix := cfTsCommit + ":"
+ for _, col := range row[cfTsCommit] {
+ hash := strings.TrimPrefix(col.Column, prefix)
+ timeStamp := col.Timestamp
+
+ // Parse the index
+ idxStr := string(col.Value)
+ idx := parseIndex(idxStr)
+ if idx < 0 {
+ return skerr.Fmt("Unable to parse index key %q. Invalid index", idxStr)
+ }
+
+ s.results[shard] = append(s.results[shard], &vcsinfo.IndexCommit{
+ Hash: hash,
+ Timestamp: timeStamp.Time().UTC(),
+ Index: idx,
+ })
+ }
+ return nil
+}
+
+// rawNodesResults implements the shardedResults interface for rows necessary to build the
+// commit graph. It collects slices of strings, where the first string is the commit hash and
+// all subsequent strings are its parents.
+type rawNodesResult struct {
+ errs []*multierror.Error
+ results [][][]string
+ timeStamps [][]time.Time
+ retSize int64
+}
+
+// newRawNodesResult creates a new
+func newRawNodesResult(shards uint32) *rawNodesResult {
+ return &rawNodesResult{
+ results: make([][][]string, shards),
+ timeStamps: make([][]time.Time, shards),
+ errs: make([]*multierror.Error, shards),
+ }
+}
+
+// rawNodeColPrefix is the prefix of column names.
+const rawNodeColPrefix = cfCommit + ":"
+
+// Add implements the shardedResults interface.
+func (r *rawNodesResult) Add(shard uint32, row bigtable.Row) error {
+ var commitHash string
+ var parents []string
+ var timeStamp bigtable.Timestamp
+ for _, col := range row[cfCommit] {
+ switch strings.TrimPrefix(col.Column, rawNodeColPrefix) {
+ case colHash:
+ commitHash = string(col.Value)
+ timeStamp = col.Timestamp
+ case colParents:
+ if len(col.Value) > 0 {
+ parents = strings.Split(string(col.Value), ":")
+ }
+ }
+ }
+ hp := make([]string, 0, 1+len(parents))
+ hp = append(hp, commitHash)
+ hp = append(hp, parents...)
+ r.results[shard] = append(r.results[shard], hp)
+ r.timeStamps[shard] = append(r.timeStamps[shard], timeStamp.Time())
+ return nil
+}
+
+// Add implements the shardedResults interface.
+func (r *rawNodesResult) Finish(shard uint32) {
+ atomic.AddInt64(&r.retSize, int64(len(r.results)))
+}
+
+// Merge merges the results of all shards into one slice of string slices. These are unordered
+// since structure will be imposed by building a CommitGraph from it.
+func (r *rawNodesResult) Merge() ([][]string, []time.Time) {
+ ret := make([][]string, 0, r.retSize)
+ timeStamps := make([]time.Time, 0, r.retSize)
+ for idx, shardResults := range r.results {
+ ret = append(ret, shardResults...)
+ timeStamps = append(timeStamps, r.timeStamps[idx]...)
+ }
+ return ret, timeStamps
+}
diff --git a/go/gitstore_deprecated/bt_gitstore/types.go b/go/gitstore_deprecated/bt_gitstore/types.go
new file mode 100644
index 0000000..0acdb28
--- /dev/null
+++ b/go/gitstore_deprecated/bt_gitstore/types.go
@@ -0,0 +1,9 @@
+package bt_gitstore
+
+// BTConfig contains the BigTable configuration to define where the repo should be stored.
+type BTConfig struct {
+ ProjectID string
+ InstanceID string
+ TableID string
+ Shards int
+}
diff --git a/go/gitstore_deprecated/graph.go b/go/gitstore_deprecated/graph.go
new file mode 100644
index 0000000..71d1518
--- /dev/null
+++ b/go/gitstore_deprecated/graph.go
@@ -0,0 +1,85 @@
+package gitstore_deprecated
+
+import (
+ "time"
+)
+
+const (
+ // initialGraphSize is the assumed starting number of commits in a repository. Just so we
+ // don't start with an empty data structure when we building or traversing the graph.
+ initialGraphSize = 100000
+)
+
+// BuildGraph takes a rawGraph (a slice where each element contains a commit hash followed by its
+// parents) and returns an instance of CommitGraph.
+// TODO(kjlubick,borenet): can this be replaced by go/git/repograph.Graph ?
+func BuildGraph(rawGraph [][]string, timeStamps []time.Time) *CommitGraph {
+ nodeMap := make(map[string]*Node, len(rawGraph))
+ for idx, rawNode := range rawGraph {
+ hash := rawNode[0]
+ nodeMap[hash] = &Node{
+ Hash: hash,
+ Parents: make([]*Node, len(rawNode)-1),
+ Timestamp: timeStamps[idx],
+ }
+ }
+
+ for _, rawNode := range rawGraph {
+ for idx, p := range rawNode[1:] {
+ nodeMap[rawNode[0]].Parents[idx] = nodeMap[p]
+ }
+ }
+
+ return &CommitGraph{
+ Nodes: nodeMap,
+ }
+}
+
+// CommitGraph contains commits as Nodes that are connected and thus can be traversed.
+// Given a graph a client can retrieve a specific node and traverse the graph like this:
+// // First-parent traversal
+// node := graph.GetNode(someHash)
+// for node != nil {
+// // so something with the commit
+// node = node.Parents[0]
+// }
+//
+type CommitGraph struct {
+ Nodes map[string]*Node
+}
+
+// GetNode returns the node in the graph that corresponds to the given hash or nil
+func (c *CommitGraph) GetNode(hash string) *Node {
+ return c.Nodes[hash]
+}
+
+// Node is a node in the commit graph that contains the commit hash, a timestamp and pointers to
+// its parent nodes. The first parent is the immediate parent in the same branch (like in Git).
+type Node struct {
+ Hash string
+ Timestamp time.Time
+ Parents []*Node
+}
+
+// DescendantChain returns all nodes in the commit graph in the range of
+// (firstAncestor, lastDescendant) where the parameters are both commit hashes.
+// 'firstAncestor' can be "" in which case it will return all ancestors of 'lastDescendant'.
+// 'lastDescendant' must not be empty and must exist in graph or this will panic.
+func (g *CommitGraph) DecendantChain(firstAncestor, lastDescendant string) []*Node {
+ curr := g.Nodes[lastDescendant]
+ ret := make([]*Node, 0, len(g.Nodes))
+ for curr != nil {
+ ret = append(ret, curr)
+ if (len(curr.Parents) == 0) || (curr.Hash == firstAncestor) {
+ break
+ }
+ curr = curr.Parents[0]
+ }
+
+ // Reverse the result
+ for idx := 0; idx < len(ret)/2; idx++ {
+ rightIdx := len(ret) - 1 - idx
+ ret[idx], ret[rightIdx] = ret[rightIdx], ret[idx]
+ }
+ return ret
+}
diff --git a/go/gitstore_deprecated/mocks/GitStore.go b/go/gitstore_deprecated/mocks/GitStore.go
new file mode 100644
index 0000000..c3998bd
--- /dev/null
+++ b/go/gitstore_deprecated/mocks/GitStore.go
@@ -0,0 +1,162 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+package mocks
+
+import (
+ context "context"
+
+ mock "github.com/stretchr/testify/mock"
+ gitstore_deprecated "go.skia.org/infra/go/gitstore_deprecated"
+
+ time "time"
+
+ vcsinfo "go.skia.org/infra/go/vcsinfo"
+)
+
+// GitStore is an autogenerated mock type for the GitStore type
+type GitStore struct {
+ mock.Mock
+}
+
+// Get provides a mock function with given fields: ctx, hashes
+func (_m *GitStore) Get(ctx context.Context, hashes []string) ([]*vcsinfo.LongCommit, error) {
+ ret := _m.Called(ctx, hashes)
+
+ var r0 []*vcsinfo.LongCommit
+ if rf, ok := ret.Get(0).(func(context.Context, []string) []*vcsinfo.LongCommit); ok {
+ r0 = rf(ctx, hashes)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).([]*vcsinfo.LongCommit)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, []string) error); ok {
+ r1 = rf(ctx, hashes)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// GetBranches provides a mock function with given fields: ctx
+func (_m *GitStore) GetBranches(ctx context.Context) (map[string]*gitstore_deprecated.BranchPointer, error) {
+ ret := _m.Called(ctx)
+
+ var r0 map[string]*gitstore_deprecated.BranchPointer
+ if rf, ok := ret.Get(0).(func(context.Context) map[string]*gitstore_deprecated.BranchPointer); ok {
+ r0 = rf(ctx)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).(map[string]*gitstore_deprecated.BranchPointer)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+ r1 = rf(ctx)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// GetGraph provides a mock function with given fields: ctx
+func (_m *GitStore) GetGraph(ctx context.Context) (*gitstore_deprecated.CommitGraph, error) {
+ ret := _m.Called(ctx)
+
+ var r0 *gitstore_deprecated.CommitGraph
+ if rf, ok := ret.Get(0).(func(context.Context) *gitstore_deprecated.CommitGraph); ok {
+ r0 = rf(ctx)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).(*gitstore_deprecated.CommitGraph)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+ r1 = rf(ctx)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// Put provides a mock function with given fields: ctx, commits
+func (_m *GitStore) Put(ctx context.Context, commits []*vcsinfo.LongCommit) error {
+ ret := _m.Called(ctx, commits)
+
+ var r0 error
+ if rf, ok := ret.Get(0).(func(context.Context, []*vcsinfo.LongCommit) error); ok {
+ r0 = rf(ctx, commits)
+ } else {
+ r0 = ret.Error(0)
+ }
+
+ return r0
+}
+
+// PutBranches provides a mock function with given fields: ctx, branches
+func (_m *GitStore) PutBranches(ctx context.Context, branches map[string]string) error {
+ ret := _m.Called(ctx, branches)
+
+ var r0 error
+ if rf, ok := ret.Get(0).(func(context.Context, map[string]string) error); ok {
+ r0 = rf(ctx, branches)
+ } else {
+ r0 = ret.Error(0)
+ }
+
+ return r0
+}
+
+// RangeByTime provides a mock function with given fields: ctx, start, end, branch
+func (_m *GitStore) RangeByTime(ctx context.Context, start time.Time, end time.Time, branch string) ([]*vcsinfo.IndexCommit, error) {
+ ret := _m.Called(ctx, start, end, branch)
+
+ var r0 []*vcsinfo.IndexCommit
+ if rf, ok := ret.Get(0).(func(context.Context, time.Time, time.Time, string) []*vcsinfo.IndexCommit); ok {
+ r0 = rf(ctx, start, end, branch)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).([]*vcsinfo.IndexCommit)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, time.Time, time.Time, string) error); ok {
+ r1 = rf(ctx, start, end, branch)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// RangeN provides a mock function with given fields: ctx, startIndex, endIndex, branch
+func (_m *GitStore) RangeN(ctx context.Context, startIndex int, endIndex int, branch string) ([]*vcsinfo.IndexCommit, error) {
+ ret := _m.Called(ctx, startIndex, endIndex, branch)
+
+ var r0 []*vcsinfo.IndexCommit
+ if rf, ok := ret.Get(0).(func(context.Context, int, int, string) []*vcsinfo.IndexCommit); ok {
+ r0 = rf(ctx, startIndex, endIndex, branch)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).([]*vcsinfo.IndexCommit)
+ }
+ }
+
+ var r1 error
+ if rf, ok := ret.Get(1).(func(context.Context, int, int, string) error); ok {
+ r1 = rf(ctx, startIndex, endIndex, branch)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
diff --git a/go/gitstore_deprecated/mocks/generate.go b/go/gitstore_deprecated/mocks/generate.go
new file mode 100644
index 0000000..38a73f2
--- /dev/null
+++ b/go/gitstore_deprecated/mocks/generate.go
@@ -0,0 +1,3 @@
+package mocks
+
+//go:generate mockery -name GitStore -dir ../ -output .
diff --git a/go/gitstore_deprecated/repo_impl.go b/go/gitstore_deprecated/repo_impl.go
new file mode 100644
index 0000000..6ed33e6
--- /dev/null
+++ b/go/gitstore_deprecated/repo_impl.go
@@ -0,0 +1,107 @@
+package gitstore_deprecated
+
+import (
+ "context"
+ "time"
+
+ "go.skia.org/infra/go/git"
+ "go.skia.org/infra/go/git/repograph"
+ "go.skia.org/infra/go/skerr"
+ "go.skia.org/infra/go/sklog"
+ "go.skia.org/infra/go/vcsinfo"
+)
+
+// GetRepoGraph returns *repograph.Graph backed by the given GitStore.
+func GetRepoGraph(ctx context.Context, gs GitStore) (*repograph.Graph, error) {
+ ri, err := NewGitStoreRepoImpl(ctx, gs)
+ if err != nil {
+ return nil, err
+ }
+ return repograph.NewWithRepoImpl(ctx, ri)
+}
+
+// gitStoreRepoImpl is an implementation of the repograph.RepoImpl interface
+// which uses a GitStore to interact with a git repo.
+type gitStoreRepoImpl struct {
+ *repograph.MemCacheRepoImpl
+ gs GitStore
+ lastUpdate time.Time
+}
+
+// NewGitStoreRepoImpl returns a repograph.RepoImpl instance which uses the
+// given GitStore.
+func NewGitStoreRepoImpl(ctx context.Context, gs GitStore) (repograph.RepoImpl, error) {
+ rv := &gitStoreRepoImpl{
+ MemCacheRepoImpl: repograph.NewMemCacheRepoImpl(nil, nil),
+ gs: gs,
+ }
+ if err := rv.Update(ctx); err != nil {
+ return nil, err
+ }
+ return rv, nil
+}
+
+// See documentation for repograph.RepoImpl interface.
+func (g *gitStoreRepoImpl) Update(ctx context.Context) error {
+ branchPtrs, err := g.gs.GetBranches(ctx)
+ if err != nil {
+ return skerr.Wrapf(err, "Failed to read branches from GitStore")
+ }
+ branches := make([]*git.Branch, 0, len(branchPtrs))
+ for name, ptr := range branchPtrs {
+ if name != "" {
+ branches = append(branches, &git.Branch{
+ Name: name,
+ Head: ptr.Head,
+ })
+ }
+ }
+
+ from := g.lastUpdate.Add(-10 * time.Minute)
+ now := time.Now()
+ to := now.Add(time.Second)
+ indexCommits, err := g.gs.RangeByTime(ctx, from, to, "")
+ if err != nil {
+ return skerr.Wrapf(err, "Failed to read IndexCommits from GitStore")
+ }
+ hashes := make([]string, 0, len(indexCommits))
+ for _, c := range indexCommits {
+ hashes = append(hashes, c.Hash)
+ }
+ commits, err := g.gs.Get(ctx, hashes)
+ if err != nil {
+ return skerr.Wrapf(err, "Failed to read LongCommits from GitStore")
+ }
+ commitsMap := make(map[string]*vcsinfo.LongCommit, len(commits))
+ for _, c := range commits {
+ commitsMap[c.Hash] = c
+ }
+
+ g.lastUpdate = now
+ g.BranchList = branches
+ g.Commits = commitsMap
+ return nil
+}
+
+// See documentation for repograph.RepoImpl interface.
+func (g *gitStoreRepoImpl) Details(ctx context.Context, hash string) (*vcsinfo.LongCommit, error) {
+ d, err := g.MemCacheRepoImpl.Details(ctx, hash)
+ if err == nil {
+ return d, nil
+ }
+ // Update() should have pre-fetched all of the commits for us, so we
+ // shouldn't have hit this code. Log a warning and fall back to
+ // retrieving the commit from GitStore.
+ sklog.Warningf("Commit %q not found in results; performing explicit lookup.", hash)
+ got, err := g.gs.Get(ctx, []string{hash})
+ if err != nil {
+ return nil, skerr.Wrapf(err, "Failed to read commit %s from GitStore", hash)
+ }
+ for _, c := range got {
+ if c == nil {
+ return nil, skerr.Fmt("Commit %s not present in GitStore.", hash)
+ }
+ g.Commits[c.Hash] = c
+ }
+ return got[0], nil
+}
diff --git a/go/gitstore_deprecated/repo_impl_test.go b/go/gitstore_deprecated/repo_impl_test.go
new file mode 100644
index 0000000..28fad31
--- /dev/null
+++ b/go/gitstore_deprecated/repo_impl_test.go
@@ -0,0 +1,202 @@
+package gitstore_deprecated
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ assert "github.com/stretchr/testify/require"
+ "go.skia.org/infra/go/git"
+ "go.skia.org/infra/go/git/repograph"
+ repograph_shared_tests "go.skia.org/infra/go/git/repograph/shared_tests"
+ git_testutils "go.skia.org/infra/go/git/testutils"
+ "go.skia.org/infra/go/skerr"
+ "go.skia.org/infra/go/testutils/unittest"
+ "go.skia.org/infra/go/vcsinfo"
+)
+
+// testGitStore is an in-memory GitStore implementation used for testing
+// gitStoreRepoImpl. It is incomplete and does not support indexed commits
+// at all.
+type testGitStore struct {
+ // commits maps commit hashes to LongCommits.
+ commits map[string]*vcsinfo.LongCommit
+ // branches maps branch names to the commit hashes of their HEADs.
+ branches map[string]string
+}
+
+// See documentation for GitStore interface.
+func (gs *testGitStore) Put(ctx context.Context, commits []*vcsinfo.LongCommit) error {
+ for _, c := range commits {
+ gs.commits[c.Hash] = c
+ }
+ return nil
+}
+
+// See documentation for GitStore interface.
+func (gs *testGitStore) Get(ctx context.Context, hashes []string) ([]*vcsinfo.LongCommit, error) {
+ rv := make([]*vcsinfo.LongCommit, len(hashes))
+ for idx, h := range hashes {
+ rv[idx] = gs.commits[h]
+ }
+ return rv, nil
+}
+
+// See documentation for GitStore interface.
+func (gs *testGitStore) PutBranches(ctx context.Context, branches map[string]string) error {
+ for name, hash := range branches {
+ if hash == DELETE_BRANCH {
+ delete(gs.branches, name)
+ } else {
+ gs.branches[name] = hash
+ }
+ }
+ return nil
+}
+
+// See documentation for GitStore interface.
+func (gs *testGitStore) GetBranches(ctx context.Context) (map[string]*BranchPointer, error) {
+ rv := make(map[string]*BranchPointer, len(gs.branches))
+ for name, hash := range gs.branches {
+ rv[name] = &BranchPointer{
+ Head: hash,
+ }
+ }
+ return rv, nil
+}
+
+// See documentation for GitStore interface.
+func (gs *testGitStore) RangeN(ctx context.Context, startIndex, endIndex int, branch string) ([]*vcsinfo.IndexCommit, error) {
+ return nil, skerr.Fmt("RangeN not implemented for testGitStore.")
+}
+
+// See documentation for GitStore interface.
+func (gs *testGitStore) RangeByTime(ctx context.Context, start, end time.Time, branch string) ([]*vcsinfo.IndexCommit, error) {
+ if branch != "" {
+ return nil, skerr.Fmt("RangeByTime not implemented for single branches in testGitStore.")
+ }
+ rv := make([]*vcsinfo.IndexCommit, 0, len(gs.commits))
+ for _, c := range gs.commits {
+ if c.Timestamp.Before(end) && !c.Timestamp.Before(start) {
+ rv = append(rv, &vcsinfo.IndexCommit{
+ Hash: c.Hash,
+ Timestamp: c.Timestamp,
+ })
+ }
+ }
+ return rv, nil
+}
+
+// See documentation for GitStore interface.
+func (gs *testGitStore) GetGraph(ctx context.Context) (*CommitGraph, error) {
+ return nil, skerr.Fmt("GetGraph not implemented for testGitStore.")
+}
+
+// gitstoreRefresher is an implementation of repograph_shared_tests.RepoImplRefresher
+// used for testing a GitStore.
+type gitstoreRefresher struct {
+ gs GitStore
+ repo git.GitDir
+ t *testing.T
+}
+
+func newGitstoreUpdater(t *testing.T, gs GitStore, gb *git_testutils.GitBuilder) repograph_shared_tests.RepoImplRefresher {
+ return &gitstoreRefresher{
+ gs: gs,
+ repo: git.GitDir(gb.Dir()),
+ t: t,
+ }
+}
+
+func (u *gitstoreRefresher) Refresh(commits ...*vcsinfo.LongCommit) {
+ ctx := context.Background()
+ // Add the commits.
+ assert.NoError(u.t, u.gs.Put(ctx, commits))
+ branches, err := u.repo.Branches(ctx)
+ assert.NoError(u.t, err)
+ putBranches := make(map[string]string, len(branches))
+ for _, branch := range branches {
+ putBranches[branch.Name] = branch.Head
+ }
+ oldBranches, err := u.gs.GetBranches(ctx)
+ assert.NoError(u.t, err)
+ for name := range oldBranches {
+ if name == "" {
+ continue
+ }
+ if _, ok := putBranches[name]; !ok {
+ putBranches[name] = DELETE_BRANCH
+ }
+ }
+ assert.NoError(u.t, u.gs.PutBranches(ctx, putBranches))
+}
+
+// setupGitStore performs common setup for GitStore based Graphs.
+func setupGitStore(t *testing.T) (context.Context, *git_testutils.GitBuilder, *repograph.Graph, repograph_shared_tests.RepoImplRefresher, func()) {
+ ctx, g, cleanup := repograph_shared_tests.CommonSetup(t)
+
+ gs := &testGitStore{
+ commits: map[string]*vcsinfo.LongCommit{},
+ branches: map[string]string{},
+ }
+ ud := newGitstoreUpdater(t, gs, g)
+ repo, err := GetRepoGraph(ctx, gs)
+ assert.NoError(t, err)
+ return ctx, g, repo, ud, cleanup
+}
+
+func TestGraphWellFormedGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestGraphWellFormed(t, ctx, g, repo, ud)
+}
+
+func TestRecurseGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestRecurse(t, ctx, g, repo, ud)
+}
+
+func TestRecurseAllBranchesGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestRecurseAllBranches(t, ctx, g, repo, ud)
+}
+
+func TestLogLinearGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestLogLinear(t, ctx, g, repo, ud)
+}
+
+func TestUpdateHistoryChangedGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestUpdateHistoryChanged(t, ctx, g, repo, ud)
+}
+
+func TestUpdateAndReturnCommitDiffsGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestUpdateAndReturnCommitDiffs(t, ctx, g, repo, ud)
+}
+
+func TestRevListGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestRevList(t, ctx, g, repo, ud)
+}
+
+func TestBranchMembershipGitStore(t *testing.T) {
+ unittest.LargeTest(t)
+ ctx, g, repo, ud, cleanup := setupGitStore(t)
+ defer cleanup()
+ repograph_shared_tests.TestBranchMembership(t, ctx, g, repo, ud)
+}
diff --git a/go/gitstore_deprecated/testutils/bt_testutils.go b/go/gitstore_deprecated/testutils/bt_testutils.go
new file mode 100644
index 0000000..5fe5496
--- /dev/null
+++ b/go/gitstore_deprecated/testutils/bt_testutils.go
@@ -0,0 +1,141 @@
+package testutils
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "go.skia.org/infra/go/bt"
+ "go.skia.org/infra/go/git/gitinfo"
+ "go.skia.org/infra/go/gitstore_deprecated"
+ "go.skia.org/infra/go/gitstore_deprecated/bt_gitstore"
+ "go.skia.org/infra/go/sklog"
+ "go.skia.org/infra/go/sktest"
+ "go.skia.org/infra/go/timer"
+ "go.skia.org/infra/go/vcsinfo"
+)
+
+const (
+ concurrentWrites = 1000
+)
+
+var (
+ btConf = &bt_gitstore.BTConfig{
+ ProjectID: "skia-public",
+ InstanceID: "staging",
+ TableID: bt_gitstore.TESTING_TABLE_ID,
+ }
+)
+
+// SetupAndLoadBTGitStore loads the Git repo in repoDir into the Gitstore. It assumes that the
+// repo is checked out. repoURL is only used for creating the GitStore.
+func SetupAndLoadBTGitStore(t sktest.TestingT, repoURL, repoDir string, load bool) ([]*vcsinfo.IndexCommit, []*vcsinfo.LongCommit, *bt_gitstore.BigTableGitStore) {
+ if load {
+ // Delete the tables.
+ assert.NoError(t, bt.DeleteTables(btConf.ProjectID, btConf.InstanceID, btConf.TableID))
+ assert.NoError(t, bt_gitstore.InitBT(btConf))
+ }
+
+ // Get a new gitstore.
+ gitStore, err := bt_gitstore.New(context.TODO(), btConf, repoURL)
+ assert.NoError(t, err)
+
+ // Get the commits of the last ~20 years and load them into the GitStore
+ timeDelta := time.Hour * 24 * 365 * 20
+ tLoad := timer.New("Loading all commits")
+ indexCommits, longCommits := loadGitRepo(t, repoDir, gitStore, timeDelta, load)
+ tLoad.Stop()
+
+ return indexCommits, longCommits, gitStore
+}
+
+type commitInfo struct {
+ commits []*vcsinfo.LongCommit
+ indices []int
+}
+
+func loadGitRepo(t sktest.TestingT, repoDir string, gitStore gitstore_deprecated.GitStore, timeDelta time.Duration, load bool) ([]*vcsinfo.IndexCommit, []*vcsinfo.LongCommit) {
+ ctx := context.TODO()
+ commitCh := make(chan *commitInfo, 10)
+ indexCommits, branches := iterateCommits(t, repoDir, concurrentWrites, commitCh, timeDelta)
+ longCommits := make([]*vcsinfo.LongCommit, 0, len(indexCommits))
+
+ for ci := range commitCh {
+ assert.True(t, len(ci.commits) > 0)
+ longCommits = append(longCommits, ci.commits...)
+ if load {
+ // Add the commits.
+ putT := timer.New(fmt.Sprintf("Put %d commits.", len(ci.commits)))
+ assert.NoError(t, gitStore.Put(ctx, ci.commits))
+ putT.Stop()
+ }
+ }
+
+ for name, head := range branches {
+ details, err := gitStore.Get(ctx, []string{head})
+ assert.NoError(t, err)
+ if details[0] == nil {
+ delete(branches, name)
+ } else {
+ sklog.Infof("Found branches: %40s : %s", name, head)
+ }
+ }
+
+ if load {
+ assert.NoError(t, gitStore.PutBranches(ctx, branches))
+ }
+ return indexCommits, longCommits
+}
+
+// iterateCommits returns batches of commits via a channel. It returns all IndexCommits within
+// the given timeDelta.
+func iterateCommits(t sktest.TestingT, repoDir string, maxCount int, targetCh chan<- *commitInfo, timeDelta time.Duration) ([]*vcsinfo.IndexCommit, map[string]string) {
+ gitInfo, err := gitinfo.NewGitInfo(context.TODO(), repoDir, false, true)
+ assert.NoError(t, err)
+
+ start := time.Now().Add(-timeDelta)
+ indexCommits := gitInfo.Range(start, time.Now().Add(time.Hour))
+ sklog.Infof("Index commits: %d", len(indexCommits))
+
+ gitBranches, err := gitInfo.GetBranches(context.TODO())
+ assert.NoError(t, err)
+
+ // Keep track of the branches.
+ branches := map[string]string{}
+ for _, gb := range gitBranches {
+ branches[gb.Name] = gb.Head
+ }
+
+ go func() {
+ ctx := context.TODO()
+ longCommits := make([]*vcsinfo.LongCommit, 0, maxCount)
+ indices := make([]int, 0, maxCount)
+ retIdx := 0
+ batchTimer := timer.New("Fetching commits starting with 0")
+ for idx, indexCommit := range indexCommits {
+ commitDetails, err := gitInfo.Details(ctx, indexCommit.Hash, false)
+ if err != nil {
+ sklog.Fatalf("Error fetching commits: %s", err)
+ }
+ longCommits = append(longCommits, commitDetails)
+ indices = append(indices, indexCommit.Index)
+ if len(longCommits) >= maxCount || idx == (len(indexCommits)-1) {
+ batchTimer.Stop()
+ targetCh <- &commitInfo{
+ commits: longCommits,
+ indices: indices,
+ }
+ longCommits = make([]*vcsinfo.LongCommit, 0, maxCount)
+ indices = make([]int, 0, maxCount)
+ retIdx = 0
+ batchTimer = timer.New(fmt.Sprintf("Fetching commits starting with %d", idx))
+ } else {
+ retIdx++
+ }
+ }
+ batchTimer.Stop()
+ close(targetCh)
+ }()
+ return indexCommits, branches
+}
diff --git a/go/gitstore_deprecated/types.go b/go/gitstore_deprecated/types.go
new file mode 100644
index 0000000..9edb020
--- /dev/null
+++ b/go/gitstore_deprecated/types.go
@@ -0,0 +1,93 @@
+package gitstore_deprecated
+
+import (
+ "context"
+ "time"
+
+ "go.skia.org/infra/go/vcsinfo"
+)
+
+const (
+ // DELETE_BRANCH is a placeholder which can be used as a value in the
+ // branch map passed to GitStore.PutBranches to signify that the branch
+ // should be deleted.
+ DELETE_BRANCH = "@DELETE"
+
+ ALL_BRANCHES = ""
+)
+
+// GitStore defines the functions of a data store for Git metadata (aka vcsinfo.LongCommit)
+// Each GitStore instance relates to one repository that is defined in the constructor of the
+// implementation.
+type GitStore interface {
+ // Put stores the given commits. They can be retrieved in order of timestamps by using
+ // RangeByTime or RangeN (no topological ordering).
+ Put(ctx context.Context, commits []*vcsinfo.LongCommit) error
+
+ // Get retrieves the commits identified by 'hashes'. The return value will always have the
+ // length of the input value and the results will line up by index. If a commit does not exist
+ // the corresponding entry in the result is nil.
+ // The function will only return an error if the retrieval operation (the I/O) fails, not
+ // if the given hashes do not exist or are invalid.
+ Get(ctx context.Context, hashes []string) ([]*vcsinfo.LongCommit, error)
+
+ // PutBranches updates branches in the repository. It writes indices for the branches so they
+ // can be retrieved via RangeN and RangeByTime. These are ordered in toplogical order with only
+ // first-parents included.
+ // 'branches' maps branchName -> commit_hash to indicate the head of a branch. The store then
+ // calculates the commits of the branch and updates the indices accordingly. Branches which
+ // already exist in the GitStore are not removed if not present in 'branches'; if DELETE_BRANCH
+ // string is used as the head instead of a commit hash, then the branch is removed.
+ PutBranches(ctx context.Context, branches map[string]string) error
+
+ // GetBranches returns the current branches in the store. It maps[branchName]->BranchPointer.
+ // A BranchPointer contains the HEAD commit and also the Index of the HEAD commit, which is
+ // usually the total number of commits in the branch minus 1.
+ GetBranches(ctx context.Context) (map[string]*BranchPointer, error)
+
+ // RangeN returns all commits in the half open index range [startIndex, endIndex).
+ // Thus not including endIndex. It returns the commits in the given branch sorted in ascending
+ // order by Index and the commits are topologically sorted only including first-parent commits.
+ RangeN(ctx context.Context, startIndex, endIndex int, branch string) ([]*vcsinfo.IndexCommit, error)
+
+ // RangeByTime returns all commits in the half open time range [start, end). Thus not
+ // including commits at 'end' time.
+ // Caveat: The returned results will match the requested range, but will be sorted by Index.
+ // So if the timestamps within a commit are not in order they will be unordered in the results.
+ RangeByTime(ctx context.Context, start, end time.Time, branch string) ([]*vcsinfo.IndexCommit, error)
+
+ // GetGraph returns the commit graph of the entire repository.
+ GetGraph(ctx context.Context) (*CommitGraph, error)
+}
+
+// GitStoreBased is an interface that tags an object as being based on GitStore and the
+// underlying GitStore instance can be retrieved. e.g.
+//
+// if gsb, ok := someInstance.(GitStoreBased); ok {
+// gitStore := gsb.GetGitStore()
+// ... do something with gitStore
+// }
+//
+type GitStoreBased interface {
+ // GetGitStore returns the underlying GitStore instances.
+ GetGitStore() GitStore
+}
+
+// BranchPointer captures the HEAD of a branch and the index of that commit.
+type BranchPointer struct {
+ Head string
+ Index int
+}
+
+// RepoInfo contains information about one repo in the GitStore.
+type RepoInfo struct {
+ // Numeric id of the repo. This is unique within all repos in a BT table. This ID is uniquely
+ // assigned whenever a new repo is added.
+ ID int64
+
+ // RepoURL contains the URL of the repo as returned by git.NormalizeURL(...).
+ RepoURL string
+
+ // Branches contain all the branches in the repo, mapping branch_name -> branch_pointer.
+ Branches map[string]*BranchPointer
+}
diff --git a/go/ingestion/helpers.go b/go/ingestion/helpers.go
index 15d77cf..e0ecf8e 100644
--- a/go/ingestion/helpers.go
+++ b/go/ingestion/helpers.go
@@ -20,7 +20,7 @@
"go.skia.org/infra/go/gcs"
"go.skia.org/infra/go/git/gitinfo"
"go.skia.org/infra/go/gitiles"
- "go.skia.org/infra/go/gitstore/bt_gitstore"
+ "go.skia.org/infra/go/gitstore_deprecated/bt_gitstore"
"go.skia.org/infra/go/sharedconfig"
"go.skia.org/infra/go/skerr"
"go.skia.org/infra/go/sklog"
diff --git a/go/vcsinfo/bt_vcs/bt_vcs.go b/go/vcsinfo/bt_vcs/bt_vcs.go
index e0a014d..73aac6e 100644
--- a/go/vcsinfo/bt_vcs/bt_vcs.go
+++ b/go/vcsinfo/bt_vcs/bt_vcs.go
@@ -12,8 +12,8 @@
"go.skia.org/infra/go/depot_tools"
"go.skia.org/infra/go/eventbus"
"go.skia.org/infra/go/gitiles"
- "go.skia.org/infra/go/gitstore"
- "go.skia.org/infra/go/gitstore/bt_gitstore"
+ "go.skia.org/infra/go/gitstore_deprecated"
+ "go.skia.org/infra/go/gitstore_deprecated/bt_gitstore"
"go.skia.org/infra/go/skerr"
"go.skia.org/infra/go/sklog"
"go.skia.org/infra/go/util"
@@ -38,13 +38,13 @@
// BigTableVCS implements the vcsinfo.VCS interface based on a BT-backed GitStore.
type BigTableVCS struct {
- gitStore gitstore.GitStore
+ gitStore gitstore_deprecated.GitStore
repo *gitiles.Repo
defaultBranch string
secondaryVCS vcsinfo.VCS
secondaryExtractor depot_tools.DEPSExtractor
- branchInfo *gitstore.BranchPointer
+ branchInfo *gitstore_deprecated.BranchPointer
// This mutex protects detailsCache and indexCommits
mutex sync.RWMutex
@@ -57,7 +57,7 @@
// gittiles.Repo to retrieve files. Each instance provides an interface to one branch.
// If defaultBranch is gitstore.ALL_BRANCHES all commits in the repository are considered.
// The instances of gitiles.Repo is only used to fetch files.
-func New(ctx context.Context, gitStore gitstore.GitStore, defaultBranch string, repo *gitiles.Repo) (*BigTableVCS, error) {
+func New(ctx context.Context, gitStore gitstore_deprecated.GitStore, defaultBranch string, repo *gitiles.Repo) (*BigTableVCS, error) {
if gitStore == nil {
return nil, errors.New("Cannot have nil gitStore")
}
@@ -91,7 +91,7 @@
// Check if we need to pull across all branches.
targetBranch := b.defaultBranch
if allBranches {
- targetBranch = gitstore.ALL_BRANCHES
+ targetBranch = gitstore_deprecated.ALL_BRANCHES
}
// Simulate a pull by fetching the latest head of the target branch.
@@ -283,12 +283,12 @@
// getBranchInfo determines which branches contain the given commit 'c'.
// This function can potentially spawn a huge number of goroutines (one per branch).
-func (b *BigTableVCS) getBranchInfo(ctx context.Context, c *vcsinfo.LongCommit, allBranches map[string]*gitstore.BranchPointer) (map[string]bool, error) {
+func (b *BigTableVCS) getBranchInfo(ctx context.Context, c *vcsinfo.LongCommit, allBranches map[string]*gitstore_deprecated.BranchPointer) (map[string]bool, error) {
ret := make(map[string]bool, len(allBranches))
var mutex sync.Mutex
var egroup errgroup.Group
for branchName := range allBranches {
- if branchName != gitstore.ALL_BRANCHES {
+ if branchName != gitstore_deprecated.ALL_BRANCHES {
func(branchName string) {
egroup.Go(func() error {
// Since we cannot look up a commit in a branch directly we query for all commits that
@@ -401,8 +401,8 @@
return foundCommit, nil
}
-// GetGitStore implements the gitstore.GitStoreBased interface
-func (b *BigTableVCS) GetGitStore() gitstore.GitStore {
+// GetGitStore implements the gitstore_deprecated.GitStoreBased interface
+func (b *BigTableVCS) GetGitStore() gitstore_deprecated.GitStore {
return b.gitStore
}
diff --git a/go/vcsinfo/bt_vcs/bt_vcs_test.go b/go/vcsinfo/bt_vcs/bt_vcs_test.go
index a3018d4..cc5c4a7 100644
--- a/go/vcsinfo/bt_vcs/bt_vcs_test.go
+++ b/go/vcsinfo/bt_vcs/bt_vcs_test.go
@@ -2,18 +2,16 @@
import (
"context"
- "io/ioutil"
"testing"
"time"
"github.com/stretchr/testify/mock"
assert "github.com/stretchr/testify/require"
"go.skia.org/infra/go/gitiles"
- "go.skia.org/infra/go/gitstore"
- "go.skia.org/infra/go/gitstore/mocks"
- gs_testutils "go.skia.org/infra/go/gitstore/testutils"
+ "go.skia.org/infra/go/gitstore_deprecated"
+ "go.skia.org/infra/go/gitstore_deprecated/mocks"
+ gs_testutils "go.skia.org/infra/go/gitstore_deprecated/testutils"
"go.skia.org/infra/go/testutils/unittest"
- "go.skia.org/infra/go/util"
"go.skia.org/infra/go/vcsinfo"
vcs_testutils "go.skia.org/infra/go/vcsinfo/testutils"
)
@@ -39,14 +37,14 @@
func TestBranchInfo(t *testing.T) {
unittest.LargeTest(t)
- vcs, gitStore, cleanup := setupVCSLocalRepo(t, gitstore.ALL_BRANCHES)
+ vcs, gitStore, cleanup := setupVCSLocalRepo(t, gitstore_deprecated.ALL_BRANCHES)
defer cleanup()
branchPointers, err := gitStore.GetBranches(context.Background())
assert.NoError(t, err)
branches := []string{}
for branchName := range branchPointers {
- if branchName != gitstore.ALL_BRANCHES {
+ if branchName != gitstore_deprecated.ALL_BRANCHES {
branches = append(branches, branchName)
}
}
@@ -260,18 +258,12 @@
}
// setupVCSLocalRepo loads the test repo into a new GitStore and returns an instance of vcsinfo.VCS.
-func setupVCSLocalRepo(t *testing.T, branch string) (vcsinfo.VCS, gitstore.GitStore, func()) {
+func setupVCSLocalRepo(t *testing.T, branch string) (vcsinfo.VCS, gitstore_deprecated.GitStore, func()) {
repoDir, cleanup := vcs_testutils.InitTempRepo()
- wd, err := ioutil.TempDir("", "")
+ _, _, btgs := gs_testutils.SetupAndLoadBTGitStore(t, localRepoURL, repoDir, true)
+ vcs, err := New(context.Background(), btgs, branch, nil)
assert.NoError(t, err)
- ctx := context.Background()
- _, _, btgs := gs_testutils.SetupAndLoadBTGitStore(t, ctx, wd, "file://"+repoDir, true)
- vcs, err := New(ctx, btgs, branch, nil)
- assert.NoError(t, err)
- return vcs, btgs, func() {
- util.RemoveAll(wd)
- cleanup()
- }
+ return vcs, btgs, cleanup
}
func startWithEmptyCache(mg *mocks.GitStore) {
@@ -363,8 +355,8 @@
}
}
-func makeTestBranchPointerMap() map[string]*gitstore.BranchPointer {
- return map[string]*gitstore.BranchPointer{
+func makeTestBranchPointerMap() map[string]*gitstore_deprecated.BranchPointer {
+ return map[string]*gitstore_deprecated.BranchPointer{
"master": {
Head: "master",
Index: 3,
diff --git a/golden/cmd/gold_ingestion/main.go b/golden/cmd/gold_ingestion/main.go
index 635324f..012a967 100644
--- a/golden/cmd/gold_ingestion/main.go
+++ b/golden/cmd/gold_ingestion/main.go
@@ -22,7 +22,7 @@
"go.skia.org/infra/go/eventbus"
"go.skia.org/infra/go/firestore"
"go.skia.org/infra/go/gevent"
- "go.skia.org/infra/go/gitstore/bt_gitstore"
+ "go.skia.org/infra/go/gitstore_deprecated/bt_gitstore"
"go.skia.org/infra/go/httputils"
"go.skia.org/infra/go/ingestion"
"go.skia.org/infra/go/ingestion/fs_ingestionstore"
diff --git a/golden/cmd/skiacorrectness/main.go b/golden/cmd/skiacorrectness/main.go
index 547c2bd..dd921ec 100644
--- a/golden/cmd/skiacorrectness/main.go
+++ b/golden/cmd/skiacorrectness/main.go
@@ -33,7 +33,7 @@
"go.skia.org/infra/go/gevent"
"go.skia.org/infra/go/git/gitinfo"
"go.skia.org/infra/go/gitiles"
- "go.skia.org/infra/go/gitstore/bt_gitstore"
+ "go.skia.org/infra/go/gitstore_deprecated/bt_gitstore"
"go.skia.org/infra/go/httputils"
"go.skia.org/infra/go/issues"
"go.skia.org/infra/go/login"
diff --git a/status/go/status/main.go b/status/go/status/main.go
index 8b3ad85..2f5d608 100644
--- a/status/go/status/main.go
+++ b/status/go/status/main.go
@@ -32,7 +32,7 @@
"go.skia.org/infra/go/ds"
"go.skia.org/infra/go/git/repograph"
"go.skia.org/infra/go/gitauth"
- "go.skia.org/infra/go/gitstore/bt_gitstore"
+ "go.skia.org/infra/go/gitstore_deprecated/bt_gitstore"
"go.skia.org/infra/go/httputils"
"go.skia.org/infra/go/login"
"go.skia.org/infra/go/metrics2"
diff --git a/task_scheduler/go/task_scheduler/main.go b/task_scheduler/go/task_scheduler/main.go
index eabe3d8..dc810b8 100644
--- a/task_scheduler/go/task_scheduler/main.go
+++ b/task_scheduler/go/task_scheduler/main.go
@@ -28,7 +28,7 @@
"go.skia.org/infra/go/git"
"go.skia.org/infra/go/git/repograph"
"go.skia.org/infra/go/gitauth"
- "go.skia.org/infra/go/gitstore/bt_gitstore"
+ "go.skia.org/infra/go/gitstore_deprecated/bt_gitstore"
"go.skia.org/infra/go/httputils"
"go.skia.org/infra/go/human"
"go.skia.org/infra/go/isolate"
