go/androidbuild/androidbuild.go - buildbot - Git at Google

 // androidbuild implements a simple interface to look up skia git commit
 // hashes from android buildIDs.
 package androidbuild

 import (
 	"bytes"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"sort"
 	"strconv"
 	"strings"
 	"time"

 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/errors"
 	lutil "github.com/syndtr/goleveldb/leveldb/util"
 	"go.skia.org/infra/go/sklog"
 	"go.skia.org/infra/go/util"
 	"go.skia.org/infra/go/vcsinfo"
 )

 const (
 	NUM_RETRIES    = 10
 	PAGE_SIZE      = 5
 	SLEEP_DURATION = 3

 	// TARGETS_KEY is the key for the list of space separated branch:targets we are monitoring.
 	TARGETS_KEY = "android_ingest_targets"

 	// LAST_BUILD_FOR_TARGET_TEMPLATE is expanded with the branch and target name to build they key for the last buildID seen for that target.
 	LAST_BUILD_FOR_TARGET_TEMPLATE = "android_ingest_last_build:%s:%s"
 )

 // Info is an inferface for querying for commit info from an Android branch, target, and buildID.
 type Info interface {
 	Get(branch, target, buildID string) (*vcsinfo.ShortCommit, error)
 }

 // info implements Info.
 //
 // It uses a leveldb database to store information it has already read.
 //
 // The first time a caller tries to Get(branch, target, buildID) and the
 // (branch, target) pair has never been seen before it will be added to
 // the list of (branch, target) pairs to periodically refresh. The pairs
 // are stored in the leveldb at TARGETS_KEY in the leveldb.
 //
 // Periodically info will request the build info for all (branch:target)
 // pairs it is monitoring. The request will be for all builds from the most
 // current back to the last one seen. The last buildID seen for each (branch:target)
 // pair is stored at LAST_BUILD_FOR_TARGET_TEMPLATE.
 type info struct {
 	db *leveldb.DB

 	// commits is an interface for fetching data from the Android Build API,
 	// broken out as an interface for testability.
 	commits commits
 }

 // New creates a new *info as Info.
 //
 // dir is the directory where the leveldb that caches responses will be written.
 // client must be an authenticated client to make requests to the Android Build API.
 func New(dir string, client *http.Client) (Info, error) {
 	db, err := leveldb.OpenFile(dir, nil)
 	if err != nil && errors.IsCorrupted(err) {
 		db, err = leveldb.RecoverFile(dir, nil)
 	}
 	if err != nil {
 		return nil, fmt.Errorf("Failed to open leveldb at %s: %s", dir, err)
 	}
 	c, err := newAndroidCommits(client)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to create commits: %s", err)
 	}

 	i := &info{db: db, commits: c}
 	go i.poll()

 	return i, nil
 }

 // Get returns the ShortCommit info for the given branch, target, and buildID.
 func (i *info) Get(branch, target, buildID string) (*vcsinfo.ShortCommit, error) {
 	// Get the list of targets and confirm that this target is in it, otherwise add it to the list of targets.
 	branchtargets := i.branchtargets()
 	branchtarget := fmt.Sprintf("%s:%s", branch, target)
 	if !util.In(branchtarget, branchtargets) {
 		// If we aren't currently scanning results for this (branch, target) pair
 		// then add it to the list.
 		branchtargets = append(branchtargets, branchtarget)
 		err := i.db.Put([]byte(TARGETS_KEY), []byte(strings.Join(branchtargets, " ")), nil)
 		if err != nil {
 			sklog.Errorf("Failed to add new target %s: %s", branchtarget, err)
 		}
 		// Always try to fetch the information from the Android Build API directly if
 		// we don't have it yet.
 		return i.single_get(branch, target, buildID)
 	} else {
 		key, err := toKey(branch, target, buildID)
 		if err != nil {
 			return nil, fmt.Errorf("Can't Get with an invalid build ID %q: %s", buildID, err)
 		}
 		// Scan backwards through the build info until we find a buildID that is equal to or
 		// comes before the buildID we are looking for.
 		iter := i.db.NewIterator(lutil.BytesPrefix([]byte(toPrefix(branch, target))), nil)
 		defer iter.Release()
 		if found := iter.Seek([]byte(key)); found {
 			value := &vcsinfo.ShortCommit{}
 			if err := json.Unmarshal(iter.Value(), value); err != nil {
 				return nil, fmt.Errorf("Unable to deserialize value: %s", err)
 			}
 			return value, nil
 		} else {
 			return i.single_get(branch, target, buildID)
 		}
 	}
 }

 // single_get uses i.commits to attempt to retrieve a single commit, storing the results
 // in the leveldb if successful.
 func (i *info) single_get(branch, target, buildID string) (*vcsinfo.ShortCommit, error) {
 	c, err := i.commits.Get(branch, target, buildID)
 	if err != nil {
 		return nil, err
 	}
 	i.store(branch, target, buildID, c)
 	return c, nil
 }

 // branchtargets returns the list of (branch:target)s we are monitoring.
 func (i *info) branchtargets() []string {
 	b, err := i.db.Get([]byte(TARGETS_KEY), nil)
 	if err != nil {
 		sklog.Errorf("Failed to get TARGETS_KEY: %s", err)
 		return []string{}
 	}
 	parts := strings.Split(string(b), " ")
 	ret := []string{}
 	for _, s := range parts {
 		if s != "" {
 			ret = append(ret, s)
 		}
 	}
 	return ret
 }

 // store the given commit in the leveldb.
 func (i *info) store(branch, target, buildID string, commit *vcsinfo.ShortCommit) {
 	key, err := toKey(branch, target, buildID)
 	if err != nil {
 		sklog.Errorf("store: invalid build ID %s: %s", key, err)
 		return
 	}
 	b, err := json.Marshal(commit)
 	if err != nil {
 		sklog.Errorf("store: can't encode %#v: %s", commit, err)
 		return
 	}
 	if err := i.db.Put([]byte(key), b, nil); err != nil {
 		sklog.Errorf("Failed to store commit: %s", err)
 	}
 }

 // single_poll does a single loop of polling the API for each target we are monitoring.
 func (i *info) single_poll() {
 	for _, branchtarget := range i.branchtargets() {
 		parts := strings.Split(branchtarget, ":")
 		if len(parts) != 2 {
 			sklog.Errorf("Found an invalid branchtarget: %s", branchtarget)
 			continue
 		}
 		branch := parts[0]
 		target := parts[1]
 		// Find the last buildID we've seen so far.
 		lastBuildID := ""
 		lastBuildIDKey := []byte(fmt.Sprintf(LAST_BUILD_FOR_TARGET_TEMPLATE, branch, target))
 		b, err := i.db.Get(lastBuildIDKey, nil)
 		if err == nil {
 			lastBuildID = string(b)
 		}

 		// Query for commits from latest to lastBuildID.
 		builds, err := i.commits.List(branch, target, lastBuildID)
 		if err != nil {
 			sklog.Errorf("Failed to get commits for %s %s %s: %s", branch, target, lastBuildID, err)
 			continue
 		}
 		// Save each buildID we found.
 		for k, v := range builds {
 			i.store(branch, target, k, v)
 		}
 		// Now find the largest buildID and store it.
 		buildIDs := []int{}
 		for id := range builds {
 			i, err := strconv.Atoi(id)
 			if err == nil {
 				buildIDs = append(buildIDs, i)
 			}
 		}
 		sort.Sort(sort.IntSlice(buildIDs))
 		if len(buildIDs) > 0 {
 			lastBuildID = strconv.Itoa(buildIDs[len(buildIDs)-1])
 			err := i.db.Put(lastBuildIDKey, []byte(lastBuildID), nil)
 			if err != nil {
 				sklog.Errorf("Failed to write last build ID: %s", err)
 			}
 		}
 	}
 }

 // poll periodically polls each target we are monitoring.
 func (i *info) poll() {
 	for range time.Tick(time.Minute) {
 		i.single_poll()
 	}
 }

 // invertBSlice inverts the bytes in a slice so that sorting
 // will work in the reverse order.
 //
 // So leveldb will only sort keys in ascending order, but we want to sort based
 // in the buildID in reverse order, so we will invert the bytes so that sorting
 // works in the direction that we want.
 func invertBSlice(a []byte) []byte {
 	inverted := make([]byte, len(a))
 	copy(inverted, a)
 	for i := 0; i < len(a); i++ {
 		inverted[i] = 0xff - a[i]
 	}
 	return inverted
 }

 // toKey converts branch, target, and buildID into a string usable as a leveldb key.
 //
 // For example:
 //
 //     toKey("git_master-skia", "razor-userdebug", "1814540")
 //
 // Returns
 //
 //     "git_master-skia:razor-userdebug:<inverted bytes of 00000000000001814540>"
 func toKey(branch, target, buildID string) ([]byte, error) {
 	id, err := strconv.Atoi(buildID)
 	if err != nil {
 		return []byte{}, err
 	}
 	paddedID := []byte(fmt.Sprintf("%020d", id))
 	return append([]byte(fmt.Sprintf("%s:%s:", branch, target)), invertBSlice(paddedID)...), nil
 }

 // toPrefix returns a prefix for restricting searches to one particular target in the leveldb.
 func toPrefix(branch, target string) []byte {
 	return []byte(fmt.Sprintf("%s:%s:", branch, target))
 }

 // fromKey converts a key returned from toKey back into (branch, target, buildID).
 func fromKey(key []byte) (string, string, string, error) {
 	parts := bytes.Split(key, []byte(":"))
 	if len(parts) != 3 {
 		return "", "", "", fmt.Errorf("Invalid key format, wrong number of parts: %q", key)
 	}
 	id, err := strconv.Atoi(string(invertBSlice(parts[2])))
 	if err != nil {
 		return "", "", "", fmt.Errorf("Invalid key format, buildID not a valid int: %q", key)
 	}
 	return string(parts[0]), string(parts[1]), strconv.Itoa(id), nil
 }
	// androidbuild implements a simple interface to look up skia git commit
	// hashes from android buildIDs.
	package androidbuild

	import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/errors"
	lutil "github.com/syndtr/goleveldb/leveldb/util"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/vcsinfo"
	)

	const (
	NUM_RETRIES = 10
	PAGE_SIZE = 5
	SLEEP_DURATION = 3

	// TARGETS_KEY is the key for the list of space separated branch:targets we are monitoring.
	TARGETS_KEY = "android_ingest_targets"

	// LAST_BUILD_FOR_TARGET_TEMPLATE is expanded with the branch and target name to build they key for the last buildID seen for that target.
	LAST_BUILD_FOR_TARGET_TEMPLATE = "android_ingest_last_build:%s:%s"
	)

	// Info is an inferface for querying for commit info from an Android branch, target, and buildID.
	type Info interface {
	Get(branch, target, buildID string) (*vcsinfo.ShortCommit, error)
	}

	// info implements Info.
	//
	// It uses a leveldb database to store information it has already read.
	//
	// The first time a caller tries to Get(branch, target, buildID) and the
	// (branch, target) pair has never been seen before it will be added to
	// the list of (branch, target) pairs to periodically refresh. The pairs
	// are stored in the leveldb at TARGETS_KEY in the leveldb.
	//
	// Periodically info will request the build info for all (branch:target)
	// pairs it is monitoring. The request will be for all builds from the most
	// current back to the last one seen. The last buildID seen for each (branch:target)
	// pair is stored at LAST_BUILD_FOR_TARGET_TEMPLATE.
	type info struct {
	db *leveldb.DB

	// commits is an interface for fetching data from the Android Build API,
	// broken out as an interface for testability.
	commits commits
	}

	// New creates a new *info as Info.
	//
	// dir is the directory where the leveldb that caches responses will be written.
	// client must be an authenticated client to make requests to the Android Build API.
	func New(dir string, client *http.Client) (Info, error) {
	db, err := leveldb.OpenFile(dir, nil)
	if err != nil && errors.IsCorrupted(err) {
	db, err = leveldb.RecoverFile(dir, nil)
	}
	if err != nil {
	return nil, fmt.Errorf("Failed to open leveldb at %s: %s", dir, err)
	}
	c, err := newAndroidCommits(client)
	if err != nil {
	return nil, fmt.Errorf("Failed to create commits: %s", err)
	}

	i := &info{db: db, commits: c}
	go i.poll()

	return i, nil
	}

	// Get returns the ShortCommit info for the given branch, target, and buildID.
	func (i info) Get(branch, target, buildID string) (vcsinfo.ShortCommit, error) {
	// Get the list of targets and confirm that this target is in it, otherwise add it to the list of targets.
	branchtargets := i.branchtargets()
	branchtarget := fmt.Sprintf("%s:%s", branch, target)
	if !util.In(branchtarget, branchtargets) {
	// If we aren't currently scanning results for this (branch, target) pair
	// then add it to the list.
	branchtargets = append(branchtargets, branchtarget)
	err := i.db.Put([]byte(TARGETS_KEY), []byte(strings.Join(branchtargets, " ")), nil)
	if err != nil {
	sklog.Errorf("Failed to add new target %s: %s", branchtarget, err)
	}
	// Always try to fetch the information from the Android Build API directly if
	// we don't have it yet.
	return i.single_get(branch, target, buildID)
	} else {
	key, err := toKey(branch, target, buildID)
	if err != nil {
	return nil, fmt.Errorf("Can't Get with an invalid build ID %q: %s", buildID, err)
	}
	// Scan backwards through the build info until we find a buildID that is equal to or
	// comes before the buildID we are looking for.
	iter := i.db.NewIterator(lutil.BytesPrefix([]byte(toPrefix(branch, target))), nil)
	defer iter.Release()
	if found := iter.Seek([]byte(key)); found {
	value := &vcsinfo.ShortCommit{}
	if err := json.Unmarshal(iter.Value(), value); err != nil {
	return nil, fmt.Errorf("Unable to deserialize value: %s", err)
	}
	return value, nil
	} else {
	return i.single_get(branch, target, buildID)
	}
	}
	}

	// single_get uses i.commits to attempt to retrieve a single commit, storing the results
	// in the leveldb if successful.
	func (i info) single_get(branch, target, buildID string) (vcsinfo.ShortCommit, error) {
	c, err := i.commits.Get(branch, target, buildID)
	if err != nil {
	return nil, err
	}
	i.store(branch, target, buildID, c)
	return c, nil
	}

	// branchtargets returns the list of (branch:target)s we are monitoring.
	func (i *info) branchtargets() []string {
	b, err := i.db.Get([]byte(TARGETS_KEY), nil)
	if err != nil {
	sklog.Errorf("Failed to get TARGETS_KEY: %s", err)
	return []string{}
	}
	parts := strings.Split(string(b), " ")
	ret := []string{}
	for _, s := range parts {
	if s != "" {
	ret = append(ret, s)
	}
	}
	return ret
	}

	// store the given commit in the leveldb.
	func (i info) store(branch, target, buildID string, commit vcsinfo.ShortCommit) {
	key, err := toKey(branch, target, buildID)
	if err != nil {
	sklog.Errorf("store: invalid build ID %s: %s", key, err)
	return
	}
	b, err := json.Marshal(commit)
	if err != nil {
	sklog.Errorf("store: can't encode %#v: %s", commit, err)
	return
	}
	if err := i.db.Put([]byte(key), b, nil); err != nil {
	sklog.Errorf("Failed to store commit: %s", err)
	}
	}

	// single_poll does a single loop of polling the API for each target we are monitoring.
	func (i *info) single_poll() {
	for _, branchtarget := range i.branchtargets() {
	parts := strings.Split(branchtarget, ":")
	if len(parts) != 2 {
	sklog.Errorf("Found an invalid branchtarget: %s", branchtarget)
	continue
	}
	branch := parts[0]
	target := parts[1]
	// Find the last buildID we've seen so far.
	lastBuildID := ""
	lastBuildIDKey := []byte(fmt.Sprintf(LAST_BUILD_FOR_TARGET_TEMPLATE, branch, target))
	b, err := i.db.Get(lastBuildIDKey, nil)
	if err == nil {
	lastBuildID = string(b)
	}

	// Query for commits from latest to lastBuildID.
	builds, err := i.commits.List(branch, target, lastBuildID)
	if err != nil {
	sklog.Errorf("Failed to get commits for %s %s %s: %s", branch, target, lastBuildID, err)
	continue
	}
	// Save each buildID we found.
	for k, v := range builds {
	i.store(branch, target, k, v)
	}
	// Now find the largest buildID and store it.
	buildIDs := []int{}
	for id := range builds {
	i, err := strconv.Atoi(id)
	if err == nil {
	buildIDs = append(buildIDs, i)
	}
	}
	sort.Sort(sort.IntSlice(buildIDs))
	if len(buildIDs) > 0 {
	lastBuildID = strconv.Itoa(buildIDs[len(buildIDs)-1])
	err := i.db.Put(lastBuildIDKey, []byte(lastBuildID), nil)
	if err != nil {
	sklog.Errorf("Failed to write last build ID: %s", err)
	}
	}
	}
	}

	// poll periodically polls each target we are monitoring.
	func (i *info) poll() {
	for range time.Tick(time.Minute) {
	i.single_poll()
	}
	}

	// invertBSlice inverts the bytes in a slice so that sorting
	// will work in the reverse order.
	//
	// So leveldb will only sort keys in ascending order, but we want to sort based
	// in the buildID in reverse order, so we will invert the bytes so that sorting
	// works in the direction that we want.
	func invertBSlice(a []byte) []byte {
	inverted := make([]byte, len(a))
	copy(inverted, a)
	for i := 0; i < len(a); i++ {
	inverted[i] = 0xff - a[i]
	}
	return inverted
	}

	// toKey converts branch, target, and buildID into a string usable as a leveldb key.
	//
	// For example:
	//
	// toKey("git_master-skia", "razor-userdebug", "1814540")
	//
	// Returns
	//
	// "git_master-skia:razor-userdebug:<inverted bytes of 00000000000001814540>"
	func toKey(branch, target, buildID string) ([]byte, error) {
	id, err := strconv.Atoi(buildID)
	if err != nil {
	return []byte{}, err
	}
	paddedID := []byte(fmt.Sprintf("%020d", id))
	return append([]byte(fmt.Sprintf("%s:%s:", branch, target)), invertBSlice(paddedID)...), nil
	}

	// toPrefix returns a prefix for restricting searches to one particular target in the leveldb.
	func toPrefix(branch, target string) []byte {
	return []byte(fmt.Sprintf("%s:%s:", branch, target))
	}

	// fromKey converts a key returned from toKey back into (branch, target, buildID).
	func fromKey(key []byte) (string, string, string, error) {
	parts := bytes.Split(key, []byte(":"))
	if len(parts) != 3 {
	return "", "", "", fmt.Errorf("Invalid key format, wrong number of parts: %q", key)
	}
	id, err := strconv.Atoi(string(invertBSlice(parts[2])))
	if err != nil {
	return "", "", "", fmt.Errorf("Invalid key format, buildID not a valid int: %q", key)
	}
	return string(parts[0]), string(parts[1]), strconv.Itoa(id), nil
	}