| // Package lookup provides ... |
| package lookup |
| |
| import ( |
| "context" |
| "fmt" |
| "net/url" |
| "strconv" |
| "strings" |
| "sync" |
| |
| "go.skia.org/infra/go/git" |
| "go.skia.org/infra/go/sklog" |
| ) |
| |
| // Cache keeps a cache of recent buildids and their associated git hashes. Since |
| // this is only used for ingesting test results this window is fine since we can |
| // presume the tests successfully finish running in under a year. |
| type Cache struct { |
| mutex sync.Mutex |
| |
| // hashes maps buildids to git hashes. We don't get fancy and use an lru |
| // cache because the amount of data we store is so small. Git hashes are 40 |
| // chars and int64's are 8, so if we assume 100 commits to the repo per day, |
| // and the application ran without restart for three years straight then this |
| // data structure would grow to 48*100*365*3 bytes, which is ~5MB. Even if builds |
| // arrived at the maximum of one build per second for a year we'd only use |
| // 48*86400*365 bytes = 1.5 GB. |
| hashes map[int64]string |
| } |
| |
| // New returns a newly populated *Cache with buildids for the last 2 weeks. |
| // |
| // The 'checkout' is only used during the construction of *Cache. |
| func New(ctx context.Context, checkout *git.Checkout) (*Cache, error) { |
| // Runs |
| // |
| // git log main --format=oneline --since="4 weeks ago" |
| // |
| // to prepopulate hashes. |
| log, err := checkout.Git(ctx, "log", git.MainBranch, "--format=oneline", "--since=\"1 year ago\"") |
| if err != nil { |
| return nil, fmt.Errorf("Failed to prime cache from checkout: %s", err) |
| } |
| c := &Cache{ |
| hashes: map[int64]string{}, |
| } |
| if err := c.parseLog(log); err != nil { |
| return nil, fmt.Errorf("Failed to parse git log from checkout: %s", err) |
| } |
| return c, nil |
| } |
| |
| func (c *Cache) parseLog(log string) error { |
| // The oneline log format looks like |
| // |
| // 6dab50c23b3927daf7487b4a6f105fc74aff5fa7 https://android-build.googleplex.com/builds/jump-to-build/7432561 |
| // 3133350e05eb07629d681c3bb61a91a51e2ff2ef https://android-build.googleplex.com/builds/jump-to-build/7432560 |
| // |
| // if you include the commit message that poprepo adds. |
| count := 0 |
| lines := strings.Split(log, "\n") |
| for _, line := range lines { |
| if line == "" { |
| continue |
| } |
| // Split at the space between hash and url. |
| parts := strings.Split(line, " ") |
| if len(parts) != 2 { |
| sklog.Warningf("Found invalid line: %q", line) |
| continue |
| } |
| hash := parts[0] |
| u, err := url.Parse(parts[1]) |
| if err != nil { |
| return fmt.Errorf("Found invalid url: %q", parts[1]) |
| } |
| |
| // Split the URL on the slashes. |
| urlParts := strings.Split(u.Path, "/") |
| |
| buildIDAsString := "" |
| if len(urlParts) == 3 { |
| buildIDAsString = urlParts[2] |
| } else if len(urlParts) == 4 { |
| buildIDAsString = urlParts[3] |
| } else { |
| sklog.Errorf("Found invalid url: %q", urlParts) |
| continue |
| } |
| buildid, err := strconv.ParseInt(buildIDAsString, 10, 64) |
| if err != nil { |
| sklog.Errorf("Found invalid buildid: %q", urlParts[2]) |
| continue |
| } |
| c.hashes[buildid] = hash |
| count++ |
| } |
| sklog.Infof("Prepopulated lookup.Cache with %d buildids.", count) |
| return nil |
| } |
| |
| // Lookup returns the git hash for the given buildid. |
| func (c *Cache) Lookup(buildid int64) (string, error) { |
| c.mutex.Lock() |
| defer c.mutex.Unlock() |
| |
| if hash, ok := c.hashes[buildid]; !ok { |
| return "", fmt.Errorf("BuildId not found in cache: %d", buildid) |
| } else { |
| return hash, nil |
| } |
| } |
| |
| // Add a new buildid, githash to the cache. |
| func (c *Cache) Add(buildid int64, hash string) { |
| c.mutex.Lock() |
| defer c.mutex.Unlock() |
| |
| c.hashes[buildid] = hash |
| } |