[gold] Add service for gitiles follower
Tested it on the 4 instances for which there is a json5 file.
A follow up CL will land it for all instances.
Bug: skia:10582
Change-Id: I40bffde6596dafb1ac54220f6494bd8483c0a754
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/376836
Reviewed-by: Leandro Lovisolo <lovisolo@google.com>
diff --git a/golden/Makefile b/golden/Makefile
index 03b7278..ec57a91 100644
--- a/golden/Makefile
+++ b/golden/Makefile
@@ -80,6 +80,10 @@
k8s-release-diffcalculator: build-static-diffcalculator
./k8s_release_diffcalculator
+.PHONY: k8s-release-gitilesfollower
+k8s-release-gitilesfollower: build-static-gitilesfollower
+ ./k8s_release_gitilesfollower
+
.PHONY: k8s-release-ingestion
k8s-release-ingestion: build-static-ingestion
./k8s_release_ingestion
@@ -110,6 +114,12 @@
rm -f ./build/diffcalculator_k8s
$(KGO) -o build/diffcalculator_k8s -a ./cmd/diffcalculator/...
+.PHONY: build-static-gitilesfollower
+build-static-gitilesfollower:
+ mkdir -p ./build
+ rm -f ./build/gitilesfollower_k8s
+ $(KGO) -o build/gitilesfollower_k8s -a ./cmd/gitilesfollower/gitilesfollower.go
+
.PHONY: build-static-diffserver
build-static-diffserver:
mkdir -p ./build
diff --git a/golden/cmd/gitilesfollower/gitilesfollower.go b/golden/cmd/gitilesfollower/gitilesfollower.go
index 40fbd0c..2213b03 100644
--- a/golden/cmd/gitilesfollower/gitilesfollower.go
+++ b/golden/cmd/gitilesfollower/gitilesfollower.go
@@ -25,6 +25,7 @@
"go.skia.org/infra/go/httputils"
"go.skia.org/infra/go/skerr"
"go.skia.org/infra/go/sklog"
+ "go.skia.org/infra/go/util"
"go.skia.org/infra/go/vcsinfo"
"go.skia.org/infra/golden/go/config"
"go.skia.org/infra/golden/go/sql"
@@ -107,10 +108,11 @@
}
client := httputils.DefaultClientConfig().WithTokenSource(ts).Client()
gitilesClient := gitiles.NewRepo(rfc.GitRepoURL, client)
- go pollRepo(ctx, db, gitilesClient, rfc)
-
- // Wait at least 5 seconds for polling to start before signaling all is well.
- time.Sleep(5 * time.Second)
+ // This starts a goroutine in the background
+ if err := pollRepo(ctx, db, gitilesClient, rfc); err != nil {
+ sklog.Fatalf("Could not do initial update: %s", err)
+ }
+ sklog.Infof("Initial update complete")
http.HandleFunc("/healthz", httputils.ReadyHandleFunc)
sklog.Fatal(http.ListenAndServe(rfc.ReadyPort, nil))
}
@@ -134,22 +136,31 @@
return db
}
-// pollRepo polls the gitiles repo according to the provided duration for as long as the
-// context remains ok.
-func pollRepo(ctx context.Context, db *pgxpool.Pool, client *gitiles.Repo, rfc repoFollowerConfig) {
- ct := time.Tick(rfc.PollPeriod.Duration)
- for {
- select {
- case <-ctx.Done():
- sklog.Errorf("Stopping polling due to context error: %s", ctx.Err())
- return
- case <-ct:
- err := updateCycle(ctx, db, client, rfc)
- if err != nil {
- sklog.Errorf("Error on this cycle for talking to %s: %s", rfc.GitRepoURL, rfc)
+// pollRepo does an initial updateCycle and starts a goroutine to continue updating according
+// to the provided duration for as long as the context remains ok.
+func pollRepo(ctx context.Context, db *pgxpool.Pool, client *gitiles.Repo, rfc repoFollowerConfig) error {
+ sklog.Infof("Doing initial update")
+ err := updateCycle(ctx, db, client, rfc)
+ if err != nil {
+ return skerr.Wrap(err)
+ }
+ go func() {
+ ct := time.Tick(rfc.PollPeriod.Duration)
+ sklog.Infof("Polling every %s", rfc.PollPeriod.Duration)
+ for {
+ select {
+ case <-ctx.Done():
+ sklog.Errorf("Stopping polling due to context error: %s", ctx.Err())
+ return
+ case <-ct:
+ err := updateCycle(ctx, db, client, rfc)
+ if err != nil {
+ sklog.Errorf("Error on this cycle for talking to %s: %s", rfc.GitRepoURL, rfc)
+ }
}
}
- }
+ }()
+ return nil
}
// GitilesLogger is a subset of the gitiles client library that we need. This allows us to mock
@@ -190,6 +201,7 @@
return skerr.Wrapf(err, "getting backlog of commits from %s..%s", previousHash, latestHash)
}
// commits is backwards and LogFirstParent does not respect gitiles.LogReverse()
+ reverse(commits)
sklog.Infof("Got %d commits to store", len(commits))
if err := storeCommits(ctx, db, previousID, commits); err != nil {
return skerr.Wrapf(err, "storing %d commits to GitCommits table", len(commits))
@@ -197,6 +209,14 @@
return nil
}
+// reverses the order of the slice.
+func reverse(commits []*vcsinfo.LongCommit) {
+ total := len(commits)
+ for i := 0; i < total/2; i++ {
+ commits[i], commits[total-i-1] = commits[total-i-1], commits[i]
+ }
+}
+
// getLatestCommitFromRepo returns the git hash of the latest git commit known on the configured
// branch. If overrideLatestCommitKey has a value set, that will be used instead.
func getLatestCommitFromRepo(ctx context.Context, client GitilesLogger, rfc repoFollowerConfig) (string, error) {
@@ -212,7 +232,6 @@
if len(latestCommit) < 1 {
return "", skerr.Fmt("No commits returned")
}
- sklog.Debugf("latest commit: %#v", latestCommit[0])
return latestCommit[0].Hash, nil
}
@@ -245,26 +264,31 @@
}
// storeCommits writes the given commits to the SQL database, assigning them commitIDs in
-// monotonically increasing order. The commits slice is expected to be sorted with the most recent
-// commit first (as is returned by gitiles).
+// monotonically increasing order. The commits slice is expected to be sorted with the oldest
+// commit first (the opposite of how gitiles returns it).
func storeCommits(ctx context.Context, db *pgxpool.Pool, lastCommitID int64, commits []*vcsinfo.LongCommit) error {
ctx, span := trace.StartSpan(ctx, "gitilesfollower_storeCommits")
defer span.End()
+ commitID := lastCommitID + 1
+ // batchSize is only really relevant in the initial load. But we need it to avoid going over
+ // the 65k limit of placeholder indexes.
+ const batchSize = 1000
const statement = `UPSERT INTO GitCommits (git_hash, commit_id, commit_time, author_email, subject) VALUES `
const valuesPerRow = 5
- arguments := make([]interface{}, 0, len(commits)*valuesPerRow)
- commitID := lastCommitID + 1
- for i := range commits {
- // commits is in backwards order. This reverses things.
- c := commits[len(commits)-i-1]
- cid := fmt.Sprintf("%012d", commitID)
- arguments = append(arguments, c.Hash, cid, c.Timestamp, c.Author, c.Subject)
- commitID++
- }
- vp := sql.ValuesPlaceholders(valuesPerRow, len(commits))
- if _, err := db.Exec(ctx, statement+vp, arguments...); err != nil {
- return skerr.Wrap(err)
- }
- return nil
+ err := util.ChunkIter(len(commits), batchSize, func(startIdx int, endIdx int) error {
+ chunk := commits[startIdx:endIdx]
+ arguments := make([]interface{}, 0, len(chunk)*valuesPerRow)
+ for _, c := range chunk {
+ cid := fmt.Sprintf("%012d", commitID)
+ arguments = append(arguments, c.Hash, cid, c.Timestamp, c.Author, c.Subject)
+ commitID++
+ }
+ vp := sql.ValuesPlaceholders(valuesPerRow, len(chunk))
+ if _, err := db.Exec(ctx, statement+vp, arguments...); err != nil {
+ return skerr.Wrap(err)
+ }
+ return nil
+ })
+ return skerr.Wrap(err)
}
diff --git a/golden/cmd/goldpushk/goldpushk/services_map.go b/golden/cmd/goldpushk/goldpushk/services_map.go
index 1cbdea3..a5eee4f 100644
--- a/golden/cmd/goldpushk/goldpushk/services_map.go
+++ b/golden/cmd/goldpushk/goldpushk/services_map.go
@@ -20,11 +20,12 @@
SkiaPublic Instance = "skia-public"
// Gold services.
- BaselineServer Service = "baselineserver"
- DiffCalculator Service = "diffcalculator"
- DiffServer Service = "diffserver"
- IngestionBT Service = "ingestion-bt"
- Frontend Service = "frontend"
+ BaselineServer Service = "baselineserver"
+ DiffCalculator Service = "diffcalculator"
+ DiffServer Service = "diffserver"
+ IngestionBT Service = "ingestion-bt"
+ Frontend Service = "frontend"
+ GitilesFollower Service = "gitilesfollower"
// Testing Gold instances.
TestInstance1 Instance = "goldpushk-test1"
@@ -69,10 +70,11 @@
},
knownServices: []Service{
BaselineServer,
- DiffServer,
- IngestionBT,
- Frontend,
DiffCalculator,
+ DiffServer,
+ Frontend,
+ GitilesFollower,
+ IngestionBT,
},
}
@@ -83,10 +85,14 @@
s.add(instance, Frontend)
} else {
// Add common services for regular instances.
- s.add(instance, DiffServer)
- s.add(instance, IngestionBT)
- s.add(instance, Frontend)
s.add(instance, DiffCalculator)
+ s.add(instance, DiffServer)
+ s.add(instance, Frontend)
+ // See skbug.com/11367
+ if instance != ChromiumOSTastDev {
+ s.add(instance, GitilesFollower)
+ }
+ s.add(instance, IngestionBT)
}
}
@@ -97,18 +103,18 @@
for _, instance := range publicInstancesNeedingBaselineServer {
s.add(instance, BaselineServer)
}
+
// Internal baseline options.
s.addWithOptions(Fuchsia, BaselineServer, DeploymentOptions{
internal: true,
})
// Overwrite common services for "fuchsia" instance, which need to run on skia-corp.
- s.addWithOptions(Fuchsia, DiffServer, DeploymentOptions{
- internal: true,
- })
- s.addWithOptions(Fuchsia, IngestionBT, DeploymentOptions{internal: true})
- s.addWithOptions(Fuchsia, Frontend, DeploymentOptions{internal: true})
s.addWithOptions(Fuchsia, DiffCalculator, DeploymentOptions{internal: true})
+ s.addWithOptions(Fuchsia, DiffServer, DeploymentOptions{internal: true})
+ s.addWithOptions(Fuchsia, Frontend, DeploymentOptions{internal: true})
+ s.addWithOptions(Fuchsia, GitilesFollower, DeploymentOptions{internal: true})
+ s.addWithOptions(Fuchsia, IngestionBT, DeploymentOptions{internal: true})
return s
}
diff --git a/golden/cmd/goldpushk/main_test.go b/golden/cmd/goldpushk/main_test.go
index d1a5000..7d195db 100644
--- a/golden/cmd/goldpushk/main_test.go
+++ b/golden/cmd/goldpushk/main_test.go
@@ -130,6 +130,7 @@
chromeDiffServer := makeID(goldpushk.Chrome, goldpushk.DiffServer)
chromeIngestionBT := makeID(goldpushk.Chrome, goldpushk.IngestionBT)
chromeFrontend := makeID(goldpushk.Chrome, goldpushk.Frontend)
+ chromeGitilesFollower := makeID(goldpushk.Chrome, goldpushk.GitilesFollower)
chromePublicFrontend := makeID(goldpushk.ChromePublic, goldpushk.Frontend)
chromiumTastFrontend := makeID(goldpushk.ChromiumOSTastDev, goldpushk.Frontend)
chromiumTastDiffServer := makeID(goldpushk.ChromiumOSTastDev, goldpushk.DiffServer)
@@ -152,6 +153,7 @@
skiaIngestionBT := makeID(goldpushk.Skia, goldpushk.IngestionBT)
skiaPublicFrontend := makeID(goldpushk.SkiaPublic, goldpushk.Frontend)
skiaFrontend := makeID(goldpushk.Skia, goldpushk.Frontend)
+ skiaGitilesFollower := makeID(goldpushk.Skia, goldpushk.GitilesFollower)
test := func(name string, flagInstances, flagServices, flagCanaries []string, expectedDeployableUnitIDs, expectedCanariedDeployableUnitIDs []goldpushk.DeployableUnitID) {
t.Run(name, func(t *testing.T) {
@@ -212,27 +214,27 @@
////////////////////////////////////////////////////////////////////////////////////////////////
test("Single instance, all services, no canary",
[]string{"chrome"}, []string{"all"}, nil,
- []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeDiffServer, chromeIngestionBT, chromeFrontend},
+ []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeDiffServer, chromeIngestionBT, chromeFrontend, chromeGitilesFollower},
nil)
test("Single instance, all services, one canary",
[]string{"chrome"}, []string{"all"}, []string{"chrome:frontend"},
- []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeDiffServer, chromeIngestionBT},
+ []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeDiffServer, chromeIngestionBT, chromeGitilesFollower},
[]goldpushk.DeployableUnitID{chromeFrontend})
test("Single instance, all services, multiple canaries",
[]string{"chrome"}, []string{"all"}, []string{"chrome:ingestion-bt", "chrome:frontend"},
- []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeDiffServer},
+ []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeDiffServer, chromeGitilesFollower},
[]goldpushk.DeployableUnitID{chromeIngestionBT, chromeFrontend})
test("Multiple instances, all services, no canary",
[]string{"chrome", "skia"}, []string{"all"}, nil,
- []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeDiffServer, chromeIngestionBT, chromeFrontend, skiaDiffCalculator, skiaDiffServer, skiaIngestionBT, skiaFrontend},
+ []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeDiffServer, chromeIngestionBT, chromeFrontend, chromeGitilesFollower, skiaDiffCalculator, skiaDiffServer, skiaIngestionBT, skiaFrontend, skiaGitilesFollower},
nil)
test("Multiple instances, all services, one canary",
[]string{"chrome", "skia"}, []string{"all"}, []string{"skia:frontend"},
- []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeDiffServer, chromeIngestionBT, chromeFrontend, skiaDiffCalculator, skiaDiffServer, skiaIngestionBT},
+ []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeDiffServer, chromeIngestionBT, chromeFrontend, chromeGitilesFollower, skiaDiffCalculator, skiaDiffServer, skiaIngestionBT, skiaGitilesFollower},
[]goldpushk.DeployableUnitID{skiaFrontend})
test("Multiple instances, all services, multiple canaries",
[]string{"chrome", "skia"}, []string{"all"}, []string{"skia:ingestion-bt", "skia:frontend"},
- []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeDiffServer, chromeIngestionBT, chromeFrontend, skiaDiffCalculator, skiaDiffServer},
+ []goldpushk.DeployableUnitID{chromeBaselineServer, chromeDiffCalculator, chromeDiffServer, chromeIngestionBT, chromeFrontend, chromeGitilesFollower, skiaDiffCalculator, skiaDiffServer, skiaGitilesFollower},
[]goldpushk.DeployableUnitID{skiaIngestionBT, skiaFrontend})
////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/golden/dockerfiles/Dockerfile_gitilesfollower b/golden/dockerfiles/Dockerfile_gitilesfollower
new file mode 100644
index 0000000..9605669
--- /dev/null
+++ b/golden/dockerfiles/Dockerfile_gitilesfollower
@@ -0,0 +1,9 @@
+FROM gcr.io/skia-public/basealpine:3.8
+
+USER root
+
+COPY . /
+
+USER skia
+
+ENTRYPOINT ["/usr/local/bin/gold-gitilesfollower"]
diff --git a/golden/k8s-config-templates/gold-common.json5 b/golden/k8s-config-templates/gold-common.json5
index 198ae99..9bf02ed 100644
--- a/golden/k8s-config-templates/gold-common.json5
+++ b/golden/k8s-config-templates/gold-common.json5
@@ -3,6 +3,7 @@
"DIFFCALCULATOR_IMAGE": "gcr.io/skia-public/gold-diffcalculator:2021-02-22T12_23_04Z-kjlubick-d23f6c7-clean",
"DIFF_SERVER_IMAGE": "gcr.io/skia-public/gold-diff-server:2021-01-19T14_27_42Z-kjlubick-450bea3-clean",
"FRONTEND_IMAGE": "gcr.io/skia-public/gold-frontend:2021-02-23T13_50_54Z-kjlubick-6d6951a-clean",
+ "GITILESFOLLOWER_IMAGE": "gcr.io/skia-public/gold-gitilesfollower:2021-02-26T18_42_54Z-kjlubick-dacfd05-dirty",
"INGESTION_BT_IMAGE": "gcr.io/skia-public/gold-ingestion:2021-02-15T14_17_39Z-kjlubick-b37d281-clean",
// Services for testing goldpushk.
diff --git a/golden/k8s-config-templates/gold-gitilesfollower-template.yaml b/golden/k8s-config-templates/gold-gitilesfollower-template.yaml
new file mode 100644
index 0000000..697a10bc
--- /dev/null
+++ b/golden/k8s-config-templates/gold-gitilesfollower-template.yaml
@@ -0,0 +1,79 @@
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app: gold-{{.INSTANCE_ID}}-gitilesfollower
+ name: gold-{{.INSTANCE_ID}}-gitilesfollower
+spec:
+ ports:
+ - name: metrics
+ port: 20000
+ selector:
+ app: gold-{{.INSTANCE_ID}}-gitilesfollower
+ type: NodePort
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: gold-{{.INSTANCE_ID}}-gitilesfollower
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: gold-{{.INSTANCE_ID}}-gitilesfollower
+ strategy:
+ type: RollingUpdate
+ template:
+ metadata:
+ labels:
+ app: gold-{{.INSTANCE_ID}}-gitilesfollower
+ appgroup: gold-{{.INSTANCE_ID}}
+ date: "{{.NOW}}" # Forces a re-deploy even if just the config file changes.
+ annotations:
+ prometheus.io.scrape: "true"
+ prometheus.io.port: "20000"
+ spec:
+ automountServiceAccountToken: false
+ securityContext:
+ runAsUser: 2000 # aka skia
+ fsGroup: 2000 # aka skia
+ containers:
+ - name: gold-{{.INSTANCE_ID}}-gitilesfollower
+ image: {{.GITILESFOLLOWER_IMAGE}}
+ args:
+ - "--common_instance_config=/etc/gold-config/{{.INSTANCE_ID}}.json5"
+ - "--config=/etc/gold-config/{{.INSTANCE_ID}}-gitilesfollower.json5"
+ - "--logtostderr"
+ ports:
+ - containerPort: 20000
+ name: prom
+ volumeMounts:
+ - name: gold-{{.INSTANCE_ID}}-config-volume
+ mountPath: /etc/gold-config/
+ - name: gold-service-account-secrets
+ mountPath: /etc/gold-secrets/
+ env:
+ - name: GOOGLE_APPLICATION_CREDENTIALS
+ value: /etc/gold-secrets/service-account.json
+ - name: K8S_POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ resources:
+ requests:
+ memory: "100Mi"
+ cpu: "10m"
+ readinessProbe:
+ httpGet:
+ path: /healthz
+ port: 8000
+ initialDelaySeconds: 5
+ periodSeconds: 3
+ volumes:
+ - name: gold-{{.INSTANCE_ID}}-config-volume
+ configMap:
+ defaultMode: 400
+ name: gold-{{.INSTANCE_ID}}-config
+ - name: gold-service-account-secrets
+ secret:
+ secretName: gold-service-account-secrets
diff --git a/golden/k8s-instances/chrome/chrome-gitilesfollower.json5 b/golden/k8s-instances/chrome/chrome-gitilesfollower.json5
new file mode 100644
index 0000000..d45aa53
--- /dev/null
+++ b/golden/k8s-instances/chrome/chrome-gitilesfollower.json5
@@ -0,0 +1,7 @@
+{
+ // Arbitrary commit from 17 Dec 2020
+ initial_commit: "971e0b95aba2fe33d26f14cc19e346e81d6b6a34",
+ poll_period: "1m",
+ prom_port: ":20000",
+ ready_port: ":8000"
+}
\ No newline at end of file
diff --git a/golden/k8s-instances/flutter/flutter-gitilesfollower.json5 b/golden/k8s-instances/flutter/flutter-gitilesfollower.json5
new file mode 100644
index 0000000..2fa5c1b
--- /dev/null
+++ b/golden/k8s-instances/flutter/flutter-gitilesfollower.json5
@@ -0,0 +1,9 @@
+{
+ // Arbitrary commit from 28 Dec 2020
+ initial_commit: "36373f8d08742468541cbaca35f20810662b2436",
+ // Need to use a gitiles mirror
+ git_repo_url: "https://chromium.googlesource.com/external/github.com/flutter/flutter",
+ poll_period: "1m",
+ prom_port: ":20000",
+ ready_port: ":8000"
+}
\ No newline at end of file
diff --git a/golden/k8s-instances/skia-infra/skia-infra-gitilesfollower.json5 b/golden/k8s-instances/skia-infra/skia-infra-gitilesfollower.json5
new file mode 100644
index 0000000..ccaeaf4
--- /dev/null
+++ b/golden/k8s-instances/skia-infra/skia-infra-gitilesfollower.json5
@@ -0,0 +1,7 @@
+{
+ // Arbitrary commit from 9 Dec 2020
+ initial_commit: "7f4ddfca8eb543bdc427173a7c3e10d69f960722",
+ poll_period: "1m",
+ prom_port: ":20000",
+ ready_port: ":8000"
+}
\ No newline at end of file
diff --git a/golden/k8s-instances/skia/skia-gitilesfollower.json5 b/golden/k8s-instances/skia/skia-gitilesfollower.json5
new file mode 100644
index 0000000..410c1bd
--- /dev/null
+++ b/golden/k8s-instances/skia/skia-gitilesfollower.json5
@@ -0,0 +1,7 @@
+{
+ // Arbitrary commit from 14 Dec 2020
+ initial_commit: "9b395f55ea0f8f92103d33f1ea8e8217bee8aaea",
+ poll_period: "1m",
+ prom_port: ":20000",
+ ready_port: ":8000"
+}
\ No newline at end of file
diff --git a/golden/k8s_release_gitilesfollower b/golden/k8s_release_gitilesfollower
new file mode 100755
index 0000000..9fbbda5
--- /dev/null
+++ b/golden/k8s_release_gitilesfollower
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -x -e
+
+# Create and upload a container image for a server that syncs the gitiles repo into SQL.
+APPNAME=gold-gitilesfollower
+
+# Copy files into the right locations in ${ROOT}.
+copy_release_files()
+{
+INSTALL="install -D --verbose --backup=none"
+INSTALL_DIR="install -d --verbose --backup=none"
+
+# Add the dockerfile and binary.
+${INSTALL} --mode=644 -T ./dockerfiles/Dockerfile_gitilesfollower ${ROOT}/Dockerfile
+${INSTALL} --mode=755 -T ./build/gitilesfollower_k8s ${ROOT}/usr/local/bin/${APPNAME}
+}
+
+source ../bash/docker_build.sh
