[gold] Move ingestion stuff to gold proper
Change-Id: I9556291b6b810ef1fe0c235777a07b6079884d8a
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/220525
Commit-Queue: Kevin Lubick <kjlubick@google.com>
Reviewed-by: Joe Gregorio <jcgregorio@google.com>
diff --git a/golden/Makefile b/golden/Makefile
index 6ff338c..5b7f074 100644
--- a/golden/Makefile
+++ b/golden/Makefile
@@ -45,6 +45,10 @@
k8s-release-baseline-server: build-static-baseline-server
./k8s_release_baseline_server
+.PHONY: k8s-release-ingestion
+k8s-release-ingestion: build-static-ingestion
+ ./k8s_release_ingestion
+
.PHONY: build-static-skiacorrectness
build-static-skiacorrectness: skiaversion
mkdir -p ./build
@@ -63,6 +67,12 @@
rm -f ./build/diff_server_k8s
$(KGO) -o build/diff_server_k8s -a ./cmd/skia_diff_server/main.go
+.PHONY: build-static-ingestion
+build-static-ingestion: skiaversion
+ mkdir -p ./build
+ rm -f ./build/gold_ingestion_k8s
+ $(KGO) -o build/gold_ingestion_k8s -a ./cmd/gold_ingestion/main.go
+
.PHONY: release
release: allgo frontend
./build_release "release" "all"
diff --git a/tracedb/build_release_ingestiond b/golden/build_release_ingestiond
similarity index 95%
rename from tracedb/build_release_ingestiond
rename to golden/build_release_ingestiond
index 513234e..00e2a76 100755
--- a/tracedb/build_release_ingestiond
+++ b/golden/build_release_ingestiond
@@ -38,7 +38,7 @@
INSTALL_DIR="fakeroot install -d --verbose --backup=none --group=root --owner=root"
${INSTALL_DIR} --mode=777 ${ROOT}/mnt/pd0/${APPNAME}/repo
${INSTALL} --mode=644 -T ${SERVICE_FILE} ${ROOT}/etc/systemd/system/${SYSTEMD}
- ${INSTALL} --mode=755 -T ${GOPATH}/bin/skia_ingestion ${ROOT}/usr/local/bin/${APPNAME}
+ ${INSTALL} --mode=755 -T ${GOPATH}/bin/gold_ingestion ${ROOT}/usr/local/bin/${APPNAME}
${INSTALL} --mode=644 -T ./sys/${APPNAME}.json5 ${ROOT}/etc/${APPNAME}/config.json5
}
diff --git a/golden/cmd/bt_reingester/main.go b/golden/cmd/bt_reingester/main.go
new file mode 100644
index 0000000..b0fe97f
--- /dev/null
+++ b/golden/cmd/bt_reingester/main.go
@@ -0,0 +1,27 @@
+package main
+
+// bt_reingester will scan through all the files in a GCS bucket and ingest
+// them into the bt_tracestore.
+import (
+ "flag"
+
+ "go.skia.org/infra/golden/go/tracestore/bt_tracestore"
+)
+
+func main() {
+ var (
+ btInstance = flag.String("bt_instance", "", "Bigtable instance to use in the project identified by 'project_id'")
+ projectID = flag.String("project_id", "skia-public", "GCP project ID.")
+ btTableID = flag.String("bt_table_id", "production", "BigTable table ID.")
+ )
+ flag.Parse()
+
+ btc := bt_tracestore.BTConfig{
+ ProjectID: *projectID,
+ InstanceID: *btInstance,
+ TableID: *btTableID,
+ // TODO(kjlubick): VCS
+ }
+
+ _ = bt_tracestore.InitBT(btc)
+}
diff --git a/tracedb/go/skia_ingestion/main.go b/golden/cmd/gold_ingestion/main.go
similarity index 100%
rename from tracedb/go/skia_ingestion/main.go
rename to golden/cmd/gold_ingestion/main.go
diff --git a/golden/cmd/gold_init/main.go b/golden/cmd/gold_init/main.go
deleted file mode 100644
index 3a9aeef..0000000
--- a/golden/cmd/gold_init/main.go
+++ /dev/null
@@ -1,29 +0,0 @@
-package main
-
-// Gold init initializes the cloud so it's ready to host a gold instances,
-// e.g. create the Bigtable tables required.
-import (
- "flag"
-
- "go.skia.org/infra/go/bt"
- "go.skia.org/infra/go/common"
- "go.skia.org/infra/go/ingestion"
- "go.skia.org/infra/go/sklog"
-)
-
-// TODO(stephana): Look into whether this should be done by a script.
-
-// Command line flags.
-var (
- btInstance = flag.String("bt_instance", "", "Bigtable instance to use in the project identified by 'project_id'")
- projectID = flag.String("project_id", common.PROJECT_ID, "GCP project ID.")
- btTableID = flag.String("bt_table_id", "production", "BigTable table ID.")
-)
-
-func main() {
- common.Init()
-
- if err := bt.InitBigtable(*projectID, *btInstance, *btTableID, ingestion.ColumnFamilies); err != nil {
- sklog.Fatalf("Error initializing bigtable: %s", err)
- }
-}
diff --git a/tracedb/dockerfiles/Dockerfile_ingestion b/golden/dockerfiles/Dockerfile_ingestion
similarity index 100%
rename from tracedb/dockerfiles/Dockerfile_ingestion
rename to golden/dockerfiles/Dockerfile_ingestion
diff --git a/tracedb/k8s_push_ingestion b/golden/k8s_release_ingestion
similarity index 86%
rename from tracedb/k8s_push_ingestion
rename to golden/k8s_release_ingestion
index 94fbedd..7851f5d 100755
--- a/tracedb/k8s_push_ingestion
+++ b/golden/k8s_release_ingestion
@@ -12,7 +12,7 @@
INSTALL_DIR="install -d --verbose --backup=none"
${INSTALL} --mode=644 -T ./dockerfiles/Dockerfile_ingestion ${ROOT}/Dockerfile
-${INSTALL} --mode=755 -T ./build/skia_ingestion_k8s ${ROOT}/usr/local/bin/gold-ingestion
+${INSTALL} --mode=755 -T ./build/gold_ingestion_k8s ${ROOT}/usr/local/bin/gold-ingestion
}
source ../bash/docker_build.sh
diff --git a/tracedb/sys/chromevr-gold-ingestion.json5 b/golden/sys/chromevr-gold-ingestion.json5
similarity index 100%
rename from tracedb/sys/chromevr-gold-ingestion.json5
rename to golden/sys/chromevr-gold-ingestion.json5
diff --git a/tracedb/sys/chromevr-gold-ingestion.service b/golden/sys/chromevr-gold-ingestion.service
similarity index 100%
rename from tracedb/sys/chromevr-gold-ingestion.service
rename to golden/sys/chromevr-gold-ingestion.service
diff --git a/tracedb/sys/pdfium-gold-ingestion.json5 b/golden/sys/pdfium-gold-ingestion.json5
similarity index 100%
rename from tracedb/sys/pdfium-gold-ingestion.json5
rename to golden/sys/pdfium-gold-ingestion.json5
diff --git a/tracedb/sys/pdfium-gold-ingestion.service b/golden/sys/pdfium-gold-ingestion.service
similarity index 100%
rename from tracedb/sys/pdfium-gold-ingestion.service
rename to golden/sys/pdfium-gold-ingestion.service
diff --git a/tracedb/sys/skia-gold-ingestion.json5 b/golden/sys/skia-gold-ingestion.json5
similarity index 100%
rename from tracedb/sys/skia-gold-ingestion.json5
rename to golden/sys/skia-gold-ingestion.json5
diff --git a/tracedb/sys/skia-gold-ingestion.service b/golden/sys/skia-gold-ingestion.service
similarity index 100%
rename from tracedb/sys/skia-gold-ingestion.service
rename to golden/sys/skia-gold-ingestion.service
diff --git a/tracedb/Makefile b/tracedb/Makefile
index f8e0c50..429110d 100644
--- a/tracedb/Makefile
+++ b/tracedb/Makefile
@@ -4,32 +4,13 @@
go install -v ./go/traceserver
go install -v ./go/tracetool
go install -v ./go/importtile
- go install -v ./go/skia_ingestion
go install -v ./go/difftile
- # Make local copies of the binaries. It makes running locally easier.
- mv $(GOPATH)/bin/gold_ingestion $(GOPATH)/bin/gold_ingestion_old
- cp $(GOPATH)/bin/skia_ingestion $(GOPATH)/bin/gold_ingestion
- rm $(GOPATH)/bin/gold_ingestion_old
-
- mv $(GOPATH)/bin/pdfium_gold_ingestion $(GOPATH)/bin/pdfium_gold_ingestion_old
- cp $(GOPATH)/bin/skia_ingestion $(GOPATH)/bin/pdfium_gold_ingestion
- rm $(GOPATH)/bin/pdfium_gold_ingestion_old
-
- mv $(GOPATH)/bin/chromevr_gold_ingestion $(GOPATH)/bin/chromevr_gold_ingestion_old
- cp $(GOPATH)/bin/skia_ingestion $(GOPATH)/bin/chromevr_gold_ingestion
- rm $(GOPATH)/bin/chromevr_gold_ingestion_old
-
.PHONY: k8s-binaries
k8s-binaries:
mkdir -p ./build
rm -f ./build/*
$(KGO) -o build/traceserver_k8s -a ./go/traceserver/main.go
- $(KGO) -o build/skia_ingestion_k8s -a ./go/skia_ingestion/main.go
-
-.PHONY: k8s-release-ingestion
-k8s-release-ingestion: k8s-binaries
- ./k8s_push_ingestion
.PHONY: k8s-release-traceserver
k8s-release-traceserver: k8s-binaries
