Revert "[CD] Reland changes to use gitiles/gerrit"
This reverts commit 968d68c94d96110044833465d080a47c77bf433e.
Reason for revert: Still working on quota.
Original change's description:
> [CD] Reland changes to use gitiles/gerrit
>
> This reverts commit 80dcc5fbc3b068a4a1056cb71f693d7dc9c269d4.
>
> Change-Id: I2532718516e6c03e21476ff0f81d5347c57440aa
> Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/631316
> Commit-Queue: Eric Boren <borenet@google.com>
> Reviewed-by: Ravi Mistry <rmistry@google.com>
Change-Id: I409ff02e416003842a03530b8b91d7f9c16c2f3e
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/633996
Reviewed-by: Ravi Mistry <rmistry@google.com>
Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com>
Commit-Queue: Eric Boren <borenet@google.com>
diff --git a/autoroll/go/autoroll-config-converter/BUILD.bazel b/autoroll/go/autoroll-config-converter/BUILD.bazel
index dc3d683..11d6a58 100644
--- a/autoroll/go/autoroll-config-converter/BUILD.bazel
+++ b/autoroll/go/autoroll-config-converter/BUILD.bazel
@@ -14,19 +14,21 @@
"//autoroll/go/config",
"//autoroll/go/config_vars",
"//cd/go/cd",
- "//go/auth",
"//go/chrome_branch",
- "//go/gerrit",
+ "//go/exec",
"//go/git",
+ "//go/gitauth",
"//go/gitiles",
"//go/httputils",
"//go/skerr",
"//go/sklog",
"//kube/go/kube_conf_gen_lib",
+ "//task_driver/go/lib/git_steps",
"//task_driver/go/td",
+ "@org_golang_google_api//oauth2/v2:oauth2",
+ "@org_golang_google_api//option",
"@org_golang_google_protobuf//encoding/protojson",
"@org_golang_google_protobuf//encoding/prototext",
- "@org_golang_x_oauth2//google",
"@org_golang_x_sync//errgroup",
],
)
diff --git a/autoroll/go/autoroll-config-converter/main.go b/autoroll/go/autoroll-config-converter/main.go
index 4f0c91a..7826f4e 100644
--- a/autoroll/go/autoroll-config-converter/main.go
+++ b/autoroll/go/autoroll-config-converter/main.go
@@ -8,7 +8,9 @@
"encoding/json"
"flag"
"fmt"
- "path"
+ "io/fs"
+ "io/ioutil"
+ "os"
"path/filepath"
"sort"
"strings"
@@ -18,18 +20,20 @@
"go.skia.org/infra/autoroll/go/config"
"go.skia.org/infra/autoroll/go/config_vars"
"go.skia.org/infra/cd/go/cd"
- "go.skia.org/infra/go/auth"
"go.skia.org/infra/go/chrome_branch"
- "go.skia.org/infra/go/gerrit"
+ "go.skia.org/infra/go/exec"
"go.skia.org/infra/go/git"
+ "go.skia.org/infra/go/gitauth"
"go.skia.org/infra/go/gitiles"
"go.skia.org/infra/go/httputils"
"go.skia.org/infra/go/skerr"
"go.skia.org/infra/go/sklog"
"go.skia.org/infra/kube/go/kube_conf_gen_lib"
+ "go.skia.org/infra/task_driver/go/lib/git_steps"
"go.skia.org/infra/task_driver/go/td"
- "golang.org/x/oauth2/google"
"golang.org/x/sync/errgroup"
+ "google.golang.org/api/oauth2/v2"
+ "google.golang.org/api/option"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/encoding/prototext"
)
@@ -63,12 +67,16 @@
func main() {
// Flags.
+ src := flag.String("src", "", "Source directory.")
+ dst := flag.String("dst", "", "Destination directory. Outputs will mimic the structure of the source.")
privacySandboxAndroidRepoURL := flag.String("privacy_sandbox_android_repo_url", "", "Repo URL for privacy sandbox on Android.")
privacySandboxAndroidVersionsPath := flag.String("privacy_sandbox_android_versions_path", "", "Path to the file containing the versions of privacy sandbox on Android.")
- srcRepo := flag.String("source-repo", "https://skia.googlesource.com/skia-autoroll-internal-config.git", "URL of the repo which triggered this run.")
+ createCL := flag.Bool("create-cl", false, "If true, creates a CL if any changes were made.")
+ srcRepo := flag.String("source-repo", "", "URL of the repo which triggered this run.")
srcCommit := flag.String("source-commit", "", "Commit hash which triggered this run.")
louhiExecutionID := flag.String("louhi-execution-id", "", "Execution ID of the Louhi flow.")
louhiPubsubProject := flag.String("louhi-pubsub-project", "", "GCP project used for sending Louhi pub/sub notifications.")
+ local := flag.Bool("local", false, "True if running locally.")
flag.Parse()
@@ -82,18 +90,35 @@
ctx := td.StartRun(&fakeProjectId, &fakeTaskId, &fakeTaskName, &output, &tdLocal)
defer td.EndRun(ctx)
+ if *src == "" {
+ td.Fatalf(ctx, "--src is required.")
+ }
+ if *dst == "" {
+ td.Fatalf(ctx, "--dst is required.")
+ }
if backendTemplate == "" {
td.Fatalf(ctx, "internal error; embedded template is empty.")
}
- if *srcCommit == "" {
- td.Fatalf(ctx, "--source-commit is required.")
- }
// Set up auth, load config variables.
- ts, err := google.DefaultTokenSource(ctx, auth.ScopeUserinfoEmail, gerrit.AuthScope)
+ ts, err := git_steps.Init(ctx, true)
if err != nil {
td.Fatal(ctx, err)
}
+ if !*local {
+ srv, err := oauth2.NewService(ctx, option.WithTokenSource(ts))
+ if err != nil {
+ td.Fatal(ctx, err)
+ }
+ info, err := srv.Userinfo.V2.Me.Get().Do()
+ if err != nil {
+ td.Fatal(ctx, err)
+ }
+ sklog.Infof("Authenticated as %s", info.Email)
+ if _, err := gitauth.New(ts, "/tmp/.gitcookies", true, info.Email); err != nil {
+ td.Fatal(ctx, err)
+ }
+ }
client := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()
reg, err := config_vars.NewRegistry(ctx, chrome_branch.NewClient(client))
if err != nil {
@@ -155,87 +180,62 @@
}
sklog.Infof("Using variables: %s", string(b))
- // Walk through all files from the k8s-config repo. Read autoroll-related
- // config files.
- dst := gitiles.NewRepo("https://skia.googlesource.com/k8s-config.git", client)
- dstBaseCommit, err := dst.ResolveRef(ctx, git.DefaultRef)
- if err != nil {
- td.Fatal(ctx, err)
- }
- dstFiles, err := dst.ListFilesRecursiveAtRef(ctx, ".", dstBaseCommit)
- if err != nil {
- td.Fatal(ctx, err)
- }
- dstExistingContents := map[string][]byte{}
- for _, dstFile := range dstFiles {
- if strings.HasSuffix(dstFile, "-autoroll-ns.yaml") || (strings.HasPrefix(dstFile, "autoroll-be-") && strings.HasSuffix(dstFile, ".yaml")) {
- contents, err := dst.ReadFileAtRef(ctx, dstFile, dstBaseCommit)
- if err != nil {
- td.Fatal(ctx, err)
- }
- dstExistingContents[dstFile] = contents
- }
- }
-
- // Walk through the autoroller config repo. Create roller configs from
+ // Walk through the autoroller config directory. Create roller configs from
// templates and convert roller configs to k8s configs.
- src := gitiles.NewRepo(*srcRepo, client)
- srcFiles, err := src.ListFilesRecursiveAtRef(ctx, ".", *srcCommit)
- if err != nil {
- td.Fatal(ctx, err)
- }
- generatedContents := map[string][]byte{}
- for _, srcFile := range srcFiles {
- if strings.HasSuffix(srcFile, ".cfg") {
- sklog.Infof("Converting %s", srcFile)
- cfgBytes, err := src.ReadFileAtRef(ctx, srcFile, *srcCommit)
+ fsys := os.DirFS(*src)
+ if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() {
+ return nil
+ }
+ if strings.HasSuffix(d.Name(), ".cfg") {
+ srcPath := filepath.Join(*src, path)
+ sklog.Infof("Converting %s", srcPath)
+ cfgBytes, err := ioutil.ReadFile(srcPath)
if err != nil {
- td.Fatalf(ctx, "failed to read roller config %s: %s", srcFile, err)
+ return skerr.Wrapf(err, "failed to read roller config %s", srcPath)
}
- if err := convertConfig(ctx, cfgBytes, srcFile, generatedContents); err != nil {
- td.Fatalf(ctx, "failed to convert config %s: %s", srcFile, err)
+
+ if err := convertConfig(ctx, cfgBytes, path, *dst); err != nil {
+ return skerr.Wrapf(err, "failed to convert config %s", path)
}
- } else if strings.HasSuffix(srcFile, ".tmpl") {
- sklog.Infof("Processing %s", srcFile)
- tmplBytes, err := src.ReadFileAtRef(ctx, srcFile, *srcCommit)
+ } else if strings.HasSuffix(d.Name(), ".tmpl") {
+ tmplPath := filepath.Join(*src, path)
+ sklog.Infof("Processing %s", tmplPath)
+ tmplContents, err := ioutil.ReadFile(tmplPath)
if err != nil {
- td.Fatalf(ctx, "failed to read template file %s: %s", srcFile, err)
+ return skerr.Wrapf(err, "failed to read template file %s", tmplPath)
}
- generatedConfigs, err := processTemplate(ctx, srcFile, string(tmplBytes), vars)
+ generatedConfigs, err := processTemplate(ctx, path, string(tmplContents), vars)
if err != nil {
- td.Fatalf(ctx, "failed to process template file %s: %s", srcFile, err)
+ return skerr.Wrapf(err, "failed to process template file %s", path)
}
for path, cfgBytes := range generatedConfigs {
- if err := convertConfig(ctx, cfgBytes, path, generatedContents); err != nil {
- td.Fatalf(ctx, "failed to convert config %s: %s", srcFile, err)
+ if err := convertConfig(ctx, cfgBytes, path, *dst); err != nil {
+ return skerr.Wrapf(err, "failed to convert config %s", path)
}
}
}
+ return nil
+ }); err != nil {
+ td.Fatalf(ctx, "Failed to read configs: %s", err)
}
- // Find the actual changes between the existing and the generated configs.
- changes := make(map[string]string, len(generatedContents))
- // First, "delete" all of the old contents, to ensure that we remove any
- // no-longer-generated rollers.
- for path := range dstExistingContents {
- changes[path] = ""
+ // "git add" the directory.
+ gitExec, err := git.Executable(ctx)
+ if err != nil {
+ td.Fatal(ctx, err)
}
- // Next, overwrite the old contents with the generated contents.
- for path, newContents := range generatedContents {
- changes[path] = string(newContents)
- }
- // Finally, remove any files which didn't actually change.
- for path, newContents := range generatedContents {
- oldContents, ok := dstExistingContents[path]
- if ok && bytes.Equal(oldContents, newContents) {
- delete(changes, path)
- }
+ if _, err := exec.RunCwd(ctx, *dst, gitExec, "add", "-A"); err != nil {
+ td.Fatal(ctx, err)
}
// Upload a CL.
- if len(changes) > 0 {
+ if *createCL {
commitSubject := "Update autoroll k8s configs"
- if err := cd.UploadCL(ctx, changes, "https://skia.googlesource.com/k8s-config.git", dstBaseCommit, commitSubject, *srcRepo, *srcCommit, *louhiPubsubProject, *louhiExecutionID); err != nil {
+ if err := cd.MaybeUploadCL(ctx, *dst, commitSubject, *srcRepo, *srcCommit, *louhiPubsubProject, *louhiExecutionID); err != nil {
td.Fatalf(ctx, "Failed to create CL: %s", err)
}
}
@@ -305,11 +305,11 @@
PrivacySandboxVersions []*PrivacySandboxVersion
}
-func convertConfig(ctx context.Context, cfgBytes []byte, relPath string, generatedContents map[string][]byte) error {
+func convertConfig(ctx context.Context, cfgBytes []byte, relPath, dstDir string) error {
// Decode the config file.
var cfg config.Config
if err := prototext.Unmarshal(cfgBytes, &cfg); err != nil {
- return skerr.Wrapf(err, "failed to parse roller config: %s", string(cfgBytes))
+ return skerr.Wrapf(err, "failed to parse roller config")
}
// Google3 uses a different type of backend.
if cfg.ParentDisplayName == google3ParentName {
@@ -358,24 +358,22 @@
cfgMap["configBase64"] = cfgFileBase64
// Run kube-conf-gen to generate the backend config file.
- relDir, baseName := path.Split(relPath)
- dstCfgPath := path.Join(relDir, fmt.Sprintf("autoroll-be-%s.yaml", strings.Split(baseName, ".")[0]))
- rollerCfg, err := kube_conf_gen_lib.GenerateOutputFromTemplateString(backendTemplate, false, cfgMap)
- if err != nil {
+ relDir, baseName := filepath.Split(relPath)
+ dstPath := filepath.Join(dstDir, relDir, fmt.Sprintf("autoroll-be-%s.yaml", strings.Split(baseName, ".")[0]))
+ if err := kube_conf_gen_lib.GenerateOutputFromTemplateString(backendTemplate, false, cfgMap, dstPath); err != nil {
return skerr.Wrapf(err, "failed to write output")
}
- generatedContents[dstCfgPath] = rollerCfg
+ sklog.Infof("Wrote %s", dstPath)
// Run kube-conf-gen to generate the namespace config file. Note that we'll
// overwrite this file for every roller in the namespace, but that shouldn't
// be a problem, since the generated files will be the same.
namespace := strings.Split(cfg.ServiceAccount, "@")[0]
- dstNsPath := path.Join(relDir, fmt.Sprintf("%s-ns.yaml", namespace))
- nsCfg, err := kube_conf_gen_lib.GenerateOutputFromTemplateString(namespaceTemplate, false, cfgMap)
- if err != nil {
+ dstNsPath := filepath.Join(dstDir, relDir, fmt.Sprintf("%s-ns.yaml", namespace))
+ if err := kube_conf_gen_lib.GenerateOutputFromTemplateString(namespaceTemplate, false, cfgMap, dstNsPath); err != nil {
return skerr.Wrapf(err, "failed to write output")
}
- generatedContents[dstNsPath] = nsCfg
+ sklog.Infof("Wrote %s", dstNsPath)
return nil
}
diff --git a/cd/go/build-images/BUILD.bazel b/cd/go/build-images/BUILD.bazel
index c71b8f8..a26eefd 100644
--- a/cd/go/build-images/BUILD.bazel
+++ b/cd/go/build-images/BUILD.bazel
@@ -12,19 +12,14 @@
visibility = ["//visibility:private"],
deps = [
"//cd/go/cd",
- "//go/auth",
"//go/exec",
- "//go/gerrit",
"//go/git",
"//go/gitauth",
- "//go/gitiles",
- "//go/httputils",
"//go/skerr",
"//go/util",
"//task_driver/go/lib/git_steps",
"//task_driver/go/td",
"@com_github_urfave_cli_v2//:cli",
- "@org_golang_x_oauth2//google",
],
)
diff --git a/cd/go/build-images/update-refs.go b/cd/go/build-images/update-refs.go
index 462c73d..3fe1703 100644
--- a/cd/go/build-images/update-refs.go
+++ b/cd/go/build-images/update-refs.go
@@ -3,33 +3,34 @@
import (
"context"
"fmt"
+ "io/fs"
+ "io/ioutil"
"path"
+ "path/filepath"
"regexp"
"strings"
"go.skia.org/infra/cd/go/cd"
- "go.skia.org/infra/go/auth"
"go.skia.org/infra/go/exec"
- "go.skia.org/infra/go/gerrit"
"go.skia.org/infra/go/git"
- "go.skia.org/infra/go/gitiles"
- "go.skia.org/infra/go/httputils"
+ "go.skia.org/infra/go/gitauth"
"go.skia.org/infra/go/skerr"
+ "go.skia.org/infra/task_driver/go/lib/git_steps"
"go.skia.org/infra/task_driver/go/td"
- "golang.org/x/oauth2/google"
)
-func updateRefs(ctx context.Context, repoURL, workspace, username, email, louhiPubsubProject, louhiExecutionID, srcRepo, srcCommit string) error {
+func updateRefs(ctx context.Context, repo, workspace, username, email, louhiPubsubProject, executionID, srcRepo, srcCommit string) error {
ctx = td.StartStep(ctx, td.Props("Update References"))
defer td.EndStep(ctx)
- // Create the git repo.
- ts, err := google.DefaultTokenSource(ctx, auth.ScopeUserinfoEmail, gerrit.AuthScope)
+ // Initialize git authentication.
+ ts, err := git_steps.Init(ctx, true)
if err != nil {
return td.FailStep(ctx, err)
}
- client := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()
- repo := gitiles.NewRepo(repoURL, client)
+ if _, err := gitauth.New(ts, "/tmp/.gitcookies", true, email); err != nil {
+ return td.FailStep(ctx, err)
+ }
imageInfo, err := readBuildImagesJSON(ctx, workspace)
if err != nil {
@@ -53,58 +54,75 @@
image.Sha256 = strings.TrimSuffix(strings.TrimPrefix(split[1], "sha256:"), "'")
}
- // Obtain the current contents of all files in the repo.
- baseCommit, err := repo.ResolveRef(ctx, git.DefaultRef)
+ // Create a shallow clone of the repo.
+ checkoutDir, err := shallowClone(ctx, repo, git.DefaultRef)
if err != nil {
return td.FailStep(ctx, err)
}
- oldFiles, err := repo.ListFilesRecursiveAtRef(ctx, ".", baseCommit)
- if err != nil {
- return td.FailStep(ctx, err)
- }
- oldContents := map[string][]byte{}
- for _, f := range oldFiles {
- contents, err := repo.ReadFileAtRef(ctx, f, baseCommit)
- if err != nil {
- return td.FailStep(ctx, err)
- }
- oldContents[f] = contents
- }
- // Create regexes for each of the images.
- imageRegexes := make([]*regexp.Regexp, 0, len(imageInfo.Images))
- imageReplace := make([]string, 0, len(imageInfo.Images))
- for _, image := range imageInfo.Images {
- imageRegexes = append(imageRegexes, regexp.MustCompile(fmt.Sprintf(`%s@sha256:[a-f0-9]+`, image.Image)))
- imageReplace = append(imageReplace, fmt.Sprintf("%s@sha256:%s", image.Image, image.Sha256))
+ // Create a branch.
+ gitExec, err := git.Executable(ctx)
+ if err != nil {
+ return td.FailStep(ctx, err)
+ }
+ if _, err := exec.RunCwd(ctx, checkoutDir, gitExec, "checkout", "-b", "update", "-t", git.DefaultRemoteBranch); err != nil {
+ return td.FailStep(ctx, err)
}
// Find-and-replace each of the image references.
- changes := map[string]string{}
- for f, oldFileContents := range oldContents {
- // Replace all instances of the old image specification with the new.
- contentsStr := string(oldFileContents)
- for idx, re := range imageRegexes {
- contentsStr = re.ReplaceAllString(contentsStr, imageReplace[idx])
- }
+ if err := td.Do(ctx, td.Props("Update Image References"), func(ctx context.Context) error {
+ imageRegexes := make([]*regexp.Regexp, 0, len(imageInfo.Images))
+ imageReplace := make([]string, 0, len(imageInfo.Images))
+ for _, image := range imageInfo.Images {
+ // Update instances of "image/path@sha256:digest"
+ imageRegexes = append(imageRegexes, regexp.MustCompile(fmt.Sprintf(`%s@sha256:[a-f0-9]+`, image.Image)))
+ imageReplace = append(imageReplace, fmt.Sprintf("%s@sha256:%s", image.Image, image.Sha256))
- // Write out the updated file.
- newFileContents := contentsStr
- if string(oldFileContents) != newFileContents {
- changes[f] = newFileContents
+ // Replace Bazel container_pull specifications.
+ bazelRegex, bazelReplace := bazelRegexAndReplaceForImage(image)
+ imageRegexes = append(imageRegexes, bazelRegex)
+ imageReplace = append(imageReplace, bazelReplace)
}
+ return filepath.WalkDir(checkoutDir, func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ } else if d.IsDir() {
+ if d.Name() == ".git" {
+ return fs.SkipDir
+ } else {
+ return nil
+ }
+ }
+ // Read the file.
+ contents, err := ioutil.ReadFile(path)
+ if err != nil {
+ return err
+ }
+ contentsStr := string(contents)
+
+ // Replace all instances of the old image specification with the new.
+ for idx, re := range imageRegexes {
+ contentsStr = re.ReplaceAllString(contentsStr, imageReplace[idx])
+ }
+
+ // Write out the updated file.
+ contents = []byte(contentsStr)
+ if err := ioutil.WriteFile(path, contents, d.Type().Perm()); err != nil {
+ return err
+ }
+ return nil
+ })
+ }); err != nil {
+ return td.FailStep(ctx, err)
}
// Upload a CL.
- if len(changes) > 0 {
- imageList := make([]string, 0, len(imageInfo.Images))
- for _, image := range imageInfo.Images {
- imageList = append(imageList, path.Base(image.Image))
- }
- commitSubject := fmt.Sprintf("Update %s", strings.Join(imageList, ", "))
- return cd.UploadCL(ctx, changes, repoURL, baseCommit, commitSubject, srcRepo, srcCommit, louhiPubsubProject, louhiExecutionID)
+ imageList := make([]string, 0, len(imageInfo.Images))
+ for _, image := range imageInfo.Images {
+ imageList = append(imageList, path.Base(image.Image))
}
- return nil
+ commitSubject := fmt.Sprintf("Update %s", strings.Join(imageList, ", "))
+ return cd.MaybeUploadCL(ctx, checkoutDir, commitSubject, srcRepo, srcCommit, louhiPubsubProject, executionID)
}
func bazelRegexAndReplaceForImage(image *SingleImageInfo) (*regexp.Regexp, string) {
diff --git a/cd/go/cd/BUILD.bazel b/cd/go/cd/BUILD.bazel
index aab1a0c..780d35a 100644
--- a/cd/go/cd/BUILD.bazel
+++ b/cd/go/cd/BUILD.bazel
@@ -1,3 +1,4 @@
+load("//bazel/go:go_test.bzl", "go_test")
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
@@ -7,7 +8,7 @@
visibility = ["//visibility:public"],
deps = [
"//go/auth",
- "//go/gerrit",
+ "//go/exec",
"//go/gerrit/rubberstamper",
"//go/git",
"//go/gitiles",
@@ -19,3 +20,10 @@
"@org_golang_x_oauth2//google",
],
)
+
+go_test(
+ name = "cd_test",
+ srcs = ["cd_test.go"],
+ embed = [":cd"],
+ deps = ["@com_github_stretchr_testify//require"],
+)
diff --git a/cd/go/cd/cd.go b/cd/go/cd/cd.go
index 5e2e36d..0115b2e 100644
--- a/cd/go/cd/cd.go
+++ b/cd/go/cd/cd.go
@@ -3,9 +3,10 @@
import (
"context"
"fmt"
+ "regexp"
"go.skia.org/infra/go/auth"
- "go.skia.org/infra/go/gerrit"
+ "go.skia.org/infra/go/exec"
"go.skia.org/infra/go/gerrit/rubberstamper"
"go.skia.org/infra/go/git"
"go.skia.org/infra/go/gitiles"
@@ -17,73 +18,79 @@
"golang.org/x/oauth2/google"
)
-// UploadCL uploads a CL with the given changes. It builds the commit message
-// starting with the given commitSubject. If srcRepo and srcCommit are provided,
-// a link back to the source commit is added to the commit message. If
-// louhiPubsubProject and louhiExecutionID are provided, a pub/sub message is
-// sent after the CL is uploaded.
-func UploadCL(ctx context.Context, changes map[string]string, dstRepo, baseCommit, commitSubject, srcRepo, srcCommit, louhiPubsubProject, louhiExecutionID string) error {
- ctx = td.StartStep(ctx, td.Props("UploadCL"))
+var uploadedCLRegex = regexp.MustCompile(`https://.*review\.googlesource\.com.*\d+`)
+
+// MaybeUploadCL uploads a CL if there are any diffs in checkoutDir. It builds
+// the commit message starting with the given commitSubject. If srcRepo and
+// srcCommit are provided, a link back to the source commit is added to the
+// commit message. If louhiPubsubProject and louhiExecutionID are provided,
+// a pub/sub message is sent after the CL is uploaded.
+func MaybeUploadCL(ctx context.Context, checkoutDir, commitSubject, srcRepo, srcCommit, louhiPubsubProject, louhiExecutionID string) error {
+ ctx = td.StartStep(ctx, td.Props("MaybeUploadCL"))
defer td.EndStep(ctx)
- // Build the commit message.
- commitMsg := commitSubject
- if srcCommit != "" {
- shortCommit := srcCommit
- if len(shortCommit) > 12 {
- shortCommit = shortCommit[:12]
- }
- commitMsg += " for " + shortCommit
+ gitExec, err := git.Executable(ctx)
+ if err != nil {
+ return skerr.Wrap(err)
}
- commitMsg += "\n\n"
- if srcRepo != "" && srcCommit != "" {
- ts, err := google.DefaultTokenSource(ctx, auth.ScopeUserinfoEmail)
- if err != nil {
- return skerr.Wrap(err)
+
+ // Did we change anything?
+ if _, err := exec.RunCwd(ctx, checkoutDir, gitExec, "diff", "HEAD", "--exit-code"); err != nil {
+ // If so, create a CL.
+
+ // Build the commit message.
+ commitMsg := commitSubject
+ if srcCommit != "" {
+ shortCommit := srcCommit
+ if len(shortCommit) > 12 {
+ shortCommit = shortCommit[:12]
+ }
+ commitMsg += " for " + shortCommit
}
- client := httputils.DefaultClientConfig().WithTokenSource(ts).Client()
- gitilesRepo := gitiles.NewRepo(srcRepo, client)
- commitDetails, err := gitilesRepo.Details(ctx, srcCommit)
- if err != nil {
- return skerr.Wrap(err)
- }
- commitMsg += fmt.Sprintf("%s/+/%s\n\n", srcRepo, srcCommit)
- commitMsg += commitDetails.Subject
commitMsg += "\n\n"
- }
+ if srcRepo != "" && srcCommit != "" {
+ ts, err := google.DefaultTokenSource(ctx, auth.ScopeUserinfoEmail)
+ if err != nil {
+ return skerr.Wrap(err)
+ }
+ client := httputils.DefaultClientConfig().WithTokenSource(ts).Client()
+ gitilesRepo := gitiles.NewRepo(srcRepo, client)
+ commitDetails, err := gitilesRepo.Details(ctx, srcCommit)
+ if err != nil {
+ return skerr.Wrap(err)
+ }
+ commitMsg += fmt.Sprintf("%s/+/%s\n\n", srcRepo, srcCommit)
+ commitMsg += commitDetails.Subject
+ commitMsg += "\n\n"
+ }
+ commitMsg += rubberstamper.RandomChangeID()
- // Create the CL.
- gerritURL, gerritProject, err := gerrit.ParseGerritURLAndProject(dstRepo)
- if err != nil {
- return td.FailStep(ctx, err)
- }
- ts, err := google.DefaultTokenSource(ctx, gerrit.AuthScope)
- if err != nil {
- return td.FailStep(ctx, err)
- }
- client := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()
- g, err := gerrit.NewGerrit(gerritURL, client)
- if err != nil {
- return td.FailStep(ctx, err)
- }
- reviewers := []string{rubberstamper.RubberStamperUser}
- ci, err := gerrit.CreateCLWithChanges(ctx, g, gerritProject, git.MainBranch, commitMsg, baseCommit, changes, reviewers)
- if err != nil {
- return td.FailStep(ctx, err)
- }
-
- // Send a pub/sub message.
- if louhiPubsubProject != "" && louhiExecutionID != "" {
- sender, err := pubsub.NewPubSubSender(ctx, louhiPubsubProject)
+ // Commit and push.
+ if _, err := exec.RunCwd(ctx, checkoutDir, gitExec, "commit", "-a", "-m", commitMsg); err != nil {
+ return skerr.Wrap(err)
+ }
+ output, err := exec.RunCwd(ctx, checkoutDir, gitExec, "push", git.DefaultRemote, rubberstamper.PushRequestAutoSubmit)
if err != nil {
return skerr.Wrap(err)
}
- if err := sender.Send(ctx, &louhi.Notification{
- EventAction: louhi.EventAction_CREATED_ARTIFACT,
- GeneratedCls: []string{g.Url(ci.Issue)},
- PipelineExecutionId: louhiExecutionID,
- }); err != nil {
- return skerr.Wrap(err)
+
+ // Send a pub/sub message.
+ if louhiPubsubProject != "" && louhiExecutionID != "" {
+ match := uploadedCLRegex.FindString(output)
+ if match == "" {
+ return skerr.Fmt("Failed to parse CL link from:\n%s", output)
+ }
+ sender, err := pubsub.NewPubSubSender(ctx, louhiPubsubProject)
+ if err != nil {
+ return skerr.Wrap(err)
+ }
+ if err := sender.Send(ctx, &louhi.Notification{
+ EventAction: louhi.EventAction_CREATED_ARTIFACT,
+ GeneratedCls: []string{match},
+ PipelineExecutionId: louhiExecutionID,
+ }); err != nil {
+ return skerr.Wrap(err)
+ }
}
}
return nil
diff --git a/cd/go/cd/cd_test.go b/cd/go/cd/cd_test.go
new file mode 100644
index 0000000..bdb6bc6
--- /dev/null
+++ b/cd/go/cd/cd_test.go
@@ -0,0 +1,36 @@
+package cd
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestUploadedCLRegex(t *testing.T) {
+ const logOutput = `
+ path/to/my-file.txt | 2 +
+1 files changed, 376 insertions(+), 172 deletions(-)
+Waiting for editor...
+
+Enumerating objects: 68, done.
+Counting objects: 100% (68/68), done.
+Delta compression using up to 48 threads
+Compressing objects: 100% (53/53), done.
+Writing objects: 100% (54/54), 18.60 KiB | 3.72 MiB/s, done.
+Total 54 (delta 29), reused 0 (delta 0), pack-reused 0
+remote: Resolving deltas: 100% (29/29)
+remote: Waiting for private key checker: 5/30 objects left
+remote: Processing changes: refs: 1, new: 1, done
+remote: commit 321f209: warning: subject >50 characters; use shorter first paragraph
+remote: commit 321f209: warning: too many message lines longer than 72 characters; manually wrap lines
+remote:
+remote: SUCCESS
+remote:
+remote: https://my-project-review.googlesource.com/c/my-repo/+/12345 Commit Message [WIP] [NEW]
+remote:
+To https://my-project.googlesource.com/my-repo.git
+* [new reference] 321f209aecdaf8f0a276d7d8413fd5b524f1c985 -> refs/for/refs/heads/main%wip,m=Initial_upload,cc=reviews@my-project.org,l=Commit-Queue+1,hashtag=blah
+`
+ const expect = "https://my-project-review.googlesource.com/c/my-repo/+/12345"
+ require.Equal(t, expect, uploadedCLRegex.FindString(logOutput))
+}
diff --git a/go/gerrit/gerrit.go b/go/gerrit/gerrit.go
index e9fcc67..f2e0aa5 100644
--- a/go/gerrit/gerrit.go
+++ b/go/gerrit/gerrit.go
@@ -1681,19 +1681,3 @@
}
return resp, err
}
-
-// ParseGerritURLAndProject extracts the Gerrit URL and project name from the
-// given Gitiles repo URL.
-func ParseGerritURLAndProject(gitilesRepoURL string) (string, string, error) {
- parsed, err := url.Parse(gitilesRepoURL)
- if err != nil {
- return "", "", skerr.Wrap(err)
- }
- project := strings.TrimPrefix(strings.TrimSuffix(parsed.Path, ".git"), "/")
- splitHost := strings.Split(parsed.Host, ".")
- splitHost[0] = splitHost[0] + "-review"
- parsed.Host = strings.Join(splitHost, ".")
- parsed.Path = ""
- gerritURL := parsed.String()
- return gerritURL, project, nil
-}
diff --git a/go/gerrit/gerrit_test.go b/go/gerrit/gerrit_test.go
index 6013b92..47e177f 100644
--- a/go/gerrit/gerrit_test.go
+++ b/go/gerrit/gerrit_test.go
@@ -646,14 +646,3 @@
ci.Branch = "refs/heads/chrome/m90"
require.Equal(t, "chromium%2Fsrc~chrome%2Fm90~abc", FullChangeId(ci))
}
-
-func TestParseGerritURLAndProject(t *testing.T) {
- test := func(repoURL, expectGerritURL, expectProject string) {
- actualGerritURL, actualProject, err := ParseGerritURLAndProject(repoURL)
- require.NoError(t, err)
- require.Equal(t, expectGerritURL, actualGerritURL)
- require.Equal(t, expectProject, actualProject)
- }
- test("https://skia.googlesource.com/k8s-config.git", "https://skia-review.googlesource.com", "k8s-config")
- test("https://chromium.googlesource.com/chromium/src", "https://chromium-review.googlesource.com", "chromium/src")
-}
diff --git a/go/gitiles/gitiles.go b/go/gitiles/gitiles.go
index 5bf464a..80f1b39 100644
--- a/go/gitiles/gitiles.go
+++ b/go/gitiles/gitiles.go
@@ -12,7 +12,6 @@
"net/http"
"net/url"
"os"
- "path"
"sort"
"strconv"
"strings"
@@ -187,9 +186,6 @@
// and FileInfo.
func (r *Repo) ReadObject(ctx context.Context, path, ref string) (os.FileInfo, []byte, error) {
path = strings.TrimSuffix(path, "/")
- if path == "." {
- path = ""
- }
resp, err := r.get(ctx, fmt.Sprintf(DownloadURL, r.url, ref, path))
if err != nil {
return nil, nil, skerr.Wrap(err)
@@ -292,13 +288,12 @@
return err
}
for _, fi := range infos {
- fullPath := path.Join(dir, fi.Name())
if fi.IsDir() {
- if err := helper(fullPath); err != nil {
+ if err := helper(dir + "/" + fi.Name()); err != nil {
return err
}
} else {
- rv = append(rv, strings.TrimPrefix(fullPath, topDir+"/"))
+ rv = append(rv, strings.TrimPrefix(dir+"/"+fi.Name(), topDir+"/"))
}
}
return nil
diff --git a/go/gitiles/testutils/testutils.go b/go/gitiles/testutils/testutils.go
index d45b032..c16622b 100644
--- a/go/gitiles/testutils/testutils.go
+++ b/go/gitiles/testutils/testutils.go
@@ -53,11 +53,7 @@
require.NoError(mr.t, err)
body := make([]byte, base64.StdEncoding.EncodedLen(len(contents)))
base64.StdEncoding.Encode(body, contents)
- mockURLPath := srcPath
- if srcPath == "." {
- mockURLPath = ""
- }
- url := fmt.Sprintf(gitiles.DownloadURL, mr.url, ref, mockURLPath)
+ url := fmt.Sprintf(gitiles.DownloadURL, mr.url, ref, srcPath)
md := mockhttpclient.MockGetDialogue(body)
typ := git.ObjectTypeBlob
if st.IsDir() {
diff --git a/kube/go/kube-conf-gen/main.go b/kube/go/kube-conf-gen/main.go
index abd9ee6..f35835d 100644
--- a/kube/go/kube-conf-gen/main.go
+++ b/kube/go/kube-conf-gen/main.go
@@ -47,7 +47,7 @@
}
// Generate the output.
- if err := kube_conf_gen_lib.WriteOutputFromTemplateFile(*templateFileName, *strict, config, *outputFileName); err != nil {
+ if err := kube_conf_gen_lib.GenerateOutputFromTemplateFile(*templateFileName, *strict, config, *outputFileName); err != nil {
sklog.Fatal(err)
}
}
diff --git a/kube/go/kube_conf_gen_lib/kube_conf_gen_lib.go b/kube/go/kube_conf_gen_lib/kube_conf_gen_lib.go
index dc8ca30..4f1a946 100644
--- a/kube/go/kube_conf_gen_lib/kube_conf_gen_lib.go
+++ b/kube/go/kube_conf_gen_lib/kube_conf_gen_lib.go
@@ -97,59 +97,37 @@
}
// GenerateOutputFromTemplateString executes the template string with config as
-// its environment and returns the generated contents.
-func GenerateOutputFromTemplateString(tmplString string, strict bool, config map[string]interface{}) ([]byte, error) {
+// its environment and writes the result to outFile.
+func GenerateOutputFromTemplateString(tmplString string, strict bool, config map[string]interface{}, outFile string) error {
tmpl, err := template.New("kube-conf-gen-tmpl").Funcs(sprig.TxtFuncMap()).Parse(tmplString)
if err != nil {
- return nil, skerr.Wrapf(err, "error parsing template %s", tmplString)
+ return skerr.Wrapf(err, "error parsing template %s", tmplString)
}
- return generateOutputHelper(tmpl, strict, config)
+ return generateOutputHelper(tmpl, strict, config, outFile)
}
// GenerateOutputFromTemplateFile executes the template file with config as its
-// environment and returns the generated contents.
-func GenerateOutputFromTemplateFile(templateFileName string, strict bool, config map[string]interface{}) ([]byte, error) {
+// environment and writes the result to outFile.
+func GenerateOutputFromTemplateFile(templateFileName string, strict bool, config map[string]interface{}, outFile string) error {
tmpl, err := template.New(path.Base(templateFileName)).Funcs(sprig.TxtFuncMap()).ParseFiles(templateFileName)
if err != nil {
- return nil, skerr.Wrapf(err, "error parsing template '%s'", templateFileName)
+ return skerr.Wrapf(err, "error parsing template '%s'", templateFileName)
}
- return generateOutputHelper(tmpl, strict, config)
+ return generateOutputHelper(tmpl, strict, config, outFile)
}
-// WriteOutputFromTemplateString executes the template string with config as its
-// environment and writes the result to outFile.
-func WriteOutputFromTemplateString(tmplString string, strict bool, config map[string]interface{}, outFile string) error {
- contents, err := GenerateOutputFromTemplateString(tmplString, strict, config)
- if err != nil {
- return err
- }
- return writeOutputHelper(outFile, contents)
-}
-
-// WriteOutputFromTemplateFile executes the template file with config as its
-// environment and writes the result to outFile.
-func WriteOutputFromTemplateFile(templateFileName string, strict bool, config map[string]interface{}, outFile string) error {
- contents, err := GenerateOutputFromTemplateFile(templateFileName, strict, config)
- if err != nil {
- return err
- }
- return writeOutputHelper(outFile, contents)
-}
-
-func generateOutputHelper(tmpl *template.Template, strict bool, config map[string]interface{}) ([]byte, error) {
+func generateOutputHelper(tmpl *template.Template, strict bool, config map[string]interface{}, outFile string) error {
if strict {
tmpl.Option("missingkey=error")
}
+
var buf bytes.Buffer
if err := tmpl.Execute(&buf, config); err != nil {
- return nil, skerr.Wrap(err)
+ return skerr.Wrap(err)
}
- return buf.Bytes(), nil
-}
-func writeOutputHelper(outFile string, contents []byte) error {
if outFile == "_" {
- fmt.Println(string(contents))
+ fmt.Println(string(buf.Bytes()))
return nil
} else {
dir, _ := filepath.Split(outFile)
@@ -158,6 +136,6 @@
return skerr.Wrapf(err, "failed to create destination directory")
}
}
- return skerr.Wrap(ioutil.WriteFile(outFile, contents, 0644))
+ return skerr.Wrap(ioutil.WriteFile(outFile, buf.Bytes(), 0644))
}
}
