kube/go/pushk/main.go - buildbot - Git at Google

 // pushk pushes a new version of an app.
 //
 // pushk docserver
 // pushk --rollback docserver
 // pushk --cluster=skia-public docserver
 // pushk --rollback --cluster=skia-corp docserver
 // pushk --dry-run my-new-service
 package main

 import (
 	"context"
 	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
 	"sort"
 	"strings"

 	"go.skia.org/infra/go/auth"
 	"go.skia.org/infra/go/common"
 	"go.skia.org/infra/go/exec"
 	"go.skia.org/infra/go/gcr"
 	"go.skia.org/infra/go/git"
 	"go.skia.org/infra/go/sklog"
 	"go.skia.org/infra/go/util"
 )

 const (
 	repoURLTemplate = "https://skia.googlesource.com/%s-config"
 	repoBaseDir     = "/tmp"
 	repoDirTemplate = "/tmp/%s-config"

 	containerRegistryProject = "skia-public"

 	maxListSize = 10
 )

 // Project is used to map cluster name to GCE project info.
 type Project struct {
 	Zone    string
 	Project string // The full project name, e.g. google.com:skia-corp.
 }

 var (
 	clusters = map[string]*Project{
 		"skia-public": {
 			Zone:    "us-central1-a",
 			Project: "skia-public",
 		},
 		"skia-corp": {
 			Zone:    "us-central1-a",
 			Project: "google.com:skia-corp",
 		},
 	}
 )

 func init() {
 	flag.Usage = func() {
 		fmt.Printf("Usage: pushk <flags> [one or more image names]\n\n")
 		fmt.Printf(`pushk pushes a new version of an app.

 The command:
   1. Modifies the kubernetes yaml files with the new image.
   2. Commits the changes to the config repo.
   3. Applies the changes with kubectl.

 The config is stored in a separate repo that will automaticaly be checked out
 under /tmp.

 The command applies the changes by default, or just changes the local yaml files
 if --dry-run is supplied.

 Examples:
   # Pusk an exact tag.
   pushk gcr.io/skia-public/fiddler:694900e3ca9468784a5794dc53382d1c8411ab07

   # Push the latest version of docserver.
   pushk docserver --message="Fix bug #1234"

   # Push the latest version of docserver to the skia-corp cluster.
   pushk docserver --cluster=skia-corp --message="Fix bug #1234"

   # Push the latest version of docserver and iap-proxy
   pushk docserver iap-proxy

   # Rollback docserver.
   pushk --rollback docserver

   # List the last few versions of the docserver image. Doesn't apply anything.
   pushk --list docserver

   # Compute any changes a push to docserver will make, but do not apply them.
   pushk --dry-run docserver

 `)
 		flag.PrintDefaults()
 	}
 }

 // toFullRepoURL converts the project name into a git repo URL.
 func toFullRepoURL(s string) string {
 	return fmt.Sprintf(repoURLTemplate, s)

 }

 // toRepoDir converts the project name into a git repo directory name.
 func toRepoDir(s string) string {
 	return fmt.Sprintf(repoDirTemplate, s)
 }

 // flags
 var (
 	cluster     = flag.String("cluster", "skia-public", "Either 'skia-public' or 'skia-corp'.")
 	dryRun      = flag.Bool("dry-run", false, "If true then do not run the kubectl command to apply the changes, and do not commit the changes to the config repo.")
 	ignoreDirty = flag.Bool("ignore-dirty", false, "If true, then do not fail out if the git repo is dirty.")
 	message     = flag.String("message", "Push", "Message to go along with the change.")
 	rollback    = flag.Bool("rollback", false, "If true go back to the second most recent image, otherwise use most recent image.")
 	list        = flag.Bool("list", false, "List the last few versions of the given image.")
 )

 var (
 	validTag = regexp.MustCompile(`^\d\d\d\d-\d\d-\d\dT\d\d_\d\d_\d\dZ-.+$`)
 )

 // filter strips the list of tags down to only the ones that conform to our
 // constraints and also checks that there are enough tags. The results
 // are sorted in ascending order, so oldest tags are first, newest tags
 // are last.
 func filter(tags []string) ([]string, error) {
 	validTags := []string{}
 	for _, t := range tags {
 		if validTag.MatchString(t) {
 			validTags = append(validTags, t)
 		}
 	}
 	sort.Strings(validTags)
 	if len(validTags) == 0 {
 		return nil, fmt.Errorf("Not enough tags returned.")
 	}
 	return validTags, nil
 }

 // tagProvider is a type that returns the correct tag to push for the given imageName.
 type tagProvider func(imageName string) ([]string, error)

 // imageFromCmdLineImage handles image names, which can be either short, ala 'fiddler', or exact,
 // such as gcr.io/skia-public/fiddler:694900e3ca9468784a5794dc53382d1c8411ab07, both of which can
 // appear on the command-line.
 func imageFromCmdLineImage(imageName string, tp tagProvider) (string, error) {
 	if strings.HasPrefix(imageName, "gcr.io/") {
 		if *rollback {
 			return "", fmt.Errorf("Supplying a fully qualified image name and the --rollback flag are mutually exclusive.")
 		}
 		if *list {
 			return "", fmt.Errorf("Supplying a fully qualified image name and the --list flag are mutually exclusive.")
 		}
 		return imageName, nil
 	}
 	// Get all the tags for the selected image.
 	tags, err := tp(imageName)
 	if err != nil {
 		return "", fmt.Errorf("Tag provider failed: %s", err)
 	}

 	// Filter the tags
 	tags, err = filter(tags)
 	if err != nil {
 		return "", fmt.Errorf("Failed to filter: %s", err)
 	}

 	if *list {
 		if len(tags) > maxListSize {
 			tags = tags[len(tags)-maxListSize:]
 		}
 		for _, tag := range tags {
 			fmt.Println(tag)
 		}
 	}

 	// Pick the target tag we want to move to.
 	tag := tags[len(tags)-1]
 	if *rollback {
 		if len(tags) < 2 {
 			return "", fmt.Errorf("No version to rollback to.")
 		}
 		tag = tags[len(tags)-2]
 	}

 	// The full docker image name and tag of the image we want to deploy.
 	return fmt.Sprintf("%s/%s/%s:%s", gcr.SERVER, containerRegistryProject, imageName, tag), nil
 }

 func main() {
 	common.Init()

 	ctx := context.Background()
 	repoDir := toRepoDir(*cluster)
 	checkout, err := git.NewCheckout(ctx, toFullRepoURL(*cluster), repoBaseDir)
 	if err != nil {
 		sklog.Fatalf("Failed to check out config repo: %s", err)
 	}
 	output, err := checkout.Git(ctx, "status", "-s")
 	if err != nil {
 		sklog.Fatal(err)
 	}
 	if strings.TrimSpace(output) != "" {
 		if !*ignoreDirty {
 			sklog.Fatalf("Found dirty checkout in %s:\n%s", repoBaseDir, output)
 		}
 	} else {
 		if err := checkout.Update(ctx); err != nil {
 			sklog.Fatal(err)
 		}
 	}

 	// Switch kubectl to the right project.
 	p := clusters[*cluster]
 	if p == nil {
 		fmt.Printf("Invalid value for --cluster flag: %q", *cluster)
 		flag.Usage()
 		os.Exit(1)
 	}

 	if err := exec.Run(context.Background(), &exec.Command{
 		Name: "gcloud",
 		Args: []string{
 			"container",
 			"clusters",
 			"get-credentials",
 			*cluster,
 			"--zone",
 			p.Zone,
 			"--project",
 			p.Project,
 		},
 		LogStderr: true,
 		LogStdout: true,
 	}); err != nil {
 		sklog.Errorf("Failed to run: %s", err)
 	}

 	// Get all the yaml files.
 	filenames, err := filepath.Glob(filepath.Join(repoDir, "*.yaml"))
 	if err != nil {
 		sklog.Fatal(err)
 	}

 	tokenSource := auth.NewGCloudTokenSource(containerRegistryProject)
 	imageNames := flag.Args()
 	if len(imageNames) == 0 {
 		fmt.Printf("At least one image name needs to be supplied.")
 		flag.Usage()
 		os.Exit(1)
 	}
 	sklog.Infof("Pushing the following images: %q", imageNames)

 	gcrTagProvider := func(imageName string) ([]string, error) {
 		return gcr.NewClient(tokenSource, containerRegistryProject, imageName).Tags()
 	}

 	changed := util.StringSet{}
 	for _, imageName := range imageNames {
 		image, err := imageFromCmdLineImage(imageName, gcrTagProvider)
 		if err != nil {
 			sklog.Fatal(err)
 		}
 		if *list {
 			// imageFromCmdLineImage printed out the tags, so nothing more to do.
 			continue
 		}

 		// imageRegex has the following groups returned on match:
 		// 0 - the entire line
 		// 1 - the prefix, i.e. image:, with correct spacing.
 		// 2 - full image name
 		// 3 - just the tag
 		//
 		// We pull out the 'prefix' so we can use it when
 		// we rewrite the image: line so the indent level is
 		// unchanged.
 		parts := strings.SplitN(image, ":", 2)
 		if len(parts) != 2 {
 			sklog.Fatalf("Failed to split imageName: %v", parts)
 		}
 		imageNoTag := parts[0]
 		imageRegex := regexp.MustCompile(fmt.Sprintf(`^(\s+image:\s+)(%s):.*$`, imageNoTag))

 		// Loop over all the yaml files and update tags for the given imageName.
 		for _, filename := range filenames {
 			b, err := ioutil.ReadFile(filename)
 			if err != nil {
 				sklog.Errorf("Failed to read %q (skipping): %s", filename, err)
 				continue
 			}
 			lines := strings.Split(string(b), "\n")
 			for i, line := range lines {
 				matches := imageRegex.FindStringSubmatch(line)
 				if len(matches) != 3 {
 					continue
 				}
 				changed[filename] = true
 				lines[i] = matches[1] + image
 			}
 			if changed[filename] {
 				err := util.WithWriteFile(filename, func(w io.Writer) error {
 					_, err := w.Write([]byte(strings.Join(lines, "\n")))
 					return err
 				})
 				if err != nil {
 					sklog.Fatalf("Failed to write update config file %q: %s", filename, err)
 				}
 			}
 		}
 	}

 	// Were any files updated?
 	if len(changed) != 0 {
 		filenameFlag := fmt.Sprintf("--filename=%s\n", strings.Join(changed.Keys(), ","))
 		if !*dryRun {
 			for filename := range changed {
 				msg, err := checkout.Git(ctx, "add", filepath.Base(filename))
 				if err != nil {
 					sklog.Fatalf("Failed to stage changes to the config repo: %s: %q", err, msg)
 				}
 			}
 			msg, err := checkout.Git(ctx, "diff", "--cached", "--name-only")
 			if err != nil {
 				sklog.Fatalf("Failed to diff :%s: %q", err, msg)
 			}
 			if msg == "" {
 				sklog.Infof("Not pushing since no files changed.")
 				return
 			}
 			msg, err = checkout.Git(ctx, "commit", "-m", *message)
 			if err != nil {
 				sklog.Fatalf("Failed to commit to the config repo: %s: %q", err, msg)
 			}
 			msg, err = checkout.Git(ctx, "push", "origin", "master")
 			if err != nil {
 				sklog.Fatalf("Failed to push the config repo: %s: %q", err, msg)
 			}
 			if err := exec.Run(context.Background(), &exec.Command{
 				Name:      "kubectl",
 				Args:      []string{"apply", filenameFlag},
 				LogStderr: true,
 				LogStdout: true,
 			}); err != nil {
 				sklog.Errorf("Failed to run: %s", err)
 			}
 		} else {
 			fmt.Printf("\nkubectl apply %s\n", filenameFlag)
 		}
 	} else {
 		fmt.Println("Nothing to do.")
 	}
 }
	// pushk pushes a new version of an app.
	//
	// pushk docserver
	// pushk --rollback docserver
	// pushk --cluster=skia-public docserver
	// pushk --rollback --cluster=skia-corp docserver
	// pushk --dry-run my-new-service
	package main

	import (
	"context"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/gcr"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	)

	const (
	repoURLTemplate = "https://skia.googlesource.com/%s-config"
	repoBaseDir = "/tmp"
	repoDirTemplate = "/tmp/%s-config"

	containerRegistryProject = "skia-public"

	maxListSize = 10
	)

	// Project is used to map cluster name to GCE project info.
	type Project struct {
	Zone string
	Project string // The full project name, e.g. google.com:skia-corp.
	}

	var (
	clusters = map[string]*Project{
	"skia-public": {
	Zone: "us-central1-a",
	Project: "skia-public",
	},
	"skia-corp": {
	Zone: "us-central1-a",
	Project: "google.com:skia-corp",
	},
	}
	)

	func init() {
	flag.Usage = func() {
	fmt.Printf("Usage: pushk <flags> [one or more image names]\n\n")
	fmt.Printf(`pushk pushes a new version of an app.

	The command:
	1. Modifies the kubernetes yaml files with the new image.
	2. Commits the changes to the config repo.
	3. Applies the changes with kubectl.

	The config is stored in a separate repo that will automaticaly be checked out
	under /tmp.

	The command applies the changes by default, or just changes the local yaml files
	if --dry-run is supplied.

	Examples:
	# Pusk an exact tag.
	pushk gcr.io/skia-public/fiddler:694900e3ca9468784a5794dc53382d1c8411ab07

	# Push the latest version of docserver.
	pushk docserver --message="Fix bug #1234"

	# Push the latest version of docserver to the skia-corp cluster.
	pushk docserver --cluster=skia-corp --message="Fix bug #1234"

	# Push the latest version of docserver and iap-proxy
	pushk docserver iap-proxy

	# Rollback docserver.
	pushk --rollback docserver

	# List the last few versions of the docserver image. Doesn't apply anything.
	pushk --list docserver

	# Compute any changes a push to docserver will make, but do not apply them.
	pushk --dry-run docserver

	`)
	flag.PrintDefaults()
	}
	}

	// toFullRepoURL converts the project name into a git repo URL.
	func toFullRepoURL(s string) string {
	return fmt.Sprintf(repoURLTemplate, s)

	}

	// toRepoDir converts the project name into a git repo directory name.
	func toRepoDir(s string) string {
	return fmt.Sprintf(repoDirTemplate, s)
	}

	// flags
	var (
	cluster = flag.String("cluster", "skia-public", "Either 'skia-public' or 'skia-corp'.")
	dryRun = flag.Bool("dry-run", false, "If true then do not run the kubectl command to apply the changes, and do not commit the changes to the config repo.")
	ignoreDirty = flag.Bool("ignore-dirty", false, "If true, then do not fail out if the git repo is dirty.")
	message = flag.String("message", "Push", "Message to go along with the change.")
	rollback = flag.Bool("rollback", false, "If true go back to the second most recent image, otherwise use most recent image.")
	list = flag.Bool("list", false, "List the last few versions of the given image.")
	)

	var (
	validTag = regexp.MustCompile(`^\d\d\d\d-\d\d-\d\dT\d\d_\d\d_\d\dZ-.+$`)
	)

	// filter strips the list of tags down to only the ones that conform to our
	// constraints and also checks that there are enough tags. The results
	// are sorted in ascending order, so oldest tags are first, newest tags
	// are last.
	func filter(tags []string) ([]string, error) {
	validTags := []string{}
	for _, t := range tags {
	if validTag.MatchString(t) {
	validTags = append(validTags, t)
	}
	}
	sort.Strings(validTags)
	if len(validTags) == 0 {
	return nil, fmt.Errorf("Not enough tags returned.")
	}
	return validTags, nil
	}

	// tagProvider is a type that returns the correct tag to push for the given imageName.
	type tagProvider func(imageName string) ([]string, error)

	// imageFromCmdLineImage handles image names, which can be either short, ala 'fiddler', or exact,
	// such as gcr.io/skia-public/fiddler:694900e3ca9468784a5794dc53382d1c8411ab07, both of which can
	// appear on the command-line.
	func imageFromCmdLineImage(imageName string, tp tagProvider) (string, error) {
	if strings.HasPrefix(imageName, "gcr.io/") {
	if *rollback {
	return "", fmt.Errorf("Supplying a fully qualified image name and the --rollback flag are mutually exclusive.")
	}
	if *list {
	return "", fmt.Errorf("Supplying a fully qualified image name and the --list flag are mutually exclusive.")
	}
	return imageName, nil
	}
	// Get all the tags for the selected image.
	tags, err := tp(imageName)
	if err != nil {
	return "", fmt.Errorf("Tag provider failed: %s", err)
	}

	// Filter the tags
	tags, err = filter(tags)
	if err != nil {
	return "", fmt.Errorf("Failed to filter: %s", err)
	}

	if *list {
	if len(tags) > maxListSize {
	tags = tags[len(tags)-maxListSize:]
	}
	for _, tag := range tags {
	fmt.Println(tag)
	}
	}

	// Pick the target tag we want to move to.
	tag := tags[len(tags)-1]
	if *rollback {
	if len(tags) < 2 {
	return "", fmt.Errorf("No version to rollback to.")
	}
	tag = tags[len(tags)-2]
	}

	// The full docker image name and tag of the image we want to deploy.
	return fmt.Sprintf("%s/%s/%s:%s", gcr.SERVER, containerRegistryProject, imageName, tag), nil
	}

	func main() {
	common.Init()

	ctx := context.Background()
	repoDir := toRepoDir(*cluster)
	checkout, err := git.NewCheckout(ctx, toFullRepoURL(*cluster), repoBaseDir)
	if err != nil {
	sklog.Fatalf("Failed to check out config repo: %s", err)
	}
	output, err := checkout.Git(ctx, "status", "-s")
	if err != nil {
	sklog.Fatal(err)
	}
	if strings.TrimSpace(output) != "" {
	if !*ignoreDirty {
	sklog.Fatalf("Found dirty checkout in %s:\n%s", repoBaseDir, output)
	}
	} else {
	if err := checkout.Update(ctx); err != nil {
	sklog.Fatal(err)
	}
	}

	// Switch kubectl to the right project.
	p := clusters[*cluster]
	if p == nil {
	fmt.Printf("Invalid value for --cluster flag: %q", *cluster)
	flag.Usage()
	os.Exit(1)
	}

	if err := exec.Run(context.Background(), &exec.Command{
	Name: "gcloud",
	Args: []string{
	"container",
	"clusters",
	"get-credentials",
	*cluster,
	"--zone",
	p.Zone,
	"--project",
	p.Project,
	},
	LogStderr: true,
	LogStdout: true,
	}); err != nil {
	sklog.Errorf("Failed to run: %s", err)
	}

	// Get all the yaml files.
	filenames, err := filepath.Glob(filepath.Join(repoDir, "*.yaml"))
	if err != nil {
	sklog.Fatal(err)
	}

	tokenSource := auth.NewGCloudTokenSource(containerRegistryProject)
	imageNames := flag.Args()
	if len(imageNames) == 0 {
	fmt.Printf("At least one image name needs to be supplied.")
	flag.Usage()
	os.Exit(1)
	}
	sklog.Infof("Pushing the following images: %q", imageNames)

	gcrTagProvider := func(imageName string) ([]string, error) {
	return gcr.NewClient(tokenSource, containerRegistryProject, imageName).Tags()
	}

	changed := util.StringSet{}
	for _, imageName := range imageNames {
	image, err := imageFromCmdLineImage(imageName, gcrTagProvider)
	if err != nil {
	sklog.Fatal(err)
	}
	if *list {
	// imageFromCmdLineImage printed out the tags, so nothing more to do.
	continue
	}

	// imageRegex has the following groups returned on match:
	// 0 - the entire line
	// 1 - the prefix, i.e. image:, with correct spacing.
	// 2 - full image name
	// 3 - just the tag
	//
	// We pull out the 'prefix' so we can use it when
	// we rewrite the image: line so the indent level is
	// unchanged.
	parts := strings.SplitN(image, ":", 2)
	if len(parts) != 2 {
	sklog.Fatalf("Failed to split imageName: %v", parts)
	}
	imageNoTag := parts[0]
	imageRegex := regexp.MustCompile(fmt.Sprintf(`^(\s+image:\s+)(%s):.*$`, imageNoTag))

	// Loop over all the yaml files and update tags for the given imageName.
	for _, filename := range filenames {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
	sklog.Errorf("Failed to read %q (skipping): %s", filename, err)
	continue
	}
	lines := strings.Split(string(b), "\n")
	for i, line := range lines {
	matches := imageRegex.FindStringSubmatch(line)
	if len(matches) != 3 {
	continue
	}
	changed[filename] = true
	lines[i] = matches[1] + image
	}
	if changed[filename] {
	err := util.WithWriteFile(filename, func(w io.Writer) error {
	_, err := w.Write([]byte(strings.Join(lines, "\n")))
	return err
	})
	if err != nil {
	sklog.Fatalf("Failed to write update config file %q: %s", filename, err)
	}
	}
	}
	}

	// Were any files updated?
	if len(changed) != 0 {
	filenameFlag := fmt.Sprintf("--filename=%s\n", strings.Join(changed.Keys(), ","))
	if !*dryRun {
	for filename := range changed {
	msg, err := checkout.Git(ctx, "add", filepath.Base(filename))
	if err != nil {
	sklog.Fatalf("Failed to stage changes to the config repo: %s: %q", err, msg)
	}
	}
	msg, err := checkout.Git(ctx, "diff", "--cached", "--name-only")
	if err != nil {
	sklog.Fatalf("Failed to diff :%s: %q", err, msg)
	}
	if msg == "" {
	sklog.Infof("Not pushing since no files changed.")
	return
	}
	msg, err = checkout.Git(ctx, "commit", "-m", *message)
	if err != nil {
	sklog.Fatalf("Failed to commit to the config repo: %s: %q", err, msg)
	}
	msg, err = checkout.Git(ctx, "push", "origin", "master")
	if err != nil {
	sklog.Fatalf("Failed to push the config repo: %s: %q", err, msg)
	}
	if err := exec.Run(context.Background(), &exec.Command{
	Name: "kubectl",
	Args: []string{"apply", filenameFlag},
	LogStderr: true,
	LogStdout: true,
	}); err != nil {
	sklog.Errorf("Failed to run: %s", err)
	}
	} else {
	fmt.Printf("\nkubectl apply %s\n", filenameFlag)
	}
	} else {
	fmt.Println("Nothing to do.")
	}
	}