gitsync/go/gitsync/main.go

package main

import (
	"context"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"path/filepath"
	"strings"
	"time"

	"github.com/flynn/json5"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/fileutil"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/gitauth"
	"go.skia.org/infra/go/gitstore/bt_gitstore"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/human"
	"go.skia.org/infra/go/sklog"
)

// This server watches a list of git repos for changes and syncs the meta data of all commits
// to a BigTable backed datastore.

// Default config/flag values
var defaultConf = gitSyncConfig{
	BTInstanceID:    "production",
	BTTableID:       "git-repos2",
	HttpPort:        ":9091",
	Local:           false,
	ProjectID:       "skia-public",
	PromPort:        ":20000",
	RepoURLs:        []string{},
	RefreshInterval: human.JSONDuration(10 * time.Minute),
	WorkDir:         "",
}

func main() {
	// config holds the configuration values either from flags or from parsing the config files.
	config := defaultConf

	// Flags that cause the flags below to be disregarded.
	configFile := flag.String("config", "", "Disregard flags and load the configuration from this JSON5 config file. The keys and types of the config file match the flags.")
	runInit := flag.Bool("init", false, "Initialize the BigTable instance and quit. This should be run with a different different user who has admin rights.")

	// Define flags that map to field in the configuration struct.
	flag.StringVar(&config.BTInstanceID, "bt_instance", defaultConf.BTInstanceID, "Big Table instance")
	flag.StringVar(&config.BTTableID, "bt_table", defaultConf.BTTableID, "BigTable table ID")
	flag.StringVar(&config.HttpPort, "http_port", defaultConf.HttpPort, "The http port where ready-ness endpoints are served.")
	flag.BoolVar(&config.Local, "local", defaultConf.Local, "Running locally if true. As opposed to in production.")
	flag.StringVar(&config.ProjectID, "project", defaultConf.ProjectID, "ID of the GCP project")
	flag.StringVar(&config.PromPort, "prom_port", defaultConf.PromPort, "Metrics service address (e.g., ':10110')")
	common.MultiStringFlagVar(&config.RepoURLs, "repo_url", defaultConf.RepoURLs, "Repo url")
	flag.DurationVar((*time.Duration)(&config.RefreshInterval), "refresh", time.Duration(defaultConf.RefreshInterval), "Interval in which to poll git and refresh the GitStore.")
	flag.StringVar(&config.WorkDir, "workdir", defaultConf.WorkDir, "Working directory where repos are cached. Use the same directory between calls to speed up checkout time.")

	common.InitWithMust(
		"gitsync",
		common.PrometheusOpt(&config.PromPort),
		common.MetricsLoggingOpt(),
	)
	defer common.Defer()

	// If a configuration file was given we load it into config.
	if *configFile != "" {
		confBytes, err := ioutil.ReadFile(*configFile)
		if err != nil {
			sklog.Fatalf("Error reading config file %s: %s", *configFile, err)
		}

		if err := json5.Unmarshal(confBytes, &config); err != nil {
			sklog.Fatalf("Error parsing config file %s: %s", *configFile, err)
		}
	}

	// Dump the configuration since it might be different than the flags that are dumped by default.
	sklog.Infof("\n\n  Effective configuration: \n%s \n", config.String())

	// Configure the bigtable instance.
	btConfig := &bt_gitstore.BTConfig{
		ProjectID:  config.ProjectID,
		InstanceID: config.BTInstanceID,
		TableID:    config.BTTableID,
	}

	// Initialize bigtable if invoked with --init and quit.
	// This should be invoked with a user that has admin privileges, so that the production user that
	// wants to write to the instance does not need admin privileges.
	if *runInit {
		if err := bt_gitstore.InitBT(btConfig); err != nil {
			sklog.Fatalf("Error initializing BT: %s", err)
		}
		sklog.Infof("BigTable instance %s and table %s in project %s initialized.", btConfig.InstanceID, btConfig.TableID, btConfig.ProjectID)
		return
	}

	// Make sure we have a data directory and it exists or can be created.
	if config.WorkDir == "" {
		sklog.Fatal("No workdir specified.")
	}
	useWorkDir := fileutil.Must(fileutil.EnsureDirExists(config.WorkDir))

	// Make sure we have at least one repo configured.
	if len(config.RepoURLs) == 0 {
		sklog.Fatalf("At least one repository URL must be configured.")
	}

	// TODO(stephana): Pass the token source explicitly to the BigTable related functions below.

	// Create token source.
	ts, err := auth.NewDefaultTokenSource(false, auth.SCOPE_USERINFO_EMAIL, auth.SCOPE_GERRIT)
	if err != nil {
		sklog.Fatalf("Problem setting up default token source: %s", err)
	}

	// Set up Git authentication if a service account email was set.
	if !config.Local {
		// Use the gitcookie created by the gitauth package.
		gitcookiesPath := "/tmp/gitcookies"
		sklog.Infof("Writing gitcookies to %s", gitcookiesPath)
		if _, err := gitauth.New(ts, gitcookiesPath, true, ""); err != nil {
			sklog.Fatalf("Failed to create git cookie updater: %s", err)
		}
		sklog.Infof("Git authentication set up successfully.")
	}

	// Start all repo watchers in the background.
	ctx := context.Background()
	for _, repoURL := range config.RepoURLs {
		go func(repoURL string) {
			repoDir, err := git.NormalizeURL(repoURL)
			if err != nil {
				sklog.Fatalf("Error getting normalized URL for %q:  %s", repoURL, err)
			}
			repoDir = strings.Replace(repoDir, "/", "_", -1)
			repoDir = filepath.Join(useWorkDir, repoDir)
			sklog.Infof("Checking out %s into %s", repoURL, repoDir)

			watcher, err := NewRepoWatcher(ctx, btConfig, repoURL, repoDir)
			if err != nil {
				sklog.Fatalf("Error initializing repo watcher: %s", err)
			}
			watcher.Start(ctx, time.Duration(config.RefreshInterval))
		}(repoURL)
	}

	// Set up the http handler to indicate ready-ness and start serving.
	http.HandleFunc("/healthz", httputils.ReadyHandleFunc)
	sklog.Infof("Listening on port: %s", config.HttpPort)
	log.Fatal(http.ListenAndServe(config.HttpPort, nil))
}