golden/cmd/skiacorrectness/main.go

// skiacorrectness implements the process that exposes a RESTful API used by the JS frontend.
package main

import (
	"context"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"math/rand"
	"net/http"
	netpprof "net/http/pprof"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"cloud.google.com/go/datastore"
	"github.com/flynn/json5"
	"github.com/gorilla/mux"
	"google.golang.org/api/option"
	gstorage "google.golang.org/api/storage/v1"
	"google.golang.org/grpc"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/ds"
	"go.skia.org/infra/go/eventbus"
	"go.skia.org/infra/go/firestore"
	"go.skia.org/infra/go/gerrit"
	"go.skia.org/infra/go/gevent"
	"go.skia.org/infra/go/git/gitinfo"
	"go.skia.org/infra/go/gitiles"
	"go.skia.org/infra/go/gitstore_deprecated/bt_gitstore"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/issues"
	"go.skia.org/infra/go/login"
	"go.skia.org/infra/go/paramtools"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/skiaversion"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/timer"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/vcsinfo"
	"go.skia.org/infra/go/vcsinfo/bt_vcs"
	"go.skia.org/infra/golden/go/baseline/simple_baseliner"
	"go.skia.org/infra/golden/go/diff"
	"go.skia.org/infra/golden/go/diffstore"
	"go.skia.org/infra/golden/go/diffstore/mapper/disk_mapper"
	"go.skia.org/infra/golden/go/expstorage/fs_expstore"
	"go.skia.org/infra/golden/go/ignore"
	"go.skia.org/infra/golden/go/ignore/ds_ignorestore"
	"go.skia.org/infra/golden/go/indexer"
	"go.skia.org/infra/golden/go/search"
	"go.skia.org/infra/golden/go/shared"
	"go.skia.org/infra/golden/go/status"
	"go.skia.org/infra/golden/go/storage"
	"go.skia.org/infra/golden/go/tilesource"
	"go.skia.org/infra/golden/go/tracestore/bt_tracestore"
	"go.skia.org/infra/golden/go/tryjobs/gerrit_tryjob_monitor"
	"go.skia.org/infra/golden/go/tryjobstore/ds_tryjobstore"
	"go.skia.org/infra/golden/go/warmer"
	"go.skia.org/infra/golden/go/web"
)

const (
	// IMAGE_URL_PREFIX is path prefix used for all images (digests and diffs)
	IMAGE_URL_PREFIX = "/img/"

	// OAUTH2_CALLBACK_PATH is callback endpoint used for the Oauth2 flow
	OAUTH2_CALLBACK_PATH = "/oauth2callback/"

	// EVERYTHING_PUBLIC can be provided as the value for the whitelist file to whitelist all configurations
	EVERYTHING_PUBLIC = "all"
)

func main() {
	// Command line flags.
	var (
		appTitle            = flag.String("app_title", "Skia Gold", "Title of the deployed up on the front end.")
		authoritative       = flag.Bool("authoritative", false, "Indicates that this instance should write changes that could be triggered on multiple instances running in parallel.")
		authorizedUsers     = flag.String("auth_users", login.DEFAULT_DOMAIN_WHITELIST, "White space separated list of domains and email addresses that are allowed to login.")
		btInstanceID        = flag.String("bt_instance", "production", "ID of the BigTable instance that contains Git metadata")
		btProjectID         = flag.String("bt_project_id", "skia-public", "project id with BigTable instance")
		cacheSize           = flag.Int("cache_size", 1, "Approximate cachesize used to cache images and diff metrics in GiB. This is just a way to limit caching. 0 means no caching at all. Use default for testing.")
		clientSecretFile    = flag.String("client_secret", "", "Client secret file for OAuth2 authentication.")
		cpuProfile          = flag.Duration("cpu_profile", 0, "Duration for which to profile the CPU usage. After this duration the program writes the CPU profile and exits.")
		defaultCorpus       = flag.String("default_corpus", "gm", "The corpus identifier shown by default on the frontend.")
		defaultMatchFields  = flag.String("match_fields", "name", "A comma separated list of fields that need to match when finding closest images.")
		diffServerGRPCAddr  = flag.String("diff_server_grpc", "", "The grpc port of the diff server. 'diff_server_http also needs to be set.")
		diffServerImageAddr = flag.String("diff_server_http", "", "The images serving address of the diff server. 'diff_server_grpc has to be set as well.")
		dsNamespace         = flag.String("ds_namespace", "", "Cloud datastore namespace to be used by this instance.")
		dsProjectID         = flag.String("ds_project_id", "", "Project id that houses the datastore instance.")
		eventTopic          = flag.String("event_topic", "", "The pubsub topic to use for distributed events.")
		forceLogin          = flag.Bool("force_login", true, "Force the user to be authenticated for all requests.")
		fsLegacyAuth        = flag.Bool("fs_legacy_auth", false, "use legacy credentials to auth Firestore")
		fsNamespace         = flag.String("fs_namespace", "", "Typically the instance id. e.g. 'flutter', 'skia', etc")
		fsProjectID         = flag.String("fs_project_id", "skia-firestore", "The project with the firestore instance. Datastore and Firestore can't be in the same project.")
		gerritURL           = flag.String("gerrit_url", gerrit.GERRIT_SKIA_URL, "URL of the Gerrit instance where we retrieve CL metadata.")
		gitBTTableID        = flag.String("git_bt_table", "", "ID of the BigTable table that contains Git metadata")
		gitRepoDir          = flag.String("git_repo_dir", "../../../skia", "Directory location for the Skia repo.")
		gitRepoURL          = flag.String("git_repo_url", "https://skia.googlesource.com/skia", "The URL to pass to git clone for the source repository.")
		gsBucketNames       = flag.String("gs_buckets", "", "Comma-separated list of google storage bucket that hold uploaded images.")
		hashesGSPath        = flag.String("hashes_gs_path", "", "GS path, where the known hashes file should be stored. If empty no file will be written. Format: <bucket>/<path>.")
		imageDir            = flag.String("image_dir", "/tmp/imagedir", "What directory to store test and diff images in.")
		indexInterval       = flag.Duration("idx_interval", 5*time.Minute, "Interval at which the indexer calculates the search index.")
		internalPort        = flag.String("internal_port", "", "HTTP service address for internal clients, e.g. probers. No authentication on this port.")
		local               = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
		memProfile          = flag.Duration("memprofile", 0, "Duration for which to profile memory. After this duration the program writes the memory profile and exits.")
		nCommits            = flag.Int("n_commits", 50, "Number of recent commits to include in the analysis.")
		noCloudLog          = flag.Bool("no_cloud_log", false, "Disables cloud logging. Primarily for running locally and in K8s.")
		port                = flag.String("port", ":9000", "HTTP service address (e.g., ':9000')")
		promPort            = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':10110')")
		pubWhiteList        = flag.String("public_whitelist", "", fmt.Sprintf("File name of a JSON5 file that contains a query with the traces to white list. If set to '%s' everything is included. This is required if force_login is false.", EVERYTHING_PUBLIC))
		pubsubProjectID     = flag.String("pubsub_project_id", "", "Project ID that houses the pubsub topics (e.g. for ingestion).")
		redirectURL         = flag.String("redirect_url", "https://gold.skia.org/oauth2callback/", "OAuth2 redirect url. Only used when local=false.")
		resourcesDir        = flag.String("resources_dir", "", "The directory to find templates, JS, and CSS files. If blank the directory relative to the source code files will be used.")
		showBotProgress     = flag.Bool("show_bot_progress", true, "Query status.skia.org for the progress of bot results.")
		siteURL             = flag.String("site_url", "https://gold.skia.org", "URL where this app is hosted.")
		tileFreshness       = flag.Duration("tile_freshness", time.Minute, "How often to re-fetch the tile")
		traceBTTableID      = flag.String("trace_bt_table", "", "BigTable table ID for the traces.")
	)

	var err error

	// Needed to use TimeSortableKey(...) which relies on an RNG. See docs there.
	rand.Seed(time.Now().UnixNano())

	mainTimer := timer.New("main init")

	// Parse the options. So we can configure logging.
	flag.Parse()

	// Set up the logging options.
	logOpts := []common.Opt{
		common.PrometheusOpt(promPort),
	}

	// Should we disable cloud logging.
	if !*noCloudLog {
		logOpts = append(logOpts, common.CloudLoggingOpt())
	}
	_, appName := filepath.Split(os.Args[0])
	common.InitWithMust(appName, logOpts...)
	skiaversion.MustLogVersion()

	ctx := context.Background()
	skiaversion.MustLogVersion()

	// Enable the memory profiler if memProfile was set.
	// TODO(stephana): This should be moved to a HTTP endpoint that
	// only responds to internal IP addresses/ports.
	if *memProfile > 0 {
		time.AfterFunc(*memProfile, func() {
			sklog.Infof("Writing Memory Profile")
			f, err := ioutil.TempFile("./", "memory-profile")
			if err != nil {
				sklog.Fatalf("Unable to create memory profile file: %s", err)
			}
			if err := pprof.WriteHeapProfile(f); err != nil {
				sklog.Fatalf("Unable to write memory profile file: %v", err)
			}
			util.Close(f)
			sklog.Infof("Memory profile written to %s", f.Name())

			os.Exit(0)
		})
	}

	if *cpuProfile > 0 {
		f, err := ioutil.TempFile("./", "cpu-profile")
		if err != nil {
			sklog.Fatalf("Unable to create cpu profile file: %s", err)
		}
		sklog.Infof("Writing CPU Profile to %s", f.Name())

		if err := pprof.StartCPUProfile(f); err != nil {
			sklog.Fatalf("Unable to write cpu profile file: %v", err)
		}
		time.AfterFunc(*cpuProfile, func() {
			pprof.StopCPUProfile()
			util.Close(f)
			sklog.Infof("CPU profile written to %s", f.Name())
			os.Exit(0)
		})
	}

	// Start the internal server on the internal port if requested.
	if *internalPort != "" {
		// Add the profiling endpoints to the internal router.
		internalRouter := mux.NewRouter()

		// Set up the health check endpoint.
		internalRouter.HandleFunc("/healthz", httputils.ReadyHandleFunc)

		// Register pprof handlers
		internalRouter.HandleFunc("/debug/pprof/", netpprof.Index)
		internalRouter.HandleFunc("/debug/pprof/symbol", netpprof.Symbol)
		internalRouter.HandleFunc("/debug/pprof/profile", netpprof.Profile)
		internalRouter.HandleFunc("/debug/pprof/{profile}", netpprof.Index)

		go func() {
			sklog.Infof("Internal server on  http://127.0.0.1" + *internalPort)
			sklog.Fatal(http.ListenAndServe(*internalPort, internalRouter))
		}()
	}

	// Set the resource directory if it's empty. Useful for running locally.
	if *resourcesDir == "" {
		_, filename, _, _ := runtime.Caller(0)
		*resourcesDir = filepath.Join(filepath.Dir(filename), "../..")
		*resourcesDir += "/frontend"
	}

	// Set up login
	useRedirectURL := *redirectURL
	if *local {
		useRedirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
	}
	sklog.Infof("The allowed list of users is: %q", *authorizedUsers)
	if err := login.Init(useRedirectURL, *authorizedUsers, *clientSecretFile); err != nil {
		sklog.Fatalf("Failed to initialize the login system: %s", err)
	}

	// Get the token source for the service account with access to GCS, the Monorail issue tracker,
	// cloud pubsub, and datastore.
	tokenSource, err := auth.NewDefaultTokenSource(*local, datastore.ScopeDatastore, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email")
	if err != nil {
		sklog.Fatalf("Failed to authenticate service account: %s", err)
	}
	// TODO(dogben): Ok to add request/dial timeouts?
	client := httputils.DefaultClientConfig().WithTokenSource(tokenSource).WithoutRetries().Client()

	// serviceName uniquely identifies this host and app and is used as ID for other services.
	nodeName, err := gevent.GetNodeName(appName, *local)
	if err != nil {
		sklog.Fatalf("Error getting unique service name: %s", err)
	}

	// If the addresses for a remote DiffStore were given, then set it up
	// otherwise create an embedded DiffStore instance.
	var diffStore diff.DiffStore = nil
	if (*diffServerGRPCAddr != "") || (*diffServerImageAddr != "") {
		// Create the client connection and connect to the server.
		conn, err := grpc.Dial(*diffServerGRPCAddr,
			grpc.WithInsecure(),
			grpc.WithDefaultCallOptions(
				grpc.MaxCallSendMsgSize(diffstore.MAX_MESSAGE_SIZE),
				grpc.MaxCallRecvMsgSize(diffstore.MAX_MESSAGE_SIZE)))
		if err != nil {
			sklog.Fatalf("Unable to connect to grpc service: %s", err)
		}

		codec := diffstore.MetricMapCodec{}
		diffStore, err = diffstore.NewNetDiffStore(conn, *diffServerImageAddr, codec)
		if err != nil {
			sklog.Fatalf("Unable to initialize NetDiffStore: %s", err)
		}
		sklog.Infof("DiffStore: NetDiffStore initiated.")
	} else {
		if *gsBucketNames == "" {
			sklog.Fatalf("Must specify --gs_buckets or (--diff_server_http and --diff_server_grpc)")
		}
		mapper := disk_mapper.New(&diff.DiffMetrics{})
		diffStore, err = diffstore.NewMemDiffStore(client, *imageDir, strings.Split(*gsBucketNames, ","), diffstore.DEFAULT_GCS_IMG_DIR_NAME, *cacheSize, mapper)
		if err != nil {
			sklog.Fatalf("Allocating local DiffStore failed: %s", err)
		}
		sklog.Infof("DiffStore: MemDiffStore initiated.")
	}

	// Set up the event bus which can either be in-process or distributed
	// depending whether an PubSub topic was defined.
	var evt eventbus.EventBus = nil
	if *eventTopic != "" {
		evt, err = gevent.New(*pubsubProjectID, *eventTopic, nodeName, option.WithTokenSource(tokenSource))
		if err != nil {
			sklog.Fatalf("Unable to create global event client. Got error: %s", err)
		}
		sklog.Infof("Global eventbus for topic '%s' and subscriber '%s' created.", *eventTopic, nodeName)
	} else {
		evt = eventbus.New()
	}

	var vcs vcsinfo.VCS
	if *btInstanceID != "" && *gitBTTableID != "" {
		btConf := &bt_gitstore.BTConfig{
			ProjectID:  *btProjectID,
			InstanceID: *btInstanceID,
			TableID:    *gitBTTableID,
		}

		// If the repoURL is numeric then it is treated like the numeric ID of a repository and
		// we look up the corresponding repo URL.
		useRepoURL := *gitRepoURL
		if foundRepoURL, ok := bt_gitstore.RepoURLFromID(ctx, btConf, *gitRepoURL); ok {
			useRepoURL = foundRepoURL
		}
		gitStore, err := bt_gitstore.New(ctx, btConf, useRepoURL)
		if err != nil {
			sklog.Fatalf("Error instantiating gitstore: %s", err)
		}

		gitilesRepo := gitiles.NewRepo("", "", nil)
		bvcs, err := bt_vcs.New(ctx, gitStore, "master", gitilesRepo)
		if err != nil {
			sklog.Fatalf("Error creating BT-backed VCS instance: %s", err)
		}

		bvcs.StartTracking(ctx, evt)
		vcs = bvcs
	} else {
		vcs, err = gitinfo.CloneOrUpdate(ctx, *gitRepoURL, *gitRepoDir, false)
		if err != nil {
			sklog.Fatalf("Error creating on-disk VCS instance: %s", err)
		}
	}

	// If this is an authoritative instance we need an authenticated Gerrit client
	// because it needs to write.
	gitcookiesPath := ""
	if *authoritative {
		// Set up an authenticated Gerrit client.
		gitcookiesPath = gerrit.DefaultGitCookiesPath()
		if !*local {
			if gitcookiesPath, err = gerrit.GitCookieAuthDaemonPath(); err != nil {
				sklog.Fatalf("Error retrieving git_cookie_authdaemon path: %s", err)
			}
		}
	}
	gerritAPI, err := gerrit.NewGerrit(*gerritURL, gitcookiesPath, nil)
	if err != nil {
		sklog.Fatalf("Failed to create Gerrit client: %s", err)
	}

	btc := bt_tracestore.BTConfig{
		ProjectID:  *btProjectID,
		InstanceID: *btInstanceID,
		TableID:    *traceBTTableID,
		VCS:        vcs,
	}

	err = bt_tracestore.InitBT(context.Background(), btc)
	if err != nil {
		sklog.Fatalf("Could not initialize BigTable tracestore with config %#v: %s", btc, err)
	}

	traceStore, err := bt_tracestore.New(context.Background(), btc, false)
	if err != nil {
		sklog.Fatalf("Could not instantiate BT tracestore: %s", err)
	}

	gsClientOpt := storage.GCSClientOptions{
		HashesGSPath: *hashesGSPath,
		Dryrun:       *local,
	}

	gsClient, err := storage.NewGCSClient(client, gsClientOpt)
	if err != nil {
		sklog.Fatalf("Unable to create GCSClient: %s", err)
	}

	if err := ds.InitWithOpt(*dsProjectID, *dsNamespace, option.WithTokenSource(tokenSource)); err != nil {
		sklog.Fatalf("Unable to configure cloud datastore: %s", err)
	}

	if *fsNamespace == "" {
		sklog.Fatalf("--fs_namespace must be set")
	}

	var fsClient *firestore.Client
	if *fsLegacyAuth {
		fsClient, err = firestore.NewClient(context.Background(), *fsProjectID, "gold", *fsNamespace, tokenSource)
		if err != nil {
			sklog.Fatalf("Unable to configure Firestore: %s", err)
		}
	} else {
		// Auth note: the underlying firestore.NewClient looks at the
		// GOOGLE_APPLICATION_CREDENTIALS env variable, so we don't need to supply
		// a token source.
		fsClient, err = firestore.NewClient(context.Background(), *fsProjectID, "gold", *fsNamespace, nil)
		if err != nil {
			sklog.Fatalf("Unable to configure Firestore: %s", err)
		}
	}

	// Set up the cloud expectations store
	expStore, err := fs_expstore.New(fsClient, evt, fs_expstore.ReadWrite)
	if err != nil {
		sklog.Fatalf("Unable to initialize fs_expstore: %s", err)
	}

	tryjobStore, err := ds_tryjobstore.New(ds.DS, evt)
	if err != nil {
		sklog.Fatalf("Unable to instantiate tryjob store: %s", err)
	}
	tryjobMonitor := gerrit_tryjob_monitor.New(tryjobStore, expStore, gerritAPI, *siteURL, evt, *authoritative)

	baseliner := simple_baseliner.New(expStore)

	publiclyViewableParams := paramtools.ParamSet{}
	// Load the publiclyViewable params if configured and disable querying for issues.
	if *pubWhiteList != "" && *pubWhiteList != EVERYTHING_PUBLIC {
		if publiclyViewableParams, err = loadParamFile(*pubWhiteList); err != nil {
			sklog.Fatalf("Could not load list of public params: %s", err)
		}
	}

	// Check if this is public instance. If so, make sure a list of public params
	// has been specified - can be EVERYTHING_PUBLIC.
	if !*forceLogin && (*pubWhiteList == "") {
		sklog.Fatalf("Empty whitelist file. A non-empty white list must be provided if force_login=false.")
	}

	// openSite indicates whether this can expose all end-points. The user still has to be authenticated.
	openSite := (*pubWhiteList == EVERYTHING_PUBLIC) || *forceLogin

	ignoreStore, err := ds_ignorestore.New(ds.DS)
	if err != nil {
		sklog.Fatalf("Unable to create ignorestore: %s", err)
	}

	if err := ignore.StartMonitoring(ignoreStore, *tileFreshness); err != nil {
		sklog.Fatalf("Failed to start monitoring for expired ignore rules: %s", err)
	}

	ctc := tilesource.CachedTileSourceConfig{
		EventBus:               evt,
		GerritAPI:              gerritAPI,
		IgnoreStore:            ignoreStore,
		NCommits:               *nCommits,
		PubliclyViewableParams: publiclyViewableParams,
		TraceStore:             traceStore,
		TryjobMonitor:          tryjobMonitor,
		VCS:                    vcs,
	}

	tileSource := tilesource.New(ctc)
	sklog.Infof("Fetching tile")
	// Blocks until tile is fetched
	err = tileSource.StartUpdater(context.Background(), 2*time.Minute)
	if err != nil {
		sklog.Fatalf("Could not fetch initial tile: %s", err)
	}

	ic := indexer.IndexerConfig{
		DiffStore:         diffStore,
		EventBus:          evt,
		ExpectationsStore: expStore,
		GCSClient:         gsClient,
		TileSource:        tileSource,
		Warmer:            warmer.New(),
	}

	// Rebuild the index every few minutes.
	sklog.Infof("Starting indexer to run every %s", *indexInterval)
	ixr, err := indexer.New(ic, *indexInterval)
	if err != nil {
		sklog.Fatalf("Failed to create indexer: %s", err)
	}
	sklog.Infof("Indexer created.")

	searchAPI := search.NewSearchAPI(diffStore, expStore, ixr, tryjobStore, publiclyViewableParams)

	sklog.Infof("Search API created")

	issueTracker := issues.NewMonorailIssueTracker(client, issues.PROJECT_SKIA)

	swc := status.StatusWatcherConfig{
		VCS:               vcs,
		EventBus:          evt,
		TileSource:        tileSource,
		ExpectationsStore: expStore,
	}

	statusWatcher, err := status.New(swc)
	if err != nil {
		sklog.Fatalf("Failed to initialize status watcher: %s", err)
	}
	sklog.Infof("statusWatcher created")

	handlers := web.WebHandlers{
		Baseliner:         baseliner,
		DiffStore:         diffStore,
		ExpectationsStore: expStore,
		GCSClient:         gsClient,
		IgnoreStore:       ignoreStore,
		Indexer:           ixr,
		IssueTracker:      issueTracker,
		SearchAPI:         searchAPI,
		StatusWatcher:     statusWatcher,
		TileSource:        tileSource,
		TryjobMonitor:     tryjobMonitor,
		TryjobStore:       tryjobStore,
		VCS:               vcs,
	}

	mainTimer.Stop()

	// loggedRouter contains all the endpoints that are logged. See the call below to
	// LoggingGzipRequestResponse.
	loggedRouter := mux.NewRouter()

	// Set up the resource to serve the image files.
	imgHandler, err := diffStore.ImageHandler(IMAGE_URL_PREFIX)
	if err != nil {
		sklog.Fatalf("Unable to get image handler: %s", err)
	}

	// New Polymer based UI endpoints.
	loggedRouter.PathPrefix("/res/").HandlerFunc(web.MakeResourceHandler(*resourcesDir))
	loggedRouter.HandleFunc(OAUTH2_CALLBACK_PATH, login.OAuth2CallbackHandler)

	// TODO(stephana): remove "/_/hashes" in favor of "/json/hashes" once all clients have switched.

	// /_/hashes is used by the bots to find hashes it does not need to upload.
	loggedRouter.HandleFunc(shared.LEGACY_KNOWN_HASHES_ROUTE, handlers.TextKnownHashesProxy).Methods("GET")
	loggedRouter.HandleFunc("/json/version", skiaversion.JsonHandler)
	loggedRouter.HandleFunc("/loginstatus/", login.StatusHandler)
	loggedRouter.HandleFunc("/logout/", login.LogoutHandler)

	// Set up a subrouter for the '/json' routes which make up the Gold API.
	// This makes routing faster, but also returns a failure when an /json route is
	// requested that doesn't exit. If we did this differently a call to a non-existing endpoint
	// would be handled by the route that handles the returning the index template and make debugging
	// confusing.
	jsonRouter := loggedRouter.PathPrefix("/json").Subrouter()
	trim := func(r string) string { return strings.TrimPrefix(r, "/json") }

	jsonRouter.HandleFunc(trim(shared.KNOWN_HASHES_ROUTE), handlers.TextKnownHashesProxy).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/byblame"), handlers.JsonByBlameHandler).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/cleardigests"), handlers.JsonClearDigests).Methods("POST")
	jsonRouter.HandleFunc(trim("/json/clusterdiff"), handlers.JsonClusterDiffHandler).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/cmp"), handlers.JsonCompareTestHandler).Methods("POST")
	jsonRouter.HandleFunc(trim("/json/commits"), handlers.JsonCommitsHandler).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/details"), handlers.JsonDetailsHandler).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/diff"), handlers.JsonDiffHandler).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/export"), handlers.JsonExportHandler).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/failure"), handlers.JsonListFailureHandler).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/failure/clear"), handlers.JsonClearFailureHandler).Methods("POST")
	jsonRouter.HandleFunc(trim("/json/gitlog"), handlers.JsonGitLogHandler).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/list"), handlers.JsonListTestsHandler).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/paramset"), handlers.JsonParamsHandler).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/search"), handlers.JsonSearchHandler).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/triage"), handlers.JsonTriageHandler).Methods("POST")
	jsonRouter.HandleFunc(trim("/json/triagelog"), handlers.JsonTriageLogHandler).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/triagelog/undo"), handlers.JsonTriageUndoHandler).Methods("POST")
	jsonRouter.HandleFunc(trim("/json/tryjob"), handlers.JsonTryjobListHandler).Methods("GET")
	jsonRouter.HandleFunc(trim("/json/tryjob/{id}"), handlers.JsonTryjobSummaryHandler).Methods("GET")

	// Retrieving that baseline for master and an Gerrit issue are handled the same way
	// These routes can be served with baseline_server for higher availability.
	jsonRouter.HandleFunc(trim(shared.EXPECTATIONS_ROUTE), handlers.JsonBaselineHandler).Methods("GET")
	jsonRouter.HandleFunc(trim(shared.EXPECTATIONS_ISSUE_ROUTE), handlers.JsonBaselineHandler).Methods("GET")

	jsonRouter.HandleFunc(trim("/json/refresh/{id}"), handlers.JsonRefreshIssue).Methods("GET")

	// Only expose these endpoints if login is enforced across the app or this an open site.
	if openSite {
		jsonRouter.HandleFunc(trim("/json/ignores"), handlers.JsonIgnoresHandler).Methods("GET")
		jsonRouter.HandleFunc(trim("/json/ignores/add/"), handlers.JsonIgnoresAddHandler).Methods("POST")
		jsonRouter.HandleFunc(trim("/json/ignores/del/{id}"), handlers.JsonIgnoresDeleteHandler).Methods("POST")
		jsonRouter.HandleFunc(trim("/json/ignores/save/{id}"), handlers.JsonIgnoresUpdateHandler).Methods("POST")
	}

	// Make sure we return a 404 for anything that starts with /json and could not be found.
	jsonRouter.HandleFunc("/{ignore:.*}", http.NotFound)
	loggedRouter.HandleFunc("/json", http.NotFound)

	// For everything else serve the same markup.
	indexFile := *resourcesDir + "/index.html"
	indexTemplate := template.Must(template.New("").ParseFiles(indexFile)).Lookup("index.html")

	// appConfig is injected into the header of the index file.
	appConfig := &struct {
		BaseRepoURL        string   `json:"baseRepoURL"`
		DefaultCorpus      string   `json:"defaultCorpus"`
		DefaultMatchFields []string `json:"defaultMatchFields"`
		ShowBotProgress    bool     `json:"showBotProgress"`
		Title              string   `json:"title"`
		IsPublic           bool     `json:"isPublic"` // If true this is not open but restrictions apply.
	}{
		BaseRepoURL:        *gitRepoURL,
		DefaultCorpus:      *defaultCorpus,
		DefaultMatchFields: strings.Split(*defaultMatchFields, ","),
		ShowBotProgress:    *showBotProgress,
		Title:              *appTitle,
		IsPublic:           !openSite,
	}

	loggedRouter.PathPrefix("/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html")

		// Reload the template if we are running locally.
		if *local {
			indexTemplate = template.Must(template.New("").ParseFiles(indexFile)).Lookup("index.html")
		}
		if err := indexTemplate.Execute(w, appConfig); err != nil {
			sklog.Errorf("Failed to expand template: %s", err)
			return
		}
	})

	// set up the app router that might be authenticated and logs almost everything.
	appRouter := mux.NewRouter()
	// Images should not be served gzipped, which can sometimes have issues
	// when serving an image from a NetDiffstore with HTTP2. Additionally, is wasteful
	// given PNGs typically have zlib compression anyway.
	appRouter.PathPrefix(IMAGE_URL_PREFIX).Handler(imgHandler)
	appRouter.PathPrefix("/").Handler(httputils.LoggingGzipRequestResponse(loggedRouter))

	// Use the appRouter as a handler and wrap it into middleware that enforces authentication if
	// necessary it was requested via the force_login flag.
	appHandler := http.Handler(appRouter)
	if *forceLogin {
		appHandler = login.ForceAuth(appRouter, OAUTH2_CALLBACK_PATH)
	}

	// The appHandler contains all application specific routes that are have logging and
	// authentication configured. Now we wrap it into the router that is exposed to the host
	// (aka the K8s container) which requires that some routes are never logged or authenticated.
	rootRouter := mux.NewRouter()
	rootRouter.HandleFunc("/healthz", httputils.ReadyHandleFunc)
	rootRouter.HandleFunc("/json/trstatus", httputils.CorsHandler(handlers.JsonStatusHandler))

	rootRouter.PathPrefix("/").Handler(appHandler)

	// Start the server
	sklog.Infof("Serving on http://127.0.0.1" + *port)
	sklog.Fatal(http.ListenAndServe(*port, rootRouter))
}

// loadParamFile loads the given JSON5 file that defines the query to
// make traces publicly viewable. If the given file is empty or otherwise
// cannot be parsed an error will be returned.
func loadParamFile(fName string) (paramtools.ParamSet, error) {
	params := paramtools.ParamSet{}

	f, err := os.Open(fName)
	if err != nil {
		return params, skerr.Fmt("unable open file %s: %s", fName, err)
	}
	defer util.Close(f)

	if err := json5.NewDecoder(f).Decode(&params); err != nil {
		return params, skerr.Fmt("invalid JSON5 in %s: %s", fName, err)
	}

	// Make sure the param file is not empty.
	empty := true
	for _, values := range params {
		if empty = len(values) == 0; !empty {
			break
		}
	}
	if empty {
		return params, fmt.Errorf("publicly viewable params in %s cannot be empty.", fName)
	}
	sklog.Infof("publicly viewable params loaded from %s", fName)
	sklog.Debugf("%#v", params)
	return params, nil
}