blob: 4bc748845a791002ff95ae22a5d95d001aa37b4b [file] [log] [blame]
/*
Server that collects and displays bug data for Skia's clients from different issue frameworks
*/
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"html/template"
"net/http"
"os"
"path/filepath"
"sort"
"time"
"cloud.google.com/go/datastore"
"github.com/gorilla/mux"
"github.com/unrolled/secure"
"go.skia.org/infra/bugs-central/go/aliases_watcher"
"go.skia.org/infra/bugs-central/go/db"
"go.skia.org/infra/bugs-central/go/poller"
"go.skia.org/infra/bugs-central/go/types"
"go.skia.org/infra/go/allowed"
"go.skia.org/infra/go/auth"
"go.skia.org/infra/go/baseapp"
"go.skia.org/infra/go/httputils"
"go.skia.org/infra/go/login"
"go.skia.org/infra/go/sklog"
)
var (
// Flags
host = flag.String("host", "bugs-central.skia.org", "HTTP service host")
workdir = flag.String("workdir", ".", "Directory to use for scratch work.")
fsNamespace = flag.String("fs_namespace", "", "Typically the instance id. e.g. 'bugs-central'")
fsProjectID = flag.String("fs_project_id", "skia-firestore", "The project with the firestore instance. Datastore and Firestore can't be in the same project.")
serviceAccountFile = flag.String("service_account_file", "/var/secrets/google/key.json", "Service account JSON file.")
authAllowList = flag.String("auth_allowlist", "google.com", "White space separated list of domains and email addresses that are allowed to login.")
pollInterval = flag.Duration("poll_interval", 2*time.Hour, "How often the server will poll the different issue frameworks for open issues.")
aliasesWatcherInterval = flag.Duration("aliases_watcher_interval", 2*time.Minute, "How often the server will look for issues assigned to rotation aliases.")
)
type ClientConfig struct {
ClientID string `json:"client_id"`
ClientSecret string `json:"client_secret"`
}
type ClientSecretJSON struct {
Installed ClientConfig `json:"installed"`
}
// See baseapp.Constructor.
func New() (baseapp.App, error) {
// Create workdir if it does not exist.
if err := os.MkdirAll(*workdir, 0755); err != nil {
sklog.Fatalf("Could not create %s: %s", *workdir, err)
}
var allow allowed.Allow
if !*baseapp.Local {
allow = allowed.NewAllowedFromList([]string{*authAllowList})
} else {
allow = allowed.NewAllowedFromList([]string{"fred@example.org", "barney@example.org", "wilma@example.org"})
}
login.SimpleInitWithAllow(*baseapp.Port, *baseapp.Local, nil, nil, allow)
ctx := context.Background()
ts, err := auth.NewDefaultTokenSource(*baseapp.Local, auth.SCOPE_USERINFO_EMAIL, auth.SCOPE_FULL_CONTROL, datastore.ScopeDatastore)
dbClient, err := db.New(ctx, ts, *fsNamespace, *fsProjectID)
if err != nil {
sklog.Fatalf("Could not init DB: %s", err)
}
// Instantiate poller and turn it on.
pollerClient, err := poller.New(ctx, ts, *serviceAccountFile, dbClient)
if err != nil {
sklog.Fatalf("Could not init poller: %s", err)
}
if err := pollerClient.Start(ctx, *pollInterval); err != nil {
sklog.Fatalf("Could not start poller: %s", err)
}
// Instantiate aliases_watcher and turn it on.
aliasesWatcher, err := aliases_watcher.New(ctx, ts, *serviceAccountFile)
if err != nil {
sklog.Fatalf("Could not init watcher: %s", err)
}
if err := aliasesWatcher.Start(ctx, *aliasesWatcherInterval); err != nil {
sklog.Fatalf("Could not start aliases watcher: %s", err)
}
srv := &Server{
pollerClient: pollerClient,
dbClient: dbClient,
}
srv.loadTemplates()
return srv, nil
}
// Server is the state of the server.
type Server struct {
pollerClient *poller.IssuesPoller
dbClient *db.FirestoreDB
templates *template.Template
}
func (srv *Server) loadTemplates() {
srv.templates = template.Must(template.New("").Delims("{%", "%}").ParseFiles(
filepath.Join(*baseapp.ResourcesDir, "index.html"),
))
}
// user returns the currently logged in user, or a placeholder if running locally.
func (srv *Server) user(r *http.Request) string {
user := "barney@example.org"
if !*baseapp.Local {
user = login.LoggedInAs(r)
}
return user
}
// See baseapp.App.
func (srv *Server) AddHandlers(r *mux.Router) {
// For login/logout.
r.HandleFunc(login.DEFAULT_OAUTH2_CALLBACK, login.OAuth2CallbackHandler)
r.HandleFunc("/logout/", login.LogoutHandler)
r.HandleFunc("/loginstatus/", login.StatusHandler)
// All endpoints that require authentication should be added to this router.
appRouter := mux.NewRouter()
appRouter.HandleFunc("/", srv.indexHandler)
appRouter.HandleFunc("/_/get_issue_counts", srv.getIssueCountsHandler).Methods("POST")
appRouter.HandleFunc("/_/get_clients_sources_queries", srv.getClients).Methods("POST")
appRouter.HandleFunc("/_/get_charts_data", srv.getChartsData).Methods("POST")
appRouter.HandleFunc("/_/get_issues_outside_slo", srv.getIssuesOutsideSLO).Methods("POST")
// Endpoints that status will use to get client counts.
r.HandleFunc("/get_client_counts", httputils.CorsHandler(srv.getClientCounts)).Methods("GET")
// Use the appRouter as a handler and wrap it into middleware that enforces authentication.
appHandler := http.Handler(appRouter)
if !*baseapp.Local {
appHandler = login.ForceAuth(appRouter, login.DEFAULT_REDIRECT_URL)
}
r.PathPrefix("/").Handler(appHandler)
}
func (srv *Server) indexHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
if err := srv.templates.ExecuteTemplate(w, "index.html", map[string]string{
// Look in webpack.config.js for where the nonce templates are injected.
"Nonce": secure.CSPNonce(r.Context()),
}); err != nil {
httputils.ReportError(w, err, "Failed to expand template.", http.StatusInternalServerError)
return
}
return
}
func (srv *Server) getClients(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
clients, err := srv.dbClient.GetClientsFromDB(r.Context())
if err != nil {
httputils.ReportError(w, err, "Failed to get clients", http.StatusInternalServerError)
return
}
resp := types.GetClientsResponse{
Clients: clients,
}
if err := json.NewEncoder(w).Encode(resp); err != nil {
sklog.Errorf("Failed to send response: %s", err)
}
}
// StatusData is used in the response of the get_client_counts endpoint.
type StatusData struct {
UntriagedCount int `json:"untriaged_count"`
Link string `json:"link"`
}
// GetClientCountsResponse is the response used by the get_client_counts endpoint.
type GetClientCountsResponse struct {
ClientsToStatusData map[types.RecognizedClient]StatusData `json:"clients_to_status_data"`
}
func (srv *Server) getClientCounts(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
clientsForStatus := []types.RecognizedClient{types.SkiaClient, types.AndroidClient, types.ChromiumClient}
clientsToStatusData := map[types.RecognizedClient]StatusData{}
for _, c := range clientsForStatus {
countsData, err := srv.dbClient.GetCountsFromDB(r.Context(), c, "", "")
if err != nil {
httputils.ReportError(w, err, "Failed to query DB.", http.StatusInternalServerError)
}
clientsToStatusData[c] = StatusData{
UntriagedCount: countsData.UntriagedCount,
Link: fmt.Sprintf("http://%s/?client=%s", *host, c),
}
}
resp := GetClientCountsResponse{
ClientsToStatusData: clientsToStatusData,
}
if err := json.NewEncoder(w).Encode(resp); err != nil {
sklog.Errorf("Failed to send response: %s", err)
}
}
func (srv *Server) getIssuesOutsideSLO(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
// Parse the request.
var req types.ClientSourceQueryRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
httputils.ReportError(w, err, "Failed to decode request.", http.StatusInternalServerError)
return
}
priToIssues := srv.pollerClient.GetOpenIssues().GetIssuesOutsideSLO(req.Client, req.Source, req.Query)
resp := types.IssuesOutsideSLOResponse{
PriToSLOIssues: priToIssues,
}
if err := json.NewEncoder(w).Encode(&resp); err != nil {
sklog.Errorf("Failed to send response: %s", err)
}
}
func (srv *Server) getChartsData(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
// Parse the request.
var req types.ClientSourceQueryRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
httputils.ReportError(w, err, "Failed to decode request.", http.StatusInternalServerError)
return
}
qds, err := srv.dbClient.GetQueryDataFromDB(context.Background(), types.RecognizedClient(req.Client), types.IssueSource(req.Source), req.Query)
if err != nil {
sklog.Fatal(err)
}
dateToCountsData := map[string]*types.IssueCountsData{}
validRunIds, err := srv.dbClient.GetAllRecognizedRunIds(r.Context())
if err != nil {
httputils.ReportError(w, err, "Failed to get valid runIds from DB", http.StatusInternalServerError)
return
}
for _, qd := range qds {
if _, ok := validRunIds[qd.RunId]; !ok {
// Ignore this query data since runId was not found.
continue
}
d := qd.RunId
if _, ok := dateToCountsData[d]; !ok {
dateToCountsData[d] = &types.IssueCountsData{}
}
dateToCountsData[d].Merge(qd.CountsData)
}
// Sort the dates in dateToCountsData.
dates := []string{}
for d := range dateToCountsData {
dates = append(dates, d)
}
sort.Slice(dates, func(i, j int) bool {
ts1, err := time.Parse(time.RFC1123, dates[i])
if err != nil {
sklog.Errorf("Could not time.Parse %s", dates[i])
}
ts2, err := time.Parse(time.RFC1123, dates[j])
if err != nil {
sklog.Errorf("Could not time.Parse %s", dates[j])
}
return ts1.Before(ts2)
})
openData := [][]interface{}{}
sloData := [][]interface{}{}
untriagedData := [][]interface{}{}
// The first row should contain column information.
openData = append(openData, []interface{}{"Date", "P0/P1", "P2", "P3+"})
sloData = append(sloData, []interface{}{"Date", "SLO: P0/P1", "SLO: P2", "SLO: P3+"})
untriagedData = append(untriagedData, []interface{}{"Date", "Untriaged"})
for _, d := range dates {
countsData := dateToCountsData[d]
openData = append(openData, []interface{}{
d, // Date
countsData.P0Count + countsData.P1Count, // P0/P1
countsData.P2Count, // P2
countsData.P3Count + countsData.P4Count + countsData.P5Count + countsData.P6Count, // P3+
})
sloData = append(sloData, []interface{}{
d, // Date
countsData.P0SLOViolationCount + countsData.P1SLOViolationCount, // SLO: P0/P1
countsData.P2SLOViolationCount, // SLO: P2
countsData.P3SLOViolationCount, // SLO: P3+
})
// We did not ingest untriaged data before the 1603288800 timestamp.
// Hack to exclude everything before so we do not see 0s in the charts.
ts, err := time.Parse(time.RFC1123, d)
if err != nil {
sklog.Errorf("Could not time.Parse %s", d)
}
if ts.After(time.Unix(1603288800, 0)) {
untriagedData = append(untriagedData, []interface{}{
d, // Date
countsData.UntriagedCount, // Untriaged
})
}
}
resp := types.GetChartsDataResponse{
OpenData: openData,
SloData: sloData,
UntriagedData: untriagedData,
}
if err := json.NewEncoder(w).Encode(&resp); err != nil {
sklog.Errorf("Failed to send response: %s", err)
}
}
func (srv *Server) getIssueCountsHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
// Parse the request.
var req types.ClientSourceQueryRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
httputils.ReportError(w, err, "Failed to decode request.", http.StatusInternalServerError)
return
}
countsData, err := srv.dbClient.GetCountsFromDB(r.Context(), req.Client, req.Source, req.Query)
if err != nil {
httputils.ReportError(w, err, "Failed to get issue counts", http.StatusInternalServerError)
return
}
if err := json.NewEncoder(w).Encode(countsData); err != nil {
sklog.Errorf("Failed to send response: %s", err)
}
}
// See baseapp.App.
func (srv *Server) AddMiddleware() []mux.MiddlewareFunc {
return []mux.MiddlewareFunc{}
}
func main() {
baseapp.Serve(New, []string{*host})
}