npm-audit-mirror/go/audits/audit.go

// audits package checks for audit issues and automatically files bugs.
package audits

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"path"
	"time"

	"go.skia.org/infra/go/executil"
	"go.skia.org/infra/go/gitiles"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/monorail/v3"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/npm-audit-mirror/go/config"
	"go.skia.org/infra/npm-audit-mirror/go/types"
)

const (
	packageFileName     = "package.json"
	packageLockFileName = "package-lock.json"

	highVulnerabilityKey = "high"

	issueSummaryTmpl     = "npm audit found high severity issues in %s’s package.json file"
	issueDescriptionTmpl = "npm audit found high severity issues in %s’s package.json file here: %s\\n\\nThis issue was automatically filed by the npm-audit-mirror framework (see go/sk-npm-audit-mirror for more information)."
	defaultIssueType     = "Task"
	defaultIssueStatus   = "Assigned"
	defaultIssuePriority = "High"

	defaultCCUser = "rmistry@google.com"
)

// File new audit issues once a quarter.
var fileAuditIssueAfterThreshold = time.Hour * 24 * 30 * 3

// NpmProjectAudit implements the types.ProjectAudit interface.
type NpmProjectAudit struct {
	projectName     string
	repoURL         string
	gitBranch       string
	packageFilesDir string
	workDir         string
	gitilesRepo     gitiles.GitilesRepo
	dbClient        types.NpmDB
	monorailConfig  *config.MonorailConfig
	monorailService monorail.IMonorailService
}

// NewNpmProjectAudit periodically downloads package.json/package-lock.json from gitiles
// and runs audit on it.
func NewNpmProjectAudit(ctx context.Context, projectName, repoURL, gitBranch, packageFilesDir, workDir, serviceAccountFilePath string, httpClient *http.Client, dbClient types.NpmDB, monorailConfig *config.MonorailConfig) (types.ProjectAudit, error) {
	gitilesRepo := gitiles.NewRepo(repoURL, httpClient)

	// Instantiate monorailService only if we have a monorailConfig.
	var monorailService *monorail.MonorailService
	var err error
	if monorailConfig != nil {
		monorailService, err = monorail.New(ctx, serviceAccountFilePath)
		if err != nil {
			return nil, skerr.Wrap(err)
		}
	}

	return &NpmProjectAudit{
		projectName:     projectName,
		repoURL:         repoURL,
		gitBranch:       gitBranch,
		packageFilesDir: packageFilesDir,
		workDir:         workDir,
		gitilesRepo:     gitilesRepo,
		dbClient:        dbClient,
		monorailConfig:  monorailConfig,
		monorailService: monorailService,
	}, nil
}

// StartAudit runs `npm audit` on the project's package.json/package-lock.json
// files. If there are any high severity issues reported then the following
// algorithm will be used:
// * Check in the DB to see if an audit issue has been filed.
// * If issue has not been filed:
//   * File a new issue and add it to the DB.
// * Else if issue has been filed:
//   * Check to see if the issue has been closed.
//   * If issue is closed:
//     * Check to see if the issue is closed more than fileAuditIssueAfterThreshold duration ago.
//     * If it is older then file a new issue and add it to the DB.
//     * Else do nothing.
//   * Else if issue is still open then do nothing.
//
// StartAudit implements the types.ProjectAudit interface.
func (a *NpmProjectAudit) StartAudit(ctx context.Context, pollInterval time.Duration) {
	liveness := metrics2.NewLiveness("npm_audit", map[string]string{
		"project": a.projectName,
	})

	go util.RepeatCtx(ctx, pollInterval, func(ctx context.Context) {
		a.oneAuditCycle(ctx, liveness)
	})
}

func (a *NpmProjectAudit) oneAuditCycle(ctx context.Context, liveness metrics2.Liveness) {
	sklog.Infof("Starting audit of %s", a.projectName)

	// Download package.json and package-lock.json
	for _, packageFile := range []string{packageFileName, packageLockFileName} {
		packageFilePath := path.Join(a.packageFilesDir, packageFile)
		dest := path.Join(a.workDir, packageFile)
		err := a.gitilesRepo.DownloadFileAtRef(ctx, packageFilePath, a.gitBranch, dest)
		if err != nil {
			sklog.Errorf("Could not download %s from %s/%s/%s: %s", packageFile, a.repoURL, a.gitBranch, a.packageFilesDir, err)
			return // return so that the liveness is not updated
		}
	}

	auditCmd := executil.CommandContext(ctx, "npm", "audit", "--json", "--audit-level=high")
	auditCmd.Dir = a.workDir
	sklog.Info(auditCmd.String())
	b, err := auditCmd.Output()
	if err != nil {
		// Ignore error here because "npm audit" exits with a non-0 code if any vulnerability is found.
	}

	var ao types.NpmAuditOutput
	if err := json.Unmarshal(b, &ao); err != nil {
		sklog.Errorf("Could not parse npm audit output for %s: %s", a.projectName, err)
		return // return so that the liveness is not updated
	}

	// Keep track of how many audit issues have high severity.
	highSeverityCounter := 0
	if issues, ok := ao.Metadata.Vulnerabilities["high"]; ok {
		highSeverityCounter = highSeverityCounter + issues
	}
	if issues, ok := ao.Metadata.Vulnerabilities["critical"]; ok {
		highSeverityCounter = highSeverityCounter + issues
	}

	sklog.Infof("Done with audit of %s. Found %d high severity issues.", a.projectName, highSeverityCounter)

	if highSeverityCounter > 0 && a.monorailConfig != nil {
		// Check in the DB to see if an audit issue has been filed.
		ad, err := a.dbClient.GetFromDB(ctx, a.projectName)
		if err != nil {
			sklog.Errorf("Could not get audit data for %s from the DB: %s", a.projectName, err)
			return // return so that the liveness is not updated
		}

		if ad == nil {
			// Issue has not been filed yet. File one and add it to the DB.
			sklog.Infof("There is no audit data for project %s in firestore. File a new issue.", a.projectName)
			if err := a.fileAndPersistAuditIssue(ctx); err != nil {
				sklog.Errorf("Could not file and persist audit issue for %s: %s", a.projectName, err)
				return // return so that the liveness is not updated
			}
		} else {
			sklog.Infof("Found audit data in firestore for project %s: %+v", a.projectName, ad)
			// Query monorail to see if the issue is closed.
			existingIssue, err := a.monorailService.GetIssue(ad.IssueName)
			if err != nil {
				sklog.Errorf("Could not query monorail for %s: %s", ad.IssueName, err)
				return // return so that the liveness is not updated
			}
			if !existingIssue.ClosedTime.IsZero() {
				// Check to see when the issue was closed.
				closedDuration := time.Now().UTC().Sub(existingIssue.ClosedTime)
				sklog.Infof("Previously filed audit issue %s was closed at %s which is %s ago.", existingIssue.Name, existingIssue.ClosedTime, closedDuration)

				if closedDuration > fileAuditIssueAfterThreshold {
					sklog.Infof("Filing new audit issue since audit issue %s was closed longer than the threshold %s.", existingIssue.Name, fileAuditIssueAfterThreshold)
					if err := a.fileAndPersistAuditIssue(ctx); err != nil {
						sklog.Errorf("Could not file and persist audit issue for %s: %s", a.projectName, err)
						return // return so that the liveness is not updated
					}
				}
			} else {
				sklog.Infof("Previously filed audit issue %s is still open. Do nothing.", existingIssue.Name)
			}
		}
	}

	liveness.Reset()
}

// fileAndPersistAuditIssue calls the monorail service to file a new monorail
// issue and then adds that issue to the DB.
func (a *NpmProjectAudit) fileAndPersistAuditIssue(ctx context.Context) error {
	mc := a.monorailConfig

	// Create a new monorail issue.
	summary := fmt.Sprintf(issueSummaryTmpl, a.projectName)
	description := fmt.Sprintf(issueDescriptionTmpl, a.projectName, path.Join(a.gitilesRepo.URL(), "+show", "refs/heads/"+a.gitBranch, a.packageFilesDir, packageFileName))

	// Always file issues with the Restrict-View-Google label.
	labels := []string{monorail.RestrictViewGoogleLabelName}
	labels = append(labels, mc.Labels...)

	newIssue, err := a.monorailService.MakeIssue(mc.InstanceName, mc.Owner, summary, description, defaultIssueStatus, defaultIssuePriority, defaultIssueType, labels, mc.ComponentDefIDs, []string{defaultCCUser})
	if err != nil {
		return skerr.Wrapf(err, "Could not create an audit issue for project %s", a.projectName)
	}
	// Add new issue data to firestore.
	if err := a.dbClient.PutInDB(ctx, a.projectName, newIssue.Name, newIssue.CreatedTime.UTC()); err != nil {
		return skerr.Wrapf(err, "Could not put issue data into firestore for project %s", a.projectName)
	}
	sklog.Infof("Filed new audit issue %s and put it in DB.", newIssue.Name)

	return nil
}