blob: 72cd1d218a6231714a99866bc61a036c1f36517f [file] [log] [blame]
// The presubmit binary runs many checks on the differences between the current commit and its
// parent branch. This is usually origin/main. If there is a chain of CLs (i.e. branches), this
// will only consider the diffs between the current commit and the parent branch.
// If all presubmits pass, this binary will output nothing (by default) and have exit code 0.
// If any presubmits fail, there will be errors logged to stdout and the exit code will be non-zero.
//
// This should be invoked from the root of the repo via Bazel like
//
// bazel run //cmd/presubmit
//
// See presubmit.sh for a helper that pipes in the correct value for repo_dir.
package main
import (
"context"
"flag"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
"go.skia.org/infra/bazel/external/buildifier"
"go.skia.org/infra/go/deepequal"
)
func main() {
var (
// https://bazel.build/docs/user-manual#running-executables
repoDir = flag.String("repo_dir", os.Getenv("BUILD_WORKSPACE_DIRECTORY"), "The root directory of the repo. Default set by BUILD_WORKSPACE_DIRECTORY env variable.")
upstream = flag.String("upstream", "origin/main", "The upstream repo to diff against.")
verbose = flag.Bool("verbose", false, "If extra logging is desired")
upload = flag.Bool("upload", false, "If true, this will skip any checks that are not suitable for an upload check (may be the empty set).")
commit = flag.Bool("commit", false, "If true, this will skip any checks that are not suitable for a commit check (may be the empty set).")
)
flag.Parse()
ctx := withOutputWriter(context.Background(), os.Stdout)
if *repoDir == "" {
logf(ctx, "Must set --repo_dir\n")
flag.PrintDefaults()
os.Exit(1)
}
if *upload && *commit {
logf(ctx, "Cannot set both --upload and --commit\n")
flag.PrintDefaults()
os.Exit(1)
}
// The pre-built binaries we need are relative to the path that Bazel starts us in.
// We need to get those paths before we re-locate to the git repo we are testing.
buildifierPath := buildifier.MustFindBuildifier()
if err := os.Chdir(*repoDir); err != nil {
logf(ctx, "Could not cd to %s\n", *repoDir)
os.Exit(1)
}
filesWithDiffs := findUncommittedChanges(ctx)
if len(filesWithDiffs) > 0 {
filesStr := strings.Join(filesWithDiffs, "\n")
logf(ctx, "Found uncommitted changes in %d files. Aborting.\n:%s", len(filesWithDiffs), filesStr)
os.Exit(1)
}
untrackedFiles := findUntrackedFiles(ctx)
for _, uf := range untrackedFiles {
if filepath.Base(uf) == "BUILD.bazel" {
logf(ctx, "Found uncommitted BUILD.bazel files. Please delete these or check them in. Aborting.\n")
os.Exit(1)
}
}
branchBaseCommit := findBranchBase(ctx, *upstream)
if branchBaseCommit == "" {
// This either means the user hasn't committed their changes or is just on the main branch
// somewhere. Either way, we don't want to run the presubmits. It could mutate those
// un-committed changes or just be a no-op, since presumably code in the past passed the
// presubmit checks.
logf(ctx, "No commits since %s. Presubmit passes by default. Did you commit all new files?\n", *upstream)
os.Exit(0)
}
if *verbose {
logf(ctx, "Base commit is %s\n", branchBaseCommit)
}
changedFiles, deletedFiles := computeDiffFiles(ctx, branchBaseCommit)
if *verbose {
logf(ctx, "Changed files:\n%v\n", changedFiles)
logf(ctx, "Deleted files:\n%s\n", deletedFiles)
}
// Run checks and keep track of errors.
//
// We run the code-mutating checks first (gofmt, gazelle, etc.), followed by those that do not
// mutate code (line length, stray whitespace, etc.). This prevents us from reporting errors that
// might go away after running a code-mutating check, e.g. a line length error that goes away
// after running Prettier.
//
// We compute a new Git diff after each code-mutating check so that subsequent code-mutating
// checks can compare against the last check, rather than against the whole CL. This allows
// code-mutating checks to only fail due to their own diffs, rather than due to diffs from
// a previous check.
anyErrors := false
trackErrors := func(ok bool) { anyErrors = anyErrors || !ok }
trackErrors(runBuildifier(ctx, buildifierPath, changedFiles, branchBaseCommit))
changedFiles, _ = computeDiffFiles(ctx, branchBaseCommit)
trackErrors(runGoimports(ctx, changedFiles, *repoDir, branchBaseCommit))
changedFiles, _ = computeDiffFiles(ctx, branchBaseCommit)
trackErrors(runGofmt(ctx, changedFiles, branchBaseCommit))
changedFiles, _ = computeDiffFiles(ctx, branchBaseCommit)
trackErrors(runGoVet(ctx, changedFiles, branchBaseCommit))
if *commit {
// When running the presubmit checks on CI, we must manually ensure that the node_modules
// directory exists because Bazel no longer manages that directory for us. Running "npm ci"
// accomplishes this. If the node_modules directory does not exist, the prettier step below
// will fail.
//
// When running the presubmit checks as part of the "git cl upload" command, it is the
// developer's responsibility to keep their node_modules directory up-to-date.
trackErrors(runNpmCi(ctx))
}
trackErrors(runPrettier(ctx, changedFiles, *repoDir, branchBaseCommit))
changedFiles, _ = computeDiffFiles(ctx, branchBaseCommit)
trackErrors(runESLint(ctx, changedFiles, *repoDir, branchBaseCommit))
changedFiles, _ = computeDiffFiles(ctx, branchBaseCommit)
trackErrors(runGazelle(ctx, changedFiles, deletedFiles, branchBaseCommit))
changedFiles, _ = computeDiffFiles(ctx, branchBaseCommit)
trackErrors(checkTODOHasOwner(ctx, changedFiles))
trackErrors(checkForStrayWhitespace(ctx, changedFiles))
trackErrors(checkPythonFilesHaveNoTabs(ctx, changedFiles))
trackErrors(checkBannedGoAPIs(ctx, changedFiles))
trackErrors(checkJSDebugging(ctx, changedFiles))
if !*commit {
// Long lines are sometimes inevitable. Ideally we would add these long line files to
// the excluded list, but sometimes that is hard to do precisely.
trackErrors(checkLongLines(ctx, changedFiles))
// Give warnings for non-ASCII characters on upload but not commit, since they may
// be intentional.
trackErrors(checkNonASCII(ctx, changedFiles))
}
if anyErrors {
logf(ctx, "Presubmit errors detected!\n")
os.Exit(1)
}
os.Exit(0)
}
const (
gitErrorMessage = `Error running git - is git on your path?
If running with Bazel, you need to invoke this like:
bazel run //cmd/presubmit --run_under="cd $PWD &&"
`
refSeperator = "$|" // Some string we hope that no users start their branch names with
)
// findUncommittedChanges returns a list of files that git says have changed (compared to HEAD).
func findUncommittedChanges(ctx context.Context) []string {
// diff-index is one of the git "plumbing" commands and the output should be relatively stable.
// https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git.html#_low_level_commands_plumbing
cmd := exec.CommandContext(ctx, "git", "diff-index", "HEAD")
output, err := cmd.CombinedOutput()
if err != nil {
logf(ctx, string(output)+"\n")
logf(ctx, err.Error()+"\n")
panic(gitErrorMessage)
}
return extractFilesWithDiffs(string(output))
}
var fileDiff = regexp.MustCompile(`^:.+\t(?P<file>[^\t]+)$`)
func extractFilesWithDiffs(output string) []string {
output = strings.TrimSpace(output)
if output == "" {
return nil
}
var files []string
lines := strings.Split(output, "\n")
for _, line := range lines {
if match := fileDiff.FindStringSubmatch(line); len(match) > 0 {
files = append(files, match[1])
}
}
return files
}
// findUntrackedFiles returns a list of files untracked and unignored by Git.
func findUntrackedFiles(ctx context.Context) []string {
// https://stackoverflow.com/a/2659808/1447621
// This will list all untracked, unignored files on their own lines
cmd := exec.CommandContext(ctx, "git", "ls-files", "--others", "--exclude-standard")
output, err := cmd.CombinedOutput()
if err != nil {
logf(ctx, string(output)+"\n")
logf(ctx, err.Error()+"\n")
panic(gitErrorMessage)
}
return strings.Split(string(output), "\n")
}
// findBranchBase returns the git commit of the parent branch. If there is a chain of CLs
// (i.e. branches), this will return the parent branch's most recent commit. If we are on the
// upstream branch, this will return empty string. Otherwise, it will return the commit on the
// upstream branch where branching occurred. It shells out to git, which is presumed to be on
// PATH in order to find this information.
func findBranchBase(ctx context.Context, upstream string) string {
// rev-list is one of the git "plumbing" commands and the output should be relatively stable.
// https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git.html#_low_level_commands_plumbing
cmd := exec.CommandContext(ctx, "git", "rev-list", "HEAD", "^"+upstream,
// %D means "ref names", which is the commit hash and any branch name associated with it
// %P means the parent hash
`--format=%D`+refSeperator+`%P`)
output, err := cmd.CombinedOutput()
if err != nil {
logf(ctx, string(output)+"\n")
logf(ctx, err.Error()+"\n")
panic(gitErrorMessage)
}
return extractBranchBase(string(output))
}
type revEntry struct {
commit string
branch string
parents []string
// depth is a monotonically increasing number for each commit we see as we are going back
// in time.
depth int
}
// extractBranchBase looks for the most recent branch, as indicated by the "ref name". Failing to find
// that, it will return the last parent commit, which will connect to the upstream branch.
func extractBranchBase(output string) string {
output = strings.TrimSpace(output)
if output == "" {
return ""
}
lines := strings.Split(output, "\n")
// Create a graph of commits. The entries map holds onto all the nodes (using the commit as
// a key). After we create the graph, we'll look for the key features.
entries := map[string]*revEntry{}
var currentEntry *revEntry
depth := 0
for _, line := range lines {
if strings.HasPrefix(line, "commit ") {
c := strings.TrimPrefix(line, "commit ")
entry := entries[c]
if entry == nil {
entry = &revEntry{commit: c, depth: depth}
entries[c] = entry
} else {
entry.depth = depth
}
currentEntry = entry
depth++
continue
}
parts := strings.Split(line, refSeperator)
// First part is the possibly empty branch name
currentEntry.branch = parts[0]
// second part is the possibly multiple parents, seperated by spaces
parents := strings.Split(parts[1], " ")
currentEntry.parents = parents
for _, parent := range parents {
// Associate the parents with the depth of the child they apply to.
entries[parent] = &revEntry{commit: parent, depth: depth}
}
}
// Go through the created graph and find commits of interest
var shallowestCommitWithNoParents *revEntry
var shallowestCommitWithBranch *revEntry
for _, entry := range entries {
if len(entry.parents) == 0 {
if shallowestCommitWithNoParents == nil || shallowestCommitWithNoParents.depth > entry.depth {
shallowestCommitWithNoParents = entry
}
}
if entry.branch != "" && !strings.HasPrefix(entry.branch, "HEAD -> ") {
if shallowestCommitWithBranch == nil || shallowestCommitWithBranch.depth > entry.depth {
shallowestCommitWithBranch = entry
}
}
}
// If we found a branch that HEAD descends from, compare to the shallowest commit belonging
// to that branch.
if shallowestCommitWithBranch != nil {
return shallowestCommitWithBranch.commit
}
// Otherwise, go with the shallowest commit that we didn't find parents for. These parent-less
// commits correspond to commits on the main branch, and the shallowest one will be the newest.
if shallowestCommitWithNoParents != nil {
return shallowestCommitWithNoParents.commit
}
// This should not happen unless we are parsing things wrong.
panic("Could not find a branch to compare to")
}
// computeDiffFiles returns a slice of changed (modified or added) files with the lines touched
// and a slice of deleted files. It shells out to git, which is presumed to be on PATH in order
// to find this information.
func computeDiffFiles(ctx context.Context, branchBase string) ([]fileWithChanges, []string) {
// git diff-index is considered to be a "git plumbing" API, so its output should be pretty
// stable across git version (unlike ordinary git diff, which can do things like show
// tabs as multiple spaces).
cmd := exec.CommandContext(ctx, "git", "diff-index", branchBase,
// Don't show any surrounding context (i.e. lines which did not change)
"--unified=0",
"--patch-with-raw",
"--no-color")
output, err := cmd.CombinedOutput()
if err != nil {
logf(ctx, string(output)+"\n")
logf(ctx, err.Error()+"\n")
panic(gitErrorMessage)
}
return extractChangedAndDeletedFiles(string(output))
}
var (
gitDiffLine = regexp.MustCompile(`^diff --git (?P<fileA>.*) (?P<fileB>.*)$`)
lineAnchor = regexp.MustCompile(`^@@ -(?P<deleted>\d+)(?P<delLines>,\d+)? \+(?P<added>\d+)(?P<addLines>,\d+)? @@.*$`)
)
const (
deletedFilePrefix = "deleted file mode"
addedLinePrefix = "+"
)
// fileWithChanges represents an added or modified files along with the lines changed (touched).
// Lines that were deleted are not tracked.
type fileWithChanges struct {
fileName string
touchedLines []lineOfCode
}
func (f fileWithChanges) String() string {
rv := f.fileName + "\n"
for _, line := range f.touchedLines {
rv += " " + line.String() + "\n"
}
return rv
}
type lineOfCode struct {
contents string
num int
}
func (c lineOfCode) String() string {
return fmt.Sprintf("% 4d:%s", c.num, c.contents)
}
// extractChangedAndDeletedFiles looks through the provided `git diff` output and finds the files
// that were added or modified, as well as the new version of any lines touched. It also returns
// a slice of deleted files.
func extractChangedAndDeletedFiles(diffOutput string) ([]fileWithChanges, []string) {
var changed []fileWithChanges
var deleted []string
lines := strings.Split(diffOutput, "\n")
currFileDeleted := false
lastLineIndex := -1
for i := 0; i < len(lines); i++ {
line := lines[i]
if match := gitDiffLine.FindStringSubmatch(line); len(match) > 0 {
// A new file was changed, reset our counters, trackers, and add an entry for it.
var newFile fileWithChanges
newFile.fileName = strings.TrimPrefix(match[2], "b/")
changed = append(changed, newFile)
currFileDeleted = false
lastLineIndex = -1
}
if currFileDeleted || len(changed) == 0 {
continue
}
currFile := &changed[len(changed)-1]
if strings.HasPrefix(line, deletedFilePrefix) {
deleted = append(deleted, currFile.fileName)
changed = changed[:len(changed)-1] // trim off changed file
currFileDeleted = true
continue
}
if match := lineAnchor.FindStringSubmatch(line); len(match) > 0 {
// The added group has the line number corresponding to the next line starting with a +
var err error
lastLineIndex, err = strconv.Atoi(match[3])
if err != nil {
panic("Got an integer where none was expected: " + line + "\n" + err.Error())
}
continue
}
if lastLineIndex < 0 {
// Have not found a line index yet, ignore the lines like:
// ++ b/PRESUBMIT.py
continue
}
if strings.HasPrefix(line, addedLinePrefix) {
currFile.touchedLines = append(currFile.touchedLines, lineOfCode{
contents: strings.TrimPrefix(line, addedLinePrefix),
num: lastLineIndex,
})
lastLineIndex++
}
}
return changed, deleted
}
// checkLongLines looks through all touched lines and returns false if any of them (not covered by
// exceptions) have lines longer than 100 lines, an arbitrary measurement.
// Based on https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/19b3eb5adbe00e9da40375cb5dc47380a46f3041/presubmit_canned_checks.py#488
func checkLongLines(ctx context.Context, files []fileWithChanges) bool {
const maxLineLength = 100
ignoreFileExts := []string{".go", ".html", ".py"}
ignoreFiles := []string{"package-lock.json", "pnpm-lock.yaml", "go.sum", "infra/bots/tasks.json", "WORKSPACE",
"golden/k8s-config-templates/gold-common.json5"}
ok := true
for _, f := range files {
if contains(ignoreFiles, f.fileName) {
continue
}
if contains(ignoreFileExts, filepath.Ext(f.fileName)) {
continue
}
for _, line := range f.touchedLines {
if len(line.contents) > maxLineLength {
logf(ctx, "%s:%d Line too long (%d/%d)\n", f.fileName, line.num, len(line.contents), maxLineLength)
ok = false
}
}
}
return ok
}
var todoWithoutOwner = regexp.MustCompile(`TODO[^(]`)
// checkTODOHasOwner looks through all touched lines and returns false if any of them (not covered
// by exceptions) have a TODO without an owner or a bug.
// Based on https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/19b3eb5adbe00e9da40375cb5dc47380a46f3041/presubmit_canned_checks.py#464
func checkTODOHasOwner(ctx context.Context, files []fileWithChanges) bool {
ignoreFiles := []string{
// These files have TODO in their function names and test data.
"cmd/presubmit/presubmit.go", "cmd/presubmit/presubmit_test.go",
}
ok := true
for _, f := range files {
if contains(ignoreFiles, f.fileName) {
continue
}
for _, line := range f.touchedLines {
if todoWithoutOwner.MatchString(line.contents) || strings.HasSuffix(line.contents, "TODO") {
logf(ctx, "%s:%d TODO without owner or bug\n", f.fileName, line.num)
ok = false
}
}
}
return ok
}
var trailingWhitespace = regexp.MustCompile(`\s+$`)
// checkForStrayWhitespace goes through all touched lines and returns false if any of them end with
// a whitespace character (e.g. tabs, spaces). newlines should have been stripped out of
// the content of a line earlier.
// Based on https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/19b3eb5adbe00e9da40375cb5dc47380a46f3041/presubmit_canned_checks.py#476
func checkForStrayWhitespace(ctx context.Context, files []fileWithChanges) bool {
ok := true
for _, f := range files {
for _, line := range f.touchedLines {
if trailingWhitespace.MatchString(line.contents) {
logf(ctx, "%s:%d Trailing whitespace\n", f.fileName, line.num)
ok = false
}
}
}
return ok
}
// checkPythonFilesHaveNoTabs goes through the touched lines of all Python files and returns false
// if any of them have tabs anywhere.
// Based on https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/19b3eb5adbe00e9da40375cb5dc47380a46f3041/presubmit_canned_checks.py#441
func checkPythonFilesHaveNoTabs(ctx context.Context, files []fileWithChanges) bool {
ok := true
for _, f := range files {
if filepath.Ext(f.fileName) != ".py" {
continue
}
for _, line := range f.touchedLines {
if strings.Contains(line.contents, "\t") {
logf(ctx, "%s:%d Tab character not allowed\n", f.fileName, line.num)
ok = false
}
}
}
return ok
}
type bannedGoAPI struct {
regex *regexp.Regexp
suggestion string
exceptions []*regexp.Regexp
}
// checkBannedGoAPIs goes through all touched lines in go files and returns false if any of them
// have APIs that we wish not to use. It logs suggested replacements in that case.
func checkBannedGoAPIs(ctx context.Context, files []fileWithChanges) bool {
bannedAPIs := []bannedGoAPI{
{regex: regexp.MustCompile(`reflect\.DeepEqual`), suggestion: "Equal in go.skia.org/infra/go/deepequal/assertdeep"},
{regex: regexp.MustCompile(`github\.com/golang/glog`), suggestion: "go.skia.org/infra/go/sklog"},
{regex: regexp.MustCompile(`github\.com/skia-dev/glog`), suggestion: "go.skia.org/infra/go/sklog"},
{regex: regexp.MustCompile(`http\.Get`), suggestion: "NewTimeoutClient in go.skia.org/infra/go/httputils"},
{regex: regexp.MustCompile(`http\.Head`), suggestion: "NewTimeoutClient in go.skia.org/infra/go/httputils",
exceptions: []*regexp.Regexp{
// generated file we have no control over.
regexp.MustCompile(`go/issuetracker/.*\.go`),
}},
{regex: regexp.MustCompile(`http\.Post`), suggestion: "NewTimeoutClient in go.skia.org/infra/go/httputils"},
{regex: regexp.MustCompile(`http\.PostForm`), suggestion: "NewTimeoutClient in go.skia.org/infra/go/httputils"},
{regex: regexp.MustCompile(`os\.Interrupt`), suggestion: "AtExit in go.skia.org/go/cleanup"},
{regex: regexp.MustCompile(`signal\.Notify`), suggestion: "AtExit in go.skia.org/go/cleanup"},
{regex: regexp.MustCompile(`syscall\.SIGINT`), suggestion: "AtExit in go.skia.org/go/cleanup"},
{regex: regexp.MustCompile(`syscall\.SIGTERM`), suggestion: "AtExit in go.skia.org/go/cleanup"},
{regex: regexp.MustCompile(`syncmap\.Map`), suggestion: "sync.Map, added in go 1.9"},
{regex: regexp.MustCompile(`assert\s+"github\.com/stretchr/testify/require"`), suggestion: `non-aliased import; this can be confused with package "github.com/stretchr/testify/assert"`},
{
regex: regexp.MustCompile(`"git"`),
suggestion: `Executable in go.skia.org/infra/go/git`,
exceptions: []*regexp.Regexp{
// These don't actually shell out to git; the tests look for "git" in the
// command line and mock stdout accordingly.
regexp.MustCompile(`autoroll/go/repo_manager/.*_test\.go`),
// This doesn't shell out to git; it's referring to a CIPD package with
// the same name.
regexp.MustCompile(`infra/bots/gen_tasks\.go`),
// This doesn't shell out to git; it retrieves the path to the Git binary
// in the corresponding Bazel-downloaded CIPD packages.
regexp.MustCompile(`bazel/external/cipd/git/git\.go`),
// Our presubmits invoke git directly because git is a necessary
// executable for all devs, and we do not want our presubmit code to
// depend on the code it is checking.
regexp.MustCompile(`cmd/presubmit/.*`),
// This doesn't shell out to git.
regexp.MustCompile(`go/depot_tools/deps_parser/.*\.go`),
// This is the one place where we are allowed to shell out to git; all
// others should go through here.
regexp.MustCompile(`go/git/git_common/.*\.go`),
// Just using the word "git" as a config value.
regexp.MustCompile(`perf/go/config/.*\.go`),
// Just using the word "git" as a directory name.
regexp.MustCompile(`perf/go/git/providers/git_checkout/.*\.go`),
},
},
}
ok := true
for _, f := range files {
if filepath.Ext(f.fileName) != ".go" {
continue
}
if f.fileName == "cmd/presubmit/presubmit_test.go" {
// We don't want our own test cases to trigger any of these.
continue
}
for _, line := range f.touchedLines {
bannedAPILoop:
for _, bannedAPI := range bannedAPIs {
for _, exception := range bannedAPI.exceptions {
if exception.MatchString(f.fileName) {
continue bannedAPILoop
}
}
if match := bannedAPI.regex.FindStringSubmatch(line.contents); len(match) > 0 {
logf(ctx, "%s:%d Instead of %s, please use %s\n", f.fileName, line.num, match[0], bannedAPI.suggestion)
ok = false
}
}
}
}
return ok
}
// checkJSDebugging goes through all touched lines and returns false if any TS or JS files contain
// refinements of debugging that we don't want to check in.
func checkJSDebugging(ctx context.Context, files []fileWithChanges) bool {
debuggingCalls := []string{"debugger;", "it.only(", "describe.only("}
targetFileExts := []string{".ts", ".js"}
ok := true
for _, f := range files {
if !contains(targetFileExts, filepath.Ext(f.fileName)) {
continue
}
for _, line := range f.touchedLines {
for _, call := range debuggingCalls {
if strings.Contains(line.contents, call) {
logf(ctx, "%s:%d debugging code found (%s)\n", f.fileName, line.num, call)
ok = false
}
}
}
}
return ok
}
// checkNonASCII goes through all touched lines and returns false if any of them contain non-ASCII
// characters (except for file formats that support things like UTF-8).
func checkNonASCII(ctx context.Context, files []fileWithChanges) bool {
// This list can grow if other file extensions are OK with non-ascii (UTF-8) characters
ignoreFileExts := []string{".go"}
ok := true
for _, f := range files {
if contains(ignoreFileExts, filepath.Ext(f.fileName)) {
continue
}
for _, line := range f.touchedLines {
// https://stackoverflow.com/a/53069799/1447621
for i := 0; i < len(line.contents); i++ {
if line.contents[i] > '\u007F' { // unicode.MaxASCII
// Report both line number and (1-indexed) byte offset
logf(ctx, "%s:%d:%d Non ASCII character found\n", f.fileName, line.num, i+1)
ok = false
break
}
}
}
}
return ok
}
// runBuildifier uses a provided buildifier path to reformat our BUILD.bazel and .bzl files as
// well as check them for linting errors. If buildifier has no output and a non-zero exit code,
// that is interpreted as "all good" and we return true. If there are issues, we print the
// buildifier output (which has files and line numbers) and return false.
func runBuildifier(ctx context.Context, buildifierPath string, files []fileWithChanges, branchBaseCommit string) bool {
args := []string{"-lint=warn", "-mode=fix"}
foundAny := false
for _, f := range files {
if filepath.Base(f.fileName) == "BUILD.bazel" || filepath.Ext(f.fileName) == ".bzl" {
args = append(args, f.fileName)
foundAny = true
}
}
if !foundAny {
return true
}
cmd := exec.CommandContext(ctx, buildifierPath, args...)
output, err := cmd.CombinedOutput()
if err != nil {
logf(ctx, string(output))
logf(ctx, "Buildifier linting errors detected!\n")
return false
}
if changedFiles, _ := computeDiffFiles(ctx, branchBaseCommit); !deepequal.DeepEqual(files, changedFiles) {
logf(ctx, "Buildifier caused additional changes. %+v \n", changedFiles)
return false
}
return true
}
// runGoimports runs goimports on any changed golang files. It returns false if goimports fails or
// produces any diffs.
func runGoimports(ctx context.Context, files []fileWithChanges, workspaceRoot, branchBaseCommit string) bool {
// -w means "write", as in, modify the files that need formatting.
args := []string{"run", "--config=mayberemote", "//:goimports", "--run_under=cd " + workspaceRoot + " &&", "--", "-w"}
foundAny := false
for _, f := range files {
if filepath.Ext(f.fileName) == ".go" {
args = append(args, f.fileName)
foundAny = true
}
}
if !foundAny {
return true
}
cmd := exec.CommandContext(ctx, "bazelisk", args...)
output, err := cmd.CombinedOutput()
if err != nil {
logf(ctx, string(output))
logf(ctx, "goimports failed!\n")
return false
}
if changedFiles, _ := computeDiffFiles(ctx, branchBaseCommit); !deepequal.DeepEqual(files, changedFiles) {
logf(ctx, "goimports caused additional changes. %+v \n", changedFiles)
return false
}
return true
}
// runNpmCi runs "npm ci" to ensure that the "node_modules" directory exists.
func runNpmCi(ctx context.Context) bool {
cmd := exec.CommandContext(ctx, "bazelisk", "run", "--config=mayberemote", "//:npm", "--", "ci")
output, err := cmd.CombinedOutput()
if err != nil {
logf(ctx, "Command \"npm ci\" failed. Output:\n")
logf(ctx, string(output))
return false
}
return true
}
// runESLint runs eslint on any changed files. It returns false if prettier
// returns a non-zero error code.
func runESLint(ctx context.Context, files []fileWithChanges, workspaceRoot, branchBaseCommit string) bool {
args := []string{"run", "--config=mayberemote", "//:npx", "--", "eslint", "--quiet"}
numFiles := 0
for _, f := range files {
if filepath.Ext(f.fileName) == ".ts" {
args = append(args, f.fileName)
numFiles++
}
}
if numFiles == 0 {
return true
}
cmd := exec.CommandContext(ctx, "bazelisk", args...)
output, err := cmd.CombinedOutput()
if err != nil {
logf(ctx, string(output))
logf(ctx, "eslint failed!\n")
return false
}
return true
}
// runPrettier runs prettier --write on any changed files. It returns false if
// prettier returns a non-zero error code.
func runPrettier(ctx context.Context, files []fileWithChanges, workspaceRoot, branchBaseCommit string) bool {
args := []string{"run", "--config=mayberemote", "//:npx", "--", "prettier", "--write", "--ignore-unknown"}
for _, f := range files {
args = append(args, f.fileName)
}
cmd := exec.CommandContext(ctx, "bazelisk", args...)
output, err := cmd.CombinedOutput()
if err != nil {
logf(ctx, string(output))
logf(ctx, "prettier failed!\n")
return false
}
if changedFiles, _ := computeDiffFiles(ctx, branchBaseCommit); !deepequal.DeepEqual(files, changedFiles) {
logf(ctx, "Prettier caused additional changes. %+v \n", changedFiles)
return false
}
return true
}
// runGofmt runs gofmt on any changed golang files. It returns false if gofmt fails or
// produces any diffs.
func runGofmt(ctx context.Context, files []fileWithChanges, branchBaseCommit string) bool {
// -s means "simplify"
// -w means "write", as in, modify the files that need formatting.
args := []string{"run", "--config=mayberemote", "//:gofmt", "--", "-s", "-w"}
foundAny := false
for _, f := range files {
if filepath.Ext(f.fileName) == ".go" {
args = append(args, f.fileName)
}
}
if !foundAny {
return true
}
cmd := exec.CommandContext(ctx, "bazelisk", args...)
output, err := cmd.CombinedOutput()
if err != nil {
logf(ctx, string(output))
logf(ctx, "gofmt failed!\n")
return false
}
if changedFiles, _ := computeDiffFiles(ctx, branchBaseCommit); !deepequal.DeepEqual(files, changedFiles) {
logf(ctx, "gofmt caused additional changes. %+v \n", changedFiles)
return false
}
return true
}
// runGoVet runs `go vet` on all golang files. It returns false if go vet fails.
func runGoVet(ctx context.Context, files []fileWithChanges, branchBaseCommit string) bool {
args := []string{"run", "--config=mayberemote", "//:go", "--", "vet", "./..."}
cmd := exec.CommandContext(ctx, "bazelisk", args...)
output, err := cmd.CombinedOutput()
if err != nil {
logf(ctx, string(output))
logf(ctx, "go vet failed!\n")
return false
}
return true
}
// runGazelle uses gazelle (and our custom gazelle plugin) to regenerate BUILD.bazel files for our
// go files as well as our Typescript and SCSS rules. Gazelle is idempotent, so if a user has
// already generated BUILD.bazel files, there should be no diffs. If the user forgot to do so,
// there could be diffs or new files added. This function returns false if that is the case or true
// if Gazelle made no modifications.
func runGazelle(ctx context.Context, changedFiles []fileWithChanges, deletedFiles []string, branchBaseCommit string) bool {
// If these change, we should regenerate everything (slower, but more sound)
globalFilesToCheck := []string{"WORKSPACE", "WORKSPACE.bazel", "go.mod", "go.sum"}
globalExtensionsToCheck := []string{".bzl"}
// If these change, then we should only need to update their containing folders.
localFilesToCheck := []string{"BUILD.bazel"}
localExtensionsToCheck := []string{".go", ".ts", ".scss"}
var foldersToCheck []string
regenEverything := len(deletedFiles) > 0
if !regenEverything {
for _, f := range changedFiles {
if contains(globalFilesToCheck, filepath.Base(f.fileName)) || contains(globalExtensionsToCheck, filepath.Ext(f.fileName)) {
regenEverything = true
break
}
if contains(localFilesToCheck, filepath.Base(f.fileName)) || contains(localExtensionsToCheck, filepath.Ext(f.fileName)) {
folder := filepath.Dir(f.fileName)
if !contains(foldersToCheck, folder) {
foldersToCheck = append(foldersToCheck, folder)
}
}
}
}
// No need to run gazelle
if !regenEverything && len(foldersToCheck) == 0 {
return true
}
if regenEverything {
cmd := exec.CommandContext(ctx, "bazelisk", "run", "--config=mayberemote", "//:gazelle", "--", "update-repos",
"-from_file=go.mod", "-to_macro=go_repositories.bzl%go_repositories")
output, err := cmd.CombinedOutput()
if err != nil {
logf(ctx, string(output))
logf(ctx, "Could not regenerate go_repositories.bzl!\n")
return false
}
}
args := []string{"run", "--config=mayberemote", "//:gazelle", "--", "update"}
if regenEverything {
// Reminder: we have changed directory into the repo root
args = append(args, "./")
} else {
args = append(args, foldersToCheck...)
}
cmd := exec.CommandContext(ctx, "bazelisk", args...)
output, err := cmd.CombinedOutput()
if err != nil {
logf(ctx, string(output))
logf(ctx, "Could not regenerate BUILD.bazel changedFiles using gazelle!\n")
return false
}
if newChangedFiles, _ := computeDiffFiles(ctx, branchBaseCommit); !deepequal.DeepEqual(changedFiles, newChangedFiles) {
logf(ctx, "Gazelle caused changes. Please inspect them (git diff) and commit if ok.\n")
return false
}
for _, f := range findUntrackedFiles(ctx) {
if filepath.Base(f) == "BUILD.bazel" {
logf(ctx, "Gazelle created new BUILD.bazel files. Please inspect these and check them in.\n")
return false
}
}
return true
}
// contains returns true if the given slice has the provided element in it.
func contains[T string](a []T, s T) bool {
for _, x := range a {
if x == s {
return true
}
}
return false
}
type contextKeyType string
const outputWriterKey contextKeyType = "outputWriter"
// withOutputWriter registers the given writer to the context. See also logf.
func withOutputWriter(ctx context.Context, w io.Writer) context.Context {
return context.WithValue(ctx, outputWriterKey, w)
}
// logf takes the writer on the context and writes the given format and arguments to it. This allows
// unit tests to intercept logged output if necessary. It panics if a writer was not registered
// using withOutputWriter.
func logf(ctx context.Context, format string, args ...interface{}) {
w, ok := ctx.Value(outputWriterKey).(io.Writer)
if !ok {
panic("Must set outputWriter on ctx")
}
_, err := fmt.Fprintf(w, format, args...)
if err != nil {
panic("Error while logging " + err.Error())
}
}