| package main |
| |
| import ( |
| "encoding/json" |
| "os" |
| |
| "github.com/spf13/cobra" |
| "go.skia.org/infra/gold-client/go/goldclient" |
| "go.skia.org/infra/golden/go/jsonio" |
| "go.skia.org/infra/golden/go/shared" |
| ) |
| |
| // imgTestEnv is the environment for the imgtest command ant its sub-commands. |
| type imgTestEnv struct { |
| // Flags used by imgtest:init and imgtest:add. |
| flagCommit string // flag containing the commit hash |
| flagKeysFile string |
| flagIssueID string |
| flagPatchsetID string |
| flagJobID string |
| flagInstanceID string |
| flagWorkDir string |
| flagPassFailStep bool |
| flagFailureFile string |
| flagURL string |
| |
| // Flags used by imgtest:add |
| flagTestName string |
| flagPNGFile string |
| } |
| |
| // getImgTestCmd returns the definition of the imgtest command. |
| func getImgTestCmd() *cobra.Command { |
| env := &imgTestEnv{} |
| |
| // imgtest command and its sub commands |
| imgTestCmd := &cobra.Command{ |
| Use: "imgtest", |
| Short: "Collect and upload test results as images", |
| Long: ` |
| Collect and upload test results to the Gold backend.`, |
| } |
| |
| // cmd: imgtest init |
| imgTestInitCmd := &cobra.Command{ |
| Use: "init", |
| Short: "Initialize a testing environment", |
| Long: ` |
| Start a testing session during which tests are added. This initializes the environment. |
| It gathers whether the 'add' command returns a pass/fail value and the common |
| keys shared by all tests that are added via 'add'.`, |
| Run: env.runImgTestInitCmd, |
| } |
| env.addCommonFlags(imgTestInitCmd, false) |
| |
| imgTestAddCmd := &cobra.Command{ |
| Use: "add", |
| Short: "Adds a test image to the results.", |
| Long: ` |
| Add images generated by the tests to the test results. This requires two arguments: |
| - The test name |
| - The path to the resulting PNG. |
| `, |
| Run: env.runImgTestAddCmd, |
| Args: cobra.NoArgs, |
| } |
| env.addCommonFlags(imgTestAddCmd, false) |
| imgTestAddCmd.Flags().StringVarP(&env.flagTestName, "test-name", "", "", "Unique name of the test, must not contain spaces.") |
| imgTestAddCmd.Flags().StringVarP(&env.flagPNGFile, "png-file", "", "", "Path to the PNG file that contains the test results.") |
| |
| imgTestFinalizeCmd := &cobra.Command{ |
| Use: "finalize", |
| Short: "Finish adding tests and process results.", |
| Long: ` |
| All tests have been added. Upload images and generate and upload the JSON file that captures |
| test results.`, |
| Run: env.runImgTestFinalizeCmd, |
| } |
| |
| imgTestPassFailCmd := &cobra.Command{ |
| Use: "passfail", |
| Short: "Checks whether the results match expectations", |
| Long: ` |
| Check against Gold or local baseline whether the results match the expectations`, |
| Run: env.runImgTestPassFailCmd, |
| } |
| |
| // assemble the imgtest command. |
| imgTestCmd.AddCommand( |
| imgTestInitCmd, |
| imgTestAddCmd, |
| imgTestFinalizeCmd, |
| imgTestPassFailCmd, |
| ) |
| return imgTestCmd |
| } |
| |
| func (i *imgTestEnv) addCommonFlags(cmd *cobra.Command, optional bool) { |
| cmd.Flags().StringVarP(&i.flagInstanceID, "instance", "", "", "ID of the Gold instance.") |
| cmd.Flags().StringVarP(&i.flagWorkDir, fstrWorkDir, "", "", "Temporary work directory") |
| cmd.Flags().BoolVarP(&i.flagPassFailStep, "passfail", "", false, "Whether the 'add' call returns a pass/fail for each test.") |
| |
| cmd.Flags().StringVarP(&i.flagCommit, "commit", "", "", "Git commit hash") |
| cmd.Flags().StringVarP(&i.flagKeysFile, "keys-file", "", "", "JSON file containing key/value pairs commmon to all tests") |
| cmd.Flags().StringVarP(&i.flagIssueID, "issue", "", "", "Gerrit issue if this is trybot run. ") |
| cmd.Flags().StringVarP(&i.flagPatchsetID, "patchset", "", "", "Gerrit patchset number if this is a trybot run. ") |
| cmd.Flags().StringVarP(&i.flagJobID, "jobid", "", "", "Job ID if this is a tryjob run. Current the BuildBucket id.") |
| cmd.Flags().StringVarP(&i.flagFailureFile, "failure-file", "", "", "Path to the file where to write failure information") |
| cmd.Flags().StringVarP(&i.flagURL, "url", "", "", "URL of the Gold instance. Used for testing, if empty the URL will be derived from the value of 'instance'") |
| |
| if !optional { |
| _ = cmd.MarkFlagRequired("instance") |
| _ = cmd.MarkFlagRequired(fstrWorkDir) |
| _ = cmd.MarkFlagRequired("passfail") |
| _ = cmd.MarkFlagRequired("commit") |
| _ = cmd.MarkFlagRequired("keys-file") |
| } |
| } |
| |
| // TODO(stephana): Implement these stubbed out sub-commands. |
| func (i *imgTestEnv) runImgTestInitCmd(cmd *cobra.Command, args []string) { notImplemented(cmd) } |
| func (i *imgTestEnv) runImgTestFinalizeCmd(cmd *cobra.Command, args []string) { notImplemented(cmd) } |
| func (i *imgTestEnv) runImgTestPassFailCmd(cmd *cobra.Command, args []string) { notImplemented(cmd) } |
| |
| // runImgTestCommand processes and uploads test results to Gold. |
| func (i *imgTestEnv) runImgTestAddCmd(cmd *cobra.Command, args []string) { |
| keyMap, err := readKeysFile(i.flagKeysFile) |
| ifErrLogExit(cmd, err) |
| |
| validation := shared.Validation{} |
| issueID := validation.Int64Value("issue", i.flagIssueID, 0) |
| patchsetID := validation.Int64Value("patchset", i.flagPatchsetID, 0) |
| jobID := validation.Int64Value("jobid", i.flagJobID, 0) |
| ifErrLogExit(cmd, validation.Errors()) |
| |
| // Define the meta data of the result that is shared by all tests. |
| gr := &jsonio.GoldResults{ |
| GitHash: i.flagCommit, |
| Key: keyMap, |
| Issue: issueID, |
| Patchset: patchsetID, |
| BuildBucketID: jobID, |
| } |
| |
| config := &goldclient.GoldClientConfig{ |
| InstanceID: i.flagInstanceID, |
| WorkDir: i.flagWorkDir, |
| PassFailStep: i.flagPassFailStep, |
| OverrideGoldURL: i.flagURL, |
| } |
| goldClient, err := goldclient.NewCloudClient(config, gr) |
| ifErrLogExit(cmd, err) |
| |
| pass, err := goldClient.Test(i.flagTestName, i.flagPNGFile) |
| ifErrLogExit(cmd, err) |
| |
| if !pass { |
| logErrf(cmd, "Test: %s FAIL\n", i.flagTestName) |
| exitProcess(cmd, 1) |
| } |
| logInfof(cmd, "Test: %s PASS\n", i.flagTestName) |
| exitProcess(cmd, 0) |
| } |
| |
| // readKeysFile is a helper function to read a JSON file with key/value pairs. |
| func readKeysFile(keysFile string) (map[string]string, error) { |
| reader, err := os.Open(keysFile) |
| if err != nil { |
| return nil, err |
| } |
| |
| ret := map[string]string{} |
| err = json.NewDecoder(reader).Decode(&ret) |
| return ret, err |
| } |