infra/bots/task_drivers/perf_puppeteer_canvas/perf_puppeteer_canvas_test.go

// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"context"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/testutils"
	"go.skia.org/infra/task_driver/go/td"
)

func TestSetup_NPMInitializedBenchmarkOutCreated(t *testing.T) {
	benchmarkPath, err := ioutil.TempDir("", "benchmark")
	require.NoError(t, err)
	defer testutils.RemoveAll(t, benchmarkPath)

	const fakeNodeBinPath = "/fake/path/to/node/bin"

	res := td.RunTestSteps(t, false, func(ctx context.Context) error {
		mock := exec.CommandCollector{}
		ctx = td.WithExecRunFn(ctx, mock.Run)
		err := setup(ctx, benchmarkPath, fakeNodeBinPath)
		if err != nil {
			assert.NoError(t, err)
			return err
		}
		require.Len(t, mock.Commands(), 1)
		cmd := mock.Commands()[0]
		assert.Equal(t, "/fake/path/to/node/bin/npm", cmd.Name)
		assert.Equal(t, []string{"ci"}, cmd.Args)
		return nil
	})
	require.Empty(t, res.Errors)
	require.Empty(t, res.Exceptions)

	fi, err := os.Stat(filepath.Join(benchmarkPath, "out"))
	require.NoError(t, err)
	assert.True(t, fi.IsDir())
}

func TestBenchCanvas_CPUHasNoUseGPUFlag(t *testing.T) {
	const fakeNodeBinPath = "/fake/path/to/node/bin"
	const fakeCanvasKitPath = "/fake/path/to/canvaskit"
	const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer"

	perfObj := perfJSONFormat{
		Key: map[string]string{
			perfKeyCpuOrGPU: "CPU",
		},
	}

	res := td.RunTestSteps(t, false, func(ctx context.Context) error {
		mock := exec.CommandCollector{}
		ctx = td.WithExecRunFn(ctx, mock.Run)
		err := benchCanvas(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, fakeNodeBinPath)
		if err != nil {
			assert.NoError(t, err)
			return err
		}
		require.Len(t, mock.Commands(), 1)
		cmd := mock.Commands()[0]
		assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name)
		assert.Equal(t, []string{"perf-canvaskit-with-puppeteer",
			"--bench_html", "canvas_perf.html",
			"--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js",
			"--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm",
			"--assets", "canvas_perf_assets",
			"--output", "/fake/path/to/perf-puppeteer/out/perf.json",
			"--timeout", "240",
		}, cmd.Args)
		return nil
	})
	require.Empty(t, res.Errors)
	require.Empty(t, res.Exceptions)
}

func TestBenchCanvas_GPUHasFlag(t *testing.T) {
	const fakeNodeBinPath = "/fake/path/to/node/bin"
	const fakeCanvasKitPath = "/fake/path/to/canvaskit"
	const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer"

	perfObj := perfJSONFormat{
		Key: map[string]string{
			perfKeyCpuOrGPU: "GPU",
		},
	}

	res := td.RunTestSteps(t, false, func(ctx context.Context) error {
		mock := exec.CommandCollector{}
		ctx = td.WithExecRunFn(ctx, mock.Run)
		err := benchCanvas(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, fakeNodeBinPath)
		if err != nil {
			assert.NoError(t, err)
			return err
		}
		require.Len(t, mock.Commands(), 1)
		cmd := mock.Commands()[0]
		assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name)
		assert.Equal(t, []string{"perf-canvaskit-with-puppeteer",
			"--bench_html", "canvas_perf.html",
			"--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js",
			"--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm",
			"--assets", "canvas_perf_assets",
			"--output", "/fake/path/to/perf-puppeteer/out/perf.json",
			"--timeout", "240",
			"--use_gpu"}, cmd.Args)
		return nil
	})
	require.Empty(t, res.Errors)
	require.Empty(t, res.Exceptions)
}

// TestProcessFramesData_GPUTwoInputsGetSummarizedAndCombined tests the reading of the
// output JSON file, and reorganizing and computing perf results and checks the result format.
func TestProcessFramesData_GPUTwoInputsGetSummarizedAndCombined(t *testing.T) {
	input, err := ioutil.TempDir("", "inputs")
	require.NoError(t, err)
	defer testutils.RemoveAll(t, input)
	err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{
		"perf.json": sampleData,
	})
	require.NoError(t, err)
	output, err := ioutil.TempDir("", "perf")
	require.NoError(t, err)
	defer testutils.RemoveAll(t, output)

	// These are based off of realistic values.
	keys := map[string]string{
		"os":               "Ubuntu18",
		"model":            "Golo",
		perfKeyCpuOrGPU:    "GPU",
		"cpu_or_gpu_value": "QuadroP400",
	}

	perfObj, err := makePerfObj(someGitHash, someTaskID, someMachineID, keys)
	require.NoError(t, err)

	outputFile := filepath.Join(output, "perf-taskid1352.json")
	res := td.RunTestSteps(t, false, func(ctx context.Context) error {
		return processFramesData(ctx, perfObj, input, outputFile)
	})
	require.Empty(t, res.Errors)

	b, err := ioutil.ReadFile(outputFile)
	require.NoError(t, err)

	assert.Equal(t, `{
  "gitHash": "032631e490db494128e0610a19adce4cab9706d1",
  "swarming_task_id": "4bdd43ed7c906c11",
  "swarming_machine_id": "skia-e-gce-203",
  "key": {
    "arch": "wasm",
    "binary": "CanvasKit",
    "browser": "Chromium",
    "configuration": "Release",
    "cpu_or_gpu": "GPU",
    "cpu_or_gpu_value": "QuadroP400",
    "extra_config": "CanvasPerf",
    "model": "Golo",
    "os": "Ubuntu18"
  },
  "results": {
    "canvas_drawColor": {
      "webgl2": {
        "90th_percentile_frame_ms": 4.455,
        "95th_percentile_frame_ms": 31.555,
        "99th_percentile_frame_ms": 87.795,
        "avg_render_frame_ms": 5.662692,
        "avg_render_with_flush_ms": 1.75,
        "avg_render_without_flush_ms": 1.875,
        "median_render_frame_ms": 0.795,
        "median_render_with_flush_ms": 1.8,
        "median_render_without_flush_ms": 1.88,
        "stddev_render_frame_ms": 17.463467,
        "stddev_render_with_flush_ms": 0.74999994,
        "stddev_render_without_flush_ms": 0.07500001
      }
    },
    "canvas_drawHugeGradient": {
      "webgl2": {
        "90th_percentile_frame_ms": 210.555,
        "95th_percentile_frame_ms": 400.455,
        "99th_percentile_frame_ms": 770.795,
        "avg_render_frame_ms": 55.58577,
        "avg_render_with_flush_ms": 3.75,
        "avg_render_without_flush_ms": 5.125,
        "median_render_frame_ms": 0.8,
        "median_render_with_flush_ms": 3.8,
        "median_render_without_flush_ms": 5.13,
        "stddev_render_frame_ms": 166.36926,
        "stddev_render_with_flush_ms": 0.75,
        "stddev_render_without_flush_ms": 0.074999936
      }
    }
  }
}`, string(b))
}

func writeFilesToDisk(path string, fileNamesToContent map[string]string) error {
	if err := os.MkdirAll(path, 0777); err != nil {
		return err
	}
	for name, content := range fileNamesToContent {
		if err := ioutil.WriteFile(filepath.Join(path, name), []byte(content), 0666); err != nil {
			return err
		}
	}
	return nil
}

const (
	someGitHash   = "032631e490db494128e0610a19adce4cab9706d1"
	someTaskID    = "4bdd43ed7c906c11"
	someMachineID = "skia-e-gce-203"
)

const sampleData = `{
"canvas_drawColor": {
  "total_frame_ms": [
    31.555,
    87.795,
    0.430,
    1.845,
    3.610,
    1.105,
    0.545,
    2.315,
    1.685,
    0.615,
    0.425,
    0.815,
    0.355,
    0.655,
    0.390,
    4.455,
    0.800,
    0.685,
    2.630,
    0.325,
    0.355,
    0.740,
    0.785,
    0.795,
    0.72,
    0.80
  ],
  "without_flush_ms": [
    2.0,
    1.99,
    1.98,
    1.97,
    1.96,
    1.95,
    1.94,
    1.93,
    1.92,
    1.91,
    1.9,
    1.89,
    1.88,
    1.87,
    1.86,
    1.85,
    1.84,
    1.83,
    1.82,
    1.81,
    1.8,
    1.79,
    1.78,
    1.77,
    1.76,
    1.75
  ],
  "with_flush_ms": [
    3.0,
    2.9,
    2.8,
    2.7,
    2.6,
    2.5,
    2.4,
    2.3,
    2.2,
    2.1,
    2.0,
    1.9,
    1.8,
    1.7,
    1.6,
    1.5,
    1.4,
    1.3,
    1.2,
    1.1,
    1.0,
    0.9,
    0.8,
    0.7,
    0.6,
    0.5
  ]
},
"canvas_drawHugeGradient": {
  "total_frame_ms": [
    210.555,
    770.795,
    10.430,
    31.845,
    3.610,
    1.105,
    0.545,
    2.315,
    1.685,
    0.615,
    0.425,
    0.815,
    0.355,
    0.655,
    0.390,
    400.455,
    0.800,
    0.685,
    2.630,
    0.325,
    0.355,
    0.740,
    0.785,
    0.795,
    0.72,
    0.80
  ],
  "without_flush_ms": [
    5.0,
    5.01,
    5.02,
    5.03,
    5.04,
    5.05,
    5.06,
    5.07,
    5.08,
    5.09,
    5.1,
    5.11,
    5.12,
    5.13,
    5.14,
    5.15,
    5.16,
    5.17,
    5.18,
    5.19,
    5.2,
    5.21,
    5.22,
    5.23,
    5.24,
    5.25
  ],
  "with_flush_ms": [
    5.0,
    4.9,
    4.8,
    4.7,
    4.6,
    4.5,
    4.4,
    4.3,
    4.2,
    4.1,
    4.0,
    3.9,
    3.8,
    3.7,
    3.6,
    3.5,
    3.4,
    3.3,
    3.2,
    3.1,
    3.0,
    2.9,
    2.8,
    2.7,
    2.6,
    2.5
  ]
}
}`