infra/bots/task_drivers/perf_puppeteer_render_skps/perf_puppeteer_render_skps_test.go

// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"context"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/testutils"
	"go.skia.org/infra/task_driver/go/td"
)

func TestSetup_NPMInitializedChromeStoppedBenchmarkOutCreated(t *testing.T) {
	benchmarkPath, err := ioutil.TempDir("", "benchmark")
	require.NoError(t, err)
	defer testutils.RemoveAll(t, benchmarkPath)

	const fakeNodeBinPath = "/fake/path/to/node/bin"

	res := td.RunTestSteps(t, false, func(ctx context.Context) error {
		mock := exec.CommandCollector{}
		ctx = td.WithExecRunFn(ctx, mock.Run)
		err := setup(ctx, benchmarkPath, fakeNodeBinPath)
		if err != nil {
			assert.NoError(t, err)
			return err
		}
		cmds := mock.Commands()
		require.Len(t, cmds, 2)
		cmd := cmds[0]
		assert.Equal(t, "/fake/path/to/node/bin/npm", cmd.Name)
		assert.Equal(t, []string{"ci"}, cmd.Args)

		cmd = cmds[1]
		assert.Equal(t, "killall", cmd.Name)
		assert.Equal(t, []string{"chrome"}, cmd.Args)
		return nil
	})
	require.Empty(t, res.Errors)
	require.Empty(t, res.Exceptions)

	fi, err := os.Stat(filepath.Join(benchmarkPath, "out"))
	require.NoError(t, err)
	assert.True(t, fi.IsDir())
}

func TestBenchSKPs_CPUHasNoUseGPUFlag(t *testing.T) {
	skps, err := ioutil.TempDir("", "skps")
	require.NoError(t, err)
	defer testutils.RemoveAll(t, skps)

	require.NoError(t, ioutil.WriteFile(filepath.Join(skps, "first_skp"), []byte("doesnt matter"), 0777))

	const fakeNodeBinPath = "/fake/path/to/node/bin"
	const fakeCanvasKitPath = "/fake/path/to/canvaskit"
	const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer"

	perfObj := perfJSONFormat{
		Key: map[string]string{
			perfKeyCpuOrGPU: "CPU",
		},
	}

	res := td.RunTestSteps(t, false, func(ctx context.Context) error {
		mock := exec.CommandCollector{}
		ctx = td.WithExecRunFn(ctx, mock.Run)
		err := benchSKPs(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, skps, fakeNodeBinPath)
		if err != nil {
			assert.NoError(t, err)
			return err
		}
		require.Len(t, mock.Commands(), 1)
		cmd := mock.Commands()[0]
		assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name)
		assert.Equal(t, []string{"perf-canvaskit-with-puppeteer",
			"--bench_html", "render-skp.html",
			"--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js",
			"--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm",
			"--timeout", "90",
			"--input_skp", filepath.Join(skps, "first_skp"),
			"--output", "/fake/path/to/perf-puppeteer/out/first_skp.json"}, cmd.Args)
		return nil
	})
	require.Empty(t, res.Errors)
	require.Empty(t, res.Exceptions)
}

func TestBenchSKPs_SkiplistIsUsed(t *testing.T) {
	skps, err := ioutil.TempDir("", "skps")
	require.NoError(t, err)
	defer testutils.RemoveAll(t, skps)

	require.NoError(t, ioutil.WriteFile(filepath.Join(skps, "desk_carsvg.skp"), []byte("doesnt matter"), 0777))

	const fakeNodeBinPath = "/fake/path/to/node/bin"
	const fakeCanvasKitPath = "/fake/path/to/canvaskit"
	const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer"

	perfObj := perfJSONFormat{
		Key: map[string]string{
			perfKeyCpuOrGPU: "CPU",
		},
	}

	res := td.RunTestSteps(t, false, func(ctx context.Context) error {
		mock := exec.CommandCollector{}
		ctx = td.WithExecRunFn(ctx, mock.Run)
		err := benchSKPs(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, skps, fakeNodeBinPath)
		if err != nil {
			assert.NoError(t, err)
			return err
		}
		// Should be skipped
		require.Len(t, mock.Commands(), 0)
		return nil
	})
	require.Empty(t, res.Errors)
	require.Empty(t, res.Exceptions)
}

func TestBenchSKPs_GPUHasFlag(t *testing.T) {
	skps, err := ioutil.TempDir("", "skps")
	require.NoError(t, err)
	defer testutils.RemoveAll(t, skps)

	require.NoError(t, ioutil.WriteFile(filepath.Join(skps, "first_skp"), []byte("doesnt matter"), 0777))

	const fakeNodeBinPath = "/fake/path/to/node/bin"
	const fakeCanvasKitPath = "/fake/path/to/canvaskit"
	const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer"

	perfObj := perfJSONFormat{
		Key: map[string]string{
			perfKeyCpuOrGPU: "GPU",
		},
	}

	res := td.RunTestSteps(t, false, func(ctx context.Context) error {
		mock := exec.CommandCollector{}
		ctx = td.WithExecRunFn(ctx, mock.Run)
		err := benchSKPs(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, skps, fakeNodeBinPath)
		if err != nil {
			assert.NoError(t, err)
			return err
		}
		require.Len(t, mock.Commands(), 1)
		cmd := mock.Commands()[0]
		assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name)
		assert.Equal(t, []string{"perf-canvaskit-with-puppeteer",
			"--bench_html", "render-skp.html",
			"--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js",
			"--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm",
			"--timeout", "90",
			"--input_skp", filepath.Join(skps, "first_skp"),
			"--output", "/fake/path/to/perf-puppeteer/out/first_skp.json",
			"--use_gpu"}, cmd.Args)
		return nil
	})
	require.Empty(t, res.Errors)
	require.Empty(t, res.Exceptions)
}

func TestBenchSKPs_WebGL1(t *testing.T) {
	skps, err := ioutil.TempDir("", "skps")
	require.NoError(t, err)
	defer testutils.RemoveAll(t, skps)

	require.NoError(t, ioutil.WriteFile(filepath.Join(skps, "first_skp"), []byte("doesnt matter"), 0777))

	const fakeNodeBinPath = "/fake/path/to/node/bin"
	const fakeCanvasKitPath = "/fake/path/to/canvaskit"
	const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer"

	perfObj := perfJSONFormat{
		Key: map[string]string{
			perfKeyCpuOrGPU:     "GPU",
			perfKeyWebGLVersion: "1",
		},
	}

	res := td.RunTestSteps(t, false, func(ctx context.Context) error {
		mock := exec.CommandCollector{}
		ctx = td.WithExecRunFn(ctx, mock.Run)
		err := benchSKPs(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, skps, fakeNodeBinPath)
		if err != nil {
			assert.NoError(t, err)
			return err
		}
		require.Len(t, mock.Commands(), 1)
		cmd := mock.Commands()[0]
		assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name)
		assert.Equal(t, []string{"perf-canvaskit-with-puppeteer",
			"--bench_html", "render-skp.html",
			"--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js",
			"--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm",
			"--timeout", "90",
			"--input_skp", filepath.Join(skps, "first_skp"),
			"--output", "/fake/path/to/perf-puppeteer/out/first_skp.json",
			"--use_gpu",
			"--query_params webgl1"}, cmd.Args)
		return nil
	})
	require.Empty(t, res.Errors)
	require.Empty(t, res.Exceptions)
}

func TestProcessSkottieFramesData_GPUTwoInputsGetSummarizedAndCombined(t *testing.T) {
	input, err := ioutil.TempDir("", "inputs")
	require.NoError(t, err)
	defer testutils.RemoveAll(t, input)
	err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{
		"first_skp.json":  firstSKP,
		"second_skp.json": secondSKP,
	})
	require.NoError(t, err)
	output, err := ioutil.TempDir("", "perf")
	require.NoError(t, err)
	defer testutils.RemoveAll(t, output)

	// These are based off of realistic values.
	keys := map[string]string{
		"os":               "Ubuntu18",
		"model":            "Golo",
		perfKeyCpuOrGPU:    "GPU",
		"cpu_or_gpu_value": "QuadroP400",
	}

	perfObj, err := makePerfObj(someGitHash, someTaskID, someMachineID, keys)
	require.NoError(t, err)

	outputFile := filepath.Join(output, "perf-taskid1352.json")
	res := td.RunTestSteps(t, false, func(ctx context.Context) error {
		return processSKPData(ctx, perfObj, input, outputFile)
	})
	require.Empty(t, res.Errors)
	require.Empty(t, res.Exceptions)

	b, err := ioutil.ReadFile(outputFile)
	require.NoError(t, err)

	assert.Equal(t, `{
  "gitHash": "032631e490db494128e0610a19adce4cab9706d1",
  "swarming_task_id": "4bdd43ed7c906c11",
  "swarming_machine_id": "skia-e-gce-203",
  "key": {
    "arch": "wasm",
    "binary": "CanvasKit",
    "browser": "Chromium",
    "configuration": "Release",
    "cpu_or_gpu": "GPU",
    "cpu_or_gpu_value": "QuadroP400",
    "extra_config": "RenderSKP",
    "model": "Golo",
    "os": "Ubuntu18"
  },
  "results": {
    "first_skp": {
      "webgl2": {
        "avg_render_frame_ms": 150.065,
        "avg_render_with_flush_ms": 133.91167,
        "avg_render_without_flush_ms": 45.398335,
        "median_render_frame_ms": 143.71,
        "median_render_with_flush_ms": 125.185,
        "median_render_without_flush_ms": 37.445,
        "skp_load_ms": 1.715,
        "stddev_render_frame_ms": 15.210527,
        "stddev_render_with_flush_ms": 15.47429,
        "stddev_render_without_flush_ms": 21.69691
      }
    },
    "second_skp": {
      "webgl2": {
        "avg_render_frame_ms": 316.7317,
        "avg_render_with_flush_ms": 233.91167,
        "avg_render_without_flush_ms": 85.39834,
        "median_render_frame_ms": 243.71,
        "median_render_with_flush_ms": 225.185,
        "median_render_without_flush_ms": 67.445,
        "skp_load_ms": 3.715,
        "stddev_render_frame_ms": 109.164635,
        "stddev_render_with_flush_ms": 15.474287,
        "stddev_render_without_flush_ms": 43.27188
      }
    }
  }
}`, string(b))
}

func writeFilesToDisk(path string, fileNamesToContent map[string]string) error {
	if err := os.MkdirAll(path, 0777); err != nil {
		return err
	}
	for name, content := range fileNamesToContent {
		if err := ioutil.WriteFile(filepath.Join(path, name), []byte(content), 0666); err != nil {
			return err
		}
	}
	return nil
}

const (
	someGitHash   = "032631e490db494128e0610a19adce4cab9706d1"
	someTaskID    = "4bdd43ed7c906c11"
	someMachineID = "skia-e-gce-203"
)

const firstSKP = `{
  "without_flush_ms": [23.71, 37.445, 75.04],
  "with_flush_ms": [125.185, 120.895, 155.655],
  "total_frame_ms": [143.71, 135.445, 171.04],
  "skp_load_ms":1.715
}`

const secondSKP = `{
  "without_flush_ms": [43.71, 67.445, 145.04],
  "with_flush_ms": [225.185, 220.895, 255.655],
  "total_frame_ms": [243.71, 235.445, 471.04],
  "skp_load_ms":3.715
}`