blob: e3735f45acc5a9b6233471f66796859b5ee3cba9 [file] [log] [blame]
/*
* Copyright 2023 Google LLC
*
* Use of this source code is governed by a BSD-style license that can be
* found in the LICENSE file.
*/
#include "bench/Benchmark.h"
#include "include/core/SkBitmap.h"
#include "include/core/SkCanvas.h"
#include "include/core/SkColorType.h"
#include "include/core/SkGraphics.h"
#include "include/core/SkStream.h"
#include "include/core/SkString.h"
#include "include/core/SkSurface.h"
#include "include/encode/SkPngEncoder.h"
#include "include/private/base/SkAssert.h"
#include "include/private/base/SkTArray.h"
#include "src/base/SkTime.h"
#include "src/utils/SkJSONWriter.h"
#include "src/utils/SkOSPath.h"
#include "tools/AutoreleasePool.h"
#include "tools/ProcStats.h"
#include "tools/Stats.h"
#include "tools/flags/CommandLineFlags.h"
#include "tools/testrunners/benchmark/target/BenchmarkTarget.h"
#include "tools/testrunners/common/TestRunner.h"
#include "tools/testrunners/common/compilation_mode_keys/CompilationModeKeys.h"
#include "tools/testrunners/common/surface_manager/SurfaceManager.h"
#include "tools/timer/Timer.h"
#include <cinttypes>
#include <sstream>
static DEFINE_string(skip, "", "Space-separated list of test cases (regexps) to skip.");
static DEFINE_string(
match,
"",
"Space-separated list of test cases (regexps) to run. Will run all tests if omitted.");
// TODO(lovisolo): Should we check that this is a valid Git hash?
static DEFINE_string(
gitHash,
"",
"Git hash to include in the results.json output file, which can be ingested by Perf.");
static DEFINE_string(issue,
"",
"Changelist ID (e.g. a Gerrit changelist number) to include in the "
"results.json output file, which can be ingested by Perf.");
static DEFINE_string(patchset,
"",
"Patchset ID (e.g. a Gerrit patchset number) to include in the results.json "
"output file, which can be ingested by Perf.");
static DEFINE_string(key,
"",
"Space-separated key/value pairs common to all benchmarks. These will be "
"included in the results.json output file, which can be ingested by Perf.");
static DEFINE_string(
links,
"",
"Space-separated name/URL pairs with additional information about the benchmark execution, "
"for example links to the Swarming bot and task pages, named \"swarming_bot\" and "
"\"swarming_task\", respectively. These links are included in the "
"results.json output file, which can be ingested by Perf.");
// When running under Bazel and overriding the output directory, you might encounter errors
// such as "No such file or directory" and "Read-only file system". The former can happen
// when running on RBE because the passed in output dir might not exist on the remote
// worker, whereas the latter can happen when running locally in sandboxed mode, which is
// the default strategy when running outside of RBE. One possible workaround is to run the
// test as a local subprocess, which can be done by passing flag --strategy=TestRunner=local
// to Bazel.
//
// Reference: https://bazel.build/docs/user-manual#execution-strategy.
static DEFINE_string(outputDir,
"",
"Directory where to write any output JSON and PNG files. "
"Optional when running under Bazel "
"(e.g. \"bazel test //path/to:test\") as it defaults to "
"$TEST_UNDECLARED_OUTPUTS_DIR.");
static DEFINE_string(surfaceConfig,
"",
"Name of the Surface configuration to use (e.g. \"8888\"). This determines "
"how we construct the SkSurface from which we get the SkCanvas that "
"benchmarks will draw on. See file "
"//tools/testrunners/common/surface_manager/SurfaceManager.h for details.");
static DEFINE_string(
cpuName,
"",
"Contents of the \"cpu_or_gpu_value\" dimension for CPU-bound traces (e.g. \"AVX512\").");
static DEFINE_string(
gpuName,
"",
"Contents of the \"cpu_or_gpu_value\" dimension for GPU-bound traces (e.g. \"RTX3060\").");
static DEFINE_bool(
writePNGs,
false,
"Whether or not to write to the output directory any bitmaps produced by benchmarks.");
// Mutually exclusive with --autoTuneLoops.
static DEFINE_int(loops, 0, "The number of benchmark runs that constitutes a single sample.");
// Mutually exclusive with --loops.
static DEFINE_bool(autoTuneLoops,
false,
"Auto-tune (automatically determine) the number of benchmark runs that "
"constitutes a single sample. Timings are only reported when auto-tuning.");
static DEFINE_int(
autoTuneLoopsMax,
1000000,
"Maximum number of benchmark runs per single sample when auto-tuning. Ignored unless flag "
"--autoTuneLoops is set.");
// Mutually exclusive with --ms.
static DEFINE_int(samples, 10, "Number of samples to collect for each benchmark.");
// Mutually exclusive with --samples.
static DEFINE_int(ms, 0, "For each benchmark, collect samples for this many milliseconds.");
static DEFINE_int(flushEvery,
10,
"Flush the results.json output file every n-th run. This file "
"can be ingested by Perf.");
static DEFINE_bool(csv, false, "Print status in CSV format.");
static DEFINE_bool2(quiet, q, false, "if true, do not print status updates.");
static DEFINE_bool2(verbose, v, false, "Enable verbose output from the test runner.");
// Set in //bazel/devicesrc but only consumed by adb_test_runner.go. We cannot use the
// DEFINE_string macro because the flag name includes dashes.
[[maybe_unused]] static bool unused =
SkFlagInfo::CreateStringFlag("device-specific-bazel-config",
nullptr,
new CommandLineFlags::StringArray(),
nullptr,
"Ignored by this test runner.",
nullptr);
static void validate_flags(bool isBazelTest) {
TestRunner::FlagValidators::AllOrNone(
{{"--issue", FLAGS_issue.size() > 0}, {"--patchset", FLAGS_patchset.size() > 0}});
TestRunner::FlagValidators::StringAtMostOne("--issue", FLAGS_issue);
TestRunner::FlagValidators::StringAtMostOne("--patchset", FLAGS_patchset);
TestRunner::FlagValidators::StringEven("--key", FLAGS_key);
TestRunner::FlagValidators::StringEven("--links", FLAGS_links);
if (!isBazelTest) {
TestRunner::FlagValidators::StringNonEmpty("--outputDir", FLAGS_outputDir);
}
TestRunner::FlagValidators::StringAtMostOne("--outputDir", FLAGS_outputDir);
TestRunner::FlagValidators::StringNonEmpty("--surfaceConfig", FLAGS_surfaceConfig);
TestRunner::FlagValidators::StringAtMostOne("--surfaceConfig", FLAGS_surfaceConfig);
TestRunner::FlagValidators::StringAtMostOne("--cpuName", FLAGS_cpuName);
TestRunner::FlagValidators::StringAtMostOne("--gpuName", FLAGS_gpuName);
TestRunner::FlagValidators::ExactlyOne(
{{"--loops", FLAGS_loops != 0}, {"--autoTuneLoops", FLAGS_autoTuneLoops}});
if (!FLAGS_autoTuneLoops) {
TestRunner::FlagValidators::IntGreaterOrEqual("--loops", FLAGS_loops, 1);
}
TestRunner::FlagValidators::IntGreaterOrEqual("--autoTuneLoopsMax", FLAGS_autoTuneLoopsMax, 1);
TestRunner::FlagValidators::ExactlyOne(
{{"--samples", FLAGS_samples != 0}, {"--ms", FLAGS_ms != 0}});
if (FLAGS_ms == 0) {
TestRunner::FlagValidators::IntGreaterOrEqual("--samples", FLAGS_samples, 1);
}
if (FLAGS_samples == 0) {
TestRunner::FlagValidators::IntGreaterOrEqual("--ms", FLAGS_ms, 1);
}
}
// Helper class to produce JSON files in Perf ingestion format. The format is determined by Perf's
// format.Format Go struct:
//
// https://skia.googlesource.com/buildbot/+/e12f70e0a3249af6dd7754d55958ee64a22e0957/perf/go/ingest/format/format.go#168
//
// Note that the JSON format produced by this class differs from Nanobench. The latter follows
// Perf's legacy format, which is determined by the format.BenchData Go struct:
//
// https://skia.googlesource.com/buildbot/+/e12f70e0a3249af6dd7754d55958ee64a22e0957/perf/go/ingest/format/leagacyformat.go#26
class ResultsJSONWriter {
public:
// This struct mirrors Perf's format.SingleMeasurement Go struct:
// https://skia.googlesource.com/buildbot/+/e12f70e0a3249af6dd7754d55958ee64a22e0957/perf/go/ingest/format/format.go#31.
struct SingleMeasurement {
std::string value;
double measurement;
};
// This struct mirrors Perf's format.Result Go struct:
// https://skia.googlesource.com/buildbot/+/e12f70e0a3249af6dd7754d55958ee64a22e0957/perf/go/ingest/format/format.go#69.
//
// Note that the format.Result Go struct supports either one single measurement, or multiple
// measurements represented as a dictionary from arbitrary string keys to an array of
// format.SingleMeasurement Go structs. This class focuses on the latter variant.
struct Result {
std::map<std::string, std::string> key;
std::map<std::string, std::vector<SingleMeasurement>> measurements;
};
ResultsJSONWriter(const char* path)
: fFileWStream(path)
, fJson(&fFileWStream, SkJSONWriter::Mode::kPretty)
, fAddingResults(false) {
fJson.beginObject(); // Root object.
fJson.appendS32("version", 1);
}
void addGitHash(std::string gitHash) {
assertNotAddingResults();
fJson.appendCString("git_hash", gitHash.c_str());
}
void addChangelistInfo(std::string issue, std::string patchset) {
assertNotAddingResults();
fJson.appendCString("issue", issue.c_str());
fJson.appendCString("patchset", patchset.c_str());
}
void addKey(std::map<std::string, std::string> key) {
assertNotAddingResults();
fJson.beginObject("key");
for (auto const& [name, value] : key) {
fJson.appendCString(name.c_str(), value.c_str());
}
fJson.endObject();
}
void addLinks(std::map<std::string, std::string> links) {
assertNotAddingResults();
fJson.beginObject("links");
for (auto const& [key, value] : links) {
fJson.appendCString(key.c_str(), value.c_str());
}
fJson.endObject();
}
void addResult(Result result) {
if (!fAddingResults) {
fAddingResults = true;
fJson.beginArray("results"); // "results" array.
}
fJson.beginObject(); // Result object.
// Key.
fJson.beginObject("key"); // "key" dictionary.
for (auto const& [name, value] : result.key) {
fJson.appendCString(name.c_str(), value.c_str());
}
fJson.endObject(); // "key" dictionary.
// Measurements.
fJson.beginObject("measurements"); // "measurements" dictionary.
for (auto const& [name, singleMeasurements] : result.measurements) {
fJson.beginArray(name.c_str()); // <name> array.
for (const SingleMeasurement& singleMeasurement : singleMeasurements) {
// Based on
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/ResultsWriter.h#51.
//
// Don't record if NaN or Inf.
if (std::isfinite(singleMeasurement.measurement)) {
fJson.beginObject();
fJson.appendCString("value", singleMeasurement.value.c_str());
fJson.appendDoubleDigits("measurement", singleMeasurement.measurement, 16);
fJson.endObject();
}
}
fJson.endArray(); // <name> array.
}
fJson.endObject(); // "measurements" dictionary.
fJson.endObject(); // Result object.
}
void flush() { fJson.flush(); }
~ResultsJSONWriter() {
if (fAddingResults) {
fJson.endArray(); // "results" array;
}
fJson.endObject(); // Root object.
}
private:
void assertNotAddingResults() {
if (fAddingResults) {
SK_ABORT("Cannot perform this operation after addResults() is called.");
}
}
SkFILEWStream fFileWStream;
SkJSONWriter fJson;
bool fAddingResults;
};
// Manages an autorelease pool for Metal. On other platforms, pool.drain() is a no-op.
AutoreleasePool pool;
static double now_ms() { return SkTime::GetNSecs() * 1e-6; }
// Based on
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1503.
static void warm_up_test_runner_once(BenchmarkTarget* target, int loops) {
static bool warm = false;
if (warm) {
return;
}
if (FLAGS_ms < 1000) {
// Run the first bench for 1000ms to warm up the test runner if FLAGS_ms < 1000.
// Otherwise, the first few benches' measurements will be inaccurate.
auto stop = now_ms() + 1000;
do {
target->time(loops);
pool.drain();
} while (now_ms() < stop);
}
warm = true;
}
// Collects samples for the given benchmark. Returns a boolean indicating success or failure, the
// number of benchmark runs used for each sample, the samples and any statistics produced by the
// benchmark and/or target.
//
// Based on
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1489.
static int sample_benchmark(BenchmarkTarget* target,
int* loops,
skia_private::TArray<double>* samples,
skia_private::TArray<SkString>* statKeys,
skia_private::TArray<double>* statValues) {
target->setup();
if (FLAGS_autoTuneLoops) {
auto [autoTunedLoops, ok] = target->autoTuneLoops();
if (!ok) {
// Can't be timed. A warning has already been printed.
target->tearDown();
return false;
}
*loops = autoTunedLoops;
if (*loops > FLAGS_autoTuneLoopsMax) {
TestRunner::Log(
"Warning: Clamping loops from %d to %d (per the --autoTuneLoopsMax flag) for "
"benchmark \"%s\".",
*loops,
FLAGS_autoTuneLoopsMax,
target->getBenchmark()->getUniqueName());
*loops = FLAGS_autoTuneLoopsMax;
}
} else {
*loops = FLAGS_loops;
}
// Warm up the test runner to increase the chances of getting consistent measurements. Only
// done once for the entire lifecycle of the test runner.
warm_up_test_runner_once(target, *loops);
// Each individual BenchmarkTarget must also be warmed up.
target->warmUp(*loops);
if (FLAGS_ms) {
// Collect as many samples as possible for a specific duration of time.
auto stop = now_ms() + FLAGS_ms;
do {
samples->push_back(target->time(*loops) / *loops);
pool.drain();
} while (now_ms() < stop);
} else {
// Collect an exact number of samples.
samples->reset(FLAGS_samples);
for (int s = 0; s < FLAGS_samples; s++) {
(*samples)[s] = target->time(*loops) / *loops;
pool.drain();
}
}
// Scale each sample to the benchmark's own units, time/unit.
for (double& sample : *samples) {
sample *= (1.0 / target->getBenchmark()->getUnits());
}
target->dumpStats(statKeys, statValues);
target->tearDown();
return true;
}
// Based on
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#432.
static void maybe_write_png(BenchmarkTarget* target, std::string outputDir) {
if (target->getBackend() == Benchmark::Backend::kNonRendering) {
return;
}
SkString filename = SkStringPrintf("%s.png", target->getBenchmark()->getUniqueName());
filename = SkOSPath::Join(outputDir.c_str(), filename.c_str());
if (!target->getCanvas() ||
target->getCanvas()->imageInfo().colorType() == kUnknown_SkColorType) {
return;
}
SkBitmap bmp;
bmp.allocPixels(target->getCanvas()->imageInfo());
if (!target->getCanvas()->readPixels(bmp, 0, 0)) {
TestRunner::Log("Warning: Could not read canvas pixels for benchmark \"%s\".",
target->getBenchmark()->getUniqueName());
return;
}
SkFILEWStream stream(filename.c_str());
if (!stream.isValid()) {
TestRunner::Log("Warning: Could not write file \"%s\".", filename.c_str());
return;
}
if (!SkPngEncoder::Encode(&stream, bmp.pixmap(), {})) {
TestRunner::Log("Warning: Could not encode pixels from benchmark \"%s\" as PNG.",
target->getBenchmark()->getUniqueName());
return;
}
if (FLAGS_verbose) {
TestRunner::Log("PNG for benchmark \"%s\" written to: %s",
target->getBenchmark()->getUniqueName(),
filename.c_str());
}
}
// Non-static because it is used from RasterBenchmarkTarget.cpp.
SkString humanize(double ms) {
if (FLAGS_verbose) return SkStringPrintf("%" PRIu64, (uint64_t)(ms * 1e6));
return HumanizeMs(ms);
}
#define HUMANIZE(ms) humanize(ms).c_str()
static SkString to_string(int n) {
SkString str;
str.appendS32(n);
return str;
}
// Based on
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1593.
static void print_benchmark_stats(Stats* stats,
skia_private::TArray<double>* samples,
BenchmarkTarget* target,
std::string surfaceConfig,
int loops) {
if (!FLAGS_autoTuneLoops) {
TestRunner::Log("%4d/%-4dMB\t%s\t%s",
sk_tools::getCurrResidentSetSizeMB(),
sk_tools::getMaxResidentSetSizeMB(),
target->getBenchmark()->getUniqueName(),
surfaceConfig.c_str());
} else if (FLAGS_quiet) {
const char* mark = " ";
const double stddev_percent = sk_ieee_double_divide(100 * sqrt(stats->var), stats->mean);
if (stddev_percent > 5) mark = "?";
if (stddev_percent > 10) mark = "!";
TestRunner::Log("%10.2f %s\t%s\t%s",
stats->median * 1e3,
mark,
target->getBenchmark()->getUniqueName(),
surfaceConfig.c_str());
} else if (FLAGS_csv) {
const double stddev_percent = sk_ieee_double_divide(100 * sqrt(stats->var), stats->mean);
SkDebugf("%g,%g,%g,%g,%g,%s,%s\n",
stats->min,
stats->median,
stats->mean,
stats->max,
stddev_percent,
surfaceConfig.c_str(),
target->getBenchmark()->getUniqueName());
} else {
const double stddev_percent = sk_ieee_double_divide(100 * sqrt(stats->var), stats->mean);
TestRunner::Log("%4d/%-4dMB\t%d\t%s\t%s\t%s\t%s\t%.0f%%\t%s\t%s\t%s",
sk_tools::getCurrResidentSetSizeMB(),
sk_tools::getMaxResidentSetSizeMB(),
loops,
HUMANIZE(stats->min),
HUMANIZE(stats->median),
HUMANIZE(stats->mean),
HUMANIZE(stats->max),
stddev_percent,
FLAGS_ms ? to_string(samples->size()).c_str() : stats->plot.c_str(),
surfaceConfig.c_str(),
target->getBenchmark()->getUniqueName());
}
target->printStats();
if (FLAGS_verbose) {
std::ostringstream oss;
oss << "Samples: ";
for (int j = 0; j < samples->size(); j++) {
oss << HUMANIZE((*samples)[j]) << " ";
}
oss << target->getBenchmark()->getUniqueName();
TestRunner::Log("%s", oss.str().c_str());
}
}
// Based on
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1337.
int main(int argc, char** argv) {
TestRunner::InitAndLogCmdlineArgs(argc, argv);
// When running under Bazel (e.g. "bazel test //path/to:test"), we'll store output files in
// $TEST_UNDECLARED_OUTPUTS_DIR unless overridden via the --outputDir flag.
//
// See https://bazel.build/reference/test-encyclopedia#initial-conditions.
std::string testUndeclaredOutputsDir;
if (char* envVar = std::getenv("TEST_UNDECLARED_OUTPUTS_DIR")) {
testUndeclaredOutputsDir = envVar;
}
bool isBazelTest = !testUndeclaredOutputsDir.empty();
// Parse and validate flags.
CommandLineFlags::Parse(argc, argv);
validate_flags(isBazelTest);
// TODO(lovisolo): Define an enum for surface configs and turn this flag into an enum value.
std::string surfaceConfig = FLAGS_surfaceConfig[0];
// Directory where we will write the output JSON file and any PNGs produced by the benchmarks.
std::string outputDir =
FLAGS_outputDir.isEmpty() ? testUndeclaredOutputsDir : FLAGS_outputDir[0];
std::string cpuName = FLAGS_cpuName.isEmpty() ? "" : FLAGS_cpuName[0];
std::string gpuName = FLAGS_gpuName.isEmpty() ? "" : FLAGS_gpuName[0];
// Output JSON file.
//
// TODO(lovisolo): Define a constant with the file name, use it here and in flag descriptions.
SkString jsonPath = SkOSPath::Join(outputDir.c_str(), "results.json");
ResultsJSONWriter jsonWriter(jsonPath.c_str());
if (FLAGS_gitHash.size() == 1) {
jsonWriter.addGitHash(FLAGS_gitHash[0]);
} else {
TestRunner::Log(
"Warning: No --gitHash flag was specified. Perf ingestion ignores JSON files that "
"do not specify a Git hash. This is fine for local debugging, but CI tasks should "
"always set the --gitHash flag.");
}
if (FLAGS_issue.size() == 1 && FLAGS_patchset.size() == 1) {
jsonWriter.addChangelistInfo(FLAGS_issue[0], FLAGS_patchset[0]);
}
// Key.
std::map<std::string, std::string> keyValuePairs = {
// Add a key/value pair that nanobench will never use in order to avoid accidentally
// polluting an existing trace.
{"build_system", "bazel"},
};
for (int i = 1; i < FLAGS_key.size(); i += 2) {
keyValuePairs[FLAGS_key[i - 1]] = FLAGS_key[i];
}
keyValuePairs.merge(GetCompilationModeGoldAndPerfKeyValuePairs());
jsonWriter.addKey(keyValuePairs);
// Links.
if (FLAGS_links.size()) {
std::map<std::string, std::string> links;
for (int i = 1; i < FLAGS_links.size(); i += 2) {
links[FLAGS_links[i - 1]] = FLAGS_links[i];
}
jsonWriter.addLinks(links);
}
int runs = 0;
bool missingCpuOrGpuWarningLogged = false;
for (auto benchmarkFactory : BenchRegistry::Range()) {
std::unique_ptr<Benchmark> benchmark(benchmarkFactory(nullptr));
if (!TestRunner::ShouldRunTestCase(benchmark->getUniqueName(), FLAGS_match, FLAGS_skip)) {
TestRunner::Log("Skipping %s", benchmark->getName());
continue;
}
benchmark->delayedSetup();
std::unique_ptr<BenchmarkTarget> target =
BenchmarkTarget::FromConfig(surfaceConfig, benchmark.get());
// Print warning about missing cpu_or_gpu key if necessary.
if (target->isCpuOrGpuBound() == SurfaceManager::CpuOrGpu::kCPU && cpuName == "" &&
!missingCpuOrGpuWarningLogged) {
TestRunner::Log(
"Warning: The surface is CPU-bound, but flag --cpuName was not provided. "
"Perf traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\".");
missingCpuOrGpuWarningLogged = true;
}
if (target->isCpuOrGpuBound() == SurfaceManager::CpuOrGpu::kGPU && gpuName == "" &&
!missingCpuOrGpuWarningLogged) {
TestRunner::Log(
"Warning: The surface is GPU-bound, but flag --gpuName was not provided. "
"Perf traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\".");
missingCpuOrGpuWarningLogged = true;
}
if (benchmark->isSuitableFor(target->getBackend())) {
// Run benchmark and collect samples.
int loops;
skia_private::TArray<double> samples;
skia_private::TArray<SkString> statKeys;
skia_private::TArray<double> statValues;
if (!sample_benchmark(target.get(), &loops, &samples, &statKeys, &statValues)) {
// Sampling failed. A warning has alredy been printed.
pool.drain();
continue;
}
if (FLAGS_writePNGs) {
// Save the bitmap produced by the benchmark to disk, if applicable. Not all
// benchmarks produce bitmaps, e.g. those that use the "nonrendering" config.
maybe_write_png(target.get(), outputDir);
}
// Building stats.plot often shows up in profiles, so skip building it when we're
// not going to print it anyway.
const bool want_plot = !FLAGS_quiet && !FLAGS_ms;
Stats stats(samples, want_plot);
print_benchmark_stats(&stats, &samples, target.get(), surfaceConfig, loops);
ResultsJSONWriter::Result result;
result.key = {
// Based on
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1566.
{"name", std::string(benchmark->getName())},
// Replaces the "config" and "extra_config" keys set by nanobench.
{"surface_config", surfaceConfig},
// Based on
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1578.
//
// TODO(lovisolo): Determine these dynamically when we add support for GMBench,
// SKPBench, etc.
{"source_type", "bench"},
{"bench_type", "micro"},
// Nanobench adds a "test" key consisting of "<unique name>_<width>_<height>",
// presumably with the goal of making the trace ID unique, see
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1456.
//
// However, we can accomplish unique trace IDs by expressing "<width>" and
// "<height>" as their own keys.
//
// Regarding the "<unique name>" part of the "test" key:
//
// - Nanobench sets "<unique name>" to the result of
// Benchmark::getUniqueName():
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/Benchmark.h#41.
//
// - Benchmark::getUniqueName() returns Benchmark::getName() except for the
// following two cases.
//
// - SKPBench::getUniqueName() returns "<name>_<scale>":
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/SKPBench.cpp#33.
//
// - SKPAnimationBench returns "<name>_<tag>":
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/SKPAnimationBench.cpp#18.
//
// Therefore it is important that we add "<scale>" and "<tag>" as their own
// keys when we eventually add support for these kinds of benchmarks.
//
// Based on
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1456.
{"width", SkStringPrintf("%d", benchmark->getSize().width()).c_str()},
{"height", SkStringPrintf("%d", benchmark->getSize().height()).c_str()},
};
result.key.merge(target->getKeyValuePairs(cpuName, gpuName));
result.measurements["ms"] = {
// Based on
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1571.
{.value = "min", .measurement = stats.min},
{.value = "ratio",
.measurement = sk_ieee_double_divide(stats.median, stats.min)},
};
if (!statKeys.empty()) {
// Based on
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1580.
//
// Only SKPBench currently returns valid key/value pairs.
SkASSERT(statKeys.size() == statValues.size());
result.measurements["stats"] = {};
for (int i = 0; i < statKeys.size(); i++) {
result.measurements["stats"].push_back(
{.value = statKeys[i].c_str(), .measurement = statValues[i]});
}
}
jsonWriter.addResult(result);
runs++;
if (runs % FLAGS_flushEvery == 0) {
jsonWriter.flush();
}
pool.drain();
}
}
BenchmarkTarget::printGlobalStats();
SkGraphics::PurgeAllCaches();
// Based on
// https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1668.
jsonWriter.addResult({
.key =
{
{"name", "memory_usage"},
},
.measurements =
{
{"resident_set_size_mb",
{{.value = "max",
.measurement = double(sk_tools::getMaxResidentSetSizeMB())}}},
},
});
TestRunner::Log("JSON file written to: %s", jsonPath.c_str());
TestRunner::Log("PNGs (if any) written to: %s", outputDir.c_str());
return 0;
}