blob: d900ecb13c6829a17e39aee13330e2bfc10d52f3 [file] [log] [blame]
/*
* Copyright 2023 Google LLC
*
* Use of this source code is governed by a BSD-style license that can be
* found in the LICENSE file.
*
* This program runs all GMs registered via macros such as DEF_GM, and for each GM, it saves the
* resulting SkBitmap as a .png file to disk, along with a .json file with the hash of the pixels.
*/
#include "gm/gm.h"
#include "include/core/SkBitmap.h"
#include "include/core/SkCanvas.h"
#include "include/core/SkColorSpace.h"
#include "include/core/SkColorType.h"
#include "include/core/SkImageInfo.h"
#include "include/core/SkStream.h"
#include "include/core/SkSurface.h"
#include "include/encode/SkPngEncoder.h"
#include "include/private/base/SkAssert.h"
#include "include/private/base/SkDebug.h"
#include "src/core/SkMD5.h"
#include "src/utils/SkJSONWriter.h"
#include "src/utils/SkOSPath.h"
#include "tools/HashAndEncode.h"
#include "tools/testrunners/common/TestRunner.h"
#include "tools/testrunners/common/compilation_mode_keys/CompilationModeKeys.h"
#include "tools/testrunners/common/surface_manager/SurfaceManager.h"
#include "tools/testrunners/gm/vias/Draw.h"
#include <ctime>
#include <filesystem>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <string>
// TODO(lovisolo): Add flag --omitDigestIfHashInFile (provides the known hashes file).
static DEFINE_string(skip, "", "Space-separated list of test cases (regexps) to skip.");
static DEFINE_string(
match,
"",
"Space-separated list of test cases (regexps) to run. Will run all tests if omitted.");
// When running under Bazel and overriding the output directory, you might encounter errors such
// as "No such file or directory" and "Read-only file system". The former can happen when running
// on RBE because the passed in output dir might not exist on the remote worker, whereas the latter
// can happen when running locally in sandboxed mode, which is the default strategy when running
// outside of RBE. One possible workaround is to run the test as a local subprocess, which can be
// done by passing flag --strategy=TestRunner=local to Bazel.
//
// Reference: https://bazel.build/docs/user-manual#execution-strategy.
static DEFINE_string(outputDir,
"",
"Directory where to write any output .png and .json files. "
"Optional when running under Bazel "
"(e.g. \"bazel test //path/to:test\") as it defaults to "
"$TEST_UNDECLARED_OUTPUTS_DIR.");
static DEFINE_string(key, "", "Space-separated key/value pairs common to all traces.");
// We named this flag --surfaceConfig rather than --config to avoid confusion with the --config
// Bazel flag.
static DEFINE_string(
surfaceConfig,
"",
"Name of the Surface configuration to use (e.g. \"8888\"). This determines "
"how we construct the SkSurface from which we get the SkCanvas that GMs will "
"draw on. See file //tools/testrunners/common/surface_manager/SurfaceManager.h for "
"details.");
static DEFINE_string(
cpuName,
"",
"Contents of the \"cpu_or_gpu_value\" dimension for CPU-bound traces (e.g. \"AVX512\").");
static DEFINE_string(
gpuName,
"",
"Contents of the \"cpu_or_gpu_value\" dimension for GPU-bound traces (e.g. \"RTX3060\").");
static DEFINE_string(via,
"direct", // Equivalent to running DM without a via.
"Name of the \"via\" to use (e.g. \"picture_serialization\"). Optional.");
// Set in //bazel/devicesrc but only consumed by adb_test_runner.go. We cannot use the
// DEFINE_string macro because the flag name includes dashes.
[[maybe_unused]] static bool unused =
SkFlagInfo::CreateStringFlag("device-specific-bazel-config",
nullptr,
new CommandLineFlags::StringArray(),
nullptr,
"Ignored by this test runner.",
nullptr);
// Takes a SkBitmap and writes the resulting PNG and MD5 hash into the given files. Returns an
// empty string on success, or an error message in the case of failures.
static std::string write_png_and_json_files(std::string name,
std::map<std::string, std::string> commonKeys,
std::map<std::string, std::string> gmGoldKeys,
std::map<std::string, std::string> surfaceGoldKeys,
const SkBitmap& bitmap,
const char* pngPath,
const char* jsonPath) {
HashAndEncode hashAndEncode(bitmap);
// Compute MD5 hash.
SkMD5 hash;
hashAndEncode.feedHash(&hash);
SkMD5::Digest digest = hash.finish();
SkString md5 = digest.toLowercaseHexString();
// Write PNG file.
SkFILEWStream pngFile(pngPath);
bool result = hashAndEncode.encodePNG(&pngFile,
md5.c_str(),
/* key= */ CommandLineFlags::StringArray(),
/* properties= */ CommandLineFlags::StringArray());
if (!result) {
return "Error encoding or writing PNG to " + std::string(pngPath);
}
// Validate GM-related Gold keys.
if (gmGoldKeys.find("name") == gmGoldKeys.end()) {
SK_ABORT("gmGoldKeys does not contain key \"name\"");
}
if (gmGoldKeys.find("source_type") == gmGoldKeys.end()) {
SK_ABORT("gmGoldKeys does not contain key \"source_type\"");
}
// Validate surface-related Gold keys.
if (surfaceGoldKeys.find("surface_config") == surfaceGoldKeys.end()) {
SK_ABORT("surfaceGoldKeys does not contain key \"surface_config\"");
}
// Gather all Gold keys.
std::map<std::string, std::string> keys = {
{"build_system", "bazel"},
};
keys.merge(GetCompilationModeGoldAndPerfKeyValuePairs());
keys.merge(commonKeys);
keys.merge(surfaceGoldKeys);
keys.merge(gmGoldKeys);
// Write JSON file with MD5 hash and Gold key-value pairs.
SkFILEWStream jsonFile(jsonPath);
SkJSONWriter jsonWriter(&jsonFile, SkJSONWriter::Mode::kPretty);
jsonWriter.beginObject(); // Root object.
jsonWriter.appendString("md5", md5);
jsonWriter.beginObject("keys"); // "keys" dictionary.
for (auto const& [param, value] : keys) {
jsonWriter.appendString(param.c_str(), SkString(value));
}
jsonWriter.endObject(); // "keys" dictionary.
jsonWriter.endObject(); // Root object.
return "";
}
static std::string draw_result_to_string(skiagm::DrawResult result) {
switch (result) {
case skiagm::DrawResult::kOk:
return "Ok";
case skiagm::DrawResult::kFail:
return "Fail";
case skiagm::DrawResult::kSkip:
return "Skip";
default:
SkUNREACHABLE;
}
}
static int gNumSuccessfulGMs = 0;
static int gNumFailedGMs = 0;
static int gNumSkippedGMs = 0;
static bool gMissingCpuOrGpuWarningLogged = false;
// Runs a GM under the given surface config, and saves its output PNG file (and accompanying JSON
// file with metadata) to the given output directory.
void run_gm(std::unique_ptr<skiagm::GM> gm,
std::string config,
std::map<std::string, std::string> keyValuePairs,
std::string cpuName,
std::string gpuName,
std::string outputDir) {
TestRunner::Log("GM: %s", gm->getName().c_str());
// Create surface and canvas.
std::unique_ptr<SurfaceManager> surfaceManager = SurfaceManager::FromConfig(
config, SurfaceOptions{gm->getISize().width(), gm->getISize().height()});
if (surfaceManager == nullptr) {
SK_ABORT("Unknown --surfaceConfig flag value: %s.", config.c_str());
}
// Print warning about missing cpu_or_gpu key if necessary.
if ((surfaceManager->isCpuOrGpuBound() == SurfaceManager::CpuOrGpu::kCPU && cpuName == "" &&
!gMissingCpuOrGpuWarningLogged)) {
TestRunner::Log(
"\tWarning: The surface is CPU-bound, but flag --cpuName was not provided. "
"Gold traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\".");
gMissingCpuOrGpuWarningLogged = true;
}
if ((surfaceManager->isCpuOrGpuBound() == SurfaceManager::CpuOrGpu::kGPU && gpuName == "" &&
!gMissingCpuOrGpuWarningLogged)) {
TestRunner::Log(
"\tWarning: The surface is GPU-bound, but flag --gpuName was not provided. "
"Gold traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\".");
gMissingCpuOrGpuWarningLogged = true;
}
// Set up GPU.
TestRunner::Log("\tSetting up GPU...");
SkString msg;
skiagm::DrawResult result = gm->gpuSetup(surfaceManager->getSurface()->getCanvas(), &msg);
// Draw GM into canvas if GPU setup was successful.
SkBitmap bitmap;
if (result == skiagm::DrawResult::kOk) {
GMOutput output;
std::string viaName = FLAGS_via.size() == 0 ? "" : (FLAGS_via[0]);
TestRunner::Log("\tDrawing GM via \"%s\"...", viaName.c_str());
output = draw(gm.get(), surfaceManager->getSurface().get(), viaName);
result = output.result;
msg = SkString(output.msg.c_str());
bitmap = output.bitmap;
}
// Keep track of results. We will exit with a non-zero exit code in the case of failures.
switch (result) {
case skiagm::DrawResult::kOk:
// We don't increment numSuccessfulGMs just yet. We still need to successfully save
// its output bitmap to disk.
TestRunner::Log("\tFlushing surface...");
surfaceManager->flush();
break;
case skiagm::DrawResult::kFail:
gNumFailedGMs++;
break;
case skiagm::DrawResult::kSkip:
gNumSkippedGMs++;
break;
default:
SkUNREACHABLE;
}
// Report GM result and optional message.
TestRunner::Log("\tResult: %s", draw_result_to_string(result).c_str());
if (!msg.isEmpty()) {
TestRunner::Log("\tMessage: \"%s\"", msg.c_str());
}
// Save PNG and JSON file with MD5 hash to disk if the GM was successful.
if (result == skiagm::DrawResult::kOk) {
std::string name = std::string(gm->getName().c_str());
SkString pngPath = SkOSPath::Join(outputDir.c_str(), (name + ".png").c_str());
SkString jsonPath = SkOSPath::Join(outputDir.c_str(), (name + ".json").c_str());
std::string pngAndJSONResult =
write_png_and_json_files(gm->getName().c_str(),
keyValuePairs,
gm->getGoldKeys(),
surfaceManager->getGoldKeyValuePairs(cpuName, gpuName),
bitmap,
pngPath.c_str(),
jsonPath.c_str());
if (pngAndJSONResult != "") {
TestRunner::Log("%s", pngAndJSONResult.c_str());
gNumFailedGMs++;
} else {
gNumSuccessfulGMs++;
TestRunner::Log("\tPNG file written to: %s", pngPath.c_str());
TestRunner::Log("\tJSON file written to: %s", jsonPath.c_str());
}
}
}
int main(int argc, char** argv) {
TestRunner::InitAndLogCmdlineArgs(argc, argv);
// When running under Bazel (e.g. "bazel test //path/to:test"), we'll store output files in
// $TEST_UNDECLARED_OUTPUTS_DIR unless overridden via the --outputDir flag.
//
// See https://bazel.build/reference/test-encyclopedia#initial-conditions.
std::string testUndeclaredOutputsDir;
if (char* envVar = std::getenv("TEST_UNDECLARED_OUTPUTS_DIR")) {
testUndeclaredOutputsDir = envVar;
}
bool isBazelTest = !testUndeclaredOutputsDir.empty();
// Parse and validate flags.
CommandLineFlags::Parse(argc, argv);
if (!isBazelTest) {
TestRunner::FlagValidators::StringNonEmpty("--outputDir", FLAGS_outputDir);
}
TestRunner::FlagValidators::StringAtMostOne("--outputDir", FLAGS_outputDir);
TestRunner::FlagValidators::StringEven("--key", FLAGS_key);
TestRunner::FlagValidators::StringNonEmpty("--surfaceConfig", FLAGS_surfaceConfig);
TestRunner::FlagValidators::StringAtMostOne("--surfaceConfig", FLAGS_surfaceConfig);
TestRunner::FlagValidators::StringAtMostOne("--cpuName", FLAGS_cpuName);
TestRunner::FlagValidators::StringAtMostOne("--gpuName", FLAGS_gpuName);
TestRunner::FlagValidators::StringAtMostOne("--via", FLAGS_via);
std::string outputDir =
FLAGS_outputDir.isEmpty() ? testUndeclaredOutputsDir : FLAGS_outputDir[0];
std::map<std::string, std::string> keyValuePairs;
for (int i = 1; i < FLAGS_key.size(); i += 2) {
keyValuePairs[FLAGS_key[i - 1]] = FLAGS_key[i];
}
std::string config = FLAGS_surfaceConfig[0];
std::string cpuName = FLAGS_cpuName.isEmpty() ? "" : FLAGS_cpuName[0];
std::string gpuName = FLAGS_gpuName.isEmpty() ? "" : FLAGS_gpuName[0];
// Execute all GM registerer functions, then run all registered GMs.
for (const skiagm::GMRegistererFn& f : skiagm::GMRegistererFnRegistry::Range()) {
std::string errorMsg = f();
if (errorMsg != "") {
SK_ABORT("Error while gathering GMs: %s", errorMsg.c_str());
}
}
for (const skiagm::GMFactory& f : skiagm::GMRegistry::Range()) {
std::unique_ptr<skiagm::GM> gm = f();
if (!TestRunner::ShouldRunTestCase(gm->getName().c_str(), FLAGS_match, FLAGS_skip)) {
TestRunner::Log("Skipping %s", gm->getName().c_str());
continue;
}
run_gm(std::move(gm), config, keyValuePairs, cpuName, gpuName, outputDir);
}
// TODO(lovisolo): If running under Bazel, print command to display output files.
TestRunner::Log(gNumFailedGMs > 0 ? "FAIL" : "PASS");
TestRunner::Log(
"%d successful GMs (images written to %s).", gNumSuccessfulGMs, outputDir.c_str());
TestRunner::Log("%d failed GMs.", gNumFailedGMs);
TestRunner::Log("%d skipped GMs.", gNumSkippedGMs);
return gNumFailedGMs > 0 ? 1 : 0;
}