| /* |
| * Copyright 2023 Google LLC |
| * |
| * Use of this source code is governed by a BSD-style license that can be |
| * found in the LICENSE file. |
| * |
| * This program runs all GMs registered via macros such as DEF_GM, and for each GM, it saves the |
| * resulting SkBitmap as a .png file to disk, along with a .json file with the hash of the pixels. |
| */ |
| |
| #include "gm/gm.h" |
| #include "include/core/SkBitmap.h" |
| #include "include/core/SkCanvas.h" |
| #include "include/core/SkColorSpace.h" |
| #include "include/core/SkColorType.h" |
| #include "include/core/SkImageInfo.h" |
| #include "include/core/SkStream.h" |
| #include "include/core/SkSurface.h" |
| #include "include/encode/SkPngEncoder.h" |
| #include "include/private/base/SkAssert.h" |
| #include "include/private/base/SkDebug.h" |
| #include "src/core/SkMD5.h" |
| #include "src/utils/SkJSONWriter.h" |
| #include "src/utils/SkOSPath.h" |
| #include "tools/HashAndEncode.h" |
| #include "tools/testrunners/common/TestRunner.h" |
| #include "tools/testrunners/common/compilation_mode_keys/CompilationModeKeys.h" |
| #include "tools/testrunners/common/surface_manager/SurfaceManager.h" |
| #include "tools/testrunners/gm/vias/Draw.h" |
| |
| #include <ctime> |
| #include <filesystem> |
| #include <iomanip> |
| #include <iostream> |
| #include <sstream> |
| #include <string> |
| |
| // TODO(lovisolo): Add flag --omitDigestIfHashInFile (provides the known hashes file). |
| |
| static DEFINE_string(skip, "", "Space-separated list of test cases (regexps) to skip."); |
| static DEFINE_string( |
| match, |
| "", |
| "Space-separated list of test cases (regexps) to run. Will run all tests if omitted."); |
| |
| // When running under Bazel and overriding the output directory, you might encounter errors such |
| // as "No such file or directory" and "Read-only file system". The former can happen when running |
| // on RBE because the passed in output dir might not exist on the remote worker, whereas the latter |
| // can happen when running locally in sandboxed mode, which is the default strategy when running |
| // outside of RBE. One possible workaround is to run the test as a local subprocess, which can be |
| // done by passing flag --strategy=TestRunner=local to Bazel. |
| // |
| // Reference: https://bazel.build/docs/user-manual#execution-strategy. |
| static DEFINE_string(outputDir, |
| "", |
| "Directory where to write any output .png and .json files. " |
| "Optional when running under Bazel " |
| "(e.g. \"bazel test //path/to:test\") as it defaults to " |
| "$TEST_UNDECLARED_OUTPUTS_DIR."); |
| |
| static DEFINE_string(key, "", "Space-separated key/value pairs common to all traces."); |
| |
| // We named this flag --surfaceConfig rather than --config to avoid confusion with the --config |
| // Bazel flag. |
| static DEFINE_string( |
| surfaceConfig, |
| "", |
| "Name of the Surface configuration to use (e.g. \"8888\"). This determines " |
| "how we construct the SkSurface from which we get the SkCanvas that GMs will " |
| "draw on. See file //tools/testrunners/common/surface_manager/SurfaceManager.h for " |
| "details."); |
| |
| static DEFINE_string( |
| cpuName, |
| "", |
| "Contents of the \"cpu_or_gpu_value\" dimension for CPU-bound traces (e.g. \"AVX512\")."); |
| |
| static DEFINE_string( |
| gpuName, |
| "", |
| "Contents of the \"cpu_or_gpu_value\" dimension for GPU-bound traces (e.g. \"RTX3060\")."); |
| |
| static DEFINE_string(via, |
| "direct", // Equivalent to running DM without a via. |
| "Name of the \"via\" to use (e.g. \"picture_serialization\"). Optional."); |
| |
| // Set in //bazel/devicesrc but only consumed by adb_test_runner.go. We cannot use the |
| // DEFINE_string macro because the flag name includes dashes. |
| [[maybe_unused]] static bool unused = |
| SkFlagInfo::CreateStringFlag("device-specific-bazel-config", |
| nullptr, |
| new CommandLineFlags::StringArray(), |
| nullptr, |
| "Ignored by this test runner.", |
| nullptr); |
| |
| // Takes a SkBitmap and writes the resulting PNG and MD5 hash into the given files. Returns an |
| // empty string on success, or an error message in the case of failures. |
| static std::string write_png_and_json_files(std::string name, |
| std::map<std::string, std::string> commonKeys, |
| std::map<std::string, std::string> gmGoldKeys, |
| std::map<std::string, std::string> surfaceGoldKeys, |
| const SkBitmap& bitmap, |
| const char* pngPath, |
| const char* jsonPath) { |
| HashAndEncode hashAndEncode(bitmap); |
| |
| // Compute MD5 hash. |
| SkMD5 hash; |
| hashAndEncode.feedHash(&hash); |
| SkMD5::Digest digest = hash.finish(); |
| SkString md5 = digest.toLowercaseHexString(); |
| |
| // Write PNG file. |
| SkFILEWStream pngFile(pngPath); |
| bool result = hashAndEncode.encodePNG(&pngFile, |
| md5.c_str(), |
| /* key= */ CommandLineFlags::StringArray(), |
| /* properties= */ CommandLineFlags::StringArray()); |
| if (!result) { |
| return "Error encoding or writing PNG to " + std::string(pngPath); |
| } |
| |
| // Validate GM-related Gold keys. |
| if (gmGoldKeys.find("name") == gmGoldKeys.end()) { |
| SK_ABORT("gmGoldKeys does not contain key \"name\""); |
| } |
| if (gmGoldKeys.find("source_type") == gmGoldKeys.end()) { |
| SK_ABORT("gmGoldKeys does not contain key \"source_type\""); |
| } |
| |
| // Validate surface-related Gold keys. |
| if (surfaceGoldKeys.find("surface_config") == surfaceGoldKeys.end()) { |
| SK_ABORT("surfaceGoldKeys does not contain key \"surface_config\""); |
| } |
| |
| // Gather all Gold keys. |
| std::map<std::string, std::string> keys = { |
| {"build_system", "bazel"}, |
| }; |
| keys.merge(GetCompilationModeGoldAndPerfKeyValuePairs()); |
| keys.merge(commonKeys); |
| keys.merge(surfaceGoldKeys); |
| keys.merge(gmGoldKeys); |
| |
| // Write JSON file with MD5 hash and Gold key-value pairs. |
| SkFILEWStream jsonFile(jsonPath); |
| SkJSONWriter jsonWriter(&jsonFile, SkJSONWriter::Mode::kPretty); |
| jsonWriter.beginObject(); // Root object. |
| jsonWriter.appendString("md5", md5); |
| jsonWriter.beginObject("keys"); // "keys" dictionary. |
| for (auto const& [param, value] : keys) { |
| jsonWriter.appendString(param.c_str(), SkString(value)); |
| } |
| jsonWriter.endObject(); // "keys" dictionary. |
| jsonWriter.endObject(); // Root object. |
| |
| return ""; |
| } |
| |
| static std::string draw_result_to_string(skiagm::DrawResult result) { |
| switch (result) { |
| case skiagm::DrawResult::kOk: |
| return "Ok"; |
| case skiagm::DrawResult::kFail: |
| return "Fail"; |
| case skiagm::DrawResult::kSkip: |
| return "Skip"; |
| default: |
| SkUNREACHABLE; |
| } |
| } |
| |
| static int gNumSuccessfulGMs = 0; |
| static int gNumFailedGMs = 0; |
| static int gNumSkippedGMs = 0; |
| |
| static bool gMissingCpuOrGpuWarningLogged = false; |
| |
| // Runs a GM under the given surface config, and saves its output PNG file (and accompanying JSON |
| // file with metadata) to the given output directory. |
| void run_gm(std::unique_ptr<skiagm::GM> gm, |
| std::string config, |
| std::map<std::string, std::string> keyValuePairs, |
| std::string cpuName, |
| std::string gpuName, |
| std::string outputDir) { |
| TestRunner::Log("GM: %s", gm->getName().c_str()); |
| |
| // Create surface and canvas. |
| std::unique_ptr<SurfaceManager> surfaceManager = SurfaceManager::FromConfig( |
| config, SurfaceOptions{gm->getISize().width(), gm->getISize().height()}); |
| if (surfaceManager == nullptr) { |
| SK_ABORT("Unknown --surfaceConfig flag value: %s.", config.c_str()); |
| } |
| |
| // Print warning about missing cpu_or_gpu key if necessary. |
| if ((surfaceManager->isCpuOrGpuBound() == SurfaceManager::CpuOrGpu::kCPU && cpuName == "" && |
| !gMissingCpuOrGpuWarningLogged)) { |
| TestRunner::Log( |
| "\tWarning: The surface is CPU-bound, but flag --cpuName was not provided. " |
| "Gold traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\"."); |
| gMissingCpuOrGpuWarningLogged = true; |
| } |
| if ((surfaceManager->isCpuOrGpuBound() == SurfaceManager::CpuOrGpu::kGPU && gpuName == "" && |
| !gMissingCpuOrGpuWarningLogged)) { |
| TestRunner::Log( |
| "\tWarning: The surface is GPU-bound, but flag --gpuName was not provided. " |
| "Gold traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\"."); |
| gMissingCpuOrGpuWarningLogged = true; |
| } |
| |
| // Set up GPU. |
| TestRunner::Log("\tSetting up GPU..."); |
| SkString msg; |
| skiagm::DrawResult result = gm->gpuSetup(surfaceManager->getSurface()->getCanvas(), &msg); |
| |
| // Draw GM into canvas if GPU setup was successful. |
| SkBitmap bitmap; |
| if (result == skiagm::DrawResult::kOk) { |
| GMOutput output; |
| std::string viaName = FLAGS_via.size() == 0 ? "" : (FLAGS_via[0]); |
| TestRunner::Log("\tDrawing GM via \"%s\"...", viaName.c_str()); |
| output = draw(gm.get(), surfaceManager->getSurface().get(), viaName); |
| result = output.result; |
| msg = SkString(output.msg.c_str()); |
| bitmap = output.bitmap; |
| } |
| |
| // Keep track of results. We will exit with a non-zero exit code in the case of failures. |
| switch (result) { |
| case skiagm::DrawResult::kOk: |
| // We don't increment numSuccessfulGMs just yet. We still need to successfully save |
| // its output bitmap to disk. |
| TestRunner::Log("\tFlushing surface..."); |
| surfaceManager->flush(); |
| break; |
| case skiagm::DrawResult::kFail: |
| gNumFailedGMs++; |
| break; |
| case skiagm::DrawResult::kSkip: |
| gNumSkippedGMs++; |
| break; |
| default: |
| SkUNREACHABLE; |
| } |
| |
| // Report GM result and optional message. |
| TestRunner::Log("\tResult: %s", draw_result_to_string(result).c_str()); |
| if (!msg.isEmpty()) { |
| TestRunner::Log("\tMessage: \"%s\"", msg.c_str()); |
| } |
| |
| // Save PNG and JSON file with MD5 hash to disk if the GM was successful. |
| if (result == skiagm::DrawResult::kOk) { |
| std::string name = std::string(gm->getName().c_str()); |
| SkString pngPath = SkOSPath::Join(outputDir.c_str(), (name + ".png").c_str()); |
| SkString jsonPath = SkOSPath::Join(outputDir.c_str(), (name + ".json").c_str()); |
| |
| std::string pngAndJSONResult = |
| write_png_and_json_files(gm->getName().c_str(), |
| keyValuePairs, |
| gm->getGoldKeys(), |
| surfaceManager->getGoldKeyValuePairs(cpuName, gpuName), |
| bitmap, |
| pngPath.c_str(), |
| jsonPath.c_str()); |
| if (pngAndJSONResult != "") { |
| TestRunner::Log("%s", pngAndJSONResult.c_str()); |
| gNumFailedGMs++; |
| } else { |
| gNumSuccessfulGMs++; |
| TestRunner::Log("\tPNG file written to: %s", pngPath.c_str()); |
| TestRunner::Log("\tJSON file written to: %s", jsonPath.c_str()); |
| } |
| } |
| } |
| |
| int main(int argc, char** argv) { |
| TestRunner::InitAndLogCmdlineArgs(argc, argv); |
| |
| // When running under Bazel (e.g. "bazel test //path/to:test"), we'll store output files in |
| // $TEST_UNDECLARED_OUTPUTS_DIR unless overridden via the --outputDir flag. |
| // |
| // See https://bazel.build/reference/test-encyclopedia#initial-conditions. |
| std::string testUndeclaredOutputsDir; |
| if (char* envVar = std::getenv("TEST_UNDECLARED_OUTPUTS_DIR")) { |
| testUndeclaredOutputsDir = envVar; |
| } |
| bool isBazelTest = !testUndeclaredOutputsDir.empty(); |
| |
| // Parse and validate flags. |
| CommandLineFlags::Parse(argc, argv); |
| if (!isBazelTest) { |
| TestRunner::FlagValidators::StringNonEmpty("--outputDir", FLAGS_outputDir); |
| } |
| TestRunner::FlagValidators::StringAtMostOne("--outputDir", FLAGS_outputDir); |
| TestRunner::FlagValidators::StringEven("--key", FLAGS_key); |
| TestRunner::FlagValidators::StringNonEmpty("--surfaceConfig", FLAGS_surfaceConfig); |
| TestRunner::FlagValidators::StringAtMostOne("--surfaceConfig", FLAGS_surfaceConfig); |
| TestRunner::FlagValidators::StringAtMostOne("--cpuName", FLAGS_cpuName); |
| TestRunner::FlagValidators::StringAtMostOne("--gpuName", FLAGS_gpuName); |
| TestRunner::FlagValidators::StringAtMostOne("--via", FLAGS_via); |
| |
| std::string outputDir = |
| FLAGS_outputDir.isEmpty() ? testUndeclaredOutputsDir : FLAGS_outputDir[0]; |
| std::map<std::string, std::string> keyValuePairs; |
| for (int i = 1; i < FLAGS_key.size(); i += 2) { |
| keyValuePairs[FLAGS_key[i - 1]] = FLAGS_key[i]; |
| } |
| std::string config = FLAGS_surfaceConfig[0]; |
| std::string cpuName = FLAGS_cpuName.isEmpty() ? "" : FLAGS_cpuName[0]; |
| std::string gpuName = FLAGS_gpuName.isEmpty() ? "" : FLAGS_gpuName[0]; |
| |
| // Execute all GM registerer functions, then run all registered GMs. |
| for (const skiagm::GMRegistererFn& f : skiagm::GMRegistererFnRegistry::Range()) { |
| std::string errorMsg = f(); |
| if (errorMsg != "") { |
| SK_ABORT("Error while gathering GMs: %s", errorMsg.c_str()); |
| } |
| } |
| for (const skiagm::GMFactory& f : skiagm::GMRegistry::Range()) { |
| std::unique_ptr<skiagm::GM> gm = f(); |
| |
| if (!TestRunner::ShouldRunTestCase(gm->getName().c_str(), FLAGS_match, FLAGS_skip)) { |
| TestRunner::Log("Skipping %s", gm->getName().c_str()); |
| continue; |
| } |
| |
| run_gm(std::move(gm), config, keyValuePairs, cpuName, gpuName, outputDir); |
| } |
| |
| // TODO(lovisolo): If running under Bazel, print command to display output files. |
| |
| TestRunner::Log(gNumFailedGMs > 0 ? "FAIL" : "PASS"); |
| TestRunner::Log( |
| "%d successful GMs (images written to %s).", gNumSuccessfulGMs, outputDir.c_str()); |
| TestRunner::Log("%d failed GMs.", gNumFailedGMs); |
| TestRunner::Log("%d skipped GMs.", gNumSkippedGMs); |
| return gNumFailedGMs > 0 ? 1 : 0; |
| } |