SkQP: fix model colorspace (16-bit gold images)
Problem: `make_gmkb.go `was ignoring the ignoring the embedded ICC
profile in the images it was getting from Gold.
Replace make_gmkb.go with two small programs: `goldgetter.py` and
`make_skqp_model.cpp`.
`make_skqp_model` uses Skia to create the model from a bunch of images.
`goldgetter` wraps `make_skqp_model` and handles:
- json parsing
- downloading images from gold
- multiprocessing
CQ_INCLUDE_TRYBOTS=skia.primary:Build-Debian9-Clang-x86-devrel-Android_SKQP,Test-Debian9-Clang-NUC7i5BNK-CPU-Emulator-x86-devrel-All-Android_SKQP
Change-Id: I7add1a1dfd83bbd0ab07ab126d4183c36325263c
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/209101
Commit-Queue: Hal Canary <halcanary@google.com>
Reviewed-by: Mike Klein <mtklein@google.com>
diff --git a/BUILD.gn b/BUILD.gn
index d5f43f9..51d3510 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -1812,6 +1812,15 @@
}
}
+ test_app("make_skqp_model") {
+ sources = [
+ "tools/skqp/make_skqp_model.cpp",
+ ]
+ deps = [
+ ":skia",
+ ]
+ }
+
if (target_cpu != "wasm") {
import("gn/samples.gni")
test_lib("samples") {
diff --git a/tools/skqp/cut_release b/tools/skqp/cut_release
index cc92072..a6b199c 100755
--- a/tools/skqp/cut_release
+++ b/tools/skqp/cut_release
@@ -11,21 +11,16 @@
set -e
META_JSON="$1"
cd "$(dirname "$0")/../.."
-
-if [ -z "$SKQP_SKIP_INFRA_UPDATE" ]; then
- go get -u go.skia.org/infra/golden/go/search
-fi
-go run tools/skqp/make_gmkb.go \
- "$META_JSON" \
- platform_tools/android/apps/skqp/src/main/assets/gmkb
env GIT_SYNC_DEPS_QUIET=1 python tools/git-sync-deps
O='out/ndebug'
mkdir -p $O
bin/gn gen $O --args='cc="clang" cxx="clang++" is_debug=false'
-ninja -C $O jitter_gms list_gpu_unit_tests
+ninja -C $O jitter_gms list_gpu_unit_tests make_skqp_model
+GMKB='platform_tools/android/apps/skqp/src/main/assets/gmkb'
+python tools/skqp/goldgetter.py "$META_JSON" "$GMKB" $O/make_skqp_model
$O/jitter_gms tools/skqp/bad_gms.txt
python tools/skqp/make_rendertests_list.py
-rm 'bad.txt' 'good.txt'
+rm 'bad.txt' 'good.txt' "$GMKB"/models.txt
sh tools/skqp/upload_model
$O/list_gpu_unit_tests \
> platform_tools/android/apps/skqp/src/main/assets/skqp/unittests.txt
diff --git a/tools/skqp/goldgetter.py b/tools/skqp/goldgetter.py
new file mode 100755
index 0000000..ba58eb9
--- /dev/null
+++ b/tools/skqp/goldgetter.py
@@ -0,0 +1,48 @@
+#! /usr/bin/env python
+# Copyright 2019 Google LLC.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import multiprocessing
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import urllib
+
+def make_skqp_model(arg):
+ name, urls, dst_dir, exe = arg
+ tmp = tempfile.mkdtemp()
+ for url in urls:
+ urllib.urlretrieve(url, tmp + '/' + url[url.rindex('/') + 1:])
+ subprocess.check_call([exe, tmp, dst_dir + '/' + name])
+ shutil.rmtree(tmp)
+ sys.stdout.write(name + ' ')
+ sys.stdout.flush()
+
+def main(meta, dst, exe):
+ assert os.path.exists(exe)
+ jobs = []
+ with open(meta, 'r') as f:
+ for rec in json.load(f):
+ urls = [d['URL'] for d in rec['digests']
+ if d['status'] == 'positive' and
+ (set(d['paramset']['config']) & set(['vk', 'gles']))]
+ if urls:
+ jobs.append((rec['testName'], urls, dst, exe))
+ if not os.path.exists(dst):
+ os.mkdir(dst)
+ pool = multiprocessing.Pool(processes=20)
+ pool.map(make_skqp_model, jobs)
+ sys.stdout.write('\n')
+ with open(dst + '/models.txt', 'w') as o:
+ for n, _, _, _ in jobs:
+ o.write(n + '\n')
+
+if __name__ == '__main__':
+ if len(sys.argv) != 4:
+ sys.stderr.write('Usage:\n %s META.JSON DST_DIR MAKE_SKQP_MODEL_EXE\n\n' % sys.argv[0])
+ sys.exit(1)
+ main(sys.argv[1], sys.argv[2], sys.argv[3])
diff --git a/tools/skqp/make_gmkb.go b/tools/skqp/make_gmkb.go
deleted file mode 100644
index 2b5dda5..0000000
--- a/tools/skqp/make_gmkb.go
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Copyright 2017 Google Inc.
- *
- * Use of this source code is governed by a BSD-style license that can be
- * found in the LICENSE file.
- */
-package main
-
-import (
- "encoding/json"
- "errors"
- "fmt"
- "image"
- "image/draw"
- "image/png"
- "log"
- "net/http"
- "os"
- "path"
- "sort"
- "strings"
- "sync"
-
- "go.skia.org/infra/golden/go/search"
-)
-
-const (
- min_png = "min.png"
- max_png = "max.png"
-)
-
-type ExportTestRecordArray []search.ExportTestRecord
-
-func (a ExportTestRecordArray) Len() int { return len(a) }
-func (a ExportTestRecordArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
-func (a ExportTestRecordArray) Less(i, j int) bool { return a[i].TestName < a[j].TestName }
-
-func in(v string, a []string) bool {
- for _, u := range a {
- if u == v {
- return true
- }
- }
- return false
-}
-
-func clampU8(v int) uint8 {
- if v < 0 {
- return 0
- } else if v > 255 {
- return 255
- }
- return uint8(v)
-}
-
-func processTest(testName string, imgUrls []string, output string) (bool, error) {
- if strings.ContainsRune(testName, '/') {
- return false, nil
- }
- output_directory := path.Join(output, testName)
- var img_max image.NRGBA
- var img_min image.NRGBA
- for _, url := range imgUrls {
- resp, err := http.Get(url)
- if err != nil {
- return false, err
- }
- img, err := png.Decode(resp.Body)
- resp.Body.Close()
- if err != nil {
- return false, err
- }
- if img_max.Rect.Max.X == 0 {
- // N.B. img_max.Pix may alias img.Pix (if they're already NRGBA).
- img_max = toNrgba(img)
- img_min = copyNrgba(img_max)
- continue
- }
- w := img.Bounds().Max.X - img.Bounds().Min.X
- h := img.Bounds().Max.Y - img.Bounds().Min.Y
- if img_max.Rect.Max.X != w || img_max.Rect.Max.Y != h {
- return false, errors.New("size mismatch")
- }
- img_nrgba := toNrgba(img)
- for i, value := range img_nrgba.Pix {
- if value > img_max.Pix[i] {
- img_max.Pix[i] = value
- } else if value < img_min.Pix[i] {
- img_min.Pix[i] = value
- }
- }
- }
- if img_max.Rect.Max.X == 0 {
- return false, nil
- }
-
- if err := os.Mkdir(output_directory, os.ModePerm); err != nil && !os.IsExist(err) {
- return false, err
- }
- if err := writePngToFile(path.Join(output_directory, min_png), &img_min); err != nil {
- return false, err
- }
- if err := writePngToFile(path.Join(output_directory, max_png), &img_max); err != nil {
- return false, err
- }
- return true, nil
-}
-
-type LockedStringList struct {
- List []string
- mux sync.Mutex
-}
-
-func (l *LockedStringList) add(v string) {
- l.mux.Lock()
- defer l.mux.Unlock()
- l.List = append(l.List, v)
-}
-
-
-func readMetaJsonFile(filename string) ([]search.ExportTestRecord, error) {
- file, err := os.Open(filename)
- if err != nil {
- return nil, err
- }
- dec := json.NewDecoder(file)
- var records []search.ExportTestRecord
- err = dec.Decode(&records)
- return records, err
-}
-
-func writePngToFile(path string, img image.Image) error {
- file, err := os.Create(path)
- if err != nil {
- return err
- }
- defer file.Close()
- return png.Encode(file, img)
-}
-
-// to_nrgb() may return a shallow copy of img if it's already NRGBA.
-func toNrgba(img image.Image) image.NRGBA {
- switch v := img.(type) {
- case *image.NRGBA:
- return *v
- }
- nimg := *image.NewNRGBA(img.Bounds())
- draw.Draw(&nimg, img.Bounds(), img, image.Point{0, 0}, draw.Src)
- return nimg
-}
-
-func copyNrgba(src image.NRGBA) image.NRGBA {
- dst := image.NRGBA{make([]uint8, len(src.Pix)), src.Stride, src.Rect}
- copy(dst.Pix, src.Pix)
- return dst
-}
-
-func main() {
- if len(os.Args) != 3 {
- log.Printf("Usage:\n %s INPUT.json OUTPUT_DIRECTORY\n\n", os.Args[0])
- os.Exit(1)
- }
- input := os.Args[1]
- output := os.Args[2]
- // output is removed and replaced with a clean directory.
- if err := os.RemoveAll(output); err != nil && !os.IsNotExist(err) {
- log.Fatal(err)
- }
- if err := os.MkdirAll(output, os.ModePerm); err != nil && !os.IsExist(err) {
- log.Fatal(err)
- }
-
- records, err := readMetaJsonFile(input)
- if err != nil {
- log.Fatal(err)
- }
- sort.Sort(ExportTestRecordArray(records))
-
- var results LockedStringList
- var wg sync.WaitGroup
- for _, record := range records {
- var goodUrls []string
- for _, digest := range record.Digests {
- if (in("vk", digest.ParamSet["config"]) ||
- in("gles", digest.ParamSet["config"])) &&
- digest.Status == "positive" {
- goodUrls = append(goodUrls, digest.URL)
- }
- }
- wg.Add(1)
- go func(testName string, imgUrls []string, output string, results* LockedStringList) {
- defer wg.Done()
- success, err := processTest(testName, imgUrls, output)
- if err != nil {
- log.Fatal(err)
- }
- if success {
- results.add(testName)
- }
- fmt.Printf("\r%-60s", testName)
- }(record.TestName, goodUrls, output, &results)
- }
- wg.Wait()
- fmt.Printf("\r%60s\n", "")
- sort.Strings(results.List)
- modelFile, err := os.Create(path.Join(output, "models.txt"))
- if err != nil {
- log.Fatal(err)
- }
- for _, v := range results.List {
- fmt.Fprintln(modelFile, v)
- }
-}
diff --git a/tools/skqp/make_skqp_model.cpp b/tools/skqp/make_skqp_model.cpp
new file mode 100644
index 0000000..4bf7e13
--- /dev/null
+++ b/tools/skqp/make_skqp_model.cpp
@@ -0,0 +1,82 @@
+// Copyright 2019 Google LLC.
+// Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.
+
+#include "SkBitmap.h"
+#include "SkCodec.h"
+#include "SkPngEncoder.h"
+#include "SkOSFile.h"
+
+static void update(SkBitmap* maxBitmap, SkBitmap* minBitmap, const SkBitmap& bm) {
+ SkASSERT(!bm.drawsNothing());
+ SkASSERT(4 == bm.bytesPerPixel());
+ if (maxBitmap->drawsNothing()) {
+ maxBitmap->allocPixels(bm.info());
+ maxBitmap->eraseColor(0x00000000);
+ minBitmap->allocPixels(bm.info());
+ minBitmap->eraseColor(0xFFFFFFFF);
+ }
+ SkASSERT_RELEASE(maxBitmap->info() == bm.info());
+ const SkPixmap& pmin = minBitmap->pixmap();
+ const SkPixmap& pmax = maxBitmap->pixmap();
+ const SkPixmap& pm = bm.pixmap();
+ for (int y = 0; y < pm.height(); ++y) {
+ for (int x = 0; x < pm.width(); ++x) {
+ uint32_t* minPtr = pmin.writable_addr32(x, y);
+ uint32_t* maxPtr = pmax.writable_addr32(x, y);
+ uint8_t minColor[4], maxColor[4], color[4];
+ memcpy(minColor, minPtr, 4);
+ memcpy(maxColor, maxPtr, 4);
+ memcpy(color, pm.addr32(x, y), 4);
+ for (unsigned i = 0; i < 4; ++i) {
+ minColor[i] = std::min(minColor[i], color[i]);
+ maxColor[i] = std::max(maxColor[i], color[i]);
+ }
+ memcpy(minPtr, minColor, 4);
+ memcpy(maxPtr, maxColor, 4);
+ }
+ }
+}
+
+static SkBitmap decode_to_srgb_8888_unpremul(const char* path) {
+ SkBitmap dst;
+ if (auto codec = SkCodec::MakeFromData(SkData::MakeFromFileName(path))) {
+ SkISize size = codec->getInfo().dimensions();
+ SkASSERT(!size.isEmpty());
+ dst.allocPixels(SkImageInfo::Make(
+ size.width(), size.height(), kRGBA_8888_SkColorType,
+ kUnpremul_SkAlphaType, SkColorSpace::MakeSRGB()));
+ if (SkCodec::kSuccess != codec->getPixels(dst.pixmap())) {
+ dst.reset();
+ }
+ }
+ return dst;
+}
+
+bool encode_png(const char* path, const SkPixmap& pixmap) {
+ if (!pixmap.addr()) {
+ return false;
+ }
+ SkPngEncoder::Options encOpts;
+ encOpts.fZLibLevel = 9; // slow encode;
+ SkFILEWStream o(path);
+ return o.isValid() && SkPngEncoder::Encode(&o, pixmap, encOpts);
+}
+
+int main(int argc, char** argv) {
+ SkASSERT_RELEASE(argc > 2);
+ const char* src_dir = argv[1];
+ const char* dst_dir = argv[2];
+ SkBitmap maxBitmap, minBitmap;
+ SkOSFile::Iter iter(src_dir);
+ SkString name;
+ while (iter.next(&name)) {
+ name.prependf("%s/", src_dir);
+ SkBitmap bm = decode_to_srgb_8888_unpremul(name.c_str());
+ if (!bm.drawsNothing()) {
+ update(&maxBitmap, &minBitmap, bm);
+ }
+ }
+ SkASSERT_RELEASE(sk_mkdir(dst_dir));
+ encode_png(SkStringPrintf("%s/min.png", dst_dir).c_str(), minBitmap.pixmap());
+ encode_png(SkStringPrintf("%s/max.png", dst_dir).c_str(), maxBitmap.pixmap());
+}
diff --git a/tools/skqp/test_apk.sh b/tools/skqp/test_apk.sh
index 4a155d7..42a6382 100755
--- a/tools/skqp/test_apk.sh
+++ b/tools/skqp/test_apk.sh
@@ -34,7 +34,8 @@
TDIR="$(mktemp -d "${TMPDIR:-/tmp}/skqp_report.XXXXXXXXXX")"
-adb install -r "$APK" || exit 2
+adb uninstall org.skia.skqp
+adb install "$APK" || exit 2
adb logcat -c
adb logcat TestRunner org.skia.skqp skia DEBUG "*:S" | tee "${TDIR}/logcat.txt" &
@@ -72,7 +73,7 @@
if [ -f "$REPORT" ]; then
grep 'f(.*;' "$REPORT"
echo "$REPORT"
- "$(dirname "$0")"/../../bin/sysopen "$REPORT"
+ "$(dirname "$0")"/../../bin/sysopen "$REPORT" > /dev/null 2>&1 &
else
echo "$TDIR"
fi
