123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495 |
- /*
- * Copyright 2018 Google Inc.
- *
- * Use of this source code is governed by a BSD-style license that can be
- * found in the LICENSE file.
- */
- #include "tools/skqp/src/skqp.h"
- #include "gm/gm.h"
- #include "include/core/SkFontStyle.h"
- #include "include/core/SkGraphics.h"
- #include "include/core/SkStream.h"
- #include "include/core/SkSurface.h"
- #include "include/encode/SkPngEncoder.h"
- #include "include/gpu/GrContext.h"
- #include "include/gpu/GrContextOptions.h"
- #include "include/private/SkImageInfoPriv.h"
- #include "src/core/SkFontMgrPriv.h"
- #include "src/core/SkOSFile.h"
- #include "src/core/SkStreamPriv.h"
- #include "src/gpu/GrContextPriv.h"
- #include "src/utils/SkOSPath.h"
- #include "tests/Test.h"
- #include "tools/fonts/TestFontMgr.h"
- #include "tools/gpu/gl/GLTestContext.h"
- #include "tools/gpu/vk/VkTestContext.h"
- #include <limits.h>
- #include <algorithm>
- #include <cinttypes>
- #include <sstream>
- #include "tools/skqp/src/skqp_model.h"
- #define IMAGES_DIRECTORY_PATH "images"
- #define PATH_MAX_PNG "max.png"
- #define PATH_MIN_PNG "min.png"
- #define PATH_IMG_PNG "image.png"
- #define PATH_ERR_PNG "errors.png"
- #define PATH_MODEL "model"
- static constexpr char kRenderTestCSVReport[] = "out.csv";
- static constexpr char kRenderTestReportPath[] = "report.html";
- static constexpr char kRenderTestsPath[] = "skqp/rendertests.txt";
- static constexpr char kUnitTestReportPath[] = "unit_tests.txt";
- static constexpr char kUnitTestsPath[] = "skqp/unittests.txt";
- // Kind of like Python's readlines(), but without any allocation.
- // Calls f() on each line.
- // F is [](const char*, size_t) -> void
- template <typename F>
- static void readlines(const void* data, size_t size, F f) {
- const char* start = (const char*)data;
- const char* end = start + size;
- const char* ptr = start;
- while (ptr < end) {
- while (*ptr++ != '\n' && ptr < end) {}
- size_t len = ptr - start;
- f(start, len);
- start = ptr;
- }
- }
- static void get_unit_tests(SkQPAssetManager* mgr, std::vector<SkQP::UnitTest>* unitTests) {
- std::unordered_set<std::string> testset;
- auto insert = [&testset](const char* s, size_t l) {
- SkASSERT(l > 1) ;
- if (l > 0 && s[l - 1] == '\n') { // strip line endings.
- --l;
- }
- if (l > 0) { // only add non-empty strings.
- testset.insert(std::string(s, l));
- }
- };
- if (sk_sp<SkData> dat = mgr->open(kUnitTestsPath)) {
- readlines(dat->data(), dat->size(), insert);
- }
- for (const skiatest::Test& test : skiatest::TestRegistry::Range()) {
- if ((testset.empty() || testset.count(std::string(test.name)) > 0) && test.needsGpu) {
- unitTests->push_back(&test);
- }
- }
- auto lt = [](SkQP::UnitTest u, SkQP::UnitTest v) { return strcmp(u->name, v->name) < 0; };
- std::sort(unitTests->begin(), unitTests->end(), lt);
- }
- static void get_render_tests(SkQPAssetManager* mgr,
- std::vector<SkQP::GMFactory>* gmlist,
- std::unordered_map<std::string, int64_t>* gmThresholds) {
- auto insert = [gmThresholds](const char* s, size_t l) {
- SkASSERT(l > 1) ;
- if (l > 0 && s[l - 1] == '\n') { // strip line endings.
- --l;
- }
- if (l == 0) {
- return;
- }
- const char* end = s + l;
- const char* ptr = s;
- constexpr char kDelimeter = ',';
- while (ptr < end && *ptr != kDelimeter) { ++ptr; }
- if (ptr + 1 >= end) {
- SkASSERT(false); // missing delimeter
- return;
- }
- std::string key(s, ptr - s);
- ++ptr; // skip delimeter
- std::string number(ptr, end - ptr); // null-terminated copy.
- int64_t value = 0;
- if (1 != sscanf(number.c_str(), "%" SCNd64 , &value)) {
- SkASSERT(false); // Not a number
- return;
- }
- gmThresholds->insert({std::move(key), value}); // (*gmThresholds)[s] = value;
- };
- if (sk_sp<SkData> dat = mgr->open(kRenderTestsPath)) {
- readlines(dat->data(), dat->size(), insert);
- }
- using GmAndName = std::pair<SkQP::GMFactory, std::string>;
- std::vector<GmAndName> gmsWithNames;
- for (skiagm::GMFactory f : skiagm::GMRegistry::Range()) {
- std::string name = SkQP::GetGMName(f);
- if ((gmThresholds->empty() || gmThresholds->count(name) > 0)) {
- gmsWithNames.push_back(std::make_pair(f, std::move(name)));
- }
- }
- std::sort(gmsWithNames.begin(), gmsWithNames.end(),
- [](GmAndName u, GmAndName v) { return u.second < v.second; });
- gmlist->reserve(gmsWithNames.size());
- for (const GmAndName& gmn : gmsWithNames) {
- gmlist->push_back(gmn.first);
- }
- }
- static std::unique_ptr<sk_gpu_test::TestContext> make_test_context(SkQP::SkiaBackend backend) {
- using U = std::unique_ptr<sk_gpu_test::TestContext>;
- switch (backend) {
- case SkQP::SkiaBackend::kGL:
- return U(sk_gpu_test::CreatePlatformGLTestContext(kGL_GrGLStandard, nullptr));
- case SkQP::SkiaBackend::kGLES:
- return U(sk_gpu_test::CreatePlatformGLTestContext(kGLES_GrGLStandard, nullptr));
- #ifdef SK_VULKAN
- case SkQP::SkiaBackend::kVulkan:
- return U(sk_gpu_test::CreatePlatformVkTestContext(nullptr));
- #endif
- default:
- return nullptr;
- }
- }
- static GrContextOptions context_options(skiagm::GM* gm = nullptr) {
- GrContextOptions grContextOptions;
- grContextOptions.fAllowPathMaskCaching = true;
- grContextOptions.fDisableDriverCorrectnessWorkarounds = true;
- if (gm) {
- gm->modifyGrContextOptions(&grContextOptions);
- }
- return grContextOptions;
- }
- static std::vector<SkQP::SkiaBackend> get_backends() {
- std::vector<SkQP::SkiaBackend> result;
- SkQP::SkiaBackend backends[] = {
- #ifndef SK_BUILD_FOR_ANDROID
- SkQP::SkiaBackend::kGL, // Used for testing on desktop machines.
- #endif
- SkQP::SkiaBackend::kGLES,
- #ifdef SK_VULKAN
- SkQP::SkiaBackend::kVulkan,
- #endif
- };
- for (SkQP::SkiaBackend backend : backends) {
- std::unique_ptr<sk_gpu_test::TestContext> testCtx = make_test_context(backend);
- if (testCtx) {
- testCtx->makeCurrent();
- if (nullptr != testCtx->makeGrContext(context_options())) {
- result.push_back(backend);
- }
- }
- }
- SkASSERT_RELEASE(result.size() > 0);
- return result;
- }
- static void print_backend_info(const char* dstPath,
- const std::vector<SkQP::SkiaBackend>& backends) {
- #ifdef SK_ENABLE_DUMP_GPU
- SkFILEWStream out(dstPath);
- out.writeText("[\n");
- for (SkQP::SkiaBackend backend : backends) {
- if (std::unique_ptr<sk_gpu_test::TestContext> testCtx = make_test_context(backend)) {
- testCtx->makeCurrent();
- if (sk_sp<GrContext> ctx = testCtx->makeGrContext(context_options())) {
- SkString info = ctx->priv().dump();
- // remove null
- out.write(info.c_str(), info.size());
- out.writeText(",\n");
- }
- }
- }
- out.writeText("]\n");
- #endif
- }
- static void encode_png(const SkBitmap& src, const std::string& dst) {
- SkFILEWStream wStream(dst.c_str());
- SkPngEncoder::Options options;
- bool success = wStream.isValid() && SkPngEncoder::Encode(&wStream, src.pixmap(), options);
- SkASSERT_RELEASE(success);
- }
- static void write_to_file(const sk_sp<SkData>& src, const std::string& dst) {
- SkFILEWStream wStream(dst.c_str());
- bool success = wStream.isValid() && wStream.write(src->data(), src->size());
- SkASSERT_RELEASE(success);
- }
- ////////////////////////////////////////////////////////////////////////////////
- const char* SkQP::GetBackendName(SkQP::SkiaBackend b) {
- switch (b) {
- case SkQP::SkiaBackend::kGL: return "gl";
- case SkQP::SkiaBackend::kGLES: return "gles";
- case SkQP::SkiaBackend::kVulkan: return "vk";
- }
- return "";
- }
- std::string SkQP::GetGMName(SkQP::GMFactory f) {
- std::unique_ptr<skiagm::GM> gm(f ? f(nullptr) : nullptr);
- return std::string(gm ? gm->getName() : "");
- }
- const char* SkQP::GetUnitTestName(SkQP::UnitTest t) { return t->name; }
- SkQP::SkQP() {}
- SkQP::~SkQP() {}
- void SkQP::init(SkQPAssetManager* am, const char* reportDirectory) {
- SkASSERT_RELEASE(!fAssetManager);
- SkASSERT_RELEASE(am);
- fAssetManager = am;
- fReportDirectory = reportDirectory;
- SkGraphics::Init();
- gSkFontMgr_DefaultFactory = &ToolUtils::MakePortableFontMgr;
- /* If the file "skqp/rendertests.txt" does not exist or is empty, run all
- render tests. Otherwise only run tests mentioned in that file. */
- get_render_tests(fAssetManager, &fGMs, &fGMThresholds);
- /* If the file "skqp/unittests.txt" does not exist or is empty, run all gpu
- unit tests. Otherwise only run tests mentioned in that file. */
- get_unit_tests(fAssetManager, &fUnitTests);
- fSupportedBackends = get_backends();
- print_backend_info((fReportDirectory + "/grdump.txt").c_str(), fSupportedBackends);
- }
- std::tuple<SkQP::RenderOutcome, std::string> SkQP::evaluateGM(SkQP::SkiaBackend backend,
- SkQP::GMFactory gmFact) {
- SkASSERT_RELEASE(fAssetManager);
- static constexpr SkQP::RenderOutcome kError = {INT_MAX, INT_MAX, INT64_MAX};
- static constexpr SkQP::RenderOutcome kPass = {0, 0, 0};
- std::unique_ptr<sk_gpu_test::TestContext> testCtx = make_test_context(backend);
- if (!testCtx) {
- return std::make_tuple(kError, "Skia Failure: test context");
- }
- testCtx->makeCurrent();
- SkASSERT(gmFact);
- std::unique_ptr<skiagm::GM> gm(gmFact(nullptr));
- SkASSERT(gm);
- const char* const name = gm->getName();
- const SkISize size = gm->getISize();
- const int w = size.width();
- const int h = size.height();
- const SkImageInfo info =
- SkImageInfo::Make(w, h, skqp::kColorType, kPremul_SkAlphaType, nullptr);
- const SkSurfaceProps props(0, SkSurfaceProps::kLegacyFontHost_InitType);
- sk_sp<SkSurface> surf = SkSurface::MakeRenderTarget(
- testCtx->makeGrContext(context_options(gm.get())).get(),
- SkBudgeted::kNo, info, 0, &props);
- if (!surf) {
- return std::make_tuple(kError, "Skia Failure: gr-context");
- }
- gm->draw(surf->getCanvas());
- SkBitmap image;
- image.allocPixels(SkImageInfo::Make(w, h, skqp::kColorType, skqp::kAlphaType));
- // SkColorTypeBytesPerPixel should be constexpr, but is not.
- SkASSERT(SkColorTypeBytesPerPixel(skqp::kColorType) == sizeof(uint32_t));
- // Call readPixels because we need to compare pixels.
- if (!surf->readPixels(image.pixmap(), 0, 0)) {
- return std::make_tuple(kError, "Skia Failure: read pixels");
- }
- int64_t passingThreshold = fGMThresholds.empty() ? -1 : fGMThresholds[std::string(name)];
- if (-1 == passingThreshold) {
- return std::make_tuple(kPass, "");
- }
- skqp::ModelResult modelResult =
- skqp::CheckAgainstModel(name, image.pixmap(), fAssetManager);
- if (!modelResult.fErrorString.empty()) {
- return std::make_tuple(kError, std::move(modelResult.fErrorString));
- }
- fRenderResults.push_back(SkQP::RenderResult{backend, gmFact, modelResult.fOutcome});
- if (modelResult.fOutcome.fMaxError <= passingThreshold) {
- return std::make_tuple(kPass, "");
- }
- std::string imagesDirectory = fReportDirectory + "/" IMAGES_DIRECTORY_PATH;
- if (!sk_mkdir(imagesDirectory.c_str())) {
- SkDebugf("ERROR: sk_mkdir('%s');\n", imagesDirectory.c_str());
- return std::make_tuple(modelResult.fOutcome, "");
- }
- std::ostringstream tmp;
- tmp << imagesDirectory << '/' << SkQP::GetBackendName(backend) << '_' << name << '_';
- std::string imagesPathPrefix1 = tmp.str();
- tmp = std::ostringstream();
- tmp << imagesDirectory << '/' << PATH_MODEL << '_' << name << '_';
- std::string imagesPathPrefix2 = tmp.str();
- encode_png(image, imagesPathPrefix1 + PATH_IMG_PNG);
- encode_png(modelResult.fErrors, imagesPathPrefix1 + PATH_ERR_PNG);
- write_to_file(modelResult.fMaxPng, imagesPathPrefix2 + PATH_MAX_PNG);
- write_to_file(modelResult.fMinPng, imagesPathPrefix2 + PATH_MIN_PNG);
- return std::make_tuple(modelResult.fOutcome, "");
- }
- std::vector<std::string> SkQP::executeTest(SkQP::UnitTest test) {
- SkASSERT_RELEASE(fAssetManager);
- struct : public skiatest::Reporter {
- std::vector<std::string> fErrors;
- void reportFailed(const skiatest::Failure& failure) override {
- SkString desc = failure.toString();
- fErrors.push_back(std::string(desc.c_str(), desc.size()));
- }
- } r;
- GrContextOptions options;
- options.fDisableDriverCorrectnessWorkarounds = true;
- if (test->fContextOptionsProc) {
- test->fContextOptionsProc(&options);
- }
- test->proc(&r, options);
- fUnitTestResults.push_back(UnitTestResult{test, r.fErrors});
- return r.fErrors;
- }
- ////////////////////////////////////////////////////////////////////////////////
- static constexpr char kDocHead[] =
- "<!doctype html>\n"
- "<html lang=\"en\">\n"
- "<head>\n"
- "<meta charset=\"UTF-8\">\n"
- "<title>SkQP Report</title>\n"
- "<style>\n"
- "img { max-width:48%; border:1px green solid;\n"
- " image-rendering: pixelated;\n"
- " background-image:url('data:image/png;base64,iVBORw0KGgoA"
- "AAANSUhEUgAAABAAAAAQCAAAAAA6mKC9AAAAAXNSR0IArs4c6QAAAAJiS0dEAP+H"
- "j8y/AAAACXBIWXMAAA7DAAAOwwHHb6hkAAAAB3RJTUUH3gUBEi4DGRAQYgAAAB1J"
- "REFUGNNjfMoAAVJQmokBDdBHgPE/lPFsYN0BABdaAwN6tehMAAAAAElFTkSuQmCC"
- "'); }\n"
- "</style>\n"
- "<script>\n"
- "function ce(t) { return document.createElement(t); }\n"
- "function ct(n) { return document.createTextNode(n); }\n"
- "function ac(u,v) { return u.appendChild(v); }\n"
- "function br(u) { ac(u, ce(\"br\")); }\n"
- "function ma(s, c) { var a = ce(\"a\"); a.href = s; ac(a, c); return a; }\n"
- "function f(backend, gm, e1, e2, e3) {\n"
- " var b = ce(\"div\");\n"
- " var x = ce(\"h2\");\n"
- " var t = backend + \"_\" + gm;\n"
- " ac(x, ct(t));\n"
- " ac(b, x);\n"
- " ac(b, ct(\"backend: \" + backend));\n"
- " br(b);\n"
- " ac(b, ct(\"gm name: \" + gm));\n"
- " br(b);\n"
- " ac(b, ct(\"maximum error: \" + e1));\n"
- " br(b);\n"
- " ac(b, ct(\"bad pixel counts: \" + e2));\n"
- " br(b);\n"
- " ac(b, ct(\"total error: \" + e3));\n"
- " br(b);\n"
- " var q = \"" IMAGES_DIRECTORY_PATH "/\" + backend + \"_\" + gm + \"_\";\n"
- " var p = \"" IMAGES_DIRECTORY_PATH "/" PATH_MODEL "_\" + gm + \"_\";\n"
- " var i = ce(\"img\");\n"
- " i.src = q + \"" PATH_IMG_PNG "\";\n"
- " i.alt = \"img\";\n"
- " ac(b, ma(i.src, i));\n"
- " i = ce(\"img\");\n"
- " i.src = q + \"" PATH_ERR_PNG "\";\n"
- " i.alt = \"err\";\n"
- " ac(b, ma(i.src, i));\n"
- " br(b);\n"
- " ac(b, ct(\"Expectation: \"));\n"
- " ac(b, ma(p + \"" PATH_MAX_PNG "\", ct(\"max\")));\n"
- " ac(b, ct(\" | \"));\n"
- " ac(b, ma(p + \"" PATH_MIN_PNG "\", ct(\"min\")));\n"
- " ac(b, ce(\"hr\"));\n"
- " b.id = backend + \":\" + gm;\n"
- " ac(document.body, b);\n"
- " l = ce(\"li\");\n"
- " ac(l, ct(\"[\" + e3 + \"] \"));\n"
- " ac(l, ma(\"#\" + backend +\":\"+ gm , ct(t)));\n"
- " ac(document.getElementById(\"toc\"), l);\n"
- "}\n"
- "function main() {\n";
- static constexpr char kDocMiddle[] =
- "}\n"
- "</script>\n"
- "</head>\n"
- "<body onload=\"main()\">\n"
- "<h1>SkQP Report</h1>\n";
- static constexpr char kDocTail[] =
- "<ul id=\"toc\"></ul>\n"
- "<hr>\n"
- "<p>Left image: test result<br>\n"
- "Right image: errors (white = no error, black = smallest error, red = biggest error; "
- "other errors are a color between black and red.)</p>\n"
- "<hr>\n"
- "</body>\n"
- "</html>\n";
- template <typename T>
- inline void write(SkWStream* wStream, const T& text) {
- wStream->write(text.c_str(), text.size());
- }
- void SkQP::makeReport() {
- SkASSERT_RELEASE(fAssetManager);
- int glesErrorCount = 0, vkErrorCount = 0, gles = 0, vk = 0;
- if (!sk_isdir(fReportDirectory.c_str())) {
- SkDebugf("Report destination does not exist: '%s'\n", fReportDirectory.c_str());
- return;
- }
- SkFILEWStream csvOut(SkOSPath::Join(fReportDirectory.c_str(), kRenderTestCSVReport).c_str());
- SkFILEWStream htmOut(SkOSPath::Join(fReportDirectory.c_str(), kRenderTestReportPath).c_str());
- SkASSERT_RELEASE(csvOut.isValid() && htmOut.isValid());
- htmOut.writeText(kDocHead);
- for (const SkQP::RenderResult& run : fRenderResults) {
- switch (run.fBackend) {
- case SkQP::SkiaBackend::kGLES: ++gles; break;
- case SkQP::SkiaBackend::kVulkan: ++vk; break;
- default: break;
- }
- const char* backendName = SkQP::GetBackendName(run.fBackend);
- std::string gmName = SkQP::GetGMName(run.fGM);
- const SkQP::RenderOutcome& outcome = run.fOutcome;
- auto str = SkStringPrintf("\"%s\",\"%s\",%d,%d,%" PRId64, backendName, gmName.c_str(),
- outcome.fMaxError, outcome.fBadPixelCount, outcome.fTotalError);
- write(&csvOut, SkStringPrintf("%s\n", str.c_str()));
- int64_t passingThreshold = fGMThresholds.empty() ? 0 : fGMThresholds[gmName];
- if (passingThreshold == -1 || outcome.fMaxError <= passingThreshold) {
- continue;
- }
- write(&htmOut, SkStringPrintf(" f(%s);\n", str.c_str()));
- switch (run.fBackend) {
- case SkQP::SkiaBackend::kGLES: ++glesErrorCount; break;
- case SkQP::SkiaBackend::kVulkan: ++vkErrorCount; break;
- default: break;
- }
- }
- htmOut.writeText(kDocMiddle);
- write(&htmOut, SkStringPrintf("<p>gles errors: %d (of %d)</br>\n"
- "vk errors: %d (of %d)</p>\n",
- glesErrorCount, gles, vkErrorCount, vk));
- htmOut.writeText(kDocTail);
- SkFILEWStream unitOut(SkOSPath::Join(fReportDirectory.c_str(), kUnitTestReportPath).c_str());
- SkASSERT_RELEASE(unitOut.isValid());
- for (const SkQP::UnitTestResult& result : fUnitTestResults) {
- unitOut.writeText(GetUnitTestName(result.fUnitTest));
- if (result.fErrors.empty()) {
- unitOut.writeText(" PASSED\n* * *\n");
- } else {
- write(&unitOut, SkStringPrintf(" FAILED (%u errors)\n", result.fErrors.size()));
- for (const std::string& err : result.fErrors) {
- write(&unitOut, err);
- unitOut.newline();
- }
- unitOut.writeText("* * *\n");
- }
- }
- }
|