skia_gold_pixel_diff.cc 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355
  1. // Copyright 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "ui/base/test/skia_gold_pixel_diff.h"
  5. #include "build/build_config.h"
  6. #if BUILDFLAG(IS_WIN)
  7. #include <windows.h>
  8. #endif
  9. #include "third_party/skia/include/core/SkBitmap.h"
  10. #include "base/command_line.h"
  11. #include "base/environment.h"
  12. #include "base/files/file.h"
  13. #include "base/files/file_util.h"
  14. #include "base/json/json_reader.h"
  15. #include "base/json/json_writer.h"
  16. #include "base/logging.h"
  17. #include "base/path_service.h"
  18. #include "base/process/launch.h"
  19. #include "base/process/process.h"
  20. #include "base/strings/string_util.h"
  21. #include "base/test/test_switches.h"
  22. #include "base/threading/thread_restrictions.h"
  23. #include "base/values.h"
  24. #include "build/chromeos_buildflags.h"
  25. #include "testing/gtest/include/gtest/gtest.h"
  26. #include "ui/base/test/skia_gold_matching_algorithm.h"
  27. #include "ui/gfx/codec/png_codec.h"
  28. #include "ui/gfx/image/image.h"
  29. #include "ui/snapshot/snapshot.h"
  30. namespace ui {
  31. namespace test {
  32. const char* kSkiaGoldInstance = "chrome";
  33. #if BUILDFLAG(IS_WIN)
  34. const wchar_t* kSkiaGoldCtl = L"tools/skia_goldctl/win/goldctl.exe";
  35. #elif BUILDFLAG(IS_APPLE)
  36. #if defined(ARCH_CPU_ARM64)
  37. const char* kSkiaGoldCtl = "tools/skia_goldctl/mac_arm64/goldctl";
  38. #else
  39. const char* kSkiaGoldCtl = "tools/skia_goldctl/mac_amd64/goldctl";
  40. #endif // defined(ARCH_CPU_ARM64)
  41. #else
  42. const char* kSkiaGoldCtl = "tools/skia_goldctl/linux/goldctl";
  43. #endif
  44. const char* kBuildRevisionKey = "git-revision";
  45. // The switch keys for tryjob.
  46. const char* kIssueKey = "gerrit-issue";
  47. const char* kPatchSetKey = "gerrit-patchset";
  48. const char* kJobIdKey = "buildbucket-id";
  49. const char* kCodeReviewSystemKey = "code-review-system";
  50. const char* kNoLuciAuth = "no-luci-auth";
  51. const char* kBypassSkiaGoldFunctionality = "bypass-skia-gold-functionality";
  52. const char* kDryRun = "dryrun";
  53. // The switch key for saving png file locally for debugging. This will allow
  54. // the framework to save the screenshot png file to this path.
  55. const char* kPngFilePathDebugging = "skia-gold-local-png-write-directory";
  56. namespace {
  57. base::FilePath GetAbsoluteSrcRelativePath(base::FilePath::StringType path) {
  58. base::FilePath root_path;
  59. base::PathService::Get(base::BasePathKey::DIR_SOURCE_ROOT, &root_path);
  60. return base::MakeAbsoluteFilePath(root_path.Append(path));
  61. }
  62. // Append args after program.
  63. // The base::Commandline.AppendArg append the arg at
  64. // the end which doesn't work for us.
  65. void AppendArgsJustAfterProgram(base::CommandLine& cmd,
  66. base::CommandLine::StringVector args) {
  67. base::CommandLine::StringVector& argv =
  68. const_cast<base::CommandLine::StringVector&>(cmd.argv());
  69. argv.insert(argv.begin() + 1, args.begin(), args.end());
  70. }
  71. void FillInSystemEnvironment(base::Value::Dict& ds) {
  72. std::string processor = "unknown";
  73. #if defined(ARCH_CPU_X86)
  74. processor = "x86";
  75. #elif defined(ARCH_CPU_X86_64)
  76. processor = "x86_64";
  77. #else
  78. LOG(WARNING) << "Unknown Processor.";
  79. #endif
  80. ds.Set("system", SkiaGoldPixelDiff::GetPlatform());
  81. ds.Set("processor", processor);
  82. }
  83. // Fill in test environment to the keys_file. The format is json.
  84. // We need the system information to determine whether a new screenshot
  85. // is good or not. All the information that can affect the output of pixels
  86. // should be filled in. Eg: operating system, graphics card, processor
  87. // architecture, screen resolution, etc.
  88. bool FillInTestEnvironment(const base::FilePath& keys_file) {
  89. base::Value::Dict ds;
  90. FillInSystemEnvironment(ds);
  91. base::Value root(std::move(ds));
  92. std::string content;
  93. base::JSONWriter::Write(root, &content);
  94. base::ScopedAllowBlockingForTesting allow_blocking;
  95. base::File file(keys_file, base::File::Flags::FLAG_CREATE_ALWAYS |
  96. base::File::Flags::FLAG_WRITE);
  97. int ret_code = file.Write(0, content.c_str(), content.size());
  98. file.Close();
  99. if (ret_code <= 0) {
  100. LOG(ERROR) << "Writing the keys file to temporary file failed."
  101. << "File path:" << keys_file.AsUTF8Unsafe()
  102. << ". Return code: " << ret_code;
  103. return false;
  104. }
  105. return true;
  106. }
  107. bool BotModeEnabled(const base::CommandLine* command_line) {
  108. std::unique_ptr<base::Environment> env(base::Environment::Create());
  109. return command_line->HasSwitch(switches::kTestLauncherBotMode) ||
  110. env->HasVar("CHROMIUM_TEST_LAUNCHER_BOT_MODE");
  111. }
  112. // Returns true if it's running on CQ under 'without patch'. Otherwise
  113. // returns false.
  114. // The implementation is a bit hacky because there's no good indicator.
  115. bool IsTryjobWithoutPatch(const base::CommandLine* command_line) {
  116. return BotModeEnabled(command_line) &&
  117. command_line->HasSwitch(switches::kTestLauncherBatchLimit);
  118. }
  119. } // namespace
  120. SkiaGoldPixelDiff::SkiaGoldPixelDiff() = default;
  121. SkiaGoldPixelDiff::~SkiaGoldPixelDiff() = default;
  122. // static
  123. std::string SkiaGoldPixelDiff::GetPlatform() {
  124. #if BUILDFLAG(IS_WIN)
  125. return "windows";
  126. #elif BUILDFLAG(IS_APPLE)
  127. return "macOS";
  128. // TODO(crbug.com/1052397): Revisit the macro expression once build flag switch
  129. // of lacros-chrome is complete.
  130. #elif BUILDFLAG(IS_LINUX)
  131. return "linux";
  132. #elif BUILDFLAG(IS_CHROMEOS_LACROS)
  133. return "lacros";
  134. #elif BUILDFLAG(IS_CHROMEOS_ASH)
  135. return "ash";
  136. #endif
  137. }
  138. int SkiaGoldPixelDiff::LaunchProcess(const base::CommandLine& cmdline) const {
  139. base::Process sub_process =
  140. base::LaunchProcess(cmdline, base::LaunchOptionsForTest());
  141. int exit_code = 0;
  142. if (!sub_process.WaitForExit(&exit_code)) {
  143. ADD_FAILURE() << "Failed to wait for process.";
  144. // Return a non zero code indicating an error.
  145. return 1;
  146. }
  147. return exit_code;
  148. }
  149. void SkiaGoldPixelDiff::InitSkiaGold() {
  150. if (base::CommandLine::ForCurrentProcess()->HasSwitch(
  151. kBypassSkiaGoldFunctionality)) {
  152. LOG(WARNING) << "Bypassing Skia Gold initialization due to "
  153. << "--bypass-skia-gold-functionality being present.";
  154. return;
  155. }
  156. base::ScopedAllowBlockingForTesting allow_blocking;
  157. base::CommandLine cmd(GetAbsoluteSrcRelativePath(kSkiaGoldCtl));
  158. cmd.AppendSwitchPath("work-dir", working_dir_);
  159. if (luci_auth_) {
  160. cmd.AppendArg("--luci");
  161. }
  162. AppendArgsJustAfterProgram(cmd, {FILE_PATH_LITERAL("auth")});
  163. base::CommandLine::StringType cmd_str = cmd.GetCommandLineString();
  164. LOG(INFO) << "Skia Gold Auth Commandline: " << cmd_str;
  165. int exit_code = LaunchProcess(cmd);
  166. ASSERT_EQ(exit_code, 0);
  167. base::FilePath json_temp_file =
  168. working_dir_.Append(FILE_PATH_LITERAL("keys_file.txt"));
  169. FillInTestEnvironment(json_temp_file);
  170. base::FilePath failure_temp_file =
  171. working_dir_.Append(FILE_PATH_LITERAL("failure.log"));
  172. cmd = base::CommandLine(GetAbsoluteSrcRelativePath(kSkiaGoldCtl));
  173. cmd.AppendSwitchASCII("instance", kSkiaGoldInstance);
  174. cmd.AppendSwitchPath("work-dir", working_dir_);
  175. cmd.AppendSwitchPath("keys-file", json_temp_file);
  176. cmd.AppendSwitchPath("failure-file", failure_temp_file);
  177. cmd.AppendSwitch("passfail");
  178. cmd.AppendSwitchASCII("commit", build_revision_);
  179. // This handles the logic for tryjob.
  180. if (issue_.length()) {
  181. cmd.AppendSwitchASCII("issue", issue_);
  182. cmd.AppendSwitchASCII("patchset", patchset_);
  183. cmd.AppendSwitchASCII("jobid", job_id_);
  184. cmd.AppendSwitchASCII("crs", code_review_system_);
  185. cmd.AppendSwitchASCII("cis", "buildbucket");
  186. }
  187. AppendArgsJustAfterProgram(
  188. cmd, {FILE_PATH_LITERAL("imgtest"), FILE_PATH_LITERAL("init")});
  189. cmd_str = cmd.GetCommandLineString();
  190. LOG(INFO) << "Skia Gold imgtest init Commandline: " << cmd_str;
  191. exit_code = LaunchProcess(cmd);
  192. ASSERT_EQ(exit_code, 0);
  193. }
  194. void SkiaGoldPixelDiff::Init(const std::string& screenshot_prefix,
  195. const std::string& corpus) {
  196. auto* cmd_line = base::CommandLine::ForCurrentProcess();
  197. ASSERT_TRUE(cmd_line->HasSwitch(kBuildRevisionKey))
  198. << "Missing switch " << kBuildRevisionKey;
  199. ASSERT_TRUE(
  200. cmd_line->HasSwitch(kIssueKey) && cmd_line->HasSwitch(kPatchSetKey) &&
  201. cmd_line->HasSwitch(kJobIdKey) ||
  202. !cmd_line->HasSwitch(kIssueKey) && !cmd_line->HasSwitch(kPatchSetKey) &&
  203. !cmd_line->HasSwitch(kJobIdKey))
  204. << "Missing switch. If it's running for tryjob, you should pass --"
  205. << kIssueKey << " --" << kPatchSetKey << " --" << kJobIdKey
  206. << ". Otherwise, do not pass any one of them.";
  207. build_revision_ = cmd_line->GetSwitchValueASCII(kBuildRevisionKey);
  208. if (cmd_line->HasSwitch(kIssueKey)) {
  209. issue_ = cmd_line->GetSwitchValueASCII(kIssueKey);
  210. patchset_ = cmd_line->GetSwitchValueASCII(kPatchSetKey);
  211. job_id_ = cmd_line->GetSwitchValueASCII(kJobIdKey);
  212. code_review_system_ = cmd_line->GetSwitchValueASCII(kCodeReviewSystemKey);
  213. if (code_review_system_.empty()) {
  214. code_review_system_ = "gerrit";
  215. }
  216. }
  217. if (cmd_line->HasSwitch(kNoLuciAuth) || !BotModeEnabled(cmd_line)) {
  218. luci_auth_ = false;
  219. }
  220. initialized_ = true;
  221. prefix_ = screenshot_prefix;
  222. corpus_ = corpus.length() ? corpus : "gtest-pixeltests";
  223. base::ScopedAllowBlockingForTesting allow_blocking;
  224. base::CreateNewTempDirectory(FILE_PATH_LITERAL("SkiaGoldTemp"),
  225. &working_dir_);
  226. InitSkiaGold();
  227. }
  228. bool SkiaGoldPixelDiff::UploadToSkiaGoldServer(
  229. const base::FilePath& local_file_path,
  230. const std::string& remote_golden_image_name,
  231. const SkiaGoldMatchingAlgorithm* algorithm) const {
  232. if (base::CommandLine::ForCurrentProcess()->HasSwitch(
  233. kBypassSkiaGoldFunctionality)) {
  234. LOG(WARNING) << "Bypassing Skia Gold comparison due to "
  235. << "--bypass-skia-gold-functionality being present.";
  236. return true;
  237. }
  238. // Copy the png file to another place for local debugging.
  239. if (base::CommandLine::ForCurrentProcess()->HasSwitch(
  240. kPngFilePathDebugging)) {
  241. base::FilePath path =
  242. base::CommandLine::ForCurrentProcess()->GetSwitchValuePath(
  243. kPngFilePathDebugging);
  244. if (!base::PathExists(path)) {
  245. base::CreateDirectory(path);
  246. }
  247. base::FilePath filepath;
  248. if (remote_golden_image_name.length() <= 4 ||
  249. (remote_golden_image_name.length() > 4 &&
  250. remote_golden_image_name.substr(remote_golden_image_name.length() -
  251. 4) != ".png")) {
  252. filepath = path.AppendASCII(remote_golden_image_name + ".png");
  253. } else {
  254. filepath = path.AppendASCII(remote_golden_image_name);
  255. }
  256. base::CopyFile(local_file_path, filepath);
  257. }
  258. base::ScopedAllowBlockingForTesting allow_blocking;
  259. base::CommandLine cmd(GetAbsoluteSrcRelativePath(kSkiaGoldCtl));
  260. cmd.AppendSwitchASCII("test-name", remote_golden_image_name);
  261. cmd.AppendSwitchASCII("corpus", corpus_);
  262. cmd.AppendSwitchPath("png-file", local_file_path);
  263. cmd.AppendSwitchPath("work-dir", working_dir_);
  264. if (!BotModeEnabled(base::CommandLine::ForCurrentProcess())) {
  265. cmd.AppendSwitch(kDryRun);
  266. }
  267. // For CQ, if a Skia Gold gtest fails, then swarming runs the suite
  268. // without patch, and the test succeed. The success job will override
  269. // the failed job, and the failed test will not show on the triage dashboard.
  270. // To resolve this, we use dryrun mode for 'run without patch'.
  271. if (IsTryjobWithoutPatch(base::CommandLine::ForCurrentProcess())) {
  272. cmd.AppendSwitch(kDryRun);
  273. }
  274. if (algorithm)
  275. algorithm->AppendAlgorithmToCmdline(cmd);
  276. AppendArgsJustAfterProgram(
  277. cmd, {FILE_PATH_LITERAL("imgtest"), FILE_PATH_LITERAL("add")});
  278. base::CommandLine::StringType cmd_str = cmd.GetCommandLineString();
  279. LOG(INFO) << "Skia Gold Commandline: " << cmd_str;
  280. int exit_code = LaunchProcess(cmd);
  281. return exit_code == 0;
  282. }
  283. bool SkiaGoldPixelDiff::CompareScreenshot(
  284. const std::string& screenshot_name,
  285. const SkBitmap& bitmap,
  286. const SkiaGoldMatchingAlgorithm* algorithm) const {
  287. DCHECK(Initialized()) << "Initialize the class before using this method.";
  288. std::vector<unsigned char> output;
  289. bool ret = gfx::PNGCodec::EncodeBGRASkBitmap(bitmap, true, &output);
  290. if (!ret) {
  291. LOG(ERROR) << "Encoding SkBitmap to PNG format failed.";
  292. return false;
  293. }
  294. // The golden image name should be unique on GCS per platform. And also the
  295. // name should be valid across all systems.
  296. std::string suffix = GetPlatform();
  297. std::string normalized_screenshot_name;
  298. // Parameterized tests have "/" in their names which isn't allowed in file
  299. // names. Replace with "_".
  300. base::ReplaceChars(screenshot_name, "/", "_", &normalized_screenshot_name);
  301. std::string name = prefix_ + "_" + normalized_screenshot_name + "_" + suffix;
  302. base::ScopedAllowBlockingForTesting allow_blocking;
  303. base::FilePath temporary_path =
  304. working_dir_.Append(base::FilePath::FromUTF8Unsafe(name + ".png"));
  305. base::File file(temporary_path, base::File::Flags::FLAG_CREATE_ALWAYS |
  306. base::File::Flags::FLAG_WRITE);
  307. int ret_code = file.Write(0, (char*)output.data(), output.size());
  308. file.Close();
  309. if (ret_code <= 0) {
  310. LOG(ERROR) << "Writing the PNG image to temporary file failed."
  311. << "File path:" << temporary_path.AsUTF8Unsafe()
  312. << ". Return code: " << ret_code;
  313. return false;
  314. }
  315. return UploadToSkiaGoldServer(temporary_path, name, algorithm);
  316. }
  317. } // namespace test
  318. } // namespace ui