baseline_optimizer.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700
  1. # Copyright (C) 2011, Google Inc. All rights reserved.
  2. #
  3. # Redistribution and use in source and binary forms, with or without
  4. # modification, are permitted provided that the following conditions are
  5. # met:
  6. #
  7. # * Redistributions of source code must retain the above copyright
  8. # notice, this list of conditions and the following disclaimer.
  9. # * Redistributions in binary form must reproduce the above
  10. # copyright notice, this list of conditions and the following disclaimer
  11. # in the documentation and/or other materials provided with the
  12. # distribution.
  13. # * Neither the name of Google Inc. nor the names of its
  14. # contributors may be used to endorse or promote products derived from
  15. # this software without specific prior written permission.
  16. #
  17. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28. import copy
  29. import logging
  30. from blinkpy.common.memoized import memoized
  31. from blinkpy.web_tests.models.testharness_results import is_all_pass_testharness_result
  32. _log = logging.getLogger(__name__)
  33. class BaselineOptimizer(object):
  34. def __init__(self, host, default_port, port_names):
  35. self._filesystem = host.filesystem
  36. self._default_port = default_port
  37. # To ensure the flag-specific baselines join the fallback graph in the
  38. # same location each time the optimizer runs, we create a separate port.
  39. self._flag_spec_port = self._make_flag_spec_port(host, default_port)
  40. self._ports = {}
  41. for port_name in port_names:
  42. self._ports[port_name] = host.port_factory.get(port_name)
  43. self._web_tests_dir = default_port.web_tests_dir()
  44. self._generic_baselines_dir = default_port.generic_baselines_dir()
  45. self._parent_of_tests = self._filesystem.dirname(self._web_tests_dir)
  46. self._web_tests_dir_name = self._filesystem.relpath(
  47. self._web_tests_dir, self._parent_of_tests)
  48. # Only used by unit tests.
  49. self.new_results_by_directory = []
  50. def _make_flag_spec_port(self, host, default_port):
  51. option = default_port.flag_specific_config_name()
  52. if not option:
  53. return None
  54. port_name = host.builders.port_name_for_flag_specific_option(option)
  55. port = host.port_factory.get(port_name)
  56. port.set_option_default('flag_specific', option)
  57. return port
  58. def optimize(self, test_name, suffix):
  59. # A visualization of baseline fallback:
  60. # https://docs.google.com/drawings/d/13l3IUlSE99RoKjDwEWuY1O77simAhhF6Wi0fZdkSaMA/
  61. # The full document with more details:
  62. # https://chromium.googlesource.com/chromium/src/+/main/docs/testing/web_test_baseline_fallback.md
  63. # The virtual and non-virtual subtrees are identical, with the virtual
  64. # root being the special node having multiple parents and connecting the
  65. # two trees. We patch the virtual subtree to cut its dependencies on the
  66. # non-virtual one and optimze the two independently. Finally, we treat
  67. # the virtual subtree specially to remove any duplication between the
  68. # two subtrees.
  69. # For CLI compatibility, "suffix" is an extension without the leading
  70. # dot. Yet we use dotted extension everywhere else in the codebase.
  71. # TODO(robertma): Investigate changing the CLI.
  72. assert not suffix.startswith('.')
  73. extension = '.' + suffix
  74. succeeded = True
  75. if self._flag_spec_port:
  76. self._optimize_flag_specific_baselines(test_name, extension)
  77. return True
  78. baseline_name = self._default_port.output_filename(
  79. test_name, self._default_port.BASELINE_SUFFIX, extension)
  80. non_virtual_test_name = self._virtual_test_base(test_name)
  81. if non_virtual_test_name:
  82. # The test belongs to a virtual suite.
  83. _log.debug('Optimizing virtual fallback path.')
  84. self._patch_virtual_subtree(test_name, extension, baseline_name)
  85. succeeded &= self._optimize_subtree(test_name, baseline_name)
  86. # Update the non-virtual baseline name
  87. non_virtual_baseline_name = self._default_port.output_filename(
  88. non_virtual_test_name, self._default_port.BASELINE_SUFFIX,
  89. extension)
  90. self._optimize_virtual_root(test_name, extension, baseline_name,
  91. non_virtual_baseline_name)
  92. else:
  93. # The given baseline is already non-virtual.
  94. non_virtual_baseline_name = baseline_name
  95. _log.debug('Optimizing non-virtual fallback path.')
  96. succeeded &= self._optimize_subtree(test_name,
  97. non_virtual_baseline_name)
  98. self._remove_extra_result_at_root(test_name, non_virtual_baseline_name)
  99. if not succeeded:
  100. _log.error('Heuristics failed to optimize %s', baseline_name)
  101. return succeeded
  102. def _optimize_flag_specific_baselines(self, test_name, extension):
  103. """Optimize flag-specific baselines."""
  104. flag_specific = self._flag_spec_port.flag_specific_config_name()
  105. non_virtual_test_name = self._virtual_test_base(test_name)
  106. if non_virtual_test_name:
  107. _log.debug(
  108. 'Optimizing flag-specific virtual fallback path '
  109. 'for "%s".', flag_specific)
  110. self._optimize_single_baseline(test_name, extension,
  111. self._flag_spec_port)
  112. else:
  113. non_virtual_test_name = test_name
  114. _log.debug(
  115. 'Optimizing flag-specific non-virtual fallback path '
  116. 'for "%s".', flag_specific)
  117. self._optimize_single_baseline(non_virtual_test_name, extension,
  118. self._flag_spec_port)
  119. def _get_baseline_paths(self, test_name, extension, port):
  120. """Get paths to baselines that the provided port would search.
  121. Returns:
  122. list[str]: A list of absolute paths (symbolically generated, may
  123. not actually exist on disk).
  124. """
  125. baselines = port.expected_baselines(test_name,
  126. extension,
  127. all_baselines=True)
  128. non_virtual_test_name = self._virtual_test_base(test_name)
  129. if non_virtual_test_name:
  130. baselines.extend(
  131. port.expected_baselines(non_virtual_test_name,
  132. extension,
  133. all_baselines=True))
  134. # `baseline_dir` is `None` when the search path is empty and the generic
  135. # baseline is also missing.
  136. baseline_paths = [
  137. self._filesystem.join(baseline_dir, baseline_filename)
  138. for baseline_dir, baseline_filename in baselines if baseline_dir
  139. ]
  140. return baseline_paths
  141. def _optimize_single_baseline(self, test_name, extension, port):
  142. """Optimize a baseline directly by simulating the fallback algorithm."""
  143. baseline_paths = self._get_baseline_paths(test_name, extension, port)
  144. if not baseline_paths:
  145. # The baseline for this test does not exist.
  146. return
  147. baseline_to_optimize = baseline_paths[0]
  148. basename = self._filesystem.basename(baseline_to_optimize)
  149. if len(baseline_paths) < 2:
  150. _log.debug(' %s: (no baselines found)', basename)
  151. return
  152. fallback_baseline = baseline_paths[1]
  153. is_reftest = self._is_reftest(test_name)
  154. target_digest = ResultDigest(self._filesystem, baseline_to_optimize,
  155. is_reftest)
  156. fallback_digest = ResultDigest(self._filesystem, fallback_baseline,
  157. is_reftest)
  158. if target_digest == fallback_digest:
  159. _log.debug(' %s:', basename)
  160. _log.debug(' Deleting (file system): %s', baseline_to_optimize)
  161. self._filesystem.remove(baseline_to_optimize)
  162. else:
  163. _log.debug(' %s: (already optimal)', basename)
  164. def write_by_directory(self, results_by_directory, writer, indent):
  165. """Logs results_by_directory in a pretty format."""
  166. for path in sorted(results_by_directory):
  167. writer('%s%s: %s' % (indent, self._platform(path),
  168. results_by_directory[path]))
  169. def read_results_by_directory(self, test_name, baseline_name):
  170. """Reads the baselines with the given file name in all directories.
  171. Returns:
  172. A dict from directory names to the digest of file content.
  173. """
  174. results_by_directory = {}
  175. directories = set()
  176. for port in self._ports.values():
  177. directories.update(set(self._relative_baseline_search_path(port)))
  178. for directory in directories:
  179. path = self._join_directory(directory, baseline_name)
  180. if self._filesystem.exists(path):
  181. results_by_directory[directory] = ResultDigest(
  182. self._filesystem, path, self._is_reftest(test_name))
  183. return results_by_directory
  184. def _is_reftest(self, test_name):
  185. return bool(self._default_port.reference_files(test_name))
  186. def _optimize_subtree(self, test_name, baseline_name):
  187. basename = self._filesystem.basename(baseline_name)
  188. results_by_directory, new_results_by_directory = self._find_optimal_result_placement(
  189. test_name, baseline_name)
  190. if new_results_by_directory == results_by_directory:
  191. if new_results_by_directory:
  192. _log.debug(' %s: (already optimal)', basename)
  193. self.write_by_directory(results_by_directory, _log.debug,
  194. ' ')
  195. else:
  196. _log.debug(' %s: (no baselines found)', basename)
  197. # This is just used for unit tests.
  198. # Intentionally set it to the old data if we don't modify anything.
  199. self.new_results_by_directory.append(results_by_directory)
  200. return True
  201. # Check if the results before and after optimization are equivalent.
  202. if (self._results_by_port_name(results_by_directory) !=
  203. self._results_by_port_name(new_results_by_directory)):
  204. # This really should never happen. Just a sanity check to make
  205. # sure the script fails in the case of bugs instead of committing
  206. # incorrect baselines.
  207. _log.error(' %s: optimization failed', basename)
  208. self.write_by_directory(results_by_directory, _log.warning,
  209. ' ')
  210. return False
  211. _log.debug(' %s:', basename)
  212. _log.debug(' Before: ')
  213. self.write_by_directory(results_by_directory, _log.debug, ' ')
  214. _log.debug(' After: ')
  215. self.write_by_directory(new_results_by_directory, _log.debug, ' ')
  216. self._move_baselines(baseline_name, results_by_directory,
  217. new_results_by_directory)
  218. return True
  219. def _move_baselines(self, baseline_name, results_by_directory,
  220. new_results_by_directory):
  221. data_for_result = {}
  222. for directory, result in results_by_directory.items():
  223. if str(result) not in data_for_result:
  224. source = self._join_directory(directory, baseline_name)
  225. data_for_result[str(
  226. result)] = self._filesystem.read_binary_file(source)
  227. fs_files = []
  228. for directory, result in results_by_directory.items():
  229. if new_results_by_directory.get(directory) != result:
  230. file_name = self._join_directory(directory, baseline_name)
  231. if self._filesystem.exists(file_name):
  232. fs_files.append(file_name)
  233. if fs_files:
  234. _log.debug(' Deleting (file system):')
  235. for platform_dir in sorted(
  236. self._platform(filename) for filename in fs_files):
  237. _log.debug(' ' + platform_dir)
  238. for filename in fs_files:
  239. self._filesystem.remove(filename)
  240. else:
  241. _log.debug(' (Nothing to delete)')
  242. file_names = []
  243. for directory, result in new_results_by_directory.items():
  244. if results_by_directory.get(directory) != result:
  245. destination = self._join_directory(directory, baseline_name)
  246. self._filesystem.maybe_make_directory(
  247. self._filesystem.split(destination)[0])
  248. self._filesystem.write_binary_file(
  249. destination, data_for_result[result.__str__()])
  250. file_names.append(destination)
  251. if file_names:
  252. _log.debug(' Adding:')
  253. for platform_dir in sorted(
  254. self._platform(filename) for filename in file_names):
  255. _log.debug(' ' + platform_dir)
  256. else:
  257. _log.debug(' (Nothing to add)')
  258. def _platform(self, filename):
  259. """Guesses the platform from a path (absolute or relative).
  260. Returns:
  261. The platform name, or '(generic)' if unable to make a guess.
  262. """
  263. platform_dir = self._web_tests_dir_name + self._filesystem.sep + 'platform' + self._filesystem.sep
  264. if filename.startswith(platform_dir):
  265. return filename.replace(platform_dir,
  266. '').split(self._filesystem.sep)[0]
  267. platform_dir = self._filesystem.join(self._parent_of_tests,
  268. platform_dir)
  269. if filename.startswith(platform_dir):
  270. return filename.replace(platform_dir,
  271. '').split(self._filesystem.sep)[0]
  272. return '(generic)'
  273. def _port_from_baseline_dir(self, baseline_dir):
  274. """Returns a Port object from the given baseline directory."""
  275. baseline_dir = self._filesystem.basename(baseline_dir)
  276. for port in self._ports.values():
  277. if self._filesystem.basename(
  278. port.baseline_version_dir()) == baseline_dir:
  279. return port
  280. raise Exception(
  281. 'Failed to find port for primary baseline %s.' % baseline_dir)
  282. def _walk_immediate_predecessors_of_virtual_root(
  283. self, test_name, extension, baseline_name, worker_func):
  284. """Maps a function onto each immediate predecessor of the virtual root.
  285. For each immediate predecessor, we call
  286. worker_func(virtual_baseline, non_virtual_fallback)
  287. where the two arguments are the absolute paths to the virtual platform
  288. baseline and the non-virtual fallback respectively.
  289. """
  290. actual_test_name = self._virtual_test_base(test_name)
  291. assert actual_test_name, '%s is not a virtual test.' % test_name
  292. for directory in self._directories_immediately_preceding_root():
  293. port = self._port_from_baseline_dir(directory)
  294. virtual_baseline = self._join_directory(directory, baseline_name)
  295. # return_default=False mandates expected_filename() to return None
  296. # instead of a non-existing generic path when nothing is found.
  297. non_virtual_fallback = port.expected_filename(
  298. actual_test_name, extension, return_default=False)
  299. if not non_virtual_fallback:
  300. # Unable to find a non-virtual fallback baseline, skipping.
  301. continue
  302. worker_func(virtual_baseline, non_virtual_fallback)
  303. def _patch_virtual_subtree(self, test_name, extension, baseline_name):
  304. # Ensure all immediate predecessors of the root have a baseline for this
  305. # virtual suite so that the virtual subtree can be treated completely
  306. # independently. If an immediate predecessor is missing a baseline, find
  307. # its non-virtual fallback and copy over.
  308. _log.debug(
  309. 'Copying non-virtual baselines to the virtual subtree to make it independent.'
  310. )
  311. virtual_root_baseline_path = self._filesystem.join(
  312. self._generic_baselines_dir, baseline_name)
  313. if self._filesystem.exists(virtual_root_baseline_path):
  314. return
  315. def patcher(virtual_baseline, non_virtual_fallback):
  316. if not self._filesystem.exists(virtual_baseline):
  317. _log.debug(' Copying (file system): %s -> %s.',
  318. non_virtual_fallback, virtual_baseline)
  319. self._filesystem.maybe_make_directory(
  320. self._filesystem.split(virtual_baseline)[0])
  321. self._filesystem.copyfile(non_virtual_fallback,
  322. virtual_baseline)
  323. self._walk_immediate_predecessors_of_virtual_root(
  324. test_name, extension, baseline_name, patcher)
  325. def _optimize_virtual_root(self, test_name, extension, baseline_name,
  326. non_virtual_baseline_name):
  327. virtual_root_baseline_path = self._filesystem.join(
  328. self._generic_baselines_dir, baseline_name)
  329. if self._filesystem.exists(virtual_root_baseline_path):
  330. _log.debug(
  331. 'Virtual root baseline found. Checking if we can remove it.')
  332. self._try_to_remove_virtual_root(test_name,
  333. non_virtual_baseline_name,
  334. virtual_root_baseline_path)
  335. else:
  336. _log.debug(
  337. 'Virtual root baseline not found. Searching for virtual baselines redundant with non-virtual ones.'
  338. )
  339. self._unpatch_virtual_subtree(test_name, extension, baseline_name)
  340. def _try_to_remove_virtual_root(self, test_name, non_virtual_baseline_name,
  341. virtual_root_baseline_path):
  342. # See if all the successors of the virtual root (i.e. all non-virtual
  343. # platforms) have the same baseline as the virtual root. If so, the
  344. # virtual root is redundant and can be safely removed.
  345. virtual_root_digest = ResultDigest(self._filesystem,
  346. virtual_root_baseline_path,
  347. self._is_reftest(test_name))
  348. # Read the base (non-virtual) results.
  349. results_by_directory = self.read_results_by_directory(
  350. test_name, non_virtual_baseline_name)
  351. results_by_port_name = self._results_by_port_name(results_by_directory)
  352. for port_name in self._ports.keys():
  353. assert port_name in results_by_port_name
  354. if results_by_port_name[port_name] != virtual_root_digest:
  355. return
  356. _log.debug('Deleting redundant virtual root baseline.')
  357. _log.debug(' Deleting (file system): ' + virtual_root_baseline_path)
  358. self._filesystem.remove(virtual_root_baseline_path)
  359. def _unpatch_virtual_subtree(self, test_name, extension, baseline_name):
  360. # Check all immediate predecessors of the virtual root and delete those
  361. # duplicate with their non-virtual fallback, essentially undoing some
  362. # of the work done in _patch_virtual_subtree.
  363. is_reftest = self._is_reftest(test_name)
  364. def unpatcher(virtual_baseline, non_virtual_fallback):
  365. if self._filesystem.exists(virtual_baseline) and \
  366. (ResultDigest(self._filesystem, virtual_baseline, is_reftest) ==
  367. ResultDigest(self._filesystem, non_virtual_fallback, is_reftest)):
  368. _log.debug(
  369. ' Deleting (file system): %s (redundant with %s).',
  370. virtual_baseline, non_virtual_fallback)
  371. self._filesystem.remove(virtual_baseline)
  372. self._walk_immediate_predecessors_of_virtual_root(
  373. test_name, extension, baseline_name, unpatcher)
  374. def _baseline_root(self):
  375. """Returns the name of the root (generic) baseline directory."""
  376. return self._filesystem.join(self._web_tests_dir_name, "platform", "generic")
  377. def _baseline_search_path(self, port):
  378. """Returns the baseline search path (a list of absolute paths) of the
  379. given port."""
  380. return port.baseline_search_path()
  381. @memoized
  382. def _relative_baseline_search_path(self, port):
  383. """Returns a list of paths to check for baselines in order.
  384. The generic baseline path is appended to the list. All paths are
  385. relative to the parent of the test directory.
  386. """
  387. baseline_search_path = self._baseline_search_path(port)
  388. relative_paths = [
  389. self._filesystem.relpath(path, self._parent_of_tests)
  390. for path in baseline_search_path
  391. ]
  392. relative_baseline_root = self._baseline_root()
  393. return relative_paths + [relative_baseline_root]
  394. def _virtual_test_base(self, test_name):
  395. """Returns the base (non-virtual) version of test_name, or None if
  396. test_name is not virtual."""
  397. # This function should only accept a test name. Use baseline won't work
  398. # because some bases in VirtualTestSuites are full test name which has
  399. # .html as extension.
  400. return self._default_port.lookup_virtual_test_base(test_name)
  401. def _join_directory(self, directory, baseline_name):
  402. """Returns the absolute path to the baseline in the given directory."""
  403. return self._filesystem.join(self._parent_of_tests, directory,
  404. baseline_name)
  405. def _results_by_port_name(self, results_by_directory):
  406. """Transforms a by-directory result dict to by-port-name.
  407. The method mimicks the baseline search behaviour, i.e. results[port] is
  408. the first baseline found on the baseline search path of the port. If no
  409. baseline is found on the search path, the test is assumed to be an all-
  410. PASS testharness.js test.
  411. Args:
  412. results_by_directory: A dictionary returned by read_results_by_directory().
  413. Returns:
  414. A dictionary mapping port names to their baselines.
  415. """
  416. results_by_port_name = {}
  417. for port_name, port in self._ports.items():
  418. for directory in self._relative_baseline_search_path(port):
  419. if directory in results_by_directory:
  420. results_by_port_name[port_name] = results_by_directory[
  421. directory]
  422. break
  423. if port_name not in results_by_port_name:
  424. # Implicit extra result.
  425. results_by_port_name[port_name] = ResultDigest(None, None)
  426. return results_by_port_name
  427. @memoized
  428. def _directories_immediately_preceding_root(self):
  429. """Returns a list of directories immediately preceding the root on
  430. search paths."""
  431. directories = set()
  432. for port in self._ports.values():
  433. directory = self._filesystem.relpath(
  434. self._baseline_search_path(port)[-1], self._parent_of_tests)
  435. directories.add(directory)
  436. return frozenset(directories)
  437. def _optimize_result_for_root(self, new_results_by_directory):
  438. # The root directory (i.e. web_tests) is the only one not
  439. # corresponding to a specific platform. As such, baselines in
  440. # directories that immediately precede the root on search paths may
  441. # be promoted up if they are all the same.
  442. # Example: if win and mac have the same baselines, then they can be
  443. # promoted up to be the root baseline.
  444. # All other baselines can only be removed if they're redundant with a
  445. # baseline later on the search path. They can never be promoted up.
  446. immediately_preceding_root = self._directories_immediately_preceding_root(
  447. )
  448. shared_result = None
  449. root_baseline_unused = False
  450. for directory in immediately_preceding_root:
  451. this_result = new_results_by_directory.get(directory)
  452. # If any of these directories don't have a baseline, there's no optimization we can do.
  453. if not this_result:
  454. return
  455. if not shared_result:
  456. shared_result = this_result
  457. elif shared_result != this_result:
  458. root_baseline_unused = True
  459. baseline_root = self._baseline_root()
  460. # The root baseline is unused if all the directories immediately preceding the root
  461. # have a baseline, but have different baselines, so the baselines can't be promoted up.
  462. if root_baseline_unused:
  463. if baseline_root in new_results_by_directory:
  464. del new_results_by_directory[baseline_root]
  465. return
  466. new_results_by_directory[baseline_root] = shared_result
  467. for directory in immediately_preceding_root:
  468. del new_results_by_directory[directory]
  469. def _find_optimal_result_placement(self, test_name, baseline_name):
  470. results_by_directory = self.read_results_by_directory(
  471. test_name, baseline_name)
  472. results_by_port_name = self._results_by_port_name(results_by_directory)
  473. new_results_by_directory = self._remove_redundant_results(
  474. results_by_directory, results_by_port_name)
  475. self._optimize_result_for_root(new_results_by_directory)
  476. return results_by_directory, new_results_by_directory
  477. def _remove_redundant_results(self, results_by_directory,
  478. results_by_port_name):
  479. # For every port, traverse its search path in the fallback order (from
  480. # specific to generic). Remove duplicate baselines until a different
  481. # baseline is found (or the root is reached), i.e., keep the most
  482. # generic one among duplicate baselines.
  483. new_results_by_directory = copy.copy(results_by_directory)
  484. for port_name, port in self._ports.items():
  485. current_result = results_by_port_name.get(port_name)
  486. # This happens if we're missing baselines for a port.
  487. if not current_result:
  488. continue
  489. search_path = self._relative_baseline_search_path(port)
  490. current_index, current_directory = self._find_in_search_path(
  491. search_path, current_result, new_results_by_directory)
  492. found_different_result = False
  493. for index in range(current_index + 1, len(search_path)):
  494. new_directory = search_path[index]
  495. if new_directory not in new_results_by_directory:
  496. # No baseline in this directory.
  497. continue
  498. elif new_results_by_directory[new_directory] == current_result:
  499. # The baseline in current_directory is redundant with the
  500. # baseline in new_directory which is later in the search
  501. # path. Remove the earlier one and point current to new.
  502. if current_directory in new_results_by_directory:
  503. del new_results_by_directory[current_directory]
  504. current_directory = new_directory
  505. else:
  506. # A different result is found, so stop.
  507. found_different_result = True
  508. break
  509. # If we did not find a different fallback and current_result is
  510. # an extra result, we can safely remove it.
  511. # Note that we do not remove the generic extra result here.
  512. # Roots (virtual and non-virtual) are treated specially later.
  513. if (not found_different_result and current_result.is_extra_result
  514. and current_directory != self._baseline_root()
  515. and current_directory in new_results_by_directory):
  516. del new_results_by_directory[current_directory]
  517. return new_results_by_directory
  518. def _find_in_search_path(self, search_path, current_result,
  519. results_by_directory):
  520. """Finds the index and the directory of a result on a search path."""
  521. for index, directory in enumerate(search_path):
  522. if (directory in results_by_directory
  523. and (results_by_directory[directory] == current_result)):
  524. return index, directory
  525. assert current_result.is_extra_result, (
  526. 'result %s not found in search path %s, %s' %
  527. (current_result, search_path, results_by_directory))
  528. # Implicit extra result at the root.
  529. return len(search_path) - 1, search_path[-1]
  530. def _remove_extra_result_at_root(self, test_name,
  531. non_virtual_baseline_name):
  532. """Removes extra result at the non-virtual root."""
  533. path = self._join_directory(self._baseline_root(),
  534. non_virtual_baseline_name)
  535. if (self._filesystem.exists(path)
  536. and ResultDigest(self._filesystem, path,
  537. self._is_reftest(test_name)).is_extra_result):
  538. _log.debug(
  539. 'Deleting extra baseline (empty, -expected.png for reftest, or all-PASS testharness JS result)'
  540. )
  541. _log.debug(' Deleting (file system): ' + path)
  542. self._filesystem.remove(path)
  543. class ResultDigest(object):
  544. """Digest of a result file for fast comparison.
  545. A result file can be any actual or expected output from a web test,
  546. including text and image. SHA1 is used internally to digest the file.
  547. """
  548. # A baseline is extra in the following cases:
  549. # 1. if the result is an all-PASS testharness result,
  550. # 2. if the result is empty,
  551. # 3. if the test is a reftest and the baseline is an -expected-png file.
  552. #
  553. # An extra baseline should be deleted if doesn't override the fallback
  554. # baseline. To check that, we compare the ResultDigests of the baseline and
  555. # the fallback baseline. If the baseline is not the root baseline and the
  556. # fallback baseline doesn't exist, we assume that the fallback baseline is
  557. # an *implicit extra result* which equals to any extra baselines, so that
  558. # the extra baseline will be treated as not overriding the fallback baseline
  559. # thus will be removed.
  560. _IMPLICIT_EXTRA_RESULT = '<EXTRA>'
  561. def __init__(self, fs, path, is_reftest=False):
  562. """Constructs the digest for a result.
  563. Args:
  564. fs: An instance of common.system.FileSystem.
  565. path: The path to a result file. If None is provided, the result is
  566. an *implicit* extra result.
  567. is_reftest: Whether the test is a reftest.
  568. """
  569. self.path = path
  570. if path is None:
  571. self.sha = self._IMPLICIT_EXTRA_RESULT
  572. self.is_extra_result = True
  573. return
  574. assert fs.exists(path), path + " does not exist"
  575. if path.endswith('.txt'):
  576. try:
  577. content = fs.read_text_file(path)
  578. self.is_extra_result = not content or is_all_pass_testharness_result(
  579. content)
  580. except UnicodeDecodeError as e:
  581. self.is_extra_result = False
  582. # Unfortunately, we may read the file twice, once in text mode
  583. # and once in binary mode.
  584. self.sha = fs.sha1(path)
  585. return
  586. if path.endswith('.png') and is_reftest:
  587. self.is_extra_result = True
  588. self.sha = ''
  589. return
  590. self.is_extra_result = not fs.read_binary_file(path)
  591. self.sha = fs.sha1(path)
  592. def __eq__(self, other):
  593. if other is None:
  594. return False
  595. # Implicit extra result is equal to any extra results.
  596. if self.sha == self._IMPLICIT_EXTRA_RESULT or other.sha == self._IMPLICIT_EXTRA_RESULT:
  597. return self.is_extra_result and other.is_extra_result
  598. return self.sha == other.sha
  599. # Python 2 does not automatically delegate __ne__ to not __eq__.
  600. def __ne__(self, other):
  601. return not self.__eq__(other)
  602. def __str__(self):
  603. return self.sha[0:7]
  604. def __repr__(self):
  605. is_extra_result = ' EXTRA' if self.is_extra_result else ''
  606. return '<ResultDigest %s%s %s>' % (self.sha, is_extra_result,
  607. self.path)