expectations.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562
  1. # Copyright 2020 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. """Methods related to test expectations/expectation files."""
  5. from __future__ import print_function
  6. import collections
  7. import datetime
  8. import logging
  9. import os
  10. import re
  11. import subprocess
  12. import sys
  13. from typing import Iterable, List, Optional, Set, Tuple, Union
  14. import six
  15. from typ import expectations_parser
  16. from unexpected_passes_common import data_types
  17. from unexpected_passes_common import result_output
  18. FINDER_DISABLE_COMMENT_BASE = 'finder:disable'
  19. FINDER_ENABLE_COMMENT_BASE = 'finder:enable'
  20. FINDER_COMMENT_SUFFIX_GENERAL = '-general'
  21. FINDER_COMMENT_SUFFIX_STALE = '-stale'
  22. FINDER_COMMENT_SUFFIX_UNUSED = '-unused'
  23. ALL_FINDER_SUFFIXES = frozenset([
  24. FINDER_COMMENT_SUFFIX_GENERAL,
  25. FINDER_COMMENT_SUFFIX_STALE,
  26. FINDER_COMMENT_SUFFIX_UNUSED,
  27. ])
  28. FINDER_DISABLE_COMMENT_GENERAL = (FINDER_DISABLE_COMMENT_BASE +
  29. FINDER_COMMENT_SUFFIX_GENERAL)
  30. FINDER_DISABLE_COMMENT_STALE = (FINDER_DISABLE_COMMENT_BASE +
  31. FINDER_COMMENT_SUFFIX_STALE)
  32. FINDER_DISABLE_COMMENT_UNUSED = (FINDER_DISABLE_COMMENT_BASE +
  33. FINDER_COMMENT_SUFFIX_UNUSED)
  34. FINDER_ENABLE_COMMENT_GENERAL = (FINDER_ENABLE_COMMENT_BASE +
  35. FINDER_COMMENT_SUFFIX_GENERAL)
  36. FINDER_ENABLE_COMMENT_STALE = (FINDER_ENABLE_COMMENT_BASE +
  37. FINDER_COMMENT_SUFFIX_STALE)
  38. FINDER_ENABLE_COMMENT_UNUSED = (FINDER_ENABLE_COMMENT_BASE +
  39. FINDER_COMMENT_SUFFIX_UNUSED)
  40. FINDER_DISABLE_COMMENTS = frozenset([
  41. FINDER_DISABLE_COMMENT_GENERAL, FINDER_DISABLE_COMMENT_STALE,
  42. FINDER_DISABLE_COMMENT_UNUSED
  43. ])
  44. FINDER_ENABLE_COMMENTS = frozenset([
  45. FINDER_ENABLE_COMMENT_GENERAL,
  46. FINDER_ENABLE_COMMENT_STALE,
  47. FINDER_ENABLE_COMMENT_UNUSED,
  48. ])
  49. ALL_FINDER_COMMENTS = frozenset(FINDER_DISABLE_COMMENTS
  50. | FINDER_ENABLE_COMMENTS)
  51. GIT_BLAME_REGEX = re.compile(
  52. r'^[\w\s]+\(.+(?P<date>\d\d\d\d-\d\d-\d\d)[^\)]+\)(?P<content>.*)$',
  53. re.DOTALL)
  54. EXPECTATION_LINE_REGEX = re.compile(r'^.*\[ .* \] .* \[ \w* \].*$', re.DOTALL)
  55. # pylint: disable=useless-object-inheritance
  56. class RemovalType(object):
  57. STALE = FINDER_COMMENT_SUFFIX_STALE
  58. UNUSED = FINDER_COMMENT_SUFFIX_UNUSED
  59. class Expectations(object):
  60. def CreateTestExpectationMap(
  61. self, expectation_files: Optional[Union[str, List[str]]],
  62. tests: Optional[Iterable[str]],
  63. grace_period: int) -> data_types.TestExpectationMap:
  64. """Creates an expectation map based off a file or list of tests.
  65. Args:
  66. expectation_files: A filepath or list of filepaths to expectation files to
  67. read from, or None. If a filepath is specified, |tests| must be None.
  68. tests: An iterable of strings containing test names to check. If
  69. specified, |expectation_file| must be None.
  70. grace_period: An int specifying how many days old an expectation must
  71. be in order to be parsed, i.e. how many days old an expectation must
  72. be before it is a candidate for removal/modification.
  73. Returns:
  74. A data_types.TestExpectationMap, although all its BuilderStepMap contents
  75. will be empty.
  76. """
  77. def AddContentToMap(content: str, ex_map: data_types.TestExpectationMap,
  78. expectation_file_name: str) -> None:
  79. list_parser = expectations_parser.TaggedTestListParser(content)
  80. expectations_for_file = ex_map.setdefault(
  81. expectation_file_name, data_types.ExpectationBuilderMap())
  82. logging.debug('Parsed %d expectations', len(list_parser.expectations))
  83. for e in list_parser.expectations:
  84. if 'Skip' in e.raw_results:
  85. continue
  86. # Expectations that only have a Pass expectation (usually used to
  87. # override a broader, failing expectation) are not handled by the
  88. # unexpected pass finder, so ignore those.
  89. if e.raw_results == ['Pass']:
  90. continue
  91. expectation = data_types.Expectation(e.test, e.tags, e.raw_results,
  92. e.reason)
  93. assert expectation not in expectations_for_file
  94. expectations_for_file[expectation] = data_types.BuilderStepMap()
  95. logging.info('Creating test expectation map')
  96. assert expectation_files or tests
  97. assert not (expectation_files and tests)
  98. expectation_map = data_types.TestExpectationMap()
  99. if expectation_files:
  100. if not isinstance(expectation_files, list):
  101. expectation_files = [expectation_files]
  102. for ef in expectation_files:
  103. # Normalize to '/' as the path separator.
  104. expectation_file_name = os.path.normpath(ef).replace(os.path.sep, '/')
  105. content = self._GetNonRecentExpectationContent(expectation_file_name,
  106. grace_period)
  107. AddContentToMap(content, expectation_map, expectation_file_name)
  108. else:
  109. expectation_file_name = ''
  110. content = '# results: [ RetryOnFailure ]\n'
  111. for t in tests:
  112. content += '%s [ RetryOnFailure ]\n' % t
  113. AddContentToMap(content, expectation_map, expectation_file_name)
  114. return expectation_map
  115. def _GetNonRecentExpectationContent(self, expectation_file_path: str,
  116. num_days: int) -> str:
  117. """Gets content from |expectation_file_path| older than |num_days| days.
  118. Args:
  119. expectation_file_path: A string containing a filepath pointing to an
  120. expectation file.
  121. num_days: An int containing how old an expectation in the given
  122. expectation file must be to be included.
  123. Returns:
  124. The contents of the expectation file located at |expectation_file_path|
  125. as a string with any recent expectations removed.
  126. """
  127. num_days = datetime.timedelta(days=num_days)
  128. content = ''
  129. # `git blame` output is normally in the format:
  130. # revision optional_filename (author date time timezone lineno) line_content
  131. # The --porcelain option is meant to be more machine readable, but is much
  132. # more difficult to parse for what we need to do here. In order to
  133. # guarantee that the filename won't be included in the output (by default,
  134. # it will be shown if there is content from a renamed file), pass -c to
  135. # use the same format as `git annotate`, which is:
  136. # revision (author date time timezone lineno)line_content
  137. # (Note the lack of space between the ) and the content).
  138. cmd = ['git', 'blame', '-c', expectation_file_path]
  139. with open(os.devnull, 'w') as devnull:
  140. blame_output = subprocess.check_output(cmd,
  141. stderr=devnull).decode('utf-8')
  142. for line in blame_output.splitlines(True):
  143. match = GIT_BLAME_REGEX.match(line)
  144. assert match
  145. date = match.groupdict()['date']
  146. line_content = match.groupdict()['content']
  147. if EXPECTATION_LINE_REGEX.match(line):
  148. if six.PY2:
  149. date_parts = date.split('-')
  150. date = datetime.date(year=int(date_parts[0]),
  151. month=int(date_parts[1]),
  152. day=int(date_parts[2]))
  153. else:
  154. date = datetime.date.fromisoformat(date)
  155. date_diff = datetime.date.today() - date
  156. if date_diff > num_days:
  157. content += line_content
  158. else:
  159. logging.debug('Omitting expectation %s because it is too new',
  160. line_content.rstrip())
  161. else:
  162. content += line_content
  163. return content
  164. def RemoveExpectationsFromFile(self,
  165. expectations: List[data_types.Expectation],
  166. expectation_file: str,
  167. removal_type: str) -> Set[str]:
  168. """Removes lines corresponding to |expectations| from |expectation_file|.
  169. Ignores any lines that match but are within a disable block or have an
  170. inline disable comment.
  171. Args:
  172. expectations: A list of data_types.Expectations to remove.
  173. expectation_file: A filepath pointing to an expectation file to remove
  174. lines from.
  175. removal_type: A RemovalType enum corresponding to the type of expectations
  176. being removed.
  177. Returns:
  178. A set of strings containing URLs of bugs associated with the removed
  179. expectations.
  180. """
  181. with open(expectation_file) as f:
  182. input_contents = f.read()
  183. output_contents = ''
  184. in_disable_block = False
  185. disable_block_reason = ''
  186. disable_block_suffix = ''
  187. removed_urls = set()
  188. for line in input_contents.splitlines(True):
  189. # Auto-add any comments or empty lines
  190. stripped_line = line.strip()
  191. if _IsCommentOrBlankLine(stripped_line):
  192. output_contents += line
  193. # Only allow one enable/disable per line.
  194. assert len([c for c in ALL_FINDER_COMMENTS if c in line]) <= 1
  195. # Handle disable/enable block comments.
  196. if _LineContainsDisableComment(line):
  197. if in_disable_block:
  198. raise RuntimeError(
  199. 'Invalid expectation file %s - contains a disable comment "%s" '
  200. 'that is in another disable block.' %
  201. (expectation_file, stripped_line))
  202. in_disable_block = True
  203. disable_block_reason = _GetDisableReasonFromComment(line)
  204. disable_block_suffix = _GetFinderCommentSuffix(line)
  205. if _LineContainsEnableComment(line):
  206. if not in_disable_block:
  207. raise RuntimeError(
  208. 'Invalid expectation file %s - contains an enable comment "%s" '
  209. 'that is outside of a disable block.' %
  210. (expectation_file, stripped_line))
  211. in_disable_block = False
  212. continue
  213. current_expectation = self._CreateExpectationFromExpectationFileLine(
  214. line, expectation_file)
  215. # Add any lines containing expectations that don't match any of the given
  216. # expectations to remove.
  217. if any(e for e in expectations if e == current_expectation):
  218. # Skip any expectations that match if we're in a disable block or there
  219. # is an inline disable comment.
  220. if in_disable_block and _DisableSuffixIsRelevant(
  221. disable_block_suffix, removal_type):
  222. output_contents += line
  223. logging.info(
  224. 'Would have removed expectation %s, but inside a disable block '
  225. 'with reason %s', stripped_line, disable_block_reason)
  226. elif _LineContainsRelevantDisableComment(line, removal_type):
  227. output_contents += line
  228. logging.info(
  229. 'Would have removed expectation %s, but it has an inline disable '
  230. 'comment with reason %s',
  231. stripped_line.split('#')[0], _GetDisableReasonFromComment(line))
  232. else:
  233. bug = current_expectation.bug
  234. if bug:
  235. # It's possible to have multiple whitespace-separated bugs per
  236. # expectation, so treat each one separately.
  237. removed_urls |= set(bug.split())
  238. else:
  239. output_contents += line
  240. with open(expectation_file, 'w') as f:
  241. f.write(output_contents)
  242. return removed_urls
  243. def _CreateExpectationFromExpectationFileLine(self, line: str,
  244. expectation_file: str
  245. ) -> data_types.Expectation:
  246. """Creates a data_types.Expectation from |line|.
  247. Args:
  248. line: A string containing a single line from an expectation file.
  249. expectation_file: A filepath pointing to an expectation file |line| came
  250. from.
  251. Returns:
  252. A data_types.Expectation containing the same information as |line|.
  253. """
  254. header = self._GetExpectationFileTagHeader(expectation_file)
  255. single_line_content = header + line
  256. list_parser = expectations_parser.TaggedTestListParser(single_line_content)
  257. assert len(list_parser.expectations) == 1
  258. typ_expectation = list_parser.expectations[0]
  259. return data_types.Expectation(typ_expectation.test, typ_expectation.tags,
  260. typ_expectation.raw_results,
  261. typ_expectation.reason)
  262. def _GetExpectationFileTagHeader(self, expectation_file: str) -> str:
  263. """Gets the tag header used for expectation files.
  264. Args:
  265. expectation_file: A filepath pointing to an expectation file to get the
  266. tag header from.
  267. Returns:
  268. A string containing an expectation file header, i.e. the comment block at
  269. the top of the file defining possible tags and expected results.
  270. """
  271. raise NotImplementedError()
  272. def ModifySemiStaleExpectations(
  273. self, stale_expectation_map: data_types.TestExpectationMap) -> Set[str]:
  274. """Modifies lines from |stale_expectation_map| in |expectation_file|.
  275. Prompts the user for each modification and provides debug information since
  276. semi-stale expectations cannot be blindly removed like fully stale ones.
  277. Args:
  278. stale_expectation_map: A data_types.TestExpectationMap containing stale
  279. expectations.
  280. file_handle: An optional open file-like object to output to. If not
  281. specified, stdout will be used.
  282. Returns:
  283. A set of strings containing URLs of bugs associated with the modified
  284. (manually modified by the user or removed by the script) expectations.
  285. """
  286. expectations_to_remove = []
  287. expectations_to_modify = []
  288. modified_urls = set()
  289. for expectation_file, e, builder_map in (
  290. stale_expectation_map.IterBuilderStepMaps()):
  291. with open(expectation_file) as infile:
  292. file_contents = infile.read()
  293. line, line_number = self._GetExpectationLine(e, file_contents,
  294. expectation_file)
  295. expectation_str = None
  296. if not line:
  297. logging.error(
  298. 'Could not find line corresponding to semi-stale expectation for '
  299. '%s with tags %s and expected results %s', e.test, e.tags,
  300. e.expected_results)
  301. expectation_str = '[ %s ] %s [ %s ]' % (' '.join(
  302. e.tags), e.test, ' '.join(e.expected_results))
  303. else:
  304. expectation_str = '%s (approx. line %d)' % (line, line_number)
  305. str_dict = result_output.ConvertBuilderMapToPassOrderedStringDict(
  306. builder_map)
  307. print('\nSemi-stale expectation:\n%s' % expectation_str)
  308. result_output.RecursivePrintToFile(str_dict, 1, sys.stdout)
  309. response = _WaitForUserInputOnModification()
  310. if response == 'r':
  311. expectations_to_remove.append(e)
  312. elif response == 'm':
  313. expectations_to_modify.append(e)
  314. # It's possible that the user will introduce a typo while manually
  315. # modifying an expectation, which will cause a parser error. Catch that
  316. # now and give them chances to fix it so that they don't lose all of their
  317. # work due to an early exit.
  318. while True:
  319. try:
  320. with open(expectation_file) as infile:
  321. file_contents = infile.read()
  322. _ = expectations_parser.TaggedTestListParser(file_contents)
  323. break
  324. except expectations_parser.ParseError as error:
  325. logging.error('Got parser error: %s', error)
  326. logging.error(
  327. 'This probably means you introduced a typo, please fix it.')
  328. _WaitForAnyUserInput()
  329. modified_urls |= self.RemoveExpectationsFromFile(expectations_to_remove,
  330. expectation_file,
  331. RemovalType.STALE)
  332. for e in expectations_to_modify:
  333. modified_urls |= set(e.bug.split())
  334. return modified_urls
  335. def _GetExpectationLine(self, expectation: data_types.Expectation,
  336. file_contents: str, expectation_file: str
  337. ) -> Union[Tuple[None, None], Tuple[str, int]]:
  338. """Gets the line and line number of |expectation| in |file_contents|.
  339. Args:
  340. expectation: A data_types.Expectation.
  341. file_contents: A string containing the contents read from an expectation
  342. file.
  343. expectation_file: A string containing the path to the expectation file
  344. that |file_contents| came from.
  345. Returns:
  346. A tuple (line, line_number). |line| is a string containing the exact line
  347. in |file_contents| corresponding to |expectation|. |line_number| is an int
  348. corresponding to where |line| is in |file_contents|. |line_number| may be
  349. off if the file on disk has changed since |file_contents| was read. If a
  350. corresponding line cannot be found, both |line| and |line_number| are
  351. None.
  352. """
  353. # We have all the information necessary to recreate the expectation line and
  354. # line number can be pulled during the initial expectation parsing. However,
  355. # the information we have is not necessarily in the same order as the
  356. # text file (e.g. tag ordering), and line numbers can change pretty
  357. # dramatically between the initial parse and now due to stale expectations
  358. # being removed. So, parse this way in order to improve the user experience.
  359. file_lines = file_contents.splitlines()
  360. for line_number, line in enumerate(file_lines):
  361. if _IsCommentOrBlankLine(line.strip()):
  362. continue
  363. current_expectation = self._CreateExpectationFromExpectationFileLine(
  364. line, expectation_file)
  365. if expectation == current_expectation:
  366. return line, line_number + 1
  367. return None, None
  368. def FindOrphanedBugs(self, affected_urls: Iterable[str]) -> Set[str]:
  369. """Finds cases where expectations for bugs no longer exist.
  370. Args:
  371. affected_urls: An iterable of affected bug URLs, as returned by functions
  372. such as RemoveExpectationsFromFile.
  373. Returns:
  374. A set containing a subset of |affected_urls| who no longer have any
  375. associated expectations in any expectation files.
  376. """
  377. seen_bugs = set()
  378. expectation_files = self.GetExpectationFilepaths()
  379. for ef in expectation_files:
  380. with open(ef) as infile:
  381. contents = infile.read()
  382. for url in affected_urls:
  383. if url in seen_bugs:
  384. continue
  385. if url in contents:
  386. seen_bugs.add(url)
  387. return set(affected_urls) - seen_bugs
  388. def GetExpectationFilepaths(self) -> List[str]:
  389. """Gets all the filepaths to expectation files of interest.
  390. Returns:
  391. A list of strings, each element being a filepath pointing towards an
  392. expectation file.
  393. """
  394. raise NotImplementedError()
  395. def _WaitForAnyUserInput() -> None:
  396. """Waits for any user input.
  397. Split out for testing purposes.
  398. """
  399. _get_input('Press any key to continue')
  400. def _WaitForUserInputOnModification() -> str:
  401. """Waits for user input on how to modify a semi-stale expectation.
  402. Returns:
  403. One of the following string values:
  404. i - Expectation should be ignored and left alone.
  405. m - Expectation will be manually modified by the user.
  406. r - Expectation should be removed by the script.
  407. """
  408. valid_inputs = ['i', 'm', 'r']
  409. prompt = ('How should this expectation be handled? (i)gnore/(m)anually '
  410. 'modify/(r)emove: ')
  411. response = _get_input(prompt).lower()
  412. while response not in valid_inputs:
  413. print('Invalid input, valid inputs are %s' % (', '.join(valid_inputs)))
  414. response = _get_input(prompt).lower()
  415. return response
  416. def _LineContainsDisableComment(line: str) -> bool:
  417. return FINDER_DISABLE_COMMENT_BASE in line
  418. def _LineContainsEnableComment(line: str) -> bool:
  419. return FINDER_ENABLE_COMMENT_BASE in line
  420. def _GetFinderCommentSuffix(line: str) -> str:
  421. """Gets the suffix of the finder comment on the given line.
  422. Examples:
  423. 'foo # finder:disable' -> ''
  424. 'foo # finder:disable-stale some_reason' -> '-stale'
  425. """
  426. target_str = None
  427. if _LineContainsDisableComment(line):
  428. target_str = FINDER_DISABLE_COMMENT_BASE
  429. elif _LineContainsEnableComment(line):
  430. target_str = FINDER_ENABLE_COMMENT_BASE
  431. else:
  432. raise RuntimeError('Given line %s did not have a finder comment.' % line)
  433. line = line[line.find(target_str):]
  434. line = line.split()[0]
  435. suffix = line.replace(target_str, '')
  436. assert suffix in ALL_FINDER_SUFFIXES
  437. return suffix
  438. def _LineContainsRelevantDisableComment(line: str, removal_type: str) -> bool:
  439. """Returns whether the given line contains a relevant disable comment.
  440. Args:
  441. line: A string containing the line to check.
  442. removal_type: A RemovalType enum corresponding to the type of expectations
  443. being removed.
  444. Returns:
  445. A bool denoting whether |line| contains a relevant disable comment given
  446. |removal_type|.
  447. """
  448. if FINDER_DISABLE_COMMENT_GENERAL in line:
  449. return True
  450. if FINDER_DISABLE_COMMENT_BASE + removal_type in line:
  451. return True
  452. return False
  453. def _DisableSuffixIsRelevant(suffix: str, removal_type: str) -> bool:
  454. """Returns whether the given suffix is relevant given the removal type.
  455. Args:
  456. suffix: A string containing a disable comment suffix.
  457. removal_type: A RemovalType enum corresponding to the type of expectations
  458. being removed.
  459. Returns:
  460. True if suffix is relevant and its disable request should be honored.
  461. """
  462. if suffix == FINDER_COMMENT_SUFFIX_GENERAL:
  463. return True
  464. if suffix == removal_type:
  465. return True
  466. return False
  467. def _GetDisableReasonFromComment(line: str) -> str:
  468. suffix = _GetFinderCommentSuffix(line)
  469. return line.split(FINDER_DISABLE_COMMENT_BASE + suffix, 1)[1].strip()
  470. def _IsCommentOrBlankLine(line: str) -> bool:
  471. return (not line or line.startswith('#'))
  472. def _get_input(prompt: str) -> str:
  473. if sys.version_info[0] == 2:
  474. return raw_input(prompt)
  475. return input(prompt)