compare_codereview.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414
  1. #!/usr/bin/python2
  2. # Copyright 2014 Google Inc.
  3. #
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. """Skia's Chromium Codereview Comparison Script.
  7. This script takes two Codereview URLs, looks at the trybot results for
  8. the two codereviews and compares the results.
  9. Usage:
  10. compare_codereview.py CONTROL_URL ROLL_URL
  11. """
  12. import collections
  13. import os
  14. import re
  15. import sys
  16. import urllib2
  17. import HTMLParser
  18. class CodeReviewHTMLParser(HTMLParser.HTMLParser):
  19. """Parses CodeReview web page.
  20. Use the CodeReviewHTMLParser.parse static function to make use of
  21. this class.
  22. This uses the HTMLParser class because it's the best thing in
  23. Python's standard library. We need a little more power than a
  24. regex. [Search for "You can't parse [X]HTML with regex." for more
  25. information.
  26. """
  27. # pylint: disable=I0011,R0904
  28. @staticmethod
  29. def parse(url):
  30. """Parses a CodeReview web pages.
  31. Args:
  32. url (string), a codereview URL like this:
  33. 'https://codereview.chromium.org/?????????'.
  34. Returns:
  35. A dictionary; the keys are bot_name strings, the values
  36. are CodeReviewHTMLParser.Status objects
  37. """
  38. parser = CodeReviewHTMLParser()
  39. try:
  40. parser.feed(urllib2.urlopen(url).read())
  41. except (urllib2.URLError,):
  42. print >> sys.stderr, 'Error getting', url
  43. return None
  44. parser.close()
  45. return parser.statuses
  46. # namedtuples are like lightweight structs in Python. The low
  47. # overhead of a tuple, but the ease of use of an object.
  48. Status = collections.namedtuple('Status', ['status', 'url'])
  49. def __init__(self):
  50. HTMLParser.HTMLParser.__init__(self)
  51. self._id = None
  52. self._status = None
  53. self._href = None
  54. self._anchor_data = ''
  55. self._currently_parsing_trybotdiv = False
  56. # statuses is a dictionary of CodeReviewHTMLParser.Status
  57. self.statuses = {}
  58. def handle_starttag(self, tag, attrs):
  59. """Overrides the HTMLParser method to implement functionality.
  60. [[begin standard library documentation]]
  61. This method is called to handle the start of a tag
  62. (e.g. <div id="main">).
  63. The tag argument is the name of the tag converted to lower
  64. case. The attrs argument is a list of (name, value) pairs
  65. containing the attributes found inside the tag's <>
  66. brackets. The name will be translated to lower case, and
  67. quotes in the value have been removed, and character and
  68. entity references have been replaced.
  69. For instance, for the tag <A HREF="http://www.cwi.nl/">, this
  70. method would be called as handle_starttag('a', [('href',
  71. 'http://www.cwi.nl/')]).
  72. [[end standard library documentation]]
  73. """
  74. attrs = dict(attrs)
  75. if tag == 'div':
  76. # We are looking for <div id="tryjobdiv*">.
  77. id_attr = attrs.get('id','')
  78. if id_attr.startswith('tryjobdiv'):
  79. self._id = id_attr
  80. if (self._id and tag == 'a'
  81. and 'build-result' in attrs.get('class', '').split()):
  82. # If we are already inside a <div id="tryjobdiv*">, we
  83. # look for a link if the form
  84. # <a class="build-result" href="*">. Then we save the
  85. # (non-standard) status attribute and the URL.
  86. self._status = attrs.get('status')
  87. self._href = attrs.get('href')
  88. self._currently_parsing_trybotdiv = True
  89. # Start saving anchor data.
  90. def handle_data(self, data):
  91. """Overrides the HTMLParser method to implement functionality.
  92. [[begin standard library documentation]]
  93. This method is called to process arbitrary data (e.g. text
  94. nodes and the content of <script>...</script> and
  95. <style>...</style>).
  96. [[end standard library documentation]]
  97. """
  98. # Save the text inside the <a></a> tags. Assume <a> tags
  99. # aren't nested.
  100. if self._currently_parsing_trybotdiv:
  101. self._anchor_data += data
  102. def handle_endtag(self, tag):
  103. """Overrides the HTMLParser method to implement functionality.
  104. [[begin standard library documentation]]
  105. This method is called to handle the end tag of an element
  106. (e.g. </div>). The tag argument is the name of the tag
  107. converted to lower case.
  108. [[end standard library documentation]]
  109. """
  110. if tag == 'a' and self._status:
  111. # We take the accumulated self._anchor_data and save it as
  112. # the bot name.
  113. bot = self._anchor_data.strip()
  114. stat = CodeReviewHTMLParser.Status(status=self._status,
  115. url=self._href)
  116. if bot:
  117. # Add to accumulating dictionary.
  118. self.statuses[bot] = stat
  119. # Reset state to search for the next bot.
  120. self._currently_parsing_trybotdiv = False
  121. self._anchor_data = ''
  122. self._status = None
  123. self._href = None
  124. class BuilderHTMLParser(HTMLParser.HTMLParser):
  125. """parses Trybot web pages.
  126. Use the BuilderHTMLParser.parse static function to make use of
  127. this class.
  128. This uses the HTMLParser class because it's the best thing in
  129. Python's standard library. We need a little more power than a
  130. regex. [Search for "You can't parse [X]HTML with regex." for more
  131. information.
  132. """
  133. # pylint: disable=I0011,R0904
  134. @staticmethod
  135. def parse(url):
  136. """Parses a Trybot web page.
  137. Args:
  138. url (string), a trybot result URL.
  139. Returns:
  140. An array of BuilderHTMLParser.Results, each a description
  141. of failure results, along with an optional url
  142. """
  143. parser = BuilderHTMLParser()
  144. try:
  145. parser.feed(urllib2.urlopen(url).read())
  146. except (urllib2.URLError,):
  147. print >> sys.stderr, 'Error getting', url
  148. return []
  149. parser.close()
  150. return parser.failure_results
  151. Result = collections.namedtuple('Result', ['text', 'url'])
  152. def __init__(self):
  153. HTMLParser.HTMLParser.__init__(self)
  154. self.failure_results = []
  155. self._current_failure_result = None
  156. self._divlevel = None
  157. self._li_level = 0
  158. self._li_data = ''
  159. self._current_failure = False
  160. self._failure_results_url = ''
  161. def handle_starttag(self, tag, attrs):
  162. """Overrides the HTMLParser method to implement functionality.
  163. [[begin standard library documentation]]
  164. This method is called to handle the start of a tag
  165. (e.g. <div id="main">).
  166. The tag argument is the name of the tag converted to lower
  167. case. The attrs argument is a list of (name, value) pairs
  168. containing the attributes found inside the tag's <>
  169. brackets. The name will be translated to lower case, and
  170. quotes in the value have been removed, and character and
  171. entity references have been replaced.
  172. For instance, for the tag <A HREF="http://www.cwi.nl/">, this
  173. method would be called as handle_starttag('a', [('href',
  174. 'http://www.cwi.nl/')]).
  175. [[end standard library documentation]]
  176. """
  177. attrs = dict(attrs)
  178. if tag == 'li':
  179. # <li> tags can be nested. So we have to count the
  180. # nest-level for backing out.
  181. self._li_level += 1
  182. return
  183. if tag == 'div' and attrs.get('class') == 'failure result':
  184. # We care about this sort of thing:
  185. # <li>
  186. # <li>
  187. # <li>
  188. # <div class="failure result">...</div>
  189. # </li>
  190. # </li>
  191. # We want this text here.
  192. # </li>
  193. if self._li_level > 0:
  194. self._current_failure = True # Tells us to keep text.
  195. return
  196. if tag == 'a' and self._current_failure:
  197. href = attrs.get('href')
  198. # Sometimes we want to keep the stdio url. We always
  199. # return it, just in case.
  200. if href.endswith('/logs/stdio'):
  201. self._failure_results_url = href
  202. def handle_data(self, data):
  203. """Overrides the HTMLParser method to implement functionality.
  204. [[begin standard library documentation]]
  205. This method is called to process arbitrary data (e.g. text
  206. nodes and the content of <script>...</script> and
  207. <style>...</style>).
  208. [[end standard library documentation]]
  209. """
  210. if self._current_failure:
  211. self._li_data += data
  212. def handle_endtag(self, tag):
  213. """Overrides the HTMLParser method to implement functionality.
  214. [[begin standard library documentation]]
  215. This method is called to handle the end tag of an element
  216. (e.g. </div>). The tag argument is the name of the tag
  217. converted to lower case.
  218. [[end standard library documentation]]
  219. """
  220. if tag == 'li':
  221. self._li_level -= 1
  222. if 0 == self._li_level:
  223. if self._current_failure:
  224. result = self._li_data.strip()
  225. first = result.split()[0]
  226. if first:
  227. result = re.sub(
  228. r'^%s(\s+%s)+' % (first, first), first, result)
  229. # Sometimes, it repeats the same thing
  230. # multiple times.
  231. result = re.sub(r'unexpected flaky.*', '', result)
  232. # Remove some extra unnecessary text.
  233. result = re.sub(r'\bpreamble\b', '', result)
  234. result = re.sub(r'\bstdio\b', '', result)
  235. url = self._failure_results_url
  236. self.failure_results.append(
  237. BuilderHTMLParser.Result(result, url))
  238. self._current_failure_result = None
  239. # Reset the state.
  240. self._current_failure = False
  241. self._li_data = ''
  242. self._failure_results_url = ''
  243. def printer(indent, string):
  244. """Print indented, wrapped text.
  245. """
  246. def wrap_to(line, columns):
  247. """Wrap a line to the given number of columns, return a list
  248. of strings.
  249. """
  250. ret = []
  251. nextline = ''
  252. for word in line.split():
  253. if nextline:
  254. if len(nextline) + 1 + len(word) > columns:
  255. ret.append(nextline)
  256. nextline = word
  257. else:
  258. nextline += (' ' + word)
  259. else:
  260. nextline = word
  261. if nextline:
  262. ret.append(nextline)
  263. return ret
  264. out = sys.stdout
  265. spacer = ' '
  266. for line in string.split('\n'):
  267. for i, wrapped_line in enumerate(wrap_to(line, 68 - (2 * indent))):
  268. out.write(spacer * indent)
  269. if i > 0:
  270. out.write(spacer)
  271. out.write(wrapped_line)
  272. out.write('\n')
  273. out.flush()
  274. def main(control_url, roll_url, verbosity=1):
  275. """Compare two Codereview URLs
  276. Args:
  277. control_url, roll_url: (strings) URL of the format
  278. https://codereview.chromium.org/?????????
  279. verbosity: (int) verbose level. 0, 1, or 2.
  280. """
  281. # pylint: disable=I0011,R0914,R0912
  282. control = CodeReviewHTMLParser.parse(control_url)
  283. roll = CodeReviewHTMLParser.parse(roll_url)
  284. all_bots = set(control) & set(roll) # Set intersection.
  285. if not all_bots:
  286. print >> sys.stderr, (
  287. 'Error: control %s and roll %s have no common trybots.'
  288. % (list(control), list(roll)))
  289. return
  290. control_name = '[control %s]' % control_url.split('/')[-1]
  291. roll_name = '[roll %s]' % roll_url.split('/')[-1]
  292. out = sys.stdout
  293. for bot in sorted(all_bots):
  294. if (roll[bot].status == 'success'):
  295. if verbosity > 1:
  296. printer(0, '==%s==' % bot)
  297. printer(1, 'OK')
  298. continue
  299. if control[bot].status != 'failure' and roll[bot].status != 'failure':
  300. continue
  301. printer(0, '==%s==' % bot)
  302. formatted_results = []
  303. for (status, name, url) in [
  304. (control[bot].status, control_name, control[bot].url),
  305. ( roll[bot].status, roll_name, roll[bot].url)]:
  306. lines = []
  307. if status == 'failure':
  308. results = BuilderHTMLParser.parse(url)
  309. for result in results:
  310. formatted_result = re.sub(r'(\S*\.html) ', '\n__\g<1>\n', result.text)
  311. # Strip runtimes.
  312. formatted_result = re.sub(r'\(.*\)', '', formatted_result)
  313. lines.append((2, formatted_result))
  314. if ('compile' in result.text or '...and more' in result.text):
  315. lines.append((3, re.sub('/[^/]*$', '/', url) + result.url))
  316. formatted_results.append(lines)
  317. identical = formatted_results[0] == formatted_results[1]
  318. for (formatted_result, (status, name, url)) in zip(
  319. formatted_results,
  320. [(control[bot].status, control_name, control[bot].url),
  321. (roll[bot].status, roll_name, roll[bot].url)]):
  322. if status != 'failure' and not identical:
  323. printer(1, name)
  324. printer(2, status)
  325. elif status == 'failure':
  326. if identical:
  327. printer(1, control_name + ' and ' + roll_name + ' failed identically')
  328. else:
  329. printer(1, name)
  330. for (indent, line) in formatted_result:
  331. printer(indent, line)
  332. if identical:
  333. break
  334. out.write('\n')
  335. if verbosity > 0:
  336. # Print out summary of all of the bots.
  337. out.write('%11s %11s %4s %s\n\n' %
  338. ('CONTROL', 'ROLL', 'DIFF', 'BOT'))
  339. for bot in sorted(all_bots):
  340. if roll[bot].status == 'success':
  341. diff = ''
  342. elif (control[bot].status == 'success' and
  343. roll[bot].status == 'failure'):
  344. diff = '!!!!'
  345. elif ('pending' in control[bot].status or
  346. 'pending' in roll[bot].status):
  347. diff = '....'
  348. else:
  349. diff = '****'
  350. out.write('%11s %11s %4s %s\n' % (
  351. control[bot].status, roll[bot].status, diff, bot))
  352. out.write('\n')
  353. out.flush()
  354. if __name__ == '__main__':
  355. if len(sys.argv) < 3:
  356. print >> sys.stderr, __doc__
  357. exit(1)
  358. main(sys.argv[1], sys.argv[2],
  359. int(os.environ.get('COMPARE_CODEREVIEW_VERBOSITY', 1)))