dromaeo_benchmark_runner.py 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268
  1. #!/usr/bin/env python
  2. # Copyright (c) 2011 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Dromaeo benchmark automation script.
  6. Script runs dromaeo tests in browsers specified by --browser switch and saves
  7. results to a spreadsheet on docs.google.com.
  8. Prerequisites:
  9. 1. Install Google Data APIs Python Client Library from
  10. http://code.google.com/p/gdata-python-client.
  11. 2. Checkout Dromaeo benchmark from
  12. http://src.chromium.org/svn/trunk/src/chrome/test/data/dromaeo and provide
  13. local path to it in --dromaeo_home switch.
  14. 3. Create a spreadsheet at http://docs.google.com and specify its name in
  15. --spreadsheet switch
  16. Benchmark results are presented in the following format:
  17. browser | date time
  18. test 1 name|m11|...|m1n|test 1 average mean| |e11|...|e1n|test 1 average error
  19. test 2 name|m21|...|m2n|test 2 average mean| |e21|...|e2n|test 2 average error
  20. ...
  21. Here mij is mean run/s in individual dromaeo test i during benchmark run j,
  22. eij is error in individual dromaeo test i during benchmark run j.
  23. Example usage:
  24. dromaeo_benchmark_runner.py -b "E:\chromium\src\chrome\Release\chrome.exe"
  25. -b "C:\Program Files (x86)\Safari\safari.exe"
  26. -b "C:\Program Files (x86)\Opera 10.50 pre-alpha\opera.exe" -n 1
  27. -d "E:\chromium\src\chrome\test\data\dromaeo" -f dom -e example@gmail.com
  28. """
  29. from __future__ import print_function
  30. import getpass
  31. import json
  32. import os
  33. import re
  34. import subprocess
  35. import time
  36. import urlparse
  37. from optparse import OptionParser
  38. from BaseHTTPServer import HTTPServer
  39. import SimpleHTTPServer
  40. import gdata.spreadsheet.service
  41. max_spreadsheet_columns = 20
  42. test_props = ['mean', 'error']
  43. def ParseArguments():
  44. parser = OptionParser()
  45. parser.add_option("-b", "--browser",
  46. action="append", dest="browsers",
  47. help="list of browsers to test")
  48. parser.add_option("-n", "--run_count", dest="run_count", type="int",
  49. default=5, help="number of runs")
  50. parser.add_option("-d", "--dromaeo_home", dest="dromaeo_home",
  51. help="directory with your dromaeo files")
  52. parser.add_option("-p", "--port", dest="port", type="int",
  53. default=8080, help="http server port")
  54. parser.add_option("-f", "--filter", dest="filter",
  55. default="dom", help="dromaeo suite filter")
  56. parser.add_option("-e", "--email", dest="email",
  57. help="your google docs account")
  58. parser.add_option("-s", "--spreadsheet", dest="spreadsheet_title",
  59. default="dromaeo",
  60. help="your google docs spreadsheet name")
  61. options = parser.parse_args()[0]
  62. if not options.dromaeo_home:
  63. raise Exception('please specify dromaeo_home')
  64. return options
  65. def KillProcessByName(process_name):
  66. process = subprocess.Popen('wmic process get processid, executablepath',
  67. stdout=subprocess.PIPE)
  68. stdout = str(process.communicate()[0])
  69. match = re.search(re.escape(process_name) + '\s+(\d+)', stdout)
  70. if match:
  71. pid = match.group(1)
  72. subprocess.call('taskkill /pid %s' % pid)
  73. class SpreadsheetWriter(object):
  74. "Utility class for storing benchmarking results in Google spreadsheets."
  75. def __init__(self, email, spreadsheet_title):
  76. '''Login to google docs and search for spreadsheet'''
  77. self.token_file = os.path.expanduser("~/.dromaeo_bot_auth_token")
  78. self.gd_client = gdata.spreadsheet.service.SpreadsheetsService()
  79. authenticated = False
  80. if os.path.exists(self.token_file):
  81. token = ''
  82. try:
  83. file = open(self.token_file, 'r')
  84. token = file.read()
  85. file.close()
  86. self.gd_client.SetClientLoginToken(token)
  87. self.gd_client.GetSpreadsheetsFeed()
  88. authenticated = True
  89. except (IOError, gdata.service.RequestError):
  90. pass
  91. if not authenticated:
  92. self.gd_client.email = email
  93. self.gd_client.password = getpass.getpass('Password for %s: ' % email)
  94. self.gd_client.source = 'python robot for dromaeo'
  95. self.gd_client.ProgrammaticLogin()
  96. token = self.gd_client.GetClientLoginToken()
  97. try:
  98. file = open(self.token_file, 'w')
  99. file.write(token)
  100. file.close()
  101. except (IOError):
  102. pass
  103. os.chmod(self.token_file, 0600)
  104. # Search for the spreadsheet with title = spreadsheet_title.
  105. spreadsheet_feed = self.gd_client.GetSpreadsheetsFeed()
  106. for spreadsheet in spreadsheet_feed.entry:
  107. if spreadsheet.title.text == spreadsheet_title:
  108. self.spreadsheet_key = spreadsheet.id.text.rsplit('/', 1)[1]
  109. if not self.spreadsheet_key:
  110. raise Exception('Spreadsheet %s not found' % spreadsheet_title)
  111. # Get the key of the first worksheet in spreadsheet.
  112. worksheet_feed = self.gd_client.GetWorksheetsFeed(self.spreadsheet_key)
  113. self.worksheet_key = worksheet_feed.entry[0].id.text.rsplit('/', 1)[1]
  114. def _InsertRow(self, row):
  115. row = dict([('c' + str(i), row[i]) for i in xrange(len(row))])
  116. self.gd_client.InsertRow(row, self.spreadsheet_key, self.worksheet_key)
  117. def _InsertBlankRow(self):
  118. self._InsertRow('-' * self.columns_count)
  119. def PrepareSpreadsheet(self, run_count):
  120. """Update cells in worksheet topmost row with service information.
  121. Calculate column count corresponding to run_count and create worksheet
  122. column titles [c0, c1, ...] in the topmost row to speed up spreadsheet
  123. updates (it allows to insert a whole row with a single request)
  124. """
  125. # Calculate the number of columns we need to present all test results.
  126. self.columns_count = (run_count + 2) * len(test_props)
  127. if self.columns_count > max_spreadsheet_columns:
  128. # Google spreadsheet has just max_spreadsheet_columns columns.
  129. max_run_count = max_spreadsheet_columns / len(test_props) - 2
  130. raise Exception('maximum run count is %i' % max_run_count)
  131. # Create worksheet column titles [c0, c1, ..., cn].
  132. for i in xrange(self.columns_count):
  133. self.gd_client.UpdateCell(1, i + 1, 'c' + str(i), self.spreadsheet_key,
  134. self.worksheet_key)
  135. def WriteColumnTitles(self, run_count):
  136. "Create titles for test results (mean 1, mean 2, ..., average mean, ...)"
  137. row = []
  138. for prop in test_props:
  139. row.append('')
  140. for i in xrange(run_count):
  141. row.append('%s %i' % (prop, i + 1))
  142. row.append('average ' + prop)
  143. self._InsertRow(row)
  144. def WriteBrowserBenchmarkTitle(self, browser_name):
  145. "Create browser benchmark title (browser name, date time)"
  146. self._InsertBlankRow()
  147. self._InsertRow([browser_name, time.strftime('%d.%m.%Y %H:%M:%S')])
  148. def WriteBrowserBenchmarkResults(self, test_name, test_data):
  149. "Insert a row with single test results"
  150. row = []
  151. for prop in test_props:
  152. if not row:
  153. row.append(test_name)
  154. else:
  155. row.append('')
  156. row.extend([str(x) for x in test_data[prop]])
  157. row.append(str(sum(test_data[prop]) / len(test_data[prop])))
  158. self._InsertRow(row)
  159. class DromaeoHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
  160. def do_POST(self):
  161. self.send_response(200)
  162. self.end_headers()
  163. self.wfile.write("<HTML>POST OK.<BR><BR>");
  164. length = int(self.headers.getheader('content-length'))
  165. parameters = urlparse.parse_qs(self.rfile.read(length))
  166. self.server.got_post = True
  167. self.server.post_data = parameters['data']
  168. class BenchmarkResults(object):
  169. "Storage class for dromaeo benchmark results"
  170. def __init__(self):
  171. self.data = {}
  172. def ProcessBrowserPostData(self, data):
  173. "Convert dromaeo test results in internal format"
  174. tests = json.loads(data[0])
  175. for test in tests:
  176. test_name = test['name']
  177. if test_name not in self.data:
  178. # Test is encountered for the first time.
  179. self.data[test_name] = dict([(prop, []) for prop in test_props])
  180. # Append current run results.
  181. for prop in test_props:
  182. value = -1
  183. if prop in test: value = test[prop] # workaround for Opera 10.5
  184. self.data[test_name][prop].append(value)
  185. def main():
  186. options = ParseArguments()
  187. # Start sever with dromaeo.
  188. os.chdir(options.dromaeo_home)
  189. server = HTTPServer(('', options.port), DromaeoHandler)
  190. # Open and prepare spreadsheet on google docs.
  191. spreadsheet_writer = SpreadsheetWriter(options.email,
  192. options.spreadsheet_title)
  193. spreadsheet_writer.PrepareSpreadsheet(options.run_count)
  194. spreadsheet_writer.WriteColumnTitles(options.run_count)
  195. for browser in options.browsers:
  196. browser_name = os.path.splitext(os.path.basename(browser))[0]
  197. spreadsheet_writer.WriteBrowserBenchmarkTitle(browser_name)
  198. benchmark_results = BenchmarkResults()
  199. for run_number in xrange(options.run_count):
  200. print('%s run %i' % (browser_name, run_number + 1))
  201. # Run browser.
  202. test_page = 'http://localhost:%i/index.html?%s&automated&post_json' % (
  203. options.port, options.filter)
  204. browser_process = subprocess.Popen('%s "%s"' % (browser, test_page))
  205. server.got_post = False
  206. server.post_data = None
  207. # Wait until POST request from browser.
  208. while not server.got_post:
  209. server.handle_request()
  210. benchmark_results.ProcessBrowserPostData(server.post_data)
  211. # Kill browser.
  212. KillProcessByName(browser)
  213. browser_process.wait()
  214. # Insert test results into spreadsheet.
  215. for (test_name, test_data) in benchmark_results.data.iteritems():
  216. spreadsheet_writer.WriteBrowserBenchmarkResults(test_name, test_data)
  217. server.socket.close()
  218. return 0
  219. if __name__ == '__main__':
  220. sys.exit(main())