webpages_playback.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623
  1. #!/usr/bin/env python
  2. # Copyright (c) 2012 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Archives or replays webpages and creates SKPs in a Google Storage location.
  6. To archive webpages and store SKP files (archives should be rarely updated):
  7. cd skia
  8. python tools/skp/webpages_playback.py --data_store=gs://rmistry --record \
  9. --page_sets=all --skia_tools=/home/default/trunk/out/Debug/ \
  10. --browser_executable=/tmp/chromium/out/Release/chrome
  11. The above command uses Google Storage bucket 'rmistry' to download needed files.
  12. To replay archived webpages and re-generate SKP files (should be run whenever
  13. SkPicture.PICTURE_VERSION changes):
  14. cd skia
  15. python tools/skp/webpages_playback.py --data_store=gs://rmistry \
  16. --page_sets=all --skia_tools=/home/default/trunk/out/Debug/ \
  17. --browser_executable=/tmp/chromium/out/Release/chrome
  18. Specify the --page_sets flag (default value is 'all') to pick a list of which
  19. webpages should be archived and/or replayed. Eg:
  20. --page_sets=tools/skp/page_sets/skia_yahooanswers_desktop.py,\
  21. tools/skp/page_sets/skia_googlecalendar_nexus10.py
  22. The --browser_executable flag should point to the browser binary you want to use
  23. to capture archives and/or capture SKP files. Majority of the time it should be
  24. a newly built chrome binary.
  25. The --data_store flag controls where the needed artifacts are downloaded from.
  26. It also controls where the generated artifacts, such as recorded webpages and
  27. resulting skp renderings, are uploaded to. URLs with scheme 'gs://' use Google
  28. Storage. Otherwise use local filesystem.
  29. The --upload=True flag means generated artifacts will be
  30. uploaded or copied to the location specified by --data_store. (default value is
  31. False if not specified).
  32. The --non-interactive flag controls whether the script will prompt the user
  33. (default value is False if not specified).
  34. The --skia_tools flag if specified will allow this script to run
  35. debugger, render_pictures, and render_pdfs on the captured
  36. SKP(s). The tools are run after all SKPs are succesfully captured to make sure
  37. they can be added to the buildbots with no breakages.
  38. """
  39. import datetime
  40. import glob
  41. import optparse
  42. import os
  43. import posixpath
  44. import shutil
  45. import subprocess
  46. import sys
  47. import tempfile
  48. import time
  49. import traceback
  50. ROOT_PLAYBACK_DIR_NAME = 'playback'
  51. SKPICTURES_DIR_NAME = 'skps'
  52. GS_PREFIX = 'gs://'
  53. PARTNERS_GS_BUCKET = 'gs://chrome-partner-telemetry'
  54. # Local archive and SKP directories.
  55. LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR = os.path.join(
  56. os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data')
  57. TMP_SKP_DIR = tempfile.mkdtemp()
  58. # Location of the credentials.json file and the string that represents missing
  59. # passwords.
  60. CREDENTIALS_FILE_PATH = os.path.join(
  61. os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data',
  62. 'credentials.json'
  63. )
  64. # Name of the SKP benchmark
  65. SKP_BENCHMARK = 'skpicture_printer'
  66. # The max base name length of Skp files.
  67. MAX_SKP_BASE_NAME_LEN = 31
  68. # Dictionary of device to platform prefixes for SKP files.
  69. DEVICE_TO_PLATFORM_PREFIX = {
  70. 'desktop': 'desk',
  71. 'mobile': 'mobi',
  72. 'tablet': 'tabl'
  73. }
  74. # How many times the record_wpr binary should be retried.
  75. RETRY_RECORD_WPR_COUNT = 5
  76. # How many times the run_benchmark binary should be retried.
  77. RETRY_RUN_MEASUREMENT_COUNT = 3
  78. # Location of the credentials.json file in Google Storage.
  79. CREDENTIALS_GS_PATH = 'playback/credentials/credentials.json'
  80. X11_DISPLAY = os.getenv('DISPLAY', ':0')
  81. # Path to Chromium's page sets.
  82. CHROMIUM_PAGE_SETS_PATH = os.path.join('tools', 'perf', 'page_sets')
  83. # Dictionary of supported Chromium page sets to their file prefixes.
  84. CHROMIUM_PAGE_SETS_TO_PREFIX = {
  85. }
  86. PAGE_SETS_TO_EXCLUSIONS = {
  87. # See skbug.com/7348
  88. 'key_mobile_sites_smooth.py': '"(digg|worldjournal|twitter|espn)"',
  89. # See skbug.com/7421
  90. 'top_25_smooth.py': '"(mail\.google\.com)"',
  91. }
  92. class InvalidSKPException(Exception):
  93. """Raised when the created SKP is invalid."""
  94. pass
  95. def remove_prefix(s, prefix):
  96. if s.startswith(prefix):
  97. return s[len(prefix):]
  98. return s
  99. class SkPicturePlayback(object):
  100. """Class that archives or replays webpages and creates SKPs."""
  101. def __init__(self, parse_options):
  102. """Constructs a SkPicturePlayback BuildStep instance."""
  103. assert parse_options.browser_executable, 'Must specify --browser_executable'
  104. self._browser_executable = parse_options.browser_executable
  105. self._browser_args = '--disable-setuid-sandbox'
  106. if parse_options.browser_extra_args:
  107. self._browser_args = '%s %s' % (
  108. self._browser_args, parse_options.browser_extra_args)
  109. self._chrome_page_sets_path = os.path.join(parse_options.chrome_src_path,
  110. CHROMIUM_PAGE_SETS_PATH)
  111. self._all_page_sets_specified = parse_options.page_sets == 'all'
  112. self._page_sets = self._ParsePageSets(parse_options.page_sets)
  113. self._record = parse_options.record
  114. self._skia_tools = parse_options.skia_tools
  115. self._non_interactive = parse_options.non_interactive
  116. self._upload = parse_options.upload
  117. self._skp_prefix = parse_options.skp_prefix
  118. data_store_location = parse_options.data_store
  119. if data_store_location.startswith(GS_PREFIX):
  120. self.gs = GoogleStorageDataStore(data_store_location)
  121. else:
  122. self.gs = LocalFileSystemDataStore(data_store_location)
  123. self._upload_to_partner_bucket = parse_options.upload_to_partner_bucket
  124. self._alternate_upload_dir = parse_options.alternate_upload_dir
  125. self._telemetry_binaries_dir = os.path.join(parse_options.chrome_src_path,
  126. 'tools', 'perf')
  127. self._catapult_dir = os.path.join(parse_options.chrome_src_path,
  128. 'third_party', 'catapult')
  129. self._local_skp_dir = os.path.join(
  130. parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, SKPICTURES_DIR_NAME)
  131. self._local_record_webpages_archive_dir = os.path.join(
  132. parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive')
  133. # List of SKP files generated by this script.
  134. self._skp_files = []
  135. def _ParsePageSets(self, page_sets):
  136. if not page_sets:
  137. raise ValueError('Must specify at least one page_set!')
  138. elif self._all_page_sets_specified:
  139. # Get everything from the page_sets directory.
  140. page_sets_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
  141. 'page_sets')
  142. ps = [os.path.join(page_sets_dir, page_set)
  143. for page_set in os.listdir(page_sets_dir)
  144. if not os.path.isdir(os.path.join(page_sets_dir, page_set)) and
  145. page_set.endswith('.py')]
  146. chromium_ps = [
  147. os.path.join(self._chrome_page_sets_path, cr_page_set)
  148. for cr_page_set in CHROMIUM_PAGE_SETS_TO_PREFIX]
  149. ps.extend(chromium_ps)
  150. elif '*' in page_sets:
  151. # Explode and return the glob.
  152. ps = glob.glob(page_sets)
  153. else:
  154. ps = page_sets.split(',')
  155. ps.sort()
  156. return ps
  157. def _IsChromiumPageSet(self, page_set):
  158. """Returns true if the specified page set is a Chromium page set."""
  159. return page_set.startswith(self._chrome_page_sets_path)
  160. def Run(self):
  161. """Run the SkPicturePlayback BuildStep."""
  162. # Download the credentials file if it was not previously downloaded.
  163. if not os.path.isfile(CREDENTIALS_FILE_PATH):
  164. # Download the credentials.json file from Google Storage.
  165. self.gs.download_file(CREDENTIALS_GS_PATH, CREDENTIALS_FILE_PATH)
  166. if not os.path.isfile(CREDENTIALS_FILE_PATH):
  167. raise Exception("""Could not locate credentials file in the storage.
  168. Please create a credentials file in gs://%s that contains:
  169. {
  170. "google": {
  171. "username": "google_testing_account_username",
  172. "password": "google_testing_account_password"
  173. }
  174. }\n\n""" % CREDENTIALS_GS_PATH)
  175. # Delete any left over data files in the data directory.
  176. for archive_file in glob.glob(
  177. os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')):
  178. os.remove(archive_file)
  179. # Create the required local storage directories.
  180. self._CreateLocalStorageDirs()
  181. # Start the timer.
  182. start_time = time.time()
  183. # Loop through all page_sets.
  184. for page_set in self._page_sets:
  185. if os.path.basename(page_set) == '__init__.py':
  186. continue
  187. page_set_basename = os.path.basename(page_set).split('.')[0]
  188. page_set_json_name = page_set_basename + '.json'
  189. wpr_data_file_glob = (
  190. page_set.split(os.path.sep)[-1].split('.')[0] + '_*.wprgo')
  191. page_set_dir = os.path.dirname(page_set)
  192. if self._IsChromiumPageSet(page_set):
  193. print 'Using Chromium\'s captured archives for Chromium\'s page sets.'
  194. elif self._record:
  195. # Create an archive of the specified webpages if '--record=True' is
  196. # specified.
  197. record_wpr_cmd = (
  198. 'PYTHONPATH=%s:%s:$PYTHONPATH' % (page_set_dir, self._catapult_dir),
  199. 'DISPLAY=%s' % X11_DISPLAY,
  200. os.path.join(self._telemetry_binaries_dir, 'record_wpr'),
  201. '--extra-browser-args="%s"' % self._browser_args,
  202. '--browser=exact',
  203. '--browser-executable=%s' % self._browser_executable,
  204. '%s_page_set' % page_set_basename,
  205. '--page-set-base-dir=%s' % page_set_dir
  206. )
  207. for _ in range(RETRY_RECORD_WPR_COUNT):
  208. try:
  209. subprocess.check_call(' '.join(record_wpr_cmd), shell=True)
  210. # Copy over the created archive into the local webpages archive
  211. # directory.
  212. for wpr_data_file in glob.glob(os.path.join(
  213. LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file_glob)):
  214. shutil.copy(
  215. os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file),
  216. self._local_record_webpages_archive_dir)
  217. shutil.copy(
  218. os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
  219. page_set_json_name),
  220. self._local_record_webpages_archive_dir)
  221. # Break out of the retry loop since there were no errors.
  222. break
  223. except Exception:
  224. # There was a failure continue with the loop.
  225. traceback.print_exc()
  226. else:
  227. # If we get here then record_wpr did not succeed and thus did not
  228. # break out of the loop.
  229. raise Exception('record_wpr failed for page_set: %s' % page_set)
  230. else:
  231. # Get the webpages archive so that it can be replayed.
  232. self._DownloadWebpagesArchive(wpr_data_file_glob, page_set_json_name)
  233. run_benchmark_cmd = [
  234. 'PYTHONPATH=%s:%s:$PYTHONPATH' % (page_set_dir, self._catapult_dir),
  235. 'DISPLAY=%s' % X11_DISPLAY,
  236. 'timeout', '1800',
  237. os.path.join(self._telemetry_binaries_dir, 'run_benchmark'),
  238. '--extra-browser-args="%s"' % self._browser_args,
  239. '--browser=exact',
  240. '--browser-executable=%s' % self._browser_executable,
  241. SKP_BENCHMARK,
  242. '--page-set-name=%s' % page_set_basename,
  243. '--page-set-base-dir=%s' % page_set_dir,
  244. '--skp-outdir=%s' % TMP_SKP_DIR,
  245. '--also-run-disabled-tests',
  246. ]
  247. exclusions = PAGE_SETS_TO_EXCLUSIONS.get(os.path.basename(page_set))
  248. if exclusions:
  249. run_benchmark_cmd.append('--story-filter-exclude=' + exclusions)
  250. for _ in range(RETRY_RUN_MEASUREMENT_COUNT):
  251. try:
  252. print '\n\n=======Capturing SKP of %s=======\n\n' % page_set
  253. subprocess.check_call(' '.join(run_benchmark_cmd), shell=True)
  254. except subprocess.CalledProcessError:
  255. # There was a failure continue with the loop.
  256. traceback.print_exc()
  257. print '\n\n=======Retrying %s=======\n\n' % page_set
  258. time.sleep(10)
  259. continue
  260. try:
  261. # Rename generated SKP files into more descriptive names.
  262. self._RenameSkpFiles(page_set)
  263. except InvalidSKPException:
  264. # There was a failure continue with the loop.
  265. traceback.print_exc()
  266. print '\n\n=======Retrying %s=======\n\n' % page_set
  267. time.sleep(10)
  268. continue
  269. # Break out of the retry loop since there were no errors.
  270. break
  271. else:
  272. # If we get here then run_benchmark did not succeed and thus did not
  273. # break out of the loop.
  274. raise Exception('run_benchmark failed for page_set: %s' % page_set)
  275. print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % (
  276. time.time() - start_time)
  277. if self._skia_tools:
  278. render_pictures_cmd = [
  279. os.path.join(self._skia_tools, 'render_pictures'),
  280. '-r', self._local_skp_dir
  281. ]
  282. render_pdfs_cmd = [
  283. os.path.join(self._skia_tools, 'render_pdfs'),
  284. '-r', self._local_skp_dir
  285. ]
  286. for tools_cmd in (render_pictures_cmd, render_pdfs_cmd):
  287. print '\n\n=======Running %s=======' % ' '.join(tools_cmd)
  288. subprocess.check_call(tools_cmd)
  289. if not self._non_interactive:
  290. print '\n\n=======Running debugger======='
  291. os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'),
  292. self._local_skp_dir))
  293. print '\n\n'
  294. if self._upload:
  295. print '\n\n=======Uploading to %s=======\n\n' % self.gs.target_type()
  296. # Copy the directory structure in the root directory into Google Storage.
  297. dest_dir_name = ROOT_PLAYBACK_DIR_NAME
  298. if self._alternate_upload_dir:
  299. dest_dir_name = self._alternate_upload_dir
  300. self.gs.upload_dir_contents(
  301. self._local_skp_dir, dest_dir=dest_dir_name)
  302. print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
  303. posixpath.join(self.gs.target_name(), dest_dir_name,
  304. SKPICTURES_DIR_NAME))
  305. else:
  306. print '\n\n=======Not Uploading to %s=======\n\n' % self.gs.target_type()
  307. print 'Generated resources are available in %s\n\n' % (
  308. self._local_skp_dir)
  309. if self._upload_to_partner_bucket:
  310. print '\n\n=======Uploading to Partner bucket %s =======\n\n' % (
  311. PARTNERS_GS_BUCKET)
  312. partner_gs = GoogleStorageDataStore(PARTNERS_GS_BUCKET)
  313. timestamp = datetime.datetime.utcnow().strftime('%Y-%m-%d')
  314. upload_dir = posixpath.join(SKPICTURES_DIR_NAME, timestamp)
  315. try:
  316. partner_gs.delete_path(upload_dir)
  317. except subprocess.CalledProcessError:
  318. print 'Cannot delete %s because it does not exist yet.' % upload_dir
  319. print 'Uploading %s to %s' % (self._local_skp_dir, upload_dir)
  320. partner_gs.upload_dir_contents(self._local_skp_dir, upload_dir)
  321. print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
  322. posixpath.join(partner_gs.target_name(), upload_dir))
  323. return 0
  324. def _GetSkiaSkpFileName(self, page_set):
  325. """Returns the SKP file name for Skia page sets."""
  326. # /path/to/skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop.py
  327. ps_filename = os.path.basename(page_set)
  328. # skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop
  329. ps_basename, _ = os.path.splitext(ps_filename)
  330. # skia_yahooanswers_desktop -> skia, yahooanswers, desktop
  331. _, page_name, device = ps_basename.split('_')
  332. basename = '%s_%s' % (DEVICE_TO_PLATFORM_PREFIX[device], page_name)
  333. return basename[:MAX_SKP_BASE_NAME_LEN] + '.skp'
  334. def _GetChromiumSkpFileName(self, page_set, site):
  335. """Returns the SKP file name for Chromium page sets."""
  336. # /path/to/http___mobile_news_sandbox_pt0 -> http___mobile_news_sandbox_pt0
  337. _, webpage = os.path.split(site)
  338. # http___mobile_news_sandbox_pt0 -> mobile_news_sandbox_pt0
  339. for prefix in ('http___', 'https___', 'www_'):
  340. if webpage.startswith(prefix):
  341. webpage = webpage[len(prefix):]
  342. # /path/to/skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop.py
  343. ps_filename = os.path.basename(page_set)
  344. # http___mobile_news_sandbox -> pagesetprefix_http___mobile_news_sandbox
  345. basename = '%s_%s' % (CHROMIUM_PAGE_SETS_TO_PREFIX[ps_filename], webpage)
  346. return basename[:MAX_SKP_BASE_NAME_LEN] + '.skp'
  347. def _RenameSkpFiles(self, page_set):
  348. """Rename generated SKP files into more descriptive names.
  349. Look into the subdirectory of TMP_SKP_DIR and find the most interesting
  350. .skp in there to be this page_set's representative .skp.
  351. Throws InvalidSKPException if the chosen .skp is less than 1KB. This
  352. typically happens when there is a 404 or a redirect loop. Anything greater
  353. than 1KB seems to have captured at least some useful information.
  354. """
  355. subdirs = glob.glob(os.path.join(TMP_SKP_DIR, '*'))
  356. for site in subdirs:
  357. if self._IsChromiumPageSet(page_set):
  358. filename = self._GetChromiumSkpFileName(page_set, site)
  359. else:
  360. filename = self._GetSkiaSkpFileName(page_set)
  361. filename = filename.lower()
  362. if self._skp_prefix:
  363. filename = '%s%s' % (self._skp_prefix, filename)
  364. # We choose the largest .skp as the most likely to be interesting.
  365. largest_skp = max(glob.glob(os.path.join(site, '*.skp')),
  366. key=lambda path: os.stat(path).st_size)
  367. dest = os.path.join(self._local_skp_dir, filename)
  368. print 'Moving', largest_skp, 'to', dest
  369. shutil.move(largest_skp, dest)
  370. self._skp_files.append(filename)
  371. shutil.rmtree(site)
  372. skp_size = os.path.getsize(dest)
  373. if skp_size < 1024:
  374. raise InvalidSKPException(
  375. 'Size of %s is only %d. Something is wrong.' % (dest, skp_size))
  376. def _CreateLocalStorageDirs(self):
  377. """Creates required local storage directories for this script."""
  378. for d in (self._local_record_webpages_archive_dir,
  379. self._local_skp_dir):
  380. if os.path.exists(d):
  381. shutil.rmtree(d)
  382. os.makedirs(d)
  383. def _DownloadWebpagesArchive(self, wpr_data_file, page_set_json_name):
  384. """Downloads the webpages archive and its required page set from GS."""
  385. wpr_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME, 'webpages_archive',
  386. wpr_data_file)
  387. page_set_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME,
  388. 'webpages_archive',
  389. page_set_json_name)
  390. gs = self.gs
  391. if (gs.does_storage_object_exist(wpr_source) and
  392. gs.does_storage_object_exist(page_set_source)):
  393. gs.download_file(wpr_source, LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR)
  394. gs.download_file(page_set_source,
  395. os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
  396. page_set_json_name))
  397. else:
  398. raise Exception('%s and %s do not exist in %s!' % (gs.target_type(),
  399. wpr_source, page_set_source))
  400. class DataStore:
  401. """An abstract base class for uploading recordings to a data storage.
  402. The interface emulates the google storage api."""
  403. def target_name(self):
  404. raise NotImplementedError()
  405. def target_type(self):
  406. raise NotImplementedError()
  407. def does_storage_object_exist(self, name):
  408. raise NotImplementedError()
  409. def download_file(self, name, local_path):
  410. raise NotImplementedError()
  411. def upload_dir_contents(self, source_dir, dest_dir):
  412. raise NotImplementedError()
  413. class GoogleStorageDataStore(DataStore):
  414. def __init__(self, data_store_url):
  415. self._url = data_store_url.rstrip('/')
  416. def target_name(self):
  417. return self._url
  418. def target_type(self):
  419. return 'Google Storage'
  420. def does_storage_object_exist(self, name):
  421. try:
  422. output = subprocess.check_output([
  423. 'gsutil', 'ls', '/'.join((self._url, name))])
  424. except subprocess.CalledProcessError:
  425. return False
  426. if len(output.splitlines()) != 1:
  427. return False
  428. return True
  429. def delete_path(self, path):
  430. subprocess.check_call(['gsutil', 'rm', '-r', '/'.join((self._url, path))])
  431. def download_file(self, name, local_path):
  432. subprocess.check_call([
  433. 'gsutil', 'cp', '/'.join((self._url, name)), local_path])
  434. def upload_dir_contents(self, source_dir, dest_dir):
  435. subprocess.check_call([
  436. 'gsutil', 'cp', '-r', source_dir, '/'.join((self._url, dest_dir))])
  437. class LocalFileSystemDataStore(DataStore):
  438. def __init__(self, data_store_location):
  439. self._base_dir = data_store_location
  440. def target_name(self):
  441. return self._base_dir
  442. def target_type(self):
  443. return self._base_dir
  444. def does_storage_object_exist(self, name):
  445. return os.path.isfile(os.path.join(self._base_dir, name))
  446. def delete_path(self, path):
  447. shutil.rmtree(path)
  448. def download_file(self, name, local_path):
  449. shutil.copyfile(os.path.join(self._base_dir, name), local_path)
  450. def upload_dir_contents(self, source_dir, dest_dir):
  451. def copytree(source_dir, dest_dir):
  452. if not os.path.exists(dest_dir):
  453. os.makedirs(dest_dir)
  454. for item in os.listdir(source_dir):
  455. source = os.path.join(source_dir, item)
  456. dest = os.path.join(dest_dir, item)
  457. if os.path.isdir(source):
  458. copytree(source, dest)
  459. else:
  460. shutil.copy2(source, dest)
  461. copytree(source_dir, os.path.join(self._base_dir, dest_dir))
  462. if '__main__' == __name__:
  463. option_parser = optparse.OptionParser()
  464. option_parser.add_option(
  465. '', '--page_sets',
  466. help='Specifies the page sets to use to archive. Supports globs.',
  467. default='all')
  468. option_parser.add_option(
  469. '', '--record', action='store_true',
  470. help='Specifies whether a new website archive should be created.',
  471. default=False)
  472. option_parser.add_option(
  473. '', '--skia_tools',
  474. help=('Path to compiled Skia executable tools. '
  475. 'render_pictures/render_pdfs is run on the set '
  476. 'after all SKPs are captured. If the script is run without '
  477. '--non-interactive then the debugger is also run at the end. Debug '
  478. 'builds are recommended because they seem to catch more failures '
  479. 'than Release builds.'),
  480. default=None)
  481. option_parser.add_option(
  482. '', '--upload', action='store_true',
  483. help=('Uploads to Google Storage or copies to local filesystem storage '
  484. ' if this is True.'),
  485. default=False)
  486. option_parser.add_option(
  487. '', '--upload_to_partner_bucket', action='store_true',
  488. help=('Uploads SKPs to the chrome-partner-telemetry Google Storage '
  489. 'bucket if true.'),
  490. default=False)
  491. option_parser.add_option(
  492. '', '--data_store',
  493. help=('The location of the file storage to use to download and upload '
  494. 'files. Can be \'gs://<bucket>\' for Google Storage, or '
  495. 'a directory for local filesystem storage'),
  496. default='gs://skia-skps')
  497. option_parser.add_option(
  498. '', '--alternate_upload_dir',
  499. help= ('Uploads to a different directory in Google Storage or local '
  500. 'storage if this flag is specified'),
  501. default=None)
  502. option_parser.add_option(
  503. '', '--output_dir',
  504. help=('Temporary directory where SKPs and webpage archives will be '
  505. 'outputted to.'),
  506. default=tempfile.gettempdir())
  507. option_parser.add_option(
  508. '', '--browser_executable',
  509. help='The exact browser executable to run.',
  510. default=None)
  511. option_parser.add_option(
  512. '', '--browser_extra_args',
  513. help='Additional arguments to pass to the browser.',
  514. default=None)
  515. option_parser.add_option(
  516. '', '--chrome_src_path',
  517. help='Path to the chromium src directory.',
  518. default=None)
  519. option_parser.add_option(
  520. '', '--non-interactive', action='store_true',
  521. help='Runs the script without any prompts. If this flag is specified and '
  522. '--skia_tools is specified then the debugger is not run.',
  523. default=False)
  524. option_parser.add_option(
  525. '', '--skp_prefix',
  526. help='Prefix to add to the names of generated SKPs.',
  527. default=None)
  528. options, unused_args = option_parser.parse_args()
  529. playback = SkPicturePlayback(options)
  530. sys.exit(playback.Run())