update_pgo_profiles.py 4.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154
  1. #!/usr/bin/env python
  2. # Copyright 2020 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Downloads pgo profiles for optimizing official Chrome.
  6. This script has the following responsibilities:
  7. 1. Download a requested profile if necessary.
  8. 2. Return a path to the current profile to feed to the build system.
  9. 3. Removed stale profiles (2 days) to save disk spaces because profiles are
  10. large (~1GB) and updated frequently (~4 times a day).
  11. """
  12. from __future__ import print_function
  13. import argparse
  14. import os
  15. import sys
  16. import time
  17. _SRC_ROOT = os.path.abspath(
  18. os.path.join(os.path.dirname(__file__), os.path.pardir))
  19. sys.path.append(os.path.join(_SRC_ROOT, 'third_party', 'depot_tools'))
  20. import download_from_google_storage
  21. sys.path.append(os.path.join(_SRC_ROOT, 'build'))
  22. import gn_helpers
  23. # Absolute path to the directory that stores pgo related state files, which
  24. # specifcies which profile to update and use.
  25. _PGO_DIR = os.path.join(_SRC_ROOT, 'chrome', 'build')
  26. # Absolute path to the directory that stores pgo profiles.
  27. _PGO_PROFILE_DIR = os.path.join(_PGO_DIR, 'pgo_profiles')
  28. def _read_profile_name(target):
  29. """Read profile name given a target.
  30. Args:
  31. target(str): The target name, such as win32, mac.
  32. Returns:
  33. Name of the profile to update and use, such as:
  34. chrome-win32-master-67ad3c89d2017131cc9ce664a1580315517550d1.profdata.
  35. """
  36. state_file = os.path.join(_PGO_DIR, '%s.pgo.txt' % target)
  37. with open(state_file, 'r') as f:
  38. profile_name = f.read().strip()
  39. return profile_name
  40. def _remove_unused_profiles(current_profile_name):
  41. """Removes unused profiles, except the current one, to save disk space."""
  42. days = 2
  43. expiration_duration = 60 * 60 * 24 * days
  44. for f in os.listdir(_PGO_PROFILE_DIR):
  45. if f == current_profile_name:
  46. continue
  47. p = os.path.join(_PGO_PROFILE_DIR, f)
  48. age = time.time() - os.path.getmtime(p)
  49. if age > expiration_duration:
  50. print('Removing profile %s as it hasn\'t been used in the past %d days' %
  51. (p, days))
  52. os.remove(p)
  53. def _update(args):
  54. """Update profile if necessary according to the state file.
  55. Args:
  56. args(dict): A dict of cmd arguments, such as target and gs_url_base.
  57. Raises:
  58. RuntimeError: If failed to download profiles from gcs.
  59. """
  60. profile_name = _read_profile_name(args.target)
  61. profile_path = os.path.join(_PGO_PROFILE_DIR, profile_name)
  62. if os.path.isfile(profile_path):
  63. os.utime(profile_path, None)
  64. return
  65. gsutil = download_from_google_storage.Gsutil(
  66. download_from_google_storage.GSUTIL_DEFAULT_PATH)
  67. gs_path = 'gs://' + args.gs_url_base.strip('/') + '/' + profile_name
  68. code = gsutil.call('cp', gs_path, profile_path)
  69. if code != 0:
  70. raise RuntimeError('gsutil failed to download "%s"' % gs_path)
  71. _remove_unused_profiles(profile_name)
  72. def _get_profile_path(args):
  73. """Returns an absolute path to the current profile.
  74. Args:
  75. args(dict): A dict of cmd arguments, such as target and gs_url_base.
  76. Raises:
  77. RuntimeError: If the current profile is missing.
  78. """
  79. profile_path = os.path.join(_PGO_PROFILE_DIR, _read_profile_name(args.target))
  80. if not os.path.isfile(profile_path):
  81. raise RuntimeError(
  82. 'requested profile "%s" doesn\'t exist, please make sure '
  83. '"checkout_pgo_profiles" is set to True in the "custom_vars" section '
  84. 'of your .gclient file, e.g.: \n'
  85. 'solutions = [ \n'
  86. ' { \n'
  87. ' "name": "src", \n'
  88. ' # ... \n'
  89. ' "custom_vars": { \n'
  90. ' "checkout_pgo_profiles": True, \n'
  91. ' }, \n'
  92. ' }, \n'
  93. '], \n'
  94. 'and then run "gclient runhooks" to download it. You can also simply '
  95. 'disable the PGO optimizations by setting |chrome_pgo_phase = 0| in '
  96. 'your GN arguments.'%
  97. profile_path)
  98. os.utime(profile_path, None)
  99. profile_path.rstrip(os.sep)
  100. print(gn_helpers.ToGNString(profile_path))
  101. def main():
  102. parser = argparse.ArgumentParser(
  103. description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
  104. parser.add_argument(
  105. '--target',
  106. required=True,
  107. choices=['win32', 'win64', 'mac', 'mac-arm', 'linux'],
  108. help='Identifier of a specific target platform + architecture.')
  109. subparsers = parser.add_subparsers()
  110. parser_update = subparsers.add_parser('update')
  111. parser_update.add_argument(
  112. '--gs-url-base',
  113. required=True,
  114. help='The base GS URL to search for the profile.')
  115. parser_update.set_defaults(func=_update)
  116. parser_get_profile_path = subparsers.add_parser('get_profile_path')
  117. parser_get_profile_path.set_defaults(func=_get_profile_path)
  118. args = parser.parse_args()
  119. return args.func(args)
  120. if __name__ == '__main__':
  121. sys.exit(main())