get_concurrent_links.py 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157
  1. #!/usr/bin/env python
  2. # Copyright 2014 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. # This script computs the number of concurrent links we want to run in the build
  6. # as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP.
  7. from __future__ import print_function
  8. import argparse
  9. import multiprocessing
  10. import os
  11. import re
  12. import subprocess
  13. import sys
  14. sys.path.insert(1, os.path.join(os.path.dirname(__file__), '..'))
  15. import gn_helpers
  16. def _GetTotalMemoryInBytes():
  17. if sys.platform in ('win32', 'cygwin'):
  18. import ctypes
  19. class MEMORYSTATUSEX(ctypes.Structure):
  20. _fields_ = [
  21. ("dwLength", ctypes.c_ulong),
  22. ("dwMemoryLoad", ctypes.c_ulong),
  23. ("ullTotalPhys", ctypes.c_ulonglong),
  24. ("ullAvailPhys", ctypes.c_ulonglong),
  25. ("ullTotalPageFile", ctypes.c_ulonglong),
  26. ("ullAvailPageFile", ctypes.c_ulonglong),
  27. ("ullTotalVirtual", ctypes.c_ulonglong),
  28. ("ullAvailVirtual", ctypes.c_ulonglong),
  29. ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
  30. ]
  31. stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
  32. ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
  33. return stat.ullTotalPhys
  34. elif sys.platform.startswith('linux'):
  35. if os.path.exists("/proc/meminfo"):
  36. with open("/proc/meminfo") as meminfo:
  37. memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
  38. for line in meminfo:
  39. match = memtotal_re.match(line)
  40. if not match:
  41. continue
  42. return float(match.group(1)) * 2**10
  43. elif sys.platform == 'darwin':
  44. try:
  45. return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
  46. except Exception:
  47. return 0
  48. # TODO(scottmg): Implement this for other platforms.
  49. return 0
  50. def _GetDefaultConcurrentLinks(per_link_gb, reserve_gb, thin_lto_type,
  51. secondary_per_link_gb, override_ram_in_gb):
  52. explanation = []
  53. explanation.append(
  54. 'per_link_gb={} reserve_gb={} secondary_per_link_gb={}'.format(
  55. per_link_gb, reserve_gb, secondary_per_link_gb))
  56. if override_ram_in_gb:
  57. mem_total_gb = override_ram_in_gb
  58. else:
  59. mem_total_gb = float(_GetTotalMemoryInBytes()) / 2**30
  60. mem_total_gb = max(0, mem_total_gb - reserve_gb)
  61. # Ensure that there is at least as many links allocated for the secondary as
  62. # there is for the primary. The secondary link usually uses fewer gbs.
  63. mem_cap = int(max(1, mem_total_gb / (per_link_gb + secondary_per_link_gb)))
  64. try:
  65. cpu_count = multiprocessing.cpu_count()
  66. except:
  67. cpu_count = 1
  68. # A local LTO links saturate all cores, but only for some amount of the link.
  69. # Goma LTO runs LTO codegen on goma, only run one of these tasks at once.
  70. cpu_cap = cpu_count
  71. if thin_lto_type is not None:
  72. if thin_lto_type == 'goma':
  73. cpu_cap = 1
  74. else:
  75. assert thin_lto_type == 'local'
  76. cpu_cap = min(cpu_count, 6)
  77. explanation.append('cpu_count={} cpu_cap={} mem_total_gb={:.1f}GiB'.format(
  78. cpu_count, cpu_cap, mem_total_gb))
  79. num_links = min(mem_cap, cpu_cap)
  80. if num_links == cpu_cap:
  81. if cpu_cap == cpu_count:
  82. reason = 'cpu_count'
  83. else:
  84. reason = 'cpu_cap (thinlto)'
  85. else:
  86. reason = 'RAM'
  87. # static link see too many open files if we have many concurrent links.
  88. # ref: http://b/233068481
  89. if num_links > 30:
  90. num_links = 30
  91. reason = 'nofile'
  92. explanation.append('concurrent_links={} (reason: {})'.format(
  93. num_links, reason))
  94. # Use remaining RAM for a secondary pool if needed.
  95. if secondary_per_link_gb:
  96. mem_remaining = mem_total_gb - num_links * per_link_gb
  97. secondary_size = int(max(0, mem_remaining / secondary_per_link_gb))
  98. if secondary_size > cpu_count:
  99. secondary_size = cpu_count
  100. reason = 'cpu_count'
  101. else:
  102. reason = 'mem_remaining={:.1f}GiB'.format(mem_remaining)
  103. explanation.append('secondary_size={} (reason: {})'.format(
  104. secondary_size, reason))
  105. else:
  106. secondary_size = 0
  107. return num_links, secondary_size, explanation
  108. def main():
  109. parser = argparse.ArgumentParser()
  110. parser.add_argument('--mem_per_link_gb', type=int, default=8)
  111. parser.add_argument('--reserve_mem_gb', type=int, default=0)
  112. parser.add_argument('--secondary_mem_per_link', type=int, default=0)
  113. parser.add_argument('--override-ram-in-gb-for-testing', type=float, default=0)
  114. parser.add_argument('--thin-lto')
  115. options = parser.parse_args()
  116. primary_pool_size, secondary_pool_size, explanation = (
  117. _GetDefaultConcurrentLinks(options.mem_per_link_gb,
  118. options.reserve_mem_gb, options.thin_lto,
  119. options.secondary_mem_per_link,
  120. options.override_ram_in_gb_for_testing))
  121. if options.override_ram_in_gb_for_testing:
  122. print('primary={} secondary={}'.format(primary_pool_size,
  123. secondary_pool_size))
  124. else:
  125. sys.stdout.write(
  126. gn_helpers.ToGNString({
  127. 'primary_pool_size': primary_pool_size,
  128. 'secondary_pool_size': secondary_pool_size,
  129. 'explanation': explanation,
  130. }))
  131. return 0
  132. if __name__ == '__main__':
  133. sys.exit(main())