123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157 |
- #!/usr/bin/env python
- # Copyright 2014 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- # This script computs the number of concurrent links we want to run in the build
- # as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP.
- from __future__ import print_function
- import argparse
- import multiprocessing
- import os
- import re
- import subprocess
- import sys
- sys.path.insert(1, os.path.join(os.path.dirname(__file__), '..'))
- import gn_helpers
- def _GetTotalMemoryInBytes():
- if sys.platform in ('win32', 'cygwin'):
- import ctypes
- class MEMORYSTATUSEX(ctypes.Structure):
- _fields_ = [
- ("dwLength", ctypes.c_ulong),
- ("dwMemoryLoad", ctypes.c_ulong),
- ("ullTotalPhys", ctypes.c_ulonglong),
- ("ullAvailPhys", ctypes.c_ulonglong),
- ("ullTotalPageFile", ctypes.c_ulonglong),
- ("ullAvailPageFile", ctypes.c_ulonglong),
- ("ullTotalVirtual", ctypes.c_ulonglong),
- ("ullAvailVirtual", ctypes.c_ulonglong),
- ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
- ]
- stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
- ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
- return stat.ullTotalPhys
- elif sys.platform.startswith('linux'):
- if os.path.exists("/proc/meminfo"):
- with open("/proc/meminfo") as meminfo:
- memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
- for line in meminfo:
- match = memtotal_re.match(line)
- if not match:
- continue
- return float(match.group(1)) * 2**10
- elif sys.platform == 'darwin':
- try:
- return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
- except Exception:
- return 0
- # TODO(scottmg): Implement this for other platforms.
- return 0
- def _GetDefaultConcurrentLinks(per_link_gb, reserve_gb, thin_lto_type,
- secondary_per_link_gb, override_ram_in_gb):
- explanation = []
- explanation.append(
- 'per_link_gb={} reserve_gb={} secondary_per_link_gb={}'.format(
- per_link_gb, reserve_gb, secondary_per_link_gb))
- if override_ram_in_gb:
- mem_total_gb = override_ram_in_gb
- else:
- mem_total_gb = float(_GetTotalMemoryInBytes()) / 2**30
- mem_total_gb = max(0, mem_total_gb - reserve_gb)
- # Ensure that there is at least as many links allocated for the secondary as
- # there is for the primary. The secondary link usually uses fewer gbs.
- mem_cap = int(max(1, mem_total_gb / (per_link_gb + secondary_per_link_gb)))
- try:
- cpu_count = multiprocessing.cpu_count()
- except:
- cpu_count = 1
- # A local LTO links saturate all cores, but only for some amount of the link.
- # Goma LTO runs LTO codegen on goma, only run one of these tasks at once.
- cpu_cap = cpu_count
- if thin_lto_type is not None:
- if thin_lto_type == 'goma':
- cpu_cap = 1
- else:
- assert thin_lto_type == 'local'
- cpu_cap = min(cpu_count, 6)
- explanation.append('cpu_count={} cpu_cap={} mem_total_gb={:.1f}GiB'.format(
- cpu_count, cpu_cap, mem_total_gb))
- num_links = min(mem_cap, cpu_cap)
- if num_links == cpu_cap:
- if cpu_cap == cpu_count:
- reason = 'cpu_count'
- else:
- reason = 'cpu_cap (thinlto)'
- else:
- reason = 'RAM'
- # static link see too many open files if we have many concurrent links.
- # ref: http://b/233068481
- if num_links > 30:
- num_links = 30
- reason = 'nofile'
- explanation.append('concurrent_links={} (reason: {})'.format(
- num_links, reason))
- # Use remaining RAM for a secondary pool if needed.
- if secondary_per_link_gb:
- mem_remaining = mem_total_gb - num_links * per_link_gb
- secondary_size = int(max(0, mem_remaining / secondary_per_link_gb))
- if secondary_size > cpu_count:
- secondary_size = cpu_count
- reason = 'cpu_count'
- else:
- reason = 'mem_remaining={:.1f}GiB'.format(mem_remaining)
- explanation.append('secondary_size={} (reason: {})'.format(
- secondary_size, reason))
- else:
- secondary_size = 0
- return num_links, secondary_size, explanation
- def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--mem_per_link_gb', type=int, default=8)
- parser.add_argument('--reserve_mem_gb', type=int, default=0)
- parser.add_argument('--secondary_mem_per_link', type=int, default=0)
- parser.add_argument('--override-ram-in-gb-for-testing', type=float, default=0)
- parser.add_argument('--thin-lto')
- options = parser.parse_args()
- primary_pool_size, secondary_pool_size, explanation = (
- _GetDefaultConcurrentLinks(options.mem_per_link_gb,
- options.reserve_mem_gb, options.thin_lto,
- options.secondary_mem_per_link,
- options.override_ram_in_gb_for_testing))
- if options.override_ram_in_gb_for_testing:
- print('primary={} secondary={}'.format(primary_pool_size,
- secondary_pool_size))
- else:
- sys.stdout.write(
- gn_helpers.ToGNString({
- 'primary_pool_size': primary_pool_size,
- 'secondary_pool_size': secondary_pool_size,
- 'explanation': explanation,
- }))
- return 0
- if __name__ == '__main__':
- sys.exit(main())
|