xcode_util.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355
  1. # Copyright 2020 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. import distutils.version
  5. import glob
  6. import logging
  7. import os
  8. import shutil
  9. import subprocess
  10. import test_runner_errors
  11. LOGGER = logging.getLogger(__name__)
  12. XcodeIOSSimulatorDefaultRuntimeFilename = 'iOS.simruntime'
  13. XcodeIOSSimulatorRuntimeRelPath = ('Contents/Developer/Platforms/'
  14. 'iPhoneOS.platform/Library/Developer/'
  15. 'CoreSimulator/Profiles/Runtimes')
  16. def _using_new_mac_toolchain(mac_toolchain):
  17. """Returns if the mac_toolchain command passed in is new version.
  18. New mac_toolchain can download an Xcode without bundled runtime, and can
  19. download single runtimes. Legacy mac_toolchain can only download Xcode package
  20. as a whole package. The function tells the difference by checking the
  21. existence of a new command line switch in new version.
  22. TODO(crbug.com/1191260): Remove this util function when the new mac_toolchain
  23. version is rolled to everywhere using this script.
  24. """
  25. cmd = [
  26. mac_toolchain,
  27. 'help',
  28. ]
  29. output = subprocess.check_output(
  30. cmd, stderr=subprocess.STDOUT).decode('utf-8')
  31. # "install-runtime" presents as a command line switch in help output in the
  32. # new mac_toolchain.
  33. using_new_mac_toolchain = 'install-runtime' in output
  34. return using_new_mac_toolchain
  35. def _is_legacy_xcode_package(xcode_app_path):
  36. """Checks and returns if the installed Xcode package is legacy version.
  37. Legacy Xcode package are uploaded with legacy version of mac_toolchain.
  38. Typically, multiple iOS runtimes are bundled into legacy Xcode packages. No
  39. runtime is bundled into new format Xcode packages.
  40. Args:
  41. xcode_app_path: (string) Path to install the contents of Xcode.app.
  42. Returns:
  43. (bool) True if the package is legacy(with runtime bundled). False otherwise.
  44. """
  45. # More than one iOS runtimes indicate the downloaded Xcode is a legacy one.
  46. # If no runtimes are found in the package, it's a new format package. If only
  47. # one runtime is found in package, it typically means it's an incorrectly
  48. # cached new format Xcode package. (The single runtime wasn't moved out from
  49. # Xcode in the end of last task, because last task was killed before moving.)
  50. runtimes_in_xcode = glob.glob(
  51. os.path.join(xcode_app_path, XcodeIOSSimulatorRuntimeRelPath,
  52. '*.simruntime'))
  53. is_legacy = len(runtimes_in_xcode) >= 2
  54. if not is_legacy:
  55. for runtime in runtimes_in_xcode:
  56. LOGGER.warning('Removing %s from incorrectly cached Xcode.', runtime)
  57. shutil.rmtree(runtime)
  58. return is_legacy
  59. def _install_runtime(mac_toolchain, install_path, xcode_build_version,
  60. ios_version):
  61. """Invokes mac_toolchain to install the runtime.
  62. mac_toolchain will resolve & find the best suitable runtime and install to the
  63. path, with Xcode and ios version as input.
  64. Args:
  65. install_path: (string) Path to install the runtime package into.
  66. xcode_build_version: (string) Xcode build version, e.g. 12d4e.
  67. ios_version: (string) Runtime version (number only), e.g. 13.4.
  68. """
  69. existing_runtimes = glob.glob(os.path.join(install_path, '*.simruntime'))
  70. # When no runtime file exists, remove any remaining .cipd or .xcode_versions
  71. # status folders, so mac_toolchain(underlying CIPD) will work to download a
  72. # new one.
  73. if len(existing_runtimes) == 0:
  74. for dir_name in ['.cipd', '.xcode_versions']:
  75. dir_path = os.path.join(install_path, dir_name)
  76. if os.path.exists(dir_path):
  77. LOGGER.warning('Removing %s in runtime cache folder.', dir_path)
  78. shutil.rmtree(dir_path)
  79. # Transform iOS version to the runtime version format required my the tool.
  80. # e.g. "14.4" -> "ios-14-4"
  81. runtime_version = 'ios-' + ios_version.replace('.', '-')
  82. cmd = [
  83. mac_toolchain,
  84. 'install-runtime',
  85. '-xcode-version',
  86. xcode_build_version.lower(),
  87. '-runtime-version',
  88. runtime_version,
  89. '-output-dir',
  90. install_path,
  91. ]
  92. LOGGER.debug('Installing runtime with command: %s' % cmd)
  93. output = subprocess.check_call(cmd, stderr=subprocess.STDOUT)
  94. return output
  95. def construct_runtime_cache_folder(runtime_cache_prefix, ios_version):
  96. """Composes runtime cache folder from it's prefix and ios_version.
  97. Note: Please keep the pattern consistent between what's being passed into
  98. runner script in gn(build/config/ios/ios_test_runner_wrapper.gni), and what's
  99. being configured for swarming cache in test configs (testing/buildbot/*).
  100. """
  101. return runtime_cache_prefix + ios_version
  102. def move_runtime(runtime_cache_folder, xcode_app_path, into_xcode):
  103. """Moves runtime from runtime cache into xcode or vice versa.
  104. The function is intended to only work with new Xcode packages.
  105. The function assumes that there's exactly one *.simruntime file in the source
  106. folder. It also removes existing runtimes in the destination folder. The above
  107. assumption & handling can ensure no incorrect Xcode package is cached from
  108. corner cases.
  109. Args:
  110. runtime_cache_folder: (string) Path to the runtime cache directory.
  111. xcode_app_path: (string) Path to install the contents of Xcode.app.
  112. into_xcode: (bool) Whether the function moves from cache dir into Xcode or
  113. from Xcode to cache dir.
  114. Raises:
  115. IOSRuntimeHandlingError for issues moving runtime around.
  116. shutil.Error for exceptions from shutil when moving files around.
  117. """
  118. xcode_runtime_folder = os.path.join(xcode_app_path,
  119. XcodeIOSSimulatorRuntimeRelPath)
  120. src_folder = runtime_cache_folder if into_xcode else xcode_runtime_folder
  121. dst_folder = xcode_runtime_folder if into_xcode else runtime_cache_folder
  122. runtimes_in_src = glob.glob(os.path.join(src_folder, '*.simruntime'))
  123. if len(runtimes_in_src) != 1:
  124. raise test_runner_errors.IOSRuntimeHandlingError(
  125. 'Not exactly one runtime files (files: %s) to move from %s!' %
  126. (runtimes_in_src, src_folder))
  127. runtimes_in_dst = glob.glob(os.path.join(dst_folder, '*.simruntime'))
  128. for runtime in runtimes_in_dst:
  129. LOGGER.warning('Removing existing %s in destination folder.', runtime)
  130. shutil.rmtree(runtime)
  131. # Get the runtime package filename. It might not be the default name.
  132. runtime_name = os.path.basename(runtimes_in_src[0])
  133. dst_runtime = os.path.join(dst_folder, runtime_name)
  134. LOGGER.debug('Moving %s from %s to %s.' %
  135. (runtime_name, src_folder, dst_folder))
  136. shutil.move(os.path.join(src_folder, runtime_name), dst_runtime)
  137. return
  138. def remove_runtimes(xcode_app_path):
  139. """Removes all runtimes in given xcode path."""
  140. runtimes = glob.glob(
  141. os.path.join(xcode_app_path, XcodeIOSSimulatorRuntimeRelPath,
  142. '*.simruntime'))
  143. for runtime in runtimes:
  144. LOGGER.warning('Removing existing %s in xcode.', runtime)
  145. shutil.rmtree(runtime)
  146. def select(xcode_app_path):
  147. """Invokes sudo xcode-select -s {xcode_app_path}
  148. Raises:
  149. subprocess.CalledProcessError on exit codes non zero
  150. """
  151. cmd = [
  152. 'sudo',
  153. 'xcode-select',
  154. '-s',
  155. xcode_app_path,
  156. ]
  157. LOGGER.debug('Selecting XCode with command %s and "xcrun simctl list".' % cmd)
  158. output = subprocess.check_output(
  159. cmd, stderr=subprocess.STDOUT).decode('utf-8')
  160. # This is to avoid issues caused by mixed usage of different Xcode versions on
  161. # one machine.
  162. xcrun_simctl_cmd = ['xcrun', 'simctl', 'list']
  163. output += subprocess.check_output(
  164. xcrun_simctl_cmd, stderr=subprocess.STDOUT).decode('utf-8')
  165. return output
  166. def _install_xcode(mac_toolchain, xcode_build_version, xcode_path,
  167. using_new_mac_toolchain):
  168. """Invokes mac_toolchain to install the given xcode version.
  169. If using legacy mac_toolchain, install the whole Xcode package. If using the
  170. new mac_toolchain, add a command line switch to try to install an Xcode
  171. without runtime. However, the existence of runtime depends on the actual Xcode
  172. package in CIPD. e.g. An Xcode package uploaded with legacy mac_toolchain will
  173. include runtimes, even though it's installed with new mac_toolchain and
  174. "-with-runtime=False" switch.
  175. TODO(crbug.com/1191260): Remove the last argument when the new mac_toolchain
  176. version is rolled to everywhere using this script.
  177. Args:
  178. xcode_build_version: (string) Xcode build version to install.
  179. mac_toolchain: (string) Path to mac_toolchain command to install Xcode
  180. See https://chromium.googlesource.com/infra/infra/+/main/go/src/infra/cmd/mac_toolchain/
  181. xcode_path: (string) Path to install the contents of Xcode.app.
  182. using_new_mac_toolchain: (bool) Using new mac_toolchain.
  183. Raises:
  184. subprocess.CalledProcessError on exit codes non zero
  185. """
  186. cmd = [
  187. mac_toolchain,
  188. 'install',
  189. '-kind',
  190. 'ios',
  191. '-xcode-version',
  192. xcode_build_version.lower(),
  193. '-output-dir',
  194. xcode_path,
  195. ]
  196. if using_new_mac_toolchain:
  197. cmd.append('-with-runtime=False')
  198. LOGGER.debug('Installing xcode with command: %s' % cmd)
  199. output = subprocess.check_call(cmd, stderr=subprocess.STDOUT)
  200. return output
  201. def install(mac_toolchain, xcode_build_version, xcode_app_path, **runtime_args):
  202. """Installs the Xcode and returns if the installed one is a legacy package.
  203. Installs the Xcode of given version to path. Returns if the Xcode package
  204. of the version is a legacy package (with runtimes bundled in). Runtime related
  205. arguments will only work when |mac_toolchain| is a new version (with runtime
  206. features), and the |xcode_build_version| in CIPD is a new package (uploaded
  207. by new mac_toolchain).
  208. If using legacy mac_toolchain, install the whole legacy Xcode package. (Will
  209. raise if the Xcode package isn't legacy.)
  210. If using new mac_toolchain, first install the Xcode package:
  211. * If installed Xcode is legacy one (with runtimes bundled), return.
  212. * If installed Xcode isn't legacy (without runtime bundled), install and copy
  213. * the specified runtime version into Xcode.
  214. Args:
  215. xcode_build_version: (string) Xcode build version to install.
  216. mac_toolchain: (string) Path to mac_toolchain command to install Xcode
  217. See https://chromium.googlesource.com/infra/infra/+/main/go/src/infra/cmd/mac_toolchain/
  218. xcode_app_path: (string) Path to install the contents of Xcode.app.
  219. runtime_args: Keyword arguments related with runtime installation. Can be
  220. empty when installing an Xcode w/o runtime (for real device tasks). Namely:
  221. runtime_cache_folder: (string) Path to the folder where runtime package
  222. file (e.g. iOS.simruntime) is stored.
  223. ios_version: (string) iOS version requested to be in Xcode package.
  224. Raises:
  225. subprocess.CalledProcessError on exit codes non zero
  226. XcodeMacToolchainMismatchError if an Xcode without runtime is installed with
  227. a legacy mac_toolchain.
  228. Returns:
  229. True, if the Xcode package in CIPD is legacy (bundled with runtimes).
  230. False, if the Xcode package in CIPD is new (not bundled with runtimes).
  231. """
  232. using_new_mac_toolchain = _using_new_mac_toolchain(mac_toolchain)
  233. _install_xcode(mac_toolchain, xcode_build_version, xcode_app_path,
  234. using_new_mac_toolchain)
  235. is_legacy_xcode_package = _is_legacy_xcode_package(xcode_app_path)
  236. if not using_new_mac_toolchain and not is_legacy_xcode_package:
  237. # Legacy mac_toolchain can't handle the situation when no runtime is in
  238. # Xcode package.
  239. raise test_runner_errors.XcodeMacToolchainMismatchError(xcode_build_version)
  240. # Install & move the runtime to Xcode. Can only work with new mac_toolchain.
  241. # Only install runtime when it's working for a simulator task.
  242. if not is_legacy_xcode_package and runtime_args.get('ios_version'):
  243. runtime_cache_folder = runtime_args.get('runtime_cache_folder')
  244. ios_version = runtime_args.get('ios_version')
  245. if not runtime_cache_folder or not ios_version:
  246. raise test_runner_errors.IOSRuntimeHandlingError(
  247. 'Insufficient runtime_args. runtime_cache_folder: %s, ios_version: %s'
  248. % s(runtime_cache_folder, ios_version))
  249. # Try to install the runtime to it's cache folder. mac_toolchain will test
  250. # and install only when the runtime doesn't exist in cache.
  251. _install_runtime(mac_toolchain, runtime_cache_folder, xcode_build_version,
  252. ios_version)
  253. move_runtime(runtime_cache_folder, xcode_app_path, into_xcode=True)
  254. return is_legacy_xcode_package
  255. def version():
  256. """Invokes xcodebuild -version
  257. Raises:
  258. subprocess.CalledProcessError on exit codes non zero
  259. Returns:
  260. version (12.0), build_version (12a6163b)
  261. """
  262. cmd = [
  263. 'xcodebuild',
  264. '-version',
  265. ]
  266. LOGGER.debug('Checking XCode version with command: %s' % cmd)
  267. output = subprocess.check_output(cmd).decode('utf-8')
  268. output = output.splitlines()
  269. # output sample:
  270. # Xcode 12.0
  271. # Build version 12A6159
  272. LOGGER.info(output)
  273. version = output[0].split(' ')[1]
  274. build_version = output[1].split(' ')[2].lower()
  275. return version, build_version
  276. def using_xcode_11_or_higher():
  277. """Returns true if using Xcode version 11 or higher."""
  278. LOGGER.debug('Checking if Xcode version is 11 or higher')
  279. return distutils.version.LooseVersion(
  280. '11.0') <= distutils.version.LooseVersion(version()[0])
  281. def using_xcode_13_or_higher():
  282. """Returns true if using Xcode version 13 or higher."""
  283. LOGGER.debug('Checking if Xcode version is 13 or higher')
  284. return distutils.version.LooseVersion(
  285. '13.0') <= distutils.version.LooseVersion(version()[0])