generate_build_gn.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377
  1. # Copyright 2022 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. #
  5. #
  6. # LOCAL SETUP:
  7. # Make sure you have bazel installed first.
  8. # `sudo apt-get install bazel`
  9. # This script is designed to work on Linux only, but may work on other platforms
  10. # with small changes.
  11. #
  12. # WHAT:
  13. # This script creates the BUILD.gn file for XNNPACK by reverse-engineering the
  14. # bazel build.
  15. # HOW:
  16. # By setting the -s option on the bazel build command, bazel logs each compiler
  17. # invocation to the console which is then scraped and put into a configuration
  18. # that gn will accept.
  19. #
  20. # WHY:
  21. # The biggest difficulty of this process is that gn expects each source's
  22. # basename to be unique within a single source set. For example, including both
  23. # "bar/foo.c" and "baz/foo.c" in the same source set is illegal. Therefore, each
  24. # source set will only contain sources from a single directory.
  25. # However, some sources within the same directory may need different compiler
  26. # flags set, so source sets are further split by their flags.
  27. import collections
  28. import logging
  29. import os
  30. import shutil
  31. import subprocess
  32. from operator import attrgetter
  33. _HEADER = '''
  34. # Copyright 2022 The Chromium Authors. All rights reserved.
  35. # Use of this source code is governed by a BSD-style license that can be
  36. # found in the LICENSE file.
  37. # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
  38. # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
  39. # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
  40. #
  41. # THIS FILE IS AUTO-GENERATED. DO NOT EDIT.
  42. #
  43. # See //third_party/xnnpack/generate_build_gn.py
  44. #
  45. # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
  46. # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
  47. # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
  48. config("xnnpack_config") {
  49. include_dirs = [
  50. "src/deps/clog/include",
  51. "src/include",
  52. "src/src",
  53. ]
  54. cflags=[
  55. "-Wno-unused-function",
  56. ]
  57. defines = [
  58. # Don't enable this without first talking to Chrome Security!
  59. # XNNPACK runs in the browser process. The hardening and fuzzing needed
  60. # to ensure JIT can be used safely is not in place yet.
  61. "XNN_ENABLE_JIT=0",
  62. "XNN_ENABLE_ASSEMBLY=1",
  63. "XNN_ENABLE_GEMM_M_SPECIALIZATION=1",
  64. "XNN_ENABLE_MEMOPT=1",
  65. "XNN_ENABLE_SPARSE=1",
  66. "XNN_LOG_LEVEL=0",
  67. "XNN_LOG_TO_STDIO=0",
  68. ]
  69. }
  70. '''.strip()
  71. _MAIN_TMPL = '''
  72. source_set("xnnpack") {
  73. public = [ "src/include/xnnpack.h" ]
  74. configs -= [ "//build/config/compiler:chromium_code" ]
  75. configs += [ "//build/config/compiler:no_chromium_code" ]
  76. configs += [ "//build/config/sanitizers:cfi_icall_generalize_pointers" ]
  77. sources = [
  78. %SRCS%
  79. ]
  80. deps = [
  81. %TARGETS%,
  82. "//third_party/cpuinfo",
  83. "//third_party/fp16",
  84. "//third_party/fxdiv",
  85. "//third_party/pthreadpool",
  86. ]
  87. public_configs = [ ":xnnpack_config" ]
  88. }
  89. '''.strip()
  90. _TARGET_TMPL = '''
  91. source_set("%TARGET_NAME%") {
  92. cflags = [
  93. %ARGS%
  94. ]
  95. sources = [
  96. %SRCS%
  97. ]
  98. configs -= [ "//build/config/compiler:chromium_code" ]
  99. configs += [ "//build/config/compiler:no_chromium_code" ]
  100. configs += [ "//build/config/sanitizers:cfi_icall_generalize_pointers" ]
  101. deps = [
  102. "//third_party/cpuinfo",
  103. "//third_party/fp16",
  104. "//third_party/fxdiv",
  105. "//third_party/pthreadpool",
  106. ]
  107. public_configs = [ ":xnnpack_config" ]
  108. }
  109. '''.strip()
  110. # This is the latest version of the Android NDK that is compatible with
  111. # XNNPACK.
  112. _ANDROID_NDK_VERSION = 'android-ndk-r19c'
  113. _ANDROID_NDK_URL = 'https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip'
  114. g_android_ndk = None
  115. def _ensure_android_ndk_available():
  116. """
  117. Ensures that the Android NDK is available to bazel, downloading a new copy if
  118. needed. Raises an Exception if any command fails.
  119. Returns: the full path to the Android NDK
  120. """
  121. global g_android_ndk
  122. if g_android_ndk:
  123. return g_android_ndk
  124. g_android_ndk = '/tmp/'+_ANDROID_NDK_VERSION
  125. if os.path.exists(g_android_ndk):
  126. logging.info('Using existing Android NDK at ' + g_android_ndk)
  127. return g_android_ndk
  128. logging.info('Downloading new copy of the Android NDK')
  129. zipfile = '/tmp/{ndk}.zip'.format(ndk=_ANDROID_NDK_VERSION)
  130. subprocess.run(['wget', '-O', zipfile, _ANDROID_NDK_URL],
  131. stdout=subprocess.DEVNULL,
  132. stderr=subprocess.DEVNULL,
  133. check=True)
  134. subprocess.run(['unzip', '-o', zipfile, '-d', '/tmp'],
  135. stdout=subprocess.DEVNULL,
  136. stderr=subprocess.DEVNULL,
  137. check=True)
  138. return g_android_ndk
  139. # A SourceSet corresponds to a single source_set() gn tuple.
  140. SourceSet = collections.namedtuple(
  141. 'SourceSet',
  142. ['dir', 'srcs', 'args'],
  143. defaults=['', [], []])
  144. def NameForSourceSet(source_set):
  145. """
  146. Returns the name to use for a SourceSet in the gn target.
  147. """
  148. if source_set.dir == 'xnnpack':
  149. return 'xnnpack'
  150. if len(source_set.args) == 0:
  151. return source_set.dir
  152. return '{dir}_{args}'.format(**{
  153. 'dir': source_set.dir,
  154. 'args': '-'.join([arg[2:] for arg in source_set.args]),
  155. })
  156. # An ObjectBuild corresponds to a single built object, which is parsed from a
  157. # single bazel compiler invocation on a single source file.
  158. ObjectBuild = collections.namedtuple(
  159. 'ObjectBuild',
  160. ['src', 'dir', 'args'],
  161. defaults=['', '', []])
  162. def _objectbuild_from_bazel_log(log_line):
  163. """
  164. Attempts to scrape a compiler invocation from a single bazel build output
  165. line. If no invocation is present, None is returned.
  166. """
  167. split = log_line.strip().split(' ')
  168. if not split[0].endswith('gcc'):
  169. return None
  170. src = ''
  171. dir = ''
  172. args = []
  173. for i, arg in enumerate(split):
  174. if arg == '-c':
  175. src = os.path.join('src', split[i + 1])
  176. src_path = src.split('/')
  177. if len(src_path) == 3:
  178. dir = 'xnnpack'
  179. else:
  180. dir = src_path[2]
  181. if arg.startswith('-m'):
  182. args.append(arg)
  183. return ObjectBuild(src=src, dir=dir, args=args)
  184. def _cwd():
  185. """
  186. Returns the absolute path of //third_party/xnnpack/.
  187. """
  188. return os.path.dirname(os.path.realpath(__file__))
  189. def _run_bazel_cmd(args):
  190. """
  191. Runs a bazel command in the form of bazel <args...>. Returns the stdout and
  192. stderr concatenated, raising an Exception if the command failed.
  193. """
  194. exec_path = shutil.which("bazel")
  195. if not exec_path:
  196. raise Exception("bazel is not installed. Please run `sudo apt-get install "
  197. + "bazel` or put the bazel executable in $PATH")
  198. cmd = [exec_path]
  199. cmd.extend(args)
  200. env = os.environ
  201. env.update({
  202. 'ANDROID_NDK_HOME': _ensure_android_ndk_available(),
  203. })
  204. proc = subprocess.Popen(cmd,
  205. text=True,
  206. cwd=os.path.join(_cwd(), 'src'),
  207. stdout=subprocess.PIPE,
  208. stderr=subprocess.PIPE,
  209. env=env)
  210. stdout, stderr = proc.communicate()
  211. if proc.returncode != 0:
  212. raise Exception("bazel command returned non-zero return code:\n"
  213. "cmd: {cmd}\n"
  214. "status: {status}\n"
  215. "stdout: {stdout}\n"
  216. "stderr: {stderr}".format(**{
  217. 'cmd': str(cmd),
  218. 'status': proc.returncode,
  219. 'stdout': stdout,
  220. 'stderr': stderr,
  221. })
  222. )
  223. return stdout + "\n" + stderr
  224. def ListAllSrcs():
  225. """
  226. Runs a bazel command to query and and return all source files for XNNPACK, but
  227. not any dependancies, as relative paths to //third_party/xnnpack/.
  228. """
  229. logging.info('Querying for the list of all srcs in :xnnpack_for_tflite...')
  230. out = _run_bazel_cmd([
  231. 'cquery',
  232. 'kind("source file", deps(:xnnpack_for_tflite))',
  233. '--define',
  234. 'xnn_enable_jit=false',
  235. ])
  236. srcs = []
  237. for line in out.split('\n'):
  238. if line.startswith('//:'):
  239. srcs.append(os.path.join('src', line.split()[0][3:]))
  240. return srcs
  241. def GenerateObjectBuilds(srcs):
  242. """
  243. Builds XNNPACK with bazel and scrapes out all the ObjectBuild's for all
  244. source files in srcs.
  245. """
  246. logging.info('Running `bazel clean`')
  247. _run_bazel_cmd(['clean'])
  248. logging.info('Building xnnpack with bazel...')
  249. logs = _run_bazel_cmd([
  250. 'build',
  251. ':xnnpack_for_tflite',
  252. '-s',
  253. '-c', 'opt',
  254. '--define',
  255. 'xnn_enable_jit=false',
  256. ])
  257. logging.info('scraping %d log lines from bazel build...'
  258. % len(logs.split('\n')))
  259. obs = []
  260. for log in logs.split('\n'):
  261. ob = _objectbuild_from_bazel_log(log)
  262. if ob and ob.src in srcs:
  263. obs.append(ob)
  264. logging.info('Scraped %d built objects' % len(obs))
  265. return obs
  266. def CombineObjectBuildsIntoSourceSets(obs):
  267. """
  268. Combines all the given ObjectBuild's into SourceSet's by combining source
  269. files whose SourceSet name's (that is thier directory and compiler flags)
  270. match.
  271. """
  272. sss = {}
  273. for ob in obs:
  274. single = SourceSet(dir=ob.dir, srcs=[ob.src], args=ob.args)
  275. name = NameForSourceSet(single)
  276. if name not in sss:
  277. sss[name] = single
  278. else:
  279. ss = sss[name]
  280. ss = ss._replace(srcs=list(set(ss.srcs + [ob.src])))
  281. sss[name] = ss
  282. xxnpack_ss = sss.pop('xnnpack')
  283. logging.info('Generated %d sub targets for xnnpack' % len(sss))
  284. return xxnpack_ss, sorted(
  285. list(sss.values()), key=lambda ss: NameForSourceSet(ss))
  286. def MakeTargetSourceSet(ss):
  287. """
  288. Generates the BUILD file text for a build target that supports the main
  289. XNNPACK target, returning it as a string.
  290. """
  291. target = _TARGET_TMPL
  292. target = target.replace('%ARGS%', ',\n'.join([
  293. ' "%s"' % arg for arg in sorted(ss.args)
  294. ]))
  295. target = target.replace('%SRCS%', ',\n'.join([
  296. ' "%s"' % src for src in sorted(ss.srcs)
  297. ]))
  298. target = target.replace('%TARGET_NAME%', NameForSourceSet(ss))
  299. return target
  300. def MakeXNNPACKSourceSet(ss, other_targets):
  301. """
  302. Generates the BUILD file text for the main XNNPACK build target, given the
  303. XNNPACK SourceSet and the names of all its supporting targets.
  304. """
  305. target = _MAIN_TMPL
  306. target = target.replace('%SRCS%', ',\n'.join([
  307. ' "%s"' % src for src in sorted(ss.srcs)
  308. ]))
  309. target = target.replace('%TARGETS%', ',\n'.join([
  310. ' ":%s"' % t for t in sorted(other_targets)
  311. ]))
  312. return target
  313. def main():
  314. logging.basicConfig(level=logging.INFO)
  315. srcs = ListAllSrcs()
  316. obs = GenerateObjectBuilds(srcs)
  317. xnnpack_ss, other_sss = CombineObjectBuildsIntoSourceSets(obs)
  318. sub_targets = []
  319. for ss in other_sss:
  320. sub_targets.append(MakeTargetSourceSet(ss))
  321. xnnpack_target = MakeXNNPACKSourceSet(
  322. xnnpack_ss,
  323. [NameForSourceSet(ss) for ss in other_sss])
  324. out_path = os.path.join(_cwd(), 'BUILD.gn')
  325. logging.info('Writing to ' + out_path)
  326. with open(out_path, 'w') as f:
  327. f.write(_HEADER)
  328. f.write('\n')
  329. f.write(xnnpack_target)
  330. f.write('\n\n')
  331. for target in sub_targets:
  332. f.write(target)
  333. f.write('\n\n')
  334. logging.info('Done! Please run `git cl format`')
  335. if __name__ == "__main__":
  336. main()