licenses.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899
  1. #!/usr/bin/env python3
  2. # Copyright (c) 2012 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Utility for checking and processing licensing information in third_party
  6. directories.
  7. Usage: licenses.py <command>
  8. Commands:
  9. scan scan third_party directories, verifying that we have licensing info
  10. credits generate about:credits on stdout
  11. (You can also import this as a module.)
  12. """
  13. from __future__ import print_function
  14. import argparse
  15. import codecs
  16. import json
  17. import os
  18. import shutil
  19. import re
  20. import subprocess
  21. import sys
  22. import tempfile
  23. if sys.version_info.major == 2:
  24. import cgi as html
  25. else:
  26. import html
  27. # TODO(agrieve): Move build_utils.WriteDepFile into a non-android directory.
  28. _REPOSITORY_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
  29. sys.path.insert(0, os.path.join(_REPOSITORY_ROOT, 'build/android/gyp'))
  30. from util import build_utils
  31. # Paths from the root of the tree to directories to skip.
  32. PRUNE_PATHS = set([
  33. # Placeholder directory only, not third-party code.
  34. os.path.join('third_party', 'adobe'),
  35. # Will remove it once converted private sdk using cipd.
  36. os.path.join('third_party', 'android_tools_internal'),
  37. # Build files only, not third-party code.
  38. os.path.join('third_party', 'widevine'),
  39. # Only binaries, used during development.
  40. os.path.join('third_party', 'valgrind'),
  41. # Used for development and test, not in the shipping product.
  42. os.path.join('build', 'secondary'),
  43. os.path.join('third_party', 'bison'),
  44. os.path.join('third_party', 'blanketjs'),
  45. os.path.join('third_party', 'chromite'),
  46. os.path.join('third_party', 'cygwin'),
  47. os.path.join('third_party', 'gles2_conform'),
  48. os.path.join('third_party', 'gnu_binutils'),
  49. os.path.join('third_party', 'gold'),
  50. os.path.join('third_party', 'gperf'),
  51. os.path.join('third_party', 'lighttpd'),
  52. os.path.join('third_party', 'llvm'),
  53. os.path.join('third_party', 'llvm-build'),
  54. os.path.join('third_party', 'mingw-w64'),
  55. os.path.join('third_party', 'nacl_sdk_binaries'),
  56. os.path.join('third_party', 'pefile'),
  57. os.path.join('third_party', 'perl'),
  58. os.path.join('third_party', 'psyco_win32'),
  59. os.path.join('third_party', 'pyelftools'),
  60. os.path.join('third_party', 'pylib'),
  61. os.path.join('third_party', 'pywebsocket'),
  62. os.path.join('third_party', 'syzygy'),
  63. # Stuff pulled in from chrome-internal for official builds/tools.
  64. os.path.join('third_party', 'clear_cache'),
  65. os.path.join('third_party', 'gnu'),
  66. os.path.join('third_party', 'googlemac'),
  67. os.path.join('third_party', 'pcre'),
  68. os.path.join('third_party', 'psutils'),
  69. os.path.join('third_party', 'sawbuck'),
  70. # See crbug.com/350472
  71. os.path.join('chrome', 'browser', 'resources', 'chromeos', 'quickoffice'),
  72. # Chrome for Android proprietary code.
  73. os.path.join('clank'),
  74. # Proprietary barcode detection library.
  75. os.path.join('third_party', 'barhopper'),
  76. # Proprietary DevTools code.
  77. os.path.join('third_party', 'devtools-frontend-internal'),
  78. # Redistribution does not require attribution in documentation.
  79. os.path.join('third_party', 'directxsdk'),
  80. # For testing only, presents on some bots.
  81. os.path.join('isolate_deps_dir'),
  82. # Mock test data.
  83. os.path.join('tools', 'binary_size', 'libsupersize', 'testdata'),
  84. # Overrides some WebRTC files, same license. Skip this one.
  85. os.path.join('third_party', 'webrtc_overrides'),
  86. ])
  87. # Directories we don't scan through.
  88. VCS_METADATA_DIRS = ('.svn', '.git')
  89. PRUNE_DIRS = VCS_METADATA_DIRS + ('layout_tests', ) # lots of subdirs
  90. # A third_party directory can define this file, containing a list of
  91. # subdirectories to process in addition to itself. Intended for directories
  92. # that contain multiple others as transitive dependencies.
  93. ADDITIONAL_PATHS_FILENAME = 'additional_readme_paths.json'
  94. # A list of paths that contain license information but that would otherwise
  95. # not be included. Possible reasons include:
  96. # - Third party directories in //clank which are considered to be Google-owned
  97. # - Directories that are directly checked out from upstream, and thus
  98. # don't have a README.chromium
  99. # - Directories that contain example code, or build tooling.
  100. # - Nested third_party code inside other third_party libraries.
  101. ADDITIONAL_PATHS = (
  102. os.path.join('chrome', 'common', 'extensions', 'docs', 'examples'),
  103. os.path.join('chrome', 'test', 'chromeos', 'autotest'),
  104. os.path.join('chrome', 'test', 'data'),
  105. os.path.join('native_client'),
  106. os.path.join('testing', 'gmock'),
  107. os.path.join('testing', 'gtest'),
  108. os.path.join('third_party', 'boringssl', 'src', 'third_party', 'fiat'),
  109. os.path.join('third_party', 'devtools-frontend', 'src', 'front_end',
  110. 'third_party'),
  111. os.path.join('third_party', 'devtools-frontend-internal', 'front_end',
  112. 'third_party'),
  113. os.path.join('tools', 'gyp'),
  114. os.path.join('tools', 'page_cycler', 'acid3'),
  115. os.path.join('url', 'third_party', 'mozilla'),
  116. os.path.join('v8'),
  117. # Fake directories to include the strongtalk and fdlibm licenses.
  118. os.path.join('v8', 'strongtalk'),
  119. os.path.join('v8', 'fdlibm'),
  120. )
  121. # Directories where we check out directly from upstream, and therefore
  122. # can't provide a README.chromium. Please prefer a README.chromium
  123. # wherever possible.
  124. SPECIAL_CASES = {
  125. os.path.join('native_client'): {
  126. "Name": "native client",
  127. "URL": "http://code.google.com/p/nativeclient",
  128. "License": "BSD",
  129. },
  130. os.path.join('testing', 'gmock'): {
  131. "Name": "gmock",
  132. "URL": "http://code.google.com/p/googlemock",
  133. "License": "BSD",
  134. "License File": "NOT_SHIPPED",
  135. },
  136. os.path.join('testing', 'gtest'): {
  137. "Name": "gtest",
  138. "URL": "http://code.google.com/p/googletest",
  139. "License": "BSD",
  140. "License File": "NOT_SHIPPED",
  141. },
  142. os.path.join('third_party', 'angle'): {
  143. "Name": "Almost Native Graphics Layer Engine",
  144. "URL": "http://code.google.com/p/angleproject/",
  145. "License": "BSD",
  146. },
  147. os.path.join('third_party', 'cros_system_api'): {
  148. "Name": "Chromium OS system API",
  149. "URL": "http://www.chromium.org/chromium-os",
  150. "License": "BSD",
  151. # Absolute path here is resolved as relative to the source root.
  152. "License File": "/LICENSE.chromium_os",
  153. },
  154. os.path.join('third_party', 'ipcz'): {
  155. "Name": "ipcz",
  156. "URL" : "https://chromium.googlesource.com/chromium/src/third_party/ipcz",
  157. "License": "BSD",
  158. "License File": "/third_party/ipcz/LICENSE",
  159. },
  160. os.path.join('third_party', 'lss'): {
  161. "Name": "linux-syscall-support",
  162. "URL": "http://code.google.com/p/linux-syscall-support/",
  163. "License": "BSD",
  164. "License File": "/LICENSE",
  165. },
  166. os.path.join('third_party', 'openscreen', 'src', 'third_party', 'abseil'): {
  167. "Name": "abseil",
  168. "URL": "https://github.com/abseil/abseil-cpp/",
  169. "License": "Apache 2.0",
  170. "License File": "/third_party/abseil-cpp/LICENSE",
  171. },
  172. os.path.join('third_party', 'openscreen', 'src', 'third_party',
  173. 'boringssl'):
  174. {
  175. "Name": "BoringSSL",
  176. "URL": "https://boringssl.googlesource.com/boringssl/",
  177. "License": "BSDish",
  178. "License File": "/third_party/boringssl/src/LICENSE",
  179. },
  180. os.path.join('third_party', 'openscreen', 'src', 'third_party', 'jsoncpp'):
  181. {
  182. "Name": "jsoncpp",
  183. "URL": "https://github.com/open-source-parsers/jsoncpp",
  184. "License": "MIT",
  185. "License File": "/third_party/jsoncpp/LICENSE",
  186. },
  187. os.path.join('third_party', 'openscreen', 'src', 'third_party', 'mozilla'):
  188. {
  189. "Name": "mozilla",
  190. "URL": "https://github.com/mozilla",
  191. "License": "MPL 1.1/GPL 2.0/LGPL 2.1",
  192. "License File": "LICENSE.txt",
  193. },
  194. os.path.join('third_party', 'pdfium'): {
  195. "Name": "PDFium",
  196. "URL": "http://code.google.com/p/pdfium/",
  197. "License": "BSD",
  198. },
  199. os.path.join('third_party', 'ppapi'): {
  200. "Name": "ppapi",
  201. "URL": "http://code.google.com/p/ppapi/",
  202. },
  203. os.path.join('third_party', 'scons-2.0.1'): {
  204. "Name": "scons-2.0.1",
  205. "URL": "http://www.scons.org",
  206. "License": "MIT",
  207. "License File": "NOT_SHIPPED",
  208. },
  209. os.path.join('third_party', 'catapult'): {
  210. "Name": "catapult",
  211. "URL": "https://github.com/catapult-project/catapult",
  212. "License": "BSD",
  213. "License File": "NOT_SHIPPED",
  214. },
  215. os.path.join('third_party', 'crashpad', 'crashpad', 'third_party', 'lss'): {
  216. "Name": "linux-syscall-support",
  217. "URL": "https://chromium.googlesource.com/linux-syscall-support/",
  218. "License": "BSD",
  219. "License File": "NOT_SHIPPED",
  220. },
  221. os.path.join('third_party', 'crashpad', 'crashpad', 'third_party',
  222. 'mini_chromium'):
  223. {
  224. "Name": "mini_chromium",
  225. "URL": "https://chromium.googlesource.com/chromium/mini_chromium/",
  226. "License": "BSD",
  227. "License File": "NOT_SHIPPED",
  228. },
  229. os.path.join('third_party', 'crashpad', 'crashpad', 'third_party', 'xnu'): {
  230. "Name": "xnu",
  231. "URL": "https://opensource.apple.com/source/xnu/",
  232. "License": "Apple Public Source License 2.0",
  233. "License File": "APPLE_LICENSE",
  234. },
  235. os.path.join('third_party', 'crashpad', 'crashpad', 'third_party', 'zlib'):
  236. {
  237. "Name": "zlib",
  238. "URL": "https://zlib.net/",
  239. "License": "zlib",
  240. "License File": "NOT_SHIPPED",
  241. },
  242. os.path.join('third_party', 'v8-i18n'): {
  243. "Name": "Internationalization Library for v8",
  244. "URL": "http://code.google.com/p/v8-i18n/",
  245. "License": "Apache 2.0",
  246. },
  247. os.path.join('third_party', 'blink'): {
  248. # about:credits doesn't show "Blink" but "WebKit".
  249. # Blink is a fork of WebKit, and Chromium project has maintained it
  250. # since the fork. about:credits needs to mention the code before
  251. # the fork.
  252. "Name": "WebKit",
  253. "URL": "http://webkit.org/",
  254. "License": "BSD and LGPL v2 and LGPL v2.1",
  255. # Absolute path here is resolved as relative to the source root.
  256. "License File": "/third_party/blink/LICENSE_FOR_ABOUT_CREDITS",
  257. },
  258. os.path.join('third_party', 'webpagereplay'): {
  259. "Name": "webpagereplay",
  260. "URL": "http://code.google.com/p/web-page-replay",
  261. "License": "Apache 2.0",
  262. "License File": "NOT_SHIPPED",
  263. },
  264. os.path.join('tools', 'gyp'): {
  265. "Name": "gyp",
  266. "URL": "http://code.google.com/p/gyp",
  267. "License": "BSD",
  268. "License File": "NOT_SHIPPED",
  269. },
  270. os.path.join('v8'): {
  271. "Name": "V8 JavaScript Engine",
  272. "URL": "http://code.google.com/p/v8",
  273. "License": "BSD",
  274. },
  275. os.path.join('v8', 'strongtalk'): {
  276. "Name": "Strongtalk",
  277. "URL": "http://www.strongtalk.org/",
  278. "License": "BSD",
  279. # Absolute path here is resolved as relative to the source root.
  280. "License File": "/v8/LICENSE.strongtalk",
  281. },
  282. os.path.join('v8', 'fdlibm'): {
  283. "Name": "fdlibm",
  284. "URL": "http://www.netlib.org/fdlibm/",
  285. "License": "Freely Distributable",
  286. # Absolute path here is resolved as relative to the source root.
  287. "License File": "/v8/LICENSE.fdlibm",
  288. "License Android Compatible": "yes",
  289. },
  290. os.path.join('third_party', 'khronos_glcts'): {
  291. # These sources are not shipped, are not public, and it isn't
  292. # clear why they're tripping the license check.
  293. "Name": "khronos_glcts",
  294. "URL": "http://no-public-url",
  295. "License": "Khronos",
  296. "License File": "NOT_SHIPPED",
  297. },
  298. os.path.join('tools', 'telemetry', 'third_party', 'gsutil'): {
  299. "Name": "gsutil",
  300. "URL": "https://cloud.google.com/storage/docs/gsutil",
  301. "License": "Apache 2.0",
  302. "License File": "NOT_SHIPPED",
  303. },
  304. os.path.join('third_party', 'swiftshader'): {
  305. "Name": "SwiftShader",
  306. "URL": "https://swiftshader.googlesource.com/SwiftShader",
  307. "License": "Apache 2.0 and compatible licenses",
  308. "License Android Compatible": "yes",
  309. "License File": "/third_party/swiftshader/LICENSE.txt",
  310. },
  311. os.path.join('third_party', 'swiftshader', 'third_party', 'SPIRV-Tools'): {
  312. "Name": "SPIRV-Tools",
  313. "URL": "https://github.com/KhronosGroup/SPIRV-Tools",
  314. "License": "Apache 2.0",
  315. "License File":
  316. "/third_party/swiftshader/third_party/SPIRV-Tools/LICENSE",
  317. },
  318. os.path.join('third_party', 'swiftshader', 'third_party', 'SPIRV-Headers'): {
  319. "Name": "SPIRV-Headers",
  320. "URL": "https://github.com/KhronosGroup/SPIRV-Headers",
  321. "License": "Apache 2.0",
  322. "License File":
  323. "/third_party/swiftshader/third_party/SPIRV-Headers/LICENSE",
  324. },
  325. }
  326. # Special value for 'License File' field used to indicate that the license file
  327. # should not be used in about:credits.
  328. NOT_SHIPPED = "NOT_SHIPPED"
  329. # Paths for libraries that we have checked are not shipped on iOS. These are
  330. # left out of the licenses file primarily because we don't want to cause a
  331. # firedrill due to someone thinking that Chrome for iOS is using LGPL code
  332. # when it isn't.
  333. # This is a temporary hack; the real solution is crbug.com/178215
  334. KNOWN_NON_IOS_LIBRARIES = set([
  335. os.path.join('base', 'third_party', 'symbolize'),
  336. os.path.join('base', 'third_party', 'xdg_mime'),
  337. os.path.join('base', 'third_party', 'xdg_user_dirs'),
  338. os.path.join('buildtools', 'third_party', 'libc++'),
  339. os.path.join('buildtools', 'third_party', 'libc++abi'),
  340. os.path.join('chrome', 'installer', 'mac', 'third_party', 'bsdiff'),
  341. os.path.join('chrome', 'installer', 'mac', 'third_party', 'xz'),
  342. os.path.join('chrome', 'test', 'data', 'third_party', 'kraken'),
  343. os.path.join('chrome', 'test', 'data', 'third_party', 'spaceport'),
  344. os.path.join('chrome', 'third_party', 'mozilla_security_manager'),
  345. os.path.join('third_party', 'angle'),
  346. os.path.join('third_party', 'apple_apsl'),
  347. os.path.join('third_party', 'apple_sample_code'),
  348. os.path.join('third_party', 'ashmem'),
  349. os.path.join('third_party', 'blink'),
  350. os.path.join('third_party', 'bspatch'),
  351. os.path.join('third_party', 'cld'),
  352. os.path.join('third_party', 'flot'),
  353. os.path.join('third_party', 'gtk+'),
  354. os.path.join('third_party', 'iaccessible2'),
  355. os.path.join('third_party', 'iccjpeg'),
  356. os.path.join('third_party', 'isimpledom'),
  357. os.path.join('third_party', 'jsoncpp'),
  358. os.path.join('third_party', 'khronos'),
  359. os.path.join('third_party', 'libevent'),
  360. os.path.join('third_party', 'libjpeg'),
  361. os.path.join('third_party', 'libusb'),
  362. os.path.join('third_party', 'libxslt'),
  363. os.path.join('third_party', 'lss'),
  364. os.path.join('third_party', 'lzma_sdk'),
  365. os.path.join('third_party', 'mesa'),
  366. os.path.join('third_party', 'motemplate'),
  367. os.path.join('third_party', 'mozc'),
  368. os.path.join('third_party', 'mozilla'),
  369. os.path.join('third_party', 'npapi'),
  370. os.path.join('third_party', 'ots'),
  371. os.path.join('third_party', 'perfetto'),
  372. os.path.join('third_party', 'ppapi'),
  373. os.path.join('third_party', 'qcms'),
  374. os.path.join('third_party', 're2'),
  375. os.path.join('third_party', 'safe_browsing'),
  376. os.path.join('third_party', 'smhasher'),
  377. os.path.join('third_party', 'sudden_motion_sensor'),
  378. os.path.join('third_party', 'swiftshader'),
  379. os.path.join('third_party', 'swig'),
  380. os.path.join('third_party', 'talloc'),
  381. os.path.join('third_party', 'usb_ids'),
  382. os.path.join('third_party', 'v8-i18n'),
  383. os.path.join('third_party', 'wtl'),
  384. os.path.join('third_party', 'yasm'),
  385. os.path.join('v8', 'strongtalk'),
  386. ])
  387. class LicenseError(Exception):
  388. """We raise this exception when a directory's licensing info isn't
  389. fully filled out."""
  390. pass
  391. def AbsolutePath(path, filename, root):
  392. """Convert a path in README.chromium to be absolute based on the source
  393. root."""
  394. if filename.startswith('/'):
  395. # Absolute-looking paths are relative to the source root
  396. # (which is the directory we're run from).
  397. absolute_path = os.path.join(root, filename[1:])
  398. else:
  399. absolute_path = os.path.join(root, path, filename)
  400. if os.path.exists(absolute_path):
  401. return absolute_path
  402. return None
  403. def ParseDir(path, root, require_license_file=True, optional_keys=None):
  404. """Examine a third_party/foo component and extract its metadata."""
  405. # Parse metadata fields out of README.chromium.
  406. # We examine "LICENSE" for the license file by default.
  407. metadata = {
  408. "License File": "LICENSE", # Relative path to license text.
  409. "Name": None, # Short name (for header on about:credits).
  410. "URL": None, # Project home page.
  411. "License": None, # Software license.
  412. }
  413. if optional_keys is None:
  414. optional_keys = []
  415. if path in SPECIAL_CASES:
  416. metadata.update(SPECIAL_CASES[path])
  417. else:
  418. # Try to find README.chromium.
  419. readme_path = os.path.join(root, path, 'README.chromium')
  420. if not os.path.exists(readme_path):
  421. raise LicenseError("missing README.chromium or licenses.py "
  422. "SPECIAL_CASES entry in %s\n" % path)
  423. for line in codecs.open(readme_path, encoding='utf-8'):
  424. line = line.strip()
  425. if not line:
  426. break
  427. for key in list(metadata.keys()) + optional_keys:
  428. field = key + ": "
  429. if line.startswith(field):
  430. metadata[key] = line[len(field):]
  431. # Check that all expected metadata is present.
  432. errors = []
  433. for key, value in metadata.items():
  434. if not value:
  435. errors.append("couldn't find '" + key + "' line "
  436. "in README.chromium or licences.py "
  437. "SPECIAL_CASES")
  438. # Special-case modules that aren't in the shipping product, so don't need
  439. # their license in about:credits.
  440. if metadata["License File"] != NOT_SHIPPED:
  441. # Check that the license file exists.
  442. for filename in (metadata["License File"], "COPYING"):
  443. license_path = AbsolutePath(path, filename, root)
  444. if license_path is not None:
  445. break
  446. if require_license_file and not license_path:
  447. errors.append("License file not found. "
  448. "Either add a file named LICENSE, "
  449. "import upstream's COPYING if available, "
  450. "or add a 'License File:' line to "
  451. "README.chromium with the appropriate path.")
  452. metadata["License File"] = license_path
  453. if errors:
  454. raise LicenseError("Errors in %s:\n %s\n" % (path, ";\n ".join(errors)))
  455. return metadata
  456. def ContainsFiles(path, root):
  457. """Determines whether any files exist in a directory or in any of its
  458. subdirectories."""
  459. for _, dirs, files in os.walk(os.path.join(root, path)):
  460. if files:
  461. return True
  462. for vcs_metadata in VCS_METADATA_DIRS:
  463. if vcs_metadata in dirs:
  464. dirs.remove(vcs_metadata)
  465. return False
  466. def FilterDirsWithFiles(dirs_list, root):
  467. # If a directory contains no files, assume it's a DEPS directory for a
  468. # project not used by our current configuration and skip it.
  469. return [x for x in dirs_list if ContainsFiles(x, root)]
  470. def ProcessAdditionalReadmePathsJson(root, dirname, third_party_dirs):
  471. """For a given directory, process the additional readme paths, and add to
  472. third_party_dirs."""
  473. additional_paths_file = os.path.join(root, dirname, ADDITIONAL_PATHS_FILENAME)
  474. if os.path.exists(additional_paths_file):
  475. with codecs.open(additional_paths_file, encoding='utf-8') as paths_file:
  476. extra_paths = json.load(paths_file)
  477. third_party_dirs.update([os.path.join(dirname, p) for p in extra_paths])
  478. def FindThirdPartyDirs(prune_paths, root, extra_third_party_dirs=None):
  479. """Find all third_party directories underneath the source root."""
  480. third_party_dirs = set()
  481. for path, dirs, files in os.walk(root):
  482. path = path[len(root) + 1:] # Pretty up the path.
  483. # .gitignore ignores /out*/, so do the same here.
  484. if path in prune_paths or path.startswith('out'):
  485. dirs[:] = []
  486. continue
  487. # Prune out directories we want to skip.
  488. # (Note that we loop over PRUNE_DIRS so we're not iterating over a
  489. # list that we're simultaneously mutating.)
  490. for skip in PRUNE_DIRS:
  491. if skip in dirs:
  492. dirs.remove(skip)
  493. if os.path.basename(path) == 'third_party':
  494. # Add all subdirectories that are not marked for skipping.
  495. for dir in dirs:
  496. dirpath = os.path.join(path, dir)
  497. if dirpath not in prune_paths:
  498. third_party_dirs.add(dirpath)
  499. ProcessAdditionalReadmePathsJson(root, dirpath, third_party_dirs)
  500. # Don't recurse into any subdirs from here.
  501. dirs[:] = []
  502. continue
  503. # Don't recurse into paths in ADDITIONAL_PATHS, like we do with regular
  504. # third_party/foo paths.
  505. if path in ADDITIONAL_PATHS:
  506. dirs[:] = []
  507. extra_paths = set(ADDITIONAL_PATHS)
  508. if extra_third_party_dirs:
  509. extra_paths.update(extra_third_party_dirs)
  510. for dir in extra_paths:
  511. if dir not in prune_paths:
  512. third_party_dirs.add(dir)
  513. ProcessAdditionalReadmePathsJson(root, dir, third_party_dirs)
  514. return sorted(third_party_dirs)
  515. def FindThirdPartyDirsWithFiles(root):
  516. third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, root)
  517. return FilterDirsWithFiles(third_party_dirs, root)
  518. # Many builders do not contain 'gn' in their PATH, so use the GN binary from
  519. # //buildtools.
  520. def _GnBinary():
  521. exe = 'gn'
  522. if sys.platform.startswith('linux'):
  523. subdir = 'linux64'
  524. elif sys.platform == 'darwin':
  525. subdir = 'mac'
  526. elif sys.platform == 'win32':
  527. subdir, exe = 'win', 'gn.exe'
  528. else:
  529. raise RuntimeError("Unsupported platform '%s'." % sys.platform)
  530. return os.path.join(_REPOSITORY_ROOT, 'buildtools', subdir, exe)
  531. def GetThirdPartyDepsFromGNDepsOutput(gn_deps, target_os):
  532. """Returns third_party/foo directories given the output of "gn desc deps".
  533. Note that it always returns the direct sub-directory of third_party
  534. where README.chromium and LICENSE files are, so that it can be passed to
  535. ParseDir(). e.g.:
  536. third_party/cld_3/src/src/BUILD.gn -> third_party/cld_3
  537. It returns relative paths from _REPOSITORY_ROOT, not absolute paths.
  538. """
  539. third_party_deps = set()
  540. for absolute_build_dep in gn_deps.split():
  541. relative_build_dep = os.path.relpath(absolute_build_dep, _REPOSITORY_ROOT)
  542. m = re.search(
  543. r'^((.+[/\\])?third_party[/\\][^/\\]+[/\\])(.+[/\\])?BUILD\.gn$',
  544. relative_build_dep)
  545. if not m:
  546. continue
  547. third_party_path = m.group(1)
  548. if any(third_party_path.startswith(p + os.sep) for p in PRUNE_PATHS):
  549. continue
  550. if (target_os == 'ios' and any(
  551. third_party_path.startswith(p + os.sep)
  552. for p in KNOWN_NON_IOS_LIBRARIES)):
  553. # Skip over files that are known not to be used on iOS.
  554. continue
  555. third_party_deps.add(third_party_path[:-1])
  556. return sorted(third_party_deps)
  557. def FindThirdPartyDeps(gn_out_dir, gn_target, target_os):
  558. if not gn_out_dir:
  559. raise RuntimeError("--gn-out-dir is required if --gn-target is used.")
  560. # Generate gn project in temp directory and use it to find dependencies.
  561. # Current gn directory cannot be used when we run this script in a gn action
  562. # rule, because gn doesn't allow recursive invocations due to potential side
  563. # effects.
  564. try:
  565. with tempfile.TemporaryDirectory(dir=gn_out_dir) as tmp_dir:
  566. shutil.copy(os.path.join(gn_out_dir, "args.gn"), tmp_dir)
  567. subprocess.check_output(
  568. [_GnBinary(), "gen",
  569. "--root=%s" % _REPOSITORY_ROOT, tmp_dir])
  570. gn_deps = subprocess.check_output([
  571. _GnBinary(), "desc",
  572. "--root=%s" % _REPOSITORY_ROOT, tmp_dir, gn_target, "deps",
  573. "--as=buildfile", "--all"
  574. ])
  575. if isinstance(gn_deps, bytes):
  576. gn_deps = gn_deps.decode("utf-8")
  577. except:
  578. if sys.platform == 'win32':
  579. print("""
  580. ############################################################################
  581. This is known issue, please report the failure to https://crbug.com/1208393.
  582. ############################################################################
  583. """)
  584. subprocess.check_call(['tasklist.exe'])
  585. raise
  586. return GetThirdPartyDepsFromGNDepsOutput(gn_deps, target_os)
  587. def ScanThirdPartyDirs(root=None):
  588. """Scan a list of directories and report on any problems we find."""
  589. if root is None:
  590. root = os.getcwd()
  591. third_party_dirs = FindThirdPartyDirsWithFiles(root)
  592. errors = []
  593. for path in sorted(third_party_dirs):
  594. try:
  595. metadata = ParseDir(path, root)
  596. except LicenseError as e:
  597. errors.append((path, e.args[0]))
  598. continue
  599. for path, error in sorted(errors):
  600. print(path + ": " + error)
  601. return len(errors) == 0
  602. def GenerateCredits(file_template_file,
  603. entry_template_file,
  604. output_file,
  605. target_os,
  606. gn_out_dir,
  607. gn_target,
  608. extra_third_party_dirs=None,
  609. depfile=None):
  610. """Generate about:credits."""
  611. def EvaluateTemplate(template, env, escape=True):
  612. """Expand a template with variables like {{foo}} using a
  613. dictionary of expansions."""
  614. for key, val in env.items():
  615. if escape:
  616. val = html.escape(val)
  617. template = template.replace('{{%s}}' % key, val)
  618. return template
  619. def MetadataToTemplateEntry(metadata, entry_template):
  620. env = {
  621. 'name': metadata['Name'],
  622. 'url': metadata['URL'],
  623. 'license': codecs.open(metadata['License File'],
  624. encoding='utf-8').read(),
  625. }
  626. return {
  627. 'name': metadata['Name'],
  628. 'content': EvaluateTemplate(entry_template, env),
  629. 'license_file': metadata['License File'],
  630. }
  631. if gn_target:
  632. third_party_dirs = FindThirdPartyDeps(gn_out_dir, gn_target, target_os)
  633. # Sanity-check to raise a build error if invalid gn_... settings are
  634. # somehow passed to this script.
  635. if not third_party_dirs:
  636. raise RuntimeError("No deps found.")
  637. else:
  638. third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, _REPOSITORY_ROOT,
  639. extra_third_party_dirs)
  640. if not file_template_file:
  641. file_template_file = os.path.join(_REPOSITORY_ROOT, 'components',
  642. 'about_ui', 'resources',
  643. 'about_credits.tmpl')
  644. if not entry_template_file:
  645. entry_template_file = os.path.join(_REPOSITORY_ROOT, 'components',
  646. 'about_ui', 'resources',
  647. 'about_credits_entry.tmpl')
  648. entry_template = codecs.open(entry_template_file, encoding='utf-8').read()
  649. entries = []
  650. # Start from Chromium's LICENSE file
  651. chromium_license_metadata = {
  652. 'Name': 'The Chromium Project',
  653. 'URL': 'http://www.chromium.org',
  654. 'License File': os.path.join(_REPOSITORY_ROOT, 'LICENSE')
  655. }
  656. entries.append(
  657. MetadataToTemplateEntry(chromium_license_metadata, entry_template))
  658. entries_by_name = {}
  659. for path in third_party_dirs:
  660. try:
  661. metadata = ParseDir(path, _REPOSITORY_ROOT)
  662. except LicenseError:
  663. # TODO(phajdan.jr): Convert to fatal error (http://crbug.com/39240).
  664. continue
  665. if metadata['License File'] == NOT_SHIPPED:
  666. continue
  667. if target_os == 'ios' and not gn_target:
  668. # Skip over files that are known not to be used on iOS. But
  669. # skipping is unnecessary if GN was used to query the actual
  670. # dependencies.
  671. # TODO(lambroslambrou): Remove this step once the iOS build is
  672. # updated to provide --gn-target to this script.
  673. if path in KNOWN_NON_IOS_LIBRARIES:
  674. continue
  675. new_entry = MetadataToTemplateEntry(metadata, entry_template)
  676. # Skip entries that we've already seen (it exists in multiple directories).
  677. prev_entry = entries_by_name.setdefault(new_entry['name'], new_entry)
  678. if prev_entry is not new_entry and (
  679. prev_entry['content'] == new_entry['content']):
  680. continue
  681. entries.append(new_entry)
  682. entries.sort(key=lambda entry: (entry['name'].lower(), entry['content']))
  683. for entry_id, entry in enumerate(entries):
  684. entry['content'] = entry['content'].replace('{{id}}', str(entry_id))
  685. entries_contents = '\n'.join([entry['content'] for entry in entries])
  686. file_template = codecs.open(file_template_file, encoding='utf-8').read()
  687. template_contents = "<!-- Generated by licenses.py; do not edit. -->"
  688. template_contents += EvaluateTemplate(
  689. file_template, {'entries': entries_contents}, escape=False)
  690. if output_file:
  691. changed = True
  692. try:
  693. old_output = codecs.open(output_file, 'r', encoding='utf-8').read()
  694. if old_output == template_contents:
  695. changed = False
  696. except:
  697. pass
  698. if changed:
  699. with codecs.open(output_file, 'w', encoding='utf-8') as output:
  700. output.write(template_contents)
  701. else:
  702. print(template_contents)
  703. if depfile:
  704. assert output_file
  705. # Add in build.ninja so that the target will be considered dirty whenever
  706. # gn gen is run. Otherwise, it will fail to notice new files being added.
  707. # This is still no perfect, as it will fail if no build files are changed,
  708. # but a new README.chromium / LICENSE is added. This shouldn't happen in
  709. # practice however.
  710. license_file_list = (entry['license_file'] for entry in entries)
  711. license_file_list = (os.path.relpath(p) for p in license_file_list)
  712. license_file_list = sorted(set(license_file_list))
  713. build_utils.WriteDepfile(depfile, output_file,
  714. license_file_list + ['build.ninja'])
  715. return True
  716. def _ReadFile(path):
  717. """Reads a file from disk.
  718. Args:
  719. path: The path of the file to read, relative to the root of the
  720. repository.
  721. Returns:
  722. The contents of the file as a string.
  723. """
  724. with codecs.open(os.path.join(_REPOSITORY_ROOT, path), 'r', 'utf-8') as f:
  725. return f.read()
  726. def GenerateLicenseFile(output_file, gn_out_dir, gn_target, target_os):
  727. """Generate a plain-text LICENSE file which can be used when you ship a part
  728. of Chromium code (specified by gn_target) as a stand-alone library
  729. (e.g., //ios/web_view).
  730. The LICENSE file contains licenses of both Chromium and third-party
  731. libraries which gn_target depends on. """
  732. third_party_dirs = FindThirdPartyDeps(gn_out_dir, gn_target, target_os)
  733. # Start with Chromium's LICENSE file.
  734. content = [_ReadFile('LICENSE')]
  735. # Add necessary third_party.
  736. for directory in sorted(third_party_dirs):
  737. metadata = ParseDir(directory, _REPOSITORY_ROOT, require_license_file=True)
  738. license_file = metadata['License File']
  739. if license_file and license_file != NOT_SHIPPED:
  740. content.append('-' * 20)
  741. content.append(directory.split(os.sep)[-1])
  742. content.append('-' * 20)
  743. content.append(_ReadFile(license_file))
  744. content_text = '\n'.join(content)
  745. if output_file:
  746. with codecs.open(output_file, 'w', 'utf-8') as output:
  747. output.write(content_text)
  748. else:
  749. print(content_text)
  750. def main():
  751. parser = argparse.ArgumentParser()
  752. parser.add_argument(
  753. '--file-template', help='Template HTML to use for the license page.')
  754. parser.add_argument(
  755. '--entry-template', help='Template HTML to use for each license.')
  756. parser.add_argument(
  757. '--extra-third-party-dirs',
  758. help='Gn list of additional third_party dirs to look through.')
  759. parser.add_argument('--target-os', help='OS that this build is targeting.')
  760. parser.add_argument(
  761. '--gn-out-dir', help='GN output directory for scanning dependencies.')
  762. parser.add_argument('--gn-target', help='GN target to scan for dependencies.')
  763. parser.add_argument(
  764. 'command', choices=['help', 'scan', 'credits', 'license_file'])
  765. parser.add_argument('output_file', nargs='?')
  766. build_utils.AddDepfileOption(parser)
  767. args = parser.parse_args()
  768. args.extra_third_party_dirs = build_utils.ParseGnList(
  769. args.extra_third_party_dirs)
  770. if args.command == 'scan':
  771. if not ScanThirdPartyDirs():
  772. return 1
  773. elif args.command == 'credits':
  774. if not GenerateCredits(args.file_template, args.entry_template,
  775. args.output_file, args.target_os, args.gn_out_dir,
  776. args.gn_target, args.extra_third_party_dirs,
  777. args.depfile):
  778. return 1
  779. elif args.command == 'license_file':
  780. try:
  781. GenerateLicenseFile(args.output_file, args.gn_out_dir, args.gn_target,
  782. args.target_os)
  783. except LicenseError as e:
  784. print("Failed to parse README.chromium: {}".format(e))
  785. return 1
  786. else:
  787. print(__doc__)
  788. return 1
  789. if __name__ == '__main__':
  790. sys.exit(main())