analyze_bcpf.py 59 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477
  1. #!/usr/bin/env -S python -u
  2. #
  3. # Copyright (C) 2022 The Android Open Source Project
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. """Analyze bootclasspath_fragment usage."""
  17. import argparse
  18. import dataclasses
  19. import enum
  20. import json
  21. import logging
  22. import os
  23. import re
  24. import shutil
  25. import subprocess
  26. import tempfile
  27. import textwrap
  28. import typing
  29. from enum import Enum
  30. import sys
  31. from signature_trie import signature_trie
  32. _STUB_FLAGS_FILE = "out/soong/hiddenapi/hiddenapi-stub-flags.txt"
  33. _FLAGS_FILE = "out/soong/hiddenapi/hiddenapi-flags.csv"
  34. _INCONSISTENT_FLAGS = "ERROR: Hidden API flags are inconsistent:"
  35. class BuildOperation:
  36. def __init__(self, popen):
  37. self.popen = popen
  38. self.returncode = None
  39. def lines(self):
  40. """Return an iterator over the lines output by the build operation.
  41. The lines have had any trailing white space, including the newline
  42. stripped.
  43. """
  44. return newline_stripping_iter(self.popen.stdout.readline)
  45. def wait(self, *args, **kwargs):
  46. self.popen.wait(*args, **kwargs)
  47. self.returncode = self.popen.returncode
  48. @dataclasses.dataclass()
  49. class FlagDiffs:
  50. """Encapsulates differences in flags reported by the build"""
  51. # Map from member signature to the (module flags, monolithic flags)
  52. diffs: typing.Dict[str, typing.Tuple[str, str]]
  53. @dataclasses.dataclass()
  54. class ModuleInfo:
  55. """Provides access to the generated module-info.json file.
  56. This is used to find the location of the file within which specific modules
  57. are defined.
  58. """
  59. modules: typing.Dict[str, typing.Dict[str, typing.Any]]
  60. @staticmethod
  61. def load(filename):
  62. with open(filename, "r", encoding="utf8") as f:
  63. j = json.load(f)
  64. return ModuleInfo(j)
  65. def _module(self, module_name):
  66. """Find module by name in module-info.json file"""
  67. if module_name in self.modules:
  68. return self.modules[module_name]
  69. raise Exception(f"Module {module_name} could not be found")
  70. def module_path(self, module_name):
  71. module = self._module(module_name)
  72. # The "path" is actually a list of paths, one for each class of module
  73. # but as the modules are all created from bp files if a module does
  74. # create multiple classes of make modules they should all have the same
  75. # path.
  76. paths = module["path"]
  77. unique_paths = set(paths)
  78. if len(unique_paths) != 1:
  79. raise Exception(f"Expected module '{module_name}' to have a "
  80. f"single unique path but found {unique_paths}")
  81. return paths[0]
  82. def extract_indent(line):
  83. return re.match(r"([ \t]*)", line).group(1)
  84. _SPECIAL_PLACEHOLDER: str = "SPECIAL_PLACEHOLDER"
  85. @dataclasses.dataclass
  86. class BpModifyRunner:
  87. bpmodify_path: str
  88. def add_values_to_property(self, property_name, values, module_name,
  89. bp_file):
  90. cmd = [
  91. self.bpmodify_path, "-a", values, "-property", property_name, "-m",
  92. module_name, "-w", bp_file, bp_file
  93. ]
  94. logging.debug(" ".join(cmd))
  95. subprocess.run(
  96. cmd,
  97. stderr=subprocess.STDOUT,
  98. stdout=log_stream_for_subprocess(),
  99. check=True)
  100. @dataclasses.dataclass
  101. class FileChange:
  102. path: str
  103. description: str
  104. def __lt__(self, other):
  105. return self.path < other.path
  106. class PropertyChangeAction(Enum):
  107. """Allowable actions that are supported by HiddenApiPropertyChange."""
  108. # New values are appended to any existing values.
  109. APPEND = 1
  110. # New values replace any existing values.
  111. REPLACE = 2
  112. @dataclasses.dataclass
  113. class HiddenApiPropertyChange:
  114. property_name: str
  115. values: typing.List[str]
  116. property_comment: str = ""
  117. # The action that indicates how this change is applied.
  118. action: PropertyChangeAction = PropertyChangeAction.APPEND
  119. def snippet(self, indent):
  120. snippet = "\n"
  121. snippet += format_comment_as_text(self.property_comment, indent)
  122. snippet += f"{indent}{self.property_name}: ["
  123. if self.values:
  124. snippet += "\n"
  125. for value in self.values:
  126. snippet += f'{indent} "{value}",\n'
  127. snippet += f"{indent}"
  128. snippet += "],\n"
  129. return snippet
  130. def fix_bp_file(self, bcpf_bp_file, bcpf, bpmodify_runner: BpModifyRunner):
  131. # Add an additional placeholder value to identify the modification that
  132. # bpmodify makes.
  133. bpmodify_values = [_SPECIAL_PLACEHOLDER]
  134. if self.action == PropertyChangeAction.APPEND:
  135. # If adding the values to the existing values then pass the new
  136. # values to bpmodify.
  137. bpmodify_values.extend(self.values)
  138. elif self.action == PropertyChangeAction.REPLACE:
  139. # If replacing the existing values then it is not possible to use
  140. # bpmodify for that directly. It could be used twice to remove the
  141. # existing property and then add a new one but that does not remove
  142. # any related comments and loses the position of the existing
  143. # property as the new property is always added to the end of the
  144. # containing block.
  145. #
  146. # So, instead of passing the new values to bpmodify this this just
  147. # adds an extra placeholder to force bpmodify to format the list
  148. # across multiple lines to ensure a consistent structure for the
  149. # code that removes all the existing values and adds the new ones.
  150. #
  151. # This placeholder has to be different to the other placeholder as
  152. # bpmodify dedups values.
  153. bpmodify_values.append(_SPECIAL_PLACEHOLDER + "_REPLACE")
  154. else:
  155. raise ValueError(f"unknown action {self.action}")
  156. packages = ",".join(bpmodify_values)
  157. bpmodify_runner.add_values_to_property(
  158. f"hidden_api.{self.property_name}", packages, bcpf, bcpf_bp_file)
  159. with open(bcpf_bp_file, "r", encoding="utf8") as tio:
  160. lines = tio.readlines()
  161. lines = [line.rstrip("\n") for line in lines]
  162. if self.fixup_bpmodify_changes(bcpf_bp_file, lines):
  163. with open(bcpf_bp_file, "w", encoding="utf8") as tio:
  164. for line in lines:
  165. print(line, file=tio)
  166. def fixup_bpmodify_changes(self, bcpf_bp_file, lines):
  167. """Fixup the output of bpmodify.
  168. The bpmodify tool does not support all the capabilities that this needs
  169. so it is used to do what it can, including marking the place in the
  170. Android.bp file where it makes its changes and then this gets passed a
  171. list of lines from that file which it then modifies to complete the
  172. change.
  173. This analyzes the list of lines to find the indices of the significant
  174. lines and then applies some changes. As those changes can insert and
  175. delete lines (changing the indices of following lines) the changes are
  176. generally done in reverse order starting from the end and working
  177. towards the beginning. That ensures that the changes do not invalidate
  178. the indices of following lines.
  179. """
  180. # Find the line containing the placeholder that has been inserted.
  181. place_holder_index = -1
  182. for i, line in enumerate(lines):
  183. if _SPECIAL_PLACEHOLDER in line:
  184. place_holder_index = i
  185. break
  186. if place_holder_index == -1:
  187. logging.debug("Could not find %s in %s", _SPECIAL_PLACEHOLDER,
  188. bcpf_bp_file)
  189. return False
  190. # Remove the place holder. Do this before inserting the comment as that
  191. # would change the location of the place holder in the list.
  192. place_holder_line = lines[place_holder_index]
  193. if place_holder_line.endswith("],"):
  194. place_holder_line = place_holder_line.replace(
  195. f'"{_SPECIAL_PLACEHOLDER}"', "")
  196. lines[place_holder_index] = place_holder_line
  197. else:
  198. del lines[place_holder_index]
  199. # Scan forward to the end of the property block to remove a blank line
  200. # that bpmodify inserts.
  201. end_property_array_index = -1
  202. for i in range(place_holder_index, len(lines)):
  203. line = lines[i]
  204. if line.endswith("],"):
  205. end_property_array_index = i
  206. break
  207. if end_property_array_index == -1:
  208. logging.debug("Could not find end of property array in %s",
  209. bcpf_bp_file)
  210. return False
  211. # If bdmodify inserted a blank line afterwards then remove it.
  212. if (not lines[end_property_array_index + 1] and
  213. lines[end_property_array_index + 2].endswith("},")):
  214. del lines[end_property_array_index + 1]
  215. # Scan back to find the preceding property line.
  216. property_line_index = -1
  217. for i in range(place_holder_index, 0, -1):
  218. line = lines[i]
  219. if line.lstrip().startswith(f"{self.property_name}: ["):
  220. property_line_index = i
  221. break
  222. if property_line_index == -1:
  223. logging.debug("Could not find property line in %s", bcpf_bp_file)
  224. return False
  225. # If this change is replacing the existing values then they need to be
  226. # removed and replaced with the new values. That will change the lines
  227. # after the property but it is necessary to do here as the following
  228. # code operates on earlier lines.
  229. if self.action == PropertyChangeAction.REPLACE:
  230. # This removes the existing values and replaces them with the new
  231. # values.
  232. indent = extract_indent(lines[property_line_index + 1])
  233. insert = [f'{indent}"{x}",' for x in self.values]
  234. lines[property_line_index + 1:end_property_array_index] = insert
  235. if not self.values:
  236. # If the property has no values then merge the ], onto the
  237. # same line as the property name.
  238. del lines[property_line_index + 1]
  239. lines[property_line_index] = lines[property_line_index] + "],"
  240. # Only insert a comment if the property does not already have a comment.
  241. line_preceding_property = lines[(property_line_index - 1)]
  242. if (self.property_comment and
  243. not re.match("([ \t]+)// ", line_preceding_property)):
  244. # Extract the indent from the property line and use it to format the
  245. # comment.
  246. indent = extract_indent(lines[property_line_index])
  247. comment_lines = format_comment_as_lines(self.property_comment,
  248. indent)
  249. # If the line before the comment is not blank then insert an extra
  250. # blank line at the beginning of the comment.
  251. if line_preceding_property:
  252. comment_lines.insert(0, "")
  253. # Insert the comment before the property.
  254. lines[property_line_index:property_line_index] = comment_lines
  255. return True
  256. @dataclasses.dataclass()
  257. class PackagePropertyReason:
  258. """Provides the reasons why a package was added to a specific property.
  259. A split package is one that contains classes from the bootclasspath_fragment
  260. and other bootclasspath modules. So, for a split package this contains the
  261. corresponding lists of classes.
  262. A single package is one that contains classes sub-packages from the
  263. For a split package this contains a list of classes in that package that are
  264. provided by the bootclasspath_fragment and a list of classes
  265. """
  266. # The list of classes/sub-packages that is provided by the
  267. # bootclasspath_fragment.
  268. bcpf: typing.List[str]
  269. # The list of classes/sub-packages that is provided by other modules on the
  270. # bootclasspath.
  271. other: typing.List[str]
  272. @dataclasses.dataclass()
  273. class Result:
  274. """Encapsulates the result of the analysis."""
  275. # The diffs in the flags.
  276. diffs: typing.Optional[FlagDiffs] = None
  277. # A map from package name to the reason why it belongs in the
  278. # split_packages property.
  279. split_packages: typing.Dict[str, PackagePropertyReason] = dataclasses.field(
  280. default_factory=dict)
  281. # A map from package name to the reason why it belongs in the
  282. # single_packages property.
  283. single_packages: typing.Dict[str,
  284. PackagePropertyReason] = dataclasses.field(
  285. default_factory=dict)
  286. # The list of packages to add to the package_prefixes property.
  287. package_prefixes: typing.List[str] = dataclasses.field(default_factory=list)
  288. # The bootclasspath_fragment hidden API properties changes.
  289. property_changes: typing.List[HiddenApiPropertyChange] = dataclasses.field(
  290. default_factory=list)
  291. # The list of file changes.
  292. file_changes: typing.List[FileChange] = dataclasses.field(
  293. default_factory=list)
  294. class ClassProvider(enum.Enum):
  295. """The source of a class found during the hidden API processing"""
  296. BCPF = "bcpf"
  297. OTHER = "other"
  298. # A fake member to use when using the signature trie to compute the package
  299. # properties from hidden API flags. This is needed because while that
  300. # computation only cares about classes the trie expects a class to be an
  301. # interior node but without a member it makes the class a leaf node. That causes
  302. # problems when analyzing inner classes as the outer class is a leaf node for
  303. # its own entry but is used as an interior node for inner classes.
  304. _FAKE_MEMBER = ";->fake()V"
  305. @dataclasses.dataclass()
  306. class BcpfAnalyzer:
  307. # Path to this tool.
  308. tool_path: str
  309. # Directory pointed to by ANDROID_BUILD_OUT
  310. top_dir: str
  311. # Directory pointed to by OUT_DIR of {top_dir}/out if that is not set.
  312. out_dir: str
  313. # Directory pointed to by ANDROID_PRODUCT_OUT.
  314. product_out_dir: str
  315. # The name of the bootclasspath_fragment module.
  316. bcpf: str
  317. # The name of the apex module containing {bcpf}, only used for
  318. # informational purposes.
  319. apex: str
  320. # The name of the sdk module containing {bcpf}, only used for
  321. # informational purposes.
  322. sdk: str
  323. # If true then this will attempt to automatically fix any issues that are
  324. # found.
  325. fix: bool = False
  326. # All the signatures, loaded from all-flags.csv, initialized by
  327. # load_all_flags().
  328. _signatures: typing.Set[str] = dataclasses.field(default_factory=set)
  329. # All the classes, loaded from all-flags.csv, initialized by
  330. # load_all_flags().
  331. _classes: typing.Set[str] = dataclasses.field(default_factory=set)
  332. # Information loaded from module-info.json, initialized by
  333. # load_module_info().
  334. module_info: ModuleInfo = None
  335. @staticmethod
  336. def reformat_report_test(text):
  337. return re.sub(r"(.)\n([^\s])", r"\1 \2", text)
  338. def report(self, text="", **kwargs):
  339. # Concatenate lines that are not separated by a blank line together to
  340. # eliminate formatting applied to the supplied text to adhere to python
  341. # line length limitations.
  342. text = self.reformat_report_test(text)
  343. logging.info("%s", text, **kwargs)
  344. def report_dedent(self, text, **kwargs):
  345. text = textwrap.dedent(text)
  346. self.report(text, **kwargs)
  347. def run_command(self, cmd, *args, **kwargs):
  348. cmd_line = " ".join(cmd)
  349. logging.debug("Running %s", cmd_line)
  350. subprocess.run(
  351. cmd,
  352. *args,
  353. check=True,
  354. cwd=self.top_dir,
  355. stderr=subprocess.STDOUT,
  356. stdout=log_stream_for_subprocess(),
  357. text=True,
  358. **kwargs)
  359. @property
  360. def signatures(self):
  361. if not self._signatures:
  362. raise Exception("signatures has not been initialized")
  363. return self._signatures
  364. @property
  365. def classes(self):
  366. if not self._classes:
  367. raise Exception("classes has not been initialized")
  368. return self._classes
  369. def load_all_flags(self):
  370. all_flags = self.find_bootclasspath_fragment_output_file(
  371. "all-flags.csv")
  372. # Extract the set of signatures and a separate set of classes produced
  373. # by the bootclasspath_fragment.
  374. with open(all_flags, "r", encoding="utf8") as f:
  375. for line in newline_stripping_iter(f.readline):
  376. signature = self.line_to_signature(line)
  377. self._signatures.add(signature)
  378. class_name = self.signature_to_class(signature)
  379. self._classes.add(class_name)
  380. def load_module_info(self):
  381. module_info_file = os.path.join(self.product_out_dir,
  382. "module-info.json")
  383. self.report(f"\nMaking sure that {module_info_file} is up to date.\n")
  384. output = self.build_file_read_output(module_info_file)
  385. lines = output.lines()
  386. for line in lines:
  387. logging.debug("%s", line)
  388. output.wait(timeout=10)
  389. if output.returncode:
  390. raise Exception(f"Error building {module_info_file}")
  391. abs_module_info_file = os.path.join(self.top_dir, module_info_file)
  392. self.module_info = ModuleInfo.load(abs_module_info_file)
  393. @staticmethod
  394. def line_to_signature(line):
  395. return line.split(",")[0]
  396. @staticmethod
  397. def signature_to_class(signature):
  398. return signature.split(";->")[0]
  399. @staticmethod
  400. def to_parent_package(pkg_or_class):
  401. return pkg_or_class.rsplit("/", 1)[0]
  402. def module_path(self, module_name):
  403. return self.module_info.module_path(module_name)
  404. def module_out_dir(self, module_name):
  405. module_path = self.module_path(module_name)
  406. return os.path.join(self.out_dir, "soong/.intermediates", module_path,
  407. module_name)
  408. def find_bootclasspath_fragment_output_file(self, basename, required=True):
  409. # Find the output file of the bootclasspath_fragment with the specified
  410. # base name.
  411. found_file = ""
  412. bcpf_out_dir = self.module_out_dir(self.bcpf)
  413. for (dirpath, _, filenames) in os.walk(bcpf_out_dir):
  414. for f in filenames:
  415. if f == basename:
  416. found_file = os.path.join(dirpath, f)
  417. break
  418. if not found_file and required:
  419. raise Exception(f"Could not find {basename} in {bcpf_out_dir}")
  420. return found_file
  421. def analyze(self):
  422. """Analyze a bootclasspath_fragment module.
  423. Provides help in resolving any existing issues and provides
  424. optimizations that can be applied.
  425. """
  426. self.report(f"Analyzing bootclasspath_fragment module {self.bcpf}")
  427. self.report_dedent(f"""
  428. Run this tool to help initialize a bootclasspath_fragment module.
  429. Before you start make sure that:
  430. 1. The current checkout is up to date.
  431. 2. The environment has been initialized using lunch, e.g.
  432. lunch aosp_arm64-userdebug
  433. 3. You have added a bootclasspath_fragment module to the appropriate
  434. Android.bp file. Something like this:
  435. bootclasspath_fragment {{
  436. name: "{self.bcpf}",
  437. contents: [
  438. "...",
  439. ],
  440. // The bootclasspath_fragments that provide APIs on which this
  441. // depends.
  442. fragments: [
  443. {{
  444. apex: "com.android.art",
  445. module: "art-bootclasspath-fragment",
  446. }},
  447. ],
  448. }}
  449. 4. You have added it to the platform_bootclasspath module in
  450. frameworks/base/boot/Android.bp. Something like this:
  451. platform_bootclasspath {{
  452. name: "platform-bootclasspath",
  453. fragments: [
  454. ...
  455. {{
  456. apex: "{self.apex}",
  457. module: "{self.bcpf}",
  458. }},
  459. ],
  460. }}
  461. 5. You have added an sdk module. Something like this:
  462. sdk {{
  463. name: "{self.sdk}",
  464. bootclasspath_fragments: ["{self.bcpf}"],
  465. }}
  466. """)
  467. # Make sure that the module-info.json file is up to date.
  468. self.load_module_info()
  469. self.report_dedent("""
  470. Cleaning potentially stale files.
  471. """)
  472. # Remove the out/soong/hiddenapi files.
  473. shutil.rmtree(f"{self.out_dir}/soong/hiddenapi", ignore_errors=True)
  474. # Remove any bootclasspath_fragment output files.
  475. shutil.rmtree(self.module_out_dir(self.bcpf), ignore_errors=True)
  476. self.build_monolithic_stubs_flags()
  477. result = Result()
  478. self.build_monolithic_flags(result)
  479. self.analyze_hiddenapi_package_properties(result)
  480. self.explain_how_to_check_signature_patterns()
  481. # If there were any changes that need to be made to the Android.bp
  482. # file then either apply or report them.
  483. if result.property_changes:
  484. bcpf_dir = self.module_info.module_path(self.bcpf)
  485. bcpf_bp_file = os.path.join(self.top_dir, bcpf_dir, "Android.bp")
  486. if self.fix:
  487. tool_dir = os.path.dirname(self.tool_path)
  488. bpmodify_path = os.path.join(tool_dir, "bpmodify")
  489. bpmodify_runner = BpModifyRunner(bpmodify_path)
  490. for property_change in result.property_changes:
  491. property_change.fix_bp_file(bcpf_bp_file, self.bcpf,
  492. bpmodify_runner)
  493. result.file_changes.append(
  494. self.new_file_change(
  495. bcpf_bp_file,
  496. f"Updated hidden_api properties of '{self.bcpf}'"))
  497. else:
  498. hiddenapi_snippet = ""
  499. for property_change in result.property_changes:
  500. hiddenapi_snippet += property_change.snippet(" ")
  501. # Remove leading and trailing blank lines.
  502. hiddenapi_snippet = hiddenapi_snippet.strip("\n")
  503. result.file_changes.append(
  504. self.new_file_change(
  505. bcpf_bp_file, f"""
  506. Add the following snippet into the {self.bcpf} bootclasspath_fragment module
  507. in the {bcpf_dir}/Android.bp file. If the hidden_api block already exists then
  508. merge these properties into it.
  509. hidden_api: {{
  510. {hiddenapi_snippet}
  511. }},
  512. """))
  513. if result.file_changes:
  514. if self.fix:
  515. file_change_message = textwrap.dedent("""
  516. The following files were modified by this script:
  517. """)
  518. else:
  519. file_change_message = textwrap.dedent("""
  520. The following modifications need to be made:
  521. """)
  522. self.report(file_change_message)
  523. result.file_changes.sort()
  524. for file_change in result.file_changes:
  525. self.report(f" {file_change.path}")
  526. self.report(f" {file_change.description}")
  527. self.report()
  528. if not self.fix:
  529. self.report_dedent("""
  530. Run the command again with the --fix option to automatically
  531. make the above changes.
  532. """.lstrip("\n"))
  533. def new_file_change(self, file, description):
  534. return FileChange(
  535. path=os.path.relpath(file, self.top_dir), description=description)
  536. def check_inconsistent_flag_lines(self, significant, module_line,
  537. monolithic_line, separator_line):
  538. if not (module_line.startswith("< ") and
  539. monolithic_line.startswith("> ") and not separator_line):
  540. # Something went wrong.
  541. self.report("Invalid build output detected:")
  542. self.report(f" module_line: '{module_line}'")
  543. self.report(f" monolithic_line: '{monolithic_line}'")
  544. self.report(f" separator_line: '{separator_line}'")
  545. sys.exit(1)
  546. if significant:
  547. logging.debug("%s", module_line)
  548. logging.debug("%s", monolithic_line)
  549. logging.debug("%s", separator_line)
  550. def scan_inconsistent_flags_report(self, lines):
  551. """Scans a hidden API flags report
  552. The hidden API inconsistent flags report which looks something like
  553. this.
  554. < out/soong/.intermediates/.../filtered-stub-flags.csv
  555. > out/soong/hiddenapi/hiddenapi-stub-flags.txt
  556. < Landroid/compat/Compatibility;->clearOverrides()V
  557. > Landroid/compat/Compatibility;->clearOverrides()V,core-platform-api
  558. """
  559. # The basic format of an entry in the inconsistent flags report is:
  560. # <module specific flag>
  561. # <monolithic flag>
  562. # <separator>
  563. #
  564. # Wrap the lines iterator in an iterator which returns a tuple
  565. # consisting of the three separate lines.
  566. triples = zip(lines, lines, lines)
  567. module_line, monolithic_line, separator_line = next(triples)
  568. significant = False
  569. bcpf_dir = self.module_info.module_path(self.bcpf)
  570. if os.path.join(bcpf_dir, self.bcpf) in module_line:
  571. # These errors are related to the bcpf being analyzed so
  572. # keep them.
  573. significant = True
  574. else:
  575. self.report(f"Filtering out errors related to {module_line}")
  576. self.check_inconsistent_flag_lines(significant, module_line,
  577. monolithic_line, separator_line)
  578. diffs = {}
  579. for module_line, monolithic_line, separator_line in triples:
  580. self.check_inconsistent_flag_lines(significant, module_line,
  581. monolithic_line, "")
  582. module_parts = module_line.removeprefix("< ").split(",")
  583. module_signature = module_parts[0]
  584. module_flags = module_parts[1:]
  585. monolithic_parts = monolithic_line.removeprefix("> ").split(",")
  586. monolithic_signature = monolithic_parts[0]
  587. monolithic_flags = monolithic_parts[1:]
  588. if module_signature != monolithic_signature:
  589. # Something went wrong.
  590. self.report("Inconsistent signatures detected:")
  591. self.report(f" module_signature: '{module_signature}'")
  592. self.report(f" monolithic_signature: '{monolithic_signature}'")
  593. sys.exit(1)
  594. diffs[module_signature] = (module_flags, monolithic_flags)
  595. if separator_line:
  596. # If the separator line is not blank then it is the end of the
  597. # current report, and possibly the start of another.
  598. return separator_line, diffs
  599. return "", diffs
  600. def build_file_read_output(self, filename):
  601. # Make sure the filename is relative to top if possible as the build
  602. # may be using relative paths as the target.
  603. rel_filename = filename.removeprefix(self.top_dir)
  604. cmd = ["build/soong/soong_ui.bash", "--make-mode", rel_filename]
  605. cmd_line = " ".join(cmd)
  606. logging.debug("%s", cmd_line)
  607. # pylint: disable=consider-using-with
  608. output = subprocess.Popen(
  609. cmd,
  610. cwd=self.top_dir,
  611. stderr=subprocess.STDOUT,
  612. stdout=subprocess.PIPE,
  613. text=True,
  614. )
  615. return BuildOperation(popen=output)
  616. def build_hiddenapi_flags(self, filename):
  617. output = self.build_file_read_output(filename)
  618. lines = output.lines()
  619. diffs = None
  620. for line in lines:
  621. logging.debug("%s", line)
  622. while line == _INCONSISTENT_FLAGS:
  623. line, diffs = self.scan_inconsistent_flags_report(lines)
  624. output.wait(timeout=10)
  625. if output.returncode != 0:
  626. logging.debug("Command failed with %s", output.returncode)
  627. else:
  628. logging.debug("Command succeeded")
  629. return diffs
  630. def build_monolithic_stubs_flags(self):
  631. self.report_dedent(f"""
  632. Attempting to build {_STUB_FLAGS_FILE} to verify that the
  633. bootclasspath_fragment has the correct API stubs available...
  634. """)
  635. # Build the hiddenapi-stubs-flags.txt file.
  636. diffs = self.build_hiddenapi_flags(_STUB_FLAGS_FILE)
  637. if diffs:
  638. self.report_dedent(f"""
  639. There is a discrepancy between the stub API derived flags
  640. created by the bootclasspath_fragment and the
  641. platform_bootclasspath. See preceding error messages to see
  642. which flags are inconsistent. The inconsistencies can occur for
  643. a couple of reasons:
  644. If you are building against prebuilts of the Android SDK, e.g.
  645. by using TARGET_BUILD_APPS then the prebuilt versions of the
  646. APIs this bootclasspath_fragment depends upon are out of date
  647. and need updating. See go/update-prebuilts for help.
  648. Otherwise, this is happening because there are some stub APIs
  649. that are either provided by or used by the contents of the
  650. bootclasspath_fragment but which are not available to it. There
  651. are 4 ways to handle this:
  652. 1. A java_sdk_library in the contents property will
  653. automatically make its stub APIs available to the
  654. bootclasspath_fragment so nothing needs to be done.
  655. 2. If the API provided by the bootclasspath_fragment is created
  656. by an api_only java_sdk_library (or a java_library that compiles
  657. files generated by a separate droidstubs module then it cannot
  658. be added to the contents and instead must be added to the
  659. api.stubs property, e.g.
  660. bootclasspath_fragment {{
  661. name: "{self.bcpf}",
  662. ...
  663. api: {{
  664. stubs: ["$MODULE-api-only"],"
  665. }},
  666. }}
  667. 3. If the contents use APIs provided by another
  668. bootclasspath_fragment then it needs to be added to the
  669. fragments property, e.g.
  670. bootclasspath_fragment {{
  671. name: "{self.bcpf}",
  672. ...
  673. // The bootclasspath_fragments that provide APIs on which this depends.
  674. fragments: [
  675. ...
  676. {{
  677. apex: "com.android.other",
  678. module: "com.android.other-bootclasspath-fragment",
  679. }},
  680. ],
  681. }}
  682. 4. If the contents use APIs from a module that is not part of
  683. another bootclasspath_fragment then it must be added to the
  684. additional_stubs property, e.g.
  685. bootclasspath_fragment {{
  686. name: "{self.bcpf}",
  687. ...
  688. additional_stubs: ["android-non-updatable"],
  689. }}
  690. Like the api.stubs property these are typically
  691. java_sdk_library modules but can be java_library too.
  692. Note: The "android-non-updatable" is treated as if it was a
  693. java_sdk_library which it is not at the moment but will be in
  694. future.
  695. """)
  696. return diffs
  697. def build_monolithic_flags(self, result):
  698. self.report_dedent(f"""
  699. Attempting to build {_FLAGS_FILE} to verify that the
  700. bootclasspath_fragment has the correct hidden API flags...
  701. """)
  702. # Build the hiddenapi-flags.csv file and extract any differences in
  703. # the flags between this bootclasspath_fragment and the monolithic
  704. # files.
  705. result.diffs = self.build_hiddenapi_flags(_FLAGS_FILE)
  706. # Load information from the bootclasspath_fragment's all-flags.csv file.
  707. self.load_all_flags()
  708. if result.diffs:
  709. self.report_dedent(f"""
  710. There is a discrepancy between the hidden API flags created by
  711. the bootclasspath_fragment and the platform_bootclasspath. See
  712. preceding error messages to see which flags are inconsistent.
  713. The inconsistencies can occur for a couple of reasons:
  714. If you are building against prebuilts of this
  715. bootclasspath_fragment then the prebuilt version of the sdk
  716. snapshot (specifically the hidden API flag files) are
  717. inconsistent with the prebuilt version of the apex {self.apex}.
  718. Please ensure that they are both updated from the same build.
  719. 1. There are custom hidden API flags specified in the one of the
  720. files in frameworks/base/boot/hiddenapi which apply to the
  721. bootclasspath_fragment but which are not supplied to the
  722. bootclasspath_fragment module.
  723. 2. The bootclasspath_fragment specifies invalid
  724. "split_packages", "single_packages" and/of "package_prefixes"
  725. properties that match packages and classes that it does not
  726. provide.
  727. """)
  728. # Check to see if there are any hiddenapi related properties that
  729. # need to be added to the
  730. self.report_dedent("""
  731. Checking custom hidden API flags....
  732. """)
  733. self.check_frameworks_base_boot_hidden_api_files(result)
  734. def report_hidden_api_flag_file_changes(self, result, property_name,
  735. flags_file, rel_bcpf_flags_file,
  736. bcpf_flags_file):
  737. matched_signatures = set()
  738. # Open the flags file to read the flags from.
  739. with open(flags_file, "r", encoding="utf8") as f:
  740. for signature in newline_stripping_iter(f.readline):
  741. if signature in self.signatures:
  742. # The signature is provided by the bootclasspath_fragment so
  743. # it will need to be moved to the bootclasspath_fragment
  744. # specific file.
  745. matched_signatures.add(signature)
  746. # If the bootclasspath_fragment specific flags file is not empty
  747. # then it contains flags. That could either be new flags just moved
  748. # from frameworks/base or previous contents of the file. In either
  749. # case the file must not be removed.
  750. if matched_signatures:
  751. insert = textwrap.indent("\n".join(matched_signatures),
  752. " ")
  753. result.file_changes.append(
  754. self.new_file_change(
  755. flags_file, f"""Remove the following entries:
  756. {insert}
  757. """))
  758. result.file_changes.append(
  759. self.new_file_change(
  760. bcpf_flags_file, f"""Add the following entries:
  761. {insert}
  762. """))
  763. result.property_changes.append(
  764. HiddenApiPropertyChange(
  765. property_name=property_name,
  766. values=[rel_bcpf_flags_file],
  767. ))
  768. def fix_hidden_api_flag_files(self, result, property_name, flags_file,
  769. rel_bcpf_flags_file, bcpf_flags_file):
  770. # Read the file in frameworks/base/boot/hiddenapi/<file> copy any
  771. # flags that relate to the bootclasspath_fragment into a local
  772. # file in the hiddenapi subdirectory.
  773. tmp_flags_file = flags_file + ".tmp"
  774. # Make sure the directory containing the bootclasspath_fragment specific
  775. # hidden api flags exists.
  776. os.makedirs(os.path.dirname(bcpf_flags_file), exist_ok=True)
  777. bcpf_flags_file_exists = os.path.exists(bcpf_flags_file)
  778. matched_signatures = set()
  779. # Open the flags file to read the flags from.
  780. with open(flags_file, "r", encoding="utf8") as f:
  781. # Open a temporary file to write the flags (minus any removed
  782. # flags).
  783. with open(tmp_flags_file, "w", encoding="utf8") as t:
  784. # Open the bootclasspath_fragment file for append just in
  785. # case it already exists.
  786. with open(bcpf_flags_file, "a", encoding="utf8") as b:
  787. for line in iter(f.readline, ""):
  788. signature = line.rstrip()
  789. if signature in self.signatures:
  790. # The signature is provided by the
  791. # bootclasspath_fragment so write it to the new
  792. # bootclasspath_fragment specific file.
  793. print(line, file=b, end="")
  794. matched_signatures.add(signature)
  795. else:
  796. # The signature is NOT provided by the
  797. # bootclasspath_fragment. Copy it to the new
  798. # monolithic file.
  799. print(line, file=t, end="")
  800. # If the bootclasspath_fragment specific flags file is not empty
  801. # then it contains flags. That could either be new flags just moved
  802. # from frameworks/base or previous contents of the file. In either
  803. # case the file must not be removed.
  804. if matched_signatures:
  805. # There are custom flags related to the bootclasspath_fragment
  806. # so replace the frameworks/base/boot/hiddenapi file with the
  807. # file that does not contain those flags.
  808. shutil.move(tmp_flags_file, flags_file)
  809. result.file_changes.append(
  810. self.new_file_change(flags_file,
  811. f"Removed '{self.bcpf}' specific entries"))
  812. result.property_changes.append(
  813. HiddenApiPropertyChange(
  814. property_name=property_name,
  815. values=[rel_bcpf_flags_file],
  816. ))
  817. # Make sure that the files are sorted.
  818. self.run_command([
  819. "tools/platform-compat/hiddenapi/sort_api.sh",
  820. bcpf_flags_file,
  821. ])
  822. if bcpf_flags_file_exists:
  823. desc = f"Added '{self.bcpf}' specific entries"
  824. else:
  825. desc = f"Created with '{self.bcpf}' specific entries"
  826. result.file_changes.append(
  827. self.new_file_change(bcpf_flags_file, desc))
  828. else:
  829. # There are no custom flags related to the
  830. # bootclasspath_fragment so clean up the working files.
  831. os.remove(tmp_flags_file)
  832. if not bcpf_flags_file_exists:
  833. os.remove(bcpf_flags_file)
  834. def check_frameworks_base_boot_hidden_api_files(self, result):
  835. hiddenapi_dir = os.path.join(self.top_dir,
  836. "frameworks/base/boot/hiddenapi")
  837. for basename in sorted(os.listdir(hiddenapi_dir)):
  838. if not (basename.startswith("hiddenapi-") and
  839. basename.endswith(".txt")):
  840. continue
  841. flags_file = os.path.join(hiddenapi_dir, basename)
  842. logging.debug("Checking %s for flags related to %s", flags_file,
  843. self.bcpf)
  844. # Map the file name in frameworks/base/boot/hiddenapi into a
  845. # slightly more meaningful name for use by the
  846. # bootclasspath_fragment.
  847. if basename == "hiddenapi-max-target-o.txt":
  848. basename = "hiddenapi-max-target-o-low-priority.txt"
  849. elif basename == "hiddenapi-max-target-r-loprio.txt":
  850. basename = "hiddenapi-max-target-r-low-priority.txt"
  851. property_name = basename.removeprefix("hiddenapi-")
  852. property_name = property_name.removesuffix(".txt")
  853. property_name = property_name.replace("-", "_")
  854. rel_bcpf_flags_file = f"hiddenapi/{basename}"
  855. bcpf_dir = self.module_info.module_path(self.bcpf)
  856. bcpf_flags_file = os.path.join(self.top_dir, bcpf_dir,
  857. rel_bcpf_flags_file)
  858. if self.fix:
  859. self.fix_hidden_api_flag_files(result, property_name,
  860. flags_file, rel_bcpf_flags_file,
  861. bcpf_flags_file)
  862. else:
  863. self.report_hidden_api_flag_file_changes(
  864. result, property_name, flags_file, rel_bcpf_flags_file,
  865. bcpf_flags_file)
  866. @staticmethod
  867. def split_package_comment(split_packages):
  868. if split_packages:
  869. return textwrap.dedent("""
  870. The following packages contain classes from other modules on the
  871. bootclasspath. That means that the hidden API flags for this
  872. module has to explicitly list every single class this module
  873. provides in that package to differentiate them from the classes
  874. provided by other modules. That can include private classes that
  875. are not part of the API.
  876. """).strip("\n")
  877. return "This module does not contain any split packages."
  878. @staticmethod
  879. def package_prefixes_comment():
  880. return textwrap.dedent("""
  881. The following packages and all their subpackages currently only
  882. contain classes from this bootclasspath_fragment. Listing a package
  883. here won't prevent other bootclasspath modules from adding classes
  884. in any of those packages but it will prevent them from adding those
  885. classes into an API surface, e.g. public, system, etc.. Doing so
  886. will result in a build failure due to inconsistent flags.
  887. """).strip("\n")
  888. def analyze_hiddenapi_package_properties(self, result):
  889. self.compute_hiddenapi_package_properties(result)
  890. def indent_lines(lines):
  891. return "\n".join([f" {cls}" for cls in lines])
  892. # TODO(b/202154151): Find those classes in split packages that are not
  893. # part of an API, i.e. are an internal implementation class, and so
  894. # can, and should, be safely moved out of the split packages.
  895. split_packages = result.split_packages.keys()
  896. result.property_changes.append(
  897. HiddenApiPropertyChange(
  898. property_name="split_packages",
  899. values=split_packages,
  900. property_comment=self.split_package_comment(split_packages),
  901. action=PropertyChangeAction.REPLACE,
  902. ))
  903. if split_packages:
  904. self.report_dedent(f"""
  905. bootclasspath_fragment {self.bcpf} contains classes in packages
  906. that also contain classes provided by other bootclasspath
  907. modules. Those packages are called split packages. Split
  908. packages should be avoided where possible but are often
  909. unavoidable when modularizing existing code.
  910. The hidden api processing needs to know which packages are split
  911. (and conversely which are not) so that it can optimize the
  912. hidden API flags to remove unnecessary implementation details.
  913. By default (for backwards compatibility) the
  914. bootclasspath_fragment assumes that all packages are split
  915. unless one of the package_prefixes or split_packages properties
  916. are specified. While that is safe it is not optimal and can lead
  917. to unnecessary implementation details leaking into the hidden
  918. API flags. Adding an empty split_packages property allows the
  919. flags to be optimized and remove any unnecessary implementation
  920. details.
  921. """)
  922. for package in split_packages:
  923. reason = result.split_packages[package]
  924. self.report(f"""
  925. Package {package} is split because while this bootclasspath_fragment
  926. provides the following classes:
  927. {indent_lines(reason.bcpf)}
  928. Other module(s) on the bootclasspath provides the following classes in
  929. that package:
  930. {indent_lines(reason.other)}
  931. """)
  932. single_packages = result.single_packages.keys()
  933. if single_packages:
  934. result.property_changes.append(
  935. HiddenApiPropertyChange(
  936. property_name="single_packages",
  937. values=single_packages,
  938. property_comment=textwrap.dedent("""
  939. The following packages currently only contain classes from
  940. this bootclasspath_fragment but some of their sub-packages
  941. contain classes from other bootclasspath modules. Packages
  942. should only be listed here when necessary for legacy
  943. purposes, new packages should match a package prefix.
  944. """),
  945. action=PropertyChangeAction.REPLACE,
  946. ))
  947. self.report_dedent(f"""
  948. bootclasspath_fragment {self.bcpf} contains classes from
  949. packages that has sub-packages which contain classes provided by
  950. other bootclasspath modules. Those packages are called single
  951. packages. Single packages should be avoided where possible but
  952. are often unavoidable when modularizing existing code.
  953. Because some sub-packages contains classes from other
  954. bootclasspath modules it is not possible to use the package as a
  955. package prefix as that treats the package and all its
  956. sub-packages as being provided by this module.
  957. """)
  958. for package in single_packages:
  959. reason = result.single_packages[package]
  960. self.report(f"""
  961. Package {package} is not a package prefix because while this
  962. bootclasspath_fragment provides the following sub-packages:
  963. {indent_lines(reason.bcpf)}
  964. Other module(s) on the bootclasspath provide the following sub-packages:
  965. {indent_lines(reason.other)}
  966. """)
  967. package_prefixes = result.package_prefixes
  968. if package_prefixes:
  969. result.property_changes.append(
  970. HiddenApiPropertyChange(
  971. property_name="package_prefixes",
  972. values=package_prefixes,
  973. property_comment=self.package_prefixes_comment(),
  974. action=PropertyChangeAction.REPLACE,
  975. ))
  976. def explain_how_to_check_signature_patterns(self):
  977. signature_patterns_files = self.find_bootclasspath_fragment_output_file(
  978. "signature-patterns.csv", required=False)
  979. if signature_patterns_files:
  980. signature_patterns_files = signature_patterns_files.removeprefix(
  981. self.top_dir)
  982. self.report_dedent(f"""
  983. The purpose of the hiddenapi split_packages and package_prefixes
  984. properties is to allow the removal of implementation details
  985. from the hidden API flags to reduce the coupling between sdk
  986. snapshots and the APEX runtime. It cannot eliminate that
  987. coupling completely though. Doing so may require changes to the
  988. code.
  989. This tool provides support for managing those properties but it
  990. cannot decide whether the set of package prefixes suggested is
  991. appropriate that needs the input of the developer.
  992. Please run the following command:
  993. m {signature_patterns_files}
  994. And then check the '{signature_patterns_files}' for any mention
  995. of implementation classes and packages (i.e. those
  996. classes/packages that do not contain any part of an API surface,
  997. including the hidden API). If they are found then the code
  998. should ideally be moved to a package unique to this module that
  999. is contained within a package that is part of an API surface.
  1000. The format of the file is a list of patterns:
  1001. * Patterns for split packages will list every class in that package.
  1002. * Patterns for package prefixes will end with .../**.
  1003. * Patterns for packages which are not split but cannot use a
  1004. package prefix because there are sub-packages which are provided
  1005. by another module will end with .../*.
  1006. """)
  1007. def compute_hiddenapi_package_properties(self, result):
  1008. trie = signature_trie()
  1009. # Populate the trie with the classes that are provided by the
  1010. # bootclasspath_fragment tagging them to make it clear where they
  1011. # are from.
  1012. sorted_classes = sorted(self.classes)
  1013. for class_name in sorted_classes:
  1014. trie.add(class_name + _FAKE_MEMBER, ClassProvider.BCPF)
  1015. # Now the same for monolithic classes.
  1016. monolithic_classes = set()
  1017. abs_flags_file = os.path.join(self.top_dir, _FLAGS_FILE)
  1018. with open(abs_flags_file, "r", encoding="utf8") as f:
  1019. for line in iter(f.readline, ""):
  1020. signature = self.line_to_signature(line)
  1021. class_name = self.signature_to_class(signature)
  1022. if (class_name not in monolithic_classes and
  1023. class_name not in self.classes):
  1024. trie.add(
  1025. class_name + _FAKE_MEMBER,
  1026. ClassProvider.OTHER,
  1027. only_if_matches=True)
  1028. monolithic_classes.add(class_name)
  1029. self.recurse_hiddenapi_packages_trie(trie, result)
  1030. @staticmethod
  1031. def selector_to_java_reference(node):
  1032. return node.selector.replace("/", ".")
  1033. @staticmethod
  1034. def determine_reason_for_single_package(node):
  1035. bcpf_packages = []
  1036. other_packages = []
  1037. def recurse(n):
  1038. if n.type != "package":
  1039. return
  1040. providers = n.get_matching_rows("*")
  1041. package_ref = BcpfAnalyzer.selector_to_java_reference(n)
  1042. if ClassProvider.BCPF in providers:
  1043. bcpf_packages.append(package_ref)
  1044. else:
  1045. other_packages.append(package_ref)
  1046. children = n.child_nodes()
  1047. if children:
  1048. for child in children:
  1049. recurse(child)
  1050. recurse(node)
  1051. return PackagePropertyReason(bcpf=bcpf_packages, other=other_packages)
  1052. @staticmethod
  1053. def determine_reason_for_split_package(node):
  1054. bcpf_classes = []
  1055. other_classes = []
  1056. for child in node.child_nodes():
  1057. if child.type != "class":
  1058. continue
  1059. providers = child.values(lambda _: True)
  1060. class_ref = BcpfAnalyzer.selector_to_java_reference(child)
  1061. if ClassProvider.BCPF in providers:
  1062. bcpf_classes.append(class_ref)
  1063. else:
  1064. other_classes.append(class_ref)
  1065. return PackagePropertyReason(bcpf=bcpf_classes, other=other_classes)
  1066. def recurse_hiddenapi_packages_trie(self, node, result):
  1067. nodes = node.child_nodes()
  1068. if nodes:
  1069. for child in nodes:
  1070. # Ignore any non-package nodes.
  1071. if child.type != "package":
  1072. continue
  1073. package = self.selector_to_java_reference(child)
  1074. providers = set(child.get_matching_rows("**"))
  1075. if not providers:
  1076. # The package and all its sub packages contain no
  1077. # classes. This should never happen.
  1078. pass
  1079. elif providers == {ClassProvider.BCPF}:
  1080. # The package and all its sub packages only contain
  1081. # classes provided by the bootclasspath_fragment.
  1082. logging.debug("Package '%s.**' is not split", package)
  1083. result.package_prefixes.append(package)
  1084. # There is no point traversing into the sub packages.
  1085. continue
  1086. elif providers == {ClassProvider.OTHER}:
  1087. # The package and all its sub packages contain no
  1088. # classes provided by the bootclasspath_fragment.
  1089. # There is no point traversing into the sub packages.
  1090. logging.debug("Package '%s.**' contains no classes from %s",
  1091. package, self.bcpf)
  1092. continue
  1093. elif ClassProvider.BCPF in providers:
  1094. # The package and all its sub packages contain classes
  1095. # provided by the bootclasspath_fragment and other
  1096. # sources.
  1097. logging.debug(
  1098. "Package '%s.**' contains classes from "
  1099. "%s and other sources", package, self.bcpf)
  1100. providers = set(child.get_matching_rows("*"))
  1101. if not providers:
  1102. # The package contains no classes.
  1103. logging.debug("Package: %s contains no classes", package)
  1104. elif providers == {ClassProvider.BCPF}:
  1105. # The package only contains classes provided by the
  1106. # bootclasspath_fragment.
  1107. logging.debug(
  1108. "Package '%s.*' is not split but does have "
  1109. "sub-packages from other modules", package)
  1110. # Partition the sub-packages into those that are provided by
  1111. # this bootclasspath_fragment and those provided by other
  1112. # modules. They can be used to explain the reason for the
  1113. # single package to developers.
  1114. reason = self.determine_reason_for_single_package(child)
  1115. result.single_packages[package] = reason
  1116. elif providers == {ClassProvider.OTHER}:
  1117. # The package contains no classes provided by the
  1118. # bootclasspath_fragment. Child nodes make contain such
  1119. # classes.
  1120. logging.debug("Package '%s.*' contains no classes from %s",
  1121. package, self.bcpf)
  1122. elif ClassProvider.BCPF in providers:
  1123. # The package contains classes provided by both the
  1124. # bootclasspath_fragment and some other source.
  1125. logging.debug("Package '%s.*' is split", package)
  1126. # Partition the classes in this split package into those
  1127. # that come from this bootclasspath_fragment and those that
  1128. # come from other modules. That can be used to explain the
  1129. # reason for the split package to developers.
  1130. reason = self.determine_reason_for_split_package(child)
  1131. result.split_packages[package] = reason
  1132. self.recurse_hiddenapi_packages_trie(child, result)
  1133. def newline_stripping_iter(iterator):
  1134. """Return an iterator over the iterator that strips trailing white space."""
  1135. lines = iter(iterator, "")
  1136. lines = (line.rstrip() for line in lines)
  1137. return lines
  1138. def format_comment_as_text(text, indent):
  1139. return "".join(
  1140. [f"{line}\n" for line in format_comment_as_lines(text, indent)])
  1141. def format_comment_as_lines(text, indent):
  1142. lines = textwrap.wrap(text.strip("\n"), width=77 - len(indent))
  1143. lines = [f"{indent}// {line}" for line in lines]
  1144. return lines
  1145. def log_stream_for_subprocess():
  1146. stream = subprocess.DEVNULL
  1147. for handler in logging.root.handlers:
  1148. if handler.level == logging.DEBUG:
  1149. if isinstance(handler, logging.StreamHandler):
  1150. stream = handler.stream
  1151. return stream
  1152. def main(argv):
  1153. args_parser = argparse.ArgumentParser(
  1154. description="Analyze a bootclasspath_fragment module.")
  1155. args_parser.add_argument(
  1156. "--bcpf",
  1157. help="The bootclasspath_fragment module to analyze",
  1158. required=True,
  1159. )
  1160. args_parser.add_argument(
  1161. "--apex",
  1162. help="The apex module to which the bootclasspath_fragment belongs. It "
  1163. "is not strictly necessary at the moment but providing it will "
  1164. "allow this script to give more useful messages and it may be"
  1165. "required in future.",
  1166. default="SPECIFY-APEX-OPTION")
  1167. args_parser.add_argument(
  1168. "--sdk",
  1169. help="The sdk module to which the bootclasspath_fragment belongs. It "
  1170. "is not strictly necessary at the moment but providing it will "
  1171. "allow this script to give more useful messages and it may be"
  1172. "required in future.",
  1173. default="SPECIFY-SDK-OPTION")
  1174. args_parser.add_argument(
  1175. "--fix",
  1176. help="Attempt to fix any issues found automatically.",
  1177. action="store_true",
  1178. default=False)
  1179. args = args_parser.parse_args(argv[1:])
  1180. top_dir = os.environ["ANDROID_BUILD_TOP"] + "/"
  1181. out_dir = os.environ.get("OUT_DIR", os.path.join(top_dir, "out"))
  1182. product_out_dir = os.environ.get("ANDROID_PRODUCT_OUT", top_dir)
  1183. # Make product_out_dir relative to the top so it can be used as part of a
  1184. # build target.
  1185. product_out_dir = product_out_dir.removeprefix(top_dir)
  1186. log_fd, abs_log_file = tempfile.mkstemp(
  1187. suffix="_analyze_bcpf.log", text=True)
  1188. with os.fdopen(log_fd, "w") as log_file:
  1189. # Set up debug logging to the log file.
  1190. logging.basicConfig(
  1191. level=logging.DEBUG,
  1192. format="%(levelname)-8s %(message)s",
  1193. stream=log_file)
  1194. # define a Handler which writes INFO messages or higher to the
  1195. # sys.stdout with just the message.
  1196. console = logging.StreamHandler()
  1197. console.setLevel(logging.INFO)
  1198. console.setFormatter(logging.Formatter("%(message)s"))
  1199. # add the handler to the root logger
  1200. logging.getLogger("").addHandler(console)
  1201. print(f"Writing log to {abs_log_file}")
  1202. try:
  1203. analyzer = BcpfAnalyzer(
  1204. tool_path=argv[0],
  1205. top_dir=top_dir,
  1206. out_dir=out_dir,
  1207. product_out_dir=product_out_dir,
  1208. bcpf=args.bcpf,
  1209. apex=args.apex,
  1210. sdk=args.sdk,
  1211. fix=args.fix,
  1212. )
  1213. analyzer.analyze()
  1214. finally:
  1215. print(f"Log written to {abs_log_file}")
  1216. if __name__ == "__main__":
  1217. main(sys.argv)