generate_hiddenapi_lists.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429
  1. #!/usr/bin/env python
  2. #
  3. # Copyright (C) 2018 The Android Open Source Project
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. """Generate API lists for non-SDK API enforcement."""
  17. import argparse
  18. from collections import defaultdict, namedtuple
  19. import re
  20. import sys
  21. # Names of flags recognized by the `hiddenapi` tool.
  22. FLAG_SDK = 'sdk'
  23. FLAG_UNSUPPORTED = 'unsupported'
  24. FLAG_BLOCKED = 'blocked'
  25. FLAG_MAX_TARGET_O = 'max-target-o'
  26. FLAG_MAX_TARGET_P = 'max-target-p'
  27. FLAG_MAX_TARGET_Q = 'max-target-q'
  28. FLAG_MAX_TARGET_R = 'max-target-r'
  29. FLAG_MAX_TARGET_S = 'max-target-s'
  30. FLAG_CORE_PLATFORM_API = 'core-platform-api'
  31. FLAG_PUBLIC_API = 'public-api'
  32. FLAG_SYSTEM_API = 'system-api'
  33. FLAG_TEST_API = 'test-api'
  34. # List of all known flags.
  35. FLAGS_API_LIST = [
  36. FLAG_SDK,
  37. FLAG_UNSUPPORTED,
  38. FLAG_BLOCKED,
  39. FLAG_MAX_TARGET_O,
  40. FLAG_MAX_TARGET_P,
  41. FLAG_MAX_TARGET_Q,
  42. FLAG_MAX_TARGET_R,
  43. FLAG_MAX_TARGET_S,
  44. ]
  45. ALL_FLAGS = FLAGS_API_LIST + [
  46. FLAG_CORE_PLATFORM_API,
  47. FLAG_PUBLIC_API,
  48. FLAG_SYSTEM_API,
  49. FLAG_TEST_API,
  50. ]
  51. FLAGS_API_LIST_SET = set(FLAGS_API_LIST)
  52. ALL_FLAGS_SET = set(ALL_FLAGS)
  53. # Option specified after one of FLAGS_API_LIST to indicate that
  54. # only known and otherwise unassigned entries should be assign the
  55. # given flag.
  56. # For example, the max-target-P list is checked in as it was in P,
  57. # but signatures have changes since then. The flag instructs this
  58. # script to skip any entries which do not exist any more.
  59. FLAG_IGNORE_CONFLICTS = 'ignore-conflicts'
  60. # Option specified after one of FLAGS_API_LIST to express that all
  61. # apis within a given set of packages should be assign the given flag.
  62. FLAG_PACKAGES = 'packages'
  63. # Option specified after one of FLAGS_API_LIST to indicate an extra
  64. # tag that should be added to the matching APIs.
  65. FLAG_TAG = 'tag'
  66. # Regex patterns of fields/methods used in serialization. These are
  67. # considered public API despite being hidden.
  68. SERIALIZATION_PATTERNS = [
  69. r'readObject\(Ljava/io/ObjectInputStream;\)V',
  70. r'readObjectNoData\(\)V',
  71. r'readResolve\(\)Ljava/lang/Object;',
  72. r'serialVersionUID:J',
  73. r'serialPersistentFields:\[Ljava/io/ObjectStreamField;',
  74. r'writeObject\(Ljava/io/ObjectOutputStream;\)V',
  75. r'writeReplace\(\)Ljava/lang/Object;',
  76. ]
  77. # Single regex used to match serialization API. It combines all the
  78. # SERIALIZATION_PATTERNS into a single regular expression.
  79. SERIALIZATION_REGEX = re.compile(r'.*->(' + '|'.join(SERIALIZATION_PATTERNS) +
  80. r')$')
  81. # Predicates to be used with filter_apis.
  82. HAS_NO_API_LIST_ASSIGNED = \
  83. lambda api,flags: not FLAGS_API_LIST_SET.intersection(flags)
  84. IS_SERIALIZATION = lambda api, flags: SERIALIZATION_REGEX.match(api)
  85. class StoreOrderedOptions(argparse.Action):
  86. """An argparse action that stores a number of option arguments in the order
  87. that they were specified.
  88. """
  89. def __call__(self, parser, args, values, option_string=None):
  90. items = getattr(args, self.dest, None)
  91. if items is None:
  92. items = []
  93. items.append([option_string.lstrip('-'), values])
  94. setattr(args, self.dest, items)
  95. def get_args():
  96. """Parses command line arguments.
  97. Returns:
  98. Namespace: dictionary of parsed arguments
  99. """
  100. parser = argparse.ArgumentParser()
  101. parser.add_argument('--output', required=True)
  102. parser.add_argument(
  103. '--csv',
  104. nargs='*',
  105. default=[],
  106. metavar='CSV_FILE',
  107. help='CSV files to be merged into output')
  108. for flag in ALL_FLAGS:
  109. parser.add_argument(
  110. '--' + flag,
  111. dest='ordered_flags',
  112. metavar='TXT_FILE',
  113. action=StoreOrderedOptions,
  114. help='lists of entries with flag "' + flag + '"')
  115. parser.add_argument(
  116. '--' + FLAG_IGNORE_CONFLICTS,
  117. dest='ordered_flags',
  118. nargs=0,
  119. action=StoreOrderedOptions,
  120. help='Indicates that only known and otherwise unassigned '
  121. 'entries should be assign the given flag. Must follow a list of '
  122. 'entries and applies to the preceding such list.')
  123. parser.add_argument(
  124. '--' + FLAG_PACKAGES,
  125. dest='ordered_flags',
  126. nargs=0,
  127. action=StoreOrderedOptions,
  128. help='Indicates that the previous list of entries '
  129. 'is a list of packages. All members in those packages will be given '
  130. 'the flag. Must follow a list of entries and applies to the preceding '
  131. 'such list.')
  132. parser.add_argument(
  133. '--' + FLAG_TAG,
  134. dest='ordered_flags',
  135. nargs=1,
  136. action=StoreOrderedOptions,
  137. help='Adds an extra tag to the previous list of entries. '
  138. 'Must follow a list of entries and applies to the preceding such list.')
  139. return parser.parse_args()
  140. def read_lines(filename):
  141. """Reads entire file and return it as a list of lines.
  142. Lines which begin with a hash are ignored.
  143. Args:
  144. filename (string): Path to the file to read from.
  145. Returns:
  146. Lines of the file as a list of string.
  147. """
  148. with open(filename, 'r') as f:
  149. lines = f.readlines()
  150. lines = [line for line in lines if not line.startswith('#')]
  151. lines = [line.strip() for line in lines]
  152. return set(lines)
  153. def write_lines(filename, lines):
  154. """Writes list of lines into a file, overwriting the file if it exists.
  155. Args:
  156. filename (string): Path to the file to be writing into.
  157. lines (list): List of strings to write into the file.
  158. """
  159. lines = [line + '\n' for line in lines]
  160. with open(filename, 'w') as f:
  161. f.writelines(lines)
  162. def extract_package(signature):
  163. """Extracts the package from a signature.
  164. Args:
  165. signature (string): JNI signature of a method or field.
  166. Returns:
  167. The package name of the class containing the field/method.
  168. """
  169. full_class_name = signature.split(';->')[0]
  170. # Example: Landroid/hardware/radio/V1_2/IRadio$Proxy
  171. if full_class_name[0] != 'L':
  172. raise ValueError("Expected to start with 'L': %s"
  173. % full_class_name)
  174. full_class_name = full_class_name[1:]
  175. # If full_class_name doesn't contain '/', then package_name will be ''.
  176. package_name = full_class_name.rpartition('/')[0]
  177. return package_name.replace('/', '.')
  178. class FlagsDict:
  179. def __init__(self):
  180. self._dict_keyset = set()
  181. self._dict = defaultdict(set)
  182. def _check_entries_set(self, keys_subset, source):
  183. assert isinstance(keys_subset, set)
  184. assert keys_subset.issubset(self._dict_keyset), (
  185. 'Error: {} specifies signatures not present in code:\n'
  186. '{}'
  187. 'Please visit go/hiddenapi for more information.').format(
  188. source, ''.join(
  189. [' ' + str(x) + '\n' for x in
  190. keys_subset - self._dict_keyset]))
  191. def _check_flags_set(self, flags_subset, source):
  192. assert isinstance(flags_subset, set)
  193. assert flags_subset.issubset(ALL_FLAGS_SET), (
  194. 'Error processing: {}\n'
  195. 'The following flags were not recognized: \n'
  196. '{}\n'
  197. 'Please visit go/hiddenapi for more information.').format(
  198. source, '\n'.join(flags_subset - ALL_FLAGS_SET))
  199. def filter_apis(self, filter_fn):
  200. """Returns APIs which match a given predicate.
  201. This is a helper function which allows to filter on both signatures
  202. (keys) and
  203. flags (values). The built-in filter() invokes the lambda only with
  204. dict's keys.
  205. Args:
  206. filter_fn : Function which takes two arguments (signature/flags) and
  207. returns a boolean.
  208. Returns:
  209. A set of APIs which match the predicate.
  210. """
  211. return {x for x in self._dict_keyset if filter_fn(x, self._dict[x])}
  212. def get_valid_subset_of_unassigned_apis(self, api_subset):
  213. """Sanitizes a key set input to only include keys which exist in the
  214. dictionary and have not been assigned any API list flags.
  215. Args:
  216. entries_subset (set/list): Key set to be sanitized.
  217. Returns:
  218. Sanitized key set.
  219. """
  220. assert isinstance(api_subset, set)
  221. return api_subset.intersection(
  222. self.filter_apis(HAS_NO_API_LIST_ASSIGNED))
  223. def generate_csv(self):
  224. """Constructs CSV entries from a dictionary.
  225. Old versions of flags are used to generate the file.
  226. Returns:
  227. List of lines comprising a CSV file. See "parse_and_merge_csv" for
  228. format description.
  229. """
  230. lines = []
  231. for api in self._dict:
  232. flags = sorted(self._dict[api])
  233. lines.append(','.join([api] + flags))
  234. return sorted(lines)
  235. def parse_and_merge_csv(self, csv_lines, source='<unknown>'):
  236. """Parses CSV entries and merges them into a given dictionary.
  237. The expected CSV format is:
  238. <api signature>,<flag1>,<flag2>,...,<flagN>
  239. Args:
  240. csv_lines (list of strings): Lines read from a CSV file.
  241. source (string): Origin of `csv_lines`. Will be printed in error
  242. messages.
  243. Throws: AssertionError if parsed flags are invalid.
  244. """
  245. # Split CSV lines into arrays of values.
  246. csv_values = [line.split(',') for line in csv_lines]
  247. # Update the full set of API signatures.
  248. self._dict_keyset.update([csv[0] for csv in csv_values])
  249. # Check that all flags are known.
  250. csv_flags = set()
  251. for csv in csv_values:
  252. csv_flags.update(csv[1:])
  253. self._check_flags_set(csv_flags, source)
  254. # Iterate over all CSV lines, find entry in dict and append flags to it.
  255. for csv in csv_values:
  256. flags = csv[1:]
  257. if (FLAG_PUBLIC_API in flags) or (FLAG_SYSTEM_API in flags):
  258. flags.append(FLAG_SDK)
  259. self._dict[csv[0]].update(flags)
  260. def assign_flag(self, flag, apis, source='<unknown>', tag=None):
  261. """Assigns a flag to given subset of entries.
  262. Args:
  263. flag (string): One of ALL_FLAGS.
  264. apis (set): Subset of APIs to receive the flag.
  265. source (string): Origin of `entries_subset`. Will be printed in
  266. error messages.
  267. Throws: AssertionError if parsed API signatures of flags are invalid.
  268. """
  269. # Check that all APIs exist in the dict.
  270. self._check_entries_set(apis, source)
  271. # Check that the flag is known.
  272. self._check_flags_set(set([flag]), source)
  273. # Iterate over the API subset, find each entry in dict and assign the
  274. # flag to it.
  275. for api in apis:
  276. self._dict[api].add(flag)
  277. if tag:
  278. self._dict[api].add(tag)
  279. FlagFile = namedtuple('FlagFile',
  280. ('flag', 'file', 'ignore_conflicts', 'packages', 'tag'))
  281. def parse_ordered_flags(ordered_flags):
  282. r = []
  283. currentflag, file, ignore_conflicts, packages, tag = None, None, False, \
  284. False, None
  285. for flag_value in ordered_flags:
  286. flag, value = flag_value[0], flag_value[1]
  287. if flag in ALL_FLAGS_SET:
  288. if currentflag:
  289. r.append(
  290. FlagFile(currentflag, file, ignore_conflicts, packages,
  291. tag))
  292. ignore_conflicts, packages, tag = False, False, None
  293. currentflag = flag
  294. file = value
  295. else:
  296. if currentflag is None:
  297. raise argparse.ArgumentError( #pylint: disable=no-value-for-parameter
  298. '--%s is only allowed after one of %s' %
  299. (flag, ' '.join(['--%s' % f for f in ALL_FLAGS_SET])))
  300. if flag == FLAG_IGNORE_CONFLICTS:
  301. ignore_conflicts = True
  302. elif flag == FLAG_PACKAGES:
  303. packages = True
  304. elif flag == FLAG_TAG:
  305. tag = value[0]
  306. if currentflag:
  307. r.append(FlagFile(currentflag, file, ignore_conflicts, packages, tag))
  308. return r
  309. def main(argv): #pylint: disable=unused-argument
  310. # Parse arguments.
  311. args = vars(get_args())
  312. flagfiles = parse_ordered_flags(args['ordered_flags'] or [])
  313. # Initialize API->flags dictionary.
  314. flags = FlagsDict()
  315. # Merge input CSV files into the dictionary.
  316. # Do this first because CSV files produced by parsing API stubs will
  317. # contain the full set of APIs. Subsequent additions from text files
  318. # will be able to detect invalid entries, and/or filter all as-yet
  319. # unassigned entries.
  320. for filename in args['csv']:
  321. flags.parse_and_merge_csv(read_lines(filename), filename)
  322. # Combine inputs which do not require any particular order.
  323. # (1) Assign serialization API to SDK.
  324. flags.assign_flag(FLAG_SDK, flags.filter_apis(IS_SERIALIZATION))
  325. # (2) Merge text files with a known flag into the dictionary.
  326. for info in flagfiles:
  327. if (not info.ignore_conflicts) and (not info.packages):
  328. flags.assign_flag(info.flag, read_lines(info.file), info.file,
  329. info.tag)
  330. # Merge text files where conflicts should be ignored.
  331. # This will only assign the given flag if:
  332. # (a) the entry exists, and
  333. # (b) it has not been assigned any other flag.
  334. # Because of (b), this must run after all strict assignments have been
  335. # performed.
  336. for info in flagfiles:
  337. if info.ignore_conflicts:
  338. valid_entries = flags.get_valid_subset_of_unassigned_apis(
  339. read_lines(info.file))
  340. flags.assign_flag(info.flag, valid_entries, filename, info.tag) #pylint: disable=undefined-loop-variable
  341. # All members in the specified packages will be assigned the appropriate
  342. # flag.
  343. for info in flagfiles:
  344. if info.packages:
  345. packages_needing_list = set(read_lines(info.file))
  346. should_add_signature_to_list = lambda sig, lists: extract_package(
  347. sig) in packages_needing_list and not lists #pylint: disable=cell-var-from-loop
  348. valid_entries = flags.filter_apis(should_add_signature_to_list)
  349. flags.assign_flag(info.flag, valid_entries, info.file, info.tag)
  350. # Mark all remaining entries as blocked.
  351. flags.assign_flag(FLAG_BLOCKED, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED))
  352. # Write output.
  353. write_lines(args['output'], flags.generate_csv())
  354. if __name__ == '__main__':
  355. main(sys.argv)