generate_flag_labels.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345
  1. #!/usr/bin/env vpython3
  2. #
  3. # Copyright 2020 The Chromium Authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. """Helps generate enums.xml from ProductionSupportedFlagList.
  7. This is only a best-effort attempt to generate enums.xml values for the
  8. LoginCustomFlags enum. You need to verify this script picks the right string
  9. value for the new features and double check the hash value by running
  10. "AboutFlagsHistogramTest.*".
  11. """
  12. from __future__ import print_function
  13. import argparse
  14. import os
  15. import re
  16. import hashlib
  17. import ctypes
  18. import xml.etree.ElementTree as ET
  19. import logging
  20. import sys
  21. _CHROMIUM_SRC = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
  22. sys.path.append(os.path.join(_CHROMIUM_SRC, 'third_party', 'catapult', 'devil'))
  23. from devil.utils import logging_common # pylint: disable=wrong-import-position
  24. _FLAG_LIST_FILE = os.path.join(_CHROMIUM_SRC, 'android_webview', 'java', 'src',
  25. 'org', 'chromium', 'android_webview', 'common',
  26. 'ProductionSupportedFlagList.java')
  27. _ENUMS_XML_FILE = os.path.join(_CHROMIUM_SRC, 'tools', 'metrics', 'histograms',
  28. 'enums.xml')
  29. _SCRIPT_PATH = '//android_webview/tools/generate_flag_labels.py'
  30. # This script tries to guess the commandline switch/base::Feature name from the
  31. # generated Java constant (assuming the constant name follows typical
  32. # conventions), but sometimes the script generates the incorrect name.
  33. # Once you update the KNOWN_MISTAKES dictionary, re-run the tool (either invoke
  34. # this script directly or run the git-cl presubmit check) and it should generate
  35. # the correct integer hash value for enums.xml.
  36. #
  37. # Keys are the names autogenerated by this script's logic, values are the actual
  38. # string names used to define the base::Feature or commandline switch in C++
  39. # code.
  40. #
  41. # pylint: disable=line-too-long
  42. # yapf: disable
  43. KNOWN_MISTAKES = {
  44. # 'AutogeneratedName': 'CorrectName',
  45. 'WebViewAccelerateSmallCanvases': 'WebviewAccelerateSmallCanvases',
  46. 'Canvas2dStaysGpuOnReadback': 'Canvas2dStaysGPUOnReadback',
  47. 'EnableSharedImageForWebView': 'EnableSharedImageForWebview',
  48. 'GmsCoreEmoji': 'GMSCoreEmoji',
  49. 'KeyboardAccessoryPaymentVirtualCardFeature': 'IPH_KeyboardAccessoryPaymentVirtualCard',
  50. 'CompositeBgColorAnimation': 'CompositeBGColorAnimation',
  51. 'enable-http2-grease-settings': 'http2-grease-settings',
  52. 'AvoidUnnecessaryBeforeUnloadCheckPostTask': 'AvoidUnnecessaryBeforeUnloadCheck',
  53. 'AutofillShadowDom': 'AutofillShadowDOM',
  54. 'HtmlParamElementUrlSupport': 'HTMLParamElementUrlSupport',
  55. 'webview-mp-arch-fenced-frames': 'webview-mparch-fenced-frames',
  56. 'ThrottleIntersectionObserverUma': 'ThrottleIntersectionObserverUMA',
  57. 'RemoveCanceledTasksInTaskQueue': 'RemoveCanceledTasksInTaskQueue2',
  58. 'CssOverflowForReplacedElements': 'CSSOverflowForReplacedElements',
  59. # The final entry should have a trailing comma for cleaner diffs. You may
  60. # remove the entry from this dictionary when it's time to delete the
  61. # corresponding base::Feature or commandline switch.
  62. }
  63. # yapf: enable
  64. # pylint: enable=line-too-long
  65. def GetSwitchId(label):
  66. """Generate a hash consistent with flags_ui::GetSwitchUMAId()."""
  67. digest = hashlib.md5(label.encode('utf-8')).hexdigest()
  68. first_eight_bytes = digest[:16]
  69. long_value = int(first_eight_bytes, 16)
  70. signed_32bit = ctypes.c_int(long_value).value
  71. return signed_32bit
  72. def _Capitalize(value):
  73. value = value[0].upper() + value[1:].lower()
  74. if value == 'Webview':
  75. value = 'WebView'
  76. return value
  77. def FormatName(name, convert_to_pascal_case):
  78. """Converts name to the correct format.
  79. If name is shouty-case (ex. 'SOME_NAME') like a Java constant, then:
  80. * it converts to pascal case (camel case, with the first letter capitalized)
  81. if convert_to_pascal_case == True (ex. 'SomeName')
  82. * it converts to hyphenates name and converts to lower case (ex.
  83. 'some-name')
  84. raises
  85. ValueError if name contains quotation marks like a Java literal (ex.
  86. '"SomeName"')
  87. """
  88. has_quotes_re = re.compile(r'".*"')
  89. if has_quotes_re.match(name):
  90. raise ValueError('String literals are not supported (got {})'.format(name))
  91. name = re.sub(r'^[^.]+\.', '', name)
  92. sections = name.split('_')
  93. if convert_to_pascal_case:
  94. sections = [_Capitalize(section) for section in sections]
  95. return ''.join(sections)
  96. sections = [section.lower() for section in sections]
  97. return '-'.join(sections)
  98. def ConvertNameIfNecessary(name):
  99. """Fixes any names which are known to be autogenerated incorrectly."""
  100. if name in KNOWN_MISTAKES.keys():
  101. return KNOWN_MISTAKES.get(name)
  102. return name
  103. class Flag:
  104. """Simplified python equivalent of the Flag java class.
  105. See //android_webview/java/src/org/chromium/android_webview/common/Flag.java
  106. """
  107. def __init__(self, name, is_base_feature):
  108. self.name = name
  109. self.is_base_feature = is_base_feature
  110. class EnumValue:
  111. """Represents a label/value pair consistent with enums.xml."""
  112. def __init__(self, label):
  113. self.label = label
  114. self.value = GetSwitchId(label)
  115. def ToXml(self):
  116. return '<int value="{value}" label="{label}"/>'.format(value=self.value,
  117. label=self.label)
  118. class UncheckedEnumValue(EnumValue):
  119. """Like an EnumValue, but value may not be correct."""
  120. def __init__(self, label, value):
  121. super().__init__(label)
  122. self.label = label
  123. self.value = value
  124. def _GetExistingFlagEnums():
  125. with open(_ENUMS_XML_FILE) as f:
  126. root = ET.fromstring(f.read())
  127. all_enums = root.find('enums')
  128. login_custom_flags = all_enums.find('enum[@name="LoginCustomFlags"]')
  129. return {
  130. UncheckedEnumValue(item.get('label'), int(item.get('value')))
  131. for item in login_custom_flags
  132. }
  133. def _RemoveDuplicates(enums, existing_labels):
  134. return [enum for enum in enums if enum.label not in existing_labels]
  135. def ExtractFlagsFromJavaLines(lines):
  136. flags = []
  137. hanging_name_re = re.compile(
  138. r'(?:\s*Flag\.(?:baseFeature|commandLine)\()?(\S+),')
  139. pending_feature = False
  140. pending_commandline = False
  141. for line in lines:
  142. if 'baseFeature(' in line:
  143. pending_feature = True
  144. if 'commandLine(' in line:
  145. pending_commandline = True
  146. if pending_feature and pending_commandline:
  147. raise RuntimeError('Somehow this is both a baseFeature and commandLine '
  148. 'switch: ({})'.format(line))
  149. # This means we saw Flag.baseFeature() or Flag.commandLine() on this or a
  150. # previous line but haven't found that flag's name yet. Check if we can
  151. # find a name in this line.
  152. if pending_feature or pending_commandline:
  153. m = hanging_name_re.search(line)
  154. if m:
  155. name = m.group(1)
  156. try:
  157. formatted_name = FormatName(name, pending_feature)
  158. formatted_name = ConvertNameIfNecessary(formatted_name)
  159. flags.append(Flag(formatted_name, pending_feature))
  160. pending_feature = False
  161. pending_commandline = False
  162. except ValueError:
  163. logging.warning('String literals are not supported, skipping %s',
  164. name)
  165. return flags
  166. def _GetIncorrectWebViewEnums():
  167. """Find incorrect WebView EnumValue pairs and return the correct output
  168. This iterates through both ProductionSupportedFlagList and enums.xml and
  169. returns EnumValue pairs for:
  170. * Any ProductionSupportedFlagList entries which are already in enums.xml but
  171. with the wrong integer hash value
  172. * Any ProductionSupportedFlagList entries which are not yet in enums.xml
  173. Returns the tuple (enums_need_fixing, enums_to_add):
  174. enums_need_fixing: a set of (correct) EnumValues for any existing enums.xml
  175. values which are currently incorrect.
  176. enums_to_add: a set of EnumValues for anything which isn't already in
  177. enums.xml.
  178. """
  179. with open(_FLAG_LIST_FILE, 'r') as f:
  180. lines = f.readlines()
  181. flags = ExtractFlagsFromJavaLines(lines)
  182. enums = []
  183. for flag in flags:
  184. if flag.is_base_feature:
  185. enums.append(EnumValue(flag.name + ':enabled'))
  186. enums.append(EnumValue(flag.name + ':disabled'))
  187. else:
  188. enums.append(EnumValue(flag.name))
  189. production_supported_flag_labels = {enum.label for enum in enums}
  190. # Don't bother checking non-WebView flags. Folks modifying
  191. # ProductionSupportedFlagList shouldn't be responsible for updating
  192. # non-WebView flags.
  193. def is_webview_flag(unchecked_enum):
  194. return unchecked_enum.label in production_supported_flag_labels
  195. existing_flag_enums = set(filter(is_webview_flag, _GetExistingFlagEnums()))
  196. # Find the invalid enums and generate the corresponding correct enums
  197. def is_invalid_enum(unchecked_enum):
  198. correct_enum = EnumValue(unchecked_enum.label)
  199. return correct_enum.value != unchecked_enum.value
  200. enums_need_fixing = {
  201. EnumValue(unchecked_enum.label)
  202. for unchecked_enum in filter(is_invalid_enum, existing_flag_enums)
  203. }
  204. existing_labels = {enum.label for enum in existing_flag_enums}
  205. enums_to_add = _RemoveDuplicates(enums, existing_labels)
  206. return (enums_need_fixing, enums_to_add)
  207. def _GenerateEnumFlagMessage(enums_need_fixing, enums_to_add):
  208. output = ''
  209. if enums_need_fixing:
  210. output += """\
  211. It looks like some flags in enums.xml have the wrong 'int value'. This is
  212. probably a mistake in enums.xml. Please update these flags in enums.xml
  213. to use the following (correct) int values:
  214. """
  215. output += '\n'.join(
  216. sorted([' ' + enum.ToXml() for enum in enums_need_fixing]))
  217. output += '\n\n'
  218. if enums_to_add:
  219. output += """\
  220. It looks like you added flags to ProductionSupportedFlagList but didn't yet add
  221. the flags to enums.xml. Please double-check that the following flag labels are
  222. spelled correctly (case-sensitive). You can correct any spelling mistakes by
  223. editing KNOWN_MISTAKES in {script_path}.
  224. Once the spelling is correct, please run this tool again and add the following
  225. to enums.xml:
  226. """.format(script_path=_SCRIPT_PATH)
  227. output += '\n'.join(sorted([' ' + enum.ToXml() for enum in enums_to_add]))
  228. output += '\n\n'
  229. return output
  230. def CheckMissingWebViewEnums(input_api, output_api):
  231. """A presubmit check to find missing flag enums."""
  232. sources = input_api.AffectedSourceFiles(
  233. lambda affected_file: input_api.FilterSourceFile(
  234. affected_file,
  235. files_to_check=(r'.*\bProductionSupportedFlagList\.java$', )))
  236. if not sources:
  237. return []
  238. enums_need_fixing, enums_to_add = _GetIncorrectWebViewEnums()
  239. if not enums_need_fixing and not enums_to_add:
  240. # Return empty list to tell git-cl presubmit there were no errors.
  241. return []
  242. enums_path = '//tools/metrics/histograms/enums.xml'
  243. return [
  244. output_api.PresubmitPromptWarning("""
  245. It looks like new flags have been added to ProductionSupportedFlagList but the
  246. labels still need to be added to LoginCustomFlags enum in {enums_path}.
  247. If you believe this warning is correct, please update enums.xml by pasting the
  248. following lines under LoginCustomFlags and running `git-cl format` to correctly
  249. sort the changes:
  250. {enums}
  251. You can run this check again by running the {script_path} tool.
  252. """.format(enums=_GenerateEnumFlagMessage(enums_need_fixing, enums_to_add),
  253. enums_path=enums_path,
  254. script_path=_SCRIPT_PATH))
  255. ]
  256. def main():
  257. parser = argparse.ArgumentParser()
  258. logging_common.AddLoggingArguments(parser)
  259. args = parser.parse_args()
  260. logging_common.InitializeLogging(args)
  261. enums_need_fixing, enums_to_add = _GetIncorrectWebViewEnums()
  262. if not enums_need_fixing and not enums_to_add:
  263. print('enums.xml is already up-to-date!')
  264. return
  265. message = """\
  266. This is a best-effort attempt to generate missing enums.xml entries. Please
  267. double-check this picked the correct labels for your new features (labels are
  268. case-sensitive!), add these to enums.xml, run `git-cl format`, and then follow
  269. these steps as a final check:
  270. https://chromium.googlesource.com/chromium/src/+/main/tools/metrics/histograms/README.md#flag-histograms
  271. """
  272. print(message)
  273. print(_GenerateEnumFlagMessage(enums_need_fixing, enums_to_add))
  274. if __name__ == '__main__':
  275. main()