PRESUBMIT.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294
  1. # Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. # If this presubmit check fails or misbehaves, please complain to
  5. # mnissler@chromium.org, bartfab@chromium.org or atwilson@chromium.org.
  6. import sys
  7. from xml.dom import minidom
  8. from xml.parsers import expat
  9. USE_PYTHON3 = True
  10. def _GetPolicyTemplates(template_path):
  11. # Read list of policies in the template. eval() is used instead of a JSON
  12. # parser because policy_templates.json is not quite JSON, and uses some
  13. # python features such as #-comments and '''strings'''. policy_templates.json
  14. # is actually maintained as a python dictionary.
  15. with open(template_path, encoding='utf-8') as f:
  16. template_data = eval(f.read(), {})
  17. policies = [ policy
  18. for policy in template_data['policy_definitions']
  19. if policy['type'] != 'group' ]
  20. return policies
  21. def _CheckPolicyTemplatesSyntax(input_api, output_api):
  22. local_path = input_api.PresubmitLocalPath()
  23. filepath = input_api.os_path.join(input_api.change.RepositoryRoot(),
  24. 'components','policy','resources','policy_templates.json')
  25. try:
  26. template_affected_file = next(iter(f \
  27. for f in input_api.change.AffectedFiles() \
  28. if f.AbsoluteLocalPath() == filepath))
  29. except:
  30. template_affected_file = None
  31. old_sys_path = sys.path
  32. try:
  33. tools_path = input_api.os_path.normpath(
  34. input_api.os_path.join(local_path, input_api.os_path.pardir, 'tools'))
  35. sys.path = [ tools_path ] + sys.path
  36. # Optimization: only load this when it's needed.
  37. import syntax_check_policy_template_json
  38. device_policy_proto_path = input_api.os_path.join(
  39. local_path, '..','proto','chrome_device_policy.proto')
  40. args = ["--device_policy_proto_path=" + device_policy_proto_path]
  41. root = input_api.change.RepositoryRoot()
  42. # Get the current version from the VERSION file so that we can check
  43. # which policies are un-released and thus can be changed at will.
  44. current_version = None
  45. try:
  46. version_path = input_api.os_path.join(root, 'chrome', 'VERSION')
  47. with open(version_path, "rb") as f:
  48. current_version = int(f.readline().split(b"=")[1])
  49. print('Checking policies against current version: ' +
  50. current_version)
  51. except:
  52. pass
  53. # Get the original file contents of the policy file so that we can check
  54. # the compatibility of template changes in it
  55. original_file_contents = None
  56. if template_affected_file is not None:
  57. original_file_contents = '\n'.join(template_affected_file.OldContents())
  58. # Check if there is a tag that allows us to bypass compatibility checks.
  59. # This can be used in situations where there is a bug in the validation
  60. # code or if a policy change needs to urgently be submitted.
  61. skip_compatibility_check = \
  62. 'BYPASS_POLICY_COMPATIBILITY_CHECK' in input_api.change.tags
  63. checker = syntax_check_policy_template_json.PolicyTemplateChecker()
  64. errors, warnings = checker.Run(args, filepath, original_file_contents,
  65. current_version, skip_compatibility_check)
  66. # PRESUBMIT won't print warning if there is any error. Append warnings to
  67. # error for policy_templates.json so that they can always be printed
  68. # together.
  69. if errors:
  70. return [output_api.PresubmitError('Syntax error(s) in file:',
  71. [filepath],
  72. "\n".join(errors+warnings))]
  73. elif warnings:
  74. return [output_api.PresubmitPromptWarning('Syntax warning(s) in file:',
  75. [filepath],
  76. "\n".join(warnings))]
  77. finally:
  78. sys.path = old_sys_path
  79. return []
  80. def _CheckPolicyTestCases(input_api, output_api, policies):
  81. # Read list of policies in chrome/test/data/policy/policy_test_cases.json.
  82. root = input_api.change.RepositoryRoot()
  83. test_cases_depot_path = input_api.os_path.join(
  84. 'chrome', 'test', 'data', 'policy', 'policy_test_cases.json')
  85. policy_test_cases_file = input_api.os_path.join(
  86. root, test_cases_depot_path)
  87. with open(policy_test_cases_file, encoding='utf-8') as f:
  88. test_names = input_api.json.load(f).keys()
  89. tested_policies = frozenset(name.partition('.')[0]
  90. for name in test_names
  91. if name[:2] != '--')
  92. policy_names = frozenset(policy['name'] for policy in policies)
  93. # Finally check if any policies are missing.
  94. missing = policy_names - tested_policies
  95. extra = tested_policies - policy_names
  96. error_missing = ('Policy \'%s\' was added to policy_templates.json but not '
  97. 'to src/chrome/test/data/policy/policy_test_cases.json. '
  98. 'Please update both files.')
  99. error_extra = ('Policy \'%s\' is tested by '
  100. 'src/chrome/test/data/policy/policy_test_cases.json but is not'
  101. ' defined in policy_templates.json. Please update both files.')
  102. results = []
  103. for policy in missing:
  104. results.append(output_api.PresubmitError(error_missing % policy))
  105. for policy in extra:
  106. results.append(output_api.PresubmitError(error_extra % policy))
  107. results.extend(
  108. input_api.canned_checks.CheckChangeHasNoTabs(
  109. input_api,
  110. output_api,
  111. source_file_filter=lambda x: x.LocalPath() == test_cases_depot_path))
  112. return results
  113. def _CheckPolicyHistograms(input_api, output_api, policies):
  114. root = input_api.change.RepositoryRoot()
  115. histograms = input_api.os_path.join(
  116. root, 'tools', 'metrics', 'histograms', 'enums.xml')
  117. with open(histograms, encoding='utf-8') as f:
  118. tree = minidom.parseString(f.read())
  119. enums = (tree.getElementsByTagName('histogram-configuration')[0]
  120. .getElementsByTagName('enums')[0]
  121. .getElementsByTagName('enum'))
  122. policy_enum = [e for e in enums
  123. if e.getAttribute('name') == 'EnterprisePolicies'][0]
  124. policy_enum_ids = frozenset(int(e.getAttribute('value'))
  125. for e in policy_enum.getElementsByTagName('int'))
  126. policy_id_to_name = {policy['id']: policy['name'] for policy in policies}
  127. policy_ids = frozenset(policy_id_to_name.keys())
  128. missing_ids = policy_ids - policy_enum_ids
  129. extra_ids = policy_enum_ids - policy_ids
  130. error_missing = ('Policy \'%s\' (id %d) was added to '
  131. 'policy_templates.json but not to '
  132. 'src/tools/metrics/histograms/enums.xml. Please update '
  133. 'both files. To regenerate the policy part of enums.xml, '
  134. 'run:\n'
  135. 'python tools/metrics/histograms/update_policies.py')
  136. error_extra = ('Policy id %d was found in '
  137. 'src/tools/metrics/histograms/enums.xml, but no policy with '
  138. 'this id exists in policy_templates.json. To regenerate the '
  139. 'policy part of enums.xml, run:\n'
  140. 'python tools/metrics/histograms/update_policies.py')
  141. results = []
  142. for policy_id in missing_ids:
  143. results.append(
  144. output_api.PresubmitError(error_missing %
  145. (policy_id_to_name[policy_id], policy_id)))
  146. for policy_id in extra_ids:
  147. results.append(output_api.PresubmitError(error_extra % policy_id))
  148. return results
  149. def _CheckPolicyAtomicGroupsHistograms(input_api, output_api, atomic_groups):
  150. root = input_api.change.RepositoryRoot()
  151. histograms = input_api.os_path.join(
  152. root, 'tools', 'metrics', 'histograms', 'enums.xml')
  153. with open(histograms, encoding='utf-8') as f:
  154. tree = minidom.parseString(f.read())
  155. enums = (tree.getElementsByTagName('histogram-configuration')[0]
  156. .getElementsByTagName('enums')[0]
  157. .getElementsByTagName('enum'))
  158. atomic_group_enum = [e for e in enums
  159. if e.getAttribute('name') == 'PolicyAtomicGroups'][0]
  160. atomic_group_enum_ids = frozenset(int(e.getAttribute('value'))
  161. for e in atomic_group_enum
  162. .getElementsByTagName('int'))
  163. atomic_group_id_to_name = {policy['id']: policy['name']
  164. for policy in atomic_groups}
  165. atomic_group_ids = frozenset(atomic_group_id_to_name.keys())
  166. missing_ids = atomic_group_ids - atomic_group_enum_ids
  167. extra_ids = atomic_group_enum_ids - atomic_group_ids
  168. error_missing = ('Policy atomic group \'%s\' (id %d) was added to '
  169. 'policy_templates.json but not to '
  170. 'src/tools/metrics/histograms/enums.xml. Please update '
  171. 'both files. To regenerate the policy part of enums.xml, '
  172. 'run:\n'
  173. 'python tools/metrics/histograms/update_policies.py')
  174. error_extra = ('Policy atomic group id %d was found in '
  175. 'src/tools/metrics/histograms/enums.xml, but no policy with '
  176. 'this id exists in policy_templates.json. To regenerate the '
  177. 'policy part of enums.xml, run:\n'
  178. 'python tools/metrics/histograms/update_policies.py')
  179. results = []
  180. for atomic_group_id in missing_ids:
  181. results.append(output_api.PresubmitError(error_missing %
  182. (atomic_group_id_to_name[atomic_group_id],
  183. atomic_group_id)))
  184. for atomic_group_id in extra_ids:
  185. results.append(output_api.PresubmitError(error_extra % atomic_group_id))
  186. return results
  187. def _CheckMissingPlaceholders(input_api, output_api, template_path):
  188. with open(template_path, encoding='utf-8') as f:
  189. template_data = eval(f.read(), {})
  190. results = []
  191. items = template_data['policy_definitions'] \
  192. + [msg for msg in template_data['messages'].values()]
  193. for item in items:
  194. for key in ['desc', 'text']:
  195. if not key in item:
  196. continue
  197. try:
  198. node = minidom.parseString('<msg>%s</msg>' % item[key]).childNodes[0]
  199. except expat.ExpatError as e:
  200. error = (
  201. 'Error when checking for missing placeholders: %s in:\n'
  202. '!<Policy Start>!\n%s\n<Policy End>!' %
  203. (e, item[key]))
  204. results.append(output_api.PresubmitError(error))
  205. continue
  206. for child in node.childNodes:
  207. if child.nodeType == minidom.Node.TEXT_NODE and '$' in child.data:
  208. warning = ('Character \'$\' found outside of a placeholder in "%s". '
  209. 'Should it be in a placeholder ?') % item[key]
  210. results.append(output_api.PresubmitPromptWarning(warning))
  211. return results
  212. def _CommonChecks(input_api, output_api):
  213. results = []
  214. root = input_api.change.RepositoryRoot()
  215. template_path = input_api.os_path.join(
  216. root, 'components', 'policy', 'resources', 'policy_templates.json')
  217. device_policy_proto_path = input_api.os_path.join(
  218. root, 'components', 'policy', 'proto', 'chrome_device_policy.proto')
  219. # policies in chrome/test/data/policy/policy_test_cases.json.
  220. test_cases_path = input_api.os_path.join(
  221. root, 'chrome', 'test', 'data', 'policy', 'policy_test_cases.json')
  222. syntax_check_path = input_api.os_path.join(
  223. root, 'components', 'policy', 'tools',
  224. 'syntax_check_policy_template_json.py')
  225. affected_files = input_api.change.AffectedFiles()
  226. results.extend(_CheckMissingPlaceholders(input_api, output_api,
  227. template_path))
  228. template_changed = any(f.AbsoluteLocalPath() == template_path \
  229. for f in affected_files)
  230. device_policy_proto_changed = \
  231. any(f.AbsoluteLocalPath() == device_policy_proto_path \
  232. for f in affected_files)
  233. tests_changed = any(f.AbsoluteLocalPath() == test_cases_path \
  234. for f in affected_files)
  235. syntax_check_changed = any(f.AbsoluteLocalPath() == syntax_check_path \
  236. for f in affected_files)
  237. if (template_changed or device_policy_proto_changed or tests_changed or
  238. syntax_check_changed):
  239. try:
  240. policies = _GetPolicyTemplates(template_path)
  241. except:
  242. results.append(output_api.PresubmitError('Invalid Python/JSON syntax.'))
  243. return results
  244. if template_changed or tests_changed:
  245. results.extend(_CheckPolicyTestCases(input_api, output_api, policies))
  246. if template_changed:
  247. results.extend(_CheckPolicyHistograms(input_api, output_api, policies))
  248. # chrome_device_policy.proto is hand crafted. When it is changed, we need
  249. # to check if it still corresponds to policy_templates.json.
  250. if template_changed or device_policy_proto_changed or syntax_check_changed:
  251. results.extend(_CheckPolicyTemplatesSyntax(input_api, output_api))
  252. return results
  253. def CheckChangeOnUpload(input_api, output_api):
  254. return _CommonChecks(input_api, output_api)
  255. def CheckChangeOnCommit(input_api, output_api):
  256. return _CommonChecks(input_api, output_api)