checkperms.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520
  1. #!/usr/bin/env python3
  2. # Copyright (c) 2012 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Makes sure files have the right permissions.
  6. Some developers have broken SCM configurations that flip the executable
  7. permission on for no good reason. Unix developers who run ls --color will then
  8. see .cc files in green and get confused.
  9. - For file extensions that must be executable, add it to EXECUTABLE_EXTENSIONS.
  10. - For file extensions that must not be executable, add it to
  11. NOT_EXECUTABLE_EXTENSIONS.
  12. - To ignore all the files inside a directory, add it to IGNORED_PATHS.
  13. - For file base name with ambiguous state and that should not be checked for
  14. shebang, add it to IGNORED_FILENAMES.
  15. Any file not matching the above will be opened and looked if it has a shebang
  16. or an ELF or Mach-O header. If this does not match the executable bit on the
  17. file, the file will be flagged. Mach-O files are allowed to exist with or
  18. without an executable bit set, as there are many examples of it appearing as
  19. test data, and as Mach-O types such as dSYM that canonically do not have their
  20. executable bits set.
  21. Note that all directory separators must be slashes (Unix-style) and not
  22. backslashes. All directories should be relative to the source root and all
  23. file paths should be only lowercase.
  24. """
  25. from __future__ import print_function
  26. import json
  27. import logging
  28. import optparse
  29. import os
  30. import stat
  31. import string
  32. import subprocess
  33. import sys
  34. #### USER EDITABLE SECTION STARTS HERE ####
  35. # Files with these extensions must have executable bit set.
  36. #
  37. # Case-sensitive.
  38. EXECUTABLE_EXTENSIONS = (
  39. 'bat',
  40. 'dll',
  41. 'exe',
  42. )
  43. # Files for which the executable bit may or may not be set.
  44. IGNORED_EXTENSIONS = (
  45. 'dylib',
  46. )
  47. # These files must have executable bit set.
  48. #
  49. # Case-insensitive, lower-case only.
  50. EXECUTABLE_PATHS = (
  51. 'chrome/test/data/app_shim/app_shim_32_bit.app/contents/'
  52. 'macos/app_mode_loader',
  53. )
  54. # These files must not have the executable bit set. This is mainly a performance
  55. # optimization as these files are not checked for shebang. The list was
  56. # partially generated from:
  57. # git ls-files | grep "\\." | sed 's/.*\.//' | sort | uniq -c | sort -b -g
  58. #
  59. # Case-sensitive.
  60. NON_EXECUTABLE_EXTENSIONS = (
  61. '1',
  62. '3ds',
  63. 'S',
  64. 'am',
  65. 'applescript',
  66. 'asm',
  67. 'c',
  68. 'cc',
  69. 'cfg',
  70. 'chromium',
  71. 'cpp',
  72. 'crx',
  73. 'cs',
  74. 'css',
  75. 'cur',
  76. 'def',
  77. 'der',
  78. 'expected',
  79. 'gif',
  80. 'grd',
  81. 'gyp',
  82. 'gypi',
  83. 'h',
  84. 'hh',
  85. 'htm',
  86. 'html',
  87. 'hyph',
  88. 'ico',
  89. 'idl',
  90. 'java',
  91. 'jpg',
  92. 'js',
  93. 'json',
  94. 'm',
  95. 'm4',
  96. 'mm',
  97. 'mms',
  98. 'mock-http-headers',
  99. 'nexe',
  100. 'nmf',
  101. 'onc',
  102. 'pat',
  103. 'patch',
  104. 'pdf',
  105. 'pem',
  106. 'plist',
  107. 'png',
  108. 'proto',
  109. 'rc',
  110. 'rfx',
  111. 'rgs',
  112. 'rules',
  113. 'spec',
  114. 'sql',
  115. 'srpc',
  116. 'svg',
  117. 'tcl',
  118. 'test',
  119. 'tga',
  120. 'txt',
  121. 'vcproj',
  122. 'vsprops',
  123. 'webm',
  124. 'word',
  125. 'xib',
  126. 'xml',
  127. 'xtb',
  128. 'zip',
  129. )
  130. # These files must not have executable bit set.
  131. #
  132. # Case-insensitive, lower-case only.
  133. NON_EXECUTABLE_PATHS = (
  134. 'build/android/tests/symbolize/liba.so',
  135. 'build/android/tests/symbolize/libb.so',
  136. 'chrome/installer/mac/sign_app.sh.in',
  137. 'chrome/installer/mac/sign_versioned_dir.sh.in',
  138. 'courgette/testdata/elf-32-1',
  139. 'courgette/testdata/elf-32-2',
  140. 'courgette/testdata/elf-64',
  141. )
  142. # File names that are always whitelisted. (These are mostly autoconf spew.)
  143. #
  144. # Case-sensitive.
  145. IGNORED_FILENAMES = (
  146. 'config.guess',
  147. 'config.sub',
  148. 'configure',
  149. 'depcomp',
  150. 'install-sh',
  151. 'missing',
  152. 'mkinstalldirs',
  153. 'naclsdk',
  154. 'scons',
  155. )
  156. # File paths starting with one of these will be ignored as well.
  157. # Please consider fixing your file permissions, rather than adding to this list.
  158. #
  159. # Case-insensitive, lower-case only.
  160. IGNORED_PATHS = (
  161. 'native_client_sdk/src/build_tools/sdk_tools/third_party/fancy_urllib/'
  162. '__init__.py',
  163. 'out/',
  164. 'third_party/wpt_tools/wpt/tools/third_party/',
  165. # TODO(maruel): Fix these.
  166. 'third_party/devscripts/licensecheck.pl.vanilla',
  167. 'third_party/libxml/linux/xml2-config',
  168. 'third_party/protobuf/',
  169. 'third_party/sqlite/',
  170. )
  171. #### USER EDITABLE SECTION ENDS HERE ####
  172. assert (set(EXECUTABLE_EXTENSIONS) & set(IGNORED_EXTENSIONS) &
  173. set(NON_EXECUTABLE_EXTENSIONS) == set())
  174. assert set(EXECUTABLE_PATHS) & set(NON_EXECUTABLE_PATHS) == set()
  175. VALID_CHARS = set(string.ascii_lowercase + string.digits + '/-_.')
  176. for paths in (EXECUTABLE_PATHS, NON_EXECUTABLE_PATHS, IGNORED_PATHS):
  177. assert all(set(path).issubset(VALID_CHARS) for path in paths)
  178. git_name = 'git.bat' if sys.platform.startswith('win') else 'git'
  179. def capture(cmd, cwd):
  180. """Returns the output of a command.
  181. Ignores the error code or stderr.
  182. """
  183. logging.debug('%s; cwd=%s' % (' '.join(cmd), cwd))
  184. env = os.environ.copy()
  185. env['LANGUAGE'] = 'en_US.UTF-8'
  186. p = subprocess.Popen(
  187. cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env)
  188. return p.communicate()[0].decode('utf-8', 'ignore')
  189. def get_git_root(dir_path):
  190. """Returns the git checkout root or None."""
  191. root = capture([git_name, 'rev-parse', '--show-toplevel'], dir_path).strip()
  192. if root:
  193. return root
  194. # Should not be reached.
  195. return None
  196. def is_ignored(rel_path):
  197. """Returns True if rel_path is in our whitelist of files to ignore."""
  198. rel_path = rel_path.lower()
  199. return (
  200. os.path.basename(rel_path) in IGNORED_FILENAMES or
  201. rel_path.lower().startswith(IGNORED_PATHS))
  202. def must_be_executable(rel_path):
  203. """The file name represents a file type that must have the executable bit
  204. set.
  205. """
  206. return (os.path.splitext(rel_path)[1][1:] in EXECUTABLE_EXTENSIONS or
  207. rel_path.lower() in EXECUTABLE_PATHS)
  208. def ignored_extension(rel_path):
  209. """The file name represents a file type that may or may not have the
  210. executable set.
  211. """
  212. return os.path.splitext(rel_path)[1][1:] in IGNORED_EXTENSIONS
  213. def must_not_be_executable(rel_path):
  214. """The file name represents a file type that must not have the executable
  215. bit set.
  216. """
  217. return (os.path.splitext(rel_path)[1][1:] in NON_EXECUTABLE_EXTENSIONS or
  218. rel_path.lower() in NON_EXECUTABLE_PATHS)
  219. def has_executable_bit(full_path):
  220. """Returns if any executable bit is set."""
  221. if sys.platform.startswith('win'):
  222. # Using stat doesn't work on Windows, we have to ask git what the
  223. # permissions are.
  224. dir_part, file_part = os.path.split(full_path)
  225. bits = capture([git_name, 'ls-files', '-s', file_part], dir_part).strip()
  226. return bits.startswith('100755')
  227. permission = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
  228. return bool(permission & os.stat(full_path).st_mode)
  229. def has_shebang_or_is_elf_or_mach_o(full_path):
  230. """Returns a three-element tuple that indicates if the file starts with #!/,
  231. is an ELF binary, or Mach-O binary.
  232. full_path is the absolute path to the file.
  233. """
  234. with open(full_path, 'rb') as f:
  235. data = f.read(4)
  236. return (
  237. data[:3] == b'#!/' or data == b'#! /',
  238. data == b'\x7fELF', # ELFMAG
  239. data in (
  240. b'\xfe\xed\xfa\xce', # MH_MAGIC
  241. b'\xce\xfa\xed\xfe', # MH_CIGAM
  242. b'\xfe\xed\xfa\xcf', # MH_MAGIC_64
  243. b'\xcf\xfa\xed\xfe', # MH_CIGAM_64
  244. b'\xca\xfe\xba\xbe', # FAT_MAGIC
  245. b'\xbe\xba\xfe\xca', # FAT_CIGAM
  246. b'\xca\xfe\xba\xbf', # FAT_MAGIC_64
  247. b'\xbf\xba\xfe\xca')) # FAT_CIGAM_64
  248. def check_file(root_path, rel_path):
  249. """Checks the permissions of the file whose path is root_path + rel_path and
  250. returns an error if it is inconsistent. Returns None on success.
  251. It is assumed that the file is not ignored by is_ignored().
  252. If the file name is matched with must_be_executable() or
  253. must_not_be_executable(), only its executable bit is checked.
  254. Otherwise, the first few bytes of the file are read to verify if it has a
  255. shebang or ELF or Mach-O header and compares this with the executable bit on
  256. the file.
  257. """
  258. full_path = os.path.join(root_path, rel_path)
  259. def result_dict(error):
  260. return {
  261. 'error': error,
  262. 'full_path': full_path,
  263. 'rel_path': rel_path,
  264. }
  265. try:
  266. bit = has_executable_bit(full_path)
  267. except OSError:
  268. # It's faster to catch exception than call os.path.islink(). The Chromium
  269. # tree may have invalid symlinks.
  270. return None
  271. exec_add = 'git add --chmod=+x %s' % rel_path
  272. exec_remove = 'git add --chmod=-x %s' % rel_path
  273. if must_be_executable(rel_path):
  274. if not bit:
  275. return result_dict('Must have executable bit set: %s' % exec_add)
  276. return None
  277. if must_not_be_executable(rel_path):
  278. if bit:
  279. return result_dict('Must not have executable bit set: %s' % exec_remove)
  280. return None
  281. if ignored_extension(rel_path):
  282. return None
  283. # For the others, it depends on the file header.
  284. (shebang, elf, mach_o) = has_shebang_or_is_elf_or_mach_o(full_path)
  285. if bit != (shebang or elf or mach_o):
  286. if bit:
  287. return result_dict(
  288. 'Has executable bit but not shebang or ELF or Mach-O header: %s' %
  289. exec_remove)
  290. if shebang:
  291. return result_dict('Has shebang but not executable bit: %s' % exec_add)
  292. if elf:
  293. return result_dict('Has ELF header but not executable bit: %s' % exec_add)
  294. # Mach-O is allowed to exist in the tree with or without an executable bit.
  295. return None
  296. def check_files(root, files):
  297. gen = (check_file(root, f) for f in files
  298. if not is_ignored(f) and not os.path.isdir(f))
  299. return filter(None, gen)
  300. class ApiBase:
  301. def __init__(self, root_dir, bare_output):
  302. self.root_dir = root_dir
  303. self.bare_output = bare_output
  304. self.count = 0
  305. self.count_read_header = 0
  306. def check_file(self, rel_path):
  307. logging.debug('check_file(%s)' % rel_path)
  308. self.count += 1
  309. if (not must_be_executable(rel_path) and
  310. not must_not_be_executable(rel_path)):
  311. self.count_read_header += 1
  312. return check_file(self.root_dir, rel_path)
  313. def check_dir(self, rel_path):
  314. return self.check(rel_path)
  315. def check(self, start_dir):
  316. """Check the files in start_dir, recursively check its subdirectories."""
  317. errors = []
  318. items = self.list_dir(start_dir)
  319. logging.info('check(%s) -> %d' % (start_dir, len(items)))
  320. for item in items:
  321. full_path = os.path.join(self.root_dir, start_dir, item)
  322. rel_path = full_path[len(self.root_dir) + 1:]
  323. if is_ignored(rel_path):
  324. continue
  325. if os.path.isdir(full_path):
  326. # Depth first.
  327. errors.extend(self.check_dir(rel_path))
  328. else:
  329. error = self.check_file(rel_path)
  330. if error:
  331. errors.append(error)
  332. return errors
  333. def list_dir(self, start_dir):
  334. """Lists all the files and directory inside start_dir."""
  335. return sorted(
  336. x for x in os.listdir(os.path.join(self.root_dir, start_dir))
  337. if not x.startswith('.')
  338. )
  339. class ApiAllFilesAtOnceBase(ApiBase):
  340. _files = None
  341. def list_dir(self, start_dir):
  342. """Lists all the files and directory inside start_dir."""
  343. if self._files is None:
  344. self._files = sorted(self._get_all_files())
  345. if not self.bare_output:
  346. print('Found %s files' % len(self._files))
  347. start_dir = start_dir[len(self.root_dir) + 1:]
  348. return [
  349. x[len(start_dir):] for x in self._files if x.startswith(start_dir)
  350. ]
  351. def _get_all_files(self):
  352. """Lists all the files and directory inside self._root_dir."""
  353. raise NotImplementedError()
  354. class ApiGit(ApiAllFilesAtOnceBase):
  355. def _get_all_files(self):
  356. return capture([git_name, 'ls-files'], cwd=self.root_dir).splitlines()
  357. def get_scm(dir_path, bare):
  358. """Returns a properly configured ApiBase instance."""
  359. cwd = os.getcwd()
  360. root = get_git_root(dir_path or cwd)
  361. if root:
  362. if not bare:
  363. print('Found git repository at %s' % root)
  364. return ApiGit(dir_path or root, bare)
  365. # Returns a non-scm aware checker.
  366. if not bare:
  367. print('Failed to determine the SCM for %s' % dir_path)
  368. return ApiBase(dir_path or cwd, bare)
  369. def main():
  370. usage = """Usage: python %prog [--root <root>] [tocheck]
  371. tocheck Specifies the directory, relative to root, to check. This defaults
  372. to "." so it checks everything.
  373. Examples:
  374. python %prog
  375. python %prog --root /path/to/source chrome"""
  376. parser = optparse.OptionParser(usage=usage)
  377. parser.add_option(
  378. '--root',
  379. help='Specifies the repository root. This defaults '
  380. 'to the checkout repository root')
  381. parser.add_option(
  382. '-v', '--verbose', action='count', default=0, help='Print debug logging')
  383. parser.add_option(
  384. '--bare',
  385. action='store_true',
  386. default=False,
  387. help='Prints the bare filename triggering the checks')
  388. parser.add_option(
  389. '--file', action='append', dest='files',
  390. help='Specifics a list of files to check the permissions of. Only these '
  391. 'files will be checked')
  392. parser.add_option(
  393. '--file-list',
  394. help='Specifies a file with a list of files (one per line) to check the '
  395. 'permissions of. Only these files will be checked')
  396. parser.add_option('--json', help='Path to JSON output file')
  397. options, args = parser.parse_args()
  398. levels = [logging.ERROR, logging.INFO, logging.DEBUG]
  399. logging.basicConfig(level=levels[min(len(levels) - 1, options.verbose)])
  400. if len(args) > 1:
  401. parser.error('Too many arguments used')
  402. if options.files and options.file_list:
  403. parser.error('--file and --file-list are mutually exclusive options')
  404. if sys.platform.startswith(
  405. 'win') and not options.files and not options.file_list:
  406. # checkperms of the entire tree on Windows takes many hours so is not
  407. # supported. Instead just check this script.
  408. options.files = [sys.argv[0]]
  409. options.root = '.'
  410. print('Full-tree checkperms not supported on Windows.')
  411. if options.root:
  412. options.root = os.path.abspath(options.root)
  413. if options.files:
  414. errors = check_files(options.root, options.files)
  415. elif options.file_list:
  416. with open(options.file_list) as file_list:
  417. files = file_list.read().splitlines()
  418. errors = check_files(options.root, files)
  419. else:
  420. api = get_scm(options.root, options.bare)
  421. start_dir = args[0] if args else api.root_dir
  422. errors = api.check(start_dir)
  423. if not options.bare:
  424. print('Processed %s files, %d files were tested for shebang/ELF/Mach-O '
  425. 'header' % (api.count, api.count_read_header))
  426. # Convert to an actual list.
  427. errors = list(errors)
  428. if options.json:
  429. with open(options.json, 'w') as f:
  430. json.dump(errors, f)
  431. if errors:
  432. if options.bare:
  433. print('\n'.join(e['full_path'] for e in errors))
  434. else:
  435. print('\nFAILED\n')
  436. print('\n'.join('%s: %s' % (e['full_path'], e['error']) for e in errors))
  437. return 1
  438. if not options.bare:
  439. print('\nSUCCESS\n')
  440. return 0
  441. if '__main__' == __name__:
  442. sys.exit(main())