123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520 |
- #!/usr/bin/env python3
- # Copyright (c) 2012 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Makes sure files have the right permissions.
- Some developers have broken SCM configurations that flip the executable
- permission on for no good reason. Unix developers who run ls --color will then
- see .cc files in green and get confused.
- - For file extensions that must be executable, add it to EXECUTABLE_EXTENSIONS.
- - For file extensions that must not be executable, add it to
- NOT_EXECUTABLE_EXTENSIONS.
- - To ignore all the files inside a directory, add it to IGNORED_PATHS.
- - For file base name with ambiguous state and that should not be checked for
- shebang, add it to IGNORED_FILENAMES.
- Any file not matching the above will be opened and looked if it has a shebang
- or an ELF or Mach-O header. If this does not match the executable bit on the
- file, the file will be flagged. Mach-O files are allowed to exist with or
- without an executable bit set, as there are many examples of it appearing as
- test data, and as Mach-O types such as dSYM that canonically do not have their
- executable bits set.
- Note that all directory separators must be slashes (Unix-style) and not
- backslashes. All directories should be relative to the source root and all
- file paths should be only lowercase.
- """
- from __future__ import print_function
- import json
- import logging
- import optparse
- import os
- import stat
- import string
- import subprocess
- import sys
- #### USER EDITABLE SECTION STARTS HERE ####
- # Files with these extensions must have executable bit set.
- #
- # Case-sensitive.
- EXECUTABLE_EXTENSIONS = (
- 'bat',
- 'dll',
- 'exe',
- )
- # Files for which the executable bit may or may not be set.
- IGNORED_EXTENSIONS = (
- 'dylib',
- )
- # These files must have executable bit set.
- #
- # Case-insensitive, lower-case only.
- EXECUTABLE_PATHS = (
- 'chrome/test/data/app_shim/app_shim_32_bit.app/contents/'
- 'macos/app_mode_loader',
- )
- # These files must not have the executable bit set. This is mainly a performance
- # optimization as these files are not checked for shebang. The list was
- # partially generated from:
- # git ls-files | grep "\\." | sed 's/.*\.//' | sort | uniq -c | sort -b -g
- #
- # Case-sensitive.
- NON_EXECUTABLE_EXTENSIONS = (
- '1',
- '3ds',
- 'S',
- 'am',
- 'applescript',
- 'asm',
- 'c',
- 'cc',
- 'cfg',
- 'chromium',
- 'cpp',
- 'crx',
- 'cs',
- 'css',
- 'cur',
- 'def',
- 'der',
- 'expected',
- 'gif',
- 'grd',
- 'gyp',
- 'gypi',
- 'h',
- 'hh',
- 'htm',
- 'html',
- 'hyph',
- 'ico',
- 'idl',
- 'java',
- 'jpg',
- 'js',
- 'json',
- 'm',
- 'm4',
- 'mm',
- 'mms',
- 'mock-http-headers',
- 'nexe',
- 'nmf',
- 'onc',
- 'pat',
- 'patch',
- 'pdf',
- 'pem',
- 'plist',
- 'png',
- 'proto',
- 'rc',
- 'rfx',
- 'rgs',
- 'rules',
- 'spec',
- 'sql',
- 'srpc',
- 'svg',
- 'tcl',
- 'test',
- 'tga',
- 'txt',
- 'vcproj',
- 'vsprops',
- 'webm',
- 'word',
- 'xib',
- 'xml',
- 'xtb',
- 'zip',
- )
- # These files must not have executable bit set.
- #
- # Case-insensitive, lower-case only.
- NON_EXECUTABLE_PATHS = (
- 'build/android/tests/symbolize/liba.so',
- 'build/android/tests/symbolize/libb.so',
- 'chrome/installer/mac/sign_app.sh.in',
- 'chrome/installer/mac/sign_versioned_dir.sh.in',
- 'courgette/testdata/elf-32-1',
- 'courgette/testdata/elf-32-2',
- 'courgette/testdata/elf-64',
- )
- # File names that are always whitelisted. (These are mostly autoconf spew.)
- #
- # Case-sensitive.
- IGNORED_FILENAMES = (
- 'config.guess',
- 'config.sub',
- 'configure',
- 'depcomp',
- 'install-sh',
- 'missing',
- 'mkinstalldirs',
- 'naclsdk',
- 'scons',
- )
- # File paths starting with one of these will be ignored as well.
- # Please consider fixing your file permissions, rather than adding to this list.
- #
- # Case-insensitive, lower-case only.
- IGNORED_PATHS = (
- 'native_client_sdk/src/build_tools/sdk_tools/third_party/fancy_urllib/'
- '__init__.py',
- 'out/',
- 'third_party/wpt_tools/wpt/tools/third_party/',
- # TODO(maruel): Fix these.
- 'third_party/devscripts/licensecheck.pl.vanilla',
- 'third_party/libxml/linux/xml2-config',
- 'third_party/protobuf/',
- 'third_party/sqlite/',
- )
- #### USER EDITABLE SECTION ENDS HERE ####
- assert (set(EXECUTABLE_EXTENSIONS) & set(IGNORED_EXTENSIONS) &
- set(NON_EXECUTABLE_EXTENSIONS) == set())
- assert set(EXECUTABLE_PATHS) & set(NON_EXECUTABLE_PATHS) == set()
- VALID_CHARS = set(string.ascii_lowercase + string.digits + '/-_.')
- for paths in (EXECUTABLE_PATHS, NON_EXECUTABLE_PATHS, IGNORED_PATHS):
- assert all(set(path).issubset(VALID_CHARS) for path in paths)
- git_name = 'git.bat' if sys.platform.startswith('win') else 'git'
- def capture(cmd, cwd):
- """Returns the output of a command.
- Ignores the error code or stderr.
- """
- logging.debug('%s; cwd=%s' % (' '.join(cmd), cwd))
- env = os.environ.copy()
- env['LANGUAGE'] = 'en_US.UTF-8'
- p = subprocess.Popen(
- cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env)
- return p.communicate()[0].decode('utf-8', 'ignore')
- def get_git_root(dir_path):
- """Returns the git checkout root or None."""
- root = capture([git_name, 'rev-parse', '--show-toplevel'], dir_path).strip()
- if root:
- return root
- # Should not be reached.
- return None
- def is_ignored(rel_path):
- """Returns True if rel_path is in our whitelist of files to ignore."""
- rel_path = rel_path.lower()
- return (
- os.path.basename(rel_path) in IGNORED_FILENAMES or
- rel_path.lower().startswith(IGNORED_PATHS))
- def must_be_executable(rel_path):
- """The file name represents a file type that must have the executable bit
- set.
- """
- return (os.path.splitext(rel_path)[1][1:] in EXECUTABLE_EXTENSIONS or
- rel_path.lower() in EXECUTABLE_PATHS)
- def ignored_extension(rel_path):
- """The file name represents a file type that may or may not have the
- executable set.
- """
- return os.path.splitext(rel_path)[1][1:] in IGNORED_EXTENSIONS
- def must_not_be_executable(rel_path):
- """The file name represents a file type that must not have the executable
- bit set.
- """
- return (os.path.splitext(rel_path)[1][1:] in NON_EXECUTABLE_EXTENSIONS or
- rel_path.lower() in NON_EXECUTABLE_PATHS)
- def has_executable_bit(full_path):
- """Returns if any executable bit is set."""
- if sys.platform.startswith('win'):
- # Using stat doesn't work on Windows, we have to ask git what the
- # permissions are.
- dir_part, file_part = os.path.split(full_path)
- bits = capture([git_name, 'ls-files', '-s', file_part], dir_part).strip()
- return bits.startswith('100755')
- permission = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
- return bool(permission & os.stat(full_path).st_mode)
- def has_shebang_or_is_elf_or_mach_o(full_path):
- """Returns a three-element tuple that indicates if the file starts with #!/,
- is an ELF binary, or Mach-O binary.
- full_path is the absolute path to the file.
- """
- with open(full_path, 'rb') as f:
- data = f.read(4)
- return (
- data[:3] == b'#!/' or data == b'#! /',
- data == b'\x7fELF', # ELFMAG
- data in (
- b'\xfe\xed\xfa\xce', # MH_MAGIC
- b'\xce\xfa\xed\xfe', # MH_CIGAM
- b'\xfe\xed\xfa\xcf', # MH_MAGIC_64
- b'\xcf\xfa\xed\xfe', # MH_CIGAM_64
- b'\xca\xfe\xba\xbe', # FAT_MAGIC
- b'\xbe\xba\xfe\xca', # FAT_CIGAM
- b'\xca\xfe\xba\xbf', # FAT_MAGIC_64
- b'\xbf\xba\xfe\xca')) # FAT_CIGAM_64
- def check_file(root_path, rel_path):
- """Checks the permissions of the file whose path is root_path + rel_path and
- returns an error if it is inconsistent. Returns None on success.
- It is assumed that the file is not ignored by is_ignored().
- If the file name is matched with must_be_executable() or
- must_not_be_executable(), only its executable bit is checked.
- Otherwise, the first few bytes of the file are read to verify if it has a
- shebang or ELF or Mach-O header and compares this with the executable bit on
- the file.
- """
- full_path = os.path.join(root_path, rel_path)
- def result_dict(error):
- return {
- 'error': error,
- 'full_path': full_path,
- 'rel_path': rel_path,
- }
- try:
- bit = has_executable_bit(full_path)
- except OSError:
- # It's faster to catch exception than call os.path.islink(). The Chromium
- # tree may have invalid symlinks.
- return None
- exec_add = 'git add --chmod=+x %s' % rel_path
- exec_remove = 'git add --chmod=-x %s' % rel_path
- if must_be_executable(rel_path):
- if not bit:
- return result_dict('Must have executable bit set: %s' % exec_add)
- return None
- if must_not_be_executable(rel_path):
- if bit:
- return result_dict('Must not have executable bit set: %s' % exec_remove)
- return None
- if ignored_extension(rel_path):
- return None
- # For the others, it depends on the file header.
- (shebang, elf, mach_o) = has_shebang_or_is_elf_or_mach_o(full_path)
- if bit != (shebang or elf or mach_o):
- if bit:
- return result_dict(
- 'Has executable bit but not shebang or ELF or Mach-O header: %s' %
- exec_remove)
- if shebang:
- return result_dict('Has shebang but not executable bit: %s' % exec_add)
- if elf:
- return result_dict('Has ELF header but not executable bit: %s' % exec_add)
- # Mach-O is allowed to exist in the tree with or without an executable bit.
- return None
- def check_files(root, files):
- gen = (check_file(root, f) for f in files
- if not is_ignored(f) and not os.path.isdir(f))
- return filter(None, gen)
- class ApiBase:
- def __init__(self, root_dir, bare_output):
- self.root_dir = root_dir
- self.bare_output = bare_output
- self.count = 0
- self.count_read_header = 0
- def check_file(self, rel_path):
- logging.debug('check_file(%s)' % rel_path)
- self.count += 1
- if (not must_be_executable(rel_path) and
- not must_not_be_executable(rel_path)):
- self.count_read_header += 1
- return check_file(self.root_dir, rel_path)
- def check_dir(self, rel_path):
- return self.check(rel_path)
- def check(self, start_dir):
- """Check the files in start_dir, recursively check its subdirectories."""
- errors = []
- items = self.list_dir(start_dir)
- logging.info('check(%s) -> %d' % (start_dir, len(items)))
- for item in items:
- full_path = os.path.join(self.root_dir, start_dir, item)
- rel_path = full_path[len(self.root_dir) + 1:]
- if is_ignored(rel_path):
- continue
- if os.path.isdir(full_path):
- # Depth first.
- errors.extend(self.check_dir(rel_path))
- else:
- error = self.check_file(rel_path)
- if error:
- errors.append(error)
- return errors
- def list_dir(self, start_dir):
- """Lists all the files and directory inside start_dir."""
- return sorted(
- x for x in os.listdir(os.path.join(self.root_dir, start_dir))
- if not x.startswith('.')
- )
- class ApiAllFilesAtOnceBase(ApiBase):
- _files = None
- def list_dir(self, start_dir):
- """Lists all the files and directory inside start_dir."""
- if self._files is None:
- self._files = sorted(self._get_all_files())
- if not self.bare_output:
- print('Found %s files' % len(self._files))
- start_dir = start_dir[len(self.root_dir) + 1:]
- return [
- x[len(start_dir):] for x in self._files if x.startswith(start_dir)
- ]
- def _get_all_files(self):
- """Lists all the files and directory inside self._root_dir."""
- raise NotImplementedError()
- class ApiGit(ApiAllFilesAtOnceBase):
- def _get_all_files(self):
- return capture([git_name, 'ls-files'], cwd=self.root_dir).splitlines()
- def get_scm(dir_path, bare):
- """Returns a properly configured ApiBase instance."""
- cwd = os.getcwd()
- root = get_git_root(dir_path or cwd)
- if root:
- if not bare:
- print('Found git repository at %s' % root)
- return ApiGit(dir_path or root, bare)
- # Returns a non-scm aware checker.
- if not bare:
- print('Failed to determine the SCM for %s' % dir_path)
- return ApiBase(dir_path or cwd, bare)
- def main():
- usage = """Usage: python %prog [--root <root>] [tocheck]
- tocheck Specifies the directory, relative to root, to check. This defaults
- to "." so it checks everything.
- Examples:
- python %prog
- python %prog --root /path/to/source chrome"""
- parser = optparse.OptionParser(usage=usage)
- parser.add_option(
- '--root',
- help='Specifies the repository root. This defaults '
- 'to the checkout repository root')
- parser.add_option(
- '-v', '--verbose', action='count', default=0, help='Print debug logging')
- parser.add_option(
- '--bare',
- action='store_true',
- default=False,
- help='Prints the bare filename triggering the checks')
- parser.add_option(
- '--file', action='append', dest='files',
- help='Specifics a list of files to check the permissions of. Only these '
- 'files will be checked')
- parser.add_option(
- '--file-list',
- help='Specifies a file with a list of files (one per line) to check the '
- 'permissions of. Only these files will be checked')
- parser.add_option('--json', help='Path to JSON output file')
- options, args = parser.parse_args()
- levels = [logging.ERROR, logging.INFO, logging.DEBUG]
- logging.basicConfig(level=levels[min(len(levels) - 1, options.verbose)])
- if len(args) > 1:
- parser.error('Too many arguments used')
- if options.files and options.file_list:
- parser.error('--file and --file-list are mutually exclusive options')
- if sys.platform.startswith(
- 'win') and not options.files and not options.file_list:
- # checkperms of the entire tree on Windows takes many hours so is not
- # supported. Instead just check this script.
- options.files = [sys.argv[0]]
- options.root = '.'
- print('Full-tree checkperms not supported on Windows.')
- if options.root:
- options.root = os.path.abspath(options.root)
- if options.files:
- errors = check_files(options.root, options.files)
- elif options.file_list:
- with open(options.file_list) as file_list:
- files = file_list.read().splitlines()
- errors = check_files(options.root, files)
- else:
- api = get_scm(options.root, options.bare)
- start_dir = args[0] if args else api.root_dir
- errors = api.check(start_dir)
- if not options.bare:
- print('Processed %s files, %d files were tested for shebang/ELF/Mach-O '
- 'header' % (api.count, api.count_read_header))
- # Convert to an actual list.
- errors = list(errors)
- if options.json:
- with open(options.json, 'w') as f:
- json.dump(errors, f)
- if errors:
- if options.bare:
- print('\n'.join(e['full_path'] for e in errors))
- else:
- print('\nFAILED\n')
- print('\n'.join('%s: %s' % (e['full_path'], e['error']) for e in errors))
- return 1
- if not options.bare:
- print('\nSUCCESS\n')
- return 0
- if '__main__' == __name__:
- sys.exit(main())
|