123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540 |
- #!/usr/bin/env python
- # Copyright 2014 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Script that attempts to push to a special git repository to verify that git
- credentials are configured correctly. It also verifies that gclient solution is
- configured to use git checkout.
- It will be added as gclient hook shortly before Chromium switches to git and
- removed after the switch.
- When running as hook in *.corp.google.com network it will also report status
- of the push attempt to the server (on appengine), so that chrome-infra team can
- collect information about misconfigured Git accounts.
- """
- from __future__ import print_function
- import contextlib
- import datetime
- import errno
- import getpass
- import json
- import logging
- import netrc
- import optparse
- import os
- import pprint
- import shutil
- import socket
- import ssl
- import subprocess
- import sys
- import tempfile
- import time
- import urllib2
- import urlparse
- # Absolute path to src/ directory.
- REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- # Absolute path to a file with gclient solutions.
- GCLIENT_CONFIG = os.path.join(os.path.dirname(REPO_ROOT), '.gclient')
- # Incremented whenever some changes to scrip logic are made. Change in version
- # will cause the check to be rerun on next gclient runhooks invocation.
- CHECKER_VERSION = 1
- # Do not attempt to upload a report after this date.
- UPLOAD_DISABLE_TS = datetime.datetime(2014, 10, 1)
- # URL to POST json with results to.
- MOTHERSHIP_URL = (
- 'https://chromium-git-access.appspot.com/'
- 'git_access/api/v1/reports/access_check')
- # Repository to push test commits to.
- TEST_REPO_URL = 'https://chromium.googlesource.com/a/playground/access_test'
- # Git-compatible gclient solution.
- GOOD_GCLIENT_SOLUTION = {
- 'name': 'src',
- 'deps_file': 'DEPS',
- 'managed': False,
- 'url': 'https://chromium.googlesource.com/chromium/src.git',
- }
- # Possible chunks of git push response in case .netrc is misconfigured.
- BAD_ACL_ERRORS = (
- '(prohibited by Gerrit)',
- 'does not match your user account',
- 'Git repository not found',
- 'Invalid user name or password',
- 'Please make sure you have the correct access rights',
- )
- # Git executable to call.
- GIT_EXE = 'git.bat' if sys.platform == 'win32' else 'git'
- def is_on_bot():
- """True when running under buildbot."""
- return os.environ.get('CHROME_HEADLESS') == '1'
- def is_in_google_corp():
- """True when running in google corp network."""
- try:
- return socket.getfqdn().endswith('.corp.google.com')
- except socket.error:
- logging.exception('Failed to get FQDN')
- return False
- def is_using_git():
- """True if git checkout is used."""
- return os.path.exists(os.path.join(REPO_ROOT, '.git', 'objects'))
- def is_using_svn():
- """True if svn checkout is used."""
- return os.path.exists(os.path.join(REPO_ROOT, '.svn'))
- def read_git_config(prop):
- """Reads git config property of src.git repo.
- Returns empty string in case of errors.
- """
- try:
- proc = subprocess.Popen(
- [GIT_EXE, 'config', prop], stdout=subprocess.PIPE, cwd=REPO_ROOT)
- out, _ = proc.communicate()
- return out.strip().decode('utf-8')
- except OSError as exc:
- if exc.errno != errno.ENOENT:
- logging.exception('Unexpected error when calling git')
- return ''
- def read_netrc_user(netrc_obj, host):
- """Reads 'user' field of a host entry in netrc.
- Returns empty string if netrc is missing, or host is not there.
- """
- if not netrc_obj:
- return ''
- entry = netrc_obj.authenticators(host)
- if not entry:
- return ''
- return entry[0]
- def get_git_version():
- """Returns version of git or None if git is not available."""
- try:
- proc = subprocess.Popen([GIT_EXE, '--version'], stdout=subprocess.PIPE)
- out, _ = proc.communicate()
- return out.strip() if proc.returncode == 0 else ''
- except OSError as exc:
- if exc.errno != errno.ENOENT:
- logging.exception('Unexpected error when calling git')
- return ''
- def read_gclient_solution():
- """Read information about 'src' gclient solution from .gclient file.
- Returns tuple:
- (url, deps_file, managed)
- or
- (None, None, None) if no such solution.
- """
- try:
- env = {}
- execfile(GCLIENT_CONFIG, env, env)
- for sol in (env.get('solutions') or []):
- if sol.get('name') == 'src':
- return sol.get('url'), sol.get('deps_file'), sol.get('managed')
- return None, None, None
- except Exception:
- logging.exception('Failed to read .gclient solution')
- return None, None, None
- def read_git_insteadof(host):
- """Reads relevant insteadOf config entries."""
- try:
- proc = subprocess.Popen([GIT_EXE, 'config', '-l'], stdout=subprocess.PIPE)
- out, _ = proc.communicate()
- lines = []
- for line in out.strip().split('\n'):
- line = line.lower()
- if 'insteadof=' in line and host in line:
- lines.append(line)
- return '\n'.join(lines)
- except OSError as exc:
- if exc.errno != errno.ENOENT:
- logging.exception('Unexpected error when calling git')
- return ''
- def scan_configuration():
- """Scans local environment for git related configuration values."""
- # Git checkout?
- is_git = is_using_git()
- # On Windows HOME should be set.
- if 'HOME' in os.environ:
- netrc_path = os.path.join(
- os.environ['HOME'],
- '_netrc' if sys.platform.startswith('win') else '.netrc')
- else:
- netrc_path = None
- # Netrc exists?
- is_using_netrc = netrc_path and os.path.exists(netrc_path)
- # Read it.
- netrc_obj = None
- if is_using_netrc:
- try:
- netrc_obj = netrc.netrc(netrc_path)
- except Exception:
- logging.exception('Failed to read netrc from %s', netrc_path)
- netrc_obj = None
- # Read gclient 'src' solution.
- gclient_url, gclient_deps, gclient_managed = read_gclient_solution()
- return {
- 'checker_version': CHECKER_VERSION,
- 'is_git': is_git,
- 'is_home_set': 'HOME' in os.environ,
- 'is_using_netrc': is_using_netrc,
- 'netrc_file_mode': os.stat(netrc_path).st_mode if is_using_netrc else 0,
- 'git_version': get_git_version(),
- 'platform': sys.platform,
- 'username': getpass.getuser(),
- 'git_user_email': read_git_config('user.email') if is_git else '',
- 'git_user_name': read_git_config('user.name') if is_git else '',
- 'git_insteadof': read_git_insteadof('chromium.googlesource.com'),
- 'chromium_netrc_email':
- read_netrc_user(netrc_obj, 'chromium.googlesource.com'),
- 'chrome_internal_netrc_email':
- read_netrc_user(netrc_obj, 'chrome-internal.googlesource.com'),
- 'gclient_deps': gclient_deps,
- 'gclient_managed': gclient_managed,
- 'gclient_url': gclient_url,
- }
- def last_configuration_path():
- """Path to store last checked configuration."""
- if is_using_git():
- return os.path.join(REPO_ROOT, '.git', 'check_git_push_access_conf.json')
- elif is_using_svn():
- return os.path.join(REPO_ROOT, '.svn', 'check_git_push_access_conf.json')
- else:
- return os.path.join(REPO_ROOT, '.check_git_push_access_conf.json')
- def read_last_configuration():
- """Reads last checked configuration if it exists."""
- try:
- with open(last_configuration_path(), 'r') as f:
- return json.load(f)
- except (IOError, ValueError):
- return None
- def write_last_configuration(conf):
- """Writes last checked configuration to a file."""
- try:
- with open(last_configuration_path(), 'w') as f:
- json.dump(conf, f, indent=2, sort_keys=True)
- except IOError:
- logging.exception('Failed to write JSON to %s', path)
- @contextlib.contextmanager
- def temp_directory():
- """Creates a temp directory, then nukes it."""
- tmp = tempfile.mkdtemp()
- try:
- yield tmp
- finally:
- try:
- shutil.rmtree(tmp)
- except (OSError, IOError):
- logging.exception('Failed to remove temp directory %s', tmp)
- class Runner(object):
- """Runs a bunch of commands in some directory, collects logs from them."""
- def __init__(self, cwd, verbose):
- self.cwd = cwd
- self.verbose = verbose
- self.log = []
- def run(self, cmd):
- self.append_to_log('> ' + ' '.join(cmd))
- retcode = -1
- try:
- proc = subprocess.Popen(
- cmd,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- cwd=self.cwd)
- out, _ = proc.communicate()
- out = out.strip()
- retcode = proc.returncode
- except OSError as exc:
- out = str(exc)
- if retcode:
- out += '\n(exit code: %d)' % retcode
- self.append_to_log(out)
- return retcode
- def append_to_log(self, text):
- if text:
- self.log.append(text)
- if self.verbose:
- logging.warning(text)
- def check_git_config(conf, report_url, verbose):
- """Attempts to push to a git repository, reports results to a server.
- Returns True if the check finished without incidents (push itself may
- have failed) and should NOT be retried on next invocation of the hook.
- """
- # Don't even try to push if netrc is not configured.
- if not conf['chromium_netrc_email']:
- return upload_report(
- conf,
- report_url,
- verbose,
- push_works=False,
- push_log='',
- push_duration_ms=0)
- # Ref to push to, each user has its own ref.
- ref = 'refs/push-test/%s' % conf['chromium_netrc_email']
- push_works = False
- flake = False
- started = time.time()
- try:
- logging.warning('Checking push access to the git repository...')
- with temp_directory() as tmp:
- # Prepare a simple commit on a new timeline.
- runner = Runner(tmp, verbose)
- runner.run([GIT_EXE, 'init', '.'])
- if conf['git_user_name']:
- runner.run([GIT_EXE, 'config', 'user.name', conf['git_user_name']])
- if conf['git_user_email']:
- runner.run([GIT_EXE, 'config', 'user.email', conf['git_user_email']])
- with open(os.path.join(tmp, 'timestamp'), 'w') as f:
- f.write(str(int(time.time() * 1000)))
- runner.run([GIT_EXE, 'add', 'timestamp'])
- runner.run([GIT_EXE, 'commit', '-m', 'Push test.'])
- # Try to push multiple times if it fails due to issues other than ACLs.
- attempt = 0
- while attempt < 5:
- attempt += 1
- logging.info('Pushing to %s %s', TEST_REPO_URL, ref)
- ret = runner.run(
- [GIT_EXE, 'push', TEST_REPO_URL, 'HEAD:%s' % ref, '-f'])
- if not ret:
- push_works = True
- break
- if any(x in runner.log[-1] for x in BAD_ACL_ERRORS):
- push_works = False
- break
- except Exception:
- logging.exception('Unexpected exception when pushing')
- flake = True
- if push_works:
- logging.warning('Git push works!')
- else:
- logging.warning(
- 'Git push doesn\'t work, which is fine if you are not a committer.')
- uploaded = upload_report(
- conf,
- report_url,
- verbose,
- push_works=push_works,
- push_log='\n'.join(runner.log),
- push_duration_ms=int((time.time() - started) * 1000))
- return uploaded and not flake
- def check_gclient_config(conf):
- """Shows warning if gclient solution is not properly configured for git."""
- # Ignore configs that do not have 'src' solution at all.
- if not conf['gclient_url']:
- return
- current = {
- 'name': 'src',
- 'deps_file': conf['gclient_deps'] or 'DEPS',
- 'managed': conf['gclient_managed'] or False,
- 'url': conf['gclient_url'],
- }
- # After depot_tools r291592 both DEPS and .DEPS.git are valid.
- good = GOOD_GCLIENT_SOLUTION.copy()
- good['deps_file'] = current['deps_file']
- if current == good:
- return
- # Show big warning if url or deps_file is wrong.
- if current['url'] != good['url'] or current['deps_file'] != good['deps_file']:
- print('-' * 80)
- print('Your gclient solution is not set to use supported git workflow!')
- print()
- print('Your \'src\' solution (in %s):' % GCLIENT_CONFIG)
- print(pprint.pformat(current, indent=2))
- print()
- print('Correct \'src\' solution to use git:')
- print(pprint.pformat(good, indent=2))
- print()
- print('Please update your .gclient file ASAP.')
- print('-' * 80)
- # Show smaller (additional) warning about managed workflow.
- if current['managed']:
- print('-' * 80)
- print('You are using managed gclient mode with git, which was deprecated '
- 'on 8/22/13:')
- print('https://groups.google.com/a/chromium.org/'
- 'forum/#!topic/chromium-dev/n9N5N3JL2_U')
- print()
- print('It is strongly advised to switch to unmanaged mode. For more '
- 'information about managed mode and reasons for its deprecation see:')
- print(
- 'http://www.chromium.org/developers/how-tos/get-the-code/gclient-managed-mode'
- )
- print()
- print('There\'s also a large suite of tools to assist managing git '
- 'checkouts.\nSee \'man depot_tools\' (or read '
- 'depot_tools/man/html/depot_tools.html).')
- print('-' * 80)
- def upload_report(
- conf, report_url, verbose, push_works, push_log, push_duration_ms):
- """Posts report to the server, returns True if server accepted it.
- Uploads the report only if script is running in Google corp network. Otherwise
- just prints the report.
- """
- report = conf.copy()
- report.update(
- push_works=push_works,
- push_log=push_log,
- push_duration_ms=push_duration_ms)
- as_bytes = json.dumps({'access_check': report}, indent=2, sort_keys=True)
- if verbose:
- print('Status of git push attempt:')
- print(as_bytes)
- # Do not upload it outside of corp or if server side is already disabled.
- if not is_in_google_corp() or datetime.datetime.now() > UPLOAD_DISABLE_TS:
- if verbose:
- print (
- 'You can send the above report to chrome-git-migration@google.com '
- 'if you need help to set up you committer git account.')
- return True
- req = urllib2.Request(
- url=report_url,
- data=as_bytes,
- headers={'Content-Type': 'application/json; charset=utf-8'})
- attempt = 0
- success = False
- while not success and attempt < 10:
- attempt += 1
- try:
- logging.warning(
- 'Attempting to upload the report to %s...',
- urlparse.urlparse(report_url).netloc)
- resp = urllib2.urlopen(req, timeout=5)
- report_id = None
- try:
- report_id = json.load(resp)['report_id']
- except (ValueError, TypeError, KeyError):
- pass
- logging.warning('Report uploaded: %s', report_id)
- success = True
- except (urllib2.URLError, socket.error, ssl.SSLError) as exc:
- logging.warning('Failed to upload the report: %s', exc)
- return success
- def main(args):
- parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
- parser.add_option(
- '--running-as-hook',
- action='store_true',
- help='Set when invoked from gclient hook')
- parser.add_option(
- '--report-url',
- default=MOTHERSHIP_URL,
- help='URL to submit the report to')
- parser.add_option(
- '--verbose',
- action='store_true',
- help='More logging')
- options, args = parser.parse_args()
- if args:
- parser.error('Unknown argument %s' % args)
- logging.basicConfig(
- format='%(message)s',
- level=logging.INFO if options.verbose else logging.WARN)
- # When invoked not as a hook, always run the check.
- if not options.running_as_hook:
- config = scan_configuration()
- check_gclient_config(config)
- check_git_config(config, options.report_url, True)
- return 0
- # Always do nothing on bots.
- if is_on_bot():
- return 0
- # Read current config, verify gclient solution looks correct.
- config = scan_configuration()
- check_gclient_config(config)
- # Do not attempt to push from non-google owned machines.
- if not is_in_google_corp():
- logging.info('Skipping git push check: non *.corp.google.com machine.')
- return 0
- # Skip git push check if current configuration was already checked.
- if config == read_last_configuration():
- logging.info('Check already performed, skipping.')
- return 0
- # Run the check. Mark configuration as checked only on success. Ignore any
- # exceptions or errors. This check must not break gclient runhooks.
- try:
- ok = check_git_config(config, options.report_url, False)
- if ok:
- write_last_configuration(config)
- else:
- logging.warning('Check failed and will be retried on the next run')
- except Exception:
- logging.exception('Unexpected exception when performing git access check')
- return 0
- if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
|