check_git_config.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540
  1. #!/usr/bin/env python
  2. # Copyright 2014 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Script that attempts to push to a special git repository to verify that git
  6. credentials are configured correctly. It also verifies that gclient solution is
  7. configured to use git checkout.
  8. It will be added as gclient hook shortly before Chromium switches to git and
  9. removed after the switch.
  10. When running as hook in *.corp.google.com network it will also report status
  11. of the push attempt to the server (on appengine), so that chrome-infra team can
  12. collect information about misconfigured Git accounts.
  13. """
  14. from __future__ import print_function
  15. import contextlib
  16. import datetime
  17. import errno
  18. import getpass
  19. import json
  20. import logging
  21. import netrc
  22. import optparse
  23. import os
  24. import pprint
  25. import shutil
  26. import socket
  27. import ssl
  28. import subprocess
  29. import sys
  30. import tempfile
  31. import time
  32. import urllib2
  33. import urlparse
  34. # Absolute path to src/ directory.
  35. REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  36. # Absolute path to a file with gclient solutions.
  37. GCLIENT_CONFIG = os.path.join(os.path.dirname(REPO_ROOT), '.gclient')
  38. # Incremented whenever some changes to scrip logic are made. Change in version
  39. # will cause the check to be rerun on next gclient runhooks invocation.
  40. CHECKER_VERSION = 1
  41. # Do not attempt to upload a report after this date.
  42. UPLOAD_DISABLE_TS = datetime.datetime(2014, 10, 1)
  43. # URL to POST json with results to.
  44. MOTHERSHIP_URL = (
  45. 'https://chromium-git-access.appspot.com/'
  46. 'git_access/api/v1/reports/access_check')
  47. # Repository to push test commits to.
  48. TEST_REPO_URL = 'https://chromium.googlesource.com/a/playground/access_test'
  49. # Git-compatible gclient solution.
  50. GOOD_GCLIENT_SOLUTION = {
  51. 'name': 'src',
  52. 'deps_file': 'DEPS',
  53. 'managed': False,
  54. 'url': 'https://chromium.googlesource.com/chromium/src.git',
  55. }
  56. # Possible chunks of git push response in case .netrc is misconfigured.
  57. BAD_ACL_ERRORS = (
  58. '(prohibited by Gerrit)',
  59. 'does not match your user account',
  60. 'Git repository not found',
  61. 'Invalid user name or password',
  62. 'Please make sure you have the correct access rights',
  63. )
  64. # Git executable to call.
  65. GIT_EXE = 'git.bat' if sys.platform == 'win32' else 'git'
  66. def is_on_bot():
  67. """True when running under buildbot."""
  68. return os.environ.get('CHROME_HEADLESS') == '1'
  69. def is_in_google_corp():
  70. """True when running in google corp network."""
  71. try:
  72. return socket.getfqdn().endswith('.corp.google.com')
  73. except socket.error:
  74. logging.exception('Failed to get FQDN')
  75. return False
  76. def is_using_git():
  77. """True if git checkout is used."""
  78. return os.path.exists(os.path.join(REPO_ROOT, '.git', 'objects'))
  79. def is_using_svn():
  80. """True if svn checkout is used."""
  81. return os.path.exists(os.path.join(REPO_ROOT, '.svn'))
  82. def read_git_config(prop):
  83. """Reads git config property of src.git repo.
  84. Returns empty string in case of errors.
  85. """
  86. try:
  87. proc = subprocess.Popen(
  88. [GIT_EXE, 'config', prop], stdout=subprocess.PIPE, cwd=REPO_ROOT)
  89. out, _ = proc.communicate()
  90. return out.strip().decode('utf-8')
  91. except OSError as exc:
  92. if exc.errno != errno.ENOENT:
  93. logging.exception('Unexpected error when calling git')
  94. return ''
  95. def read_netrc_user(netrc_obj, host):
  96. """Reads 'user' field of a host entry in netrc.
  97. Returns empty string if netrc is missing, or host is not there.
  98. """
  99. if not netrc_obj:
  100. return ''
  101. entry = netrc_obj.authenticators(host)
  102. if not entry:
  103. return ''
  104. return entry[0]
  105. def get_git_version():
  106. """Returns version of git or None if git is not available."""
  107. try:
  108. proc = subprocess.Popen([GIT_EXE, '--version'], stdout=subprocess.PIPE)
  109. out, _ = proc.communicate()
  110. return out.strip() if proc.returncode == 0 else ''
  111. except OSError as exc:
  112. if exc.errno != errno.ENOENT:
  113. logging.exception('Unexpected error when calling git')
  114. return ''
  115. def read_gclient_solution():
  116. """Read information about 'src' gclient solution from .gclient file.
  117. Returns tuple:
  118. (url, deps_file, managed)
  119. or
  120. (None, None, None) if no such solution.
  121. """
  122. try:
  123. env = {}
  124. execfile(GCLIENT_CONFIG, env, env)
  125. for sol in (env.get('solutions') or []):
  126. if sol.get('name') == 'src':
  127. return sol.get('url'), sol.get('deps_file'), sol.get('managed')
  128. return None, None, None
  129. except Exception:
  130. logging.exception('Failed to read .gclient solution')
  131. return None, None, None
  132. def read_git_insteadof(host):
  133. """Reads relevant insteadOf config entries."""
  134. try:
  135. proc = subprocess.Popen([GIT_EXE, 'config', '-l'], stdout=subprocess.PIPE)
  136. out, _ = proc.communicate()
  137. lines = []
  138. for line in out.strip().split('\n'):
  139. line = line.lower()
  140. if 'insteadof=' in line and host in line:
  141. lines.append(line)
  142. return '\n'.join(lines)
  143. except OSError as exc:
  144. if exc.errno != errno.ENOENT:
  145. logging.exception('Unexpected error when calling git')
  146. return ''
  147. def scan_configuration():
  148. """Scans local environment for git related configuration values."""
  149. # Git checkout?
  150. is_git = is_using_git()
  151. # On Windows HOME should be set.
  152. if 'HOME' in os.environ:
  153. netrc_path = os.path.join(
  154. os.environ['HOME'],
  155. '_netrc' if sys.platform.startswith('win') else '.netrc')
  156. else:
  157. netrc_path = None
  158. # Netrc exists?
  159. is_using_netrc = netrc_path and os.path.exists(netrc_path)
  160. # Read it.
  161. netrc_obj = None
  162. if is_using_netrc:
  163. try:
  164. netrc_obj = netrc.netrc(netrc_path)
  165. except Exception:
  166. logging.exception('Failed to read netrc from %s', netrc_path)
  167. netrc_obj = None
  168. # Read gclient 'src' solution.
  169. gclient_url, gclient_deps, gclient_managed = read_gclient_solution()
  170. return {
  171. 'checker_version': CHECKER_VERSION,
  172. 'is_git': is_git,
  173. 'is_home_set': 'HOME' in os.environ,
  174. 'is_using_netrc': is_using_netrc,
  175. 'netrc_file_mode': os.stat(netrc_path).st_mode if is_using_netrc else 0,
  176. 'git_version': get_git_version(),
  177. 'platform': sys.platform,
  178. 'username': getpass.getuser(),
  179. 'git_user_email': read_git_config('user.email') if is_git else '',
  180. 'git_user_name': read_git_config('user.name') if is_git else '',
  181. 'git_insteadof': read_git_insteadof('chromium.googlesource.com'),
  182. 'chromium_netrc_email':
  183. read_netrc_user(netrc_obj, 'chromium.googlesource.com'),
  184. 'chrome_internal_netrc_email':
  185. read_netrc_user(netrc_obj, 'chrome-internal.googlesource.com'),
  186. 'gclient_deps': gclient_deps,
  187. 'gclient_managed': gclient_managed,
  188. 'gclient_url': gclient_url,
  189. }
  190. def last_configuration_path():
  191. """Path to store last checked configuration."""
  192. if is_using_git():
  193. return os.path.join(REPO_ROOT, '.git', 'check_git_push_access_conf.json')
  194. elif is_using_svn():
  195. return os.path.join(REPO_ROOT, '.svn', 'check_git_push_access_conf.json')
  196. else:
  197. return os.path.join(REPO_ROOT, '.check_git_push_access_conf.json')
  198. def read_last_configuration():
  199. """Reads last checked configuration if it exists."""
  200. try:
  201. with open(last_configuration_path(), 'r') as f:
  202. return json.load(f)
  203. except (IOError, ValueError):
  204. return None
  205. def write_last_configuration(conf):
  206. """Writes last checked configuration to a file."""
  207. try:
  208. with open(last_configuration_path(), 'w') as f:
  209. json.dump(conf, f, indent=2, sort_keys=True)
  210. except IOError:
  211. logging.exception('Failed to write JSON to %s', path)
  212. @contextlib.contextmanager
  213. def temp_directory():
  214. """Creates a temp directory, then nukes it."""
  215. tmp = tempfile.mkdtemp()
  216. try:
  217. yield tmp
  218. finally:
  219. try:
  220. shutil.rmtree(tmp)
  221. except (OSError, IOError):
  222. logging.exception('Failed to remove temp directory %s', tmp)
  223. class Runner(object):
  224. """Runs a bunch of commands in some directory, collects logs from them."""
  225. def __init__(self, cwd, verbose):
  226. self.cwd = cwd
  227. self.verbose = verbose
  228. self.log = []
  229. def run(self, cmd):
  230. self.append_to_log('> ' + ' '.join(cmd))
  231. retcode = -1
  232. try:
  233. proc = subprocess.Popen(
  234. cmd,
  235. stdout=subprocess.PIPE,
  236. stderr=subprocess.STDOUT,
  237. cwd=self.cwd)
  238. out, _ = proc.communicate()
  239. out = out.strip()
  240. retcode = proc.returncode
  241. except OSError as exc:
  242. out = str(exc)
  243. if retcode:
  244. out += '\n(exit code: %d)' % retcode
  245. self.append_to_log(out)
  246. return retcode
  247. def append_to_log(self, text):
  248. if text:
  249. self.log.append(text)
  250. if self.verbose:
  251. logging.warning(text)
  252. def check_git_config(conf, report_url, verbose):
  253. """Attempts to push to a git repository, reports results to a server.
  254. Returns True if the check finished without incidents (push itself may
  255. have failed) and should NOT be retried on next invocation of the hook.
  256. """
  257. # Don't even try to push if netrc is not configured.
  258. if not conf['chromium_netrc_email']:
  259. return upload_report(
  260. conf,
  261. report_url,
  262. verbose,
  263. push_works=False,
  264. push_log='',
  265. push_duration_ms=0)
  266. # Ref to push to, each user has its own ref.
  267. ref = 'refs/push-test/%s' % conf['chromium_netrc_email']
  268. push_works = False
  269. flake = False
  270. started = time.time()
  271. try:
  272. logging.warning('Checking push access to the git repository...')
  273. with temp_directory() as tmp:
  274. # Prepare a simple commit on a new timeline.
  275. runner = Runner(tmp, verbose)
  276. runner.run([GIT_EXE, 'init', '.'])
  277. if conf['git_user_name']:
  278. runner.run([GIT_EXE, 'config', 'user.name', conf['git_user_name']])
  279. if conf['git_user_email']:
  280. runner.run([GIT_EXE, 'config', 'user.email', conf['git_user_email']])
  281. with open(os.path.join(tmp, 'timestamp'), 'w') as f:
  282. f.write(str(int(time.time() * 1000)))
  283. runner.run([GIT_EXE, 'add', 'timestamp'])
  284. runner.run([GIT_EXE, 'commit', '-m', 'Push test.'])
  285. # Try to push multiple times if it fails due to issues other than ACLs.
  286. attempt = 0
  287. while attempt < 5:
  288. attempt += 1
  289. logging.info('Pushing to %s %s', TEST_REPO_URL, ref)
  290. ret = runner.run(
  291. [GIT_EXE, 'push', TEST_REPO_URL, 'HEAD:%s' % ref, '-f'])
  292. if not ret:
  293. push_works = True
  294. break
  295. if any(x in runner.log[-1] for x in BAD_ACL_ERRORS):
  296. push_works = False
  297. break
  298. except Exception:
  299. logging.exception('Unexpected exception when pushing')
  300. flake = True
  301. if push_works:
  302. logging.warning('Git push works!')
  303. else:
  304. logging.warning(
  305. 'Git push doesn\'t work, which is fine if you are not a committer.')
  306. uploaded = upload_report(
  307. conf,
  308. report_url,
  309. verbose,
  310. push_works=push_works,
  311. push_log='\n'.join(runner.log),
  312. push_duration_ms=int((time.time() - started) * 1000))
  313. return uploaded and not flake
  314. def check_gclient_config(conf):
  315. """Shows warning if gclient solution is not properly configured for git."""
  316. # Ignore configs that do not have 'src' solution at all.
  317. if not conf['gclient_url']:
  318. return
  319. current = {
  320. 'name': 'src',
  321. 'deps_file': conf['gclient_deps'] or 'DEPS',
  322. 'managed': conf['gclient_managed'] or False,
  323. 'url': conf['gclient_url'],
  324. }
  325. # After depot_tools r291592 both DEPS and .DEPS.git are valid.
  326. good = GOOD_GCLIENT_SOLUTION.copy()
  327. good['deps_file'] = current['deps_file']
  328. if current == good:
  329. return
  330. # Show big warning if url or deps_file is wrong.
  331. if current['url'] != good['url'] or current['deps_file'] != good['deps_file']:
  332. print('-' * 80)
  333. print('Your gclient solution is not set to use supported git workflow!')
  334. print()
  335. print('Your \'src\' solution (in %s):' % GCLIENT_CONFIG)
  336. print(pprint.pformat(current, indent=2))
  337. print()
  338. print('Correct \'src\' solution to use git:')
  339. print(pprint.pformat(good, indent=2))
  340. print()
  341. print('Please update your .gclient file ASAP.')
  342. print('-' * 80)
  343. # Show smaller (additional) warning about managed workflow.
  344. if current['managed']:
  345. print('-' * 80)
  346. print('You are using managed gclient mode with git, which was deprecated '
  347. 'on 8/22/13:')
  348. print('https://groups.google.com/a/chromium.org/'
  349. 'forum/#!topic/chromium-dev/n9N5N3JL2_U')
  350. print()
  351. print('It is strongly advised to switch to unmanaged mode. For more '
  352. 'information about managed mode and reasons for its deprecation see:')
  353. print(
  354. 'http://www.chromium.org/developers/how-tos/get-the-code/gclient-managed-mode'
  355. )
  356. print()
  357. print('There\'s also a large suite of tools to assist managing git '
  358. 'checkouts.\nSee \'man depot_tools\' (or read '
  359. 'depot_tools/man/html/depot_tools.html).')
  360. print('-' * 80)
  361. def upload_report(
  362. conf, report_url, verbose, push_works, push_log, push_duration_ms):
  363. """Posts report to the server, returns True if server accepted it.
  364. Uploads the report only if script is running in Google corp network. Otherwise
  365. just prints the report.
  366. """
  367. report = conf.copy()
  368. report.update(
  369. push_works=push_works,
  370. push_log=push_log,
  371. push_duration_ms=push_duration_ms)
  372. as_bytes = json.dumps({'access_check': report}, indent=2, sort_keys=True)
  373. if verbose:
  374. print('Status of git push attempt:')
  375. print(as_bytes)
  376. # Do not upload it outside of corp or if server side is already disabled.
  377. if not is_in_google_corp() or datetime.datetime.now() > UPLOAD_DISABLE_TS:
  378. if verbose:
  379. print (
  380. 'You can send the above report to chrome-git-migration@google.com '
  381. 'if you need help to set up you committer git account.')
  382. return True
  383. req = urllib2.Request(
  384. url=report_url,
  385. data=as_bytes,
  386. headers={'Content-Type': 'application/json; charset=utf-8'})
  387. attempt = 0
  388. success = False
  389. while not success and attempt < 10:
  390. attempt += 1
  391. try:
  392. logging.warning(
  393. 'Attempting to upload the report to %s...',
  394. urlparse.urlparse(report_url).netloc)
  395. resp = urllib2.urlopen(req, timeout=5)
  396. report_id = None
  397. try:
  398. report_id = json.load(resp)['report_id']
  399. except (ValueError, TypeError, KeyError):
  400. pass
  401. logging.warning('Report uploaded: %s', report_id)
  402. success = True
  403. except (urllib2.URLError, socket.error, ssl.SSLError) as exc:
  404. logging.warning('Failed to upload the report: %s', exc)
  405. return success
  406. def main(args):
  407. parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
  408. parser.add_option(
  409. '--running-as-hook',
  410. action='store_true',
  411. help='Set when invoked from gclient hook')
  412. parser.add_option(
  413. '--report-url',
  414. default=MOTHERSHIP_URL,
  415. help='URL to submit the report to')
  416. parser.add_option(
  417. '--verbose',
  418. action='store_true',
  419. help='More logging')
  420. options, args = parser.parse_args()
  421. if args:
  422. parser.error('Unknown argument %s' % args)
  423. logging.basicConfig(
  424. format='%(message)s',
  425. level=logging.INFO if options.verbose else logging.WARN)
  426. # When invoked not as a hook, always run the check.
  427. if not options.running_as_hook:
  428. config = scan_configuration()
  429. check_gclient_config(config)
  430. check_git_config(config, options.report_url, True)
  431. return 0
  432. # Always do nothing on bots.
  433. if is_on_bot():
  434. return 0
  435. # Read current config, verify gclient solution looks correct.
  436. config = scan_configuration()
  437. check_gclient_config(config)
  438. # Do not attempt to push from non-google owned machines.
  439. if not is_in_google_corp():
  440. logging.info('Skipping git push check: non *.corp.google.com machine.')
  441. return 0
  442. # Skip git push check if current configuration was already checked.
  443. if config == read_last_configuration():
  444. logging.info('Check already performed, skipping.')
  445. return 0
  446. # Run the check. Mark configuration as checked only on success. Ignore any
  447. # exceptions or errors. This check must not break gclient runhooks.
  448. try:
  449. ok = check_git_config(config, options.report_url, False)
  450. if ok:
  451. write_last_configuration(config)
  452. else:
  453. logging.warning('Check failed and will be retried on the next run')
  454. except Exception:
  455. logging.exception('Unexpected exception when performing git access check')
  456. return 0
  457. if __name__ == '__main__':
  458. sys.exit(main(sys.argv[1:]))