combo-layer 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924
  1. #!/usr/bin/env python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. #
  5. # Copyright 2011 Intel Corporation
  6. # Authored-by: Yu Ke <ke.yu@intel.com>
  7. # Paul Eggleton <paul.eggleton@intel.com>
  8. # Richard Purdie <richard.purdie@intel.com>
  9. #
  10. # This program is free software; you can redistribute it and/or modify
  11. # it under the terms of the GNU General Public License version 2 as
  12. # published by the Free Software Foundation.
  13. #
  14. # This program is distributed in the hope that it will be useful,
  15. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17. # GNU General Public License for more details.
  18. #
  19. # You should have received a copy of the GNU General Public License along
  20. # with this program; if not, write to the Free Software Foundation, Inc.,
  21. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  22. import fnmatch
  23. import os, sys
  24. import optparse
  25. import logging
  26. import subprocess
  27. import tempfile
  28. import ConfigParser
  29. import re
  30. from collections import OrderedDict
  31. from string import Template
  32. __version__ = "0.2.1"
  33. def logger_create():
  34. logger = logging.getLogger("")
  35. loggerhandler = logging.StreamHandler()
  36. loggerhandler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s","%H:%M:%S"))
  37. logger.addHandler(loggerhandler)
  38. logger.setLevel(logging.INFO)
  39. return logger
  40. logger = logger_create()
  41. def get_current_branch(repodir=None):
  42. try:
  43. if not os.path.exists(os.path.join(repodir if repodir else '', ".git")):
  44. # Repo not created yet (i.e. during init) so just assume master
  45. return "master"
  46. branchname = runcmd("git symbolic-ref HEAD 2>/dev/null", repodir).strip()
  47. if branchname.startswith("refs/heads/"):
  48. branchname = branchname[11:]
  49. return branchname
  50. except subprocess.CalledProcessError:
  51. return ""
  52. class Configuration(object):
  53. """
  54. Manages the configuration
  55. For an example config file, see combo-layer.conf.example
  56. """
  57. def __init__(self, options):
  58. for key, val in options.__dict__.items():
  59. setattr(self, key, val)
  60. def readsection(parser, section, repo):
  61. for (name, value) in parser.items(section):
  62. if value.startswith("@"):
  63. self.repos[repo][name] = eval(value.strip("@"))
  64. else:
  65. # Apply special type transformations for some properties.
  66. # Type matches the RawConfigParser.get*() methods.
  67. types = {'signoff': 'boolean', 'update': 'boolean'}
  68. if name in types:
  69. value = getattr(parser, 'get' + types[name])(section, name)
  70. self.repos[repo][name] = value
  71. def readglobalsection(parser, section):
  72. for (name, value) in parser.items(section):
  73. if name == "commit_msg":
  74. self.commit_msg_template = value
  75. logger.debug("Loading config file %s" % self.conffile)
  76. self.parser = ConfigParser.ConfigParser()
  77. with open(self.conffile) as f:
  78. self.parser.readfp(f)
  79. # initialize default values
  80. self.commit_msg_template = "Automatic commit to update last_revision"
  81. self.repos = {}
  82. for repo in self.parser.sections():
  83. if repo == "combo-layer-settings":
  84. # special handling for global settings
  85. readglobalsection(self.parser, repo)
  86. else:
  87. self.repos[repo] = {}
  88. readsection(self.parser, repo, repo)
  89. # Load local configuration, if available
  90. self.localconffile = None
  91. self.localparser = None
  92. self.combobranch = None
  93. if self.conffile.endswith('.conf'):
  94. lcfile = self.conffile.replace('.conf', '-local.conf')
  95. if os.path.exists(lcfile):
  96. # Read combo layer branch
  97. self.combobranch = get_current_branch()
  98. logger.debug("Combo layer branch is %s" % self.combobranch)
  99. self.localconffile = lcfile
  100. logger.debug("Loading local config file %s" % self.localconffile)
  101. self.localparser = ConfigParser.ConfigParser()
  102. with open(self.localconffile) as f:
  103. self.localparser.readfp(f)
  104. for section in self.localparser.sections():
  105. if '|' in section:
  106. sectionvals = section.split('|')
  107. repo = sectionvals[0]
  108. if sectionvals[1] != self.combobranch:
  109. continue
  110. else:
  111. repo = section
  112. if repo in self.repos:
  113. readsection(self.localparser, section, repo)
  114. def update(self, repo, option, value, initmode=False):
  115. # If the main config has the option already, that is what we
  116. # are expected to modify.
  117. if self.localparser and not self.parser.has_option(repo, option):
  118. parser = self.localparser
  119. section = "%s|%s" % (repo, self.combobranch)
  120. conffile = self.localconffile
  121. if initmode and not parser.has_section(section):
  122. parser.add_section(section)
  123. else:
  124. parser = self.parser
  125. section = repo
  126. conffile = self.conffile
  127. parser.set(section, option, value)
  128. with open(conffile, "w") as f:
  129. parser.write(f)
  130. self.repos[repo][option] = value
  131. def sanity_check(self, initmode=False):
  132. required_options=["src_uri", "local_repo_dir", "dest_dir", "last_revision"]
  133. if initmode:
  134. required_options.remove("last_revision")
  135. msg = ""
  136. missing_options = []
  137. for name in self.repos:
  138. for option in required_options:
  139. if option not in self.repos[name]:
  140. msg = "%s\nOption %s is not defined for component %s" %(msg, option, name)
  141. missing_options.append(option)
  142. # Sanitize dest_dir so that we do not have to deal with edge cases
  143. # (unset, empty string, double slashes) in the rest of the code.
  144. # It not being set will still be flagged as error because it is
  145. # listed as required option above; that could be changed now.
  146. dest_dir = os.path.normpath(self.repos[name].get("dest_dir", "."))
  147. self.repos[name]["dest_dir"] = "." if not dest_dir else dest_dir
  148. if msg != "":
  149. logger.error("configuration file %s has the following error: %s" % (self.conffile,msg))
  150. if self.localconffile and 'last_revision' in missing_options:
  151. logger.error("local configuration file %s may be missing configuration for combo branch %s" % (self.localconffile, self.combobranch))
  152. sys.exit(1)
  153. # filterdiff is required by action_splitpatch, so check its availability
  154. if subprocess.call("which filterdiff > /dev/null 2>&1", shell=True) != 0:
  155. logger.error("ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)")
  156. sys.exit(1)
  157. def runcmd(cmd,destdir=None,printerr=True,out=None):
  158. """
  159. execute command, raise CalledProcessError if fail
  160. return output if succeed
  161. """
  162. logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
  163. if not out:
  164. out = os.tmpfile()
  165. err = out
  166. else:
  167. err = os.tmpfile()
  168. try:
  169. subprocess.check_call(cmd, stdout=out, stderr=err, cwd=destdir, shell=isinstance(cmd, str))
  170. except subprocess.CalledProcessError,e:
  171. err.seek(0)
  172. if printerr:
  173. logger.error("%s" % err.read())
  174. raise e
  175. err.seek(0)
  176. output = err.read()
  177. logger.debug("output: %s" % output )
  178. return output
  179. def action_init(conf, args):
  180. """
  181. Clone component repositories
  182. Check git is initialised; if not, copy initial data from component repos
  183. """
  184. for name in conf.repos:
  185. ldir = conf.repos[name]['local_repo_dir']
  186. if not os.path.exists(ldir):
  187. logger.info("cloning %s to %s" %(conf.repos[name]['src_uri'], ldir))
  188. subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True)
  189. if not os.path.exists(".git"):
  190. runcmd("git init")
  191. if conf.history:
  192. # Need a common ref for all trees.
  193. runcmd('git commit -m "initial empty commit" --allow-empty')
  194. startrev = runcmd('git rev-parse master').strip()
  195. for name in conf.repos:
  196. repo = conf.repos[name]
  197. ldir = repo['local_repo_dir']
  198. branch = repo.get('branch', "master")
  199. lastrev = repo.get('last_revision', None)
  200. if lastrev and lastrev != "HEAD":
  201. initialrev = lastrev
  202. if branch:
  203. if not check_rev_branch(name, ldir, lastrev, branch):
  204. sys.exit(1)
  205. logger.info("Copying data from %s at specified revision %s..." % (name, lastrev))
  206. else:
  207. lastrev = None
  208. initialrev = branch
  209. logger.info("Copying data from %s..." % name)
  210. # Sanity check initialrev and turn it into hash (required for copying history,
  211. # because resolving a name ref only works in the component repo).
  212. rev = runcmd('git rev-parse %s' % initialrev, ldir).strip()
  213. if rev != initialrev:
  214. try:
  215. refs = runcmd('git show-ref -s %s' % initialrev, ldir).split('\n')
  216. if len(set(refs)) > 1:
  217. # Happens for example when configured to track
  218. # "master" and there is a refs/heads/master. The
  219. # traditional behavior from "git archive" (preserved
  220. # here) it to choose the first one. This might not be
  221. # intended, so at least warn about it.
  222. logger.warn("%s: initial revision '%s' not unique, picking result of rev-parse = %s" %
  223. (name, initialrev, refs[0]))
  224. initialrev = rev
  225. except:
  226. # show-ref fails for hashes. Skip the sanity warning in that case.
  227. pass
  228. initialrev = rev
  229. dest_dir = repo['dest_dir']
  230. if dest_dir != ".":
  231. extract_dir = os.path.join(os.getcwd(), dest_dir)
  232. if not os.path.exists(extract_dir):
  233. os.makedirs(extract_dir)
  234. else:
  235. extract_dir = os.getcwd()
  236. file_filter = repo.get('file_filter', "")
  237. exclude_patterns = repo.get('file_exclude', '').split()
  238. def copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir,
  239. subdir=""):
  240. # When working inside a filtered branch which had the
  241. # files already moved, we need to prepend the
  242. # subdirectory to all filters, otherwise they would
  243. # not match.
  244. if subdir == '.':
  245. subdir = ''
  246. elif subdir:
  247. subdir = os.path.normpath(subdir)
  248. file_filter = ' '.join([subdir + '/' + x for x in file_filter.split()])
  249. exclude_patterns = [subdir + '/' + x for x in exclude_patterns]
  250. # To handle both cases, we cd into the target
  251. # directory and optionally tell tar to strip the path
  252. # prefix when the files were already moved.
  253. subdir_components = len(subdir.split(os.path.sep)) if subdir else 0
  254. strip=('--strip-components=%d' % subdir_components) if subdir else ''
  255. # TODO: file_filter wild cards do not work (and haven't worked before either), because
  256. # a) GNU tar requires a --wildcards parameter before turning on wild card matching.
  257. # b) The semantic is not as intendend (src/*.c also matches src/foo/bar.c,
  258. # in contrast to the other use of file_filter as parameter of "git archive"
  259. # where it only matches .c files directly in src).
  260. files = runcmd("git archive %s %s | tar -x -v %s -C %s %s" %
  261. (initialrev, subdir,
  262. strip, extract_dir, file_filter),
  263. ldir)
  264. if exclude_patterns:
  265. # Implement file removal by letting tar create the
  266. # file and then deleting it in the file system
  267. # again. Uses the list of files created by tar (easier
  268. # than walking the tree).
  269. for file in files.split('\n'):
  270. for pattern in exclude_patterns:
  271. if fnmatch.fnmatch(file, pattern):
  272. os.unlink(os.path.join(*([extract_dir] + ['..'] * subdir_components + [file])))
  273. break
  274. if not conf.history:
  275. copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir)
  276. else:
  277. # First fetch remote history into local repository.
  278. # We need a ref for that, so ensure that there is one.
  279. refname = "combo-layer-init-%s" % name
  280. runcmd("git branch -f %s %s" % (refname, initialrev), ldir)
  281. runcmd("git fetch %s %s" % (ldir, refname))
  282. runcmd("git branch -D %s" % refname, ldir)
  283. # Make that the head revision.
  284. runcmd("git checkout -b %s %s" % (name, initialrev))
  285. # Optional: cut the history by replacing the given
  286. # start point(s) with commits providing the same
  287. # content (aka tree), but with commit information that
  288. # makes it clear that this is an artifically created
  289. # commit and nothing the original authors had anything
  290. # to do with.
  291. since_rev = repo.get('since_revision', '')
  292. if since_rev:
  293. committer = runcmd('git var GIT_AUTHOR_IDENT').strip()
  294. # Same time stamp, no name.
  295. author = re.sub('.* (\d+ [+-]\d+)', r'unknown <unknown> \1', committer)
  296. logger.info('author %s' % author)
  297. for rev in since_rev.split():
  298. # Resolve in component repo...
  299. rev = runcmd('git log --oneline --no-abbrev-commit -n1 %s' % rev, ldir).split()[0]
  300. # ... and then get the tree in current
  301. # one. The commit should be in both repos with
  302. # the same tree, but better check here.
  303. tree = runcmd('git show -s --pretty=format:%%T %s' % rev).strip()
  304. with tempfile.NamedTemporaryFile() as editor:
  305. editor.write('''cat >$1 <<EOF
  306. tree %s
  307. author %s
  308. committer %s
  309. %s: squashed import of component
  310. This commit copies the entire set of files as found in
  311. %s %s
  312. For more information about previous commits, see the
  313. upstream repository.
  314. Commit created by combo-layer.
  315. EOF
  316. ''' % (tree, author, committer, name, name, since_rev))
  317. editor.flush()
  318. os.environ['GIT_EDITOR'] = 'sh %s' % editor.name
  319. runcmd('git replace --edit %s' % rev)
  320. # Optional: rewrite history to change commit messages or to move files.
  321. if 'hook' in repo or dest_dir != ".":
  322. filter_branch = ['git', 'filter-branch', '--force']
  323. with tempfile.NamedTemporaryFile() as hookwrapper:
  324. if 'hook' in repo:
  325. # Create a shell script wrapper around the original hook that
  326. # can be used by git filter-branch. Hook may or may not have
  327. # an absolute path.
  328. hook = repo['hook']
  329. hook = os.path.join(os.path.dirname(conf.conffile), '..', hook)
  330. # The wrappers turns the commit message
  331. # from stdin into a fake patch header.
  332. # This is good enough for changing Subject
  333. # and commit msg body with normal
  334. # combo-layer hooks.
  335. hookwrapper.write('''set -e
  336. tmpname=$(mktemp)
  337. trap "rm $tmpname" EXIT
  338. echo -n 'Subject: [PATCH] ' >>$tmpname
  339. cat >>$tmpname
  340. if ! [ $(tail -c 1 $tmpname | od -A n -t x1) == '0a' ]; then
  341. echo >>$tmpname
  342. fi
  343. echo '---' >>$tmpname
  344. %s $tmpname $GIT_COMMIT %s
  345. tail -c +18 $tmpname | head -c -4
  346. ''' % (hook, name))
  347. hookwrapper.flush()
  348. filter_branch.extend(['--msg-filter', 'bash %s' % hookwrapper.name])
  349. if dest_dir != ".":
  350. parent = os.path.dirname(dest_dir)
  351. if not parent:
  352. parent = '.'
  353. # May run outside of the current directory, so do not assume that .git exists.
  354. filter_branch.extend(['--tree-filter', 'mkdir -p .git/tmptree && find . -mindepth 1 -maxdepth 1 ! -name .git -print0 | xargs -0 -I SOURCE mv SOURCE .git/tmptree && mkdir -p %s && mv .git/tmptree %s' % (parent, dest_dir)])
  355. filter_branch.append('HEAD')
  356. runcmd(filter_branch)
  357. runcmd('git update-ref -d refs/original/refs/heads/%s' % name)
  358. repo['rewritten_revision'] = runcmd('git rev-parse HEAD').strip()
  359. repo['stripped_revision'] = repo['rewritten_revision']
  360. # Optional filter files: remove everything and re-populate using the normal filtering code.
  361. # Override any potential .gitignore.
  362. if file_filter or exclude_patterns:
  363. runcmd('git rm -rf .')
  364. if not os.path.exists(extract_dir):
  365. os.makedirs(extract_dir)
  366. copy_selected_files('HEAD', extract_dir, file_filter, exclude_patterns, '.',
  367. subdir=dest_dir)
  368. runcmd('git add --all --force .')
  369. if runcmd('git status --porcelain'):
  370. # Something to commit.
  371. runcmd(['git', 'commit', '-m',
  372. '''%s: select file subset
  373. Files from the component repository were chosen based on
  374. the following filters:
  375. file_filter = %s
  376. file_exclude = %s''' % (name, file_filter or '<empty>', repo.get('file_exclude', '<empty>'))])
  377. repo['stripped_revision'] = runcmd('git rev-parse HEAD').strip()
  378. if not lastrev:
  379. lastrev = runcmd('git rev-parse %s' % initialrev, ldir).strip()
  380. conf.update(name, "last_revision", lastrev, initmode=True)
  381. if not conf.history:
  382. runcmd("git add .")
  383. else:
  384. # Create Octopus merge commit according to http://stackoverflow.com/questions/10874149/git-octopus-merge-with-unrelated-repositoies
  385. runcmd('git checkout master')
  386. merge = ['git', 'merge', '--no-commit']
  387. for name in conf.repos:
  388. repo = conf.repos[name]
  389. # Use branch created earlier.
  390. merge.append(name)
  391. # Root all commits which have no parent in the common
  392. # ancestor in the new repository.
  393. for start in runcmd('git log --pretty=format:%%H --max-parents=0 %s' % name).split('\n'):
  394. runcmd('git replace --graft %s %s' % (start, startrev))
  395. try:
  396. runcmd(merge)
  397. except Exception, error:
  398. logger.info('''Merging component repository history failed, perhaps because of merge conflicts.
  399. It may be possible to commit anyway after resolving these conflicts.
  400. %s''' % error)
  401. # Create MERGE_HEAD and MERGE_MSG. "git merge" itself
  402. # does not create MERGE_HEAD in case of a (harmless) failure,
  403. # and we want certain auto-generated information in the
  404. # commit message for future reference and/or automation.
  405. with open('.git/MERGE_HEAD', 'w') as head:
  406. with open('.git/MERGE_MSG', 'w') as msg:
  407. msg.write('repo: initial import of components\n\n')
  408. # head.write('%s\n' % startrev)
  409. for name in conf.repos:
  410. repo = conf.repos[name]
  411. # <upstream ref> <rewritten ref> <rewritten + files removed>
  412. msg.write('combo-layer-%s: %s %s %s\n' % (name,
  413. repo['last_revision'],
  414. repo['rewritten_revision'],
  415. repo['stripped_revision']))
  416. rev = runcmd('git rev-parse %s' % name).strip()
  417. head.write('%s\n' % rev)
  418. if conf.localconffile:
  419. localadded = True
  420. try:
  421. runcmd("git rm --cached %s" % conf.localconffile, printerr=False)
  422. except subprocess.CalledProcessError:
  423. localadded = False
  424. if localadded:
  425. localrelpath = os.path.relpath(conf.localconffile)
  426. runcmd("grep -q %s .gitignore || echo %s >> .gitignore" % (localrelpath, localrelpath))
  427. runcmd("git add .gitignore")
  428. logger.info("Added local configuration file %s to .gitignore", localrelpath)
  429. logger.info("Initial combo layer repository data has been created; please make any changes if desired and then use 'git commit' to make the initial commit.")
  430. else:
  431. logger.info("Repository already initialised, nothing to do.")
  432. def check_repo_clean(repodir):
  433. """
  434. check if the repo is clean
  435. exit if repo is dirty
  436. """
  437. output=runcmd("git status --porcelain", repodir)
  438. r = re.compile('\?\? patch-.*/')
  439. dirtyout = [item for item in output.splitlines() if not r.match(item)]
  440. if dirtyout:
  441. logger.error("git repo %s is dirty, please fix it first", repodir)
  442. sys.exit(1)
  443. def check_patch(patchfile):
  444. f = open(patchfile)
  445. ln = f.readline()
  446. of = None
  447. in_patch = False
  448. beyond_msg = False
  449. pre_buf = ''
  450. while ln:
  451. if not beyond_msg:
  452. if ln == '---\n':
  453. if not of:
  454. break
  455. in_patch = False
  456. beyond_msg = True
  457. elif ln.startswith('--- '):
  458. # We have a diff in the commit message
  459. in_patch = True
  460. if not of:
  461. print('WARNING: %s contains a diff in its commit message, indenting to avoid failure during apply' % patchfile)
  462. of = open(patchfile + '.tmp', 'w')
  463. of.write(pre_buf)
  464. pre_buf = ''
  465. elif in_patch and not ln[0] in '+-@ \n\r':
  466. in_patch = False
  467. if of:
  468. if in_patch:
  469. of.write(' ' + ln)
  470. else:
  471. of.write(ln)
  472. else:
  473. pre_buf += ln
  474. ln = f.readline()
  475. f.close()
  476. if of:
  477. of.close()
  478. os.rename(patchfile + '.tmp', patchfile)
  479. def drop_to_shell(workdir=None):
  480. if not sys.stdin.isatty():
  481. print "Not a TTY so can't drop to shell for resolution, exiting."
  482. return False
  483. shell = os.environ.get('SHELL', 'bash')
  484. print('Dropping to shell "%s"\n' \
  485. 'When you are finished, run the following to continue:\n' \
  486. ' exit -- continue to apply the patches\n' \
  487. ' exit 1 -- abort\n' % shell);
  488. ret = subprocess.call([shell], cwd=workdir)
  489. if ret != 0:
  490. print "Aborting"
  491. return False
  492. else:
  493. return True
  494. def check_rev_branch(component, repodir, rev, branch):
  495. try:
  496. actualbranch = runcmd("git branch --contains %s" % rev, repodir, printerr=False)
  497. except subprocess.CalledProcessError as e:
  498. if e.returncode == 129:
  499. actualbranch = ""
  500. else:
  501. raise
  502. if not actualbranch:
  503. logger.error("%s: specified revision %s is invalid!" % (component, rev))
  504. return False
  505. branches = []
  506. branchlist = actualbranch.split("\n")
  507. for b in branchlist:
  508. branches.append(b.strip().split(' ')[-1])
  509. if branch not in branches:
  510. logger.error("%s: specified revision %s is not on specified branch %s!" % (component, rev, branch))
  511. return False
  512. return True
  513. def get_repos(conf, repo_names):
  514. repos = []
  515. for name in repo_names:
  516. if name.startswith('-'):
  517. break
  518. else:
  519. repos.append(name)
  520. for repo in repos:
  521. if not repo in conf.repos:
  522. logger.error("Specified component '%s' not found in configuration" % repo)
  523. sys.exit(1)
  524. if not repos:
  525. repos = [ repo for repo in conf.repos if conf.repos[repo].get("update", True) ]
  526. return repos
  527. def action_pull(conf, args):
  528. """
  529. update the component repos only
  530. """
  531. repos = get_repos(conf, args[1:])
  532. # make sure all repos are clean
  533. for name in repos:
  534. check_repo_clean(conf.repos[name]['local_repo_dir'])
  535. for name in repos:
  536. repo = conf.repos[name]
  537. ldir = repo['local_repo_dir']
  538. branch = repo.get('branch', "master")
  539. logger.info("update branch %s of component repo %s in %s ..." % (branch, name, ldir))
  540. if not conf.hard_reset:
  541. # Try to pull only the configured branch. Beware that this may fail
  542. # when the branch is currently unknown (for example, after reconfiguring
  543. # combo-layer). In that case we need to fetch everything and try the check out
  544. # and pull again.
  545. try:
  546. runcmd("git checkout %s" % branch, ldir, printerr=False)
  547. except subprocess.CalledProcessError:
  548. output=runcmd("git fetch", ldir)
  549. logger.info(output)
  550. runcmd("git checkout %s" % branch, ldir)
  551. runcmd("git pull --ff-only", ldir)
  552. else:
  553. output=runcmd("git pull --ff-only", ldir)
  554. logger.info(output)
  555. else:
  556. output=runcmd("git fetch", ldir)
  557. logger.info(output)
  558. runcmd("git checkout %s" % branch, ldir)
  559. runcmd("git reset --hard FETCH_HEAD", ldir)
  560. def action_update(conf, args):
  561. """
  562. update the component repos
  563. generate the patch list
  564. apply the generated patches
  565. """
  566. components = [arg.split(':')[0] for arg in args[1:]]
  567. revisions = {}
  568. for arg in args[1:]:
  569. if ':' in arg:
  570. a = arg.split(':', 1)
  571. revisions[a[0]] = a[1]
  572. repos = get_repos(conf, components)
  573. # make sure combo repo is clean
  574. check_repo_clean(os.getcwd())
  575. import uuid
  576. patch_dir = "patch-%s" % uuid.uuid4()
  577. if not os.path.exists(patch_dir):
  578. os.mkdir(patch_dir)
  579. # Step 1: update the component repos
  580. if conf.nopull:
  581. logger.info("Skipping pull (-n)")
  582. else:
  583. action_pull(conf, ['arg0'] + components)
  584. for name in repos:
  585. revision = revisions.get(name, None)
  586. repo = conf.repos[name]
  587. ldir = repo['local_repo_dir']
  588. dest_dir = repo['dest_dir']
  589. branch = repo.get('branch', "master")
  590. repo_patch_dir = os.path.join(os.getcwd(), patch_dir, name)
  591. # Step 2: generate the patch list and store to patch dir
  592. logger.info("Generating patches from %s..." % name)
  593. top_revision = revision or branch
  594. if not check_rev_branch(name, ldir, top_revision, branch):
  595. sys.exit(1)
  596. if dest_dir != ".":
  597. prefix = "--src-prefix=a/%s/ --dst-prefix=b/%s/" % (dest_dir, dest_dir)
  598. else:
  599. prefix = ""
  600. if repo['last_revision'] == "":
  601. logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name)
  602. patch_cmd_range = "--root %s" % top_revision
  603. rev_cmd_range = top_revision
  604. else:
  605. if not check_rev_branch(name, ldir, repo['last_revision'], branch):
  606. sys.exit(1)
  607. patch_cmd_range = "%s..%s" % (repo['last_revision'], top_revision)
  608. rev_cmd_range = patch_cmd_range
  609. file_filter = repo.get('file_filter',".")
  610. # Filter out unwanted files
  611. exclude = repo.get('file_exclude', '')
  612. if exclude:
  613. for path in exclude.split():
  614. p = "%s/%s" % (dest_dir, path) if dest_dir != '.' else path
  615. file_filter += " ':!%s'" % p
  616. patch_cmd = "git format-patch -N %s --output-directory %s %s -- %s" % \
  617. (prefix,repo_patch_dir, patch_cmd_range, file_filter)
  618. output = runcmd(patch_cmd, ldir)
  619. logger.debug("generated patch set:\n%s" % output)
  620. patchlist = output.splitlines()
  621. rev_cmd = "git rev-list --no-merges %s -- %s" % (rev_cmd_range, file_filter)
  622. revlist = runcmd(rev_cmd, ldir).splitlines()
  623. # Step 3: Call repo specific hook to adjust patch
  624. if 'hook' in repo:
  625. # hook parameter is: ./hook patchpath revision reponame
  626. count=len(revlist)-1
  627. for patch in patchlist:
  628. runcmd("%s %s %s %s" % (repo['hook'], patch, revlist[count], name))
  629. count=count-1
  630. # Step 4: write patch list and revision list to file, for user to edit later
  631. patchlist_file = os.path.join(os.getcwd(), patch_dir, "patchlist-%s" % name)
  632. repo['patchlist'] = patchlist_file
  633. f = open(patchlist_file, 'w')
  634. count=len(revlist)-1
  635. for patch in patchlist:
  636. f.write("%s %s\n" % (patch, revlist[count]))
  637. check_patch(os.path.join(patch_dir, patch))
  638. count=count-1
  639. f.close()
  640. # Step 5: invoke bash for user to edit patch and patch list
  641. if conf.interactive:
  642. print('You may now edit the patch and patch list in %s\n' \
  643. 'For example, you can remove unwanted patch entries from patchlist-*, so that they will be not applied later' % patch_dir);
  644. if not drop_to_shell(patch_dir):
  645. sys.exit(1)
  646. # Step 6: apply the generated and revised patch
  647. apply_patchlist(conf, repos)
  648. runcmd("rm -rf %s" % patch_dir)
  649. # Step 7: commit the updated config file if it's being tracked
  650. relpath = os.path.relpath(conf.conffile)
  651. try:
  652. output = runcmd("git status --porcelain %s" % relpath, printerr=False)
  653. except:
  654. # Outside the repository
  655. output = None
  656. if output:
  657. logger.info("Committing updated configuration file")
  658. if output.lstrip().startswith("M"):
  659. # create the "components" string
  660. component_str = "all components"
  661. if len(components) > 0:
  662. # otherwise tell which components were actually changed
  663. component_str = ", ".join(components)
  664. # expand the template with known values
  665. template = Template(conf.commit_msg_template)
  666. raw_msg = template.substitute(components = component_str)
  667. # sanitize the string before using it in command line
  668. msg = raw_msg.replace('"', '\\"')
  669. runcmd('git commit -m "%s" %s' % (msg, relpath))
  670. def apply_patchlist(conf, repos):
  671. """
  672. apply the generated patch list to combo repo
  673. """
  674. for name in repos:
  675. repo = conf.repos[name]
  676. lastrev = repo["last_revision"]
  677. prevrev = lastrev
  678. # Get non-blank lines from patch list file
  679. patchlist = []
  680. if os.path.exists(repo['patchlist']) or not conf.interactive:
  681. # Note: we want this to fail here if the file doesn't exist and we're not in
  682. # interactive mode since the file should exist in this case
  683. with open(repo['patchlist']) as f:
  684. for line in f:
  685. line = line.rstrip()
  686. if line:
  687. patchlist.append(line)
  688. ldir = conf.repos[name]['local_repo_dir']
  689. branch = conf.repos[name].get('branch', "master")
  690. branchrev = runcmd("git rev-parse %s" % branch, ldir).strip()
  691. if patchlist:
  692. logger.info("Applying patches from %s..." % name)
  693. linecount = len(patchlist)
  694. i = 1
  695. for line in patchlist:
  696. patchfile = line.split()[0]
  697. lastrev = line.split()[1]
  698. patchdisp = os.path.relpath(patchfile)
  699. if os.path.getsize(patchfile) == 0:
  700. logger.info("(skipping %d/%d %s - no changes)" % (i, linecount, patchdisp))
  701. else:
  702. cmd = "git am --keep-cr %s-p1 %s" % ('-s ' if repo.get('signoff', True) else '', patchfile)
  703. logger.info("Applying %d/%d: %s" % (i, linecount, patchdisp))
  704. try:
  705. runcmd(cmd)
  706. except subprocess.CalledProcessError:
  707. logger.info('Running "git am --abort" to cleanup repo')
  708. runcmd("git am --abort")
  709. logger.error('"%s" failed' % cmd)
  710. logger.info("Please manually apply patch %s" % patchdisp)
  711. logger.info("Note: if you exit and continue applying without manually applying the patch, it will be skipped")
  712. if not drop_to_shell():
  713. if prevrev != repo['last_revision']:
  714. conf.update(name, "last_revision", prevrev)
  715. sys.exit(1)
  716. prevrev = lastrev
  717. i += 1
  718. # Once all patches are applied, we should update
  719. # last_revision to the branch head instead of the last
  720. # applied patch. The two are not necessarily the same when
  721. # the last commit is a merge commit or when the patches at
  722. # the branch head were intentionally excluded.
  723. #
  724. # If we do not do that for a merge commit, the next
  725. # combo-layer run will only exclude patches reachable from
  726. # one of the merged branches and try to re-apply patches
  727. # from other branches even though they were already
  728. # copied.
  729. #
  730. # If patches were intentionally excluded, the next run will
  731. # present them again instead of skipping over them. This
  732. # may or may not be intended, so the code here is conservative
  733. # and only addresses the "head is merge commit" case.
  734. if lastrev != branchrev and \
  735. len(runcmd("git show --pretty=format:%%P --no-patch %s" % branch, ldir).split()) > 1:
  736. lastrev = branchrev
  737. else:
  738. logger.info("No patches to apply from %s" % name)
  739. lastrev = branchrev
  740. if lastrev != repo['last_revision']:
  741. conf.update(name, "last_revision", lastrev)
  742. def action_splitpatch(conf, args):
  743. """
  744. generate the commit patch and
  745. split the patch per repo
  746. """
  747. logger.debug("action_splitpatch")
  748. if len(args) > 1:
  749. commit = args[1]
  750. else:
  751. commit = "HEAD"
  752. patchdir = "splitpatch-%s" % commit
  753. if not os.path.exists(patchdir):
  754. os.mkdir(patchdir)
  755. # filerange_root is for the repo whose dest_dir is root "."
  756. # and it should be specified by excluding all other repo dest dir
  757. # like "-x repo1 -x repo2 -x repo3 ..."
  758. filerange_root = ""
  759. for name in conf.repos:
  760. dest_dir = conf.repos[name]['dest_dir']
  761. if dest_dir != ".":
  762. filerange_root = '%s -x "%s/*"' % (filerange_root, dest_dir)
  763. for name in conf.repos:
  764. dest_dir = conf.repos[name]['dest_dir']
  765. patch_filename = "%s/%s.patch" % (patchdir, name)
  766. if dest_dir == ".":
  767. cmd = "git format-patch -n1 --stdout %s^..%s | filterdiff -p1 %s > %s" % (commit, commit, filerange_root, patch_filename)
  768. else:
  769. cmd = "git format-patch --no-prefix -n1 --stdout %s^..%s -- %s > %s" % (commit, commit, dest_dir, patch_filename)
  770. runcmd(cmd)
  771. # Detect empty patches (including those produced by filterdiff above
  772. # that contain only preamble text)
  773. if os.path.getsize(patch_filename) == 0 or runcmd("filterdiff %s" % patch_filename) == "":
  774. os.remove(patch_filename)
  775. logger.info("(skipping %s - no changes)", name)
  776. else:
  777. logger.info(patch_filename)
  778. def action_error(conf, args):
  779. logger.info("invalid action %s" % args[0])
  780. actions = {
  781. "init": action_init,
  782. "update": action_update,
  783. "pull": action_pull,
  784. "splitpatch": action_splitpatch,
  785. }
  786. def main():
  787. parser = optparse.OptionParser(
  788. version = "Combo Layer Repo Tool version %s" % __version__,
  789. usage = """%prog [options] action
  790. Create and update a combination layer repository from multiple component repositories.
  791. Action:
  792. init initialise the combo layer repo
  793. update [components] get patches from component repos and apply them to the combo repo
  794. pull [components] just pull component repos only
  795. splitpatch [commit] generate commit patch and split per component, default commit is HEAD""")
  796. parser.add_option("-c", "--conf", help = "specify the config file (conf/combo-layer.conf is the default).",
  797. action = "store", dest = "conffile", default = "conf/combo-layer.conf")
  798. parser.add_option("-i", "--interactive", help = "interactive mode, user can edit the patch list and patches",
  799. action = "store_true", dest = "interactive", default = False)
  800. parser.add_option("-D", "--debug", help = "output debug information",
  801. action = "store_true", dest = "debug", default = False)
  802. parser.add_option("-n", "--no-pull", help = "skip pulling component repos during update",
  803. action = "store_true", dest = "nopull", default = False)
  804. parser.add_option("--hard-reset",
  805. help = "instead of pull do fetch and hard-reset in component repos",
  806. action = "store_true", dest = "hard_reset", default = False)
  807. parser.add_option("-H", "--history", help = "import full history of components during init",
  808. action = "store_true", default = False)
  809. options, args = parser.parse_args(sys.argv)
  810. # Dispatch to action handler
  811. if len(args) == 1:
  812. logger.error("No action specified, exiting")
  813. parser.print_help()
  814. elif args[1] not in actions:
  815. logger.error("Unsupported action %s, exiting\n" % (args[1]))
  816. parser.print_help()
  817. elif not os.path.exists(options.conffile):
  818. logger.error("No valid config file, exiting\n")
  819. parser.print_help()
  820. else:
  821. if options.debug:
  822. logger.setLevel(logging.DEBUG)
  823. confdata = Configuration(options)
  824. initmode = (args[1] == 'init')
  825. confdata.sanity_check(initmode)
  826. actions.get(args[1], action_error)(confdata, args[1:])
  827. if __name__ == "__main__":
  828. try:
  829. ret = main()
  830. except Exception:
  831. ret = 1
  832. import traceback
  833. traceback.print_exc()
  834. sys.exit(ret)