combo-layer 60 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376
  1. #!/usr/bin/env python3
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. #
  5. # Copyright 2011 Intel Corporation
  6. # Authored-by: Yu Ke <ke.yu@intel.com>
  7. # Paul Eggleton <paul.eggleton@intel.com>
  8. # Richard Purdie <richard.purdie@intel.com>
  9. #
  10. # This program is free software; you can redistribute it and/or modify
  11. # it under the terms of the GNU General Public License version 2 as
  12. # published by the Free Software Foundation.
  13. #
  14. # This program is distributed in the hope that it will be useful,
  15. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17. # GNU General Public License for more details.
  18. #
  19. # You should have received a copy of the GNU General Public License along
  20. # with this program; if not, write to the Free Software Foundation, Inc.,
  21. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  22. import fnmatch
  23. import os, sys
  24. import optparse
  25. import logging
  26. import subprocess
  27. import tempfile
  28. import configparser
  29. import re
  30. import copy
  31. import pipes
  32. import shutil
  33. from collections import OrderedDict
  34. from string import Template
  35. from functools import reduce
  36. __version__ = "0.2.1"
  37. def logger_create():
  38. logger = logging.getLogger("")
  39. loggerhandler = logging.StreamHandler()
  40. loggerhandler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s","%H:%M:%S"))
  41. logger.addHandler(loggerhandler)
  42. logger.setLevel(logging.INFO)
  43. return logger
  44. logger = logger_create()
  45. def get_current_branch(repodir=None):
  46. try:
  47. if not os.path.exists(os.path.join(repodir if repodir else '', ".git")):
  48. # Repo not created yet (i.e. during init) so just assume master
  49. return "master"
  50. branchname = runcmd("git symbolic-ref HEAD 2>/dev/null", repodir).strip()
  51. if branchname.startswith("refs/heads/"):
  52. branchname = branchname[11:]
  53. return branchname
  54. except subprocess.CalledProcessError:
  55. return ""
  56. class Configuration(object):
  57. """
  58. Manages the configuration
  59. For an example config file, see combo-layer.conf.example
  60. """
  61. def __init__(self, options):
  62. for key, val in options.__dict__.items():
  63. setattr(self, key, val)
  64. def readsection(parser, section, repo):
  65. for (name, value) in parser.items(section):
  66. if value.startswith("@"):
  67. self.repos[repo][name] = eval(value.strip("@"))
  68. else:
  69. # Apply special type transformations for some properties.
  70. # Type matches the RawConfigParser.get*() methods.
  71. types = {'signoff': 'boolean', 'update': 'boolean', 'history': 'boolean'}
  72. if name in types:
  73. value = getattr(parser, 'get' + types[name])(section, name)
  74. self.repos[repo][name] = value
  75. def readglobalsection(parser, section):
  76. for (name, value) in parser.items(section):
  77. if name == "commit_msg":
  78. self.commit_msg_template = value
  79. logger.debug("Loading config file %s" % self.conffile)
  80. self.parser = configparser.ConfigParser()
  81. with open(self.conffile) as f:
  82. self.parser.readfp(f)
  83. # initialize default values
  84. self.commit_msg_template = "Automatic commit to update last_revision"
  85. self.repos = {}
  86. for repo in self.parser.sections():
  87. if repo == "combo-layer-settings":
  88. # special handling for global settings
  89. readglobalsection(self.parser, repo)
  90. else:
  91. self.repos[repo] = {}
  92. readsection(self.parser, repo, repo)
  93. # Load local configuration, if available
  94. self.localconffile = None
  95. self.localparser = None
  96. self.combobranch = None
  97. if self.conffile.endswith('.conf'):
  98. lcfile = self.conffile.replace('.conf', '-local.conf')
  99. if os.path.exists(lcfile):
  100. # Read combo layer branch
  101. self.combobranch = get_current_branch()
  102. logger.debug("Combo layer branch is %s" % self.combobranch)
  103. self.localconffile = lcfile
  104. logger.debug("Loading local config file %s" % self.localconffile)
  105. self.localparser = configparser.ConfigParser()
  106. with open(self.localconffile) as f:
  107. self.localparser.readfp(f)
  108. for section in self.localparser.sections():
  109. if '|' in section:
  110. sectionvals = section.split('|')
  111. repo = sectionvals[0]
  112. if sectionvals[1] != self.combobranch:
  113. continue
  114. else:
  115. repo = section
  116. if repo in self.repos:
  117. readsection(self.localparser, section, repo)
  118. def update(self, repo, option, value, initmode=False):
  119. # If the main config has the option already, that is what we
  120. # are expected to modify.
  121. if self.localparser and not self.parser.has_option(repo, option):
  122. parser = self.localparser
  123. section = "%s|%s" % (repo, self.combobranch)
  124. conffile = self.localconffile
  125. if initmode and not parser.has_section(section):
  126. parser.add_section(section)
  127. else:
  128. parser = self.parser
  129. section = repo
  130. conffile = self.conffile
  131. parser.set(section, option, value)
  132. with open(conffile, "w") as f:
  133. parser.write(f)
  134. self.repos[repo][option] = value
  135. def sanity_check(self, initmode=False):
  136. required_options=["src_uri", "local_repo_dir", "dest_dir", "last_revision"]
  137. if initmode:
  138. required_options.remove("last_revision")
  139. msg = ""
  140. missing_options = []
  141. for name in self.repos:
  142. for option in required_options:
  143. if option not in self.repos[name]:
  144. msg = "%s\nOption %s is not defined for component %s" %(msg, option, name)
  145. missing_options.append(option)
  146. # Sanitize dest_dir so that we do not have to deal with edge cases
  147. # (unset, empty string, double slashes) in the rest of the code.
  148. # It not being set will still be flagged as error because it is
  149. # listed as required option above; that could be changed now.
  150. dest_dir = os.path.normpath(self.repos[name].get("dest_dir", "."))
  151. self.repos[name]["dest_dir"] = "." if not dest_dir else dest_dir
  152. if msg != "":
  153. logger.error("configuration file %s has the following error: %s" % (self.conffile,msg))
  154. if self.localconffile and 'last_revision' in missing_options:
  155. logger.error("local configuration file %s may be missing configuration for combo branch %s" % (self.localconffile, self.combobranch))
  156. sys.exit(1)
  157. # filterdiff is required by action_splitpatch, so check its availability
  158. if subprocess.call("which filterdiff > /dev/null 2>&1", shell=True) != 0:
  159. logger.error("ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)")
  160. sys.exit(1)
  161. def runcmd(cmd,destdir=None,printerr=True,out=None,env=None):
  162. """
  163. execute command, raise CalledProcessError if fail
  164. return output if succeed
  165. """
  166. logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
  167. if not out:
  168. out = tempfile.TemporaryFile()
  169. err = out
  170. else:
  171. err = tempfile.TemporaryFile()
  172. try:
  173. subprocess.check_call(cmd, stdout=out, stderr=err, cwd=destdir, shell=isinstance(cmd, str), env=env or os.environ)
  174. except subprocess.CalledProcessError as e:
  175. err.seek(0)
  176. if printerr:
  177. logger.error("%s" % err.read())
  178. raise e
  179. err.seek(0)
  180. output = err.read().decode('utf-8')
  181. logger.debug("output: %s" % output.replace(chr(0), '\\0'))
  182. return output
  183. def action_init(conf, args):
  184. """
  185. Clone component repositories
  186. Check git is initialised; if not, copy initial data from component repos
  187. """
  188. for name in conf.repos:
  189. ldir = conf.repos[name]['local_repo_dir']
  190. if not os.path.exists(ldir):
  191. logger.info("cloning %s to %s" %(conf.repos[name]['src_uri'], ldir))
  192. subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True)
  193. if not os.path.exists(".git"):
  194. runcmd("git init")
  195. if conf.history:
  196. # Need a common ref for all trees.
  197. runcmd('git commit -m "initial empty commit" --allow-empty')
  198. startrev = runcmd('git rev-parse master').strip()
  199. for name in conf.repos:
  200. repo = conf.repos[name]
  201. ldir = repo['local_repo_dir']
  202. branch = repo.get('branch', "master")
  203. lastrev = repo.get('last_revision', None)
  204. if lastrev and lastrev != "HEAD":
  205. initialrev = lastrev
  206. if branch:
  207. if not check_rev_branch(name, ldir, lastrev, branch):
  208. sys.exit(1)
  209. logger.info("Copying data from %s at specified revision %s..." % (name, lastrev))
  210. else:
  211. lastrev = None
  212. initialrev = branch
  213. logger.info("Copying data from %s..." % name)
  214. # Sanity check initialrev and turn it into hash (required for copying history,
  215. # because resolving a name ref only works in the component repo).
  216. rev = runcmd('git rev-parse %s' % initialrev, ldir).strip()
  217. if rev != initialrev:
  218. try:
  219. refs = runcmd('git show-ref -s %s' % initialrev, ldir).split('\n')
  220. if len(set(refs)) > 1:
  221. # Happens for example when configured to track
  222. # "master" and there is a refs/heads/master. The
  223. # traditional behavior from "git archive" (preserved
  224. # here) it to choose the first one. This might not be
  225. # intended, so at least warn about it.
  226. logger.warn("%s: initial revision '%s' not unique, picking result of rev-parse = %s" %
  227. (name, initialrev, refs[0]))
  228. initialrev = rev
  229. except:
  230. # show-ref fails for hashes. Skip the sanity warning in that case.
  231. pass
  232. initialrev = rev
  233. dest_dir = repo['dest_dir']
  234. if dest_dir != ".":
  235. extract_dir = os.path.join(os.getcwd(), dest_dir)
  236. if not os.path.exists(extract_dir):
  237. os.makedirs(extract_dir)
  238. else:
  239. extract_dir = os.getcwd()
  240. file_filter = repo.get('file_filter', "")
  241. exclude_patterns = repo.get('file_exclude', '').split()
  242. def copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir,
  243. subdir=""):
  244. # When working inside a filtered branch which had the
  245. # files already moved, we need to prepend the
  246. # subdirectory to all filters, otherwise they would
  247. # not match.
  248. if subdir == '.':
  249. subdir = ''
  250. elif subdir:
  251. subdir = os.path.normpath(subdir)
  252. file_filter = ' '.join([subdir + '/' + x for x in file_filter.split()])
  253. exclude_patterns = [subdir + '/' + x for x in exclude_patterns]
  254. # To handle both cases, we cd into the target
  255. # directory and optionally tell tar to strip the path
  256. # prefix when the files were already moved.
  257. subdir_components = len(subdir.split(os.path.sep)) if subdir else 0
  258. strip=('--strip-components=%d' % subdir_components) if subdir else ''
  259. # TODO: file_filter wild cards do not work (and haven't worked before either), because
  260. # a) GNU tar requires a --wildcards parameter before turning on wild card matching.
  261. # b) The semantic is not as intendend (src/*.c also matches src/foo/bar.c,
  262. # in contrast to the other use of file_filter as parameter of "git archive"
  263. # where it only matches .c files directly in src).
  264. files = runcmd("git archive %s %s | tar -x -v %s -C %s %s" %
  265. (initialrev, subdir,
  266. strip, extract_dir, file_filter),
  267. ldir)
  268. if exclude_patterns:
  269. # Implement file removal by letting tar create the
  270. # file and then deleting it in the file system
  271. # again. Uses the list of files created by tar (easier
  272. # than walking the tree).
  273. for file in files.split('\n'):
  274. if file.endswith(os.path.sep):
  275. continue
  276. for pattern in exclude_patterns:
  277. if fnmatch.fnmatch(file, pattern):
  278. os.unlink(os.path.join(*([extract_dir] + ['..'] * subdir_components + [file])))
  279. break
  280. if not conf.history:
  281. copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir)
  282. else:
  283. # First fetch remote history into local repository.
  284. # We need a ref for that, so ensure that there is one.
  285. refname = "combo-layer-init-%s" % name
  286. runcmd("git branch -f %s %s" % (refname, initialrev), ldir)
  287. runcmd("git fetch %s %s" % (ldir, refname))
  288. runcmd("git branch -D %s" % refname, ldir)
  289. # Make that the head revision.
  290. runcmd("git checkout -b %s %s" % (name, initialrev))
  291. # Optional: cut the history by replacing the given
  292. # start point(s) with commits providing the same
  293. # content (aka tree), but with commit information that
  294. # makes it clear that this is an artifically created
  295. # commit and nothing the original authors had anything
  296. # to do with.
  297. since_rev = repo.get('since_revision', '')
  298. if since_rev:
  299. committer = runcmd('git var GIT_AUTHOR_IDENT').strip()
  300. # Same time stamp, no name.
  301. author = re.sub('.* (\d+ [+-]\d+)', r'unknown <unknown> \1', committer)
  302. logger.info('author %s' % author)
  303. for rev in since_rev.split():
  304. # Resolve in component repo...
  305. rev = runcmd('git log --oneline --no-abbrev-commit -n1 %s' % rev, ldir).split()[0]
  306. # ... and then get the tree in current
  307. # one. The commit should be in both repos with
  308. # the same tree, but better check here.
  309. tree = runcmd('git show -s --pretty=format:%%T %s' % rev).strip()
  310. with tempfile.NamedTemporaryFile(mode='wt') as editor:
  311. editor.write('''cat >$1 <<EOF
  312. tree %s
  313. author %s
  314. committer %s
  315. %s: squashed import of component
  316. This commit copies the entire set of files as found in
  317. %s %s
  318. For more information about previous commits, see the
  319. upstream repository.
  320. Commit created by combo-layer.
  321. EOF
  322. ''' % (tree, author, committer, name, name, since_rev))
  323. editor.flush()
  324. os.environ['GIT_EDITOR'] = 'sh %s' % editor.name
  325. runcmd('git replace --edit %s' % rev)
  326. # Optional: rewrite history to change commit messages or to move files.
  327. if 'hook' in repo or dest_dir != ".":
  328. filter_branch = ['git', 'filter-branch', '--force']
  329. with tempfile.NamedTemporaryFile(mode='wt') as hookwrapper:
  330. if 'hook' in repo:
  331. # Create a shell script wrapper around the original hook that
  332. # can be used by git filter-branch. Hook may or may not have
  333. # an absolute path.
  334. hook = repo['hook']
  335. hook = os.path.join(os.path.dirname(conf.conffile), '..', hook)
  336. # The wrappers turns the commit message
  337. # from stdin into a fake patch header.
  338. # This is good enough for changing Subject
  339. # and commit msg body with normal
  340. # combo-layer hooks.
  341. hookwrapper.write('''set -e
  342. tmpname=$(mktemp)
  343. trap "rm $tmpname" EXIT
  344. echo -n 'Subject: [PATCH] ' >>$tmpname
  345. cat >>$tmpname
  346. if ! [ $(tail -c 1 $tmpname | od -A n -t x1) == '0a' ]; then
  347. echo >>$tmpname
  348. fi
  349. echo '---' >>$tmpname
  350. %s $tmpname $GIT_COMMIT %s
  351. tail -c +18 $tmpname | head -c -4
  352. ''' % (hook, name))
  353. hookwrapper.flush()
  354. filter_branch.extend(['--msg-filter', 'bash %s' % hookwrapper.name])
  355. if dest_dir != ".":
  356. parent = os.path.dirname(dest_dir)
  357. if not parent:
  358. parent = '.'
  359. # May run outside of the current directory, so do not assume that .git exists.
  360. filter_branch.extend(['--tree-filter', 'mkdir -p .git/tmptree && find . -mindepth 1 -maxdepth 1 ! -name .git -print0 | xargs -0 -I SOURCE mv SOURCE .git/tmptree && mkdir -p %s && mv .git/tmptree %s' % (parent, dest_dir)])
  361. filter_branch.append('HEAD')
  362. runcmd(filter_branch)
  363. runcmd('git update-ref -d refs/original/refs/heads/%s' % name)
  364. repo['rewritten_revision'] = runcmd('git rev-parse HEAD').strip()
  365. repo['stripped_revision'] = repo['rewritten_revision']
  366. # Optional filter files: remove everything and re-populate using the normal filtering code.
  367. # Override any potential .gitignore.
  368. if file_filter or exclude_patterns:
  369. runcmd('git rm -rf .')
  370. if not os.path.exists(extract_dir):
  371. os.makedirs(extract_dir)
  372. copy_selected_files('HEAD', extract_dir, file_filter, exclude_patterns, '.',
  373. subdir=dest_dir)
  374. runcmd('git add --all --force .')
  375. if runcmd('git status --porcelain'):
  376. # Something to commit.
  377. runcmd(['git', 'commit', '-m',
  378. '''%s: select file subset
  379. Files from the component repository were chosen based on
  380. the following filters:
  381. file_filter = %s
  382. file_exclude = %s''' % (name, file_filter or '<empty>', repo.get('file_exclude', '<empty>'))])
  383. repo['stripped_revision'] = runcmd('git rev-parse HEAD').strip()
  384. if not lastrev:
  385. lastrev = runcmd('git rev-parse %s' % initialrev, ldir).strip()
  386. conf.update(name, "last_revision", lastrev, initmode=True)
  387. if not conf.history:
  388. runcmd("git add .")
  389. else:
  390. # Create Octopus merge commit according to http://stackoverflow.com/questions/10874149/git-octopus-merge-with-unrelated-repositoies
  391. runcmd('git checkout master')
  392. merge = ['git', 'merge', '--no-commit']
  393. for name in conf.repos:
  394. repo = conf.repos[name]
  395. # Use branch created earlier.
  396. merge.append(name)
  397. # Root all commits which have no parent in the common
  398. # ancestor in the new repository.
  399. for start in runcmd('git log --pretty=format:%%H --max-parents=0 %s --' % name).split('\n'):
  400. runcmd('git replace --graft %s %s' % (start, startrev))
  401. try:
  402. runcmd(merge)
  403. except Exception as error:
  404. logger.info('''Merging component repository history failed, perhaps because of merge conflicts.
  405. It may be possible to commit anyway after resolving these conflicts.
  406. %s''' % error)
  407. # Create MERGE_HEAD and MERGE_MSG. "git merge" itself
  408. # does not create MERGE_HEAD in case of a (harmless) failure,
  409. # and we want certain auto-generated information in the
  410. # commit message for future reference and/or automation.
  411. with open('.git/MERGE_HEAD', 'w') as head:
  412. with open('.git/MERGE_MSG', 'w') as msg:
  413. msg.write('repo: initial import of components\n\n')
  414. # head.write('%s\n' % startrev)
  415. for name in conf.repos:
  416. repo = conf.repos[name]
  417. # <upstream ref> <rewritten ref> <rewritten + files removed>
  418. msg.write('combo-layer-%s: %s %s %s\n' % (name,
  419. repo['last_revision'],
  420. repo['rewritten_revision'],
  421. repo['stripped_revision']))
  422. rev = runcmd('git rev-parse %s' % name).strip()
  423. head.write('%s\n' % rev)
  424. if conf.localconffile:
  425. localadded = True
  426. try:
  427. runcmd("git rm --cached %s" % conf.localconffile, printerr=False)
  428. except subprocess.CalledProcessError:
  429. localadded = False
  430. if localadded:
  431. localrelpath = os.path.relpath(conf.localconffile)
  432. runcmd("grep -q %s .gitignore || echo %s >> .gitignore" % (localrelpath, localrelpath))
  433. runcmd("git add .gitignore")
  434. logger.info("Added local configuration file %s to .gitignore", localrelpath)
  435. logger.info("Initial combo layer repository data has been created; please make any changes if desired and then use 'git commit' to make the initial commit.")
  436. else:
  437. logger.info("Repository already initialised, nothing to do.")
  438. def check_repo_clean(repodir):
  439. """
  440. check if the repo is clean
  441. exit if repo is dirty
  442. """
  443. output=runcmd("git status --porcelain", repodir)
  444. r = re.compile('\?\? patch-.*/')
  445. dirtyout = [item for item in output.splitlines() if not r.match(item)]
  446. if dirtyout:
  447. logger.error("git repo %s is dirty, please fix it first", repodir)
  448. sys.exit(1)
  449. def check_patch(patchfile):
  450. f = open(patchfile, 'rb')
  451. ln = f.readline()
  452. of = None
  453. in_patch = False
  454. beyond_msg = False
  455. pre_buf = b''
  456. while ln:
  457. if not beyond_msg:
  458. if ln == b'---\n':
  459. if not of:
  460. break
  461. in_patch = False
  462. beyond_msg = True
  463. elif ln.startswith(b'--- '):
  464. # We have a diff in the commit message
  465. in_patch = True
  466. if not of:
  467. print('WARNING: %s contains a diff in its commit message, indenting to avoid failure during apply' % patchfile)
  468. of = open(patchfile + '.tmp', 'wb')
  469. of.write(pre_buf)
  470. pre_buf = b''
  471. elif in_patch and not ln[0] in b'+-@ \n\r':
  472. in_patch = False
  473. if of:
  474. if in_patch:
  475. of.write(b' ' + ln)
  476. else:
  477. of.write(ln)
  478. else:
  479. pre_buf += ln
  480. ln = f.readline()
  481. f.close()
  482. if of:
  483. of.close()
  484. os.rename(patchfile + '.tmp', patchfile)
  485. def drop_to_shell(workdir=None):
  486. if not sys.stdin.isatty():
  487. print("Not a TTY so can't drop to shell for resolution, exiting.")
  488. return False
  489. shell = os.environ.get('SHELL', 'bash')
  490. print('Dropping to shell "%s"\n' \
  491. 'When you are finished, run the following to continue:\n' \
  492. ' exit -- continue to apply the patches\n' \
  493. ' exit 1 -- abort\n' % shell);
  494. ret = subprocess.call([shell], cwd=workdir)
  495. if ret != 0:
  496. print("Aborting")
  497. return False
  498. else:
  499. return True
  500. def check_rev_branch(component, repodir, rev, branch):
  501. try:
  502. actualbranch = runcmd("git branch --contains %s" % rev, repodir, printerr=False)
  503. except subprocess.CalledProcessError as e:
  504. if e.returncode == 129:
  505. actualbranch = ""
  506. else:
  507. raise
  508. if not actualbranch:
  509. logger.error("%s: specified revision %s is invalid!" % (component, rev))
  510. return False
  511. branches = []
  512. branchlist = actualbranch.split("\n")
  513. for b in branchlist:
  514. branches.append(b.strip().split(' ')[-1])
  515. if branch not in branches:
  516. logger.error("%s: specified revision %s is not on specified branch %s!" % (component, rev, branch))
  517. return False
  518. return True
  519. def get_repos(conf, repo_names):
  520. repos = []
  521. for name in repo_names:
  522. if name.startswith('-'):
  523. break
  524. else:
  525. repos.append(name)
  526. for repo in repos:
  527. if not repo in conf.repos:
  528. logger.error("Specified component '%s' not found in configuration" % repo)
  529. sys.exit(1)
  530. if not repos:
  531. repos = [ repo for repo in conf.repos if conf.repos[repo].get("update", True) ]
  532. return repos
  533. def action_pull(conf, args):
  534. """
  535. update the component repos only
  536. """
  537. repos = get_repos(conf, args[1:])
  538. # make sure all repos are clean
  539. for name in repos:
  540. check_repo_clean(conf.repos[name]['local_repo_dir'])
  541. for name in repos:
  542. repo = conf.repos[name]
  543. ldir = repo['local_repo_dir']
  544. branch = repo.get('branch', "master")
  545. logger.info("update branch %s of component repo %s in %s ..." % (branch, name, ldir))
  546. if not conf.hard_reset:
  547. # Try to pull only the configured branch. Beware that this may fail
  548. # when the branch is currently unknown (for example, after reconfiguring
  549. # combo-layer). In that case we need to fetch everything and try the check out
  550. # and pull again.
  551. try:
  552. runcmd("git checkout %s" % branch, ldir, printerr=False)
  553. except subprocess.CalledProcessError:
  554. output=runcmd("git fetch", ldir)
  555. logger.info(output)
  556. runcmd("git checkout %s" % branch, ldir)
  557. runcmd("git pull --ff-only", ldir)
  558. else:
  559. output=runcmd("git pull --ff-only", ldir)
  560. logger.info(output)
  561. else:
  562. output=runcmd("git fetch", ldir)
  563. logger.info(output)
  564. runcmd("git checkout %s" % branch, ldir)
  565. runcmd("git reset --hard FETCH_HEAD", ldir)
  566. def action_update(conf, args):
  567. """
  568. update the component repos
  569. either:
  570. generate the patch list
  571. apply the generated patches
  572. or:
  573. re-creates the entire component history and merges them
  574. into the current branch with a merge commit
  575. """
  576. components = [arg.split(':')[0] for arg in args[1:]]
  577. revisions = {}
  578. for arg in args[1:]:
  579. if ':' in arg:
  580. a = arg.split(':', 1)
  581. revisions[a[0]] = a[1]
  582. repos = get_repos(conf, components)
  583. # make sure combo repo is clean
  584. check_repo_clean(os.getcwd())
  585. # Check whether we keep the component histories. Must be
  586. # set either via --history command line parameter or consistently
  587. # in combo-layer.conf. Mixing modes is (currently, and probably
  588. # permanently because it would be complicated) not supported.
  589. if conf.history:
  590. history = True
  591. else:
  592. history = None
  593. for name in repos:
  594. repo = conf.repos[name]
  595. repo_history = repo.get('history', False)
  596. if history is None:
  597. history = repo_history
  598. elif history != repo_history:
  599. logger.error("'history' property is set inconsistently")
  600. sys.exit(1)
  601. # Step 1: update the component repos
  602. if conf.nopull:
  603. logger.info("Skipping pull (-n)")
  604. else:
  605. action_pull(conf, ['arg0'] + components)
  606. if history:
  607. update_with_history(conf, components, revisions, repos)
  608. else:
  609. update_with_patches(conf, components, revisions, repos)
  610. def update_with_patches(conf, components, revisions, repos):
  611. import uuid
  612. patch_dir = "patch-%s" % uuid.uuid4()
  613. if not os.path.exists(patch_dir):
  614. os.mkdir(patch_dir)
  615. for name in repos:
  616. revision = revisions.get(name, None)
  617. repo = conf.repos[name]
  618. ldir = repo['local_repo_dir']
  619. dest_dir = repo['dest_dir']
  620. branch = repo.get('branch', "master")
  621. repo_patch_dir = os.path.join(os.getcwd(), patch_dir, name)
  622. # Step 2: generate the patch list and store to patch dir
  623. logger.info("Generating patches from %s..." % name)
  624. top_revision = revision or branch
  625. if not check_rev_branch(name, ldir, top_revision, branch):
  626. sys.exit(1)
  627. if dest_dir != ".":
  628. prefix = "--src-prefix=a/%s/ --dst-prefix=b/%s/" % (dest_dir, dest_dir)
  629. else:
  630. prefix = ""
  631. if repo['last_revision'] == "":
  632. logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name)
  633. patch_cmd_range = "--root %s" % top_revision
  634. rev_cmd_range = top_revision
  635. else:
  636. if not check_rev_branch(name, ldir, repo['last_revision'], branch):
  637. sys.exit(1)
  638. patch_cmd_range = "%s..%s" % (repo['last_revision'], top_revision)
  639. rev_cmd_range = patch_cmd_range
  640. file_filter = repo.get('file_filter',".")
  641. # Filter out unwanted files
  642. exclude = repo.get('file_exclude', '')
  643. if exclude:
  644. for path in exclude.split():
  645. p = "%s/%s" % (dest_dir, path) if dest_dir != '.' else path
  646. file_filter += " ':!%s'" % p
  647. patch_cmd = "git format-patch -N %s --output-directory %s %s -- %s" % \
  648. (prefix,repo_patch_dir, patch_cmd_range, file_filter)
  649. output = runcmd(patch_cmd, ldir)
  650. logger.debug("generated patch set:\n%s" % output)
  651. patchlist = output.splitlines()
  652. rev_cmd = "git rev-list --no-merges %s -- %s" % (rev_cmd_range, file_filter)
  653. revlist = runcmd(rev_cmd, ldir).splitlines()
  654. # Step 3: Call repo specific hook to adjust patch
  655. if 'hook' in repo:
  656. # hook parameter is: ./hook patchpath revision reponame
  657. count=len(revlist)-1
  658. for patch in patchlist:
  659. runcmd("%s %s %s %s" % (repo['hook'], patch, revlist[count], name))
  660. count=count-1
  661. # Step 4: write patch list and revision list to file, for user to edit later
  662. patchlist_file = os.path.join(os.getcwd(), patch_dir, "patchlist-%s" % name)
  663. repo['patchlist'] = patchlist_file
  664. f = open(patchlist_file, 'w')
  665. count=len(revlist)-1
  666. for patch in patchlist:
  667. f.write("%s %s\n" % (patch, revlist[count]))
  668. check_patch(os.path.join(patch_dir, patch))
  669. count=count-1
  670. f.close()
  671. # Step 5: invoke bash for user to edit patch and patch list
  672. if conf.interactive:
  673. print('You may now edit the patch and patch list in %s\n' \
  674. 'For example, you can remove unwanted patch entries from patchlist-*, so that they will be not applied later' % patch_dir);
  675. if not drop_to_shell(patch_dir):
  676. sys.exit(1)
  677. # Step 6: apply the generated and revised patch
  678. apply_patchlist(conf, repos)
  679. runcmd("rm -rf %s" % patch_dir)
  680. # Step 7: commit the updated config file if it's being tracked
  681. commit_conf_file(conf, components)
  682. def conf_commit_msg(conf, components):
  683. # create the "components" string
  684. component_str = "all components"
  685. if len(components) > 0:
  686. # otherwise tell which components were actually changed
  687. component_str = ", ".join(components)
  688. # expand the template with known values
  689. template = Template(conf.commit_msg_template)
  690. msg = template.substitute(components = component_str)
  691. return msg
  692. def commit_conf_file(conf, components, commit=True):
  693. relpath = os.path.relpath(conf.conffile)
  694. try:
  695. output = runcmd("git status --porcelain %s" % relpath, printerr=False)
  696. except:
  697. # Outside the repository
  698. output = None
  699. if output:
  700. if output.lstrip().startswith("M"):
  701. logger.info("Committing updated configuration file")
  702. if commit:
  703. msg = conf_commit_msg(conf, components)
  704. runcmd('git commit -m'.split() + [msg, relpath])
  705. else:
  706. runcmd('git add %s' % relpath)
  707. return True
  708. return False
  709. def apply_patchlist(conf, repos):
  710. """
  711. apply the generated patch list to combo repo
  712. """
  713. for name in repos:
  714. repo = conf.repos[name]
  715. lastrev = repo["last_revision"]
  716. prevrev = lastrev
  717. # Get non-blank lines from patch list file
  718. patchlist = []
  719. if os.path.exists(repo['patchlist']) or not conf.interactive:
  720. # Note: we want this to fail here if the file doesn't exist and we're not in
  721. # interactive mode since the file should exist in this case
  722. with open(repo['patchlist']) as f:
  723. for line in f:
  724. line = line.rstrip()
  725. if line:
  726. patchlist.append(line)
  727. ldir = conf.repos[name]['local_repo_dir']
  728. branch = conf.repos[name].get('branch', "master")
  729. branchrev = runcmd("git rev-parse %s" % branch, ldir).strip()
  730. if patchlist:
  731. logger.info("Applying patches from %s..." % name)
  732. linecount = len(patchlist)
  733. i = 1
  734. for line in patchlist:
  735. patchfile = line.split()[0]
  736. lastrev = line.split()[1]
  737. patchdisp = os.path.relpath(patchfile)
  738. if os.path.getsize(patchfile) == 0:
  739. logger.info("(skipping %d/%d %s - no changes)" % (i, linecount, patchdisp))
  740. else:
  741. cmd = "git am --keep-cr %s-p1 %s" % ('-s ' if repo.get('signoff', True) else '', patchfile)
  742. logger.info("Applying %d/%d: %s" % (i, linecount, patchdisp))
  743. try:
  744. runcmd(cmd)
  745. except subprocess.CalledProcessError:
  746. logger.info('Running "git am --abort" to cleanup repo')
  747. runcmd("git am --abort")
  748. logger.error('"%s" failed' % cmd)
  749. logger.info("Please manually apply patch %s" % patchdisp)
  750. logger.info("Note: if you exit and continue applying without manually applying the patch, it will be skipped")
  751. if not drop_to_shell():
  752. if prevrev != repo['last_revision']:
  753. conf.update(name, "last_revision", prevrev)
  754. sys.exit(1)
  755. prevrev = lastrev
  756. i += 1
  757. # Once all patches are applied, we should update
  758. # last_revision to the branch head instead of the last
  759. # applied patch. The two are not necessarily the same when
  760. # the last commit is a merge commit or when the patches at
  761. # the branch head were intentionally excluded.
  762. #
  763. # If we do not do that for a merge commit, the next
  764. # combo-layer run will only exclude patches reachable from
  765. # one of the merged branches and try to re-apply patches
  766. # from other branches even though they were already
  767. # copied.
  768. #
  769. # If patches were intentionally excluded, the next run will
  770. # present them again instead of skipping over them. This
  771. # may or may not be intended, so the code here is conservative
  772. # and only addresses the "head is merge commit" case.
  773. if lastrev != branchrev and \
  774. len(runcmd("git show --pretty=format:%%P --no-patch %s" % branch, ldir).split()) > 1:
  775. lastrev = branchrev
  776. else:
  777. logger.info("No patches to apply from %s" % name)
  778. lastrev = branchrev
  779. if lastrev != repo['last_revision']:
  780. conf.update(name, "last_revision", lastrev)
  781. def action_splitpatch(conf, args):
  782. """
  783. generate the commit patch and
  784. split the patch per repo
  785. """
  786. logger.debug("action_splitpatch")
  787. if len(args) > 1:
  788. commit = args[1]
  789. else:
  790. commit = "HEAD"
  791. patchdir = "splitpatch-%s" % commit
  792. if not os.path.exists(patchdir):
  793. os.mkdir(patchdir)
  794. # filerange_root is for the repo whose dest_dir is root "."
  795. # and it should be specified by excluding all other repo dest dir
  796. # like "-x repo1 -x repo2 -x repo3 ..."
  797. filerange_root = ""
  798. for name in conf.repos:
  799. dest_dir = conf.repos[name]['dest_dir']
  800. if dest_dir != ".":
  801. filerange_root = '%s -x "%s/*"' % (filerange_root, dest_dir)
  802. for name in conf.repos:
  803. dest_dir = conf.repos[name]['dest_dir']
  804. patch_filename = "%s/%s.patch" % (patchdir, name)
  805. if dest_dir == ".":
  806. cmd = "git format-patch -n1 --stdout %s^..%s | filterdiff -p1 %s > %s" % (commit, commit, filerange_root, patch_filename)
  807. else:
  808. cmd = "git format-patch --no-prefix -n1 --stdout %s^..%s -- %s > %s" % (commit, commit, dest_dir, patch_filename)
  809. runcmd(cmd)
  810. # Detect empty patches (including those produced by filterdiff above
  811. # that contain only preamble text)
  812. if os.path.getsize(patch_filename) == 0 or runcmd("filterdiff %s" % patch_filename) == "":
  813. os.remove(patch_filename)
  814. logger.info("(skipping %s - no changes)", name)
  815. else:
  816. logger.info(patch_filename)
  817. def update_with_history(conf, components, revisions, repos):
  818. '''Update all components with full history.
  819. Works by importing all commits reachable from a component's
  820. current head revision. If those commits are rooted in an already
  821. imported commit, their content gets mixed with the content of the
  822. combined repo of that commit (new or modified files overwritten,
  823. removed files removed).
  824. The last commit is an artificial merge commit that merges all the
  825. updated components into the combined repository.
  826. The HEAD ref only gets updated at the very end. All intermediate work
  827. happens in a worktree which will get garbage collected by git eventually
  828. after a failure.
  829. '''
  830. # Remember current HEAD and what we need to add to it.
  831. head = runcmd("git rev-parse HEAD").strip()
  832. additional_heads = {}
  833. # Track the mapping between original commit and commit in the
  834. # combined repo. We do not have to distinguish between components,
  835. # because commit hashes are different anyway. Often we can
  836. # skip find_revs() entirely (for example, when all new commits
  837. # are derived from the last imported revision).
  838. #
  839. # Using "head" (typically the merge commit) instead of the actual
  840. # commit for the component leads to a nicer history in the combined
  841. # repo.
  842. old2new_revs = {}
  843. for name in repos:
  844. repo = conf.repos[name]
  845. revision = repo['last_revision']
  846. if revision:
  847. old2new_revs[revision] = head
  848. def add_p(parents):
  849. '''Insert -p before each entry.'''
  850. parameters = []
  851. for p in parents:
  852. parameters.append('-p')
  853. parameters.append(p)
  854. return parameters
  855. # Do all intermediate work with a separate work dir and index,
  856. # chosen via env variables (can't use "git worktree", it is too
  857. # new). This is useful (no changes to current work tree unless the
  858. # update succeeds) and required (otherwise we end up temporarily
  859. # removing the combo-layer hooks that we currently use when
  860. # importing a new component).
  861. #
  862. # Not cleaned up after a failure at the moment.
  863. wdir = os.path.join(os.getcwd(), ".git", "combo-layer")
  864. windex = wdir + ".index"
  865. if os.path.isdir(wdir):
  866. shutil.rmtree(wdir)
  867. os.mkdir(wdir)
  868. wenv = copy.deepcopy(os.environ)
  869. wenv["GIT_WORK_TREE"] = wdir
  870. wenv["GIT_INDEX_FILE"] = windex
  871. # This one turned out to be needed in practice.
  872. wenv["GIT_OBJECT_DIRECTORY"] = os.path.join(os.getcwd(), ".git", "objects")
  873. wargs = {"destdir": wdir, "env": wenv}
  874. for name in repos:
  875. revision = revisions.get(name, None)
  876. repo = conf.repos[name]
  877. ldir = repo['local_repo_dir']
  878. dest_dir = repo['dest_dir']
  879. branch = repo.get('branch', "master")
  880. hook = repo.get('hook', None)
  881. largs = {"destdir": ldir, "env": None}
  882. file_include = repo.get('file_filter', '').split()
  883. file_include.sort() # make sure that short entries like '.' come first.
  884. file_exclude = repo.get('file_exclude', '').split()
  885. def include_file(file):
  886. if not file_include:
  887. # No explicit filter set, include file.
  888. return True
  889. for filter in file_include:
  890. if filter == '.':
  891. # Another special case: include current directory and thus all files.
  892. return True
  893. if os.path.commonprefix((filter, file)) == filter:
  894. # Included in directory or direct file match.
  895. return True
  896. # Check for wildcard match *with* allowing * to match /, i.e.
  897. # src/*.c does match src/foobar/*.c. That's not how it is done elsewhere
  898. # when passing the filtering to "git archive", but it is unclear what
  899. # the intended semantic is (the comment on file_exclude that "append a * wildcard
  900. # at the end" to match the full content of a directories implies that
  901. # slashes are indeed not special), so here we simply do what's easy to
  902. # implement in Python.
  903. logger.debug('fnmatch(%s, %s)' % (file, filter))
  904. if fnmatch.fnmatchcase(file, filter):
  905. return True
  906. return False
  907. def exclude_file(file):
  908. for filter in file_exclude:
  909. if fnmatch.fnmatchcase(file, filter):
  910. return True
  911. return False
  912. def file_filter(files):
  913. '''Clean up file list so that only included files remain.'''
  914. index = 0
  915. while index < len(files):
  916. file = files[index]
  917. if not include_file(file) or exclude_file(file):
  918. del files[index]
  919. else:
  920. index += 1
  921. # Generate the revision list.
  922. logger.info("Analyzing commits from %s..." % name)
  923. top_revision = revision or branch
  924. if not check_rev_branch(name, ldir, top_revision, branch):
  925. sys.exit(1)
  926. last_revision = repo['last_revision']
  927. rev_list_args = "--full-history --sparse --topo-order --reverse"
  928. if not last_revision:
  929. logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name)
  930. rev_list_args = rev_list_args + ' ' + top_revision
  931. else:
  932. if not check_rev_branch(name, ldir, last_revision, branch):
  933. sys.exit(1)
  934. rev_list_args = "%s %s..%s" % (rev_list_args, last_revision, top_revision)
  935. # By definition, the current HEAD contains the latest imported
  936. # commit of each component. We use that as initial mapping even
  937. # though the commits do not match exactly because
  938. # a) it always works (in contrast to find_revs, which relies on special
  939. # commit messages)
  940. # b) it is faster than find_revs, which will only be called on demand
  941. # and can be skipped entirely in most cases
  942. # c) last but not least, the combined history looks nicer when all
  943. # new commits are rooted in the same merge commit
  944. old2new_revs[last_revision] = head
  945. # We care about all commits (--full-history and --sparse) and
  946. # we want reconstruct the topology and thus do not care
  947. # about ordering by time (--topo-order). We ask for the ones
  948. # we need to import first to be listed first (--reverse).
  949. revs = runcmd("git rev-list %s" % rev_list_args, **largs).split()
  950. logger.debug("To be imported: %s" % revs)
  951. # Now 'revs' contains all revisions reachable from the top revision.
  952. # All revisions derived from the 'last_revision' definitely are new,
  953. # whereas the others may or may not have been imported before. For
  954. # a linear history in the component, that second set will be empty.
  955. # To distinguish between them, we also get the shorter list
  956. # of revisions starting at the ancestor.
  957. if last_revision:
  958. ancestor_revs = runcmd("git rev-list --ancestry-path %s" % rev_list_args, **largs).split()
  959. else:
  960. ancestor_revs = []
  961. logger.debug("Ancestors: %s" % ancestor_revs)
  962. # Now import each revision.
  963. logger.info("Importing commits from %s..." % name)
  964. def import_rev(rev):
  965. global scanned_revs
  966. # If it is part of the new commits, we definitely need
  967. # to import it. Otherwise we need to check, we might have
  968. # imported it before. If it was imported and we merely
  969. # fail to find it because commit messages did not track
  970. # the mapping, then we end up importing it again. So
  971. # combined repos using "updating with history" really should
  972. # enable the "From ... rev:" commit header modifications.
  973. if rev not in ancestor_revs and rev not in old2new_revs and not scanned_revs:
  974. logger.debug("Revision %s triggers log analysis." % rev)
  975. find_revs(old2new_revs, head)
  976. scanned_revs = True
  977. new_rev = old2new_revs.get(rev, None)
  978. if new_rev:
  979. return new_rev
  980. # If the commit is not in the original list of revisions
  981. # to be imported, then it must be a parent of one of those
  982. # commits and it was skipped during earlier imports or not
  983. # found. Importing such merge commits leads to very ugly
  984. # history (long cascade of merge commits which all point
  985. # to to older commits) when switching from "update via
  986. # patches" to "update with history".
  987. #
  988. # We can avoid importing merge commits if all non-merge commits
  989. # reachable from it were already imported. In that case we
  990. # can root the new commits in the current head revision.
  991. def is_imported(prev):
  992. parents = runcmd("git show --no-patch --pretty=format:%P " + prev, **largs).split()
  993. if len(parents) > 1:
  994. for p in parents:
  995. if not is_imported(p):
  996. logger.debug("Must import %s because %s is not imported." % (rev, p))
  997. return False
  998. return True
  999. elif prev in old2new_revs:
  1000. return True
  1001. else:
  1002. logger.debug("Must import %s because %s is not imported." % (rev, prev))
  1003. return False
  1004. if rev not in revs and is_imported(rev):
  1005. old2new_revs[rev] = head
  1006. return head
  1007. # Need to import rev. Collect some information about it.
  1008. logger.debug("Importing %s" % rev)
  1009. (parents, author_name, author_email, author_timestamp, body) = \
  1010. runcmd("git show --no-patch --pretty=format:%P%x00%an%x00%ae%x00%at%x00%B " + rev, **largs).split(chr(0))
  1011. parents = parents.split()
  1012. if parents:
  1013. # Arbitrarily pick the first parent as base. It may or may not have
  1014. # been imported before. For example, if the parent is a merge commit
  1015. # and previously the combined repository used patching as update
  1016. # method, then the actual merge commit parent never was imported.
  1017. # To cover this, We recursively import parents.
  1018. parent = parents[0]
  1019. new_parent = import_rev(parent)
  1020. # Clean index and working tree. TODO: can we combine this and the
  1021. # next into one command with less file IO?
  1022. # "git reset --hard" does not work, it changes HEAD of the parent
  1023. # repo, which we wanted to avoid. Probably need to keep
  1024. # track of the rev that corresponds to the index and use apply_commit().
  1025. runcmd("git rm -q --ignore-unmatch -rf .", **wargs)
  1026. # Update index and working tree to match the parent.
  1027. runcmd("git checkout -q -f %s ." % new_parent, **wargs)
  1028. else:
  1029. parent = None
  1030. # Clean index and working tree.
  1031. runcmd("git rm -q --ignore-unmatch -rf .", **wargs)
  1032. # Modify index and working tree such that it mirrors the commit.
  1033. apply_commit(parent, rev, largs, wargs, dest_dir, file_filter=file_filter)
  1034. # Now commit.
  1035. new_tree = runcmd("git write-tree", **wargs).strip()
  1036. env = copy.deepcopy(wenv)
  1037. env['GIT_AUTHOR_NAME'] = author_name
  1038. env['GIT_AUTHOR_EMAIL'] = author_email
  1039. env['GIT_AUTHOR_DATE'] = author_timestamp
  1040. if hook:
  1041. # Need to turn the verbatim commit message into something resembling a patch header
  1042. # for the hook.
  1043. with tempfile.NamedTemporaryFile(mode='wt', delete=False) as patch:
  1044. patch.write('Subject: [PATCH] ')
  1045. patch.write(body)
  1046. patch.write('\n---\n')
  1047. patch.close()
  1048. runcmd([hook, patch.name, rev, name])
  1049. with open(patch.name) as f:
  1050. body = f.read()[len('Subject: [PATCH] '):][:-len('\n---\n')]
  1051. # We can skip non-merge commits that did not change any files. Those are typically
  1052. # the result of file filtering, although they could also have been introduced
  1053. # intentionally upstream, in which case we drop some information here.
  1054. if len(parents) == 1:
  1055. parent_rev = import_rev(parents[0])
  1056. old_tree = runcmd("git show -s --pretty=format:%T " + parent_rev, **wargs).strip()
  1057. commit = old_tree != new_tree
  1058. if not commit:
  1059. new_rev = parent_rev
  1060. else:
  1061. commit = True
  1062. if commit:
  1063. new_rev = runcmd("git commit-tree".split() + add_p([import_rev(p) for p in parents]) +
  1064. ["-m", body, new_tree],
  1065. env=env).strip()
  1066. old2new_revs[rev] = new_rev
  1067. return new_rev
  1068. if revs:
  1069. for rev in revs:
  1070. import_rev(rev)
  1071. # Remember how to update our current head. New components get added,
  1072. # updated components get the delta between current head and the updated component
  1073. # applied.
  1074. additional_heads[old2new_revs[revs[-1]]] = head if repo['last_revision'] else None
  1075. repo['last_revision'] = revs[-1]
  1076. # Now construct the final merge commit. We create the tree by
  1077. # starting with the head and applying the changes from each
  1078. # components imported head revision.
  1079. if additional_heads:
  1080. runcmd("git reset --hard", **wargs)
  1081. for rev, base in additional_heads.items():
  1082. apply_commit(base, rev, wargs, wargs, None)
  1083. # Commit with all component branches as parents as well as the previous head.
  1084. logger.info("Writing final merge commit...")
  1085. msg = conf_commit_msg(conf, components)
  1086. new_tree = runcmd("git write-tree", **wargs).strip()
  1087. new_rev = runcmd("git commit-tree".split() +
  1088. add_p([head] + list(additional_heads.keys())) +
  1089. ["-m", msg, new_tree],
  1090. **wargs).strip()
  1091. # And done! This is the first time we change the HEAD in the actual work tree.
  1092. runcmd("git reset --hard %s" % new_rev)
  1093. # Update and stage the (potentially modified)
  1094. # combo-layer.conf, but do not commit separately.
  1095. for name in repos:
  1096. repo = conf.repos[name]
  1097. rev = repo['last_revision']
  1098. conf.update(name, "last_revision", rev)
  1099. if commit_conf_file(conf, components, False):
  1100. # Must augment the previous commit.
  1101. runcmd("git commit --amend -C HEAD")
  1102. scanned_revs = False
  1103. def find_revs(old2new, head):
  1104. '''Construct mapping from original commit hash to commit hash in
  1105. combined repo by looking at the commit messages. Depends on the
  1106. "From ... rev: ..." convention.'''
  1107. logger.info("Analyzing log messages to find previously imported commits...")
  1108. num_known = len(old2new)
  1109. log = runcmd("git log --grep='From .* rev: [a-fA-F0-9][a-fA-F0-9]*' --pretty=format:%H%x00%B%x00 " + head).split(chr(0))
  1110. regex = re.compile(r'From .* rev: ([a-fA-F0-9]+)')
  1111. for new_rev, body in zip(*[iter(log)]* 2):
  1112. # Use the last one, in the unlikely case there are more than one.
  1113. rev = regex.findall(body)[-1]
  1114. if rev not in old2new:
  1115. old2new[rev] = new_rev.strip()
  1116. logger.info("Found %d additional commits, leading to: %s" % (len(old2new) - num_known, old2new))
  1117. def apply_commit(parent, rev, largs, wargs, dest_dir, file_filter=None):
  1118. '''Compare revision against parent, remove files deleted in the
  1119. commit, re-write new or modified ones. Moves them into dest_dir.
  1120. Optionally filters files.
  1121. '''
  1122. if not dest_dir:
  1123. dest_dir = "."
  1124. # -r recurses into sub-directories, given is the full overview of
  1125. # what changed. We do not care about copy/edits or renames, so we
  1126. # can disable those with --no-renames (but we still parse them,
  1127. # because it was not clear from git documentation whether C and M
  1128. # lines can still occur).
  1129. logger.debug("Applying changes between %s and %s in %s" % (parent, rev, largs["destdir"]))
  1130. delete = []
  1131. update = []
  1132. if parent:
  1133. # Apply delta.
  1134. changes = runcmd("git diff-tree --no-commit-id --no-renames --name-status -r --raw -z %s %s" % (parent, rev), **largs).split(chr(0))
  1135. for status, name in zip(*[iter(changes)]*2):
  1136. if status[0] in "ACMRT":
  1137. update.append(name)
  1138. elif status[0] in "D":
  1139. delete.append(name)
  1140. else:
  1141. logger.error("Unknown status %s of file %s in revision %s" % (status, name, rev))
  1142. sys.exit(1)
  1143. else:
  1144. # Copy all files.
  1145. update.extend(runcmd("git ls-tree -r --name-only -z %s" % rev, **largs).split(chr(0)))
  1146. # Include/exclude files as define in the component config.
  1147. # Both updated and deleted file lists get filtered, because it might happen
  1148. # that a file gets excluded, pulled from a different component, and then the
  1149. # excluded file gets deleted. In that case we must keep the copy.
  1150. if file_filter:
  1151. file_filter(update)
  1152. file_filter(delete)
  1153. # We export into a tar archive here and extract with tar because it is simple (no
  1154. # need to implement file and symlink writing ourselves) and gives us some degree
  1155. # of parallel IO. The downside is that we have to pass the list of files via
  1156. # command line parameters - hopefully there will never be too many at once.
  1157. if update:
  1158. target = os.path.join(wargs["destdir"], dest_dir)
  1159. if not os.path.isdir(target):
  1160. os.makedirs(target)
  1161. quoted_target = pipes.quote(target)
  1162. # os.sysconf('SC_ARG_MAX') is lying: running a command with
  1163. # string length 629343 already failed with "Argument list too
  1164. # long" although SC_ARG_MAX = 2097152. "man execve" explains
  1165. # the limitations, but those are pretty complicated. So here
  1166. # we just hard-code a fixed value which is more likely to work.
  1167. max_cmdsize = 64 * 1024
  1168. while update:
  1169. quoted_args = []
  1170. unquoted_args = []
  1171. cmdsize = 100 + len(quoted_target)
  1172. while update:
  1173. quoted_next = pipes.quote(update[0])
  1174. size_next = len(quoted_next) + len(dest_dir) + 1
  1175. logger.debug('cmdline length %d + %d < %d?' % (cmdsize, size_next, os.sysconf('SC_ARG_MAX')))
  1176. if cmdsize + size_next < max_cmdsize:
  1177. quoted_args.append(quoted_next)
  1178. unquoted_args.append(update.pop(0))
  1179. cmdsize += size_next
  1180. else:
  1181. logger.debug('Breaking the cmdline at length %d' % cmdsize)
  1182. break
  1183. logger.debug('Final cmdline length %d / %d' % (cmdsize, os.sysconf('SC_ARG_MAX')))
  1184. cmd = "git archive %s %s | tar -C %s -xf -" % (rev, ' '.join(quoted_args), quoted_target)
  1185. logger.debug('First cmdline length %d' % len(cmd))
  1186. runcmd(cmd, **largs)
  1187. cmd = "git add -f".split() + [os.path.join(dest_dir, x) for x in unquoted_args]
  1188. logger.debug('Second cmdline length %d' % reduce(lambda x, y: x + len(y), cmd, 0))
  1189. runcmd(cmd, **wargs)
  1190. if delete:
  1191. for path in delete:
  1192. if dest_dir:
  1193. path = os.path.join(dest_dir, path)
  1194. runcmd("git rm -f --ignore-unmatch".split() + [os.path.join(dest_dir, x) for x in delete], **wargs)
  1195. def action_error(conf, args):
  1196. logger.info("invalid action %s" % args[0])
  1197. actions = {
  1198. "init": action_init,
  1199. "update": action_update,
  1200. "pull": action_pull,
  1201. "splitpatch": action_splitpatch,
  1202. }
  1203. def main():
  1204. parser = optparse.OptionParser(
  1205. version = "Combo Layer Repo Tool version %s" % __version__,
  1206. usage = """%prog [options] action
  1207. Create and update a combination layer repository from multiple component repositories.
  1208. Action:
  1209. init initialise the combo layer repo
  1210. update [components] get patches from component repos and apply them to the combo repo
  1211. pull [components] just pull component repos only
  1212. splitpatch [commit] generate commit patch and split per component, default commit is HEAD""")
  1213. parser.add_option("-c", "--conf", help = "specify the config file (conf/combo-layer.conf is the default).",
  1214. action = "store", dest = "conffile", default = "conf/combo-layer.conf")
  1215. parser.add_option("-i", "--interactive", help = "interactive mode, user can edit the patch list and patches",
  1216. action = "store_true", dest = "interactive", default = False)
  1217. parser.add_option("-D", "--debug", help = "output debug information",
  1218. action = "store_true", dest = "debug", default = False)
  1219. parser.add_option("-n", "--no-pull", help = "skip pulling component repos during update",
  1220. action = "store_true", dest = "nopull", default = False)
  1221. parser.add_option("--hard-reset",
  1222. help = "instead of pull do fetch and hard-reset in component repos",
  1223. action = "store_true", dest = "hard_reset", default = False)
  1224. parser.add_option("-H", "--history", help = "import full history of components during init",
  1225. action = "store_true", default = False)
  1226. options, args = parser.parse_args(sys.argv)
  1227. # Dispatch to action handler
  1228. if len(args) == 1:
  1229. logger.error("No action specified, exiting")
  1230. parser.print_help()
  1231. elif args[1] not in actions:
  1232. logger.error("Unsupported action %s, exiting\n" % (args[1]))
  1233. parser.print_help()
  1234. elif not os.path.exists(options.conffile):
  1235. logger.error("No valid config file, exiting\n")
  1236. parser.print_help()
  1237. else:
  1238. if options.debug:
  1239. logger.setLevel(logging.DEBUG)
  1240. confdata = Configuration(options)
  1241. initmode = (args[1] == 'init')
  1242. confdata.sanity_check(initmode)
  1243. actions.get(args[1], action_error)(confdata, args[1:])
  1244. if __name__ == "__main__":
  1245. try:
  1246. ret = main()
  1247. except Exception:
  1248. ret = 1
  1249. import traceback
  1250. traceback.print_exc()
  1251. sys.exit(ret)