combo-layer 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598
  1. #!/usr/bin/env python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. #
  5. # Copyright 2011 Intel Corporation
  6. # Authored-by: Yu Ke <ke.yu@intel.com>
  7. # Paul Eggleton <paul.eggleton@intel.com>
  8. # Richard Purdie <richard.purdie@intel.com>
  9. #
  10. # This program is free software; you can redistribute it and/or modify
  11. # it under the terms of the GNU General Public License version 2 as
  12. # published by the Free Software Foundation.
  13. #
  14. # This program is distributed in the hope that it will be useful,
  15. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17. # GNU General Public License for more details.
  18. #
  19. # You should have received a copy of the GNU General Public License along
  20. # with this program; if not, write to the Free Software Foundation, Inc.,
  21. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  22. import os, sys
  23. import optparse
  24. import logging
  25. import subprocess
  26. import ConfigParser
  27. import re
  28. __version__ = "0.2.1"
  29. def logger_create():
  30. logger = logging.getLogger("")
  31. loggerhandler = logging.StreamHandler()
  32. loggerhandler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s","%H:%M:%S"))
  33. logger.addHandler(loggerhandler)
  34. logger.setLevel(logging.INFO)
  35. return logger
  36. logger = logger_create()
  37. def get_current_branch(repodir=None):
  38. try:
  39. if not os.path.exists(os.path.join(repodir if repodir else '', ".git")):
  40. # Repo not created yet (i.e. during init) so just assume master
  41. return "master"
  42. branchname = runcmd("git symbolic-ref HEAD 2>/dev/null", repodir).strip()
  43. if branchname.startswith("refs/heads/"):
  44. branchname = branchname[11:]
  45. return branchname
  46. except subprocess.CalledProcessError:
  47. return ""
  48. class Configuration(object):
  49. """
  50. Manages the configuration
  51. For an example config file, see combo-layer.conf.example
  52. """
  53. def __init__(self, options):
  54. for key, val in options.__dict__.items():
  55. setattr(self, key, val)
  56. def readsection(parser, section, repo):
  57. for (name, value) in parser.items(section):
  58. if value.startswith("@"):
  59. self.repos[repo][name] = eval(value.strip("@"))
  60. else:
  61. self.repos[repo][name] = value
  62. logger.debug("Loading config file %s" % self.conffile)
  63. self.parser = ConfigParser.ConfigParser()
  64. with open(self.conffile) as f:
  65. self.parser.readfp(f)
  66. self.repos = {}
  67. for repo in self.parser.sections():
  68. self.repos[repo] = {}
  69. readsection(self.parser, repo, repo)
  70. # Load local configuration, if available
  71. self.localconffile = None
  72. self.localparser = None
  73. self.combobranch = None
  74. if self.conffile.endswith('.conf'):
  75. lcfile = self.conffile.replace('.conf', '-local.conf')
  76. if os.path.exists(lcfile):
  77. # Read combo layer branch
  78. self.combobranch = get_current_branch()
  79. logger.debug("Combo layer branch is %s" % self.combobranch)
  80. self.localconffile = lcfile
  81. logger.debug("Loading local config file %s" % self.localconffile)
  82. self.localparser = ConfigParser.ConfigParser()
  83. with open(self.localconffile) as f:
  84. self.localparser.readfp(f)
  85. for section in self.localparser.sections():
  86. if '|' in section:
  87. sectionvals = section.split('|')
  88. repo = sectionvals[0]
  89. if sectionvals[1] != self.combobranch:
  90. continue
  91. else:
  92. repo = section
  93. if repo in self.repos:
  94. readsection(self.localparser, section, repo)
  95. def update(self, repo, option, value, initmode=False):
  96. if self.localparser:
  97. parser = self.localparser
  98. section = "%s|%s" % (repo, self.combobranch)
  99. conffile = self.localconffile
  100. if initmode and not parser.has_section(section):
  101. parser.add_section(section)
  102. else:
  103. parser = self.parser
  104. section = repo
  105. conffile = self.conffile
  106. parser.set(section, option, value)
  107. with open(conffile, "w") as f:
  108. parser.write(f)
  109. def sanity_check(self, initmode=False):
  110. required_options=["src_uri", "local_repo_dir", "dest_dir", "last_revision"]
  111. if initmode:
  112. required_options.remove("last_revision")
  113. msg = ""
  114. missing_options = []
  115. for name in self.repos:
  116. for option in required_options:
  117. if option not in self.repos[name]:
  118. msg = "%s\nOption %s is not defined for component %s" %(msg, option, name)
  119. missing_options.append(option)
  120. if msg != "":
  121. logger.error("configuration file %s has the following error: %s" % (self.conffile,msg))
  122. if self.localconffile and 'last_revision' in missing_options:
  123. logger.error("local configuration file %s may be missing configuration for combo branch %s" % (self.localconffile, self.combobranch))
  124. sys.exit(1)
  125. # filterdiff is required by action_splitpatch, so check its availability
  126. if subprocess.call("which filterdiff > /dev/null 2>&1", shell=True) != 0:
  127. logger.error("ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)")
  128. sys.exit(1)
  129. def runcmd(cmd,destdir=None,printerr=True):
  130. """
  131. execute command, raise CalledProcessError if fail
  132. return output if succeed
  133. """
  134. logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
  135. out = os.tmpfile()
  136. try:
  137. subprocess.check_call(cmd, stdout=out, stderr=out, cwd=destdir, shell=True)
  138. except subprocess.CalledProcessError,e:
  139. out.seek(0)
  140. if printerr:
  141. logger.error("%s" % out.read())
  142. raise e
  143. out.seek(0)
  144. output = out.read()
  145. logger.debug("output: %s" % output )
  146. return output
  147. def action_init(conf, args):
  148. """
  149. Clone component repositories
  150. Check git is initialised; if not, copy initial data from component repos
  151. """
  152. for name in conf.repos:
  153. ldir = conf.repos[name]['local_repo_dir']
  154. if not os.path.exists(ldir):
  155. logger.info("cloning %s to %s" %(conf.repos[name]['src_uri'], ldir))
  156. subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True)
  157. if not os.path.exists(".git"):
  158. runcmd("git init")
  159. for name in conf.repos:
  160. repo = conf.repos[name]
  161. ldir = repo['local_repo_dir']
  162. branch = repo.get('branch', "master")
  163. lastrev = repo.get('last_revision', None)
  164. if lastrev and lastrev != "HEAD":
  165. initialrev = lastrev
  166. if branch:
  167. if not check_rev_branch(name, ldir, lastrev, branch):
  168. sys.exit(1)
  169. logger.info("Copying data from %s at specified revision %s..." % (name, lastrev))
  170. else:
  171. lastrev = None
  172. initialrev = branch
  173. logger.info("Copying data from %s..." % name)
  174. dest_dir = repo['dest_dir']
  175. if dest_dir and dest_dir != ".":
  176. extract_dir = os.path.join(os.getcwd(), dest_dir)
  177. os.makedirs(extract_dir)
  178. else:
  179. extract_dir = os.getcwd()
  180. file_filter = repo.get('file_filter', "")
  181. runcmd("git archive %s | tar -x -C %s %s" % (initialrev, extract_dir, file_filter), ldir)
  182. if not lastrev:
  183. lastrev = runcmd("git rev-parse %s" % initialrev, ldir).strip()
  184. conf.update(name, "last_revision", lastrev, initmode=True)
  185. runcmd("git add .")
  186. if conf.localconffile:
  187. localadded = True
  188. try:
  189. runcmd("git rm --cached %s" % conf.localconffile, printerr=False)
  190. except subprocess.CalledProcessError:
  191. localadded = False
  192. if localadded:
  193. localrelpath = os.path.relpath(conf.localconffile)
  194. runcmd("grep -q %s .gitignore || echo %s >> .gitignore" % (localrelpath, localrelpath))
  195. runcmd("git add .gitignore")
  196. logger.info("Added local configuration file %s to .gitignore", localrelpath)
  197. logger.info("Initial combo layer repository data has been created; please make any changes if desired and then use 'git commit' to make the initial commit.")
  198. else:
  199. logger.info("Repository already initialised, nothing to do.")
  200. def check_repo_clean(repodir):
  201. """
  202. check if the repo is clean
  203. exit if repo is dirty
  204. """
  205. output=runcmd("git status --porcelain", repodir)
  206. r = re.compile('\?\? patch-.*/')
  207. dirtyout = [item for item in output.splitlines() if not r.match(item)]
  208. if dirtyout:
  209. logger.error("git repo %s is dirty, please fix it first", repodir)
  210. sys.exit(1)
  211. def check_patch(patchfile):
  212. f = open(patchfile)
  213. ln = f.readline()
  214. of = None
  215. in_patch = False
  216. beyond_msg = False
  217. pre_buf = ''
  218. while ln:
  219. if not beyond_msg:
  220. if ln == '---\n':
  221. if not of:
  222. break
  223. in_patch = False
  224. beyond_msg = True
  225. elif ln.startswith('--- '):
  226. # We have a diff in the commit message
  227. in_patch = True
  228. if not of:
  229. print('WARNING: %s contains a diff in its commit message, indenting to avoid failure during apply' % patchfile)
  230. of = open(patchfile + '.tmp', 'w')
  231. of.write(pre_buf)
  232. pre_buf = ''
  233. elif in_patch and not ln[0] in '+-@ \n\r':
  234. in_patch = False
  235. if of:
  236. if in_patch:
  237. of.write(' ' + ln)
  238. else:
  239. of.write(ln)
  240. else:
  241. pre_buf += ln
  242. ln = f.readline()
  243. f.close()
  244. if of:
  245. of.close()
  246. os.rename(patchfile + '.tmp', patchfile)
  247. def drop_to_shell(workdir=None):
  248. shell = os.environ.get('SHELL', 'bash')
  249. print('Dropping to shell "%s"\n' \
  250. 'When you are finished, run the following to continue:\n' \
  251. ' exit -- continue to apply the patches\n' \
  252. ' exit 1 -- abort\n' % shell);
  253. ret = subprocess.call([shell], cwd=workdir)
  254. if ret != 0:
  255. print "Aborting"
  256. return False
  257. else:
  258. return True
  259. def check_rev_branch(component, repodir, rev, branch):
  260. try:
  261. actualbranch = runcmd("git branch --contains %s" % rev, repodir, printerr=False)
  262. except subprocess.CalledProcessError as e:
  263. if e.returncode == 129:
  264. actualbranch = ""
  265. else:
  266. raise
  267. if not actualbranch:
  268. logger.error("%s: specified revision %s is invalid!" % (component, rev))
  269. return False
  270. branches = []
  271. branchlist = actualbranch.split("\n")
  272. for b in branchlist:
  273. branches.append(b.strip().split(' ')[-1])
  274. if branch not in branches:
  275. logger.error("%s: specified revision %s is not on specified branch %s!" % (component, rev, branch))
  276. return False
  277. return True
  278. def get_repos(conf, args):
  279. repos = []
  280. if len(args) > 1:
  281. for arg in args[1:]:
  282. if arg.startswith('-'):
  283. break
  284. else:
  285. repos.append(arg)
  286. for repo in repos:
  287. if not repo in conf.repos:
  288. logger.error("Specified component '%s' not found in configuration" % repo)
  289. sys.exit(0)
  290. if not repos:
  291. repos = conf.repos
  292. return repos
  293. def action_pull(conf, args):
  294. """
  295. update the component repos only
  296. """
  297. repos = get_repos(conf, args)
  298. # make sure all repos are clean
  299. for name in repos:
  300. check_repo_clean(conf.repos[name]['local_repo_dir'])
  301. for name in repos:
  302. repo = conf.repos[name]
  303. ldir = repo['local_repo_dir']
  304. branch = repo.get('branch', "master")
  305. runcmd("git checkout %s" % branch, ldir)
  306. logger.info("git pull for component repo %s in %s ..." % (name, ldir))
  307. output=runcmd("git pull", ldir)
  308. logger.info(output)
  309. def action_update(conf, args):
  310. """
  311. update the component repos
  312. generate the patch list
  313. apply the generated patches
  314. """
  315. repos = get_repos(conf, args)
  316. # make sure combo repo is clean
  317. check_repo_clean(os.getcwd())
  318. import uuid
  319. patch_dir = "patch-%s" % uuid.uuid4()
  320. os.mkdir(patch_dir)
  321. # Step 1: update the component repos
  322. if conf.nopull:
  323. logger.info("Skipping pull (-n)")
  324. else:
  325. action_pull(conf, args)
  326. for name in repos:
  327. repo = conf.repos[name]
  328. ldir = repo['local_repo_dir']
  329. dest_dir = repo['dest_dir']
  330. branch = repo.get('branch', "master")
  331. repo_patch_dir = os.path.join(os.getcwd(), patch_dir, name)
  332. # Step 2: generate the patch list and store to patch dir
  333. logger.info("Generating patches from %s..." % name)
  334. if dest_dir != ".":
  335. prefix = "--src-prefix=a/%s/ --dst-prefix=b/%s/" % (dest_dir, dest_dir)
  336. else:
  337. prefix = ""
  338. if repo['last_revision'] == "":
  339. logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name)
  340. patch_cmd_range = "--root %s" % branch
  341. rev_cmd_range = branch
  342. else:
  343. if not check_rev_branch(name, ldir, repo['last_revision'], branch):
  344. sys.exit(1)
  345. patch_cmd_range = "%s..%s" % (repo['last_revision'], branch)
  346. rev_cmd_range = patch_cmd_range
  347. file_filter = repo.get('file_filter',"")
  348. patch_cmd = "git format-patch -N %s --output-directory %s %s -- %s" % \
  349. (prefix,repo_patch_dir, patch_cmd_range, file_filter)
  350. output = runcmd(patch_cmd, ldir)
  351. logger.debug("generated patch set:\n%s" % output)
  352. patchlist = output.splitlines()
  353. rev_cmd = 'git rev-list --no-merges ' + rev_cmd_range
  354. revlist = runcmd(rev_cmd, ldir).splitlines()
  355. # Step 3: Call repo specific hook to adjust patch
  356. if 'hook' in repo:
  357. # hook parameter is: ./hook patchpath revision reponame
  358. count=len(revlist)-1
  359. for patch in patchlist:
  360. runcmd("%s %s %s %s" % (repo['hook'], patch, revlist[count], name))
  361. count=count-1
  362. # Step 4: write patch list and revision list to file, for user to edit later
  363. patchlist_file = os.path.join(os.getcwd(), patch_dir, "patchlist-%s" % name)
  364. repo['patchlist'] = patchlist_file
  365. f = open(patchlist_file, 'w')
  366. count=len(revlist)-1
  367. for patch in patchlist:
  368. f.write("%s %s\n" % (patch, revlist[count]))
  369. check_patch(os.path.join(patch_dir, patch))
  370. count=count-1
  371. f.close()
  372. # Step 5: invoke bash for user to edit patch and patch list
  373. if conf.interactive:
  374. print('You may now edit the patch and patch list in %s\n' \
  375. 'For example, you can remove unwanted patch entries from patchlist-*, so that they will be not applied later' % patch_dir);
  376. if not drop_to_shell(patch_dir):
  377. sys.exit(0)
  378. # Step 6: apply the generated and revised patch
  379. apply_patchlist(conf, repos)
  380. runcmd("rm -rf %s" % patch_dir)
  381. # Step 7: commit the updated config file if it's being tracked
  382. relpath = os.path.relpath(conf.conffile)
  383. try:
  384. output = runcmd("git status --porcelain %s" % relpath, printerr=False)
  385. except:
  386. # Outside the repository
  387. output = None
  388. if output:
  389. logger.info("Committing updated configuration file")
  390. if output.lstrip().startswith("M"):
  391. runcmd('git commit -m "Automatic commit to update last_revision" %s' % relpath)
  392. def apply_patchlist(conf, repos):
  393. """
  394. apply the generated patch list to combo repo
  395. """
  396. for name in repos:
  397. repo = conf.repos[name]
  398. lastrev = repo["last_revision"]
  399. prevrev = lastrev
  400. # Get non-blank lines from patch list file
  401. patchlist = []
  402. if os.path.exists(repo['patchlist']) or not conf.interactive:
  403. # Note: we want this to fail here if the file doesn't exist and we're not in
  404. # interactive mode since the file should exist in this case
  405. with open(repo['patchlist']) as f:
  406. for line in f:
  407. line = line.rstrip()
  408. if line:
  409. patchlist.append(line)
  410. if patchlist:
  411. logger.info("Applying patches from %s..." % name)
  412. linecount = len(patchlist)
  413. i = 1
  414. for line in patchlist:
  415. patchfile = line.split()[0]
  416. lastrev = line.split()[1]
  417. patchdisp = os.path.relpath(patchfile)
  418. if os.path.getsize(patchfile) == 0:
  419. logger.info("(skipping %d/%d %s - no changes)" % (i, linecount, patchdisp))
  420. else:
  421. cmd = "git am --keep-cr -s -p1 %s" % patchfile
  422. logger.info("Applying %d/%d: %s" % (i, linecount, patchdisp))
  423. try:
  424. runcmd(cmd)
  425. except subprocess.CalledProcessError:
  426. logger.info('Running "git am --abort" to cleanup repo')
  427. runcmd("git am --abort")
  428. logger.error('"%s" failed' % cmd)
  429. logger.info("Please manually apply patch %s" % patchdisp)
  430. logger.info("Note: if you exit and continue applying without manually applying the patch, it will be skipped")
  431. if not drop_to_shell():
  432. if prevrev != repo['last_revision']:
  433. conf.update(name, "last_revision", prevrev)
  434. sys.exit(0)
  435. prevrev = lastrev
  436. i += 1
  437. else:
  438. logger.info("No patches to apply from %s" % name)
  439. ldir = conf.repos[name]['local_repo_dir']
  440. branch = conf.repos[name].get('branch', "master")
  441. lastrev = runcmd("git rev-parse %s" % branch, ldir).strip()
  442. if lastrev != repo['last_revision']:
  443. conf.update(name, "last_revision", lastrev)
  444. def action_splitpatch(conf, args):
  445. """
  446. generate the commit patch and
  447. split the patch per repo
  448. """
  449. logger.debug("action_splitpatch")
  450. if len(args) > 1:
  451. commit = args[1]
  452. else:
  453. commit = "HEAD"
  454. patchdir = "splitpatch-%s" % commit
  455. if not os.path.exists(patchdir):
  456. os.mkdir(patchdir)
  457. # filerange_root is for the repo whose dest_dir is root "."
  458. # and it should be specified by excluding all other repo dest dir
  459. # like "-x repo1 -x repo2 -x repo3 ..."
  460. filerange_root = ""
  461. for name in conf.repos:
  462. dest_dir = conf.repos[name]['dest_dir']
  463. if dest_dir != ".":
  464. filerange_root = '%s -x "%s/*"' % (filerange_root, dest_dir)
  465. for name in conf.repos:
  466. dest_dir = conf.repos[name]['dest_dir']
  467. patch_filename = "%s/%s.patch" % (patchdir, name)
  468. if dest_dir == ".":
  469. cmd = "git format-patch -n1 --stdout %s^..%s | filterdiff -p1 %s > %s" % (commit, commit, filerange_root, patch_filename)
  470. else:
  471. cmd = "git format-patch --no-prefix -n1 --stdout %s^..%s -- %s > %s" % (commit, commit, dest_dir, patch_filename)
  472. runcmd(cmd)
  473. # Detect empty patches (including those produced by filterdiff above
  474. # that contain only preamble text)
  475. if os.path.getsize(patch_filename) == 0 or runcmd("filterdiff %s" % patch_filename) == "":
  476. os.remove(patch_filename)
  477. logger.info("(skipping %s - no changes)", name)
  478. else:
  479. logger.info(patch_filename)
  480. def action_error(conf, args):
  481. logger.info("invalid action %s" % args[0])
  482. actions = {
  483. "init": action_init,
  484. "update": action_update,
  485. "pull": action_pull,
  486. "splitpatch": action_splitpatch,
  487. }
  488. def main():
  489. parser = optparse.OptionParser(
  490. version = "Combo Layer Repo Tool version %s" % __version__,
  491. usage = """%prog [options] action
  492. Create and update a combination layer repository from multiple component repositories.
  493. Action:
  494. init initialise the combo layer repo
  495. update [components] get patches from component repos and apply them to the combo repo
  496. pull [components] just pull component repos only
  497. splitpatch [commit] generate commit patch and split per component, default commit is HEAD""")
  498. parser.add_option("-c", "--conf", help = "specify the config file (conf/combo-layer.conf is the default).",
  499. action = "store", dest = "conffile", default = "conf/combo-layer.conf")
  500. parser.add_option("-i", "--interactive", help = "interactive mode, user can edit the patch list and patches",
  501. action = "store_true", dest = "interactive", default = False)
  502. parser.add_option("-D", "--debug", help = "output debug information",
  503. action = "store_true", dest = "debug", default = False)
  504. parser.add_option("-n", "--no-pull", help = "skip pulling component repos during update",
  505. action = "store_true", dest = "nopull", default = False)
  506. options, args = parser.parse_args(sys.argv)
  507. # Dispatch to action handler
  508. if len(args) == 1:
  509. logger.error("No action specified, exiting")
  510. parser.print_help()
  511. elif args[1] not in actions:
  512. logger.error("Unsupported action %s, exiting\n" % (args[1]))
  513. parser.print_help()
  514. elif not os.path.exists(options.conffile):
  515. logger.error("No valid config file, exiting\n")
  516. parser.print_help()
  517. else:
  518. if options.debug:
  519. logger.setLevel(logging.DEBUG)
  520. confdata = Configuration(options)
  521. initmode = (args[1] == 'init')
  522. confdata.sanity_check(initmode)
  523. actions.get(args[1], action_error)(confdata, args[1:])
  524. if __name__ == "__main__":
  525. try:
  526. ret = main()
  527. except Exception:
  528. ret = 1
  529. import traceback
  530. traceback.print_exc(5)
  531. sys.exit(ret)