gitsm.py 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227
  1. # ex:ts=4:sw=4:sts=4:et
  2. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  3. """
  4. BitBake 'Fetch' git submodules implementation
  5. Inherits from and extends the Git fetcher to retrieve submodules of a git repository
  6. after cloning.
  7. SRC_URI = "gitsm://<see Git fetcher for syntax>"
  8. See the Git fetcher, git://, for usage documentation.
  9. NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe.
  10. """
  11. # Copyright (C) 2013 Richard Purdie
  12. #
  13. # SPDX-License-Identifier: GPL-2.0-only
  14. #
  15. # This program is free software; you can redistribute it and/or modify
  16. # it under the terms of the GNU General Public License version 2 as
  17. # published by the Free Software Foundation.
  18. #
  19. # This program is distributed in the hope that it will be useful,
  20. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  21. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  22. # GNU General Public License for more details.
  23. #
  24. # You should have received a copy of the GNU General Public License along
  25. # with this program; if not, write to the Free Software Foundation, Inc.,
  26. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  27. import os
  28. import bb
  29. import copy
  30. from bb.fetch2.git import Git
  31. from bb.fetch2 import runfetchcmd
  32. from bb.fetch2 import logger
  33. from bb.fetch2 import Fetch
  34. from bb.fetch2 import BBFetchException
  35. class GitSM(Git):
  36. def supports(self, ud, d):
  37. """
  38. Check to see if a given url can be fetched with git.
  39. """
  40. return ud.type in ['gitsm']
  41. def process_submodules(self, ud, workdir, function, d):
  42. """
  43. Iterate over all of the submodules in this repository and execute
  44. the 'function' for each of them.
  45. """
  46. submodules = []
  47. paths = {}
  48. revision = {}
  49. uris = {}
  50. subrevision = {}
  51. def parse_gitmodules(gitmodules):
  52. modules = {}
  53. module = ""
  54. for line in gitmodules.splitlines():
  55. if line.startswith('[submodule'):
  56. module = line.split('"')[1]
  57. modules[module] = {}
  58. elif module and line.strip().startswith('path'):
  59. path = line.split('=')[1].strip()
  60. modules[module]['path'] = path
  61. elif module and line.strip().startswith('url'):
  62. url = line.split('=')[1].strip()
  63. modules[module]['url'] = url
  64. return modules
  65. # Collect the defined submodules, and their attributes
  66. for name in ud.names:
  67. try:
  68. gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
  69. except:
  70. # No submodules to update
  71. continue
  72. for m, md in parse_gitmodules(gitmodules).items():
  73. try:
  74. module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
  75. except:
  76. # If the command fails, we don't have a valid file to check. If it doesn't
  77. # fail -- it still might be a failure, see next check...
  78. module_hash = ""
  79. if not module_hash:
  80. logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m)
  81. continue
  82. submodules.append(m)
  83. paths[m] = md['path']
  84. revision[m] = ud.revisions[name]
  85. uris[m] = md['url']
  86. subrevision[m] = module_hash.split()[2]
  87. # Convert relative to absolute uri based on parent uri
  88. if uris[m].startswith('..'):
  89. newud = copy.copy(ud)
  90. newud.path = os.path.realpath(os.path.join(newud.path, uris[m]))
  91. uris[m] = Git._get_repo_url(self, newud)
  92. for module in submodules:
  93. # Translate the module url into a SRC_URI
  94. if "://" in uris[module]:
  95. # Properly formated URL already
  96. proto = uris[module].split(':', 1)[0]
  97. url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
  98. else:
  99. if ":" in uris[module]:
  100. # Most likely an SSH style reference
  101. proto = "ssh"
  102. if ":/" in uris[module]:
  103. # Absolute reference, easy to convert..
  104. url = "gitsm://" + uris[module].replace(':/', '/', 1)
  105. else:
  106. # Relative reference, no way to know if this is right!
  107. logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module]))
  108. url = "gitsm://" + uris[module].replace(':', '/', 1)
  109. else:
  110. # This has to be a file reference
  111. proto = "file"
  112. url = "gitsm://" + uris[module]
  113. url += ';protocol=%s' % proto
  114. url += ";name=%s" % module
  115. url += ";subpath=%s" % paths[module]
  116. ld = d.createCopy()
  117. # Not necessary to set SRC_URI, since we're passing the URI to
  118. # Fetch.
  119. #ld.setVar('SRC_URI', url)
  120. ld.setVar('SRCREV_%s' % module, subrevision[module])
  121. # Workaround for issues with SRCPV/SRCREV_FORMAT errors
  122. # error refer to 'multiple' repositories. Only the repository
  123. # in the original SRC_URI actually matters...
  124. ld.setVar('SRCPV', d.getVar('SRCPV'))
  125. ld.setVar('SRCREV_FORMAT', module)
  126. function(ud, url, module, paths[module], ld)
  127. return submodules != []
  128. def need_update(self, ud, d):
  129. if Git.need_update(self, ud, d):
  130. return True
  131. try:
  132. # Check for the nugget dropped by the download operation
  133. known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \
  134. (ud.basecmd), d, workdir=ud.clonedir)
  135. if ud.revisions[ud.names[0]] not in known_srcrevs.split():
  136. return True
  137. except bb.fetch2.FetchError:
  138. # No srcrev nuggets, so this is new and needs to be updated
  139. return True
  140. return False
  141. def download(self, ud, d):
  142. def download_submodule(ud, url, module, modpath, d):
  143. url += ";bareclone=1;nobranch=1"
  144. # Is the following still needed?
  145. #url += ";nocheckout=1"
  146. try:
  147. newfetch = Fetch([url], d, cache=False)
  148. newfetch.download()
  149. # Drop a nugget to add each of the srcrevs we've fetched (used by need_update)
  150. runfetchcmd("%s config --add bitbake.srcrev %s" % \
  151. (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
  152. except Exception as e:
  153. logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
  154. raise
  155. Git.download(self, ud, d)
  156. self.process_submodules(ud, ud.clonedir, download_submodule, d)
  157. def unpack(self, ud, destdir, d):
  158. def unpack_submodules(ud, url, module, modpath, d):
  159. url += ";bareclone=1;nobranch=1"
  160. # Figure out where we clone over the bare submodules...
  161. if ud.bareclone:
  162. repo_conf = ud.destdir
  163. else:
  164. repo_conf = os.path.join(ud.destdir, '.git')
  165. try:
  166. newfetch = Fetch([url], d, cache=False)
  167. newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', modpath)))
  168. except Exception as e:
  169. logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
  170. raise
  171. local_path = newfetch.localpath(url)
  172. # Correct the submodule references to the local download version...
  173. runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir)
  174. if ud.shallow:
  175. runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
  176. # Ensure the submodule repository is NOT set to bare, since we're checking it out...
  177. try:
  178. runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath))
  179. except:
  180. logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', modpath))
  181. raise
  182. Git.unpack(self, ud, destdir, d)
  183. ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
  184. if not ud.bareclone and ret:
  185. # Run submodule update, this sets up the directories -- without touching the config
  186. runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)