gitsm.py 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215
  1. # ex:ts=4:sw=4:sts=4:et
  2. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  3. """
  4. BitBake 'Fetch' git submodules implementation
  5. Inherits from and extends the Git fetcher to retrieve submodules of a git repository
  6. after cloning.
  7. SRC_URI = "gitsm://<see Git fetcher for syntax>"
  8. See the Git fetcher, git://, for usage documentation.
  9. NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe.
  10. """
  11. # Copyright (C) 2013 Richard Purdie
  12. #
  13. # SPDX-License-Identifier: GPL-2.0-only
  14. #
  15. import os
  16. import bb
  17. import copy
  18. from bb.fetch2.git import Git
  19. from bb.fetch2 import runfetchcmd
  20. from bb.fetch2 import logger
  21. from bb.fetch2 import Fetch
  22. from bb.fetch2 import BBFetchException
  23. class GitSM(Git):
  24. def supports(self, ud, d):
  25. """
  26. Check to see if a given url can be fetched with git.
  27. """
  28. return ud.type in ['gitsm']
  29. def process_submodules(self, ud, workdir, function, d):
  30. """
  31. Iterate over all of the submodules in this repository and execute
  32. the 'function' for each of them.
  33. """
  34. submodules = []
  35. paths = {}
  36. revision = {}
  37. uris = {}
  38. subrevision = {}
  39. def parse_gitmodules(gitmodules):
  40. modules = {}
  41. module = ""
  42. for line in gitmodules.splitlines():
  43. if line.startswith('[submodule'):
  44. module = line.split('"')[1]
  45. modules[module] = {}
  46. elif module and line.strip().startswith('path'):
  47. path = line.split('=')[1].strip()
  48. modules[module]['path'] = path
  49. elif module and line.strip().startswith('url'):
  50. url = line.split('=')[1].strip()
  51. modules[module]['url'] = url
  52. return modules
  53. # Collect the defined submodules, and their attributes
  54. for name in ud.names:
  55. try:
  56. gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
  57. except:
  58. # No submodules to update
  59. continue
  60. for m, md in parse_gitmodules(gitmodules).items():
  61. try:
  62. module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
  63. except:
  64. # If the command fails, we don't have a valid file to check. If it doesn't
  65. # fail -- it still might be a failure, see next check...
  66. module_hash = ""
  67. if not module_hash:
  68. logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m)
  69. continue
  70. submodules.append(m)
  71. paths[m] = md['path']
  72. revision[m] = ud.revisions[name]
  73. uris[m] = md['url']
  74. subrevision[m] = module_hash.split()[2]
  75. # Convert relative to absolute uri based on parent uri
  76. if uris[m].startswith('..'):
  77. newud = copy.copy(ud)
  78. newud.path = os.path.realpath(os.path.join(newud.path, uris[m]))
  79. uris[m] = Git._get_repo_url(self, newud)
  80. for module in submodules:
  81. # Translate the module url into a SRC_URI
  82. if "://" in uris[module]:
  83. # Properly formated URL already
  84. proto = uris[module].split(':', 1)[0]
  85. url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
  86. else:
  87. if ":" in uris[module]:
  88. # Most likely an SSH style reference
  89. proto = "ssh"
  90. if ":/" in uris[module]:
  91. # Absolute reference, easy to convert..
  92. url = "gitsm://" + uris[module].replace(':/', '/', 1)
  93. else:
  94. # Relative reference, no way to know if this is right!
  95. logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module]))
  96. url = "gitsm://" + uris[module].replace(':', '/', 1)
  97. else:
  98. # This has to be a file reference
  99. proto = "file"
  100. url = "gitsm://" + uris[module]
  101. url += ';protocol=%s' % proto
  102. url += ";name=%s" % module
  103. url += ";subpath=%s" % paths[module]
  104. ld = d.createCopy()
  105. # Not necessary to set SRC_URI, since we're passing the URI to
  106. # Fetch.
  107. #ld.setVar('SRC_URI', url)
  108. ld.setVar('SRCREV_%s' % module, subrevision[module])
  109. # Workaround for issues with SRCPV/SRCREV_FORMAT errors
  110. # error refer to 'multiple' repositories. Only the repository
  111. # in the original SRC_URI actually matters...
  112. ld.setVar('SRCPV', d.getVar('SRCPV'))
  113. ld.setVar('SRCREV_FORMAT', module)
  114. function(ud, url, module, paths[module], ld)
  115. return submodules != []
  116. def need_update(self, ud, d):
  117. if Git.need_update(self, ud, d):
  118. return True
  119. try:
  120. # Check for the nugget dropped by the download operation
  121. known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \
  122. (ud.basecmd), d, workdir=ud.clonedir)
  123. if ud.revisions[ud.names[0]] not in known_srcrevs.split():
  124. return True
  125. except bb.fetch2.FetchError:
  126. # No srcrev nuggets, so this is new and needs to be updated
  127. return True
  128. return False
  129. def download(self, ud, d):
  130. def download_submodule(ud, url, module, modpath, d):
  131. url += ";bareclone=1;nobranch=1"
  132. # Is the following still needed?
  133. #url += ";nocheckout=1"
  134. try:
  135. newfetch = Fetch([url], d, cache=False)
  136. newfetch.download()
  137. # Drop a nugget to add each of the srcrevs we've fetched (used by need_update)
  138. runfetchcmd("%s config --add bitbake.srcrev %s" % \
  139. (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
  140. except Exception as e:
  141. logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
  142. raise
  143. Git.download(self, ud, d)
  144. self.process_submodules(ud, ud.clonedir, download_submodule, d)
  145. def unpack(self, ud, destdir, d):
  146. def unpack_submodules(ud, url, module, modpath, d):
  147. url += ";bareclone=1;nobranch=1"
  148. # Figure out where we clone over the bare submodules...
  149. if ud.bareclone:
  150. repo_conf = ud.destdir
  151. else:
  152. repo_conf = os.path.join(ud.destdir, '.git')
  153. try:
  154. newfetch = Fetch([url], d, cache=False)
  155. newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', modpath)))
  156. except Exception as e:
  157. logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
  158. raise
  159. local_path = newfetch.localpath(url)
  160. # Correct the submodule references to the local download version...
  161. runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir)
  162. if ud.shallow:
  163. runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
  164. # Ensure the submodule repository is NOT set to bare, since we're checking it out...
  165. try:
  166. runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath))
  167. except:
  168. logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', modpath))
  169. raise
  170. Git.unpack(self, ud, destdir, d)
  171. ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
  172. if not ud.bareclone and ret:
  173. # Run submodule update, this sets up the directories -- without touching the config
  174. runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)