gitsm.py 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213
  1. """
  2. BitBake 'Fetch' git submodules implementation
  3. Inherits from and extends the Git fetcher to retrieve submodules of a git repository
  4. after cloning.
  5. SRC_URI = "gitsm://<see Git fetcher for syntax>"
  6. See the Git fetcher, git://, for usage documentation.
  7. NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe.
  8. """
  9. # Copyright (C) 2013 Richard Purdie
  10. #
  11. # SPDX-License-Identifier: GPL-2.0-only
  12. #
  13. import os
  14. import bb
  15. import copy
  16. from bb.fetch2.git import Git
  17. from bb.fetch2 import runfetchcmd
  18. from bb.fetch2 import logger
  19. from bb.fetch2 import Fetch
  20. from bb.fetch2 import BBFetchException
  21. class GitSM(Git):
  22. def supports(self, ud, d):
  23. """
  24. Check to see if a given url can be fetched with git.
  25. """
  26. return ud.type in ['gitsm']
  27. def process_submodules(self, ud, workdir, function, d):
  28. """
  29. Iterate over all of the submodules in this repository and execute
  30. the 'function' for each of them.
  31. """
  32. submodules = []
  33. paths = {}
  34. revision = {}
  35. uris = {}
  36. subrevision = {}
  37. def parse_gitmodules(gitmodules):
  38. modules = {}
  39. module = ""
  40. for line in gitmodules.splitlines():
  41. if line.startswith('[submodule'):
  42. module = line.split('"')[1]
  43. modules[module] = {}
  44. elif module and line.strip().startswith('path'):
  45. path = line.split('=')[1].strip()
  46. modules[module]['path'] = path
  47. elif module and line.strip().startswith('url'):
  48. url = line.split('=')[1].strip()
  49. modules[module]['url'] = url
  50. return modules
  51. # Collect the defined submodules, and their attributes
  52. for name in ud.names:
  53. try:
  54. gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
  55. except:
  56. # No submodules to update
  57. continue
  58. for m, md in parse_gitmodules(gitmodules).items():
  59. try:
  60. module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
  61. except:
  62. # If the command fails, we don't have a valid file to check. If it doesn't
  63. # fail -- it still might be a failure, see next check...
  64. module_hash = ""
  65. if not module_hash:
  66. logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m)
  67. continue
  68. submodules.append(m)
  69. paths[m] = md['path']
  70. revision[m] = ud.revisions[name]
  71. uris[m] = md['url']
  72. subrevision[m] = module_hash.split()[2]
  73. # Convert relative to absolute uri based on parent uri
  74. if uris[m].startswith('..'):
  75. newud = copy.copy(ud)
  76. newud.path = os.path.realpath(os.path.join(newud.path, uris[m]))
  77. uris[m] = Git._get_repo_url(self, newud)
  78. for module in submodules:
  79. # Translate the module url into a SRC_URI
  80. if "://" in uris[module]:
  81. # Properly formated URL already
  82. proto = uris[module].split(':', 1)[0]
  83. url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
  84. else:
  85. if ":" in uris[module]:
  86. # Most likely an SSH style reference
  87. proto = "ssh"
  88. if ":/" in uris[module]:
  89. # Absolute reference, easy to convert..
  90. url = "gitsm://" + uris[module].replace(':/', '/', 1)
  91. else:
  92. # Relative reference, no way to know if this is right!
  93. logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module]))
  94. url = "gitsm://" + uris[module].replace(':', '/', 1)
  95. else:
  96. # This has to be a file reference
  97. proto = "file"
  98. url = "gitsm://" + uris[module]
  99. url += ';protocol=%s' % proto
  100. url += ";name=%s" % module
  101. url += ";subpath=%s" % paths[module]
  102. ld = d.createCopy()
  103. # Not necessary to set SRC_URI, since we're passing the URI to
  104. # Fetch.
  105. #ld.setVar('SRC_URI', url)
  106. ld.setVar('SRCREV_%s' % module, subrevision[module])
  107. # Workaround for issues with SRCPV/SRCREV_FORMAT errors
  108. # error refer to 'multiple' repositories. Only the repository
  109. # in the original SRC_URI actually matters...
  110. ld.setVar('SRCPV', d.getVar('SRCPV'))
  111. ld.setVar('SRCREV_FORMAT', module)
  112. function(ud, url, module, paths[module], ld)
  113. return submodules != []
  114. def need_update(self, ud, d):
  115. if Git.need_update(self, ud, d):
  116. return True
  117. try:
  118. # Check for the nugget dropped by the download operation
  119. known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \
  120. (ud.basecmd), d, workdir=ud.clonedir)
  121. if ud.revisions[ud.names[0]] not in known_srcrevs.split():
  122. return True
  123. except bb.fetch2.FetchError:
  124. # No srcrev nuggets, so this is new and needs to be updated
  125. return True
  126. return False
  127. def download(self, ud, d):
  128. def download_submodule(ud, url, module, modpath, d):
  129. url += ";bareclone=1;nobranch=1"
  130. # Is the following still needed?
  131. #url += ";nocheckout=1"
  132. try:
  133. newfetch = Fetch([url], d, cache=False)
  134. newfetch.download()
  135. # Drop a nugget to add each of the srcrevs we've fetched (used by need_update)
  136. runfetchcmd("%s config --add bitbake.srcrev %s" % \
  137. (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
  138. except Exception as e:
  139. logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
  140. raise
  141. Git.download(self, ud, d)
  142. self.process_submodules(ud, ud.clonedir, download_submodule, d)
  143. def unpack(self, ud, destdir, d):
  144. def unpack_submodules(ud, url, module, modpath, d):
  145. url += ";bareclone=1;nobranch=1"
  146. # Figure out where we clone over the bare submodules...
  147. if ud.bareclone:
  148. repo_conf = ud.destdir
  149. else:
  150. repo_conf = os.path.join(ud.destdir, '.git')
  151. try:
  152. newfetch = Fetch([url], d, cache=False)
  153. newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', modpath)))
  154. except Exception as e:
  155. logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
  156. raise
  157. local_path = newfetch.localpath(url)
  158. # Correct the submodule references to the local download version...
  159. runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir)
  160. if ud.shallow:
  161. runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
  162. # Ensure the submodule repository is NOT set to bare, since we're checking it out...
  163. try:
  164. runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath))
  165. except:
  166. logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', modpath))
  167. raise
  168. Git.unpack(self, ud, destdir, d)
  169. ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
  170. if not ud.bareclone and ret:
  171. # Run submodule update, this sets up the directories -- without touching the config
  172. runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)