git 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216
  1. #!/usr/bin/env bash
  2. # NOTE: if the output of this backend has to change (e.g. we change what gets
  3. # included in the archive (e.g. LFS), or we change the format of the archive
  4. # (e.g. tar options, compression ratio or method)), we MUST update the format
  5. # version in the variable BR_FMT_VERSION_git, in package/pkg-download.mk.
  6. # We want to catch any unexpected failure, and exit immediately
  7. set -E
  8. # Download helper for git, to be called from the download wrapper script
  9. #
  10. # Options:
  11. # -q Be quiet.
  12. # -r Clone and archive sub-modules.
  13. # -o FILE Generate archive in FILE.
  14. # -u URI Clone from repository at URI.
  15. # -c CSET Use changeset CSET.
  16. # -n NAME Use basename NAME.
  17. #
  18. # Environment:
  19. # GIT : the git command to call
  20. . "${0%/*}/helpers"
  21. # Save our path and options in case we need to call ourselves again
  22. myname="${0}"
  23. declare -a OPTS=("${@}")
  24. # This function is called when an error occurs. Its job is to attempt a
  25. # clone from scratch (only once!) in case the git tree is borked, or in
  26. # case an unexpected and unsupported situation arises with submodules
  27. # or uncommitted stuff (e.g. if the user manually mucked around in the
  28. # git cache).
  29. _on_error() {
  30. local ret=${?}
  31. printf "Detected a corrupted git cache.\n" >&2
  32. if ${BR_GIT_BACKEND_FIRST_FAULT:-false}; then
  33. printf "This is the second time in a row; bailing out\n" >&2
  34. exit ${ret}
  35. fi
  36. export BR_GIT_BACKEND_FIRST_FAULT=true
  37. printf "Removing it and starting afresh.\n" >&2
  38. popd >/dev/null
  39. rm -rf "${git_cache}"
  40. exec "${myname}" "${OPTS[@]}" || exit ${ret}
  41. }
  42. quiet=
  43. recurse=0
  44. while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
  45. case "${OPT}" in
  46. q) quiet=-q; exec >/dev/null;;
  47. r) recurse=1;;
  48. o) output="${OPTARG}";;
  49. u) uri="${OPTARG}";;
  50. c) cset="${OPTARG}";;
  51. d) dl_dir="${OPTARG}";;
  52. n) basename="${OPTARG}";;
  53. :) printf "option '%s' expects a mandatory argument\n" "${OPTARG}"; exit 1;;
  54. \?) printf "unknown option '%s'\n" "${OPTARG}" >&2; exit 1;;
  55. esac
  56. done
  57. shift $((OPTIND-1)) # Get rid of our options
  58. # Create and cd into the directory that will contain the local git cache
  59. git_cache="${dl_dir}/git"
  60. mkdir -p "${git_cache}"
  61. pushd "${git_cache}" >/dev/null
  62. # Any error now should try to recover
  63. trap _on_error ERR
  64. # Caller needs to single-quote its arguments to prevent them from
  65. # being expanded a second time (in case there are spaces in them)
  66. _git() {
  67. if [ -z "${quiet}" ]; then
  68. printf '%s ' GIT_DIR="${git_cache}/.git" ${GIT} "${@}"; printf '\n'
  69. fi
  70. _plain_git "$@"
  71. }
  72. # Note: please keep command below aligned with what is printed above
  73. _plain_git() {
  74. eval GIT_DIR="${git_cache}/.git" ${GIT} "${@}"
  75. }
  76. # Create a warning file, that the user should not use the git cache.
  77. # It's ours. Our precious.
  78. cat <<-_EOF_ >"${dl_dir}/git.readme"
  79. IMPORTANT NOTE!
  80. The git tree located in this directory is for the exclusive use
  81. by Buildroot, which uses it as a local cache to reduce bandwidth
  82. usage.
  83. Buildroot *will* trash any changes in that tree whenever it needs
  84. to use it. Buildroot may even remove it in case it detects the
  85. repository may have been damaged or corrupted.
  86. Do *not* work in that directory; your changes will eventually get
  87. lost. Do *not* even use it as a remote, or as the source for new
  88. worktrees; your commits will eventually get lost.
  89. _EOF_
  90. # Initialise a repository in the git cache. If the repository already
  91. # existed, this is a noop, unless the repository was broken, in which
  92. # case this magically restores it to working conditions. In the latter
  93. # case, we might be missing blobs, but that's not a problem: we'll
  94. # fetch what we need later anyway.
  95. #
  96. # We can still go through the wrapper, because 'init' does not use the
  97. # path pointed to by GIT_DIR, but really uses the directory passed as
  98. # argument.
  99. _git init .
  100. # Ensure the repo has an origin (in case a previous run was killed).
  101. if ! _plain_git remote |grep -q -E '^origin$'; then
  102. _git remote add origin "'${uri}'"
  103. fi
  104. _git remote set-url origin "'${uri}'"
  105. printf "Fetching all references\n"
  106. _git fetch origin
  107. _git fetch origin -t
  108. # Try to get the special refs exposed by some forges (pull-requests for
  109. # github, changes for gerrit...). There is no easy way to know whether
  110. # the cset the user passed us is such a special ref or a tag or a sha1
  111. # or whatever else. We'll eventually fail at checking out that cset,
  112. # below, if there is an issue anyway. Since most of the cset we're gonna
  113. # have to clone are not such special refs, consign the output to oblivion
  114. # so as not to alarm unsuspecting users, but still trace it as a warning.
  115. if ! _git fetch origin "'${cset}:${cset}'" >/dev/null 2>&1; then
  116. printf "Could not fetch special ref '%s'; assuming it is not special.\n" "${cset}"
  117. fi
  118. # Check that the changeset does exist. If it does not, re-cloning from
  119. # scratch won't help, so we don't want to trash the repository for a
  120. # missing commit. We just exit without going through the ERR trap.
  121. if ! _git rev-parse --quiet --verify "'${cset}^{commit}'" >/dev/null 2>&1; then
  122. printf "Commit '%s' does not exist in this repository.\n" "${cset}"
  123. exit 1
  124. fi
  125. # The new cset we want to checkout might have different submodules, or
  126. # have sub-dirs converted to/from a submodule. So we would need to
  127. # deregister _current_ submodules before we checkout.
  128. #
  129. # Using "git submodule deinit --all" would remove all the files for
  130. # all submodules, including the corresponding .git files or directories.
  131. # However, it was only introduced with git-1.8.3, which is too recent
  132. # for some enterprise-grade distros.
  133. #
  134. # So, we fall-back to just removing all submodules directories. We do
  135. # not need to be recursive, as removing a submodule will de-facto remove
  136. # its own submodules.
  137. #
  138. # For recent git versions, the repository for submodules is stored
  139. # inside the repository of the super repository, so the following will
  140. # only remove the working copies of submodules, effectively caching the
  141. # submodules.
  142. #
  143. # For older versions however, the repository is stored in the .git/ of
  144. # the submodule directory, so the following will effectively remove the
  145. # the working copy as well as the repository, which means submodules
  146. # will not be cached for older versions.
  147. #
  148. cmd='printf "Deregistering submodule \"%s\"\n" "${path}" && cd .. && rm -rf "${path##*/}"'
  149. _git submodule --quiet foreach "'${cmd}'"
  150. # Checkout the required changeset, so that we can update the required
  151. # submodules.
  152. _git checkout -f -q "'${cset}'"
  153. # Get rid of now-untracked directories (in case a git operation was
  154. # interrupted in a previous run, or to get rid of empty directories
  155. # that were parents of submodules removed above).
  156. _git clean -ffdx
  157. # Get date of commit to generate a reproducible archive.
  158. # %ci is ISO 8601, so it's fully qualified, with TZ and all.
  159. date="$( _plain_git log -1 --pretty=format:%ci )"
  160. # There might be submodules, so fetch them.
  161. if [ ${recurse} -eq 1 ]; then
  162. _git submodule update --init --recursive
  163. # Older versions of git will store the absolute path of the git tree
  164. # in the .git of submodules, while newer versions just use relative
  165. # paths. Detect and fix the older variants to use relative paths, so
  166. # that the archives are reproducible across a wider range of git
  167. # versions. However, we can't do that if git is too old and uses
  168. # full repositories for submodules.
  169. cmd='printf "%s\n" "${path}/"'
  170. for module_dir in $( _plain_git submodule --quiet foreach "'${cmd}'" ); do
  171. [ -f "${module_dir}/.git" ] || continue
  172. relative_dir="$( sed -r -e 's,/+,/,g; s,[^/]+/,../,g' <<<"${module_dir}" )"
  173. sed -r -i -e "s:^gitdir\: $(pwd)/:gitdir\: "${relative_dir}":" "${module_dir}/.git"
  174. done
  175. fi
  176. popd >/dev/null
  177. # Generate the archive.
  178. # We do not want the .git dir; we keep other .git files, in case they are the
  179. # only files in their directory.
  180. # The .git dir would generate non reproducible tarballs as it depends on
  181. # the state of the remote server. It also would generate large tarballs
  182. # (gigabytes for some linux trees) when a full clone took place.
  183. mk_tar_gz "${git_cache}" "${basename}" "${date}" "${output}" ".git/*"