dl-wrapper 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. #!/usr/bin/env bash
  2. # This script is a wrapper to the other download backends.
  3. # Its role is to ensure atomicity when saving downloaded files
  4. # back to BR2_DL_DIR, and not clutter BR2_DL_DIR with partial,
  5. # failed downloads.
  6. #
  7. # Call it with -h to see some help.
  8. # To avoid cluttering BR2_DL_DIR, we download to a trashable
  9. # location, namely in $(BUILD_DIR).
  10. # Then, we move the downloaded file to a temporary file in the
  11. # same directory as the final output file.
  12. # This allows us to finally atomically rename it to its final
  13. # name.
  14. # If anything goes wrong, we just remove all the temporaries
  15. # created so far.
  16. # We want to catch any unexpected failure, and exit immediately.
  17. set -e
  18. export BR_BACKEND_DL_GETOPTS=":hc:o:n:N:H:ru:qf:e"
  19. main() {
  20. local OPT OPTARG
  21. local backend output hfile recurse quiet rc
  22. local -a uris
  23. # Parse our options; anything after '--' is for the backend
  24. while getopts ":hc:o:n:N:H:rf:u:q" OPT; do
  25. case "${OPT}" in
  26. h) help; exit 0;;
  27. c) cset="${OPTARG}";;
  28. o) output="${OPTARG}";;
  29. n) raw_base_name="${OPTARG}";;
  30. N) base_name="${OPTARG}";;
  31. H) hfile="${OPTARG}";;
  32. r) recurse="-r";;
  33. f) filename="${OPTARG}";;
  34. u) uris+=( "${OPTARG}" );;
  35. q) quiet="-q";;
  36. :) error "option '%s' expects a mandatory argument\n" "${OPTARG}";;
  37. \?) error "unknown option '%s'\n" "${OPTARG}";;
  38. esac
  39. done
  40. # Forget our options, and keep only those for the backend
  41. shift $((OPTIND-1))
  42. if [ -z "${output}" ]; then
  43. error "no output specified, use -o\n"
  44. fi
  45. # If the output file already exists and:
  46. # - there's no .hash file: do not download it again and exit promptly
  47. # - matches all its hashes: do not download it again and exit promptly
  48. # - fails at least one of its hashes: force a re-download
  49. # - there's no hash (but a .hash file): consider it a hard error
  50. if [ -e "${output}" ]; then
  51. if support/download/check-hash ${quiet} "${hfile}" "${output}" "${output##*/}"; then
  52. exit 0
  53. elif [ ${?} -ne 2 ]; then
  54. # Do not remove the file, otherwise it might get re-downloaded
  55. # from a later location (i.e. primary -> upstream -> mirror).
  56. # Do not print a message, check-hash already did.
  57. exit 1
  58. fi
  59. rm -f "${output}"
  60. warn "Re-downloading '%s'...\n" "${output##*/}"
  61. fi
  62. # Look through all the uris that we were given to download the package
  63. # source
  64. download_and_check=0
  65. rc=1
  66. for uri in "${uris[@]}"; do
  67. backend=${uri%+*}
  68. case "${backend}" in
  69. git|svn|cvs|bzr|file|scp|hg) ;;
  70. *) backend="wget" ;;
  71. esac
  72. uri=${uri#*+}
  73. urlencode=${backend#*|}
  74. # urlencode must be "urlencode"
  75. [ "${urlencode}" != "urlencode" ] && urlencode=""
  76. # tmpd is a temporary directory in which backends may store
  77. # intermediate by-products of the download.
  78. # tmpf is the file in which the backends should put the downloaded
  79. # content.
  80. # tmpd is located in $(BUILD_DIR), so as not to clutter the (precious)
  81. # $(BR2_DL_DIR)
  82. # We let the backends create tmpf, so they are able to set whatever
  83. # permission bits they want (although we're only really interested in
  84. # the executable bit.)
  85. tmpd="$(mktemp -d "${BUILD_DIR}/.${output##*/}.XXXXXX")"
  86. tmpf="${tmpd}/output"
  87. # Helpers expect to run in a directory that is *really* trashable, so
  88. # they are free to create whatever files and/or sub-dirs they might need.
  89. # Doing the 'cd' here rather than in all backends is easier.
  90. cd "${tmpd}"
  91. # If the backend fails, we can just remove the content of the temporary
  92. # directory to remove all the cruft it may have left behind, and try
  93. # the next URI until it succeeds. Once out of URI to try, we need to
  94. # cleanup and exit.
  95. if ! "${OLDPWD}/support/download/${backend}" \
  96. $([ -n "${urlencode}" ] && printf %s '-e') \
  97. -c "${cset}" \
  98. -n "${raw_base_name}" \
  99. -N "${raw_name}" \
  100. -f "${filename}" \
  101. -u "${uri}" \
  102. -o "${tmpf}" \
  103. ${quiet} ${recurse} "${@}"
  104. then
  105. # cd back to keep path coherence
  106. cd "${OLDPWD}"
  107. rm -rf "${tmpd}"
  108. continue
  109. fi
  110. # cd back to free the temp-dir, so we can remove it later
  111. cd "${OLDPWD}"
  112. # Check if the downloaded file is sane, and matches the stored hashes
  113. # for that file
  114. if support/download/check-hash ${quiet} "${hfile}" "${tmpf}" "${output##*/}"; then
  115. rc=0
  116. else
  117. if [ ${?} -ne 3 ]; then
  118. rm -rf "${tmpd}"
  119. continue
  120. fi
  121. # the hash file exists and there was no hash to check the file
  122. # against
  123. rc=1
  124. fi
  125. download_and_check=1
  126. break
  127. done
  128. # We tried every URI possible, none seems to work or to check against the
  129. # available hash. *ABORT MISSION*
  130. if [ "${download_and_check}" -eq 0 ]; then
  131. rm -rf "${tmpd}"
  132. exit 1
  133. fi
  134. # tmp_output is in the same directory as the final output, so we can
  135. # later move it atomically.
  136. tmp_output="$(mktemp "${output}.XXXXXX")"
  137. # 'mktemp' creates files with 'go=-rwx', so the files are not accessible
  138. # to users other than the one doing the download (and root, of course).
  139. # This can be problematic when a shared BR2_DL_DIR is used by different
  140. # users (e.g. on a build server), where all users may write to the shared
  141. # location, since other users would not be allowed to read the files
  142. # another user downloaded.
  143. # So, we restore the 'go' access rights to a more sensible value, while
  144. # still abiding by the current user's umask. We must do that before the
  145. # final 'mv', so just do it now.
  146. # Some backends (cp and scp) may create executable files, so we need to
  147. # carry the executable bit if needed.
  148. [ -x "${tmpf}" ] && new_mode=755 || new_mode=644
  149. new_mode=$(printf "%04o" $((0${new_mode} & ~0$(umask))))
  150. chmod ${new_mode} "${tmp_output}"
  151. # We must *not* unlink tmp_output, otherwise there is a small window
  152. # during which another download process may create the same tmp_output
  153. # name (very, very unlikely; but not impossible.)
  154. # Using 'cp' is not reliable, since 'cp' may unlink the destination file
  155. # if it is unable to open it with O_WRONLY|O_TRUNC; see:
  156. # http://pubs.opengroup.org/onlinepubs/9699919799/utilities/cp.html
  157. # Since the destination filesystem can be anything, it might not support
  158. # O_TRUNC, so 'cp' would unlink it first.
  159. # Use 'cat' and append-redirection '>>' to save to the final location,
  160. # since that is the only way we can be 100% sure of the behaviour.
  161. if ! cat "${tmpf}" >>"${tmp_output}"; then
  162. rm -rf "${tmpd}" "${tmp_output}"
  163. exit 1
  164. fi
  165. rm -rf "${tmpd}"
  166. # tmp_output and output are on the same filesystem, so POSIX guarantees
  167. # that 'mv' is atomic, because it then uses rename() that POSIX mandates
  168. # to be atomic, see:
  169. # http://pubs.opengroup.org/onlinepubs/9699919799/functions/rename.html
  170. if ! mv -f "${tmp_output}" "${output}"; then
  171. rm -f "${tmp_output}"
  172. exit 1
  173. fi
  174. return ${rc}
  175. }
  176. help() {
  177. cat <<_EOF_
  178. NAME
  179. ${my_name} - download wrapper for Buildroot
  180. SYNOPSIS
  181. ${my_name} [OPTION]... -- [BACKEND OPTION]...
  182. DESCRIPTION
  183. Wrapper script around different download mechanisms. Ensures that
  184. concurrent downloads do not conflict, that partial downloads are
  185. properly evicted without leaving temporary files, and that access
  186. rights are maintained.
  187. -h This help text.
  188. -u URIs
  189. The URI to get the file from, the URI must respect the format given in
  190. the example.
  191. You may give as many '-u URI' as you want, the script will stop at the
  192. frist successful download.
  193. Example: backend+URI; git+http://example.com or http+http://example.com
  194. -o FILE
  195. Store the downloaded archive in FILE.
  196. -H FILE
  197. Use FILE to read hashes from, and check them against the downloaded
  198. archive.
  199. Exit status:
  200. 0 if OK
  201. !0 in case of error
  202. ENVIRONMENT
  203. BUILD_DIR
  204. The path to Buildroot's build dir
  205. _EOF_
  206. }
  207. trace() { local msg="${1}"; shift; printf "%s: ${msg}" "${my_name}" "${@}"; }
  208. warn() { trace "${@}" >&2; }
  209. errorN() { local ret="${1}"; shift; warn "${@}"; exit ${ret}; }
  210. error() { errorN 1 "${@}"; }
  211. my_name="${0##*/}"
  212. main "${@}"