dl-wrapper 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219
  1. #!/usr/bin/env bash
  2. # This script is a wrapper to the other download backends.
  3. # Its role is to ensure atomicity when saving downloaded files
  4. # back to BR2_DL_DIR, and not clutter BR2_DL_DIR with partial,
  5. # failed downloads.
  6. # To avoid cluttering BR2_DL_DIR, we download to a trashable
  7. # location, namely in $(BUILD_DIR).
  8. # Then, we move the downloaded file to a temporary file in the
  9. # same directory as the final output file.
  10. # This allows us to finally atomically rename it to its final
  11. # name.
  12. # If anything goes wrong, we just remove all the temporaries
  13. # created so far.
  14. # We want to catch any unexpected failure, and exit immediately.
  15. set -e
  16. export BR_BACKEND_DL_GETOPTS=":hc:d:o:n:N:H:ru:qf:e"
  17. main() {
  18. local OPT OPTARG
  19. local backend output hfile recurse quiet rc
  20. local -a uris
  21. # Parse our options; anything after '--' is for the backend
  22. while getopts ":c:d:D:o:n:N:H:rf:u:q" OPT; do
  23. case "${OPT}" in
  24. c) cset="${OPTARG}";;
  25. d) dl_dir="${OPTARG}";;
  26. D) old_dl_dir="${OPTARG}";;
  27. o) output="${OPTARG}";;
  28. n) raw_base_name="${OPTARG}";;
  29. N) base_name="${OPTARG}";;
  30. H) hfile="${OPTARG}";;
  31. r) recurse="-r";;
  32. f) filename="${OPTARG}";;
  33. u) uris+=( "${OPTARG}" );;
  34. q) quiet="-q";;
  35. :) error "option '%s' expects a mandatory argument\n" "${OPTARG}";;
  36. \?) error "unknown option '%s'\n" "${OPTARG}";;
  37. esac
  38. done
  39. # Forget our options, and keep only those for the backend
  40. shift $((OPTIND-1))
  41. if [ -z "${output}" ]; then
  42. error "no output specified, use -o\n"
  43. fi
  44. # Legacy handling: check if the file already exists in the global
  45. # download directory. If it does, hard-link it. If it turns out it
  46. # was an incorrect download, we'd still check it below anyway.
  47. # If we can neither link nor copy, fallback to doing a download.
  48. # NOTE! This is not atomic, is subject to TOCTTOU, but the whole
  49. # dl-wrapper runs under an flock, so we're safe.
  50. if [ ! -e "${output}" -a -e "${old_dl_dir}/${filename}" ]; then
  51. ln "${old_dl_dir}/${filename}" "${output}" || \
  52. cp "${old_dl_dir}/${filename}" "${output}" || \
  53. true
  54. fi
  55. # If the output file already exists and:
  56. # - there's no .hash file: do not download it again and exit promptly
  57. # - matches all its hashes: do not download it again and exit promptly
  58. # - fails at least one of its hashes: force a re-download
  59. # - there's no hash (but a .hash file): consider it a hard error
  60. if [ -e "${output}" ]; then
  61. if support/download/check-hash ${quiet} "${hfile}" "${output}" "${output##*/}"; then
  62. exit 0
  63. elif [ ${?} -ne 2 ]; then
  64. # Do not remove the file, otherwise it might get re-downloaded
  65. # from a later location (i.e. primary -> upstream -> mirror).
  66. # Do not print a message, check-hash already did.
  67. exit 1
  68. fi
  69. rm -f "${output}"
  70. warn "Re-downloading '%s'...\n" "${output##*/}"
  71. fi
  72. # Look through all the uris that we were given to download the package
  73. # source
  74. download_and_check=0
  75. rc=1
  76. for uri in "${uris[@]}"; do
  77. backend_urlencode="${uri%%+*}"
  78. backend="${backend_urlencode%|*}"
  79. case "${backend}" in
  80. git|svn|cvs|bzr|file|scp|hg) ;;
  81. *) backend="wget" ;;
  82. esac
  83. uri=${uri#*+}
  84. urlencode=${backend_urlencode#*|}
  85. # urlencode must be "urlencode"
  86. [ "${urlencode}" != "urlencode" ] && urlencode=""
  87. # tmpd is a temporary directory in which backends may store
  88. # intermediate by-products of the download.
  89. # tmpf is the file in which the backends should put the downloaded
  90. # content.
  91. # tmpd is located in $(BUILD_DIR), so as not to clutter the (precious)
  92. # $(BR2_DL_DIR)
  93. # We let the backends create tmpf, so they are able to set whatever
  94. # permission bits they want (although we're only really interested in
  95. # the executable bit.)
  96. tmpd="$(mktemp -d "${BUILD_DIR}/.${output##*/}.XXXXXX")"
  97. tmpf="${tmpd}/output"
  98. # Helpers expect to run in a directory that is *really* trashable, so
  99. # they are free to create whatever files and/or sub-dirs they might need.
  100. # Doing the 'cd' here rather than in all backends is easier.
  101. cd "${tmpd}"
  102. # If the backend fails, we can just remove the content of the temporary
  103. # directory to remove all the cruft it may have left behind, and try
  104. # the next URI until it succeeds. Once out of URI to try, we need to
  105. # cleanup and exit.
  106. if ! "${OLDPWD}/support/download/${backend}" \
  107. $([ -n "${urlencode}" ] && printf %s '-e') \
  108. -c "${cset}" \
  109. -d "${dl_dir}" \
  110. -n "${raw_base_name}" \
  111. -N "${base_name}" \
  112. -f "${filename}" \
  113. -u "${uri}" \
  114. -o "${tmpf}" \
  115. ${quiet} ${recurse} -- "${@}"
  116. then
  117. # cd back to keep path coherence
  118. cd "${OLDPWD}"
  119. rm -rf "${tmpd}"
  120. continue
  121. fi
  122. # cd back to free the temp-dir, so we can remove it later
  123. cd "${OLDPWD}"
  124. # Check if the downloaded file is sane, and matches the stored hashes
  125. # for that file
  126. if support/download/check-hash ${quiet} "${hfile}" "${tmpf}" "${output##*/}"; then
  127. rc=0
  128. else
  129. if [ ${?} -ne 3 ]; then
  130. rm -rf "${tmpd}"
  131. continue
  132. fi
  133. # the hash file exists and there was no hash to check the file
  134. # against
  135. rc=1
  136. fi
  137. download_and_check=1
  138. break
  139. done
  140. # We tried every URI possible, none seems to work or to check against the
  141. # available hash. *ABORT MISSION*
  142. if [ "${download_and_check}" -eq 0 ]; then
  143. rm -rf "${tmpd}"
  144. exit 1
  145. fi
  146. # tmp_output is in the same directory as the final output, so we can
  147. # later move it atomically.
  148. tmp_output="$(mktemp "${output}.XXXXXX")"
  149. # 'mktemp' creates files with 'go=-rwx', so the files are not accessible
  150. # to users other than the one doing the download (and root, of course).
  151. # This can be problematic when a shared BR2_DL_DIR is used by different
  152. # users (e.g. on a build server), where all users may write to the shared
  153. # location, since other users would not be allowed to read the files
  154. # another user downloaded.
  155. # So, we restore the 'go' access rights to a more sensible value, while
  156. # still abiding by the current user's umask. We must do that before the
  157. # final 'mv', so just do it now.
  158. # Some backends (cp and scp) may create executable files, so we need to
  159. # carry the executable bit if needed.
  160. [ -x "${tmpf}" ] && new_mode=755 || new_mode=644
  161. new_mode=$(printf "%04o" $((0${new_mode} & ~0$(umask))))
  162. chmod ${new_mode} "${tmp_output}"
  163. # We must *not* unlink tmp_output, otherwise there is a small window
  164. # during which another download process may create the same tmp_output
  165. # name (very, very unlikely; but not impossible.)
  166. # Using 'cp' is not reliable, since 'cp' may unlink the destination file
  167. # if it is unable to open it with O_WRONLY|O_TRUNC; see:
  168. # http://pubs.opengroup.org/onlinepubs/9699919799/utilities/cp.html
  169. # Since the destination filesystem can be anything, it might not support
  170. # O_TRUNC, so 'cp' would unlink it first.
  171. # Use 'cat' and append-redirection '>>' to save to the final location,
  172. # since that is the only way we can be 100% sure of the behaviour.
  173. if ! cat "${tmpf}" >>"${tmp_output}"; then
  174. rm -rf "${tmpd}" "${tmp_output}"
  175. exit 1
  176. fi
  177. rm -rf "${tmpd}"
  178. # tmp_output and output are on the same filesystem, so POSIX guarantees
  179. # that 'mv' is atomic, because it then uses rename() that POSIX mandates
  180. # to be atomic, see:
  181. # http://pubs.opengroup.org/onlinepubs/9699919799/functions/rename.html
  182. if ! mv -f "${tmp_output}" "${output}"; then
  183. rm -f "${tmp_output}"
  184. exit 1
  185. fi
  186. return ${rc}
  187. }
  188. trace() { local msg="${1}"; shift; printf "%s: ${msg}" "${my_name}" "${@}"; }
  189. warn() { trace "${@}" >&2; }
  190. errorN() { local ret="${1}"; shift; warn "${@}"; exit ${ret}; }
  191. error() { errorN 1 "${@}"; }
  192. my_name="${0##*/}"
  193. main "${@}"