pkg-download.mk 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258
  1. ############################################################################
  2. #
  3. # This file contains the download helpers for the various package
  4. # infrastructures. It is used to handle downloads from HTTP servers,
  5. # FTP servers, Git repositories, Subversion repositories, Mercurial
  6. # repositories, Bazaar repositories, and SCP servers.
  7. #
  8. ############################################################################
  9. # Download method commands
  10. WGET:=$(call qstrip,$(BR2_WGET)) $(QUIET)
  11. SVN:=$(call qstrip,$(BR2_SVN))
  12. BZR:=$(call qstrip,$(BR2_BZR))
  13. GIT:=$(call qstrip,$(BR2_GIT))
  14. HG:=$(call qstrip,$(BR2_HG)) $(QUIET)
  15. SCP:=$(call qstrip,$(BR2_SCP)) $(QUIET)
  16. SSH:=$(call qstrip,$(BR2_SSH)) $(QUIET)
  17. LOCALFILES:=$(call qstrip,$(BR2_LOCALFILES))
  18. # Default spider mode is 'DOWNLOAD'. Other possible values are 'SOURCE_CHECK'
  19. # used by the _source-check target and 'SHOW_EXTERNAL_DEPS', used by the
  20. # external-deps target.
  21. DL_MODE=DOWNLOAD
  22. # Override BR2_DL_DIR if shell variable defined
  23. ifneq ($(BUILDROOT_DL_DIR),)
  24. DL_DIR:=$(BUILDROOT_DL_DIR)
  25. else
  26. DL_DIR:=$(call qstrip,$(BR2_DL_DIR))
  27. endif
  28. ifeq ($(DL_DIR),)
  29. DL_DIR:=$(TOPDIR)/dl
  30. endif
  31. # ensure it exists and a absolute path
  32. DL_DIR:=$(shell mkdir -p $(DL_DIR) && cd $(DL_DIR) >/dev/null && pwd)
  33. #
  34. # URI scheme helper functions
  35. # Example URIs:
  36. # * http://www.example.com/dir/file
  37. # * scp://www.example.com:dir/file (with domainseparator :)
  38. #
  39. # geturischeme: http
  40. geturischeme=$(firstword $(subst ://, ,$(call qstrip,$(1))))
  41. # stripurischeme: www.example.com/dir/file
  42. stripurischeme=$(lastword $(subst ://, ,$(call qstrip,$(1))))
  43. # domain: www.example.com
  44. domain=$(firstword $(subst $(call domainseparator,$(2)), ,$(call stripurischeme,$(1))))
  45. # notdomain: dir/file
  46. notdomain=$(patsubst $(call domain,$(1),$(2))$(call domainseparator,$(2))%,%,$(call stripurischeme,$(1)))
  47. #
  48. # default domainseparator is /, specify alternative value as first argument
  49. domainseparator=$(if $(1),$(1),/)
  50. ################################################################################
  51. # The DOWNLOAD_{GIT,SVN,BZR,HG,LOCALFILES} helpers are in charge of getting a
  52. # working copy of the source repository for their corresponding SCM,
  53. # checking out the requested version / commit / tag, and create an
  54. # archive out of it. DOWNLOAD_SCP uses scp to obtain a remote file with
  55. # ssh authentication. DOWNLOAD_WGET is the normal wget-based download
  56. # mechanism.
  57. #
  58. # The SOURCE_CHECK_{GIT,SVN,BZR,HG,WGET,LOCALFILES,SCP} helpers are in charge of
  59. # simply checking that the source is available for download. This can be used
  60. # to make sure one will be able to get all the sources needed for
  61. # one's build configuration.
  62. #
  63. # The SHOW_EXTERNAL_DEPS_{GIT,SVN,BZR,HG,WGET,LOCALFILES,SCP} helpers simply
  64. # output to the console the names of the files that will be downloaded, or path
  65. # and revision of the source repositories, producing a list of all the
  66. # "external dependencies" of a given build configuration.
  67. ################################################################################
  68. # Try a shallow clone - but that only works if the version is a ref (tag or
  69. # branch). Before trying to do a shallow clone we check if $($(PKG)_DL_VERSION)
  70. # is in the list provided by git ls-remote. If not we fall back on a full clone.
  71. #
  72. # Messages for the type of clone used are provided to ease debugging in case of
  73. # problems
  74. define DOWNLOAD_GIT
  75. test -e $(DL_DIR)/$($(PKG)_SOURCE) || \
  76. (pushd $(DL_DIR) > /dev/null && \
  77. ((test `git ls-remote $($(PKG)_SITE) | cut -f 2- | grep $($(PKG)_DL_VERSION)` && \
  78. echo "Doing shallow clone" && \
  79. $(GIT) clone --depth 1 -b $($(PKG)_DL_VERSION) --bare $($(PKG)_SITE) $($(PKG)_BASE_NAME)) || \
  80. (echo "Doing full clone" && \
  81. $(GIT) clone --bare $($(PKG)_SITE) $($(PKG)_BASE_NAME))) && \
  82. pushd $($(PKG)_BASE_NAME) > /dev/null && \
  83. $(GIT) archive --format=tar --prefix=$($(PKG)_BASE_NAME)/ $($(PKG)_DL_VERSION) | \
  84. gzip -c > $(DL_DIR)/$($(PKG)_SOURCE) && \
  85. popd > /dev/null && \
  86. rm -rf $($(PKG)_DL_DIR) && \
  87. popd > /dev/null)
  88. endef
  89. # TODO: improve to check that the given PKG_DL_VERSION exists on the remote
  90. # repository
  91. define SOURCE_CHECK_GIT
  92. $(GIT) ls-remote --heads $($(PKG)_SITE) > /dev/null
  93. endef
  94. define SHOW_EXTERNAL_DEPS_GIT
  95. echo $($(PKG)_SOURCE)
  96. endef
  97. define DOWNLOAD_BZR
  98. test -e $(DL_DIR)/$($(PKG)_SOURCE) || \
  99. $(BZR) export $(DL_DIR)/$($(PKG)_SOURCE) $($(PKG)_SITE) -r $($(PKG)_DL_VERSION)
  100. endef
  101. define SOURCE_CHECK_BZR
  102. $(BZR) ls --quiet $($(PKG)_SITE) > /dev/null
  103. endef
  104. define SHOW_EXTERNAL_DEPS_BZR
  105. echo $($(PKG)_SOURCE)
  106. endef
  107. define DOWNLOAD_SVN
  108. test -e $(DL_DIR)/$($(PKG)_SOURCE) || \
  109. (pushd $(DL_DIR) > /dev/null && \
  110. $(SVN) export -r $($(PKG)_DL_VERSION) $($(PKG)_SITE) $($(PKG)_DL_DIR) && \
  111. $(TAR) czf $($(PKG)_SOURCE) $($(PKG)_BASE_NAME)/ && \
  112. rm -rf $($(PKG)_DL_DIR) && \
  113. popd > /dev/null)
  114. endef
  115. define SOURCE_CHECK_SVN
  116. $(SVN) ls $($(PKG)_SITE) > /dev/null
  117. endef
  118. define SHOW_EXTERNAL_DEPS_SVN
  119. echo $($(PKG)_SOURCE)
  120. endef
  121. # SCP URIs should be of the form scp://[user@]host:filepath
  122. # Note that filepath is relative to the user's home directory, so you may want
  123. # to prepend the path with a slash: scp://[user@]host:/absolutepath
  124. define DOWNLOAD_SCP
  125. test -e $(DL_DIR)/$(2) || \
  126. $(SCP) '$(call stripurischeme,$(call qstrip,$(1)))' $(DL_DIR)/$(2)
  127. endef
  128. define SOURCE_CHECK_SCP
  129. $(SSH) $(call domain,$(1),:) ls '$(call notdomain,$(1),:)' > /dev/null
  130. endef
  131. define SHOW_EXTERNAL_DEPS_SCP
  132. echo $(2)
  133. endef
  134. define DOWNLOAD_HG
  135. test -e $(DL_DIR)/$($(PKG)_SOURCE) || \
  136. (pushd $(DL_DIR) > /dev/null && \
  137. $(HG) clone --noupdate --rev $($(PKG)_DL_VERSION) $($(PKG)_SITE) $($(PKG)_BASE_NAME) && \
  138. $(HG) archive --repository $($(PKG)_BASE_NAME) --type tgz --prefix $($(PKG)_BASE_NAME)/ \
  139. --rev $($(PKG)_DL_VERSION) $(DL_DIR)/$($(PKG)_SOURCE) && \
  140. rm -rf $($(PKG)_DL_DIR) && \
  141. popd > /dev/null)
  142. endef
  143. # TODO: improve to check that the given PKG_DL_VERSION exists on the remote
  144. # repository
  145. define SOURCE_CHECK_HG
  146. $(HG) incoming --force -l1 $($(PKG)_SITE) > /dev/null
  147. endef
  148. define SHOW_EXTERNAL_DEPS_HG
  149. echo $($(PKG)_SOURCE)
  150. endef
  151. # Download a file using wget. Only download the file if it doesn't
  152. # already exist in the download directory. If the download fails,
  153. # remove the file (because wget -O creates a 0-byte file even if the
  154. # download fails). To handle an interrupted download as well, download
  155. # to a temporary file first. The temporary file will be overwritten
  156. # the next time the download is tried.
  157. define DOWNLOAD_WGET
  158. test -e $(DL_DIR)/$(2) || \
  159. ($(WGET) -O $(DL_DIR)/$(2).tmp '$(call qstrip,$(1))' && \
  160. mv $(DL_DIR)/$(2).tmp $(DL_DIR)/$(2)) || \
  161. (rm -f $(DL_DIR)/$(2).tmp ; exit 1)
  162. endef
  163. define SOURCE_CHECK_WGET
  164. $(WGET) --spider '$(call qstrip,$(1))'
  165. endef
  166. define SHOW_EXTERNAL_DEPS_WGET
  167. echo $(2)
  168. endef
  169. define DOWNLOAD_LOCALFILES
  170. test -e $(DL_DIR)/$(2) || \
  171. $(LOCALFILES) $(call stripurischeme,$(call qstrip,$(1))) $(DL_DIR)
  172. endef
  173. define SOURCE_CHECK_LOCALFILES
  174. test -e $(call stripurischeme,$(call qstrip,$(1)))
  175. endef
  176. define SHOW_EXTERNAL_DEPS_LOCALFILES
  177. echo $(2)
  178. endef
  179. ################################################################################
  180. # DOWNLOAD -- Download helper. Will try to download source from:
  181. # 1) BR2_PRIMARY_SITE if enabled
  182. # 2) Download site, unless BR2_PRIMARY_SITE_ONLY is set
  183. # 3) BR2_BACKUP_SITE if enabled, unless BR2_PRIMARY_SITE_ONLY is set
  184. #
  185. # Argument 1 is the source location
  186. # Argument 2 is the source filename
  187. #
  188. # E.G. use like this:
  189. # $(call DOWNLOAD,$(FOO_SITE),$(FOO_SOURCE))
  190. ################################################################################
  191. define DOWNLOAD
  192. $(call DOWNLOAD_INNER,$(1),$(if $(2),$(2),$(notdir $(1))))
  193. endef
  194. define DOWNLOAD_INNER
  195. $(Q)if test -n "$(call qstrip,$(BR2_PRIMARY_SITE))" ; then \
  196. case "$(call geturischeme,$(BR2_PRIMARY_SITE))" in \
  197. scp) $(call $(DL_MODE)_SCP,$(BR2_PRIMARY_SITE)/$(2),$(2)) && exit ;; \
  198. *) $(call $(DL_MODE)_WGET,$(BR2_PRIMARY_SITE)/$(2),$(2)) && exit ;; \
  199. esac ; \
  200. fi ; \
  201. if test "$(BR2_PRIMARY_SITE_ONLY)" = "y" ; then \
  202. exit 1 ; \
  203. fi ; \
  204. if test -n "$(1)" ; then \
  205. if test -z "$($(PKG)_SITE_METHOD)" ; then \
  206. scheme="$(call geturischeme,$(1))" ; \
  207. else \
  208. scheme="$($(PKG)_SITE_METHOD)" ; \
  209. fi ; \
  210. case "$$scheme" in \
  211. git) $($(DL_MODE)_GIT) && exit ;; \
  212. svn) $($(DL_MODE)_SVN) && exit ;; \
  213. bzr) $($(DL_MODE)_BZR) && exit ;; \
  214. file) $($(DL_MODE)_LOCALFILES) && exit ;; \
  215. scp) $($(DL_MODE)_SCP) && exit ;; \
  216. hg) $($(DL_MODE)_HG) && exit ;; \
  217. *) $(call $(DL_MODE)_WGET,$(1),$(2)) && exit ;; \
  218. esac ; \
  219. fi ; \
  220. if test -n "$(call qstrip,$(BR2_BACKUP_SITE))" ; then \
  221. $(call $(DL_MODE)_WGET,$(BR2_BACKUP_SITE)/$(2),$(2)) && exit ; \
  222. fi ; \
  223. exit 1
  224. endef