download.py 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219
  1. import os
  2. import pycurl
  3. import sys
  4. import time
  5. import urllib2
  6. import hashlib
  7. from threading import Thread
  8. from StringIO import StringIO
  9. class Download(Thread):
  10. _dst_path = ""
  11. _is_finished = False
  12. _is_successful = False
  13. _errors = []
  14. _HashFunc = None
  15. _HashValue = ""
  16. def __init__(self, url, path, cookies=False, useragent=False):
  17. super(Download, self).__init__()
  18. self.url = url
  19. self.path = path
  20. self.useragent = useragent
  21. self.cookies = cookies
  22. self.downloaded = 0
  23. self.progress = { 'downloaded': 0, 'total': 0, 'percent': 0,'stopped':False }
  24. self._stop = False
  25. self.filename = ""
  26. def isFinished(self):
  27. return self._is_finished
  28. def isSuccessful(self):
  29. return self._is_successful
  30. def get_dest(self):
  31. return self._dst_path
  32. def get_errors(self):
  33. return self._errors
  34. def run(self):
  35. c = pycurl.Curl()
  36. c.setopt(pycurl.URL, self.url)
  37. c.setopt(pycurl.FOLLOWLOCATION, True)
  38. c.setopt(pycurl.MAXREDIRS, 4)
  39. #c.setopt(pycurl.NOBODY, 1)
  40. c.setopt(c.VERBOSE, True)
  41. #c.setopt(pycurl.CONNECTTIMEOUT, 20)
  42. if self.useragent:
  43. c.setopt(pycurl.USERAGENT, self.useragent)
  44. # add cookies, if available
  45. if self.cookies:
  46. c.setopt(pycurl.COOKIE, self.cookies)
  47. #realurl = c.getinfo(pycurl.EFFECTIVE_URL)
  48. realurl = self.url
  49. print("realurl",realurl)
  50. self.filename = realurl.split("/")[-1].strip()
  51. c.setopt(pycurl.URL, realurl)
  52. c.setopt(pycurl.FOLLOWLOCATION, True)
  53. c.setopt(pycurl.NOPROGRESS, False)
  54. c.setopt(pycurl.XFERINFOFUNCTION, self.getProgress)
  55. c.setopt(pycurl.SSL_VERIFYPEER, False)
  56. c.setopt(pycurl.SSL_VERIFYHOST, False)
  57. # configure pycurl output file
  58. if self.path == False:
  59. self.path = os.getcwd()
  60. filepath = os.path.join(self.path, self.filename)
  61. if os.path.exists(filepath):## remove old file,restart download
  62. os.system("rm -rf " + filepath)
  63. buffer = StringIO()
  64. c.setopt(pycurl.WRITEDATA, buffer)
  65. self._dst_path = filepath
  66. # add cookies, if available
  67. if self.cookies:
  68. c.setopt(pycurl.COOKIE, self.cookies)
  69. self._stop = False
  70. self.progress["stopped"] = False
  71. # download file
  72. try:
  73. c.perform()
  74. except pycurl.error, error:
  75. errno,errstr = error
  76. print("curl error: %s" % errstr)
  77. self._errors.append(errstr)
  78. self._stop = True
  79. self.progress["stopped"] = True
  80. self.stop()
  81. finally:
  82. code = c.getinfo( c.RESPONSE_CODE )
  83. c.close()
  84. self._is_finished = True
  85. with open(filepath, mode='w') as f:
  86. f.write(buffer.getvalue())
  87. if self.progress["percent"] < 100:
  88. self._is_successful = False
  89. else:
  90. if self._HashFunc != None:
  91. hashed = self.hashlib_hash(self._HashFunc, self._dst_path)
  92. if hashed == self._HashValue:
  93. self._is_successful= True
  94. else:
  95. self._is_successful = False
  96. self._errors.append("hash failed")
  97. else:
  98. if code != 200:
  99. self._is_successful = False
  100. os.system("rm -rf " + self._dst_path) ## clear garbage file
  101. self._errors.append("response error %d " % code)
  102. else:
  103. self._is_successful = True ## 100% downloaded without hash check
  104. def getProgress(self,download_t, download_d, upload_t, upload_d):
  105. if download_t and download_d:
  106. self.progress['downloaded'] = download_d + self.downloaded
  107. self.progress['total'] = download_t + self.downloaded
  108. self.progress['percent'] = ( float(self.progress['downloaded']) / float(self.progress['total'])) * 100.0
  109. self.progress["stopped"] = False
  110. if self._stop:
  111. self.progress["stopped"] = True
  112. return 1
  113. def hashlib_hash(method,fname): #eg: method == hashlib.md5(),function pointer
  114. hash_ = method
  115. with open(fname, "rb") as f:
  116. for chunk in iter(lambda: f.read(4096), b""):
  117. hash_.update(chunk)
  118. return hash_.hexdigest()
  119. def add_hash_verification(self,method_name,method_value):
  120. if method_name == "md5":
  121. self._HashFunc = hashlib.md5()
  122. else:
  123. self._HashFunc = None
  124. self._HashValue = method_value
  125. def get_progress(self):
  126. return self.progress["percent"]
  127. def stop(self):
  128. self._stop = True
  129. def cancel(self):
  130. # sets the boolean to stop the thread.
  131. self._stop = True
  132. def main():
  133. from optparse import OptionParser
  134. parser = OptionParser(usage="%prog [options] <url>")
  135. parser.add_option( "-p", "--path", default=False, dest="path", help="download file to PATH", metavar="PATH")
  136. parser.add_option( "-c", "--cookies", default=False, dest="cookies", help="specify cookie(s)", metavar="COOKIES")
  137. opts, args = parser.parse_args()
  138. if len(args) == 0:
  139. parser.error("No url supplied")
  140. for url in args:
  141. print("Downloading: %s" % url)
  142. if opts.path:
  143. print("to: %s" % opts.path)
  144. else:
  145. print("to current directory")
  146. d = Download(url, opts.path, opts.cookies)
  147. d.start()
  148. last_downloaded = 0
  149. sleep_time = 0.05
  150. while 1:
  151. try:
  152. progress = d.progress['percent']
  153. download_dx = d.progress["downloaded"] - last_downloaded
  154. speed = float(download_dx) / ( sleep_time * 1000.0)
  155. last_downloaded = d.progress["downloaded"]
  156. if d.progress["stopped"] == True:
  157. break
  158. print("%.2f percent | %d of %d | %.1f KB/s" % (progress, d.progress['downloaded'], d.progress['total'], speed))
  159. if progress == 100:
  160. print("")
  161. print("Download complete: %s" % d.filename)
  162. break
  163. time.sleep(sleep_time)
  164. # tell thread to terminate on keyboard interrupt,
  165. # otherwise the process has to be killed manually
  166. except KeyboardInterrupt:
  167. d.cancel()
  168. break
  169. except:
  170. raise
  171. if __name__ == "__main__":
  172. main()