uploader.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482
  1. # -*- coding: utf-8 -*-
  2. # Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
  3. """Main functionality for nodemcu-uploader"""
  4. # Not sure about it, because UnicodeEncodeError throws anyway
  5. # from __future__ import unicode_literals
  6. import time
  7. import logging
  8. import hashlib
  9. import os
  10. import errno
  11. import serial
  12. from .exceptions import CommunicationTimeout, DeviceNotFoundException, \
  13. BadResponseException, VerificationError, NoAckException
  14. from .utils import default_port, system, wrap, hexify, from_file, ENCODING
  15. from .luacode import RECV_LUA, SEND_LUA, LUA_FUNCTIONS, \
  16. LIST_FILES, UART_SETUP, PRINT_FILE
  17. log = logging.getLogger(__name__) # pylint: disable=C0103
  18. __all__ = ['Uploader', 'default_port']
  19. SYSTEM = system()
  20. MINIMAL_TIMEOUT = 0.001
  21. BLOCK_START = '\x01'
  22. NUL = '\x00'
  23. ACK = '\x06'
  24. class Uploader(object):
  25. """Uploader is the class for communicating with the nodemcu and
  26. that will allow various tasks like uploading files, formating the filesystem etc.
  27. """
  28. BAUD = 115200
  29. START_BAUD = 115200
  30. TIMEOUT = 5
  31. AUTOBAUD_TIME = 0.3
  32. PORT = default_port()
  33. def __init__(self, port=PORT, baud=BAUD, start_baud=START_BAUD, timeout=TIMEOUT, autobaud_time=AUTOBAUD_TIME):
  34. self._timeout = Uploader.TIMEOUT
  35. self.set_timeout(timeout)
  36. log.info('opening port %s with %s baud', port, start_baud)
  37. if port == 'loop://':
  38. self._port = serial.serial_for_url(port, start_baud, timeout=timeout)
  39. else:
  40. self._port = serial.Serial(port, start_baud, timeout=timeout)
  41. # black magic aka proxifying
  42. self._port = wrap(self._port)
  43. self.start_baud = start_baud
  44. self.baud = baud
  45. self.autobaud_time = autobaud_time
  46. # Keeps things working, if following conections are made:
  47. ## RTS = CH_PD (i.e reset)
  48. ## DTR = GPIO0
  49. self._port.setRTS(False)
  50. self._port.setDTR(False)
  51. def __sync():
  52. """Get in sync with LUA (this assumes that NodeMCU gets reset by the previous two lines)"""
  53. log.debug('getting in sync with LUA')
  54. self.__clear_buffers()
  55. try:
  56. self.__writeln('UUUUUUUUUUUU') # Send enough characters for auto-baud
  57. self.__clear_buffers()
  58. time.sleep(self.autobaud_time) # Wait for autobaud timer to expire
  59. self.__exchange(';') # Get a defined state
  60. self.__writeln('print("%sync%");')
  61. self.__expect('%sync%\r\n> ')
  62. except CommunicationTimeout:
  63. raise DeviceNotFoundException('Device not found or wrong port')
  64. __sync()
  65. if baud != start_baud:
  66. self.__set_baudrate(baud)
  67. # Get in sync again
  68. __sync()
  69. self.line_number = 0
  70. def __set_baudrate(self, baud):
  71. """setting baudrate if supported"""
  72. log.info('Changing communication to %s baud', baud)
  73. self.__writeln(UART_SETUP.format(baud=baud))
  74. # Wait for the string to be sent before switching baud
  75. time.sleep(0.1)
  76. try:
  77. self._port.setBaudrate(baud)
  78. except AttributeError:
  79. #pySerial 2.7
  80. self._port.baudrate = baud
  81. def set_timeout(self, timeout):
  82. """Set the timeout for the communication with the device."""
  83. timeout = int(timeout) # will raise on Error
  84. self._timeout = timeout == 0 and 999999 or timeout
  85. def __clear_buffers(self):
  86. """Clears the input and output buffers"""
  87. try:
  88. self._port.reset_input_buffer()
  89. self._port.reset_output_buffer()
  90. except AttributeError:
  91. #pySerial 2.7
  92. self._port.flushInput()
  93. self._port.flushOutput()
  94. def __expect(self, exp='> ', timeout=None):
  95. """will wait for exp to be returned from nodemcu or timeout"""
  96. timeout_before = self._port.timeout
  97. timeout = timeout or self._timeout
  98. #do NOT set timeout on Windows
  99. if SYSTEM != 'Windows':
  100. # Checking for new data every 100us is fast enough
  101. if self._port.timeout != MINIMAL_TIMEOUT:
  102. self._port.timeout = MINIMAL_TIMEOUT
  103. end = time.time() + timeout
  104. # Finish as soon as either exp matches or we run out of time (work like dump, but faster on success)
  105. data = ''
  106. while not data.endswith(exp) and time.time() <= end:
  107. data += self._port.read()
  108. log.debug('expect returned: `{0}`'.format(data))
  109. if time.time() > end:
  110. raise CommunicationTimeout('Timeout waiting for data', data)
  111. if not data.endswith(exp) and len(exp) > 0:
  112. raise BadResponseException('Bad response.', exp, data)
  113. if SYSTEM != 'Windows':
  114. self._port.timeout = timeout_before
  115. return data
  116. def __write(self, output, binary=False):
  117. """write data on the nodemcu port. If 'binary' is True the debug log
  118. will show the intended output as hex, otherwise as string"""
  119. if not binary:
  120. log.debug('write: %s', output)
  121. else:
  122. log.debug('write binary: %s', hexify(output))
  123. self._port.write(output)
  124. self._port.flush()
  125. def __writeln(self, output):
  126. """write, with linefeed"""
  127. self.__write(output + '\n')
  128. def __exchange(self, output, timeout=None):
  129. """Write output to the port and wait for response"""
  130. self.__writeln(output)
  131. self._port.flush()
  132. return self.__expect(timeout=timeout or self._timeout)
  133. def close(self):
  134. """restores the nodemcu to default baudrate and then closes the port"""
  135. try:
  136. if self.baud != self.start_baud:
  137. self.__set_baudrate(self.start_baud)
  138. self._port.flush()
  139. self.__clear_buffers()
  140. except serial.serialutil.SerialException:
  141. pass
  142. log.debug('closing port')
  143. self._port.close()
  144. def prepare(self):
  145. """
  146. This uploads the protocol functions nessecary to do binary
  147. chunked transfer
  148. """
  149. log.info('Preparing esp for transfer.')
  150. for func in LUA_FUNCTIONS:
  151. detected = self.__exchange('print({0})'.format(func))
  152. if detected.find('function:') == -1:
  153. break
  154. else:
  155. log.info('Preparation already done. Not adding functions again.')
  156. return True
  157. functions = RECV_LUA + '\n' + SEND_LUA
  158. data = functions.format(baud=self._port.baudrate)
  159. ##change any \r\n to just \n and split on that
  160. lines = data.replace('\r', '').split('\n')
  161. #remove some unneccesary spaces to conserve some bytes
  162. for line in lines:
  163. line = line.strip().replace(', ', ',').replace(' = ', '=')
  164. if len(line) == 0:
  165. continue
  166. resp = self.__exchange(line)
  167. #do some basic test of the result
  168. if ('unexpected' in resp) or ('stdin' in resp) or len(resp) > len(functions)+10:
  169. log.error('error when preparing "%s"', resp)
  170. return False
  171. return True
  172. def download_file(self, filename):
  173. """Download a file from device to local filesystem"""
  174. res = self.__exchange('send("{filename}")'.format(filename=filename))
  175. if ('unexpected' in res) or ('stdin' in res):
  176. log.error('Unexpected error downloading file: %s', res)
  177. raise Exception('Unexpected error downloading file')
  178. #tell device we are ready to receive
  179. self.__write('C')
  180. #we should get a NUL terminated filename to start with
  181. sent_filename = self.__expect(NUL).strip()
  182. log.info('receiveing ' + sent_filename)
  183. #ACK to start download
  184. self.__write(ACK, True)
  185. buf = ''
  186. data = ''
  187. chunk, buf = self.__read_chunk(buf)
  188. #read chunks until we get an empty which is the end
  189. while chunk != '':
  190. self.__write(ACK, True)
  191. data = data + chunk
  192. chunk, buf = self.__read_chunk(buf)
  193. return data
  194. def read_file(self, filename, destination=''):
  195. """reading data from device into local file"""
  196. if not destination:
  197. destination = filename
  198. log.info('Transferring %s to %s', filename, destination)
  199. data = self.download_file(filename)
  200. # Just in case, the filename may contain folder, so create it if needed.
  201. log.info(destination)
  202. if not os.path.exists(os.path.dirname(destination)):
  203. try:
  204. os.makedirs(os.path.dirname(destination))
  205. except OSError as e: # Guard against race condition
  206. if e.errno != errno.EEXIST:
  207. raise
  208. with open(destination, 'w') as fil:
  209. fil.write(data)
  210. def write_file(self, path, destination='', verify='none'):
  211. """sends a file to the device using the transfer protocol"""
  212. filename = os.path.basename(path)
  213. if not destination:
  214. destination = filename
  215. log.info('Transferring %s as %s', path, destination)
  216. self.__writeln("recv()")
  217. res = self.__expect('C> ')
  218. if not res.endswith('C> '):
  219. log.error('Error waiting for esp "%s"', res)
  220. raise CommunicationTimeout('Error waiting for device to start receiving', res)
  221. log.debug('sending destination filename "%s"', destination)
  222. self.__write(destination + '\x00', True)
  223. if not self.__got_ack():
  224. log.error('did not ack destination filename')
  225. raise NoAckException('Device did not ACK destination filename')
  226. content = from_file(path)
  227. log.debug('sending %d bytes in %s', len(content), filename)
  228. pos = 0
  229. chunk_size = 128
  230. while pos < len(content):
  231. rest = len(content) - pos
  232. if rest > chunk_size:
  233. rest = chunk_size
  234. data = content[pos:pos+rest]
  235. if not self.__write_chunk(data):
  236. resp = self.__expect()
  237. log.error('Bad chunk response "%s" %s', resp, hexify(resp))
  238. raise BadResponseException('Bad chunk response', ACK, resp)
  239. pos += chunk_size
  240. log.debug('sending zero block')
  241. #zero size block
  242. self.__write_chunk('')
  243. if verify != 'none':
  244. self.verify_file(path, destination, verify)
  245. def verify_file(self, path, destination, verify='none'):
  246. """Tries to verify if path has same checksum as destination.
  247. Valid options for verify is 'raw', 'sha1' or 'none'
  248. """
  249. content = from_file(path)
  250. log.info('Verifying using %s...' % verify)
  251. if verify == 'raw':
  252. data = self.download_file(destination)
  253. if content != data:
  254. log.error('Raw verification failed.')
  255. raise VerificationError('Verification failed.')
  256. else:
  257. log.info('Verification successful. Contents are identical.')
  258. elif verify == 'sha1':
  259. #Calculate SHA1 on remote file. Extract just hash from result
  260. data = self.__exchange('shafile("'+destination+'")').splitlines()[1]
  261. log.info('Remote SHA1: %s', data)
  262. #Calculate hash of local data
  263. filehashhex = hashlib.sha1(content.encode(ENCODING)).hexdigest()
  264. log.info('Local SHA1: %s', filehashhex)
  265. if data != filehashhex:
  266. log.error('SHA1 verification failed.')
  267. raise VerificationError('SHA1 Verification failed.')
  268. else:
  269. log.info('Verification successful. Checksums match')
  270. elif verify != 'none':
  271. raise Exception(verify + ' is not a valid verification method.')
  272. def exec_file(self, path):
  273. """execute the lines in the local file 'path'"""
  274. filename = os.path.basename(path)
  275. log.info('Execute %s', filename)
  276. content = from_file(path).replace('\r', '').split('\n')
  277. res = '> '
  278. for line in content:
  279. line = line.rstrip('\n')
  280. retlines = (res + self.__exchange(line)).splitlines()
  281. # Log all but the last line
  282. res = retlines.pop()
  283. for lin in retlines:
  284. log.info(lin)
  285. # last line
  286. log.info(res)
  287. def __got_ack(self):
  288. """Returns true if ACK is received"""
  289. log.debug('waiting for ack')
  290. res = self._port.read(1)
  291. log.debug('ack read %s', hexify(res))
  292. return res == ACK
  293. def write_lines(self, data):
  294. """write lines, one by one, separated by \n to device"""
  295. lines = data.replace('\r', '').split('\n')
  296. for line in lines:
  297. self.__exchange(line)
  298. def __write_chunk(self, chunk):
  299. """formats and sends a chunk of data to the device according
  300. to transfer protocol"""
  301. log.debug('writing %d bytes chunk', len(chunk))
  302. data = BLOCK_START + chr(len(chunk)) + chunk
  303. if len(chunk) < 128:
  304. padding = 128 - len(chunk)
  305. log.debug('pad with %d characters', padding)
  306. data = data + (' ' * padding)
  307. log.debug("packet size %d", len(data))
  308. self.__write(data)
  309. self._port.flush()
  310. return self.__got_ack()
  311. def __read_chunk(self, buf):
  312. """Read a chunk of data"""
  313. log.debug('reading chunk')
  314. timeout_before = self._port.timeout
  315. if SYSTEM != 'Windows':
  316. # Checking for new data every 100us is fast enough
  317. if self._port.timeout != MINIMAL_TIMEOUT:
  318. self._port.timeout = MINIMAL_TIMEOUT
  319. end = time.time() + timeout_before
  320. while len(buf) < 130 and time.time() <= end:
  321. buf = buf + self._port.read()
  322. if buf[0] != BLOCK_START or len(buf) < 130:
  323. log.debug('buffer binary: %s ', hexify(buf))
  324. raise Exception('Bad blocksize or start byte')
  325. if SYSTEM != 'Windows':
  326. self._port.timeout = timeout_before
  327. chunk_size = ord(buf[1])
  328. data = buf[2:chunk_size+2]
  329. buf = buf[130:]
  330. return (data, buf)
  331. def file_list(self):
  332. """list files on the device"""
  333. log.info('Listing files')
  334. res = self.__exchange(LIST_FILES)
  335. res = res.split('\r\n')
  336. # skip first and last lines
  337. res = res[1:-1]
  338. files = []
  339. for line in res:
  340. files.append(line.split('\t'))
  341. return files
  342. def file_do(self, filename):
  343. """Execute a file on the device using 'do'"""
  344. log.info('Executing '+filename)
  345. res = self.__exchange('dofile("'+filename+'")')
  346. log.info(res)
  347. return res
  348. def file_format(self):
  349. """Formats device filesystem"""
  350. log.info('Formating, can take minutes depending on flash size...')
  351. res = self.__exchange('file.format()', timeout=300)
  352. if 'format done' not in res:
  353. log.error(res)
  354. else:
  355. log.info(res)
  356. return res
  357. def file_print(self, filename):
  358. """Prints a file on the device to console"""
  359. log.info('Printing ' + filename)
  360. res = self.__exchange(PRINT_FILE.format(filename=filename))
  361. log.info(res)
  362. return res
  363. def node_heap(self):
  364. """Show device heap size"""
  365. log.info('Heap')
  366. res = self.__exchange('print(node.heap())')
  367. log.info(res)
  368. return int(res.split('\r\n')[1])
  369. def node_restart(self):
  370. """Restarts device"""
  371. log.info('Restart')
  372. res = self.__exchange('node.restart()')
  373. log.info(res)
  374. return res
  375. def file_compile(self, path):
  376. """Compiles a file specified by path on the device"""
  377. log.info('Compile '+path)
  378. cmd = 'node.compile("%s")' % path
  379. res = self.__exchange(cmd)
  380. log.info(res)
  381. return res
  382. def file_remove(self, path):
  383. """Removes a file on the device"""
  384. log.info('Remove '+path)
  385. cmd = 'file.remove("%s")' % path
  386. res = self.__exchange(cmd)
  387. log.info(res)
  388. return res
  389. def backup(self, path):
  390. """Backup all files from the device"""
  391. log.info('Backing up in '+path)
  392. # List file to backup
  393. files = self.file_list()
  394. # then download each of then
  395. self.prepare()
  396. for f in files:
  397. self.read_file(f[0], os.path.join(path, f[0]))