uploader.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. # Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
  4. import time
  5. import logging
  6. import hashlib
  7. import os
  8. import serial
  9. from .exceptions import CommunicationTimeout, DeviceNotFoundException, \
  10. BadResponseException
  11. from .utils import default_port, system
  12. from .luacode import DOWNLOAD_FILE, RECV_LUA, SEND_LUA, LUA_FUNCTIONS, \
  13. LIST_FILES, UART_SETUP, PRINT_FILE
  14. log = logging.getLogger(__name__)
  15. __all__ = ['Uploader', 'default_port']
  16. SYSTEM = system()
  17. BLOCK_START = '\x01'
  18. NUL = '\x00'
  19. ACK = '\x06'
  20. class Uploader(object):
  21. """Uploader is the class for communicating with the nodemcu and
  22. that will allow various tasks like uploading files, formating the filesystem etc.
  23. """
  24. BAUD = 115200
  25. START_BAUD = 9600
  26. TIMEOUT = 5
  27. PORT = default_port()
  28. def __init__(self, port=PORT, baud=BAUD, start_baud=START_BAUD, timeout=TIMEOUT):
  29. self.set_timeout(timeout)
  30. log.info('opening port %s with %s baud', port, start_baud)
  31. if port == 'loop://':
  32. self._port = serial.serial_for_url(port, start_baud, timeout=timeout)
  33. else:
  34. self._port = serial.Serial(port, start_baud, timeout=timeout)
  35. self.start_baud = start_baud
  36. self.baud = baud
  37. # Keeps things working, if following conections are made:
  38. ## RTS = CH_PD (i.e reset)
  39. ## DTR = GPIO0
  40. self._port.setRTS(False)
  41. self._port.setDTR(False)
  42. def sync():
  43. # Get in sync with LUA (this assumes that NodeMCU gets reset by the previous two lines)
  44. log.debug('getting in sync with LUA')
  45. self.clear_buffers()
  46. try:
  47. self.exchange(';') # Get a defined state
  48. self.writeln('print("%sync%");')
  49. self.expect('%sync%\r\n> ')
  50. except CommunicationTimeout:
  51. raise DeviceNotFoundException('Device not found or wrong port')
  52. sync()
  53. if baud != start_baud:
  54. self.set_baudrate(baud)
  55. # Get in sync again
  56. sync()
  57. self.line_number = 0
  58. def set_baudrate(self, baud):
  59. log.info('Changing communication to %s baud', baud)
  60. self.writeln(UART_SETUP.format(baud=baud))
  61. # Wait for the string to be sent before switching baud
  62. time.sleep(0.1)
  63. try:
  64. self._port.setBaudrate(baud)
  65. except AttributeError:
  66. #pySerial 2.7
  67. self._port.baudrate = baud
  68. def set_timeout(self, timeout):
  69. """Set the timeout for the communication with the device."""
  70. timeout = int(timeout) # will raise on Error
  71. self._timeout = timeout == 0 and 999999 or timeout
  72. def clear_buffers(self):
  73. try:
  74. self._port.reset_input_buffer()
  75. self._port.reset_output_buffer()
  76. except AttributeError:
  77. #pySerial 2.7
  78. self._port.flushInput()
  79. self._port.flushOutput()
  80. def expect(self, exp='> ', timeout=None):
  81. """will wait for exp to be returned from nodemcu or timeout"""
  82. timeout = timeout or self._timeout
  83. #do NOT set timeout on Windows
  84. if SYSTEM != 'Windows':
  85. timer = self._port.timeout
  86. # Checking for new data every 100us is fast enough
  87. lt = 0.0001
  88. if self._port.timeout != lt:
  89. self._port.timeout = lt
  90. end = time.time() + timeout
  91. # Finish as soon as either exp matches or we run out of time (work like dump, but faster on success)
  92. data = ''
  93. while not data.endswith(exp) and time.time() <= end:
  94. data += self._port.read()
  95. log.debug('expect returned: `{0}`'.format(data))
  96. if time.time() > end:
  97. raise CommunicationTimeout('Timeout waiting for data', data)
  98. if not data.endswith(exp) and len(exp) > 0:
  99. raise BadResponseException('Bad response.', exp, data)
  100. if SYSTEM != 'Windows':
  101. self._port.timeout = timer
  102. return data
  103. def write(self, output, binary=False):
  104. """write data on the nodemcu port. If 'binary' is True the debug log
  105. will show the intended output as hex, otherwise as string"""
  106. if not binary:
  107. log.debug('write: %s', output)
  108. else:
  109. log.debug('write binary: %s', ':'.join(x.encode('hex') for x in output))
  110. self._port.write(output)
  111. self._port.flush()
  112. def writeln(self, output):
  113. """write, with linefeed"""
  114. self.write(output + '\n')
  115. def exchange(self, output, timeout=None):
  116. self.writeln(output)
  117. self._port.flush()
  118. return self.expect(timeout=timeout or self._timeout)
  119. def close(self):
  120. """restores the nodemcu to default baudrate and then closes the port"""
  121. try:
  122. if self.baud != self.start_baud:
  123. self.set_baudrate(self.start_baud)
  124. self._port.flush()
  125. self.clear_buffers()
  126. except serial.serialutil.SerialException:
  127. pass
  128. log.debug('closing port')
  129. self._port.close()
  130. def prepare(self):
  131. """
  132. This uploads the protocol functions nessecary to do binary
  133. chunked transfer
  134. """
  135. log.info('Preparing esp for transfer.')
  136. for fn in LUA_FUNCTIONS:
  137. d = self.exchange('print({0})'.format(fn))
  138. if d.find('function:') == -1:
  139. break
  140. else:
  141. log.info('Preparation already done. Not adding functions again.')
  142. return True
  143. functions = RECV_LUA + '\n' + SEND_LUA
  144. data = functions.format(baud=self._port.baudrate)
  145. ##change any \r\n to just \n and split on that
  146. lines = data.replace('\r', '').split('\n')
  147. #remove some unneccesary spaces to conserve some bytes
  148. for line in lines:
  149. line = line.strip().replace(', ', ',').replace(' = ', '=')
  150. if len(line) == 0:
  151. continue
  152. d = self.exchange(line)
  153. #do some basic test of the result
  154. if ('unexpected' in d) or ('stdin' in d) or len(d) > len(functions)+10:
  155. log.error('error when preparing "%s"', d)
  156. return False
  157. return True
  158. def download_file(self, filename):
  159. res = self.exchange('send("{filename}")'.format(filename=filename))
  160. if ('unexpected' in res) or ('stdin' in res):
  161. log.error('Unexpected error downloading file', res)
  162. raise Exception('Unexpected error downloading file')
  163. #tell device we are ready to receive
  164. self.write('C')
  165. #we should get a NUL terminated filename to start with
  166. sent_filename = self.expect(NUL).strip()
  167. log.info('receiveing ' + sent_filename)
  168. #ACK to start download
  169. self.write(ACK, True)
  170. buffer = ''
  171. data = ''
  172. chunk, buffer = self.read_chunk(buffer)
  173. #read chunks until we get an empty which is the end
  174. while chunk != '':
  175. self.write(ACK, True)
  176. data = data + chunk
  177. chunk, buffer = self.read_chunk(buffer)
  178. return data
  179. def read_file(self, filename, destination=''):
  180. if not destination:
  181. destination = filename
  182. log.info('Transfering %s to %s', filename, destination)
  183. data = self.download_file(filename)
  184. with open(destination, 'w') as f:
  185. f.write(data)
  186. def write_file(self, path, destination='', verify='none'):
  187. filename = os.path.basename(path)
  188. if not destination:
  189. destination = filename
  190. log.info('Transfering %s as %s', path, destination)
  191. self.writeln("recv()")
  192. res = self.expect('C> ')
  193. if not res.endswith('C> '):
  194. log.error('Error waiting for esp "%s"', res)
  195. return
  196. log.debug('sending destination filename "%s"', destination)
  197. self.write(destination + '\x00', True)
  198. if not self.got_ack():
  199. log.error('did not ack destination filename')
  200. return
  201. f = open(path, 'rb')
  202. content = f.read()
  203. f.close()
  204. log.debug('sending %d bytes in %s', len(content), filename)
  205. pos = 0
  206. chunk_size = 128
  207. while pos < len(content):
  208. rest = len(content) - pos
  209. if rest > chunk_size:
  210. rest = chunk_size
  211. data = content[pos:pos+rest]
  212. if not self.write_chunk(data):
  213. d = self.expect()
  214. log.error('Bad chunk response "%s" %s', d, ':'.join(x.encode('hex') for x in d))
  215. return
  216. pos += chunk_size
  217. log.debug('sending zero block')
  218. #zero size block
  219. self.write_chunk('')
  220. if verify == 'raw':
  221. log.info('Verifying...')
  222. data = self.download_file(destination)
  223. if content != data:
  224. log.error('Verification failed.')
  225. else:
  226. log.info('Verification successfull. Contents are identical.')
  227. elif verify == 'sha1':
  228. #Calculate SHA1 on remote file. Extract just hash from result
  229. data = self.exchange('shafile("'+destination+'")').splitlines()[1]
  230. log.info('Remote SHA1: %s', data)
  231. #Calculate hash of local data
  232. filehashhex = hashlib.sha1(content.encode()).hexdigest()
  233. log.info('Local SHA1: %s', filehashhex)
  234. if data != filehashhex:
  235. log.error('Verification failed.')
  236. else:
  237. log.info('Verification successfull. Checksums match')
  238. elif verify != 'none':
  239. raise Exception(verify + ' is not a valid verification method.')
  240. def exec_file(self, path):
  241. filename = os.path.basename(path)
  242. log.info('Execute %s', filename)
  243. f = open(path, 'rt')
  244. res = '> '
  245. for line in f:
  246. line = line.rstrip('\r\n')
  247. retlines = (res + self.exchange(line)).splitlines()
  248. # Log all but the last line
  249. res = retlines.pop()
  250. for lin in retlines:
  251. log.info(lin)
  252. # last line
  253. log.info(res)
  254. f.close()
  255. def got_ack(self):
  256. log.debug('waiting for ack')
  257. res = self._port.read(1)
  258. log.debug('ack read %s', res.encode('hex'))
  259. return res == '\x06' #ACK
  260. def write_lines(self, data):
  261. lines = data.replace('\r', '').split('\n')
  262. for line in lines:
  263. self.exchange(line)
  264. return
  265. def write_chunk(self, chunk):
  266. log.debug('writing %d bytes chunk', len(chunk))
  267. data = BLOCK_START + chr(len(chunk)) + chunk
  268. if len(chunk) < 128:
  269. padding = 128 - len(chunk)
  270. log.debug('pad with %d characters', padding)
  271. data = data + (' ' * padding)
  272. log.debug("packet size %d", len(data))
  273. self.write(data)
  274. self._port.flush()
  275. return self.got_ack()
  276. def read_chunk(self, buffer):
  277. log.debug('reading chunk')
  278. timeout = self._port.timeout
  279. if SYSTEM != 'Windows':
  280. timer = self._port.timeout
  281. # Checking for new data every 100us is fast enough
  282. lt = 0.0001
  283. if self._port.timeout != lt:
  284. self._port.timeout = lt
  285. end = time.time() + timeout
  286. while len(buffer) < 130 and time.time() <= end:
  287. buffer = buffer + self._port.read()
  288. if buffer[0] != BLOCK_START or len(buffer) < 130:
  289. print 'buffer size:', len(buffer)
  290. log.debug('buffer binary: ', ':'.join(x.encode('hex') for x in buffer))
  291. raise Exception('Bad blocksize or start byte')
  292. chunk_size = ord(buffer[1])
  293. data = buffer[2:chunk_size+2]
  294. buffer = buffer[130:]
  295. return (data, buffer)
  296. def file_list(self):
  297. log.info('Listing files')
  298. res = self.exchange(LIST_FILES)
  299. log.info(res)
  300. return res
  301. def file_do(self, f):
  302. log.info('Executing '+f)
  303. res = self.exchange('dofile("'+f+'")')
  304. log.info(res)
  305. return res
  306. def file_format(self):
  307. log.info('Formating, can take up to 1 minute...')
  308. res = self.exchange('file.format()', timeout=60)
  309. if 'format done' not in res:
  310. log.error(res)
  311. else:
  312. log.info(res)
  313. return res
  314. def file_print(self, f):
  315. log.info('Printing ' + f)
  316. res = self.exchange(PRINT_FILE.format(filename=f))
  317. log.info(res)
  318. return res
  319. def node_heap(self):
  320. log.info('Heap')
  321. res = self.exchange('print(node.heap())')
  322. log.info(res)
  323. return res
  324. def node_restart(self):
  325. log.info('Restart')
  326. res = self.exchange('node.restart()')
  327. log.info(res)
  328. return res
  329. def file_compile(self, path):
  330. log.info('Compile '+path)
  331. cmd = 'node.compile("%s")' % path
  332. res = self.exchange(cmd)
  333. log.info(res)
  334. return res
  335. def file_remove(self, path):
  336. log.info('Remove '+path)
  337. cmd = 'file.remove("%s")' % path
  338. res = self.exchange(cmd)
  339. log.info(res)
  340. return res