nodemcu-uploader.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557
  1. #!/usr/bin/env python
  2. # Copyright (C) 2015 Peter Magnusson
  3. # For NodeMCU version 0.9.4 build 2014-12-30 and newer.
  4. import os
  5. import serial
  6. import sys
  7. import argparse
  8. import time
  9. import logging
  10. import hashlib
  11. import warnings
  12. log = logging.getLogger(__name__)
  13. __version__='0.1.2'
  14. save_lua = \
  15. r"""
  16. function recv_block(d)
  17. if string.byte(d, 1) == 1 then
  18. size = string.byte(d, 2)
  19. uart.write(0,'\006')
  20. if size > 0 then
  21. file.write(string.sub(d, 3, 3+size-1))
  22. else
  23. file.close()
  24. uart.on('data')
  25. uart.setup(0,9600,8,0,1,1)
  26. end
  27. else
  28. uart.write(0, '\021' .. d)
  29. uart.setup(0,9600,8,0,1,1)
  30. uart.on('data')
  31. end
  32. end
  33. function recv_name(d) d = string.gsub(d, '\000', '') file.remove(d) file.open(d, 'w') uart.on('data', 130, recv_block, 0) uart.write(0, '\006') end
  34. function recv() uart.setup(0,9600,8,0,1,0) uart.on('data', '\000', recv_name, 0) uart.write(0, 'C') end
  35. function shafile(f) file.open(f, "r") print(crypto.toHex(crypto.hash("sha1",file.read()))) file.close() end
  36. """
  37. CHUNK_END = '\v'
  38. CHUNK_REPLY = '\v'
  39. try:
  40. from serial.tools.miniterm import Miniterm, console, NEWLINE_CONVERISON_MAP
  41. MINITERM_AVAILABLE=True
  42. except ImportError:
  43. MINITERM_AVAILABLE=False
  44. def deprecated(func):
  45. """This is a decorator which can be used to mark functions
  46. as deprecated. It will result in a warning being emmitted
  47. when the function is used."""
  48. def newFunc(*args, **kwargs):
  49. warnings.warn("Call to deprecated function %s." % func.__name__,
  50. category=DeprecationWarning)
  51. return func(*args, **kwargs)
  52. newFunc.__name__ = func.__name__
  53. newFunc.__doc__ = func.__doc__
  54. newFunc.__dict__.update(func.__dict__)
  55. return newFunc
  56. @deprecated
  57. class MyMiniterm(Miniterm):
  58. def __init__(self, serial):
  59. if not MINITERM_AVAILABLE:
  60. print "Miniterm is not available on this system"
  61. return
  62. self.serial = serial
  63. self.echo = False
  64. self.convert_outgoing = 2
  65. self.repr_mode = 1
  66. self.newline = NEWLINE_CONVERISON_MAP[self.convert_outgoing]
  67. self.dtr_state = True
  68. self.rts_state = True
  69. self.break_state = False
  70. class Uploader:
  71. BAUD = 9600
  72. import platform
  73. PORT = '/dev/tty.SLAB_USBtoUART' if platform.system() == 'Darwin' else '/dev/ttyUSB0'
  74. TIMEOUT = 5
  75. def expect(self, exp='> ', timeout=TIMEOUT):
  76. t = self._port.timeout
  77. # Checking for new data every 100us is fast enough
  78. lt = 0.0001
  79. if self._port.timeout != lt:
  80. self._port.timeout = lt
  81. end = time.time() + timeout
  82. # Finish as soon as either exp matches or we run out of time (work like dump, but faster on success)
  83. data = ''
  84. while not data.endswith(exp) and time.time() <= end:
  85. data += self._port.read()
  86. self._port.timeout = t
  87. log.debug('expect return: %s', data)
  88. return data
  89. def write(self, output, binary=False):
  90. if not binary:
  91. log.debug('write: %s', output)
  92. else:
  93. log.debug('write binary: %s' % ':'.join(x.encode('hex') for x in output))
  94. self._port.write(output)
  95. self._port.flush()
  96. def writeln(self, output):
  97. self.write(output + '\n')
  98. def exchange(self, output):
  99. self.writeln(output)
  100. return self.expect()
  101. def __init__(self, port = 0, baud = BAUD):
  102. self._port = serial.Serial(port, Uploader.BAUD, timeout=Uploader.TIMEOUT)
  103. # Keeps things working, if following conections are made:
  104. ## RTS = CH_PD (i.e reset)
  105. ## DTR = GPIO0
  106. self._port.setRTS(False)
  107. self._port.setDTR(False)
  108. # Get in sync with LUA (this assumes that NodeMCU gets reset by the previous two lines)
  109. self.exchange(';'); # Get a defined state
  110. self.writeln('print("%sync%");');
  111. self.expect('%sync%\r\n> ');
  112. if baud != Uploader.BAUD:
  113. log.info('Changing communication to %s baud', baud)
  114. self.writeln('uart.setup(0,%s,8,0,1,1)' % baud)
  115. # Wait for the string to be sent before switching baud
  116. time.sleep(0.1)
  117. self._port.setBaudrate(baud)
  118. # Get in sync again
  119. self.exchange('')
  120. self.exchange('')
  121. self.line_number = 0
  122. def close(self):
  123. self.writeln('uart.setup(0,%s,8,0,1,1)' % Uploader.BAUD)
  124. self._port.close()
  125. def prepare(self):
  126. log.info('Preparing esp for transfer.')
  127. data = save_lua.replace('9600', '%d' % self._port.baudrate)
  128. lines = data.replace('\r', '').split('\n')
  129. for line in lines:
  130. line = line.strip().replace(', ', ',').replace(' = ', '=')
  131. if len(line) == 0:
  132. continue
  133. d = self.exchange(line)
  134. if 'unexpected' in d or len(d) > len(save_lua)+10:
  135. log.error('error in save_lua "%s"' % d)
  136. return
  137. def download_file(self, filename):
  138. chunk_size=256
  139. bytes_read = 0
  140. data=""
  141. while True:
  142. d = self.exchange("file.open('" + filename + r"') print(file.seek('end', 0)) file.seek('set', %d) uart.write(0, file.read(%d))file.close()" % (bytes_read, chunk_size))
  143. cmd, size, tmp_data = d.split('\n', 2)
  144. data=data+tmp_data[0:chunk_size]
  145. bytes_read=bytes_read+chunk_size
  146. if bytes_read > int(size):
  147. break
  148. data = data[0:int(size)]
  149. return data
  150. def read_file(self, filename, destination = ''):
  151. if not destination:
  152. destination = filename
  153. log.info('Transfering %s to %s' %(filename, destination))
  154. data = self.download_file(filename)
  155. with open(destination, 'w') as f:
  156. f.write(data)
  157. def write_file(self, path, destination = '', verify = 'none'):
  158. filename = os.path.basename(path)
  159. if not destination:
  160. destination = filename
  161. log.info('Transfering %s as %s' %(path, destination))
  162. self.writeln("recv()")
  163. r = self.expect('C> ')
  164. if not r.endswith('C> '):
  165. log.error('Error waiting for esp "%s"' % r)
  166. return
  167. log.debug('sending destination filename "%s"', destination)
  168. self.write(destination + '\x00', True)
  169. if not self.got_ack():
  170. log.error('did not ack destination filename')
  171. return
  172. f = open( path, 'rb' ); content = f.read(); f.close()
  173. log.debug('sending %d bytes in %s' % (len(content), filename))
  174. pos = 0
  175. chunk_size = 128
  176. error = False
  177. while pos < len(content):
  178. rest = len(content) - pos
  179. if rest > chunk_size:
  180. rest = chunk_size
  181. data = content[pos:pos+rest]
  182. if not self.write_chunk(data):
  183. d = self.expect()
  184. log.error('Bad chunk response "%s" %s' % (d, ':'.join(x.encode('hex') for x in d)))
  185. return
  186. pos += chunk_size
  187. log.debug('sending zero block')
  188. #zero size block
  189. self.write_chunk('')
  190. if verify == 'standard':
  191. log.info('Verifying...')
  192. data = self.download_file(destination)
  193. if content != data:
  194. log.error('Verification failed.')
  195. elif verify == 'sha1':
  196. #Calculate SHA1 on remote file. Extract just hash from result
  197. data = self.exchange('shafile("'+destination+'")').splitlines()[1]
  198. log.info('Remote SHA1: %s',data)
  199. #Calculate hash of local data
  200. filehashhex = hashlib.sha1(content.encode()).hexdigest()
  201. log.info('Local SHA1: %s',filehashhex)
  202. if data != filehashhex:
  203. log.error('Verification failed.')
  204. def exec_file(self, path):
  205. filename = os.path.basename(path)
  206. log.info('Execute %s' %(filename,))
  207. f = open( path, 'rt' );
  208. res = '> '
  209. for line in f:
  210. line = line.rstrip('\r\n')
  211. retlines = (res + self.exchange(line)).splitlines()
  212. # Log all but the last line
  213. res = retlines.pop()
  214. for l in retlines:
  215. log.info(l)
  216. # last line
  217. log.info(res)
  218. f.close()
  219. def got_ack(self):
  220. log.debug('waiting for ack')
  221. r = self._port.read(1)
  222. log.debug('ack read %s', r.encode('hex'))
  223. return r == '\x06' #ACK
  224. def write_lines(self, data):
  225. lines = data.replace('\r', '').split('\n')
  226. for line in lines:
  227. self.exchange(line)
  228. return
  229. def write_chunk(self, chunk):
  230. log.debug('writing %d bytes chunk' % len(chunk))
  231. data = '\x01' + chr(len(chunk)) + chunk
  232. if len(chunk) < 128:
  233. padding = 128 - len(chunk)
  234. log.debug('pad with %d characters' % padding)
  235. data = data + (' ' * padding)
  236. log.debug("packet size %d" % len(data))
  237. self.write(data)
  238. return self.got_ack()
  239. def file_list(self):
  240. log.info('Listing files')
  241. r = self.exchange('for key,value in pairs(file.list()) do print(key,value) end')
  242. log.info(r)
  243. return r
  244. def file_do(self, f):
  245. log.info('Executing '+f)
  246. r = self.exchange('dofile("'+f+'")')
  247. log.info(r)
  248. return r
  249. def file_format(self):
  250. log.info('Formating...')
  251. r = self.exchange('file.format()')
  252. if 'format done' not in r:
  253. log.error(r)
  254. else:
  255. log.info(r)
  256. return r
  257. def file_remove(self):
  258. log.info('Removing...')
  259. r = self.exchange('file.remove("'+f+'")')
  260. log.info(r)
  261. return r
  262. def node_heap(self):
  263. log.info('Heap')
  264. r = self.exchange('print(node.heap())')
  265. log.info(r)
  266. return r
  267. def node_restart(self):
  268. log.info('Restart')
  269. r = self.exchange('node.restart()')
  270. log.info(r)
  271. return r
  272. def file_compile(self, path):
  273. log.info('Compile '+path)
  274. cmd = 'node.compile("%s")' % path
  275. r = self.exchange(cmd)
  276. log.info(r)
  277. return r
  278. def file_remove(self, path):
  279. log.info('Remove '+path)
  280. cmd = 'file.remove("%s")' % path
  281. r = self.exchange(cmd)
  282. log.info(r)
  283. return r
  284. @deprecated
  285. def terminal(self):
  286. if not MINITERM_AVAILABLE:
  287. print "Miniterm is not available on this system"
  288. return
  289. miniterm = MyMiniterm(self._port)
  290. log.info('Started terminal. Hit ctrl-] to leave terminal')
  291. console.setup()
  292. miniterm.start()
  293. try:
  294. miniterm.join(True)
  295. except KeyboardInterrupt:
  296. pass
  297. miniterm.join()
  298. def arg_auto_int(x):
  299. return int(x, 0)
  300. if __name__ == '__main__':
  301. parser = argparse.ArgumentParser(description = 'NodeMCU Lua file uploader', prog = 'nodemcu-uploader')
  302. parser.add_argument(
  303. '--verbose',
  304. help = 'verbose output',
  305. action = 'store_true',
  306. default = False)
  307. parser.add_argument(
  308. '--port', '-p',
  309. help = 'Serial port device',
  310. default = Uploader.PORT)
  311. parser.add_argument(
  312. '--baud', '-b',
  313. help = 'Serial port baudrate',
  314. type = arg_auto_int,
  315. default = Uploader.BAUD)
  316. subparsers = parser.add_subparsers(
  317. dest='operation',
  318. help = 'Run nodemcu-uploader {command} -h for additional help')
  319. upload_parser = subparsers.add_parser(
  320. 'upload',
  321. help = 'Path to one or more files to be uploaded. Destination name will be the same as the file name.')
  322. # upload_parser.add_argument(
  323. # '--filename', '-f',
  324. # help = 'File to upload. You can specify this option multiple times.',
  325. # action='append')
  326. # upload_parser.add_argument(
  327. # '--destination', '-d',
  328. # help = 'Name to be used when saving in NodeMCU. You should specify one per file.',
  329. # action='append')
  330. upload_parser.add_argument('filename', nargs='+', help = 'Lua file to upload. Use colon to give alternate destination.')
  331. upload_parser.add_argument(
  332. '--compile', '-c',
  333. help = 'If file should be uploaded as compiled',
  334. action='store_true',
  335. default=False
  336. )
  337. upload_parser.add_argument(
  338. '--verify', '-v',
  339. help = 'To verify the uploaded data.',
  340. action='store',
  341. nargs='?',
  342. choices=['standard','sha1'],
  343. default='standard'
  344. )
  345. upload_parser.add_argument(
  346. '--dofile', '-e',
  347. help = 'If file should be run after upload.',
  348. action='store_true',
  349. default=False
  350. )
  351. upload_parser.add_argument(
  352. '--terminal', '-t',
  353. help = 'If miniterm should claim the port after all uploading is done.',
  354. action='store_true',
  355. default=False
  356. )
  357. upload_parser.add_argument(
  358. '--restart', '-r',
  359. help = 'If esp should be restarted',
  360. action='store_true',
  361. default=False
  362. )
  363. exec_parser = subparsers.add_parser(
  364. 'exec',
  365. help = 'Path to one or more files to be executed line by line.')
  366. exec_parser.add_argument('filename', nargs='+', help = 'Lua file to execute.')
  367. download_parser = subparsers.add_parser(
  368. 'download',
  369. help = 'Path to one or more files to be downloaded. Destination name will be the same as the file name.')
  370. # download_parser.add_argument(
  371. # '--filename', '-f',
  372. # help = 'File to download. You can specify this option multiple times.',
  373. # action='append')
  374. # download_parser.add_argument(
  375. # '--destination', '-d',
  376. # help = 'Name to be used when saving in NodeMCU. You should specify one per file.',
  377. # action='append')
  378. download_parser.add_argument('filename', nargs='+', help = 'Lua file to download. Use colon to give alternate destination.')
  379. file_parser = subparsers.add_parser(
  380. 'file',
  381. help = 'File functions')
  382. file_parser.add_argument('cmd', choices=('list', 'do', 'format', 'remove'))
  383. file_parser.add_argument('filename', nargs='*', help = 'Lua file to run.')
  384. node_parse = subparsers.add_parser(
  385. 'node',
  386. help = 'Node functions')
  387. node_parse.add_argument('ncmd', choices=('heap', 'restart'))
  388. args = parser.parse_args()
  389. formatter = logging.Formatter('%(message)s')
  390. logging.basicConfig(level=logging.INFO, format='%(message)s')
  391. if args.verbose:
  392. log.setLevel(logging.DEBUG)
  393. uploader = Uploader(args.port, args.baud)
  394. if args.operation == 'upload' or args.operation == 'download':
  395. sources = args.filename
  396. destinations = []
  397. for i in range(0, len(sources)):
  398. sd = sources[i].split(':')
  399. if len(sd) == 2:
  400. destinations.append(sd[1])
  401. sources[i]=sd[0]
  402. else:
  403. destinations.append(sd[0])
  404. if args.operation == 'upload':
  405. if len(destinations) == len(sources):
  406. uploader.prepare()
  407. for f, d in zip(sources, destinations):
  408. if args.compile:
  409. uploader.file_remove(os.path.splitext(d)[0]+'.lc')
  410. uploader.write_file(f, d, args.verify)
  411. if args.compile and d != 'init.lua':
  412. uploader.file_compile(d)
  413. uploader.file_remove(d)
  414. if args.dofile:
  415. uploader.file_do(os.path.splitext(d)[0]+'.lc')
  416. elif args.dofile:
  417. uploader.file_do(d)
  418. else:
  419. raise Exception('You must specify a destination filename for each file you want to upload.')
  420. if args.terminal:
  421. uploader.terminal()
  422. if args.restart:
  423. uploader.node_restart()
  424. log.info('All done!')
  425. if args.operation == 'download':
  426. if len(destinations) == len(sources):
  427. for f, d in zip(sources, destinations):
  428. uploader.read_file(f, d)
  429. else:
  430. raise Exception('You must specify a destination filename for each file you want to download.')
  431. log.info('All done!')
  432. elif args.operation == 'exec':
  433. sources = args.filename
  434. for f in sources:
  435. uploader.exec_file(f)
  436. elif args.operation == 'file':
  437. if args.cmd == 'list':
  438. uploader.file_list()
  439. if args.cmd == 'do':
  440. for f in args.filename:
  441. uploader.file_do(f)
  442. elif args.cmd == 'format':
  443. uploader.file_format()
  444. elif args.cmd == 'remove':
  445. for f in args.filename:
  446. uploader.file_remove(f)
  447. elif args.operation == 'node':
  448. if args.ncmd == 'heap':
  449. uploader.node_heap()
  450. elif args.ncmd == 'restart':
  451. uploader.node_restart()
  452. uploader.close()