Browse Source

unicode and LF

kmpm 8 years ago
parent
commit
f32e8a6d99
11 changed files with 705 additions and 691 deletions
  1. 33 33
      lib/luacode.py
  2. 239 239
      lib/main.py
  3. 50 50
      lib/term.py
  4. 307 304
      lib/uploader.py
  5. 13 13
      lib/utils.py
  6. 3 1
      lib/version.py
  7. 7 6
      nodemcu-uploader.py
  8. 2 0
      setup.py
  9. 16 14
      tests/__init__.py
  10. 18 16
      tests/misc.py
  11. 17 15
      tests/uploader.py

+ 33 - 33
lib/luacode.py

@@ -1,33 +1,33 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
-
-
-DOWNLOAD_FILE = "file.open('{filename}') print(file.seek('end', 0)) file.seek('set', {bytes_read}) uart.write(0, file.read({chunk_size}))file.close()"
-
-LIST_FILES = 'for key,value in pairs(file.list()) do print(key,value) end'
-
-SAVE_LUA = \
-r"""
-function recv_block(d)
-  if string.byte(d, 1) == 1 then
-    size = string.byte(d, 2)
-    uart.write(0,'\006')
-    if size > 0 then
-      file.write(string.sub(d, 3, 3+size-1))
-    else
-      file.close()
-      uart.on('data')
-      uart.setup(0,{baud},8,0,1,1)
-    end
-  else
-    uart.write(0, '\021' .. d)
-    uart.setup(0,{baud},8,0,1,1)
-    uart.on('data')
-  end
-end
-function recv_name(d) d = string.gsub(d, '\000', '') file.remove(d) file.open(d, 'w') uart.on('data', 130, recv_block, 0) uart.write(0, '\006') end
-function recv() uart.setup(0,{baud},8,0,1,0) uart.on('data', '\000', recv_name, 0) uart.write(0, 'C') end
-function shafile(f) file.open(f, "r") print(crypto.toHex(crypto.hash("sha1",file.read()))) file.close() end
-"""
-UART_SETUP = 'uart.setup(0,{baud},8,0,1,1)'
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
+
+
+DOWNLOAD_FILE = "file.open('{filename}') print(file.seek('end', 0)) file.seek('set', {bytes_read}) uart.write(0, file.read({chunk_size}))file.close()"
+
+LIST_FILES = 'for key,value in pairs(file.list()) do print(key,value) end'
+
+SAVE_LUA = \
+r"""
+function recv_block(d)
+  if string.byte(d, 1) == 1 then
+    size = string.byte(d, 2)
+    uart.write(0,'\006')
+    if size > 0 then
+      file.write(string.sub(d, 3, 3+size-1))
+    else
+      file.close()
+      uart.on('data')
+      uart.setup(0,{baud},8,0,1,1)
+    end
+  else
+    uart.write(0, '\021' .. d)
+    uart.setup(0,{baud},8,0,1,1)
+    uart.on('data')
+  end
+end
+function recv_name(d) d = string.gsub(d, '\000', '') file.remove(d) file.open(d, 'w') uart.on('data', 130, recv_block, 0) uart.write(0, '\006') end
+function recv() uart.setup(0,{baud},8,0,1,0) uart.on('data', '\000', recv_name, 0) uart.write(0, 'C') end
+function shafile(f) file.open(f, "r") print(crypto.toHex(crypto.hash("sha1",file.read()))) file.close() end
+"""
+UART_SETUP = 'uart.setup(0,{baud},8,0,1,1)'

+ 239 - 239
lib/main.py

@@ -1,239 +1,239 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
-
-import argparse
-import logging
-import os
-from .uploader import Uploader
-from .term import terminal
-from serial import VERSION as serialversion
-
-log = logging.getLogger(__name__)
-from .version import __version__
-
-def destination_from_source(sources):
-    """
-    Split each of the sources in the array on ':'
-    First part will be source, second will be destination.
-    Modifies the the original array to contain only sources
-    and returns an array of destinations.
-    """
-    destinations = []
-    for i in range(0, len(sources)):
-        sd = sources[i].split(':')
-        if len(sd) == 2:
-            destinations.append(sd[1])
-            sources[i] = sd[0]
-        else:
-            destinations.append(sd[0])
-    return destinations
-
-
-def operation_upload(uploader, sources, verify, do_compile, do_file, do_restart):
-    """The upload operation"""
-    destinations = destination_from_source(sources)
-    if len(destinations) == len(sources):
-        uploader.prepare()
-        for f, d in zip(sources, destinations):
-            if do_compile:
-                uploader.file_remove(os.path.splitext(d)[0]+'.lc')
-            uploader.write_file(f, d, verify)
-            if do_compile and d != 'init.lua':
-                uploader.file_compile(d)
-                uploader.file_remove(d)
-                if do_file:
-                    uploader.file_do(os.path.splitext(d)[0]+'.lc')
-            elif do_file:
-                uploader.file_do(d)
-    else:
-        raise Exception('You must specify a destination filename for each file you want to upload.')
-
-    if do_restart:
-        uploader.node_restart()
-    log.info('All done!')
-
-
-def operation_download(uploader, sources):
-    """The download operation"""
-    destinations = destination_from_source(sources)
-    if len(destinations) == len(sources):
-        for f, d in zip(sources, destinations):
-            uploader.read_file(f, d)
-    else:
-        raise Exception('You must specify a destination filename for each file you want to download.')
-    log.info('All done!')
-
-
-def operation_file(uploader, cmd, filename=''):
-    """File operations"""
-    if cmd == 'list':
-        uploader.file_list()
-    if cmd == 'do':
-        for f in filename:
-            uploader.file_do(f)
-    elif cmd == 'format':
-        uploader.file_format()
-    elif cmd == 'remove':
-        for f in filename:
-            uploader.file_remove(f)
-
-
-
-def arg_auto_int(value):
-    """parsing function for integer arguments"""
-    return int(value, 0)
-
-
-def main_func():
-    parser = argparse.ArgumentParser(
-        description='NodeMCU Lua file uploader',
-        prog='nodemcu-uploader'
-        )
-
-    parser.add_argument(
-        '--verbose',
-        help='verbose output',
-        action='store_true',
-        default=False)
-
-    parser.add_argument(
-        '--version',
-        help='prints the version and exists',
-        action='version',
-        version='%(prog)s {version} (serial {serialversion})'.format(version=__version__, serialversion=serialversion)
-    )
-
-    parser.add_argument(
-        '--port', '-p',
-        help='Serial port device',
-        default=Uploader.PORT)
-
-    parser.add_argument(
-        '--baud', '-b',
-        help='Serial port baudrate',
-        type=arg_auto_int,
-        default=Uploader.BAUD)
-
-    subparsers = parser.add_subparsers(
-        dest='operation',
-        help='Run nodemcu-uploader {command} -h for additional help')
-
-    upload_parser = subparsers.add_parser(
-        'upload',
-        help='Path to one or more files to be uploaded. Destination name will be the same as the file name.')
-
-    upload_parser.add_argument(
-        'filename',
-        nargs='+',
-        help='Lua file to upload. Use colon to give alternate destination.'
-        )
-
-    upload_parser.add_argument(
-        '--compile', '-c',
-        help='If file should be uploaded as compiled',
-        action='store_true',
-        default=False
-        )
-
-    upload_parser.add_argument(
-        '--verify', '-v',
-        help='To verify the uploaded data.',
-        action='store',
-        nargs='?',
-        choices=['standard', 'sha1'],
-        default='standard'
-        )
-
-    upload_parser.add_argument(
-        '--dofile', '-e',
-        help='If file should be run after upload.',
-        action='store_true',
-        default=False
-        )
-
-    upload_parser.add_argument(
-        '--restart', '-r',
-        help='If esp should be restarted',
-        action='store_true',
-        default=False
-    )
-
-    exec_parser = subparsers.add_parser(
-        'exec',
-        help='Path to one or more files to be executed line by line.')
-
-    exec_parser.add_argument('filename', nargs='+', help='Lua file to execute.')
-
-    download_parser = subparsers.add_parser(
-        'download',
-        help='Path to one or more files to be downloaded. Destination name will be the same as the file name.')
-
-    download_parser.add_argument('filename', nargs='+', help='Lua file to download. Use colon to give alternate destination.')
-
-
-    file_parser = subparsers.add_parser(
-        'file',
-        help='File functions')
-
-    file_parser.add_argument(
-        'cmd',
-        choices=('list', 'do', 'format', 'remove'),
-        help="list=list files, do=dofile given path, format=formate file area, remove=remove given path")
-
-    file_parser.add_argument('filename', nargs='*', help='path for cmd')
-
-    node_parse = subparsers.add_parser(
-        'node',
-        help='Node functions')
-
-    node_parse.add_argument('ncmd', choices=('heap', 'restart'), help="heap=print heap memory, restart=restart nodemcu")
-
-    terminal_parser = subparsers.add_parser(
-        'terminal',
-        help='Run pySerials miniterm'
-    )
-
-    args = parser.parse_args()
-
-    default_level = logging.INFO
-    if args.verbose:
-        default_level = logging.DEBUG
-
-    #formatter = logging.Formatter('%(message)s')
-
-    logging.basicConfig(level=default_level, format='%(message)s')
-
-    uploader = Uploader(args.port, args.baud)
-
-
-
-    if args.operation == 'upload':
-        operation_upload(uploader, args.filename, args.verify, args.compile, args.dofile,
-                         args.restart)
-
-    elif args.operation == 'download':
-        operation_download(uploader, args.filename)
-
-    elif args.operation == 'exec':
-        sources = args.filename
-        for f in sources:
-            uploader.exec_file(f)
-
-    elif args.operation == 'file':
-        operation_file(uploader, args.cmd, args.filename)
-
-    elif args.operation == 'node':
-        if args.ncmd == 'heap':
-            uploader.node_heap()
-        elif args.ncmd == 'restart':
-            uploader.node_restart()
-    #no uploader related commands after this point
-    uploader.close()
-
-    if args.operation == 'terminal':
-        #uploader can not claim the port
-        uploader.close()
-        terminal(args.port)
-
-
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
+
+import argparse
+import logging
+import os
+from .uploader import Uploader
+from .term import terminal
+from serial import VERSION as serialversion
+
+log = logging.getLogger(__name__)
+from .version import __version__
+
+def destination_from_source(sources):
+    """
+    Split each of the sources in the array on ':'
+    First part will be source, second will be destination.
+    Modifies the the original array to contain only sources
+    and returns an array of destinations.
+    """
+    destinations = []
+    for i in range(0, len(sources)):
+        sd = sources[i].split(':')
+        if len(sd) == 2:
+            destinations.append(sd[1])
+            sources[i] = sd[0]
+        else:
+            destinations.append(sd[0])
+    return destinations
+
+
+def operation_upload(uploader, sources, verify, do_compile, do_file, do_restart):
+    """The upload operation"""
+    destinations = destination_from_source(sources)
+    if len(destinations) == len(sources):
+        uploader.prepare()
+        for f, d in zip(sources, destinations):
+            if do_compile:
+                uploader.file_remove(os.path.splitext(d)[0]+'.lc')
+            uploader.write_file(f, d, verify)
+            if do_compile and d != 'init.lua':
+                uploader.file_compile(d)
+                uploader.file_remove(d)
+                if do_file:
+                    uploader.file_do(os.path.splitext(d)[0]+'.lc')
+            elif do_file:
+                uploader.file_do(d)
+    else:
+        raise Exception('You must specify a destination filename for each file you want to upload.')
+
+    if do_restart:
+        uploader.node_restart()
+    log.info('All done!')
+
+
+def operation_download(uploader, sources):
+    """The download operation"""
+    destinations = destination_from_source(sources)
+    if len(destinations) == len(sources):
+        for f, d in zip(sources, destinations):
+            uploader.read_file(f, d)
+    else:
+        raise Exception('You must specify a destination filename for each file you want to download.')
+    log.info('All done!')
+
+
+def operation_file(uploader, cmd, filename=''):
+    """File operations"""
+    if cmd == 'list':
+        uploader.file_list()
+    if cmd == 'do':
+        for f in filename:
+            uploader.file_do(f)
+    elif cmd == 'format':
+        uploader.file_format()
+    elif cmd == 'remove':
+        for f in filename:
+            uploader.file_remove(f)
+
+
+
+def arg_auto_int(value):
+    """parsing function for integer arguments"""
+    return int(value, 0)
+
+
+def main_func():
+    parser = argparse.ArgumentParser(
+        description='NodeMCU Lua file uploader',
+        prog='nodemcu-uploader'
+        )
+
+    parser.add_argument(
+        '--verbose',
+        help='verbose output',
+        action='store_true',
+        default=False)
+
+    parser.add_argument(
+        '--version',
+        help='prints the version and exists',
+        action='version',
+        version='%(prog)s {version} (serial {serialversion})'.format(version=__version__, serialversion=serialversion)
+    )
+
+    parser.add_argument(
+        '--port', '-p',
+        help='Serial port device',
+        default=Uploader.PORT)
+
+    parser.add_argument(
+        '--baud', '-b',
+        help='Serial port baudrate',
+        type=arg_auto_int,
+        default=Uploader.BAUD)
+
+    subparsers = parser.add_subparsers(
+        dest='operation',
+        help='Run nodemcu-uploader {command} -h for additional help')
+
+    upload_parser = subparsers.add_parser(
+        'upload',
+        help='Path to one or more files to be uploaded. Destination name will be the same as the file name.')
+
+    upload_parser.add_argument(
+        'filename',
+        nargs='+',
+        help='Lua file to upload. Use colon to give alternate destination.'
+        )
+
+    upload_parser.add_argument(
+        '--compile', '-c',
+        help='If file should be uploaded as compiled',
+        action='store_true',
+        default=False
+        )
+
+    upload_parser.add_argument(
+        '--verify', '-v',
+        help='To verify the uploaded data.',
+        action='store',
+        nargs='?',
+        choices=['standard', 'sha1'],
+        default='standard'
+        )
+
+    upload_parser.add_argument(
+        '--dofile', '-e',
+        help='If file should be run after upload.',
+        action='store_true',
+        default=False
+        )
+
+    upload_parser.add_argument(
+        '--restart', '-r',
+        help='If esp should be restarted',
+        action='store_true',
+        default=False
+    )
+
+    exec_parser = subparsers.add_parser(
+        'exec',
+        help='Path to one or more files to be executed line by line.')
+
+    exec_parser.add_argument('filename', nargs='+', help='Lua file to execute.')
+
+    download_parser = subparsers.add_parser(
+        'download',
+        help='Path to one or more files to be downloaded. Destination name will be the same as the file name.')
+
+    download_parser.add_argument('filename', nargs='+', help='Lua file to download. Use colon to give alternate destination.')
+
+
+    file_parser = subparsers.add_parser(
+        'file',
+        help='File functions')
+
+    file_parser.add_argument(
+        'cmd',
+        choices=('list', 'do', 'format', 'remove'),
+        help="list=list files, do=dofile given path, format=formate file area, remove=remove given path")
+
+    file_parser.add_argument('filename', nargs='*', help='path for cmd')
+
+    node_parse = subparsers.add_parser(
+        'node',
+        help='Node functions')
+
+    node_parse.add_argument('ncmd', choices=('heap', 'restart'), help="heap=print heap memory, restart=restart nodemcu")
+
+    terminal_parser = subparsers.add_parser(
+        'terminal',
+        help='Run pySerials miniterm'
+    )
+
+    args = parser.parse_args()
+
+    default_level = logging.INFO
+    if args.verbose:
+        default_level = logging.DEBUG
+
+    #formatter = logging.Formatter('%(message)s')
+
+    logging.basicConfig(level=default_level, format='%(message)s')
+
+    uploader = Uploader(args.port, args.baud)
+
+
+
+    if args.operation == 'upload':
+        operation_upload(uploader, args.filename, args.verify, args.compile, args.dofile,
+                         args.restart)
+
+    elif args.operation == 'download':
+        operation_download(uploader, args.filename)
+
+    elif args.operation == 'exec':
+        sources = args.filename
+        for f in sources:
+            uploader.exec_file(f)
+
+    elif args.operation == 'file':
+        operation_file(uploader, args.cmd, args.filename)
+
+    elif args.operation == 'node':
+        if args.ncmd == 'heap':
+            uploader.node_heap()
+        elif args.ncmd == 'restart':
+            uploader.node_restart()
+    #no uploader related commands after this point
+    uploader.close()
+
+    if args.operation == 'terminal':
+        #uploader can not claim the port
+        uploader.close()
+        terminal(args.port)
+
+

+ 50 - 50
lib/term.py

@@ -1,51 +1,51 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from .utils import default_port
-try:
-    from serial.tools.miniterm import Miniterm, console, NEWLINE_CONVERISON_MAP
-    import serial
-    MINITERM_AVAILABLE=True
-except ImportError:
-    
-    MINITERM_AVAILABLE=False
-    
-    
-class McuMiniterm(Miniterm):
-    def __init__(self, serial):
-        if not MINITERM_AVAILABLE:
-            print "Miniterm is not available on this system"
-            return 
-        self.serial = serial
-        self.echo = False
-        self.convert_outgoing = 2
-        self.repr_mode = 1
-        self.newline = NEWLINE_CONVERISON_MAP[self.convert_outgoing]
-        self.dtr_state = True
-        self.rts_state = True
-        self.break_state = False
-        
-        
-def terminal(port=default_port()):
-    if not MINITERM_AVAILABLE:
-        print "Miniterm is not available on this system"
-        return False
-    sp = serial.Serial(port, 9600)
-
-    # Keeps things working, if following conections are made:
-    ## RTS = CH_PD (i.e reset)
-    ## DTR = GPIO0
-    sp.setRTS(False)
-    sp.setDTR(False)
-    miniterm = McuMiniterm(sp)
-
-    log.info('Started terminal. Hit ctrl-] to leave terminal')
-
-    console.setup()
-    miniterm.start()
-    try:
-            miniterm.join(True)
-    except KeyboardInterrupt:
-            pass
-    miniterm.join()
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from .utils import default_port
+try:
+    from serial.tools.miniterm import Miniterm, console, NEWLINE_CONVERISON_MAP
+    import serial
+    MINITERM_AVAILABLE=True
+except ImportError:
+
+    MINITERM_AVAILABLE=False
+
+
+class McuMiniterm(Miniterm):
+    def __init__(self, serial):
+        if not MINITERM_AVAILABLE:
+            print "Miniterm is not available on this system"
+            return
+        self.serial = serial
+        self.echo = False
+        self.convert_outgoing = 2
+        self.repr_mode = 1
+        self.newline = NEWLINE_CONVERISON_MAP[self.convert_outgoing]
+        self.dtr_state = True
+        self.rts_state = True
+        self.break_state = False
+
+
+def terminal(port=default_port()):
+    if not MINITERM_AVAILABLE:
+        print "Miniterm is not available on this system"
+        return False
+    sp = serial.Serial(port, 9600)
+
+    # Keeps things working, if following conections are made:
+    ## RTS = CH_PD (i.e reset)
+    ## DTR = GPIO0
+    sp.setRTS(False)
+    sp.setDTR(False)
+    miniterm = McuMiniterm(sp)
+
+    log.info('Started terminal. Hit ctrl-] to leave terminal')
+
+    console.setup()
+    miniterm.start()
+    try:
+            miniterm.join(True)
+    except KeyboardInterrupt:
+            pass
+    miniterm.join()
     sp.close()

+ 307 - 304
lib/uploader.py

@@ -1,304 +1,307 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
-
-import time
-import logging
-import hashlib
-import os
-import serial
-from .utils import default_port
-from .luacode import SAVE_LUA, LIST_FILES, UART_SETUP
-
-log = logging.getLogger(__name__)
-
-__all__ = ['Uploader', 'default_port']
-
-
-class Uploader(object):
-    """Uploader is the class for communicating with the nodemcu and
-    that will allow various tasks like uploading files, formating the filesystem etc.
-    """
-    BAUD = 9600
-    TIMEOUT = 5
-    PORT = default_port()
-
-    def __init__(self, port=PORT, baud=BAUD):
-        log.info('opening port %s', port)
-        if port == 'loop://':
-            self._port = serial.serial_for_url(port, baud, timeout=Uploader.TIMEOUT)
-        else:
-            self._port = serial.Serial(port, baud, timeout=Uploader.TIMEOUT)
-
-        # Keeps things working, if following conections are made:
-        ## RTS = CH_PD (i.e reset)
-        ## DTR = GPIO0
-        self._port.setRTS(False)
-        self._port.setDTR(False)
-
-        def sync():
-            # Get in sync with LUA (this assumes that NodeMCU gets reset by the previous two lines)
-            log.debug('getting in sync with LUA');
-            self.exchange(';') # Get a defined state
-            self.writeln('print("%sync%");')
-            self.expect('%sync%\r\n> ')
-        sync()
-        if baud != Uploader.BAUD:
-            log.info('Changing communication to %s baud', baud)
-            self.writeln(UART_SETUP.format(baud=baud))
-
-            # Wait for the string to be sent before switching baud
-            time.sleep(0.1)
-            self.set_baudrate(baud)
-
-            # Get in sync again
-            sync()
-
-        self.line_number = 0
-
-    def set_baudrate(self, baud):
-        try:
-            self._port.setBaudrate(baud)
-        except AttributeError:
-            self._port.baudrate = baud
-
-    def expect(self, exp='> ', timeout=TIMEOUT):
-        """will wait for exp to be returned from nodemcu or timeout"""
-        timer = self._port.timeout
-
-        # Checking for new data every 100us is fast enough
-        lt = 0.0001
-        if self._port.timeout != lt:
-            self._port.timeout = lt
-
-        end = time.time() + timeout
-
-        # Finish as soon as either exp matches or we run out of time (work like dump, but faster on success)
-        data = ''
-        while not data.endswith(exp) and time.time() <= end:
-            data += self._port.read()
-
-        self._port.timeout = timer
-        log.debug('expect return: %s', data)
-        return data
-
-    def write(self, output, binary=False):
-        """write data on the nodemcu port. If 'binary' is True the debug log
-        will show the intended output as hex, otherwise as string"""
-        if not binary:
-            log.debug('write: %s', output)
-        else:
-            log.debug('write binary: %s', ':'.join(x.encode('hex') for x in output))
-        self._port.write(output)
-        self._port.flush()
-
-    def writeln(self, output):
-        """write, with linefeed"""
-        self.write(output + '\n')
-
-    def exchange(self, output):
-        self.writeln(output)
-        return self.expect()
-
-    def close(self):
-        """restores the nodemcu to default baudrate and then closes the port"""
-        self.writeln(UART_SETUP.format(baud=Uploader.BAUD))
-        self._port.close()
-
-    def prepare(self):
-        """
-        This uploads the protocol functions nessecary to do binary
-        chunked transfer
-        """
-        log.info('Preparing esp for transfer.')
-
-        data = SAVE_LUA.format(baud=self._port.baudrate)
-        ##change any \r\n to just \n and split on that
-        lines = data.replace('\r', '').split('\n')
-
-        #remove some unneccesary spaces to conserve some bytes
-        for line in lines:
-            line = line.strip().replace(', ', ',').replace(' = ', '=')
-
-            if len(line) == 0:
-                continue
-
-            d = self.exchange(line)
-            #do some basic test of the result
-            if 'unexpected' in d or len(d) > len(SAVE_LUA)+10:
-                log.error('error in save_lua "%s"', d)
-                return
-
-    def download_file(self, filename):
-        chunk_size = 256
-        bytes_read = 0
-        data = ""
-        while True:
-            d = self.exchange(DOWNLOAD_FILE.format(filename=filename, bytes_read=bytes_read, chunk_size=chunk_size))
-            cmd, size, tmp_data = d.split('\n', 2)
-            data = data + tmp_data[0:chunk_size]
-            bytes_read = bytes_read + chunk_size
-            if bytes_read > int(size):
-                break
-        data = data[0:int(size)]
-        return data
-
-    def read_file(self, filename, destination=''):
-        if not destination:
-            destination = filename
-        log.info('Transfering %s to %s', filename, destination)
-        data = self.download_file(filename)
-        with open(destination, 'w') as f:
-            f.write(data)
-
-    def write_file(self, path, destination='', verify='none'):
-        filename = os.path.basename(path)
-        if not destination:
-            destination = filename
-        log.info('Transfering %s as %s', path, destination)
-        self.writeln("recv()")
-
-        res = self.expect('C> ')
-        if not res.endswith('C> '):
-            log.error('Error waiting for esp "%s"', res)
-            return
-        log.debug('sending destination filename "%s"', destination)
-        self.write(destination + '\x00', True)
-        if not self.got_ack():
-            log.error('did not ack destination filename')
-            return
-
-        f = open(path, 'rb')
-        content = f.read()
-        f.close()
-
-        log.debug('sending %d bytes in %s', len(content), filename)
-        pos = 0
-        chunk_size = 128
-        while pos < len(content):
-            rest = len(content) - pos
-            if rest > chunk_size:
-                rest = chunk_size
-
-            data = content[pos:pos+rest]
-            if not self.write_chunk(data):
-                d = self.expect()
-                log.error('Bad chunk response "%s" %s', d, ':'.join(x.encode('hex') for x in d))
-                return
-
-            pos += chunk_size
-
-        log.debug('sending zero block')
-        #zero size block
-        self.write_chunk('')
-
-        if verify == 'standard':
-            log.info('Verifying...')
-            data = self.download_file(destination)
-            if content != data:
-                log.error('Verification failed.')
-        elif verify == 'sha1':
-            #Calculate SHA1 on remote file. Extract just hash from result
-            data = self.exchange('shafile("'+destination+'")').splitlines()[1]
-            log.info('Remote SHA1: %s', data)
-
-            #Calculate hash of local data
-            filehashhex = hashlib.sha1(content.encode()).hexdigest()
-            log.info('Local SHA1: %s', filehashhex)
-            if data != filehashhex:
-                log.error('Verification failed.')
-
-    def exec_file(self, path):
-        filename = os.path.basename(path)
-        log.info('Execute %s', filename)
-
-        f = open(path, 'rt')
-
-        res = '> '
-        for line in f:
-            line = line.rstrip('\r\n')
-            retlines = (res + self.exchange(line)).splitlines()
-            # Log all but the last line
-            res = retlines.pop()
-            for lin in retlines:
-                log.info(lin)
-        # last line
-        log.info(res)
-        f.close()
-
-    def got_ack(self):
-        log.debug('waiting for ack')
-        res = self._port.read(1)
-        log.debug('ack read %s', res.encode('hex'))
-        return res == '\x06' #ACK
-
-
-    def write_lines(self, data):
-        lines = data.replace('\r', '').split('\n')
-
-        for line in lines:
-            self.exchange(line)
-
-        return
-
-
-    def write_chunk(self, chunk):
-        log.debug('writing %d bytes chunk', len(chunk))
-        data = '\x01' + chr(len(chunk)) + chunk
-        if len(chunk) < 128:
-            padding = 128 - len(chunk)
-            log.debug('pad with %d characters', padding)
-            data = data + (' ' * padding)
-        log.debug("packet size %d", len(data))
-        self.write(data)
-
-        return self.got_ack()
-
-
-    def file_list(self):
-        log.info('Listing files')
-        res = self.exchange(LIST_FILES)
-        log.info(res)
-        return res
-
-    def file_do(self, f):
-        log.info('Executing '+f)
-        res = self.exchange('dofile("'+f+'")')
-        log.info(res)
-        return res
-
-    def file_format(self):
-        log.info('Formating...')
-        res = self.exchange('file.format()')
-        if 'format done' not in res:
-            log.error(res)
-        else:
-            log.info(res)
-        return res
-
-    def node_heap(self):
-        log.info('Heap')
-        res = self.exchange('print(node.heap())')
-        log.info(res)
-        return res
-
-    def node_restart(self):
-        log.info('Restart')
-        res = self.exchange('node.restart()')
-        log.info(res)
-        return res
-
-    def file_compile(self, path):
-        log.info('Compile '+path)
-        cmd = 'node.compile("%s")' % path
-        res = self.exchange(cmd)
-        log.info(res)
-        return res
-
-    def file_remove(self, path):
-        log.info('Remove '+path)
-        cmd = 'file.remove("%s")' % path
-        res = self.exchange(cmd)
-        log.info(res)
-        return res
-
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
+
+import time
+import logging
+import hashlib
+import os
+import serial
+# -*- coding: utf-8 -*-
+# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
+
+from .utils import default_port
+from .luacode import SAVE_LUA, LIST_FILES, UART_SETUP
+
+log = logging.getLogger(__name__)
+
+__all__ = ['Uploader', 'default_port']
+
+
+class Uploader(object):
+    """Uploader is the class for communicating with the nodemcu and
+    that will allow various tasks like uploading files, formating the filesystem etc.
+    """
+    BAUD = 9600
+    TIMEOUT = 5
+    PORT = default_port()
+
+    def __init__(self, port=PORT, baud=BAUD):
+        log.info('opening port %s', port)
+        if port == 'loop://':
+            self._port = serial.serial_for_url(port, baud, timeout=Uploader.TIMEOUT)
+        else:
+            self._port = serial.Serial(port, baud, timeout=Uploader.TIMEOUT)
+
+        # Keeps things working, if following conections are made:
+        ## RTS = CH_PD (i.e reset)
+        ## DTR = GPIO0
+        self._port.setRTS(False)
+        self._port.setDTR(False)
+
+        def sync():
+            # Get in sync with LUA (this assumes that NodeMCU gets reset by the previous two lines)
+            log.debug('getting in sync with LUA');
+            self.exchange(';') # Get a defined state
+            self.writeln('print("%sync%");')
+            self.expect('%sync%\r\n> ')
+        sync()
+        if baud != Uploader.BAUD:
+            log.info('Changing communication to %s baud', baud)
+            self.writeln(UART_SETUP.format(baud=baud))
+
+            # Wait for the string to be sent before switching baud
+            time.sleep(0.1)
+            self.set_baudrate(baud)
+
+            # Get in sync again
+            sync()
+
+        self.line_number = 0
+
+    def set_baudrate(self, baud):
+        try:
+            self._port.setBaudrate(baud)
+        except AttributeError:
+            self._port.baudrate = baud
+
+    def expect(self, exp='> ', timeout=TIMEOUT):
+        """will wait for exp to be returned from nodemcu or timeout"""
+        timer = self._port.timeout
+
+        # Checking for new data every 100us is fast enough
+        lt = 0.0001
+        if self._port.timeout != lt:
+            self._port.timeout = lt
+
+        end = time.time() + timeout
+
+        # Finish as soon as either exp matches or we run out of time (work like dump, but faster on success)
+        data = ''
+        while not data.endswith(exp) and time.time() <= end:
+            data += self._port.read()
+
+        self._port.timeout = timer
+        log.debug('expect return: %s', data)
+        return data
+
+    def write(self, output, binary=False):
+        """write data on the nodemcu port. If 'binary' is True the debug log
+        will show the intended output as hex, otherwise as string"""
+        if not binary:
+            log.debug('write: %s', output)
+        else:
+            log.debug('write binary: %s', ':'.join(x.encode('hex') for x in output))
+        self._port.write(output)
+        self._port.flush()
+
+    def writeln(self, output):
+        """write, with linefeed"""
+        self.write(output + '\n')
+
+    def exchange(self, output):
+        self.writeln(output)
+        return self.expect()
+
+    def close(self):
+        """restores the nodemcu to default baudrate and then closes the port"""
+        self.writeln(UART_SETUP.format(baud=Uploader.BAUD))
+        self._port.close()
+
+    def prepare(self):
+        """
+        This uploads the protocol functions nessecary to do binary
+        chunked transfer
+        """
+        log.info('Preparing esp for transfer.')
+
+        data = SAVE_LUA.format(baud=self._port.baudrate)
+        ##change any \r\n to just \n and split on that
+        lines = data.replace('\r', '').split('\n')
+
+        #remove some unneccesary spaces to conserve some bytes
+        for line in lines:
+            line = line.strip().replace(', ', ',').replace(' = ', '=')
+
+            if len(line) == 0:
+                continue
+
+            d = self.exchange(line)
+            #do some basic test of the result
+            if 'unexpected' in d or len(d) > len(SAVE_LUA)+10:
+                log.error('error in save_lua "%s"', d)
+                return
+
+    def download_file(self, filename):
+        chunk_size = 256
+        bytes_read = 0
+        data = ""
+        while True:
+            d = self.exchange(DOWNLOAD_FILE.format(filename=filename, bytes_read=bytes_read, chunk_size=chunk_size))
+            cmd, size, tmp_data = d.split('\n', 2)
+            data = data + tmp_data[0:chunk_size]
+            bytes_read = bytes_read + chunk_size
+            if bytes_read > int(size):
+                break
+        data = data[0:int(size)]
+        return data
+
+    def read_file(self, filename, destination=''):
+        if not destination:
+            destination = filename
+        log.info('Transfering %s to %s', filename, destination)
+        data = self.download_file(filename)
+        with open(destination, 'w') as f:
+            f.write(data)
+
+    def write_file(self, path, destination='', verify='none'):
+        filename = os.path.basename(path)
+        if not destination:
+            destination = filename
+        log.info('Transfering %s as %s', path, destination)
+        self.writeln("recv()")
+
+        res = self.expect('C> ')
+        if not res.endswith('C> '):
+            log.error('Error waiting for esp "%s"', res)
+            return
+        log.debug('sending destination filename "%s"', destination)
+        self.write(destination + '\x00', True)
+        if not self.got_ack():
+            log.error('did not ack destination filename')
+            return
+
+        f = open(path, 'rb')
+        content = f.read()
+        f.close()
+
+        log.debug('sending %d bytes in %s', len(content), filename)
+        pos = 0
+        chunk_size = 128
+        while pos < len(content):
+            rest = len(content) - pos
+            if rest > chunk_size:
+                rest = chunk_size
+
+            data = content[pos:pos+rest]
+            if not self.write_chunk(data):
+                d = self.expect()
+                log.error('Bad chunk response "%s" %s', d, ':'.join(x.encode('hex') for x in d))
+                return
+
+            pos += chunk_size
+
+        log.debug('sending zero block')
+        #zero size block
+        self.write_chunk('')
+
+        if verify == 'standard':
+            log.info('Verifying...')
+            data = self.download_file(destination)
+            if content != data:
+                log.error('Verification failed.')
+        elif verify == 'sha1':
+            #Calculate SHA1 on remote file. Extract just hash from result
+            data = self.exchange('shafile("'+destination+'")').splitlines()[1]
+            log.info('Remote SHA1: %s', data)
+
+            #Calculate hash of local data
+            filehashhex = hashlib.sha1(content.encode()).hexdigest()
+            log.info('Local SHA1: %s', filehashhex)
+            if data != filehashhex:
+                log.error('Verification failed.')
+
+    def exec_file(self, path):
+        filename = os.path.basename(path)
+        log.info('Execute %s', filename)
+
+        f = open(path, 'rt')
+
+        res = '> '
+        for line in f:
+            line = line.rstrip('\r\n')
+            retlines = (res + self.exchange(line)).splitlines()
+            # Log all but the last line
+            res = retlines.pop()
+            for lin in retlines:
+                log.info(lin)
+        # last line
+        log.info(res)
+        f.close()
+
+    def got_ack(self):
+        log.debug('waiting for ack')
+        res = self._port.read(1)
+        log.debug('ack read %s', res.encode('hex'))
+        return res == '\x06' #ACK
+
+
+    def write_lines(self, data):
+        lines = data.replace('\r', '').split('\n')
+
+        for line in lines:
+            self.exchange(line)
+
+        return
+
+
+    def write_chunk(self, chunk):
+        log.debug('writing %d bytes chunk', len(chunk))
+        data = '\x01' + chr(len(chunk)) + chunk
+        if len(chunk) < 128:
+            padding = 128 - len(chunk)
+            log.debug('pad with %d characters', padding)
+            data = data + (' ' * padding)
+        log.debug("packet size %d", len(data))
+        self.write(data)
+
+        return self.got_ack()
+
+
+    def file_list(self):
+        log.info('Listing files')
+        res = self.exchange(LIST_FILES)
+        log.info(res)
+        return res
+
+    def file_do(self, f):
+        log.info('Executing '+f)
+        res = self.exchange('dofile("'+f+'")')
+        log.info(res)
+        return res
+
+    def file_format(self):
+        log.info('Formating...')
+        res = self.exchange('file.format()')
+        if 'format done' not in res:
+            log.error(res)
+        else:
+            log.info(res)
+        return res
+
+    def node_heap(self):
+        log.info('Heap')
+        res = self.exchange('print(node.heap())')
+        log.info(res)
+        return res
+
+    def node_restart(self):
+        log.info('Restart')
+        res = self.exchange('node.restart()')
+        log.info(res)
+        return res
+
+    def file_compile(self, path):
+        log.info('Compile '+path)
+        cmd = 'node.compile("%s")' % path
+        res = self.exchange(cmd)
+        log.info(res)
+        return res
+
+    def file_remove(self, path):
+        log.info('Remove '+path)
+        cmd = 'file.remove("%s")' % path
+        res = self.exchange(cmd)
+        log.info(res)
+        return res
+

+ 13 - 13
lib/utils.py

@@ -1,13 +1,13 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
-
-from platform import system
-
-def default_port(sysname = system()):
-    """This returns the default port used for different systems"""
-    return {
-        'Windows': 'COM1',
-        'Darwin': '/dev/tty.SLAB_USBtoUART'
-    }.get(sysname, '/dev/ttyUSB0')
-
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
+
+from platform import system
+
+def default_port(sysname = system()):
+    """This returns the default port used for different systems"""
+    return {
+        'Windows': 'COM1',
+        'Darwin': '/dev/tty.SLAB_USBtoUART'
+    }.get(sysname, '/dev/ttyUSB0')
+

+ 3 - 1
lib/version.py

@@ -1 +1,3 @@
-__version__ = '0.2.0'
+# -*- coding: utf-8 -*-
+# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
+__version__ = '0.2.0'

+ 7 - 6
nodemcu-uploader.py

@@ -1,6 +1,7 @@
-from lib import main
-
-
-if __name__ == '__main__':
-    main.main_func()
-    
+# -*- coding: utf-8 -*-
+# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
+from lib import main
+
+
+if __name__ == '__main__':
+    main.main_func()

+ 2 - 0
setup.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
 from setuptools import setup
 
 exec(open('lib/version.py').read())

+ 16 - 14
tests/__init__.py

@@ -1,14 +1,16 @@
-import unittest
-
-def get_tests():
-    return full_suite()
-
-def full_suite():
-    from .misc import MiscTestCase
-    from .uploader import UploaderTestCase
-    # from .serializer import ResourceTestCase as SerializerTestCase
-    # from .utils import UtilsTestCase
-
-    miscsuite = unittest.TestLoader().loadTestsFromTestCase(MiscTestCase)
-    uploadersuite = unittest.TestLoader().loadTestsFromTestCase(UploaderTestCase)
-    return unittest.TestSuite([miscsuite, uploadersuite])
+# -*- coding: utf-8 -*-
+# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
+import unittest
+
+def get_tests():
+    return full_suite()
+
+def full_suite():
+    from .misc import MiscTestCase
+    from .uploader import UploaderTestCase
+    # from .serializer import ResourceTestCase as SerializerTestCase
+    # from .utils import UtilsTestCase
+
+    miscsuite = unittest.TestLoader().loadTestsFromTestCase(MiscTestCase)
+    uploadersuite = unittest.TestLoader().loadTestsFromTestCase(UploaderTestCase)
+    return unittest.TestSuite([miscsuite, uploadersuite])

+ 18 - 16
tests/misc.py

@@ -1,16 +1,18 @@
-import unittest
-from lib.utils import default_port
-from lib import __version__
-
-class MiscTestCase(unittest.TestCase):
-
-    def test_version(self):
-        self.assertEqual(__version__, '0.2.0')
-
-    def test_default_port(self):
-        #Test as if it were given system
-        self.assertEqual(default_port('Linux'), '/dev/ttyUSB0')
-        self.assertEqual(default_port('Windows'), 'COM1')
-        self.assertEqual(default_port('Darwin'), '/dev/tty.SLAB_USBtoUART')
-
-        self.assertTrue(len(default_port()) >= 3)
+# -*- coding: utf-8 -*-
+# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
+import unittest
+from lib.utils import default_port
+from lib import __version__
+
+class MiscTestCase(unittest.TestCase):
+
+    def test_version(self):
+        self.assertEqual(__version__, '0.2.0')
+
+    def test_default_port(self):
+        #Test as if it were given system
+        self.assertEqual(default_port('Linux'), '/dev/ttyUSB0')
+        self.assertEqual(default_port('Windows'), 'COM1')
+        self.assertEqual(default_port('Darwin'), '/dev/tty.SLAB_USBtoUART')
+
+        self.assertTrue(len(default_port()) >= 3)

+ 17 - 15
tests/uploader.py

@@ -1,15 +1,17 @@
-import unittest
-from lib import Uploader, __version__
-
-#on which port should the tests be performed
-PORT = 'loop://'
-
-#which speed should the tests use
-BAUD = 115200
-
-class UploaderTestCase(unittest.TestCase):
-
-
-
-    def test_initialize(self):
-        uploader = Uploader(PORT, BAUD)
+# -*- coding: utf-8 -*-
+# Copyright (C) 2015-2016 Peter Magnusson <peter@birchroad.net>
+import unittest
+from lib import Uploader, __version__
+
+#on which port should the tests be performed
+PORT = 'loop://'
+
+#which speed should the tests use
+BAUD = 115200
+
+class UploaderTestCase(unittest.TestCase):
+
+
+
+    def test_initialize(self):
+        uploader = Uploader(PORT, BAUD)