Browse Source

Chunked download of files.

--verify=text changed to --verify=raw

Closes #19 and  #33
kmpm 8 years ago
parent
commit
8a61d17b68
4 changed files with 79 additions and 27 deletions
  1. 5 8
      README.md
  2. 10 4
      lib/luacode.py
  3. 1 1
      lib/main.py
  4. 63 14
      lib/uploader.py

+ 5 - 8
README.md

@@ -33,11 +33,8 @@ not work depending on version used. This is a known issue.
 
 
 ### Notes for Windows
-There are some 
-[significant issues with Windows](https://github.com/kmpm/nodemcu-uploader/issues?q=is%3Aissue+is%3Aopen+label%3Aos%3Awindows)
-that might be related to serial port hardware but it can be things like
-complete hang (you have to kill the python.exe process) to random
-errors and/or corrupted file transfers.
+There might be some
+[significant issues with Windows](https://github.com/kmpm/nodemcu-uploader/issues?q=is%3Aissue+is%3Aopen+label%3Aos%3Awindows).
 
 
 Issues
@@ -79,14 +76,14 @@ Uploading a number of files, but saving with a different file name.
 ```
 
 Uploading a number of files and verify successful uploading by downloading the file
-and comparing contents. Might fail because of Issue #33 - 'Upload verification breaks if file has "> " in it'
+and comparing contents.
 
 ```
-./nodemcu-uploader.py upload init.lua README.md nodemcu-uploader.py --verify=text
+./nodemcu-uploader.py upload init.lua README.md nodemcu-uploader.py --verify=raw
 ```
 
 Uploading a number of files and verify successful uploading by doing a sha1 checksum.
-__Requires crypto module on the device__
+__Requires crypto module on the device__ and currently files not to big (~1000 bytes)
 
 ```
 ./nodemcu-uploader.py upload init.lua README.md nodemcu-uploader.py --verify=sha1

+ 10 - 4
lib/luacode.py

@@ -5,14 +5,14 @@
 
 # these functions are needed on the device, otherwise they will be
 # uploaded during prepare
-LUA_FUNCTIONS = ['recv_block', 'recv_name','recv','shafile', 'send_block', 'send_file']
+LUA_FUNCTIONS = ['recv_block', 'recv_name','recv','shafile', 'send_block', 'send_file', 'send']
 
 DOWNLOAD_FILE = "file.open('{filename}') print(file.seek('end', 0)) file.seek('set', {bytes_read}) uart.write(0, file.read({chunk_size}))file.close()"
 
 PRINT_FILE = "file.open('{filename}') print('---{filename}---') print(file.read()) file.close() print('---')"
 
 LIST_FILES = 'for key,value in pairs(file.list()) do print(key,value) end'
-
+#NUL = \000, ACK = \006
 RECV_LUA = \
 r"""
 function recv_block(d)
@@ -39,8 +39,14 @@ function shafile(f) file.open(f, "r") print(crypto.toHex(crypto.hash("sha1",file
 
 SEND_LUA = \
 r"""
-function send_block(d) l = string.len(d) uart.write(0, '\001' + string.char(l) + string.rep(' ', 128 - l)) return l end
-function send_file(f) file.open(f) s=file.seek('end', 0) p=0 while (p<s) do file.seek('set',p) p=p+send_block(file.read(128)) end send_block('') file.close() end
+function send_block(d) l = string.len(d) uart.write(0, '\001' .. string.char(l) .. d .. string.rep('#', 128 - l)) return l end
+function send_file(f) file.open(f) s=file.seek('end', 0) p=0 uart.on('data', 1, function(data)
+if data == '\006' and p<s then file.seek('set',p) p=p+send_block(file.read(128)) else
+send_block('') file.close() uart.on('data') print('interrupted') end end, 0) uart.write(0, f .. '\000')
+end
+function send(f) uart.on('data', 1, function (data)
+    uart.on('data') if data == 'C' then send_file(f) else print('transfer interrupted') end end, 0)
+end
 """
 
 UART_SETUP = 'uart.setup(0,{baud},8,0,1,1)'

+ 1 - 1
lib/main.py

@@ -146,7 +146,7 @@ def main_func():
         help='To verify the uploaded data.',
         action='store',
         nargs='?',
-        choices=['none', 'text', 'sha1'],
+        choices=['none', 'raw', 'sha1'],
         default='none'
         )
 

+ 63 - 14
lib/uploader.py

@@ -17,6 +17,10 @@ __all__ = ['Uploader', 'default_port']
 
 SYSTEM = system()
 
+BLOCK_START='\x01';
+NUL = '\x00';
+ACK = '\x06';
+
 class Uploader(object):
     """Uploader is the class for communicating with the nodemcu and
     that will allow various tasks like uploading files, formating the filesystem etc.
@@ -94,11 +98,12 @@ class Uploader(object):
         while not data.endswith(exp) and time.time() <= end:
             data += self._port.read()
 
+        log.debug('expect returned: `{0}`'.format(data))
         if time.time() > end and not data.endswith(exp) and len(exp) > 0:
             raise Exception('Timeout expecting ' + exp)
         if SYSTEM != 'Windows':
             self._port.timeout = timer
-        log.debug('expect returned: `{0}`'.format(data))
+
         return data
 
     def write(self, output, binary=False):
@@ -165,17 +170,27 @@ class Uploader(object):
         return True
 
     def download_file(self, filename):
-        chunk_size = 256
-        bytes_read = 0
-        data = ""
-        while True:
-            d = self.exchange(DOWNLOAD_FILE.format(filename=filename, bytes_read=bytes_read, chunk_size=chunk_size))
-            cmd, size, tmp_data = d.split('\n', 2)
-            data = data + tmp_data[0:chunk_size]
-            bytes_read = bytes_read + chunk_size
-            if bytes_read > int(size):
-                break
-        data = data[0:int(size)]
+        res = self.exchange('send("{filename}")'.format(filename=filename))
+        if ('unexpected' in res) or ('stdin' in res):
+            log.error('Unexpected error downloading file', res)
+            raise Exception('Unexpected error downloading file')
+
+        #tell device we are ready to receive
+        self.write('C')
+        #we should get a NUL terminated filename to start with
+        sent_filename = self.expect(NUL).strip()
+        log.info('receiveing ' + sent_filename)
+
+        #ACK to start download
+        self.write(ACK, True);
+        buffer = ''
+        data = ''
+        chunk, buffer = self.read_chunk(buffer)
+        #read chunks until we get an empty which is the end
+        while chunk != '':
+            self.write(ACK,True);
+            data = data + chunk
+            chunk, buffer = self.read_chunk(buffer)
         return data
 
     def read_file(self, filename, destination=''):
@@ -227,11 +242,13 @@ class Uploader(object):
         #zero size block
         self.write_chunk('')
 
-        if verify == 'text':
+        if verify == 'raw':
             log.info('Verifying...')
             data = self.download_file(destination)
             if content != data:
                 log.error('Verification failed.')
+            else:
+                log.info('Verification successfull. Contents are identical.')
         elif verify == 'sha1':
             #Calculate SHA1 on remote file. Extract just hash from result
             data = self.exchange('shafile("'+destination+'")').splitlines()[1]
@@ -242,6 +259,12 @@ class Uploader(object):
             log.info('Local SHA1: %s', filehashhex)
             if data != filehashhex:
                 log.error('Verification failed.')
+            else:
+                log.info('Verification successfull. Checksums match')
+
+        elif verify != 'none':
+            raise Exception(verify + ' is not a valid verification method.')
+
 
     def exec_file(self, path):
         filename = os.path.basename(path)
@@ -279,7 +302,7 @@ class Uploader(object):
 
     def write_chunk(self, chunk):
         log.debug('writing %d bytes chunk', len(chunk))
-        data = '\x01' + chr(len(chunk)) + chunk
+        data = BLOCK_START + chr(len(chunk)) + chunk
         if len(chunk) < 128:
             padding = 128 - len(chunk)
             log.debug('pad with %d characters', padding)
@@ -290,6 +313,32 @@ class Uploader(object):
         return self.got_ack()
 
 
+    def read_chunk(self, buffer):
+        log.debug('reading chunk')
+        timeout = self._port.timeout
+        if SYSTEM != 'Windows':
+            timer = self._port.timeout
+            # Checking for new data every 100us is fast enough
+            lt = 0.0001
+            if self._port.timeout != lt:
+                self._port.timeout = lt
+
+        end = time.time() + timeout
+
+        while len(buffer) < 130 and time.time() <= end:
+            buffer = buffer + self._port.read()
+
+        if buffer[0] != BLOCK_START or len(buffer) < 130:
+            print 'buffer size:', len(buffer)
+            log.debug('buffer binary: ', ':'.join(x.encode('hex') for x in buffer))
+            raise Exception('Bad blocksize or start byte')
+
+        chunk_size = ord(buffer[1])
+        data = buffer[2:chunk_size+2]
+        buffer = buffer[130:]
+        return (data, buffer)
+
+
     def file_list(self):
         log.info('Listing files')
         res = self.exchange(LIST_FILES)