Przeglądaj źródła

Chunked download of files.

--verify=text changed to --verify=raw

Closes #19 and  #33
kmpm 8 lat temu
rodzic
commit
6a9d9281c3
4 zmienionych plików z 79 dodań i 27 usunięć
  1. 5 8
      README.md
  2. 10 4
      lib/luacode.py
  3. 1 1
      lib/main.py
  4. 63 14
      lib/uploader.py

+ 5 - 8
README.md

@@ -33,11 +33,8 @@ not work depending on version used. This is a known issue.
 
 
 
 
 ### Notes for Windows
 ### Notes for Windows
-There are some 
-[significant issues with Windows](https://github.com/kmpm/nodemcu-uploader/issues?q=is%3Aissue+is%3Aopen+label%3Aos%3Awindows)
-that might be related to serial port hardware but it can be things like
-complete hang (you have to kill the python.exe process) to random
-errors and/or corrupted file transfers.
+There might be some
+[significant issues with Windows](https://github.com/kmpm/nodemcu-uploader/issues?q=is%3Aissue+is%3Aopen+label%3Aos%3Awindows).
 
 
 
 
 Issues
 Issues
@@ -79,14 +76,14 @@ Uploading a number of files, but saving with a different file name.
 ```
 ```
 
 
 Uploading a number of files and verify successful uploading by downloading the file
 Uploading a number of files and verify successful uploading by downloading the file
-and comparing contents. Might fail because of Issue #33 - 'Upload verification breaks if file has "> " in it'
+and comparing contents.
 
 
 ```
 ```
-./nodemcu-uploader.py upload init.lua README.md nodemcu-uploader.py --verify=text
+./nodemcu-uploader.py upload init.lua README.md nodemcu-uploader.py --verify=raw
 ```
 ```
 
 
 Uploading a number of files and verify successful uploading by doing a sha1 checksum.
 Uploading a number of files and verify successful uploading by doing a sha1 checksum.
-__Requires crypto module on the device__
+__Requires crypto module on the device__ and currently files not to big (~1000 bytes)
 
 
 ```
 ```
 ./nodemcu-uploader.py upload init.lua README.md nodemcu-uploader.py --verify=sha1
 ./nodemcu-uploader.py upload init.lua README.md nodemcu-uploader.py --verify=sha1

+ 10 - 4
lib/luacode.py

@@ -5,14 +5,14 @@
 
 
 # these functions are needed on the device, otherwise they will be
 # these functions are needed on the device, otherwise they will be
 # uploaded during prepare
 # uploaded during prepare
-LUA_FUNCTIONS = ['recv_block', 'recv_name','recv','shafile', 'send_block', 'send_file']
+LUA_FUNCTIONS = ['recv_block', 'recv_name','recv','shafile', 'send_block', 'send_file', 'send']
 
 
 DOWNLOAD_FILE = "file.open('{filename}') print(file.seek('end', 0)) file.seek('set', {bytes_read}) uart.write(0, file.read({chunk_size}))file.close()"
 DOWNLOAD_FILE = "file.open('{filename}') print(file.seek('end', 0)) file.seek('set', {bytes_read}) uart.write(0, file.read({chunk_size}))file.close()"
 
 
 PRINT_FILE = "file.open('{filename}') print('---{filename}---') print(file.read()) file.close() print('---')"
 PRINT_FILE = "file.open('{filename}') print('---{filename}---') print(file.read()) file.close() print('---')"
 
 
 LIST_FILES = 'for key,value in pairs(file.list()) do print(key,value) end'
 LIST_FILES = 'for key,value in pairs(file.list()) do print(key,value) end'
-
+#NUL = \000, ACK = \006
 RECV_LUA = \
 RECV_LUA = \
 r"""
 r"""
 function recv_block(d)
 function recv_block(d)
@@ -39,8 +39,14 @@ function shafile(f) file.open(f, "r") print(crypto.toHex(crypto.hash("sha1",file
 
 
 SEND_LUA = \
 SEND_LUA = \
 r"""
 r"""
-function send_block(d) l = string.len(d) uart.write(0, '\001' + string.char(l) + string.rep(' ', 128 - l)) return l end
-function send_file(f) file.open(f) s=file.seek('end', 0) p=0 while (p<s) do file.seek('set',p) p=p+send_block(file.read(128)) end send_block('') file.close() end
+function send_block(d) l = string.len(d) uart.write(0, '\001' .. string.char(l) .. d .. string.rep('#', 128 - l)) return l end
+function send_file(f) file.open(f) s=file.seek('end', 0) p=0 uart.on('data', 1, function(data)
+if data == '\006' and p<s then file.seek('set',p) p=p+send_block(file.read(128)) else
+send_block('') file.close() uart.on('data') print('interrupted') end end, 0) uart.write(0, f .. '\000')
+end
+function send(f) uart.on('data', 1, function (data)
+    uart.on('data') if data == 'C' then send_file(f) else print('transfer interrupted') end end, 0)
+end
 """
 """
 
 
 UART_SETUP = 'uart.setup(0,{baud},8,0,1,1)'
 UART_SETUP = 'uart.setup(0,{baud},8,0,1,1)'

+ 1 - 1
lib/main.py

@@ -146,7 +146,7 @@ def main_func():
         help='To verify the uploaded data.',
         help='To verify the uploaded data.',
         action='store',
         action='store',
         nargs='?',
         nargs='?',
-        choices=['none', 'text', 'sha1'],
+        choices=['none', 'raw', 'sha1'],
         default='none'
         default='none'
         )
         )
 
 

+ 63 - 14
lib/uploader.py

@@ -17,6 +17,10 @@ __all__ = ['Uploader', 'default_port']
 
 
 SYSTEM = system()
 SYSTEM = system()
 
 
+BLOCK_START='\x01';
+NUL = '\x00';
+ACK = '\x06';
+
 class Uploader(object):
 class Uploader(object):
     """Uploader is the class for communicating with the nodemcu and
     """Uploader is the class for communicating with the nodemcu and
     that will allow various tasks like uploading files, formating the filesystem etc.
     that will allow various tasks like uploading files, formating the filesystem etc.
@@ -94,11 +98,12 @@ class Uploader(object):
         while not data.endswith(exp) and time.time() <= end:
         while not data.endswith(exp) and time.time() <= end:
             data += self._port.read()
             data += self._port.read()
 
 
+        log.debug('expect returned: `{0}`'.format(data))
         if time.time() > end and not data.endswith(exp) and len(exp) > 0:
         if time.time() > end and not data.endswith(exp) and len(exp) > 0:
             raise Exception('Timeout expecting ' + exp)
             raise Exception('Timeout expecting ' + exp)
         if SYSTEM != 'Windows':
         if SYSTEM != 'Windows':
             self._port.timeout = timer
             self._port.timeout = timer
-        log.debug('expect returned: `{0}`'.format(data))
+
         return data
         return data
 
 
     def write(self, output, binary=False):
     def write(self, output, binary=False):
@@ -165,17 +170,27 @@ class Uploader(object):
         return True
         return True
 
 
     def download_file(self, filename):
     def download_file(self, filename):
-        chunk_size = 256
-        bytes_read = 0
-        data = ""
-        while True:
-            d = self.exchange(DOWNLOAD_FILE.format(filename=filename, bytes_read=bytes_read, chunk_size=chunk_size))
-            cmd, size, tmp_data = d.split('\n', 2)
-            data = data + tmp_data[0:chunk_size]
-            bytes_read = bytes_read + chunk_size
-            if bytes_read > int(size):
-                break
-        data = data[0:int(size)]
+        res = self.exchange('send("{filename}")'.format(filename=filename))
+        if ('unexpected' in res) or ('stdin' in res):
+            log.error('Unexpected error downloading file', res)
+            raise Exception('Unexpected error downloading file')
+
+        #tell device we are ready to receive
+        self.write('C')
+        #we should get a NUL terminated filename to start with
+        sent_filename = self.expect(NUL).strip()
+        log.info('receiveing ' + sent_filename)
+
+        #ACK to start download
+        self.write(ACK, True);
+        buffer = ''
+        data = ''
+        chunk, buffer = self.read_chunk(buffer)
+        #read chunks until we get an empty which is the end
+        while chunk != '':
+            self.write(ACK,True);
+            data = data + chunk
+            chunk, buffer = self.read_chunk(buffer)
         return data
         return data
 
 
     def read_file(self, filename, destination=''):
     def read_file(self, filename, destination=''):
@@ -227,11 +242,13 @@ class Uploader(object):
         #zero size block
         #zero size block
         self.write_chunk('')
         self.write_chunk('')
 
 
-        if verify == 'text':
+        if verify == 'raw':
             log.info('Verifying...')
             log.info('Verifying...')
             data = self.download_file(destination)
             data = self.download_file(destination)
             if content != data:
             if content != data:
                 log.error('Verification failed.')
                 log.error('Verification failed.')
+            else:
+                log.info('Verification successfull. Contents are identical.')
         elif verify == 'sha1':
         elif verify == 'sha1':
             #Calculate SHA1 on remote file. Extract just hash from result
             #Calculate SHA1 on remote file. Extract just hash from result
             data = self.exchange('shafile("'+destination+'")').splitlines()[1]
             data = self.exchange('shafile("'+destination+'")').splitlines()[1]
@@ -242,6 +259,12 @@ class Uploader(object):
             log.info('Local SHA1: %s', filehashhex)
             log.info('Local SHA1: %s', filehashhex)
             if data != filehashhex:
             if data != filehashhex:
                 log.error('Verification failed.')
                 log.error('Verification failed.')
+            else:
+                log.info('Verification successfull. Checksums match')
+
+        elif verify != 'none':
+            raise Exception(verify + ' is not a valid verification method.')
+
 
 
     def exec_file(self, path):
     def exec_file(self, path):
         filename = os.path.basename(path)
         filename = os.path.basename(path)
@@ -279,7 +302,7 @@ class Uploader(object):
 
 
     def write_chunk(self, chunk):
     def write_chunk(self, chunk):
         log.debug('writing %d bytes chunk', len(chunk))
         log.debug('writing %d bytes chunk', len(chunk))
-        data = '\x01' + chr(len(chunk)) + chunk
+        data = BLOCK_START + chr(len(chunk)) + chunk
         if len(chunk) < 128:
         if len(chunk) < 128:
             padding = 128 - len(chunk)
             padding = 128 - len(chunk)
             log.debug('pad with %d characters', padding)
             log.debug('pad with %d characters', padding)
@@ -290,6 +313,32 @@ class Uploader(object):
         return self.got_ack()
         return self.got_ack()
 
 
 
 
+    def read_chunk(self, buffer):
+        log.debug('reading chunk')
+        timeout = self._port.timeout
+        if SYSTEM != 'Windows':
+            timer = self._port.timeout
+            # Checking for new data every 100us is fast enough
+            lt = 0.0001
+            if self._port.timeout != lt:
+                self._port.timeout = lt
+
+        end = time.time() + timeout
+
+        while len(buffer) < 130 and time.time() <= end:
+            buffer = buffer + self._port.read()
+
+        if buffer[0] != BLOCK_START or len(buffer) < 130:
+            print 'buffer size:', len(buffer)
+            log.debug('buffer binary: ', ':'.join(x.encode('hex') for x in buffer))
+            raise Exception('Bad blocksize or start byte')
+
+        chunk_size = ord(buffer[1])
+        data = buffer[2:chunk_size+2]
+        buffer = buffer[130:]
+        return (data, buffer)
+
+
     def file_list(self):
     def file_list(self):
         log.info('Listing files')
         log.info('Listing files')
         res = self.exchange(LIST_FILES)
         res = self.exchange(LIST_FILES)