nodemcu-partition.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391
  1. #!/usr/bin/env python
  2. #
  3. # ESP8266 LFS Loader Utility
  4. #
  5. # Copyright (C) 2019 Terry Ellison, NodeMCU Firmware Community Project. drawing
  6. # heavily from and including content from esptool.py with full acknowledgement
  7. # under GPL 2.0, with said content: Copyright (C) 2014-2016 Fredrik Ahlberg, Angus
  8. # Gratton, Espressif Systems (Shanghai) PTE LTD, other contributors as noted.
  9. # https:# github.com/espressif/esptool
  10. #
  11. # This program is free software; you can redistribute it and/or modify it under
  12. # the terms of the GNU General Public License as published by the Free Software
  13. # Foundation; either version 2 of the License, or (at your option) any later version.
  14. #
  15. # This program is distributed in the hope that it will be useful, but WITHOUT
  16. # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17. # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  18. #
  19. # You should have received a copy of the GNU General Public License along with
  20. # this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
  21. # Street, Fifth Floor, Boston, MA 02110-1301 USA.
  22. import os
  23. import sys
  24. print os.path.dirname(os.path.realpath(__file__))
  25. sys.path.append(os.path.dirname(os.path.realpath(__file__)) + '/toolchains/')
  26. import esptool
  27. import io
  28. import tempfile
  29. import shutil
  30. from pprint import pprint
  31. import argparse
  32. import gzip
  33. import copy
  34. import inspect
  35. import struct
  36. import string
  37. import math
  38. __version__ = '1.0'
  39. __program__ = 'nodemcu-partition.py'
  40. ROM0_Seg = 0x010000
  41. FLASH_PAGESIZE = 0x001000
  42. FLASH_BASE_ADDR = 0x40200000
  43. PARTITION_TYPE = {
  44. 4: 'RF_CAL',
  45. 5: 'PHY_DATA',
  46. 6: 'SYSTEM_PARAMETER',
  47. 101: 'EAGLEROM',
  48. 102: 'IROM0TEXT',
  49. 103: 'LFS0',
  50. 104: 'LFS1',
  51. 105: 'TLSCERT',
  52. 106: 'SPIFFS0',
  53. 107: 'SPIFFS1'}
  54. SYSTEM_PARAMETER = 6
  55. IROM0TEXT = 102
  56. LFS = 103
  57. SPIFFS = 106
  58. MAX_PT_SIZE = 20*3
  59. FLASH_SIG = 0xfafaa150
  60. FLASH_SIG_MASK = 0xfffffff0
  61. FLASH_SIG_ABSOLUTE = 0x00000001
  62. WORDSIZE = 4
  63. WORDBITS = 32
  64. DEFAULT_FLASH_SIZE = 4*1024*1024
  65. PLATFORM_RCR_DELETED = 0x0
  66. PLATFORM_RCR_PT = 0x1
  67. PLATFORM_RCR_FREE = 0xFF
  68. SPIFFS_USE_ALL = 0xFFFFFFFF
  69. PACK_INT = struct.Struct("<I")
  70. class FatalError(RuntimeError):
  71. def __init__(self, message):
  72. RuntimeError.__init__(self, message)
  73. def WithResult(message, result):
  74. message += " (result was %s)" % hexify(result)
  75. return FatalError(message)
  76. def alignPT(n):
  77. return 2*FLASH_PAGESIZE*int(math.ceil(n/2/FLASH_PAGESIZE))
  78. def unpack_RCR(data):
  79. RCRword,recs, i = [PACK_INT.unpack_from(data,i)[0] \
  80. for i in range(0, FLASH_PAGESIZE, WORDSIZE)], \
  81. [],0
  82. while RCRword[i] % 256 != PLATFORM_RCR_FREE:
  83. Rlen, Rtype = RCRword[i] % 256, (RCRword[i]/256) % 256
  84. if Rtype != PLATFORM_RCR_DELETED:
  85. rec = [Rtype,[RCRword[j] for j in range(i+1,i+1+Rlen)]]
  86. if Rtype == PLATFORM_RCR_PT:
  87. PTrec = rec[1]
  88. else:
  89. recs.append(rec)
  90. i = i + Rlen + 1
  91. if PTrec is not None:
  92. return PTrec,recs
  93. FatalError("No partition table found")
  94. def repack_RCR(recs):
  95. data = []
  96. for r in recs:
  97. Rtype, Rdata = r
  98. data.append(256*Rtype + len(Rdata))
  99. data.extend(Rdata)
  100. return ''.join([PACK_INT.pack(i) for i in data])
  101. def load_PT(data, args):
  102. """
  103. Load the Flash copy of the Partition Table from the first segment of the IROM0
  104. segment, that is at 0x10000. If nececessary the LFS partition is then correctly
  105. positioned and adjusted according to the optional start and len arguments.
  106. The (possibly) updated PT is then returned with the LFS sizing.
  107. """
  108. PTrec,recs = unpack_RCR(data)
  109. flash_size = args.fs if args.fs is not None else DEFAULT_FLASH_SIZE
  110. # The partition table format is a set of 3*uint32 fields (type, addr, size),
  111. # with the optional last slot being an end marker (0,size,0) where size is
  112. # of the firmware image.
  113. if PTrec[-3] == 0: # Pick out the ROM size and remove the marker
  114. defaultIROM0size = PTrec[-2] - FLASH_BASE_ADDR
  115. del PTrec[-3:]
  116. else:
  117. defaultIROM0size = None
  118. # The SDK objects to zero-length partitions so if the developer sets the
  119. # size of the LFS and/or the SPIFFS partition to 0 then this is removed.
  120. # If it is subsequently set back to non-zero then it needs to be reinserted.
  121. # In reality the sizing algos assume that the LFS follows the IROM0TEXT one
  122. # and SPIFFS is the last partition. We will need to revisit these algos if
  123. # we adopt a more flexible partiton allocation policy. *** BOTCH WARNING ***
  124. i = 0
  125. while i < len(PTrec):
  126. if PTrec[i] == IROM0TEXT and args.ls is not None and \
  127. (len(PTrec) == i+3 or PTrec[i+3] != LFS):
  128. PTrec[i+3:i+3] = [LFS, 0, 0]
  129. i += 3
  130. if PTrec[-6] != SPIFFS:
  131. PTrec[-6:-6] = [SPIFFS, PTrec[-5] + PTrec[-4], 0x1000]
  132. lastEnd, newPT, map = 0,[], dict()
  133. print " Partition Start Size \n ------------------ ------ ------"
  134. for i in range (0, len(PTrec), 3):
  135. Ptype, Paddr, Psize = PTrec[i:i+3]
  136. if Ptype == IROM0TEXT:
  137. # If the IROM0 partition size is 0 then compute from the IROM0_SIZE.
  138. # Note that this script uses the size in the end-marker as a default
  139. if Psize == 0:
  140. if defaultIROM0size is None:
  141. raise FatalError("Cannot set the IROM0 partition size")
  142. Psize = alignPT(defaultIROM0size)
  143. elif Ptype == LFS:
  144. # Properly align the LFS partition size and make it consecutive to
  145. # the previous partition.
  146. if args.la is not None:
  147. Paddr = args.la
  148. if args.ls is not None:
  149. Psize = args.ls
  150. Psize = alignPT(Psize)
  151. if Paddr == 0:
  152. Paddr = lastEnd
  153. if Psize > 0:
  154. map['LFS'] = {"addr" : Paddr, "size" : Psize}
  155. elif Ptype == SPIFFS:
  156. # The logic here is convolved. Explicit start and length can be
  157. # set, but the SPIFFS region is aslo contrained by the end of the
  158. # previos partition and the end of Flash. The size = -1 value
  159. # means use up remaining flash and the SPIFFS will be moved to the
  160. # 1Mb boundary if the address is default and the specified size
  161. # allows this.
  162. if args.sa is not None:
  163. Paddr = args.sa
  164. if args.ss is not None:
  165. Psize = args.ss if args.ss >= 0 else SPIFFS_USE_ALL
  166. if Psize == SPIFFS_USE_ALL:
  167. # This allocate all the remaining flash to SPIFFS
  168. if Paddr < lastEnd:
  169. Paddr = lastEnd
  170. Psize = flash_size - Paddr
  171. else:
  172. if Paddr == 0:
  173. # if the is addr not specified then start SPIFFS at 1Mb
  174. # boundary if the size will fit otherwise make it consecutive
  175. # to the previous partition.
  176. Paddr = 0x100000 if Psize <= flash_size - 0x100000 else lastEnd
  177. elif Paddr < lastEnd:
  178. Paddr = lastEnd
  179. if Psize > flash_size - Paddr:
  180. Psize = flash_size - Paddr
  181. if Psize > 0:
  182. map['SPIFFS'] = {"addr" : Paddr, "size" : Psize}
  183. elif Ptype == SYSTEM_PARAMETER and Paddr == 0:
  184. Paddr = flash_size - Psize
  185. if Psize > 0:
  186. Pname = PARTITION_TYPE[Ptype] if Ptype in PARTITION_TYPE \
  187. else ("Type %d" % Ptype)
  188. print(" %-18s %06x %06x"% (Pname, Paddr, Psize))
  189. # Do consistency tests on the partition
  190. if (Paddr & (FLASH_PAGESIZE - 1)) > 0 or \
  191. (Psize & (FLASH_PAGESIZE - 1)) > 0 or \
  192. Paddr < lastEnd or \
  193. Paddr + Psize > flash_size:
  194. print (lastEnd, flash_size)
  195. raise FatalError("Partition %u invalid alignment\n" % (i/3))
  196. newPT.extend([Ptype, Paddr, Psize])
  197. lastEnd = Paddr + Psize
  198. recs.append([PLATFORM_RCR_PT,newPT])
  199. return recs, map
  200. def relocate_lfs(data, addr, size):
  201. """
  202. The unpacked LFS image comprises the relocatable image itself, followed by a bit
  203. map (one bit per word) flagging if the corresponding word of the image needs
  204. relocating. The image and bitmap are enumerated with any addresses being
  205. relocated by the LFS base address. (Note that the PIC format of addresses is word
  206. aligned and so first needs scaling by the wordsize.)
  207. """
  208. addr += FLASH_BASE_ADDR
  209. w = [PACK_INT.unpack_from(data,i)[0] for i in range(0, len(data),WORDSIZE)]
  210. flash_sig, flash_size = w[0], w[1]
  211. assert ((flash_sig & FLASH_SIG_MASK) == FLASH_SIG and
  212. (flash_sig & FLASH_SIG_ABSOLUTE) == 0 and
  213. flash_size % WORDSIZE == 0)
  214. flash_size //= WORDSIZE
  215. flags_size = (flash_size + WORDBITS - 1) // WORDBITS
  216. print WORDSIZE*flash_size, size, len(data), WORDSIZE*(flash_size + flags_size)
  217. assert (WORDSIZE*flash_size <= size and
  218. len(data) == WORDSIZE*(flash_size + flags_size))
  219. image,flags,j = w[0:flash_size], w[flash_size:], 0
  220. for i in range(0,len(image)):
  221. if i % WORDBITS == 0:
  222. flag_word = flags[j]
  223. j += 1
  224. if (flag_word & 1) == 1:
  225. o = image[i]
  226. image[i] = WORDSIZE*image[i] + addr
  227. flag_word >>= 1
  228. return ''.join([PACK_INT.pack(i) for i in image])
  229. def main():
  230. def arg_auto_int(x):
  231. ux = x.upper()
  232. if "M" in ux:
  233. return int(ux[:ux.index("M")]) * 1024 * 1024
  234. elif "K" in ux:
  235. return int(ux[:ux.index("K")]) * 1024
  236. else:
  237. return int(ux, 0)
  238. print('%s V%s' %(__program__, __version__))
  239. # ---------- process the arguments ---------- #
  240. a = argparse.ArgumentParser(
  241. description='%s V%s - ESP8266 NodeMCU Loader Utility' %
  242. (__program__, __version__),
  243. prog=__program__)
  244. a.add_argument('--port', '-p', help='Serial port device')
  245. a.add_argument('--baud', '-b', type=arg_auto_int,
  246. help='Serial port baud rate used when flashing/reading')
  247. a.add_argument('--flash_size', '-fs', dest="fs", type=arg_auto_int,
  248. help='Flash size used in SPIFFS allocation (Default 4MB)')
  249. a.add_argument('--lfs_addr', '-la', dest="la", type=arg_auto_int,
  250. help='(Overwrite) start address of LFS partition')
  251. a.add_argument('--lfs_size', '-ls', dest="ls", type=arg_auto_int,
  252. help='(Overwrite) length of LFS partition')
  253. a.add_argument('--lfs_file', '-lf', dest="lf", help='LFS image file')
  254. a.add_argument('--spiffs_addr', '-sa', dest="sa", type=arg_auto_int,
  255. help='(Overwrite) start address of SPIFFS partition')
  256. a.add_argument('--spiffs_size', '-ss', dest="ss", type=arg_auto_int,
  257. help='(Overwrite) length of SPIFFS partition')
  258. a.add_argument('--spiffs_file', '-sf', dest="sf", help='SPIFFS image file')
  259. arg = a.parse_args()
  260. if arg.lf is not None:
  261. if not os.path.exists(arg.lf):
  262. raise FatalError("LFS image %s does not exist" % arg.lf)
  263. if arg.sf is not None:
  264. if not os.path.exists(arg.sf):
  265. raise FatalError("SPIFFS image %s does not exist" % arg.sf)
  266. base = [] if arg.port is None else ['--port',arg.port]
  267. if arg.baud is not None: base.extend(['--baud',str(arg.baud)])
  268. # ---------- Use esptool to read the PT ---------- #
  269. tmpdir = tempfile.mkdtemp()
  270. pt_file = tmpdir + '/pt.dmp'
  271. espargs = base+['--after', 'no_reset', 'read_flash', '--no-progress',
  272. str(ROM0_Seg), str(FLASH_PAGESIZE), pt_file]
  273. esptool.main(espargs)
  274. with open(pt_file,"rb") as f:
  275. data = f.read()
  276. # ---------- Update the PT if necessary ---------- #
  277. recs, pt_map = load_PT(data, arg)
  278. odata = repack_RCR(recs)
  279. odata = odata + "\xFF" * (FLASH_PAGESIZE - len(odata))
  280. # ---------- If the PT has changed then use esptool to rewrite it ---------- #
  281. if odata != data:
  282. print("PT updated")
  283. pt_file = tmpdir + '/opt.dmp'
  284. with open(pt_file,"wb") as f:
  285. f.write(odata)
  286. espargs = base+['--after', 'no_reset', 'write_flash', '--no-progress',
  287. str(ROM0_Seg), pt_file]
  288. esptool.main(espargs)
  289. if arg.lf is not None:
  290. if 'LFS' not in pt_map:
  291. raise FatalError("No LFS partition; cannot write LFS image")
  292. la,ls = pt_map['LFS']['addr'], pt_map['LFS']['size']
  293. # ---------- Read and relocate the LFS image ---------- #
  294. with gzip.open(arg.lf) as f:
  295. lfs = f.read()
  296. if len(lfs) > ls:
  297. raise FatalError("LFS partition to small for LFS image")
  298. lfs = relocate_lfs(lfs, la, ls)
  299. # ---------- Write to a temp file and use esptool to write it to flash ---------- #
  300. img_file = tmpdir + '/lfs.img'
  301. espargs = base + ['write_flash', str(la), img_file]
  302. with open(img_file,"wb") as f:
  303. f.write(lfs)
  304. esptool.main(espargs)
  305. if arg.sf is not None:
  306. if 'SPIFFS' not in pt_map:
  307. raise FatalError("No SPIFSS partition; cannot write SPIFFS image")
  308. sa,ss = pt_map['SPIFFS']['addr'], pt_map['SPIFFS']['size']
  309. # ---------- Write to a temp file and use esptool to write it to flash ---------- #
  310. spiffs_file = arg.sf
  311. espargs = base + ['write_flash', str(sa), spiffs_file]
  312. esptool.main(espargs)
  313. # ---------- Clean up temp directory ---------- #
  314. # espargs = base + ['--after', 'hard_reset', 'flash_id']
  315. # esptool.main(espargs)
  316. shutil.rmtree(tmpdir)
  317. def _main():
  318. main()
  319. if __name__ == '__main__':
  320. _main()