lflashimg.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446
  1. /***--
  2. ** lflashimg.c
  3. ** Dump a compiled Proto hiearchy to a RO (FLash) image file
  4. ** See Copyright Notice in lua.h
  5. */
  6. #define LUAC_CROSS_FILE
  7. #include "luac_cross.h"
  8. #include C_HEADER_CTYPE
  9. #include C_HEADER_STDIO
  10. #include C_HEADER_STDLIB
  11. #include C_HEADER_STRING
  12. #define lflashimg_c
  13. #define LUA_CORE
  14. #include "lobject.h"
  15. #include "lstring.h"
  16. #undef LUA_FLASH_STORE
  17. #define LUA_FLASH_STORE
  18. #include "lflash.h"
  19. #include "uzlib.h"
  20. //#define LOCAL_DEBUG
  21. #if INT_MAX != 2147483647
  22. # error "luac.cross requires C toolchain with 4 byte word size"
  23. #endif
  24. #define WORDSIZE ((int) sizeof(int))
  25. #define ALIGN(s) (((s)+(WORDSIZE-1)) & (-(signed) WORDSIZE))
  26. #define WORDSHIFT 2
  27. typedef unsigned int uint;
  28. #define FLASH_WORDS(t) (sizeof(t)/sizeof(FlashAddr))
  29. /*
  30. *
  31. * This dumper is a variant of the standard ldump, in that instead of producing a
  32. * binary loader format that lundump can load, it produces an image file that can be
  33. * directly mapped or copied into addressable memory. The typical application is on
  34. * small memory IoT devices which support programmable flash storage such as the
  35. * ESP8266. A 64 Kb LFS image has 16Kb words and will enable all program-related
  36. * storage to be accessed directly from flash, leaving the RAM for true R/W
  37. * application data.
  38. *
  39. * The start address of the Lua Flash Store (LFS) is build-dependent, and the cross
  40. * compiler '-a' option allows the developer to fix the LFS at a defined flash memory
  41. * address. Alternatively and by default the cross compilation adopts a position
  42. * independent image format, which permits the on-device image loader to load the LFS
  43. * image at an appropriate base within the flash address space. As all objects in the
  44. * LFS can be treated as multiples of 4-byte words, also all address fields are both
  45. * word aligned, and any address references within the LFS are also word-aligned.
  46. *
  47. * This version adds gzip compression of the generated LFS image for more efficient
  48. * over-the-air (OTA) transfer, so the method of tagging address words has been
  49. * replaced by a scheme which achieves better compression: an additional bitmap
  50. * has been added to the image, with each bit corresponding to a word in the image
  51. * and set if the corresponding work is an address. The addresses are stored as
  52. * signed relative word offsets.
  53. *
  54. * The unloader is documented in lflash.c Note that his relocation process is
  55. * skipped for absolute addressed images (which are identified by the
  56. * FLASH_SIG_ABSOLUTE bit setting in the flash signature).
  57. *
  58. * The flash image has a standard header detailed in lflash.h
  59. *
  60. * Note that luac.cross may be compiled on any little-endian machine with 32 or 64 bit
  61. * word length so Flash addresses can't be handled as standard C pointers as size_t
  62. * and int may not have the same size. Hence addresses with the must be declared as
  63. * the FlashAddr type rather than typed C pointers and must be accessed through macros.
  64. *
  65. * Also note that image built with a given LUA_PACK_TVALUES / LUA_NUNBER_INTEGRAL
  66. * combination must be loaded into a corresponding firmware build. Hence these
  67. * configuration options are also included in the FLash Signature.
  68. *
  69. * The Flash image is assembled up by first building the RO stringtable containing
  70. * all strings used in the compiled proto hierarchy. This is followed by the Protos.
  71. *
  72. * The storage is allocated bottom up using a serial allocator and the algortihm for
  73. * building the image essentially does a bottom-uo serial enumeration so that any
  74. * referenced storage has already been allocated in the image, and therefore (with the
  75. * exception of the Flash Header) all pointer references are backwards.
  76. *
  77. * As addresses are 4 byte on the target and either 4 or (typically) 8 bytes on the
  78. * host so any structures containing address fields (TStrings, TValues, Protos, other
  79. * address vectors) need repacking.
  80. */
  81. typedef struct flashts { /* This is the fixed 32-bit equivalent of TString */
  82. FlashAddr next;
  83. lu_byte tt;
  84. lu_byte marked;
  85. int hash;
  86. int len;
  87. } FlashTS;
  88. #ifndef LUA_MAX_FLASH_SIZE
  89. #define LUA_MAX_FLASH_SIZE 0x10000 //in words
  90. #endif
  91. static uint curOffset = 0;
  92. /*
  93. * The flashAddrTag is a bit array, one bit per flashImage word denoting
  94. * whether the corresponding word is a relative address. The defines
  95. * are access methods for this bit array.
  96. */
  97. static uint flashImage[LUA_MAX_FLASH_SIZE + LUA_MAX_FLASH_SIZE/32];
  98. static uint *flashAddrTag = flashImage + LUA_MAX_FLASH_SIZE;
  99. #define _TW(v) (v)>>5
  100. #define _TB(v) (1<<((v)&0x1F))
  101. #define setFlashAddrTag(v) flashAddrTag[_TW(v)] |= _TB(v)
  102. #define getFlashAddrTag(v) ((flashAddrTag[_TW(v)]&_TB(v)) != 0)
  103. #define fatal luac_fatal
  104. extern void __attribute__((noreturn)) luac_fatal(const char* message);
  105. #ifdef LOCAL_DEBUG
  106. #define DBG_PRINT(...) printf(__VA_ARGS__)
  107. #else
  108. #define DBG_PRINT(...) ((void)0)
  109. #endif
  110. /*
  111. * Serial allocator. Throw a luac-style out of memory error is allocaiton fails.
  112. */
  113. static void *flashAlloc(lua_State* L, size_t n) {
  114. void *p = (void *)(flashImage + curOffset);
  115. curOffset += ALIGN(n)>>WORDSHIFT;
  116. if (curOffset > LUA_MAX_FLASH_SIZE) {
  117. fatal("Out of Flash memory");
  118. }
  119. return p;
  120. }
  121. /*
  122. * Convert an absolute address pointing inside the flash image to offset form.
  123. * This macro form also takes the lvalue destination so that this can be tagged
  124. * as a relocatable address.
  125. */
  126. #define toFlashAddr(l, pd, s) _toFlashAddr(l, &(pd), s)
  127. static void _toFlashAddr(lua_State* L, FlashAddr *a, void *p) {
  128. uint doffset = cast(char *, a) - cast(char *,flashImage);
  129. lua_assert(!(doffset & (WORDSIZE-1))); // check word aligned
  130. doffset >>= WORDSHIFT; // and convert to a word offset
  131. lua_assert(doffset <= curOffset);
  132. if (p) {
  133. uint poffset = cast(char *, p) - cast(char *,flashImage);
  134. lua_assert(!(poffset & (WORDSIZE-1)));
  135. poffset >>= WORDSHIFT;
  136. lua_assert(poffset <= curOffset);
  137. flashImage[doffset] = poffset; // Set the pointer to the offset
  138. setFlashAddrTag(doffset); // And tag as an address
  139. } /* else leave clear */ // Special case for NULL pointer
  140. }
  141. /*
  142. * Convert an image address in offset form back to (host) absolute form
  143. */
  144. static void *fromFashAddr(FlashAddr a) {
  145. return a ? cast(void *, flashImage + a) : NULL;
  146. }
  147. /*
  148. * Add a TS found in the Proto Load to the table at the ToS
  149. */
  150. static void addTS(lua_State *L, TString *ts) {
  151. lua_assert(ts->tsv.tt==LUA_TSTRING);
  152. lua_pushnil(L);
  153. setsvalue(L, L->top-1, ts);
  154. lua_pushinteger(L, 1);
  155. lua_rawset(L, -3);
  156. DBG_PRINT("Adding string: %s\n",getstr(ts));
  157. }
  158. /*
  159. * Enumerate all of the Protos in the Proto hiearchy and scan contents to collect
  160. * all referenced strings in a Lua Array at ToS.
  161. */
  162. static void scanProtoStrings(lua_State *L, const Proto* f) {
  163. /* Table at L->Top[-1] is used to collect the strings */
  164. int i;
  165. if (f->source)
  166. addTS(L, f->source);
  167. #ifdef LUA_OPTIMIZE_DEBUG
  168. if (f->packedlineinfo)
  169. addTS(L, luaS_new(L, cast(const char *, f->packedlineinfo)));
  170. #endif
  171. for (i = 0; i < f->sizek; i++) {
  172. if (ttisstring(f->k + i))
  173. addTS(L, rawtsvalue(f->k + i));
  174. }
  175. for (i = 0; i < f->sizeupvalues; i++) addTS(L, f->upvalues[i]);
  176. for (i = 0; i < f->sizelocvars; i++) addTS(L, f->locvars[i].varname);
  177. for (i = 0; i < f->sizep; i++) scanProtoStrings(L, f->p[i]);
  178. }
  179. /*
  180. * Use the collected strings table to build the new ROstrt in the Flash Image
  181. *
  182. * The input is an array of {"SomeString" = 1, ...} on the ToS.
  183. * The output is an array of {"SomeString" = FlashOffset("SomeString"), ...} on ToS
  184. */
  185. static void createROstrt(lua_State *L, FlashHeader *fh) {
  186. /* Table at L->Top[-1] on input is hash used to collect the strings */
  187. /* Count the number of strings. Can't use objlen as this is a hash */
  188. fh->nROuse = 0;
  189. lua_pushnil(L); /* first key */
  190. while (lua_next(L, -2) != 0) {
  191. fh->nROuse++;
  192. DBG_PRINT("Found: %s\n",getstr(rawtsvalue(L->top-2)));
  193. lua_pop(L, 1); // dump the value
  194. }
  195. fh->nROsize = 2<<luaO_log2(fh->nROuse);
  196. FlashAddr *hashTab = flashAlloc(L, fh->nROsize * WORDSIZE);
  197. toFlashAddr(L, fh->pROhash, hashTab);
  198. /* Now iterate over the strings to be added to the RO string table and build it */
  199. lua_newtable(L); // add output table
  200. lua_pushnil(L); // First key
  201. while (lua_next(L, -3) != 0) { // replaces key, pushes value
  202. TString *ts = rawtsvalue(L->top - 2); // key.ts
  203. const char *p = getstr(ts); // C string of key
  204. uint hash = ts->tsv.hash; // hash of key
  205. size_t len = ts->tsv.len; // and length
  206. DBG_PRINT("2nd pass: %s\n",p);
  207. FlashAddr *e = hashTab + lmod(hash, fh->nROsize);
  208. FlashTS *last = cast(FlashTS *, fromFashAddr(*e));
  209. FlashTS *fts = cast(FlashTS *, flashAlloc(L, sizeof(FlashTS)));
  210. toFlashAddr(L, *e, fts); // add reference to TS to lookup vector
  211. toFlashAddr(L, fts->next, last); // and chain to previous entry if any
  212. fts->tt = LUA_TSTRING; // Set as String
  213. fts->marked = bitmask(LFSBIT); // LFS string with no Whitebits set
  214. fts->hash = hash; // add hash
  215. fts->len = len; // and length
  216. memcpy(flashAlloc(L, len+1), p, len+1); // copy string
  217. // include the trailing null char
  218. lua_pop(L, 1); // Junk the value
  219. lua_pushvalue(L, -1); // Dup the key as rawset dumps its copy
  220. lua_pushinteger(L, cast(FlashAddr*,fts)-flashImage); // Value is new TS offset.
  221. lua_rawset(L, -4); // Add to new table
  222. }
  223. /* At this point the old hash is done to derefence for GC */
  224. lua_remove(L, -2);
  225. }
  226. /*
  227. * Convert a TString reference in the host G(L)->strt entry into the corresponding
  228. * TString address in the flashImage using the lookup table at ToS
  229. */
  230. static void *resolveTString(lua_State* L, TString *s) {
  231. if (!s)
  232. return NULL;
  233. lua_pushnil(L);
  234. setsvalue(L, L->top-1, s);
  235. lua_rawget(L, -2);
  236. lua_assert(!lua_isnil(L, -1));
  237. void *ts = fromFashAddr(lua_tointeger(L, -1));
  238. lua_pop(L, 1);
  239. return ts;
  240. }
  241. /*
  242. * In order to simplify repacking of structures from the host format to that target
  243. * format, this simple copy routine is data-driven by a simple format specifier.
  244. * n Number of consecutive records to be processed
  245. * fmt A string of A, I, S, V specifiers spanning the record.
  246. * src Source of record
  247. * returns Address of destination record
  248. */
  249. #if defined(LUA_PACK_TVALUES)
  250. #define TARGET_TV_SIZE (sizeof(lua_Number)+sizeof(lu_int32))
  251. #else
  252. #define TARGET_TV_SIZE (2*sizeof(lua_Number))
  253. #endif
  254. static void *flashCopy(lua_State* L, int n, const char *fmt, void *src) {
  255. /* ToS is the string address mapping table */
  256. if (n == 0)
  257. return NULL;
  258. int i, recsize;
  259. void *newts;
  260. /* A bit of a botch because fmt is either "V" or a string of WORDSIZE specifiers */
  261. /* The size 8 / 12 / 16 bytes for integer builds, packed TV and default TVs resp */
  262. if (fmt[0]=='V') {
  263. lua_assert(fmt[1] == 0); /* V formats must be singetons */
  264. recsize = TARGET_TV_SIZE;
  265. } else {
  266. recsize = WORDSIZE * strlen(fmt);
  267. }
  268. uint *d = cast(uint *, flashAlloc(L, n * recsize));
  269. uint *dest = d;
  270. uint *s = cast(uint *, src);
  271. for (i = 0; i < n; i++) {
  272. const char *p = fmt;
  273. while (*p) {
  274. /* All input address types (A,S,V) are aligned to size_t boundaries */
  275. if (*p != 'I' && ((size_t)s)&(sizeof(size_t)-1))
  276. s++;
  277. switch (*p++) {
  278. case 'A':
  279. toFlashAddr(L, *d, *cast(void**, s));
  280. s += FLASH_WORDS(size_t);
  281. d++;
  282. break;
  283. case 'I':
  284. *d++ = *s++;
  285. break;
  286. case 'H':
  287. *d++ = (*s++) & 0;
  288. break;
  289. case 'S':
  290. newts = resolveTString(L, *cast(TString **, s));
  291. toFlashAddr(L, *d, newts);
  292. s += FLASH_WORDS(size_t);
  293. d++;
  294. break;
  295. case 'V':
  296. /* This code has to work for both Integer and Float build variants */
  297. memset(d, 0, TARGET_TV_SIZE);
  298. TValue *sv = cast(TValue *, s);
  299. /* The value is 0, 4 or 8 bytes depending on type */
  300. if (ttisstring(sv)) {
  301. toFlashAddr(L, *d, resolveTString(L, rawtsvalue(sv)));
  302. } else if (ttisnumber(sv)) {
  303. *cast(lua_Number*,d) = *cast(lua_Number*,s);
  304. } else if (!ttisnil(sv)){
  305. /* all other types are 4 byte */
  306. lua_assert(!iscollectable(sv));
  307. *cast(uint *,d) = *cast(uint *,s);
  308. }
  309. *cast(int *,cast(lua_Number*,d)+1) = ttype(sv);
  310. s += FLASH_WORDS(TValue);
  311. d += TARGET_TV_SIZE/WORDSIZE;
  312. break;
  313. default:
  314. lua_assert (0);
  315. }
  316. }
  317. }
  318. return dest;
  319. }
  320. /* The debug optimised version has a different Proto layout */
  321. #ifdef LUA_OPTIMIZE_DEBUG
  322. #define PROTO_COPY_MASK "AHAAAAAASIIIIIIIAI"
  323. #else
  324. #define PROTO_COPY_MASK "AHAAAAAASIIIIIIIIAI"
  325. #endif
  326. /*
  327. * Do the actual prototype copy.
  328. */
  329. static void *functionToFlash(lua_State* L, const Proto* orig) {
  330. Proto f;
  331. int i;
  332. memcpy (&f, orig, sizeof(Proto));
  333. f.gclist = NULL;
  334. f.next = NULL;
  335. l_setbit(f.marked, LFSBIT); /* OK to set the LFSBIT on a stack-cloned copy */
  336. if (f.sizep) { /* clone included Protos */
  337. Proto **p = luaM_newvector(L, f.sizep, Proto *);
  338. for (i=0; i<f.sizep; i++)
  339. p[i] = cast(Proto *, functionToFlash(L, f.p[i]));
  340. f.p = cast(Proto **, flashCopy(L, f.sizep, "A", p));
  341. luaM_freearray(L, p, f.sizep, Proto *);
  342. }
  343. f.k = cast(TValue *, flashCopy(L, f.sizek, "V", f.k));
  344. f.code = cast(Instruction *, flashCopy(L, f.sizecode, "I", f.code));
  345. #ifdef LUA_OPTIMIZE_DEBUG
  346. if (f.packedlineinfo) {
  347. TString *ts=luaS_new(L, cast(const char *,f.packedlineinfo));
  348. f.packedlineinfo = cast(unsigned char *, resolveTString(L, ts)) + sizeof (FlashTS);
  349. }
  350. #else
  351. f.lineinfo = cast(int *, flashCopy(L, f.sizelineinfo, "I", f.lineinfo));
  352. #endif
  353. f.locvars = cast(struct LocVar *, flashCopy(L, f.sizelocvars, "SII", f.locvars));
  354. f.upvalues = cast(TString **, flashCopy(L, f.sizeupvalues, "S", f.upvalues));
  355. return cast(void *, flashCopy(L, 1, PROTO_COPY_MASK, &f));
  356. }
  357. uint dumpToFlashImage (lua_State* L, const Proto *main, lua_Writer w,
  358. void* data, int strip,
  359. lu_int32 address, lu_int32 maxSize) {
  360. // parameter strip is ignored for now
  361. FlashHeader *fh = cast(FlashHeader *, flashAlloc(L, sizeof(FlashHeader)));
  362. int i, status;
  363. lua_newtable(L);
  364. scanProtoStrings(L, main);
  365. createROstrt(L, fh);
  366. toFlashAddr(L, fh->mainProto, functionToFlash(L, main));
  367. fh->flash_sig = FLASH_SIG + (address ? FLASH_SIG_ABSOLUTE : 0);
  368. fh->flash_size = curOffset*WORDSIZE;
  369. if (fh->flash_size>maxSize) {
  370. fatal ("The image is too large for specfied LFS size");
  371. }
  372. if (address) { /* in absolute mode convert addresses to mapped address */
  373. for (i = 0 ; i < curOffset; i++)
  374. if (getFlashAddrTag(i))
  375. flashImage[i] = 4*flashImage[i] + address;
  376. lua_unlock(L);
  377. status = w(L, flashImage, fh->flash_size, data);
  378. } else { /* compressed PI mode */
  379. /*
  380. * In image mode, shift the relocation bitmap down directly above
  381. * the used flashimage. This consolidated array is then gzipped.
  382. */
  383. uint oLen;
  384. uint8_t *oBuf;
  385. int bmLen = sizeof(uint)*((curOffset+31)/32); /* 32 flags to a word */
  386. memmove(flashImage+curOffset, flashAddrTag, bmLen);
  387. status = uzlib_compress (&oBuf, &oLen,
  388. (const uint8_t *)flashImage, bmLen+fh->flash_size);
  389. if (status != UZLIB_OK) {
  390. luac_fatal("Out of memory during image compression");
  391. }
  392. lua_unlock(L);
  393. #if 0
  394. status = w(L, flashImage, bmLen+fh->flash_size, data);
  395. #else
  396. status = w(L, oBuf, oLen, data);
  397. free(oBuf);
  398. #endif
  399. }
  400. lua_lock(L);
  401. return status;
  402. }