ram_core.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2012 Google, Inc.
  4. */
  5. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  6. #include <linux/device.h>
  7. #include <linux/err.h>
  8. #include <linux/errno.h>
  9. #include <linux/init.h>
  10. #include <linux/io.h>
  11. #include <linux/kernel.h>
  12. #include <linux/list.h>
  13. #include <linux/memblock.h>
  14. #include <linux/pstore_ram.h>
  15. #include <linux/rslib.h>
  16. #include <linux/slab.h>
  17. #include <linux/uaccess.h>
  18. #include <linux/vmalloc.h>
  19. #include <asm/page.h>
  20. /**
  21. * struct persistent_ram_buffer - persistent circular RAM buffer
  22. *
  23. * @sig:
  24. * signature to indicate header (PERSISTENT_RAM_SIG xor PRZ-type value)
  25. * @start:
  26. * offset into @data where the beginning of the stored bytes begin
  27. * @size:
  28. * number of valid bytes stored in @data
  29. */
  30. struct persistent_ram_buffer {
  31. uint32_t sig;
  32. atomic_t start;
  33. atomic_t size;
  34. uint8_t data[];
  35. };
  36. #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */
  37. static inline size_t buffer_size(struct persistent_ram_zone *prz)
  38. {
  39. return atomic_read(&prz->buffer->size);
  40. }
  41. static inline size_t buffer_start(struct persistent_ram_zone *prz)
  42. {
  43. return atomic_read(&prz->buffer->start);
  44. }
  45. /* increase and wrap the start pointer, returning the old value */
  46. static size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a)
  47. {
  48. int old;
  49. int new;
  50. unsigned long flags = 0;
  51. if (!(prz->flags & PRZ_FLAG_NO_LOCK))
  52. raw_spin_lock_irqsave(&prz->buffer_lock, flags);
  53. old = atomic_read(&prz->buffer->start);
  54. new = old + a;
  55. while (unlikely(new >= prz->buffer_size))
  56. new -= prz->buffer_size;
  57. atomic_set(&prz->buffer->start, new);
  58. if (!(prz->flags & PRZ_FLAG_NO_LOCK))
  59. raw_spin_unlock_irqrestore(&prz->buffer_lock, flags);
  60. return old;
  61. }
  62. /* increase the size counter until it hits the max size */
  63. static void buffer_size_add(struct persistent_ram_zone *prz, size_t a)
  64. {
  65. size_t old;
  66. size_t new;
  67. unsigned long flags = 0;
  68. if (!(prz->flags & PRZ_FLAG_NO_LOCK))
  69. raw_spin_lock_irqsave(&prz->buffer_lock, flags);
  70. old = atomic_read(&prz->buffer->size);
  71. if (old == prz->buffer_size)
  72. goto exit;
  73. new = old + a;
  74. if (new > prz->buffer_size)
  75. new = prz->buffer_size;
  76. atomic_set(&prz->buffer->size, new);
  77. exit:
  78. if (!(prz->flags & PRZ_FLAG_NO_LOCK))
  79. raw_spin_unlock_irqrestore(&prz->buffer_lock, flags);
  80. }
  81. static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz,
  82. uint8_t *data, size_t len, uint8_t *ecc)
  83. {
  84. int i;
  85. /* Initialize the parity buffer */
  86. memset(prz->ecc_info.par, 0,
  87. prz->ecc_info.ecc_size * sizeof(prz->ecc_info.par[0]));
  88. encode_rs8(prz->rs_decoder, data, len, prz->ecc_info.par, 0);
  89. for (i = 0; i < prz->ecc_info.ecc_size; i++)
  90. ecc[i] = prz->ecc_info.par[i];
  91. }
  92. static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz,
  93. void *data, size_t len, uint8_t *ecc)
  94. {
  95. int i;
  96. for (i = 0; i < prz->ecc_info.ecc_size; i++)
  97. prz->ecc_info.par[i] = ecc[i];
  98. return decode_rs8(prz->rs_decoder, data, prz->ecc_info.par, len,
  99. NULL, 0, NULL, 0, NULL);
  100. }
  101. static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz,
  102. unsigned int start, unsigned int count)
  103. {
  104. struct persistent_ram_buffer *buffer = prz->buffer;
  105. uint8_t *buffer_end = buffer->data + prz->buffer_size;
  106. uint8_t *block;
  107. uint8_t *par;
  108. int ecc_block_size = prz->ecc_info.block_size;
  109. int ecc_size = prz->ecc_info.ecc_size;
  110. int size = ecc_block_size;
  111. if (!ecc_size)
  112. return;
  113. block = buffer->data + (start & ~(ecc_block_size - 1));
  114. par = prz->par_buffer + (start / ecc_block_size) * ecc_size;
  115. do {
  116. if (block + ecc_block_size > buffer_end)
  117. size = buffer_end - block;
  118. persistent_ram_encode_rs8(prz, block, size, par);
  119. block += ecc_block_size;
  120. par += ecc_size;
  121. } while (block < buffer->data + start + count);
  122. }
  123. static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz)
  124. {
  125. struct persistent_ram_buffer *buffer = prz->buffer;
  126. if (!prz->ecc_info.ecc_size)
  127. return;
  128. persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer),
  129. prz->par_header);
  130. }
  131. static void persistent_ram_ecc_old(struct persistent_ram_zone *prz)
  132. {
  133. struct persistent_ram_buffer *buffer = prz->buffer;
  134. uint8_t *block;
  135. uint8_t *par;
  136. if (!prz->ecc_info.ecc_size)
  137. return;
  138. block = buffer->data;
  139. par = prz->par_buffer;
  140. while (block < buffer->data + buffer_size(prz)) {
  141. int numerr;
  142. int size = prz->ecc_info.block_size;
  143. if (block + size > buffer->data + prz->buffer_size)
  144. size = buffer->data + prz->buffer_size - block;
  145. numerr = persistent_ram_decode_rs8(prz, block, size, par);
  146. if (numerr > 0) {
  147. pr_devel("error in block %p, %d\n", block, numerr);
  148. prz->corrected_bytes += numerr;
  149. } else if (numerr < 0) {
  150. pr_devel("uncorrectable error in block %p\n", block);
  151. prz->bad_blocks++;
  152. }
  153. block += prz->ecc_info.block_size;
  154. par += prz->ecc_info.ecc_size;
  155. }
  156. }
  157. static int persistent_ram_init_ecc(struct persistent_ram_zone *prz,
  158. struct persistent_ram_ecc_info *ecc_info)
  159. {
  160. int numerr;
  161. struct persistent_ram_buffer *buffer = prz->buffer;
  162. int ecc_blocks;
  163. size_t ecc_total;
  164. if (!ecc_info || !ecc_info->ecc_size)
  165. return 0;
  166. prz->ecc_info.block_size = ecc_info->block_size ?: 128;
  167. prz->ecc_info.ecc_size = ecc_info->ecc_size ?: 16;
  168. prz->ecc_info.symsize = ecc_info->symsize ?: 8;
  169. prz->ecc_info.poly = ecc_info->poly ?: 0x11d;
  170. ecc_blocks = DIV_ROUND_UP(prz->buffer_size - prz->ecc_info.ecc_size,
  171. prz->ecc_info.block_size +
  172. prz->ecc_info.ecc_size);
  173. ecc_total = (ecc_blocks + 1) * prz->ecc_info.ecc_size;
  174. if (ecc_total >= prz->buffer_size) {
  175. pr_err("%s: invalid ecc_size %u (total %zu, buffer size %zu)\n",
  176. __func__, prz->ecc_info.ecc_size,
  177. ecc_total, prz->buffer_size);
  178. return -EINVAL;
  179. }
  180. prz->buffer_size -= ecc_total;
  181. prz->par_buffer = buffer->data + prz->buffer_size;
  182. prz->par_header = prz->par_buffer +
  183. ecc_blocks * prz->ecc_info.ecc_size;
  184. /*
  185. * first consecutive root is 0
  186. * primitive element to generate roots = 1
  187. */
  188. prz->rs_decoder = init_rs(prz->ecc_info.symsize, prz->ecc_info.poly,
  189. 0, 1, prz->ecc_info.ecc_size);
  190. if (prz->rs_decoder == NULL) {
  191. pr_info("init_rs failed\n");
  192. return -EINVAL;
  193. }
  194. /* allocate workspace instead of using stack VLA */
  195. prz->ecc_info.par = kmalloc_array(prz->ecc_info.ecc_size,
  196. sizeof(*prz->ecc_info.par),
  197. GFP_KERNEL);
  198. if (!prz->ecc_info.par) {
  199. pr_err("cannot allocate ECC parity workspace\n");
  200. return -ENOMEM;
  201. }
  202. prz->corrected_bytes = 0;
  203. prz->bad_blocks = 0;
  204. numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer),
  205. prz->par_header);
  206. if (numerr > 0) {
  207. pr_info("error in header, %d\n", numerr);
  208. prz->corrected_bytes += numerr;
  209. } else if (numerr < 0) {
  210. pr_info("uncorrectable error in header\n");
  211. prz->bad_blocks++;
  212. }
  213. return 0;
  214. }
  215. ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz,
  216. char *str, size_t len)
  217. {
  218. ssize_t ret;
  219. if (!prz->ecc_info.ecc_size)
  220. return 0;
  221. if (prz->corrected_bytes || prz->bad_blocks)
  222. ret = snprintf(str, len, ""
  223. "\n%d Corrected bytes, %d unrecoverable blocks\n",
  224. prz->corrected_bytes, prz->bad_blocks);
  225. else
  226. ret = snprintf(str, len, "\nNo errors detected\n");
  227. return ret;
  228. }
  229. static void notrace persistent_ram_update(struct persistent_ram_zone *prz,
  230. const void *s, unsigned int start, unsigned int count)
  231. {
  232. struct persistent_ram_buffer *buffer = prz->buffer;
  233. memcpy_toio(buffer->data + start, s, count);
  234. persistent_ram_update_ecc(prz, start, count);
  235. }
  236. static int notrace persistent_ram_update_user(struct persistent_ram_zone *prz,
  237. const void __user *s, unsigned int start, unsigned int count)
  238. {
  239. struct persistent_ram_buffer *buffer = prz->buffer;
  240. int ret = unlikely(copy_from_user(buffer->data + start, s, count)) ?
  241. -EFAULT : 0;
  242. persistent_ram_update_ecc(prz, start, count);
  243. return ret;
  244. }
  245. void persistent_ram_save_old(struct persistent_ram_zone *prz)
  246. {
  247. struct persistent_ram_buffer *buffer = prz->buffer;
  248. size_t size = buffer_size(prz);
  249. size_t start = buffer_start(prz);
  250. if (!size)
  251. return;
  252. if (!prz->old_log) {
  253. persistent_ram_ecc_old(prz);
  254. prz->old_log = kmalloc(size, GFP_KERNEL);
  255. }
  256. if (!prz->old_log) {
  257. pr_err("failed to allocate buffer\n");
  258. return;
  259. }
  260. prz->old_log_size = size;
  261. memcpy_fromio(prz->old_log, &buffer->data[start], size - start);
  262. memcpy_fromio(prz->old_log + size - start, &buffer->data[0], start);
  263. }
  264. int notrace persistent_ram_write(struct persistent_ram_zone *prz,
  265. const void *s, unsigned int count)
  266. {
  267. int rem;
  268. int c = count;
  269. size_t start;
  270. if (unlikely(c > prz->buffer_size)) {
  271. s += c - prz->buffer_size;
  272. c = prz->buffer_size;
  273. }
  274. buffer_size_add(prz, c);
  275. start = buffer_start_add(prz, c);
  276. rem = prz->buffer_size - start;
  277. if (unlikely(rem < c)) {
  278. persistent_ram_update(prz, s, start, rem);
  279. s += rem;
  280. c -= rem;
  281. start = 0;
  282. }
  283. persistent_ram_update(prz, s, start, c);
  284. persistent_ram_update_header_ecc(prz);
  285. return count;
  286. }
  287. int notrace persistent_ram_write_user(struct persistent_ram_zone *prz,
  288. const void __user *s, unsigned int count)
  289. {
  290. int rem, ret = 0, c = count;
  291. size_t start;
  292. if (unlikely(c > prz->buffer_size)) {
  293. s += c - prz->buffer_size;
  294. c = prz->buffer_size;
  295. }
  296. buffer_size_add(prz, c);
  297. start = buffer_start_add(prz, c);
  298. rem = prz->buffer_size - start;
  299. if (unlikely(rem < c)) {
  300. ret = persistent_ram_update_user(prz, s, start, rem);
  301. s += rem;
  302. c -= rem;
  303. start = 0;
  304. }
  305. if (likely(!ret))
  306. ret = persistent_ram_update_user(prz, s, start, c);
  307. persistent_ram_update_header_ecc(prz);
  308. return unlikely(ret) ? ret : count;
  309. }
  310. size_t persistent_ram_old_size(struct persistent_ram_zone *prz)
  311. {
  312. return prz->old_log_size;
  313. }
  314. void *persistent_ram_old(struct persistent_ram_zone *prz)
  315. {
  316. return prz->old_log;
  317. }
  318. void persistent_ram_free_old(struct persistent_ram_zone *prz)
  319. {
  320. kfree(prz->old_log);
  321. prz->old_log = NULL;
  322. prz->old_log_size = 0;
  323. }
  324. void persistent_ram_zap(struct persistent_ram_zone *prz)
  325. {
  326. atomic_set(&prz->buffer->start, 0);
  327. atomic_set(&prz->buffer->size, 0);
  328. persistent_ram_update_header_ecc(prz);
  329. }
  330. #define MEM_TYPE_WCOMBINE 0
  331. #define MEM_TYPE_NONCACHED 1
  332. #define MEM_TYPE_NORMAL 2
  333. static void *persistent_ram_vmap(phys_addr_t start, size_t size,
  334. unsigned int memtype)
  335. {
  336. struct page **pages;
  337. phys_addr_t page_start;
  338. unsigned int page_count;
  339. pgprot_t prot;
  340. unsigned int i;
  341. void *vaddr;
  342. page_start = start - offset_in_page(start);
  343. page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE);
  344. switch (memtype) {
  345. case MEM_TYPE_NORMAL:
  346. prot = PAGE_KERNEL;
  347. break;
  348. case MEM_TYPE_NONCACHED:
  349. prot = pgprot_noncached(PAGE_KERNEL);
  350. break;
  351. case MEM_TYPE_WCOMBINE:
  352. prot = pgprot_writecombine(PAGE_KERNEL);
  353. break;
  354. default:
  355. pr_err("invalid mem_type=%d\n", memtype);
  356. return NULL;
  357. }
  358. pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL);
  359. if (!pages) {
  360. pr_err("%s: Failed to allocate array for %u pages\n",
  361. __func__, page_count);
  362. return NULL;
  363. }
  364. for (i = 0; i < page_count; i++) {
  365. phys_addr_t addr = page_start + i * PAGE_SIZE;
  366. pages[i] = pfn_to_page(addr >> PAGE_SHIFT);
  367. }
  368. vaddr = vmap(pages, page_count, VM_MAP, prot);
  369. kfree(pages);
  370. /*
  371. * Since vmap() uses page granularity, we must add the offset
  372. * into the page here, to get the byte granularity address
  373. * into the mapping to represent the actual "start" location.
  374. */
  375. return vaddr + offset_in_page(start);
  376. }
  377. static void *persistent_ram_iomap(phys_addr_t start, size_t size,
  378. unsigned int memtype, char *label)
  379. {
  380. void *va;
  381. if (!request_mem_region(start, size, label ?: "ramoops")) {
  382. pr_err("request mem region (%s 0x%llx@0x%llx) failed\n",
  383. label ?: "ramoops",
  384. (unsigned long long)size, (unsigned long long)start);
  385. return NULL;
  386. }
  387. if (memtype)
  388. va = ioremap(start, size);
  389. else
  390. va = ioremap_wc(start, size);
  391. /*
  392. * Since request_mem_region() and ioremap() are byte-granularity
  393. * there is no need handle anything special like we do when the
  394. * vmap() case in persistent_ram_vmap() above.
  395. */
  396. return va;
  397. }
  398. static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size,
  399. struct persistent_ram_zone *prz, int memtype)
  400. {
  401. prz->paddr = start;
  402. prz->size = size;
  403. if (pfn_valid(start >> PAGE_SHIFT))
  404. prz->vaddr = persistent_ram_vmap(start, size, memtype);
  405. else
  406. prz->vaddr = persistent_ram_iomap(start, size, memtype,
  407. prz->label);
  408. if (!prz->vaddr) {
  409. pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__,
  410. (unsigned long long)size, (unsigned long long)start);
  411. return -ENOMEM;
  412. }
  413. prz->buffer = prz->vaddr;
  414. prz->buffer_size = size - sizeof(struct persistent_ram_buffer);
  415. return 0;
  416. }
  417. static int persistent_ram_post_init(struct persistent_ram_zone *prz, u32 sig,
  418. struct persistent_ram_ecc_info *ecc_info)
  419. {
  420. int ret;
  421. bool zap = !!(prz->flags & PRZ_FLAG_ZAP_OLD);
  422. ret = persistent_ram_init_ecc(prz, ecc_info);
  423. if (ret) {
  424. pr_warn("ECC failed %s\n", prz->label);
  425. return ret;
  426. }
  427. sig ^= PERSISTENT_RAM_SIG;
  428. if (prz->buffer->sig == sig) {
  429. if (buffer_size(prz) == 0) {
  430. pr_debug("found existing empty buffer\n");
  431. return 0;
  432. }
  433. if (buffer_size(prz) > prz->buffer_size ||
  434. buffer_start(prz) > buffer_size(prz)) {
  435. pr_info("found existing invalid buffer, size %zu, start %zu\n",
  436. buffer_size(prz), buffer_start(prz));
  437. zap = true;
  438. } else {
  439. pr_debug("found existing buffer, size %zu, start %zu\n",
  440. buffer_size(prz), buffer_start(prz));
  441. persistent_ram_save_old(prz);
  442. }
  443. } else {
  444. pr_debug("no valid data in buffer (sig = 0x%08x)\n",
  445. prz->buffer->sig);
  446. prz->buffer->sig = sig;
  447. zap = true;
  448. }
  449. /* Reset missing, invalid, or single-use memory area. */
  450. if (zap)
  451. persistent_ram_zap(prz);
  452. return 0;
  453. }
  454. void persistent_ram_free(struct persistent_ram_zone *prz)
  455. {
  456. if (!prz)
  457. return;
  458. if (prz->vaddr) {
  459. if (pfn_valid(prz->paddr >> PAGE_SHIFT)) {
  460. /* We must vunmap() at page-granularity. */
  461. vunmap(prz->vaddr - offset_in_page(prz->paddr));
  462. } else {
  463. iounmap(prz->vaddr);
  464. release_mem_region(prz->paddr, prz->size);
  465. }
  466. prz->vaddr = NULL;
  467. }
  468. if (prz->rs_decoder) {
  469. free_rs(prz->rs_decoder);
  470. prz->rs_decoder = NULL;
  471. }
  472. kfree(prz->ecc_info.par);
  473. prz->ecc_info.par = NULL;
  474. persistent_ram_free_old(prz);
  475. kfree(prz->label);
  476. kfree(prz);
  477. }
  478. struct persistent_ram_zone *persistent_ram_new(phys_addr_t start, size_t size,
  479. u32 sig, struct persistent_ram_ecc_info *ecc_info,
  480. unsigned int memtype, u32 flags, char *label)
  481. {
  482. struct persistent_ram_zone *prz;
  483. int ret = -ENOMEM;
  484. prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL);
  485. if (!prz) {
  486. pr_err("failed to allocate persistent ram zone\n");
  487. goto err;
  488. }
  489. /* Initialize general buffer state. */
  490. raw_spin_lock_init(&prz->buffer_lock);
  491. prz->flags = flags;
  492. prz->label = kstrdup(label, GFP_KERNEL);
  493. ret = persistent_ram_buffer_map(start, size, prz, memtype);
  494. if (ret)
  495. goto err;
  496. ret = persistent_ram_post_init(prz, sig, ecc_info);
  497. if (ret)
  498. goto err;
  499. pr_debug("attached %s 0x%zx@0x%llx: %zu header, %zu data, %zu ecc (%d/%d)\n",
  500. prz->label, prz->size, (unsigned long long)prz->paddr,
  501. sizeof(*prz->buffer), prz->buffer_size,
  502. prz->size - sizeof(*prz->buffer) - prz->buffer_size,
  503. prz->ecc_info.ecc_size, prz->ecc_info.block_size);
  504. return prz;
  505. err:
  506. persistent_ram_free(prz);
  507. return ERR_PTR(ret);
  508. }