cec.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2017-2019 Borislav Petkov, SUSE Labs.
  4. */
  5. #include <linux/mm.h>
  6. #include <linux/gfp.h>
  7. #include <linux/ras.h>
  8. #include <linux/kernel.h>
  9. #include <linux/workqueue.h>
  10. #include <asm/mce.h>
  11. #include "debugfs.h"
  12. /*
  13. * RAS Correctable Errors Collector
  14. *
  15. * This is a simple gadget which collects correctable errors and counts their
  16. * occurrence per physical page address.
  17. *
  18. * We've opted for possibly the simplest data structure to collect those - an
  19. * array of the size of a memory page. It stores 512 u64's with the following
  20. * structure:
  21. *
  22. * [63 ... PFN ... 12 | 11 ... generation ... 10 | 9 ... count ... 0]
  23. *
  24. * The generation in the two highest order bits is two bits which are set to 11b
  25. * on every insertion. During the course of each entry's existence, the
  26. * generation field gets decremented during spring cleaning to 10b, then 01b and
  27. * then 00b.
  28. *
  29. * This way we're employing the natural numeric ordering to make sure that newly
  30. * inserted/touched elements have higher 12-bit counts (which we've manufactured)
  31. * and thus iterating over the array initially won't kick out those elements
  32. * which were inserted last.
  33. *
  34. * Spring cleaning is what we do when we reach a certain number CLEAN_ELEMS of
  35. * elements entered into the array, during which, we're decaying all elements.
  36. * If, after decay, an element gets inserted again, its generation is set to 11b
  37. * to make sure it has higher numerical count than other, older elements and
  38. * thus emulate an an LRU-like behavior when deleting elements to free up space
  39. * in the page.
  40. *
  41. * When an element reaches it's max count of action_threshold, we try to poison
  42. * it by assuming that errors triggered action_threshold times in a single page
  43. * are excessive and that page shouldn't be used anymore. action_threshold is
  44. * initialized to COUNT_MASK which is the maximum.
  45. *
  46. * That error event entry causes cec_add_elem() to return !0 value and thus
  47. * signal to its callers to log the error.
  48. *
  49. * To the question why we've chosen a page and moving elements around with
  50. * memmove(), it is because it is a very simple structure to handle and max data
  51. * movement is 4K which on highly optimized modern CPUs is almost unnoticeable.
  52. * We wanted to avoid the pointer traversal of more complex structures like a
  53. * linked list or some sort of a balancing search tree.
  54. *
  55. * Deleting an element takes O(n) but since it is only a single page, it should
  56. * be fast enough and it shouldn't happen all too often depending on error
  57. * patterns.
  58. */
  59. #undef pr_fmt
  60. #define pr_fmt(fmt) "RAS: " fmt
  61. /*
  62. * We use DECAY_BITS bits of PAGE_SHIFT bits for counting decay, i.e., how long
  63. * elements have stayed in the array without having been accessed again.
  64. */
  65. #define DECAY_BITS 2
  66. #define DECAY_MASK ((1ULL << DECAY_BITS) - 1)
  67. #define MAX_ELEMS (PAGE_SIZE / sizeof(u64))
  68. /*
  69. * Threshold amount of inserted elements after which we start spring
  70. * cleaning.
  71. */
  72. #define CLEAN_ELEMS (MAX_ELEMS >> DECAY_BITS)
  73. /* Bits which count the number of errors happened in this 4K page. */
  74. #define COUNT_BITS (PAGE_SHIFT - DECAY_BITS)
  75. #define COUNT_MASK ((1ULL << COUNT_BITS) - 1)
  76. #define FULL_COUNT_MASK (PAGE_SIZE - 1)
  77. /*
  78. * u64: [ 63 ... 12 | DECAY_BITS | COUNT_BITS ]
  79. */
  80. #define PFN(e) ((e) >> PAGE_SHIFT)
  81. #define DECAY(e) (((e) >> COUNT_BITS) & DECAY_MASK)
  82. #define COUNT(e) ((unsigned int)(e) & COUNT_MASK)
  83. #define FULL_COUNT(e) ((e) & (PAGE_SIZE - 1))
  84. static struct ce_array {
  85. u64 *array; /* container page */
  86. unsigned int n; /* number of elements in the array */
  87. unsigned int decay_count; /*
  88. * number of element insertions/increments
  89. * since the last spring cleaning.
  90. */
  91. u64 pfns_poisoned; /*
  92. * number of PFNs which got poisoned.
  93. */
  94. u64 ces_entered; /*
  95. * The number of correctable errors
  96. * entered into the collector.
  97. */
  98. u64 decays_done; /*
  99. * Times we did spring cleaning.
  100. */
  101. union {
  102. struct {
  103. __u32 disabled : 1, /* cmdline disabled */
  104. __resv : 31;
  105. };
  106. __u32 flags;
  107. };
  108. } ce_arr;
  109. static DEFINE_MUTEX(ce_mutex);
  110. static u64 dfs_pfn;
  111. /* Amount of errors after which we offline */
  112. static u64 action_threshold = COUNT_MASK;
  113. /* Each element "decays" each decay_interval which is 24hrs by default. */
  114. #define CEC_DECAY_DEFAULT_INTERVAL 24 * 60 * 60 /* 24 hrs */
  115. #define CEC_DECAY_MIN_INTERVAL 1 * 60 * 60 /* 1h */
  116. #define CEC_DECAY_MAX_INTERVAL 30 * 24 * 60 * 60 /* one month */
  117. static struct delayed_work cec_work;
  118. static u64 decay_interval = CEC_DECAY_DEFAULT_INTERVAL;
  119. /*
  120. * Decrement decay value. We're using DECAY_BITS bits to denote decay of an
  121. * element in the array. On insertion and any access, it gets reset to max.
  122. */
  123. static void do_spring_cleaning(struct ce_array *ca)
  124. {
  125. int i;
  126. for (i = 0; i < ca->n; i++) {
  127. u8 decay = DECAY(ca->array[i]);
  128. if (!decay)
  129. continue;
  130. decay--;
  131. ca->array[i] &= ~(DECAY_MASK << COUNT_BITS);
  132. ca->array[i] |= (decay << COUNT_BITS);
  133. }
  134. ca->decay_count = 0;
  135. ca->decays_done++;
  136. }
  137. /*
  138. * @interval in seconds
  139. */
  140. static void cec_mod_work(unsigned long interval)
  141. {
  142. unsigned long iv;
  143. iv = interval * HZ;
  144. mod_delayed_work(system_wq, &cec_work, round_jiffies(iv));
  145. }
  146. static void cec_work_fn(struct work_struct *work)
  147. {
  148. mutex_lock(&ce_mutex);
  149. do_spring_cleaning(&ce_arr);
  150. mutex_unlock(&ce_mutex);
  151. cec_mod_work(decay_interval);
  152. }
  153. /*
  154. * @to: index of the smallest element which is >= then @pfn.
  155. *
  156. * Return the index of the pfn if found, otherwise negative value.
  157. */
  158. static int __find_elem(struct ce_array *ca, u64 pfn, unsigned int *to)
  159. {
  160. int min = 0, max = ca->n - 1;
  161. u64 this_pfn;
  162. while (min <= max) {
  163. int i = (min + max) >> 1;
  164. this_pfn = PFN(ca->array[i]);
  165. if (this_pfn < pfn)
  166. min = i + 1;
  167. else if (this_pfn > pfn)
  168. max = i - 1;
  169. else if (this_pfn == pfn) {
  170. if (to)
  171. *to = i;
  172. return i;
  173. }
  174. }
  175. /*
  176. * When the loop terminates without finding @pfn, min has the index of
  177. * the element slot where the new @pfn should be inserted. The loop
  178. * terminates when min > max, which means the min index points to the
  179. * bigger element while the max index to the smaller element, in-between
  180. * which the new @pfn belongs to.
  181. *
  182. * For more details, see exercise 1, Section 6.2.1 in TAOCP, vol. 3.
  183. */
  184. if (to)
  185. *to = min;
  186. return -ENOKEY;
  187. }
  188. static int find_elem(struct ce_array *ca, u64 pfn, unsigned int *to)
  189. {
  190. WARN_ON(!to);
  191. if (!ca->n) {
  192. *to = 0;
  193. return -ENOKEY;
  194. }
  195. return __find_elem(ca, pfn, to);
  196. }
  197. static void del_elem(struct ce_array *ca, int idx)
  198. {
  199. /* Save us a function call when deleting the last element. */
  200. if (ca->n - (idx + 1))
  201. memmove((void *)&ca->array[idx],
  202. (void *)&ca->array[idx + 1],
  203. (ca->n - (idx + 1)) * sizeof(u64));
  204. ca->n--;
  205. }
  206. static u64 del_lru_elem_unlocked(struct ce_array *ca)
  207. {
  208. unsigned int min = FULL_COUNT_MASK;
  209. int i, min_idx = 0;
  210. for (i = 0; i < ca->n; i++) {
  211. unsigned int this = FULL_COUNT(ca->array[i]);
  212. if (min > this) {
  213. min = this;
  214. min_idx = i;
  215. }
  216. }
  217. del_elem(ca, min_idx);
  218. return PFN(ca->array[min_idx]);
  219. }
  220. /*
  221. * We return the 0th pfn in the error case under the assumption that it cannot
  222. * be poisoned and excessive CEs in there are a serious deal anyway.
  223. */
  224. static u64 __maybe_unused del_lru_elem(void)
  225. {
  226. struct ce_array *ca = &ce_arr;
  227. u64 pfn;
  228. if (!ca->n)
  229. return 0;
  230. mutex_lock(&ce_mutex);
  231. pfn = del_lru_elem_unlocked(ca);
  232. mutex_unlock(&ce_mutex);
  233. return pfn;
  234. }
  235. static bool sanity_check(struct ce_array *ca)
  236. {
  237. bool ret = false;
  238. u64 prev = 0;
  239. int i;
  240. for (i = 0; i < ca->n; i++) {
  241. u64 this = PFN(ca->array[i]);
  242. if (WARN(prev > this, "prev: 0x%016llx <-> this: 0x%016llx\n", prev, this))
  243. ret = true;
  244. prev = this;
  245. }
  246. if (!ret)
  247. return ret;
  248. pr_info("Sanity check dump:\n{ n: %d\n", ca->n);
  249. for (i = 0; i < ca->n; i++) {
  250. u64 this = PFN(ca->array[i]);
  251. pr_info(" %03d: [%016llx|%03llx]\n", i, this, FULL_COUNT(ca->array[i]));
  252. }
  253. pr_info("}\n");
  254. return ret;
  255. }
  256. /**
  257. * cec_add_elem - Add an element to the CEC array.
  258. * @pfn: page frame number to insert
  259. *
  260. * Return values:
  261. * - <0: on error
  262. * - 0: on success
  263. * - >0: when the inserted pfn was offlined
  264. */
  265. static int cec_add_elem(u64 pfn)
  266. {
  267. struct ce_array *ca = &ce_arr;
  268. int count, err, ret = 0;
  269. unsigned int to = 0;
  270. /*
  271. * We can be called very early on the identify_cpu() path where we are
  272. * not initialized yet. We ignore the error for simplicity.
  273. */
  274. if (!ce_arr.array || ce_arr.disabled)
  275. return -ENODEV;
  276. mutex_lock(&ce_mutex);
  277. ca->ces_entered++;
  278. /* Array full, free the LRU slot. */
  279. if (ca->n == MAX_ELEMS)
  280. WARN_ON(!del_lru_elem_unlocked(ca));
  281. err = find_elem(ca, pfn, &to);
  282. if (err < 0) {
  283. /*
  284. * Shift range [to-end] to make room for one more element.
  285. */
  286. memmove((void *)&ca->array[to + 1],
  287. (void *)&ca->array[to],
  288. (ca->n - to) * sizeof(u64));
  289. ca->array[to] = pfn << PAGE_SHIFT;
  290. ca->n++;
  291. }
  292. /* Add/refresh element generation and increment count */
  293. ca->array[to] |= DECAY_MASK << COUNT_BITS;
  294. ca->array[to]++;
  295. /* Check action threshold and soft-offline, if reached. */
  296. count = COUNT(ca->array[to]);
  297. if (count >= action_threshold) {
  298. u64 pfn = ca->array[to] >> PAGE_SHIFT;
  299. if (!pfn_valid(pfn)) {
  300. pr_warn("CEC: Invalid pfn: 0x%llx\n", pfn);
  301. } else {
  302. /* We have reached max count for this page, soft-offline it. */
  303. pr_err("Soft-offlining pfn: 0x%llx\n", pfn);
  304. memory_failure_queue(pfn, MF_SOFT_OFFLINE);
  305. ca->pfns_poisoned++;
  306. }
  307. del_elem(ca, to);
  308. /*
  309. * Return a >0 value to callers, to denote that we've reached
  310. * the offlining threshold.
  311. */
  312. ret = 1;
  313. goto unlock;
  314. }
  315. ca->decay_count++;
  316. if (ca->decay_count >= CLEAN_ELEMS)
  317. do_spring_cleaning(ca);
  318. WARN_ON_ONCE(sanity_check(ca));
  319. unlock:
  320. mutex_unlock(&ce_mutex);
  321. return ret;
  322. }
  323. static int u64_get(void *data, u64 *val)
  324. {
  325. *val = *(u64 *)data;
  326. return 0;
  327. }
  328. static int pfn_set(void *data, u64 val)
  329. {
  330. *(u64 *)data = val;
  331. cec_add_elem(val);
  332. return 0;
  333. }
  334. DEFINE_DEBUGFS_ATTRIBUTE(pfn_ops, u64_get, pfn_set, "0x%llx\n");
  335. static int decay_interval_set(void *data, u64 val)
  336. {
  337. if (val < CEC_DECAY_MIN_INTERVAL)
  338. return -EINVAL;
  339. if (val > CEC_DECAY_MAX_INTERVAL)
  340. return -EINVAL;
  341. *(u64 *)data = val;
  342. decay_interval = val;
  343. cec_mod_work(decay_interval);
  344. return 0;
  345. }
  346. DEFINE_DEBUGFS_ATTRIBUTE(decay_interval_ops, u64_get, decay_interval_set, "%lld\n");
  347. static int action_threshold_set(void *data, u64 val)
  348. {
  349. *(u64 *)data = val;
  350. if (val > COUNT_MASK)
  351. val = COUNT_MASK;
  352. action_threshold = val;
  353. return 0;
  354. }
  355. DEFINE_DEBUGFS_ATTRIBUTE(action_threshold_ops, u64_get, action_threshold_set, "%lld\n");
  356. static const char * const bins[] = { "00", "01", "10", "11" };
  357. static int array_show(struct seq_file *m, void *v)
  358. {
  359. struct ce_array *ca = &ce_arr;
  360. int i;
  361. mutex_lock(&ce_mutex);
  362. seq_printf(m, "{ n: %d\n", ca->n);
  363. for (i = 0; i < ca->n; i++) {
  364. u64 this = PFN(ca->array[i]);
  365. seq_printf(m, " %3d: [%016llx|%s|%03llx]\n",
  366. i, this, bins[DECAY(ca->array[i])], COUNT(ca->array[i]));
  367. }
  368. seq_printf(m, "}\n");
  369. seq_printf(m, "Stats:\nCEs: %llu\nofflined pages: %llu\n",
  370. ca->ces_entered, ca->pfns_poisoned);
  371. seq_printf(m, "Flags: 0x%x\n", ca->flags);
  372. seq_printf(m, "Decay interval: %lld seconds\n", decay_interval);
  373. seq_printf(m, "Decays: %lld\n", ca->decays_done);
  374. seq_printf(m, "Action threshold: %lld\n", action_threshold);
  375. mutex_unlock(&ce_mutex);
  376. return 0;
  377. }
  378. DEFINE_SHOW_ATTRIBUTE(array);
  379. static int __init create_debugfs_nodes(void)
  380. {
  381. struct dentry *d, *pfn, *decay, *count, *array;
  382. d = debugfs_create_dir("cec", ras_debugfs_dir);
  383. if (!d) {
  384. pr_warn("Error creating cec debugfs node!\n");
  385. return -1;
  386. }
  387. decay = debugfs_create_file("decay_interval", S_IRUSR | S_IWUSR, d,
  388. &decay_interval, &decay_interval_ops);
  389. if (!decay) {
  390. pr_warn("Error creating decay_interval debugfs node!\n");
  391. goto err;
  392. }
  393. count = debugfs_create_file("action_threshold", S_IRUSR | S_IWUSR, d,
  394. &action_threshold, &action_threshold_ops);
  395. if (!count) {
  396. pr_warn("Error creating action_threshold debugfs node!\n");
  397. goto err;
  398. }
  399. if (!IS_ENABLED(CONFIG_RAS_CEC_DEBUG))
  400. return 0;
  401. pfn = debugfs_create_file("pfn", S_IRUSR | S_IWUSR, d, &dfs_pfn, &pfn_ops);
  402. if (!pfn) {
  403. pr_warn("Error creating pfn debugfs node!\n");
  404. goto err;
  405. }
  406. array = debugfs_create_file("array", S_IRUSR, d, NULL, &array_fops);
  407. if (!array) {
  408. pr_warn("Error creating array debugfs node!\n");
  409. goto err;
  410. }
  411. return 0;
  412. err:
  413. debugfs_remove_recursive(d);
  414. return 1;
  415. }
  416. static int cec_notifier(struct notifier_block *nb, unsigned long val,
  417. void *data)
  418. {
  419. struct mce *m = (struct mce *)data;
  420. if (!m)
  421. return NOTIFY_DONE;
  422. /* We eat only correctable DRAM errors with usable addresses. */
  423. if (mce_is_memory_error(m) &&
  424. mce_is_correctable(m) &&
  425. mce_usable_address(m)) {
  426. if (!cec_add_elem(m->addr >> PAGE_SHIFT)) {
  427. m->kflags |= MCE_HANDLED_CEC;
  428. return NOTIFY_OK;
  429. }
  430. }
  431. return NOTIFY_DONE;
  432. }
  433. static struct notifier_block cec_nb = {
  434. .notifier_call = cec_notifier,
  435. .priority = MCE_PRIO_CEC,
  436. };
  437. static int __init cec_init(void)
  438. {
  439. if (ce_arr.disabled)
  440. return -ENODEV;
  441. ce_arr.array = (void *)get_zeroed_page(GFP_KERNEL);
  442. if (!ce_arr.array) {
  443. pr_err("Error allocating CE array page!\n");
  444. return -ENOMEM;
  445. }
  446. if (create_debugfs_nodes()) {
  447. free_page((unsigned long)ce_arr.array);
  448. return -ENOMEM;
  449. }
  450. INIT_DELAYED_WORK(&cec_work, cec_work_fn);
  451. schedule_delayed_work(&cec_work, CEC_DECAY_DEFAULT_INTERVAL);
  452. mce_register_decode_chain(&cec_nb);
  453. pr_info("Correctable Errors collector initialized.\n");
  454. return 0;
  455. }
  456. late_initcall(cec_init);
  457. int __init parse_cec_param(char *str)
  458. {
  459. if (!str)
  460. return 0;
  461. if (*str == '=')
  462. str++;
  463. if (!strcmp(str, "cec_disable"))
  464. ce_arr.disabled = 1;
  465. else
  466. return 0;
  467. return 1;
  468. }