algos.c 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /* -*- linux-c -*- ------------------------------------------------------- *
  3. *
  4. * Copyright 2002 H. Peter Anvin - All Rights Reserved
  5. *
  6. * ----------------------------------------------------------------------- */
  7. /*
  8. * raid6/algos.c
  9. *
  10. * Algorithm list and algorithm selection for RAID-6
  11. */
  12. #include <linux/raid/pq.h>
  13. #ifndef __KERNEL__
  14. #include <sys/mman.h>
  15. #include <stdio.h>
  16. #else
  17. #include <linux/module.h>
  18. #include <linux/gfp.h>
  19. #if !RAID6_USE_EMPTY_ZERO_PAGE
  20. /* In .bss so it's zeroed */
  21. const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256)));
  22. EXPORT_SYMBOL(raid6_empty_zero_page);
  23. #endif
  24. #endif
  25. struct raid6_calls raid6_call;
  26. EXPORT_SYMBOL_GPL(raid6_call);
  27. const struct raid6_calls * const raid6_algos[] = {
  28. #if defined(__i386__) && !defined(__arch_um__)
  29. #ifdef CONFIG_AS_AVX512
  30. &raid6_avx512x2,
  31. &raid6_avx512x1,
  32. #endif
  33. &raid6_avx2x2,
  34. &raid6_avx2x1,
  35. &raid6_sse2x2,
  36. &raid6_sse2x1,
  37. &raid6_sse1x2,
  38. &raid6_sse1x1,
  39. &raid6_mmxx2,
  40. &raid6_mmxx1,
  41. #endif
  42. #if defined(__x86_64__) && !defined(__arch_um__)
  43. #ifdef CONFIG_AS_AVX512
  44. &raid6_avx512x4,
  45. &raid6_avx512x2,
  46. &raid6_avx512x1,
  47. #endif
  48. &raid6_avx2x4,
  49. &raid6_avx2x2,
  50. &raid6_avx2x1,
  51. &raid6_sse2x4,
  52. &raid6_sse2x2,
  53. &raid6_sse2x1,
  54. #endif
  55. #ifdef CONFIG_ALTIVEC
  56. &raid6_vpermxor8,
  57. &raid6_vpermxor4,
  58. &raid6_vpermxor2,
  59. &raid6_vpermxor1,
  60. &raid6_altivec8,
  61. &raid6_altivec4,
  62. &raid6_altivec2,
  63. &raid6_altivec1,
  64. #endif
  65. #if defined(CONFIG_S390)
  66. &raid6_s390vx8,
  67. #endif
  68. #ifdef CONFIG_KERNEL_MODE_NEON
  69. &raid6_neonx8,
  70. &raid6_neonx4,
  71. &raid6_neonx2,
  72. &raid6_neonx1,
  73. #endif
  74. #if defined(__ia64__)
  75. &raid6_intx32,
  76. &raid6_intx16,
  77. #endif
  78. &raid6_intx8,
  79. &raid6_intx4,
  80. &raid6_intx2,
  81. &raid6_intx1,
  82. NULL
  83. };
  84. void (*raid6_2data_recov)(int, size_t, int, int, void **);
  85. EXPORT_SYMBOL_GPL(raid6_2data_recov);
  86. void (*raid6_datap_recov)(int, size_t, int, void **);
  87. EXPORT_SYMBOL_GPL(raid6_datap_recov);
  88. const struct raid6_recov_calls *const raid6_recov_algos[] = {
  89. #ifdef CONFIG_X86
  90. #ifdef CONFIG_AS_AVX512
  91. &raid6_recov_avx512,
  92. #endif
  93. &raid6_recov_avx2,
  94. &raid6_recov_ssse3,
  95. #endif
  96. #ifdef CONFIG_S390
  97. &raid6_recov_s390xc,
  98. #endif
  99. #if defined(CONFIG_KERNEL_MODE_NEON)
  100. &raid6_recov_neon,
  101. #endif
  102. &raid6_recov_intx1,
  103. NULL
  104. };
  105. #ifdef __KERNEL__
  106. #define RAID6_TIME_JIFFIES_LG2 4
  107. #else
  108. /* Need more time to be stable in userspace */
  109. #define RAID6_TIME_JIFFIES_LG2 9
  110. #define time_before(x, y) ((x) < (y))
  111. #endif
  112. #define RAID6_TEST_DISKS 8
  113. #define RAID6_TEST_DISKS_ORDER 3
  114. static inline const struct raid6_recov_calls *raid6_choose_recov(void)
  115. {
  116. const struct raid6_recov_calls *const *algo;
  117. const struct raid6_recov_calls *best;
  118. for (best = NULL, algo = raid6_recov_algos; *algo; algo++)
  119. if (!best || (*algo)->priority > best->priority)
  120. if (!(*algo)->valid || (*algo)->valid())
  121. best = *algo;
  122. if (best) {
  123. raid6_2data_recov = best->data2;
  124. raid6_datap_recov = best->datap;
  125. pr_info("raid6: using %s recovery algorithm\n", best->name);
  126. } else
  127. pr_err("raid6: Yikes! No recovery algorithm found!\n");
  128. return best;
  129. }
  130. static inline const struct raid6_calls *raid6_choose_gen(
  131. void *(*const dptrs)[RAID6_TEST_DISKS], const int disks)
  132. {
  133. unsigned long perf, bestgenperf, bestxorperf, j0, j1;
  134. int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */
  135. const struct raid6_calls *const *algo;
  136. const struct raid6_calls *best;
  137. for (bestgenperf = 0, bestxorperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) {
  138. if (!best || (*algo)->prefer >= best->prefer) {
  139. if ((*algo)->valid && !(*algo)->valid())
  140. continue;
  141. if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
  142. best = *algo;
  143. break;
  144. }
  145. perf = 0;
  146. preempt_disable();
  147. j0 = jiffies;
  148. while ((j1 = jiffies) == j0)
  149. cpu_relax();
  150. while (time_before(jiffies,
  151. j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
  152. (*algo)->gen_syndrome(disks, PAGE_SIZE, *dptrs);
  153. perf++;
  154. }
  155. preempt_enable();
  156. if (perf > bestgenperf) {
  157. bestgenperf = perf;
  158. best = *algo;
  159. }
  160. pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name,
  161. (perf * HZ * (disks-2)) >>
  162. (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
  163. if (!(*algo)->xor_syndrome)
  164. continue;
  165. perf = 0;
  166. preempt_disable();
  167. j0 = jiffies;
  168. while ((j1 = jiffies) == j0)
  169. cpu_relax();
  170. while (time_before(jiffies,
  171. j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
  172. (*algo)->xor_syndrome(disks, start, stop,
  173. PAGE_SIZE, *dptrs);
  174. perf++;
  175. }
  176. preempt_enable();
  177. if (best == *algo)
  178. bestxorperf = perf;
  179. pr_info("raid6: %-8s xor() %5ld MB/s\n", (*algo)->name,
  180. (perf * HZ * (disks-2)) >>
  181. (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
  182. }
  183. }
  184. if (best) {
  185. if (IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
  186. pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
  187. best->name,
  188. (bestgenperf * HZ * (disks-2)) >>
  189. (20 - PAGE_SHIFT+RAID6_TIME_JIFFIES_LG2));
  190. if (best->xor_syndrome)
  191. pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",
  192. (bestxorperf * HZ * (disks-2)) >>
  193. (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
  194. } else
  195. pr_info("raid6: skip pq benchmark and using algorithm %s\n",
  196. best->name);
  197. raid6_call = *best;
  198. } else
  199. pr_err("raid6: Yikes! No algorithm found!\n");
  200. return best;
  201. }
  202. /* Try to pick the best algorithm */
  203. /* This code uses the gfmul table as convenient data set to abuse */
  204. int __init raid6_select_algo(void)
  205. {
  206. const int disks = RAID6_TEST_DISKS;
  207. const struct raid6_calls *gen_best;
  208. const struct raid6_recov_calls *rec_best;
  209. char *disk_ptr, *p;
  210. void *dptrs[RAID6_TEST_DISKS];
  211. int i, cycle;
  212. /* prepare the buffer and fill it circularly with gfmul table */
  213. disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER);
  214. if (!disk_ptr) {
  215. pr_err("raid6: Yikes! No memory available.\n");
  216. return -ENOMEM;
  217. }
  218. p = disk_ptr;
  219. for (i = 0; i < disks; i++)
  220. dptrs[i] = p + PAGE_SIZE * i;
  221. cycle = ((disks - 2) * PAGE_SIZE) / 65536;
  222. for (i = 0; i < cycle; i++) {
  223. memcpy(p, raid6_gfmul, 65536);
  224. p += 65536;
  225. }
  226. if ((disks - 2) * PAGE_SIZE % 65536)
  227. memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536);
  228. /* select raid gen_syndrome function */
  229. gen_best = raid6_choose_gen(&dptrs, disks);
  230. /* select raid recover functions */
  231. rec_best = raid6_choose_recov();
  232. free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER);
  233. return gen_best && rec_best ? 0 : -EINVAL;
  234. }
  235. static void raid6_exit(void)
  236. {
  237. do { } while (0);
  238. }
  239. subsys_initcall(raid6_select_algo);
  240. module_exit(raid6_exit);
  241. MODULE_LICENSE("GPL");
  242. MODULE_DESCRIPTION("RAID6 Q-syndrome calculations");