xor.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710
  1. /* SPDX-License-Identifier: GPL-2.0-or-later */
  2. /*
  3. * include/asm-generic/xor.h
  4. *
  5. * Generic optimized RAID-5 checksumming functions.
  6. */
  7. #include <linux/prefetch.h>
  8. static void
  9. xor_8regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
  10. {
  11. long lines = bytes / (sizeof (long)) / 8;
  12. do {
  13. p1[0] ^= p2[0];
  14. p1[1] ^= p2[1];
  15. p1[2] ^= p2[2];
  16. p1[3] ^= p2[3];
  17. p1[4] ^= p2[4];
  18. p1[5] ^= p2[5];
  19. p1[6] ^= p2[6];
  20. p1[7] ^= p2[7];
  21. p1 += 8;
  22. p2 += 8;
  23. } while (--lines > 0);
  24. }
  25. static void
  26. xor_8regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
  27. unsigned long *p3)
  28. {
  29. long lines = bytes / (sizeof (long)) / 8;
  30. do {
  31. p1[0] ^= p2[0] ^ p3[0];
  32. p1[1] ^= p2[1] ^ p3[1];
  33. p1[2] ^= p2[2] ^ p3[2];
  34. p1[3] ^= p2[3] ^ p3[3];
  35. p1[4] ^= p2[4] ^ p3[4];
  36. p1[5] ^= p2[5] ^ p3[5];
  37. p1[6] ^= p2[6] ^ p3[6];
  38. p1[7] ^= p2[7] ^ p3[7];
  39. p1 += 8;
  40. p2 += 8;
  41. p3 += 8;
  42. } while (--lines > 0);
  43. }
  44. static void
  45. xor_8regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
  46. unsigned long *p3, unsigned long *p4)
  47. {
  48. long lines = bytes / (sizeof (long)) / 8;
  49. do {
  50. p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
  51. p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
  52. p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
  53. p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
  54. p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
  55. p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
  56. p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
  57. p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
  58. p1 += 8;
  59. p2 += 8;
  60. p3 += 8;
  61. p4 += 8;
  62. } while (--lines > 0);
  63. }
  64. static void
  65. xor_8regs_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
  66. unsigned long *p3, unsigned long *p4, unsigned long *p5)
  67. {
  68. long lines = bytes / (sizeof (long)) / 8;
  69. do {
  70. p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
  71. p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
  72. p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
  73. p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
  74. p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
  75. p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
  76. p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
  77. p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
  78. p1 += 8;
  79. p2 += 8;
  80. p3 += 8;
  81. p4 += 8;
  82. p5 += 8;
  83. } while (--lines > 0);
  84. }
  85. static void
  86. xor_32regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
  87. {
  88. long lines = bytes / (sizeof (long)) / 8;
  89. do {
  90. register long d0, d1, d2, d3, d4, d5, d6, d7;
  91. d0 = p1[0]; /* Pull the stuff into registers */
  92. d1 = p1[1]; /* ... in bursts, if possible. */
  93. d2 = p1[2];
  94. d3 = p1[3];
  95. d4 = p1[4];
  96. d5 = p1[5];
  97. d6 = p1[6];
  98. d7 = p1[7];
  99. d0 ^= p2[0];
  100. d1 ^= p2[1];
  101. d2 ^= p2[2];
  102. d3 ^= p2[3];
  103. d4 ^= p2[4];
  104. d5 ^= p2[5];
  105. d6 ^= p2[6];
  106. d7 ^= p2[7];
  107. p1[0] = d0; /* Store the result (in bursts) */
  108. p1[1] = d1;
  109. p1[2] = d2;
  110. p1[3] = d3;
  111. p1[4] = d4;
  112. p1[5] = d5;
  113. p1[6] = d6;
  114. p1[7] = d7;
  115. p1 += 8;
  116. p2 += 8;
  117. } while (--lines > 0);
  118. }
  119. static void
  120. xor_32regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
  121. unsigned long *p3)
  122. {
  123. long lines = bytes / (sizeof (long)) / 8;
  124. do {
  125. register long d0, d1, d2, d3, d4, d5, d6, d7;
  126. d0 = p1[0]; /* Pull the stuff into registers */
  127. d1 = p1[1]; /* ... in bursts, if possible. */
  128. d2 = p1[2];
  129. d3 = p1[3];
  130. d4 = p1[4];
  131. d5 = p1[5];
  132. d6 = p1[6];
  133. d7 = p1[7];
  134. d0 ^= p2[0];
  135. d1 ^= p2[1];
  136. d2 ^= p2[2];
  137. d3 ^= p2[3];
  138. d4 ^= p2[4];
  139. d5 ^= p2[5];
  140. d6 ^= p2[6];
  141. d7 ^= p2[7];
  142. d0 ^= p3[0];
  143. d1 ^= p3[1];
  144. d2 ^= p3[2];
  145. d3 ^= p3[3];
  146. d4 ^= p3[4];
  147. d5 ^= p3[5];
  148. d6 ^= p3[6];
  149. d7 ^= p3[7];
  150. p1[0] = d0; /* Store the result (in bursts) */
  151. p1[1] = d1;
  152. p1[2] = d2;
  153. p1[3] = d3;
  154. p1[4] = d4;
  155. p1[5] = d5;
  156. p1[6] = d6;
  157. p1[7] = d7;
  158. p1 += 8;
  159. p2 += 8;
  160. p3 += 8;
  161. } while (--lines > 0);
  162. }
  163. static void
  164. xor_32regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
  165. unsigned long *p3, unsigned long *p4)
  166. {
  167. long lines = bytes / (sizeof (long)) / 8;
  168. do {
  169. register long d0, d1, d2, d3, d4, d5, d6, d7;
  170. d0 = p1[0]; /* Pull the stuff into registers */
  171. d1 = p1[1]; /* ... in bursts, if possible. */
  172. d2 = p1[2];
  173. d3 = p1[3];
  174. d4 = p1[4];
  175. d5 = p1[5];
  176. d6 = p1[6];
  177. d7 = p1[7];
  178. d0 ^= p2[0];
  179. d1 ^= p2[1];
  180. d2 ^= p2[2];
  181. d3 ^= p2[3];
  182. d4 ^= p2[4];
  183. d5 ^= p2[5];
  184. d6 ^= p2[6];
  185. d7 ^= p2[7];
  186. d0 ^= p3[0];
  187. d1 ^= p3[1];
  188. d2 ^= p3[2];
  189. d3 ^= p3[3];
  190. d4 ^= p3[4];
  191. d5 ^= p3[5];
  192. d6 ^= p3[6];
  193. d7 ^= p3[7];
  194. d0 ^= p4[0];
  195. d1 ^= p4[1];
  196. d2 ^= p4[2];
  197. d3 ^= p4[3];
  198. d4 ^= p4[4];
  199. d5 ^= p4[5];
  200. d6 ^= p4[6];
  201. d7 ^= p4[7];
  202. p1[0] = d0; /* Store the result (in bursts) */
  203. p1[1] = d1;
  204. p1[2] = d2;
  205. p1[3] = d3;
  206. p1[4] = d4;
  207. p1[5] = d5;
  208. p1[6] = d6;
  209. p1[7] = d7;
  210. p1 += 8;
  211. p2 += 8;
  212. p3 += 8;
  213. p4 += 8;
  214. } while (--lines > 0);
  215. }
  216. static void
  217. xor_32regs_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
  218. unsigned long *p3, unsigned long *p4, unsigned long *p5)
  219. {
  220. long lines = bytes / (sizeof (long)) / 8;
  221. do {
  222. register long d0, d1, d2, d3, d4, d5, d6, d7;
  223. d0 = p1[0]; /* Pull the stuff into registers */
  224. d1 = p1[1]; /* ... in bursts, if possible. */
  225. d2 = p1[2];
  226. d3 = p1[3];
  227. d4 = p1[4];
  228. d5 = p1[5];
  229. d6 = p1[6];
  230. d7 = p1[7];
  231. d0 ^= p2[0];
  232. d1 ^= p2[1];
  233. d2 ^= p2[2];
  234. d3 ^= p2[3];
  235. d4 ^= p2[4];
  236. d5 ^= p2[5];
  237. d6 ^= p2[6];
  238. d7 ^= p2[7];
  239. d0 ^= p3[0];
  240. d1 ^= p3[1];
  241. d2 ^= p3[2];
  242. d3 ^= p3[3];
  243. d4 ^= p3[4];
  244. d5 ^= p3[5];
  245. d6 ^= p3[6];
  246. d7 ^= p3[7];
  247. d0 ^= p4[0];
  248. d1 ^= p4[1];
  249. d2 ^= p4[2];
  250. d3 ^= p4[3];
  251. d4 ^= p4[4];
  252. d5 ^= p4[5];
  253. d6 ^= p4[6];
  254. d7 ^= p4[7];
  255. d0 ^= p5[0];
  256. d1 ^= p5[1];
  257. d2 ^= p5[2];
  258. d3 ^= p5[3];
  259. d4 ^= p5[4];
  260. d5 ^= p5[5];
  261. d6 ^= p5[6];
  262. d7 ^= p5[7];
  263. p1[0] = d0; /* Store the result (in bursts) */
  264. p1[1] = d1;
  265. p1[2] = d2;
  266. p1[3] = d3;
  267. p1[4] = d4;
  268. p1[5] = d5;
  269. p1[6] = d6;
  270. p1[7] = d7;
  271. p1 += 8;
  272. p2 += 8;
  273. p3 += 8;
  274. p4 += 8;
  275. p5 += 8;
  276. } while (--lines > 0);
  277. }
  278. static void
  279. xor_8regs_p_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
  280. {
  281. long lines = bytes / (sizeof (long)) / 8 - 1;
  282. prefetchw(p1);
  283. prefetch(p2);
  284. do {
  285. prefetchw(p1+8);
  286. prefetch(p2+8);
  287. once_more:
  288. p1[0] ^= p2[0];
  289. p1[1] ^= p2[1];
  290. p1[2] ^= p2[2];
  291. p1[3] ^= p2[3];
  292. p1[4] ^= p2[4];
  293. p1[5] ^= p2[5];
  294. p1[6] ^= p2[6];
  295. p1[7] ^= p2[7];
  296. p1 += 8;
  297. p2 += 8;
  298. } while (--lines > 0);
  299. if (lines == 0)
  300. goto once_more;
  301. }
  302. static void
  303. xor_8regs_p_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
  304. unsigned long *p3)
  305. {
  306. long lines = bytes / (sizeof (long)) / 8 - 1;
  307. prefetchw(p1);
  308. prefetch(p2);
  309. prefetch(p3);
  310. do {
  311. prefetchw(p1+8);
  312. prefetch(p2+8);
  313. prefetch(p3+8);
  314. once_more:
  315. p1[0] ^= p2[0] ^ p3[0];
  316. p1[1] ^= p2[1] ^ p3[1];
  317. p1[2] ^= p2[2] ^ p3[2];
  318. p1[3] ^= p2[3] ^ p3[3];
  319. p1[4] ^= p2[4] ^ p3[4];
  320. p1[5] ^= p2[5] ^ p3[5];
  321. p1[6] ^= p2[6] ^ p3[6];
  322. p1[7] ^= p2[7] ^ p3[7];
  323. p1 += 8;
  324. p2 += 8;
  325. p3 += 8;
  326. } while (--lines > 0);
  327. if (lines == 0)
  328. goto once_more;
  329. }
  330. static void
  331. xor_8regs_p_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
  332. unsigned long *p3, unsigned long *p4)
  333. {
  334. long lines = bytes / (sizeof (long)) / 8 - 1;
  335. prefetchw(p1);
  336. prefetch(p2);
  337. prefetch(p3);
  338. prefetch(p4);
  339. do {
  340. prefetchw(p1+8);
  341. prefetch(p2+8);
  342. prefetch(p3+8);
  343. prefetch(p4+8);
  344. once_more:
  345. p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
  346. p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
  347. p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
  348. p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
  349. p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
  350. p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
  351. p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
  352. p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
  353. p1 += 8;
  354. p2 += 8;
  355. p3 += 8;
  356. p4 += 8;
  357. } while (--lines > 0);
  358. if (lines == 0)
  359. goto once_more;
  360. }
  361. static void
  362. xor_8regs_p_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
  363. unsigned long *p3, unsigned long *p4, unsigned long *p5)
  364. {
  365. long lines = bytes / (sizeof (long)) / 8 - 1;
  366. prefetchw(p1);
  367. prefetch(p2);
  368. prefetch(p3);
  369. prefetch(p4);
  370. prefetch(p5);
  371. do {
  372. prefetchw(p1+8);
  373. prefetch(p2+8);
  374. prefetch(p3+8);
  375. prefetch(p4+8);
  376. prefetch(p5+8);
  377. once_more:
  378. p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
  379. p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
  380. p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
  381. p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
  382. p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
  383. p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
  384. p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
  385. p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
  386. p1 += 8;
  387. p2 += 8;
  388. p3 += 8;
  389. p4 += 8;
  390. p5 += 8;
  391. } while (--lines > 0);
  392. if (lines == 0)
  393. goto once_more;
  394. }
  395. static void
  396. xor_32regs_p_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
  397. {
  398. long lines = bytes / (sizeof (long)) / 8 - 1;
  399. prefetchw(p1);
  400. prefetch(p2);
  401. do {
  402. register long d0, d1, d2, d3, d4, d5, d6, d7;
  403. prefetchw(p1+8);
  404. prefetch(p2+8);
  405. once_more:
  406. d0 = p1[0]; /* Pull the stuff into registers */
  407. d1 = p1[1]; /* ... in bursts, if possible. */
  408. d2 = p1[2];
  409. d3 = p1[3];
  410. d4 = p1[4];
  411. d5 = p1[5];
  412. d6 = p1[6];
  413. d7 = p1[7];
  414. d0 ^= p2[0];
  415. d1 ^= p2[1];
  416. d2 ^= p2[2];
  417. d3 ^= p2[3];
  418. d4 ^= p2[4];
  419. d5 ^= p2[5];
  420. d6 ^= p2[6];
  421. d7 ^= p2[7];
  422. p1[0] = d0; /* Store the result (in bursts) */
  423. p1[1] = d1;
  424. p1[2] = d2;
  425. p1[3] = d3;
  426. p1[4] = d4;
  427. p1[5] = d5;
  428. p1[6] = d6;
  429. p1[7] = d7;
  430. p1 += 8;
  431. p2 += 8;
  432. } while (--lines > 0);
  433. if (lines == 0)
  434. goto once_more;
  435. }
  436. static void
  437. xor_32regs_p_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
  438. unsigned long *p3)
  439. {
  440. long lines = bytes / (sizeof (long)) / 8 - 1;
  441. prefetchw(p1);
  442. prefetch(p2);
  443. prefetch(p3);
  444. do {
  445. register long d0, d1, d2, d3, d4, d5, d6, d7;
  446. prefetchw(p1+8);
  447. prefetch(p2+8);
  448. prefetch(p3+8);
  449. once_more:
  450. d0 = p1[0]; /* Pull the stuff into registers */
  451. d1 = p1[1]; /* ... in bursts, if possible. */
  452. d2 = p1[2];
  453. d3 = p1[3];
  454. d4 = p1[4];
  455. d5 = p1[5];
  456. d6 = p1[6];
  457. d7 = p1[7];
  458. d0 ^= p2[0];
  459. d1 ^= p2[1];
  460. d2 ^= p2[2];
  461. d3 ^= p2[3];
  462. d4 ^= p2[4];
  463. d5 ^= p2[5];
  464. d6 ^= p2[6];
  465. d7 ^= p2[7];
  466. d0 ^= p3[0];
  467. d1 ^= p3[1];
  468. d2 ^= p3[2];
  469. d3 ^= p3[3];
  470. d4 ^= p3[4];
  471. d5 ^= p3[5];
  472. d6 ^= p3[6];
  473. d7 ^= p3[7];
  474. p1[0] = d0; /* Store the result (in bursts) */
  475. p1[1] = d1;
  476. p1[2] = d2;
  477. p1[3] = d3;
  478. p1[4] = d4;
  479. p1[5] = d5;
  480. p1[6] = d6;
  481. p1[7] = d7;
  482. p1 += 8;
  483. p2 += 8;
  484. p3 += 8;
  485. } while (--lines > 0);
  486. if (lines == 0)
  487. goto once_more;
  488. }
  489. static void
  490. xor_32regs_p_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
  491. unsigned long *p3, unsigned long *p4)
  492. {
  493. long lines = bytes / (sizeof (long)) / 8 - 1;
  494. prefetchw(p1);
  495. prefetch(p2);
  496. prefetch(p3);
  497. prefetch(p4);
  498. do {
  499. register long d0, d1, d2, d3, d4, d5, d6, d7;
  500. prefetchw(p1+8);
  501. prefetch(p2+8);
  502. prefetch(p3+8);
  503. prefetch(p4+8);
  504. once_more:
  505. d0 = p1[0]; /* Pull the stuff into registers */
  506. d1 = p1[1]; /* ... in bursts, if possible. */
  507. d2 = p1[2];
  508. d3 = p1[3];
  509. d4 = p1[4];
  510. d5 = p1[5];
  511. d6 = p1[6];
  512. d7 = p1[7];
  513. d0 ^= p2[0];
  514. d1 ^= p2[1];
  515. d2 ^= p2[2];
  516. d3 ^= p2[3];
  517. d4 ^= p2[4];
  518. d5 ^= p2[5];
  519. d6 ^= p2[6];
  520. d7 ^= p2[7];
  521. d0 ^= p3[0];
  522. d1 ^= p3[1];
  523. d2 ^= p3[2];
  524. d3 ^= p3[3];
  525. d4 ^= p3[4];
  526. d5 ^= p3[5];
  527. d6 ^= p3[6];
  528. d7 ^= p3[7];
  529. d0 ^= p4[0];
  530. d1 ^= p4[1];
  531. d2 ^= p4[2];
  532. d3 ^= p4[3];
  533. d4 ^= p4[4];
  534. d5 ^= p4[5];
  535. d6 ^= p4[6];
  536. d7 ^= p4[7];
  537. p1[0] = d0; /* Store the result (in bursts) */
  538. p1[1] = d1;
  539. p1[2] = d2;
  540. p1[3] = d3;
  541. p1[4] = d4;
  542. p1[5] = d5;
  543. p1[6] = d6;
  544. p1[7] = d7;
  545. p1 += 8;
  546. p2 += 8;
  547. p3 += 8;
  548. p4 += 8;
  549. } while (--lines > 0);
  550. if (lines == 0)
  551. goto once_more;
  552. }
  553. static void
  554. xor_32regs_p_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
  555. unsigned long *p3, unsigned long *p4, unsigned long *p5)
  556. {
  557. long lines = bytes / (sizeof (long)) / 8 - 1;
  558. prefetchw(p1);
  559. prefetch(p2);
  560. prefetch(p3);
  561. prefetch(p4);
  562. prefetch(p5);
  563. do {
  564. register long d0, d1, d2, d3, d4, d5, d6, d7;
  565. prefetchw(p1+8);
  566. prefetch(p2+8);
  567. prefetch(p3+8);
  568. prefetch(p4+8);
  569. prefetch(p5+8);
  570. once_more:
  571. d0 = p1[0]; /* Pull the stuff into registers */
  572. d1 = p1[1]; /* ... in bursts, if possible. */
  573. d2 = p1[2];
  574. d3 = p1[3];
  575. d4 = p1[4];
  576. d5 = p1[5];
  577. d6 = p1[6];
  578. d7 = p1[7];
  579. d0 ^= p2[0];
  580. d1 ^= p2[1];
  581. d2 ^= p2[2];
  582. d3 ^= p2[3];
  583. d4 ^= p2[4];
  584. d5 ^= p2[5];
  585. d6 ^= p2[6];
  586. d7 ^= p2[7];
  587. d0 ^= p3[0];
  588. d1 ^= p3[1];
  589. d2 ^= p3[2];
  590. d3 ^= p3[3];
  591. d4 ^= p3[4];
  592. d5 ^= p3[5];
  593. d6 ^= p3[6];
  594. d7 ^= p3[7];
  595. d0 ^= p4[0];
  596. d1 ^= p4[1];
  597. d2 ^= p4[2];
  598. d3 ^= p4[3];
  599. d4 ^= p4[4];
  600. d5 ^= p4[5];
  601. d6 ^= p4[6];
  602. d7 ^= p4[7];
  603. d0 ^= p5[0];
  604. d1 ^= p5[1];
  605. d2 ^= p5[2];
  606. d3 ^= p5[3];
  607. d4 ^= p5[4];
  608. d5 ^= p5[5];
  609. d6 ^= p5[6];
  610. d7 ^= p5[7];
  611. p1[0] = d0; /* Store the result (in bursts) */
  612. p1[1] = d1;
  613. p1[2] = d2;
  614. p1[3] = d3;
  615. p1[4] = d4;
  616. p1[5] = d5;
  617. p1[6] = d6;
  618. p1[7] = d7;
  619. p1 += 8;
  620. p2 += 8;
  621. p3 += 8;
  622. p4 += 8;
  623. p5 += 8;
  624. } while (--lines > 0);
  625. if (lines == 0)
  626. goto once_more;
  627. }
  628. static struct xor_block_template xor_block_8regs = {
  629. .name = "8regs",
  630. .do_2 = xor_8regs_2,
  631. .do_3 = xor_8regs_3,
  632. .do_4 = xor_8regs_4,
  633. .do_5 = xor_8regs_5,
  634. };
  635. static struct xor_block_template xor_block_32regs = {
  636. .name = "32regs",
  637. .do_2 = xor_32regs_2,
  638. .do_3 = xor_32regs_3,
  639. .do_4 = xor_32regs_4,
  640. .do_5 = xor_32regs_5,
  641. };
  642. static struct xor_block_template xor_block_8regs_p __maybe_unused = {
  643. .name = "8regs_prefetch",
  644. .do_2 = xor_8regs_p_2,
  645. .do_3 = xor_8regs_p_3,
  646. .do_4 = xor_8regs_p_4,
  647. .do_5 = xor_8regs_p_5,
  648. };
  649. static struct xor_block_template xor_block_32regs_p __maybe_unused = {
  650. .name = "32regs_prefetch",
  651. .do_2 = xor_32regs_p_2,
  652. .do_3 = xor_32regs_p_3,
  653. .do_4 = xor_32regs_p_4,
  654. .do_5 = xor_32regs_p_5,
  655. };
  656. #define XOR_TRY_TEMPLATES \
  657. do { \
  658. xor_speed(&xor_block_8regs); \
  659. xor_speed(&xor_block_8regs_p); \
  660. xor_speed(&xor_block_32regs); \
  661. xor_speed(&xor_block_32regs_p); \
  662. } while (0)