crc32-vpmsum_core.S 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751
  1. /* SPDX-License-Identifier: GPL-2.0-or-later */
  2. /*
  3. * Core of the accelerated CRC algorithm.
  4. * In your file, define the constants and CRC_FUNCTION_NAME
  5. * Then include this file.
  6. *
  7. * Calculate the checksum of data that is 16 byte aligned and a multiple of
  8. * 16 bytes.
  9. *
  10. * The first step is to reduce it to 1024 bits. We do this in 8 parallel
  11. * chunks in order to mask the latency of the vpmsum instructions. If we
  12. * have more than 32 kB of data to checksum we repeat this step multiple
  13. * times, passing in the previous 1024 bits.
  14. *
  15. * The next step is to reduce the 1024 bits to 64 bits. This step adds
  16. * 32 bits of 0s to the end - this matches what a CRC does. We just
  17. * calculate constants that land the data in this 32 bits.
  18. *
  19. * We then use fixed point Barrett reduction to compute a mod n over GF(2)
  20. * for n = CRC using POWER8 instructions. We use x = 32.
  21. *
  22. * https://en.wikipedia.org/wiki/Barrett_reduction
  23. *
  24. * Copyright (C) 2015 Anton Blanchard <anton@au.ibm.com>, IBM
  25. */
  26. #include <asm/ppc_asm.h>
  27. #include <asm/ppc-opcode.h>
  28. #define MAX_SIZE 32768
  29. .text
  30. #if defined(__BIG_ENDIAN__) && defined(REFLECT)
  31. #define BYTESWAP_DATA
  32. #elif defined(__LITTLE_ENDIAN__) && !defined(REFLECT)
  33. #define BYTESWAP_DATA
  34. #else
  35. #undef BYTESWAP_DATA
  36. #endif
  37. #define off16 r25
  38. #define off32 r26
  39. #define off48 r27
  40. #define off64 r28
  41. #define off80 r29
  42. #define off96 r30
  43. #define off112 r31
  44. #define const1 v24
  45. #define const2 v25
  46. #define byteswap v26
  47. #define mask_32bit v27
  48. #define mask_64bit v28
  49. #define zeroes v29
  50. #ifdef BYTESWAP_DATA
  51. #define VPERM(A, B, C, D) vperm A, B, C, D
  52. #else
  53. #define VPERM(A, B, C, D)
  54. #endif
  55. /* unsigned int CRC_FUNCTION_NAME(unsigned int crc, void *p, unsigned long len) */
  56. FUNC_START(CRC_FUNCTION_NAME)
  57. std r31,-8(r1)
  58. std r30,-16(r1)
  59. std r29,-24(r1)
  60. std r28,-32(r1)
  61. std r27,-40(r1)
  62. std r26,-48(r1)
  63. std r25,-56(r1)
  64. li off16,16
  65. li off32,32
  66. li off48,48
  67. li off64,64
  68. li off80,80
  69. li off96,96
  70. li off112,112
  71. li r0,0
  72. /* Enough room for saving 10 non volatile VMX registers */
  73. subi r6,r1,56+10*16
  74. subi r7,r1,56+2*16
  75. stvx v20,0,r6
  76. stvx v21,off16,r6
  77. stvx v22,off32,r6
  78. stvx v23,off48,r6
  79. stvx v24,off64,r6
  80. stvx v25,off80,r6
  81. stvx v26,off96,r6
  82. stvx v27,off112,r6
  83. stvx v28,0,r7
  84. stvx v29,off16,r7
  85. mr r10,r3
  86. vxor zeroes,zeroes,zeroes
  87. vspltisw v0,-1
  88. vsldoi mask_32bit,zeroes,v0,4
  89. vsldoi mask_64bit,zeroes,v0,8
  90. /* Get the initial value into v8 */
  91. vxor v8,v8,v8
  92. MTVRD(v8, R3)
  93. #ifdef REFLECT
  94. vsldoi v8,zeroes,v8,8 /* shift into bottom 32 bits */
  95. #else
  96. vsldoi v8,v8,zeroes,4 /* shift into top 32 bits */
  97. #endif
  98. #ifdef BYTESWAP_DATA
  99. addis r3,r2,.byteswap_constant@toc@ha
  100. addi r3,r3,.byteswap_constant@toc@l
  101. lvx byteswap,0,r3
  102. addi r3,r3,16
  103. #endif
  104. cmpdi r5,256
  105. blt .Lshort
  106. rldicr r6,r5,0,56
  107. /* Checksum in blocks of MAX_SIZE */
  108. 1: lis r7,MAX_SIZE@h
  109. ori r7,r7,MAX_SIZE@l
  110. mr r9,r7
  111. cmpd r6,r7
  112. bgt 2f
  113. mr r7,r6
  114. 2: subf r6,r7,r6
  115. /* our main loop does 128 bytes at a time */
  116. srdi r7,r7,7
  117. /*
  118. * Work out the offset into the constants table to start at. Each
  119. * constant is 16 bytes, and it is used against 128 bytes of input
  120. * data - 128 / 16 = 8
  121. */
  122. sldi r8,r7,4
  123. srdi r9,r9,3
  124. subf r8,r8,r9
  125. /* We reduce our final 128 bytes in a separate step */
  126. addi r7,r7,-1
  127. mtctr r7
  128. addis r3,r2,.constants@toc@ha
  129. addi r3,r3,.constants@toc@l
  130. /* Find the start of our constants */
  131. add r3,r3,r8
  132. /* zero v0-v7 which will contain our checksums */
  133. vxor v0,v0,v0
  134. vxor v1,v1,v1
  135. vxor v2,v2,v2
  136. vxor v3,v3,v3
  137. vxor v4,v4,v4
  138. vxor v5,v5,v5
  139. vxor v6,v6,v6
  140. vxor v7,v7,v7
  141. lvx const1,0,r3
  142. /*
  143. * If we are looping back to consume more data we use the values
  144. * already in v16-v23.
  145. */
  146. cmpdi r0,1
  147. beq 2f
  148. /* First warm up pass */
  149. lvx v16,0,r4
  150. lvx v17,off16,r4
  151. VPERM(v16,v16,v16,byteswap)
  152. VPERM(v17,v17,v17,byteswap)
  153. lvx v18,off32,r4
  154. lvx v19,off48,r4
  155. VPERM(v18,v18,v18,byteswap)
  156. VPERM(v19,v19,v19,byteswap)
  157. lvx v20,off64,r4
  158. lvx v21,off80,r4
  159. VPERM(v20,v20,v20,byteswap)
  160. VPERM(v21,v21,v21,byteswap)
  161. lvx v22,off96,r4
  162. lvx v23,off112,r4
  163. VPERM(v22,v22,v22,byteswap)
  164. VPERM(v23,v23,v23,byteswap)
  165. addi r4,r4,8*16
  166. /* xor in initial value */
  167. vxor v16,v16,v8
  168. 2: bdz .Lfirst_warm_up_done
  169. addi r3,r3,16
  170. lvx const2,0,r3
  171. /* Second warm up pass */
  172. VPMSUMD(v8,v16,const1)
  173. lvx v16,0,r4
  174. VPERM(v16,v16,v16,byteswap)
  175. ori r2,r2,0
  176. VPMSUMD(v9,v17,const1)
  177. lvx v17,off16,r4
  178. VPERM(v17,v17,v17,byteswap)
  179. ori r2,r2,0
  180. VPMSUMD(v10,v18,const1)
  181. lvx v18,off32,r4
  182. VPERM(v18,v18,v18,byteswap)
  183. ori r2,r2,0
  184. VPMSUMD(v11,v19,const1)
  185. lvx v19,off48,r4
  186. VPERM(v19,v19,v19,byteswap)
  187. ori r2,r2,0
  188. VPMSUMD(v12,v20,const1)
  189. lvx v20,off64,r4
  190. VPERM(v20,v20,v20,byteswap)
  191. ori r2,r2,0
  192. VPMSUMD(v13,v21,const1)
  193. lvx v21,off80,r4
  194. VPERM(v21,v21,v21,byteswap)
  195. ori r2,r2,0
  196. VPMSUMD(v14,v22,const1)
  197. lvx v22,off96,r4
  198. VPERM(v22,v22,v22,byteswap)
  199. ori r2,r2,0
  200. VPMSUMD(v15,v23,const1)
  201. lvx v23,off112,r4
  202. VPERM(v23,v23,v23,byteswap)
  203. addi r4,r4,8*16
  204. bdz .Lfirst_cool_down
  205. /*
  206. * main loop. We modulo schedule it such that it takes three iterations
  207. * to complete - first iteration load, second iteration vpmsum, third
  208. * iteration xor.
  209. */
  210. .balign 16
  211. 4: lvx const1,0,r3
  212. addi r3,r3,16
  213. ori r2,r2,0
  214. vxor v0,v0,v8
  215. VPMSUMD(v8,v16,const2)
  216. lvx v16,0,r4
  217. VPERM(v16,v16,v16,byteswap)
  218. ori r2,r2,0
  219. vxor v1,v1,v9
  220. VPMSUMD(v9,v17,const2)
  221. lvx v17,off16,r4
  222. VPERM(v17,v17,v17,byteswap)
  223. ori r2,r2,0
  224. vxor v2,v2,v10
  225. VPMSUMD(v10,v18,const2)
  226. lvx v18,off32,r4
  227. VPERM(v18,v18,v18,byteswap)
  228. ori r2,r2,0
  229. vxor v3,v3,v11
  230. VPMSUMD(v11,v19,const2)
  231. lvx v19,off48,r4
  232. VPERM(v19,v19,v19,byteswap)
  233. lvx const2,0,r3
  234. ori r2,r2,0
  235. vxor v4,v4,v12
  236. VPMSUMD(v12,v20,const1)
  237. lvx v20,off64,r4
  238. VPERM(v20,v20,v20,byteswap)
  239. ori r2,r2,0
  240. vxor v5,v5,v13
  241. VPMSUMD(v13,v21,const1)
  242. lvx v21,off80,r4
  243. VPERM(v21,v21,v21,byteswap)
  244. ori r2,r2,0
  245. vxor v6,v6,v14
  246. VPMSUMD(v14,v22,const1)
  247. lvx v22,off96,r4
  248. VPERM(v22,v22,v22,byteswap)
  249. ori r2,r2,0
  250. vxor v7,v7,v15
  251. VPMSUMD(v15,v23,const1)
  252. lvx v23,off112,r4
  253. VPERM(v23,v23,v23,byteswap)
  254. addi r4,r4,8*16
  255. bdnz 4b
  256. .Lfirst_cool_down:
  257. /* First cool down pass */
  258. lvx const1,0,r3
  259. addi r3,r3,16
  260. vxor v0,v0,v8
  261. VPMSUMD(v8,v16,const1)
  262. ori r2,r2,0
  263. vxor v1,v1,v9
  264. VPMSUMD(v9,v17,const1)
  265. ori r2,r2,0
  266. vxor v2,v2,v10
  267. VPMSUMD(v10,v18,const1)
  268. ori r2,r2,0
  269. vxor v3,v3,v11
  270. VPMSUMD(v11,v19,const1)
  271. ori r2,r2,0
  272. vxor v4,v4,v12
  273. VPMSUMD(v12,v20,const1)
  274. ori r2,r2,0
  275. vxor v5,v5,v13
  276. VPMSUMD(v13,v21,const1)
  277. ori r2,r2,0
  278. vxor v6,v6,v14
  279. VPMSUMD(v14,v22,const1)
  280. ori r2,r2,0
  281. vxor v7,v7,v15
  282. VPMSUMD(v15,v23,const1)
  283. ori r2,r2,0
  284. .Lsecond_cool_down:
  285. /* Second cool down pass */
  286. vxor v0,v0,v8
  287. vxor v1,v1,v9
  288. vxor v2,v2,v10
  289. vxor v3,v3,v11
  290. vxor v4,v4,v12
  291. vxor v5,v5,v13
  292. vxor v6,v6,v14
  293. vxor v7,v7,v15
  294. #ifdef REFLECT
  295. /*
  296. * vpmsumd produces a 96 bit result in the least significant bits
  297. * of the register. Since we are bit reflected we have to shift it
  298. * left 32 bits so it occupies the least significant bits in the
  299. * bit reflected domain.
  300. */
  301. vsldoi v0,v0,zeroes,4
  302. vsldoi v1,v1,zeroes,4
  303. vsldoi v2,v2,zeroes,4
  304. vsldoi v3,v3,zeroes,4
  305. vsldoi v4,v4,zeroes,4
  306. vsldoi v5,v5,zeroes,4
  307. vsldoi v6,v6,zeroes,4
  308. vsldoi v7,v7,zeroes,4
  309. #endif
  310. /* xor with last 1024 bits */
  311. lvx v8,0,r4
  312. lvx v9,off16,r4
  313. VPERM(v8,v8,v8,byteswap)
  314. VPERM(v9,v9,v9,byteswap)
  315. lvx v10,off32,r4
  316. lvx v11,off48,r4
  317. VPERM(v10,v10,v10,byteswap)
  318. VPERM(v11,v11,v11,byteswap)
  319. lvx v12,off64,r4
  320. lvx v13,off80,r4
  321. VPERM(v12,v12,v12,byteswap)
  322. VPERM(v13,v13,v13,byteswap)
  323. lvx v14,off96,r4
  324. lvx v15,off112,r4
  325. VPERM(v14,v14,v14,byteswap)
  326. VPERM(v15,v15,v15,byteswap)
  327. addi r4,r4,8*16
  328. vxor v16,v0,v8
  329. vxor v17,v1,v9
  330. vxor v18,v2,v10
  331. vxor v19,v3,v11
  332. vxor v20,v4,v12
  333. vxor v21,v5,v13
  334. vxor v22,v6,v14
  335. vxor v23,v7,v15
  336. li r0,1
  337. cmpdi r6,0
  338. addi r6,r6,128
  339. bne 1b
  340. /* Work out how many bytes we have left */
  341. andi. r5,r5,127
  342. /* Calculate where in the constant table we need to start */
  343. subfic r6,r5,128
  344. add r3,r3,r6
  345. /* How many 16 byte chunks are in the tail */
  346. srdi r7,r5,4
  347. mtctr r7
  348. /*
  349. * Reduce the previously calculated 1024 bits to 64 bits, shifting
  350. * 32 bits to include the trailing 32 bits of zeros
  351. */
  352. lvx v0,0,r3
  353. lvx v1,off16,r3
  354. lvx v2,off32,r3
  355. lvx v3,off48,r3
  356. lvx v4,off64,r3
  357. lvx v5,off80,r3
  358. lvx v6,off96,r3
  359. lvx v7,off112,r3
  360. addi r3,r3,8*16
  361. VPMSUMW(v0,v16,v0)
  362. VPMSUMW(v1,v17,v1)
  363. VPMSUMW(v2,v18,v2)
  364. VPMSUMW(v3,v19,v3)
  365. VPMSUMW(v4,v20,v4)
  366. VPMSUMW(v5,v21,v5)
  367. VPMSUMW(v6,v22,v6)
  368. VPMSUMW(v7,v23,v7)
  369. /* Now reduce the tail (0 - 112 bytes) */
  370. cmpdi r7,0
  371. beq 1f
  372. lvx v16,0,r4
  373. lvx v17,0,r3
  374. VPERM(v16,v16,v16,byteswap)
  375. VPMSUMW(v16,v16,v17)
  376. vxor v0,v0,v16
  377. bdz 1f
  378. lvx v16,off16,r4
  379. lvx v17,off16,r3
  380. VPERM(v16,v16,v16,byteswap)
  381. VPMSUMW(v16,v16,v17)
  382. vxor v0,v0,v16
  383. bdz 1f
  384. lvx v16,off32,r4
  385. lvx v17,off32,r3
  386. VPERM(v16,v16,v16,byteswap)
  387. VPMSUMW(v16,v16,v17)
  388. vxor v0,v0,v16
  389. bdz 1f
  390. lvx v16,off48,r4
  391. lvx v17,off48,r3
  392. VPERM(v16,v16,v16,byteswap)
  393. VPMSUMW(v16,v16,v17)
  394. vxor v0,v0,v16
  395. bdz 1f
  396. lvx v16,off64,r4
  397. lvx v17,off64,r3
  398. VPERM(v16,v16,v16,byteswap)
  399. VPMSUMW(v16,v16,v17)
  400. vxor v0,v0,v16
  401. bdz 1f
  402. lvx v16,off80,r4
  403. lvx v17,off80,r3
  404. VPERM(v16,v16,v16,byteswap)
  405. VPMSUMW(v16,v16,v17)
  406. vxor v0,v0,v16
  407. bdz 1f
  408. lvx v16,off96,r4
  409. lvx v17,off96,r3
  410. VPERM(v16,v16,v16,byteswap)
  411. VPMSUMW(v16,v16,v17)
  412. vxor v0,v0,v16
  413. /* Now xor all the parallel chunks together */
  414. 1: vxor v0,v0,v1
  415. vxor v2,v2,v3
  416. vxor v4,v4,v5
  417. vxor v6,v6,v7
  418. vxor v0,v0,v2
  419. vxor v4,v4,v6
  420. vxor v0,v0,v4
  421. .Lbarrett_reduction:
  422. /* Barrett constants */
  423. addis r3,r2,.barrett_constants@toc@ha
  424. addi r3,r3,.barrett_constants@toc@l
  425. lvx const1,0,r3
  426. lvx const2,off16,r3
  427. vsldoi v1,v0,v0,8
  428. vxor v0,v0,v1 /* xor two 64 bit results together */
  429. #ifdef REFLECT
  430. /* shift left one bit */
  431. vspltisb v1,1
  432. vsl v0,v0,v1
  433. #endif
  434. vand v0,v0,mask_64bit
  435. #ifndef REFLECT
  436. /*
  437. * Now for the Barrett reduction algorithm. The idea is to calculate q,
  438. * the multiple of our polynomial that we need to subtract. By
  439. * doing the computation 2x bits higher (ie 64 bits) and shifting the
  440. * result back down 2x bits, we round down to the nearest multiple.
  441. */
  442. VPMSUMD(v1,v0,const1) /* ma */
  443. vsldoi v1,zeroes,v1,8 /* q = floor(ma/(2^64)) */
  444. VPMSUMD(v1,v1,const2) /* qn */
  445. vxor v0,v0,v1 /* a - qn, subtraction is xor in GF(2) */
  446. /*
  447. * Get the result into r3. We need to shift it left 8 bytes:
  448. * V0 [ 0 1 2 X ]
  449. * V0 [ 0 X 2 3 ]
  450. */
  451. vsldoi v0,v0,zeroes,8 /* shift result into top 64 bits */
  452. #else
  453. /*
  454. * The reflected version of Barrett reduction. Instead of bit
  455. * reflecting our data (which is expensive to do), we bit reflect our
  456. * constants and our algorithm, which means the intermediate data in
  457. * our vector registers goes from 0-63 instead of 63-0. We can reflect
  458. * the algorithm because we don't carry in mod 2 arithmetic.
  459. */
  460. vand v1,v0,mask_32bit /* bottom 32 bits of a */
  461. VPMSUMD(v1,v1,const1) /* ma */
  462. vand v1,v1,mask_32bit /* bottom 32bits of ma */
  463. VPMSUMD(v1,v1,const2) /* qn */
  464. vxor v0,v0,v1 /* a - qn, subtraction is xor in GF(2) */
  465. /*
  466. * Since we are bit reflected, the result (ie the low 32 bits) is in
  467. * the high 32 bits. We just need to shift it left 4 bytes
  468. * V0 [ 0 1 X 3 ]
  469. * V0 [ 0 X 2 3 ]
  470. */
  471. vsldoi v0,v0,zeroes,4 /* shift result into top 64 bits of */
  472. #endif
  473. /* Get it into r3 */
  474. MFVRD(R3, v0)
  475. .Lout:
  476. subi r6,r1,56+10*16
  477. subi r7,r1,56+2*16
  478. lvx v20,0,r6
  479. lvx v21,off16,r6
  480. lvx v22,off32,r6
  481. lvx v23,off48,r6
  482. lvx v24,off64,r6
  483. lvx v25,off80,r6
  484. lvx v26,off96,r6
  485. lvx v27,off112,r6
  486. lvx v28,0,r7
  487. lvx v29,off16,r7
  488. ld r31,-8(r1)
  489. ld r30,-16(r1)
  490. ld r29,-24(r1)
  491. ld r28,-32(r1)
  492. ld r27,-40(r1)
  493. ld r26,-48(r1)
  494. ld r25,-56(r1)
  495. blr
  496. .Lfirst_warm_up_done:
  497. lvx const1,0,r3
  498. addi r3,r3,16
  499. VPMSUMD(v8,v16,const1)
  500. VPMSUMD(v9,v17,const1)
  501. VPMSUMD(v10,v18,const1)
  502. VPMSUMD(v11,v19,const1)
  503. VPMSUMD(v12,v20,const1)
  504. VPMSUMD(v13,v21,const1)
  505. VPMSUMD(v14,v22,const1)
  506. VPMSUMD(v15,v23,const1)
  507. b .Lsecond_cool_down
  508. .Lshort:
  509. cmpdi r5,0
  510. beq .Lzero
  511. addis r3,r2,.short_constants@toc@ha
  512. addi r3,r3,.short_constants@toc@l
  513. /* Calculate where in the constant table we need to start */
  514. subfic r6,r5,256
  515. add r3,r3,r6
  516. /* How many 16 byte chunks? */
  517. srdi r7,r5,4
  518. mtctr r7
  519. vxor v19,v19,v19
  520. vxor v20,v20,v20
  521. lvx v0,0,r4
  522. lvx v16,0,r3
  523. VPERM(v0,v0,v16,byteswap)
  524. vxor v0,v0,v8 /* xor in initial value */
  525. VPMSUMW(v0,v0,v16)
  526. bdz .Lv0
  527. lvx v1,off16,r4
  528. lvx v17,off16,r3
  529. VPERM(v1,v1,v17,byteswap)
  530. VPMSUMW(v1,v1,v17)
  531. bdz .Lv1
  532. lvx v2,off32,r4
  533. lvx v16,off32,r3
  534. VPERM(v2,v2,v16,byteswap)
  535. VPMSUMW(v2,v2,v16)
  536. bdz .Lv2
  537. lvx v3,off48,r4
  538. lvx v17,off48,r3
  539. VPERM(v3,v3,v17,byteswap)
  540. VPMSUMW(v3,v3,v17)
  541. bdz .Lv3
  542. lvx v4,off64,r4
  543. lvx v16,off64,r3
  544. VPERM(v4,v4,v16,byteswap)
  545. VPMSUMW(v4,v4,v16)
  546. bdz .Lv4
  547. lvx v5,off80,r4
  548. lvx v17,off80,r3
  549. VPERM(v5,v5,v17,byteswap)
  550. VPMSUMW(v5,v5,v17)
  551. bdz .Lv5
  552. lvx v6,off96,r4
  553. lvx v16,off96,r3
  554. VPERM(v6,v6,v16,byteswap)
  555. VPMSUMW(v6,v6,v16)
  556. bdz .Lv6
  557. lvx v7,off112,r4
  558. lvx v17,off112,r3
  559. VPERM(v7,v7,v17,byteswap)
  560. VPMSUMW(v7,v7,v17)
  561. bdz .Lv7
  562. addi r3,r3,128
  563. addi r4,r4,128
  564. lvx v8,0,r4
  565. lvx v16,0,r3
  566. VPERM(v8,v8,v16,byteswap)
  567. VPMSUMW(v8,v8,v16)
  568. bdz .Lv8
  569. lvx v9,off16,r4
  570. lvx v17,off16,r3
  571. VPERM(v9,v9,v17,byteswap)
  572. VPMSUMW(v9,v9,v17)
  573. bdz .Lv9
  574. lvx v10,off32,r4
  575. lvx v16,off32,r3
  576. VPERM(v10,v10,v16,byteswap)
  577. VPMSUMW(v10,v10,v16)
  578. bdz .Lv10
  579. lvx v11,off48,r4
  580. lvx v17,off48,r3
  581. VPERM(v11,v11,v17,byteswap)
  582. VPMSUMW(v11,v11,v17)
  583. bdz .Lv11
  584. lvx v12,off64,r4
  585. lvx v16,off64,r3
  586. VPERM(v12,v12,v16,byteswap)
  587. VPMSUMW(v12,v12,v16)
  588. bdz .Lv12
  589. lvx v13,off80,r4
  590. lvx v17,off80,r3
  591. VPERM(v13,v13,v17,byteswap)
  592. VPMSUMW(v13,v13,v17)
  593. bdz .Lv13
  594. lvx v14,off96,r4
  595. lvx v16,off96,r3
  596. VPERM(v14,v14,v16,byteswap)
  597. VPMSUMW(v14,v14,v16)
  598. bdz .Lv14
  599. lvx v15,off112,r4
  600. lvx v17,off112,r3
  601. VPERM(v15,v15,v17,byteswap)
  602. VPMSUMW(v15,v15,v17)
  603. .Lv15: vxor v19,v19,v15
  604. .Lv14: vxor v20,v20,v14
  605. .Lv13: vxor v19,v19,v13
  606. .Lv12: vxor v20,v20,v12
  607. .Lv11: vxor v19,v19,v11
  608. .Lv10: vxor v20,v20,v10
  609. .Lv9: vxor v19,v19,v9
  610. .Lv8: vxor v20,v20,v8
  611. .Lv7: vxor v19,v19,v7
  612. .Lv6: vxor v20,v20,v6
  613. .Lv5: vxor v19,v19,v5
  614. .Lv4: vxor v20,v20,v4
  615. .Lv3: vxor v19,v19,v3
  616. .Lv2: vxor v20,v20,v2
  617. .Lv1: vxor v19,v19,v1
  618. .Lv0: vxor v20,v20,v0
  619. vxor v0,v19,v20
  620. b .Lbarrett_reduction
  621. .Lzero:
  622. mr r3,r10
  623. b .Lout
  624. FUNC_END(CRC_FUNCTION_NAME)