aes-spe-modes.S 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625
  1. /* SPDX-License-Identifier: GPL-2.0-or-later */
  2. /*
  3. * AES modes (ECB/CBC/CTR/XTS) for PPC AES implementation
  4. *
  5. * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
  6. */
  7. #include <asm/ppc_asm.h>
  8. #include "aes-spe-regs.h"
  9. #ifdef __BIG_ENDIAN__ /* Macros for big endian builds */
  10. #define LOAD_DATA(reg, off) \
  11. lwz reg,off(rSP); /* load with offset */
  12. #define SAVE_DATA(reg, off) \
  13. stw reg,off(rDP); /* save with offset */
  14. #define NEXT_BLOCK \
  15. addi rSP,rSP,16; /* increment pointers per bloc */ \
  16. addi rDP,rDP,16;
  17. #define LOAD_IV(reg, off) \
  18. lwz reg,off(rIP); /* IV loading with offset */
  19. #define SAVE_IV(reg, off) \
  20. stw reg,off(rIP); /* IV saving with offset */
  21. #define START_IV /* nothing to reset */
  22. #define CBC_DEC 16 /* CBC decrement per block */
  23. #define CTR_DEC 1 /* CTR decrement one byte */
  24. #else /* Macros for little endian */
  25. #define LOAD_DATA(reg, off) \
  26. lwbrx reg,0,rSP; /* load reversed */ \
  27. addi rSP,rSP,4; /* and increment pointer */
  28. #define SAVE_DATA(reg, off) \
  29. stwbrx reg,0,rDP; /* save reversed */ \
  30. addi rDP,rDP,4; /* and increment pointer */
  31. #define NEXT_BLOCK /* nothing todo */
  32. #define LOAD_IV(reg, off) \
  33. lwbrx reg,0,rIP; /* load reversed */ \
  34. addi rIP,rIP,4; /* and increment pointer */
  35. #define SAVE_IV(reg, off) \
  36. stwbrx reg,0,rIP; /* load reversed */ \
  37. addi rIP,rIP,4; /* and increment pointer */
  38. #define START_IV \
  39. subi rIP,rIP,16; /* must reset pointer */
  40. #define CBC_DEC 32 /* 2 blocks because of incs */
  41. #define CTR_DEC 17 /* 1 block because of incs */
  42. #endif
  43. #define SAVE_0_REGS
  44. #define LOAD_0_REGS
  45. #define SAVE_4_REGS \
  46. stw rI0,96(r1); /* save 32 bit registers */ \
  47. stw rI1,100(r1); \
  48. stw rI2,104(r1); \
  49. stw rI3,108(r1);
  50. #define LOAD_4_REGS \
  51. lwz rI0,96(r1); /* restore 32 bit registers */ \
  52. lwz rI1,100(r1); \
  53. lwz rI2,104(r1); \
  54. lwz rI3,108(r1);
  55. #define SAVE_8_REGS \
  56. SAVE_4_REGS \
  57. stw rG0,112(r1); /* save 32 bit registers */ \
  58. stw rG1,116(r1); \
  59. stw rG2,120(r1); \
  60. stw rG3,124(r1);
  61. #define LOAD_8_REGS \
  62. LOAD_4_REGS \
  63. lwz rG0,112(r1); /* restore 32 bit registers */ \
  64. lwz rG1,116(r1); \
  65. lwz rG2,120(r1); \
  66. lwz rG3,124(r1);
  67. #define INITIALIZE_CRYPT(tab,nr32bitregs) \
  68. mflr r0; \
  69. stwu r1,-160(r1); /* create stack frame */ \
  70. lis rT0,tab@h; /* en-/decryption table pointer */ \
  71. stw r0,8(r1); /* save link register */ \
  72. ori rT0,rT0,tab@l; \
  73. evstdw r14,16(r1); \
  74. mr rKS,rKP; \
  75. evstdw r15,24(r1); /* We must save non volatile */ \
  76. evstdw r16,32(r1); /* registers. Take the chance */ \
  77. evstdw r17,40(r1); /* and save the SPE part too */ \
  78. evstdw r18,48(r1); \
  79. evstdw r19,56(r1); \
  80. evstdw r20,64(r1); \
  81. evstdw r21,72(r1); \
  82. evstdw r22,80(r1); \
  83. evstdw r23,88(r1); \
  84. SAVE_##nr32bitregs##_REGS
  85. #define FINALIZE_CRYPT(nr32bitregs) \
  86. lwz r0,8(r1); \
  87. evldw r14,16(r1); /* restore SPE registers */ \
  88. evldw r15,24(r1); \
  89. evldw r16,32(r1); \
  90. evldw r17,40(r1); \
  91. evldw r18,48(r1); \
  92. evldw r19,56(r1); \
  93. evldw r20,64(r1); \
  94. evldw r21,72(r1); \
  95. evldw r22,80(r1); \
  96. evldw r23,88(r1); \
  97. LOAD_##nr32bitregs##_REGS \
  98. mtlr r0; /* restore link register */ \
  99. xor r0,r0,r0; \
  100. stw r0,16(r1); /* delete sensitive data */ \
  101. stw r0,24(r1); /* that we might have pushed */ \
  102. stw r0,32(r1); /* from other context that runs */ \
  103. stw r0,40(r1); /* the same code */ \
  104. stw r0,48(r1); \
  105. stw r0,56(r1); \
  106. stw r0,64(r1); \
  107. stw r0,72(r1); \
  108. stw r0,80(r1); \
  109. stw r0,88(r1); \
  110. addi r1,r1,160; /* cleanup stack frame */
  111. #define ENDIAN_SWAP(t0, t1, s0, s1) \
  112. rotrwi t0,s0,8; /* swap endianness for 2 GPRs */ \
  113. rotrwi t1,s1,8; \
  114. rlwimi t0,s0,8,8,15; \
  115. rlwimi t1,s1,8,8,15; \
  116. rlwimi t0,s0,8,24,31; \
  117. rlwimi t1,s1,8,24,31;
  118. #define GF128_MUL(d0, d1, d2, d3, t0) \
  119. li t0,0x87; /* multiplication in GF128 */ \
  120. cmpwi d3,-1; \
  121. iselgt t0,0,t0; \
  122. rlwimi d3,d2,0,0,0; /* propagate "carry" bits */ \
  123. rotlwi d3,d3,1; \
  124. rlwimi d2,d1,0,0,0; \
  125. rotlwi d2,d2,1; \
  126. rlwimi d1,d0,0,0,0; \
  127. slwi d0,d0,1; /* shift left 128 bit */ \
  128. rotlwi d1,d1,1; \
  129. xor d0,d0,t0;
  130. #define START_KEY(d0, d1, d2, d3) \
  131. lwz rW0,0(rKP); \
  132. mtctr rRR; \
  133. lwz rW1,4(rKP); \
  134. lwz rW2,8(rKP); \
  135. lwz rW3,12(rKP); \
  136. xor rD0,d0,rW0; \
  137. xor rD1,d1,rW1; \
  138. xor rD2,d2,rW2; \
  139. xor rD3,d3,rW3;
  140. /*
  141. * ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc,
  142. * u32 rounds)
  143. *
  144. * called from glue layer to encrypt a single 16 byte block
  145. * round values are AES128 = 4, AES192 = 5, AES256 = 6
  146. *
  147. */
  148. _GLOBAL(ppc_encrypt_aes)
  149. INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0)
  150. LOAD_DATA(rD0, 0)
  151. LOAD_DATA(rD1, 4)
  152. LOAD_DATA(rD2, 8)
  153. LOAD_DATA(rD3, 12)
  154. START_KEY(rD0, rD1, rD2, rD3)
  155. bl ppc_encrypt_block
  156. xor rD0,rD0,rW0
  157. SAVE_DATA(rD0, 0)
  158. xor rD1,rD1,rW1
  159. SAVE_DATA(rD1, 4)
  160. xor rD2,rD2,rW2
  161. SAVE_DATA(rD2, 8)
  162. xor rD3,rD3,rW3
  163. SAVE_DATA(rD3, 12)
  164. FINALIZE_CRYPT(0)
  165. blr
  166. /*
  167. * ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec,
  168. * u32 rounds)
  169. *
  170. * called from glue layer to decrypt a single 16 byte block
  171. * round values are AES128 = 4, AES192 = 5, AES256 = 6
  172. *
  173. */
  174. _GLOBAL(ppc_decrypt_aes)
  175. INITIALIZE_CRYPT(PPC_AES_4K_DECTAB,0)
  176. LOAD_DATA(rD0, 0)
  177. addi rT1,rT0,4096
  178. LOAD_DATA(rD1, 4)
  179. LOAD_DATA(rD2, 8)
  180. LOAD_DATA(rD3, 12)
  181. START_KEY(rD0, rD1, rD2, rD3)
  182. bl ppc_decrypt_block
  183. xor rD0,rD0,rW0
  184. SAVE_DATA(rD0, 0)
  185. xor rD1,rD1,rW1
  186. SAVE_DATA(rD1, 4)
  187. xor rD2,rD2,rW2
  188. SAVE_DATA(rD2, 8)
  189. xor rD3,rD3,rW3
  190. SAVE_DATA(rD3, 12)
  191. FINALIZE_CRYPT(0)
  192. blr
  193. /*
  194. * ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc,
  195. * u32 rounds, u32 bytes);
  196. *
  197. * called from glue layer to encrypt multiple blocks via ECB
  198. * Bytes must be larger or equal 16 and only whole blocks are
  199. * processed. round values are AES128 = 4, AES192 = 5 and
  200. * AES256 = 6
  201. *
  202. */
  203. _GLOBAL(ppc_encrypt_ecb)
  204. INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0)
  205. ppc_encrypt_ecb_loop:
  206. LOAD_DATA(rD0, 0)
  207. mr rKP,rKS
  208. LOAD_DATA(rD1, 4)
  209. subi rLN,rLN,16
  210. LOAD_DATA(rD2, 8)
  211. cmpwi rLN,15
  212. LOAD_DATA(rD3, 12)
  213. START_KEY(rD0, rD1, rD2, rD3)
  214. bl ppc_encrypt_block
  215. xor rD0,rD0,rW0
  216. SAVE_DATA(rD0, 0)
  217. xor rD1,rD1,rW1
  218. SAVE_DATA(rD1, 4)
  219. xor rD2,rD2,rW2
  220. SAVE_DATA(rD2, 8)
  221. xor rD3,rD3,rW3
  222. SAVE_DATA(rD3, 12)
  223. NEXT_BLOCK
  224. bt gt,ppc_encrypt_ecb_loop
  225. FINALIZE_CRYPT(0)
  226. blr
  227. /*
  228. * ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec,
  229. * u32 rounds, u32 bytes);
  230. *
  231. * called from glue layer to decrypt multiple blocks via ECB
  232. * Bytes must be larger or equal 16 and only whole blocks are
  233. * processed. round values are AES128 = 4, AES192 = 5 and
  234. * AES256 = 6
  235. *
  236. */
  237. _GLOBAL(ppc_decrypt_ecb)
  238. INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 0)
  239. addi rT1,rT0,4096
  240. ppc_decrypt_ecb_loop:
  241. LOAD_DATA(rD0, 0)
  242. mr rKP,rKS
  243. LOAD_DATA(rD1, 4)
  244. subi rLN,rLN,16
  245. LOAD_DATA(rD2, 8)
  246. cmpwi rLN,15
  247. LOAD_DATA(rD3, 12)
  248. START_KEY(rD0, rD1, rD2, rD3)
  249. bl ppc_decrypt_block
  250. xor rD0,rD0,rW0
  251. SAVE_DATA(rD0, 0)
  252. xor rD1,rD1,rW1
  253. SAVE_DATA(rD1, 4)
  254. xor rD2,rD2,rW2
  255. SAVE_DATA(rD2, 8)
  256. xor rD3,rD3,rW3
  257. SAVE_DATA(rD3, 12)
  258. NEXT_BLOCK
  259. bt gt,ppc_decrypt_ecb_loop
  260. FINALIZE_CRYPT(0)
  261. blr
  262. /*
  263. * ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc,
  264. * 32 rounds, u32 bytes, u8 *iv);
  265. *
  266. * called from glue layer to encrypt multiple blocks via CBC
  267. * Bytes must be larger or equal 16 and only whole blocks are
  268. * processed. round values are AES128 = 4, AES192 = 5 and
  269. * AES256 = 6
  270. *
  271. */
  272. _GLOBAL(ppc_encrypt_cbc)
  273. INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4)
  274. LOAD_IV(rI0, 0)
  275. LOAD_IV(rI1, 4)
  276. LOAD_IV(rI2, 8)
  277. LOAD_IV(rI3, 12)
  278. ppc_encrypt_cbc_loop:
  279. LOAD_DATA(rD0, 0)
  280. mr rKP,rKS
  281. LOAD_DATA(rD1, 4)
  282. subi rLN,rLN,16
  283. LOAD_DATA(rD2, 8)
  284. cmpwi rLN,15
  285. LOAD_DATA(rD3, 12)
  286. xor rD0,rD0,rI0
  287. xor rD1,rD1,rI1
  288. xor rD2,rD2,rI2
  289. xor rD3,rD3,rI3
  290. START_KEY(rD0, rD1, rD2, rD3)
  291. bl ppc_encrypt_block
  292. xor rI0,rD0,rW0
  293. SAVE_DATA(rI0, 0)
  294. xor rI1,rD1,rW1
  295. SAVE_DATA(rI1, 4)
  296. xor rI2,rD2,rW2
  297. SAVE_DATA(rI2, 8)
  298. xor rI3,rD3,rW3
  299. SAVE_DATA(rI3, 12)
  300. NEXT_BLOCK
  301. bt gt,ppc_encrypt_cbc_loop
  302. START_IV
  303. SAVE_IV(rI0, 0)
  304. SAVE_IV(rI1, 4)
  305. SAVE_IV(rI2, 8)
  306. SAVE_IV(rI3, 12)
  307. FINALIZE_CRYPT(4)
  308. blr
  309. /*
  310. * ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec,
  311. * u32 rounds, u32 bytes, u8 *iv);
  312. *
  313. * called from glue layer to decrypt multiple blocks via CBC
  314. * round values are AES128 = 4, AES192 = 5, AES256 = 6
  315. *
  316. */
  317. _GLOBAL(ppc_decrypt_cbc)
  318. INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 4)
  319. li rT1,15
  320. LOAD_IV(rI0, 0)
  321. andc rLN,rLN,rT1
  322. LOAD_IV(rI1, 4)
  323. subi rLN,rLN,16
  324. LOAD_IV(rI2, 8)
  325. add rSP,rSP,rLN /* reverse processing */
  326. LOAD_IV(rI3, 12)
  327. add rDP,rDP,rLN
  328. LOAD_DATA(rD0, 0)
  329. addi rT1,rT0,4096
  330. LOAD_DATA(rD1, 4)
  331. LOAD_DATA(rD2, 8)
  332. LOAD_DATA(rD3, 12)
  333. START_IV
  334. SAVE_IV(rD0, 0)
  335. SAVE_IV(rD1, 4)
  336. SAVE_IV(rD2, 8)
  337. cmpwi rLN,16
  338. SAVE_IV(rD3, 12)
  339. bt lt,ppc_decrypt_cbc_end
  340. ppc_decrypt_cbc_loop:
  341. mr rKP,rKS
  342. START_KEY(rD0, rD1, rD2, rD3)
  343. bl ppc_decrypt_block
  344. subi rLN,rLN,16
  345. subi rSP,rSP,CBC_DEC
  346. xor rW0,rD0,rW0
  347. LOAD_DATA(rD0, 0)
  348. xor rW1,rD1,rW1
  349. LOAD_DATA(rD1, 4)
  350. xor rW2,rD2,rW2
  351. LOAD_DATA(rD2, 8)
  352. xor rW3,rD3,rW3
  353. LOAD_DATA(rD3, 12)
  354. xor rW0,rW0,rD0
  355. SAVE_DATA(rW0, 0)
  356. xor rW1,rW1,rD1
  357. SAVE_DATA(rW1, 4)
  358. xor rW2,rW2,rD2
  359. SAVE_DATA(rW2, 8)
  360. xor rW3,rW3,rD3
  361. SAVE_DATA(rW3, 12)
  362. cmpwi rLN,15
  363. subi rDP,rDP,CBC_DEC
  364. bt gt,ppc_decrypt_cbc_loop
  365. ppc_decrypt_cbc_end:
  366. mr rKP,rKS
  367. START_KEY(rD0, rD1, rD2, rD3)
  368. bl ppc_decrypt_block
  369. xor rW0,rW0,rD0
  370. xor rW1,rW1,rD1
  371. xor rW2,rW2,rD2
  372. xor rW3,rW3,rD3
  373. xor rW0,rW0,rI0 /* decrypt with initial IV */
  374. SAVE_DATA(rW0, 0)
  375. xor rW1,rW1,rI1
  376. SAVE_DATA(rW1, 4)
  377. xor rW2,rW2,rI2
  378. SAVE_DATA(rW2, 8)
  379. xor rW3,rW3,rI3
  380. SAVE_DATA(rW3, 12)
  381. FINALIZE_CRYPT(4)
  382. blr
  383. /*
  384. * ppc_crypt_ctr(u8 *out, const u8 *in, u32 *key_enc,
  385. * u32 rounds, u32 bytes, u8 *iv);
  386. *
  387. * called from glue layer to encrypt/decrypt multiple blocks
  388. * via CTR. Number of bytes does not need to be a multiple of
  389. * 16. Round values are AES128 = 4, AES192 = 5, AES256 = 6
  390. *
  391. */
  392. _GLOBAL(ppc_crypt_ctr)
  393. INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4)
  394. LOAD_IV(rI0, 0)
  395. LOAD_IV(rI1, 4)
  396. LOAD_IV(rI2, 8)
  397. cmpwi rLN,16
  398. LOAD_IV(rI3, 12)
  399. START_IV
  400. bt lt,ppc_crypt_ctr_partial
  401. ppc_crypt_ctr_loop:
  402. mr rKP,rKS
  403. START_KEY(rI0, rI1, rI2, rI3)
  404. bl ppc_encrypt_block
  405. xor rW0,rD0,rW0
  406. xor rW1,rD1,rW1
  407. xor rW2,rD2,rW2
  408. xor rW3,rD3,rW3
  409. LOAD_DATA(rD0, 0)
  410. subi rLN,rLN,16
  411. LOAD_DATA(rD1, 4)
  412. LOAD_DATA(rD2, 8)
  413. LOAD_DATA(rD3, 12)
  414. xor rD0,rD0,rW0
  415. SAVE_DATA(rD0, 0)
  416. xor rD1,rD1,rW1
  417. SAVE_DATA(rD1, 4)
  418. xor rD2,rD2,rW2
  419. SAVE_DATA(rD2, 8)
  420. xor rD3,rD3,rW3
  421. SAVE_DATA(rD3, 12)
  422. addic rI3,rI3,1 /* increase counter */
  423. addze rI2,rI2
  424. addze rI1,rI1
  425. addze rI0,rI0
  426. NEXT_BLOCK
  427. cmpwi rLN,15
  428. bt gt,ppc_crypt_ctr_loop
  429. ppc_crypt_ctr_partial:
  430. cmpwi rLN,0
  431. bt eq,ppc_crypt_ctr_end
  432. mr rKP,rKS
  433. START_KEY(rI0, rI1, rI2, rI3)
  434. bl ppc_encrypt_block
  435. xor rW0,rD0,rW0
  436. SAVE_IV(rW0, 0)
  437. xor rW1,rD1,rW1
  438. SAVE_IV(rW1, 4)
  439. xor rW2,rD2,rW2
  440. SAVE_IV(rW2, 8)
  441. xor rW3,rD3,rW3
  442. SAVE_IV(rW3, 12)
  443. mtctr rLN
  444. subi rIP,rIP,CTR_DEC
  445. subi rSP,rSP,1
  446. subi rDP,rDP,1
  447. ppc_crypt_ctr_xorbyte:
  448. lbzu rW4,1(rIP) /* bytewise xor for partial block */
  449. lbzu rW5,1(rSP)
  450. xor rW4,rW4,rW5
  451. stbu rW4,1(rDP)
  452. bdnz ppc_crypt_ctr_xorbyte
  453. subf rIP,rLN,rIP
  454. addi rIP,rIP,1
  455. addic rI3,rI3,1
  456. addze rI2,rI2
  457. addze rI1,rI1
  458. addze rI0,rI0
  459. ppc_crypt_ctr_end:
  460. SAVE_IV(rI0, 0)
  461. SAVE_IV(rI1, 4)
  462. SAVE_IV(rI2, 8)
  463. SAVE_IV(rI3, 12)
  464. FINALIZE_CRYPT(4)
  465. blr
  466. /*
  467. * ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc,
  468. * u32 rounds, u32 bytes, u8 *iv, u32 *key_twk);
  469. *
  470. * called from glue layer to encrypt multiple blocks via XTS
  471. * If key_twk is given, the initial IV encryption will be
  472. * processed too. Round values are AES128 = 4, AES192 = 5,
  473. * AES256 = 6
  474. *
  475. */
  476. _GLOBAL(ppc_encrypt_xts)
  477. INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 8)
  478. LOAD_IV(rI0, 0)
  479. LOAD_IV(rI1, 4)
  480. LOAD_IV(rI2, 8)
  481. cmpwi rKT,0
  482. LOAD_IV(rI3, 12)
  483. bt eq,ppc_encrypt_xts_notweak
  484. mr rKP,rKT
  485. START_KEY(rI0, rI1, rI2, rI3)
  486. bl ppc_encrypt_block
  487. xor rI0,rD0,rW0
  488. xor rI1,rD1,rW1
  489. xor rI2,rD2,rW2
  490. xor rI3,rD3,rW3
  491. ppc_encrypt_xts_notweak:
  492. ENDIAN_SWAP(rG0, rG1, rI0, rI1)
  493. ENDIAN_SWAP(rG2, rG3, rI2, rI3)
  494. ppc_encrypt_xts_loop:
  495. LOAD_DATA(rD0, 0)
  496. mr rKP,rKS
  497. LOAD_DATA(rD1, 4)
  498. subi rLN,rLN,16
  499. LOAD_DATA(rD2, 8)
  500. LOAD_DATA(rD3, 12)
  501. xor rD0,rD0,rI0
  502. xor rD1,rD1,rI1
  503. xor rD2,rD2,rI2
  504. xor rD3,rD3,rI3
  505. START_KEY(rD0, rD1, rD2, rD3)
  506. bl ppc_encrypt_block
  507. xor rD0,rD0,rW0
  508. xor rD1,rD1,rW1
  509. xor rD2,rD2,rW2
  510. xor rD3,rD3,rW3
  511. xor rD0,rD0,rI0
  512. SAVE_DATA(rD0, 0)
  513. xor rD1,rD1,rI1
  514. SAVE_DATA(rD1, 4)
  515. xor rD2,rD2,rI2
  516. SAVE_DATA(rD2, 8)
  517. xor rD3,rD3,rI3
  518. SAVE_DATA(rD3, 12)
  519. GF128_MUL(rG0, rG1, rG2, rG3, rW0)
  520. ENDIAN_SWAP(rI0, rI1, rG0, rG1)
  521. ENDIAN_SWAP(rI2, rI3, rG2, rG3)
  522. cmpwi rLN,0
  523. NEXT_BLOCK
  524. bt gt,ppc_encrypt_xts_loop
  525. START_IV
  526. SAVE_IV(rI0, 0)
  527. SAVE_IV(rI1, 4)
  528. SAVE_IV(rI2, 8)
  529. SAVE_IV(rI3, 12)
  530. FINALIZE_CRYPT(8)
  531. blr
  532. /*
  533. * ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec,
  534. * u32 rounds, u32 blocks, u8 *iv, u32 *key_twk);
  535. *
  536. * called from glue layer to decrypt multiple blocks via XTS
  537. * If key_twk is given, the initial IV encryption will be
  538. * processed too. Round values are AES128 = 4, AES192 = 5,
  539. * AES256 = 6
  540. *
  541. */
  542. _GLOBAL(ppc_decrypt_xts)
  543. INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 8)
  544. LOAD_IV(rI0, 0)
  545. addi rT1,rT0,4096
  546. LOAD_IV(rI1, 4)
  547. LOAD_IV(rI2, 8)
  548. cmpwi rKT,0
  549. LOAD_IV(rI3, 12)
  550. bt eq,ppc_decrypt_xts_notweak
  551. subi rT0,rT0,4096
  552. mr rKP,rKT
  553. START_KEY(rI0, rI1, rI2, rI3)
  554. bl ppc_encrypt_block
  555. xor rI0,rD0,rW0
  556. xor rI1,rD1,rW1
  557. xor rI2,rD2,rW2
  558. xor rI3,rD3,rW3
  559. addi rT0,rT0,4096
  560. ppc_decrypt_xts_notweak:
  561. ENDIAN_SWAP(rG0, rG1, rI0, rI1)
  562. ENDIAN_SWAP(rG2, rG3, rI2, rI3)
  563. ppc_decrypt_xts_loop:
  564. LOAD_DATA(rD0, 0)
  565. mr rKP,rKS
  566. LOAD_DATA(rD1, 4)
  567. subi rLN,rLN,16
  568. LOAD_DATA(rD2, 8)
  569. LOAD_DATA(rD3, 12)
  570. xor rD0,rD0,rI0
  571. xor rD1,rD1,rI1
  572. xor rD2,rD2,rI2
  573. xor rD3,rD3,rI3
  574. START_KEY(rD0, rD1, rD2, rD3)
  575. bl ppc_decrypt_block
  576. xor rD0,rD0,rW0
  577. xor rD1,rD1,rW1
  578. xor rD2,rD2,rW2
  579. xor rD3,rD3,rW3
  580. xor rD0,rD0,rI0
  581. SAVE_DATA(rD0, 0)
  582. xor rD1,rD1,rI1
  583. SAVE_DATA(rD1, 4)
  584. xor rD2,rD2,rI2
  585. SAVE_DATA(rD2, 8)
  586. xor rD3,rD3,rI3
  587. SAVE_DATA(rD3, 12)
  588. GF128_MUL(rG0, rG1, rG2, rG3, rW0)
  589. ENDIAN_SWAP(rI0, rI1, rG0, rG1)
  590. ENDIAN_SWAP(rI2, rI3, rG2, rG3)
  591. cmpwi rLN,0
  592. NEXT_BLOCK
  593. bt gt,ppc_decrypt_xts_loop
  594. START_IV
  595. SAVE_IV(rI0, 0)
  596. SAVE_IV(rI1, 4)
  597. SAVE_IV(rI2, 8)
  598. SAVE_IV(rI3, 12)
  599. FINALIZE_CRYPT(8)
  600. blr