padlock-aes.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Cryptographic API.
  4. *
  5. * Support for VIA PadLock hardware crypto engine.
  6. *
  7. * Copyright (c) 2004 Michal Ludvig <michal@logix.cz>
  8. *
  9. */
  10. #include <crypto/algapi.h>
  11. #include <crypto/aes.h>
  12. #include <crypto/internal/skcipher.h>
  13. #include <crypto/padlock.h>
  14. #include <linux/module.h>
  15. #include <linux/init.h>
  16. #include <linux/types.h>
  17. #include <linux/errno.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/kernel.h>
  20. #include <linux/mm.h>
  21. #include <linux/percpu.h>
  22. #include <linux/smp.h>
  23. #include <linux/slab.h>
  24. #include <asm/cpu_device_id.h>
  25. #include <asm/byteorder.h>
  26. #include <asm/processor.h>
  27. #include <asm/fpu/api.h>
  28. /*
  29. * Number of data blocks actually fetched for each xcrypt insn.
  30. * Processors with prefetch errata will fetch extra blocks.
  31. */
  32. static unsigned int ecb_fetch_blocks = 2;
  33. #define MAX_ECB_FETCH_BLOCKS (8)
  34. #define ecb_fetch_bytes (ecb_fetch_blocks * AES_BLOCK_SIZE)
  35. static unsigned int cbc_fetch_blocks = 1;
  36. #define MAX_CBC_FETCH_BLOCKS (4)
  37. #define cbc_fetch_bytes (cbc_fetch_blocks * AES_BLOCK_SIZE)
  38. /* Control word. */
  39. struct cword {
  40. unsigned int __attribute__ ((__packed__))
  41. rounds:4,
  42. algo:3,
  43. keygen:1,
  44. interm:1,
  45. encdec:1,
  46. ksize:2;
  47. } __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
  48. /* Whenever making any changes to the following
  49. * structure *make sure* you keep E, d_data
  50. * and cword aligned on 16 Bytes boundaries and
  51. * the Hardware can access 16 * 16 bytes of E and d_data
  52. * (only the first 15 * 16 bytes matter but the HW reads
  53. * more).
  54. */
  55. struct aes_ctx {
  56. u32 E[AES_MAX_KEYLENGTH_U32]
  57. __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
  58. u32 d_data[AES_MAX_KEYLENGTH_U32]
  59. __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
  60. struct {
  61. struct cword encrypt;
  62. struct cword decrypt;
  63. } cword;
  64. u32 *D;
  65. };
  66. static DEFINE_PER_CPU(struct cword *, paes_last_cword);
  67. /* Tells whether the ACE is capable to generate
  68. the extended key for a given key_len. */
  69. static inline int
  70. aes_hw_extkey_available(uint8_t key_len)
  71. {
  72. /* TODO: We should check the actual CPU model/stepping
  73. as it's possible that the capability will be
  74. added in the next CPU revisions. */
  75. if (key_len == 16)
  76. return 1;
  77. return 0;
  78. }
  79. static inline struct aes_ctx *aes_ctx_common(void *ctx)
  80. {
  81. unsigned long addr = (unsigned long)ctx;
  82. unsigned long align = PADLOCK_ALIGNMENT;
  83. if (align <= crypto_tfm_ctx_alignment())
  84. align = 1;
  85. return (struct aes_ctx *)ALIGN(addr, align);
  86. }
  87. static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm)
  88. {
  89. return aes_ctx_common(crypto_tfm_ctx(tfm));
  90. }
  91. static inline struct aes_ctx *skcipher_aes_ctx(struct crypto_skcipher *tfm)
  92. {
  93. return aes_ctx_common(crypto_skcipher_ctx(tfm));
  94. }
  95. static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
  96. unsigned int key_len)
  97. {
  98. struct aes_ctx *ctx = aes_ctx(tfm);
  99. const __le32 *key = (const __le32 *)in_key;
  100. struct crypto_aes_ctx gen_aes;
  101. int cpu;
  102. if (key_len % 8)
  103. return -EINVAL;
  104. /*
  105. * If the hardware is capable of generating the extended key
  106. * itself we must supply the plain key for both encryption
  107. * and decryption.
  108. */
  109. ctx->D = ctx->E;
  110. ctx->E[0] = le32_to_cpu(key[0]);
  111. ctx->E[1] = le32_to_cpu(key[1]);
  112. ctx->E[2] = le32_to_cpu(key[2]);
  113. ctx->E[3] = le32_to_cpu(key[3]);
  114. /* Prepare control words. */
  115. memset(&ctx->cword, 0, sizeof(ctx->cword));
  116. ctx->cword.decrypt.encdec = 1;
  117. ctx->cword.encrypt.rounds = 10 + (key_len - 16) / 4;
  118. ctx->cword.decrypt.rounds = ctx->cword.encrypt.rounds;
  119. ctx->cword.encrypt.ksize = (key_len - 16) / 8;
  120. ctx->cword.decrypt.ksize = ctx->cword.encrypt.ksize;
  121. /* Don't generate extended keys if the hardware can do it. */
  122. if (aes_hw_extkey_available(key_len))
  123. goto ok;
  124. ctx->D = ctx->d_data;
  125. ctx->cword.encrypt.keygen = 1;
  126. ctx->cword.decrypt.keygen = 1;
  127. if (aes_expandkey(&gen_aes, in_key, key_len))
  128. return -EINVAL;
  129. memcpy(ctx->E, gen_aes.key_enc, AES_MAX_KEYLENGTH);
  130. memcpy(ctx->D, gen_aes.key_dec, AES_MAX_KEYLENGTH);
  131. ok:
  132. for_each_online_cpu(cpu)
  133. if (&ctx->cword.encrypt == per_cpu(paes_last_cword, cpu) ||
  134. &ctx->cword.decrypt == per_cpu(paes_last_cword, cpu))
  135. per_cpu(paes_last_cword, cpu) = NULL;
  136. return 0;
  137. }
  138. static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
  139. unsigned int key_len)
  140. {
  141. return aes_set_key(crypto_skcipher_tfm(tfm), in_key, key_len);
  142. }
  143. /* ====== Encryption/decryption routines ====== */
  144. /* These are the real call to PadLock. */
  145. static inline void padlock_reset_key(struct cword *cword)
  146. {
  147. int cpu = raw_smp_processor_id();
  148. if (cword != per_cpu(paes_last_cword, cpu))
  149. #ifndef CONFIG_X86_64
  150. asm volatile ("pushfl; popfl");
  151. #else
  152. asm volatile ("pushfq; popfq");
  153. #endif
  154. }
  155. static inline void padlock_store_cword(struct cword *cword)
  156. {
  157. per_cpu(paes_last_cword, raw_smp_processor_id()) = cword;
  158. }
  159. /*
  160. * While the padlock instructions don't use FP/SSE registers, they
  161. * generate a spurious DNA fault when CR0.TS is '1'. Fortunately,
  162. * the kernel doesn't use CR0.TS.
  163. */
  164. static inline void rep_xcrypt_ecb(const u8 *input, u8 *output, void *key,
  165. struct cword *control_word, int count)
  166. {
  167. asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
  168. : "+S"(input), "+D"(output)
  169. : "d"(control_word), "b"(key), "c"(count));
  170. }
  171. static inline u8 *rep_xcrypt_cbc(const u8 *input, u8 *output, void *key,
  172. u8 *iv, struct cword *control_word, int count)
  173. {
  174. asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
  175. : "+S" (input), "+D" (output), "+a" (iv)
  176. : "d" (control_word), "b" (key), "c" (count));
  177. return iv;
  178. }
  179. static void ecb_crypt_copy(const u8 *in, u8 *out, u32 *key,
  180. struct cword *cword, int count)
  181. {
  182. /*
  183. * Padlock prefetches extra data so we must provide mapped input buffers.
  184. * Assume there are at least 16 bytes of stack already in use.
  185. */
  186. u8 buf[AES_BLOCK_SIZE * (MAX_ECB_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
  187. u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
  188. memcpy(tmp, in, count * AES_BLOCK_SIZE);
  189. rep_xcrypt_ecb(tmp, out, key, cword, count);
  190. }
  191. static u8 *cbc_crypt_copy(const u8 *in, u8 *out, u32 *key,
  192. u8 *iv, struct cword *cword, int count)
  193. {
  194. /*
  195. * Padlock prefetches extra data so we must provide mapped input buffers.
  196. * Assume there are at least 16 bytes of stack already in use.
  197. */
  198. u8 buf[AES_BLOCK_SIZE * (MAX_CBC_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
  199. u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
  200. memcpy(tmp, in, count * AES_BLOCK_SIZE);
  201. return rep_xcrypt_cbc(tmp, out, key, iv, cword, count);
  202. }
  203. static inline void ecb_crypt(const u8 *in, u8 *out, u32 *key,
  204. struct cword *cword, int count)
  205. {
  206. /* Padlock in ECB mode fetches at least ecb_fetch_bytes of data.
  207. * We could avoid some copying here but it's probably not worth it.
  208. */
  209. if (unlikely(offset_in_page(in) + ecb_fetch_bytes > PAGE_SIZE)) {
  210. ecb_crypt_copy(in, out, key, cword, count);
  211. return;
  212. }
  213. rep_xcrypt_ecb(in, out, key, cword, count);
  214. }
  215. static inline u8 *cbc_crypt(const u8 *in, u8 *out, u32 *key,
  216. u8 *iv, struct cword *cword, int count)
  217. {
  218. /* Padlock in CBC mode fetches at least cbc_fetch_bytes of data. */
  219. if (unlikely(offset_in_page(in) + cbc_fetch_bytes > PAGE_SIZE))
  220. return cbc_crypt_copy(in, out, key, iv, cword, count);
  221. return rep_xcrypt_cbc(in, out, key, iv, cword, count);
  222. }
  223. static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
  224. void *control_word, u32 count)
  225. {
  226. u32 initial = count & (ecb_fetch_blocks - 1);
  227. if (count < ecb_fetch_blocks) {
  228. ecb_crypt(input, output, key, control_word, count);
  229. return;
  230. }
  231. count -= initial;
  232. if (initial)
  233. asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
  234. : "+S"(input), "+D"(output)
  235. : "d"(control_word), "b"(key), "c"(initial));
  236. asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
  237. : "+S"(input), "+D"(output)
  238. : "d"(control_word), "b"(key), "c"(count));
  239. }
  240. static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
  241. u8 *iv, void *control_word, u32 count)
  242. {
  243. u32 initial = count & (cbc_fetch_blocks - 1);
  244. if (count < cbc_fetch_blocks)
  245. return cbc_crypt(input, output, key, iv, control_word, count);
  246. count -= initial;
  247. if (initial)
  248. asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
  249. : "+S" (input), "+D" (output), "+a" (iv)
  250. : "d" (control_word), "b" (key), "c" (initial));
  251. asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
  252. : "+S" (input), "+D" (output), "+a" (iv)
  253. : "d" (control_word), "b" (key), "c" (count));
  254. return iv;
  255. }
  256. static void padlock_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
  257. {
  258. struct aes_ctx *ctx = aes_ctx(tfm);
  259. padlock_reset_key(&ctx->cword.encrypt);
  260. ecb_crypt(in, out, ctx->E, &ctx->cword.encrypt, 1);
  261. padlock_store_cword(&ctx->cword.encrypt);
  262. }
  263. static void padlock_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
  264. {
  265. struct aes_ctx *ctx = aes_ctx(tfm);
  266. padlock_reset_key(&ctx->cword.encrypt);
  267. ecb_crypt(in, out, ctx->D, &ctx->cword.decrypt, 1);
  268. padlock_store_cword(&ctx->cword.encrypt);
  269. }
  270. static struct crypto_alg aes_alg = {
  271. .cra_name = "aes",
  272. .cra_driver_name = "aes-padlock",
  273. .cra_priority = PADLOCK_CRA_PRIORITY,
  274. .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
  275. .cra_blocksize = AES_BLOCK_SIZE,
  276. .cra_ctxsize = sizeof(struct aes_ctx),
  277. .cra_alignmask = PADLOCK_ALIGNMENT - 1,
  278. .cra_module = THIS_MODULE,
  279. .cra_u = {
  280. .cipher = {
  281. .cia_min_keysize = AES_MIN_KEY_SIZE,
  282. .cia_max_keysize = AES_MAX_KEY_SIZE,
  283. .cia_setkey = aes_set_key,
  284. .cia_encrypt = padlock_aes_encrypt,
  285. .cia_decrypt = padlock_aes_decrypt,
  286. }
  287. }
  288. };
  289. static int ecb_aes_encrypt(struct skcipher_request *req)
  290. {
  291. struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  292. struct aes_ctx *ctx = skcipher_aes_ctx(tfm);
  293. struct skcipher_walk walk;
  294. unsigned int nbytes;
  295. int err;
  296. padlock_reset_key(&ctx->cword.encrypt);
  297. err = skcipher_walk_virt(&walk, req, false);
  298. while ((nbytes = walk.nbytes) != 0) {
  299. padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
  300. ctx->E, &ctx->cword.encrypt,
  301. nbytes / AES_BLOCK_SIZE);
  302. nbytes &= AES_BLOCK_SIZE - 1;
  303. err = skcipher_walk_done(&walk, nbytes);
  304. }
  305. padlock_store_cword(&ctx->cword.encrypt);
  306. return err;
  307. }
  308. static int ecb_aes_decrypt(struct skcipher_request *req)
  309. {
  310. struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  311. struct aes_ctx *ctx = skcipher_aes_ctx(tfm);
  312. struct skcipher_walk walk;
  313. unsigned int nbytes;
  314. int err;
  315. padlock_reset_key(&ctx->cword.decrypt);
  316. err = skcipher_walk_virt(&walk, req, false);
  317. while ((nbytes = walk.nbytes) != 0) {
  318. padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
  319. ctx->D, &ctx->cword.decrypt,
  320. nbytes / AES_BLOCK_SIZE);
  321. nbytes &= AES_BLOCK_SIZE - 1;
  322. err = skcipher_walk_done(&walk, nbytes);
  323. }
  324. padlock_store_cword(&ctx->cword.encrypt);
  325. return err;
  326. }
  327. static struct skcipher_alg ecb_aes_alg = {
  328. .base.cra_name = "ecb(aes)",
  329. .base.cra_driver_name = "ecb-aes-padlock",
  330. .base.cra_priority = PADLOCK_COMPOSITE_PRIORITY,
  331. .base.cra_blocksize = AES_BLOCK_SIZE,
  332. .base.cra_ctxsize = sizeof(struct aes_ctx),
  333. .base.cra_alignmask = PADLOCK_ALIGNMENT - 1,
  334. .base.cra_module = THIS_MODULE,
  335. .min_keysize = AES_MIN_KEY_SIZE,
  336. .max_keysize = AES_MAX_KEY_SIZE,
  337. .setkey = aes_set_key_skcipher,
  338. .encrypt = ecb_aes_encrypt,
  339. .decrypt = ecb_aes_decrypt,
  340. };
  341. static int cbc_aes_encrypt(struct skcipher_request *req)
  342. {
  343. struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  344. struct aes_ctx *ctx = skcipher_aes_ctx(tfm);
  345. struct skcipher_walk walk;
  346. unsigned int nbytes;
  347. int err;
  348. padlock_reset_key(&ctx->cword.encrypt);
  349. err = skcipher_walk_virt(&walk, req, false);
  350. while ((nbytes = walk.nbytes) != 0) {
  351. u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr,
  352. walk.dst.virt.addr, ctx->E,
  353. walk.iv, &ctx->cword.encrypt,
  354. nbytes / AES_BLOCK_SIZE);
  355. memcpy(walk.iv, iv, AES_BLOCK_SIZE);
  356. nbytes &= AES_BLOCK_SIZE - 1;
  357. err = skcipher_walk_done(&walk, nbytes);
  358. }
  359. padlock_store_cword(&ctx->cword.decrypt);
  360. return err;
  361. }
  362. static int cbc_aes_decrypt(struct skcipher_request *req)
  363. {
  364. struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  365. struct aes_ctx *ctx = skcipher_aes_ctx(tfm);
  366. struct skcipher_walk walk;
  367. unsigned int nbytes;
  368. int err;
  369. padlock_reset_key(&ctx->cword.encrypt);
  370. err = skcipher_walk_virt(&walk, req, false);
  371. while ((nbytes = walk.nbytes) != 0) {
  372. padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr,
  373. ctx->D, walk.iv, &ctx->cword.decrypt,
  374. nbytes / AES_BLOCK_SIZE);
  375. nbytes &= AES_BLOCK_SIZE - 1;
  376. err = skcipher_walk_done(&walk, nbytes);
  377. }
  378. padlock_store_cword(&ctx->cword.encrypt);
  379. return err;
  380. }
  381. static struct skcipher_alg cbc_aes_alg = {
  382. .base.cra_name = "cbc(aes)",
  383. .base.cra_driver_name = "cbc-aes-padlock",
  384. .base.cra_priority = PADLOCK_COMPOSITE_PRIORITY,
  385. .base.cra_blocksize = AES_BLOCK_SIZE,
  386. .base.cra_ctxsize = sizeof(struct aes_ctx),
  387. .base.cra_alignmask = PADLOCK_ALIGNMENT - 1,
  388. .base.cra_module = THIS_MODULE,
  389. .min_keysize = AES_MIN_KEY_SIZE,
  390. .max_keysize = AES_MAX_KEY_SIZE,
  391. .ivsize = AES_BLOCK_SIZE,
  392. .setkey = aes_set_key_skcipher,
  393. .encrypt = cbc_aes_encrypt,
  394. .decrypt = cbc_aes_decrypt,
  395. };
  396. static const struct x86_cpu_id padlock_cpu_id[] = {
  397. X86_MATCH_FEATURE(X86_FEATURE_XCRYPT, NULL),
  398. {}
  399. };
  400. MODULE_DEVICE_TABLE(x86cpu, padlock_cpu_id);
  401. static int __init padlock_init(void)
  402. {
  403. int ret;
  404. struct cpuinfo_x86 *c = &cpu_data(0);
  405. if (!x86_match_cpu(padlock_cpu_id))
  406. return -ENODEV;
  407. if (!boot_cpu_has(X86_FEATURE_XCRYPT_EN)) {
  408. printk(KERN_NOTICE PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
  409. return -ENODEV;
  410. }
  411. if ((ret = crypto_register_alg(&aes_alg)) != 0)
  412. goto aes_err;
  413. if ((ret = crypto_register_skcipher(&ecb_aes_alg)) != 0)
  414. goto ecb_aes_err;
  415. if ((ret = crypto_register_skcipher(&cbc_aes_alg)) != 0)
  416. goto cbc_aes_err;
  417. printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
  418. if (c->x86 == 6 && c->x86_model == 15 && c->x86_stepping == 2) {
  419. ecb_fetch_blocks = MAX_ECB_FETCH_BLOCKS;
  420. cbc_fetch_blocks = MAX_CBC_FETCH_BLOCKS;
  421. printk(KERN_NOTICE PFX "VIA Nano stepping 2 detected: enabling workaround.\n");
  422. }
  423. out:
  424. return ret;
  425. cbc_aes_err:
  426. crypto_unregister_skcipher(&ecb_aes_alg);
  427. ecb_aes_err:
  428. crypto_unregister_alg(&aes_alg);
  429. aes_err:
  430. printk(KERN_ERR PFX "VIA PadLock AES initialization failed.\n");
  431. goto out;
  432. }
  433. static void __exit padlock_fini(void)
  434. {
  435. crypto_unregister_skcipher(&cbc_aes_alg);
  436. crypto_unregister_skcipher(&ecb_aes_alg);
  437. crypto_unregister_alg(&aes_alg);
  438. }
  439. module_init(padlock_init);
  440. module_exit(padlock_fini);
  441. MODULE_DESCRIPTION("VIA PadLock AES algorithm support");
  442. MODULE_LICENSE("GPL");
  443. MODULE_AUTHOR("Michal Ludvig");
  444. MODULE_ALIAS_CRYPTO("aes");