aes-ce-glue.c 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * aes-ce-cipher.c - core AES cipher using ARMv8 Crypto Extensions
  4. *
  5. * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
  6. */
  7. #include <asm/neon.h>
  8. #include <asm/simd.h>
  9. #include <asm/unaligned.h>
  10. #include <crypto/aes.h>
  11. #include <crypto/internal/simd.h>
  12. #include <linux/cpufeature.h>
  13. #include <linux/crypto.h>
  14. #include <linux/module.h>
  15. #include "aes-ce-setkey.h"
  16. MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions");
  17. MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
  18. MODULE_LICENSE("GPL v2");
  19. struct aes_block {
  20. u8 b[AES_BLOCK_SIZE];
  21. };
  22. asmlinkage void __aes_ce_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
  23. asmlinkage void __aes_ce_decrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
  24. asmlinkage u32 __aes_ce_sub(u32 l);
  25. asmlinkage void __aes_ce_invert(struct aes_block *out,
  26. const struct aes_block *in);
  27. static int num_rounds(struct crypto_aes_ctx *ctx)
  28. {
  29. /*
  30. * # of rounds specified by AES:
  31. * 128 bit key 10 rounds
  32. * 192 bit key 12 rounds
  33. * 256 bit key 14 rounds
  34. * => n byte key => 6 + (n/4) rounds
  35. */
  36. return 6 + ctx->key_length / 4;
  37. }
  38. static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
  39. {
  40. struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
  41. if (!crypto_simd_usable()) {
  42. aes_encrypt(ctx, dst, src);
  43. return;
  44. }
  45. kernel_neon_begin();
  46. __aes_ce_encrypt(ctx->key_enc, dst, src, num_rounds(ctx));
  47. kernel_neon_end();
  48. }
  49. static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
  50. {
  51. struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
  52. if (!crypto_simd_usable()) {
  53. aes_decrypt(ctx, dst, src);
  54. return;
  55. }
  56. kernel_neon_begin();
  57. __aes_ce_decrypt(ctx->key_dec, dst, src, num_rounds(ctx));
  58. kernel_neon_end();
  59. }
  60. int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
  61. unsigned int key_len)
  62. {
  63. /*
  64. * The AES key schedule round constants
  65. */
  66. static u8 const rcon[] = {
  67. 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
  68. };
  69. u32 kwords = key_len / sizeof(u32);
  70. struct aes_block *key_enc, *key_dec;
  71. int i, j;
  72. if (key_len != AES_KEYSIZE_128 &&
  73. key_len != AES_KEYSIZE_192 &&
  74. key_len != AES_KEYSIZE_256)
  75. return -EINVAL;
  76. ctx->key_length = key_len;
  77. for (i = 0; i < kwords; i++)
  78. ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));
  79. kernel_neon_begin();
  80. for (i = 0; i < sizeof(rcon); i++) {
  81. u32 *rki = ctx->key_enc + (i * kwords);
  82. u32 *rko = rki + kwords;
  83. rko[0] = ror32(__aes_ce_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0];
  84. rko[1] = rko[0] ^ rki[1];
  85. rko[2] = rko[1] ^ rki[2];
  86. rko[3] = rko[2] ^ rki[3];
  87. if (key_len == AES_KEYSIZE_192) {
  88. if (i >= 7)
  89. break;
  90. rko[4] = rko[3] ^ rki[4];
  91. rko[5] = rko[4] ^ rki[5];
  92. } else if (key_len == AES_KEYSIZE_256) {
  93. if (i >= 6)
  94. break;
  95. rko[4] = __aes_ce_sub(rko[3]) ^ rki[4];
  96. rko[5] = rko[4] ^ rki[5];
  97. rko[6] = rko[5] ^ rki[6];
  98. rko[7] = rko[6] ^ rki[7];
  99. }
  100. }
  101. /*
  102. * Generate the decryption keys for the Equivalent Inverse Cipher.
  103. * This involves reversing the order of the round keys, and applying
  104. * the Inverse Mix Columns transformation on all but the first and
  105. * the last one.
  106. */
  107. key_enc = (struct aes_block *)ctx->key_enc;
  108. key_dec = (struct aes_block *)ctx->key_dec;
  109. j = num_rounds(ctx);
  110. key_dec[0] = key_enc[j];
  111. for (i = 1, j--; j > 0; i++, j--)
  112. __aes_ce_invert(key_dec + i, key_enc + j);
  113. key_dec[i] = key_enc[0];
  114. kernel_neon_end();
  115. return 0;
  116. }
  117. EXPORT_SYMBOL(ce_aes_expandkey);
  118. int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
  119. unsigned int key_len)
  120. {
  121. struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
  122. return ce_aes_expandkey(ctx, in_key, key_len);
  123. }
  124. EXPORT_SYMBOL(ce_aes_setkey);
  125. static struct crypto_alg aes_alg = {
  126. .cra_name = "aes",
  127. .cra_driver_name = "aes-ce",
  128. .cra_priority = 250,
  129. .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
  130. .cra_blocksize = AES_BLOCK_SIZE,
  131. .cra_ctxsize = sizeof(struct crypto_aes_ctx),
  132. .cra_module = THIS_MODULE,
  133. .cra_cipher = {
  134. .cia_min_keysize = AES_MIN_KEY_SIZE,
  135. .cia_max_keysize = AES_MAX_KEY_SIZE,
  136. .cia_setkey = ce_aes_setkey,
  137. .cia_encrypt = aes_cipher_encrypt,
  138. .cia_decrypt = aes_cipher_decrypt
  139. }
  140. };
  141. static int __init aes_mod_init(void)
  142. {
  143. return crypto_register_alg(&aes_alg);
  144. }
  145. static void __exit aes_mod_exit(void)
  146. {
  147. crypto_unregister_alg(&aes_alg);
  148. }
  149. module_cpu_feature_match(AES, aes_mod_init);
  150. module_exit(aes_mod_exit);