chacha20poly1305.c 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368
  1. // SPDX-License-Identifier: GPL-2.0 OR MIT
  2. /*
  3. * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
  4. *
  5. * This is an implementation of the ChaCha20Poly1305 AEAD construction.
  6. *
  7. * Information: https://tools.ietf.org/html/rfc8439
  8. */
  9. #include <crypto/algapi.h>
  10. #include <crypto/chacha20poly1305.h>
  11. #include <crypto/chacha.h>
  12. #include <crypto/poly1305.h>
  13. #include <crypto/scatterwalk.h>
  14. #include <asm/unaligned.h>
  15. #include <linux/kernel.h>
  16. #include <linux/init.h>
  17. #include <linux/mm.h>
  18. #include <linux/module.h>
  19. #define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32))
  20. static void chacha_load_key(u32 *k, const u8 *in)
  21. {
  22. k[0] = get_unaligned_le32(in);
  23. k[1] = get_unaligned_le32(in + 4);
  24. k[2] = get_unaligned_le32(in + 8);
  25. k[3] = get_unaligned_le32(in + 12);
  26. k[4] = get_unaligned_le32(in + 16);
  27. k[5] = get_unaligned_le32(in + 20);
  28. k[6] = get_unaligned_le32(in + 24);
  29. k[7] = get_unaligned_le32(in + 28);
  30. }
  31. static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)
  32. {
  33. u32 k[CHACHA_KEY_WORDS];
  34. u8 iv[CHACHA_IV_SIZE];
  35. memset(iv, 0, 8);
  36. memcpy(iv + 8, nonce + 16, 8);
  37. chacha_load_key(k, key);
  38. /* Compute the subkey given the original key and first 128 nonce bits */
  39. chacha_init(chacha_state, k, nonce);
  40. hchacha_block(chacha_state, k, 20);
  41. chacha_init(chacha_state, k, iv);
  42. memzero_explicit(k, sizeof(k));
  43. memzero_explicit(iv, sizeof(iv));
  44. }
  45. static void
  46. __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
  47. const u8 *ad, const size_t ad_len, u32 *chacha_state)
  48. {
  49. const u8 *pad0 = page_address(ZERO_PAGE(0));
  50. struct poly1305_desc_ctx poly1305_state;
  51. union {
  52. u8 block0[POLY1305_KEY_SIZE];
  53. __le64 lens[2];
  54. } b;
  55. chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
  56. poly1305_init(&poly1305_state, b.block0);
  57. poly1305_update(&poly1305_state, ad, ad_len);
  58. if (ad_len & 0xf)
  59. poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
  60. chacha20_crypt(chacha_state, dst, src, src_len);
  61. poly1305_update(&poly1305_state, dst, src_len);
  62. if (src_len & 0xf)
  63. poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
  64. b.lens[0] = cpu_to_le64(ad_len);
  65. b.lens[1] = cpu_to_le64(src_len);
  66. poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
  67. poly1305_final(&poly1305_state, dst + src_len);
  68. memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
  69. memzero_explicit(&b, sizeof(b));
  70. }
  71. void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
  72. const u8 *ad, const size_t ad_len,
  73. const u64 nonce,
  74. const u8 key[CHACHA20POLY1305_KEY_SIZE])
  75. {
  76. u32 chacha_state[CHACHA_STATE_WORDS];
  77. u32 k[CHACHA_KEY_WORDS];
  78. __le64 iv[2];
  79. chacha_load_key(k, key);
  80. iv[0] = 0;
  81. iv[1] = cpu_to_le64(nonce);
  82. chacha_init(chacha_state, k, (u8 *)iv);
  83. __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
  84. memzero_explicit(iv, sizeof(iv));
  85. memzero_explicit(k, sizeof(k));
  86. }
  87. EXPORT_SYMBOL(chacha20poly1305_encrypt);
  88. void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
  89. const u8 *ad, const size_t ad_len,
  90. const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
  91. const u8 key[CHACHA20POLY1305_KEY_SIZE])
  92. {
  93. u32 chacha_state[CHACHA_STATE_WORDS];
  94. xchacha_init(chacha_state, key, nonce);
  95. __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
  96. }
  97. EXPORT_SYMBOL(xchacha20poly1305_encrypt);
  98. static bool
  99. __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
  100. const u8 *ad, const size_t ad_len, u32 *chacha_state)
  101. {
  102. const u8 *pad0 = page_address(ZERO_PAGE(0));
  103. struct poly1305_desc_ctx poly1305_state;
  104. size_t dst_len;
  105. int ret;
  106. union {
  107. u8 block0[POLY1305_KEY_SIZE];
  108. u8 mac[POLY1305_DIGEST_SIZE];
  109. __le64 lens[2];
  110. } b;
  111. if (unlikely(src_len < POLY1305_DIGEST_SIZE))
  112. return false;
  113. chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
  114. poly1305_init(&poly1305_state, b.block0);
  115. poly1305_update(&poly1305_state, ad, ad_len);
  116. if (ad_len & 0xf)
  117. poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
  118. dst_len = src_len - POLY1305_DIGEST_SIZE;
  119. poly1305_update(&poly1305_state, src, dst_len);
  120. if (dst_len & 0xf)
  121. poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
  122. b.lens[0] = cpu_to_le64(ad_len);
  123. b.lens[1] = cpu_to_le64(dst_len);
  124. poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
  125. poly1305_final(&poly1305_state, b.mac);
  126. ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
  127. if (likely(!ret))
  128. chacha20_crypt(chacha_state, dst, src, dst_len);
  129. memzero_explicit(&b, sizeof(b));
  130. return !ret;
  131. }
  132. bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
  133. const u8 *ad, const size_t ad_len,
  134. const u64 nonce,
  135. const u8 key[CHACHA20POLY1305_KEY_SIZE])
  136. {
  137. u32 chacha_state[CHACHA_STATE_WORDS];
  138. u32 k[CHACHA_KEY_WORDS];
  139. __le64 iv[2];
  140. bool ret;
  141. chacha_load_key(k, key);
  142. iv[0] = 0;
  143. iv[1] = cpu_to_le64(nonce);
  144. chacha_init(chacha_state, k, (u8 *)iv);
  145. ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
  146. chacha_state);
  147. memzero_explicit(chacha_state, sizeof(chacha_state));
  148. memzero_explicit(iv, sizeof(iv));
  149. memzero_explicit(k, sizeof(k));
  150. return ret;
  151. }
  152. EXPORT_SYMBOL(chacha20poly1305_decrypt);
  153. bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
  154. const u8 *ad, const size_t ad_len,
  155. const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
  156. const u8 key[CHACHA20POLY1305_KEY_SIZE])
  157. {
  158. u32 chacha_state[CHACHA_STATE_WORDS];
  159. xchacha_init(chacha_state, key, nonce);
  160. return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
  161. chacha_state);
  162. }
  163. EXPORT_SYMBOL(xchacha20poly1305_decrypt);
  164. static
  165. bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
  166. const size_t src_len,
  167. const u8 *ad, const size_t ad_len,
  168. const u64 nonce,
  169. const u8 key[CHACHA20POLY1305_KEY_SIZE],
  170. int encrypt)
  171. {
  172. const u8 *pad0 = page_address(ZERO_PAGE(0));
  173. struct poly1305_desc_ctx poly1305_state;
  174. u32 chacha_state[CHACHA_STATE_WORDS];
  175. struct sg_mapping_iter miter;
  176. size_t partial = 0;
  177. unsigned int flags;
  178. bool ret = true;
  179. int sl;
  180. union {
  181. struct {
  182. u32 k[CHACHA_KEY_WORDS];
  183. __le64 iv[2];
  184. };
  185. u8 block0[POLY1305_KEY_SIZE];
  186. u8 chacha_stream[CHACHA_BLOCK_SIZE];
  187. struct {
  188. u8 mac[2][POLY1305_DIGEST_SIZE];
  189. };
  190. __le64 lens[2];
  191. } b __aligned(16);
  192. if (WARN_ON(src_len > INT_MAX))
  193. return false;
  194. chacha_load_key(b.k, key);
  195. b.iv[0] = 0;
  196. b.iv[1] = cpu_to_le64(nonce);
  197. chacha_init(chacha_state, b.k, (u8 *)b.iv);
  198. chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
  199. poly1305_init(&poly1305_state, b.block0);
  200. if (unlikely(ad_len)) {
  201. poly1305_update(&poly1305_state, ad, ad_len);
  202. if (ad_len & 0xf)
  203. poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
  204. }
  205. flags = SG_MITER_TO_SG | SG_MITER_ATOMIC;
  206. sg_miter_start(&miter, src, sg_nents(src), flags);
  207. for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
  208. u8 *addr = miter.addr;
  209. size_t length = min_t(size_t, sl, miter.length);
  210. if (!encrypt)
  211. poly1305_update(&poly1305_state, addr, length);
  212. if (unlikely(partial)) {
  213. size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
  214. crypto_xor(addr, b.chacha_stream + partial, l);
  215. partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
  216. addr += l;
  217. length -= l;
  218. }
  219. if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
  220. size_t l = length;
  221. if (unlikely(length < sl))
  222. l &= ~(CHACHA_BLOCK_SIZE - 1);
  223. chacha20_crypt(chacha_state, addr, addr, l);
  224. addr += l;
  225. length -= l;
  226. }
  227. if (unlikely(length > 0)) {
  228. chacha20_crypt(chacha_state, b.chacha_stream, pad0,
  229. CHACHA_BLOCK_SIZE);
  230. crypto_xor(addr, b.chacha_stream, length);
  231. partial = length;
  232. }
  233. if (encrypt)
  234. poly1305_update(&poly1305_state, miter.addr,
  235. min_t(size_t, sl, miter.length));
  236. }
  237. if (src_len & 0xf)
  238. poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
  239. b.lens[0] = cpu_to_le64(ad_len);
  240. b.lens[1] = cpu_to_le64(src_len);
  241. poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
  242. if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
  243. if (encrypt) {
  244. poly1305_final(&poly1305_state,
  245. miter.addr + miter.length + sl);
  246. ret = true;
  247. } else {
  248. poly1305_final(&poly1305_state, b.mac[0]);
  249. ret = !crypto_memneq(b.mac[0],
  250. miter.addr + miter.length + sl,
  251. POLY1305_DIGEST_SIZE);
  252. }
  253. }
  254. sg_miter_stop(&miter);
  255. if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
  256. poly1305_final(&poly1305_state, b.mac[1]);
  257. scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
  258. sizeof(b.mac[1]), encrypt);
  259. ret = encrypt ||
  260. !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
  261. }
  262. memzero_explicit(chacha_state, sizeof(chacha_state));
  263. memzero_explicit(&b, sizeof(b));
  264. return ret;
  265. }
  266. bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
  267. const u8 *ad, const size_t ad_len,
  268. const u64 nonce,
  269. const u8 key[CHACHA20POLY1305_KEY_SIZE])
  270. {
  271. return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
  272. nonce, key, 1);
  273. }
  274. EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
  275. bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
  276. const u8 *ad, const size_t ad_len,
  277. const u64 nonce,
  278. const u8 key[CHACHA20POLY1305_KEY_SIZE])
  279. {
  280. if (unlikely(src_len < POLY1305_DIGEST_SIZE))
  281. return false;
  282. return chacha20poly1305_crypt_sg_inplace(src,
  283. src_len - POLY1305_DIGEST_SIZE,
  284. ad, ad_len, nonce, key, 0);
  285. }
  286. EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
  287. static int __init mod_init(void)
  288. {
  289. if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
  290. WARN_ON(!chacha20poly1305_selftest()))
  291. return -ENODEV;
  292. return 0;
  293. }
  294. module_init(mod_init);
  295. MODULE_LICENSE("GPL v2");
  296. MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
  297. MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");