xts.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /* XTS: as defined in IEEE1619/D16
  3. * http://grouper.ieee.org/groups/1619/email/pdf00086.pdf
  4. *
  5. * Copyright (c) 2007 Rik Snel <rsnel@cube.dyndns.org>
  6. *
  7. * Based on ecb.c
  8. * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  9. */
  10. #include <crypto/internal/cipher.h>
  11. #include <crypto/internal/skcipher.h>
  12. #include <crypto/scatterwalk.h>
  13. #include <linux/err.h>
  14. #include <linux/init.h>
  15. #include <linux/kernel.h>
  16. #include <linux/module.h>
  17. #include <linux/scatterlist.h>
  18. #include <linux/slab.h>
  19. #include <crypto/xts.h>
  20. #include <crypto/b128ops.h>
  21. #include <crypto/gf128mul.h>
  22. struct xts_tfm_ctx {
  23. struct crypto_skcipher *child;
  24. struct crypto_cipher *tweak;
  25. };
  26. struct xts_instance_ctx {
  27. struct crypto_skcipher_spawn spawn;
  28. char name[CRYPTO_MAX_ALG_NAME];
  29. };
  30. struct xts_request_ctx {
  31. le128 t;
  32. struct scatterlist *tail;
  33. struct scatterlist sg[2];
  34. struct skcipher_request subreq;
  35. };
  36. static int xts_setkey(struct crypto_skcipher *parent, const u8 *key,
  37. unsigned int keylen)
  38. {
  39. struct xts_tfm_ctx *ctx = crypto_skcipher_ctx(parent);
  40. struct crypto_skcipher *child;
  41. struct crypto_cipher *tweak;
  42. int err;
  43. err = xts_verify_key(parent, key, keylen);
  44. if (err)
  45. return err;
  46. keylen /= 2;
  47. /* we need two cipher instances: one to compute the initial 'tweak'
  48. * by encrypting the IV (usually the 'plain' iv) and the other
  49. * one to encrypt and decrypt the data */
  50. /* tweak cipher, uses Key2 i.e. the second half of *key */
  51. tweak = ctx->tweak;
  52. crypto_cipher_clear_flags(tweak, CRYPTO_TFM_REQ_MASK);
  53. crypto_cipher_set_flags(tweak, crypto_skcipher_get_flags(parent) &
  54. CRYPTO_TFM_REQ_MASK);
  55. err = crypto_cipher_setkey(tweak, key + keylen, keylen);
  56. if (err)
  57. return err;
  58. /* data cipher, uses Key1 i.e. the first half of *key */
  59. child = ctx->child;
  60. crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
  61. crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &
  62. CRYPTO_TFM_REQ_MASK);
  63. return crypto_skcipher_setkey(child, key, keylen);
  64. }
  65. /*
  66. * We compute the tweak masks twice (both before and after the ECB encryption or
  67. * decryption) to avoid having to allocate a temporary buffer and/or make
  68. * mutliple calls to the 'ecb(..)' instance, which usually would be slower than
  69. * just doing the gf128mul_x_ble() calls again.
  70. */
  71. static int xts_xor_tweak(struct skcipher_request *req, bool second_pass,
  72. bool enc)
  73. {
  74. struct xts_request_ctx *rctx = skcipher_request_ctx(req);
  75. struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  76. const bool cts = (req->cryptlen % XTS_BLOCK_SIZE);
  77. const int bs = XTS_BLOCK_SIZE;
  78. struct skcipher_walk w;
  79. le128 t = rctx->t;
  80. int err;
  81. if (second_pass) {
  82. req = &rctx->subreq;
  83. /* set to our TFM to enforce correct alignment: */
  84. skcipher_request_set_tfm(req, tfm);
  85. }
  86. err = skcipher_walk_virt(&w, req, false);
  87. while (w.nbytes) {
  88. unsigned int avail = w.nbytes;
  89. le128 *wsrc;
  90. le128 *wdst;
  91. wsrc = w.src.virt.addr;
  92. wdst = w.dst.virt.addr;
  93. do {
  94. if (unlikely(cts) &&
  95. w.total - w.nbytes + avail < 2 * XTS_BLOCK_SIZE) {
  96. if (!enc) {
  97. if (second_pass)
  98. rctx->t = t;
  99. gf128mul_x_ble(&t, &t);
  100. }
  101. le128_xor(wdst, &t, wsrc);
  102. if (enc && second_pass)
  103. gf128mul_x_ble(&rctx->t, &t);
  104. skcipher_walk_done(&w, avail - bs);
  105. return 0;
  106. }
  107. le128_xor(wdst++, &t, wsrc++);
  108. gf128mul_x_ble(&t, &t);
  109. } while ((avail -= bs) >= bs);
  110. err = skcipher_walk_done(&w, avail);
  111. }
  112. return err;
  113. }
  114. static int xts_xor_tweak_pre(struct skcipher_request *req, bool enc)
  115. {
  116. return xts_xor_tweak(req, false, enc);
  117. }
  118. static int xts_xor_tweak_post(struct skcipher_request *req, bool enc)
  119. {
  120. return xts_xor_tweak(req, true, enc);
  121. }
  122. static void xts_cts_done(struct crypto_async_request *areq, int err)
  123. {
  124. struct skcipher_request *req = areq->data;
  125. le128 b;
  126. if (!err) {
  127. struct xts_request_ctx *rctx = skcipher_request_ctx(req);
  128. scatterwalk_map_and_copy(&b, rctx->tail, 0, XTS_BLOCK_SIZE, 0);
  129. le128_xor(&b, &rctx->t, &b);
  130. scatterwalk_map_and_copy(&b, rctx->tail, 0, XTS_BLOCK_SIZE, 1);
  131. }
  132. skcipher_request_complete(req, err);
  133. }
  134. static int xts_cts_final(struct skcipher_request *req,
  135. int (*crypt)(struct skcipher_request *req))
  136. {
  137. const struct xts_tfm_ctx *ctx =
  138. crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
  139. int offset = req->cryptlen & ~(XTS_BLOCK_SIZE - 1);
  140. struct xts_request_ctx *rctx = skcipher_request_ctx(req);
  141. struct skcipher_request *subreq = &rctx->subreq;
  142. int tail = req->cryptlen % XTS_BLOCK_SIZE;
  143. le128 b[2];
  144. int err;
  145. rctx->tail = scatterwalk_ffwd(rctx->sg, req->dst,
  146. offset - XTS_BLOCK_SIZE);
  147. scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 0);
  148. b[1] = b[0];
  149. scatterwalk_map_and_copy(b, req->src, offset, tail, 0);
  150. le128_xor(b, &rctx->t, b);
  151. scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE + tail, 1);
  152. skcipher_request_set_tfm(subreq, ctx->child);
  153. skcipher_request_set_callback(subreq, req->base.flags, xts_cts_done,
  154. req);
  155. skcipher_request_set_crypt(subreq, rctx->tail, rctx->tail,
  156. XTS_BLOCK_SIZE, NULL);
  157. err = crypt(subreq);
  158. if (err)
  159. return err;
  160. scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 0);
  161. le128_xor(b, &rctx->t, b);
  162. scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 1);
  163. return 0;
  164. }
  165. static void xts_encrypt_done(struct crypto_async_request *areq, int err)
  166. {
  167. struct skcipher_request *req = areq->data;
  168. if (!err) {
  169. struct xts_request_ctx *rctx = skcipher_request_ctx(req);
  170. rctx->subreq.base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  171. err = xts_xor_tweak_post(req, true);
  172. if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) {
  173. err = xts_cts_final(req, crypto_skcipher_encrypt);
  174. if (err == -EINPROGRESS)
  175. return;
  176. }
  177. }
  178. skcipher_request_complete(req, err);
  179. }
  180. static void xts_decrypt_done(struct crypto_async_request *areq, int err)
  181. {
  182. struct skcipher_request *req = areq->data;
  183. if (!err) {
  184. struct xts_request_ctx *rctx = skcipher_request_ctx(req);
  185. rctx->subreq.base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  186. err = xts_xor_tweak_post(req, false);
  187. if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) {
  188. err = xts_cts_final(req, crypto_skcipher_decrypt);
  189. if (err == -EINPROGRESS)
  190. return;
  191. }
  192. }
  193. skcipher_request_complete(req, err);
  194. }
  195. static int xts_init_crypt(struct skcipher_request *req,
  196. crypto_completion_t compl)
  197. {
  198. const struct xts_tfm_ctx *ctx =
  199. crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
  200. struct xts_request_ctx *rctx = skcipher_request_ctx(req);
  201. struct skcipher_request *subreq = &rctx->subreq;
  202. if (req->cryptlen < XTS_BLOCK_SIZE)
  203. return -EINVAL;
  204. skcipher_request_set_tfm(subreq, ctx->child);
  205. skcipher_request_set_callback(subreq, req->base.flags, compl, req);
  206. skcipher_request_set_crypt(subreq, req->dst, req->dst,
  207. req->cryptlen & ~(XTS_BLOCK_SIZE - 1), NULL);
  208. /* calculate first value of T */
  209. crypto_cipher_encrypt_one(ctx->tweak, (u8 *)&rctx->t, req->iv);
  210. return 0;
  211. }
  212. static int xts_encrypt(struct skcipher_request *req)
  213. {
  214. struct xts_request_ctx *rctx = skcipher_request_ctx(req);
  215. struct skcipher_request *subreq = &rctx->subreq;
  216. int err;
  217. err = xts_init_crypt(req, xts_encrypt_done) ?:
  218. xts_xor_tweak_pre(req, true) ?:
  219. crypto_skcipher_encrypt(subreq) ?:
  220. xts_xor_tweak_post(req, true);
  221. if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0))
  222. return err;
  223. return xts_cts_final(req, crypto_skcipher_encrypt);
  224. }
  225. static int xts_decrypt(struct skcipher_request *req)
  226. {
  227. struct xts_request_ctx *rctx = skcipher_request_ctx(req);
  228. struct skcipher_request *subreq = &rctx->subreq;
  229. int err;
  230. err = xts_init_crypt(req, xts_decrypt_done) ?:
  231. xts_xor_tweak_pre(req, false) ?:
  232. crypto_skcipher_decrypt(subreq) ?:
  233. xts_xor_tweak_post(req, false);
  234. if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0))
  235. return err;
  236. return xts_cts_final(req, crypto_skcipher_decrypt);
  237. }
  238. static int xts_init_tfm(struct crypto_skcipher *tfm)
  239. {
  240. struct skcipher_instance *inst = skcipher_alg_instance(tfm);
  241. struct xts_instance_ctx *ictx = skcipher_instance_ctx(inst);
  242. struct xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
  243. struct crypto_skcipher *child;
  244. struct crypto_cipher *tweak;
  245. child = crypto_spawn_skcipher(&ictx->spawn);
  246. if (IS_ERR(child))
  247. return PTR_ERR(child);
  248. ctx->child = child;
  249. tweak = crypto_alloc_cipher(ictx->name, 0, 0);
  250. if (IS_ERR(tweak)) {
  251. crypto_free_skcipher(ctx->child);
  252. return PTR_ERR(tweak);
  253. }
  254. ctx->tweak = tweak;
  255. crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) +
  256. sizeof(struct xts_request_ctx));
  257. return 0;
  258. }
  259. static void xts_exit_tfm(struct crypto_skcipher *tfm)
  260. {
  261. struct xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
  262. crypto_free_skcipher(ctx->child);
  263. crypto_free_cipher(ctx->tweak);
  264. }
  265. static void xts_free_instance(struct skcipher_instance *inst)
  266. {
  267. struct xts_instance_ctx *ictx = skcipher_instance_ctx(inst);
  268. crypto_drop_skcipher(&ictx->spawn);
  269. kfree(inst);
  270. }
  271. static int xts_create(struct crypto_template *tmpl, struct rtattr **tb)
  272. {
  273. struct skcipher_instance *inst;
  274. struct xts_instance_ctx *ctx;
  275. struct skcipher_alg *alg;
  276. const char *cipher_name;
  277. u32 mask;
  278. int err;
  279. err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SKCIPHER, &mask);
  280. if (err)
  281. return err;
  282. cipher_name = crypto_attr_alg_name(tb[1]);
  283. if (IS_ERR(cipher_name))
  284. return PTR_ERR(cipher_name);
  285. inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
  286. if (!inst)
  287. return -ENOMEM;
  288. ctx = skcipher_instance_ctx(inst);
  289. err = crypto_grab_skcipher(&ctx->spawn, skcipher_crypto_instance(inst),
  290. cipher_name, 0, mask);
  291. if (err == -ENOENT) {
  292. err = -ENAMETOOLONG;
  293. if (snprintf(ctx->name, CRYPTO_MAX_ALG_NAME, "ecb(%s)",
  294. cipher_name) >= CRYPTO_MAX_ALG_NAME)
  295. goto err_free_inst;
  296. err = crypto_grab_skcipher(&ctx->spawn,
  297. skcipher_crypto_instance(inst),
  298. ctx->name, 0, mask);
  299. }
  300. if (err)
  301. goto err_free_inst;
  302. alg = crypto_skcipher_spawn_alg(&ctx->spawn);
  303. err = -EINVAL;
  304. if (alg->base.cra_blocksize != XTS_BLOCK_SIZE)
  305. goto err_free_inst;
  306. if (crypto_skcipher_alg_ivsize(alg))
  307. goto err_free_inst;
  308. err = crypto_inst_setname(skcipher_crypto_instance(inst), "xts",
  309. &alg->base);
  310. if (err)
  311. goto err_free_inst;
  312. err = -EINVAL;
  313. cipher_name = alg->base.cra_name;
  314. /* Alas we screwed up the naming so we have to mangle the
  315. * cipher name.
  316. */
  317. if (!strncmp(cipher_name, "ecb(", 4)) {
  318. unsigned len;
  319. len = strlcpy(ctx->name, cipher_name + 4, sizeof(ctx->name));
  320. if (len < 2 || len >= sizeof(ctx->name))
  321. goto err_free_inst;
  322. if (ctx->name[len - 1] != ')')
  323. goto err_free_inst;
  324. ctx->name[len - 1] = 0;
  325. if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
  326. "xts(%s)", ctx->name) >= CRYPTO_MAX_ALG_NAME) {
  327. err = -ENAMETOOLONG;
  328. goto err_free_inst;
  329. }
  330. } else
  331. goto err_free_inst;
  332. inst->alg.base.cra_priority = alg->base.cra_priority;
  333. inst->alg.base.cra_blocksize = XTS_BLOCK_SIZE;
  334. inst->alg.base.cra_alignmask = alg->base.cra_alignmask |
  335. (__alignof__(u64) - 1);
  336. inst->alg.ivsize = XTS_BLOCK_SIZE;
  337. inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg) * 2;
  338. inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg) * 2;
  339. inst->alg.base.cra_ctxsize = sizeof(struct xts_tfm_ctx);
  340. inst->alg.init = xts_init_tfm;
  341. inst->alg.exit = xts_exit_tfm;
  342. inst->alg.setkey = xts_setkey;
  343. inst->alg.encrypt = xts_encrypt;
  344. inst->alg.decrypt = xts_decrypt;
  345. inst->free = xts_free_instance;
  346. err = skcipher_register_instance(tmpl, inst);
  347. if (err) {
  348. err_free_inst:
  349. xts_free_instance(inst);
  350. }
  351. return err;
  352. }
  353. static struct crypto_template xts_tmpl = {
  354. .name = "xts",
  355. .create = xts_create,
  356. .module = THIS_MODULE,
  357. };
  358. static int __init xts_module_init(void)
  359. {
  360. return crypto_register_template(&xts_tmpl);
  361. }
  362. static void __exit xts_module_exit(void)
  363. {
  364. crypto_unregister_template(&xts_tmpl);
  365. }
  366. subsys_initcall(xts_module_init);
  367. module_exit(xts_module_exit);
  368. MODULE_LICENSE("GPL");
  369. MODULE_DESCRIPTION("XTS block cipher mode");
  370. MODULE_ALIAS_CRYPTO("xts");
  371. MODULE_IMPORT_NS(CRYPTO_INTERNAL);