pcrypt.c 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * pcrypt - Parallel crypto wrapper.
  4. *
  5. * Copyright (C) 2009 secunet Security Networks AG
  6. * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
  7. */
  8. #include <crypto/algapi.h>
  9. #include <crypto/internal/aead.h>
  10. #include <linux/atomic.h>
  11. #include <linux/err.h>
  12. #include <linux/init.h>
  13. #include <linux/module.h>
  14. #include <linux/slab.h>
  15. #include <linux/kobject.h>
  16. #include <linux/cpu.h>
  17. #include <crypto/pcrypt.h>
  18. static struct padata_instance *pencrypt;
  19. static struct padata_instance *pdecrypt;
  20. static struct kset *pcrypt_kset;
  21. struct pcrypt_instance_ctx {
  22. struct crypto_aead_spawn spawn;
  23. struct padata_shell *psenc;
  24. struct padata_shell *psdec;
  25. atomic_t tfm_count;
  26. };
  27. struct pcrypt_aead_ctx {
  28. struct crypto_aead *child;
  29. unsigned int cb_cpu;
  30. };
  31. static inline struct pcrypt_instance_ctx *pcrypt_tfm_ictx(
  32. struct crypto_aead *tfm)
  33. {
  34. return aead_instance_ctx(aead_alg_instance(tfm));
  35. }
  36. static int pcrypt_aead_setkey(struct crypto_aead *parent,
  37. const u8 *key, unsigned int keylen)
  38. {
  39. struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
  40. return crypto_aead_setkey(ctx->child, key, keylen);
  41. }
  42. static int pcrypt_aead_setauthsize(struct crypto_aead *parent,
  43. unsigned int authsize)
  44. {
  45. struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
  46. return crypto_aead_setauthsize(ctx->child, authsize);
  47. }
  48. static void pcrypt_aead_serial(struct padata_priv *padata)
  49. {
  50. struct pcrypt_request *preq = pcrypt_padata_request(padata);
  51. struct aead_request *req = pcrypt_request_ctx(preq);
  52. aead_request_complete(req->base.data, padata->info);
  53. }
  54. static void pcrypt_aead_done(struct crypto_async_request *areq, int err)
  55. {
  56. struct aead_request *req = areq->data;
  57. struct pcrypt_request *preq = aead_request_ctx(req);
  58. struct padata_priv *padata = pcrypt_request_padata(preq);
  59. padata->info = err;
  60. padata_do_serial(padata);
  61. }
  62. static void pcrypt_aead_enc(struct padata_priv *padata)
  63. {
  64. struct pcrypt_request *preq = pcrypt_padata_request(padata);
  65. struct aead_request *req = pcrypt_request_ctx(preq);
  66. int ret;
  67. ret = crypto_aead_encrypt(req);
  68. if (ret == -EINPROGRESS)
  69. return;
  70. padata->info = ret;
  71. padata_do_serial(padata);
  72. }
  73. static int pcrypt_aead_encrypt(struct aead_request *req)
  74. {
  75. int err;
  76. struct pcrypt_request *preq = aead_request_ctx(req);
  77. struct aead_request *creq = pcrypt_request_ctx(preq);
  78. struct padata_priv *padata = pcrypt_request_padata(preq);
  79. struct crypto_aead *aead = crypto_aead_reqtfm(req);
  80. struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
  81. u32 flags = aead_request_flags(req);
  82. struct pcrypt_instance_ctx *ictx;
  83. ictx = pcrypt_tfm_ictx(aead);
  84. memset(padata, 0, sizeof(struct padata_priv));
  85. padata->parallel = pcrypt_aead_enc;
  86. padata->serial = pcrypt_aead_serial;
  87. aead_request_set_tfm(creq, ctx->child);
  88. aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
  89. pcrypt_aead_done, req);
  90. aead_request_set_crypt(creq, req->src, req->dst,
  91. req->cryptlen, req->iv);
  92. aead_request_set_ad(creq, req->assoclen);
  93. err = padata_do_parallel(ictx->psenc, padata, &ctx->cb_cpu);
  94. if (!err)
  95. return -EINPROGRESS;
  96. return err;
  97. }
  98. static void pcrypt_aead_dec(struct padata_priv *padata)
  99. {
  100. struct pcrypt_request *preq = pcrypt_padata_request(padata);
  101. struct aead_request *req = pcrypt_request_ctx(preq);
  102. int ret;
  103. ret = crypto_aead_decrypt(req);
  104. if (ret == -EINPROGRESS)
  105. return;
  106. padata->info = ret;
  107. padata_do_serial(padata);
  108. }
  109. static int pcrypt_aead_decrypt(struct aead_request *req)
  110. {
  111. int err;
  112. struct pcrypt_request *preq = aead_request_ctx(req);
  113. struct aead_request *creq = pcrypt_request_ctx(preq);
  114. struct padata_priv *padata = pcrypt_request_padata(preq);
  115. struct crypto_aead *aead = crypto_aead_reqtfm(req);
  116. struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
  117. u32 flags = aead_request_flags(req);
  118. struct pcrypt_instance_ctx *ictx;
  119. ictx = pcrypt_tfm_ictx(aead);
  120. memset(padata, 0, sizeof(struct padata_priv));
  121. padata->parallel = pcrypt_aead_dec;
  122. padata->serial = pcrypt_aead_serial;
  123. aead_request_set_tfm(creq, ctx->child);
  124. aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
  125. pcrypt_aead_done, req);
  126. aead_request_set_crypt(creq, req->src, req->dst,
  127. req->cryptlen, req->iv);
  128. aead_request_set_ad(creq, req->assoclen);
  129. err = padata_do_parallel(ictx->psdec, padata, &ctx->cb_cpu);
  130. if (!err)
  131. return -EINPROGRESS;
  132. return err;
  133. }
  134. static int pcrypt_aead_init_tfm(struct crypto_aead *tfm)
  135. {
  136. int cpu, cpu_index;
  137. struct aead_instance *inst = aead_alg_instance(tfm);
  138. struct pcrypt_instance_ctx *ictx = aead_instance_ctx(inst);
  139. struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm);
  140. struct crypto_aead *cipher;
  141. cpu_index = (unsigned int)atomic_inc_return(&ictx->tfm_count) %
  142. cpumask_weight(cpu_online_mask);
  143. ctx->cb_cpu = cpumask_first(cpu_online_mask);
  144. for (cpu = 0; cpu < cpu_index; cpu++)
  145. ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask);
  146. cipher = crypto_spawn_aead(&ictx->spawn);
  147. if (IS_ERR(cipher))
  148. return PTR_ERR(cipher);
  149. ctx->child = cipher;
  150. crypto_aead_set_reqsize(tfm, sizeof(struct pcrypt_request) +
  151. sizeof(struct aead_request) +
  152. crypto_aead_reqsize(cipher));
  153. return 0;
  154. }
  155. static void pcrypt_aead_exit_tfm(struct crypto_aead *tfm)
  156. {
  157. struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm);
  158. crypto_free_aead(ctx->child);
  159. }
  160. static void pcrypt_free(struct aead_instance *inst)
  161. {
  162. struct pcrypt_instance_ctx *ctx = aead_instance_ctx(inst);
  163. crypto_drop_aead(&ctx->spawn);
  164. padata_free_shell(ctx->psdec);
  165. padata_free_shell(ctx->psenc);
  166. kfree(inst);
  167. }
  168. static int pcrypt_init_instance(struct crypto_instance *inst,
  169. struct crypto_alg *alg)
  170. {
  171. if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
  172. "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
  173. return -ENAMETOOLONG;
  174. memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
  175. inst->alg.cra_priority = alg->cra_priority + 100;
  176. inst->alg.cra_blocksize = alg->cra_blocksize;
  177. inst->alg.cra_alignmask = alg->cra_alignmask;
  178. return 0;
  179. }
  180. static int pcrypt_create_aead(struct crypto_template *tmpl, struct rtattr **tb,
  181. struct crypto_attr_type *algt)
  182. {
  183. struct pcrypt_instance_ctx *ctx;
  184. struct aead_instance *inst;
  185. struct aead_alg *alg;
  186. u32 mask = crypto_algt_inherited_mask(algt);
  187. int err;
  188. inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
  189. if (!inst)
  190. return -ENOMEM;
  191. err = -ENOMEM;
  192. ctx = aead_instance_ctx(inst);
  193. ctx->psenc = padata_alloc_shell(pencrypt);
  194. if (!ctx->psenc)
  195. goto err_free_inst;
  196. ctx->psdec = padata_alloc_shell(pdecrypt);
  197. if (!ctx->psdec)
  198. goto err_free_inst;
  199. err = crypto_grab_aead(&ctx->spawn, aead_crypto_instance(inst),
  200. crypto_attr_alg_name(tb[1]), 0, mask);
  201. if (err)
  202. goto err_free_inst;
  203. alg = crypto_spawn_aead_alg(&ctx->spawn);
  204. err = pcrypt_init_instance(aead_crypto_instance(inst), &alg->base);
  205. if (err)
  206. goto err_free_inst;
  207. inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC;
  208. inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
  209. inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
  210. inst->alg.base.cra_ctxsize = sizeof(struct pcrypt_aead_ctx);
  211. inst->alg.init = pcrypt_aead_init_tfm;
  212. inst->alg.exit = pcrypt_aead_exit_tfm;
  213. inst->alg.setkey = pcrypt_aead_setkey;
  214. inst->alg.setauthsize = pcrypt_aead_setauthsize;
  215. inst->alg.encrypt = pcrypt_aead_encrypt;
  216. inst->alg.decrypt = pcrypt_aead_decrypt;
  217. inst->free = pcrypt_free;
  218. err = aead_register_instance(tmpl, inst);
  219. if (err) {
  220. err_free_inst:
  221. pcrypt_free(inst);
  222. }
  223. return err;
  224. }
  225. static int pcrypt_create(struct crypto_template *tmpl, struct rtattr **tb)
  226. {
  227. struct crypto_attr_type *algt;
  228. algt = crypto_get_attr_type(tb);
  229. if (IS_ERR(algt))
  230. return PTR_ERR(algt);
  231. switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
  232. case CRYPTO_ALG_TYPE_AEAD:
  233. return pcrypt_create_aead(tmpl, tb, algt);
  234. }
  235. return -EINVAL;
  236. }
  237. static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name)
  238. {
  239. int ret;
  240. pinst->kobj.kset = pcrypt_kset;
  241. ret = kobject_add(&pinst->kobj, NULL, "%s", name);
  242. if (!ret)
  243. kobject_uevent(&pinst->kobj, KOBJ_ADD);
  244. return ret;
  245. }
  246. static int pcrypt_init_padata(struct padata_instance **pinst, const char *name)
  247. {
  248. int ret = -ENOMEM;
  249. *pinst = padata_alloc(name);
  250. if (!*pinst)
  251. return ret;
  252. ret = pcrypt_sysfs_add(*pinst, name);
  253. if (ret)
  254. padata_free(*pinst);
  255. return ret;
  256. }
  257. static struct crypto_template pcrypt_tmpl = {
  258. .name = "pcrypt",
  259. .create = pcrypt_create,
  260. .module = THIS_MODULE,
  261. };
  262. static int __init pcrypt_init(void)
  263. {
  264. int err = -ENOMEM;
  265. pcrypt_kset = kset_create_and_add("pcrypt", NULL, kernel_kobj);
  266. if (!pcrypt_kset)
  267. goto err;
  268. err = pcrypt_init_padata(&pencrypt, "pencrypt");
  269. if (err)
  270. goto err_unreg_kset;
  271. err = pcrypt_init_padata(&pdecrypt, "pdecrypt");
  272. if (err)
  273. goto err_deinit_pencrypt;
  274. return crypto_register_template(&pcrypt_tmpl);
  275. err_deinit_pencrypt:
  276. padata_free(pencrypt);
  277. err_unreg_kset:
  278. kset_unregister(pcrypt_kset);
  279. err:
  280. return err;
  281. }
  282. static void __exit pcrypt_exit(void)
  283. {
  284. crypto_unregister_template(&pcrypt_tmpl);
  285. padata_free(pencrypt);
  286. padata_free(pdecrypt);
  287. kset_unregister(pcrypt_kset);
  288. }
  289. subsys_initcall(pcrypt_init);
  290. module_exit(pcrypt_exit);
  291. MODULE_LICENSE("GPL");
  292. MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
  293. MODULE_DESCRIPTION("Parallel crypto wrapper");
  294. MODULE_ALIAS_CRYPTO("pcrypt");