omap-aes.c 31 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Cryptographic API.
  4. *
  5. * Support for OMAP AES HW acceleration.
  6. *
  7. * Copyright (c) 2010 Nokia Corporation
  8. * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
  9. * Copyright (c) 2011 Texas Instruments Incorporated
  10. */
  11. #define pr_fmt(fmt) "%20s: " fmt, __func__
  12. #define prn(num) pr_debug(#num "=%d\n", num)
  13. #define prx(num) pr_debug(#num "=%x\n", num)
  14. #include <linux/err.h>
  15. #include <linux/module.h>
  16. #include <linux/init.h>
  17. #include <linux/errno.h>
  18. #include <linux/kernel.h>
  19. #include <linux/platform_device.h>
  20. #include <linux/scatterlist.h>
  21. #include <linux/dma-mapping.h>
  22. #include <linux/dmaengine.h>
  23. #include <linux/pm_runtime.h>
  24. #include <linux/of.h>
  25. #include <linux/of_device.h>
  26. #include <linux/of_address.h>
  27. #include <linux/io.h>
  28. #include <linux/crypto.h>
  29. #include <linux/interrupt.h>
  30. #include <crypto/scatterwalk.h>
  31. #include <crypto/aes.h>
  32. #include <crypto/gcm.h>
  33. #include <crypto/engine.h>
  34. #include <crypto/internal/skcipher.h>
  35. #include <crypto/internal/aead.h>
  36. #include "omap-crypto.h"
  37. #include "omap-aes.h"
  38. /* keep registered devices data here */
  39. static LIST_HEAD(dev_list);
  40. static DEFINE_SPINLOCK(list_lock);
  41. static int aes_fallback_sz = 200;
  42. #ifdef DEBUG
  43. #define omap_aes_read(dd, offset) \
  44. ({ \
  45. int _read_ret; \
  46. _read_ret = __raw_readl(dd->io_base + offset); \
  47. pr_debug("omap_aes_read(" #offset "=%#x)= %#x\n", \
  48. offset, _read_ret); \
  49. _read_ret; \
  50. })
  51. #else
  52. inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
  53. {
  54. return __raw_readl(dd->io_base + offset);
  55. }
  56. #endif
  57. #ifdef DEBUG
  58. #define omap_aes_write(dd, offset, value) \
  59. do { \
  60. pr_debug("omap_aes_write(" #offset "=%#x) value=%#x\n", \
  61. offset, value); \
  62. __raw_writel(value, dd->io_base + offset); \
  63. } while (0)
  64. #else
  65. inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,
  66. u32 value)
  67. {
  68. __raw_writel(value, dd->io_base + offset);
  69. }
  70. #endif
  71. static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset,
  72. u32 value, u32 mask)
  73. {
  74. u32 val;
  75. val = omap_aes_read(dd, offset);
  76. val &= ~mask;
  77. val |= value;
  78. omap_aes_write(dd, offset, val);
  79. }
  80. static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset,
  81. u32 *value, int count)
  82. {
  83. for (; count--; value++, offset += 4)
  84. omap_aes_write(dd, offset, *value);
  85. }
  86. static int omap_aes_hw_init(struct omap_aes_dev *dd)
  87. {
  88. int err;
  89. if (!(dd->flags & FLAGS_INIT)) {
  90. dd->flags |= FLAGS_INIT;
  91. dd->err = 0;
  92. }
  93. err = pm_runtime_resume_and_get(dd->dev);
  94. if (err < 0) {
  95. dev_err(dd->dev, "failed to get sync: %d\n", err);
  96. return err;
  97. }
  98. return 0;
  99. }
  100. void omap_aes_clear_copy_flags(struct omap_aes_dev *dd)
  101. {
  102. dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_IN_DATA_ST_SHIFT);
  103. dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_OUT_DATA_ST_SHIFT);
  104. dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_ASSOC_DATA_ST_SHIFT);
  105. }
  106. int omap_aes_write_ctrl(struct omap_aes_dev *dd)
  107. {
  108. struct omap_aes_reqctx *rctx;
  109. unsigned int key32;
  110. int i, err;
  111. u32 val;
  112. err = omap_aes_hw_init(dd);
  113. if (err)
  114. return err;
  115. key32 = dd->ctx->keylen / sizeof(u32);
  116. /* RESET the key as previous HASH keys should not get affected*/
  117. if (dd->flags & FLAGS_GCM)
  118. for (i = 0; i < 0x40; i = i + 4)
  119. omap_aes_write(dd, i, 0x0);
  120. for (i = 0; i < key32; i++) {
  121. omap_aes_write(dd, AES_REG_KEY(dd, i),
  122. (__force u32)cpu_to_le32(dd->ctx->key[i]));
  123. }
  124. if ((dd->flags & (FLAGS_CBC | FLAGS_CTR)) && dd->req->iv)
  125. omap_aes_write_n(dd, AES_REG_IV(dd, 0), (void *)dd->req->iv, 4);
  126. if ((dd->flags & (FLAGS_GCM)) && dd->aead_req->iv) {
  127. rctx = aead_request_ctx(dd->aead_req);
  128. omap_aes_write_n(dd, AES_REG_IV(dd, 0), (u32 *)rctx->iv, 4);
  129. }
  130. val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3);
  131. if (dd->flags & FLAGS_CBC)
  132. val |= AES_REG_CTRL_CBC;
  133. if (dd->flags & (FLAGS_CTR | FLAGS_GCM))
  134. val |= AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_128;
  135. if (dd->flags & FLAGS_GCM)
  136. val |= AES_REG_CTRL_GCM;
  137. if (dd->flags & FLAGS_ENCRYPT)
  138. val |= AES_REG_CTRL_DIRECTION;
  139. omap_aes_write_mask(dd, AES_REG_CTRL(dd), val, AES_REG_CTRL_MASK);
  140. return 0;
  141. }
  142. static void omap_aes_dma_trigger_omap2(struct omap_aes_dev *dd, int length)
  143. {
  144. u32 mask, val;
  145. val = dd->pdata->dma_start;
  146. if (dd->dma_lch_out != NULL)
  147. val |= dd->pdata->dma_enable_out;
  148. if (dd->dma_lch_in != NULL)
  149. val |= dd->pdata->dma_enable_in;
  150. mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
  151. dd->pdata->dma_start;
  152. omap_aes_write_mask(dd, AES_REG_MASK(dd), val, mask);
  153. }
  154. static void omap_aes_dma_trigger_omap4(struct omap_aes_dev *dd, int length)
  155. {
  156. omap_aes_write(dd, AES_REG_LENGTH_N(0), length);
  157. omap_aes_write(dd, AES_REG_LENGTH_N(1), 0);
  158. if (dd->flags & FLAGS_GCM)
  159. omap_aes_write(dd, AES_REG_A_LEN, dd->assoc_len);
  160. omap_aes_dma_trigger_omap2(dd, length);
  161. }
  162. static void omap_aes_dma_stop(struct omap_aes_dev *dd)
  163. {
  164. u32 mask;
  165. mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
  166. dd->pdata->dma_start;
  167. omap_aes_write_mask(dd, AES_REG_MASK(dd), 0, mask);
  168. }
  169. struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_reqctx *rctx)
  170. {
  171. struct omap_aes_dev *dd;
  172. spin_lock_bh(&list_lock);
  173. dd = list_first_entry(&dev_list, struct omap_aes_dev, list);
  174. list_move_tail(&dd->list, &dev_list);
  175. rctx->dd = dd;
  176. spin_unlock_bh(&list_lock);
  177. return dd;
  178. }
  179. static void omap_aes_dma_out_callback(void *data)
  180. {
  181. struct omap_aes_dev *dd = data;
  182. /* dma_lch_out - completed */
  183. tasklet_schedule(&dd->done_task);
  184. }
  185. static int omap_aes_dma_init(struct omap_aes_dev *dd)
  186. {
  187. int err;
  188. dd->dma_lch_out = NULL;
  189. dd->dma_lch_in = NULL;
  190. dd->dma_lch_in = dma_request_chan(dd->dev, "rx");
  191. if (IS_ERR(dd->dma_lch_in)) {
  192. dev_err(dd->dev, "Unable to request in DMA channel\n");
  193. return PTR_ERR(dd->dma_lch_in);
  194. }
  195. dd->dma_lch_out = dma_request_chan(dd->dev, "tx");
  196. if (IS_ERR(dd->dma_lch_out)) {
  197. dev_err(dd->dev, "Unable to request out DMA channel\n");
  198. err = PTR_ERR(dd->dma_lch_out);
  199. goto err_dma_out;
  200. }
  201. return 0;
  202. err_dma_out:
  203. dma_release_channel(dd->dma_lch_in);
  204. return err;
  205. }
  206. static void omap_aes_dma_cleanup(struct omap_aes_dev *dd)
  207. {
  208. if (dd->pio_only)
  209. return;
  210. dma_release_channel(dd->dma_lch_out);
  211. dma_release_channel(dd->dma_lch_in);
  212. }
  213. static int omap_aes_crypt_dma(struct omap_aes_dev *dd,
  214. struct scatterlist *in_sg,
  215. struct scatterlist *out_sg,
  216. int in_sg_len, int out_sg_len)
  217. {
  218. struct dma_async_tx_descriptor *tx_in, *tx_out = NULL, *cb_desc;
  219. struct dma_slave_config cfg;
  220. int ret;
  221. if (dd->pio_only) {
  222. scatterwalk_start(&dd->in_walk, dd->in_sg);
  223. if (out_sg_len)
  224. scatterwalk_start(&dd->out_walk, dd->out_sg);
  225. /* Enable DATAIN interrupt and let it take
  226. care of the rest */
  227. omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);
  228. return 0;
  229. }
  230. dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE);
  231. memset(&cfg, 0, sizeof(cfg));
  232. cfg.src_addr = dd->phys_base + AES_REG_DATA_N(dd, 0);
  233. cfg.dst_addr = dd->phys_base + AES_REG_DATA_N(dd, 0);
  234. cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
  235. cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
  236. cfg.src_maxburst = DST_MAXBURST;
  237. cfg.dst_maxburst = DST_MAXBURST;
  238. /* IN */
  239. ret = dmaengine_slave_config(dd->dma_lch_in, &cfg);
  240. if (ret) {
  241. dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
  242. ret);
  243. return ret;
  244. }
  245. tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len,
  246. DMA_MEM_TO_DEV,
  247. DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
  248. if (!tx_in) {
  249. dev_err(dd->dev, "IN prep_slave_sg() failed\n");
  250. return -EINVAL;
  251. }
  252. /* No callback necessary */
  253. tx_in->callback_param = dd;
  254. tx_in->callback = NULL;
  255. /* OUT */
  256. if (out_sg_len) {
  257. ret = dmaengine_slave_config(dd->dma_lch_out, &cfg);
  258. if (ret) {
  259. dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
  260. ret);
  261. return ret;
  262. }
  263. tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg,
  264. out_sg_len,
  265. DMA_DEV_TO_MEM,
  266. DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
  267. if (!tx_out) {
  268. dev_err(dd->dev, "OUT prep_slave_sg() failed\n");
  269. return -EINVAL;
  270. }
  271. cb_desc = tx_out;
  272. } else {
  273. cb_desc = tx_in;
  274. }
  275. if (dd->flags & FLAGS_GCM)
  276. cb_desc->callback = omap_aes_gcm_dma_out_callback;
  277. else
  278. cb_desc->callback = omap_aes_dma_out_callback;
  279. cb_desc->callback_param = dd;
  280. dmaengine_submit(tx_in);
  281. if (tx_out)
  282. dmaengine_submit(tx_out);
  283. dma_async_issue_pending(dd->dma_lch_in);
  284. if (out_sg_len)
  285. dma_async_issue_pending(dd->dma_lch_out);
  286. /* start DMA */
  287. dd->pdata->trigger(dd, dd->total);
  288. return 0;
  289. }
  290. int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
  291. {
  292. int err;
  293. pr_debug("total: %zu\n", dd->total);
  294. if (!dd->pio_only) {
  295. err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len,
  296. DMA_TO_DEVICE);
  297. if (!err) {
  298. dev_err(dd->dev, "dma_map_sg() error\n");
  299. return -EINVAL;
  300. }
  301. if (dd->out_sg_len) {
  302. err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len,
  303. DMA_FROM_DEVICE);
  304. if (!err) {
  305. dev_err(dd->dev, "dma_map_sg() error\n");
  306. return -EINVAL;
  307. }
  308. }
  309. }
  310. err = omap_aes_crypt_dma(dd, dd->in_sg, dd->out_sg, dd->in_sg_len,
  311. dd->out_sg_len);
  312. if (err && !dd->pio_only) {
  313. dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
  314. if (dd->out_sg_len)
  315. dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
  316. DMA_FROM_DEVICE);
  317. }
  318. return err;
  319. }
  320. static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
  321. {
  322. struct skcipher_request *req = dd->req;
  323. pr_debug("err: %d\n", err);
  324. crypto_finalize_skcipher_request(dd->engine, req, err);
  325. pm_runtime_mark_last_busy(dd->dev);
  326. pm_runtime_put_autosuspend(dd->dev);
  327. }
  328. int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
  329. {
  330. pr_debug("total: %zu\n", dd->total);
  331. omap_aes_dma_stop(dd);
  332. return 0;
  333. }
  334. static int omap_aes_handle_queue(struct omap_aes_dev *dd,
  335. struct skcipher_request *req)
  336. {
  337. if (req)
  338. return crypto_transfer_skcipher_request_to_engine(dd->engine, req);
  339. return 0;
  340. }
  341. static int omap_aes_prepare_req(struct crypto_engine *engine,
  342. void *areq)
  343. {
  344. struct skcipher_request *req = container_of(areq, struct skcipher_request, base);
  345. struct omap_aes_ctx *ctx = crypto_skcipher_ctx(
  346. crypto_skcipher_reqtfm(req));
  347. struct omap_aes_reqctx *rctx = skcipher_request_ctx(req);
  348. struct omap_aes_dev *dd = rctx->dd;
  349. int ret;
  350. u16 flags;
  351. if (!dd)
  352. return -ENODEV;
  353. /* assign new request to device */
  354. dd->req = req;
  355. dd->total = req->cryptlen;
  356. dd->total_save = req->cryptlen;
  357. dd->in_sg = req->src;
  358. dd->out_sg = req->dst;
  359. dd->orig_out = req->dst;
  360. flags = OMAP_CRYPTO_COPY_DATA;
  361. if (req->src == req->dst)
  362. flags |= OMAP_CRYPTO_FORCE_COPY;
  363. ret = omap_crypto_align_sg(&dd->in_sg, dd->total, AES_BLOCK_SIZE,
  364. dd->in_sgl, flags,
  365. FLAGS_IN_DATA_ST_SHIFT, &dd->flags);
  366. if (ret)
  367. return ret;
  368. ret = omap_crypto_align_sg(&dd->out_sg, dd->total, AES_BLOCK_SIZE,
  369. &dd->out_sgl, 0,
  370. FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
  371. if (ret)
  372. return ret;
  373. dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total);
  374. if (dd->in_sg_len < 0)
  375. return dd->in_sg_len;
  376. dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total);
  377. if (dd->out_sg_len < 0)
  378. return dd->out_sg_len;
  379. rctx->mode &= FLAGS_MODE_MASK;
  380. dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
  381. dd->ctx = ctx;
  382. rctx->dd = dd;
  383. return omap_aes_write_ctrl(dd);
  384. }
  385. static int omap_aes_crypt_req(struct crypto_engine *engine,
  386. void *areq)
  387. {
  388. struct skcipher_request *req = container_of(areq, struct skcipher_request, base);
  389. struct omap_aes_reqctx *rctx = skcipher_request_ctx(req);
  390. struct omap_aes_dev *dd = rctx->dd;
  391. if (!dd)
  392. return -ENODEV;
  393. return omap_aes_crypt_dma_start(dd);
  394. }
  395. static void omap_aes_copy_ivout(struct omap_aes_dev *dd, u8 *ivbuf)
  396. {
  397. int i;
  398. for (i = 0; i < 4; i++)
  399. ((u32 *)ivbuf)[i] = omap_aes_read(dd, AES_REG_IV(dd, i));
  400. }
  401. static void omap_aes_done_task(unsigned long data)
  402. {
  403. struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
  404. pr_debug("enter done_task\n");
  405. if (!dd->pio_only) {
  406. dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
  407. DMA_FROM_DEVICE);
  408. dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
  409. dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
  410. DMA_FROM_DEVICE);
  411. omap_aes_crypt_dma_stop(dd);
  412. }
  413. omap_crypto_cleanup(dd->in_sg, NULL, 0, dd->total_save,
  414. FLAGS_IN_DATA_ST_SHIFT, dd->flags);
  415. omap_crypto_cleanup(dd->out_sg, dd->orig_out, 0, dd->total_save,
  416. FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
  417. /* Update IV output */
  418. if (dd->flags & (FLAGS_CBC | FLAGS_CTR))
  419. omap_aes_copy_ivout(dd, dd->req->iv);
  420. omap_aes_finish_req(dd, 0);
  421. pr_debug("exit\n");
  422. }
  423. static int omap_aes_crypt(struct skcipher_request *req, unsigned long mode)
  424. {
  425. struct omap_aes_ctx *ctx = crypto_skcipher_ctx(
  426. crypto_skcipher_reqtfm(req));
  427. struct omap_aes_reqctx *rctx = skcipher_request_ctx(req);
  428. struct omap_aes_dev *dd;
  429. int ret;
  430. if ((req->cryptlen % AES_BLOCK_SIZE) && !(mode & FLAGS_CTR))
  431. return -EINVAL;
  432. pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->cryptlen,
  433. !!(mode & FLAGS_ENCRYPT),
  434. !!(mode & FLAGS_CBC));
  435. if (req->cryptlen < aes_fallback_sz) {
  436. skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
  437. skcipher_request_set_callback(&rctx->fallback_req,
  438. req->base.flags,
  439. req->base.complete,
  440. req->base.data);
  441. skcipher_request_set_crypt(&rctx->fallback_req, req->src,
  442. req->dst, req->cryptlen, req->iv);
  443. if (mode & FLAGS_ENCRYPT)
  444. ret = crypto_skcipher_encrypt(&rctx->fallback_req);
  445. else
  446. ret = crypto_skcipher_decrypt(&rctx->fallback_req);
  447. return ret;
  448. }
  449. dd = omap_aes_find_dev(rctx);
  450. if (!dd)
  451. return -ENODEV;
  452. rctx->mode = mode;
  453. return omap_aes_handle_queue(dd, req);
  454. }
  455. /* ********************** ALG API ************************************ */
  456. static int omap_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
  457. unsigned int keylen)
  458. {
  459. struct omap_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
  460. int ret;
  461. if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
  462. keylen != AES_KEYSIZE_256)
  463. return -EINVAL;
  464. pr_debug("enter, keylen: %d\n", keylen);
  465. memcpy(ctx->key, key, keylen);
  466. ctx->keylen = keylen;
  467. crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
  468. crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
  469. CRYPTO_TFM_REQ_MASK);
  470. ret = crypto_skcipher_setkey(ctx->fallback, key, keylen);
  471. if (!ret)
  472. return 0;
  473. return 0;
  474. }
  475. static int omap_aes_ecb_encrypt(struct skcipher_request *req)
  476. {
  477. return omap_aes_crypt(req, FLAGS_ENCRYPT);
  478. }
  479. static int omap_aes_ecb_decrypt(struct skcipher_request *req)
  480. {
  481. return omap_aes_crypt(req, 0);
  482. }
  483. static int omap_aes_cbc_encrypt(struct skcipher_request *req)
  484. {
  485. return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
  486. }
  487. static int omap_aes_cbc_decrypt(struct skcipher_request *req)
  488. {
  489. return omap_aes_crypt(req, FLAGS_CBC);
  490. }
  491. static int omap_aes_ctr_encrypt(struct skcipher_request *req)
  492. {
  493. return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CTR);
  494. }
  495. static int omap_aes_ctr_decrypt(struct skcipher_request *req)
  496. {
  497. return omap_aes_crypt(req, FLAGS_CTR);
  498. }
  499. static int omap_aes_prepare_req(struct crypto_engine *engine,
  500. void *req);
  501. static int omap_aes_crypt_req(struct crypto_engine *engine,
  502. void *req);
  503. static int omap_aes_init_tfm(struct crypto_skcipher *tfm)
  504. {
  505. const char *name = crypto_tfm_alg_name(&tfm->base);
  506. struct omap_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
  507. struct crypto_skcipher *blk;
  508. blk = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
  509. if (IS_ERR(blk))
  510. return PTR_ERR(blk);
  511. ctx->fallback = blk;
  512. crypto_skcipher_set_reqsize(tfm, sizeof(struct omap_aes_reqctx) +
  513. crypto_skcipher_reqsize(blk));
  514. ctx->enginectx.op.prepare_request = omap_aes_prepare_req;
  515. ctx->enginectx.op.unprepare_request = NULL;
  516. ctx->enginectx.op.do_one_request = omap_aes_crypt_req;
  517. return 0;
  518. }
  519. static void omap_aes_exit_tfm(struct crypto_skcipher *tfm)
  520. {
  521. struct omap_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
  522. if (ctx->fallback)
  523. crypto_free_skcipher(ctx->fallback);
  524. ctx->fallback = NULL;
  525. }
  526. /* ********************** ALGS ************************************ */
  527. static struct skcipher_alg algs_ecb_cbc[] = {
  528. {
  529. .base.cra_name = "ecb(aes)",
  530. .base.cra_driver_name = "ecb-aes-omap",
  531. .base.cra_priority = 300,
  532. .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  533. CRYPTO_ALG_ASYNC |
  534. CRYPTO_ALG_NEED_FALLBACK,
  535. .base.cra_blocksize = AES_BLOCK_SIZE,
  536. .base.cra_ctxsize = sizeof(struct omap_aes_ctx),
  537. .base.cra_module = THIS_MODULE,
  538. .min_keysize = AES_MIN_KEY_SIZE,
  539. .max_keysize = AES_MAX_KEY_SIZE,
  540. .setkey = omap_aes_setkey,
  541. .encrypt = omap_aes_ecb_encrypt,
  542. .decrypt = omap_aes_ecb_decrypt,
  543. .init = omap_aes_init_tfm,
  544. .exit = omap_aes_exit_tfm,
  545. },
  546. {
  547. .base.cra_name = "cbc(aes)",
  548. .base.cra_driver_name = "cbc-aes-omap",
  549. .base.cra_priority = 300,
  550. .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  551. CRYPTO_ALG_ASYNC |
  552. CRYPTO_ALG_NEED_FALLBACK,
  553. .base.cra_blocksize = AES_BLOCK_SIZE,
  554. .base.cra_ctxsize = sizeof(struct omap_aes_ctx),
  555. .base.cra_module = THIS_MODULE,
  556. .min_keysize = AES_MIN_KEY_SIZE,
  557. .max_keysize = AES_MAX_KEY_SIZE,
  558. .ivsize = AES_BLOCK_SIZE,
  559. .setkey = omap_aes_setkey,
  560. .encrypt = omap_aes_cbc_encrypt,
  561. .decrypt = omap_aes_cbc_decrypt,
  562. .init = omap_aes_init_tfm,
  563. .exit = omap_aes_exit_tfm,
  564. }
  565. };
  566. static struct skcipher_alg algs_ctr[] = {
  567. {
  568. .base.cra_name = "ctr(aes)",
  569. .base.cra_driver_name = "ctr-aes-omap",
  570. .base.cra_priority = 300,
  571. .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  572. CRYPTO_ALG_ASYNC |
  573. CRYPTO_ALG_NEED_FALLBACK,
  574. .base.cra_blocksize = 1,
  575. .base.cra_ctxsize = sizeof(struct omap_aes_ctx),
  576. .base.cra_module = THIS_MODULE,
  577. .min_keysize = AES_MIN_KEY_SIZE,
  578. .max_keysize = AES_MAX_KEY_SIZE,
  579. .ivsize = AES_BLOCK_SIZE,
  580. .setkey = omap_aes_setkey,
  581. .encrypt = omap_aes_ctr_encrypt,
  582. .decrypt = omap_aes_ctr_decrypt,
  583. .init = omap_aes_init_tfm,
  584. .exit = omap_aes_exit_tfm,
  585. }
  586. };
  587. static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc[] = {
  588. {
  589. .algs_list = algs_ecb_cbc,
  590. .size = ARRAY_SIZE(algs_ecb_cbc),
  591. },
  592. };
  593. static struct aead_alg algs_aead_gcm[] = {
  594. {
  595. .base = {
  596. .cra_name = "gcm(aes)",
  597. .cra_driver_name = "gcm-aes-omap",
  598. .cra_priority = 300,
  599. .cra_flags = CRYPTO_ALG_ASYNC |
  600. CRYPTO_ALG_KERN_DRIVER_ONLY,
  601. .cra_blocksize = 1,
  602. .cra_ctxsize = sizeof(struct omap_aes_gcm_ctx),
  603. .cra_alignmask = 0xf,
  604. .cra_module = THIS_MODULE,
  605. },
  606. .init = omap_aes_gcm_cra_init,
  607. .ivsize = GCM_AES_IV_SIZE,
  608. .maxauthsize = AES_BLOCK_SIZE,
  609. .setkey = omap_aes_gcm_setkey,
  610. .setauthsize = omap_aes_gcm_setauthsize,
  611. .encrypt = omap_aes_gcm_encrypt,
  612. .decrypt = omap_aes_gcm_decrypt,
  613. },
  614. {
  615. .base = {
  616. .cra_name = "rfc4106(gcm(aes))",
  617. .cra_driver_name = "rfc4106-gcm-aes-omap",
  618. .cra_priority = 300,
  619. .cra_flags = CRYPTO_ALG_ASYNC |
  620. CRYPTO_ALG_KERN_DRIVER_ONLY,
  621. .cra_blocksize = 1,
  622. .cra_ctxsize = sizeof(struct omap_aes_gcm_ctx),
  623. .cra_alignmask = 0xf,
  624. .cra_module = THIS_MODULE,
  625. },
  626. .init = omap_aes_gcm_cra_init,
  627. .maxauthsize = AES_BLOCK_SIZE,
  628. .ivsize = GCM_RFC4106_IV_SIZE,
  629. .setkey = omap_aes_4106gcm_setkey,
  630. .setauthsize = omap_aes_4106gcm_setauthsize,
  631. .encrypt = omap_aes_4106gcm_encrypt,
  632. .decrypt = omap_aes_4106gcm_decrypt,
  633. },
  634. };
  635. static struct omap_aes_aead_algs omap_aes_aead_info = {
  636. .algs_list = algs_aead_gcm,
  637. .size = ARRAY_SIZE(algs_aead_gcm),
  638. };
  639. static const struct omap_aes_pdata omap_aes_pdata_omap2 = {
  640. .algs_info = omap_aes_algs_info_ecb_cbc,
  641. .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc),
  642. .trigger = omap_aes_dma_trigger_omap2,
  643. .key_ofs = 0x1c,
  644. .iv_ofs = 0x20,
  645. .ctrl_ofs = 0x30,
  646. .data_ofs = 0x34,
  647. .rev_ofs = 0x44,
  648. .mask_ofs = 0x48,
  649. .dma_enable_in = BIT(2),
  650. .dma_enable_out = BIT(3),
  651. .dma_start = BIT(5),
  652. .major_mask = 0xf0,
  653. .major_shift = 4,
  654. .minor_mask = 0x0f,
  655. .minor_shift = 0,
  656. };
  657. #ifdef CONFIG_OF
  658. static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc_ctr[] = {
  659. {
  660. .algs_list = algs_ecb_cbc,
  661. .size = ARRAY_SIZE(algs_ecb_cbc),
  662. },
  663. {
  664. .algs_list = algs_ctr,
  665. .size = ARRAY_SIZE(algs_ctr),
  666. },
  667. };
  668. static const struct omap_aes_pdata omap_aes_pdata_omap3 = {
  669. .algs_info = omap_aes_algs_info_ecb_cbc_ctr,
  670. .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr),
  671. .trigger = omap_aes_dma_trigger_omap2,
  672. .key_ofs = 0x1c,
  673. .iv_ofs = 0x20,
  674. .ctrl_ofs = 0x30,
  675. .data_ofs = 0x34,
  676. .rev_ofs = 0x44,
  677. .mask_ofs = 0x48,
  678. .dma_enable_in = BIT(2),
  679. .dma_enable_out = BIT(3),
  680. .dma_start = BIT(5),
  681. .major_mask = 0xf0,
  682. .major_shift = 4,
  683. .minor_mask = 0x0f,
  684. .minor_shift = 0,
  685. };
  686. static const struct omap_aes_pdata omap_aes_pdata_omap4 = {
  687. .algs_info = omap_aes_algs_info_ecb_cbc_ctr,
  688. .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr),
  689. .aead_algs_info = &omap_aes_aead_info,
  690. .trigger = omap_aes_dma_trigger_omap4,
  691. .key_ofs = 0x3c,
  692. .iv_ofs = 0x40,
  693. .ctrl_ofs = 0x50,
  694. .data_ofs = 0x60,
  695. .rev_ofs = 0x80,
  696. .mask_ofs = 0x84,
  697. .irq_status_ofs = 0x8c,
  698. .irq_enable_ofs = 0x90,
  699. .dma_enable_in = BIT(5),
  700. .dma_enable_out = BIT(6),
  701. .major_mask = 0x0700,
  702. .major_shift = 8,
  703. .minor_mask = 0x003f,
  704. .minor_shift = 0,
  705. };
  706. static irqreturn_t omap_aes_irq(int irq, void *dev_id)
  707. {
  708. struct omap_aes_dev *dd = dev_id;
  709. u32 status, i;
  710. u32 *src, *dst;
  711. status = omap_aes_read(dd, AES_REG_IRQ_STATUS(dd));
  712. if (status & AES_REG_IRQ_DATA_IN) {
  713. omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);
  714. BUG_ON(!dd->in_sg);
  715. BUG_ON(_calc_walked(in) > dd->in_sg->length);
  716. src = sg_virt(dd->in_sg) + _calc_walked(in);
  717. for (i = 0; i < AES_BLOCK_WORDS; i++) {
  718. omap_aes_write(dd, AES_REG_DATA_N(dd, i), *src);
  719. scatterwalk_advance(&dd->in_walk, 4);
  720. if (dd->in_sg->length == _calc_walked(in)) {
  721. dd->in_sg = sg_next(dd->in_sg);
  722. if (dd->in_sg) {
  723. scatterwalk_start(&dd->in_walk,
  724. dd->in_sg);
  725. src = sg_virt(dd->in_sg) +
  726. _calc_walked(in);
  727. }
  728. } else {
  729. src++;
  730. }
  731. }
  732. /* Clear IRQ status */
  733. status &= ~AES_REG_IRQ_DATA_IN;
  734. omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);
  735. /* Enable DATA_OUT interrupt */
  736. omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x4);
  737. } else if (status & AES_REG_IRQ_DATA_OUT) {
  738. omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);
  739. BUG_ON(!dd->out_sg);
  740. BUG_ON(_calc_walked(out) > dd->out_sg->length);
  741. dst = sg_virt(dd->out_sg) + _calc_walked(out);
  742. for (i = 0; i < AES_BLOCK_WORDS; i++) {
  743. *dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i));
  744. scatterwalk_advance(&dd->out_walk, 4);
  745. if (dd->out_sg->length == _calc_walked(out)) {
  746. dd->out_sg = sg_next(dd->out_sg);
  747. if (dd->out_sg) {
  748. scatterwalk_start(&dd->out_walk,
  749. dd->out_sg);
  750. dst = sg_virt(dd->out_sg) +
  751. _calc_walked(out);
  752. }
  753. } else {
  754. dst++;
  755. }
  756. }
  757. dd->total -= min_t(size_t, AES_BLOCK_SIZE, dd->total);
  758. /* Clear IRQ status */
  759. status &= ~AES_REG_IRQ_DATA_OUT;
  760. omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);
  761. if (!dd->total)
  762. /* All bytes read! */
  763. tasklet_schedule(&dd->done_task);
  764. else
  765. /* Enable DATA_IN interrupt for next block */
  766. omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);
  767. }
  768. return IRQ_HANDLED;
  769. }
  770. static const struct of_device_id omap_aes_of_match[] = {
  771. {
  772. .compatible = "ti,omap2-aes",
  773. .data = &omap_aes_pdata_omap2,
  774. },
  775. {
  776. .compatible = "ti,omap3-aes",
  777. .data = &omap_aes_pdata_omap3,
  778. },
  779. {
  780. .compatible = "ti,omap4-aes",
  781. .data = &omap_aes_pdata_omap4,
  782. },
  783. {},
  784. };
  785. MODULE_DEVICE_TABLE(of, omap_aes_of_match);
  786. static int omap_aes_get_res_of(struct omap_aes_dev *dd,
  787. struct device *dev, struct resource *res)
  788. {
  789. struct device_node *node = dev->of_node;
  790. int err = 0;
  791. dd->pdata = of_device_get_match_data(dev);
  792. if (!dd->pdata) {
  793. dev_err(dev, "no compatible OF match\n");
  794. err = -EINVAL;
  795. goto err;
  796. }
  797. err = of_address_to_resource(node, 0, res);
  798. if (err < 0) {
  799. dev_err(dev, "can't translate OF node address\n");
  800. err = -EINVAL;
  801. goto err;
  802. }
  803. err:
  804. return err;
  805. }
  806. #else
  807. static const struct of_device_id omap_aes_of_match[] = {
  808. {},
  809. };
  810. static int omap_aes_get_res_of(struct omap_aes_dev *dd,
  811. struct device *dev, struct resource *res)
  812. {
  813. return -EINVAL;
  814. }
  815. #endif
  816. static int omap_aes_get_res_pdev(struct omap_aes_dev *dd,
  817. struct platform_device *pdev, struct resource *res)
  818. {
  819. struct device *dev = &pdev->dev;
  820. struct resource *r;
  821. int err = 0;
  822. /* Get the base address */
  823. r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  824. if (!r) {
  825. dev_err(dev, "no MEM resource info\n");
  826. err = -ENODEV;
  827. goto err;
  828. }
  829. memcpy(res, r, sizeof(*res));
  830. /* Only OMAP2/3 can be non-DT */
  831. dd->pdata = &omap_aes_pdata_omap2;
  832. err:
  833. return err;
  834. }
  835. static ssize_t fallback_show(struct device *dev, struct device_attribute *attr,
  836. char *buf)
  837. {
  838. return sprintf(buf, "%d\n", aes_fallback_sz);
  839. }
  840. static ssize_t fallback_store(struct device *dev, struct device_attribute *attr,
  841. const char *buf, size_t size)
  842. {
  843. ssize_t status;
  844. long value;
  845. status = kstrtol(buf, 0, &value);
  846. if (status)
  847. return status;
  848. /* HW accelerator only works with buffers > 9 */
  849. if (value < 9) {
  850. dev_err(dev, "minimum fallback size 9\n");
  851. return -EINVAL;
  852. }
  853. aes_fallback_sz = value;
  854. return size;
  855. }
  856. static ssize_t queue_len_show(struct device *dev, struct device_attribute *attr,
  857. char *buf)
  858. {
  859. struct omap_aes_dev *dd = dev_get_drvdata(dev);
  860. return sprintf(buf, "%d\n", dd->engine->queue.max_qlen);
  861. }
  862. static ssize_t queue_len_store(struct device *dev,
  863. struct device_attribute *attr, const char *buf,
  864. size_t size)
  865. {
  866. struct omap_aes_dev *dd;
  867. ssize_t status;
  868. long value;
  869. unsigned long flags;
  870. status = kstrtol(buf, 0, &value);
  871. if (status)
  872. return status;
  873. if (value < 1)
  874. return -EINVAL;
  875. /*
  876. * Changing the queue size in fly is safe, if size becomes smaller
  877. * than current size, it will just not accept new entries until
  878. * it has shrank enough.
  879. */
  880. spin_lock_bh(&list_lock);
  881. list_for_each_entry(dd, &dev_list, list) {
  882. spin_lock_irqsave(&dd->lock, flags);
  883. dd->engine->queue.max_qlen = value;
  884. dd->aead_queue.base.max_qlen = value;
  885. spin_unlock_irqrestore(&dd->lock, flags);
  886. }
  887. spin_unlock_bh(&list_lock);
  888. return size;
  889. }
  890. static DEVICE_ATTR_RW(queue_len);
  891. static DEVICE_ATTR_RW(fallback);
  892. static struct attribute *omap_aes_attrs[] = {
  893. &dev_attr_queue_len.attr,
  894. &dev_attr_fallback.attr,
  895. NULL,
  896. };
  897. static struct attribute_group omap_aes_attr_group = {
  898. .attrs = omap_aes_attrs,
  899. };
  900. static int omap_aes_probe(struct platform_device *pdev)
  901. {
  902. struct device *dev = &pdev->dev;
  903. struct omap_aes_dev *dd;
  904. struct skcipher_alg *algp;
  905. struct aead_alg *aalg;
  906. struct resource res;
  907. int err = -ENOMEM, i, j, irq = -1;
  908. u32 reg;
  909. dd = devm_kzalloc(dev, sizeof(struct omap_aes_dev), GFP_KERNEL);
  910. if (dd == NULL) {
  911. dev_err(dev, "unable to alloc data struct.\n");
  912. goto err_data;
  913. }
  914. dd->dev = dev;
  915. platform_set_drvdata(pdev, dd);
  916. aead_init_queue(&dd->aead_queue, OMAP_AES_QUEUE_LENGTH);
  917. err = (dev->of_node) ? omap_aes_get_res_of(dd, dev, &res) :
  918. omap_aes_get_res_pdev(dd, pdev, &res);
  919. if (err)
  920. goto err_res;
  921. dd->io_base = devm_ioremap_resource(dev, &res);
  922. if (IS_ERR(dd->io_base)) {
  923. err = PTR_ERR(dd->io_base);
  924. goto err_res;
  925. }
  926. dd->phys_base = res.start;
  927. pm_runtime_use_autosuspend(dev);
  928. pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY);
  929. pm_runtime_enable(dev);
  930. err = pm_runtime_resume_and_get(dev);
  931. if (err < 0) {
  932. dev_err(dev, "%s: failed to get_sync(%d)\n",
  933. __func__, err);
  934. goto err_pm_disable;
  935. }
  936. omap_aes_dma_stop(dd);
  937. reg = omap_aes_read(dd, AES_REG_REV(dd));
  938. pm_runtime_put_sync(dev);
  939. dev_info(dev, "OMAP AES hw accel rev: %u.%u\n",
  940. (reg & dd->pdata->major_mask) >> dd->pdata->major_shift,
  941. (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
  942. tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd);
  943. err = omap_aes_dma_init(dd);
  944. if (err == -EPROBE_DEFER) {
  945. goto err_irq;
  946. } else if (err && AES_REG_IRQ_STATUS(dd) && AES_REG_IRQ_ENABLE(dd)) {
  947. dd->pio_only = 1;
  948. irq = platform_get_irq(pdev, 0);
  949. if (irq < 0) {
  950. err = irq;
  951. goto err_irq;
  952. }
  953. err = devm_request_irq(dev, irq, omap_aes_irq, 0,
  954. dev_name(dev), dd);
  955. if (err) {
  956. dev_err(dev, "Unable to grab omap-aes IRQ\n");
  957. goto err_irq;
  958. }
  959. }
  960. spin_lock_init(&dd->lock);
  961. INIT_LIST_HEAD(&dd->list);
  962. spin_lock_bh(&list_lock);
  963. list_add_tail(&dd->list, &dev_list);
  964. spin_unlock_bh(&list_lock);
  965. /* Initialize crypto engine */
  966. dd->engine = crypto_engine_alloc_init(dev, 1);
  967. if (!dd->engine) {
  968. err = -ENOMEM;
  969. goto err_engine;
  970. }
  971. err = crypto_engine_start(dd->engine);
  972. if (err)
  973. goto err_engine;
  974. for (i = 0; i < dd->pdata->algs_info_size; i++) {
  975. if (!dd->pdata->algs_info[i].registered) {
  976. for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
  977. algp = &dd->pdata->algs_info[i].algs_list[j];
  978. pr_debug("reg alg: %s\n", algp->base.cra_name);
  979. err = crypto_register_skcipher(algp);
  980. if (err)
  981. goto err_algs;
  982. dd->pdata->algs_info[i].registered++;
  983. }
  984. }
  985. }
  986. if (dd->pdata->aead_algs_info &&
  987. !dd->pdata->aead_algs_info->registered) {
  988. for (i = 0; i < dd->pdata->aead_algs_info->size; i++) {
  989. aalg = &dd->pdata->aead_algs_info->algs_list[i];
  990. pr_debug("reg alg: %s\n", aalg->base.cra_name);
  991. err = crypto_register_aead(aalg);
  992. if (err)
  993. goto err_aead_algs;
  994. dd->pdata->aead_algs_info->registered++;
  995. }
  996. }
  997. err = sysfs_create_group(&dev->kobj, &omap_aes_attr_group);
  998. if (err) {
  999. dev_err(dev, "could not create sysfs device attrs\n");
  1000. goto err_aead_algs;
  1001. }
  1002. return 0;
  1003. err_aead_algs:
  1004. for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) {
  1005. aalg = &dd->pdata->aead_algs_info->algs_list[i];
  1006. crypto_unregister_aead(aalg);
  1007. }
  1008. err_algs:
  1009. for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
  1010. for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
  1011. crypto_unregister_skcipher(
  1012. &dd->pdata->algs_info[i].algs_list[j]);
  1013. err_engine:
  1014. if (dd->engine)
  1015. crypto_engine_exit(dd->engine);
  1016. omap_aes_dma_cleanup(dd);
  1017. err_irq:
  1018. tasklet_kill(&dd->done_task);
  1019. err_pm_disable:
  1020. pm_runtime_disable(dev);
  1021. err_res:
  1022. dd = NULL;
  1023. err_data:
  1024. dev_err(dev, "initialization failed.\n");
  1025. return err;
  1026. }
  1027. static int omap_aes_remove(struct platform_device *pdev)
  1028. {
  1029. struct omap_aes_dev *dd = platform_get_drvdata(pdev);
  1030. struct aead_alg *aalg;
  1031. int i, j;
  1032. if (!dd)
  1033. return -ENODEV;
  1034. spin_lock_bh(&list_lock);
  1035. list_del(&dd->list);
  1036. spin_unlock_bh(&list_lock);
  1037. for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
  1038. for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) {
  1039. crypto_unregister_skcipher(
  1040. &dd->pdata->algs_info[i].algs_list[j]);
  1041. dd->pdata->algs_info[i].registered--;
  1042. }
  1043. for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) {
  1044. aalg = &dd->pdata->aead_algs_info->algs_list[i];
  1045. crypto_unregister_aead(aalg);
  1046. dd->pdata->aead_algs_info->registered--;
  1047. }
  1048. crypto_engine_exit(dd->engine);
  1049. tasklet_kill(&dd->done_task);
  1050. omap_aes_dma_cleanup(dd);
  1051. pm_runtime_disable(dd->dev);
  1052. sysfs_remove_group(&dd->dev->kobj, &omap_aes_attr_group);
  1053. return 0;
  1054. }
  1055. #ifdef CONFIG_PM_SLEEP
  1056. static int omap_aes_suspend(struct device *dev)
  1057. {
  1058. pm_runtime_put_sync(dev);
  1059. return 0;
  1060. }
  1061. static int omap_aes_resume(struct device *dev)
  1062. {
  1063. pm_runtime_get_sync(dev);
  1064. return 0;
  1065. }
  1066. #endif
  1067. static SIMPLE_DEV_PM_OPS(omap_aes_pm_ops, omap_aes_suspend, omap_aes_resume);
  1068. static struct platform_driver omap_aes_driver = {
  1069. .probe = omap_aes_probe,
  1070. .remove = omap_aes_remove,
  1071. .driver = {
  1072. .name = "omap-aes",
  1073. .pm = &omap_aes_pm_ops,
  1074. .of_match_table = omap_aes_of_match,
  1075. },
  1076. };
  1077. module_platform_driver(omap_aes_driver);
  1078. MODULE_DESCRIPTION("OMAP AES hw acceleration support.");
  1079. MODULE_LICENSE("GPL v2");
  1080. MODULE_AUTHOR("Dmitry Kasatkin");