atmel-aes.c 65 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Cryptographic API.
  4. *
  5. * Support for ATMEL AES HW acceleration.
  6. *
  7. * Copyright (c) 2012 Eukréa Electromatique - ATMEL
  8. * Author: Nicolas Royer <nicolas@eukrea.com>
  9. *
  10. * Some ideas are from omap-aes.c driver.
  11. */
  12. #include <linux/kernel.h>
  13. #include <linux/module.h>
  14. #include <linux/slab.h>
  15. #include <linux/err.h>
  16. #include <linux/clk.h>
  17. #include <linux/io.h>
  18. #include <linux/hw_random.h>
  19. #include <linux/platform_device.h>
  20. #include <linux/device.h>
  21. #include <linux/dmaengine.h>
  22. #include <linux/init.h>
  23. #include <linux/errno.h>
  24. #include <linux/interrupt.h>
  25. #include <linux/irq.h>
  26. #include <linux/scatterlist.h>
  27. #include <linux/dma-mapping.h>
  28. #include <linux/of_device.h>
  29. #include <linux/delay.h>
  30. #include <linux/crypto.h>
  31. #include <crypto/scatterwalk.h>
  32. #include <crypto/algapi.h>
  33. #include <crypto/aes.h>
  34. #include <crypto/gcm.h>
  35. #include <crypto/xts.h>
  36. #include <crypto/internal/aead.h>
  37. #include <crypto/internal/skcipher.h>
  38. #include "atmel-aes-regs.h"
  39. #include "atmel-authenc.h"
  40. #define ATMEL_AES_PRIORITY 300
  41. #define ATMEL_AES_BUFFER_ORDER 2
  42. #define ATMEL_AES_BUFFER_SIZE (PAGE_SIZE << ATMEL_AES_BUFFER_ORDER)
  43. #define CFB8_BLOCK_SIZE 1
  44. #define CFB16_BLOCK_SIZE 2
  45. #define CFB32_BLOCK_SIZE 4
  46. #define CFB64_BLOCK_SIZE 8
  47. #define SIZE_IN_WORDS(x) ((x) >> 2)
  48. /* AES flags */
  49. /* Reserve bits [18:16] [14:12] [1:0] for mode (same as for AES_MR) */
  50. #define AES_FLAGS_ENCRYPT AES_MR_CYPHER_ENC
  51. #define AES_FLAGS_GTAGEN AES_MR_GTAGEN
  52. #define AES_FLAGS_OPMODE_MASK (AES_MR_OPMOD_MASK | AES_MR_CFBS_MASK)
  53. #define AES_FLAGS_ECB AES_MR_OPMOD_ECB
  54. #define AES_FLAGS_CBC AES_MR_OPMOD_CBC
  55. #define AES_FLAGS_OFB AES_MR_OPMOD_OFB
  56. #define AES_FLAGS_CFB128 (AES_MR_OPMOD_CFB | AES_MR_CFBS_128b)
  57. #define AES_FLAGS_CFB64 (AES_MR_OPMOD_CFB | AES_MR_CFBS_64b)
  58. #define AES_FLAGS_CFB32 (AES_MR_OPMOD_CFB | AES_MR_CFBS_32b)
  59. #define AES_FLAGS_CFB16 (AES_MR_OPMOD_CFB | AES_MR_CFBS_16b)
  60. #define AES_FLAGS_CFB8 (AES_MR_OPMOD_CFB | AES_MR_CFBS_8b)
  61. #define AES_FLAGS_CTR AES_MR_OPMOD_CTR
  62. #define AES_FLAGS_GCM AES_MR_OPMOD_GCM
  63. #define AES_FLAGS_XTS AES_MR_OPMOD_XTS
  64. #define AES_FLAGS_MODE_MASK (AES_FLAGS_OPMODE_MASK | \
  65. AES_FLAGS_ENCRYPT | \
  66. AES_FLAGS_GTAGEN)
  67. #define AES_FLAGS_BUSY BIT(3)
  68. #define AES_FLAGS_DUMP_REG BIT(4)
  69. #define AES_FLAGS_OWN_SHA BIT(5)
  70. #define AES_FLAGS_PERSISTENT AES_FLAGS_BUSY
  71. #define ATMEL_AES_QUEUE_LENGTH 50
  72. #define ATMEL_AES_DMA_THRESHOLD 256
  73. struct atmel_aes_caps {
  74. bool has_dualbuff;
  75. bool has_cfb64;
  76. bool has_gcm;
  77. bool has_xts;
  78. bool has_authenc;
  79. u32 max_burst_size;
  80. };
  81. struct atmel_aes_dev;
  82. typedef int (*atmel_aes_fn_t)(struct atmel_aes_dev *);
  83. struct atmel_aes_base_ctx {
  84. struct atmel_aes_dev *dd;
  85. atmel_aes_fn_t start;
  86. int keylen;
  87. u32 key[AES_KEYSIZE_256 / sizeof(u32)];
  88. u16 block_size;
  89. bool is_aead;
  90. };
  91. struct atmel_aes_ctx {
  92. struct atmel_aes_base_ctx base;
  93. };
  94. struct atmel_aes_ctr_ctx {
  95. struct atmel_aes_base_ctx base;
  96. __be32 iv[AES_BLOCK_SIZE / sizeof(u32)];
  97. size_t offset;
  98. struct scatterlist src[2];
  99. struct scatterlist dst[2];
  100. u32 blocks;
  101. };
  102. struct atmel_aes_gcm_ctx {
  103. struct atmel_aes_base_ctx base;
  104. struct scatterlist src[2];
  105. struct scatterlist dst[2];
  106. __be32 j0[AES_BLOCK_SIZE / sizeof(u32)];
  107. u32 tag[AES_BLOCK_SIZE / sizeof(u32)];
  108. __be32 ghash[AES_BLOCK_SIZE / sizeof(u32)];
  109. size_t textlen;
  110. const __be32 *ghash_in;
  111. __be32 *ghash_out;
  112. atmel_aes_fn_t ghash_resume;
  113. };
  114. struct atmel_aes_xts_ctx {
  115. struct atmel_aes_base_ctx base;
  116. u32 key2[AES_KEYSIZE_256 / sizeof(u32)];
  117. };
  118. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  119. struct atmel_aes_authenc_ctx {
  120. struct atmel_aes_base_ctx base;
  121. struct atmel_sha_authenc_ctx *auth;
  122. };
  123. #endif
  124. struct atmel_aes_reqctx {
  125. unsigned long mode;
  126. u8 lastc[AES_BLOCK_SIZE];
  127. };
  128. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  129. struct atmel_aes_authenc_reqctx {
  130. struct atmel_aes_reqctx base;
  131. struct scatterlist src[2];
  132. struct scatterlist dst[2];
  133. size_t textlen;
  134. u32 digest[SHA512_DIGEST_SIZE / sizeof(u32)];
  135. /* auth_req MUST be place last. */
  136. struct ahash_request auth_req;
  137. };
  138. #endif
  139. struct atmel_aes_dma {
  140. struct dma_chan *chan;
  141. struct scatterlist *sg;
  142. int nents;
  143. unsigned int remainder;
  144. unsigned int sg_len;
  145. };
  146. struct atmel_aes_dev {
  147. struct list_head list;
  148. unsigned long phys_base;
  149. void __iomem *io_base;
  150. struct crypto_async_request *areq;
  151. struct atmel_aes_base_ctx *ctx;
  152. bool is_async;
  153. atmel_aes_fn_t resume;
  154. atmel_aes_fn_t cpu_transfer_complete;
  155. struct device *dev;
  156. struct clk *iclk;
  157. int irq;
  158. unsigned long flags;
  159. spinlock_t lock;
  160. struct crypto_queue queue;
  161. struct tasklet_struct done_task;
  162. struct tasklet_struct queue_task;
  163. size_t total;
  164. size_t datalen;
  165. u32 *data;
  166. struct atmel_aes_dma src;
  167. struct atmel_aes_dma dst;
  168. size_t buflen;
  169. void *buf;
  170. struct scatterlist aligned_sg;
  171. struct scatterlist *real_dst;
  172. struct atmel_aes_caps caps;
  173. u32 hw_version;
  174. };
  175. struct atmel_aes_drv {
  176. struct list_head dev_list;
  177. spinlock_t lock;
  178. };
  179. static struct atmel_aes_drv atmel_aes = {
  180. .dev_list = LIST_HEAD_INIT(atmel_aes.dev_list),
  181. .lock = __SPIN_LOCK_UNLOCKED(atmel_aes.lock),
  182. };
  183. #ifdef VERBOSE_DEBUG
  184. static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz)
  185. {
  186. switch (offset) {
  187. case AES_CR:
  188. return "CR";
  189. case AES_MR:
  190. return "MR";
  191. case AES_ISR:
  192. return "ISR";
  193. case AES_IMR:
  194. return "IMR";
  195. case AES_IER:
  196. return "IER";
  197. case AES_IDR:
  198. return "IDR";
  199. case AES_KEYWR(0):
  200. case AES_KEYWR(1):
  201. case AES_KEYWR(2):
  202. case AES_KEYWR(3):
  203. case AES_KEYWR(4):
  204. case AES_KEYWR(5):
  205. case AES_KEYWR(6):
  206. case AES_KEYWR(7):
  207. snprintf(tmp, sz, "KEYWR[%u]", (offset - AES_KEYWR(0)) >> 2);
  208. break;
  209. case AES_IDATAR(0):
  210. case AES_IDATAR(1):
  211. case AES_IDATAR(2):
  212. case AES_IDATAR(3):
  213. snprintf(tmp, sz, "IDATAR[%u]", (offset - AES_IDATAR(0)) >> 2);
  214. break;
  215. case AES_ODATAR(0):
  216. case AES_ODATAR(1):
  217. case AES_ODATAR(2):
  218. case AES_ODATAR(3):
  219. snprintf(tmp, sz, "ODATAR[%u]", (offset - AES_ODATAR(0)) >> 2);
  220. break;
  221. case AES_IVR(0):
  222. case AES_IVR(1):
  223. case AES_IVR(2):
  224. case AES_IVR(3):
  225. snprintf(tmp, sz, "IVR[%u]", (offset - AES_IVR(0)) >> 2);
  226. break;
  227. case AES_AADLENR:
  228. return "AADLENR";
  229. case AES_CLENR:
  230. return "CLENR";
  231. case AES_GHASHR(0):
  232. case AES_GHASHR(1):
  233. case AES_GHASHR(2):
  234. case AES_GHASHR(3):
  235. snprintf(tmp, sz, "GHASHR[%u]", (offset - AES_GHASHR(0)) >> 2);
  236. break;
  237. case AES_TAGR(0):
  238. case AES_TAGR(1):
  239. case AES_TAGR(2):
  240. case AES_TAGR(3):
  241. snprintf(tmp, sz, "TAGR[%u]", (offset - AES_TAGR(0)) >> 2);
  242. break;
  243. case AES_CTRR:
  244. return "CTRR";
  245. case AES_GCMHR(0):
  246. case AES_GCMHR(1):
  247. case AES_GCMHR(2):
  248. case AES_GCMHR(3):
  249. snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2);
  250. break;
  251. case AES_EMR:
  252. return "EMR";
  253. case AES_TWR(0):
  254. case AES_TWR(1):
  255. case AES_TWR(2):
  256. case AES_TWR(3):
  257. snprintf(tmp, sz, "TWR[%u]", (offset - AES_TWR(0)) >> 2);
  258. break;
  259. case AES_ALPHAR(0):
  260. case AES_ALPHAR(1):
  261. case AES_ALPHAR(2):
  262. case AES_ALPHAR(3):
  263. snprintf(tmp, sz, "ALPHAR[%u]", (offset - AES_ALPHAR(0)) >> 2);
  264. break;
  265. default:
  266. snprintf(tmp, sz, "0x%02x", offset);
  267. break;
  268. }
  269. return tmp;
  270. }
  271. #endif /* VERBOSE_DEBUG */
  272. /* Shared functions */
  273. static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset)
  274. {
  275. u32 value = readl_relaxed(dd->io_base + offset);
  276. #ifdef VERBOSE_DEBUG
  277. if (dd->flags & AES_FLAGS_DUMP_REG) {
  278. char tmp[16];
  279. dev_vdbg(dd->dev, "read 0x%08x from %s\n", value,
  280. atmel_aes_reg_name(offset, tmp, sizeof(tmp)));
  281. }
  282. #endif /* VERBOSE_DEBUG */
  283. return value;
  284. }
  285. static inline void atmel_aes_write(struct atmel_aes_dev *dd,
  286. u32 offset, u32 value)
  287. {
  288. #ifdef VERBOSE_DEBUG
  289. if (dd->flags & AES_FLAGS_DUMP_REG) {
  290. char tmp[16];
  291. dev_vdbg(dd->dev, "write 0x%08x into %s\n", value,
  292. atmel_aes_reg_name(offset, tmp, sizeof(tmp)));
  293. }
  294. #endif /* VERBOSE_DEBUG */
  295. writel_relaxed(value, dd->io_base + offset);
  296. }
  297. static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset,
  298. u32 *value, int count)
  299. {
  300. for (; count--; value++, offset += 4)
  301. *value = atmel_aes_read(dd, offset);
  302. }
  303. static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset,
  304. const u32 *value, int count)
  305. {
  306. for (; count--; value++, offset += 4)
  307. atmel_aes_write(dd, offset, *value);
  308. }
  309. static inline void atmel_aes_read_block(struct atmel_aes_dev *dd, u32 offset,
  310. void *value)
  311. {
  312. atmel_aes_read_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE));
  313. }
  314. static inline void atmel_aes_write_block(struct atmel_aes_dev *dd, u32 offset,
  315. const void *value)
  316. {
  317. atmel_aes_write_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE));
  318. }
  319. static inline int atmel_aes_wait_for_data_ready(struct atmel_aes_dev *dd,
  320. atmel_aes_fn_t resume)
  321. {
  322. u32 isr = atmel_aes_read(dd, AES_ISR);
  323. if (unlikely(isr & AES_INT_DATARDY))
  324. return resume(dd);
  325. dd->resume = resume;
  326. atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);
  327. return -EINPROGRESS;
  328. }
  329. static inline size_t atmel_aes_padlen(size_t len, size_t block_size)
  330. {
  331. len &= block_size - 1;
  332. return len ? block_size - len : 0;
  333. }
  334. static struct atmel_aes_dev *atmel_aes_find_dev(struct atmel_aes_base_ctx *ctx)
  335. {
  336. struct atmel_aes_dev *aes_dd = NULL;
  337. struct atmel_aes_dev *tmp;
  338. spin_lock_bh(&atmel_aes.lock);
  339. if (!ctx->dd) {
  340. list_for_each_entry(tmp, &atmel_aes.dev_list, list) {
  341. aes_dd = tmp;
  342. break;
  343. }
  344. ctx->dd = aes_dd;
  345. } else {
  346. aes_dd = ctx->dd;
  347. }
  348. spin_unlock_bh(&atmel_aes.lock);
  349. return aes_dd;
  350. }
  351. static int atmel_aes_hw_init(struct atmel_aes_dev *dd)
  352. {
  353. int err;
  354. err = clk_enable(dd->iclk);
  355. if (err)
  356. return err;
  357. atmel_aes_write(dd, AES_CR, AES_CR_SWRST);
  358. atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET);
  359. return 0;
  360. }
  361. static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd)
  362. {
  363. return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff;
  364. }
  365. static int atmel_aes_hw_version_init(struct atmel_aes_dev *dd)
  366. {
  367. int err;
  368. err = atmel_aes_hw_init(dd);
  369. if (err)
  370. return err;
  371. dd->hw_version = atmel_aes_get_version(dd);
  372. dev_info(dd->dev, "version: 0x%x\n", dd->hw_version);
  373. clk_disable(dd->iclk);
  374. return 0;
  375. }
  376. static inline void atmel_aes_set_mode(struct atmel_aes_dev *dd,
  377. const struct atmel_aes_reqctx *rctx)
  378. {
  379. /* Clear all but persistent flags and set request flags. */
  380. dd->flags = (dd->flags & AES_FLAGS_PERSISTENT) | rctx->mode;
  381. }
  382. static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd)
  383. {
  384. return (dd->flags & AES_FLAGS_ENCRYPT);
  385. }
  386. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  387. static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err);
  388. #endif
  389. static void atmel_aes_set_iv_as_last_ciphertext_block(struct atmel_aes_dev *dd)
  390. {
  391. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  392. struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req);
  393. struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
  394. unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
  395. if (req->cryptlen < ivsize)
  396. return;
  397. if (rctx->mode & AES_FLAGS_ENCRYPT) {
  398. scatterwalk_map_and_copy(req->iv, req->dst,
  399. req->cryptlen - ivsize, ivsize, 0);
  400. } else {
  401. if (req->src == req->dst)
  402. memcpy(req->iv, rctx->lastc, ivsize);
  403. else
  404. scatterwalk_map_and_copy(req->iv, req->src,
  405. req->cryptlen - ivsize,
  406. ivsize, 0);
  407. }
  408. }
  409. static inline struct atmel_aes_ctr_ctx *
  410. atmel_aes_ctr_ctx_cast(struct atmel_aes_base_ctx *ctx)
  411. {
  412. return container_of(ctx, struct atmel_aes_ctr_ctx, base);
  413. }
  414. static void atmel_aes_ctr_update_req_iv(struct atmel_aes_dev *dd)
  415. {
  416. struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);
  417. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  418. struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
  419. unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
  420. int i;
  421. /*
  422. * The CTR transfer works in fragments of data of maximum 1 MByte
  423. * because of the 16 bit CTR counter embedded in the IP. When reaching
  424. * here, ctx->blocks contains the number of blocks of the last fragment
  425. * processed, there is no need to explicit cast it to u16.
  426. */
  427. for (i = 0; i < ctx->blocks; i++)
  428. crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE);
  429. memcpy(req->iv, ctx->iv, ivsize);
  430. }
  431. static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err)
  432. {
  433. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  434. struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req);
  435. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  436. if (dd->ctx->is_aead)
  437. atmel_aes_authenc_complete(dd, err);
  438. #endif
  439. clk_disable(dd->iclk);
  440. dd->flags &= ~AES_FLAGS_BUSY;
  441. if (!err && !dd->ctx->is_aead &&
  442. (rctx->mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_ECB) {
  443. if ((rctx->mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_CTR)
  444. atmel_aes_set_iv_as_last_ciphertext_block(dd);
  445. else
  446. atmel_aes_ctr_update_req_iv(dd);
  447. }
  448. if (dd->is_async)
  449. dd->areq->complete(dd->areq, err);
  450. tasklet_schedule(&dd->queue_task);
  451. return err;
  452. }
  453. static void atmel_aes_write_ctrl_key(struct atmel_aes_dev *dd, bool use_dma,
  454. const __be32 *iv, const u32 *key, int keylen)
  455. {
  456. u32 valmr = 0;
  457. /* MR register must be set before IV registers */
  458. if (keylen == AES_KEYSIZE_128)
  459. valmr |= AES_MR_KEYSIZE_128;
  460. else if (keylen == AES_KEYSIZE_192)
  461. valmr |= AES_MR_KEYSIZE_192;
  462. else
  463. valmr |= AES_MR_KEYSIZE_256;
  464. valmr |= dd->flags & AES_FLAGS_MODE_MASK;
  465. if (use_dma) {
  466. valmr |= AES_MR_SMOD_IDATAR0;
  467. if (dd->caps.has_dualbuff)
  468. valmr |= AES_MR_DUALBUFF;
  469. } else {
  470. valmr |= AES_MR_SMOD_AUTO;
  471. }
  472. atmel_aes_write(dd, AES_MR, valmr);
  473. atmel_aes_write_n(dd, AES_KEYWR(0), key, SIZE_IN_WORDS(keylen));
  474. if (iv && (valmr & AES_MR_OPMOD_MASK) != AES_MR_OPMOD_ECB)
  475. atmel_aes_write_block(dd, AES_IVR(0), iv);
  476. }
  477. static inline void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma,
  478. const __be32 *iv)
  479. {
  480. atmel_aes_write_ctrl_key(dd, use_dma, iv,
  481. dd->ctx->key, dd->ctx->keylen);
  482. }
  483. /* CPU transfer */
  484. static int atmel_aes_cpu_transfer(struct atmel_aes_dev *dd)
  485. {
  486. int err = 0;
  487. u32 isr;
  488. for (;;) {
  489. atmel_aes_read_block(dd, AES_ODATAR(0), dd->data);
  490. dd->data += 4;
  491. dd->datalen -= AES_BLOCK_SIZE;
  492. if (dd->datalen < AES_BLOCK_SIZE)
  493. break;
  494. atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);
  495. isr = atmel_aes_read(dd, AES_ISR);
  496. if (!(isr & AES_INT_DATARDY)) {
  497. dd->resume = atmel_aes_cpu_transfer;
  498. atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);
  499. return -EINPROGRESS;
  500. }
  501. }
  502. if (!sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst),
  503. dd->buf, dd->total))
  504. err = -EINVAL;
  505. if (err)
  506. return atmel_aes_complete(dd, err);
  507. return dd->cpu_transfer_complete(dd);
  508. }
  509. static int atmel_aes_cpu_start(struct atmel_aes_dev *dd,
  510. struct scatterlist *src,
  511. struct scatterlist *dst,
  512. size_t len,
  513. atmel_aes_fn_t resume)
  514. {
  515. size_t padlen = atmel_aes_padlen(len, AES_BLOCK_SIZE);
  516. if (unlikely(len == 0))
  517. return -EINVAL;
  518. sg_copy_to_buffer(src, sg_nents(src), dd->buf, len);
  519. dd->total = len;
  520. dd->real_dst = dst;
  521. dd->cpu_transfer_complete = resume;
  522. dd->datalen = len + padlen;
  523. dd->data = (u32 *)dd->buf;
  524. atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);
  525. return atmel_aes_wait_for_data_ready(dd, atmel_aes_cpu_transfer);
  526. }
  527. /* DMA transfer */
  528. static void atmel_aes_dma_callback(void *data);
  529. static bool atmel_aes_check_aligned(struct atmel_aes_dev *dd,
  530. struct scatterlist *sg,
  531. size_t len,
  532. struct atmel_aes_dma *dma)
  533. {
  534. int nents;
  535. if (!IS_ALIGNED(len, dd->ctx->block_size))
  536. return false;
  537. for (nents = 0; sg; sg = sg_next(sg), ++nents) {
  538. if (!IS_ALIGNED(sg->offset, sizeof(u32)))
  539. return false;
  540. if (len <= sg->length) {
  541. if (!IS_ALIGNED(len, dd->ctx->block_size))
  542. return false;
  543. dma->nents = nents+1;
  544. dma->remainder = sg->length - len;
  545. sg->length = len;
  546. return true;
  547. }
  548. if (!IS_ALIGNED(sg->length, dd->ctx->block_size))
  549. return false;
  550. len -= sg->length;
  551. }
  552. return false;
  553. }
  554. static inline void atmel_aes_restore_sg(const struct atmel_aes_dma *dma)
  555. {
  556. struct scatterlist *sg = dma->sg;
  557. int nents = dma->nents;
  558. if (!dma->remainder)
  559. return;
  560. while (--nents > 0 && sg)
  561. sg = sg_next(sg);
  562. if (!sg)
  563. return;
  564. sg->length += dma->remainder;
  565. }
  566. static int atmel_aes_map(struct atmel_aes_dev *dd,
  567. struct scatterlist *src,
  568. struct scatterlist *dst,
  569. size_t len)
  570. {
  571. bool src_aligned, dst_aligned;
  572. size_t padlen;
  573. dd->total = len;
  574. dd->src.sg = src;
  575. dd->dst.sg = dst;
  576. dd->real_dst = dst;
  577. src_aligned = atmel_aes_check_aligned(dd, src, len, &dd->src);
  578. if (src == dst)
  579. dst_aligned = src_aligned;
  580. else
  581. dst_aligned = atmel_aes_check_aligned(dd, dst, len, &dd->dst);
  582. if (!src_aligned || !dst_aligned) {
  583. padlen = atmel_aes_padlen(len, dd->ctx->block_size);
  584. if (dd->buflen < len + padlen)
  585. return -ENOMEM;
  586. if (!src_aligned) {
  587. sg_copy_to_buffer(src, sg_nents(src), dd->buf, len);
  588. dd->src.sg = &dd->aligned_sg;
  589. dd->src.nents = 1;
  590. dd->src.remainder = 0;
  591. }
  592. if (!dst_aligned) {
  593. dd->dst.sg = &dd->aligned_sg;
  594. dd->dst.nents = 1;
  595. dd->dst.remainder = 0;
  596. }
  597. sg_init_table(&dd->aligned_sg, 1);
  598. sg_set_buf(&dd->aligned_sg, dd->buf, len + padlen);
  599. }
  600. if (dd->src.sg == dd->dst.sg) {
  601. dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents,
  602. DMA_BIDIRECTIONAL);
  603. dd->dst.sg_len = dd->src.sg_len;
  604. if (!dd->src.sg_len)
  605. return -EFAULT;
  606. } else {
  607. dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents,
  608. DMA_TO_DEVICE);
  609. if (!dd->src.sg_len)
  610. return -EFAULT;
  611. dd->dst.sg_len = dma_map_sg(dd->dev, dd->dst.sg, dd->dst.nents,
  612. DMA_FROM_DEVICE);
  613. if (!dd->dst.sg_len) {
  614. dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,
  615. DMA_TO_DEVICE);
  616. return -EFAULT;
  617. }
  618. }
  619. return 0;
  620. }
  621. static void atmel_aes_unmap(struct atmel_aes_dev *dd)
  622. {
  623. if (dd->src.sg == dd->dst.sg) {
  624. dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,
  625. DMA_BIDIRECTIONAL);
  626. if (dd->src.sg != &dd->aligned_sg)
  627. atmel_aes_restore_sg(&dd->src);
  628. } else {
  629. dma_unmap_sg(dd->dev, dd->dst.sg, dd->dst.nents,
  630. DMA_FROM_DEVICE);
  631. if (dd->dst.sg != &dd->aligned_sg)
  632. atmel_aes_restore_sg(&dd->dst);
  633. dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,
  634. DMA_TO_DEVICE);
  635. if (dd->src.sg != &dd->aligned_sg)
  636. atmel_aes_restore_sg(&dd->src);
  637. }
  638. if (dd->dst.sg == &dd->aligned_sg)
  639. sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst),
  640. dd->buf, dd->total);
  641. }
  642. static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd,
  643. enum dma_slave_buswidth addr_width,
  644. enum dma_transfer_direction dir,
  645. u32 maxburst)
  646. {
  647. struct dma_async_tx_descriptor *desc;
  648. struct dma_slave_config config;
  649. dma_async_tx_callback callback;
  650. struct atmel_aes_dma *dma;
  651. int err;
  652. memset(&config, 0, sizeof(config));
  653. config.src_addr_width = addr_width;
  654. config.dst_addr_width = addr_width;
  655. config.src_maxburst = maxburst;
  656. config.dst_maxburst = maxburst;
  657. switch (dir) {
  658. case DMA_MEM_TO_DEV:
  659. dma = &dd->src;
  660. callback = NULL;
  661. config.dst_addr = dd->phys_base + AES_IDATAR(0);
  662. break;
  663. case DMA_DEV_TO_MEM:
  664. dma = &dd->dst;
  665. callback = atmel_aes_dma_callback;
  666. config.src_addr = dd->phys_base + AES_ODATAR(0);
  667. break;
  668. default:
  669. return -EINVAL;
  670. }
  671. err = dmaengine_slave_config(dma->chan, &config);
  672. if (err)
  673. return err;
  674. desc = dmaengine_prep_slave_sg(dma->chan, dma->sg, dma->sg_len, dir,
  675. DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
  676. if (!desc)
  677. return -ENOMEM;
  678. desc->callback = callback;
  679. desc->callback_param = dd;
  680. dmaengine_submit(desc);
  681. dma_async_issue_pending(dma->chan);
  682. return 0;
  683. }
  684. static int atmel_aes_dma_start(struct atmel_aes_dev *dd,
  685. struct scatterlist *src,
  686. struct scatterlist *dst,
  687. size_t len,
  688. atmel_aes_fn_t resume)
  689. {
  690. enum dma_slave_buswidth addr_width;
  691. u32 maxburst;
  692. int err;
  693. switch (dd->ctx->block_size) {
  694. case CFB8_BLOCK_SIZE:
  695. addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
  696. maxburst = 1;
  697. break;
  698. case CFB16_BLOCK_SIZE:
  699. addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
  700. maxburst = 1;
  701. break;
  702. case CFB32_BLOCK_SIZE:
  703. case CFB64_BLOCK_SIZE:
  704. addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
  705. maxburst = 1;
  706. break;
  707. case AES_BLOCK_SIZE:
  708. addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
  709. maxburst = dd->caps.max_burst_size;
  710. break;
  711. default:
  712. err = -EINVAL;
  713. goto exit;
  714. }
  715. err = atmel_aes_map(dd, src, dst, len);
  716. if (err)
  717. goto exit;
  718. dd->resume = resume;
  719. /* Set output DMA transfer first */
  720. err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_DEV_TO_MEM,
  721. maxburst);
  722. if (err)
  723. goto unmap;
  724. /* Then set input DMA transfer */
  725. err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_MEM_TO_DEV,
  726. maxburst);
  727. if (err)
  728. goto output_transfer_stop;
  729. return -EINPROGRESS;
  730. output_transfer_stop:
  731. dmaengine_terminate_sync(dd->dst.chan);
  732. unmap:
  733. atmel_aes_unmap(dd);
  734. exit:
  735. return atmel_aes_complete(dd, err);
  736. }
  737. static void atmel_aes_dma_callback(void *data)
  738. {
  739. struct atmel_aes_dev *dd = data;
  740. atmel_aes_unmap(dd);
  741. dd->is_async = true;
  742. (void)dd->resume(dd);
  743. }
  744. static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,
  745. struct crypto_async_request *new_areq)
  746. {
  747. struct crypto_async_request *areq, *backlog;
  748. struct atmel_aes_base_ctx *ctx;
  749. unsigned long flags;
  750. bool start_async;
  751. int err, ret = 0;
  752. spin_lock_irqsave(&dd->lock, flags);
  753. if (new_areq)
  754. ret = crypto_enqueue_request(&dd->queue, new_areq);
  755. if (dd->flags & AES_FLAGS_BUSY) {
  756. spin_unlock_irqrestore(&dd->lock, flags);
  757. return ret;
  758. }
  759. backlog = crypto_get_backlog(&dd->queue);
  760. areq = crypto_dequeue_request(&dd->queue);
  761. if (areq)
  762. dd->flags |= AES_FLAGS_BUSY;
  763. spin_unlock_irqrestore(&dd->lock, flags);
  764. if (!areq)
  765. return ret;
  766. if (backlog)
  767. backlog->complete(backlog, -EINPROGRESS);
  768. ctx = crypto_tfm_ctx(areq->tfm);
  769. dd->areq = areq;
  770. dd->ctx = ctx;
  771. start_async = (areq != new_areq);
  772. dd->is_async = start_async;
  773. /* WARNING: ctx->start() MAY change dd->is_async. */
  774. err = ctx->start(dd);
  775. return (start_async) ? ret : err;
  776. }
  777. /* AES async block ciphers */
  778. static int atmel_aes_transfer_complete(struct atmel_aes_dev *dd)
  779. {
  780. return atmel_aes_complete(dd, 0);
  781. }
  782. static int atmel_aes_start(struct atmel_aes_dev *dd)
  783. {
  784. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  785. struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req);
  786. bool use_dma = (req->cryptlen >= ATMEL_AES_DMA_THRESHOLD ||
  787. dd->ctx->block_size != AES_BLOCK_SIZE);
  788. int err;
  789. atmel_aes_set_mode(dd, rctx);
  790. err = atmel_aes_hw_init(dd);
  791. if (err)
  792. return atmel_aes_complete(dd, err);
  793. atmel_aes_write_ctrl(dd, use_dma, (void *)req->iv);
  794. if (use_dma)
  795. return atmel_aes_dma_start(dd, req->src, req->dst,
  796. req->cryptlen,
  797. atmel_aes_transfer_complete);
  798. return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen,
  799. atmel_aes_transfer_complete);
  800. }
  801. static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd)
  802. {
  803. struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);
  804. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  805. struct scatterlist *src, *dst;
  806. size_t datalen;
  807. u32 ctr;
  808. u16 start, end;
  809. bool use_dma, fragmented = false;
  810. /* Check for transfer completion. */
  811. ctx->offset += dd->total;
  812. if (ctx->offset >= req->cryptlen)
  813. return atmel_aes_transfer_complete(dd);
  814. /* Compute data length. */
  815. datalen = req->cryptlen - ctx->offset;
  816. ctx->blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE);
  817. ctr = be32_to_cpu(ctx->iv[3]);
  818. /* Check 16bit counter overflow. */
  819. start = ctr & 0xffff;
  820. end = start + ctx->blocks - 1;
  821. if (ctx->blocks >> 16 || end < start) {
  822. ctr |= 0xffff;
  823. datalen = AES_BLOCK_SIZE * (0x10000 - start);
  824. fragmented = true;
  825. }
  826. use_dma = (datalen >= ATMEL_AES_DMA_THRESHOLD);
  827. /* Jump to offset. */
  828. src = scatterwalk_ffwd(ctx->src, req->src, ctx->offset);
  829. dst = ((req->src == req->dst) ? src :
  830. scatterwalk_ffwd(ctx->dst, req->dst, ctx->offset));
  831. /* Configure hardware. */
  832. atmel_aes_write_ctrl(dd, use_dma, ctx->iv);
  833. if (unlikely(fragmented)) {
  834. /*
  835. * Increment the counter manually to cope with the hardware
  836. * counter overflow.
  837. */
  838. ctx->iv[3] = cpu_to_be32(ctr);
  839. crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE);
  840. }
  841. if (use_dma)
  842. return atmel_aes_dma_start(dd, src, dst, datalen,
  843. atmel_aes_ctr_transfer);
  844. return atmel_aes_cpu_start(dd, src, dst, datalen,
  845. atmel_aes_ctr_transfer);
  846. }
  847. static int atmel_aes_ctr_start(struct atmel_aes_dev *dd)
  848. {
  849. struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);
  850. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  851. struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req);
  852. int err;
  853. atmel_aes_set_mode(dd, rctx);
  854. err = atmel_aes_hw_init(dd);
  855. if (err)
  856. return atmel_aes_complete(dd, err);
  857. memcpy(ctx->iv, req->iv, AES_BLOCK_SIZE);
  858. ctx->offset = 0;
  859. dd->total = 0;
  860. return atmel_aes_ctr_transfer(dd);
  861. }
  862. static int atmel_aes_crypt(struct skcipher_request *req, unsigned long mode)
  863. {
  864. struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
  865. struct atmel_aes_base_ctx *ctx = crypto_skcipher_ctx(skcipher);
  866. struct atmel_aes_reqctx *rctx;
  867. struct atmel_aes_dev *dd;
  868. switch (mode & AES_FLAGS_OPMODE_MASK) {
  869. case AES_FLAGS_CFB8:
  870. ctx->block_size = CFB8_BLOCK_SIZE;
  871. break;
  872. case AES_FLAGS_CFB16:
  873. ctx->block_size = CFB16_BLOCK_SIZE;
  874. break;
  875. case AES_FLAGS_CFB32:
  876. ctx->block_size = CFB32_BLOCK_SIZE;
  877. break;
  878. case AES_FLAGS_CFB64:
  879. ctx->block_size = CFB64_BLOCK_SIZE;
  880. break;
  881. default:
  882. ctx->block_size = AES_BLOCK_SIZE;
  883. break;
  884. }
  885. ctx->is_aead = false;
  886. dd = atmel_aes_find_dev(ctx);
  887. if (!dd)
  888. return -ENODEV;
  889. rctx = skcipher_request_ctx(req);
  890. rctx->mode = mode;
  891. if ((mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_ECB &&
  892. !(mode & AES_FLAGS_ENCRYPT) && req->src == req->dst) {
  893. unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
  894. if (req->cryptlen >= ivsize)
  895. scatterwalk_map_and_copy(rctx->lastc, req->src,
  896. req->cryptlen - ivsize,
  897. ivsize, 0);
  898. }
  899. return atmel_aes_handle_queue(dd, &req->base);
  900. }
  901. static int atmel_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
  902. unsigned int keylen)
  903. {
  904. struct atmel_aes_base_ctx *ctx = crypto_skcipher_ctx(tfm);
  905. if (keylen != AES_KEYSIZE_128 &&
  906. keylen != AES_KEYSIZE_192 &&
  907. keylen != AES_KEYSIZE_256)
  908. return -EINVAL;
  909. memcpy(ctx->key, key, keylen);
  910. ctx->keylen = keylen;
  911. return 0;
  912. }
  913. static int atmel_aes_ecb_encrypt(struct skcipher_request *req)
  914. {
  915. return atmel_aes_crypt(req, AES_FLAGS_ECB | AES_FLAGS_ENCRYPT);
  916. }
  917. static int atmel_aes_ecb_decrypt(struct skcipher_request *req)
  918. {
  919. return atmel_aes_crypt(req, AES_FLAGS_ECB);
  920. }
  921. static int atmel_aes_cbc_encrypt(struct skcipher_request *req)
  922. {
  923. return atmel_aes_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT);
  924. }
  925. static int atmel_aes_cbc_decrypt(struct skcipher_request *req)
  926. {
  927. return atmel_aes_crypt(req, AES_FLAGS_CBC);
  928. }
  929. static int atmel_aes_ofb_encrypt(struct skcipher_request *req)
  930. {
  931. return atmel_aes_crypt(req, AES_FLAGS_OFB | AES_FLAGS_ENCRYPT);
  932. }
  933. static int atmel_aes_ofb_decrypt(struct skcipher_request *req)
  934. {
  935. return atmel_aes_crypt(req, AES_FLAGS_OFB);
  936. }
  937. static int atmel_aes_cfb_encrypt(struct skcipher_request *req)
  938. {
  939. return atmel_aes_crypt(req, AES_FLAGS_CFB128 | AES_FLAGS_ENCRYPT);
  940. }
  941. static int atmel_aes_cfb_decrypt(struct skcipher_request *req)
  942. {
  943. return atmel_aes_crypt(req, AES_FLAGS_CFB128);
  944. }
  945. static int atmel_aes_cfb64_encrypt(struct skcipher_request *req)
  946. {
  947. return atmel_aes_crypt(req, AES_FLAGS_CFB64 | AES_FLAGS_ENCRYPT);
  948. }
  949. static int atmel_aes_cfb64_decrypt(struct skcipher_request *req)
  950. {
  951. return atmel_aes_crypt(req, AES_FLAGS_CFB64);
  952. }
  953. static int atmel_aes_cfb32_encrypt(struct skcipher_request *req)
  954. {
  955. return atmel_aes_crypt(req, AES_FLAGS_CFB32 | AES_FLAGS_ENCRYPT);
  956. }
  957. static int atmel_aes_cfb32_decrypt(struct skcipher_request *req)
  958. {
  959. return atmel_aes_crypt(req, AES_FLAGS_CFB32);
  960. }
  961. static int atmel_aes_cfb16_encrypt(struct skcipher_request *req)
  962. {
  963. return atmel_aes_crypt(req, AES_FLAGS_CFB16 | AES_FLAGS_ENCRYPT);
  964. }
  965. static int atmel_aes_cfb16_decrypt(struct skcipher_request *req)
  966. {
  967. return atmel_aes_crypt(req, AES_FLAGS_CFB16);
  968. }
  969. static int atmel_aes_cfb8_encrypt(struct skcipher_request *req)
  970. {
  971. return atmel_aes_crypt(req, AES_FLAGS_CFB8 | AES_FLAGS_ENCRYPT);
  972. }
  973. static int atmel_aes_cfb8_decrypt(struct skcipher_request *req)
  974. {
  975. return atmel_aes_crypt(req, AES_FLAGS_CFB8);
  976. }
  977. static int atmel_aes_ctr_encrypt(struct skcipher_request *req)
  978. {
  979. return atmel_aes_crypt(req, AES_FLAGS_CTR | AES_FLAGS_ENCRYPT);
  980. }
  981. static int atmel_aes_ctr_decrypt(struct skcipher_request *req)
  982. {
  983. return atmel_aes_crypt(req, AES_FLAGS_CTR);
  984. }
  985. static int atmel_aes_init_tfm(struct crypto_skcipher *tfm)
  986. {
  987. struct atmel_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
  988. crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
  989. ctx->base.start = atmel_aes_start;
  990. return 0;
  991. }
  992. static int atmel_aes_ctr_init_tfm(struct crypto_skcipher *tfm)
  993. {
  994. struct atmel_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
  995. crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
  996. ctx->base.start = atmel_aes_ctr_start;
  997. return 0;
  998. }
  999. static struct skcipher_alg aes_algs[] = {
  1000. {
  1001. .base.cra_name = "ecb(aes)",
  1002. .base.cra_driver_name = "atmel-ecb-aes",
  1003. .base.cra_blocksize = AES_BLOCK_SIZE,
  1004. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1005. .init = atmel_aes_init_tfm,
  1006. .min_keysize = AES_MIN_KEY_SIZE,
  1007. .max_keysize = AES_MAX_KEY_SIZE,
  1008. .setkey = atmel_aes_setkey,
  1009. .encrypt = atmel_aes_ecb_encrypt,
  1010. .decrypt = atmel_aes_ecb_decrypt,
  1011. },
  1012. {
  1013. .base.cra_name = "cbc(aes)",
  1014. .base.cra_driver_name = "atmel-cbc-aes",
  1015. .base.cra_blocksize = AES_BLOCK_SIZE,
  1016. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1017. .init = atmel_aes_init_tfm,
  1018. .min_keysize = AES_MIN_KEY_SIZE,
  1019. .max_keysize = AES_MAX_KEY_SIZE,
  1020. .setkey = atmel_aes_setkey,
  1021. .encrypt = atmel_aes_cbc_encrypt,
  1022. .decrypt = atmel_aes_cbc_decrypt,
  1023. .ivsize = AES_BLOCK_SIZE,
  1024. },
  1025. {
  1026. .base.cra_name = "ofb(aes)",
  1027. .base.cra_driver_name = "atmel-ofb-aes",
  1028. .base.cra_blocksize = AES_BLOCK_SIZE,
  1029. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1030. .init = atmel_aes_init_tfm,
  1031. .min_keysize = AES_MIN_KEY_SIZE,
  1032. .max_keysize = AES_MAX_KEY_SIZE,
  1033. .setkey = atmel_aes_setkey,
  1034. .encrypt = atmel_aes_ofb_encrypt,
  1035. .decrypt = atmel_aes_ofb_decrypt,
  1036. .ivsize = AES_BLOCK_SIZE,
  1037. },
  1038. {
  1039. .base.cra_name = "cfb(aes)",
  1040. .base.cra_driver_name = "atmel-cfb-aes",
  1041. .base.cra_blocksize = AES_BLOCK_SIZE,
  1042. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1043. .init = atmel_aes_init_tfm,
  1044. .min_keysize = AES_MIN_KEY_SIZE,
  1045. .max_keysize = AES_MAX_KEY_SIZE,
  1046. .setkey = atmel_aes_setkey,
  1047. .encrypt = atmel_aes_cfb_encrypt,
  1048. .decrypt = atmel_aes_cfb_decrypt,
  1049. .ivsize = AES_BLOCK_SIZE,
  1050. },
  1051. {
  1052. .base.cra_name = "cfb32(aes)",
  1053. .base.cra_driver_name = "atmel-cfb32-aes",
  1054. .base.cra_blocksize = CFB32_BLOCK_SIZE,
  1055. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1056. .init = atmel_aes_init_tfm,
  1057. .min_keysize = AES_MIN_KEY_SIZE,
  1058. .max_keysize = AES_MAX_KEY_SIZE,
  1059. .setkey = atmel_aes_setkey,
  1060. .encrypt = atmel_aes_cfb32_encrypt,
  1061. .decrypt = atmel_aes_cfb32_decrypt,
  1062. .ivsize = AES_BLOCK_SIZE,
  1063. },
  1064. {
  1065. .base.cra_name = "cfb16(aes)",
  1066. .base.cra_driver_name = "atmel-cfb16-aes",
  1067. .base.cra_blocksize = CFB16_BLOCK_SIZE,
  1068. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1069. .init = atmel_aes_init_tfm,
  1070. .min_keysize = AES_MIN_KEY_SIZE,
  1071. .max_keysize = AES_MAX_KEY_SIZE,
  1072. .setkey = atmel_aes_setkey,
  1073. .encrypt = atmel_aes_cfb16_encrypt,
  1074. .decrypt = atmel_aes_cfb16_decrypt,
  1075. .ivsize = AES_BLOCK_SIZE,
  1076. },
  1077. {
  1078. .base.cra_name = "cfb8(aes)",
  1079. .base.cra_driver_name = "atmel-cfb8-aes",
  1080. .base.cra_blocksize = CFB8_BLOCK_SIZE,
  1081. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1082. .init = atmel_aes_init_tfm,
  1083. .min_keysize = AES_MIN_KEY_SIZE,
  1084. .max_keysize = AES_MAX_KEY_SIZE,
  1085. .setkey = atmel_aes_setkey,
  1086. .encrypt = atmel_aes_cfb8_encrypt,
  1087. .decrypt = atmel_aes_cfb8_decrypt,
  1088. .ivsize = AES_BLOCK_SIZE,
  1089. },
  1090. {
  1091. .base.cra_name = "ctr(aes)",
  1092. .base.cra_driver_name = "atmel-ctr-aes",
  1093. .base.cra_blocksize = 1,
  1094. .base.cra_ctxsize = sizeof(struct atmel_aes_ctr_ctx),
  1095. .init = atmel_aes_ctr_init_tfm,
  1096. .min_keysize = AES_MIN_KEY_SIZE,
  1097. .max_keysize = AES_MAX_KEY_SIZE,
  1098. .setkey = atmel_aes_setkey,
  1099. .encrypt = atmel_aes_ctr_encrypt,
  1100. .decrypt = atmel_aes_ctr_decrypt,
  1101. .ivsize = AES_BLOCK_SIZE,
  1102. },
  1103. };
  1104. static struct skcipher_alg aes_cfb64_alg = {
  1105. .base.cra_name = "cfb64(aes)",
  1106. .base.cra_driver_name = "atmel-cfb64-aes",
  1107. .base.cra_blocksize = CFB64_BLOCK_SIZE,
  1108. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1109. .init = atmel_aes_init_tfm,
  1110. .min_keysize = AES_MIN_KEY_SIZE,
  1111. .max_keysize = AES_MAX_KEY_SIZE,
  1112. .setkey = atmel_aes_setkey,
  1113. .encrypt = atmel_aes_cfb64_encrypt,
  1114. .decrypt = atmel_aes_cfb64_decrypt,
  1115. .ivsize = AES_BLOCK_SIZE,
  1116. };
  1117. /* gcm aead functions */
  1118. static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd,
  1119. const u32 *data, size_t datalen,
  1120. const __be32 *ghash_in, __be32 *ghash_out,
  1121. atmel_aes_fn_t resume);
  1122. static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd);
  1123. static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd);
  1124. static int atmel_aes_gcm_start(struct atmel_aes_dev *dd);
  1125. static int atmel_aes_gcm_process(struct atmel_aes_dev *dd);
  1126. static int atmel_aes_gcm_length(struct atmel_aes_dev *dd);
  1127. static int atmel_aes_gcm_data(struct atmel_aes_dev *dd);
  1128. static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd);
  1129. static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd);
  1130. static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd);
  1131. static inline struct atmel_aes_gcm_ctx *
  1132. atmel_aes_gcm_ctx_cast(struct atmel_aes_base_ctx *ctx)
  1133. {
  1134. return container_of(ctx, struct atmel_aes_gcm_ctx, base);
  1135. }
  1136. static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd,
  1137. const u32 *data, size_t datalen,
  1138. const __be32 *ghash_in, __be32 *ghash_out,
  1139. atmel_aes_fn_t resume)
  1140. {
  1141. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1142. dd->data = (u32 *)data;
  1143. dd->datalen = datalen;
  1144. ctx->ghash_in = ghash_in;
  1145. ctx->ghash_out = ghash_out;
  1146. ctx->ghash_resume = resume;
  1147. atmel_aes_write_ctrl(dd, false, NULL);
  1148. return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_ghash_init);
  1149. }
  1150. static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd)
  1151. {
  1152. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1153. /* Set the data length. */
  1154. atmel_aes_write(dd, AES_AADLENR, dd->total);
  1155. atmel_aes_write(dd, AES_CLENR, 0);
  1156. /* If needed, overwrite the GCM Intermediate Hash Word Registers */
  1157. if (ctx->ghash_in)
  1158. atmel_aes_write_block(dd, AES_GHASHR(0), ctx->ghash_in);
  1159. return atmel_aes_gcm_ghash_finalize(dd);
  1160. }
  1161. static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd)
  1162. {
  1163. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1164. u32 isr;
  1165. /* Write data into the Input Data Registers. */
  1166. while (dd->datalen > 0) {
  1167. atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);
  1168. dd->data += 4;
  1169. dd->datalen -= AES_BLOCK_SIZE;
  1170. isr = atmel_aes_read(dd, AES_ISR);
  1171. if (!(isr & AES_INT_DATARDY)) {
  1172. dd->resume = atmel_aes_gcm_ghash_finalize;
  1173. atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);
  1174. return -EINPROGRESS;
  1175. }
  1176. }
  1177. /* Read the computed hash from GHASHRx. */
  1178. atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash_out);
  1179. return ctx->ghash_resume(dd);
  1180. }
  1181. static int atmel_aes_gcm_start(struct atmel_aes_dev *dd)
  1182. {
  1183. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1184. struct aead_request *req = aead_request_cast(dd->areq);
  1185. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1186. struct atmel_aes_reqctx *rctx = aead_request_ctx(req);
  1187. size_t ivsize = crypto_aead_ivsize(tfm);
  1188. size_t datalen, padlen;
  1189. const void *iv = req->iv;
  1190. u8 *data = dd->buf;
  1191. int err;
  1192. atmel_aes_set_mode(dd, rctx);
  1193. err = atmel_aes_hw_init(dd);
  1194. if (err)
  1195. return atmel_aes_complete(dd, err);
  1196. if (likely(ivsize == GCM_AES_IV_SIZE)) {
  1197. memcpy(ctx->j0, iv, ivsize);
  1198. ctx->j0[3] = cpu_to_be32(1);
  1199. return atmel_aes_gcm_process(dd);
  1200. }
  1201. padlen = atmel_aes_padlen(ivsize, AES_BLOCK_SIZE);
  1202. datalen = ivsize + padlen + AES_BLOCK_SIZE;
  1203. if (datalen > dd->buflen)
  1204. return atmel_aes_complete(dd, -EINVAL);
  1205. memcpy(data, iv, ivsize);
  1206. memset(data + ivsize, 0, padlen + sizeof(u64));
  1207. ((__be64 *)(data + datalen))[-1] = cpu_to_be64(ivsize * 8);
  1208. return atmel_aes_gcm_ghash(dd, (const u32 *)data, datalen,
  1209. NULL, ctx->j0, atmel_aes_gcm_process);
  1210. }
  1211. static int atmel_aes_gcm_process(struct atmel_aes_dev *dd)
  1212. {
  1213. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1214. struct aead_request *req = aead_request_cast(dd->areq);
  1215. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1216. bool enc = atmel_aes_is_encrypt(dd);
  1217. u32 authsize;
  1218. /* Compute text length. */
  1219. authsize = crypto_aead_authsize(tfm);
  1220. ctx->textlen = req->cryptlen - (enc ? 0 : authsize);
  1221. /*
  1222. * According to tcrypt test suite, the GCM Automatic Tag Generation
  1223. * fails when both the message and its associated data are empty.
  1224. */
  1225. if (likely(req->assoclen != 0 || ctx->textlen != 0))
  1226. dd->flags |= AES_FLAGS_GTAGEN;
  1227. atmel_aes_write_ctrl(dd, false, NULL);
  1228. return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_length);
  1229. }
  1230. static int atmel_aes_gcm_length(struct atmel_aes_dev *dd)
  1231. {
  1232. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1233. struct aead_request *req = aead_request_cast(dd->areq);
  1234. __be32 j0_lsw, *j0 = ctx->j0;
  1235. size_t padlen;
  1236. /* Write incr32(J0) into IV. */
  1237. j0_lsw = j0[3];
  1238. be32_add_cpu(&j0[3], 1);
  1239. atmel_aes_write_block(dd, AES_IVR(0), j0);
  1240. j0[3] = j0_lsw;
  1241. /* Set aad and text lengths. */
  1242. atmel_aes_write(dd, AES_AADLENR, req->assoclen);
  1243. atmel_aes_write(dd, AES_CLENR, ctx->textlen);
  1244. /* Check whether AAD are present. */
  1245. if (unlikely(req->assoclen == 0)) {
  1246. dd->datalen = 0;
  1247. return atmel_aes_gcm_data(dd);
  1248. }
  1249. /* Copy assoc data and add padding. */
  1250. padlen = atmel_aes_padlen(req->assoclen, AES_BLOCK_SIZE);
  1251. if (unlikely(req->assoclen + padlen > dd->buflen))
  1252. return atmel_aes_complete(dd, -EINVAL);
  1253. sg_copy_to_buffer(req->src, sg_nents(req->src), dd->buf, req->assoclen);
  1254. /* Write assoc data into the Input Data register. */
  1255. dd->data = (u32 *)dd->buf;
  1256. dd->datalen = req->assoclen + padlen;
  1257. return atmel_aes_gcm_data(dd);
  1258. }
  1259. static int atmel_aes_gcm_data(struct atmel_aes_dev *dd)
  1260. {
  1261. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1262. struct aead_request *req = aead_request_cast(dd->areq);
  1263. bool use_dma = (ctx->textlen >= ATMEL_AES_DMA_THRESHOLD);
  1264. struct scatterlist *src, *dst;
  1265. u32 isr, mr;
  1266. /* Write AAD first. */
  1267. while (dd->datalen > 0) {
  1268. atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);
  1269. dd->data += 4;
  1270. dd->datalen -= AES_BLOCK_SIZE;
  1271. isr = atmel_aes_read(dd, AES_ISR);
  1272. if (!(isr & AES_INT_DATARDY)) {
  1273. dd->resume = atmel_aes_gcm_data;
  1274. atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);
  1275. return -EINPROGRESS;
  1276. }
  1277. }
  1278. /* GMAC only. */
  1279. if (unlikely(ctx->textlen == 0))
  1280. return atmel_aes_gcm_tag_init(dd);
  1281. /* Prepare src and dst scatter lists to transfer cipher/plain texts */
  1282. src = scatterwalk_ffwd(ctx->src, req->src, req->assoclen);
  1283. dst = ((req->src == req->dst) ? src :
  1284. scatterwalk_ffwd(ctx->dst, req->dst, req->assoclen));
  1285. if (use_dma) {
  1286. /* Update the Mode Register for DMA transfers. */
  1287. mr = atmel_aes_read(dd, AES_MR);
  1288. mr &= ~(AES_MR_SMOD_MASK | AES_MR_DUALBUFF);
  1289. mr |= AES_MR_SMOD_IDATAR0;
  1290. if (dd->caps.has_dualbuff)
  1291. mr |= AES_MR_DUALBUFF;
  1292. atmel_aes_write(dd, AES_MR, mr);
  1293. return atmel_aes_dma_start(dd, src, dst, ctx->textlen,
  1294. atmel_aes_gcm_tag_init);
  1295. }
  1296. return atmel_aes_cpu_start(dd, src, dst, ctx->textlen,
  1297. atmel_aes_gcm_tag_init);
  1298. }
  1299. static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd)
  1300. {
  1301. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1302. struct aead_request *req = aead_request_cast(dd->areq);
  1303. __be64 *data = dd->buf;
  1304. if (likely(dd->flags & AES_FLAGS_GTAGEN)) {
  1305. if (!(atmel_aes_read(dd, AES_ISR) & AES_INT_TAGRDY)) {
  1306. dd->resume = atmel_aes_gcm_tag_init;
  1307. atmel_aes_write(dd, AES_IER, AES_INT_TAGRDY);
  1308. return -EINPROGRESS;
  1309. }
  1310. return atmel_aes_gcm_finalize(dd);
  1311. }
  1312. /* Read the GCM Intermediate Hash Word Registers. */
  1313. atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash);
  1314. data[0] = cpu_to_be64(req->assoclen * 8);
  1315. data[1] = cpu_to_be64(ctx->textlen * 8);
  1316. return atmel_aes_gcm_ghash(dd, (const u32 *)data, AES_BLOCK_SIZE,
  1317. ctx->ghash, ctx->ghash, atmel_aes_gcm_tag);
  1318. }
  1319. static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd)
  1320. {
  1321. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1322. unsigned long flags;
  1323. /*
  1324. * Change mode to CTR to complete the tag generation.
  1325. * Use J0 as Initialization Vector.
  1326. */
  1327. flags = dd->flags;
  1328. dd->flags &= ~(AES_FLAGS_OPMODE_MASK | AES_FLAGS_GTAGEN);
  1329. dd->flags |= AES_FLAGS_CTR;
  1330. atmel_aes_write_ctrl(dd, false, ctx->j0);
  1331. dd->flags = flags;
  1332. atmel_aes_write_block(dd, AES_IDATAR(0), ctx->ghash);
  1333. return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_finalize);
  1334. }
  1335. static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd)
  1336. {
  1337. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1338. struct aead_request *req = aead_request_cast(dd->areq);
  1339. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1340. bool enc = atmel_aes_is_encrypt(dd);
  1341. u32 offset, authsize, itag[4], *otag = ctx->tag;
  1342. int err;
  1343. /* Read the computed tag. */
  1344. if (likely(dd->flags & AES_FLAGS_GTAGEN))
  1345. atmel_aes_read_block(dd, AES_TAGR(0), ctx->tag);
  1346. else
  1347. atmel_aes_read_block(dd, AES_ODATAR(0), ctx->tag);
  1348. offset = req->assoclen + ctx->textlen;
  1349. authsize = crypto_aead_authsize(tfm);
  1350. if (enc) {
  1351. scatterwalk_map_and_copy(otag, req->dst, offset, authsize, 1);
  1352. err = 0;
  1353. } else {
  1354. scatterwalk_map_and_copy(itag, req->src, offset, authsize, 0);
  1355. err = crypto_memneq(itag, otag, authsize) ? -EBADMSG : 0;
  1356. }
  1357. return atmel_aes_complete(dd, err);
  1358. }
  1359. static int atmel_aes_gcm_crypt(struct aead_request *req,
  1360. unsigned long mode)
  1361. {
  1362. struct atmel_aes_base_ctx *ctx;
  1363. struct atmel_aes_reqctx *rctx;
  1364. struct atmel_aes_dev *dd;
  1365. ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
  1366. ctx->block_size = AES_BLOCK_SIZE;
  1367. ctx->is_aead = true;
  1368. dd = atmel_aes_find_dev(ctx);
  1369. if (!dd)
  1370. return -ENODEV;
  1371. rctx = aead_request_ctx(req);
  1372. rctx->mode = AES_FLAGS_GCM | mode;
  1373. return atmel_aes_handle_queue(dd, &req->base);
  1374. }
  1375. static int atmel_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
  1376. unsigned int keylen)
  1377. {
  1378. struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm);
  1379. if (keylen != AES_KEYSIZE_256 &&
  1380. keylen != AES_KEYSIZE_192 &&
  1381. keylen != AES_KEYSIZE_128)
  1382. return -EINVAL;
  1383. memcpy(ctx->key, key, keylen);
  1384. ctx->keylen = keylen;
  1385. return 0;
  1386. }
  1387. static int atmel_aes_gcm_setauthsize(struct crypto_aead *tfm,
  1388. unsigned int authsize)
  1389. {
  1390. return crypto_gcm_check_authsize(authsize);
  1391. }
  1392. static int atmel_aes_gcm_encrypt(struct aead_request *req)
  1393. {
  1394. return atmel_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT);
  1395. }
  1396. static int atmel_aes_gcm_decrypt(struct aead_request *req)
  1397. {
  1398. return atmel_aes_gcm_crypt(req, 0);
  1399. }
  1400. static int atmel_aes_gcm_init(struct crypto_aead *tfm)
  1401. {
  1402. struct atmel_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
  1403. crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
  1404. ctx->base.start = atmel_aes_gcm_start;
  1405. return 0;
  1406. }
  1407. static struct aead_alg aes_gcm_alg = {
  1408. .setkey = atmel_aes_gcm_setkey,
  1409. .setauthsize = atmel_aes_gcm_setauthsize,
  1410. .encrypt = atmel_aes_gcm_encrypt,
  1411. .decrypt = atmel_aes_gcm_decrypt,
  1412. .init = atmel_aes_gcm_init,
  1413. .ivsize = GCM_AES_IV_SIZE,
  1414. .maxauthsize = AES_BLOCK_SIZE,
  1415. .base = {
  1416. .cra_name = "gcm(aes)",
  1417. .cra_driver_name = "atmel-gcm-aes",
  1418. .cra_blocksize = 1,
  1419. .cra_ctxsize = sizeof(struct atmel_aes_gcm_ctx),
  1420. },
  1421. };
  1422. /* xts functions */
  1423. static inline struct atmel_aes_xts_ctx *
  1424. atmel_aes_xts_ctx_cast(struct atmel_aes_base_ctx *ctx)
  1425. {
  1426. return container_of(ctx, struct atmel_aes_xts_ctx, base);
  1427. }
  1428. static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd);
  1429. static int atmel_aes_xts_start(struct atmel_aes_dev *dd)
  1430. {
  1431. struct atmel_aes_xts_ctx *ctx = atmel_aes_xts_ctx_cast(dd->ctx);
  1432. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  1433. struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req);
  1434. unsigned long flags;
  1435. int err;
  1436. atmel_aes_set_mode(dd, rctx);
  1437. err = atmel_aes_hw_init(dd);
  1438. if (err)
  1439. return atmel_aes_complete(dd, err);
  1440. /* Compute the tweak value from req->iv with ecb(aes). */
  1441. flags = dd->flags;
  1442. dd->flags &= ~AES_FLAGS_MODE_MASK;
  1443. dd->flags |= (AES_FLAGS_ECB | AES_FLAGS_ENCRYPT);
  1444. atmel_aes_write_ctrl_key(dd, false, NULL,
  1445. ctx->key2, ctx->base.keylen);
  1446. dd->flags = flags;
  1447. atmel_aes_write_block(dd, AES_IDATAR(0), req->iv);
  1448. return atmel_aes_wait_for_data_ready(dd, atmel_aes_xts_process_data);
  1449. }
  1450. static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd)
  1451. {
  1452. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  1453. bool use_dma = (req->cryptlen >= ATMEL_AES_DMA_THRESHOLD);
  1454. u32 tweak[AES_BLOCK_SIZE / sizeof(u32)];
  1455. static const __le32 one[AES_BLOCK_SIZE / sizeof(u32)] = {cpu_to_le32(1), };
  1456. u8 *tweak_bytes = (u8 *)tweak;
  1457. int i;
  1458. /* Read the computed ciphered tweak value. */
  1459. atmel_aes_read_block(dd, AES_ODATAR(0), tweak);
  1460. /*
  1461. * Hardware quirk:
  1462. * the order of the ciphered tweak bytes need to be reversed before
  1463. * writing them into the ODATARx registers.
  1464. */
  1465. for (i = 0; i < AES_BLOCK_SIZE/2; ++i) {
  1466. u8 tmp = tweak_bytes[AES_BLOCK_SIZE - 1 - i];
  1467. tweak_bytes[AES_BLOCK_SIZE - 1 - i] = tweak_bytes[i];
  1468. tweak_bytes[i] = tmp;
  1469. }
  1470. /* Process the data. */
  1471. atmel_aes_write_ctrl(dd, use_dma, NULL);
  1472. atmel_aes_write_block(dd, AES_TWR(0), tweak);
  1473. atmel_aes_write_block(dd, AES_ALPHAR(0), one);
  1474. if (use_dma)
  1475. return atmel_aes_dma_start(dd, req->src, req->dst,
  1476. req->cryptlen,
  1477. atmel_aes_transfer_complete);
  1478. return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen,
  1479. atmel_aes_transfer_complete);
  1480. }
  1481. static int atmel_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
  1482. unsigned int keylen)
  1483. {
  1484. struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  1485. int err;
  1486. err = xts_check_key(crypto_skcipher_tfm(tfm), key, keylen);
  1487. if (err)
  1488. return err;
  1489. memcpy(ctx->base.key, key, keylen/2);
  1490. memcpy(ctx->key2, key + keylen/2, keylen/2);
  1491. ctx->base.keylen = keylen/2;
  1492. return 0;
  1493. }
  1494. static int atmel_aes_xts_encrypt(struct skcipher_request *req)
  1495. {
  1496. return atmel_aes_crypt(req, AES_FLAGS_XTS | AES_FLAGS_ENCRYPT);
  1497. }
  1498. static int atmel_aes_xts_decrypt(struct skcipher_request *req)
  1499. {
  1500. return atmel_aes_crypt(req, AES_FLAGS_XTS);
  1501. }
  1502. static int atmel_aes_xts_init_tfm(struct crypto_skcipher *tfm)
  1503. {
  1504. struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  1505. crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
  1506. ctx->base.start = atmel_aes_xts_start;
  1507. return 0;
  1508. }
  1509. static struct skcipher_alg aes_xts_alg = {
  1510. .base.cra_name = "xts(aes)",
  1511. .base.cra_driver_name = "atmel-xts-aes",
  1512. .base.cra_blocksize = AES_BLOCK_SIZE,
  1513. .base.cra_ctxsize = sizeof(struct atmel_aes_xts_ctx),
  1514. .min_keysize = 2 * AES_MIN_KEY_SIZE,
  1515. .max_keysize = 2 * AES_MAX_KEY_SIZE,
  1516. .ivsize = AES_BLOCK_SIZE,
  1517. .setkey = atmel_aes_xts_setkey,
  1518. .encrypt = atmel_aes_xts_encrypt,
  1519. .decrypt = atmel_aes_xts_decrypt,
  1520. .init = atmel_aes_xts_init_tfm,
  1521. };
  1522. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  1523. /* authenc aead functions */
  1524. static int atmel_aes_authenc_start(struct atmel_aes_dev *dd);
  1525. static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,
  1526. bool is_async);
  1527. static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,
  1528. bool is_async);
  1529. static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd);
  1530. static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,
  1531. bool is_async);
  1532. static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err)
  1533. {
  1534. struct aead_request *req = aead_request_cast(dd->areq);
  1535. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1536. if (err && (dd->flags & AES_FLAGS_OWN_SHA))
  1537. atmel_sha_authenc_abort(&rctx->auth_req);
  1538. dd->flags &= ~AES_FLAGS_OWN_SHA;
  1539. }
  1540. static int atmel_aes_authenc_start(struct atmel_aes_dev *dd)
  1541. {
  1542. struct aead_request *req = aead_request_cast(dd->areq);
  1543. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1544. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1545. struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
  1546. int err;
  1547. atmel_aes_set_mode(dd, &rctx->base);
  1548. err = atmel_aes_hw_init(dd);
  1549. if (err)
  1550. return atmel_aes_complete(dd, err);
  1551. return atmel_sha_authenc_schedule(&rctx->auth_req, ctx->auth,
  1552. atmel_aes_authenc_init, dd);
  1553. }
  1554. static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,
  1555. bool is_async)
  1556. {
  1557. struct aead_request *req = aead_request_cast(dd->areq);
  1558. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1559. if (is_async)
  1560. dd->is_async = true;
  1561. if (err)
  1562. return atmel_aes_complete(dd, err);
  1563. /* If here, we've got the ownership of the SHA device. */
  1564. dd->flags |= AES_FLAGS_OWN_SHA;
  1565. /* Configure the SHA device. */
  1566. return atmel_sha_authenc_init(&rctx->auth_req,
  1567. req->src, req->assoclen,
  1568. rctx->textlen,
  1569. atmel_aes_authenc_transfer, dd);
  1570. }
  1571. static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,
  1572. bool is_async)
  1573. {
  1574. struct aead_request *req = aead_request_cast(dd->areq);
  1575. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1576. bool enc = atmel_aes_is_encrypt(dd);
  1577. struct scatterlist *src, *dst;
  1578. __be32 iv[AES_BLOCK_SIZE / sizeof(u32)];
  1579. u32 emr;
  1580. if (is_async)
  1581. dd->is_async = true;
  1582. if (err)
  1583. return atmel_aes_complete(dd, err);
  1584. /* Prepare src and dst scatter-lists to transfer cipher/plain texts. */
  1585. src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
  1586. dst = src;
  1587. if (req->src != req->dst)
  1588. dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
  1589. /* Configure the AES device. */
  1590. memcpy(iv, req->iv, sizeof(iv));
  1591. /*
  1592. * Here we always set the 2nd parameter of atmel_aes_write_ctrl() to
  1593. * 'true' even if the data transfer is actually performed by the CPU (so
  1594. * not by the DMA) because we must force the AES_MR_SMOD bitfield to the
  1595. * value AES_MR_SMOD_IDATAR0. Indeed, both AES_MR_SMOD and SHA_MR_SMOD
  1596. * must be set to *_MR_SMOD_IDATAR0.
  1597. */
  1598. atmel_aes_write_ctrl(dd, true, iv);
  1599. emr = AES_EMR_PLIPEN;
  1600. if (!enc)
  1601. emr |= AES_EMR_PLIPD;
  1602. atmel_aes_write(dd, AES_EMR, emr);
  1603. /* Transfer data. */
  1604. return atmel_aes_dma_start(dd, src, dst, rctx->textlen,
  1605. atmel_aes_authenc_digest);
  1606. }
  1607. static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd)
  1608. {
  1609. struct aead_request *req = aead_request_cast(dd->areq);
  1610. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1611. /* atmel_sha_authenc_final() releases the SHA device. */
  1612. dd->flags &= ~AES_FLAGS_OWN_SHA;
  1613. return atmel_sha_authenc_final(&rctx->auth_req,
  1614. rctx->digest, sizeof(rctx->digest),
  1615. atmel_aes_authenc_final, dd);
  1616. }
  1617. static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,
  1618. bool is_async)
  1619. {
  1620. struct aead_request *req = aead_request_cast(dd->areq);
  1621. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1622. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1623. bool enc = atmel_aes_is_encrypt(dd);
  1624. u32 idigest[SHA512_DIGEST_SIZE / sizeof(u32)], *odigest = rctx->digest;
  1625. u32 offs, authsize;
  1626. if (is_async)
  1627. dd->is_async = true;
  1628. if (err)
  1629. goto complete;
  1630. offs = req->assoclen + rctx->textlen;
  1631. authsize = crypto_aead_authsize(tfm);
  1632. if (enc) {
  1633. scatterwalk_map_and_copy(odigest, req->dst, offs, authsize, 1);
  1634. } else {
  1635. scatterwalk_map_and_copy(idigest, req->src, offs, authsize, 0);
  1636. if (crypto_memneq(idigest, odigest, authsize))
  1637. err = -EBADMSG;
  1638. }
  1639. complete:
  1640. return atmel_aes_complete(dd, err);
  1641. }
  1642. static int atmel_aes_authenc_setkey(struct crypto_aead *tfm, const u8 *key,
  1643. unsigned int keylen)
  1644. {
  1645. struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
  1646. struct crypto_authenc_keys keys;
  1647. int err;
  1648. if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
  1649. goto badkey;
  1650. if (keys.enckeylen > sizeof(ctx->base.key))
  1651. goto badkey;
  1652. /* Save auth key. */
  1653. err = atmel_sha_authenc_setkey(ctx->auth,
  1654. keys.authkey, keys.authkeylen,
  1655. crypto_aead_get_flags(tfm));
  1656. if (err) {
  1657. memzero_explicit(&keys, sizeof(keys));
  1658. return err;
  1659. }
  1660. /* Save enc key. */
  1661. ctx->base.keylen = keys.enckeylen;
  1662. memcpy(ctx->base.key, keys.enckey, keys.enckeylen);
  1663. memzero_explicit(&keys, sizeof(keys));
  1664. return 0;
  1665. badkey:
  1666. memzero_explicit(&keys, sizeof(keys));
  1667. return -EINVAL;
  1668. }
  1669. static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm,
  1670. unsigned long auth_mode)
  1671. {
  1672. struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
  1673. unsigned int auth_reqsize = atmel_sha_authenc_get_reqsize();
  1674. ctx->auth = atmel_sha_authenc_spawn(auth_mode);
  1675. if (IS_ERR(ctx->auth))
  1676. return PTR_ERR(ctx->auth);
  1677. crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) +
  1678. auth_reqsize));
  1679. ctx->base.start = atmel_aes_authenc_start;
  1680. return 0;
  1681. }
  1682. static int atmel_aes_authenc_hmac_sha1_init_tfm(struct crypto_aead *tfm)
  1683. {
  1684. return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA1);
  1685. }
  1686. static int atmel_aes_authenc_hmac_sha224_init_tfm(struct crypto_aead *tfm)
  1687. {
  1688. return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA224);
  1689. }
  1690. static int atmel_aes_authenc_hmac_sha256_init_tfm(struct crypto_aead *tfm)
  1691. {
  1692. return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA256);
  1693. }
  1694. static int atmel_aes_authenc_hmac_sha384_init_tfm(struct crypto_aead *tfm)
  1695. {
  1696. return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA384);
  1697. }
  1698. static int atmel_aes_authenc_hmac_sha512_init_tfm(struct crypto_aead *tfm)
  1699. {
  1700. return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA512);
  1701. }
  1702. static void atmel_aes_authenc_exit_tfm(struct crypto_aead *tfm)
  1703. {
  1704. struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
  1705. atmel_sha_authenc_free(ctx->auth);
  1706. }
  1707. static int atmel_aes_authenc_crypt(struct aead_request *req,
  1708. unsigned long mode)
  1709. {
  1710. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1711. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1712. struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm);
  1713. u32 authsize = crypto_aead_authsize(tfm);
  1714. bool enc = (mode & AES_FLAGS_ENCRYPT);
  1715. struct atmel_aes_dev *dd;
  1716. /* Compute text length. */
  1717. if (!enc && req->cryptlen < authsize)
  1718. return -EINVAL;
  1719. rctx->textlen = req->cryptlen - (enc ? 0 : authsize);
  1720. /*
  1721. * Currently, empty messages are not supported yet:
  1722. * the SHA auto-padding can be used only on non-empty messages.
  1723. * Hence a special case needs to be implemented for empty message.
  1724. */
  1725. if (!rctx->textlen && !req->assoclen)
  1726. return -EINVAL;
  1727. rctx->base.mode = mode;
  1728. ctx->block_size = AES_BLOCK_SIZE;
  1729. ctx->is_aead = true;
  1730. dd = atmel_aes_find_dev(ctx);
  1731. if (!dd)
  1732. return -ENODEV;
  1733. return atmel_aes_handle_queue(dd, &req->base);
  1734. }
  1735. static int atmel_aes_authenc_cbc_aes_encrypt(struct aead_request *req)
  1736. {
  1737. return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT);
  1738. }
  1739. static int atmel_aes_authenc_cbc_aes_decrypt(struct aead_request *req)
  1740. {
  1741. return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC);
  1742. }
  1743. static struct aead_alg aes_authenc_algs[] = {
  1744. {
  1745. .setkey = atmel_aes_authenc_setkey,
  1746. .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
  1747. .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
  1748. .init = atmel_aes_authenc_hmac_sha1_init_tfm,
  1749. .exit = atmel_aes_authenc_exit_tfm,
  1750. .ivsize = AES_BLOCK_SIZE,
  1751. .maxauthsize = SHA1_DIGEST_SIZE,
  1752. .base = {
  1753. .cra_name = "authenc(hmac(sha1),cbc(aes))",
  1754. .cra_driver_name = "atmel-authenc-hmac-sha1-cbc-aes",
  1755. .cra_blocksize = AES_BLOCK_SIZE,
  1756. .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
  1757. },
  1758. },
  1759. {
  1760. .setkey = atmel_aes_authenc_setkey,
  1761. .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
  1762. .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
  1763. .init = atmel_aes_authenc_hmac_sha224_init_tfm,
  1764. .exit = atmel_aes_authenc_exit_tfm,
  1765. .ivsize = AES_BLOCK_SIZE,
  1766. .maxauthsize = SHA224_DIGEST_SIZE,
  1767. .base = {
  1768. .cra_name = "authenc(hmac(sha224),cbc(aes))",
  1769. .cra_driver_name = "atmel-authenc-hmac-sha224-cbc-aes",
  1770. .cra_blocksize = AES_BLOCK_SIZE,
  1771. .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
  1772. },
  1773. },
  1774. {
  1775. .setkey = atmel_aes_authenc_setkey,
  1776. .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
  1777. .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
  1778. .init = atmel_aes_authenc_hmac_sha256_init_tfm,
  1779. .exit = atmel_aes_authenc_exit_tfm,
  1780. .ivsize = AES_BLOCK_SIZE,
  1781. .maxauthsize = SHA256_DIGEST_SIZE,
  1782. .base = {
  1783. .cra_name = "authenc(hmac(sha256),cbc(aes))",
  1784. .cra_driver_name = "atmel-authenc-hmac-sha256-cbc-aes",
  1785. .cra_blocksize = AES_BLOCK_SIZE,
  1786. .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
  1787. },
  1788. },
  1789. {
  1790. .setkey = atmel_aes_authenc_setkey,
  1791. .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
  1792. .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
  1793. .init = atmel_aes_authenc_hmac_sha384_init_tfm,
  1794. .exit = atmel_aes_authenc_exit_tfm,
  1795. .ivsize = AES_BLOCK_SIZE,
  1796. .maxauthsize = SHA384_DIGEST_SIZE,
  1797. .base = {
  1798. .cra_name = "authenc(hmac(sha384),cbc(aes))",
  1799. .cra_driver_name = "atmel-authenc-hmac-sha384-cbc-aes",
  1800. .cra_blocksize = AES_BLOCK_SIZE,
  1801. .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
  1802. },
  1803. },
  1804. {
  1805. .setkey = atmel_aes_authenc_setkey,
  1806. .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
  1807. .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
  1808. .init = atmel_aes_authenc_hmac_sha512_init_tfm,
  1809. .exit = atmel_aes_authenc_exit_tfm,
  1810. .ivsize = AES_BLOCK_SIZE,
  1811. .maxauthsize = SHA512_DIGEST_SIZE,
  1812. .base = {
  1813. .cra_name = "authenc(hmac(sha512),cbc(aes))",
  1814. .cra_driver_name = "atmel-authenc-hmac-sha512-cbc-aes",
  1815. .cra_blocksize = AES_BLOCK_SIZE,
  1816. .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
  1817. },
  1818. },
  1819. };
  1820. #endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */
  1821. /* Probe functions */
  1822. static int atmel_aes_buff_init(struct atmel_aes_dev *dd)
  1823. {
  1824. dd->buf = (void *)__get_free_pages(GFP_KERNEL, ATMEL_AES_BUFFER_ORDER);
  1825. dd->buflen = ATMEL_AES_BUFFER_SIZE;
  1826. dd->buflen &= ~(AES_BLOCK_SIZE - 1);
  1827. if (!dd->buf) {
  1828. dev_err(dd->dev, "unable to alloc pages.\n");
  1829. return -ENOMEM;
  1830. }
  1831. return 0;
  1832. }
  1833. static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd)
  1834. {
  1835. free_page((unsigned long)dd->buf);
  1836. }
  1837. static int atmel_aes_dma_init(struct atmel_aes_dev *dd)
  1838. {
  1839. int ret;
  1840. /* Try to grab 2 DMA channels */
  1841. dd->src.chan = dma_request_chan(dd->dev, "tx");
  1842. if (IS_ERR(dd->src.chan)) {
  1843. ret = PTR_ERR(dd->src.chan);
  1844. goto err_dma_in;
  1845. }
  1846. dd->dst.chan = dma_request_chan(dd->dev, "rx");
  1847. if (IS_ERR(dd->dst.chan)) {
  1848. ret = PTR_ERR(dd->dst.chan);
  1849. goto err_dma_out;
  1850. }
  1851. return 0;
  1852. err_dma_out:
  1853. dma_release_channel(dd->src.chan);
  1854. err_dma_in:
  1855. dev_err(dd->dev, "no DMA channel available\n");
  1856. return ret;
  1857. }
  1858. static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd)
  1859. {
  1860. dma_release_channel(dd->dst.chan);
  1861. dma_release_channel(dd->src.chan);
  1862. }
  1863. static void atmel_aes_queue_task(unsigned long data)
  1864. {
  1865. struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data;
  1866. atmel_aes_handle_queue(dd, NULL);
  1867. }
  1868. static void atmel_aes_done_task(unsigned long data)
  1869. {
  1870. struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data;
  1871. dd->is_async = true;
  1872. (void)dd->resume(dd);
  1873. }
  1874. static irqreturn_t atmel_aes_irq(int irq, void *dev_id)
  1875. {
  1876. struct atmel_aes_dev *aes_dd = dev_id;
  1877. u32 reg;
  1878. reg = atmel_aes_read(aes_dd, AES_ISR);
  1879. if (reg & atmel_aes_read(aes_dd, AES_IMR)) {
  1880. atmel_aes_write(aes_dd, AES_IDR, reg);
  1881. if (AES_FLAGS_BUSY & aes_dd->flags)
  1882. tasklet_schedule(&aes_dd->done_task);
  1883. else
  1884. dev_warn(aes_dd->dev, "AES interrupt when no active requests.\n");
  1885. return IRQ_HANDLED;
  1886. }
  1887. return IRQ_NONE;
  1888. }
  1889. static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd)
  1890. {
  1891. int i;
  1892. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  1893. if (dd->caps.has_authenc)
  1894. for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++)
  1895. crypto_unregister_aead(&aes_authenc_algs[i]);
  1896. #endif
  1897. if (dd->caps.has_xts)
  1898. crypto_unregister_skcipher(&aes_xts_alg);
  1899. if (dd->caps.has_gcm)
  1900. crypto_unregister_aead(&aes_gcm_alg);
  1901. if (dd->caps.has_cfb64)
  1902. crypto_unregister_skcipher(&aes_cfb64_alg);
  1903. for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
  1904. crypto_unregister_skcipher(&aes_algs[i]);
  1905. }
  1906. static void atmel_aes_crypto_alg_init(struct crypto_alg *alg)
  1907. {
  1908. alg->cra_flags = CRYPTO_ALG_ASYNC;
  1909. alg->cra_alignmask = 0xf;
  1910. alg->cra_priority = ATMEL_AES_PRIORITY;
  1911. alg->cra_module = THIS_MODULE;
  1912. }
  1913. static int atmel_aes_register_algs(struct atmel_aes_dev *dd)
  1914. {
  1915. int err, i, j;
  1916. for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
  1917. atmel_aes_crypto_alg_init(&aes_algs[i].base);
  1918. err = crypto_register_skcipher(&aes_algs[i]);
  1919. if (err)
  1920. goto err_aes_algs;
  1921. }
  1922. if (dd->caps.has_cfb64) {
  1923. atmel_aes_crypto_alg_init(&aes_cfb64_alg.base);
  1924. err = crypto_register_skcipher(&aes_cfb64_alg);
  1925. if (err)
  1926. goto err_aes_cfb64_alg;
  1927. }
  1928. if (dd->caps.has_gcm) {
  1929. atmel_aes_crypto_alg_init(&aes_gcm_alg.base);
  1930. err = crypto_register_aead(&aes_gcm_alg);
  1931. if (err)
  1932. goto err_aes_gcm_alg;
  1933. }
  1934. if (dd->caps.has_xts) {
  1935. atmel_aes_crypto_alg_init(&aes_xts_alg.base);
  1936. err = crypto_register_skcipher(&aes_xts_alg);
  1937. if (err)
  1938. goto err_aes_xts_alg;
  1939. }
  1940. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  1941. if (dd->caps.has_authenc) {
  1942. for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) {
  1943. atmel_aes_crypto_alg_init(&aes_authenc_algs[i].base);
  1944. err = crypto_register_aead(&aes_authenc_algs[i]);
  1945. if (err)
  1946. goto err_aes_authenc_alg;
  1947. }
  1948. }
  1949. #endif
  1950. return 0;
  1951. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  1952. /* i = ARRAY_SIZE(aes_authenc_algs); */
  1953. err_aes_authenc_alg:
  1954. for (j = 0; j < i; j++)
  1955. crypto_unregister_aead(&aes_authenc_algs[j]);
  1956. crypto_unregister_skcipher(&aes_xts_alg);
  1957. #endif
  1958. err_aes_xts_alg:
  1959. crypto_unregister_aead(&aes_gcm_alg);
  1960. err_aes_gcm_alg:
  1961. crypto_unregister_skcipher(&aes_cfb64_alg);
  1962. err_aes_cfb64_alg:
  1963. i = ARRAY_SIZE(aes_algs);
  1964. err_aes_algs:
  1965. for (j = 0; j < i; j++)
  1966. crypto_unregister_skcipher(&aes_algs[j]);
  1967. return err;
  1968. }
  1969. static void atmel_aes_get_cap(struct atmel_aes_dev *dd)
  1970. {
  1971. dd->caps.has_dualbuff = 0;
  1972. dd->caps.has_cfb64 = 0;
  1973. dd->caps.has_gcm = 0;
  1974. dd->caps.has_xts = 0;
  1975. dd->caps.has_authenc = 0;
  1976. dd->caps.max_burst_size = 1;
  1977. /* keep only major version number */
  1978. switch (dd->hw_version & 0xff0) {
  1979. case 0x500:
  1980. dd->caps.has_dualbuff = 1;
  1981. dd->caps.has_cfb64 = 1;
  1982. dd->caps.has_gcm = 1;
  1983. dd->caps.has_xts = 1;
  1984. dd->caps.has_authenc = 1;
  1985. dd->caps.max_burst_size = 4;
  1986. break;
  1987. case 0x200:
  1988. dd->caps.has_dualbuff = 1;
  1989. dd->caps.has_cfb64 = 1;
  1990. dd->caps.has_gcm = 1;
  1991. dd->caps.max_burst_size = 4;
  1992. break;
  1993. case 0x130:
  1994. dd->caps.has_dualbuff = 1;
  1995. dd->caps.has_cfb64 = 1;
  1996. dd->caps.max_burst_size = 4;
  1997. break;
  1998. case 0x120:
  1999. break;
  2000. default:
  2001. dev_warn(dd->dev,
  2002. "Unmanaged aes version, set minimum capabilities\n");
  2003. break;
  2004. }
  2005. }
  2006. #if defined(CONFIG_OF)
  2007. static const struct of_device_id atmel_aes_dt_ids[] = {
  2008. { .compatible = "atmel,at91sam9g46-aes" },
  2009. { /* sentinel */ }
  2010. };
  2011. MODULE_DEVICE_TABLE(of, atmel_aes_dt_ids);
  2012. #endif
  2013. static int atmel_aes_probe(struct platform_device *pdev)
  2014. {
  2015. struct atmel_aes_dev *aes_dd;
  2016. struct device *dev = &pdev->dev;
  2017. struct resource *aes_res;
  2018. int err;
  2019. aes_dd = devm_kzalloc(&pdev->dev, sizeof(*aes_dd), GFP_KERNEL);
  2020. if (!aes_dd)
  2021. return -ENOMEM;
  2022. aes_dd->dev = dev;
  2023. platform_set_drvdata(pdev, aes_dd);
  2024. INIT_LIST_HEAD(&aes_dd->list);
  2025. spin_lock_init(&aes_dd->lock);
  2026. tasklet_init(&aes_dd->done_task, atmel_aes_done_task,
  2027. (unsigned long)aes_dd);
  2028. tasklet_init(&aes_dd->queue_task, atmel_aes_queue_task,
  2029. (unsigned long)aes_dd);
  2030. crypto_init_queue(&aes_dd->queue, ATMEL_AES_QUEUE_LENGTH);
  2031. /* Get the base address */
  2032. aes_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  2033. if (!aes_res) {
  2034. dev_err(dev, "no MEM resource info\n");
  2035. err = -ENODEV;
  2036. goto err_tasklet_kill;
  2037. }
  2038. aes_dd->phys_base = aes_res->start;
  2039. /* Get the IRQ */
  2040. aes_dd->irq = platform_get_irq(pdev, 0);
  2041. if (aes_dd->irq < 0) {
  2042. err = aes_dd->irq;
  2043. goto err_tasklet_kill;
  2044. }
  2045. err = devm_request_irq(&pdev->dev, aes_dd->irq, atmel_aes_irq,
  2046. IRQF_SHARED, "atmel-aes", aes_dd);
  2047. if (err) {
  2048. dev_err(dev, "unable to request aes irq.\n");
  2049. goto err_tasklet_kill;
  2050. }
  2051. /* Initializing the clock */
  2052. aes_dd->iclk = devm_clk_get(&pdev->dev, "aes_clk");
  2053. if (IS_ERR(aes_dd->iclk)) {
  2054. dev_err(dev, "clock initialization failed.\n");
  2055. err = PTR_ERR(aes_dd->iclk);
  2056. goto err_tasklet_kill;
  2057. }
  2058. aes_dd->io_base = devm_ioremap_resource(&pdev->dev, aes_res);
  2059. if (IS_ERR(aes_dd->io_base)) {
  2060. dev_err(dev, "can't ioremap\n");
  2061. err = PTR_ERR(aes_dd->io_base);
  2062. goto err_tasklet_kill;
  2063. }
  2064. err = clk_prepare(aes_dd->iclk);
  2065. if (err)
  2066. goto err_tasklet_kill;
  2067. err = atmel_aes_hw_version_init(aes_dd);
  2068. if (err)
  2069. goto err_iclk_unprepare;
  2070. atmel_aes_get_cap(aes_dd);
  2071. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  2072. if (aes_dd->caps.has_authenc && !atmel_sha_authenc_is_ready()) {
  2073. err = -EPROBE_DEFER;
  2074. goto err_iclk_unprepare;
  2075. }
  2076. #endif
  2077. err = atmel_aes_buff_init(aes_dd);
  2078. if (err)
  2079. goto err_iclk_unprepare;
  2080. err = atmel_aes_dma_init(aes_dd);
  2081. if (err)
  2082. goto err_buff_cleanup;
  2083. spin_lock(&atmel_aes.lock);
  2084. list_add_tail(&aes_dd->list, &atmel_aes.dev_list);
  2085. spin_unlock(&atmel_aes.lock);
  2086. err = atmel_aes_register_algs(aes_dd);
  2087. if (err)
  2088. goto err_algs;
  2089. dev_info(dev, "Atmel AES - Using %s, %s for DMA transfers\n",
  2090. dma_chan_name(aes_dd->src.chan),
  2091. dma_chan_name(aes_dd->dst.chan));
  2092. return 0;
  2093. err_algs:
  2094. spin_lock(&atmel_aes.lock);
  2095. list_del(&aes_dd->list);
  2096. spin_unlock(&atmel_aes.lock);
  2097. atmel_aes_dma_cleanup(aes_dd);
  2098. err_buff_cleanup:
  2099. atmel_aes_buff_cleanup(aes_dd);
  2100. err_iclk_unprepare:
  2101. clk_unprepare(aes_dd->iclk);
  2102. err_tasklet_kill:
  2103. tasklet_kill(&aes_dd->done_task);
  2104. tasklet_kill(&aes_dd->queue_task);
  2105. return err;
  2106. }
  2107. static int atmel_aes_remove(struct platform_device *pdev)
  2108. {
  2109. struct atmel_aes_dev *aes_dd;
  2110. aes_dd = platform_get_drvdata(pdev);
  2111. if (!aes_dd)
  2112. return -ENODEV;
  2113. spin_lock(&atmel_aes.lock);
  2114. list_del(&aes_dd->list);
  2115. spin_unlock(&atmel_aes.lock);
  2116. atmel_aes_unregister_algs(aes_dd);
  2117. tasklet_kill(&aes_dd->done_task);
  2118. tasklet_kill(&aes_dd->queue_task);
  2119. atmel_aes_dma_cleanup(aes_dd);
  2120. atmel_aes_buff_cleanup(aes_dd);
  2121. clk_unprepare(aes_dd->iclk);
  2122. return 0;
  2123. }
  2124. static struct platform_driver atmel_aes_driver = {
  2125. .probe = atmel_aes_probe,
  2126. .remove = atmel_aes_remove,
  2127. .driver = {
  2128. .name = "atmel_aes",
  2129. .of_match_table = of_match_ptr(atmel_aes_dt_ids),
  2130. },
  2131. };
  2132. module_platform_driver(atmel_aes_driver);
  2133. MODULE_DESCRIPTION("Atmel AES hw acceleration support.");
  2134. MODULE_LICENSE("GPL v2");
  2135. MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique");