sa2ul.c 65 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * K3 SA2UL crypto accelerator driver
  4. *
  5. * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com
  6. *
  7. * Authors: Keerthy
  8. * Vitaly Andrianov
  9. * Tero Kristo
  10. */
  11. #include <linux/clk.h>
  12. #include <linux/dmaengine.h>
  13. #include <linux/dmapool.h>
  14. #include <linux/module.h>
  15. #include <linux/of_device.h>
  16. #include <linux/platform_device.h>
  17. #include <linux/pm_runtime.h>
  18. #include <crypto/aes.h>
  19. #include <crypto/authenc.h>
  20. #include <crypto/des.h>
  21. #include <crypto/internal/aead.h>
  22. #include <crypto/internal/hash.h>
  23. #include <crypto/internal/skcipher.h>
  24. #include <crypto/scatterwalk.h>
  25. #include <crypto/sha.h>
  26. #include "sa2ul.h"
  27. /* Byte offset for key in encryption security context */
  28. #define SC_ENC_KEY_OFFSET (1 + 27 + 4)
  29. /* Byte offset for Aux-1 in encryption security context */
  30. #define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32)
  31. #define SA_CMDL_UPD_ENC 0x0001
  32. #define SA_CMDL_UPD_AUTH 0x0002
  33. #define SA_CMDL_UPD_ENC_IV 0x0004
  34. #define SA_CMDL_UPD_AUTH_IV 0x0008
  35. #define SA_CMDL_UPD_AUX_KEY 0x0010
  36. #define SA_AUTH_SUBKEY_LEN 16
  37. #define SA_CMDL_PAYLOAD_LENGTH_MASK 0xFFFF
  38. #define SA_CMDL_SOP_BYPASS_LEN_MASK 0xFF000000
  39. #define MODE_CONTROL_BYTES 27
  40. #define SA_HASH_PROCESSING 0
  41. #define SA_CRYPTO_PROCESSING 0
  42. #define SA_UPLOAD_HASH_TO_TLR BIT(6)
  43. #define SA_SW0_FLAGS_MASK 0xF0000
  44. #define SA_SW0_CMDL_INFO_MASK 0x1F00000
  45. #define SA_SW0_CMDL_PRESENT BIT(4)
  46. #define SA_SW0_ENG_ID_MASK 0x3E000000
  47. #define SA_SW0_DEST_INFO_PRESENT BIT(30)
  48. #define SA_SW2_EGRESS_LENGTH 0xFF000000
  49. #define SA_BASIC_HASH 0x10
  50. #define SHA256_DIGEST_WORDS 8
  51. /* Make 32-bit word from 4 bytes */
  52. #define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \
  53. ((b2) << 8) | (b3))
  54. /* size of SCCTL structure in bytes */
  55. #define SA_SCCTL_SZ 16
  56. /* Max Authentication tag size */
  57. #define SA_MAX_AUTH_TAG_SZ 64
  58. #define PRIV_ID 0x1
  59. #define PRIV 0x1
  60. static struct device *sa_k3_dev;
  61. /**
  62. * struct sa_cmdl_cfg - Command label configuration descriptor
  63. * @aalg: authentication algorithm ID
  64. * @enc_eng_id: Encryption Engine ID supported by the SA hardware
  65. * @auth_eng_id: Authentication Engine ID
  66. * @iv_size: Initialization Vector size
  67. * @akey: Authentication key
  68. * @akey_len: Authentication key length
  69. * @enc: True, if this is an encode request
  70. */
  71. struct sa_cmdl_cfg {
  72. int aalg;
  73. u8 enc_eng_id;
  74. u8 auth_eng_id;
  75. u8 iv_size;
  76. const u8 *akey;
  77. u16 akey_len;
  78. bool enc;
  79. };
  80. /**
  81. * struct algo_data - Crypto algorithm specific data
  82. * @enc_eng: Encryption engine info structure
  83. * @auth_eng: Authentication engine info structure
  84. * @auth_ctrl: Authentication control word
  85. * @hash_size: Size of digest
  86. * @iv_idx: iv index in psdata
  87. * @iv_out_size: iv out size
  88. * @ealg_id: Encryption Algorithm ID
  89. * @aalg_id: Authentication algorithm ID
  90. * @mci_enc: Mode Control Instruction for Encryption algorithm
  91. * @mci_dec: Mode Control Instruction for Decryption
  92. * @inv_key: Whether the encryption algorithm demands key inversion
  93. * @ctx: Pointer to the algorithm context
  94. * @keyed_mac: Whether the authentication algorithm has key
  95. * @prep_iopad: Function pointer to generate intermediate ipad/opad
  96. */
  97. struct algo_data {
  98. struct sa_eng_info enc_eng;
  99. struct sa_eng_info auth_eng;
  100. u8 auth_ctrl;
  101. u8 hash_size;
  102. u8 iv_idx;
  103. u8 iv_out_size;
  104. u8 ealg_id;
  105. u8 aalg_id;
  106. u8 *mci_enc;
  107. u8 *mci_dec;
  108. bool inv_key;
  109. struct sa_tfm_ctx *ctx;
  110. bool keyed_mac;
  111. void (*prep_iopad)(struct algo_data *algo, const u8 *key,
  112. u16 key_sz, __be32 *ipad, __be32 *opad);
  113. };
  114. /**
  115. * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms
  116. * @type: Type of the crypto algorithm.
  117. * @alg: Union of crypto algorithm definitions.
  118. * @registered: Flag indicating if the crypto algorithm is already registered
  119. */
  120. struct sa_alg_tmpl {
  121. u32 type; /* CRYPTO_ALG_TYPE from <linux/crypto.h> */
  122. union {
  123. struct skcipher_alg skcipher;
  124. struct ahash_alg ahash;
  125. struct aead_alg aead;
  126. } alg;
  127. bool registered;
  128. };
  129. /**
  130. * struct sa_mapped_sg: scatterlist information for tx and rx
  131. * @mapped: Set to true if the @sgt is mapped
  132. * @dir: mapping direction used for @sgt
  133. * @split_sg: Set if the sg is split and needs to be freed up
  134. * @static_sg: Static scatterlist entry for overriding data
  135. * @sgt: scatterlist table for DMA API use
  136. */
  137. struct sa_mapped_sg {
  138. bool mapped;
  139. enum dma_data_direction dir;
  140. struct scatterlist static_sg;
  141. struct scatterlist *split_sg;
  142. struct sg_table sgt;
  143. };
  144. /**
  145. * struct sa_rx_data: RX Packet miscellaneous data place holder
  146. * @req: crypto request data pointer
  147. * @ddev: pointer to the DMA device
  148. * @tx_in: dma_async_tx_descriptor pointer for rx channel
  149. * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping
  150. * @enc: Flag indicating either encryption or decryption
  151. * @enc_iv_size: Initialisation vector size
  152. * @iv_idx: Initialisation vector index
  153. */
  154. struct sa_rx_data {
  155. void *req;
  156. struct device *ddev;
  157. struct dma_async_tx_descriptor *tx_in;
  158. struct sa_mapped_sg mapped_sg[2];
  159. u8 enc;
  160. u8 enc_iv_size;
  161. u8 iv_idx;
  162. };
  163. /**
  164. * struct sa_req: SA request definition
  165. * @dev: device for the request
  166. * @size: total data to the xmitted via DMA
  167. * @enc_offset: offset of cipher data
  168. * @enc_size: data to be passed to cipher engine
  169. * @enc_iv: cipher IV
  170. * @auth_offset: offset of the authentication data
  171. * @auth_size: size of the authentication data
  172. * @auth_iv: authentication IV
  173. * @type: algorithm type for the request
  174. * @cmdl: command label pointer
  175. * @base: pointer to the base request
  176. * @ctx: pointer to the algorithm context data
  177. * @enc: true if this is an encode request
  178. * @src: source data
  179. * @dst: destination data
  180. * @callback: DMA callback for the request
  181. * @mdata_size: metadata size passed to DMA
  182. */
  183. struct sa_req {
  184. struct device *dev;
  185. u16 size;
  186. u8 enc_offset;
  187. u16 enc_size;
  188. u8 *enc_iv;
  189. u8 auth_offset;
  190. u16 auth_size;
  191. u8 *auth_iv;
  192. u32 type;
  193. u32 *cmdl;
  194. struct crypto_async_request *base;
  195. struct sa_tfm_ctx *ctx;
  196. bool enc;
  197. struct scatterlist *src;
  198. struct scatterlist *dst;
  199. dma_async_tx_callback callback;
  200. u16 mdata_size;
  201. };
  202. /*
  203. * Mode Control Instructions for various Key lengths 128, 192, 256
  204. * For CBC (Cipher Block Chaining) mode for encryption
  205. */
  206. static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = {
  207. { 0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
  208. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  209. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  210. { 0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
  211. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  212. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  213. { 0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
  214. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  215. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  216. };
  217. /*
  218. * Mode Control Instructions for various Key lengths 128, 192, 256
  219. * For CBC (Cipher Block Chaining) mode for decryption
  220. */
  221. static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = {
  222. { 0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
  223. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  224. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  225. { 0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
  226. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  227. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  228. { 0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
  229. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  230. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  231. };
  232. /*
  233. * Mode Control Instructions for various Key lengths 128, 192, 256
  234. * For CBC (Cipher Block Chaining) mode for encryption
  235. */
  236. static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = {
  237. { 0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
  238. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  239. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  240. { 0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
  241. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  242. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  243. { 0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
  244. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  245. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  246. };
  247. /*
  248. * Mode Control Instructions for various Key lengths 128, 192, 256
  249. * For CBC (Cipher Block Chaining) mode for decryption
  250. */
  251. static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = {
  252. { 0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
  253. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  254. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  255. { 0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
  256. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  257. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  258. { 0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
  259. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  260. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  261. };
  262. /*
  263. * Mode Control Instructions for various Key lengths 128, 192, 256
  264. * For ECB (Electronic Code Book) mode for encryption
  265. */
  266. static u8 mci_ecb_enc_array[3][27] = {
  267. { 0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
  268. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  269. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  270. { 0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
  271. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  272. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  273. { 0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
  274. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  275. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  276. };
  277. /*
  278. * Mode Control Instructions for various Key lengths 128, 192, 256
  279. * For ECB (Electronic Code Book) mode for decryption
  280. */
  281. static u8 mci_ecb_dec_array[3][27] = {
  282. { 0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
  283. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  284. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  285. { 0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
  286. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  287. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  288. { 0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
  289. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  290. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  291. };
  292. /*
  293. * Mode Control Instructions for DES algorithm
  294. * For CBC (Cipher Block Chaining) mode and ECB mode
  295. * encryption and for decryption respectively
  296. */
  297. static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = {
  298. 0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00,
  299. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  300. 0x00, 0x00, 0x00,
  301. };
  302. static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = {
  303. 0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00,
  304. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  305. 0x00, 0x00, 0x00,
  306. };
  307. static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = {
  308. 0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
  309. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  310. 0x00, 0x00, 0x00,
  311. };
  312. static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = {
  313. 0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
  314. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  315. 0x00, 0x00, 0x00,
  316. };
  317. /*
  318. * Perform 16 byte or 128 bit swizzling
  319. * The SA2UL Expects the security context to
  320. * be in little Endian and the bus width is 128 bits or 16 bytes
  321. * Hence swap 16 bytes at a time from higher to lower address
  322. */
  323. static void sa_swiz_128(u8 *in, u16 len)
  324. {
  325. u8 data[16];
  326. int i, j;
  327. for (i = 0; i < len; i += 16) {
  328. memcpy(data, &in[i], 16);
  329. for (j = 0; j < 16; j++)
  330. in[i + j] = data[15 - j];
  331. }
  332. }
  333. /* Prepare the ipad and opad from key as per SHA algorithm step 1*/
  334. static void prepare_kiopad(u8 *k_ipad, u8 *k_opad, const u8 *key, u16 key_sz)
  335. {
  336. int i;
  337. for (i = 0; i < key_sz; i++) {
  338. k_ipad[i] = key[i] ^ 0x36;
  339. k_opad[i] = key[i] ^ 0x5c;
  340. }
  341. /* Instead of XOR with 0 */
  342. for (; i < SHA1_BLOCK_SIZE; i++) {
  343. k_ipad[i] = 0x36;
  344. k_opad[i] = 0x5c;
  345. }
  346. }
  347. static void sa_export_shash(struct shash_desc *hash, int block_size,
  348. int digest_size, __be32 *out)
  349. {
  350. union {
  351. struct sha1_state sha1;
  352. struct sha256_state sha256;
  353. struct sha512_state sha512;
  354. } sha;
  355. void *state;
  356. u32 *result;
  357. int i;
  358. switch (digest_size) {
  359. case SHA1_DIGEST_SIZE:
  360. state = &sha.sha1;
  361. result = sha.sha1.state;
  362. break;
  363. case SHA256_DIGEST_SIZE:
  364. state = &sha.sha256;
  365. result = sha.sha256.state;
  366. break;
  367. default:
  368. dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__,
  369. digest_size);
  370. return;
  371. }
  372. crypto_shash_export(hash, state);
  373. for (i = 0; i < digest_size >> 2; i++)
  374. out[i] = cpu_to_be32(result[i]);
  375. }
  376. static void sa_prepare_iopads(struct algo_data *data, const u8 *key,
  377. u16 key_sz, __be32 *ipad, __be32 *opad)
  378. {
  379. SHASH_DESC_ON_STACK(shash, data->ctx->shash);
  380. int block_size = crypto_shash_blocksize(data->ctx->shash);
  381. int digest_size = crypto_shash_digestsize(data->ctx->shash);
  382. u8 k_ipad[SHA1_BLOCK_SIZE];
  383. u8 k_opad[SHA1_BLOCK_SIZE];
  384. shash->tfm = data->ctx->shash;
  385. prepare_kiopad(k_ipad, k_opad, key, key_sz);
  386. memzero_explicit(ipad, block_size);
  387. memzero_explicit(opad, block_size);
  388. crypto_shash_init(shash);
  389. crypto_shash_update(shash, k_ipad, block_size);
  390. sa_export_shash(shash, block_size, digest_size, ipad);
  391. crypto_shash_init(shash);
  392. crypto_shash_update(shash, k_opad, block_size);
  393. sa_export_shash(shash, block_size, digest_size, opad);
  394. }
  395. /* Derive the inverse key used in AES-CBC decryption operation */
  396. static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz)
  397. {
  398. struct crypto_aes_ctx ctx;
  399. int key_pos;
  400. if (aes_expandkey(&ctx, key, key_sz)) {
  401. dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
  402. return -EINVAL;
  403. }
  404. /* work around to get the right inverse for AES_KEYSIZE_192 size keys */
  405. if (key_sz == AES_KEYSIZE_192) {
  406. ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46];
  407. ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47];
  408. }
  409. /* Based crypto_aes_expand_key logic */
  410. switch (key_sz) {
  411. case AES_KEYSIZE_128:
  412. case AES_KEYSIZE_192:
  413. key_pos = key_sz + 24;
  414. break;
  415. case AES_KEYSIZE_256:
  416. key_pos = key_sz + 24 - 4;
  417. break;
  418. default:
  419. dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
  420. return -EINVAL;
  421. }
  422. memcpy(inv_key, &ctx.key_enc[key_pos], key_sz);
  423. return 0;
  424. }
  425. /* Set Security context for the encryption engine */
  426. static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz,
  427. u8 enc, u8 *sc_buf)
  428. {
  429. const u8 *mci = NULL;
  430. /* Set Encryption mode selector to crypto processing */
  431. sc_buf[0] = SA_CRYPTO_PROCESSING;
  432. if (enc)
  433. mci = ad->mci_enc;
  434. else
  435. mci = ad->mci_dec;
  436. /* Set the mode control instructions in security context */
  437. if (mci)
  438. memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES);
  439. /* For AES-CBC decryption get the inverse key */
  440. if (ad->inv_key && !enc) {
  441. if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz))
  442. return -EINVAL;
  443. /* For all other cases: key is used */
  444. } else {
  445. memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz);
  446. }
  447. return 0;
  448. }
  449. /* Set Security context for the authentication engine */
  450. static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz,
  451. u8 *sc_buf)
  452. {
  453. __be32 ipad[64], opad[64];
  454. /* Set Authentication mode selector to hash processing */
  455. sc_buf[0] = SA_HASH_PROCESSING;
  456. /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */
  457. sc_buf[1] = SA_UPLOAD_HASH_TO_TLR;
  458. sc_buf[1] |= ad->auth_ctrl;
  459. /* Copy the keys or ipad/opad */
  460. if (ad->keyed_mac) {
  461. ad->prep_iopad(ad, key, key_sz, ipad, opad);
  462. /* Copy ipad to AuthKey */
  463. memcpy(&sc_buf[32], ipad, ad->hash_size);
  464. /* Copy opad to Aux-1 */
  465. memcpy(&sc_buf[64], opad, ad->hash_size);
  466. } else {
  467. /* basic hash */
  468. sc_buf[1] |= SA_BASIC_HASH;
  469. }
  470. }
  471. static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16)
  472. {
  473. int j;
  474. for (j = 0; j < ((size16) ? 4 : 2); j++) {
  475. *out = cpu_to_be32(*((u32 *)iv));
  476. iv += 4;
  477. out++;
  478. }
  479. }
  480. /* Format general command label */
  481. static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl,
  482. struct sa_cmdl_upd_info *upd_info)
  483. {
  484. u8 enc_offset = 0, auth_offset = 0, total = 0;
  485. u8 enc_next_eng = SA_ENG_ID_OUTPORT2;
  486. u8 auth_next_eng = SA_ENG_ID_OUTPORT2;
  487. u32 *word_ptr = (u32 *)cmdl;
  488. int i;
  489. /* Clear the command label */
  490. memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32)));
  491. /* Iniialize the command update structure */
  492. memzero_explicit(upd_info, sizeof(*upd_info));
  493. if (cfg->enc_eng_id && cfg->auth_eng_id) {
  494. if (cfg->enc) {
  495. auth_offset = SA_CMDL_HEADER_SIZE_BYTES;
  496. enc_next_eng = cfg->auth_eng_id;
  497. if (cfg->iv_size)
  498. auth_offset += cfg->iv_size;
  499. } else {
  500. enc_offset = SA_CMDL_HEADER_SIZE_BYTES;
  501. auth_next_eng = cfg->enc_eng_id;
  502. }
  503. }
  504. if (cfg->enc_eng_id) {
  505. upd_info->flags |= SA_CMDL_UPD_ENC;
  506. upd_info->enc_size.index = enc_offset >> 2;
  507. upd_info->enc_offset.index = upd_info->enc_size.index + 1;
  508. /* Encryption command label */
  509. cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng;
  510. /* Encryption modes requiring IV */
  511. if (cfg->iv_size) {
  512. upd_info->flags |= SA_CMDL_UPD_ENC_IV;
  513. upd_info->enc_iv.index =
  514. (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2;
  515. upd_info->enc_iv.size = cfg->iv_size;
  516. cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
  517. SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
  518. cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] =
  519. (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3));
  520. total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
  521. } else {
  522. cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
  523. SA_CMDL_HEADER_SIZE_BYTES;
  524. total += SA_CMDL_HEADER_SIZE_BYTES;
  525. }
  526. }
  527. if (cfg->auth_eng_id) {
  528. upd_info->flags |= SA_CMDL_UPD_AUTH;
  529. upd_info->auth_size.index = auth_offset >> 2;
  530. upd_info->auth_offset.index = upd_info->auth_size.index + 1;
  531. cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng;
  532. cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] =
  533. SA_CMDL_HEADER_SIZE_BYTES;
  534. total += SA_CMDL_HEADER_SIZE_BYTES;
  535. }
  536. total = roundup(total, 8);
  537. for (i = 0; i < total / 4; i++)
  538. word_ptr[i] = swab32(word_ptr[i]);
  539. return total;
  540. }
  541. /* Update Command label */
  542. static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl,
  543. struct sa_cmdl_upd_info *upd_info)
  544. {
  545. int i = 0, j;
  546. if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) {
  547. cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
  548. cmdl[upd_info->enc_size.index] |= req->enc_size;
  549. cmdl[upd_info->enc_offset.index] &=
  550. ~SA_CMDL_SOP_BYPASS_LEN_MASK;
  551. cmdl[upd_info->enc_offset.index] |=
  552. ((u32)req->enc_offset <<
  553. __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
  554. if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) {
  555. __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index];
  556. u32 *enc_iv = (u32 *)req->enc_iv;
  557. for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) {
  558. data[j] = cpu_to_be32(*enc_iv);
  559. enc_iv++;
  560. }
  561. }
  562. }
  563. if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) {
  564. cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
  565. cmdl[upd_info->auth_size.index] |= req->auth_size;
  566. cmdl[upd_info->auth_offset.index] &=
  567. ~SA_CMDL_SOP_BYPASS_LEN_MASK;
  568. cmdl[upd_info->auth_offset.index] |=
  569. ((u32)req->auth_offset <<
  570. __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
  571. if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) {
  572. sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index],
  573. req->auth_iv,
  574. (upd_info->auth_iv.size > 8));
  575. }
  576. if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) {
  577. int offset = (req->auth_size & 0xF) ? 4 : 0;
  578. memcpy(&cmdl[upd_info->aux_key_info.index],
  579. &upd_info->aux_key[offset], 16);
  580. }
  581. }
  582. }
  583. /* Format SWINFO words to be sent to SA */
  584. static
  585. void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys,
  586. u8 cmdl_present, u8 cmdl_offset, u8 flags,
  587. u8 hash_size, u32 *swinfo)
  588. {
  589. swinfo[0] = sc_id;
  590. swinfo[0] |= (flags << __ffs(SA_SW0_FLAGS_MASK));
  591. if (likely(cmdl_present))
  592. swinfo[0] |= ((cmdl_offset | SA_SW0_CMDL_PRESENT) <<
  593. __ffs(SA_SW0_CMDL_INFO_MASK));
  594. swinfo[0] |= (eng_id << __ffs(SA_SW0_ENG_ID_MASK));
  595. swinfo[0] |= SA_SW0_DEST_INFO_PRESENT;
  596. swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL);
  597. swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32);
  598. swinfo[2] |= (hash_size << __ffs(SA_SW2_EGRESS_LENGTH));
  599. }
  600. /* Dump the security context */
  601. static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr)
  602. {
  603. #ifdef DEBUG
  604. dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr);
  605. print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
  606. 16, 1, buf, SA_CTX_MAX_SZ, false);
  607. #endif
  608. }
  609. static
  610. int sa_init_sc(struct sa_ctx_info *ctx, const u8 *enc_key,
  611. u16 enc_key_sz, const u8 *auth_key, u16 auth_key_sz,
  612. struct algo_data *ad, u8 enc, u32 *swinfo)
  613. {
  614. int enc_sc_offset = 0;
  615. int auth_sc_offset = 0;
  616. u8 *sc_buf = ctx->sc;
  617. u16 sc_id = ctx->sc_id;
  618. u8 first_engine = 0;
  619. memzero_explicit(sc_buf, SA_CTX_MAX_SZ);
  620. if (ad->auth_eng.eng_id) {
  621. if (enc)
  622. first_engine = ad->enc_eng.eng_id;
  623. else
  624. first_engine = ad->auth_eng.eng_id;
  625. enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
  626. auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size;
  627. sc_buf[1] = SA_SCCTL_FE_AUTH_ENC;
  628. if (!ad->hash_size)
  629. return -EINVAL;
  630. ad->hash_size = roundup(ad->hash_size, 8);
  631. } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) {
  632. enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
  633. first_engine = ad->enc_eng.eng_id;
  634. sc_buf[1] = SA_SCCTL_FE_ENC;
  635. ad->hash_size = ad->iv_out_size;
  636. }
  637. /* SCCTL Owner info: 0=host, 1=CP_ACE */
  638. sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0;
  639. memcpy(&sc_buf[2], &sc_id, 2);
  640. sc_buf[4] = 0x0;
  641. sc_buf[5] = PRIV_ID;
  642. sc_buf[6] = PRIV;
  643. sc_buf[7] = 0x0;
  644. /* Prepare context for encryption engine */
  645. if (ad->enc_eng.sc_size) {
  646. if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc,
  647. &sc_buf[enc_sc_offset]))
  648. return -EINVAL;
  649. }
  650. /* Prepare context for authentication engine */
  651. if (ad->auth_eng.sc_size)
  652. sa_set_sc_auth(ad, auth_key, auth_key_sz,
  653. &sc_buf[auth_sc_offset]);
  654. /* Set the ownership of context to CP_ACE */
  655. sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80;
  656. /* swizzle the security context */
  657. sa_swiz_128(sc_buf, SA_CTX_MAX_SZ);
  658. sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0,
  659. SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo);
  660. sa_dump_sc(sc_buf, ctx->sc_phys);
  661. return 0;
  662. }
  663. /* Free the per direction context memory */
  664. static void sa_free_ctx_info(struct sa_ctx_info *ctx,
  665. struct sa_crypto_data *data)
  666. {
  667. unsigned long bn;
  668. bn = ctx->sc_id - data->sc_id_start;
  669. spin_lock(&data->scid_lock);
  670. __clear_bit(bn, data->ctx_bm);
  671. data->sc_id--;
  672. spin_unlock(&data->scid_lock);
  673. if (ctx->sc) {
  674. dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys);
  675. ctx->sc = NULL;
  676. }
  677. }
  678. static int sa_init_ctx_info(struct sa_ctx_info *ctx,
  679. struct sa_crypto_data *data)
  680. {
  681. unsigned long bn;
  682. int err;
  683. spin_lock(&data->scid_lock);
  684. bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX);
  685. __set_bit(bn, data->ctx_bm);
  686. data->sc_id++;
  687. spin_unlock(&data->scid_lock);
  688. ctx->sc_id = (u16)(data->sc_id_start + bn);
  689. ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys);
  690. if (!ctx->sc) {
  691. dev_err(&data->pdev->dev, "Failed to allocate SC memory\n");
  692. err = -ENOMEM;
  693. goto scid_rollback;
  694. }
  695. return 0;
  696. scid_rollback:
  697. spin_lock(&data->scid_lock);
  698. __clear_bit(bn, data->ctx_bm);
  699. data->sc_id--;
  700. spin_unlock(&data->scid_lock);
  701. return err;
  702. }
  703. static void sa_cipher_cra_exit(struct crypto_skcipher *tfm)
  704. {
  705. struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
  706. struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
  707. dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
  708. __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
  709. ctx->dec.sc_id, &ctx->dec.sc_phys);
  710. sa_free_ctx_info(&ctx->enc, data);
  711. sa_free_ctx_info(&ctx->dec, data);
  712. crypto_free_sync_skcipher(ctx->fallback.skcipher);
  713. }
  714. static int sa_cipher_cra_init(struct crypto_skcipher *tfm)
  715. {
  716. struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
  717. struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
  718. const char *name = crypto_tfm_alg_name(&tfm->base);
  719. int ret;
  720. memzero_explicit(ctx, sizeof(*ctx));
  721. ctx->dev_data = data;
  722. ret = sa_init_ctx_info(&ctx->enc, data);
  723. if (ret)
  724. return ret;
  725. ret = sa_init_ctx_info(&ctx->dec, data);
  726. if (ret) {
  727. sa_free_ctx_info(&ctx->enc, data);
  728. return ret;
  729. }
  730. ctx->fallback.skcipher =
  731. crypto_alloc_sync_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
  732. if (IS_ERR(ctx->fallback.skcipher)) {
  733. dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name);
  734. return PTR_ERR(ctx->fallback.skcipher);
  735. }
  736. dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
  737. __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
  738. ctx->dec.sc_id, &ctx->dec.sc_phys);
  739. return 0;
  740. }
  741. static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
  742. unsigned int keylen, struct algo_data *ad)
  743. {
  744. struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
  745. int cmdl_len;
  746. struct sa_cmdl_cfg cfg;
  747. int ret;
  748. if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
  749. keylen != AES_KEYSIZE_256)
  750. return -EINVAL;
  751. ad->enc_eng.eng_id = SA_ENG_ID_EM1;
  752. ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
  753. memzero_explicit(&cfg, sizeof(cfg));
  754. cfg.enc_eng_id = ad->enc_eng.eng_id;
  755. cfg.iv_size = crypto_skcipher_ivsize(tfm);
  756. crypto_sync_skcipher_clear_flags(ctx->fallback.skcipher,
  757. CRYPTO_TFM_REQ_MASK);
  758. crypto_sync_skcipher_set_flags(ctx->fallback.skcipher,
  759. tfm->base.crt_flags &
  760. CRYPTO_TFM_REQ_MASK);
  761. ret = crypto_sync_skcipher_setkey(ctx->fallback.skcipher, key, keylen);
  762. if (ret)
  763. return ret;
  764. /* Setup Encryption Security Context & Command label template */
  765. if (sa_init_sc(&ctx->enc, key, keylen, NULL, 0, ad, 1,
  766. &ctx->enc.epib[1]))
  767. goto badkey;
  768. cmdl_len = sa_format_cmdl_gen(&cfg,
  769. (u8 *)ctx->enc.cmdl,
  770. &ctx->enc.cmdl_upd_info);
  771. if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
  772. goto badkey;
  773. ctx->enc.cmdl_size = cmdl_len;
  774. /* Setup Decryption Security Context & Command label template */
  775. if (sa_init_sc(&ctx->dec, key, keylen, NULL, 0, ad, 0,
  776. &ctx->dec.epib[1]))
  777. goto badkey;
  778. cfg.enc_eng_id = ad->enc_eng.eng_id;
  779. cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
  780. &ctx->dec.cmdl_upd_info);
  781. if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
  782. goto badkey;
  783. ctx->dec.cmdl_size = cmdl_len;
  784. ctx->iv_idx = ad->iv_idx;
  785. return 0;
  786. badkey:
  787. dev_err(sa_k3_dev, "%s: badkey\n", __func__);
  788. return -EINVAL;
  789. }
  790. static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
  791. unsigned int keylen)
  792. {
  793. struct algo_data ad = { 0 };
  794. /* Convert the key size (16/24/32) to the key size index (0/1/2) */
  795. int key_idx = (keylen >> 3) - 2;
  796. if (key_idx >= 3)
  797. return -EINVAL;
  798. ad.mci_enc = mci_cbc_enc_array[key_idx];
  799. ad.mci_dec = mci_cbc_dec_array[key_idx];
  800. ad.inv_key = true;
  801. ad.ealg_id = SA_EALG_ID_AES_CBC;
  802. ad.iv_idx = 4;
  803. ad.iv_out_size = 16;
  804. return sa_cipher_setkey(tfm, key, keylen, &ad);
  805. }
  806. static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
  807. unsigned int keylen)
  808. {
  809. struct algo_data ad = { 0 };
  810. /* Convert the key size (16/24/32) to the key size index (0/1/2) */
  811. int key_idx = (keylen >> 3) - 2;
  812. if (key_idx >= 3)
  813. return -EINVAL;
  814. ad.mci_enc = mci_ecb_enc_array[key_idx];
  815. ad.mci_dec = mci_ecb_dec_array[key_idx];
  816. ad.inv_key = true;
  817. ad.ealg_id = SA_EALG_ID_AES_ECB;
  818. return sa_cipher_setkey(tfm, key, keylen, &ad);
  819. }
  820. static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
  821. unsigned int keylen)
  822. {
  823. struct algo_data ad = { 0 };
  824. ad.mci_enc = mci_cbc_3des_enc_array;
  825. ad.mci_dec = mci_cbc_3des_dec_array;
  826. ad.ealg_id = SA_EALG_ID_3DES_CBC;
  827. ad.iv_idx = 6;
  828. ad.iv_out_size = 8;
  829. return sa_cipher_setkey(tfm, key, keylen, &ad);
  830. }
  831. static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
  832. unsigned int keylen)
  833. {
  834. struct algo_data ad = { 0 };
  835. ad.mci_enc = mci_ecb_3des_enc_array;
  836. ad.mci_dec = mci_ecb_3des_dec_array;
  837. return sa_cipher_setkey(tfm, key, keylen, &ad);
  838. }
  839. static void sa_sync_from_device(struct sa_rx_data *rxd)
  840. {
  841. struct sg_table *sgt;
  842. if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL)
  843. sgt = &rxd->mapped_sg[0].sgt;
  844. else
  845. sgt = &rxd->mapped_sg[1].sgt;
  846. dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE);
  847. }
  848. static void sa_free_sa_rx_data(struct sa_rx_data *rxd)
  849. {
  850. int i;
  851. for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) {
  852. struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i];
  853. if (mapped_sg->mapped) {
  854. dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt,
  855. mapped_sg->dir, 0);
  856. kfree(mapped_sg->split_sg);
  857. }
  858. }
  859. kfree(rxd);
  860. }
  861. static void sa_aes_dma_in_callback(void *data)
  862. {
  863. struct sa_rx_data *rxd = (struct sa_rx_data *)data;
  864. struct skcipher_request *req;
  865. u32 *result;
  866. __be32 *mdptr;
  867. size_t ml, pl;
  868. int i;
  869. sa_sync_from_device(rxd);
  870. req = container_of(rxd->req, struct skcipher_request, base);
  871. if (req->iv) {
  872. mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl,
  873. &ml);
  874. result = (u32 *)req->iv;
  875. for (i = 0; i < (rxd->enc_iv_size / 4); i++)
  876. result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]);
  877. }
  878. sa_free_sa_rx_data(rxd);
  879. skcipher_request_complete(req, 0);
  880. }
  881. static void
  882. sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib)
  883. {
  884. u32 *out, *in;
  885. int i;
  886. for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++)
  887. *out++ = *in++;
  888. mdptr[4] = (0xFFFF << 16);
  889. for (out = &mdptr[5], in = psdata, i = 0;
  890. i < pslen / sizeof(u32); i++)
  891. *out++ = *in++;
  892. }
  893. static int sa_run(struct sa_req *req)
  894. {
  895. struct sa_rx_data *rxd;
  896. gfp_t gfp_flags;
  897. u32 cmdl[SA_MAX_CMDL_WORDS];
  898. struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
  899. struct device *ddev;
  900. struct dma_chan *dma_rx;
  901. int sg_nents, src_nents, dst_nents;
  902. struct scatterlist *src, *dst;
  903. size_t pl, ml, split_size;
  904. struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec;
  905. int ret;
  906. struct dma_async_tx_descriptor *tx_out;
  907. u32 *mdptr;
  908. bool diff_dst;
  909. enum dma_data_direction dir_src;
  910. struct sa_mapped_sg *mapped_sg;
  911. gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
  912. GFP_KERNEL : GFP_ATOMIC;
  913. rxd = kzalloc(sizeof(*rxd), gfp_flags);
  914. if (!rxd)
  915. return -ENOMEM;
  916. if (req->src != req->dst) {
  917. diff_dst = true;
  918. dir_src = DMA_TO_DEVICE;
  919. } else {
  920. diff_dst = false;
  921. dir_src = DMA_BIDIRECTIONAL;
  922. }
  923. /*
  924. * SA2UL has an interesting feature where the receive DMA channel
  925. * is selected based on the data passed to the engine. Within the
  926. * transition range, there is also a space where it is impossible
  927. * to determine where the data will end up, and this should be
  928. * avoided. This will be handled by the SW fallback mechanism by
  929. * the individual algorithm implementations.
  930. */
  931. if (req->size >= 256)
  932. dma_rx = pdata->dma_rx2;
  933. else
  934. dma_rx = pdata->dma_rx1;
  935. ddev = dma_rx->device->dev;
  936. rxd->ddev = ddev;
  937. memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
  938. sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info);
  939. if (req->type != CRYPTO_ALG_TYPE_AHASH) {
  940. if (req->enc)
  941. req->type |=
  942. (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT);
  943. else
  944. req->type |=
  945. (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT);
  946. }
  947. cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type;
  948. /*
  949. * Map the packets, first we check if the data fits into a single
  950. * sg entry and use that if possible. If it does not fit, we check
  951. * if we need to do sg_split to align the scatterlist data on the
  952. * actual data size being processed by the crypto engine.
  953. */
  954. src = req->src;
  955. sg_nents = sg_nents_for_len(src, req->size);
  956. split_size = req->size;
  957. mapped_sg = &rxd->mapped_sg[0];
  958. if (sg_nents == 1 && split_size <= req->src->length) {
  959. src = &mapped_sg->static_sg;
  960. src_nents = 1;
  961. sg_init_table(src, 1);
  962. sg_set_page(src, sg_page(req->src), split_size,
  963. req->src->offset);
  964. mapped_sg->sgt.sgl = src;
  965. mapped_sg->sgt.orig_nents = src_nents;
  966. ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
  967. if (ret) {
  968. kfree(rxd);
  969. return ret;
  970. }
  971. mapped_sg->dir = dir_src;
  972. mapped_sg->mapped = true;
  973. } else {
  974. mapped_sg->sgt.sgl = req->src;
  975. mapped_sg->sgt.orig_nents = sg_nents;
  976. ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
  977. if (ret) {
  978. kfree(rxd);
  979. return ret;
  980. }
  981. mapped_sg->dir = dir_src;
  982. mapped_sg->mapped = true;
  983. ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1,
  984. &split_size, &src, &src_nents, gfp_flags);
  985. if (ret) {
  986. src_nents = mapped_sg->sgt.nents;
  987. src = mapped_sg->sgt.sgl;
  988. } else {
  989. mapped_sg->split_sg = src;
  990. }
  991. }
  992. dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE);
  993. if (!diff_dst) {
  994. dst_nents = src_nents;
  995. dst = src;
  996. } else {
  997. dst_nents = sg_nents_for_len(req->dst, req->size);
  998. mapped_sg = &rxd->mapped_sg[1];
  999. if (dst_nents == 1 && split_size <= req->dst->length) {
  1000. dst = &mapped_sg->static_sg;
  1001. dst_nents = 1;
  1002. sg_init_table(dst, 1);
  1003. sg_set_page(dst, sg_page(req->dst), split_size,
  1004. req->dst->offset);
  1005. mapped_sg->sgt.sgl = dst;
  1006. mapped_sg->sgt.orig_nents = dst_nents;
  1007. ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
  1008. DMA_FROM_DEVICE, 0);
  1009. if (ret)
  1010. goto err_cleanup;
  1011. mapped_sg->dir = DMA_FROM_DEVICE;
  1012. mapped_sg->mapped = true;
  1013. } else {
  1014. mapped_sg->sgt.sgl = req->dst;
  1015. mapped_sg->sgt.orig_nents = dst_nents;
  1016. ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
  1017. DMA_FROM_DEVICE, 0);
  1018. if (ret)
  1019. goto err_cleanup;
  1020. mapped_sg->dir = DMA_FROM_DEVICE;
  1021. mapped_sg->mapped = true;
  1022. ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents,
  1023. 0, 1, &split_size, &dst, &dst_nents,
  1024. gfp_flags);
  1025. if (ret) {
  1026. dst_nents = mapped_sg->sgt.nents;
  1027. dst = mapped_sg->sgt.sgl;
  1028. } else {
  1029. mapped_sg->split_sg = dst;
  1030. }
  1031. }
  1032. }
  1033. rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents,
  1034. DMA_DEV_TO_MEM,
  1035. DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
  1036. if (!rxd->tx_in) {
  1037. dev_err(pdata->dev, "IN prep_slave_sg() failed\n");
  1038. ret = -EINVAL;
  1039. goto err_cleanup;
  1040. }
  1041. rxd->req = (void *)req->base;
  1042. rxd->enc = req->enc;
  1043. rxd->iv_idx = req->ctx->iv_idx;
  1044. rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size;
  1045. rxd->tx_in->callback = req->callback;
  1046. rxd->tx_in->callback_param = rxd;
  1047. tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src,
  1048. src_nents, DMA_MEM_TO_DEV,
  1049. DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
  1050. if (!tx_out) {
  1051. dev_err(pdata->dev, "OUT prep_slave_sg() failed\n");
  1052. ret = -EINVAL;
  1053. goto err_cleanup;
  1054. }
  1055. /*
  1056. * Prepare metadata for DMA engine. This essentially describes the
  1057. * crypto algorithm to be used, data sizes, different keys etc.
  1058. */
  1059. mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml);
  1060. sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS *
  1061. sizeof(u32))), cmdl, sizeof(sa_ctx->epib),
  1062. sa_ctx->epib);
  1063. ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32));
  1064. dmaengine_desc_set_metadata_len(tx_out, req->mdata_size);
  1065. dmaengine_submit(tx_out);
  1066. dmaengine_submit(rxd->tx_in);
  1067. dma_async_issue_pending(dma_rx);
  1068. dma_async_issue_pending(pdata->dma_tx);
  1069. return -EINPROGRESS;
  1070. err_cleanup:
  1071. sa_free_sa_rx_data(rxd);
  1072. return ret;
  1073. }
  1074. static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc)
  1075. {
  1076. struct sa_tfm_ctx *ctx =
  1077. crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
  1078. struct crypto_alg *alg = req->base.tfm->__crt_alg;
  1079. struct sa_req sa_req = { 0 };
  1080. int ret;
  1081. if (!req->cryptlen)
  1082. return 0;
  1083. if (req->cryptlen % alg->cra_blocksize)
  1084. return -EINVAL;
  1085. /* Use SW fallback if the data size is not supported */
  1086. if (req->cryptlen > SA_MAX_DATA_SZ ||
  1087. (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN &&
  1088. req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) {
  1089. SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback.skcipher);
  1090. skcipher_request_set_sync_tfm(subreq, ctx->fallback.skcipher);
  1091. skcipher_request_set_callback(subreq, req->base.flags,
  1092. NULL, NULL);
  1093. skcipher_request_set_crypt(subreq, req->src, req->dst,
  1094. req->cryptlen, req->iv);
  1095. if (enc)
  1096. ret = crypto_skcipher_encrypt(subreq);
  1097. else
  1098. ret = crypto_skcipher_decrypt(subreq);
  1099. skcipher_request_zero(subreq);
  1100. return ret;
  1101. }
  1102. sa_req.size = req->cryptlen;
  1103. sa_req.enc_size = req->cryptlen;
  1104. sa_req.src = req->src;
  1105. sa_req.dst = req->dst;
  1106. sa_req.enc_iv = iv;
  1107. sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER;
  1108. sa_req.enc = enc;
  1109. sa_req.callback = sa_aes_dma_in_callback;
  1110. sa_req.mdata_size = 44;
  1111. sa_req.base = &req->base;
  1112. sa_req.ctx = ctx;
  1113. return sa_run(&sa_req);
  1114. }
  1115. static int sa_encrypt(struct skcipher_request *req)
  1116. {
  1117. return sa_cipher_run(req, req->iv, 1);
  1118. }
  1119. static int sa_decrypt(struct skcipher_request *req)
  1120. {
  1121. return sa_cipher_run(req, req->iv, 0);
  1122. }
  1123. static void sa_sha_dma_in_callback(void *data)
  1124. {
  1125. struct sa_rx_data *rxd = (struct sa_rx_data *)data;
  1126. struct ahash_request *req;
  1127. struct crypto_ahash *tfm;
  1128. unsigned int authsize;
  1129. int i;
  1130. size_t ml, pl;
  1131. u32 *result;
  1132. __be32 *mdptr;
  1133. sa_sync_from_device(rxd);
  1134. req = container_of(rxd->req, struct ahash_request, base);
  1135. tfm = crypto_ahash_reqtfm(req);
  1136. authsize = crypto_ahash_digestsize(tfm);
  1137. mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
  1138. result = (u32 *)req->result;
  1139. for (i = 0; i < (authsize / 4); i++)
  1140. result[i] = be32_to_cpu(mdptr[i + 4]);
  1141. sa_free_sa_rx_data(rxd);
  1142. ahash_request_complete(req, 0);
  1143. }
  1144. static int zero_message_process(struct ahash_request *req)
  1145. {
  1146. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1147. int sa_digest_size = crypto_ahash_digestsize(tfm);
  1148. switch (sa_digest_size) {
  1149. case SHA1_DIGEST_SIZE:
  1150. memcpy(req->result, sha1_zero_message_hash, sa_digest_size);
  1151. break;
  1152. case SHA256_DIGEST_SIZE:
  1153. memcpy(req->result, sha256_zero_message_hash, sa_digest_size);
  1154. break;
  1155. case SHA512_DIGEST_SIZE:
  1156. memcpy(req->result, sha512_zero_message_hash, sa_digest_size);
  1157. break;
  1158. default:
  1159. return -EINVAL;
  1160. }
  1161. return 0;
  1162. }
  1163. static int sa_sha_run(struct ahash_request *req)
  1164. {
  1165. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
  1166. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1167. struct sa_req sa_req = { 0 };
  1168. size_t auth_len;
  1169. auth_len = req->nbytes;
  1170. if (!auth_len)
  1171. return zero_message_process(req);
  1172. if (auth_len > SA_MAX_DATA_SZ ||
  1173. (auth_len >= SA_UNSAFE_DATA_SZ_MIN &&
  1174. auth_len <= SA_UNSAFE_DATA_SZ_MAX)) {
  1175. struct ahash_request *subreq = &rctx->fallback_req;
  1176. int ret = 0;
  1177. ahash_request_set_tfm(subreq, ctx->fallback.ahash);
  1178. subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  1179. crypto_ahash_init(subreq);
  1180. subreq->nbytes = auth_len;
  1181. subreq->src = req->src;
  1182. subreq->result = req->result;
  1183. ret |= crypto_ahash_update(subreq);
  1184. subreq->nbytes = 0;
  1185. ret |= crypto_ahash_final(subreq);
  1186. return ret;
  1187. }
  1188. sa_req.size = auth_len;
  1189. sa_req.auth_size = auth_len;
  1190. sa_req.src = req->src;
  1191. sa_req.dst = req->src;
  1192. sa_req.enc = true;
  1193. sa_req.type = CRYPTO_ALG_TYPE_AHASH;
  1194. sa_req.callback = sa_sha_dma_in_callback;
  1195. sa_req.mdata_size = 28;
  1196. sa_req.ctx = ctx;
  1197. sa_req.base = &req->base;
  1198. return sa_run(&sa_req);
  1199. }
  1200. static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct algo_data *ad)
  1201. {
  1202. int bs = crypto_shash_blocksize(ctx->shash);
  1203. int cmdl_len;
  1204. struct sa_cmdl_cfg cfg;
  1205. ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
  1206. ad->auth_eng.eng_id = SA_ENG_ID_AM1;
  1207. ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
  1208. memset(ctx->authkey, 0, bs);
  1209. memset(&cfg, 0, sizeof(cfg));
  1210. cfg.aalg = ad->aalg_id;
  1211. cfg.enc_eng_id = ad->enc_eng.eng_id;
  1212. cfg.auth_eng_id = ad->auth_eng.eng_id;
  1213. cfg.iv_size = 0;
  1214. cfg.akey = NULL;
  1215. cfg.akey_len = 0;
  1216. /* Setup Encryption Security Context & Command label template */
  1217. if (sa_init_sc(&ctx->enc, NULL, 0, NULL, 0, ad, 0,
  1218. &ctx->enc.epib[1]))
  1219. goto badkey;
  1220. cmdl_len = sa_format_cmdl_gen(&cfg,
  1221. (u8 *)ctx->enc.cmdl,
  1222. &ctx->enc.cmdl_upd_info);
  1223. if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
  1224. goto badkey;
  1225. ctx->enc.cmdl_size = cmdl_len;
  1226. return 0;
  1227. badkey:
  1228. dev_err(sa_k3_dev, "%s: badkey\n", __func__);
  1229. return -EINVAL;
  1230. }
  1231. static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
  1232. {
  1233. struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
  1234. struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
  1235. int ret;
  1236. memset(ctx, 0, sizeof(*ctx));
  1237. ctx->dev_data = data;
  1238. ret = sa_init_ctx_info(&ctx->enc, data);
  1239. if (ret)
  1240. return ret;
  1241. if (alg_base) {
  1242. ctx->shash = crypto_alloc_shash(alg_base, 0,
  1243. CRYPTO_ALG_NEED_FALLBACK);
  1244. if (IS_ERR(ctx->shash)) {
  1245. dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n",
  1246. alg_base);
  1247. return PTR_ERR(ctx->shash);
  1248. }
  1249. /* for fallback */
  1250. ctx->fallback.ahash =
  1251. crypto_alloc_ahash(alg_base, 0,
  1252. CRYPTO_ALG_NEED_FALLBACK);
  1253. if (IS_ERR(ctx->fallback.ahash)) {
  1254. dev_err(ctx->dev_data->dev,
  1255. "Could not load fallback driver\n");
  1256. return PTR_ERR(ctx->fallback.ahash);
  1257. }
  1258. }
  1259. dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
  1260. __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
  1261. ctx->dec.sc_id, &ctx->dec.sc_phys);
  1262. crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
  1263. sizeof(struct sa_sha_req_ctx) +
  1264. crypto_ahash_reqsize(ctx->fallback.ahash));
  1265. return 0;
  1266. }
  1267. static int sa_sha_digest(struct ahash_request *req)
  1268. {
  1269. return sa_sha_run(req);
  1270. }
  1271. static int sa_sha_init(struct ahash_request *req)
  1272. {
  1273. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1274. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1275. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
  1276. dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n",
  1277. crypto_ahash_digestsize(tfm), rctx);
  1278. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
  1279. rctx->fallback_req.base.flags =
  1280. req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  1281. return crypto_ahash_init(&rctx->fallback_req);
  1282. }
  1283. static int sa_sha_update(struct ahash_request *req)
  1284. {
  1285. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1286. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1287. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
  1288. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
  1289. rctx->fallback_req.base.flags =
  1290. req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  1291. rctx->fallback_req.nbytes = req->nbytes;
  1292. rctx->fallback_req.src = req->src;
  1293. return crypto_ahash_update(&rctx->fallback_req);
  1294. }
  1295. static int sa_sha_final(struct ahash_request *req)
  1296. {
  1297. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1298. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1299. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
  1300. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
  1301. rctx->fallback_req.base.flags =
  1302. req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  1303. rctx->fallback_req.result = req->result;
  1304. return crypto_ahash_final(&rctx->fallback_req);
  1305. }
  1306. static int sa_sha_finup(struct ahash_request *req)
  1307. {
  1308. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1309. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1310. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
  1311. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
  1312. rctx->fallback_req.base.flags =
  1313. req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  1314. rctx->fallback_req.nbytes = req->nbytes;
  1315. rctx->fallback_req.src = req->src;
  1316. rctx->fallback_req.result = req->result;
  1317. return crypto_ahash_finup(&rctx->fallback_req);
  1318. }
  1319. static int sa_sha_import(struct ahash_request *req, const void *in)
  1320. {
  1321. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1322. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1323. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
  1324. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
  1325. rctx->fallback_req.base.flags = req->base.flags &
  1326. CRYPTO_TFM_REQ_MAY_SLEEP;
  1327. return crypto_ahash_import(&rctx->fallback_req, in);
  1328. }
  1329. static int sa_sha_export(struct ahash_request *req, void *out)
  1330. {
  1331. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1332. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1333. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
  1334. struct ahash_request *subreq = &rctx->fallback_req;
  1335. ahash_request_set_tfm(subreq, ctx->fallback.ahash);
  1336. subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  1337. return crypto_ahash_export(subreq, out);
  1338. }
  1339. static int sa_sha1_cra_init(struct crypto_tfm *tfm)
  1340. {
  1341. struct algo_data ad = { 0 };
  1342. struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
  1343. sa_sha_cra_init_alg(tfm, "sha1");
  1344. ad.aalg_id = SA_AALG_ID_SHA1;
  1345. ad.hash_size = SHA1_DIGEST_SIZE;
  1346. ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
  1347. sa_sha_setup(ctx, &ad);
  1348. return 0;
  1349. }
  1350. static int sa_sha256_cra_init(struct crypto_tfm *tfm)
  1351. {
  1352. struct algo_data ad = { 0 };
  1353. struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
  1354. sa_sha_cra_init_alg(tfm, "sha256");
  1355. ad.aalg_id = SA_AALG_ID_SHA2_256;
  1356. ad.hash_size = SHA256_DIGEST_SIZE;
  1357. ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
  1358. sa_sha_setup(ctx, &ad);
  1359. return 0;
  1360. }
  1361. static int sa_sha512_cra_init(struct crypto_tfm *tfm)
  1362. {
  1363. struct algo_data ad = { 0 };
  1364. struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
  1365. sa_sha_cra_init_alg(tfm, "sha512");
  1366. ad.aalg_id = SA_AALG_ID_SHA2_512;
  1367. ad.hash_size = SHA512_DIGEST_SIZE;
  1368. ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512;
  1369. sa_sha_setup(ctx, &ad);
  1370. return 0;
  1371. }
  1372. static void sa_sha_cra_exit(struct crypto_tfm *tfm)
  1373. {
  1374. struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
  1375. struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
  1376. dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
  1377. __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
  1378. ctx->dec.sc_id, &ctx->dec.sc_phys);
  1379. if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH)
  1380. sa_free_ctx_info(&ctx->enc, data);
  1381. crypto_free_shash(ctx->shash);
  1382. crypto_free_ahash(ctx->fallback.ahash);
  1383. }
  1384. static void sa_aead_dma_in_callback(void *data)
  1385. {
  1386. struct sa_rx_data *rxd = (struct sa_rx_data *)data;
  1387. struct aead_request *req;
  1388. struct crypto_aead *tfm;
  1389. unsigned int start;
  1390. unsigned int authsize;
  1391. u8 auth_tag[SA_MAX_AUTH_TAG_SZ];
  1392. size_t pl, ml;
  1393. int i;
  1394. int err = 0;
  1395. u16 auth_len;
  1396. u32 *mdptr;
  1397. sa_sync_from_device(rxd);
  1398. req = container_of(rxd->req, struct aead_request, base);
  1399. tfm = crypto_aead_reqtfm(req);
  1400. start = req->assoclen + req->cryptlen;
  1401. authsize = crypto_aead_authsize(tfm);
  1402. mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
  1403. for (i = 0; i < (authsize / 4); i++)
  1404. mdptr[i + 4] = swab32(mdptr[i + 4]);
  1405. auth_len = req->assoclen + req->cryptlen;
  1406. if (rxd->enc) {
  1407. scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize,
  1408. 1);
  1409. } else {
  1410. auth_len -= authsize;
  1411. start -= authsize;
  1412. scatterwalk_map_and_copy(auth_tag, req->src, start, authsize,
  1413. 0);
  1414. err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0;
  1415. }
  1416. sa_free_sa_rx_data(rxd);
  1417. aead_request_complete(req, err);
  1418. }
  1419. static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash,
  1420. const char *fallback)
  1421. {
  1422. struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
  1423. struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
  1424. int ret;
  1425. memzero_explicit(ctx, sizeof(*ctx));
  1426. ctx->shash = crypto_alloc_shash(hash, 0, CRYPTO_ALG_NEED_FALLBACK);
  1427. if (IS_ERR(ctx->shash)) {
  1428. dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", hash);
  1429. return PTR_ERR(ctx->shash);
  1430. }
  1431. ctx->fallback.aead = crypto_alloc_aead(fallback, 0,
  1432. CRYPTO_ALG_NEED_FALLBACK);
  1433. if (IS_ERR(ctx->fallback.aead)) {
  1434. dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n",
  1435. fallback);
  1436. return PTR_ERR(ctx->fallback.aead);
  1437. }
  1438. crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
  1439. crypto_aead_reqsize(ctx->fallback.aead));
  1440. ret = sa_init_ctx_info(&ctx->enc, data);
  1441. if (ret)
  1442. return ret;
  1443. ret = sa_init_ctx_info(&ctx->dec, data);
  1444. if (ret) {
  1445. sa_free_ctx_info(&ctx->enc, data);
  1446. return ret;
  1447. }
  1448. dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
  1449. __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
  1450. ctx->dec.sc_id, &ctx->dec.sc_phys);
  1451. return ret;
  1452. }
  1453. static int sa_cra_init_aead_sha1(struct crypto_aead *tfm)
  1454. {
  1455. return sa_cra_init_aead(tfm, "sha1",
  1456. "authenc(hmac(sha1-ce),cbc(aes-ce))");
  1457. }
  1458. static int sa_cra_init_aead_sha256(struct crypto_aead *tfm)
  1459. {
  1460. return sa_cra_init_aead(tfm, "sha256",
  1461. "authenc(hmac(sha256-ce),cbc(aes-ce))");
  1462. }
  1463. static void sa_exit_tfm_aead(struct crypto_aead *tfm)
  1464. {
  1465. struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
  1466. struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
  1467. crypto_free_shash(ctx->shash);
  1468. crypto_free_aead(ctx->fallback.aead);
  1469. sa_free_ctx_info(&ctx->enc, data);
  1470. sa_free_ctx_info(&ctx->dec, data);
  1471. }
  1472. /* AEAD algorithm configuration interface function */
  1473. static int sa_aead_setkey(struct crypto_aead *authenc,
  1474. const u8 *key, unsigned int keylen,
  1475. struct algo_data *ad)
  1476. {
  1477. struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc);
  1478. struct crypto_authenc_keys keys;
  1479. int cmdl_len;
  1480. struct sa_cmdl_cfg cfg;
  1481. int key_idx;
  1482. if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
  1483. return -EINVAL;
  1484. /* Convert the key size (16/24/32) to the key size index (0/1/2) */
  1485. key_idx = (keys.enckeylen >> 3) - 2;
  1486. if (key_idx >= 3)
  1487. return -EINVAL;
  1488. ad->ctx = ctx;
  1489. ad->enc_eng.eng_id = SA_ENG_ID_EM1;
  1490. ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
  1491. ad->auth_eng.eng_id = SA_ENG_ID_AM1;
  1492. ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
  1493. ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx];
  1494. ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx];
  1495. ad->inv_key = true;
  1496. ad->keyed_mac = true;
  1497. ad->ealg_id = SA_EALG_ID_AES_CBC;
  1498. ad->prep_iopad = sa_prepare_iopads;
  1499. memset(&cfg, 0, sizeof(cfg));
  1500. cfg.enc = true;
  1501. cfg.aalg = ad->aalg_id;
  1502. cfg.enc_eng_id = ad->enc_eng.eng_id;
  1503. cfg.auth_eng_id = ad->auth_eng.eng_id;
  1504. cfg.iv_size = crypto_aead_ivsize(authenc);
  1505. cfg.akey = keys.authkey;
  1506. cfg.akey_len = keys.authkeylen;
  1507. /* Setup Encryption Security Context & Command label template */
  1508. if (sa_init_sc(&ctx->enc, keys.enckey, keys.enckeylen,
  1509. keys.authkey, keys.authkeylen,
  1510. ad, 1, &ctx->enc.epib[1]))
  1511. return -EINVAL;
  1512. cmdl_len = sa_format_cmdl_gen(&cfg,
  1513. (u8 *)ctx->enc.cmdl,
  1514. &ctx->enc.cmdl_upd_info);
  1515. if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
  1516. return -EINVAL;
  1517. ctx->enc.cmdl_size = cmdl_len;
  1518. /* Setup Decryption Security Context & Command label template */
  1519. if (sa_init_sc(&ctx->dec, keys.enckey, keys.enckeylen,
  1520. keys.authkey, keys.authkeylen,
  1521. ad, 0, &ctx->dec.epib[1]))
  1522. return -EINVAL;
  1523. cfg.enc = false;
  1524. cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
  1525. &ctx->dec.cmdl_upd_info);
  1526. if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
  1527. return -EINVAL;
  1528. ctx->dec.cmdl_size = cmdl_len;
  1529. crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK);
  1530. crypto_aead_set_flags(ctx->fallback.aead,
  1531. crypto_aead_get_flags(authenc) &
  1532. CRYPTO_TFM_REQ_MASK);
  1533. crypto_aead_setkey(ctx->fallback.aead, key, keylen);
  1534. return 0;
  1535. }
  1536. static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
  1537. {
  1538. struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm));
  1539. return crypto_aead_setauthsize(ctx->fallback.aead, authsize);
  1540. }
  1541. static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc,
  1542. const u8 *key, unsigned int keylen)
  1543. {
  1544. struct algo_data ad = { 0 };
  1545. ad.ealg_id = SA_EALG_ID_AES_CBC;
  1546. ad.aalg_id = SA_AALG_ID_HMAC_SHA1;
  1547. ad.hash_size = SHA1_DIGEST_SIZE;
  1548. ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
  1549. return sa_aead_setkey(authenc, key, keylen, &ad);
  1550. }
  1551. static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc,
  1552. const u8 *key, unsigned int keylen)
  1553. {
  1554. struct algo_data ad = { 0 };
  1555. ad.ealg_id = SA_EALG_ID_AES_CBC;
  1556. ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256;
  1557. ad.hash_size = SHA256_DIGEST_SIZE;
  1558. ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
  1559. return sa_aead_setkey(authenc, key, keylen, &ad);
  1560. }
  1561. static int sa_aead_run(struct aead_request *req, u8 *iv, int enc)
  1562. {
  1563. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1564. struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
  1565. struct sa_req sa_req = { 0 };
  1566. size_t auth_size, enc_size;
  1567. enc_size = req->cryptlen;
  1568. auth_size = req->assoclen + req->cryptlen;
  1569. if (!enc) {
  1570. enc_size -= crypto_aead_authsize(tfm);
  1571. auth_size -= crypto_aead_authsize(tfm);
  1572. }
  1573. if (auth_size > SA_MAX_DATA_SZ ||
  1574. (auth_size >= SA_UNSAFE_DATA_SZ_MIN &&
  1575. auth_size <= SA_UNSAFE_DATA_SZ_MAX)) {
  1576. struct aead_request *subreq = aead_request_ctx(req);
  1577. int ret;
  1578. aead_request_set_tfm(subreq, ctx->fallback.aead);
  1579. aead_request_set_callback(subreq, req->base.flags,
  1580. req->base.complete, req->base.data);
  1581. aead_request_set_crypt(subreq, req->src, req->dst,
  1582. req->cryptlen, req->iv);
  1583. aead_request_set_ad(subreq, req->assoclen);
  1584. ret = enc ? crypto_aead_encrypt(subreq) :
  1585. crypto_aead_decrypt(subreq);
  1586. return ret;
  1587. }
  1588. sa_req.enc_offset = req->assoclen;
  1589. sa_req.enc_size = enc_size;
  1590. sa_req.auth_size = auth_size;
  1591. sa_req.size = auth_size;
  1592. sa_req.enc_iv = iv;
  1593. sa_req.type = CRYPTO_ALG_TYPE_AEAD;
  1594. sa_req.enc = enc;
  1595. sa_req.callback = sa_aead_dma_in_callback;
  1596. sa_req.mdata_size = 52;
  1597. sa_req.base = &req->base;
  1598. sa_req.ctx = ctx;
  1599. sa_req.src = req->src;
  1600. sa_req.dst = req->dst;
  1601. return sa_run(&sa_req);
  1602. }
  1603. /* AEAD algorithm encrypt interface function */
  1604. static int sa_aead_encrypt(struct aead_request *req)
  1605. {
  1606. return sa_aead_run(req, req->iv, 1);
  1607. }
  1608. /* AEAD algorithm decrypt interface function */
  1609. static int sa_aead_decrypt(struct aead_request *req)
  1610. {
  1611. return sa_aead_run(req, req->iv, 0);
  1612. }
  1613. static struct sa_alg_tmpl sa_algs[] = {
  1614. {
  1615. .type = CRYPTO_ALG_TYPE_SKCIPHER,
  1616. .alg.skcipher = {
  1617. .base.cra_name = "cbc(aes)",
  1618. .base.cra_driver_name = "cbc-aes-sa2ul",
  1619. .base.cra_priority = 30000,
  1620. .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
  1621. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1622. CRYPTO_ALG_ASYNC |
  1623. CRYPTO_ALG_NEED_FALLBACK,
  1624. .base.cra_blocksize = AES_BLOCK_SIZE,
  1625. .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1626. .base.cra_module = THIS_MODULE,
  1627. .init = sa_cipher_cra_init,
  1628. .exit = sa_cipher_cra_exit,
  1629. .min_keysize = AES_MIN_KEY_SIZE,
  1630. .max_keysize = AES_MAX_KEY_SIZE,
  1631. .ivsize = AES_BLOCK_SIZE,
  1632. .setkey = sa_aes_cbc_setkey,
  1633. .encrypt = sa_encrypt,
  1634. .decrypt = sa_decrypt,
  1635. }
  1636. },
  1637. {
  1638. .type = CRYPTO_ALG_TYPE_SKCIPHER,
  1639. .alg.skcipher = {
  1640. .base.cra_name = "ecb(aes)",
  1641. .base.cra_driver_name = "ecb-aes-sa2ul",
  1642. .base.cra_priority = 30000,
  1643. .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
  1644. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1645. CRYPTO_ALG_ASYNC |
  1646. CRYPTO_ALG_NEED_FALLBACK,
  1647. .base.cra_blocksize = AES_BLOCK_SIZE,
  1648. .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1649. .base.cra_module = THIS_MODULE,
  1650. .init = sa_cipher_cra_init,
  1651. .exit = sa_cipher_cra_exit,
  1652. .min_keysize = AES_MIN_KEY_SIZE,
  1653. .max_keysize = AES_MAX_KEY_SIZE,
  1654. .setkey = sa_aes_ecb_setkey,
  1655. .encrypt = sa_encrypt,
  1656. .decrypt = sa_decrypt,
  1657. }
  1658. },
  1659. {
  1660. .type = CRYPTO_ALG_TYPE_SKCIPHER,
  1661. .alg.skcipher = {
  1662. .base.cra_name = "cbc(des3_ede)",
  1663. .base.cra_driver_name = "cbc-des3-sa2ul",
  1664. .base.cra_priority = 30000,
  1665. .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
  1666. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1667. CRYPTO_ALG_ASYNC |
  1668. CRYPTO_ALG_NEED_FALLBACK,
  1669. .base.cra_blocksize = DES_BLOCK_SIZE,
  1670. .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1671. .base.cra_module = THIS_MODULE,
  1672. .init = sa_cipher_cra_init,
  1673. .exit = sa_cipher_cra_exit,
  1674. .min_keysize = 3 * DES_KEY_SIZE,
  1675. .max_keysize = 3 * DES_KEY_SIZE,
  1676. .ivsize = DES_BLOCK_SIZE,
  1677. .setkey = sa_3des_cbc_setkey,
  1678. .encrypt = sa_encrypt,
  1679. .decrypt = sa_decrypt,
  1680. }
  1681. },
  1682. {
  1683. .type = CRYPTO_ALG_TYPE_SKCIPHER,
  1684. .alg.skcipher = {
  1685. .base.cra_name = "ecb(des3_ede)",
  1686. .base.cra_driver_name = "ecb-des3-sa2ul",
  1687. .base.cra_priority = 30000,
  1688. .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
  1689. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1690. CRYPTO_ALG_ASYNC |
  1691. CRYPTO_ALG_NEED_FALLBACK,
  1692. .base.cra_blocksize = DES_BLOCK_SIZE,
  1693. .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1694. .base.cra_module = THIS_MODULE,
  1695. .init = sa_cipher_cra_init,
  1696. .exit = sa_cipher_cra_exit,
  1697. .min_keysize = 3 * DES_KEY_SIZE,
  1698. .max_keysize = 3 * DES_KEY_SIZE,
  1699. .setkey = sa_3des_ecb_setkey,
  1700. .encrypt = sa_encrypt,
  1701. .decrypt = sa_decrypt,
  1702. }
  1703. },
  1704. {
  1705. .type = CRYPTO_ALG_TYPE_AHASH,
  1706. .alg.ahash = {
  1707. .halg.base = {
  1708. .cra_name = "sha1",
  1709. .cra_driver_name = "sha1-sa2ul",
  1710. .cra_priority = 400,
  1711. .cra_flags = CRYPTO_ALG_TYPE_AHASH |
  1712. CRYPTO_ALG_ASYNC |
  1713. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1714. CRYPTO_ALG_NEED_FALLBACK,
  1715. .cra_blocksize = SHA1_BLOCK_SIZE,
  1716. .cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1717. .cra_module = THIS_MODULE,
  1718. .cra_init = sa_sha1_cra_init,
  1719. .cra_exit = sa_sha_cra_exit,
  1720. },
  1721. .halg.digestsize = SHA1_DIGEST_SIZE,
  1722. .halg.statesize = sizeof(struct sa_sha_req_ctx) +
  1723. sizeof(struct sha1_state),
  1724. .init = sa_sha_init,
  1725. .update = sa_sha_update,
  1726. .final = sa_sha_final,
  1727. .finup = sa_sha_finup,
  1728. .digest = sa_sha_digest,
  1729. .export = sa_sha_export,
  1730. .import = sa_sha_import,
  1731. },
  1732. },
  1733. {
  1734. .type = CRYPTO_ALG_TYPE_AHASH,
  1735. .alg.ahash = {
  1736. .halg.base = {
  1737. .cra_name = "sha256",
  1738. .cra_driver_name = "sha256-sa2ul",
  1739. .cra_priority = 400,
  1740. .cra_flags = CRYPTO_ALG_TYPE_AHASH |
  1741. CRYPTO_ALG_ASYNC |
  1742. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1743. CRYPTO_ALG_NEED_FALLBACK,
  1744. .cra_blocksize = SHA256_BLOCK_SIZE,
  1745. .cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1746. .cra_module = THIS_MODULE,
  1747. .cra_init = sa_sha256_cra_init,
  1748. .cra_exit = sa_sha_cra_exit,
  1749. },
  1750. .halg.digestsize = SHA256_DIGEST_SIZE,
  1751. .halg.statesize = sizeof(struct sa_sha_req_ctx) +
  1752. sizeof(struct sha256_state),
  1753. .init = sa_sha_init,
  1754. .update = sa_sha_update,
  1755. .final = sa_sha_final,
  1756. .finup = sa_sha_finup,
  1757. .digest = sa_sha_digest,
  1758. .export = sa_sha_export,
  1759. .import = sa_sha_import,
  1760. },
  1761. },
  1762. {
  1763. .type = CRYPTO_ALG_TYPE_AHASH,
  1764. .alg.ahash = {
  1765. .halg.base = {
  1766. .cra_name = "sha512",
  1767. .cra_driver_name = "sha512-sa2ul",
  1768. .cra_priority = 400,
  1769. .cra_flags = CRYPTO_ALG_TYPE_AHASH |
  1770. CRYPTO_ALG_ASYNC |
  1771. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1772. CRYPTO_ALG_NEED_FALLBACK,
  1773. .cra_blocksize = SHA512_BLOCK_SIZE,
  1774. .cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1775. .cra_module = THIS_MODULE,
  1776. .cra_init = sa_sha512_cra_init,
  1777. .cra_exit = sa_sha_cra_exit,
  1778. },
  1779. .halg.digestsize = SHA512_DIGEST_SIZE,
  1780. .halg.statesize = sizeof(struct sa_sha_req_ctx) +
  1781. sizeof(struct sha512_state),
  1782. .init = sa_sha_init,
  1783. .update = sa_sha_update,
  1784. .final = sa_sha_final,
  1785. .finup = sa_sha_finup,
  1786. .digest = sa_sha_digest,
  1787. .export = sa_sha_export,
  1788. .import = sa_sha_import,
  1789. },
  1790. },
  1791. {
  1792. .type = CRYPTO_ALG_TYPE_AEAD,
  1793. .alg.aead = {
  1794. .base = {
  1795. .cra_name = "authenc(hmac(sha1),cbc(aes))",
  1796. .cra_driver_name =
  1797. "authenc(hmac(sha1),cbc(aes))-sa2ul",
  1798. .cra_blocksize = AES_BLOCK_SIZE,
  1799. .cra_flags = CRYPTO_ALG_TYPE_AEAD |
  1800. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1801. CRYPTO_ALG_ASYNC |
  1802. CRYPTO_ALG_NEED_FALLBACK,
  1803. .cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1804. .cra_module = THIS_MODULE,
  1805. .cra_priority = 3000,
  1806. },
  1807. .ivsize = AES_BLOCK_SIZE,
  1808. .maxauthsize = SHA1_DIGEST_SIZE,
  1809. .init = sa_cra_init_aead_sha1,
  1810. .exit = sa_exit_tfm_aead,
  1811. .setkey = sa_aead_cbc_sha1_setkey,
  1812. .setauthsize = sa_aead_setauthsize,
  1813. .encrypt = sa_aead_encrypt,
  1814. .decrypt = sa_aead_decrypt,
  1815. },
  1816. },
  1817. {
  1818. .type = CRYPTO_ALG_TYPE_AEAD,
  1819. .alg.aead = {
  1820. .base = {
  1821. .cra_name = "authenc(hmac(sha256),cbc(aes))",
  1822. .cra_driver_name =
  1823. "authenc(hmac(sha256),cbc(aes))-sa2ul",
  1824. .cra_blocksize = AES_BLOCK_SIZE,
  1825. .cra_flags = CRYPTO_ALG_TYPE_AEAD |
  1826. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1827. CRYPTO_ALG_ASYNC |
  1828. CRYPTO_ALG_NEED_FALLBACK,
  1829. .cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1830. .cra_module = THIS_MODULE,
  1831. .cra_alignmask = 0,
  1832. .cra_priority = 3000,
  1833. },
  1834. .ivsize = AES_BLOCK_SIZE,
  1835. .maxauthsize = SHA256_DIGEST_SIZE,
  1836. .init = sa_cra_init_aead_sha256,
  1837. .exit = sa_exit_tfm_aead,
  1838. .setkey = sa_aead_cbc_sha256_setkey,
  1839. .setauthsize = sa_aead_setauthsize,
  1840. .encrypt = sa_aead_encrypt,
  1841. .decrypt = sa_aead_decrypt,
  1842. },
  1843. },
  1844. };
  1845. /* Register the algorithms in crypto framework */
  1846. static void sa_register_algos(const struct device *dev)
  1847. {
  1848. char *alg_name;
  1849. u32 type;
  1850. int i, err;
  1851. for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
  1852. type = sa_algs[i].type;
  1853. if (type == CRYPTO_ALG_TYPE_SKCIPHER) {
  1854. alg_name = sa_algs[i].alg.skcipher.base.cra_name;
  1855. err = crypto_register_skcipher(&sa_algs[i].alg.skcipher);
  1856. } else if (type == CRYPTO_ALG_TYPE_AHASH) {
  1857. alg_name = sa_algs[i].alg.ahash.halg.base.cra_name;
  1858. err = crypto_register_ahash(&sa_algs[i].alg.ahash);
  1859. } else if (type == CRYPTO_ALG_TYPE_AEAD) {
  1860. alg_name = sa_algs[i].alg.aead.base.cra_name;
  1861. err = crypto_register_aead(&sa_algs[i].alg.aead);
  1862. } else {
  1863. dev_err(dev,
  1864. "un-supported crypto algorithm (%d)",
  1865. sa_algs[i].type);
  1866. continue;
  1867. }
  1868. if (err)
  1869. dev_err(dev, "Failed to register '%s'\n", alg_name);
  1870. else
  1871. sa_algs[i].registered = true;
  1872. }
  1873. }
  1874. /* Unregister the algorithms in crypto framework */
  1875. static void sa_unregister_algos(const struct device *dev)
  1876. {
  1877. u32 type;
  1878. int i;
  1879. for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
  1880. type = sa_algs[i].type;
  1881. if (!sa_algs[i].registered)
  1882. continue;
  1883. if (type == CRYPTO_ALG_TYPE_SKCIPHER)
  1884. crypto_unregister_skcipher(&sa_algs[i].alg.skcipher);
  1885. else if (type == CRYPTO_ALG_TYPE_AHASH)
  1886. crypto_unregister_ahash(&sa_algs[i].alg.ahash);
  1887. else if (type == CRYPTO_ALG_TYPE_AEAD)
  1888. crypto_unregister_aead(&sa_algs[i].alg.aead);
  1889. sa_algs[i].registered = false;
  1890. }
  1891. }
  1892. static int sa_init_mem(struct sa_crypto_data *dev_data)
  1893. {
  1894. struct device *dev = &dev_data->pdev->dev;
  1895. /* Setup dma pool for security context buffers */
  1896. dev_data->sc_pool = dma_pool_create("keystone-sc", dev,
  1897. SA_CTX_MAX_SZ, 64, 0);
  1898. if (!dev_data->sc_pool) {
  1899. dev_err(dev, "Failed to create dma pool");
  1900. return -ENOMEM;
  1901. }
  1902. return 0;
  1903. }
  1904. static int sa_dma_init(struct sa_crypto_data *dd)
  1905. {
  1906. int ret;
  1907. struct dma_slave_config cfg;
  1908. dd->dma_rx1 = NULL;
  1909. dd->dma_tx = NULL;
  1910. dd->dma_rx2 = NULL;
  1911. ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48));
  1912. if (ret)
  1913. return ret;
  1914. dd->dma_rx1 = dma_request_chan(dd->dev, "rx1");
  1915. if (IS_ERR(dd->dma_rx1))
  1916. return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1),
  1917. "Unable to request rx1 DMA channel\n");
  1918. dd->dma_rx2 = dma_request_chan(dd->dev, "rx2");
  1919. if (IS_ERR(dd->dma_rx2)) {
  1920. ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2),
  1921. "Unable to request rx2 DMA channel\n");
  1922. goto err_dma_rx2;
  1923. }
  1924. dd->dma_tx = dma_request_chan(dd->dev, "tx");
  1925. if (IS_ERR(dd->dma_tx)) {
  1926. ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx),
  1927. "Unable to request tx DMA channel\n");
  1928. goto err_dma_tx;
  1929. }
  1930. memzero_explicit(&cfg, sizeof(cfg));
  1931. cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
  1932. cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
  1933. cfg.src_maxburst = 4;
  1934. cfg.dst_maxburst = 4;
  1935. ret = dmaengine_slave_config(dd->dma_rx1, &cfg);
  1936. if (ret) {
  1937. dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
  1938. ret);
  1939. goto err_dma_config;
  1940. }
  1941. ret = dmaengine_slave_config(dd->dma_rx2, &cfg);
  1942. if (ret) {
  1943. dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
  1944. ret);
  1945. goto err_dma_config;
  1946. }
  1947. ret = dmaengine_slave_config(dd->dma_tx, &cfg);
  1948. if (ret) {
  1949. dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
  1950. ret);
  1951. goto err_dma_config;
  1952. }
  1953. return 0;
  1954. err_dma_config:
  1955. dma_release_channel(dd->dma_tx);
  1956. err_dma_tx:
  1957. dma_release_channel(dd->dma_rx2);
  1958. err_dma_rx2:
  1959. dma_release_channel(dd->dma_rx1);
  1960. return ret;
  1961. }
  1962. static int sa_link_child(struct device *dev, void *data)
  1963. {
  1964. struct device *parent = data;
  1965. device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER);
  1966. return 0;
  1967. }
  1968. static int sa_ul_probe(struct platform_device *pdev)
  1969. {
  1970. struct device *dev = &pdev->dev;
  1971. struct device_node *node = dev->of_node;
  1972. struct resource *res;
  1973. static void __iomem *saul_base;
  1974. struct sa_crypto_data *dev_data;
  1975. u32 val;
  1976. int ret;
  1977. dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL);
  1978. if (!dev_data)
  1979. return -ENOMEM;
  1980. sa_k3_dev = dev;
  1981. dev_data->dev = dev;
  1982. dev_data->pdev = pdev;
  1983. platform_set_drvdata(pdev, dev_data);
  1984. dev_set_drvdata(sa_k3_dev, dev_data);
  1985. pm_runtime_enable(dev);
  1986. ret = pm_runtime_resume_and_get(dev);
  1987. if (ret < 0) {
  1988. dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__,
  1989. ret);
  1990. pm_runtime_disable(dev);
  1991. return ret;
  1992. }
  1993. sa_init_mem(dev_data);
  1994. ret = sa_dma_init(dev_data);
  1995. if (ret)
  1996. goto destroy_dma_pool;
  1997. spin_lock_init(&dev_data->scid_lock);
  1998. res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  1999. saul_base = devm_ioremap_resource(dev, res);
  2000. dev_data->base = saul_base;
  2001. val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN |
  2002. SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN |
  2003. SA_EEC_TRNG_EN;
  2004. writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL);
  2005. sa_register_algos(dev);
  2006. ret = of_platform_populate(node, NULL, NULL, &pdev->dev);
  2007. if (ret)
  2008. goto release_dma;
  2009. device_for_each_child(&pdev->dev, &pdev->dev, sa_link_child);
  2010. return 0;
  2011. release_dma:
  2012. sa_unregister_algos(&pdev->dev);
  2013. dma_release_channel(dev_data->dma_rx2);
  2014. dma_release_channel(dev_data->dma_rx1);
  2015. dma_release_channel(dev_data->dma_tx);
  2016. destroy_dma_pool:
  2017. dma_pool_destroy(dev_data->sc_pool);
  2018. pm_runtime_put_sync(&pdev->dev);
  2019. pm_runtime_disable(&pdev->dev);
  2020. return ret;
  2021. }
  2022. static int sa_ul_remove(struct platform_device *pdev)
  2023. {
  2024. struct sa_crypto_data *dev_data = platform_get_drvdata(pdev);
  2025. sa_unregister_algos(&pdev->dev);
  2026. dma_release_channel(dev_data->dma_rx2);
  2027. dma_release_channel(dev_data->dma_rx1);
  2028. dma_release_channel(dev_data->dma_tx);
  2029. dma_pool_destroy(dev_data->sc_pool);
  2030. platform_set_drvdata(pdev, NULL);
  2031. pm_runtime_put_sync(&pdev->dev);
  2032. pm_runtime_disable(&pdev->dev);
  2033. return 0;
  2034. }
  2035. static const struct of_device_id of_match[] = {
  2036. {.compatible = "ti,j721e-sa2ul",},
  2037. {.compatible = "ti,am654-sa2ul",},
  2038. {},
  2039. };
  2040. MODULE_DEVICE_TABLE(of, of_match);
  2041. static struct platform_driver sa_ul_driver = {
  2042. .probe = sa_ul_probe,
  2043. .remove = sa_ul_remove,
  2044. .driver = {
  2045. .name = "saul-crypto",
  2046. .of_match_table = of_match,
  2047. },
  2048. };
  2049. module_platform_driver(sa_ul_driver);
  2050. MODULE_LICENSE("GPL v2");