n2_core.c 51 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
  3. *
  4. * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net>
  5. */
  6. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  7. #include <linux/kernel.h>
  8. #include <linux/module.h>
  9. #include <linux/of.h>
  10. #include <linux/of_device.h>
  11. #include <linux/cpumask.h>
  12. #include <linux/slab.h>
  13. #include <linux/interrupt.h>
  14. #include <linux/crypto.h>
  15. #include <crypto/md5.h>
  16. #include <crypto/sha.h>
  17. #include <crypto/aes.h>
  18. #include <crypto/internal/des.h>
  19. #include <linux/mutex.h>
  20. #include <linux/delay.h>
  21. #include <linux/sched.h>
  22. #include <crypto/internal/hash.h>
  23. #include <crypto/internal/skcipher.h>
  24. #include <crypto/scatterwalk.h>
  25. #include <crypto/algapi.h>
  26. #include <asm/hypervisor.h>
  27. #include <asm/mdesc.h>
  28. #include "n2_core.h"
  29. #define DRV_MODULE_NAME "n2_crypto"
  30. #define DRV_MODULE_VERSION "0.2"
  31. #define DRV_MODULE_RELDATE "July 28, 2011"
  32. static const char version[] =
  33. DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
  34. MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
  35. MODULE_DESCRIPTION("Niagara2 Crypto driver");
  36. MODULE_LICENSE("GPL");
  37. MODULE_VERSION(DRV_MODULE_VERSION);
  38. #define N2_CRA_PRIORITY 200
  39. static DEFINE_MUTEX(spu_lock);
  40. struct spu_queue {
  41. cpumask_t sharing;
  42. unsigned long qhandle;
  43. spinlock_t lock;
  44. u8 q_type;
  45. void *q;
  46. unsigned long head;
  47. unsigned long tail;
  48. struct list_head jobs;
  49. unsigned long devino;
  50. char irq_name[32];
  51. unsigned int irq;
  52. struct list_head list;
  53. };
  54. struct spu_qreg {
  55. struct spu_queue *queue;
  56. unsigned long type;
  57. };
  58. static struct spu_queue **cpu_to_cwq;
  59. static struct spu_queue **cpu_to_mau;
  60. static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off)
  61. {
  62. if (q->q_type == HV_NCS_QTYPE_MAU) {
  63. off += MAU_ENTRY_SIZE;
  64. if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES))
  65. off = 0;
  66. } else {
  67. off += CWQ_ENTRY_SIZE;
  68. if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES))
  69. off = 0;
  70. }
  71. return off;
  72. }
  73. struct n2_request_common {
  74. struct list_head entry;
  75. unsigned int offset;
  76. };
  77. #define OFFSET_NOT_RUNNING (~(unsigned int)0)
  78. /* An async job request records the final tail value it used in
  79. * n2_request_common->offset, test to see if that offset is in
  80. * the range old_head, new_head, inclusive.
  81. */
  82. static inline bool job_finished(struct spu_queue *q, unsigned int offset,
  83. unsigned long old_head, unsigned long new_head)
  84. {
  85. if (old_head <= new_head) {
  86. if (offset > old_head && offset <= new_head)
  87. return true;
  88. } else {
  89. if (offset > old_head || offset <= new_head)
  90. return true;
  91. }
  92. return false;
  93. }
  94. /* When the HEAD marker is unequal to the actual HEAD, we get
  95. * a virtual device INO interrupt. We should process the
  96. * completed CWQ entries and adjust the HEAD marker to clear
  97. * the IRQ.
  98. */
  99. static irqreturn_t cwq_intr(int irq, void *dev_id)
  100. {
  101. unsigned long off, new_head, hv_ret;
  102. struct spu_queue *q = dev_id;
  103. pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
  104. smp_processor_id(), q->qhandle);
  105. spin_lock(&q->lock);
  106. hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head);
  107. pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
  108. smp_processor_id(), new_head, hv_ret);
  109. for (off = q->head; off != new_head; off = spu_next_offset(q, off)) {
  110. /* XXX ... XXX */
  111. }
  112. hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head);
  113. if (hv_ret == HV_EOK)
  114. q->head = new_head;
  115. spin_unlock(&q->lock);
  116. return IRQ_HANDLED;
  117. }
  118. static irqreturn_t mau_intr(int irq, void *dev_id)
  119. {
  120. struct spu_queue *q = dev_id;
  121. unsigned long head, hv_ret;
  122. spin_lock(&q->lock);
  123. pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
  124. smp_processor_id(), q->qhandle);
  125. hv_ret = sun4v_ncs_gethead(q->qhandle, &head);
  126. pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
  127. smp_processor_id(), head, hv_ret);
  128. sun4v_ncs_sethead_marker(q->qhandle, head);
  129. spin_unlock(&q->lock);
  130. return IRQ_HANDLED;
  131. }
  132. static void *spu_queue_next(struct spu_queue *q, void *cur)
  133. {
  134. return q->q + spu_next_offset(q, cur - q->q);
  135. }
  136. static int spu_queue_num_free(struct spu_queue *q)
  137. {
  138. unsigned long head = q->head;
  139. unsigned long tail = q->tail;
  140. unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES);
  141. unsigned long diff;
  142. if (head > tail)
  143. diff = head - tail;
  144. else
  145. diff = (end - tail) + head;
  146. return (diff / CWQ_ENTRY_SIZE) - 1;
  147. }
  148. static void *spu_queue_alloc(struct spu_queue *q, int num_entries)
  149. {
  150. int avail = spu_queue_num_free(q);
  151. if (avail >= num_entries)
  152. return q->q + q->tail;
  153. return NULL;
  154. }
  155. static unsigned long spu_queue_submit(struct spu_queue *q, void *last)
  156. {
  157. unsigned long hv_ret, new_tail;
  158. new_tail = spu_next_offset(q, last - q->q);
  159. hv_ret = sun4v_ncs_settail(q->qhandle, new_tail);
  160. if (hv_ret == HV_EOK)
  161. q->tail = new_tail;
  162. return hv_ret;
  163. }
  164. static u64 control_word_base(unsigned int len, unsigned int hmac_key_len,
  165. int enc_type, int auth_type,
  166. unsigned int hash_len,
  167. bool sfas, bool sob, bool eob, bool encrypt,
  168. int opcode)
  169. {
  170. u64 word = (len - 1) & CONTROL_LEN;
  171. word |= ((u64) opcode << CONTROL_OPCODE_SHIFT);
  172. word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT);
  173. word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT);
  174. if (sfas)
  175. word |= CONTROL_STORE_FINAL_AUTH_STATE;
  176. if (sob)
  177. word |= CONTROL_START_OF_BLOCK;
  178. if (eob)
  179. word |= CONTROL_END_OF_BLOCK;
  180. if (encrypt)
  181. word |= CONTROL_ENCRYPT;
  182. if (hmac_key_len)
  183. word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT;
  184. if (hash_len)
  185. word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT;
  186. return word;
  187. }
  188. #if 0
  189. static inline bool n2_should_run_async(struct spu_queue *qp, int this_len)
  190. {
  191. if (this_len >= 64 ||
  192. qp->head != qp->tail)
  193. return true;
  194. return false;
  195. }
  196. #endif
  197. struct n2_ahash_alg {
  198. struct list_head entry;
  199. const u8 *hash_zero;
  200. const u8 *hash_init;
  201. u8 hw_op_hashsz;
  202. u8 digest_size;
  203. u8 auth_type;
  204. u8 hmac_type;
  205. struct ahash_alg alg;
  206. };
  207. static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm)
  208. {
  209. struct crypto_alg *alg = tfm->__crt_alg;
  210. struct ahash_alg *ahash_alg;
  211. ahash_alg = container_of(alg, struct ahash_alg, halg.base);
  212. return container_of(ahash_alg, struct n2_ahash_alg, alg);
  213. }
  214. struct n2_hmac_alg {
  215. const char *child_alg;
  216. struct n2_ahash_alg derived;
  217. };
  218. static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm)
  219. {
  220. struct crypto_alg *alg = tfm->__crt_alg;
  221. struct ahash_alg *ahash_alg;
  222. ahash_alg = container_of(alg, struct ahash_alg, halg.base);
  223. return container_of(ahash_alg, struct n2_hmac_alg, derived.alg);
  224. }
  225. struct n2_hash_ctx {
  226. struct crypto_ahash *fallback_tfm;
  227. };
  228. #define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */
  229. struct n2_hmac_ctx {
  230. struct n2_hash_ctx base;
  231. struct crypto_shash *child_shash;
  232. int hash_key_len;
  233. unsigned char hash_key[N2_HASH_KEY_MAX];
  234. };
  235. struct n2_hash_req_ctx {
  236. union {
  237. struct md5_state md5;
  238. struct sha1_state sha1;
  239. struct sha256_state sha256;
  240. } u;
  241. struct ahash_request fallback_req;
  242. };
  243. static int n2_hash_async_init(struct ahash_request *req)
  244. {
  245. struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
  246. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  247. struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
  248. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
  249. rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  250. return crypto_ahash_init(&rctx->fallback_req);
  251. }
  252. static int n2_hash_async_update(struct ahash_request *req)
  253. {
  254. struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
  255. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  256. struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
  257. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
  258. rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  259. rctx->fallback_req.nbytes = req->nbytes;
  260. rctx->fallback_req.src = req->src;
  261. return crypto_ahash_update(&rctx->fallback_req);
  262. }
  263. static int n2_hash_async_final(struct ahash_request *req)
  264. {
  265. struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
  266. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  267. struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
  268. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
  269. rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  270. rctx->fallback_req.result = req->result;
  271. return crypto_ahash_final(&rctx->fallback_req);
  272. }
  273. static int n2_hash_async_finup(struct ahash_request *req)
  274. {
  275. struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
  276. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  277. struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
  278. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
  279. rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  280. rctx->fallback_req.nbytes = req->nbytes;
  281. rctx->fallback_req.src = req->src;
  282. rctx->fallback_req.result = req->result;
  283. return crypto_ahash_finup(&rctx->fallback_req);
  284. }
  285. static int n2_hash_async_noimport(struct ahash_request *req, const void *in)
  286. {
  287. return -ENOSYS;
  288. }
  289. static int n2_hash_async_noexport(struct ahash_request *req, void *out)
  290. {
  291. return -ENOSYS;
  292. }
  293. static int n2_hash_cra_init(struct crypto_tfm *tfm)
  294. {
  295. const char *fallback_driver_name = crypto_tfm_alg_name(tfm);
  296. struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
  297. struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  298. struct crypto_ahash *fallback_tfm;
  299. int err;
  300. fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
  301. CRYPTO_ALG_NEED_FALLBACK);
  302. if (IS_ERR(fallback_tfm)) {
  303. pr_warn("Fallback driver '%s' could not be loaded!\n",
  304. fallback_driver_name);
  305. err = PTR_ERR(fallback_tfm);
  306. goto out;
  307. }
  308. crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
  309. crypto_ahash_reqsize(fallback_tfm)));
  310. ctx->fallback_tfm = fallback_tfm;
  311. return 0;
  312. out:
  313. return err;
  314. }
  315. static void n2_hash_cra_exit(struct crypto_tfm *tfm)
  316. {
  317. struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
  318. struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  319. crypto_free_ahash(ctx->fallback_tfm);
  320. }
  321. static int n2_hmac_cra_init(struct crypto_tfm *tfm)
  322. {
  323. const char *fallback_driver_name = crypto_tfm_alg_name(tfm);
  324. struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
  325. struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
  326. struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm);
  327. struct crypto_ahash *fallback_tfm;
  328. struct crypto_shash *child_shash;
  329. int err;
  330. fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
  331. CRYPTO_ALG_NEED_FALLBACK);
  332. if (IS_ERR(fallback_tfm)) {
  333. pr_warn("Fallback driver '%s' could not be loaded!\n",
  334. fallback_driver_name);
  335. err = PTR_ERR(fallback_tfm);
  336. goto out;
  337. }
  338. child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0);
  339. if (IS_ERR(child_shash)) {
  340. pr_warn("Child shash '%s' could not be loaded!\n",
  341. n2alg->child_alg);
  342. err = PTR_ERR(child_shash);
  343. goto out_free_fallback;
  344. }
  345. crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
  346. crypto_ahash_reqsize(fallback_tfm)));
  347. ctx->child_shash = child_shash;
  348. ctx->base.fallback_tfm = fallback_tfm;
  349. return 0;
  350. out_free_fallback:
  351. crypto_free_ahash(fallback_tfm);
  352. out:
  353. return err;
  354. }
  355. static void n2_hmac_cra_exit(struct crypto_tfm *tfm)
  356. {
  357. struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
  358. struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
  359. crypto_free_ahash(ctx->base.fallback_tfm);
  360. crypto_free_shash(ctx->child_shash);
  361. }
  362. static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key,
  363. unsigned int keylen)
  364. {
  365. struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
  366. struct crypto_shash *child_shash = ctx->child_shash;
  367. struct crypto_ahash *fallback_tfm;
  368. int err, bs, ds;
  369. fallback_tfm = ctx->base.fallback_tfm;
  370. err = crypto_ahash_setkey(fallback_tfm, key, keylen);
  371. if (err)
  372. return err;
  373. bs = crypto_shash_blocksize(child_shash);
  374. ds = crypto_shash_digestsize(child_shash);
  375. BUG_ON(ds > N2_HASH_KEY_MAX);
  376. if (keylen > bs) {
  377. err = crypto_shash_tfm_digest(child_shash, key, keylen,
  378. ctx->hash_key);
  379. if (err)
  380. return err;
  381. keylen = ds;
  382. } else if (keylen <= N2_HASH_KEY_MAX)
  383. memcpy(ctx->hash_key, key, keylen);
  384. ctx->hash_key_len = keylen;
  385. return err;
  386. }
  387. static unsigned long wait_for_tail(struct spu_queue *qp)
  388. {
  389. unsigned long head, hv_ret;
  390. do {
  391. hv_ret = sun4v_ncs_gethead(qp->qhandle, &head);
  392. if (hv_ret != HV_EOK) {
  393. pr_err("Hypervisor error on gethead\n");
  394. break;
  395. }
  396. if (head == qp->tail) {
  397. qp->head = head;
  398. break;
  399. }
  400. } while (1);
  401. return hv_ret;
  402. }
  403. static unsigned long submit_and_wait_for_tail(struct spu_queue *qp,
  404. struct cwq_initial_entry *ent)
  405. {
  406. unsigned long hv_ret = spu_queue_submit(qp, ent);
  407. if (hv_ret == HV_EOK)
  408. hv_ret = wait_for_tail(qp);
  409. return hv_ret;
  410. }
  411. static int n2_do_async_digest(struct ahash_request *req,
  412. unsigned int auth_type, unsigned int digest_size,
  413. unsigned int result_size, void *hash_loc,
  414. unsigned long auth_key, unsigned int auth_key_len)
  415. {
  416. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  417. struct cwq_initial_entry *ent;
  418. struct crypto_hash_walk walk;
  419. struct spu_queue *qp;
  420. unsigned long flags;
  421. int err = -ENODEV;
  422. int nbytes, cpu;
  423. /* The total effective length of the operation may not
  424. * exceed 2^16.
  425. */
  426. if (unlikely(req->nbytes > (1 << 16))) {
  427. struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
  428. struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
  429. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
  430. rctx->fallback_req.base.flags =
  431. req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  432. rctx->fallback_req.nbytes = req->nbytes;
  433. rctx->fallback_req.src = req->src;
  434. rctx->fallback_req.result = req->result;
  435. return crypto_ahash_digest(&rctx->fallback_req);
  436. }
  437. nbytes = crypto_hash_walk_first(req, &walk);
  438. cpu = get_cpu();
  439. qp = cpu_to_cwq[cpu];
  440. if (!qp)
  441. goto out;
  442. spin_lock_irqsave(&qp->lock, flags);
  443. /* XXX can do better, improve this later by doing a by-hand scatterlist
  444. * XXX walk, etc.
  445. */
  446. ent = qp->q + qp->tail;
  447. ent->control = control_word_base(nbytes, auth_key_len, 0,
  448. auth_type, digest_size,
  449. false, true, false, false,
  450. OPCODE_INPLACE_BIT |
  451. OPCODE_AUTH_MAC);
  452. ent->src_addr = __pa(walk.data);
  453. ent->auth_key_addr = auth_key;
  454. ent->auth_iv_addr = __pa(hash_loc);
  455. ent->final_auth_state_addr = 0UL;
  456. ent->enc_key_addr = 0UL;
  457. ent->enc_iv_addr = 0UL;
  458. ent->dest_addr = __pa(hash_loc);
  459. nbytes = crypto_hash_walk_done(&walk, 0);
  460. while (nbytes > 0) {
  461. ent = spu_queue_next(qp, ent);
  462. ent->control = (nbytes - 1);
  463. ent->src_addr = __pa(walk.data);
  464. ent->auth_key_addr = 0UL;
  465. ent->auth_iv_addr = 0UL;
  466. ent->final_auth_state_addr = 0UL;
  467. ent->enc_key_addr = 0UL;
  468. ent->enc_iv_addr = 0UL;
  469. ent->dest_addr = 0UL;
  470. nbytes = crypto_hash_walk_done(&walk, 0);
  471. }
  472. ent->control |= CONTROL_END_OF_BLOCK;
  473. if (submit_and_wait_for_tail(qp, ent) != HV_EOK)
  474. err = -EINVAL;
  475. else
  476. err = 0;
  477. spin_unlock_irqrestore(&qp->lock, flags);
  478. if (!err)
  479. memcpy(req->result, hash_loc, result_size);
  480. out:
  481. put_cpu();
  482. return err;
  483. }
  484. static int n2_hash_async_digest(struct ahash_request *req)
  485. {
  486. struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm);
  487. struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
  488. int ds;
  489. ds = n2alg->digest_size;
  490. if (unlikely(req->nbytes == 0)) {
  491. memcpy(req->result, n2alg->hash_zero, ds);
  492. return 0;
  493. }
  494. memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz);
  495. return n2_do_async_digest(req, n2alg->auth_type,
  496. n2alg->hw_op_hashsz, ds,
  497. &rctx->u, 0UL, 0);
  498. }
  499. static int n2_hmac_async_digest(struct ahash_request *req)
  500. {
  501. struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm);
  502. struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
  503. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  504. struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
  505. int ds;
  506. ds = n2alg->derived.digest_size;
  507. if (unlikely(req->nbytes == 0) ||
  508. unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) {
  509. struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
  510. struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
  511. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
  512. rctx->fallback_req.base.flags =
  513. req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  514. rctx->fallback_req.nbytes = req->nbytes;
  515. rctx->fallback_req.src = req->src;
  516. rctx->fallback_req.result = req->result;
  517. return crypto_ahash_digest(&rctx->fallback_req);
  518. }
  519. memcpy(&rctx->u, n2alg->derived.hash_init,
  520. n2alg->derived.hw_op_hashsz);
  521. return n2_do_async_digest(req, n2alg->derived.hmac_type,
  522. n2alg->derived.hw_op_hashsz, ds,
  523. &rctx->u,
  524. __pa(&ctx->hash_key),
  525. ctx->hash_key_len);
  526. }
  527. struct n2_skcipher_context {
  528. int key_len;
  529. int enc_type;
  530. union {
  531. u8 aes[AES_MAX_KEY_SIZE];
  532. u8 des[DES_KEY_SIZE];
  533. u8 des3[3 * DES_KEY_SIZE];
  534. } key;
  535. };
  536. #define N2_CHUNK_ARR_LEN 16
  537. struct n2_crypto_chunk {
  538. struct list_head entry;
  539. unsigned long iv_paddr : 44;
  540. unsigned long arr_len : 20;
  541. unsigned long dest_paddr;
  542. unsigned long dest_final;
  543. struct {
  544. unsigned long src_paddr : 44;
  545. unsigned long src_len : 20;
  546. } arr[N2_CHUNK_ARR_LEN];
  547. };
  548. struct n2_request_context {
  549. struct skcipher_walk walk;
  550. struct list_head chunk_list;
  551. struct n2_crypto_chunk chunk;
  552. u8 temp_iv[16];
  553. };
  554. /* The SPU allows some level of flexibility for partial cipher blocks
  555. * being specified in a descriptor.
  556. *
  557. * It merely requires that every descriptor's length field is at least
  558. * as large as the cipher block size. This means that a cipher block
  559. * can span at most 2 descriptors. However, this does not allow a
  560. * partial block to span into the final descriptor as that would
  561. * violate the rule (since every descriptor's length must be at lest
  562. * the block size). So, for example, assuming an 8 byte block size:
  563. *
  564. * 0xe --> 0xa --> 0x8
  565. *
  566. * is a valid length sequence, whereas:
  567. *
  568. * 0xe --> 0xb --> 0x7
  569. *
  570. * is not a valid sequence.
  571. */
  572. struct n2_skcipher_alg {
  573. struct list_head entry;
  574. u8 enc_type;
  575. struct skcipher_alg skcipher;
  576. };
  577. static inline struct n2_skcipher_alg *n2_skcipher_alg(struct crypto_skcipher *tfm)
  578. {
  579. struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
  580. return container_of(alg, struct n2_skcipher_alg, skcipher);
  581. }
  582. struct n2_skcipher_request_context {
  583. struct skcipher_walk walk;
  584. };
  585. static int n2_aes_setkey(struct crypto_skcipher *skcipher, const u8 *key,
  586. unsigned int keylen)
  587. {
  588. struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
  589. struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm);
  590. struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher);
  591. ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK);
  592. switch (keylen) {
  593. case AES_KEYSIZE_128:
  594. ctx->enc_type |= ENC_TYPE_ALG_AES128;
  595. break;
  596. case AES_KEYSIZE_192:
  597. ctx->enc_type |= ENC_TYPE_ALG_AES192;
  598. break;
  599. case AES_KEYSIZE_256:
  600. ctx->enc_type |= ENC_TYPE_ALG_AES256;
  601. break;
  602. default:
  603. return -EINVAL;
  604. }
  605. ctx->key_len = keylen;
  606. memcpy(ctx->key.aes, key, keylen);
  607. return 0;
  608. }
  609. static int n2_des_setkey(struct crypto_skcipher *skcipher, const u8 *key,
  610. unsigned int keylen)
  611. {
  612. struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
  613. struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm);
  614. struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher);
  615. int err;
  616. err = verify_skcipher_des_key(skcipher, key);
  617. if (err)
  618. return err;
  619. ctx->enc_type = n2alg->enc_type;
  620. ctx->key_len = keylen;
  621. memcpy(ctx->key.des, key, keylen);
  622. return 0;
  623. }
  624. static int n2_3des_setkey(struct crypto_skcipher *skcipher, const u8 *key,
  625. unsigned int keylen)
  626. {
  627. struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
  628. struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm);
  629. struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher);
  630. int err;
  631. err = verify_skcipher_des3_key(skcipher, key);
  632. if (err)
  633. return err;
  634. ctx->enc_type = n2alg->enc_type;
  635. ctx->key_len = keylen;
  636. memcpy(ctx->key.des3, key, keylen);
  637. return 0;
  638. }
  639. static inline int skcipher_descriptor_len(int nbytes, unsigned int block_size)
  640. {
  641. int this_len = nbytes;
  642. this_len -= (nbytes & (block_size - 1));
  643. return this_len > (1 << 16) ? (1 << 16) : this_len;
  644. }
  645. static int __n2_crypt_chunk(struct crypto_skcipher *skcipher,
  646. struct n2_crypto_chunk *cp,
  647. struct spu_queue *qp, bool encrypt)
  648. {
  649. struct n2_skcipher_context *ctx = crypto_skcipher_ctx(skcipher);
  650. struct cwq_initial_entry *ent;
  651. bool in_place;
  652. int i;
  653. ent = spu_queue_alloc(qp, cp->arr_len);
  654. if (!ent) {
  655. pr_info("queue_alloc() of %d fails\n",
  656. cp->arr_len);
  657. return -EBUSY;
  658. }
  659. in_place = (cp->dest_paddr == cp->arr[0].src_paddr);
  660. ent->control = control_word_base(cp->arr[0].src_len,
  661. 0, ctx->enc_type, 0, 0,
  662. false, true, false, encrypt,
  663. OPCODE_ENCRYPT |
  664. (in_place ? OPCODE_INPLACE_BIT : 0));
  665. ent->src_addr = cp->arr[0].src_paddr;
  666. ent->auth_key_addr = 0UL;
  667. ent->auth_iv_addr = 0UL;
  668. ent->final_auth_state_addr = 0UL;
  669. ent->enc_key_addr = __pa(&ctx->key);
  670. ent->enc_iv_addr = cp->iv_paddr;
  671. ent->dest_addr = (in_place ? 0UL : cp->dest_paddr);
  672. for (i = 1; i < cp->arr_len; i++) {
  673. ent = spu_queue_next(qp, ent);
  674. ent->control = cp->arr[i].src_len - 1;
  675. ent->src_addr = cp->arr[i].src_paddr;
  676. ent->auth_key_addr = 0UL;
  677. ent->auth_iv_addr = 0UL;
  678. ent->final_auth_state_addr = 0UL;
  679. ent->enc_key_addr = 0UL;
  680. ent->enc_iv_addr = 0UL;
  681. ent->dest_addr = 0UL;
  682. }
  683. ent->control |= CONTROL_END_OF_BLOCK;
  684. return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0;
  685. }
  686. static int n2_compute_chunks(struct skcipher_request *req)
  687. {
  688. struct n2_request_context *rctx = skcipher_request_ctx(req);
  689. struct skcipher_walk *walk = &rctx->walk;
  690. struct n2_crypto_chunk *chunk;
  691. unsigned long dest_prev;
  692. unsigned int tot_len;
  693. bool prev_in_place;
  694. int err, nbytes;
  695. err = skcipher_walk_async(walk, req);
  696. if (err)
  697. return err;
  698. INIT_LIST_HEAD(&rctx->chunk_list);
  699. chunk = &rctx->chunk;
  700. INIT_LIST_HEAD(&chunk->entry);
  701. chunk->iv_paddr = 0UL;
  702. chunk->arr_len = 0;
  703. chunk->dest_paddr = 0UL;
  704. prev_in_place = false;
  705. dest_prev = ~0UL;
  706. tot_len = 0;
  707. while ((nbytes = walk->nbytes) != 0) {
  708. unsigned long dest_paddr, src_paddr;
  709. bool in_place;
  710. int this_len;
  711. src_paddr = (page_to_phys(walk->src.phys.page) +
  712. walk->src.phys.offset);
  713. dest_paddr = (page_to_phys(walk->dst.phys.page) +
  714. walk->dst.phys.offset);
  715. in_place = (src_paddr == dest_paddr);
  716. this_len = skcipher_descriptor_len(nbytes, walk->blocksize);
  717. if (chunk->arr_len != 0) {
  718. if (in_place != prev_in_place ||
  719. (!prev_in_place &&
  720. dest_paddr != dest_prev) ||
  721. chunk->arr_len == N2_CHUNK_ARR_LEN ||
  722. tot_len + this_len > (1 << 16)) {
  723. chunk->dest_final = dest_prev;
  724. list_add_tail(&chunk->entry,
  725. &rctx->chunk_list);
  726. chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
  727. if (!chunk) {
  728. err = -ENOMEM;
  729. break;
  730. }
  731. INIT_LIST_HEAD(&chunk->entry);
  732. }
  733. }
  734. if (chunk->arr_len == 0) {
  735. chunk->dest_paddr = dest_paddr;
  736. tot_len = 0;
  737. }
  738. chunk->arr[chunk->arr_len].src_paddr = src_paddr;
  739. chunk->arr[chunk->arr_len].src_len = this_len;
  740. chunk->arr_len++;
  741. dest_prev = dest_paddr + this_len;
  742. prev_in_place = in_place;
  743. tot_len += this_len;
  744. err = skcipher_walk_done(walk, nbytes - this_len);
  745. if (err)
  746. break;
  747. }
  748. if (!err && chunk->arr_len != 0) {
  749. chunk->dest_final = dest_prev;
  750. list_add_tail(&chunk->entry, &rctx->chunk_list);
  751. }
  752. return err;
  753. }
  754. static void n2_chunk_complete(struct skcipher_request *req, void *final_iv)
  755. {
  756. struct n2_request_context *rctx = skcipher_request_ctx(req);
  757. struct n2_crypto_chunk *c, *tmp;
  758. if (final_iv)
  759. memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize);
  760. list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
  761. list_del(&c->entry);
  762. if (unlikely(c != &rctx->chunk))
  763. kfree(c);
  764. }
  765. }
  766. static int n2_do_ecb(struct skcipher_request *req, bool encrypt)
  767. {
  768. struct n2_request_context *rctx = skcipher_request_ctx(req);
  769. struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  770. int err = n2_compute_chunks(req);
  771. struct n2_crypto_chunk *c, *tmp;
  772. unsigned long flags, hv_ret;
  773. struct spu_queue *qp;
  774. if (err)
  775. return err;
  776. qp = cpu_to_cwq[get_cpu()];
  777. err = -ENODEV;
  778. if (!qp)
  779. goto out;
  780. spin_lock_irqsave(&qp->lock, flags);
  781. list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
  782. err = __n2_crypt_chunk(tfm, c, qp, encrypt);
  783. if (err)
  784. break;
  785. list_del(&c->entry);
  786. if (unlikely(c != &rctx->chunk))
  787. kfree(c);
  788. }
  789. if (!err) {
  790. hv_ret = wait_for_tail(qp);
  791. if (hv_ret != HV_EOK)
  792. err = -EINVAL;
  793. }
  794. spin_unlock_irqrestore(&qp->lock, flags);
  795. out:
  796. put_cpu();
  797. n2_chunk_complete(req, NULL);
  798. return err;
  799. }
  800. static int n2_encrypt_ecb(struct skcipher_request *req)
  801. {
  802. return n2_do_ecb(req, true);
  803. }
  804. static int n2_decrypt_ecb(struct skcipher_request *req)
  805. {
  806. return n2_do_ecb(req, false);
  807. }
  808. static int n2_do_chaining(struct skcipher_request *req, bool encrypt)
  809. {
  810. struct n2_request_context *rctx = skcipher_request_ctx(req);
  811. struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  812. unsigned long flags, hv_ret, iv_paddr;
  813. int err = n2_compute_chunks(req);
  814. struct n2_crypto_chunk *c, *tmp;
  815. struct spu_queue *qp;
  816. void *final_iv_addr;
  817. final_iv_addr = NULL;
  818. if (err)
  819. return err;
  820. qp = cpu_to_cwq[get_cpu()];
  821. err = -ENODEV;
  822. if (!qp)
  823. goto out;
  824. spin_lock_irqsave(&qp->lock, flags);
  825. if (encrypt) {
  826. iv_paddr = __pa(rctx->walk.iv);
  827. list_for_each_entry_safe(c, tmp, &rctx->chunk_list,
  828. entry) {
  829. c->iv_paddr = iv_paddr;
  830. err = __n2_crypt_chunk(tfm, c, qp, true);
  831. if (err)
  832. break;
  833. iv_paddr = c->dest_final - rctx->walk.blocksize;
  834. list_del(&c->entry);
  835. if (unlikely(c != &rctx->chunk))
  836. kfree(c);
  837. }
  838. final_iv_addr = __va(iv_paddr);
  839. } else {
  840. list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list,
  841. entry) {
  842. if (c == &rctx->chunk) {
  843. iv_paddr = __pa(rctx->walk.iv);
  844. } else {
  845. iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr +
  846. tmp->arr[tmp->arr_len-1].src_len -
  847. rctx->walk.blocksize);
  848. }
  849. if (!final_iv_addr) {
  850. unsigned long pa;
  851. pa = (c->arr[c->arr_len-1].src_paddr +
  852. c->arr[c->arr_len-1].src_len -
  853. rctx->walk.blocksize);
  854. final_iv_addr = rctx->temp_iv;
  855. memcpy(rctx->temp_iv, __va(pa),
  856. rctx->walk.blocksize);
  857. }
  858. c->iv_paddr = iv_paddr;
  859. err = __n2_crypt_chunk(tfm, c, qp, false);
  860. if (err)
  861. break;
  862. list_del(&c->entry);
  863. if (unlikely(c != &rctx->chunk))
  864. kfree(c);
  865. }
  866. }
  867. if (!err) {
  868. hv_ret = wait_for_tail(qp);
  869. if (hv_ret != HV_EOK)
  870. err = -EINVAL;
  871. }
  872. spin_unlock_irqrestore(&qp->lock, flags);
  873. out:
  874. put_cpu();
  875. n2_chunk_complete(req, err ? NULL : final_iv_addr);
  876. return err;
  877. }
  878. static int n2_encrypt_chaining(struct skcipher_request *req)
  879. {
  880. return n2_do_chaining(req, true);
  881. }
  882. static int n2_decrypt_chaining(struct skcipher_request *req)
  883. {
  884. return n2_do_chaining(req, false);
  885. }
  886. struct n2_skcipher_tmpl {
  887. const char *name;
  888. const char *drv_name;
  889. u8 block_size;
  890. u8 enc_type;
  891. struct skcipher_alg skcipher;
  892. };
  893. static const struct n2_skcipher_tmpl skcipher_tmpls[] = {
  894. /* DES: ECB CBC and CFB are supported */
  895. { .name = "ecb(des)",
  896. .drv_name = "ecb-des",
  897. .block_size = DES_BLOCK_SIZE,
  898. .enc_type = (ENC_TYPE_ALG_DES |
  899. ENC_TYPE_CHAINING_ECB),
  900. .skcipher = {
  901. .min_keysize = DES_KEY_SIZE,
  902. .max_keysize = DES_KEY_SIZE,
  903. .setkey = n2_des_setkey,
  904. .encrypt = n2_encrypt_ecb,
  905. .decrypt = n2_decrypt_ecb,
  906. },
  907. },
  908. { .name = "cbc(des)",
  909. .drv_name = "cbc-des",
  910. .block_size = DES_BLOCK_SIZE,
  911. .enc_type = (ENC_TYPE_ALG_DES |
  912. ENC_TYPE_CHAINING_CBC),
  913. .skcipher = {
  914. .ivsize = DES_BLOCK_SIZE,
  915. .min_keysize = DES_KEY_SIZE,
  916. .max_keysize = DES_KEY_SIZE,
  917. .setkey = n2_des_setkey,
  918. .encrypt = n2_encrypt_chaining,
  919. .decrypt = n2_decrypt_chaining,
  920. },
  921. },
  922. { .name = "cfb(des)",
  923. .drv_name = "cfb-des",
  924. .block_size = DES_BLOCK_SIZE,
  925. .enc_type = (ENC_TYPE_ALG_DES |
  926. ENC_TYPE_CHAINING_CFB),
  927. .skcipher = {
  928. .min_keysize = DES_KEY_SIZE,
  929. .max_keysize = DES_KEY_SIZE,
  930. .setkey = n2_des_setkey,
  931. .encrypt = n2_encrypt_chaining,
  932. .decrypt = n2_decrypt_chaining,
  933. },
  934. },
  935. /* 3DES: ECB CBC and CFB are supported */
  936. { .name = "ecb(des3_ede)",
  937. .drv_name = "ecb-3des",
  938. .block_size = DES_BLOCK_SIZE,
  939. .enc_type = (ENC_TYPE_ALG_3DES |
  940. ENC_TYPE_CHAINING_ECB),
  941. .skcipher = {
  942. .min_keysize = 3 * DES_KEY_SIZE,
  943. .max_keysize = 3 * DES_KEY_SIZE,
  944. .setkey = n2_3des_setkey,
  945. .encrypt = n2_encrypt_ecb,
  946. .decrypt = n2_decrypt_ecb,
  947. },
  948. },
  949. { .name = "cbc(des3_ede)",
  950. .drv_name = "cbc-3des",
  951. .block_size = DES_BLOCK_SIZE,
  952. .enc_type = (ENC_TYPE_ALG_3DES |
  953. ENC_TYPE_CHAINING_CBC),
  954. .skcipher = {
  955. .ivsize = DES_BLOCK_SIZE,
  956. .min_keysize = 3 * DES_KEY_SIZE,
  957. .max_keysize = 3 * DES_KEY_SIZE,
  958. .setkey = n2_3des_setkey,
  959. .encrypt = n2_encrypt_chaining,
  960. .decrypt = n2_decrypt_chaining,
  961. },
  962. },
  963. { .name = "cfb(des3_ede)",
  964. .drv_name = "cfb-3des",
  965. .block_size = DES_BLOCK_SIZE,
  966. .enc_type = (ENC_TYPE_ALG_3DES |
  967. ENC_TYPE_CHAINING_CFB),
  968. .skcipher = {
  969. .min_keysize = 3 * DES_KEY_SIZE,
  970. .max_keysize = 3 * DES_KEY_SIZE,
  971. .setkey = n2_3des_setkey,
  972. .encrypt = n2_encrypt_chaining,
  973. .decrypt = n2_decrypt_chaining,
  974. },
  975. },
  976. /* AES: ECB CBC and CTR are supported */
  977. { .name = "ecb(aes)",
  978. .drv_name = "ecb-aes",
  979. .block_size = AES_BLOCK_SIZE,
  980. .enc_type = (ENC_TYPE_ALG_AES128 |
  981. ENC_TYPE_CHAINING_ECB),
  982. .skcipher = {
  983. .min_keysize = AES_MIN_KEY_SIZE,
  984. .max_keysize = AES_MAX_KEY_SIZE,
  985. .setkey = n2_aes_setkey,
  986. .encrypt = n2_encrypt_ecb,
  987. .decrypt = n2_decrypt_ecb,
  988. },
  989. },
  990. { .name = "cbc(aes)",
  991. .drv_name = "cbc-aes",
  992. .block_size = AES_BLOCK_SIZE,
  993. .enc_type = (ENC_TYPE_ALG_AES128 |
  994. ENC_TYPE_CHAINING_CBC),
  995. .skcipher = {
  996. .ivsize = AES_BLOCK_SIZE,
  997. .min_keysize = AES_MIN_KEY_SIZE,
  998. .max_keysize = AES_MAX_KEY_SIZE,
  999. .setkey = n2_aes_setkey,
  1000. .encrypt = n2_encrypt_chaining,
  1001. .decrypt = n2_decrypt_chaining,
  1002. },
  1003. },
  1004. { .name = "ctr(aes)",
  1005. .drv_name = "ctr-aes",
  1006. .block_size = AES_BLOCK_SIZE,
  1007. .enc_type = (ENC_TYPE_ALG_AES128 |
  1008. ENC_TYPE_CHAINING_COUNTER),
  1009. .skcipher = {
  1010. .ivsize = AES_BLOCK_SIZE,
  1011. .min_keysize = AES_MIN_KEY_SIZE,
  1012. .max_keysize = AES_MAX_KEY_SIZE,
  1013. .setkey = n2_aes_setkey,
  1014. .encrypt = n2_encrypt_chaining,
  1015. .decrypt = n2_encrypt_chaining,
  1016. },
  1017. },
  1018. };
  1019. #define NUM_CIPHER_TMPLS ARRAY_SIZE(skcipher_tmpls)
  1020. static LIST_HEAD(skcipher_algs);
  1021. struct n2_hash_tmpl {
  1022. const char *name;
  1023. const u8 *hash_zero;
  1024. const u8 *hash_init;
  1025. u8 hw_op_hashsz;
  1026. u8 digest_size;
  1027. u8 block_size;
  1028. u8 auth_type;
  1029. u8 hmac_type;
  1030. };
  1031. static const __le32 n2_md5_init[MD5_HASH_WORDS] = {
  1032. cpu_to_le32(MD5_H0),
  1033. cpu_to_le32(MD5_H1),
  1034. cpu_to_le32(MD5_H2),
  1035. cpu_to_le32(MD5_H3),
  1036. };
  1037. static const u32 n2_sha1_init[SHA1_DIGEST_SIZE / 4] = {
  1038. SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
  1039. };
  1040. static const u32 n2_sha256_init[SHA256_DIGEST_SIZE / 4] = {
  1041. SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
  1042. SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
  1043. };
  1044. static const u32 n2_sha224_init[SHA256_DIGEST_SIZE / 4] = {
  1045. SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
  1046. SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
  1047. };
  1048. static const struct n2_hash_tmpl hash_tmpls[] = {
  1049. { .name = "md5",
  1050. .hash_zero = md5_zero_message_hash,
  1051. .hash_init = (u8 *)n2_md5_init,
  1052. .auth_type = AUTH_TYPE_MD5,
  1053. .hmac_type = AUTH_TYPE_HMAC_MD5,
  1054. .hw_op_hashsz = MD5_DIGEST_SIZE,
  1055. .digest_size = MD5_DIGEST_SIZE,
  1056. .block_size = MD5_HMAC_BLOCK_SIZE },
  1057. { .name = "sha1",
  1058. .hash_zero = sha1_zero_message_hash,
  1059. .hash_init = (u8 *)n2_sha1_init,
  1060. .auth_type = AUTH_TYPE_SHA1,
  1061. .hmac_type = AUTH_TYPE_HMAC_SHA1,
  1062. .hw_op_hashsz = SHA1_DIGEST_SIZE,
  1063. .digest_size = SHA1_DIGEST_SIZE,
  1064. .block_size = SHA1_BLOCK_SIZE },
  1065. { .name = "sha256",
  1066. .hash_zero = sha256_zero_message_hash,
  1067. .hash_init = (u8 *)n2_sha256_init,
  1068. .auth_type = AUTH_TYPE_SHA256,
  1069. .hmac_type = AUTH_TYPE_HMAC_SHA256,
  1070. .hw_op_hashsz = SHA256_DIGEST_SIZE,
  1071. .digest_size = SHA256_DIGEST_SIZE,
  1072. .block_size = SHA256_BLOCK_SIZE },
  1073. { .name = "sha224",
  1074. .hash_zero = sha224_zero_message_hash,
  1075. .hash_init = (u8 *)n2_sha224_init,
  1076. .auth_type = AUTH_TYPE_SHA256,
  1077. .hmac_type = AUTH_TYPE_RESERVED,
  1078. .hw_op_hashsz = SHA256_DIGEST_SIZE,
  1079. .digest_size = SHA224_DIGEST_SIZE,
  1080. .block_size = SHA224_BLOCK_SIZE },
  1081. };
  1082. #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
  1083. static LIST_HEAD(ahash_algs);
  1084. static LIST_HEAD(hmac_algs);
  1085. static int algs_registered;
  1086. static void __n2_unregister_algs(void)
  1087. {
  1088. struct n2_skcipher_alg *skcipher, *skcipher_tmp;
  1089. struct n2_ahash_alg *alg, *alg_tmp;
  1090. struct n2_hmac_alg *hmac, *hmac_tmp;
  1091. list_for_each_entry_safe(skcipher, skcipher_tmp, &skcipher_algs, entry) {
  1092. crypto_unregister_skcipher(&skcipher->skcipher);
  1093. list_del(&skcipher->entry);
  1094. kfree(skcipher);
  1095. }
  1096. list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) {
  1097. crypto_unregister_ahash(&hmac->derived.alg);
  1098. list_del(&hmac->derived.entry);
  1099. kfree(hmac);
  1100. }
  1101. list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) {
  1102. crypto_unregister_ahash(&alg->alg);
  1103. list_del(&alg->entry);
  1104. kfree(alg);
  1105. }
  1106. }
  1107. static int n2_skcipher_init_tfm(struct crypto_skcipher *tfm)
  1108. {
  1109. crypto_skcipher_set_reqsize(tfm, sizeof(struct n2_request_context));
  1110. return 0;
  1111. }
  1112. static int __n2_register_one_skcipher(const struct n2_skcipher_tmpl *tmpl)
  1113. {
  1114. struct n2_skcipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
  1115. struct skcipher_alg *alg;
  1116. int err;
  1117. if (!p)
  1118. return -ENOMEM;
  1119. alg = &p->skcipher;
  1120. *alg = tmpl->skcipher;
  1121. snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
  1122. snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name);
  1123. alg->base.cra_priority = N2_CRA_PRIORITY;
  1124. alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC |
  1125. CRYPTO_ALG_ALLOCATES_MEMORY;
  1126. alg->base.cra_blocksize = tmpl->block_size;
  1127. p->enc_type = tmpl->enc_type;
  1128. alg->base.cra_ctxsize = sizeof(struct n2_skcipher_context);
  1129. alg->base.cra_module = THIS_MODULE;
  1130. alg->init = n2_skcipher_init_tfm;
  1131. list_add(&p->entry, &skcipher_algs);
  1132. err = crypto_register_skcipher(alg);
  1133. if (err) {
  1134. pr_err("%s alg registration failed\n", alg->base.cra_name);
  1135. list_del(&p->entry);
  1136. kfree(p);
  1137. } else {
  1138. pr_info("%s alg registered\n", alg->base.cra_name);
  1139. }
  1140. return err;
  1141. }
  1142. static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash)
  1143. {
  1144. struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
  1145. struct ahash_alg *ahash;
  1146. struct crypto_alg *base;
  1147. int err;
  1148. if (!p)
  1149. return -ENOMEM;
  1150. p->child_alg = n2ahash->alg.halg.base.cra_name;
  1151. memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg));
  1152. INIT_LIST_HEAD(&p->derived.entry);
  1153. ahash = &p->derived.alg;
  1154. ahash->digest = n2_hmac_async_digest;
  1155. ahash->setkey = n2_hmac_async_setkey;
  1156. base = &ahash->halg.base;
  1157. snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg);
  1158. snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg);
  1159. base->cra_ctxsize = sizeof(struct n2_hmac_ctx);
  1160. base->cra_init = n2_hmac_cra_init;
  1161. base->cra_exit = n2_hmac_cra_exit;
  1162. list_add(&p->derived.entry, &hmac_algs);
  1163. err = crypto_register_ahash(ahash);
  1164. if (err) {
  1165. pr_err("%s alg registration failed\n", base->cra_name);
  1166. list_del(&p->derived.entry);
  1167. kfree(p);
  1168. } else {
  1169. pr_info("%s alg registered\n", base->cra_name);
  1170. }
  1171. return err;
  1172. }
  1173. static int __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)
  1174. {
  1175. struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
  1176. struct hash_alg_common *halg;
  1177. struct crypto_alg *base;
  1178. struct ahash_alg *ahash;
  1179. int err;
  1180. if (!p)
  1181. return -ENOMEM;
  1182. p->hash_zero = tmpl->hash_zero;
  1183. p->hash_init = tmpl->hash_init;
  1184. p->auth_type = tmpl->auth_type;
  1185. p->hmac_type = tmpl->hmac_type;
  1186. p->hw_op_hashsz = tmpl->hw_op_hashsz;
  1187. p->digest_size = tmpl->digest_size;
  1188. ahash = &p->alg;
  1189. ahash->init = n2_hash_async_init;
  1190. ahash->update = n2_hash_async_update;
  1191. ahash->final = n2_hash_async_final;
  1192. ahash->finup = n2_hash_async_finup;
  1193. ahash->digest = n2_hash_async_digest;
  1194. ahash->export = n2_hash_async_noexport;
  1195. ahash->import = n2_hash_async_noimport;
  1196. halg = &ahash->halg;
  1197. halg->digestsize = tmpl->digest_size;
  1198. base = &halg->base;
  1199. snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
  1200. snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);
  1201. base->cra_priority = N2_CRA_PRIORITY;
  1202. base->cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  1203. CRYPTO_ALG_NEED_FALLBACK;
  1204. base->cra_blocksize = tmpl->block_size;
  1205. base->cra_ctxsize = sizeof(struct n2_hash_ctx);
  1206. base->cra_module = THIS_MODULE;
  1207. base->cra_init = n2_hash_cra_init;
  1208. base->cra_exit = n2_hash_cra_exit;
  1209. list_add(&p->entry, &ahash_algs);
  1210. err = crypto_register_ahash(ahash);
  1211. if (err) {
  1212. pr_err("%s alg registration failed\n", base->cra_name);
  1213. list_del(&p->entry);
  1214. kfree(p);
  1215. } else {
  1216. pr_info("%s alg registered\n", base->cra_name);
  1217. }
  1218. if (!err && p->hmac_type != AUTH_TYPE_RESERVED)
  1219. err = __n2_register_one_hmac(p);
  1220. return err;
  1221. }
  1222. static int n2_register_algs(void)
  1223. {
  1224. int i, err = 0;
  1225. mutex_lock(&spu_lock);
  1226. if (algs_registered++)
  1227. goto out;
  1228. for (i = 0; i < NUM_HASH_TMPLS; i++) {
  1229. err = __n2_register_one_ahash(&hash_tmpls[i]);
  1230. if (err) {
  1231. __n2_unregister_algs();
  1232. goto out;
  1233. }
  1234. }
  1235. for (i = 0; i < NUM_CIPHER_TMPLS; i++) {
  1236. err = __n2_register_one_skcipher(&skcipher_tmpls[i]);
  1237. if (err) {
  1238. __n2_unregister_algs();
  1239. goto out;
  1240. }
  1241. }
  1242. out:
  1243. mutex_unlock(&spu_lock);
  1244. return err;
  1245. }
  1246. static void n2_unregister_algs(void)
  1247. {
  1248. mutex_lock(&spu_lock);
  1249. if (!--algs_registered)
  1250. __n2_unregister_algs();
  1251. mutex_unlock(&spu_lock);
  1252. }
  1253. /* To map CWQ queues to interrupt sources, the hypervisor API provides
  1254. * a devino. This isn't very useful to us because all of the
  1255. * interrupts listed in the device_node have been translated to
  1256. * Linux virtual IRQ cookie numbers.
  1257. *
  1258. * So we have to back-translate, going through the 'intr' and 'ino'
  1259. * property tables of the n2cp MDESC node, matching it with the OF
  1260. * 'interrupts' property entries, in order to to figure out which
  1261. * devino goes to which already-translated IRQ.
  1262. */
  1263. static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip,
  1264. unsigned long dev_ino)
  1265. {
  1266. const unsigned int *dev_intrs;
  1267. unsigned int intr;
  1268. int i;
  1269. for (i = 0; i < ip->num_intrs; i++) {
  1270. if (ip->ino_table[i].ino == dev_ino)
  1271. break;
  1272. }
  1273. if (i == ip->num_intrs)
  1274. return -ENODEV;
  1275. intr = ip->ino_table[i].intr;
  1276. dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL);
  1277. if (!dev_intrs)
  1278. return -ENODEV;
  1279. for (i = 0; i < dev->archdata.num_irqs; i++) {
  1280. if (dev_intrs[i] == intr)
  1281. return i;
  1282. }
  1283. return -ENODEV;
  1284. }
  1285. static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip,
  1286. const char *irq_name, struct spu_queue *p,
  1287. irq_handler_t handler)
  1288. {
  1289. unsigned long herr;
  1290. int index;
  1291. herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino);
  1292. if (herr)
  1293. return -EINVAL;
  1294. index = find_devino_index(dev, ip, p->devino);
  1295. if (index < 0)
  1296. return index;
  1297. p->irq = dev->archdata.irqs[index];
  1298. sprintf(p->irq_name, "%s-%d", irq_name, index);
  1299. return request_irq(p->irq, handler, 0, p->irq_name, p);
  1300. }
  1301. static struct kmem_cache *queue_cache[2];
  1302. static void *new_queue(unsigned long q_type)
  1303. {
  1304. return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL);
  1305. }
  1306. static void free_queue(void *p, unsigned long q_type)
  1307. {
  1308. kmem_cache_free(queue_cache[q_type - 1], p);
  1309. }
  1310. static int queue_cache_init(void)
  1311. {
  1312. if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
  1313. queue_cache[HV_NCS_QTYPE_MAU - 1] =
  1314. kmem_cache_create("mau_queue",
  1315. (MAU_NUM_ENTRIES *
  1316. MAU_ENTRY_SIZE),
  1317. MAU_ENTRY_SIZE, 0, NULL);
  1318. if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
  1319. return -ENOMEM;
  1320. if (!queue_cache[HV_NCS_QTYPE_CWQ - 1])
  1321. queue_cache[HV_NCS_QTYPE_CWQ - 1] =
  1322. kmem_cache_create("cwq_queue",
  1323. (CWQ_NUM_ENTRIES *
  1324. CWQ_ENTRY_SIZE),
  1325. CWQ_ENTRY_SIZE, 0, NULL);
  1326. if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) {
  1327. kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
  1328. queue_cache[HV_NCS_QTYPE_MAU - 1] = NULL;
  1329. return -ENOMEM;
  1330. }
  1331. return 0;
  1332. }
  1333. static void queue_cache_destroy(void)
  1334. {
  1335. kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
  1336. kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]);
  1337. queue_cache[HV_NCS_QTYPE_MAU - 1] = NULL;
  1338. queue_cache[HV_NCS_QTYPE_CWQ - 1] = NULL;
  1339. }
  1340. static long spu_queue_register_workfn(void *arg)
  1341. {
  1342. struct spu_qreg *qr = arg;
  1343. struct spu_queue *p = qr->queue;
  1344. unsigned long q_type = qr->type;
  1345. unsigned long hv_ret;
  1346. hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q),
  1347. CWQ_NUM_ENTRIES, &p->qhandle);
  1348. if (!hv_ret)
  1349. sun4v_ncs_sethead_marker(p->qhandle, 0);
  1350. return hv_ret ? -EINVAL : 0;
  1351. }
  1352. static int spu_queue_register(struct spu_queue *p, unsigned long q_type)
  1353. {
  1354. int cpu = cpumask_any_and(&p->sharing, cpu_online_mask);
  1355. struct spu_qreg qr = { .queue = p, .type = q_type };
  1356. return work_on_cpu_safe(cpu, spu_queue_register_workfn, &qr);
  1357. }
  1358. static int spu_queue_setup(struct spu_queue *p)
  1359. {
  1360. int err;
  1361. p->q = new_queue(p->q_type);
  1362. if (!p->q)
  1363. return -ENOMEM;
  1364. err = spu_queue_register(p, p->q_type);
  1365. if (err) {
  1366. free_queue(p->q, p->q_type);
  1367. p->q = NULL;
  1368. }
  1369. return err;
  1370. }
  1371. static void spu_queue_destroy(struct spu_queue *p)
  1372. {
  1373. unsigned long hv_ret;
  1374. if (!p->q)
  1375. return;
  1376. hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle);
  1377. if (!hv_ret)
  1378. free_queue(p->q, p->q_type);
  1379. }
  1380. static void spu_list_destroy(struct list_head *list)
  1381. {
  1382. struct spu_queue *p, *n;
  1383. list_for_each_entry_safe(p, n, list, list) {
  1384. int i;
  1385. for (i = 0; i < NR_CPUS; i++) {
  1386. if (cpu_to_cwq[i] == p)
  1387. cpu_to_cwq[i] = NULL;
  1388. }
  1389. if (p->irq) {
  1390. free_irq(p->irq, p);
  1391. p->irq = 0;
  1392. }
  1393. spu_queue_destroy(p);
  1394. list_del(&p->list);
  1395. kfree(p);
  1396. }
  1397. }
  1398. /* Walk the backward arcs of a CWQ 'exec-unit' node,
  1399. * gathering cpu membership information.
  1400. */
  1401. static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
  1402. struct platform_device *dev,
  1403. u64 node, struct spu_queue *p,
  1404. struct spu_queue **table)
  1405. {
  1406. u64 arc;
  1407. mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) {
  1408. u64 tgt = mdesc_arc_target(mdesc, arc);
  1409. const char *name = mdesc_node_name(mdesc, tgt);
  1410. const u64 *id;
  1411. if (strcmp(name, "cpu"))
  1412. continue;
  1413. id = mdesc_get_property(mdesc, tgt, "id", NULL);
  1414. if (table[*id] != NULL) {
  1415. dev_err(&dev->dev, "%pOF: SPU cpu slot already set.\n",
  1416. dev->dev.of_node);
  1417. return -EINVAL;
  1418. }
  1419. cpumask_set_cpu(*id, &p->sharing);
  1420. table[*id] = p;
  1421. }
  1422. return 0;
  1423. }
  1424. /* Process an 'exec-unit' MDESC node of type 'cwq'. */
  1425. static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
  1426. struct platform_device *dev, struct mdesc_handle *mdesc,
  1427. u64 node, const char *iname, unsigned long q_type,
  1428. irq_handler_t handler, struct spu_queue **table)
  1429. {
  1430. struct spu_queue *p;
  1431. int err;
  1432. p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);
  1433. if (!p) {
  1434. dev_err(&dev->dev, "%pOF: Could not allocate SPU queue.\n",
  1435. dev->dev.of_node);
  1436. return -ENOMEM;
  1437. }
  1438. cpumask_clear(&p->sharing);
  1439. spin_lock_init(&p->lock);
  1440. p->q_type = q_type;
  1441. INIT_LIST_HEAD(&p->jobs);
  1442. list_add(&p->list, list);
  1443. err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table);
  1444. if (err)
  1445. return err;
  1446. err = spu_queue_setup(p);
  1447. if (err)
  1448. return err;
  1449. return spu_map_ino(dev, ip, iname, p, handler);
  1450. }
  1451. static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev,
  1452. struct spu_mdesc_info *ip, struct list_head *list,
  1453. const char *exec_name, unsigned long q_type,
  1454. irq_handler_t handler, struct spu_queue **table)
  1455. {
  1456. int err = 0;
  1457. u64 node;
  1458. mdesc_for_each_node_by_name(mdesc, node, "exec-unit") {
  1459. const char *type;
  1460. type = mdesc_get_property(mdesc, node, "type", NULL);
  1461. if (!type || strcmp(type, exec_name))
  1462. continue;
  1463. err = handle_exec_unit(ip, list, dev, mdesc, node,
  1464. exec_name, q_type, handler, table);
  1465. if (err) {
  1466. spu_list_destroy(list);
  1467. break;
  1468. }
  1469. }
  1470. return err;
  1471. }
  1472. static int get_irq_props(struct mdesc_handle *mdesc, u64 node,
  1473. struct spu_mdesc_info *ip)
  1474. {
  1475. const u64 *ino;
  1476. int ino_len;
  1477. int i;
  1478. ino = mdesc_get_property(mdesc, node, "ino", &ino_len);
  1479. if (!ino) {
  1480. printk("NO 'ino'\n");
  1481. return -ENODEV;
  1482. }
  1483. ip->num_intrs = ino_len / sizeof(u64);
  1484. ip->ino_table = kzalloc((sizeof(struct ino_blob) *
  1485. ip->num_intrs),
  1486. GFP_KERNEL);
  1487. if (!ip->ino_table)
  1488. return -ENOMEM;
  1489. for (i = 0; i < ip->num_intrs; i++) {
  1490. struct ino_blob *b = &ip->ino_table[i];
  1491. b->intr = i + 1;
  1492. b->ino = ino[i];
  1493. }
  1494. return 0;
  1495. }
  1496. static int grab_mdesc_irq_props(struct mdesc_handle *mdesc,
  1497. struct platform_device *dev,
  1498. struct spu_mdesc_info *ip,
  1499. const char *node_name)
  1500. {
  1501. const unsigned int *reg;
  1502. u64 node;
  1503. reg = of_get_property(dev->dev.of_node, "reg", NULL);
  1504. if (!reg)
  1505. return -ENODEV;
  1506. mdesc_for_each_node_by_name(mdesc, node, "virtual-device") {
  1507. const char *name;
  1508. const u64 *chdl;
  1509. name = mdesc_get_property(mdesc, node, "name", NULL);
  1510. if (!name || strcmp(name, node_name))
  1511. continue;
  1512. chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL);
  1513. if (!chdl || (*chdl != *reg))
  1514. continue;
  1515. ip->cfg_handle = *chdl;
  1516. return get_irq_props(mdesc, node, ip);
  1517. }
  1518. return -ENODEV;
  1519. }
  1520. static unsigned long n2_spu_hvapi_major;
  1521. static unsigned long n2_spu_hvapi_minor;
  1522. static int n2_spu_hvapi_register(void)
  1523. {
  1524. int err;
  1525. n2_spu_hvapi_major = 2;
  1526. n2_spu_hvapi_minor = 0;
  1527. err = sun4v_hvapi_register(HV_GRP_NCS,
  1528. n2_spu_hvapi_major,
  1529. &n2_spu_hvapi_minor);
  1530. if (!err)
  1531. pr_info("Registered NCS HVAPI version %lu.%lu\n",
  1532. n2_spu_hvapi_major,
  1533. n2_spu_hvapi_minor);
  1534. return err;
  1535. }
  1536. static void n2_spu_hvapi_unregister(void)
  1537. {
  1538. sun4v_hvapi_unregister(HV_GRP_NCS);
  1539. }
  1540. static int global_ref;
  1541. static int grab_global_resources(void)
  1542. {
  1543. int err = 0;
  1544. mutex_lock(&spu_lock);
  1545. if (global_ref++)
  1546. goto out;
  1547. err = n2_spu_hvapi_register();
  1548. if (err)
  1549. goto out;
  1550. err = queue_cache_init();
  1551. if (err)
  1552. goto out_hvapi_release;
  1553. err = -ENOMEM;
  1554. cpu_to_cwq = kcalloc(NR_CPUS, sizeof(struct spu_queue *),
  1555. GFP_KERNEL);
  1556. if (!cpu_to_cwq)
  1557. goto out_queue_cache_destroy;
  1558. cpu_to_mau = kcalloc(NR_CPUS, sizeof(struct spu_queue *),
  1559. GFP_KERNEL);
  1560. if (!cpu_to_mau)
  1561. goto out_free_cwq_table;
  1562. err = 0;
  1563. out:
  1564. if (err)
  1565. global_ref--;
  1566. mutex_unlock(&spu_lock);
  1567. return err;
  1568. out_free_cwq_table:
  1569. kfree(cpu_to_cwq);
  1570. cpu_to_cwq = NULL;
  1571. out_queue_cache_destroy:
  1572. queue_cache_destroy();
  1573. out_hvapi_release:
  1574. n2_spu_hvapi_unregister();
  1575. goto out;
  1576. }
  1577. static void release_global_resources(void)
  1578. {
  1579. mutex_lock(&spu_lock);
  1580. if (!--global_ref) {
  1581. kfree(cpu_to_cwq);
  1582. cpu_to_cwq = NULL;
  1583. kfree(cpu_to_mau);
  1584. cpu_to_mau = NULL;
  1585. queue_cache_destroy();
  1586. n2_spu_hvapi_unregister();
  1587. }
  1588. mutex_unlock(&spu_lock);
  1589. }
  1590. static struct n2_crypto *alloc_n2cp(void)
  1591. {
  1592. struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL);
  1593. if (np)
  1594. INIT_LIST_HEAD(&np->cwq_list);
  1595. return np;
  1596. }
  1597. static void free_n2cp(struct n2_crypto *np)
  1598. {
  1599. kfree(np->cwq_info.ino_table);
  1600. np->cwq_info.ino_table = NULL;
  1601. kfree(np);
  1602. }
  1603. static void n2_spu_driver_version(void)
  1604. {
  1605. static int n2_spu_version_printed;
  1606. if (n2_spu_version_printed++ == 0)
  1607. pr_info("%s", version);
  1608. }
  1609. static int n2_crypto_probe(struct platform_device *dev)
  1610. {
  1611. struct mdesc_handle *mdesc;
  1612. struct n2_crypto *np;
  1613. int err;
  1614. n2_spu_driver_version();
  1615. pr_info("Found N2CP at %pOF\n", dev->dev.of_node);
  1616. np = alloc_n2cp();
  1617. if (!np) {
  1618. dev_err(&dev->dev, "%pOF: Unable to allocate n2cp.\n",
  1619. dev->dev.of_node);
  1620. return -ENOMEM;
  1621. }
  1622. err = grab_global_resources();
  1623. if (err) {
  1624. dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n",
  1625. dev->dev.of_node);
  1626. goto out_free_n2cp;
  1627. }
  1628. mdesc = mdesc_grab();
  1629. if (!mdesc) {
  1630. dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n",
  1631. dev->dev.of_node);
  1632. err = -ENODEV;
  1633. goto out_free_global;
  1634. }
  1635. err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");
  1636. if (err) {
  1637. dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n",
  1638. dev->dev.of_node);
  1639. mdesc_release(mdesc);
  1640. goto out_free_global;
  1641. }
  1642. err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list,
  1643. "cwq", HV_NCS_QTYPE_CWQ, cwq_intr,
  1644. cpu_to_cwq);
  1645. mdesc_release(mdesc);
  1646. if (err) {
  1647. dev_err(&dev->dev, "%pOF: CWQ MDESC scan failed.\n",
  1648. dev->dev.of_node);
  1649. goto out_free_global;
  1650. }
  1651. err = n2_register_algs();
  1652. if (err) {
  1653. dev_err(&dev->dev, "%pOF: Unable to register algorithms.\n",
  1654. dev->dev.of_node);
  1655. goto out_free_spu_list;
  1656. }
  1657. dev_set_drvdata(&dev->dev, np);
  1658. return 0;
  1659. out_free_spu_list:
  1660. spu_list_destroy(&np->cwq_list);
  1661. out_free_global:
  1662. release_global_resources();
  1663. out_free_n2cp:
  1664. free_n2cp(np);
  1665. return err;
  1666. }
  1667. static int n2_crypto_remove(struct platform_device *dev)
  1668. {
  1669. struct n2_crypto *np = dev_get_drvdata(&dev->dev);
  1670. n2_unregister_algs();
  1671. spu_list_destroy(&np->cwq_list);
  1672. release_global_resources();
  1673. free_n2cp(np);
  1674. return 0;
  1675. }
  1676. static struct n2_mau *alloc_ncp(void)
  1677. {
  1678. struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL);
  1679. if (mp)
  1680. INIT_LIST_HEAD(&mp->mau_list);
  1681. return mp;
  1682. }
  1683. static void free_ncp(struct n2_mau *mp)
  1684. {
  1685. kfree(mp->mau_info.ino_table);
  1686. mp->mau_info.ino_table = NULL;
  1687. kfree(mp);
  1688. }
  1689. static int n2_mau_probe(struct platform_device *dev)
  1690. {
  1691. struct mdesc_handle *mdesc;
  1692. struct n2_mau *mp;
  1693. int err;
  1694. n2_spu_driver_version();
  1695. pr_info("Found NCP at %pOF\n", dev->dev.of_node);
  1696. mp = alloc_ncp();
  1697. if (!mp) {
  1698. dev_err(&dev->dev, "%pOF: Unable to allocate ncp.\n",
  1699. dev->dev.of_node);
  1700. return -ENOMEM;
  1701. }
  1702. err = grab_global_resources();
  1703. if (err) {
  1704. dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n",
  1705. dev->dev.of_node);
  1706. goto out_free_ncp;
  1707. }
  1708. mdesc = mdesc_grab();
  1709. if (!mdesc) {
  1710. dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n",
  1711. dev->dev.of_node);
  1712. err = -ENODEV;
  1713. goto out_free_global;
  1714. }
  1715. err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");
  1716. if (err) {
  1717. dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n",
  1718. dev->dev.of_node);
  1719. mdesc_release(mdesc);
  1720. goto out_free_global;
  1721. }
  1722. err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list,
  1723. "mau", HV_NCS_QTYPE_MAU, mau_intr,
  1724. cpu_to_mau);
  1725. mdesc_release(mdesc);
  1726. if (err) {
  1727. dev_err(&dev->dev, "%pOF: MAU MDESC scan failed.\n",
  1728. dev->dev.of_node);
  1729. goto out_free_global;
  1730. }
  1731. dev_set_drvdata(&dev->dev, mp);
  1732. return 0;
  1733. out_free_global:
  1734. release_global_resources();
  1735. out_free_ncp:
  1736. free_ncp(mp);
  1737. return err;
  1738. }
  1739. static int n2_mau_remove(struct platform_device *dev)
  1740. {
  1741. struct n2_mau *mp = dev_get_drvdata(&dev->dev);
  1742. spu_list_destroy(&mp->mau_list);
  1743. release_global_resources();
  1744. free_ncp(mp);
  1745. return 0;
  1746. }
  1747. static const struct of_device_id n2_crypto_match[] = {
  1748. {
  1749. .name = "n2cp",
  1750. .compatible = "SUNW,n2-cwq",
  1751. },
  1752. {
  1753. .name = "n2cp",
  1754. .compatible = "SUNW,vf-cwq",
  1755. },
  1756. {
  1757. .name = "n2cp",
  1758. .compatible = "SUNW,kt-cwq",
  1759. },
  1760. {},
  1761. };
  1762. MODULE_DEVICE_TABLE(of, n2_crypto_match);
  1763. static struct platform_driver n2_crypto_driver = {
  1764. .driver = {
  1765. .name = "n2cp",
  1766. .of_match_table = n2_crypto_match,
  1767. },
  1768. .probe = n2_crypto_probe,
  1769. .remove = n2_crypto_remove,
  1770. };
  1771. static const struct of_device_id n2_mau_match[] = {
  1772. {
  1773. .name = "ncp",
  1774. .compatible = "SUNW,n2-mau",
  1775. },
  1776. {
  1777. .name = "ncp",
  1778. .compatible = "SUNW,vf-mau",
  1779. },
  1780. {
  1781. .name = "ncp",
  1782. .compatible = "SUNW,kt-mau",
  1783. },
  1784. {},
  1785. };
  1786. MODULE_DEVICE_TABLE(of, n2_mau_match);
  1787. static struct platform_driver n2_mau_driver = {
  1788. .driver = {
  1789. .name = "ncp",
  1790. .of_match_table = n2_mau_match,
  1791. },
  1792. .probe = n2_mau_probe,
  1793. .remove = n2_mau_remove,
  1794. };
  1795. static struct platform_driver * const drivers[] = {
  1796. &n2_crypto_driver,
  1797. &n2_mau_driver,
  1798. };
  1799. static int __init n2_init(void)
  1800. {
  1801. return platform_register_drivers(drivers, ARRAY_SIZE(drivers));
  1802. }
  1803. static void __exit n2_exit(void)
  1804. {
  1805. platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
  1806. }
  1807. module_init(n2_init);
  1808. module_exit(n2_exit);