mad.c 88 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171
  1. /*
  2. * Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved.
  3. * Copyright (c) 2005 Intel Corporation. All rights reserved.
  4. * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved.
  5. * Copyright (c) 2009 HNR Consulting. All rights reserved.
  6. * Copyright (c) 2014,2018 Intel Corporation. All rights reserved.
  7. *
  8. * This software is available to you under a choice of one of two
  9. * licenses. You may choose to be licensed under the terms of the GNU
  10. * General Public License (GPL) Version 2, available from the file
  11. * COPYING in the main directory of this source tree, or the
  12. * OpenIB.org BSD license below:
  13. *
  14. * Redistribution and use in source and binary forms, with or
  15. * without modification, are permitted provided that the following
  16. * conditions are met:
  17. *
  18. * - Redistributions of source code must retain the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer.
  21. *
  22. * - Redistributions in binary form must reproduce the above
  23. * copyright notice, this list of conditions and the following
  24. * disclaimer in the documentation and/or other materials
  25. * provided with the distribution.
  26. *
  27. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34. * SOFTWARE.
  35. *
  36. */
  37. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  38. #include <linux/dma-mapping.h>
  39. #include <linux/slab.h>
  40. #include <linux/module.h>
  41. #include <linux/security.h>
  42. #include <linux/xarray.h>
  43. #include <rdma/ib_cache.h>
  44. #include "mad_priv.h"
  45. #include "core_priv.h"
  46. #include "mad_rmpp.h"
  47. #include "smi.h"
  48. #include "opa_smi.h"
  49. #include "agent.h"
  50. #define CREATE_TRACE_POINTS
  51. #include <trace/events/ib_mad.h>
  52. #ifdef CONFIG_TRACEPOINTS
  53. static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr,
  54. struct ib_mad_qp_info *qp_info,
  55. struct trace_event_raw_ib_mad_send_template *entry)
  56. {
  57. u16 pkey;
  58. struct ib_device *dev = qp_info->port_priv->device;
  59. u8 pnum = qp_info->port_priv->port_num;
  60. struct ib_ud_wr *wr = &mad_send_wr->send_wr;
  61. struct rdma_ah_attr attr = {};
  62. rdma_query_ah(wr->ah, &attr);
  63. /* These are common */
  64. entry->sl = attr.sl;
  65. ib_query_pkey(dev, pnum, wr->pkey_index, &pkey);
  66. entry->pkey = pkey;
  67. entry->rqpn = wr->remote_qpn;
  68. entry->rqkey = wr->remote_qkey;
  69. entry->dlid = rdma_ah_get_dlid(&attr);
  70. }
  71. #endif
  72. static int mad_sendq_size = IB_MAD_QP_SEND_SIZE;
  73. static int mad_recvq_size = IB_MAD_QP_RECV_SIZE;
  74. module_param_named(send_queue_size, mad_sendq_size, int, 0444);
  75. MODULE_PARM_DESC(send_queue_size, "Size of send queue in number of work requests");
  76. module_param_named(recv_queue_size, mad_recvq_size, int, 0444);
  77. MODULE_PARM_DESC(recv_queue_size, "Size of receive queue in number of work requests");
  78. static DEFINE_XARRAY_ALLOC1(ib_mad_clients);
  79. static u32 ib_mad_client_next;
  80. static struct list_head ib_mad_port_list;
  81. /* Port list lock */
  82. static DEFINE_SPINLOCK(ib_mad_port_list_lock);
  83. /* Forward declarations */
  84. static int method_in_use(struct ib_mad_mgmt_method_table **method,
  85. struct ib_mad_reg_req *mad_reg_req);
  86. static void remove_mad_reg_req(struct ib_mad_agent_private *priv);
  87. static struct ib_mad_agent_private *find_mad_agent(
  88. struct ib_mad_port_private *port_priv,
  89. const struct ib_mad_hdr *mad);
  90. static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,
  91. struct ib_mad_private *mad);
  92. static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv);
  93. static void timeout_sends(struct work_struct *work);
  94. static void local_completions(struct work_struct *work);
  95. static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req,
  96. struct ib_mad_agent_private *agent_priv,
  97. u8 mgmt_class);
  98. static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req,
  99. struct ib_mad_agent_private *agent_priv);
  100. static bool ib_mad_send_error(struct ib_mad_port_private *port_priv,
  101. struct ib_wc *wc);
  102. static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc);
  103. /*
  104. * Returns a ib_mad_port_private structure or NULL for a device/port
  105. * Assumes ib_mad_port_list_lock is being held
  106. */
  107. static inline struct ib_mad_port_private *
  108. __ib_get_mad_port(struct ib_device *device, int port_num)
  109. {
  110. struct ib_mad_port_private *entry;
  111. list_for_each_entry(entry, &ib_mad_port_list, port_list) {
  112. if (entry->device == device && entry->port_num == port_num)
  113. return entry;
  114. }
  115. return NULL;
  116. }
  117. /*
  118. * Wrapper function to return a ib_mad_port_private structure or NULL
  119. * for a device/port
  120. */
  121. static inline struct ib_mad_port_private *
  122. ib_get_mad_port(struct ib_device *device, int port_num)
  123. {
  124. struct ib_mad_port_private *entry;
  125. unsigned long flags;
  126. spin_lock_irqsave(&ib_mad_port_list_lock, flags);
  127. entry = __ib_get_mad_port(device, port_num);
  128. spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
  129. return entry;
  130. }
  131. static inline u8 convert_mgmt_class(u8 mgmt_class)
  132. {
  133. /* Alias IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE to 0 */
  134. return mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ?
  135. 0 : mgmt_class;
  136. }
  137. static int get_spl_qp_index(enum ib_qp_type qp_type)
  138. {
  139. switch (qp_type)
  140. {
  141. case IB_QPT_SMI:
  142. return 0;
  143. case IB_QPT_GSI:
  144. return 1;
  145. default:
  146. return -1;
  147. }
  148. }
  149. static int vendor_class_index(u8 mgmt_class)
  150. {
  151. return mgmt_class - IB_MGMT_CLASS_VENDOR_RANGE2_START;
  152. }
  153. static int is_vendor_class(u8 mgmt_class)
  154. {
  155. if ((mgmt_class < IB_MGMT_CLASS_VENDOR_RANGE2_START) ||
  156. (mgmt_class > IB_MGMT_CLASS_VENDOR_RANGE2_END))
  157. return 0;
  158. return 1;
  159. }
  160. static int is_vendor_oui(char *oui)
  161. {
  162. if (oui[0] || oui[1] || oui[2])
  163. return 1;
  164. return 0;
  165. }
  166. static int is_vendor_method_in_use(
  167. struct ib_mad_mgmt_vendor_class *vendor_class,
  168. struct ib_mad_reg_req *mad_reg_req)
  169. {
  170. struct ib_mad_mgmt_method_table *method;
  171. int i;
  172. for (i = 0; i < MAX_MGMT_OUI; i++) {
  173. if (!memcmp(vendor_class->oui[i], mad_reg_req->oui, 3)) {
  174. method = vendor_class->method_table[i];
  175. if (method) {
  176. if (method_in_use(&method, mad_reg_req))
  177. return 1;
  178. else
  179. break;
  180. }
  181. }
  182. }
  183. return 0;
  184. }
  185. int ib_response_mad(const struct ib_mad_hdr *hdr)
  186. {
  187. return ((hdr->method & IB_MGMT_METHOD_RESP) ||
  188. (hdr->method == IB_MGMT_METHOD_TRAP_REPRESS) ||
  189. ((hdr->mgmt_class == IB_MGMT_CLASS_BM) &&
  190. (hdr->attr_mod & IB_BM_ATTR_MOD_RESP)));
  191. }
  192. EXPORT_SYMBOL(ib_response_mad);
  193. /*
  194. * ib_register_mad_agent - Register to send/receive MADs
  195. *
  196. * Context: Process context.
  197. */
  198. struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
  199. u8 port_num,
  200. enum ib_qp_type qp_type,
  201. struct ib_mad_reg_req *mad_reg_req,
  202. u8 rmpp_version,
  203. ib_mad_send_handler send_handler,
  204. ib_mad_recv_handler recv_handler,
  205. void *context,
  206. u32 registration_flags)
  207. {
  208. struct ib_mad_port_private *port_priv;
  209. struct ib_mad_agent *ret = ERR_PTR(-EINVAL);
  210. struct ib_mad_agent_private *mad_agent_priv;
  211. struct ib_mad_reg_req *reg_req = NULL;
  212. struct ib_mad_mgmt_class_table *class;
  213. struct ib_mad_mgmt_vendor_class_table *vendor;
  214. struct ib_mad_mgmt_vendor_class *vendor_class;
  215. struct ib_mad_mgmt_method_table *method;
  216. int ret2, qpn;
  217. u8 mgmt_class, vclass;
  218. if ((qp_type == IB_QPT_SMI && !rdma_cap_ib_smi(device, port_num)) ||
  219. (qp_type == IB_QPT_GSI && !rdma_cap_ib_cm(device, port_num)))
  220. return ERR_PTR(-EPROTONOSUPPORT);
  221. /* Validate parameters */
  222. qpn = get_spl_qp_index(qp_type);
  223. if (qpn == -1) {
  224. dev_dbg_ratelimited(&device->dev, "%s: invalid QP Type %d\n",
  225. __func__, qp_type);
  226. goto error1;
  227. }
  228. if (rmpp_version && rmpp_version != IB_MGMT_RMPP_VERSION) {
  229. dev_dbg_ratelimited(&device->dev,
  230. "%s: invalid RMPP Version %u\n",
  231. __func__, rmpp_version);
  232. goto error1;
  233. }
  234. /* Validate MAD registration request if supplied */
  235. if (mad_reg_req) {
  236. if (mad_reg_req->mgmt_class_version >= MAX_MGMT_VERSION) {
  237. dev_dbg_ratelimited(&device->dev,
  238. "%s: invalid Class Version %u\n",
  239. __func__,
  240. mad_reg_req->mgmt_class_version);
  241. goto error1;
  242. }
  243. if (!recv_handler) {
  244. dev_dbg_ratelimited(&device->dev,
  245. "%s: no recv_handler\n", __func__);
  246. goto error1;
  247. }
  248. if (mad_reg_req->mgmt_class >= MAX_MGMT_CLASS) {
  249. /*
  250. * IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE is the only
  251. * one in this range currently allowed
  252. */
  253. if (mad_reg_req->mgmt_class !=
  254. IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
  255. dev_dbg_ratelimited(&device->dev,
  256. "%s: Invalid Mgmt Class 0x%x\n",
  257. __func__, mad_reg_req->mgmt_class);
  258. goto error1;
  259. }
  260. } else if (mad_reg_req->mgmt_class == 0) {
  261. /*
  262. * Class 0 is reserved in IBA and is used for
  263. * aliasing of IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
  264. */
  265. dev_dbg_ratelimited(&device->dev,
  266. "%s: Invalid Mgmt Class 0\n",
  267. __func__);
  268. goto error1;
  269. } else if (is_vendor_class(mad_reg_req->mgmt_class)) {
  270. /*
  271. * If class is in "new" vendor range,
  272. * ensure supplied OUI is not zero
  273. */
  274. if (!is_vendor_oui(mad_reg_req->oui)) {
  275. dev_dbg_ratelimited(&device->dev,
  276. "%s: No OUI specified for class 0x%x\n",
  277. __func__,
  278. mad_reg_req->mgmt_class);
  279. goto error1;
  280. }
  281. }
  282. /* Make sure class supplied is consistent with RMPP */
  283. if (!ib_is_mad_class_rmpp(mad_reg_req->mgmt_class)) {
  284. if (rmpp_version) {
  285. dev_dbg_ratelimited(&device->dev,
  286. "%s: RMPP version for non-RMPP class 0x%x\n",
  287. __func__, mad_reg_req->mgmt_class);
  288. goto error1;
  289. }
  290. }
  291. /* Make sure class supplied is consistent with QP type */
  292. if (qp_type == IB_QPT_SMI) {
  293. if ((mad_reg_req->mgmt_class !=
  294. IB_MGMT_CLASS_SUBN_LID_ROUTED) &&
  295. (mad_reg_req->mgmt_class !=
  296. IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
  297. dev_dbg_ratelimited(&device->dev,
  298. "%s: Invalid SM QP type: class 0x%x\n",
  299. __func__, mad_reg_req->mgmt_class);
  300. goto error1;
  301. }
  302. } else {
  303. if ((mad_reg_req->mgmt_class ==
  304. IB_MGMT_CLASS_SUBN_LID_ROUTED) ||
  305. (mad_reg_req->mgmt_class ==
  306. IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
  307. dev_dbg_ratelimited(&device->dev,
  308. "%s: Invalid GS QP type: class 0x%x\n",
  309. __func__, mad_reg_req->mgmt_class);
  310. goto error1;
  311. }
  312. }
  313. } else {
  314. /* No registration request supplied */
  315. if (!send_handler)
  316. goto error1;
  317. if (registration_flags & IB_MAD_USER_RMPP)
  318. goto error1;
  319. }
  320. /* Validate device and port */
  321. port_priv = ib_get_mad_port(device, port_num);
  322. if (!port_priv) {
  323. dev_dbg_ratelimited(&device->dev, "%s: Invalid port %d\n",
  324. __func__, port_num);
  325. ret = ERR_PTR(-ENODEV);
  326. goto error1;
  327. }
  328. /* Verify the QP requested is supported. For example, Ethernet devices
  329. * will not have QP0.
  330. */
  331. if (!port_priv->qp_info[qpn].qp) {
  332. dev_dbg_ratelimited(&device->dev, "%s: QP %d not supported\n",
  333. __func__, qpn);
  334. ret = ERR_PTR(-EPROTONOSUPPORT);
  335. goto error1;
  336. }
  337. /* Allocate structures */
  338. mad_agent_priv = kzalloc(sizeof *mad_agent_priv, GFP_KERNEL);
  339. if (!mad_agent_priv) {
  340. ret = ERR_PTR(-ENOMEM);
  341. goto error1;
  342. }
  343. if (mad_reg_req) {
  344. reg_req = kmemdup(mad_reg_req, sizeof *reg_req, GFP_KERNEL);
  345. if (!reg_req) {
  346. ret = ERR_PTR(-ENOMEM);
  347. goto error3;
  348. }
  349. }
  350. /* Now, fill in the various structures */
  351. mad_agent_priv->qp_info = &port_priv->qp_info[qpn];
  352. mad_agent_priv->reg_req = reg_req;
  353. mad_agent_priv->agent.rmpp_version = rmpp_version;
  354. mad_agent_priv->agent.device = device;
  355. mad_agent_priv->agent.recv_handler = recv_handler;
  356. mad_agent_priv->agent.send_handler = send_handler;
  357. mad_agent_priv->agent.context = context;
  358. mad_agent_priv->agent.qp = port_priv->qp_info[qpn].qp;
  359. mad_agent_priv->agent.port_num = port_num;
  360. mad_agent_priv->agent.flags = registration_flags;
  361. spin_lock_init(&mad_agent_priv->lock);
  362. INIT_LIST_HEAD(&mad_agent_priv->send_list);
  363. INIT_LIST_HEAD(&mad_agent_priv->wait_list);
  364. INIT_LIST_HEAD(&mad_agent_priv->done_list);
  365. INIT_LIST_HEAD(&mad_agent_priv->rmpp_list);
  366. INIT_DELAYED_WORK(&mad_agent_priv->timed_work, timeout_sends);
  367. INIT_LIST_HEAD(&mad_agent_priv->local_list);
  368. INIT_WORK(&mad_agent_priv->local_work, local_completions);
  369. refcount_set(&mad_agent_priv->refcount, 1);
  370. init_completion(&mad_agent_priv->comp);
  371. ret2 = ib_mad_agent_security_setup(&mad_agent_priv->agent, qp_type);
  372. if (ret2) {
  373. ret = ERR_PTR(ret2);
  374. goto error4;
  375. }
  376. /*
  377. * The mlx4 driver uses the top byte to distinguish which virtual
  378. * function generated the MAD, so we must avoid using it.
  379. */
  380. ret2 = xa_alloc_cyclic(&ib_mad_clients, &mad_agent_priv->agent.hi_tid,
  381. mad_agent_priv, XA_LIMIT(0, (1 << 24) - 1),
  382. &ib_mad_client_next, GFP_KERNEL);
  383. if (ret2 < 0) {
  384. ret = ERR_PTR(ret2);
  385. goto error5;
  386. }
  387. /*
  388. * Make sure MAD registration (if supplied)
  389. * is non overlapping with any existing ones
  390. */
  391. spin_lock_irq(&port_priv->reg_lock);
  392. if (mad_reg_req) {
  393. mgmt_class = convert_mgmt_class(mad_reg_req->mgmt_class);
  394. if (!is_vendor_class(mgmt_class)) {
  395. class = port_priv->version[mad_reg_req->
  396. mgmt_class_version].class;
  397. if (class) {
  398. method = class->method_table[mgmt_class];
  399. if (method) {
  400. if (method_in_use(&method,
  401. mad_reg_req))
  402. goto error6;
  403. }
  404. }
  405. ret2 = add_nonoui_reg_req(mad_reg_req, mad_agent_priv,
  406. mgmt_class);
  407. } else {
  408. /* "New" vendor class range */
  409. vendor = port_priv->version[mad_reg_req->
  410. mgmt_class_version].vendor;
  411. if (vendor) {
  412. vclass = vendor_class_index(mgmt_class);
  413. vendor_class = vendor->vendor_class[vclass];
  414. if (vendor_class) {
  415. if (is_vendor_method_in_use(
  416. vendor_class,
  417. mad_reg_req))
  418. goto error6;
  419. }
  420. }
  421. ret2 = add_oui_reg_req(mad_reg_req, mad_agent_priv);
  422. }
  423. if (ret2) {
  424. ret = ERR_PTR(ret2);
  425. goto error6;
  426. }
  427. }
  428. spin_unlock_irq(&port_priv->reg_lock);
  429. trace_ib_mad_create_agent(mad_agent_priv);
  430. return &mad_agent_priv->agent;
  431. error6:
  432. spin_unlock_irq(&port_priv->reg_lock);
  433. xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid);
  434. error5:
  435. ib_mad_agent_security_cleanup(&mad_agent_priv->agent);
  436. error4:
  437. kfree(reg_req);
  438. error3:
  439. kfree(mad_agent_priv);
  440. error1:
  441. return ret;
  442. }
  443. EXPORT_SYMBOL(ib_register_mad_agent);
  444. static inline void deref_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
  445. {
  446. if (refcount_dec_and_test(&mad_agent_priv->refcount))
  447. complete(&mad_agent_priv->comp);
  448. }
  449. static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
  450. {
  451. struct ib_mad_port_private *port_priv;
  452. /* Note that we could still be handling received MADs */
  453. trace_ib_mad_unregister_agent(mad_agent_priv);
  454. /*
  455. * Canceling all sends results in dropping received response
  456. * MADs, preventing us from queuing additional work
  457. */
  458. cancel_mads(mad_agent_priv);
  459. port_priv = mad_agent_priv->qp_info->port_priv;
  460. cancel_delayed_work(&mad_agent_priv->timed_work);
  461. spin_lock_irq(&port_priv->reg_lock);
  462. remove_mad_reg_req(mad_agent_priv);
  463. spin_unlock_irq(&port_priv->reg_lock);
  464. xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid);
  465. flush_workqueue(port_priv->wq);
  466. deref_mad_agent(mad_agent_priv);
  467. wait_for_completion(&mad_agent_priv->comp);
  468. ib_cancel_rmpp_recvs(mad_agent_priv);
  469. ib_mad_agent_security_cleanup(&mad_agent_priv->agent);
  470. kfree(mad_agent_priv->reg_req);
  471. kfree_rcu(mad_agent_priv, rcu);
  472. }
  473. /*
  474. * ib_unregister_mad_agent - Unregisters a client from using MAD services
  475. *
  476. * Context: Process context.
  477. */
  478. void ib_unregister_mad_agent(struct ib_mad_agent *mad_agent)
  479. {
  480. struct ib_mad_agent_private *mad_agent_priv;
  481. mad_agent_priv = container_of(mad_agent,
  482. struct ib_mad_agent_private,
  483. agent);
  484. unregister_mad_agent(mad_agent_priv);
  485. }
  486. EXPORT_SYMBOL(ib_unregister_mad_agent);
  487. static void dequeue_mad(struct ib_mad_list_head *mad_list)
  488. {
  489. struct ib_mad_queue *mad_queue;
  490. unsigned long flags;
  491. mad_queue = mad_list->mad_queue;
  492. spin_lock_irqsave(&mad_queue->lock, flags);
  493. list_del(&mad_list->list);
  494. mad_queue->count--;
  495. spin_unlock_irqrestore(&mad_queue->lock, flags);
  496. }
  497. static void build_smp_wc(struct ib_qp *qp, struct ib_cqe *cqe, u16 slid,
  498. u16 pkey_index, u8 port_num, struct ib_wc *wc)
  499. {
  500. memset(wc, 0, sizeof *wc);
  501. wc->wr_cqe = cqe;
  502. wc->status = IB_WC_SUCCESS;
  503. wc->opcode = IB_WC_RECV;
  504. wc->pkey_index = pkey_index;
  505. wc->byte_len = sizeof(struct ib_mad) + sizeof(struct ib_grh);
  506. wc->src_qp = IB_QP0;
  507. wc->qp = qp;
  508. wc->slid = slid;
  509. wc->sl = 0;
  510. wc->dlid_path_bits = 0;
  511. wc->port_num = port_num;
  512. }
  513. static size_t mad_priv_size(const struct ib_mad_private *mp)
  514. {
  515. return sizeof(struct ib_mad_private) + mp->mad_size;
  516. }
  517. static struct ib_mad_private *alloc_mad_private(size_t mad_size, gfp_t flags)
  518. {
  519. size_t size = sizeof(struct ib_mad_private) + mad_size;
  520. struct ib_mad_private *ret = kzalloc(size, flags);
  521. if (ret)
  522. ret->mad_size = mad_size;
  523. return ret;
  524. }
  525. static size_t port_mad_size(const struct ib_mad_port_private *port_priv)
  526. {
  527. return rdma_max_mad_size(port_priv->device, port_priv->port_num);
  528. }
  529. static size_t mad_priv_dma_size(const struct ib_mad_private *mp)
  530. {
  531. return sizeof(struct ib_grh) + mp->mad_size;
  532. }
  533. /*
  534. * Return 0 if SMP is to be sent
  535. * Return 1 if SMP was consumed locally (whether or not solicited)
  536. * Return < 0 if error
  537. */
  538. static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
  539. struct ib_mad_send_wr_private *mad_send_wr)
  540. {
  541. int ret = 0;
  542. struct ib_smp *smp = mad_send_wr->send_buf.mad;
  543. struct opa_smp *opa_smp = (struct opa_smp *)smp;
  544. unsigned long flags;
  545. struct ib_mad_local_private *local;
  546. struct ib_mad_private *mad_priv;
  547. struct ib_mad_port_private *port_priv;
  548. struct ib_mad_agent_private *recv_mad_agent = NULL;
  549. struct ib_device *device = mad_agent_priv->agent.device;
  550. u8 port_num;
  551. struct ib_wc mad_wc;
  552. struct ib_ud_wr *send_wr = &mad_send_wr->send_wr;
  553. size_t mad_size = port_mad_size(mad_agent_priv->qp_info->port_priv);
  554. u16 out_mad_pkey_index = 0;
  555. u16 drslid;
  556. bool opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
  557. mad_agent_priv->qp_info->port_priv->port_num);
  558. if (rdma_cap_ib_switch(device) &&
  559. smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
  560. port_num = send_wr->port_num;
  561. else
  562. port_num = mad_agent_priv->agent.port_num;
  563. /*
  564. * Directed route handling starts if the initial LID routed part of
  565. * a request or the ending LID routed part of a response is empty.
  566. * If we are at the start of the LID routed part, don't update the
  567. * hop_ptr or hop_cnt. See section 14.2.2, Vol 1 IB spec.
  568. */
  569. if (opa && smp->class_version == OPA_SM_CLASS_VERSION) {
  570. u32 opa_drslid;
  571. trace_ib_mad_handle_out_opa_smi(opa_smp);
  572. if ((opa_get_smp_direction(opa_smp)
  573. ? opa_smp->route.dr.dr_dlid : opa_smp->route.dr.dr_slid) ==
  574. OPA_LID_PERMISSIVE &&
  575. opa_smi_handle_dr_smp_send(opa_smp,
  576. rdma_cap_ib_switch(device),
  577. port_num) == IB_SMI_DISCARD) {
  578. ret = -EINVAL;
  579. dev_err(&device->dev, "OPA Invalid directed route\n");
  580. goto out;
  581. }
  582. opa_drslid = be32_to_cpu(opa_smp->route.dr.dr_slid);
  583. if (opa_drslid != be32_to_cpu(OPA_LID_PERMISSIVE) &&
  584. opa_drslid & 0xffff0000) {
  585. ret = -EINVAL;
  586. dev_err(&device->dev, "OPA Invalid dr_slid 0x%x\n",
  587. opa_drslid);
  588. goto out;
  589. }
  590. drslid = (u16)(opa_drslid & 0x0000ffff);
  591. /* Check to post send on QP or process locally */
  592. if (opa_smi_check_local_smp(opa_smp, device) == IB_SMI_DISCARD &&
  593. opa_smi_check_local_returning_smp(opa_smp, device) == IB_SMI_DISCARD)
  594. goto out;
  595. } else {
  596. trace_ib_mad_handle_out_ib_smi(smp);
  597. if ((ib_get_smp_direction(smp) ? smp->dr_dlid : smp->dr_slid) ==
  598. IB_LID_PERMISSIVE &&
  599. smi_handle_dr_smp_send(smp, rdma_cap_ib_switch(device), port_num) ==
  600. IB_SMI_DISCARD) {
  601. ret = -EINVAL;
  602. dev_err(&device->dev, "Invalid directed route\n");
  603. goto out;
  604. }
  605. drslid = be16_to_cpu(smp->dr_slid);
  606. /* Check to post send on QP or process locally */
  607. if (smi_check_local_smp(smp, device) == IB_SMI_DISCARD &&
  608. smi_check_local_returning_smp(smp, device) == IB_SMI_DISCARD)
  609. goto out;
  610. }
  611. local = kmalloc(sizeof *local, GFP_ATOMIC);
  612. if (!local) {
  613. ret = -ENOMEM;
  614. goto out;
  615. }
  616. local->mad_priv = NULL;
  617. local->recv_mad_agent = NULL;
  618. mad_priv = alloc_mad_private(mad_size, GFP_ATOMIC);
  619. if (!mad_priv) {
  620. ret = -ENOMEM;
  621. kfree(local);
  622. goto out;
  623. }
  624. build_smp_wc(mad_agent_priv->agent.qp,
  625. send_wr->wr.wr_cqe, drslid,
  626. send_wr->pkey_index,
  627. send_wr->port_num, &mad_wc);
  628. if (opa && smp->base_version == OPA_MGMT_BASE_VERSION) {
  629. mad_wc.byte_len = mad_send_wr->send_buf.hdr_len
  630. + mad_send_wr->send_buf.data_len
  631. + sizeof(struct ib_grh);
  632. }
  633. /* No GRH for DR SMP */
  634. ret = device->ops.process_mad(device, 0, port_num, &mad_wc, NULL,
  635. (const struct ib_mad *)smp,
  636. (struct ib_mad *)mad_priv->mad, &mad_size,
  637. &out_mad_pkey_index);
  638. switch (ret)
  639. {
  640. case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY:
  641. if (ib_response_mad((const struct ib_mad_hdr *)mad_priv->mad) &&
  642. mad_agent_priv->agent.recv_handler) {
  643. local->mad_priv = mad_priv;
  644. local->recv_mad_agent = mad_agent_priv;
  645. /*
  646. * Reference MAD agent until receive
  647. * side of local completion handled
  648. */
  649. refcount_inc(&mad_agent_priv->refcount);
  650. } else
  651. kfree(mad_priv);
  652. break;
  653. case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED:
  654. kfree(mad_priv);
  655. break;
  656. case IB_MAD_RESULT_SUCCESS:
  657. /* Treat like an incoming receive MAD */
  658. port_priv = ib_get_mad_port(mad_agent_priv->agent.device,
  659. mad_agent_priv->agent.port_num);
  660. if (port_priv) {
  661. memcpy(mad_priv->mad, smp, mad_priv->mad_size);
  662. recv_mad_agent = find_mad_agent(port_priv,
  663. (const struct ib_mad_hdr *)mad_priv->mad);
  664. }
  665. if (!port_priv || !recv_mad_agent) {
  666. /*
  667. * No receiving agent so drop packet and
  668. * generate send completion.
  669. */
  670. kfree(mad_priv);
  671. break;
  672. }
  673. local->mad_priv = mad_priv;
  674. local->recv_mad_agent = recv_mad_agent;
  675. break;
  676. default:
  677. kfree(mad_priv);
  678. kfree(local);
  679. ret = -EINVAL;
  680. goto out;
  681. }
  682. local->mad_send_wr = mad_send_wr;
  683. if (opa) {
  684. local->mad_send_wr->send_wr.pkey_index = out_mad_pkey_index;
  685. local->return_wc_byte_len = mad_size;
  686. }
  687. /* Reference MAD agent until send side of local completion handled */
  688. refcount_inc(&mad_agent_priv->refcount);
  689. /* Queue local completion to local list */
  690. spin_lock_irqsave(&mad_agent_priv->lock, flags);
  691. list_add_tail(&local->completion_list, &mad_agent_priv->local_list);
  692. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  693. queue_work(mad_agent_priv->qp_info->port_priv->wq,
  694. &mad_agent_priv->local_work);
  695. ret = 1;
  696. out:
  697. return ret;
  698. }
  699. static int get_pad_size(int hdr_len, int data_len, size_t mad_size)
  700. {
  701. int seg_size, pad;
  702. seg_size = mad_size - hdr_len;
  703. if (data_len && seg_size) {
  704. pad = seg_size - data_len % seg_size;
  705. return pad == seg_size ? 0 : pad;
  706. } else
  707. return seg_size;
  708. }
  709. static void free_send_rmpp_list(struct ib_mad_send_wr_private *mad_send_wr)
  710. {
  711. struct ib_rmpp_segment *s, *t;
  712. list_for_each_entry_safe(s, t, &mad_send_wr->rmpp_list, list) {
  713. list_del(&s->list);
  714. kfree(s);
  715. }
  716. }
  717. static int alloc_send_rmpp_list(struct ib_mad_send_wr_private *send_wr,
  718. size_t mad_size, gfp_t gfp_mask)
  719. {
  720. struct ib_mad_send_buf *send_buf = &send_wr->send_buf;
  721. struct ib_rmpp_mad *rmpp_mad = send_buf->mad;
  722. struct ib_rmpp_segment *seg = NULL;
  723. int left, seg_size, pad;
  724. send_buf->seg_size = mad_size - send_buf->hdr_len;
  725. send_buf->seg_rmpp_size = mad_size - IB_MGMT_RMPP_HDR;
  726. seg_size = send_buf->seg_size;
  727. pad = send_wr->pad;
  728. /* Allocate data segments. */
  729. for (left = send_buf->data_len + pad; left > 0; left -= seg_size) {
  730. seg = kmalloc(sizeof (*seg) + seg_size, gfp_mask);
  731. if (!seg) {
  732. free_send_rmpp_list(send_wr);
  733. return -ENOMEM;
  734. }
  735. seg->num = ++send_buf->seg_count;
  736. list_add_tail(&seg->list, &send_wr->rmpp_list);
  737. }
  738. /* Zero any padding */
  739. if (pad)
  740. memset(seg->data + seg_size - pad, 0, pad);
  741. rmpp_mad->rmpp_hdr.rmpp_version = send_wr->mad_agent_priv->
  742. agent.rmpp_version;
  743. rmpp_mad->rmpp_hdr.rmpp_type = IB_MGMT_RMPP_TYPE_DATA;
  744. ib_set_rmpp_flags(&rmpp_mad->rmpp_hdr, IB_MGMT_RMPP_FLAG_ACTIVE);
  745. send_wr->cur_seg = container_of(send_wr->rmpp_list.next,
  746. struct ib_rmpp_segment, list);
  747. send_wr->last_ack_seg = send_wr->cur_seg;
  748. return 0;
  749. }
  750. int ib_mad_kernel_rmpp_agent(const struct ib_mad_agent *agent)
  751. {
  752. return agent->rmpp_version && !(agent->flags & IB_MAD_USER_RMPP);
  753. }
  754. EXPORT_SYMBOL(ib_mad_kernel_rmpp_agent);
  755. struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent,
  756. u32 remote_qpn, u16 pkey_index,
  757. int rmpp_active,
  758. int hdr_len, int data_len,
  759. gfp_t gfp_mask,
  760. u8 base_version)
  761. {
  762. struct ib_mad_agent_private *mad_agent_priv;
  763. struct ib_mad_send_wr_private *mad_send_wr;
  764. int pad, message_size, ret, size;
  765. void *buf;
  766. size_t mad_size;
  767. bool opa;
  768. mad_agent_priv = container_of(mad_agent, struct ib_mad_agent_private,
  769. agent);
  770. opa = rdma_cap_opa_mad(mad_agent->device, mad_agent->port_num);
  771. if (opa && base_version == OPA_MGMT_BASE_VERSION)
  772. mad_size = sizeof(struct opa_mad);
  773. else
  774. mad_size = sizeof(struct ib_mad);
  775. pad = get_pad_size(hdr_len, data_len, mad_size);
  776. message_size = hdr_len + data_len + pad;
  777. if (ib_mad_kernel_rmpp_agent(mad_agent)) {
  778. if (!rmpp_active && message_size > mad_size)
  779. return ERR_PTR(-EINVAL);
  780. } else
  781. if (rmpp_active || message_size > mad_size)
  782. return ERR_PTR(-EINVAL);
  783. size = rmpp_active ? hdr_len : mad_size;
  784. buf = kzalloc(sizeof *mad_send_wr + size, gfp_mask);
  785. if (!buf)
  786. return ERR_PTR(-ENOMEM);
  787. mad_send_wr = buf + size;
  788. INIT_LIST_HEAD(&mad_send_wr->rmpp_list);
  789. mad_send_wr->send_buf.mad = buf;
  790. mad_send_wr->send_buf.hdr_len = hdr_len;
  791. mad_send_wr->send_buf.data_len = data_len;
  792. mad_send_wr->pad = pad;
  793. mad_send_wr->mad_agent_priv = mad_agent_priv;
  794. mad_send_wr->sg_list[0].length = hdr_len;
  795. mad_send_wr->sg_list[0].lkey = mad_agent->qp->pd->local_dma_lkey;
  796. /* OPA MADs don't have to be the full 2048 bytes */
  797. if (opa && base_version == OPA_MGMT_BASE_VERSION &&
  798. data_len < mad_size - hdr_len)
  799. mad_send_wr->sg_list[1].length = data_len;
  800. else
  801. mad_send_wr->sg_list[1].length = mad_size - hdr_len;
  802. mad_send_wr->sg_list[1].lkey = mad_agent->qp->pd->local_dma_lkey;
  803. mad_send_wr->mad_list.cqe.done = ib_mad_send_done;
  804. mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe;
  805. mad_send_wr->send_wr.wr.sg_list = mad_send_wr->sg_list;
  806. mad_send_wr->send_wr.wr.num_sge = 2;
  807. mad_send_wr->send_wr.wr.opcode = IB_WR_SEND;
  808. mad_send_wr->send_wr.wr.send_flags = IB_SEND_SIGNALED;
  809. mad_send_wr->send_wr.remote_qpn = remote_qpn;
  810. mad_send_wr->send_wr.remote_qkey = IB_QP_SET_QKEY;
  811. mad_send_wr->send_wr.pkey_index = pkey_index;
  812. if (rmpp_active) {
  813. ret = alloc_send_rmpp_list(mad_send_wr, mad_size, gfp_mask);
  814. if (ret) {
  815. kfree(buf);
  816. return ERR_PTR(ret);
  817. }
  818. }
  819. mad_send_wr->send_buf.mad_agent = mad_agent;
  820. refcount_inc(&mad_agent_priv->refcount);
  821. return &mad_send_wr->send_buf;
  822. }
  823. EXPORT_SYMBOL(ib_create_send_mad);
  824. int ib_get_mad_data_offset(u8 mgmt_class)
  825. {
  826. if (mgmt_class == IB_MGMT_CLASS_SUBN_ADM)
  827. return IB_MGMT_SA_HDR;
  828. else if ((mgmt_class == IB_MGMT_CLASS_DEVICE_MGMT) ||
  829. (mgmt_class == IB_MGMT_CLASS_DEVICE_ADM) ||
  830. (mgmt_class == IB_MGMT_CLASS_BIS))
  831. return IB_MGMT_DEVICE_HDR;
  832. else if ((mgmt_class >= IB_MGMT_CLASS_VENDOR_RANGE2_START) &&
  833. (mgmt_class <= IB_MGMT_CLASS_VENDOR_RANGE2_END))
  834. return IB_MGMT_VENDOR_HDR;
  835. else
  836. return IB_MGMT_MAD_HDR;
  837. }
  838. EXPORT_SYMBOL(ib_get_mad_data_offset);
  839. int ib_is_mad_class_rmpp(u8 mgmt_class)
  840. {
  841. if ((mgmt_class == IB_MGMT_CLASS_SUBN_ADM) ||
  842. (mgmt_class == IB_MGMT_CLASS_DEVICE_MGMT) ||
  843. (mgmt_class == IB_MGMT_CLASS_DEVICE_ADM) ||
  844. (mgmt_class == IB_MGMT_CLASS_BIS) ||
  845. ((mgmt_class >= IB_MGMT_CLASS_VENDOR_RANGE2_START) &&
  846. (mgmt_class <= IB_MGMT_CLASS_VENDOR_RANGE2_END)))
  847. return 1;
  848. return 0;
  849. }
  850. EXPORT_SYMBOL(ib_is_mad_class_rmpp);
  851. void *ib_get_rmpp_segment(struct ib_mad_send_buf *send_buf, int seg_num)
  852. {
  853. struct ib_mad_send_wr_private *mad_send_wr;
  854. struct list_head *list;
  855. mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private,
  856. send_buf);
  857. list = &mad_send_wr->cur_seg->list;
  858. if (mad_send_wr->cur_seg->num < seg_num) {
  859. list_for_each_entry(mad_send_wr->cur_seg, list, list)
  860. if (mad_send_wr->cur_seg->num == seg_num)
  861. break;
  862. } else if (mad_send_wr->cur_seg->num > seg_num) {
  863. list_for_each_entry_reverse(mad_send_wr->cur_seg, list, list)
  864. if (mad_send_wr->cur_seg->num == seg_num)
  865. break;
  866. }
  867. return mad_send_wr->cur_seg->data;
  868. }
  869. EXPORT_SYMBOL(ib_get_rmpp_segment);
  870. static inline void *ib_get_payload(struct ib_mad_send_wr_private *mad_send_wr)
  871. {
  872. if (mad_send_wr->send_buf.seg_count)
  873. return ib_get_rmpp_segment(&mad_send_wr->send_buf,
  874. mad_send_wr->seg_num);
  875. else
  876. return mad_send_wr->send_buf.mad +
  877. mad_send_wr->send_buf.hdr_len;
  878. }
  879. void ib_free_send_mad(struct ib_mad_send_buf *send_buf)
  880. {
  881. struct ib_mad_agent_private *mad_agent_priv;
  882. struct ib_mad_send_wr_private *mad_send_wr;
  883. mad_agent_priv = container_of(send_buf->mad_agent,
  884. struct ib_mad_agent_private, agent);
  885. mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private,
  886. send_buf);
  887. free_send_rmpp_list(mad_send_wr);
  888. kfree(send_buf->mad);
  889. deref_mad_agent(mad_agent_priv);
  890. }
  891. EXPORT_SYMBOL(ib_free_send_mad);
  892. int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr)
  893. {
  894. struct ib_mad_qp_info *qp_info;
  895. struct list_head *list;
  896. struct ib_mad_agent *mad_agent;
  897. struct ib_sge *sge;
  898. unsigned long flags;
  899. int ret;
  900. /* Set WR ID to find mad_send_wr upon completion */
  901. qp_info = mad_send_wr->mad_agent_priv->qp_info;
  902. mad_send_wr->mad_list.mad_queue = &qp_info->send_queue;
  903. mad_send_wr->mad_list.cqe.done = ib_mad_send_done;
  904. mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe;
  905. mad_agent = mad_send_wr->send_buf.mad_agent;
  906. sge = mad_send_wr->sg_list;
  907. sge[0].addr = ib_dma_map_single(mad_agent->device,
  908. mad_send_wr->send_buf.mad,
  909. sge[0].length,
  910. DMA_TO_DEVICE);
  911. if (unlikely(ib_dma_mapping_error(mad_agent->device, sge[0].addr)))
  912. return -ENOMEM;
  913. mad_send_wr->header_mapping = sge[0].addr;
  914. sge[1].addr = ib_dma_map_single(mad_agent->device,
  915. ib_get_payload(mad_send_wr),
  916. sge[1].length,
  917. DMA_TO_DEVICE);
  918. if (unlikely(ib_dma_mapping_error(mad_agent->device, sge[1].addr))) {
  919. ib_dma_unmap_single(mad_agent->device,
  920. mad_send_wr->header_mapping,
  921. sge[0].length, DMA_TO_DEVICE);
  922. return -ENOMEM;
  923. }
  924. mad_send_wr->payload_mapping = sge[1].addr;
  925. spin_lock_irqsave(&qp_info->send_queue.lock, flags);
  926. if (qp_info->send_queue.count < qp_info->send_queue.max_active) {
  927. trace_ib_mad_ib_send_mad(mad_send_wr, qp_info);
  928. ret = ib_post_send(mad_agent->qp, &mad_send_wr->send_wr.wr,
  929. NULL);
  930. list = &qp_info->send_queue.list;
  931. } else {
  932. ret = 0;
  933. list = &qp_info->overflow_list;
  934. }
  935. if (!ret) {
  936. qp_info->send_queue.count++;
  937. list_add_tail(&mad_send_wr->mad_list.list, list);
  938. }
  939. spin_unlock_irqrestore(&qp_info->send_queue.lock, flags);
  940. if (ret) {
  941. ib_dma_unmap_single(mad_agent->device,
  942. mad_send_wr->header_mapping,
  943. sge[0].length, DMA_TO_DEVICE);
  944. ib_dma_unmap_single(mad_agent->device,
  945. mad_send_wr->payload_mapping,
  946. sge[1].length, DMA_TO_DEVICE);
  947. }
  948. return ret;
  949. }
  950. /*
  951. * ib_post_send_mad - Posts MAD(s) to the send queue of the QP associated
  952. * with the registered client
  953. */
  954. int ib_post_send_mad(struct ib_mad_send_buf *send_buf,
  955. struct ib_mad_send_buf **bad_send_buf)
  956. {
  957. struct ib_mad_agent_private *mad_agent_priv;
  958. struct ib_mad_send_buf *next_send_buf;
  959. struct ib_mad_send_wr_private *mad_send_wr;
  960. unsigned long flags;
  961. int ret = -EINVAL;
  962. /* Walk list of send WRs and post each on send list */
  963. for (; send_buf; send_buf = next_send_buf) {
  964. mad_send_wr = container_of(send_buf,
  965. struct ib_mad_send_wr_private,
  966. send_buf);
  967. mad_agent_priv = mad_send_wr->mad_agent_priv;
  968. ret = ib_mad_enforce_security(mad_agent_priv,
  969. mad_send_wr->send_wr.pkey_index);
  970. if (ret)
  971. goto error;
  972. if (!send_buf->mad_agent->send_handler ||
  973. (send_buf->timeout_ms &&
  974. !send_buf->mad_agent->recv_handler)) {
  975. ret = -EINVAL;
  976. goto error;
  977. }
  978. if (!ib_is_mad_class_rmpp(((struct ib_mad_hdr *) send_buf->mad)->mgmt_class)) {
  979. if (mad_agent_priv->agent.rmpp_version) {
  980. ret = -EINVAL;
  981. goto error;
  982. }
  983. }
  984. /*
  985. * Save pointer to next work request to post in case the
  986. * current one completes, and the user modifies the work
  987. * request associated with the completion
  988. */
  989. next_send_buf = send_buf->next;
  990. mad_send_wr->send_wr.ah = send_buf->ah;
  991. if (((struct ib_mad_hdr *) send_buf->mad)->mgmt_class ==
  992. IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
  993. ret = handle_outgoing_dr_smp(mad_agent_priv,
  994. mad_send_wr);
  995. if (ret < 0) /* error */
  996. goto error;
  997. else if (ret == 1) /* locally consumed */
  998. continue;
  999. }
  1000. mad_send_wr->tid = ((struct ib_mad_hdr *) send_buf->mad)->tid;
  1001. /* Timeout will be updated after send completes */
  1002. mad_send_wr->timeout = msecs_to_jiffies(send_buf->timeout_ms);
  1003. mad_send_wr->max_retries = send_buf->retries;
  1004. mad_send_wr->retries_left = send_buf->retries;
  1005. send_buf->retries = 0;
  1006. /* Reference for work request to QP + response */
  1007. mad_send_wr->refcount = 1 + (mad_send_wr->timeout > 0);
  1008. mad_send_wr->status = IB_WC_SUCCESS;
  1009. /* Reference MAD agent until send completes */
  1010. refcount_inc(&mad_agent_priv->refcount);
  1011. spin_lock_irqsave(&mad_agent_priv->lock, flags);
  1012. list_add_tail(&mad_send_wr->agent_list,
  1013. &mad_agent_priv->send_list);
  1014. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  1015. if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
  1016. ret = ib_send_rmpp_mad(mad_send_wr);
  1017. if (ret >= 0 && ret != IB_RMPP_RESULT_CONSUMED)
  1018. ret = ib_send_mad(mad_send_wr);
  1019. } else
  1020. ret = ib_send_mad(mad_send_wr);
  1021. if (ret < 0) {
  1022. /* Fail send request */
  1023. spin_lock_irqsave(&mad_agent_priv->lock, flags);
  1024. list_del(&mad_send_wr->agent_list);
  1025. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  1026. deref_mad_agent(mad_agent_priv);
  1027. goto error;
  1028. }
  1029. }
  1030. return 0;
  1031. error:
  1032. if (bad_send_buf)
  1033. *bad_send_buf = send_buf;
  1034. return ret;
  1035. }
  1036. EXPORT_SYMBOL(ib_post_send_mad);
  1037. /*
  1038. * ib_free_recv_mad - Returns data buffers used to receive
  1039. * a MAD to the access layer
  1040. */
  1041. void ib_free_recv_mad(struct ib_mad_recv_wc *mad_recv_wc)
  1042. {
  1043. struct ib_mad_recv_buf *mad_recv_buf, *temp_recv_buf;
  1044. struct ib_mad_private_header *mad_priv_hdr;
  1045. struct ib_mad_private *priv;
  1046. struct list_head free_list;
  1047. INIT_LIST_HEAD(&free_list);
  1048. list_splice_init(&mad_recv_wc->rmpp_list, &free_list);
  1049. list_for_each_entry_safe(mad_recv_buf, temp_recv_buf,
  1050. &free_list, list) {
  1051. mad_recv_wc = container_of(mad_recv_buf, struct ib_mad_recv_wc,
  1052. recv_buf);
  1053. mad_priv_hdr = container_of(mad_recv_wc,
  1054. struct ib_mad_private_header,
  1055. recv_wc);
  1056. priv = container_of(mad_priv_hdr, struct ib_mad_private,
  1057. header);
  1058. kfree(priv);
  1059. }
  1060. }
  1061. EXPORT_SYMBOL(ib_free_recv_mad);
  1062. static int method_in_use(struct ib_mad_mgmt_method_table **method,
  1063. struct ib_mad_reg_req *mad_reg_req)
  1064. {
  1065. int i;
  1066. for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS) {
  1067. if ((*method)->agent[i]) {
  1068. pr_err("Method %d already in use\n", i);
  1069. return -EINVAL;
  1070. }
  1071. }
  1072. return 0;
  1073. }
  1074. static int allocate_method_table(struct ib_mad_mgmt_method_table **method)
  1075. {
  1076. /* Allocate management method table */
  1077. *method = kzalloc(sizeof **method, GFP_ATOMIC);
  1078. return (*method) ? 0 : (-ENOMEM);
  1079. }
  1080. /*
  1081. * Check to see if there are any methods still in use
  1082. */
  1083. static int check_method_table(struct ib_mad_mgmt_method_table *method)
  1084. {
  1085. int i;
  1086. for (i = 0; i < IB_MGMT_MAX_METHODS; i++)
  1087. if (method->agent[i])
  1088. return 1;
  1089. return 0;
  1090. }
  1091. /*
  1092. * Check to see if there are any method tables for this class still in use
  1093. */
  1094. static int check_class_table(struct ib_mad_mgmt_class_table *class)
  1095. {
  1096. int i;
  1097. for (i = 0; i < MAX_MGMT_CLASS; i++)
  1098. if (class->method_table[i])
  1099. return 1;
  1100. return 0;
  1101. }
  1102. static int check_vendor_class(struct ib_mad_mgmt_vendor_class *vendor_class)
  1103. {
  1104. int i;
  1105. for (i = 0; i < MAX_MGMT_OUI; i++)
  1106. if (vendor_class->method_table[i])
  1107. return 1;
  1108. return 0;
  1109. }
  1110. static int find_vendor_oui(struct ib_mad_mgmt_vendor_class *vendor_class,
  1111. const char *oui)
  1112. {
  1113. int i;
  1114. for (i = 0; i < MAX_MGMT_OUI; i++)
  1115. /* Is there matching OUI for this vendor class ? */
  1116. if (!memcmp(vendor_class->oui[i], oui, 3))
  1117. return i;
  1118. return -1;
  1119. }
  1120. static int check_vendor_table(struct ib_mad_mgmt_vendor_class_table *vendor)
  1121. {
  1122. int i;
  1123. for (i = 0; i < MAX_MGMT_VENDOR_RANGE2; i++)
  1124. if (vendor->vendor_class[i])
  1125. return 1;
  1126. return 0;
  1127. }
  1128. static void remove_methods_mad_agent(struct ib_mad_mgmt_method_table *method,
  1129. struct ib_mad_agent_private *agent)
  1130. {
  1131. int i;
  1132. /* Remove any methods for this mad agent */
  1133. for (i = 0; i < IB_MGMT_MAX_METHODS; i++) {
  1134. if (method->agent[i] == agent) {
  1135. method->agent[i] = NULL;
  1136. }
  1137. }
  1138. }
  1139. static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req,
  1140. struct ib_mad_agent_private *agent_priv,
  1141. u8 mgmt_class)
  1142. {
  1143. struct ib_mad_port_private *port_priv;
  1144. struct ib_mad_mgmt_class_table **class;
  1145. struct ib_mad_mgmt_method_table **method;
  1146. int i, ret;
  1147. port_priv = agent_priv->qp_info->port_priv;
  1148. class = &port_priv->version[mad_reg_req->mgmt_class_version].class;
  1149. if (!*class) {
  1150. /* Allocate management class table for "new" class version */
  1151. *class = kzalloc(sizeof **class, GFP_ATOMIC);
  1152. if (!*class) {
  1153. ret = -ENOMEM;
  1154. goto error1;
  1155. }
  1156. /* Allocate method table for this management class */
  1157. method = &(*class)->method_table[mgmt_class];
  1158. if ((ret = allocate_method_table(method)))
  1159. goto error2;
  1160. } else {
  1161. method = &(*class)->method_table[mgmt_class];
  1162. if (!*method) {
  1163. /* Allocate method table for this management class */
  1164. if ((ret = allocate_method_table(method)))
  1165. goto error1;
  1166. }
  1167. }
  1168. /* Now, make sure methods are not already in use */
  1169. if (method_in_use(method, mad_reg_req))
  1170. goto error3;
  1171. /* Finally, add in methods being registered */
  1172. for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS)
  1173. (*method)->agent[i] = agent_priv;
  1174. return 0;
  1175. error3:
  1176. /* Remove any methods for this mad agent */
  1177. remove_methods_mad_agent(*method, agent_priv);
  1178. /* Now, check to see if there are any methods in use */
  1179. if (!check_method_table(*method)) {
  1180. /* If not, release management method table */
  1181. kfree(*method);
  1182. *method = NULL;
  1183. }
  1184. ret = -EINVAL;
  1185. goto error1;
  1186. error2:
  1187. kfree(*class);
  1188. *class = NULL;
  1189. error1:
  1190. return ret;
  1191. }
  1192. static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req,
  1193. struct ib_mad_agent_private *agent_priv)
  1194. {
  1195. struct ib_mad_port_private *port_priv;
  1196. struct ib_mad_mgmt_vendor_class_table **vendor_table;
  1197. struct ib_mad_mgmt_vendor_class_table *vendor = NULL;
  1198. struct ib_mad_mgmt_vendor_class *vendor_class = NULL;
  1199. struct ib_mad_mgmt_method_table **method;
  1200. int i, ret = -ENOMEM;
  1201. u8 vclass;
  1202. /* "New" vendor (with OUI) class */
  1203. vclass = vendor_class_index(mad_reg_req->mgmt_class);
  1204. port_priv = agent_priv->qp_info->port_priv;
  1205. vendor_table = &port_priv->version[
  1206. mad_reg_req->mgmt_class_version].vendor;
  1207. if (!*vendor_table) {
  1208. /* Allocate mgmt vendor class table for "new" class version */
  1209. vendor = kzalloc(sizeof *vendor, GFP_ATOMIC);
  1210. if (!vendor)
  1211. goto error1;
  1212. *vendor_table = vendor;
  1213. }
  1214. if (!(*vendor_table)->vendor_class[vclass]) {
  1215. /* Allocate table for this management vendor class */
  1216. vendor_class = kzalloc(sizeof *vendor_class, GFP_ATOMIC);
  1217. if (!vendor_class)
  1218. goto error2;
  1219. (*vendor_table)->vendor_class[vclass] = vendor_class;
  1220. }
  1221. for (i = 0; i < MAX_MGMT_OUI; i++) {
  1222. /* Is there matching OUI for this vendor class ? */
  1223. if (!memcmp((*vendor_table)->vendor_class[vclass]->oui[i],
  1224. mad_reg_req->oui, 3)) {
  1225. method = &(*vendor_table)->vendor_class[
  1226. vclass]->method_table[i];
  1227. if (!*method)
  1228. goto error3;
  1229. goto check_in_use;
  1230. }
  1231. }
  1232. for (i = 0; i < MAX_MGMT_OUI; i++) {
  1233. /* OUI slot available ? */
  1234. if (!is_vendor_oui((*vendor_table)->vendor_class[
  1235. vclass]->oui[i])) {
  1236. method = &(*vendor_table)->vendor_class[
  1237. vclass]->method_table[i];
  1238. /* Allocate method table for this OUI */
  1239. if (!*method) {
  1240. ret = allocate_method_table(method);
  1241. if (ret)
  1242. goto error3;
  1243. }
  1244. memcpy((*vendor_table)->vendor_class[vclass]->oui[i],
  1245. mad_reg_req->oui, 3);
  1246. goto check_in_use;
  1247. }
  1248. }
  1249. dev_err(&agent_priv->agent.device->dev, "All OUI slots in use\n");
  1250. goto error3;
  1251. check_in_use:
  1252. /* Now, make sure methods are not already in use */
  1253. if (method_in_use(method, mad_reg_req))
  1254. goto error4;
  1255. /* Finally, add in methods being registered */
  1256. for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS)
  1257. (*method)->agent[i] = agent_priv;
  1258. return 0;
  1259. error4:
  1260. /* Remove any methods for this mad agent */
  1261. remove_methods_mad_agent(*method, agent_priv);
  1262. /* Now, check to see if there are any methods in use */
  1263. if (!check_method_table(*method)) {
  1264. /* If not, release management method table */
  1265. kfree(*method);
  1266. *method = NULL;
  1267. }
  1268. ret = -EINVAL;
  1269. error3:
  1270. if (vendor_class) {
  1271. (*vendor_table)->vendor_class[vclass] = NULL;
  1272. kfree(vendor_class);
  1273. }
  1274. error2:
  1275. if (vendor) {
  1276. *vendor_table = NULL;
  1277. kfree(vendor);
  1278. }
  1279. error1:
  1280. return ret;
  1281. }
  1282. static void remove_mad_reg_req(struct ib_mad_agent_private *agent_priv)
  1283. {
  1284. struct ib_mad_port_private *port_priv;
  1285. struct ib_mad_mgmt_class_table *class;
  1286. struct ib_mad_mgmt_method_table *method;
  1287. struct ib_mad_mgmt_vendor_class_table *vendor;
  1288. struct ib_mad_mgmt_vendor_class *vendor_class;
  1289. int index;
  1290. u8 mgmt_class;
  1291. /*
  1292. * Was MAD registration request supplied
  1293. * with original registration ?
  1294. */
  1295. if (!agent_priv->reg_req) {
  1296. goto out;
  1297. }
  1298. port_priv = agent_priv->qp_info->port_priv;
  1299. mgmt_class = convert_mgmt_class(agent_priv->reg_req->mgmt_class);
  1300. class = port_priv->version[
  1301. agent_priv->reg_req->mgmt_class_version].class;
  1302. if (!class)
  1303. goto vendor_check;
  1304. method = class->method_table[mgmt_class];
  1305. if (method) {
  1306. /* Remove any methods for this mad agent */
  1307. remove_methods_mad_agent(method, agent_priv);
  1308. /* Now, check to see if there are any methods still in use */
  1309. if (!check_method_table(method)) {
  1310. /* If not, release management method table */
  1311. kfree(method);
  1312. class->method_table[mgmt_class] = NULL;
  1313. /* Any management classes left ? */
  1314. if (!check_class_table(class)) {
  1315. /* If not, release management class table */
  1316. kfree(class);
  1317. port_priv->version[
  1318. agent_priv->reg_req->
  1319. mgmt_class_version].class = NULL;
  1320. }
  1321. }
  1322. }
  1323. vendor_check:
  1324. if (!is_vendor_class(mgmt_class))
  1325. goto out;
  1326. /* normalize mgmt_class to vendor range 2 */
  1327. mgmt_class = vendor_class_index(agent_priv->reg_req->mgmt_class);
  1328. vendor = port_priv->version[
  1329. agent_priv->reg_req->mgmt_class_version].vendor;
  1330. if (!vendor)
  1331. goto out;
  1332. vendor_class = vendor->vendor_class[mgmt_class];
  1333. if (vendor_class) {
  1334. index = find_vendor_oui(vendor_class, agent_priv->reg_req->oui);
  1335. if (index < 0)
  1336. goto out;
  1337. method = vendor_class->method_table[index];
  1338. if (method) {
  1339. /* Remove any methods for this mad agent */
  1340. remove_methods_mad_agent(method, agent_priv);
  1341. /*
  1342. * Now, check to see if there are
  1343. * any methods still in use
  1344. */
  1345. if (!check_method_table(method)) {
  1346. /* If not, release management method table */
  1347. kfree(method);
  1348. vendor_class->method_table[index] = NULL;
  1349. memset(vendor_class->oui[index], 0, 3);
  1350. /* Any OUIs left ? */
  1351. if (!check_vendor_class(vendor_class)) {
  1352. /* If not, release vendor class table */
  1353. kfree(vendor_class);
  1354. vendor->vendor_class[mgmt_class] = NULL;
  1355. /* Any other vendor classes left ? */
  1356. if (!check_vendor_table(vendor)) {
  1357. kfree(vendor);
  1358. port_priv->version[
  1359. agent_priv->reg_req->
  1360. mgmt_class_version].
  1361. vendor = NULL;
  1362. }
  1363. }
  1364. }
  1365. }
  1366. }
  1367. out:
  1368. return;
  1369. }
  1370. static struct ib_mad_agent_private *
  1371. find_mad_agent(struct ib_mad_port_private *port_priv,
  1372. const struct ib_mad_hdr *mad_hdr)
  1373. {
  1374. struct ib_mad_agent_private *mad_agent = NULL;
  1375. unsigned long flags;
  1376. if (ib_response_mad(mad_hdr)) {
  1377. u32 hi_tid;
  1378. /*
  1379. * Routing is based on high 32 bits of transaction ID
  1380. * of MAD.
  1381. */
  1382. hi_tid = be64_to_cpu(mad_hdr->tid) >> 32;
  1383. rcu_read_lock();
  1384. mad_agent = xa_load(&ib_mad_clients, hi_tid);
  1385. if (mad_agent && !refcount_inc_not_zero(&mad_agent->refcount))
  1386. mad_agent = NULL;
  1387. rcu_read_unlock();
  1388. } else {
  1389. struct ib_mad_mgmt_class_table *class;
  1390. struct ib_mad_mgmt_method_table *method;
  1391. struct ib_mad_mgmt_vendor_class_table *vendor;
  1392. struct ib_mad_mgmt_vendor_class *vendor_class;
  1393. const struct ib_vendor_mad *vendor_mad;
  1394. int index;
  1395. spin_lock_irqsave(&port_priv->reg_lock, flags);
  1396. /*
  1397. * Routing is based on version, class, and method
  1398. * For "newer" vendor MADs, also based on OUI
  1399. */
  1400. if (mad_hdr->class_version >= MAX_MGMT_VERSION)
  1401. goto out;
  1402. if (!is_vendor_class(mad_hdr->mgmt_class)) {
  1403. class = port_priv->version[
  1404. mad_hdr->class_version].class;
  1405. if (!class)
  1406. goto out;
  1407. if (convert_mgmt_class(mad_hdr->mgmt_class) >=
  1408. ARRAY_SIZE(class->method_table))
  1409. goto out;
  1410. method = class->method_table[convert_mgmt_class(
  1411. mad_hdr->mgmt_class)];
  1412. if (method)
  1413. mad_agent = method->agent[mad_hdr->method &
  1414. ~IB_MGMT_METHOD_RESP];
  1415. } else {
  1416. vendor = port_priv->version[
  1417. mad_hdr->class_version].vendor;
  1418. if (!vendor)
  1419. goto out;
  1420. vendor_class = vendor->vendor_class[vendor_class_index(
  1421. mad_hdr->mgmt_class)];
  1422. if (!vendor_class)
  1423. goto out;
  1424. /* Find matching OUI */
  1425. vendor_mad = (const struct ib_vendor_mad *)mad_hdr;
  1426. index = find_vendor_oui(vendor_class, vendor_mad->oui);
  1427. if (index == -1)
  1428. goto out;
  1429. method = vendor_class->method_table[index];
  1430. if (method) {
  1431. mad_agent = method->agent[mad_hdr->method &
  1432. ~IB_MGMT_METHOD_RESP];
  1433. }
  1434. }
  1435. if (mad_agent)
  1436. refcount_inc(&mad_agent->refcount);
  1437. out:
  1438. spin_unlock_irqrestore(&port_priv->reg_lock, flags);
  1439. }
  1440. if (mad_agent && !mad_agent->agent.recv_handler) {
  1441. dev_notice(&port_priv->device->dev,
  1442. "No receive handler for client %p on port %d\n",
  1443. &mad_agent->agent, port_priv->port_num);
  1444. deref_mad_agent(mad_agent);
  1445. mad_agent = NULL;
  1446. }
  1447. return mad_agent;
  1448. }
  1449. static int validate_mad(const struct ib_mad_hdr *mad_hdr,
  1450. const struct ib_mad_qp_info *qp_info,
  1451. bool opa)
  1452. {
  1453. int valid = 0;
  1454. u32 qp_num = qp_info->qp->qp_num;
  1455. /* Make sure MAD base version is understood */
  1456. if (mad_hdr->base_version != IB_MGMT_BASE_VERSION &&
  1457. (!opa || mad_hdr->base_version != OPA_MGMT_BASE_VERSION)) {
  1458. pr_err("MAD received with unsupported base version %d %s\n",
  1459. mad_hdr->base_version, opa ? "(opa)" : "");
  1460. goto out;
  1461. }
  1462. /* Filter SMI packets sent to other than QP0 */
  1463. if ((mad_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED) ||
  1464. (mad_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
  1465. if (qp_num == 0)
  1466. valid = 1;
  1467. } else {
  1468. /* CM attributes other than ClassPortInfo only use Send method */
  1469. if ((mad_hdr->mgmt_class == IB_MGMT_CLASS_CM) &&
  1470. (mad_hdr->attr_id != IB_MGMT_CLASSPORTINFO_ATTR_ID) &&
  1471. (mad_hdr->method != IB_MGMT_METHOD_SEND))
  1472. goto out;
  1473. /* Filter GSI packets sent to QP0 */
  1474. if (qp_num != 0)
  1475. valid = 1;
  1476. }
  1477. out:
  1478. return valid;
  1479. }
  1480. static int is_rmpp_data_mad(const struct ib_mad_agent_private *mad_agent_priv,
  1481. const struct ib_mad_hdr *mad_hdr)
  1482. {
  1483. struct ib_rmpp_mad *rmpp_mad;
  1484. rmpp_mad = (struct ib_rmpp_mad *)mad_hdr;
  1485. return !mad_agent_priv->agent.rmpp_version ||
  1486. !ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent) ||
  1487. !(ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) &
  1488. IB_MGMT_RMPP_FLAG_ACTIVE) ||
  1489. (rmpp_mad->rmpp_hdr.rmpp_type == IB_MGMT_RMPP_TYPE_DATA);
  1490. }
  1491. static inline int rcv_has_same_class(const struct ib_mad_send_wr_private *wr,
  1492. const struct ib_mad_recv_wc *rwc)
  1493. {
  1494. return ((struct ib_mad_hdr *)(wr->send_buf.mad))->mgmt_class ==
  1495. rwc->recv_buf.mad->mad_hdr.mgmt_class;
  1496. }
  1497. static inline int rcv_has_same_gid(const struct ib_mad_agent_private *mad_agent_priv,
  1498. const struct ib_mad_send_wr_private *wr,
  1499. const struct ib_mad_recv_wc *rwc )
  1500. {
  1501. struct rdma_ah_attr attr;
  1502. u8 send_resp, rcv_resp;
  1503. union ib_gid sgid;
  1504. struct ib_device *device = mad_agent_priv->agent.device;
  1505. u8 port_num = mad_agent_priv->agent.port_num;
  1506. u8 lmc;
  1507. bool has_grh;
  1508. send_resp = ib_response_mad((struct ib_mad_hdr *)wr->send_buf.mad);
  1509. rcv_resp = ib_response_mad(&rwc->recv_buf.mad->mad_hdr);
  1510. if (send_resp == rcv_resp)
  1511. /* both requests, or both responses. GIDs different */
  1512. return 0;
  1513. if (rdma_query_ah(wr->send_buf.ah, &attr))
  1514. /* Assume not equal, to avoid false positives. */
  1515. return 0;
  1516. has_grh = !!(rdma_ah_get_ah_flags(&attr) & IB_AH_GRH);
  1517. if (has_grh != !!(rwc->wc->wc_flags & IB_WC_GRH))
  1518. /* one has GID, other does not. Assume different */
  1519. return 0;
  1520. if (!send_resp && rcv_resp) {
  1521. /* is request/response. */
  1522. if (!has_grh) {
  1523. if (ib_get_cached_lmc(device, port_num, &lmc))
  1524. return 0;
  1525. return (!lmc || !((rdma_ah_get_path_bits(&attr) ^
  1526. rwc->wc->dlid_path_bits) &
  1527. ((1 << lmc) - 1)));
  1528. } else {
  1529. const struct ib_global_route *grh =
  1530. rdma_ah_read_grh(&attr);
  1531. if (rdma_query_gid(device, port_num,
  1532. grh->sgid_index, &sgid))
  1533. return 0;
  1534. return !memcmp(sgid.raw, rwc->recv_buf.grh->dgid.raw,
  1535. 16);
  1536. }
  1537. }
  1538. if (!has_grh)
  1539. return rdma_ah_get_dlid(&attr) == rwc->wc->slid;
  1540. else
  1541. return !memcmp(rdma_ah_read_grh(&attr)->dgid.raw,
  1542. rwc->recv_buf.grh->sgid.raw,
  1543. 16);
  1544. }
  1545. static inline int is_direct(u8 class)
  1546. {
  1547. return (class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE);
  1548. }
  1549. struct ib_mad_send_wr_private*
  1550. ib_find_send_mad(const struct ib_mad_agent_private *mad_agent_priv,
  1551. const struct ib_mad_recv_wc *wc)
  1552. {
  1553. struct ib_mad_send_wr_private *wr;
  1554. const struct ib_mad_hdr *mad_hdr;
  1555. mad_hdr = &wc->recv_buf.mad->mad_hdr;
  1556. list_for_each_entry(wr, &mad_agent_priv->wait_list, agent_list) {
  1557. if ((wr->tid == mad_hdr->tid) &&
  1558. rcv_has_same_class(wr, wc) &&
  1559. /*
  1560. * Don't check GID for direct routed MADs.
  1561. * These might have permissive LIDs.
  1562. */
  1563. (is_direct(mad_hdr->mgmt_class) ||
  1564. rcv_has_same_gid(mad_agent_priv, wr, wc)))
  1565. return (wr->status == IB_WC_SUCCESS) ? wr : NULL;
  1566. }
  1567. /*
  1568. * It's possible to receive the response before we've
  1569. * been notified that the send has completed
  1570. */
  1571. list_for_each_entry(wr, &mad_agent_priv->send_list, agent_list) {
  1572. if (is_rmpp_data_mad(mad_agent_priv, wr->send_buf.mad) &&
  1573. wr->tid == mad_hdr->tid &&
  1574. wr->timeout &&
  1575. rcv_has_same_class(wr, wc) &&
  1576. /*
  1577. * Don't check GID for direct routed MADs.
  1578. * These might have permissive LIDs.
  1579. */
  1580. (is_direct(mad_hdr->mgmt_class) ||
  1581. rcv_has_same_gid(mad_agent_priv, wr, wc)))
  1582. /* Verify request has not been canceled */
  1583. return (wr->status == IB_WC_SUCCESS) ? wr : NULL;
  1584. }
  1585. return NULL;
  1586. }
  1587. void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr)
  1588. {
  1589. mad_send_wr->timeout = 0;
  1590. if (mad_send_wr->refcount == 1)
  1591. list_move_tail(&mad_send_wr->agent_list,
  1592. &mad_send_wr->mad_agent_priv->done_list);
  1593. }
  1594. static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
  1595. struct ib_mad_recv_wc *mad_recv_wc)
  1596. {
  1597. struct ib_mad_send_wr_private *mad_send_wr;
  1598. struct ib_mad_send_wc mad_send_wc;
  1599. unsigned long flags;
  1600. int ret;
  1601. INIT_LIST_HEAD(&mad_recv_wc->rmpp_list);
  1602. ret = ib_mad_enforce_security(mad_agent_priv,
  1603. mad_recv_wc->wc->pkey_index);
  1604. if (ret) {
  1605. ib_free_recv_mad(mad_recv_wc);
  1606. deref_mad_agent(mad_agent_priv);
  1607. return;
  1608. }
  1609. list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list);
  1610. if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
  1611. mad_recv_wc = ib_process_rmpp_recv_wc(mad_agent_priv,
  1612. mad_recv_wc);
  1613. if (!mad_recv_wc) {
  1614. deref_mad_agent(mad_agent_priv);
  1615. return;
  1616. }
  1617. }
  1618. /* Complete corresponding request */
  1619. if (ib_response_mad(&mad_recv_wc->recv_buf.mad->mad_hdr)) {
  1620. spin_lock_irqsave(&mad_agent_priv->lock, flags);
  1621. mad_send_wr = ib_find_send_mad(mad_agent_priv, mad_recv_wc);
  1622. if (!mad_send_wr) {
  1623. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  1624. if (!ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)
  1625. && ib_is_mad_class_rmpp(mad_recv_wc->recv_buf.mad->mad_hdr.mgmt_class)
  1626. && (ib_get_rmpp_flags(&((struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad)->rmpp_hdr)
  1627. & IB_MGMT_RMPP_FLAG_ACTIVE)) {
  1628. /* user rmpp is in effect
  1629. * and this is an active RMPP MAD
  1630. */
  1631. mad_agent_priv->agent.recv_handler(
  1632. &mad_agent_priv->agent, NULL,
  1633. mad_recv_wc);
  1634. deref_mad_agent(mad_agent_priv);
  1635. } else {
  1636. /* not user rmpp, revert to normal behavior and
  1637. * drop the mad */
  1638. ib_free_recv_mad(mad_recv_wc);
  1639. deref_mad_agent(mad_agent_priv);
  1640. return;
  1641. }
  1642. } else {
  1643. ib_mark_mad_done(mad_send_wr);
  1644. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  1645. /* Defined behavior is to complete response before request */
  1646. mad_agent_priv->agent.recv_handler(
  1647. &mad_agent_priv->agent,
  1648. &mad_send_wr->send_buf,
  1649. mad_recv_wc);
  1650. deref_mad_agent(mad_agent_priv);
  1651. mad_send_wc.status = IB_WC_SUCCESS;
  1652. mad_send_wc.vendor_err = 0;
  1653. mad_send_wc.send_buf = &mad_send_wr->send_buf;
  1654. ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc);
  1655. }
  1656. } else {
  1657. mad_agent_priv->agent.recv_handler(&mad_agent_priv->agent, NULL,
  1658. mad_recv_wc);
  1659. deref_mad_agent(mad_agent_priv);
  1660. }
  1661. return;
  1662. }
  1663. static enum smi_action handle_ib_smi(const struct ib_mad_port_private *port_priv,
  1664. const struct ib_mad_qp_info *qp_info,
  1665. const struct ib_wc *wc,
  1666. int port_num,
  1667. struct ib_mad_private *recv,
  1668. struct ib_mad_private *response)
  1669. {
  1670. enum smi_forward_action retsmi;
  1671. struct ib_smp *smp = (struct ib_smp *)recv->mad;
  1672. trace_ib_mad_handle_ib_smi(smp);
  1673. if (smi_handle_dr_smp_recv(smp,
  1674. rdma_cap_ib_switch(port_priv->device),
  1675. port_num,
  1676. port_priv->device->phys_port_cnt) ==
  1677. IB_SMI_DISCARD)
  1678. return IB_SMI_DISCARD;
  1679. retsmi = smi_check_forward_dr_smp(smp);
  1680. if (retsmi == IB_SMI_LOCAL)
  1681. return IB_SMI_HANDLE;
  1682. if (retsmi == IB_SMI_SEND) { /* don't forward */
  1683. if (smi_handle_dr_smp_send(smp,
  1684. rdma_cap_ib_switch(port_priv->device),
  1685. port_num) == IB_SMI_DISCARD)
  1686. return IB_SMI_DISCARD;
  1687. if (smi_check_local_smp(smp, port_priv->device) == IB_SMI_DISCARD)
  1688. return IB_SMI_DISCARD;
  1689. } else if (rdma_cap_ib_switch(port_priv->device)) {
  1690. /* forward case for switches */
  1691. memcpy(response, recv, mad_priv_size(response));
  1692. response->header.recv_wc.wc = &response->header.wc;
  1693. response->header.recv_wc.recv_buf.mad = (struct ib_mad *)response->mad;
  1694. response->header.recv_wc.recv_buf.grh = &response->grh;
  1695. agent_send_response((const struct ib_mad_hdr *)response->mad,
  1696. &response->grh, wc,
  1697. port_priv->device,
  1698. smi_get_fwd_port(smp),
  1699. qp_info->qp->qp_num,
  1700. response->mad_size,
  1701. false);
  1702. return IB_SMI_DISCARD;
  1703. }
  1704. return IB_SMI_HANDLE;
  1705. }
  1706. static bool generate_unmatched_resp(const struct ib_mad_private *recv,
  1707. struct ib_mad_private *response,
  1708. size_t *resp_len, bool opa)
  1709. {
  1710. const struct ib_mad_hdr *recv_hdr = (const struct ib_mad_hdr *)recv->mad;
  1711. struct ib_mad_hdr *resp_hdr = (struct ib_mad_hdr *)response->mad;
  1712. if (recv_hdr->method == IB_MGMT_METHOD_GET ||
  1713. recv_hdr->method == IB_MGMT_METHOD_SET) {
  1714. memcpy(response, recv, mad_priv_size(response));
  1715. response->header.recv_wc.wc = &response->header.wc;
  1716. response->header.recv_wc.recv_buf.mad = (struct ib_mad *)response->mad;
  1717. response->header.recv_wc.recv_buf.grh = &response->grh;
  1718. resp_hdr->method = IB_MGMT_METHOD_GET_RESP;
  1719. resp_hdr->status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB);
  1720. if (recv_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
  1721. resp_hdr->status |= IB_SMP_DIRECTION;
  1722. if (opa && recv_hdr->base_version == OPA_MGMT_BASE_VERSION) {
  1723. if (recv_hdr->mgmt_class ==
  1724. IB_MGMT_CLASS_SUBN_LID_ROUTED ||
  1725. recv_hdr->mgmt_class ==
  1726. IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
  1727. *resp_len = opa_get_smp_header_size(
  1728. (struct opa_smp *)recv->mad);
  1729. else
  1730. *resp_len = sizeof(struct ib_mad_hdr);
  1731. }
  1732. return true;
  1733. } else {
  1734. return false;
  1735. }
  1736. }
  1737. static enum smi_action
  1738. handle_opa_smi(struct ib_mad_port_private *port_priv,
  1739. struct ib_mad_qp_info *qp_info,
  1740. struct ib_wc *wc,
  1741. int port_num,
  1742. struct ib_mad_private *recv,
  1743. struct ib_mad_private *response)
  1744. {
  1745. enum smi_forward_action retsmi;
  1746. struct opa_smp *smp = (struct opa_smp *)recv->mad;
  1747. trace_ib_mad_handle_opa_smi(smp);
  1748. if (opa_smi_handle_dr_smp_recv(smp,
  1749. rdma_cap_ib_switch(port_priv->device),
  1750. port_num,
  1751. port_priv->device->phys_port_cnt) ==
  1752. IB_SMI_DISCARD)
  1753. return IB_SMI_DISCARD;
  1754. retsmi = opa_smi_check_forward_dr_smp(smp);
  1755. if (retsmi == IB_SMI_LOCAL)
  1756. return IB_SMI_HANDLE;
  1757. if (retsmi == IB_SMI_SEND) { /* don't forward */
  1758. if (opa_smi_handle_dr_smp_send(smp,
  1759. rdma_cap_ib_switch(port_priv->device),
  1760. port_num) == IB_SMI_DISCARD)
  1761. return IB_SMI_DISCARD;
  1762. if (opa_smi_check_local_smp(smp, port_priv->device) ==
  1763. IB_SMI_DISCARD)
  1764. return IB_SMI_DISCARD;
  1765. } else if (rdma_cap_ib_switch(port_priv->device)) {
  1766. /* forward case for switches */
  1767. memcpy(response, recv, mad_priv_size(response));
  1768. response->header.recv_wc.wc = &response->header.wc;
  1769. response->header.recv_wc.recv_buf.opa_mad =
  1770. (struct opa_mad *)response->mad;
  1771. response->header.recv_wc.recv_buf.grh = &response->grh;
  1772. agent_send_response((const struct ib_mad_hdr *)response->mad,
  1773. &response->grh, wc,
  1774. port_priv->device,
  1775. opa_smi_get_fwd_port(smp),
  1776. qp_info->qp->qp_num,
  1777. recv->header.wc.byte_len,
  1778. true);
  1779. return IB_SMI_DISCARD;
  1780. }
  1781. return IB_SMI_HANDLE;
  1782. }
  1783. static enum smi_action
  1784. handle_smi(struct ib_mad_port_private *port_priv,
  1785. struct ib_mad_qp_info *qp_info,
  1786. struct ib_wc *wc,
  1787. int port_num,
  1788. struct ib_mad_private *recv,
  1789. struct ib_mad_private *response,
  1790. bool opa)
  1791. {
  1792. struct ib_mad_hdr *mad_hdr = (struct ib_mad_hdr *)recv->mad;
  1793. if (opa && mad_hdr->base_version == OPA_MGMT_BASE_VERSION &&
  1794. mad_hdr->class_version == OPA_SM_CLASS_VERSION)
  1795. return handle_opa_smi(port_priv, qp_info, wc, port_num, recv,
  1796. response);
  1797. return handle_ib_smi(port_priv, qp_info, wc, port_num, recv, response);
  1798. }
  1799. static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc)
  1800. {
  1801. struct ib_mad_port_private *port_priv = cq->cq_context;
  1802. struct ib_mad_list_head *mad_list =
  1803. container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
  1804. struct ib_mad_qp_info *qp_info;
  1805. struct ib_mad_private_header *mad_priv_hdr;
  1806. struct ib_mad_private *recv, *response = NULL;
  1807. struct ib_mad_agent_private *mad_agent;
  1808. int port_num;
  1809. int ret = IB_MAD_RESULT_SUCCESS;
  1810. size_t mad_size;
  1811. u16 resp_mad_pkey_index = 0;
  1812. bool opa;
  1813. if (list_empty_careful(&port_priv->port_list))
  1814. return;
  1815. if (wc->status != IB_WC_SUCCESS) {
  1816. /*
  1817. * Receive errors indicate that the QP has entered the error
  1818. * state - error handling/shutdown code will cleanup
  1819. */
  1820. return;
  1821. }
  1822. qp_info = mad_list->mad_queue->qp_info;
  1823. dequeue_mad(mad_list);
  1824. opa = rdma_cap_opa_mad(qp_info->port_priv->device,
  1825. qp_info->port_priv->port_num);
  1826. mad_priv_hdr = container_of(mad_list, struct ib_mad_private_header,
  1827. mad_list);
  1828. recv = container_of(mad_priv_hdr, struct ib_mad_private, header);
  1829. ib_dma_unmap_single(port_priv->device,
  1830. recv->header.mapping,
  1831. mad_priv_dma_size(recv),
  1832. DMA_FROM_DEVICE);
  1833. /* Setup MAD receive work completion from "normal" work completion */
  1834. recv->header.wc = *wc;
  1835. recv->header.recv_wc.wc = &recv->header.wc;
  1836. if (opa && ((struct ib_mad_hdr *)(recv->mad))->base_version == OPA_MGMT_BASE_VERSION) {
  1837. recv->header.recv_wc.mad_len = wc->byte_len - sizeof(struct ib_grh);
  1838. recv->header.recv_wc.mad_seg_size = sizeof(struct opa_mad);
  1839. } else {
  1840. recv->header.recv_wc.mad_len = sizeof(struct ib_mad);
  1841. recv->header.recv_wc.mad_seg_size = sizeof(struct ib_mad);
  1842. }
  1843. recv->header.recv_wc.recv_buf.mad = (struct ib_mad *)recv->mad;
  1844. recv->header.recv_wc.recv_buf.grh = &recv->grh;
  1845. /* Validate MAD */
  1846. if (!validate_mad((const struct ib_mad_hdr *)recv->mad, qp_info, opa))
  1847. goto out;
  1848. trace_ib_mad_recv_done_handler(qp_info, wc,
  1849. (struct ib_mad_hdr *)recv->mad);
  1850. mad_size = recv->mad_size;
  1851. response = alloc_mad_private(mad_size, GFP_KERNEL);
  1852. if (!response)
  1853. goto out;
  1854. if (rdma_cap_ib_switch(port_priv->device))
  1855. port_num = wc->port_num;
  1856. else
  1857. port_num = port_priv->port_num;
  1858. if (((struct ib_mad_hdr *)recv->mad)->mgmt_class ==
  1859. IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
  1860. if (handle_smi(port_priv, qp_info, wc, port_num, recv,
  1861. response, opa)
  1862. == IB_SMI_DISCARD)
  1863. goto out;
  1864. }
  1865. /* Give driver "right of first refusal" on incoming MAD */
  1866. if (port_priv->device->ops.process_mad) {
  1867. ret = port_priv->device->ops.process_mad(
  1868. port_priv->device, 0, port_priv->port_num, wc,
  1869. &recv->grh, (const struct ib_mad *)recv->mad,
  1870. (struct ib_mad *)response->mad, &mad_size,
  1871. &resp_mad_pkey_index);
  1872. if (opa)
  1873. wc->pkey_index = resp_mad_pkey_index;
  1874. if (ret & IB_MAD_RESULT_SUCCESS) {
  1875. if (ret & IB_MAD_RESULT_CONSUMED)
  1876. goto out;
  1877. if (ret & IB_MAD_RESULT_REPLY) {
  1878. agent_send_response((const struct ib_mad_hdr *)response->mad,
  1879. &recv->grh, wc,
  1880. port_priv->device,
  1881. port_num,
  1882. qp_info->qp->qp_num,
  1883. mad_size, opa);
  1884. goto out;
  1885. }
  1886. }
  1887. }
  1888. mad_agent = find_mad_agent(port_priv, (const struct ib_mad_hdr *)recv->mad);
  1889. if (mad_agent) {
  1890. trace_ib_mad_recv_done_agent(mad_agent);
  1891. ib_mad_complete_recv(mad_agent, &recv->header.recv_wc);
  1892. /*
  1893. * recv is freed up in error cases in ib_mad_complete_recv
  1894. * or via recv_handler in ib_mad_complete_recv()
  1895. */
  1896. recv = NULL;
  1897. } else if ((ret & IB_MAD_RESULT_SUCCESS) &&
  1898. generate_unmatched_resp(recv, response, &mad_size, opa)) {
  1899. agent_send_response((const struct ib_mad_hdr *)response->mad, &recv->grh, wc,
  1900. port_priv->device, port_num,
  1901. qp_info->qp->qp_num, mad_size, opa);
  1902. }
  1903. out:
  1904. /* Post another receive request for this QP */
  1905. if (response) {
  1906. ib_mad_post_receive_mads(qp_info, response);
  1907. kfree(recv);
  1908. } else
  1909. ib_mad_post_receive_mads(qp_info, recv);
  1910. }
  1911. static void adjust_timeout(struct ib_mad_agent_private *mad_agent_priv)
  1912. {
  1913. struct ib_mad_send_wr_private *mad_send_wr;
  1914. unsigned long delay;
  1915. if (list_empty(&mad_agent_priv->wait_list)) {
  1916. cancel_delayed_work(&mad_agent_priv->timed_work);
  1917. } else {
  1918. mad_send_wr = list_entry(mad_agent_priv->wait_list.next,
  1919. struct ib_mad_send_wr_private,
  1920. agent_list);
  1921. if (time_after(mad_agent_priv->timeout,
  1922. mad_send_wr->timeout)) {
  1923. mad_agent_priv->timeout = mad_send_wr->timeout;
  1924. delay = mad_send_wr->timeout - jiffies;
  1925. if ((long)delay <= 0)
  1926. delay = 1;
  1927. mod_delayed_work(mad_agent_priv->qp_info->port_priv->wq,
  1928. &mad_agent_priv->timed_work, delay);
  1929. }
  1930. }
  1931. }
  1932. static void wait_for_response(struct ib_mad_send_wr_private *mad_send_wr)
  1933. {
  1934. struct ib_mad_agent_private *mad_agent_priv;
  1935. struct ib_mad_send_wr_private *temp_mad_send_wr;
  1936. struct list_head *list_item;
  1937. unsigned long delay;
  1938. mad_agent_priv = mad_send_wr->mad_agent_priv;
  1939. list_del(&mad_send_wr->agent_list);
  1940. delay = mad_send_wr->timeout;
  1941. mad_send_wr->timeout += jiffies;
  1942. if (delay) {
  1943. list_for_each_prev(list_item, &mad_agent_priv->wait_list) {
  1944. temp_mad_send_wr = list_entry(list_item,
  1945. struct ib_mad_send_wr_private,
  1946. agent_list);
  1947. if (time_after(mad_send_wr->timeout,
  1948. temp_mad_send_wr->timeout))
  1949. break;
  1950. }
  1951. }
  1952. else
  1953. list_item = &mad_agent_priv->wait_list;
  1954. list_add(&mad_send_wr->agent_list, list_item);
  1955. /* Reschedule a work item if we have a shorter timeout */
  1956. if (mad_agent_priv->wait_list.next == &mad_send_wr->agent_list)
  1957. mod_delayed_work(mad_agent_priv->qp_info->port_priv->wq,
  1958. &mad_agent_priv->timed_work, delay);
  1959. }
  1960. void ib_reset_mad_timeout(struct ib_mad_send_wr_private *mad_send_wr,
  1961. unsigned long timeout_ms)
  1962. {
  1963. mad_send_wr->timeout = msecs_to_jiffies(timeout_ms);
  1964. wait_for_response(mad_send_wr);
  1965. }
  1966. /*
  1967. * Process a send work completion
  1968. */
  1969. void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
  1970. struct ib_mad_send_wc *mad_send_wc)
  1971. {
  1972. struct ib_mad_agent_private *mad_agent_priv;
  1973. unsigned long flags;
  1974. int ret;
  1975. mad_agent_priv = mad_send_wr->mad_agent_priv;
  1976. spin_lock_irqsave(&mad_agent_priv->lock, flags);
  1977. if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
  1978. ret = ib_process_rmpp_send_wc(mad_send_wr, mad_send_wc);
  1979. if (ret == IB_RMPP_RESULT_CONSUMED)
  1980. goto done;
  1981. } else
  1982. ret = IB_RMPP_RESULT_UNHANDLED;
  1983. if (mad_send_wc->status != IB_WC_SUCCESS &&
  1984. mad_send_wr->status == IB_WC_SUCCESS) {
  1985. mad_send_wr->status = mad_send_wc->status;
  1986. mad_send_wr->refcount -= (mad_send_wr->timeout > 0);
  1987. }
  1988. if (--mad_send_wr->refcount > 0) {
  1989. if (mad_send_wr->refcount == 1 && mad_send_wr->timeout &&
  1990. mad_send_wr->status == IB_WC_SUCCESS) {
  1991. wait_for_response(mad_send_wr);
  1992. }
  1993. goto done;
  1994. }
  1995. /* Remove send from MAD agent and notify client of completion */
  1996. list_del(&mad_send_wr->agent_list);
  1997. adjust_timeout(mad_agent_priv);
  1998. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  1999. if (mad_send_wr->status != IB_WC_SUCCESS )
  2000. mad_send_wc->status = mad_send_wr->status;
  2001. if (ret == IB_RMPP_RESULT_INTERNAL)
  2002. ib_rmpp_send_handler(mad_send_wc);
  2003. else
  2004. mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
  2005. mad_send_wc);
  2006. /* Release reference on agent taken when sending */
  2007. deref_mad_agent(mad_agent_priv);
  2008. return;
  2009. done:
  2010. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  2011. }
  2012. static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc)
  2013. {
  2014. struct ib_mad_port_private *port_priv = cq->cq_context;
  2015. struct ib_mad_list_head *mad_list =
  2016. container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
  2017. struct ib_mad_send_wr_private *mad_send_wr, *queued_send_wr;
  2018. struct ib_mad_qp_info *qp_info;
  2019. struct ib_mad_queue *send_queue;
  2020. struct ib_mad_send_wc mad_send_wc;
  2021. unsigned long flags;
  2022. int ret;
  2023. if (list_empty_careful(&port_priv->port_list))
  2024. return;
  2025. if (wc->status != IB_WC_SUCCESS) {
  2026. if (!ib_mad_send_error(port_priv, wc))
  2027. return;
  2028. }
  2029. mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private,
  2030. mad_list);
  2031. send_queue = mad_list->mad_queue;
  2032. qp_info = send_queue->qp_info;
  2033. trace_ib_mad_send_done_agent(mad_send_wr->mad_agent_priv);
  2034. trace_ib_mad_send_done_handler(mad_send_wr, wc);
  2035. retry:
  2036. ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device,
  2037. mad_send_wr->header_mapping,
  2038. mad_send_wr->sg_list[0].length, DMA_TO_DEVICE);
  2039. ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device,
  2040. mad_send_wr->payload_mapping,
  2041. mad_send_wr->sg_list[1].length, DMA_TO_DEVICE);
  2042. queued_send_wr = NULL;
  2043. spin_lock_irqsave(&send_queue->lock, flags);
  2044. list_del(&mad_list->list);
  2045. /* Move queued send to the send queue */
  2046. if (send_queue->count-- > send_queue->max_active) {
  2047. mad_list = container_of(qp_info->overflow_list.next,
  2048. struct ib_mad_list_head, list);
  2049. queued_send_wr = container_of(mad_list,
  2050. struct ib_mad_send_wr_private,
  2051. mad_list);
  2052. list_move_tail(&mad_list->list, &send_queue->list);
  2053. }
  2054. spin_unlock_irqrestore(&send_queue->lock, flags);
  2055. mad_send_wc.send_buf = &mad_send_wr->send_buf;
  2056. mad_send_wc.status = wc->status;
  2057. mad_send_wc.vendor_err = wc->vendor_err;
  2058. ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc);
  2059. if (queued_send_wr) {
  2060. trace_ib_mad_send_done_resend(queued_send_wr, qp_info);
  2061. ret = ib_post_send(qp_info->qp, &queued_send_wr->send_wr.wr,
  2062. NULL);
  2063. if (ret) {
  2064. dev_err(&port_priv->device->dev,
  2065. "ib_post_send failed: %d\n", ret);
  2066. mad_send_wr = queued_send_wr;
  2067. wc->status = IB_WC_LOC_QP_OP_ERR;
  2068. goto retry;
  2069. }
  2070. }
  2071. }
  2072. static void mark_sends_for_retry(struct ib_mad_qp_info *qp_info)
  2073. {
  2074. struct ib_mad_send_wr_private *mad_send_wr;
  2075. struct ib_mad_list_head *mad_list;
  2076. unsigned long flags;
  2077. spin_lock_irqsave(&qp_info->send_queue.lock, flags);
  2078. list_for_each_entry(mad_list, &qp_info->send_queue.list, list) {
  2079. mad_send_wr = container_of(mad_list,
  2080. struct ib_mad_send_wr_private,
  2081. mad_list);
  2082. mad_send_wr->retry = 1;
  2083. }
  2084. spin_unlock_irqrestore(&qp_info->send_queue.lock, flags);
  2085. }
  2086. static bool ib_mad_send_error(struct ib_mad_port_private *port_priv,
  2087. struct ib_wc *wc)
  2088. {
  2089. struct ib_mad_list_head *mad_list =
  2090. container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
  2091. struct ib_mad_qp_info *qp_info = mad_list->mad_queue->qp_info;
  2092. struct ib_mad_send_wr_private *mad_send_wr;
  2093. int ret;
  2094. /*
  2095. * Send errors will transition the QP to SQE - move
  2096. * QP to RTS and repost flushed work requests
  2097. */
  2098. mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private,
  2099. mad_list);
  2100. if (wc->status == IB_WC_WR_FLUSH_ERR) {
  2101. if (mad_send_wr->retry) {
  2102. /* Repost send */
  2103. mad_send_wr->retry = 0;
  2104. trace_ib_mad_error_handler(mad_send_wr, qp_info);
  2105. ret = ib_post_send(qp_info->qp, &mad_send_wr->send_wr.wr,
  2106. NULL);
  2107. if (!ret)
  2108. return false;
  2109. }
  2110. } else {
  2111. struct ib_qp_attr *attr;
  2112. /* Transition QP to RTS and fail offending send */
  2113. attr = kmalloc(sizeof *attr, GFP_KERNEL);
  2114. if (attr) {
  2115. attr->qp_state = IB_QPS_RTS;
  2116. attr->cur_qp_state = IB_QPS_SQE;
  2117. ret = ib_modify_qp(qp_info->qp, attr,
  2118. IB_QP_STATE | IB_QP_CUR_STATE);
  2119. kfree(attr);
  2120. if (ret)
  2121. dev_err(&port_priv->device->dev,
  2122. "%s - ib_modify_qp to RTS: %d\n",
  2123. __func__, ret);
  2124. else
  2125. mark_sends_for_retry(qp_info);
  2126. }
  2127. }
  2128. return true;
  2129. }
  2130. static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv)
  2131. {
  2132. unsigned long flags;
  2133. struct ib_mad_send_wr_private *mad_send_wr, *temp_mad_send_wr;
  2134. struct ib_mad_send_wc mad_send_wc;
  2135. struct list_head cancel_list;
  2136. INIT_LIST_HEAD(&cancel_list);
  2137. spin_lock_irqsave(&mad_agent_priv->lock, flags);
  2138. list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
  2139. &mad_agent_priv->send_list, agent_list) {
  2140. if (mad_send_wr->status == IB_WC_SUCCESS) {
  2141. mad_send_wr->status = IB_WC_WR_FLUSH_ERR;
  2142. mad_send_wr->refcount -= (mad_send_wr->timeout > 0);
  2143. }
  2144. }
  2145. /* Empty wait list to prevent receives from finding a request */
  2146. list_splice_init(&mad_agent_priv->wait_list, &cancel_list);
  2147. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  2148. /* Report all cancelled requests */
  2149. mad_send_wc.status = IB_WC_WR_FLUSH_ERR;
  2150. mad_send_wc.vendor_err = 0;
  2151. list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
  2152. &cancel_list, agent_list) {
  2153. mad_send_wc.send_buf = &mad_send_wr->send_buf;
  2154. list_del(&mad_send_wr->agent_list);
  2155. mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
  2156. &mad_send_wc);
  2157. deref_mad_agent(mad_agent_priv);
  2158. }
  2159. }
  2160. static struct ib_mad_send_wr_private*
  2161. find_send_wr(struct ib_mad_agent_private *mad_agent_priv,
  2162. struct ib_mad_send_buf *send_buf)
  2163. {
  2164. struct ib_mad_send_wr_private *mad_send_wr;
  2165. list_for_each_entry(mad_send_wr, &mad_agent_priv->wait_list,
  2166. agent_list) {
  2167. if (&mad_send_wr->send_buf == send_buf)
  2168. return mad_send_wr;
  2169. }
  2170. list_for_each_entry(mad_send_wr, &mad_agent_priv->send_list,
  2171. agent_list) {
  2172. if (is_rmpp_data_mad(mad_agent_priv,
  2173. mad_send_wr->send_buf.mad) &&
  2174. &mad_send_wr->send_buf == send_buf)
  2175. return mad_send_wr;
  2176. }
  2177. return NULL;
  2178. }
  2179. int ib_modify_mad(struct ib_mad_agent *mad_agent,
  2180. struct ib_mad_send_buf *send_buf, u32 timeout_ms)
  2181. {
  2182. struct ib_mad_agent_private *mad_agent_priv;
  2183. struct ib_mad_send_wr_private *mad_send_wr;
  2184. unsigned long flags;
  2185. int active;
  2186. mad_agent_priv = container_of(mad_agent, struct ib_mad_agent_private,
  2187. agent);
  2188. spin_lock_irqsave(&mad_agent_priv->lock, flags);
  2189. mad_send_wr = find_send_wr(mad_agent_priv, send_buf);
  2190. if (!mad_send_wr || mad_send_wr->status != IB_WC_SUCCESS) {
  2191. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  2192. return -EINVAL;
  2193. }
  2194. active = (!mad_send_wr->timeout || mad_send_wr->refcount > 1);
  2195. if (!timeout_ms) {
  2196. mad_send_wr->status = IB_WC_WR_FLUSH_ERR;
  2197. mad_send_wr->refcount -= (mad_send_wr->timeout > 0);
  2198. }
  2199. mad_send_wr->send_buf.timeout_ms = timeout_ms;
  2200. if (active)
  2201. mad_send_wr->timeout = msecs_to_jiffies(timeout_ms);
  2202. else
  2203. ib_reset_mad_timeout(mad_send_wr, timeout_ms);
  2204. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  2205. return 0;
  2206. }
  2207. EXPORT_SYMBOL(ib_modify_mad);
  2208. void ib_cancel_mad(struct ib_mad_agent *mad_agent,
  2209. struct ib_mad_send_buf *send_buf)
  2210. {
  2211. ib_modify_mad(mad_agent, send_buf, 0);
  2212. }
  2213. EXPORT_SYMBOL(ib_cancel_mad);
  2214. static void local_completions(struct work_struct *work)
  2215. {
  2216. struct ib_mad_agent_private *mad_agent_priv;
  2217. struct ib_mad_local_private *local;
  2218. struct ib_mad_agent_private *recv_mad_agent;
  2219. unsigned long flags;
  2220. int free_mad;
  2221. struct ib_wc wc;
  2222. struct ib_mad_send_wc mad_send_wc;
  2223. bool opa;
  2224. mad_agent_priv =
  2225. container_of(work, struct ib_mad_agent_private, local_work);
  2226. opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
  2227. mad_agent_priv->qp_info->port_priv->port_num);
  2228. spin_lock_irqsave(&mad_agent_priv->lock, flags);
  2229. while (!list_empty(&mad_agent_priv->local_list)) {
  2230. local = list_entry(mad_agent_priv->local_list.next,
  2231. struct ib_mad_local_private,
  2232. completion_list);
  2233. list_del(&local->completion_list);
  2234. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  2235. free_mad = 0;
  2236. if (local->mad_priv) {
  2237. u8 base_version;
  2238. recv_mad_agent = local->recv_mad_agent;
  2239. if (!recv_mad_agent) {
  2240. dev_err(&mad_agent_priv->agent.device->dev,
  2241. "No receive MAD agent for local completion\n");
  2242. free_mad = 1;
  2243. goto local_send_completion;
  2244. }
  2245. /*
  2246. * Defined behavior is to complete response
  2247. * before request
  2248. */
  2249. build_smp_wc(recv_mad_agent->agent.qp,
  2250. local->mad_send_wr->send_wr.wr.wr_cqe,
  2251. be16_to_cpu(IB_LID_PERMISSIVE),
  2252. local->mad_send_wr->send_wr.pkey_index,
  2253. recv_mad_agent->agent.port_num, &wc);
  2254. local->mad_priv->header.recv_wc.wc = &wc;
  2255. base_version = ((struct ib_mad_hdr *)(local->mad_priv->mad))->base_version;
  2256. if (opa && base_version == OPA_MGMT_BASE_VERSION) {
  2257. local->mad_priv->header.recv_wc.mad_len = local->return_wc_byte_len;
  2258. local->mad_priv->header.recv_wc.mad_seg_size = sizeof(struct opa_mad);
  2259. } else {
  2260. local->mad_priv->header.recv_wc.mad_len = sizeof(struct ib_mad);
  2261. local->mad_priv->header.recv_wc.mad_seg_size = sizeof(struct ib_mad);
  2262. }
  2263. INIT_LIST_HEAD(&local->mad_priv->header.recv_wc.rmpp_list);
  2264. list_add(&local->mad_priv->header.recv_wc.recv_buf.list,
  2265. &local->mad_priv->header.recv_wc.rmpp_list);
  2266. local->mad_priv->header.recv_wc.recv_buf.grh = NULL;
  2267. local->mad_priv->header.recv_wc.recv_buf.mad =
  2268. (struct ib_mad *)local->mad_priv->mad;
  2269. recv_mad_agent->agent.recv_handler(
  2270. &recv_mad_agent->agent,
  2271. &local->mad_send_wr->send_buf,
  2272. &local->mad_priv->header.recv_wc);
  2273. spin_lock_irqsave(&recv_mad_agent->lock, flags);
  2274. deref_mad_agent(recv_mad_agent);
  2275. spin_unlock_irqrestore(&recv_mad_agent->lock, flags);
  2276. }
  2277. local_send_completion:
  2278. /* Complete send */
  2279. mad_send_wc.status = IB_WC_SUCCESS;
  2280. mad_send_wc.vendor_err = 0;
  2281. mad_send_wc.send_buf = &local->mad_send_wr->send_buf;
  2282. mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
  2283. &mad_send_wc);
  2284. spin_lock_irqsave(&mad_agent_priv->lock, flags);
  2285. deref_mad_agent(mad_agent_priv);
  2286. if (free_mad)
  2287. kfree(local->mad_priv);
  2288. kfree(local);
  2289. }
  2290. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  2291. }
  2292. static int retry_send(struct ib_mad_send_wr_private *mad_send_wr)
  2293. {
  2294. int ret;
  2295. if (!mad_send_wr->retries_left)
  2296. return -ETIMEDOUT;
  2297. mad_send_wr->retries_left--;
  2298. mad_send_wr->send_buf.retries++;
  2299. mad_send_wr->timeout = msecs_to_jiffies(mad_send_wr->send_buf.timeout_ms);
  2300. if (ib_mad_kernel_rmpp_agent(&mad_send_wr->mad_agent_priv->agent)) {
  2301. ret = ib_retry_rmpp(mad_send_wr);
  2302. switch (ret) {
  2303. case IB_RMPP_RESULT_UNHANDLED:
  2304. ret = ib_send_mad(mad_send_wr);
  2305. break;
  2306. case IB_RMPP_RESULT_CONSUMED:
  2307. ret = 0;
  2308. break;
  2309. default:
  2310. ret = -ECOMM;
  2311. break;
  2312. }
  2313. } else
  2314. ret = ib_send_mad(mad_send_wr);
  2315. if (!ret) {
  2316. mad_send_wr->refcount++;
  2317. list_add_tail(&mad_send_wr->agent_list,
  2318. &mad_send_wr->mad_agent_priv->send_list);
  2319. }
  2320. return ret;
  2321. }
  2322. static void timeout_sends(struct work_struct *work)
  2323. {
  2324. struct ib_mad_agent_private *mad_agent_priv;
  2325. struct ib_mad_send_wr_private *mad_send_wr;
  2326. struct ib_mad_send_wc mad_send_wc;
  2327. unsigned long flags, delay;
  2328. mad_agent_priv = container_of(work, struct ib_mad_agent_private,
  2329. timed_work.work);
  2330. mad_send_wc.vendor_err = 0;
  2331. spin_lock_irqsave(&mad_agent_priv->lock, flags);
  2332. while (!list_empty(&mad_agent_priv->wait_list)) {
  2333. mad_send_wr = list_entry(mad_agent_priv->wait_list.next,
  2334. struct ib_mad_send_wr_private,
  2335. agent_list);
  2336. if (time_after(mad_send_wr->timeout, jiffies)) {
  2337. delay = mad_send_wr->timeout - jiffies;
  2338. if ((long)delay <= 0)
  2339. delay = 1;
  2340. queue_delayed_work(mad_agent_priv->qp_info->
  2341. port_priv->wq,
  2342. &mad_agent_priv->timed_work, delay);
  2343. break;
  2344. }
  2345. list_del(&mad_send_wr->agent_list);
  2346. if (mad_send_wr->status == IB_WC_SUCCESS &&
  2347. !retry_send(mad_send_wr))
  2348. continue;
  2349. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  2350. if (mad_send_wr->status == IB_WC_SUCCESS)
  2351. mad_send_wc.status = IB_WC_RESP_TIMEOUT_ERR;
  2352. else
  2353. mad_send_wc.status = mad_send_wr->status;
  2354. mad_send_wc.send_buf = &mad_send_wr->send_buf;
  2355. mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
  2356. &mad_send_wc);
  2357. deref_mad_agent(mad_agent_priv);
  2358. spin_lock_irqsave(&mad_agent_priv->lock, flags);
  2359. }
  2360. spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
  2361. }
  2362. /*
  2363. * Allocate receive MADs and post receive WRs for them
  2364. */
  2365. static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,
  2366. struct ib_mad_private *mad)
  2367. {
  2368. unsigned long flags;
  2369. int post, ret;
  2370. struct ib_mad_private *mad_priv;
  2371. struct ib_sge sg_list;
  2372. struct ib_recv_wr recv_wr;
  2373. struct ib_mad_queue *recv_queue = &qp_info->recv_queue;
  2374. /* Initialize common scatter list fields */
  2375. sg_list.lkey = qp_info->port_priv->pd->local_dma_lkey;
  2376. /* Initialize common receive WR fields */
  2377. recv_wr.next = NULL;
  2378. recv_wr.sg_list = &sg_list;
  2379. recv_wr.num_sge = 1;
  2380. do {
  2381. /* Allocate and map receive buffer */
  2382. if (mad) {
  2383. mad_priv = mad;
  2384. mad = NULL;
  2385. } else {
  2386. mad_priv = alloc_mad_private(port_mad_size(qp_info->port_priv),
  2387. GFP_ATOMIC);
  2388. if (!mad_priv) {
  2389. ret = -ENOMEM;
  2390. break;
  2391. }
  2392. }
  2393. sg_list.length = mad_priv_dma_size(mad_priv);
  2394. sg_list.addr = ib_dma_map_single(qp_info->port_priv->device,
  2395. &mad_priv->grh,
  2396. mad_priv_dma_size(mad_priv),
  2397. DMA_FROM_DEVICE);
  2398. if (unlikely(ib_dma_mapping_error(qp_info->port_priv->device,
  2399. sg_list.addr))) {
  2400. kfree(mad_priv);
  2401. ret = -ENOMEM;
  2402. break;
  2403. }
  2404. mad_priv->header.mapping = sg_list.addr;
  2405. mad_priv->header.mad_list.mad_queue = recv_queue;
  2406. mad_priv->header.mad_list.cqe.done = ib_mad_recv_done;
  2407. recv_wr.wr_cqe = &mad_priv->header.mad_list.cqe;
  2408. /* Post receive WR */
  2409. spin_lock_irqsave(&recv_queue->lock, flags);
  2410. post = (++recv_queue->count < recv_queue->max_active);
  2411. list_add_tail(&mad_priv->header.mad_list.list, &recv_queue->list);
  2412. spin_unlock_irqrestore(&recv_queue->lock, flags);
  2413. ret = ib_post_recv(qp_info->qp, &recv_wr, NULL);
  2414. if (ret) {
  2415. spin_lock_irqsave(&recv_queue->lock, flags);
  2416. list_del(&mad_priv->header.mad_list.list);
  2417. recv_queue->count--;
  2418. spin_unlock_irqrestore(&recv_queue->lock, flags);
  2419. ib_dma_unmap_single(qp_info->port_priv->device,
  2420. mad_priv->header.mapping,
  2421. mad_priv_dma_size(mad_priv),
  2422. DMA_FROM_DEVICE);
  2423. kfree(mad_priv);
  2424. dev_err(&qp_info->port_priv->device->dev,
  2425. "ib_post_recv failed: %d\n", ret);
  2426. break;
  2427. }
  2428. } while (post);
  2429. return ret;
  2430. }
  2431. /*
  2432. * Return all the posted receive MADs
  2433. */
  2434. static void cleanup_recv_queue(struct ib_mad_qp_info *qp_info)
  2435. {
  2436. struct ib_mad_private_header *mad_priv_hdr;
  2437. struct ib_mad_private *recv;
  2438. struct ib_mad_list_head *mad_list;
  2439. if (!qp_info->qp)
  2440. return;
  2441. while (!list_empty(&qp_info->recv_queue.list)) {
  2442. mad_list = list_entry(qp_info->recv_queue.list.next,
  2443. struct ib_mad_list_head, list);
  2444. mad_priv_hdr = container_of(mad_list,
  2445. struct ib_mad_private_header,
  2446. mad_list);
  2447. recv = container_of(mad_priv_hdr, struct ib_mad_private,
  2448. header);
  2449. /* Remove from posted receive MAD list */
  2450. list_del(&mad_list->list);
  2451. ib_dma_unmap_single(qp_info->port_priv->device,
  2452. recv->header.mapping,
  2453. mad_priv_dma_size(recv),
  2454. DMA_FROM_DEVICE);
  2455. kfree(recv);
  2456. }
  2457. qp_info->recv_queue.count = 0;
  2458. }
  2459. /*
  2460. * Start the port
  2461. */
  2462. static int ib_mad_port_start(struct ib_mad_port_private *port_priv)
  2463. {
  2464. int ret, i;
  2465. struct ib_qp_attr *attr;
  2466. struct ib_qp *qp;
  2467. u16 pkey_index;
  2468. attr = kmalloc(sizeof *attr, GFP_KERNEL);
  2469. if (!attr)
  2470. return -ENOMEM;
  2471. ret = ib_find_pkey(port_priv->device, port_priv->port_num,
  2472. IB_DEFAULT_PKEY_FULL, &pkey_index);
  2473. if (ret)
  2474. pkey_index = 0;
  2475. for (i = 0; i < IB_MAD_QPS_CORE; i++) {
  2476. qp = port_priv->qp_info[i].qp;
  2477. if (!qp)
  2478. continue;
  2479. /*
  2480. * PKey index for QP1 is irrelevant but
  2481. * one is needed for the Reset to Init transition
  2482. */
  2483. attr->qp_state = IB_QPS_INIT;
  2484. attr->pkey_index = pkey_index;
  2485. attr->qkey = (qp->qp_num == 0) ? 0 : IB_QP1_QKEY;
  2486. ret = ib_modify_qp(qp, attr, IB_QP_STATE |
  2487. IB_QP_PKEY_INDEX | IB_QP_QKEY);
  2488. if (ret) {
  2489. dev_err(&port_priv->device->dev,
  2490. "Couldn't change QP%d state to INIT: %d\n",
  2491. i, ret);
  2492. goto out;
  2493. }
  2494. attr->qp_state = IB_QPS_RTR;
  2495. ret = ib_modify_qp(qp, attr, IB_QP_STATE);
  2496. if (ret) {
  2497. dev_err(&port_priv->device->dev,
  2498. "Couldn't change QP%d state to RTR: %d\n",
  2499. i, ret);
  2500. goto out;
  2501. }
  2502. attr->qp_state = IB_QPS_RTS;
  2503. attr->sq_psn = IB_MAD_SEND_Q_PSN;
  2504. ret = ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_SQ_PSN);
  2505. if (ret) {
  2506. dev_err(&port_priv->device->dev,
  2507. "Couldn't change QP%d state to RTS: %d\n",
  2508. i, ret);
  2509. goto out;
  2510. }
  2511. }
  2512. ret = ib_req_notify_cq(port_priv->cq, IB_CQ_NEXT_COMP);
  2513. if (ret) {
  2514. dev_err(&port_priv->device->dev,
  2515. "Failed to request completion notification: %d\n",
  2516. ret);
  2517. goto out;
  2518. }
  2519. for (i = 0; i < IB_MAD_QPS_CORE; i++) {
  2520. if (!port_priv->qp_info[i].qp)
  2521. continue;
  2522. ret = ib_mad_post_receive_mads(&port_priv->qp_info[i], NULL);
  2523. if (ret) {
  2524. dev_err(&port_priv->device->dev,
  2525. "Couldn't post receive WRs\n");
  2526. goto out;
  2527. }
  2528. }
  2529. out:
  2530. kfree(attr);
  2531. return ret;
  2532. }
  2533. static void qp_event_handler(struct ib_event *event, void *qp_context)
  2534. {
  2535. struct ib_mad_qp_info *qp_info = qp_context;
  2536. /* It's worse than that! He's dead, Jim! */
  2537. dev_err(&qp_info->port_priv->device->dev,
  2538. "Fatal error (%d) on MAD QP (%d)\n",
  2539. event->event, qp_info->qp->qp_num);
  2540. }
  2541. static void init_mad_queue(struct ib_mad_qp_info *qp_info,
  2542. struct ib_mad_queue *mad_queue)
  2543. {
  2544. mad_queue->qp_info = qp_info;
  2545. mad_queue->count = 0;
  2546. spin_lock_init(&mad_queue->lock);
  2547. INIT_LIST_HEAD(&mad_queue->list);
  2548. }
  2549. static void init_mad_qp(struct ib_mad_port_private *port_priv,
  2550. struct ib_mad_qp_info *qp_info)
  2551. {
  2552. qp_info->port_priv = port_priv;
  2553. init_mad_queue(qp_info, &qp_info->send_queue);
  2554. init_mad_queue(qp_info, &qp_info->recv_queue);
  2555. INIT_LIST_HEAD(&qp_info->overflow_list);
  2556. }
  2557. static int create_mad_qp(struct ib_mad_qp_info *qp_info,
  2558. enum ib_qp_type qp_type)
  2559. {
  2560. struct ib_qp_init_attr qp_init_attr;
  2561. int ret;
  2562. memset(&qp_init_attr, 0, sizeof qp_init_attr);
  2563. qp_init_attr.send_cq = qp_info->port_priv->cq;
  2564. qp_init_attr.recv_cq = qp_info->port_priv->cq;
  2565. qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
  2566. qp_init_attr.cap.max_send_wr = mad_sendq_size;
  2567. qp_init_attr.cap.max_recv_wr = mad_recvq_size;
  2568. qp_init_attr.cap.max_send_sge = IB_MAD_SEND_REQ_MAX_SG;
  2569. qp_init_attr.cap.max_recv_sge = IB_MAD_RECV_REQ_MAX_SG;
  2570. qp_init_attr.qp_type = qp_type;
  2571. qp_init_attr.port_num = qp_info->port_priv->port_num;
  2572. qp_init_attr.qp_context = qp_info;
  2573. qp_init_attr.event_handler = qp_event_handler;
  2574. qp_info->qp = ib_create_qp(qp_info->port_priv->pd, &qp_init_attr);
  2575. if (IS_ERR(qp_info->qp)) {
  2576. dev_err(&qp_info->port_priv->device->dev,
  2577. "Couldn't create ib_mad QP%d\n",
  2578. get_spl_qp_index(qp_type));
  2579. ret = PTR_ERR(qp_info->qp);
  2580. goto error;
  2581. }
  2582. /* Use minimum queue sizes unless the CQ is resized */
  2583. qp_info->send_queue.max_active = mad_sendq_size;
  2584. qp_info->recv_queue.max_active = mad_recvq_size;
  2585. return 0;
  2586. error:
  2587. return ret;
  2588. }
  2589. static void destroy_mad_qp(struct ib_mad_qp_info *qp_info)
  2590. {
  2591. if (!qp_info->qp)
  2592. return;
  2593. ib_destroy_qp(qp_info->qp);
  2594. }
  2595. /*
  2596. * Open the port
  2597. * Create the QP, PD, MR, and CQ if needed
  2598. */
  2599. static int ib_mad_port_open(struct ib_device *device,
  2600. int port_num)
  2601. {
  2602. int ret, cq_size;
  2603. struct ib_mad_port_private *port_priv;
  2604. unsigned long flags;
  2605. char name[sizeof "ib_mad123"];
  2606. int has_smi;
  2607. if (WARN_ON(rdma_max_mad_size(device, port_num) < IB_MGMT_MAD_SIZE))
  2608. return -EFAULT;
  2609. if (WARN_ON(rdma_cap_opa_mad(device, port_num) &&
  2610. rdma_max_mad_size(device, port_num) < OPA_MGMT_MAD_SIZE))
  2611. return -EFAULT;
  2612. /* Create new device info */
  2613. port_priv = kzalloc(sizeof *port_priv, GFP_KERNEL);
  2614. if (!port_priv)
  2615. return -ENOMEM;
  2616. port_priv->device = device;
  2617. port_priv->port_num = port_num;
  2618. spin_lock_init(&port_priv->reg_lock);
  2619. init_mad_qp(port_priv, &port_priv->qp_info[0]);
  2620. init_mad_qp(port_priv, &port_priv->qp_info[1]);
  2621. cq_size = mad_sendq_size + mad_recvq_size;
  2622. has_smi = rdma_cap_ib_smi(device, port_num);
  2623. if (has_smi)
  2624. cq_size *= 2;
  2625. port_priv->pd = ib_alloc_pd(device, 0);
  2626. if (IS_ERR(port_priv->pd)) {
  2627. dev_err(&device->dev, "Couldn't create ib_mad PD\n");
  2628. ret = PTR_ERR(port_priv->pd);
  2629. goto error3;
  2630. }
  2631. port_priv->cq = ib_alloc_cq(port_priv->device, port_priv, cq_size, 0,
  2632. IB_POLL_UNBOUND_WORKQUEUE);
  2633. if (IS_ERR(port_priv->cq)) {
  2634. dev_err(&device->dev, "Couldn't create ib_mad CQ\n");
  2635. ret = PTR_ERR(port_priv->cq);
  2636. goto error4;
  2637. }
  2638. if (has_smi) {
  2639. ret = create_mad_qp(&port_priv->qp_info[0], IB_QPT_SMI);
  2640. if (ret)
  2641. goto error6;
  2642. }
  2643. ret = create_mad_qp(&port_priv->qp_info[1], IB_QPT_GSI);
  2644. if (ret)
  2645. goto error7;
  2646. snprintf(name, sizeof name, "ib_mad%d", port_num);
  2647. port_priv->wq = alloc_ordered_workqueue(name, WQ_MEM_RECLAIM);
  2648. if (!port_priv->wq) {
  2649. ret = -ENOMEM;
  2650. goto error8;
  2651. }
  2652. spin_lock_irqsave(&ib_mad_port_list_lock, flags);
  2653. list_add_tail(&port_priv->port_list, &ib_mad_port_list);
  2654. spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
  2655. ret = ib_mad_port_start(port_priv);
  2656. if (ret) {
  2657. dev_err(&device->dev, "Couldn't start port\n");
  2658. goto error9;
  2659. }
  2660. return 0;
  2661. error9:
  2662. spin_lock_irqsave(&ib_mad_port_list_lock, flags);
  2663. list_del_init(&port_priv->port_list);
  2664. spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
  2665. destroy_workqueue(port_priv->wq);
  2666. error8:
  2667. destroy_mad_qp(&port_priv->qp_info[1]);
  2668. error7:
  2669. destroy_mad_qp(&port_priv->qp_info[0]);
  2670. error6:
  2671. ib_free_cq(port_priv->cq);
  2672. cleanup_recv_queue(&port_priv->qp_info[1]);
  2673. cleanup_recv_queue(&port_priv->qp_info[0]);
  2674. error4:
  2675. ib_dealloc_pd(port_priv->pd);
  2676. error3:
  2677. kfree(port_priv);
  2678. return ret;
  2679. }
  2680. /*
  2681. * Close the port
  2682. * If there are no classes using the port, free the port
  2683. * resources (CQ, MR, PD, QP) and remove the port's info structure
  2684. */
  2685. static int ib_mad_port_close(struct ib_device *device, int port_num)
  2686. {
  2687. struct ib_mad_port_private *port_priv;
  2688. unsigned long flags;
  2689. spin_lock_irqsave(&ib_mad_port_list_lock, flags);
  2690. port_priv = __ib_get_mad_port(device, port_num);
  2691. if (port_priv == NULL) {
  2692. spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
  2693. dev_err(&device->dev, "Port %d not found\n", port_num);
  2694. return -ENODEV;
  2695. }
  2696. list_del_init(&port_priv->port_list);
  2697. spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
  2698. destroy_workqueue(port_priv->wq);
  2699. destroy_mad_qp(&port_priv->qp_info[1]);
  2700. destroy_mad_qp(&port_priv->qp_info[0]);
  2701. ib_free_cq(port_priv->cq);
  2702. ib_dealloc_pd(port_priv->pd);
  2703. cleanup_recv_queue(&port_priv->qp_info[1]);
  2704. cleanup_recv_queue(&port_priv->qp_info[0]);
  2705. /* XXX: Handle deallocation of MAD registration tables */
  2706. kfree(port_priv);
  2707. return 0;
  2708. }
  2709. static int ib_mad_init_device(struct ib_device *device)
  2710. {
  2711. int start, i;
  2712. unsigned int count = 0;
  2713. int ret;
  2714. start = rdma_start_port(device);
  2715. for (i = start; i <= rdma_end_port(device); i++) {
  2716. if (!rdma_cap_ib_mad(device, i))
  2717. continue;
  2718. ret = ib_mad_port_open(device, i);
  2719. if (ret) {
  2720. dev_err(&device->dev, "Couldn't open port %d\n", i);
  2721. goto error;
  2722. }
  2723. ret = ib_agent_port_open(device, i);
  2724. if (ret) {
  2725. dev_err(&device->dev,
  2726. "Couldn't open port %d for agents\n", i);
  2727. goto error_agent;
  2728. }
  2729. count++;
  2730. }
  2731. if (!count)
  2732. return -EOPNOTSUPP;
  2733. return 0;
  2734. error_agent:
  2735. if (ib_mad_port_close(device, i))
  2736. dev_err(&device->dev, "Couldn't close port %d\n", i);
  2737. error:
  2738. while (--i >= start) {
  2739. if (!rdma_cap_ib_mad(device, i))
  2740. continue;
  2741. if (ib_agent_port_close(device, i))
  2742. dev_err(&device->dev,
  2743. "Couldn't close port %d for agents\n", i);
  2744. if (ib_mad_port_close(device, i))
  2745. dev_err(&device->dev, "Couldn't close port %d\n", i);
  2746. }
  2747. return ret;
  2748. }
  2749. static void ib_mad_remove_device(struct ib_device *device, void *client_data)
  2750. {
  2751. unsigned int i;
  2752. rdma_for_each_port (device, i) {
  2753. if (!rdma_cap_ib_mad(device, i))
  2754. continue;
  2755. if (ib_agent_port_close(device, i))
  2756. dev_err(&device->dev,
  2757. "Couldn't close port %d for agents\n", i);
  2758. if (ib_mad_port_close(device, i))
  2759. dev_err(&device->dev, "Couldn't close port %d\n", i);
  2760. }
  2761. }
  2762. static struct ib_client mad_client = {
  2763. .name = "mad",
  2764. .add = ib_mad_init_device,
  2765. .remove = ib_mad_remove_device
  2766. };
  2767. int ib_mad_init(void)
  2768. {
  2769. mad_recvq_size = min(mad_recvq_size, IB_MAD_QP_MAX_SIZE);
  2770. mad_recvq_size = max(mad_recvq_size, IB_MAD_QP_MIN_SIZE);
  2771. mad_sendq_size = min(mad_sendq_size, IB_MAD_QP_MAX_SIZE);
  2772. mad_sendq_size = max(mad_sendq_size, IB_MAD_QP_MIN_SIZE);
  2773. INIT_LIST_HEAD(&ib_mad_port_list);
  2774. if (ib_register_client(&mad_client)) {
  2775. pr_err("Couldn't register ib_mad client\n");
  2776. return -EINVAL;
  2777. }
  2778. return 0;
  2779. }
  2780. void ib_mad_cleanup(void)
  2781. {
  2782. ib_unregister_client(&mad_client);
  2783. }