ib_verbs.h 138 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716
  1. /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
  2. /*
  3. * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
  4. * Copyright (c) 2004 Infinicon Corporation. All rights reserved.
  5. * Copyright (c) 2004 Intel Corporation. All rights reserved.
  6. * Copyright (c) 2004 Topspin Corporation. All rights reserved.
  7. * Copyright (c) 2004 Voltaire Corporation. All rights reserved.
  8. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
  9. * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
  10. */
  11. #ifndef IB_VERBS_H
  12. #define IB_VERBS_H
  13. #include <linux/types.h>
  14. #include <linux/device.h>
  15. #include <linux/dma-mapping.h>
  16. #include <linux/kref.h>
  17. #include <linux/list.h>
  18. #include <linux/rwsem.h>
  19. #include <linux/workqueue.h>
  20. #include <linux/irq_poll.h>
  21. #include <uapi/linux/if_ether.h>
  22. #include <net/ipv6.h>
  23. #include <net/ip.h>
  24. #include <linux/string.h>
  25. #include <linux/slab.h>
  26. #include <linux/netdevice.h>
  27. #include <linux/refcount.h>
  28. #include <linux/if_link.h>
  29. #include <linux/atomic.h>
  30. #include <linux/mmu_notifier.h>
  31. #include <linux/uaccess.h>
  32. #include <linux/cgroup_rdma.h>
  33. #include <linux/irqflags.h>
  34. #include <linux/preempt.h>
  35. #include <linux/dim.h>
  36. #include <uapi/rdma/ib_user_verbs.h>
  37. #include <rdma/rdma_counter.h>
  38. #include <rdma/restrack.h>
  39. #include <rdma/signature.h>
  40. #include <uapi/rdma/rdma_user_ioctl.h>
  41. #include <uapi/rdma/ib_user_ioctl_verbs.h>
  42. #define IB_FW_VERSION_NAME_MAX ETHTOOL_FWVERS_LEN
  43. struct ib_umem_odp;
  44. struct ib_uqp_object;
  45. struct ib_usrq_object;
  46. struct ib_uwq_object;
  47. struct rdma_cm_id;
  48. extern struct workqueue_struct *ib_wq;
  49. extern struct workqueue_struct *ib_comp_wq;
  50. extern struct workqueue_struct *ib_comp_unbound_wq;
  51. struct ib_ucq_object;
  52. __printf(3, 4) __cold
  53. void ibdev_printk(const char *level, const struct ib_device *ibdev,
  54. const char *format, ...);
  55. __printf(2, 3) __cold
  56. void ibdev_emerg(const struct ib_device *ibdev, const char *format, ...);
  57. __printf(2, 3) __cold
  58. void ibdev_alert(const struct ib_device *ibdev, const char *format, ...);
  59. __printf(2, 3) __cold
  60. void ibdev_crit(const struct ib_device *ibdev, const char *format, ...);
  61. __printf(2, 3) __cold
  62. void ibdev_err(const struct ib_device *ibdev, const char *format, ...);
  63. __printf(2, 3) __cold
  64. void ibdev_warn(const struct ib_device *ibdev, const char *format, ...);
  65. __printf(2, 3) __cold
  66. void ibdev_notice(const struct ib_device *ibdev, const char *format, ...);
  67. __printf(2, 3) __cold
  68. void ibdev_info(const struct ib_device *ibdev, const char *format, ...);
  69. #if defined(CONFIG_DYNAMIC_DEBUG) || \
  70. (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
  71. #define ibdev_dbg(__dev, format, args...) \
  72. dynamic_ibdev_dbg(__dev, format, ##args)
  73. #else
  74. __printf(2, 3) __cold
  75. static inline
  76. void ibdev_dbg(const struct ib_device *ibdev, const char *format, ...) {}
  77. #endif
  78. #define ibdev_level_ratelimited(ibdev_level, ibdev, fmt, ...) \
  79. do { \
  80. static DEFINE_RATELIMIT_STATE(_rs, \
  81. DEFAULT_RATELIMIT_INTERVAL, \
  82. DEFAULT_RATELIMIT_BURST); \
  83. if (__ratelimit(&_rs)) \
  84. ibdev_level(ibdev, fmt, ##__VA_ARGS__); \
  85. } while (0)
  86. #define ibdev_emerg_ratelimited(ibdev, fmt, ...) \
  87. ibdev_level_ratelimited(ibdev_emerg, ibdev, fmt, ##__VA_ARGS__)
  88. #define ibdev_alert_ratelimited(ibdev, fmt, ...) \
  89. ibdev_level_ratelimited(ibdev_alert, ibdev, fmt, ##__VA_ARGS__)
  90. #define ibdev_crit_ratelimited(ibdev, fmt, ...) \
  91. ibdev_level_ratelimited(ibdev_crit, ibdev, fmt, ##__VA_ARGS__)
  92. #define ibdev_err_ratelimited(ibdev, fmt, ...) \
  93. ibdev_level_ratelimited(ibdev_err, ibdev, fmt, ##__VA_ARGS__)
  94. #define ibdev_warn_ratelimited(ibdev, fmt, ...) \
  95. ibdev_level_ratelimited(ibdev_warn, ibdev, fmt, ##__VA_ARGS__)
  96. #define ibdev_notice_ratelimited(ibdev, fmt, ...) \
  97. ibdev_level_ratelimited(ibdev_notice, ibdev, fmt, ##__VA_ARGS__)
  98. #define ibdev_info_ratelimited(ibdev, fmt, ...) \
  99. ibdev_level_ratelimited(ibdev_info, ibdev, fmt, ##__VA_ARGS__)
  100. #if defined(CONFIG_DYNAMIC_DEBUG) || \
  101. (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
  102. /* descriptor check is first to prevent flooding with "callbacks suppressed" */
  103. #define ibdev_dbg_ratelimited(ibdev, fmt, ...) \
  104. do { \
  105. static DEFINE_RATELIMIT_STATE(_rs, \
  106. DEFAULT_RATELIMIT_INTERVAL, \
  107. DEFAULT_RATELIMIT_BURST); \
  108. DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, fmt); \
  109. if (DYNAMIC_DEBUG_BRANCH(descriptor) && __ratelimit(&_rs)) \
  110. __dynamic_ibdev_dbg(&descriptor, ibdev, fmt, \
  111. ##__VA_ARGS__); \
  112. } while (0)
  113. #else
  114. __printf(2, 3) __cold
  115. static inline
  116. void ibdev_dbg_ratelimited(const struct ib_device *ibdev, const char *format, ...) {}
  117. #endif
  118. union ib_gid {
  119. u8 raw[16];
  120. struct {
  121. __be64 subnet_prefix;
  122. __be64 interface_id;
  123. } global;
  124. };
  125. extern union ib_gid zgid;
  126. enum ib_gid_type {
  127. IB_GID_TYPE_IB = IB_UVERBS_GID_TYPE_IB,
  128. IB_GID_TYPE_ROCE = IB_UVERBS_GID_TYPE_ROCE_V1,
  129. IB_GID_TYPE_ROCE_UDP_ENCAP = IB_UVERBS_GID_TYPE_ROCE_V2,
  130. IB_GID_TYPE_SIZE
  131. };
  132. #define ROCE_V2_UDP_DPORT 4791
  133. struct ib_gid_attr {
  134. struct net_device __rcu *ndev;
  135. struct ib_device *device;
  136. union ib_gid gid;
  137. enum ib_gid_type gid_type;
  138. u16 index;
  139. u8 port_num;
  140. };
  141. enum {
  142. /* set the local administered indication */
  143. IB_SA_WELL_KNOWN_GUID = BIT_ULL(57) | 2,
  144. };
  145. enum rdma_transport_type {
  146. RDMA_TRANSPORT_IB,
  147. RDMA_TRANSPORT_IWARP,
  148. RDMA_TRANSPORT_USNIC,
  149. RDMA_TRANSPORT_USNIC_UDP,
  150. RDMA_TRANSPORT_UNSPECIFIED,
  151. };
  152. enum rdma_protocol_type {
  153. RDMA_PROTOCOL_IB,
  154. RDMA_PROTOCOL_IBOE,
  155. RDMA_PROTOCOL_IWARP,
  156. RDMA_PROTOCOL_USNIC_UDP
  157. };
  158. __attribute_const__ enum rdma_transport_type
  159. rdma_node_get_transport(unsigned int node_type);
  160. enum rdma_network_type {
  161. RDMA_NETWORK_IB,
  162. RDMA_NETWORK_ROCE_V1,
  163. RDMA_NETWORK_IPV4,
  164. RDMA_NETWORK_IPV6
  165. };
  166. static inline enum ib_gid_type ib_network_to_gid_type(enum rdma_network_type network_type)
  167. {
  168. if (network_type == RDMA_NETWORK_IPV4 ||
  169. network_type == RDMA_NETWORK_IPV6)
  170. return IB_GID_TYPE_ROCE_UDP_ENCAP;
  171. else if (network_type == RDMA_NETWORK_ROCE_V1)
  172. return IB_GID_TYPE_ROCE;
  173. else
  174. return IB_GID_TYPE_IB;
  175. }
  176. static inline enum rdma_network_type
  177. rdma_gid_attr_network_type(const struct ib_gid_attr *attr)
  178. {
  179. if (attr->gid_type == IB_GID_TYPE_IB)
  180. return RDMA_NETWORK_IB;
  181. if (attr->gid_type == IB_GID_TYPE_ROCE)
  182. return RDMA_NETWORK_ROCE_V1;
  183. if (ipv6_addr_v4mapped((struct in6_addr *)&attr->gid))
  184. return RDMA_NETWORK_IPV4;
  185. else
  186. return RDMA_NETWORK_IPV6;
  187. }
  188. enum rdma_link_layer {
  189. IB_LINK_LAYER_UNSPECIFIED,
  190. IB_LINK_LAYER_INFINIBAND,
  191. IB_LINK_LAYER_ETHERNET,
  192. };
  193. enum ib_device_cap_flags {
  194. IB_DEVICE_RESIZE_MAX_WR = (1 << 0),
  195. IB_DEVICE_BAD_PKEY_CNTR = (1 << 1),
  196. IB_DEVICE_BAD_QKEY_CNTR = (1 << 2),
  197. IB_DEVICE_RAW_MULTI = (1 << 3),
  198. IB_DEVICE_AUTO_PATH_MIG = (1 << 4),
  199. IB_DEVICE_CHANGE_PHY_PORT = (1 << 5),
  200. IB_DEVICE_UD_AV_PORT_ENFORCE = (1 << 6),
  201. IB_DEVICE_CURR_QP_STATE_MOD = (1 << 7),
  202. IB_DEVICE_SHUTDOWN_PORT = (1 << 8),
  203. /* Not in use, former INIT_TYPE = (1 << 9),*/
  204. IB_DEVICE_PORT_ACTIVE_EVENT = (1 << 10),
  205. IB_DEVICE_SYS_IMAGE_GUID = (1 << 11),
  206. IB_DEVICE_RC_RNR_NAK_GEN = (1 << 12),
  207. IB_DEVICE_SRQ_RESIZE = (1 << 13),
  208. IB_DEVICE_N_NOTIFY_CQ = (1 << 14),
  209. /*
  210. * This device supports a per-device lkey or stag that can be
  211. * used without performing a memory registration for the local
  212. * memory. Note that ULPs should never check this flag, but
  213. * instead of use the local_dma_lkey flag in the ib_pd structure,
  214. * which will always contain a usable lkey.
  215. */
  216. IB_DEVICE_LOCAL_DMA_LKEY = (1 << 15),
  217. /* Reserved, old SEND_W_INV = (1 << 16),*/
  218. IB_DEVICE_MEM_WINDOW = (1 << 17),
  219. /*
  220. * Devices should set IB_DEVICE_UD_IP_SUM if they support
  221. * insertion of UDP and TCP checksum on outgoing UD IPoIB
  222. * messages and can verify the validity of checksum for
  223. * incoming messages. Setting this flag implies that the
  224. * IPoIB driver may set NETIF_F_IP_CSUM for datagram mode.
  225. */
  226. IB_DEVICE_UD_IP_CSUM = (1 << 18),
  227. IB_DEVICE_UD_TSO = (1 << 19),
  228. IB_DEVICE_XRC = (1 << 20),
  229. /*
  230. * This device supports the IB "base memory management extension",
  231. * which includes support for fast registrations (IB_WR_REG_MR,
  232. * IB_WR_LOCAL_INV and IB_WR_SEND_WITH_INV verbs). This flag should
  233. * also be set by any iWarp device which must support FRs to comply
  234. * to the iWarp verbs spec. iWarp devices also support the
  235. * IB_WR_RDMA_READ_WITH_INV verb for RDMA READs that invalidate the
  236. * stag.
  237. */
  238. IB_DEVICE_MEM_MGT_EXTENSIONS = (1 << 21),
  239. IB_DEVICE_BLOCK_MULTICAST_LOOPBACK = (1 << 22),
  240. IB_DEVICE_MEM_WINDOW_TYPE_2A = (1 << 23),
  241. IB_DEVICE_MEM_WINDOW_TYPE_2B = (1 << 24),
  242. IB_DEVICE_RC_IP_CSUM = (1 << 25),
  243. /* Deprecated. Please use IB_RAW_PACKET_CAP_IP_CSUM. */
  244. IB_DEVICE_RAW_IP_CSUM = (1 << 26),
  245. /*
  246. * Devices should set IB_DEVICE_CROSS_CHANNEL if they
  247. * support execution of WQEs that involve synchronization
  248. * of I/O operations with single completion queue managed
  249. * by hardware.
  250. */
  251. IB_DEVICE_CROSS_CHANNEL = (1 << 27),
  252. IB_DEVICE_MANAGED_FLOW_STEERING = (1 << 29),
  253. IB_DEVICE_INTEGRITY_HANDOVER = (1 << 30),
  254. IB_DEVICE_ON_DEMAND_PAGING = (1ULL << 31),
  255. IB_DEVICE_SG_GAPS_REG = (1ULL << 32),
  256. IB_DEVICE_VIRTUAL_FUNCTION = (1ULL << 33),
  257. /* Deprecated. Please use IB_RAW_PACKET_CAP_SCATTER_FCS. */
  258. IB_DEVICE_RAW_SCATTER_FCS = (1ULL << 34),
  259. IB_DEVICE_RDMA_NETDEV_OPA = (1ULL << 35),
  260. /* The device supports padding incoming writes to cacheline. */
  261. IB_DEVICE_PCI_WRITE_END_PADDING = (1ULL << 36),
  262. IB_DEVICE_ALLOW_USER_UNREG = (1ULL << 37),
  263. };
  264. enum ib_atomic_cap {
  265. IB_ATOMIC_NONE,
  266. IB_ATOMIC_HCA,
  267. IB_ATOMIC_GLOB
  268. };
  269. enum ib_odp_general_cap_bits {
  270. IB_ODP_SUPPORT = 1 << 0,
  271. IB_ODP_SUPPORT_IMPLICIT = 1 << 1,
  272. };
  273. enum ib_odp_transport_cap_bits {
  274. IB_ODP_SUPPORT_SEND = 1 << 0,
  275. IB_ODP_SUPPORT_RECV = 1 << 1,
  276. IB_ODP_SUPPORT_WRITE = 1 << 2,
  277. IB_ODP_SUPPORT_READ = 1 << 3,
  278. IB_ODP_SUPPORT_ATOMIC = 1 << 4,
  279. IB_ODP_SUPPORT_SRQ_RECV = 1 << 5,
  280. };
  281. struct ib_odp_caps {
  282. uint64_t general_caps;
  283. struct {
  284. uint32_t rc_odp_caps;
  285. uint32_t uc_odp_caps;
  286. uint32_t ud_odp_caps;
  287. uint32_t xrc_odp_caps;
  288. } per_transport_caps;
  289. };
  290. struct ib_rss_caps {
  291. /* Corresponding bit will be set if qp type from
  292. * 'enum ib_qp_type' is supported, e.g.
  293. * supported_qpts |= 1 << IB_QPT_UD
  294. */
  295. u32 supported_qpts;
  296. u32 max_rwq_indirection_tables;
  297. u32 max_rwq_indirection_table_size;
  298. };
  299. enum ib_tm_cap_flags {
  300. /* Support tag matching with rendezvous offload for RC transport */
  301. IB_TM_CAP_RNDV_RC = 1 << 0,
  302. };
  303. struct ib_tm_caps {
  304. /* Max size of RNDV header */
  305. u32 max_rndv_hdr_size;
  306. /* Max number of entries in tag matching list */
  307. u32 max_num_tags;
  308. /* From enum ib_tm_cap_flags */
  309. u32 flags;
  310. /* Max number of outstanding list operations */
  311. u32 max_ops;
  312. /* Max number of SGE in tag matching entry */
  313. u32 max_sge;
  314. };
  315. struct ib_cq_init_attr {
  316. unsigned int cqe;
  317. u32 comp_vector;
  318. u32 flags;
  319. };
  320. enum ib_cq_attr_mask {
  321. IB_CQ_MODERATE = 1 << 0,
  322. };
  323. struct ib_cq_caps {
  324. u16 max_cq_moderation_count;
  325. u16 max_cq_moderation_period;
  326. };
  327. struct ib_dm_mr_attr {
  328. u64 length;
  329. u64 offset;
  330. u32 access_flags;
  331. };
  332. struct ib_dm_alloc_attr {
  333. u64 length;
  334. u32 alignment;
  335. u32 flags;
  336. };
  337. struct ib_device_attr {
  338. u64 fw_ver;
  339. __be64 sys_image_guid;
  340. u64 max_mr_size;
  341. u64 page_size_cap;
  342. u32 vendor_id;
  343. u32 vendor_part_id;
  344. u32 hw_ver;
  345. int max_qp;
  346. int max_qp_wr;
  347. u64 device_cap_flags;
  348. int max_send_sge;
  349. int max_recv_sge;
  350. int max_sge_rd;
  351. int max_cq;
  352. int max_cqe;
  353. int max_mr;
  354. int max_pd;
  355. int max_qp_rd_atom;
  356. int max_ee_rd_atom;
  357. int max_res_rd_atom;
  358. int max_qp_init_rd_atom;
  359. int max_ee_init_rd_atom;
  360. enum ib_atomic_cap atomic_cap;
  361. enum ib_atomic_cap masked_atomic_cap;
  362. int max_ee;
  363. int max_rdd;
  364. int max_mw;
  365. int max_raw_ipv6_qp;
  366. int max_raw_ethy_qp;
  367. int max_mcast_grp;
  368. int max_mcast_qp_attach;
  369. int max_total_mcast_qp_attach;
  370. int max_ah;
  371. int max_srq;
  372. int max_srq_wr;
  373. int max_srq_sge;
  374. unsigned int max_fast_reg_page_list_len;
  375. unsigned int max_pi_fast_reg_page_list_len;
  376. u16 max_pkeys;
  377. u8 local_ca_ack_delay;
  378. int sig_prot_cap;
  379. int sig_guard_cap;
  380. struct ib_odp_caps odp_caps;
  381. uint64_t timestamp_mask;
  382. uint64_t hca_core_clock; /* in KHZ */
  383. struct ib_rss_caps rss_caps;
  384. u32 max_wq_type_rq;
  385. u32 raw_packet_caps; /* Use ib_raw_packet_caps enum */
  386. struct ib_tm_caps tm_caps;
  387. struct ib_cq_caps cq_caps;
  388. u64 max_dm_size;
  389. /* Max entries for sgl for optimized performance per READ */
  390. u32 max_sgl_rd;
  391. };
  392. enum ib_mtu {
  393. IB_MTU_256 = 1,
  394. IB_MTU_512 = 2,
  395. IB_MTU_1024 = 3,
  396. IB_MTU_2048 = 4,
  397. IB_MTU_4096 = 5
  398. };
  399. enum opa_mtu {
  400. OPA_MTU_8192 = 6,
  401. OPA_MTU_10240 = 7
  402. };
  403. static inline int ib_mtu_enum_to_int(enum ib_mtu mtu)
  404. {
  405. switch (mtu) {
  406. case IB_MTU_256: return 256;
  407. case IB_MTU_512: return 512;
  408. case IB_MTU_1024: return 1024;
  409. case IB_MTU_2048: return 2048;
  410. case IB_MTU_4096: return 4096;
  411. default: return -1;
  412. }
  413. }
  414. static inline enum ib_mtu ib_mtu_int_to_enum(int mtu)
  415. {
  416. if (mtu >= 4096)
  417. return IB_MTU_4096;
  418. else if (mtu >= 2048)
  419. return IB_MTU_2048;
  420. else if (mtu >= 1024)
  421. return IB_MTU_1024;
  422. else if (mtu >= 512)
  423. return IB_MTU_512;
  424. else
  425. return IB_MTU_256;
  426. }
  427. static inline int opa_mtu_enum_to_int(enum opa_mtu mtu)
  428. {
  429. switch (mtu) {
  430. case OPA_MTU_8192:
  431. return 8192;
  432. case OPA_MTU_10240:
  433. return 10240;
  434. default:
  435. return(ib_mtu_enum_to_int((enum ib_mtu)mtu));
  436. }
  437. }
  438. static inline enum opa_mtu opa_mtu_int_to_enum(int mtu)
  439. {
  440. if (mtu >= 10240)
  441. return OPA_MTU_10240;
  442. else if (mtu >= 8192)
  443. return OPA_MTU_8192;
  444. else
  445. return ((enum opa_mtu)ib_mtu_int_to_enum(mtu));
  446. }
  447. enum ib_port_state {
  448. IB_PORT_NOP = 0,
  449. IB_PORT_DOWN = 1,
  450. IB_PORT_INIT = 2,
  451. IB_PORT_ARMED = 3,
  452. IB_PORT_ACTIVE = 4,
  453. IB_PORT_ACTIVE_DEFER = 5
  454. };
  455. enum ib_port_phys_state {
  456. IB_PORT_PHYS_STATE_SLEEP = 1,
  457. IB_PORT_PHYS_STATE_POLLING = 2,
  458. IB_PORT_PHYS_STATE_DISABLED = 3,
  459. IB_PORT_PHYS_STATE_PORT_CONFIGURATION_TRAINING = 4,
  460. IB_PORT_PHYS_STATE_LINK_UP = 5,
  461. IB_PORT_PHYS_STATE_LINK_ERROR_RECOVERY = 6,
  462. IB_PORT_PHYS_STATE_PHY_TEST = 7,
  463. };
  464. enum ib_port_width {
  465. IB_WIDTH_1X = 1,
  466. IB_WIDTH_2X = 16,
  467. IB_WIDTH_4X = 2,
  468. IB_WIDTH_8X = 4,
  469. IB_WIDTH_12X = 8
  470. };
  471. static inline int ib_width_enum_to_int(enum ib_port_width width)
  472. {
  473. switch (width) {
  474. case IB_WIDTH_1X: return 1;
  475. case IB_WIDTH_2X: return 2;
  476. case IB_WIDTH_4X: return 4;
  477. case IB_WIDTH_8X: return 8;
  478. case IB_WIDTH_12X: return 12;
  479. default: return -1;
  480. }
  481. }
  482. enum ib_port_speed {
  483. IB_SPEED_SDR = 1,
  484. IB_SPEED_DDR = 2,
  485. IB_SPEED_QDR = 4,
  486. IB_SPEED_FDR10 = 8,
  487. IB_SPEED_FDR = 16,
  488. IB_SPEED_EDR = 32,
  489. IB_SPEED_HDR = 64,
  490. IB_SPEED_NDR = 128,
  491. };
  492. /**
  493. * struct rdma_hw_stats
  494. * @lock - Mutex to protect parallel write access to lifespan and values
  495. * of counters, which are 64bits and not guaranteeed to be written
  496. * atomicaly on 32bits systems.
  497. * @timestamp - Used by the core code to track when the last update was
  498. * @lifespan - Used by the core code to determine how old the counters
  499. * should be before being updated again. Stored in jiffies, defaults
  500. * to 10 milliseconds, drivers can override the default be specifying
  501. * their own value during their allocation routine.
  502. * @name - Array of pointers to static names used for the counters in
  503. * directory.
  504. * @num_counters - How many hardware counters there are. If name is
  505. * shorter than this number, a kernel oops will result. Driver authors
  506. * are encouraged to leave BUILD_BUG_ON(ARRAY_SIZE(@name) < num_counters)
  507. * in their code to prevent this.
  508. * @value - Array of u64 counters that are accessed by the sysfs code and
  509. * filled in by the drivers get_stats routine
  510. */
  511. struct rdma_hw_stats {
  512. struct mutex lock; /* Protect lifespan and values[] */
  513. unsigned long timestamp;
  514. unsigned long lifespan;
  515. const char * const *names;
  516. int num_counters;
  517. u64 value[];
  518. };
  519. #define RDMA_HW_STATS_DEFAULT_LIFESPAN 10
  520. /**
  521. * rdma_alloc_hw_stats_struct - Helper function to allocate dynamic struct
  522. * for drivers.
  523. * @names - Array of static const char *
  524. * @num_counters - How many elements in array
  525. * @lifespan - How many milliseconds between updates
  526. */
  527. static inline struct rdma_hw_stats *rdma_alloc_hw_stats_struct(
  528. const char * const *names, int num_counters,
  529. unsigned long lifespan)
  530. {
  531. struct rdma_hw_stats *stats;
  532. stats = kzalloc(sizeof(*stats) + num_counters * sizeof(u64),
  533. GFP_KERNEL);
  534. if (!stats)
  535. return NULL;
  536. stats->names = names;
  537. stats->num_counters = num_counters;
  538. stats->lifespan = msecs_to_jiffies(lifespan);
  539. return stats;
  540. }
  541. /* Define bits for the various functionality this port needs to be supported by
  542. * the core.
  543. */
  544. /* Management 0x00000FFF */
  545. #define RDMA_CORE_CAP_IB_MAD 0x00000001
  546. #define RDMA_CORE_CAP_IB_SMI 0x00000002
  547. #define RDMA_CORE_CAP_IB_CM 0x00000004
  548. #define RDMA_CORE_CAP_IW_CM 0x00000008
  549. #define RDMA_CORE_CAP_IB_SA 0x00000010
  550. #define RDMA_CORE_CAP_OPA_MAD 0x00000020
  551. /* Address format 0x000FF000 */
  552. #define RDMA_CORE_CAP_AF_IB 0x00001000
  553. #define RDMA_CORE_CAP_ETH_AH 0x00002000
  554. #define RDMA_CORE_CAP_OPA_AH 0x00004000
  555. #define RDMA_CORE_CAP_IB_GRH_REQUIRED 0x00008000
  556. /* Protocol 0xFFF00000 */
  557. #define RDMA_CORE_CAP_PROT_IB 0x00100000
  558. #define RDMA_CORE_CAP_PROT_ROCE 0x00200000
  559. #define RDMA_CORE_CAP_PROT_IWARP 0x00400000
  560. #define RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP 0x00800000
  561. #define RDMA_CORE_CAP_PROT_RAW_PACKET 0x01000000
  562. #define RDMA_CORE_CAP_PROT_USNIC 0x02000000
  563. #define RDMA_CORE_PORT_IB_GRH_REQUIRED (RDMA_CORE_CAP_IB_GRH_REQUIRED \
  564. | RDMA_CORE_CAP_PROT_ROCE \
  565. | RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP)
  566. #define RDMA_CORE_PORT_IBA_IB (RDMA_CORE_CAP_PROT_IB \
  567. | RDMA_CORE_CAP_IB_MAD \
  568. | RDMA_CORE_CAP_IB_SMI \
  569. | RDMA_CORE_CAP_IB_CM \
  570. | RDMA_CORE_CAP_IB_SA \
  571. | RDMA_CORE_CAP_AF_IB)
  572. #define RDMA_CORE_PORT_IBA_ROCE (RDMA_CORE_CAP_PROT_ROCE \
  573. | RDMA_CORE_CAP_IB_MAD \
  574. | RDMA_CORE_CAP_IB_CM \
  575. | RDMA_CORE_CAP_AF_IB \
  576. | RDMA_CORE_CAP_ETH_AH)
  577. #define RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP \
  578. (RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP \
  579. | RDMA_CORE_CAP_IB_MAD \
  580. | RDMA_CORE_CAP_IB_CM \
  581. | RDMA_CORE_CAP_AF_IB \
  582. | RDMA_CORE_CAP_ETH_AH)
  583. #define RDMA_CORE_PORT_IWARP (RDMA_CORE_CAP_PROT_IWARP \
  584. | RDMA_CORE_CAP_IW_CM)
  585. #define RDMA_CORE_PORT_INTEL_OPA (RDMA_CORE_PORT_IBA_IB \
  586. | RDMA_CORE_CAP_OPA_MAD)
  587. #define RDMA_CORE_PORT_RAW_PACKET (RDMA_CORE_CAP_PROT_RAW_PACKET)
  588. #define RDMA_CORE_PORT_USNIC (RDMA_CORE_CAP_PROT_USNIC)
  589. struct ib_port_attr {
  590. u64 subnet_prefix;
  591. enum ib_port_state state;
  592. enum ib_mtu max_mtu;
  593. enum ib_mtu active_mtu;
  594. u32 phys_mtu;
  595. int gid_tbl_len;
  596. unsigned int ip_gids:1;
  597. /* This is the value from PortInfo CapabilityMask, defined by IBA */
  598. u32 port_cap_flags;
  599. u32 max_msg_sz;
  600. u32 bad_pkey_cntr;
  601. u32 qkey_viol_cntr;
  602. u16 pkey_tbl_len;
  603. u32 sm_lid;
  604. u32 lid;
  605. u8 lmc;
  606. u8 max_vl_num;
  607. u8 sm_sl;
  608. u8 subnet_timeout;
  609. u8 init_type_reply;
  610. u8 active_width;
  611. u16 active_speed;
  612. u8 phys_state;
  613. u16 port_cap_flags2;
  614. };
  615. enum ib_device_modify_flags {
  616. IB_DEVICE_MODIFY_SYS_IMAGE_GUID = 1 << 0,
  617. IB_DEVICE_MODIFY_NODE_DESC = 1 << 1
  618. };
  619. #define IB_DEVICE_NODE_DESC_MAX 64
  620. struct ib_device_modify {
  621. u64 sys_image_guid;
  622. char node_desc[IB_DEVICE_NODE_DESC_MAX];
  623. };
  624. enum ib_port_modify_flags {
  625. IB_PORT_SHUTDOWN = 1,
  626. IB_PORT_INIT_TYPE = (1<<2),
  627. IB_PORT_RESET_QKEY_CNTR = (1<<3),
  628. IB_PORT_OPA_MASK_CHG = (1<<4)
  629. };
  630. struct ib_port_modify {
  631. u32 set_port_cap_mask;
  632. u32 clr_port_cap_mask;
  633. u8 init_type;
  634. };
  635. enum ib_event_type {
  636. IB_EVENT_CQ_ERR,
  637. IB_EVENT_QP_FATAL,
  638. IB_EVENT_QP_REQ_ERR,
  639. IB_EVENT_QP_ACCESS_ERR,
  640. IB_EVENT_COMM_EST,
  641. IB_EVENT_SQ_DRAINED,
  642. IB_EVENT_PATH_MIG,
  643. IB_EVENT_PATH_MIG_ERR,
  644. IB_EVENT_DEVICE_FATAL,
  645. IB_EVENT_PORT_ACTIVE,
  646. IB_EVENT_PORT_ERR,
  647. IB_EVENT_LID_CHANGE,
  648. IB_EVENT_PKEY_CHANGE,
  649. IB_EVENT_SM_CHANGE,
  650. IB_EVENT_SRQ_ERR,
  651. IB_EVENT_SRQ_LIMIT_REACHED,
  652. IB_EVENT_QP_LAST_WQE_REACHED,
  653. IB_EVENT_CLIENT_REREGISTER,
  654. IB_EVENT_GID_CHANGE,
  655. IB_EVENT_WQ_FATAL,
  656. };
  657. const char *__attribute_const__ ib_event_msg(enum ib_event_type event);
  658. struct ib_event {
  659. struct ib_device *device;
  660. union {
  661. struct ib_cq *cq;
  662. struct ib_qp *qp;
  663. struct ib_srq *srq;
  664. struct ib_wq *wq;
  665. u8 port_num;
  666. } element;
  667. enum ib_event_type event;
  668. };
  669. struct ib_event_handler {
  670. struct ib_device *device;
  671. void (*handler)(struct ib_event_handler *, struct ib_event *);
  672. struct list_head list;
  673. };
  674. #define INIT_IB_EVENT_HANDLER(_ptr, _device, _handler) \
  675. do { \
  676. (_ptr)->device = _device; \
  677. (_ptr)->handler = _handler; \
  678. INIT_LIST_HEAD(&(_ptr)->list); \
  679. } while (0)
  680. struct ib_global_route {
  681. const struct ib_gid_attr *sgid_attr;
  682. union ib_gid dgid;
  683. u32 flow_label;
  684. u8 sgid_index;
  685. u8 hop_limit;
  686. u8 traffic_class;
  687. };
  688. struct ib_grh {
  689. __be32 version_tclass_flow;
  690. __be16 paylen;
  691. u8 next_hdr;
  692. u8 hop_limit;
  693. union ib_gid sgid;
  694. union ib_gid dgid;
  695. };
  696. union rdma_network_hdr {
  697. struct ib_grh ibgrh;
  698. struct {
  699. /* The IB spec states that if it's IPv4, the header
  700. * is located in the last 20 bytes of the header.
  701. */
  702. u8 reserved[20];
  703. struct iphdr roce4grh;
  704. };
  705. };
  706. #define IB_QPN_MASK 0xFFFFFF
  707. enum {
  708. IB_MULTICAST_QPN = 0xffffff
  709. };
  710. #define IB_LID_PERMISSIVE cpu_to_be16(0xFFFF)
  711. #define IB_MULTICAST_LID_BASE cpu_to_be16(0xC000)
  712. enum ib_ah_flags {
  713. IB_AH_GRH = 1
  714. };
  715. enum ib_rate {
  716. IB_RATE_PORT_CURRENT = 0,
  717. IB_RATE_2_5_GBPS = 2,
  718. IB_RATE_5_GBPS = 5,
  719. IB_RATE_10_GBPS = 3,
  720. IB_RATE_20_GBPS = 6,
  721. IB_RATE_30_GBPS = 4,
  722. IB_RATE_40_GBPS = 7,
  723. IB_RATE_60_GBPS = 8,
  724. IB_RATE_80_GBPS = 9,
  725. IB_RATE_120_GBPS = 10,
  726. IB_RATE_14_GBPS = 11,
  727. IB_RATE_56_GBPS = 12,
  728. IB_RATE_112_GBPS = 13,
  729. IB_RATE_168_GBPS = 14,
  730. IB_RATE_25_GBPS = 15,
  731. IB_RATE_100_GBPS = 16,
  732. IB_RATE_200_GBPS = 17,
  733. IB_RATE_300_GBPS = 18,
  734. IB_RATE_28_GBPS = 19,
  735. IB_RATE_50_GBPS = 20,
  736. IB_RATE_400_GBPS = 21,
  737. IB_RATE_600_GBPS = 22,
  738. };
  739. /**
  740. * ib_rate_to_mult - Convert the IB rate enum to a multiple of the
  741. * base rate of 2.5 Gbit/sec. For example, IB_RATE_5_GBPS will be
  742. * converted to 2, since 5 Gbit/sec is 2 * 2.5 Gbit/sec.
  743. * @rate: rate to convert.
  744. */
  745. __attribute_const__ int ib_rate_to_mult(enum ib_rate rate);
  746. /**
  747. * ib_rate_to_mbps - Convert the IB rate enum to Mbps.
  748. * For example, IB_RATE_2_5_GBPS will be converted to 2500.
  749. * @rate: rate to convert.
  750. */
  751. __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate);
  752. /**
  753. * enum ib_mr_type - memory region type
  754. * @IB_MR_TYPE_MEM_REG: memory region that is used for
  755. * normal registration
  756. * @IB_MR_TYPE_SG_GAPS: memory region that is capable to
  757. * register any arbitrary sg lists (without
  758. * the normal mr constraints - see
  759. * ib_map_mr_sg)
  760. * @IB_MR_TYPE_DM: memory region that is used for device
  761. * memory registration
  762. * @IB_MR_TYPE_USER: memory region that is used for the user-space
  763. * application
  764. * @IB_MR_TYPE_DMA: memory region that is used for DMA operations
  765. * without address translations (VA=PA)
  766. * @IB_MR_TYPE_INTEGRITY: memory region that is used for
  767. * data integrity operations
  768. */
  769. enum ib_mr_type {
  770. IB_MR_TYPE_MEM_REG,
  771. IB_MR_TYPE_SG_GAPS,
  772. IB_MR_TYPE_DM,
  773. IB_MR_TYPE_USER,
  774. IB_MR_TYPE_DMA,
  775. IB_MR_TYPE_INTEGRITY,
  776. };
  777. enum ib_mr_status_check {
  778. IB_MR_CHECK_SIG_STATUS = 1,
  779. };
  780. /**
  781. * struct ib_mr_status - Memory region status container
  782. *
  783. * @fail_status: Bitmask of MR checks status. For each
  784. * failed check a corresponding status bit is set.
  785. * @sig_err: Additional info for IB_MR_CEHCK_SIG_STATUS
  786. * failure.
  787. */
  788. struct ib_mr_status {
  789. u32 fail_status;
  790. struct ib_sig_err sig_err;
  791. };
  792. /**
  793. * mult_to_ib_rate - Convert a multiple of 2.5 Gbit/sec to an IB rate
  794. * enum.
  795. * @mult: multiple to convert.
  796. */
  797. __attribute_const__ enum ib_rate mult_to_ib_rate(int mult);
  798. struct rdma_ah_init_attr {
  799. struct rdma_ah_attr *ah_attr;
  800. u32 flags;
  801. struct net_device *xmit_slave;
  802. };
  803. enum rdma_ah_attr_type {
  804. RDMA_AH_ATTR_TYPE_UNDEFINED,
  805. RDMA_AH_ATTR_TYPE_IB,
  806. RDMA_AH_ATTR_TYPE_ROCE,
  807. RDMA_AH_ATTR_TYPE_OPA,
  808. };
  809. struct ib_ah_attr {
  810. u16 dlid;
  811. u8 src_path_bits;
  812. };
  813. struct roce_ah_attr {
  814. u8 dmac[ETH_ALEN];
  815. };
  816. struct opa_ah_attr {
  817. u32 dlid;
  818. u8 src_path_bits;
  819. bool make_grd;
  820. };
  821. struct rdma_ah_attr {
  822. struct ib_global_route grh;
  823. u8 sl;
  824. u8 static_rate;
  825. u8 port_num;
  826. u8 ah_flags;
  827. enum rdma_ah_attr_type type;
  828. union {
  829. struct ib_ah_attr ib;
  830. struct roce_ah_attr roce;
  831. struct opa_ah_attr opa;
  832. };
  833. };
  834. enum ib_wc_status {
  835. IB_WC_SUCCESS,
  836. IB_WC_LOC_LEN_ERR,
  837. IB_WC_LOC_QP_OP_ERR,
  838. IB_WC_LOC_EEC_OP_ERR,
  839. IB_WC_LOC_PROT_ERR,
  840. IB_WC_WR_FLUSH_ERR,
  841. IB_WC_MW_BIND_ERR,
  842. IB_WC_BAD_RESP_ERR,
  843. IB_WC_LOC_ACCESS_ERR,
  844. IB_WC_REM_INV_REQ_ERR,
  845. IB_WC_REM_ACCESS_ERR,
  846. IB_WC_REM_OP_ERR,
  847. IB_WC_RETRY_EXC_ERR,
  848. IB_WC_RNR_RETRY_EXC_ERR,
  849. IB_WC_LOC_RDD_VIOL_ERR,
  850. IB_WC_REM_INV_RD_REQ_ERR,
  851. IB_WC_REM_ABORT_ERR,
  852. IB_WC_INV_EECN_ERR,
  853. IB_WC_INV_EEC_STATE_ERR,
  854. IB_WC_FATAL_ERR,
  855. IB_WC_RESP_TIMEOUT_ERR,
  856. IB_WC_GENERAL_ERR
  857. };
  858. const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status);
  859. enum ib_wc_opcode {
  860. IB_WC_SEND = IB_UVERBS_WC_SEND,
  861. IB_WC_RDMA_WRITE = IB_UVERBS_WC_RDMA_WRITE,
  862. IB_WC_RDMA_READ = IB_UVERBS_WC_RDMA_READ,
  863. IB_WC_COMP_SWAP = IB_UVERBS_WC_COMP_SWAP,
  864. IB_WC_FETCH_ADD = IB_UVERBS_WC_FETCH_ADD,
  865. IB_WC_BIND_MW = IB_UVERBS_WC_BIND_MW,
  866. IB_WC_LOCAL_INV = IB_UVERBS_WC_LOCAL_INV,
  867. IB_WC_LSO = IB_UVERBS_WC_TSO,
  868. IB_WC_REG_MR,
  869. IB_WC_MASKED_COMP_SWAP,
  870. IB_WC_MASKED_FETCH_ADD,
  871. /*
  872. * Set value of IB_WC_RECV so consumers can test if a completion is a
  873. * receive by testing (opcode & IB_WC_RECV).
  874. */
  875. IB_WC_RECV = 1 << 7,
  876. IB_WC_RECV_RDMA_WITH_IMM
  877. };
  878. enum ib_wc_flags {
  879. IB_WC_GRH = 1,
  880. IB_WC_WITH_IMM = (1<<1),
  881. IB_WC_WITH_INVALIDATE = (1<<2),
  882. IB_WC_IP_CSUM_OK = (1<<3),
  883. IB_WC_WITH_SMAC = (1<<4),
  884. IB_WC_WITH_VLAN = (1<<5),
  885. IB_WC_WITH_NETWORK_HDR_TYPE = (1<<6),
  886. };
  887. struct ib_wc {
  888. union {
  889. u64 wr_id;
  890. struct ib_cqe *wr_cqe;
  891. };
  892. enum ib_wc_status status;
  893. enum ib_wc_opcode opcode;
  894. u32 vendor_err;
  895. u32 byte_len;
  896. struct ib_qp *qp;
  897. union {
  898. __be32 imm_data;
  899. u32 invalidate_rkey;
  900. } ex;
  901. u32 src_qp;
  902. u32 slid;
  903. int wc_flags;
  904. u16 pkey_index;
  905. u8 sl;
  906. u8 dlid_path_bits;
  907. u8 port_num; /* valid only for DR SMPs on switches */
  908. u8 smac[ETH_ALEN];
  909. u16 vlan_id;
  910. u8 network_hdr_type;
  911. };
  912. enum ib_cq_notify_flags {
  913. IB_CQ_SOLICITED = 1 << 0,
  914. IB_CQ_NEXT_COMP = 1 << 1,
  915. IB_CQ_SOLICITED_MASK = IB_CQ_SOLICITED | IB_CQ_NEXT_COMP,
  916. IB_CQ_REPORT_MISSED_EVENTS = 1 << 2,
  917. };
  918. enum ib_srq_type {
  919. IB_SRQT_BASIC = IB_UVERBS_SRQT_BASIC,
  920. IB_SRQT_XRC = IB_UVERBS_SRQT_XRC,
  921. IB_SRQT_TM = IB_UVERBS_SRQT_TM,
  922. };
  923. static inline bool ib_srq_has_cq(enum ib_srq_type srq_type)
  924. {
  925. return srq_type == IB_SRQT_XRC ||
  926. srq_type == IB_SRQT_TM;
  927. }
  928. enum ib_srq_attr_mask {
  929. IB_SRQ_MAX_WR = 1 << 0,
  930. IB_SRQ_LIMIT = 1 << 1,
  931. };
  932. struct ib_srq_attr {
  933. u32 max_wr;
  934. u32 max_sge;
  935. u32 srq_limit;
  936. };
  937. struct ib_srq_init_attr {
  938. void (*event_handler)(struct ib_event *, void *);
  939. void *srq_context;
  940. struct ib_srq_attr attr;
  941. enum ib_srq_type srq_type;
  942. struct {
  943. struct ib_cq *cq;
  944. union {
  945. struct {
  946. struct ib_xrcd *xrcd;
  947. } xrc;
  948. struct {
  949. u32 max_num_tags;
  950. } tag_matching;
  951. };
  952. } ext;
  953. };
  954. struct ib_qp_cap {
  955. u32 max_send_wr;
  956. u32 max_recv_wr;
  957. u32 max_send_sge;
  958. u32 max_recv_sge;
  959. u32 max_inline_data;
  960. /*
  961. * Maximum number of rdma_rw_ctx structures in flight at a time.
  962. * ib_create_qp() will calculate the right amount of neededed WRs
  963. * and MRs based on this.
  964. */
  965. u32 max_rdma_ctxs;
  966. };
  967. enum ib_sig_type {
  968. IB_SIGNAL_ALL_WR,
  969. IB_SIGNAL_REQ_WR
  970. };
  971. enum ib_qp_type {
  972. /*
  973. * IB_QPT_SMI and IB_QPT_GSI have to be the first two entries
  974. * here (and in that order) since the MAD layer uses them as
  975. * indices into a 2-entry table.
  976. */
  977. IB_QPT_SMI,
  978. IB_QPT_GSI,
  979. IB_QPT_RC = IB_UVERBS_QPT_RC,
  980. IB_QPT_UC = IB_UVERBS_QPT_UC,
  981. IB_QPT_UD = IB_UVERBS_QPT_UD,
  982. IB_QPT_RAW_IPV6,
  983. IB_QPT_RAW_ETHERTYPE,
  984. IB_QPT_RAW_PACKET = IB_UVERBS_QPT_RAW_PACKET,
  985. IB_QPT_XRC_INI = IB_UVERBS_QPT_XRC_INI,
  986. IB_QPT_XRC_TGT = IB_UVERBS_QPT_XRC_TGT,
  987. IB_QPT_MAX,
  988. IB_QPT_DRIVER = IB_UVERBS_QPT_DRIVER,
  989. /* Reserve a range for qp types internal to the low level driver.
  990. * These qp types will not be visible at the IB core layer, so the
  991. * IB_QPT_MAX usages should not be affected in the core layer
  992. */
  993. IB_QPT_RESERVED1 = 0x1000,
  994. IB_QPT_RESERVED2,
  995. IB_QPT_RESERVED3,
  996. IB_QPT_RESERVED4,
  997. IB_QPT_RESERVED5,
  998. IB_QPT_RESERVED6,
  999. IB_QPT_RESERVED7,
  1000. IB_QPT_RESERVED8,
  1001. IB_QPT_RESERVED9,
  1002. IB_QPT_RESERVED10,
  1003. };
  1004. enum ib_qp_create_flags {
  1005. IB_QP_CREATE_IPOIB_UD_LSO = 1 << 0,
  1006. IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK =
  1007. IB_UVERBS_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
  1008. IB_QP_CREATE_CROSS_CHANNEL = 1 << 2,
  1009. IB_QP_CREATE_MANAGED_SEND = 1 << 3,
  1010. IB_QP_CREATE_MANAGED_RECV = 1 << 4,
  1011. IB_QP_CREATE_NETIF_QP = 1 << 5,
  1012. IB_QP_CREATE_INTEGRITY_EN = 1 << 6,
  1013. IB_QP_CREATE_NETDEV_USE = 1 << 7,
  1014. IB_QP_CREATE_SCATTER_FCS =
  1015. IB_UVERBS_QP_CREATE_SCATTER_FCS,
  1016. IB_QP_CREATE_CVLAN_STRIPPING =
  1017. IB_UVERBS_QP_CREATE_CVLAN_STRIPPING,
  1018. IB_QP_CREATE_SOURCE_QPN = 1 << 10,
  1019. IB_QP_CREATE_PCI_WRITE_END_PADDING =
  1020. IB_UVERBS_QP_CREATE_PCI_WRITE_END_PADDING,
  1021. /* reserve bits 26-31 for low level drivers' internal use */
  1022. IB_QP_CREATE_RESERVED_START = 1 << 26,
  1023. IB_QP_CREATE_RESERVED_END = 1 << 31,
  1024. };
  1025. /*
  1026. * Note: users may not call ib_close_qp or ib_destroy_qp from the event_handler
  1027. * callback to destroy the passed in QP.
  1028. */
  1029. struct ib_qp_init_attr {
  1030. /* Consumer's event_handler callback must not block */
  1031. void (*event_handler)(struct ib_event *, void *);
  1032. void *qp_context;
  1033. struct ib_cq *send_cq;
  1034. struct ib_cq *recv_cq;
  1035. struct ib_srq *srq;
  1036. struct ib_xrcd *xrcd; /* XRC TGT QPs only */
  1037. struct ib_qp_cap cap;
  1038. enum ib_sig_type sq_sig_type;
  1039. enum ib_qp_type qp_type;
  1040. u32 create_flags;
  1041. /*
  1042. * Only needed for special QP types, or when using the RW API.
  1043. */
  1044. u8 port_num;
  1045. struct ib_rwq_ind_table *rwq_ind_tbl;
  1046. u32 source_qpn;
  1047. };
  1048. struct ib_qp_open_attr {
  1049. void (*event_handler)(struct ib_event *, void *);
  1050. void *qp_context;
  1051. u32 qp_num;
  1052. enum ib_qp_type qp_type;
  1053. };
  1054. enum ib_rnr_timeout {
  1055. IB_RNR_TIMER_655_36 = 0,
  1056. IB_RNR_TIMER_000_01 = 1,
  1057. IB_RNR_TIMER_000_02 = 2,
  1058. IB_RNR_TIMER_000_03 = 3,
  1059. IB_RNR_TIMER_000_04 = 4,
  1060. IB_RNR_TIMER_000_06 = 5,
  1061. IB_RNR_TIMER_000_08 = 6,
  1062. IB_RNR_TIMER_000_12 = 7,
  1063. IB_RNR_TIMER_000_16 = 8,
  1064. IB_RNR_TIMER_000_24 = 9,
  1065. IB_RNR_TIMER_000_32 = 10,
  1066. IB_RNR_TIMER_000_48 = 11,
  1067. IB_RNR_TIMER_000_64 = 12,
  1068. IB_RNR_TIMER_000_96 = 13,
  1069. IB_RNR_TIMER_001_28 = 14,
  1070. IB_RNR_TIMER_001_92 = 15,
  1071. IB_RNR_TIMER_002_56 = 16,
  1072. IB_RNR_TIMER_003_84 = 17,
  1073. IB_RNR_TIMER_005_12 = 18,
  1074. IB_RNR_TIMER_007_68 = 19,
  1075. IB_RNR_TIMER_010_24 = 20,
  1076. IB_RNR_TIMER_015_36 = 21,
  1077. IB_RNR_TIMER_020_48 = 22,
  1078. IB_RNR_TIMER_030_72 = 23,
  1079. IB_RNR_TIMER_040_96 = 24,
  1080. IB_RNR_TIMER_061_44 = 25,
  1081. IB_RNR_TIMER_081_92 = 26,
  1082. IB_RNR_TIMER_122_88 = 27,
  1083. IB_RNR_TIMER_163_84 = 28,
  1084. IB_RNR_TIMER_245_76 = 29,
  1085. IB_RNR_TIMER_327_68 = 30,
  1086. IB_RNR_TIMER_491_52 = 31
  1087. };
  1088. enum ib_qp_attr_mask {
  1089. IB_QP_STATE = 1,
  1090. IB_QP_CUR_STATE = (1<<1),
  1091. IB_QP_EN_SQD_ASYNC_NOTIFY = (1<<2),
  1092. IB_QP_ACCESS_FLAGS = (1<<3),
  1093. IB_QP_PKEY_INDEX = (1<<4),
  1094. IB_QP_PORT = (1<<5),
  1095. IB_QP_QKEY = (1<<6),
  1096. IB_QP_AV = (1<<7),
  1097. IB_QP_PATH_MTU = (1<<8),
  1098. IB_QP_TIMEOUT = (1<<9),
  1099. IB_QP_RETRY_CNT = (1<<10),
  1100. IB_QP_RNR_RETRY = (1<<11),
  1101. IB_QP_RQ_PSN = (1<<12),
  1102. IB_QP_MAX_QP_RD_ATOMIC = (1<<13),
  1103. IB_QP_ALT_PATH = (1<<14),
  1104. IB_QP_MIN_RNR_TIMER = (1<<15),
  1105. IB_QP_SQ_PSN = (1<<16),
  1106. IB_QP_MAX_DEST_RD_ATOMIC = (1<<17),
  1107. IB_QP_PATH_MIG_STATE = (1<<18),
  1108. IB_QP_CAP = (1<<19),
  1109. IB_QP_DEST_QPN = (1<<20),
  1110. IB_QP_RESERVED1 = (1<<21),
  1111. IB_QP_RESERVED2 = (1<<22),
  1112. IB_QP_RESERVED3 = (1<<23),
  1113. IB_QP_RESERVED4 = (1<<24),
  1114. IB_QP_RATE_LIMIT = (1<<25),
  1115. };
  1116. enum ib_qp_state {
  1117. IB_QPS_RESET,
  1118. IB_QPS_INIT,
  1119. IB_QPS_RTR,
  1120. IB_QPS_RTS,
  1121. IB_QPS_SQD,
  1122. IB_QPS_SQE,
  1123. IB_QPS_ERR
  1124. };
  1125. enum ib_mig_state {
  1126. IB_MIG_MIGRATED,
  1127. IB_MIG_REARM,
  1128. IB_MIG_ARMED
  1129. };
  1130. enum ib_mw_type {
  1131. IB_MW_TYPE_1 = 1,
  1132. IB_MW_TYPE_2 = 2
  1133. };
  1134. struct ib_qp_attr {
  1135. enum ib_qp_state qp_state;
  1136. enum ib_qp_state cur_qp_state;
  1137. enum ib_mtu path_mtu;
  1138. enum ib_mig_state path_mig_state;
  1139. u32 qkey;
  1140. u32 rq_psn;
  1141. u32 sq_psn;
  1142. u32 dest_qp_num;
  1143. int qp_access_flags;
  1144. struct ib_qp_cap cap;
  1145. struct rdma_ah_attr ah_attr;
  1146. struct rdma_ah_attr alt_ah_attr;
  1147. u16 pkey_index;
  1148. u16 alt_pkey_index;
  1149. u8 en_sqd_async_notify;
  1150. u8 sq_draining;
  1151. u8 max_rd_atomic;
  1152. u8 max_dest_rd_atomic;
  1153. u8 min_rnr_timer;
  1154. u8 port_num;
  1155. u8 timeout;
  1156. u8 retry_cnt;
  1157. u8 rnr_retry;
  1158. u8 alt_port_num;
  1159. u8 alt_timeout;
  1160. u32 rate_limit;
  1161. struct net_device *xmit_slave;
  1162. };
  1163. enum ib_wr_opcode {
  1164. /* These are shared with userspace */
  1165. IB_WR_RDMA_WRITE = IB_UVERBS_WR_RDMA_WRITE,
  1166. IB_WR_RDMA_WRITE_WITH_IMM = IB_UVERBS_WR_RDMA_WRITE_WITH_IMM,
  1167. IB_WR_SEND = IB_UVERBS_WR_SEND,
  1168. IB_WR_SEND_WITH_IMM = IB_UVERBS_WR_SEND_WITH_IMM,
  1169. IB_WR_RDMA_READ = IB_UVERBS_WR_RDMA_READ,
  1170. IB_WR_ATOMIC_CMP_AND_SWP = IB_UVERBS_WR_ATOMIC_CMP_AND_SWP,
  1171. IB_WR_ATOMIC_FETCH_AND_ADD = IB_UVERBS_WR_ATOMIC_FETCH_AND_ADD,
  1172. IB_WR_BIND_MW = IB_UVERBS_WR_BIND_MW,
  1173. IB_WR_LSO = IB_UVERBS_WR_TSO,
  1174. IB_WR_SEND_WITH_INV = IB_UVERBS_WR_SEND_WITH_INV,
  1175. IB_WR_RDMA_READ_WITH_INV = IB_UVERBS_WR_RDMA_READ_WITH_INV,
  1176. IB_WR_LOCAL_INV = IB_UVERBS_WR_LOCAL_INV,
  1177. IB_WR_MASKED_ATOMIC_CMP_AND_SWP =
  1178. IB_UVERBS_WR_MASKED_ATOMIC_CMP_AND_SWP,
  1179. IB_WR_MASKED_ATOMIC_FETCH_AND_ADD =
  1180. IB_UVERBS_WR_MASKED_ATOMIC_FETCH_AND_ADD,
  1181. /* These are kernel only and can not be issued by userspace */
  1182. IB_WR_REG_MR = 0x20,
  1183. IB_WR_REG_MR_INTEGRITY,
  1184. /* reserve values for low level drivers' internal use.
  1185. * These values will not be used at all in the ib core layer.
  1186. */
  1187. IB_WR_RESERVED1 = 0xf0,
  1188. IB_WR_RESERVED2,
  1189. IB_WR_RESERVED3,
  1190. IB_WR_RESERVED4,
  1191. IB_WR_RESERVED5,
  1192. IB_WR_RESERVED6,
  1193. IB_WR_RESERVED7,
  1194. IB_WR_RESERVED8,
  1195. IB_WR_RESERVED9,
  1196. IB_WR_RESERVED10,
  1197. };
  1198. enum ib_send_flags {
  1199. IB_SEND_FENCE = 1,
  1200. IB_SEND_SIGNALED = (1<<1),
  1201. IB_SEND_SOLICITED = (1<<2),
  1202. IB_SEND_INLINE = (1<<3),
  1203. IB_SEND_IP_CSUM = (1<<4),
  1204. /* reserve bits 26-31 for low level drivers' internal use */
  1205. IB_SEND_RESERVED_START = (1 << 26),
  1206. IB_SEND_RESERVED_END = (1 << 31),
  1207. };
  1208. struct ib_sge {
  1209. u64 addr;
  1210. u32 length;
  1211. u32 lkey;
  1212. };
  1213. struct ib_cqe {
  1214. void (*done)(struct ib_cq *cq, struct ib_wc *wc);
  1215. };
  1216. struct ib_send_wr {
  1217. struct ib_send_wr *next;
  1218. union {
  1219. u64 wr_id;
  1220. struct ib_cqe *wr_cqe;
  1221. };
  1222. struct ib_sge *sg_list;
  1223. int num_sge;
  1224. enum ib_wr_opcode opcode;
  1225. int send_flags;
  1226. union {
  1227. __be32 imm_data;
  1228. u32 invalidate_rkey;
  1229. } ex;
  1230. };
  1231. struct ib_rdma_wr {
  1232. struct ib_send_wr wr;
  1233. u64 remote_addr;
  1234. u32 rkey;
  1235. };
  1236. static inline const struct ib_rdma_wr *rdma_wr(const struct ib_send_wr *wr)
  1237. {
  1238. return container_of(wr, struct ib_rdma_wr, wr);
  1239. }
  1240. struct ib_atomic_wr {
  1241. struct ib_send_wr wr;
  1242. u64 remote_addr;
  1243. u64 compare_add;
  1244. u64 swap;
  1245. u64 compare_add_mask;
  1246. u64 swap_mask;
  1247. u32 rkey;
  1248. };
  1249. static inline const struct ib_atomic_wr *atomic_wr(const struct ib_send_wr *wr)
  1250. {
  1251. return container_of(wr, struct ib_atomic_wr, wr);
  1252. }
  1253. struct ib_ud_wr {
  1254. struct ib_send_wr wr;
  1255. struct ib_ah *ah;
  1256. void *header;
  1257. int hlen;
  1258. int mss;
  1259. u32 remote_qpn;
  1260. u32 remote_qkey;
  1261. u16 pkey_index; /* valid for GSI only */
  1262. u8 port_num; /* valid for DR SMPs on switch only */
  1263. };
  1264. static inline const struct ib_ud_wr *ud_wr(const struct ib_send_wr *wr)
  1265. {
  1266. return container_of(wr, struct ib_ud_wr, wr);
  1267. }
  1268. struct ib_reg_wr {
  1269. struct ib_send_wr wr;
  1270. struct ib_mr *mr;
  1271. u32 key;
  1272. int access;
  1273. };
  1274. static inline const struct ib_reg_wr *reg_wr(const struct ib_send_wr *wr)
  1275. {
  1276. return container_of(wr, struct ib_reg_wr, wr);
  1277. }
  1278. struct ib_recv_wr {
  1279. struct ib_recv_wr *next;
  1280. union {
  1281. u64 wr_id;
  1282. struct ib_cqe *wr_cqe;
  1283. };
  1284. struct ib_sge *sg_list;
  1285. int num_sge;
  1286. };
  1287. enum ib_access_flags {
  1288. IB_ACCESS_LOCAL_WRITE = IB_UVERBS_ACCESS_LOCAL_WRITE,
  1289. IB_ACCESS_REMOTE_WRITE = IB_UVERBS_ACCESS_REMOTE_WRITE,
  1290. IB_ACCESS_REMOTE_READ = IB_UVERBS_ACCESS_REMOTE_READ,
  1291. IB_ACCESS_REMOTE_ATOMIC = IB_UVERBS_ACCESS_REMOTE_ATOMIC,
  1292. IB_ACCESS_MW_BIND = IB_UVERBS_ACCESS_MW_BIND,
  1293. IB_ZERO_BASED = IB_UVERBS_ACCESS_ZERO_BASED,
  1294. IB_ACCESS_ON_DEMAND = IB_UVERBS_ACCESS_ON_DEMAND,
  1295. IB_ACCESS_HUGETLB = IB_UVERBS_ACCESS_HUGETLB,
  1296. IB_ACCESS_RELAXED_ORDERING = IB_UVERBS_ACCESS_RELAXED_ORDERING,
  1297. IB_ACCESS_OPTIONAL = IB_UVERBS_ACCESS_OPTIONAL_RANGE,
  1298. IB_ACCESS_SUPPORTED =
  1299. ((IB_ACCESS_HUGETLB << 1) - 1) | IB_ACCESS_OPTIONAL,
  1300. };
  1301. /*
  1302. * XXX: these are apparently used for ->rereg_user_mr, no idea why they
  1303. * are hidden here instead of a uapi header!
  1304. */
  1305. enum ib_mr_rereg_flags {
  1306. IB_MR_REREG_TRANS = 1,
  1307. IB_MR_REREG_PD = (1<<1),
  1308. IB_MR_REREG_ACCESS = (1<<2),
  1309. IB_MR_REREG_SUPPORTED = ((IB_MR_REREG_ACCESS << 1) - 1)
  1310. };
  1311. struct ib_umem;
  1312. enum rdma_remove_reason {
  1313. /*
  1314. * Userspace requested uobject deletion or initial try
  1315. * to remove uobject via cleanup. Call could fail
  1316. */
  1317. RDMA_REMOVE_DESTROY,
  1318. /* Context deletion. This call should delete the actual object itself */
  1319. RDMA_REMOVE_CLOSE,
  1320. /* Driver is being hot-unplugged. This call should delete the actual object itself */
  1321. RDMA_REMOVE_DRIVER_REMOVE,
  1322. /* uobj is being cleaned-up before being committed */
  1323. RDMA_REMOVE_ABORT,
  1324. };
  1325. struct ib_rdmacg_object {
  1326. #ifdef CONFIG_CGROUP_RDMA
  1327. struct rdma_cgroup *cg; /* owner rdma cgroup */
  1328. #endif
  1329. };
  1330. struct ib_ucontext {
  1331. struct ib_device *device;
  1332. struct ib_uverbs_file *ufile;
  1333. bool cleanup_retryable;
  1334. struct ib_rdmacg_object cg_obj;
  1335. /*
  1336. * Implementation details of the RDMA core, don't use in drivers:
  1337. */
  1338. struct rdma_restrack_entry res;
  1339. struct xarray mmap_xa;
  1340. };
  1341. struct ib_uobject {
  1342. u64 user_handle; /* handle given to us by userspace */
  1343. /* ufile & ucontext owning this object */
  1344. struct ib_uverbs_file *ufile;
  1345. /* FIXME, save memory: ufile->context == context */
  1346. struct ib_ucontext *context; /* associated user context */
  1347. void *object; /* containing object */
  1348. struct list_head list; /* link to context's list */
  1349. struct ib_rdmacg_object cg_obj; /* rdmacg object */
  1350. int id; /* index into kernel idr */
  1351. struct kref ref;
  1352. atomic_t usecnt; /* protects exclusive access */
  1353. struct rcu_head rcu; /* kfree_rcu() overhead */
  1354. const struct uverbs_api_object *uapi_object;
  1355. };
  1356. struct ib_udata {
  1357. const void __user *inbuf;
  1358. void __user *outbuf;
  1359. size_t inlen;
  1360. size_t outlen;
  1361. };
  1362. struct ib_pd {
  1363. u32 local_dma_lkey;
  1364. u32 flags;
  1365. struct ib_device *device;
  1366. struct ib_uobject *uobject;
  1367. atomic_t usecnt; /* count all resources */
  1368. u32 unsafe_global_rkey;
  1369. /*
  1370. * Implementation details of the RDMA core, don't use in drivers:
  1371. */
  1372. struct ib_mr *__internal_mr;
  1373. struct rdma_restrack_entry res;
  1374. };
  1375. struct ib_xrcd {
  1376. struct ib_device *device;
  1377. atomic_t usecnt; /* count all exposed resources */
  1378. struct inode *inode;
  1379. struct rw_semaphore tgt_qps_rwsem;
  1380. struct xarray tgt_qps;
  1381. };
  1382. struct ib_ah {
  1383. struct ib_device *device;
  1384. struct ib_pd *pd;
  1385. struct ib_uobject *uobject;
  1386. const struct ib_gid_attr *sgid_attr;
  1387. enum rdma_ah_attr_type type;
  1388. };
  1389. typedef void (*ib_comp_handler)(struct ib_cq *cq, void *cq_context);
  1390. enum ib_poll_context {
  1391. IB_POLL_SOFTIRQ, /* poll from softirq context */
  1392. IB_POLL_WORKQUEUE, /* poll from workqueue */
  1393. IB_POLL_UNBOUND_WORKQUEUE, /* poll from unbound workqueue */
  1394. IB_POLL_LAST_POOL_TYPE = IB_POLL_UNBOUND_WORKQUEUE,
  1395. IB_POLL_DIRECT, /* caller context, no hw completions */
  1396. };
  1397. struct ib_cq {
  1398. struct ib_device *device;
  1399. struct ib_ucq_object *uobject;
  1400. ib_comp_handler comp_handler;
  1401. void (*event_handler)(struct ib_event *, void *);
  1402. void *cq_context;
  1403. int cqe;
  1404. unsigned int cqe_used;
  1405. atomic_t usecnt; /* count number of work queues */
  1406. enum ib_poll_context poll_ctx;
  1407. struct ib_wc *wc;
  1408. struct list_head pool_entry;
  1409. union {
  1410. struct irq_poll iop;
  1411. struct work_struct work;
  1412. };
  1413. struct workqueue_struct *comp_wq;
  1414. struct dim *dim;
  1415. /* updated only by trace points */
  1416. ktime_t timestamp;
  1417. u8 interrupt:1;
  1418. u8 shared:1;
  1419. unsigned int comp_vector;
  1420. /*
  1421. * Implementation details of the RDMA core, don't use in drivers:
  1422. */
  1423. struct rdma_restrack_entry res;
  1424. };
  1425. struct ib_srq {
  1426. struct ib_device *device;
  1427. struct ib_pd *pd;
  1428. struct ib_usrq_object *uobject;
  1429. void (*event_handler)(struct ib_event *, void *);
  1430. void *srq_context;
  1431. enum ib_srq_type srq_type;
  1432. atomic_t usecnt;
  1433. struct {
  1434. struct ib_cq *cq;
  1435. union {
  1436. struct {
  1437. struct ib_xrcd *xrcd;
  1438. u32 srq_num;
  1439. } xrc;
  1440. };
  1441. } ext;
  1442. };
  1443. enum ib_raw_packet_caps {
  1444. /* Strip cvlan from incoming packet and report it in the matching work
  1445. * completion is supported.
  1446. */
  1447. IB_RAW_PACKET_CAP_CVLAN_STRIPPING = (1 << 0),
  1448. /* Scatter FCS field of an incoming packet to host memory is supported.
  1449. */
  1450. IB_RAW_PACKET_CAP_SCATTER_FCS = (1 << 1),
  1451. /* Checksum offloads are supported (for both send and receive). */
  1452. IB_RAW_PACKET_CAP_IP_CSUM = (1 << 2),
  1453. /* When a packet is received for an RQ with no receive WQEs, the
  1454. * packet processing is delayed.
  1455. */
  1456. IB_RAW_PACKET_CAP_DELAY_DROP = (1 << 3),
  1457. };
  1458. enum ib_wq_type {
  1459. IB_WQT_RQ = IB_UVERBS_WQT_RQ,
  1460. };
  1461. enum ib_wq_state {
  1462. IB_WQS_RESET,
  1463. IB_WQS_RDY,
  1464. IB_WQS_ERR
  1465. };
  1466. struct ib_wq {
  1467. struct ib_device *device;
  1468. struct ib_uwq_object *uobject;
  1469. void *wq_context;
  1470. void (*event_handler)(struct ib_event *, void *);
  1471. struct ib_pd *pd;
  1472. struct ib_cq *cq;
  1473. u32 wq_num;
  1474. enum ib_wq_state state;
  1475. enum ib_wq_type wq_type;
  1476. atomic_t usecnt;
  1477. };
  1478. enum ib_wq_flags {
  1479. IB_WQ_FLAGS_CVLAN_STRIPPING = IB_UVERBS_WQ_FLAGS_CVLAN_STRIPPING,
  1480. IB_WQ_FLAGS_SCATTER_FCS = IB_UVERBS_WQ_FLAGS_SCATTER_FCS,
  1481. IB_WQ_FLAGS_DELAY_DROP = IB_UVERBS_WQ_FLAGS_DELAY_DROP,
  1482. IB_WQ_FLAGS_PCI_WRITE_END_PADDING =
  1483. IB_UVERBS_WQ_FLAGS_PCI_WRITE_END_PADDING,
  1484. };
  1485. struct ib_wq_init_attr {
  1486. void *wq_context;
  1487. enum ib_wq_type wq_type;
  1488. u32 max_wr;
  1489. u32 max_sge;
  1490. struct ib_cq *cq;
  1491. void (*event_handler)(struct ib_event *, void *);
  1492. u32 create_flags; /* Use enum ib_wq_flags */
  1493. };
  1494. enum ib_wq_attr_mask {
  1495. IB_WQ_STATE = 1 << 0,
  1496. IB_WQ_CUR_STATE = 1 << 1,
  1497. IB_WQ_FLAGS = 1 << 2,
  1498. };
  1499. struct ib_wq_attr {
  1500. enum ib_wq_state wq_state;
  1501. enum ib_wq_state curr_wq_state;
  1502. u32 flags; /* Use enum ib_wq_flags */
  1503. u32 flags_mask; /* Use enum ib_wq_flags */
  1504. };
  1505. struct ib_rwq_ind_table {
  1506. struct ib_device *device;
  1507. struct ib_uobject *uobject;
  1508. atomic_t usecnt;
  1509. u32 ind_tbl_num;
  1510. u32 log_ind_tbl_size;
  1511. struct ib_wq **ind_tbl;
  1512. };
  1513. struct ib_rwq_ind_table_init_attr {
  1514. u32 log_ind_tbl_size;
  1515. /* Each entry is a pointer to Receive Work Queue */
  1516. struct ib_wq **ind_tbl;
  1517. };
  1518. enum port_pkey_state {
  1519. IB_PORT_PKEY_NOT_VALID = 0,
  1520. IB_PORT_PKEY_VALID = 1,
  1521. IB_PORT_PKEY_LISTED = 2,
  1522. };
  1523. struct ib_qp_security;
  1524. struct ib_port_pkey {
  1525. enum port_pkey_state state;
  1526. u16 pkey_index;
  1527. u8 port_num;
  1528. struct list_head qp_list;
  1529. struct list_head to_error_list;
  1530. struct ib_qp_security *sec;
  1531. };
  1532. struct ib_ports_pkeys {
  1533. struct ib_port_pkey main;
  1534. struct ib_port_pkey alt;
  1535. };
  1536. struct ib_qp_security {
  1537. struct ib_qp *qp;
  1538. struct ib_device *dev;
  1539. /* Hold this mutex when changing port and pkey settings. */
  1540. struct mutex mutex;
  1541. struct ib_ports_pkeys *ports_pkeys;
  1542. /* A list of all open shared QP handles. Required to enforce security
  1543. * properly for all users of a shared QP.
  1544. */
  1545. struct list_head shared_qp_list;
  1546. void *security;
  1547. bool destroying;
  1548. atomic_t error_list_count;
  1549. struct completion error_complete;
  1550. int error_comps_pending;
  1551. };
  1552. /*
  1553. * @max_write_sge: Maximum SGE elements per RDMA WRITE request.
  1554. * @max_read_sge: Maximum SGE elements per RDMA READ request.
  1555. */
  1556. struct ib_qp {
  1557. struct ib_device *device;
  1558. struct ib_pd *pd;
  1559. struct ib_cq *send_cq;
  1560. struct ib_cq *recv_cq;
  1561. spinlock_t mr_lock;
  1562. int mrs_used;
  1563. struct list_head rdma_mrs;
  1564. struct list_head sig_mrs;
  1565. struct ib_srq *srq;
  1566. struct ib_xrcd *xrcd; /* XRC TGT QPs only */
  1567. struct list_head xrcd_list;
  1568. /* count times opened, mcast attaches, flow attaches */
  1569. atomic_t usecnt;
  1570. struct list_head open_list;
  1571. struct ib_qp *real_qp;
  1572. struct ib_uqp_object *uobject;
  1573. void (*event_handler)(struct ib_event *, void *);
  1574. void *qp_context;
  1575. /* sgid_attrs associated with the AV's */
  1576. const struct ib_gid_attr *av_sgid_attr;
  1577. const struct ib_gid_attr *alt_path_sgid_attr;
  1578. u32 qp_num;
  1579. u32 max_write_sge;
  1580. u32 max_read_sge;
  1581. enum ib_qp_type qp_type;
  1582. struct ib_rwq_ind_table *rwq_ind_tbl;
  1583. struct ib_qp_security *qp_sec;
  1584. u8 port;
  1585. bool integrity_en;
  1586. /*
  1587. * Implementation details of the RDMA core, don't use in drivers:
  1588. */
  1589. struct rdma_restrack_entry res;
  1590. /* The counter the qp is bind to */
  1591. struct rdma_counter *counter;
  1592. };
  1593. struct ib_dm {
  1594. struct ib_device *device;
  1595. u32 length;
  1596. u32 flags;
  1597. struct ib_uobject *uobject;
  1598. atomic_t usecnt;
  1599. };
  1600. struct ib_mr {
  1601. struct ib_device *device;
  1602. struct ib_pd *pd;
  1603. u32 lkey;
  1604. u32 rkey;
  1605. u64 iova;
  1606. u64 length;
  1607. unsigned int page_size;
  1608. enum ib_mr_type type;
  1609. bool need_inval;
  1610. union {
  1611. struct ib_uobject *uobject; /* user */
  1612. struct list_head qp_entry; /* FR */
  1613. };
  1614. struct ib_dm *dm;
  1615. struct ib_sig_attrs *sig_attrs; /* only for IB_MR_TYPE_INTEGRITY MRs */
  1616. /*
  1617. * Implementation details of the RDMA core, don't use in drivers:
  1618. */
  1619. struct rdma_restrack_entry res;
  1620. };
  1621. struct ib_mw {
  1622. struct ib_device *device;
  1623. struct ib_pd *pd;
  1624. struct ib_uobject *uobject;
  1625. u32 rkey;
  1626. enum ib_mw_type type;
  1627. };
  1628. /* Supported steering options */
  1629. enum ib_flow_attr_type {
  1630. /* steering according to rule specifications */
  1631. IB_FLOW_ATTR_NORMAL = 0x0,
  1632. /* default unicast and multicast rule -
  1633. * receive all Eth traffic which isn't steered to any QP
  1634. */
  1635. IB_FLOW_ATTR_ALL_DEFAULT = 0x1,
  1636. /* default multicast rule -
  1637. * receive all Eth multicast traffic which isn't steered to any QP
  1638. */
  1639. IB_FLOW_ATTR_MC_DEFAULT = 0x2,
  1640. /* sniffer rule - receive all port traffic */
  1641. IB_FLOW_ATTR_SNIFFER = 0x3
  1642. };
  1643. /* Supported steering header types */
  1644. enum ib_flow_spec_type {
  1645. /* L2 headers*/
  1646. IB_FLOW_SPEC_ETH = 0x20,
  1647. IB_FLOW_SPEC_IB = 0x22,
  1648. /* L3 header*/
  1649. IB_FLOW_SPEC_IPV4 = 0x30,
  1650. IB_FLOW_SPEC_IPV6 = 0x31,
  1651. IB_FLOW_SPEC_ESP = 0x34,
  1652. /* L4 headers*/
  1653. IB_FLOW_SPEC_TCP = 0x40,
  1654. IB_FLOW_SPEC_UDP = 0x41,
  1655. IB_FLOW_SPEC_VXLAN_TUNNEL = 0x50,
  1656. IB_FLOW_SPEC_GRE = 0x51,
  1657. IB_FLOW_SPEC_MPLS = 0x60,
  1658. IB_FLOW_SPEC_INNER = 0x100,
  1659. /* Actions */
  1660. IB_FLOW_SPEC_ACTION_TAG = 0x1000,
  1661. IB_FLOW_SPEC_ACTION_DROP = 0x1001,
  1662. IB_FLOW_SPEC_ACTION_HANDLE = 0x1002,
  1663. IB_FLOW_SPEC_ACTION_COUNT = 0x1003,
  1664. };
  1665. #define IB_FLOW_SPEC_LAYER_MASK 0xF0
  1666. #define IB_FLOW_SPEC_SUPPORT_LAYERS 10
  1667. enum ib_flow_flags {
  1668. IB_FLOW_ATTR_FLAGS_DONT_TRAP = 1UL << 1, /* Continue match, no steal */
  1669. IB_FLOW_ATTR_FLAGS_EGRESS = 1UL << 2, /* Egress flow */
  1670. IB_FLOW_ATTR_FLAGS_RESERVED = 1UL << 3 /* Must be last */
  1671. };
  1672. struct ib_flow_eth_filter {
  1673. u8 dst_mac[6];
  1674. u8 src_mac[6];
  1675. __be16 ether_type;
  1676. __be16 vlan_tag;
  1677. /* Must be last */
  1678. u8 real_sz[];
  1679. };
  1680. struct ib_flow_spec_eth {
  1681. u32 type;
  1682. u16 size;
  1683. struct ib_flow_eth_filter val;
  1684. struct ib_flow_eth_filter mask;
  1685. };
  1686. struct ib_flow_ib_filter {
  1687. __be16 dlid;
  1688. __u8 sl;
  1689. /* Must be last */
  1690. u8 real_sz[];
  1691. };
  1692. struct ib_flow_spec_ib {
  1693. u32 type;
  1694. u16 size;
  1695. struct ib_flow_ib_filter val;
  1696. struct ib_flow_ib_filter mask;
  1697. };
  1698. /* IPv4 header flags */
  1699. enum ib_ipv4_flags {
  1700. IB_IPV4_DONT_FRAG = 0x2, /* Don't enable packet fragmentation */
  1701. IB_IPV4_MORE_FRAG = 0X4 /* For All fragmented packets except the
  1702. last have this flag set */
  1703. };
  1704. struct ib_flow_ipv4_filter {
  1705. __be32 src_ip;
  1706. __be32 dst_ip;
  1707. u8 proto;
  1708. u8 tos;
  1709. u8 ttl;
  1710. u8 flags;
  1711. /* Must be last */
  1712. u8 real_sz[];
  1713. };
  1714. struct ib_flow_spec_ipv4 {
  1715. u32 type;
  1716. u16 size;
  1717. struct ib_flow_ipv4_filter val;
  1718. struct ib_flow_ipv4_filter mask;
  1719. };
  1720. struct ib_flow_ipv6_filter {
  1721. u8 src_ip[16];
  1722. u8 dst_ip[16];
  1723. __be32 flow_label;
  1724. u8 next_hdr;
  1725. u8 traffic_class;
  1726. u8 hop_limit;
  1727. /* Must be last */
  1728. u8 real_sz[];
  1729. };
  1730. struct ib_flow_spec_ipv6 {
  1731. u32 type;
  1732. u16 size;
  1733. struct ib_flow_ipv6_filter val;
  1734. struct ib_flow_ipv6_filter mask;
  1735. };
  1736. struct ib_flow_tcp_udp_filter {
  1737. __be16 dst_port;
  1738. __be16 src_port;
  1739. /* Must be last */
  1740. u8 real_sz[];
  1741. };
  1742. struct ib_flow_spec_tcp_udp {
  1743. u32 type;
  1744. u16 size;
  1745. struct ib_flow_tcp_udp_filter val;
  1746. struct ib_flow_tcp_udp_filter mask;
  1747. };
  1748. struct ib_flow_tunnel_filter {
  1749. __be32 tunnel_id;
  1750. u8 real_sz[];
  1751. };
  1752. /* ib_flow_spec_tunnel describes the Vxlan tunnel
  1753. * the tunnel_id from val has the vni value
  1754. */
  1755. struct ib_flow_spec_tunnel {
  1756. u32 type;
  1757. u16 size;
  1758. struct ib_flow_tunnel_filter val;
  1759. struct ib_flow_tunnel_filter mask;
  1760. };
  1761. struct ib_flow_esp_filter {
  1762. __be32 spi;
  1763. __be32 seq;
  1764. /* Must be last */
  1765. u8 real_sz[];
  1766. };
  1767. struct ib_flow_spec_esp {
  1768. u32 type;
  1769. u16 size;
  1770. struct ib_flow_esp_filter val;
  1771. struct ib_flow_esp_filter mask;
  1772. };
  1773. struct ib_flow_gre_filter {
  1774. __be16 c_ks_res0_ver;
  1775. __be16 protocol;
  1776. __be32 key;
  1777. /* Must be last */
  1778. u8 real_sz[];
  1779. };
  1780. struct ib_flow_spec_gre {
  1781. u32 type;
  1782. u16 size;
  1783. struct ib_flow_gre_filter val;
  1784. struct ib_flow_gre_filter mask;
  1785. };
  1786. struct ib_flow_mpls_filter {
  1787. __be32 tag;
  1788. /* Must be last */
  1789. u8 real_sz[];
  1790. };
  1791. struct ib_flow_spec_mpls {
  1792. u32 type;
  1793. u16 size;
  1794. struct ib_flow_mpls_filter val;
  1795. struct ib_flow_mpls_filter mask;
  1796. };
  1797. struct ib_flow_spec_action_tag {
  1798. enum ib_flow_spec_type type;
  1799. u16 size;
  1800. u32 tag_id;
  1801. };
  1802. struct ib_flow_spec_action_drop {
  1803. enum ib_flow_spec_type type;
  1804. u16 size;
  1805. };
  1806. struct ib_flow_spec_action_handle {
  1807. enum ib_flow_spec_type type;
  1808. u16 size;
  1809. struct ib_flow_action *act;
  1810. };
  1811. enum ib_counters_description {
  1812. IB_COUNTER_PACKETS,
  1813. IB_COUNTER_BYTES,
  1814. };
  1815. struct ib_flow_spec_action_count {
  1816. enum ib_flow_spec_type type;
  1817. u16 size;
  1818. struct ib_counters *counters;
  1819. };
  1820. union ib_flow_spec {
  1821. struct {
  1822. u32 type;
  1823. u16 size;
  1824. };
  1825. struct ib_flow_spec_eth eth;
  1826. struct ib_flow_spec_ib ib;
  1827. struct ib_flow_spec_ipv4 ipv4;
  1828. struct ib_flow_spec_tcp_udp tcp_udp;
  1829. struct ib_flow_spec_ipv6 ipv6;
  1830. struct ib_flow_spec_tunnel tunnel;
  1831. struct ib_flow_spec_esp esp;
  1832. struct ib_flow_spec_gre gre;
  1833. struct ib_flow_spec_mpls mpls;
  1834. struct ib_flow_spec_action_tag flow_tag;
  1835. struct ib_flow_spec_action_drop drop;
  1836. struct ib_flow_spec_action_handle action;
  1837. struct ib_flow_spec_action_count flow_count;
  1838. };
  1839. struct ib_flow_attr {
  1840. enum ib_flow_attr_type type;
  1841. u16 size;
  1842. u16 priority;
  1843. u32 flags;
  1844. u8 num_of_specs;
  1845. u8 port;
  1846. union ib_flow_spec flows[];
  1847. };
  1848. struct ib_flow {
  1849. struct ib_qp *qp;
  1850. struct ib_device *device;
  1851. struct ib_uobject *uobject;
  1852. };
  1853. enum ib_flow_action_type {
  1854. IB_FLOW_ACTION_UNSPECIFIED,
  1855. IB_FLOW_ACTION_ESP = 1,
  1856. };
  1857. struct ib_flow_action_attrs_esp_keymats {
  1858. enum ib_uverbs_flow_action_esp_keymat protocol;
  1859. union {
  1860. struct ib_uverbs_flow_action_esp_keymat_aes_gcm aes_gcm;
  1861. } keymat;
  1862. };
  1863. struct ib_flow_action_attrs_esp_replays {
  1864. enum ib_uverbs_flow_action_esp_replay protocol;
  1865. union {
  1866. struct ib_uverbs_flow_action_esp_replay_bmp bmp;
  1867. } replay;
  1868. };
  1869. enum ib_flow_action_attrs_esp_flags {
  1870. /* All user-space flags at the top: Use enum ib_uverbs_flow_action_esp_flags
  1871. * This is done in order to share the same flags between user-space and
  1872. * kernel and spare an unnecessary translation.
  1873. */
  1874. /* Kernel flags */
  1875. IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED = 1ULL << 32,
  1876. IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS = 1ULL << 33,
  1877. };
  1878. struct ib_flow_spec_list {
  1879. struct ib_flow_spec_list *next;
  1880. union ib_flow_spec spec;
  1881. };
  1882. struct ib_flow_action_attrs_esp {
  1883. struct ib_flow_action_attrs_esp_keymats *keymat;
  1884. struct ib_flow_action_attrs_esp_replays *replay;
  1885. struct ib_flow_spec_list *encap;
  1886. /* Used only if IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED is enabled.
  1887. * Value of 0 is a valid value.
  1888. */
  1889. u32 esn;
  1890. u32 spi;
  1891. u32 seq;
  1892. u32 tfc_pad;
  1893. /* Use enum ib_flow_action_attrs_esp_flags */
  1894. u64 flags;
  1895. u64 hard_limit_pkts;
  1896. };
  1897. struct ib_flow_action {
  1898. struct ib_device *device;
  1899. struct ib_uobject *uobject;
  1900. enum ib_flow_action_type type;
  1901. atomic_t usecnt;
  1902. };
  1903. struct ib_mad;
  1904. struct ib_grh;
  1905. enum ib_process_mad_flags {
  1906. IB_MAD_IGNORE_MKEY = 1,
  1907. IB_MAD_IGNORE_BKEY = 2,
  1908. IB_MAD_IGNORE_ALL = IB_MAD_IGNORE_MKEY | IB_MAD_IGNORE_BKEY
  1909. };
  1910. enum ib_mad_result {
  1911. IB_MAD_RESULT_FAILURE = 0, /* (!SUCCESS is the important flag) */
  1912. IB_MAD_RESULT_SUCCESS = 1 << 0, /* MAD was successfully processed */
  1913. IB_MAD_RESULT_REPLY = 1 << 1, /* Reply packet needs to be sent */
  1914. IB_MAD_RESULT_CONSUMED = 1 << 2 /* Packet consumed: stop processing */
  1915. };
  1916. struct ib_port_cache {
  1917. u64 subnet_prefix;
  1918. struct ib_pkey_cache *pkey;
  1919. struct ib_gid_table *gid;
  1920. u8 lmc;
  1921. enum ib_port_state port_state;
  1922. };
  1923. struct ib_port_immutable {
  1924. int pkey_tbl_len;
  1925. int gid_tbl_len;
  1926. u32 core_cap_flags;
  1927. u32 max_mad_size;
  1928. };
  1929. struct ib_port_data {
  1930. struct ib_device *ib_dev;
  1931. struct ib_port_immutable immutable;
  1932. spinlock_t pkey_list_lock;
  1933. struct list_head pkey_list;
  1934. struct ib_port_cache cache;
  1935. spinlock_t netdev_lock;
  1936. struct net_device __rcu *netdev;
  1937. struct hlist_node ndev_hash_link;
  1938. struct rdma_port_counter port_counter;
  1939. struct rdma_hw_stats *hw_stats;
  1940. };
  1941. /* rdma netdev type - specifies protocol type */
  1942. enum rdma_netdev_t {
  1943. RDMA_NETDEV_OPA_VNIC,
  1944. RDMA_NETDEV_IPOIB,
  1945. };
  1946. /**
  1947. * struct rdma_netdev - rdma netdev
  1948. * For cases where netstack interfacing is required.
  1949. */
  1950. struct rdma_netdev {
  1951. void *clnt_priv;
  1952. struct ib_device *hca;
  1953. u8 port_num;
  1954. int mtu;
  1955. /*
  1956. * cleanup function must be specified.
  1957. * FIXME: This is only used for OPA_VNIC and that usage should be
  1958. * removed too.
  1959. */
  1960. void (*free_rdma_netdev)(struct net_device *netdev);
  1961. /* control functions */
  1962. void (*set_id)(struct net_device *netdev, int id);
  1963. /* send packet */
  1964. int (*send)(struct net_device *dev, struct sk_buff *skb,
  1965. struct ib_ah *address, u32 dqpn);
  1966. /* multicast */
  1967. int (*attach_mcast)(struct net_device *dev, struct ib_device *hca,
  1968. union ib_gid *gid, u16 mlid,
  1969. int set_qkey, u32 qkey);
  1970. int (*detach_mcast)(struct net_device *dev, struct ib_device *hca,
  1971. union ib_gid *gid, u16 mlid);
  1972. };
  1973. struct rdma_netdev_alloc_params {
  1974. size_t sizeof_priv;
  1975. unsigned int txqs;
  1976. unsigned int rxqs;
  1977. void *param;
  1978. int (*initialize_rdma_netdev)(struct ib_device *device, u8 port_num,
  1979. struct net_device *netdev, void *param);
  1980. };
  1981. struct ib_odp_counters {
  1982. atomic64_t faults;
  1983. atomic64_t invalidations;
  1984. atomic64_t prefetch;
  1985. };
  1986. struct ib_counters {
  1987. struct ib_device *device;
  1988. struct ib_uobject *uobject;
  1989. /* num of objects attached */
  1990. atomic_t usecnt;
  1991. };
  1992. struct ib_counters_read_attr {
  1993. u64 *counters_buff;
  1994. u32 ncounters;
  1995. u32 flags; /* use enum ib_read_counters_flags */
  1996. };
  1997. struct uverbs_attr_bundle;
  1998. struct iw_cm_id;
  1999. struct iw_cm_conn_param;
  2000. #define INIT_RDMA_OBJ_SIZE(ib_struct, drv_struct, member) \
  2001. .size_##ib_struct = \
  2002. (sizeof(struct drv_struct) + \
  2003. BUILD_BUG_ON_ZERO(offsetof(struct drv_struct, member)) + \
  2004. BUILD_BUG_ON_ZERO( \
  2005. !__same_type(((struct drv_struct *)NULL)->member, \
  2006. struct ib_struct)))
  2007. #define rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, gfp) \
  2008. ((struct ib_type *)kzalloc(ib_dev->ops.size_##ib_type, gfp))
  2009. #define rdma_zalloc_drv_obj(ib_dev, ib_type) \
  2010. rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, GFP_KERNEL)
  2011. #define DECLARE_RDMA_OBJ_SIZE(ib_struct) size_t size_##ib_struct
  2012. struct rdma_user_mmap_entry {
  2013. struct kref ref;
  2014. struct ib_ucontext *ucontext;
  2015. unsigned long start_pgoff;
  2016. size_t npages;
  2017. bool driver_removed;
  2018. };
  2019. /* Return the offset (in bytes) the user should pass to libc's mmap() */
  2020. static inline u64
  2021. rdma_user_mmap_get_offset(const struct rdma_user_mmap_entry *entry)
  2022. {
  2023. return (u64)entry->start_pgoff << PAGE_SHIFT;
  2024. }
  2025. /**
  2026. * struct ib_device_ops - InfiniBand device operations
  2027. * This structure defines all the InfiniBand device operations, providers will
  2028. * need to define the supported operations, otherwise they will be set to null.
  2029. */
  2030. struct ib_device_ops {
  2031. struct module *owner;
  2032. enum rdma_driver_id driver_id;
  2033. u32 uverbs_abi_ver;
  2034. unsigned int uverbs_no_driver_id_binding:1;
  2035. int (*post_send)(struct ib_qp *qp, const struct ib_send_wr *send_wr,
  2036. const struct ib_send_wr **bad_send_wr);
  2037. int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,
  2038. const struct ib_recv_wr **bad_recv_wr);
  2039. void (*drain_rq)(struct ib_qp *qp);
  2040. void (*drain_sq)(struct ib_qp *qp);
  2041. int (*poll_cq)(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
  2042. int (*peek_cq)(struct ib_cq *cq, int wc_cnt);
  2043. int (*req_notify_cq)(struct ib_cq *cq, enum ib_cq_notify_flags flags);
  2044. int (*req_ncomp_notif)(struct ib_cq *cq, int wc_cnt);
  2045. int (*post_srq_recv)(struct ib_srq *srq,
  2046. const struct ib_recv_wr *recv_wr,
  2047. const struct ib_recv_wr **bad_recv_wr);
  2048. int (*process_mad)(struct ib_device *device, int process_mad_flags,
  2049. u8 port_num, const struct ib_wc *in_wc,
  2050. const struct ib_grh *in_grh,
  2051. const struct ib_mad *in_mad, struct ib_mad *out_mad,
  2052. size_t *out_mad_size, u16 *out_mad_pkey_index);
  2053. int (*query_device)(struct ib_device *device,
  2054. struct ib_device_attr *device_attr,
  2055. struct ib_udata *udata);
  2056. int (*modify_device)(struct ib_device *device, int device_modify_mask,
  2057. struct ib_device_modify *device_modify);
  2058. void (*get_dev_fw_str)(struct ib_device *device, char *str);
  2059. const struct cpumask *(*get_vector_affinity)(struct ib_device *ibdev,
  2060. int comp_vector);
  2061. int (*query_port)(struct ib_device *device, u8 port_num,
  2062. struct ib_port_attr *port_attr);
  2063. int (*modify_port)(struct ib_device *device, u8 port_num,
  2064. int port_modify_mask,
  2065. struct ib_port_modify *port_modify);
  2066. /**
  2067. * The following mandatory functions are used only at device
  2068. * registration. Keep functions such as these at the end of this
  2069. * structure to avoid cache line misses when accessing struct ib_device
  2070. * in fast paths.
  2071. */
  2072. int (*get_port_immutable)(struct ib_device *device, u8 port_num,
  2073. struct ib_port_immutable *immutable);
  2074. enum rdma_link_layer (*get_link_layer)(struct ib_device *device,
  2075. u8 port_num);
  2076. /**
  2077. * When calling get_netdev, the HW vendor's driver should return the
  2078. * net device of device @device at port @port_num or NULL if such
  2079. * a net device doesn't exist. The vendor driver should call dev_hold
  2080. * on this net device. The HW vendor's device driver must guarantee
  2081. * that this function returns NULL before the net device has finished
  2082. * NETDEV_UNREGISTER state.
  2083. */
  2084. struct net_device *(*get_netdev)(struct ib_device *device, u8 port_num);
  2085. /**
  2086. * rdma netdev operation
  2087. *
  2088. * Driver implementing alloc_rdma_netdev or rdma_netdev_get_params
  2089. * must return -EOPNOTSUPP if it doesn't support the specified type.
  2090. */
  2091. struct net_device *(*alloc_rdma_netdev)(
  2092. struct ib_device *device, u8 port_num, enum rdma_netdev_t type,
  2093. const char *name, unsigned char name_assign_type,
  2094. void (*setup)(struct net_device *));
  2095. int (*rdma_netdev_get_params)(struct ib_device *device, u8 port_num,
  2096. enum rdma_netdev_t type,
  2097. struct rdma_netdev_alloc_params *params);
  2098. /**
  2099. * query_gid should be return GID value for @device, when @port_num
  2100. * link layer is either IB or iWarp. It is no-op if @port_num port
  2101. * is RoCE link layer.
  2102. */
  2103. int (*query_gid)(struct ib_device *device, u8 port_num, int index,
  2104. union ib_gid *gid);
  2105. /**
  2106. * When calling add_gid, the HW vendor's driver should add the gid
  2107. * of device of port at gid index available at @attr. Meta-info of
  2108. * that gid (for example, the network device related to this gid) is
  2109. * available at @attr. @context allows the HW vendor driver to store
  2110. * extra information together with a GID entry. The HW vendor driver may
  2111. * allocate memory to contain this information and store it in @context
  2112. * when a new GID entry is written to. Params are consistent until the
  2113. * next call of add_gid or delete_gid. The function should return 0 on
  2114. * success or error otherwise. The function could be called
  2115. * concurrently for different ports. This function is only called when
  2116. * roce_gid_table is used.
  2117. */
  2118. int (*add_gid)(const struct ib_gid_attr *attr, void **context);
  2119. /**
  2120. * When calling del_gid, the HW vendor's driver should delete the
  2121. * gid of device @device at gid index gid_index of port port_num
  2122. * available in @attr.
  2123. * Upon the deletion of a GID entry, the HW vendor must free any
  2124. * allocated memory. The caller will clear @context afterwards.
  2125. * This function is only called when roce_gid_table is used.
  2126. */
  2127. int (*del_gid)(const struct ib_gid_attr *attr, void **context);
  2128. int (*query_pkey)(struct ib_device *device, u8 port_num, u16 index,
  2129. u16 *pkey);
  2130. int (*alloc_ucontext)(struct ib_ucontext *context,
  2131. struct ib_udata *udata);
  2132. void (*dealloc_ucontext)(struct ib_ucontext *context);
  2133. int (*mmap)(struct ib_ucontext *context, struct vm_area_struct *vma);
  2134. /**
  2135. * This will be called once refcount of an entry in mmap_xa reaches
  2136. * zero. The type of the memory that was mapped may differ between
  2137. * entries and is opaque to the rdma_user_mmap interface.
  2138. * Therefore needs to be implemented by the driver in mmap_free.
  2139. */
  2140. void (*mmap_free)(struct rdma_user_mmap_entry *entry);
  2141. void (*disassociate_ucontext)(struct ib_ucontext *ibcontext);
  2142. int (*alloc_pd)(struct ib_pd *pd, struct ib_udata *udata);
  2143. int (*dealloc_pd)(struct ib_pd *pd, struct ib_udata *udata);
  2144. int (*create_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr,
  2145. struct ib_udata *udata);
  2146. int (*modify_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
  2147. int (*query_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
  2148. int (*destroy_ah)(struct ib_ah *ah, u32 flags);
  2149. int (*create_srq)(struct ib_srq *srq,
  2150. struct ib_srq_init_attr *srq_init_attr,
  2151. struct ib_udata *udata);
  2152. int (*modify_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr,
  2153. enum ib_srq_attr_mask srq_attr_mask,
  2154. struct ib_udata *udata);
  2155. int (*query_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
  2156. int (*destroy_srq)(struct ib_srq *srq, struct ib_udata *udata);
  2157. struct ib_qp *(*create_qp)(struct ib_pd *pd,
  2158. struct ib_qp_init_attr *qp_init_attr,
  2159. struct ib_udata *udata);
  2160. int (*modify_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
  2161. int qp_attr_mask, struct ib_udata *udata);
  2162. int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
  2163. int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
  2164. int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata);
  2165. int (*create_cq)(struct ib_cq *cq, const struct ib_cq_init_attr *attr,
  2166. struct ib_udata *udata);
  2167. int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
  2168. int (*destroy_cq)(struct ib_cq *cq, struct ib_udata *udata);
  2169. int (*resize_cq)(struct ib_cq *cq, int cqe, struct ib_udata *udata);
  2170. struct ib_mr *(*get_dma_mr)(struct ib_pd *pd, int mr_access_flags);
  2171. struct ib_mr *(*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length,
  2172. u64 virt_addr, int mr_access_flags,
  2173. struct ib_udata *udata);
  2174. int (*rereg_user_mr)(struct ib_mr *mr, int flags, u64 start, u64 length,
  2175. u64 virt_addr, int mr_access_flags,
  2176. struct ib_pd *pd, struct ib_udata *udata);
  2177. int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata);
  2178. struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type,
  2179. u32 max_num_sg);
  2180. struct ib_mr *(*alloc_mr_integrity)(struct ib_pd *pd,
  2181. u32 max_num_data_sg,
  2182. u32 max_num_meta_sg);
  2183. int (*advise_mr)(struct ib_pd *pd,
  2184. enum ib_uverbs_advise_mr_advice advice, u32 flags,
  2185. struct ib_sge *sg_list, u32 num_sge,
  2186. struct uverbs_attr_bundle *attrs);
  2187. int (*map_mr_sg)(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
  2188. unsigned int *sg_offset);
  2189. int (*check_mr_status)(struct ib_mr *mr, u32 check_mask,
  2190. struct ib_mr_status *mr_status);
  2191. int (*alloc_mw)(struct ib_mw *mw, struct ib_udata *udata);
  2192. int (*dealloc_mw)(struct ib_mw *mw);
  2193. int (*attach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
  2194. int (*detach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
  2195. int (*alloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
  2196. int (*dealloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
  2197. struct ib_flow *(*create_flow)(struct ib_qp *qp,
  2198. struct ib_flow_attr *flow_attr,
  2199. struct ib_udata *udata);
  2200. int (*destroy_flow)(struct ib_flow *flow_id);
  2201. struct ib_flow_action *(*create_flow_action_esp)(
  2202. struct ib_device *device,
  2203. const struct ib_flow_action_attrs_esp *attr,
  2204. struct uverbs_attr_bundle *attrs);
  2205. int (*destroy_flow_action)(struct ib_flow_action *action);
  2206. int (*modify_flow_action_esp)(
  2207. struct ib_flow_action *action,
  2208. const struct ib_flow_action_attrs_esp *attr,
  2209. struct uverbs_attr_bundle *attrs);
  2210. int (*set_vf_link_state)(struct ib_device *device, int vf, u8 port,
  2211. int state);
  2212. int (*get_vf_config)(struct ib_device *device, int vf, u8 port,
  2213. struct ifla_vf_info *ivf);
  2214. int (*get_vf_stats)(struct ib_device *device, int vf, u8 port,
  2215. struct ifla_vf_stats *stats);
  2216. int (*get_vf_guid)(struct ib_device *device, int vf, u8 port,
  2217. struct ifla_vf_guid *node_guid,
  2218. struct ifla_vf_guid *port_guid);
  2219. int (*set_vf_guid)(struct ib_device *device, int vf, u8 port, u64 guid,
  2220. int type);
  2221. struct ib_wq *(*create_wq)(struct ib_pd *pd,
  2222. struct ib_wq_init_attr *init_attr,
  2223. struct ib_udata *udata);
  2224. int (*destroy_wq)(struct ib_wq *wq, struct ib_udata *udata);
  2225. int (*modify_wq)(struct ib_wq *wq, struct ib_wq_attr *attr,
  2226. u32 wq_attr_mask, struct ib_udata *udata);
  2227. int (*create_rwq_ind_table)(struct ib_rwq_ind_table *ib_rwq_ind_table,
  2228. struct ib_rwq_ind_table_init_attr *init_attr,
  2229. struct ib_udata *udata);
  2230. int (*destroy_rwq_ind_table)(struct ib_rwq_ind_table *wq_ind_table);
  2231. struct ib_dm *(*alloc_dm)(struct ib_device *device,
  2232. struct ib_ucontext *context,
  2233. struct ib_dm_alloc_attr *attr,
  2234. struct uverbs_attr_bundle *attrs);
  2235. int (*dealloc_dm)(struct ib_dm *dm, struct uverbs_attr_bundle *attrs);
  2236. struct ib_mr *(*reg_dm_mr)(struct ib_pd *pd, struct ib_dm *dm,
  2237. struct ib_dm_mr_attr *attr,
  2238. struct uverbs_attr_bundle *attrs);
  2239. int (*create_counters)(struct ib_counters *counters,
  2240. struct uverbs_attr_bundle *attrs);
  2241. int (*destroy_counters)(struct ib_counters *counters);
  2242. int (*read_counters)(struct ib_counters *counters,
  2243. struct ib_counters_read_attr *counters_read_attr,
  2244. struct uverbs_attr_bundle *attrs);
  2245. int (*map_mr_sg_pi)(struct ib_mr *mr, struct scatterlist *data_sg,
  2246. int data_sg_nents, unsigned int *data_sg_offset,
  2247. struct scatterlist *meta_sg, int meta_sg_nents,
  2248. unsigned int *meta_sg_offset);
  2249. /**
  2250. * alloc_hw_stats - Allocate a struct rdma_hw_stats and fill in the
  2251. * driver initialized data. The struct is kfree()'ed by the sysfs
  2252. * core when the device is removed. A lifespan of -1 in the return
  2253. * struct tells the core to set a default lifespan.
  2254. */
  2255. struct rdma_hw_stats *(*alloc_hw_stats)(struct ib_device *device,
  2256. u8 port_num);
  2257. /**
  2258. * get_hw_stats - Fill in the counter value(s) in the stats struct.
  2259. * @index - The index in the value array we wish to have updated, or
  2260. * num_counters if we want all stats updated
  2261. * Return codes -
  2262. * < 0 - Error, no counters updated
  2263. * index - Updated the single counter pointed to by index
  2264. * num_counters - Updated all counters (will reset the timestamp
  2265. * and prevent further calls for lifespan milliseconds)
  2266. * Drivers are allowed to update all counters in leiu of just the
  2267. * one given in index at their option
  2268. */
  2269. int (*get_hw_stats)(struct ib_device *device,
  2270. struct rdma_hw_stats *stats, u8 port, int index);
  2271. /*
  2272. * This function is called once for each port when a ib device is
  2273. * registered.
  2274. */
  2275. int (*init_port)(struct ib_device *device, u8 port_num,
  2276. struct kobject *port_sysfs);
  2277. /**
  2278. * Allows rdma drivers to add their own restrack attributes.
  2279. */
  2280. int (*fill_res_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);
  2281. int (*fill_res_mr_entry_raw)(struct sk_buff *msg, struct ib_mr *ibmr);
  2282. int (*fill_res_cq_entry)(struct sk_buff *msg, struct ib_cq *ibcq);
  2283. int (*fill_res_cq_entry_raw)(struct sk_buff *msg, struct ib_cq *ibcq);
  2284. int (*fill_res_qp_entry)(struct sk_buff *msg, struct ib_qp *ibqp);
  2285. int (*fill_res_qp_entry_raw)(struct sk_buff *msg, struct ib_qp *ibqp);
  2286. int (*fill_res_cm_id_entry)(struct sk_buff *msg, struct rdma_cm_id *id);
  2287. /* Device lifecycle callbacks */
  2288. /*
  2289. * Called after the device becomes registered, before clients are
  2290. * attached
  2291. */
  2292. int (*enable_driver)(struct ib_device *dev);
  2293. /*
  2294. * This is called as part of ib_dealloc_device().
  2295. */
  2296. void (*dealloc_driver)(struct ib_device *dev);
  2297. /* iWarp CM callbacks */
  2298. void (*iw_add_ref)(struct ib_qp *qp);
  2299. void (*iw_rem_ref)(struct ib_qp *qp);
  2300. struct ib_qp *(*iw_get_qp)(struct ib_device *device, int qpn);
  2301. int (*iw_connect)(struct iw_cm_id *cm_id,
  2302. struct iw_cm_conn_param *conn_param);
  2303. int (*iw_accept)(struct iw_cm_id *cm_id,
  2304. struct iw_cm_conn_param *conn_param);
  2305. int (*iw_reject)(struct iw_cm_id *cm_id, const void *pdata,
  2306. u8 pdata_len);
  2307. int (*iw_create_listen)(struct iw_cm_id *cm_id, int backlog);
  2308. int (*iw_destroy_listen)(struct iw_cm_id *cm_id);
  2309. /**
  2310. * counter_bind_qp - Bind a QP to a counter.
  2311. * @counter - The counter to be bound. If counter->id is zero then
  2312. * the driver needs to allocate a new counter and set counter->id
  2313. */
  2314. int (*counter_bind_qp)(struct rdma_counter *counter, struct ib_qp *qp);
  2315. /**
  2316. * counter_unbind_qp - Unbind the qp from the dynamically-allocated
  2317. * counter and bind it onto the default one
  2318. */
  2319. int (*counter_unbind_qp)(struct ib_qp *qp);
  2320. /**
  2321. * counter_dealloc -De-allocate the hw counter
  2322. */
  2323. int (*counter_dealloc)(struct rdma_counter *counter);
  2324. /**
  2325. * counter_alloc_stats - Allocate a struct rdma_hw_stats and fill in
  2326. * the driver initialized data.
  2327. */
  2328. struct rdma_hw_stats *(*counter_alloc_stats)(
  2329. struct rdma_counter *counter);
  2330. /**
  2331. * counter_update_stats - Query the stats value of this counter
  2332. */
  2333. int (*counter_update_stats)(struct rdma_counter *counter);
  2334. /**
  2335. * Allows rdma drivers to add their own restrack attributes
  2336. * dumped via 'rdma stat' iproute2 command.
  2337. */
  2338. int (*fill_stat_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);
  2339. /* query driver for its ucontext properties */
  2340. int (*query_ucontext)(struct ib_ucontext *context,
  2341. struct uverbs_attr_bundle *attrs);
  2342. DECLARE_RDMA_OBJ_SIZE(ib_ah);
  2343. DECLARE_RDMA_OBJ_SIZE(ib_counters);
  2344. DECLARE_RDMA_OBJ_SIZE(ib_cq);
  2345. DECLARE_RDMA_OBJ_SIZE(ib_mw);
  2346. DECLARE_RDMA_OBJ_SIZE(ib_pd);
  2347. DECLARE_RDMA_OBJ_SIZE(ib_rwq_ind_table);
  2348. DECLARE_RDMA_OBJ_SIZE(ib_srq);
  2349. DECLARE_RDMA_OBJ_SIZE(ib_ucontext);
  2350. DECLARE_RDMA_OBJ_SIZE(ib_xrcd);
  2351. };
  2352. struct ib_core_device {
  2353. /* device must be the first element in structure until,
  2354. * union of ib_core_device and device exists in ib_device.
  2355. */
  2356. struct device dev;
  2357. possible_net_t rdma_net;
  2358. struct kobject *ports_kobj;
  2359. struct list_head port_list;
  2360. struct ib_device *owner; /* reach back to owner ib_device */
  2361. };
  2362. struct rdma_restrack_root;
  2363. struct ib_device {
  2364. /* Do not access @dma_device directly from ULP nor from HW drivers. */
  2365. struct device *dma_device;
  2366. struct ib_device_ops ops;
  2367. char name[IB_DEVICE_NAME_MAX];
  2368. struct rcu_head rcu_head;
  2369. struct list_head event_handler_list;
  2370. /* Protects event_handler_list */
  2371. struct rw_semaphore event_handler_rwsem;
  2372. /* Protects QP's event_handler calls and open_qp list */
  2373. spinlock_t qp_open_list_lock;
  2374. struct rw_semaphore client_data_rwsem;
  2375. struct xarray client_data;
  2376. struct mutex unregistration_lock;
  2377. /* Synchronize GID, Pkey cache entries, subnet prefix, LMC */
  2378. rwlock_t cache_lock;
  2379. /**
  2380. * port_data is indexed by port number
  2381. */
  2382. struct ib_port_data *port_data;
  2383. int num_comp_vectors;
  2384. union {
  2385. struct device dev;
  2386. struct ib_core_device coredev;
  2387. };
  2388. /* First group for device attributes,
  2389. * Second group for driver provided attributes (optional).
  2390. * It is NULL terminated array.
  2391. */
  2392. const struct attribute_group *groups[3];
  2393. u64 uverbs_cmd_mask;
  2394. u64 uverbs_ex_cmd_mask;
  2395. char node_desc[IB_DEVICE_NODE_DESC_MAX];
  2396. __be64 node_guid;
  2397. u32 local_dma_lkey;
  2398. u16 is_switch:1;
  2399. /* Indicates kernel verbs support, should not be used in drivers */
  2400. u16 kverbs_provider:1;
  2401. /* CQ adaptive moderation (RDMA DIM) */
  2402. u16 use_cq_dim:1;
  2403. u8 node_type;
  2404. u8 phys_port_cnt;
  2405. struct ib_device_attr attrs;
  2406. struct attribute_group *hw_stats_ag;
  2407. struct rdma_hw_stats *hw_stats;
  2408. #ifdef CONFIG_CGROUP_RDMA
  2409. struct rdmacg_device cg_device;
  2410. #endif
  2411. u32 index;
  2412. spinlock_t cq_pools_lock;
  2413. struct list_head cq_pools[IB_POLL_LAST_POOL_TYPE + 1];
  2414. struct rdma_restrack_root *res;
  2415. const struct uapi_definition *driver_def;
  2416. /*
  2417. * Positive refcount indicates that the device is currently
  2418. * registered and cannot be unregistered.
  2419. */
  2420. refcount_t refcount;
  2421. struct completion unreg_completion;
  2422. struct work_struct unregistration_work;
  2423. const struct rdma_link_ops *link_ops;
  2424. /* Protects compat_devs xarray modifications */
  2425. struct mutex compat_devs_mutex;
  2426. /* Maintains compat devices for each net namespace */
  2427. struct xarray compat_devs;
  2428. /* Used by iWarp CM */
  2429. char iw_ifname[IFNAMSIZ];
  2430. u32 iw_driver_flags;
  2431. u32 lag_flags;
  2432. };
  2433. struct ib_client_nl_info;
  2434. struct ib_client {
  2435. const char *name;
  2436. int (*add)(struct ib_device *ibdev);
  2437. void (*remove)(struct ib_device *, void *client_data);
  2438. void (*rename)(struct ib_device *dev, void *client_data);
  2439. int (*get_nl_info)(struct ib_device *ibdev, void *client_data,
  2440. struct ib_client_nl_info *res);
  2441. int (*get_global_nl_info)(struct ib_client_nl_info *res);
  2442. /* Returns the net_dev belonging to this ib_client and matching the
  2443. * given parameters.
  2444. * @dev: An RDMA device that the net_dev use for communication.
  2445. * @port: A physical port number on the RDMA device.
  2446. * @pkey: P_Key that the net_dev uses if applicable.
  2447. * @gid: A GID that the net_dev uses to communicate.
  2448. * @addr: An IP address the net_dev is configured with.
  2449. * @client_data: The device's client data set by ib_set_client_data().
  2450. *
  2451. * An ib_client that implements a net_dev on top of RDMA devices
  2452. * (such as IP over IB) should implement this callback, allowing the
  2453. * rdma_cm module to find the right net_dev for a given request.
  2454. *
  2455. * The caller is responsible for calling dev_put on the returned
  2456. * netdev. */
  2457. struct net_device *(*get_net_dev_by_params)(
  2458. struct ib_device *dev,
  2459. u8 port,
  2460. u16 pkey,
  2461. const union ib_gid *gid,
  2462. const struct sockaddr *addr,
  2463. void *client_data);
  2464. refcount_t uses;
  2465. struct completion uses_zero;
  2466. u32 client_id;
  2467. /* kverbs are not required by the client */
  2468. u8 no_kverbs_req:1;
  2469. };
  2470. /*
  2471. * IB block DMA iterator
  2472. *
  2473. * Iterates the DMA-mapped SGL in contiguous memory blocks aligned
  2474. * to a HW supported page size.
  2475. */
  2476. struct ib_block_iter {
  2477. /* internal states */
  2478. struct scatterlist *__sg; /* sg holding the current aligned block */
  2479. dma_addr_t __dma_addr; /* unaligned DMA address of this block */
  2480. unsigned int __sg_nents; /* number of SG entries */
  2481. unsigned int __sg_advance; /* number of bytes to advance in sg in next step */
  2482. unsigned int __pg_bit; /* alignment of current block */
  2483. };
  2484. struct ib_device *_ib_alloc_device(size_t size);
  2485. #define ib_alloc_device(drv_struct, member) \
  2486. container_of(_ib_alloc_device(sizeof(struct drv_struct) + \
  2487. BUILD_BUG_ON_ZERO(offsetof( \
  2488. struct drv_struct, member))), \
  2489. struct drv_struct, member)
  2490. void ib_dealloc_device(struct ib_device *device);
  2491. void ib_get_device_fw_str(struct ib_device *device, char *str);
  2492. int ib_register_device(struct ib_device *device, const char *name,
  2493. struct device *dma_device);
  2494. void ib_unregister_device(struct ib_device *device);
  2495. void ib_unregister_driver(enum rdma_driver_id driver_id);
  2496. void ib_unregister_device_and_put(struct ib_device *device);
  2497. void ib_unregister_device_queued(struct ib_device *ib_dev);
  2498. int ib_register_client (struct ib_client *client);
  2499. void ib_unregister_client(struct ib_client *client);
  2500. void __rdma_block_iter_start(struct ib_block_iter *biter,
  2501. struct scatterlist *sglist,
  2502. unsigned int nents,
  2503. unsigned long pgsz);
  2504. bool __rdma_block_iter_next(struct ib_block_iter *biter);
  2505. /**
  2506. * rdma_block_iter_dma_address - get the aligned dma address of the current
  2507. * block held by the block iterator.
  2508. * @biter: block iterator holding the memory block
  2509. */
  2510. static inline dma_addr_t
  2511. rdma_block_iter_dma_address(struct ib_block_iter *biter)
  2512. {
  2513. return biter->__dma_addr & ~(BIT_ULL(biter->__pg_bit) - 1);
  2514. }
  2515. /**
  2516. * rdma_for_each_block - iterate over contiguous memory blocks of the sg list
  2517. * @sglist: sglist to iterate over
  2518. * @biter: block iterator holding the memory block
  2519. * @nents: maximum number of sg entries to iterate over
  2520. * @pgsz: best HW supported page size to use
  2521. *
  2522. * Callers may use rdma_block_iter_dma_address() to get each
  2523. * blocks aligned DMA address.
  2524. */
  2525. #define rdma_for_each_block(sglist, biter, nents, pgsz) \
  2526. for (__rdma_block_iter_start(biter, sglist, nents, \
  2527. pgsz); \
  2528. __rdma_block_iter_next(biter);)
  2529. /**
  2530. * ib_get_client_data - Get IB client context
  2531. * @device:Device to get context for
  2532. * @client:Client to get context for
  2533. *
  2534. * ib_get_client_data() returns the client context data set with
  2535. * ib_set_client_data(). This can only be called while the client is
  2536. * registered to the device, once the ib_client remove() callback returns this
  2537. * cannot be called.
  2538. */
  2539. static inline void *ib_get_client_data(struct ib_device *device,
  2540. struct ib_client *client)
  2541. {
  2542. return xa_load(&device->client_data, client->client_id);
  2543. }
  2544. void ib_set_client_data(struct ib_device *device, struct ib_client *client,
  2545. void *data);
  2546. void ib_set_device_ops(struct ib_device *device,
  2547. const struct ib_device_ops *ops);
  2548. int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,
  2549. unsigned long pfn, unsigned long size, pgprot_t prot,
  2550. struct rdma_user_mmap_entry *entry);
  2551. int rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext,
  2552. struct rdma_user_mmap_entry *entry,
  2553. size_t length);
  2554. int rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext,
  2555. struct rdma_user_mmap_entry *entry,
  2556. size_t length, u32 min_pgoff,
  2557. u32 max_pgoff);
  2558. struct rdma_user_mmap_entry *
  2559. rdma_user_mmap_entry_get_pgoff(struct ib_ucontext *ucontext,
  2560. unsigned long pgoff);
  2561. struct rdma_user_mmap_entry *
  2562. rdma_user_mmap_entry_get(struct ib_ucontext *ucontext,
  2563. struct vm_area_struct *vma);
  2564. void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry);
  2565. void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry);
  2566. static inline int ib_copy_from_udata(void *dest, struct ib_udata *udata, size_t len)
  2567. {
  2568. return copy_from_user(dest, udata->inbuf, len) ? -EFAULT : 0;
  2569. }
  2570. static inline int ib_copy_to_udata(struct ib_udata *udata, void *src, size_t len)
  2571. {
  2572. return copy_to_user(udata->outbuf, src, len) ? -EFAULT : 0;
  2573. }
  2574. static inline bool ib_is_buffer_cleared(const void __user *p,
  2575. size_t len)
  2576. {
  2577. bool ret;
  2578. u8 *buf;
  2579. if (len > USHRT_MAX)
  2580. return false;
  2581. buf = memdup_user(p, len);
  2582. if (IS_ERR(buf))
  2583. return false;
  2584. ret = !memchr_inv(buf, 0, len);
  2585. kfree(buf);
  2586. return ret;
  2587. }
  2588. static inline bool ib_is_udata_cleared(struct ib_udata *udata,
  2589. size_t offset,
  2590. size_t len)
  2591. {
  2592. return ib_is_buffer_cleared(udata->inbuf + offset, len);
  2593. }
  2594. /**
  2595. * ib_is_destroy_retryable - Check whether the uobject destruction
  2596. * is retryable.
  2597. * @ret: The initial destruction return code
  2598. * @why: remove reason
  2599. * @uobj: The uobject that is destroyed
  2600. *
  2601. * This function is a helper function that IB layer and low-level drivers
  2602. * can use to consider whether the destruction of the given uobject is
  2603. * retry-able.
  2604. * It checks the original return code, if it wasn't success the destruction
  2605. * is retryable according to the ucontext state (i.e. cleanup_retryable) and
  2606. * the remove reason. (i.e. why).
  2607. * Must be called with the object locked for destroy.
  2608. */
  2609. static inline bool ib_is_destroy_retryable(int ret, enum rdma_remove_reason why,
  2610. struct ib_uobject *uobj)
  2611. {
  2612. return ret && (why == RDMA_REMOVE_DESTROY ||
  2613. uobj->context->cleanup_retryable);
  2614. }
  2615. /**
  2616. * ib_destroy_usecnt - Called during destruction to check the usecnt
  2617. * @usecnt: The usecnt atomic
  2618. * @why: remove reason
  2619. * @uobj: The uobject that is destroyed
  2620. *
  2621. * Non-zero usecnts will block destruction unless destruction was triggered by
  2622. * a ucontext cleanup.
  2623. */
  2624. static inline int ib_destroy_usecnt(atomic_t *usecnt,
  2625. enum rdma_remove_reason why,
  2626. struct ib_uobject *uobj)
  2627. {
  2628. if (atomic_read(usecnt) && ib_is_destroy_retryable(-EBUSY, why, uobj))
  2629. return -EBUSY;
  2630. return 0;
  2631. }
  2632. /**
  2633. * ib_modify_qp_is_ok - Check that the supplied attribute mask
  2634. * contains all required attributes and no attributes not allowed for
  2635. * the given QP state transition.
  2636. * @cur_state: Current QP state
  2637. * @next_state: Next QP state
  2638. * @type: QP type
  2639. * @mask: Mask of supplied QP attributes
  2640. *
  2641. * This function is a helper function that a low-level driver's
  2642. * modify_qp method can use to validate the consumer's input. It
  2643. * checks that cur_state and next_state are valid QP states, that a
  2644. * transition from cur_state to next_state is allowed by the IB spec,
  2645. * and that the attribute mask supplied is allowed for the transition.
  2646. */
  2647. bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
  2648. enum ib_qp_type type, enum ib_qp_attr_mask mask);
  2649. void ib_register_event_handler(struct ib_event_handler *event_handler);
  2650. void ib_unregister_event_handler(struct ib_event_handler *event_handler);
  2651. void ib_dispatch_event(const struct ib_event *event);
  2652. int ib_query_port(struct ib_device *device,
  2653. u8 port_num, struct ib_port_attr *port_attr);
  2654. enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device,
  2655. u8 port_num);
  2656. /**
  2657. * rdma_cap_ib_switch - Check if the device is IB switch
  2658. * @device: Device to check
  2659. *
  2660. * Device driver is responsible for setting is_switch bit on
  2661. * in ib_device structure at init time.
  2662. *
  2663. * Return: true if the device is IB switch.
  2664. */
  2665. static inline bool rdma_cap_ib_switch(const struct ib_device *device)
  2666. {
  2667. return device->is_switch;
  2668. }
  2669. /**
  2670. * rdma_start_port - Return the first valid port number for the device
  2671. * specified
  2672. *
  2673. * @device: Device to be checked
  2674. *
  2675. * Return start port number
  2676. */
  2677. static inline u8 rdma_start_port(const struct ib_device *device)
  2678. {
  2679. return rdma_cap_ib_switch(device) ? 0 : 1;
  2680. }
  2681. /**
  2682. * rdma_for_each_port - Iterate over all valid port numbers of the IB device
  2683. * @device - The struct ib_device * to iterate over
  2684. * @iter - The unsigned int to store the port number
  2685. */
  2686. #define rdma_for_each_port(device, iter) \
  2687. for (iter = rdma_start_port(device + BUILD_BUG_ON_ZERO(!__same_type( \
  2688. unsigned int, iter))); \
  2689. iter <= rdma_end_port(device); (iter)++)
  2690. /**
  2691. * rdma_end_port - Return the last valid port number for the device
  2692. * specified
  2693. *
  2694. * @device: Device to be checked
  2695. *
  2696. * Return last port number
  2697. */
  2698. static inline u8 rdma_end_port(const struct ib_device *device)
  2699. {
  2700. return rdma_cap_ib_switch(device) ? 0 : device->phys_port_cnt;
  2701. }
  2702. static inline int rdma_is_port_valid(const struct ib_device *device,
  2703. unsigned int port)
  2704. {
  2705. return (port >= rdma_start_port(device) &&
  2706. port <= rdma_end_port(device));
  2707. }
  2708. static inline bool rdma_is_grh_required(const struct ib_device *device,
  2709. u8 port_num)
  2710. {
  2711. return device->port_data[port_num].immutable.core_cap_flags &
  2712. RDMA_CORE_PORT_IB_GRH_REQUIRED;
  2713. }
  2714. static inline bool rdma_protocol_ib(const struct ib_device *device, u8 port_num)
  2715. {
  2716. return device->port_data[port_num].immutable.core_cap_flags &
  2717. RDMA_CORE_CAP_PROT_IB;
  2718. }
  2719. static inline bool rdma_protocol_roce(const struct ib_device *device, u8 port_num)
  2720. {
  2721. return device->port_data[port_num].immutable.core_cap_flags &
  2722. (RDMA_CORE_CAP_PROT_ROCE | RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP);
  2723. }
  2724. static inline bool rdma_protocol_roce_udp_encap(const struct ib_device *device, u8 port_num)
  2725. {
  2726. return device->port_data[port_num].immutable.core_cap_flags &
  2727. RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
  2728. }
  2729. static inline bool rdma_protocol_roce_eth_encap(const struct ib_device *device, u8 port_num)
  2730. {
  2731. return device->port_data[port_num].immutable.core_cap_flags &
  2732. RDMA_CORE_CAP_PROT_ROCE;
  2733. }
  2734. static inline bool rdma_protocol_iwarp(const struct ib_device *device, u8 port_num)
  2735. {
  2736. return device->port_data[port_num].immutable.core_cap_flags &
  2737. RDMA_CORE_CAP_PROT_IWARP;
  2738. }
  2739. static inline bool rdma_ib_or_roce(const struct ib_device *device, u8 port_num)
  2740. {
  2741. return rdma_protocol_ib(device, port_num) ||
  2742. rdma_protocol_roce(device, port_num);
  2743. }
  2744. static inline bool rdma_protocol_raw_packet(const struct ib_device *device, u8 port_num)
  2745. {
  2746. return device->port_data[port_num].immutable.core_cap_flags &
  2747. RDMA_CORE_CAP_PROT_RAW_PACKET;
  2748. }
  2749. static inline bool rdma_protocol_usnic(const struct ib_device *device, u8 port_num)
  2750. {
  2751. return device->port_data[port_num].immutable.core_cap_flags &
  2752. RDMA_CORE_CAP_PROT_USNIC;
  2753. }
  2754. /**
  2755. * rdma_cap_ib_mad - Check if the port of a device supports Infiniband
  2756. * Management Datagrams.
  2757. * @device: Device to check
  2758. * @port_num: Port number to check
  2759. *
  2760. * Management Datagrams (MAD) are a required part of the InfiniBand
  2761. * specification and are supported on all InfiniBand devices. A slightly
  2762. * extended version are also supported on OPA interfaces.
  2763. *
  2764. * Return: true if the port supports sending/receiving of MAD packets.
  2765. */
  2766. static inline bool rdma_cap_ib_mad(const struct ib_device *device, u8 port_num)
  2767. {
  2768. return device->port_data[port_num].immutable.core_cap_flags &
  2769. RDMA_CORE_CAP_IB_MAD;
  2770. }
  2771. /**
  2772. * rdma_cap_opa_mad - Check if the port of device provides support for OPA
  2773. * Management Datagrams.
  2774. * @device: Device to check
  2775. * @port_num: Port number to check
  2776. *
  2777. * Intel OmniPath devices extend and/or replace the InfiniBand Management
  2778. * datagrams with their own versions. These OPA MADs share many but not all of
  2779. * the characteristics of InfiniBand MADs.
  2780. *
  2781. * OPA MADs differ in the following ways:
  2782. *
  2783. * 1) MADs are variable size up to 2K
  2784. * IBTA defined MADs remain fixed at 256 bytes
  2785. * 2) OPA SMPs must carry valid PKeys
  2786. * 3) OPA SMP packets are a different format
  2787. *
  2788. * Return: true if the port supports OPA MAD packet formats.
  2789. */
  2790. static inline bool rdma_cap_opa_mad(struct ib_device *device, u8 port_num)
  2791. {
  2792. return device->port_data[port_num].immutable.core_cap_flags &
  2793. RDMA_CORE_CAP_OPA_MAD;
  2794. }
  2795. /**
  2796. * rdma_cap_ib_smi - Check if the port of a device provides an Infiniband
  2797. * Subnet Management Agent (SMA) on the Subnet Management Interface (SMI).
  2798. * @device: Device to check
  2799. * @port_num: Port number to check
  2800. *
  2801. * Each InfiniBand node is required to provide a Subnet Management Agent
  2802. * that the subnet manager can access. Prior to the fabric being fully
  2803. * configured by the subnet manager, the SMA is accessed via a well known
  2804. * interface called the Subnet Management Interface (SMI). This interface
  2805. * uses directed route packets to communicate with the SM to get around the
  2806. * chicken and egg problem of the SM needing to know what's on the fabric
  2807. * in order to configure the fabric, and needing to configure the fabric in
  2808. * order to send packets to the devices on the fabric. These directed
  2809. * route packets do not need the fabric fully configured in order to reach
  2810. * their destination. The SMI is the only method allowed to send
  2811. * directed route packets on an InfiniBand fabric.
  2812. *
  2813. * Return: true if the port provides an SMI.
  2814. */
  2815. static inline bool rdma_cap_ib_smi(const struct ib_device *device, u8 port_num)
  2816. {
  2817. return device->port_data[port_num].immutable.core_cap_flags &
  2818. RDMA_CORE_CAP_IB_SMI;
  2819. }
  2820. /**
  2821. * rdma_cap_ib_cm - Check if the port of device has the capability Infiniband
  2822. * Communication Manager.
  2823. * @device: Device to check
  2824. * @port_num: Port number to check
  2825. *
  2826. * The InfiniBand Communication Manager is one of many pre-defined General
  2827. * Service Agents (GSA) that are accessed via the General Service
  2828. * Interface (GSI). It's role is to facilitate establishment of connections
  2829. * between nodes as well as other management related tasks for established
  2830. * connections.
  2831. *
  2832. * Return: true if the port supports an IB CM (this does not guarantee that
  2833. * a CM is actually running however).
  2834. */
  2835. static inline bool rdma_cap_ib_cm(const struct ib_device *device, u8 port_num)
  2836. {
  2837. return device->port_data[port_num].immutable.core_cap_flags &
  2838. RDMA_CORE_CAP_IB_CM;
  2839. }
  2840. /**
  2841. * rdma_cap_iw_cm - Check if the port of device has the capability IWARP
  2842. * Communication Manager.
  2843. * @device: Device to check
  2844. * @port_num: Port number to check
  2845. *
  2846. * Similar to above, but specific to iWARP connections which have a different
  2847. * managment protocol than InfiniBand.
  2848. *
  2849. * Return: true if the port supports an iWARP CM (this does not guarantee that
  2850. * a CM is actually running however).
  2851. */
  2852. static inline bool rdma_cap_iw_cm(const struct ib_device *device, u8 port_num)
  2853. {
  2854. return device->port_data[port_num].immutable.core_cap_flags &
  2855. RDMA_CORE_CAP_IW_CM;
  2856. }
  2857. /**
  2858. * rdma_cap_ib_sa - Check if the port of device has the capability Infiniband
  2859. * Subnet Administration.
  2860. * @device: Device to check
  2861. * @port_num: Port number to check
  2862. *
  2863. * An InfiniBand Subnet Administration (SA) service is a pre-defined General
  2864. * Service Agent (GSA) provided by the Subnet Manager (SM). On InfiniBand
  2865. * fabrics, devices should resolve routes to other hosts by contacting the
  2866. * SA to query the proper route.
  2867. *
  2868. * Return: true if the port should act as a client to the fabric Subnet
  2869. * Administration interface. This does not imply that the SA service is
  2870. * running locally.
  2871. */
  2872. static inline bool rdma_cap_ib_sa(const struct ib_device *device, u8 port_num)
  2873. {
  2874. return device->port_data[port_num].immutable.core_cap_flags &
  2875. RDMA_CORE_CAP_IB_SA;
  2876. }
  2877. /**
  2878. * rdma_cap_ib_mcast - Check if the port of device has the capability Infiniband
  2879. * Multicast.
  2880. * @device: Device to check
  2881. * @port_num: Port number to check
  2882. *
  2883. * InfiniBand multicast registration is more complex than normal IPv4 or
  2884. * IPv6 multicast registration. Each Host Channel Adapter must register
  2885. * with the Subnet Manager when it wishes to join a multicast group. It
  2886. * should do so only once regardless of how many queue pairs it subscribes
  2887. * to this group. And it should leave the group only after all queue pairs
  2888. * attached to the group have been detached.
  2889. *
  2890. * Return: true if the port must undertake the additional adminstrative
  2891. * overhead of registering/unregistering with the SM and tracking of the
  2892. * total number of queue pairs attached to the multicast group.
  2893. */
  2894. static inline bool rdma_cap_ib_mcast(const struct ib_device *device, u8 port_num)
  2895. {
  2896. return rdma_cap_ib_sa(device, port_num);
  2897. }
  2898. /**
  2899. * rdma_cap_af_ib - Check if the port of device has the capability
  2900. * Native Infiniband Address.
  2901. * @device: Device to check
  2902. * @port_num: Port number to check
  2903. *
  2904. * InfiniBand addressing uses a port's GUID + Subnet Prefix to make a default
  2905. * GID. RoCE uses a different mechanism, but still generates a GID via
  2906. * a prescribed mechanism and port specific data.
  2907. *
  2908. * Return: true if the port uses a GID address to identify devices on the
  2909. * network.
  2910. */
  2911. static inline bool rdma_cap_af_ib(const struct ib_device *device, u8 port_num)
  2912. {
  2913. return device->port_data[port_num].immutable.core_cap_flags &
  2914. RDMA_CORE_CAP_AF_IB;
  2915. }
  2916. /**
  2917. * rdma_cap_eth_ah - Check if the port of device has the capability
  2918. * Ethernet Address Handle.
  2919. * @device: Device to check
  2920. * @port_num: Port number to check
  2921. *
  2922. * RoCE is InfiniBand over Ethernet, and it uses a well defined technique
  2923. * to fabricate GIDs over Ethernet/IP specific addresses native to the
  2924. * port. Normally, packet headers are generated by the sending host
  2925. * adapter, but when sending connectionless datagrams, we must manually
  2926. * inject the proper headers for the fabric we are communicating over.
  2927. *
  2928. * Return: true if we are running as a RoCE port and must force the
  2929. * addition of a Global Route Header built from our Ethernet Address
  2930. * Handle into our header list for connectionless packets.
  2931. */
  2932. static inline bool rdma_cap_eth_ah(const struct ib_device *device, u8 port_num)
  2933. {
  2934. return device->port_data[port_num].immutable.core_cap_flags &
  2935. RDMA_CORE_CAP_ETH_AH;
  2936. }
  2937. /**
  2938. * rdma_cap_opa_ah - Check if the port of device supports
  2939. * OPA Address handles
  2940. * @device: Device to check
  2941. * @port_num: Port number to check
  2942. *
  2943. * Return: true if we are running on an OPA device which supports
  2944. * the extended OPA addressing.
  2945. */
  2946. static inline bool rdma_cap_opa_ah(struct ib_device *device, u8 port_num)
  2947. {
  2948. return (device->port_data[port_num].immutable.core_cap_flags &
  2949. RDMA_CORE_CAP_OPA_AH) == RDMA_CORE_CAP_OPA_AH;
  2950. }
  2951. /**
  2952. * rdma_max_mad_size - Return the max MAD size required by this RDMA Port.
  2953. *
  2954. * @device: Device
  2955. * @port_num: Port number
  2956. *
  2957. * This MAD size includes the MAD headers and MAD payload. No other headers
  2958. * are included.
  2959. *
  2960. * Return the max MAD size required by the Port. Will return 0 if the port
  2961. * does not support MADs
  2962. */
  2963. static inline size_t rdma_max_mad_size(const struct ib_device *device, u8 port_num)
  2964. {
  2965. return device->port_data[port_num].immutable.max_mad_size;
  2966. }
  2967. /**
  2968. * rdma_cap_roce_gid_table - Check if the port of device uses roce_gid_table
  2969. * @device: Device to check
  2970. * @port_num: Port number to check
  2971. *
  2972. * RoCE GID table mechanism manages the various GIDs for a device.
  2973. *
  2974. * NOTE: if allocating the port's GID table has failed, this call will still
  2975. * return true, but any RoCE GID table API will fail.
  2976. *
  2977. * Return: true if the port uses RoCE GID table mechanism in order to manage
  2978. * its GIDs.
  2979. */
  2980. static inline bool rdma_cap_roce_gid_table(const struct ib_device *device,
  2981. u8 port_num)
  2982. {
  2983. return rdma_protocol_roce(device, port_num) &&
  2984. device->ops.add_gid && device->ops.del_gid;
  2985. }
  2986. /*
  2987. * Check if the device supports READ W/ INVALIDATE.
  2988. */
  2989. static inline bool rdma_cap_read_inv(struct ib_device *dev, u32 port_num)
  2990. {
  2991. /*
  2992. * iWarp drivers must support READ W/ INVALIDATE. No other protocol
  2993. * has support for it yet.
  2994. */
  2995. return rdma_protocol_iwarp(dev, port_num);
  2996. }
  2997. /**
  2998. * rdma_core_cap_opa_port - Return whether the RDMA Port is OPA or not.
  2999. * @device: Device
  3000. * @port_num: 1 based Port number
  3001. *
  3002. * Return true if port is an Intel OPA port , false if not
  3003. */
  3004. static inline bool rdma_core_cap_opa_port(struct ib_device *device,
  3005. u32 port_num)
  3006. {
  3007. return (device->port_data[port_num].immutable.core_cap_flags &
  3008. RDMA_CORE_PORT_INTEL_OPA) == RDMA_CORE_PORT_INTEL_OPA;
  3009. }
  3010. /**
  3011. * rdma_mtu_enum_to_int - Return the mtu of the port as an integer value.
  3012. * @device: Device
  3013. * @port_num: Port number
  3014. * @mtu: enum value of MTU
  3015. *
  3016. * Return the MTU size supported by the port as an integer value. Will return
  3017. * -1 if enum value of mtu is not supported.
  3018. */
  3019. static inline int rdma_mtu_enum_to_int(struct ib_device *device, u8 port,
  3020. int mtu)
  3021. {
  3022. if (rdma_core_cap_opa_port(device, port))
  3023. return opa_mtu_enum_to_int((enum opa_mtu)mtu);
  3024. else
  3025. return ib_mtu_enum_to_int((enum ib_mtu)mtu);
  3026. }
  3027. /**
  3028. * rdma_mtu_from_attr - Return the mtu of the port from the port attribute.
  3029. * @device: Device
  3030. * @port_num: Port number
  3031. * @attr: port attribute
  3032. *
  3033. * Return the MTU size supported by the port as an integer value.
  3034. */
  3035. static inline int rdma_mtu_from_attr(struct ib_device *device, u8 port,
  3036. struct ib_port_attr *attr)
  3037. {
  3038. if (rdma_core_cap_opa_port(device, port))
  3039. return attr->phys_mtu;
  3040. else
  3041. return ib_mtu_enum_to_int(attr->max_mtu);
  3042. }
  3043. int ib_set_vf_link_state(struct ib_device *device, int vf, u8 port,
  3044. int state);
  3045. int ib_get_vf_config(struct ib_device *device, int vf, u8 port,
  3046. struct ifla_vf_info *info);
  3047. int ib_get_vf_stats(struct ib_device *device, int vf, u8 port,
  3048. struct ifla_vf_stats *stats);
  3049. int ib_get_vf_guid(struct ib_device *device, int vf, u8 port,
  3050. struct ifla_vf_guid *node_guid,
  3051. struct ifla_vf_guid *port_guid);
  3052. int ib_set_vf_guid(struct ib_device *device, int vf, u8 port, u64 guid,
  3053. int type);
  3054. int ib_query_pkey(struct ib_device *device,
  3055. u8 port_num, u16 index, u16 *pkey);
  3056. int ib_modify_device(struct ib_device *device,
  3057. int device_modify_mask,
  3058. struct ib_device_modify *device_modify);
  3059. int ib_modify_port(struct ib_device *device,
  3060. u8 port_num, int port_modify_mask,
  3061. struct ib_port_modify *port_modify);
  3062. int ib_find_gid(struct ib_device *device, union ib_gid *gid,
  3063. u8 *port_num, u16 *index);
  3064. int ib_find_pkey(struct ib_device *device,
  3065. u8 port_num, u16 pkey, u16 *index);
  3066. enum ib_pd_flags {
  3067. /*
  3068. * Create a memory registration for all memory in the system and place
  3069. * the rkey for it into pd->unsafe_global_rkey. This can be used by
  3070. * ULPs to avoid the overhead of dynamic MRs.
  3071. *
  3072. * This flag is generally considered unsafe and must only be used in
  3073. * extremly trusted environments. Every use of it will log a warning
  3074. * in the kernel log.
  3075. */
  3076. IB_PD_UNSAFE_GLOBAL_RKEY = 0x01,
  3077. };
  3078. struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
  3079. const char *caller);
  3080. #define ib_alloc_pd(device, flags) \
  3081. __ib_alloc_pd((device), (flags), KBUILD_MODNAME)
  3082. int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata);
  3083. /**
  3084. * ib_dealloc_pd - Deallocate kernel PD
  3085. * @pd: The protection domain
  3086. *
  3087. * NOTE: for user PD use ib_dealloc_pd_user with valid udata!
  3088. */
  3089. static inline void ib_dealloc_pd(struct ib_pd *pd)
  3090. {
  3091. int ret = ib_dealloc_pd_user(pd, NULL);
  3092. WARN_ONCE(ret, "Destroy of kernel PD shouldn't fail");
  3093. }
  3094. enum rdma_create_ah_flags {
  3095. /* In a sleepable context */
  3096. RDMA_CREATE_AH_SLEEPABLE = BIT(0),
  3097. };
  3098. /**
  3099. * rdma_create_ah - Creates an address handle for the given address vector.
  3100. * @pd: The protection domain associated with the address handle.
  3101. * @ah_attr: The attributes of the address vector.
  3102. * @flags: Create address handle flags (see enum rdma_create_ah_flags).
  3103. *
  3104. * The address handle is used to reference a local or global destination
  3105. * in all UD QP post sends.
  3106. */
  3107. struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
  3108. u32 flags);
  3109. /**
  3110. * rdma_create_user_ah - Creates an address handle for the given address vector.
  3111. * It resolves destination mac address for ah attribute of RoCE type.
  3112. * @pd: The protection domain associated with the address handle.
  3113. * @ah_attr: The attributes of the address vector.
  3114. * @udata: pointer to user's input output buffer information need by
  3115. * provider driver.
  3116. *
  3117. * It returns 0 on success and returns appropriate error code on error.
  3118. * The address handle is used to reference a local or global destination
  3119. * in all UD QP post sends.
  3120. */
  3121. struct ib_ah *rdma_create_user_ah(struct ib_pd *pd,
  3122. struct rdma_ah_attr *ah_attr,
  3123. struct ib_udata *udata);
  3124. /**
  3125. * ib_get_gids_from_rdma_hdr - Get sgid and dgid from GRH or IPv4 header
  3126. * work completion.
  3127. * @hdr: the L3 header to parse
  3128. * @net_type: type of header to parse
  3129. * @sgid: place to store source gid
  3130. * @dgid: place to store destination gid
  3131. */
  3132. int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,
  3133. enum rdma_network_type net_type,
  3134. union ib_gid *sgid, union ib_gid *dgid);
  3135. /**
  3136. * ib_get_rdma_header_version - Get the header version
  3137. * @hdr: the L3 header to parse
  3138. */
  3139. int ib_get_rdma_header_version(const union rdma_network_hdr *hdr);
  3140. /**
  3141. * ib_init_ah_attr_from_wc - Initializes address handle attributes from a
  3142. * work completion.
  3143. * @device: Device on which the received message arrived.
  3144. * @port_num: Port on which the received message arrived.
  3145. * @wc: Work completion associated with the received message.
  3146. * @grh: References the received global route header. This parameter is
  3147. * ignored unless the work completion indicates that the GRH is valid.
  3148. * @ah_attr: Returned attributes that can be used when creating an address
  3149. * handle for replying to the message.
  3150. * When ib_init_ah_attr_from_wc() returns success,
  3151. * (a) for IB link layer it optionally contains a reference to SGID attribute
  3152. * when GRH is present for IB link layer.
  3153. * (b) for RoCE link layer it contains a reference to SGID attribute.
  3154. * User must invoke rdma_cleanup_ah_attr_gid_attr() to release reference to SGID
  3155. * attributes which are initialized using ib_init_ah_attr_from_wc().
  3156. *
  3157. */
  3158. int ib_init_ah_attr_from_wc(struct ib_device *device, u8 port_num,
  3159. const struct ib_wc *wc, const struct ib_grh *grh,
  3160. struct rdma_ah_attr *ah_attr);
  3161. /**
  3162. * ib_create_ah_from_wc - Creates an address handle associated with the
  3163. * sender of the specified work completion.
  3164. * @pd: The protection domain associated with the address handle.
  3165. * @wc: Work completion information associated with a received message.
  3166. * @grh: References the received global route header. This parameter is
  3167. * ignored unless the work completion indicates that the GRH is valid.
  3168. * @port_num: The outbound port number to associate with the address.
  3169. *
  3170. * The address handle is used to reference a local or global destination
  3171. * in all UD QP post sends.
  3172. */
  3173. struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,
  3174. const struct ib_grh *grh, u8 port_num);
  3175. /**
  3176. * rdma_modify_ah - Modifies the address vector associated with an address
  3177. * handle.
  3178. * @ah: The address handle to modify.
  3179. * @ah_attr: The new address vector attributes to associate with the
  3180. * address handle.
  3181. */
  3182. int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
  3183. /**
  3184. * rdma_query_ah - Queries the address vector associated with an address
  3185. * handle.
  3186. * @ah: The address handle to query.
  3187. * @ah_attr: The address vector attributes associated with the address
  3188. * handle.
  3189. */
  3190. int rdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
  3191. enum rdma_destroy_ah_flags {
  3192. /* In a sleepable context */
  3193. RDMA_DESTROY_AH_SLEEPABLE = BIT(0),
  3194. };
  3195. /**
  3196. * rdma_destroy_ah_user - Destroys an address handle.
  3197. * @ah: The address handle to destroy.
  3198. * @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).
  3199. * @udata: Valid user data or NULL for kernel objects
  3200. */
  3201. int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata);
  3202. /**
  3203. * rdma_destroy_ah - Destroys an kernel address handle.
  3204. * @ah: The address handle to destroy.
  3205. * @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).
  3206. *
  3207. * NOTE: for user ah use rdma_destroy_ah_user with valid udata!
  3208. */
  3209. static inline void rdma_destroy_ah(struct ib_ah *ah, u32 flags)
  3210. {
  3211. int ret = rdma_destroy_ah_user(ah, flags, NULL);
  3212. WARN_ONCE(ret, "Destroy of kernel AH shouldn't fail");
  3213. }
  3214. struct ib_srq *ib_create_srq_user(struct ib_pd *pd,
  3215. struct ib_srq_init_attr *srq_init_attr,
  3216. struct ib_usrq_object *uobject,
  3217. struct ib_udata *udata);
  3218. static inline struct ib_srq *
  3219. ib_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *srq_init_attr)
  3220. {
  3221. if (!pd->device->ops.create_srq)
  3222. return ERR_PTR(-EOPNOTSUPP);
  3223. return ib_create_srq_user(pd, srq_init_attr, NULL, NULL);
  3224. }
  3225. /**
  3226. * ib_modify_srq - Modifies the attributes for the specified SRQ.
  3227. * @srq: The SRQ to modify.
  3228. * @srq_attr: On input, specifies the SRQ attributes to modify. On output,
  3229. * the current values of selected SRQ attributes are returned.
  3230. * @srq_attr_mask: A bit-mask used to specify which attributes of the SRQ
  3231. * are being modified.
  3232. *
  3233. * The mask may contain IB_SRQ_MAX_WR to resize the SRQ and/or
  3234. * IB_SRQ_LIMIT to set the SRQ's limit and request notification when
  3235. * the number of receives queued drops below the limit.
  3236. */
  3237. int ib_modify_srq(struct ib_srq *srq,
  3238. struct ib_srq_attr *srq_attr,
  3239. enum ib_srq_attr_mask srq_attr_mask);
  3240. /**
  3241. * ib_query_srq - Returns the attribute list and current values for the
  3242. * specified SRQ.
  3243. * @srq: The SRQ to query.
  3244. * @srq_attr: The attributes of the specified SRQ.
  3245. */
  3246. int ib_query_srq(struct ib_srq *srq,
  3247. struct ib_srq_attr *srq_attr);
  3248. /**
  3249. * ib_destroy_srq_user - Destroys the specified SRQ.
  3250. * @srq: The SRQ to destroy.
  3251. * @udata: Valid user data or NULL for kernel objects
  3252. */
  3253. int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata);
  3254. /**
  3255. * ib_destroy_srq - Destroys the specified kernel SRQ.
  3256. * @srq: The SRQ to destroy.
  3257. *
  3258. * NOTE: for user srq use ib_destroy_srq_user with valid udata!
  3259. */
  3260. static inline void ib_destroy_srq(struct ib_srq *srq)
  3261. {
  3262. int ret = ib_destroy_srq_user(srq, NULL);
  3263. WARN_ONCE(ret, "Destroy of kernel SRQ shouldn't fail");
  3264. }
  3265. /**
  3266. * ib_post_srq_recv - Posts a list of work requests to the specified SRQ.
  3267. * @srq: The SRQ to post the work request on.
  3268. * @recv_wr: A list of work requests to post on the receive queue.
  3269. * @bad_recv_wr: On an immediate failure, this parameter will reference
  3270. * the work request that failed to be posted on the QP.
  3271. */
  3272. static inline int ib_post_srq_recv(struct ib_srq *srq,
  3273. const struct ib_recv_wr *recv_wr,
  3274. const struct ib_recv_wr **bad_recv_wr)
  3275. {
  3276. const struct ib_recv_wr *dummy;
  3277. return srq->device->ops.post_srq_recv(srq, recv_wr,
  3278. bad_recv_wr ? : &dummy);
  3279. }
  3280. struct ib_qp *ib_create_qp(struct ib_pd *pd,
  3281. struct ib_qp_init_attr *qp_init_attr);
  3282. /**
  3283. * ib_modify_qp_with_udata - Modifies the attributes for the specified QP.
  3284. * @qp: The QP to modify.
  3285. * @attr: On input, specifies the QP attributes to modify. On output,
  3286. * the current values of selected QP attributes are returned.
  3287. * @attr_mask: A bit-mask used to specify which attributes of the QP
  3288. * are being modified.
  3289. * @udata: pointer to user's input output buffer information
  3290. * are being modified.
  3291. * It returns 0 on success and returns appropriate error code on error.
  3292. */
  3293. int ib_modify_qp_with_udata(struct ib_qp *qp,
  3294. struct ib_qp_attr *attr,
  3295. int attr_mask,
  3296. struct ib_udata *udata);
  3297. /**
  3298. * ib_modify_qp - Modifies the attributes for the specified QP and then
  3299. * transitions the QP to the given state.
  3300. * @qp: The QP to modify.
  3301. * @qp_attr: On input, specifies the QP attributes to modify. On output,
  3302. * the current values of selected QP attributes are returned.
  3303. * @qp_attr_mask: A bit-mask used to specify which attributes of the QP
  3304. * are being modified.
  3305. */
  3306. int ib_modify_qp(struct ib_qp *qp,
  3307. struct ib_qp_attr *qp_attr,
  3308. int qp_attr_mask);
  3309. /**
  3310. * ib_query_qp - Returns the attribute list and current values for the
  3311. * specified QP.
  3312. * @qp: The QP to query.
  3313. * @qp_attr: The attributes of the specified QP.
  3314. * @qp_attr_mask: A bit-mask used to select specific attributes to query.
  3315. * @qp_init_attr: Additional attributes of the selected QP.
  3316. *
  3317. * The qp_attr_mask may be used to limit the query to gathering only the
  3318. * selected attributes.
  3319. */
  3320. int ib_query_qp(struct ib_qp *qp,
  3321. struct ib_qp_attr *qp_attr,
  3322. int qp_attr_mask,
  3323. struct ib_qp_init_attr *qp_init_attr);
  3324. /**
  3325. * ib_destroy_qp - Destroys the specified QP.
  3326. * @qp: The QP to destroy.
  3327. * @udata: Valid udata or NULL for kernel objects
  3328. */
  3329. int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata);
  3330. /**
  3331. * ib_destroy_qp - Destroys the specified kernel QP.
  3332. * @qp: The QP to destroy.
  3333. *
  3334. * NOTE: for user qp use ib_destroy_qp_user with valid udata!
  3335. */
  3336. static inline int ib_destroy_qp(struct ib_qp *qp)
  3337. {
  3338. return ib_destroy_qp_user(qp, NULL);
  3339. }
  3340. /**
  3341. * ib_open_qp - Obtain a reference to an existing sharable QP.
  3342. * @xrcd - XRC domain
  3343. * @qp_open_attr: Attributes identifying the QP to open.
  3344. *
  3345. * Returns a reference to a sharable QP.
  3346. */
  3347. struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
  3348. struct ib_qp_open_attr *qp_open_attr);
  3349. /**
  3350. * ib_close_qp - Release an external reference to a QP.
  3351. * @qp: The QP handle to release
  3352. *
  3353. * The opened QP handle is released by the caller. The underlying
  3354. * shared QP is not destroyed until all internal references are released.
  3355. */
  3356. int ib_close_qp(struct ib_qp *qp);
  3357. /**
  3358. * ib_post_send - Posts a list of work requests to the send queue of
  3359. * the specified QP.
  3360. * @qp: The QP to post the work request on.
  3361. * @send_wr: A list of work requests to post on the send queue.
  3362. * @bad_send_wr: On an immediate failure, this parameter will reference
  3363. * the work request that failed to be posted on the QP.
  3364. *
  3365. * While IBA Vol. 1 section 11.4.1.1 specifies that if an immediate
  3366. * error is returned, the QP state shall not be affected,
  3367. * ib_post_send() will return an immediate error after queueing any
  3368. * earlier work requests in the list.
  3369. */
  3370. static inline int ib_post_send(struct ib_qp *qp,
  3371. const struct ib_send_wr *send_wr,
  3372. const struct ib_send_wr **bad_send_wr)
  3373. {
  3374. const struct ib_send_wr *dummy;
  3375. return qp->device->ops.post_send(qp, send_wr, bad_send_wr ? : &dummy);
  3376. }
  3377. /**
  3378. * ib_post_recv - Posts a list of work requests to the receive queue of
  3379. * the specified QP.
  3380. * @qp: The QP to post the work request on.
  3381. * @recv_wr: A list of work requests to post on the receive queue.
  3382. * @bad_recv_wr: On an immediate failure, this parameter will reference
  3383. * the work request that failed to be posted on the QP.
  3384. */
  3385. static inline int ib_post_recv(struct ib_qp *qp,
  3386. const struct ib_recv_wr *recv_wr,
  3387. const struct ib_recv_wr **bad_recv_wr)
  3388. {
  3389. const struct ib_recv_wr *dummy;
  3390. return qp->device->ops.post_recv(qp, recv_wr, bad_recv_wr ? : &dummy);
  3391. }
  3392. struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe,
  3393. int comp_vector, enum ib_poll_context poll_ctx,
  3394. const char *caller);
  3395. static inline struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private,
  3396. int nr_cqe, int comp_vector,
  3397. enum ib_poll_context poll_ctx)
  3398. {
  3399. return __ib_alloc_cq(dev, private, nr_cqe, comp_vector, poll_ctx,
  3400. KBUILD_MODNAME);
  3401. }
  3402. struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, void *private,
  3403. int nr_cqe, enum ib_poll_context poll_ctx,
  3404. const char *caller);
  3405. /**
  3406. * ib_alloc_cq_any: Allocate kernel CQ
  3407. * @dev: The IB device
  3408. * @private: Private data attached to the CQE
  3409. * @nr_cqe: Number of CQEs in the CQ
  3410. * @poll_ctx: Context used for polling the CQ
  3411. */
  3412. static inline struct ib_cq *ib_alloc_cq_any(struct ib_device *dev,
  3413. void *private, int nr_cqe,
  3414. enum ib_poll_context poll_ctx)
  3415. {
  3416. return __ib_alloc_cq_any(dev, private, nr_cqe, poll_ctx,
  3417. KBUILD_MODNAME);
  3418. }
  3419. void ib_free_cq(struct ib_cq *cq);
  3420. int ib_process_cq_direct(struct ib_cq *cq, int budget);
  3421. /**
  3422. * ib_create_cq - Creates a CQ on the specified device.
  3423. * @device: The device on which to create the CQ.
  3424. * @comp_handler: A user-specified callback that is invoked when a
  3425. * completion event occurs on the CQ.
  3426. * @event_handler: A user-specified callback that is invoked when an
  3427. * asynchronous event not associated with a completion occurs on the CQ.
  3428. * @cq_context: Context associated with the CQ returned to the user via
  3429. * the associated completion and event handlers.
  3430. * @cq_attr: The attributes the CQ should be created upon.
  3431. *
  3432. * Users can examine the cq structure to determine the actual CQ size.
  3433. */
  3434. struct ib_cq *__ib_create_cq(struct ib_device *device,
  3435. ib_comp_handler comp_handler,
  3436. void (*event_handler)(struct ib_event *, void *),
  3437. void *cq_context,
  3438. const struct ib_cq_init_attr *cq_attr,
  3439. const char *caller);
  3440. #define ib_create_cq(device, cmp_hndlr, evt_hndlr, cq_ctxt, cq_attr) \
  3441. __ib_create_cq((device), (cmp_hndlr), (evt_hndlr), (cq_ctxt), (cq_attr), KBUILD_MODNAME)
  3442. /**
  3443. * ib_resize_cq - Modifies the capacity of the CQ.
  3444. * @cq: The CQ to resize.
  3445. * @cqe: The minimum size of the CQ.
  3446. *
  3447. * Users can examine the cq structure to determine the actual CQ size.
  3448. */
  3449. int ib_resize_cq(struct ib_cq *cq, int cqe);
  3450. /**
  3451. * rdma_set_cq_moderation - Modifies moderation params of the CQ
  3452. * @cq: The CQ to modify.
  3453. * @cq_count: number of CQEs that will trigger an event
  3454. * @cq_period: max period of time in usec before triggering an event
  3455. *
  3456. */
  3457. int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period);
  3458. /**
  3459. * ib_destroy_cq_user - Destroys the specified CQ.
  3460. * @cq: The CQ to destroy.
  3461. * @udata: Valid user data or NULL for kernel objects
  3462. */
  3463. int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata);
  3464. /**
  3465. * ib_destroy_cq - Destroys the specified kernel CQ.
  3466. * @cq: The CQ to destroy.
  3467. *
  3468. * NOTE: for user cq use ib_destroy_cq_user with valid udata!
  3469. */
  3470. static inline void ib_destroy_cq(struct ib_cq *cq)
  3471. {
  3472. int ret = ib_destroy_cq_user(cq, NULL);
  3473. WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail");
  3474. }
  3475. /**
  3476. * ib_poll_cq - poll a CQ for completion(s)
  3477. * @cq:the CQ being polled
  3478. * @num_entries:maximum number of completions to return
  3479. * @wc:array of at least @num_entries &struct ib_wc where completions
  3480. * will be returned
  3481. *
  3482. * Poll a CQ for (possibly multiple) completions. If the return value
  3483. * is < 0, an error occurred. If the return value is >= 0, it is the
  3484. * number of completions returned. If the return value is
  3485. * non-negative and < num_entries, then the CQ was emptied.
  3486. */
  3487. static inline int ib_poll_cq(struct ib_cq *cq, int num_entries,
  3488. struct ib_wc *wc)
  3489. {
  3490. return cq->device->ops.poll_cq(cq, num_entries, wc);
  3491. }
  3492. /**
  3493. * ib_req_notify_cq - Request completion notification on a CQ.
  3494. * @cq: The CQ to generate an event for.
  3495. * @flags:
  3496. * Must contain exactly one of %IB_CQ_SOLICITED or %IB_CQ_NEXT_COMP
  3497. * to request an event on the next solicited event or next work
  3498. * completion at any type, respectively. %IB_CQ_REPORT_MISSED_EVENTS
  3499. * may also be |ed in to request a hint about missed events, as
  3500. * described below.
  3501. *
  3502. * Return Value:
  3503. * < 0 means an error occurred while requesting notification
  3504. * == 0 means notification was requested successfully, and if
  3505. * IB_CQ_REPORT_MISSED_EVENTS was passed in, then no events
  3506. * were missed and it is safe to wait for another event. In
  3507. * this case is it guaranteed that any work completions added
  3508. * to the CQ since the last CQ poll will trigger a completion
  3509. * notification event.
  3510. * > 0 is only returned if IB_CQ_REPORT_MISSED_EVENTS was passed
  3511. * in. It means that the consumer must poll the CQ again to
  3512. * make sure it is empty to avoid missing an event because of a
  3513. * race between requesting notification and an entry being
  3514. * added to the CQ. This return value means it is possible
  3515. * (but not guaranteed) that a work completion has been added
  3516. * to the CQ since the last poll without triggering a
  3517. * completion notification event.
  3518. */
  3519. static inline int ib_req_notify_cq(struct ib_cq *cq,
  3520. enum ib_cq_notify_flags flags)
  3521. {
  3522. return cq->device->ops.req_notify_cq(cq, flags);
  3523. }
  3524. struct ib_cq *ib_cq_pool_get(struct ib_device *dev, unsigned int nr_cqe,
  3525. int comp_vector_hint,
  3526. enum ib_poll_context poll_ctx);
  3527. void ib_cq_pool_put(struct ib_cq *cq, unsigned int nr_cqe);
  3528. /**
  3529. * ib_req_ncomp_notif - Request completion notification when there are
  3530. * at least the specified number of unreaped completions on the CQ.
  3531. * @cq: The CQ to generate an event for.
  3532. * @wc_cnt: The number of unreaped completions that should be on the
  3533. * CQ before an event is generated.
  3534. */
  3535. static inline int ib_req_ncomp_notif(struct ib_cq *cq, int wc_cnt)
  3536. {
  3537. return cq->device->ops.req_ncomp_notif ?
  3538. cq->device->ops.req_ncomp_notif(cq, wc_cnt) :
  3539. -ENOSYS;
  3540. }
  3541. /*
  3542. * Drivers that don't need a DMA mapping at the RDMA layer, set dma_device to
  3543. * NULL. This causes the ib_dma* helpers to just stash the kernel virtual
  3544. * address into the dma address.
  3545. */
  3546. static inline bool ib_uses_virt_dma(struct ib_device *dev)
  3547. {
  3548. return IS_ENABLED(CONFIG_INFINIBAND_VIRT_DMA) && !dev->dma_device;
  3549. }
  3550. /**
  3551. * ib_dma_mapping_error - check a DMA addr for error
  3552. * @dev: The device for which the dma_addr was created
  3553. * @dma_addr: The DMA address to check
  3554. */
  3555. static inline int ib_dma_mapping_error(struct ib_device *dev, u64 dma_addr)
  3556. {
  3557. if (ib_uses_virt_dma(dev))
  3558. return 0;
  3559. return dma_mapping_error(dev->dma_device, dma_addr);
  3560. }
  3561. /**
  3562. * ib_dma_map_single - Map a kernel virtual address to DMA address
  3563. * @dev: The device for which the dma_addr is to be created
  3564. * @cpu_addr: The kernel virtual address
  3565. * @size: The size of the region in bytes
  3566. * @direction: The direction of the DMA
  3567. */
  3568. static inline u64 ib_dma_map_single(struct ib_device *dev,
  3569. void *cpu_addr, size_t size,
  3570. enum dma_data_direction direction)
  3571. {
  3572. if (ib_uses_virt_dma(dev))
  3573. return (uintptr_t)cpu_addr;
  3574. return dma_map_single(dev->dma_device, cpu_addr, size, direction);
  3575. }
  3576. /**
  3577. * ib_dma_unmap_single - Destroy a mapping created by ib_dma_map_single()
  3578. * @dev: The device for which the DMA address was created
  3579. * @addr: The DMA address
  3580. * @size: The size of the region in bytes
  3581. * @direction: The direction of the DMA
  3582. */
  3583. static inline void ib_dma_unmap_single(struct ib_device *dev,
  3584. u64 addr, size_t size,
  3585. enum dma_data_direction direction)
  3586. {
  3587. if (!ib_uses_virt_dma(dev))
  3588. dma_unmap_single(dev->dma_device, addr, size, direction);
  3589. }
  3590. /**
  3591. * ib_dma_map_page - Map a physical page to DMA address
  3592. * @dev: The device for which the dma_addr is to be created
  3593. * @page: The page to be mapped
  3594. * @offset: The offset within the page
  3595. * @size: The size of the region in bytes
  3596. * @direction: The direction of the DMA
  3597. */
  3598. static inline u64 ib_dma_map_page(struct ib_device *dev,
  3599. struct page *page,
  3600. unsigned long offset,
  3601. size_t size,
  3602. enum dma_data_direction direction)
  3603. {
  3604. if (ib_uses_virt_dma(dev))
  3605. return (uintptr_t)(page_address(page) + offset);
  3606. return dma_map_page(dev->dma_device, page, offset, size, direction);
  3607. }
  3608. /**
  3609. * ib_dma_unmap_page - Destroy a mapping created by ib_dma_map_page()
  3610. * @dev: The device for which the DMA address was created
  3611. * @addr: The DMA address
  3612. * @size: The size of the region in bytes
  3613. * @direction: The direction of the DMA
  3614. */
  3615. static inline void ib_dma_unmap_page(struct ib_device *dev,
  3616. u64 addr, size_t size,
  3617. enum dma_data_direction direction)
  3618. {
  3619. if (!ib_uses_virt_dma(dev))
  3620. dma_unmap_page(dev->dma_device, addr, size, direction);
  3621. }
  3622. int ib_dma_virt_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents);
  3623. static inline int ib_dma_map_sg_attrs(struct ib_device *dev,
  3624. struct scatterlist *sg, int nents,
  3625. enum dma_data_direction direction,
  3626. unsigned long dma_attrs)
  3627. {
  3628. if (ib_uses_virt_dma(dev))
  3629. return ib_dma_virt_map_sg(dev, sg, nents);
  3630. return dma_map_sg_attrs(dev->dma_device, sg, nents, direction,
  3631. dma_attrs);
  3632. }
  3633. static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev,
  3634. struct scatterlist *sg, int nents,
  3635. enum dma_data_direction direction,
  3636. unsigned long dma_attrs)
  3637. {
  3638. if (!ib_uses_virt_dma(dev))
  3639. dma_unmap_sg_attrs(dev->dma_device, sg, nents, direction,
  3640. dma_attrs);
  3641. }
  3642. /**
  3643. * ib_dma_map_sg - Map a scatter/gather list to DMA addresses
  3644. * @dev: The device for which the DMA addresses are to be created
  3645. * @sg: The array of scatter/gather entries
  3646. * @nents: The number of scatter/gather entries
  3647. * @direction: The direction of the DMA
  3648. */
  3649. static inline int ib_dma_map_sg(struct ib_device *dev,
  3650. struct scatterlist *sg, int nents,
  3651. enum dma_data_direction direction)
  3652. {
  3653. return ib_dma_map_sg_attrs(dev, sg, nents, direction, 0);
  3654. }
  3655. /**
  3656. * ib_dma_unmap_sg - Unmap a scatter/gather list of DMA addresses
  3657. * @dev: The device for which the DMA addresses were created
  3658. * @sg: The array of scatter/gather entries
  3659. * @nents: The number of scatter/gather entries
  3660. * @direction: The direction of the DMA
  3661. */
  3662. static inline void ib_dma_unmap_sg(struct ib_device *dev,
  3663. struct scatterlist *sg, int nents,
  3664. enum dma_data_direction direction)
  3665. {
  3666. ib_dma_unmap_sg_attrs(dev, sg, nents, direction, 0);
  3667. }
  3668. /**
  3669. * ib_dma_max_seg_size - Return the size limit of a single DMA transfer
  3670. * @dev: The device to query
  3671. *
  3672. * The returned value represents a size in bytes.
  3673. */
  3674. static inline unsigned int ib_dma_max_seg_size(struct ib_device *dev)
  3675. {
  3676. if (ib_uses_virt_dma(dev))
  3677. return UINT_MAX;
  3678. return dma_get_max_seg_size(dev->dma_device);
  3679. }
  3680. /**
  3681. * ib_dma_sync_single_for_cpu - Prepare DMA region to be accessed by CPU
  3682. * @dev: The device for which the DMA address was created
  3683. * @addr: The DMA address
  3684. * @size: The size of the region in bytes
  3685. * @dir: The direction of the DMA
  3686. */
  3687. static inline void ib_dma_sync_single_for_cpu(struct ib_device *dev,
  3688. u64 addr,
  3689. size_t size,
  3690. enum dma_data_direction dir)
  3691. {
  3692. if (!ib_uses_virt_dma(dev))
  3693. dma_sync_single_for_cpu(dev->dma_device, addr, size, dir);
  3694. }
  3695. /**
  3696. * ib_dma_sync_single_for_device - Prepare DMA region to be accessed by device
  3697. * @dev: The device for which the DMA address was created
  3698. * @addr: The DMA address
  3699. * @size: The size of the region in bytes
  3700. * @dir: The direction of the DMA
  3701. */
  3702. static inline void ib_dma_sync_single_for_device(struct ib_device *dev,
  3703. u64 addr,
  3704. size_t size,
  3705. enum dma_data_direction dir)
  3706. {
  3707. if (!ib_uses_virt_dma(dev))
  3708. dma_sync_single_for_device(dev->dma_device, addr, size, dir);
  3709. }
  3710. /**
  3711. * ib_dma_alloc_coherent - Allocate memory and map it for DMA
  3712. * @dev: The device for which the DMA address is requested
  3713. * @size: The size of the region to allocate in bytes
  3714. * @dma_handle: A pointer for returning the DMA address of the region
  3715. * @flag: memory allocator flags
  3716. */
  3717. static inline void *ib_dma_alloc_coherent(struct ib_device *dev,
  3718. size_t size,
  3719. dma_addr_t *dma_handle,
  3720. gfp_t flag)
  3721. {
  3722. return dma_alloc_coherent(dev->dma_device, size, dma_handle, flag);
  3723. }
  3724. /**
  3725. * ib_dma_free_coherent - Free memory allocated by ib_dma_alloc_coherent()
  3726. * @dev: The device for which the DMA addresses were allocated
  3727. * @size: The size of the region
  3728. * @cpu_addr: the address returned by ib_dma_alloc_coherent()
  3729. * @dma_handle: the DMA address returned by ib_dma_alloc_coherent()
  3730. */
  3731. static inline void ib_dma_free_coherent(struct ib_device *dev,
  3732. size_t size, void *cpu_addr,
  3733. dma_addr_t dma_handle)
  3734. {
  3735. dma_free_coherent(dev->dma_device, size, cpu_addr, dma_handle);
  3736. }
  3737. /* ib_reg_user_mr - register a memory region for virtual addresses from kernel
  3738. * space. This function should be called when 'current' is the owning MM.
  3739. */
  3740. struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
  3741. u64 virt_addr, int mr_access_flags);
  3742. /* ib_advise_mr - give an advice about an address range in a memory region */
  3743. int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,
  3744. u32 flags, struct ib_sge *sg_list, u32 num_sge);
  3745. /**
  3746. * ib_dereg_mr_user - Deregisters a memory region and removes it from the
  3747. * HCA translation table.
  3748. * @mr: The memory region to deregister.
  3749. * @udata: Valid user data or NULL for kernel object
  3750. *
  3751. * This function can fail, if the memory region has memory windows bound to it.
  3752. */
  3753. int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata);
  3754. /**
  3755. * ib_dereg_mr - Deregisters a kernel memory region and removes it from the
  3756. * HCA translation table.
  3757. * @mr: The memory region to deregister.
  3758. *
  3759. * This function can fail, if the memory region has memory windows bound to it.
  3760. *
  3761. * NOTE: for user mr use ib_dereg_mr_user with valid udata!
  3762. */
  3763. static inline int ib_dereg_mr(struct ib_mr *mr)
  3764. {
  3765. return ib_dereg_mr_user(mr, NULL);
  3766. }
  3767. struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
  3768. u32 max_num_sg);
  3769. struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
  3770. u32 max_num_data_sg,
  3771. u32 max_num_meta_sg);
  3772. /**
  3773. * ib_update_fast_reg_key - updates the key portion of the fast_reg MR
  3774. * R_Key and L_Key.
  3775. * @mr - struct ib_mr pointer to be updated.
  3776. * @newkey - new key to be used.
  3777. */
  3778. static inline void ib_update_fast_reg_key(struct ib_mr *mr, u8 newkey)
  3779. {
  3780. mr->lkey = (mr->lkey & 0xffffff00) | newkey;
  3781. mr->rkey = (mr->rkey & 0xffffff00) | newkey;
  3782. }
  3783. /**
  3784. * ib_inc_rkey - increments the key portion of the given rkey. Can be used
  3785. * for calculating a new rkey for type 2 memory windows.
  3786. * @rkey - the rkey to increment.
  3787. */
  3788. static inline u32 ib_inc_rkey(u32 rkey)
  3789. {
  3790. const u32 mask = 0x000000ff;
  3791. return ((rkey + 1) & mask) | (rkey & ~mask);
  3792. }
  3793. /**
  3794. * ib_attach_mcast - Attaches the specified QP to a multicast group.
  3795. * @qp: QP to attach to the multicast group. The QP must be type
  3796. * IB_QPT_UD.
  3797. * @gid: Multicast group GID.
  3798. * @lid: Multicast group LID in host byte order.
  3799. *
  3800. * In order to send and receive multicast packets, subnet
  3801. * administration must have created the multicast group and configured
  3802. * the fabric appropriately. The port associated with the specified
  3803. * QP must also be a member of the multicast group.
  3804. */
  3805. int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
  3806. /**
  3807. * ib_detach_mcast - Detaches the specified QP from a multicast group.
  3808. * @qp: QP to detach from the multicast group.
  3809. * @gid: Multicast group GID.
  3810. * @lid: Multicast group LID in host byte order.
  3811. */
  3812. int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
  3813. struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device,
  3814. struct inode *inode, struct ib_udata *udata);
  3815. int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata);
  3816. static inline int ib_check_mr_access(int flags)
  3817. {
  3818. /*
  3819. * Local write permission is required if remote write or
  3820. * remote atomic permission is also requested.
  3821. */
  3822. if (flags & (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE) &&
  3823. !(flags & IB_ACCESS_LOCAL_WRITE))
  3824. return -EINVAL;
  3825. if (flags & ~IB_ACCESS_SUPPORTED)
  3826. return -EINVAL;
  3827. return 0;
  3828. }
  3829. static inline bool ib_access_writable(int access_flags)
  3830. {
  3831. /*
  3832. * We have writable memory backing the MR if any of the following
  3833. * access flags are set. "Local write" and "remote write" obviously
  3834. * require write access. "Remote atomic" can do things like fetch and
  3835. * add, which will modify memory, and "MW bind" can change permissions
  3836. * by binding a window.
  3837. */
  3838. return access_flags &
  3839. (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE |
  3840. IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_MW_BIND);
  3841. }
  3842. /**
  3843. * ib_check_mr_status: lightweight check of MR status.
  3844. * This routine may provide status checks on a selected
  3845. * ib_mr. first use is for signature status check.
  3846. *
  3847. * @mr: A memory region.
  3848. * @check_mask: Bitmask of which checks to perform from
  3849. * ib_mr_status_check enumeration.
  3850. * @mr_status: The container of relevant status checks.
  3851. * failed checks will be indicated in the status bitmask
  3852. * and the relevant info shall be in the error item.
  3853. */
  3854. int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
  3855. struct ib_mr_status *mr_status);
  3856. /**
  3857. * ib_device_try_get: Hold a registration lock
  3858. * device: The device to lock
  3859. *
  3860. * A device under an active registration lock cannot become unregistered. It
  3861. * is only possible to obtain a registration lock on a device that is fully
  3862. * registered, otherwise this function returns false.
  3863. *
  3864. * The registration lock is only necessary for actions which require the
  3865. * device to still be registered. Uses that only require the device pointer to
  3866. * be valid should use get_device(&ibdev->dev) to hold the memory.
  3867. *
  3868. */
  3869. static inline bool ib_device_try_get(struct ib_device *dev)
  3870. {
  3871. return refcount_inc_not_zero(&dev->refcount);
  3872. }
  3873. void ib_device_put(struct ib_device *device);
  3874. struct ib_device *ib_device_get_by_netdev(struct net_device *ndev,
  3875. enum rdma_driver_id driver_id);
  3876. struct ib_device *ib_device_get_by_name(const char *name,
  3877. enum rdma_driver_id driver_id);
  3878. struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, u8 port,
  3879. u16 pkey, const union ib_gid *gid,
  3880. const struct sockaddr *addr);
  3881. int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev,
  3882. unsigned int port);
  3883. struct net_device *ib_device_netdev(struct ib_device *dev, u8 port);
  3884. struct ib_wq *ib_create_wq(struct ib_pd *pd,
  3885. struct ib_wq_init_attr *init_attr);
  3886. int ib_destroy_wq_user(struct ib_wq *wq, struct ib_udata *udata);
  3887. int ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *attr,
  3888. u32 wq_attr_mask);
  3889. int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
  3890. unsigned int *sg_offset, unsigned int page_size);
  3891. int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,
  3892. int data_sg_nents, unsigned int *data_sg_offset,
  3893. struct scatterlist *meta_sg, int meta_sg_nents,
  3894. unsigned int *meta_sg_offset, unsigned int page_size);
  3895. static inline int
  3896. ib_map_mr_sg_zbva(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
  3897. unsigned int *sg_offset, unsigned int page_size)
  3898. {
  3899. int n;
  3900. n = ib_map_mr_sg(mr, sg, sg_nents, sg_offset, page_size);
  3901. mr->iova = 0;
  3902. return n;
  3903. }
  3904. int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
  3905. unsigned int *sg_offset, int (*set_page)(struct ib_mr *, u64));
  3906. void ib_drain_rq(struct ib_qp *qp);
  3907. void ib_drain_sq(struct ib_qp *qp);
  3908. void ib_drain_qp(struct ib_qp *qp);
  3909. int ib_get_eth_speed(struct ib_device *dev, u8 port_num, u16 *speed, u8 *width);
  3910. static inline u8 *rdma_ah_retrieve_dmac(struct rdma_ah_attr *attr)
  3911. {
  3912. if (attr->type == RDMA_AH_ATTR_TYPE_ROCE)
  3913. return attr->roce.dmac;
  3914. return NULL;
  3915. }
  3916. static inline void rdma_ah_set_dlid(struct rdma_ah_attr *attr, u32 dlid)
  3917. {
  3918. if (attr->type == RDMA_AH_ATTR_TYPE_IB)
  3919. attr->ib.dlid = (u16)dlid;
  3920. else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
  3921. attr->opa.dlid = dlid;
  3922. }
  3923. static inline u32 rdma_ah_get_dlid(const struct rdma_ah_attr *attr)
  3924. {
  3925. if (attr->type == RDMA_AH_ATTR_TYPE_IB)
  3926. return attr->ib.dlid;
  3927. else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
  3928. return attr->opa.dlid;
  3929. return 0;
  3930. }
  3931. static inline void rdma_ah_set_sl(struct rdma_ah_attr *attr, u8 sl)
  3932. {
  3933. attr->sl = sl;
  3934. }
  3935. static inline u8 rdma_ah_get_sl(const struct rdma_ah_attr *attr)
  3936. {
  3937. return attr->sl;
  3938. }
  3939. static inline void rdma_ah_set_path_bits(struct rdma_ah_attr *attr,
  3940. u8 src_path_bits)
  3941. {
  3942. if (attr->type == RDMA_AH_ATTR_TYPE_IB)
  3943. attr->ib.src_path_bits = src_path_bits;
  3944. else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
  3945. attr->opa.src_path_bits = src_path_bits;
  3946. }
  3947. static inline u8 rdma_ah_get_path_bits(const struct rdma_ah_attr *attr)
  3948. {
  3949. if (attr->type == RDMA_AH_ATTR_TYPE_IB)
  3950. return attr->ib.src_path_bits;
  3951. else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
  3952. return attr->opa.src_path_bits;
  3953. return 0;
  3954. }
  3955. static inline void rdma_ah_set_make_grd(struct rdma_ah_attr *attr,
  3956. bool make_grd)
  3957. {
  3958. if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
  3959. attr->opa.make_grd = make_grd;
  3960. }
  3961. static inline bool rdma_ah_get_make_grd(const struct rdma_ah_attr *attr)
  3962. {
  3963. if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
  3964. return attr->opa.make_grd;
  3965. return false;
  3966. }
  3967. static inline void rdma_ah_set_port_num(struct rdma_ah_attr *attr, u8 port_num)
  3968. {
  3969. attr->port_num = port_num;
  3970. }
  3971. static inline u8 rdma_ah_get_port_num(const struct rdma_ah_attr *attr)
  3972. {
  3973. return attr->port_num;
  3974. }
  3975. static inline void rdma_ah_set_static_rate(struct rdma_ah_attr *attr,
  3976. u8 static_rate)
  3977. {
  3978. attr->static_rate = static_rate;
  3979. }
  3980. static inline u8 rdma_ah_get_static_rate(const struct rdma_ah_attr *attr)
  3981. {
  3982. return attr->static_rate;
  3983. }
  3984. static inline void rdma_ah_set_ah_flags(struct rdma_ah_attr *attr,
  3985. enum ib_ah_flags flag)
  3986. {
  3987. attr->ah_flags = flag;
  3988. }
  3989. static inline enum ib_ah_flags
  3990. rdma_ah_get_ah_flags(const struct rdma_ah_attr *attr)
  3991. {
  3992. return attr->ah_flags;
  3993. }
  3994. static inline const struct ib_global_route
  3995. *rdma_ah_read_grh(const struct rdma_ah_attr *attr)
  3996. {
  3997. return &attr->grh;
  3998. }
  3999. /*To retrieve and modify the grh */
  4000. static inline struct ib_global_route
  4001. *rdma_ah_retrieve_grh(struct rdma_ah_attr *attr)
  4002. {
  4003. return &attr->grh;
  4004. }
  4005. static inline void rdma_ah_set_dgid_raw(struct rdma_ah_attr *attr, void *dgid)
  4006. {
  4007. struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
  4008. memcpy(grh->dgid.raw, dgid, sizeof(grh->dgid));
  4009. }
  4010. static inline void rdma_ah_set_subnet_prefix(struct rdma_ah_attr *attr,
  4011. __be64 prefix)
  4012. {
  4013. struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
  4014. grh->dgid.global.subnet_prefix = prefix;
  4015. }
  4016. static inline void rdma_ah_set_interface_id(struct rdma_ah_attr *attr,
  4017. __be64 if_id)
  4018. {
  4019. struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
  4020. grh->dgid.global.interface_id = if_id;
  4021. }
  4022. static inline void rdma_ah_set_grh(struct rdma_ah_attr *attr,
  4023. union ib_gid *dgid, u32 flow_label,
  4024. u8 sgid_index, u8 hop_limit,
  4025. u8 traffic_class)
  4026. {
  4027. struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
  4028. attr->ah_flags = IB_AH_GRH;
  4029. if (dgid)
  4030. grh->dgid = *dgid;
  4031. grh->flow_label = flow_label;
  4032. grh->sgid_index = sgid_index;
  4033. grh->hop_limit = hop_limit;
  4034. grh->traffic_class = traffic_class;
  4035. grh->sgid_attr = NULL;
  4036. }
  4037. void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr);
  4038. void rdma_move_grh_sgid_attr(struct rdma_ah_attr *attr, union ib_gid *dgid,
  4039. u32 flow_label, u8 hop_limit, u8 traffic_class,
  4040. const struct ib_gid_attr *sgid_attr);
  4041. void rdma_copy_ah_attr(struct rdma_ah_attr *dest,
  4042. const struct rdma_ah_attr *src);
  4043. void rdma_replace_ah_attr(struct rdma_ah_attr *old,
  4044. const struct rdma_ah_attr *new);
  4045. void rdma_move_ah_attr(struct rdma_ah_attr *dest, struct rdma_ah_attr *src);
  4046. /**
  4047. * rdma_ah_find_type - Return address handle type.
  4048. *
  4049. * @dev: Device to be checked
  4050. * @port_num: Port number
  4051. */
  4052. static inline enum rdma_ah_attr_type rdma_ah_find_type(struct ib_device *dev,
  4053. u8 port_num)
  4054. {
  4055. if (rdma_protocol_roce(dev, port_num))
  4056. return RDMA_AH_ATTR_TYPE_ROCE;
  4057. if (rdma_protocol_ib(dev, port_num)) {
  4058. if (rdma_cap_opa_ah(dev, port_num))
  4059. return RDMA_AH_ATTR_TYPE_OPA;
  4060. return RDMA_AH_ATTR_TYPE_IB;
  4061. }
  4062. return RDMA_AH_ATTR_TYPE_UNDEFINED;
  4063. }
  4064. /**
  4065. * ib_lid_cpu16 - Return lid in 16bit CPU encoding.
  4066. * In the current implementation the only way to get
  4067. * get the 32bit lid is from other sources for OPA.
  4068. * For IB, lids will always be 16bits so cast the
  4069. * value accordingly.
  4070. *
  4071. * @lid: A 32bit LID
  4072. */
  4073. static inline u16 ib_lid_cpu16(u32 lid)
  4074. {
  4075. WARN_ON_ONCE(lid & 0xFFFF0000);
  4076. return (u16)lid;
  4077. }
  4078. /**
  4079. * ib_lid_be16 - Return lid in 16bit BE encoding.
  4080. *
  4081. * @lid: A 32bit LID
  4082. */
  4083. static inline __be16 ib_lid_be16(u32 lid)
  4084. {
  4085. WARN_ON_ONCE(lid & 0xFFFF0000);
  4086. return cpu_to_be16((u16)lid);
  4087. }
  4088. /**
  4089. * ib_get_vector_affinity - Get the affinity mappings of a given completion
  4090. * vector
  4091. * @device: the rdma device
  4092. * @comp_vector: index of completion vector
  4093. *
  4094. * Returns NULL on failure, otherwise a corresponding cpu map of the
  4095. * completion vector (returns all-cpus map if the device driver doesn't
  4096. * implement get_vector_affinity).
  4097. */
  4098. static inline const struct cpumask *
  4099. ib_get_vector_affinity(struct ib_device *device, int comp_vector)
  4100. {
  4101. if (comp_vector < 0 || comp_vector >= device->num_comp_vectors ||
  4102. !device->ops.get_vector_affinity)
  4103. return NULL;
  4104. return device->ops.get_vector_affinity(device, comp_vector);
  4105. }
  4106. /**
  4107. * rdma_roce_rescan_device - Rescan all of the network devices in the system
  4108. * and add their gids, as needed, to the relevant RoCE devices.
  4109. *
  4110. * @device: the rdma device
  4111. */
  4112. void rdma_roce_rescan_device(struct ib_device *ibdev);
  4113. struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile);
  4114. int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs);
  4115. struct net_device *rdma_alloc_netdev(struct ib_device *device, u8 port_num,
  4116. enum rdma_netdev_t type, const char *name,
  4117. unsigned char name_assign_type,
  4118. void (*setup)(struct net_device *));
  4119. int rdma_init_netdev(struct ib_device *device, u8 port_num,
  4120. enum rdma_netdev_t type, const char *name,
  4121. unsigned char name_assign_type,
  4122. void (*setup)(struct net_device *),
  4123. struct net_device *netdev);
  4124. /**
  4125. * rdma_set_device_sysfs_group - Set device attributes group to have
  4126. * driver specific sysfs entries at
  4127. * for infiniband class.
  4128. *
  4129. * @device: device pointer for which attributes to be created
  4130. * @group: Pointer to group which should be added when device
  4131. * is registered with sysfs.
  4132. * rdma_set_device_sysfs_group() allows existing drivers to expose one
  4133. * group per device to have sysfs attributes.
  4134. *
  4135. * NOTE: New drivers should not make use of this API; instead new device
  4136. * parameter should be exposed via netlink command. This API and mechanism
  4137. * exist only for existing drivers.
  4138. */
  4139. static inline void
  4140. rdma_set_device_sysfs_group(struct ib_device *dev,
  4141. const struct attribute_group *group)
  4142. {
  4143. dev->groups[1] = group;
  4144. }
  4145. /**
  4146. * rdma_device_to_ibdev - Get ib_device pointer from device pointer
  4147. *
  4148. * @device: device pointer for which ib_device pointer to retrieve
  4149. *
  4150. * rdma_device_to_ibdev() retrieves ib_device pointer from device.
  4151. *
  4152. */
  4153. static inline struct ib_device *rdma_device_to_ibdev(struct device *device)
  4154. {
  4155. struct ib_core_device *coredev =
  4156. container_of(device, struct ib_core_device, dev);
  4157. return coredev->owner;
  4158. }
  4159. /**
  4160. * ibdev_to_node - return the NUMA node for a given ib_device
  4161. * @dev: device to get the NUMA node for.
  4162. */
  4163. static inline int ibdev_to_node(struct ib_device *ibdev)
  4164. {
  4165. struct device *parent = ibdev->dev.parent;
  4166. if (!parent)
  4167. return NUMA_NO_NODE;
  4168. return dev_to_node(parent);
  4169. }
  4170. /**
  4171. * rdma_device_to_drv_device - Helper macro to reach back to driver's
  4172. * ib_device holder structure from device pointer.
  4173. *
  4174. * NOTE: New drivers should not make use of this API; This API is only for
  4175. * existing drivers who have exposed sysfs entries using
  4176. * rdma_set_device_sysfs_group().
  4177. */
  4178. #define rdma_device_to_drv_device(dev, drv_dev_struct, ibdev_member) \
  4179. container_of(rdma_device_to_ibdev(dev), drv_dev_struct, ibdev_member)
  4180. bool rdma_dev_access_netns(const struct ib_device *device,
  4181. const struct net *net);
  4182. #define IB_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000)
  4183. #define IB_ROCE_UDP_ENCAP_VALID_PORT_MAX (0xFFFF)
  4184. #define IB_GRH_FLOWLABEL_MASK (0x000FFFFF)
  4185. /**
  4186. * rdma_flow_label_to_udp_sport - generate a RoCE v2 UDP src port value based
  4187. * on the flow_label
  4188. *
  4189. * This function will convert the 20 bit flow_label input to a valid RoCE v2
  4190. * UDP src port 14 bit value. All RoCE V2 drivers should use this same
  4191. * convention.
  4192. */
  4193. static inline u16 rdma_flow_label_to_udp_sport(u32 fl)
  4194. {
  4195. u32 fl_low = fl & 0x03fff, fl_high = fl & 0xFC000;
  4196. fl_low ^= fl_high >> 14;
  4197. return (u16)(fl_low | IB_ROCE_UDP_ENCAP_VALID_PORT_MIN);
  4198. }
  4199. /**
  4200. * rdma_calc_flow_label - generate a RDMA symmetric flow label value based on
  4201. * local and remote qpn values
  4202. *
  4203. * This function folded the multiplication results of two qpns, 24 bit each,
  4204. * fields, and converts it to a 20 bit results.
  4205. *
  4206. * This function will create symmetric flow_label value based on the local
  4207. * and remote qpn values. this will allow both the requester and responder
  4208. * to calculate the same flow_label for a given connection.
  4209. *
  4210. * This helper function should be used by driver in case the upper layer
  4211. * provide a zero flow_label value. This is to improve entropy of RDMA
  4212. * traffic in the network.
  4213. */
  4214. static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn)
  4215. {
  4216. u64 v = (u64)lqpn * rqpn;
  4217. v ^= v >> 20;
  4218. v ^= v >> 40;
  4219. return (u32)(v & IB_GRH_FLOWLABEL_MASK);
  4220. }
  4221. #endif /* IB_VERBS_H */