rdma.c 51 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * NVMe over Fabrics RDMA target.
  4. * Copyright (c) 2015-2016 HGST, a Western Digital Company.
  5. */
  6. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  7. #include <linux/atomic.h>
  8. #include <linux/ctype.h>
  9. #include <linux/delay.h>
  10. #include <linux/err.h>
  11. #include <linux/init.h>
  12. #include <linux/module.h>
  13. #include <linux/nvme.h>
  14. #include <linux/slab.h>
  15. #include <linux/string.h>
  16. #include <linux/wait.h>
  17. #include <linux/inet.h>
  18. #include <asm/unaligned.h>
  19. #include <rdma/ib_verbs.h>
  20. #include <rdma/rdma_cm.h>
  21. #include <rdma/rw.h>
  22. #include <rdma/ib_cm.h>
  23. #include <linux/nvme-rdma.h>
  24. #include "nvmet.h"
  25. /*
  26. * We allow at least 1 page, up to 4 SGEs, and up to 16KB of inline data
  27. */
  28. #define NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE PAGE_SIZE
  29. #define NVMET_RDMA_MAX_INLINE_SGE 4
  30. #define NVMET_RDMA_MAX_INLINE_DATA_SIZE max_t(int, SZ_16K, PAGE_SIZE)
  31. /* Assume mpsmin == device_page_size == 4KB */
  32. #define NVMET_RDMA_MAX_MDTS 8
  33. #define NVMET_RDMA_MAX_METADATA_MDTS 5
  34. struct nvmet_rdma_srq;
  35. struct nvmet_rdma_cmd {
  36. struct ib_sge sge[NVMET_RDMA_MAX_INLINE_SGE + 1];
  37. struct ib_cqe cqe;
  38. struct ib_recv_wr wr;
  39. struct scatterlist inline_sg[NVMET_RDMA_MAX_INLINE_SGE];
  40. struct nvme_command *nvme_cmd;
  41. struct nvmet_rdma_queue *queue;
  42. struct nvmet_rdma_srq *nsrq;
  43. };
  44. enum {
  45. NVMET_RDMA_REQ_INLINE_DATA = (1 << 0),
  46. NVMET_RDMA_REQ_INVALIDATE_RKEY = (1 << 1),
  47. };
  48. struct nvmet_rdma_rsp {
  49. struct ib_sge send_sge;
  50. struct ib_cqe send_cqe;
  51. struct ib_send_wr send_wr;
  52. struct nvmet_rdma_cmd *cmd;
  53. struct nvmet_rdma_queue *queue;
  54. struct ib_cqe read_cqe;
  55. struct ib_cqe write_cqe;
  56. struct rdma_rw_ctx rw;
  57. struct nvmet_req req;
  58. bool allocated;
  59. u8 n_rdma;
  60. u32 flags;
  61. u32 invalidate_rkey;
  62. struct list_head wait_list;
  63. struct list_head free_list;
  64. };
  65. enum nvmet_rdma_queue_state {
  66. NVMET_RDMA_Q_CONNECTING,
  67. NVMET_RDMA_Q_LIVE,
  68. NVMET_RDMA_Q_DISCONNECTING,
  69. };
  70. struct nvmet_rdma_queue {
  71. struct rdma_cm_id *cm_id;
  72. struct ib_qp *qp;
  73. struct nvmet_port *port;
  74. struct ib_cq *cq;
  75. atomic_t sq_wr_avail;
  76. struct nvmet_rdma_device *dev;
  77. struct nvmet_rdma_srq *nsrq;
  78. spinlock_t state_lock;
  79. enum nvmet_rdma_queue_state state;
  80. struct nvmet_cq nvme_cq;
  81. struct nvmet_sq nvme_sq;
  82. struct nvmet_rdma_rsp *rsps;
  83. struct list_head free_rsps;
  84. spinlock_t rsps_lock;
  85. struct nvmet_rdma_cmd *cmds;
  86. struct work_struct release_work;
  87. struct list_head rsp_wait_list;
  88. struct list_head rsp_wr_wait_list;
  89. spinlock_t rsp_wr_wait_lock;
  90. int idx;
  91. int host_qid;
  92. int comp_vector;
  93. int recv_queue_size;
  94. int send_queue_size;
  95. struct list_head queue_list;
  96. };
  97. struct nvmet_rdma_port {
  98. struct nvmet_port *nport;
  99. struct sockaddr_storage addr;
  100. struct rdma_cm_id *cm_id;
  101. struct delayed_work repair_work;
  102. };
  103. struct nvmet_rdma_srq {
  104. struct ib_srq *srq;
  105. struct nvmet_rdma_cmd *cmds;
  106. struct nvmet_rdma_device *ndev;
  107. };
  108. struct nvmet_rdma_device {
  109. struct ib_device *device;
  110. struct ib_pd *pd;
  111. struct nvmet_rdma_srq **srqs;
  112. int srq_count;
  113. size_t srq_size;
  114. struct kref ref;
  115. struct list_head entry;
  116. int inline_data_size;
  117. int inline_page_count;
  118. };
  119. static bool nvmet_rdma_use_srq;
  120. module_param_named(use_srq, nvmet_rdma_use_srq, bool, 0444);
  121. MODULE_PARM_DESC(use_srq, "Use shared receive queue.");
  122. static int srq_size_set(const char *val, const struct kernel_param *kp);
  123. static const struct kernel_param_ops srq_size_ops = {
  124. .set = srq_size_set,
  125. .get = param_get_int,
  126. };
  127. static int nvmet_rdma_srq_size = 1024;
  128. module_param_cb(srq_size, &srq_size_ops, &nvmet_rdma_srq_size, 0644);
  129. MODULE_PARM_DESC(srq_size, "set Shared Receive Queue (SRQ) size, should >= 256 (default: 1024)");
  130. static DEFINE_IDA(nvmet_rdma_queue_ida);
  131. static LIST_HEAD(nvmet_rdma_queue_list);
  132. static DEFINE_MUTEX(nvmet_rdma_queue_mutex);
  133. static LIST_HEAD(device_list);
  134. static DEFINE_MUTEX(device_list_mutex);
  135. static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp);
  136. static void nvmet_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc);
  137. static void nvmet_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);
  138. static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc);
  139. static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc);
  140. static void nvmet_rdma_qp_event(struct ib_event *event, void *priv);
  141. static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue);
  142. static void nvmet_rdma_free_rsp(struct nvmet_rdma_device *ndev,
  143. struct nvmet_rdma_rsp *r);
  144. static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device *ndev,
  145. struct nvmet_rdma_rsp *r);
  146. static const struct nvmet_fabrics_ops nvmet_rdma_ops;
  147. static int srq_size_set(const char *val, const struct kernel_param *kp)
  148. {
  149. int n = 0, ret;
  150. ret = kstrtoint(val, 10, &n);
  151. if (ret != 0 || n < 256)
  152. return -EINVAL;
  153. return param_set_int(val, kp);
  154. }
  155. static int num_pages(int len)
  156. {
  157. return 1 + (((len - 1) & PAGE_MASK) >> PAGE_SHIFT);
  158. }
  159. static inline bool nvmet_rdma_need_data_in(struct nvmet_rdma_rsp *rsp)
  160. {
  161. return nvme_is_write(rsp->req.cmd) &&
  162. rsp->req.transfer_len &&
  163. !(rsp->flags & NVMET_RDMA_REQ_INLINE_DATA);
  164. }
  165. static inline bool nvmet_rdma_need_data_out(struct nvmet_rdma_rsp *rsp)
  166. {
  167. return !nvme_is_write(rsp->req.cmd) &&
  168. rsp->req.transfer_len &&
  169. !rsp->req.cqe->status &&
  170. !(rsp->flags & NVMET_RDMA_REQ_INLINE_DATA);
  171. }
  172. static inline struct nvmet_rdma_rsp *
  173. nvmet_rdma_get_rsp(struct nvmet_rdma_queue *queue)
  174. {
  175. struct nvmet_rdma_rsp *rsp;
  176. unsigned long flags;
  177. spin_lock_irqsave(&queue->rsps_lock, flags);
  178. rsp = list_first_entry_or_null(&queue->free_rsps,
  179. struct nvmet_rdma_rsp, free_list);
  180. if (likely(rsp))
  181. list_del(&rsp->free_list);
  182. spin_unlock_irqrestore(&queue->rsps_lock, flags);
  183. if (unlikely(!rsp)) {
  184. int ret;
  185. rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
  186. if (unlikely(!rsp))
  187. return NULL;
  188. ret = nvmet_rdma_alloc_rsp(queue->dev, rsp);
  189. if (unlikely(ret)) {
  190. kfree(rsp);
  191. return NULL;
  192. }
  193. rsp->allocated = true;
  194. }
  195. return rsp;
  196. }
  197. static inline void
  198. nvmet_rdma_put_rsp(struct nvmet_rdma_rsp *rsp)
  199. {
  200. unsigned long flags;
  201. if (unlikely(rsp->allocated)) {
  202. nvmet_rdma_free_rsp(rsp->queue->dev, rsp);
  203. kfree(rsp);
  204. return;
  205. }
  206. spin_lock_irqsave(&rsp->queue->rsps_lock, flags);
  207. list_add_tail(&rsp->free_list, &rsp->queue->free_rsps);
  208. spin_unlock_irqrestore(&rsp->queue->rsps_lock, flags);
  209. }
  210. static void nvmet_rdma_free_inline_pages(struct nvmet_rdma_device *ndev,
  211. struct nvmet_rdma_cmd *c)
  212. {
  213. struct scatterlist *sg;
  214. struct ib_sge *sge;
  215. int i;
  216. if (!ndev->inline_data_size)
  217. return;
  218. sg = c->inline_sg;
  219. sge = &c->sge[1];
  220. for (i = 0; i < ndev->inline_page_count; i++, sg++, sge++) {
  221. if (sge->length)
  222. ib_dma_unmap_page(ndev->device, sge->addr,
  223. sge->length, DMA_FROM_DEVICE);
  224. if (sg_page(sg))
  225. __free_page(sg_page(sg));
  226. }
  227. }
  228. static int nvmet_rdma_alloc_inline_pages(struct nvmet_rdma_device *ndev,
  229. struct nvmet_rdma_cmd *c)
  230. {
  231. struct scatterlist *sg;
  232. struct ib_sge *sge;
  233. struct page *pg;
  234. int len;
  235. int i;
  236. if (!ndev->inline_data_size)
  237. return 0;
  238. sg = c->inline_sg;
  239. sg_init_table(sg, ndev->inline_page_count);
  240. sge = &c->sge[1];
  241. len = ndev->inline_data_size;
  242. for (i = 0; i < ndev->inline_page_count; i++, sg++, sge++) {
  243. pg = alloc_page(GFP_KERNEL);
  244. if (!pg)
  245. goto out_err;
  246. sg_assign_page(sg, pg);
  247. sge->addr = ib_dma_map_page(ndev->device,
  248. pg, 0, PAGE_SIZE, DMA_FROM_DEVICE);
  249. if (ib_dma_mapping_error(ndev->device, sge->addr))
  250. goto out_err;
  251. sge->length = min_t(int, len, PAGE_SIZE);
  252. sge->lkey = ndev->pd->local_dma_lkey;
  253. len -= sge->length;
  254. }
  255. return 0;
  256. out_err:
  257. for (; i >= 0; i--, sg--, sge--) {
  258. if (sge->length)
  259. ib_dma_unmap_page(ndev->device, sge->addr,
  260. sge->length, DMA_FROM_DEVICE);
  261. if (sg_page(sg))
  262. __free_page(sg_page(sg));
  263. }
  264. return -ENOMEM;
  265. }
  266. static int nvmet_rdma_alloc_cmd(struct nvmet_rdma_device *ndev,
  267. struct nvmet_rdma_cmd *c, bool admin)
  268. {
  269. /* NVMe command / RDMA RECV */
  270. c->nvme_cmd = kmalloc(sizeof(*c->nvme_cmd), GFP_KERNEL);
  271. if (!c->nvme_cmd)
  272. goto out;
  273. c->sge[0].addr = ib_dma_map_single(ndev->device, c->nvme_cmd,
  274. sizeof(*c->nvme_cmd), DMA_FROM_DEVICE);
  275. if (ib_dma_mapping_error(ndev->device, c->sge[0].addr))
  276. goto out_free_cmd;
  277. c->sge[0].length = sizeof(*c->nvme_cmd);
  278. c->sge[0].lkey = ndev->pd->local_dma_lkey;
  279. if (!admin && nvmet_rdma_alloc_inline_pages(ndev, c))
  280. goto out_unmap_cmd;
  281. c->cqe.done = nvmet_rdma_recv_done;
  282. c->wr.wr_cqe = &c->cqe;
  283. c->wr.sg_list = c->sge;
  284. c->wr.num_sge = admin ? 1 : ndev->inline_page_count + 1;
  285. return 0;
  286. out_unmap_cmd:
  287. ib_dma_unmap_single(ndev->device, c->sge[0].addr,
  288. sizeof(*c->nvme_cmd), DMA_FROM_DEVICE);
  289. out_free_cmd:
  290. kfree(c->nvme_cmd);
  291. out:
  292. return -ENOMEM;
  293. }
  294. static void nvmet_rdma_free_cmd(struct nvmet_rdma_device *ndev,
  295. struct nvmet_rdma_cmd *c, bool admin)
  296. {
  297. if (!admin)
  298. nvmet_rdma_free_inline_pages(ndev, c);
  299. ib_dma_unmap_single(ndev->device, c->sge[0].addr,
  300. sizeof(*c->nvme_cmd), DMA_FROM_DEVICE);
  301. kfree(c->nvme_cmd);
  302. }
  303. static struct nvmet_rdma_cmd *
  304. nvmet_rdma_alloc_cmds(struct nvmet_rdma_device *ndev,
  305. int nr_cmds, bool admin)
  306. {
  307. struct nvmet_rdma_cmd *cmds;
  308. int ret = -EINVAL, i;
  309. cmds = kcalloc(nr_cmds, sizeof(struct nvmet_rdma_cmd), GFP_KERNEL);
  310. if (!cmds)
  311. goto out;
  312. for (i = 0; i < nr_cmds; i++) {
  313. ret = nvmet_rdma_alloc_cmd(ndev, cmds + i, admin);
  314. if (ret)
  315. goto out_free;
  316. }
  317. return cmds;
  318. out_free:
  319. while (--i >= 0)
  320. nvmet_rdma_free_cmd(ndev, cmds + i, admin);
  321. kfree(cmds);
  322. out:
  323. return ERR_PTR(ret);
  324. }
  325. static void nvmet_rdma_free_cmds(struct nvmet_rdma_device *ndev,
  326. struct nvmet_rdma_cmd *cmds, int nr_cmds, bool admin)
  327. {
  328. int i;
  329. for (i = 0; i < nr_cmds; i++)
  330. nvmet_rdma_free_cmd(ndev, cmds + i, admin);
  331. kfree(cmds);
  332. }
  333. static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device *ndev,
  334. struct nvmet_rdma_rsp *r)
  335. {
  336. /* NVMe CQE / RDMA SEND */
  337. r->req.cqe = kmalloc(sizeof(*r->req.cqe), GFP_KERNEL);
  338. if (!r->req.cqe)
  339. goto out;
  340. r->send_sge.addr = ib_dma_map_single(ndev->device, r->req.cqe,
  341. sizeof(*r->req.cqe), DMA_TO_DEVICE);
  342. if (ib_dma_mapping_error(ndev->device, r->send_sge.addr))
  343. goto out_free_rsp;
  344. if (!ib_uses_virt_dma(ndev->device))
  345. r->req.p2p_client = &ndev->device->dev;
  346. r->send_sge.length = sizeof(*r->req.cqe);
  347. r->send_sge.lkey = ndev->pd->local_dma_lkey;
  348. r->send_cqe.done = nvmet_rdma_send_done;
  349. r->send_wr.wr_cqe = &r->send_cqe;
  350. r->send_wr.sg_list = &r->send_sge;
  351. r->send_wr.num_sge = 1;
  352. r->send_wr.send_flags = IB_SEND_SIGNALED;
  353. /* Data In / RDMA READ */
  354. r->read_cqe.done = nvmet_rdma_read_data_done;
  355. /* Data Out / RDMA WRITE */
  356. r->write_cqe.done = nvmet_rdma_write_data_done;
  357. return 0;
  358. out_free_rsp:
  359. kfree(r->req.cqe);
  360. out:
  361. return -ENOMEM;
  362. }
  363. static void nvmet_rdma_free_rsp(struct nvmet_rdma_device *ndev,
  364. struct nvmet_rdma_rsp *r)
  365. {
  366. ib_dma_unmap_single(ndev->device, r->send_sge.addr,
  367. sizeof(*r->req.cqe), DMA_TO_DEVICE);
  368. kfree(r->req.cqe);
  369. }
  370. static int
  371. nvmet_rdma_alloc_rsps(struct nvmet_rdma_queue *queue)
  372. {
  373. struct nvmet_rdma_device *ndev = queue->dev;
  374. int nr_rsps = queue->recv_queue_size * 2;
  375. int ret = -EINVAL, i;
  376. queue->rsps = kcalloc(nr_rsps, sizeof(struct nvmet_rdma_rsp),
  377. GFP_KERNEL);
  378. if (!queue->rsps)
  379. goto out;
  380. for (i = 0; i < nr_rsps; i++) {
  381. struct nvmet_rdma_rsp *rsp = &queue->rsps[i];
  382. ret = nvmet_rdma_alloc_rsp(ndev, rsp);
  383. if (ret)
  384. goto out_free;
  385. list_add_tail(&rsp->free_list, &queue->free_rsps);
  386. }
  387. return 0;
  388. out_free:
  389. while (--i >= 0) {
  390. struct nvmet_rdma_rsp *rsp = &queue->rsps[i];
  391. list_del(&rsp->free_list);
  392. nvmet_rdma_free_rsp(ndev, rsp);
  393. }
  394. kfree(queue->rsps);
  395. out:
  396. return ret;
  397. }
  398. static void nvmet_rdma_free_rsps(struct nvmet_rdma_queue *queue)
  399. {
  400. struct nvmet_rdma_device *ndev = queue->dev;
  401. int i, nr_rsps = queue->recv_queue_size * 2;
  402. for (i = 0; i < nr_rsps; i++) {
  403. struct nvmet_rdma_rsp *rsp = &queue->rsps[i];
  404. list_del(&rsp->free_list);
  405. nvmet_rdma_free_rsp(ndev, rsp);
  406. }
  407. kfree(queue->rsps);
  408. }
  409. static int nvmet_rdma_post_recv(struct nvmet_rdma_device *ndev,
  410. struct nvmet_rdma_cmd *cmd)
  411. {
  412. int ret;
  413. ib_dma_sync_single_for_device(ndev->device,
  414. cmd->sge[0].addr, cmd->sge[0].length,
  415. DMA_FROM_DEVICE);
  416. if (cmd->nsrq)
  417. ret = ib_post_srq_recv(cmd->nsrq->srq, &cmd->wr, NULL);
  418. else
  419. ret = ib_post_recv(cmd->queue->qp, &cmd->wr, NULL);
  420. if (unlikely(ret))
  421. pr_err("post_recv cmd failed\n");
  422. return ret;
  423. }
  424. static void nvmet_rdma_process_wr_wait_list(struct nvmet_rdma_queue *queue)
  425. {
  426. spin_lock(&queue->rsp_wr_wait_lock);
  427. while (!list_empty(&queue->rsp_wr_wait_list)) {
  428. struct nvmet_rdma_rsp *rsp;
  429. bool ret;
  430. rsp = list_entry(queue->rsp_wr_wait_list.next,
  431. struct nvmet_rdma_rsp, wait_list);
  432. list_del(&rsp->wait_list);
  433. spin_unlock(&queue->rsp_wr_wait_lock);
  434. ret = nvmet_rdma_execute_command(rsp);
  435. spin_lock(&queue->rsp_wr_wait_lock);
  436. if (!ret) {
  437. list_add(&rsp->wait_list, &queue->rsp_wr_wait_list);
  438. break;
  439. }
  440. }
  441. spin_unlock(&queue->rsp_wr_wait_lock);
  442. }
  443. static u16 nvmet_rdma_check_pi_status(struct ib_mr *sig_mr)
  444. {
  445. struct ib_mr_status mr_status;
  446. int ret;
  447. u16 status = 0;
  448. ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status);
  449. if (ret) {
  450. pr_err("ib_check_mr_status failed, ret %d\n", ret);
  451. return NVME_SC_INVALID_PI;
  452. }
  453. if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) {
  454. switch (mr_status.sig_err.err_type) {
  455. case IB_SIG_BAD_GUARD:
  456. status = NVME_SC_GUARD_CHECK;
  457. break;
  458. case IB_SIG_BAD_REFTAG:
  459. status = NVME_SC_REFTAG_CHECK;
  460. break;
  461. case IB_SIG_BAD_APPTAG:
  462. status = NVME_SC_APPTAG_CHECK;
  463. break;
  464. }
  465. pr_err("PI error found type %d expected 0x%x vs actual 0x%x\n",
  466. mr_status.sig_err.err_type,
  467. mr_status.sig_err.expected,
  468. mr_status.sig_err.actual);
  469. }
  470. return status;
  471. }
  472. static void nvmet_rdma_set_sig_domain(struct blk_integrity *bi,
  473. struct nvme_command *cmd, struct ib_sig_domain *domain,
  474. u16 control, u8 pi_type)
  475. {
  476. domain->sig_type = IB_SIG_TYPE_T10_DIF;
  477. domain->sig.dif.bg_type = IB_T10DIF_CRC;
  478. domain->sig.dif.pi_interval = 1 << bi->interval_exp;
  479. domain->sig.dif.ref_tag = le32_to_cpu(cmd->rw.reftag);
  480. if (control & NVME_RW_PRINFO_PRCHK_REF)
  481. domain->sig.dif.ref_remap = true;
  482. domain->sig.dif.app_tag = le16_to_cpu(cmd->rw.apptag);
  483. domain->sig.dif.apptag_check_mask = le16_to_cpu(cmd->rw.appmask);
  484. domain->sig.dif.app_escape = true;
  485. if (pi_type == NVME_NS_DPS_PI_TYPE3)
  486. domain->sig.dif.ref_escape = true;
  487. }
  488. static void nvmet_rdma_set_sig_attrs(struct nvmet_req *req,
  489. struct ib_sig_attrs *sig_attrs)
  490. {
  491. struct nvme_command *cmd = req->cmd;
  492. u16 control = le16_to_cpu(cmd->rw.control);
  493. u8 pi_type = req->ns->pi_type;
  494. struct blk_integrity *bi;
  495. bi = bdev_get_integrity(req->ns->bdev);
  496. memset(sig_attrs, 0, sizeof(*sig_attrs));
  497. if (control & NVME_RW_PRINFO_PRACT) {
  498. /* for WRITE_INSERT/READ_STRIP no wire domain */
  499. sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
  500. nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->mem, control,
  501. pi_type);
  502. /* Clear the PRACT bit since HCA will generate/verify the PI */
  503. control &= ~NVME_RW_PRINFO_PRACT;
  504. cmd->rw.control = cpu_to_le16(control);
  505. /* PI is added by the HW */
  506. req->transfer_len += req->metadata_len;
  507. } else {
  508. /* for WRITE_PASS/READ_PASS both wire/memory domains exist */
  509. nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->wire, control,
  510. pi_type);
  511. nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->mem, control,
  512. pi_type);
  513. }
  514. if (control & NVME_RW_PRINFO_PRCHK_REF)
  515. sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG;
  516. if (control & NVME_RW_PRINFO_PRCHK_GUARD)
  517. sig_attrs->check_mask |= IB_SIG_CHECK_GUARD;
  518. if (control & NVME_RW_PRINFO_PRCHK_APP)
  519. sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG;
  520. }
  521. static int nvmet_rdma_rw_ctx_init(struct nvmet_rdma_rsp *rsp, u64 addr, u32 key,
  522. struct ib_sig_attrs *sig_attrs)
  523. {
  524. struct rdma_cm_id *cm_id = rsp->queue->cm_id;
  525. struct nvmet_req *req = &rsp->req;
  526. int ret;
  527. if (req->metadata_len)
  528. ret = rdma_rw_ctx_signature_init(&rsp->rw, cm_id->qp,
  529. cm_id->port_num, req->sg, req->sg_cnt,
  530. req->metadata_sg, req->metadata_sg_cnt, sig_attrs,
  531. addr, key, nvmet_data_dir(req));
  532. else
  533. ret = rdma_rw_ctx_init(&rsp->rw, cm_id->qp, cm_id->port_num,
  534. req->sg, req->sg_cnt, 0, addr, key,
  535. nvmet_data_dir(req));
  536. return ret;
  537. }
  538. static void nvmet_rdma_rw_ctx_destroy(struct nvmet_rdma_rsp *rsp)
  539. {
  540. struct rdma_cm_id *cm_id = rsp->queue->cm_id;
  541. struct nvmet_req *req = &rsp->req;
  542. if (req->metadata_len)
  543. rdma_rw_ctx_destroy_signature(&rsp->rw, cm_id->qp,
  544. cm_id->port_num, req->sg, req->sg_cnt,
  545. req->metadata_sg, req->metadata_sg_cnt,
  546. nvmet_data_dir(req));
  547. else
  548. rdma_rw_ctx_destroy(&rsp->rw, cm_id->qp, cm_id->port_num,
  549. req->sg, req->sg_cnt, nvmet_data_dir(req));
  550. }
  551. static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp)
  552. {
  553. struct nvmet_rdma_queue *queue = rsp->queue;
  554. atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail);
  555. if (rsp->n_rdma)
  556. nvmet_rdma_rw_ctx_destroy(rsp);
  557. if (rsp->req.sg != rsp->cmd->inline_sg)
  558. nvmet_req_free_sgls(&rsp->req);
  559. if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list)))
  560. nvmet_rdma_process_wr_wait_list(queue);
  561. nvmet_rdma_put_rsp(rsp);
  562. }
  563. static void nvmet_rdma_error_comp(struct nvmet_rdma_queue *queue)
  564. {
  565. if (queue->nvme_sq.ctrl) {
  566. nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl);
  567. } else {
  568. /*
  569. * we didn't setup the controller yet in case
  570. * of admin connect error, just disconnect and
  571. * cleanup the queue
  572. */
  573. nvmet_rdma_queue_disconnect(queue);
  574. }
  575. }
  576. static void nvmet_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc)
  577. {
  578. struct nvmet_rdma_rsp *rsp =
  579. container_of(wc->wr_cqe, struct nvmet_rdma_rsp, send_cqe);
  580. struct nvmet_rdma_queue *queue = wc->qp->qp_context;
  581. nvmet_rdma_release_rsp(rsp);
  582. if (unlikely(wc->status != IB_WC_SUCCESS &&
  583. wc->status != IB_WC_WR_FLUSH_ERR)) {
  584. pr_err("SEND for CQE 0x%p failed with status %s (%d).\n",
  585. wc->wr_cqe, ib_wc_status_msg(wc->status), wc->status);
  586. nvmet_rdma_error_comp(queue);
  587. }
  588. }
  589. static void nvmet_rdma_queue_response(struct nvmet_req *req)
  590. {
  591. struct nvmet_rdma_rsp *rsp =
  592. container_of(req, struct nvmet_rdma_rsp, req);
  593. struct rdma_cm_id *cm_id = rsp->queue->cm_id;
  594. struct ib_send_wr *first_wr;
  595. if (rsp->flags & NVMET_RDMA_REQ_INVALIDATE_RKEY) {
  596. rsp->send_wr.opcode = IB_WR_SEND_WITH_INV;
  597. rsp->send_wr.ex.invalidate_rkey = rsp->invalidate_rkey;
  598. } else {
  599. rsp->send_wr.opcode = IB_WR_SEND;
  600. }
  601. if (nvmet_rdma_need_data_out(rsp)) {
  602. if (rsp->req.metadata_len)
  603. first_wr = rdma_rw_ctx_wrs(&rsp->rw, cm_id->qp,
  604. cm_id->port_num, &rsp->write_cqe, NULL);
  605. else
  606. first_wr = rdma_rw_ctx_wrs(&rsp->rw, cm_id->qp,
  607. cm_id->port_num, NULL, &rsp->send_wr);
  608. } else {
  609. first_wr = &rsp->send_wr;
  610. }
  611. nvmet_rdma_post_recv(rsp->queue->dev, rsp->cmd);
  612. ib_dma_sync_single_for_device(rsp->queue->dev->device,
  613. rsp->send_sge.addr, rsp->send_sge.length,
  614. DMA_TO_DEVICE);
  615. if (unlikely(ib_post_send(cm_id->qp, first_wr, NULL))) {
  616. pr_err("sending cmd response failed\n");
  617. nvmet_rdma_release_rsp(rsp);
  618. }
  619. }
  620. static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc)
  621. {
  622. struct nvmet_rdma_rsp *rsp =
  623. container_of(wc->wr_cqe, struct nvmet_rdma_rsp, read_cqe);
  624. struct nvmet_rdma_queue *queue = wc->qp->qp_context;
  625. u16 status = 0;
  626. WARN_ON(rsp->n_rdma <= 0);
  627. atomic_add(rsp->n_rdma, &queue->sq_wr_avail);
  628. rsp->n_rdma = 0;
  629. if (unlikely(wc->status != IB_WC_SUCCESS)) {
  630. nvmet_rdma_rw_ctx_destroy(rsp);
  631. nvmet_req_uninit(&rsp->req);
  632. nvmet_rdma_release_rsp(rsp);
  633. if (wc->status != IB_WC_WR_FLUSH_ERR) {
  634. pr_info("RDMA READ for CQE 0x%p failed with status %s (%d).\n",
  635. wc->wr_cqe, ib_wc_status_msg(wc->status), wc->status);
  636. nvmet_rdma_error_comp(queue);
  637. }
  638. return;
  639. }
  640. if (rsp->req.metadata_len)
  641. status = nvmet_rdma_check_pi_status(rsp->rw.reg->mr);
  642. nvmet_rdma_rw_ctx_destroy(rsp);
  643. if (unlikely(status))
  644. nvmet_req_complete(&rsp->req, status);
  645. else
  646. rsp->req.execute(&rsp->req);
  647. }
  648. static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc)
  649. {
  650. struct nvmet_rdma_rsp *rsp =
  651. container_of(wc->wr_cqe, struct nvmet_rdma_rsp, write_cqe);
  652. struct nvmet_rdma_queue *queue = wc->qp->qp_context;
  653. struct rdma_cm_id *cm_id = rsp->queue->cm_id;
  654. u16 status;
  655. if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY))
  656. return;
  657. WARN_ON(rsp->n_rdma <= 0);
  658. atomic_add(rsp->n_rdma, &queue->sq_wr_avail);
  659. rsp->n_rdma = 0;
  660. if (unlikely(wc->status != IB_WC_SUCCESS)) {
  661. nvmet_rdma_rw_ctx_destroy(rsp);
  662. nvmet_req_uninit(&rsp->req);
  663. nvmet_rdma_release_rsp(rsp);
  664. if (wc->status != IB_WC_WR_FLUSH_ERR) {
  665. pr_info("RDMA WRITE for CQE failed with status %s (%d).\n",
  666. ib_wc_status_msg(wc->status), wc->status);
  667. nvmet_rdma_error_comp(queue);
  668. }
  669. return;
  670. }
  671. /*
  672. * Upon RDMA completion check the signature status
  673. * - if succeeded send good NVMe response
  674. * - if failed send bad NVMe response with appropriate error
  675. */
  676. status = nvmet_rdma_check_pi_status(rsp->rw.reg->mr);
  677. if (unlikely(status))
  678. rsp->req.cqe->status = cpu_to_le16(status << 1);
  679. nvmet_rdma_rw_ctx_destroy(rsp);
  680. if (unlikely(ib_post_send(cm_id->qp, &rsp->send_wr, NULL))) {
  681. pr_err("sending cmd response failed\n");
  682. nvmet_rdma_release_rsp(rsp);
  683. }
  684. }
  685. static void nvmet_rdma_use_inline_sg(struct nvmet_rdma_rsp *rsp, u32 len,
  686. u64 off)
  687. {
  688. int sg_count = num_pages(len);
  689. struct scatterlist *sg;
  690. int i;
  691. sg = rsp->cmd->inline_sg;
  692. for (i = 0; i < sg_count; i++, sg++) {
  693. if (i < sg_count - 1)
  694. sg_unmark_end(sg);
  695. else
  696. sg_mark_end(sg);
  697. sg->offset = off;
  698. sg->length = min_t(int, len, PAGE_SIZE - off);
  699. len -= sg->length;
  700. if (!i)
  701. off = 0;
  702. }
  703. rsp->req.sg = rsp->cmd->inline_sg;
  704. rsp->req.sg_cnt = sg_count;
  705. }
  706. static u16 nvmet_rdma_map_sgl_inline(struct nvmet_rdma_rsp *rsp)
  707. {
  708. struct nvme_sgl_desc *sgl = &rsp->req.cmd->common.dptr.sgl;
  709. u64 off = le64_to_cpu(sgl->addr);
  710. u32 len = le32_to_cpu(sgl->length);
  711. if (!nvme_is_write(rsp->req.cmd)) {
  712. rsp->req.error_loc =
  713. offsetof(struct nvme_common_command, opcode);
  714. return NVME_SC_INVALID_FIELD | NVME_SC_DNR;
  715. }
  716. if (off + len > rsp->queue->dev->inline_data_size) {
  717. pr_err("invalid inline data offset!\n");
  718. return NVME_SC_SGL_INVALID_OFFSET | NVME_SC_DNR;
  719. }
  720. /* no data command? */
  721. if (!len)
  722. return 0;
  723. nvmet_rdma_use_inline_sg(rsp, len, off);
  724. rsp->flags |= NVMET_RDMA_REQ_INLINE_DATA;
  725. rsp->req.transfer_len += len;
  726. return 0;
  727. }
  728. static u16 nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp *rsp,
  729. struct nvme_keyed_sgl_desc *sgl, bool invalidate)
  730. {
  731. u64 addr = le64_to_cpu(sgl->addr);
  732. u32 key = get_unaligned_le32(sgl->key);
  733. struct ib_sig_attrs sig_attrs;
  734. int ret;
  735. rsp->req.transfer_len = get_unaligned_le24(sgl->length);
  736. /* no data command? */
  737. if (!rsp->req.transfer_len)
  738. return 0;
  739. if (rsp->req.metadata_len)
  740. nvmet_rdma_set_sig_attrs(&rsp->req, &sig_attrs);
  741. ret = nvmet_req_alloc_sgls(&rsp->req);
  742. if (unlikely(ret < 0))
  743. goto error_out;
  744. ret = nvmet_rdma_rw_ctx_init(rsp, addr, key, &sig_attrs);
  745. if (unlikely(ret < 0))
  746. goto error_out;
  747. rsp->n_rdma += ret;
  748. if (invalidate) {
  749. rsp->invalidate_rkey = key;
  750. rsp->flags |= NVMET_RDMA_REQ_INVALIDATE_RKEY;
  751. }
  752. return 0;
  753. error_out:
  754. rsp->req.transfer_len = 0;
  755. return NVME_SC_INTERNAL;
  756. }
  757. static u16 nvmet_rdma_map_sgl(struct nvmet_rdma_rsp *rsp)
  758. {
  759. struct nvme_keyed_sgl_desc *sgl = &rsp->req.cmd->common.dptr.ksgl;
  760. switch (sgl->type >> 4) {
  761. case NVME_SGL_FMT_DATA_DESC:
  762. switch (sgl->type & 0xf) {
  763. case NVME_SGL_FMT_OFFSET:
  764. return nvmet_rdma_map_sgl_inline(rsp);
  765. default:
  766. pr_err("invalid SGL subtype: %#x\n", sgl->type);
  767. rsp->req.error_loc =
  768. offsetof(struct nvme_common_command, dptr);
  769. return NVME_SC_INVALID_FIELD | NVME_SC_DNR;
  770. }
  771. case NVME_KEY_SGL_FMT_DATA_DESC:
  772. switch (sgl->type & 0xf) {
  773. case NVME_SGL_FMT_ADDRESS | NVME_SGL_FMT_INVALIDATE:
  774. return nvmet_rdma_map_sgl_keyed(rsp, sgl, true);
  775. case NVME_SGL_FMT_ADDRESS:
  776. return nvmet_rdma_map_sgl_keyed(rsp, sgl, false);
  777. default:
  778. pr_err("invalid SGL subtype: %#x\n", sgl->type);
  779. rsp->req.error_loc =
  780. offsetof(struct nvme_common_command, dptr);
  781. return NVME_SC_INVALID_FIELD | NVME_SC_DNR;
  782. }
  783. default:
  784. pr_err("invalid SGL type: %#x\n", sgl->type);
  785. rsp->req.error_loc = offsetof(struct nvme_common_command, dptr);
  786. return NVME_SC_SGL_INVALID_TYPE | NVME_SC_DNR;
  787. }
  788. }
  789. static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp)
  790. {
  791. struct nvmet_rdma_queue *queue = rsp->queue;
  792. if (unlikely(atomic_sub_return(1 + rsp->n_rdma,
  793. &queue->sq_wr_avail) < 0)) {
  794. pr_debug("IB send queue full (needed %d): queue %u cntlid %u\n",
  795. 1 + rsp->n_rdma, queue->idx,
  796. queue->nvme_sq.ctrl->cntlid);
  797. atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail);
  798. return false;
  799. }
  800. if (nvmet_rdma_need_data_in(rsp)) {
  801. if (rdma_rw_ctx_post(&rsp->rw, queue->qp,
  802. queue->cm_id->port_num, &rsp->read_cqe, NULL))
  803. nvmet_req_complete(&rsp->req, NVME_SC_DATA_XFER_ERROR);
  804. } else {
  805. rsp->req.execute(&rsp->req);
  806. }
  807. return true;
  808. }
  809. static void nvmet_rdma_handle_command(struct nvmet_rdma_queue *queue,
  810. struct nvmet_rdma_rsp *cmd)
  811. {
  812. u16 status;
  813. ib_dma_sync_single_for_cpu(queue->dev->device,
  814. cmd->cmd->sge[0].addr, cmd->cmd->sge[0].length,
  815. DMA_FROM_DEVICE);
  816. ib_dma_sync_single_for_cpu(queue->dev->device,
  817. cmd->send_sge.addr, cmd->send_sge.length,
  818. DMA_TO_DEVICE);
  819. if (!nvmet_req_init(&cmd->req, &queue->nvme_cq,
  820. &queue->nvme_sq, &nvmet_rdma_ops))
  821. return;
  822. status = nvmet_rdma_map_sgl(cmd);
  823. if (status)
  824. goto out_err;
  825. if (unlikely(!nvmet_rdma_execute_command(cmd))) {
  826. spin_lock(&queue->rsp_wr_wait_lock);
  827. list_add_tail(&cmd->wait_list, &queue->rsp_wr_wait_list);
  828. spin_unlock(&queue->rsp_wr_wait_lock);
  829. }
  830. return;
  831. out_err:
  832. nvmet_req_complete(&cmd->req, status);
  833. }
  834. static void nvmet_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc)
  835. {
  836. struct nvmet_rdma_cmd *cmd =
  837. container_of(wc->wr_cqe, struct nvmet_rdma_cmd, cqe);
  838. struct nvmet_rdma_queue *queue = wc->qp->qp_context;
  839. struct nvmet_rdma_rsp *rsp;
  840. if (unlikely(wc->status != IB_WC_SUCCESS)) {
  841. if (wc->status != IB_WC_WR_FLUSH_ERR) {
  842. pr_err("RECV for CQE 0x%p failed with status %s (%d)\n",
  843. wc->wr_cqe, ib_wc_status_msg(wc->status),
  844. wc->status);
  845. nvmet_rdma_error_comp(queue);
  846. }
  847. return;
  848. }
  849. if (unlikely(wc->byte_len < sizeof(struct nvme_command))) {
  850. pr_err("Ctrl Fatal Error: capsule size less than 64 bytes\n");
  851. nvmet_rdma_error_comp(queue);
  852. return;
  853. }
  854. cmd->queue = queue;
  855. rsp = nvmet_rdma_get_rsp(queue);
  856. if (unlikely(!rsp)) {
  857. /*
  858. * we get here only under memory pressure,
  859. * silently drop and have the host retry
  860. * as we can't even fail it.
  861. */
  862. nvmet_rdma_post_recv(queue->dev, cmd);
  863. return;
  864. }
  865. rsp->queue = queue;
  866. rsp->cmd = cmd;
  867. rsp->flags = 0;
  868. rsp->req.cmd = cmd->nvme_cmd;
  869. rsp->req.port = queue->port;
  870. rsp->n_rdma = 0;
  871. if (unlikely(queue->state != NVMET_RDMA_Q_LIVE)) {
  872. unsigned long flags;
  873. spin_lock_irqsave(&queue->state_lock, flags);
  874. if (queue->state == NVMET_RDMA_Q_CONNECTING)
  875. list_add_tail(&rsp->wait_list, &queue->rsp_wait_list);
  876. else
  877. nvmet_rdma_put_rsp(rsp);
  878. spin_unlock_irqrestore(&queue->state_lock, flags);
  879. return;
  880. }
  881. nvmet_rdma_handle_command(queue, rsp);
  882. }
  883. static void nvmet_rdma_destroy_srq(struct nvmet_rdma_srq *nsrq)
  884. {
  885. nvmet_rdma_free_cmds(nsrq->ndev, nsrq->cmds, nsrq->ndev->srq_size,
  886. false);
  887. ib_destroy_srq(nsrq->srq);
  888. kfree(nsrq);
  889. }
  890. static void nvmet_rdma_destroy_srqs(struct nvmet_rdma_device *ndev)
  891. {
  892. int i;
  893. if (!ndev->srqs)
  894. return;
  895. for (i = 0; i < ndev->srq_count; i++)
  896. nvmet_rdma_destroy_srq(ndev->srqs[i]);
  897. kfree(ndev->srqs);
  898. }
  899. static struct nvmet_rdma_srq *
  900. nvmet_rdma_init_srq(struct nvmet_rdma_device *ndev)
  901. {
  902. struct ib_srq_init_attr srq_attr = { NULL, };
  903. size_t srq_size = ndev->srq_size;
  904. struct nvmet_rdma_srq *nsrq;
  905. struct ib_srq *srq;
  906. int ret, i;
  907. nsrq = kzalloc(sizeof(*nsrq), GFP_KERNEL);
  908. if (!nsrq)
  909. return ERR_PTR(-ENOMEM);
  910. srq_attr.attr.max_wr = srq_size;
  911. srq_attr.attr.max_sge = 1 + ndev->inline_page_count;
  912. srq_attr.attr.srq_limit = 0;
  913. srq_attr.srq_type = IB_SRQT_BASIC;
  914. srq = ib_create_srq(ndev->pd, &srq_attr);
  915. if (IS_ERR(srq)) {
  916. ret = PTR_ERR(srq);
  917. goto out_free;
  918. }
  919. nsrq->cmds = nvmet_rdma_alloc_cmds(ndev, srq_size, false);
  920. if (IS_ERR(nsrq->cmds)) {
  921. ret = PTR_ERR(nsrq->cmds);
  922. goto out_destroy_srq;
  923. }
  924. nsrq->srq = srq;
  925. nsrq->ndev = ndev;
  926. for (i = 0; i < srq_size; i++) {
  927. nsrq->cmds[i].nsrq = nsrq;
  928. ret = nvmet_rdma_post_recv(ndev, &nsrq->cmds[i]);
  929. if (ret)
  930. goto out_free_cmds;
  931. }
  932. return nsrq;
  933. out_free_cmds:
  934. nvmet_rdma_free_cmds(ndev, nsrq->cmds, srq_size, false);
  935. out_destroy_srq:
  936. ib_destroy_srq(srq);
  937. out_free:
  938. kfree(nsrq);
  939. return ERR_PTR(ret);
  940. }
  941. static int nvmet_rdma_init_srqs(struct nvmet_rdma_device *ndev)
  942. {
  943. int i, ret;
  944. if (!ndev->device->attrs.max_srq_wr || !ndev->device->attrs.max_srq) {
  945. /*
  946. * If SRQs aren't supported we just go ahead and use normal
  947. * non-shared receive queues.
  948. */
  949. pr_info("SRQ requested but not supported.\n");
  950. return 0;
  951. }
  952. ndev->srq_size = min(ndev->device->attrs.max_srq_wr,
  953. nvmet_rdma_srq_size);
  954. ndev->srq_count = min(ndev->device->num_comp_vectors,
  955. ndev->device->attrs.max_srq);
  956. ndev->srqs = kcalloc(ndev->srq_count, sizeof(*ndev->srqs), GFP_KERNEL);
  957. if (!ndev->srqs)
  958. return -ENOMEM;
  959. for (i = 0; i < ndev->srq_count; i++) {
  960. ndev->srqs[i] = nvmet_rdma_init_srq(ndev);
  961. if (IS_ERR(ndev->srqs[i])) {
  962. ret = PTR_ERR(ndev->srqs[i]);
  963. goto err_srq;
  964. }
  965. }
  966. return 0;
  967. err_srq:
  968. while (--i >= 0)
  969. nvmet_rdma_destroy_srq(ndev->srqs[i]);
  970. kfree(ndev->srqs);
  971. return ret;
  972. }
  973. static void nvmet_rdma_free_dev(struct kref *ref)
  974. {
  975. struct nvmet_rdma_device *ndev =
  976. container_of(ref, struct nvmet_rdma_device, ref);
  977. mutex_lock(&device_list_mutex);
  978. list_del(&ndev->entry);
  979. mutex_unlock(&device_list_mutex);
  980. nvmet_rdma_destroy_srqs(ndev);
  981. ib_dealloc_pd(ndev->pd);
  982. kfree(ndev);
  983. }
  984. static struct nvmet_rdma_device *
  985. nvmet_rdma_find_get_device(struct rdma_cm_id *cm_id)
  986. {
  987. struct nvmet_rdma_port *port = cm_id->context;
  988. struct nvmet_port *nport = port->nport;
  989. struct nvmet_rdma_device *ndev;
  990. int inline_page_count;
  991. int inline_sge_count;
  992. int ret;
  993. mutex_lock(&device_list_mutex);
  994. list_for_each_entry(ndev, &device_list, entry) {
  995. if (ndev->device->node_guid == cm_id->device->node_guid &&
  996. kref_get_unless_zero(&ndev->ref))
  997. goto out_unlock;
  998. }
  999. ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
  1000. if (!ndev)
  1001. goto out_err;
  1002. inline_page_count = num_pages(nport->inline_data_size);
  1003. inline_sge_count = max(cm_id->device->attrs.max_sge_rd,
  1004. cm_id->device->attrs.max_recv_sge) - 1;
  1005. if (inline_page_count > inline_sge_count) {
  1006. pr_warn("inline_data_size %d cannot be supported by device %s. Reducing to %lu.\n",
  1007. nport->inline_data_size, cm_id->device->name,
  1008. inline_sge_count * PAGE_SIZE);
  1009. nport->inline_data_size = inline_sge_count * PAGE_SIZE;
  1010. inline_page_count = inline_sge_count;
  1011. }
  1012. ndev->inline_data_size = nport->inline_data_size;
  1013. ndev->inline_page_count = inline_page_count;
  1014. if (nport->pi_enable && !(cm_id->device->attrs.device_cap_flags &
  1015. IB_DEVICE_INTEGRITY_HANDOVER)) {
  1016. pr_warn("T10-PI is not supported by device %s. Disabling it\n",
  1017. cm_id->device->name);
  1018. nport->pi_enable = false;
  1019. }
  1020. ndev->device = cm_id->device;
  1021. kref_init(&ndev->ref);
  1022. ndev->pd = ib_alloc_pd(ndev->device, 0);
  1023. if (IS_ERR(ndev->pd))
  1024. goto out_free_dev;
  1025. if (nvmet_rdma_use_srq) {
  1026. ret = nvmet_rdma_init_srqs(ndev);
  1027. if (ret)
  1028. goto out_free_pd;
  1029. }
  1030. list_add(&ndev->entry, &device_list);
  1031. out_unlock:
  1032. mutex_unlock(&device_list_mutex);
  1033. pr_debug("added %s.\n", ndev->device->name);
  1034. return ndev;
  1035. out_free_pd:
  1036. ib_dealloc_pd(ndev->pd);
  1037. out_free_dev:
  1038. kfree(ndev);
  1039. out_err:
  1040. mutex_unlock(&device_list_mutex);
  1041. return NULL;
  1042. }
  1043. static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue *queue)
  1044. {
  1045. struct ib_qp_init_attr qp_attr;
  1046. struct nvmet_rdma_device *ndev = queue->dev;
  1047. int nr_cqe, ret, i, factor;
  1048. /*
  1049. * Reserve CQ slots for RECV + RDMA_READ/RDMA_WRITE + RDMA_SEND.
  1050. */
  1051. nr_cqe = queue->recv_queue_size + 2 * queue->send_queue_size;
  1052. queue->cq = ib_cq_pool_get(ndev->device, nr_cqe + 1,
  1053. queue->comp_vector, IB_POLL_WORKQUEUE);
  1054. if (IS_ERR(queue->cq)) {
  1055. ret = PTR_ERR(queue->cq);
  1056. pr_err("failed to create CQ cqe= %d ret= %d\n",
  1057. nr_cqe + 1, ret);
  1058. goto out;
  1059. }
  1060. memset(&qp_attr, 0, sizeof(qp_attr));
  1061. qp_attr.qp_context = queue;
  1062. qp_attr.event_handler = nvmet_rdma_qp_event;
  1063. qp_attr.send_cq = queue->cq;
  1064. qp_attr.recv_cq = queue->cq;
  1065. qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
  1066. qp_attr.qp_type = IB_QPT_RC;
  1067. /* +1 for drain */
  1068. qp_attr.cap.max_send_wr = queue->send_queue_size + 1;
  1069. factor = rdma_rw_mr_factor(ndev->device, queue->cm_id->port_num,
  1070. 1 << NVMET_RDMA_MAX_MDTS);
  1071. qp_attr.cap.max_rdma_ctxs = queue->send_queue_size * factor;
  1072. qp_attr.cap.max_send_sge = max(ndev->device->attrs.max_sge_rd,
  1073. ndev->device->attrs.max_send_sge);
  1074. if (queue->nsrq) {
  1075. qp_attr.srq = queue->nsrq->srq;
  1076. } else {
  1077. /* +1 for drain */
  1078. qp_attr.cap.max_recv_wr = 1 + queue->recv_queue_size;
  1079. qp_attr.cap.max_recv_sge = 1 + ndev->inline_page_count;
  1080. }
  1081. if (queue->port->pi_enable && queue->host_qid)
  1082. qp_attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN;
  1083. ret = rdma_create_qp(queue->cm_id, ndev->pd, &qp_attr);
  1084. if (ret) {
  1085. pr_err("failed to create_qp ret= %d\n", ret);
  1086. goto err_destroy_cq;
  1087. }
  1088. queue->qp = queue->cm_id->qp;
  1089. atomic_set(&queue->sq_wr_avail, qp_attr.cap.max_send_wr);
  1090. pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
  1091. __func__, queue->cq->cqe, qp_attr.cap.max_send_sge,
  1092. qp_attr.cap.max_send_wr, queue->cm_id);
  1093. if (!queue->nsrq) {
  1094. for (i = 0; i < queue->recv_queue_size; i++) {
  1095. queue->cmds[i].queue = queue;
  1096. ret = nvmet_rdma_post_recv(ndev, &queue->cmds[i]);
  1097. if (ret)
  1098. goto err_destroy_qp;
  1099. }
  1100. }
  1101. out:
  1102. return ret;
  1103. err_destroy_qp:
  1104. rdma_destroy_qp(queue->cm_id);
  1105. err_destroy_cq:
  1106. ib_cq_pool_put(queue->cq, nr_cqe + 1);
  1107. goto out;
  1108. }
  1109. static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue *queue)
  1110. {
  1111. ib_drain_qp(queue->qp);
  1112. if (queue->cm_id)
  1113. rdma_destroy_id(queue->cm_id);
  1114. ib_destroy_qp(queue->qp);
  1115. ib_cq_pool_put(queue->cq, queue->recv_queue_size + 2 *
  1116. queue->send_queue_size + 1);
  1117. }
  1118. static void nvmet_rdma_free_queue(struct nvmet_rdma_queue *queue)
  1119. {
  1120. pr_debug("freeing queue %d\n", queue->idx);
  1121. nvmet_sq_destroy(&queue->nvme_sq);
  1122. nvmet_rdma_destroy_queue_ib(queue);
  1123. if (!queue->nsrq) {
  1124. nvmet_rdma_free_cmds(queue->dev, queue->cmds,
  1125. queue->recv_queue_size,
  1126. !queue->host_qid);
  1127. }
  1128. nvmet_rdma_free_rsps(queue);
  1129. ida_simple_remove(&nvmet_rdma_queue_ida, queue->idx);
  1130. kfree(queue);
  1131. }
  1132. static void nvmet_rdma_release_queue_work(struct work_struct *w)
  1133. {
  1134. struct nvmet_rdma_queue *queue =
  1135. container_of(w, struct nvmet_rdma_queue, release_work);
  1136. struct nvmet_rdma_device *dev = queue->dev;
  1137. nvmet_rdma_free_queue(queue);
  1138. kref_put(&dev->ref, nvmet_rdma_free_dev);
  1139. }
  1140. static int
  1141. nvmet_rdma_parse_cm_connect_req(struct rdma_conn_param *conn,
  1142. struct nvmet_rdma_queue *queue)
  1143. {
  1144. struct nvme_rdma_cm_req *req;
  1145. req = (struct nvme_rdma_cm_req *)conn->private_data;
  1146. if (!req || conn->private_data_len == 0)
  1147. return NVME_RDMA_CM_INVALID_LEN;
  1148. if (le16_to_cpu(req->recfmt) != NVME_RDMA_CM_FMT_1_0)
  1149. return NVME_RDMA_CM_INVALID_RECFMT;
  1150. queue->host_qid = le16_to_cpu(req->qid);
  1151. /*
  1152. * req->hsqsize corresponds to our recv queue size plus 1
  1153. * req->hrqsize corresponds to our send queue size
  1154. */
  1155. queue->recv_queue_size = le16_to_cpu(req->hsqsize) + 1;
  1156. queue->send_queue_size = le16_to_cpu(req->hrqsize);
  1157. if (!queue->host_qid && queue->recv_queue_size > NVME_AQ_DEPTH)
  1158. return NVME_RDMA_CM_INVALID_HSQSIZE;
  1159. /* XXX: Should we enforce some kind of max for IO queues? */
  1160. return 0;
  1161. }
  1162. static int nvmet_rdma_cm_reject(struct rdma_cm_id *cm_id,
  1163. enum nvme_rdma_cm_status status)
  1164. {
  1165. struct nvme_rdma_cm_rej rej;
  1166. pr_debug("rejecting connect request: status %d (%s)\n",
  1167. status, nvme_rdma_cm_msg(status));
  1168. rej.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
  1169. rej.sts = cpu_to_le16(status);
  1170. return rdma_reject(cm_id, (void *)&rej, sizeof(rej),
  1171. IB_CM_REJ_CONSUMER_DEFINED);
  1172. }
  1173. static struct nvmet_rdma_queue *
  1174. nvmet_rdma_alloc_queue(struct nvmet_rdma_device *ndev,
  1175. struct rdma_cm_id *cm_id,
  1176. struct rdma_cm_event *event)
  1177. {
  1178. struct nvmet_rdma_port *port = cm_id->context;
  1179. struct nvmet_rdma_queue *queue;
  1180. int ret;
  1181. queue = kzalloc(sizeof(*queue), GFP_KERNEL);
  1182. if (!queue) {
  1183. ret = NVME_RDMA_CM_NO_RSC;
  1184. goto out_reject;
  1185. }
  1186. ret = nvmet_sq_init(&queue->nvme_sq);
  1187. if (ret) {
  1188. ret = NVME_RDMA_CM_NO_RSC;
  1189. goto out_free_queue;
  1190. }
  1191. ret = nvmet_rdma_parse_cm_connect_req(&event->param.conn, queue);
  1192. if (ret)
  1193. goto out_destroy_sq;
  1194. /*
  1195. * Schedules the actual release because calling rdma_destroy_id from
  1196. * inside a CM callback would trigger a deadlock. (great API design..)
  1197. */
  1198. INIT_WORK(&queue->release_work, nvmet_rdma_release_queue_work);
  1199. queue->dev = ndev;
  1200. queue->cm_id = cm_id;
  1201. queue->port = port->nport;
  1202. spin_lock_init(&queue->state_lock);
  1203. queue->state = NVMET_RDMA_Q_CONNECTING;
  1204. INIT_LIST_HEAD(&queue->rsp_wait_list);
  1205. INIT_LIST_HEAD(&queue->rsp_wr_wait_list);
  1206. spin_lock_init(&queue->rsp_wr_wait_lock);
  1207. INIT_LIST_HEAD(&queue->free_rsps);
  1208. spin_lock_init(&queue->rsps_lock);
  1209. INIT_LIST_HEAD(&queue->queue_list);
  1210. queue->idx = ida_simple_get(&nvmet_rdma_queue_ida, 0, 0, GFP_KERNEL);
  1211. if (queue->idx < 0) {
  1212. ret = NVME_RDMA_CM_NO_RSC;
  1213. goto out_destroy_sq;
  1214. }
  1215. /*
  1216. * Spread the io queues across completion vectors,
  1217. * but still keep all admin queues on vector 0.
  1218. */
  1219. queue->comp_vector = !queue->host_qid ? 0 :
  1220. queue->idx % ndev->device->num_comp_vectors;
  1221. ret = nvmet_rdma_alloc_rsps(queue);
  1222. if (ret) {
  1223. ret = NVME_RDMA_CM_NO_RSC;
  1224. goto out_ida_remove;
  1225. }
  1226. if (ndev->srqs) {
  1227. queue->nsrq = ndev->srqs[queue->comp_vector % ndev->srq_count];
  1228. } else {
  1229. queue->cmds = nvmet_rdma_alloc_cmds(ndev,
  1230. queue->recv_queue_size,
  1231. !queue->host_qid);
  1232. if (IS_ERR(queue->cmds)) {
  1233. ret = NVME_RDMA_CM_NO_RSC;
  1234. goto out_free_responses;
  1235. }
  1236. }
  1237. ret = nvmet_rdma_create_queue_ib(queue);
  1238. if (ret) {
  1239. pr_err("%s: creating RDMA queue failed (%d).\n",
  1240. __func__, ret);
  1241. ret = NVME_RDMA_CM_NO_RSC;
  1242. goto out_free_cmds;
  1243. }
  1244. return queue;
  1245. out_free_cmds:
  1246. if (!queue->nsrq) {
  1247. nvmet_rdma_free_cmds(queue->dev, queue->cmds,
  1248. queue->recv_queue_size,
  1249. !queue->host_qid);
  1250. }
  1251. out_free_responses:
  1252. nvmet_rdma_free_rsps(queue);
  1253. out_ida_remove:
  1254. ida_simple_remove(&nvmet_rdma_queue_ida, queue->idx);
  1255. out_destroy_sq:
  1256. nvmet_sq_destroy(&queue->nvme_sq);
  1257. out_free_queue:
  1258. kfree(queue);
  1259. out_reject:
  1260. nvmet_rdma_cm_reject(cm_id, ret);
  1261. return NULL;
  1262. }
  1263. static void nvmet_rdma_qp_event(struct ib_event *event, void *priv)
  1264. {
  1265. struct nvmet_rdma_queue *queue = priv;
  1266. switch (event->event) {
  1267. case IB_EVENT_COMM_EST:
  1268. rdma_notify(queue->cm_id, event->event);
  1269. break;
  1270. case IB_EVENT_QP_LAST_WQE_REACHED:
  1271. pr_debug("received last WQE reached event for queue=0x%p\n",
  1272. queue);
  1273. break;
  1274. default:
  1275. pr_err("received IB QP event: %s (%d)\n",
  1276. ib_event_msg(event->event), event->event);
  1277. break;
  1278. }
  1279. }
  1280. static int nvmet_rdma_cm_accept(struct rdma_cm_id *cm_id,
  1281. struct nvmet_rdma_queue *queue,
  1282. struct rdma_conn_param *p)
  1283. {
  1284. struct rdma_conn_param param = { };
  1285. struct nvme_rdma_cm_rep priv = { };
  1286. int ret = -ENOMEM;
  1287. param.rnr_retry_count = 7;
  1288. param.flow_control = 1;
  1289. param.initiator_depth = min_t(u8, p->initiator_depth,
  1290. queue->dev->device->attrs.max_qp_init_rd_atom);
  1291. param.private_data = &priv;
  1292. param.private_data_len = sizeof(priv);
  1293. priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
  1294. priv.crqsize = cpu_to_le16(queue->recv_queue_size);
  1295. ret = rdma_accept(cm_id, &param);
  1296. if (ret)
  1297. pr_err("rdma_accept failed (error code = %d)\n", ret);
  1298. return ret;
  1299. }
  1300. static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id,
  1301. struct rdma_cm_event *event)
  1302. {
  1303. struct nvmet_rdma_device *ndev;
  1304. struct nvmet_rdma_queue *queue;
  1305. int ret = -EINVAL;
  1306. ndev = nvmet_rdma_find_get_device(cm_id);
  1307. if (!ndev) {
  1308. nvmet_rdma_cm_reject(cm_id, NVME_RDMA_CM_NO_RSC);
  1309. return -ECONNREFUSED;
  1310. }
  1311. queue = nvmet_rdma_alloc_queue(ndev, cm_id, event);
  1312. if (!queue) {
  1313. ret = -ENOMEM;
  1314. goto put_device;
  1315. }
  1316. if (queue->host_qid == 0) {
  1317. /* Let inflight controller teardown complete */
  1318. flush_scheduled_work();
  1319. }
  1320. ret = nvmet_rdma_cm_accept(cm_id, queue, &event->param.conn);
  1321. if (ret) {
  1322. /*
  1323. * Don't destroy the cm_id in free path, as we implicitly
  1324. * destroy the cm_id here with non-zero ret code.
  1325. */
  1326. queue->cm_id = NULL;
  1327. goto free_queue;
  1328. }
  1329. mutex_lock(&nvmet_rdma_queue_mutex);
  1330. list_add_tail(&queue->queue_list, &nvmet_rdma_queue_list);
  1331. mutex_unlock(&nvmet_rdma_queue_mutex);
  1332. return 0;
  1333. free_queue:
  1334. nvmet_rdma_free_queue(queue);
  1335. put_device:
  1336. kref_put(&ndev->ref, nvmet_rdma_free_dev);
  1337. return ret;
  1338. }
  1339. static void nvmet_rdma_queue_established(struct nvmet_rdma_queue *queue)
  1340. {
  1341. unsigned long flags;
  1342. spin_lock_irqsave(&queue->state_lock, flags);
  1343. if (queue->state != NVMET_RDMA_Q_CONNECTING) {
  1344. pr_warn("trying to establish a connected queue\n");
  1345. goto out_unlock;
  1346. }
  1347. queue->state = NVMET_RDMA_Q_LIVE;
  1348. while (!list_empty(&queue->rsp_wait_list)) {
  1349. struct nvmet_rdma_rsp *cmd;
  1350. cmd = list_first_entry(&queue->rsp_wait_list,
  1351. struct nvmet_rdma_rsp, wait_list);
  1352. list_del(&cmd->wait_list);
  1353. spin_unlock_irqrestore(&queue->state_lock, flags);
  1354. nvmet_rdma_handle_command(queue, cmd);
  1355. spin_lock_irqsave(&queue->state_lock, flags);
  1356. }
  1357. out_unlock:
  1358. spin_unlock_irqrestore(&queue->state_lock, flags);
  1359. }
  1360. static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue)
  1361. {
  1362. bool disconnect = false;
  1363. unsigned long flags;
  1364. pr_debug("cm_id= %p queue->state= %d\n", queue->cm_id, queue->state);
  1365. spin_lock_irqsave(&queue->state_lock, flags);
  1366. switch (queue->state) {
  1367. case NVMET_RDMA_Q_CONNECTING:
  1368. while (!list_empty(&queue->rsp_wait_list)) {
  1369. struct nvmet_rdma_rsp *rsp;
  1370. rsp = list_first_entry(&queue->rsp_wait_list,
  1371. struct nvmet_rdma_rsp,
  1372. wait_list);
  1373. list_del(&rsp->wait_list);
  1374. nvmet_rdma_put_rsp(rsp);
  1375. }
  1376. fallthrough;
  1377. case NVMET_RDMA_Q_LIVE:
  1378. queue->state = NVMET_RDMA_Q_DISCONNECTING;
  1379. disconnect = true;
  1380. break;
  1381. case NVMET_RDMA_Q_DISCONNECTING:
  1382. break;
  1383. }
  1384. spin_unlock_irqrestore(&queue->state_lock, flags);
  1385. if (disconnect) {
  1386. rdma_disconnect(queue->cm_id);
  1387. schedule_work(&queue->release_work);
  1388. }
  1389. }
  1390. static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue)
  1391. {
  1392. bool disconnect = false;
  1393. mutex_lock(&nvmet_rdma_queue_mutex);
  1394. if (!list_empty(&queue->queue_list)) {
  1395. list_del_init(&queue->queue_list);
  1396. disconnect = true;
  1397. }
  1398. mutex_unlock(&nvmet_rdma_queue_mutex);
  1399. if (disconnect)
  1400. __nvmet_rdma_queue_disconnect(queue);
  1401. }
  1402. static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id,
  1403. struct nvmet_rdma_queue *queue)
  1404. {
  1405. WARN_ON_ONCE(queue->state != NVMET_RDMA_Q_CONNECTING);
  1406. mutex_lock(&nvmet_rdma_queue_mutex);
  1407. if (!list_empty(&queue->queue_list))
  1408. list_del_init(&queue->queue_list);
  1409. mutex_unlock(&nvmet_rdma_queue_mutex);
  1410. pr_err("failed to connect queue %d\n", queue->idx);
  1411. schedule_work(&queue->release_work);
  1412. }
  1413. /**
  1414. * nvme_rdma_device_removal() - Handle RDMA device removal
  1415. * @cm_id: rdma_cm id, used for nvmet port
  1416. * @queue: nvmet rdma queue (cm id qp_context)
  1417. *
  1418. * DEVICE_REMOVAL event notifies us that the RDMA device is about
  1419. * to unplug. Note that this event can be generated on a normal
  1420. * queue cm_id and/or a device bound listener cm_id (where in this
  1421. * case queue will be null).
  1422. *
  1423. * We registered an ib_client to handle device removal for queues,
  1424. * so we only need to handle the listening port cm_ids. In this case
  1425. * we nullify the priv to prevent double cm_id destruction and destroying
  1426. * the cm_id implicitely by returning a non-zero rc to the callout.
  1427. */
  1428. static int nvmet_rdma_device_removal(struct rdma_cm_id *cm_id,
  1429. struct nvmet_rdma_queue *queue)
  1430. {
  1431. struct nvmet_rdma_port *port;
  1432. if (queue) {
  1433. /*
  1434. * This is a queue cm_id. we have registered
  1435. * an ib_client to handle queues removal
  1436. * so don't interfear and just return.
  1437. */
  1438. return 0;
  1439. }
  1440. port = cm_id->context;
  1441. /*
  1442. * This is a listener cm_id. Make sure that
  1443. * future remove_port won't invoke a double
  1444. * cm_id destroy. use atomic xchg to make sure
  1445. * we don't compete with remove_port.
  1446. */
  1447. if (xchg(&port->cm_id, NULL) != cm_id)
  1448. return 0;
  1449. /*
  1450. * We need to return 1 so that the core will destroy
  1451. * it's own ID. What a great API design..
  1452. */
  1453. return 1;
  1454. }
  1455. static int nvmet_rdma_cm_handler(struct rdma_cm_id *cm_id,
  1456. struct rdma_cm_event *event)
  1457. {
  1458. struct nvmet_rdma_queue *queue = NULL;
  1459. int ret = 0;
  1460. if (cm_id->qp)
  1461. queue = cm_id->qp->qp_context;
  1462. pr_debug("%s (%d): status %d id %p\n",
  1463. rdma_event_msg(event->event), event->event,
  1464. event->status, cm_id);
  1465. switch (event->event) {
  1466. case RDMA_CM_EVENT_CONNECT_REQUEST:
  1467. ret = nvmet_rdma_queue_connect(cm_id, event);
  1468. break;
  1469. case RDMA_CM_EVENT_ESTABLISHED:
  1470. nvmet_rdma_queue_established(queue);
  1471. break;
  1472. case RDMA_CM_EVENT_ADDR_CHANGE:
  1473. if (!queue) {
  1474. struct nvmet_rdma_port *port = cm_id->context;
  1475. schedule_delayed_work(&port->repair_work, 0);
  1476. break;
  1477. }
  1478. fallthrough;
  1479. case RDMA_CM_EVENT_DISCONNECTED:
  1480. case RDMA_CM_EVENT_TIMEWAIT_EXIT:
  1481. nvmet_rdma_queue_disconnect(queue);
  1482. break;
  1483. case RDMA_CM_EVENT_DEVICE_REMOVAL:
  1484. ret = nvmet_rdma_device_removal(cm_id, queue);
  1485. break;
  1486. case RDMA_CM_EVENT_REJECTED:
  1487. pr_debug("Connection rejected: %s\n",
  1488. rdma_reject_msg(cm_id, event->status));
  1489. fallthrough;
  1490. case RDMA_CM_EVENT_UNREACHABLE:
  1491. case RDMA_CM_EVENT_CONNECT_ERROR:
  1492. nvmet_rdma_queue_connect_fail(cm_id, queue);
  1493. break;
  1494. default:
  1495. pr_err("received unrecognized RDMA CM event %d\n",
  1496. event->event);
  1497. break;
  1498. }
  1499. return ret;
  1500. }
  1501. static void nvmet_rdma_delete_ctrl(struct nvmet_ctrl *ctrl)
  1502. {
  1503. struct nvmet_rdma_queue *queue;
  1504. restart:
  1505. mutex_lock(&nvmet_rdma_queue_mutex);
  1506. list_for_each_entry(queue, &nvmet_rdma_queue_list, queue_list) {
  1507. if (queue->nvme_sq.ctrl == ctrl) {
  1508. list_del_init(&queue->queue_list);
  1509. mutex_unlock(&nvmet_rdma_queue_mutex);
  1510. __nvmet_rdma_queue_disconnect(queue);
  1511. goto restart;
  1512. }
  1513. }
  1514. mutex_unlock(&nvmet_rdma_queue_mutex);
  1515. }
  1516. static void nvmet_rdma_destroy_port_queues(struct nvmet_rdma_port *port)
  1517. {
  1518. struct nvmet_rdma_queue *queue, *tmp;
  1519. struct nvmet_port *nport = port->nport;
  1520. mutex_lock(&nvmet_rdma_queue_mutex);
  1521. list_for_each_entry_safe(queue, tmp, &nvmet_rdma_queue_list,
  1522. queue_list) {
  1523. if (queue->port != nport)
  1524. continue;
  1525. list_del_init(&queue->queue_list);
  1526. __nvmet_rdma_queue_disconnect(queue);
  1527. }
  1528. mutex_unlock(&nvmet_rdma_queue_mutex);
  1529. }
  1530. static void nvmet_rdma_disable_port(struct nvmet_rdma_port *port)
  1531. {
  1532. struct rdma_cm_id *cm_id = xchg(&port->cm_id, NULL);
  1533. if (cm_id)
  1534. rdma_destroy_id(cm_id);
  1535. /*
  1536. * Destroy the remaining queues, which are not belong to any
  1537. * controller yet. Do it here after the RDMA-CM was destroyed
  1538. * guarantees that no new queue will be created.
  1539. */
  1540. nvmet_rdma_destroy_port_queues(port);
  1541. }
  1542. static int nvmet_rdma_enable_port(struct nvmet_rdma_port *port)
  1543. {
  1544. struct sockaddr *addr = (struct sockaddr *)&port->addr;
  1545. struct rdma_cm_id *cm_id;
  1546. int ret;
  1547. cm_id = rdma_create_id(&init_net, nvmet_rdma_cm_handler, port,
  1548. RDMA_PS_TCP, IB_QPT_RC);
  1549. if (IS_ERR(cm_id)) {
  1550. pr_err("CM ID creation failed\n");
  1551. return PTR_ERR(cm_id);
  1552. }
  1553. /*
  1554. * Allow both IPv4 and IPv6 sockets to bind a single port
  1555. * at the same time.
  1556. */
  1557. ret = rdma_set_afonly(cm_id, 1);
  1558. if (ret) {
  1559. pr_err("rdma_set_afonly failed (%d)\n", ret);
  1560. goto out_destroy_id;
  1561. }
  1562. ret = rdma_bind_addr(cm_id, addr);
  1563. if (ret) {
  1564. pr_err("binding CM ID to %pISpcs failed (%d)\n", addr, ret);
  1565. goto out_destroy_id;
  1566. }
  1567. ret = rdma_listen(cm_id, 128);
  1568. if (ret) {
  1569. pr_err("listening to %pISpcs failed (%d)\n", addr, ret);
  1570. goto out_destroy_id;
  1571. }
  1572. port->cm_id = cm_id;
  1573. return 0;
  1574. out_destroy_id:
  1575. rdma_destroy_id(cm_id);
  1576. return ret;
  1577. }
  1578. static void nvmet_rdma_repair_port_work(struct work_struct *w)
  1579. {
  1580. struct nvmet_rdma_port *port = container_of(to_delayed_work(w),
  1581. struct nvmet_rdma_port, repair_work);
  1582. int ret;
  1583. nvmet_rdma_disable_port(port);
  1584. ret = nvmet_rdma_enable_port(port);
  1585. if (ret)
  1586. schedule_delayed_work(&port->repair_work, 5 * HZ);
  1587. }
  1588. static int nvmet_rdma_add_port(struct nvmet_port *nport)
  1589. {
  1590. struct nvmet_rdma_port *port;
  1591. __kernel_sa_family_t af;
  1592. int ret;
  1593. port = kzalloc(sizeof(*port), GFP_KERNEL);
  1594. if (!port)
  1595. return -ENOMEM;
  1596. nport->priv = port;
  1597. port->nport = nport;
  1598. INIT_DELAYED_WORK(&port->repair_work, nvmet_rdma_repair_port_work);
  1599. switch (nport->disc_addr.adrfam) {
  1600. case NVMF_ADDR_FAMILY_IP4:
  1601. af = AF_INET;
  1602. break;
  1603. case NVMF_ADDR_FAMILY_IP6:
  1604. af = AF_INET6;
  1605. break;
  1606. default:
  1607. pr_err("address family %d not supported\n",
  1608. nport->disc_addr.adrfam);
  1609. ret = -EINVAL;
  1610. goto out_free_port;
  1611. }
  1612. if (nport->inline_data_size < 0) {
  1613. nport->inline_data_size = NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE;
  1614. } else if (nport->inline_data_size > NVMET_RDMA_MAX_INLINE_DATA_SIZE) {
  1615. pr_warn("inline_data_size %u is too large, reducing to %u\n",
  1616. nport->inline_data_size,
  1617. NVMET_RDMA_MAX_INLINE_DATA_SIZE);
  1618. nport->inline_data_size = NVMET_RDMA_MAX_INLINE_DATA_SIZE;
  1619. }
  1620. ret = inet_pton_with_scope(&init_net, af, nport->disc_addr.traddr,
  1621. nport->disc_addr.trsvcid, &port->addr);
  1622. if (ret) {
  1623. pr_err("malformed ip/port passed: %s:%s\n",
  1624. nport->disc_addr.traddr, nport->disc_addr.trsvcid);
  1625. goto out_free_port;
  1626. }
  1627. ret = nvmet_rdma_enable_port(port);
  1628. if (ret)
  1629. goto out_free_port;
  1630. pr_info("enabling port %d (%pISpcs)\n",
  1631. le16_to_cpu(nport->disc_addr.portid),
  1632. (struct sockaddr *)&port->addr);
  1633. return 0;
  1634. out_free_port:
  1635. kfree(port);
  1636. return ret;
  1637. }
  1638. static void nvmet_rdma_remove_port(struct nvmet_port *nport)
  1639. {
  1640. struct nvmet_rdma_port *port = nport->priv;
  1641. cancel_delayed_work_sync(&port->repair_work);
  1642. nvmet_rdma_disable_port(port);
  1643. kfree(port);
  1644. }
  1645. static void nvmet_rdma_disc_port_addr(struct nvmet_req *req,
  1646. struct nvmet_port *nport, char *traddr)
  1647. {
  1648. struct nvmet_rdma_port *port = nport->priv;
  1649. struct rdma_cm_id *cm_id = port->cm_id;
  1650. if (inet_addr_is_any((struct sockaddr *)&cm_id->route.addr.src_addr)) {
  1651. struct nvmet_rdma_rsp *rsp =
  1652. container_of(req, struct nvmet_rdma_rsp, req);
  1653. struct rdma_cm_id *req_cm_id = rsp->queue->cm_id;
  1654. struct sockaddr *addr = (void *)&req_cm_id->route.addr.src_addr;
  1655. sprintf(traddr, "%pISc", addr);
  1656. } else {
  1657. memcpy(traddr, nport->disc_addr.traddr, NVMF_TRADDR_SIZE);
  1658. }
  1659. }
  1660. static u8 nvmet_rdma_get_mdts(const struct nvmet_ctrl *ctrl)
  1661. {
  1662. if (ctrl->pi_support)
  1663. return NVMET_RDMA_MAX_METADATA_MDTS;
  1664. return NVMET_RDMA_MAX_MDTS;
  1665. }
  1666. static const struct nvmet_fabrics_ops nvmet_rdma_ops = {
  1667. .owner = THIS_MODULE,
  1668. .type = NVMF_TRTYPE_RDMA,
  1669. .msdbd = 1,
  1670. .flags = NVMF_KEYED_SGLS | NVMF_METADATA_SUPPORTED,
  1671. .add_port = nvmet_rdma_add_port,
  1672. .remove_port = nvmet_rdma_remove_port,
  1673. .queue_response = nvmet_rdma_queue_response,
  1674. .delete_ctrl = nvmet_rdma_delete_ctrl,
  1675. .disc_traddr = nvmet_rdma_disc_port_addr,
  1676. .get_mdts = nvmet_rdma_get_mdts,
  1677. };
  1678. static void nvmet_rdma_remove_one(struct ib_device *ib_device, void *client_data)
  1679. {
  1680. struct nvmet_rdma_queue *queue, *tmp;
  1681. struct nvmet_rdma_device *ndev;
  1682. bool found = false;
  1683. mutex_lock(&device_list_mutex);
  1684. list_for_each_entry(ndev, &device_list, entry) {
  1685. if (ndev->device == ib_device) {
  1686. found = true;
  1687. break;
  1688. }
  1689. }
  1690. mutex_unlock(&device_list_mutex);
  1691. if (!found)
  1692. return;
  1693. /*
  1694. * IB Device that is used by nvmet controllers is being removed,
  1695. * delete all queues using this device.
  1696. */
  1697. mutex_lock(&nvmet_rdma_queue_mutex);
  1698. list_for_each_entry_safe(queue, tmp, &nvmet_rdma_queue_list,
  1699. queue_list) {
  1700. if (queue->dev->device != ib_device)
  1701. continue;
  1702. pr_info("Removing queue %d\n", queue->idx);
  1703. list_del_init(&queue->queue_list);
  1704. __nvmet_rdma_queue_disconnect(queue);
  1705. }
  1706. mutex_unlock(&nvmet_rdma_queue_mutex);
  1707. flush_scheduled_work();
  1708. }
  1709. static struct ib_client nvmet_rdma_ib_client = {
  1710. .name = "nvmet_rdma",
  1711. .remove = nvmet_rdma_remove_one
  1712. };
  1713. static int __init nvmet_rdma_init(void)
  1714. {
  1715. int ret;
  1716. ret = ib_register_client(&nvmet_rdma_ib_client);
  1717. if (ret)
  1718. return ret;
  1719. ret = nvmet_register_transport(&nvmet_rdma_ops);
  1720. if (ret)
  1721. goto err_ib_client;
  1722. return 0;
  1723. err_ib_client:
  1724. ib_unregister_client(&nvmet_rdma_ib_client);
  1725. return ret;
  1726. }
  1727. static void __exit nvmet_rdma_exit(void)
  1728. {
  1729. nvmet_unregister_transport(&nvmet_rdma_ops);
  1730. ib_unregister_client(&nvmet_rdma_ib_client);
  1731. WARN_ON_ONCE(!list_empty(&nvmet_rdma_queue_list));
  1732. ida_destroy(&nvmet_rdma_queue_ida);
  1733. }
  1734. module_init(nvmet_rdma_init);
  1735. module_exit(nvmet_rdma_exit);
  1736. MODULE_LICENSE("GPL v2");
  1737. MODULE_ALIAS("nvmet-transport-1"); /* 1 == NVMF_TRTYPE_RDMA */