tcp.c 44 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * NVMe over Fabrics TCP target.
  4. * Copyright (c) 2018 Lightbits Labs. All rights reserved.
  5. */
  6. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  7. #include <linux/module.h>
  8. #include <linux/init.h>
  9. #include <linux/slab.h>
  10. #include <linux/err.h>
  11. #include <linux/nvme-tcp.h>
  12. #include <net/sock.h>
  13. #include <net/tcp.h>
  14. #include <linux/inet.h>
  15. #include <linux/llist.h>
  16. #include <crypto/hash.h>
  17. #include "nvmet.h"
  18. #define NVMET_TCP_DEF_INLINE_DATA_SIZE (4 * PAGE_SIZE)
  19. /* Define the socket priority to use for connections were it is desirable
  20. * that the NIC consider performing optimized packet processing or filtering.
  21. * A non-zero value being sufficient to indicate general consideration of any
  22. * possible optimization. Making it a module param allows for alternative
  23. * values that may be unique for some NIC implementations.
  24. */
  25. static int so_priority;
  26. module_param(so_priority, int, 0644);
  27. MODULE_PARM_DESC(so_priority, "nvmet tcp socket optimize priority");
  28. #define NVMET_TCP_RECV_BUDGET 8
  29. #define NVMET_TCP_SEND_BUDGET 8
  30. #define NVMET_TCP_IO_WORK_BUDGET 64
  31. enum nvmet_tcp_send_state {
  32. NVMET_TCP_SEND_DATA_PDU,
  33. NVMET_TCP_SEND_DATA,
  34. NVMET_TCP_SEND_R2T,
  35. NVMET_TCP_SEND_DDGST,
  36. NVMET_TCP_SEND_RESPONSE
  37. };
  38. enum nvmet_tcp_recv_state {
  39. NVMET_TCP_RECV_PDU,
  40. NVMET_TCP_RECV_DATA,
  41. NVMET_TCP_RECV_DDGST,
  42. NVMET_TCP_RECV_ERR,
  43. };
  44. enum {
  45. NVMET_TCP_F_INIT_FAILED = (1 << 0),
  46. };
  47. struct nvmet_tcp_cmd {
  48. struct nvmet_tcp_queue *queue;
  49. struct nvmet_req req;
  50. struct nvme_tcp_cmd_pdu *cmd_pdu;
  51. struct nvme_tcp_rsp_pdu *rsp_pdu;
  52. struct nvme_tcp_data_pdu *data_pdu;
  53. struct nvme_tcp_r2t_pdu *r2t_pdu;
  54. u32 rbytes_done;
  55. u32 wbytes_done;
  56. u32 pdu_len;
  57. u32 pdu_recv;
  58. int sg_idx;
  59. int nr_mapped;
  60. struct msghdr recv_msg;
  61. struct kvec *iov;
  62. u32 flags;
  63. struct list_head entry;
  64. struct llist_node lentry;
  65. /* send state */
  66. u32 offset;
  67. struct scatterlist *cur_sg;
  68. enum nvmet_tcp_send_state state;
  69. __le32 exp_ddgst;
  70. __le32 recv_ddgst;
  71. };
  72. enum nvmet_tcp_queue_state {
  73. NVMET_TCP_Q_CONNECTING,
  74. NVMET_TCP_Q_LIVE,
  75. NVMET_TCP_Q_DISCONNECTING,
  76. };
  77. struct nvmet_tcp_queue {
  78. struct socket *sock;
  79. struct nvmet_tcp_port *port;
  80. struct work_struct io_work;
  81. struct nvmet_cq nvme_cq;
  82. struct nvmet_sq nvme_sq;
  83. /* send state */
  84. struct nvmet_tcp_cmd *cmds;
  85. unsigned int nr_cmds;
  86. struct list_head free_list;
  87. struct llist_head resp_list;
  88. struct list_head resp_send_list;
  89. int send_list_len;
  90. struct nvmet_tcp_cmd *snd_cmd;
  91. /* recv state */
  92. int offset;
  93. int left;
  94. enum nvmet_tcp_recv_state rcv_state;
  95. struct nvmet_tcp_cmd *cmd;
  96. union nvme_tcp_pdu pdu;
  97. /* digest state */
  98. bool hdr_digest;
  99. bool data_digest;
  100. struct ahash_request *snd_hash;
  101. struct ahash_request *rcv_hash;
  102. spinlock_t state_lock;
  103. enum nvmet_tcp_queue_state state;
  104. struct sockaddr_storage sockaddr;
  105. struct sockaddr_storage sockaddr_peer;
  106. struct work_struct release_work;
  107. int idx;
  108. struct list_head queue_list;
  109. struct nvmet_tcp_cmd connect;
  110. struct page_frag_cache pf_cache;
  111. void (*data_ready)(struct sock *);
  112. void (*state_change)(struct sock *);
  113. void (*write_space)(struct sock *);
  114. };
  115. struct nvmet_tcp_port {
  116. struct socket *sock;
  117. struct work_struct accept_work;
  118. struct nvmet_port *nport;
  119. struct sockaddr_storage addr;
  120. void (*data_ready)(struct sock *);
  121. };
  122. static DEFINE_IDA(nvmet_tcp_queue_ida);
  123. static LIST_HEAD(nvmet_tcp_queue_list);
  124. static DEFINE_MUTEX(nvmet_tcp_queue_mutex);
  125. static struct workqueue_struct *nvmet_tcp_wq;
  126. static const struct nvmet_fabrics_ops nvmet_tcp_ops;
  127. static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c);
  128. static void nvmet_tcp_finish_cmd(struct nvmet_tcp_cmd *cmd);
  129. static inline u16 nvmet_tcp_cmd_tag(struct nvmet_tcp_queue *queue,
  130. struct nvmet_tcp_cmd *cmd)
  131. {
  132. if (unlikely(!queue->nr_cmds)) {
  133. /* We didn't allocate cmds yet, send 0xffff */
  134. return USHRT_MAX;
  135. }
  136. return cmd - queue->cmds;
  137. }
  138. static inline bool nvmet_tcp_has_data_in(struct nvmet_tcp_cmd *cmd)
  139. {
  140. return nvme_is_write(cmd->req.cmd) &&
  141. cmd->rbytes_done < cmd->req.transfer_len;
  142. }
  143. static inline bool nvmet_tcp_need_data_in(struct nvmet_tcp_cmd *cmd)
  144. {
  145. return nvmet_tcp_has_data_in(cmd) && !cmd->req.cqe->status;
  146. }
  147. static inline bool nvmet_tcp_need_data_out(struct nvmet_tcp_cmd *cmd)
  148. {
  149. return !nvme_is_write(cmd->req.cmd) &&
  150. cmd->req.transfer_len > 0 &&
  151. !cmd->req.cqe->status;
  152. }
  153. static inline bool nvmet_tcp_has_inline_data(struct nvmet_tcp_cmd *cmd)
  154. {
  155. return nvme_is_write(cmd->req.cmd) && cmd->pdu_len &&
  156. !cmd->rbytes_done;
  157. }
  158. static inline struct nvmet_tcp_cmd *
  159. nvmet_tcp_get_cmd(struct nvmet_tcp_queue *queue)
  160. {
  161. struct nvmet_tcp_cmd *cmd;
  162. cmd = list_first_entry_or_null(&queue->free_list,
  163. struct nvmet_tcp_cmd, entry);
  164. if (!cmd)
  165. return NULL;
  166. list_del_init(&cmd->entry);
  167. cmd->rbytes_done = cmd->wbytes_done = 0;
  168. cmd->pdu_len = 0;
  169. cmd->pdu_recv = 0;
  170. cmd->iov = NULL;
  171. cmd->flags = 0;
  172. return cmd;
  173. }
  174. static inline void nvmet_tcp_put_cmd(struct nvmet_tcp_cmd *cmd)
  175. {
  176. if (unlikely(cmd == &cmd->queue->connect))
  177. return;
  178. list_add_tail(&cmd->entry, &cmd->queue->free_list);
  179. }
  180. static inline int queue_cpu(struct nvmet_tcp_queue *queue)
  181. {
  182. return queue->sock->sk->sk_incoming_cpu;
  183. }
  184. static inline u8 nvmet_tcp_hdgst_len(struct nvmet_tcp_queue *queue)
  185. {
  186. return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
  187. }
  188. static inline u8 nvmet_tcp_ddgst_len(struct nvmet_tcp_queue *queue)
  189. {
  190. return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
  191. }
  192. static inline void nvmet_tcp_hdgst(struct ahash_request *hash,
  193. void *pdu, size_t len)
  194. {
  195. struct scatterlist sg;
  196. sg_init_one(&sg, pdu, len);
  197. ahash_request_set_crypt(hash, &sg, pdu + len, len);
  198. crypto_ahash_digest(hash);
  199. }
  200. static int nvmet_tcp_verify_hdgst(struct nvmet_tcp_queue *queue,
  201. void *pdu, size_t len)
  202. {
  203. struct nvme_tcp_hdr *hdr = pdu;
  204. __le32 recv_digest;
  205. __le32 exp_digest;
  206. if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
  207. pr_err("queue %d: header digest enabled but no header digest\n",
  208. queue->idx);
  209. return -EPROTO;
  210. }
  211. recv_digest = *(__le32 *)(pdu + hdr->hlen);
  212. nvmet_tcp_hdgst(queue->rcv_hash, pdu, len);
  213. exp_digest = *(__le32 *)(pdu + hdr->hlen);
  214. if (recv_digest != exp_digest) {
  215. pr_err("queue %d: header digest error: recv %#x expected %#x\n",
  216. queue->idx, le32_to_cpu(recv_digest),
  217. le32_to_cpu(exp_digest));
  218. return -EPROTO;
  219. }
  220. return 0;
  221. }
  222. static int nvmet_tcp_check_ddgst(struct nvmet_tcp_queue *queue, void *pdu)
  223. {
  224. struct nvme_tcp_hdr *hdr = pdu;
  225. u8 digest_len = nvmet_tcp_hdgst_len(queue);
  226. u32 len;
  227. len = le32_to_cpu(hdr->plen) - hdr->hlen -
  228. (hdr->flags & NVME_TCP_F_HDGST ? digest_len : 0);
  229. if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
  230. pr_err("queue %d: data digest flag is cleared\n", queue->idx);
  231. return -EPROTO;
  232. }
  233. return 0;
  234. }
  235. static void nvmet_tcp_unmap_pdu_iovec(struct nvmet_tcp_cmd *cmd)
  236. {
  237. struct scatterlist *sg;
  238. int i;
  239. sg = &cmd->req.sg[cmd->sg_idx];
  240. for (i = 0; i < cmd->nr_mapped; i++)
  241. kunmap(sg_page(&sg[i]));
  242. }
  243. static void nvmet_tcp_map_pdu_iovec(struct nvmet_tcp_cmd *cmd)
  244. {
  245. struct kvec *iov = cmd->iov;
  246. struct scatterlist *sg;
  247. u32 length, offset, sg_offset;
  248. length = cmd->pdu_len;
  249. cmd->nr_mapped = DIV_ROUND_UP(length, PAGE_SIZE);
  250. offset = cmd->rbytes_done;
  251. cmd->sg_idx = offset / PAGE_SIZE;
  252. sg_offset = offset % PAGE_SIZE;
  253. sg = &cmd->req.sg[cmd->sg_idx];
  254. while (length) {
  255. u32 iov_len = min_t(u32, length, sg->length - sg_offset);
  256. iov->iov_base = kmap(sg_page(sg)) + sg->offset + sg_offset;
  257. iov->iov_len = iov_len;
  258. length -= iov_len;
  259. sg = sg_next(sg);
  260. iov++;
  261. sg_offset = 0;
  262. }
  263. iov_iter_kvec(&cmd->recv_msg.msg_iter, READ, cmd->iov,
  264. cmd->nr_mapped, cmd->pdu_len);
  265. }
  266. static void nvmet_tcp_fatal_error(struct nvmet_tcp_queue *queue)
  267. {
  268. queue->rcv_state = NVMET_TCP_RECV_ERR;
  269. if (queue->nvme_sq.ctrl)
  270. nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl);
  271. else
  272. kernel_sock_shutdown(queue->sock, SHUT_RDWR);
  273. }
  274. static void nvmet_tcp_socket_error(struct nvmet_tcp_queue *queue, int status)
  275. {
  276. if (status == -EPIPE || status == -ECONNRESET)
  277. kernel_sock_shutdown(queue->sock, SHUT_RDWR);
  278. else
  279. nvmet_tcp_fatal_error(queue);
  280. }
  281. static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd)
  282. {
  283. struct nvme_sgl_desc *sgl = &cmd->req.cmd->common.dptr.sgl;
  284. u32 len = le32_to_cpu(sgl->length);
  285. if (!len)
  286. return 0;
  287. if (sgl->type == ((NVME_SGL_FMT_DATA_DESC << 4) |
  288. NVME_SGL_FMT_OFFSET)) {
  289. if (!nvme_is_write(cmd->req.cmd))
  290. return NVME_SC_INVALID_FIELD | NVME_SC_DNR;
  291. if (len > cmd->req.port->inline_data_size)
  292. return NVME_SC_SGL_INVALID_OFFSET | NVME_SC_DNR;
  293. cmd->pdu_len = len;
  294. }
  295. cmd->req.transfer_len += len;
  296. cmd->req.sg = sgl_alloc(len, GFP_KERNEL, &cmd->req.sg_cnt);
  297. if (!cmd->req.sg)
  298. return NVME_SC_INTERNAL;
  299. cmd->cur_sg = cmd->req.sg;
  300. if (nvmet_tcp_has_data_in(cmd)) {
  301. cmd->iov = kmalloc_array(cmd->req.sg_cnt,
  302. sizeof(*cmd->iov), GFP_KERNEL);
  303. if (!cmd->iov)
  304. goto err;
  305. }
  306. return 0;
  307. err:
  308. sgl_free(cmd->req.sg);
  309. return NVME_SC_INTERNAL;
  310. }
  311. static void nvmet_tcp_send_ddgst(struct ahash_request *hash,
  312. struct nvmet_tcp_cmd *cmd)
  313. {
  314. ahash_request_set_crypt(hash, cmd->req.sg,
  315. (void *)&cmd->exp_ddgst, cmd->req.transfer_len);
  316. crypto_ahash_digest(hash);
  317. }
  318. static void nvmet_tcp_recv_ddgst(struct ahash_request *hash,
  319. struct nvmet_tcp_cmd *cmd)
  320. {
  321. struct scatterlist sg;
  322. struct kvec *iov;
  323. int i;
  324. crypto_ahash_init(hash);
  325. for (i = 0, iov = cmd->iov; i < cmd->nr_mapped; i++, iov++) {
  326. sg_init_one(&sg, iov->iov_base, iov->iov_len);
  327. ahash_request_set_crypt(hash, &sg, NULL, iov->iov_len);
  328. crypto_ahash_update(hash);
  329. }
  330. ahash_request_set_crypt(hash, NULL, (void *)&cmd->exp_ddgst, 0);
  331. crypto_ahash_final(hash);
  332. }
  333. static void nvmet_setup_c2h_data_pdu(struct nvmet_tcp_cmd *cmd)
  334. {
  335. struct nvme_tcp_data_pdu *pdu = cmd->data_pdu;
  336. struct nvmet_tcp_queue *queue = cmd->queue;
  337. u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
  338. u8 ddgst = nvmet_tcp_ddgst_len(cmd->queue);
  339. cmd->offset = 0;
  340. cmd->state = NVMET_TCP_SEND_DATA_PDU;
  341. pdu->hdr.type = nvme_tcp_c2h_data;
  342. pdu->hdr.flags = NVME_TCP_F_DATA_LAST | (queue->nvme_sq.sqhd_disabled ?
  343. NVME_TCP_F_DATA_SUCCESS : 0);
  344. pdu->hdr.hlen = sizeof(*pdu);
  345. pdu->hdr.pdo = pdu->hdr.hlen + hdgst;
  346. pdu->hdr.plen =
  347. cpu_to_le32(pdu->hdr.hlen + hdgst +
  348. cmd->req.transfer_len + ddgst);
  349. pdu->command_id = cmd->req.cqe->command_id;
  350. pdu->data_length = cpu_to_le32(cmd->req.transfer_len);
  351. pdu->data_offset = cpu_to_le32(cmd->wbytes_done);
  352. if (queue->data_digest) {
  353. pdu->hdr.flags |= NVME_TCP_F_DDGST;
  354. nvmet_tcp_send_ddgst(queue->snd_hash, cmd);
  355. }
  356. if (cmd->queue->hdr_digest) {
  357. pdu->hdr.flags |= NVME_TCP_F_HDGST;
  358. nvmet_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
  359. }
  360. }
  361. static void nvmet_setup_r2t_pdu(struct nvmet_tcp_cmd *cmd)
  362. {
  363. struct nvme_tcp_r2t_pdu *pdu = cmd->r2t_pdu;
  364. struct nvmet_tcp_queue *queue = cmd->queue;
  365. u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
  366. cmd->offset = 0;
  367. cmd->state = NVMET_TCP_SEND_R2T;
  368. pdu->hdr.type = nvme_tcp_r2t;
  369. pdu->hdr.flags = 0;
  370. pdu->hdr.hlen = sizeof(*pdu);
  371. pdu->hdr.pdo = 0;
  372. pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
  373. pdu->command_id = cmd->req.cmd->common.command_id;
  374. pdu->ttag = nvmet_tcp_cmd_tag(cmd->queue, cmd);
  375. pdu->r2t_length = cpu_to_le32(cmd->req.transfer_len - cmd->rbytes_done);
  376. pdu->r2t_offset = cpu_to_le32(cmd->rbytes_done);
  377. if (cmd->queue->hdr_digest) {
  378. pdu->hdr.flags |= NVME_TCP_F_HDGST;
  379. nvmet_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
  380. }
  381. }
  382. static void nvmet_setup_response_pdu(struct nvmet_tcp_cmd *cmd)
  383. {
  384. struct nvme_tcp_rsp_pdu *pdu = cmd->rsp_pdu;
  385. struct nvmet_tcp_queue *queue = cmd->queue;
  386. u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
  387. cmd->offset = 0;
  388. cmd->state = NVMET_TCP_SEND_RESPONSE;
  389. pdu->hdr.type = nvme_tcp_rsp;
  390. pdu->hdr.flags = 0;
  391. pdu->hdr.hlen = sizeof(*pdu);
  392. pdu->hdr.pdo = 0;
  393. pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
  394. if (cmd->queue->hdr_digest) {
  395. pdu->hdr.flags |= NVME_TCP_F_HDGST;
  396. nvmet_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
  397. }
  398. }
  399. static void nvmet_tcp_process_resp_list(struct nvmet_tcp_queue *queue)
  400. {
  401. struct llist_node *node;
  402. struct nvmet_tcp_cmd *cmd;
  403. for (node = llist_del_all(&queue->resp_list); node; node = node->next) {
  404. cmd = llist_entry(node, struct nvmet_tcp_cmd, lentry);
  405. list_add(&cmd->entry, &queue->resp_send_list);
  406. queue->send_list_len++;
  407. }
  408. }
  409. static struct nvmet_tcp_cmd *nvmet_tcp_fetch_cmd(struct nvmet_tcp_queue *queue)
  410. {
  411. queue->snd_cmd = list_first_entry_or_null(&queue->resp_send_list,
  412. struct nvmet_tcp_cmd, entry);
  413. if (!queue->snd_cmd) {
  414. nvmet_tcp_process_resp_list(queue);
  415. queue->snd_cmd =
  416. list_first_entry_or_null(&queue->resp_send_list,
  417. struct nvmet_tcp_cmd, entry);
  418. if (unlikely(!queue->snd_cmd))
  419. return NULL;
  420. }
  421. list_del_init(&queue->snd_cmd->entry);
  422. queue->send_list_len--;
  423. if (nvmet_tcp_need_data_out(queue->snd_cmd))
  424. nvmet_setup_c2h_data_pdu(queue->snd_cmd);
  425. else if (nvmet_tcp_need_data_in(queue->snd_cmd))
  426. nvmet_setup_r2t_pdu(queue->snd_cmd);
  427. else
  428. nvmet_setup_response_pdu(queue->snd_cmd);
  429. return queue->snd_cmd;
  430. }
  431. static void nvmet_tcp_queue_response(struct nvmet_req *req)
  432. {
  433. struct nvmet_tcp_cmd *cmd =
  434. container_of(req, struct nvmet_tcp_cmd, req);
  435. struct nvmet_tcp_queue *queue = cmd->queue;
  436. struct nvme_sgl_desc *sgl;
  437. u32 len;
  438. if (unlikely(cmd == queue->cmd)) {
  439. sgl = &cmd->req.cmd->common.dptr.sgl;
  440. len = le32_to_cpu(sgl->length);
  441. /*
  442. * Wait for inline data before processing the response.
  443. * Avoid using helpers, this might happen before
  444. * nvmet_req_init is completed.
  445. */
  446. if (queue->rcv_state == NVMET_TCP_RECV_PDU &&
  447. len && len <= cmd->req.port->inline_data_size &&
  448. nvme_is_write(cmd->req.cmd))
  449. return;
  450. }
  451. llist_add(&cmd->lentry, &queue->resp_list);
  452. queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &cmd->queue->io_work);
  453. }
  454. static void nvmet_tcp_execute_request(struct nvmet_tcp_cmd *cmd)
  455. {
  456. if (unlikely(cmd->flags & NVMET_TCP_F_INIT_FAILED))
  457. nvmet_tcp_queue_response(&cmd->req);
  458. else
  459. cmd->req.execute(&cmd->req);
  460. }
  461. static int nvmet_try_send_data_pdu(struct nvmet_tcp_cmd *cmd)
  462. {
  463. u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
  464. int left = sizeof(*cmd->data_pdu) - cmd->offset + hdgst;
  465. int ret;
  466. ret = kernel_sendpage(cmd->queue->sock, virt_to_page(cmd->data_pdu),
  467. offset_in_page(cmd->data_pdu) + cmd->offset,
  468. left, MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST);
  469. if (ret <= 0)
  470. return ret;
  471. cmd->offset += ret;
  472. left -= ret;
  473. if (left)
  474. return -EAGAIN;
  475. cmd->state = NVMET_TCP_SEND_DATA;
  476. cmd->offset = 0;
  477. return 1;
  478. }
  479. static int nvmet_try_send_data(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
  480. {
  481. struct nvmet_tcp_queue *queue = cmd->queue;
  482. int ret;
  483. while (cmd->cur_sg) {
  484. struct page *page = sg_page(cmd->cur_sg);
  485. u32 left = cmd->cur_sg->length - cmd->offset;
  486. int flags = MSG_DONTWAIT;
  487. if ((!last_in_batch && cmd->queue->send_list_len) ||
  488. cmd->wbytes_done + left < cmd->req.transfer_len ||
  489. queue->data_digest || !queue->nvme_sq.sqhd_disabled)
  490. flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
  491. ret = kernel_sendpage(cmd->queue->sock, page, cmd->offset,
  492. left, flags);
  493. if (ret <= 0)
  494. return ret;
  495. cmd->offset += ret;
  496. cmd->wbytes_done += ret;
  497. /* Done with sg?*/
  498. if (cmd->offset == cmd->cur_sg->length) {
  499. cmd->cur_sg = sg_next(cmd->cur_sg);
  500. cmd->offset = 0;
  501. }
  502. }
  503. if (queue->data_digest) {
  504. cmd->state = NVMET_TCP_SEND_DDGST;
  505. cmd->offset = 0;
  506. } else {
  507. if (queue->nvme_sq.sqhd_disabled) {
  508. cmd->queue->snd_cmd = NULL;
  509. nvmet_tcp_put_cmd(cmd);
  510. } else {
  511. nvmet_setup_response_pdu(cmd);
  512. }
  513. }
  514. if (queue->nvme_sq.sqhd_disabled) {
  515. kfree(cmd->iov);
  516. sgl_free(cmd->req.sg);
  517. }
  518. return 1;
  519. }
  520. static int nvmet_try_send_response(struct nvmet_tcp_cmd *cmd,
  521. bool last_in_batch)
  522. {
  523. u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
  524. int left = sizeof(*cmd->rsp_pdu) - cmd->offset + hdgst;
  525. int flags = MSG_DONTWAIT;
  526. int ret;
  527. if (!last_in_batch && cmd->queue->send_list_len)
  528. flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
  529. else
  530. flags |= MSG_EOR;
  531. ret = kernel_sendpage(cmd->queue->sock, virt_to_page(cmd->rsp_pdu),
  532. offset_in_page(cmd->rsp_pdu) + cmd->offset, left, flags);
  533. if (ret <= 0)
  534. return ret;
  535. cmd->offset += ret;
  536. left -= ret;
  537. if (left)
  538. return -EAGAIN;
  539. kfree(cmd->iov);
  540. sgl_free(cmd->req.sg);
  541. cmd->queue->snd_cmd = NULL;
  542. nvmet_tcp_put_cmd(cmd);
  543. return 1;
  544. }
  545. static int nvmet_try_send_r2t(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
  546. {
  547. u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
  548. int left = sizeof(*cmd->r2t_pdu) - cmd->offset + hdgst;
  549. int flags = MSG_DONTWAIT;
  550. int ret;
  551. if (!last_in_batch && cmd->queue->send_list_len)
  552. flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
  553. else
  554. flags |= MSG_EOR;
  555. ret = kernel_sendpage(cmd->queue->sock, virt_to_page(cmd->r2t_pdu),
  556. offset_in_page(cmd->r2t_pdu) + cmd->offset, left, flags);
  557. if (ret <= 0)
  558. return ret;
  559. cmd->offset += ret;
  560. left -= ret;
  561. if (left)
  562. return -EAGAIN;
  563. cmd->queue->snd_cmd = NULL;
  564. return 1;
  565. }
  566. static int nvmet_try_send_ddgst(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
  567. {
  568. struct nvmet_tcp_queue *queue = cmd->queue;
  569. int left = NVME_TCP_DIGEST_LENGTH - cmd->offset;
  570. struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
  571. struct kvec iov = {
  572. .iov_base = (u8 *)&cmd->exp_ddgst + cmd->offset,
  573. .iov_len = left
  574. };
  575. int ret;
  576. if (!last_in_batch && cmd->queue->send_list_len)
  577. msg.msg_flags |= MSG_MORE;
  578. else
  579. msg.msg_flags |= MSG_EOR;
  580. ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
  581. if (unlikely(ret <= 0))
  582. return ret;
  583. cmd->offset += ret;
  584. left -= ret;
  585. if (left)
  586. return -EAGAIN;
  587. if (queue->nvme_sq.sqhd_disabled) {
  588. cmd->queue->snd_cmd = NULL;
  589. nvmet_tcp_put_cmd(cmd);
  590. } else {
  591. nvmet_setup_response_pdu(cmd);
  592. }
  593. return 1;
  594. }
  595. static int nvmet_tcp_try_send_one(struct nvmet_tcp_queue *queue,
  596. bool last_in_batch)
  597. {
  598. struct nvmet_tcp_cmd *cmd = queue->snd_cmd;
  599. int ret = 0;
  600. if (!cmd || queue->state == NVMET_TCP_Q_DISCONNECTING) {
  601. cmd = nvmet_tcp_fetch_cmd(queue);
  602. if (unlikely(!cmd))
  603. return 0;
  604. }
  605. if (cmd->state == NVMET_TCP_SEND_DATA_PDU) {
  606. ret = nvmet_try_send_data_pdu(cmd);
  607. if (ret <= 0)
  608. goto done_send;
  609. }
  610. if (cmd->state == NVMET_TCP_SEND_DATA) {
  611. ret = nvmet_try_send_data(cmd, last_in_batch);
  612. if (ret <= 0)
  613. goto done_send;
  614. }
  615. if (cmd->state == NVMET_TCP_SEND_DDGST) {
  616. ret = nvmet_try_send_ddgst(cmd, last_in_batch);
  617. if (ret <= 0)
  618. goto done_send;
  619. }
  620. if (cmd->state == NVMET_TCP_SEND_R2T) {
  621. ret = nvmet_try_send_r2t(cmd, last_in_batch);
  622. if (ret <= 0)
  623. goto done_send;
  624. }
  625. if (cmd->state == NVMET_TCP_SEND_RESPONSE)
  626. ret = nvmet_try_send_response(cmd, last_in_batch);
  627. done_send:
  628. if (ret < 0) {
  629. if (ret == -EAGAIN)
  630. return 0;
  631. return ret;
  632. }
  633. return 1;
  634. }
  635. static int nvmet_tcp_try_send(struct nvmet_tcp_queue *queue,
  636. int budget, int *sends)
  637. {
  638. int i, ret = 0;
  639. for (i = 0; i < budget; i++) {
  640. ret = nvmet_tcp_try_send_one(queue, i == budget - 1);
  641. if (unlikely(ret < 0)) {
  642. nvmet_tcp_socket_error(queue, ret);
  643. goto done;
  644. } else if (ret == 0) {
  645. break;
  646. }
  647. (*sends)++;
  648. }
  649. done:
  650. return ret;
  651. }
  652. static void nvmet_prepare_receive_pdu(struct nvmet_tcp_queue *queue)
  653. {
  654. queue->offset = 0;
  655. queue->left = sizeof(struct nvme_tcp_hdr);
  656. queue->cmd = NULL;
  657. queue->rcv_state = NVMET_TCP_RECV_PDU;
  658. }
  659. static void nvmet_tcp_free_crypto(struct nvmet_tcp_queue *queue)
  660. {
  661. struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
  662. ahash_request_free(queue->rcv_hash);
  663. ahash_request_free(queue->snd_hash);
  664. crypto_free_ahash(tfm);
  665. }
  666. static int nvmet_tcp_alloc_crypto(struct nvmet_tcp_queue *queue)
  667. {
  668. struct crypto_ahash *tfm;
  669. tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
  670. if (IS_ERR(tfm))
  671. return PTR_ERR(tfm);
  672. queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
  673. if (!queue->snd_hash)
  674. goto free_tfm;
  675. ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
  676. queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
  677. if (!queue->rcv_hash)
  678. goto free_snd_hash;
  679. ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
  680. return 0;
  681. free_snd_hash:
  682. ahash_request_free(queue->snd_hash);
  683. free_tfm:
  684. crypto_free_ahash(tfm);
  685. return -ENOMEM;
  686. }
  687. static int nvmet_tcp_handle_icreq(struct nvmet_tcp_queue *queue)
  688. {
  689. struct nvme_tcp_icreq_pdu *icreq = &queue->pdu.icreq;
  690. struct nvme_tcp_icresp_pdu *icresp = &queue->pdu.icresp;
  691. struct msghdr msg = {};
  692. struct kvec iov;
  693. int ret;
  694. if (le32_to_cpu(icreq->hdr.plen) != sizeof(struct nvme_tcp_icreq_pdu)) {
  695. pr_err("bad nvme-tcp pdu length (%d)\n",
  696. le32_to_cpu(icreq->hdr.plen));
  697. nvmet_tcp_fatal_error(queue);
  698. }
  699. if (icreq->pfv != NVME_TCP_PFV_1_0) {
  700. pr_err("queue %d: bad pfv %d\n", queue->idx, icreq->pfv);
  701. return -EPROTO;
  702. }
  703. if (icreq->hpda != 0) {
  704. pr_err("queue %d: unsupported hpda %d\n", queue->idx,
  705. icreq->hpda);
  706. return -EPROTO;
  707. }
  708. queue->hdr_digest = !!(icreq->digest & NVME_TCP_HDR_DIGEST_ENABLE);
  709. queue->data_digest = !!(icreq->digest & NVME_TCP_DATA_DIGEST_ENABLE);
  710. if (queue->hdr_digest || queue->data_digest) {
  711. ret = nvmet_tcp_alloc_crypto(queue);
  712. if (ret)
  713. return ret;
  714. }
  715. memset(icresp, 0, sizeof(*icresp));
  716. icresp->hdr.type = nvme_tcp_icresp;
  717. icresp->hdr.hlen = sizeof(*icresp);
  718. icresp->hdr.pdo = 0;
  719. icresp->hdr.plen = cpu_to_le32(icresp->hdr.hlen);
  720. icresp->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
  721. icresp->maxdata = cpu_to_le32(0x400000); /* 16M arbitrary limit */
  722. icresp->cpda = 0;
  723. if (queue->hdr_digest)
  724. icresp->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
  725. if (queue->data_digest)
  726. icresp->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
  727. iov.iov_base = icresp;
  728. iov.iov_len = sizeof(*icresp);
  729. ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
  730. if (ret < 0)
  731. goto free_crypto;
  732. queue->state = NVMET_TCP_Q_LIVE;
  733. nvmet_prepare_receive_pdu(queue);
  734. return 0;
  735. free_crypto:
  736. if (queue->hdr_digest || queue->data_digest)
  737. nvmet_tcp_free_crypto(queue);
  738. return ret;
  739. }
  740. static void nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue,
  741. struct nvmet_tcp_cmd *cmd, struct nvmet_req *req)
  742. {
  743. size_t data_len = le32_to_cpu(req->cmd->common.dptr.sgl.length);
  744. int ret;
  745. if (!nvme_is_write(cmd->req.cmd) ||
  746. data_len > cmd->req.port->inline_data_size) {
  747. nvmet_prepare_receive_pdu(queue);
  748. return;
  749. }
  750. ret = nvmet_tcp_map_data(cmd);
  751. if (unlikely(ret)) {
  752. pr_err("queue %d: failed to map data\n", queue->idx);
  753. nvmet_tcp_fatal_error(queue);
  754. return;
  755. }
  756. queue->rcv_state = NVMET_TCP_RECV_DATA;
  757. nvmet_tcp_map_pdu_iovec(cmd);
  758. cmd->flags |= NVMET_TCP_F_INIT_FAILED;
  759. }
  760. static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue)
  761. {
  762. struct nvme_tcp_data_pdu *data = &queue->pdu.data;
  763. struct nvmet_tcp_cmd *cmd;
  764. if (likely(queue->nr_cmds))
  765. cmd = &queue->cmds[data->ttag];
  766. else
  767. cmd = &queue->connect;
  768. if (le32_to_cpu(data->data_offset) != cmd->rbytes_done) {
  769. pr_err("ttag %u unexpected data offset %u (expected %u)\n",
  770. data->ttag, le32_to_cpu(data->data_offset),
  771. cmd->rbytes_done);
  772. /* FIXME: use path and transport errors */
  773. nvmet_req_complete(&cmd->req,
  774. NVME_SC_INVALID_FIELD | NVME_SC_DNR);
  775. return -EPROTO;
  776. }
  777. cmd->pdu_len = le32_to_cpu(data->data_length);
  778. cmd->pdu_recv = 0;
  779. nvmet_tcp_map_pdu_iovec(cmd);
  780. queue->cmd = cmd;
  781. queue->rcv_state = NVMET_TCP_RECV_DATA;
  782. return 0;
  783. }
  784. static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
  785. {
  786. struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;
  787. struct nvme_command *nvme_cmd = &queue->pdu.cmd.cmd;
  788. struct nvmet_req *req;
  789. int ret;
  790. if (unlikely(queue->state == NVMET_TCP_Q_CONNECTING)) {
  791. if (hdr->type != nvme_tcp_icreq) {
  792. pr_err("unexpected pdu type (%d) before icreq\n",
  793. hdr->type);
  794. nvmet_tcp_fatal_error(queue);
  795. return -EPROTO;
  796. }
  797. return nvmet_tcp_handle_icreq(queue);
  798. }
  799. if (hdr->type == nvme_tcp_h2c_data) {
  800. ret = nvmet_tcp_handle_h2c_data_pdu(queue);
  801. if (unlikely(ret))
  802. return ret;
  803. return 0;
  804. }
  805. queue->cmd = nvmet_tcp_get_cmd(queue);
  806. if (unlikely(!queue->cmd)) {
  807. /* This should never happen */
  808. pr_err("queue %d: out of commands (%d) send_list_len: %d, opcode: %d",
  809. queue->idx, queue->nr_cmds, queue->send_list_len,
  810. nvme_cmd->common.opcode);
  811. nvmet_tcp_fatal_error(queue);
  812. return -ENOMEM;
  813. }
  814. req = &queue->cmd->req;
  815. memcpy(req->cmd, nvme_cmd, sizeof(*nvme_cmd));
  816. if (unlikely(!nvmet_req_init(req, &queue->nvme_cq,
  817. &queue->nvme_sq, &nvmet_tcp_ops))) {
  818. pr_err("failed cmd %p id %d opcode %d, data_len: %d\n",
  819. req->cmd, req->cmd->common.command_id,
  820. req->cmd->common.opcode,
  821. le32_to_cpu(req->cmd->common.dptr.sgl.length));
  822. nvmet_tcp_handle_req_failure(queue, queue->cmd, req);
  823. return 0;
  824. }
  825. ret = nvmet_tcp_map_data(queue->cmd);
  826. if (unlikely(ret)) {
  827. pr_err("queue %d: failed to map data\n", queue->idx);
  828. if (nvmet_tcp_has_inline_data(queue->cmd))
  829. nvmet_tcp_fatal_error(queue);
  830. else
  831. nvmet_req_complete(req, ret);
  832. ret = -EAGAIN;
  833. goto out;
  834. }
  835. if (nvmet_tcp_need_data_in(queue->cmd)) {
  836. if (nvmet_tcp_has_inline_data(queue->cmd)) {
  837. queue->rcv_state = NVMET_TCP_RECV_DATA;
  838. nvmet_tcp_map_pdu_iovec(queue->cmd);
  839. return 0;
  840. }
  841. /* send back R2T */
  842. nvmet_tcp_queue_response(&queue->cmd->req);
  843. goto out;
  844. }
  845. queue->cmd->req.execute(&queue->cmd->req);
  846. out:
  847. nvmet_prepare_receive_pdu(queue);
  848. return ret;
  849. }
  850. static const u8 nvme_tcp_pdu_sizes[] = {
  851. [nvme_tcp_icreq] = sizeof(struct nvme_tcp_icreq_pdu),
  852. [nvme_tcp_cmd] = sizeof(struct nvme_tcp_cmd_pdu),
  853. [nvme_tcp_h2c_data] = sizeof(struct nvme_tcp_data_pdu),
  854. };
  855. static inline u8 nvmet_tcp_pdu_size(u8 type)
  856. {
  857. size_t idx = type;
  858. return (idx < ARRAY_SIZE(nvme_tcp_pdu_sizes) &&
  859. nvme_tcp_pdu_sizes[idx]) ?
  860. nvme_tcp_pdu_sizes[idx] : 0;
  861. }
  862. static inline bool nvmet_tcp_pdu_valid(u8 type)
  863. {
  864. switch (type) {
  865. case nvme_tcp_icreq:
  866. case nvme_tcp_cmd:
  867. case nvme_tcp_h2c_data:
  868. /* fallthru */
  869. return true;
  870. }
  871. return false;
  872. }
  873. static int nvmet_tcp_try_recv_pdu(struct nvmet_tcp_queue *queue)
  874. {
  875. struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;
  876. int len;
  877. struct kvec iov;
  878. struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
  879. recv:
  880. iov.iov_base = (void *)&queue->pdu + queue->offset;
  881. iov.iov_len = queue->left;
  882. len = kernel_recvmsg(queue->sock, &msg, &iov, 1,
  883. iov.iov_len, msg.msg_flags);
  884. if (unlikely(len < 0))
  885. return len;
  886. queue->offset += len;
  887. queue->left -= len;
  888. if (queue->left)
  889. return -EAGAIN;
  890. if (queue->offset == sizeof(struct nvme_tcp_hdr)) {
  891. u8 hdgst = nvmet_tcp_hdgst_len(queue);
  892. if (unlikely(!nvmet_tcp_pdu_valid(hdr->type))) {
  893. pr_err("unexpected pdu type %d\n", hdr->type);
  894. nvmet_tcp_fatal_error(queue);
  895. return -EIO;
  896. }
  897. if (unlikely(hdr->hlen != nvmet_tcp_pdu_size(hdr->type))) {
  898. pr_err("pdu %d bad hlen %d\n", hdr->type, hdr->hlen);
  899. return -EIO;
  900. }
  901. queue->left = hdr->hlen - queue->offset + hdgst;
  902. goto recv;
  903. }
  904. if (queue->hdr_digest &&
  905. nvmet_tcp_verify_hdgst(queue, &queue->pdu, hdr->hlen)) {
  906. nvmet_tcp_fatal_error(queue); /* fatal */
  907. return -EPROTO;
  908. }
  909. if (queue->data_digest &&
  910. nvmet_tcp_check_ddgst(queue, &queue->pdu)) {
  911. nvmet_tcp_fatal_error(queue); /* fatal */
  912. return -EPROTO;
  913. }
  914. return nvmet_tcp_done_recv_pdu(queue);
  915. }
  916. static void nvmet_tcp_prep_recv_ddgst(struct nvmet_tcp_cmd *cmd)
  917. {
  918. struct nvmet_tcp_queue *queue = cmd->queue;
  919. nvmet_tcp_recv_ddgst(queue->rcv_hash, cmd);
  920. queue->offset = 0;
  921. queue->left = NVME_TCP_DIGEST_LENGTH;
  922. queue->rcv_state = NVMET_TCP_RECV_DDGST;
  923. }
  924. static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue)
  925. {
  926. struct nvmet_tcp_cmd *cmd = queue->cmd;
  927. int ret;
  928. while (msg_data_left(&cmd->recv_msg)) {
  929. ret = sock_recvmsg(cmd->queue->sock, &cmd->recv_msg,
  930. cmd->recv_msg.msg_flags);
  931. if (ret <= 0)
  932. return ret;
  933. cmd->pdu_recv += ret;
  934. cmd->rbytes_done += ret;
  935. }
  936. nvmet_tcp_unmap_pdu_iovec(cmd);
  937. if (queue->data_digest) {
  938. nvmet_tcp_prep_recv_ddgst(cmd);
  939. return 0;
  940. }
  941. if (cmd->rbytes_done == cmd->req.transfer_len)
  942. nvmet_tcp_execute_request(cmd);
  943. nvmet_prepare_receive_pdu(queue);
  944. return 0;
  945. }
  946. static int nvmet_tcp_try_recv_ddgst(struct nvmet_tcp_queue *queue)
  947. {
  948. struct nvmet_tcp_cmd *cmd = queue->cmd;
  949. int ret;
  950. struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
  951. struct kvec iov = {
  952. .iov_base = (void *)&cmd->recv_ddgst + queue->offset,
  953. .iov_len = queue->left
  954. };
  955. ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
  956. iov.iov_len, msg.msg_flags);
  957. if (unlikely(ret < 0))
  958. return ret;
  959. queue->offset += ret;
  960. queue->left -= ret;
  961. if (queue->left)
  962. return -EAGAIN;
  963. if (queue->data_digest && cmd->exp_ddgst != cmd->recv_ddgst) {
  964. pr_err("queue %d: cmd %d pdu (%d) data digest error: recv %#x expected %#x\n",
  965. queue->idx, cmd->req.cmd->common.command_id,
  966. queue->pdu.cmd.hdr.type, le32_to_cpu(cmd->recv_ddgst),
  967. le32_to_cpu(cmd->exp_ddgst));
  968. nvmet_tcp_finish_cmd(cmd);
  969. nvmet_tcp_fatal_error(queue);
  970. ret = -EPROTO;
  971. goto out;
  972. }
  973. if (cmd->rbytes_done == cmd->req.transfer_len)
  974. nvmet_tcp_execute_request(cmd);
  975. ret = 0;
  976. out:
  977. nvmet_prepare_receive_pdu(queue);
  978. return ret;
  979. }
  980. static int nvmet_tcp_try_recv_one(struct nvmet_tcp_queue *queue)
  981. {
  982. int result = 0;
  983. if (unlikely(queue->rcv_state == NVMET_TCP_RECV_ERR))
  984. return 0;
  985. if (queue->rcv_state == NVMET_TCP_RECV_PDU) {
  986. result = nvmet_tcp_try_recv_pdu(queue);
  987. if (result != 0)
  988. goto done_recv;
  989. }
  990. if (queue->rcv_state == NVMET_TCP_RECV_DATA) {
  991. result = nvmet_tcp_try_recv_data(queue);
  992. if (result != 0)
  993. goto done_recv;
  994. }
  995. if (queue->rcv_state == NVMET_TCP_RECV_DDGST) {
  996. result = nvmet_tcp_try_recv_ddgst(queue);
  997. if (result != 0)
  998. goto done_recv;
  999. }
  1000. done_recv:
  1001. if (result < 0) {
  1002. if (result == -EAGAIN)
  1003. return 0;
  1004. return result;
  1005. }
  1006. return 1;
  1007. }
  1008. static int nvmet_tcp_try_recv(struct nvmet_tcp_queue *queue,
  1009. int budget, int *recvs)
  1010. {
  1011. int i, ret = 0;
  1012. for (i = 0; i < budget; i++) {
  1013. ret = nvmet_tcp_try_recv_one(queue);
  1014. if (unlikely(ret < 0)) {
  1015. nvmet_tcp_socket_error(queue, ret);
  1016. goto done;
  1017. } else if (ret == 0) {
  1018. break;
  1019. }
  1020. (*recvs)++;
  1021. }
  1022. done:
  1023. return ret;
  1024. }
  1025. static void nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue *queue)
  1026. {
  1027. spin_lock(&queue->state_lock);
  1028. if (queue->state != NVMET_TCP_Q_DISCONNECTING) {
  1029. queue->state = NVMET_TCP_Q_DISCONNECTING;
  1030. schedule_work(&queue->release_work);
  1031. }
  1032. spin_unlock(&queue->state_lock);
  1033. }
  1034. static void nvmet_tcp_io_work(struct work_struct *w)
  1035. {
  1036. struct nvmet_tcp_queue *queue =
  1037. container_of(w, struct nvmet_tcp_queue, io_work);
  1038. bool pending;
  1039. int ret, ops = 0;
  1040. do {
  1041. pending = false;
  1042. ret = nvmet_tcp_try_recv(queue, NVMET_TCP_RECV_BUDGET, &ops);
  1043. if (ret > 0)
  1044. pending = true;
  1045. else if (ret < 0)
  1046. return;
  1047. ret = nvmet_tcp_try_send(queue, NVMET_TCP_SEND_BUDGET, &ops);
  1048. if (ret > 0)
  1049. pending = true;
  1050. else if (ret < 0)
  1051. return;
  1052. } while (pending && ops < NVMET_TCP_IO_WORK_BUDGET);
  1053. /*
  1054. * We exahusted our budget, requeue our selves
  1055. */
  1056. if (pending)
  1057. queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work);
  1058. }
  1059. static int nvmet_tcp_alloc_cmd(struct nvmet_tcp_queue *queue,
  1060. struct nvmet_tcp_cmd *c)
  1061. {
  1062. u8 hdgst = nvmet_tcp_hdgst_len(queue);
  1063. c->queue = queue;
  1064. c->req.port = queue->port->nport;
  1065. c->cmd_pdu = page_frag_alloc(&queue->pf_cache,
  1066. sizeof(*c->cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
  1067. if (!c->cmd_pdu)
  1068. return -ENOMEM;
  1069. c->req.cmd = &c->cmd_pdu->cmd;
  1070. c->rsp_pdu = page_frag_alloc(&queue->pf_cache,
  1071. sizeof(*c->rsp_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
  1072. if (!c->rsp_pdu)
  1073. goto out_free_cmd;
  1074. c->req.cqe = &c->rsp_pdu->cqe;
  1075. c->data_pdu = page_frag_alloc(&queue->pf_cache,
  1076. sizeof(*c->data_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
  1077. if (!c->data_pdu)
  1078. goto out_free_rsp;
  1079. c->r2t_pdu = page_frag_alloc(&queue->pf_cache,
  1080. sizeof(*c->r2t_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
  1081. if (!c->r2t_pdu)
  1082. goto out_free_data;
  1083. c->recv_msg.msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL;
  1084. list_add_tail(&c->entry, &queue->free_list);
  1085. return 0;
  1086. out_free_data:
  1087. page_frag_free(c->data_pdu);
  1088. out_free_rsp:
  1089. page_frag_free(c->rsp_pdu);
  1090. out_free_cmd:
  1091. page_frag_free(c->cmd_pdu);
  1092. return -ENOMEM;
  1093. }
  1094. static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c)
  1095. {
  1096. page_frag_free(c->r2t_pdu);
  1097. page_frag_free(c->data_pdu);
  1098. page_frag_free(c->rsp_pdu);
  1099. page_frag_free(c->cmd_pdu);
  1100. }
  1101. static int nvmet_tcp_alloc_cmds(struct nvmet_tcp_queue *queue)
  1102. {
  1103. struct nvmet_tcp_cmd *cmds;
  1104. int i, ret = -EINVAL, nr_cmds = queue->nr_cmds;
  1105. cmds = kcalloc(nr_cmds, sizeof(struct nvmet_tcp_cmd), GFP_KERNEL);
  1106. if (!cmds)
  1107. goto out;
  1108. for (i = 0; i < nr_cmds; i++) {
  1109. ret = nvmet_tcp_alloc_cmd(queue, cmds + i);
  1110. if (ret)
  1111. goto out_free;
  1112. }
  1113. queue->cmds = cmds;
  1114. return 0;
  1115. out_free:
  1116. while (--i >= 0)
  1117. nvmet_tcp_free_cmd(cmds + i);
  1118. kfree(cmds);
  1119. out:
  1120. return ret;
  1121. }
  1122. static void nvmet_tcp_free_cmds(struct nvmet_tcp_queue *queue)
  1123. {
  1124. struct nvmet_tcp_cmd *cmds = queue->cmds;
  1125. int i;
  1126. for (i = 0; i < queue->nr_cmds; i++)
  1127. nvmet_tcp_free_cmd(cmds + i);
  1128. nvmet_tcp_free_cmd(&queue->connect);
  1129. kfree(cmds);
  1130. }
  1131. static void nvmet_tcp_restore_socket_callbacks(struct nvmet_tcp_queue *queue)
  1132. {
  1133. struct socket *sock = queue->sock;
  1134. write_lock_bh(&sock->sk->sk_callback_lock);
  1135. sock->sk->sk_data_ready = queue->data_ready;
  1136. sock->sk->sk_state_change = queue->state_change;
  1137. sock->sk->sk_write_space = queue->write_space;
  1138. sock->sk->sk_user_data = NULL;
  1139. write_unlock_bh(&sock->sk->sk_callback_lock);
  1140. }
  1141. static void nvmet_tcp_finish_cmd(struct nvmet_tcp_cmd *cmd)
  1142. {
  1143. nvmet_req_uninit(&cmd->req);
  1144. nvmet_tcp_unmap_pdu_iovec(cmd);
  1145. kfree(cmd->iov);
  1146. sgl_free(cmd->req.sg);
  1147. }
  1148. static void nvmet_tcp_uninit_data_in_cmds(struct nvmet_tcp_queue *queue)
  1149. {
  1150. struct nvmet_tcp_cmd *cmd = queue->cmds;
  1151. int i;
  1152. for (i = 0; i < queue->nr_cmds; i++, cmd++) {
  1153. if (nvmet_tcp_need_data_in(cmd))
  1154. nvmet_tcp_finish_cmd(cmd);
  1155. }
  1156. if (!queue->nr_cmds && nvmet_tcp_need_data_in(&queue->connect)) {
  1157. /* failed in connect */
  1158. nvmet_tcp_finish_cmd(&queue->connect);
  1159. }
  1160. }
  1161. static void nvmet_tcp_release_queue_work(struct work_struct *w)
  1162. {
  1163. struct page *page;
  1164. struct nvmet_tcp_queue *queue =
  1165. container_of(w, struct nvmet_tcp_queue, release_work);
  1166. mutex_lock(&nvmet_tcp_queue_mutex);
  1167. list_del_init(&queue->queue_list);
  1168. mutex_unlock(&nvmet_tcp_queue_mutex);
  1169. nvmet_tcp_restore_socket_callbacks(queue);
  1170. flush_work(&queue->io_work);
  1171. nvmet_tcp_uninit_data_in_cmds(queue);
  1172. nvmet_sq_destroy(&queue->nvme_sq);
  1173. cancel_work_sync(&queue->io_work);
  1174. sock_release(queue->sock);
  1175. nvmet_tcp_free_cmds(queue);
  1176. if (queue->hdr_digest || queue->data_digest)
  1177. nvmet_tcp_free_crypto(queue);
  1178. ida_simple_remove(&nvmet_tcp_queue_ida, queue->idx);
  1179. page = virt_to_head_page(queue->pf_cache.va);
  1180. __page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias);
  1181. kfree(queue);
  1182. }
  1183. static void nvmet_tcp_data_ready(struct sock *sk)
  1184. {
  1185. struct nvmet_tcp_queue *queue;
  1186. read_lock_bh(&sk->sk_callback_lock);
  1187. queue = sk->sk_user_data;
  1188. if (likely(queue))
  1189. queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work);
  1190. read_unlock_bh(&sk->sk_callback_lock);
  1191. }
  1192. static void nvmet_tcp_write_space(struct sock *sk)
  1193. {
  1194. struct nvmet_tcp_queue *queue;
  1195. read_lock_bh(&sk->sk_callback_lock);
  1196. queue = sk->sk_user_data;
  1197. if (unlikely(!queue))
  1198. goto out;
  1199. if (unlikely(queue->state == NVMET_TCP_Q_CONNECTING)) {
  1200. queue->write_space(sk);
  1201. goto out;
  1202. }
  1203. if (sk_stream_is_writeable(sk)) {
  1204. clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
  1205. queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work);
  1206. }
  1207. out:
  1208. read_unlock_bh(&sk->sk_callback_lock);
  1209. }
  1210. static void nvmet_tcp_state_change(struct sock *sk)
  1211. {
  1212. struct nvmet_tcp_queue *queue;
  1213. read_lock_bh(&sk->sk_callback_lock);
  1214. queue = sk->sk_user_data;
  1215. if (!queue)
  1216. goto done;
  1217. switch (sk->sk_state) {
  1218. case TCP_FIN_WAIT1:
  1219. case TCP_CLOSE_WAIT:
  1220. case TCP_CLOSE:
  1221. /* FALLTHRU */
  1222. nvmet_tcp_schedule_release_queue(queue);
  1223. break;
  1224. default:
  1225. pr_warn("queue %d unhandled state %d\n",
  1226. queue->idx, sk->sk_state);
  1227. }
  1228. done:
  1229. read_unlock_bh(&sk->sk_callback_lock);
  1230. }
  1231. static int nvmet_tcp_set_queue_sock(struct nvmet_tcp_queue *queue)
  1232. {
  1233. struct socket *sock = queue->sock;
  1234. struct inet_sock *inet = inet_sk(sock->sk);
  1235. int ret;
  1236. ret = kernel_getsockname(sock,
  1237. (struct sockaddr *)&queue->sockaddr);
  1238. if (ret < 0)
  1239. return ret;
  1240. ret = kernel_getpeername(sock,
  1241. (struct sockaddr *)&queue->sockaddr_peer);
  1242. if (ret < 0)
  1243. return ret;
  1244. /*
  1245. * Cleanup whatever is sitting in the TCP transmit queue on socket
  1246. * close. This is done to prevent stale data from being sent should
  1247. * the network connection be restored before TCP times out.
  1248. */
  1249. sock_no_linger(sock->sk);
  1250. if (so_priority > 0)
  1251. sock_set_priority(sock->sk, so_priority);
  1252. /* Set socket type of service */
  1253. if (inet->rcv_tos > 0)
  1254. ip_sock_set_tos(sock->sk, inet->rcv_tos);
  1255. ret = 0;
  1256. write_lock_bh(&sock->sk->sk_callback_lock);
  1257. if (sock->sk->sk_state != TCP_ESTABLISHED) {
  1258. /*
  1259. * If the socket is already closing, don't even start
  1260. * consuming it
  1261. */
  1262. ret = -ENOTCONN;
  1263. } else {
  1264. sock->sk->sk_user_data = queue;
  1265. queue->data_ready = sock->sk->sk_data_ready;
  1266. sock->sk->sk_data_ready = nvmet_tcp_data_ready;
  1267. queue->state_change = sock->sk->sk_state_change;
  1268. sock->sk->sk_state_change = nvmet_tcp_state_change;
  1269. queue->write_space = sock->sk->sk_write_space;
  1270. sock->sk->sk_write_space = nvmet_tcp_write_space;
  1271. queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work);
  1272. }
  1273. write_unlock_bh(&sock->sk->sk_callback_lock);
  1274. return ret;
  1275. }
  1276. static int nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port,
  1277. struct socket *newsock)
  1278. {
  1279. struct nvmet_tcp_queue *queue;
  1280. int ret;
  1281. queue = kzalloc(sizeof(*queue), GFP_KERNEL);
  1282. if (!queue)
  1283. return -ENOMEM;
  1284. INIT_WORK(&queue->release_work, nvmet_tcp_release_queue_work);
  1285. INIT_WORK(&queue->io_work, nvmet_tcp_io_work);
  1286. queue->sock = newsock;
  1287. queue->port = port;
  1288. queue->nr_cmds = 0;
  1289. spin_lock_init(&queue->state_lock);
  1290. queue->state = NVMET_TCP_Q_CONNECTING;
  1291. INIT_LIST_HEAD(&queue->free_list);
  1292. init_llist_head(&queue->resp_list);
  1293. INIT_LIST_HEAD(&queue->resp_send_list);
  1294. queue->idx = ida_simple_get(&nvmet_tcp_queue_ida, 0, 0, GFP_KERNEL);
  1295. if (queue->idx < 0) {
  1296. ret = queue->idx;
  1297. goto out_free_queue;
  1298. }
  1299. ret = nvmet_tcp_alloc_cmd(queue, &queue->connect);
  1300. if (ret)
  1301. goto out_ida_remove;
  1302. ret = nvmet_sq_init(&queue->nvme_sq);
  1303. if (ret)
  1304. goto out_free_connect;
  1305. nvmet_prepare_receive_pdu(queue);
  1306. mutex_lock(&nvmet_tcp_queue_mutex);
  1307. list_add_tail(&queue->queue_list, &nvmet_tcp_queue_list);
  1308. mutex_unlock(&nvmet_tcp_queue_mutex);
  1309. ret = nvmet_tcp_set_queue_sock(queue);
  1310. if (ret)
  1311. goto out_destroy_sq;
  1312. return 0;
  1313. out_destroy_sq:
  1314. mutex_lock(&nvmet_tcp_queue_mutex);
  1315. list_del_init(&queue->queue_list);
  1316. mutex_unlock(&nvmet_tcp_queue_mutex);
  1317. nvmet_sq_destroy(&queue->nvme_sq);
  1318. out_free_connect:
  1319. nvmet_tcp_free_cmd(&queue->connect);
  1320. out_ida_remove:
  1321. ida_simple_remove(&nvmet_tcp_queue_ida, queue->idx);
  1322. out_free_queue:
  1323. kfree(queue);
  1324. return ret;
  1325. }
  1326. static void nvmet_tcp_accept_work(struct work_struct *w)
  1327. {
  1328. struct nvmet_tcp_port *port =
  1329. container_of(w, struct nvmet_tcp_port, accept_work);
  1330. struct socket *newsock;
  1331. int ret;
  1332. while (true) {
  1333. ret = kernel_accept(port->sock, &newsock, O_NONBLOCK);
  1334. if (ret < 0) {
  1335. if (ret != -EAGAIN)
  1336. pr_warn("failed to accept err=%d\n", ret);
  1337. return;
  1338. }
  1339. ret = nvmet_tcp_alloc_queue(port, newsock);
  1340. if (ret) {
  1341. pr_err("failed to allocate queue\n");
  1342. sock_release(newsock);
  1343. }
  1344. }
  1345. }
  1346. static void nvmet_tcp_listen_data_ready(struct sock *sk)
  1347. {
  1348. struct nvmet_tcp_port *port;
  1349. read_lock_bh(&sk->sk_callback_lock);
  1350. port = sk->sk_user_data;
  1351. if (!port)
  1352. goto out;
  1353. if (sk->sk_state == TCP_LISTEN)
  1354. schedule_work(&port->accept_work);
  1355. out:
  1356. read_unlock_bh(&sk->sk_callback_lock);
  1357. }
  1358. static int nvmet_tcp_add_port(struct nvmet_port *nport)
  1359. {
  1360. struct nvmet_tcp_port *port;
  1361. __kernel_sa_family_t af;
  1362. int ret;
  1363. port = kzalloc(sizeof(*port), GFP_KERNEL);
  1364. if (!port)
  1365. return -ENOMEM;
  1366. switch (nport->disc_addr.adrfam) {
  1367. case NVMF_ADDR_FAMILY_IP4:
  1368. af = AF_INET;
  1369. break;
  1370. case NVMF_ADDR_FAMILY_IP6:
  1371. af = AF_INET6;
  1372. break;
  1373. default:
  1374. pr_err("address family %d not supported\n",
  1375. nport->disc_addr.adrfam);
  1376. ret = -EINVAL;
  1377. goto err_port;
  1378. }
  1379. ret = inet_pton_with_scope(&init_net, af, nport->disc_addr.traddr,
  1380. nport->disc_addr.trsvcid, &port->addr);
  1381. if (ret) {
  1382. pr_err("malformed ip/port passed: %s:%s\n",
  1383. nport->disc_addr.traddr, nport->disc_addr.trsvcid);
  1384. goto err_port;
  1385. }
  1386. port->nport = nport;
  1387. INIT_WORK(&port->accept_work, nvmet_tcp_accept_work);
  1388. if (port->nport->inline_data_size < 0)
  1389. port->nport->inline_data_size = NVMET_TCP_DEF_INLINE_DATA_SIZE;
  1390. ret = sock_create(port->addr.ss_family, SOCK_STREAM,
  1391. IPPROTO_TCP, &port->sock);
  1392. if (ret) {
  1393. pr_err("failed to create a socket\n");
  1394. goto err_port;
  1395. }
  1396. port->sock->sk->sk_user_data = port;
  1397. port->data_ready = port->sock->sk->sk_data_ready;
  1398. port->sock->sk->sk_data_ready = nvmet_tcp_listen_data_ready;
  1399. sock_set_reuseaddr(port->sock->sk);
  1400. tcp_sock_set_nodelay(port->sock->sk);
  1401. if (so_priority > 0)
  1402. sock_set_priority(port->sock->sk, so_priority);
  1403. ret = kernel_bind(port->sock, (struct sockaddr *)&port->addr,
  1404. sizeof(port->addr));
  1405. if (ret) {
  1406. pr_err("failed to bind port socket %d\n", ret);
  1407. goto err_sock;
  1408. }
  1409. ret = kernel_listen(port->sock, 128);
  1410. if (ret) {
  1411. pr_err("failed to listen %d on port sock\n", ret);
  1412. goto err_sock;
  1413. }
  1414. nport->priv = port;
  1415. pr_info("enabling port %d (%pISpc)\n",
  1416. le16_to_cpu(nport->disc_addr.portid), &port->addr);
  1417. return 0;
  1418. err_sock:
  1419. sock_release(port->sock);
  1420. err_port:
  1421. kfree(port);
  1422. return ret;
  1423. }
  1424. static void nvmet_tcp_destroy_port_queues(struct nvmet_tcp_port *port)
  1425. {
  1426. struct nvmet_tcp_queue *queue;
  1427. mutex_lock(&nvmet_tcp_queue_mutex);
  1428. list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list)
  1429. if (queue->port == port)
  1430. kernel_sock_shutdown(queue->sock, SHUT_RDWR);
  1431. mutex_unlock(&nvmet_tcp_queue_mutex);
  1432. }
  1433. static void nvmet_tcp_remove_port(struct nvmet_port *nport)
  1434. {
  1435. struct nvmet_tcp_port *port = nport->priv;
  1436. write_lock_bh(&port->sock->sk->sk_callback_lock);
  1437. port->sock->sk->sk_data_ready = port->data_ready;
  1438. port->sock->sk->sk_user_data = NULL;
  1439. write_unlock_bh(&port->sock->sk->sk_callback_lock);
  1440. cancel_work_sync(&port->accept_work);
  1441. /*
  1442. * Destroy the remaining queues, which are not belong to any
  1443. * controller yet.
  1444. */
  1445. nvmet_tcp_destroy_port_queues(port);
  1446. sock_release(port->sock);
  1447. kfree(port);
  1448. }
  1449. static void nvmet_tcp_delete_ctrl(struct nvmet_ctrl *ctrl)
  1450. {
  1451. struct nvmet_tcp_queue *queue;
  1452. mutex_lock(&nvmet_tcp_queue_mutex);
  1453. list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list)
  1454. if (queue->nvme_sq.ctrl == ctrl)
  1455. kernel_sock_shutdown(queue->sock, SHUT_RDWR);
  1456. mutex_unlock(&nvmet_tcp_queue_mutex);
  1457. }
  1458. static u16 nvmet_tcp_install_queue(struct nvmet_sq *sq)
  1459. {
  1460. struct nvmet_tcp_queue *queue =
  1461. container_of(sq, struct nvmet_tcp_queue, nvme_sq);
  1462. if (sq->qid == 0) {
  1463. /* Let inflight controller teardown complete */
  1464. flush_scheduled_work();
  1465. }
  1466. queue->nr_cmds = sq->size * 2;
  1467. if (nvmet_tcp_alloc_cmds(queue))
  1468. return NVME_SC_INTERNAL;
  1469. return 0;
  1470. }
  1471. static void nvmet_tcp_disc_port_addr(struct nvmet_req *req,
  1472. struct nvmet_port *nport, char *traddr)
  1473. {
  1474. struct nvmet_tcp_port *port = nport->priv;
  1475. if (inet_addr_is_any((struct sockaddr *)&port->addr)) {
  1476. struct nvmet_tcp_cmd *cmd =
  1477. container_of(req, struct nvmet_tcp_cmd, req);
  1478. struct nvmet_tcp_queue *queue = cmd->queue;
  1479. sprintf(traddr, "%pISc", (struct sockaddr *)&queue->sockaddr);
  1480. } else {
  1481. memcpy(traddr, nport->disc_addr.traddr, NVMF_TRADDR_SIZE);
  1482. }
  1483. }
  1484. static const struct nvmet_fabrics_ops nvmet_tcp_ops = {
  1485. .owner = THIS_MODULE,
  1486. .type = NVMF_TRTYPE_TCP,
  1487. .msdbd = 1,
  1488. .add_port = nvmet_tcp_add_port,
  1489. .remove_port = nvmet_tcp_remove_port,
  1490. .queue_response = nvmet_tcp_queue_response,
  1491. .delete_ctrl = nvmet_tcp_delete_ctrl,
  1492. .install_queue = nvmet_tcp_install_queue,
  1493. .disc_traddr = nvmet_tcp_disc_port_addr,
  1494. };
  1495. static int __init nvmet_tcp_init(void)
  1496. {
  1497. int ret;
  1498. nvmet_tcp_wq = alloc_workqueue("nvmet_tcp_wq", WQ_HIGHPRI, 0);
  1499. if (!nvmet_tcp_wq)
  1500. return -ENOMEM;
  1501. ret = nvmet_register_transport(&nvmet_tcp_ops);
  1502. if (ret)
  1503. goto err;
  1504. return 0;
  1505. err:
  1506. destroy_workqueue(nvmet_tcp_wq);
  1507. return ret;
  1508. }
  1509. static void __exit nvmet_tcp_exit(void)
  1510. {
  1511. struct nvmet_tcp_queue *queue;
  1512. nvmet_unregister_transport(&nvmet_tcp_ops);
  1513. flush_scheduled_work();
  1514. mutex_lock(&nvmet_tcp_queue_mutex);
  1515. list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list)
  1516. kernel_sock_shutdown(queue->sock, SHUT_RDWR);
  1517. mutex_unlock(&nvmet_tcp_queue_mutex);
  1518. flush_scheduled_work();
  1519. destroy_workqueue(nvmet_tcp_wq);
  1520. }
  1521. module_init(nvmet_tcp_init);
  1522. module_exit(nvmet_tcp_exit);
  1523. MODULE_LICENSE("GPL v2");
  1524. MODULE_ALIAS("nvmet-transport-3"); /* 3 == NVMF_TRTYPE_TCP */