csio_scsi.c 67 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523
  1. /*
  2. * This file is part of the Chelsio FCoE driver for Linux.
  3. *
  4. * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.
  5. *
  6. * This software is available to you under a choice of one of two
  7. * licenses. You may choose to be licensed under the terms of the GNU
  8. * General Public License (GPL) Version 2, available from the file
  9. * COPYING in the main directory of this source tree, or the
  10. * OpenIB.org BSD license below:
  11. *
  12. * Redistribution and use in source and binary forms, with or
  13. * without modification, are permitted provided that the following
  14. * conditions are met:
  15. *
  16. * - Redistributions of source code must retain the above
  17. * copyright notice, this list of conditions and the following
  18. * disclaimer.
  19. *
  20. * - Redistributions in binary form must reproduce the above
  21. * copyright notice, this list of conditions and the following
  22. * disclaimer in the documentation and/or other materials
  23. * provided with the distribution.
  24. *
  25. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32. * SOFTWARE.
  33. */
  34. #include <linux/device.h>
  35. #include <linux/delay.h>
  36. #include <linux/ctype.h>
  37. #include <linux/kernel.h>
  38. #include <linux/slab.h>
  39. #include <linux/string.h>
  40. #include <linux/compiler.h>
  41. #include <linux/export.h>
  42. #include <linux/module.h>
  43. #include <asm/unaligned.h>
  44. #include <asm/page.h>
  45. #include <scsi/scsi.h>
  46. #include <scsi/scsi_device.h>
  47. #include <scsi/scsi_transport_fc.h>
  48. #include "csio_hw.h"
  49. #include "csio_lnode.h"
  50. #include "csio_rnode.h"
  51. #include "csio_scsi.h"
  52. #include "csio_init.h"
  53. int csio_scsi_eqsize = 65536;
  54. int csio_scsi_iqlen = 128;
  55. int csio_scsi_ioreqs = 2048;
  56. uint32_t csio_max_scan_tmo;
  57. uint32_t csio_delta_scan_tmo = 5;
  58. int csio_lun_qdepth = 32;
  59. static int csio_ddp_descs = 128;
  60. static int csio_do_abrt_cls(struct csio_hw *,
  61. struct csio_ioreq *, bool);
  62. static void csio_scsis_uninit(struct csio_ioreq *, enum csio_scsi_ev);
  63. static void csio_scsis_io_active(struct csio_ioreq *, enum csio_scsi_ev);
  64. static void csio_scsis_tm_active(struct csio_ioreq *, enum csio_scsi_ev);
  65. static void csio_scsis_aborting(struct csio_ioreq *, enum csio_scsi_ev);
  66. static void csio_scsis_closing(struct csio_ioreq *, enum csio_scsi_ev);
  67. static void csio_scsis_shost_cmpl_await(struct csio_ioreq *, enum csio_scsi_ev);
  68. /*
  69. * csio_scsi_match_io - Match an ioreq with the given SCSI level data.
  70. * @ioreq: The I/O request
  71. * @sld: Level information
  72. *
  73. * Should be called with lock held.
  74. *
  75. */
  76. static bool
  77. csio_scsi_match_io(struct csio_ioreq *ioreq, struct csio_scsi_level_data *sld)
  78. {
  79. struct scsi_cmnd *scmnd = csio_scsi_cmnd(ioreq);
  80. switch (sld->level) {
  81. case CSIO_LEV_LUN:
  82. if (scmnd == NULL)
  83. return false;
  84. return ((ioreq->lnode == sld->lnode) &&
  85. (ioreq->rnode == sld->rnode) &&
  86. ((uint64_t)scmnd->device->lun == sld->oslun));
  87. case CSIO_LEV_RNODE:
  88. return ((ioreq->lnode == sld->lnode) &&
  89. (ioreq->rnode == sld->rnode));
  90. case CSIO_LEV_LNODE:
  91. return (ioreq->lnode == sld->lnode);
  92. case CSIO_LEV_ALL:
  93. return true;
  94. default:
  95. return false;
  96. }
  97. }
  98. /*
  99. * csio_scsi_gather_active_ios - Gather active I/Os based on level
  100. * @scm: SCSI module
  101. * @sld: Level information
  102. * @dest: The queue where these I/Os have to be gathered.
  103. *
  104. * Should be called with lock held.
  105. */
  106. static void
  107. csio_scsi_gather_active_ios(struct csio_scsim *scm,
  108. struct csio_scsi_level_data *sld,
  109. struct list_head *dest)
  110. {
  111. struct list_head *tmp, *next;
  112. if (list_empty(&scm->active_q))
  113. return;
  114. /* Just splice the entire active_q into dest */
  115. if (sld->level == CSIO_LEV_ALL) {
  116. list_splice_tail_init(&scm->active_q, dest);
  117. return;
  118. }
  119. list_for_each_safe(tmp, next, &scm->active_q) {
  120. if (csio_scsi_match_io((struct csio_ioreq *)tmp, sld)) {
  121. list_del_init(tmp);
  122. list_add_tail(tmp, dest);
  123. }
  124. }
  125. }
  126. static inline bool
  127. csio_scsi_itnexus_loss_error(uint16_t error)
  128. {
  129. switch (error) {
  130. case FW_ERR_LINK_DOWN:
  131. case FW_RDEV_NOT_READY:
  132. case FW_ERR_RDEV_LOST:
  133. case FW_ERR_RDEV_LOGO:
  134. case FW_ERR_RDEV_IMPL_LOGO:
  135. return 1;
  136. }
  137. return 0;
  138. }
  139. /*
  140. * csio_scsi_fcp_cmnd - Frame the SCSI FCP command paylod.
  141. * @req: IO req structure.
  142. * @addr: DMA location to place the payload.
  143. *
  144. * This routine is shared between FCP_WRITE, FCP_READ and FCP_CMD requests.
  145. */
  146. static inline void
  147. csio_scsi_fcp_cmnd(struct csio_ioreq *req, void *addr)
  148. {
  149. struct fcp_cmnd *fcp_cmnd = (struct fcp_cmnd *)addr;
  150. struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
  151. /* Check for Task Management */
  152. if (likely(scmnd->SCp.Message == 0)) {
  153. int_to_scsilun(scmnd->device->lun, &fcp_cmnd->fc_lun);
  154. fcp_cmnd->fc_tm_flags = 0;
  155. fcp_cmnd->fc_cmdref = 0;
  156. memcpy(fcp_cmnd->fc_cdb, scmnd->cmnd, 16);
  157. fcp_cmnd->fc_pri_ta = FCP_PTA_SIMPLE;
  158. fcp_cmnd->fc_dl = cpu_to_be32(scsi_bufflen(scmnd));
  159. if (req->nsge)
  160. if (req->datadir == DMA_TO_DEVICE)
  161. fcp_cmnd->fc_flags = FCP_CFL_WRDATA;
  162. else
  163. fcp_cmnd->fc_flags = FCP_CFL_RDDATA;
  164. else
  165. fcp_cmnd->fc_flags = 0;
  166. } else {
  167. memset(fcp_cmnd, 0, sizeof(*fcp_cmnd));
  168. int_to_scsilun(scmnd->device->lun, &fcp_cmnd->fc_lun);
  169. fcp_cmnd->fc_tm_flags = (uint8_t)scmnd->SCp.Message;
  170. }
  171. }
  172. /*
  173. * csio_scsi_init_cmd_wr - Initialize the SCSI CMD WR.
  174. * @req: IO req structure.
  175. * @addr: DMA location to place the payload.
  176. * @size: Size of WR (including FW WR + immed data + rsp SG entry
  177. *
  178. * Wrapper for populating fw_scsi_cmd_wr.
  179. */
  180. static inline void
  181. csio_scsi_init_cmd_wr(struct csio_ioreq *req, void *addr, uint32_t size)
  182. {
  183. struct csio_hw *hw = req->lnode->hwp;
  184. struct csio_rnode *rn = req->rnode;
  185. struct fw_scsi_cmd_wr *wr = (struct fw_scsi_cmd_wr *)addr;
  186. struct csio_dma_buf *dma_buf;
  187. uint8_t imm = csio_hw_to_scsim(hw)->proto_cmd_len;
  188. wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_SCSI_CMD_WR) |
  189. FW_SCSI_CMD_WR_IMMDLEN(imm));
  190. wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(rn->flowid) |
  191. FW_WR_LEN16_V(
  192. DIV_ROUND_UP(size, 16)));
  193. wr->cookie = (uintptr_t) req;
  194. wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
  195. wr->tmo_val = (uint8_t) req->tmo;
  196. wr->r3 = 0;
  197. memset(&wr->r5, 0, 8);
  198. /* Get RSP DMA buffer */
  199. dma_buf = &req->dma_buf;
  200. /* Prepare RSP SGL */
  201. wr->rsp_dmalen = cpu_to_be32(dma_buf->len);
  202. wr->rsp_dmaaddr = cpu_to_be64(dma_buf->paddr);
  203. wr->r6 = 0;
  204. wr->u.fcoe.ctl_pri = 0;
  205. wr->u.fcoe.cp_en_class = 0;
  206. wr->u.fcoe.r4_lo[0] = 0;
  207. wr->u.fcoe.r4_lo[1] = 0;
  208. /* Frame a FCP command */
  209. csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)addr +
  210. sizeof(struct fw_scsi_cmd_wr)));
  211. }
  212. #define CSIO_SCSI_CMD_WR_SZ(_imm) \
  213. (sizeof(struct fw_scsi_cmd_wr) + /* WR size */ \
  214. ALIGN((_imm), 16)) /* Immed data */
  215. #define CSIO_SCSI_CMD_WR_SZ_16(_imm) \
  216. (ALIGN(CSIO_SCSI_CMD_WR_SZ((_imm)), 16))
  217. /*
  218. * csio_scsi_cmd - Create a SCSI CMD WR.
  219. * @req: IO req structure.
  220. *
  221. * Gets a WR slot in the ingress queue and initializes it with SCSI CMD WR.
  222. *
  223. */
  224. static inline void
  225. csio_scsi_cmd(struct csio_ioreq *req)
  226. {
  227. struct csio_wr_pair wrp;
  228. struct csio_hw *hw = req->lnode->hwp;
  229. struct csio_scsim *scsim = csio_hw_to_scsim(hw);
  230. uint32_t size = CSIO_SCSI_CMD_WR_SZ_16(scsim->proto_cmd_len);
  231. req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
  232. if (unlikely(req->drv_status != 0))
  233. return;
  234. if (wrp.size1 >= size) {
  235. /* Initialize WR in one shot */
  236. csio_scsi_init_cmd_wr(req, wrp.addr1, size);
  237. } else {
  238. uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
  239. /*
  240. * Make a temporary copy of the WR and write back
  241. * the copy into the WR pair.
  242. */
  243. csio_scsi_init_cmd_wr(req, (void *)tmpwr, size);
  244. memcpy(wrp.addr1, tmpwr, wrp.size1);
  245. memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
  246. }
  247. }
  248. /*
  249. * csio_scsi_init_ulptx_dsgl - Fill in a ULP_TX_SC_DSGL
  250. * @hw: HW module
  251. * @req: IO request
  252. * @sgl: ULP TX SGL pointer.
  253. *
  254. */
  255. static inline void
  256. csio_scsi_init_ultptx_dsgl(struct csio_hw *hw, struct csio_ioreq *req,
  257. struct ulptx_sgl *sgl)
  258. {
  259. struct ulptx_sge_pair *sge_pair = NULL;
  260. struct scatterlist *sgel;
  261. uint32_t i = 0;
  262. uint32_t xfer_len;
  263. struct list_head *tmp;
  264. struct csio_dma_buf *dma_buf;
  265. struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
  266. sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | ULPTX_MORE_F |
  267. ULPTX_NSGE_V(req->nsge));
  268. /* Now add the data SGLs */
  269. if (likely(!req->dcopy)) {
  270. scsi_for_each_sg(scmnd, sgel, req->nsge, i) {
  271. if (i == 0) {
  272. sgl->addr0 = cpu_to_be64(sg_dma_address(sgel));
  273. sgl->len0 = cpu_to_be32(sg_dma_len(sgel));
  274. sge_pair = (struct ulptx_sge_pair *)(sgl + 1);
  275. continue;
  276. }
  277. if ((i - 1) & 0x1) {
  278. sge_pair->addr[1] = cpu_to_be64(
  279. sg_dma_address(sgel));
  280. sge_pair->len[1] = cpu_to_be32(
  281. sg_dma_len(sgel));
  282. sge_pair++;
  283. } else {
  284. sge_pair->addr[0] = cpu_to_be64(
  285. sg_dma_address(sgel));
  286. sge_pair->len[0] = cpu_to_be32(
  287. sg_dma_len(sgel));
  288. }
  289. }
  290. } else {
  291. /* Program sg elements with driver's DDP buffer */
  292. xfer_len = scsi_bufflen(scmnd);
  293. list_for_each(tmp, &req->gen_list) {
  294. dma_buf = (struct csio_dma_buf *)tmp;
  295. if (i == 0) {
  296. sgl->addr0 = cpu_to_be64(dma_buf->paddr);
  297. sgl->len0 = cpu_to_be32(
  298. min(xfer_len, dma_buf->len));
  299. sge_pair = (struct ulptx_sge_pair *)(sgl + 1);
  300. } else if ((i - 1) & 0x1) {
  301. sge_pair->addr[1] = cpu_to_be64(dma_buf->paddr);
  302. sge_pair->len[1] = cpu_to_be32(
  303. min(xfer_len, dma_buf->len));
  304. sge_pair++;
  305. } else {
  306. sge_pair->addr[0] = cpu_to_be64(dma_buf->paddr);
  307. sge_pair->len[0] = cpu_to_be32(
  308. min(xfer_len, dma_buf->len));
  309. }
  310. xfer_len -= min(xfer_len, dma_buf->len);
  311. i++;
  312. }
  313. }
  314. }
  315. /*
  316. * csio_scsi_init_read_wr - Initialize the READ SCSI WR.
  317. * @req: IO req structure.
  318. * @wrp: DMA location to place the payload.
  319. * @size: Size of WR (including FW WR + immed data + rsp SG entry + data SGL
  320. *
  321. * Wrapper for populating fw_scsi_read_wr.
  322. */
  323. static inline void
  324. csio_scsi_init_read_wr(struct csio_ioreq *req, void *wrp, uint32_t size)
  325. {
  326. struct csio_hw *hw = req->lnode->hwp;
  327. struct csio_rnode *rn = req->rnode;
  328. struct fw_scsi_read_wr *wr = (struct fw_scsi_read_wr *)wrp;
  329. struct ulptx_sgl *sgl;
  330. struct csio_dma_buf *dma_buf;
  331. uint8_t imm = csio_hw_to_scsim(hw)->proto_cmd_len;
  332. struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
  333. wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_SCSI_READ_WR) |
  334. FW_SCSI_READ_WR_IMMDLEN(imm));
  335. wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(rn->flowid) |
  336. FW_WR_LEN16_V(DIV_ROUND_UP(size, 16)));
  337. wr->cookie = (uintptr_t)req;
  338. wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
  339. wr->tmo_val = (uint8_t)(req->tmo);
  340. wr->use_xfer_cnt = 1;
  341. wr->xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
  342. wr->ini_xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
  343. /* Get RSP DMA buffer */
  344. dma_buf = &req->dma_buf;
  345. /* Prepare RSP SGL */
  346. wr->rsp_dmalen = cpu_to_be32(dma_buf->len);
  347. wr->rsp_dmaaddr = cpu_to_be64(dma_buf->paddr);
  348. wr->r4 = 0;
  349. wr->u.fcoe.ctl_pri = 0;
  350. wr->u.fcoe.cp_en_class = 0;
  351. wr->u.fcoe.r3_lo[0] = 0;
  352. wr->u.fcoe.r3_lo[1] = 0;
  353. csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)wrp +
  354. sizeof(struct fw_scsi_read_wr)));
  355. /* Move WR pointer past command and immediate data */
  356. sgl = (struct ulptx_sgl *)((uintptr_t)wrp +
  357. sizeof(struct fw_scsi_read_wr) + ALIGN(imm, 16));
  358. /* Fill in the DSGL */
  359. csio_scsi_init_ultptx_dsgl(hw, req, sgl);
  360. }
  361. /*
  362. * csio_scsi_init_write_wr - Initialize the WRITE SCSI WR.
  363. * @req: IO req structure.
  364. * @wrp: DMA location to place the payload.
  365. * @size: Size of WR (including FW WR + immed data + rsp SG entry + data SGL
  366. *
  367. * Wrapper for populating fw_scsi_write_wr.
  368. */
  369. static inline void
  370. csio_scsi_init_write_wr(struct csio_ioreq *req, void *wrp, uint32_t size)
  371. {
  372. struct csio_hw *hw = req->lnode->hwp;
  373. struct csio_rnode *rn = req->rnode;
  374. struct fw_scsi_write_wr *wr = (struct fw_scsi_write_wr *)wrp;
  375. struct ulptx_sgl *sgl;
  376. struct csio_dma_buf *dma_buf;
  377. uint8_t imm = csio_hw_to_scsim(hw)->proto_cmd_len;
  378. struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
  379. wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_SCSI_WRITE_WR) |
  380. FW_SCSI_WRITE_WR_IMMDLEN(imm));
  381. wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(rn->flowid) |
  382. FW_WR_LEN16_V(DIV_ROUND_UP(size, 16)));
  383. wr->cookie = (uintptr_t)req;
  384. wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
  385. wr->tmo_val = (uint8_t)(req->tmo);
  386. wr->use_xfer_cnt = 1;
  387. wr->xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
  388. wr->ini_xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
  389. /* Get RSP DMA buffer */
  390. dma_buf = &req->dma_buf;
  391. /* Prepare RSP SGL */
  392. wr->rsp_dmalen = cpu_to_be32(dma_buf->len);
  393. wr->rsp_dmaaddr = cpu_to_be64(dma_buf->paddr);
  394. wr->r4 = 0;
  395. wr->u.fcoe.ctl_pri = 0;
  396. wr->u.fcoe.cp_en_class = 0;
  397. wr->u.fcoe.r3_lo[0] = 0;
  398. wr->u.fcoe.r3_lo[1] = 0;
  399. csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)wrp +
  400. sizeof(struct fw_scsi_write_wr)));
  401. /* Move WR pointer past command and immediate data */
  402. sgl = (struct ulptx_sgl *)((uintptr_t)wrp +
  403. sizeof(struct fw_scsi_write_wr) + ALIGN(imm, 16));
  404. /* Fill in the DSGL */
  405. csio_scsi_init_ultptx_dsgl(hw, req, sgl);
  406. }
  407. /* Calculate WR size needed for fw_scsi_read_wr/fw_scsi_write_wr */
  408. #define CSIO_SCSI_DATA_WRSZ(req, oper, sz, imm) \
  409. do { \
  410. (sz) = sizeof(struct fw_scsi_##oper##_wr) + /* WR size */ \
  411. ALIGN((imm), 16) + /* Immed data */ \
  412. sizeof(struct ulptx_sgl); /* ulptx_sgl */ \
  413. \
  414. if (unlikely((req)->nsge > 1)) \
  415. (sz) += (sizeof(struct ulptx_sge_pair) * \
  416. (ALIGN(((req)->nsge - 1), 2) / 2)); \
  417. /* Data SGE */ \
  418. } while (0)
  419. /*
  420. * csio_scsi_read - Create a SCSI READ WR.
  421. * @req: IO req structure.
  422. *
  423. * Gets a WR slot in the ingress queue and initializes it with
  424. * SCSI READ WR.
  425. *
  426. */
  427. static inline void
  428. csio_scsi_read(struct csio_ioreq *req)
  429. {
  430. struct csio_wr_pair wrp;
  431. uint32_t size;
  432. struct csio_hw *hw = req->lnode->hwp;
  433. struct csio_scsim *scsim = csio_hw_to_scsim(hw);
  434. CSIO_SCSI_DATA_WRSZ(req, read, size, scsim->proto_cmd_len);
  435. size = ALIGN(size, 16);
  436. req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
  437. if (likely(req->drv_status == 0)) {
  438. if (likely(wrp.size1 >= size)) {
  439. /* Initialize WR in one shot */
  440. csio_scsi_init_read_wr(req, wrp.addr1, size);
  441. } else {
  442. uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
  443. /*
  444. * Make a temporary copy of the WR and write back
  445. * the copy into the WR pair.
  446. */
  447. csio_scsi_init_read_wr(req, (void *)tmpwr, size);
  448. memcpy(wrp.addr1, tmpwr, wrp.size1);
  449. memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
  450. }
  451. }
  452. }
  453. /*
  454. * csio_scsi_write - Create a SCSI WRITE WR.
  455. * @req: IO req structure.
  456. *
  457. * Gets a WR slot in the ingress queue and initializes it with
  458. * SCSI WRITE WR.
  459. *
  460. */
  461. static inline void
  462. csio_scsi_write(struct csio_ioreq *req)
  463. {
  464. struct csio_wr_pair wrp;
  465. uint32_t size;
  466. struct csio_hw *hw = req->lnode->hwp;
  467. struct csio_scsim *scsim = csio_hw_to_scsim(hw);
  468. CSIO_SCSI_DATA_WRSZ(req, write, size, scsim->proto_cmd_len);
  469. size = ALIGN(size, 16);
  470. req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
  471. if (likely(req->drv_status == 0)) {
  472. if (likely(wrp.size1 >= size)) {
  473. /* Initialize WR in one shot */
  474. csio_scsi_init_write_wr(req, wrp.addr1, size);
  475. } else {
  476. uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
  477. /*
  478. * Make a temporary copy of the WR and write back
  479. * the copy into the WR pair.
  480. */
  481. csio_scsi_init_write_wr(req, (void *)tmpwr, size);
  482. memcpy(wrp.addr1, tmpwr, wrp.size1);
  483. memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
  484. }
  485. }
  486. }
  487. /*
  488. * csio_setup_ddp - Setup DDP buffers for Read request.
  489. * @req: IO req structure.
  490. *
  491. * Checks SGLs/Data buffers are virtually contiguous required for DDP.
  492. * If contiguous,driver posts SGLs in the WR otherwise post internal
  493. * buffers for such request for DDP.
  494. */
  495. static inline void
  496. csio_setup_ddp(struct csio_scsim *scsim, struct csio_ioreq *req)
  497. {
  498. #ifdef __CSIO_DEBUG__
  499. struct csio_hw *hw = req->lnode->hwp;
  500. #endif
  501. struct scatterlist *sgel = NULL;
  502. struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
  503. uint64_t sg_addr = 0;
  504. uint32_t ddp_pagesz = 4096;
  505. uint32_t buf_off;
  506. struct csio_dma_buf *dma_buf = NULL;
  507. uint32_t alloc_len = 0;
  508. uint32_t xfer_len = 0;
  509. uint32_t sg_len = 0;
  510. uint32_t i;
  511. scsi_for_each_sg(scmnd, sgel, req->nsge, i) {
  512. sg_addr = sg_dma_address(sgel);
  513. sg_len = sg_dma_len(sgel);
  514. buf_off = sg_addr & (ddp_pagesz - 1);
  515. /* Except 1st buffer,all buffer addr have to be Page aligned */
  516. if (i != 0 && buf_off) {
  517. csio_dbg(hw, "SGL addr not DDP aligned (%llx:%d)\n",
  518. sg_addr, sg_len);
  519. goto unaligned;
  520. }
  521. /* Except last buffer,all buffer must end on page boundary */
  522. if ((i != (req->nsge - 1)) &&
  523. ((buf_off + sg_len) & (ddp_pagesz - 1))) {
  524. csio_dbg(hw,
  525. "SGL addr not ending on page boundary"
  526. "(%llx:%d)\n", sg_addr, sg_len);
  527. goto unaligned;
  528. }
  529. }
  530. /* SGL's are virtually contiguous. HW will DDP to SGLs */
  531. req->dcopy = 0;
  532. csio_scsi_read(req);
  533. return;
  534. unaligned:
  535. CSIO_INC_STATS(scsim, n_unaligned);
  536. /*
  537. * For unaligned SGLs, driver will allocate internal DDP buffer.
  538. * Once command is completed data from DDP buffer copied to SGLs
  539. */
  540. req->dcopy = 1;
  541. /* Use gen_list to store the DDP buffers */
  542. INIT_LIST_HEAD(&req->gen_list);
  543. xfer_len = scsi_bufflen(scmnd);
  544. i = 0;
  545. /* Allocate ddp buffers for this request */
  546. while (alloc_len < xfer_len) {
  547. dma_buf = csio_get_scsi_ddp(scsim);
  548. if (dma_buf == NULL || i > scsim->max_sge) {
  549. req->drv_status = -EBUSY;
  550. break;
  551. }
  552. alloc_len += dma_buf->len;
  553. /* Added to IO req */
  554. list_add_tail(&dma_buf->list, &req->gen_list);
  555. i++;
  556. }
  557. if (!req->drv_status) {
  558. /* set number of ddp bufs used */
  559. req->nsge = i;
  560. csio_scsi_read(req);
  561. return;
  562. }
  563. /* release dma descs */
  564. if (i > 0)
  565. csio_put_scsi_ddp_list(scsim, &req->gen_list, i);
  566. }
  567. /*
  568. * csio_scsi_init_abrt_cls_wr - Initialize an ABORT/CLOSE WR.
  569. * @req: IO req structure.
  570. * @addr: DMA location to place the payload.
  571. * @size: Size of WR
  572. * @abort: abort OR close
  573. *
  574. * Wrapper for populating fw_scsi_cmd_wr.
  575. */
  576. static inline void
  577. csio_scsi_init_abrt_cls_wr(struct csio_ioreq *req, void *addr, uint32_t size,
  578. bool abort)
  579. {
  580. struct csio_hw *hw = req->lnode->hwp;
  581. struct csio_rnode *rn = req->rnode;
  582. struct fw_scsi_abrt_cls_wr *wr = (struct fw_scsi_abrt_cls_wr *)addr;
  583. wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_SCSI_ABRT_CLS_WR));
  584. wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(rn->flowid) |
  585. FW_WR_LEN16_V(
  586. DIV_ROUND_UP(size, 16)));
  587. wr->cookie = (uintptr_t) req;
  588. wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
  589. wr->tmo_val = (uint8_t) req->tmo;
  590. /* 0 for CHK_ALL_IO tells FW to look up t_cookie */
  591. wr->sub_opcode_to_chk_all_io =
  592. (FW_SCSI_ABRT_CLS_WR_SUB_OPCODE(abort) |
  593. FW_SCSI_ABRT_CLS_WR_CHK_ALL_IO(0));
  594. wr->r3[0] = 0;
  595. wr->r3[1] = 0;
  596. wr->r3[2] = 0;
  597. wr->r3[3] = 0;
  598. /* Since we re-use the same ioreq for abort as well */
  599. wr->t_cookie = (uintptr_t) req;
  600. }
  601. static inline void
  602. csio_scsi_abrt_cls(struct csio_ioreq *req, bool abort)
  603. {
  604. struct csio_wr_pair wrp;
  605. struct csio_hw *hw = req->lnode->hwp;
  606. uint32_t size = ALIGN(sizeof(struct fw_scsi_abrt_cls_wr), 16);
  607. req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
  608. if (req->drv_status != 0)
  609. return;
  610. if (wrp.size1 >= size) {
  611. /* Initialize WR in one shot */
  612. csio_scsi_init_abrt_cls_wr(req, wrp.addr1, size, abort);
  613. } else {
  614. uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
  615. /*
  616. * Make a temporary copy of the WR and write back
  617. * the copy into the WR pair.
  618. */
  619. csio_scsi_init_abrt_cls_wr(req, (void *)tmpwr, size, abort);
  620. memcpy(wrp.addr1, tmpwr, wrp.size1);
  621. memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
  622. }
  623. }
  624. /*****************************************************************************/
  625. /* START: SCSI SM */
  626. /*****************************************************************************/
  627. static void
  628. csio_scsis_uninit(struct csio_ioreq *req, enum csio_scsi_ev evt)
  629. {
  630. struct csio_hw *hw = req->lnode->hwp;
  631. struct csio_scsim *scsim = csio_hw_to_scsim(hw);
  632. switch (evt) {
  633. case CSIO_SCSIE_START_IO:
  634. if (req->nsge) {
  635. if (req->datadir == DMA_TO_DEVICE) {
  636. req->dcopy = 0;
  637. csio_scsi_write(req);
  638. } else
  639. csio_setup_ddp(scsim, req);
  640. } else {
  641. csio_scsi_cmd(req);
  642. }
  643. if (likely(req->drv_status == 0)) {
  644. /* change state and enqueue on active_q */
  645. csio_set_state(&req->sm, csio_scsis_io_active);
  646. list_add_tail(&req->sm.sm_list, &scsim->active_q);
  647. csio_wr_issue(hw, req->eq_idx, false);
  648. CSIO_INC_STATS(scsim, n_active);
  649. return;
  650. }
  651. break;
  652. case CSIO_SCSIE_START_TM:
  653. csio_scsi_cmd(req);
  654. if (req->drv_status == 0) {
  655. /*
  656. * NOTE: We collect the affected I/Os prior to issuing
  657. * LUN reset, and not after it. This is to prevent
  658. * aborting I/Os that get issued after the LUN reset,
  659. * but prior to LUN reset completion (in the event that
  660. * the host stack has not blocked I/Os to a LUN that is
  661. * being reset.
  662. */
  663. csio_set_state(&req->sm, csio_scsis_tm_active);
  664. list_add_tail(&req->sm.sm_list, &scsim->active_q);
  665. csio_wr_issue(hw, req->eq_idx, false);
  666. CSIO_INC_STATS(scsim, n_tm_active);
  667. }
  668. return;
  669. case CSIO_SCSIE_ABORT:
  670. case CSIO_SCSIE_CLOSE:
  671. /*
  672. * NOTE:
  673. * We could get here due to :
  674. * - a window in the cleanup path of the SCSI module
  675. * (csio_scsi_abort_io()). Please see NOTE in this function.
  676. * - a window in the time we tried to issue an abort/close
  677. * of a request to FW, and the FW completed the request
  678. * itself.
  679. * Print a message for now, and return INVAL either way.
  680. */
  681. req->drv_status = -EINVAL;
  682. csio_warn(hw, "Trying to abort/close completed IO:%p!\n", req);
  683. break;
  684. default:
  685. csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
  686. CSIO_DB_ASSERT(0);
  687. }
  688. }
  689. static void
  690. csio_scsis_io_active(struct csio_ioreq *req, enum csio_scsi_ev evt)
  691. {
  692. struct csio_hw *hw = req->lnode->hwp;
  693. struct csio_scsim *scm = csio_hw_to_scsim(hw);
  694. struct csio_rnode *rn;
  695. switch (evt) {
  696. case CSIO_SCSIE_COMPLETED:
  697. CSIO_DEC_STATS(scm, n_active);
  698. list_del_init(&req->sm.sm_list);
  699. csio_set_state(&req->sm, csio_scsis_uninit);
  700. /*
  701. * In MSIX mode, with multiple queues, the SCSI compeltions
  702. * could reach us sooner than the FW events sent to indicate
  703. * I-T nexus loss (link down, remote device logo etc). We
  704. * dont want to be returning such I/Os to the upper layer
  705. * immediately, since we wouldnt have reported the I-T nexus
  706. * loss itself. This forces us to serialize such completions
  707. * with the reporting of the I-T nexus loss. Therefore, we
  708. * internally queue up such up such completions in the rnode.
  709. * The reporting of I-T nexus loss to the upper layer is then
  710. * followed by the returning of I/Os in this internal queue.
  711. * Having another state alongwith another queue helps us take
  712. * actions for events such as ABORT received while we are
  713. * in this rnode queue.
  714. */
  715. if (unlikely(req->wr_status != FW_SUCCESS)) {
  716. rn = req->rnode;
  717. /*
  718. * FW says remote device is lost, but rnode
  719. * doesnt reflect it.
  720. */
  721. if (csio_scsi_itnexus_loss_error(req->wr_status) &&
  722. csio_is_rnode_ready(rn)) {
  723. csio_set_state(&req->sm,
  724. csio_scsis_shost_cmpl_await);
  725. list_add_tail(&req->sm.sm_list,
  726. &rn->host_cmpl_q);
  727. }
  728. }
  729. break;
  730. case CSIO_SCSIE_ABORT:
  731. csio_scsi_abrt_cls(req, SCSI_ABORT);
  732. if (req->drv_status == 0) {
  733. csio_wr_issue(hw, req->eq_idx, false);
  734. csio_set_state(&req->sm, csio_scsis_aborting);
  735. }
  736. break;
  737. case CSIO_SCSIE_CLOSE:
  738. csio_scsi_abrt_cls(req, SCSI_CLOSE);
  739. if (req->drv_status == 0) {
  740. csio_wr_issue(hw, req->eq_idx, false);
  741. csio_set_state(&req->sm, csio_scsis_closing);
  742. }
  743. break;
  744. case CSIO_SCSIE_DRVCLEANUP:
  745. req->wr_status = FW_HOSTERROR;
  746. CSIO_DEC_STATS(scm, n_active);
  747. csio_set_state(&req->sm, csio_scsis_uninit);
  748. break;
  749. default:
  750. csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
  751. CSIO_DB_ASSERT(0);
  752. }
  753. }
  754. static void
  755. csio_scsis_tm_active(struct csio_ioreq *req, enum csio_scsi_ev evt)
  756. {
  757. struct csio_hw *hw = req->lnode->hwp;
  758. struct csio_scsim *scm = csio_hw_to_scsim(hw);
  759. switch (evt) {
  760. case CSIO_SCSIE_COMPLETED:
  761. CSIO_DEC_STATS(scm, n_tm_active);
  762. list_del_init(&req->sm.sm_list);
  763. csio_set_state(&req->sm, csio_scsis_uninit);
  764. break;
  765. case CSIO_SCSIE_ABORT:
  766. csio_scsi_abrt_cls(req, SCSI_ABORT);
  767. if (req->drv_status == 0) {
  768. csio_wr_issue(hw, req->eq_idx, false);
  769. csio_set_state(&req->sm, csio_scsis_aborting);
  770. }
  771. break;
  772. case CSIO_SCSIE_CLOSE:
  773. csio_scsi_abrt_cls(req, SCSI_CLOSE);
  774. if (req->drv_status == 0) {
  775. csio_wr_issue(hw, req->eq_idx, false);
  776. csio_set_state(&req->sm, csio_scsis_closing);
  777. }
  778. break;
  779. case CSIO_SCSIE_DRVCLEANUP:
  780. req->wr_status = FW_HOSTERROR;
  781. CSIO_DEC_STATS(scm, n_tm_active);
  782. csio_set_state(&req->sm, csio_scsis_uninit);
  783. break;
  784. default:
  785. csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
  786. CSIO_DB_ASSERT(0);
  787. }
  788. }
  789. static void
  790. csio_scsis_aborting(struct csio_ioreq *req, enum csio_scsi_ev evt)
  791. {
  792. struct csio_hw *hw = req->lnode->hwp;
  793. struct csio_scsim *scm = csio_hw_to_scsim(hw);
  794. switch (evt) {
  795. case CSIO_SCSIE_COMPLETED:
  796. csio_dbg(hw,
  797. "ioreq %p recvd cmpltd (wr_status:%d) "
  798. "in aborting st\n", req, req->wr_status);
  799. /*
  800. * Use -ECANCELED to explicitly tell the ABORTED event that
  801. * the original I/O was returned to driver by FW.
  802. * We dont really care if the I/O was returned with success by
  803. * FW (because the ABORT and completion of the I/O crossed each
  804. * other), or any other return value. Once we are in aborting
  805. * state, the success or failure of the I/O is unimportant to
  806. * us.
  807. */
  808. req->drv_status = -ECANCELED;
  809. break;
  810. case CSIO_SCSIE_ABORT:
  811. CSIO_INC_STATS(scm, n_abrt_dups);
  812. break;
  813. case CSIO_SCSIE_ABORTED:
  814. csio_dbg(hw, "abort of %p return status:0x%x drv_status:%x\n",
  815. req, req->wr_status, req->drv_status);
  816. /*
  817. * Check if original I/O WR completed before the Abort
  818. * completion.
  819. */
  820. if (req->drv_status != -ECANCELED) {
  821. csio_warn(hw,
  822. "Abort completed before original I/O,"
  823. " req:%p\n", req);
  824. CSIO_DB_ASSERT(0);
  825. }
  826. /*
  827. * There are the following possible scenarios:
  828. * 1. The abort completed successfully, FW returned FW_SUCCESS.
  829. * 2. The completion of an I/O and the receipt of
  830. * abort for that I/O by the FW crossed each other.
  831. * The FW returned FW_EINVAL. The original I/O would have
  832. * returned with FW_SUCCESS or any other SCSI error.
  833. * 3. The FW couldn't sent the abort out on the wire, as there
  834. * was an I-T nexus loss (link down, remote device logged
  835. * out etc). FW sent back an appropriate IT nexus loss status
  836. * for the abort.
  837. * 4. FW sent an abort, but abort timed out (remote device
  838. * didnt respond). FW replied back with
  839. * FW_SCSI_ABORT_TIMEDOUT.
  840. * 5. FW couldn't genuinely abort the request for some reason,
  841. * and sent us an error.
  842. *
  843. * The first 3 scenarios are treated as succesful abort
  844. * operations by the host, while the last 2 are failed attempts
  845. * to abort. Manipulate the return value of the request
  846. * appropriately, so that host can convey these results
  847. * back to the upper layer.
  848. */
  849. if ((req->wr_status == FW_SUCCESS) ||
  850. (req->wr_status == FW_EINVAL) ||
  851. csio_scsi_itnexus_loss_error(req->wr_status))
  852. req->wr_status = FW_SCSI_ABORT_REQUESTED;
  853. CSIO_DEC_STATS(scm, n_active);
  854. list_del_init(&req->sm.sm_list);
  855. csio_set_state(&req->sm, csio_scsis_uninit);
  856. break;
  857. case CSIO_SCSIE_DRVCLEANUP:
  858. req->wr_status = FW_HOSTERROR;
  859. CSIO_DEC_STATS(scm, n_active);
  860. csio_set_state(&req->sm, csio_scsis_uninit);
  861. break;
  862. case CSIO_SCSIE_CLOSE:
  863. /*
  864. * We can receive this event from the module
  865. * cleanup paths, if the FW forgot to reply to the ABORT WR
  866. * and left this ioreq in this state. For now, just ignore
  867. * the event. The CLOSE event is sent to this state, as
  868. * the LINK may have already gone down.
  869. */
  870. break;
  871. default:
  872. csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
  873. CSIO_DB_ASSERT(0);
  874. }
  875. }
  876. static void
  877. csio_scsis_closing(struct csio_ioreq *req, enum csio_scsi_ev evt)
  878. {
  879. struct csio_hw *hw = req->lnode->hwp;
  880. struct csio_scsim *scm = csio_hw_to_scsim(hw);
  881. switch (evt) {
  882. case CSIO_SCSIE_COMPLETED:
  883. csio_dbg(hw,
  884. "ioreq %p recvd cmpltd (wr_status:%d) "
  885. "in closing st\n", req, req->wr_status);
  886. /*
  887. * Use -ECANCELED to explicitly tell the CLOSED event that
  888. * the original I/O was returned to driver by FW.
  889. * We dont really care if the I/O was returned with success by
  890. * FW (because the CLOSE and completion of the I/O crossed each
  891. * other), or any other return value. Once we are in aborting
  892. * state, the success or failure of the I/O is unimportant to
  893. * us.
  894. */
  895. req->drv_status = -ECANCELED;
  896. break;
  897. case CSIO_SCSIE_CLOSED:
  898. /*
  899. * Check if original I/O WR completed before the Close
  900. * completion.
  901. */
  902. if (req->drv_status != -ECANCELED) {
  903. csio_fatal(hw,
  904. "Close completed before original I/O,"
  905. " req:%p\n", req);
  906. CSIO_DB_ASSERT(0);
  907. }
  908. /*
  909. * Either close succeeded, or we issued close to FW at the
  910. * same time FW compelted it to us. Either way, the I/O
  911. * is closed.
  912. */
  913. CSIO_DB_ASSERT((req->wr_status == FW_SUCCESS) ||
  914. (req->wr_status == FW_EINVAL));
  915. req->wr_status = FW_SCSI_CLOSE_REQUESTED;
  916. CSIO_DEC_STATS(scm, n_active);
  917. list_del_init(&req->sm.sm_list);
  918. csio_set_state(&req->sm, csio_scsis_uninit);
  919. break;
  920. case CSIO_SCSIE_CLOSE:
  921. break;
  922. case CSIO_SCSIE_DRVCLEANUP:
  923. req->wr_status = FW_HOSTERROR;
  924. CSIO_DEC_STATS(scm, n_active);
  925. csio_set_state(&req->sm, csio_scsis_uninit);
  926. break;
  927. default:
  928. csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
  929. CSIO_DB_ASSERT(0);
  930. }
  931. }
  932. static void
  933. csio_scsis_shost_cmpl_await(struct csio_ioreq *req, enum csio_scsi_ev evt)
  934. {
  935. switch (evt) {
  936. case CSIO_SCSIE_ABORT:
  937. case CSIO_SCSIE_CLOSE:
  938. /*
  939. * Just succeed the abort request, and hope that
  940. * the remote device unregister path will cleanup
  941. * this I/O to the upper layer within a sane
  942. * amount of time.
  943. */
  944. /*
  945. * A close can come in during a LINK DOWN. The FW would have
  946. * returned us the I/O back, but not the remote device lost
  947. * FW event. In this interval, if the I/O times out at the upper
  948. * layer, a close can come in. Take the same action as abort:
  949. * return success, and hope that the remote device unregister
  950. * path will cleanup this I/O. If the FW still doesnt send
  951. * the msg, the close times out, and the upper layer resorts
  952. * to the next level of error recovery.
  953. */
  954. req->drv_status = 0;
  955. break;
  956. case CSIO_SCSIE_DRVCLEANUP:
  957. csio_set_state(&req->sm, csio_scsis_uninit);
  958. break;
  959. default:
  960. csio_dbg(req->lnode->hwp, "Unhandled event:%d sent to req:%p\n",
  961. evt, req);
  962. CSIO_DB_ASSERT(0);
  963. }
  964. }
  965. /*
  966. * csio_scsi_cmpl_handler - WR completion handler for SCSI.
  967. * @hw: HW module.
  968. * @wr: The completed WR from the ingress queue.
  969. * @len: Length of the WR.
  970. * @flb: Freelist buffer array.
  971. * @priv: Private object
  972. * @scsiwr: Pointer to SCSI WR.
  973. *
  974. * This is the WR completion handler called per completion from the
  975. * ISR. It is called with lock held. It walks past the RSS and CPL message
  976. * header where the actual WR is present.
  977. * It then gets the status, WR handle (ioreq pointer) and the len of
  978. * the WR, based on WR opcode. Only on a non-good status is the entire
  979. * WR copied into the WR cache (ioreq->fw_wr).
  980. * The ioreq corresponding to the WR is returned to the caller.
  981. * NOTE: The SCSI queue doesnt allocate a freelist today, hence
  982. * no freelist buffer is expected.
  983. */
  984. struct csio_ioreq *
  985. csio_scsi_cmpl_handler(struct csio_hw *hw, void *wr, uint32_t len,
  986. struct csio_fl_dma_buf *flb, void *priv, uint8_t **scsiwr)
  987. {
  988. struct csio_ioreq *ioreq = NULL;
  989. struct cpl_fw6_msg *cpl;
  990. uint8_t *tempwr;
  991. uint8_t status;
  992. struct csio_scsim *scm = csio_hw_to_scsim(hw);
  993. /* skip RSS header */
  994. cpl = (struct cpl_fw6_msg *)((uintptr_t)wr + sizeof(__be64));
  995. if (unlikely(cpl->opcode != CPL_FW6_MSG)) {
  996. csio_warn(hw, "Error: Invalid CPL msg %x recvd on SCSI q\n",
  997. cpl->opcode);
  998. CSIO_INC_STATS(scm, n_inval_cplop);
  999. return NULL;
  1000. }
  1001. tempwr = (uint8_t *)(cpl->data);
  1002. status = csio_wr_status(tempwr);
  1003. *scsiwr = tempwr;
  1004. if (likely((*tempwr == FW_SCSI_READ_WR) ||
  1005. (*tempwr == FW_SCSI_WRITE_WR) ||
  1006. (*tempwr == FW_SCSI_CMD_WR))) {
  1007. ioreq = (struct csio_ioreq *)((uintptr_t)
  1008. (((struct fw_scsi_read_wr *)tempwr)->cookie));
  1009. CSIO_DB_ASSERT(virt_addr_valid(ioreq));
  1010. ioreq->wr_status = status;
  1011. return ioreq;
  1012. }
  1013. if (*tempwr == FW_SCSI_ABRT_CLS_WR) {
  1014. ioreq = (struct csio_ioreq *)((uintptr_t)
  1015. (((struct fw_scsi_abrt_cls_wr *)tempwr)->cookie));
  1016. CSIO_DB_ASSERT(virt_addr_valid(ioreq));
  1017. ioreq->wr_status = status;
  1018. return ioreq;
  1019. }
  1020. csio_warn(hw, "WR with invalid opcode in SCSI IQ: %x\n", *tempwr);
  1021. CSIO_INC_STATS(scm, n_inval_scsiop);
  1022. return NULL;
  1023. }
  1024. /*
  1025. * csio_scsi_cleanup_io_q - Cleanup the given queue.
  1026. * @scm: SCSI module.
  1027. * @q: Queue to be cleaned up.
  1028. *
  1029. * Called with lock held. Has to exit with lock held.
  1030. */
  1031. void
  1032. csio_scsi_cleanup_io_q(struct csio_scsim *scm, struct list_head *q)
  1033. {
  1034. struct csio_hw *hw = scm->hw;
  1035. struct csio_ioreq *ioreq;
  1036. struct list_head *tmp, *next;
  1037. struct scsi_cmnd *scmnd;
  1038. /* Call back the completion routines of the active_q */
  1039. list_for_each_safe(tmp, next, q) {
  1040. ioreq = (struct csio_ioreq *)tmp;
  1041. csio_scsi_drvcleanup(ioreq);
  1042. list_del_init(&ioreq->sm.sm_list);
  1043. scmnd = csio_scsi_cmnd(ioreq);
  1044. spin_unlock_irq(&hw->lock);
  1045. /*
  1046. * Upper layers may have cleared this command, hence this
  1047. * check to avoid accessing stale references.
  1048. */
  1049. if (scmnd != NULL)
  1050. ioreq->io_cbfn(hw, ioreq);
  1051. spin_lock_irq(&scm->freelist_lock);
  1052. csio_put_scsi_ioreq(scm, ioreq);
  1053. spin_unlock_irq(&scm->freelist_lock);
  1054. spin_lock_irq(&hw->lock);
  1055. }
  1056. }
  1057. #define CSIO_SCSI_ABORT_Q_POLL_MS 2000
  1058. static void
  1059. csio_abrt_cls(struct csio_ioreq *ioreq, struct scsi_cmnd *scmnd)
  1060. {
  1061. struct csio_lnode *ln = ioreq->lnode;
  1062. struct csio_hw *hw = ln->hwp;
  1063. int ready = 0;
  1064. struct csio_scsim *scsim = csio_hw_to_scsim(hw);
  1065. int rv;
  1066. if (csio_scsi_cmnd(ioreq) != scmnd) {
  1067. CSIO_INC_STATS(scsim, n_abrt_race_comp);
  1068. return;
  1069. }
  1070. ready = csio_is_lnode_ready(ln);
  1071. rv = csio_do_abrt_cls(hw, ioreq, (ready ? SCSI_ABORT : SCSI_CLOSE));
  1072. if (rv != 0) {
  1073. if (ready)
  1074. CSIO_INC_STATS(scsim, n_abrt_busy_error);
  1075. else
  1076. CSIO_INC_STATS(scsim, n_cls_busy_error);
  1077. }
  1078. }
  1079. /*
  1080. * csio_scsi_abort_io_q - Abort all I/Os on given queue
  1081. * @scm: SCSI module.
  1082. * @q: Queue to abort.
  1083. * @tmo: Timeout in ms
  1084. *
  1085. * Attempt to abort all I/Os on given queue, and wait for a max
  1086. * of tmo milliseconds for them to complete. Returns success
  1087. * if all I/Os are aborted. Else returns -ETIMEDOUT.
  1088. * Should be entered with lock held. Exits with lock held.
  1089. * NOTE:
  1090. * Lock has to be held across the loop that aborts I/Os, since dropping the lock
  1091. * in between can cause the list to be corrupted. As a result, the caller
  1092. * of this function has to ensure that the number of I/os to be aborted
  1093. * is finite enough to not cause lock-held-for-too-long issues.
  1094. */
  1095. static int
  1096. csio_scsi_abort_io_q(struct csio_scsim *scm, struct list_head *q, uint32_t tmo)
  1097. {
  1098. struct csio_hw *hw = scm->hw;
  1099. struct list_head *tmp, *next;
  1100. int count = DIV_ROUND_UP(tmo, CSIO_SCSI_ABORT_Q_POLL_MS);
  1101. struct scsi_cmnd *scmnd;
  1102. if (list_empty(q))
  1103. return 0;
  1104. csio_dbg(hw, "Aborting SCSI I/Os\n");
  1105. /* Now abort/close I/Os in the queue passed */
  1106. list_for_each_safe(tmp, next, q) {
  1107. scmnd = csio_scsi_cmnd((struct csio_ioreq *)tmp);
  1108. csio_abrt_cls((struct csio_ioreq *)tmp, scmnd);
  1109. }
  1110. /* Wait till all active I/Os are completed/aborted/closed */
  1111. while (!list_empty(q) && count--) {
  1112. spin_unlock_irq(&hw->lock);
  1113. msleep(CSIO_SCSI_ABORT_Q_POLL_MS);
  1114. spin_lock_irq(&hw->lock);
  1115. }
  1116. /* all aborts completed */
  1117. if (list_empty(q))
  1118. return 0;
  1119. return -ETIMEDOUT;
  1120. }
  1121. /*
  1122. * csio_scsim_cleanup_io - Cleanup all I/Os in SCSI module.
  1123. * @scm: SCSI module.
  1124. * @abort: abort required.
  1125. * Called with lock held, should exit with lock held.
  1126. * Can sleep when waiting for I/Os to complete.
  1127. */
  1128. int
  1129. csio_scsim_cleanup_io(struct csio_scsim *scm, bool abort)
  1130. {
  1131. struct csio_hw *hw = scm->hw;
  1132. int rv = 0;
  1133. int count = DIV_ROUND_UP(60 * 1000, CSIO_SCSI_ABORT_Q_POLL_MS);
  1134. /* No I/Os pending */
  1135. if (list_empty(&scm->active_q))
  1136. return 0;
  1137. /* Wait until all active I/Os are completed */
  1138. while (!list_empty(&scm->active_q) && count--) {
  1139. spin_unlock_irq(&hw->lock);
  1140. msleep(CSIO_SCSI_ABORT_Q_POLL_MS);
  1141. spin_lock_irq(&hw->lock);
  1142. }
  1143. /* all I/Os completed */
  1144. if (list_empty(&scm->active_q))
  1145. return 0;
  1146. /* Else abort */
  1147. if (abort) {
  1148. rv = csio_scsi_abort_io_q(scm, &scm->active_q, 30000);
  1149. if (rv == 0)
  1150. return rv;
  1151. csio_dbg(hw, "Some I/O aborts timed out, cleaning up..\n");
  1152. }
  1153. csio_scsi_cleanup_io_q(scm, &scm->active_q);
  1154. CSIO_DB_ASSERT(list_empty(&scm->active_q));
  1155. return rv;
  1156. }
  1157. /*
  1158. * csio_scsim_cleanup_io_lnode - Cleanup all I/Os of given lnode.
  1159. * @scm: SCSI module.
  1160. * @lnode: lnode
  1161. *
  1162. * Called with lock held, should exit with lock held.
  1163. * Can sleep (with dropped lock) when waiting for I/Os to complete.
  1164. */
  1165. int
  1166. csio_scsim_cleanup_io_lnode(struct csio_scsim *scm, struct csio_lnode *ln)
  1167. {
  1168. struct csio_hw *hw = scm->hw;
  1169. struct csio_scsi_level_data sld;
  1170. int rv;
  1171. int count = DIV_ROUND_UP(60 * 1000, CSIO_SCSI_ABORT_Q_POLL_MS);
  1172. csio_dbg(hw, "Gathering all SCSI I/Os on lnode %p\n", ln);
  1173. sld.level = CSIO_LEV_LNODE;
  1174. sld.lnode = ln;
  1175. INIT_LIST_HEAD(&ln->cmpl_q);
  1176. csio_scsi_gather_active_ios(scm, &sld, &ln->cmpl_q);
  1177. /* No I/Os pending on this lnode */
  1178. if (list_empty(&ln->cmpl_q))
  1179. return 0;
  1180. /* Wait until all active I/Os on this lnode are completed */
  1181. while (!list_empty(&ln->cmpl_q) && count--) {
  1182. spin_unlock_irq(&hw->lock);
  1183. msleep(CSIO_SCSI_ABORT_Q_POLL_MS);
  1184. spin_lock_irq(&hw->lock);
  1185. }
  1186. /* all I/Os completed */
  1187. if (list_empty(&ln->cmpl_q))
  1188. return 0;
  1189. csio_dbg(hw, "Some I/Os pending on ln:%p, aborting them..\n", ln);
  1190. /* I/Os are pending, abort them */
  1191. rv = csio_scsi_abort_io_q(scm, &ln->cmpl_q, 30000);
  1192. if (rv != 0) {
  1193. csio_dbg(hw, "Some I/O aborts timed out, cleaning up..\n");
  1194. csio_scsi_cleanup_io_q(scm, &ln->cmpl_q);
  1195. }
  1196. CSIO_DB_ASSERT(list_empty(&ln->cmpl_q));
  1197. return rv;
  1198. }
  1199. static ssize_t
  1200. csio_show_hw_state(struct device *dev,
  1201. struct device_attribute *attr, char *buf)
  1202. {
  1203. struct csio_lnode *ln = shost_priv(class_to_shost(dev));
  1204. struct csio_hw *hw = csio_lnode_to_hw(ln);
  1205. if (csio_is_hw_ready(hw))
  1206. return snprintf(buf, PAGE_SIZE, "ready\n");
  1207. else
  1208. return snprintf(buf, PAGE_SIZE, "not ready\n");
  1209. }
  1210. /* Device reset */
  1211. static ssize_t
  1212. csio_device_reset(struct device *dev,
  1213. struct device_attribute *attr, const char *buf, size_t count)
  1214. {
  1215. struct csio_lnode *ln = shost_priv(class_to_shost(dev));
  1216. struct csio_hw *hw = csio_lnode_to_hw(ln);
  1217. if (*buf != '1')
  1218. return -EINVAL;
  1219. /* Delete NPIV lnodes */
  1220. csio_lnodes_exit(hw, 1);
  1221. /* Block upper IOs */
  1222. csio_lnodes_block_request(hw);
  1223. spin_lock_irq(&hw->lock);
  1224. csio_hw_reset(hw);
  1225. spin_unlock_irq(&hw->lock);
  1226. /* Unblock upper IOs */
  1227. csio_lnodes_unblock_request(hw);
  1228. return count;
  1229. }
  1230. /* disable port */
  1231. static ssize_t
  1232. csio_disable_port(struct device *dev,
  1233. struct device_attribute *attr, const char *buf, size_t count)
  1234. {
  1235. struct csio_lnode *ln = shost_priv(class_to_shost(dev));
  1236. struct csio_hw *hw = csio_lnode_to_hw(ln);
  1237. bool disable;
  1238. if (*buf == '1' || *buf == '0')
  1239. disable = (*buf == '1') ? true : false;
  1240. else
  1241. return -EINVAL;
  1242. /* Block upper IOs */
  1243. csio_lnodes_block_by_port(hw, ln->portid);
  1244. spin_lock_irq(&hw->lock);
  1245. csio_disable_lnodes(hw, ln->portid, disable);
  1246. spin_unlock_irq(&hw->lock);
  1247. /* Unblock upper IOs */
  1248. csio_lnodes_unblock_by_port(hw, ln->portid);
  1249. return count;
  1250. }
  1251. /* Show debug level */
  1252. static ssize_t
  1253. csio_show_dbg_level(struct device *dev,
  1254. struct device_attribute *attr, char *buf)
  1255. {
  1256. struct csio_lnode *ln = shost_priv(class_to_shost(dev));
  1257. return snprintf(buf, PAGE_SIZE, "%x\n", ln->params.log_level);
  1258. }
  1259. /* Store debug level */
  1260. static ssize_t
  1261. csio_store_dbg_level(struct device *dev,
  1262. struct device_attribute *attr, const char *buf, size_t count)
  1263. {
  1264. struct csio_lnode *ln = shost_priv(class_to_shost(dev));
  1265. struct csio_hw *hw = csio_lnode_to_hw(ln);
  1266. uint32_t dbg_level = 0;
  1267. if (!isdigit(buf[0]))
  1268. return -EINVAL;
  1269. if (sscanf(buf, "%i", &dbg_level))
  1270. return -EINVAL;
  1271. ln->params.log_level = dbg_level;
  1272. hw->params.log_level = dbg_level;
  1273. return 0;
  1274. }
  1275. static DEVICE_ATTR(hw_state, S_IRUGO, csio_show_hw_state, NULL);
  1276. static DEVICE_ATTR(device_reset, S_IWUSR, NULL, csio_device_reset);
  1277. static DEVICE_ATTR(disable_port, S_IWUSR, NULL, csio_disable_port);
  1278. static DEVICE_ATTR(dbg_level, S_IRUGO | S_IWUSR, csio_show_dbg_level,
  1279. csio_store_dbg_level);
  1280. static struct device_attribute *csio_fcoe_lport_attrs[] = {
  1281. &dev_attr_hw_state,
  1282. &dev_attr_device_reset,
  1283. &dev_attr_disable_port,
  1284. &dev_attr_dbg_level,
  1285. NULL,
  1286. };
  1287. static ssize_t
  1288. csio_show_num_reg_rnodes(struct device *dev,
  1289. struct device_attribute *attr, char *buf)
  1290. {
  1291. struct csio_lnode *ln = shost_priv(class_to_shost(dev));
  1292. return snprintf(buf, PAGE_SIZE, "%d\n", ln->num_reg_rnodes);
  1293. }
  1294. static DEVICE_ATTR(num_reg_rnodes, S_IRUGO, csio_show_num_reg_rnodes, NULL);
  1295. static struct device_attribute *csio_fcoe_vport_attrs[] = {
  1296. &dev_attr_num_reg_rnodes,
  1297. &dev_attr_dbg_level,
  1298. NULL,
  1299. };
  1300. static inline uint32_t
  1301. csio_scsi_copy_to_sgl(struct csio_hw *hw, struct csio_ioreq *req)
  1302. {
  1303. struct scsi_cmnd *scmnd = (struct scsi_cmnd *)csio_scsi_cmnd(req);
  1304. struct scatterlist *sg;
  1305. uint32_t bytes_left;
  1306. uint32_t bytes_copy;
  1307. uint32_t buf_off = 0;
  1308. uint32_t start_off = 0;
  1309. uint32_t sg_off = 0;
  1310. void *sg_addr;
  1311. void *buf_addr;
  1312. struct csio_dma_buf *dma_buf;
  1313. bytes_left = scsi_bufflen(scmnd);
  1314. sg = scsi_sglist(scmnd);
  1315. dma_buf = (struct csio_dma_buf *)csio_list_next(&req->gen_list);
  1316. /* Copy data from driver buffer to SGs of SCSI CMD */
  1317. while (bytes_left > 0 && sg && dma_buf) {
  1318. if (buf_off >= dma_buf->len) {
  1319. buf_off = 0;
  1320. dma_buf = (struct csio_dma_buf *)
  1321. csio_list_next(dma_buf);
  1322. continue;
  1323. }
  1324. if (start_off >= sg->length) {
  1325. start_off -= sg->length;
  1326. sg = sg_next(sg);
  1327. continue;
  1328. }
  1329. buf_addr = dma_buf->vaddr + buf_off;
  1330. sg_off = sg->offset + start_off;
  1331. bytes_copy = min((dma_buf->len - buf_off),
  1332. sg->length - start_off);
  1333. bytes_copy = min((uint32_t)(PAGE_SIZE - (sg_off & ~PAGE_MASK)),
  1334. bytes_copy);
  1335. sg_addr = kmap_atomic(sg_page(sg) + (sg_off >> PAGE_SHIFT));
  1336. if (!sg_addr) {
  1337. csio_err(hw, "failed to kmap sg:%p of ioreq:%p\n",
  1338. sg, req);
  1339. break;
  1340. }
  1341. csio_dbg(hw, "copy_to_sgl:sg_addr %p sg_off %d buf %p len %d\n",
  1342. sg_addr, sg_off, buf_addr, bytes_copy);
  1343. memcpy(sg_addr + (sg_off & ~PAGE_MASK), buf_addr, bytes_copy);
  1344. kunmap_atomic(sg_addr);
  1345. start_off += bytes_copy;
  1346. buf_off += bytes_copy;
  1347. bytes_left -= bytes_copy;
  1348. }
  1349. if (bytes_left > 0)
  1350. return DID_ERROR;
  1351. else
  1352. return DID_OK;
  1353. }
  1354. /*
  1355. * csio_scsi_err_handler - SCSI error handler.
  1356. * @hw: HW module.
  1357. * @req: IO request.
  1358. *
  1359. */
  1360. static inline void
  1361. csio_scsi_err_handler(struct csio_hw *hw, struct csio_ioreq *req)
  1362. {
  1363. struct scsi_cmnd *cmnd = (struct scsi_cmnd *)csio_scsi_cmnd(req);
  1364. struct csio_scsim *scm = csio_hw_to_scsim(hw);
  1365. struct fcp_resp_with_ext *fcp_resp;
  1366. struct fcp_resp_rsp_info *rsp_info;
  1367. struct csio_dma_buf *dma_buf;
  1368. uint8_t flags, scsi_status = 0;
  1369. uint32_t host_status = DID_OK;
  1370. uint32_t rsp_len = 0, sns_len = 0;
  1371. struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
  1372. switch (req->wr_status) {
  1373. case FW_HOSTERROR:
  1374. if (unlikely(!csio_is_hw_ready(hw)))
  1375. return;
  1376. host_status = DID_ERROR;
  1377. CSIO_INC_STATS(scm, n_hosterror);
  1378. break;
  1379. case FW_SCSI_RSP_ERR:
  1380. dma_buf = &req->dma_buf;
  1381. fcp_resp = (struct fcp_resp_with_ext *)dma_buf->vaddr;
  1382. rsp_info = (struct fcp_resp_rsp_info *)(fcp_resp + 1);
  1383. flags = fcp_resp->resp.fr_flags;
  1384. scsi_status = fcp_resp->resp.fr_status;
  1385. if (flags & FCP_RSP_LEN_VAL) {
  1386. rsp_len = be32_to_cpu(fcp_resp->ext.fr_rsp_len);
  1387. if ((rsp_len != 0 && rsp_len != 4 && rsp_len != 8) ||
  1388. (rsp_info->rsp_code != FCP_TMF_CMPL)) {
  1389. host_status = DID_ERROR;
  1390. goto out;
  1391. }
  1392. }
  1393. if ((flags & FCP_SNS_LEN_VAL) && fcp_resp->ext.fr_sns_len) {
  1394. sns_len = be32_to_cpu(fcp_resp->ext.fr_sns_len);
  1395. if (sns_len > SCSI_SENSE_BUFFERSIZE)
  1396. sns_len = SCSI_SENSE_BUFFERSIZE;
  1397. memcpy(cmnd->sense_buffer,
  1398. &rsp_info->_fr_resvd[0] + rsp_len, sns_len);
  1399. CSIO_INC_STATS(scm, n_autosense);
  1400. }
  1401. scsi_set_resid(cmnd, 0);
  1402. /* Under run */
  1403. if (flags & FCP_RESID_UNDER) {
  1404. scsi_set_resid(cmnd,
  1405. be32_to_cpu(fcp_resp->ext.fr_resid));
  1406. if (!(flags & FCP_SNS_LEN_VAL) &&
  1407. (scsi_status == SAM_STAT_GOOD) &&
  1408. ((scsi_bufflen(cmnd) - scsi_get_resid(cmnd))
  1409. < cmnd->underflow))
  1410. host_status = DID_ERROR;
  1411. } else if (flags & FCP_RESID_OVER)
  1412. host_status = DID_ERROR;
  1413. CSIO_INC_STATS(scm, n_rsperror);
  1414. break;
  1415. case FW_SCSI_OVER_FLOW_ERR:
  1416. csio_warn(hw,
  1417. "Over-flow error,cmnd:0x%x expected len:0x%x"
  1418. " resid:0x%x\n", cmnd->cmnd[0],
  1419. scsi_bufflen(cmnd), scsi_get_resid(cmnd));
  1420. host_status = DID_ERROR;
  1421. CSIO_INC_STATS(scm, n_ovflerror);
  1422. break;
  1423. case FW_SCSI_UNDER_FLOW_ERR:
  1424. csio_warn(hw,
  1425. "Under-flow error,cmnd:0x%x expected"
  1426. " len:0x%x resid:0x%x lun:0x%llx ssn:0x%x\n",
  1427. cmnd->cmnd[0], scsi_bufflen(cmnd),
  1428. scsi_get_resid(cmnd), cmnd->device->lun,
  1429. rn->flowid);
  1430. host_status = DID_ERROR;
  1431. CSIO_INC_STATS(scm, n_unflerror);
  1432. break;
  1433. case FW_SCSI_ABORT_REQUESTED:
  1434. case FW_SCSI_ABORTED:
  1435. case FW_SCSI_CLOSE_REQUESTED:
  1436. csio_dbg(hw, "Req %p cmd:%p op:%x %s\n", req, cmnd,
  1437. cmnd->cmnd[0],
  1438. (req->wr_status == FW_SCSI_CLOSE_REQUESTED) ?
  1439. "closed" : "aborted");
  1440. /*
  1441. * csio_eh_abort_handler checks this value to
  1442. * succeed or fail the abort request.
  1443. */
  1444. host_status = DID_REQUEUE;
  1445. if (req->wr_status == FW_SCSI_CLOSE_REQUESTED)
  1446. CSIO_INC_STATS(scm, n_closed);
  1447. else
  1448. CSIO_INC_STATS(scm, n_aborted);
  1449. break;
  1450. case FW_SCSI_ABORT_TIMEDOUT:
  1451. /* FW timed out the abort itself */
  1452. csio_dbg(hw, "FW timed out abort req:%p cmnd:%p status:%x\n",
  1453. req, cmnd, req->wr_status);
  1454. host_status = DID_ERROR;
  1455. CSIO_INC_STATS(scm, n_abrt_timedout);
  1456. break;
  1457. case FW_RDEV_NOT_READY:
  1458. /*
  1459. * In firmware, a RDEV can get into this state
  1460. * temporarily, before moving into dissapeared/lost
  1461. * state. So, the driver should complete the request equivalent
  1462. * to device-disappeared!
  1463. */
  1464. CSIO_INC_STATS(scm, n_rdev_nr_error);
  1465. host_status = DID_ERROR;
  1466. break;
  1467. case FW_ERR_RDEV_LOST:
  1468. CSIO_INC_STATS(scm, n_rdev_lost_error);
  1469. host_status = DID_ERROR;
  1470. break;
  1471. case FW_ERR_RDEV_LOGO:
  1472. CSIO_INC_STATS(scm, n_rdev_logo_error);
  1473. host_status = DID_ERROR;
  1474. break;
  1475. case FW_ERR_RDEV_IMPL_LOGO:
  1476. host_status = DID_ERROR;
  1477. break;
  1478. case FW_ERR_LINK_DOWN:
  1479. CSIO_INC_STATS(scm, n_link_down_error);
  1480. host_status = DID_ERROR;
  1481. break;
  1482. case FW_FCOE_NO_XCHG:
  1483. CSIO_INC_STATS(scm, n_no_xchg_error);
  1484. host_status = DID_ERROR;
  1485. break;
  1486. default:
  1487. csio_err(hw, "Unknown SCSI FW WR status:%d req:%p cmnd:%p\n",
  1488. req->wr_status, req, cmnd);
  1489. CSIO_DB_ASSERT(0);
  1490. CSIO_INC_STATS(scm, n_unknown_error);
  1491. host_status = DID_ERROR;
  1492. break;
  1493. }
  1494. out:
  1495. if (req->nsge > 0) {
  1496. scsi_dma_unmap(cmnd);
  1497. if (req->dcopy && (host_status == DID_OK))
  1498. host_status = csio_scsi_copy_to_sgl(hw, req);
  1499. }
  1500. cmnd->result = (((host_status) << 16) | scsi_status);
  1501. cmnd->scsi_done(cmnd);
  1502. /* Wake up waiting threads */
  1503. csio_scsi_cmnd(req) = NULL;
  1504. complete(&req->cmplobj);
  1505. }
  1506. /*
  1507. * csio_scsi_cbfn - SCSI callback function.
  1508. * @hw: HW module.
  1509. * @req: IO request.
  1510. *
  1511. */
  1512. static void
  1513. csio_scsi_cbfn(struct csio_hw *hw, struct csio_ioreq *req)
  1514. {
  1515. struct scsi_cmnd *cmnd = (struct scsi_cmnd *)csio_scsi_cmnd(req);
  1516. uint8_t scsi_status = SAM_STAT_GOOD;
  1517. uint32_t host_status = DID_OK;
  1518. if (likely(req->wr_status == FW_SUCCESS)) {
  1519. if (req->nsge > 0) {
  1520. scsi_dma_unmap(cmnd);
  1521. if (req->dcopy)
  1522. host_status = csio_scsi_copy_to_sgl(hw, req);
  1523. }
  1524. cmnd->result = (((host_status) << 16) | scsi_status);
  1525. cmnd->scsi_done(cmnd);
  1526. csio_scsi_cmnd(req) = NULL;
  1527. CSIO_INC_STATS(csio_hw_to_scsim(hw), n_tot_success);
  1528. } else {
  1529. /* Error handling */
  1530. csio_scsi_err_handler(hw, req);
  1531. }
  1532. }
  1533. /**
  1534. * csio_queuecommand - Entry point to kickstart an I/O request.
  1535. * @host: The scsi_host pointer.
  1536. * @cmnd: The I/O request from ML.
  1537. *
  1538. * This routine does the following:
  1539. * - Checks for HW and Rnode module readiness.
  1540. * - Gets a free ioreq structure (which is already initialized
  1541. * to uninit during its allocation).
  1542. * - Maps SG elements.
  1543. * - Initializes ioreq members.
  1544. * - Kicks off the SCSI state machine for this IO.
  1545. * - Returns busy status on error.
  1546. */
  1547. static int
  1548. csio_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmnd)
  1549. {
  1550. struct csio_lnode *ln = shost_priv(host);
  1551. struct csio_hw *hw = csio_lnode_to_hw(ln);
  1552. struct csio_scsim *scsim = csio_hw_to_scsim(hw);
  1553. struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
  1554. struct csio_ioreq *ioreq = NULL;
  1555. unsigned long flags;
  1556. int nsge = 0;
  1557. int rv = SCSI_MLQUEUE_HOST_BUSY, nr;
  1558. int retval;
  1559. struct csio_scsi_qset *sqset;
  1560. struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
  1561. sqset = &hw->sqset[ln->portid][blk_mq_rq_cpu(cmnd->request)];
  1562. nr = fc_remote_port_chkready(rport);
  1563. if (nr) {
  1564. cmnd->result = nr;
  1565. CSIO_INC_STATS(scsim, n_rn_nr_error);
  1566. goto err_done;
  1567. }
  1568. if (unlikely(!csio_is_hw_ready(hw))) {
  1569. cmnd->result = (DID_REQUEUE << 16);
  1570. CSIO_INC_STATS(scsim, n_hw_nr_error);
  1571. goto err_done;
  1572. }
  1573. /* Get req->nsge, if there are SG elements to be mapped */
  1574. nsge = scsi_dma_map(cmnd);
  1575. if (unlikely(nsge < 0)) {
  1576. CSIO_INC_STATS(scsim, n_dmamap_error);
  1577. goto err;
  1578. }
  1579. /* Do we support so many mappings? */
  1580. if (unlikely(nsge > scsim->max_sge)) {
  1581. csio_warn(hw,
  1582. "More SGEs than can be supported."
  1583. " SGEs: %d, Max SGEs: %d\n", nsge, scsim->max_sge);
  1584. CSIO_INC_STATS(scsim, n_unsupp_sge_error);
  1585. goto err_dma_unmap;
  1586. }
  1587. /* Get a free ioreq structure - SM is already set to uninit */
  1588. ioreq = csio_get_scsi_ioreq_lock(hw, scsim);
  1589. if (!ioreq) {
  1590. csio_err(hw, "Out of I/O request elements. Active #:%d\n",
  1591. scsim->stats.n_active);
  1592. CSIO_INC_STATS(scsim, n_no_req_error);
  1593. goto err_dma_unmap;
  1594. }
  1595. ioreq->nsge = nsge;
  1596. ioreq->lnode = ln;
  1597. ioreq->rnode = rn;
  1598. ioreq->iq_idx = sqset->iq_idx;
  1599. ioreq->eq_idx = sqset->eq_idx;
  1600. ioreq->wr_status = 0;
  1601. ioreq->drv_status = 0;
  1602. csio_scsi_cmnd(ioreq) = (void *)cmnd;
  1603. ioreq->tmo = 0;
  1604. ioreq->datadir = cmnd->sc_data_direction;
  1605. if (cmnd->sc_data_direction == DMA_TO_DEVICE) {
  1606. CSIO_INC_STATS(ln, n_output_requests);
  1607. ln->stats.n_output_bytes += scsi_bufflen(cmnd);
  1608. } else if (cmnd->sc_data_direction == DMA_FROM_DEVICE) {
  1609. CSIO_INC_STATS(ln, n_input_requests);
  1610. ln->stats.n_input_bytes += scsi_bufflen(cmnd);
  1611. } else
  1612. CSIO_INC_STATS(ln, n_control_requests);
  1613. /* Set cbfn */
  1614. ioreq->io_cbfn = csio_scsi_cbfn;
  1615. /* Needed during abort */
  1616. cmnd->host_scribble = (unsigned char *)ioreq;
  1617. cmnd->SCp.Message = 0;
  1618. /* Kick off SCSI IO SM on the ioreq */
  1619. spin_lock_irqsave(&hw->lock, flags);
  1620. retval = csio_scsi_start_io(ioreq);
  1621. spin_unlock_irqrestore(&hw->lock, flags);
  1622. if (retval != 0) {
  1623. csio_err(hw, "ioreq: %p couldn't be started, status:%d\n",
  1624. ioreq, retval);
  1625. CSIO_INC_STATS(scsim, n_busy_error);
  1626. goto err_put_req;
  1627. }
  1628. return 0;
  1629. err_put_req:
  1630. csio_put_scsi_ioreq_lock(hw, scsim, ioreq);
  1631. err_dma_unmap:
  1632. if (nsge > 0)
  1633. scsi_dma_unmap(cmnd);
  1634. err:
  1635. return rv;
  1636. err_done:
  1637. cmnd->scsi_done(cmnd);
  1638. return 0;
  1639. }
  1640. static int
  1641. csio_do_abrt_cls(struct csio_hw *hw, struct csio_ioreq *ioreq, bool abort)
  1642. {
  1643. int rv;
  1644. int cpu = smp_processor_id();
  1645. struct csio_lnode *ln = ioreq->lnode;
  1646. struct csio_scsi_qset *sqset = &hw->sqset[ln->portid][cpu];
  1647. ioreq->tmo = CSIO_SCSI_ABRT_TMO_MS;
  1648. /*
  1649. * Use current processor queue for posting the abort/close, but retain
  1650. * the ingress queue ID of the original I/O being aborted/closed - we
  1651. * need the abort/close completion to be received on the same queue
  1652. * as the original I/O.
  1653. */
  1654. ioreq->eq_idx = sqset->eq_idx;
  1655. if (abort == SCSI_ABORT)
  1656. rv = csio_scsi_abort(ioreq);
  1657. else
  1658. rv = csio_scsi_close(ioreq);
  1659. return rv;
  1660. }
  1661. static int
  1662. csio_eh_abort_handler(struct scsi_cmnd *cmnd)
  1663. {
  1664. struct csio_ioreq *ioreq;
  1665. struct csio_lnode *ln = shost_priv(cmnd->device->host);
  1666. struct csio_hw *hw = csio_lnode_to_hw(ln);
  1667. struct csio_scsim *scsim = csio_hw_to_scsim(hw);
  1668. int ready = 0, ret;
  1669. unsigned long tmo = 0;
  1670. int rv;
  1671. struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
  1672. ret = fc_block_scsi_eh(cmnd);
  1673. if (ret)
  1674. return ret;
  1675. ioreq = (struct csio_ioreq *)cmnd->host_scribble;
  1676. if (!ioreq)
  1677. return SUCCESS;
  1678. if (!rn)
  1679. return FAILED;
  1680. csio_dbg(hw,
  1681. "Request to abort ioreq:%p cmd:%p cdb:%08llx"
  1682. " ssni:0x%x lun:%llu iq:0x%x\n",
  1683. ioreq, cmnd, *((uint64_t *)cmnd->cmnd), rn->flowid,
  1684. cmnd->device->lun, csio_q_physiqid(hw, ioreq->iq_idx));
  1685. if (((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) != cmnd) {
  1686. CSIO_INC_STATS(scsim, n_abrt_race_comp);
  1687. return SUCCESS;
  1688. }
  1689. ready = csio_is_lnode_ready(ln);
  1690. tmo = CSIO_SCSI_ABRT_TMO_MS;
  1691. reinit_completion(&ioreq->cmplobj);
  1692. spin_lock_irq(&hw->lock);
  1693. rv = csio_do_abrt_cls(hw, ioreq, (ready ? SCSI_ABORT : SCSI_CLOSE));
  1694. spin_unlock_irq(&hw->lock);
  1695. if (rv != 0) {
  1696. if (rv == -EINVAL) {
  1697. /* Return success, if abort/close request issued on
  1698. * already completed IO
  1699. */
  1700. return SUCCESS;
  1701. }
  1702. if (ready)
  1703. CSIO_INC_STATS(scsim, n_abrt_busy_error);
  1704. else
  1705. CSIO_INC_STATS(scsim, n_cls_busy_error);
  1706. goto inval_scmnd;
  1707. }
  1708. wait_for_completion_timeout(&ioreq->cmplobj, msecs_to_jiffies(tmo));
  1709. /* FW didnt respond to abort within our timeout */
  1710. if (((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) == cmnd) {
  1711. csio_err(hw, "Abort timed out -- req: %p\n", ioreq);
  1712. CSIO_INC_STATS(scsim, n_abrt_timedout);
  1713. inval_scmnd:
  1714. if (ioreq->nsge > 0)
  1715. scsi_dma_unmap(cmnd);
  1716. spin_lock_irq(&hw->lock);
  1717. csio_scsi_cmnd(ioreq) = NULL;
  1718. spin_unlock_irq(&hw->lock);
  1719. cmnd->result = (DID_ERROR << 16);
  1720. cmnd->scsi_done(cmnd);
  1721. return FAILED;
  1722. }
  1723. /* FW successfully aborted the request */
  1724. if (host_byte(cmnd->result) == DID_REQUEUE) {
  1725. csio_info(hw,
  1726. "Aborted SCSI command to (%d:%llu) tag %u\n",
  1727. cmnd->device->id, cmnd->device->lun,
  1728. cmnd->request->tag);
  1729. return SUCCESS;
  1730. } else {
  1731. csio_info(hw,
  1732. "Failed to abort SCSI command, (%d:%llu) tag %u\n",
  1733. cmnd->device->id, cmnd->device->lun,
  1734. cmnd->request->tag);
  1735. return FAILED;
  1736. }
  1737. }
  1738. /*
  1739. * csio_tm_cbfn - TM callback function.
  1740. * @hw: HW module.
  1741. * @req: IO request.
  1742. *
  1743. * Cache the result in 'cmnd', since ioreq will be freed soon
  1744. * after we return from here, and the waiting thread shouldnt trust
  1745. * the ioreq contents.
  1746. */
  1747. static void
  1748. csio_tm_cbfn(struct csio_hw *hw, struct csio_ioreq *req)
  1749. {
  1750. struct scsi_cmnd *cmnd = (struct scsi_cmnd *)csio_scsi_cmnd(req);
  1751. struct csio_dma_buf *dma_buf;
  1752. uint8_t flags = 0;
  1753. struct fcp_resp_with_ext *fcp_resp;
  1754. struct fcp_resp_rsp_info *rsp_info;
  1755. csio_dbg(hw, "req: %p in csio_tm_cbfn status: %d\n",
  1756. req, req->wr_status);
  1757. /* Cache FW return status */
  1758. cmnd->SCp.Status = req->wr_status;
  1759. /* Special handling based on FCP response */
  1760. /*
  1761. * FW returns us this error, if flags were set. FCP4 says
  1762. * FCP_RSP_LEN_VAL in flags shall be set for TM completions.
  1763. * So if a target were to set this bit, we expect that the
  1764. * rsp_code is set to FCP_TMF_CMPL for a successful TM
  1765. * completion. Any other rsp_code means TM operation failed.
  1766. * If a target were to just ignore setting flags, we treat
  1767. * the TM operation as success, and FW returns FW_SUCCESS.
  1768. */
  1769. if (req->wr_status == FW_SCSI_RSP_ERR) {
  1770. dma_buf = &req->dma_buf;
  1771. fcp_resp = (struct fcp_resp_with_ext *)dma_buf->vaddr;
  1772. rsp_info = (struct fcp_resp_rsp_info *)(fcp_resp + 1);
  1773. flags = fcp_resp->resp.fr_flags;
  1774. /* Modify return status if flags indicate success */
  1775. if (flags & FCP_RSP_LEN_VAL)
  1776. if (rsp_info->rsp_code == FCP_TMF_CMPL)
  1777. cmnd->SCp.Status = FW_SUCCESS;
  1778. csio_dbg(hw, "TM FCP rsp code: %d\n", rsp_info->rsp_code);
  1779. }
  1780. /* Wake up the TM handler thread */
  1781. csio_scsi_cmnd(req) = NULL;
  1782. }
  1783. static int
  1784. csio_eh_lun_reset_handler(struct scsi_cmnd *cmnd)
  1785. {
  1786. struct csio_lnode *ln = shost_priv(cmnd->device->host);
  1787. struct csio_hw *hw = csio_lnode_to_hw(ln);
  1788. struct csio_scsim *scsim = csio_hw_to_scsim(hw);
  1789. struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
  1790. struct csio_ioreq *ioreq = NULL;
  1791. struct csio_scsi_qset *sqset;
  1792. unsigned long flags;
  1793. int retval;
  1794. int count, ret;
  1795. LIST_HEAD(local_q);
  1796. struct csio_scsi_level_data sld;
  1797. if (!rn)
  1798. goto fail;
  1799. csio_dbg(hw, "Request to reset LUN:%llu (ssni:0x%x tgtid:%d)\n",
  1800. cmnd->device->lun, rn->flowid, rn->scsi_id);
  1801. if (!csio_is_lnode_ready(ln)) {
  1802. csio_err(hw,
  1803. "LUN reset cannot be issued on non-ready"
  1804. " local node vnpi:0x%x (LUN:%llu)\n",
  1805. ln->vnp_flowid, cmnd->device->lun);
  1806. goto fail;
  1807. }
  1808. /* Lnode is ready, now wait on rport node readiness */
  1809. ret = fc_block_scsi_eh(cmnd);
  1810. if (ret)
  1811. return ret;
  1812. /*
  1813. * If we have blocked in the previous call, at this point, either the
  1814. * remote node has come back online, or device loss timer has fired
  1815. * and the remote node is destroyed. Allow the LUN reset only for
  1816. * the former case, since LUN reset is a TMF I/O on the wire, and we
  1817. * need a valid session to issue it.
  1818. */
  1819. if (fc_remote_port_chkready(rn->rport)) {
  1820. csio_err(hw,
  1821. "LUN reset cannot be issued on non-ready"
  1822. " remote node ssni:0x%x (LUN:%llu)\n",
  1823. rn->flowid, cmnd->device->lun);
  1824. goto fail;
  1825. }
  1826. /* Get a free ioreq structure - SM is already set to uninit */
  1827. ioreq = csio_get_scsi_ioreq_lock(hw, scsim);
  1828. if (!ioreq) {
  1829. csio_err(hw, "Out of IO request elements. Active # :%d\n",
  1830. scsim->stats.n_active);
  1831. goto fail;
  1832. }
  1833. sqset = &hw->sqset[ln->portid][smp_processor_id()];
  1834. ioreq->nsge = 0;
  1835. ioreq->lnode = ln;
  1836. ioreq->rnode = rn;
  1837. ioreq->iq_idx = sqset->iq_idx;
  1838. ioreq->eq_idx = sqset->eq_idx;
  1839. csio_scsi_cmnd(ioreq) = cmnd;
  1840. cmnd->host_scribble = (unsigned char *)ioreq;
  1841. cmnd->SCp.Status = 0;
  1842. cmnd->SCp.Message = FCP_TMF_LUN_RESET;
  1843. ioreq->tmo = CSIO_SCSI_LUNRST_TMO_MS / 1000;
  1844. /*
  1845. * FW times the LUN reset for ioreq->tmo, so we got to wait a little
  1846. * longer (10s for now) than that to allow FW to return the timed
  1847. * out command.
  1848. */
  1849. count = DIV_ROUND_UP((ioreq->tmo + 10) * 1000, CSIO_SCSI_TM_POLL_MS);
  1850. /* Set cbfn */
  1851. ioreq->io_cbfn = csio_tm_cbfn;
  1852. /* Save of the ioreq info for later use */
  1853. sld.level = CSIO_LEV_LUN;
  1854. sld.lnode = ioreq->lnode;
  1855. sld.rnode = ioreq->rnode;
  1856. sld.oslun = cmnd->device->lun;
  1857. spin_lock_irqsave(&hw->lock, flags);
  1858. /* Kick off TM SM on the ioreq */
  1859. retval = csio_scsi_start_tm(ioreq);
  1860. spin_unlock_irqrestore(&hw->lock, flags);
  1861. if (retval != 0) {
  1862. csio_err(hw, "Failed to issue LUN reset, req:%p, status:%d\n",
  1863. ioreq, retval);
  1864. goto fail_ret_ioreq;
  1865. }
  1866. csio_dbg(hw, "Waiting max %d secs for LUN reset completion\n",
  1867. count * (CSIO_SCSI_TM_POLL_MS / 1000));
  1868. /* Wait for completion */
  1869. while ((((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) == cmnd)
  1870. && count--)
  1871. msleep(CSIO_SCSI_TM_POLL_MS);
  1872. /* LUN reset timed-out */
  1873. if (((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) == cmnd) {
  1874. csio_err(hw, "LUN reset (%d:%llu) timed out\n",
  1875. cmnd->device->id, cmnd->device->lun);
  1876. spin_lock_irq(&hw->lock);
  1877. csio_scsi_drvcleanup(ioreq);
  1878. list_del_init(&ioreq->sm.sm_list);
  1879. spin_unlock_irq(&hw->lock);
  1880. goto fail_ret_ioreq;
  1881. }
  1882. /* LUN reset returned, check cached status */
  1883. if (cmnd->SCp.Status != FW_SUCCESS) {
  1884. csio_err(hw, "LUN reset failed (%d:%llu), status: %d\n",
  1885. cmnd->device->id, cmnd->device->lun, cmnd->SCp.Status);
  1886. goto fail;
  1887. }
  1888. /* LUN reset succeeded, Start aborting affected I/Os */
  1889. /*
  1890. * Since the host guarantees during LUN reset that there
  1891. * will not be any more I/Os to that LUN, until the LUN reset
  1892. * completes, we gather pending I/Os after the LUN reset.
  1893. */
  1894. spin_lock_irq(&hw->lock);
  1895. csio_scsi_gather_active_ios(scsim, &sld, &local_q);
  1896. retval = csio_scsi_abort_io_q(scsim, &local_q, 30000);
  1897. spin_unlock_irq(&hw->lock);
  1898. /* Aborts may have timed out */
  1899. if (retval != 0) {
  1900. csio_err(hw,
  1901. "Attempt to abort I/Os during LUN reset of %llu"
  1902. " returned %d\n", cmnd->device->lun, retval);
  1903. /* Return I/Os back to active_q */
  1904. spin_lock_irq(&hw->lock);
  1905. list_splice_tail_init(&local_q, &scsim->active_q);
  1906. spin_unlock_irq(&hw->lock);
  1907. goto fail;
  1908. }
  1909. CSIO_INC_STATS(rn, n_lun_rst);
  1910. csio_info(hw, "LUN reset occurred (%d:%llu)\n",
  1911. cmnd->device->id, cmnd->device->lun);
  1912. return SUCCESS;
  1913. fail_ret_ioreq:
  1914. csio_put_scsi_ioreq_lock(hw, scsim, ioreq);
  1915. fail:
  1916. CSIO_INC_STATS(rn, n_lun_rst_fail);
  1917. return FAILED;
  1918. }
  1919. static int
  1920. csio_slave_alloc(struct scsi_device *sdev)
  1921. {
  1922. struct fc_rport *rport = starget_to_rport(scsi_target(sdev));
  1923. if (!rport || fc_remote_port_chkready(rport))
  1924. return -ENXIO;
  1925. sdev->hostdata = *((struct csio_lnode **)(rport->dd_data));
  1926. return 0;
  1927. }
  1928. static int
  1929. csio_slave_configure(struct scsi_device *sdev)
  1930. {
  1931. scsi_change_queue_depth(sdev, csio_lun_qdepth);
  1932. return 0;
  1933. }
  1934. static void
  1935. csio_slave_destroy(struct scsi_device *sdev)
  1936. {
  1937. sdev->hostdata = NULL;
  1938. }
  1939. static int
  1940. csio_scan_finished(struct Scsi_Host *shost, unsigned long time)
  1941. {
  1942. struct csio_lnode *ln = shost_priv(shost);
  1943. int rv = 1;
  1944. spin_lock_irq(shost->host_lock);
  1945. if (!ln->hwp || csio_list_deleted(&ln->sm.sm_list))
  1946. goto out;
  1947. rv = csio_scan_done(ln, jiffies, time, csio_max_scan_tmo * HZ,
  1948. csio_delta_scan_tmo * HZ);
  1949. out:
  1950. spin_unlock_irq(shost->host_lock);
  1951. return rv;
  1952. }
  1953. struct scsi_host_template csio_fcoe_shost_template = {
  1954. .module = THIS_MODULE,
  1955. .name = CSIO_DRV_DESC,
  1956. .proc_name = KBUILD_MODNAME,
  1957. .queuecommand = csio_queuecommand,
  1958. .eh_timed_out = fc_eh_timed_out,
  1959. .eh_abort_handler = csio_eh_abort_handler,
  1960. .eh_device_reset_handler = csio_eh_lun_reset_handler,
  1961. .slave_alloc = csio_slave_alloc,
  1962. .slave_configure = csio_slave_configure,
  1963. .slave_destroy = csio_slave_destroy,
  1964. .scan_finished = csio_scan_finished,
  1965. .this_id = -1,
  1966. .sg_tablesize = CSIO_SCSI_MAX_SGE,
  1967. .cmd_per_lun = CSIO_MAX_CMD_PER_LUN,
  1968. .shost_attrs = csio_fcoe_lport_attrs,
  1969. .max_sectors = CSIO_MAX_SECTOR_SIZE,
  1970. };
  1971. struct scsi_host_template csio_fcoe_shost_vport_template = {
  1972. .module = THIS_MODULE,
  1973. .name = CSIO_DRV_DESC,
  1974. .proc_name = KBUILD_MODNAME,
  1975. .queuecommand = csio_queuecommand,
  1976. .eh_timed_out = fc_eh_timed_out,
  1977. .eh_abort_handler = csio_eh_abort_handler,
  1978. .eh_device_reset_handler = csio_eh_lun_reset_handler,
  1979. .slave_alloc = csio_slave_alloc,
  1980. .slave_configure = csio_slave_configure,
  1981. .slave_destroy = csio_slave_destroy,
  1982. .scan_finished = csio_scan_finished,
  1983. .this_id = -1,
  1984. .sg_tablesize = CSIO_SCSI_MAX_SGE,
  1985. .cmd_per_lun = CSIO_MAX_CMD_PER_LUN,
  1986. .shost_attrs = csio_fcoe_vport_attrs,
  1987. .max_sectors = CSIO_MAX_SECTOR_SIZE,
  1988. };
  1989. /*
  1990. * csio_scsi_alloc_ddp_bufs - Allocate buffers for DDP of unaligned SGLs.
  1991. * @scm: SCSI Module
  1992. * @hw: HW device.
  1993. * @buf_size: buffer size
  1994. * @num_buf : Number of buffers.
  1995. *
  1996. * This routine allocates DMA buffers required for SCSI Data xfer, if
  1997. * each SGL buffer for a SCSI Read request posted by SCSI midlayer are
  1998. * not virtually contiguous.
  1999. */
  2000. static int
  2001. csio_scsi_alloc_ddp_bufs(struct csio_scsim *scm, struct csio_hw *hw,
  2002. int buf_size, int num_buf)
  2003. {
  2004. int n = 0;
  2005. struct list_head *tmp;
  2006. struct csio_dma_buf *ddp_desc = NULL;
  2007. uint32_t unit_size = 0;
  2008. if (!num_buf)
  2009. return 0;
  2010. if (!buf_size)
  2011. return -EINVAL;
  2012. INIT_LIST_HEAD(&scm->ddp_freelist);
  2013. /* Align buf size to page size */
  2014. buf_size = (buf_size + PAGE_SIZE - 1) & PAGE_MASK;
  2015. /* Initialize dma descriptors */
  2016. for (n = 0; n < num_buf; n++) {
  2017. /* Set unit size to request size */
  2018. unit_size = buf_size;
  2019. ddp_desc = kzalloc(sizeof(struct csio_dma_buf), GFP_KERNEL);
  2020. if (!ddp_desc) {
  2021. csio_err(hw,
  2022. "Failed to allocate ddp descriptors,"
  2023. " Num allocated = %d.\n",
  2024. scm->stats.n_free_ddp);
  2025. goto no_mem;
  2026. }
  2027. /* Allocate Dma buffers for DDP */
  2028. ddp_desc->vaddr = dma_alloc_coherent(&hw->pdev->dev, unit_size,
  2029. &ddp_desc->paddr, GFP_KERNEL);
  2030. if (!ddp_desc->vaddr) {
  2031. csio_err(hw,
  2032. "SCSI response DMA buffer (ddp) allocation"
  2033. " failed!\n");
  2034. kfree(ddp_desc);
  2035. goto no_mem;
  2036. }
  2037. ddp_desc->len = unit_size;
  2038. /* Added it to scsi ddp freelist */
  2039. list_add_tail(&ddp_desc->list, &scm->ddp_freelist);
  2040. CSIO_INC_STATS(scm, n_free_ddp);
  2041. }
  2042. return 0;
  2043. no_mem:
  2044. /* release dma descs back to freelist and free dma memory */
  2045. list_for_each(tmp, &scm->ddp_freelist) {
  2046. ddp_desc = (struct csio_dma_buf *) tmp;
  2047. tmp = csio_list_prev(tmp);
  2048. dma_free_coherent(&hw->pdev->dev, ddp_desc->len,
  2049. ddp_desc->vaddr, ddp_desc->paddr);
  2050. list_del_init(&ddp_desc->list);
  2051. kfree(ddp_desc);
  2052. }
  2053. scm->stats.n_free_ddp = 0;
  2054. return -ENOMEM;
  2055. }
  2056. /*
  2057. * csio_scsi_free_ddp_bufs - free DDP buffers of unaligned SGLs.
  2058. * @scm: SCSI Module
  2059. * @hw: HW device.
  2060. *
  2061. * This routine frees ddp buffers.
  2062. */
  2063. static void
  2064. csio_scsi_free_ddp_bufs(struct csio_scsim *scm, struct csio_hw *hw)
  2065. {
  2066. struct list_head *tmp;
  2067. struct csio_dma_buf *ddp_desc;
  2068. /* release dma descs back to freelist and free dma memory */
  2069. list_for_each(tmp, &scm->ddp_freelist) {
  2070. ddp_desc = (struct csio_dma_buf *) tmp;
  2071. tmp = csio_list_prev(tmp);
  2072. dma_free_coherent(&hw->pdev->dev, ddp_desc->len,
  2073. ddp_desc->vaddr, ddp_desc->paddr);
  2074. list_del_init(&ddp_desc->list);
  2075. kfree(ddp_desc);
  2076. }
  2077. scm->stats.n_free_ddp = 0;
  2078. }
  2079. /**
  2080. * csio_scsim_init - Initialize SCSI Module
  2081. * @scm: SCSI Module
  2082. * @hw: HW module
  2083. *
  2084. */
  2085. int
  2086. csio_scsim_init(struct csio_scsim *scm, struct csio_hw *hw)
  2087. {
  2088. int i;
  2089. struct csio_ioreq *ioreq;
  2090. struct csio_dma_buf *dma_buf;
  2091. INIT_LIST_HEAD(&scm->active_q);
  2092. scm->hw = hw;
  2093. scm->proto_cmd_len = sizeof(struct fcp_cmnd);
  2094. scm->proto_rsp_len = CSIO_SCSI_RSP_LEN;
  2095. scm->max_sge = CSIO_SCSI_MAX_SGE;
  2096. spin_lock_init(&scm->freelist_lock);
  2097. /* Pre-allocate ioreqs and initialize them */
  2098. INIT_LIST_HEAD(&scm->ioreq_freelist);
  2099. for (i = 0; i < csio_scsi_ioreqs; i++) {
  2100. ioreq = kzalloc(sizeof(struct csio_ioreq), GFP_KERNEL);
  2101. if (!ioreq) {
  2102. csio_err(hw,
  2103. "I/O request element allocation failed, "
  2104. " Num allocated = %d.\n",
  2105. scm->stats.n_free_ioreq);
  2106. goto free_ioreq;
  2107. }
  2108. /* Allocate Dma buffers for Response Payload */
  2109. dma_buf = &ioreq->dma_buf;
  2110. dma_buf->vaddr = dma_pool_alloc(hw->scsi_dma_pool, GFP_KERNEL,
  2111. &dma_buf->paddr);
  2112. if (!dma_buf->vaddr) {
  2113. csio_err(hw,
  2114. "SCSI response DMA buffer allocation"
  2115. " failed!\n");
  2116. kfree(ioreq);
  2117. goto free_ioreq;
  2118. }
  2119. dma_buf->len = scm->proto_rsp_len;
  2120. /* Set state to uninit */
  2121. csio_init_state(&ioreq->sm, csio_scsis_uninit);
  2122. INIT_LIST_HEAD(&ioreq->gen_list);
  2123. init_completion(&ioreq->cmplobj);
  2124. list_add_tail(&ioreq->sm.sm_list, &scm->ioreq_freelist);
  2125. CSIO_INC_STATS(scm, n_free_ioreq);
  2126. }
  2127. if (csio_scsi_alloc_ddp_bufs(scm, hw, PAGE_SIZE, csio_ddp_descs))
  2128. goto free_ioreq;
  2129. return 0;
  2130. free_ioreq:
  2131. /*
  2132. * Free up existing allocations, since an error
  2133. * from here means we are returning for good
  2134. */
  2135. while (!list_empty(&scm->ioreq_freelist)) {
  2136. struct csio_sm *tmp;
  2137. tmp = list_first_entry(&scm->ioreq_freelist,
  2138. struct csio_sm, sm_list);
  2139. list_del_init(&tmp->sm_list);
  2140. ioreq = (struct csio_ioreq *)tmp;
  2141. dma_buf = &ioreq->dma_buf;
  2142. dma_pool_free(hw->scsi_dma_pool, dma_buf->vaddr,
  2143. dma_buf->paddr);
  2144. kfree(ioreq);
  2145. }
  2146. scm->stats.n_free_ioreq = 0;
  2147. return -ENOMEM;
  2148. }
  2149. /**
  2150. * csio_scsim_exit: Uninitialize SCSI Module
  2151. * @scm: SCSI Module
  2152. *
  2153. */
  2154. void
  2155. csio_scsim_exit(struct csio_scsim *scm)
  2156. {
  2157. struct csio_ioreq *ioreq;
  2158. struct csio_dma_buf *dma_buf;
  2159. while (!list_empty(&scm->ioreq_freelist)) {
  2160. struct csio_sm *tmp;
  2161. tmp = list_first_entry(&scm->ioreq_freelist,
  2162. struct csio_sm, sm_list);
  2163. list_del_init(&tmp->sm_list);
  2164. ioreq = (struct csio_ioreq *)tmp;
  2165. dma_buf = &ioreq->dma_buf;
  2166. dma_pool_free(scm->hw->scsi_dma_pool, dma_buf->vaddr,
  2167. dma_buf->paddr);
  2168. kfree(ioreq);
  2169. }
  2170. scm->stats.n_free_ioreq = 0;
  2171. csio_scsi_free_ddp_bufs(scm, scm->hw);
  2172. }