vmw_pvscsi.c 44 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631
  1. /*
  2. * Linux driver for VMware's para-virtualized SCSI HBA.
  3. *
  4. * Copyright (C) 2008-2014, VMware, Inc. All Rights Reserved.
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License as published by the
  8. * Free Software Foundation; version 2 of the License and no later version.
  9. *
  10. * This program is distributed in the hope that it will be useful, but
  11. * WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  13. * NON INFRINGEMENT. See the GNU General Public License for more
  14. * details.
  15. *
  16. * You should have received a copy of the GNU General Public License
  17. * along with this program; if not, write to the Free Software
  18. * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19. *
  20. * Maintained by: Jim Gill <jgill@vmware.com>
  21. *
  22. */
  23. #include <linux/kernel.h>
  24. #include <linux/module.h>
  25. #include <linux/interrupt.h>
  26. #include <linux/slab.h>
  27. #include <linux/workqueue.h>
  28. #include <linux/pci.h>
  29. #include <scsi/scsi.h>
  30. #include <scsi/scsi_host.h>
  31. #include <scsi/scsi_cmnd.h>
  32. #include <scsi/scsi_device.h>
  33. #include <scsi/scsi_tcq.h>
  34. #include "vmw_pvscsi.h"
  35. #define PVSCSI_LINUX_DRIVER_DESC "VMware PVSCSI driver"
  36. MODULE_DESCRIPTION(PVSCSI_LINUX_DRIVER_DESC);
  37. MODULE_AUTHOR("VMware, Inc.");
  38. MODULE_LICENSE("GPL");
  39. MODULE_VERSION(PVSCSI_DRIVER_VERSION_STRING);
  40. #define PVSCSI_DEFAULT_NUM_PAGES_PER_RING 8
  41. #define PVSCSI_DEFAULT_NUM_PAGES_MSG_RING 1
  42. #define PVSCSI_DEFAULT_QUEUE_DEPTH 254
  43. #define SGL_SIZE PAGE_SIZE
  44. struct pvscsi_sg_list {
  45. struct PVSCSISGElement sge[PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT];
  46. };
  47. struct pvscsi_ctx {
  48. /*
  49. * The index of the context in cmd_map serves as the context ID for a
  50. * 1-to-1 mapping completions back to requests.
  51. */
  52. struct scsi_cmnd *cmd;
  53. struct pvscsi_sg_list *sgl;
  54. struct list_head list;
  55. dma_addr_t dataPA;
  56. dma_addr_t sensePA;
  57. dma_addr_t sglPA;
  58. struct completion *abort_cmp;
  59. };
  60. struct pvscsi_adapter {
  61. char *mmioBase;
  62. u8 rev;
  63. bool use_msg;
  64. bool use_req_threshold;
  65. spinlock_t hw_lock;
  66. struct workqueue_struct *workqueue;
  67. struct work_struct work;
  68. struct PVSCSIRingReqDesc *req_ring;
  69. unsigned req_pages;
  70. unsigned req_depth;
  71. dma_addr_t reqRingPA;
  72. struct PVSCSIRingCmpDesc *cmp_ring;
  73. unsigned cmp_pages;
  74. dma_addr_t cmpRingPA;
  75. struct PVSCSIRingMsgDesc *msg_ring;
  76. unsigned msg_pages;
  77. dma_addr_t msgRingPA;
  78. struct PVSCSIRingsState *rings_state;
  79. dma_addr_t ringStatePA;
  80. struct pci_dev *dev;
  81. struct Scsi_Host *host;
  82. struct list_head cmd_pool;
  83. struct pvscsi_ctx *cmd_map;
  84. };
  85. /* Command line parameters */
  86. static int pvscsi_ring_pages;
  87. static int pvscsi_msg_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_MSG_RING;
  88. static int pvscsi_cmd_per_lun = PVSCSI_DEFAULT_QUEUE_DEPTH;
  89. static bool pvscsi_disable_msi;
  90. static bool pvscsi_disable_msix;
  91. static bool pvscsi_use_msg = true;
  92. static bool pvscsi_use_req_threshold = true;
  93. #define PVSCSI_RW (S_IRUSR | S_IWUSR)
  94. module_param_named(ring_pages, pvscsi_ring_pages, int, PVSCSI_RW);
  95. MODULE_PARM_DESC(ring_pages, "Number of pages per req/cmp ring - (default="
  96. __stringify(PVSCSI_DEFAULT_NUM_PAGES_PER_RING)
  97. "[up to 16 targets],"
  98. __stringify(PVSCSI_SETUP_RINGS_MAX_NUM_PAGES)
  99. "[for 16+ targets])");
  100. module_param_named(msg_ring_pages, pvscsi_msg_ring_pages, int, PVSCSI_RW);
  101. MODULE_PARM_DESC(msg_ring_pages, "Number of pages for the msg ring - (default="
  102. __stringify(PVSCSI_DEFAULT_NUM_PAGES_MSG_RING) ")");
  103. module_param_named(cmd_per_lun, pvscsi_cmd_per_lun, int, PVSCSI_RW);
  104. MODULE_PARM_DESC(cmd_per_lun, "Maximum commands per lun - (default="
  105. __stringify(PVSCSI_DEFAULT_QUEUE_DEPTH) ")");
  106. module_param_named(disable_msi, pvscsi_disable_msi, bool, PVSCSI_RW);
  107. MODULE_PARM_DESC(disable_msi, "Disable MSI use in driver - (default=0)");
  108. module_param_named(disable_msix, pvscsi_disable_msix, bool, PVSCSI_RW);
  109. MODULE_PARM_DESC(disable_msix, "Disable MSI-X use in driver - (default=0)");
  110. module_param_named(use_msg, pvscsi_use_msg, bool, PVSCSI_RW);
  111. MODULE_PARM_DESC(use_msg, "Use msg ring when available - (default=1)");
  112. module_param_named(use_req_threshold, pvscsi_use_req_threshold,
  113. bool, PVSCSI_RW);
  114. MODULE_PARM_DESC(use_req_threshold, "Use driver-based request coalescing if configured - (default=1)");
  115. static const struct pci_device_id pvscsi_pci_tbl[] = {
  116. { PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_PVSCSI) },
  117. { 0 }
  118. };
  119. MODULE_DEVICE_TABLE(pci, pvscsi_pci_tbl);
  120. static struct device *
  121. pvscsi_dev(const struct pvscsi_adapter *adapter)
  122. {
  123. return &(adapter->dev->dev);
  124. }
  125. static struct pvscsi_ctx *
  126. pvscsi_find_context(const struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd)
  127. {
  128. struct pvscsi_ctx *ctx, *end;
  129. end = &adapter->cmd_map[adapter->req_depth];
  130. for (ctx = adapter->cmd_map; ctx < end; ctx++)
  131. if (ctx->cmd == cmd)
  132. return ctx;
  133. return NULL;
  134. }
  135. static struct pvscsi_ctx *
  136. pvscsi_acquire_context(struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd)
  137. {
  138. struct pvscsi_ctx *ctx;
  139. if (list_empty(&adapter->cmd_pool))
  140. return NULL;
  141. ctx = list_first_entry(&adapter->cmd_pool, struct pvscsi_ctx, list);
  142. ctx->cmd = cmd;
  143. list_del(&ctx->list);
  144. return ctx;
  145. }
  146. static void pvscsi_release_context(struct pvscsi_adapter *adapter,
  147. struct pvscsi_ctx *ctx)
  148. {
  149. ctx->cmd = NULL;
  150. ctx->abort_cmp = NULL;
  151. list_add(&ctx->list, &adapter->cmd_pool);
  152. }
  153. /*
  154. * Map a pvscsi_ctx struct to a context ID field value; we map to a simple
  155. * non-zero integer. ctx always points to an entry in cmd_map array, hence
  156. * the return value is always >=1.
  157. */
  158. static u64 pvscsi_map_context(const struct pvscsi_adapter *adapter,
  159. const struct pvscsi_ctx *ctx)
  160. {
  161. return ctx - adapter->cmd_map + 1;
  162. }
  163. static struct pvscsi_ctx *
  164. pvscsi_get_context(const struct pvscsi_adapter *adapter, u64 context)
  165. {
  166. return &adapter->cmd_map[context - 1];
  167. }
  168. static void pvscsi_reg_write(const struct pvscsi_adapter *adapter,
  169. u32 offset, u32 val)
  170. {
  171. writel(val, adapter->mmioBase + offset);
  172. }
  173. static u32 pvscsi_reg_read(const struct pvscsi_adapter *adapter, u32 offset)
  174. {
  175. return readl(adapter->mmioBase + offset);
  176. }
  177. static u32 pvscsi_read_intr_status(const struct pvscsi_adapter *adapter)
  178. {
  179. return pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_INTR_STATUS);
  180. }
  181. static void pvscsi_write_intr_status(const struct pvscsi_adapter *adapter,
  182. u32 val)
  183. {
  184. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_STATUS, val);
  185. }
  186. static void pvscsi_unmask_intr(const struct pvscsi_adapter *adapter)
  187. {
  188. u32 intr_bits;
  189. intr_bits = PVSCSI_INTR_CMPL_MASK;
  190. if (adapter->use_msg)
  191. intr_bits |= PVSCSI_INTR_MSG_MASK;
  192. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, intr_bits);
  193. }
  194. static void pvscsi_mask_intr(const struct pvscsi_adapter *adapter)
  195. {
  196. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, 0);
  197. }
  198. static void pvscsi_write_cmd_desc(const struct pvscsi_adapter *adapter,
  199. u32 cmd, const void *desc, size_t len)
  200. {
  201. const u32 *ptr = desc;
  202. size_t i;
  203. len /= sizeof(*ptr);
  204. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND, cmd);
  205. for (i = 0; i < len; i++)
  206. pvscsi_reg_write(adapter,
  207. PVSCSI_REG_OFFSET_COMMAND_DATA, ptr[i]);
  208. }
  209. static void pvscsi_abort_cmd(const struct pvscsi_adapter *adapter,
  210. const struct pvscsi_ctx *ctx)
  211. {
  212. struct PVSCSICmdDescAbortCmd cmd = { 0 };
  213. cmd.target = ctx->cmd->device->id;
  214. cmd.context = pvscsi_map_context(adapter, ctx);
  215. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ABORT_CMD, &cmd, sizeof(cmd));
  216. }
  217. static void pvscsi_kick_rw_io(const struct pvscsi_adapter *adapter)
  218. {
  219. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_RW_IO, 0);
  220. }
  221. static void pvscsi_process_request_ring(const struct pvscsi_adapter *adapter)
  222. {
  223. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0);
  224. }
  225. static int scsi_is_rw(unsigned char op)
  226. {
  227. return op == READ_6 || op == WRITE_6 ||
  228. op == READ_10 || op == WRITE_10 ||
  229. op == READ_12 || op == WRITE_12 ||
  230. op == READ_16 || op == WRITE_16;
  231. }
  232. static void pvscsi_kick_io(const struct pvscsi_adapter *adapter,
  233. unsigned char op)
  234. {
  235. if (scsi_is_rw(op)) {
  236. struct PVSCSIRingsState *s = adapter->rings_state;
  237. if (!adapter->use_req_threshold ||
  238. s->reqProdIdx - s->reqConsIdx >= s->reqCallThreshold)
  239. pvscsi_kick_rw_io(adapter);
  240. } else {
  241. pvscsi_process_request_ring(adapter);
  242. }
  243. }
  244. static void ll_adapter_reset(const struct pvscsi_adapter *adapter)
  245. {
  246. dev_dbg(pvscsi_dev(adapter), "Adapter Reset on %p\n", adapter);
  247. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ADAPTER_RESET, NULL, 0);
  248. }
  249. static void ll_bus_reset(const struct pvscsi_adapter *adapter)
  250. {
  251. dev_dbg(pvscsi_dev(adapter), "Resetting bus on %p\n", adapter);
  252. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_BUS, NULL, 0);
  253. }
  254. static void ll_device_reset(const struct pvscsi_adapter *adapter, u32 target)
  255. {
  256. struct PVSCSICmdDescResetDevice cmd = { 0 };
  257. dev_dbg(pvscsi_dev(adapter), "Resetting device: target=%u\n", target);
  258. cmd.target = target;
  259. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_DEVICE,
  260. &cmd, sizeof(cmd));
  261. }
  262. static void pvscsi_create_sg(struct pvscsi_ctx *ctx,
  263. struct scatterlist *sg, unsigned count)
  264. {
  265. unsigned i;
  266. struct PVSCSISGElement *sge;
  267. BUG_ON(count > PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT);
  268. sge = &ctx->sgl->sge[0];
  269. for (i = 0; i < count; i++, sg = sg_next(sg)) {
  270. sge[i].addr = sg_dma_address(sg);
  271. sge[i].length = sg_dma_len(sg);
  272. sge[i].flags = 0;
  273. }
  274. }
  275. /*
  276. * Map all data buffers for a command into PCI space and
  277. * setup the scatter/gather list if needed.
  278. */
  279. static int pvscsi_map_buffers(struct pvscsi_adapter *adapter,
  280. struct pvscsi_ctx *ctx, struct scsi_cmnd *cmd,
  281. struct PVSCSIRingReqDesc *e)
  282. {
  283. unsigned count;
  284. unsigned bufflen = scsi_bufflen(cmd);
  285. struct scatterlist *sg;
  286. e->dataLen = bufflen;
  287. e->dataAddr = 0;
  288. if (bufflen == 0)
  289. return 0;
  290. sg = scsi_sglist(cmd);
  291. count = scsi_sg_count(cmd);
  292. if (count != 0) {
  293. int segs = scsi_dma_map(cmd);
  294. if (segs == -ENOMEM) {
  295. scmd_printk(KERN_DEBUG, cmd,
  296. "vmw_pvscsi: Failed to map cmd sglist for DMA.\n");
  297. return -ENOMEM;
  298. } else if (segs > 1) {
  299. pvscsi_create_sg(ctx, sg, segs);
  300. e->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST;
  301. ctx->sglPA = dma_map_single(&adapter->dev->dev,
  302. ctx->sgl, SGL_SIZE, DMA_TO_DEVICE);
  303. if (dma_mapping_error(&adapter->dev->dev, ctx->sglPA)) {
  304. scmd_printk(KERN_ERR, cmd,
  305. "vmw_pvscsi: Failed to map ctx sglist for DMA.\n");
  306. scsi_dma_unmap(cmd);
  307. ctx->sglPA = 0;
  308. return -ENOMEM;
  309. }
  310. e->dataAddr = ctx->sglPA;
  311. } else
  312. e->dataAddr = sg_dma_address(sg);
  313. } else {
  314. /*
  315. * In case there is no S/G list, scsi_sglist points
  316. * directly to the buffer.
  317. */
  318. ctx->dataPA = dma_map_single(&adapter->dev->dev, sg, bufflen,
  319. cmd->sc_data_direction);
  320. if (dma_mapping_error(&adapter->dev->dev, ctx->dataPA)) {
  321. scmd_printk(KERN_DEBUG, cmd,
  322. "vmw_pvscsi: Failed to map direct data buffer for DMA.\n");
  323. return -ENOMEM;
  324. }
  325. e->dataAddr = ctx->dataPA;
  326. }
  327. return 0;
  328. }
  329. /*
  330. * The device incorrectly doesn't clear the first byte of the sense
  331. * buffer in some cases. We have to do it ourselves.
  332. * Otherwise we run into trouble when SWIOTLB is forced.
  333. */
  334. static void pvscsi_patch_sense(struct scsi_cmnd *cmd)
  335. {
  336. if (cmd->sense_buffer)
  337. cmd->sense_buffer[0] = 0;
  338. }
  339. static void pvscsi_unmap_buffers(const struct pvscsi_adapter *adapter,
  340. struct pvscsi_ctx *ctx)
  341. {
  342. struct scsi_cmnd *cmd;
  343. unsigned bufflen;
  344. cmd = ctx->cmd;
  345. bufflen = scsi_bufflen(cmd);
  346. if (bufflen != 0) {
  347. unsigned count = scsi_sg_count(cmd);
  348. if (count != 0) {
  349. scsi_dma_unmap(cmd);
  350. if (ctx->sglPA) {
  351. dma_unmap_single(&adapter->dev->dev, ctx->sglPA,
  352. SGL_SIZE, DMA_TO_DEVICE);
  353. ctx->sglPA = 0;
  354. }
  355. } else
  356. dma_unmap_single(&adapter->dev->dev, ctx->dataPA,
  357. bufflen, cmd->sc_data_direction);
  358. }
  359. if (cmd->sense_buffer)
  360. dma_unmap_single(&adapter->dev->dev, ctx->sensePA,
  361. SCSI_SENSE_BUFFERSIZE, DMA_FROM_DEVICE);
  362. }
  363. static int pvscsi_allocate_rings(struct pvscsi_adapter *adapter)
  364. {
  365. adapter->rings_state = dma_alloc_coherent(&adapter->dev->dev, PAGE_SIZE,
  366. &adapter->ringStatePA, GFP_KERNEL);
  367. if (!adapter->rings_state)
  368. return -ENOMEM;
  369. adapter->req_pages = min(PVSCSI_MAX_NUM_PAGES_REQ_RING,
  370. pvscsi_ring_pages);
  371. adapter->req_depth = adapter->req_pages
  372. * PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
  373. adapter->req_ring = dma_alloc_coherent(&adapter->dev->dev,
  374. adapter->req_pages * PAGE_SIZE, &adapter->reqRingPA,
  375. GFP_KERNEL);
  376. if (!adapter->req_ring)
  377. return -ENOMEM;
  378. adapter->cmp_pages = min(PVSCSI_MAX_NUM_PAGES_CMP_RING,
  379. pvscsi_ring_pages);
  380. adapter->cmp_ring = dma_alloc_coherent(&adapter->dev->dev,
  381. adapter->cmp_pages * PAGE_SIZE, &adapter->cmpRingPA,
  382. GFP_KERNEL);
  383. if (!adapter->cmp_ring)
  384. return -ENOMEM;
  385. BUG_ON(!IS_ALIGNED(adapter->ringStatePA, PAGE_SIZE));
  386. BUG_ON(!IS_ALIGNED(adapter->reqRingPA, PAGE_SIZE));
  387. BUG_ON(!IS_ALIGNED(adapter->cmpRingPA, PAGE_SIZE));
  388. if (!adapter->use_msg)
  389. return 0;
  390. adapter->msg_pages = min(PVSCSI_MAX_NUM_PAGES_MSG_RING,
  391. pvscsi_msg_ring_pages);
  392. adapter->msg_ring = dma_alloc_coherent(&adapter->dev->dev,
  393. adapter->msg_pages * PAGE_SIZE, &adapter->msgRingPA,
  394. GFP_KERNEL);
  395. if (!adapter->msg_ring)
  396. return -ENOMEM;
  397. BUG_ON(!IS_ALIGNED(adapter->msgRingPA, PAGE_SIZE));
  398. return 0;
  399. }
  400. static void pvscsi_setup_all_rings(const struct pvscsi_adapter *adapter)
  401. {
  402. struct PVSCSICmdDescSetupRings cmd = { 0 };
  403. dma_addr_t base;
  404. unsigned i;
  405. cmd.ringsStatePPN = adapter->ringStatePA >> PAGE_SHIFT;
  406. cmd.reqRingNumPages = adapter->req_pages;
  407. cmd.cmpRingNumPages = adapter->cmp_pages;
  408. base = adapter->reqRingPA;
  409. for (i = 0; i < adapter->req_pages; i++) {
  410. cmd.reqRingPPNs[i] = base >> PAGE_SHIFT;
  411. base += PAGE_SIZE;
  412. }
  413. base = adapter->cmpRingPA;
  414. for (i = 0; i < adapter->cmp_pages; i++) {
  415. cmd.cmpRingPPNs[i] = base >> PAGE_SHIFT;
  416. base += PAGE_SIZE;
  417. }
  418. memset(adapter->rings_state, 0, PAGE_SIZE);
  419. memset(adapter->req_ring, 0, adapter->req_pages * PAGE_SIZE);
  420. memset(adapter->cmp_ring, 0, adapter->cmp_pages * PAGE_SIZE);
  421. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_RINGS,
  422. &cmd, sizeof(cmd));
  423. if (adapter->use_msg) {
  424. struct PVSCSICmdDescSetupMsgRing cmd_msg = { 0 };
  425. cmd_msg.numPages = adapter->msg_pages;
  426. base = adapter->msgRingPA;
  427. for (i = 0; i < adapter->msg_pages; i++) {
  428. cmd_msg.ringPPNs[i] = base >> PAGE_SHIFT;
  429. base += PAGE_SIZE;
  430. }
  431. memset(adapter->msg_ring, 0, adapter->msg_pages * PAGE_SIZE);
  432. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_MSG_RING,
  433. &cmd_msg, sizeof(cmd_msg));
  434. }
  435. }
  436. static int pvscsi_change_queue_depth(struct scsi_device *sdev, int qdepth)
  437. {
  438. if (!sdev->tagged_supported)
  439. qdepth = 1;
  440. return scsi_change_queue_depth(sdev, qdepth);
  441. }
  442. /*
  443. * Pull a completion descriptor off and pass the completion back
  444. * to the SCSI mid layer.
  445. */
  446. static void pvscsi_complete_request(struct pvscsi_adapter *adapter,
  447. const struct PVSCSIRingCmpDesc *e)
  448. {
  449. struct pvscsi_ctx *ctx;
  450. struct scsi_cmnd *cmd;
  451. struct completion *abort_cmp;
  452. u32 btstat = e->hostStatus;
  453. u32 sdstat = e->scsiStatus;
  454. ctx = pvscsi_get_context(adapter, e->context);
  455. cmd = ctx->cmd;
  456. abort_cmp = ctx->abort_cmp;
  457. pvscsi_unmap_buffers(adapter, ctx);
  458. if (sdstat != SAM_STAT_CHECK_CONDITION)
  459. pvscsi_patch_sense(cmd);
  460. pvscsi_release_context(adapter, ctx);
  461. if (abort_cmp) {
  462. /*
  463. * The command was requested to be aborted. Just signal that
  464. * the request completed and swallow the actual cmd completion
  465. * here. The abort handler will post a completion for this
  466. * command indicating that it got successfully aborted.
  467. */
  468. complete(abort_cmp);
  469. return;
  470. }
  471. cmd->result = 0;
  472. if (sdstat != SAM_STAT_GOOD &&
  473. (btstat == BTSTAT_SUCCESS ||
  474. btstat == BTSTAT_LINKED_COMMAND_COMPLETED ||
  475. btstat == BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG)) {
  476. if (sdstat == SAM_STAT_COMMAND_TERMINATED) {
  477. cmd->result = (DID_RESET << 16);
  478. } else {
  479. cmd->result = (DID_OK << 16) | sdstat;
  480. if (sdstat == SAM_STAT_CHECK_CONDITION &&
  481. cmd->sense_buffer)
  482. cmd->result |= (DRIVER_SENSE << 24);
  483. }
  484. } else
  485. switch (btstat) {
  486. case BTSTAT_SUCCESS:
  487. case BTSTAT_LINKED_COMMAND_COMPLETED:
  488. case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG:
  489. /*
  490. * Commands like INQUIRY may transfer less data than
  491. * requested by the initiator via bufflen. Set residual
  492. * count to make upper layer aware of the actual amount
  493. * of data returned. There are cases when controller
  494. * returns zero dataLen with non zero data - do not set
  495. * residual count in that case.
  496. */
  497. if (e->dataLen && (e->dataLen < scsi_bufflen(cmd)))
  498. scsi_set_resid(cmd, scsi_bufflen(cmd) - e->dataLen);
  499. cmd->result = (DID_OK << 16);
  500. break;
  501. case BTSTAT_DATARUN:
  502. case BTSTAT_DATA_UNDERRUN:
  503. /* Report residual data in underruns */
  504. scsi_set_resid(cmd, scsi_bufflen(cmd) - e->dataLen);
  505. cmd->result = (DID_ERROR << 16);
  506. break;
  507. case BTSTAT_SELTIMEO:
  508. /* Our emulation returns this for non-connected devs */
  509. cmd->result = (DID_BAD_TARGET << 16);
  510. break;
  511. case BTSTAT_LUNMISMATCH:
  512. case BTSTAT_TAGREJECT:
  513. case BTSTAT_BADMSG:
  514. cmd->result = (DRIVER_INVALID << 24);
  515. fallthrough;
  516. case BTSTAT_HAHARDWARE:
  517. case BTSTAT_INVPHASE:
  518. case BTSTAT_HATIMEOUT:
  519. case BTSTAT_NORESPONSE:
  520. case BTSTAT_DISCONNECT:
  521. case BTSTAT_HASOFTWARE:
  522. case BTSTAT_BUSFREE:
  523. case BTSTAT_SENSFAILED:
  524. cmd->result |= (DID_ERROR << 16);
  525. break;
  526. case BTSTAT_SENTRST:
  527. case BTSTAT_RECVRST:
  528. case BTSTAT_BUSRESET:
  529. cmd->result = (DID_RESET << 16);
  530. break;
  531. case BTSTAT_ABORTQUEUE:
  532. cmd->result = (DID_BUS_BUSY << 16);
  533. break;
  534. case BTSTAT_SCSIPARITY:
  535. cmd->result = (DID_PARITY << 16);
  536. break;
  537. default:
  538. cmd->result = (DID_ERROR << 16);
  539. scmd_printk(KERN_DEBUG, cmd,
  540. "Unknown completion status: 0x%x\n",
  541. btstat);
  542. }
  543. dev_dbg(&cmd->device->sdev_gendev,
  544. "cmd=%p %x ctx=%p result=0x%x status=0x%x,%x\n",
  545. cmd, cmd->cmnd[0], ctx, cmd->result, btstat, sdstat);
  546. cmd->scsi_done(cmd);
  547. }
  548. /*
  549. * barrier usage : Since the PVSCSI device is emulated, there could be cases
  550. * where we may want to serialize some accesses between the driver and the
  551. * emulation layer. We use compiler barriers instead of the more expensive
  552. * memory barriers because PVSCSI is only supported on X86 which has strong
  553. * memory access ordering.
  554. */
  555. static void pvscsi_process_completion_ring(struct pvscsi_adapter *adapter)
  556. {
  557. struct PVSCSIRingsState *s = adapter->rings_state;
  558. struct PVSCSIRingCmpDesc *ring = adapter->cmp_ring;
  559. u32 cmp_entries = s->cmpNumEntriesLog2;
  560. while (s->cmpConsIdx != s->cmpProdIdx) {
  561. struct PVSCSIRingCmpDesc *e = ring + (s->cmpConsIdx &
  562. MASK(cmp_entries));
  563. /*
  564. * This barrier() ensures that *e is not dereferenced while
  565. * the device emulation still writes data into the slot.
  566. * Since the device emulation advances s->cmpProdIdx only after
  567. * updating the slot we want to check it first.
  568. */
  569. barrier();
  570. pvscsi_complete_request(adapter, e);
  571. /*
  572. * This barrier() ensures that compiler doesn't reorder write
  573. * to s->cmpConsIdx before the read of (*e) inside
  574. * pvscsi_complete_request. Otherwise, device emulation may
  575. * overwrite *e before we had a chance to read it.
  576. */
  577. barrier();
  578. s->cmpConsIdx++;
  579. }
  580. }
  581. /*
  582. * Translate a Linux SCSI request into a request ring entry.
  583. */
  584. static int pvscsi_queue_ring(struct pvscsi_adapter *adapter,
  585. struct pvscsi_ctx *ctx, struct scsi_cmnd *cmd)
  586. {
  587. struct PVSCSIRingsState *s;
  588. struct PVSCSIRingReqDesc *e;
  589. struct scsi_device *sdev;
  590. u32 req_entries;
  591. s = adapter->rings_state;
  592. sdev = cmd->device;
  593. req_entries = s->reqNumEntriesLog2;
  594. /*
  595. * If this condition holds, we might have room on the request ring, but
  596. * we might not have room on the completion ring for the response.
  597. * However, we have already ruled out this possibility - we would not
  598. * have successfully allocated a context if it were true, since we only
  599. * have one context per request entry. Check for it anyway, since it
  600. * would be a serious bug.
  601. */
  602. if (s->reqProdIdx - s->cmpConsIdx >= 1 << req_entries) {
  603. scmd_printk(KERN_ERR, cmd, "vmw_pvscsi: "
  604. "ring full: reqProdIdx=%d cmpConsIdx=%d\n",
  605. s->reqProdIdx, s->cmpConsIdx);
  606. return -1;
  607. }
  608. e = adapter->req_ring + (s->reqProdIdx & MASK(req_entries));
  609. e->bus = sdev->channel;
  610. e->target = sdev->id;
  611. memset(e->lun, 0, sizeof(e->lun));
  612. e->lun[1] = sdev->lun;
  613. if (cmd->sense_buffer) {
  614. ctx->sensePA = dma_map_single(&adapter->dev->dev,
  615. cmd->sense_buffer, SCSI_SENSE_BUFFERSIZE,
  616. DMA_FROM_DEVICE);
  617. if (dma_mapping_error(&adapter->dev->dev, ctx->sensePA)) {
  618. scmd_printk(KERN_DEBUG, cmd,
  619. "vmw_pvscsi: Failed to map sense buffer for DMA.\n");
  620. ctx->sensePA = 0;
  621. return -ENOMEM;
  622. }
  623. e->senseAddr = ctx->sensePA;
  624. e->senseLen = SCSI_SENSE_BUFFERSIZE;
  625. } else {
  626. e->senseLen = 0;
  627. e->senseAddr = 0;
  628. }
  629. e->cdbLen = cmd->cmd_len;
  630. e->vcpuHint = smp_processor_id();
  631. memcpy(e->cdb, cmd->cmnd, e->cdbLen);
  632. e->tag = SIMPLE_QUEUE_TAG;
  633. if (cmd->sc_data_direction == DMA_FROM_DEVICE)
  634. e->flags = PVSCSI_FLAG_CMD_DIR_TOHOST;
  635. else if (cmd->sc_data_direction == DMA_TO_DEVICE)
  636. e->flags = PVSCSI_FLAG_CMD_DIR_TODEVICE;
  637. else if (cmd->sc_data_direction == DMA_NONE)
  638. e->flags = PVSCSI_FLAG_CMD_DIR_NONE;
  639. else
  640. e->flags = 0;
  641. if (pvscsi_map_buffers(adapter, ctx, cmd, e) != 0) {
  642. if (cmd->sense_buffer) {
  643. dma_unmap_single(&adapter->dev->dev, ctx->sensePA,
  644. SCSI_SENSE_BUFFERSIZE,
  645. DMA_FROM_DEVICE);
  646. ctx->sensePA = 0;
  647. }
  648. return -ENOMEM;
  649. }
  650. e->context = pvscsi_map_context(adapter, ctx);
  651. barrier();
  652. s->reqProdIdx++;
  653. return 0;
  654. }
  655. static int pvscsi_queue_lck(struct scsi_cmnd *cmd, void (*done)(struct scsi_cmnd *))
  656. {
  657. struct Scsi_Host *host = cmd->device->host;
  658. struct pvscsi_adapter *adapter = shost_priv(host);
  659. struct pvscsi_ctx *ctx;
  660. unsigned long flags;
  661. unsigned char op;
  662. spin_lock_irqsave(&adapter->hw_lock, flags);
  663. ctx = pvscsi_acquire_context(adapter, cmd);
  664. if (!ctx || pvscsi_queue_ring(adapter, ctx, cmd) != 0) {
  665. if (ctx)
  666. pvscsi_release_context(adapter, ctx);
  667. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  668. return SCSI_MLQUEUE_HOST_BUSY;
  669. }
  670. cmd->scsi_done = done;
  671. op = cmd->cmnd[0];
  672. dev_dbg(&cmd->device->sdev_gendev,
  673. "queued cmd %p, ctx %p, op=%x\n", cmd, ctx, op);
  674. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  675. pvscsi_kick_io(adapter, op);
  676. return 0;
  677. }
  678. static DEF_SCSI_QCMD(pvscsi_queue)
  679. static int pvscsi_abort(struct scsi_cmnd *cmd)
  680. {
  681. struct pvscsi_adapter *adapter = shost_priv(cmd->device->host);
  682. struct pvscsi_ctx *ctx;
  683. unsigned long flags;
  684. int result = SUCCESS;
  685. DECLARE_COMPLETION_ONSTACK(abort_cmp);
  686. int done;
  687. scmd_printk(KERN_DEBUG, cmd, "task abort on host %u, %p\n",
  688. adapter->host->host_no, cmd);
  689. spin_lock_irqsave(&adapter->hw_lock, flags);
  690. /*
  691. * Poll the completion ring first - we might be trying to abort
  692. * a command that is waiting to be dispatched in the completion ring.
  693. */
  694. pvscsi_process_completion_ring(adapter);
  695. /*
  696. * If there is no context for the command, it either already succeeded
  697. * or else was never properly issued. Not our problem.
  698. */
  699. ctx = pvscsi_find_context(adapter, cmd);
  700. if (!ctx) {
  701. scmd_printk(KERN_DEBUG, cmd, "Failed to abort cmd %p\n", cmd);
  702. goto out;
  703. }
  704. /*
  705. * Mark that the command has been requested to be aborted and issue
  706. * the abort.
  707. */
  708. ctx->abort_cmp = &abort_cmp;
  709. pvscsi_abort_cmd(adapter, ctx);
  710. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  711. /* Wait for 2 secs for the completion. */
  712. done = wait_for_completion_timeout(&abort_cmp, msecs_to_jiffies(2000));
  713. spin_lock_irqsave(&adapter->hw_lock, flags);
  714. if (!done) {
  715. /*
  716. * Failed to abort the command, unmark the fact that it
  717. * was requested to be aborted.
  718. */
  719. ctx->abort_cmp = NULL;
  720. result = FAILED;
  721. scmd_printk(KERN_DEBUG, cmd,
  722. "Failed to get completion for aborted cmd %p\n",
  723. cmd);
  724. goto out;
  725. }
  726. /*
  727. * Successfully aborted the command.
  728. */
  729. cmd->result = (DID_ABORT << 16);
  730. cmd->scsi_done(cmd);
  731. out:
  732. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  733. return result;
  734. }
  735. /*
  736. * Abort all outstanding requests. This is only safe to use if the completion
  737. * ring will never be walked again or the device has been reset, because it
  738. * destroys the 1-1 mapping between context field passed to emulation and our
  739. * request structure.
  740. */
  741. static void pvscsi_reset_all(struct pvscsi_adapter *adapter)
  742. {
  743. unsigned i;
  744. for (i = 0; i < adapter->req_depth; i++) {
  745. struct pvscsi_ctx *ctx = &adapter->cmd_map[i];
  746. struct scsi_cmnd *cmd = ctx->cmd;
  747. if (cmd) {
  748. scmd_printk(KERN_ERR, cmd,
  749. "Forced reset on cmd %p\n", cmd);
  750. pvscsi_unmap_buffers(adapter, ctx);
  751. pvscsi_patch_sense(cmd);
  752. pvscsi_release_context(adapter, ctx);
  753. cmd->result = (DID_RESET << 16);
  754. cmd->scsi_done(cmd);
  755. }
  756. }
  757. }
  758. static int pvscsi_host_reset(struct scsi_cmnd *cmd)
  759. {
  760. struct Scsi_Host *host = cmd->device->host;
  761. struct pvscsi_adapter *adapter = shost_priv(host);
  762. unsigned long flags;
  763. bool use_msg;
  764. scmd_printk(KERN_INFO, cmd, "SCSI Host reset\n");
  765. spin_lock_irqsave(&adapter->hw_lock, flags);
  766. use_msg = adapter->use_msg;
  767. if (use_msg) {
  768. adapter->use_msg = false;
  769. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  770. /*
  771. * Now that we know that the ISR won't add more work on the
  772. * workqueue we can safely flush any outstanding work.
  773. */
  774. flush_workqueue(adapter->workqueue);
  775. spin_lock_irqsave(&adapter->hw_lock, flags);
  776. }
  777. /*
  778. * We're going to tear down the entire ring structure and set it back
  779. * up, so stalling new requests until all completions are flushed and
  780. * the rings are back in place.
  781. */
  782. pvscsi_process_request_ring(adapter);
  783. ll_adapter_reset(adapter);
  784. /*
  785. * Now process any completions. Note we do this AFTER adapter reset,
  786. * which is strange, but stops races where completions get posted
  787. * between processing the ring and issuing the reset. The backend will
  788. * not touch the ring memory after reset, so the immediately pre-reset
  789. * completion ring state is still valid.
  790. */
  791. pvscsi_process_completion_ring(adapter);
  792. pvscsi_reset_all(adapter);
  793. adapter->use_msg = use_msg;
  794. pvscsi_setup_all_rings(adapter);
  795. pvscsi_unmask_intr(adapter);
  796. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  797. return SUCCESS;
  798. }
  799. static int pvscsi_bus_reset(struct scsi_cmnd *cmd)
  800. {
  801. struct Scsi_Host *host = cmd->device->host;
  802. struct pvscsi_adapter *adapter = shost_priv(host);
  803. unsigned long flags;
  804. scmd_printk(KERN_INFO, cmd, "SCSI Bus reset\n");
  805. /*
  806. * We don't want to queue new requests for this bus after
  807. * flushing all pending requests to emulation, since new
  808. * requests could then sneak in during this bus reset phase,
  809. * so take the lock now.
  810. */
  811. spin_lock_irqsave(&adapter->hw_lock, flags);
  812. pvscsi_process_request_ring(adapter);
  813. ll_bus_reset(adapter);
  814. pvscsi_process_completion_ring(adapter);
  815. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  816. return SUCCESS;
  817. }
  818. static int pvscsi_device_reset(struct scsi_cmnd *cmd)
  819. {
  820. struct Scsi_Host *host = cmd->device->host;
  821. struct pvscsi_adapter *adapter = shost_priv(host);
  822. unsigned long flags;
  823. scmd_printk(KERN_INFO, cmd, "SCSI device reset on scsi%u:%u\n",
  824. host->host_no, cmd->device->id);
  825. /*
  826. * We don't want to queue new requests for this device after flushing
  827. * all pending requests to emulation, since new requests could then
  828. * sneak in during this device reset phase, so take the lock now.
  829. */
  830. spin_lock_irqsave(&adapter->hw_lock, flags);
  831. pvscsi_process_request_ring(adapter);
  832. ll_device_reset(adapter, cmd->device->id);
  833. pvscsi_process_completion_ring(adapter);
  834. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  835. return SUCCESS;
  836. }
  837. static struct scsi_host_template pvscsi_template;
  838. static const char *pvscsi_info(struct Scsi_Host *host)
  839. {
  840. struct pvscsi_adapter *adapter = shost_priv(host);
  841. static char buf[256];
  842. sprintf(buf, "VMware PVSCSI storage adapter rev %d, req/cmp/msg rings: "
  843. "%u/%u/%u pages, cmd_per_lun=%u", adapter->rev,
  844. adapter->req_pages, adapter->cmp_pages, adapter->msg_pages,
  845. pvscsi_template.cmd_per_lun);
  846. return buf;
  847. }
  848. static struct scsi_host_template pvscsi_template = {
  849. .module = THIS_MODULE,
  850. .name = "VMware PVSCSI Host Adapter",
  851. .proc_name = "vmw_pvscsi",
  852. .info = pvscsi_info,
  853. .queuecommand = pvscsi_queue,
  854. .this_id = -1,
  855. .sg_tablesize = PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT,
  856. .dma_boundary = UINT_MAX,
  857. .max_sectors = 0xffff,
  858. .change_queue_depth = pvscsi_change_queue_depth,
  859. .eh_abort_handler = pvscsi_abort,
  860. .eh_device_reset_handler = pvscsi_device_reset,
  861. .eh_bus_reset_handler = pvscsi_bus_reset,
  862. .eh_host_reset_handler = pvscsi_host_reset,
  863. };
  864. static void pvscsi_process_msg(const struct pvscsi_adapter *adapter,
  865. const struct PVSCSIRingMsgDesc *e)
  866. {
  867. struct PVSCSIRingsState *s = adapter->rings_state;
  868. struct Scsi_Host *host = adapter->host;
  869. struct scsi_device *sdev;
  870. printk(KERN_INFO "vmw_pvscsi: msg type: 0x%x - MSG RING: %u/%u (%u) \n",
  871. e->type, s->msgProdIdx, s->msgConsIdx, s->msgNumEntriesLog2);
  872. BUILD_BUG_ON(PVSCSI_MSG_LAST != 2);
  873. if (e->type == PVSCSI_MSG_DEV_ADDED) {
  874. struct PVSCSIMsgDescDevStatusChanged *desc;
  875. desc = (struct PVSCSIMsgDescDevStatusChanged *)e;
  876. printk(KERN_INFO
  877. "vmw_pvscsi: msg: device added at scsi%u:%u:%u\n",
  878. desc->bus, desc->target, desc->lun[1]);
  879. if (!scsi_host_get(host))
  880. return;
  881. sdev = scsi_device_lookup(host, desc->bus, desc->target,
  882. desc->lun[1]);
  883. if (sdev) {
  884. printk(KERN_INFO "vmw_pvscsi: device already exists\n");
  885. scsi_device_put(sdev);
  886. } else
  887. scsi_add_device(adapter->host, desc->bus,
  888. desc->target, desc->lun[1]);
  889. scsi_host_put(host);
  890. } else if (e->type == PVSCSI_MSG_DEV_REMOVED) {
  891. struct PVSCSIMsgDescDevStatusChanged *desc;
  892. desc = (struct PVSCSIMsgDescDevStatusChanged *)e;
  893. printk(KERN_INFO
  894. "vmw_pvscsi: msg: device removed at scsi%u:%u:%u\n",
  895. desc->bus, desc->target, desc->lun[1]);
  896. if (!scsi_host_get(host))
  897. return;
  898. sdev = scsi_device_lookup(host, desc->bus, desc->target,
  899. desc->lun[1]);
  900. if (sdev) {
  901. scsi_remove_device(sdev);
  902. scsi_device_put(sdev);
  903. } else
  904. printk(KERN_INFO
  905. "vmw_pvscsi: failed to lookup scsi%u:%u:%u\n",
  906. desc->bus, desc->target, desc->lun[1]);
  907. scsi_host_put(host);
  908. }
  909. }
  910. static int pvscsi_msg_pending(const struct pvscsi_adapter *adapter)
  911. {
  912. struct PVSCSIRingsState *s = adapter->rings_state;
  913. return s->msgProdIdx != s->msgConsIdx;
  914. }
  915. static void pvscsi_process_msg_ring(const struct pvscsi_adapter *adapter)
  916. {
  917. struct PVSCSIRingsState *s = adapter->rings_state;
  918. struct PVSCSIRingMsgDesc *ring = adapter->msg_ring;
  919. u32 msg_entries = s->msgNumEntriesLog2;
  920. while (pvscsi_msg_pending(adapter)) {
  921. struct PVSCSIRingMsgDesc *e = ring + (s->msgConsIdx &
  922. MASK(msg_entries));
  923. barrier();
  924. pvscsi_process_msg(adapter, e);
  925. barrier();
  926. s->msgConsIdx++;
  927. }
  928. }
  929. static void pvscsi_msg_workqueue_handler(struct work_struct *data)
  930. {
  931. struct pvscsi_adapter *adapter;
  932. adapter = container_of(data, struct pvscsi_adapter, work);
  933. pvscsi_process_msg_ring(adapter);
  934. }
  935. static int pvscsi_setup_msg_workqueue(struct pvscsi_adapter *adapter)
  936. {
  937. char name[32];
  938. if (!pvscsi_use_msg)
  939. return 0;
  940. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND,
  941. PVSCSI_CMD_SETUP_MSG_RING);
  942. if (pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS) == -1)
  943. return 0;
  944. snprintf(name, sizeof(name),
  945. "vmw_pvscsi_wq_%u", adapter->host->host_no);
  946. adapter->workqueue = create_singlethread_workqueue(name);
  947. if (!adapter->workqueue) {
  948. printk(KERN_ERR "vmw_pvscsi: failed to create work queue\n");
  949. return 0;
  950. }
  951. INIT_WORK(&adapter->work, pvscsi_msg_workqueue_handler);
  952. return 1;
  953. }
  954. static bool pvscsi_setup_req_threshold(struct pvscsi_adapter *adapter,
  955. bool enable)
  956. {
  957. u32 val;
  958. if (!pvscsi_use_req_threshold)
  959. return false;
  960. pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND,
  961. PVSCSI_CMD_SETUP_REQCALLTHRESHOLD);
  962. val = pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS);
  963. if (val == -1) {
  964. printk(KERN_INFO "vmw_pvscsi: device does not support req_threshold\n");
  965. return false;
  966. } else {
  967. struct PVSCSICmdDescSetupReqCall cmd_msg = { 0 };
  968. cmd_msg.enable = enable;
  969. printk(KERN_INFO
  970. "vmw_pvscsi: %sabling reqCallThreshold\n",
  971. enable ? "en" : "dis");
  972. pvscsi_write_cmd_desc(adapter,
  973. PVSCSI_CMD_SETUP_REQCALLTHRESHOLD,
  974. &cmd_msg, sizeof(cmd_msg));
  975. return pvscsi_reg_read(adapter,
  976. PVSCSI_REG_OFFSET_COMMAND_STATUS) != 0;
  977. }
  978. }
  979. static irqreturn_t pvscsi_isr(int irq, void *devp)
  980. {
  981. struct pvscsi_adapter *adapter = devp;
  982. unsigned long flags;
  983. spin_lock_irqsave(&adapter->hw_lock, flags);
  984. pvscsi_process_completion_ring(adapter);
  985. if (adapter->use_msg && pvscsi_msg_pending(adapter))
  986. queue_work(adapter->workqueue, &adapter->work);
  987. spin_unlock_irqrestore(&adapter->hw_lock, flags);
  988. return IRQ_HANDLED;
  989. }
  990. static irqreturn_t pvscsi_shared_isr(int irq, void *devp)
  991. {
  992. struct pvscsi_adapter *adapter = devp;
  993. u32 val = pvscsi_read_intr_status(adapter);
  994. if (!(val & PVSCSI_INTR_ALL_SUPPORTED))
  995. return IRQ_NONE;
  996. pvscsi_write_intr_status(devp, val);
  997. return pvscsi_isr(irq, devp);
  998. }
  999. static void pvscsi_free_sgls(const struct pvscsi_adapter *adapter)
  1000. {
  1001. struct pvscsi_ctx *ctx = adapter->cmd_map;
  1002. unsigned i;
  1003. for (i = 0; i < adapter->req_depth; ++i, ++ctx)
  1004. free_pages((unsigned long)ctx->sgl, get_order(SGL_SIZE));
  1005. }
  1006. static void pvscsi_shutdown_intr(struct pvscsi_adapter *adapter)
  1007. {
  1008. free_irq(pci_irq_vector(adapter->dev, 0), adapter);
  1009. pci_free_irq_vectors(adapter->dev);
  1010. }
  1011. static void pvscsi_release_resources(struct pvscsi_adapter *adapter)
  1012. {
  1013. if (adapter->workqueue)
  1014. destroy_workqueue(adapter->workqueue);
  1015. if (adapter->mmioBase)
  1016. pci_iounmap(adapter->dev, adapter->mmioBase);
  1017. pci_release_regions(adapter->dev);
  1018. if (adapter->cmd_map) {
  1019. pvscsi_free_sgls(adapter);
  1020. kfree(adapter->cmd_map);
  1021. }
  1022. if (adapter->rings_state)
  1023. dma_free_coherent(&adapter->dev->dev, PAGE_SIZE,
  1024. adapter->rings_state, adapter->ringStatePA);
  1025. if (adapter->req_ring)
  1026. dma_free_coherent(&adapter->dev->dev,
  1027. adapter->req_pages * PAGE_SIZE,
  1028. adapter->req_ring, adapter->reqRingPA);
  1029. if (adapter->cmp_ring)
  1030. dma_free_coherent(&adapter->dev->dev,
  1031. adapter->cmp_pages * PAGE_SIZE,
  1032. adapter->cmp_ring, adapter->cmpRingPA);
  1033. if (adapter->msg_ring)
  1034. dma_free_coherent(&adapter->dev->dev,
  1035. adapter->msg_pages * PAGE_SIZE,
  1036. adapter->msg_ring, adapter->msgRingPA);
  1037. }
  1038. /*
  1039. * Allocate scatter gather lists.
  1040. *
  1041. * These are statically allocated. Trying to be clever was not worth it.
  1042. *
  1043. * Dynamic allocation can fail, and we can't go deep into the memory
  1044. * allocator, since we're a SCSI driver, and trying too hard to allocate
  1045. * memory might generate disk I/O. We also don't want to fail disk I/O
  1046. * in that case because we can't get an allocation - the I/O could be
  1047. * trying to swap out data to free memory. Since that is pathological,
  1048. * just use a statically allocated scatter list.
  1049. *
  1050. */
  1051. static int pvscsi_allocate_sg(struct pvscsi_adapter *adapter)
  1052. {
  1053. struct pvscsi_ctx *ctx;
  1054. int i;
  1055. ctx = adapter->cmd_map;
  1056. BUILD_BUG_ON(sizeof(struct pvscsi_sg_list) > SGL_SIZE);
  1057. for (i = 0; i < adapter->req_depth; ++i, ++ctx) {
  1058. ctx->sgl = (void *)__get_free_pages(GFP_KERNEL,
  1059. get_order(SGL_SIZE));
  1060. ctx->sglPA = 0;
  1061. BUG_ON(!IS_ALIGNED(((unsigned long)ctx->sgl), PAGE_SIZE));
  1062. if (!ctx->sgl) {
  1063. for (; i >= 0; --i, --ctx) {
  1064. free_pages((unsigned long)ctx->sgl,
  1065. get_order(SGL_SIZE));
  1066. ctx->sgl = NULL;
  1067. }
  1068. return -ENOMEM;
  1069. }
  1070. }
  1071. return 0;
  1072. }
  1073. /*
  1074. * Query the device, fetch the config info and return the
  1075. * maximum number of targets on the adapter. In case of
  1076. * failure due to any reason return default i.e. 16.
  1077. */
  1078. static u32 pvscsi_get_max_targets(struct pvscsi_adapter *adapter)
  1079. {
  1080. struct PVSCSICmdDescConfigCmd cmd;
  1081. struct PVSCSIConfigPageHeader *header;
  1082. struct device *dev;
  1083. dma_addr_t configPagePA;
  1084. void *config_page;
  1085. u32 numPhys = 16;
  1086. dev = pvscsi_dev(adapter);
  1087. config_page = dma_alloc_coherent(&adapter->dev->dev, PAGE_SIZE,
  1088. &configPagePA, GFP_KERNEL);
  1089. if (!config_page) {
  1090. dev_warn(dev, "vmw_pvscsi: failed to allocate memory for config page\n");
  1091. goto exit;
  1092. }
  1093. BUG_ON(configPagePA & ~PAGE_MASK);
  1094. /* Fetch config info from the device. */
  1095. cmd.configPageAddress = ((u64)PVSCSI_CONFIG_CONTROLLER_ADDRESS) << 32;
  1096. cmd.configPageNum = PVSCSI_CONFIG_PAGE_CONTROLLER;
  1097. cmd.cmpAddr = configPagePA;
  1098. cmd._pad = 0;
  1099. /*
  1100. * Mark the completion page header with error values. If the device
  1101. * completes the command successfully, it sets the status values to
  1102. * indicate success.
  1103. */
  1104. header = config_page;
  1105. memset(header, 0, sizeof *header);
  1106. header->hostStatus = BTSTAT_INVPARAM;
  1107. header->scsiStatus = SDSTAT_CHECK;
  1108. pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_CONFIG, &cmd, sizeof cmd);
  1109. if (header->hostStatus == BTSTAT_SUCCESS &&
  1110. header->scsiStatus == SDSTAT_GOOD) {
  1111. struct PVSCSIConfigPageController *config;
  1112. config = config_page;
  1113. numPhys = config->numPhys;
  1114. } else
  1115. dev_warn(dev, "vmw_pvscsi: PVSCSI_CMD_CONFIG failed. hostStatus = 0x%x, scsiStatus = 0x%x\n",
  1116. header->hostStatus, header->scsiStatus);
  1117. dma_free_coherent(&adapter->dev->dev, PAGE_SIZE, config_page,
  1118. configPagePA);
  1119. exit:
  1120. return numPhys;
  1121. }
  1122. static int pvscsi_probe(struct pci_dev *pdev, const struct pci_device_id *id)
  1123. {
  1124. unsigned int irq_flag = PCI_IRQ_MSIX | PCI_IRQ_MSI | PCI_IRQ_LEGACY;
  1125. struct pvscsi_adapter *adapter;
  1126. struct pvscsi_adapter adapter_temp;
  1127. struct Scsi_Host *host = NULL;
  1128. unsigned int i;
  1129. int error;
  1130. u32 max_id;
  1131. error = -ENODEV;
  1132. if (pci_enable_device(pdev))
  1133. return error;
  1134. if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
  1135. printk(KERN_INFO "vmw_pvscsi: using 64bit dma\n");
  1136. } else if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) {
  1137. printk(KERN_INFO "vmw_pvscsi: using 32bit dma\n");
  1138. } else {
  1139. printk(KERN_ERR "vmw_pvscsi: failed to set DMA mask\n");
  1140. goto out_disable_device;
  1141. }
  1142. /*
  1143. * Let's use a temp pvscsi_adapter struct until we find the number of
  1144. * targets on the adapter, after that we will switch to the real
  1145. * allocated struct.
  1146. */
  1147. adapter = &adapter_temp;
  1148. memset(adapter, 0, sizeof(*adapter));
  1149. adapter->dev = pdev;
  1150. adapter->rev = pdev->revision;
  1151. if (pci_request_regions(pdev, "vmw_pvscsi")) {
  1152. printk(KERN_ERR "vmw_pvscsi: pci memory selection failed\n");
  1153. goto out_disable_device;
  1154. }
  1155. for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
  1156. if ((pci_resource_flags(pdev, i) & PCI_BASE_ADDRESS_SPACE_IO))
  1157. continue;
  1158. if (pci_resource_len(pdev, i) < PVSCSI_MEM_SPACE_SIZE)
  1159. continue;
  1160. break;
  1161. }
  1162. if (i == DEVICE_COUNT_RESOURCE) {
  1163. printk(KERN_ERR
  1164. "vmw_pvscsi: adapter has no suitable MMIO region\n");
  1165. goto out_release_resources_and_disable;
  1166. }
  1167. adapter->mmioBase = pci_iomap(pdev, i, PVSCSI_MEM_SPACE_SIZE);
  1168. if (!adapter->mmioBase) {
  1169. printk(KERN_ERR
  1170. "vmw_pvscsi: can't iomap for BAR %d memsize %lu\n",
  1171. i, PVSCSI_MEM_SPACE_SIZE);
  1172. goto out_release_resources_and_disable;
  1173. }
  1174. pci_set_master(pdev);
  1175. /*
  1176. * Ask the device for max number of targets before deciding the
  1177. * default pvscsi_ring_pages value.
  1178. */
  1179. max_id = pvscsi_get_max_targets(adapter);
  1180. printk(KERN_INFO "vmw_pvscsi: max_id: %u\n", max_id);
  1181. if (pvscsi_ring_pages == 0)
  1182. /*
  1183. * Set the right default value. Up to 16 it is 8, above it is
  1184. * max.
  1185. */
  1186. pvscsi_ring_pages = (max_id > 16) ?
  1187. PVSCSI_SETUP_RINGS_MAX_NUM_PAGES :
  1188. PVSCSI_DEFAULT_NUM_PAGES_PER_RING;
  1189. printk(KERN_INFO
  1190. "vmw_pvscsi: setting ring_pages to %d\n",
  1191. pvscsi_ring_pages);
  1192. pvscsi_template.can_queue =
  1193. min(PVSCSI_MAX_NUM_PAGES_REQ_RING, pvscsi_ring_pages) *
  1194. PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
  1195. pvscsi_template.cmd_per_lun =
  1196. min(pvscsi_template.can_queue, pvscsi_cmd_per_lun);
  1197. host = scsi_host_alloc(&pvscsi_template, sizeof(struct pvscsi_adapter));
  1198. if (!host) {
  1199. printk(KERN_ERR "vmw_pvscsi: failed to allocate host\n");
  1200. goto out_release_resources_and_disable;
  1201. }
  1202. /*
  1203. * Let's use the real pvscsi_adapter struct here onwards.
  1204. */
  1205. adapter = shost_priv(host);
  1206. memset(adapter, 0, sizeof(*adapter));
  1207. adapter->dev = pdev;
  1208. adapter->host = host;
  1209. /*
  1210. * Copy back what we already have to the allocated adapter struct.
  1211. */
  1212. adapter->rev = adapter_temp.rev;
  1213. adapter->mmioBase = adapter_temp.mmioBase;
  1214. spin_lock_init(&adapter->hw_lock);
  1215. host->max_channel = 0;
  1216. host->max_lun = 1;
  1217. host->max_cmd_len = 16;
  1218. host->max_id = max_id;
  1219. pci_set_drvdata(pdev, host);
  1220. ll_adapter_reset(adapter);
  1221. adapter->use_msg = pvscsi_setup_msg_workqueue(adapter);
  1222. error = pvscsi_allocate_rings(adapter);
  1223. if (error) {
  1224. printk(KERN_ERR "vmw_pvscsi: unable to allocate ring memory\n");
  1225. goto out_release_resources;
  1226. }
  1227. /*
  1228. * From this point on we should reset the adapter if anything goes
  1229. * wrong.
  1230. */
  1231. pvscsi_setup_all_rings(adapter);
  1232. adapter->cmd_map = kcalloc(adapter->req_depth,
  1233. sizeof(struct pvscsi_ctx), GFP_KERNEL);
  1234. if (!adapter->cmd_map) {
  1235. printk(KERN_ERR "vmw_pvscsi: failed to allocate memory.\n");
  1236. error = -ENOMEM;
  1237. goto out_reset_adapter;
  1238. }
  1239. INIT_LIST_HEAD(&adapter->cmd_pool);
  1240. for (i = 0; i < adapter->req_depth; i++) {
  1241. struct pvscsi_ctx *ctx = adapter->cmd_map + i;
  1242. list_add(&ctx->list, &adapter->cmd_pool);
  1243. }
  1244. error = pvscsi_allocate_sg(adapter);
  1245. if (error) {
  1246. printk(KERN_ERR "vmw_pvscsi: unable to allocate s/g table\n");
  1247. goto out_reset_adapter;
  1248. }
  1249. if (pvscsi_disable_msix)
  1250. irq_flag &= ~PCI_IRQ_MSIX;
  1251. if (pvscsi_disable_msi)
  1252. irq_flag &= ~PCI_IRQ_MSI;
  1253. error = pci_alloc_irq_vectors(adapter->dev, 1, 1, irq_flag);
  1254. if (error < 0)
  1255. goto out_reset_adapter;
  1256. adapter->use_req_threshold = pvscsi_setup_req_threshold(adapter, true);
  1257. printk(KERN_DEBUG "vmw_pvscsi: driver-based request coalescing %sabled\n",
  1258. adapter->use_req_threshold ? "en" : "dis");
  1259. if (adapter->dev->msix_enabled || adapter->dev->msi_enabled) {
  1260. printk(KERN_INFO "vmw_pvscsi: using MSI%s\n",
  1261. adapter->dev->msix_enabled ? "-X" : "");
  1262. error = request_irq(pci_irq_vector(pdev, 0), pvscsi_isr,
  1263. 0, "vmw_pvscsi", adapter);
  1264. } else {
  1265. printk(KERN_INFO "vmw_pvscsi: using INTx\n");
  1266. error = request_irq(pci_irq_vector(pdev, 0), pvscsi_shared_isr,
  1267. IRQF_SHARED, "vmw_pvscsi", adapter);
  1268. }
  1269. if (error) {
  1270. printk(KERN_ERR
  1271. "vmw_pvscsi: unable to request IRQ: %d\n", error);
  1272. goto out_reset_adapter;
  1273. }
  1274. error = scsi_add_host(host, &pdev->dev);
  1275. if (error) {
  1276. printk(KERN_ERR
  1277. "vmw_pvscsi: scsi_add_host failed: %d\n", error);
  1278. goto out_reset_adapter;
  1279. }
  1280. dev_info(&pdev->dev, "VMware PVSCSI rev %d host #%u\n",
  1281. adapter->rev, host->host_no);
  1282. pvscsi_unmask_intr(adapter);
  1283. scsi_scan_host(host);
  1284. return 0;
  1285. out_reset_adapter:
  1286. ll_adapter_reset(adapter);
  1287. out_release_resources:
  1288. pvscsi_shutdown_intr(adapter);
  1289. pvscsi_release_resources(adapter);
  1290. scsi_host_put(host);
  1291. out_disable_device:
  1292. pci_disable_device(pdev);
  1293. return error;
  1294. out_release_resources_and_disable:
  1295. pvscsi_shutdown_intr(adapter);
  1296. pvscsi_release_resources(adapter);
  1297. goto out_disable_device;
  1298. }
  1299. static void __pvscsi_shutdown(struct pvscsi_adapter *adapter)
  1300. {
  1301. pvscsi_mask_intr(adapter);
  1302. if (adapter->workqueue)
  1303. flush_workqueue(adapter->workqueue);
  1304. pvscsi_shutdown_intr(adapter);
  1305. pvscsi_process_request_ring(adapter);
  1306. pvscsi_process_completion_ring(adapter);
  1307. ll_adapter_reset(adapter);
  1308. }
  1309. static void pvscsi_shutdown(struct pci_dev *dev)
  1310. {
  1311. struct Scsi_Host *host = pci_get_drvdata(dev);
  1312. struct pvscsi_adapter *adapter = shost_priv(host);
  1313. __pvscsi_shutdown(adapter);
  1314. }
  1315. static void pvscsi_remove(struct pci_dev *pdev)
  1316. {
  1317. struct Scsi_Host *host = pci_get_drvdata(pdev);
  1318. struct pvscsi_adapter *adapter = shost_priv(host);
  1319. scsi_remove_host(host);
  1320. __pvscsi_shutdown(adapter);
  1321. pvscsi_release_resources(adapter);
  1322. scsi_host_put(host);
  1323. pci_disable_device(pdev);
  1324. }
  1325. static struct pci_driver pvscsi_pci_driver = {
  1326. .name = "vmw_pvscsi",
  1327. .id_table = pvscsi_pci_tbl,
  1328. .probe = pvscsi_probe,
  1329. .remove = pvscsi_remove,
  1330. .shutdown = pvscsi_shutdown,
  1331. };
  1332. static int __init pvscsi_init(void)
  1333. {
  1334. pr_info("%s - version %s\n",
  1335. PVSCSI_LINUX_DRIVER_DESC, PVSCSI_DRIVER_VERSION_STRING);
  1336. return pci_register_driver(&pvscsi_pci_driver);
  1337. }
  1338. static void __exit pvscsi_exit(void)
  1339. {
  1340. pci_unregister_driver(&pvscsi_pci_driver);
  1341. }
  1342. module_init(pvscsi_init);
  1343. module_exit(pvscsi_exit);