ntb_transport.c 61 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498
  1. /*
  2. * This file is provided under a dual BSD/GPLv2 license. When using or
  3. * redistributing this file, you may do so under either license.
  4. *
  5. * GPL LICENSE SUMMARY
  6. *
  7. * Copyright(c) 2012 Intel Corporation. All rights reserved.
  8. * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
  9. *
  10. * This program is free software; you can redistribute it and/or modify
  11. * it under the terms of version 2 of the GNU General Public License as
  12. * published by the Free Software Foundation.
  13. *
  14. * BSD LICENSE
  15. *
  16. * Copyright(c) 2012 Intel Corporation. All rights reserved.
  17. * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
  18. *
  19. * Redistribution and use in source and binary forms, with or without
  20. * modification, are permitted provided that the following conditions
  21. * are met:
  22. *
  23. * * Redistributions of source code must retain the above copyright
  24. * notice, this list of conditions and the following disclaimer.
  25. * * Redistributions in binary form must reproduce the above copy
  26. * notice, this list of conditions and the following disclaimer in
  27. * the documentation and/or other materials provided with the
  28. * distribution.
  29. * * Neither the name of Intel Corporation nor the names of its
  30. * contributors may be used to endorse or promote products derived
  31. * from this software without specific prior written permission.
  32. *
  33. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  34. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  35. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  36. * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  37. * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  38. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  39. * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  40. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  41. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  42. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  43. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  44. *
  45. * PCIe NTB Transport Linux driver
  46. *
  47. * Contact Information:
  48. * Jon Mason <jon.mason@intel.com>
  49. */
  50. #include <linux/debugfs.h>
  51. #include <linux/delay.h>
  52. #include <linux/dmaengine.h>
  53. #include <linux/dma-mapping.h>
  54. #include <linux/errno.h>
  55. #include <linux/export.h>
  56. #include <linux/interrupt.h>
  57. #include <linux/module.h>
  58. #include <linux/pci.h>
  59. #include <linux/slab.h>
  60. #include <linux/types.h>
  61. #include <linux/uaccess.h>
  62. #include "linux/ntb.h"
  63. #include "linux/ntb_transport.h"
  64. #define NTB_TRANSPORT_VERSION 4
  65. #define NTB_TRANSPORT_VER "4"
  66. #define NTB_TRANSPORT_NAME "ntb_transport"
  67. #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB"
  68. #define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2)
  69. MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
  70. MODULE_VERSION(NTB_TRANSPORT_VER);
  71. MODULE_LICENSE("Dual BSD/GPL");
  72. MODULE_AUTHOR("Intel Corporation");
  73. static unsigned long max_mw_size;
  74. module_param(max_mw_size, ulong, 0644);
  75. MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
  76. static unsigned int transport_mtu = 0x10000;
  77. module_param(transport_mtu, uint, 0644);
  78. MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets");
  79. static unsigned char max_num_clients;
  80. module_param(max_num_clients, byte, 0644);
  81. MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");
  82. static unsigned int copy_bytes = 1024;
  83. module_param(copy_bytes, uint, 0644);
  84. MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");
  85. static bool use_dma;
  86. module_param(use_dma, bool, 0644);
  87. MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy");
  88. static bool use_msi;
  89. #ifdef CONFIG_NTB_MSI
  90. module_param(use_msi, bool, 0644);
  91. MODULE_PARM_DESC(use_msi, "Use MSI interrupts instead of doorbells");
  92. #endif
  93. static struct dentry *nt_debugfs_dir;
  94. /* Only two-ports NTB devices are supported */
  95. #define PIDX NTB_DEF_PEER_IDX
  96. struct ntb_queue_entry {
  97. /* ntb_queue list reference */
  98. struct list_head entry;
  99. /* pointers to data to be transferred */
  100. void *cb_data;
  101. void *buf;
  102. unsigned int len;
  103. unsigned int flags;
  104. int retries;
  105. int errors;
  106. unsigned int tx_index;
  107. unsigned int rx_index;
  108. struct ntb_transport_qp *qp;
  109. union {
  110. struct ntb_payload_header __iomem *tx_hdr;
  111. struct ntb_payload_header *rx_hdr;
  112. };
  113. };
  114. struct ntb_rx_info {
  115. unsigned int entry;
  116. };
  117. struct ntb_transport_qp {
  118. struct ntb_transport_ctx *transport;
  119. struct ntb_dev *ndev;
  120. void *cb_data;
  121. struct dma_chan *tx_dma_chan;
  122. struct dma_chan *rx_dma_chan;
  123. bool client_ready;
  124. bool link_is_up;
  125. bool active;
  126. u8 qp_num; /* Only 64 QP's are allowed. 0-63 */
  127. u64 qp_bit;
  128. struct ntb_rx_info __iomem *rx_info;
  129. struct ntb_rx_info *remote_rx_info;
  130. void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
  131. void *data, int len);
  132. struct list_head tx_free_q;
  133. spinlock_t ntb_tx_free_q_lock;
  134. void __iomem *tx_mw;
  135. phys_addr_t tx_mw_phys;
  136. size_t tx_mw_size;
  137. dma_addr_t tx_mw_dma_addr;
  138. unsigned int tx_index;
  139. unsigned int tx_max_entry;
  140. unsigned int tx_max_frame;
  141. void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
  142. void *data, int len);
  143. struct list_head rx_post_q;
  144. struct list_head rx_pend_q;
  145. struct list_head rx_free_q;
  146. /* ntb_rx_q_lock: synchronize access to rx_XXXX_q */
  147. spinlock_t ntb_rx_q_lock;
  148. void *rx_buff;
  149. unsigned int rx_index;
  150. unsigned int rx_max_entry;
  151. unsigned int rx_max_frame;
  152. unsigned int rx_alloc_entry;
  153. dma_cookie_t last_cookie;
  154. struct tasklet_struct rxc_db_work;
  155. void (*event_handler)(void *data, int status);
  156. struct delayed_work link_work;
  157. struct work_struct link_cleanup;
  158. struct dentry *debugfs_dir;
  159. struct dentry *debugfs_stats;
  160. /* Stats */
  161. u64 rx_bytes;
  162. u64 rx_pkts;
  163. u64 rx_ring_empty;
  164. u64 rx_err_no_buf;
  165. u64 rx_err_oflow;
  166. u64 rx_err_ver;
  167. u64 rx_memcpy;
  168. u64 rx_async;
  169. u64 tx_bytes;
  170. u64 tx_pkts;
  171. u64 tx_ring_full;
  172. u64 tx_err_no_buf;
  173. u64 tx_memcpy;
  174. u64 tx_async;
  175. bool use_msi;
  176. int msi_irq;
  177. struct ntb_msi_desc msi_desc;
  178. struct ntb_msi_desc peer_msi_desc;
  179. };
  180. struct ntb_transport_mw {
  181. phys_addr_t phys_addr;
  182. resource_size_t phys_size;
  183. void __iomem *vbase;
  184. size_t xlat_size;
  185. size_t buff_size;
  186. size_t alloc_size;
  187. void *alloc_addr;
  188. void *virt_addr;
  189. dma_addr_t dma_addr;
  190. };
  191. struct ntb_transport_client_dev {
  192. struct list_head entry;
  193. struct ntb_transport_ctx *nt;
  194. struct device dev;
  195. };
  196. struct ntb_transport_ctx {
  197. struct list_head entry;
  198. struct list_head client_devs;
  199. struct ntb_dev *ndev;
  200. struct ntb_transport_mw *mw_vec;
  201. struct ntb_transport_qp *qp_vec;
  202. unsigned int mw_count;
  203. unsigned int qp_count;
  204. u64 qp_bitmap;
  205. u64 qp_bitmap_free;
  206. bool use_msi;
  207. unsigned int msi_spad_offset;
  208. u64 msi_db_mask;
  209. bool link_is_up;
  210. struct delayed_work link_work;
  211. struct work_struct link_cleanup;
  212. struct dentry *debugfs_node_dir;
  213. };
  214. enum {
  215. DESC_DONE_FLAG = BIT(0),
  216. LINK_DOWN_FLAG = BIT(1),
  217. };
  218. struct ntb_payload_header {
  219. unsigned int ver;
  220. unsigned int len;
  221. unsigned int flags;
  222. };
  223. enum {
  224. VERSION = 0,
  225. QP_LINKS,
  226. NUM_QPS,
  227. NUM_MWS,
  228. MW0_SZ_HIGH,
  229. MW0_SZ_LOW,
  230. };
  231. #define dev_client_dev(__dev) \
  232. container_of((__dev), struct ntb_transport_client_dev, dev)
  233. #define drv_client(__drv) \
  234. container_of((__drv), struct ntb_transport_client, driver)
  235. #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count)
  236. #define NTB_QP_DEF_NUM_ENTRIES 100
  237. #define NTB_LINK_DOWN_TIMEOUT 10
  238. static void ntb_transport_rxc_db(unsigned long data);
  239. static const struct ntb_ctx_ops ntb_transport_ops;
  240. static struct ntb_client ntb_transport_client;
  241. static int ntb_async_tx_submit(struct ntb_transport_qp *qp,
  242. struct ntb_queue_entry *entry);
  243. static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset);
  244. static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset);
  245. static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset);
  246. static int ntb_transport_bus_match(struct device *dev,
  247. struct device_driver *drv)
  248. {
  249. return !strncmp(dev_name(dev), drv->name, strlen(drv->name));
  250. }
  251. static int ntb_transport_bus_probe(struct device *dev)
  252. {
  253. const struct ntb_transport_client *client;
  254. int rc;
  255. get_device(dev);
  256. client = drv_client(dev->driver);
  257. rc = client->probe(dev);
  258. if (rc)
  259. put_device(dev);
  260. return rc;
  261. }
  262. static int ntb_transport_bus_remove(struct device *dev)
  263. {
  264. const struct ntb_transport_client *client;
  265. client = drv_client(dev->driver);
  266. client->remove(dev);
  267. put_device(dev);
  268. return 0;
  269. }
  270. static struct bus_type ntb_transport_bus = {
  271. .name = "ntb_transport",
  272. .match = ntb_transport_bus_match,
  273. .probe = ntb_transport_bus_probe,
  274. .remove = ntb_transport_bus_remove,
  275. };
  276. static LIST_HEAD(ntb_transport_list);
  277. static int ntb_bus_init(struct ntb_transport_ctx *nt)
  278. {
  279. list_add_tail(&nt->entry, &ntb_transport_list);
  280. return 0;
  281. }
  282. static void ntb_bus_remove(struct ntb_transport_ctx *nt)
  283. {
  284. struct ntb_transport_client_dev *client_dev, *cd;
  285. list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {
  286. dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n",
  287. dev_name(&client_dev->dev));
  288. list_del(&client_dev->entry);
  289. device_unregister(&client_dev->dev);
  290. }
  291. list_del(&nt->entry);
  292. }
  293. static void ntb_transport_client_release(struct device *dev)
  294. {
  295. struct ntb_transport_client_dev *client_dev;
  296. client_dev = dev_client_dev(dev);
  297. kfree(client_dev);
  298. }
  299. /**
  300. * ntb_transport_unregister_client_dev - Unregister NTB client device
  301. * @device_name: Name of NTB client device
  302. *
  303. * Unregister an NTB client device with the NTB transport layer
  304. */
  305. void ntb_transport_unregister_client_dev(char *device_name)
  306. {
  307. struct ntb_transport_client_dev *client, *cd;
  308. struct ntb_transport_ctx *nt;
  309. list_for_each_entry(nt, &ntb_transport_list, entry)
  310. list_for_each_entry_safe(client, cd, &nt->client_devs, entry)
  311. if (!strncmp(dev_name(&client->dev), device_name,
  312. strlen(device_name))) {
  313. list_del(&client->entry);
  314. device_unregister(&client->dev);
  315. }
  316. }
  317. EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev);
  318. /**
  319. * ntb_transport_register_client_dev - Register NTB client device
  320. * @device_name: Name of NTB client device
  321. *
  322. * Register an NTB client device with the NTB transport layer
  323. */
  324. int ntb_transport_register_client_dev(char *device_name)
  325. {
  326. struct ntb_transport_client_dev *client_dev;
  327. struct ntb_transport_ctx *nt;
  328. int node;
  329. int rc, i = 0;
  330. if (list_empty(&ntb_transport_list))
  331. return -ENODEV;
  332. list_for_each_entry(nt, &ntb_transport_list, entry) {
  333. struct device *dev;
  334. node = dev_to_node(&nt->ndev->dev);
  335. client_dev = kzalloc_node(sizeof(*client_dev),
  336. GFP_KERNEL, node);
  337. if (!client_dev) {
  338. rc = -ENOMEM;
  339. goto err;
  340. }
  341. dev = &client_dev->dev;
  342. /* setup and register client devices */
  343. dev_set_name(dev, "%s%d", device_name, i);
  344. dev->bus = &ntb_transport_bus;
  345. dev->release = ntb_transport_client_release;
  346. dev->parent = &nt->ndev->dev;
  347. rc = device_register(dev);
  348. if (rc) {
  349. kfree(client_dev);
  350. goto err;
  351. }
  352. list_add_tail(&client_dev->entry, &nt->client_devs);
  353. i++;
  354. }
  355. return 0;
  356. err:
  357. ntb_transport_unregister_client_dev(device_name);
  358. return rc;
  359. }
  360. EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev);
  361. /**
  362. * ntb_transport_register_client - Register NTB client driver
  363. * @drv: NTB client driver to be registered
  364. *
  365. * Register an NTB client driver with the NTB transport layer
  366. *
  367. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  368. */
  369. int ntb_transport_register_client(struct ntb_transport_client *drv)
  370. {
  371. drv->driver.bus = &ntb_transport_bus;
  372. if (list_empty(&ntb_transport_list))
  373. return -ENODEV;
  374. return driver_register(&drv->driver);
  375. }
  376. EXPORT_SYMBOL_GPL(ntb_transport_register_client);
  377. /**
  378. * ntb_transport_unregister_client - Unregister NTB client driver
  379. * @drv: NTB client driver to be unregistered
  380. *
  381. * Unregister an NTB client driver with the NTB transport layer
  382. *
  383. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  384. */
  385. void ntb_transport_unregister_client(struct ntb_transport_client *drv)
  386. {
  387. driver_unregister(&drv->driver);
  388. }
  389. EXPORT_SYMBOL_GPL(ntb_transport_unregister_client);
  390. static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
  391. loff_t *offp)
  392. {
  393. struct ntb_transport_qp *qp;
  394. char *buf;
  395. ssize_t ret, out_offset, out_count;
  396. qp = filp->private_data;
  397. if (!qp || !qp->link_is_up)
  398. return 0;
  399. out_count = 1000;
  400. buf = kmalloc(out_count, GFP_KERNEL);
  401. if (!buf)
  402. return -ENOMEM;
  403. out_offset = 0;
  404. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  405. "\nNTB QP stats:\n\n");
  406. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  407. "rx_bytes - \t%llu\n", qp->rx_bytes);
  408. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  409. "rx_pkts - \t%llu\n", qp->rx_pkts);
  410. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  411. "rx_memcpy - \t%llu\n", qp->rx_memcpy);
  412. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  413. "rx_async - \t%llu\n", qp->rx_async);
  414. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  415. "rx_ring_empty - %llu\n", qp->rx_ring_empty);
  416. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  417. "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
  418. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  419. "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);
  420. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  421. "rx_err_ver - \t%llu\n", qp->rx_err_ver);
  422. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  423. "rx_buff - \t0x%p\n", qp->rx_buff);
  424. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  425. "rx_index - \t%u\n", qp->rx_index);
  426. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  427. "rx_max_entry - \t%u\n", qp->rx_max_entry);
  428. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  429. "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry);
  430. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  431. "tx_bytes - \t%llu\n", qp->tx_bytes);
  432. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  433. "tx_pkts - \t%llu\n", qp->tx_pkts);
  434. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  435. "tx_memcpy - \t%llu\n", qp->tx_memcpy);
  436. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  437. "tx_async - \t%llu\n", qp->tx_async);
  438. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  439. "tx_ring_full - \t%llu\n", qp->tx_ring_full);
  440. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  441. "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
  442. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  443. "tx_mw - \t0x%p\n", qp->tx_mw);
  444. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  445. "tx_index (H) - \t%u\n", qp->tx_index);
  446. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  447. "RRI (T) - \t%u\n",
  448. qp->remote_rx_info->entry);
  449. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  450. "tx_max_entry - \t%u\n", qp->tx_max_entry);
  451. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  452. "free tx - \t%u\n",
  453. ntb_transport_tx_free_entry(qp));
  454. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  455. "\n");
  456. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  457. "Using TX DMA - \t%s\n",
  458. qp->tx_dma_chan ? "Yes" : "No");
  459. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  460. "Using RX DMA - \t%s\n",
  461. qp->rx_dma_chan ? "Yes" : "No");
  462. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  463. "QP Link - \t%s\n",
  464. qp->link_is_up ? "Up" : "Down");
  465. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  466. "\n");
  467. if (out_offset > out_count)
  468. out_offset = out_count;
  469. ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
  470. kfree(buf);
  471. return ret;
  472. }
  473. static const struct file_operations ntb_qp_debugfs_stats = {
  474. .owner = THIS_MODULE,
  475. .open = simple_open,
  476. .read = debugfs_read,
  477. };
  478. static void ntb_list_add(spinlock_t *lock, struct list_head *entry,
  479. struct list_head *list)
  480. {
  481. unsigned long flags;
  482. spin_lock_irqsave(lock, flags);
  483. list_add_tail(entry, list);
  484. spin_unlock_irqrestore(lock, flags);
  485. }
  486. static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock,
  487. struct list_head *list)
  488. {
  489. struct ntb_queue_entry *entry;
  490. unsigned long flags;
  491. spin_lock_irqsave(lock, flags);
  492. if (list_empty(list)) {
  493. entry = NULL;
  494. goto out;
  495. }
  496. entry = list_first_entry(list, struct ntb_queue_entry, entry);
  497. list_del(&entry->entry);
  498. out:
  499. spin_unlock_irqrestore(lock, flags);
  500. return entry;
  501. }
  502. static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock,
  503. struct list_head *list,
  504. struct list_head *to_list)
  505. {
  506. struct ntb_queue_entry *entry;
  507. unsigned long flags;
  508. spin_lock_irqsave(lock, flags);
  509. if (list_empty(list)) {
  510. entry = NULL;
  511. } else {
  512. entry = list_first_entry(list, struct ntb_queue_entry, entry);
  513. list_move_tail(&entry->entry, to_list);
  514. }
  515. spin_unlock_irqrestore(lock, flags);
  516. return entry;
  517. }
  518. static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
  519. unsigned int qp_num)
  520. {
  521. struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
  522. struct ntb_transport_mw *mw;
  523. struct ntb_dev *ndev = nt->ndev;
  524. struct ntb_queue_entry *entry;
  525. unsigned int rx_size, num_qps_mw;
  526. unsigned int mw_num, mw_count, qp_count;
  527. unsigned int i;
  528. int node;
  529. mw_count = nt->mw_count;
  530. qp_count = nt->qp_count;
  531. mw_num = QP_TO_MW(nt, qp_num);
  532. mw = &nt->mw_vec[mw_num];
  533. if (!mw->virt_addr)
  534. return -ENOMEM;
  535. if (mw_num < qp_count % mw_count)
  536. num_qps_mw = qp_count / mw_count + 1;
  537. else
  538. num_qps_mw = qp_count / mw_count;
  539. rx_size = (unsigned int)mw->xlat_size / num_qps_mw;
  540. qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count);
  541. rx_size -= sizeof(struct ntb_rx_info);
  542. qp->remote_rx_info = qp->rx_buff + rx_size;
  543. /* Due to housekeeping, there must be atleast 2 buffs */
  544. qp->rx_max_frame = min(transport_mtu, rx_size / 2);
  545. qp->rx_max_entry = rx_size / qp->rx_max_frame;
  546. qp->rx_index = 0;
  547. /*
  548. * Checking to see if we have more entries than the default.
  549. * We should add additional entries if that is the case so we
  550. * can be in sync with the transport frames.
  551. */
  552. node = dev_to_node(&ndev->dev);
  553. for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) {
  554. entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
  555. if (!entry)
  556. return -ENOMEM;
  557. entry->qp = qp;
  558. ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
  559. &qp->rx_free_q);
  560. qp->rx_alloc_entry++;
  561. }
  562. qp->remote_rx_info->entry = qp->rx_max_entry - 1;
  563. /* setup the hdr offsets with 0's */
  564. for (i = 0; i < qp->rx_max_entry; i++) {
  565. void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) -
  566. sizeof(struct ntb_payload_header));
  567. memset(offset, 0, sizeof(struct ntb_payload_header));
  568. }
  569. qp->rx_pkts = 0;
  570. qp->tx_pkts = 0;
  571. qp->tx_index = 0;
  572. return 0;
  573. }
  574. static irqreturn_t ntb_transport_isr(int irq, void *dev)
  575. {
  576. struct ntb_transport_qp *qp = dev;
  577. tasklet_schedule(&qp->rxc_db_work);
  578. return IRQ_HANDLED;
  579. }
  580. static void ntb_transport_setup_qp_peer_msi(struct ntb_transport_ctx *nt,
  581. unsigned int qp_num)
  582. {
  583. struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
  584. int spad = qp_num * 2 + nt->msi_spad_offset;
  585. if (!nt->use_msi)
  586. return;
  587. if (spad >= ntb_spad_count(nt->ndev))
  588. return;
  589. qp->peer_msi_desc.addr_offset =
  590. ntb_peer_spad_read(qp->ndev, PIDX, spad);
  591. qp->peer_msi_desc.data =
  592. ntb_peer_spad_read(qp->ndev, PIDX, spad + 1);
  593. dev_dbg(&qp->ndev->pdev->dev, "QP%d Peer MSI addr=%x data=%x\n",
  594. qp_num, qp->peer_msi_desc.addr_offset, qp->peer_msi_desc.data);
  595. if (qp->peer_msi_desc.addr_offset) {
  596. qp->use_msi = true;
  597. dev_info(&qp->ndev->pdev->dev,
  598. "Using MSI interrupts for QP%d\n", qp_num);
  599. }
  600. }
  601. static void ntb_transport_setup_qp_msi(struct ntb_transport_ctx *nt,
  602. unsigned int qp_num)
  603. {
  604. struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
  605. int spad = qp_num * 2 + nt->msi_spad_offset;
  606. int rc;
  607. if (!nt->use_msi)
  608. return;
  609. if (spad >= ntb_spad_count(nt->ndev)) {
  610. dev_warn_once(&qp->ndev->pdev->dev,
  611. "Not enough SPADS to use MSI interrupts\n");
  612. return;
  613. }
  614. ntb_spad_write(qp->ndev, spad, 0);
  615. ntb_spad_write(qp->ndev, spad + 1, 0);
  616. if (!qp->msi_irq) {
  617. qp->msi_irq = ntbm_msi_request_irq(qp->ndev, ntb_transport_isr,
  618. KBUILD_MODNAME, qp,
  619. &qp->msi_desc);
  620. if (qp->msi_irq < 0) {
  621. dev_warn(&qp->ndev->pdev->dev,
  622. "Unable to allocate MSI interrupt for qp%d\n",
  623. qp_num);
  624. return;
  625. }
  626. }
  627. rc = ntb_spad_write(qp->ndev, spad, qp->msi_desc.addr_offset);
  628. if (rc)
  629. goto err_free_interrupt;
  630. rc = ntb_spad_write(qp->ndev, spad + 1, qp->msi_desc.data);
  631. if (rc)
  632. goto err_free_interrupt;
  633. dev_dbg(&qp->ndev->pdev->dev, "QP%d MSI %d addr=%x data=%x\n",
  634. qp_num, qp->msi_irq, qp->msi_desc.addr_offset,
  635. qp->msi_desc.data);
  636. return;
  637. err_free_interrupt:
  638. devm_free_irq(&nt->ndev->dev, qp->msi_irq, qp);
  639. }
  640. static void ntb_transport_msi_peer_desc_changed(struct ntb_transport_ctx *nt)
  641. {
  642. int i;
  643. dev_dbg(&nt->ndev->pdev->dev, "Peer MSI descriptors changed");
  644. for (i = 0; i < nt->qp_count; i++)
  645. ntb_transport_setup_qp_peer_msi(nt, i);
  646. }
  647. static void ntb_transport_msi_desc_changed(void *data)
  648. {
  649. struct ntb_transport_ctx *nt = data;
  650. int i;
  651. dev_dbg(&nt->ndev->pdev->dev, "MSI descriptors changed");
  652. for (i = 0; i < nt->qp_count; i++)
  653. ntb_transport_setup_qp_msi(nt, i);
  654. ntb_peer_db_set(nt->ndev, nt->msi_db_mask);
  655. }
  656. static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
  657. {
  658. struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
  659. struct pci_dev *pdev = nt->ndev->pdev;
  660. if (!mw->virt_addr)
  661. return;
  662. ntb_mw_clear_trans(nt->ndev, PIDX, num_mw);
  663. dma_free_coherent(&pdev->dev, mw->alloc_size,
  664. mw->alloc_addr, mw->dma_addr);
  665. mw->xlat_size = 0;
  666. mw->buff_size = 0;
  667. mw->alloc_size = 0;
  668. mw->alloc_addr = NULL;
  669. mw->virt_addr = NULL;
  670. }
  671. static int ntb_alloc_mw_buffer(struct ntb_transport_mw *mw,
  672. struct device *dma_dev, size_t align)
  673. {
  674. dma_addr_t dma_addr;
  675. void *alloc_addr, *virt_addr;
  676. int rc;
  677. alloc_addr = dma_alloc_coherent(dma_dev, mw->alloc_size,
  678. &dma_addr, GFP_KERNEL);
  679. if (!alloc_addr) {
  680. dev_err(dma_dev, "Unable to alloc MW buff of size %zu\n",
  681. mw->alloc_size);
  682. return -ENOMEM;
  683. }
  684. virt_addr = alloc_addr;
  685. /*
  686. * we must ensure that the memory address allocated is BAR size
  687. * aligned in order for the XLAT register to take the value. This
  688. * is a requirement of the hardware. It is recommended to setup CMA
  689. * for BAR sizes equal or greater than 4MB.
  690. */
  691. if (!IS_ALIGNED(dma_addr, align)) {
  692. if (mw->alloc_size > mw->buff_size) {
  693. virt_addr = PTR_ALIGN(alloc_addr, align);
  694. dma_addr = ALIGN(dma_addr, align);
  695. } else {
  696. rc = -ENOMEM;
  697. goto err;
  698. }
  699. }
  700. mw->alloc_addr = alloc_addr;
  701. mw->virt_addr = virt_addr;
  702. mw->dma_addr = dma_addr;
  703. return 0;
  704. err:
  705. dma_free_coherent(dma_dev, mw->alloc_size, alloc_addr, dma_addr);
  706. return rc;
  707. }
  708. static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,
  709. resource_size_t size)
  710. {
  711. struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
  712. struct pci_dev *pdev = nt->ndev->pdev;
  713. size_t xlat_size, buff_size;
  714. resource_size_t xlat_align;
  715. resource_size_t xlat_align_size;
  716. int rc;
  717. if (!size)
  718. return -EINVAL;
  719. rc = ntb_mw_get_align(nt->ndev, PIDX, num_mw, &xlat_align,
  720. &xlat_align_size, NULL);
  721. if (rc)
  722. return rc;
  723. xlat_size = round_up(size, xlat_align_size);
  724. buff_size = round_up(size, xlat_align);
  725. /* No need to re-setup */
  726. if (mw->xlat_size == xlat_size)
  727. return 0;
  728. if (mw->buff_size)
  729. ntb_free_mw(nt, num_mw);
  730. /* Alloc memory for receiving data. Must be aligned */
  731. mw->xlat_size = xlat_size;
  732. mw->buff_size = buff_size;
  733. mw->alloc_size = buff_size;
  734. rc = ntb_alloc_mw_buffer(mw, &pdev->dev, xlat_align);
  735. if (rc) {
  736. mw->alloc_size *= 2;
  737. rc = ntb_alloc_mw_buffer(mw, &pdev->dev, xlat_align);
  738. if (rc) {
  739. dev_err(&pdev->dev,
  740. "Unable to alloc aligned MW buff\n");
  741. mw->xlat_size = 0;
  742. mw->buff_size = 0;
  743. mw->alloc_size = 0;
  744. return rc;
  745. }
  746. }
  747. /* Notify HW the memory location of the receive buffer */
  748. rc = ntb_mw_set_trans(nt->ndev, PIDX, num_mw, mw->dma_addr,
  749. mw->xlat_size);
  750. if (rc) {
  751. dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw);
  752. ntb_free_mw(nt, num_mw);
  753. return -EIO;
  754. }
  755. return 0;
  756. }
  757. static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)
  758. {
  759. qp->link_is_up = false;
  760. qp->active = false;
  761. qp->tx_index = 0;
  762. qp->rx_index = 0;
  763. qp->rx_bytes = 0;
  764. qp->rx_pkts = 0;
  765. qp->rx_ring_empty = 0;
  766. qp->rx_err_no_buf = 0;
  767. qp->rx_err_oflow = 0;
  768. qp->rx_err_ver = 0;
  769. qp->rx_memcpy = 0;
  770. qp->rx_async = 0;
  771. qp->tx_bytes = 0;
  772. qp->tx_pkts = 0;
  773. qp->tx_ring_full = 0;
  774. qp->tx_err_no_buf = 0;
  775. qp->tx_memcpy = 0;
  776. qp->tx_async = 0;
  777. }
  778. static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
  779. {
  780. struct ntb_transport_ctx *nt = qp->transport;
  781. struct pci_dev *pdev = nt->ndev->pdev;
  782. dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num);
  783. cancel_delayed_work_sync(&qp->link_work);
  784. ntb_qp_link_down_reset(qp);
  785. if (qp->event_handler)
  786. qp->event_handler(qp->cb_data, qp->link_is_up);
  787. }
  788. static void ntb_qp_link_cleanup_work(struct work_struct *work)
  789. {
  790. struct ntb_transport_qp *qp = container_of(work,
  791. struct ntb_transport_qp,
  792. link_cleanup);
  793. struct ntb_transport_ctx *nt = qp->transport;
  794. ntb_qp_link_cleanup(qp);
  795. if (nt->link_is_up)
  796. schedule_delayed_work(&qp->link_work,
  797. msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
  798. }
  799. static void ntb_qp_link_down(struct ntb_transport_qp *qp)
  800. {
  801. schedule_work(&qp->link_cleanup);
  802. }
  803. static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
  804. {
  805. struct ntb_transport_qp *qp;
  806. u64 qp_bitmap_alloc;
  807. unsigned int i, count;
  808. qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;
  809. /* Pass along the info to any clients */
  810. for (i = 0; i < nt->qp_count; i++)
  811. if (qp_bitmap_alloc & BIT_ULL(i)) {
  812. qp = &nt->qp_vec[i];
  813. ntb_qp_link_cleanup(qp);
  814. cancel_work_sync(&qp->link_cleanup);
  815. cancel_delayed_work_sync(&qp->link_work);
  816. }
  817. if (!nt->link_is_up)
  818. cancel_delayed_work_sync(&nt->link_work);
  819. for (i = 0; i < nt->mw_count; i++)
  820. ntb_free_mw(nt, i);
  821. /* The scratchpad registers keep the values if the remote side
  822. * goes down, blast them now to give them a sane value the next
  823. * time they are accessed
  824. */
  825. count = ntb_spad_count(nt->ndev);
  826. for (i = 0; i < count; i++)
  827. ntb_spad_write(nt->ndev, i, 0);
  828. }
  829. static void ntb_transport_link_cleanup_work(struct work_struct *work)
  830. {
  831. struct ntb_transport_ctx *nt =
  832. container_of(work, struct ntb_transport_ctx, link_cleanup);
  833. ntb_transport_link_cleanup(nt);
  834. }
  835. static void ntb_transport_event_callback(void *data)
  836. {
  837. struct ntb_transport_ctx *nt = data;
  838. if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)
  839. schedule_delayed_work(&nt->link_work, 0);
  840. else
  841. schedule_work(&nt->link_cleanup);
  842. }
  843. static void ntb_transport_link_work(struct work_struct *work)
  844. {
  845. struct ntb_transport_ctx *nt =
  846. container_of(work, struct ntb_transport_ctx, link_work.work);
  847. struct ntb_dev *ndev = nt->ndev;
  848. struct pci_dev *pdev = ndev->pdev;
  849. resource_size_t size;
  850. u32 val;
  851. int rc = 0, i, spad;
  852. /* send the local info, in the opposite order of the way we read it */
  853. if (nt->use_msi) {
  854. rc = ntb_msi_setup_mws(ndev);
  855. if (rc) {
  856. dev_warn(&pdev->dev,
  857. "Failed to register MSI memory window: %d\n",
  858. rc);
  859. nt->use_msi = false;
  860. }
  861. }
  862. for (i = 0; i < nt->qp_count; i++)
  863. ntb_transport_setup_qp_msi(nt, i);
  864. for (i = 0; i < nt->mw_count; i++) {
  865. size = nt->mw_vec[i].phys_size;
  866. if (max_mw_size && size > max_mw_size)
  867. size = max_mw_size;
  868. spad = MW0_SZ_HIGH + (i * 2);
  869. ntb_peer_spad_write(ndev, PIDX, spad, upper_32_bits(size));
  870. spad = MW0_SZ_LOW + (i * 2);
  871. ntb_peer_spad_write(ndev, PIDX, spad, lower_32_bits(size));
  872. }
  873. ntb_peer_spad_write(ndev, PIDX, NUM_MWS, nt->mw_count);
  874. ntb_peer_spad_write(ndev, PIDX, NUM_QPS, nt->qp_count);
  875. ntb_peer_spad_write(ndev, PIDX, VERSION, NTB_TRANSPORT_VERSION);
  876. /* Query the remote side for its info */
  877. val = ntb_spad_read(ndev, VERSION);
  878. dev_dbg(&pdev->dev, "Remote version = %d\n", val);
  879. if (val != NTB_TRANSPORT_VERSION)
  880. goto out;
  881. val = ntb_spad_read(ndev, NUM_QPS);
  882. dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val);
  883. if (val != nt->qp_count)
  884. goto out;
  885. val = ntb_spad_read(ndev, NUM_MWS);
  886. dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val);
  887. if (val != nt->mw_count)
  888. goto out;
  889. for (i = 0; i < nt->mw_count; i++) {
  890. u64 val64;
  891. val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2));
  892. val64 = (u64)val << 32;
  893. val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2));
  894. val64 |= val;
  895. dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64);
  896. rc = ntb_set_mw(nt, i, val64);
  897. if (rc)
  898. goto out1;
  899. }
  900. nt->link_is_up = true;
  901. for (i = 0; i < nt->qp_count; i++) {
  902. struct ntb_transport_qp *qp = &nt->qp_vec[i];
  903. ntb_transport_setup_qp_mw(nt, i);
  904. ntb_transport_setup_qp_peer_msi(nt, i);
  905. if (qp->client_ready)
  906. schedule_delayed_work(&qp->link_work, 0);
  907. }
  908. return;
  909. out1:
  910. for (i = 0; i < nt->mw_count; i++)
  911. ntb_free_mw(nt, i);
  912. /* if there's an actual failure, we should just bail */
  913. if (rc < 0)
  914. return;
  915. out:
  916. if (ntb_link_is_up(ndev, NULL, NULL) == 1)
  917. schedule_delayed_work(&nt->link_work,
  918. msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
  919. }
  920. static void ntb_qp_link_work(struct work_struct *work)
  921. {
  922. struct ntb_transport_qp *qp = container_of(work,
  923. struct ntb_transport_qp,
  924. link_work.work);
  925. struct pci_dev *pdev = qp->ndev->pdev;
  926. struct ntb_transport_ctx *nt = qp->transport;
  927. int val;
  928. WARN_ON(!nt->link_is_up);
  929. val = ntb_spad_read(nt->ndev, QP_LINKS);
  930. ntb_peer_spad_write(nt->ndev, PIDX, QP_LINKS, val | BIT(qp->qp_num));
  931. /* query remote spad for qp ready bits */
  932. dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val);
  933. /* See if the remote side is up */
  934. if (val & BIT(qp->qp_num)) {
  935. dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num);
  936. qp->link_is_up = true;
  937. qp->active = true;
  938. if (qp->event_handler)
  939. qp->event_handler(qp->cb_data, qp->link_is_up);
  940. if (qp->active)
  941. tasklet_schedule(&qp->rxc_db_work);
  942. } else if (nt->link_is_up)
  943. schedule_delayed_work(&qp->link_work,
  944. msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
  945. }
  946. static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,
  947. unsigned int qp_num)
  948. {
  949. struct ntb_transport_qp *qp;
  950. phys_addr_t mw_base;
  951. resource_size_t mw_size;
  952. unsigned int num_qps_mw, tx_size;
  953. unsigned int mw_num, mw_count, qp_count;
  954. u64 qp_offset;
  955. mw_count = nt->mw_count;
  956. qp_count = nt->qp_count;
  957. mw_num = QP_TO_MW(nt, qp_num);
  958. qp = &nt->qp_vec[qp_num];
  959. qp->qp_num = qp_num;
  960. qp->transport = nt;
  961. qp->ndev = nt->ndev;
  962. qp->client_ready = false;
  963. qp->event_handler = NULL;
  964. ntb_qp_link_down_reset(qp);
  965. if (mw_num < qp_count % mw_count)
  966. num_qps_mw = qp_count / mw_count + 1;
  967. else
  968. num_qps_mw = qp_count / mw_count;
  969. mw_base = nt->mw_vec[mw_num].phys_addr;
  970. mw_size = nt->mw_vec[mw_num].phys_size;
  971. if (max_mw_size && mw_size > max_mw_size)
  972. mw_size = max_mw_size;
  973. tx_size = (unsigned int)mw_size / num_qps_mw;
  974. qp_offset = tx_size * (qp_num / mw_count);
  975. qp->tx_mw_size = tx_size;
  976. qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;
  977. if (!qp->tx_mw)
  978. return -EINVAL;
  979. qp->tx_mw_phys = mw_base + qp_offset;
  980. if (!qp->tx_mw_phys)
  981. return -EINVAL;
  982. tx_size -= sizeof(struct ntb_rx_info);
  983. qp->rx_info = qp->tx_mw + tx_size;
  984. /* Due to housekeeping, there must be atleast 2 buffs */
  985. qp->tx_max_frame = min(transport_mtu, tx_size / 2);
  986. qp->tx_max_entry = tx_size / qp->tx_max_frame;
  987. if (nt->debugfs_node_dir) {
  988. char debugfs_name[4];
  989. snprintf(debugfs_name, 4, "qp%d", qp_num);
  990. qp->debugfs_dir = debugfs_create_dir(debugfs_name,
  991. nt->debugfs_node_dir);
  992. qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR,
  993. qp->debugfs_dir, qp,
  994. &ntb_qp_debugfs_stats);
  995. } else {
  996. qp->debugfs_dir = NULL;
  997. qp->debugfs_stats = NULL;
  998. }
  999. INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work);
  1000. INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work);
  1001. spin_lock_init(&qp->ntb_rx_q_lock);
  1002. spin_lock_init(&qp->ntb_tx_free_q_lock);
  1003. INIT_LIST_HEAD(&qp->rx_post_q);
  1004. INIT_LIST_HEAD(&qp->rx_pend_q);
  1005. INIT_LIST_HEAD(&qp->rx_free_q);
  1006. INIT_LIST_HEAD(&qp->tx_free_q);
  1007. tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db,
  1008. (unsigned long)qp);
  1009. return 0;
  1010. }
  1011. static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
  1012. {
  1013. struct ntb_transport_ctx *nt;
  1014. struct ntb_transport_mw *mw;
  1015. unsigned int mw_count, qp_count, spad_count, max_mw_count_for_spads;
  1016. u64 qp_bitmap;
  1017. int node;
  1018. int rc, i;
  1019. mw_count = ntb_peer_mw_count(ndev);
  1020. if (!ndev->ops->mw_set_trans) {
  1021. dev_err(&ndev->dev, "Inbound MW based NTB API is required\n");
  1022. return -EINVAL;
  1023. }
  1024. if (ntb_db_is_unsafe(ndev))
  1025. dev_dbg(&ndev->dev,
  1026. "doorbell is unsafe, proceed anyway...\n");
  1027. if (ntb_spad_is_unsafe(ndev))
  1028. dev_dbg(&ndev->dev,
  1029. "scratchpad is unsafe, proceed anyway...\n");
  1030. if (ntb_peer_port_count(ndev) != NTB_DEF_PEER_CNT)
  1031. dev_warn(&ndev->dev, "Multi-port NTB devices unsupported\n");
  1032. node = dev_to_node(&ndev->dev);
  1033. nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node);
  1034. if (!nt)
  1035. return -ENOMEM;
  1036. nt->ndev = ndev;
  1037. /*
  1038. * If we are using MSI, and have at least one extra memory window,
  1039. * we will reserve the last MW for the MSI window.
  1040. */
  1041. if (use_msi && mw_count > 1) {
  1042. rc = ntb_msi_init(ndev, ntb_transport_msi_desc_changed);
  1043. if (!rc) {
  1044. mw_count -= 1;
  1045. nt->use_msi = true;
  1046. }
  1047. }
  1048. spad_count = ntb_spad_count(ndev);
  1049. /* Limit the MW's based on the availability of scratchpads */
  1050. if (spad_count < NTB_TRANSPORT_MIN_SPADS) {
  1051. nt->mw_count = 0;
  1052. rc = -EINVAL;
  1053. goto err;
  1054. }
  1055. max_mw_count_for_spads = (spad_count - MW0_SZ_HIGH) / 2;
  1056. nt->mw_count = min(mw_count, max_mw_count_for_spads);
  1057. nt->msi_spad_offset = nt->mw_count * 2 + MW0_SZ_HIGH;
  1058. nt->mw_vec = kcalloc_node(mw_count, sizeof(*nt->mw_vec),
  1059. GFP_KERNEL, node);
  1060. if (!nt->mw_vec) {
  1061. rc = -ENOMEM;
  1062. goto err;
  1063. }
  1064. for (i = 0; i < mw_count; i++) {
  1065. mw = &nt->mw_vec[i];
  1066. rc = ntb_peer_mw_get_addr(ndev, i, &mw->phys_addr,
  1067. &mw->phys_size);
  1068. if (rc)
  1069. goto err1;
  1070. mw->vbase = ioremap_wc(mw->phys_addr, mw->phys_size);
  1071. if (!mw->vbase) {
  1072. rc = -ENOMEM;
  1073. goto err1;
  1074. }
  1075. mw->buff_size = 0;
  1076. mw->xlat_size = 0;
  1077. mw->virt_addr = NULL;
  1078. mw->dma_addr = 0;
  1079. }
  1080. qp_bitmap = ntb_db_valid_mask(ndev);
  1081. qp_count = ilog2(qp_bitmap);
  1082. if (nt->use_msi) {
  1083. qp_count -= 1;
  1084. nt->msi_db_mask = 1 << qp_count;
  1085. ntb_db_clear_mask(ndev, nt->msi_db_mask);
  1086. }
  1087. if (max_num_clients && max_num_clients < qp_count)
  1088. qp_count = max_num_clients;
  1089. else if (nt->mw_count < qp_count)
  1090. qp_count = nt->mw_count;
  1091. qp_bitmap &= BIT_ULL(qp_count) - 1;
  1092. nt->qp_count = qp_count;
  1093. nt->qp_bitmap = qp_bitmap;
  1094. nt->qp_bitmap_free = qp_bitmap;
  1095. nt->qp_vec = kcalloc_node(qp_count, sizeof(*nt->qp_vec),
  1096. GFP_KERNEL, node);
  1097. if (!nt->qp_vec) {
  1098. rc = -ENOMEM;
  1099. goto err1;
  1100. }
  1101. if (nt_debugfs_dir) {
  1102. nt->debugfs_node_dir =
  1103. debugfs_create_dir(pci_name(ndev->pdev),
  1104. nt_debugfs_dir);
  1105. }
  1106. for (i = 0; i < qp_count; i++) {
  1107. rc = ntb_transport_init_queue(nt, i);
  1108. if (rc)
  1109. goto err2;
  1110. }
  1111. INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);
  1112. INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work);
  1113. rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops);
  1114. if (rc)
  1115. goto err2;
  1116. INIT_LIST_HEAD(&nt->client_devs);
  1117. rc = ntb_bus_init(nt);
  1118. if (rc)
  1119. goto err3;
  1120. nt->link_is_up = false;
  1121. ntb_link_enable(ndev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
  1122. ntb_link_event(ndev);
  1123. return 0;
  1124. err3:
  1125. ntb_clear_ctx(ndev);
  1126. err2:
  1127. kfree(nt->qp_vec);
  1128. err1:
  1129. while (i--) {
  1130. mw = &nt->mw_vec[i];
  1131. iounmap(mw->vbase);
  1132. }
  1133. kfree(nt->mw_vec);
  1134. err:
  1135. kfree(nt);
  1136. return rc;
  1137. }
  1138. static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev)
  1139. {
  1140. struct ntb_transport_ctx *nt = ndev->ctx;
  1141. struct ntb_transport_qp *qp;
  1142. u64 qp_bitmap_alloc;
  1143. int i;
  1144. ntb_transport_link_cleanup(nt);
  1145. cancel_work_sync(&nt->link_cleanup);
  1146. cancel_delayed_work_sync(&nt->link_work);
  1147. qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;
  1148. /* verify that all the qp's are freed */
  1149. for (i = 0; i < nt->qp_count; i++) {
  1150. qp = &nt->qp_vec[i];
  1151. if (qp_bitmap_alloc & BIT_ULL(i))
  1152. ntb_transport_free_queue(qp);
  1153. debugfs_remove_recursive(qp->debugfs_dir);
  1154. }
  1155. ntb_link_disable(ndev);
  1156. ntb_clear_ctx(ndev);
  1157. ntb_bus_remove(nt);
  1158. for (i = nt->mw_count; i--; ) {
  1159. ntb_free_mw(nt, i);
  1160. iounmap(nt->mw_vec[i].vbase);
  1161. }
  1162. kfree(nt->qp_vec);
  1163. kfree(nt->mw_vec);
  1164. kfree(nt);
  1165. }
  1166. static void ntb_complete_rxc(struct ntb_transport_qp *qp)
  1167. {
  1168. struct ntb_queue_entry *entry;
  1169. void *cb_data;
  1170. unsigned int len;
  1171. unsigned long irqflags;
  1172. spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags);
  1173. while (!list_empty(&qp->rx_post_q)) {
  1174. entry = list_first_entry(&qp->rx_post_q,
  1175. struct ntb_queue_entry, entry);
  1176. if (!(entry->flags & DESC_DONE_FLAG))
  1177. break;
  1178. entry->rx_hdr->flags = 0;
  1179. iowrite32(entry->rx_index, &qp->rx_info->entry);
  1180. cb_data = entry->cb_data;
  1181. len = entry->len;
  1182. list_move_tail(&entry->entry, &qp->rx_free_q);
  1183. spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags);
  1184. if (qp->rx_handler && qp->client_ready)
  1185. qp->rx_handler(qp, qp->cb_data, cb_data, len);
  1186. spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags);
  1187. }
  1188. spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags);
  1189. }
  1190. static void ntb_rx_copy_callback(void *data,
  1191. const struct dmaengine_result *res)
  1192. {
  1193. struct ntb_queue_entry *entry = data;
  1194. /* we need to check DMA results if we are using DMA */
  1195. if (res) {
  1196. enum dmaengine_tx_result dma_err = res->result;
  1197. switch (dma_err) {
  1198. case DMA_TRANS_READ_FAILED:
  1199. case DMA_TRANS_WRITE_FAILED:
  1200. entry->errors++;
  1201. fallthrough;
  1202. case DMA_TRANS_ABORTED:
  1203. {
  1204. struct ntb_transport_qp *qp = entry->qp;
  1205. void *offset = qp->rx_buff + qp->rx_max_frame *
  1206. qp->rx_index;
  1207. ntb_memcpy_rx(entry, offset);
  1208. qp->rx_memcpy++;
  1209. return;
  1210. }
  1211. case DMA_TRANS_NOERROR:
  1212. default:
  1213. break;
  1214. }
  1215. }
  1216. entry->flags |= DESC_DONE_FLAG;
  1217. ntb_complete_rxc(entry->qp);
  1218. }
  1219. static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset)
  1220. {
  1221. void *buf = entry->buf;
  1222. size_t len = entry->len;
  1223. memcpy(buf, offset, len);
  1224. /* Ensure that the data is fully copied out before clearing the flag */
  1225. wmb();
  1226. ntb_rx_copy_callback(entry, NULL);
  1227. }
  1228. static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset)
  1229. {
  1230. struct dma_async_tx_descriptor *txd;
  1231. struct ntb_transport_qp *qp = entry->qp;
  1232. struct dma_chan *chan = qp->rx_dma_chan;
  1233. struct dma_device *device;
  1234. size_t pay_off, buff_off, len;
  1235. struct dmaengine_unmap_data *unmap;
  1236. dma_cookie_t cookie;
  1237. void *buf = entry->buf;
  1238. len = entry->len;
  1239. device = chan->device;
  1240. pay_off = (size_t)offset & ~PAGE_MASK;
  1241. buff_off = (size_t)buf & ~PAGE_MASK;
  1242. if (!is_dma_copy_aligned(device, pay_off, buff_off, len))
  1243. goto err;
  1244. unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT);
  1245. if (!unmap)
  1246. goto err;
  1247. unmap->len = len;
  1248. unmap->addr[0] = dma_map_page(device->dev, virt_to_page(offset),
  1249. pay_off, len, DMA_TO_DEVICE);
  1250. if (dma_mapping_error(device->dev, unmap->addr[0]))
  1251. goto err_get_unmap;
  1252. unmap->to_cnt = 1;
  1253. unmap->addr[1] = dma_map_page(device->dev, virt_to_page(buf),
  1254. buff_off, len, DMA_FROM_DEVICE);
  1255. if (dma_mapping_error(device->dev, unmap->addr[1]))
  1256. goto err_get_unmap;
  1257. unmap->from_cnt = 1;
  1258. txd = device->device_prep_dma_memcpy(chan, unmap->addr[1],
  1259. unmap->addr[0], len,
  1260. DMA_PREP_INTERRUPT);
  1261. if (!txd)
  1262. goto err_get_unmap;
  1263. txd->callback_result = ntb_rx_copy_callback;
  1264. txd->callback_param = entry;
  1265. dma_set_unmap(txd, unmap);
  1266. cookie = dmaengine_submit(txd);
  1267. if (dma_submit_error(cookie))
  1268. goto err_set_unmap;
  1269. dmaengine_unmap_put(unmap);
  1270. qp->last_cookie = cookie;
  1271. qp->rx_async++;
  1272. return 0;
  1273. err_set_unmap:
  1274. dmaengine_unmap_put(unmap);
  1275. err_get_unmap:
  1276. dmaengine_unmap_put(unmap);
  1277. err:
  1278. return -ENXIO;
  1279. }
  1280. static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
  1281. {
  1282. struct ntb_transport_qp *qp = entry->qp;
  1283. struct dma_chan *chan = qp->rx_dma_chan;
  1284. int res;
  1285. if (!chan)
  1286. goto err;
  1287. if (entry->len < copy_bytes)
  1288. goto err;
  1289. res = ntb_async_rx_submit(entry, offset);
  1290. if (res < 0)
  1291. goto err;
  1292. if (!entry->retries)
  1293. qp->rx_async++;
  1294. return;
  1295. err:
  1296. ntb_memcpy_rx(entry, offset);
  1297. qp->rx_memcpy++;
  1298. }
  1299. static int ntb_process_rxc(struct ntb_transport_qp *qp)
  1300. {
  1301. struct ntb_payload_header *hdr;
  1302. struct ntb_queue_entry *entry;
  1303. void *offset;
  1304. offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index;
  1305. hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header);
  1306. dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n",
  1307. qp->qp_num, hdr->ver, hdr->len, hdr->flags);
  1308. if (!(hdr->flags & DESC_DONE_FLAG)) {
  1309. dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n");
  1310. qp->rx_ring_empty++;
  1311. return -EAGAIN;
  1312. }
  1313. if (hdr->flags & LINK_DOWN_FLAG) {
  1314. dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n");
  1315. ntb_qp_link_down(qp);
  1316. hdr->flags = 0;
  1317. return -EAGAIN;
  1318. }
  1319. if (hdr->ver != (u32)qp->rx_pkts) {
  1320. dev_dbg(&qp->ndev->pdev->dev,
  1321. "version mismatch, expected %llu - got %u\n",
  1322. qp->rx_pkts, hdr->ver);
  1323. qp->rx_err_ver++;
  1324. return -EIO;
  1325. }
  1326. entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q);
  1327. if (!entry) {
  1328. dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n");
  1329. qp->rx_err_no_buf++;
  1330. return -EAGAIN;
  1331. }
  1332. entry->rx_hdr = hdr;
  1333. entry->rx_index = qp->rx_index;
  1334. if (hdr->len > entry->len) {
  1335. dev_dbg(&qp->ndev->pdev->dev,
  1336. "receive buffer overflow! Wanted %d got %d\n",
  1337. hdr->len, entry->len);
  1338. qp->rx_err_oflow++;
  1339. entry->len = -EIO;
  1340. entry->flags |= DESC_DONE_FLAG;
  1341. ntb_complete_rxc(qp);
  1342. } else {
  1343. dev_dbg(&qp->ndev->pdev->dev,
  1344. "RX OK index %u ver %u size %d into buf size %d\n",
  1345. qp->rx_index, hdr->ver, hdr->len, entry->len);
  1346. qp->rx_bytes += hdr->len;
  1347. qp->rx_pkts++;
  1348. entry->len = hdr->len;
  1349. ntb_async_rx(entry, offset);
  1350. }
  1351. qp->rx_index++;
  1352. qp->rx_index %= qp->rx_max_entry;
  1353. return 0;
  1354. }
  1355. static void ntb_transport_rxc_db(unsigned long data)
  1356. {
  1357. struct ntb_transport_qp *qp = (void *)data;
  1358. int rc, i;
  1359. dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n",
  1360. __func__, qp->qp_num);
  1361. /* Limit the number of packets processed in a single interrupt to
  1362. * provide fairness to others
  1363. */
  1364. for (i = 0; i < qp->rx_max_entry; i++) {
  1365. rc = ntb_process_rxc(qp);
  1366. if (rc)
  1367. break;
  1368. }
  1369. if (i && qp->rx_dma_chan)
  1370. dma_async_issue_pending(qp->rx_dma_chan);
  1371. if (i == qp->rx_max_entry) {
  1372. /* there is more work to do */
  1373. if (qp->active)
  1374. tasklet_schedule(&qp->rxc_db_work);
  1375. } else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) {
  1376. /* the doorbell bit is set: clear it */
  1377. ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num));
  1378. /* ntb_db_read ensures ntb_db_clear write is committed */
  1379. ntb_db_read(qp->ndev);
  1380. /* an interrupt may have arrived between finishing
  1381. * ntb_process_rxc and clearing the doorbell bit:
  1382. * there might be some more work to do.
  1383. */
  1384. if (qp->active)
  1385. tasklet_schedule(&qp->rxc_db_work);
  1386. }
  1387. }
  1388. static void ntb_tx_copy_callback(void *data,
  1389. const struct dmaengine_result *res)
  1390. {
  1391. struct ntb_queue_entry *entry = data;
  1392. struct ntb_transport_qp *qp = entry->qp;
  1393. struct ntb_payload_header __iomem *hdr = entry->tx_hdr;
  1394. /* we need to check DMA results if we are using DMA */
  1395. if (res) {
  1396. enum dmaengine_tx_result dma_err = res->result;
  1397. switch (dma_err) {
  1398. case DMA_TRANS_READ_FAILED:
  1399. case DMA_TRANS_WRITE_FAILED:
  1400. entry->errors++;
  1401. fallthrough;
  1402. case DMA_TRANS_ABORTED:
  1403. {
  1404. void __iomem *offset =
  1405. qp->tx_mw + qp->tx_max_frame *
  1406. entry->tx_index;
  1407. /* resubmit via CPU */
  1408. ntb_memcpy_tx(entry, offset);
  1409. qp->tx_memcpy++;
  1410. return;
  1411. }
  1412. case DMA_TRANS_NOERROR:
  1413. default:
  1414. break;
  1415. }
  1416. }
  1417. iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags);
  1418. if (qp->use_msi)
  1419. ntb_msi_peer_trigger(qp->ndev, PIDX, &qp->peer_msi_desc);
  1420. else
  1421. ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num));
  1422. /* The entry length can only be zero if the packet is intended to be a
  1423. * "link down" or similar. Since no payload is being sent in these
  1424. * cases, there is nothing to add to the completion queue.
  1425. */
  1426. if (entry->len > 0) {
  1427. qp->tx_bytes += entry->len;
  1428. if (qp->tx_handler)
  1429. qp->tx_handler(qp, qp->cb_data, entry->cb_data,
  1430. entry->len);
  1431. }
  1432. ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q);
  1433. }
  1434. static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset)
  1435. {
  1436. #ifdef ARCH_HAS_NOCACHE_UACCESS
  1437. /*
  1438. * Using non-temporal mov to improve performance on non-cached
  1439. * writes, even though we aren't actually copying from user space.
  1440. */
  1441. __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len);
  1442. #else
  1443. memcpy_toio(offset, entry->buf, entry->len);
  1444. #endif
  1445. /* Ensure that the data is fully copied out before setting the flags */
  1446. wmb();
  1447. ntb_tx_copy_callback(entry, NULL);
  1448. }
  1449. static int ntb_async_tx_submit(struct ntb_transport_qp *qp,
  1450. struct ntb_queue_entry *entry)
  1451. {
  1452. struct dma_async_tx_descriptor *txd;
  1453. struct dma_chan *chan = qp->tx_dma_chan;
  1454. struct dma_device *device;
  1455. size_t len = entry->len;
  1456. void *buf = entry->buf;
  1457. size_t dest_off, buff_off;
  1458. struct dmaengine_unmap_data *unmap;
  1459. dma_addr_t dest;
  1460. dma_cookie_t cookie;
  1461. device = chan->device;
  1462. dest = qp->tx_mw_dma_addr + qp->tx_max_frame * entry->tx_index;
  1463. buff_off = (size_t)buf & ~PAGE_MASK;
  1464. dest_off = (size_t)dest & ~PAGE_MASK;
  1465. if (!is_dma_copy_aligned(device, buff_off, dest_off, len))
  1466. goto err;
  1467. unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT);
  1468. if (!unmap)
  1469. goto err;
  1470. unmap->len = len;
  1471. unmap->addr[0] = dma_map_page(device->dev, virt_to_page(buf),
  1472. buff_off, len, DMA_TO_DEVICE);
  1473. if (dma_mapping_error(device->dev, unmap->addr[0]))
  1474. goto err_get_unmap;
  1475. unmap->to_cnt = 1;
  1476. txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len,
  1477. DMA_PREP_INTERRUPT);
  1478. if (!txd)
  1479. goto err_get_unmap;
  1480. txd->callback_result = ntb_tx_copy_callback;
  1481. txd->callback_param = entry;
  1482. dma_set_unmap(txd, unmap);
  1483. cookie = dmaengine_submit(txd);
  1484. if (dma_submit_error(cookie))
  1485. goto err_set_unmap;
  1486. dmaengine_unmap_put(unmap);
  1487. dma_async_issue_pending(chan);
  1488. return 0;
  1489. err_set_unmap:
  1490. dmaengine_unmap_put(unmap);
  1491. err_get_unmap:
  1492. dmaengine_unmap_put(unmap);
  1493. err:
  1494. return -ENXIO;
  1495. }
  1496. static void ntb_async_tx(struct ntb_transport_qp *qp,
  1497. struct ntb_queue_entry *entry)
  1498. {
  1499. struct ntb_payload_header __iomem *hdr;
  1500. struct dma_chan *chan = qp->tx_dma_chan;
  1501. void __iomem *offset;
  1502. int res;
  1503. entry->tx_index = qp->tx_index;
  1504. offset = qp->tx_mw + qp->tx_max_frame * entry->tx_index;
  1505. hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header);
  1506. entry->tx_hdr = hdr;
  1507. iowrite32(entry->len, &hdr->len);
  1508. iowrite32((u32)qp->tx_pkts, &hdr->ver);
  1509. if (!chan)
  1510. goto err;
  1511. if (entry->len < copy_bytes)
  1512. goto err;
  1513. res = ntb_async_tx_submit(qp, entry);
  1514. if (res < 0)
  1515. goto err;
  1516. if (!entry->retries)
  1517. qp->tx_async++;
  1518. return;
  1519. err:
  1520. ntb_memcpy_tx(entry, offset);
  1521. qp->tx_memcpy++;
  1522. }
  1523. static int ntb_process_tx(struct ntb_transport_qp *qp,
  1524. struct ntb_queue_entry *entry)
  1525. {
  1526. if (qp->tx_index == qp->remote_rx_info->entry) {
  1527. qp->tx_ring_full++;
  1528. return -EAGAIN;
  1529. }
  1530. if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) {
  1531. if (qp->tx_handler)
  1532. qp->tx_handler(qp, qp->cb_data, NULL, -EIO);
  1533. ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
  1534. &qp->tx_free_q);
  1535. return 0;
  1536. }
  1537. ntb_async_tx(qp, entry);
  1538. qp->tx_index++;
  1539. qp->tx_index %= qp->tx_max_entry;
  1540. qp->tx_pkts++;
  1541. return 0;
  1542. }
  1543. static void ntb_send_link_down(struct ntb_transport_qp *qp)
  1544. {
  1545. struct pci_dev *pdev = qp->ndev->pdev;
  1546. struct ntb_queue_entry *entry;
  1547. int i, rc;
  1548. if (!qp->link_is_up)
  1549. return;
  1550. dev_info(&pdev->dev, "qp %d: Send Link Down\n", qp->qp_num);
  1551. for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) {
  1552. entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);
  1553. if (entry)
  1554. break;
  1555. msleep(100);
  1556. }
  1557. if (!entry)
  1558. return;
  1559. entry->cb_data = NULL;
  1560. entry->buf = NULL;
  1561. entry->len = 0;
  1562. entry->flags = LINK_DOWN_FLAG;
  1563. rc = ntb_process_tx(qp, entry);
  1564. if (rc)
  1565. dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n",
  1566. qp->qp_num);
  1567. ntb_qp_link_down_reset(qp);
  1568. }
  1569. static bool ntb_dma_filter_fn(struct dma_chan *chan, void *node)
  1570. {
  1571. return dev_to_node(&chan->dev->device) == (int)(unsigned long)node;
  1572. }
  1573. /**
  1574. * ntb_transport_create_queue - Create a new NTB transport layer queue
  1575. * @rx_handler: receive callback function
  1576. * @tx_handler: transmit callback function
  1577. * @event_handler: event callback function
  1578. *
  1579. * Create a new NTB transport layer queue and provide the queue with a callback
  1580. * routine for both transmit and receive. The receive callback routine will be
  1581. * used to pass up data when the transport has received it on the queue. The
  1582. * transmit callback routine will be called when the transport has completed the
  1583. * transmission of the data on the queue and the data is ready to be freed.
  1584. *
  1585. * RETURNS: pointer to newly created ntb_queue, NULL on error.
  1586. */
  1587. struct ntb_transport_qp *
  1588. ntb_transport_create_queue(void *data, struct device *client_dev,
  1589. const struct ntb_queue_handlers *handlers)
  1590. {
  1591. struct ntb_dev *ndev;
  1592. struct pci_dev *pdev;
  1593. struct ntb_transport_ctx *nt;
  1594. struct ntb_queue_entry *entry;
  1595. struct ntb_transport_qp *qp;
  1596. u64 qp_bit;
  1597. unsigned int free_queue;
  1598. dma_cap_mask_t dma_mask;
  1599. int node;
  1600. int i;
  1601. ndev = dev_ntb(client_dev->parent);
  1602. pdev = ndev->pdev;
  1603. nt = ndev->ctx;
  1604. node = dev_to_node(&ndev->dev);
  1605. free_queue = ffs(nt->qp_bitmap_free);
  1606. if (!free_queue)
  1607. goto err;
  1608. /* decrement free_queue to make it zero based */
  1609. free_queue--;
  1610. qp = &nt->qp_vec[free_queue];
  1611. qp_bit = BIT_ULL(qp->qp_num);
  1612. nt->qp_bitmap_free &= ~qp_bit;
  1613. qp->cb_data = data;
  1614. qp->rx_handler = handlers->rx_handler;
  1615. qp->tx_handler = handlers->tx_handler;
  1616. qp->event_handler = handlers->event_handler;
  1617. dma_cap_zero(dma_mask);
  1618. dma_cap_set(DMA_MEMCPY, dma_mask);
  1619. if (use_dma) {
  1620. qp->tx_dma_chan =
  1621. dma_request_channel(dma_mask, ntb_dma_filter_fn,
  1622. (void *)(unsigned long)node);
  1623. if (!qp->tx_dma_chan)
  1624. dev_info(&pdev->dev, "Unable to allocate TX DMA channel\n");
  1625. qp->rx_dma_chan =
  1626. dma_request_channel(dma_mask, ntb_dma_filter_fn,
  1627. (void *)(unsigned long)node);
  1628. if (!qp->rx_dma_chan)
  1629. dev_info(&pdev->dev, "Unable to allocate RX DMA channel\n");
  1630. } else {
  1631. qp->tx_dma_chan = NULL;
  1632. qp->rx_dma_chan = NULL;
  1633. }
  1634. qp->tx_mw_dma_addr = 0;
  1635. if (qp->tx_dma_chan) {
  1636. qp->tx_mw_dma_addr =
  1637. dma_map_resource(qp->tx_dma_chan->device->dev,
  1638. qp->tx_mw_phys, qp->tx_mw_size,
  1639. DMA_FROM_DEVICE, 0);
  1640. if (dma_mapping_error(qp->tx_dma_chan->device->dev,
  1641. qp->tx_mw_dma_addr)) {
  1642. qp->tx_mw_dma_addr = 0;
  1643. goto err1;
  1644. }
  1645. }
  1646. dev_dbg(&pdev->dev, "Using %s memcpy for TX\n",
  1647. qp->tx_dma_chan ? "DMA" : "CPU");
  1648. dev_dbg(&pdev->dev, "Using %s memcpy for RX\n",
  1649. qp->rx_dma_chan ? "DMA" : "CPU");
  1650. for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
  1651. entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
  1652. if (!entry)
  1653. goto err1;
  1654. entry->qp = qp;
  1655. ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
  1656. &qp->rx_free_q);
  1657. }
  1658. qp->rx_alloc_entry = NTB_QP_DEF_NUM_ENTRIES;
  1659. for (i = 0; i < qp->tx_max_entry; i++) {
  1660. entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
  1661. if (!entry)
  1662. goto err2;
  1663. entry->qp = qp;
  1664. ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
  1665. &qp->tx_free_q);
  1666. }
  1667. ntb_db_clear(qp->ndev, qp_bit);
  1668. ntb_db_clear_mask(qp->ndev, qp_bit);
  1669. dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num);
  1670. return qp;
  1671. err2:
  1672. while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
  1673. kfree(entry);
  1674. err1:
  1675. qp->rx_alloc_entry = 0;
  1676. while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q)))
  1677. kfree(entry);
  1678. if (qp->tx_mw_dma_addr)
  1679. dma_unmap_resource(qp->tx_dma_chan->device->dev,
  1680. qp->tx_mw_dma_addr, qp->tx_mw_size,
  1681. DMA_FROM_DEVICE, 0);
  1682. if (qp->tx_dma_chan)
  1683. dma_release_channel(qp->tx_dma_chan);
  1684. if (qp->rx_dma_chan)
  1685. dma_release_channel(qp->rx_dma_chan);
  1686. nt->qp_bitmap_free |= qp_bit;
  1687. err:
  1688. return NULL;
  1689. }
  1690. EXPORT_SYMBOL_GPL(ntb_transport_create_queue);
  1691. /**
  1692. * ntb_transport_free_queue - Frees NTB transport queue
  1693. * @qp: NTB queue to be freed
  1694. *
  1695. * Frees NTB transport queue
  1696. */
  1697. void ntb_transport_free_queue(struct ntb_transport_qp *qp)
  1698. {
  1699. struct pci_dev *pdev;
  1700. struct ntb_queue_entry *entry;
  1701. u64 qp_bit;
  1702. if (!qp)
  1703. return;
  1704. pdev = qp->ndev->pdev;
  1705. qp->active = false;
  1706. if (qp->tx_dma_chan) {
  1707. struct dma_chan *chan = qp->tx_dma_chan;
  1708. /* Putting the dma_chan to NULL will force any new traffic to be
  1709. * processed by the CPU instead of the DAM engine
  1710. */
  1711. qp->tx_dma_chan = NULL;
  1712. /* Try to be nice and wait for any queued DMA engine
  1713. * transactions to process before smashing it with a rock
  1714. */
  1715. dma_sync_wait(chan, qp->last_cookie);
  1716. dmaengine_terminate_all(chan);
  1717. dma_unmap_resource(chan->device->dev,
  1718. qp->tx_mw_dma_addr, qp->tx_mw_size,
  1719. DMA_FROM_DEVICE, 0);
  1720. dma_release_channel(chan);
  1721. }
  1722. if (qp->rx_dma_chan) {
  1723. struct dma_chan *chan = qp->rx_dma_chan;
  1724. /* Putting the dma_chan to NULL will force any new traffic to be
  1725. * processed by the CPU instead of the DAM engine
  1726. */
  1727. qp->rx_dma_chan = NULL;
  1728. /* Try to be nice and wait for any queued DMA engine
  1729. * transactions to process before smashing it with a rock
  1730. */
  1731. dma_sync_wait(chan, qp->last_cookie);
  1732. dmaengine_terminate_all(chan);
  1733. dma_release_channel(chan);
  1734. }
  1735. qp_bit = BIT_ULL(qp->qp_num);
  1736. ntb_db_set_mask(qp->ndev, qp_bit);
  1737. tasklet_kill(&qp->rxc_db_work);
  1738. cancel_delayed_work_sync(&qp->link_work);
  1739. qp->cb_data = NULL;
  1740. qp->rx_handler = NULL;
  1741. qp->tx_handler = NULL;
  1742. qp->event_handler = NULL;
  1743. while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q)))
  1744. kfree(entry);
  1745. while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) {
  1746. dev_warn(&pdev->dev, "Freeing item from non-empty rx_pend_q\n");
  1747. kfree(entry);
  1748. }
  1749. while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) {
  1750. dev_warn(&pdev->dev, "Freeing item from non-empty rx_post_q\n");
  1751. kfree(entry);
  1752. }
  1753. while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
  1754. kfree(entry);
  1755. qp->transport->qp_bitmap_free |= qp_bit;
  1756. dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num);
  1757. }
  1758. EXPORT_SYMBOL_GPL(ntb_transport_free_queue);
  1759. /**
  1760. * ntb_transport_rx_remove - Dequeues enqueued rx packet
  1761. * @qp: NTB queue to be freed
  1762. * @len: pointer to variable to write enqueued buffers length
  1763. *
  1764. * Dequeues unused buffers from receive queue. Should only be used during
  1765. * shutdown of qp.
  1766. *
  1767. * RETURNS: NULL error value on error, or void* for success.
  1768. */
  1769. void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len)
  1770. {
  1771. struct ntb_queue_entry *entry;
  1772. void *buf;
  1773. if (!qp || qp->client_ready)
  1774. return NULL;
  1775. entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q);
  1776. if (!entry)
  1777. return NULL;
  1778. buf = entry->cb_data;
  1779. *len = entry->len;
  1780. ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
  1781. return buf;
  1782. }
  1783. EXPORT_SYMBOL_GPL(ntb_transport_rx_remove);
  1784. /**
  1785. * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry
  1786. * @qp: NTB transport layer queue the entry is to be enqueued on
  1787. * @cb: per buffer pointer for callback function to use
  1788. * @data: pointer to data buffer that incoming packets will be copied into
  1789. * @len: length of the data buffer
  1790. *
  1791. * Enqueue a new receive buffer onto the transport queue into which a NTB
  1792. * payload can be received into.
  1793. *
  1794. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  1795. */
  1796. int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
  1797. unsigned int len)
  1798. {
  1799. struct ntb_queue_entry *entry;
  1800. if (!qp)
  1801. return -EINVAL;
  1802. entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q);
  1803. if (!entry)
  1804. return -ENOMEM;
  1805. entry->cb_data = cb;
  1806. entry->buf = data;
  1807. entry->len = len;
  1808. entry->flags = 0;
  1809. entry->retries = 0;
  1810. entry->errors = 0;
  1811. entry->rx_index = 0;
  1812. ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q);
  1813. if (qp->active)
  1814. tasklet_schedule(&qp->rxc_db_work);
  1815. return 0;
  1816. }
  1817. EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue);
  1818. /**
  1819. * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry
  1820. * @qp: NTB transport layer queue the entry is to be enqueued on
  1821. * @cb: per buffer pointer for callback function to use
  1822. * @data: pointer to data buffer that will be sent
  1823. * @len: length of the data buffer
  1824. *
  1825. * Enqueue a new transmit buffer onto the transport queue from which a NTB
  1826. * payload will be transmitted. This assumes that a lock is being held to
  1827. * serialize access to the qp.
  1828. *
  1829. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  1830. */
  1831. int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
  1832. unsigned int len)
  1833. {
  1834. struct ntb_queue_entry *entry;
  1835. int rc;
  1836. if (!qp || !qp->link_is_up || !len)
  1837. return -EINVAL;
  1838. entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);
  1839. if (!entry) {
  1840. qp->tx_err_no_buf++;
  1841. return -EBUSY;
  1842. }
  1843. entry->cb_data = cb;
  1844. entry->buf = data;
  1845. entry->len = len;
  1846. entry->flags = 0;
  1847. entry->errors = 0;
  1848. entry->retries = 0;
  1849. entry->tx_index = 0;
  1850. rc = ntb_process_tx(qp, entry);
  1851. if (rc)
  1852. ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
  1853. &qp->tx_free_q);
  1854. return rc;
  1855. }
  1856. EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue);
  1857. /**
  1858. * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
  1859. * @qp: NTB transport layer queue to be enabled
  1860. *
  1861. * Notify NTB transport layer of client readiness to use queue
  1862. */
  1863. void ntb_transport_link_up(struct ntb_transport_qp *qp)
  1864. {
  1865. if (!qp)
  1866. return;
  1867. qp->client_ready = true;
  1868. if (qp->transport->link_is_up)
  1869. schedule_delayed_work(&qp->link_work, 0);
  1870. }
  1871. EXPORT_SYMBOL_GPL(ntb_transport_link_up);
  1872. /**
  1873. * ntb_transport_link_down - Notify NTB transport to no longer enqueue data
  1874. * @qp: NTB transport layer queue to be disabled
  1875. *
  1876. * Notify NTB transport layer of client's desire to no longer receive data on
  1877. * transport queue specified. It is the client's responsibility to ensure all
  1878. * entries on queue are purged or otherwise handled appropriately.
  1879. */
  1880. void ntb_transport_link_down(struct ntb_transport_qp *qp)
  1881. {
  1882. int val;
  1883. if (!qp)
  1884. return;
  1885. qp->client_ready = false;
  1886. val = ntb_spad_read(qp->ndev, QP_LINKS);
  1887. ntb_peer_spad_write(qp->ndev, PIDX, QP_LINKS, val & ~BIT(qp->qp_num));
  1888. if (qp->link_is_up)
  1889. ntb_send_link_down(qp);
  1890. else
  1891. cancel_delayed_work_sync(&qp->link_work);
  1892. }
  1893. EXPORT_SYMBOL_GPL(ntb_transport_link_down);
  1894. /**
  1895. * ntb_transport_link_query - Query transport link state
  1896. * @qp: NTB transport layer queue to be queried
  1897. *
  1898. * Query connectivity to the remote system of the NTB transport queue
  1899. *
  1900. * RETURNS: true for link up or false for link down
  1901. */
  1902. bool ntb_transport_link_query(struct ntb_transport_qp *qp)
  1903. {
  1904. if (!qp)
  1905. return false;
  1906. return qp->link_is_up;
  1907. }
  1908. EXPORT_SYMBOL_GPL(ntb_transport_link_query);
  1909. /**
  1910. * ntb_transport_qp_num - Query the qp number
  1911. * @qp: NTB transport layer queue to be queried
  1912. *
  1913. * Query qp number of the NTB transport queue
  1914. *
  1915. * RETURNS: a zero based number specifying the qp number
  1916. */
  1917. unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp)
  1918. {
  1919. if (!qp)
  1920. return 0;
  1921. return qp->qp_num;
  1922. }
  1923. EXPORT_SYMBOL_GPL(ntb_transport_qp_num);
  1924. /**
  1925. * ntb_transport_max_size - Query the max payload size of a qp
  1926. * @qp: NTB transport layer queue to be queried
  1927. *
  1928. * Query the maximum payload size permissible on the given qp
  1929. *
  1930. * RETURNS: the max payload size of a qp
  1931. */
  1932. unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp)
  1933. {
  1934. unsigned int max_size;
  1935. unsigned int copy_align;
  1936. struct dma_chan *rx_chan, *tx_chan;
  1937. if (!qp)
  1938. return 0;
  1939. rx_chan = qp->rx_dma_chan;
  1940. tx_chan = qp->tx_dma_chan;
  1941. copy_align = max(rx_chan ? rx_chan->device->copy_align : 0,
  1942. tx_chan ? tx_chan->device->copy_align : 0);
  1943. /* If DMA engine usage is possible, try to find the max size for that */
  1944. max_size = qp->tx_max_frame - sizeof(struct ntb_payload_header);
  1945. max_size = round_down(max_size, 1 << copy_align);
  1946. return max_size;
  1947. }
  1948. EXPORT_SYMBOL_GPL(ntb_transport_max_size);
  1949. unsigned int ntb_transport_tx_free_entry(struct ntb_transport_qp *qp)
  1950. {
  1951. unsigned int head = qp->tx_index;
  1952. unsigned int tail = qp->remote_rx_info->entry;
  1953. return tail > head ? tail - head : qp->tx_max_entry + tail - head;
  1954. }
  1955. EXPORT_SYMBOL_GPL(ntb_transport_tx_free_entry);
  1956. static void ntb_transport_doorbell_callback(void *data, int vector)
  1957. {
  1958. struct ntb_transport_ctx *nt = data;
  1959. struct ntb_transport_qp *qp;
  1960. u64 db_bits;
  1961. unsigned int qp_num;
  1962. if (ntb_db_read(nt->ndev) & nt->msi_db_mask) {
  1963. ntb_transport_msi_peer_desc_changed(nt);
  1964. ntb_db_clear(nt->ndev, nt->msi_db_mask);
  1965. }
  1966. db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free &
  1967. ntb_db_vector_mask(nt->ndev, vector));
  1968. while (db_bits) {
  1969. qp_num = __ffs(db_bits);
  1970. qp = &nt->qp_vec[qp_num];
  1971. if (qp->active)
  1972. tasklet_schedule(&qp->rxc_db_work);
  1973. db_bits &= ~BIT_ULL(qp_num);
  1974. }
  1975. }
  1976. static const struct ntb_ctx_ops ntb_transport_ops = {
  1977. .link_event = ntb_transport_event_callback,
  1978. .db_event = ntb_transport_doorbell_callback,
  1979. };
  1980. static struct ntb_client ntb_transport_client = {
  1981. .ops = {
  1982. .probe = ntb_transport_probe,
  1983. .remove = ntb_transport_free,
  1984. },
  1985. };
  1986. static int __init ntb_transport_init(void)
  1987. {
  1988. int rc;
  1989. pr_info("%s, version %s\n", NTB_TRANSPORT_DESC, NTB_TRANSPORT_VER);
  1990. if (debugfs_initialized())
  1991. nt_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
  1992. rc = bus_register(&ntb_transport_bus);
  1993. if (rc)
  1994. goto err_bus;
  1995. rc = ntb_register_client(&ntb_transport_client);
  1996. if (rc)
  1997. goto err_client;
  1998. return 0;
  1999. err_client:
  2000. bus_unregister(&ntb_transport_bus);
  2001. err_bus:
  2002. debugfs_remove_recursive(nt_debugfs_dir);
  2003. return rc;
  2004. }
  2005. module_init(ntb_transport_init);
  2006. static void __exit ntb_transport_exit(void)
  2007. {
  2008. ntb_unregister_client(&ntb_transport_client);
  2009. bus_unregister(&ntb_transport_bus);
  2010. debugfs_remove_recursive(nt_debugfs_dir);
  2011. }
  2012. module_exit(ntb_transport_exit);