ntb_perf.c 39 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567
  1. /*
  2. * This file is provided under a dual BSD/GPLv2 license. When using or
  3. * redistributing this file, you may do so under either license.
  4. *
  5. * GPL LICENSE SUMMARY
  6. *
  7. * Copyright(c) 2015 Intel Corporation. All rights reserved.
  8. * Copyright(c) 2017 T-Platforms. All Rights Reserved.
  9. *
  10. * This program is free software; you can redistribute it and/or modify
  11. * it under the terms of version 2 of the GNU General Public License as
  12. * published by the Free Software Foundation.
  13. *
  14. * BSD LICENSE
  15. *
  16. * Copyright(c) 2015 Intel Corporation. All rights reserved.
  17. * Copyright(c) 2017 T-Platforms. All Rights Reserved.
  18. *
  19. * Redistribution and use in source and binary forms, with or without
  20. * modification, are permitted provided that the following conditions
  21. * are met:
  22. *
  23. * * Redistributions of source code must retain the above copyright
  24. * notice, this list of conditions and the following disclaimer.
  25. * * Redistributions in binary form must reproduce the above copy
  26. * notice, this list of conditions and the following disclaimer in
  27. * the documentation and/or other materials provided with the
  28. * distribution.
  29. * * Neither the name of Intel Corporation nor the names of its
  30. * contributors may be used to endorse or promote products derived
  31. * from this software without specific prior written permission.
  32. *
  33. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  34. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  35. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  36. * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  37. * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  38. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  39. * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  40. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  41. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  42. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  43. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  44. *
  45. * PCIe NTB Perf Linux driver
  46. */
  47. /*
  48. * How to use this tool, by example.
  49. *
  50. * Assuming $DBG_DIR is something like:
  51. * '/sys/kernel/debug/ntb_perf/0000:00:03.0'
  52. * Suppose aside from local device there is at least one remote device
  53. * connected to NTB with index 0.
  54. *-----------------------------------------------------------------------------
  55. * Eg: install driver with specified chunk/total orders and dma-enabled flag
  56. *
  57. * root@self# insmod ntb_perf.ko chunk_order=19 total_order=28 use_dma
  58. *-----------------------------------------------------------------------------
  59. * Eg: check NTB ports (index) and MW mapping information
  60. *
  61. * root@self# cat $DBG_DIR/info
  62. *-----------------------------------------------------------------------------
  63. * Eg: start performance test with peer (index 0) and get the test metrics
  64. *
  65. * root@self# echo 0 > $DBG_DIR/run
  66. * root@self# cat $DBG_DIR/run
  67. */
  68. #include <linux/init.h>
  69. #include <linux/kernel.h>
  70. #include <linux/module.h>
  71. #include <linux/sched.h>
  72. #include <linux/wait.h>
  73. #include <linux/dma-mapping.h>
  74. #include <linux/dmaengine.h>
  75. #include <linux/pci.h>
  76. #include <linux/ktime.h>
  77. #include <linux/slab.h>
  78. #include <linux/delay.h>
  79. #include <linux/sizes.h>
  80. #include <linux/workqueue.h>
  81. #include <linux/debugfs.h>
  82. #include <linux/random.h>
  83. #include <linux/ntb.h>
  84. #define DRIVER_NAME "ntb_perf"
  85. #define DRIVER_VERSION "2.0"
  86. MODULE_LICENSE("Dual BSD/GPL");
  87. MODULE_VERSION(DRIVER_VERSION);
  88. MODULE_AUTHOR("Dave Jiang <dave.jiang@intel.com>");
  89. MODULE_DESCRIPTION("PCIe NTB Performance Measurement Tool");
  90. #define MAX_THREADS_CNT 32
  91. #define DEF_THREADS_CNT 1
  92. #define MAX_CHUNK_SIZE SZ_1M
  93. #define MAX_CHUNK_ORDER 20 /* no larger than 1M */
  94. #define DMA_TRIES 100
  95. #define DMA_MDELAY 10
  96. #define MSG_TRIES 1000
  97. #define MSG_UDELAY_LOW 1000000
  98. #define MSG_UDELAY_HIGH 2000000
  99. #define PERF_BUF_LEN 1024
  100. static unsigned long max_mw_size;
  101. module_param(max_mw_size, ulong, 0644);
  102. MODULE_PARM_DESC(max_mw_size, "Upper limit of memory window size");
  103. static unsigned char chunk_order = 19; /* 512K */
  104. module_param(chunk_order, byte, 0644);
  105. MODULE_PARM_DESC(chunk_order, "Data chunk order [2^n] to transfer");
  106. static unsigned char total_order = 30; /* 1G */
  107. module_param(total_order, byte, 0644);
  108. MODULE_PARM_DESC(total_order, "Total data order [2^n] to transfer");
  109. static bool use_dma; /* default to 0 */
  110. module_param(use_dma, bool, 0644);
  111. MODULE_PARM_DESC(use_dma, "Use DMA engine to measure performance");
  112. /*==============================================================================
  113. * Perf driver data definition
  114. *==============================================================================
  115. */
  116. enum perf_cmd {
  117. PERF_CMD_INVAL = -1,/* invalid spad command */
  118. PERF_CMD_SSIZE = 0, /* send out buffer size */
  119. PERF_CMD_RSIZE = 1, /* recv in buffer size */
  120. PERF_CMD_SXLAT = 2, /* send in buffer xlat */
  121. PERF_CMD_RXLAT = 3, /* recv out buffer xlat */
  122. PERF_CMD_CLEAR = 4, /* clear allocated memory */
  123. PERF_STS_DONE = 5, /* init is done */
  124. PERF_STS_LNKUP = 6, /* link up state flag */
  125. };
  126. struct perf_ctx;
  127. struct perf_peer {
  128. struct perf_ctx *perf;
  129. int pidx;
  130. int gidx;
  131. /* Outbound MW params */
  132. u64 outbuf_xlat;
  133. resource_size_t outbuf_size;
  134. void __iomem *outbuf;
  135. phys_addr_t out_phys_addr;
  136. dma_addr_t dma_dst_addr;
  137. /* Inbound MW params */
  138. dma_addr_t inbuf_xlat;
  139. resource_size_t inbuf_size;
  140. void *inbuf;
  141. /* NTB connection setup service */
  142. struct work_struct service;
  143. unsigned long sts;
  144. struct completion init_comp;
  145. };
  146. #define to_peer_service(__work) \
  147. container_of(__work, struct perf_peer, service)
  148. struct perf_thread {
  149. struct perf_ctx *perf;
  150. int tidx;
  151. /* DMA-based test sync parameters */
  152. atomic_t dma_sync;
  153. wait_queue_head_t dma_wait;
  154. struct dma_chan *dma_chan;
  155. /* Data source and measured statistics */
  156. void *src;
  157. u64 copied;
  158. ktime_t duration;
  159. int status;
  160. struct work_struct work;
  161. };
  162. #define to_thread_work(__work) \
  163. container_of(__work, struct perf_thread, work)
  164. struct perf_ctx {
  165. struct ntb_dev *ntb;
  166. /* Global device index and peers descriptors */
  167. int gidx;
  168. int pcnt;
  169. struct perf_peer *peers;
  170. /* Performance measuring work-threads interface */
  171. unsigned long busy_flag;
  172. wait_queue_head_t twait;
  173. atomic_t tsync;
  174. u8 tcnt;
  175. struct perf_peer *test_peer;
  176. struct perf_thread threads[MAX_THREADS_CNT];
  177. /* Scratchpad/Message IO operations */
  178. int (*cmd_send)(struct perf_peer *peer, enum perf_cmd cmd, u64 data);
  179. int (*cmd_recv)(struct perf_ctx *perf, int *pidx, enum perf_cmd *cmd,
  180. u64 *data);
  181. struct dentry *dbgfs_dir;
  182. };
  183. /*
  184. * Scratchpads-base commands interface
  185. */
  186. #define PERF_SPAD_CNT(_pcnt) \
  187. (3*((_pcnt) + 1))
  188. #define PERF_SPAD_CMD(_gidx) \
  189. (3*(_gidx))
  190. #define PERF_SPAD_LDATA(_gidx) \
  191. (3*(_gidx) + 1)
  192. #define PERF_SPAD_HDATA(_gidx) \
  193. (3*(_gidx) + 2)
  194. #define PERF_SPAD_NOTIFY(_gidx) \
  195. (BIT_ULL(_gidx))
  196. /*
  197. * Messages-base commands interface
  198. */
  199. #define PERF_MSG_CNT 3
  200. #define PERF_MSG_CMD 0
  201. #define PERF_MSG_LDATA 1
  202. #define PERF_MSG_HDATA 2
  203. /*==============================================================================
  204. * Static data declarations
  205. *==============================================================================
  206. */
  207. static struct dentry *perf_dbgfs_topdir;
  208. static struct workqueue_struct *perf_wq __read_mostly;
  209. /*==============================================================================
  210. * NTB cross-link commands execution service
  211. *==============================================================================
  212. */
  213. static void perf_terminate_test(struct perf_ctx *perf);
  214. static inline bool perf_link_is_up(struct perf_peer *peer)
  215. {
  216. u64 link;
  217. link = ntb_link_is_up(peer->perf->ntb, NULL, NULL);
  218. return !!(link & BIT_ULL_MASK(peer->pidx));
  219. }
  220. static int perf_spad_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,
  221. u64 data)
  222. {
  223. struct perf_ctx *perf = peer->perf;
  224. int try;
  225. u32 sts;
  226. dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
  227. /*
  228. * Perform predefined number of attempts before give up.
  229. * We are sending the data to the port specific scratchpad, so
  230. * to prevent a multi-port access race-condition. Additionally
  231. * there is no need in local locking since only thread-safe
  232. * service work is using this method.
  233. */
  234. for (try = 0; try < MSG_TRIES; try++) {
  235. if (!perf_link_is_up(peer))
  236. return -ENOLINK;
  237. sts = ntb_peer_spad_read(perf->ntb, peer->pidx,
  238. PERF_SPAD_CMD(perf->gidx));
  239. if (sts != PERF_CMD_INVAL) {
  240. usleep_range(MSG_UDELAY_LOW, MSG_UDELAY_HIGH);
  241. continue;
  242. }
  243. ntb_peer_spad_write(perf->ntb, peer->pidx,
  244. PERF_SPAD_LDATA(perf->gidx),
  245. lower_32_bits(data));
  246. ntb_peer_spad_write(perf->ntb, peer->pidx,
  247. PERF_SPAD_HDATA(perf->gidx),
  248. upper_32_bits(data));
  249. ntb_peer_spad_write(perf->ntb, peer->pidx,
  250. PERF_SPAD_CMD(perf->gidx),
  251. cmd);
  252. ntb_peer_db_set(perf->ntb, PERF_SPAD_NOTIFY(peer->gidx));
  253. dev_dbg(&perf->ntb->dev, "DB ring peer %#llx\n",
  254. PERF_SPAD_NOTIFY(peer->gidx));
  255. break;
  256. }
  257. return try < MSG_TRIES ? 0 : -EAGAIN;
  258. }
  259. static int perf_spad_cmd_recv(struct perf_ctx *perf, int *pidx,
  260. enum perf_cmd *cmd, u64 *data)
  261. {
  262. struct perf_peer *peer;
  263. u32 val;
  264. ntb_db_clear(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
  265. /*
  266. * We start scanning all over, since cleared DB may have been set
  267. * by any peer. Yes, it makes peer with smaller index being
  268. * serviced with greater priority, but it's convenient for spad
  269. * and message code unification and simplicity.
  270. */
  271. for (*pidx = 0; *pidx < perf->pcnt; (*pidx)++) {
  272. peer = &perf->peers[*pidx];
  273. if (!perf_link_is_up(peer))
  274. continue;
  275. val = ntb_spad_read(perf->ntb, PERF_SPAD_CMD(peer->gidx));
  276. if (val == PERF_CMD_INVAL)
  277. continue;
  278. *cmd = val;
  279. val = ntb_spad_read(perf->ntb, PERF_SPAD_LDATA(peer->gidx));
  280. *data = val;
  281. val = ntb_spad_read(perf->ntb, PERF_SPAD_HDATA(peer->gidx));
  282. *data |= (u64)val << 32;
  283. /* Next command can be retrieved from now */
  284. ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx),
  285. PERF_CMD_INVAL);
  286. dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
  287. return 0;
  288. }
  289. return -ENODATA;
  290. }
  291. static int perf_msg_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,
  292. u64 data)
  293. {
  294. struct perf_ctx *perf = peer->perf;
  295. int try, ret;
  296. u64 outbits;
  297. dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
  298. /*
  299. * Perform predefined number of attempts before give up. Message
  300. * registers are free of race-condition problem when accessed
  301. * from different ports, so we don't need splitting registers
  302. * by global device index. We also won't have local locking,
  303. * since the method is used from service work only.
  304. */
  305. outbits = ntb_msg_outbits(perf->ntb);
  306. for (try = 0; try < MSG_TRIES; try++) {
  307. if (!perf_link_is_up(peer))
  308. return -ENOLINK;
  309. ret = ntb_msg_clear_sts(perf->ntb, outbits);
  310. if (ret)
  311. return ret;
  312. ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_LDATA,
  313. lower_32_bits(data));
  314. if (ntb_msg_read_sts(perf->ntb) & outbits) {
  315. usleep_range(MSG_UDELAY_LOW, MSG_UDELAY_HIGH);
  316. continue;
  317. }
  318. ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_HDATA,
  319. upper_32_bits(data));
  320. /* This call shall trigger peer message event */
  321. ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_CMD, cmd);
  322. break;
  323. }
  324. return try < MSG_TRIES ? 0 : -EAGAIN;
  325. }
  326. static int perf_msg_cmd_recv(struct perf_ctx *perf, int *pidx,
  327. enum perf_cmd *cmd, u64 *data)
  328. {
  329. u64 inbits;
  330. u32 val;
  331. inbits = ntb_msg_inbits(perf->ntb);
  332. if (hweight64(ntb_msg_read_sts(perf->ntb) & inbits) < 3)
  333. return -ENODATA;
  334. val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_CMD);
  335. *cmd = val;
  336. val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_LDATA);
  337. *data = val;
  338. val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_HDATA);
  339. *data |= (u64)val << 32;
  340. /* Next command can be retrieved from now */
  341. ntb_msg_clear_sts(perf->ntb, inbits);
  342. dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
  343. return 0;
  344. }
  345. static int perf_cmd_send(struct perf_peer *peer, enum perf_cmd cmd, u64 data)
  346. {
  347. struct perf_ctx *perf = peer->perf;
  348. if (cmd == PERF_CMD_SSIZE || cmd == PERF_CMD_SXLAT)
  349. return perf->cmd_send(peer, cmd, data);
  350. dev_err(&perf->ntb->dev, "Send invalid command\n");
  351. return -EINVAL;
  352. }
  353. static int perf_cmd_exec(struct perf_peer *peer, enum perf_cmd cmd)
  354. {
  355. switch (cmd) {
  356. case PERF_CMD_SSIZE:
  357. case PERF_CMD_RSIZE:
  358. case PERF_CMD_SXLAT:
  359. case PERF_CMD_RXLAT:
  360. case PERF_CMD_CLEAR:
  361. break;
  362. default:
  363. dev_err(&peer->perf->ntb->dev, "Exec invalid command\n");
  364. return -EINVAL;
  365. }
  366. /* No need of memory barrier, since bit ops have invernal lock */
  367. set_bit(cmd, &peer->sts);
  368. dev_dbg(&peer->perf->ntb->dev, "CMD exec: %d\n", cmd);
  369. (void)queue_work(system_highpri_wq, &peer->service);
  370. return 0;
  371. }
  372. static int perf_cmd_recv(struct perf_ctx *perf)
  373. {
  374. struct perf_peer *peer;
  375. int ret, pidx, cmd;
  376. u64 data;
  377. while (!(ret = perf->cmd_recv(perf, &pidx, &cmd, &data))) {
  378. peer = &perf->peers[pidx];
  379. switch (cmd) {
  380. case PERF_CMD_SSIZE:
  381. peer->inbuf_size = data;
  382. return perf_cmd_exec(peer, PERF_CMD_RSIZE);
  383. case PERF_CMD_SXLAT:
  384. peer->outbuf_xlat = data;
  385. return perf_cmd_exec(peer, PERF_CMD_RXLAT);
  386. default:
  387. dev_err(&perf->ntb->dev, "Recv invalid command\n");
  388. return -EINVAL;
  389. }
  390. }
  391. /* Return 0 if no data left to process, otherwise an error */
  392. return ret == -ENODATA ? 0 : ret;
  393. }
  394. static void perf_link_event(void *ctx)
  395. {
  396. struct perf_ctx *perf = ctx;
  397. struct perf_peer *peer;
  398. bool lnk_up;
  399. int pidx;
  400. for (pidx = 0; pidx < perf->pcnt; pidx++) {
  401. peer = &perf->peers[pidx];
  402. lnk_up = perf_link_is_up(peer);
  403. if (lnk_up &&
  404. !test_and_set_bit(PERF_STS_LNKUP, &peer->sts)) {
  405. perf_cmd_exec(peer, PERF_CMD_SSIZE);
  406. } else if (!lnk_up &&
  407. test_and_clear_bit(PERF_STS_LNKUP, &peer->sts)) {
  408. perf_cmd_exec(peer, PERF_CMD_CLEAR);
  409. }
  410. }
  411. }
  412. static void perf_db_event(void *ctx, int vec)
  413. {
  414. struct perf_ctx *perf = ctx;
  415. dev_dbg(&perf->ntb->dev, "DB vec %d mask %#llx bits %#llx\n", vec,
  416. ntb_db_vector_mask(perf->ntb, vec), ntb_db_read(perf->ntb));
  417. /* Just receive all available commands */
  418. (void)perf_cmd_recv(perf);
  419. }
  420. static void perf_msg_event(void *ctx)
  421. {
  422. struct perf_ctx *perf = ctx;
  423. dev_dbg(&perf->ntb->dev, "Msg status bits %#llx\n",
  424. ntb_msg_read_sts(perf->ntb));
  425. /* Messages are only sent one-by-one */
  426. (void)perf_cmd_recv(perf);
  427. }
  428. static const struct ntb_ctx_ops perf_ops = {
  429. .link_event = perf_link_event,
  430. .db_event = perf_db_event,
  431. .msg_event = perf_msg_event
  432. };
  433. static void perf_free_outbuf(struct perf_peer *peer)
  434. {
  435. (void)ntb_peer_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
  436. }
  437. static int perf_setup_outbuf(struct perf_peer *peer)
  438. {
  439. struct perf_ctx *perf = peer->perf;
  440. int ret;
  441. /* Outbuf size can be unaligned due to custom max_mw_size */
  442. ret = ntb_peer_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
  443. peer->outbuf_xlat, peer->outbuf_size);
  444. if (ret) {
  445. dev_err(&perf->ntb->dev, "Failed to set outbuf translation\n");
  446. return ret;
  447. }
  448. /* Initialization is finally done */
  449. set_bit(PERF_STS_DONE, &peer->sts);
  450. complete_all(&peer->init_comp);
  451. return 0;
  452. }
  453. static void perf_free_inbuf(struct perf_peer *peer)
  454. {
  455. if (!peer->inbuf)
  456. return;
  457. (void)ntb_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
  458. dma_free_coherent(&peer->perf->ntb->pdev->dev, peer->inbuf_size,
  459. peer->inbuf, peer->inbuf_xlat);
  460. peer->inbuf = NULL;
  461. }
  462. static int perf_setup_inbuf(struct perf_peer *peer)
  463. {
  464. resource_size_t xlat_align, size_align, size_max;
  465. struct perf_ctx *perf = peer->perf;
  466. int ret;
  467. /* Get inbound MW parameters */
  468. ret = ntb_mw_get_align(perf->ntb, peer->pidx, perf->gidx,
  469. &xlat_align, &size_align, &size_max);
  470. if (ret) {
  471. dev_err(&perf->ntb->dev, "Couldn't get inbuf restrictions\n");
  472. return ret;
  473. }
  474. if (peer->inbuf_size > size_max) {
  475. dev_err(&perf->ntb->dev, "Too big inbuf size %pa > %pa\n",
  476. &peer->inbuf_size, &size_max);
  477. return -EINVAL;
  478. }
  479. peer->inbuf_size = round_up(peer->inbuf_size, size_align);
  480. perf_free_inbuf(peer);
  481. peer->inbuf = dma_alloc_coherent(&perf->ntb->pdev->dev,
  482. peer->inbuf_size, &peer->inbuf_xlat,
  483. GFP_KERNEL);
  484. if (!peer->inbuf) {
  485. dev_err(&perf->ntb->dev, "Failed to alloc inbuf of %pa\n",
  486. &peer->inbuf_size);
  487. return -ENOMEM;
  488. }
  489. if (!IS_ALIGNED(peer->inbuf_xlat, xlat_align)) {
  490. ret = -EINVAL;
  491. dev_err(&perf->ntb->dev, "Unaligned inbuf allocated\n");
  492. goto err_free_inbuf;
  493. }
  494. ret = ntb_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
  495. peer->inbuf_xlat, peer->inbuf_size);
  496. if (ret) {
  497. dev_err(&perf->ntb->dev, "Failed to set inbuf translation\n");
  498. goto err_free_inbuf;
  499. }
  500. /*
  501. * We submit inbuf xlat transmission cmd for execution here to follow
  502. * the code architecture, even though this method is called from service
  503. * work itself so the command will be executed right after it returns.
  504. */
  505. (void)perf_cmd_exec(peer, PERF_CMD_SXLAT);
  506. return 0;
  507. err_free_inbuf:
  508. perf_free_inbuf(peer);
  509. return ret;
  510. }
  511. static void perf_service_work(struct work_struct *work)
  512. {
  513. struct perf_peer *peer = to_peer_service(work);
  514. if (test_and_clear_bit(PERF_CMD_SSIZE, &peer->sts))
  515. perf_cmd_send(peer, PERF_CMD_SSIZE, peer->outbuf_size);
  516. if (test_and_clear_bit(PERF_CMD_RSIZE, &peer->sts))
  517. perf_setup_inbuf(peer);
  518. if (test_and_clear_bit(PERF_CMD_SXLAT, &peer->sts))
  519. perf_cmd_send(peer, PERF_CMD_SXLAT, peer->inbuf_xlat);
  520. if (test_and_clear_bit(PERF_CMD_RXLAT, &peer->sts))
  521. perf_setup_outbuf(peer);
  522. if (test_and_clear_bit(PERF_CMD_CLEAR, &peer->sts)) {
  523. init_completion(&peer->init_comp);
  524. clear_bit(PERF_STS_DONE, &peer->sts);
  525. if (test_bit(0, &peer->perf->busy_flag) &&
  526. peer == peer->perf->test_peer) {
  527. dev_warn(&peer->perf->ntb->dev,
  528. "Freeing while test on-fly\n");
  529. perf_terminate_test(peer->perf);
  530. }
  531. perf_free_outbuf(peer);
  532. perf_free_inbuf(peer);
  533. }
  534. }
  535. static int perf_init_service(struct perf_ctx *perf)
  536. {
  537. u64 mask;
  538. if (ntb_peer_mw_count(perf->ntb) < perf->pcnt) {
  539. dev_err(&perf->ntb->dev, "Not enough memory windows\n");
  540. return -EINVAL;
  541. }
  542. if (ntb_msg_count(perf->ntb) >= PERF_MSG_CNT) {
  543. perf->cmd_send = perf_msg_cmd_send;
  544. perf->cmd_recv = perf_msg_cmd_recv;
  545. dev_dbg(&perf->ntb->dev, "Message service initialized\n");
  546. return 0;
  547. }
  548. dev_dbg(&perf->ntb->dev, "Message service unsupported\n");
  549. mask = GENMASK_ULL(perf->pcnt, 0);
  550. if (ntb_spad_count(perf->ntb) >= PERF_SPAD_CNT(perf->pcnt) &&
  551. (ntb_db_valid_mask(perf->ntb) & mask) == mask) {
  552. perf->cmd_send = perf_spad_cmd_send;
  553. perf->cmd_recv = perf_spad_cmd_recv;
  554. dev_dbg(&perf->ntb->dev, "Scratchpad service initialized\n");
  555. return 0;
  556. }
  557. dev_dbg(&perf->ntb->dev, "Scratchpad service unsupported\n");
  558. dev_err(&perf->ntb->dev, "Command services unsupported\n");
  559. return -EINVAL;
  560. }
  561. static int perf_enable_service(struct perf_ctx *perf)
  562. {
  563. u64 mask, incmd_bit;
  564. int ret, sidx, scnt;
  565. mask = ntb_db_valid_mask(perf->ntb);
  566. (void)ntb_db_set_mask(perf->ntb, mask);
  567. ret = ntb_set_ctx(perf->ntb, perf, &perf_ops);
  568. if (ret)
  569. return ret;
  570. if (perf->cmd_send == perf_msg_cmd_send) {
  571. u64 inbits, outbits;
  572. inbits = ntb_msg_inbits(perf->ntb);
  573. outbits = ntb_msg_outbits(perf->ntb);
  574. (void)ntb_msg_set_mask(perf->ntb, inbits | outbits);
  575. incmd_bit = BIT_ULL(__ffs64(inbits));
  576. ret = ntb_msg_clear_mask(perf->ntb, incmd_bit);
  577. dev_dbg(&perf->ntb->dev, "MSG sts unmasked %#llx\n", incmd_bit);
  578. } else {
  579. scnt = ntb_spad_count(perf->ntb);
  580. for (sidx = 0; sidx < scnt; sidx++)
  581. ntb_spad_write(perf->ntb, sidx, PERF_CMD_INVAL);
  582. incmd_bit = PERF_SPAD_NOTIFY(perf->gidx);
  583. ret = ntb_db_clear_mask(perf->ntb, incmd_bit);
  584. dev_dbg(&perf->ntb->dev, "DB bits unmasked %#llx\n", incmd_bit);
  585. }
  586. if (ret) {
  587. ntb_clear_ctx(perf->ntb);
  588. return ret;
  589. }
  590. ntb_link_enable(perf->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
  591. /* Might be not necessary */
  592. ntb_link_event(perf->ntb);
  593. return 0;
  594. }
  595. static void perf_disable_service(struct perf_ctx *perf)
  596. {
  597. int pidx;
  598. if (perf->cmd_send == perf_msg_cmd_send) {
  599. u64 inbits;
  600. inbits = ntb_msg_inbits(perf->ntb);
  601. (void)ntb_msg_set_mask(perf->ntb, inbits);
  602. } else {
  603. (void)ntb_db_set_mask(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
  604. }
  605. ntb_clear_ctx(perf->ntb);
  606. for (pidx = 0; pidx < perf->pcnt; pidx++)
  607. perf_cmd_exec(&perf->peers[pidx], PERF_CMD_CLEAR);
  608. for (pidx = 0; pidx < perf->pcnt; pidx++)
  609. flush_work(&perf->peers[pidx].service);
  610. for (pidx = 0; pidx < perf->pcnt; pidx++) {
  611. struct perf_peer *peer = &perf->peers[pidx];
  612. ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx), 0);
  613. }
  614. ntb_db_clear(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
  615. ntb_link_disable(perf->ntb);
  616. }
  617. /*==============================================================================
  618. * Performance measuring work-thread
  619. *==============================================================================
  620. */
  621. static void perf_dma_copy_callback(void *data)
  622. {
  623. struct perf_thread *pthr = data;
  624. atomic_dec(&pthr->dma_sync);
  625. wake_up(&pthr->dma_wait);
  626. }
  627. static int perf_copy_chunk(struct perf_thread *pthr,
  628. void __iomem *dst, void *src, size_t len)
  629. {
  630. struct dma_async_tx_descriptor *tx;
  631. struct dmaengine_unmap_data *unmap;
  632. struct device *dma_dev;
  633. int try = 0, ret = 0;
  634. struct perf_peer *peer = pthr->perf->test_peer;
  635. void __iomem *vbase;
  636. void __iomem *dst_vaddr;
  637. dma_addr_t dst_dma_addr;
  638. if (!use_dma) {
  639. memcpy_toio(dst, src, len);
  640. goto ret_check_tsync;
  641. }
  642. dma_dev = pthr->dma_chan->device->dev;
  643. if (!is_dma_copy_aligned(pthr->dma_chan->device, offset_in_page(src),
  644. offset_in_page(dst), len))
  645. return -EIO;
  646. vbase = peer->outbuf;
  647. dst_vaddr = dst;
  648. dst_dma_addr = peer->dma_dst_addr + (dst_vaddr - vbase);
  649. unmap = dmaengine_get_unmap_data(dma_dev, 1, GFP_NOWAIT);
  650. if (!unmap)
  651. return -ENOMEM;
  652. unmap->len = len;
  653. unmap->addr[0] = dma_map_page(dma_dev, virt_to_page(src),
  654. offset_in_page(src), len, DMA_TO_DEVICE);
  655. if (dma_mapping_error(dma_dev, unmap->addr[0])) {
  656. ret = -EIO;
  657. goto err_free_resource;
  658. }
  659. unmap->to_cnt = 1;
  660. do {
  661. tx = dmaengine_prep_dma_memcpy(pthr->dma_chan, dst_dma_addr,
  662. unmap->addr[0], len, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
  663. if (!tx)
  664. msleep(DMA_MDELAY);
  665. } while (!tx && (try++ < DMA_TRIES));
  666. if (!tx) {
  667. ret = -EIO;
  668. goto err_free_resource;
  669. }
  670. tx->callback = perf_dma_copy_callback;
  671. tx->callback_param = pthr;
  672. dma_set_unmap(tx, unmap);
  673. ret = dma_submit_error(dmaengine_submit(tx));
  674. if (ret) {
  675. dmaengine_unmap_put(unmap);
  676. goto err_free_resource;
  677. }
  678. dmaengine_unmap_put(unmap);
  679. atomic_inc(&pthr->dma_sync);
  680. dma_async_issue_pending(pthr->dma_chan);
  681. ret_check_tsync:
  682. return likely(atomic_read(&pthr->perf->tsync) > 0) ? 0 : -EINTR;
  683. err_free_resource:
  684. dmaengine_unmap_put(unmap);
  685. return ret;
  686. }
  687. static bool perf_dma_filter(struct dma_chan *chan, void *data)
  688. {
  689. struct perf_ctx *perf = data;
  690. int node;
  691. node = dev_to_node(&perf->ntb->dev);
  692. return node == NUMA_NO_NODE || node == dev_to_node(chan->device->dev);
  693. }
  694. static int perf_init_test(struct perf_thread *pthr)
  695. {
  696. struct perf_ctx *perf = pthr->perf;
  697. dma_cap_mask_t dma_mask;
  698. struct perf_peer *peer = pthr->perf->test_peer;
  699. pthr->src = kmalloc_node(perf->test_peer->outbuf_size, GFP_KERNEL,
  700. dev_to_node(&perf->ntb->dev));
  701. if (!pthr->src)
  702. return -ENOMEM;
  703. get_random_bytes(pthr->src, perf->test_peer->outbuf_size);
  704. if (!use_dma)
  705. return 0;
  706. dma_cap_zero(dma_mask);
  707. dma_cap_set(DMA_MEMCPY, dma_mask);
  708. pthr->dma_chan = dma_request_channel(dma_mask, perf_dma_filter, perf);
  709. if (!pthr->dma_chan) {
  710. dev_err(&perf->ntb->dev, "%d: Failed to get DMA channel\n",
  711. pthr->tidx);
  712. goto err_free;
  713. }
  714. peer->dma_dst_addr =
  715. dma_map_resource(pthr->dma_chan->device->dev,
  716. peer->out_phys_addr, peer->outbuf_size,
  717. DMA_FROM_DEVICE, 0);
  718. if (dma_mapping_error(pthr->dma_chan->device->dev,
  719. peer->dma_dst_addr)) {
  720. dev_err(pthr->dma_chan->device->dev, "%d: Failed to map DMA addr\n",
  721. pthr->tidx);
  722. peer->dma_dst_addr = 0;
  723. dma_release_channel(pthr->dma_chan);
  724. goto err_free;
  725. }
  726. dev_dbg(pthr->dma_chan->device->dev, "%d: Map MMIO %pa to DMA addr %pad\n",
  727. pthr->tidx,
  728. &peer->out_phys_addr,
  729. &peer->dma_dst_addr);
  730. atomic_set(&pthr->dma_sync, 0);
  731. return 0;
  732. err_free:
  733. atomic_dec(&perf->tsync);
  734. wake_up(&perf->twait);
  735. kfree(pthr->src);
  736. return -ENODEV;
  737. }
  738. static int perf_run_test(struct perf_thread *pthr)
  739. {
  740. struct perf_peer *peer = pthr->perf->test_peer;
  741. struct perf_ctx *perf = pthr->perf;
  742. void __iomem *flt_dst, *bnd_dst;
  743. u64 total_size, chunk_size;
  744. void *flt_src;
  745. int ret = 0;
  746. total_size = 1ULL << total_order;
  747. chunk_size = 1ULL << chunk_order;
  748. chunk_size = min_t(u64, peer->outbuf_size, chunk_size);
  749. flt_src = pthr->src;
  750. bnd_dst = peer->outbuf + peer->outbuf_size;
  751. flt_dst = peer->outbuf;
  752. pthr->duration = ktime_get();
  753. /* Copied field is cleared on test launch stage */
  754. while (pthr->copied < total_size) {
  755. ret = perf_copy_chunk(pthr, flt_dst, flt_src, chunk_size);
  756. if (ret) {
  757. dev_err(&perf->ntb->dev, "%d: Got error %d on test\n",
  758. pthr->tidx, ret);
  759. return ret;
  760. }
  761. pthr->copied += chunk_size;
  762. flt_dst += chunk_size;
  763. flt_src += chunk_size;
  764. if (flt_dst >= bnd_dst || flt_dst < peer->outbuf) {
  765. flt_dst = peer->outbuf;
  766. flt_src = pthr->src;
  767. }
  768. /* Give up CPU to give a chance for other threads to use it */
  769. schedule();
  770. }
  771. return 0;
  772. }
  773. static int perf_sync_test(struct perf_thread *pthr)
  774. {
  775. struct perf_ctx *perf = pthr->perf;
  776. if (!use_dma)
  777. goto no_dma_ret;
  778. wait_event(pthr->dma_wait,
  779. (atomic_read(&pthr->dma_sync) == 0 ||
  780. atomic_read(&perf->tsync) < 0));
  781. if (atomic_read(&perf->tsync) < 0)
  782. return -EINTR;
  783. no_dma_ret:
  784. pthr->duration = ktime_sub(ktime_get(), pthr->duration);
  785. dev_dbg(&perf->ntb->dev, "%d: copied %llu bytes\n",
  786. pthr->tidx, pthr->copied);
  787. dev_dbg(&perf->ntb->dev, "%d: lasted %llu usecs\n",
  788. pthr->tidx, ktime_to_us(pthr->duration));
  789. dev_dbg(&perf->ntb->dev, "%d: %llu MBytes/s\n", pthr->tidx,
  790. div64_u64(pthr->copied, ktime_to_us(pthr->duration)));
  791. return 0;
  792. }
  793. static void perf_clear_test(struct perf_thread *pthr)
  794. {
  795. struct perf_ctx *perf = pthr->perf;
  796. if (!use_dma)
  797. goto no_dma_notify;
  798. /*
  799. * If test finished without errors, termination isn't needed.
  800. * We call it anyway just to be sure of the transfers completion.
  801. */
  802. (void)dmaengine_terminate_sync(pthr->dma_chan);
  803. if (pthr->perf->test_peer->dma_dst_addr)
  804. dma_unmap_resource(pthr->dma_chan->device->dev,
  805. pthr->perf->test_peer->dma_dst_addr,
  806. pthr->perf->test_peer->outbuf_size,
  807. DMA_FROM_DEVICE, 0);
  808. dma_release_channel(pthr->dma_chan);
  809. no_dma_notify:
  810. atomic_dec(&perf->tsync);
  811. wake_up(&perf->twait);
  812. kfree(pthr->src);
  813. }
  814. static void perf_thread_work(struct work_struct *work)
  815. {
  816. struct perf_thread *pthr = to_thread_work(work);
  817. int ret;
  818. /*
  819. * Perform stages in compliance with use_dma flag value.
  820. * Test status is changed only if error happened, otherwise
  821. * status -ENODATA is kept while test is on-fly. Results
  822. * synchronization is performed only if test fininshed
  823. * without an error or interruption.
  824. */
  825. ret = perf_init_test(pthr);
  826. if (ret) {
  827. pthr->status = ret;
  828. return;
  829. }
  830. ret = perf_run_test(pthr);
  831. if (ret) {
  832. pthr->status = ret;
  833. goto err_clear_test;
  834. }
  835. pthr->status = perf_sync_test(pthr);
  836. err_clear_test:
  837. perf_clear_test(pthr);
  838. }
  839. static int perf_set_tcnt(struct perf_ctx *perf, u8 tcnt)
  840. {
  841. if (tcnt == 0 || tcnt > MAX_THREADS_CNT)
  842. return -EINVAL;
  843. if (test_and_set_bit_lock(0, &perf->busy_flag))
  844. return -EBUSY;
  845. perf->tcnt = tcnt;
  846. clear_bit_unlock(0, &perf->busy_flag);
  847. return 0;
  848. }
  849. static void perf_terminate_test(struct perf_ctx *perf)
  850. {
  851. int tidx;
  852. atomic_set(&perf->tsync, -1);
  853. wake_up(&perf->twait);
  854. for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
  855. wake_up(&perf->threads[tidx].dma_wait);
  856. cancel_work_sync(&perf->threads[tidx].work);
  857. }
  858. }
  859. static int perf_submit_test(struct perf_peer *peer)
  860. {
  861. struct perf_ctx *perf = peer->perf;
  862. struct perf_thread *pthr;
  863. int tidx, ret;
  864. ret = wait_for_completion_interruptible(&peer->init_comp);
  865. if (ret < 0)
  866. return ret;
  867. if (test_and_set_bit_lock(0, &perf->busy_flag))
  868. return -EBUSY;
  869. perf->test_peer = peer;
  870. atomic_set(&perf->tsync, perf->tcnt);
  871. for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
  872. pthr = &perf->threads[tidx];
  873. pthr->status = -ENODATA;
  874. pthr->copied = 0;
  875. pthr->duration = ktime_set(0, 0);
  876. if (tidx < perf->tcnt)
  877. (void)queue_work(perf_wq, &pthr->work);
  878. }
  879. ret = wait_event_interruptible(perf->twait,
  880. atomic_read(&perf->tsync) <= 0);
  881. if (ret == -ERESTARTSYS) {
  882. perf_terminate_test(perf);
  883. ret = -EINTR;
  884. }
  885. clear_bit_unlock(0, &perf->busy_flag);
  886. return ret;
  887. }
  888. static int perf_read_stats(struct perf_ctx *perf, char *buf,
  889. size_t size, ssize_t *pos)
  890. {
  891. struct perf_thread *pthr;
  892. int tidx;
  893. if (test_and_set_bit_lock(0, &perf->busy_flag))
  894. return -EBUSY;
  895. (*pos) += scnprintf(buf + *pos, size - *pos,
  896. " Peer %d test statistics:\n", perf->test_peer->pidx);
  897. for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
  898. pthr = &perf->threads[tidx];
  899. if (pthr->status == -ENODATA)
  900. continue;
  901. if (pthr->status) {
  902. (*pos) += scnprintf(buf + *pos, size - *pos,
  903. "%d: error status %d\n", tidx, pthr->status);
  904. continue;
  905. }
  906. (*pos) += scnprintf(buf + *pos, size - *pos,
  907. "%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n",
  908. tidx, pthr->copied, ktime_to_us(pthr->duration),
  909. div64_u64(pthr->copied, ktime_to_us(pthr->duration)));
  910. }
  911. clear_bit_unlock(0, &perf->busy_flag);
  912. return 0;
  913. }
  914. static void perf_init_threads(struct perf_ctx *perf)
  915. {
  916. struct perf_thread *pthr;
  917. int tidx;
  918. perf->tcnt = DEF_THREADS_CNT;
  919. perf->test_peer = &perf->peers[0];
  920. init_waitqueue_head(&perf->twait);
  921. for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
  922. pthr = &perf->threads[tidx];
  923. pthr->perf = perf;
  924. pthr->tidx = tidx;
  925. pthr->status = -ENODATA;
  926. init_waitqueue_head(&pthr->dma_wait);
  927. INIT_WORK(&pthr->work, perf_thread_work);
  928. }
  929. }
  930. static void perf_clear_threads(struct perf_ctx *perf)
  931. {
  932. perf_terminate_test(perf);
  933. }
  934. /*==============================================================================
  935. * DebugFS nodes
  936. *==============================================================================
  937. */
  938. static ssize_t perf_dbgfs_read_info(struct file *filep, char __user *ubuf,
  939. size_t size, loff_t *offp)
  940. {
  941. struct perf_ctx *perf = filep->private_data;
  942. struct perf_peer *peer;
  943. size_t buf_size;
  944. ssize_t pos = 0;
  945. int ret, pidx;
  946. char *buf;
  947. buf_size = min_t(size_t, size, 0x1000U);
  948. buf = kmalloc(buf_size, GFP_KERNEL);
  949. if (!buf)
  950. return -ENOMEM;
  951. pos += scnprintf(buf + pos, buf_size - pos,
  952. " Performance measuring tool info:\n\n");
  953. pos += scnprintf(buf + pos, buf_size - pos,
  954. "Local port %d, Global index %d\n", ntb_port_number(perf->ntb),
  955. perf->gidx);
  956. pos += scnprintf(buf + pos, buf_size - pos, "Test status: ");
  957. if (test_bit(0, &perf->busy_flag)) {
  958. pos += scnprintf(buf + pos, buf_size - pos,
  959. "on-fly with port %d (%d)\n",
  960. ntb_peer_port_number(perf->ntb, perf->test_peer->pidx),
  961. perf->test_peer->pidx);
  962. } else {
  963. pos += scnprintf(buf + pos, buf_size - pos, "idle\n");
  964. }
  965. for (pidx = 0; pidx < perf->pcnt; pidx++) {
  966. peer = &perf->peers[pidx];
  967. pos += scnprintf(buf + pos, buf_size - pos,
  968. "Port %d (%d), Global index %d:\n",
  969. ntb_peer_port_number(perf->ntb, peer->pidx), peer->pidx,
  970. peer->gidx);
  971. pos += scnprintf(buf + pos, buf_size - pos,
  972. "\tLink status: %s\n",
  973. test_bit(PERF_STS_LNKUP, &peer->sts) ? "up" : "down");
  974. pos += scnprintf(buf + pos, buf_size - pos,
  975. "\tOut buffer addr 0x%pK\n", peer->outbuf);
  976. pos += scnprintf(buf + pos, buf_size - pos,
  977. "\tOut buff phys addr %pa[p]\n", &peer->out_phys_addr);
  978. pos += scnprintf(buf + pos, buf_size - pos,
  979. "\tOut buffer size %pa\n", &peer->outbuf_size);
  980. pos += scnprintf(buf + pos, buf_size - pos,
  981. "\tOut buffer xlat 0x%016llx[p]\n", peer->outbuf_xlat);
  982. if (!peer->inbuf) {
  983. pos += scnprintf(buf + pos, buf_size - pos,
  984. "\tIn buffer addr: unallocated\n");
  985. continue;
  986. }
  987. pos += scnprintf(buf + pos, buf_size - pos,
  988. "\tIn buffer addr 0x%pK\n", peer->inbuf);
  989. pos += scnprintf(buf + pos, buf_size - pos,
  990. "\tIn buffer size %pa\n", &peer->inbuf_size);
  991. pos += scnprintf(buf + pos, buf_size - pos,
  992. "\tIn buffer xlat %pad[p]\n", &peer->inbuf_xlat);
  993. }
  994. ret = simple_read_from_buffer(ubuf, size, offp, buf, pos);
  995. kfree(buf);
  996. return ret;
  997. }
  998. static const struct file_operations perf_dbgfs_info = {
  999. .open = simple_open,
  1000. .read = perf_dbgfs_read_info
  1001. };
  1002. static ssize_t perf_dbgfs_read_run(struct file *filep, char __user *ubuf,
  1003. size_t size, loff_t *offp)
  1004. {
  1005. struct perf_ctx *perf = filep->private_data;
  1006. ssize_t ret, pos = 0;
  1007. char *buf;
  1008. buf = kmalloc(PERF_BUF_LEN, GFP_KERNEL);
  1009. if (!buf)
  1010. return -ENOMEM;
  1011. ret = perf_read_stats(perf, buf, PERF_BUF_LEN, &pos);
  1012. if (ret)
  1013. goto err_free;
  1014. ret = simple_read_from_buffer(ubuf, size, offp, buf, pos);
  1015. err_free:
  1016. kfree(buf);
  1017. return ret;
  1018. }
  1019. static ssize_t perf_dbgfs_write_run(struct file *filep, const char __user *ubuf,
  1020. size_t size, loff_t *offp)
  1021. {
  1022. struct perf_ctx *perf = filep->private_data;
  1023. struct perf_peer *peer;
  1024. int pidx, ret;
  1025. ret = kstrtoint_from_user(ubuf, size, 0, &pidx);
  1026. if (ret)
  1027. return ret;
  1028. if (pidx < 0 || pidx >= perf->pcnt)
  1029. return -EINVAL;
  1030. peer = &perf->peers[pidx];
  1031. ret = perf_submit_test(peer);
  1032. if (ret)
  1033. return ret;
  1034. return size;
  1035. }
  1036. static const struct file_operations perf_dbgfs_run = {
  1037. .open = simple_open,
  1038. .read = perf_dbgfs_read_run,
  1039. .write = perf_dbgfs_write_run
  1040. };
  1041. static ssize_t perf_dbgfs_read_tcnt(struct file *filep, char __user *ubuf,
  1042. size_t size, loff_t *offp)
  1043. {
  1044. struct perf_ctx *perf = filep->private_data;
  1045. char buf[8];
  1046. ssize_t pos;
  1047. pos = scnprintf(buf, sizeof(buf), "%hhu\n", perf->tcnt);
  1048. return simple_read_from_buffer(ubuf, size, offp, buf, pos);
  1049. }
  1050. static ssize_t perf_dbgfs_write_tcnt(struct file *filep,
  1051. const char __user *ubuf,
  1052. size_t size, loff_t *offp)
  1053. {
  1054. struct perf_ctx *perf = filep->private_data;
  1055. int ret;
  1056. u8 val;
  1057. ret = kstrtou8_from_user(ubuf, size, 0, &val);
  1058. if (ret)
  1059. return ret;
  1060. ret = perf_set_tcnt(perf, val);
  1061. if (ret)
  1062. return ret;
  1063. return size;
  1064. }
  1065. static const struct file_operations perf_dbgfs_tcnt = {
  1066. .open = simple_open,
  1067. .read = perf_dbgfs_read_tcnt,
  1068. .write = perf_dbgfs_write_tcnt
  1069. };
  1070. static void perf_setup_dbgfs(struct perf_ctx *perf)
  1071. {
  1072. struct pci_dev *pdev = perf->ntb->pdev;
  1073. perf->dbgfs_dir = debugfs_create_dir(pci_name(pdev), perf_dbgfs_topdir);
  1074. if (!perf->dbgfs_dir) {
  1075. dev_warn(&perf->ntb->dev, "DebugFS unsupported\n");
  1076. return;
  1077. }
  1078. debugfs_create_file("info", 0600, perf->dbgfs_dir, perf,
  1079. &perf_dbgfs_info);
  1080. debugfs_create_file("run", 0600, perf->dbgfs_dir, perf,
  1081. &perf_dbgfs_run);
  1082. debugfs_create_file("threads_count", 0600, perf->dbgfs_dir, perf,
  1083. &perf_dbgfs_tcnt);
  1084. /* They are made read-only for test exec safety and integrity */
  1085. debugfs_create_u8("chunk_order", 0500, perf->dbgfs_dir, &chunk_order);
  1086. debugfs_create_u8("total_order", 0500, perf->dbgfs_dir, &total_order);
  1087. debugfs_create_bool("use_dma", 0500, perf->dbgfs_dir, &use_dma);
  1088. }
  1089. static void perf_clear_dbgfs(struct perf_ctx *perf)
  1090. {
  1091. debugfs_remove_recursive(perf->dbgfs_dir);
  1092. }
  1093. /*==============================================================================
  1094. * Basic driver initialization
  1095. *==============================================================================
  1096. */
  1097. static struct perf_ctx *perf_create_data(struct ntb_dev *ntb)
  1098. {
  1099. struct perf_ctx *perf;
  1100. perf = devm_kzalloc(&ntb->dev, sizeof(*perf), GFP_KERNEL);
  1101. if (!perf)
  1102. return ERR_PTR(-ENOMEM);
  1103. perf->pcnt = ntb_peer_port_count(ntb);
  1104. perf->peers = devm_kcalloc(&ntb->dev, perf->pcnt, sizeof(*perf->peers),
  1105. GFP_KERNEL);
  1106. if (!perf->peers)
  1107. return ERR_PTR(-ENOMEM);
  1108. perf->ntb = ntb;
  1109. return perf;
  1110. }
  1111. static int perf_setup_peer_mw(struct perf_peer *peer)
  1112. {
  1113. struct perf_ctx *perf = peer->perf;
  1114. phys_addr_t phys_addr;
  1115. int ret;
  1116. /* Get outbound MW parameters and map it */
  1117. ret = ntb_peer_mw_get_addr(perf->ntb, perf->gidx, &phys_addr,
  1118. &peer->outbuf_size);
  1119. if (ret)
  1120. return ret;
  1121. peer->outbuf = devm_ioremap_wc(&perf->ntb->dev, phys_addr,
  1122. peer->outbuf_size);
  1123. if (!peer->outbuf)
  1124. return -ENOMEM;
  1125. peer->out_phys_addr = phys_addr;
  1126. if (max_mw_size && peer->outbuf_size > max_mw_size) {
  1127. peer->outbuf_size = max_mw_size;
  1128. dev_warn(&peer->perf->ntb->dev,
  1129. "Peer %d outbuf reduced to %pa\n", peer->pidx,
  1130. &peer->outbuf_size);
  1131. }
  1132. return 0;
  1133. }
  1134. static int perf_init_peers(struct perf_ctx *perf)
  1135. {
  1136. struct perf_peer *peer;
  1137. int pidx, lport, ret;
  1138. lport = ntb_port_number(perf->ntb);
  1139. perf->gidx = -1;
  1140. for (pidx = 0; pidx < perf->pcnt; pidx++) {
  1141. peer = &perf->peers[pidx];
  1142. peer->perf = perf;
  1143. peer->pidx = pidx;
  1144. if (lport < ntb_peer_port_number(perf->ntb, pidx)) {
  1145. if (perf->gidx == -1)
  1146. perf->gidx = pidx;
  1147. peer->gidx = pidx + 1;
  1148. } else {
  1149. peer->gidx = pidx;
  1150. }
  1151. INIT_WORK(&peer->service, perf_service_work);
  1152. init_completion(&peer->init_comp);
  1153. }
  1154. if (perf->gidx == -1)
  1155. perf->gidx = pidx;
  1156. /*
  1157. * Hardware with only two ports may not have unique port
  1158. * numbers. In this case, the gidxs should all be zero.
  1159. */
  1160. if (perf->pcnt == 1 && ntb_port_number(perf->ntb) == 0 &&
  1161. ntb_peer_port_number(perf->ntb, 0) == 0) {
  1162. perf->gidx = 0;
  1163. perf->peers[0].gidx = 0;
  1164. }
  1165. for (pidx = 0; pidx < perf->pcnt; pidx++) {
  1166. ret = perf_setup_peer_mw(&perf->peers[pidx]);
  1167. if (ret)
  1168. return ret;
  1169. }
  1170. dev_dbg(&perf->ntb->dev, "Global port index %d\n", perf->gidx);
  1171. return 0;
  1172. }
  1173. static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb)
  1174. {
  1175. struct perf_ctx *perf;
  1176. int ret;
  1177. perf = perf_create_data(ntb);
  1178. if (IS_ERR(perf))
  1179. return PTR_ERR(perf);
  1180. ret = perf_init_peers(perf);
  1181. if (ret)
  1182. return ret;
  1183. perf_init_threads(perf);
  1184. ret = perf_init_service(perf);
  1185. if (ret)
  1186. return ret;
  1187. ret = perf_enable_service(perf);
  1188. if (ret)
  1189. return ret;
  1190. perf_setup_dbgfs(perf);
  1191. return 0;
  1192. }
  1193. static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb)
  1194. {
  1195. struct perf_ctx *perf = ntb->ctx;
  1196. perf_clear_dbgfs(perf);
  1197. perf_disable_service(perf);
  1198. perf_clear_threads(perf);
  1199. }
  1200. static struct ntb_client perf_client = {
  1201. .ops = {
  1202. .probe = perf_probe,
  1203. .remove = perf_remove
  1204. }
  1205. };
  1206. static int __init perf_init(void)
  1207. {
  1208. int ret;
  1209. if (chunk_order > MAX_CHUNK_ORDER) {
  1210. chunk_order = MAX_CHUNK_ORDER;
  1211. pr_info("Chunk order reduced to %hhu\n", chunk_order);
  1212. }
  1213. if (total_order < chunk_order) {
  1214. total_order = chunk_order;
  1215. pr_info("Total data order reduced to %hhu\n", total_order);
  1216. }
  1217. perf_wq = alloc_workqueue("perf_wq", WQ_UNBOUND | WQ_SYSFS, 0);
  1218. if (!perf_wq)
  1219. return -ENOMEM;
  1220. if (debugfs_initialized())
  1221. perf_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
  1222. ret = ntb_register_client(&perf_client);
  1223. if (ret) {
  1224. debugfs_remove_recursive(perf_dbgfs_topdir);
  1225. destroy_workqueue(perf_wq);
  1226. }
  1227. return ret;
  1228. }
  1229. module_init(perf_init);
  1230. static void __exit perf_exit(void)
  1231. {
  1232. ntb_unregister_client(&perf_client);
  1233. debugfs_remove_recursive(perf_dbgfs_topdir);
  1234. destroy_workqueue(perf_wq);
  1235. }
  1236. module_exit(perf_exit);