dfl-fme-perf.c 30 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Driver for FPGA Management Engine (FME) Global Performance Reporting
  4. *
  5. * Copyright 2019 Intel Corporation, Inc.
  6. *
  7. * Authors:
  8. * Kang Luwei <luwei.kang@intel.com>
  9. * Xiao Guangrong <guangrong.xiao@linux.intel.com>
  10. * Wu Hao <hao.wu@intel.com>
  11. * Xu Yilun <yilun.xu@intel.com>
  12. * Joseph Grecco <joe.grecco@intel.com>
  13. * Enno Luebbers <enno.luebbers@intel.com>
  14. * Tim Whisonant <tim.whisonant@intel.com>
  15. * Ananda Ravuri <ananda.ravuri@intel.com>
  16. * Mitchel, Henry <henry.mitchel@intel.com>
  17. */
  18. #include <linux/perf_event.h>
  19. #include "dfl.h"
  20. #include "dfl-fme.h"
  21. /*
  22. * Performance Counter Registers for Cache.
  23. *
  24. * Cache Events are listed below as CACHE_EVNT_*.
  25. */
  26. #define CACHE_CTRL 0x8
  27. #define CACHE_RESET_CNTR BIT_ULL(0)
  28. #define CACHE_FREEZE_CNTR BIT_ULL(8)
  29. #define CACHE_CTRL_EVNT GENMASK_ULL(19, 16)
  30. #define CACHE_EVNT_RD_HIT 0x0
  31. #define CACHE_EVNT_WR_HIT 0x1
  32. #define CACHE_EVNT_RD_MISS 0x2
  33. #define CACHE_EVNT_WR_MISS 0x3
  34. #define CACHE_EVNT_RSVD 0x4
  35. #define CACHE_EVNT_HOLD_REQ 0x5
  36. #define CACHE_EVNT_DATA_WR_PORT_CONTEN 0x6
  37. #define CACHE_EVNT_TAG_WR_PORT_CONTEN 0x7
  38. #define CACHE_EVNT_TX_REQ_STALL 0x8
  39. #define CACHE_EVNT_RX_REQ_STALL 0x9
  40. #define CACHE_EVNT_EVICTIONS 0xa
  41. #define CACHE_EVNT_MAX CACHE_EVNT_EVICTIONS
  42. #define CACHE_CHANNEL_SEL BIT_ULL(20)
  43. #define CACHE_CHANNEL_RD 0
  44. #define CACHE_CHANNEL_WR 1
  45. #define CACHE_CNTR0 0x10
  46. #define CACHE_CNTR1 0x18
  47. #define CACHE_CNTR_EVNT_CNTR GENMASK_ULL(47, 0)
  48. #define CACHE_CNTR_EVNT GENMASK_ULL(63, 60)
  49. /*
  50. * Performance Counter Registers for Fabric.
  51. *
  52. * Fabric Events are listed below as FAB_EVNT_*
  53. */
  54. #define FAB_CTRL 0x20
  55. #define FAB_RESET_CNTR BIT_ULL(0)
  56. #define FAB_FREEZE_CNTR BIT_ULL(8)
  57. #define FAB_CTRL_EVNT GENMASK_ULL(19, 16)
  58. #define FAB_EVNT_PCIE0_RD 0x0
  59. #define FAB_EVNT_PCIE0_WR 0x1
  60. #define FAB_EVNT_PCIE1_RD 0x2
  61. #define FAB_EVNT_PCIE1_WR 0x3
  62. #define FAB_EVNT_UPI_RD 0x4
  63. #define FAB_EVNT_UPI_WR 0x5
  64. #define FAB_EVNT_MMIO_RD 0x6
  65. #define FAB_EVNT_MMIO_WR 0x7
  66. #define FAB_EVNT_MAX FAB_EVNT_MMIO_WR
  67. #define FAB_PORT_ID GENMASK_ULL(21, 20)
  68. #define FAB_PORT_FILTER BIT_ULL(23)
  69. #define FAB_PORT_FILTER_DISABLE 0
  70. #define FAB_PORT_FILTER_ENABLE 1
  71. #define FAB_CNTR 0x28
  72. #define FAB_CNTR_EVNT_CNTR GENMASK_ULL(59, 0)
  73. #define FAB_CNTR_EVNT GENMASK_ULL(63, 60)
  74. /*
  75. * Performance Counter Registers for Clock.
  76. *
  77. * Clock Counter can't be reset or frozen by SW.
  78. */
  79. #define CLK_CNTR 0x30
  80. #define BASIC_EVNT_CLK 0x0
  81. #define BASIC_EVNT_MAX BASIC_EVNT_CLK
  82. /*
  83. * Performance Counter Registers for IOMMU / VT-D.
  84. *
  85. * VT-D Events are listed below as VTD_EVNT_* and VTD_SIP_EVNT_*
  86. */
  87. #define VTD_CTRL 0x38
  88. #define VTD_RESET_CNTR BIT_ULL(0)
  89. #define VTD_FREEZE_CNTR BIT_ULL(8)
  90. #define VTD_CTRL_EVNT GENMASK_ULL(19, 16)
  91. #define VTD_EVNT_AFU_MEM_RD_TRANS 0x0
  92. #define VTD_EVNT_AFU_MEM_WR_TRANS 0x1
  93. #define VTD_EVNT_AFU_DEVTLB_RD_HIT 0x2
  94. #define VTD_EVNT_AFU_DEVTLB_WR_HIT 0x3
  95. #define VTD_EVNT_DEVTLB_4K_FILL 0x4
  96. #define VTD_EVNT_DEVTLB_2M_FILL 0x5
  97. #define VTD_EVNT_DEVTLB_1G_FILL 0x6
  98. #define VTD_EVNT_MAX VTD_EVNT_DEVTLB_1G_FILL
  99. #define VTD_CNTR 0x40
  100. #define VTD_CNTR_EVNT_CNTR GENMASK_ULL(47, 0)
  101. #define VTD_CNTR_EVNT GENMASK_ULL(63, 60)
  102. #define VTD_SIP_CTRL 0x48
  103. #define VTD_SIP_RESET_CNTR BIT_ULL(0)
  104. #define VTD_SIP_FREEZE_CNTR BIT_ULL(8)
  105. #define VTD_SIP_CTRL_EVNT GENMASK_ULL(19, 16)
  106. #define VTD_SIP_EVNT_IOTLB_4K_HIT 0x0
  107. #define VTD_SIP_EVNT_IOTLB_2M_HIT 0x1
  108. #define VTD_SIP_EVNT_IOTLB_1G_HIT 0x2
  109. #define VTD_SIP_EVNT_SLPWC_L3_HIT 0x3
  110. #define VTD_SIP_EVNT_SLPWC_L4_HIT 0x4
  111. #define VTD_SIP_EVNT_RCC_HIT 0x5
  112. #define VTD_SIP_EVNT_IOTLB_4K_MISS 0x6
  113. #define VTD_SIP_EVNT_IOTLB_2M_MISS 0x7
  114. #define VTD_SIP_EVNT_IOTLB_1G_MISS 0x8
  115. #define VTD_SIP_EVNT_SLPWC_L3_MISS 0x9
  116. #define VTD_SIP_EVNT_SLPWC_L4_MISS 0xa
  117. #define VTD_SIP_EVNT_RCC_MISS 0xb
  118. #define VTD_SIP_EVNT_MAX VTD_SIP_EVNT_SLPWC_L4_MISS
  119. #define VTD_SIP_CNTR 0X50
  120. #define VTD_SIP_CNTR_EVNT_CNTR GENMASK_ULL(47, 0)
  121. #define VTD_SIP_CNTR_EVNT GENMASK_ULL(63, 60)
  122. #define PERF_TIMEOUT 30
  123. #define PERF_MAX_PORT_NUM 1U
  124. /**
  125. * struct fme_perf_priv - priv data structure for fme perf driver
  126. *
  127. * @dev: parent device.
  128. * @ioaddr: mapped base address of mmio region.
  129. * @pmu: pmu data structure for fme perf counters.
  130. * @id: id of this fme performance report private feature.
  131. * @fab_users: current user number on fabric counters.
  132. * @fab_port_id: used to indicate current working mode of fabric counters.
  133. * @fab_lock: lock to protect fabric counters working mode.
  134. * @cpu: active CPU to which the PMU is bound for accesses.
  135. * @cpuhp_node: node for CPU hotplug notifier link.
  136. * @cpuhp_state: state for CPU hotplug notification;
  137. */
  138. struct fme_perf_priv {
  139. struct device *dev;
  140. void __iomem *ioaddr;
  141. struct pmu pmu;
  142. u16 id;
  143. u32 fab_users;
  144. u32 fab_port_id;
  145. spinlock_t fab_lock;
  146. unsigned int cpu;
  147. struct hlist_node node;
  148. enum cpuhp_state cpuhp_state;
  149. };
  150. /**
  151. * struct fme_perf_event_ops - callbacks for fme perf events
  152. *
  153. * @event_init: callback invoked during event init.
  154. * @event_destroy: callback invoked during event destroy.
  155. * @read_counter: callback to read hardware counters.
  156. */
  157. struct fme_perf_event_ops {
  158. int (*event_init)(struct fme_perf_priv *priv, u32 event, u32 portid);
  159. void (*event_destroy)(struct fme_perf_priv *priv, u32 event,
  160. u32 portid);
  161. u64 (*read_counter)(struct fme_perf_priv *priv, u32 event, u32 portid);
  162. };
  163. #define to_fme_perf_priv(_pmu) container_of(_pmu, struct fme_perf_priv, pmu)
  164. static ssize_t cpumask_show(struct device *dev,
  165. struct device_attribute *attr, char *buf)
  166. {
  167. struct pmu *pmu = dev_get_drvdata(dev);
  168. struct fme_perf_priv *priv;
  169. priv = to_fme_perf_priv(pmu);
  170. return cpumap_print_to_pagebuf(true, buf, cpumask_of(priv->cpu));
  171. }
  172. static DEVICE_ATTR_RO(cpumask);
  173. static struct attribute *fme_perf_cpumask_attrs[] = {
  174. &dev_attr_cpumask.attr,
  175. NULL,
  176. };
  177. static struct attribute_group fme_perf_cpumask_group = {
  178. .attrs = fme_perf_cpumask_attrs,
  179. };
  180. #define FME_EVENT_MASK GENMASK_ULL(11, 0)
  181. #define FME_EVENT_SHIFT 0
  182. #define FME_EVTYPE_MASK GENMASK_ULL(15, 12)
  183. #define FME_EVTYPE_SHIFT 12
  184. #define FME_EVTYPE_BASIC 0
  185. #define FME_EVTYPE_CACHE 1
  186. #define FME_EVTYPE_FABRIC 2
  187. #define FME_EVTYPE_VTD 3
  188. #define FME_EVTYPE_VTD_SIP 4
  189. #define FME_EVTYPE_MAX FME_EVTYPE_VTD_SIP
  190. #define FME_PORTID_MASK GENMASK_ULL(23, 16)
  191. #define FME_PORTID_SHIFT 16
  192. #define FME_PORTID_ROOT (0xffU)
  193. #define get_event(_config) FIELD_GET(FME_EVENT_MASK, _config)
  194. #define get_evtype(_config) FIELD_GET(FME_EVTYPE_MASK, _config)
  195. #define get_portid(_config) FIELD_GET(FME_PORTID_MASK, _config)
  196. PMU_FORMAT_ATTR(event, "config:0-11");
  197. PMU_FORMAT_ATTR(evtype, "config:12-15");
  198. PMU_FORMAT_ATTR(portid, "config:16-23");
  199. static struct attribute *fme_perf_format_attrs[] = {
  200. &format_attr_event.attr,
  201. &format_attr_evtype.attr,
  202. &format_attr_portid.attr,
  203. NULL,
  204. };
  205. static struct attribute_group fme_perf_format_group = {
  206. .name = "format",
  207. .attrs = fme_perf_format_attrs,
  208. };
  209. /*
  210. * There are no default events, but we need to create
  211. * "events" group (with empty attrs) before updating
  212. * it with detected events (using pmu->attr_update).
  213. */
  214. static struct attribute *fme_perf_events_attrs_empty[] = {
  215. NULL,
  216. };
  217. static struct attribute_group fme_perf_events_group = {
  218. .name = "events",
  219. .attrs = fme_perf_events_attrs_empty,
  220. };
  221. static const struct attribute_group *fme_perf_groups[] = {
  222. &fme_perf_format_group,
  223. &fme_perf_cpumask_group,
  224. &fme_perf_events_group,
  225. NULL,
  226. };
  227. static bool is_portid_root(u32 portid)
  228. {
  229. return portid == FME_PORTID_ROOT;
  230. }
  231. static bool is_portid_port(u32 portid)
  232. {
  233. return portid < PERF_MAX_PORT_NUM;
  234. }
  235. static bool is_portid_root_or_port(u32 portid)
  236. {
  237. return is_portid_root(portid) || is_portid_port(portid);
  238. }
  239. static u64 fme_read_perf_cntr_reg(void __iomem *addr)
  240. {
  241. u32 low;
  242. u64 v;
  243. /*
  244. * For 64bit counter registers, the counter may increases and carries
  245. * out of bit [31] between 2 32bit reads. So add extra reads to help
  246. * to prevent this issue. This only happens in platforms which don't
  247. * support 64bit read - readq is split into 2 readl.
  248. */
  249. do {
  250. v = readq(addr);
  251. low = readl(addr);
  252. } while (((u32)v) > low);
  253. return v;
  254. }
  255. static int basic_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
  256. {
  257. if (event <= BASIC_EVNT_MAX && is_portid_root(portid))
  258. return 0;
  259. return -EINVAL;
  260. }
  261. static u64 basic_read_event_counter(struct fme_perf_priv *priv,
  262. u32 event, u32 portid)
  263. {
  264. void __iomem *base = priv->ioaddr;
  265. return fme_read_perf_cntr_reg(base + CLK_CNTR);
  266. }
  267. static int cache_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
  268. {
  269. if (priv->id == FME_FEATURE_ID_GLOBAL_IPERF &&
  270. event <= CACHE_EVNT_MAX && is_portid_root(portid))
  271. return 0;
  272. return -EINVAL;
  273. }
  274. static u64 cache_read_event_counter(struct fme_perf_priv *priv,
  275. u32 event, u32 portid)
  276. {
  277. void __iomem *base = priv->ioaddr;
  278. u64 v, count;
  279. u8 channel;
  280. if (event == CACHE_EVNT_WR_HIT || event == CACHE_EVNT_WR_MISS ||
  281. event == CACHE_EVNT_DATA_WR_PORT_CONTEN ||
  282. event == CACHE_EVNT_TAG_WR_PORT_CONTEN)
  283. channel = CACHE_CHANNEL_WR;
  284. else
  285. channel = CACHE_CHANNEL_RD;
  286. /* set channel access type and cache event code. */
  287. v = readq(base + CACHE_CTRL);
  288. v &= ~(CACHE_CHANNEL_SEL | CACHE_CTRL_EVNT);
  289. v |= FIELD_PREP(CACHE_CHANNEL_SEL, channel);
  290. v |= FIELD_PREP(CACHE_CTRL_EVNT, event);
  291. writeq(v, base + CACHE_CTRL);
  292. if (readq_poll_timeout_atomic(base + CACHE_CNTR0, v,
  293. FIELD_GET(CACHE_CNTR_EVNT, v) == event,
  294. 1, PERF_TIMEOUT)) {
  295. dev_err(priv->dev, "timeout, unmatched cache event code in counter register.\n");
  296. return 0;
  297. }
  298. v = fme_read_perf_cntr_reg(base + CACHE_CNTR0);
  299. count = FIELD_GET(CACHE_CNTR_EVNT_CNTR, v);
  300. v = fme_read_perf_cntr_reg(base + CACHE_CNTR1);
  301. count += FIELD_GET(CACHE_CNTR_EVNT_CNTR, v);
  302. return count;
  303. }
  304. static bool is_fabric_event_supported(struct fme_perf_priv *priv, u32 event,
  305. u32 portid)
  306. {
  307. if (event > FAB_EVNT_MAX || !is_portid_root_or_port(portid))
  308. return false;
  309. if (priv->id == FME_FEATURE_ID_GLOBAL_DPERF &&
  310. (event == FAB_EVNT_PCIE1_RD || event == FAB_EVNT_UPI_RD ||
  311. event == FAB_EVNT_PCIE1_WR || event == FAB_EVNT_UPI_WR))
  312. return false;
  313. return true;
  314. }
  315. static int fabric_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
  316. {
  317. void __iomem *base = priv->ioaddr;
  318. int ret = 0;
  319. u64 v;
  320. if (!is_fabric_event_supported(priv, event, portid))
  321. return -EINVAL;
  322. /*
  323. * as fabric counter set only can be in either overall or port mode.
  324. * In overall mode, it counts overall data for FPGA, and in port mode,
  325. * it is configured to monitor on one individual port.
  326. *
  327. * so every time, a new event is initialized, driver checks
  328. * current working mode and if someone is using this counter set.
  329. */
  330. spin_lock(&priv->fab_lock);
  331. if (priv->fab_users && priv->fab_port_id != portid) {
  332. dev_dbg(priv->dev, "conflict fabric event monitoring mode.\n");
  333. ret = -EOPNOTSUPP;
  334. goto exit;
  335. }
  336. priv->fab_users++;
  337. /*
  338. * skip if current working mode matches, otherwise change the working
  339. * mode per input port_id, to monitor overall data or another port.
  340. */
  341. if (priv->fab_port_id == portid)
  342. goto exit;
  343. priv->fab_port_id = portid;
  344. v = readq(base + FAB_CTRL);
  345. v &= ~(FAB_PORT_FILTER | FAB_PORT_ID);
  346. if (is_portid_root(portid)) {
  347. v |= FIELD_PREP(FAB_PORT_FILTER, FAB_PORT_FILTER_DISABLE);
  348. } else {
  349. v |= FIELD_PREP(FAB_PORT_FILTER, FAB_PORT_FILTER_ENABLE);
  350. v |= FIELD_PREP(FAB_PORT_ID, portid);
  351. }
  352. writeq(v, base + FAB_CTRL);
  353. exit:
  354. spin_unlock(&priv->fab_lock);
  355. return ret;
  356. }
  357. static void fabric_event_destroy(struct fme_perf_priv *priv, u32 event,
  358. u32 portid)
  359. {
  360. spin_lock(&priv->fab_lock);
  361. priv->fab_users--;
  362. spin_unlock(&priv->fab_lock);
  363. }
  364. static u64 fabric_read_event_counter(struct fme_perf_priv *priv, u32 event,
  365. u32 portid)
  366. {
  367. void __iomem *base = priv->ioaddr;
  368. u64 v;
  369. v = readq(base + FAB_CTRL);
  370. v &= ~FAB_CTRL_EVNT;
  371. v |= FIELD_PREP(FAB_CTRL_EVNT, event);
  372. writeq(v, base + FAB_CTRL);
  373. if (readq_poll_timeout_atomic(base + FAB_CNTR, v,
  374. FIELD_GET(FAB_CNTR_EVNT, v) == event,
  375. 1, PERF_TIMEOUT)) {
  376. dev_err(priv->dev, "timeout, unmatched fab event code in counter register.\n");
  377. return 0;
  378. }
  379. v = fme_read_perf_cntr_reg(base + FAB_CNTR);
  380. return FIELD_GET(FAB_CNTR_EVNT_CNTR, v);
  381. }
  382. static int vtd_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
  383. {
  384. if (priv->id == FME_FEATURE_ID_GLOBAL_IPERF &&
  385. event <= VTD_EVNT_MAX && is_portid_port(portid))
  386. return 0;
  387. return -EINVAL;
  388. }
  389. static u64 vtd_read_event_counter(struct fme_perf_priv *priv, u32 event,
  390. u32 portid)
  391. {
  392. void __iomem *base = priv->ioaddr;
  393. u64 v;
  394. event += (portid * (VTD_EVNT_MAX + 1));
  395. v = readq(base + VTD_CTRL);
  396. v &= ~VTD_CTRL_EVNT;
  397. v |= FIELD_PREP(VTD_CTRL_EVNT, event);
  398. writeq(v, base + VTD_CTRL);
  399. if (readq_poll_timeout_atomic(base + VTD_CNTR, v,
  400. FIELD_GET(VTD_CNTR_EVNT, v) == event,
  401. 1, PERF_TIMEOUT)) {
  402. dev_err(priv->dev, "timeout, unmatched vtd event code in counter register.\n");
  403. return 0;
  404. }
  405. v = fme_read_perf_cntr_reg(base + VTD_CNTR);
  406. return FIELD_GET(VTD_CNTR_EVNT_CNTR, v);
  407. }
  408. static int vtd_sip_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
  409. {
  410. if (priv->id == FME_FEATURE_ID_GLOBAL_IPERF &&
  411. event <= VTD_SIP_EVNT_MAX && is_portid_root(portid))
  412. return 0;
  413. return -EINVAL;
  414. }
  415. static u64 vtd_sip_read_event_counter(struct fme_perf_priv *priv, u32 event,
  416. u32 portid)
  417. {
  418. void __iomem *base = priv->ioaddr;
  419. u64 v;
  420. v = readq(base + VTD_SIP_CTRL);
  421. v &= ~VTD_SIP_CTRL_EVNT;
  422. v |= FIELD_PREP(VTD_SIP_CTRL_EVNT, event);
  423. writeq(v, base + VTD_SIP_CTRL);
  424. if (readq_poll_timeout_atomic(base + VTD_SIP_CNTR, v,
  425. FIELD_GET(VTD_SIP_CNTR_EVNT, v) == event,
  426. 1, PERF_TIMEOUT)) {
  427. dev_err(priv->dev, "timeout, unmatched vtd sip event code in counter register\n");
  428. return 0;
  429. }
  430. v = fme_read_perf_cntr_reg(base + VTD_SIP_CNTR);
  431. return FIELD_GET(VTD_SIP_CNTR_EVNT_CNTR, v);
  432. }
  433. static struct fme_perf_event_ops fme_perf_event_ops[] = {
  434. [FME_EVTYPE_BASIC] = {.event_init = basic_event_init,
  435. .read_counter = basic_read_event_counter,},
  436. [FME_EVTYPE_CACHE] = {.event_init = cache_event_init,
  437. .read_counter = cache_read_event_counter,},
  438. [FME_EVTYPE_FABRIC] = {.event_init = fabric_event_init,
  439. .event_destroy = fabric_event_destroy,
  440. .read_counter = fabric_read_event_counter,},
  441. [FME_EVTYPE_VTD] = {.event_init = vtd_event_init,
  442. .read_counter = vtd_read_event_counter,},
  443. [FME_EVTYPE_VTD_SIP] = {.event_init = vtd_sip_event_init,
  444. .read_counter = vtd_sip_read_event_counter,},
  445. };
  446. static ssize_t fme_perf_event_show(struct device *dev,
  447. struct device_attribute *attr, char *buf)
  448. {
  449. struct dev_ext_attribute *eattr;
  450. unsigned long config;
  451. char *ptr = buf;
  452. eattr = container_of(attr, struct dev_ext_attribute, attr);
  453. config = (unsigned long)eattr->var;
  454. ptr += sprintf(ptr, "event=0x%02x", (unsigned int)get_event(config));
  455. ptr += sprintf(ptr, ",evtype=0x%02x", (unsigned int)get_evtype(config));
  456. if (is_portid_root(get_portid(config)))
  457. ptr += sprintf(ptr, ",portid=0x%02x\n", FME_PORTID_ROOT);
  458. else
  459. ptr += sprintf(ptr, ",portid=?\n");
  460. return (ssize_t)(ptr - buf);
  461. }
  462. #define FME_EVENT_ATTR(_name) \
  463. __ATTR(_name, 0444, fme_perf_event_show, NULL)
  464. #define FME_PORT_EVENT_CONFIG(_event, _type) \
  465. (void *)((((_event) << FME_EVENT_SHIFT) & FME_EVENT_MASK) | \
  466. (((_type) << FME_EVTYPE_SHIFT) & FME_EVTYPE_MASK))
  467. #define FME_EVENT_CONFIG(_event, _type) \
  468. (void *)((((_event) << FME_EVENT_SHIFT) & FME_EVENT_MASK) | \
  469. (((_type) << FME_EVTYPE_SHIFT) & FME_EVTYPE_MASK) | \
  470. (FME_PORTID_ROOT << FME_PORTID_SHIFT))
  471. /* FME Perf Basic Events */
  472. #define FME_EVENT_BASIC(_name, _event) \
  473. static struct dev_ext_attribute fme_perf_event_##_name = { \
  474. .attr = FME_EVENT_ATTR(_name), \
  475. .var = FME_EVENT_CONFIG(_event, FME_EVTYPE_BASIC), \
  476. }
  477. FME_EVENT_BASIC(clock, BASIC_EVNT_CLK);
  478. static struct attribute *fme_perf_basic_events_attrs[] = {
  479. &fme_perf_event_clock.attr.attr,
  480. NULL,
  481. };
  482. static const struct attribute_group fme_perf_basic_events_group = {
  483. .name = "events",
  484. .attrs = fme_perf_basic_events_attrs,
  485. };
  486. /* FME Perf Cache Events */
  487. #define FME_EVENT_CACHE(_name, _event) \
  488. static struct dev_ext_attribute fme_perf_event_cache_##_name = { \
  489. .attr = FME_EVENT_ATTR(cache_##_name), \
  490. .var = FME_EVENT_CONFIG(_event, FME_EVTYPE_CACHE), \
  491. }
  492. FME_EVENT_CACHE(read_hit, CACHE_EVNT_RD_HIT);
  493. FME_EVENT_CACHE(read_miss, CACHE_EVNT_RD_MISS);
  494. FME_EVENT_CACHE(write_hit, CACHE_EVNT_WR_HIT);
  495. FME_EVENT_CACHE(write_miss, CACHE_EVNT_WR_MISS);
  496. FME_EVENT_CACHE(hold_request, CACHE_EVNT_HOLD_REQ);
  497. FME_EVENT_CACHE(tx_req_stall, CACHE_EVNT_TX_REQ_STALL);
  498. FME_EVENT_CACHE(rx_req_stall, CACHE_EVNT_RX_REQ_STALL);
  499. FME_EVENT_CACHE(eviction, CACHE_EVNT_EVICTIONS);
  500. FME_EVENT_CACHE(data_write_port_contention, CACHE_EVNT_DATA_WR_PORT_CONTEN);
  501. FME_EVENT_CACHE(tag_write_port_contention, CACHE_EVNT_TAG_WR_PORT_CONTEN);
  502. static struct attribute *fme_perf_cache_events_attrs[] = {
  503. &fme_perf_event_cache_read_hit.attr.attr,
  504. &fme_perf_event_cache_read_miss.attr.attr,
  505. &fme_perf_event_cache_write_hit.attr.attr,
  506. &fme_perf_event_cache_write_miss.attr.attr,
  507. &fme_perf_event_cache_hold_request.attr.attr,
  508. &fme_perf_event_cache_tx_req_stall.attr.attr,
  509. &fme_perf_event_cache_rx_req_stall.attr.attr,
  510. &fme_perf_event_cache_eviction.attr.attr,
  511. &fme_perf_event_cache_data_write_port_contention.attr.attr,
  512. &fme_perf_event_cache_tag_write_port_contention.attr.attr,
  513. NULL,
  514. };
  515. static umode_t fme_perf_events_visible(struct kobject *kobj,
  516. struct attribute *attr, int n)
  517. {
  518. struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
  519. struct fme_perf_priv *priv = to_fme_perf_priv(pmu);
  520. return (priv->id == FME_FEATURE_ID_GLOBAL_IPERF) ? attr->mode : 0;
  521. }
  522. static const struct attribute_group fme_perf_cache_events_group = {
  523. .name = "events",
  524. .attrs = fme_perf_cache_events_attrs,
  525. .is_visible = fme_perf_events_visible,
  526. };
  527. /* FME Perf Fabric Events */
  528. #define FME_EVENT_FABRIC(_name, _event) \
  529. static struct dev_ext_attribute fme_perf_event_fab_##_name = { \
  530. .attr = FME_EVENT_ATTR(fab_##_name), \
  531. .var = FME_EVENT_CONFIG(_event, FME_EVTYPE_FABRIC), \
  532. }
  533. #define FME_EVENT_FABRIC_PORT(_name, _event) \
  534. static struct dev_ext_attribute fme_perf_event_fab_port_##_name = { \
  535. .attr = FME_EVENT_ATTR(fab_port_##_name), \
  536. .var = FME_PORT_EVENT_CONFIG(_event, FME_EVTYPE_FABRIC), \
  537. }
  538. FME_EVENT_FABRIC(pcie0_read, FAB_EVNT_PCIE0_RD);
  539. FME_EVENT_FABRIC(pcie0_write, FAB_EVNT_PCIE0_WR);
  540. FME_EVENT_FABRIC(pcie1_read, FAB_EVNT_PCIE1_RD);
  541. FME_EVENT_FABRIC(pcie1_write, FAB_EVNT_PCIE1_WR);
  542. FME_EVENT_FABRIC(upi_read, FAB_EVNT_UPI_RD);
  543. FME_EVENT_FABRIC(upi_write, FAB_EVNT_UPI_WR);
  544. FME_EVENT_FABRIC(mmio_read, FAB_EVNT_MMIO_RD);
  545. FME_EVENT_FABRIC(mmio_write, FAB_EVNT_MMIO_WR);
  546. FME_EVENT_FABRIC_PORT(pcie0_read, FAB_EVNT_PCIE0_RD);
  547. FME_EVENT_FABRIC_PORT(pcie0_write, FAB_EVNT_PCIE0_WR);
  548. FME_EVENT_FABRIC_PORT(pcie1_read, FAB_EVNT_PCIE1_RD);
  549. FME_EVENT_FABRIC_PORT(pcie1_write, FAB_EVNT_PCIE1_WR);
  550. FME_EVENT_FABRIC_PORT(upi_read, FAB_EVNT_UPI_RD);
  551. FME_EVENT_FABRIC_PORT(upi_write, FAB_EVNT_UPI_WR);
  552. FME_EVENT_FABRIC_PORT(mmio_read, FAB_EVNT_MMIO_RD);
  553. FME_EVENT_FABRIC_PORT(mmio_write, FAB_EVNT_MMIO_WR);
  554. static struct attribute *fme_perf_fabric_events_attrs[] = {
  555. &fme_perf_event_fab_pcie0_read.attr.attr,
  556. &fme_perf_event_fab_pcie0_write.attr.attr,
  557. &fme_perf_event_fab_pcie1_read.attr.attr,
  558. &fme_perf_event_fab_pcie1_write.attr.attr,
  559. &fme_perf_event_fab_upi_read.attr.attr,
  560. &fme_perf_event_fab_upi_write.attr.attr,
  561. &fme_perf_event_fab_mmio_read.attr.attr,
  562. &fme_perf_event_fab_mmio_write.attr.attr,
  563. &fme_perf_event_fab_port_pcie0_read.attr.attr,
  564. &fme_perf_event_fab_port_pcie0_write.attr.attr,
  565. &fme_perf_event_fab_port_pcie1_read.attr.attr,
  566. &fme_perf_event_fab_port_pcie1_write.attr.attr,
  567. &fme_perf_event_fab_port_upi_read.attr.attr,
  568. &fme_perf_event_fab_port_upi_write.attr.attr,
  569. &fme_perf_event_fab_port_mmio_read.attr.attr,
  570. &fme_perf_event_fab_port_mmio_write.attr.attr,
  571. NULL,
  572. };
  573. static umode_t fme_perf_fabric_events_visible(struct kobject *kobj,
  574. struct attribute *attr, int n)
  575. {
  576. struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
  577. struct fme_perf_priv *priv = to_fme_perf_priv(pmu);
  578. struct dev_ext_attribute *eattr;
  579. unsigned long var;
  580. eattr = container_of(attr, struct dev_ext_attribute, attr.attr);
  581. var = (unsigned long)eattr->var;
  582. if (is_fabric_event_supported(priv, get_event(var), get_portid(var)))
  583. return attr->mode;
  584. return 0;
  585. }
  586. static const struct attribute_group fme_perf_fabric_events_group = {
  587. .name = "events",
  588. .attrs = fme_perf_fabric_events_attrs,
  589. .is_visible = fme_perf_fabric_events_visible,
  590. };
  591. /* FME Perf VTD Events */
  592. #define FME_EVENT_VTD_PORT(_name, _event) \
  593. static struct dev_ext_attribute fme_perf_event_vtd_port_##_name = { \
  594. .attr = FME_EVENT_ATTR(vtd_port_##_name), \
  595. .var = FME_PORT_EVENT_CONFIG(_event, FME_EVTYPE_VTD), \
  596. }
  597. FME_EVENT_VTD_PORT(read_transaction, VTD_EVNT_AFU_MEM_RD_TRANS);
  598. FME_EVENT_VTD_PORT(write_transaction, VTD_EVNT_AFU_MEM_WR_TRANS);
  599. FME_EVENT_VTD_PORT(devtlb_read_hit, VTD_EVNT_AFU_DEVTLB_RD_HIT);
  600. FME_EVENT_VTD_PORT(devtlb_write_hit, VTD_EVNT_AFU_DEVTLB_WR_HIT);
  601. FME_EVENT_VTD_PORT(devtlb_4k_fill, VTD_EVNT_DEVTLB_4K_FILL);
  602. FME_EVENT_VTD_PORT(devtlb_2m_fill, VTD_EVNT_DEVTLB_2M_FILL);
  603. FME_EVENT_VTD_PORT(devtlb_1g_fill, VTD_EVNT_DEVTLB_1G_FILL);
  604. static struct attribute *fme_perf_vtd_events_attrs[] = {
  605. &fme_perf_event_vtd_port_read_transaction.attr.attr,
  606. &fme_perf_event_vtd_port_write_transaction.attr.attr,
  607. &fme_perf_event_vtd_port_devtlb_read_hit.attr.attr,
  608. &fme_perf_event_vtd_port_devtlb_write_hit.attr.attr,
  609. &fme_perf_event_vtd_port_devtlb_4k_fill.attr.attr,
  610. &fme_perf_event_vtd_port_devtlb_2m_fill.attr.attr,
  611. &fme_perf_event_vtd_port_devtlb_1g_fill.attr.attr,
  612. NULL,
  613. };
  614. static const struct attribute_group fme_perf_vtd_events_group = {
  615. .name = "events",
  616. .attrs = fme_perf_vtd_events_attrs,
  617. .is_visible = fme_perf_events_visible,
  618. };
  619. /* FME Perf VTD SIP Events */
  620. #define FME_EVENT_VTD_SIP(_name, _event) \
  621. static struct dev_ext_attribute fme_perf_event_vtd_sip_##_name = { \
  622. .attr = FME_EVENT_ATTR(vtd_sip_##_name), \
  623. .var = FME_EVENT_CONFIG(_event, FME_EVTYPE_VTD_SIP), \
  624. }
  625. FME_EVENT_VTD_SIP(iotlb_4k_hit, VTD_SIP_EVNT_IOTLB_4K_HIT);
  626. FME_EVENT_VTD_SIP(iotlb_2m_hit, VTD_SIP_EVNT_IOTLB_2M_HIT);
  627. FME_EVENT_VTD_SIP(iotlb_1g_hit, VTD_SIP_EVNT_IOTLB_1G_HIT);
  628. FME_EVENT_VTD_SIP(slpwc_l3_hit, VTD_SIP_EVNT_SLPWC_L3_HIT);
  629. FME_EVENT_VTD_SIP(slpwc_l4_hit, VTD_SIP_EVNT_SLPWC_L4_HIT);
  630. FME_EVENT_VTD_SIP(rcc_hit, VTD_SIP_EVNT_RCC_HIT);
  631. FME_EVENT_VTD_SIP(iotlb_4k_miss, VTD_SIP_EVNT_IOTLB_4K_MISS);
  632. FME_EVENT_VTD_SIP(iotlb_2m_miss, VTD_SIP_EVNT_IOTLB_2M_MISS);
  633. FME_EVENT_VTD_SIP(iotlb_1g_miss, VTD_SIP_EVNT_IOTLB_1G_MISS);
  634. FME_EVENT_VTD_SIP(slpwc_l3_miss, VTD_SIP_EVNT_SLPWC_L3_MISS);
  635. FME_EVENT_VTD_SIP(slpwc_l4_miss, VTD_SIP_EVNT_SLPWC_L4_MISS);
  636. FME_EVENT_VTD_SIP(rcc_miss, VTD_SIP_EVNT_RCC_MISS);
  637. static struct attribute *fme_perf_vtd_sip_events_attrs[] = {
  638. &fme_perf_event_vtd_sip_iotlb_4k_hit.attr.attr,
  639. &fme_perf_event_vtd_sip_iotlb_2m_hit.attr.attr,
  640. &fme_perf_event_vtd_sip_iotlb_1g_hit.attr.attr,
  641. &fme_perf_event_vtd_sip_slpwc_l3_hit.attr.attr,
  642. &fme_perf_event_vtd_sip_slpwc_l4_hit.attr.attr,
  643. &fme_perf_event_vtd_sip_rcc_hit.attr.attr,
  644. &fme_perf_event_vtd_sip_iotlb_4k_miss.attr.attr,
  645. &fme_perf_event_vtd_sip_iotlb_2m_miss.attr.attr,
  646. &fme_perf_event_vtd_sip_iotlb_1g_miss.attr.attr,
  647. &fme_perf_event_vtd_sip_slpwc_l3_miss.attr.attr,
  648. &fme_perf_event_vtd_sip_slpwc_l4_miss.attr.attr,
  649. &fme_perf_event_vtd_sip_rcc_miss.attr.attr,
  650. NULL,
  651. };
  652. static const struct attribute_group fme_perf_vtd_sip_events_group = {
  653. .name = "events",
  654. .attrs = fme_perf_vtd_sip_events_attrs,
  655. .is_visible = fme_perf_events_visible,
  656. };
  657. static const struct attribute_group *fme_perf_events_groups[] = {
  658. &fme_perf_basic_events_group,
  659. &fme_perf_cache_events_group,
  660. &fme_perf_fabric_events_group,
  661. &fme_perf_vtd_events_group,
  662. &fme_perf_vtd_sip_events_group,
  663. NULL,
  664. };
  665. static struct fme_perf_event_ops *get_event_ops(u32 evtype)
  666. {
  667. if (evtype > FME_EVTYPE_MAX)
  668. return NULL;
  669. return &fme_perf_event_ops[evtype];
  670. }
  671. static void fme_perf_event_destroy(struct perf_event *event)
  672. {
  673. struct fme_perf_event_ops *ops = get_event_ops(event->hw.event_base);
  674. struct fme_perf_priv *priv = to_fme_perf_priv(event->pmu);
  675. if (ops->event_destroy)
  676. ops->event_destroy(priv, event->hw.idx, event->hw.config_base);
  677. }
  678. static int fme_perf_event_init(struct perf_event *event)
  679. {
  680. struct fme_perf_priv *priv = to_fme_perf_priv(event->pmu);
  681. struct hw_perf_event *hwc = &event->hw;
  682. struct fme_perf_event_ops *ops;
  683. u32 eventid, evtype, portid;
  684. /* test the event attr type check for PMU enumeration */
  685. if (event->attr.type != event->pmu->type)
  686. return -ENOENT;
  687. /*
  688. * fme counters are shared across all cores.
  689. * Therefore, it does not support per-process mode.
  690. * Also, it does not support event sampling mode.
  691. */
  692. if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
  693. return -EINVAL;
  694. if (event->cpu < 0)
  695. return -EINVAL;
  696. if (event->cpu != priv->cpu)
  697. return -EINVAL;
  698. eventid = get_event(event->attr.config);
  699. portid = get_portid(event->attr.config);
  700. evtype = get_evtype(event->attr.config);
  701. if (evtype > FME_EVTYPE_MAX)
  702. return -EINVAL;
  703. hwc->event_base = evtype;
  704. hwc->idx = (int)eventid;
  705. hwc->config_base = portid;
  706. event->destroy = fme_perf_event_destroy;
  707. dev_dbg(priv->dev, "%s event=0x%x, evtype=0x%x, portid=0x%x,\n",
  708. __func__, eventid, evtype, portid);
  709. ops = get_event_ops(evtype);
  710. if (ops->event_init)
  711. return ops->event_init(priv, eventid, portid);
  712. return 0;
  713. }
  714. static void fme_perf_event_update(struct perf_event *event)
  715. {
  716. struct fme_perf_event_ops *ops = get_event_ops(event->hw.event_base);
  717. struct fme_perf_priv *priv = to_fme_perf_priv(event->pmu);
  718. struct hw_perf_event *hwc = &event->hw;
  719. u64 now, prev, delta;
  720. now = ops->read_counter(priv, (u32)hwc->idx, hwc->config_base);
  721. prev = local64_read(&hwc->prev_count);
  722. delta = now - prev;
  723. local64_add(delta, &event->count);
  724. }
  725. static void fme_perf_event_start(struct perf_event *event, int flags)
  726. {
  727. struct fme_perf_event_ops *ops = get_event_ops(event->hw.event_base);
  728. struct fme_perf_priv *priv = to_fme_perf_priv(event->pmu);
  729. struct hw_perf_event *hwc = &event->hw;
  730. u64 count;
  731. count = ops->read_counter(priv, (u32)hwc->idx, hwc->config_base);
  732. local64_set(&hwc->prev_count, count);
  733. }
  734. static void fme_perf_event_stop(struct perf_event *event, int flags)
  735. {
  736. fme_perf_event_update(event);
  737. }
  738. static int fme_perf_event_add(struct perf_event *event, int flags)
  739. {
  740. if (flags & PERF_EF_START)
  741. fme_perf_event_start(event, flags);
  742. return 0;
  743. }
  744. static void fme_perf_event_del(struct perf_event *event, int flags)
  745. {
  746. fme_perf_event_stop(event, PERF_EF_UPDATE);
  747. }
  748. static void fme_perf_event_read(struct perf_event *event)
  749. {
  750. fme_perf_event_update(event);
  751. }
  752. static void fme_perf_setup_hardware(struct fme_perf_priv *priv)
  753. {
  754. void __iomem *base = priv->ioaddr;
  755. u64 v;
  756. /* read and save current working mode for fabric counters */
  757. v = readq(base + FAB_CTRL);
  758. if (FIELD_GET(FAB_PORT_FILTER, v) == FAB_PORT_FILTER_DISABLE)
  759. priv->fab_port_id = FME_PORTID_ROOT;
  760. else
  761. priv->fab_port_id = FIELD_GET(FAB_PORT_ID, v);
  762. }
  763. static int fme_perf_pmu_register(struct platform_device *pdev,
  764. struct fme_perf_priv *priv)
  765. {
  766. struct pmu *pmu = &priv->pmu;
  767. char *name;
  768. int ret;
  769. spin_lock_init(&priv->fab_lock);
  770. fme_perf_setup_hardware(priv);
  771. pmu->task_ctx_nr = perf_invalid_context;
  772. pmu->attr_groups = fme_perf_groups;
  773. pmu->attr_update = fme_perf_events_groups;
  774. pmu->event_init = fme_perf_event_init;
  775. pmu->add = fme_perf_event_add;
  776. pmu->del = fme_perf_event_del;
  777. pmu->start = fme_perf_event_start;
  778. pmu->stop = fme_perf_event_stop;
  779. pmu->read = fme_perf_event_read;
  780. pmu->capabilities = PERF_PMU_CAP_NO_INTERRUPT |
  781. PERF_PMU_CAP_NO_EXCLUDE;
  782. name = devm_kasprintf(priv->dev, GFP_KERNEL, "dfl_fme%d", pdev->id);
  783. ret = perf_pmu_register(pmu, name, -1);
  784. if (ret)
  785. return ret;
  786. return 0;
  787. }
  788. static void fme_perf_pmu_unregister(struct fme_perf_priv *priv)
  789. {
  790. perf_pmu_unregister(&priv->pmu);
  791. }
  792. static int fme_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)
  793. {
  794. struct fme_perf_priv *priv;
  795. int target;
  796. priv = hlist_entry_safe(node, struct fme_perf_priv, node);
  797. if (cpu != priv->cpu)
  798. return 0;
  799. target = cpumask_any_but(cpu_online_mask, cpu);
  800. if (target >= nr_cpu_ids)
  801. return 0;
  802. priv->cpu = target;
  803. perf_pmu_migrate_context(&priv->pmu, cpu, target);
  804. return 0;
  805. }
  806. static int fme_perf_init(struct platform_device *pdev,
  807. struct dfl_feature *feature)
  808. {
  809. struct fme_perf_priv *priv;
  810. int ret;
  811. priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
  812. if (!priv)
  813. return -ENOMEM;
  814. priv->dev = &pdev->dev;
  815. priv->ioaddr = feature->ioaddr;
  816. priv->id = feature->id;
  817. priv->cpu = raw_smp_processor_id();
  818. ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
  819. "perf/fpga/dfl_fme:online",
  820. NULL, fme_perf_offline_cpu);
  821. if (ret < 0)
  822. return ret;
  823. priv->cpuhp_state = ret;
  824. /* Register the pmu instance for cpu hotplug */
  825. ret = cpuhp_state_add_instance_nocalls(priv->cpuhp_state, &priv->node);
  826. if (ret)
  827. goto cpuhp_instance_err;
  828. ret = fme_perf_pmu_register(pdev, priv);
  829. if (ret)
  830. goto pmu_register_err;
  831. feature->priv = priv;
  832. return 0;
  833. pmu_register_err:
  834. cpuhp_state_remove_instance_nocalls(priv->cpuhp_state, &priv->node);
  835. cpuhp_instance_err:
  836. cpuhp_remove_multi_state(priv->cpuhp_state);
  837. return ret;
  838. }
  839. static void fme_perf_uinit(struct platform_device *pdev,
  840. struct dfl_feature *feature)
  841. {
  842. struct fme_perf_priv *priv = feature->priv;
  843. fme_perf_pmu_unregister(priv);
  844. cpuhp_state_remove_instance_nocalls(priv->cpuhp_state, &priv->node);
  845. cpuhp_remove_multi_state(priv->cpuhp_state);
  846. }
  847. const struct dfl_feature_id fme_perf_id_table[] = {
  848. {.id = FME_FEATURE_ID_GLOBAL_IPERF,},
  849. {.id = FME_FEATURE_ID_GLOBAL_DPERF,},
  850. {0,}
  851. };
  852. const struct dfl_feature_ops fme_perf_ops = {
  853. .init = fme_perf_init,
  854. .uinit = fme_perf_uinit,
  855. };