arm_spe_pmu.c 33 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Perf support for the Statistical Profiling Extension, introduced as
  4. * part of ARMv8.2.
  5. *
  6. * Copyright (C) 2016 ARM Limited
  7. *
  8. * Author: Will Deacon <will.deacon@arm.com>
  9. */
  10. #define PMUNAME "arm_spe"
  11. #define DRVNAME PMUNAME "_pmu"
  12. #define pr_fmt(fmt) DRVNAME ": " fmt
  13. #include <linux/bitops.h>
  14. #include <linux/bug.h>
  15. #include <linux/capability.h>
  16. #include <linux/cpuhotplug.h>
  17. #include <linux/cpumask.h>
  18. #include <linux/device.h>
  19. #include <linux/errno.h>
  20. #include <linux/interrupt.h>
  21. #include <linux/irq.h>
  22. #include <linux/kernel.h>
  23. #include <linux/list.h>
  24. #include <linux/module.h>
  25. #include <linux/of_address.h>
  26. #include <linux/of_device.h>
  27. #include <linux/perf_event.h>
  28. #include <linux/perf/arm_pmu.h>
  29. #include <linux/platform_device.h>
  30. #include <linux/printk.h>
  31. #include <linux/slab.h>
  32. #include <linux/smp.h>
  33. #include <linux/vmalloc.h>
  34. #include <asm/barrier.h>
  35. #include <asm/cpufeature.h>
  36. #include <asm/mmu.h>
  37. #include <asm/sysreg.h>
  38. #define ARM_SPE_BUF_PAD_BYTE 0
  39. struct arm_spe_pmu_buf {
  40. int nr_pages;
  41. bool snapshot;
  42. void *base;
  43. };
  44. struct arm_spe_pmu {
  45. struct pmu pmu;
  46. struct platform_device *pdev;
  47. cpumask_t supported_cpus;
  48. struct hlist_node hotplug_node;
  49. int irq; /* PPI */
  50. u16 min_period;
  51. u16 counter_sz;
  52. #define SPE_PMU_FEAT_FILT_EVT (1UL << 0)
  53. #define SPE_PMU_FEAT_FILT_TYP (1UL << 1)
  54. #define SPE_PMU_FEAT_FILT_LAT (1UL << 2)
  55. #define SPE_PMU_FEAT_ARCH_INST (1UL << 3)
  56. #define SPE_PMU_FEAT_LDS (1UL << 4)
  57. #define SPE_PMU_FEAT_ERND (1UL << 5)
  58. #define SPE_PMU_FEAT_DEV_PROBED (1UL << 63)
  59. u64 features;
  60. u16 max_record_sz;
  61. u16 align;
  62. struct perf_output_handle __percpu *handle;
  63. };
  64. #define to_spe_pmu(p) (container_of(p, struct arm_spe_pmu, pmu))
  65. /* Convert a free-running index from perf into an SPE buffer offset */
  66. #define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
  67. /* Keep track of our dynamic hotplug state */
  68. static enum cpuhp_state arm_spe_pmu_online;
  69. enum arm_spe_pmu_buf_fault_action {
  70. SPE_PMU_BUF_FAULT_ACT_SPURIOUS,
  71. SPE_PMU_BUF_FAULT_ACT_FATAL,
  72. SPE_PMU_BUF_FAULT_ACT_OK,
  73. };
  74. /* This sysfs gunk was really good fun to write. */
  75. enum arm_spe_pmu_capabilities {
  76. SPE_PMU_CAP_ARCH_INST = 0,
  77. SPE_PMU_CAP_ERND,
  78. SPE_PMU_CAP_FEAT_MAX,
  79. SPE_PMU_CAP_CNT_SZ = SPE_PMU_CAP_FEAT_MAX,
  80. SPE_PMU_CAP_MIN_IVAL,
  81. };
  82. static int arm_spe_pmu_feat_caps[SPE_PMU_CAP_FEAT_MAX] = {
  83. [SPE_PMU_CAP_ARCH_INST] = SPE_PMU_FEAT_ARCH_INST,
  84. [SPE_PMU_CAP_ERND] = SPE_PMU_FEAT_ERND,
  85. };
  86. static u32 arm_spe_pmu_cap_get(struct arm_spe_pmu *spe_pmu, int cap)
  87. {
  88. if (cap < SPE_PMU_CAP_FEAT_MAX)
  89. return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]);
  90. switch (cap) {
  91. case SPE_PMU_CAP_CNT_SZ:
  92. return spe_pmu->counter_sz;
  93. case SPE_PMU_CAP_MIN_IVAL:
  94. return spe_pmu->min_period;
  95. default:
  96. WARN(1, "unknown cap %d\n", cap);
  97. }
  98. return 0;
  99. }
  100. static ssize_t arm_spe_pmu_cap_show(struct device *dev,
  101. struct device_attribute *attr,
  102. char *buf)
  103. {
  104. struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev);
  105. struct dev_ext_attribute *ea =
  106. container_of(attr, struct dev_ext_attribute, attr);
  107. int cap = (long)ea->var;
  108. return snprintf(buf, PAGE_SIZE, "%u\n",
  109. arm_spe_pmu_cap_get(spe_pmu, cap));
  110. }
  111. #define SPE_EXT_ATTR_ENTRY(_name, _func, _var) \
  112. &((struct dev_ext_attribute[]) { \
  113. { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_var } \
  114. })[0].attr.attr
  115. #define SPE_CAP_EXT_ATTR_ENTRY(_name, _var) \
  116. SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show, _var)
  117. static struct attribute *arm_spe_pmu_cap_attr[] = {
  118. SPE_CAP_EXT_ATTR_ENTRY(arch_inst, SPE_PMU_CAP_ARCH_INST),
  119. SPE_CAP_EXT_ATTR_ENTRY(ernd, SPE_PMU_CAP_ERND),
  120. SPE_CAP_EXT_ATTR_ENTRY(count_size, SPE_PMU_CAP_CNT_SZ),
  121. SPE_CAP_EXT_ATTR_ENTRY(min_interval, SPE_PMU_CAP_MIN_IVAL),
  122. NULL,
  123. };
  124. static struct attribute_group arm_spe_pmu_cap_group = {
  125. .name = "caps",
  126. .attrs = arm_spe_pmu_cap_attr,
  127. };
  128. /* User ABI */
  129. #define ATTR_CFG_FLD_ts_enable_CFG config /* PMSCR_EL1.TS */
  130. #define ATTR_CFG_FLD_ts_enable_LO 0
  131. #define ATTR_CFG_FLD_ts_enable_HI 0
  132. #define ATTR_CFG_FLD_pa_enable_CFG config /* PMSCR_EL1.PA */
  133. #define ATTR_CFG_FLD_pa_enable_LO 1
  134. #define ATTR_CFG_FLD_pa_enable_HI 1
  135. #define ATTR_CFG_FLD_pct_enable_CFG config /* PMSCR_EL1.PCT */
  136. #define ATTR_CFG_FLD_pct_enable_LO 2
  137. #define ATTR_CFG_FLD_pct_enable_HI 2
  138. #define ATTR_CFG_FLD_jitter_CFG config /* PMSIRR_EL1.RND */
  139. #define ATTR_CFG_FLD_jitter_LO 16
  140. #define ATTR_CFG_FLD_jitter_HI 16
  141. #define ATTR_CFG_FLD_branch_filter_CFG config /* PMSFCR_EL1.B */
  142. #define ATTR_CFG_FLD_branch_filter_LO 32
  143. #define ATTR_CFG_FLD_branch_filter_HI 32
  144. #define ATTR_CFG_FLD_load_filter_CFG config /* PMSFCR_EL1.LD */
  145. #define ATTR_CFG_FLD_load_filter_LO 33
  146. #define ATTR_CFG_FLD_load_filter_HI 33
  147. #define ATTR_CFG_FLD_store_filter_CFG config /* PMSFCR_EL1.ST */
  148. #define ATTR_CFG_FLD_store_filter_LO 34
  149. #define ATTR_CFG_FLD_store_filter_HI 34
  150. #define ATTR_CFG_FLD_event_filter_CFG config1 /* PMSEVFR_EL1 */
  151. #define ATTR_CFG_FLD_event_filter_LO 0
  152. #define ATTR_CFG_FLD_event_filter_HI 63
  153. #define ATTR_CFG_FLD_min_latency_CFG config2 /* PMSLATFR_EL1.MINLAT */
  154. #define ATTR_CFG_FLD_min_latency_LO 0
  155. #define ATTR_CFG_FLD_min_latency_HI 11
  156. /* Why does everything I do descend into this? */
  157. #define __GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \
  158. (lo) == (hi) ? #cfg ":" #lo "\n" : #cfg ":" #lo "-" #hi
  159. #define _GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \
  160. __GEN_PMU_FORMAT_ATTR(cfg, lo, hi)
  161. #define GEN_PMU_FORMAT_ATTR(name) \
  162. PMU_FORMAT_ATTR(name, \
  163. _GEN_PMU_FORMAT_ATTR(ATTR_CFG_FLD_##name##_CFG, \
  164. ATTR_CFG_FLD_##name##_LO, \
  165. ATTR_CFG_FLD_##name##_HI))
  166. #define _ATTR_CFG_GET_FLD(attr, cfg, lo, hi) \
  167. ((((attr)->cfg) >> lo) & GENMASK(hi - lo, 0))
  168. #define ATTR_CFG_GET_FLD(attr, name) \
  169. _ATTR_CFG_GET_FLD(attr, \
  170. ATTR_CFG_FLD_##name##_CFG, \
  171. ATTR_CFG_FLD_##name##_LO, \
  172. ATTR_CFG_FLD_##name##_HI)
  173. GEN_PMU_FORMAT_ATTR(ts_enable);
  174. GEN_PMU_FORMAT_ATTR(pa_enable);
  175. GEN_PMU_FORMAT_ATTR(pct_enable);
  176. GEN_PMU_FORMAT_ATTR(jitter);
  177. GEN_PMU_FORMAT_ATTR(branch_filter);
  178. GEN_PMU_FORMAT_ATTR(load_filter);
  179. GEN_PMU_FORMAT_ATTR(store_filter);
  180. GEN_PMU_FORMAT_ATTR(event_filter);
  181. GEN_PMU_FORMAT_ATTR(min_latency);
  182. static struct attribute *arm_spe_pmu_formats_attr[] = {
  183. &format_attr_ts_enable.attr,
  184. &format_attr_pa_enable.attr,
  185. &format_attr_pct_enable.attr,
  186. &format_attr_jitter.attr,
  187. &format_attr_branch_filter.attr,
  188. &format_attr_load_filter.attr,
  189. &format_attr_store_filter.attr,
  190. &format_attr_event_filter.attr,
  191. &format_attr_min_latency.attr,
  192. NULL,
  193. };
  194. static struct attribute_group arm_spe_pmu_format_group = {
  195. .name = "format",
  196. .attrs = arm_spe_pmu_formats_attr,
  197. };
  198. static ssize_t arm_spe_pmu_get_attr_cpumask(struct device *dev,
  199. struct device_attribute *attr,
  200. char *buf)
  201. {
  202. struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev);
  203. return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus);
  204. }
  205. static DEVICE_ATTR(cpumask, S_IRUGO, arm_spe_pmu_get_attr_cpumask, NULL);
  206. static struct attribute *arm_spe_pmu_attrs[] = {
  207. &dev_attr_cpumask.attr,
  208. NULL,
  209. };
  210. static struct attribute_group arm_spe_pmu_group = {
  211. .attrs = arm_spe_pmu_attrs,
  212. };
  213. static const struct attribute_group *arm_spe_pmu_attr_groups[] = {
  214. &arm_spe_pmu_group,
  215. &arm_spe_pmu_cap_group,
  216. &arm_spe_pmu_format_group,
  217. NULL,
  218. };
  219. /* Convert between user ABI and register values */
  220. static u64 arm_spe_event_to_pmscr(struct perf_event *event)
  221. {
  222. struct perf_event_attr *attr = &event->attr;
  223. u64 reg = 0;
  224. reg |= ATTR_CFG_GET_FLD(attr, ts_enable) << SYS_PMSCR_EL1_TS_SHIFT;
  225. reg |= ATTR_CFG_GET_FLD(attr, pa_enable) << SYS_PMSCR_EL1_PA_SHIFT;
  226. reg |= ATTR_CFG_GET_FLD(attr, pct_enable) << SYS_PMSCR_EL1_PCT_SHIFT;
  227. if (!attr->exclude_user)
  228. reg |= BIT(SYS_PMSCR_EL1_E0SPE_SHIFT);
  229. if (!attr->exclude_kernel)
  230. reg |= BIT(SYS_PMSCR_EL1_E1SPE_SHIFT);
  231. if (IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR) && perfmon_capable())
  232. reg |= BIT(SYS_PMSCR_EL1_CX_SHIFT);
  233. return reg;
  234. }
  235. static void arm_spe_event_sanitise_period(struct perf_event *event)
  236. {
  237. struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
  238. u64 period = event->hw.sample_period;
  239. u64 max_period = SYS_PMSIRR_EL1_INTERVAL_MASK
  240. << SYS_PMSIRR_EL1_INTERVAL_SHIFT;
  241. if (period < spe_pmu->min_period)
  242. period = spe_pmu->min_period;
  243. else if (period > max_period)
  244. period = max_period;
  245. else
  246. period &= max_period;
  247. event->hw.sample_period = period;
  248. }
  249. static u64 arm_spe_event_to_pmsirr(struct perf_event *event)
  250. {
  251. struct perf_event_attr *attr = &event->attr;
  252. u64 reg = 0;
  253. arm_spe_event_sanitise_period(event);
  254. reg |= ATTR_CFG_GET_FLD(attr, jitter) << SYS_PMSIRR_EL1_RND_SHIFT;
  255. reg |= event->hw.sample_period;
  256. return reg;
  257. }
  258. static u64 arm_spe_event_to_pmsfcr(struct perf_event *event)
  259. {
  260. struct perf_event_attr *attr = &event->attr;
  261. u64 reg = 0;
  262. reg |= ATTR_CFG_GET_FLD(attr, load_filter) << SYS_PMSFCR_EL1_LD_SHIFT;
  263. reg |= ATTR_CFG_GET_FLD(attr, store_filter) << SYS_PMSFCR_EL1_ST_SHIFT;
  264. reg |= ATTR_CFG_GET_FLD(attr, branch_filter) << SYS_PMSFCR_EL1_B_SHIFT;
  265. if (reg)
  266. reg |= BIT(SYS_PMSFCR_EL1_FT_SHIFT);
  267. if (ATTR_CFG_GET_FLD(attr, event_filter))
  268. reg |= BIT(SYS_PMSFCR_EL1_FE_SHIFT);
  269. if (ATTR_CFG_GET_FLD(attr, min_latency))
  270. reg |= BIT(SYS_PMSFCR_EL1_FL_SHIFT);
  271. return reg;
  272. }
  273. static u64 arm_spe_event_to_pmsevfr(struct perf_event *event)
  274. {
  275. struct perf_event_attr *attr = &event->attr;
  276. return ATTR_CFG_GET_FLD(attr, event_filter);
  277. }
  278. static u64 arm_spe_event_to_pmslatfr(struct perf_event *event)
  279. {
  280. struct perf_event_attr *attr = &event->attr;
  281. return ATTR_CFG_GET_FLD(attr, min_latency)
  282. << SYS_PMSLATFR_EL1_MINLAT_SHIFT;
  283. }
  284. static void arm_spe_pmu_pad_buf(struct perf_output_handle *handle, int len)
  285. {
  286. struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
  287. u64 head = PERF_IDX2OFF(handle->head, buf);
  288. memset(buf->base + head, ARM_SPE_BUF_PAD_BYTE, len);
  289. if (!buf->snapshot)
  290. perf_aux_output_skip(handle, len);
  291. }
  292. static u64 arm_spe_pmu_next_snapshot_off(struct perf_output_handle *handle)
  293. {
  294. struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
  295. struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);
  296. u64 head = PERF_IDX2OFF(handle->head, buf);
  297. u64 limit = buf->nr_pages * PAGE_SIZE;
  298. /*
  299. * The trace format isn't parseable in reverse, so clamp
  300. * the limit to half of the buffer size in snapshot mode
  301. * so that the worst case is half a buffer of records, as
  302. * opposed to a single record.
  303. */
  304. if (head < limit >> 1)
  305. limit >>= 1;
  306. /*
  307. * If we're within max_record_sz of the limit, we must
  308. * pad, move the head index and recompute the limit.
  309. */
  310. if (limit - head < spe_pmu->max_record_sz) {
  311. arm_spe_pmu_pad_buf(handle, limit - head);
  312. handle->head = PERF_IDX2OFF(limit, buf);
  313. limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head;
  314. }
  315. return limit;
  316. }
  317. static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle)
  318. {
  319. struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);
  320. struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
  321. const u64 bufsize = buf->nr_pages * PAGE_SIZE;
  322. u64 limit = bufsize;
  323. u64 head, tail, wakeup;
  324. /*
  325. * The head can be misaligned for two reasons:
  326. *
  327. * 1. The hardware left PMBPTR pointing to the first byte after
  328. * a record when generating a buffer management event.
  329. *
  330. * 2. We used perf_aux_output_skip to consume handle->size bytes
  331. * and CIRC_SPACE was used to compute the size, which always
  332. * leaves one entry free.
  333. *
  334. * Deal with this by padding to the next alignment boundary and
  335. * moving the head index. If we run out of buffer space, we'll
  336. * reduce handle->size to zero and end up reporting truncation.
  337. */
  338. head = PERF_IDX2OFF(handle->head, buf);
  339. if (!IS_ALIGNED(head, spe_pmu->align)) {
  340. unsigned long delta = roundup(head, spe_pmu->align) - head;
  341. delta = min(delta, handle->size);
  342. arm_spe_pmu_pad_buf(handle, delta);
  343. head = PERF_IDX2OFF(handle->head, buf);
  344. }
  345. /* If we've run out of free space, then nothing more to do */
  346. if (!handle->size)
  347. goto no_space;
  348. /* Compute the tail and wakeup indices now that we've aligned head */
  349. tail = PERF_IDX2OFF(handle->head + handle->size, buf);
  350. wakeup = PERF_IDX2OFF(handle->wakeup, buf);
  351. /*
  352. * Avoid clobbering unconsumed data. We know we have space, so
  353. * if we see head == tail we know that the buffer is empty. If
  354. * head > tail, then there's nothing to clobber prior to
  355. * wrapping.
  356. */
  357. if (head < tail)
  358. limit = round_down(tail, PAGE_SIZE);
  359. /*
  360. * Wakeup may be arbitrarily far into the future. If it's not in
  361. * the current generation, either we'll wrap before hitting it,
  362. * or it's in the past and has been handled already.
  363. *
  364. * If there's a wakeup before we wrap, arrange to be woken up by
  365. * the page boundary following it. Keep the tail boundary if
  366. * that's lower.
  367. */
  368. if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
  369. limit = min(limit, round_up(wakeup, PAGE_SIZE));
  370. if (limit > head)
  371. return limit;
  372. arm_spe_pmu_pad_buf(handle, handle->size);
  373. no_space:
  374. perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
  375. perf_aux_output_end(handle, 0);
  376. return 0;
  377. }
  378. static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle)
  379. {
  380. struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
  381. struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);
  382. u64 limit = __arm_spe_pmu_next_off(handle);
  383. u64 head = PERF_IDX2OFF(handle->head, buf);
  384. /*
  385. * If the head has come too close to the end of the buffer,
  386. * then pad to the end and recompute the limit.
  387. */
  388. if (limit && (limit - head < spe_pmu->max_record_sz)) {
  389. arm_spe_pmu_pad_buf(handle, limit - head);
  390. limit = __arm_spe_pmu_next_off(handle);
  391. }
  392. return limit;
  393. }
  394. static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle,
  395. struct perf_event *event)
  396. {
  397. u64 base, limit;
  398. struct arm_spe_pmu_buf *buf;
  399. /* Start a new aux session */
  400. buf = perf_aux_output_begin(handle, event);
  401. if (!buf) {
  402. event->hw.state |= PERF_HES_STOPPED;
  403. /*
  404. * We still need to clear the limit pointer, since the
  405. * profiler might only be disabled by virtue of a fault.
  406. */
  407. limit = 0;
  408. goto out_write_limit;
  409. }
  410. limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle)
  411. : arm_spe_pmu_next_off(handle);
  412. if (limit)
  413. limit |= BIT(SYS_PMBLIMITR_EL1_E_SHIFT);
  414. limit += (u64)buf->base;
  415. base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf);
  416. write_sysreg_s(base, SYS_PMBPTR_EL1);
  417. out_write_limit:
  418. write_sysreg_s(limit, SYS_PMBLIMITR_EL1);
  419. }
  420. static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle)
  421. {
  422. struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
  423. u64 offset, size;
  424. offset = read_sysreg_s(SYS_PMBPTR_EL1) - (u64)buf->base;
  425. size = offset - PERF_IDX2OFF(handle->head, buf);
  426. if (buf->snapshot)
  427. handle->head = offset;
  428. perf_aux_output_end(handle, size);
  429. }
  430. static void arm_spe_pmu_disable_and_drain_local(void)
  431. {
  432. /* Disable profiling at EL0 and EL1 */
  433. write_sysreg_s(0, SYS_PMSCR_EL1);
  434. isb();
  435. /* Drain any buffered data */
  436. psb_csync();
  437. dsb(nsh);
  438. /* Disable the profiling buffer */
  439. write_sysreg_s(0, SYS_PMBLIMITR_EL1);
  440. isb();
  441. }
  442. /* IRQ handling */
  443. static enum arm_spe_pmu_buf_fault_action
  444. arm_spe_pmu_buf_get_fault_act(struct perf_output_handle *handle)
  445. {
  446. const char *err_str;
  447. u64 pmbsr;
  448. enum arm_spe_pmu_buf_fault_action ret;
  449. /*
  450. * Ensure new profiling data is visible to the CPU and any external
  451. * aborts have been resolved.
  452. */
  453. psb_csync();
  454. dsb(nsh);
  455. /* Ensure hardware updates to PMBPTR_EL1 are visible */
  456. isb();
  457. /* Service required? */
  458. pmbsr = read_sysreg_s(SYS_PMBSR_EL1);
  459. if (!(pmbsr & BIT(SYS_PMBSR_EL1_S_SHIFT)))
  460. return SPE_PMU_BUF_FAULT_ACT_SPURIOUS;
  461. /*
  462. * If we've lost data, disable profiling and also set the PARTIAL
  463. * flag to indicate that the last record is corrupted.
  464. */
  465. if (pmbsr & BIT(SYS_PMBSR_EL1_DL_SHIFT))
  466. perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED |
  467. PERF_AUX_FLAG_PARTIAL);
  468. /* Report collisions to userspace so that it can up the period */
  469. if (pmbsr & BIT(SYS_PMBSR_EL1_COLL_SHIFT))
  470. perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);
  471. /* We only expect buffer management events */
  472. switch (pmbsr & (SYS_PMBSR_EL1_EC_MASK << SYS_PMBSR_EL1_EC_SHIFT)) {
  473. case SYS_PMBSR_EL1_EC_BUF:
  474. /* Handled below */
  475. break;
  476. case SYS_PMBSR_EL1_EC_FAULT_S1:
  477. case SYS_PMBSR_EL1_EC_FAULT_S2:
  478. err_str = "Unexpected buffer fault";
  479. goto out_err;
  480. default:
  481. err_str = "Unknown error code";
  482. goto out_err;
  483. }
  484. /* Buffer management event */
  485. switch (pmbsr &
  486. (SYS_PMBSR_EL1_BUF_BSC_MASK << SYS_PMBSR_EL1_BUF_BSC_SHIFT)) {
  487. case SYS_PMBSR_EL1_BUF_BSC_FULL:
  488. ret = SPE_PMU_BUF_FAULT_ACT_OK;
  489. goto out_stop;
  490. default:
  491. err_str = "Unknown buffer status code";
  492. }
  493. out_err:
  494. pr_err_ratelimited("%s on CPU %d [PMBSR=0x%016llx, PMBPTR=0x%016llx, PMBLIMITR=0x%016llx]\n",
  495. err_str, smp_processor_id(), pmbsr,
  496. read_sysreg_s(SYS_PMBPTR_EL1),
  497. read_sysreg_s(SYS_PMBLIMITR_EL1));
  498. ret = SPE_PMU_BUF_FAULT_ACT_FATAL;
  499. out_stop:
  500. arm_spe_perf_aux_output_end(handle);
  501. return ret;
  502. }
  503. static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev)
  504. {
  505. struct perf_output_handle *handle = dev;
  506. struct perf_event *event = handle->event;
  507. enum arm_spe_pmu_buf_fault_action act;
  508. if (!perf_get_aux(handle))
  509. return IRQ_NONE;
  510. act = arm_spe_pmu_buf_get_fault_act(handle);
  511. if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS)
  512. return IRQ_NONE;
  513. /*
  514. * Ensure perf callbacks have completed, which may disable the
  515. * profiling buffer in response to a TRUNCATION flag.
  516. */
  517. irq_work_run();
  518. switch (act) {
  519. case SPE_PMU_BUF_FAULT_ACT_FATAL:
  520. /*
  521. * If a fatal exception occurred then leaving the profiling
  522. * buffer enabled is a recipe waiting to happen. Since
  523. * fatal faults don't always imply truncation, make sure
  524. * that the profiling buffer is disabled explicitly before
  525. * clearing the syndrome register.
  526. */
  527. arm_spe_pmu_disable_and_drain_local();
  528. break;
  529. case SPE_PMU_BUF_FAULT_ACT_OK:
  530. /*
  531. * We handled the fault (the buffer was full), so resume
  532. * profiling as long as we didn't detect truncation.
  533. * PMBPTR might be misaligned, but we'll burn that bridge
  534. * when we get to it.
  535. */
  536. if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) {
  537. arm_spe_perf_aux_output_begin(handle, event);
  538. isb();
  539. }
  540. break;
  541. case SPE_PMU_BUF_FAULT_ACT_SPURIOUS:
  542. /* We've seen you before, but GCC has the memory of a sieve. */
  543. break;
  544. }
  545. /* The buffer pointers are now sane, so resume profiling. */
  546. write_sysreg_s(0, SYS_PMBSR_EL1);
  547. return IRQ_HANDLED;
  548. }
  549. /* Perf callbacks */
  550. static int arm_spe_pmu_event_init(struct perf_event *event)
  551. {
  552. u64 reg;
  553. struct perf_event_attr *attr = &event->attr;
  554. struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
  555. /* This is, of course, deeply driver-specific */
  556. if (attr->type != event->pmu->type)
  557. return -ENOENT;
  558. if (event->cpu >= 0 &&
  559. !cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus))
  560. return -ENOENT;
  561. if (arm_spe_event_to_pmsevfr(event) & SYS_PMSEVFR_EL1_RES0)
  562. return -EOPNOTSUPP;
  563. if (attr->exclude_idle)
  564. return -EOPNOTSUPP;
  565. /*
  566. * Feedback-directed frequency throttling doesn't work when we
  567. * have a buffer of samples. We'd need to manually count the
  568. * samples in the buffer when it fills up and adjust the event
  569. * count to reflect that. Instead, just force the user to specify
  570. * a sample period.
  571. */
  572. if (attr->freq)
  573. return -EINVAL;
  574. reg = arm_spe_event_to_pmsfcr(event);
  575. if ((reg & BIT(SYS_PMSFCR_EL1_FE_SHIFT)) &&
  576. !(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT))
  577. return -EOPNOTSUPP;
  578. if ((reg & BIT(SYS_PMSFCR_EL1_FT_SHIFT)) &&
  579. !(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP))
  580. return -EOPNOTSUPP;
  581. if ((reg & BIT(SYS_PMSFCR_EL1_FL_SHIFT)) &&
  582. !(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT))
  583. return -EOPNOTSUPP;
  584. reg = arm_spe_event_to_pmscr(event);
  585. if (!perfmon_capable() &&
  586. (reg & (BIT(SYS_PMSCR_EL1_PA_SHIFT) |
  587. BIT(SYS_PMSCR_EL1_CX_SHIFT) |
  588. BIT(SYS_PMSCR_EL1_PCT_SHIFT))))
  589. return -EACCES;
  590. return 0;
  591. }
  592. static void arm_spe_pmu_start(struct perf_event *event, int flags)
  593. {
  594. u64 reg;
  595. struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
  596. struct hw_perf_event *hwc = &event->hw;
  597. struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);
  598. hwc->state = 0;
  599. arm_spe_perf_aux_output_begin(handle, event);
  600. if (hwc->state)
  601. return;
  602. reg = arm_spe_event_to_pmsfcr(event);
  603. write_sysreg_s(reg, SYS_PMSFCR_EL1);
  604. reg = arm_spe_event_to_pmsevfr(event);
  605. write_sysreg_s(reg, SYS_PMSEVFR_EL1);
  606. reg = arm_spe_event_to_pmslatfr(event);
  607. write_sysreg_s(reg, SYS_PMSLATFR_EL1);
  608. if (flags & PERF_EF_RELOAD) {
  609. reg = arm_spe_event_to_pmsirr(event);
  610. write_sysreg_s(reg, SYS_PMSIRR_EL1);
  611. isb();
  612. reg = local64_read(&hwc->period_left);
  613. write_sysreg_s(reg, SYS_PMSICR_EL1);
  614. }
  615. reg = arm_spe_event_to_pmscr(event);
  616. isb();
  617. write_sysreg_s(reg, SYS_PMSCR_EL1);
  618. }
  619. static void arm_spe_pmu_stop(struct perf_event *event, int flags)
  620. {
  621. struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
  622. struct hw_perf_event *hwc = &event->hw;
  623. struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);
  624. /* If we're already stopped, then nothing to do */
  625. if (hwc->state & PERF_HES_STOPPED)
  626. return;
  627. /* Stop all trace generation */
  628. arm_spe_pmu_disable_and_drain_local();
  629. if (flags & PERF_EF_UPDATE) {
  630. /*
  631. * If there's a fault pending then ensure we contain it
  632. * to this buffer, since we might be on the context-switch
  633. * path.
  634. */
  635. if (perf_get_aux(handle)) {
  636. enum arm_spe_pmu_buf_fault_action act;
  637. act = arm_spe_pmu_buf_get_fault_act(handle);
  638. if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS)
  639. arm_spe_perf_aux_output_end(handle);
  640. else
  641. write_sysreg_s(0, SYS_PMBSR_EL1);
  642. }
  643. /*
  644. * This may also contain ECOUNT, but nobody else should
  645. * be looking at period_left, since we forbid frequency
  646. * based sampling.
  647. */
  648. local64_set(&hwc->period_left, read_sysreg_s(SYS_PMSICR_EL1));
  649. hwc->state |= PERF_HES_UPTODATE;
  650. }
  651. hwc->state |= PERF_HES_STOPPED;
  652. }
  653. static int arm_spe_pmu_add(struct perf_event *event, int flags)
  654. {
  655. int ret = 0;
  656. struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
  657. struct hw_perf_event *hwc = &event->hw;
  658. int cpu = event->cpu == -1 ? smp_processor_id() : event->cpu;
  659. if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))
  660. return -ENOENT;
  661. hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
  662. if (flags & PERF_EF_START) {
  663. arm_spe_pmu_start(event, PERF_EF_RELOAD);
  664. if (hwc->state & PERF_HES_STOPPED)
  665. ret = -EINVAL;
  666. }
  667. return ret;
  668. }
  669. static void arm_spe_pmu_del(struct perf_event *event, int flags)
  670. {
  671. arm_spe_pmu_stop(event, PERF_EF_UPDATE);
  672. }
  673. static void arm_spe_pmu_read(struct perf_event *event)
  674. {
  675. }
  676. static void *arm_spe_pmu_setup_aux(struct perf_event *event, void **pages,
  677. int nr_pages, bool snapshot)
  678. {
  679. int i, cpu = event->cpu;
  680. struct page **pglist;
  681. struct arm_spe_pmu_buf *buf;
  682. /* We need at least two pages for this to work. */
  683. if (nr_pages < 2)
  684. return NULL;
  685. /*
  686. * We require an even number of pages for snapshot mode, so that
  687. * we can effectively treat the buffer as consisting of two equal
  688. * parts and give userspace a fighting chance of getting some
  689. * useful data out of it.
  690. */
  691. if (snapshot && (nr_pages & 1))
  692. return NULL;
  693. if (cpu == -1)
  694. cpu = raw_smp_processor_id();
  695. buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, cpu_to_node(cpu));
  696. if (!buf)
  697. return NULL;
  698. pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL);
  699. if (!pglist)
  700. goto out_free_buf;
  701. for (i = 0; i < nr_pages; ++i)
  702. pglist[i] = virt_to_page(pages[i]);
  703. buf->base = vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);
  704. if (!buf->base)
  705. goto out_free_pglist;
  706. buf->nr_pages = nr_pages;
  707. buf->snapshot = snapshot;
  708. kfree(pglist);
  709. return buf;
  710. out_free_pglist:
  711. kfree(pglist);
  712. out_free_buf:
  713. kfree(buf);
  714. return NULL;
  715. }
  716. static void arm_spe_pmu_free_aux(void *aux)
  717. {
  718. struct arm_spe_pmu_buf *buf = aux;
  719. vunmap(buf->base);
  720. kfree(buf);
  721. }
  722. /* Initialisation and teardown functions */
  723. static int arm_spe_pmu_perf_init(struct arm_spe_pmu *spe_pmu)
  724. {
  725. static atomic_t pmu_idx = ATOMIC_INIT(-1);
  726. int idx;
  727. char *name;
  728. struct device *dev = &spe_pmu->pdev->dev;
  729. spe_pmu->pmu = (struct pmu) {
  730. .module = THIS_MODULE,
  731. .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE,
  732. .attr_groups = arm_spe_pmu_attr_groups,
  733. /*
  734. * We hitch a ride on the software context here, so that
  735. * we can support per-task profiling (which is not possible
  736. * with the invalid context as it doesn't get sched callbacks).
  737. * This requires that userspace either uses a dummy event for
  738. * perf_event_open, since the aux buffer is not setup until
  739. * a subsequent mmap, or creates the profiling event in a
  740. * disabled state and explicitly PERF_EVENT_IOC_ENABLEs it
  741. * once the buffer has been created.
  742. */
  743. .task_ctx_nr = perf_sw_context,
  744. .event_init = arm_spe_pmu_event_init,
  745. .add = arm_spe_pmu_add,
  746. .del = arm_spe_pmu_del,
  747. .start = arm_spe_pmu_start,
  748. .stop = arm_spe_pmu_stop,
  749. .read = arm_spe_pmu_read,
  750. .setup_aux = arm_spe_pmu_setup_aux,
  751. .free_aux = arm_spe_pmu_free_aux,
  752. };
  753. idx = atomic_inc_return(&pmu_idx);
  754. name = devm_kasprintf(dev, GFP_KERNEL, "%s_%d", PMUNAME, idx);
  755. if (!name) {
  756. dev_err(dev, "failed to allocate name for pmu %d\n", idx);
  757. return -ENOMEM;
  758. }
  759. return perf_pmu_register(&spe_pmu->pmu, name, -1);
  760. }
  761. static void arm_spe_pmu_perf_destroy(struct arm_spe_pmu *spe_pmu)
  762. {
  763. perf_pmu_unregister(&spe_pmu->pmu);
  764. }
  765. static void __arm_spe_pmu_dev_probe(void *info)
  766. {
  767. int fld;
  768. u64 reg;
  769. struct arm_spe_pmu *spe_pmu = info;
  770. struct device *dev = &spe_pmu->pdev->dev;
  771. fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64DFR0_EL1),
  772. ID_AA64DFR0_PMSVER_SHIFT);
  773. if (!fld) {
  774. dev_err(dev,
  775. "unsupported ID_AA64DFR0_EL1.PMSVer [%d] on CPU %d\n",
  776. fld, smp_processor_id());
  777. return;
  778. }
  779. /* Read PMBIDR first to determine whether or not we have access */
  780. reg = read_sysreg_s(SYS_PMBIDR_EL1);
  781. if (reg & BIT(SYS_PMBIDR_EL1_P_SHIFT)) {
  782. dev_err(dev,
  783. "profiling buffer owned by higher exception level\n");
  784. return;
  785. }
  786. /* Minimum alignment. If it's out-of-range, then fail the probe */
  787. fld = reg >> SYS_PMBIDR_EL1_ALIGN_SHIFT & SYS_PMBIDR_EL1_ALIGN_MASK;
  788. spe_pmu->align = 1 << fld;
  789. if (spe_pmu->align > SZ_2K) {
  790. dev_err(dev, "unsupported PMBIDR.Align [%d] on CPU %d\n",
  791. fld, smp_processor_id());
  792. return;
  793. }
  794. /* It's now safe to read PMSIDR and figure out what we've got */
  795. reg = read_sysreg_s(SYS_PMSIDR_EL1);
  796. if (reg & BIT(SYS_PMSIDR_EL1_FE_SHIFT))
  797. spe_pmu->features |= SPE_PMU_FEAT_FILT_EVT;
  798. if (reg & BIT(SYS_PMSIDR_EL1_FT_SHIFT))
  799. spe_pmu->features |= SPE_PMU_FEAT_FILT_TYP;
  800. if (reg & BIT(SYS_PMSIDR_EL1_FL_SHIFT))
  801. spe_pmu->features |= SPE_PMU_FEAT_FILT_LAT;
  802. if (reg & BIT(SYS_PMSIDR_EL1_ARCHINST_SHIFT))
  803. spe_pmu->features |= SPE_PMU_FEAT_ARCH_INST;
  804. if (reg & BIT(SYS_PMSIDR_EL1_LDS_SHIFT))
  805. spe_pmu->features |= SPE_PMU_FEAT_LDS;
  806. if (reg & BIT(SYS_PMSIDR_EL1_ERND_SHIFT))
  807. spe_pmu->features |= SPE_PMU_FEAT_ERND;
  808. /* This field has a spaced out encoding, so just use a look-up */
  809. fld = reg >> SYS_PMSIDR_EL1_INTERVAL_SHIFT & SYS_PMSIDR_EL1_INTERVAL_MASK;
  810. switch (fld) {
  811. case 0:
  812. spe_pmu->min_period = 256;
  813. break;
  814. case 2:
  815. spe_pmu->min_period = 512;
  816. break;
  817. case 3:
  818. spe_pmu->min_period = 768;
  819. break;
  820. case 4:
  821. spe_pmu->min_period = 1024;
  822. break;
  823. case 5:
  824. spe_pmu->min_period = 1536;
  825. break;
  826. case 6:
  827. spe_pmu->min_period = 2048;
  828. break;
  829. case 7:
  830. spe_pmu->min_period = 3072;
  831. break;
  832. default:
  833. dev_warn(dev, "unknown PMSIDR_EL1.Interval [%d]; assuming 8\n",
  834. fld);
  835. fallthrough;
  836. case 8:
  837. spe_pmu->min_period = 4096;
  838. }
  839. /* Maximum record size. If it's out-of-range, then fail the probe */
  840. fld = reg >> SYS_PMSIDR_EL1_MAXSIZE_SHIFT & SYS_PMSIDR_EL1_MAXSIZE_MASK;
  841. spe_pmu->max_record_sz = 1 << fld;
  842. if (spe_pmu->max_record_sz > SZ_2K || spe_pmu->max_record_sz < 16) {
  843. dev_err(dev, "unsupported PMSIDR_EL1.MaxSize [%d] on CPU %d\n",
  844. fld, smp_processor_id());
  845. return;
  846. }
  847. fld = reg >> SYS_PMSIDR_EL1_COUNTSIZE_SHIFT & SYS_PMSIDR_EL1_COUNTSIZE_MASK;
  848. switch (fld) {
  849. default:
  850. dev_warn(dev, "unknown PMSIDR_EL1.CountSize [%d]; assuming 2\n",
  851. fld);
  852. fallthrough;
  853. case 2:
  854. spe_pmu->counter_sz = 12;
  855. }
  856. dev_info(dev,
  857. "probed for CPUs %*pbl [max_record_sz %u, align %u, features 0x%llx]\n",
  858. cpumask_pr_args(&spe_pmu->supported_cpus),
  859. spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features);
  860. spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED;
  861. return;
  862. }
  863. static void __arm_spe_pmu_reset_local(void)
  864. {
  865. /*
  866. * This is probably overkill, as we have no idea where we're
  867. * draining any buffered data to...
  868. */
  869. arm_spe_pmu_disable_and_drain_local();
  870. /* Reset the buffer base pointer */
  871. write_sysreg_s(0, SYS_PMBPTR_EL1);
  872. isb();
  873. /* Clear any pending management interrupts */
  874. write_sysreg_s(0, SYS_PMBSR_EL1);
  875. isb();
  876. }
  877. static void __arm_spe_pmu_setup_one(void *info)
  878. {
  879. struct arm_spe_pmu *spe_pmu = info;
  880. __arm_spe_pmu_reset_local();
  881. enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE);
  882. }
  883. static void __arm_spe_pmu_stop_one(void *info)
  884. {
  885. struct arm_spe_pmu *spe_pmu = info;
  886. disable_percpu_irq(spe_pmu->irq);
  887. __arm_spe_pmu_reset_local();
  888. }
  889. static int arm_spe_pmu_cpu_startup(unsigned int cpu, struct hlist_node *node)
  890. {
  891. struct arm_spe_pmu *spe_pmu;
  892. spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node);
  893. if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))
  894. return 0;
  895. __arm_spe_pmu_setup_one(spe_pmu);
  896. return 0;
  897. }
  898. static int arm_spe_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
  899. {
  900. struct arm_spe_pmu *spe_pmu;
  901. spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node);
  902. if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))
  903. return 0;
  904. __arm_spe_pmu_stop_one(spe_pmu);
  905. return 0;
  906. }
  907. static int arm_spe_pmu_dev_init(struct arm_spe_pmu *spe_pmu)
  908. {
  909. int ret;
  910. cpumask_t *mask = &spe_pmu->supported_cpus;
  911. /* Make sure we probe the hardware on a relevant CPU */
  912. ret = smp_call_function_any(mask, __arm_spe_pmu_dev_probe, spe_pmu, 1);
  913. if (ret || !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED))
  914. return -ENXIO;
  915. /* Request our PPIs (note that the IRQ is still disabled) */
  916. ret = request_percpu_irq(spe_pmu->irq, arm_spe_pmu_irq_handler, DRVNAME,
  917. spe_pmu->handle);
  918. if (ret)
  919. return ret;
  920. /*
  921. * Register our hotplug notifier now so we don't miss any events.
  922. * This will enable the IRQ for any supported CPUs that are already
  923. * up.
  924. */
  925. ret = cpuhp_state_add_instance(arm_spe_pmu_online,
  926. &spe_pmu->hotplug_node);
  927. if (ret)
  928. free_percpu_irq(spe_pmu->irq, spe_pmu->handle);
  929. return ret;
  930. }
  931. static void arm_spe_pmu_dev_teardown(struct arm_spe_pmu *spe_pmu)
  932. {
  933. cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node);
  934. free_percpu_irq(spe_pmu->irq, spe_pmu->handle);
  935. }
  936. /* Driver and device probing */
  937. static int arm_spe_pmu_irq_probe(struct arm_spe_pmu *spe_pmu)
  938. {
  939. struct platform_device *pdev = spe_pmu->pdev;
  940. int irq = platform_get_irq(pdev, 0);
  941. if (irq < 0)
  942. return -ENXIO;
  943. if (!irq_is_percpu(irq)) {
  944. dev_err(&pdev->dev, "expected PPI but got SPI (%d)\n", irq);
  945. return -EINVAL;
  946. }
  947. if (irq_get_percpu_devid_partition(irq, &spe_pmu->supported_cpus)) {
  948. dev_err(&pdev->dev, "failed to get PPI partition (%d)\n", irq);
  949. return -EINVAL;
  950. }
  951. spe_pmu->irq = irq;
  952. return 0;
  953. }
  954. static const struct of_device_id arm_spe_pmu_of_match[] = {
  955. { .compatible = "arm,statistical-profiling-extension-v1", .data = (void *)1 },
  956. { /* Sentinel */ },
  957. };
  958. MODULE_DEVICE_TABLE(of, arm_spe_pmu_of_match);
  959. static const struct platform_device_id arm_spe_match[] = {
  960. { ARMV8_SPE_PDEV_NAME, 0},
  961. { }
  962. };
  963. MODULE_DEVICE_TABLE(platform, arm_spe_match);
  964. static int arm_spe_pmu_device_probe(struct platform_device *pdev)
  965. {
  966. int ret;
  967. struct arm_spe_pmu *spe_pmu;
  968. struct device *dev = &pdev->dev;
  969. /*
  970. * If kernelspace is unmapped when running at EL0, then the SPE
  971. * buffer will fault and prematurely terminate the AUX session.
  972. */
  973. if (arm64_kernel_unmapped_at_el0()) {
  974. dev_warn_once(dev, "profiling buffer inaccessible. Try passing \"kpti=off\" on the kernel command line\n");
  975. return -EPERM;
  976. }
  977. spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL);
  978. if (!spe_pmu) {
  979. dev_err(dev, "failed to allocate spe_pmu\n");
  980. return -ENOMEM;
  981. }
  982. spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle));
  983. if (!spe_pmu->handle)
  984. return -ENOMEM;
  985. spe_pmu->pdev = pdev;
  986. platform_set_drvdata(pdev, spe_pmu);
  987. ret = arm_spe_pmu_irq_probe(spe_pmu);
  988. if (ret)
  989. goto out_free_handle;
  990. ret = arm_spe_pmu_dev_init(spe_pmu);
  991. if (ret)
  992. goto out_free_handle;
  993. ret = arm_spe_pmu_perf_init(spe_pmu);
  994. if (ret)
  995. goto out_teardown_dev;
  996. return 0;
  997. out_teardown_dev:
  998. arm_spe_pmu_dev_teardown(spe_pmu);
  999. out_free_handle:
  1000. free_percpu(spe_pmu->handle);
  1001. return ret;
  1002. }
  1003. static int arm_spe_pmu_device_remove(struct platform_device *pdev)
  1004. {
  1005. struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev);
  1006. arm_spe_pmu_perf_destroy(spe_pmu);
  1007. arm_spe_pmu_dev_teardown(spe_pmu);
  1008. free_percpu(spe_pmu->handle);
  1009. return 0;
  1010. }
  1011. static struct platform_driver arm_spe_pmu_driver = {
  1012. .id_table = arm_spe_match,
  1013. .driver = {
  1014. .name = DRVNAME,
  1015. .of_match_table = of_match_ptr(arm_spe_pmu_of_match),
  1016. .suppress_bind_attrs = true,
  1017. },
  1018. .probe = arm_spe_pmu_device_probe,
  1019. .remove = arm_spe_pmu_device_remove,
  1020. };
  1021. static int __init arm_spe_pmu_init(void)
  1022. {
  1023. int ret;
  1024. ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME,
  1025. arm_spe_pmu_cpu_startup,
  1026. arm_spe_pmu_cpu_teardown);
  1027. if (ret < 0)
  1028. return ret;
  1029. arm_spe_pmu_online = ret;
  1030. ret = platform_driver_register(&arm_spe_pmu_driver);
  1031. if (ret)
  1032. cpuhp_remove_multi_state(arm_spe_pmu_online);
  1033. return ret;
  1034. }
  1035. static void __exit arm_spe_pmu_exit(void)
  1036. {
  1037. platform_driver_unregister(&arm_spe_pmu_driver);
  1038. cpuhp_remove_multi_state(arm_spe_pmu_online);
  1039. }
  1040. module_init(arm_spe_pmu_init);
  1041. module_exit(arm_spe_pmu_exit);
  1042. MODULE_DESCRIPTION("Perf driver for the ARMv8.2 Statistical Profiling Extension");
  1043. MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
  1044. MODULE_LICENSE("GPL v2");