arm_dsu_pmu.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * ARM DynamIQ Shared Unit (DSU) PMU driver
  4. *
  5. * Copyright (C) ARM Limited, 2017.
  6. *
  7. * Based on ARM CCI-PMU, ARMv8 PMU-v3 drivers.
  8. */
  9. #define PMUNAME "arm_dsu"
  10. #define DRVNAME PMUNAME "_pmu"
  11. #define pr_fmt(fmt) DRVNAME ": " fmt
  12. #include <linux/acpi.h>
  13. #include <linux/bitmap.h>
  14. #include <linux/bitops.h>
  15. #include <linux/bug.h>
  16. #include <linux/cpumask.h>
  17. #include <linux/device.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/kernel.h>
  20. #include <linux/module.h>
  21. #include <linux/of_device.h>
  22. #include <linux/perf_event.h>
  23. #include <linux/platform_device.h>
  24. #include <linux/spinlock.h>
  25. #include <linux/smp.h>
  26. #include <linux/sysfs.h>
  27. #include <linux/types.h>
  28. #include <asm/arm_dsu_pmu.h>
  29. #include <asm/local64.h>
  30. /* PMU event codes */
  31. #define DSU_PMU_EVT_CYCLES 0x11
  32. #define DSU_PMU_EVT_CHAIN 0x1e
  33. #define DSU_PMU_MAX_COMMON_EVENTS 0x40
  34. #define DSU_PMU_MAX_HW_CNTRS 32
  35. #define DSU_PMU_HW_COUNTER_MASK (DSU_PMU_MAX_HW_CNTRS - 1)
  36. #define CLUSTERPMCR_E BIT(0)
  37. #define CLUSTERPMCR_P BIT(1)
  38. #define CLUSTERPMCR_C BIT(2)
  39. #define CLUSTERPMCR_N_SHIFT 11
  40. #define CLUSTERPMCR_N_MASK 0x1f
  41. #define CLUSTERPMCR_IDCODE_SHIFT 16
  42. #define CLUSTERPMCR_IDCODE_MASK 0xff
  43. #define CLUSTERPMCR_IMP_SHIFT 24
  44. #define CLUSTERPMCR_IMP_MASK 0xff
  45. #define CLUSTERPMCR_RES_MASK 0x7e8
  46. #define CLUSTERPMCR_RES_VAL 0x40
  47. #define DSU_ACTIVE_CPU_MASK 0x0
  48. #define DSU_ASSOCIATED_CPU_MASK 0x1
  49. /*
  50. * We use the index of the counters as they appear in the counter
  51. * bit maps in the PMU registers (e.g CLUSTERPMSELR).
  52. * i.e,
  53. * counter 0 - Bit 0
  54. * counter 1 - Bit 1
  55. * ...
  56. * Cycle counter - Bit 31
  57. */
  58. #define DSU_PMU_IDX_CYCLE_COUNTER 31
  59. /* All event counters are 32bit, with a 64bit Cycle counter */
  60. #define DSU_PMU_COUNTER_WIDTH(idx) \
  61. (((idx) == DSU_PMU_IDX_CYCLE_COUNTER) ? 64 : 32)
  62. #define DSU_PMU_COUNTER_MASK(idx) \
  63. GENMASK_ULL((DSU_PMU_COUNTER_WIDTH((idx)) - 1), 0)
  64. #define DSU_EXT_ATTR(_name, _func, _config) \
  65. (&((struct dev_ext_attribute[]) { \
  66. { \
  67. .attr = __ATTR(_name, 0444, _func, NULL), \
  68. .var = (void *)_config \
  69. } \
  70. })[0].attr.attr)
  71. #define DSU_EVENT_ATTR(_name, _config) \
  72. DSU_EXT_ATTR(_name, dsu_pmu_sysfs_event_show, (unsigned long)_config)
  73. #define DSU_FORMAT_ATTR(_name, _config) \
  74. DSU_EXT_ATTR(_name, dsu_pmu_sysfs_format_show, (char *)_config)
  75. #define DSU_CPUMASK_ATTR(_name, _config) \
  76. DSU_EXT_ATTR(_name, dsu_pmu_cpumask_show, (unsigned long)_config)
  77. struct dsu_hw_events {
  78. DECLARE_BITMAP(used_mask, DSU_PMU_MAX_HW_CNTRS);
  79. struct perf_event *events[DSU_PMU_MAX_HW_CNTRS];
  80. };
  81. /*
  82. * struct dsu_pmu - DSU PMU descriptor
  83. *
  84. * @pmu_lock : Protects accesses to DSU PMU register from normal vs
  85. * interrupt handler contexts.
  86. * @hw_events : Holds the event counter state.
  87. * @associated_cpus : CPUs attached to the DSU.
  88. * @active_cpu : CPU to which the PMU is bound for accesses.
  89. * @cpuhp_node : Node for CPU hotplug notifier link.
  90. * @num_counters : Number of event counters implemented by the PMU,
  91. * excluding the cycle counter.
  92. * @irq : Interrupt line for counter overflow.
  93. * @cpmceid_bitmap : Bitmap for the availability of architected common
  94. * events (event_code < 0x40).
  95. */
  96. struct dsu_pmu {
  97. struct pmu pmu;
  98. struct device *dev;
  99. raw_spinlock_t pmu_lock;
  100. struct dsu_hw_events hw_events;
  101. cpumask_t associated_cpus;
  102. cpumask_t active_cpu;
  103. struct hlist_node cpuhp_node;
  104. s8 num_counters;
  105. int irq;
  106. DECLARE_BITMAP(cpmceid_bitmap, DSU_PMU_MAX_COMMON_EVENTS);
  107. };
  108. static unsigned long dsu_pmu_cpuhp_state;
  109. static inline struct dsu_pmu *to_dsu_pmu(struct pmu *pmu)
  110. {
  111. return container_of(pmu, struct dsu_pmu, pmu);
  112. }
  113. static ssize_t dsu_pmu_sysfs_event_show(struct device *dev,
  114. struct device_attribute *attr,
  115. char *buf)
  116. {
  117. struct dev_ext_attribute *eattr = container_of(attr,
  118. struct dev_ext_attribute, attr);
  119. return snprintf(buf, PAGE_SIZE, "event=0x%lx\n",
  120. (unsigned long)eattr->var);
  121. }
  122. static ssize_t dsu_pmu_sysfs_format_show(struct device *dev,
  123. struct device_attribute *attr,
  124. char *buf)
  125. {
  126. struct dev_ext_attribute *eattr = container_of(attr,
  127. struct dev_ext_attribute, attr);
  128. return snprintf(buf, PAGE_SIZE, "%s\n", (char *)eattr->var);
  129. }
  130. static ssize_t dsu_pmu_cpumask_show(struct device *dev,
  131. struct device_attribute *attr,
  132. char *buf)
  133. {
  134. struct pmu *pmu = dev_get_drvdata(dev);
  135. struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
  136. struct dev_ext_attribute *eattr = container_of(attr,
  137. struct dev_ext_attribute, attr);
  138. unsigned long mask_id = (unsigned long)eattr->var;
  139. const cpumask_t *cpumask;
  140. switch (mask_id) {
  141. case DSU_ACTIVE_CPU_MASK:
  142. cpumask = &dsu_pmu->active_cpu;
  143. break;
  144. case DSU_ASSOCIATED_CPU_MASK:
  145. cpumask = &dsu_pmu->associated_cpus;
  146. break;
  147. default:
  148. return 0;
  149. }
  150. return cpumap_print_to_pagebuf(true, buf, cpumask);
  151. }
  152. static struct attribute *dsu_pmu_format_attrs[] = {
  153. DSU_FORMAT_ATTR(event, "config:0-31"),
  154. NULL,
  155. };
  156. static const struct attribute_group dsu_pmu_format_attr_group = {
  157. .name = "format",
  158. .attrs = dsu_pmu_format_attrs,
  159. };
  160. static struct attribute *dsu_pmu_event_attrs[] = {
  161. DSU_EVENT_ATTR(cycles, 0x11),
  162. DSU_EVENT_ATTR(bus_access, 0x19),
  163. DSU_EVENT_ATTR(memory_error, 0x1a),
  164. DSU_EVENT_ATTR(bus_cycles, 0x1d),
  165. DSU_EVENT_ATTR(l3d_cache_allocate, 0x29),
  166. DSU_EVENT_ATTR(l3d_cache_refill, 0x2a),
  167. DSU_EVENT_ATTR(l3d_cache, 0x2b),
  168. DSU_EVENT_ATTR(l3d_cache_wb, 0x2c),
  169. NULL,
  170. };
  171. static umode_t
  172. dsu_pmu_event_attr_is_visible(struct kobject *kobj, struct attribute *attr,
  173. int unused)
  174. {
  175. struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
  176. struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
  177. struct dev_ext_attribute *eattr = container_of(attr,
  178. struct dev_ext_attribute, attr.attr);
  179. unsigned long evt = (unsigned long)eattr->var;
  180. return test_bit(evt, dsu_pmu->cpmceid_bitmap) ? attr->mode : 0;
  181. }
  182. static const struct attribute_group dsu_pmu_events_attr_group = {
  183. .name = "events",
  184. .attrs = dsu_pmu_event_attrs,
  185. .is_visible = dsu_pmu_event_attr_is_visible,
  186. };
  187. static struct attribute *dsu_pmu_cpumask_attrs[] = {
  188. DSU_CPUMASK_ATTR(cpumask, DSU_ACTIVE_CPU_MASK),
  189. DSU_CPUMASK_ATTR(associated_cpus, DSU_ASSOCIATED_CPU_MASK),
  190. NULL,
  191. };
  192. static const struct attribute_group dsu_pmu_cpumask_attr_group = {
  193. .attrs = dsu_pmu_cpumask_attrs,
  194. };
  195. static const struct attribute_group *dsu_pmu_attr_groups[] = {
  196. &dsu_pmu_cpumask_attr_group,
  197. &dsu_pmu_events_attr_group,
  198. &dsu_pmu_format_attr_group,
  199. NULL,
  200. };
  201. static int dsu_pmu_get_online_cpu_any_but(struct dsu_pmu *dsu_pmu, int cpu)
  202. {
  203. struct cpumask online_supported;
  204. cpumask_and(&online_supported,
  205. &dsu_pmu->associated_cpus, cpu_online_mask);
  206. return cpumask_any_but(&online_supported, cpu);
  207. }
  208. static inline bool dsu_pmu_counter_valid(struct dsu_pmu *dsu_pmu, u32 idx)
  209. {
  210. return (idx < dsu_pmu->num_counters) ||
  211. (idx == DSU_PMU_IDX_CYCLE_COUNTER);
  212. }
  213. static inline u64 dsu_pmu_read_counter(struct perf_event *event)
  214. {
  215. u64 val;
  216. unsigned long flags;
  217. struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
  218. int idx = event->hw.idx;
  219. if (WARN_ON(!cpumask_test_cpu(smp_processor_id(),
  220. &dsu_pmu->associated_cpus)))
  221. return 0;
  222. if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
  223. dev_err(event->pmu->dev,
  224. "Trying reading invalid counter %d\n", idx);
  225. return 0;
  226. }
  227. raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
  228. if (idx == DSU_PMU_IDX_CYCLE_COUNTER)
  229. val = __dsu_pmu_read_pmccntr();
  230. else
  231. val = __dsu_pmu_read_counter(idx);
  232. raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
  233. return val;
  234. }
  235. static void dsu_pmu_write_counter(struct perf_event *event, u64 val)
  236. {
  237. unsigned long flags;
  238. struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
  239. int idx = event->hw.idx;
  240. if (WARN_ON(!cpumask_test_cpu(smp_processor_id(),
  241. &dsu_pmu->associated_cpus)))
  242. return;
  243. if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
  244. dev_err(event->pmu->dev,
  245. "writing to invalid counter %d\n", idx);
  246. return;
  247. }
  248. raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
  249. if (idx == DSU_PMU_IDX_CYCLE_COUNTER)
  250. __dsu_pmu_write_pmccntr(val);
  251. else
  252. __dsu_pmu_write_counter(idx, val);
  253. raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
  254. }
  255. static int dsu_pmu_get_event_idx(struct dsu_hw_events *hw_events,
  256. struct perf_event *event)
  257. {
  258. int idx;
  259. unsigned long evtype = event->attr.config;
  260. struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
  261. unsigned long *used_mask = hw_events->used_mask;
  262. if (evtype == DSU_PMU_EVT_CYCLES) {
  263. if (test_and_set_bit(DSU_PMU_IDX_CYCLE_COUNTER, used_mask))
  264. return -EAGAIN;
  265. return DSU_PMU_IDX_CYCLE_COUNTER;
  266. }
  267. idx = find_first_zero_bit(used_mask, dsu_pmu->num_counters);
  268. if (idx >= dsu_pmu->num_counters)
  269. return -EAGAIN;
  270. set_bit(idx, hw_events->used_mask);
  271. return idx;
  272. }
  273. static void dsu_pmu_enable_counter(struct dsu_pmu *dsu_pmu, int idx)
  274. {
  275. __dsu_pmu_counter_interrupt_enable(idx);
  276. __dsu_pmu_enable_counter(idx);
  277. }
  278. static void dsu_pmu_disable_counter(struct dsu_pmu *dsu_pmu, int idx)
  279. {
  280. __dsu_pmu_disable_counter(idx);
  281. __dsu_pmu_counter_interrupt_disable(idx);
  282. }
  283. static inline void dsu_pmu_set_event(struct dsu_pmu *dsu_pmu,
  284. struct perf_event *event)
  285. {
  286. int idx = event->hw.idx;
  287. unsigned long flags;
  288. if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
  289. dev_err(event->pmu->dev,
  290. "Trying to set invalid counter %d\n", idx);
  291. return;
  292. }
  293. raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
  294. __dsu_pmu_set_event(idx, event->hw.config_base);
  295. raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
  296. }
  297. static void dsu_pmu_event_update(struct perf_event *event)
  298. {
  299. struct hw_perf_event *hwc = &event->hw;
  300. u64 delta, prev_count, new_count;
  301. do {
  302. /* We may also be called from the irq handler */
  303. prev_count = local64_read(&hwc->prev_count);
  304. new_count = dsu_pmu_read_counter(event);
  305. } while (local64_cmpxchg(&hwc->prev_count, prev_count, new_count) !=
  306. prev_count);
  307. delta = (new_count - prev_count) & DSU_PMU_COUNTER_MASK(hwc->idx);
  308. local64_add(delta, &event->count);
  309. }
  310. static void dsu_pmu_read(struct perf_event *event)
  311. {
  312. dsu_pmu_event_update(event);
  313. }
  314. static inline u32 dsu_pmu_get_reset_overflow(void)
  315. {
  316. return __dsu_pmu_get_reset_overflow();
  317. }
  318. /**
  319. * dsu_pmu_set_event_period: Set the period for the counter.
  320. *
  321. * All DSU PMU event counters, except the cycle counter are 32bit
  322. * counters. To handle cases of extreme interrupt latency, we program
  323. * the counter with half of the max count for the counters.
  324. */
  325. static void dsu_pmu_set_event_period(struct perf_event *event)
  326. {
  327. int idx = event->hw.idx;
  328. u64 val = DSU_PMU_COUNTER_MASK(idx) >> 1;
  329. local64_set(&event->hw.prev_count, val);
  330. dsu_pmu_write_counter(event, val);
  331. }
  332. static irqreturn_t dsu_pmu_handle_irq(int irq_num, void *dev)
  333. {
  334. int i;
  335. bool handled = false;
  336. struct dsu_pmu *dsu_pmu = dev;
  337. struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
  338. unsigned long overflow;
  339. overflow = dsu_pmu_get_reset_overflow();
  340. if (!overflow)
  341. return IRQ_NONE;
  342. for_each_set_bit(i, &overflow, DSU_PMU_MAX_HW_CNTRS) {
  343. struct perf_event *event = hw_events->events[i];
  344. if (!event)
  345. continue;
  346. dsu_pmu_event_update(event);
  347. dsu_pmu_set_event_period(event);
  348. handled = true;
  349. }
  350. return IRQ_RETVAL(handled);
  351. }
  352. static void dsu_pmu_start(struct perf_event *event, int pmu_flags)
  353. {
  354. struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
  355. /* We always reprogram the counter */
  356. if (pmu_flags & PERF_EF_RELOAD)
  357. WARN_ON(!(event->hw.state & PERF_HES_UPTODATE));
  358. dsu_pmu_set_event_period(event);
  359. if (event->hw.idx != DSU_PMU_IDX_CYCLE_COUNTER)
  360. dsu_pmu_set_event(dsu_pmu, event);
  361. event->hw.state = 0;
  362. dsu_pmu_enable_counter(dsu_pmu, event->hw.idx);
  363. }
  364. static void dsu_pmu_stop(struct perf_event *event, int pmu_flags)
  365. {
  366. struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
  367. if (event->hw.state & PERF_HES_STOPPED)
  368. return;
  369. dsu_pmu_disable_counter(dsu_pmu, event->hw.idx);
  370. dsu_pmu_event_update(event);
  371. event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
  372. }
  373. static int dsu_pmu_add(struct perf_event *event, int flags)
  374. {
  375. struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
  376. struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
  377. struct hw_perf_event *hwc = &event->hw;
  378. int idx;
  379. if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
  380. &dsu_pmu->associated_cpus)))
  381. return -ENOENT;
  382. idx = dsu_pmu_get_event_idx(hw_events, event);
  383. if (idx < 0)
  384. return idx;
  385. hwc->idx = idx;
  386. hw_events->events[idx] = event;
  387. hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
  388. if (flags & PERF_EF_START)
  389. dsu_pmu_start(event, PERF_EF_RELOAD);
  390. perf_event_update_userpage(event);
  391. return 0;
  392. }
  393. static void dsu_pmu_del(struct perf_event *event, int flags)
  394. {
  395. struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
  396. struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
  397. struct hw_perf_event *hwc = &event->hw;
  398. int idx = hwc->idx;
  399. dsu_pmu_stop(event, PERF_EF_UPDATE);
  400. hw_events->events[idx] = NULL;
  401. clear_bit(idx, hw_events->used_mask);
  402. perf_event_update_userpage(event);
  403. }
  404. static void dsu_pmu_enable(struct pmu *pmu)
  405. {
  406. u32 pmcr;
  407. unsigned long flags;
  408. struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
  409. /* If no counters are added, skip enabling the PMU */
  410. if (bitmap_empty(dsu_pmu->hw_events.used_mask, DSU_PMU_MAX_HW_CNTRS))
  411. return;
  412. raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
  413. pmcr = __dsu_pmu_read_pmcr();
  414. pmcr |= CLUSTERPMCR_E;
  415. __dsu_pmu_write_pmcr(pmcr);
  416. raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
  417. }
  418. static void dsu_pmu_disable(struct pmu *pmu)
  419. {
  420. u32 pmcr;
  421. unsigned long flags;
  422. struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
  423. raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
  424. pmcr = __dsu_pmu_read_pmcr();
  425. pmcr &= ~CLUSTERPMCR_E;
  426. __dsu_pmu_write_pmcr(pmcr);
  427. raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
  428. }
  429. static bool dsu_pmu_validate_event(struct pmu *pmu,
  430. struct dsu_hw_events *hw_events,
  431. struct perf_event *event)
  432. {
  433. if (is_software_event(event))
  434. return true;
  435. /* Reject groups spanning multiple HW PMUs. */
  436. if (event->pmu != pmu)
  437. return false;
  438. return dsu_pmu_get_event_idx(hw_events, event) >= 0;
  439. }
  440. /*
  441. * Make sure the group of events can be scheduled at once
  442. * on the PMU.
  443. */
  444. static bool dsu_pmu_validate_group(struct perf_event *event)
  445. {
  446. struct perf_event *sibling, *leader = event->group_leader;
  447. struct dsu_hw_events fake_hw;
  448. if (event->group_leader == event)
  449. return true;
  450. memset(fake_hw.used_mask, 0, sizeof(fake_hw.used_mask));
  451. if (!dsu_pmu_validate_event(event->pmu, &fake_hw, leader))
  452. return false;
  453. for_each_sibling_event(sibling, leader) {
  454. if (!dsu_pmu_validate_event(event->pmu, &fake_hw, sibling))
  455. return false;
  456. }
  457. return dsu_pmu_validate_event(event->pmu, &fake_hw, event);
  458. }
  459. static int dsu_pmu_event_init(struct perf_event *event)
  460. {
  461. struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
  462. if (event->attr.type != event->pmu->type)
  463. return -ENOENT;
  464. /* We don't support sampling */
  465. if (is_sampling_event(event)) {
  466. dev_dbg(dsu_pmu->pmu.dev, "Can't support sampling events\n");
  467. return -EOPNOTSUPP;
  468. }
  469. /* We cannot support task bound events */
  470. if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) {
  471. dev_dbg(dsu_pmu->pmu.dev, "Can't support per-task counters\n");
  472. return -EINVAL;
  473. }
  474. if (has_branch_stack(event)) {
  475. dev_dbg(dsu_pmu->pmu.dev, "Can't support filtering\n");
  476. return -EINVAL;
  477. }
  478. if (!cpumask_test_cpu(event->cpu, &dsu_pmu->associated_cpus)) {
  479. dev_dbg(dsu_pmu->pmu.dev,
  480. "Requested cpu is not associated with the DSU\n");
  481. return -EINVAL;
  482. }
  483. /*
  484. * Choose the current active CPU to read the events. We don't want
  485. * to migrate the event contexts, irq handling etc to the requested
  486. * CPU. As long as the requested CPU is within the same DSU, we
  487. * are fine.
  488. */
  489. event->cpu = cpumask_first(&dsu_pmu->active_cpu);
  490. if (event->cpu >= nr_cpu_ids)
  491. return -EINVAL;
  492. if (!dsu_pmu_validate_group(event))
  493. return -EINVAL;
  494. event->hw.config_base = event->attr.config;
  495. return 0;
  496. }
  497. static struct dsu_pmu *dsu_pmu_alloc(struct platform_device *pdev)
  498. {
  499. struct dsu_pmu *dsu_pmu;
  500. dsu_pmu = devm_kzalloc(&pdev->dev, sizeof(*dsu_pmu), GFP_KERNEL);
  501. if (!dsu_pmu)
  502. return ERR_PTR(-ENOMEM);
  503. raw_spin_lock_init(&dsu_pmu->pmu_lock);
  504. /*
  505. * Initialise the number of counters to -1, until we probe
  506. * the real number on a connected CPU.
  507. */
  508. dsu_pmu->num_counters = -1;
  509. return dsu_pmu;
  510. }
  511. /**
  512. * dsu_pmu_dt_get_cpus: Get the list of CPUs in the cluster
  513. * from device tree.
  514. */
  515. static int dsu_pmu_dt_get_cpus(struct device *dev, cpumask_t *mask)
  516. {
  517. int i = 0, n, cpu;
  518. struct device_node *cpu_node;
  519. n = of_count_phandle_with_args(dev->of_node, "cpus", NULL);
  520. if (n <= 0)
  521. return -ENODEV;
  522. for (; i < n; i++) {
  523. cpu_node = of_parse_phandle(dev->of_node, "cpus", i);
  524. if (!cpu_node)
  525. break;
  526. cpu = of_cpu_node_to_id(cpu_node);
  527. of_node_put(cpu_node);
  528. /*
  529. * We have to ignore the failures here and continue scanning
  530. * the list to handle cases where the nr_cpus could be capped
  531. * in the running kernel.
  532. */
  533. if (cpu < 0)
  534. continue;
  535. cpumask_set_cpu(cpu, mask);
  536. }
  537. return 0;
  538. }
  539. /**
  540. * dsu_pmu_acpi_get_cpus: Get the list of CPUs in the cluster
  541. * from ACPI.
  542. */
  543. static int dsu_pmu_acpi_get_cpus(struct device *dev, cpumask_t *mask)
  544. {
  545. #ifdef CONFIG_ACPI
  546. int cpu;
  547. /*
  548. * A dsu pmu node is inside a cluster parent node along with cpu nodes.
  549. * We need to find out all cpus that have the same parent with this pmu.
  550. */
  551. for_each_possible_cpu(cpu) {
  552. struct acpi_device *acpi_dev;
  553. struct device *cpu_dev = get_cpu_device(cpu);
  554. if (!cpu_dev)
  555. continue;
  556. acpi_dev = ACPI_COMPANION(cpu_dev);
  557. if (acpi_dev &&
  558. acpi_dev->parent == ACPI_COMPANION(dev)->parent)
  559. cpumask_set_cpu(cpu, mask);
  560. }
  561. #endif
  562. return 0;
  563. }
  564. /*
  565. * dsu_pmu_probe_pmu: Probe the PMU details on a CPU in the cluster.
  566. */
  567. static void dsu_pmu_probe_pmu(struct dsu_pmu *dsu_pmu)
  568. {
  569. u64 num_counters;
  570. u32 cpmceid[2];
  571. num_counters = (__dsu_pmu_read_pmcr() >> CLUSTERPMCR_N_SHIFT) &
  572. CLUSTERPMCR_N_MASK;
  573. /* We can only support up to 31 independent counters */
  574. if (WARN_ON(num_counters > 31))
  575. num_counters = 31;
  576. dsu_pmu->num_counters = num_counters;
  577. if (!dsu_pmu->num_counters)
  578. return;
  579. cpmceid[0] = __dsu_pmu_read_pmceid(0);
  580. cpmceid[1] = __dsu_pmu_read_pmceid(1);
  581. bitmap_from_arr32(dsu_pmu->cpmceid_bitmap, cpmceid,
  582. DSU_PMU_MAX_COMMON_EVENTS);
  583. }
  584. static void dsu_pmu_set_active_cpu(int cpu, struct dsu_pmu *dsu_pmu)
  585. {
  586. cpumask_set_cpu(cpu, &dsu_pmu->active_cpu);
  587. if (irq_set_affinity_hint(dsu_pmu->irq, &dsu_pmu->active_cpu))
  588. pr_warn("Failed to set irq affinity to %d\n", cpu);
  589. }
  590. /*
  591. * dsu_pmu_init_pmu: Initialise the DSU PMU configurations if
  592. * we haven't done it already.
  593. */
  594. static void dsu_pmu_init_pmu(struct dsu_pmu *dsu_pmu)
  595. {
  596. if (dsu_pmu->num_counters == -1)
  597. dsu_pmu_probe_pmu(dsu_pmu);
  598. /* Reset the interrupt overflow mask */
  599. dsu_pmu_get_reset_overflow();
  600. }
  601. static int dsu_pmu_device_probe(struct platform_device *pdev)
  602. {
  603. int irq, rc;
  604. struct dsu_pmu *dsu_pmu;
  605. struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev);
  606. char *name;
  607. static atomic_t pmu_idx = ATOMIC_INIT(-1);
  608. dsu_pmu = dsu_pmu_alloc(pdev);
  609. if (IS_ERR(dsu_pmu))
  610. return PTR_ERR(dsu_pmu);
  611. if (IS_ERR_OR_NULL(fwnode))
  612. return -ENOENT;
  613. if (is_of_node(fwnode))
  614. rc = dsu_pmu_dt_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus);
  615. else if (is_acpi_device_node(fwnode))
  616. rc = dsu_pmu_acpi_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus);
  617. else
  618. return -ENOENT;
  619. if (rc) {
  620. dev_warn(&pdev->dev, "Failed to parse the CPUs\n");
  621. return rc;
  622. }
  623. irq = platform_get_irq(pdev, 0);
  624. if (irq < 0)
  625. return -EINVAL;
  626. name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d",
  627. PMUNAME, atomic_inc_return(&pmu_idx));
  628. if (!name)
  629. return -ENOMEM;
  630. rc = devm_request_irq(&pdev->dev, irq, dsu_pmu_handle_irq,
  631. IRQF_NOBALANCING, name, dsu_pmu);
  632. if (rc) {
  633. dev_warn(&pdev->dev, "Failed to request IRQ %d\n", irq);
  634. return rc;
  635. }
  636. dsu_pmu->irq = irq;
  637. platform_set_drvdata(pdev, dsu_pmu);
  638. rc = cpuhp_state_add_instance(dsu_pmu_cpuhp_state,
  639. &dsu_pmu->cpuhp_node);
  640. if (rc)
  641. return rc;
  642. dsu_pmu->pmu = (struct pmu) {
  643. .task_ctx_nr = perf_invalid_context,
  644. .module = THIS_MODULE,
  645. .pmu_enable = dsu_pmu_enable,
  646. .pmu_disable = dsu_pmu_disable,
  647. .event_init = dsu_pmu_event_init,
  648. .add = dsu_pmu_add,
  649. .del = dsu_pmu_del,
  650. .start = dsu_pmu_start,
  651. .stop = dsu_pmu_stop,
  652. .read = dsu_pmu_read,
  653. .attr_groups = dsu_pmu_attr_groups,
  654. .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
  655. };
  656. rc = perf_pmu_register(&dsu_pmu->pmu, name, -1);
  657. if (rc) {
  658. cpuhp_state_remove_instance(dsu_pmu_cpuhp_state,
  659. &dsu_pmu->cpuhp_node);
  660. irq_set_affinity_hint(dsu_pmu->irq, NULL);
  661. }
  662. return rc;
  663. }
  664. static int dsu_pmu_device_remove(struct platform_device *pdev)
  665. {
  666. struct dsu_pmu *dsu_pmu = platform_get_drvdata(pdev);
  667. perf_pmu_unregister(&dsu_pmu->pmu);
  668. cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node);
  669. irq_set_affinity_hint(dsu_pmu->irq, NULL);
  670. return 0;
  671. }
  672. static const struct of_device_id dsu_pmu_of_match[] = {
  673. { .compatible = "arm,dsu-pmu", },
  674. {},
  675. };
  676. MODULE_DEVICE_TABLE(of, dsu_pmu_of_match);
  677. #ifdef CONFIG_ACPI
  678. static const struct acpi_device_id dsu_pmu_acpi_match[] = {
  679. { "ARMHD500", 0},
  680. {},
  681. };
  682. MODULE_DEVICE_TABLE(acpi, dsu_pmu_acpi_match);
  683. #endif
  684. static struct platform_driver dsu_pmu_driver = {
  685. .driver = {
  686. .name = DRVNAME,
  687. .of_match_table = of_match_ptr(dsu_pmu_of_match),
  688. .acpi_match_table = ACPI_PTR(dsu_pmu_acpi_match),
  689. .suppress_bind_attrs = true,
  690. },
  691. .probe = dsu_pmu_device_probe,
  692. .remove = dsu_pmu_device_remove,
  693. };
  694. static int dsu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
  695. {
  696. struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu,
  697. cpuhp_node);
  698. if (!cpumask_test_cpu(cpu, &dsu_pmu->associated_cpus))
  699. return 0;
  700. /* If the PMU is already managed, there is nothing to do */
  701. if (!cpumask_empty(&dsu_pmu->active_cpu))
  702. return 0;
  703. dsu_pmu_init_pmu(dsu_pmu);
  704. dsu_pmu_set_active_cpu(cpu, dsu_pmu);
  705. return 0;
  706. }
  707. static int dsu_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
  708. {
  709. int dst;
  710. struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu,
  711. cpuhp_node);
  712. if (!cpumask_test_and_clear_cpu(cpu, &dsu_pmu->active_cpu))
  713. return 0;
  714. dst = dsu_pmu_get_online_cpu_any_but(dsu_pmu, cpu);
  715. /* If there are no active CPUs in the DSU, leave IRQ disabled */
  716. if (dst >= nr_cpu_ids) {
  717. irq_set_affinity_hint(dsu_pmu->irq, NULL);
  718. return 0;
  719. }
  720. perf_pmu_migrate_context(&dsu_pmu->pmu, cpu, dst);
  721. dsu_pmu_set_active_cpu(dst, dsu_pmu);
  722. return 0;
  723. }
  724. static int __init dsu_pmu_init(void)
  725. {
  726. int ret;
  727. ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
  728. DRVNAME,
  729. dsu_pmu_cpu_online,
  730. dsu_pmu_cpu_teardown);
  731. if (ret < 0)
  732. return ret;
  733. dsu_pmu_cpuhp_state = ret;
  734. return platform_driver_register(&dsu_pmu_driver);
  735. }
  736. static void __exit dsu_pmu_exit(void)
  737. {
  738. platform_driver_unregister(&dsu_pmu_driver);
  739. cpuhp_remove_multi_state(dsu_pmu_cpuhp_state);
  740. }
  741. module_init(dsu_pmu_init);
  742. module_exit(dsu_pmu_exit);
  743. MODULE_DESCRIPTION("Perf driver for ARM DynamIQ Shared Unit");
  744. MODULE_AUTHOR("Suzuki K Poulose <suzuki.poulose@arm.com>");
  745. MODULE_LICENSE("GPL v2");