qcom_l3_pmu.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Driver for the L3 cache PMUs in Qualcomm Technologies chips.
  4. *
  5. * The driver supports a distributed cache architecture where the overall
  6. * cache for a socket is comprised of multiple slices each with its own PMU.
  7. * Access to each individual PMU is provided even though all CPUs share all
  8. * the slices. User space needs to aggregate to individual counts to provide
  9. * a global picture.
  10. *
  11. * See Documentation/admin-guide/perf/qcom_l3_pmu.rst for more details.
  12. *
  13. * Copyright (c) 2015-2017, The Linux Foundation. All rights reserved.
  14. */
  15. #include <linux/acpi.h>
  16. #include <linux/bitops.h>
  17. #include <linux/interrupt.h>
  18. #include <linux/io.h>
  19. #include <linux/list.h>
  20. #include <linux/module.h>
  21. #include <linux/perf_event.h>
  22. #include <linux/platform_device.h>
  23. /*
  24. * General constants
  25. */
  26. /* Number of counters on each PMU */
  27. #define L3_NUM_COUNTERS 8
  28. /* Mask for the event type field within perf_event_attr.config and EVTYPE reg */
  29. #define L3_EVTYPE_MASK 0xFF
  30. /*
  31. * Bit position of the 'long counter' flag within perf_event_attr.config.
  32. * Reserve some space between the event type and this flag to allow expansion
  33. * in the event type field.
  34. */
  35. #define L3_EVENT_LC_BIT 32
  36. /*
  37. * Register offsets
  38. */
  39. /* Perfmon registers */
  40. #define L3_HML3_PM_CR 0x000
  41. #define L3_HML3_PM_EVCNTR(__cntr) (0x420 + ((__cntr) & 0x7) * 8)
  42. #define L3_HML3_PM_CNTCTL(__cntr) (0x120 + ((__cntr) & 0x7) * 8)
  43. #define L3_HML3_PM_EVTYPE(__cntr) (0x220 + ((__cntr) & 0x7) * 8)
  44. #define L3_HML3_PM_FILTRA 0x300
  45. #define L3_HML3_PM_FILTRB 0x308
  46. #define L3_HML3_PM_FILTRC 0x310
  47. #define L3_HML3_PM_FILTRAM 0x304
  48. #define L3_HML3_PM_FILTRBM 0x30C
  49. #define L3_HML3_PM_FILTRCM 0x314
  50. /* Basic counter registers */
  51. #define L3_M_BC_CR 0x500
  52. #define L3_M_BC_SATROLL_CR 0x504
  53. #define L3_M_BC_CNTENSET 0x508
  54. #define L3_M_BC_CNTENCLR 0x50C
  55. #define L3_M_BC_INTENSET 0x510
  56. #define L3_M_BC_INTENCLR 0x514
  57. #define L3_M_BC_GANG 0x718
  58. #define L3_M_BC_OVSR 0x740
  59. #define L3_M_BC_IRQCTL 0x96C
  60. /*
  61. * Bit field definitions
  62. */
  63. /* L3_HML3_PM_CR */
  64. #define PM_CR_RESET (0)
  65. /* L3_HML3_PM_XCNTCTL/L3_HML3_PM_CNTCTLx */
  66. #define PMCNT_RESET (0)
  67. /* L3_HML3_PM_EVTYPEx */
  68. #define EVSEL(__val) ((__val) & L3_EVTYPE_MASK)
  69. /* Reset value for all the filter registers */
  70. #define PM_FLTR_RESET (0)
  71. /* L3_M_BC_CR */
  72. #define BC_RESET (1UL << 1)
  73. #define BC_ENABLE (1UL << 0)
  74. /* L3_M_BC_SATROLL_CR */
  75. #define BC_SATROLL_CR_RESET (0)
  76. /* L3_M_BC_CNTENSET */
  77. #define PMCNTENSET(__cntr) (1UL << ((__cntr) & 0x7))
  78. /* L3_M_BC_CNTENCLR */
  79. #define PMCNTENCLR(__cntr) (1UL << ((__cntr) & 0x7))
  80. #define BC_CNTENCLR_RESET (0xFF)
  81. /* L3_M_BC_INTENSET */
  82. #define PMINTENSET(__cntr) (1UL << ((__cntr) & 0x7))
  83. /* L3_M_BC_INTENCLR */
  84. #define PMINTENCLR(__cntr) (1UL << ((__cntr) & 0x7))
  85. #define BC_INTENCLR_RESET (0xFF)
  86. /* L3_M_BC_GANG */
  87. #define GANG_EN(__cntr) (1UL << ((__cntr) & 0x7))
  88. #define BC_GANG_RESET (0)
  89. /* L3_M_BC_OVSR */
  90. #define PMOVSRCLR(__cntr) (1UL << ((__cntr) & 0x7))
  91. #define PMOVSRCLR_RESET (0xFF)
  92. /* L3_M_BC_IRQCTL */
  93. #define PMIRQONMSBEN(__cntr) (1UL << ((__cntr) & 0x7))
  94. #define BC_IRQCTL_RESET (0x0)
  95. /*
  96. * Events
  97. */
  98. #define L3_EVENT_CYCLES 0x01
  99. #define L3_EVENT_READ_HIT 0x20
  100. #define L3_EVENT_READ_MISS 0x21
  101. #define L3_EVENT_READ_HIT_D 0x22
  102. #define L3_EVENT_READ_MISS_D 0x23
  103. #define L3_EVENT_WRITE_HIT 0x24
  104. #define L3_EVENT_WRITE_MISS 0x25
  105. /*
  106. * Decoding of settings from perf_event_attr
  107. *
  108. * The config format for perf events is:
  109. * - config: bits 0-7: event type
  110. * bit 32: HW counter size requested, 0: 32 bits, 1: 64 bits
  111. */
  112. static inline u32 get_event_type(struct perf_event *event)
  113. {
  114. return (event->attr.config) & L3_EVTYPE_MASK;
  115. }
  116. static inline bool event_uses_long_counter(struct perf_event *event)
  117. {
  118. return !!(event->attr.config & BIT_ULL(L3_EVENT_LC_BIT));
  119. }
  120. static inline int event_num_counters(struct perf_event *event)
  121. {
  122. return event_uses_long_counter(event) ? 2 : 1;
  123. }
  124. /*
  125. * Main PMU, inherits from the core perf PMU type
  126. */
  127. struct l3cache_pmu {
  128. struct pmu pmu;
  129. struct hlist_node node;
  130. void __iomem *regs;
  131. struct perf_event *events[L3_NUM_COUNTERS];
  132. unsigned long used_mask[BITS_TO_LONGS(L3_NUM_COUNTERS)];
  133. cpumask_t cpumask;
  134. };
  135. #define to_l3cache_pmu(p) (container_of(p, struct l3cache_pmu, pmu))
  136. /*
  137. * Type used to group hardware counter operations
  138. *
  139. * Used to implement two types of hardware counters, standard (32bits) and
  140. * long (64bits). The hardware supports counter chaining which we use to
  141. * implement long counters. This support is exposed via the 'lc' flag field
  142. * in perf_event_attr.config.
  143. */
  144. struct l3cache_event_ops {
  145. /* Called to start event monitoring */
  146. void (*start)(struct perf_event *event);
  147. /* Called to stop event monitoring */
  148. void (*stop)(struct perf_event *event, int flags);
  149. /* Called to update the perf_event */
  150. void (*update)(struct perf_event *event);
  151. };
  152. /*
  153. * Implementation of long counter operations
  154. *
  155. * 64bit counters are implemented by chaining two of the 32bit physical
  156. * counters. The PMU only supports chaining of adjacent even/odd pairs
  157. * and for simplicity the driver always configures the odd counter to
  158. * count the overflows of the lower-numbered even counter. Note that since
  159. * the resulting hardware counter is 64bits no IRQs are required to maintain
  160. * the software counter which is also 64bits.
  161. */
  162. static void qcom_l3_cache__64bit_counter_start(struct perf_event *event)
  163. {
  164. struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
  165. int idx = event->hw.idx;
  166. u32 evsel = get_event_type(event);
  167. u32 gang;
  168. /* Set the odd counter to count the overflows of the even counter */
  169. gang = readl_relaxed(l3pmu->regs + L3_M_BC_GANG);
  170. gang |= GANG_EN(idx + 1);
  171. writel_relaxed(gang, l3pmu->regs + L3_M_BC_GANG);
  172. /* Initialize the hardware counters and reset prev_count*/
  173. local64_set(&event->hw.prev_count, 0);
  174. writel_relaxed(0, l3pmu->regs + L3_HML3_PM_EVCNTR(idx + 1));
  175. writel_relaxed(0, l3pmu->regs + L3_HML3_PM_EVCNTR(idx));
  176. /*
  177. * Set the event types, the upper half must use zero and the lower
  178. * half the actual event type
  179. */
  180. writel_relaxed(EVSEL(0), l3pmu->regs + L3_HML3_PM_EVTYPE(idx + 1));
  181. writel_relaxed(EVSEL(evsel), l3pmu->regs + L3_HML3_PM_EVTYPE(idx));
  182. /* Finally, enable the counters */
  183. writel_relaxed(PMCNT_RESET, l3pmu->regs + L3_HML3_PM_CNTCTL(idx + 1));
  184. writel_relaxed(PMCNTENSET(idx + 1), l3pmu->regs + L3_M_BC_CNTENSET);
  185. writel_relaxed(PMCNT_RESET, l3pmu->regs + L3_HML3_PM_CNTCTL(idx));
  186. writel_relaxed(PMCNTENSET(idx), l3pmu->regs + L3_M_BC_CNTENSET);
  187. }
  188. static void qcom_l3_cache__64bit_counter_stop(struct perf_event *event,
  189. int flags)
  190. {
  191. struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
  192. int idx = event->hw.idx;
  193. u32 gang = readl_relaxed(l3pmu->regs + L3_M_BC_GANG);
  194. /* Disable the counters */
  195. writel_relaxed(PMCNTENCLR(idx), l3pmu->regs + L3_M_BC_CNTENCLR);
  196. writel_relaxed(PMCNTENCLR(idx + 1), l3pmu->regs + L3_M_BC_CNTENCLR);
  197. /* Disable chaining */
  198. writel_relaxed(gang & ~GANG_EN(idx + 1), l3pmu->regs + L3_M_BC_GANG);
  199. }
  200. static void qcom_l3_cache__64bit_counter_update(struct perf_event *event)
  201. {
  202. struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
  203. int idx = event->hw.idx;
  204. u32 hi, lo;
  205. u64 prev, new;
  206. do {
  207. prev = local64_read(&event->hw.prev_count);
  208. do {
  209. hi = readl_relaxed(l3pmu->regs + L3_HML3_PM_EVCNTR(idx + 1));
  210. lo = readl_relaxed(l3pmu->regs + L3_HML3_PM_EVCNTR(idx));
  211. } while (hi != readl_relaxed(l3pmu->regs + L3_HML3_PM_EVCNTR(idx + 1)));
  212. new = ((u64)hi << 32) | lo;
  213. } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev);
  214. local64_add(new - prev, &event->count);
  215. }
  216. static const struct l3cache_event_ops event_ops_long = {
  217. .start = qcom_l3_cache__64bit_counter_start,
  218. .stop = qcom_l3_cache__64bit_counter_stop,
  219. .update = qcom_l3_cache__64bit_counter_update,
  220. };
  221. /*
  222. * Implementation of standard counter operations
  223. *
  224. * 32bit counters use a single physical counter and a hardware feature that
  225. * asserts the overflow IRQ on the toggling of the most significant bit in
  226. * the counter. This feature allows the counters to be left free-running
  227. * without needing the usual reprogramming required to properly handle races
  228. * during concurrent calls to update.
  229. */
  230. static void qcom_l3_cache__32bit_counter_start(struct perf_event *event)
  231. {
  232. struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
  233. int idx = event->hw.idx;
  234. u32 evsel = get_event_type(event);
  235. u32 irqctl = readl_relaxed(l3pmu->regs + L3_M_BC_IRQCTL);
  236. /* Set the counter to assert the overflow IRQ on MSB toggling */
  237. writel_relaxed(irqctl | PMIRQONMSBEN(idx), l3pmu->regs + L3_M_BC_IRQCTL);
  238. /* Initialize the hardware counter and reset prev_count*/
  239. local64_set(&event->hw.prev_count, 0);
  240. writel_relaxed(0, l3pmu->regs + L3_HML3_PM_EVCNTR(idx));
  241. /* Set the event type */
  242. writel_relaxed(EVSEL(evsel), l3pmu->regs + L3_HML3_PM_EVTYPE(idx));
  243. /* Enable interrupt generation by this counter */
  244. writel_relaxed(PMINTENSET(idx), l3pmu->regs + L3_M_BC_INTENSET);
  245. /* Finally, enable the counter */
  246. writel_relaxed(PMCNT_RESET, l3pmu->regs + L3_HML3_PM_CNTCTL(idx));
  247. writel_relaxed(PMCNTENSET(idx), l3pmu->regs + L3_M_BC_CNTENSET);
  248. }
  249. static void qcom_l3_cache__32bit_counter_stop(struct perf_event *event,
  250. int flags)
  251. {
  252. struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
  253. int idx = event->hw.idx;
  254. u32 irqctl = readl_relaxed(l3pmu->regs + L3_M_BC_IRQCTL);
  255. /* Disable the counter */
  256. writel_relaxed(PMCNTENCLR(idx), l3pmu->regs + L3_M_BC_CNTENCLR);
  257. /* Disable interrupt generation by this counter */
  258. writel_relaxed(PMINTENCLR(idx), l3pmu->regs + L3_M_BC_INTENCLR);
  259. /* Set the counter to not assert the overflow IRQ on MSB toggling */
  260. writel_relaxed(irqctl & ~PMIRQONMSBEN(idx), l3pmu->regs + L3_M_BC_IRQCTL);
  261. }
  262. static void qcom_l3_cache__32bit_counter_update(struct perf_event *event)
  263. {
  264. struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
  265. int idx = event->hw.idx;
  266. u32 prev, new;
  267. do {
  268. prev = local64_read(&event->hw.prev_count);
  269. new = readl_relaxed(l3pmu->regs + L3_HML3_PM_EVCNTR(idx));
  270. } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev);
  271. local64_add(new - prev, &event->count);
  272. }
  273. static const struct l3cache_event_ops event_ops_std = {
  274. .start = qcom_l3_cache__32bit_counter_start,
  275. .stop = qcom_l3_cache__32bit_counter_stop,
  276. .update = qcom_l3_cache__32bit_counter_update,
  277. };
  278. /* Retrieve the appropriate operations for the given event */
  279. static
  280. const struct l3cache_event_ops *l3cache_event_get_ops(struct perf_event *event)
  281. {
  282. if (event_uses_long_counter(event))
  283. return &event_ops_long;
  284. else
  285. return &event_ops_std;
  286. }
  287. /*
  288. * Top level PMU functions.
  289. */
  290. static inline void qcom_l3_cache__init(struct l3cache_pmu *l3pmu)
  291. {
  292. int i;
  293. writel_relaxed(BC_RESET, l3pmu->regs + L3_M_BC_CR);
  294. /*
  295. * Use writel for the first programming command to ensure the basic
  296. * counter unit is stopped before proceeding
  297. */
  298. writel(BC_SATROLL_CR_RESET, l3pmu->regs + L3_M_BC_SATROLL_CR);
  299. writel_relaxed(BC_CNTENCLR_RESET, l3pmu->regs + L3_M_BC_CNTENCLR);
  300. writel_relaxed(BC_INTENCLR_RESET, l3pmu->regs + L3_M_BC_INTENCLR);
  301. writel_relaxed(PMOVSRCLR_RESET, l3pmu->regs + L3_M_BC_OVSR);
  302. writel_relaxed(BC_GANG_RESET, l3pmu->regs + L3_M_BC_GANG);
  303. writel_relaxed(BC_IRQCTL_RESET, l3pmu->regs + L3_M_BC_IRQCTL);
  304. writel_relaxed(PM_CR_RESET, l3pmu->regs + L3_HML3_PM_CR);
  305. for (i = 0; i < L3_NUM_COUNTERS; ++i) {
  306. writel_relaxed(PMCNT_RESET, l3pmu->regs + L3_HML3_PM_CNTCTL(i));
  307. writel_relaxed(EVSEL(0), l3pmu->regs + L3_HML3_PM_EVTYPE(i));
  308. }
  309. writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRA);
  310. writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRAM);
  311. writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRB);
  312. writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRBM);
  313. writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRC);
  314. writel_relaxed(PM_FLTR_RESET, l3pmu->regs + L3_HML3_PM_FILTRCM);
  315. /*
  316. * Use writel here to ensure all programming commands are done
  317. * before proceeding
  318. */
  319. writel(BC_ENABLE, l3pmu->regs + L3_M_BC_CR);
  320. }
  321. static irqreturn_t qcom_l3_cache__handle_irq(int irq_num, void *data)
  322. {
  323. struct l3cache_pmu *l3pmu = data;
  324. /* Read the overflow status register */
  325. long status = readl_relaxed(l3pmu->regs + L3_M_BC_OVSR);
  326. int idx;
  327. if (status == 0)
  328. return IRQ_NONE;
  329. /* Clear the bits we read on the overflow status register */
  330. writel_relaxed(status, l3pmu->regs + L3_M_BC_OVSR);
  331. for_each_set_bit(idx, &status, L3_NUM_COUNTERS) {
  332. struct perf_event *event;
  333. const struct l3cache_event_ops *ops;
  334. event = l3pmu->events[idx];
  335. if (!event)
  336. continue;
  337. /*
  338. * Since the IRQ is not enabled for events using long counters
  339. * we should never see one of those here, however, be consistent
  340. * and use the ops indirections like in the other operations.
  341. */
  342. ops = l3cache_event_get_ops(event);
  343. ops->update(event);
  344. }
  345. return IRQ_HANDLED;
  346. }
  347. /*
  348. * Implementation of abstract pmu functionality required by
  349. * the core perf events code.
  350. */
  351. static void qcom_l3_cache__pmu_enable(struct pmu *pmu)
  352. {
  353. struct l3cache_pmu *l3pmu = to_l3cache_pmu(pmu);
  354. /* Ensure the other programming commands are observed before enabling */
  355. wmb();
  356. writel_relaxed(BC_ENABLE, l3pmu->regs + L3_M_BC_CR);
  357. }
  358. static void qcom_l3_cache__pmu_disable(struct pmu *pmu)
  359. {
  360. struct l3cache_pmu *l3pmu = to_l3cache_pmu(pmu);
  361. writel_relaxed(0, l3pmu->regs + L3_M_BC_CR);
  362. /* Ensure the basic counter unit is stopped before proceeding */
  363. wmb();
  364. }
  365. /*
  366. * We must NOT create groups containing events from multiple hardware PMUs,
  367. * although mixing different software and hardware PMUs is allowed.
  368. */
  369. static bool qcom_l3_cache__validate_event_group(struct perf_event *event)
  370. {
  371. struct perf_event *leader = event->group_leader;
  372. struct perf_event *sibling;
  373. int counters = 0;
  374. if (leader->pmu != event->pmu && !is_software_event(leader))
  375. return false;
  376. counters = event_num_counters(event);
  377. counters += event_num_counters(leader);
  378. for_each_sibling_event(sibling, leader) {
  379. if (is_software_event(sibling))
  380. continue;
  381. if (sibling->pmu != event->pmu)
  382. return false;
  383. counters += event_num_counters(sibling);
  384. }
  385. /*
  386. * If the group requires more counters than the HW has, it
  387. * cannot ever be scheduled.
  388. */
  389. return counters <= L3_NUM_COUNTERS;
  390. }
  391. static int qcom_l3_cache__event_init(struct perf_event *event)
  392. {
  393. struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
  394. struct hw_perf_event *hwc = &event->hw;
  395. /*
  396. * Is the event for this PMU?
  397. */
  398. if (event->attr.type != event->pmu->type)
  399. return -ENOENT;
  400. /*
  401. * Sampling not supported since these events are not core-attributable.
  402. */
  403. if (hwc->sample_period)
  404. return -EINVAL;
  405. /*
  406. * Task mode not available, we run the counters as socket counters,
  407. * not attributable to any CPU and therefore cannot attribute per-task.
  408. */
  409. if (event->cpu < 0)
  410. return -EINVAL;
  411. /* Validate the group */
  412. if (!qcom_l3_cache__validate_event_group(event))
  413. return -EINVAL;
  414. hwc->idx = -1;
  415. /*
  416. * Many perf core operations (eg. events rotation) operate on a
  417. * single CPU context. This is obvious for CPU PMUs, where one
  418. * expects the same sets of events being observed on all CPUs,
  419. * but can lead to issues for off-core PMUs, like this one, where
  420. * each event could be theoretically assigned to a different CPU.
  421. * To mitigate this, we enforce CPU assignment to one designated
  422. * processor (the one described in the "cpumask" attribute exported
  423. * by the PMU device). perf user space tools honor this and avoid
  424. * opening more than one copy of the events.
  425. */
  426. event->cpu = cpumask_first(&l3pmu->cpumask);
  427. return 0;
  428. }
  429. static void qcom_l3_cache__event_start(struct perf_event *event, int flags)
  430. {
  431. struct hw_perf_event *hwc = &event->hw;
  432. const struct l3cache_event_ops *ops = l3cache_event_get_ops(event);
  433. hwc->state = 0;
  434. ops->start(event);
  435. }
  436. static void qcom_l3_cache__event_stop(struct perf_event *event, int flags)
  437. {
  438. struct hw_perf_event *hwc = &event->hw;
  439. const struct l3cache_event_ops *ops = l3cache_event_get_ops(event);
  440. if (hwc->state & PERF_HES_STOPPED)
  441. return;
  442. ops->stop(event, flags);
  443. if (flags & PERF_EF_UPDATE)
  444. ops->update(event);
  445. hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
  446. }
  447. static int qcom_l3_cache__event_add(struct perf_event *event, int flags)
  448. {
  449. struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
  450. struct hw_perf_event *hwc = &event->hw;
  451. int order = event_uses_long_counter(event) ? 1 : 0;
  452. int idx;
  453. /*
  454. * Try to allocate a counter.
  455. */
  456. idx = bitmap_find_free_region(l3pmu->used_mask, L3_NUM_COUNTERS, order);
  457. if (idx < 0)
  458. /* The counters are all in use. */
  459. return -EAGAIN;
  460. hwc->idx = idx;
  461. hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
  462. l3pmu->events[idx] = event;
  463. if (flags & PERF_EF_START)
  464. qcom_l3_cache__event_start(event, 0);
  465. /* Propagate changes to the userspace mapping. */
  466. perf_event_update_userpage(event);
  467. return 0;
  468. }
  469. static void qcom_l3_cache__event_del(struct perf_event *event, int flags)
  470. {
  471. struct l3cache_pmu *l3pmu = to_l3cache_pmu(event->pmu);
  472. struct hw_perf_event *hwc = &event->hw;
  473. int order = event_uses_long_counter(event) ? 1 : 0;
  474. /* Stop and clean up */
  475. qcom_l3_cache__event_stop(event, flags | PERF_EF_UPDATE);
  476. l3pmu->events[hwc->idx] = NULL;
  477. bitmap_release_region(l3pmu->used_mask, hwc->idx, order);
  478. /* Propagate changes to the userspace mapping. */
  479. perf_event_update_userpage(event);
  480. }
  481. static void qcom_l3_cache__event_read(struct perf_event *event)
  482. {
  483. const struct l3cache_event_ops *ops = l3cache_event_get_ops(event);
  484. ops->update(event);
  485. }
  486. /*
  487. * Add sysfs attributes
  488. *
  489. * We export:
  490. * - formats, used by perf user space and other tools to configure events
  491. * - events, used by perf user space and other tools to create events
  492. * symbolically, e.g.:
  493. * perf stat -a -e l3cache_0_0/event=read-miss/ ls
  494. * perf stat -a -e l3cache_0_0/event=0x21/ ls
  495. * - cpumask, used by perf user space and other tools to know on which CPUs
  496. * to open the events
  497. */
  498. /* formats */
  499. static ssize_t l3cache_pmu_format_show(struct device *dev,
  500. struct device_attribute *attr, char *buf)
  501. {
  502. struct dev_ext_attribute *eattr;
  503. eattr = container_of(attr, struct dev_ext_attribute, attr);
  504. return sprintf(buf, "%s\n", (char *) eattr->var);
  505. }
  506. #define L3CACHE_PMU_FORMAT_ATTR(_name, _config) \
  507. (&((struct dev_ext_attribute[]) { \
  508. { .attr = __ATTR(_name, 0444, l3cache_pmu_format_show, NULL), \
  509. .var = (void *) _config, } \
  510. })[0].attr.attr)
  511. static struct attribute *qcom_l3_cache_pmu_formats[] = {
  512. L3CACHE_PMU_FORMAT_ATTR(event, "config:0-7"),
  513. L3CACHE_PMU_FORMAT_ATTR(lc, "config:" __stringify(L3_EVENT_LC_BIT)),
  514. NULL,
  515. };
  516. static struct attribute_group qcom_l3_cache_pmu_format_group = {
  517. .name = "format",
  518. .attrs = qcom_l3_cache_pmu_formats,
  519. };
  520. /* events */
  521. static ssize_t l3cache_pmu_event_show(struct device *dev,
  522. struct device_attribute *attr, char *page)
  523. {
  524. struct perf_pmu_events_attr *pmu_attr;
  525. pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
  526. return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
  527. }
  528. #define L3CACHE_EVENT_ATTR(_name, _id) \
  529. (&((struct perf_pmu_events_attr[]) { \
  530. { .attr = __ATTR(_name, 0444, l3cache_pmu_event_show, NULL), \
  531. .id = _id, } \
  532. })[0].attr.attr)
  533. static struct attribute *qcom_l3_cache_pmu_events[] = {
  534. L3CACHE_EVENT_ATTR(cycles, L3_EVENT_CYCLES),
  535. L3CACHE_EVENT_ATTR(read-hit, L3_EVENT_READ_HIT),
  536. L3CACHE_EVENT_ATTR(read-miss, L3_EVENT_READ_MISS),
  537. L3CACHE_EVENT_ATTR(read-hit-d-side, L3_EVENT_READ_HIT_D),
  538. L3CACHE_EVENT_ATTR(read-miss-d-side, L3_EVENT_READ_MISS_D),
  539. L3CACHE_EVENT_ATTR(write-hit, L3_EVENT_WRITE_HIT),
  540. L3CACHE_EVENT_ATTR(write-miss, L3_EVENT_WRITE_MISS),
  541. NULL
  542. };
  543. static struct attribute_group qcom_l3_cache_pmu_events_group = {
  544. .name = "events",
  545. .attrs = qcom_l3_cache_pmu_events,
  546. };
  547. /* cpumask */
  548. static ssize_t qcom_l3_cache_pmu_cpumask_show(struct device *dev,
  549. struct device_attribute *attr, char *buf)
  550. {
  551. struct l3cache_pmu *l3pmu = to_l3cache_pmu(dev_get_drvdata(dev));
  552. return cpumap_print_to_pagebuf(true, buf, &l3pmu->cpumask);
  553. }
  554. static DEVICE_ATTR(cpumask, 0444, qcom_l3_cache_pmu_cpumask_show, NULL);
  555. static struct attribute *qcom_l3_cache_pmu_cpumask_attrs[] = {
  556. &dev_attr_cpumask.attr,
  557. NULL,
  558. };
  559. static struct attribute_group qcom_l3_cache_pmu_cpumask_attr_group = {
  560. .attrs = qcom_l3_cache_pmu_cpumask_attrs,
  561. };
  562. /*
  563. * Per PMU device attribute groups
  564. */
  565. static const struct attribute_group *qcom_l3_cache_pmu_attr_grps[] = {
  566. &qcom_l3_cache_pmu_format_group,
  567. &qcom_l3_cache_pmu_events_group,
  568. &qcom_l3_cache_pmu_cpumask_attr_group,
  569. NULL,
  570. };
  571. /*
  572. * Probing functions and data.
  573. */
  574. static int qcom_l3_cache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
  575. {
  576. struct l3cache_pmu *l3pmu = hlist_entry_safe(node, struct l3cache_pmu, node);
  577. /* If there is not a CPU/PMU association pick this CPU */
  578. if (cpumask_empty(&l3pmu->cpumask))
  579. cpumask_set_cpu(cpu, &l3pmu->cpumask);
  580. return 0;
  581. }
  582. static int qcom_l3_cache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
  583. {
  584. struct l3cache_pmu *l3pmu = hlist_entry_safe(node, struct l3cache_pmu, node);
  585. unsigned int target;
  586. if (!cpumask_test_and_clear_cpu(cpu, &l3pmu->cpumask))
  587. return 0;
  588. target = cpumask_any_but(cpu_online_mask, cpu);
  589. if (target >= nr_cpu_ids)
  590. return 0;
  591. perf_pmu_migrate_context(&l3pmu->pmu, cpu, target);
  592. cpumask_set_cpu(target, &l3pmu->cpumask);
  593. return 0;
  594. }
  595. static int qcom_l3_cache_pmu_probe(struct platform_device *pdev)
  596. {
  597. struct l3cache_pmu *l3pmu;
  598. struct acpi_device *acpi_dev;
  599. struct resource *memrc;
  600. int ret;
  601. char *name;
  602. /* Initialize the PMU data structures */
  603. acpi_dev = ACPI_COMPANION(&pdev->dev);
  604. if (!acpi_dev)
  605. return -ENODEV;
  606. l3pmu = devm_kzalloc(&pdev->dev, sizeof(*l3pmu), GFP_KERNEL);
  607. name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "l3cache_%s_%s",
  608. acpi_dev->parent->pnp.unique_id, acpi_dev->pnp.unique_id);
  609. if (!l3pmu || !name)
  610. return -ENOMEM;
  611. l3pmu->pmu = (struct pmu) {
  612. .task_ctx_nr = perf_invalid_context,
  613. .pmu_enable = qcom_l3_cache__pmu_enable,
  614. .pmu_disable = qcom_l3_cache__pmu_disable,
  615. .event_init = qcom_l3_cache__event_init,
  616. .add = qcom_l3_cache__event_add,
  617. .del = qcom_l3_cache__event_del,
  618. .start = qcom_l3_cache__event_start,
  619. .stop = qcom_l3_cache__event_stop,
  620. .read = qcom_l3_cache__event_read,
  621. .attr_groups = qcom_l3_cache_pmu_attr_grps,
  622. .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
  623. };
  624. memrc = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  625. l3pmu->regs = devm_ioremap_resource(&pdev->dev, memrc);
  626. if (IS_ERR(l3pmu->regs)) {
  627. dev_err(&pdev->dev, "Can't map PMU @%pa\n", &memrc->start);
  628. return PTR_ERR(l3pmu->regs);
  629. }
  630. qcom_l3_cache__init(l3pmu);
  631. ret = platform_get_irq(pdev, 0);
  632. if (ret <= 0)
  633. return ret;
  634. ret = devm_request_irq(&pdev->dev, ret, qcom_l3_cache__handle_irq, 0,
  635. name, l3pmu);
  636. if (ret) {
  637. dev_err(&pdev->dev, "Request for IRQ failed for slice @%pa\n",
  638. &memrc->start);
  639. return ret;
  640. }
  641. /* Add this instance to the list used by the offline callback */
  642. ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE, &l3pmu->node);
  643. if (ret) {
  644. dev_err(&pdev->dev, "Error %d registering hotplug", ret);
  645. return ret;
  646. }
  647. ret = perf_pmu_register(&l3pmu->pmu, name, -1);
  648. if (ret < 0) {
  649. dev_err(&pdev->dev, "Failed to register L3 cache PMU (%d)\n", ret);
  650. return ret;
  651. }
  652. dev_info(&pdev->dev, "Registered %s, type: %d\n", name, l3pmu->pmu.type);
  653. return 0;
  654. }
  655. static const struct acpi_device_id qcom_l3_cache_pmu_acpi_match[] = {
  656. { "QCOM8081", },
  657. { }
  658. };
  659. MODULE_DEVICE_TABLE(acpi, qcom_l3_cache_pmu_acpi_match);
  660. static struct platform_driver qcom_l3_cache_pmu_driver = {
  661. .driver = {
  662. .name = "qcom-l3cache-pmu",
  663. .acpi_match_table = ACPI_PTR(qcom_l3_cache_pmu_acpi_match),
  664. .suppress_bind_attrs = true,
  665. },
  666. .probe = qcom_l3_cache_pmu_probe,
  667. };
  668. static int __init register_qcom_l3_cache_pmu_driver(void)
  669. {
  670. int ret;
  671. /* Install a hook to update the reader CPU in case it goes offline */
  672. ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE,
  673. "perf/qcom/l3cache:online",
  674. qcom_l3_cache_pmu_online_cpu,
  675. qcom_l3_cache_pmu_offline_cpu);
  676. if (ret)
  677. return ret;
  678. return platform_driver_register(&qcom_l3_cache_pmu_driver);
  679. }
  680. device_initcall(register_qcom_l3_cache_pmu_driver);