perf_event.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Xtensa Performance Monitor Module driver
  4. * See Tensilica Debug User's Guide for PMU registers documentation.
  5. *
  6. * Copyright (C) 2015 Cadence Design Systems Inc.
  7. */
  8. #include <linux/interrupt.h>
  9. #include <linux/irqdomain.h>
  10. #include <linux/module.h>
  11. #include <linux/of.h>
  12. #include <linux/perf_event.h>
  13. #include <linux/platform_device.h>
  14. #include <asm/processor.h>
  15. #include <asm/stacktrace.h>
  16. /* Global control/status for all perf counters */
  17. #define XTENSA_PMU_PMG 0x1000
  18. /* Perf counter values */
  19. #define XTENSA_PMU_PM(i) (0x1080 + (i) * 4)
  20. /* Perf counter control registers */
  21. #define XTENSA_PMU_PMCTRL(i) (0x1100 + (i) * 4)
  22. /* Perf counter status registers */
  23. #define XTENSA_PMU_PMSTAT(i) (0x1180 + (i) * 4)
  24. #define XTENSA_PMU_PMG_PMEN 0x1
  25. #define XTENSA_PMU_COUNTER_MASK 0xffffffffULL
  26. #define XTENSA_PMU_COUNTER_MAX 0x7fffffff
  27. #define XTENSA_PMU_PMCTRL_INTEN 0x00000001
  28. #define XTENSA_PMU_PMCTRL_KRNLCNT 0x00000008
  29. #define XTENSA_PMU_PMCTRL_TRACELEVEL 0x000000f0
  30. #define XTENSA_PMU_PMCTRL_SELECT_SHIFT 8
  31. #define XTENSA_PMU_PMCTRL_SELECT 0x00001f00
  32. #define XTENSA_PMU_PMCTRL_MASK_SHIFT 16
  33. #define XTENSA_PMU_PMCTRL_MASK 0xffff0000
  34. #define XTENSA_PMU_MASK(select, mask) \
  35. (((select) << XTENSA_PMU_PMCTRL_SELECT_SHIFT) | \
  36. ((mask) << XTENSA_PMU_PMCTRL_MASK_SHIFT) | \
  37. XTENSA_PMU_PMCTRL_TRACELEVEL | \
  38. XTENSA_PMU_PMCTRL_INTEN)
  39. #define XTENSA_PMU_PMSTAT_OVFL 0x00000001
  40. #define XTENSA_PMU_PMSTAT_INTASRT 0x00000010
  41. struct xtensa_pmu_events {
  42. /* Array of events currently on this core */
  43. struct perf_event *event[XCHAL_NUM_PERF_COUNTERS];
  44. /* Bitmap of used hardware counters */
  45. unsigned long used_mask[BITS_TO_LONGS(XCHAL_NUM_PERF_COUNTERS)];
  46. };
  47. static DEFINE_PER_CPU(struct xtensa_pmu_events, xtensa_pmu_events);
  48. static const u32 xtensa_hw_ctl[] = {
  49. [PERF_COUNT_HW_CPU_CYCLES] = XTENSA_PMU_MASK(0, 0x1),
  50. [PERF_COUNT_HW_INSTRUCTIONS] = XTENSA_PMU_MASK(2, 0xffff),
  51. [PERF_COUNT_HW_CACHE_REFERENCES] = XTENSA_PMU_MASK(10, 0x1),
  52. [PERF_COUNT_HW_CACHE_MISSES] = XTENSA_PMU_MASK(12, 0x1),
  53. /* Taken and non-taken branches + taken loop ends */
  54. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = XTENSA_PMU_MASK(2, 0x490),
  55. /* Instruction-related + other global stall cycles */
  56. [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = XTENSA_PMU_MASK(4, 0x1ff),
  57. /* Data-related global stall cycles */
  58. [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = XTENSA_PMU_MASK(3, 0x1ff),
  59. };
  60. #define C(_x) PERF_COUNT_HW_CACHE_##_x
  61. static const u32 xtensa_cache_ctl[][C(OP_MAX)][C(RESULT_MAX)] = {
  62. [C(L1D)] = {
  63. [C(OP_READ)] = {
  64. [C(RESULT_ACCESS)] = XTENSA_PMU_MASK(10, 0x1),
  65. [C(RESULT_MISS)] = XTENSA_PMU_MASK(10, 0x2),
  66. },
  67. [C(OP_WRITE)] = {
  68. [C(RESULT_ACCESS)] = XTENSA_PMU_MASK(11, 0x1),
  69. [C(RESULT_MISS)] = XTENSA_PMU_MASK(11, 0x2),
  70. },
  71. },
  72. [C(L1I)] = {
  73. [C(OP_READ)] = {
  74. [C(RESULT_ACCESS)] = XTENSA_PMU_MASK(8, 0x1),
  75. [C(RESULT_MISS)] = XTENSA_PMU_MASK(8, 0x2),
  76. },
  77. },
  78. [C(DTLB)] = {
  79. [C(OP_READ)] = {
  80. [C(RESULT_ACCESS)] = XTENSA_PMU_MASK(9, 0x1),
  81. [C(RESULT_MISS)] = XTENSA_PMU_MASK(9, 0x8),
  82. },
  83. },
  84. [C(ITLB)] = {
  85. [C(OP_READ)] = {
  86. [C(RESULT_ACCESS)] = XTENSA_PMU_MASK(7, 0x1),
  87. [C(RESULT_MISS)] = XTENSA_PMU_MASK(7, 0x8),
  88. },
  89. },
  90. };
  91. static int xtensa_pmu_cache_event(u64 config)
  92. {
  93. unsigned int cache_type, cache_op, cache_result;
  94. int ret;
  95. cache_type = (config >> 0) & 0xff;
  96. cache_op = (config >> 8) & 0xff;
  97. cache_result = (config >> 16) & 0xff;
  98. if (cache_type >= ARRAY_SIZE(xtensa_cache_ctl) ||
  99. cache_op >= C(OP_MAX) ||
  100. cache_result >= C(RESULT_MAX))
  101. return -EINVAL;
  102. ret = xtensa_cache_ctl[cache_type][cache_op][cache_result];
  103. if (ret == 0)
  104. return -EINVAL;
  105. return ret;
  106. }
  107. static inline uint32_t xtensa_pmu_read_counter(int idx)
  108. {
  109. return get_er(XTENSA_PMU_PM(idx));
  110. }
  111. static inline void xtensa_pmu_write_counter(int idx, uint32_t v)
  112. {
  113. set_er(v, XTENSA_PMU_PM(idx));
  114. }
  115. static void xtensa_perf_event_update(struct perf_event *event,
  116. struct hw_perf_event *hwc, int idx)
  117. {
  118. uint64_t prev_raw_count, new_raw_count;
  119. int64_t delta;
  120. do {
  121. prev_raw_count = local64_read(&hwc->prev_count);
  122. new_raw_count = xtensa_pmu_read_counter(event->hw.idx);
  123. } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
  124. new_raw_count) != prev_raw_count);
  125. delta = (new_raw_count - prev_raw_count) & XTENSA_PMU_COUNTER_MASK;
  126. local64_add(delta, &event->count);
  127. local64_sub(delta, &hwc->period_left);
  128. }
  129. static bool xtensa_perf_event_set_period(struct perf_event *event,
  130. struct hw_perf_event *hwc, int idx)
  131. {
  132. bool rc = false;
  133. s64 left;
  134. if (!is_sampling_event(event)) {
  135. left = XTENSA_PMU_COUNTER_MAX;
  136. } else {
  137. s64 period = hwc->sample_period;
  138. left = local64_read(&hwc->period_left);
  139. if (left <= -period) {
  140. left = period;
  141. local64_set(&hwc->period_left, left);
  142. hwc->last_period = period;
  143. rc = true;
  144. } else if (left <= 0) {
  145. left += period;
  146. local64_set(&hwc->period_left, left);
  147. hwc->last_period = period;
  148. rc = true;
  149. }
  150. if (left > XTENSA_PMU_COUNTER_MAX)
  151. left = XTENSA_PMU_COUNTER_MAX;
  152. }
  153. local64_set(&hwc->prev_count, -left);
  154. xtensa_pmu_write_counter(idx, -left);
  155. perf_event_update_userpage(event);
  156. return rc;
  157. }
  158. static void xtensa_pmu_enable(struct pmu *pmu)
  159. {
  160. set_er(get_er(XTENSA_PMU_PMG) | XTENSA_PMU_PMG_PMEN, XTENSA_PMU_PMG);
  161. }
  162. static void xtensa_pmu_disable(struct pmu *pmu)
  163. {
  164. set_er(get_er(XTENSA_PMU_PMG) & ~XTENSA_PMU_PMG_PMEN, XTENSA_PMU_PMG);
  165. }
  166. static int xtensa_pmu_event_init(struct perf_event *event)
  167. {
  168. int ret;
  169. switch (event->attr.type) {
  170. case PERF_TYPE_HARDWARE:
  171. if (event->attr.config >= ARRAY_SIZE(xtensa_hw_ctl) ||
  172. xtensa_hw_ctl[event->attr.config] == 0)
  173. return -EINVAL;
  174. event->hw.config = xtensa_hw_ctl[event->attr.config];
  175. return 0;
  176. case PERF_TYPE_HW_CACHE:
  177. ret = xtensa_pmu_cache_event(event->attr.config);
  178. if (ret < 0)
  179. return ret;
  180. event->hw.config = ret;
  181. return 0;
  182. case PERF_TYPE_RAW:
  183. /* Not 'previous counter' select */
  184. if ((event->attr.config & XTENSA_PMU_PMCTRL_SELECT) ==
  185. (1 << XTENSA_PMU_PMCTRL_SELECT_SHIFT))
  186. return -EINVAL;
  187. event->hw.config = (event->attr.config &
  188. (XTENSA_PMU_PMCTRL_KRNLCNT |
  189. XTENSA_PMU_PMCTRL_TRACELEVEL |
  190. XTENSA_PMU_PMCTRL_SELECT |
  191. XTENSA_PMU_PMCTRL_MASK)) |
  192. XTENSA_PMU_PMCTRL_INTEN;
  193. return 0;
  194. default:
  195. return -ENOENT;
  196. }
  197. }
  198. /*
  199. * Starts/Stops a counter present on the PMU. The PMI handler
  200. * should stop the counter when perf_event_overflow() returns
  201. * !0. ->start() will be used to continue.
  202. */
  203. static void xtensa_pmu_start(struct perf_event *event, int flags)
  204. {
  205. struct hw_perf_event *hwc = &event->hw;
  206. int idx = hwc->idx;
  207. if (WARN_ON_ONCE(idx == -1))
  208. return;
  209. if (flags & PERF_EF_RELOAD) {
  210. WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
  211. xtensa_perf_event_set_period(event, hwc, idx);
  212. }
  213. hwc->state = 0;
  214. set_er(hwc->config, XTENSA_PMU_PMCTRL(idx));
  215. }
  216. static void xtensa_pmu_stop(struct perf_event *event, int flags)
  217. {
  218. struct hw_perf_event *hwc = &event->hw;
  219. int idx = hwc->idx;
  220. if (!(hwc->state & PERF_HES_STOPPED)) {
  221. set_er(0, XTENSA_PMU_PMCTRL(idx));
  222. set_er(get_er(XTENSA_PMU_PMSTAT(idx)),
  223. XTENSA_PMU_PMSTAT(idx));
  224. hwc->state |= PERF_HES_STOPPED;
  225. }
  226. if ((flags & PERF_EF_UPDATE) &&
  227. !(event->hw.state & PERF_HES_UPTODATE)) {
  228. xtensa_perf_event_update(event, &event->hw, idx);
  229. event->hw.state |= PERF_HES_UPTODATE;
  230. }
  231. }
  232. /*
  233. * Adds/Removes a counter to/from the PMU, can be done inside
  234. * a transaction, see the ->*_txn() methods.
  235. */
  236. static int xtensa_pmu_add(struct perf_event *event, int flags)
  237. {
  238. struct xtensa_pmu_events *ev = this_cpu_ptr(&xtensa_pmu_events);
  239. struct hw_perf_event *hwc = &event->hw;
  240. int idx = hwc->idx;
  241. if (__test_and_set_bit(idx, ev->used_mask)) {
  242. idx = find_first_zero_bit(ev->used_mask,
  243. XCHAL_NUM_PERF_COUNTERS);
  244. if (idx == XCHAL_NUM_PERF_COUNTERS)
  245. return -EAGAIN;
  246. __set_bit(idx, ev->used_mask);
  247. hwc->idx = idx;
  248. }
  249. ev->event[idx] = event;
  250. hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
  251. if (flags & PERF_EF_START)
  252. xtensa_pmu_start(event, PERF_EF_RELOAD);
  253. perf_event_update_userpage(event);
  254. return 0;
  255. }
  256. static void xtensa_pmu_del(struct perf_event *event, int flags)
  257. {
  258. struct xtensa_pmu_events *ev = this_cpu_ptr(&xtensa_pmu_events);
  259. xtensa_pmu_stop(event, PERF_EF_UPDATE);
  260. __clear_bit(event->hw.idx, ev->used_mask);
  261. perf_event_update_userpage(event);
  262. }
  263. static void xtensa_pmu_read(struct perf_event *event)
  264. {
  265. xtensa_perf_event_update(event, &event->hw, event->hw.idx);
  266. }
  267. static int callchain_trace(struct stackframe *frame, void *data)
  268. {
  269. struct perf_callchain_entry_ctx *entry = data;
  270. perf_callchain_store(entry, frame->pc);
  271. return 0;
  272. }
  273. void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
  274. struct pt_regs *regs)
  275. {
  276. xtensa_backtrace_kernel(regs, entry->max_stack,
  277. callchain_trace, NULL, entry);
  278. }
  279. void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
  280. struct pt_regs *regs)
  281. {
  282. xtensa_backtrace_user(regs, entry->max_stack,
  283. callchain_trace, entry);
  284. }
  285. void perf_event_print_debug(void)
  286. {
  287. unsigned long flags;
  288. unsigned i;
  289. local_irq_save(flags);
  290. pr_info("CPU#%d: PMG: 0x%08lx\n", smp_processor_id(),
  291. get_er(XTENSA_PMU_PMG));
  292. for (i = 0; i < XCHAL_NUM_PERF_COUNTERS; ++i)
  293. pr_info("PM%d: 0x%08lx, PMCTRL%d: 0x%08lx, PMSTAT%d: 0x%08lx\n",
  294. i, get_er(XTENSA_PMU_PM(i)),
  295. i, get_er(XTENSA_PMU_PMCTRL(i)),
  296. i, get_er(XTENSA_PMU_PMSTAT(i)));
  297. local_irq_restore(flags);
  298. }
  299. irqreturn_t xtensa_pmu_irq_handler(int irq, void *dev_id)
  300. {
  301. irqreturn_t rc = IRQ_NONE;
  302. struct xtensa_pmu_events *ev = this_cpu_ptr(&xtensa_pmu_events);
  303. unsigned i;
  304. for_each_set_bit(i, ev->used_mask, XCHAL_NUM_PERF_COUNTERS) {
  305. uint32_t v = get_er(XTENSA_PMU_PMSTAT(i));
  306. struct perf_event *event = ev->event[i];
  307. struct hw_perf_event *hwc = &event->hw;
  308. u64 last_period;
  309. if (!(v & XTENSA_PMU_PMSTAT_OVFL))
  310. continue;
  311. set_er(v, XTENSA_PMU_PMSTAT(i));
  312. xtensa_perf_event_update(event, hwc, i);
  313. last_period = hwc->last_period;
  314. if (xtensa_perf_event_set_period(event, hwc, i)) {
  315. struct perf_sample_data data;
  316. struct pt_regs *regs = get_irq_regs();
  317. perf_sample_data_init(&data, 0, last_period);
  318. if (perf_event_overflow(event, &data, regs))
  319. xtensa_pmu_stop(event, 0);
  320. }
  321. rc = IRQ_HANDLED;
  322. }
  323. return rc;
  324. }
  325. static struct pmu xtensa_pmu = {
  326. .pmu_enable = xtensa_pmu_enable,
  327. .pmu_disable = xtensa_pmu_disable,
  328. .event_init = xtensa_pmu_event_init,
  329. .add = xtensa_pmu_add,
  330. .del = xtensa_pmu_del,
  331. .start = xtensa_pmu_start,
  332. .stop = xtensa_pmu_stop,
  333. .read = xtensa_pmu_read,
  334. };
  335. static int xtensa_pmu_setup(unsigned int cpu)
  336. {
  337. unsigned i;
  338. set_er(0, XTENSA_PMU_PMG);
  339. for (i = 0; i < XCHAL_NUM_PERF_COUNTERS; ++i) {
  340. set_er(0, XTENSA_PMU_PMCTRL(i));
  341. set_er(get_er(XTENSA_PMU_PMSTAT(i)), XTENSA_PMU_PMSTAT(i));
  342. }
  343. return 0;
  344. }
  345. static int __init xtensa_pmu_init(void)
  346. {
  347. int ret;
  348. int irq = irq_create_mapping(NULL, XCHAL_PROFILING_INTERRUPT);
  349. ret = cpuhp_setup_state(CPUHP_AP_PERF_XTENSA_STARTING,
  350. "perf/xtensa:starting", xtensa_pmu_setup,
  351. NULL);
  352. if (ret) {
  353. pr_err("xtensa_pmu: failed to register CPU-hotplug.\n");
  354. return ret;
  355. }
  356. #if XTENSA_FAKE_NMI
  357. enable_irq(irq);
  358. #else
  359. ret = request_irq(irq, xtensa_pmu_irq_handler, IRQF_PERCPU,
  360. "pmu", NULL);
  361. if (ret < 0)
  362. return ret;
  363. #endif
  364. ret = perf_pmu_register(&xtensa_pmu, "cpu", PERF_TYPE_RAW);
  365. if (ret)
  366. free_irq(irq, NULL);
  367. return ret;
  368. }
  369. early_initcall(xtensa_pmu_init);