thunderx2_pmu.c 27 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * CAVIUM THUNDERX2 SoC PMU UNCORE
  4. * Copyright (C) 2018 Cavium Inc.
  5. * Author: Ganapatrao Kulkarni <gkulkarni@cavium.com>
  6. */
  7. #include <linux/acpi.h>
  8. #include <linux/cpuhotplug.h>
  9. #include <linux/perf_event.h>
  10. #include <linux/platform_device.h>
  11. /* Each ThunderX2(TX2) Socket has a L3C and DMC UNCORE PMU device.
  12. * Each UNCORE PMU device consists of 4 independent programmable counters.
  13. * Counters are 32 bit and do not support overflow interrupt,
  14. * they need to be sampled before overflow(i.e, at every 2 seconds).
  15. */
  16. #define TX2_PMU_DMC_L3C_MAX_COUNTERS 4
  17. #define TX2_PMU_CCPI2_MAX_COUNTERS 8
  18. #define TX2_PMU_MAX_COUNTERS TX2_PMU_CCPI2_MAX_COUNTERS
  19. #define TX2_PMU_DMC_CHANNELS 8
  20. #define TX2_PMU_L3_TILES 16
  21. #define TX2_PMU_HRTIMER_INTERVAL (2 * NSEC_PER_SEC)
  22. #define GET_EVENTID(ev, mask) ((ev->hw.config) & mask)
  23. #define GET_COUNTERID(ev, mask) ((ev->hw.idx) & mask)
  24. /* 1 byte per counter(4 counters).
  25. * Event id is encoded in bits [5:1] of a byte,
  26. */
  27. #define DMC_EVENT_CFG(idx, val) ((val) << (((idx) * 8) + 1))
  28. /* bits[3:0] to select counters, are indexed from 8 to 15. */
  29. #define CCPI2_COUNTER_OFFSET 8
  30. #define L3C_COUNTER_CTL 0xA8
  31. #define L3C_COUNTER_DATA 0xAC
  32. #define DMC_COUNTER_CTL 0x234
  33. #define DMC_COUNTER_DATA 0x240
  34. #define CCPI2_PERF_CTL 0x108
  35. #define CCPI2_COUNTER_CTL 0x10C
  36. #define CCPI2_COUNTER_SEL 0x12c
  37. #define CCPI2_COUNTER_DATA_L 0x130
  38. #define CCPI2_COUNTER_DATA_H 0x134
  39. /* L3C event IDs */
  40. #define L3_EVENT_READ_REQ 0xD
  41. #define L3_EVENT_WRITEBACK_REQ 0xE
  42. #define L3_EVENT_INV_N_WRITE_REQ 0xF
  43. #define L3_EVENT_INV_REQ 0x10
  44. #define L3_EVENT_EVICT_REQ 0x13
  45. #define L3_EVENT_INV_N_WRITE_HIT 0x14
  46. #define L3_EVENT_INV_HIT 0x15
  47. #define L3_EVENT_READ_HIT 0x17
  48. #define L3_EVENT_MAX 0x18
  49. /* DMC event IDs */
  50. #define DMC_EVENT_COUNT_CYCLES 0x1
  51. #define DMC_EVENT_WRITE_TXNS 0xB
  52. #define DMC_EVENT_DATA_TRANSFERS 0xD
  53. #define DMC_EVENT_READ_TXNS 0xF
  54. #define DMC_EVENT_MAX 0x10
  55. #define CCPI2_EVENT_REQ_PKT_SENT 0x3D
  56. #define CCPI2_EVENT_SNOOP_PKT_SENT 0x65
  57. #define CCPI2_EVENT_DATA_PKT_SENT 0x105
  58. #define CCPI2_EVENT_GIC_PKT_SENT 0x12D
  59. #define CCPI2_EVENT_MAX 0x200
  60. #define CCPI2_PERF_CTL_ENABLE BIT(0)
  61. #define CCPI2_PERF_CTL_START BIT(1)
  62. #define CCPI2_PERF_CTL_RESET BIT(4)
  63. #define CCPI2_EVENT_LEVEL_RISING_EDGE BIT(10)
  64. #define CCPI2_EVENT_TYPE_EDGE_SENSITIVE BIT(11)
  65. enum tx2_uncore_type {
  66. PMU_TYPE_L3C,
  67. PMU_TYPE_DMC,
  68. PMU_TYPE_CCPI2,
  69. PMU_TYPE_INVALID,
  70. };
  71. /*
  72. * Each socket has 3 uncore devices associated with a PMU. The DMC and
  73. * L3C have 4 32-bit counters and the CCPI2 has 8 64-bit counters.
  74. */
  75. struct tx2_uncore_pmu {
  76. struct hlist_node hpnode;
  77. struct list_head entry;
  78. struct pmu pmu;
  79. char *name;
  80. int node;
  81. int cpu;
  82. u32 max_counters;
  83. u32 counters_mask;
  84. u32 prorate_factor;
  85. u32 max_events;
  86. u32 events_mask;
  87. u64 hrtimer_interval;
  88. void __iomem *base;
  89. DECLARE_BITMAP(active_counters, TX2_PMU_MAX_COUNTERS);
  90. struct perf_event *events[TX2_PMU_MAX_COUNTERS];
  91. struct device *dev;
  92. struct hrtimer hrtimer;
  93. const struct attribute_group **attr_groups;
  94. enum tx2_uncore_type type;
  95. enum hrtimer_restart (*hrtimer_callback)(struct hrtimer *cb);
  96. void (*init_cntr_base)(struct perf_event *event,
  97. struct tx2_uncore_pmu *tx2_pmu);
  98. void (*stop_event)(struct perf_event *event);
  99. void (*start_event)(struct perf_event *event, int flags);
  100. };
  101. static LIST_HEAD(tx2_pmus);
  102. static inline struct tx2_uncore_pmu *pmu_to_tx2_pmu(struct pmu *pmu)
  103. {
  104. return container_of(pmu, struct tx2_uncore_pmu, pmu);
  105. }
  106. #define TX2_PMU_FORMAT_ATTR(_var, _name, _format) \
  107. static ssize_t \
  108. __tx2_pmu_##_var##_show(struct device *dev, \
  109. struct device_attribute *attr, \
  110. char *page) \
  111. { \
  112. BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
  113. return sprintf(page, _format "\n"); \
  114. } \
  115. \
  116. static struct device_attribute format_attr_##_var = \
  117. __ATTR(_name, 0444, __tx2_pmu_##_var##_show, NULL)
  118. TX2_PMU_FORMAT_ATTR(event, event, "config:0-4");
  119. TX2_PMU_FORMAT_ATTR(event_ccpi2, event, "config:0-9");
  120. static struct attribute *l3c_pmu_format_attrs[] = {
  121. &format_attr_event.attr,
  122. NULL,
  123. };
  124. static struct attribute *dmc_pmu_format_attrs[] = {
  125. &format_attr_event.attr,
  126. NULL,
  127. };
  128. static struct attribute *ccpi2_pmu_format_attrs[] = {
  129. &format_attr_event_ccpi2.attr,
  130. NULL,
  131. };
  132. static const struct attribute_group l3c_pmu_format_attr_group = {
  133. .name = "format",
  134. .attrs = l3c_pmu_format_attrs,
  135. };
  136. static const struct attribute_group dmc_pmu_format_attr_group = {
  137. .name = "format",
  138. .attrs = dmc_pmu_format_attrs,
  139. };
  140. static const struct attribute_group ccpi2_pmu_format_attr_group = {
  141. .name = "format",
  142. .attrs = ccpi2_pmu_format_attrs,
  143. };
  144. /*
  145. * sysfs event attributes
  146. */
  147. static ssize_t tx2_pmu_event_show(struct device *dev,
  148. struct device_attribute *attr, char *buf)
  149. {
  150. struct dev_ext_attribute *eattr;
  151. eattr = container_of(attr, struct dev_ext_attribute, attr);
  152. return sprintf(buf, "event=0x%lx\n", (unsigned long) eattr->var);
  153. }
  154. #define TX2_EVENT_ATTR(name, config) \
  155. PMU_EVENT_ATTR(name, tx2_pmu_event_attr_##name, \
  156. config, tx2_pmu_event_show)
  157. TX2_EVENT_ATTR(read_request, L3_EVENT_READ_REQ);
  158. TX2_EVENT_ATTR(writeback_request, L3_EVENT_WRITEBACK_REQ);
  159. TX2_EVENT_ATTR(inv_nwrite_request, L3_EVENT_INV_N_WRITE_REQ);
  160. TX2_EVENT_ATTR(inv_request, L3_EVENT_INV_REQ);
  161. TX2_EVENT_ATTR(evict_request, L3_EVENT_EVICT_REQ);
  162. TX2_EVENT_ATTR(inv_nwrite_hit, L3_EVENT_INV_N_WRITE_HIT);
  163. TX2_EVENT_ATTR(inv_hit, L3_EVENT_INV_HIT);
  164. TX2_EVENT_ATTR(read_hit, L3_EVENT_READ_HIT);
  165. static struct attribute *l3c_pmu_events_attrs[] = {
  166. &tx2_pmu_event_attr_read_request.attr.attr,
  167. &tx2_pmu_event_attr_writeback_request.attr.attr,
  168. &tx2_pmu_event_attr_inv_nwrite_request.attr.attr,
  169. &tx2_pmu_event_attr_inv_request.attr.attr,
  170. &tx2_pmu_event_attr_evict_request.attr.attr,
  171. &tx2_pmu_event_attr_inv_nwrite_hit.attr.attr,
  172. &tx2_pmu_event_attr_inv_hit.attr.attr,
  173. &tx2_pmu_event_attr_read_hit.attr.attr,
  174. NULL,
  175. };
  176. TX2_EVENT_ATTR(cnt_cycles, DMC_EVENT_COUNT_CYCLES);
  177. TX2_EVENT_ATTR(write_txns, DMC_EVENT_WRITE_TXNS);
  178. TX2_EVENT_ATTR(data_transfers, DMC_EVENT_DATA_TRANSFERS);
  179. TX2_EVENT_ATTR(read_txns, DMC_EVENT_READ_TXNS);
  180. static struct attribute *dmc_pmu_events_attrs[] = {
  181. &tx2_pmu_event_attr_cnt_cycles.attr.attr,
  182. &tx2_pmu_event_attr_write_txns.attr.attr,
  183. &tx2_pmu_event_attr_data_transfers.attr.attr,
  184. &tx2_pmu_event_attr_read_txns.attr.attr,
  185. NULL,
  186. };
  187. TX2_EVENT_ATTR(req_pktsent, CCPI2_EVENT_REQ_PKT_SENT);
  188. TX2_EVENT_ATTR(snoop_pktsent, CCPI2_EVENT_SNOOP_PKT_SENT);
  189. TX2_EVENT_ATTR(data_pktsent, CCPI2_EVENT_DATA_PKT_SENT);
  190. TX2_EVENT_ATTR(gic_pktsent, CCPI2_EVENT_GIC_PKT_SENT);
  191. static struct attribute *ccpi2_pmu_events_attrs[] = {
  192. &tx2_pmu_event_attr_req_pktsent.attr.attr,
  193. &tx2_pmu_event_attr_snoop_pktsent.attr.attr,
  194. &tx2_pmu_event_attr_data_pktsent.attr.attr,
  195. &tx2_pmu_event_attr_gic_pktsent.attr.attr,
  196. NULL,
  197. };
  198. static const struct attribute_group l3c_pmu_events_attr_group = {
  199. .name = "events",
  200. .attrs = l3c_pmu_events_attrs,
  201. };
  202. static const struct attribute_group dmc_pmu_events_attr_group = {
  203. .name = "events",
  204. .attrs = dmc_pmu_events_attrs,
  205. };
  206. static const struct attribute_group ccpi2_pmu_events_attr_group = {
  207. .name = "events",
  208. .attrs = ccpi2_pmu_events_attrs,
  209. };
  210. /*
  211. * sysfs cpumask attributes
  212. */
  213. static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
  214. char *buf)
  215. {
  216. struct tx2_uncore_pmu *tx2_pmu;
  217. tx2_pmu = pmu_to_tx2_pmu(dev_get_drvdata(dev));
  218. return cpumap_print_to_pagebuf(true, buf, cpumask_of(tx2_pmu->cpu));
  219. }
  220. static DEVICE_ATTR_RO(cpumask);
  221. static struct attribute *tx2_pmu_cpumask_attrs[] = {
  222. &dev_attr_cpumask.attr,
  223. NULL,
  224. };
  225. static const struct attribute_group pmu_cpumask_attr_group = {
  226. .attrs = tx2_pmu_cpumask_attrs,
  227. };
  228. /*
  229. * Per PMU device attribute groups
  230. */
  231. static const struct attribute_group *l3c_pmu_attr_groups[] = {
  232. &l3c_pmu_format_attr_group,
  233. &pmu_cpumask_attr_group,
  234. &l3c_pmu_events_attr_group,
  235. NULL
  236. };
  237. static const struct attribute_group *dmc_pmu_attr_groups[] = {
  238. &dmc_pmu_format_attr_group,
  239. &pmu_cpumask_attr_group,
  240. &dmc_pmu_events_attr_group,
  241. NULL
  242. };
  243. static const struct attribute_group *ccpi2_pmu_attr_groups[] = {
  244. &ccpi2_pmu_format_attr_group,
  245. &pmu_cpumask_attr_group,
  246. &ccpi2_pmu_events_attr_group,
  247. NULL
  248. };
  249. static inline u32 reg_readl(unsigned long addr)
  250. {
  251. return readl((void __iomem *)addr);
  252. }
  253. static inline void reg_writel(u32 val, unsigned long addr)
  254. {
  255. writel(val, (void __iomem *)addr);
  256. }
  257. static int alloc_counter(struct tx2_uncore_pmu *tx2_pmu)
  258. {
  259. int counter;
  260. counter = find_first_zero_bit(tx2_pmu->active_counters,
  261. tx2_pmu->max_counters);
  262. if (counter == tx2_pmu->max_counters)
  263. return -ENOSPC;
  264. set_bit(counter, tx2_pmu->active_counters);
  265. return counter;
  266. }
  267. static inline void free_counter(struct tx2_uncore_pmu *tx2_pmu, int counter)
  268. {
  269. clear_bit(counter, tx2_pmu->active_counters);
  270. }
  271. static void init_cntr_base_l3c(struct perf_event *event,
  272. struct tx2_uncore_pmu *tx2_pmu)
  273. {
  274. struct hw_perf_event *hwc = &event->hw;
  275. u32 cmask;
  276. tx2_pmu = pmu_to_tx2_pmu(event->pmu);
  277. cmask = tx2_pmu->counters_mask;
  278. /* counter ctrl/data reg offset at 8 */
  279. hwc->config_base = (unsigned long)tx2_pmu->base
  280. + L3C_COUNTER_CTL + (8 * GET_COUNTERID(event, cmask));
  281. hwc->event_base = (unsigned long)tx2_pmu->base
  282. + L3C_COUNTER_DATA + (8 * GET_COUNTERID(event, cmask));
  283. }
  284. static void init_cntr_base_dmc(struct perf_event *event,
  285. struct tx2_uncore_pmu *tx2_pmu)
  286. {
  287. struct hw_perf_event *hwc = &event->hw;
  288. u32 cmask;
  289. tx2_pmu = pmu_to_tx2_pmu(event->pmu);
  290. cmask = tx2_pmu->counters_mask;
  291. hwc->config_base = (unsigned long)tx2_pmu->base
  292. + DMC_COUNTER_CTL;
  293. /* counter data reg offset at 0xc */
  294. hwc->event_base = (unsigned long)tx2_pmu->base
  295. + DMC_COUNTER_DATA + (0xc * GET_COUNTERID(event, cmask));
  296. }
  297. static void init_cntr_base_ccpi2(struct perf_event *event,
  298. struct tx2_uncore_pmu *tx2_pmu)
  299. {
  300. struct hw_perf_event *hwc = &event->hw;
  301. u32 cmask;
  302. cmask = tx2_pmu->counters_mask;
  303. hwc->config_base = (unsigned long)tx2_pmu->base
  304. + CCPI2_COUNTER_CTL + (4 * GET_COUNTERID(event, cmask));
  305. hwc->event_base = (unsigned long)tx2_pmu->base;
  306. }
  307. static void uncore_start_event_l3c(struct perf_event *event, int flags)
  308. {
  309. u32 val, emask;
  310. struct hw_perf_event *hwc = &event->hw;
  311. struct tx2_uncore_pmu *tx2_pmu;
  312. tx2_pmu = pmu_to_tx2_pmu(event->pmu);
  313. emask = tx2_pmu->events_mask;
  314. /* event id encoded in bits [07:03] */
  315. val = GET_EVENTID(event, emask) << 3;
  316. reg_writel(val, hwc->config_base);
  317. local64_set(&hwc->prev_count, 0);
  318. reg_writel(0, hwc->event_base);
  319. }
  320. static inline void uncore_stop_event_l3c(struct perf_event *event)
  321. {
  322. reg_writel(0, event->hw.config_base);
  323. }
  324. static void uncore_start_event_dmc(struct perf_event *event, int flags)
  325. {
  326. u32 val, cmask, emask;
  327. struct hw_perf_event *hwc = &event->hw;
  328. struct tx2_uncore_pmu *tx2_pmu;
  329. int idx, event_id;
  330. tx2_pmu = pmu_to_tx2_pmu(event->pmu);
  331. cmask = tx2_pmu->counters_mask;
  332. emask = tx2_pmu->events_mask;
  333. idx = GET_COUNTERID(event, cmask);
  334. event_id = GET_EVENTID(event, emask);
  335. /* enable and start counters.
  336. * 8 bits for each counter, bits[05:01] of a counter to set event type.
  337. */
  338. val = reg_readl(hwc->config_base);
  339. val &= ~DMC_EVENT_CFG(idx, 0x1f);
  340. val |= DMC_EVENT_CFG(idx, event_id);
  341. reg_writel(val, hwc->config_base);
  342. local64_set(&hwc->prev_count, 0);
  343. reg_writel(0, hwc->event_base);
  344. }
  345. static void uncore_stop_event_dmc(struct perf_event *event)
  346. {
  347. u32 val, cmask;
  348. struct hw_perf_event *hwc = &event->hw;
  349. struct tx2_uncore_pmu *tx2_pmu;
  350. int idx;
  351. tx2_pmu = pmu_to_tx2_pmu(event->pmu);
  352. cmask = tx2_pmu->counters_mask;
  353. idx = GET_COUNTERID(event, cmask);
  354. /* clear event type(bits[05:01]) to stop counter */
  355. val = reg_readl(hwc->config_base);
  356. val &= ~DMC_EVENT_CFG(idx, 0x1f);
  357. reg_writel(val, hwc->config_base);
  358. }
  359. static void uncore_start_event_ccpi2(struct perf_event *event, int flags)
  360. {
  361. u32 emask;
  362. struct hw_perf_event *hwc = &event->hw;
  363. struct tx2_uncore_pmu *tx2_pmu;
  364. tx2_pmu = pmu_to_tx2_pmu(event->pmu);
  365. emask = tx2_pmu->events_mask;
  366. /* Bit [09:00] to set event id.
  367. * Bits [10], set level to rising edge.
  368. * Bits [11], set type to edge sensitive.
  369. */
  370. reg_writel((CCPI2_EVENT_TYPE_EDGE_SENSITIVE |
  371. CCPI2_EVENT_LEVEL_RISING_EDGE |
  372. GET_EVENTID(event, emask)), hwc->config_base);
  373. /* reset[4], enable[0] and start[1] counters */
  374. reg_writel(CCPI2_PERF_CTL_RESET |
  375. CCPI2_PERF_CTL_START |
  376. CCPI2_PERF_CTL_ENABLE,
  377. hwc->event_base + CCPI2_PERF_CTL);
  378. local64_set(&event->hw.prev_count, 0ULL);
  379. }
  380. static void uncore_stop_event_ccpi2(struct perf_event *event)
  381. {
  382. struct hw_perf_event *hwc = &event->hw;
  383. /* disable and stop counter */
  384. reg_writel(0, hwc->event_base + CCPI2_PERF_CTL);
  385. }
  386. static void tx2_uncore_event_update(struct perf_event *event)
  387. {
  388. u64 prev, delta, new = 0;
  389. struct hw_perf_event *hwc = &event->hw;
  390. struct tx2_uncore_pmu *tx2_pmu;
  391. enum tx2_uncore_type type;
  392. u32 prorate_factor;
  393. u32 cmask, emask;
  394. tx2_pmu = pmu_to_tx2_pmu(event->pmu);
  395. type = tx2_pmu->type;
  396. cmask = tx2_pmu->counters_mask;
  397. emask = tx2_pmu->events_mask;
  398. prorate_factor = tx2_pmu->prorate_factor;
  399. if (type == PMU_TYPE_CCPI2) {
  400. reg_writel(CCPI2_COUNTER_OFFSET +
  401. GET_COUNTERID(event, cmask),
  402. hwc->event_base + CCPI2_COUNTER_SEL);
  403. new = reg_readl(hwc->event_base + CCPI2_COUNTER_DATA_H);
  404. new = (new << 32) +
  405. reg_readl(hwc->event_base + CCPI2_COUNTER_DATA_L);
  406. prev = local64_xchg(&hwc->prev_count, new);
  407. delta = new - prev;
  408. } else {
  409. new = reg_readl(hwc->event_base);
  410. prev = local64_xchg(&hwc->prev_count, new);
  411. /* handles rollover of 32 bit counter */
  412. delta = (u32)(((1UL << 32) - prev) + new);
  413. }
  414. /* DMC event data_transfers granularity is 16 Bytes, convert it to 64 */
  415. if (type == PMU_TYPE_DMC &&
  416. GET_EVENTID(event, emask) == DMC_EVENT_DATA_TRANSFERS)
  417. delta = delta/4;
  418. /* L3C and DMC has 16 and 8 interleave channels respectively.
  419. * The sampled value is for channel 0 and multiplied with
  420. * prorate_factor to get the count for a device.
  421. */
  422. local64_add(delta * prorate_factor, &event->count);
  423. }
  424. static enum tx2_uncore_type get_tx2_pmu_type(struct acpi_device *adev)
  425. {
  426. int i = 0;
  427. struct acpi_tx2_pmu_device {
  428. __u8 id[ACPI_ID_LEN];
  429. enum tx2_uncore_type type;
  430. } devices[] = {
  431. {"CAV901D", PMU_TYPE_L3C},
  432. {"CAV901F", PMU_TYPE_DMC},
  433. {"CAV901E", PMU_TYPE_CCPI2},
  434. {"", PMU_TYPE_INVALID}
  435. };
  436. while (devices[i].type != PMU_TYPE_INVALID) {
  437. if (!strcmp(acpi_device_hid(adev), devices[i].id))
  438. break;
  439. i++;
  440. }
  441. return devices[i].type;
  442. }
  443. static bool tx2_uncore_validate_event(struct pmu *pmu,
  444. struct perf_event *event, int *counters)
  445. {
  446. if (is_software_event(event))
  447. return true;
  448. /* Reject groups spanning multiple HW PMUs. */
  449. if (event->pmu != pmu)
  450. return false;
  451. *counters = *counters + 1;
  452. return true;
  453. }
  454. /*
  455. * Make sure the group of events can be scheduled at once
  456. * on the PMU.
  457. */
  458. static bool tx2_uncore_validate_event_group(struct perf_event *event,
  459. int max_counters)
  460. {
  461. struct perf_event *sibling, *leader = event->group_leader;
  462. int counters = 0;
  463. if (event->group_leader == event)
  464. return true;
  465. if (!tx2_uncore_validate_event(event->pmu, leader, &counters))
  466. return false;
  467. for_each_sibling_event(sibling, leader) {
  468. if (!tx2_uncore_validate_event(event->pmu, sibling, &counters))
  469. return false;
  470. }
  471. if (!tx2_uncore_validate_event(event->pmu, event, &counters))
  472. return false;
  473. /*
  474. * If the group requires more counters than the HW has,
  475. * it cannot ever be scheduled.
  476. */
  477. return counters <= max_counters;
  478. }
  479. static int tx2_uncore_event_init(struct perf_event *event)
  480. {
  481. struct hw_perf_event *hwc = &event->hw;
  482. struct tx2_uncore_pmu *tx2_pmu;
  483. /* Test the event attr type check for PMU enumeration */
  484. if (event->attr.type != event->pmu->type)
  485. return -ENOENT;
  486. /*
  487. * SOC PMU counters are shared across all cores.
  488. * Therefore, it does not support per-process mode.
  489. * Also, it does not support event sampling mode.
  490. */
  491. if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
  492. return -EINVAL;
  493. if (event->cpu < 0)
  494. return -EINVAL;
  495. tx2_pmu = pmu_to_tx2_pmu(event->pmu);
  496. if (tx2_pmu->cpu >= nr_cpu_ids)
  497. return -EINVAL;
  498. event->cpu = tx2_pmu->cpu;
  499. if (event->attr.config >= tx2_pmu->max_events)
  500. return -EINVAL;
  501. /* store event id */
  502. hwc->config = event->attr.config;
  503. /* Validate the group */
  504. if (!tx2_uncore_validate_event_group(event, tx2_pmu->max_counters))
  505. return -EINVAL;
  506. return 0;
  507. }
  508. static void tx2_uncore_event_start(struct perf_event *event, int flags)
  509. {
  510. struct hw_perf_event *hwc = &event->hw;
  511. struct tx2_uncore_pmu *tx2_pmu;
  512. hwc->state = 0;
  513. tx2_pmu = pmu_to_tx2_pmu(event->pmu);
  514. tx2_pmu->start_event(event, flags);
  515. perf_event_update_userpage(event);
  516. /* No hrtimer needed for CCPI2, 64-bit counters */
  517. if (!tx2_pmu->hrtimer_callback)
  518. return;
  519. /* Start timer for first event */
  520. if (bitmap_weight(tx2_pmu->active_counters,
  521. tx2_pmu->max_counters) == 1) {
  522. hrtimer_start(&tx2_pmu->hrtimer,
  523. ns_to_ktime(tx2_pmu->hrtimer_interval),
  524. HRTIMER_MODE_REL_PINNED);
  525. }
  526. }
  527. static void tx2_uncore_event_stop(struct perf_event *event, int flags)
  528. {
  529. struct hw_perf_event *hwc = &event->hw;
  530. struct tx2_uncore_pmu *tx2_pmu;
  531. if (hwc->state & PERF_HES_UPTODATE)
  532. return;
  533. tx2_pmu = pmu_to_tx2_pmu(event->pmu);
  534. tx2_pmu->stop_event(event);
  535. WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
  536. hwc->state |= PERF_HES_STOPPED;
  537. if (flags & PERF_EF_UPDATE) {
  538. tx2_uncore_event_update(event);
  539. hwc->state |= PERF_HES_UPTODATE;
  540. }
  541. }
  542. static int tx2_uncore_event_add(struct perf_event *event, int flags)
  543. {
  544. struct hw_perf_event *hwc = &event->hw;
  545. struct tx2_uncore_pmu *tx2_pmu;
  546. tx2_pmu = pmu_to_tx2_pmu(event->pmu);
  547. /* Allocate a free counter */
  548. hwc->idx = alloc_counter(tx2_pmu);
  549. if (hwc->idx < 0)
  550. return -EAGAIN;
  551. tx2_pmu->events[hwc->idx] = event;
  552. /* set counter control and data registers base address */
  553. tx2_pmu->init_cntr_base(event, tx2_pmu);
  554. hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
  555. if (flags & PERF_EF_START)
  556. tx2_uncore_event_start(event, flags);
  557. return 0;
  558. }
  559. static void tx2_uncore_event_del(struct perf_event *event, int flags)
  560. {
  561. struct tx2_uncore_pmu *tx2_pmu = pmu_to_tx2_pmu(event->pmu);
  562. struct hw_perf_event *hwc = &event->hw;
  563. u32 cmask;
  564. cmask = tx2_pmu->counters_mask;
  565. tx2_uncore_event_stop(event, PERF_EF_UPDATE);
  566. /* clear the assigned counter */
  567. free_counter(tx2_pmu, GET_COUNTERID(event, cmask));
  568. perf_event_update_userpage(event);
  569. tx2_pmu->events[hwc->idx] = NULL;
  570. hwc->idx = -1;
  571. if (!tx2_pmu->hrtimer_callback)
  572. return;
  573. if (bitmap_empty(tx2_pmu->active_counters, tx2_pmu->max_counters))
  574. hrtimer_cancel(&tx2_pmu->hrtimer);
  575. }
  576. static void tx2_uncore_event_read(struct perf_event *event)
  577. {
  578. tx2_uncore_event_update(event);
  579. }
  580. static enum hrtimer_restart tx2_hrtimer_callback(struct hrtimer *timer)
  581. {
  582. struct tx2_uncore_pmu *tx2_pmu;
  583. int max_counters, idx;
  584. tx2_pmu = container_of(timer, struct tx2_uncore_pmu, hrtimer);
  585. max_counters = tx2_pmu->max_counters;
  586. if (bitmap_empty(tx2_pmu->active_counters, max_counters))
  587. return HRTIMER_NORESTART;
  588. for_each_set_bit(idx, tx2_pmu->active_counters, max_counters) {
  589. struct perf_event *event = tx2_pmu->events[idx];
  590. tx2_uncore_event_update(event);
  591. }
  592. hrtimer_forward_now(timer, ns_to_ktime(tx2_pmu->hrtimer_interval));
  593. return HRTIMER_RESTART;
  594. }
  595. static int tx2_uncore_pmu_register(
  596. struct tx2_uncore_pmu *tx2_pmu)
  597. {
  598. struct device *dev = tx2_pmu->dev;
  599. char *name = tx2_pmu->name;
  600. /* Perf event registration */
  601. tx2_pmu->pmu = (struct pmu) {
  602. .module = THIS_MODULE,
  603. .attr_groups = tx2_pmu->attr_groups,
  604. .task_ctx_nr = perf_invalid_context,
  605. .event_init = tx2_uncore_event_init,
  606. .add = tx2_uncore_event_add,
  607. .del = tx2_uncore_event_del,
  608. .start = tx2_uncore_event_start,
  609. .stop = tx2_uncore_event_stop,
  610. .read = tx2_uncore_event_read,
  611. .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
  612. };
  613. tx2_pmu->pmu.name = devm_kasprintf(dev, GFP_KERNEL,
  614. "%s", name);
  615. return perf_pmu_register(&tx2_pmu->pmu, tx2_pmu->pmu.name, -1);
  616. }
  617. static int tx2_uncore_pmu_add_dev(struct tx2_uncore_pmu *tx2_pmu)
  618. {
  619. int ret, cpu;
  620. cpu = cpumask_any_and(cpumask_of_node(tx2_pmu->node),
  621. cpu_online_mask);
  622. tx2_pmu->cpu = cpu;
  623. if (tx2_pmu->hrtimer_callback) {
  624. hrtimer_init(&tx2_pmu->hrtimer,
  625. CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  626. tx2_pmu->hrtimer.function = tx2_pmu->hrtimer_callback;
  627. }
  628. ret = tx2_uncore_pmu_register(tx2_pmu);
  629. if (ret) {
  630. dev_err(tx2_pmu->dev, "%s PMU: Failed to init driver\n",
  631. tx2_pmu->name);
  632. return -ENODEV;
  633. }
  634. /* register hotplug callback for the pmu */
  635. ret = cpuhp_state_add_instance(
  636. CPUHP_AP_PERF_ARM_CAVIUM_TX2_UNCORE_ONLINE,
  637. &tx2_pmu->hpnode);
  638. if (ret) {
  639. dev_err(tx2_pmu->dev, "Error %d registering hotplug", ret);
  640. return ret;
  641. }
  642. /* Add to list */
  643. list_add(&tx2_pmu->entry, &tx2_pmus);
  644. dev_dbg(tx2_pmu->dev, "%s PMU UNCORE registered\n",
  645. tx2_pmu->pmu.name);
  646. return ret;
  647. }
  648. static struct tx2_uncore_pmu *tx2_uncore_pmu_init_dev(struct device *dev,
  649. acpi_handle handle, struct acpi_device *adev, u32 type)
  650. {
  651. struct tx2_uncore_pmu *tx2_pmu;
  652. void __iomem *base;
  653. struct resource res;
  654. struct resource_entry *rentry;
  655. struct list_head list;
  656. int ret;
  657. INIT_LIST_HEAD(&list);
  658. ret = acpi_dev_get_resources(adev, &list, NULL, NULL);
  659. if (ret <= 0) {
  660. dev_err(dev, "failed to parse _CRS method, error %d\n", ret);
  661. return NULL;
  662. }
  663. list_for_each_entry(rentry, &list, node) {
  664. if (resource_type(rentry->res) == IORESOURCE_MEM) {
  665. res = *rentry->res;
  666. rentry = NULL;
  667. break;
  668. }
  669. }
  670. acpi_dev_free_resource_list(&list);
  671. if (rentry) {
  672. dev_err(dev, "PMU type %d: Fail to find resource\n", type);
  673. return NULL;
  674. }
  675. base = devm_ioremap_resource(dev, &res);
  676. if (IS_ERR(base)) {
  677. dev_err(dev, "PMU type %d: Fail to map resource\n", type);
  678. return NULL;
  679. }
  680. tx2_pmu = devm_kzalloc(dev, sizeof(*tx2_pmu), GFP_KERNEL);
  681. if (!tx2_pmu)
  682. return NULL;
  683. tx2_pmu->dev = dev;
  684. tx2_pmu->type = type;
  685. tx2_pmu->base = base;
  686. tx2_pmu->node = dev_to_node(dev);
  687. INIT_LIST_HEAD(&tx2_pmu->entry);
  688. switch (tx2_pmu->type) {
  689. case PMU_TYPE_L3C:
  690. tx2_pmu->max_counters = TX2_PMU_DMC_L3C_MAX_COUNTERS;
  691. tx2_pmu->counters_mask = 0x3;
  692. tx2_pmu->prorate_factor = TX2_PMU_L3_TILES;
  693. tx2_pmu->max_events = L3_EVENT_MAX;
  694. tx2_pmu->events_mask = 0x1f;
  695. tx2_pmu->hrtimer_interval = TX2_PMU_HRTIMER_INTERVAL;
  696. tx2_pmu->hrtimer_callback = tx2_hrtimer_callback;
  697. tx2_pmu->attr_groups = l3c_pmu_attr_groups;
  698. tx2_pmu->name = devm_kasprintf(dev, GFP_KERNEL,
  699. "uncore_l3c_%d", tx2_pmu->node);
  700. tx2_pmu->init_cntr_base = init_cntr_base_l3c;
  701. tx2_pmu->start_event = uncore_start_event_l3c;
  702. tx2_pmu->stop_event = uncore_stop_event_l3c;
  703. break;
  704. case PMU_TYPE_DMC:
  705. tx2_pmu->max_counters = TX2_PMU_DMC_L3C_MAX_COUNTERS;
  706. tx2_pmu->counters_mask = 0x3;
  707. tx2_pmu->prorate_factor = TX2_PMU_DMC_CHANNELS;
  708. tx2_pmu->max_events = DMC_EVENT_MAX;
  709. tx2_pmu->events_mask = 0x1f;
  710. tx2_pmu->hrtimer_interval = TX2_PMU_HRTIMER_INTERVAL;
  711. tx2_pmu->hrtimer_callback = tx2_hrtimer_callback;
  712. tx2_pmu->attr_groups = dmc_pmu_attr_groups;
  713. tx2_pmu->name = devm_kasprintf(dev, GFP_KERNEL,
  714. "uncore_dmc_%d", tx2_pmu->node);
  715. tx2_pmu->init_cntr_base = init_cntr_base_dmc;
  716. tx2_pmu->start_event = uncore_start_event_dmc;
  717. tx2_pmu->stop_event = uncore_stop_event_dmc;
  718. break;
  719. case PMU_TYPE_CCPI2:
  720. /* CCPI2 has 8 counters */
  721. tx2_pmu->max_counters = TX2_PMU_CCPI2_MAX_COUNTERS;
  722. tx2_pmu->counters_mask = 0x7;
  723. tx2_pmu->prorate_factor = 1;
  724. tx2_pmu->max_events = CCPI2_EVENT_MAX;
  725. tx2_pmu->events_mask = 0x1ff;
  726. tx2_pmu->attr_groups = ccpi2_pmu_attr_groups;
  727. tx2_pmu->name = devm_kasprintf(dev, GFP_KERNEL,
  728. "uncore_ccpi2_%d", tx2_pmu->node);
  729. tx2_pmu->init_cntr_base = init_cntr_base_ccpi2;
  730. tx2_pmu->start_event = uncore_start_event_ccpi2;
  731. tx2_pmu->stop_event = uncore_stop_event_ccpi2;
  732. tx2_pmu->hrtimer_callback = NULL;
  733. break;
  734. case PMU_TYPE_INVALID:
  735. devm_kfree(dev, tx2_pmu);
  736. return NULL;
  737. }
  738. return tx2_pmu;
  739. }
  740. static acpi_status tx2_uncore_pmu_add(acpi_handle handle, u32 level,
  741. void *data, void **return_value)
  742. {
  743. struct tx2_uncore_pmu *tx2_pmu;
  744. struct acpi_device *adev;
  745. enum tx2_uncore_type type;
  746. if (acpi_bus_get_device(handle, &adev))
  747. return AE_OK;
  748. if (acpi_bus_get_status(adev) || !adev->status.present)
  749. return AE_OK;
  750. type = get_tx2_pmu_type(adev);
  751. if (type == PMU_TYPE_INVALID)
  752. return AE_OK;
  753. tx2_pmu = tx2_uncore_pmu_init_dev((struct device *)data,
  754. handle, adev, type);
  755. if (!tx2_pmu)
  756. return AE_ERROR;
  757. if (tx2_uncore_pmu_add_dev(tx2_pmu)) {
  758. /* Can't add the PMU device, abort */
  759. return AE_ERROR;
  760. }
  761. return AE_OK;
  762. }
  763. static int tx2_uncore_pmu_online_cpu(unsigned int cpu,
  764. struct hlist_node *hpnode)
  765. {
  766. struct tx2_uncore_pmu *tx2_pmu;
  767. tx2_pmu = hlist_entry_safe(hpnode,
  768. struct tx2_uncore_pmu, hpnode);
  769. /* Pick this CPU, If there is no CPU/PMU association and both are
  770. * from same node.
  771. */
  772. if ((tx2_pmu->cpu >= nr_cpu_ids) &&
  773. (tx2_pmu->node == cpu_to_node(cpu)))
  774. tx2_pmu->cpu = cpu;
  775. return 0;
  776. }
  777. static int tx2_uncore_pmu_offline_cpu(unsigned int cpu,
  778. struct hlist_node *hpnode)
  779. {
  780. int new_cpu;
  781. struct tx2_uncore_pmu *tx2_pmu;
  782. struct cpumask cpu_online_mask_temp;
  783. tx2_pmu = hlist_entry_safe(hpnode,
  784. struct tx2_uncore_pmu, hpnode);
  785. if (cpu != tx2_pmu->cpu)
  786. return 0;
  787. if (tx2_pmu->hrtimer_callback)
  788. hrtimer_cancel(&tx2_pmu->hrtimer);
  789. cpumask_copy(&cpu_online_mask_temp, cpu_online_mask);
  790. cpumask_clear_cpu(cpu, &cpu_online_mask_temp);
  791. new_cpu = cpumask_any_and(
  792. cpumask_of_node(tx2_pmu->node),
  793. &cpu_online_mask_temp);
  794. tx2_pmu->cpu = new_cpu;
  795. if (new_cpu >= nr_cpu_ids)
  796. return 0;
  797. perf_pmu_migrate_context(&tx2_pmu->pmu, cpu, new_cpu);
  798. return 0;
  799. }
  800. static const struct acpi_device_id tx2_uncore_acpi_match[] = {
  801. {"CAV901C", 0},
  802. {},
  803. };
  804. MODULE_DEVICE_TABLE(acpi, tx2_uncore_acpi_match);
  805. static int tx2_uncore_probe(struct platform_device *pdev)
  806. {
  807. struct device *dev = &pdev->dev;
  808. acpi_handle handle;
  809. acpi_status status;
  810. set_dev_node(dev, acpi_get_node(ACPI_HANDLE(dev)));
  811. if (!has_acpi_companion(dev))
  812. return -ENODEV;
  813. handle = ACPI_HANDLE(dev);
  814. if (!handle)
  815. return -EINVAL;
  816. /* Walk through the tree for all PMU UNCORE devices */
  817. status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle, 1,
  818. tx2_uncore_pmu_add,
  819. NULL, dev, NULL);
  820. if (ACPI_FAILURE(status)) {
  821. dev_err(dev, "failed to probe PMU devices\n");
  822. return_ACPI_STATUS(status);
  823. }
  824. dev_info(dev, "node%d: pmu uncore registered\n", dev_to_node(dev));
  825. return 0;
  826. }
  827. static int tx2_uncore_remove(struct platform_device *pdev)
  828. {
  829. struct tx2_uncore_pmu *tx2_pmu, *temp;
  830. struct device *dev = &pdev->dev;
  831. if (!list_empty(&tx2_pmus)) {
  832. list_for_each_entry_safe(tx2_pmu, temp, &tx2_pmus, entry) {
  833. if (tx2_pmu->node == dev_to_node(dev)) {
  834. cpuhp_state_remove_instance_nocalls(
  835. CPUHP_AP_PERF_ARM_CAVIUM_TX2_UNCORE_ONLINE,
  836. &tx2_pmu->hpnode);
  837. perf_pmu_unregister(&tx2_pmu->pmu);
  838. list_del(&tx2_pmu->entry);
  839. }
  840. }
  841. }
  842. return 0;
  843. }
  844. static struct platform_driver tx2_uncore_driver = {
  845. .driver = {
  846. .name = "tx2-uncore-pmu",
  847. .acpi_match_table = ACPI_PTR(tx2_uncore_acpi_match),
  848. .suppress_bind_attrs = true,
  849. },
  850. .probe = tx2_uncore_probe,
  851. .remove = tx2_uncore_remove,
  852. };
  853. static int __init tx2_uncore_driver_init(void)
  854. {
  855. int ret;
  856. ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_CAVIUM_TX2_UNCORE_ONLINE,
  857. "perf/tx2/uncore:online",
  858. tx2_uncore_pmu_online_cpu,
  859. tx2_uncore_pmu_offline_cpu);
  860. if (ret) {
  861. pr_err("TX2 PMU: setup hotplug failed(%d)\n", ret);
  862. return ret;
  863. }
  864. ret = platform_driver_register(&tx2_uncore_driver);
  865. if (ret)
  866. cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_CAVIUM_TX2_UNCORE_ONLINE);
  867. return ret;
  868. }
  869. module_init(tx2_uncore_driver_init);
  870. static void __exit tx2_uncore_driver_exit(void)
  871. {
  872. platform_driver_unregister(&tx2_uncore_driver);
  873. cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_CAVIUM_TX2_UNCORE_ONLINE);
  874. }
  875. module_exit(tx2_uncore_driver_exit);
  876. MODULE_DESCRIPTION("ThunderX2 UNCORE PMU driver");
  877. MODULE_LICENSE("GPL v2");
  878. MODULE_AUTHOR("Ganapatrao Kulkarni <gkulkarni@cavium.com>");