arm-ccn.c 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. *
  4. * Copyright (C) 2014 ARM Limited
  5. */
  6. #include <linux/ctype.h>
  7. #include <linux/hrtimer.h>
  8. #include <linux/idr.h>
  9. #include <linux/interrupt.h>
  10. #include <linux/io.h>
  11. #include <linux/module.h>
  12. #include <linux/mod_devicetable.h>
  13. #include <linux/perf_event.h>
  14. #include <linux/platform_device.h>
  15. #include <linux/slab.h>
  16. #define CCN_NUM_XP_PORTS 2
  17. #define CCN_NUM_VCS 4
  18. #define CCN_NUM_REGIONS 256
  19. #define CCN_REGION_SIZE 0x10000
  20. #define CCN_ALL_OLY_ID 0xff00
  21. #define CCN_ALL_OLY_ID__OLY_ID__SHIFT 0
  22. #define CCN_ALL_OLY_ID__OLY_ID__MASK 0x1f
  23. #define CCN_ALL_OLY_ID__NODE_ID__SHIFT 8
  24. #define CCN_ALL_OLY_ID__NODE_ID__MASK 0x3f
  25. #define CCN_MN_ERRINT_STATUS 0x0008
  26. #define CCN_MN_ERRINT_STATUS__INTREQ__DESSERT 0x11
  27. #define CCN_MN_ERRINT_STATUS__ALL_ERRORS__ENABLE 0x02
  28. #define CCN_MN_ERRINT_STATUS__ALL_ERRORS__DISABLED 0x20
  29. #define CCN_MN_ERRINT_STATUS__ALL_ERRORS__DISABLE 0x22
  30. #define CCN_MN_ERRINT_STATUS__CORRECTED_ERRORS_ENABLE 0x04
  31. #define CCN_MN_ERRINT_STATUS__CORRECTED_ERRORS_DISABLED 0x40
  32. #define CCN_MN_ERRINT_STATUS__CORRECTED_ERRORS_DISABLE 0x44
  33. #define CCN_MN_ERRINT_STATUS__PMU_EVENTS__ENABLE 0x08
  34. #define CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLED 0x80
  35. #define CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLE 0x88
  36. #define CCN_MN_OLY_COMP_LIST_63_0 0x01e0
  37. #define CCN_MN_ERR_SIG_VAL_63_0 0x0300
  38. #define CCN_MN_ERR_SIG_VAL_63_0__DT (1 << 1)
  39. #define CCN_DT_ACTIVE_DSM 0x0000
  40. #define CCN_DT_ACTIVE_DSM__DSM_ID__SHIFT(n) ((n) * 8)
  41. #define CCN_DT_ACTIVE_DSM__DSM_ID__MASK 0xff
  42. #define CCN_DT_CTL 0x0028
  43. #define CCN_DT_CTL__DT_EN (1 << 0)
  44. #define CCN_DT_PMEVCNT(n) (0x0100 + (n) * 0x8)
  45. #define CCN_DT_PMCCNTR 0x0140
  46. #define CCN_DT_PMCCNTRSR 0x0190
  47. #define CCN_DT_PMOVSR 0x0198
  48. #define CCN_DT_PMOVSR_CLR 0x01a0
  49. #define CCN_DT_PMOVSR_CLR__MASK 0x1f
  50. #define CCN_DT_PMCR 0x01a8
  51. #define CCN_DT_PMCR__OVFL_INTR_EN (1 << 6)
  52. #define CCN_DT_PMCR__PMU_EN (1 << 0)
  53. #define CCN_DT_PMSR 0x01b0
  54. #define CCN_DT_PMSR_REQ 0x01b8
  55. #define CCN_DT_PMSR_CLR 0x01c0
  56. #define CCN_HNF_PMU_EVENT_SEL 0x0600
  57. #define CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 4)
  58. #define CCN_HNF_PMU_EVENT_SEL__ID__MASK 0xf
  59. #define CCN_XP_DT_CONFIG 0x0300
  60. #define CCN_XP_DT_CONFIG__DT_CFG__SHIFT(n) ((n) * 4)
  61. #define CCN_XP_DT_CONFIG__DT_CFG__MASK 0xf
  62. #define CCN_XP_DT_CONFIG__DT_CFG__PASS_THROUGH 0x0
  63. #define CCN_XP_DT_CONFIG__DT_CFG__WATCHPOINT_0_OR_1 0x1
  64. #define CCN_XP_DT_CONFIG__DT_CFG__WATCHPOINT(n) (0x2 + (n))
  65. #define CCN_XP_DT_CONFIG__DT_CFG__XP_PMU_EVENT(n) (0x4 + (n))
  66. #define CCN_XP_DT_CONFIG__DT_CFG__DEVICE_PMU_EVENT(d, n) (0x8 + (d) * 4 + (n))
  67. #define CCN_XP_DT_INTERFACE_SEL 0x0308
  68. #define CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__SHIFT(n) (0 + (n) * 8)
  69. #define CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__MASK 0x1
  70. #define CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__SHIFT(n) (1 + (n) * 8)
  71. #define CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__MASK 0x1
  72. #define CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__SHIFT(n) (2 + (n) * 8)
  73. #define CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__MASK 0x3
  74. #define CCN_XP_DT_CMP_VAL_L(n) (0x0310 + (n) * 0x40)
  75. #define CCN_XP_DT_CMP_VAL_H(n) (0x0318 + (n) * 0x40)
  76. #define CCN_XP_DT_CMP_MASK_L(n) (0x0320 + (n) * 0x40)
  77. #define CCN_XP_DT_CMP_MASK_H(n) (0x0328 + (n) * 0x40)
  78. #define CCN_XP_DT_CONTROL 0x0370
  79. #define CCN_XP_DT_CONTROL__DT_ENABLE (1 << 0)
  80. #define CCN_XP_DT_CONTROL__WP_ARM_SEL__SHIFT(n) (12 + (n) * 4)
  81. #define CCN_XP_DT_CONTROL__WP_ARM_SEL__MASK 0xf
  82. #define CCN_XP_DT_CONTROL__WP_ARM_SEL__ALWAYS 0xf
  83. #define CCN_XP_PMU_EVENT_SEL 0x0600
  84. #define CCN_XP_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 7)
  85. #define CCN_XP_PMU_EVENT_SEL__ID__MASK 0x3f
  86. #define CCN_SBAS_PMU_EVENT_SEL 0x0600
  87. #define CCN_SBAS_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 4)
  88. #define CCN_SBAS_PMU_EVENT_SEL__ID__MASK 0xf
  89. #define CCN_RNI_PMU_EVENT_SEL 0x0600
  90. #define CCN_RNI_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 4)
  91. #define CCN_RNI_PMU_EVENT_SEL__ID__MASK 0xf
  92. #define CCN_TYPE_MN 0x01
  93. #define CCN_TYPE_DT 0x02
  94. #define CCN_TYPE_HNF 0x04
  95. #define CCN_TYPE_HNI 0x05
  96. #define CCN_TYPE_XP 0x08
  97. #define CCN_TYPE_SBSX 0x0c
  98. #define CCN_TYPE_SBAS 0x10
  99. #define CCN_TYPE_RNI_1P 0x14
  100. #define CCN_TYPE_RNI_2P 0x15
  101. #define CCN_TYPE_RNI_3P 0x16
  102. #define CCN_TYPE_RND_1P 0x18 /* RN-D = RN-I + DVM */
  103. #define CCN_TYPE_RND_2P 0x19
  104. #define CCN_TYPE_RND_3P 0x1a
  105. #define CCN_TYPE_CYCLES 0xff /* Pseudotype */
  106. #define CCN_EVENT_WATCHPOINT 0xfe /* Pseudoevent */
  107. #define CCN_NUM_PMU_EVENTS 4
  108. #define CCN_NUM_XP_WATCHPOINTS 2 /* See DT.dbg_id.num_watchpoints */
  109. #define CCN_NUM_PMU_EVENT_COUNTERS 8 /* See DT.dbg_id.num_pmucntr */
  110. #define CCN_IDX_PMU_CYCLE_COUNTER CCN_NUM_PMU_EVENT_COUNTERS
  111. #define CCN_NUM_PREDEFINED_MASKS 4
  112. #define CCN_IDX_MASK_ANY (CCN_NUM_PMU_EVENT_COUNTERS + 0)
  113. #define CCN_IDX_MASK_EXACT (CCN_NUM_PMU_EVENT_COUNTERS + 1)
  114. #define CCN_IDX_MASK_ORDER (CCN_NUM_PMU_EVENT_COUNTERS + 2)
  115. #define CCN_IDX_MASK_OPCODE (CCN_NUM_PMU_EVENT_COUNTERS + 3)
  116. struct arm_ccn_component {
  117. void __iomem *base;
  118. u32 type;
  119. DECLARE_BITMAP(pmu_events_mask, CCN_NUM_PMU_EVENTS);
  120. union {
  121. struct {
  122. DECLARE_BITMAP(dt_cmp_mask, CCN_NUM_XP_WATCHPOINTS);
  123. } xp;
  124. };
  125. };
  126. #define pmu_to_arm_ccn(_pmu) container_of(container_of(_pmu, \
  127. struct arm_ccn_dt, pmu), struct arm_ccn, dt)
  128. struct arm_ccn_dt {
  129. int id;
  130. void __iomem *base;
  131. spinlock_t config_lock;
  132. DECLARE_BITMAP(pmu_counters_mask, CCN_NUM_PMU_EVENT_COUNTERS + 1);
  133. struct {
  134. struct arm_ccn_component *source;
  135. struct perf_event *event;
  136. } pmu_counters[CCN_NUM_PMU_EVENT_COUNTERS + 1];
  137. struct {
  138. u64 l, h;
  139. } cmp_mask[CCN_NUM_PMU_EVENT_COUNTERS + CCN_NUM_PREDEFINED_MASKS];
  140. struct hrtimer hrtimer;
  141. unsigned int cpu;
  142. struct hlist_node node;
  143. struct pmu pmu;
  144. };
  145. struct arm_ccn {
  146. struct device *dev;
  147. void __iomem *base;
  148. unsigned int irq;
  149. unsigned sbas_present:1;
  150. unsigned sbsx_present:1;
  151. int num_nodes;
  152. struct arm_ccn_component *node;
  153. int num_xps;
  154. struct arm_ccn_component *xp;
  155. struct arm_ccn_dt dt;
  156. int mn_id;
  157. };
  158. static int arm_ccn_node_to_xp(int node)
  159. {
  160. return node / CCN_NUM_XP_PORTS;
  161. }
  162. static int arm_ccn_node_to_xp_port(int node)
  163. {
  164. return node % CCN_NUM_XP_PORTS;
  165. }
  166. /*
  167. * Bit shifts and masks in these defines must be kept in sync with
  168. * arm_ccn_pmu_config_set() and CCN_FORMAT_ATTRs below!
  169. */
  170. #define CCN_CONFIG_NODE(_config) (((_config) >> 0) & 0xff)
  171. #define CCN_CONFIG_XP(_config) (((_config) >> 0) & 0xff)
  172. #define CCN_CONFIG_TYPE(_config) (((_config) >> 8) & 0xff)
  173. #define CCN_CONFIG_EVENT(_config) (((_config) >> 16) & 0xff)
  174. #define CCN_CONFIG_PORT(_config) (((_config) >> 24) & 0x3)
  175. #define CCN_CONFIG_BUS(_config) (((_config) >> 24) & 0x3)
  176. #define CCN_CONFIG_VC(_config) (((_config) >> 26) & 0x7)
  177. #define CCN_CONFIG_DIR(_config) (((_config) >> 29) & 0x1)
  178. #define CCN_CONFIG_MASK(_config) (((_config) >> 30) & 0xf)
  179. static void arm_ccn_pmu_config_set(u64 *config, u32 node_xp, u32 type, u32 port)
  180. {
  181. *config &= ~((0xff << 0) | (0xff << 8) | (0x3 << 24));
  182. *config |= (node_xp << 0) | (type << 8) | (port << 24);
  183. }
  184. static ssize_t arm_ccn_pmu_format_show(struct device *dev,
  185. struct device_attribute *attr, char *buf)
  186. {
  187. struct dev_ext_attribute *ea = container_of(attr,
  188. struct dev_ext_attribute, attr);
  189. return snprintf(buf, PAGE_SIZE, "%s\n", (char *)ea->var);
  190. }
  191. #define CCN_FORMAT_ATTR(_name, _config) \
  192. struct dev_ext_attribute arm_ccn_pmu_format_attr_##_name = \
  193. { __ATTR(_name, S_IRUGO, arm_ccn_pmu_format_show, \
  194. NULL), _config }
  195. static CCN_FORMAT_ATTR(node, "config:0-7");
  196. static CCN_FORMAT_ATTR(xp, "config:0-7");
  197. static CCN_FORMAT_ATTR(type, "config:8-15");
  198. static CCN_FORMAT_ATTR(event, "config:16-23");
  199. static CCN_FORMAT_ATTR(port, "config:24-25");
  200. static CCN_FORMAT_ATTR(bus, "config:24-25");
  201. static CCN_FORMAT_ATTR(vc, "config:26-28");
  202. static CCN_FORMAT_ATTR(dir, "config:29-29");
  203. static CCN_FORMAT_ATTR(mask, "config:30-33");
  204. static CCN_FORMAT_ATTR(cmp_l, "config1:0-62");
  205. static CCN_FORMAT_ATTR(cmp_h, "config2:0-59");
  206. static struct attribute *arm_ccn_pmu_format_attrs[] = {
  207. &arm_ccn_pmu_format_attr_node.attr.attr,
  208. &arm_ccn_pmu_format_attr_xp.attr.attr,
  209. &arm_ccn_pmu_format_attr_type.attr.attr,
  210. &arm_ccn_pmu_format_attr_event.attr.attr,
  211. &arm_ccn_pmu_format_attr_port.attr.attr,
  212. &arm_ccn_pmu_format_attr_bus.attr.attr,
  213. &arm_ccn_pmu_format_attr_vc.attr.attr,
  214. &arm_ccn_pmu_format_attr_dir.attr.attr,
  215. &arm_ccn_pmu_format_attr_mask.attr.attr,
  216. &arm_ccn_pmu_format_attr_cmp_l.attr.attr,
  217. &arm_ccn_pmu_format_attr_cmp_h.attr.attr,
  218. NULL
  219. };
  220. static const struct attribute_group arm_ccn_pmu_format_attr_group = {
  221. .name = "format",
  222. .attrs = arm_ccn_pmu_format_attrs,
  223. };
  224. struct arm_ccn_pmu_event {
  225. struct device_attribute attr;
  226. u32 type;
  227. u32 event;
  228. int num_ports;
  229. int num_vcs;
  230. const char *def;
  231. int mask;
  232. };
  233. #define CCN_EVENT_ATTR(_name) \
  234. __ATTR(_name, S_IRUGO, arm_ccn_pmu_event_show, NULL)
  235. /*
  236. * Events defined in TRM for MN, HN-I and SBSX are actually watchpoints set on
  237. * their ports in XP they are connected to. For the sake of usability they are
  238. * explicitly defined here (and translated into a relevant watchpoint in
  239. * arm_ccn_pmu_event_init()) so the user can easily request them without deep
  240. * knowledge of the flit format.
  241. */
  242. #define CCN_EVENT_MN(_name, _def, _mask) { .attr = CCN_EVENT_ATTR(mn_##_name), \
  243. .type = CCN_TYPE_MN, .event = CCN_EVENT_WATCHPOINT, \
  244. .num_ports = CCN_NUM_XP_PORTS, .num_vcs = CCN_NUM_VCS, \
  245. .def = _def, .mask = _mask, }
  246. #define CCN_EVENT_HNI(_name, _def, _mask) { \
  247. .attr = CCN_EVENT_ATTR(hni_##_name), .type = CCN_TYPE_HNI, \
  248. .event = CCN_EVENT_WATCHPOINT, .num_ports = CCN_NUM_XP_PORTS, \
  249. .num_vcs = CCN_NUM_VCS, .def = _def, .mask = _mask, }
  250. #define CCN_EVENT_SBSX(_name, _def, _mask) { \
  251. .attr = CCN_EVENT_ATTR(sbsx_##_name), .type = CCN_TYPE_SBSX, \
  252. .event = CCN_EVENT_WATCHPOINT, .num_ports = CCN_NUM_XP_PORTS, \
  253. .num_vcs = CCN_NUM_VCS, .def = _def, .mask = _mask, }
  254. #define CCN_EVENT_HNF(_name, _event) { .attr = CCN_EVENT_ATTR(hnf_##_name), \
  255. .type = CCN_TYPE_HNF, .event = _event, }
  256. #define CCN_EVENT_XP(_name, _event) { .attr = CCN_EVENT_ATTR(xp_##_name), \
  257. .type = CCN_TYPE_XP, .event = _event, \
  258. .num_ports = CCN_NUM_XP_PORTS, .num_vcs = CCN_NUM_VCS, }
  259. /*
  260. * RN-I & RN-D (RN-D = RN-I + DVM) nodes have different type ID depending
  261. * on configuration. One of them is picked to represent the whole group,
  262. * as they all share the same event types.
  263. */
  264. #define CCN_EVENT_RNI(_name, _event) { .attr = CCN_EVENT_ATTR(rni_##_name), \
  265. .type = CCN_TYPE_RNI_3P, .event = _event, }
  266. #define CCN_EVENT_SBAS(_name, _event) { .attr = CCN_EVENT_ATTR(sbas_##_name), \
  267. .type = CCN_TYPE_SBAS, .event = _event, }
  268. #define CCN_EVENT_CYCLES(_name) { .attr = CCN_EVENT_ATTR(_name), \
  269. .type = CCN_TYPE_CYCLES }
  270. static ssize_t arm_ccn_pmu_event_show(struct device *dev,
  271. struct device_attribute *attr, char *buf)
  272. {
  273. struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
  274. struct arm_ccn_pmu_event *event = container_of(attr,
  275. struct arm_ccn_pmu_event, attr);
  276. ssize_t res;
  277. res = scnprintf(buf, PAGE_SIZE, "type=0x%x", event->type);
  278. if (event->event)
  279. res += scnprintf(buf + res, PAGE_SIZE - res, ",event=0x%x",
  280. event->event);
  281. if (event->def)
  282. res += scnprintf(buf + res, PAGE_SIZE - res, ",%s",
  283. event->def);
  284. if (event->mask)
  285. res += scnprintf(buf + res, PAGE_SIZE - res, ",mask=0x%x",
  286. event->mask);
  287. /* Arguments required by an event */
  288. switch (event->type) {
  289. case CCN_TYPE_CYCLES:
  290. break;
  291. case CCN_TYPE_XP:
  292. res += scnprintf(buf + res, PAGE_SIZE - res,
  293. ",xp=?,vc=?");
  294. if (event->event == CCN_EVENT_WATCHPOINT)
  295. res += scnprintf(buf + res, PAGE_SIZE - res,
  296. ",port=?,dir=?,cmp_l=?,cmp_h=?,mask=?");
  297. else
  298. res += scnprintf(buf + res, PAGE_SIZE - res,
  299. ",bus=?");
  300. break;
  301. case CCN_TYPE_MN:
  302. res += scnprintf(buf + res, PAGE_SIZE - res, ",node=%d", ccn->mn_id);
  303. break;
  304. default:
  305. res += scnprintf(buf + res, PAGE_SIZE - res, ",node=?");
  306. break;
  307. }
  308. res += scnprintf(buf + res, PAGE_SIZE - res, "\n");
  309. return res;
  310. }
  311. static umode_t arm_ccn_pmu_events_is_visible(struct kobject *kobj,
  312. struct attribute *attr, int index)
  313. {
  314. struct device *dev = kobj_to_dev(kobj);
  315. struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
  316. struct device_attribute *dev_attr = container_of(attr,
  317. struct device_attribute, attr);
  318. struct arm_ccn_pmu_event *event = container_of(dev_attr,
  319. struct arm_ccn_pmu_event, attr);
  320. if (event->type == CCN_TYPE_SBAS && !ccn->sbas_present)
  321. return 0;
  322. if (event->type == CCN_TYPE_SBSX && !ccn->sbsx_present)
  323. return 0;
  324. return attr->mode;
  325. }
  326. static struct arm_ccn_pmu_event arm_ccn_pmu_events[] = {
  327. CCN_EVENT_MN(eobarrier, "dir=1,vc=0,cmp_h=0x1c00", CCN_IDX_MASK_OPCODE),
  328. CCN_EVENT_MN(ecbarrier, "dir=1,vc=0,cmp_h=0x1e00", CCN_IDX_MASK_OPCODE),
  329. CCN_EVENT_MN(dvmop, "dir=1,vc=0,cmp_h=0x2800", CCN_IDX_MASK_OPCODE),
  330. CCN_EVENT_HNI(txdatflits, "dir=1,vc=3", CCN_IDX_MASK_ANY),
  331. CCN_EVENT_HNI(rxdatflits, "dir=0,vc=3", CCN_IDX_MASK_ANY),
  332. CCN_EVENT_HNI(txreqflits, "dir=1,vc=0", CCN_IDX_MASK_ANY),
  333. CCN_EVENT_HNI(rxreqflits, "dir=0,vc=0", CCN_IDX_MASK_ANY),
  334. CCN_EVENT_HNI(rxreqflits_order, "dir=0,vc=0,cmp_h=0x8000",
  335. CCN_IDX_MASK_ORDER),
  336. CCN_EVENT_SBSX(txdatflits, "dir=1,vc=3", CCN_IDX_MASK_ANY),
  337. CCN_EVENT_SBSX(rxdatflits, "dir=0,vc=3", CCN_IDX_MASK_ANY),
  338. CCN_EVENT_SBSX(txreqflits, "dir=1,vc=0", CCN_IDX_MASK_ANY),
  339. CCN_EVENT_SBSX(rxreqflits, "dir=0,vc=0", CCN_IDX_MASK_ANY),
  340. CCN_EVENT_SBSX(rxreqflits_order, "dir=0,vc=0,cmp_h=0x8000",
  341. CCN_IDX_MASK_ORDER),
  342. CCN_EVENT_HNF(cache_miss, 0x1),
  343. CCN_EVENT_HNF(l3_sf_cache_access, 0x02),
  344. CCN_EVENT_HNF(cache_fill, 0x3),
  345. CCN_EVENT_HNF(pocq_retry, 0x4),
  346. CCN_EVENT_HNF(pocq_reqs_recvd, 0x5),
  347. CCN_EVENT_HNF(sf_hit, 0x6),
  348. CCN_EVENT_HNF(sf_evictions, 0x7),
  349. CCN_EVENT_HNF(snoops_sent, 0x8),
  350. CCN_EVENT_HNF(snoops_broadcast, 0x9),
  351. CCN_EVENT_HNF(l3_eviction, 0xa),
  352. CCN_EVENT_HNF(l3_fill_invalid_way, 0xb),
  353. CCN_EVENT_HNF(mc_retries, 0xc),
  354. CCN_EVENT_HNF(mc_reqs, 0xd),
  355. CCN_EVENT_HNF(qos_hh_retry, 0xe),
  356. CCN_EVENT_RNI(rdata_beats_p0, 0x1),
  357. CCN_EVENT_RNI(rdata_beats_p1, 0x2),
  358. CCN_EVENT_RNI(rdata_beats_p2, 0x3),
  359. CCN_EVENT_RNI(rxdat_flits, 0x4),
  360. CCN_EVENT_RNI(txdat_flits, 0x5),
  361. CCN_EVENT_RNI(txreq_flits, 0x6),
  362. CCN_EVENT_RNI(txreq_flits_retried, 0x7),
  363. CCN_EVENT_RNI(rrt_full, 0x8),
  364. CCN_EVENT_RNI(wrt_full, 0x9),
  365. CCN_EVENT_RNI(txreq_flits_replayed, 0xa),
  366. CCN_EVENT_XP(upload_starvation, 0x1),
  367. CCN_EVENT_XP(download_starvation, 0x2),
  368. CCN_EVENT_XP(respin, 0x3),
  369. CCN_EVENT_XP(valid_flit, 0x4),
  370. CCN_EVENT_XP(watchpoint, CCN_EVENT_WATCHPOINT),
  371. CCN_EVENT_SBAS(rdata_beats_p0, 0x1),
  372. CCN_EVENT_SBAS(rxdat_flits, 0x4),
  373. CCN_EVENT_SBAS(txdat_flits, 0x5),
  374. CCN_EVENT_SBAS(txreq_flits, 0x6),
  375. CCN_EVENT_SBAS(txreq_flits_retried, 0x7),
  376. CCN_EVENT_SBAS(rrt_full, 0x8),
  377. CCN_EVENT_SBAS(wrt_full, 0x9),
  378. CCN_EVENT_SBAS(txreq_flits_replayed, 0xa),
  379. CCN_EVENT_CYCLES(cycles),
  380. };
  381. /* Populated in arm_ccn_init() */
  382. static struct attribute
  383. *arm_ccn_pmu_events_attrs[ARRAY_SIZE(arm_ccn_pmu_events) + 1];
  384. static const struct attribute_group arm_ccn_pmu_events_attr_group = {
  385. .name = "events",
  386. .is_visible = arm_ccn_pmu_events_is_visible,
  387. .attrs = arm_ccn_pmu_events_attrs,
  388. };
  389. static u64 *arm_ccn_pmu_get_cmp_mask(struct arm_ccn *ccn, const char *name)
  390. {
  391. unsigned long i;
  392. if (WARN_ON(!name || !name[0] || !isxdigit(name[0]) || !name[1]))
  393. return NULL;
  394. i = isdigit(name[0]) ? name[0] - '0' : 0xa + tolower(name[0]) - 'a';
  395. switch (name[1]) {
  396. case 'l':
  397. return &ccn->dt.cmp_mask[i].l;
  398. case 'h':
  399. return &ccn->dt.cmp_mask[i].h;
  400. default:
  401. return NULL;
  402. }
  403. }
  404. static ssize_t arm_ccn_pmu_cmp_mask_show(struct device *dev,
  405. struct device_attribute *attr, char *buf)
  406. {
  407. struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
  408. u64 *mask = arm_ccn_pmu_get_cmp_mask(ccn, attr->attr.name);
  409. return mask ? snprintf(buf, PAGE_SIZE, "0x%016llx\n", *mask) : -EINVAL;
  410. }
  411. static ssize_t arm_ccn_pmu_cmp_mask_store(struct device *dev,
  412. struct device_attribute *attr, const char *buf, size_t count)
  413. {
  414. struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
  415. u64 *mask = arm_ccn_pmu_get_cmp_mask(ccn, attr->attr.name);
  416. int err = -EINVAL;
  417. if (mask)
  418. err = kstrtoull(buf, 0, mask);
  419. return err ? err : count;
  420. }
  421. #define CCN_CMP_MASK_ATTR(_name) \
  422. struct device_attribute arm_ccn_pmu_cmp_mask_attr_##_name = \
  423. __ATTR(_name, S_IRUGO | S_IWUSR, \
  424. arm_ccn_pmu_cmp_mask_show, arm_ccn_pmu_cmp_mask_store)
  425. #define CCN_CMP_MASK_ATTR_RO(_name) \
  426. struct device_attribute arm_ccn_pmu_cmp_mask_attr_##_name = \
  427. __ATTR(_name, S_IRUGO, arm_ccn_pmu_cmp_mask_show, NULL)
  428. static CCN_CMP_MASK_ATTR(0l);
  429. static CCN_CMP_MASK_ATTR(0h);
  430. static CCN_CMP_MASK_ATTR(1l);
  431. static CCN_CMP_MASK_ATTR(1h);
  432. static CCN_CMP_MASK_ATTR(2l);
  433. static CCN_CMP_MASK_ATTR(2h);
  434. static CCN_CMP_MASK_ATTR(3l);
  435. static CCN_CMP_MASK_ATTR(3h);
  436. static CCN_CMP_MASK_ATTR(4l);
  437. static CCN_CMP_MASK_ATTR(4h);
  438. static CCN_CMP_MASK_ATTR(5l);
  439. static CCN_CMP_MASK_ATTR(5h);
  440. static CCN_CMP_MASK_ATTR(6l);
  441. static CCN_CMP_MASK_ATTR(6h);
  442. static CCN_CMP_MASK_ATTR(7l);
  443. static CCN_CMP_MASK_ATTR(7h);
  444. static CCN_CMP_MASK_ATTR_RO(8l);
  445. static CCN_CMP_MASK_ATTR_RO(8h);
  446. static CCN_CMP_MASK_ATTR_RO(9l);
  447. static CCN_CMP_MASK_ATTR_RO(9h);
  448. static CCN_CMP_MASK_ATTR_RO(al);
  449. static CCN_CMP_MASK_ATTR_RO(ah);
  450. static CCN_CMP_MASK_ATTR_RO(bl);
  451. static CCN_CMP_MASK_ATTR_RO(bh);
  452. static struct attribute *arm_ccn_pmu_cmp_mask_attrs[] = {
  453. &arm_ccn_pmu_cmp_mask_attr_0l.attr, &arm_ccn_pmu_cmp_mask_attr_0h.attr,
  454. &arm_ccn_pmu_cmp_mask_attr_1l.attr, &arm_ccn_pmu_cmp_mask_attr_1h.attr,
  455. &arm_ccn_pmu_cmp_mask_attr_2l.attr, &arm_ccn_pmu_cmp_mask_attr_2h.attr,
  456. &arm_ccn_pmu_cmp_mask_attr_3l.attr, &arm_ccn_pmu_cmp_mask_attr_3h.attr,
  457. &arm_ccn_pmu_cmp_mask_attr_4l.attr, &arm_ccn_pmu_cmp_mask_attr_4h.attr,
  458. &arm_ccn_pmu_cmp_mask_attr_5l.attr, &arm_ccn_pmu_cmp_mask_attr_5h.attr,
  459. &arm_ccn_pmu_cmp_mask_attr_6l.attr, &arm_ccn_pmu_cmp_mask_attr_6h.attr,
  460. &arm_ccn_pmu_cmp_mask_attr_7l.attr, &arm_ccn_pmu_cmp_mask_attr_7h.attr,
  461. &arm_ccn_pmu_cmp_mask_attr_8l.attr, &arm_ccn_pmu_cmp_mask_attr_8h.attr,
  462. &arm_ccn_pmu_cmp_mask_attr_9l.attr, &arm_ccn_pmu_cmp_mask_attr_9h.attr,
  463. &arm_ccn_pmu_cmp_mask_attr_al.attr, &arm_ccn_pmu_cmp_mask_attr_ah.attr,
  464. &arm_ccn_pmu_cmp_mask_attr_bl.attr, &arm_ccn_pmu_cmp_mask_attr_bh.attr,
  465. NULL
  466. };
  467. static const struct attribute_group arm_ccn_pmu_cmp_mask_attr_group = {
  468. .name = "cmp_mask",
  469. .attrs = arm_ccn_pmu_cmp_mask_attrs,
  470. };
  471. static ssize_t arm_ccn_pmu_cpumask_show(struct device *dev,
  472. struct device_attribute *attr, char *buf)
  473. {
  474. struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
  475. return cpumap_print_to_pagebuf(true, buf, cpumask_of(ccn->dt.cpu));
  476. }
  477. static struct device_attribute arm_ccn_pmu_cpumask_attr =
  478. __ATTR(cpumask, S_IRUGO, arm_ccn_pmu_cpumask_show, NULL);
  479. static struct attribute *arm_ccn_pmu_cpumask_attrs[] = {
  480. &arm_ccn_pmu_cpumask_attr.attr,
  481. NULL,
  482. };
  483. static const struct attribute_group arm_ccn_pmu_cpumask_attr_group = {
  484. .attrs = arm_ccn_pmu_cpumask_attrs,
  485. };
  486. /*
  487. * Default poll period is 10ms, which is way over the top anyway,
  488. * as in the worst case scenario (an event every cycle), with 1GHz
  489. * clocked bus, the smallest, 32 bit counter will overflow in
  490. * more than 4s.
  491. */
  492. static unsigned int arm_ccn_pmu_poll_period_us = 10000;
  493. module_param_named(pmu_poll_period_us, arm_ccn_pmu_poll_period_us, uint,
  494. S_IRUGO | S_IWUSR);
  495. static ktime_t arm_ccn_pmu_timer_period(void)
  496. {
  497. return ns_to_ktime((u64)arm_ccn_pmu_poll_period_us * 1000);
  498. }
  499. static const struct attribute_group *arm_ccn_pmu_attr_groups[] = {
  500. &arm_ccn_pmu_events_attr_group,
  501. &arm_ccn_pmu_format_attr_group,
  502. &arm_ccn_pmu_cmp_mask_attr_group,
  503. &arm_ccn_pmu_cpumask_attr_group,
  504. NULL
  505. };
  506. static int arm_ccn_pmu_alloc_bit(unsigned long *bitmap, unsigned long size)
  507. {
  508. int bit;
  509. do {
  510. bit = find_first_zero_bit(bitmap, size);
  511. if (bit >= size)
  512. return -EAGAIN;
  513. } while (test_and_set_bit(bit, bitmap));
  514. return bit;
  515. }
  516. /* All RN-I and RN-D nodes have identical PMUs */
  517. static int arm_ccn_pmu_type_eq(u32 a, u32 b)
  518. {
  519. if (a == b)
  520. return 1;
  521. switch (a) {
  522. case CCN_TYPE_RNI_1P:
  523. case CCN_TYPE_RNI_2P:
  524. case CCN_TYPE_RNI_3P:
  525. case CCN_TYPE_RND_1P:
  526. case CCN_TYPE_RND_2P:
  527. case CCN_TYPE_RND_3P:
  528. switch (b) {
  529. case CCN_TYPE_RNI_1P:
  530. case CCN_TYPE_RNI_2P:
  531. case CCN_TYPE_RNI_3P:
  532. case CCN_TYPE_RND_1P:
  533. case CCN_TYPE_RND_2P:
  534. case CCN_TYPE_RND_3P:
  535. return 1;
  536. }
  537. break;
  538. }
  539. return 0;
  540. }
  541. static int arm_ccn_pmu_event_alloc(struct perf_event *event)
  542. {
  543. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  544. struct hw_perf_event *hw = &event->hw;
  545. u32 node_xp, type, event_id;
  546. struct arm_ccn_component *source;
  547. int bit;
  548. node_xp = CCN_CONFIG_NODE(event->attr.config);
  549. type = CCN_CONFIG_TYPE(event->attr.config);
  550. event_id = CCN_CONFIG_EVENT(event->attr.config);
  551. /* Allocate the cycle counter */
  552. if (type == CCN_TYPE_CYCLES) {
  553. if (test_and_set_bit(CCN_IDX_PMU_CYCLE_COUNTER,
  554. ccn->dt.pmu_counters_mask))
  555. return -EAGAIN;
  556. hw->idx = CCN_IDX_PMU_CYCLE_COUNTER;
  557. ccn->dt.pmu_counters[CCN_IDX_PMU_CYCLE_COUNTER].event = event;
  558. return 0;
  559. }
  560. /* Allocate an event counter */
  561. hw->idx = arm_ccn_pmu_alloc_bit(ccn->dt.pmu_counters_mask,
  562. CCN_NUM_PMU_EVENT_COUNTERS);
  563. if (hw->idx < 0) {
  564. dev_dbg(ccn->dev, "No more counters available!\n");
  565. return -EAGAIN;
  566. }
  567. if (type == CCN_TYPE_XP)
  568. source = &ccn->xp[node_xp];
  569. else
  570. source = &ccn->node[node_xp];
  571. ccn->dt.pmu_counters[hw->idx].source = source;
  572. /* Allocate an event source or a watchpoint */
  573. if (type == CCN_TYPE_XP && event_id == CCN_EVENT_WATCHPOINT)
  574. bit = arm_ccn_pmu_alloc_bit(source->xp.dt_cmp_mask,
  575. CCN_NUM_XP_WATCHPOINTS);
  576. else
  577. bit = arm_ccn_pmu_alloc_bit(source->pmu_events_mask,
  578. CCN_NUM_PMU_EVENTS);
  579. if (bit < 0) {
  580. dev_dbg(ccn->dev, "No more event sources/watchpoints on node/XP %d!\n",
  581. node_xp);
  582. clear_bit(hw->idx, ccn->dt.pmu_counters_mask);
  583. return -EAGAIN;
  584. }
  585. hw->config_base = bit;
  586. ccn->dt.pmu_counters[hw->idx].event = event;
  587. return 0;
  588. }
  589. static void arm_ccn_pmu_event_release(struct perf_event *event)
  590. {
  591. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  592. struct hw_perf_event *hw = &event->hw;
  593. if (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER) {
  594. clear_bit(CCN_IDX_PMU_CYCLE_COUNTER, ccn->dt.pmu_counters_mask);
  595. } else {
  596. struct arm_ccn_component *source =
  597. ccn->dt.pmu_counters[hw->idx].source;
  598. if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP &&
  599. CCN_CONFIG_EVENT(event->attr.config) ==
  600. CCN_EVENT_WATCHPOINT)
  601. clear_bit(hw->config_base, source->xp.dt_cmp_mask);
  602. else
  603. clear_bit(hw->config_base, source->pmu_events_mask);
  604. clear_bit(hw->idx, ccn->dt.pmu_counters_mask);
  605. }
  606. ccn->dt.pmu_counters[hw->idx].source = NULL;
  607. ccn->dt.pmu_counters[hw->idx].event = NULL;
  608. }
  609. static int arm_ccn_pmu_event_init(struct perf_event *event)
  610. {
  611. struct arm_ccn *ccn;
  612. struct hw_perf_event *hw = &event->hw;
  613. u32 node_xp, type, event_id;
  614. int valid;
  615. int i;
  616. struct perf_event *sibling;
  617. if (event->attr.type != event->pmu->type)
  618. return -ENOENT;
  619. ccn = pmu_to_arm_ccn(event->pmu);
  620. if (hw->sample_period) {
  621. dev_dbg(ccn->dev, "Sampling not supported!\n");
  622. return -EOPNOTSUPP;
  623. }
  624. if (has_branch_stack(event)) {
  625. dev_dbg(ccn->dev, "Can't exclude execution levels!\n");
  626. return -EINVAL;
  627. }
  628. if (event->cpu < 0) {
  629. dev_dbg(ccn->dev, "Can't provide per-task data!\n");
  630. return -EOPNOTSUPP;
  631. }
  632. /*
  633. * Many perf core operations (eg. events rotation) operate on a
  634. * single CPU context. This is obvious for CPU PMUs, where one
  635. * expects the same sets of events being observed on all CPUs,
  636. * but can lead to issues for off-core PMUs, like CCN, where each
  637. * event could be theoretically assigned to a different CPU. To
  638. * mitigate this, we enforce CPU assignment to one, selected
  639. * processor (the one described in the "cpumask" attribute).
  640. */
  641. event->cpu = ccn->dt.cpu;
  642. node_xp = CCN_CONFIG_NODE(event->attr.config);
  643. type = CCN_CONFIG_TYPE(event->attr.config);
  644. event_id = CCN_CONFIG_EVENT(event->attr.config);
  645. /* Validate node/xp vs topology */
  646. switch (type) {
  647. case CCN_TYPE_MN:
  648. if (node_xp != ccn->mn_id) {
  649. dev_dbg(ccn->dev, "Invalid MN ID %d!\n", node_xp);
  650. return -EINVAL;
  651. }
  652. break;
  653. case CCN_TYPE_XP:
  654. if (node_xp >= ccn->num_xps) {
  655. dev_dbg(ccn->dev, "Invalid XP ID %d!\n", node_xp);
  656. return -EINVAL;
  657. }
  658. break;
  659. case CCN_TYPE_CYCLES:
  660. break;
  661. default:
  662. if (node_xp >= ccn->num_nodes) {
  663. dev_dbg(ccn->dev, "Invalid node ID %d!\n", node_xp);
  664. return -EINVAL;
  665. }
  666. if (!arm_ccn_pmu_type_eq(type, ccn->node[node_xp].type)) {
  667. dev_dbg(ccn->dev, "Invalid type 0x%x for node %d!\n",
  668. type, node_xp);
  669. return -EINVAL;
  670. }
  671. break;
  672. }
  673. /* Validate event ID vs available for the type */
  674. for (i = 0, valid = 0; i < ARRAY_SIZE(arm_ccn_pmu_events) && !valid;
  675. i++) {
  676. struct arm_ccn_pmu_event *e = &arm_ccn_pmu_events[i];
  677. u32 port = CCN_CONFIG_PORT(event->attr.config);
  678. u32 vc = CCN_CONFIG_VC(event->attr.config);
  679. if (!arm_ccn_pmu_type_eq(type, e->type))
  680. continue;
  681. if (event_id != e->event)
  682. continue;
  683. if (e->num_ports && port >= e->num_ports) {
  684. dev_dbg(ccn->dev, "Invalid port %d for node/XP %d!\n",
  685. port, node_xp);
  686. return -EINVAL;
  687. }
  688. if (e->num_vcs && vc >= e->num_vcs) {
  689. dev_dbg(ccn->dev, "Invalid vc %d for node/XP %d!\n",
  690. vc, node_xp);
  691. return -EINVAL;
  692. }
  693. valid = 1;
  694. }
  695. if (!valid) {
  696. dev_dbg(ccn->dev, "Invalid event 0x%x for node/XP %d!\n",
  697. event_id, node_xp);
  698. return -EINVAL;
  699. }
  700. /* Watchpoint-based event for a node is actually set on XP */
  701. if (event_id == CCN_EVENT_WATCHPOINT && type != CCN_TYPE_XP) {
  702. u32 port;
  703. type = CCN_TYPE_XP;
  704. port = arm_ccn_node_to_xp_port(node_xp);
  705. node_xp = arm_ccn_node_to_xp(node_xp);
  706. arm_ccn_pmu_config_set(&event->attr.config,
  707. node_xp, type, port);
  708. }
  709. /*
  710. * We must NOT create groups containing mixed PMUs, although software
  711. * events are acceptable (for example to create a CCN group
  712. * periodically read when a hrtimer aka cpu-clock leader triggers).
  713. */
  714. if (event->group_leader->pmu != event->pmu &&
  715. !is_software_event(event->group_leader))
  716. return -EINVAL;
  717. for_each_sibling_event(sibling, event->group_leader) {
  718. if (sibling->pmu != event->pmu &&
  719. !is_software_event(sibling))
  720. return -EINVAL;
  721. }
  722. return 0;
  723. }
  724. static u64 arm_ccn_pmu_read_counter(struct arm_ccn *ccn, int idx)
  725. {
  726. u64 res;
  727. if (idx == CCN_IDX_PMU_CYCLE_COUNTER) {
  728. #ifdef readq
  729. res = readq(ccn->dt.base + CCN_DT_PMCCNTR);
  730. #else
  731. /* 40 bit counter, can do snapshot and read in two parts */
  732. writel(0x1, ccn->dt.base + CCN_DT_PMSR_REQ);
  733. while (!(readl(ccn->dt.base + CCN_DT_PMSR) & 0x1))
  734. ;
  735. writel(0x1, ccn->dt.base + CCN_DT_PMSR_CLR);
  736. res = readl(ccn->dt.base + CCN_DT_PMCCNTRSR + 4) & 0xff;
  737. res <<= 32;
  738. res |= readl(ccn->dt.base + CCN_DT_PMCCNTRSR);
  739. #endif
  740. } else {
  741. res = readl(ccn->dt.base + CCN_DT_PMEVCNT(idx));
  742. }
  743. return res;
  744. }
  745. static void arm_ccn_pmu_event_update(struct perf_event *event)
  746. {
  747. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  748. struct hw_perf_event *hw = &event->hw;
  749. u64 prev_count, new_count, mask;
  750. do {
  751. prev_count = local64_read(&hw->prev_count);
  752. new_count = arm_ccn_pmu_read_counter(ccn, hw->idx);
  753. } while (local64_xchg(&hw->prev_count, new_count) != prev_count);
  754. mask = (1LLU << (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER ? 40 : 32)) - 1;
  755. local64_add((new_count - prev_count) & mask, &event->count);
  756. }
  757. static void arm_ccn_pmu_xp_dt_config(struct perf_event *event, int enable)
  758. {
  759. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  760. struct hw_perf_event *hw = &event->hw;
  761. struct arm_ccn_component *xp;
  762. u32 val, dt_cfg;
  763. /* Nothing to do for cycle counter */
  764. if (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER)
  765. return;
  766. if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP)
  767. xp = &ccn->xp[CCN_CONFIG_XP(event->attr.config)];
  768. else
  769. xp = &ccn->xp[arm_ccn_node_to_xp(
  770. CCN_CONFIG_NODE(event->attr.config))];
  771. if (enable)
  772. dt_cfg = hw->event_base;
  773. else
  774. dt_cfg = CCN_XP_DT_CONFIG__DT_CFG__PASS_THROUGH;
  775. spin_lock(&ccn->dt.config_lock);
  776. val = readl(xp->base + CCN_XP_DT_CONFIG);
  777. val &= ~(CCN_XP_DT_CONFIG__DT_CFG__MASK <<
  778. CCN_XP_DT_CONFIG__DT_CFG__SHIFT(hw->idx));
  779. val |= dt_cfg << CCN_XP_DT_CONFIG__DT_CFG__SHIFT(hw->idx);
  780. writel(val, xp->base + CCN_XP_DT_CONFIG);
  781. spin_unlock(&ccn->dt.config_lock);
  782. }
  783. static void arm_ccn_pmu_event_start(struct perf_event *event, int flags)
  784. {
  785. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  786. struct hw_perf_event *hw = &event->hw;
  787. local64_set(&event->hw.prev_count,
  788. arm_ccn_pmu_read_counter(ccn, hw->idx));
  789. hw->state = 0;
  790. /* Set the DT bus input, engaging the counter */
  791. arm_ccn_pmu_xp_dt_config(event, 1);
  792. }
  793. static void arm_ccn_pmu_event_stop(struct perf_event *event, int flags)
  794. {
  795. struct hw_perf_event *hw = &event->hw;
  796. /* Disable counting, setting the DT bus to pass-through mode */
  797. arm_ccn_pmu_xp_dt_config(event, 0);
  798. if (flags & PERF_EF_UPDATE)
  799. arm_ccn_pmu_event_update(event);
  800. hw->state |= PERF_HES_STOPPED;
  801. }
  802. static void arm_ccn_pmu_xp_watchpoint_config(struct perf_event *event)
  803. {
  804. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  805. struct hw_perf_event *hw = &event->hw;
  806. struct arm_ccn_component *source =
  807. ccn->dt.pmu_counters[hw->idx].source;
  808. unsigned long wp = hw->config_base;
  809. u32 val;
  810. u64 cmp_l = event->attr.config1;
  811. u64 cmp_h = event->attr.config2;
  812. u64 mask_l = ccn->dt.cmp_mask[CCN_CONFIG_MASK(event->attr.config)].l;
  813. u64 mask_h = ccn->dt.cmp_mask[CCN_CONFIG_MASK(event->attr.config)].h;
  814. hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__WATCHPOINT(wp);
  815. /* Direction (RX/TX), device (port) & virtual channel */
  816. val = readl(source->base + CCN_XP_DT_INTERFACE_SEL);
  817. val &= ~(CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__MASK <<
  818. CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__SHIFT(wp));
  819. val |= CCN_CONFIG_DIR(event->attr.config) <<
  820. CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__SHIFT(wp);
  821. val &= ~(CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__MASK <<
  822. CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__SHIFT(wp));
  823. val |= CCN_CONFIG_PORT(event->attr.config) <<
  824. CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__SHIFT(wp);
  825. val &= ~(CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__MASK <<
  826. CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__SHIFT(wp));
  827. val |= CCN_CONFIG_VC(event->attr.config) <<
  828. CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__SHIFT(wp);
  829. writel(val, source->base + CCN_XP_DT_INTERFACE_SEL);
  830. /* Comparison values */
  831. writel(cmp_l & 0xffffffff, source->base + CCN_XP_DT_CMP_VAL_L(wp));
  832. writel((cmp_l >> 32) & 0x7fffffff,
  833. source->base + CCN_XP_DT_CMP_VAL_L(wp) + 4);
  834. writel(cmp_h & 0xffffffff, source->base + CCN_XP_DT_CMP_VAL_H(wp));
  835. writel((cmp_h >> 32) & 0x0fffffff,
  836. source->base + CCN_XP_DT_CMP_VAL_H(wp) + 4);
  837. /* Mask */
  838. writel(mask_l & 0xffffffff, source->base + CCN_XP_DT_CMP_MASK_L(wp));
  839. writel((mask_l >> 32) & 0x7fffffff,
  840. source->base + CCN_XP_DT_CMP_MASK_L(wp) + 4);
  841. writel(mask_h & 0xffffffff, source->base + CCN_XP_DT_CMP_MASK_H(wp));
  842. writel((mask_h >> 32) & 0x0fffffff,
  843. source->base + CCN_XP_DT_CMP_MASK_H(wp) + 4);
  844. }
  845. static void arm_ccn_pmu_xp_event_config(struct perf_event *event)
  846. {
  847. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  848. struct hw_perf_event *hw = &event->hw;
  849. struct arm_ccn_component *source =
  850. ccn->dt.pmu_counters[hw->idx].source;
  851. u32 val, id;
  852. hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__XP_PMU_EVENT(hw->config_base);
  853. id = (CCN_CONFIG_VC(event->attr.config) << 4) |
  854. (CCN_CONFIG_BUS(event->attr.config) << 3) |
  855. (CCN_CONFIG_EVENT(event->attr.config) << 0);
  856. val = readl(source->base + CCN_XP_PMU_EVENT_SEL);
  857. val &= ~(CCN_XP_PMU_EVENT_SEL__ID__MASK <<
  858. CCN_XP_PMU_EVENT_SEL__ID__SHIFT(hw->config_base));
  859. val |= id << CCN_XP_PMU_EVENT_SEL__ID__SHIFT(hw->config_base);
  860. writel(val, source->base + CCN_XP_PMU_EVENT_SEL);
  861. }
  862. static void arm_ccn_pmu_node_event_config(struct perf_event *event)
  863. {
  864. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  865. struct hw_perf_event *hw = &event->hw;
  866. struct arm_ccn_component *source =
  867. ccn->dt.pmu_counters[hw->idx].source;
  868. u32 type = CCN_CONFIG_TYPE(event->attr.config);
  869. u32 val, port;
  870. port = arm_ccn_node_to_xp_port(CCN_CONFIG_NODE(event->attr.config));
  871. hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__DEVICE_PMU_EVENT(port,
  872. hw->config_base);
  873. /* These *_event_sel regs should be identical, but let's make sure... */
  874. BUILD_BUG_ON(CCN_HNF_PMU_EVENT_SEL != CCN_SBAS_PMU_EVENT_SEL);
  875. BUILD_BUG_ON(CCN_SBAS_PMU_EVENT_SEL != CCN_RNI_PMU_EVENT_SEL);
  876. BUILD_BUG_ON(CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(1) !=
  877. CCN_SBAS_PMU_EVENT_SEL__ID__SHIFT(1));
  878. BUILD_BUG_ON(CCN_SBAS_PMU_EVENT_SEL__ID__SHIFT(1) !=
  879. CCN_RNI_PMU_EVENT_SEL__ID__SHIFT(1));
  880. BUILD_BUG_ON(CCN_HNF_PMU_EVENT_SEL__ID__MASK !=
  881. CCN_SBAS_PMU_EVENT_SEL__ID__MASK);
  882. BUILD_BUG_ON(CCN_SBAS_PMU_EVENT_SEL__ID__MASK !=
  883. CCN_RNI_PMU_EVENT_SEL__ID__MASK);
  884. if (WARN_ON(type != CCN_TYPE_HNF && type != CCN_TYPE_SBAS &&
  885. !arm_ccn_pmu_type_eq(type, CCN_TYPE_RNI_3P)))
  886. return;
  887. /* Set the event id for the pre-allocated counter */
  888. val = readl(source->base + CCN_HNF_PMU_EVENT_SEL);
  889. val &= ~(CCN_HNF_PMU_EVENT_SEL__ID__MASK <<
  890. CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(hw->config_base));
  891. val |= CCN_CONFIG_EVENT(event->attr.config) <<
  892. CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(hw->config_base);
  893. writel(val, source->base + CCN_HNF_PMU_EVENT_SEL);
  894. }
  895. static void arm_ccn_pmu_event_config(struct perf_event *event)
  896. {
  897. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  898. struct hw_perf_event *hw = &event->hw;
  899. u32 xp, offset, val;
  900. /* Cycle counter requires no setup */
  901. if (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER)
  902. return;
  903. if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP)
  904. xp = CCN_CONFIG_XP(event->attr.config);
  905. else
  906. xp = arm_ccn_node_to_xp(CCN_CONFIG_NODE(event->attr.config));
  907. spin_lock(&ccn->dt.config_lock);
  908. /* Set the DT bus "distance" register */
  909. offset = (hw->idx / 4) * 4;
  910. val = readl(ccn->dt.base + CCN_DT_ACTIVE_DSM + offset);
  911. val &= ~(CCN_DT_ACTIVE_DSM__DSM_ID__MASK <<
  912. CCN_DT_ACTIVE_DSM__DSM_ID__SHIFT(hw->idx % 4));
  913. val |= xp << CCN_DT_ACTIVE_DSM__DSM_ID__SHIFT(hw->idx % 4);
  914. writel(val, ccn->dt.base + CCN_DT_ACTIVE_DSM + offset);
  915. if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP) {
  916. if (CCN_CONFIG_EVENT(event->attr.config) ==
  917. CCN_EVENT_WATCHPOINT)
  918. arm_ccn_pmu_xp_watchpoint_config(event);
  919. else
  920. arm_ccn_pmu_xp_event_config(event);
  921. } else {
  922. arm_ccn_pmu_node_event_config(event);
  923. }
  924. spin_unlock(&ccn->dt.config_lock);
  925. }
  926. static int arm_ccn_pmu_active_counters(struct arm_ccn *ccn)
  927. {
  928. return bitmap_weight(ccn->dt.pmu_counters_mask,
  929. CCN_NUM_PMU_EVENT_COUNTERS + 1);
  930. }
  931. static int arm_ccn_pmu_event_add(struct perf_event *event, int flags)
  932. {
  933. int err;
  934. struct hw_perf_event *hw = &event->hw;
  935. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  936. err = arm_ccn_pmu_event_alloc(event);
  937. if (err)
  938. return err;
  939. /*
  940. * Pin the timer, so that the overflows are handled by the chosen
  941. * event->cpu (this is the same one as presented in "cpumask"
  942. * attribute).
  943. */
  944. if (!ccn->irq && arm_ccn_pmu_active_counters(ccn) == 1)
  945. hrtimer_start(&ccn->dt.hrtimer, arm_ccn_pmu_timer_period(),
  946. HRTIMER_MODE_REL_PINNED);
  947. arm_ccn_pmu_event_config(event);
  948. hw->state = PERF_HES_STOPPED;
  949. if (flags & PERF_EF_START)
  950. arm_ccn_pmu_event_start(event, PERF_EF_UPDATE);
  951. return 0;
  952. }
  953. static void arm_ccn_pmu_event_del(struct perf_event *event, int flags)
  954. {
  955. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  956. arm_ccn_pmu_event_stop(event, PERF_EF_UPDATE);
  957. arm_ccn_pmu_event_release(event);
  958. if (!ccn->irq && arm_ccn_pmu_active_counters(ccn) == 0)
  959. hrtimer_cancel(&ccn->dt.hrtimer);
  960. }
  961. static void arm_ccn_pmu_event_read(struct perf_event *event)
  962. {
  963. arm_ccn_pmu_event_update(event);
  964. }
  965. static void arm_ccn_pmu_enable(struct pmu *pmu)
  966. {
  967. struct arm_ccn *ccn = pmu_to_arm_ccn(pmu);
  968. u32 val = readl(ccn->dt.base + CCN_DT_PMCR);
  969. val |= CCN_DT_PMCR__PMU_EN;
  970. writel(val, ccn->dt.base + CCN_DT_PMCR);
  971. }
  972. static void arm_ccn_pmu_disable(struct pmu *pmu)
  973. {
  974. struct arm_ccn *ccn = pmu_to_arm_ccn(pmu);
  975. u32 val = readl(ccn->dt.base + CCN_DT_PMCR);
  976. val &= ~CCN_DT_PMCR__PMU_EN;
  977. writel(val, ccn->dt.base + CCN_DT_PMCR);
  978. }
  979. static irqreturn_t arm_ccn_pmu_overflow_handler(struct arm_ccn_dt *dt)
  980. {
  981. u32 pmovsr = readl(dt->base + CCN_DT_PMOVSR);
  982. int idx;
  983. if (!pmovsr)
  984. return IRQ_NONE;
  985. writel(pmovsr, dt->base + CCN_DT_PMOVSR_CLR);
  986. BUILD_BUG_ON(CCN_IDX_PMU_CYCLE_COUNTER != CCN_NUM_PMU_EVENT_COUNTERS);
  987. for (idx = 0; idx < CCN_NUM_PMU_EVENT_COUNTERS + 1; idx++) {
  988. struct perf_event *event = dt->pmu_counters[idx].event;
  989. int overflowed = pmovsr & BIT(idx);
  990. WARN_ON_ONCE(overflowed && !event &&
  991. idx != CCN_IDX_PMU_CYCLE_COUNTER);
  992. if (!event || !overflowed)
  993. continue;
  994. arm_ccn_pmu_event_update(event);
  995. }
  996. return IRQ_HANDLED;
  997. }
  998. static enum hrtimer_restart arm_ccn_pmu_timer_handler(struct hrtimer *hrtimer)
  999. {
  1000. struct arm_ccn_dt *dt = container_of(hrtimer, struct arm_ccn_dt,
  1001. hrtimer);
  1002. unsigned long flags;
  1003. local_irq_save(flags);
  1004. arm_ccn_pmu_overflow_handler(dt);
  1005. local_irq_restore(flags);
  1006. hrtimer_forward_now(hrtimer, arm_ccn_pmu_timer_period());
  1007. return HRTIMER_RESTART;
  1008. }
  1009. static int arm_ccn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
  1010. {
  1011. struct arm_ccn_dt *dt = hlist_entry_safe(node, struct arm_ccn_dt, node);
  1012. struct arm_ccn *ccn = container_of(dt, struct arm_ccn, dt);
  1013. unsigned int target;
  1014. if (cpu != dt->cpu)
  1015. return 0;
  1016. target = cpumask_any_but(cpu_online_mask, cpu);
  1017. if (target >= nr_cpu_ids)
  1018. return 0;
  1019. perf_pmu_migrate_context(&dt->pmu, cpu, target);
  1020. dt->cpu = target;
  1021. if (ccn->irq)
  1022. WARN_ON(irq_set_affinity_hint(ccn->irq, cpumask_of(dt->cpu)));
  1023. return 0;
  1024. }
  1025. static DEFINE_IDA(arm_ccn_pmu_ida);
  1026. static int arm_ccn_pmu_init(struct arm_ccn *ccn)
  1027. {
  1028. int i;
  1029. char *name;
  1030. int err;
  1031. /* Initialize DT subsystem */
  1032. ccn->dt.base = ccn->base + CCN_REGION_SIZE;
  1033. spin_lock_init(&ccn->dt.config_lock);
  1034. writel(CCN_DT_PMOVSR_CLR__MASK, ccn->dt.base + CCN_DT_PMOVSR_CLR);
  1035. writel(CCN_DT_CTL__DT_EN, ccn->dt.base + CCN_DT_CTL);
  1036. writel(CCN_DT_PMCR__OVFL_INTR_EN | CCN_DT_PMCR__PMU_EN,
  1037. ccn->dt.base + CCN_DT_PMCR);
  1038. writel(0x1, ccn->dt.base + CCN_DT_PMSR_CLR);
  1039. for (i = 0; i < ccn->num_xps; i++) {
  1040. writel(0, ccn->xp[i].base + CCN_XP_DT_CONFIG);
  1041. writel((CCN_XP_DT_CONTROL__WP_ARM_SEL__ALWAYS <<
  1042. CCN_XP_DT_CONTROL__WP_ARM_SEL__SHIFT(0)) |
  1043. (CCN_XP_DT_CONTROL__WP_ARM_SEL__ALWAYS <<
  1044. CCN_XP_DT_CONTROL__WP_ARM_SEL__SHIFT(1)) |
  1045. CCN_XP_DT_CONTROL__DT_ENABLE,
  1046. ccn->xp[i].base + CCN_XP_DT_CONTROL);
  1047. }
  1048. ccn->dt.cmp_mask[CCN_IDX_MASK_ANY].l = ~0;
  1049. ccn->dt.cmp_mask[CCN_IDX_MASK_ANY].h = ~0;
  1050. ccn->dt.cmp_mask[CCN_IDX_MASK_EXACT].l = 0;
  1051. ccn->dt.cmp_mask[CCN_IDX_MASK_EXACT].h = 0;
  1052. ccn->dt.cmp_mask[CCN_IDX_MASK_ORDER].l = ~0;
  1053. ccn->dt.cmp_mask[CCN_IDX_MASK_ORDER].h = ~(0x1 << 15);
  1054. ccn->dt.cmp_mask[CCN_IDX_MASK_OPCODE].l = ~0;
  1055. ccn->dt.cmp_mask[CCN_IDX_MASK_OPCODE].h = ~(0x1f << 9);
  1056. /* Get a convenient /sys/event_source/devices/ name */
  1057. ccn->dt.id = ida_simple_get(&arm_ccn_pmu_ida, 0, 0, GFP_KERNEL);
  1058. if (ccn->dt.id == 0) {
  1059. name = "ccn";
  1060. } else {
  1061. name = devm_kasprintf(ccn->dev, GFP_KERNEL, "ccn_%d",
  1062. ccn->dt.id);
  1063. if (!name) {
  1064. err = -ENOMEM;
  1065. goto error_choose_name;
  1066. }
  1067. }
  1068. /* Perf driver registration */
  1069. ccn->dt.pmu = (struct pmu) {
  1070. .module = THIS_MODULE,
  1071. .attr_groups = arm_ccn_pmu_attr_groups,
  1072. .task_ctx_nr = perf_invalid_context,
  1073. .event_init = arm_ccn_pmu_event_init,
  1074. .add = arm_ccn_pmu_event_add,
  1075. .del = arm_ccn_pmu_event_del,
  1076. .start = arm_ccn_pmu_event_start,
  1077. .stop = arm_ccn_pmu_event_stop,
  1078. .read = arm_ccn_pmu_event_read,
  1079. .pmu_enable = arm_ccn_pmu_enable,
  1080. .pmu_disable = arm_ccn_pmu_disable,
  1081. .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
  1082. };
  1083. /* No overflow interrupt? Have to use a timer instead. */
  1084. if (!ccn->irq) {
  1085. dev_info(ccn->dev, "No access to interrupts, using timer.\n");
  1086. hrtimer_init(&ccn->dt.hrtimer, CLOCK_MONOTONIC,
  1087. HRTIMER_MODE_REL);
  1088. ccn->dt.hrtimer.function = arm_ccn_pmu_timer_handler;
  1089. }
  1090. /* Pick one CPU which we will use to collect data from CCN... */
  1091. ccn->dt.cpu = raw_smp_processor_id();
  1092. /* Also make sure that the overflow interrupt is handled by this CPU */
  1093. if (ccn->irq) {
  1094. err = irq_set_affinity_hint(ccn->irq, cpumask_of(ccn->dt.cpu));
  1095. if (err) {
  1096. dev_err(ccn->dev, "Failed to set interrupt affinity!\n");
  1097. goto error_set_affinity;
  1098. }
  1099. }
  1100. cpuhp_state_add_instance_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE,
  1101. &ccn->dt.node);
  1102. err = perf_pmu_register(&ccn->dt.pmu, name, -1);
  1103. if (err)
  1104. goto error_pmu_register;
  1105. return 0;
  1106. error_pmu_register:
  1107. cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE,
  1108. &ccn->dt.node);
  1109. error_set_affinity:
  1110. error_choose_name:
  1111. ida_simple_remove(&arm_ccn_pmu_ida, ccn->dt.id);
  1112. for (i = 0; i < ccn->num_xps; i++)
  1113. writel(0, ccn->xp[i].base + CCN_XP_DT_CONTROL);
  1114. writel(0, ccn->dt.base + CCN_DT_PMCR);
  1115. return err;
  1116. }
  1117. static void arm_ccn_pmu_cleanup(struct arm_ccn *ccn)
  1118. {
  1119. int i;
  1120. cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE,
  1121. &ccn->dt.node);
  1122. if (ccn->irq)
  1123. irq_set_affinity_hint(ccn->irq, NULL);
  1124. for (i = 0; i < ccn->num_xps; i++)
  1125. writel(0, ccn->xp[i].base + CCN_XP_DT_CONTROL);
  1126. writel(0, ccn->dt.base + CCN_DT_PMCR);
  1127. perf_pmu_unregister(&ccn->dt.pmu);
  1128. ida_simple_remove(&arm_ccn_pmu_ida, ccn->dt.id);
  1129. }
  1130. static int arm_ccn_for_each_valid_region(struct arm_ccn *ccn,
  1131. int (*callback)(struct arm_ccn *ccn, int region,
  1132. void __iomem *base, u32 type, u32 id))
  1133. {
  1134. int region;
  1135. for (region = 0; region < CCN_NUM_REGIONS; region++) {
  1136. u32 val, type, id;
  1137. void __iomem *base;
  1138. int err;
  1139. val = readl(ccn->base + CCN_MN_OLY_COMP_LIST_63_0 +
  1140. 4 * (region / 32));
  1141. if (!(val & (1 << (region % 32))))
  1142. continue;
  1143. base = ccn->base + region * CCN_REGION_SIZE;
  1144. val = readl(base + CCN_ALL_OLY_ID);
  1145. type = (val >> CCN_ALL_OLY_ID__OLY_ID__SHIFT) &
  1146. CCN_ALL_OLY_ID__OLY_ID__MASK;
  1147. id = (val >> CCN_ALL_OLY_ID__NODE_ID__SHIFT) &
  1148. CCN_ALL_OLY_ID__NODE_ID__MASK;
  1149. err = callback(ccn, region, base, type, id);
  1150. if (err)
  1151. return err;
  1152. }
  1153. return 0;
  1154. }
  1155. static int arm_ccn_get_nodes_num(struct arm_ccn *ccn, int region,
  1156. void __iomem *base, u32 type, u32 id)
  1157. {
  1158. if (type == CCN_TYPE_XP && id >= ccn->num_xps)
  1159. ccn->num_xps = id + 1;
  1160. else if (id >= ccn->num_nodes)
  1161. ccn->num_nodes = id + 1;
  1162. return 0;
  1163. }
  1164. static int arm_ccn_init_nodes(struct arm_ccn *ccn, int region,
  1165. void __iomem *base, u32 type, u32 id)
  1166. {
  1167. struct arm_ccn_component *component;
  1168. dev_dbg(ccn->dev, "Region %d: id=%u, type=0x%02x\n", region, id, type);
  1169. switch (type) {
  1170. case CCN_TYPE_MN:
  1171. ccn->mn_id = id;
  1172. return 0;
  1173. case CCN_TYPE_DT:
  1174. return 0;
  1175. case CCN_TYPE_XP:
  1176. component = &ccn->xp[id];
  1177. break;
  1178. case CCN_TYPE_SBSX:
  1179. ccn->sbsx_present = 1;
  1180. component = &ccn->node[id];
  1181. break;
  1182. case CCN_TYPE_SBAS:
  1183. ccn->sbas_present = 1;
  1184. fallthrough;
  1185. default:
  1186. component = &ccn->node[id];
  1187. break;
  1188. }
  1189. component->base = base;
  1190. component->type = type;
  1191. return 0;
  1192. }
  1193. static irqreturn_t arm_ccn_error_handler(struct arm_ccn *ccn,
  1194. const u32 *err_sig_val)
  1195. {
  1196. /* This should be really handled by firmware... */
  1197. dev_err(ccn->dev, "Error reported in %08x%08x%08x%08x%08x%08x.\n",
  1198. err_sig_val[5], err_sig_val[4], err_sig_val[3],
  1199. err_sig_val[2], err_sig_val[1], err_sig_val[0]);
  1200. dev_err(ccn->dev, "Disabling interrupt generation for all errors.\n");
  1201. writel(CCN_MN_ERRINT_STATUS__ALL_ERRORS__DISABLE,
  1202. ccn->base + CCN_MN_ERRINT_STATUS);
  1203. return IRQ_HANDLED;
  1204. }
  1205. static irqreturn_t arm_ccn_irq_handler(int irq, void *dev_id)
  1206. {
  1207. irqreturn_t res = IRQ_NONE;
  1208. struct arm_ccn *ccn = dev_id;
  1209. u32 err_sig_val[6];
  1210. u32 err_or;
  1211. int i;
  1212. /* PMU overflow is a special case */
  1213. err_or = err_sig_val[0] = readl(ccn->base + CCN_MN_ERR_SIG_VAL_63_0);
  1214. if (err_or & CCN_MN_ERR_SIG_VAL_63_0__DT) {
  1215. err_or &= ~CCN_MN_ERR_SIG_VAL_63_0__DT;
  1216. res = arm_ccn_pmu_overflow_handler(&ccn->dt);
  1217. }
  1218. /* Have to read all err_sig_vals to clear them */
  1219. for (i = 1; i < ARRAY_SIZE(err_sig_val); i++) {
  1220. err_sig_val[i] = readl(ccn->base +
  1221. CCN_MN_ERR_SIG_VAL_63_0 + i * 4);
  1222. err_or |= err_sig_val[i];
  1223. }
  1224. if (err_or)
  1225. res |= arm_ccn_error_handler(ccn, err_sig_val);
  1226. if (res != IRQ_NONE)
  1227. writel(CCN_MN_ERRINT_STATUS__INTREQ__DESSERT,
  1228. ccn->base + CCN_MN_ERRINT_STATUS);
  1229. return res;
  1230. }
  1231. static int arm_ccn_probe(struct platform_device *pdev)
  1232. {
  1233. struct arm_ccn *ccn;
  1234. struct resource *res;
  1235. unsigned int irq;
  1236. int err;
  1237. ccn = devm_kzalloc(&pdev->dev, sizeof(*ccn), GFP_KERNEL);
  1238. if (!ccn)
  1239. return -ENOMEM;
  1240. ccn->dev = &pdev->dev;
  1241. platform_set_drvdata(pdev, ccn);
  1242. ccn->base = devm_platform_ioremap_resource(pdev, 0);
  1243. if (IS_ERR(ccn->base))
  1244. return PTR_ERR(ccn->base);
  1245. res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
  1246. if (!res)
  1247. return -EINVAL;
  1248. irq = res->start;
  1249. /* Check if we can use the interrupt */
  1250. writel(CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLE,
  1251. ccn->base + CCN_MN_ERRINT_STATUS);
  1252. if (readl(ccn->base + CCN_MN_ERRINT_STATUS) &
  1253. CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLED) {
  1254. /* Can set 'disable' bits, so can acknowledge interrupts */
  1255. writel(CCN_MN_ERRINT_STATUS__PMU_EVENTS__ENABLE,
  1256. ccn->base + CCN_MN_ERRINT_STATUS);
  1257. err = devm_request_irq(ccn->dev, irq, arm_ccn_irq_handler,
  1258. IRQF_NOBALANCING | IRQF_NO_THREAD,
  1259. dev_name(ccn->dev), ccn);
  1260. if (err)
  1261. return err;
  1262. ccn->irq = irq;
  1263. }
  1264. /* Build topology */
  1265. err = arm_ccn_for_each_valid_region(ccn, arm_ccn_get_nodes_num);
  1266. if (err)
  1267. return err;
  1268. ccn->node = devm_kcalloc(ccn->dev, ccn->num_nodes, sizeof(*ccn->node),
  1269. GFP_KERNEL);
  1270. ccn->xp = devm_kcalloc(ccn->dev, ccn->num_xps, sizeof(*ccn->node),
  1271. GFP_KERNEL);
  1272. if (!ccn->node || !ccn->xp)
  1273. return -ENOMEM;
  1274. err = arm_ccn_for_each_valid_region(ccn, arm_ccn_init_nodes);
  1275. if (err)
  1276. return err;
  1277. return arm_ccn_pmu_init(ccn);
  1278. }
  1279. static int arm_ccn_remove(struct platform_device *pdev)
  1280. {
  1281. struct arm_ccn *ccn = platform_get_drvdata(pdev);
  1282. arm_ccn_pmu_cleanup(ccn);
  1283. return 0;
  1284. }
  1285. static const struct of_device_id arm_ccn_match[] = {
  1286. { .compatible = "arm,ccn-502", },
  1287. { .compatible = "arm,ccn-504", },
  1288. { .compatible = "arm,ccn-512", },
  1289. {},
  1290. };
  1291. MODULE_DEVICE_TABLE(of, arm_ccn_match);
  1292. static struct platform_driver arm_ccn_driver = {
  1293. .driver = {
  1294. .name = "arm-ccn",
  1295. .of_match_table = arm_ccn_match,
  1296. .suppress_bind_attrs = true,
  1297. },
  1298. .probe = arm_ccn_probe,
  1299. .remove = arm_ccn_remove,
  1300. };
  1301. static int __init arm_ccn_init(void)
  1302. {
  1303. int i, ret;
  1304. ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_CCN_ONLINE,
  1305. "perf/arm/ccn:online", NULL,
  1306. arm_ccn_pmu_offline_cpu);
  1307. if (ret)
  1308. return ret;
  1309. for (i = 0; i < ARRAY_SIZE(arm_ccn_pmu_events); i++)
  1310. arm_ccn_pmu_events_attrs[i] = &arm_ccn_pmu_events[i].attr.attr;
  1311. ret = platform_driver_register(&arm_ccn_driver);
  1312. if (ret)
  1313. cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_CCN_ONLINE);
  1314. return ret;
  1315. }
  1316. static void __exit arm_ccn_exit(void)
  1317. {
  1318. platform_driver_unregister(&arm_ccn_driver);
  1319. cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_CCN_ONLINE);
  1320. }
  1321. module_init(arm_ccn_init);
  1322. module_exit(arm_ccn_exit);
  1323. MODULE_AUTHOR("Pawel Moll <pawel.moll@arm.com>");
  1324. MODULE_LICENSE("GPL v2");