perf_event.c 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* Performance event support for sparc64.
  3. *
  4. * Copyright (C) 2009, 2010 David S. Miller <davem@davemloft.net>
  5. *
  6. * This code is based almost entirely upon the x86 perf event
  7. * code, which is:
  8. *
  9. * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  10. * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
  11. * Copyright (C) 2009 Jaswinder Singh Rajput
  12. * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
  13. * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
  14. */
  15. #include <linux/perf_event.h>
  16. #include <linux/kprobes.h>
  17. #include <linux/ftrace.h>
  18. #include <linux/kernel.h>
  19. #include <linux/kdebug.h>
  20. #include <linux/mutex.h>
  21. #include <asm/stacktrace.h>
  22. #include <asm/cpudata.h>
  23. #include <linux/uaccess.h>
  24. #include <linux/atomic.h>
  25. #include <linux/sched/clock.h>
  26. #include <asm/nmi.h>
  27. #include <asm/pcr.h>
  28. #include <asm/cacheflush.h>
  29. #include "kernel.h"
  30. #include "kstack.h"
  31. /* Two classes of sparc64 chips currently exist. All of which have
  32. * 32-bit counters which can generate overflow interrupts on the
  33. * transition from 0xffffffff to 0.
  34. *
  35. * All chips upto and including SPARC-T3 have two performance
  36. * counters. The two 32-bit counters are accessed in one go using a
  37. * single 64-bit register.
  38. *
  39. * On these older chips both counters are controlled using a single
  40. * control register. The only way to stop all sampling is to clear
  41. * all of the context (user, supervisor, hypervisor) sampling enable
  42. * bits. But these bits apply to both counters, thus the two counters
  43. * can't be enabled/disabled individually.
  44. *
  45. * Furthermore, the control register on these older chips have two
  46. * event fields, one for each of the two counters. It's thus nearly
  47. * impossible to have one counter going while keeping the other one
  48. * stopped. Therefore it is possible to get overflow interrupts for
  49. * counters not currently "in use" and that condition must be checked
  50. * in the overflow interrupt handler.
  51. *
  52. * So we use a hack, in that we program inactive counters with the
  53. * "sw_count0" and "sw_count1" events. These count how many times
  54. * the instruction "sethi %hi(0xfc000), %g0" is executed. It's an
  55. * unusual way to encode a NOP and therefore will not trigger in
  56. * normal code.
  57. *
  58. * Starting with SPARC-T4 we have one control register per counter.
  59. * And the counters are stored in individual registers. The registers
  60. * for the counters are 64-bit but only a 32-bit counter is
  61. * implemented. The event selections on SPARC-T4 lack any
  62. * restrictions, therefore we can elide all of the complicated
  63. * conflict resolution code we have for SPARC-T3 and earlier chips.
  64. */
  65. #define MAX_HWEVENTS 4
  66. #define MAX_PCRS 4
  67. #define MAX_PERIOD ((1UL << 32) - 1)
  68. #define PIC_UPPER_INDEX 0
  69. #define PIC_LOWER_INDEX 1
  70. #define PIC_NO_INDEX -1
  71. struct cpu_hw_events {
  72. /* Number of events currently scheduled onto this cpu.
  73. * This tells how many entries in the arrays below
  74. * are valid.
  75. */
  76. int n_events;
  77. /* Number of new events added since the last hw_perf_disable().
  78. * This works because the perf event layer always adds new
  79. * events inside of a perf_{disable,enable}() sequence.
  80. */
  81. int n_added;
  82. /* Array of events current scheduled on this cpu. */
  83. struct perf_event *event[MAX_HWEVENTS];
  84. /* Array of encoded longs, specifying the %pcr register
  85. * encoding and the mask of PIC counters this even can
  86. * be scheduled on. See perf_event_encode() et al.
  87. */
  88. unsigned long events[MAX_HWEVENTS];
  89. /* The current counter index assigned to an event. When the
  90. * event hasn't been programmed into the cpu yet, this will
  91. * hold PIC_NO_INDEX. The event->hw.idx value tells us where
  92. * we ought to schedule the event.
  93. */
  94. int current_idx[MAX_HWEVENTS];
  95. /* Software copy of %pcr register(s) on this cpu. */
  96. u64 pcr[MAX_HWEVENTS];
  97. /* Enabled/disable state. */
  98. int enabled;
  99. unsigned int txn_flags;
  100. };
  101. static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, };
  102. /* An event map describes the characteristics of a performance
  103. * counter event. In particular it gives the encoding as well as
  104. * a mask telling which counters the event can be measured on.
  105. *
  106. * The mask is unused on SPARC-T4 and later.
  107. */
  108. struct perf_event_map {
  109. u16 encoding;
  110. u8 pic_mask;
  111. #define PIC_NONE 0x00
  112. #define PIC_UPPER 0x01
  113. #define PIC_LOWER 0x02
  114. };
  115. /* Encode a perf_event_map entry into a long. */
  116. static unsigned long perf_event_encode(const struct perf_event_map *pmap)
  117. {
  118. return ((unsigned long) pmap->encoding << 16) | pmap->pic_mask;
  119. }
  120. static u8 perf_event_get_msk(unsigned long val)
  121. {
  122. return val & 0xff;
  123. }
  124. static u64 perf_event_get_enc(unsigned long val)
  125. {
  126. return val >> 16;
  127. }
  128. #define C(x) PERF_COUNT_HW_CACHE_##x
  129. #define CACHE_OP_UNSUPPORTED 0xfffe
  130. #define CACHE_OP_NONSENSE 0xffff
  131. typedef struct perf_event_map cache_map_t
  132. [PERF_COUNT_HW_CACHE_MAX]
  133. [PERF_COUNT_HW_CACHE_OP_MAX]
  134. [PERF_COUNT_HW_CACHE_RESULT_MAX];
  135. struct sparc_pmu {
  136. const struct perf_event_map *(*event_map)(int);
  137. const cache_map_t *cache_map;
  138. int max_events;
  139. u32 (*read_pmc)(int);
  140. void (*write_pmc)(int, u64);
  141. int upper_shift;
  142. int lower_shift;
  143. int event_mask;
  144. int user_bit;
  145. int priv_bit;
  146. int hv_bit;
  147. int irq_bit;
  148. int upper_nop;
  149. int lower_nop;
  150. unsigned int flags;
  151. #define SPARC_PMU_ALL_EXCLUDES_SAME 0x00000001
  152. #define SPARC_PMU_HAS_CONFLICTS 0x00000002
  153. int max_hw_events;
  154. int num_pcrs;
  155. int num_pic_regs;
  156. };
  157. static u32 sparc_default_read_pmc(int idx)
  158. {
  159. u64 val;
  160. val = pcr_ops->read_pic(0);
  161. if (idx == PIC_UPPER_INDEX)
  162. val >>= 32;
  163. return val & 0xffffffff;
  164. }
  165. static void sparc_default_write_pmc(int idx, u64 val)
  166. {
  167. u64 shift, mask, pic;
  168. shift = 0;
  169. if (idx == PIC_UPPER_INDEX)
  170. shift = 32;
  171. mask = ((u64) 0xffffffff) << shift;
  172. val <<= shift;
  173. pic = pcr_ops->read_pic(0);
  174. pic &= ~mask;
  175. pic |= val;
  176. pcr_ops->write_pic(0, pic);
  177. }
  178. static const struct perf_event_map ultra3_perfmon_event_map[] = {
  179. [PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER | PIC_LOWER },
  180. [PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER | PIC_LOWER },
  181. [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0009, PIC_LOWER },
  182. [PERF_COUNT_HW_CACHE_MISSES] = { 0x0009, PIC_UPPER },
  183. };
  184. static const struct perf_event_map *ultra3_event_map(int event_id)
  185. {
  186. return &ultra3_perfmon_event_map[event_id];
  187. }
  188. static const cache_map_t ultra3_cache_map = {
  189. [C(L1D)] = {
  190. [C(OP_READ)] = {
  191. [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
  192. [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
  193. },
  194. [C(OP_WRITE)] = {
  195. [C(RESULT_ACCESS)] = { 0x0a, PIC_LOWER },
  196. [C(RESULT_MISS)] = { 0x0a, PIC_UPPER },
  197. },
  198. [C(OP_PREFETCH)] = {
  199. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  200. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  201. },
  202. },
  203. [C(L1I)] = {
  204. [C(OP_READ)] = {
  205. [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
  206. [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
  207. },
  208. [ C(OP_WRITE) ] = {
  209. [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
  210. [ C(RESULT_MISS) ] = { CACHE_OP_NONSENSE },
  211. },
  212. [ C(OP_PREFETCH) ] = {
  213. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  214. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  215. },
  216. },
  217. [C(LL)] = {
  218. [C(OP_READ)] = {
  219. [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER, },
  220. [C(RESULT_MISS)] = { 0x0c, PIC_UPPER, },
  221. },
  222. [C(OP_WRITE)] = {
  223. [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER },
  224. [C(RESULT_MISS)] = { 0x0c, PIC_UPPER },
  225. },
  226. [C(OP_PREFETCH)] = {
  227. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  228. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  229. },
  230. },
  231. [C(DTLB)] = {
  232. [C(OP_READ)] = {
  233. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  234. [C(RESULT_MISS)] = { 0x12, PIC_UPPER, },
  235. },
  236. [ C(OP_WRITE) ] = {
  237. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  238. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  239. },
  240. [ C(OP_PREFETCH) ] = {
  241. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  242. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  243. },
  244. },
  245. [C(ITLB)] = {
  246. [C(OP_READ)] = {
  247. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  248. [C(RESULT_MISS)] = { 0x11, PIC_UPPER, },
  249. },
  250. [ C(OP_WRITE) ] = {
  251. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  252. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  253. },
  254. [ C(OP_PREFETCH) ] = {
  255. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  256. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  257. },
  258. },
  259. [C(BPU)] = {
  260. [C(OP_READ)] = {
  261. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  262. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  263. },
  264. [ C(OP_WRITE) ] = {
  265. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  266. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  267. },
  268. [ C(OP_PREFETCH) ] = {
  269. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  270. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  271. },
  272. },
  273. [C(NODE)] = {
  274. [C(OP_READ)] = {
  275. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  276. [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  277. },
  278. [ C(OP_WRITE) ] = {
  279. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  280. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  281. },
  282. [ C(OP_PREFETCH) ] = {
  283. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  284. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  285. },
  286. },
  287. };
  288. static const struct sparc_pmu ultra3_pmu = {
  289. .event_map = ultra3_event_map,
  290. .cache_map = &ultra3_cache_map,
  291. .max_events = ARRAY_SIZE(ultra3_perfmon_event_map),
  292. .read_pmc = sparc_default_read_pmc,
  293. .write_pmc = sparc_default_write_pmc,
  294. .upper_shift = 11,
  295. .lower_shift = 4,
  296. .event_mask = 0x3f,
  297. .user_bit = PCR_UTRACE,
  298. .priv_bit = PCR_STRACE,
  299. .upper_nop = 0x1c,
  300. .lower_nop = 0x14,
  301. .flags = (SPARC_PMU_ALL_EXCLUDES_SAME |
  302. SPARC_PMU_HAS_CONFLICTS),
  303. .max_hw_events = 2,
  304. .num_pcrs = 1,
  305. .num_pic_regs = 1,
  306. };
  307. /* Niagara1 is very limited. The upper PIC is hard-locked to count
  308. * only instructions, so it is free running which creates all kinds of
  309. * problems. Some hardware designs make one wonder if the creator
  310. * even looked at how this stuff gets used by software.
  311. */
  312. static const struct perf_event_map niagara1_perfmon_event_map[] = {
  313. [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, PIC_UPPER },
  314. [PERF_COUNT_HW_INSTRUCTIONS] = { 0x00, PIC_UPPER },
  315. [PERF_COUNT_HW_CACHE_REFERENCES] = { 0, PIC_NONE },
  316. [PERF_COUNT_HW_CACHE_MISSES] = { 0x03, PIC_LOWER },
  317. };
  318. static const struct perf_event_map *niagara1_event_map(int event_id)
  319. {
  320. return &niagara1_perfmon_event_map[event_id];
  321. }
  322. static const cache_map_t niagara1_cache_map = {
  323. [C(L1D)] = {
  324. [C(OP_READ)] = {
  325. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  326. [C(RESULT_MISS)] = { 0x03, PIC_LOWER, },
  327. },
  328. [C(OP_WRITE)] = {
  329. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  330. [C(RESULT_MISS)] = { 0x03, PIC_LOWER, },
  331. },
  332. [C(OP_PREFETCH)] = {
  333. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  334. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  335. },
  336. },
  337. [C(L1I)] = {
  338. [C(OP_READ)] = {
  339. [C(RESULT_ACCESS)] = { 0x00, PIC_UPPER },
  340. [C(RESULT_MISS)] = { 0x02, PIC_LOWER, },
  341. },
  342. [ C(OP_WRITE) ] = {
  343. [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
  344. [ C(RESULT_MISS) ] = { CACHE_OP_NONSENSE },
  345. },
  346. [ C(OP_PREFETCH) ] = {
  347. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  348. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  349. },
  350. },
  351. [C(LL)] = {
  352. [C(OP_READ)] = {
  353. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  354. [C(RESULT_MISS)] = { 0x07, PIC_LOWER, },
  355. },
  356. [C(OP_WRITE)] = {
  357. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  358. [C(RESULT_MISS)] = { 0x07, PIC_LOWER, },
  359. },
  360. [C(OP_PREFETCH)] = {
  361. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  362. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  363. },
  364. },
  365. [C(DTLB)] = {
  366. [C(OP_READ)] = {
  367. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  368. [C(RESULT_MISS)] = { 0x05, PIC_LOWER, },
  369. },
  370. [ C(OP_WRITE) ] = {
  371. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  372. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  373. },
  374. [ C(OP_PREFETCH) ] = {
  375. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  376. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  377. },
  378. },
  379. [C(ITLB)] = {
  380. [C(OP_READ)] = {
  381. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  382. [C(RESULT_MISS)] = { 0x04, PIC_LOWER, },
  383. },
  384. [ C(OP_WRITE) ] = {
  385. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  386. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  387. },
  388. [ C(OP_PREFETCH) ] = {
  389. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  390. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  391. },
  392. },
  393. [C(BPU)] = {
  394. [C(OP_READ)] = {
  395. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  396. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  397. },
  398. [ C(OP_WRITE) ] = {
  399. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  400. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  401. },
  402. [ C(OP_PREFETCH) ] = {
  403. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  404. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  405. },
  406. },
  407. [C(NODE)] = {
  408. [C(OP_READ)] = {
  409. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  410. [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  411. },
  412. [ C(OP_WRITE) ] = {
  413. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  414. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  415. },
  416. [ C(OP_PREFETCH) ] = {
  417. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  418. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  419. },
  420. },
  421. };
  422. static const struct sparc_pmu niagara1_pmu = {
  423. .event_map = niagara1_event_map,
  424. .cache_map = &niagara1_cache_map,
  425. .max_events = ARRAY_SIZE(niagara1_perfmon_event_map),
  426. .read_pmc = sparc_default_read_pmc,
  427. .write_pmc = sparc_default_write_pmc,
  428. .upper_shift = 0,
  429. .lower_shift = 4,
  430. .event_mask = 0x7,
  431. .user_bit = PCR_UTRACE,
  432. .priv_bit = PCR_STRACE,
  433. .upper_nop = 0x0,
  434. .lower_nop = 0x0,
  435. .flags = (SPARC_PMU_ALL_EXCLUDES_SAME |
  436. SPARC_PMU_HAS_CONFLICTS),
  437. .max_hw_events = 2,
  438. .num_pcrs = 1,
  439. .num_pic_regs = 1,
  440. };
  441. static const struct perf_event_map niagara2_perfmon_event_map[] = {
  442. [PERF_COUNT_HW_CPU_CYCLES] = { 0x02ff, PIC_UPPER | PIC_LOWER },
  443. [PERF_COUNT_HW_INSTRUCTIONS] = { 0x02ff, PIC_UPPER | PIC_LOWER },
  444. [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0208, PIC_UPPER | PIC_LOWER },
  445. [PERF_COUNT_HW_CACHE_MISSES] = { 0x0302, PIC_UPPER | PIC_LOWER },
  446. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x0201, PIC_UPPER | PIC_LOWER },
  447. [PERF_COUNT_HW_BRANCH_MISSES] = { 0x0202, PIC_UPPER | PIC_LOWER },
  448. };
  449. static const struct perf_event_map *niagara2_event_map(int event_id)
  450. {
  451. return &niagara2_perfmon_event_map[event_id];
  452. }
  453. static const cache_map_t niagara2_cache_map = {
  454. [C(L1D)] = {
  455. [C(OP_READ)] = {
  456. [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, },
  457. [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, },
  458. },
  459. [C(OP_WRITE)] = {
  460. [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, },
  461. [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, },
  462. },
  463. [C(OP_PREFETCH)] = {
  464. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  465. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  466. },
  467. },
  468. [C(L1I)] = {
  469. [C(OP_READ)] = {
  470. [C(RESULT_ACCESS)] = { 0x02ff, PIC_UPPER | PIC_LOWER, },
  471. [C(RESULT_MISS)] = { 0x0301, PIC_UPPER | PIC_LOWER, },
  472. },
  473. [ C(OP_WRITE) ] = {
  474. [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
  475. [ C(RESULT_MISS) ] = { CACHE_OP_NONSENSE },
  476. },
  477. [ C(OP_PREFETCH) ] = {
  478. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  479. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  480. },
  481. },
  482. [C(LL)] = {
  483. [C(OP_READ)] = {
  484. [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, },
  485. [C(RESULT_MISS)] = { 0x0330, PIC_UPPER | PIC_LOWER, },
  486. },
  487. [C(OP_WRITE)] = {
  488. [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, },
  489. [C(RESULT_MISS)] = { 0x0320, PIC_UPPER | PIC_LOWER, },
  490. },
  491. [C(OP_PREFETCH)] = {
  492. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  493. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  494. },
  495. },
  496. [C(DTLB)] = {
  497. [C(OP_READ)] = {
  498. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  499. [C(RESULT_MISS)] = { 0x0b08, PIC_UPPER | PIC_LOWER, },
  500. },
  501. [ C(OP_WRITE) ] = {
  502. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  503. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  504. },
  505. [ C(OP_PREFETCH) ] = {
  506. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  507. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  508. },
  509. },
  510. [C(ITLB)] = {
  511. [C(OP_READ)] = {
  512. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  513. [C(RESULT_MISS)] = { 0xb04, PIC_UPPER | PIC_LOWER, },
  514. },
  515. [ C(OP_WRITE) ] = {
  516. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  517. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  518. },
  519. [ C(OP_PREFETCH) ] = {
  520. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  521. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  522. },
  523. },
  524. [C(BPU)] = {
  525. [C(OP_READ)] = {
  526. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  527. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  528. },
  529. [ C(OP_WRITE) ] = {
  530. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  531. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  532. },
  533. [ C(OP_PREFETCH) ] = {
  534. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  535. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  536. },
  537. },
  538. [C(NODE)] = {
  539. [C(OP_READ)] = {
  540. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  541. [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  542. },
  543. [ C(OP_WRITE) ] = {
  544. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  545. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  546. },
  547. [ C(OP_PREFETCH) ] = {
  548. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  549. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  550. },
  551. },
  552. };
  553. static const struct sparc_pmu niagara2_pmu = {
  554. .event_map = niagara2_event_map,
  555. .cache_map = &niagara2_cache_map,
  556. .max_events = ARRAY_SIZE(niagara2_perfmon_event_map),
  557. .read_pmc = sparc_default_read_pmc,
  558. .write_pmc = sparc_default_write_pmc,
  559. .upper_shift = 19,
  560. .lower_shift = 6,
  561. .event_mask = 0xfff,
  562. .user_bit = PCR_UTRACE,
  563. .priv_bit = PCR_STRACE,
  564. .hv_bit = PCR_N2_HTRACE,
  565. .irq_bit = 0x30,
  566. .upper_nop = 0x220,
  567. .lower_nop = 0x220,
  568. .flags = (SPARC_PMU_ALL_EXCLUDES_SAME |
  569. SPARC_PMU_HAS_CONFLICTS),
  570. .max_hw_events = 2,
  571. .num_pcrs = 1,
  572. .num_pic_regs = 1,
  573. };
  574. static const struct perf_event_map niagara4_perfmon_event_map[] = {
  575. [PERF_COUNT_HW_CPU_CYCLES] = { (26 << 6) },
  576. [PERF_COUNT_HW_INSTRUCTIONS] = { (3 << 6) | 0x3f },
  577. [PERF_COUNT_HW_CACHE_REFERENCES] = { (3 << 6) | 0x04 },
  578. [PERF_COUNT_HW_CACHE_MISSES] = { (16 << 6) | 0x07 },
  579. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { (4 << 6) | 0x01 },
  580. [PERF_COUNT_HW_BRANCH_MISSES] = { (25 << 6) | 0x0f },
  581. };
  582. static const struct perf_event_map *niagara4_event_map(int event_id)
  583. {
  584. return &niagara4_perfmon_event_map[event_id];
  585. }
  586. static const cache_map_t niagara4_cache_map = {
  587. [C(L1D)] = {
  588. [C(OP_READ)] = {
  589. [C(RESULT_ACCESS)] = { (3 << 6) | 0x04 },
  590. [C(RESULT_MISS)] = { (16 << 6) | 0x07 },
  591. },
  592. [C(OP_WRITE)] = {
  593. [C(RESULT_ACCESS)] = { (3 << 6) | 0x08 },
  594. [C(RESULT_MISS)] = { (16 << 6) | 0x07 },
  595. },
  596. [C(OP_PREFETCH)] = {
  597. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  598. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  599. },
  600. },
  601. [C(L1I)] = {
  602. [C(OP_READ)] = {
  603. [C(RESULT_ACCESS)] = { (3 << 6) | 0x3f },
  604. [C(RESULT_MISS)] = { (11 << 6) | 0x03 },
  605. },
  606. [ C(OP_WRITE) ] = {
  607. [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
  608. [ C(RESULT_MISS) ] = { CACHE_OP_NONSENSE },
  609. },
  610. [ C(OP_PREFETCH) ] = {
  611. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  612. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  613. },
  614. },
  615. [C(LL)] = {
  616. [C(OP_READ)] = {
  617. [C(RESULT_ACCESS)] = { (3 << 6) | 0x04 },
  618. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  619. },
  620. [C(OP_WRITE)] = {
  621. [C(RESULT_ACCESS)] = { (3 << 6) | 0x08 },
  622. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  623. },
  624. [C(OP_PREFETCH)] = {
  625. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  626. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  627. },
  628. },
  629. [C(DTLB)] = {
  630. [C(OP_READ)] = {
  631. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  632. [C(RESULT_MISS)] = { (17 << 6) | 0x3f },
  633. },
  634. [ C(OP_WRITE) ] = {
  635. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  636. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  637. },
  638. [ C(OP_PREFETCH) ] = {
  639. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  640. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  641. },
  642. },
  643. [C(ITLB)] = {
  644. [C(OP_READ)] = {
  645. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  646. [C(RESULT_MISS)] = { (6 << 6) | 0x3f },
  647. },
  648. [ C(OP_WRITE) ] = {
  649. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  650. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  651. },
  652. [ C(OP_PREFETCH) ] = {
  653. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  654. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  655. },
  656. },
  657. [C(BPU)] = {
  658. [C(OP_READ)] = {
  659. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  660. [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
  661. },
  662. [ C(OP_WRITE) ] = {
  663. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  664. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  665. },
  666. [ C(OP_PREFETCH) ] = {
  667. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  668. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  669. },
  670. },
  671. [C(NODE)] = {
  672. [C(OP_READ)] = {
  673. [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
  674. [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  675. },
  676. [ C(OP_WRITE) ] = {
  677. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  678. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  679. },
  680. [ C(OP_PREFETCH) ] = {
  681. [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
  682. [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
  683. },
  684. },
  685. };
  686. static u32 sparc_vt_read_pmc(int idx)
  687. {
  688. u64 val = pcr_ops->read_pic(idx);
  689. return val & 0xffffffff;
  690. }
  691. static void sparc_vt_write_pmc(int idx, u64 val)
  692. {
  693. u64 pcr;
  694. pcr = pcr_ops->read_pcr(idx);
  695. /* ensure ov and ntc are reset */
  696. pcr &= ~(PCR_N4_OV | PCR_N4_NTC);
  697. pcr_ops->write_pic(idx, val & 0xffffffff);
  698. pcr_ops->write_pcr(idx, pcr);
  699. }
  700. static const struct sparc_pmu niagara4_pmu = {
  701. .event_map = niagara4_event_map,
  702. .cache_map = &niagara4_cache_map,
  703. .max_events = ARRAY_SIZE(niagara4_perfmon_event_map),
  704. .read_pmc = sparc_vt_read_pmc,
  705. .write_pmc = sparc_vt_write_pmc,
  706. .upper_shift = 5,
  707. .lower_shift = 5,
  708. .event_mask = 0x7ff,
  709. .user_bit = PCR_N4_UTRACE,
  710. .priv_bit = PCR_N4_STRACE,
  711. /* We explicitly don't support hypervisor tracing. The T4
  712. * generates the overflow event for precise events via a trap
  713. * which will not be generated (ie. it's completely lost) if
  714. * we happen to be in the hypervisor when the event triggers.
  715. * Essentially, the overflow event reporting is completely
  716. * unusable when you have hypervisor mode tracing enabled.
  717. */
  718. .hv_bit = 0,
  719. .irq_bit = PCR_N4_TOE,
  720. .upper_nop = 0,
  721. .lower_nop = 0,
  722. .flags = 0,
  723. .max_hw_events = 4,
  724. .num_pcrs = 4,
  725. .num_pic_regs = 4,
  726. };
  727. static const struct sparc_pmu sparc_m7_pmu = {
  728. .event_map = niagara4_event_map,
  729. .cache_map = &niagara4_cache_map,
  730. .max_events = ARRAY_SIZE(niagara4_perfmon_event_map),
  731. .read_pmc = sparc_vt_read_pmc,
  732. .write_pmc = sparc_vt_write_pmc,
  733. .upper_shift = 5,
  734. .lower_shift = 5,
  735. .event_mask = 0x7ff,
  736. .user_bit = PCR_N4_UTRACE,
  737. .priv_bit = PCR_N4_STRACE,
  738. /* We explicitly don't support hypervisor tracing. */
  739. .hv_bit = 0,
  740. .irq_bit = PCR_N4_TOE,
  741. .upper_nop = 0,
  742. .lower_nop = 0,
  743. .flags = 0,
  744. .max_hw_events = 4,
  745. .num_pcrs = 4,
  746. .num_pic_regs = 4,
  747. };
  748. static const struct sparc_pmu *sparc_pmu __read_mostly;
  749. static u64 event_encoding(u64 event_id, int idx)
  750. {
  751. if (idx == PIC_UPPER_INDEX)
  752. event_id <<= sparc_pmu->upper_shift;
  753. else
  754. event_id <<= sparc_pmu->lower_shift;
  755. return event_id;
  756. }
  757. static u64 mask_for_index(int idx)
  758. {
  759. return event_encoding(sparc_pmu->event_mask, idx);
  760. }
  761. static u64 nop_for_index(int idx)
  762. {
  763. return event_encoding(idx == PIC_UPPER_INDEX ?
  764. sparc_pmu->upper_nop :
  765. sparc_pmu->lower_nop, idx);
  766. }
  767. static inline void sparc_pmu_enable_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc, int idx)
  768. {
  769. u64 enc, val, mask = mask_for_index(idx);
  770. int pcr_index = 0;
  771. if (sparc_pmu->num_pcrs > 1)
  772. pcr_index = idx;
  773. enc = perf_event_get_enc(cpuc->events[idx]);
  774. val = cpuc->pcr[pcr_index];
  775. val &= ~mask;
  776. val |= event_encoding(enc, idx);
  777. cpuc->pcr[pcr_index] = val;
  778. pcr_ops->write_pcr(pcr_index, cpuc->pcr[pcr_index]);
  779. }
  780. static inline void sparc_pmu_disable_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc, int idx)
  781. {
  782. u64 mask = mask_for_index(idx);
  783. u64 nop = nop_for_index(idx);
  784. int pcr_index = 0;
  785. u64 val;
  786. if (sparc_pmu->num_pcrs > 1)
  787. pcr_index = idx;
  788. val = cpuc->pcr[pcr_index];
  789. val &= ~mask;
  790. val |= nop;
  791. cpuc->pcr[pcr_index] = val;
  792. pcr_ops->write_pcr(pcr_index, cpuc->pcr[pcr_index]);
  793. }
  794. static u64 sparc_perf_event_update(struct perf_event *event,
  795. struct hw_perf_event *hwc, int idx)
  796. {
  797. int shift = 64 - 32;
  798. u64 prev_raw_count, new_raw_count;
  799. s64 delta;
  800. again:
  801. prev_raw_count = local64_read(&hwc->prev_count);
  802. new_raw_count = sparc_pmu->read_pmc(idx);
  803. if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
  804. new_raw_count) != prev_raw_count)
  805. goto again;
  806. delta = (new_raw_count << shift) - (prev_raw_count << shift);
  807. delta >>= shift;
  808. local64_add(delta, &event->count);
  809. local64_sub(delta, &hwc->period_left);
  810. return new_raw_count;
  811. }
  812. static int sparc_perf_event_set_period(struct perf_event *event,
  813. struct hw_perf_event *hwc, int idx)
  814. {
  815. s64 left = local64_read(&hwc->period_left);
  816. s64 period = hwc->sample_period;
  817. int ret = 0;
  818. /* The period may have been changed by PERF_EVENT_IOC_PERIOD */
  819. if (unlikely(period != hwc->last_period))
  820. left = period - (hwc->last_period - left);
  821. if (unlikely(left <= -period)) {
  822. left = period;
  823. local64_set(&hwc->period_left, left);
  824. hwc->last_period = period;
  825. ret = 1;
  826. }
  827. if (unlikely(left <= 0)) {
  828. left += period;
  829. local64_set(&hwc->period_left, left);
  830. hwc->last_period = period;
  831. ret = 1;
  832. }
  833. if (left > MAX_PERIOD)
  834. left = MAX_PERIOD;
  835. local64_set(&hwc->prev_count, (u64)-left);
  836. sparc_pmu->write_pmc(idx, (u64)(-left) & 0xffffffff);
  837. perf_event_update_userpage(event);
  838. return ret;
  839. }
  840. static void read_in_all_counters(struct cpu_hw_events *cpuc)
  841. {
  842. int i;
  843. for (i = 0; i < cpuc->n_events; i++) {
  844. struct perf_event *cp = cpuc->event[i];
  845. if (cpuc->current_idx[i] != PIC_NO_INDEX &&
  846. cpuc->current_idx[i] != cp->hw.idx) {
  847. sparc_perf_event_update(cp, &cp->hw,
  848. cpuc->current_idx[i]);
  849. cpuc->current_idx[i] = PIC_NO_INDEX;
  850. if (cp->hw.state & PERF_HES_STOPPED)
  851. cp->hw.state |= PERF_HES_ARCH;
  852. }
  853. }
  854. }
  855. /* On this PMU all PICs are programmed using a single PCR. Calculate
  856. * the combined control register value.
  857. *
  858. * For such chips we require that all of the events have the same
  859. * configuration, so just fetch the settings from the first entry.
  860. */
  861. static void calculate_single_pcr(struct cpu_hw_events *cpuc)
  862. {
  863. int i;
  864. if (!cpuc->n_added)
  865. goto out;
  866. /* Assign to counters all unassigned events. */
  867. for (i = 0; i < cpuc->n_events; i++) {
  868. struct perf_event *cp = cpuc->event[i];
  869. struct hw_perf_event *hwc = &cp->hw;
  870. int idx = hwc->idx;
  871. u64 enc;
  872. if (cpuc->current_idx[i] != PIC_NO_INDEX)
  873. continue;
  874. sparc_perf_event_set_period(cp, hwc, idx);
  875. cpuc->current_idx[i] = idx;
  876. enc = perf_event_get_enc(cpuc->events[i]);
  877. cpuc->pcr[0] &= ~mask_for_index(idx);
  878. if (hwc->state & PERF_HES_ARCH) {
  879. cpuc->pcr[0] |= nop_for_index(idx);
  880. } else {
  881. cpuc->pcr[0] |= event_encoding(enc, idx);
  882. hwc->state = 0;
  883. }
  884. }
  885. out:
  886. cpuc->pcr[0] |= cpuc->event[0]->hw.config_base;
  887. }
  888. static void sparc_pmu_start(struct perf_event *event, int flags);
  889. /* On this PMU each PIC has it's own PCR control register. */
  890. static void calculate_multiple_pcrs(struct cpu_hw_events *cpuc)
  891. {
  892. int i;
  893. if (!cpuc->n_added)
  894. goto out;
  895. for (i = 0; i < cpuc->n_events; i++) {
  896. struct perf_event *cp = cpuc->event[i];
  897. struct hw_perf_event *hwc = &cp->hw;
  898. int idx = hwc->idx;
  899. if (cpuc->current_idx[i] != PIC_NO_INDEX)
  900. continue;
  901. cpuc->current_idx[i] = idx;
  902. if (cp->hw.state & PERF_HES_ARCH)
  903. continue;
  904. sparc_pmu_start(cp, PERF_EF_RELOAD);
  905. }
  906. out:
  907. for (i = 0; i < cpuc->n_events; i++) {
  908. struct perf_event *cp = cpuc->event[i];
  909. int idx = cp->hw.idx;
  910. cpuc->pcr[idx] |= cp->hw.config_base;
  911. }
  912. }
  913. /* If performance event entries have been added, move existing events
  914. * around (if necessary) and then assign new entries to counters.
  915. */
  916. static void update_pcrs_for_enable(struct cpu_hw_events *cpuc)
  917. {
  918. if (cpuc->n_added)
  919. read_in_all_counters(cpuc);
  920. if (sparc_pmu->num_pcrs == 1) {
  921. calculate_single_pcr(cpuc);
  922. } else {
  923. calculate_multiple_pcrs(cpuc);
  924. }
  925. }
  926. static void sparc_pmu_enable(struct pmu *pmu)
  927. {
  928. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  929. int i;
  930. if (cpuc->enabled)
  931. return;
  932. cpuc->enabled = 1;
  933. barrier();
  934. if (cpuc->n_events)
  935. update_pcrs_for_enable(cpuc);
  936. for (i = 0; i < sparc_pmu->num_pcrs; i++)
  937. pcr_ops->write_pcr(i, cpuc->pcr[i]);
  938. }
  939. static void sparc_pmu_disable(struct pmu *pmu)
  940. {
  941. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  942. int i;
  943. if (!cpuc->enabled)
  944. return;
  945. cpuc->enabled = 0;
  946. cpuc->n_added = 0;
  947. for (i = 0; i < sparc_pmu->num_pcrs; i++) {
  948. u64 val = cpuc->pcr[i];
  949. val &= ~(sparc_pmu->user_bit | sparc_pmu->priv_bit |
  950. sparc_pmu->hv_bit | sparc_pmu->irq_bit);
  951. cpuc->pcr[i] = val;
  952. pcr_ops->write_pcr(i, cpuc->pcr[i]);
  953. }
  954. }
  955. static int active_event_index(struct cpu_hw_events *cpuc,
  956. struct perf_event *event)
  957. {
  958. int i;
  959. for (i = 0; i < cpuc->n_events; i++) {
  960. if (cpuc->event[i] == event)
  961. break;
  962. }
  963. BUG_ON(i == cpuc->n_events);
  964. return cpuc->current_idx[i];
  965. }
  966. static void sparc_pmu_start(struct perf_event *event, int flags)
  967. {
  968. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  969. int idx = active_event_index(cpuc, event);
  970. if (flags & PERF_EF_RELOAD) {
  971. WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
  972. sparc_perf_event_set_period(event, &event->hw, idx);
  973. }
  974. event->hw.state = 0;
  975. sparc_pmu_enable_event(cpuc, &event->hw, idx);
  976. perf_event_update_userpage(event);
  977. }
  978. static void sparc_pmu_stop(struct perf_event *event, int flags)
  979. {
  980. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  981. int idx = active_event_index(cpuc, event);
  982. if (!(event->hw.state & PERF_HES_STOPPED)) {
  983. sparc_pmu_disable_event(cpuc, &event->hw, idx);
  984. event->hw.state |= PERF_HES_STOPPED;
  985. }
  986. if (!(event->hw.state & PERF_HES_UPTODATE) && (flags & PERF_EF_UPDATE)) {
  987. sparc_perf_event_update(event, &event->hw, idx);
  988. event->hw.state |= PERF_HES_UPTODATE;
  989. }
  990. }
  991. static void sparc_pmu_del(struct perf_event *event, int _flags)
  992. {
  993. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  994. unsigned long flags;
  995. int i;
  996. local_irq_save(flags);
  997. for (i = 0; i < cpuc->n_events; i++) {
  998. if (event == cpuc->event[i]) {
  999. /* Absorb the final count and turn off the
  1000. * event.
  1001. */
  1002. sparc_pmu_stop(event, PERF_EF_UPDATE);
  1003. /* Shift remaining entries down into
  1004. * the existing slot.
  1005. */
  1006. while (++i < cpuc->n_events) {
  1007. cpuc->event[i - 1] = cpuc->event[i];
  1008. cpuc->events[i - 1] = cpuc->events[i];
  1009. cpuc->current_idx[i - 1] =
  1010. cpuc->current_idx[i];
  1011. }
  1012. perf_event_update_userpage(event);
  1013. cpuc->n_events--;
  1014. break;
  1015. }
  1016. }
  1017. local_irq_restore(flags);
  1018. }
  1019. static void sparc_pmu_read(struct perf_event *event)
  1020. {
  1021. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  1022. int idx = active_event_index(cpuc, event);
  1023. struct hw_perf_event *hwc = &event->hw;
  1024. sparc_perf_event_update(event, hwc, idx);
  1025. }
  1026. static atomic_t active_events = ATOMIC_INIT(0);
  1027. static DEFINE_MUTEX(pmc_grab_mutex);
  1028. static void perf_stop_nmi_watchdog(void *unused)
  1029. {
  1030. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  1031. int i;
  1032. stop_nmi_watchdog(NULL);
  1033. for (i = 0; i < sparc_pmu->num_pcrs; i++)
  1034. cpuc->pcr[i] = pcr_ops->read_pcr(i);
  1035. }
  1036. static void perf_event_grab_pmc(void)
  1037. {
  1038. if (atomic_inc_not_zero(&active_events))
  1039. return;
  1040. mutex_lock(&pmc_grab_mutex);
  1041. if (atomic_read(&active_events) == 0) {
  1042. if (atomic_read(&nmi_active) > 0) {
  1043. on_each_cpu(perf_stop_nmi_watchdog, NULL, 1);
  1044. BUG_ON(atomic_read(&nmi_active) != 0);
  1045. }
  1046. atomic_inc(&active_events);
  1047. }
  1048. mutex_unlock(&pmc_grab_mutex);
  1049. }
  1050. static void perf_event_release_pmc(void)
  1051. {
  1052. if (atomic_dec_and_mutex_lock(&active_events, &pmc_grab_mutex)) {
  1053. if (atomic_read(&nmi_active) == 0)
  1054. on_each_cpu(start_nmi_watchdog, NULL, 1);
  1055. mutex_unlock(&pmc_grab_mutex);
  1056. }
  1057. }
  1058. static const struct perf_event_map *sparc_map_cache_event(u64 config)
  1059. {
  1060. unsigned int cache_type, cache_op, cache_result;
  1061. const struct perf_event_map *pmap;
  1062. if (!sparc_pmu->cache_map)
  1063. return ERR_PTR(-ENOENT);
  1064. cache_type = (config >> 0) & 0xff;
  1065. if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
  1066. return ERR_PTR(-EINVAL);
  1067. cache_op = (config >> 8) & 0xff;
  1068. if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
  1069. return ERR_PTR(-EINVAL);
  1070. cache_result = (config >> 16) & 0xff;
  1071. if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
  1072. return ERR_PTR(-EINVAL);
  1073. pmap = &((*sparc_pmu->cache_map)[cache_type][cache_op][cache_result]);
  1074. if (pmap->encoding == CACHE_OP_UNSUPPORTED)
  1075. return ERR_PTR(-ENOENT);
  1076. if (pmap->encoding == CACHE_OP_NONSENSE)
  1077. return ERR_PTR(-EINVAL);
  1078. return pmap;
  1079. }
  1080. static void hw_perf_event_destroy(struct perf_event *event)
  1081. {
  1082. perf_event_release_pmc();
  1083. }
  1084. /* Make sure all events can be scheduled into the hardware at
  1085. * the same time. This is simplified by the fact that we only
  1086. * need to support 2 simultaneous HW events.
  1087. *
  1088. * As a side effect, the evts[]->hw.idx values will be assigned
  1089. * on success. These are pending indexes. When the events are
  1090. * actually programmed into the chip, these values will propagate
  1091. * to the per-cpu cpuc->current_idx[] slots, see the code in
  1092. * maybe_change_configuration() for details.
  1093. */
  1094. static int sparc_check_constraints(struct perf_event **evts,
  1095. unsigned long *events, int n_ev)
  1096. {
  1097. u8 msk0 = 0, msk1 = 0;
  1098. int idx0 = 0;
  1099. /* This case is possible when we are invoked from
  1100. * hw_perf_group_sched_in().
  1101. */
  1102. if (!n_ev)
  1103. return 0;
  1104. if (n_ev > sparc_pmu->max_hw_events)
  1105. return -1;
  1106. if (!(sparc_pmu->flags & SPARC_PMU_HAS_CONFLICTS)) {
  1107. int i;
  1108. for (i = 0; i < n_ev; i++)
  1109. evts[i]->hw.idx = i;
  1110. return 0;
  1111. }
  1112. msk0 = perf_event_get_msk(events[0]);
  1113. if (n_ev == 1) {
  1114. if (msk0 & PIC_LOWER)
  1115. idx0 = 1;
  1116. goto success;
  1117. }
  1118. BUG_ON(n_ev != 2);
  1119. msk1 = perf_event_get_msk(events[1]);
  1120. /* If both events can go on any counter, OK. */
  1121. if (msk0 == (PIC_UPPER | PIC_LOWER) &&
  1122. msk1 == (PIC_UPPER | PIC_LOWER))
  1123. goto success;
  1124. /* If one event is limited to a specific counter,
  1125. * and the other can go on both, OK.
  1126. */
  1127. if ((msk0 == PIC_UPPER || msk0 == PIC_LOWER) &&
  1128. msk1 == (PIC_UPPER | PIC_LOWER)) {
  1129. if (msk0 & PIC_LOWER)
  1130. idx0 = 1;
  1131. goto success;
  1132. }
  1133. if ((msk1 == PIC_UPPER || msk1 == PIC_LOWER) &&
  1134. msk0 == (PIC_UPPER | PIC_LOWER)) {
  1135. if (msk1 & PIC_UPPER)
  1136. idx0 = 1;
  1137. goto success;
  1138. }
  1139. /* If the events are fixed to different counters, OK. */
  1140. if ((msk0 == PIC_UPPER && msk1 == PIC_LOWER) ||
  1141. (msk0 == PIC_LOWER && msk1 == PIC_UPPER)) {
  1142. if (msk0 & PIC_LOWER)
  1143. idx0 = 1;
  1144. goto success;
  1145. }
  1146. /* Otherwise, there is a conflict. */
  1147. return -1;
  1148. success:
  1149. evts[0]->hw.idx = idx0;
  1150. if (n_ev == 2)
  1151. evts[1]->hw.idx = idx0 ^ 1;
  1152. return 0;
  1153. }
  1154. static int check_excludes(struct perf_event **evts, int n_prev, int n_new)
  1155. {
  1156. int eu = 0, ek = 0, eh = 0;
  1157. struct perf_event *event;
  1158. int i, n, first;
  1159. if (!(sparc_pmu->flags & SPARC_PMU_ALL_EXCLUDES_SAME))
  1160. return 0;
  1161. n = n_prev + n_new;
  1162. if (n <= 1)
  1163. return 0;
  1164. first = 1;
  1165. for (i = 0; i < n; i++) {
  1166. event = evts[i];
  1167. if (first) {
  1168. eu = event->attr.exclude_user;
  1169. ek = event->attr.exclude_kernel;
  1170. eh = event->attr.exclude_hv;
  1171. first = 0;
  1172. } else if (event->attr.exclude_user != eu ||
  1173. event->attr.exclude_kernel != ek ||
  1174. event->attr.exclude_hv != eh) {
  1175. return -EAGAIN;
  1176. }
  1177. }
  1178. return 0;
  1179. }
  1180. static int collect_events(struct perf_event *group, int max_count,
  1181. struct perf_event *evts[], unsigned long *events,
  1182. int *current_idx)
  1183. {
  1184. struct perf_event *event;
  1185. int n = 0;
  1186. if (!is_software_event(group)) {
  1187. if (n >= max_count)
  1188. return -1;
  1189. evts[n] = group;
  1190. events[n] = group->hw.event_base;
  1191. current_idx[n++] = PIC_NO_INDEX;
  1192. }
  1193. for_each_sibling_event(event, group) {
  1194. if (!is_software_event(event) &&
  1195. event->state != PERF_EVENT_STATE_OFF) {
  1196. if (n >= max_count)
  1197. return -1;
  1198. evts[n] = event;
  1199. events[n] = event->hw.event_base;
  1200. current_idx[n++] = PIC_NO_INDEX;
  1201. }
  1202. }
  1203. return n;
  1204. }
  1205. static int sparc_pmu_add(struct perf_event *event, int ef_flags)
  1206. {
  1207. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  1208. int n0, ret = -EAGAIN;
  1209. unsigned long flags;
  1210. local_irq_save(flags);
  1211. n0 = cpuc->n_events;
  1212. if (n0 >= sparc_pmu->max_hw_events)
  1213. goto out;
  1214. cpuc->event[n0] = event;
  1215. cpuc->events[n0] = event->hw.event_base;
  1216. cpuc->current_idx[n0] = PIC_NO_INDEX;
  1217. event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
  1218. if (!(ef_flags & PERF_EF_START))
  1219. event->hw.state |= PERF_HES_ARCH;
  1220. /*
  1221. * If group events scheduling transaction was started,
  1222. * skip the schedulability test here, it will be performed
  1223. * at commit time(->commit_txn) as a whole
  1224. */
  1225. if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
  1226. goto nocheck;
  1227. if (check_excludes(cpuc->event, n0, 1))
  1228. goto out;
  1229. if (sparc_check_constraints(cpuc->event, cpuc->events, n0 + 1))
  1230. goto out;
  1231. nocheck:
  1232. cpuc->n_events++;
  1233. cpuc->n_added++;
  1234. ret = 0;
  1235. out:
  1236. local_irq_restore(flags);
  1237. return ret;
  1238. }
  1239. static int sparc_pmu_event_init(struct perf_event *event)
  1240. {
  1241. struct perf_event_attr *attr = &event->attr;
  1242. struct perf_event *evts[MAX_HWEVENTS];
  1243. struct hw_perf_event *hwc = &event->hw;
  1244. unsigned long events[MAX_HWEVENTS];
  1245. int current_idx_dmy[MAX_HWEVENTS];
  1246. const struct perf_event_map *pmap;
  1247. int n;
  1248. if (atomic_read(&nmi_active) < 0)
  1249. return -ENODEV;
  1250. /* does not support taken branch sampling */
  1251. if (has_branch_stack(event))
  1252. return -EOPNOTSUPP;
  1253. switch (attr->type) {
  1254. case PERF_TYPE_HARDWARE:
  1255. if (attr->config >= sparc_pmu->max_events)
  1256. return -EINVAL;
  1257. pmap = sparc_pmu->event_map(attr->config);
  1258. break;
  1259. case PERF_TYPE_HW_CACHE:
  1260. pmap = sparc_map_cache_event(attr->config);
  1261. if (IS_ERR(pmap))
  1262. return PTR_ERR(pmap);
  1263. break;
  1264. case PERF_TYPE_RAW:
  1265. pmap = NULL;
  1266. break;
  1267. default:
  1268. return -ENOENT;
  1269. }
  1270. if (pmap) {
  1271. hwc->event_base = perf_event_encode(pmap);
  1272. } else {
  1273. /*
  1274. * User gives us "(encoding << 16) | pic_mask" for
  1275. * PERF_TYPE_RAW events.
  1276. */
  1277. hwc->event_base = attr->config;
  1278. }
  1279. /* We save the enable bits in the config_base. */
  1280. hwc->config_base = sparc_pmu->irq_bit;
  1281. if (!attr->exclude_user)
  1282. hwc->config_base |= sparc_pmu->user_bit;
  1283. if (!attr->exclude_kernel)
  1284. hwc->config_base |= sparc_pmu->priv_bit;
  1285. if (!attr->exclude_hv)
  1286. hwc->config_base |= sparc_pmu->hv_bit;
  1287. n = 0;
  1288. if (event->group_leader != event) {
  1289. n = collect_events(event->group_leader,
  1290. sparc_pmu->max_hw_events - 1,
  1291. evts, events, current_idx_dmy);
  1292. if (n < 0)
  1293. return -EINVAL;
  1294. }
  1295. events[n] = hwc->event_base;
  1296. evts[n] = event;
  1297. if (check_excludes(evts, n, 1))
  1298. return -EINVAL;
  1299. if (sparc_check_constraints(evts, events, n + 1))
  1300. return -EINVAL;
  1301. hwc->idx = PIC_NO_INDEX;
  1302. /* Try to do all error checking before this point, as unwinding
  1303. * state after grabbing the PMC is difficult.
  1304. */
  1305. perf_event_grab_pmc();
  1306. event->destroy = hw_perf_event_destroy;
  1307. if (!hwc->sample_period) {
  1308. hwc->sample_period = MAX_PERIOD;
  1309. hwc->last_period = hwc->sample_period;
  1310. local64_set(&hwc->period_left, hwc->sample_period);
  1311. }
  1312. return 0;
  1313. }
  1314. /*
  1315. * Start group events scheduling transaction
  1316. * Set the flag to make pmu::enable() not perform the
  1317. * schedulability test, it will be performed at commit time
  1318. */
  1319. static void sparc_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
  1320. {
  1321. struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
  1322. WARN_ON_ONCE(cpuhw->txn_flags); /* txn already in flight */
  1323. cpuhw->txn_flags = txn_flags;
  1324. if (txn_flags & ~PERF_PMU_TXN_ADD)
  1325. return;
  1326. perf_pmu_disable(pmu);
  1327. }
  1328. /*
  1329. * Stop group events scheduling transaction
  1330. * Clear the flag and pmu::enable() will perform the
  1331. * schedulability test.
  1332. */
  1333. static void sparc_pmu_cancel_txn(struct pmu *pmu)
  1334. {
  1335. struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
  1336. unsigned int txn_flags;
  1337. WARN_ON_ONCE(!cpuhw->txn_flags); /* no txn in flight */
  1338. txn_flags = cpuhw->txn_flags;
  1339. cpuhw->txn_flags = 0;
  1340. if (txn_flags & ~PERF_PMU_TXN_ADD)
  1341. return;
  1342. perf_pmu_enable(pmu);
  1343. }
  1344. /*
  1345. * Commit group events scheduling transaction
  1346. * Perform the group schedulability test as a whole
  1347. * Return 0 if success
  1348. */
  1349. static int sparc_pmu_commit_txn(struct pmu *pmu)
  1350. {
  1351. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  1352. int n;
  1353. if (!sparc_pmu)
  1354. return -EINVAL;
  1355. WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
  1356. if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) {
  1357. cpuc->txn_flags = 0;
  1358. return 0;
  1359. }
  1360. n = cpuc->n_events;
  1361. if (check_excludes(cpuc->event, 0, n))
  1362. return -EINVAL;
  1363. if (sparc_check_constraints(cpuc->event, cpuc->events, n))
  1364. return -EAGAIN;
  1365. cpuc->txn_flags = 0;
  1366. perf_pmu_enable(pmu);
  1367. return 0;
  1368. }
  1369. static struct pmu pmu = {
  1370. .pmu_enable = sparc_pmu_enable,
  1371. .pmu_disable = sparc_pmu_disable,
  1372. .event_init = sparc_pmu_event_init,
  1373. .add = sparc_pmu_add,
  1374. .del = sparc_pmu_del,
  1375. .start = sparc_pmu_start,
  1376. .stop = sparc_pmu_stop,
  1377. .read = sparc_pmu_read,
  1378. .start_txn = sparc_pmu_start_txn,
  1379. .cancel_txn = sparc_pmu_cancel_txn,
  1380. .commit_txn = sparc_pmu_commit_txn,
  1381. };
  1382. void perf_event_print_debug(void)
  1383. {
  1384. unsigned long flags;
  1385. int cpu, i;
  1386. if (!sparc_pmu)
  1387. return;
  1388. local_irq_save(flags);
  1389. cpu = smp_processor_id();
  1390. pr_info("\n");
  1391. for (i = 0; i < sparc_pmu->num_pcrs; i++)
  1392. pr_info("CPU#%d: PCR%d[%016llx]\n",
  1393. cpu, i, pcr_ops->read_pcr(i));
  1394. for (i = 0; i < sparc_pmu->num_pic_regs; i++)
  1395. pr_info("CPU#%d: PIC%d[%016llx]\n",
  1396. cpu, i, pcr_ops->read_pic(i));
  1397. local_irq_restore(flags);
  1398. }
  1399. static int __kprobes perf_event_nmi_handler(struct notifier_block *self,
  1400. unsigned long cmd, void *__args)
  1401. {
  1402. struct die_args *args = __args;
  1403. struct perf_sample_data data;
  1404. struct cpu_hw_events *cpuc;
  1405. struct pt_regs *regs;
  1406. u64 finish_clock;
  1407. u64 start_clock;
  1408. int i;
  1409. if (!atomic_read(&active_events))
  1410. return NOTIFY_DONE;
  1411. switch (cmd) {
  1412. case DIE_NMI:
  1413. break;
  1414. default:
  1415. return NOTIFY_DONE;
  1416. }
  1417. start_clock = sched_clock();
  1418. regs = args->regs;
  1419. cpuc = this_cpu_ptr(&cpu_hw_events);
  1420. /* If the PMU has the TOE IRQ enable bits, we need to do a
  1421. * dummy write to the %pcr to clear the overflow bits and thus
  1422. * the interrupt.
  1423. *
  1424. * Do this before we peek at the counters to determine
  1425. * overflow so we don't lose any events.
  1426. */
  1427. if (sparc_pmu->irq_bit &&
  1428. sparc_pmu->num_pcrs == 1)
  1429. pcr_ops->write_pcr(0, cpuc->pcr[0]);
  1430. for (i = 0; i < cpuc->n_events; i++) {
  1431. struct perf_event *event = cpuc->event[i];
  1432. int idx = cpuc->current_idx[i];
  1433. struct hw_perf_event *hwc;
  1434. u64 val;
  1435. if (sparc_pmu->irq_bit &&
  1436. sparc_pmu->num_pcrs > 1)
  1437. pcr_ops->write_pcr(idx, cpuc->pcr[idx]);
  1438. hwc = &event->hw;
  1439. val = sparc_perf_event_update(event, hwc, idx);
  1440. if (val & (1ULL << 31))
  1441. continue;
  1442. perf_sample_data_init(&data, 0, hwc->last_period);
  1443. if (!sparc_perf_event_set_period(event, hwc, idx))
  1444. continue;
  1445. if (perf_event_overflow(event, &data, regs))
  1446. sparc_pmu_stop(event, 0);
  1447. }
  1448. finish_clock = sched_clock();
  1449. perf_sample_event_took(finish_clock - start_clock);
  1450. return NOTIFY_STOP;
  1451. }
  1452. static __read_mostly struct notifier_block perf_event_nmi_notifier = {
  1453. .notifier_call = perf_event_nmi_handler,
  1454. };
  1455. static bool __init supported_pmu(void)
  1456. {
  1457. if (!strcmp(sparc_pmu_type, "ultra3") ||
  1458. !strcmp(sparc_pmu_type, "ultra3+") ||
  1459. !strcmp(sparc_pmu_type, "ultra3i") ||
  1460. !strcmp(sparc_pmu_type, "ultra4+")) {
  1461. sparc_pmu = &ultra3_pmu;
  1462. return true;
  1463. }
  1464. if (!strcmp(sparc_pmu_type, "niagara")) {
  1465. sparc_pmu = &niagara1_pmu;
  1466. return true;
  1467. }
  1468. if (!strcmp(sparc_pmu_type, "niagara2") ||
  1469. !strcmp(sparc_pmu_type, "niagara3")) {
  1470. sparc_pmu = &niagara2_pmu;
  1471. return true;
  1472. }
  1473. if (!strcmp(sparc_pmu_type, "niagara4") ||
  1474. !strcmp(sparc_pmu_type, "niagara5")) {
  1475. sparc_pmu = &niagara4_pmu;
  1476. return true;
  1477. }
  1478. if (!strcmp(sparc_pmu_type, "sparc-m7")) {
  1479. sparc_pmu = &sparc_m7_pmu;
  1480. return true;
  1481. }
  1482. return false;
  1483. }
  1484. static int __init init_hw_perf_events(void)
  1485. {
  1486. int err;
  1487. pr_info("Performance events: ");
  1488. err = pcr_arch_init();
  1489. if (err || !supported_pmu()) {
  1490. pr_cont("No support for PMU type '%s'\n", sparc_pmu_type);
  1491. return 0;
  1492. }
  1493. pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type);
  1494. perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
  1495. register_die_notifier(&perf_event_nmi_notifier);
  1496. return 0;
  1497. }
  1498. pure_initcall(init_hw_perf_events);
  1499. void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
  1500. struct pt_regs *regs)
  1501. {
  1502. unsigned long ksp, fp;
  1503. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  1504. int graph = 0;
  1505. #endif
  1506. stack_trace_flush();
  1507. perf_callchain_store(entry, regs->tpc);
  1508. ksp = regs->u_regs[UREG_I6];
  1509. fp = ksp + STACK_BIAS;
  1510. do {
  1511. struct sparc_stackf *sf;
  1512. struct pt_regs *regs;
  1513. unsigned long pc;
  1514. if (!kstack_valid(current_thread_info(), fp))
  1515. break;
  1516. sf = (struct sparc_stackf *) fp;
  1517. regs = (struct pt_regs *) (sf + 1);
  1518. if (kstack_is_trap_frame(current_thread_info(), regs)) {
  1519. if (user_mode(regs))
  1520. break;
  1521. pc = regs->tpc;
  1522. fp = regs->u_regs[UREG_I6] + STACK_BIAS;
  1523. } else {
  1524. pc = sf->callers_pc;
  1525. fp = (unsigned long)sf->fp + STACK_BIAS;
  1526. }
  1527. perf_callchain_store(entry, pc);
  1528. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  1529. if ((pc + 8UL) == (unsigned long) &return_to_handler) {
  1530. struct ftrace_ret_stack *ret_stack;
  1531. ret_stack = ftrace_graph_get_ret_stack(current,
  1532. graph);
  1533. if (ret_stack) {
  1534. pc = ret_stack->ret;
  1535. perf_callchain_store(entry, pc);
  1536. graph++;
  1537. }
  1538. }
  1539. #endif
  1540. } while (entry->nr < entry->max_stack);
  1541. }
  1542. static inline int
  1543. valid_user_frame(const void __user *fp, unsigned long size)
  1544. {
  1545. /* addresses should be at least 4-byte aligned */
  1546. if (((unsigned long) fp) & 3)
  1547. return 0;
  1548. return (__range_not_ok(fp, size, TASK_SIZE) == 0);
  1549. }
  1550. static void perf_callchain_user_64(struct perf_callchain_entry_ctx *entry,
  1551. struct pt_regs *regs)
  1552. {
  1553. unsigned long ufp;
  1554. ufp = regs->u_regs[UREG_FP] + STACK_BIAS;
  1555. do {
  1556. struct sparc_stackf __user *usf;
  1557. struct sparc_stackf sf;
  1558. unsigned long pc;
  1559. usf = (struct sparc_stackf __user *)ufp;
  1560. if (!valid_user_frame(usf, sizeof(sf)))
  1561. break;
  1562. if (__copy_from_user_inatomic(&sf, usf, sizeof(sf)))
  1563. break;
  1564. pc = sf.callers_pc;
  1565. ufp = (unsigned long)sf.fp + STACK_BIAS;
  1566. perf_callchain_store(entry, pc);
  1567. } while (entry->nr < entry->max_stack);
  1568. }
  1569. static void perf_callchain_user_32(struct perf_callchain_entry_ctx *entry,
  1570. struct pt_regs *regs)
  1571. {
  1572. unsigned long ufp;
  1573. ufp = regs->u_regs[UREG_FP] & 0xffffffffUL;
  1574. do {
  1575. unsigned long pc;
  1576. if (thread32_stack_is_64bit(ufp)) {
  1577. struct sparc_stackf __user *usf;
  1578. struct sparc_stackf sf;
  1579. ufp += STACK_BIAS;
  1580. usf = (struct sparc_stackf __user *)ufp;
  1581. if (__copy_from_user_inatomic(&sf, usf, sizeof(sf)))
  1582. break;
  1583. pc = sf.callers_pc & 0xffffffff;
  1584. ufp = ((unsigned long) sf.fp) & 0xffffffff;
  1585. } else {
  1586. struct sparc_stackf32 __user *usf;
  1587. struct sparc_stackf32 sf;
  1588. usf = (struct sparc_stackf32 __user *)ufp;
  1589. if (__copy_from_user_inatomic(&sf, usf, sizeof(sf)))
  1590. break;
  1591. pc = sf.callers_pc;
  1592. ufp = (unsigned long)sf.fp;
  1593. }
  1594. perf_callchain_store(entry, pc);
  1595. } while (entry->nr < entry->max_stack);
  1596. }
  1597. void
  1598. perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
  1599. {
  1600. u64 saved_fault_address = current_thread_info()->fault_address;
  1601. u8 saved_fault_code = get_thread_fault_code();
  1602. perf_callchain_store(entry, regs->tpc);
  1603. if (!current->mm)
  1604. return;
  1605. flushw_user();
  1606. pagefault_disable();
  1607. if (test_thread_flag(TIF_32BIT))
  1608. perf_callchain_user_32(entry, regs);
  1609. else
  1610. perf_callchain_user_64(entry, regs);
  1611. pagefault_enable();
  1612. set_thread_fault_code(saved_fault_code);
  1613. current_thread_info()->fault_address = saved_fault_address;
  1614. }