perf_event_v7.c 64 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
  4. *
  5. * ARMv7 support: Jean Pihet <jpihet@mvista.com>
  6. * 2010 (c) MontaVista Software, LLC.
  7. *
  8. * Copied from ARMv6 code, with the low level code inspired
  9. * by the ARMv7 Oprofile code.
  10. *
  11. * Cortex-A8 has up to 4 configurable performance counters and
  12. * a single cycle counter.
  13. * Cortex-A9 has up to 31 configurable performance counters and
  14. * a single cycle counter.
  15. *
  16. * All counters can be enabled/disabled and IRQ masked separately. The cycle
  17. * counter and all 4 performance counters together can be reset separately.
  18. */
  19. #ifdef CONFIG_CPU_V7
  20. #include <asm/cp15.h>
  21. #include <asm/cputype.h>
  22. #include <asm/irq_regs.h>
  23. #include <asm/vfp.h>
  24. #include "../vfp/vfpinstr.h"
  25. #include <linux/of.h>
  26. #include <linux/perf/arm_pmu.h>
  27. #include <linux/platform_device.h>
  28. /*
  29. * Common ARMv7 event types
  30. *
  31. * Note: An implementation may not be able to count all of these events
  32. * but the encodings are considered to be `reserved' in the case that
  33. * they are not available.
  34. */
  35. #define ARMV7_PERFCTR_PMNC_SW_INCR 0x00
  36. #define ARMV7_PERFCTR_L1_ICACHE_REFILL 0x01
  37. #define ARMV7_PERFCTR_ITLB_REFILL 0x02
  38. #define ARMV7_PERFCTR_L1_DCACHE_REFILL 0x03
  39. #define ARMV7_PERFCTR_L1_DCACHE_ACCESS 0x04
  40. #define ARMV7_PERFCTR_DTLB_REFILL 0x05
  41. #define ARMV7_PERFCTR_MEM_READ 0x06
  42. #define ARMV7_PERFCTR_MEM_WRITE 0x07
  43. #define ARMV7_PERFCTR_INSTR_EXECUTED 0x08
  44. #define ARMV7_PERFCTR_EXC_TAKEN 0x09
  45. #define ARMV7_PERFCTR_EXC_EXECUTED 0x0A
  46. #define ARMV7_PERFCTR_CID_WRITE 0x0B
  47. /*
  48. * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
  49. * It counts:
  50. * - all (taken) branch instructions,
  51. * - instructions that explicitly write the PC,
  52. * - exception generating instructions.
  53. */
  54. #define ARMV7_PERFCTR_PC_WRITE 0x0C
  55. #define ARMV7_PERFCTR_PC_IMM_BRANCH 0x0D
  56. #define ARMV7_PERFCTR_PC_PROC_RETURN 0x0E
  57. #define ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS 0x0F
  58. #define ARMV7_PERFCTR_PC_BRANCH_MIS_PRED 0x10
  59. #define ARMV7_PERFCTR_CLOCK_CYCLES 0x11
  60. #define ARMV7_PERFCTR_PC_BRANCH_PRED 0x12
  61. /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
  62. #define ARMV7_PERFCTR_MEM_ACCESS 0x13
  63. #define ARMV7_PERFCTR_L1_ICACHE_ACCESS 0x14
  64. #define ARMV7_PERFCTR_L1_DCACHE_WB 0x15
  65. #define ARMV7_PERFCTR_L2_CACHE_ACCESS 0x16
  66. #define ARMV7_PERFCTR_L2_CACHE_REFILL 0x17
  67. #define ARMV7_PERFCTR_L2_CACHE_WB 0x18
  68. #define ARMV7_PERFCTR_BUS_ACCESS 0x19
  69. #define ARMV7_PERFCTR_MEM_ERROR 0x1A
  70. #define ARMV7_PERFCTR_INSTR_SPEC 0x1B
  71. #define ARMV7_PERFCTR_TTBR_WRITE 0x1C
  72. #define ARMV7_PERFCTR_BUS_CYCLES 0x1D
  73. #define ARMV7_PERFCTR_CPU_CYCLES 0xFF
  74. /* ARMv7 Cortex-A8 specific event types */
  75. #define ARMV7_A8_PERFCTR_L2_CACHE_ACCESS 0x43
  76. #define ARMV7_A8_PERFCTR_L2_CACHE_REFILL 0x44
  77. #define ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS 0x50
  78. #define ARMV7_A8_PERFCTR_STALL_ISIDE 0x56
  79. /* ARMv7 Cortex-A9 specific event types */
  80. #define ARMV7_A9_PERFCTR_INSTR_CORE_RENAME 0x68
  81. #define ARMV7_A9_PERFCTR_STALL_ICACHE 0x60
  82. #define ARMV7_A9_PERFCTR_STALL_DISPATCH 0x66
  83. /* ARMv7 Cortex-A5 specific event types */
  84. #define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL 0xc2
  85. #define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP 0xc3
  86. /* ARMv7 Cortex-A15 specific event types */
  87. #define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ 0x40
  88. #define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE 0x41
  89. #define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ 0x42
  90. #define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE 0x43
  91. #define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ 0x4C
  92. #define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE 0x4D
  93. #define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ 0x50
  94. #define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE 0x51
  95. #define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ 0x52
  96. #define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE 0x53
  97. #define ARMV7_A15_PERFCTR_PC_WRITE_SPEC 0x76
  98. /* ARMv7 Cortex-A12 specific event types */
  99. #define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ 0x40
  100. #define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE 0x41
  101. #define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ 0x50
  102. #define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE 0x51
  103. #define ARMV7_A12_PERFCTR_PC_WRITE_SPEC 0x76
  104. #define ARMV7_A12_PERFCTR_PF_TLB_REFILL 0xe7
  105. /* ARMv7 Krait specific event types */
  106. #define KRAIT_PMRESR0_GROUP0 0xcc
  107. #define KRAIT_PMRESR1_GROUP0 0xd0
  108. #define KRAIT_PMRESR2_GROUP0 0xd4
  109. #define KRAIT_VPMRESR0_GROUP0 0xd8
  110. #define KRAIT_PERFCTR_L1_ICACHE_ACCESS 0x10011
  111. #define KRAIT_PERFCTR_L1_ICACHE_MISS 0x10010
  112. #define KRAIT_PERFCTR_L1_ITLB_ACCESS 0x12222
  113. #define KRAIT_PERFCTR_L1_DTLB_ACCESS 0x12210
  114. /* ARMv7 Scorpion specific event types */
  115. #define SCORPION_LPM0_GROUP0 0x4c
  116. #define SCORPION_LPM1_GROUP0 0x50
  117. #define SCORPION_LPM2_GROUP0 0x54
  118. #define SCORPION_L2LPM_GROUP0 0x58
  119. #define SCORPION_VLPM_GROUP0 0x5c
  120. #define SCORPION_ICACHE_ACCESS 0x10053
  121. #define SCORPION_ICACHE_MISS 0x10052
  122. #define SCORPION_DTLB_ACCESS 0x12013
  123. #define SCORPION_DTLB_MISS 0x12012
  124. #define SCORPION_ITLB_MISS 0x12021
  125. /*
  126. * Cortex-A8 HW events mapping
  127. *
  128. * The hardware events that we support. We do support cache operations but
  129. * we have harvard caches and no way to combine instruction and data
  130. * accesses/misses in hardware.
  131. */
  132. static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
  133. PERF_MAP_ALL_UNSUPPORTED,
  134. [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
  135. [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
  136. [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  137. [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  138. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
  139. [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  140. [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
  141. };
  142. static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  143. [PERF_COUNT_HW_CACHE_OP_MAX]
  144. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  145. PERF_CACHE_MAP_ALL_UNSUPPORTED,
  146. /*
  147. * The performance counters don't differentiate between read and write
  148. * accesses/misses so this isn't strictly correct, but it's the best we
  149. * can do. Writes and reads get combined.
  150. */
  151. [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  152. [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  153. [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  154. [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  155. [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
  156. [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
  157. [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
  158. [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
  159. [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
  160. [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
  161. [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  162. [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  163. [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
  164. [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
  165. [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  166. [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  167. [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  168. [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  169. };
  170. /*
  171. * Cortex-A9 HW events mapping
  172. */
  173. static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
  174. PERF_MAP_ALL_UNSUPPORTED,
  175. [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
  176. [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
  177. [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  178. [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  179. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
  180. [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  181. [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
  182. [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV7_A9_PERFCTR_STALL_DISPATCH,
  183. };
  184. static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  185. [PERF_COUNT_HW_CACHE_OP_MAX]
  186. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  187. PERF_CACHE_MAP_ALL_UNSUPPORTED,
  188. /*
  189. * The performance counters don't differentiate between read and write
  190. * accesses/misses so this isn't strictly correct, but it's the best we
  191. * can do. Writes and reads get combined.
  192. */
  193. [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  194. [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  195. [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  196. [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  197. [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
  198. [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  199. [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  200. [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
  201. [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
  202. [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  203. [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  204. [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  205. [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  206. };
  207. /*
  208. * Cortex-A5 HW events mapping
  209. */
  210. static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
  211. PERF_MAP_ALL_UNSUPPORTED,
  212. [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
  213. [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
  214. [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  215. [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  216. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
  217. [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  218. };
  219. static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  220. [PERF_COUNT_HW_CACHE_OP_MAX]
  221. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  222. PERF_CACHE_MAP_ALL_UNSUPPORTED,
  223. [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  224. [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  225. [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  226. [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  227. [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
  228. [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
  229. [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
  230. [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
  231. /*
  232. * The prefetch counters don't differentiate between the I side and the
  233. * D side.
  234. */
  235. [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
  236. [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
  237. [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  238. [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  239. [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
  240. [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
  241. [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  242. [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  243. [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  244. [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  245. };
  246. /*
  247. * Cortex-A15 HW events mapping
  248. */
  249. static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
  250. PERF_MAP_ALL_UNSUPPORTED,
  251. [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
  252. [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
  253. [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  254. [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  255. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
  256. [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  257. [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
  258. };
  259. static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  260. [PERF_COUNT_HW_CACHE_OP_MAX]
  261. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  262. PERF_CACHE_MAP_ALL_UNSUPPORTED,
  263. [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
  264. [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
  265. [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
  266. [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
  267. /*
  268. * Not all performance counters differentiate between read and write
  269. * accesses/misses so we're not always strictly correct, but it's the
  270. * best we can do. Writes and reads get combined in these cases.
  271. */
  272. [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
  273. [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
  274. [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
  275. [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
  276. [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
  277. [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
  278. [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
  279. [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
  280. [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
  281. [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
  282. [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  283. [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  284. [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  285. [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  286. };
  287. /*
  288. * Cortex-A7 HW events mapping
  289. */
  290. static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
  291. PERF_MAP_ALL_UNSUPPORTED,
  292. [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
  293. [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
  294. [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  295. [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  296. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
  297. [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  298. [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
  299. };
  300. static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  301. [PERF_COUNT_HW_CACHE_OP_MAX]
  302. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  303. PERF_CACHE_MAP_ALL_UNSUPPORTED,
  304. /*
  305. * The performance counters don't differentiate between read and write
  306. * accesses/misses so this isn't strictly correct, but it's the best we
  307. * can do. Writes and reads get combined.
  308. */
  309. [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  310. [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  311. [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  312. [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  313. [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
  314. [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
  315. [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS,
  316. [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
  317. [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS,
  318. [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
  319. [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  320. [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  321. [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
  322. [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
  323. [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  324. [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  325. [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  326. [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  327. };
  328. /*
  329. * Cortex-A12 HW events mapping
  330. */
  331. static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = {
  332. PERF_MAP_ALL_UNSUPPORTED,
  333. [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
  334. [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
  335. [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  336. [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  337. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A12_PERFCTR_PC_WRITE_SPEC,
  338. [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  339. [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
  340. };
  341. static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  342. [PERF_COUNT_HW_CACHE_OP_MAX]
  343. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  344. PERF_CACHE_MAP_ALL_UNSUPPORTED,
  345. [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ,
  346. [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  347. [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE,
  348. [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  349. /*
  350. * Not all performance counters differentiate between read and write
  351. * accesses/misses so we're not always strictly correct, but it's the
  352. * best we can do. Writes and reads get combined in these cases.
  353. */
  354. [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
  355. [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
  356. [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ,
  357. [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
  358. [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE,
  359. [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
  360. [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  361. [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
  362. [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A12_PERFCTR_PF_TLB_REFILL,
  363. [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
  364. [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
  365. [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  366. [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  367. [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  368. [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  369. };
  370. /*
  371. * Krait HW events mapping
  372. */
  373. static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = {
  374. PERF_MAP_ALL_UNSUPPORTED,
  375. [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
  376. [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
  377. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
  378. [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  379. [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
  380. };
  381. static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = {
  382. PERF_MAP_ALL_UNSUPPORTED,
  383. [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
  384. [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
  385. [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  386. [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
  387. };
  388. static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  389. [PERF_COUNT_HW_CACHE_OP_MAX]
  390. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  391. PERF_CACHE_MAP_ALL_UNSUPPORTED,
  392. /*
  393. * The performance counters don't differentiate between read and write
  394. * accesses/misses so this isn't strictly correct, but it's the best we
  395. * can do. Writes and reads get combined.
  396. */
  397. [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  398. [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  399. [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  400. [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  401. [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ICACHE_ACCESS,
  402. [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = KRAIT_PERFCTR_L1_ICACHE_MISS,
  403. [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,
  404. [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,
  405. [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,
  406. [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,
  407. [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  408. [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  409. [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  410. [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  411. };
  412. /*
  413. * Scorpion HW events mapping
  414. */
  415. static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = {
  416. PERF_MAP_ALL_UNSUPPORTED,
  417. [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
  418. [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
  419. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
  420. [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  421. [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
  422. };
  423. static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  424. [PERF_COUNT_HW_CACHE_OP_MAX]
  425. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  426. PERF_CACHE_MAP_ALL_UNSUPPORTED,
  427. /*
  428. * The performance counters don't differentiate between read and write
  429. * accesses/misses so this isn't strictly correct, but it's the best we
  430. * can do. Writes and reads get combined.
  431. */
  432. [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  433. [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  434. [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
  435. [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
  436. [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS,
  437. [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS,
  438. /*
  439. * Only ITLB misses and DTLB refills are supported. If users want the
  440. * DTLB refills misses a raw counter must be used.
  441. */
  442. [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
  443. [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
  444. [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
  445. [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
  446. [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
  447. [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
  448. [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  449. [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  450. [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
  451. [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
  452. };
  453. PMU_FORMAT_ATTR(event, "config:0-7");
  454. static struct attribute *armv7_pmu_format_attrs[] = {
  455. &format_attr_event.attr,
  456. NULL,
  457. };
  458. static struct attribute_group armv7_pmu_format_attr_group = {
  459. .name = "format",
  460. .attrs = armv7_pmu_format_attrs,
  461. };
  462. #define ARMV7_EVENT_ATTR_RESOLVE(m) #m
  463. #define ARMV7_EVENT_ATTR(name, config) \
  464. PMU_EVENT_ATTR_STRING(name, armv7_event_attr_##name, \
  465. "event=" ARMV7_EVENT_ATTR_RESOLVE(config))
  466. ARMV7_EVENT_ATTR(sw_incr, ARMV7_PERFCTR_PMNC_SW_INCR);
  467. ARMV7_EVENT_ATTR(l1i_cache_refill, ARMV7_PERFCTR_L1_ICACHE_REFILL);
  468. ARMV7_EVENT_ATTR(l1i_tlb_refill, ARMV7_PERFCTR_ITLB_REFILL);
  469. ARMV7_EVENT_ATTR(l1d_cache_refill, ARMV7_PERFCTR_L1_DCACHE_REFILL);
  470. ARMV7_EVENT_ATTR(l1d_cache, ARMV7_PERFCTR_L1_DCACHE_ACCESS);
  471. ARMV7_EVENT_ATTR(l1d_tlb_refill, ARMV7_PERFCTR_DTLB_REFILL);
  472. ARMV7_EVENT_ATTR(ld_retired, ARMV7_PERFCTR_MEM_READ);
  473. ARMV7_EVENT_ATTR(st_retired, ARMV7_PERFCTR_MEM_WRITE);
  474. ARMV7_EVENT_ATTR(inst_retired, ARMV7_PERFCTR_INSTR_EXECUTED);
  475. ARMV7_EVENT_ATTR(exc_taken, ARMV7_PERFCTR_EXC_TAKEN);
  476. ARMV7_EVENT_ATTR(exc_return, ARMV7_PERFCTR_EXC_EXECUTED);
  477. ARMV7_EVENT_ATTR(cid_write_retired, ARMV7_PERFCTR_CID_WRITE);
  478. ARMV7_EVENT_ATTR(pc_write_retired, ARMV7_PERFCTR_PC_WRITE);
  479. ARMV7_EVENT_ATTR(br_immed_retired, ARMV7_PERFCTR_PC_IMM_BRANCH);
  480. ARMV7_EVENT_ATTR(br_return_retired, ARMV7_PERFCTR_PC_PROC_RETURN);
  481. ARMV7_EVENT_ATTR(unaligned_ldst_retired, ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS);
  482. ARMV7_EVENT_ATTR(br_mis_pred, ARMV7_PERFCTR_PC_BRANCH_MIS_PRED);
  483. ARMV7_EVENT_ATTR(cpu_cycles, ARMV7_PERFCTR_CLOCK_CYCLES);
  484. ARMV7_EVENT_ATTR(br_pred, ARMV7_PERFCTR_PC_BRANCH_PRED);
  485. static struct attribute *armv7_pmuv1_event_attrs[] = {
  486. &armv7_event_attr_sw_incr.attr.attr,
  487. &armv7_event_attr_l1i_cache_refill.attr.attr,
  488. &armv7_event_attr_l1i_tlb_refill.attr.attr,
  489. &armv7_event_attr_l1d_cache_refill.attr.attr,
  490. &armv7_event_attr_l1d_cache.attr.attr,
  491. &armv7_event_attr_l1d_tlb_refill.attr.attr,
  492. &armv7_event_attr_ld_retired.attr.attr,
  493. &armv7_event_attr_st_retired.attr.attr,
  494. &armv7_event_attr_inst_retired.attr.attr,
  495. &armv7_event_attr_exc_taken.attr.attr,
  496. &armv7_event_attr_exc_return.attr.attr,
  497. &armv7_event_attr_cid_write_retired.attr.attr,
  498. &armv7_event_attr_pc_write_retired.attr.attr,
  499. &armv7_event_attr_br_immed_retired.attr.attr,
  500. &armv7_event_attr_br_return_retired.attr.attr,
  501. &armv7_event_attr_unaligned_ldst_retired.attr.attr,
  502. &armv7_event_attr_br_mis_pred.attr.attr,
  503. &armv7_event_attr_cpu_cycles.attr.attr,
  504. &armv7_event_attr_br_pred.attr.attr,
  505. NULL,
  506. };
  507. static struct attribute_group armv7_pmuv1_events_attr_group = {
  508. .name = "events",
  509. .attrs = armv7_pmuv1_event_attrs,
  510. };
  511. ARMV7_EVENT_ATTR(mem_access, ARMV7_PERFCTR_MEM_ACCESS);
  512. ARMV7_EVENT_ATTR(l1i_cache, ARMV7_PERFCTR_L1_ICACHE_ACCESS);
  513. ARMV7_EVENT_ATTR(l1d_cache_wb, ARMV7_PERFCTR_L1_DCACHE_WB);
  514. ARMV7_EVENT_ATTR(l2d_cache, ARMV7_PERFCTR_L2_CACHE_ACCESS);
  515. ARMV7_EVENT_ATTR(l2d_cache_refill, ARMV7_PERFCTR_L2_CACHE_REFILL);
  516. ARMV7_EVENT_ATTR(l2d_cache_wb, ARMV7_PERFCTR_L2_CACHE_WB);
  517. ARMV7_EVENT_ATTR(bus_access, ARMV7_PERFCTR_BUS_ACCESS);
  518. ARMV7_EVENT_ATTR(memory_error, ARMV7_PERFCTR_MEM_ERROR);
  519. ARMV7_EVENT_ATTR(inst_spec, ARMV7_PERFCTR_INSTR_SPEC);
  520. ARMV7_EVENT_ATTR(ttbr_write_retired, ARMV7_PERFCTR_TTBR_WRITE);
  521. ARMV7_EVENT_ATTR(bus_cycles, ARMV7_PERFCTR_BUS_CYCLES);
  522. static struct attribute *armv7_pmuv2_event_attrs[] = {
  523. &armv7_event_attr_sw_incr.attr.attr,
  524. &armv7_event_attr_l1i_cache_refill.attr.attr,
  525. &armv7_event_attr_l1i_tlb_refill.attr.attr,
  526. &armv7_event_attr_l1d_cache_refill.attr.attr,
  527. &armv7_event_attr_l1d_cache.attr.attr,
  528. &armv7_event_attr_l1d_tlb_refill.attr.attr,
  529. &armv7_event_attr_ld_retired.attr.attr,
  530. &armv7_event_attr_st_retired.attr.attr,
  531. &armv7_event_attr_inst_retired.attr.attr,
  532. &armv7_event_attr_exc_taken.attr.attr,
  533. &armv7_event_attr_exc_return.attr.attr,
  534. &armv7_event_attr_cid_write_retired.attr.attr,
  535. &armv7_event_attr_pc_write_retired.attr.attr,
  536. &armv7_event_attr_br_immed_retired.attr.attr,
  537. &armv7_event_attr_br_return_retired.attr.attr,
  538. &armv7_event_attr_unaligned_ldst_retired.attr.attr,
  539. &armv7_event_attr_br_mis_pred.attr.attr,
  540. &armv7_event_attr_cpu_cycles.attr.attr,
  541. &armv7_event_attr_br_pred.attr.attr,
  542. &armv7_event_attr_mem_access.attr.attr,
  543. &armv7_event_attr_l1i_cache.attr.attr,
  544. &armv7_event_attr_l1d_cache_wb.attr.attr,
  545. &armv7_event_attr_l2d_cache.attr.attr,
  546. &armv7_event_attr_l2d_cache_refill.attr.attr,
  547. &armv7_event_attr_l2d_cache_wb.attr.attr,
  548. &armv7_event_attr_bus_access.attr.attr,
  549. &armv7_event_attr_memory_error.attr.attr,
  550. &armv7_event_attr_inst_spec.attr.attr,
  551. &armv7_event_attr_ttbr_write_retired.attr.attr,
  552. &armv7_event_attr_bus_cycles.attr.attr,
  553. NULL,
  554. };
  555. static struct attribute_group armv7_pmuv2_events_attr_group = {
  556. .name = "events",
  557. .attrs = armv7_pmuv2_event_attrs,
  558. };
  559. /*
  560. * Perf Events' indices
  561. */
  562. #define ARMV7_IDX_CYCLE_COUNTER 0
  563. #define ARMV7_IDX_COUNTER0 1
  564. #define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \
  565. (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
  566. #define ARMV7_MAX_COUNTERS 32
  567. #define ARMV7_COUNTER_MASK (ARMV7_MAX_COUNTERS - 1)
  568. /*
  569. * ARMv7 low level PMNC access
  570. */
  571. /*
  572. * Perf Event to low level counters mapping
  573. */
  574. #define ARMV7_IDX_TO_COUNTER(x) \
  575. (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
  576. /*
  577. * Per-CPU PMNC: config reg
  578. */
  579. #define ARMV7_PMNC_E (1 << 0) /* Enable all counters */
  580. #define ARMV7_PMNC_P (1 << 1) /* Reset all counters */
  581. #define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */
  582. #define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */
  583. #define ARMV7_PMNC_X (1 << 4) /* Export to ETM */
  584. #define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/
  585. #define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */
  586. #define ARMV7_PMNC_N_MASK 0x1f
  587. #define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */
  588. /*
  589. * FLAG: counters overflow flag status reg
  590. */
  591. #define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */
  592. #define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK
  593. /*
  594. * PMXEVTYPER: Event selection reg
  595. */
  596. #define ARMV7_EVTYPE_MASK 0xc80000ff /* Mask for writable bits */
  597. #define ARMV7_EVTYPE_EVENT 0xff /* Mask for EVENT bits */
  598. /*
  599. * Event filters for PMUv2
  600. */
  601. #define ARMV7_EXCLUDE_PL1 BIT(31)
  602. #define ARMV7_EXCLUDE_USER BIT(30)
  603. #define ARMV7_INCLUDE_HYP BIT(27)
  604. /*
  605. * Secure debug enable reg
  606. */
  607. #define ARMV7_SDER_SUNIDEN BIT(1) /* Permit non-invasive debug */
  608. static inline u32 armv7_pmnc_read(void)
  609. {
  610. u32 val;
  611. asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
  612. return val;
  613. }
  614. static inline void armv7_pmnc_write(u32 val)
  615. {
  616. val &= ARMV7_PMNC_MASK;
  617. isb();
  618. asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
  619. }
  620. static inline int armv7_pmnc_has_overflowed(u32 pmnc)
  621. {
  622. return pmnc & ARMV7_OVERFLOWED_MASK;
  623. }
  624. static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx)
  625. {
  626. return idx >= ARMV7_IDX_CYCLE_COUNTER &&
  627. idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu);
  628. }
  629. static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
  630. {
  631. return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx));
  632. }
  633. static inline void armv7_pmnc_select_counter(int idx)
  634. {
  635. u32 counter = ARMV7_IDX_TO_COUNTER(idx);
  636. asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
  637. isb();
  638. }
  639. static inline u64 armv7pmu_read_counter(struct perf_event *event)
  640. {
  641. struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
  642. struct hw_perf_event *hwc = &event->hw;
  643. int idx = hwc->idx;
  644. u32 value = 0;
  645. if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
  646. pr_err("CPU%u reading wrong counter %d\n",
  647. smp_processor_id(), idx);
  648. } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
  649. asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
  650. } else {
  651. armv7_pmnc_select_counter(idx);
  652. asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
  653. }
  654. return value;
  655. }
  656. static inline void armv7pmu_write_counter(struct perf_event *event, u64 value)
  657. {
  658. struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
  659. struct hw_perf_event *hwc = &event->hw;
  660. int idx = hwc->idx;
  661. if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
  662. pr_err("CPU%u writing wrong counter %d\n",
  663. smp_processor_id(), idx);
  664. } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
  665. asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" ((u32)value));
  666. } else {
  667. armv7_pmnc_select_counter(idx);
  668. asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" ((u32)value));
  669. }
  670. }
  671. static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
  672. {
  673. armv7_pmnc_select_counter(idx);
  674. val &= ARMV7_EVTYPE_MASK;
  675. asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
  676. }
  677. static inline void armv7_pmnc_enable_counter(int idx)
  678. {
  679. u32 counter = ARMV7_IDX_TO_COUNTER(idx);
  680. asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
  681. }
  682. static inline void armv7_pmnc_disable_counter(int idx)
  683. {
  684. u32 counter = ARMV7_IDX_TO_COUNTER(idx);
  685. asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
  686. }
  687. static inline void armv7_pmnc_enable_intens(int idx)
  688. {
  689. u32 counter = ARMV7_IDX_TO_COUNTER(idx);
  690. asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
  691. }
  692. static inline void armv7_pmnc_disable_intens(int idx)
  693. {
  694. u32 counter = ARMV7_IDX_TO_COUNTER(idx);
  695. asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
  696. isb();
  697. /* Clear the overflow flag in case an interrupt is pending. */
  698. asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
  699. isb();
  700. }
  701. static inline u32 armv7_pmnc_getreset_flags(void)
  702. {
  703. u32 val;
  704. /* Read */
  705. asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
  706. /* Write to clear flags */
  707. val &= ARMV7_FLAG_MASK;
  708. asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
  709. return val;
  710. }
  711. #ifdef DEBUG
  712. static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu)
  713. {
  714. u32 val;
  715. unsigned int cnt;
  716. pr_info("PMNC registers dump:\n");
  717. asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
  718. pr_info("PMNC =0x%08x\n", val);
  719. asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
  720. pr_info("CNTENS=0x%08x\n", val);
  721. asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
  722. pr_info("INTENS=0x%08x\n", val);
  723. asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
  724. pr_info("FLAGS =0x%08x\n", val);
  725. asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
  726. pr_info("SELECT=0x%08x\n", val);
  727. asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
  728. pr_info("CCNT =0x%08x\n", val);
  729. for (cnt = ARMV7_IDX_COUNTER0;
  730. cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) {
  731. armv7_pmnc_select_counter(cnt);
  732. asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
  733. pr_info("CNT[%d] count =0x%08x\n",
  734. ARMV7_IDX_TO_COUNTER(cnt), val);
  735. asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
  736. pr_info("CNT[%d] evtsel=0x%08x\n",
  737. ARMV7_IDX_TO_COUNTER(cnt), val);
  738. }
  739. }
  740. #endif
  741. static void armv7pmu_enable_event(struct perf_event *event)
  742. {
  743. unsigned long flags;
  744. struct hw_perf_event *hwc = &event->hw;
  745. struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
  746. struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
  747. int idx = hwc->idx;
  748. if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
  749. pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
  750. smp_processor_id(), idx);
  751. return;
  752. }
  753. /*
  754. * Enable counter and interrupt, and set the counter to count
  755. * the event that we're interested in.
  756. */
  757. raw_spin_lock_irqsave(&events->pmu_lock, flags);
  758. /*
  759. * Disable counter
  760. */
  761. armv7_pmnc_disable_counter(idx);
  762. /*
  763. * Set event (if destined for PMNx counters)
  764. * We only need to set the event for the cycle counter if we
  765. * have the ability to perform event filtering.
  766. */
  767. if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
  768. armv7_pmnc_write_evtsel(idx, hwc->config_base);
  769. /*
  770. * Enable interrupt for this counter
  771. */
  772. armv7_pmnc_enable_intens(idx);
  773. /*
  774. * Enable counter
  775. */
  776. armv7_pmnc_enable_counter(idx);
  777. raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
  778. }
  779. static void armv7pmu_disable_event(struct perf_event *event)
  780. {
  781. unsigned long flags;
  782. struct hw_perf_event *hwc = &event->hw;
  783. struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
  784. struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
  785. int idx = hwc->idx;
  786. if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
  787. pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
  788. smp_processor_id(), idx);
  789. return;
  790. }
  791. /*
  792. * Disable counter and interrupt
  793. */
  794. raw_spin_lock_irqsave(&events->pmu_lock, flags);
  795. /*
  796. * Disable counter
  797. */
  798. armv7_pmnc_disable_counter(idx);
  799. /*
  800. * Disable interrupt for this counter
  801. */
  802. armv7_pmnc_disable_intens(idx);
  803. raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
  804. }
  805. static irqreturn_t armv7pmu_handle_irq(struct arm_pmu *cpu_pmu)
  806. {
  807. u32 pmnc;
  808. struct perf_sample_data data;
  809. struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
  810. struct pt_regs *regs;
  811. int idx;
  812. /*
  813. * Get and reset the IRQ flags
  814. */
  815. pmnc = armv7_pmnc_getreset_flags();
  816. /*
  817. * Did an overflow occur?
  818. */
  819. if (!armv7_pmnc_has_overflowed(pmnc))
  820. return IRQ_NONE;
  821. /*
  822. * Handle the counter(s) overflow(s)
  823. */
  824. regs = get_irq_regs();
  825. for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
  826. struct perf_event *event = cpuc->events[idx];
  827. struct hw_perf_event *hwc;
  828. /* Ignore if we don't have an event. */
  829. if (!event)
  830. continue;
  831. /*
  832. * We have a single interrupt for all counters. Check that
  833. * each counter has overflowed before we process it.
  834. */
  835. if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
  836. continue;
  837. hwc = &event->hw;
  838. armpmu_event_update(event);
  839. perf_sample_data_init(&data, 0, hwc->last_period);
  840. if (!armpmu_event_set_period(event))
  841. continue;
  842. if (perf_event_overflow(event, &data, regs))
  843. cpu_pmu->disable(event);
  844. }
  845. /*
  846. * Handle the pending perf events.
  847. *
  848. * Note: this call *must* be run with interrupts disabled. For
  849. * platforms that can have the PMU interrupts raised as an NMI, this
  850. * will not work.
  851. */
  852. irq_work_run();
  853. return IRQ_HANDLED;
  854. }
  855. static void armv7pmu_start(struct arm_pmu *cpu_pmu)
  856. {
  857. unsigned long flags;
  858. struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
  859. raw_spin_lock_irqsave(&events->pmu_lock, flags);
  860. /* Enable all counters */
  861. armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
  862. raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
  863. }
  864. static void armv7pmu_stop(struct arm_pmu *cpu_pmu)
  865. {
  866. unsigned long flags;
  867. struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
  868. raw_spin_lock_irqsave(&events->pmu_lock, flags);
  869. /* Disable all counters */
  870. armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
  871. raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
  872. }
  873. static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
  874. struct perf_event *event)
  875. {
  876. int idx;
  877. struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
  878. struct hw_perf_event *hwc = &event->hw;
  879. unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;
  880. /* Always place a cycle counter into the cycle counter. */
  881. if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
  882. if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
  883. return -EAGAIN;
  884. return ARMV7_IDX_CYCLE_COUNTER;
  885. }
  886. /*
  887. * For anything other than a cycle counter, try and use
  888. * the events counters
  889. */
  890. for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
  891. if (!test_and_set_bit(idx, cpuc->used_mask))
  892. return idx;
  893. }
  894. /* The counters are all in use. */
  895. return -EAGAIN;
  896. }
  897. static void armv7pmu_clear_event_idx(struct pmu_hw_events *cpuc,
  898. struct perf_event *event)
  899. {
  900. clear_bit(event->hw.idx, cpuc->used_mask);
  901. }
  902. /*
  903. * Add an event filter to a given event. This will only work for PMUv2 PMUs.
  904. */
  905. static int armv7pmu_set_event_filter(struct hw_perf_event *event,
  906. struct perf_event_attr *attr)
  907. {
  908. unsigned long config_base = 0;
  909. if (attr->exclude_idle)
  910. return -EPERM;
  911. if (attr->exclude_user)
  912. config_base |= ARMV7_EXCLUDE_USER;
  913. if (attr->exclude_kernel)
  914. config_base |= ARMV7_EXCLUDE_PL1;
  915. if (!attr->exclude_hv)
  916. config_base |= ARMV7_INCLUDE_HYP;
  917. /*
  918. * Install the filter into config_base as this is used to
  919. * construct the event type.
  920. */
  921. event->config_base = config_base;
  922. return 0;
  923. }
  924. static void armv7pmu_reset(void *info)
  925. {
  926. struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
  927. u32 idx, nb_cnt = cpu_pmu->num_events, val;
  928. if (cpu_pmu->secure_access) {
  929. asm volatile("mrc p15, 0, %0, c1, c1, 1" : "=r" (val));
  930. val |= ARMV7_SDER_SUNIDEN;
  931. asm volatile("mcr p15, 0, %0, c1, c1, 1" : : "r" (val));
  932. }
  933. /* The counter and interrupt enable registers are unknown at reset. */
  934. for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
  935. armv7_pmnc_disable_counter(idx);
  936. armv7_pmnc_disable_intens(idx);
  937. }
  938. /* Initialize & Reset PMNC: C and P bits */
  939. armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
  940. }
  941. static int armv7_a8_map_event(struct perf_event *event)
  942. {
  943. return armpmu_map_event(event, &armv7_a8_perf_map,
  944. &armv7_a8_perf_cache_map, 0xFF);
  945. }
  946. static int armv7_a9_map_event(struct perf_event *event)
  947. {
  948. return armpmu_map_event(event, &armv7_a9_perf_map,
  949. &armv7_a9_perf_cache_map, 0xFF);
  950. }
  951. static int armv7_a5_map_event(struct perf_event *event)
  952. {
  953. return armpmu_map_event(event, &armv7_a5_perf_map,
  954. &armv7_a5_perf_cache_map, 0xFF);
  955. }
  956. static int armv7_a15_map_event(struct perf_event *event)
  957. {
  958. return armpmu_map_event(event, &armv7_a15_perf_map,
  959. &armv7_a15_perf_cache_map, 0xFF);
  960. }
  961. static int armv7_a7_map_event(struct perf_event *event)
  962. {
  963. return armpmu_map_event(event, &armv7_a7_perf_map,
  964. &armv7_a7_perf_cache_map, 0xFF);
  965. }
  966. static int armv7_a12_map_event(struct perf_event *event)
  967. {
  968. return armpmu_map_event(event, &armv7_a12_perf_map,
  969. &armv7_a12_perf_cache_map, 0xFF);
  970. }
  971. static int krait_map_event(struct perf_event *event)
  972. {
  973. return armpmu_map_event(event, &krait_perf_map,
  974. &krait_perf_cache_map, 0xFFFFF);
  975. }
  976. static int krait_map_event_no_branch(struct perf_event *event)
  977. {
  978. return armpmu_map_event(event, &krait_perf_map_no_branch,
  979. &krait_perf_cache_map, 0xFFFFF);
  980. }
  981. static int scorpion_map_event(struct perf_event *event)
  982. {
  983. return armpmu_map_event(event, &scorpion_perf_map,
  984. &scorpion_perf_cache_map, 0xFFFFF);
  985. }
  986. static void armv7pmu_init(struct arm_pmu *cpu_pmu)
  987. {
  988. cpu_pmu->handle_irq = armv7pmu_handle_irq;
  989. cpu_pmu->enable = armv7pmu_enable_event;
  990. cpu_pmu->disable = armv7pmu_disable_event;
  991. cpu_pmu->read_counter = armv7pmu_read_counter;
  992. cpu_pmu->write_counter = armv7pmu_write_counter;
  993. cpu_pmu->get_event_idx = armv7pmu_get_event_idx;
  994. cpu_pmu->clear_event_idx = armv7pmu_clear_event_idx;
  995. cpu_pmu->start = armv7pmu_start;
  996. cpu_pmu->stop = armv7pmu_stop;
  997. cpu_pmu->reset = armv7pmu_reset;
  998. };
  999. static void armv7_read_num_pmnc_events(void *info)
  1000. {
  1001. int *nb_cnt = info;
  1002. /* Read the nb of CNTx counters supported from PMNC */
  1003. *nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
  1004. /* Add the CPU cycles counter */
  1005. *nb_cnt += 1;
  1006. }
  1007. static int armv7_probe_num_events(struct arm_pmu *arm_pmu)
  1008. {
  1009. return smp_call_function_any(&arm_pmu->supported_cpus,
  1010. armv7_read_num_pmnc_events,
  1011. &arm_pmu->num_events, 1);
  1012. }
  1013. static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
  1014. {
  1015. armv7pmu_init(cpu_pmu);
  1016. cpu_pmu->name = "armv7_cortex_a8";
  1017. cpu_pmu->map_event = armv7_a8_map_event;
  1018. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
  1019. &armv7_pmuv1_events_attr_group;
  1020. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
  1021. &armv7_pmu_format_attr_group;
  1022. return armv7_probe_num_events(cpu_pmu);
  1023. }
  1024. static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
  1025. {
  1026. armv7pmu_init(cpu_pmu);
  1027. cpu_pmu->name = "armv7_cortex_a9";
  1028. cpu_pmu->map_event = armv7_a9_map_event;
  1029. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
  1030. &armv7_pmuv1_events_attr_group;
  1031. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
  1032. &armv7_pmu_format_attr_group;
  1033. return armv7_probe_num_events(cpu_pmu);
  1034. }
  1035. static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
  1036. {
  1037. armv7pmu_init(cpu_pmu);
  1038. cpu_pmu->name = "armv7_cortex_a5";
  1039. cpu_pmu->map_event = armv7_a5_map_event;
  1040. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
  1041. &armv7_pmuv1_events_attr_group;
  1042. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
  1043. &armv7_pmu_format_attr_group;
  1044. return armv7_probe_num_events(cpu_pmu);
  1045. }
  1046. static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
  1047. {
  1048. armv7pmu_init(cpu_pmu);
  1049. cpu_pmu->name = "armv7_cortex_a15";
  1050. cpu_pmu->map_event = armv7_a15_map_event;
  1051. cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
  1052. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
  1053. &armv7_pmuv2_events_attr_group;
  1054. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
  1055. &armv7_pmu_format_attr_group;
  1056. return armv7_probe_num_events(cpu_pmu);
  1057. }
  1058. static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
  1059. {
  1060. armv7pmu_init(cpu_pmu);
  1061. cpu_pmu->name = "armv7_cortex_a7";
  1062. cpu_pmu->map_event = armv7_a7_map_event;
  1063. cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
  1064. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
  1065. &armv7_pmuv2_events_attr_group;
  1066. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
  1067. &armv7_pmu_format_attr_group;
  1068. return armv7_probe_num_events(cpu_pmu);
  1069. }
  1070. static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
  1071. {
  1072. armv7pmu_init(cpu_pmu);
  1073. cpu_pmu->name = "armv7_cortex_a12";
  1074. cpu_pmu->map_event = armv7_a12_map_event;
  1075. cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
  1076. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
  1077. &armv7_pmuv2_events_attr_group;
  1078. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
  1079. &armv7_pmu_format_attr_group;
  1080. return armv7_probe_num_events(cpu_pmu);
  1081. }
  1082. static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
  1083. {
  1084. int ret = armv7_a12_pmu_init(cpu_pmu);
  1085. cpu_pmu->name = "armv7_cortex_a17";
  1086. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
  1087. &armv7_pmuv2_events_attr_group;
  1088. cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
  1089. &armv7_pmu_format_attr_group;
  1090. return ret;
  1091. }
  1092. /*
  1093. * Krait Performance Monitor Region Event Selection Register (PMRESRn)
  1094. *
  1095. * 31 30 24 16 8 0
  1096. * +--------------------------------+
  1097. * PMRESR0 | EN | CC | CC | CC | CC | N = 1, R = 0
  1098. * +--------------------------------+
  1099. * PMRESR1 | EN | CC | CC | CC | CC | N = 1, R = 1
  1100. * +--------------------------------+
  1101. * PMRESR2 | EN | CC | CC | CC | CC | N = 1, R = 2
  1102. * +--------------------------------+
  1103. * VPMRESR0 | EN | CC | CC | CC | CC | N = 2, R = ?
  1104. * +--------------------------------+
  1105. * EN | G=3 | G=2 | G=1 | G=0
  1106. *
  1107. * Event Encoding:
  1108. *
  1109. * hwc->config_base = 0xNRCCG
  1110. *
  1111. * N = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR)
  1112. * R = region register
  1113. * CC = class of events the group G is choosing from
  1114. * G = group or particular event
  1115. *
  1116. * Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2
  1117. *
  1118. * A region (R) corresponds to a piece of the CPU (execution unit, instruction
  1119. * unit, etc.) while the event code (CC) corresponds to a particular class of
  1120. * events (interrupts for example). An event code is broken down into
  1121. * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
  1122. * example).
  1123. */
  1124. #define KRAIT_EVENT (1 << 16)
  1125. #define VENUM_EVENT (2 << 16)
  1126. #define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT)
  1127. #define PMRESRn_EN BIT(31)
  1128. #define EVENT_REGION(event) (((event) >> 12) & 0xf) /* R */
  1129. #define EVENT_GROUP(event) ((event) & 0xf) /* G */
  1130. #define EVENT_CODE(event) (((event) >> 4) & 0xff) /* CC */
  1131. #define EVENT_VENUM(event) (!!(event & VENUM_EVENT)) /* N=2 */
  1132. #define EVENT_CPU(event) (!!(event & KRAIT_EVENT)) /* N=1 */
  1133. static u32 krait_read_pmresrn(int n)
  1134. {
  1135. u32 val;
  1136. switch (n) {
  1137. case 0:
  1138. asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val));
  1139. break;
  1140. case 1:
  1141. asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val));
  1142. break;
  1143. case 2:
  1144. asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val));
  1145. break;
  1146. default:
  1147. BUG(); /* Should be validated in krait_pmu_get_event_idx() */
  1148. }
  1149. return val;
  1150. }
  1151. static void krait_write_pmresrn(int n, u32 val)
  1152. {
  1153. switch (n) {
  1154. case 0:
  1155. asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val));
  1156. break;
  1157. case 1:
  1158. asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val));
  1159. break;
  1160. case 2:
  1161. asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val));
  1162. break;
  1163. default:
  1164. BUG(); /* Should be validated in krait_pmu_get_event_idx() */
  1165. }
  1166. }
  1167. static u32 venum_read_pmresr(void)
  1168. {
  1169. u32 val;
  1170. asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val));
  1171. return val;
  1172. }
  1173. static void venum_write_pmresr(u32 val)
  1174. {
  1175. asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val));
  1176. }
  1177. static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val)
  1178. {
  1179. u32 venum_new_val;
  1180. u32 fp_new_val;
  1181. BUG_ON(preemptible());
  1182. /* CPACR Enable CP10 and CP11 access */
  1183. *venum_orig_val = get_copro_access();
  1184. venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11);
  1185. set_copro_access(venum_new_val);
  1186. /* Enable FPEXC */
  1187. *fp_orig_val = fmrx(FPEXC);
  1188. fp_new_val = *fp_orig_val | FPEXC_EN;
  1189. fmxr(FPEXC, fp_new_val);
  1190. }
  1191. static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val)
  1192. {
  1193. BUG_ON(preemptible());
  1194. /* Restore FPEXC */
  1195. fmxr(FPEXC, fp_orig_val);
  1196. isb();
  1197. /* Restore CPACR */
  1198. set_copro_access(venum_orig_val);
  1199. }
  1200. static u32 krait_get_pmresrn_event(unsigned int region)
  1201. {
  1202. static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0,
  1203. KRAIT_PMRESR1_GROUP0,
  1204. KRAIT_PMRESR2_GROUP0 };
  1205. return pmresrn_table[region];
  1206. }
  1207. static void krait_evt_setup(int idx, u32 config_base)
  1208. {
  1209. u32 val;
  1210. u32 mask;
  1211. u32 vval, fval;
  1212. unsigned int region = EVENT_REGION(config_base);
  1213. unsigned int group = EVENT_GROUP(config_base);
  1214. unsigned int code = EVENT_CODE(config_base);
  1215. unsigned int group_shift;
  1216. bool venum_event = EVENT_VENUM(config_base);
  1217. group_shift = group * 8;
  1218. mask = 0xff << group_shift;
  1219. /* Configure evtsel for the region and group */
  1220. if (venum_event)
  1221. val = KRAIT_VPMRESR0_GROUP0;
  1222. else
  1223. val = krait_get_pmresrn_event(region);
  1224. val += group;
  1225. /* Mix in mode-exclusion bits */
  1226. val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
  1227. armv7_pmnc_write_evtsel(idx, val);
  1228. if (venum_event) {
  1229. venum_pre_pmresr(&vval, &fval);
  1230. val = venum_read_pmresr();
  1231. val &= ~mask;
  1232. val |= code << group_shift;
  1233. val |= PMRESRn_EN;
  1234. venum_write_pmresr(val);
  1235. venum_post_pmresr(vval, fval);
  1236. } else {
  1237. val = krait_read_pmresrn(region);
  1238. val &= ~mask;
  1239. val |= code << group_shift;
  1240. val |= PMRESRn_EN;
  1241. krait_write_pmresrn(region, val);
  1242. }
  1243. }
  1244. static u32 clear_pmresrn_group(u32 val, int group)
  1245. {
  1246. u32 mask;
  1247. int group_shift;
  1248. group_shift = group * 8;
  1249. mask = 0xff << group_shift;
  1250. val &= ~mask;
  1251. /* Don't clear enable bit if entire region isn't disabled */
  1252. if (val & ~PMRESRn_EN)
  1253. return val |= PMRESRn_EN;
  1254. return 0;
  1255. }
  1256. static void krait_clearpmu(u32 config_base)
  1257. {
  1258. u32 val;
  1259. u32 vval, fval;
  1260. unsigned int region = EVENT_REGION(config_base);
  1261. unsigned int group = EVENT_GROUP(config_base);
  1262. bool venum_event = EVENT_VENUM(config_base);
  1263. if (venum_event) {
  1264. venum_pre_pmresr(&vval, &fval);
  1265. val = venum_read_pmresr();
  1266. val = clear_pmresrn_group(val, group);
  1267. venum_write_pmresr(val);
  1268. venum_post_pmresr(vval, fval);
  1269. } else {
  1270. val = krait_read_pmresrn(region);
  1271. val = clear_pmresrn_group(val, group);
  1272. krait_write_pmresrn(region, val);
  1273. }
  1274. }
  1275. static void krait_pmu_disable_event(struct perf_event *event)
  1276. {
  1277. unsigned long flags;
  1278. struct hw_perf_event *hwc = &event->hw;
  1279. int idx = hwc->idx;
  1280. struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
  1281. struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
  1282. /* Disable counter and interrupt */
  1283. raw_spin_lock_irqsave(&events->pmu_lock, flags);
  1284. /* Disable counter */
  1285. armv7_pmnc_disable_counter(idx);
  1286. /*
  1287. * Clear pmresr code (if destined for PMNx counters)
  1288. */
  1289. if (hwc->config_base & KRAIT_EVENT_MASK)
  1290. krait_clearpmu(hwc->config_base);
  1291. /* Disable interrupt for this counter */
  1292. armv7_pmnc_disable_intens(idx);
  1293. raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
  1294. }
  1295. static void krait_pmu_enable_event(struct perf_event *event)
  1296. {
  1297. unsigned long flags;
  1298. struct hw_perf_event *hwc = &event->hw;
  1299. int idx = hwc->idx;
  1300. struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
  1301. struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
  1302. /*
  1303. * Enable counter and interrupt, and set the counter to count
  1304. * the event that we're interested in.
  1305. */
  1306. raw_spin_lock_irqsave(&events->pmu_lock, flags);
  1307. /* Disable counter */
  1308. armv7_pmnc_disable_counter(idx);
  1309. /*
  1310. * Set event (if destined for PMNx counters)
  1311. * We set the event for the cycle counter because we
  1312. * have the ability to perform event filtering.
  1313. */
  1314. if (hwc->config_base & KRAIT_EVENT_MASK)
  1315. krait_evt_setup(idx, hwc->config_base);
  1316. else
  1317. armv7_pmnc_write_evtsel(idx, hwc->config_base);
  1318. /* Enable interrupt for this counter */
  1319. armv7_pmnc_enable_intens(idx);
  1320. /* Enable counter */
  1321. armv7_pmnc_enable_counter(idx);
  1322. raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
  1323. }
  1324. static void krait_pmu_reset(void *info)
  1325. {
  1326. u32 vval, fval;
  1327. struct arm_pmu *cpu_pmu = info;
  1328. u32 idx, nb_cnt = cpu_pmu->num_events;
  1329. armv7pmu_reset(info);
  1330. /* Clear all pmresrs */
  1331. krait_write_pmresrn(0, 0);
  1332. krait_write_pmresrn(1, 0);
  1333. krait_write_pmresrn(2, 0);
  1334. venum_pre_pmresr(&vval, &fval);
  1335. venum_write_pmresr(0);
  1336. venum_post_pmresr(vval, fval);
  1337. /* Reset PMxEVNCTCR to sane default */
  1338. for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
  1339. armv7_pmnc_select_counter(idx);
  1340. asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
  1341. }
  1342. }
  1343. static int krait_event_to_bit(struct perf_event *event, unsigned int region,
  1344. unsigned int group)
  1345. {
  1346. int bit;
  1347. struct hw_perf_event *hwc = &event->hw;
  1348. struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
  1349. if (hwc->config_base & VENUM_EVENT)
  1350. bit = KRAIT_VPMRESR0_GROUP0;
  1351. else
  1352. bit = krait_get_pmresrn_event(region);
  1353. bit -= krait_get_pmresrn_event(0);
  1354. bit += group;
  1355. /*
  1356. * Lower bits are reserved for use by the counters (see
  1357. * armv7pmu_get_event_idx() for more info)
  1358. */
  1359. bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
  1360. return bit;
  1361. }
  1362. /*
  1363. * We check for column exclusion constraints here.
  1364. * Two events cant use the same group within a pmresr register.
  1365. */
  1366. static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc,
  1367. struct perf_event *event)
  1368. {
  1369. int idx;
  1370. int bit = -1;
  1371. struct hw_perf_event *hwc = &event->hw;
  1372. unsigned int region = EVENT_REGION(hwc->config_base);
  1373. unsigned int code = EVENT_CODE(hwc->config_base);
  1374. unsigned int group = EVENT_GROUP(hwc->config_base);
  1375. bool venum_event = EVENT_VENUM(hwc->config_base);
  1376. bool krait_event = EVENT_CPU(hwc->config_base);
  1377. if (venum_event || krait_event) {
  1378. /* Ignore invalid events */
  1379. if (group > 3 || region > 2)
  1380. return -EINVAL;
  1381. if (venum_event && (code & 0xe0))
  1382. return -EINVAL;
  1383. bit = krait_event_to_bit(event, region, group);
  1384. if (test_and_set_bit(bit, cpuc->used_mask))
  1385. return -EAGAIN;
  1386. }
  1387. idx = armv7pmu_get_event_idx(cpuc, event);
  1388. if (idx < 0 && bit >= 0)
  1389. clear_bit(bit, cpuc->used_mask);
  1390. return idx;
  1391. }
  1392. static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
  1393. struct perf_event *event)
  1394. {
  1395. int bit;
  1396. struct hw_perf_event *hwc = &event->hw;
  1397. unsigned int region = EVENT_REGION(hwc->config_base);
  1398. unsigned int group = EVENT_GROUP(hwc->config_base);
  1399. bool venum_event = EVENT_VENUM(hwc->config_base);
  1400. bool krait_event = EVENT_CPU(hwc->config_base);
  1401. armv7pmu_clear_event_idx(cpuc, event);
  1402. if (venum_event || krait_event) {
  1403. bit = krait_event_to_bit(event, region, group);
  1404. clear_bit(bit, cpuc->used_mask);
  1405. }
  1406. }
  1407. static int krait_pmu_init(struct arm_pmu *cpu_pmu)
  1408. {
  1409. armv7pmu_init(cpu_pmu);
  1410. cpu_pmu->name = "armv7_krait";
  1411. /* Some early versions of Krait don't support PC write events */
  1412. if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,
  1413. "qcom,no-pc-write"))
  1414. cpu_pmu->map_event = krait_map_event_no_branch;
  1415. else
  1416. cpu_pmu->map_event = krait_map_event;
  1417. cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
  1418. cpu_pmu->reset = krait_pmu_reset;
  1419. cpu_pmu->enable = krait_pmu_enable_event;
  1420. cpu_pmu->disable = krait_pmu_disable_event;
  1421. cpu_pmu->get_event_idx = krait_pmu_get_event_idx;
  1422. cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;
  1423. return armv7_probe_num_events(cpu_pmu);
  1424. }
  1425. /*
  1426. * Scorpion Local Performance Monitor Register (LPMn)
  1427. *
  1428. * 31 30 24 16 8 0
  1429. * +--------------------------------+
  1430. * LPM0 | EN | CC | CC | CC | CC | N = 1, R = 0
  1431. * +--------------------------------+
  1432. * LPM1 | EN | CC | CC | CC | CC | N = 1, R = 1
  1433. * +--------------------------------+
  1434. * LPM2 | EN | CC | CC | CC | CC | N = 1, R = 2
  1435. * +--------------------------------+
  1436. * L2LPM | EN | CC | CC | CC | CC | N = 1, R = 3
  1437. * +--------------------------------+
  1438. * VLPM | EN | CC | CC | CC | CC | N = 2, R = ?
  1439. * +--------------------------------+
  1440. * EN | G=3 | G=2 | G=1 | G=0
  1441. *
  1442. *
  1443. * Event Encoding:
  1444. *
  1445. * hwc->config_base = 0xNRCCG
  1446. *
  1447. * N = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM)
  1448. * R = region register
  1449. * CC = class of events the group G is choosing from
  1450. * G = group or particular event
  1451. *
  1452. * Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2
  1453. *
  1454. * A region (R) corresponds to a piece of the CPU (execution unit, instruction
  1455. * unit, etc.) while the event code (CC) corresponds to a particular class of
  1456. * events (interrupts for example). An event code is broken down into
  1457. * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
  1458. * example).
  1459. */
  1460. static u32 scorpion_read_pmresrn(int n)
  1461. {
  1462. u32 val;
  1463. switch (n) {
  1464. case 0:
  1465. asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val));
  1466. break;
  1467. case 1:
  1468. asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val));
  1469. break;
  1470. case 2:
  1471. asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val));
  1472. break;
  1473. case 3:
  1474. asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val));
  1475. break;
  1476. default:
  1477. BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
  1478. }
  1479. return val;
  1480. }
  1481. static void scorpion_write_pmresrn(int n, u32 val)
  1482. {
  1483. switch (n) {
  1484. case 0:
  1485. asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val));
  1486. break;
  1487. case 1:
  1488. asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val));
  1489. break;
  1490. case 2:
  1491. asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val));
  1492. break;
  1493. case 3:
  1494. asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val));
  1495. break;
  1496. default:
  1497. BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
  1498. }
  1499. }
  1500. static u32 scorpion_get_pmresrn_event(unsigned int region)
  1501. {
  1502. static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0,
  1503. SCORPION_LPM1_GROUP0,
  1504. SCORPION_LPM2_GROUP0,
  1505. SCORPION_L2LPM_GROUP0 };
  1506. return pmresrn_table[region];
  1507. }
  1508. static void scorpion_evt_setup(int idx, u32 config_base)
  1509. {
  1510. u32 val;
  1511. u32 mask;
  1512. u32 vval, fval;
  1513. unsigned int region = EVENT_REGION(config_base);
  1514. unsigned int group = EVENT_GROUP(config_base);
  1515. unsigned int code = EVENT_CODE(config_base);
  1516. unsigned int group_shift;
  1517. bool venum_event = EVENT_VENUM(config_base);
  1518. group_shift = group * 8;
  1519. mask = 0xff << group_shift;
  1520. /* Configure evtsel for the region and group */
  1521. if (venum_event)
  1522. val = SCORPION_VLPM_GROUP0;
  1523. else
  1524. val = scorpion_get_pmresrn_event(region);
  1525. val += group;
  1526. /* Mix in mode-exclusion bits */
  1527. val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
  1528. armv7_pmnc_write_evtsel(idx, val);
  1529. asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
  1530. if (venum_event) {
  1531. venum_pre_pmresr(&vval, &fval);
  1532. val = venum_read_pmresr();
  1533. val &= ~mask;
  1534. val |= code << group_shift;
  1535. val |= PMRESRn_EN;
  1536. venum_write_pmresr(val);
  1537. venum_post_pmresr(vval, fval);
  1538. } else {
  1539. val = scorpion_read_pmresrn(region);
  1540. val &= ~mask;
  1541. val |= code << group_shift;
  1542. val |= PMRESRn_EN;
  1543. scorpion_write_pmresrn(region, val);
  1544. }
  1545. }
  1546. static void scorpion_clearpmu(u32 config_base)
  1547. {
  1548. u32 val;
  1549. u32 vval, fval;
  1550. unsigned int region = EVENT_REGION(config_base);
  1551. unsigned int group = EVENT_GROUP(config_base);
  1552. bool venum_event = EVENT_VENUM(config_base);
  1553. if (venum_event) {
  1554. venum_pre_pmresr(&vval, &fval);
  1555. val = venum_read_pmresr();
  1556. val = clear_pmresrn_group(val, group);
  1557. venum_write_pmresr(val);
  1558. venum_post_pmresr(vval, fval);
  1559. } else {
  1560. val = scorpion_read_pmresrn(region);
  1561. val = clear_pmresrn_group(val, group);
  1562. scorpion_write_pmresrn(region, val);
  1563. }
  1564. }
  1565. static void scorpion_pmu_disable_event(struct perf_event *event)
  1566. {
  1567. unsigned long flags;
  1568. struct hw_perf_event *hwc = &event->hw;
  1569. int idx = hwc->idx;
  1570. struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
  1571. struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
  1572. /* Disable counter and interrupt */
  1573. raw_spin_lock_irqsave(&events->pmu_lock, flags);
  1574. /* Disable counter */
  1575. armv7_pmnc_disable_counter(idx);
  1576. /*
  1577. * Clear pmresr code (if destined for PMNx counters)
  1578. */
  1579. if (hwc->config_base & KRAIT_EVENT_MASK)
  1580. scorpion_clearpmu(hwc->config_base);
  1581. /* Disable interrupt for this counter */
  1582. armv7_pmnc_disable_intens(idx);
  1583. raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
  1584. }
  1585. static void scorpion_pmu_enable_event(struct perf_event *event)
  1586. {
  1587. unsigned long flags;
  1588. struct hw_perf_event *hwc = &event->hw;
  1589. int idx = hwc->idx;
  1590. struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
  1591. struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
  1592. /*
  1593. * Enable counter and interrupt, and set the counter to count
  1594. * the event that we're interested in.
  1595. */
  1596. raw_spin_lock_irqsave(&events->pmu_lock, flags);
  1597. /* Disable counter */
  1598. armv7_pmnc_disable_counter(idx);
  1599. /*
  1600. * Set event (if destined for PMNx counters)
  1601. * We don't set the event for the cycle counter because we
  1602. * don't have the ability to perform event filtering.
  1603. */
  1604. if (hwc->config_base & KRAIT_EVENT_MASK)
  1605. scorpion_evt_setup(idx, hwc->config_base);
  1606. else if (idx != ARMV7_IDX_CYCLE_COUNTER)
  1607. armv7_pmnc_write_evtsel(idx, hwc->config_base);
  1608. /* Enable interrupt for this counter */
  1609. armv7_pmnc_enable_intens(idx);
  1610. /* Enable counter */
  1611. armv7_pmnc_enable_counter(idx);
  1612. raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
  1613. }
  1614. static void scorpion_pmu_reset(void *info)
  1615. {
  1616. u32 vval, fval;
  1617. struct arm_pmu *cpu_pmu = info;
  1618. u32 idx, nb_cnt = cpu_pmu->num_events;
  1619. armv7pmu_reset(info);
  1620. /* Clear all pmresrs */
  1621. scorpion_write_pmresrn(0, 0);
  1622. scorpion_write_pmresrn(1, 0);
  1623. scorpion_write_pmresrn(2, 0);
  1624. scorpion_write_pmresrn(3, 0);
  1625. venum_pre_pmresr(&vval, &fval);
  1626. venum_write_pmresr(0);
  1627. venum_post_pmresr(vval, fval);
  1628. /* Reset PMxEVNCTCR to sane default */
  1629. for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
  1630. armv7_pmnc_select_counter(idx);
  1631. asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
  1632. }
  1633. }
  1634. static int scorpion_event_to_bit(struct perf_event *event, unsigned int region,
  1635. unsigned int group)
  1636. {
  1637. int bit;
  1638. struct hw_perf_event *hwc = &event->hw;
  1639. struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
  1640. if (hwc->config_base & VENUM_EVENT)
  1641. bit = SCORPION_VLPM_GROUP0;
  1642. else
  1643. bit = scorpion_get_pmresrn_event(region);
  1644. bit -= scorpion_get_pmresrn_event(0);
  1645. bit += group;
  1646. /*
  1647. * Lower bits are reserved for use by the counters (see
  1648. * armv7pmu_get_event_idx() for more info)
  1649. */
  1650. bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
  1651. return bit;
  1652. }
  1653. /*
  1654. * We check for column exclusion constraints here.
  1655. * Two events cant use the same group within a pmresr register.
  1656. */
  1657. static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc,
  1658. struct perf_event *event)
  1659. {
  1660. int idx;
  1661. int bit = -1;
  1662. struct hw_perf_event *hwc = &event->hw;
  1663. unsigned int region = EVENT_REGION(hwc->config_base);
  1664. unsigned int group = EVENT_GROUP(hwc->config_base);
  1665. bool venum_event = EVENT_VENUM(hwc->config_base);
  1666. bool scorpion_event = EVENT_CPU(hwc->config_base);
  1667. if (venum_event || scorpion_event) {
  1668. /* Ignore invalid events */
  1669. if (group > 3 || region > 3)
  1670. return -EINVAL;
  1671. bit = scorpion_event_to_bit(event, region, group);
  1672. if (test_and_set_bit(bit, cpuc->used_mask))
  1673. return -EAGAIN;
  1674. }
  1675. idx = armv7pmu_get_event_idx(cpuc, event);
  1676. if (idx < 0 && bit >= 0)
  1677. clear_bit(bit, cpuc->used_mask);
  1678. return idx;
  1679. }
  1680. static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
  1681. struct perf_event *event)
  1682. {
  1683. int bit;
  1684. struct hw_perf_event *hwc = &event->hw;
  1685. unsigned int region = EVENT_REGION(hwc->config_base);
  1686. unsigned int group = EVENT_GROUP(hwc->config_base);
  1687. bool venum_event = EVENT_VENUM(hwc->config_base);
  1688. bool scorpion_event = EVENT_CPU(hwc->config_base);
  1689. armv7pmu_clear_event_idx(cpuc, event);
  1690. if (venum_event || scorpion_event) {
  1691. bit = scorpion_event_to_bit(event, region, group);
  1692. clear_bit(bit, cpuc->used_mask);
  1693. }
  1694. }
  1695. static int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
  1696. {
  1697. armv7pmu_init(cpu_pmu);
  1698. cpu_pmu->name = "armv7_scorpion";
  1699. cpu_pmu->map_event = scorpion_map_event;
  1700. cpu_pmu->reset = scorpion_pmu_reset;
  1701. cpu_pmu->enable = scorpion_pmu_enable_event;
  1702. cpu_pmu->disable = scorpion_pmu_disable_event;
  1703. cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx;
  1704. cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
  1705. return armv7_probe_num_events(cpu_pmu);
  1706. }
  1707. static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
  1708. {
  1709. armv7pmu_init(cpu_pmu);
  1710. cpu_pmu->name = "armv7_scorpion_mp";
  1711. cpu_pmu->map_event = scorpion_map_event;
  1712. cpu_pmu->reset = scorpion_pmu_reset;
  1713. cpu_pmu->enable = scorpion_pmu_enable_event;
  1714. cpu_pmu->disable = scorpion_pmu_disable_event;
  1715. cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx;
  1716. cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
  1717. return armv7_probe_num_events(cpu_pmu);
  1718. }
  1719. static const struct of_device_id armv7_pmu_of_device_ids[] = {
  1720. {.compatible = "arm,cortex-a17-pmu", .data = armv7_a17_pmu_init},
  1721. {.compatible = "arm,cortex-a15-pmu", .data = armv7_a15_pmu_init},
  1722. {.compatible = "arm,cortex-a12-pmu", .data = armv7_a12_pmu_init},
  1723. {.compatible = "arm,cortex-a9-pmu", .data = armv7_a9_pmu_init},
  1724. {.compatible = "arm,cortex-a8-pmu", .data = armv7_a8_pmu_init},
  1725. {.compatible = "arm,cortex-a7-pmu", .data = armv7_a7_pmu_init},
  1726. {.compatible = "arm,cortex-a5-pmu", .data = armv7_a5_pmu_init},
  1727. {.compatible = "qcom,krait-pmu", .data = krait_pmu_init},
  1728. {.compatible = "qcom,scorpion-pmu", .data = scorpion_pmu_init},
  1729. {.compatible = "qcom,scorpion-mp-pmu", .data = scorpion_mp_pmu_init},
  1730. {},
  1731. };
  1732. static const struct pmu_probe_info armv7_pmu_probe_table[] = {
  1733. ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A8, armv7_a8_pmu_init),
  1734. ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A9, armv7_a9_pmu_init),
  1735. { /* sentinel value */ }
  1736. };
  1737. static int armv7_pmu_device_probe(struct platform_device *pdev)
  1738. {
  1739. return arm_pmu_device_probe(pdev, armv7_pmu_of_device_ids,
  1740. armv7_pmu_probe_table);
  1741. }
  1742. static struct platform_driver armv7_pmu_driver = {
  1743. .driver = {
  1744. .name = "armv7-pmu",
  1745. .of_match_table = armv7_pmu_of_device_ids,
  1746. .suppress_bind_attrs = true,
  1747. },
  1748. .probe = armv7_pmu_device_probe,
  1749. };
  1750. builtin_platform_driver(armv7_pmu_driver);
  1751. #endif /* CONFIG_CPU_V7 */