irq-gic-v3.c 58 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
  4. * Author: Marc Zyngier <marc.zyngier@arm.com>
  5. */
  6. #define pr_fmt(fmt) "GICv3: " fmt
  7. #include <linux/acpi.h>
  8. #include <linux/cpu.h>
  9. #include <linux/cpu_pm.h>
  10. #include <linux/delay.h>
  11. #include <linux/interrupt.h>
  12. #include <linux/irqdomain.h>
  13. #include <linux/of.h>
  14. #include <linux/of_address.h>
  15. #include <linux/of_irq.h>
  16. #include <linux/percpu.h>
  17. #include <linux/refcount.h>
  18. #include <linux/slab.h>
  19. #include <linux/syscore_ops.h>
  20. #include <linux/wakeup_reason.h>
  21. #include <trace/hooks/gic_v3.h>
  22. #include <linux/irqchip.h>
  23. #include <linux/irqchip/arm-gic-common.h>
  24. #include <linux/irqchip/arm-gic-v3.h>
  25. #include <linux/irqchip/irq-partition-percpu.h>
  26. #include <asm/cputype.h>
  27. #include <asm/exception.h>
  28. #include <asm/smp_plat.h>
  29. #include <asm/virt.h>
  30. #include <trace/hooks/gic.h>
  31. #include "irq-gic-common.h"
  32. #define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80)
  33. #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0)
  34. #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1)
  35. #define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1)
  36. struct redist_region {
  37. void __iomem *redist_base;
  38. phys_addr_t phys_base;
  39. bool single_redist;
  40. };
  41. static struct gic_chip_data gic_data __read_mostly;
  42. static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
  43. #define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
  44. #define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U)
  45. #define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer)
  46. /*
  47. * The behaviours of RPR and PMR registers differ depending on the value of
  48. * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the
  49. * distributor and redistributors depends on whether security is enabled in the
  50. * GIC.
  51. *
  52. * When security is enabled, non-secure priority values from the (re)distributor
  53. * are presented to the GIC CPUIF as follow:
  54. * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80;
  55. *
  56. * If SCR_EL3.FIQ == 1, the values writen to/read from PMR and RPR at non-secure
  57. * EL1 are subject to a similar operation thus matching the priorities presented
  58. * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0,
  59. * these values are unchanched by the GIC.
  60. *
  61. * see GICv3/GICv4 Architecture Specification (IHI0069D):
  62. * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt
  63. * priorities.
  64. * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1
  65. * interrupt.
  66. */
  67. static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);
  68. /*
  69. * Global static key controlling whether an update to PMR allowing more
  70. * interrupts requires to be propagated to the redistributor (DSB SY).
  71. * And this needs to be exported for modules to be able to enable
  72. * interrupts...
  73. */
  74. DEFINE_STATIC_KEY_FALSE(gic_pmr_sync);
  75. EXPORT_SYMBOL(gic_pmr_sync);
  76. DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities);
  77. EXPORT_SYMBOL(gic_nonsecure_priorities);
  78. /*
  79. * When the Non-secure world has access to group 0 interrupts (as a
  80. * consequence of SCR_EL3.FIQ == 0), reading the ICC_RPR_EL1 register will
  81. * return the Distributor's view of the interrupt priority.
  82. *
  83. * When GIC security is enabled (GICD_CTLR.DS == 0), the interrupt priority
  84. * written by software is moved to the Non-secure range by the Distributor.
  85. *
  86. * If both are true (which is when gic_nonsecure_priorities gets enabled),
  87. * we need to shift down the priority programmed by software to match it
  88. * against the value returned by ICC_RPR_EL1.
  89. */
  90. #define GICD_INT_RPR_PRI(priority) \
  91. ({ \
  92. u32 __priority = (priority); \
  93. if (static_branch_unlikely(&gic_nonsecure_priorities)) \
  94. __priority = 0x80 | (__priority >> 1); \
  95. \
  96. __priority; \
  97. })
  98. /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */
  99. static refcount_t *ppi_nmi_refs;
  100. static struct gic_kvm_info gic_v3_kvm_info;
  101. static DEFINE_PER_CPU(bool, has_rss);
  102. #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4)
  103. #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist))
  104. #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
  105. #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K)
  106. /* Our default, arbitrary priority value. Linux only uses one anyway. */
  107. #define DEFAULT_PMR_VALUE 0xf0
  108. enum gic_intid_range {
  109. SGI_RANGE,
  110. PPI_RANGE,
  111. SPI_RANGE,
  112. EPPI_RANGE,
  113. ESPI_RANGE,
  114. LPI_RANGE,
  115. __INVALID_RANGE__
  116. };
  117. static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq)
  118. {
  119. switch (hwirq) {
  120. case 0 ... 15:
  121. return SGI_RANGE;
  122. case 16 ... 31:
  123. return PPI_RANGE;
  124. case 32 ... 1019:
  125. return SPI_RANGE;
  126. case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63):
  127. return EPPI_RANGE;
  128. case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023):
  129. return ESPI_RANGE;
  130. case 8192 ... GENMASK(23, 0):
  131. return LPI_RANGE;
  132. default:
  133. return __INVALID_RANGE__;
  134. }
  135. }
  136. static enum gic_intid_range get_intid_range(struct irq_data *d)
  137. {
  138. return __get_intid_range(d->hwirq);
  139. }
  140. static inline unsigned int gic_irq(struct irq_data *d)
  141. {
  142. return d->hwirq;
  143. }
  144. static inline bool gic_irq_in_rdist(struct irq_data *d)
  145. {
  146. switch (get_intid_range(d)) {
  147. case SGI_RANGE:
  148. case PPI_RANGE:
  149. case EPPI_RANGE:
  150. return true;
  151. default:
  152. return false;
  153. }
  154. }
  155. static inline void __iomem *gic_dist_base(struct irq_data *d)
  156. {
  157. switch (get_intid_range(d)) {
  158. case SGI_RANGE:
  159. case PPI_RANGE:
  160. case EPPI_RANGE:
  161. /* SGI+PPI -> SGI_base for this CPU */
  162. return gic_data_rdist_sgi_base();
  163. case SPI_RANGE:
  164. case ESPI_RANGE:
  165. /* SPI -> dist_base */
  166. return gic_data.dist_base;
  167. default:
  168. return NULL;
  169. }
  170. }
  171. static void gic_do_wait_for_rwp(void __iomem *base, u32 bit)
  172. {
  173. u32 count = 1000000; /* 1s! */
  174. while (readl_relaxed(base + GICD_CTLR) & bit) {
  175. count--;
  176. if (!count) {
  177. pr_err_ratelimited("RWP timeout, gone fishing\n");
  178. return;
  179. }
  180. cpu_relax();
  181. udelay(1);
  182. }
  183. }
  184. /* Wait for completion of a distributor change */
  185. static void gic_dist_wait_for_rwp(void)
  186. {
  187. gic_do_wait_for_rwp(gic_data.dist_base, GICD_CTLR_RWP);
  188. }
  189. /* Wait for completion of a redistributor change */
  190. static void gic_redist_wait_for_rwp(void)
  191. {
  192. gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP);
  193. }
  194. #ifdef CONFIG_ARM64
  195. static u64 __maybe_unused gic_read_iar(void)
  196. {
  197. if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154))
  198. return gic_read_iar_cavium_thunderx();
  199. else
  200. return gic_read_iar_common();
  201. }
  202. #endif
  203. static void gic_enable_redist(bool enable)
  204. {
  205. void __iomem *rbase;
  206. u32 count = 1000000; /* 1s! */
  207. u32 val;
  208. if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996)
  209. return;
  210. rbase = gic_data_rdist_rd_base();
  211. val = readl_relaxed(rbase + GICR_WAKER);
  212. if (enable)
  213. /* Wake up this CPU redistributor */
  214. val &= ~GICR_WAKER_ProcessorSleep;
  215. else
  216. val |= GICR_WAKER_ProcessorSleep;
  217. writel_relaxed(val, rbase + GICR_WAKER);
  218. if (!enable) { /* Check that GICR_WAKER is writeable */
  219. val = readl_relaxed(rbase + GICR_WAKER);
  220. if (!(val & GICR_WAKER_ProcessorSleep))
  221. return; /* No PM support in this redistributor */
  222. }
  223. while (--count) {
  224. val = readl_relaxed(rbase + GICR_WAKER);
  225. if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep))
  226. break;
  227. cpu_relax();
  228. udelay(1);
  229. }
  230. if (!count)
  231. pr_err_ratelimited("redistributor failed to %s...\n",
  232. enable ? "wakeup" : "sleep");
  233. }
  234. /*
  235. * Routines to disable, enable, EOI and route interrupts
  236. */
  237. static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index)
  238. {
  239. switch (get_intid_range(d)) {
  240. case SGI_RANGE:
  241. case PPI_RANGE:
  242. case SPI_RANGE:
  243. *index = d->hwirq;
  244. return offset;
  245. case EPPI_RANGE:
  246. /*
  247. * Contrary to the ESPI range, the EPPI range is contiguous
  248. * to the PPI range in the registers, so let's adjust the
  249. * displacement accordingly. Consistency is overrated.
  250. */
  251. *index = d->hwirq - EPPI_BASE_INTID + 32;
  252. return offset;
  253. case ESPI_RANGE:
  254. *index = d->hwirq - ESPI_BASE_INTID;
  255. switch (offset) {
  256. case GICD_ISENABLER:
  257. return GICD_ISENABLERnE;
  258. case GICD_ICENABLER:
  259. return GICD_ICENABLERnE;
  260. case GICD_ISPENDR:
  261. return GICD_ISPENDRnE;
  262. case GICD_ICPENDR:
  263. return GICD_ICPENDRnE;
  264. case GICD_ISACTIVER:
  265. return GICD_ISACTIVERnE;
  266. case GICD_ICACTIVER:
  267. return GICD_ICACTIVERnE;
  268. case GICD_IPRIORITYR:
  269. return GICD_IPRIORITYRnE;
  270. case GICD_ICFGR:
  271. return GICD_ICFGRnE;
  272. case GICD_IROUTER:
  273. return GICD_IROUTERnE;
  274. default:
  275. break;
  276. }
  277. break;
  278. default:
  279. break;
  280. }
  281. WARN_ON(1);
  282. *index = d->hwirq;
  283. return offset;
  284. }
  285. static int gic_peek_irq(struct irq_data *d, u32 offset)
  286. {
  287. void __iomem *base;
  288. u32 index, mask;
  289. offset = convert_offset_index(d, offset, &index);
  290. mask = 1 << (index % 32);
  291. if (gic_irq_in_rdist(d))
  292. base = gic_data_rdist_sgi_base();
  293. else
  294. base = gic_data.dist_base;
  295. return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask);
  296. }
  297. static void gic_poke_irq(struct irq_data *d, u32 offset)
  298. {
  299. void (*rwp_wait)(void);
  300. void __iomem *base;
  301. u32 index, mask;
  302. offset = convert_offset_index(d, offset, &index);
  303. mask = 1 << (index % 32);
  304. if (gic_irq_in_rdist(d)) {
  305. base = gic_data_rdist_sgi_base();
  306. rwp_wait = gic_redist_wait_for_rwp;
  307. } else {
  308. base = gic_data.dist_base;
  309. rwp_wait = gic_dist_wait_for_rwp;
  310. }
  311. writel_relaxed(mask, base + offset + (index / 32) * 4);
  312. rwp_wait();
  313. }
  314. static void gic_mask_irq(struct irq_data *d)
  315. {
  316. gic_poke_irq(d, GICD_ICENABLER);
  317. }
  318. static void gic_eoimode1_mask_irq(struct irq_data *d)
  319. {
  320. gic_mask_irq(d);
  321. /*
  322. * When masking a forwarded interrupt, make sure it is
  323. * deactivated as well.
  324. *
  325. * This ensures that an interrupt that is getting
  326. * disabled/masked will not get "stuck", because there is
  327. * noone to deactivate it (guest is being terminated).
  328. */
  329. if (irqd_is_forwarded_to_vcpu(d))
  330. gic_poke_irq(d, GICD_ICACTIVER);
  331. }
  332. static void gic_unmask_irq(struct irq_data *d)
  333. {
  334. gic_poke_irq(d, GICD_ISENABLER);
  335. }
  336. static inline bool gic_supports_nmi(void)
  337. {
  338. return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) &&
  339. static_branch_likely(&supports_pseudo_nmis);
  340. }
  341. static int gic_irq_set_irqchip_state(struct irq_data *d,
  342. enum irqchip_irq_state which, bool val)
  343. {
  344. u32 reg;
  345. if (d->hwirq >= 8192) /* SGI/PPI/SPI only */
  346. return -EINVAL;
  347. switch (which) {
  348. case IRQCHIP_STATE_PENDING:
  349. reg = val ? GICD_ISPENDR : GICD_ICPENDR;
  350. break;
  351. case IRQCHIP_STATE_ACTIVE:
  352. reg = val ? GICD_ISACTIVER : GICD_ICACTIVER;
  353. break;
  354. case IRQCHIP_STATE_MASKED:
  355. reg = val ? GICD_ICENABLER : GICD_ISENABLER;
  356. break;
  357. default:
  358. return -EINVAL;
  359. }
  360. gic_poke_irq(d, reg);
  361. return 0;
  362. }
  363. static int gic_irq_get_irqchip_state(struct irq_data *d,
  364. enum irqchip_irq_state which, bool *val)
  365. {
  366. if (d->hwirq >= 8192) /* PPI/SPI only */
  367. return -EINVAL;
  368. switch (which) {
  369. case IRQCHIP_STATE_PENDING:
  370. *val = gic_peek_irq(d, GICD_ISPENDR);
  371. break;
  372. case IRQCHIP_STATE_ACTIVE:
  373. *val = gic_peek_irq(d, GICD_ISACTIVER);
  374. break;
  375. case IRQCHIP_STATE_MASKED:
  376. *val = !gic_peek_irq(d, GICD_ISENABLER);
  377. break;
  378. default:
  379. return -EINVAL;
  380. }
  381. return 0;
  382. }
  383. static void gic_irq_set_prio(struct irq_data *d, u8 prio)
  384. {
  385. void __iomem *base = gic_dist_base(d);
  386. u32 offset, index;
  387. offset = convert_offset_index(d, GICD_IPRIORITYR, &index);
  388. writeb_relaxed(prio, base + offset + index);
  389. }
  390. static u32 gic_get_ppi_index(struct irq_data *d)
  391. {
  392. switch (get_intid_range(d)) {
  393. case PPI_RANGE:
  394. return d->hwirq - 16;
  395. case EPPI_RANGE:
  396. return d->hwirq - EPPI_BASE_INTID + 16;
  397. default:
  398. unreachable();
  399. }
  400. }
  401. static int gic_irq_nmi_setup(struct irq_data *d)
  402. {
  403. struct irq_desc *desc = irq_to_desc(d->irq);
  404. if (!gic_supports_nmi())
  405. return -EINVAL;
  406. if (gic_peek_irq(d, GICD_ISENABLER)) {
  407. pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
  408. return -EINVAL;
  409. }
  410. /*
  411. * A secondary irq_chip should be in charge of LPI request,
  412. * it should not be possible to get there
  413. */
  414. if (WARN_ON(gic_irq(d) >= 8192))
  415. return -EINVAL;
  416. /* desc lock should already be held */
  417. if (gic_irq_in_rdist(d)) {
  418. u32 idx = gic_get_ppi_index(d);
  419. /* Setting up PPI as NMI, only switch handler for first NMI */
  420. if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) {
  421. refcount_set(&ppi_nmi_refs[idx], 1);
  422. desc->handle_irq = handle_percpu_devid_fasteoi_nmi;
  423. }
  424. } else {
  425. desc->handle_irq = handle_fasteoi_nmi;
  426. }
  427. gic_irq_set_prio(d, GICD_INT_NMI_PRI);
  428. return 0;
  429. }
  430. static void gic_irq_nmi_teardown(struct irq_data *d)
  431. {
  432. struct irq_desc *desc = irq_to_desc(d->irq);
  433. if (WARN_ON(!gic_supports_nmi()))
  434. return;
  435. if (gic_peek_irq(d, GICD_ISENABLER)) {
  436. pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
  437. return;
  438. }
  439. /*
  440. * A secondary irq_chip should be in charge of LPI request,
  441. * it should not be possible to get there
  442. */
  443. if (WARN_ON(gic_irq(d) >= 8192))
  444. return;
  445. /* desc lock should already be held */
  446. if (gic_irq_in_rdist(d)) {
  447. u32 idx = gic_get_ppi_index(d);
  448. /* Tearing down NMI, only switch handler for last NMI */
  449. if (refcount_dec_and_test(&ppi_nmi_refs[idx]))
  450. desc->handle_irq = handle_percpu_devid_irq;
  451. } else {
  452. desc->handle_irq = handle_fasteoi_irq;
  453. }
  454. gic_irq_set_prio(d, GICD_INT_DEF_PRI);
  455. }
  456. static void gic_eoi_irq(struct irq_data *d)
  457. {
  458. gic_write_eoir(gic_irq(d));
  459. }
  460. static void gic_eoimode1_eoi_irq(struct irq_data *d)
  461. {
  462. /*
  463. * No need to deactivate an LPI, or an interrupt that
  464. * is is getting forwarded to a vcpu.
  465. */
  466. if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
  467. return;
  468. gic_write_dir(gic_irq(d));
  469. }
  470. static int gic_set_type(struct irq_data *d, unsigned int type)
  471. {
  472. enum gic_intid_range range;
  473. unsigned int irq = gic_irq(d);
  474. void (*rwp_wait)(void);
  475. void __iomem *base;
  476. u32 offset, index;
  477. int ret;
  478. range = get_intid_range(d);
  479. /* Interrupt configuration for SGIs can't be changed */
  480. if (range == SGI_RANGE)
  481. return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0;
  482. /* SPIs have restrictions on the supported types */
  483. if ((range == SPI_RANGE || range == ESPI_RANGE) &&
  484. type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
  485. return -EINVAL;
  486. if (gic_irq_in_rdist(d)) {
  487. base = gic_data_rdist_sgi_base();
  488. rwp_wait = gic_redist_wait_for_rwp;
  489. } else {
  490. base = gic_data.dist_base;
  491. rwp_wait = gic_dist_wait_for_rwp;
  492. }
  493. offset = convert_offset_index(d, GICD_ICFGR, &index);
  494. ret = gic_configure_irq(index, type, base + offset, rwp_wait);
  495. if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) {
  496. /* Misconfigured PPIs are usually not fatal */
  497. pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq);
  498. ret = 0;
  499. }
  500. return ret;
  501. }
  502. static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
  503. {
  504. if (get_intid_range(d) == SGI_RANGE)
  505. return -EINVAL;
  506. if (vcpu)
  507. irqd_set_forwarded_to_vcpu(d);
  508. else
  509. irqd_clr_forwarded_to_vcpu(d);
  510. return 0;
  511. }
  512. static u64 gic_mpidr_to_affinity(unsigned long mpidr)
  513. {
  514. u64 aff;
  515. aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
  516. MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
  517. MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
  518. MPIDR_AFFINITY_LEVEL(mpidr, 0));
  519. return aff;
  520. }
  521. static void gic_deactivate_unhandled(u32 irqnr)
  522. {
  523. if (static_branch_likely(&supports_deactivate_key)) {
  524. if (irqnr < 8192)
  525. gic_write_dir(irqnr);
  526. } else {
  527. gic_write_eoir(irqnr);
  528. }
  529. }
  530. static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs)
  531. {
  532. bool irqs_enabled = interrupts_enabled(regs);
  533. int err;
  534. if (irqs_enabled)
  535. nmi_enter();
  536. if (static_branch_likely(&supports_deactivate_key))
  537. gic_write_eoir(irqnr);
  538. /*
  539. * Leave the PSR.I bit set to prevent other NMIs to be
  540. * received while handling this one.
  541. * PSR.I will be restored when we ERET to the
  542. * interrupted context.
  543. */
  544. err = handle_domain_nmi(gic_data.domain, irqnr, regs);
  545. if (err)
  546. gic_deactivate_unhandled(irqnr);
  547. if (irqs_enabled)
  548. nmi_exit();
  549. }
  550. static u32 do_read_iar(struct pt_regs *regs)
  551. {
  552. u32 iar;
  553. if (gic_supports_nmi() && unlikely(!interrupts_enabled(regs))) {
  554. u64 pmr;
  555. /*
  556. * We were in a context with IRQs disabled. However, the
  557. * entry code has set PMR to a value that allows any
  558. * interrupt to be acknowledged, and not just NMIs. This can
  559. * lead to surprising effects if the NMI has been retired in
  560. * the meantime, and that there is an IRQ pending. The IRQ
  561. * would then be taken in NMI context, something that nobody
  562. * wants to debug twice.
  563. *
  564. * Until we sort this, drop PMR again to a level that will
  565. * actually only allow NMIs before reading IAR, and then
  566. * restore it to what it was.
  567. */
  568. pmr = gic_read_pmr();
  569. gic_pmr_mask_irqs();
  570. isb();
  571. iar = gic_read_iar();
  572. gic_write_pmr(pmr);
  573. } else {
  574. iar = gic_read_iar();
  575. }
  576. return iar;
  577. }
  578. static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
  579. {
  580. u32 irqnr;
  581. irqnr = do_read_iar(regs);
  582. /* Check for special IDs first */
  583. if ((irqnr >= 1020 && irqnr <= 1023))
  584. return;
  585. if (gic_supports_nmi() &&
  586. unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI))) {
  587. gic_handle_nmi(irqnr, regs);
  588. return;
  589. }
  590. if (gic_prio_masking_enabled()) {
  591. gic_pmr_mask_irqs();
  592. gic_arch_enable_irqs();
  593. }
  594. if (static_branch_likely(&supports_deactivate_key))
  595. gic_write_eoir(irqnr);
  596. else
  597. isb();
  598. if (handle_domain_irq(gic_data.domain, irqnr, regs)) {
  599. WARN_ONCE(true, "Unexpected interrupt received!\n");
  600. log_abnormal_wakeup_reason("unexpected HW IRQ %u", irqnr);
  601. gic_deactivate_unhandled(irqnr);
  602. }
  603. }
  604. static u32 gic_get_pribits(void)
  605. {
  606. u32 pribits;
  607. pribits = gic_read_ctlr();
  608. pribits &= ICC_CTLR_EL1_PRI_BITS_MASK;
  609. pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT;
  610. pribits++;
  611. return pribits;
  612. }
  613. static bool gic_has_group0(void)
  614. {
  615. u32 val;
  616. u32 old_pmr;
  617. old_pmr = gic_read_pmr();
  618. /*
  619. * Let's find out if Group0 is under control of EL3 or not by
  620. * setting the highest possible, non-zero priority in PMR.
  621. *
  622. * If SCR_EL3.FIQ is set, the priority gets shifted down in
  623. * order for the CPU interface to set bit 7, and keep the
  624. * actual priority in the non-secure range. In the process, it
  625. * looses the least significant bit and the actual priority
  626. * becomes 0x80. Reading it back returns 0, indicating that
  627. * we're don't have access to Group0.
  628. */
  629. gic_write_pmr(BIT(8 - gic_get_pribits()));
  630. val = gic_read_pmr();
  631. gic_write_pmr(old_pmr);
  632. return val != 0;
  633. }
  634. static void __init gic_dist_init(void)
  635. {
  636. unsigned int i;
  637. u64 affinity;
  638. void __iomem *base = gic_data.dist_base;
  639. u32 val;
  640. /* Disable the distributor */
  641. writel_relaxed(0, base + GICD_CTLR);
  642. gic_dist_wait_for_rwp();
  643. /*
  644. * Configure SPIs as non-secure Group-1. This will only matter
  645. * if the GIC only has a single security state. This will not
  646. * do the right thing if the kernel is running in secure mode,
  647. * but that's not the intended use case anyway.
  648. */
  649. for (i = 32; i < GIC_LINE_NR; i += 32)
  650. writel_relaxed(~0, base + GICD_IGROUPR + i / 8);
  651. /* Extended SPI range, not handled by the GICv2/GICv3 common code */
  652. for (i = 0; i < GIC_ESPI_NR; i += 32) {
  653. writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8);
  654. writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8);
  655. }
  656. for (i = 0; i < GIC_ESPI_NR; i += 32)
  657. writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8);
  658. for (i = 0; i < GIC_ESPI_NR; i += 16)
  659. writel_relaxed(0, base + GICD_ICFGRnE + i / 4);
  660. for (i = 0; i < GIC_ESPI_NR; i += 4)
  661. writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i);
  662. /* Now do the common stuff, and wait for the distributor to drain */
  663. gic_dist_config(base, GIC_LINE_NR, gic_dist_wait_for_rwp);
  664. val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1;
  665. if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) {
  666. pr_info("Enabling SGIs without active state\n");
  667. val |= GICD_CTLR_nASSGIreq;
  668. }
  669. /* Enable distributor with ARE, Group1 */
  670. writel_relaxed(val, base + GICD_CTLR);
  671. /*
  672. * Set all global interrupts to the boot CPU only. ARE must be
  673. * enabled.
  674. */
  675. affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
  676. for (i = 32; i < GIC_LINE_NR; i++)
  677. gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
  678. for (i = 0; i < GIC_ESPI_NR; i++)
  679. gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8);
  680. }
  681. static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
  682. {
  683. int ret = -ENODEV;
  684. int i;
  685. for (i = 0; i < gic_data.nr_redist_regions; i++) {
  686. void __iomem *ptr = gic_data.redist_regions[i].redist_base;
  687. u64 typer;
  688. u32 reg;
  689. reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
  690. if (reg != GIC_PIDR2_ARCH_GICv3 &&
  691. reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
  692. pr_warn("No redistributor present @%p\n", ptr);
  693. break;
  694. }
  695. do {
  696. typer = gic_read_typer(ptr + GICR_TYPER);
  697. ret = fn(gic_data.redist_regions + i, ptr);
  698. if (!ret)
  699. return 0;
  700. if (gic_data.redist_regions[i].single_redist)
  701. break;
  702. if (gic_data.redist_stride) {
  703. ptr += gic_data.redist_stride;
  704. } else {
  705. ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
  706. if (typer & GICR_TYPER_VLPIS)
  707. ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
  708. }
  709. } while (!(typer & GICR_TYPER_LAST));
  710. }
  711. return ret ? -ENODEV : 0;
  712. }
  713. static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
  714. {
  715. unsigned long mpidr = cpu_logical_map(smp_processor_id());
  716. u64 typer;
  717. u32 aff;
  718. /*
  719. * Convert affinity to a 32bit value that can be matched to
  720. * GICR_TYPER bits [63:32].
  721. */
  722. aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
  723. MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
  724. MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
  725. MPIDR_AFFINITY_LEVEL(mpidr, 0));
  726. typer = gic_read_typer(ptr + GICR_TYPER);
  727. if ((typer >> 32) == aff) {
  728. u64 offset = ptr - region->redist_base;
  729. raw_spin_lock_init(&gic_data_rdist()->rd_lock);
  730. gic_data_rdist_rd_base() = ptr;
  731. gic_data_rdist()->phys_base = region->phys_base + offset;
  732. pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
  733. smp_processor_id(), mpidr,
  734. (int)(region - gic_data.redist_regions),
  735. &gic_data_rdist()->phys_base);
  736. return 0;
  737. }
  738. /* Try next one */
  739. return 1;
  740. }
  741. static int gic_populate_rdist(void)
  742. {
  743. if (gic_iterate_rdists(__gic_populate_rdist) == 0)
  744. return 0;
  745. /* We couldn't even deal with ourselves... */
  746. WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
  747. smp_processor_id(),
  748. (unsigned long)cpu_logical_map(smp_processor_id()));
  749. return -ENODEV;
  750. }
  751. static int __gic_update_rdist_properties(struct redist_region *region,
  752. void __iomem *ptr)
  753. {
  754. u64 typer = gic_read_typer(ptr + GICR_TYPER);
  755. /* Boot-time cleanip */
  756. if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) {
  757. u64 val;
  758. /* Deactivate any present vPE */
  759. val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER);
  760. if (val & GICR_VPENDBASER_Valid)
  761. gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
  762. ptr + SZ_128K + GICR_VPENDBASER);
  763. /* Mark the VPE table as invalid */
  764. val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER);
  765. val &= ~GICR_VPROPBASER_4_1_VALID;
  766. gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER);
  767. }
  768. gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS);
  769. /* RVPEID implies some form of DirectLPI, no matter what the doc says... :-/ */
  770. gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID);
  771. gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) |
  772. gic_data.rdists.has_rvpeid);
  773. gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY);
  774. /* Detect non-sensical configurations */
  775. if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) {
  776. gic_data.rdists.has_direct_lpi = false;
  777. gic_data.rdists.has_vlpis = false;
  778. gic_data.rdists.has_rvpeid = false;
  779. }
  780. gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr);
  781. return 1;
  782. }
  783. static void gic_update_rdist_properties(void)
  784. {
  785. gic_data.ppi_nr = UINT_MAX;
  786. gic_iterate_rdists(__gic_update_rdist_properties);
  787. if (WARN_ON(gic_data.ppi_nr == UINT_MAX))
  788. gic_data.ppi_nr = 0;
  789. pr_info("%d PPIs implemented\n", gic_data.ppi_nr);
  790. if (gic_data.rdists.has_vlpis)
  791. pr_info("GICv4 features: %s%s%s\n",
  792. gic_data.rdists.has_direct_lpi ? "DirectLPI " : "",
  793. gic_data.rdists.has_rvpeid ? "RVPEID " : "",
  794. gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : "");
  795. }
  796. /* Check whether it's single security state view */
  797. static inline bool gic_dist_security_disabled(void)
  798. {
  799. return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS;
  800. }
  801. static void gic_cpu_sys_reg_init(void)
  802. {
  803. int i, cpu = smp_processor_id();
  804. u64 mpidr = cpu_logical_map(cpu);
  805. u64 need_rss = MPIDR_RS(mpidr);
  806. bool group0;
  807. u32 pribits;
  808. /*
  809. * Need to check that the SRE bit has actually been set. If
  810. * not, it means that SRE is disabled at EL2. We're going to
  811. * die painfully, and there is nothing we can do about it.
  812. *
  813. * Kindly inform the luser.
  814. */
  815. if (!gic_enable_sre())
  816. pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
  817. pribits = gic_get_pribits();
  818. group0 = gic_has_group0();
  819. /* Set priority mask register */
  820. if (!gic_prio_masking_enabled()) {
  821. write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
  822. } else if (gic_supports_nmi()) {
  823. /*
  824. * Mismatch configuration with boot CPU, the system is likely
  825. * to die as interrupt masking will not work properly on all
  826. * CPUs
  827. *
  828. * The boot CPU calls this function before enabling NMI support,
  829. * and as a result we'll never see this warning in the boot path
  830. * for that CPU.
  831. */
  832. if (static_branch_unlikely(&gic_nonsecure_priorities))
  833. WARN_ON(!group0 || gic_dist_security_disabled());
  834. else
  835. WARN_ON(group0 && !gic_dist_security_disabled());
  836. }
  837. /*
  838. * Some firmwares hand over to the kernel with the BPR changed from
  839. * its reset value (and with a value large enough to prevent
  840. * any pre-emptive interrupts from working at all). Writing a zero
  841. * to BPR restores is reset value.
  842. */
  843. gic_write_bpr1(0);
  844. if (static_branch_likely(&supports_deactivate_key)) {
  845. /* EOI drops priority only (mode 1) */
  846. gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
  847. } else {
  848. /* EOI deactivates interrupt too (mode 0) */
  849. gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
  850. }
  851. /* Always whack Group0 before Group1 */
  852. if (group0) {
  853. switch(pribits) {
  854. case 8:
  855. case 7:
  856. write_gicreg(0, ICC_AP0R3_EL1);
  857. write_gicreg(0, ICC_AP0R2_EL1);
  858. fallthrough;
  859. case 6:
  860. write_gicreg(0, ICC_AP0R1_EL1);
  861. fallthrough;
  862. case 5:
  863. case 4:
  864. write_gicreg(0, ICC_AP0R0_EL1);
  865. }
  866. isb();
  867. }
  868. switch(pribits) {
  869. case 8:
  870. case 7:
  871. write_gicreg(0, ICC_AP1R3_EL1);
  872. write_gicreg(0, ICC_AP1R2_EL1);
  873. fallthrough;
  874. case 6:
  875. write_gicreg(0, ICC_AP1R1_EL1);
  876. fallthrough;
  877. case 5:
  878. case 4:
  879. write_gicreg(0, ICC_AP1R0_EL1);
  880. }
  881. isb();
  882. /* ... and let's hit the road... */
  883. gic_write_grpen1(1);
  884. /* Keep the RSS capability status in per_cpu variable */
  885. per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS);
  886. /* Check all the CPUs have capable of sending SGIs to other CPUs */
  887. for_each_online_cpu(i) {
  888. bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu);
  889. need_rss |= MPIDR_RS(cpu_logical_map(i));
  890. if (need_rss && (!have_rss))
  891. pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n",
  892. cpu, (unsigned long)mpidr,
  893. i, (unsigned long)cpu_logical_map(i));
  894. }
  895. /**
  896. * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0,
  897. * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED
  898. * UNPREDICTABLE choice of :
  899. * - The write is ignored.
  900. * - The RS field is treated as 0.
  901. */
  902. if (need_rss && (!gic_data.has_rss))
  903. pr_crit_once("RSS is required but GICD doesn't support it\n");
  904. }
  905. static bool gicv3_nolpi;
  906. static int __init gicv3_nolpi_cfg(char *buf)
  907. {
  908. return strtobool(buf, &gicv3_nolpi);
  909. }
  910. early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg);
  911. static int gic_dist_supports_lpis(void)
  912. {
  913. return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) &&
  914. !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) &&
  915. !gicv3_nolpi);
  916. }
  917. static void gic_cpu_init(void)
  918. {
  919. void __iomem *rbase;
  920. int i;
  921. /* Register ourselves with the rest of the world */
  922. if (gic_populate_rdist())
  923. return;
  924. gic_enable_redist(true);
  925. WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) &&
  926. !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange),
  927. "Distributor has extended ranges, but CPU%d doesn't\n",
  928. smp_processor_id());
  929. rbase = gic_data_rdist_sgi_base();
  930. /* Configure SGIs/PPIs as non-secure Group-1 */
  931. for (i = 0; i < gic_data.ppi_nr + 16; i += 32)
  932. writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8);
  933. gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp);
  934. /* initialise system registers */
  935. gic_cpu_sys_reg_init();
  936. }
  937. #ifdef CONFIG_SMP
  938. #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT)
  939. #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL)
  940. static int gic_starting_cpu(unsigned int cpu)
  941. {
  942. gic_cpu_init();
  943. if (gic_dist_supports_lpis())
  944. its_cpu_init();
  945. return 0;
  946. }
  947. static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
  948. unsigned long cluster_id)
  949. {
  950. int next_cpu, cpu = *base_cpu;
  951. unsigned long mpidr = cpu_logical_map(cpu);
  952. u16 tlist = 0;
  953. while (cpu < nr_cpu_ids) {
  954. tlist |= 1 << (mpidr & 0xf);
  955. next_cpu = cpumask_next(cpu, mask);
  956. if (next_cpu >= nr_cpu_ids)
  957. goto out;
  958. cpu = next_cpu;
  959. mpidr = cpu_logical_map(cpu);
  960. if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) {
  961. cpu--;
  962. goto out;
  963. }
  964. }
  965. out:
  966. *base_cpu = cpu;
  967. return tlist;
  968. }
  969. #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
  970. (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
  971. << ICC_SGI1R_AFFINITY_## level ##_SHIFT)
  972. static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
  973. {
  974. u64 val;
  975. val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) |
  976. MPIDR_TO_SGI_AFFINITY(cluster_id, 2) |
  977. irq << ICC_SGI1R_SGI_ID_SHIFT |
  978. MPIDR_TO_SGI_AFFINITY(cluster_id, 1) |
  979. MPIDR_TO_SGI_RS(cluster_id) |
  980. tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
  981. pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
  982. gic_write_sgi1r(val);
  983. }
  984. static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask)
  985. {
  986. int cpu;
  987. if (WARN_ON(d->hwirq >= 16))
  988. return;
  989. /*
  990. * Ensure that stores to Normal memory are visible to the
  991. * other CPUs before issuing the IPI.
  992. */
  993. wmb();
  994. for_each_cpu(cpu, mask) {
  995. u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu));
  996. u16 tlist;
  997. tlist = gic_compute_target_list(&cpu, mask, cluster_id);
  998. gic_send_sgi(cluster_id, tlist, d->hwirq);
  999. }
  1000. /* Force the above writes to ICC_SGI1R_EL1 to be executed */
  1001. isb();
  1002. }
  1003. static void __init gic_smp_init(void)
  1004. {
  1005. struct irq_fwspec sgi_fwspec = {
  1006. .fwnode = gic_data.fwnode,
  1007. .param_count = 1,
  1008. };
  1009. int base_sgi;
  1010. cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
  1011. "irqchip/arm/gicv3:starting",
  1012. gic_starting_cpu, NULL);
  1013. /* Register all 8 non-secure SGIs */
  1014. base_sgi = __irq_domain_alloc_irqs(gic_data.domain, -1, 8,
  1015. NUMA_NO_NODE, &sgi_fwspec,
  1016. false, NULL);
  1017. if (WARN_ON(base_sgi <= 0))
  1018. return;
  1019. set_smp_ipi_range(base_sgi, 8);
  1020. }
  1021. static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
  1022. bool force)
  1023. {
  1024. unsigned int cpu;
  1025. u32 offset, index;
  1026. void __iomem *reg;
  1027. int enabled;
  1028. u64 val;
  1029. if (force)
  1030. cpu = cpumask_first(mask_val);
  1031. else
  1032. cpu = cpumask_any_and(mask_val, cpu_online_mask);
  1033. if (cpu >= nr_cpu_ids)
  1034. return -EINVAL;
  1035. if (gic_irq_in_rdist(d))
  1036. return -EINVAL;
  1037. /* If interrupt was enabled, disable it first */
  1038. enabled = gic_peek_irq(d, GICD_ISENABLER);
  1039. if (enabled)
  1040. gic_mask_irq(d);
  1041. offset = convert_offset_index(d, GICD_IROUTER, &index);
  1042. reg = gic_dist_base(d) + offset + (index * 8);
  1043. val = gic_mpidr_to_affinity(cpu_logical_map(cpu));
  1044. trace_android_rvh_gic_v3_set_affinity(d, mask_val, &val, force, gic_dist_base(d));
  1045. gic_write_irouter(val, reg);
  1046. /*
  1047. * If the interrupt was enabled, enabled it again. Otherwise,
  1048. * just wait for the distributor to have digested our changes.
  1049. */
  1050. if (enabled)
  1051. gic_unmask_irq(d);
  1052. else
  1053. gic_dist_wait_for_rwp();
  1054. irq_data_update_effective_affinity(d, cpumask_of(cpu));
  1055. return IRQ_SET_MASK_OK_DONE;
  1056. }
  1057. #else
  1058. #define gic_set_affinity NULL
  1059. #define gic_ipi_send_mask NULL
  1060. #define gic_smp_init() do { } while(0)
  1061. #endif
  1062. static int gic_retrigger(struct irq_data *data)
  1063. {
  1064. return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true);
  1065. }
  1066. #ifdef CONFIG_CPU_PM
  1067. static int gic_cpu_pm_notifier(struct notifier_block *self,
  1068. unsigned long cmd, void *v)
  1069. {
  1070. if (cmd == CPU_PM_EXIT) {
  1071. if (gic_dist_security_disabled())
  1072. gic_enable_redist(true);
  1073. gic_cpu_sys_reg_init();
  1074. } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) {
  1075. gic_write_grpen1(0);
  1076. gic_enable_redist(false);
  1077. }
  1078. return NOTIFY_OK;
  1079. }
  1080. static struct notifier_block gic_cpu_pm_notifier_block = {
  1081. .notifier_call = gic_cpu_pm_notifier,
  1082. };
  1083. static void gic_cpu_pm_init(void)
  1084. {
  1085. cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
  1086. }
  1087. #else
  1088. static inline void gic_cpu_pm_init(void) { }
  1089. #endif /* CONFIG_CPU_PM */
  1090. #ifdef CONFIG_PM
  1091. void gic_resume(void)
  1092. {
  1093. trace_android_vh_gic_resume(&gic_data);
  1094. }
  1095. EXPORT_SYMBOL_GPL(gic_resume);
  1096. static struct syscore_ops gic_syscore_ops = {
  1097. .resume = gic_resume,
  1098. };
  1099. static void gic_syscore_init(void)
  1100. {
  1101. register_syscore_ops(&gic_syscore_ops);
  1102. }
  1103. #else
  1104. static inline void gic_syscore_init(void) { }
  1105. void gic_resume(void) { }
  1106. #endif
  1107. static struct irq_chip gic_chip = {
  1108. .name = "GICv3",
  1109. .irq_mask = gic_mask_irq,
  1110. .irq_unmask = gic_unmask_irq,
  1111. .irq_eoi = gic_eoi_irq,
  1112. .irq_set_type = gic_set_type,
  1113. .irq_set_affinity = gic_set_affinity,
  1114. .irq_retrigger = gic_retrigger,
  1115. .irq_get_irqchip_state = gic_irq_get_irqchip_state,
  1116. .irq_set_irqchip_state = gic_irq_set_irqchip_state,
  1117. .irq_nmi_setup = gic_irq_nmi_setup,
  1118. .irq_nmi_teardown = gic_irq_nmi_teardown,
  1119. .ipi_send_mask = gic_ipi_send_mask,
  1120. .flags = IRQCHIP_SET_TYPE_MASKED |
  1121. IRQCHIP_SKIP_SET_WAKE |
  1122. IRQCHIP_MASK_ON_SUSPEND,
  1123. };
  1124. static struct irq_chip gic_eoimode1_chip = {
  1125. .name = "GICv3",
  1126. .irq_mask = gic_eoimode1_mask_irq,
  1127. .irq_unmask = gic_unmask_irq,
  1128. .irq_eoi = gic_eoimode1_eoi_irq,
  1129. .irq_set_type = gic_set_type,
  1130. .irq_set_affinity = gic_set_affinity,
  1131. .irq_retrigger = gic_retrigger,
  1132. .irq_get_irqchip_state = gic_irq_get_irqchip_state,
  1133. .irq_set_irqchip_state = gic_irq_set_irqchip_state,
  1134. .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity,
  1135. .irq_nmi_setup = gic_irq_nmi_setup,
  1136. .irq_nmi_teardown = gic_irq_nmi_teardown,
  1137. .ipi_send_mask = gic_ipi_send_mask,
  1138. .flags = IRQCHIP_SET_TYPE_MASKED |
  1139. IRQCHIP_SKIP_SET_WAKE |
  1140. IRQCHIP_MASK_ON_SUSPEND,
  1141. };
  1142. static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
  1143. irq_hw_number_t hw)
  1144. {
  1145. struct irq_chip *chip = &gic_chip;
  1146. struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq));
  1147. if (static_branch_likely(&supports_deactivate_key))
  1148. chip = &gic_eoimode1_chip;
  1149. switch (__get_intid_range(hw)) {
  1150. case SGI_RANGE:
  1151. irq_set_percpu_devid(irq);
  1152. irq_domain_set_info(d, irq, hw, chip, d->host_data,
  1153. handle_percpu_devid_fasteoi_ipi,
  1154. NULL, NULL);
  1155. break;
  1156. case PPI_RANGE:
  1157. case EPPI_RANGE:
  1158. irq_set_percpu_devid(irq);
  1159. irq_domain_set_info(d, irq, hw, chip, d->host_data,
  1160. handle_percpu_devid_irq, NULL, NULL);
  1161. break;
  1162. case SPI_RANGE:
  1163. case ESPI_RANGE:
  1164. irq_domain_set_info(d, irq, hw, chip, d->host_data,
  1165. handle_fasteoi_irq, NULL, NULL);
  1166. irq_set_probe(irq);
  1167. irqd_set_single_target(irqd);
  1168. break;
  1169. case LPI_RANGE:
  1170. if (!gic_dist_supports_lpis())
  1171. return -EPERM;
  1172. irq_domain_set_info(d, irq, hw, chip, d->host_data,
  1173. handle_fasteoi_irq, NULL, NULL);
  1174. break;
  1175. default:
  1176. return -EPERM;
  1177. }
  1178. /* Prevents SW retriggers which mess up the ACK/EOI ordering */
  1179. irqd_set_handle_enforce_irqctx(irqd);
  1180. return 0;
  1181. }
  1182. static int gic_irq_domain_translate(struct irq_domain *d,
  1183. struct irq_fwspec *fwspec,
  1184. unsigned long *hwirq,
  1185. unsigned int *type)
  1186. {
  1187. if (fwspec->param_count == 1 && fwspec->param[0] < 16) {
  1188. *hwirq = fwspec->param[0];
  1189. *type = IRQ_TYPE_EDGE_RISING;
  1190. return 0;
  1191. }
  1192. if (is_of_node(fwspec->fwnode)) {
  1193. if (fwspec->param_count < 3)
  1194. return -EINVAL;
  1195. switch (fwspec->param[0]) {
  1196. case 0: /* SPI */
  1197. *hwirq = fwspec->param[1] + 32;
  1198. break;
  1199. case 1: /* PPI */
  1200. *hwirq = fwspec->param[1] + 16;
  1201. break;
  1202. case 2: /* ESPI */
  1203. *hwirq = fwspec->param[1] + ESPI_BASE_INTID;
  1204. break;
  1205. case 3: /* EPPI */
  1206. *hwirq = fwspec->param[1] + EPPI_BASE_INTID;
  1207. break;
  1208. case GIC_IRQ_TYPE_LPI: /* LPI */
  1209. *hwirq = fwspec->param[1];
  1210. break;
  1211. case GIC_IRQ_TYPE_PARTITION:
  1212. *hwirq = fwspec->param[1];
  1213. if (fwspec->param[1] >= 16)
  1214. *hwirq += EPPI_BASE_INTID - 16;
  1215. else
  1216. *hwirq += 16;
  1217. break;
  1218. default:
  1219. return -EINVAL;
  1220. }
  1221. *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
  1222. /*
  1223. * Make it clear that broken DTs are... broken.
  1224. * Partitionned PPIs are an unfortunate exception.
  1225. */
  1226. WARN_ON(*type == IRQ_TYPE_NONE &&
  1227. fwspec->param[0] != GIC_IRQ_TYPE_PARTITION);
  1228. return 0;
  1229. }
  1230. if (is_fwnode_irqchip(fwspec->fwnode)) {
  1231. if(fwspec->param_count != 2)
  1232. return -EINVAL;
  1233. if (fwspec->param[0] < 16) {
  1234. pr_err(FW_BUG "Illegal GSI%d translation request\n",
  1235. fwspec->param[0]);
  1236. return -EINVAL;
  1237. }
  1238. *hwirq = fwspec->param[0];
  1239. *type = fwspec->param[1];
  1240. WARN_ON(*type == IRQ_TYPE_NONE);
  1241. return 0;
  1242. }
  1243. return -EINVAL;
  1244. }
  1245. static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
  1246. unsigned int nr_irqs, void *arg)
  1247. {
  1248. int i, ret;
  1249. irq_hw_number_t hwirq;
  1250. unsigned int type = IRQ_TYPE_NONE;
  1251. struct irq_fwspec *fwspec = arg;
  1252. ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
  1253. if (ret)
  1254. return ret;
  1255. for (i = 0; i < nr_irqs; i++) {
  1256. ret = gic_irq_domain_map(domain, virq + i, hwirq + i);
  1257. if (ret)
  1258. return ret;
  1259. }
  1260. return 0;
  1261. }
  1262. static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
  1263. unsigned int nr_irqs)
  1264. {
  1265. int i;
  1266. for (i = 0; i < nr_irqs; i++) {
  1267. struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
  1268. irq_set_handler(virq + i, NULL);
  1269. irq_domain_reset_irq_data(d);
  1270. }
  1271. }
  1272. static int gic_irq_domain_select(struct irq_domain *d,
  1273. struct irq_fwspec *fwspec,
  1274. enum irq_domain_bus_token bus_token)
  1275. {
  1276. /* Not for us */
  1277. if (fwspec->fwnode != d->fwnode)
  1278. return 0;
  1279. /* If this is not DT, then we have a single domain */
  1280. if (!is_of_node(fwspec->fwnode))
  1281. return 1;
  1282. /*
  1283. * If this is a PPI and we have a 4th (non-null) parameter,
  1284. * then we need to match the partition domain.
  1285. */
  1286. if (fwspec->param_count >= 4 &&
  1287. fwspec->param[0] == 1 && fwspec->param[3] != 0 &&
  1288. gic_data.ppi_descs)
  1289. return d == partition_get_domain(gic_data.ppi_descs[fwspec->param[1]]);
  1290. return d == gic_data.domain;
  1291. }
  1292. static const struct irq_domain_ops gic_irq_domain_ops = {
  1293. .translate = gic_irq_domain_translate,
  1294. .alloc = gic_irq_domain_alloc,
  1295. .free = gic_irq_domain_free,
  1296. .select = gic_irq_domain_select,
  1297. };
  1298. static int partition_domain_translate(struct irq_domain *d,
  1299. struct irq_fwspec *fwspec,
  1300. unsigned long *hwirq,
  1301. unsigned int *type)
  1302. {
  1303. struct device_node *np;
  1304. int ret;
  1305. if (!gic_data.ppi_descs)
  1306. return -ENOMEM;
  1307. np = of_find_node_by_phandle(fwspec->param[3]);
  1308. if (WARN_ON(!np))
  1309. return -EINVAL;
  1310. ret = partition_translate_id(gic_data.ppi_descs[fwspec->param[1]],
  1311. of_node_to_fwnode(np));
  1312. if (ret < 0)
  1313. return ret;
  1314. *hwirq = ret;
  1315. *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
  1316. return 0;
  1317. }
  1318. static const struct irq_domain_ops partition_domain_ops = {
  1319. .translate = partition_domain_translate,
  1320. .select = gic_irq_domain_select,
  1321. };
  1322. static bool gic_enable_quirk_msm8996(void *data)
  1323. {
  1324. struct gic_chip_data *d = data;
  1325. d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996;
  1326. return true;
  1327. }
  1328. static bool gic_enable_quirk_cavium_38539(void *data)
  1329. {
  1330. struct gic_chip_data *d = data;
  1331. d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539;
  1332. return true;
  1333. }
  1334. static bool gic_enable_quirk_hip06_07(void *data)
  1335. {
  1336. struct gic_chip_data *d = data;
  1337. /*
  1338. * HIP06 GICD_IIDR clashes with GIC-600 product number (despite
  1339. * not being an actual ARM implementation). The saving grace is
  1340. * that GIC-600 doesn't have ESPI, so nothing to do in that case.
  1341. * HIP07 doesn't even have a proper IIDR, and still pretends to
  1342. * have ESPI. In both cases, put them right.
  1343. */
  1344. if (d->rdists.gicd_typer & GICD_TYPER_ESPI) {
  1345. /* Zero both ESPI and the RES0 field next to it... */
  1346. d->rdists.gicd_typer &= ~GENMASK(9, 8);
  1347. return true;
  1348. }
  1349. return false;
  1350. }
  1351. static const struct gic_quirk gic_quirks[] = {
  1352. {
  1353. .desc = "GICv3: Qualcomm MSM8996 broken firmware",
  1354. .compatible = "qcom,msm8996-gic-v3",
  1355. .init = gic_enable_quirk_msm8996,
  1356. },
  1357. {
  1358. .desc = "GICv3: HIP06 erratum 161010803",
  1359. .iidr = 0x0204043b,
  1360. .mask = 0xffffffff,
  1361. .init = gic_enable_quirk_hip06_07,
  1362. },
  1363. {
  1364. .desc = "GICv3: HIP07 erratum 161010803",
  1365. .iidr = 0x00000000,
  1366. .mask = 0xffffffff,
  1367. .init = gic_enable_quirk_hip06_07,
  1368. },
  1369. {
  1370. /*
  1371. * Reserved register accesses generate a Synchronous
  1372. * External Abort. This erratum applies to:
  1373. * - ThunderX: CN88xx
  1374. * - OCTEON TX: CN83xx, CN81xx
  1375. * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx*
  1376. */
  1377. .desc = "GICv3: Cavium erratum 38539",
  1378. .iidr = 0xa000034c,
  1379. .mask = 0xe8f00fff,
  1380. .init = gic_enable_quirk_cavium_38539,
  1381. },
  1382. {
  1383. }
  1384. };
  1385. static void gic_enable_nmi_support(void)
  1386. {
  1387. int i;
  1388. if (!gic_prio_masking_enabled())
  1389. return;
  1390. ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL);
  1391. if (!ppi_nmi_refs)
  1392. return;
  1393. for (i = 0; i < gic_data.ppi_nr; i++)
  1394. refcount_set(&ppi_nmi_refs[i], 0);
  1395. /*
  1396. * Linux itself doesn't use 1:N distribution, so has no need to
  1397. * set PMHE. The only reason to have it set is if EL3 requires it
  1398. * (and we can't change it).
  1399. */
  1400. if (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK)
  1401. static_branch_enable(&gic_pmr_sync);
  1402. pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n",
  1403. static_branch_unlikely(&gic_pmr_sync) ? "forced" : "relaxed");
  1404. /*
  1405. * How priority values are used by the GIC depends on two things:
  1406. * the security state of the GIC (controlled by the GICD_CTRL.DS bit)
  1407. * and if Group 0 interrupts can be delivered to Linux in the non-secure
  1408. * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the
  1409. * the ICC_PMR_EL1 register and the priority that software assigns to
  1410. * interrupts:
  1411. *
  1412. * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority
  1413. * -----------------------------------------------------------
  1414. * 1 | - | unchanged | unchanged
  1415. * -----------------------------------------------------------
  1416. * 0 | 1 | non-secure | non-secure
  1417. * -----------------------------------------------------------
  1418. * 0 | 0 | unchanged | non-secure
  1419. *
  1420. * where non-secure means that the value is right-shifted by one and the
  1421. * MSB bit set, to make it fit in the non-secure priority range.
  1422. *
  1423. * In the first two cases, where ICC_PMR_EL1 and the interrupt priority
  1424. * are both either modified or unchanged, we can use the same set of
  1425. * priorities.
  1426. *
  1427. * In the last case, where only the interrupt priorities are modified to
  1428. * be in the non-secure range, we use a different PMR value to mask IRQs
  1429. * and the rest of the values that we use remain unchanged.
  1430. */
  1431. if (gic_has_group0() && !gic_dist_security_disabled())
  1432. static_branch_enable(&gic_nonsecure_priorities);
  1433. static_branch_enable(&supports_pseudo_nmis);
  1434. if (static_branch_likely(&supports_deactivate_key))
  1435. gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI;
  1436. else
  1437. gic_chip.flags |= IRQCHIP_SUPPORTS_NMI;
  1438. }
  1439. static int __init gic_init_bases(void __iomem *dist_base,
  1440. struct redist_region *rdist_regs,
  1441. u32 nr_redist_regions,
  1442. u64 redist_stride,
  1443. struct fwnode_handle *handle)
  1444. {
  1445. u32 typer;
  1446. int err;
  1447. if (!is_hyp_mode_available())
  1448. static_branch_disable(&supports_deactivate_key);
  1449. if (static_branch_likely(&supports_deactivate_key))
  1450. pr_info("GIC: Using split EOI/Deactivate mode\n");
  1451. gic_data.fwnode = handle;
  1452. gic_data.dist_base = dist_base;
  1453. gic_data.redist_regions = rdist_regs;
  1454. gic_data.nr_redist_regions = nr_redist_regions;
  1455. gic_data.redist_stride = redist_stride;
  1456. /*
  1457. * Find out how many interrupts are supported.
  1458. */
  1459. typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
  1460. gic_data.rdists.gicd_typer = typer;
  1461. gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR),
  1462. gic_quirks, &gic_data);
  1463. pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32);
  1464. pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR);
  1465. /*
  1466. * ThunderX1 explodes on reading GICD_TYPER2, in violation of the
  1467. * architecture spec (which says that reserved registers are RES0).
  1468. */
  1469. if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539))
  1470. gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2);
  1471. gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
  1472. &gic_data);
  1473. gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
  1474. gic_data.rdists.has_rvpeid = true;
  1475. gic_data.rdists.has_vlpis = true;
  1476. gic_data.rdists.has_direct_lpi = true;
  1477. gic_data.rdists.has_vpend_valid_dirty = true;
  1478. if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
  1479. err = -ENOMEM;
  1480. goto out_free;
  1481. }
  1482. irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);
  1483. gic_data.has_rss = !!(typer & GICD_TYPER_RSS);
  1484. pr_info("Distributor has %sRange Selector support\n",
  1485. gic_data.has_rss ? "" : "no ");
  1486. if (typer & GICD_TYPER_MBIS) {
  1487. err = mbi_init(handle, gic_data.domain);
  1488. if (err)
  1489. pr_err("Failed to initialize MBIs\n");
  1490. }
  1491. set_handle_irq(gic_handle_irq);
  1492. gic_update_rdist_properties();
  1493. gic_dist_init();
  1494. gic_cpu_init();
  1495. gic_smp_init();
  1496. gic_cpu_pm_init();
  1497. gic_syscore_init();
  1498. if (gic_dist_supports_lpis()) {
  1499. its_init(handle, &gic_data.rdists, gic_data.domain);
  1500. its_cpu_init();
  1501. } else {
  1502. if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
  1503. gicv2m_init(handle, gic_data.domain);
  1504. }
  1505. gic_enable_nmi_support();
  1506. return 0;
  1507. out_free:
  1508. if (gic_data.domain)
  1509. irq_domain_remove(gic_data.domain);
  1510. free_percpu(gic_data.rdists.rdist);
  1511. return err;
  1512. }
  1513. static int __init gic_validate_dist_version(void __iomem *dist_base)
  1514. {
  1515. u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
  1516. if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4)
  1517. return -ENODEV;
  1518. return 0;
  1519. }
  1520. /* Create all possible partitions at boot time */
  1521. static void __init gic_populate_ppi_partitions(struct device_node *gic_node)
  1522. {
  1523. struct device_node *parts_node, *child_part;
  1524. int part_idx = 0, i;
  1525. int nr_parts;
  1526. struct partition_affinity *parts;
  1527. parts_node = of_get_child_by_name(gic_node, "ppi-partitions");
  1528. if (!parts_node)
  1529. return;
  1530. gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL);
  1531. if (!gic_data.ppi_descs)
  1532. return;
  1533. nr_parts = of_get_child_count(parts_node);
  1534. if (!nr_parts)
  1535. goto out_put_node;
  1536. parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL);
  1537. if (WARN_ON(!parts))
  1538. goto out_put_node;
  1539. for_each_child_of_node(parts_node, child_part) {
  1540. struct partition_affinity *part;
  1541. int n;
  1542. part = &parts[part_idx];
  1543. part->partition_id = of_node_to_fwnode(child_part);
  1544. pr_info("GIC: PPI partition %pOFn[%d] { ",
  1545. child_part, part_idx);
  1546. n = of_property_count_elems_of_size(child_part, "affinity",
  1547. sizeof(u32));
  1548. WARN_ON(n <= 0);
  1549. for (i = 0; i < n; i++) {
  1550. int err, cpu;
  1551. u32 cpu_phandle;
  1552. struct device_node *cpu_node;
  1553. err = of_property_read_u32_index(child_part, "affinity",
  1554. i, &cpu_phandle);
  1555. if (WARN_ON(err))
  1556. continue;
  1557. cpu_node = of_find_node_by_phandle(cpu_phandle);
  1558. if (WARN_ON(!cpu_node))
  1559. continue;
  1560. cpu = of_cpu_node_to_id(cpu_node);
  1561. if (WARN_ON(cpu < 0))
  1562. continue;
  1563. pr_cont("%pOF[%d] ", cpu_node, cpu);
  1564. cpumask_set_cpu(cpu, &part->mask);
  1565. }
  1566. pr_cont("}\n");
  1567. part_idx++;
  1568. }
  1569. for (i = 0; i < gic_data.ppi_nr; i++) {
  1570. unsigned int irq;
  1571. struct partition_desc *desc;
  1572. struct irq_fwspec ppi_fwspec = {
  1573. .fwnode = gic_data.fwnode,
  1574. .param_count = 3,
  1575. .param = {
  1576. [0] = GIC_IRQ_TYPE_PARTITION,
  1577. [1] = i,
  1578. [2] = IRQ_TYPE_NONE,
  1579. },
  1580. };
  1581. irq = irq_create_fwspec_mapping(&ppi_fwspec);
  1582. if (WARN_ON(!irq))
  1583. continue;
  1584. desc = partition_create_desc(gic_data.fwnode, parts, nr_parts,
  1585. irq, &partition_domain_ops);
  1586. if (WARN_ON(!desc))
  1587. continue;
  1588. gic_data.ppi_descs[i] = desc;
  1589. }
  1590. out_put_node:
  1591. of_node_put(parts_node);
  1592. }
  1593. static void __init gic_of_setup_kvm_info(struct device_node *node)
  1594. {
  1595. int ret;
  1596. struct resource r;
  1597. u32 gicv_idx;
  1598. gic_v3_kvm_info.type = GIC_V3;
  1599. gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
  1600. if (!gic_v3_kvm_info.maint_irq)
  1601. return;
  1602. if (of_property_read_u32(node, "#redistributor-regions",
  1603. &gicv_idx))
  1604. gicv_idx = 1;
  1605. gicv_idx += 3; /* Also skip GICD, GICC, GICH */
  1606. ret = of_address_to_resource(node, gicv_idx, &r);
  1607. if (!ret)
  1608. gic_v3_kvm_info.vcpu = r;
  1609. gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
  1610. gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
  1611. gic_set_kvm_info(&gic_v3_kvm_info);
  1612. }
  1613. static int __init gic_of_init(struct device_node *node, struct device_node *parent)
  1614. {
  1615. void __iomem *dist_base;
  1616. struct redist_region *rdist_regs;
  1617. u64 redist_stride;
  1618. u32 nr_redist_regions;
  1619. int err, i;
  1620. dist_base = of_iomap(node, 0);
  1621. if (!dist_base) {
  1622. pr_err("%pOF: unable to map gic dist registers\n", node);
  1623. return -ENXIO;
  1624. }
  1625. err = gic_validate_dist_version(dist_base);
  1626. if (err) {
  1627. pr_err("%pOF: no distributor detected, giving up\n", node);
  1628. goto out_unmap_dist;
  1629. }
  1630. if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
  1631. nr_redist_regions = 1;
  1632. rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs),
  1633. GFP_KERNEL);
  1634. if (!rdist_regs) {
  1635. err = -ENOMEM;
  1636. goto out_unmap_dist;
  1637. }
  1638. for (i = 0; i < nr_redist_regions; i++) {
  1639. struct resource res;
  1640. int ret;
  1641. ret = of_address_to_resource(node, 1 + i, &res);
  1642. rdist_regs[i].redist_base = of_iomap(node, 1 + i);
  1643. if (ret || !rdist_regs[i].redist_base) {
  1644. pr_err("%pOF: couldn't map region %d\n", node, i);
  1645. err = -ENODEV;
  1646. goto out_unmap_rdist;
  1647. }
  1648. rdist_regs[i].phys_base = res.start;
  1649. }
  1650. if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
  1651. redist_stride = 0;
  1652. gic_enable_of_quirks(node, gic_quirks, &gic_data);
  1653. err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions,
  1654. redist_stride, &node->fwnode);
  1655. if (err)
  1656. goto out_unmap_rdist;
  1657. gic_populate_ppi_partitions(node);
  1658. if (static_branch_likely(&supports_deactivate_key))
  1659. gic_of_setup_kvm_info(node);
  1660. return 0;
  1661. out_unmap_rdist:
  1662. for (i = 0; i < nr_redist_regions; i++)
  1663. if (rdist_regs[i].redist_base)
  1664. iounmap(rdist_regs[i].redist_base);
  1665. kfree(rdist_regs);
  1666. out_unmap_dist:
  1667. iounmap(dist_base);
  1668. return err;
  1669. }
  1670. IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);
  1671. #ifdef CONFIG_ACPI
  1672. static struct
  1673. {
  1674. void __iomem *dist_base;
  1675. struct redist_region *redist_regs;
  1676. u32 nr_redist_regions;
  1677. bool single_redist;
  1678. int enabled_rdists;
  1679. u32 maint_irq;
  1680. int maint_irq_mode;
  1681. phys_addr_t vcpu_base;
  1682. } acpi_data __initdata;
  1683. static void __init
  1684. gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base)
  1685. {
  1686. static int count = 0;
  1687. acpi_data.redist_regs[count].phys_base = phys_base;
  1688. acpi_data.redist_regs[count].redist_base = redist_base;
  1689. acpi_data.redist_regs[count].single_redist = acpi_data.single_redist;
  1690. count++;
  1691. }
  1692. static int __init
  1693. gic_acpi_parse_madt_redist(union acpi_subtable_headers *header,
  1694. const unsigned long end)
  1695. {
  1696. struct acpi_madt_generic_redistributor *redist =
  1697. (struct acpi_madt_generic_redistributor *)header;
  1698. void __iomem *redist_base;
  1699. redist_base = ioremap(redist->base_address, redist->length);
  1700. if (!redist_base) {
  1701. pr_err("Couldn't map GICR region @%llx\n", redist->base_address);
  1702. return -ENOMEM;
  1703. }
  1704. gic_acpi_register_redist(redist->base_address, redist_base);
  1705. return 0;
  1706. }
  1707. static int __init
  1708. gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header,
  1709. const unsigned long end)
  1710. {
  1711. struct acpi_madt_generic_interrupt *gicc =
  1712. (struct acpi_madt_generic_interrupt *)header;
  1713. u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
  1714. u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2;
  1715. void __iomem *redist_base;
  1716. /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */
  1717. if (!(gicc->flags & ACPI_MADT_ENABLED))
  1718. return 0;
  1719. redist_base = ioremap(gicc->gicr_base_address, size);
  1720. if (!redist_base)
  1721. return -ENOMEM;
  1722. gic_acpi_register_redist(gicc->gicr_base_address, redist_base);
  1723. return 0;
  1724. }
  1725. static int __init gic_acpi_collect_gicr_base(void)
  1726. {
  1727. acpi_tbl_entry_handler redist_parser;
  1728. enum acpi_madt_type type;
  1729. if (acpi_data.single_redist) {
  1730. type = ACPI_MADT_TYPE_GENERIC_INTERRUPT;
  1731. redist_parser = gic_acpi_parse_madt_gicc;
  1732. } else {
  1733. type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR;
  1734. redist_parser = gic_acpi_parse_madt_redist;
  1735. }
  1736. /* Collect redistributor base addresses in GICR entries */
  1737. if (acpi_table_parse_madt(type, redist_parser, 0) > 0)
  1738. return 0;
  1739. pr_info("No valid GICR entries exist\n");
  1740. return -ENODEV;
  1741. }
  1742. static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header,
  1743. const unsigned long end)
  1744. {
  1745. /* Subtable presence means that redist exists, that's it */
  1746. return 0;
  1747. }
  1748. static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header,
  1749. const unsigned long end)
  1750. {
  1751. struct acpi_madt_generic_interrupt *gicc =
  1752. (struct acpi_madt_generic_interrupt *)header;
  1753. /*
  1754. * If GICC is enabled and has valid gicr base address, then it means
  1755. * GICR base is presented via GICC
  1756. */
  1757. if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address) {
  1758. acpi_data.enabled_rdists++;
  1759. return 0;
  1760. }
  1761. /*
  1762. * It's perfectly valid firmware can pass disabled GICC entry, driver
  1763. * should not treat as errors, skip the entry instead of probe fail.
  1764. */
  1765. if (!(gicc->flags & ACPI_MADT_ENABLED))
  1766. return 0;
  1767. return -ENODEV;
  1768. }
  1769. static int __init gic_acpi_count_gicr_regions(void)
  1770. {
  1771. int count;
  1772. /*
  1773. * Count how many redistributor regions we have. It is not allowed
  1774. * to mix redistributor description, GICR and GICC subtables have to be
  1775. * mutually exclusive.
  1776. */
  1777. count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR,
  1778. gic_acpi_match_gicr, 0);
  1779. if (count > 0) {
  1780. acpi_data.single_redist = false;
  1781. return count;
  1782. }
  1783. count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
  1784. gic_acpi_match_gicc, 0);
  1785. if (count > 0) {
  1786. acpi_data.single_redist = true;
  1787. count = acpi_data.enabled_rdists;
  1788. }
  1789. return count;
  1790. }
  1791. static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header,
  1792. struct acpi_probe_entry *ape)
  1793. {
  1794. struct acpi_madt_generic_distributor *dist;
  1795. int count;
  1796. dist = (struct acpi_madt_generic_distributor *)header;
  1797. if (dist->version != ape->driver_data)
  1798. return false;
  1799. /* We need to do that exercise anyway, the sooner the better */
  1800. count = gic_acpi_count_gicr_regions();
  1801. if (count <= 0)
  1802. return false;
  1803. acpi_data.nr_redist_regions = count;
  1804. return true;
  1805. }
  1806. static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header,
  1807. const unsigned long end)
  1808. {
  1809. struct acpi_madt_generic_interrupt *gicc =
  1810. (struct acpi_madt_generic_interrupt *)header;
  1811. int maint_irq_mode;
  1812. static int first_madt = true;
  1813. /* Skip unusable CPUs */
  1814. if (!(gicc->flags & ACPI_MADT_ENABLED))
  1815. return 0;
  1816. maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ?
  1817. ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE;
  1818. if (first_madt) {
  1819. first_madt = false;
  1820. acpi_data.maint_irq = gicc->vgic_interrupt;
  1821. acpi_data.maint_irq_mode = maint_irq_mode;
  1822. acpi_data.vcpu_base = gicc->gicv_base_address;
  1823. return 0;
  1824. }
  1825. /*
  1826. * The maintenance interrupt and GICV should be the same for every CPU
  1827. */
  1828. if ((acpi_data.maint_irq != gicc->vgic_interrupt) ||
  1829. (acpi_data.maint_irq_mode != maint_irq_mode) ||
  1830. (acpi_data.vcpu_base != gicc->gicv_base_address))
  1831. return -EINVAL;
  1832. return 0;
  1833. }
  1834. static bool __init gic_acpi_collect_virt_info(void)
  1835. {
  1836. int count;
  1837. count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
  1838. gic_acpi_parse_virt_madt_gicc, 0);
  1839. return (count > 0);
  1840. }
  1841. #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K)
  1842. #define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K)
  1843. #define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K)
  1844. static void __init gic_acpi_setup_kvm_info(void)
  1845. {
  1846. int irq;
  1847. if (!gic_acpi_collect_virt_info()) {
  1848. pr_warn("Unable to get hardware information used for virtualization\n");
  1849. return;
  1850. }
  1851. gic_v3_kvm_info.type = GIC_V3;
  1852. irq = acpi_register_gsi(NULL, acpi_data.maint_irq,
  1853. acpi_data.maint_irq_mode,
  1854. ACPI_ACTIVE_HIGH);
  1855. if (irq <= 0)
  1856. return;
  1857. gic_v3_kvm_info.maint_irq = irq;
  1858. if (acpi_data.vcpu_base) {
  1859. struct resource *vcpu = &gic_v3_kvm_info.vcpu;
  1860. vcpu->flags = IORESOURCE_MEM;
  1861. vcpu->start = acpi_data.vcpu_base;
  1862. vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1;
  1863. }
  1864. gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
  1865. gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
  1866. gic_set_kvm_info(&gic_v3_kvm_info);
  1867. }
  1868. static int __init
  1869. gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end)
  1870. {
  1871. struct acpi_madt_generic_distributor *dist;
  1872. struct fwnode_handle *domain_handle;
  1873. size_t size;
  1874. int i, err;
  1875. /* Get distributor base address */
  1876. dist = (struct acpi_madt_generic_distributor *)header;
  1877. acpi_data.dist_base = ioremap(dist->base_address,
  1878. ACPI_GICV3_DIST_MEM_SIZE);
  1879. if (!acpi_data.dist_base) {
  1880. pr_err("Unable to map GICD registers\n");
  1881. return -ENOMEM;
  1882. }
  1883. err = gic_validate_dist_version(acpi_data.dist_base);
  1884. if (err) {
  1885. pr_err("No distributor detected at @%p, giving up\n",
  1886. acpi_data.dist_base);
  1887. goto out_dist_unmap;
  1888. }
  1889. size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions;
  1890. acpi_data.redist_regs = kzalloc(size, GFP_KERNEL);
  1891. if (!acpi_data.redist_regs) {
  1892. err = -ENOMEM;
  1893. goto out_dist_unmap;
  1894. }
  1895. err = gic_acpi_collect_gicr_base();
  1896. if (err)
  1897. goto out_redist_unmap;
  1898. domain_handle = irq_domain_alloc_fwnode(&dist->base_address);
  1899. if (!domain_handle) {
  1900. err = -ENOMEM;
  1901. goto out_redist_unmap;
  1902. }
  1903. err = gic_init_bases(acpi_data.dist_base, acpi_data.redist_regs,
  1904. acpi_data.nr_redist_regions, 0, domain_handle);
  1905. if (err)
  1906. goto out_fwhandle_free;
  1907. acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, domain_handle);
  1908. if (static_branch_likely(&supports_deactivate_key))
  1909. gic_acpi_setup_kvm_info();
  1910. return 0;
  1911. out_fwhandle_free:
  1912. irq_domain_free_fwnode(domain_handle);
  1913. out_redist_unmap:
  1914. for (i = 0; i < acpi_data.nr_redist_regions; i++)
  1915. if (acpi_data.redist_regs[i].redist_base)
  1916. iounmap(acpi_data.redist_regs[i].redist_base);
  1917. kfree(acpi_data.redist_regs);
  1918. out_dist_unmap:
  1919. iounmap(acpi_data.dist_base);
  1920. return err;
  1921. }
  1922. IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
  1923. acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3,
  1924. gic_acpi_init);
  1925. IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
  1926. acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4,
  1927. gic_acpi_init);
  1928. IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
  1929. acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE,
  1930. gic_acpi_init);
  1931. #endif