chip.c 42 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
  4. * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
  5. *
  6. * This file contains the core interrupt handling code, for irq-chip based
  7. * architectures. Detailed information is available in
  8. * Documentation/core-api/genericirq.rst
  9. */
  10. #include <linux/irq.h>
  11. #include <linux/msi.h>
  12. #include <linux/module.h>
  13. #include <linux/interrupt.h>
  14. #include <linux/kernel_stat.h>
  15. #include <linux/irqdomain.h>
  16. #include <linux/wakeup_reason.h>
  17. #include <trace/events/irq.h>
  18. #include "internals.h"
  19. static irqreturn_t bad_chained_irq(int irq, void *dev_id)
  20. {
  21. WARN_ONCE(1, "Chained irq %d should not call an action\n", irq);
  22. return IRQ_NONE;
  23. }
  24. /*
  25. * Chained handlers should never call action on their IRQ. This default
  26. * action will emit warning if such thing happens.
  27. */
  28. struct irqaction chained_action = {
  29. .handler = bad_chained_irq,
  30. };
  31. /**
  32. * irq_set_chip - set the irq chip for an irq
  33. * @irq: irq number
  34. * @chip: pointer to irq chip description structure
  35. */
  36. int irq_set_chip(unsigned int irq, struct irq_chip *chip)
  37. {
  38. unsigned long flags;
  39. struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
  40. if (!desc)
  41. return -EINVAL;
  42. if (!chip)
  43. chip = &no_irq_chip;
  44. desc->irq_data.chip = chip;
  45. irq_put_desc_unlock(desc, flags);
  46. /*
  47. * For !CONFIG_SPARSE_IRQ make the irq show up in
  48. * allocated_irqs.
  49. */
  50. irq_mark_irq(irq);
  51. return 0;
  52. }
  53. EXPORT_SYMBOL(irq_set_chip);
  54. /**
  55. * irq_set_type - set the irq trigger type for an irq
  56. * @irq: irq number
  57. * @type: IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
  58. */
  59. int irq_set_irq_type(unsigned int irq, unsigned int type)
  60. {
  61. unsigned long flags;
  62. struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
  63. int ret = 0;
  64. if (!desc)
  65. return -EINVAL;
  66. ret = __irq_set_trigger(desc, type);
  67. irq_put_desc_busunlock(desc, flags);
  68. return ret;
  69. }
  70. EXPORT_SYMBOL(irq_set_irq_type);
  71. /**
  72. * irq_set_handler_data - set irq handler data for an irq
  73. * @irq: Interrupt number
  74. * @data: Pointer to interrupt specific data
  75. *
  76. * Set the hardware irq controller data for an irq
  77. */
  78. int irq_set_handler_data(unsigned int irq, void *data)
  79. {
  80. unsigned long flags;
  81. struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
  82. if (!desc)
  83. return -EINVAL;
  84. desc->irq_common_data.handler_data = data;
  85. irq_put_desc_unlock(desc, flags);
  86. return 0;
  87. }
  88. EXPORT_SYMBOL(irq_set_handler_data);
  89. /**
  90. * irq_set_msi_desc_off - set MSI descriptor data for an irq at offset
  91. * @irq_base: Interrupt number base
  92. * @irq_offset: Interrupt number offset
  93. * @entry: Pointer to MSI descriptor data
  94. *
  95. * Set the MSI descriptor entry for an irq at offset
  96. */
  97. int irq_set_msi_desc_off(unsigned int irq_base, unsigned int irq_offset,
  98. struct msi_desc *entry)
  99. {
  100. unsigned long flags;
  101. struct irq_desc *desc = irq_get_desc_lock(irq_base + irq_offset, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
  102. if (!desc)
  103. return -EINVAL;
  104. desc->irq_common_data.msi_desc = entry;
  105. if (entry && !irq_offset)
  106. entry->irq = irq_base;
  107. irq_put_desc_unlock(desc, flags);
  108. return 0;
  109. }
  110. /**
  111. * irq_set_msi_desc - set MSI descriptor data for an irq
  112. * @irq: Interrupt number
  113. * @entry: Pointer to MSI descriptor data
  114. *
  115. * Set the MSI descriptor entry for an irq
  116. */
  117. int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry)
  118. {
  119. return irq_set_msi_desc_off(irq, 0, entry);
  120. }
  121. /**
  122. * irq_set_chip_data - set irq chip data for an irq
  123. * @irq: Interrupt number
  124. * @data: Pointer to chip specific data
  125. *
  126. * Set the hardware irq chip data for an irq
  127. */
  128. int irq_set_chip_data(unsigned int irq, void *data)
  129. {
  130. unsigned long flags;
  131. struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
  132. if (!desc)
  133. return -EINVAL;
  134. desc->irq_data.chip_data = data;
  135. irq_put_desc_unlock(desc, flags);
  136. return 0;
  137. }
  138. EXPORT_SYMBOL(irq_set_chip_data);
  139. struct irq_data *irq_get_irq_data(unsigned int irq)
  140. {
  141. struct irq_desc *desc = irq_to_desc(irq);
  142. return desc ? &desc->irq_data : NULL;
  143. }
  144. EXPORT_SYMBOL_GPL(irq_get_irq_data);
  145. static void irq_state_clr_disabled(struct irq_desc *desc)
  146. {
  147. irqd_clear(&desc->irq_data, IRQD_IRQ_DISABLED);
  148. }
  149. static void irq_state_clr_masked(struct irq_desc *desc)
  150. {
  151. irqd_clear(&desc->irq_data, IRQD_IRQ_MASKED);
  152. }
  153. static void irq_state_clr_started(struct irq_desc *desc)
  154. {
  155. irqd_clear(&desc->irq_data, IRQD_IRQ_STARTED);
  156. }
  157. static void irq_state_set_started(struct irq_desc *desc)
  158. {
  159. irqd_set(&desc->irq_data, IRQD_IRQ_STARTED);
  160. }
  161. enum {
  162. IRQ_STARTUP_NORMAL,
  163. IRQ_STARTUP_MANAGED,
  164. IRQ_STARTUP_ABORT,
  165. };
  166. #ifdef CONFIG_SMP
  167. static int
  168. __irq_startup_managed(struct irq_desc *desc, struct cpumask *aff, bool force)
  169. {
  170. struct irq_data *d = irq_desc_get_irq_data(desc);
  171. if (!irqd_affinity_is_managed(d))
  172. return IRQ_STARTUP_NORMAL;
  173. irqd_clr_managed_shutdown(d);
  174. if (cpumask_any_and(aff, cpu_online_mask) >= nr_cpu_ids) {
  175. /*
  176. * Catch code which fiddles with enable_irq() on a managed
  177. * and potentially shutdown IRQ. Chained interrupt
  178. * installment or irq auto probing should not happen on
  179. * managed irqs either.
  180. */
  181. if (WARN_ON_ONCE(force))
  182. return IRQ_STARTUP_ABORT;
  183. /*
  184. * The interrupt was requested, but there is no online CPU
  185. * in it's affinity mask. Put it into managed shutdown
  186. * state and let the cpu hotplug mechanism start it up once
  187. * a CPU in the mask becomes available.
  188. */
  189. return IRQ_STARTUP_ABORT;
  190. }
  191. /*
  192. * Managed interrupts have reserved resources, so this should not
  193. * happen.
  194. */
  195. if (WARN_ON(irq_domain_activate_irq(d, false)))
  196. return IRQ_STARTUP_ABORT;
  197. return IRQ_STARTUP_MANAGED;
  198. }
  199. #else
  200. static __always_inline int
  201. __irq_startup_managed(struct irq_desc *desc, struct cpumask *aff, bool force)
  202. {
  203. return IRQ_STARTUP_NORMAL;
  204. }
  205. #endif
  206. static int __irq_startup(struct irq_desc *desc)
  207. {
  208. struct irq_data *d = irq_desc_get_irq_data(desc);
  209. int ret = 0;
  210. /* Warn if this interrupt is not activated but try nevertheless */
  211. WARN_ON_ONCE(!irqd_is_activated(d));
  212. if (d->chip->irq_startup) {
  213. ret = d->chip->irq_startup(d);
  214. irq_state_clr_disabled(desc);
  215. irq_state_clr_masked(desc);
  216. } else {
  217. irq_enable(desc);
  218. }
  219. irq_state_set_started(desc);
  220. return ret;
  221. }
  222. int irq_startup(struct irq_desc *desc, bool resend, bool force)
  223. {
  224. struct irq_data *d = irq_desc_get_irq_data(desc);
  225. struct cpumask *aff = irq_data_get_affinity_mask(d);
  226. int ret = 0;
  227. desc->depth = 0;
  228. if (irqd_is_started(d)) {
  229. irq_enable(desc);
  230. } else {
  231. switch (__irq_startup_managed(desc, aff, force)) {
  232. case IRQ_STARTUP_NORMAL:
  233. if (d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP)
  234. irq_setup_affinity(desc);
  235. ret = __irq_startup(desc);
  236. if (!(d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP))
  237. irq_setup_affinity(desc);
  238. break;
  239. case IRQ_STARTUP_MANAGED:
  240. irq_do_set_affinity(d, aff, false);
  241. ret = __irq_startup(desc);
  242. break;
  243. case IRQ_STARTUP_ABORT:
  244. irqd_set_managed_shutdown(d);
  245. return 0;
  246. }
  247. }
  248. if (resend)
  249. check_irq_resend(desc, false);
  250. return ret;
  251. }
  252. int irq_activate(struct irq_desc *desc)
  253. {
  254. struct irq_data *d = irq_desc_get_irq_data(desc);
  255. if (!irqd_affinity_is_managed(d))
  256. return irq_domain_activate_irq(d, false);
  257. return 0;
  258. }
  259. int irq_activate_and_startup(struct irq_desc *desc, bool resend)
  260. {
  261. if (WARN_ON(irq_activate(desc)))
  262. return 0;
  263. return irq_startup(desc, resend, IRQ_START_FORCE);
  264. }
  265. static void __irq_disable(struct irq_desc *desc, bool mask);
  266. void irq_shutdown(struct irq_desc *desc)
  267. {
  268. if (irqd_is_started(&desc->irq_data)) {
  269. desc->depth = 1;
  270. if (desc->irq_data.chip->irq_shutdown) {
  271. desc->irq_data.chip->irq_shutdown(&desc->irq_data);
  272. irq_state_set_disabled(desc);
  273. irq_state_set_masked(desc);
  274. } else {
  275. __irq_disable(desc, true);
  276. }
  277. irq_state_clr_started(desc);
  278. }
  279. }
  280. void irq_shutdown_and_deactivate(struct irq_desc *desc)
  281. {
  282. irq_shutdown(desc);
  283. /*
  284. * This must be called even if the interrupt was never started up,
  285. * because the activation can happen before the interrupt is
  286. * available for request/startup. It has it's own state tracking so
  287. * it's safe to call it unconditionally.
  288. */
  289. irq_domain_deactivate_irq(&desc->irq_data);
  290. }
  291. void irq_enable(struct irq_desc *desc)
  292. {
  293. if (!irqd_irq_disabled(&desc->irq_data)) {
  294. unmask_irq(desc);
  295. } else {
  296. irq_state_clr_disabled(desc);
  297. if (desc->irq_data.chip->irq_enable) {
  298. desc->irq_data.chip->irq_enable(&desc->irq_data);
  299. irq_state_clr_masked(desc);
  300. } else {
  301. unmask_irq(desc);
  302. }
  303. }
  304. }
  305. static void __irq_disable(struct irq_desc *desc, bool mask)
  306. {
  307. if (irqd_irq_disabled(&desc->irq_data)) {
  308. if (mask)
  309. mask_irq(desc);
  310. } else {
  311. irq_state_set_disabled(desc);
  312. if (desc->irq_data.chip->irq_disable) {
  313. desc->irq_data.chip->irq_disable(&desc->irq_data);
  314. irq_state_set_masked(desc);
  315. } else if (mask) {
  316. mask_irq(desc);
  317. }
  318. }
  319. }
  320. /**
  321. * irq_disable - Mark interrupt disabled
  322. * @desc: irq descriptor which should be disabled
  323. *
  324. * If the chip does not implement the irq_disable callback, we
  325. * use a lazy disable approach. That means we mark the interrupt
  326. * disabled, but leave the hardware unmasked. That's an
  327. * optimization because we avoid the hardware access for the
  328. * common case where no interrupt happens after we marked it
  329. * disabled. If an interrupt happens, then the interrupt flow
  330. * handler masks the line at the hardware level and marks it
  331. * pending.
  332. *
  333. * If the interrupt chip does not implement the irq_disable callback,
  334. * a driver can disable the lazy approach for a particular irq line by
  335. * calling 'irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY)'. This can
  336. * be used for devices which cannot disable the interrupt at the
  337. * device level under certain circumstances and have to use
  338. * disable_irq[_nosync] instead.
  339. */
  340. void irq_disable(struct irq_desc *desc)
  341. {
  342. __irq_disable(desc, irq_settings_disable_unlazy(desc));
  343. }
  344. void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
  345. {
  346. if (desc->irq_data.chip->irq_enable)
  347. desc->irq_data.chip->irq_enable(&desc->irq_data);
  348. else
  349. desc->irq_data.chip->irq_unmask(&desc->irq_data);
  350. cpumask_set_cpu(cpu, desc->percpu_enabled);
  351. }
  352. void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu)
  353. {
  354. if (desc->irq_data.chip->irq_disable)
  355. desc->irq_data.chip->irq_disable(&desc->irq_data);
  356. else
  357. desc->irq_data.chip->irq_mask(&desc->irq_data);
  358. cpumask_clear_cpu(cpu, desc->percpu_enabled);
  359. }
  360. static inline void mask_ack_irq(struct irq_desc *desc)
  361. {
  362. if (desc->irq_data.chip->irq_mask_ack) {
  363. desc->irq_data.chip->irq_mask_ack(&desc->irq_data);
  364. irq_state_set_masked(desc);
  365. } else {
  366. mask_irq(desc);
  367. if (desc->irq_data.chip->irq_ack)
  368. desc->irq_data.chip->irq_ack(&desc->irq_data);
  369. }
  370. }
  371. void mask_irq(struct irq_desc *desc)
  372. {
  373. if (irqd_irq_masked(&desc->irq_data))
  374. return;
  375. if (desc->irq_data.chip->irq_mask) {
  376. desc->irq_data.chip->irq_mask(&desc->irq_data);
  377. irq_state_set_masked(desc);
  378. }
  379. }
  380. void unmask_irq(struct irq_desc *desc)
  381. {
  382. if (!irqd_irq_masked(&desc->irq_data))
  383. return;
  384. if (desc->irq_data.chip->irq_unmask) {
  385. desc->irq_data.chip->irq_unmask(&desc->irq_data);
  386. irq_state_clr_masked(desc);
  387. }
  388. }
  389. void unmask_threaded_irq(struct irq_desc *desc)
  390. {
  391. struct irq_chip *chip = desc->irq_data.chip;
  392. if (chip->flags & IRQCHIP_EOI_THREADED)
  393. chip->irq_eoi(&desc->irq_data);
  394. unmask_irq(desc);
  395. }
  396. /*
  397. * handle_nested_irq - Handle a nested irq from a irq thread
  398. * @irq: the interrupt number
  399. *
  400. * Handle interrupts which are nested into a threaded interrupt
  401. * handler. The handler function is called inside the calling
  402. * threads context.
  403. */
  404. void handle_nested_irq(unsigned int irq)
  405. {
  406. struct irq_desc *desc = irq_to_desc(irq);
  407. struct irqaction *action;
  408. irqreturn_t action_ret;
  409. might_sleep();
  410. raw_spin_lock_irq(&desc->lock);
  411. desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
  412. action = desc->action;
  413. if (unlikely(!action || irqd_irq_disabled(&desc->irq_data))) {
  414. desc->istate |= IRQS_PENDING;
  415. goto out_unlock;
  416. }
  417. kstat_incr_irqs_this_cpu(desc);
  418. irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
  419. raw_spin_unlock_irq(&desc->lock);
  420. action_ret = IRQ_NONE;
  421. for_each_action_of_desc(desc, action)
  422. action_ret |= action->thread_fn(action->irq, action->dev_id);
  423. if (!noirqdebug)
  424. note_interrupt(desc, action_ret);
  425. raw_spin_lock_irq(&desc->lock);
  426. irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
  427. out_unlock:
  428. raw_spin_unlock_irq(&desc->lock);
  429. }
  430. EXPORT_SYMBOL_GPL(handle_nested_irq);
  431. static bool irq_check_poll(struct irq_desc *desc)
  432. {
  433. if (!(desc->istate & IRQS_POLL_INPROGRESS))
  434. return false;
  435. return irq_wait_for_poll(desc);
  436. }
  437. static bool irq_may_run(struct irq_desc *desc)
  438. {
  439. unsigned int mask = IRQD_IRQ_INPROGRESS | IRQD_WAKEUP_ARMED;
  440. /*
  441. * If the interrupt is not in progress and is not an armed
  442. * wakeup interrupt, proceed.
  443. */
  444. if (!irqd_has_set(&desc->irq_data, mask)) {
  445. #ifdef CONFIG_PM_SLEEP
  446. if (unlikely(desc->no_suspend_depth &&
  447. irqd_is_wakeup_set(&desc->irq_data))) {
  448. unsigned int irq = irq_desc_get_irq(desc);
  449. const char *name = "(unnamed)";
  450. if (desc->action && desc->action->name)
  451. name = desc->action->name;
  452. log_abnormal_wakeup_reason("misconfigured IRQ %u %s",
  453. irq, name);
  454. }
  455. #endif
  456. return true;
  457. }
  458. /*
  459. * If the interrupt is an armed wakeup source, mark it pending
  460. * and suspended, disable it and notify the pm core about the
  461. * event.
  462. */
  463. if (irq_pm_check_wakeup(desc))
  464. return false;
  465. /*
  466. * Handle a potential concurrent poll on a different core.
  467. */
  468. return irq_check_poll(desc);
  469. }
  470. /**
  471. * handle_simple_irq - Simple and software-decoded IRQs.
  472. * @desc: the interrupt description structure for this irq
  473. *
  474. * Simple interrupts are either sent from a demultiplexing interrupt
  475. * handler or come from hardware, where no interrupt hardware control
  476. * is necessary.
  477. *
  478. * Note: The caller is expected to handle the ack, clear, mask and
  479. * unmask issues if necessary.
  480. */
  481. void handle_simple_irq(struct irq_desc *desc)
  482. {
  483. raw_spin_lock(&desc->lock);
  484. if (!irq_may_run(desc))
  485. goto out_unlock;
  486. desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
  487. if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
  488. desc->istate |= IRQS_PENDING;
  489. goto out_unlock;
  490. }
  491. kstat_incr_irqs_this_cpu(desc);
  492. handle_irq_event(desc);
  493. out_unlock:
  494. raw_spin_unlock(&desc->lock);
  495. }
  496. EXPORT_SYMBOL_GPL(handle_simple_irq);
  497. /**
  498. * handle_untracked_irq - Simple and software-decoded IRQs.
  499. * @desc: the interrupt description structure for this irq
  500. *
  501. * Untracked interrupts are sent from a demultiplexing interrupt
  502. * handler when the demultiplexer does not know which device it its
  503. * multiplexed irq domain generated the interrupt. IRQ's handled
  504. * through here are not subjected to stats tracking, randomness, or
  505. * spurious interrupt detection.
  506. *
  507. * Note: Like handle_simple_irq, the caller is expected to handle
  508. * the ack, clear, mask and unmask issues if necessary.
  509. */
  510. void handle_untracked_irq(struct irq_desc *desc)
  511. {
  512. unsigned int flags = 0;
  513. raw_spin_lock(&desc->lock);
  514. if (!irq_may_run(desc))
  515. goto out_unlock;
  516. desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
  517. if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
  518. desc->istate |= IRQS_PENDING;
  519. goto out_unlock;
  520. }
  521. desc->istate &= ~IRQS_PENDING;
  522. irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
  523. raw_spin_unlock(&desc->lock);
  524. __handle_irq_event_percpu(desc, &flags);
  525. raw_spin_lock(&desc->lock);
  526. irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
  527. out_unlock:
  528. raw_spin_unlock(&desc->lock);
  529. }
  530. EXPORT_SYMBOL_GPL(handle_untracked_irq);
  531. /*
  532. * Called unconditionally from handle_level_irq() and only for oneshot
  533. * interrupts from handle_fasteoi_irq()
  534. */
  535. static void cond_unmask_irq(struct irq_desc *desc)
  536. {
  537. /*
  538. * We need to unmask in the following cases:
  539. * - Standard level irq (IRQF_ONESHOT is not set)
  540. * - Oneshot irq which did not wake the thread (caused by a
  541. * spurious interrupt or a primary handler handling it
  542. * completely).
  543. */
  544. if (!irqd_irq_disabled(&desc->irq_data) &&
  545. irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot)
  546. unmask_irq(desc);
  547. }
  548. /**
  549. * handle_level_irq - Level type irq handler
  550. * @desc: the interrupt description structure for this irq
  551. *
  552. * Level type interrupts are active as long as the hardware line has
  553. * the active level. This may require to mask the interrupt and unmask
  554. * it after the associated handler has acknowledged the device, so the
  555. * interrupt line is back to inactive.
  556. */
  557. void handle_level_irq(struct irq_desc *desc)
  558. {
  559. raw_spin_lock(&desc->lock);
  560. mask_ack_irq(desc);
  561. if (!irq_may_run(desc))
  562. goto out_unlock;
  563. desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
  564. /*
  565. * If its disabled or no action available
  566. * keep it masked and get out of here
  567. */
  568. if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
  569. desc->istate |= IRQS_PENDING;
  570. goto out_unlock;
  571. }
  572. kstat_incr_irqs_this_cpu(desc);
  573. handle_irq_event(desc);
  574. cond_unmask_irq(desc);
  575. out_unlock:
  576. raw_spin_unlock(&desc->lock);
  577. }
  578. EXPORT_SYMBOL_GPL(handle_level_irq);
  579. static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
  580. {
  581. if (!(desc->istate & IRQS_ONESHOT)) {
  582. chip->irq_eoi(&desc->irq_data);
  583. return;
  584. }
  585. /*
  586. * We need to unmask in the following cases:
  587. * - Oneshot irq which did not wake the thread (caused by a
  588. * spurious interrupt or a primary handler handling it
  589. * completely).
  590. */
  591. if (!irqd_irq_disabled(&desc->irq_data) &&
  592. irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot) {
  593. chip->irq_eoi(&desc->irq_data);
  594. unmask_irq(desc);
  595. } else if (!(chip->flags & IRQCHIP_EOI_THREADED)) {
  596. chip->irq_eoi(&desc->irq_data);
  597. }
  598. }
  599. /**
  600. * handle_fasteoi_irq - irq handler for transparent controllers
  601. * @desc: the interrupt description structure for this irq
  602. *
  603. * Only a single callback will be issued to the chip: an ->eoi()
  604. * call when the interrupt has been serviced. This enables support
  605. * for modern forms of interrupt handlers, which handle the flow
  606. * details in hardware, transparently.
  607. */
  608. void handle_fasteoi_irq(struct irq_desc *desc)
  609. {
  610. struct irq_chip *chip = desc->irq_data.chip;
  611. raw_spin_lock(&desc->lock);
  612. if (!irq_may_run(desc))
  613. goto out;
  614. desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
  615. /*
  616. * If its disabled or no action available
  617. * then mask it and get out of here:
  618. */
  619. if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
  620. desc->istate |= IRQS_PENDING;
  621. mask_irq(desc);
  622. goto out;
  623. }
  624. kstat_incr_irqs_this_cpu(desc);
  625. if (desc->istate & IRQS_ONESHOT)
  626. mask_irq(desc);
  627. handle_irq_event(desc);
  628. cond_unmask_eoi_irq(desc, chip);
  629. raw_spin_unlock(&desc->lock);
  630. return;
  631. out:
  632. if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
  633. chip->irq_eoi(&desc->irq_data);
  634. raw_spin_unlock(&desc->lock);
  635. }
  636. EXPORT_SYMBOL_GPL(handle_fasteoi_irq);
  637. /**
  638. * handle_fasteoi_nmi - irq handler for NMI interrupt lines
  639. * @desc: the interrupt description structure for this irq
  640. *
  641. * A simple NMI-safe handler, considering the restrictions
  642. * from request_nmi.
  643. *
  644. * Only a single callback will be issued to the chip: an ->eoi()
  645. * call when the interrupt has been serviced. This enables support
  646. * for modern forms of interrupt handlers, which handle the flow
  647. * details in hardware, transparently.
  648. */
  649. void handle_fasteoi_nmi(struct irq_desc *desc)
  650. {
  651. struct irq_chip *chip = irq_desc_get_chip(desc);
  652. struct irqaction *action = desc->action;
  653. unsigned int irq = irq_desc_get_irq(desc);
  654. irqreturn_t res;
  655. __kstat_incr_irqs_this_cpu(desc);
  656. trace_irq_handler_entry(irq, action);
  657. /*
  658. * NMIs cannot be shared, there is only one action.
  659. */
  660. res = action->handler(irq, action->dev_id);
  661. trace_irq_handler_exit(irq, action, res);
  662. if (chip->irq_eoi)
  663. chip->irq_eoi(&desc->irq_data);
  664. }
  665. EXPORT_SYMBOL_GPL(handle_fasteoi_nmi);
  666. /**
  667. * handle_edge_irq - edge type IRQ handler
  668. * @desc: the interrupt description structure for this irq
  669. *
  670. * Interrupt occures on the falling and/or rising edge of a hardware
  671. * signal. The occurrence is latched into the irq controller hardware
  672. * and must be acked in order to be reenabled. After the ack another
  673. * interrupt can happen on the same source even before the first one
  674. * is handled by the associated event handler. If this happens it
  675. * might be necessary to disable (mask) the interrupt depending on the
  676. * controller hardware. This requires to reenable the interrupt inside
  677. * of the loop which handles the interrupts which have arrived while
  678. * the handler was running. If all pending interrupts are handled, the
  679. * loop is left.
  680. */
  681. void handle_edge_irq(struct irq_desc *desc)
  682. {
  683. raw_spin_lock(&desc->lock);
  684. desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
  685. if (!irq_may_run(desc)) {
  686. desc->istate |= IRQS_PENDING;
  687. mask_ack_irq(desc);
  688. goto out_unlock;
  689. }
  690. /*
  691. * If its disabled or no action available then mask it and get
  692. * out of here.
  693. */
  694. if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
  695. desc->istate |= IRQS_PENDING;
  696. mask_ack_irq(desc);
  697. goto out_unlock;
  698. }
  699. kstat_incr_irqs_this_cpu(desc);
  700. /* Start handling the irq */
  701. desc->irq_data.chip->irq_ack(&desc->irq_data);
  702. do {
  703. if (unlikely(!desc->action)) {
  704. mask_irq(desc);
  705. goto out_unlock;
  706. }
  707. /*
  708. * When another irq arrived while we were handling
  709. * one, we could have masked the irq.
  710. * Renable it, if it was not disabled in meantime.
  711. */
  712. if (unlikely(desc->istate & IRQS_PENDING)) {
  713. if (!irqd_irq_disabled(&desc->irq_data) &&
  714. irqd_irq_masked(&desc->irq_data))
  715. unmask_irq(desc);
  716. }
  717. handle_irq_event(desc);
  718. } while ((desc->istate & IRQS_PENDING) &&
  719. !irqd_irq_disabled(&desc->irq_data));
  720. out_unlock:
  721. raw_spin_unlock(&desc->lock);
  722. }
  723. EXPORT_SYMBOL(handle_edge_irq);
  724. #ifdef CONFIG_IRQ_EDGE_EOI_HANDLER
  725. /**
  726. * handle_edge_eoi_irq - edge eoi type IRQ handler
  727. * @desc: the interrupt description structure for this irq
  728. *
  729. * Similar as the above handle_edge_irq, but using eoi and w/o the
  730. * mask/unmask logic.
  731. */
  732. void handle_edge_eoi_irq(struct irq_desc *desc)
  733. {
  734. struct irq_chip *chip = irq_desc_get_chip(desc);
  735. raw_spin_lock(&desc->lock);
  736. desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
  737. if (!irq_may_run(desc)) {
  738. desc->istate |= IRQS_PENDING;
  739. goto out_eoi;
  740. }
  741. /*
  742. * If its disabled or no action available then mask it and get
  743. * out of here.
  744. */
  745. if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
  746. desc->istate |= IRQS_PENDING;
  747. goto out_eoi;
  748. }
  749. kstat_incr_irqs_this_cpu(desc);
  750. do {
  751. if (unlikely(!desc->action))
  752. goto out_eoi;
  753. handle_irq_event(desc);
  754. } while ((desc->istate & IRQS_PENDING) &&
  755. !irqd_irq_disabled(&desc->irq_data));
  756. out_eoi:
  757. chip->irq_eoi(&desc->irq_data);
  758. raw_spin_unlock(&desc->lock);
  759. }
  760. #endif
  761. /**
  762. * handle_percpu_irq - Per CPU local irq handler
  763. * @desc: the interrupt description structure for this irq
  764. *
  765. * Per CPU interrupts on SMP machines without locking requirements
  766. */
  767. void handle_percpu_irq(struct irq_desc *desc)
  768. {
  769. struct irq_chip *chip = irq_desc_get_chip(desc);
  770. /*
  771. * PER CPU interrupts are not serialized. Do not touch
  772. * desc->tot_count.
  773. */
  774. __kstat_incr_irqs_this_cpu(desc);
  775. if (chip->irq_ack)
  776. chip->irq_ack(&desc->irq_data);
  777. handle_irq_event_percpu(desc);
  778. if (chip->irq_eoi)
  779. chip->irq_eoi(&desc->irq_data);
  780. }
  781. /**
  782. * handle_percpu_devid_irq - Per CPU local irq handler with per cpu dev ids
  783. * @desc: the interrupt description structure for this irq
  784. *
  785. * Per CPU interrupts on SMP machines without locking requirements. Same as
  786. * handle_percpu_irq() above but with the following extras:
  787. *
  788. * action->percpu_dev_id is a pointer to percpu variables which
  789. * contain the real device id for the cpu on which this handler is
  790. * called
  791. */
  792. void handle_percpu_devid_irq(struct irq_desc *desc)
  793. {
  794. struct irq_chip *chip = irq_desc_get_chip(desc);
  795. struct irqaction *action = desc->action;
  796. unsigned int irq = irq_desc_get_irq(desc);
  797. irqreturn_t res;
  798. /*
  799. * PER CPU interrupts are not serialized. Do not touch
  800. * desc->tot_count.
  801. */
  802. __kstat_incr_irqs_this_cpu(desc);
  803. if (chip->irq_ack)
  804. chip->irq_ack(&desc->irq_data);
  805. if (likely(action)) {
  806. trace_irq_handler_entry(irq, action);
  807. res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
  808. trace_irq_handler_exit(irq, action, res);
  809. } else {
  810. unsigned int cpu = smp_processor_id();
  811. bool enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
  812. if (enabled)
  813. irq_percpu_disable(desc, cpu);
  814. pr_err_once("Spurious%s percpu IRQ%u on CPU%u\n",
  815. enabled ? " and unmasked" : "", irq, cpu);
  816. }
  817. if (chip->irq_eoi)
  818. chip->irq_eoi(&desc->irq_data);
  819. }
  820. /**
  821. * handle_percpu_devid_fasteoi_ipi - Per CPU local IPI handler with per cpu
  822. * dev ids
  823. * @desc: the interrupt description structure for this irq
  824. *
  825. * The biggest difference with the IRQ version is that the interrupt is
  826. * EOIed early, as the IPI could result in a context switch, and we need to
  827. * make sure the IPI can fire again. We also assume that the arch code has
  828. * registered an action. If not, we are positively doomed.
  829. */
  830. void handle_percpu_devid_fasteoi_ipi(struct irq_desc *desc)
  831. {
  832. struct irq_chip *chip = irq_desc_get_chip(desc);
  833. struct irqaction *action = desc->action;
  834. unsigned int irq = irq_desc_get_irq(desc);
  835. irqreturn_t res;
  836. __kstat_incr_irqs_this_cpu(desc);
  837. if (chip->irq_eoi)
  838. chip->irq_eoi(&desc->irq_data);
  839. trace_irq_handler_entry(irq, action);
  840. res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
  841. trace_irq_handler_exit(irq, action, res);
  842. }
  843. /**
  844. * handle_percpu_devid_fasteoi_nmi - Per CPU local NMI handler with per cpu
  845. * dev ids
  846. * @desc: the interrupt description structure for this irq
  847. *
  848. * Similar to handle_fasteoi_nmi, but handling the dev_id cookie
  849. * as a percpu pointer.
  850. */
  851. void handle_percpu_devid_fasteoi_nmi(struct irq_desc *desc)
  852. {
  853. struct irq_chip *chip = irq_desc_get_chip(desc);
  854. struct irqaction *action = desc->action;
  855. unsigned int irq = irq_desc_get_irq(desc);
  856. irqreturn_t res;
  857. __kstat_incr_irqs_this_cpu(desc);
  858. trace_irq_handler_entry(irq, action);
  859. res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
  860. trace_irq_handler_exit(irq, action, res);
  861. if (chip->irq_eoi)
  862. chip->irq_eoi(&desc->irq_data);
  863. }
  864. static void
  865. __irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle,
  866. int is_chained, const char *name)
  867. {
  868. if (!handle) {
  869. handle = handle_bad_irq;
  870. } else {
  871. struct irq_data *irq_data = &desc->irq_data;
  872. #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
  873. /*
  874. * With hierarchical domains we might run into a
  875. * situation where the outermost chip is not yet set
  876. * up, but the inner chips are there. Instead of
  877. * bailing we install the handler, but obviously we
  878. * cannot enable/startup the interrupt at this point.
  879. */
  880. while (irq_data) {
  881. if (irq_data->chip != &no_irq_chip)
  882. break;
  883. /*
  884. * Bail out if the outer chip is not set up
  885. * and the interrupt supposed to be started
  886. * right away.
  887. */
  888. if (WARN_ON(is_chained))
  889. return;
  890. /* Try the parent */
  891. irq_data = irq_data->parent_data;
  892. }
  893. #endif
  894. if (WARN_ON(!irq_data || irq_data->chip == &no_irq_chip))
  895. return;
  896. }
  897. /* Uninstall? */
  898. if (handle == handle_bad_irq) {
  899. if (desc->irq_data.chip != &no_irq_chip)
  900. mask_ack_irq(desc);
  901. irq_state_set_disabled(desc);
  902. if (is_chained)
  903. desc->action = NULL;
  904. desc->depth = 1;
  905. }
  906. desc->handle_irq = handle;
  907. desc->name = name;
  908. if (handle != handle_bad_irq && is_chained) {
  909. unsigned int type = irqd_get_trigger_type(&desc->irq_data);
  910. /*
  911. * We're about to start this interrupt immediately,
  912. * hence the need to set the trigger configuration.
  913. * But the .set_type callback may have overridden the
  914. * flow handler, ignoring that we're dealing with a
  915. * chained interrupt. Reset it immediately because we
  916. * do know better.
  917. */
  918. if (type != IRQ_TYPE_NONE) {
  919. __irq_set_trigger(desc, type);
  920. desc->handle_irq = handle;
  921. }
  922. irq_settings_set_noprobe(desc);
  923. irq_settings_set_norequest(desc);
  924. irq_settings_set_nothread(desc);
  925. desc->action = &chained_action;
  926. irq_activate_and_startup(desc, IRQ_RESEND);
  927. }
  928. }
  929. void
  930. __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
  931. const char *name)
  932. {
  933. unsigned long flags;
  934. struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
  935. if (!desc)
  936. return;
  937. __irq_do_set_handler(desc, handle, is_chained, name);
  938. irq_put_desc_busunlock(desc, flags);
  939. }
  940. EXPORT_SYMBOL_GPL(__irq_set_handler);
  941. void
  942. irq_set_chained_handler_and_data(unsigned int irq, irq_flow_handler_t handle,
  943. void *data)
  944. {
  945. unsigned long flags;
  946. struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
  947. if (!desc)
  948. return;
  949. desc->irq_common_data.handler_data = data;
  950. __irq_do_set_handler(desc, handle, 1, NULL);
  951. irq_put_desc_busunlock(desc, flags);
  952. }
  953. EXPORT_SYMBOL_GPL(irq_set_chained_handler_and_data);
  954. void
  955. irq_set_chip_and_handler_name(unsigned int irq, struct irq_chip *chip,
  956. irq_flow_handler_t handle, const char *name)
  957. {
  958. irq_set_chip(irq, chip);
  959. __irq_set_handler(irq, handle, 0, name);
  960. }
  961. EXPORT_SYMBOL_GPL(irq_set_chip_and_handler_name);
  962. void __irq_modify_status(unsigned int irq, unsigned long clr,
  963. unsigned long set, unsigned long mask)
  964. {
  965. unsigned long flags, trigger, tmp;
  966. struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
  967. if (!desc)
  968. return;
  969. /*
  970. * Warn when a driver sets the no autoenable flag on an already
  971. * active interrupt.
  972. */
  973. WARN_ON_ONCE(!desc->depth && (set & _IRQ_NOAUTOEN));
  974. /* Warn when trying to clear or set a bit disallowed by the mask */
  975. WARN_ON((clr | set) & ~mask);
  976. __irq_settings_clr_and_set(desc, clr, set, mask);
  977. trigger = irqd_get_trigger_type(&desc->irq_data);
  978. irqd_clear(&desc->irq_data, IRQD_NO_BALANCING | IRQD_PER_CPU |
  979. IRQD_TRIGGER_MASK | IRQD_LEVEL | IRQD_MOVE_PCNTXT);
  980. if (irq_settings_has_no_balance_set(desc))
  981. irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
  982. if (irq_settings_is_per_cpu(desc))
  983. irqd_set(&desc->irq_data, IRQD_PER_CPU);
  984. if (irq_settings_can_move_pcntxt(desc))
  985. irqd_set(&desc->irq_data, IRQD_MOVE_PCNTXT);
  986. if (irq_settings_is_level(desc))
  987. irqd_set(&desc->irq_data, IRQD_LEVEL);
  988. tmp = irq_settings_get_trigger_mask(desc);
  989. if (tmp != IRQ_TYPE_NONE)
  990. trigger = tmp;
  991. irqd_set(&desc->irq_data, trigger);
  992. irq_put_desc_unlock(desc, flags);
  993. }
  994. void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set)
  995. {
  996. __irq_modify_status(irq, clr, set, _IRQF_MODIFY_MASK);
  997. }
  998. EXPORT_SYMBOL_GPL(irq_modify_status);
  999. /**
  1000. * irq_cpu_online - Invoke all irq_cpu_online functions.
  1001. *
  1002. * Iterate through all irqs and invoke the chip.irq_cpu_online()
  1003. * for each.
  1004. */
  1005. void irq_cpu_online(void)
  1006. {
  1007. struct irq_desc *desc;
  1008. struct irq_chip *chip;
  1009. unsigned long flags;
  1010. unsigned int irq;
  1011. for_each_active_irq(irq) {
  1012. desc = irq_to_desc(irq);
  1013. if (!desc)
  1014. continue;
  1015. raw_spin_lock_irqsave(&desc->lock, flags);
  1016. chip = irq_data_get_irq_chip(&desc->irq_data);
  1017. if (chip && chip->irq_cpu_online &&
  1018. (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
  1019. !irqd_irq_disabled(&desc->irq_data)))
  1020. chip->irq_cpu_online(&desc->irq_data);
  1021. raw_spin_unlock_irqrestore(&desc->lock, flags);
  1022. }
  1023. }
  1024. /**
  1025. * irq_cpu_offline - Invoke all irq_cpu_offline functions.
  1026. *
  1027. * Iterate through all irqs and invoke the chip.irq_cpu_offline()
  1028. * for each.
  1029. */
  1030. void irq_cpu_offline(void)
  1031. {
  1032. struct irq_desc *desc;
  1033. struct irq_chip *chip;
  1034. unsigned long flags;
  1035. unsigned int irq;
  1036. for_each_active_irq(irq) {
  1037. desc = irq_to_desc(irq);
  1038. if (!desc)
  1039. continue;
  1040. raw_spin_lock_irqsave(&desc->lock, flags);
  1041. chip = irq_data_get_irq_chip(&desc->irq_data);
  1042. if (chip && chip->irq_cpu_offline &&
  1043. (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
  1044. !irqd_irq_disabled(&desc->irq_data)))
  1045. chip->irq_cpu_offline(&desc->irq_data);
  1046. raw_spin_unlock_irqrestore(&desc->lock, flags);
  1047. }
  1048. }
  1049. #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
  1050. #ifdef CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS
  1051. /**
  1052. * handle_fasteoi_ack_irq - irq handler for edge hierarchy
  1053. * stacked on transparent controllers
  1054. *
  1055. * @desc: the interrupt description structure for this irq
  1056. *
  1057. * Like handle_fasteoi_irq(), but for use with hierarchy where
  1058. * the irq_chip also needs to have its ->irq_ack() function
  1059. * called.
  1060. */
  1061. void handle_fasteoi_ack_irq(struct irq_desc *desc)
  1062. {
  1063. struct irq_chip *chip = desc->irq_data.chip;
  1064. raw_spin_lock(&desc->lock);
  1065. if (!irq_may_run(desc))
  1066. goto out;
  1067. desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
  1068. /*
  1069. * If its disabled or no action available
  1070. * then mask it and get out of here:
  1071. */
  1072. if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
  1073. desc->istate |= IRQS_PENDING;
  1074. mask_irq(desc);
  1075. goto out;
  1076. }
  1077. kstat_incr_irqs_this_cpu(desc);
  1078. if (desc->istate & IRQS_ONESHOT)
  1079. mask_irq(desc);
  1080. /* Start handling the irq */
  1081. desc->irq_data.chip->irq_ack(&desc->irq_data);
  1082. handle_irq_event(desc);
  1083. cond_unmask_eoi_irq(desc, chip);
  1084. raw_spin_unlock(&desc->lock);
  1085. return;
  1086. out:
  1087. if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
  1088. chip->irq_eoi(&desc->irq_data);
  1089. raw_spin_unlock(&desc->lock);
  1090. }
  1091. EXPORT_SYMBOL_GPL(handle_fasteoi_ack_irq);
  1092. /**
  1093. * handle_fasteoi_mask_irq - irq handler for level hierarchy
  1094. * stacked on transparent controllers
  1095. *
  1096. * @desc: the interrupt description structure for this irq
  1097. *
  1098. * Like handle_fasteoi_irq(), but for use with hierarchy where
  1099. * the irq_chip also needs to have its ->irq_mask_ack() function
  1100. * called.
  1101. */
  1102. void handle_fasteoi_mask_irq(struct irq_desc *desc)
  1103. {
  1104. struct irq_chip *chip = desc->irq_data.chip;
  1105. raw_spin_lock(&desc->lock);
  1106. mask_ack_irq(desc);
  1107. if (!irq_may_run(desc))
  1108. goto out;
  1109. desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
  1110. /*
  1111. * If its disabled or no action available
  1112. * then mask it and get out of here:
  1113. */
  1114. if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
  1115. desc->istate |= IRQS_PENDING;
  1116. mask_irq(desc);
  1117. goto out;
  1118. }
  1119. kstat_incr_irqs_this_cpu(desc);
  1120. if (desc->istate & IRQS_ONESHOT)
  1121. mask_irq(desc);
  1122. handle_irq_event(desc);
  1123. cond_unmask_eoi_irq(desc, chip);
  1124. raw_spin_unlock(&desc->lock);
  1125. return;
  1126. out:
  1127. if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
  1128. chip->irq_eoi(&desc->irq_data);
  1129. raw_spin_unlock(&desc->lock);
  1130. }
  1131. EXPORT_SYMBOL_GPL(handle_fasteoi_mask_irq);
  1132. #endif /* CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS */
  1133. /**
  1134. * irq_chip_set_parent_state - set the state of a parent interrupt.
  1135. *
  1136. * @data: Pointer to interrupt specific data
  1137. * @which: State to be restored (one of IRQCHIP_STATE_*)
  1138. * @val: Value corresponding to @which
  1139. *
  1140. * Conditional success, if the underlying irqchip does not implement it.
  1141. */
  1142. int irq_chip_set_parent_state(struct irq_data *data,
  1143. enum irqchip_irq_state which,
  1144. bool val)
  1145. {
  1146. data = data->parent_data;
  1147. if (!data || !data->chip->irq_set_irqchip_state)
  1148. return 0;
  1149. return data->chip->irq_set_irqchip_state(data, which, val);
  1150. }
  1151. EXPORT_SYMBOL_GPL(irq_chip_set_parent_state);
  1152. /**
  1153. * irq_chip_get_parent_state - get the state of a parent interrupt.
  1154. *
  1155. * @data: Pointer to interrupt specific data
  1156. * @which: one of IRQCHIP_STATE_* the caller wants to know
  1157. * @state: a pointer to a boolean where the state is to be stored
  1158. *
  1159. * Conditional success, if the underlying irqchip does not implement it.
  1160. */
  1161. int irq_chip_get_parent_state(struct irq_data *data,
  1162. enum irqchip_irq_state which,
  1163. bool *state)
  1164. {
  1165. data = data->parent_data;
  1166. if (!data || !data->chip->irq_get_irqchip_state)
  1167. return 0;
  1168. return data->chip->irq_get_irqchip_state(data, which, state);
  1169. }
  1170. EXPORT_SYMBOL_GPL(irq_chip_get_parent_state);
  1171. /**
  1172. * irq_chip_enable_parent - Enable the parent interrupt (defaults to unmask if
  1173. * NULL)
  1174. * @data: Pointer to interrupt specific data
  1175. */
  1176. void irq_chip_enable_parent(struct irq_data *data)
  1177. {
  1178. data = data->parent_data;
  1179. if (data->chip->irq_enable)
  1180. data->chip->irq_enable(data);
  1181. else
  1182. data->chip->irq_unmask(data);
  1183. }
  1184. EXPORT_SYMBOL_GPL(irq_chip_enable_parent);
  1185. /**
  1186. * irq_chip_disable_parent - Disable the parent interrupt (defaults to mask if
  1187. * NULL)
  1188. * @data: Pointer to interrupt specific data
  1189. */
  1190. void irq_chip_disable_parent(struct irq_data *data)
  1191. {
  1192. data = data->parent_data;
  1193. if (data->chip->irq_disable)
  1194. data->chip->irq_disable(data);
  1195. else
  1196. data->chip->irq_mask(data);
  1197. }
  1198. EXPORT_SYMBOL_GPL(irq_chip_disable_parent);
  1199. /**
  1200. * irq_chip_ack_parent - Acknowledge the parent interrupt
  1201. * @data: Pointer to interrupt specific data
  1202. */
  1203. void irq_chip_ack_parent(struct irq_data *data)
  1204. {
  1205. data = data->parent_data;
  1206. data->chip->irq_ack(data);
  1207. }
  1208. EXPORT_SYMBOL_GPL(irq_chip_ack_parent);
  1209. /**
  1210. * irq_chip_mask_parent - Mask the parent interrupt
  1211. * @data: Pointer to interrupt specific data
  1212. */
  1213. void irq_chip_mask_parent(struct irq_data *data)
  1214. {
  1215. data = data->parent_data;
  1216. data->chip->irq_mask(data);
  1217. }
  1218. EXPORT_SYMBOL_GPL(irq_chip_mask_parent);
  1219. /**
  1220. * irq_chip_mask_ack_parent - Mask and acknowledge the parent interrupt
  1221. * @data: Pointer to interrupt specific data
  1222. */
  1223. void irq_chip_mask_ack_parent(struct irq_data *data)
  1224. {
  1225. data = data->parent_data;
  1226. data->chip->irq_mask_ack(data);
  1227. }
  1228. EXPORT_SYMBOL_GPL(irq_chip_mask_ack_parent);
  1229. /**
  1230. * irq_chip_unmask_parent - Unmask the parent interrupt
  1231. * @data: Pointer to interrupt specific data
  1232. */
  1233. void irq_chip_unmask_parent(struct irq_data *data)
  1234. {
  1235. data = data->parent_data;
  1236. data->chip->irq_unmask(data);
  1237. }
  1238. EXPORT_SYMBOL_GPL(irq_chip_unmask_parent);
  1239. /**
  1240. * irq_chip_eoi_parent - Invoke EOI on the parent interrupt
  1241. * @data: Pointer to interrupt specific data
  1242. */
  1243. void irq_chip_eoi_parent(struct irq_data *data)
  1244. {
  1245. data = data->parent_data;
  1246. data->chip->irq_eoi(data);
  1247. }
  1248. EXPORT_SYMBOL_GPL(irq_chip_eoi_parent);
  1249. /**
  1250. * irq_chip_set_affinity_parent - Set affinity on the parent interrupt
  1251. * @data: Pointer to interrupt specific data
  1252. * @dest: The affinity mask to set
  1253. * @force: Flag to enforce setting (disable online checks)
  1254. *
  1255. * Conditinal, as the underlying parent chip might not implement it.
  1256. */
  1257. int irq_chip_set_affinity_parent(struct irq_data *data,
  1258. const struct cpumask *dest, bool force)
  1259. {
  1260. data = data->parent_data;
  1261. if (data->chip->irq_set_affinity)
  1262. return data->chip->irq_set_affinity(data, dest, force);
  1263. return -ENOSYS;
  1264. }
  1265. EXPORT_SYMBOL_GPL(irq_chip_set_affinity_parent);
  1266. /**
  1267. * irq_chip_set_type_parent - Set IRQ type on the parent interrupt
  1268. * @data: Pointer to interrupt specific data
  1269. * @type: IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
  1270. *
  1271. * Conditional, as the underlying parent chip might not implement it.
  1272. */
  1273. int irq_chip_set_type_parent(struct irq_data *data, unsigned int type)
  1274. {
  1275. data = data->parent_data;
  1276. if (data->chip->irq_set_type)
  1277. return data->chip->irq_set_type(data, type);
  1278. return -ENOSYS;
  1279. }
  1280. EXPORT_SYMBOL_GPL(irq_chip_set_type_parent);
  1281. /**
  1282. * irq_chip_retrigger_hierarchy - Retrigger an interrupt in hardware
  1283. * @data: Pointer to interrupt specific data
  1284. *
  1285. * Iterate through the domain hierarchy of the interrupt and check
  1286. * whether a hw retrigger function exists. If yes, invoke it.
  1287. */
  1288. int irq_chip_retrigger_hierarchy(struct irq_data *data)
  1289. {
  1290. for (data = data->parent_data; data; data = data->parent_data)
  1291. if (data->chip && data->chip->irq_retrigger)
  1292. return data->chip->irq_retrigger(data);
  1293. return 0;
  1294. }
  1295. EXPORT_SYMBOL_GPL(irq_chip_retrigger_hierarchy);
  1296. /**
  1297. * irq_chip_set_vcpu_affinity_parent - Set vcpu affinity on the parent interrupt
  1298. * @data: Pointer to interrupt specific data
  1299. * @vcpu_info: The vcpu affinity information
  1300. */
  1301. int irq_chip_set_vcpu_affinity_parent(struct irq_data *data, void *vcpu_info)
  1302. {
  1303. data = data->parent_data;
  1304. if (data->chip->irq_set_vcpu_affinity)
  1305. return data->chip->irq_set_vcpu_affinity(data, vcpu_info);
  1306. return -ENOSYS;
  1307. }
  1308. EXPORT_SYMBOL_GPL(irq_chip_set_vcpu_affinity_parent);
  1309. /**
  1310. * irq_chip_set_wake_parent - Set/reset wake-up on the parent interrupt
  1311. * @data: Pointer to interrupt specific data
  1312. * @on: Whether to set or reset the wake-up capability of this irq
  1313. *
  1314. * Conditional, as the underlying parent chip might not implement it.
  1315. */
  1316. int irq_chip_set_wake_parent(struct irq_data *data, unsigned int on)
  1317. {
  1318. data = data->parent_data;
  1319. if (data->chip->flags & IRQCHIP_SKIP_SET_WAKE)
  1320. return 0;
  1321. if (data->chip->irq_set_wake)
  1322. return data->chip->irq_set_wake(data, on);
  1323. return -ENOSYS;
  1324. }
  1325. EXPORT_SYMBOL_GPL(irq_chip_set_wake_parent);
  1326. /**
  1327. * irq_chip_request_resources_parent - Request resources on the parent interrupt
  1328. * @data: Pointer to interrupt specific data
  1329. */
  1330. int irq_chip_request_resources_parent(struct irq_data *data)
  1331. {
  1332. data = data->parent_data;
  1333. if (data->chip->irq_request_resources)
  1334. return data->chip->irq_request_resources(data);
  1335. return -ENOSYS;
  1336. }
  1337. EXPORT_SYMBOL_GPL(irq_chip_request_resources_parent);
  1338. /**
  1339. * irq_chip_release_resources_parent - Release resources on the parent interrupt
  1340. * @data: Pointer to interrupt specific data
  1341. */
  1342. void irq_chip_release_resources_parent(struct irq_data *data)
  1343. {
  1344. data = data->parent_data;
  1345. if (data->chip->irq_release_resources)
  1346. data->chip->irq_release_resources(data);
  1347. }
  1348. EXPORT_SYMBOL_GPL(irq_chip_release_resources_parent);
  1349. #endif
  1350. /**
  1351. * irq_chip_compose_msi_msg - Componse msi message for a irq chip
  1352. * @data: Pointer to interrupt specific data
  1353. * @msg: Pointer to the MSI message
  1354. *
  1355. * For hierarchical domains we find the first chip in the hierarchy
  1356. * which implements the irq_compose_msi_msg callback. For non
  1357. * hierarchical we use the top level chip.
  1358. */
  1359. int irq_chip_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
  1360. {
  1361. struct irq_data *pos;
  1362. for (pos = NULL; !pos && data; data = irqd_get_parent_data(data)) {
  1363. if (data->chip && data->chip->irq_compose_msi_msg)
  1364. pos = data;
  1365. }
  1366. if (!pos)
  1367. return -ENOSYS;
  1368. pos->chip->irq_compose_msi_msg(pos, msg);
  1369. return 0;
  1370. }
  1371. /**
  1372. * irq_chip_pm_get - Enable power for an IRQ chip
  1373. * @data: Pointer to interrupt specific data
  1374. *
  1375. * Enable the power to the IRQ chip referenced by the interrupt data
  1376. * structure.
  1377. */
  1378. int irq_chip_pm_get(struct irq_data *data)
  1379. {
  1380. int retval;
  1381. if (IS_ENABLED(CONFIG_PM) && data->chip->parent_device) {
  1382. retval = pm_runtime_get_sync(data->chip->parent_device);
  1383. if (retval < 0) {
  1384. pm_runtime_put_noidle(data->chip->parent_device);
  1385. return retval;
  1386. }
  1387. }
  1388. return 0;
  1389. }
  1390. /**
  1391. * irq_chip_pm_put - Disable power for an IRQ chip
  1392. * @data: Pointer to interrupt specific data
  1393. *
  1394. * Disable the power to the IRQ chip referenced by the interrupt data
  1395. * structure, belongs. Note that power will only be disabled, once this
  1396. * function has been called for all IRQs that have called irq_chip_pm_get().
  1397. */
  1398. int irq_chip_pm_put(struct irq_data *data)
  1399. {
  1400. int retval = 0;
  1401. if (IS_ENABLED(CONFIG_PM) && data->chip->parent_device)
  1402. retval = pm_runtime_put(data->chip->parent_device);
  1403. return (retval < 0) ? retval : 0;
  1404. }