manage.c 73 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
  4. * Copyright (C) 2005-2006 Thomas Gleixner
  5. *
  6. * This file contains driver APIs to the irq subsystem.
  7. */
  8. #define pr_fmt(fmt) "genirq: " fmt
  9. #include <linux/irq.h>
  10. #include <linux/kthread.h>
  11. #include <linux/module.h>
  12. #include <linux/random.h>
  13. #include <linux/interrupt.h>
  14. #include <linux/irqdomain.h>
  15. #include <linux/slab.h>
  16. #include <linux/sched.h>
  17. #include <linux/sched/rt.h>
  18. #include <linux/sched/task.h>
  19. #include <linux/sched/isolation.h>
  20. #include <uapi/linux/sched/types.h>
  21. #include <linux/task_work.h>
  22. #include "internals.h"
  23. #if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)
  24. __read_mostly bool force_irqthreads;
  25. EXPORT_SYMBOL_GPL(force_irqthreads);
  26. static int __init setup_forced_irqthreads(char *arg)
  27. {
  28. force_irqthreads = true;
  29. return 0;
  30. }
  31. early_param("threadirqs", setup_forced_irqthreads);
  32. #endif
  33. static void __synchronize_hardirq(struct irq_desc *desc, bool sync_chip)
  34. {
  35. struct irq_data *irqd = irq_desc_get_irq_data(desc);
  36. bool inprogress;
  37. do {
  38. unsigned long flags;
  39. /*
  40. * Wait until we're out of the critical section. This might
  41. * give the wrong answer due to the lack of memory barriers.
  42. */
  43. while (irqd_irq_inprogress(&desc->irq_data))
  44. cpu_relax();
  45. /* Ok, that indicated we're done: double-check carefully. */
  46. raw_spin_lock_irqsave(&desc->lock, flags);
  47. inprogress = irqd_irq_inprogress(&desc->irq_data);
  48. /*
  49. * If requested and supported, check at the chip whether it
  50. * is in flight at the hardware level, i.e. already pending
  51. * in a CPU and waiting for service and acknowledge.
  52. */
  53. if (!inprogress && sync_chip) {
  54. /*
  55. * Ignore the return code. inprogress is only updated
  56. * when the chip supports it.
  57. */
  58. __irq_get_irqchip_state(irqd, IRQCHIP_STATE_ACTIVE,
  59. &inprogress);
  60. }
  61. raw_spin_unlock_irqrestore(&desc->lock, flags);
  62. /* Oops, that failed? */
  63. } while (inprogress);
  64. }
  65. /**
  66. * synchronize_hardirq - wait for pending hard IRQ handlers (on other CPUs)
  67. * @irq: interrupt number to wait for
  68. *
  69. * This function waits for any pending hard IRQ handlers for this
  70. * interrupt to complete before returning. If you use this
  71. * function while holding a resource the IRQ handler may need you
  72. * will deadlock. It does not take associated threaded handlers
  73. * into account.
  74. *
  75. * Do not use this for shutdown scenarios where you must be sure
  76. * that all parts (hardirq and threaded handler) have completed.
  77. *
  78. * Returns: false if a threaded handler is active.
  79. *
  80. * This function may be called - with care - from IRQ context.
  81. *
  82. * It does not check whether there is an interrupt in flight at the
  83. * hardware level, but not serviced yet, as this might deadlock when
  84. * called with interrupts disabled and the target CPU of the interrupt
  85. * is the current CPU.
  86. */
  87. bool synchronize_hardirq(unsigned int irq)
  88. {
  89. struct irq_desc *desc = irq_to_desc(irq);
  90. if (desc) {
  91. __synchronize_hardirq(desc, false);
  92. return !atomic_read(&desc->threads_active);
  93. }
  94. return true;
  95. }
  96. EXPORT_SYMBOL(synchronize_hardirq);
  97. /**
  98. * synchronize_irq - wait for pending IRQ handlers (on other CPUs)
  99. * @irq: interrupt number to wait for
  100. *
  101. * This function waits for any pending IRQ handlers for this interrupt
  102. * to complete before returning. If you use this function while
  103. * holding a resource the IRQ handler may need you will deadlock.
  104. *
  105. * Can only be called from preemptible code as it might sleep when
  106. * an interrupt thread is associated to @irq.
  107. *
  108. * It optionally makes sure (when the irq chip supports that method)
  109. * that the interrupt is not pending in any CPU and waiting for
  110. * service.
  111. */
  112. void synchronize_irq(unsigned int irq)
  113. {
  114. struct irq_desc *desc = irq_to_desc(irq);
  115. if (desc) {
  116. __synchronize_hardirq(desc, true);
  117. /*
  118. * We made sure that no hardirq handler is
  119. * running. Now verify that no threaded handlers are
  120. * active.
  121. */
  122. wait_event(desc->wait_for_threads,
  123. !atomic_read(&desc->threads_active));
  124. }
  125. }
  126. EXPORT_SYMBOL(synchronize_irq);
  127. #ifdef CONFIG_SMP
  128. cpumask_var_t irq_default_affinity;
  129. static bool __irq_can_set_affinity(struct irq_desc *desc)
  130. {
  131. if (!desc || !irqd_can_balance(&desc->irq_data) ||
  132. !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)
  133. return false;
  134. return true;
  135. }
  136. /**
  137. * irq_can_set_affinity - Check if the affinity of a given irq can be set
  138. * @irq: Interrupt to check
  139. *
  140. */
  141. int irq_can_set_affinity(unsigned int irq)
  142. {
  143. return __irq_can_set_affinity(irq_to_desc(irq));
  144. }
  145. /**
  146. * irq_can_set_affinity_usr - Check if affinity of a irq can be set from user space
  147. * @irq: Interrupt to check
  148. *
  149. * Like irq_can_set_affinity() above, but additionally checks for the
  150. * AFFINITY_MANAGED flag.
  151. */
  152. bool irq_can_set_affinity_usr(unsigned int irq)
  153. {
  154. struct irq_desc *desc = irq_to_desc(irq);
  155. return __irq_can_set_affinity(desc) &&
  156. !irqd_affinity_is_managed(&desc->irq_data);
  157. }
  158. /**
  159. * irq_set_thread_affinity - Notify irq threads to adjust affinity
  160. * @desc: irq descriptor which has affitnity changed
  161. *
  162. * We just set IRQTF_AFFINITY and delegate the affinity setting
  163. * to the interrupt thread itself. We can not call
  164. * set_cpus_allowed_ptr() here as we hold desc->lock and this
  165. * code can be called from hard interrupt context.
  166. */
  167. void irq_set_thread_affinity(struct irq_desc *desc)
  168. {
  169. struct irqaction *action;
  170. for_each_action_of_desc(desc, action)
  171. if (action->thread)
  172. set_bit(IRQTF_AFFINITY, &action->thread_flags);
  173. }
  174. #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
  175. static void irq_validate_effective_affinity(struct irq_data *data)
  176. {
  177. const struct cpumask *m = irq_data_get_effective_affinity_mask(data);
  178. struct irq_chip *chip = irq_data_get_irq_chip(data);
  179. if (!cpumask_empty(m))
  180. return;
  181. pr_warn_once("irq_chip %s did not update eff. affinity mask of irq %u\n",
  182. chip->name, data->irq);
  183. }
  184. static inline void irq_init_effective_affinity(struct irq_data *data,
  185. const struct cpumask *mask)
  186. {
  187. cpumask_copy(irq_data_get_effective_affinity_mask(data), mask);
  188. }
  189. #else
  190. static inline void irq_validate_effective_affinity(struct irq_data *data) { }
  191. static inline void irq_init_effective_affinity(struct irq_data *data,
  192. const struct cpumask *mask) { }
  193. #endif
  194. int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
  195. bool force)
  196. {
  197. struct irq_desc *desc = irq_data_to_desc(data);
  198. struct irq_chip *chip = irq_data_get_irq_chip(data);
  199. int ret;
  200. if (!chip || !chip->irq_set_affinity)
  201. return -EINVAL;
  202. /*
  203. * If this is a managed interrupt and housekeeping is enabled on
  204. * it check whether the requested affinity mask intersects with
  205. * a housekeeping CPU. If so, then remove the isolated CPUs from
  206. * the mask and just keep the housekeeping CPU(s). This prevents
  207. * the affinity setter from routing the interrupt to an isolated
  208. * CPU to avoid that I/O submitted from a housekeeping CPU causes
  209. * interrupts on an isolated one.
  210. *
  211. * If the masks do not intersect or include online CPU(s) then
  212. * keep the requested mask. The isolated target CPUs are only
  213. * receiving interrupts when the I/O operation was submitted
  214. * directly from them.
  215. *
  216. * If all housekeeping CPUs in the affinity mask are offline, the
  217. * interrupt will be migrated by the CPU hotplug code once a
  218. * housekeeping CPU which belongs to the affinity mask comes
  219. * online.
  220. */
  221. if (irqd_affinity_is_managed(data) &&
  222. housekeeping_enabled(HK_FLAG_MANAGED_IRQ)) {
  223. const struct cpumask *hk_mask, *prog_mask;
  224. static DEFINE_RAW_SPINLOCK(tmp_mask_lock);
  225. static struct cpumask tmp_mask;
  226. hk_mask = housekeeping_cpumask(HK_FLAG_MANAGED_IRQ);
  227. raw_spin_lock(&tmp_mask_lock);
  228. cpumask_and(&tmp_mask, mask, hk_mask);
  229. if (!cpumask_intersects(&tmp_mask, cpu_online_mask))
  230. prog_mask = mask;
  231. else
  232. prog_mask = &tmp_mask;
  233. ret = chip->irq_set_affinity(data, prog_mask, force);
  234. raw_spin_unlock(&tmp_mask_lock);
  235. } else {
  236. ret = chip->irq_set_affinity(data, mask, force);
  237. }
  238. switch (ret) {
  239. case IRQ_SET_MASK_OK:
  240. case IRQ_SET_MASK_OK_DONE:
  241. cpumask_copy(desc->irq_common_data.affinity, mask);
  242. fallthrough;
  243. case IRQ_SET_MASK_OK_NOCOPY:
  244. irq_validate_effective_affinity(data);
  245. irq_set_thread_affinity(desc);
  246. ret = 0;
  247. }
  248. return ret;
  249. }
  250. EXPORT_SYMBOL_GPL(irq_do_set_affinity);
  251. #ifdef CONFIG_GENERIC_PENDING_IRQ
  252. static inline int irq_set_affinity_pending(struct irq_data *data,
  253. const struct cpumask *dest)
  254. {
  255. struct irq_desc *desc = irq_data_to_desc(data);
  256. irqd_set_move_pending(data);
  257. irq_copy_pending(desc, dest);
  258. return 0;
  259. }
  260. #else
  261. static inline int irq_set_affinity_pending(struct irq_data *data,
  262. const struct cpumask *dest)
  263. {
  264. return -EBUSY;
  265. }
  266. #endif
  267. static int irq_try_set_affinity(struct irq_data *data,
  268. const struct cpumask *dest, bool force)
  269. {
  270. int ret = irq_do_set_affinity(data, dest, force);
  271. /*
  272. * In case that the underlying vector management is busy and the
  273. * architecture supports the generic pending mechanism then utilize
  274. * this to avoid returning an error to user space.
  275. */
  276. if (ret == -EBUSY && !force)
  277. ret = irq_set_affinity_pending(data, dest);
  278. return ret;
  279. }
  280. static bool irq_set_affinity_deactivated(struct irq_data *data,
  281. const struct cpumask *mask, bool force)
  282. {
  283. struct irq_desc *desc = irq_data_to_desc(data);
  284. /*
  285. * Handle irq chips which can handle affinity only in activated
  286. * state correctly
  287. *
  288. * If the interrupt is not yet activated, just store the affinity
  289. * mask and do not call the chip driver at all. On activation the
  290. * driver has to make sure anyway that the interrupt is in a
  291. * useable state so startup works.
  292. */
  293. if (!IS_ENABLED(CONFIG_IRQ_DOMAIN_HIERARCHY) ||
  294. irqd_is_activated(data) || !irqd_affinity_on_activate(data))
  295. return false;
  296. cpumask_copy(desc->irq_common_data.affinity, mask);
  297. irq_init_effective_affinity(data, mask);
  298. irqd_set(data, IRQD_AFFINITY_SET);
  299. return true;
  300. }
  301. int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,
  302. bool force)
  303. {
  304. struct irq_chip *chip = irq_data_get_irq_chip(data);
  305. struct irq_desc *desc = irq_data_to_desc(data);
  306. int ret = 0;
  307. if (!chip || !chip->irq_set_affinity)
  308. return -EINVAL;
  309. if (irq_set_affinity_deactivated(data, mask, force))
  310. return 0;
  311. if (irq_can_move_pcntxt(data) && !irqd_is_setaffinity_pending(data)) {
  312. ret = irq_try_set_affinity(data, mask, force);
  313. } else {
  314. irqd_set_move_pending(data);
  315. irq_copy_pending(desc, mask);
  316. }
  317. if (desc->affinity_notify) {
  318. kref_get(&desc->affinity_notify->kref);
  319. if (!schedule_work(&desc->affinity_notify->work)) {
  320. /* Work was already scheduled, drop our extra ref */
  321. kref_put(&desc->affinity_notify->kref,
  322. desc->affinity_notify->release);
  323. }
  324. }
  325. irqd_set(data, IRQD_AFFINITY_SET);
  326. return ret;
  327. }
  328. int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force)
  329. {
  330. struct irq_desc *desc = irq_to_desc(irq);
  331. unsigned long flags;
  332. int ret;
  333. if (!desc)
  334. return -EINVAL;
  335. raw_spin_lock_irqsave(&desc->lock, flags);
  336. ret = irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask, force);
  337. raw_spin_unlock_irqrestore(&desc->lock, flags);
  338. return ret;
  339. }
  340. int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
  341. {
  342. unsigned long flags;
  343. struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
  344. if (!desc)
  345. return -EINVAL;
  346. desc->affinity_hint = m;
  347. irq_put_desc_unlock(desc, flags);
  348. /* set the initial affinity to prevent every interrupt being on CPU0 */
  349. if (m)
  350. __irq_set_affinity(irq, m, false);
  351. return 0;
  352. }
  353. EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
  354. static void irq_affinity_notify(struct work_struct *work)
  355. {
  356. struct irq_affinity_notify *notify =
  357. container_of(work, struct irq_affinity_notify, work);
  358. struct irq_desc *desc = irq_to_desc(notify->irq);
  359. cpumask_var_t cpumask;
  360. unsigned long flags;
  361. if (!desc || !alloc_cpumask_var(&cpumask, GFP_KERNEL))
  362. goto out;
  363. raw_spin_lock_irqsave(&desc->lock, flags);
  364. if (irq_move_pending(&desc->irq_data))
  365. irq_get_pending(cpumask, desc);
  366. else
  367. cpumask_copy(cpumask, desc->irq_common_data.affinity);
  368. raw_spin_unlock_irqrestore(&desc->lock, flags);
  369. notify->notify(notify, cpumask);
  370. free_cpumask_var(cpumask);
  371. out:
  372. kref_put(&notify->kref, notify->release);
  373. }
  374. /**
  375. * irq_set_affinity_notifier - control notification of IRQ affinity changes
  376. * @irq: Interrupt for which to enable/disable notification
  377. * @notify: Context for notification, or %NULL to disable
  378. * notification. Function pointers must be initialised;
  379. * the other fields will be initialised by this function.
  380. *
  381. * Must be called in process context. Notification may only be enabled
  382. * after the IRQ is allocated and must be disabled before the IRQ is
  383. * freed using free_irq().
  384. */
  385. int
  386. irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
  387. {
  388. struct irq_desc *desc = irq_to_desc(irq);
  389. struct irq_affinity_notify *old_notify;
  390. unsigned long flags;
  391. /* The release function is promised process context */
  392. might_sleep();
  393. if (!desc || desc->istate & IRQS_NMI)
  394. return -EINVAL;
  395. /* Complete initialisation of *notify */
  396. if (notify) {
  397. notify->irq = irq;
  398. kref_init(&notify->kref);
  399. INIT_WORK(&notify->work, irq_affinity_notify);
  400. }
  401. raw_spin_lock_irqsave(&desc->lock, flags);
  402. old_notify = desc->affinity_notify;
  403. desc->affinity_notify = notify;
  404. raw_spin_unlock_irqrestore(&desc->lock, flags);
  405. if (old_notify) {
  406. if (cancel_work_sync(&old_notify->work)) {
  407. /* Pending work had a ref, put that one too */
  408. kref_put(&old_notify->kref, old_notify->release);
  409. }
  410. kref_put(&old_notify->kref, old_notify->release);
  411. }
  412. return 0;
  413. }
  414. EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
  415. #ifndef CONFIG_AUTO_IRQ_AFFINITY
  416. /*
  417. * Generic version of the affinity autoselector.
  418. */
  419. int irq_setup_affinity(struct irq_desc *desc)
  420. {
  421. struct cpumask *set = irq_default_affinity;
  422. int ret, node = irq_desc_get_node(desc);
  423. static DEFINE_RAW_SPINLOCK(mask_lock);
  424. static struct cpumask mask;
  425. /* Excludes PER_CPU and NO_BALANCE interrupts */
  426. if (!__irq_can_set_affinity(desc))
  427. return 0;
  428. raw_spin_lock(&mask_lock);
  429. /*
  430. * Preserve the managed affinity setting and a userspace affinity
  431. * setup, but make sure that one of the targets is online.
  432. */
  433. if (irqd_affinity_is_managed(&desc->irq_data) ||
  434. irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
  435. if (cpumask_intersects(desc->irq_common_data.affinity,
  436. cpu_online_mask))
  437. set = desc->irq_common_data.affinity;
  438. else
  439. irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
  440. }
  441. cpumask_and(&mask, cpu_online_mask, set);
  442. if (cpumask_empty(&mask))
  443. cpumask_copy(&mask, cpu_online_mask);
  444. if (node != NUMA_NO_NODE) {
  445. const struct cpumask *nodemask = cpumask_of_node(node);
  446. /* make sure at least one of the cpus in nodemask is online */
  447. if (cpumask_intersects(&mask, nodemask))
  448. cpumask_and(&mask, &mask, nodemask);
  449. }
  450. ret = irq_do_set_affinity(&desc->irq_data, &mask, false);
  451. raw_spin_unlock(&mask_lock);
  452. return ret;
  453. }
  454. #else
  455. /* Wrapper for ALPHA specific affinity selector magic */
  456. int irq_setup_affinity(struct irq_desc *desc)
  457. {
  458. return irq_select_affinity(irq_desc_get_irq(desc));
  459. }
  460. #endif /* CONFIG_AUTO_IRQ_AFFINITY */
  461. #endif /* CONFIG_SMP */
  462. /**
  463. * irq_set_vcpu_affinity - Set vcpu affinity for the interrupt
  464. * @irq: interrupt number to set affinity
  465. * @vcpu_info: vCPU specific data or pointer to a percpu array of vCPU
  466. * specific data for percpu_devid interrupts
  467. *
  468. * This function uses the vCPU specific data to set the vCPU
  469. * affinity for an irq. The vCPU specific data is passed from
  470. * outside, such as KVM. One example code path is as below:
  471. * KVM -> IOMMU -> irq_set_vcpu_affinity().
  472. */
  473. int irq_set_vcpu_affinity(unsigned int irq, void *vcpu_info)
  474. {
  475. unsigned long flags;
  476. struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
  477. struct irq_data *data;
  478. struct irq_chip *chip;
  479. int ret = -ENOSYS;
  480. if (!desc)
  481. return -EINVAL;
  482. data = irq_desc_get_irq_data(desc);
  483. do {
  484. chip = irq_data_get_irq_chip(data);
  485. if (chip && chip->irq_set_vcpu_affinity)
  486. break;
  487. #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
  488. data = data->parent_data;
  489. #else
  490. data = NULL;
  491. #endif
  492. } while (data);
  493. if (data)
  494. ret = chip->irq_set_vcpu_affinity(data, vcpu_info);
  495. irq_put_desc_unlock(desc, flags);
  496. return ret;
  497. }
  498. EXPORT_SYMBOL_GPL(irq_set_vcpu_affinity);
  499. void __disable_irq(struct irq_desc *desc)
  500. {
  501. if (!desc->depth++)
  502. irq_disable(desc);
  503. }
  504. static int __disable_irq_nosync(unsigned int irq)
  505. {
  506. unsigned long flags;
  507. struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
  508. if (!desc)
  509. return -EINVAL;
  510. __disable_irq(desc);
  511. irq_put_desc_busunlock(desc, flags);
  512. return 0;
  513. }
  514. /**
  515. * disable_irq_nosync - disable an irq without waiting
  516. * @irq: Interrupt to disable
  517. *
  518. * Disable the selected interrupt line. Disables and Enables are
  519. * nested.
  520. * Unlike disable_irq(), this function does not ensure existing
  521. * instances of the IRQ handler have completed before returning.
  522. *
  523. * This function may be called from IRQ context.
  524. */
  525. void disable_irq_nosync(unsigned int irq)
  526. {
  527. __disable_irq_nosync(irq);
  528. }
  529. EXPORT_SYMBOL(disable_irq_nosync);
  530. /**
  531. * disable_irq - disable an irq and wait for completion
  532. * @irq: Interrupt to disable
  533. *
  534. * Disable the selected interrupt line. Enables and Disables are
  535. * nested.
  536. * This function waits for any pending IRQ handlers for this interrupt
  537. * to complete before returning. If you use this function while
  538. * holding a resource the IRQ handler may need you will deadlock.
  539. *
  540. * This function may be called - with care - from IRQ context.
  541. */
  542. void disable_irq(unsigned int irq)
  543. {
  544. if (!__disable_irq_nosync(irq))
  545. synchronize_irq(irq);
  546. }
  547. EXPORT_SYMBOL(disable_irq);
  548. /**
  549. * disable_hardirq - disables an irq and waits for hardirq completion
  550. * @irq: Interrupt to disable
  551. *
  552. * Disable the selected interrupt line. Enables and Disables are
  553. * nested.
  554. * This function waits for any pending hard IRQ handlers for this
  555. * interrupt to complete before returning. If you use this function while
  556. * holding a resource the hard IRQ handler may need you will deadlock.
  557. *
  558. * When used to optimistically disable an interrupt from atomic context
  559. * the return value must be checked.
  560. *
  561. * Returns: false if a threaded handler is active.
  562. *
  563. * This function may be called - with care - from IRQ context.
  564. */
  565. bool disable_hardirq(unsigned int irq)
  566. {
  567. if (!__disable_irq_nosync(irq))
  568. return synchronize_hardirq(irq);
  569. return false;
  570. }
  571. EXPORT_SYMBOL_GPL(disable_hardirq);
  572. /**
  573. * disable_nmi_nosync - disable an nmi without waiting
  574. * @irq: Interrupt to disable
  575. *
  576. * Disable the selected interrupt line. Disables and enables are
  577. * nested.
  578. * The interrupt to disable must have been requested through request_nmi.
  579. * Unlike disable_nmi(), this function does not ensure existing
  580. * instances of the IRQ handler have completed before returning.
  581. */
  582. void disable_nmi_nosync(unsigned int irq)
  583. {
  584. disable_irq_nosync(irq);
  585. }
  586. void __enable_irq(struct irq_desc *desc)
  587. {
  588. switch (desc->depth) {
  589. case 0:
  590. err_out:
  591. WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n",
  592. irq_desc_get_irq(desc));
  593. break;
  594. case 1: {
  595. if (desc->istate & IRQS_SUSPENDED)
  596. goto err_out;
  597. /* Prevent probing on this irq: */
  598. irq_settings_set_noprobe(desc);
  599. /*
  600. * Call irq_startup() not irq_enable() here because the
  601. * interrupt might be marked NOAUTOEN. So irq_startup()
  602. * needs to be invoked when it gets enabled the first
  603. * time. If it was already started up, then irq_startup()
  604. * will invoke irq_enable() under the hood.
  605. */
  606. irq_startup(desc, IRQ_RESEND, IRQ_START_FORCE);
  607. break;
  608. }
  609. default:
  610. desc->depth--;
  611. }
  612. }
  613. /**
  614. * enable_irq - enable handling of an irq
  615. * @irq: Interrupt to enable
  616. *
  617. * Undoes the effect of one call to disable_irq(). If this
  618. * matches the last disable, processing of interrupts on this
  619. * IRQ line is re-enabled.
  620. *
  621. * This function may be called from IRQ context only when
  622. * desc->irq_data.chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
  623. */
  624. void enable_irq(unsigned int irq)
  625. {
  626. unsigned long flags;
  627. struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
  628. if (!desc)
  629. return;
  630. if (WARN(!desc->irq_data.chip,
  631. KERN_ERR "enable_irq before setup/request_irq: irq %u\n", irq))
  632. goto out;
  633. __enable_irq(desc);
  634. out:
  635. irq_put_desc_busunlock(desc, flags);
  636. }
  637. EXPORT_SYMBOL(enable_irq);
  638. /**
  639. * enable_nmi - enable handling of an nmi
  640. * @irq: Interrupt to enable
  641. *
  642. * The interrupt to enable must have been requested through request_nmi.
  643. * Undoes the effect of one call to disable_nmi(). If this
  644. * matches the last disable, processing of interrupts on this
  645. * IRQ line is re-enabled.
  646. */
  647. void enable_nmi(unsigned int irq)
  648. {
  649. enable_irq(irq);
  650. }
  651. static int set_irq_wake_real(unsigned int irq, unsigned int on)
  652. {
  653. struct irq_desc *desc = irq_to_desc(irq);
  654. int ret = -ENXIO;
  655. if (irq_desc_get_chip(desc)->flags & IRQCHIP_SKIP_SET_WAKE)
  656. return 0;
  657. if (desc->irq_data.chip->irq_set_wake)
  658. ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on);
  659. return ret;
  660. }
  661. /**
  662. * irq_set_irq_wake - control irq power management wakeup
  663. * @irq: interrupt to control
  664. * @on: enable/disable power management wakeup
  665. *
  666. * Enable/disable power management wakeup mode, which is
  667. * disabled by default. Enables and disables must match,
  668. * just as they match for non-wakeup mode support.
  669. *
  670. * Wakeup mode lets this IRQ wake the system from sleep
  671. * states like "suspend to RAM".
  672. *
  673. * Note: irq enable/disable state is completely orthogonal
  674. * to the enable/disable state of irq wake. An irq can be
  675. * disabled with disable_irq() and still wake the system as
  676. * long as the irq has wake enabled. If this does not hold,
  677. * then the underlying irq chip and the related driver need
  678. * to be investigated.
  679. */
  680. int irq_set_irq_wake(unsigned int irq, unsigned int on)
  681. {
  682. unsigned long flags;
  683. struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
  684. int ret = 0;
  685. if (!desc)
  686. return -EINVAL;
  687. /* Don't use NMIs as wake up interrupts please */
  688. if (desc->istate & IRQS_NMI) {
  689. ret = -EINVAL;
  690. goto out_unlock;
  691. }
  692. /* wakeup-capable irqs can be shared between drivers that
  693. * don't need to have the same sleep mode behaviors.
  694. */
  695. if (on) {
  696. if (desc->wake_depth++ == 0) {
  697. ret = set_irq_wake_real(irq, on);
  698. if (ret)
  699. desc->wake_depth = 0;
  700. else
  701. irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE);
  702. }
  703. } else {
  704. if (desc->wake_depth == 0) {
  705. WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
  706. } else if (--desc->wake_depth == 0) {
  707. ret = set_irq_wake_real(irq, on);
  708. if (ret)
  709. desc->wake_depth = 1;
  710. else
  711. irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
  712. }
  713. }
  714. out_unlock:
  715. irq_put_desc_busunlock(desc, flags);
  716. return ret;
  717. }
  718. EXPORT_SYMBOL(irq_set_irq_wake);
  719. /*
  720. * Internal function that tells the architecture code whether a
  721. * particular irq has been exclusively allocated or is available
  722. * for driver use.
  723. */
  724. int can_request_irq(unsigned int irq, unsigned long irqflags)
  725. {
  726. unsigned long flags;
  727. struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
  728. int canrequest = 0;
  729. if (!desc)
  730. return 0;
  731. if (irq_settings_can_request(desc)) {
  732. if (!desc->action ||
  733. irqflags & desc->action->flags & IRQF_SHARED)
  734. canrequest = 1;
  735. }
  736. irq_put_desc_unlock(desc, flags);
  737. return canrequest;
  738. }
  739. int __irq_set_trigger(struct irq_desc *desc, unsigned long flags)
  740. {
  741. struct irq_chip *chip = desc->irq_data.chip;
  742. int ret, unmask = 0;
  743. if (!chip || !chip->irq_set_type) {
  744. /*
  745. * IRQF_TRIGGER_* but the PIC does not support multiple
  746. * flow-types?
  747. */
  748. pr_debug("No set_type function for IRQ %d (%s)\n",
  749. irq_desc_get_irq(desc),
  750. chip ? (chip->name ? : "unknown") : "unknown");
  751. return 0;
  752. }
  753. if (chip->flags & IRQCHIP_SET_TYPE_MASKED) {
  754. if (!irqd_irq_masked(&desc->irq_data))
  755. mask_irq(desc);
  756. if (!irqd_irq_disabled(&desc->irq_data))
  757. unmask = 1;
  758. }
  759. /* Mask all flags except trigger mode */
  760. flags &= IRQ_TYPE_SENSE_MASK;
  761. ret = chip->irq_set_type(&desc->irq_data, flags);
  762. switch (ret) {
  763. case IRQ_SET_MASK_OK:
  764. case IRQ_SET_MASK_OK_DONE:
  765. irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
  766. irqd_set(&desc->irq_data, flags);
  767. fallthrough;
  768. case IRQ_SET_MASK_OK_NOCOPY:
  769. flags = irqd_get_trigger_type(&desc->irq_data);
  770. irq_settings_set_trigger_mask(desc, flags);
  771. irqd_clear(&desc->irq_data, IRQD_LEVEL);
  772. irq_settings_clr_level(desc);
  773. if (flags & IRQ_TYPE_LEVEL_MASK) {
  774. irq_settings_set_level(desc);
  775. irqd_set(&desc->irq_data, IRQD_LEVEL);
  776. }
  777. ret = 0;
  778. break;
  779. default:
  780. pr_err("Setting trigger mode %lu for irq %u failed (%pS)\n",
  781. flags, irq_desc_get_irq(desc), chip->irq_set_type);
  782. }
  783. if (unmask)
  784. unmask_irq(desc);
  785. return ret;
  786. }
  787. #ifdef CONFIG_HARDIRQS_SW_RESEND
  788. int irq_set_parent(int irq, int parent_irq)
  789. {
  790. unsigned long flags;
  791. struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
  792. if (!desc)
  793. return -EINVAL;
  794. desc->parent_irq = parent_irq;
  795. irq_put_desc_unlock(desc, flags);
  796. return 0;
  797. }
  798. EXPORT_SYMBOL_GPL(irq_set_parent);
  799. #endif
  800. /*
  801. * Default primary interrupt handler for threaded interrupts. Is
  802. * assigned as primary handler when request_threaded_irq is called
  803. * with handler == NULL. Useful for oneshot interrupts.
  804. */
  805. static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
  806. {
  807. return IRQ_WAKE_THREAD;
  808. }
  809. /*
  810. * Primary handler for nested threaded interrupts. Should never be
  811. * called.
  812. */
  813. static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
  814. {
  815. WARN(1, "Primary handler called for nested irq %d\n", irq);
  816. return IRQ_NONE;
  817. }
  818. static irqreturn_t irq_forced_secondary_handler(int irq, void *dev_id)
  819. {
  820. WARN(1, "Secondary action handler called for irq %d\n", irq);
  821. return IRQ_NONE;
  822. }
  823. static int irq_wait_for_interrupt(struct irqaction *action)
  824. {
  825. for (;;) {
  826. set_current_state(TASK_INTERRUPTIBLE);
  827. if (kthread_should_stop()) {
  828. /* may need to run one last time */
  829. if (test_and_clear_bit(IRQTF_RUNTHREAD,
  830. &action->thread_flags)) {
  831. __set_current_state(TASK_RUNNING);
  832. return 0;
  833. }
  834. __set_current_state(TASK_RUNNING);
  835. return -1;
  836. }
  837. if (test_and_clear_bit(IRQTF_RUNTHREAD,
  838. &action->thread_flags)) {
  839. __set_current_state(TASK_RUNNING);
  840. return 0;
  841. }
  842. schedule();
  843. }
  844. }
  845. /*
  846. * Oneshot interrupts keep the irq line masked until the threaded
  847. * handler finished. unmask if the interrupt has not been disabled and
  848. * is marked MASKED.
  849. */
  850. static void irq_finalize_oneshot(struct irq_desc *desc,
  851. struct irqaction *action)
  852. {
  853. if (!(desc->istate & IRQS_ONESHOT) ||
  854. action->handler == irq_forced_secondary_handler)
  855. return;
  856. again:
  857. chip_bus_lock(desc);
  858. raw_spin_lock_irq(&desc->lock);
  859. /*
  860. * Implausible though it may be we need to protect us against
  861. * the following scenario:
  862. *
  863. * The thread is faster done than the hard interrupt handler
  864. * on the other CPU. If we unmask the irq line then the
  865. * interrupt can come in again and masks the line, leaves due
  866. * to IRQS_INPROGRESS and the irq line is masked forever.
  867. *
  868. * This also serializes the state of shared oneshot handlers
  869. * versus "desc->threads_onehsot |= action->thread_mask;" in
  870. * irq_wake_thread(). See the comment there which explains the
  871. * serialization.
  872. */
  873. if (unlikely(irqd_irq_inprogress(&desc->irq_data))) {
  874. raw_spin_unlock_irq(&desc->lock);
  875. chip_bus_sync_unlock(desc);
  876. cpu_relax();
  877. goto again;
  878. }
  879. /*
  880. * Now check again, whether the thread should run. Otherwise
  881. * we would clear the threads_oneshot bit of this thread which
  882. * was just set.
  883. */
  884. if (test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
  885. goto out_unlock;
  886. desc->threads_oneshot &= ~action->thread_mask;
  887. if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) &&
  888. irqd_irq_masked(&desc->irq_data))
  889. unmask_threaded_irq(desc);
  890. out_unlock:
  891. raw_spin_unlock_irq(&desc->lock);
  892. chip_bus_sync_unlock(desc);
  893. }
  894. #ifdef CONFIG_SMP
  895. /*
  896. * Check whether we need to change the affinity of the interrupt thread.
  897. */
  898. static void
  899. irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
  900. {
  901. cpumask_var_t mask;
  902. bool valid = true;
  903. if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
  904. return;
  905. /*
  906. * In case we are out of memory we set IRQTF_AFFINITY again and
  907. * try again next time
  908. */
  909. if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
  910. set_bit(IRQTF_AFFINITY, &action->thread_flags);
  911. return;
  912. }
  913. raw_spin_lock_irq(&desc->lock);
  914. /*
  915. * This code is triggered unconditionally. Check the affinity
  916. * mask pointer. For CPU_MASK_OFFSTACK=n this is optimized out.
  917. */
  918. if (cpumask_available(desc->irq_common_data.affinity)) {
  919. const struct cpumask *m;
  920. m = irq_data_get_effective_affinity_mask(&desc->irq_data);
  921. cpumask_copy(mask, m);
  922. } else {
  923. valid = false;
  924. }
  925. raw_spin_unlock_irq(&desc->lock);
  926. if (valid)
  927. set_cpus_allowed_ptr(current, mask);
  928. free_cpumask_var(mask);
  929. }
  930. #else
  931. static inline void
  932. irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
  933. #endif
  934. /*
  935. * Interrupts which are not explicitly requested as threaded
  936. * interrupts rely on the implicit bh/preempt disable of the hard irq
  937. * context. So we need to disable bh here to avoid deadlocks and other
  938. * side effects.
  939. */
  940. static irqreturn_t
  941. irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
  942. {
  943. irqreturn_t ret;
  944. local_bh_disable();
  945. if (!IS_ENABLED(CONFIG_PREEMPT_RT))
  946. local_irq_disable();
  947. ret = action->thread_fn(action->irq, action->dev_id);
  948. if (ret == IRQ_HANDLED)
  949. atomic_inc(&desc->threads_handled);
  950. irq_finalize_oneshot(desc, action);
  951. if (!IS_ENABLED(CONFIG_PREEMPT_RT))
  952. local_irq_enable();
  953. local_bh_enable();
  954. return ret;
  955. }
  956. /*
  957. * Interrupts explicitly requested as threaded interrupts want to be
  958. * preemtible - many of them need to sleep and wait for slow busses to
  959. * complete.
  960. */
  961. static irqreturn_t irq_thread_fn(struct irq_desc *desc,
  962. struct irqaction *action)
  963. {
  964. irqreturn_t ret;
  965. ret = action->thread_fn(action->irq, action->dev_id);
  966. if (ret == IRQ_HANDLED)
  967. atomic_inc(&desc->threads_handled);
  968. irq_finalize_oneshot(desc, action);
  969. return ret;
  970. }
  971. static void wake_threads_waitq(struct irq_desc *desc)
  972. {
  973. if (atomic_dec_and_test(&desc->threads_active))
  974. wake_up(&desc->wait_for_threads);
  975. }
  976. static void irq_thread_dtor(struct callback_head *unused)
  977. {
  978. struct task_struct *tsk = current;
  979. struct irq_desc *desc;
  980. struct irqaction *action;
  981. if (WARN_ON_ONCE(!(current->flags & PF_EXITING)))
  982. return;
  983. action = kthread_data(tsk);
  984. pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
  985. tsk->comm, tsk->pid, action->irq);
  986. desc = irq_to_desc(action->irq);
  987. /*
  988. * If IRQTF_RUNTHREAD is set, we need to decrement
  989. * desc->threads_active and wake possible waiters.
  990. */
  991. if (test_and_clear_bit(IRQTF_RUNTHREAD, &action->thread_flags))
  992. wake_threads_waitq(desc);
  993. /* Prevent a stale desc->threads_oneshot */
  994. irq_finalize_oneshot(desc, action);
  995. }
  996. static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action)
  997. {
  998. struct irqaction *secondary = action->secondary;
  999. if (WARN_ON_ONCE(!secondary))
  1000. return;
  1001. raw_spin_lock_irq(&desc->lock);
  1002. __irq_wake_thread(desc, secondary);
  1003. raw_spin_unlock_irq(&desc->lock);
  1004. }
  1005. /*
  1006. * Interrupt handler thread
  1007. */
  1008. static int irq_thread(void *data)
  1009. {
  1010. struct callback_head on_exit_work;
  1011. struct irqaction *action = data;
  1012. struct irq_desc *desc = irq_to_desc(action->irq);
  1013. irqreturn_t (*handler_fn)(struct irq_desc *desc,
  1014. struct irqaction *action);
  1015. if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
  1016. &action->thread_flags))
  1017. handler_fn = irq_forced_thread_fn;
  1018. else
  1019. handler_fn = irq_thread_fn;
  1020. init_task_work(&on_exit_work, irq_thread_dtor);
  1021. task_work_add(current, &on_exit_work, TWA_NONE);
  1022. irq_thread_check_affinity(desc, action);
  1023. while (!irq_wait_for_interrupt(action)) {
  1024. irqreturn_t action_ret;
  1025. irq_thread_check_affinity(desc, action);
  1026. action_ret = handler_fn(desc, action);
  1027. if (action_ret == IRQ_WAKE_THREAD)
  1028. irq_wake_secondary(desc, action);
  1029. wake_threads_waitq(desc);
  1030. }
  1031. /*
  1032. * This is the regular exit path. __free_irq() is stopping the
  1033. * thread via kthread_stop() after calling
  1034. * synchronize_hardirq(). So neither IRQTF_RUNTHREAD nor the
  1035. * oneshot mask bit can be set.
  1036. */
  1037. task_work_cancel(current, irq_thread_dtor);
  1038. return 0;
  1039. }
  1040. /**
  1041. * irq_wake_thread - wake the irq thread for the action identified by dev_id
  1042. * @irq: Interrupt line
  1043. * @dev_id: Device identity for which the thread should be woken
  1044. *
  1045. */
  1046. void irq_wake_thread(unsigned int irq, void *dev_id)
  1047. {
  1048. struct irq_desc *desc = irq_to_desc(irq);
  1049. struct irqaction *action;
  1050. unsigned long flags;
  1051. if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
  1052. return;
  1053. raw_spin_lock_irqsave(&desc->lock, flags);
  1054. for_each_action_of_desc(desc, action) {
  1055. if (action->dev_id == dev_id) {
  1056. if (action->thread)
  1057. __irq_wake_thread(desc, action);
  1058. break;
  1059. }
  1060. }
  1061. raw_spin_unlock_irqrestore(&desc->lock, flags);
  1062. }
  1063. EXPORT_SYMBOL_GPL(irq_wake_thread);
  1064. static int irq_setup_forced_threading(struct irqaction *new)
  1065. {
  1066. if (!force_irqthreads)
  1067. return 0;
  1068. if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
  1069. return 0;
  1070. /*
  1071. * No further action required for interrupts which are requested as
  1072. * threaded interrupts already
  1073. */
  1074. if (new->handler == irq_default_primary_handler)
  1075. return 0;
  1076. new->flags |= IRQF_ONESHOT;
  1077. /*
  1078. * Handle the case where we have a real primary handler and a
  1079. * thread handler. We force thread them as well by creating a
  1080. * secondary action.
  1081. */
  1082. if (new->handler && new->thread_fn) {
  1083. /* Allocate the secondary action */
  1084. new->secondary = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
  1085. if (!new->secondary)
  1086. return -ENOMEM;
  1087. new->secondary->handler = irq_forced_secondary_handler;
  1088. new->secondary->thread_fn = new->thread_fn;
  1089. new->secondary->dev_id = new->dev_id;
  1090. new->secondary->irq = new->irq;
  1091. new->secondary->name = new->name;
  1092. }
  1093. /* Deal with the primary handler */
  1094. set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
  1095. new->thread_fn = new->handler;
  1096. new->handler = irq_default_primary_handler;
  1097. return 0;
  1098. }
  1099. static int irq_request_resources(struct irq_desc *desc)
  1100. {
  1101. struct irq_data *d = &desc->irq_data;
  1102. struct irq_chip *c = d->chip;
  1103. return c->irq_request_resources ? c->irq_request_resources(d) : 0;
  1104. }
  1105. static void irq_release_resources(struct irq_desc *desc)
  1106. {
  1107. struct irq_data *d = &desc->irq_data;
  1108. struct irq_chip *c = d->chip;
  1109. if (c->irq_release_resources)
  1110. c->irq_release_resources(d);
  1111. }
  1112. static bool irq_supports_nmi(struct irq_desc *desc)
  1113. {
  1114. struct irq_data *d = irq_desc_get_irq_data(desc);
  1115. #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
  1116. /* Only IRQs directly managed by the root irqchip can be set as NMI */
  1117. if (d->parent_data)
  1118. return false;
  1119. #endif
  1120. /* Don't support NMIs for chips behind a slow bus */
  1121. if (d->chip->irq_bus_lock || d->chip->irq_bus_sync_unlock)
  1122. return false;
  1123. return d->chip->flags & IRQCHIP_SUPPORTS_NMI;
  1124. }
  1125. static int irq_nmi_setup(struct irq_desc *desc)
  1126. {
  1127. struct irq_data *d = irq_desc_get_irq_data(desc);
  1128. struct irq_chip *c = d->chip;
  1129. return c->irq_nmi_setup ? c->irq_nmi_setup(d) : -EINVAL;
  1130. }
  1131. static void irq_nmi_teardown(struct irq_desc *desc)
  1132. {
  1133. struct irq_data *d = irq_desc_get_irq_data(desc);
  1134. struct irq_chip *c = d->chip;
  1135. if (c->irq_nmi_teardown)
  1136. c->irq_nmi_teardown(d);
  1137. }
  1138. static int
  1139. setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary)
  1140. {
  1141. struct task_struct *t;
  1142. if (!secondary) {
  1143. t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
  1144. new->name);
  1145. } else {
  1146. t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq,
  1147. new->name);
  1148. }
  1149. if (IS_ERR(t))
  1150. return PTR_ERR(t);
  1151. sched_set_fifo(t);
  1152. /*
  1153. * We keep the reference to the task struct even if
  1154. * the thread dies to avoid that the interrupt code
  1155. * references an already freed task_struct.
  1156. */
  1157. new->thread = get_task_struct(t);
  1158. /*
  1159. * Tell the thread to set its affinity. This is
  1160. * important for shared interrupt handlers as we do
  1161. * not invoke setup_affinity() for the secondary
  1162. * handlers as everything is already set up. Even for
  1163. * interrupts marked with IRQF_NO_BALANCE this is
  1164. * correct as we want the thread to move to the cpu(s)
  1165. * on which the requesting code placed the interrupt.
  1166. */
  1167. set_bit(IRQTF_AFFINITY, &new->thread_flags);
  1168. return 0;
  1169. }
  1170. /*
  1171. * Internal function to register an irqaction - typically used to
  1172. * allocate special interrupts that are part of the architecture.
  1173. *
  1174. * Locking rules:
  1175. *
  1176. * desc->request_mutex Provides serialization against a concurrent free_irq()
  1177. * chip_bus_lock Provides serialization for slow bus operations
  1178. * desc->lock Provides serialization against hard interrupts
  1179. *
  1180. * chip_bus_lock and desc->lock are sufficient for all other management and
  1181. * interrupt related functions. desc->request_mutex solely serializes
  1182. * request/free_irq().
  1183. */
  1184. static int
  1185. __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
  1186. {
  1187. struct irqaction *old, **old_ptr;
  1188. unsigned long flags, thread_mask = 0;
  1189. int ret, nested, shared = 0;
  1190. if (!desc)
  1191. return -EINVAL;
  1192. if (desc->irq_data.chip == &no_irq_chip)
  1193. return -ENOSYS;
  1194. if (!try_module_get(desc->owner))
  1195. return -ENODEV;
  1196. new->irq = irq;
  1197. /*
  1198. * If the trigger type is not specified by the caller,
  1199. * then use the default for this interrupt.
  1200. */
  1201. if (!(new->flags & IRQF_TRIGGER_MASK))
  1202. new->flags |= irqd_get_trigger_type(&desc->irq_data);
  1203. /*
  1204. * Check whether the interrupt nests into another interrupt
  1205. * thread.
  1206. */
  1207. nested = irq_settings_is_nested_thread(desc);
  1208. if (nested) {
  1209. if (!new->thread_fn) {
  1210. ret = -EINVAL;
  1211. goto out_mput;
  1212. }
  1213. /*
  1214. * Replace the primary handler which was provided from
  1215. * the driver for non nested interrupt handling by the
  1216. * dummy function which warns when called.
  1217. */
  1218. new->handler = irq_nested_primary_handler;
  1219. } else {
  1220. if (irq_settings_can_thread(desc)) {
  1221. ret = irq_setup_forced_threading(new);
  1222. if (ret)
  1223. goto out_mput;
  1224. }
  1225. }
  1226. /*
  1227. * Create a handler thread when a thread function is supplied
  1228. * and the interrupt does not nest into another interrupt
  1229. * thread.
  1230. */
  1231. if (new->thread_fn && !nested) {
  1232. ret = setup_irq_thread(new, irq, false);
  1233. if (ret)
  1234. goto out_mput;
  1235. if (new->secondary) {
  1236. ret = setup_irq_thread(new->secondary, irq, true);
  1237. if (ret)
  1238. goto out_thread;
  1239. }
  1240. }
  1241. /*
  1242. * Drivers are often written to work w/o knowledge about the
  1243. * underlying irq chip implementation, so a request for a
  1244. * threaded irq without a primary hard irq context handler
  1245. * requires the ONESHOT flag to be set. Some irq chips like
  1246. * MSI based interrupts are per se one shot safe. Check the
  1247. * chip flags, so we can avoid the unmask dance at the end of
  1248. * the threaded handler for those.
  1249. */
  1250. if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)
  1251. new->flags &= ~IRQF_ONESHOT;
  1252. /*
  1253. * Protects against a concurrent __free_irq() call which might wait
  1254. * for synchronize_hardirq() to complete without holding the optional
  1255. * chip bus lock and desc->lock. Also protects against handing out
  1256. * a recycled oneshot thread_mask bit while it's still in use by
  1257. * its previous owner.
  1258. */
  1259. mutex_lock(&desc->request_mutex);
  1260. /*
  1261. * Acquire bus lock as the irq_request_resources() callback below
  1262. * might rely on the serialization or the magic power management
  1263. * functions which are abusing the irq_bus_lock() callback,
  1264. */
  1265. chip_bus_lock(desc);
  1266. /* First installed action requests resources. */
  1267. if (!desc->action) {
  1268. ret = irq_request_resources(desc);
  1269. if (ret) {
  1270. pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n",
  1271. new->name, irq, desc->irq_data.chip->name);
  1272. goto out_bus_unlock;
  1273. }
  1274. }
  1275. /*
  1276. * The following block of code has to be executed atomically
  1277. * protected against a concurrent interrupt and any of the other
  1278. * management calls which are not serialized via
  1279. * desc->request_mutex or the optional bus lock.
  1280. */
  1281. raw_spin_lock_irqsave(&desc->lock, flags);
  1282. old_ptr = &desc->action;
  1283. old = *old_ptr;
  1284. if (old) {
  1285. /*
  1286. * Can't share interrupts unless both agree to and are
  1287. * the same type (level, edge, polarity). So both flag
  1288. * fields must have IRQF_SHARED set and the bits which
  1289. * set the trigger type must match. Also all must
  1290. * agree on ONESHOT.
  1291. * Interrupt lines used for NMIs cannot be shared.
  1292. */
  1293. unsigned int oldtype;
  1294. if (desc->istate & IRQS_NMI) {
  1295. pr_err("Invalid attempt to share NMI for %s (irq %d) on irqchip %s.\n",
  1296. new->name, irq, desc->irq_data.chip->name);
  1297. ret = -EINVAL;
  1298. goto out_unlock;
  1299. }
  1300. /*
  1301. * If nobody did set the configuration before, inherit
  1302. * the one provided by the requester.
  1303. */
  1304. if (irqd_trigger_type_was_set(&desc->irq_data)) {
  1305. oldtype = irqd_get_trigger_type(&desc->irq_data);
  1306. } else {
  1307. oldtype = new->flags & IRQF_TRIGGER_MASK;
  1308. irqd_set_trigger_type(&desc->irq_data, oldtype);
  1309. }
  1310. if (!((old->flags & new->flags) & IRQF_SHARED) ||
  1311. (oldtype != (new->flags & IRQF_TRIGGER_MASK)) ||
  1312. ((old->flags ^ new->flags) & IRQF_ONESHOT))
  1313. goto mismatch;
  1314. /* All handlers must agree on per-cpuness */
  1315. if ((old->flags & IRQF_PERCPU) !=
  1316. (new->flags & IRQF_PERCPU))
  1317. goto mismatch;
  1318. /* add new interrupt at end of irq queue */
  1319. do {
  1320. /*
  1321. * Or all existing action->thread_mask bits,
  1322. * so we can find the next zero bit for this
  1323. * new action.
  1324. */
  1325. thread_mask |= old->thread_mask;
  1326. old_ptr = &old->next;
  1327. old = *old_ptr;
  1328. } while (old);
  1329. shared = 1;
  1330. }
  1331. /*
  1332. * Setup the thread mask for this irqaction for ONESHOT. For
  1333. * !ONESHOT irqs the thread mask is 0 so we can avoid a
  1334. * conditional in irq_wake_thread().
  1335. */
  1336. if (new->flags & IRQF_ONESHOT) {
  1337. /*
  1338. * Unlikely to have 32 resp 64 irqs sharing one line,
  1339. * but who knows.
  1340. */
  1341. if (thread_mask == ~0UL) {
  1342. ret = -EBUSY;
  1343. goto out_unlock;
  1344. }
  1345. /*
  1346. * The thread_mask for the action is or'ed to
  1347. * desc->thread_active to indicate that the
  1348. * IRQF_ONESHOT thread handler has been woken, but not
  1349. * yet finished. The bit is cleared when a thread
  1350. * completes. When all threads of a shared interrupt
  1351. * line have completed desc->threads_active becomes
  1352. * zero and the interrupt line is unmasked. See
  1353. * handle.c:irq_wake_thread() for further information.
  1354. *
  1355. * If no thread is woken by primary (hard irq context)
  1356. * interrupt handlers, then desc->threads_active is
  1357. * also checked for zero to unmask the irq line in the
  1358. * affected hard irq flow handlers
  1359. * (handle_[fasteoi|level]_irq).
  1360. *
  1361. * The new action gets the first zero bit of
  1362. * thread_mask assigned. See the loop above which or's
  1363. * all existing action->thread_mask bits.
  1364. */
  1365. new->thread_mask = 1UL << ffz(thread_mask);
  1366. } else if (new->handler == irq_default_primary_handler &&
  1367. !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) {
  1368. /*
  1369. * The interrupt was requested with handler = NULL, so
  1370. * we use the default primary handler for it. But it
  1371. * does not have the oneshot flag set. In combination
  1372. * with level interrupts this is deadly, because the
  1373. * default primary handler just wakes the thread, then
  1374. * the irq lines is reenabled, but the device still
  1375. * has the level irq asserted. Rinse and repeat....
  1376. *
  1377. * While this works for edge type interrupts, we play
  1378. * it safe and reject unconditionally because we can't
  1379. * say for sure which type this interrupt really
  1380. * has. The type flags are unreliable as the
  1381. * underlying chip implementation can override them.
  1382. */
  1383. pr_err("Threaded irq requested with handler=NULL and !ONESHOT for %s (irq %d)\n",
  1384. new->name, irq);
  1385. ret = -EINVAL;
  1386. goto out_unlock;
  1387. }
  1388. if (!shared) {
  1389. init_waitqueue_head(&desc->wait_for_threads);
  1390. /* Setup the type (level, edge polarity) if configured: */
  1391. if (new->flags & IRQF_TRIGGER_MASK) {
  1392. ret = __irq_set_trigger(desc,
  1393. new->flags & IRQF_TRIGGER_MASK);
  1394. if (ret)
  1395. goto out_unlock;
  1396. }
  1397. /*
  1398. * Activate the interrupt. That activation must happen
  1399. * independently of IRQ_NOAUTOEN. request_irq() can fail
  1400. * and the callers are supposed to handle
  1401. * that. enable_irq() of an interrupt requested with
  1402. * IRQ_NOAUTOEN is not supposed to fail. The activation
  1403. * keeps it in shutdown mode, it merily associates
  1404. * resources if necessary and if that's not possible it
  1405. * fails. Interrupts which are in managed shutdown mode
  1406. * will simply ignore that activation request.
  1407. */
  1408. ret = irq_activate(desc);
  1409. if (ret)
  1410. goto out_unlock;
  1411. desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \
  1412. IRQS_ONESHOT | IRQS_WAITING);
  1413. irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
  1414. if (new->flags & IRQF_PERCPU) {
  1415. irqd_set(&desc->irq_data, IRQD_PER_CPU);
  1416. irq_settings_set_per_cpu(desc);
  1417. }
  1418. if (new->flags & IRQF_ONESHOT)
  1419. desc->istate |= IRQS_ONESHOT;
  1420. /* Exclude IRQ from balancing if requested */
  1421. if (new->flags & IRQF_NOBALANCING) {
  1422. irq_settings_set_no_balancing(desc);
  1423. irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
  1424. }
  1425. if (irq_settings_can_autoenable(desc)) {
  1426. irq_startup(desc, IRQ_RESEND, IRQ_START_COND);
  1427. } else {
  1428. /*
  1429. * Shared interrupts do not go well with disabling
  1430. * auto enable. The sharing interrupt might request
  1431. * it while it's still disabled and then wait for
  1432. * interrupts forever.
  1433. */
  1434. WARN_ON_ONCE(new->flags & IRQF_SHARED);
  1435. /* Undo nested disables: */
  1436. desc->depth = 1;
  1437. }
  1438. } else if (new->flags & IRQF_TRIGGER_MASK) {
  1439. unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
  1440. unsigned int omsk = irqd_get_trigger_type(&desc->irq_data);
  1441. if (nmsk != omsk)
  1442. /* hope the handler works with current trigger mode */
  1443. pr_warn("irq %d uses trigger mode %u; requested %u\n",
  1444. irq, omsk, nmsk);
  1445. }
  1446. *old_ptr = new;
  1447. irq_pm_install_action(desc, new);
  1448. /* Reset broken irq detection when installing new handler */
  1449. desc->irq_count = 0;
  1450. desc->irqs_unhandled = 0;
  1451. /*
  1452. * Check whether we disabled the irq via the spurious handler
  1453. * before. Reenable it and give it another chance.
  1454. */
  1455. if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) {
  1456. desc->istate &= ~IRQS_SPURIOUS_DISABLED;
  1457. __enable_irq(desc);
  1458. }
  1459. raw_spin_unlock_irqrestore(&desc->lock, flags);
  1460. chip_bus_sync_unlock(desc);
  1461. mutex_unlock(&desc->request_mutex);
  1462. irq_setup_timings(desc, new);
  1463. /*
  1464. * Strictly no need to wake it up, but hung_task complains
  1465. * when no hard interrupt wakes the thread up.
  1466. */
  1467. if (new->thread)
  1468. wake_up_process(new->thread);
  1469. if (new->secondary)
  1470. wake_up_process(new->secondary->thread);
  1471. register_irq_proc(irq, desc);
  1472. new->dir = NULL;
  1473. register_handler_proc(irq, new);
  1474. return 0;
  1475. mismatch:
  1476. if (!(new->flags & IRQF_PROBE_SHARED)) {
  1477. pr_err("Flags mismatch irq %d. %08x (%s) vs. %08x (%s)\n",
  1478. irq, new->flags, new->name, old->flags, old->name);
  1479. #ifdef CONFIG_DEBUG_SHIRQ
  1480. dump_stack();
  1481. #endif
  1482. }
  1483. ret = -EBUSY;
  1484. out_unlock:
  1485. raw_spin_unlock_irqrestore(&desc->lock, flags);
  1486. if (!desc->action)
  1487. irq_release_resources(desc);
  1488. out_bus_unlock:
  1489. chip_bus_sync_unlock(desc);
  1490. mutex_unlock(&desc->request_mutex);
  1491. out_thread:
  1492. if (new->thread) {
  1493. struct task_struct *t = new->thread;
  1494. new->thread = NULL;
  1495. kthread_stop(t);
  1496. put_task_struct(t);
  1497. }
  1498. if (new->secondary && new->secondary->thread) {
  1499. struct task_struct *t = new->secondary->thread;
  1500. new->secondary->thread = NULL;
  1501. kthread_stop(t);
  1502. put_task_struct(t);
  1503. }
  1504. out_mput:
  1505. module_put(desc->owner);
  1506. return ret;
  1507. }
  1508. /*
  1509. * Internal function to unregister an irqaction - used to free
  1510. * regular and special interrupts that are part of the architecture.
  1511. */
  1512. static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id)
  1513. {
  1514. unsigned irq = desc->irq_data.irq;
  1515. struct irqaction *action, **action_ptr;
  1516. unsigned long flags;
  1517. WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
  1518. mutex_lock(&desc->request_mutex);
  1519. chip_bus_lock(desc);
  1520. raw_spin_lock_irqsave(&desc->lock, flags);
  1521. /*
  1522. * There can be multiple actions per IRQ descriptor, find the right
  1523. * one based on the dev_id:
  1524. */
  1525. action_ptr = &desc->action;
  1526. for (;;) {
  1527. action = *action_ptr;
  1528. if (!action) {
  1529. WARN(1, "Trying to free already-free IRQ %d\n", irq);
  1530. raw_spin_unlock_irqrestore(&desc->lock, flags);
  1531. chip_bus_sync_unlock(desc);
  1532. mutex_unlock(&desc->request_mutex);
  1533. return NULL;
  1534. }
  1535. if (action->dev_id == dev_id)
  1536. break;
  1537. action_ptr = &action->next;
  1538. }
  1539. /* Found it - now remove it from the list of entries: */
  1540. *action_ptr = action->next;
  1541. irq_pm_remove_action(desc, action);
  1542. /* If this was the last handler, shut down the IRQ line: */
  1543. if (!desc->action) {
  1544. irq_settings_clr_disable_unlazy(desc);
  1545. /* Only shutdown. Deactivate after synchronize_hardirq() */
  1546. irq_shutdown(desc);
  1547. }
  1548. #ifdef CONFIG_SMP
  1549. /* make sure affinity_hint is cleaned up */
  1550. if (WARN_ON_ONCE(desc->affinity_hint))
  1551. desc->affinity_hint = NULL;
  1552. #endif
  1553. raw_spin_unlock_irqrestore(&desc->lock, flags);
  1554. /*
  1555. * Drop bus_lock here so the changes which were done in the chip
  1556. * callbacks above are synced out to the irq chips which hang
  1557. * behind a slow bus (I2C, SPI) before calling synchronize_hardirq().
  1558. *
  1559. * Aside of that the bus_lock can also be taken from the threaded
  1560. * handler in irq_finalize_oneshot() which results in a deadlock
  1561. * because kthread_stop() would wait forever for the thread to
  1562. * complete, which is blocked on the bus lock.
  1563. *
  1564. * The still held desc->request_mutex() protects against a
  1565. * concurrent request_irq() of this irq so the release of resources
  1566. * and timing data is properly serialized.
  1567. */
  1568. chip_bus_sync_unlock(desc);
  1569. unregister_handler_proc(irq, action);
  1570. /*
  1571. * Make sure it's not being used on another CPU and if the chip
  1572. * supports it also make sure that there is no (not yet serviced)
  1573. * interrupt in flight at the hardware level.
  1574. */
  1575. __synchronize_hardirq(desc, true);
  1576. #ifdef CONFIG_DEBUG_SHIRQ
  1577. /*
  1578. * It's a shared IRQ -- the driver ought to be prepared for an IRQ
  1579. * event to happen even now it's being freed, so let's make sure that
  1580. * is so by doing an extra call to the handler ....
  1581. *
  1582. * ( We do this after actually deregistering it, to make sure that a
  1583. * 'real' IRQ doesn't run in parallel with our fake. )
  1584. */
  1585. if (action->flags & IRQF_SHARED) {
  1586. local_irq_save(flags);
  1587. action->handler(irq, dev_id);
  1588. local_irq_restore(flags);
  1589. }
  1590. #endif
  1591. /*
  1592. * The action has already been removed above, but the thread writes
  1593. * its oneshot mask bit when it completes. Though request_mutex is
  1594. * held across this which prevents __setup_irq() from handing out
  1595. * the same bit to a newly requested action.
  1596. */
  1597. if (action->thread) {
  1598. kthread_stop(action->thread);
  1599. put_task_struct(action->thread);
  1600. if (action->secondary && action->secondary->thread) {
  1601. kthread_stop(action->secondary->thread);
  1602. put_task_struct(action->secondary->thread);
  1603. }
  1604. }
  1605. /* Last action releases resources */
  1606. if (!desc->action) {
  1607. /*
  1608. * Reaquire bus lock as irq_release_resources() might
  1609. * require it to deallocate resources over the slow bus.
  1610. */
  1611. chip_bus_lock(desc);
  1612. /*
  1613. * There is no interrupt on the fly anymore. Deactivate it
  1614. * completely.
  1615. */
  1616. raw_spin_lock_irqsave(&desc->lock, flags);
  1617. irq_domain_deactivate_irq(&desc->irq_data);
  1618. raw_spin_unlock_irqrestore(&desc->lock, flags);
  1619. irq_release_resources(desc);
  1620. chip_bus_sync_unlock(desc);
  1621. irq_remove_timings(desc);
  1622. }
  1623. mutex_unlock(&desc->request_mutex);
  1624. irq_chip_pm_put(&desc->irq_data);
  1625. module_put(desc->owner);
  1626. kfree(action->secondary);
  1627. return action;
  1628. }
  1629. /**
  1630. * free_irq - free an interrupt allocated with request_irq
  1631. * @irq: Interrupt line to free
  1632. * @dev_id: Device identity to free
  1633. *
  1634. * Remove an interrupt handler. The handler is removed and if the
  1635. * interrupt line is no longer in use by any driver it is disabled.
  1636. * On a shared IRQ the caller must ensure the interrupt is disabled
  1637. * on the card it drives before calling this function. The function
  1638. * does not return until any executing interrupts for this IRQ
  1639. * have completed.
  1640. *
  1641. * This function must not be called from interrupt context.
  1642. *
  1643. * Returns the devname argument passed to request_irq.
  1644. */
  1645. const void *free_irq(unsigned int irq, void *dev_id)
  1646. {
  1647. struct irq_desc *desc = irq_to_desc(irq);
  1648. struct irqaction *action;
  1649. const char *devname;
  1650. if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
  1651. return NULL;
  1652. #ifdef CONFIG_SMP
  1653. if (WARN_ON(desc->affinity_notify))
  1654. desc->affinity_notify = NULL;
  1655. #endif
  1656. action = __free_irq(desc, dev_id);
  1657. if (!action)
  1658. return NULL;
  1659. devname = action->name;
  1660. kfree(action);
  1661. return devname;
  1662. }
  1663. EXPORT_SYMBOL(free_irq);
  1664. /* This function must be called with desc->lock held */
  1665. static const void *__cleanup_nmi(unsigned int irq, struct irq_desc *desc)
  1666. {
  1667. const char *devname = NULL;
  1668. desc->istate &= ~IRQS_NMI;
  1669. if (!WARN_ON(desc->action == NULL)) {
  1670. irq_pm_remove_action(desc, desc->action);
  1671. devname = desc->action->name;
  1672. unregister_handler_proc(irq, desc->action);
  1673. kfree(desc->action);
  1674. desc->action = NULL;
  1675. }
  1676. irq_settings_clr_disable_unlazy(desc);
  1677. irq_shutdown_and_deactivate(desc);
  1678. irq_release_resources(desc);
  1679. irq_chip_pm_put(&desc->irq_data);
  1680. module_put(desc->owner);
  1681. return devname;
  1682. }
  1683. const void *free_nmi(unsigned int irq, void *dev_id)
  1684. {
  1685. struct irq_desc *desc = irq_to_desc(irq);
  1686. unsigned long flags;
  1687. const void *devname;
  1688. if (!desc || WARN_ON(!(desc->istate & IRQS_NMI)))
  1689. return NULL;
  1690. if (WARN_ON(irq_settings_is_per_cpu_devid(desc)))
  1691. return NULL;
  1692. /* NMI still enabled */
  1693. if (WARN_ON(desc->depth == 0))
  1694. disable_nmi_nosync(irq);
  1695. raw_spin_lock_irqsave(&desc->lock, flags);
  1696. irq_nmi_teardown(desc);
  1697. devname = __cleanup_nmi(irq, desc);
  1698. raw_spin_unlock_irqrestore(&desc->lock, flags);
  1699. return devname;
  1700. }
  1701. /**
  1702. * request_threaded_irq - allocate an interrupt line
  1703. * @irq: Interrupt line to allocate
  1704. * @handler: Function to be called when the IRQ occurs.
  1705. * Primary handler for threaded interrupts
  1706. * If NULL and thread_fn != NULL the default
  1707. * primary handler is installed
  1708. * @thread_fn: Function called from the irq handler thread
  1709. * If NULL, no irq thread is created
  1710. * @irqflags: Interrupt type flags
  1711. * @devname: An ascii name for the claiming device
  1712. * @dev_id: A cookie passed back to the handler function
  1713. *
  1714. * This call allocates interrupt resources and enables the
  1715. * interrupt line and IRQ handling. From the point this
  1716. * call is made your handler function may be invoked. Since
  1717. * your handler function must clear any interrupt the board
  1718. * raises, you must take care both to initialise your hardware
  1719. * and to set up the interrupt handler in the right order.
  1720. *
  1721. * If you want to set up a threaded irq handler for your device
  1722. * then you need to supply @handler and @thread_fn. @handler is
  1723. * still called in hard interrupt context and has to check
  1724. * whether the interrupt originates from the device. If yes it
  1725. * needs to disable the interrupt on the device and return
  1726. * IRQ_WAKE_THREAD which will wake up the handler thread and run
  1727. * @thread_fn. This split handler design is necessary to support
  1728. * shared interrupts.
  1729. *
  1730. * Dev_id must be globally unique. Normally the address of the
  1731. * device data structure is used as the cookie. Since the handler
  1732. * receives this value it makes sense to use it.
  1733. *
  1734. * If your interrupt is shared you must pass a non NULL dev_id
  1735. * as this is required when freeing the interrupt.
  1736. *
  1737. * Flags:
  1738. *
  1739. * IRQF_SHARED Interrupt is shared
  1740. * IRQF_TRIGGER_* Specify active edge(s) or level
  1741. *
  1742. */
  1743. int request_threaded_irq(unsigned int irq, irq_handler_t handler,
  1744. irq_handler_t thread_fn, unsigned long irqflags,
  1745. const char *devname, void *dev_id)
  1746. {
  1747. struct irqaction *action;
  1748. struct irq_desc *desc;
  1749. int retval;
  1750. if (irq == IRQ_NOTCONNECTED)
  1751. return -ENOTCONN;
  1752. /*
  1753. * Sanity-check: shared interrupts must pass in a real dev-ID,
  1754. * otherwise we'll have trouble later trying to figure out
  1755. * which interrupt is which (messes up the interrupt freeing
  1756. * logic etc).
  1757. *
  1758. * Also IRQF_COND_SUSPEND only makes sense for shared interrupts and
  1759. * it cannot be set along with IRQF_NO_SUSPEND.
  1760. */
  1761. if (((irqflags & IRQF_SHARED) && !dev_id) ||
  1762. (!(irqflags & IRQF_SHARED) && (irqflags & IRQF_COND_SUSPEND)) ||
  1763. ((irqflags & IRQF_NO_SUSPEND) && (irqflags & IRQF_COND_SUSPEND)))
  1764. return -EINVAL;
  1765. desc = irq_to_desc(irq);
  1766. if (!desc)
  1767. return -EINVAL;
  1768. if (!irq_settings_can_request(desc) ||
  1769. WARN_ON(irq_settings_is_per_cpu_devid(desc)))
  1770. return -EINVAL;
  1771. if (!handler) {
  1772. if (!thread_fn)
  1773. return -EINVAL;
  1774. handler = irq_default_primary_handler;
  1775. }
  1776. action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
  1777. if (!action)
  1778. return -ENOMEM;
  1779. action->handler = handler;
  1780. action->thread_fn = thread_fn;
  1781. action->flags = irqflags;
  1782. action->name = devname;
  1783. action->dev_id = dev_id;
  1784. retval = irq_chip_pm_get(&desc->irq_data);
  1785. if (retval < 0) {
  1786. kfree(action);
  1787. return retval;
  1788. }
  1789. retval = __setup_irq(irq, desc, action);
  1790. if (retval) {
  1791. irq_chip_pm_put(&desc->irq_data);
  1792. kfree(action->secondary);
  1793. kfree(action);
  1794. }
  1795. #ifdef CONFIG_DEBUG_SHIRQ_FIXME
  1796. if (!retval && (irqflags & IRQF_SHARED)) {
  1797. /*
  1798. * It's a shared IRQ -- the driver ought to be prepared for it
  1799. * to happen immediately, so let's make sure....
  1800. * We disable the irq to make sure that a 'real' IRQ doesn't
  1801. * run in parallel with our fake.
  1802. */
  1803. unsigned long flags;
  1804. disable_irq(irq);
  1805. local_irq_save(flags);
  1806. handler(irq, dev_id);
  1807. local_irq_restore(flags);
  1808. enable_irq(irq);
  1809. }
  1810. #endif
  1811. return retval;
  1812. }
  1813. EXPORT_SYMBOL(request_threaded_irq);
  1814. /**
  1815. * request_any_context_irq - allocate an interrupt line
  1816. * @irq: Interrupt line to allocate
  1817. * @handler: Function to be called when the IRQ occurs.
  1818. * Threaded handler for threaded interrupts.
  1819. * @flags: Interrupt type flags
  1820. * @name: An ascii name for the claiming device
  1821. * @dev_id: A cookie passed back to the handler function
  1822. *
  1823. * This call allocates interrupt resources and enables the
  1824. * interrupt line and IRQ handling. It selects either a
  1825. * hardirq or threaded handling method depending on the
  1826. * context.
  1827. *
  1828. * On failure, it returns a negative value. On success,
  1829. * it returns either IRQC_IS_HARDIRQ or IRQC_IS_NESTED.
  1830. */
  1831. int request_any_context_irq(unsigned int irq, irq_handler_t handler,
  1832. unsigned long flags, const char *name, void *dev_id)
  1833. {
  1834. struct irq_desc *desc;
  1835. int ret;
  1836. if (irq == IRQ_NOTCONNECTED)
  1837. return -ENOTCONN;
  1838. desc = irq_to_desc(irq);
  1839. if (!desc)
  1840. return -EINVAL;
  1841. if (irq_settings_is_nested_thread(desc)) {
  1842. ret = request_threaded_irq(irq, NULL, handler,
  1843. flags, name, dev_id);
  1844. return !ret ? IRQC_IS_NESTED : ret;
  1845. }
  1846. ret = request_irq(irq, handler, flags, name, dev_id);
  1847. return !ret ? IRQC_IS_HARDIRQ : ret;
  1848. }
  1849. EXPORT_SYMBOL_GPL(request_any_context_irq);
  1850. /**
  1851. * request_nmi - allocate an interrupt line for NMI delivery
  1852. * @irq: Interrupt line to allocate
  1853. * @handler: Function to be called when the IRQ occurs.
  1854. * Threaded handler for threaded interrupts.
  1855. * @irqflags: Interrupt type flags
  1856. * @name: An ascii name for the claiming device
  1857. * @dev_id: A cookie passed back to the handler function
  1858. *
  1859. * This call allocates interrupt resources and enables the
  1860. * interrupt line and IRQ handling. It sets up the IRQ line
  1861. * to be handled as an NMI.
  1862. *
  1863. * An interrupt line delivering NMIs cannot be shared and IRQ handling
  1864. * cannot be threaded.
  1865. *
  1866. * Interrupt lines requested for NMI delivering must produce per cpu
  1867. * interrupts and have auto enabling setting disabled.
  1868. *
  1869. * Dev_id must be globally unique. Normally the address of the
  1870. * device data structure is used as the cookie. Since the handler
  1871. * receives this value it makes sense to use it.
  1872. *
  1873. * If the interrupt line cannot be used to deliver NMIs, function
  1874. * will fail and return a negative value.
  1875. */
  1876. int request_nmi(unsigned int irq, irq_handler_t handler,
  1877. unsigned long irqflags, const char *name, void *dev_id)
  1878. {
  1879. struct irqaction *action;
  1880. struct irq_desc *desc;
  1881. unsigned long flags;
  1882. int retval;
  1883. if (irq == IRQ_NOTCONNECTED)
  1884. return -ENOTCONN;
  1885. /* NMI cannot be shared, used for Polling */
  1886. if (irqflags & (IRQF_SHARED | IRQF_COND_SUSPEND | IRQF_IRQPOLL))
  1887. return -EINVAL;
  1888. if (!(irqflags & IRQF_PERCPU))
  1889. return -EINVAL;
  1890. if (!handler)
  1891. return -EINVAL;
  1892. desc = irq_to_desc(irq);
  1893. if (!desc || irq_settings_can_autoenable(desc) ||
  1894. !irq_settings_can_request(desc) ||
  1895. WARN_ON(irq_settings_is_per_cpu_devid(desc)) ||
  1896. !irq_supports_nmi(desc))
  1897. return -EINVAL;
  1898. action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
  1899. if (!action)
  1900. return -ENOMEM;
  1901. action->handler = handler;
  1902. action->flags = irqflags | IRQF_NO_THREAD | IRQF_NOBALANCING;
  1903. action->name = name;
  1904. action->dev_id = dev_id;
  1905. retval = irq_chip_pm_get(&desc->irq_data);
  1906. if (retval < 0)
  1907. goto err_out;
  1908. retval = __setup_irq(irq, desc, action);
  1909. if (retval)
  1910. goto err_irq_setup;
  1911. raw_spin_lock_irqsave(&desc->lock, flags);
  1912. /* Setup NMI state */
  1913. desc->istate |= IRQS_NMI;
  1914. retval = irq_nmi_setup(desc);
  1915. if (retval) {
  1916. __cleanup_nmi(irq, desc);
  1917. raw_spin_unlock_irqrestore(&desc->lock, flags);
  1918. return -EINVAL;
  1919. }
  1920. raw_spin_unlock_irqrestore(&desc->lock, flags);
  1921. return 0;
  1922. err_irq_setup:
  1923. irq_chip_pm_put(&desc->irq_data);
  1924. err_out:
  1925. kfree(action);
  1926. return retval;
  1927. }
  1928. void enable_percpu_irq(unsigned int irq, unsigned int type)
  1929. {
  1930. unsigned int cpu = smp_processor_id();
  1931. unsigned long flags;
  1932. struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
  1933. if (!desc)
  1934. return;
  1935. /*
  1936. * If the trigger type is not specified by the caller, then
  1937. * use the default for this interrupt.
  1938. */
  1939. type &= IRQ_TYPE_SENSE_MASK;
  1940. if (type == IRQ_TYPE_NONE)
  1941. type = irqd_get_trigger_type(&desc->irq_data);
  1942. if (type != IRQ_TYPE_NONE) {
  1943. int ret;
  1944. ret = __irq_set_trigger(desc, type);
  1945. if (ret) {
  1946. WARN(1, "failed to set type for IRQ%d\n", irq);
  1947. goto out;
  1948. }
  1949. }
  1950. irq_percpu_enable(desc, cpu);
  1951. out:
  1952. irq_put_desc_unlock(desc, flags);
  1953. }
  1954. EXPORT_SYMBOL_GPL(enable_percpu_irq);
  1955. void enable_percpu_nmi(unsigned int irq, unsigned int type)
  1956. {
  1957. enable_percpu_irq(irq, type);
  1958. }
  1959. /**
  1960. * irq_percpu_is_enabled - Check whether the per cpu irq is enabled
  1961. * @irq: Linux irq number to check for
  1962. *
  1963. * Must be called from a non migratable context. Returns the enable
  1964. * state of a per cpu interrupt on the current cpu.
  1965. */
  1966. bool irq_percpu_is_enabled(unsigned int irq)
  1967. {
  1968. unsigned int cpu = smp_processor_id();
  1969. struct irq_desc *desc;
  1970. unsigned long flags;
  1971. bool is_enabled;
  1972. desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
  1973. if (!desc)
  1974. return false;
  1975. is_enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
  1976. irq_put_desc_unlock(desc, flags);
  1977. return is_enabled;
  1978. }
  1979. EXPORT_SYMBOL_GPL(irq_percpu_is_enabled);
  1980. void disable_percpu_irq(unsigned int irq)
  1981. {
  1982. unsigned int cpu = smp_processor_id();
  1983. unsigned long flags;
  1984. struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
  1985. if (!desc)
  1986. return;
  1987. irq_percpu_disable(desc, cpu);
  1988. irq_put_desc_unlock(desc, flags);
  1989. }
  1990. EXPORT_SYMBOL_GPL(disable_percpu_irq);
  1991. void disable_percpu_nmi(unsigned int irq)
  1992. {
  1993. disable_percpu_irq(irq);
  1994. }
  1995. /*
  1996. * Internal function to unregister a percpu irqaction.
  1997. */
  1998. static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_id)
  1999. {
  2000. struct irq_desc *desc = irq_to_desc(irq);
  2001. struct irqaction *action;
  2002. unsigned long flags;
  2003. WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
  2004. if (!desc)
  2005. return NULL;
  2006. raw_spin_lock_irqsave(&desc->lock, flags);
  2007. action = desc->action;
  2008. if (!action || action->percpu_dev_id != dev_id) {
  2009. WARN(1, "Trying to free already-free IRQ %d\n", irq);
  2010. goto bad;
  2011. }
  2012. if (!cpumask_empty(desc->percpu_enabled)) {
  2013. WARN(1, "percpu IRQ %d still enabled on CPU%d!\n",
  2014. irq, cpumask_first(desc->percpu_enabled));
  2015. goto bad;
  2016. }
  2017. /* Found it - now remove it from the list of entries: */
  2018. desc->action = NULL;
  2019. desc->istate &= ~IRQS_NMI;
  2020. raw_spin_unlock_irqrestore(&desc->lock, flags);
  2021. unregister_handler_proc(irq, action);
  2022. irq_chip_pm_put(&desc->irq_data);
  2023. module_put(desc->owner);
  2024. return action;
  2025. bad:
  2026. raw_spin_unlock_irqrestore(&desc->lock, flags);
  2027. return NULL;
  2028. }
  2029. /**
  2030. * remove_percpu_irq - free a per-cpu interrupt
  2031. * @irq: Interrupt line to free
  2032. * @act: irqaction for the interrupt
  2033. *
  2034. * Used to remove interrupts statically setup by the early boot process.
  2035. */
  2036. void remove_percpu_irq(unsigned int irq, struct irqaction *act)
  2037. {
  2038. struct irq_desc *desc = irq_to_desc(irq);
  2039. if (desc && irq_settings_is_per_cpu_devid(desc))
  2040. __free_percpu_irq(irq, act->percpu_dev_id);
  2041. }
  2042. /**
  2043. * free_percpu_irq - free an interrupt allocated with request_percpu_irq
  2044. * @irq: Interrupt line to free
  2045. * @dev_id: Device identity to free
  2046. *
  2047. * Remove a percpu interrupt handler. The handler is removed, but
  2048. * the interrupt line is not disabled. This must be done on each
  2049. * CPU before calling this function. The function does not return
  2050. * until any executing interrupts for this IRQ have completed.
  2051. *
  2052. * This function must not be called from interrupt context.
  2053. */
  2054. void free_percpu_irq(unsigned int irq, void __percpu *dev_id)
  2055. {
  2056. struct irq_desc *desc = irq_to_desc(irq);
  2057. if (!desc || !irq_settings_is_per_cpu_devid(desc))
  2058. return;
  2059. chip_bus_lock(desc);
  2060. kfree(__free_percpu_irq(irq, dev_id));
  2061. chip_bus_sync_unlock(desc);
  2062. }
  2063. EXPORT_SYMBOL_GPL(free_percpu_irq);
  2064. void free_percpu_nmi(unsigned int irq, void __percpu *dev_id)
  2065. {
  2066. struct irq_desc *desc = irq_to_desc(irq);
  2067. if (!desc || !irq_settings_is_per_cpu_devid(desc))
  2068. return;
  2069. if (WARN_ON(!(desc->istate & IRQS_NMI)))
  2070. return;
  2071. kfree(__free_percpu_irq(irq, dev_id));
  2072. }
  2073. /**
  2074. * setup_percpu_irq - setup a per-cpu interrupt
  2075. * @irq: Interrupt line to setup
  2076. * @act: irqaction for the interrupt
  2077. *
  2078. * Used to statically setup per-cpu interrupts in the early boot process.
  2079. */
  2080. int setup_percpu_irq(unsigned int irq, struct irqaction *act)
  2081. {
  2082. struct irq_desc *desc = irq_to_desc(irq);
  2083. int retval;
  2084. if (!desc || !irq_settings_is_per_cpu_devid(desc))
  2085. return -EINVAL;
  2086. retval = irq_chip_pm_get(&desc->irq_data);
  2087. if (retval < 0)
  2088. return retval;
  2089. retval = __setup_irq(irq, desc, act);
  2090. if (retval)
  2091. irq_chip_pm_put(&desc->irq_data);
  2092. return retval;
  2093. }
  2094. /**
  2095. * __request_percpu_irq - allocate a percpu interrupt line
  2096. * @irq: Interrupt line to allocate
  2097. * @handler: Function to be called when the IRQ occurs.
  2098. * @flags: Interrupt type flags (IRQF_TIMER only)
  2099. * @devname: An ascii name for the claiming device
  2100. * @dev_id: A percpu cookie passed back to the handler function
  2101. *
  2102. * This call allocates interrupt resources and enables the
  2103. * interrupt on the local CPU. If the interrupt is supposed to be
  2104. * enabled on other CPUs, it has to be done on each CPU using
  2105. * enable_percpu_irq().
  2106. *
  2107. * Dev_id must be globally unique. It is a per-cpu variable, and
  2108. * the handler gets called with the interrupted CPU's instance of
  2109. * that variable.
  2110. */
  2111. int __request_percpu_irq(unsigned int irq, irq_handler_t handler,
  2112. unsigned long flags, const char *devname,
  2113. void __percpu *dev_id)
  2114. {
  2115. struct irqaction *action;
  2116. struct irq_desc *desc;
  2117. int retval;
  2118. if (!dev_id)
  2119. return -EINVAL;
  2120. desc = irq_to_desc(irq);
  2121. if (!desc || !irq_settings_can_request(desc) ||
  2122. !irq_settings_is_per_cpu_devid(desc))
  2123. return -EINVAL;
  2124. if (flags && flags != IRQF_TIMER)
  2125. return -EINVAL;
  2126. action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
  2127. if (!action)
  2128. return -ENOMEM;
  2129. action->handler = handler;
  2130. action->flags = flags | IRQF_PERCPU | IRQF_NO_SUSPEND;
  2131. action->name = devname;
  2132. action->percpu_dev_id = dev_id;
  2133. retval = irq_chip_pm_get(&desc->irq_data);
  2134. if (retval < 0) {
  2135. kfree(action);
  2136. return retval;
  2137. }
  2138. retval = __setup_irq(irq, desc, action);
  2139. if (retval) {
  2140. irq_chip_pm_put(&desc->irq_data);
  2141. kfree(action);
  2142. }
  2143. return retval;
  2144. }
  2145. EXPORT_SYMBOL_GPL(__request_percpu_irq);
  2146. /**
  2147. * request_percpu_nmi - allocate a percpu interrupt line for NMI delivery
  2148. * @irq: Interrupt line to allocate
  2149. * @handler: Function to be called when the IRQ occurs.
  2150. * @name: An ascii name for the claiming device
  2151. * @dev_id: A percpu cookie passed back to the handler function
  2152. *
  2153. * This call allocates interrupt resources for a per CPU NMI. Per CPU NMIs
  2154. * have to be setup on each CPU by calling prepare_percpu_nmi() before
  2155. * being enabled on the same CPU by using enable_percpu_nmi().
  2156. *
  2157. * Dev_id must be globally unique. It is a per-cpu variable, and
  2158. * the handler gets called with the interrupted CPU's instance of
  2159. * that variable.
  2160. *
  2161. * Interrupt lines requested for NMI delivering should have auto enabling
  2162. * setting disabled.
  2163. *
  2164. * If the interrupt line cannot be used to deliver NMIs, function
  2165. * will fail returning a negative value.
  2166. */
  2167. int request_percpu_nmi(unsigned int irq, irq_handler_t handler,
  2168. const char *name, void __percpu *dev_id)
  2169. {
  2170. struct irqaction *action;
  2171. struct irq_desc *desc;
  2172. unsigned long flags;
  2173. int retval;
  2174. if (!handler)
  2175. return -EINVAL;
  2176. desc = irq_to_desc(irq);
  2177. if (!desc || !irq_settings_can_request(desc) ||
  2178. !irq_settings_is_per_cpu_devid(desc) ||
  2179. irq_settings_can_autoenable(desc) ||
  2180. !irq_supports_nmi(desc))
  2181. return -EINVAL;
  2182. /* The line cannot already be NMI */
  2183. if (desc->istate & IRQS_NMI)
  2184. return -EINVAL;
  2185. action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
  2186. if (!action)
  2187. return -ENOMEM;
  2188. action->handler = handler;
  2189. action->flags = IRQF_PERCPU | IRQF_NO_SUSPEND | IRQF_NO_THREAD
  2190. | IRQF_NOBALANCING;
  2191. action->name = name;
  2192. action->percpu_dev_id = dev_id;
  2193. retval = irq_chip_pm_get(&desc->irq_data);
  2194. if (retval < 0)
  2195. goto err_out;
  2196. retval = __setup_irq(irq, desc, action);
  2197. if (retval)
  2198. goto err_irq_setup;
  2199. raw_spin_lock_irqsave(&desc->lock, flags);
  2200. desc->istate |= IRQS_NMI;
  2201. raw_spin_unlock_irqrestore(&desc->lock, flags);
  2202. return 0;
  2203. err_irq_setup:
  2204. irq_chip_pm_put(&desc->irq_data);
  2205. err_out:
  2206. kfree(action);
  2207. return retval;
  2208. }
  2209. /**
  2210. * prepare_percpu_nmi - performs CPU local setup for NMI delivery
  2211. * @irq: Interrupt line to prepare for NMI delivery
  2212. *
  2213. * This call prepares an interrupt line to deliver NMI on the current CPU,
  2214. * before that interrupt line gets enabled with enable_percpu_nmi().
  2215. *
  2216. * As a CPU local operation, this should be called from non-preemptible
  2217. * context.
  2218. *
  2219. * If the interrupt line cannot be used to deliver NMIs, function
  2220. * will fail returning a negative value.
  2221. */
  2222. int prepare_percpu_nmi(unsigned int irq)
  2223. {
  2224. unsigned long flags;
  2225. struct irq_desc *desc;
  2226. int ret = 0;
  2227. WARN_ON(preemptible());
  2228. desc = irq_get_desc_lock(irq, &flags,
  2229. IRQ_GET_DESC_CHECK_PERCPU);
  2230. if (!desc)
  2231. return -EINVAL;
  2232. if (WARN(!(desc->istate & IRQS_NMI),
  2233. KERN_ERR "prepare_percpu_nmi called for a non-NMI interrupt: irq %u\n",
  2234. irq)) {
  2235. ret = -EINVAL;
  2236. goto out;
  2237. }
  2238. ret = irq_nmi_setup(desc);
  2239. if (ret) {
  2240. pr_err("Failed to setup NMI delivery: irq %u\n", irq);
  2241. goto out;
  2242. }
  2243. out:
  2244. irq_put_desc_unlock(desc, flags);
  2245. return ret;
  2246. }
  2247. /**
  2248. * teardown_percpu_nmi - undoes NMI setup of IRQ line
  2249. * @irq: Interrupt line from which CPU local NMI configuration should be
  2250. * removed
  2251. *
  2252. * This call undoes the setup done by prepare_percpu_nmi().
  2253. *
  2254. * IRQ line should not be enabled for the current CPU.
  2255. *
  2256. * As a CPU local operation, this should be called from non-preemptible
  2257. * context.
  2258. */
  2259. void teardown_percpu_nmi(unsigned int irq)
  2260. {
  2261. unsigned long flags;
  2262. struct irq_desc *desc;
  2263. WARN_ON(preemptible());
  2264. desc = irq_get_desc_lock(irq, &flags,
  2265. IRQ_GET_DESC_CHECK_PERCPU);
  2266. if (!desc)
  2267. return;
  2268. if (WARN_ON(!(desc->istate & IRQS_NMI)))
  2269. goto out;
  2270. irq_nmi_teardown(desc);
  2271. out:
  2272. irq_put_desc_unlock(desc, flags);
  2273. }
  2274. int __irq_get_irqchip_state(struct irq_data *data, enum irqchip_irq_state which,
  2275. bool *state)
  2276. {
  2277. struct irq_chip *chip;
  2278. int err = -EINVAL;
  2279. do {
  2280. chip = irq_data_get_irq_chip(data);
  2281. if (WARN_ON_ONCE(!chip))
  2282. return -ENODEV;
  2283. if (chip->irq_get_irqchip_state)
  2284. break;
  2285. #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
  2286. data = data->parent_data;
  2287. #else
  2288. data = NULL;
  2289. #endif
  2290. } while (data);
  2291. if (data)
  2292. err = chip->irq_get_irqchip_state(data, which, state);
  2293. return err;
  2294. }
  2295. /**
  2296. * irq_get_irqchip_state - returns the irqchip state of a interrupt.
  2297. * @irq: Interrupt line that is forwarded to a VM
  2298. * @which: One of IRQCHIP_STATE_* the caller wants to know about
  2299. * @state: a pointer to a boolean where the state is to be storeed
  2300. *
  2301. * This call snapshots the internal irqchip state of an
  2302. * interrupt, returning into @state the bit corresponding to
  2303. * stage @which
  2304. *
  2305. * This function should be called with preemption disabled if the
  2306. * interrupt controller has per-cpu registers.
  2307. */
  2308. int irq_get_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
  2309. bool *state)
  2310. {
  2311. struct irq_desc *desc;
  2312. struct irq_data *data;
  2313. unsigned long flags;
  2314. int err = -EINVAL;
  2315. desc = irq_get_desc_buslock(irq, &flags, 0);
  2316. if (!desc)
  2317. return err;
  2318. data = irq_desc_get_irq_data(desc);
  2319. err = __irq_get_irqchip_state(data, which, state);
  2320. irq_put_desc_busunlock(desc, flags);
  2321. return err;
  2322. }
  2323. EXPORT_SYMBOL_GPL(irq_get_irqchip_state);
  2324. /**
  2325. * irq_set_irqchip_state - set the state of a forwarded interrupt.
  2326. * @irq: Interrupt line that is forwarded to a VM
  2327. * @which: State to be restored (one of IRQCHIP_STATE_*)
  2328. * @val: Value corresponding to @which
  2329. *
  2330. * This call sets the internal irqchip state of an interrupt,
  2331. * depending on the value of @which.
  2332. *
  2333. * This function should be called with preemption disabled if the
  2334. * interrupt controller has per-cpu registers.
  2335. */
  2336. int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
  2337. bool val)
  2338. {
  2339. struct irq_desc *desc;
  2340. struct irq_data *data;
  2341. struct irq_chip *chip;
  2342. unsigned long flags;
  2343. int err = -EINVAL;
  2344. desc = irq_get_desc_buslock(irq, &flags, 0);
  2345. if (!desc)
  2346. return err;
  2347. data = irq_desc_get_irq_data(desc);
  2348. do {
  2349. chip = irq_data_get_irq_chip(data);
  2350. if (WARN_ON_ONCE(!chip)) {
  2351. err = -ENODEV;
  2352. goto out_unlock;
  2353. }
  2354. if (chip->irq_set_irqchip_state)
  2355. break;
  2356. #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
  2357. data = data->parent_data;
  2358. #else
  2359. data = NULL;
  2360. #endif
  2361. } while (data);
  2362. if (data)
  2363. err = chip->irq_set_irqchip_state(data, which, val);
  2364. out_unlock:
  2365. irq_put_desc_busunlock(desc, flags);
  2366. return err;
  2367. }
  2368. EXPORT_SYMBOL_GPL(irq_set_irqchip_state);