internals.h 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * IRQ subsystem internal functions and variables:
  4. *
  5. * Do not ever include this file from anything else than
  6. * kernel/irq/. Do not even think about using any information outside
  7. * of this file for your non core code.
  8. */
  9. #include <linux/irqdesc.h>
  10. #include <linux/kernel_stat.h>
  11. #include <linux/pm_runtime.h>
  12. #include <linux/sched/clock.h>
  13. #ifdef CONFIG_SPARSE_IRQ
  14. # define IRQ_BITMAP_BITS (NR_IRQS + 8196)
  15. #else
  16. # define IRQ_BITMAP_BITS NR_IRQS
  17. #endif
  18. #define istate core_internal_state__do_not_mess_with_it
  19. extern bool noirqdebug;
  20. extern struct irqaction chained_action;
  21. /*
  22. * Bits used by threaded handlers:
  23. * IRQTF_RUNTHREAD - signals that the interrupt handler thread should run
  24. * IRQTF_WARNED - warning "IRQ_WAKE_THREAD w/o thread_fn" has been printed
  25. * IRQTF_AFFINITY - irq thread is requested to adjust affinity
  26. * IRQTF_FORCED_THREAD - irq action is force threaded
  27. */
  28. enum {
  29. IRQTF_RUNTHREAD,
  30. IRQTF_WARNED,
  31. IRQTF_AFFINITY,
  32. IRQTF_FORCED_THREAD,
  33. };
  34. /*
  35. * Bit masks for desc->core_internal_state__do_not_mess_with_it
  36. *
  37. * IRQS_AUTODETECT - autodetection in progress
  38. * IRQS_SPURIOUS_DISABLED - was disabled due to spurious interrupt
  39. * detection
  40. * IRQS_POLL_INPROGRESS - polling in progress
  41. * IRQS_ONESHOT - irq is not unmasked in primary handler
  42. * IRQS_REPLAY - irq is replayed
  43. * IRQS_WAITING - irq is waiting
  44. * IRQS_PENDING - irq is pending and replayed later
  45. * IRQS_SUSPENDED - irq is suspended
  46. * IRQS_NMI - irq line is used to deliver NMIs
  47. */
  48. enum {
  49. IRQS_AUTODETECT = 0x00000001,
  50. IRQS_SPURIOUS_DISABLED = 0x00000002,
  51. IRQS_POLL_INPROGRESS = 0x00000008,
  52. IRQS_ONESHOT = 0x00000020,
  53. IRQS_REPLAY = 0x00000040,
  54. IRQS_WAITING = 0x00000080,
  55. IRQS_PENDING = 0x00000200,
  56. IRQS_SUSPENDED = 0x00000800,
  57. IRQS_TIMINGS = 0x00001000,
  58. IRQS_NMI = 0x00002000,
  59. };
  60. #include "debug.h"
  61. #include "settings.h"
  62. extern int __irq_set_trigger(struct irq_desc *desc, unsigned long flags);
  63. extern void __disable_irq(struct irq_desc *desc);
  64. extern void __enable_irq(struct irq_desc *desc);
  65. #define IRQ_RESEND true
  66. #define IRQ_NORESEND false
  67. #define IRQ_START_FORCE true
  68. #define IRQ_START_COND false
  69. extern int irq_activate(struct irq_desc *desc);
  70. extern int irq_activate_and_startup(struct irq_desc *desc, bool resend);
  71. extern int irq_startup(struct irq_desc *desc, bool resend, bool force);
  72. extern void irq_shutdown(struct irq_desc *desc);
  73. extern void irq_shutdown_and_deactivate(struct irq_desc *desc);
  74. extern void irq_enable(struct irq_desc *desc);
  75. extern void irq_disable(struct irq_desc *desc);
  76. extern void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu);
  77. extern void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu);
  78. extern void mask_irq(struct irq_desc *desc);
  79. extern void unmask_irq(struct irq_desc *desc);
  80. extern void unmask_threaded_irq(struct irq_desc *desc);
  81. #ifdef CONFIG_SPARSE_IRQ
  82. static inline void irq_mark_irq(unsigned int irq) { }
  83. #else
  84. extern void irq_mark_irq(unsigned int irq);
  85. #endif
  86. extern int __irq_get_irqchip_state(struct irq_data *data,
  87. enum irqchip_irq_state which,
  88. bool *state);
  89. extern void init_kstat_irqs(struct irq_desc *desc, int node, int nr);
  90. irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc, unsigned int *flags);
  91. irqreturn_t handle_irq_event_percpu(struct irq_desc *desc);
  92. irqreturn_t handle_irq_event(struct irq_desc *desc);
  93. /* Resending of interrupts :*/
  94. int check_irq_resend(struct irq_desc *desc, bool inject);
  95. bool irq_wait_for_poll(struct irq_desc *desc);
  96. void __irq_wake_thread(struct irq_desc *desc, struct irqaction *action);
  97. #ifdef CONFIG_PROC_FS
  98. extern void register_irq_proc(unsigned int irq, struct irq_desc *desc);
  99. extern void unregister_irq_proc(unsigned int irq, struct irq_desc *desc);
  100. extern void register_handler_proc(unsigned int irq, struct irqaction *action);
  101. extern void unregister_handler_proc(unsigned int irq, struct irqaction *action);
  102. #else
  103. static inline void register_irq_proc(unsigned int irq, struct irq_desc *desc) { }
  104. static inline void unregister_irq_proc(unsigned int irq, struct irq_desc *desc) { }
  105. static inline void register_handler_proc(unsigned int irq,
  106. struct irqaction *action) { }
  107. static inline void unregister_handler_proc(unsigned int irq,
  108. struct irqaction *action) { }
  109. #endif
  110. extern bool irq_can_set_affinity_usr(unsigned int irq);
  111. extern void irq_set_thread_affinity(struct irq_desc *desc);
  112. extern int irq_do_set_affinity(struct irq_data *data,
  113. const struct cpumask *dest, bool force);
  114. #ifdef CONFIG_SMP
  115. extern int irq_setup_affinity(struct irq_desc *desc);
  116. #else
  117. static inline int irq_setup_affinity(struct irq_desc *desc) { return 0; }
  118. #endif
  119. /* Inline functions for support of irq chips on slow busses */
  120. static inline void chip_bus_lock(struct irq_desc *desc)
  121. {
  122. if (unlikely(desc->irq_data.chip->irq_bus_lock))
  123. desc->irq_data.chip->irq_bus_lock(&desc->irq_data);
  124. }
  125. static inline void chip_bus_sync_unlock(struct irq_desc *desc)
  126. {
  127. if (unlikely(desc->irq_data.chip->irq_bus_sync_unlock))
  128. desc->irq_data.chip->irq_bus_sync_unlock(&desc->irq_data);
  129. }
  130. #define _IRQ_DESC_CHECK (1 << 0)
  131. #define _IRQ_DESC_PERCPU (1 << 1)
  132. #define IRQ_GET_DESC_CHECK_GLOBAL (_IRQ_DESC_CHECK)
  133. #define IRQ_GET_DESC_CHECK_PERCPU (_IRQ_DESC_CHECK | _IRQ_DESC_PERCPU)
  134. #define for_each_action_of_desc(desc, act) \
  135. for (act = desc->action; act; act = act->next)
  136. struct irq_desc *
  137. __irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
  138. unsigned int check);
  139. void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus);
  140. static inline struct irq_desc *
  141. irq_get_desc_buslock(unsigned int irq, unsigned long *flags, unsigned int check)
  142. {
  143. return __irq_get_desc_lock(irq, flags, true, check);
  144. }
  145. static inline void
  146. irq_put_desc_busunlock(struct irq_desc *desc, unsigned long flags)
  147. {
  148. __irq_put_desc_unlock(desc, flags, true);
  149. }
  150. static inline struct irq_desc *
  151. irq_get_desc_lock(unsigned int irq, unsigned long *flags, unsigned int check)
  152. {
  153. return __irq_get_desc_lock(irq, flags, false, check);
  154. }
  155. static inline void
  156. irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags)
  157. {
  158. __irq_put_desc_unlock(desc, flags, false);
  159. }
  160. #define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors)
  161. static inline unsigned int irqd_get(struct irq_data *d)
  162. {
  163. return __irqd_to_state(d);
  164. }
  165. /*
  166. * Manipulation functions for irq_data.state
  167. */
  168. static inline void irqd_set_move_pending(struct irq_data *d)
  169. {
  170. __irqd_to_state(d) |= IRQD_SETAFFINITY_PENDING;
  171. }
  172. static inline void irqd_clr_move_pending(struct irq_data *d)
  173. {
  174. __irqd_to_state(d) &= ~IRQD_SETAFFINITY_PENDING;
  175. }
  176. static inline void irqd_set_managed_shutdown(struct irq_data *d)
  177. {
  178. __irqd_to_state(d) |= IRQD_MANAGED_SHUTDOWN;
  179. }
  180. static inline void irqd_clr_managed_shutdown(struct irq_data *d)
  181. {
  182. __irqd_to_state(d) &= ~IRQD_MANAGED_SHUTDOWN;
  183. }
  184. static inline void irqd_clear(struct irq_data *d, unsigned int mask)
  185. {
  186. __irqd_to_state(d) &= ~mask;
  187. }
  188. static inline void irqd_set(struct irq_data *d, unsigned int mask)
  189. {
  190. __irqd_to_state(d) |= mask;
  191. }
  192. static inline bool irqd_has_set(struct irq_data *d, unsigned int mask)
  193. {
  194. return __irqd_to_state(d) & mask;
  195. }
  196. static inline void irq_state_set_disabled(struct irq_desc *desc)
  197. {
  198. irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
  199. }
  200. static inline void irq_state_set_masked(struct irq_desc *desc)
  201. {
  202. irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);
  203. }
  204. #undef __irqd_to_state
  205. static inline void __kstat_incr_irqs_this_cpu(struct irq_desc *desc)
  206. {
  207. __this_cpu_inc(*desc->kstat_irqs);
  208. __this_cpu_inc(kstat.irqs_sum);
  209. }
  210. static inline void kstat_incr_irqs_this_cpu(struct irq_desc *desc)
  211. {
  212. __kstat_incr_irqs_this_cpu(desc);
  213. desc->tot_count++;
  214. }
  215. static inline int irq_desc_get_node(struct irq_desc *desc)
  216. {
  217. return irq_common_data_get_node(&desc->irq_common_data);
  218. }
  219. static inline int irq_desc_is_chained(struct irq_desc *desc)
  220. {
  221. return (desc->action && desc->action == &chained_action);
  222. }
  223. #ifdef CONFIG_PM_SLEEP
  224. bool irq_pm_check_wakeup(struct irq_desc *desc);
  225. void irq_pm_install_action(struct irq_desc *desc, struct irqaction *action);
  226. void irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action);
  227. #else
  228. static inline bool irq_pm_check_wakeup(struct irq_desc *desc) { return false; }
  229. static inline void
  230. irq_pm_install_action(struct irq_desc *desc, struct irqaction *action) { }
  231. static inline void
  232. irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action) { }
  233. #endif
  234. #ifdef CONFIG_IRQ_TIMINGS
  235. #define IRQ_TIMINGS_SHIFT 5
  236. #define IRQ_TIMINGS_SIZE (1 << IRQ_TIMINGS_SHIFT)
  237. #define IRQ_TIMINGS_MASK (IRQ_TIMINGS_SIZE - 1)
  238. /**
  239. * struct irq_timings - irq timings storing structure
  240. * @values: a circular buffer of u64 encoded <timestamp,irq> values
  241. * @count: the number of elements in the array
  242. */
  243. struct irq_timings {
  244. u64 values[IRQ_TIMINGS_SIZE];
  245. int count;
  246. };
  247. DECLARE_PER_CPU(struct irq_timings, irq_timings);
  248. extern void irq_timings_free(int irq);
  249. extern int irq_timings_alloc(int irq);
  250. static inline void irq_remove_timings(struct irq_desc *desc)
  251. {
  252. desc->istate &= ~IRQS_TIMINGS;
  253. irq_timings_free(irq_desc_get_irq(desc));
  254. }
  255. static inline void irq_setup_timings(struct irq_desc *desc, struct irqaction *act)
  256. {
  257. int irq = irq_desc_get_irq(desc);
  258. int ret;
  259. /*
  260. * We don't need the measurement because the idle code already
  261. * knows the next expiry event.
  262. */
  263. if (act->flags & __IRQF_TIMER)
  264. return;
  265. /*
  266. * In case the timing allocation fails, we just want to warn,
  267. * not fail, so letting the system boot anyway.
  268. */
  269. ret = irq_timings_alloc(irq);
  270. if (ret) {
  271. pr_warn("Failed to allocate irq timing stats for irq%d (%d)",
  272. irq, ret);
  273. return;
  274. }
  275. desc->istate |= IRQS_TIMINGS;
  276. }
  277. extern void irq_timings_enable(void);
  278. extern void irq_timings_disable(void);
  279. DECLARE_STATIC_KEY_FALSE(irq_timing_enabled);
  280. /*
  281. * The interrupt number and the timestamp are encoded into a single
  282. * u64 variable to optimize the size.
  283. * 48 bit time stamp and 16 bit IRQ number is way sufficient.
  284. * Who cares an IRQ after 78 hours of idle time?
  285. */
  286. static inline u64 irq_timing_encode(u64 timestamp, int irq)
  287. {
  288. return (timestamp << 16) | irq;
  289. }
  290. static inline int irq_timing_decode(u64 value, u64 *timestamp)
  291. {
  292. *timestamp = value >> 16;
  293. return value & U16_MAX;
  294. }
  295. static __always_inline void irq_timings_push(u64 ts, int irq)
  296. {
  297. struct irq_timings *timings = this_cpu_ptr(&irq_timings);
  298. timings->values[timings->count & IRQ_TIMINGS_MASK] =
  299. irq_timing_encode(ts, irq);
  300. timings->count++;
  301. }
  302. /*
  303. * The function record_irq_time is only called in one place in the
  304. * interrupts handler. We want this function always inline so the code
  305. * inside is embedded in the function and the static key branching
  306. * code can act at the higher level. Without the explicit
  307. * __always_inline we can end up with a function call and a small
  308. * overhead in the hotpath for nothing.
  309. */
  310. static __always_inline void record_irq_time(struct irq_desc *desc)
  311. {
  312. if (!static_branch_likely(&irq_timing_enabled))
  313. return;
  314. if (desc->istate & IRQS_TIMINGS)
  315. irq_timings_push(local_clock(), irq_desc_get_irq(desc));
  316. }
  317. #else
  318. static inline void irq_remove_timings(struct irq_desc *desc) {}
  319. static inline void irq_setup_timings(struct irq_desc *desc,
  320. struct irqaction *act) {};
  321. static inline void record_irq_time(struct irq_desc *desc) {}
  322. #endif /* CONFIG_IRQ_TIMINGS */
  323. #ifdef CONFIG_GENERIC_IRQ_CHIP
  324. void irq_init_generic_chip(struct irq_chip_generic *gc, const char *name,
  325. int num_ct, unsigned int irq_base,
  326. void __iomem *reg_base, irq_flow_handler_t handler);
  327. #else
  328. static inline void
  329. irq_init_generic_chip(struct irq_chip_generic *gc, const char *name,
  330. int num_ct, unsigned int irq_base,
  331. void __iomem *reg_base, irq_flow_handler_t handler) { }
  332. #endif /* CONFIG_GENERIC_IRQ_CHIP */
  333. #ifdef CONFIG_GENERIC_PENDING_IRQ
  334. static inline bool irq_can_move_pcntxt(struct irq_data *data)
  335. {
  336. return irqd_can_move_in_process_context(data);
  337. }
  338. static inline bool irq_move_pending(struct irq_data *data)
  339. {
  340. return irqd_is_setaffinity_pending(data);
  341. }
  342. static inline void
  343. irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
  344. {
  345. cpumask_copy(desc->pending_mask, mask);
  346. }
  347. static inline void
  348. irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
  349. {
  350. cpumask_copy(mask, desc->pending_mask);
  351. }
  352. static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc)
  353. {
  354. return desc->pending_mask;
  355. }
  356. static inline bool handle_enforce_irqctx(struct irq_data *data)
  357. {
  358. return irqd_is_handle_enforce_irqctx(data);
  359. }
  360. bool irq_fixup_move_pending(struct irq_desc *desc, bool force_clear);
  361. #else /* CONFIG_GENERIC_PENDING_IRQ */
  362. static inline bool irq_can_move_pcntxt(struct irq_data *data)
  363. {
  364. return true;
  365. }
  366. static inline bool irq_move_pending(struct irq_data *data)
  367. {
  368. return false;
  369. }
  370. static inline void
  371. irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
  372. {
  373. }
  374. static inline void
  375. irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
  376. {
  377. }
  378. static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc)
  379. {
  380. return NULL;
  381. }
  382. static inline bool irq_fixup_move_pending(struct irq_desc *desc, bool fclear)
  383. {
  384. return false;
  385. }
  386. static inline bool handle_enforce_irqctx(struct irq_data *data)
  387. {
  388. return false;
  389. }
  390. #endif /* !CONFIG_GENERIC_PENDING_IRQ */
  391. #if !defined(CONFIG_IRQ_DOMAIN) || !defined(CONFIG_IRQ_DOMAIN_HIERARCHY)
  392. static inline int irq_domain_activate_irq(struct irq_data *data, bool reserve)
  393. {
  394. irqd_set_activated(data);
  395. return 0;
  396. }
  397. static inline void irq_domain_deactivate_irq(struct irq_data *data)
  398. {
  399. irqd_clr_activated(data);
  400. }
  401. #endif
  402. static inline struct irq_data *irqd_get_parent_data(struct irq_data *irqd)
  403. {
  404. #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
  405. return irqd->parent_data;
  406. #else
  407. return NULL;
  408. #endif
  409. }
  410. #ifdef CONFIG_GENERIC_IRQ_DEBUGFS
  411. #include <linux/debugfs.h>
  412. void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *desc);
  413. static inline void irq_remove_debugfs_entry(struct irq_desc *desc)
  414. {
  415. debugfs_remove(desc->debugfs_file);
  416. kfree(desc->dev_name);
  417. }
  418. void irq_debugfs_copy_devname(int irq, struct device *dev);
  419. # ifdef CONFIG_IRQ_DOMAIN
  420. void irq_domain_debugfs_init(struct dentry *root);
  421. # else
  422. static inline void irq_domain_debugfs_init(struct dentry *root)
  423. {
  424. }
  425. # endif
  426. #else /* CONFIG_GENERIC_IRQ_DEBUGFS */
  427. static inline void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *d)
  428. {
  429. }
  430. static inline void irq_remove_debugfs_entry(struct irq_desc *d)
  431. {
  432. }
  433. static inline void irq_debugfs_copy_devname(int irq, struct device *dev)
  434. {
  435. }
  436. #endif /* CONFIG_GENERIC_IRQ_DEBUGFS */