123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777 |
- // SPDX-License-Identifier: GPL-2.0-only
- /*
- * linux/kernel/softirq.c
- *
- * Copyright (C) 1992 Linus Torvalds
- *
- * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
- */
- #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
- #include <linux/export.h>
- #include <linux/kernel_stat.h>
- #include <linux/interrupt.h>
- #include <linux/init.h>
- #include <linux/mm.h>
- #include <linux/notifier.h>
- #include <linux/percpu.h>
- #include <linux/cpu.h>
- #include <linux/freezer.h>
- #include <linux/kthread.h>
- #include <linux/rcupdate.h>
- #include <linux/ftrace.h>
- #include <linux/smp.h>
- #include <linux/smpboot.h>
- #include <linux/tick.h>
- #include <linux/irq.h>
- #define CREATE_TRACE_POINTS
- #include <trace/events/irq.h>
- EXPORT_TRACEPOINT_SYMBOL_GPL(irq_handler_entry);
- EXPORT_TRACEPOINT_SYMBOL_GPL(irq_handler_exit);
- /*
- - No shared variables, all the data are CPU local.
- - If a softirq needs serialization, let it serialize itself
- by its own spinlocks.
- - Even if softirq is serialized, only local cpu is marked for
- execution. Hence, we get something sort of weak cpu binding.
- Though it is still not clear, will it result in better locality
- or will not.
- Examples:
- - NET RX softirq. It is multithreaded and does not require
- any global serialization.
- - NET TX softirq. It kicks software netdevice queues, hence
- it is logically serialized per device, but this serialization
- is invisible to common code.
- - Tasklets: serialized wrt itself.
- */
- #ifndef __ARCH_IRQ_STAT
- DEFINE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat);
- EXPORT_PER_CPU_SYMBOL(irq_stat);
- #endif
- static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
- DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
- EXPORT_PER_CPU_SYMBOL_GPL(ksoftirqd);
- /*
- * active_softirqs -- per cpu, a mask of softirqs that are being handled,
- * with the expectation that approximate answers are acceptable and therefore
- * no synchronization.
- */
- DEFINE_PER_CPU(__u32, active_softirqs);
- const char * const softirq_to_name[NR_SOFTIRQS] = {
- "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
- "TASKLET", "SCHED", "HRTIMER", "RCU"
- };
- /*
- * we cannot loop indefinitely here to avoid userspace starvation,
- * but we also don't want to introduce a worst case 1/HZ latency
- * to the pending events, so lets the scheduler to balance
- * the softirq load for us.
- */
- static void wakeup_softirqd(void)
- {
- /* Interrupts are disabled: no need to stop preemption */
- struct task_struct *tsk = __this_cpu_read(ksoftirqd);
- if (tsk && tsk->state != TASK_RUNNING)
- wake_up_process(tsk);
- }
- /*
- * preempt_count and SOFTIRQ_OFFSET usage:
- * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
- * softirq processing.
- * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
- * on local_bh_disable or local_bh_enable.
- * This lets us distinguish between whether we are currently processing
- * softirq and whether we just have bh disabled.
- */
- /*
- * This one is for softirq.c-internal use,
- * where hardirqs are disabled legitimately:
- */
- #ifdef CONFIG_TRACE_IRQFLAGS
- DEFINE_PER_CPU(int, hardirqs_enabled);
- DEFINE_PER_CPU(int, hardirq_context);
- EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
- EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
- void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
- {
- unsigned long flags;
- WARN_ON_ONCE(in_irq());
- raw_local_irq_save(flags);
- /*
- * The preempt tracer hooks into preempt_count_add and will break
- * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
- * is set and before current->softirq_enabled is cleared.
- * We must manually increment preempt_count here and manually
- * call the trace_preempt_off later.
- */
- __preempt_count_add(cnt);
- /*
- * Were softirqs turned off above:
- */
- if (softirq_count() == (cnt & SOFTIRQ_MASK))
- lockdep_softirqs_off(ip);
- raw_local_irq_restore(flags);
- if (preempt_count() == cnt) {
- #ifdef CONFIG_DEBUG_PREEMPT
- current->preempt_disable_ip = get_lock_parent_ip();
- #endif
- trace_preempt_off(CALLER_ADDR0, get_lock_parent_ip());
- }
- }
- EXPORT_SYMBOL(__local_bh_disable_ip);
- #endif /* CONFIG_TRACE_IRQFLAGS */
- static void __local_bh_enable(unsigned int cnt)
- {
- lockdep_assert_irqs_disabled();
- if (preempt_count() == cnt)
- trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
- if (softirq_count() == (cnt & SOFTIRQ_MASK))
- lockdep_softirqs_on(_RET_IP_);
- __preempt_count_sub(cnt);
- }
- /*
- * Special-case - softirqs can safely be enabled by __do_softirq(),
- * without processing still-pending softirqs:
- */
- void _local_bh_enable(void)
- {
- WARN_ON_ONCE(in_irq());
- __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
- }
- EXPORT_SYMBOL(_local_bh_enable);
- void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
- {
- WARN_ON_ONCE(in_irq());
- lockdep_assert_irqs_enabled();
- #ifdef CONFIG_TRACE_IRQFLAGS
- local_irq_disable();
- #endif
- /*
- * Are softirqs going to be turned on now:
- */
- if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
- lockdep_softirqs_on(ip);
- /*
- * Keep preemption disabled until we are done with
- * softirq processing:
- */
- preempt_count_sub(cnt - 1);
- if (unlikely(!in_interrupt() && local_softirq_pending())) {
- /*
- * Run softirq if any pending. And do it in its own stack
- * as we may be calling this deep in a task call stack already.
- */
- do_softirq();
- }
- preempt_count_dec();
- #ifdef CONFIG_TRACE_IRQFLAGS
- local_irq_enable();
- #endif
- preempt_check_resched();
- }
- EXPORT_SYMBOL(__local_bh_enable_ip);
- /*
- * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
- * but break the loop if need_resched() is set or after 2 ms.
- * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
- * certain cases, such as stop_machine(), jiffies may cease to
- * increment and so we need the MAX_SOFTIRQ_RESTART limit as
- * well to make sure we eventually return from this method.
- *
- * These limits have been established via experimentation.
- * The two things to balance is latency against fairness -
- * we want to handle softirqs as soon as possible, but they
- * should not be able to lock up the box.
- */
- #define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
- #define MAX_SOFTIRQ_RESTART 10
- #ifdef CONFIG_TRACE_IRQFLAGS
- /*
- * When we run softirqs from irq_exit() and thus on the hardirq stack we need
- * to keep the lockdep irq context tracking as tight as possible in order to
- * not miss-qualify lock contexts and miss possible deadlocks.
- */
- static inline bool lockdep_softirq_start(void)
- {
- bool in_hardirq = false;
- if (lockdep_hardirq_context()) {
- in_hardirq = true;
- lockdep_hardirq_exit();
- }
- lockdep_softirq_enter();
- return in_hardirq;
- }
- static inline void lockdep_softirq_end(bool in_hardirq)
- {
- lockdep_softirq_exit();
- if (in_hardirq)
- lockdep_hardirq_enter();
- }
- #else
- static inline bool lockdep_softirq_start(void) { return false; }
- static inline void lockdep_softirq_end(bool in_hardirq) { }
- #endif
- #define softirq_deferred_for_rt(pending) \
- ({ \
- __u32 deferred = 0; \
- if (cpupri_check_rt()) { \
- deferred = pending & LONG_SOFTIRQ_MASK; \
- pending &= ~LONG_SOFTIRQ_MASK; \
- } \
- deferred; \
- })
- asmlinkage __visible void __softirq_entry __do_softirq(void)
- {
- unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
- unsigned long old_flags = current->flags;
- int max_restart = MAX_SOFTIRQ_RESTART;
- struct softirq_action *h;
- bool in_hardirq;
- __u32 deferred;
- __u32 pending;
- int softirq_bit;
- /*
- * Mask out PF_MEMALLOC as the current task context is borrowed for the
- * softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
- * again if the socket is related to swapping.
- */
- current->flags &= ~PF_MEMALLOC;
- pending = local_softirq_pending();
- deferred = softirq_deferred_for_rt(pending);
- account_irq_enter_time(current);
- __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
- in_hardirq = lockdep_softirq_start();
- restart:
- /* Reset the pending bitmask before enabling irqs */
- set_softirq_pending(deferred);
- __this_cpu_write(active_softirqs, pending);
- local_irq_enable();
- h = softirq_vec;
- while ((softirq_bit = ffs(pending))) {
- unsigned int vec_nr;
- int prev_count;
- h += softirq_bit - 1;
- vec_nr = h - softirq_vec;
- prev_count = preempt_count();
- kstat_incr_softirqs_this_cpu(vec_nr);
- trace_softirq_entry(vec_nr);
- h->action(h);
- trace_softirq_exit(vec_nr);
- if (unlikely(prev_count != preempt_count())) {
- pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
- vec_nr, softirq_to_name[vec_nr], h->action,
- prev_count, preempt_count());
- preempt_count_set(prev_count);
- }
- h++;
- pending >>= softirq_bit;
- }
- __this_cpu_write(active_softirqs, 0);
- if (__this_cpu_read(ksoftirqd) == current)
- rcu_softirq_qs();
- local_irq_disable();
- pending = local_softirq_pending();
- deferred = softirq_deferred_for_rt(pending);
- if (pending) {
- if (time_before(jiffies, end) && !need_resched() &&
- --max_restart)
- goto restart;
- }
- #ifdef CONFIG_RT_SOFTINT_OPTIMIZATION
- if (pending | deferred)
- wakeup_softirqd();
- #endif
- lockdep_softirq_end(in_hardirq);
- account_irq_exit_time(current);
- __local_bh_enable(SOFTIRQ_OFFSET);
- WARN_ON_ONCE(in_interrupt());
- current_restore_flags(old_flags, PF_MEMALLOC);
- }
- asmlinkage __visible void do_softirq(void)
- {
- __u32 pending;
- unsigned long flags;
- if (in_interrupt())
- return;
- local_irq_save(flags);
- pending = local_softirq_pending();
- if (pending)
- do_softirq_own_stack();
- local_irq_restore(flags);
- }
- /**
- * irq_enter_rcu - Enter an interrupt context with RCU watching
- */
- void irq_enter_rcu(void)
- {
- if (is_idle_task(current) && !in_interrupt()) {
- /*
- * Prevent raise_softirq from needlessly waking up ksoftirqd
- * here, as softirq will be serviced on return from interrupt.
- */
- local_bh_disable();
- tick_irq_enter();
- _local_bh_enable();
- }
- __irq_enter();
- }
- /**
- * irq_enter - Enter an interrupt context including RCU update
- */
- void irq_enter(void)
- {
- rcu_irq_enter();
- irq_enter_rcu();
- }
- static inline void invoke_softirq(void)
- {
- if (!force_irqthreads) {
- #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
- /*
- * We can safely execute softirq on the current stack if
- * it is the irq stack, because it should be near empty
- * at this stage.
- */
- __do_softirq();
- #else
- /*
- * Otherwise, irq_exit() is called on the task stack that can
- * be potentially deep already. So call softirq in its own stack
- * to prevent from any overrun.
- */
- do_softirq_own_stack();
- #endif
- } else {
- wakeup_softirqd();
- }
- }
- static inline void tick_irq_exit(void)
- {
- #ifdef CONFIG_NO_HZ_COMMON
- int cpu = smp_processor_id();
- /* Make sure that timer wheel updates are propagated */
- if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
- if (!in_irq())
- tick_nohz_irq_exit();
- }
- #endif
- }
- static inline void __irq_exit_rcu(void)
- {
- #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
- local_irq_disable();
- #else
- lockdep_assert_irqs_disabled();
- #endif
- account_irq_exit_time(current);
- preempt_count_sub(HARDIRQ_OFFSET);
- if (!in_interrupt() && local_softirq_pending())
- invoke_softirq();
- tick_irq_exit();
- }
- /**
- * irq_exit_rcu() - Exit an interrupt context without updating RCU
- *
- * Also processes softirqs if needed and possible.
- */
- void irq_exit_rcu(void)
- {
- __irq_exit_rcu();
- /* must be last! */
- lockdep_hardirq_exit();
- }
- /**
- * irq_exit - Exit an interrupt context, update RCU and lockdep
- *
- * Also processes softirqs if needed and possible.
- */
- void irq_exit(void)
- {
- __irq_exit_rcu();
- rcu_irq_exit();
- /* must be last! */
- lockdep_hardirq_exit();
- }
- /*
- * This function must run with irqs disabled!
- */
- inline void raise_softirq_irqoff(unsigned int nr)
- {
- __raise_softirq_irqoff(nr);
- /*
- * If we're in an interrupt or softirq, we're done
- * (this also catches softirq-disabled code). We will
- * actually run the softirq once we return from
- * the irq or softirq.
- *
- * Otherwise we wake up ksoftirqd to make sure we
- * schedule the softirq soon.
- */
- if (!in_interrupt())
- wakeup_softirqd();
- }
- void raise_softirq(unsigned int nr)
- {
- unsigned long flags;
- local_irq_save(flags);
- raise_softirq_irqoff(nr);
- local_irq_restore(flags);
- }
- void __raise_softirq_irqoff(unsigned int nr)
- {
- lockdep_assert_irqs_disabled();
- trace_softirq_raise(nr);
- or_softirq_pending(1UL << nr);
- }
- void open_softirq(int nr, void (*action)(struct softirq_action *))
- {
- softirq_vec[nr].action = action;
- }
- /*
- * Tasklets
- */
- struct tasklet_head {
- struct tasklet_struct *head;
- struct tasklet_struct **tail;
- };
- static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
- static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
- static void __tasklet_schedule_common(struct tasklet_struct *t,
- struct tasklet_head __percpu *headp,
- unsigned int softirq_nr)
- {
- struct tasklet_head *head;
- unsigned long flags;
- local_irq_save(flags);
- head = this_cpu_ptr(headp);
- t->next = NULL;
- *head->tail = t;
- head->tail = &(t->next);
- raise_softirq_irqoff(softirq_nr);
- local_irq_restore(flags);
- }
- void __tasklet_schedule(struct tasklet_struct *t)
- {
- __tasklet_schedule_common(t, &tasklet_vec,
- TASKLET_SOFTIRQ);
- }
- EXPORT_SYMBOL(__tasklet_schedule);
- void __tasklet_hi_schedule(struct tasklet_struct *t)
- {
- __tasklet_schedule_common(t, &tasklet_hi_vec,
- HI_SOFTIRQ);
- }
- EXPORT_SYMBOL(__tasklet_hi_schedule);
- static void tasklet_action_common(struct softirq_action *a,
- struct tasklet_head *tl_head,
- unsigned int softirq_nr)
- {
- struct tasklet_struct *list;
- local_irq_disable();
- list = tl_head->head;
- tl_head->head = NULL;
- tl_head->tail = &tl_head->head;
- local_irq_enable();
- while (list) {
- struct tasklet_struct *t = list;
- list = list->next;
- if (tasklet_trylock(t)) {
- if (!atomic_read(&t->count)) {
- if (!test_and_clear_bit(TASKLET_STATE_SCHED,
- &t->state))
- BUG();
- if (t->use_callback) {
- trace_tasklet_entry(t->callback);
- t->callback(t);
- trace_tasklet_exit(t->callback);
- } else {
- trace_tasklet_entry(t->func);
- t->func(t->data);
- trace_tasklet_exit(t->func);
- }
- tasklet_unlock(t);
- continue;
- }
- tasklet_unlock(t);
- }
- local_irq_disable();
- t->next = NULL;
- *tl_head->tail = t;
- tl_head->tail = &t->next;
- __raise_softirq_irqoff(softirq_nr);
- local_irq_enable();
- }
- }
- static __latent_entropy void tasklet_action(struct softirq_action *a)
- {
- tasklet_action_common(a, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ);
- }
- static __latent_entropy void tasklet_hi_action(struct softirq_action *a)
- {
- tasklet_action_common(a, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ);
- }
- void tasklet_setup(struct tasklet_struct *t,
- void (*callback)(struct tasklet_struct *))
- {
- t->next = NULL;
- t->state = 0;
- atomic_set(&t->count, 0);
- t->callback = callback;
- t->use_callback = true;
- t->data = 0;
- }
- EXPORT_SYMBOL(tasklet_setup);
- void tasklet_init(struct tasklet_struct *t,
- void (*func)(unsigned long), unsigned long data)
- {
- t->next = NULL;
- t->state = 0;
- atomic_set(&t->count, 0);
- t->func = func;
- t->use_callback = false;
- t->data = data;
- }
- EXPORT_SYMBOL(tasklet_init);
- void tasklet_kill(struct tasklet_struct *t)
- {
- if (in_interrupt())
- pr_notice("Attempt to kill tasklet from interrupt\n");
- while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
- do {
- yield();
- } while (test_bit(TASKLET_STATE_SCHED, &t->state));
- }
- tasklet_unlock_wait(t);
- clear_bit(TASKLET_STATE_SCHED, &t->state);
- }
- EXPORT_SYMBOL(tasklet_kill);
- void __init softirq_init(void)
- {
- int cpu;
- for_each_possible_cpu(cpu) {
- per_cpu(tasklet_vec, cpu).tail =
- &per_cpu(tasklet_vec, cpu).head;
- per_cpu(tasklet_hi_vec, cpu).tail =
- &per_cpu(tasklet_hi_vec, cpu).head;
- }
- open_softirq(TASKLET_SOFTIRQ, tasklet_action);
- open_softirq(HI_SOFTIRQ, tasklet_hi_action);
- }
- static int ksoftirqd_should_run(unsigned int cpu)
- {
- return local_softirq_pending();
- }
- static void run_ksoftirqd(unsigned int cpu)
- {
- local_irq_disable();
- if (local_softirq_pending()) {
- /*
- * We can safely run softirq on inline stack, as we are not deep
- * in the task stack here.
- */
- __do_softirq();
- local_irq_enable();
- cond_resched();
- return;
- }
- local_irq_enable();
- }
- #ifdef CONFIG_HOTPLUG_CPU
- /*
- * tasklet_kill_immediate is called to remove a tasklet which can already be
- * scheduled for execution on @cpu.
- *
- * Unlike tasklet_kill, this function removes the tasklet
- * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
- *
- * When this function is called, @cpu must be in the CPU_DEAD state.
- */
- void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
- {
- struct tasklet_struct **i;
- BUG_ON(cpu_online(cpu));
- BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
- if (!test_bit(TASKLET_STATE_SCHED, &t->state))
- return;
- /* CPU is dead, so no lock needed. */
- for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
- if (*i == t) {
- *i = t->next;
- /* If this was the tail element, move the tail ptr */
- if (*i == NULL)
- per_cpu(tasklet_vec, cpu).tail = i;
- return;
- }
- }
- BUG();
- }
- static int takeover_tasklets(unsigned int cpu)
- {
- /* CPU is dead, so no lock needed. */
- local_irq_disable();
- /* Find end, append list for that CPU. */
- if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
- *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
- __this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
- per_cpu(tasklet_vec, cpu).head = NULL;
- per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
- }
- raise_softirq_irqoff(TASKLET_SOFTIRQ);
- if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
- *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
- __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
- per_cpu(tasklet_hi_vec, cpu).head = NULL;
- per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
- }
- raise_softirq_irqoff(HI_SOFTIRQ);
- local_irq_enable();
- return 0;
- }
- #else
- #define takeover_tasklets NULL
- #endif /* CONFIG_HOTPLUG_CPU */
- static struct smp_hotplug_thread softirq_threads = {
- .store = &ksoftirqd,
- .thread_should_run = ksoftirqd_should_run,
- .thread_fn = run_ksoftirqd,
- .thread_comm = "ksoftirqd/%u",
- };
- static __init int spawn_ksoftirqd(void)
- {
- cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
- takeover_tasklets);
- BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
- return 0;
- }
- early_initcall(spawn_ksoftirqd);
- /*
- * [ These __weak aliases are kept in a separate compilation unit, so that
- * GCC does not inline them incorrectly. ]
- */
- int __init __weak early_irq_init(void)
- {
- return 0;
- }
- int __init __weak arch_probe_nr_irqs(void)
- {
- return NR_IRQS_LEGACY;
- }
- int __init __weak arch_early_irq_init(void)
- {
- return 0;
- }
- unsigned int __weak arch_dynirq_lower_bound(unsigned int from)
- {
- return from;
- }
|