smpboot.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Common SMP CPU bringup/teardown functions
  4. */
  5. #include <linux/cpu.h>
  6. #include <linux/err.h>
  7. #include <linux/smp.h>
  8. #include <linux/delay.h>
  9. #include <linux/init.h>
  10. #include <linux/list.h>
  11. #include <linux/slab.h>
  12. #include <linux/sched.h>
  13. #include <linux/sched/task.h>
  14. #include <linux/export.h>
  15. #include <linux/percpu.h>
  16. #include <linux/kthread.h>
  17. #include <linux/smpboot.h>
  18. #include "smpboot.h"
  19. #ifdef CONFIG_SMP
  20. #ifdef CONFIG_GENERIC_SMP_IDLE_THREAD
  21. /*
  22. * For the hotplug case we keep the task structs around and reuse
  23. * them.
  24. */
  25. static DEFINE_PER_CPU(struct task_struct *, idle_threads);
  26. struct task_struct *idle_thread_get(unsigned int cpu)
  27. {
  28. struct task_struct *tsk = per_cpu(idle_threads, cpu);
  29. if (!tsk)
  30. return ERR_PTR(-ENOMEM);
  31. return tsk;
  32. }
  33. void __init idle_thread_set_boot_cpu(void)
  34. {
  35. per_cpu(idle_threads, smp_processor_id()) = current;
  36. }
  37. /**
  38. * idle_init - Initialize the idle thread for a cpu
  39. * @cpu: The cpu for which the idle thread should be initialized
  40. *
  41. * Creates the thread if it does not exist.
  42. */
  43. static inline void idle_init(unsigned int cpu)
  44. {
  45. struct task_struct *tsk = per_cpu(idle_threads, cpu);
  46. if (!tsk) {
  47. tsk = fork_idle(cpu);
  48. if (IS_ERR(tsk))
  49. pr_err("SMP: fork_idle() failed for CPU %u\n", cpu);
  50. else
  51. per_cpu(idle_threads, cpu) = tsk;
  52. }
  53. }
  54. /**
  55. * idle_threads_init - Initialize idle threads for all cpus
  56. */
  57. void __init idle_threads_init(void)
  58. {
  59. unsigned int cpu, boot_cpu;
  60. boot_cpu = smp_processor_id();
  61. for_each_possible_cpu(cpu) {
  62. if (cpu != boot_cpu)
  63. idle_init(cpu);
  64. }
  65. }
  66. #endif
  67. #endif /* #ifdef CONFIG_SMP */
  68. static LIST_HEAD(hotplug_threads);
  69. static DEFINE_MUTEX(smpboot_threads_lock);
  70. struct smpboot_thread_data {
  71. unsigned int cpu;
  72. unsigned int status;
  73. struct smp_hotplug_thread *ht;
  74. };
  75. enum {
  76. HP_THREAD_NONE = 0,
  77. HP_THREAD_ACTIVE,
  78. HP_THREAD_PARKED,
  79. };
  80. /**
  81. * smpboot_thread_fn - percpu hotplug thread loop function
  82. * @data: thread data pointer
  83. *
  84. * Checks for thread stop and park conditions. Calls the necessary
  85. * setup, cleanup, park and unpark functions for the registered
  86. * thread.
  87. *
  88. * Returns 1 when the thread should exit, 0 otherwise.
  89. */
  90. static int smpboot_thread_fn(void *data)
  91. {
  92. struct smpboot_thread_data *td = data;
  93. struct smp_hotplug_thread *ht = td->ht;
  94. while (1) {
  95. set_current_state(TASK_INTERRUPTIBLE);
  96. preempt_disable();
  97. if (kthread_should_stop()) {
  98. __set_current_state(TASK_RUNNING);
  99. preempt_enable();
  100. /* cleanup must mirror setup */
  101. if (ht->cleanup && td->status != HP_THREAD_NONE)
  102. ht->cleanup(td->cpu, cpu_online(td->cpu));
  103. kfree(td);
  104. return 0;
  105. }
  106. if (kthread_should_park()) {
  107. __set_current_state(TASK_RUNNING);
  108. preempt_enable();
  109. if (ht->park && td->status == HP_THREAD_ACTIVE) {
  110. BUG_ON(td->cpu != smp_processor_id());
  111. ht->park(td->cpu);
  112. td->status = HP_THREAD_PARKED;
  113. }
  114. kthread_parkme();
  115. /* We might have been woken for stop */
  116. continue;
  117. }
  118. BUG_ON(td->cpu != smp_processor_id());
  119. /* Check for state change setup */
  120. switch (td->status) {
  121. case HP_THREAD_NONE:
  122. __set_current_state(TASK_RUNNING);
  123. preempt_enable();
  124. if (ht->setup)
  125. ht->setup(td->cpu);
  126. td->status = HP_THREAD_ACTIVE;
  127. continue;
  128. case HP_THREAD_PARKED:
  129. __set_current_state(TASK_RUNNING);
  130. preempt_enable();
  131. if (ht->unpark)
  132. ht->unpark(td->cpu);
  133. td->status = HP_THREAD_ACTIVE;
  134. continue;
  135. }
  136. if (!ht->thread_should_run(td->cpu)) {
  137. preempt_enable_no_resched();
  138. schedule();
  139. } else {
  140. __set_current_state(TASK_RUNNING);
  141. preempt_enable();
  142. ht->thread_fn(td->cpu);
  143. }
  144. }
  145. }
  146. static int
  147. __smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
  148. {
  149. struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
  150. struct smpboot_thread_data *td;
  151. if (tsk)
  152. return 0;
  153. td = kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu));
  154. if (!td)
  155. return -ENOMEM;
  156. td->cpu = cpu;
  157. td->ht = ht;
  158. tsk = kthread_create_on_cpu(smpboot_thread_fn, td, cpu,
  159. ht->thread_comm);
  160. if (IS_ERR(tsk)) {
  161. kfree(td);
  162. return PTR_ERR(tsk);
  163. }
  164. kthread_set_per_cpu(tsk, cpu);
  165. /*
  166. * Park the thread so that it could start right on the CPU
  167. * when it is available.
  168. */
  169. kthread_park(tsk);
  170. get_task_struct(tsk);
  171. *per_cpu_ptr(ht->store, cpu) = tsk;
  172. if (ht->create) {
  173. /*
  174. * Make sure that the task has actually scheduled out
  175. * into park position, before calling the create
  176. * callback. At least the migration thread callback
  177. * requires that the task is off the runqueue.
  178. */
  179. if (!wait_task_inactive(tsk, TASK_PARKED))
  180. WARN_ON(1);
  181. else
  182. ht->create(cpu);
  183. }
  184. return 0;
  185. }
  186. int smpboot_create_threads(unsigned int cpu)
  187. {
  188. struct smp_hotplug_thread *cur;
  189. int ret = 0;
  190. mutex_lock(&smpboot_threads_lock);
  191. list_for_each_entry(cur, &hotplug_threads, list) {
  192. ret = __smpboot_create_thread(cur, cpu);
  193. if (ret)
  194. break;
  195. }
  196. mutex_unlock(&smpboot_threads_lock);
  197. return ret;
  198. }
  199. static void smpboot_unpark_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
  200. {
  201. struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
  202. if (!ht->selfparking)
  203. kthread_unpark(tsk);
  204. }
  205. int smpboot_unpark_threads(unsigned int cpu)
  206. {
  207. struct smp_hotplug_thread *cur;
  208. mutex_lock(&smpboot_threads_lock);
  209. list_for_each_entry(cur, &hotplug_threads, list)
  210. smpboot_unpark_thread(cur, cpu);
  211. mutex_unlock(&smpboot_threads_lock);
  212. return 0;
  213. }
  214. static void smpboot_park_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
  215. {
  216. struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
  217. if (tsk && !ht->selfparking)
  218. kthread_park(tsk);
  219. }
  220. int smpboot_park_threads(unsigned int cpu)
  221. {
  222. struct smp_hotplug_thread *cur;
  223. mutex_lock(&smpboot_threads_lock);
  224. list_for_each_entry_reverse(cur, &hotplug_threads, list)
  225. smpboot_park_thread(cur, cpu);
  226. mutex_unlock(&smpboot_threads_lock);
  227. return 0;
  228. }
  229. static void smpboot_destroy_threads(struct smp_hotplug_thread *ht)
  230. {
  231. unsigned int cpu;
  232. /* We need to destroy also the parked threads of offline cpus */
  233. for_each_possible_cpu(cpu) {
  234. struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
  235. if (tsk) {
  236. kthread_stop(tsk);
  237. put_task_struct(tsk);
  238. *per_cpu_ptr(ht->store, cpu) = NULL;
  239. }
  240. }
  241. }
  242. /**
  243. * smpboot_register_percpu_thread - Register a per_cpu thread related
  244. * to hotplug
  245. * @plug_thread: Hotplug thread descriptor
  246. *
  247. * Creates and starts the threads on all online cpus.
  248. */
  249. int smpboot_register_percpu_thread(struct smp_hotplug_thread *plug_thread)
  250. {
  251. unsigned int cpu;
  252. int ret = 0;
  253. get_online_cpus();
  254. mutex_lock(&smpboot_threads_lock);
  255. for_each_online_cpu(cpu) {
  256. ret = __smpboot_create_thread(plug_thread, cpu);
  257. if (ret) {
  258. smpboot_destroy_threads(plug_thread);
  259. goto out;
  260. }
  261. smpboot_unpark_thread(plug_thread, cpu);
  262. }
  263. list_add(&plug_thread->list, &hotplug_threads);
  264. out:
  265. mutex_unlock(&smpboot_threads_lock);
  266. put_online_cpus();
  267. return ret;
  268. }
  269. EXPORT_SYMBOL_GPL(smpboot_register_percpu_thread);
  270. /**
  271. * smpboot_unregister_percpu_thread - Unregister a per_cpu thread related to hotplug
  272. * @plug_thread: Hotplug thread descriptor
  273. *
  274. * Stops all threads on all possible cpus.
  275. */
  276. void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread)
  277. {
  278. get_online_cpus();
  279. mutex_lock(&smpboot_threads_lock);
  280. list_del(&plug_thread->list);
  281. smpboot_destroy_threads(plug_thread);
  282. mutex_unlock(&smpboot_threads_lock);
  283. put_online_cpus();
  284. }
  285. EXPORT_SYMBOL_GPL(smpboot_unregister_percpu_thread);
  286. static DEFINE_PER_CPU(atomic_t, cpu_hotplug_state) = ATOMIC_INIT(CPU_POST_DEAD);
  287. /*
  288. * Called to poll specified CPU's state, for example, when waiting for
  289. * a CPU to come online.
  290. */
  291. int cpu_report_state(int cpu)
  292. {
  293. return atomic_read(&per_cpu(cpu_hotplug_state, cpu));
  294. }
  295. /*
  296. * If CPU has died properly, set its state to CPU_UP_PREPARE and
  297. * return success. Otherwise, return -EBUSY if the CPU died after
  298. * cpu_wait_death() timed out. And yet otherwise again, return -EAGAIN
  299. * if cpu_wait_death() timed out and the CPU still hasn't gotten around
  300. * to dying. In the latter two cases, the CPU might not be set up
  301. * properly, but it is up to the arch-specific code to decide.
  302. * Finally, -EIO indicates an unanticipated problem.
  303. *
  304. * Note that it is permissible to omit this call entirely, as is
  305. * done in architectures that do no CPU-hotplug error checking.
  306. */
  307. int cpu_check_up_prepare(int cpu)
  308. {
  309. if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
  310. atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_UP_PREPARE);
  311. return 0;
  312. }
  313. switch (atomic_read(&per_cpu(cpu_hotplug_state, cpu))) {
  314. case CPU_POST_DEAD:
  315. /* The CPU died properly, so just start it up again. */
  316. atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_UP_PREPARE);
  317. return 0;
  318. case CPU_DEAD_FROZEN:
  319. /*
  320. * Timeout during CPU death, so let caller know.
  321. * The outgoing CPU completed its processing, but after
  322. * cpu_wait_death() timed out and reported the error. The
  323. * caller is free to proceed, in which case the state
  324. * will be reset properly by cpu_set_state_online().
  325. * Proceeding despite this -EBUSY return makes sense
  326. * for systems where the outgoing CPUs take themselves
  327. * offline, with no post-death manipulation required from
  328. * a surviving CPU.
  329. */
  330. return -EBUSY;
  331. case CPU_BROKEN:
  332. /*
  333. * The most likely reason we got here is that there was
  334. * a timeout during CPU death, and the outgoing CPU never
  335. * did complete its processing. This could happen on
  336. * a virtualized system if the outgoing VCPU gets preempted
  337. * for more than five seconds, and the user attempts to
  338. * immediately online that same CPU. Trying again later
  339. * might return -EBUSY above, hence -EAGAIN.
  340. */
  341. return -EAGAIN;
  342. default:
  343. /* Should not happen. Famous last words. */
  344. return -EIO;
  345. }
  346. }
  347. /*
  348. * Mark the specified CPU online.
  349. *
  350. * Note that it is permissible to omit this call entirely, as is
  351. * done in architectures that do no CPU-hotplug error checking.
  352. */
  353. void cpu_set_state_online(int cpu)
  354. {
  355. (void)atomic_xchg(&per_cpu(cpu_hotplug_state, cpu), CPU_ONLINE);
  356. }
  357. #ifdef CONFIG_HOTPLUG_CPU
  358. /*
  359. * Wait for the specified CPU to exit the idle loop and die.
  360. */
  361. bool cpu_wait_death(unsigned int cpu, int seconds)
  362. {
  363. int jf_left = seconds * HZ;
  364. int oldstate;
  365. bool ret = true;
  366. int sleep_jf = 1;
  367. might_sleep();
  368. /* The outgoing CPU will normally get done quite quickly. */
  369. if (atomic_read(&per_cpu(cpu_hotplug_state, cpu)) == CPU_DEAD)
  370. goto update_state;
  371. udelay(5);
  372. /* But if the outgoing CPU dawdles, wait increasingly long times. */
  373. while (atomic_read(&per_cpu(cpu_hotplug_state, cpu)) != CPU_DEAD) {
  374. schedule_timeout_uninterruptible(sleep_jf);
  375. jf_left -= sleep_jf;
  376. if (jf_left <= 0)
  377. break;
  378. sleep_jf = DIV_ROUND_UP(sleep_jf * 11, 10);
  379. }
  380. update_state:
  381. oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu));
  382. if (oldstate == CPU_DEAD) {
  383. /* Outgoing CPU died normally, update state. */
  384. smp_mb(); /* atomic_read() before update. */
  385. atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_POST_DEAD);
  386. } else {
  387. /* Outgoing CPU still hasn't died, set state accordingly. */
  388. if (atomic_cmpxchg(&per_cpu(cpu_hotplug_state, cpu),
  389. oldstate, CPU_BROKEN) != oldstate)
  390. goto update_state;
  391. ret = false;
  392. }
  393. return ret;
  394. }
  395. /*
  396. * Called by the outgoing CPU to report its successful death. Return
  397. * false if this report follows the surviving CPU's timing out.
  398. *
  399. * A separate "CPU_DEAD_FROZEN" is used when the surviving CPU
  400. * timed out. This approach allows architectures to omit calls to
  401. * cpu_check_up_prepare() and cpu_set_state_online() without defeating
  402. * the next cpu_wait_death()'s polling loop.
  403. */
  404. bool cpu_report_death(void)
  405. {
  406. int oldstate;
  407. int newstate;
  408. int cpu = smp_processor_id();
  409. do {
  410. oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu));
  411. if (oldstate != CPU_BROKEN)
  412. newstate = CPU_DEAD;
  413. else
  414. newstate = CPU_DEAD_FROZEN;
  415. } while (atomic_cmpxchg(&per_cpu(cpu_hotplug_state, cpu),
  416. oldstate, newstate) != oldstate);
  417. return newstate == CPU_DEAD;
  418. }
  419. #endif /* #ifdef CONFIG_HOTPLUG_CPU */