stop_machine.c 4.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210
  1. /* Copyright 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
  2. * GPL v2 and any later version.
  3. */
  4. #include <linux/stop_machine.h>
  5. #include <linux/kthread.h>
  6. #include <linux/sched.h>
  7. #include <linux/cpu.h>
  8. #include <linux/err.h>
  9. #include <linux/syscalls.h>
  10. #include <asm/atomic.h>
  11. #include <asm/semaphore.h>
  12. #include <asm/uaccess.h>
  13. /* Since we effect priority and affinity (both of which are visible
  14. * to, and settable by outside processes) we do indirection via a
  15. * kthread. */
  16. /* Thread to stop each CPU in user context. */
  17. enum stopmachine_state {
  18. STOPMACHINE_WAIT,
  19. STOPMACHINE_PREPARE,
  20. STOPMACHINE_DISABLE_IRQ,
  21. STOPMACHINE_EXIT,
  22. };
  23. static enum stopmachine_state stopmachine_state;
  24. static unsigned int stopmachine_num_threads;
  25. static atomic_t stopmachine_thread_ack;
  26. static DECLARE_MUTEX(stopmachine_mutex);
  27. static int stopmachine(void *cpu)
  28. {
  29. int irqs_disabled = 0;
  30. int prepared = 0;
  31. set_cpus_allowed(current, cpumask_of_cpu((int)(long)cpu));
  32. /* Ack: we are alive */
  33. smp_mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */
  34. atomic_inc(&stopmachine_thread_ack);
  35. /* Simple state machine */
  36. while (stopmachine_state != STOPMACHINE_EXIT) {
  37. if (stopmachine_state == STOPMACHINE_DISABLE_IRQ
  38. && !irqs_disabled) {
  39. local_irq_disable();
  40. irqs_disabled = 1;
  41. /* Ack: irqs disabled. */
  42. smp_mb(); /* Must read state first. */
  43. atomic_inc(&stopmachine_thread_ack);
  44. } else if (stopmachine_state == STOPMACHINE_PREPARE
  45. && !prepared) {
  46. /* Everyone is in place, hold CPU. */
  47. preempt_disable();
  48. prepared = 1;
  49. smp_mb(); /* Must read state first. */
  50. atomic_inc(&stopmachine_thread_ack);
  51. }
  52. /* Yield in first stage: migration threads need to
  53. * help our sisters onto their CPUs. */
  54. if (!prepared && !irqs_disabled)
  55. yield();
  56. else
  57. cpu_relax();
  58. }
  59. /* Ack: we are exiting. */
  60. smp_mb(); /* Must read state first. */
  61. atomic_inc(&stopmachine_thread_ack);
  62. if (irqs_disabled)
  63. local_irq_enable();
  64. if (prepared)
  65. preempt_enable();
  66. return 0;
  67. }
  68. /* Change the thread state */
  69. static void stopmachine_set_state(enum stopmachine_state state)
  70. {
  71. atomic_set(&stopmachine_thread_ack, 0);
  72. smp_wmb();
  73. stopmachine_state = state;
  74. while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads)
  75. cpu_relax();
  76. }
  77. static int stop_machine(void)
  78. {
  79. int i, ret = 0;
  80. struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
  81. /* One high-prio thread per cpu. We'll do this one. */
  82. sched_setscheduler(current, SCHED_FIFO, &param);
  83. atomic_set(&stopmachine_thread_ack, 0);
  84. stopmachine_num_threads = 0;
  85. stopmachine_state = STOPMACHINE_WAIT;
  86. for_each_online_cpu(i) {
  87. if (i == raw_smp_processor_id())
  88. continue;
  89. ret = kernel_thread(stopmachine, (void *)(long)i,CLONE_KERNEL);
  90. if (ret < 0)
  91. break;
  92. stopmachine_num_threads++;
  93. }
  94. /* Wait for them all to come to life. */
  95. while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads)
  96. yield();
  97. /* If some failed, kill them all. */
  98. if (ret < 0) {
  99. stopmachine_set_state(STOPMACHINE_EXIT);
  100. return ret;
  101. }
  102. /* Now they are all started, make them hold the CPUs, ready. */
  103. preempt_disable();
  104. stopmachine_set_state(STOPMACHINE_PREPARE);
  105. /* Make them disable irqs. */
  106. local_irq_disable();
  107. stopmachine_set_state(STOPMACHINE_DISABLE_IRQ);
  108. return 0;
  109. }
  110. static void restart_machine(void)
  111. {
  112. stopmachine_set_state(STOPMACHINE_EXIT);
  113. local_irq_enable();
  114. preempt_enable_no_resched();
  115. }
  116. struct stop_machine_data
  117. {
  118. int (*fn)(void *);
  119. void *data;
  120. struct completion done;
  121. };
  122. static int do_stop(void *_smdata)
  123. {
  124. struct stop_machine_data *smdata = _smdata;
  125. int ret;
  126. ret = stop_machine();
  127. if (ret == 0) {
  128. ret = smdata->fn(smdata->data);
  129. restart_machine();
  130. }
  131. /* We're done: you can kthread_stop us now */
  132. complete(&smdata->done);
  133. /* Wait for kthread_stop */
  134. set_current_state(TASK_INTERRUPTIBLE);
  135. while (!kthread_should_stop()) {
  136. schedule();
  137. set_current_state(TASK_INTERRUPTIBLE);
  138. }
  139. __set_current_state(TASK_RUNNING);
  140. return ret;
  141. }
  142. struct task_struct *__stop_machine_run(int (*fn)(void *), void *data,
  143. unsigned int cpu)
  144. {
  145. struct stop_machine_data smdata;
  146. struct task_struct *p;
  147. smdata.fn = fn;
  148. smdata.data = data;
  149. init_completion(&smdata.done);
  150. down(&stopmachine_mutex);
  151. /* If they don't care which CPU fn runs on, bind to any online one. */
  152. if (cpu == NR_CPUS)
  153. cpu = raw_smp_processor_id();
  154. p = kthread_create(do_stop, &smdata, "kstopmachine");
  155. if (!IS_ERR(p)) {
  156. kthread_bind(p, cpu);
  157. wake_up_process(p);
  158. wait_for_completion(&smdata.done);
  159. }
  160. up(&stopmachine_mutex);
  161. return p;
  162. }
  163. int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu)
  164. {
  165. struct task_struct *p;
  166. int ret;
  167. /* No CPUs can come up or down during this. */
  168. lock_cpu_hotplug();
  169. p = __stop_machine_run(fn, data, cpu);
  170. if (!IS_ERR(p))
  171. ret = kthread_stop(p);
  172. else
  173. ret = PTR_ERR(p);
  174. unlock_cpu_hotplug();
  175. return ret;
  176. }