smp.c 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * arch/sh/kernel/smp.c
  4. *
  5. * SMP support for the SuperH processors.
  6. *
  7. * Copyright (C) 2002 - 2010 Paul Mundt
  8. * Copyright (C) 2006 - 2007 Akio Idehara
  9. */
  10. #include <linux/err.h>
  11. #include <linux/cache.h>
  12. #include <linux/cpumask.h>
  13. #include <linux/delay.h>
  14. #include <linux/init.h>
  15. #include <linux/spinlock.h>
  16. #include <linux/mm.h>
  17. #include <linux/module.h>
  18. #include <linux/cpu.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/sched/mm.h>
  21. #include <linux/sched/hotplug.h>
  22. #include <linux/atomic.h>
  23. #include <linux/clockchips.h>
  24. #include <asm/processor.h>
  25. #include <asm/mmu_context.h>
  26. #include <asm/smp.h>
  27. #include <asm/cacheflush.h>
  28. #include <asm/sections.h>
  29. #include <asm/setup.h>
  30. int __cpu_number_map[NR_CPUS]; /* Map physical to logical */
  31. int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */
  32. struct plat_smp_ops *mp_ops = NULL;
  33. /* State of each CPU */
  34. DEFINE_PER_CPU(int, cpu_state) = { 0 };
  35. void register_smp_ops(struct plat_smp_ops *ops)
  36. {
  37. if (mp_ops)
  38. printk(KERN_WARNING "Overriding previously set SMP ops\n");
  39. mp_ops = ops;
  40. }
  41. static inline void smp_store_cpu_info(unsigned int cpu)
  42. {
  43. struct sh_cpuinfo *c = cpu_data + cpu;
  44. memcpy(c, &boot_cpu_data, sizeof(struct sh_cpuinfo));
  45. c->loops_per_jiffy = loops_per_jiffy;
  46. }
  47. void __init smp_prepare_cpus(unsigned int max_cpus)
  48. {
  49. unsigned int cpu = smp_processor_id();
  50. init_new_context(current, &init_mm);
  51. current_thread_info()->cpu = cpu;
  52. mp_ops->prepare_cpus(max_cpus);
  53. #ifndef CONFIG_HOTPLUG_CPU
  54. init_cpu_present(cpu_possible_mask);
  55. #endif
  56. }
  57. void __init smp_prepare_boot_cpu(void)
  58. {
  59. unsigned int cpu = smp_processor_id();
  60. __cpu_number_map[0] = cpu;
  61. __cpu_logical_map[0] = cpu;
  62. set_cpu_online(cpu, true);
  63. set_cpu_possible(cpu, true);
  64. per_cpu(cpu_state, cpu) = CPU_ONLINE;
  65. }
  66. #ifdef CONFIG_HOTPLUG_CPU
  67. void native_cpu_die(unsigned int cpu)
  68. {
  69. unsigned int i;
  70. for (i = 0; i < 10; i++) {
  71. smp_rmb();
  72. if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
  73. if (system_state == SYSTEM_RUNNING)
  74. pr_info("CPU %u is now offline\n", cpu);
  75. return;
  76. }
  77. msleep(100);
  78. }
  79. pr_err("CPU %u didn't die...\n", cpu);
  80. }
  81. int native_cpu_disable(unsigned int cpu)
  82. {
  83. return cpu == 0 ? -EPERM : 0;
  84. }
  85. void play_dead_common(void)
  86. {
  87. idle_task_exit();
  88. irq_ctx_exit(raw_smp_processor_id());
  89. mb();
  90. __this_cpu_write(cpu_state, CPU_DEAD);
  91. local_irq_disable();
  92. }
  93. void native_play_dead(void)
  94. {
  95. play_dead_common();
  96. }
  97. int __cpu_disable(void)
  98. {
  99. unsigned int cpu = smp_processor_id();
  100. int ret;
  101. ret = mp_ops->cpu_disable(cpu);
  102. if (ret)
  103. return ret;
  104. /*
  105. * Take this CPU offline. Once we clear this, we can't return,
  106. * and we must not schedule until we're ready to give up the cpu.
  107. */
  108. set_cpu_online(cpu, false);
  109. /*
  110. * OK - migrate IRQs away from this CPU
  111. */
  112. migrate_irqs();
  113. /*
  114. * Flush user cache and TLB mappings, and then remove this CPU
  115. * from the vm mask set of all processes.
  116. */
  117. flush_cache_all();
  118. #ifdef CONFIG_MMU
  119. local_flush_tlb_all();
  120. #endif
  121. clear_tasks_mm_cpumask(cpu);
  122. return 0;
  123. }
  124. #else /* ... !CONFIG_HOTPLUG_CPU */
  125. int native_cpu_disable(unsigned int cpu)
  126. {
  127. return -ENOSYS;
  128. }
  129. void native_cpu_die(unsigned int cpu)
  130. {
  131. /* We said "no" in __cpu_disable */
  132. BUG();
  133. }
  134. void native_play_dead(void)
  135. {
  136. BUG();
  137. }
  138. #endif
  139. asmlinkage void start_secondary(void)
  140. {
  141. unsigned int cpu = smp_processor_id();
  142. struct mm_struct *mm = &init_mm;
  143. enable_mmu();
  144. mmgrab(mm);
  145. mmget(mm);
  146. current->active_mm = mm;
  147. #ifdef CONFIG_MMU
  148. enter_lazy_tlb(mm, current);
  149. local_flush_tlb_all();
  150. #endif
  151. per_cpu_trap_init();
  152. notify_cpu_starting(cpu);
  153. local_irq_enable();
  154. calibrate_delay();
  155. smp_store_cpu_info(cpu);
  156. set_cpu_online(cpu, true);
  157. per_cpu(cpu_state, cpu) = CPU_ONLINE;
  158. cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
  159. }
  160. extern struct {
  161. unsigned long sp;
  162. unsigned long bss_start;
  163. unsigned long bss_end;
  164. void *start_kernel_fn;
  165. void *cpu_init_fn;
  166. void *thread_info;
  167. } stack_start;
  168. int __cpu_up(unsigned int cpu, struct task_struct *tsk)
  169. {
  170. unsigned long timeout;
  171. per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
  172. /* Fill in data in head.S for secondary cpus */
  173. stack_start.sp = tsk->thread.sp;
  174. stack_start.thread_info = tsk->stack;
  175. stack_start.bss_start = 0; /* don't clear bss for secondary cpus */
  176. stack_start.start_kernel_fn = start_secondary;
  177. flush_icache_range((unsigned long)&stack_start,
  178. (unsigned long)&stack_start + sizeof(stack_start));
  179. wmb();
  180. mp_ops->start_cpu(cpu, (unsigned long)_stext);
  181. timeout = jiffies + HZ;
  182. while (time_before(jiffies, timeout)) {
  183. if (cpu_online(cpu))
  184. break;
  185. udelay(10);
  186. barrier();
  187. }
  188. if (cpu_online(cpu))
  189. return 0;
  190. return -ENOENT;
  191. }
  192. void __init smp_cpus_done(unsigned int max_cpus)
  193. {
  194. unsigned long bogosum = 0;
  195. int cpu;
  196. for_each_online_cpu(cpu)
  197. bogosum += cpu_data[cpu].loops_per_jiffy;
  198. printk(KERN_INFO "SMP: Total of %d processors activated "
  199. "(%lu.%02lu BogoMIPS).\n", num_online_cpus(),
  200. bogosum / (500000/HZ),
  201. (bogosum / (5000/HZ)) % 100);
  202. }
  203. void smp_send_reschedule(int cpu)
  204. {
  205. mp_ops->send_ipi(cpu, SMP_MSG_RESCHEDULE);
  206. }
  207. void smp_send_stop(void)
  208. {
  209. smp_call_function(stop_this_cpu, 0, 0);
  210. }
  211. void arch_send_call_function_ipi_mask(const struct cpumask *mask)
  212. {
  213. int cpu;
  214. for_each_cpu(cpu, mask)
  215. mp_ops->send_ipi(cpu, SMP_MSG_FUNCTION);
  216. }
  217. void arch_send_call_function_single_ipi(int cpu)
  218. {
  219. mp_ops->send_ipi(cpu, SMP_MSG_FUNCTION_SINGLE);
  220. }
  221. #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
  222. void tick_broadcast(const struct cpumask *mask)
  223. {
  224. int cpu;
  225. for_each_cpu(cpu, mask)
  226. mp_ops->send_ipi(cpu, SMP_MSG_TIMER);
  227. }
  228. static void ipi_timer(void)
  229. {
  230. irq_enter();
  231. tick_receive_broadcast();
  232. irq_exit();
  233. }
  234. #endif
  235. void smp_message_recv(unsigned int msg)
  236. {
  237. switch (msg) {
  238. case SMP_MSG_FUNCTION:
  239. generic_smp_call_function_interrupt();
  240. break;
  241. case SMP_MSG_RESCHEDULE:
  242. scheduler_ipi();
  243. break;
  244. case SMP_MSG_FUNCTION_SINGLE:
  245. generic_smp_call_function_single_interrupt();
  246. break;
  247. #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
  248. case SMP_MSG_TIMER:
  249. ipi_timer();
  250. break;
  251. #endif
  252. default:
  253. printk(KERN_WARNING "SMP %d: %s(): unknown IPI %d\n",
  254. smp_processor_id(), __func__, msg);
  255. break;
  256. }
  257. }
  258. /* Not really SMP stuff ... */
  259. int setup_profiling_timer(unsigned int multiplier)
  260. {
  261. return 0;
  262. }
  263. #ifdef CONFIG_MMU
  264. static void flush_tlb_all_ipi(void *info)
  265. {
  266. local_flush_tlb_all();
  267. }
  268. void flush_tlb_all(void)
  269. {
  270. on_each_cpu(flush_tlb_all_ipi, 0, 1);
  271. }
  272. static void flush_tlb_mm_ipi(void *mm)
  273. {
  274. local_flush_tlb_mm((struct mm_struct *)mm);
  275. }
  276. /*
  277. * The following tlb flush calls are invoked when old translations are
  278. * being torn down, or pte attributes are changing. For single threaded
  279. * address spaces, a new context is obtained on the current cpu, and tlb
  280. * context on other cpus are invalidated to force a new context allocation
  281. * at switch_mm time, should the mm ever be used on other cpus. For
  282. * multithreaded address spaces, intercpu interrupts have to be sent.
  283. * Another case where intercpu interrupts are required is when the target
  284. * mm might be active on another cpu (eg debuggers doing the flushes on
  285. * behalf of debugees, kswapd stealing pages from another process etc).
  286. * Kanoj 07/00.
  287. */
  288. void flush_tlb_mm(struct mm_struct *mm)
  289. {
  290. preempt_disable();
  291. if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
  292. smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1);
  293. } else {
  294. int i;
  295. for_each_online_cpu(i)
  296. if (smp_processor_id() != i)
  297. cpu_context(i, mm) = 0;
  298. }
  299. local_flush_tlb_mm(mm);
  300. preempt_enable();
  301. }
  302. struct flush_tlb_data {
  303. struct vm_area_struct *vma;
  304. unsigned long addr1;
  305. unsigned long addr2;
  306. };
  307. static void flush_tlb_range_ipi(void *info)
  308. {
  309. struct flush_tlb_data *fd = (struct flush_tlb_data *)info;
  310. local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
  311. }
  312. void flush_tlb_range(struct vm_area_struct *vma,
  313. unsigned long start, unsigned long end)
  314. {
  315. struct mm_struct *mm = vma->vm_mm;
  316. preempt_disable();
  317. if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
  318. struct flush_tlb_data fd;
  319. fd.vma = vma;
  320. fd.addr1 = start;
  321. fd.addr2 = end;
  322. smp_call_function(flush_tlb_range_ipi, (void *)&fd, 1);
  323. } else {
  324. int i;
  325. for_each_online_cpu(i)
  326. if (smp_processor_id() != i)
  327. cpu_context(i, mm) = 0;
  328. }
  329. local_flush_tlb_range(vma, start, end);
  330. preempt_enable();
  331. }
  332. static void flush_tlb_kernel_range_ipi(void *info)
  333. {
  334. struct flush_tlb_data *fd = (struct flush_tlb_data *)info;
  335. local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
  336. }
  337. void flush_tlb_kernel_range(unsigned long start, unsigned long end)
  338. {
  339. struct flush_tlb_data fd;
  340. fd.addr1 = start;
  341. fd.addr2 = end;
  342. on_each_cpu(flush_tlb_kernel_range_ipi, (void *)&fd, 1);
  343. }
  344. static void flush_tlb_page_ipi(void *info)
  345. {
  346. struct flush_tlb_data *fd = (struct flush_tlb_data *)info;
  347. local_flush_tlb_page(fd->vma, fd->addr1);
  348. }
  349. void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
  350. {
  351. preempt_disable();
  352. if ((atomic_read(&vma->vm_mm->mm_users) != 1) ||
  353. (current->mm != vma->vm_mm)) {
  354. struct flush_tlb_data fd;
  355. fd.vma = vma;
  356. fd.addr1 = page;
  357. smp_call_function(flush_tlb_page_ipi, (void *)&fd, 1);
  358. } else {
  359. int i;
  360. for_each_online_cpu(i)
  361. if (smp_processor_id() != i)
  362. cpu_context(i, vma->vm_mm) = 0;
  363. }
  364. local_flush_tlb_page(vma, page);
  365. preempt_enable();
  366. }
  367. static void flush_tlb_one_ipi(void *info)
  368. {
  369. struct flush_tlb_data *fd = (struct flush_tlb_data *)info;
  370. local_flush_tlb_one(fd->addr1, fd->addr2);
  371. }
  372. void flush_tlb_one(unsigned long asid, unsigned long vaddr)
  373. {
  374. struct flush_tlb_data fd;
  375. fd.addr1 = asid;
  376. fd.addr2 = vaddr;
  377. smp_call_function(flush_tlb_one_ipi, (void *)&fd, 1);
  378. local_flush_tlb_one(asid, vaddr);
  379. }
  380. #endif