smp.c 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
  4. *
  5. * RajeshwarR: Dec 11, 2007
  6. * -- Added support for Inter Processor Interrupts
  7. *
  8. * Vineetg: Nov 1st, 2007
  9. * -- Initial Write (Borrowed heavily from ARM)
  10. */
  11. #include <linux/spinlock.h>
  12. #include <linux/sched/mm.h>
  13. #include <linux/interrupt.h>
  14. #include <linux/profile.h>
  15. #include <linux/mm.h>
  16. #include <linux/cpu.h>
  17. #include <linux/irq.h>
  18. #include <linux/atomic.h>
  19. #include <linux/cpumask.h>
  20. #include <linux/reboot.h>
  21. #include <linux/irqdomain.h>
  22. #include <linux/export.h>
  23. #include <linux/of_fdt.h>
  24. #include <asm/processor.h>
  25. #include <asm/setup.h>
  26. #include <asm/mach_desc.h>
  27. #ifndef CONFIG_ARC_HAS_LLSC
  28. arch_spinlock_t smp_atomic_ops_lock = __ARCH_SPIN_LOCK_UNLOCKED;
  29. arch_spinlock_t smp_bitops_lock = __ARCH_SPIN_LOCK_UNLOCKED;
  30. EXPORT_SYMBOL_GPL(smp_atomic_ops_lock);
  31. EXPORT_SYMBOL_GPL(smp_bitops_lock);
  32. #endif
  33. struct plat_smp_ops __weak plat_smp_ops;
  34. /* XXX: per cpu ? Only needed once in early seconday boot */
  35. struct task_struct *secondary_idle_tsk;
  36. /* Called from start_kernel */
  37. void __init smp_prepare_boot_cpu(void)
  38. {
  39. }
  40. static int __init arc_get_cpu_map(const char *name, struct cpumask *cpumask)
  41. {
  42. unsigned long dt_root = of_get_flat_dt_root();
  43. const char *buf;
  44. buf = of_get_flat_dt_prop(dt_root, name, NULL);
  45. if (!buf)
  46. return -EINVAL;
  47. if (cpulist_parse(buf, cpumask))
  48. return -EINVAL;
  49. return 0;
  50. }
  51. /*
  52. * Read from DeviceTree and setup cpu possible mask. If there is no
  53. * "possible-cpus" property in DeviceTree pretend all [0..NR_CPUS-1] exist.
  54. */
  55. static void __init arc_init_cpu_possible(void)
  56. {
  57. struct cpumask cpumask;
  58. if (arc_get_cpu_map("possible-cpus", &cpumask)) {
  59. pr_warn("Failed to get possible-cpus from dtb, pretending all %u cpus exist\n",
  60. NR_CPUS);
  61. cpumask_setall(&cpumask);
  62. }
  63. if (!cpumask_test_cpu(0, &cpumask))
  64. panic("Master cpu (cpu[0]) is missed in cpu possible mask!");
  65. init_cpu_possible(&cpumask);
  66. }
  67. /*
  68. * Called from setup_arch() before calling setup_processor()
  69. *
  70. * - Initialise the CPU possible map early - this describes the CPUs
  71. * which may be present or become present in the system.
  72. * - Call early smp init hook. This can initialize a specific multi-core
  73. * IP which is say common to several platforms (hence not part of
  74. * platform specific int_early() hook)
  75. */
  76. void __init smp_init_cpus(void)
  77. {
  78. arc_init_cpu_possible();
  79. if (plat_smp_ops.init_early_smp)
  80. plat_smp_ops.init_early_smp();
  81. }
  82. /* called from init ( ) => process 1 */
  83. void __init smp_prepare_cpus(unsigned int max_cpus)
  84. {
  85. /*
  86. * if platform didn't set the present map already, do it now
  87. * boot cpu is set to present already by init/main.c
  88. */
  89. if (num_present_cpus() <= 1)
  90. init_cpu_present(cpu_possible_mask);
  91. }
  92. void __init smp_cpus_done(unsigned int max_cpus)
  93. {
  94. }
  95. /*
  96. * Default smp boot helper for Run-on-reset case where all cores start off
  97. * together. Non-masters need to wait for Master to start running.
  98. * This is implemented using a flag in memory, which Non-masters spin-wait on.
  99. * Master sets it to cpu-id of core to "ungate" it.
  100. */
  101. static volatile int wake_flag;
  102. #ifdef CONFIG_ISA_ARCOMPACT
  103. #define __boot_read(f) f
  104. #define __boot_write(f, v) f = v
  105. #else
  106. #define __boot_read(f) arc_read_uncached_32(&f)
  107. #define __boot_write(f, v) arc_write_uncached_32(&f, v)
  108. #endif
  109. static void arc_default_smp_cpu_kick(int cpu, unsigned long pc)
  110. {
  111. BUG_ON(cpu == 0);
  112. __boot_write(wake_flag, cpu);
  113. }
  114. void arc_platform_smp_wait_to_boot(int cpu)
  115. {
  116. /* for halt-on-reset, we've waited already */
  117. if (IS_ENABLED(CONFIG_ARC_SMP_HALT_ON_RESET))
  118. return;
  119. while (__boot_read(wake_flag) != cpu)
  120. ;
  121. __boot_write(wake_flag, 0);
  122. }
  123. const char *arc_platform_smp_cpuinfo(void)
  124. {
  125. return plat_smp_ops.info ? : "";
  126. }
  127. /*
  128. * The very first "C" code executed by secondary
  129. * Called from asm stub in head.S
  130. * "current"/R25 already setup by low level boot code
  131. */
  132. void start_kernel_secondary(void)
  133. {
  134. struct mm_struct *mm = &init_mm;
  135. unsigned int cpu = smp_processor_id();
  136. /* MMU, Caches, Vector Table, Interrupts etc */
  137. setup_processor();
  138. mmget(mm);
  139. mmgrab(mm);
  140. current->active_mm = mm;
  141. cpumask_set_cpu(cpu, mm_cpumask(mm));
  142. /* Some SMP H/w setup - for each cpu */
  143. if (plat_smp_ops.init_per_cpu)
  144. plat_smp_ops.init_per_cpu(cpu);
  145. if (machine_desc->init_per_cpu)
  146. machine_desc->init_per_cpu(cpu);
  147. notify_cpu_starting(cpu);
  148. set_cpu_online(cpu, true);
  149. pr_info("## CPU%u LIVE ##: Executing Code...\n", cpu);
  150. local_irq_enable();
  151. cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
  152. }
  153. /*
  154. * Called from kernel_init( ) -> smp_init( ) - for each CPU
  155. *
  156. * At this point, Secondary Processor is "HALT"ed:
  157. * -It booted, but was halted in head.S
  158. * -It was configured to halt-on-reset
  159. * So need to wake it up.
  160. *
  161. * Essential requirements being where to run from (PC) and stack (SP)
  162. */
  163. int __cpu_up(unsigned int cpu, struct task_struct *idle)
  164. {
  165. unsigned long wait_till;
  166. secondary_idle_tsk = idle;
  167. pr_info("Idle Task [%d] %p", cpu, idle);
  168. pr_info("Trying to bring up CPU%u ...\n", cpu);
  169. if (plat_smp_ops.cpu_kick)
  170. plat_smp_ops.cpu_kick(cpu,
  171. (unsigned long)first_lines_of_secondary);
  172. else
  173. arc_default_smp_cpu_kick(cpu, (unsigned long)NULL);
  174. /* wait for 1 sec after kicking the secondary */
  175. wait_till = jiffies + HZ;
  176. while (time_before(jiffies, wait_till)) {
  177. if (cpu_online(cpu))
  178. break;
  179. }
  180. if (!cpu_online(cpu)) {
  181. pr_info("Timeout: CPU%u FAILED to come up !!!\n", cpu);
  182. return -1;
  183. }
  184. secondary_idle_tsk = NULL;
  185. return 0;
  186. }
  187. /*
  188. * not supported here
  189. */
  190. int setup_profiling_timer(unsigned int multiplier)
  191. {
  192. return -EINVAL;
  193. }
  194. /*****************************************************************************/
  195. /* Inter Processor Interrupt Handling */
  196. /*****************************************************************************/
  197. enum ipi_msg_type {
  198. IPI_EMPTY = 0,
  199. IPI_RESCHEDULE = 1,
  200. IPI_CALL_FUNC,
  201. IPI_CPU_STOP,
  202. };
  203. /*
  204. * In arches with IRQ for each msg type (above), receiver can use IRQ-id to
  205. * figure out what msg was sent. For those which don't (ARC has dedicated IPI
  206. * IRQ), the msg-type needs to be conveyed via per-cpu data
  207. */
  208. static DEFINE_PER_CPU(unsigned long, ipi_data);
  209. static void ipi_send_msg_one(int cpu, enum ipi_msg_type msg)
  210. {
  211. unsigned long __percpu *ipi_data_ptr = per_cpu_ptr(&ipi_data, cpu);
  212. unsigned long old, new;
  213. unsigned long flags;
  214. pr_debug("%d Sending msg [%d] to %d\n", smp_processor_id(), msg, cpu);
  215. local_irq_save(flags);
  216. /*
  217. * Atomically write new msg bit (in case others are writing too),
  218. * and read back old value
  219. */
  220. do {
  221. new = old = READ_ONCE(*ipi_data_ptr);
  222. new |= 1U << msg;
  223. } while (cmpxchg(ipi_data_ptr, old, new) != old);
  224. /*
  225. * Call the platform specific IPI kick function, but avoid if possible:
  226. * Only do so if there's no pending msg from other concurrent sender(s).
  227. * Otherwise, recevier will see this msg as well when it takes the
  228. * IPI corresponding to that msg. This is true, even if it is already in
  229. * IPI handler, because !@old means it has not yet dequeued the msg(s)
  230. * so @new msg can be a free-loader
  231. */
  232. if (plat_smp_ops.ipi_send && !old)
  233. plat_smp_ops.ipi_send(cpu);
  234. local_irq_restore(flags);
  235. }
  236. static void ipi_send_msg(const struct cpumask *callmap, enum ipi_msg_type msg)
  237. {
  238. unsigned int cpu;
  239. for_each_cpu(cpu, callmap)
  240. ipi_send_msg_one(cpu, msg);
  241. }
  242. void smp_send_reschedule(int cpu)
  243. {
  244. ipi_send_msg_one(cpu, IPI_RESCHEDULE);
  245. }
  246. void smp_send_stop(void)
  247. {
  248. struct cpumask targets;
  249. cpumask_copy(&targets, cpu_online_mask);
  250. cpumask_clear_cpu(smp_processor_id(), &targets);
  251. ipi_send_msg(&targets, IPI_CPU_STOP);
  252. }
  253. void arch_send_call_function_single_ipi(int cpu)
  254. {
  255. ipi_send_msg_one(cpu, IPI_CALL_FUNC);
  256. }
  257. void arch_send_call_function_ipi_mask(const struct cpumask *mask)
  258. {
  259. ipi_send_msg(mask, IPI_CALL_FUNC);
  260. }
  261. /*
  262. * ipi_cpu_stop - handle IPI from smp_send_stop()
  263. */
  264. static void ipi_cpu_stop(void)
  265. {
  266. machine_halt();
  267. }
  268. static inline int __do_IPI(unsigned long msg)
  269. {
  270. int rc = 0;
  271. switch (msg) {
  272. case IPI_RESCHEDULE:
  273. scheduler_ipi();
  274. break;
  275. case IPI_CALL_FUNC:
  276. generic_smp_call_function_interrupt();
  277. break;
  278. case IPI_CPU_STOP:
  279. ipi_cpu_stop();
  280. break;
  281. default:
  282. rc = 1;
  283. }
  284. return rc;
  285. }
  286. /*
  287. * arch-common ISR to handle for inter-processor interrupts
  288. * Has hooks for platform specific IPI
  289. */
  290. irqreturn_t do_IPI(int irq, void *dev_id)
  291. {
  292. unsigned long pending;
  293. unsigned long __maybe_unused copy;
  294. pr_debug("IPI [%ld] received on cpu %d\n",
  295. *this_cpu_ptr(&ipi_data), smp_processor_id());
  296. if (plat_smp_ops.ipi_clear)
  297. plat_smp_ops.ipi_clear(irq);
  298. /*
  299. * "dequeue" the msg corresponding to this IPI (and possibly other
  300. * piggybacked msg from elided IPIs: see ipi_send_msg_one() above)
  301. */
  302. copy = pending = xchg(this_cpu_ptr(&ipi_data), 0);
  303. do {
  304. unsigned long msg = __ffs(pending);
  305. int rc;
  306. rc = __do_IPI(msg);
  307. if (rc)
  308. pr_info("IPI with bogus msg %ld in %ld\n", msg, copy);
  309. pending &= ~(1U << msg);
  310. } while (pending);
  311. return IRQ_HANDLED;
  312. }
  313. /*
  314. * API called by platform code to hookup arch-common ISR to their IPI IRQ
  315. *
  316. * Note: If IPI is provided by platform (vs. say ARC MCIP), their intc setup/map
  317. * function needs to call call irq_set_percpu_devid() for IPI IRQ, otherwise
  318. * request_percpu_irq() below will fail
  319. */
  320. static DEFINE_PER_CPU(int, ipi_dev);
  321. int smp_ipi_irq_setup(int cpu, irq_hw_number_t hwirq)
  322. {
  323. int *dev = per_cpu_ptr(&ipi_dev, cpu);
  324. unsigned int virq = irq_find_mapping(NULL, hwirq);
  325. if (!virq)
  326. panic("Cannot find virq for root domain and hwirq=%lu", hwirq);
  327. /* Boot cpu calls request, all call enable */
  328. if (!cpu) {
  329. int rc;
  330. rc = request_percpu_irq(virq, do_IPI, "IPI Interrupt", dev);
  331. if (rc)
  332. panic("Percpu IRQ request failed for %u\n", virq);
  333. }
  334. enable_percpu_irq(virq, 0);
  335. return 0;
  336. }