leon_smp.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* leon_smp.c: Sparc-Leon SMP support.
  3. *
  4. * based on sun4m_smp.c
  5. * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
  6. * Copyright (C) 2009 Daniel Hellstrom (daniel@gaisler.com) Aeroflex Gaisler AB
  7. * Copyright (C) 2009 Konrad Eisele (konrad@gaisler.com) Aeroflex Gaisler AB
  8. */
  9. #include <asm/head.h>
  10. #include <linux/kernel.h>
  11. #include <linux/sched/mm.h>
  12. #include <linux/threads.h>
  13. #include <linux/smp.h>
  14. #include <linux/interrupt.h>
  15. #include <linux/kernel_stat.h>
  16. #include <linux/of.h>
  17. #include <linux/init.h>
  18. #include <linux/spinlock.h>
  19. #include <linux/mm.h>
  20. #include <linux/swap.h>
  21. #include <linux/profile.h>
  22. #include <linux/pm.h>
  23. #include <linux/delay.h>
  24. #include <linux/gfp.h>
  25. #include <linux/cpu.h>
  26. #include <linux/clockchips.h>
  27. #include <asm/cacheflush.h>
  28. #include <asm/tlbflush.h>
  29. #include <asm/ptrace.h>
  30. #include <linux/atomic.h>
  31. #include <asm/irq_regs.h>
  32. #include <asm/traps.h>
  33. #include <asm/delay.h>
  34. #include <asm/irq.h>
  35. #include <asm/page.h>
  36. #include <asm/oplib.h>
  37. #include <asm/cpudata.h>
  38. #include <asm/asi.h>
  39. #include <asm/leon.h>
  40. #include <asm/leon_amba.h>
  41. #include <asm/timer.h>
  42. #include "kernel.h"
  43. #include "irq.h"
  44. extern ctxd_t *srmmu_ctx_table_phys;
  45. static int smp_processors_ready;
  46. extern volatile unsigned long cpu_callin_map[NR_CPUS];
  47. extern cpumask_t smp_commenced_mask;
  48. void leon_configure_cache_smp(void);
  49. static void leon_ipi_init(void);
  50. /* IRQ number of LEON IPIs */
  51. int leon_ipi_irq = LEON3_IRQ_IPI_DEFAULT;
  52. static inline unsigned long do_swap(volatile unsigned long *ptr,
  53. unsigned long val)
  54. {
  55. __asm__ __volatile__("swapa [%2] %3, %0\n\t" : "=&r"(val)
  56. : "0"(val), "r"(ptr), "i"(ASI_LEON_DCACHE_MISS)
  57. : "memory");
  58. return val;
  59. }
  60. void leon_cpu_pre_starting(void *arg)
  61. {
  62. leon_configure_cache_smp();
  63. }
  64. void leon_cpu_pre_online(void *arg)
  65. {
  66. int cpuid = hard_smp_processor_id();
  67. /* Allow master to continue. The master will then give us the
  68. * go-ahead by setting the smp_commenced_mask and will wait without
  69. * timeouts until our setup is completed fully (signified by
  70. * our bit being set in the cpu_online_mask).
  71. */
  72. do_swap(&cpu_callin_map[cpuid], 1);
  73. local_ops->cache_all();
  74. local_ops->tlb_all();
  75. /* Fix idle thread fields. */
  76. __asm__ __volatile__("ld [%0], %%g6\n\t" : : "r"(&current_set[cpuid])
  77. : "memory" /* paranoid */);
  78. /* Attach to the address space of init_task. */
  79. mmgrab(&init_mm);
  80. current->active_mm = &init_mm;
  81. while (!cpumask_test_cpu(cpuid, &smp_commenced_mask))
  82. mb();
  83. }
  84. /*
  85. * Cycle through the processors asking the PROM to start each one.
  86. */
  87. extern struct linux_prom_registers smp_penguin_ctable;
  88. void leon_configure_cache_smp(void)
  89. {
  90. unsigned long cfg = sparc_leon3_get_dcachecfg();
  91. int me = smp_processor_id();
  92. if (ASI_LEON3_SYSCTRL_CFG_SSIZE(cfg) > 4) {
  93. printk(KERN_INFO "Note: SMP with snooping only works on 4k cache, found %dk(0x%x) on cpu %d, disabling caches\n",
  94. (unsigned int)ASI_LEON3_SYSCTRL_CFG_SSIZE(cfg),
  95. (unsigned int)cfg, (unsigned int)me);
  96. sparc_leon3_disable_cache();
  97. } else {
  98. if (cfg & ASI_LEON3_SYSCTRL_CFG_SNOOPING) {
  99. sparc_leon3_enable_snooping();
  100. } else {
  101. printk(KERN_INFO "Note: You have to enable snooping in the vhdl model cpu %d, disabling caches\n",
  102. me);
  103. sparc_leon3_disable_cache();
  104. }
  105. }
  106. local_ops->cache_all();
  107. local_ops->tlb_all();
  108. }
  109. static void leon_smp_setbroadcast(unsigned int mask)
  110. {
  111. int broadcast =
  112. ((LEON3_BYPASS_LOAD_PA(&(leon3_irqctrl_regs->mpstatus)) >>
  113. LEON3_IRQMPSTATUS_BROADCAST) & 1);
  114. if (!broadcast) {
  115. prom_printf("######## !!!! The irqmp-ctrl must have broadcast enabled, smp wont work !!!!! ####### nr cpus: %d\n",
  116. leon_smp_nrcpus());
  117. if (leon_smp_nrcpus() > 1) {
  118. BUG();
  119. } else {
  120. prom_printf("continue anyway\n");
  121. return;
  122. }
  123. }
  124. LEON_BYPASS_STORE_PA(&(leon3_irqctrl_regs->mpbroadcast), mask);
  125. }
  126. int leon_smp_nrcpus(void)
  127. {
  128. int nrcpu =
  129. ((LEON3_BYPASS_LOAD_PA(&(leon3_irqctrl_regs->mpstatus)) >>
  130. LEON3_IRQMPSTATUS_CPUNR) & 0xf) + 1;
  131. return nrcpu;
  132. }
  133. void __init leon_boot_cpus(void)
  134. {
  135. int nrcpu = leon_smp_nrcpus();
  136. int me = smp_processor_id();
  137. /* Setup IPI */
  138. leon_ipi_init();
  139. printk(KERN_INFO "%d:(%d:%d) cpus mpirq at 0x%x\n", (unsigned int)me,
  140. (unsigned int)nrcpu, (unsigned int)NR_CPUS,
  141. (unsigned int)&(leon3_irqctrl_regs->mpstatus));
  142. leon_enable_irq_cpu(LEON3_IRQ_CROSS_CALL, me);
  143. leon_enable_irq_cpu(LEON3_IRQ_TICKER, me);
  144. leon_enable_irq_cpu(leon_ipi_irq, me);
  145. leon_smp_setbroadcast(1 << LEON3_IRQ_TICKER);
  146. leon_configure_cache_smp();
  147. local_ops->cache_all();
  148. }
  149. int leon_boot_one_cpu(int i, struct task_struct *idle)
  150. {
  151. int timeout;
  152. current_set[i] = task_thread_info(idle);
  153. /* See trampoline.S:leon_smp_cpu_startup for details...
  154. * Initialize the contexts table
  155. * Since the call to prom_startcpu() trashes the structure,
  156. * we need to re-initialize it for each cpu
  157. */
  158. smp_penguin_ctable.which_io = 0;
  159. smp_penguin_ctable.phys_addr = (unsigned int)srmmu_ctx_table_phys;
  160. smp_penguin_ctable.reg_size = 0;
  161. /* whirrr, whirrr, whirrrrrrrrr... */
  162. printk(KERN_INFO "Starting CPU %d : (irqmp: 0x%x)\n", (unsigned int)i,
  163. (unsigned int)&leon3_irqctrl_regs->mpstatus);
  164. local_ops->cache_all();
  165. /* Make sure all IRQs are of from the start for this new CPU */
  166. LEON_BYPASS_STORE_PA(&leon3_irqctrl_regs->mask[i], 0);
  167. /* Wake one CPU */
  168. LEON_BYPASS_STORE_PA(&(leon3_irqctrl_regs->mpstatus), 1 << i);
  169. /* wheee... it's going... */
  170. for (timeout = 0; timeout < 10000; timeout++) {
  171. if (cpu_callin_map[i])
  172. break;
  173. udelay(200);
  174. }
  175. printk(KERN_INFO "Started CPU %d\n", (unsigned int)i);
  176. if (!(cpu_callin_map[i])) {
  177. printk(KERN_ERR "Processor %d is stuck.\n", i);
  178. return -ENODEV;
  179. } else {
  180. leon_enable_irq_cpu(LEON3_IRQ_CROSS_CALL, i);
  181. leon_enable_irq_cpu(LEON3_IRQ_TICKER, i);
  182. leon_enable_irq_cpu(leon_ipi_irq, i);
  183. }
  184. local_ops->cache_all();
  185. return 0;
  186. }
  187. void __init leon_smp_done(void)
  188. {
  189. int i, first;
  190. int *prev;
  191. /* setup cpu list for irq rotation */
  192. first = 0;
  193. prev = &first;
  194. for (i = 0; i < NR_CPUS; i++) {
  195. if (cpu_online(i)) {
  196. *prev = i;
  197. prev = &cpu_data(i).next;
  198. }
  199. }
  200. *prev = first;
  201. local_ops->cache_all();
  202. /* Free unneeded trap tables */
  203. if (!cpu_present(1)) {
  204. free_reserved_page(virt_to_page(&trapbase_cpu1));
  205. }
  206. if (!cpu_present(2)) {
  207. free_reserved_page(virt_to_page(&trapbase_cpu2));
  208. }
  209. if (!cpu_present(3)) {
  210. free_reserved_page(virt_to_page(&trapbase_cpu3));
  211. }
  212. /* Ok, they are spinning and ready to go. */
  213. smp_processors_ready = 1;
  214. }
  215. struct leon_ipi_work {
  216. int single;
  217. int msk;
  218. int resched;
  219. };
  220. static DEFINE_PER_CPU_SHARED_ALIGNED(struct leon_ipi_work, leon_ipi_work);
  221. /* Initialize IPIs on the LEON, in order to save IRQ resources only one IRQ
  222. * is used for all three types of IPIs.
  223. */
  224. static void __init leon_ipi_init(void)
  225. {
  226. int cpu, len;
  227. struct leon_ipi_work *work;
  228. struct property *pp;
  229. struct device_node *rootnp;
  230. struct tt_entry *trap_table;
  231. unsigned long flags;
  232. /* Find IPI IRQ or stick with default value */
  233. rootnp = of_find_node_by_path("/ambapp0");
  234. if (rootnp) {
  235. pp = of_find_property(rootnp, "ipi_num", &len);
  236. if (pp && (*(int *)pp->value))
  237. leon_ipi_irq = *(int *)pp->value;
  238. }
  239. printk(KERN_INFO "leon: SMP IPIs at IRQ %d\n", leon_ipi_irq);
  240. /* Adjust so that we jump directly to smpleon_ipi */
  241. local_irq_save(flags);
  242. trap_table = &sparc_ttable[SP_TRAP_IRQ1 + (leon_ipi_irq - 1)];
  243. trap_table->inst_three += smpleon_ipi - real_irq_entry;
  244. local_ops->cache_all();
  245. local_irq_restore(flags);
  246. for_each_possible_cpu(cpu) {
  247. work = &per_cpu(leon_ipi_work, cpu);
  248. work->single = work->msk = work->resched = 0;
  249. }
  250. }
  251. static void leon_send_ipi(int cpu, int level)
  252. {
  253. unsigned long mask;
  254. mask = leon_get_irqmask(level);
  255. LEON3_BYPASS_STORE_PA(&leon3_irqctrl_regs->force[cpu], mask);
  256. }
  257. static void leon_ipi_single(int cpu)
  258. {
  259. struct leon_ipi_work *work = &per_cpu(leon_ipi_work, cpu);
  260. /* Mark work */
  261. work->single = 1;
  262. /* Generate IRQ on the CPU */
  263. leon_send_ipi(cpu, leon_ipi_irq);
  264. }
  265. static void leon_ipi_mask_one(int cpu)
  266. {
  267. struct leon_ipi_work *work = &per_cpu(leon_ipi_work, cpu);
  268. /* Mark work */
  269. work->msk = 1;
  270. /* Generate IRQ on the CPU */
  271. leon_send_ipi(cpu, leon_ipi_irq);
  272. }
  273. static void leon_ipi_resched(int cpu)
  274. {
  275. struct leon_ipi_work *work = &per_cpu(leon_ipi_work, cpu);
  276. /* Mark work */
  277. work->resched = 1;
  278. /* Generate IRQ on the CPU (any IRQ will cause resched) */
  279. leon_send_ipi(cpu, leon_ipi_irq);
  280. }
  281. void leonsmp_ipi_interrupt(void)
  282. {
  283. struct leon_ipi_work *work = this_cpu_ptr(&leon_ipi_work);
  284. if (work->single) {
  285. work->single = 0;
  286. smp_call_function_single_interrupt();
  287. }
  288. if (work->msk) {
  289. work->msk = 0;
  290. smp_call_function_interrupt();
  291. }
  292. if (work->resched) {
  293. work->resched = 0;
  294. smp_resched_interrupt();
  295. }
  296. }
  297. static struct smp_funcall {
  298. smpfunc_t func;
  299. unsigned long arg1;
  300. unsigned long arg2;
  301. unsigned long arg3;
  302. unsigned long arg4;
  303. unsigned long arg5;
  304. unsigned long processors_in[NR_CPUS]; /* Set when ipi entered. */
  305. unsigned long processors_out[NR_CPUS]; /* Set when ipi exited. */
  306. } ccall_info __attribute__((aligned(8)));
  307. static DEFINE_SPINLOCK(cross_call_lock);
  308. /* Cross calls must be serialized, at least currently. */
  309. static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
  310. unsigned long arg2, unsigned long arg3,
  311. unsigned long arg4)
  312. {
  313. if (smp_processors_ready) {
  314. register int high = NR_CPUS - 1;
  315. unsigned long flags;
  316. spin_lock_irqsave(&cross_call_lock, flags);
  317. {
  318. /* If you make changes here, make sure gcc generates proper code... */
  319. register smpfunc_t f asm("i0") = func;
  320. register unsigned long a1 asm("i1") = arg1;
  321. register unsigned long a2 asm("i2") = arg2;
  322. register unsigned long a3 asm("i3") = arg3;
  323. register unsigned long a4 asm("i4") = arg4;
  324. register unsigned long a5 asm("i5") = 0;
  325. __asm__ __volatile__("std %0, [%6]\n\t"
  326. "std %2, [%6 + 8]\n\t"
  327. "std %4, [%6 + 16]\n\t" : :
  328. "r"(f), "r"(a1), "r"(a2), "r"(a3),
  329. "r"(a4), "r"(a5),
  330. "r"(&ccall_info.func));
  331. }
  332. /* Init receive/complete mapping, plus fire the IPI's off. */
  333. {
  334. register int i;
  335. cpumask_clear_cpu(smp_processor_id(), &mask);
  336. cpumask_and(&mask, cpu_online_mask, &mask);
  337. for (i = 0; i <= high; i++) {
  338. if (cpumask_test_cpu(i, &mask)) {
  339. ccall_info.processors_in[i] = 0;
  340. ccall_info.processors_out[i] = 0;
  341. leon_send_ipi(i, LEON3_IRQ_CROSS_CALL);
  342. }
  343. }
  344. }
  345. {
  346. register int i;
  347. i = 0;
  348. do {
  349. if (!cpumask_test_cpu(i, &mask))
  350. continue;
  351. while (!ccall_info.processors_in[i])
  352. barrier();
  353. } while (++i <= high);
  354. i = 0;
  355. do {
  356. if (!cpumask_test_cpu(i, &mask))
  357. continue;
  358. while (!ccall_info.processors_out[i])
  359. barrier();
  360. } while (++i <= high);
  361. }
  362. spin_unlock_irqrestore(&cross_call_lock, flags);
  363. }
  364. }
  365. /* Running cross calls. */
  366. void leon_cross_call_irq(void)
  367. {
  368. int i = smp_processor_id();
  369. ccall_info.processors_in[i] = 1;
  370. ccall_info.func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3,
  371. ccall_info.arg4, ccall_info.arg5);
  372. ccall_info.processors_out[i] = 1;
  373. }
  374. static const struct sparc32_ipi_ops leon_ipi_ops = {
  375. .cross_call = leon_cross_call,
  376. .resched = leon_ipi_resched,
  377. .single = leon_ipi_single,
  378. .mask_one = leon_ipi_mask_one,
  379. };
  380. void __init leon_init_smp(void)
  381. {
  382. /* Patch ipi15 trap table */
  383. t_nmi[1] = t_nmi[1] + (linux_trap_ipi15_leon - linux_trap_ipi15_sun4m);
  384. sparc32_ipi_ops = &leon_ipi_ops;
  385. }