smp.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627
  1. /*
  2. * Xtensa SMP support functions.
  3. *
  4. * This file is subject to the terms and conditions of the GNU General Public
  5. * License. See the file "COPYING" in the main directory of this archive
  6. * for more details.
  7. *
  8. * Copyright (C) 2008 - 2013 Tensilica Inc.
  9. *
  10. * Chris Zankel <chris@zankel.net>
  11. * Joe Taylor <joe@tensilica.com>
  12. * Pete Delaney <piet@tensilica.com
  13. */
  14. #include <linux/cpu.h>
  15. #include <linux/cpumask.h>
  16. #include <linux/delay.h>
  17. #include <linux/init.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/irqdomain.h>
  20. #include <linux/irq.h>
  21. #include <linux/kdebug.h>
  22. #include <linux/module.h>
  23. #include <linux/sched/mm.h>
  24. #include <linux/sched/hotplug.h>
  25. #include <linux/sched/task_stack.h>
  26. #include <linux/reboot.h>
  27. #include <linux/seq_file.h>
  28. #include <linux/smp.h>
  29. #include <linux/thread_info.h>
  30. #include <asm/cacheflush.h>
  31. #include <asm/kdebug.h>
  32. #include <asm/mmu_context.h>
  33. #include <asm/mxregs.h>
  34. #include <asm/platform.h>
  35. #include <asm/tlbflush.h>
  36. #include <asm/traps.h>
  37. #ifdef CONFIG_SMP
  38. # if XCHAL_HAVE_S32C1I == 0
  39. # error "The S32C1I option is required for SMP."
  40. # endif
  41. #endif
  42. static void system_invalidate_dcache_range(unsigned long start,
  43. unsigned long size);
  44. static void system_flush_invalidate_dcache_range(unsigned long start,
  45. unsigned long size);
  46. /* IPI (Inter Process Interrupt) */
  47. #define IPI_IRQ 0
  48. static irqreturn_t ipi_interrupt(int irq, void *dev_id);
  49. void ipi_init(void)
  50. {
  51. unsigned irq = irq_create_mapping(NULL, IPI_IRQ);
  52. if (request_irq(irq, ipi_interrupt, IRQF_PERCPU, "ipi", NULL))
  53. pr_err("Failed to request irq %u (ipi)\n", irq);
  54. }
  55. static inline unsigned int get_core_count(void)
  56. {
  57. /* Bits 18..21 of SYSCFGID contain the core count minus 1. */
  58. unsigned int syscfgid = get_er(SYSCFGID);
  59. return ((syscfgid >> 18) & 0xf) + 1;
  60. }
  61. static inline int get_core_id(void)
  62. {
  63. /* Bits 0...18 of SYSCFGID contain the core id */
  64. unsigned int core_id = get_er(SYSCFGID);
  65. return core_id & 0x3fff;
  66. }
  67. void __init smp_prepare_cpus(unsigned int max_cpus)
  68. {
  69. unsigned i;
  70. for_each_possible_cpu(i)
  71. set_cpu_present(i, true);
  72. }
  73. void __init smp_init_cpus(void)
  74. {
  75. unsigned i;
  76. unsigned int ncpus = get_core_count();
  77. unsigned int core_id = get_core_id();
  78. pr_info("%s: Core Count = %d\n", __func__, ncpus);
  79. pr_info("%s: Core Id = %d\n", __func__, core_id);
  80. if (ncpus > NR_CPUS) {
  81. ncpus = NR_CPUS;
  82. pr_info("%s: limiting core count by %d\n", __func__, ncpus);
  83. }
  84. for (i = 0; i < ncpus; ++i)
  85. set_cpu_possible(i, true);
  86. }
  87. void __init smp_prepare_boot_cpu(void)
  88. {
  89. unsigned int cpu = smp_processor_id();
  90. BUG_ON(cpu != 0);
  91. cpu_asid_cache(cpu) = ASID_USER_FIRST;
  92. }
  93. void __init smp_cpus_done(unsigned int max_cpus)
  94. {
  95. }
  96. static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */
  97. static DECLARE_COMPLETION(cpu_running);
  98. void secondary_start_kernel(void)
  99. {
  100. struct mm_struct *mm = &init_mm;
  101. unsigned int cpu = smp_processor_id();
  102. init_mmu();
  103. #ifdef CONFIG_DEBUG_MISC
  104. if (boot_secondary_processors == 0) {
  105. pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
  106. __func__, boot_secondary_processors, cpu);
  107. for (;;)
  108. __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
  109. }
  110. pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
  111. __func__, boot_secondary_processors, cpu);
  112. #endif
  113. /* Init EXCSAVE1 */
  114. secondary_trap_init();
  115. /* All kernel threads share the same mm context. */
  116. mmget(mm);
  117. mmgrab(mm);
  118. current->active_mm = mm;
  119. cpumask_set_cpu(cpu, mm_cpumask(mm));
  120. enter_lazy_tlb(mm, current);
  121. trace_hardirqs_off();
  122. calibrate_delay();
  123. notify_cpu_starting(cpu);
  124. secondary_init_irq();
  125. local_timer_setup(cpu);
  126. set_cpu_online(cpu, true);
  127. local_irq_enable();
  128. complete(&cpu_running);
  129. cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
  130. }
  131. static void mx_cpu_start(void *p)
  132. {
  133. unsigned cpu = (unsigned)p;
  134. unsigned long run_stall_mask = get_er(MPSCORE);
  135. set_er(run_stall_mask & ~(1u << cpu), MPSCORE);
  136. pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
  137. __func__, cpu, run_stall_mask, get_er(MPSCORE));
  138. }
  139. static void mx_cpu_stop(void *p)
  140. {
  141. unsigned cpu = (unsigned)p;
  142. unsigned long run_stall_mask = get_er(MPSCORE);
  143. set_er(run_stall_mask | (1u << cpu), MPSCORE);
  144. pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
  145. __func__, cpu, run_stall_mask, get_er(MPSCORE));
  146. }
  147. #ifdef CONFIG_HOTPLUG_CPU
  148. unsigned long cpu_start_id __cacheline_aligned;
  149. #endif
  150. unsigned long cpu_start_ccount;
  151. static int boot_secondary(unsigned int cpu, struct task_struct *ts)
  152. {
  153. unsigned long timeout = jiffies + msecs_to_jiffies(1000);
  154. unsigned long ccount;
  155. int i;
  156. #ifdef CONFIG_HOTPLUG_CPU
  157. WRITE_ONCE(cpu_start_id, cpu);
  158. /* Pairs with the third memw in the cpu_restart */
  159. mb();
  160. system_flush_invalidate_dcache_range((unsigned long)&cpu_start_id,
  161. sizeof(cpu_start_id));
  162. #endif
  163. smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1);
  164. for (i = 0; i < 2; ++i) {
  165. do
  166. ccount = get_ccount();
  167. while (!ccount);
  168. WRITE_ONCE(cpu_start_ccount, ccount);
  169. do {
  170. /*
  171. * Pairs with the first two memws in the
  172. * .Lboot_secondary.
  173. */
  174. mb();
  175. ccount = READ_ONCE(cpu_start_ccount);
  176. } while (ccount && time_before(jiffies, timeout));
  177. if (ccount) {
  178. smp_call_function_single(0, mx_cpu_stop,
  179. (void *)cpu, 1);
  180. WRITE_ONCE(cpu_start_ccount, 0);
  181. return -EIO;
  182. }
  183. }
  184. return 0;
  185. }
  186. int __cpu_up(unsigned int cpu, struct task_struct *idle)
  187. {
  188. int ret = 0;
  189. if (cpu_asid_cache(cpu) == 0)
  190. cpu_asid_cache(cpu) = ASID_USER_FIRST;
  191. start_info.stack = (unsigned long)task_pt_regs(idle);
  192. wmb();
  193. pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n",
  194. __func__, cpu, idle, start_info.stack);
  195. init_completion(&cpu_running);
  196. ret = boot_secondary(cpu, idle);
  197. if (ret == 0) {
  198. wait_for_completion_timeout(&cpu_running,
  199. msecs_to_jiffies(1000));
  200. if (!cpu_online(cpu))
  201. ret = -EIO;
  202. }
  203. if (ret)
  204. pr_err("CPU %u failed to boot\n", cpu);
  205. return ret;
  206. }
  207. #ifdef CONFIG_HOTPLUG_CPU
  208. /*
  209. * __cpu_disable runs on the processor to be shutdown.
  210. */
  211. int __cpu_disable(void)
  212. {
  213. unsigned int cpu = smp_processor_id();
  214. /*
  215. * Take this CPU offline. Once we clear this, we can't return,
  216. * and we must not schedule until we're ready to give up the cpu.
  217. */
  218. set_cpu_online(cpu, false);
  219. /*
  220. * OK - migrate IRQs away from this CPU
  221. */
  222. migrate_irqs();
  223. /*
  224. * Flush user cache and TLB mappings, and then remove this CPU
  225. * from the vm mask set of all processes.
  226. */
  227. local_flush_cache_all();
  228. local_flush_tlb_all();
  229. invalidate_page_directory();
  230. clear_tasks_mm_cpumask(cpu);
  231. return 0;
  232. }
  233. static void platform_cpu_kill(unsigned int cpu)
  234. {
  235. smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true);
  236. }
  237. /*
  238. * called on the thread which is asking for a CPU to be shutdown -
  239. * waits until shutdown has completed, or it is timed out.
  240. */
  241. void __cpu_die(unsigned int cpu)
  242. {
  243. unsigned long timeout = jiffies + msecs_to_jiffies(1000);
  244. while (time_before(jiffies, timeout)) {
  245. system_invalidate_dcache_range((unsigned long)&cpu_start_id,
  246. sizeof(cpu_start_id));
  247. /* Pairs with the second memw in the cpu_restart */
  248. mb();
  249. if (READ_ONCE(cpu_start_id) == -cpu) {
  250. platform_cpu_kill(cpu);
  251. return;
  252. }
  253. }
  254. pr_err("CPU%u: unable to kill\n", cpu);
  255. }
  256. void arch_cpu_idle_dead(void)
  257. {
  258. cpu_die();
  259. }
  260. /*
  261. * Called from the idle thread for the CPU which has been shutdown.
  262. *
  263. * Note that we disable IRQs here, but do not re-enable them
  264. * before returning to the caller. This is also the behaviour
  265. * of the other hotplug-cpu capable cores, so presumably coming
  266. * out of idle fixes this.
  267. */
  268. void __ref cpu_die(void)
  269. {
  270. idle_task_exit();
  271. local_irq_disable();
  272. __asm__ __volatile__(
  273. " movi a2, cpu_restart\n"
  274. " jx a2\n");
  275. }
  276. #endif /* CONFIG_HOTPLUG_CPU */
  277. enum ipi_msg_type {
  278. IPI_RESCHEDULE = 0,
  279. IPI_CALL_FUNC,
  280. IPI_CPU_STOP,
  281. IPI_MAX
  282. };
  283. static const struct {
  284. const char *short_text;
  285. const char *long_text;
  286. } ipi_text[] = {
  287. { .short_text = "RES", .long_text = "Rescheduling interrupts" },
  288. { .short_text = "CAL", .long_text = "Function call interrupts" },
  289. { .short_text = "DIE", .long_text = "CPU shutdown interrupts" },
  290. };
  291. struct ipi_data {
  292. unsigned long ipi_count[IPI_MAX];
  293. };
  294. static DEFINE_PER_CPU(struct ipi_data, ipi_data);
  295. static void send_ipi_message(const struct cpumask *callmask,
  296. enum ipi_msg_type msg_id)
  297. {
  298. int index;
  299. unsigned long mask = 0;
  300. for_each_cpu(index, callmask)
  301. mask |= 1 << index;
  302. set_er(mask, MIPISET(msg_id));
  303. }
  304. void arch_send_call_function_ipi_mask(const struct cpumask *mask)
  305. {
  306. send_ipi_message(mask, IPI_CALL_FUNC);
  307. }
  308. void arch_send_call_function_single_ipi(int cpu)
  309. {
  310. send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
  311. }
  312. void smp_send_reschedule(int cpu)
  313. {
  314. send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
  315. }
  316. void smp_send_stop(void)
  317. {
  318. struct cpumask targets;
  319. cpumask_copy(&targets, cpu_online_mask);
  320. cpumask_clear_cpu(smp_processor_id(), &targets);
  321. send_ipi_message(&targets, IPI_CPU_STOP);
  322. }
  323. static void ipi_cpu_stop(unsigned int cpu)
  324. {
  325. set_cpu_online(cpu, false);
  326. machine_halt();
  327. }
  328. irqreturn_t ipi_interrupt(int irq, void *dev_id)
  329. {
  330. unsigned int cpu = smp_processor_id();
  331. struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
  332. for (;;) {
  333. unsigned int msg;
  334. msg = get_er(MIPICAUSE(cpu));
  335. set_er(msg, MIPICAUSE(cpu));
  336. if (!msg)
  337. break;
  338. if (msg & (1 << IPI_CALL_FUNC)) {
  339. ++ipi->ipi_count[IPI_CALL_FUNC];
  340. generic_smp_call_function_interrupt();
  341. }
  342. if (msg & (1 << IPI_RESCHEDULE)) {
  343. ++ipi->ipi_count[IPI_RESCHEDULE];
  344. scheduler_ipi();
  345. }
  346. if (msg & (1 << IPI_CPU_STOP)) {
  347. ++ipi->ipi_count[IPI_CPU_STOP];
  348. ipi_cpu_stop(cpu);
  349. }
  350. }
  351. return IRQ_HANDLED;
  352. }
  353. void show_ipi_list(struct seq_file *p, int prec)
  354. {
  355. unsigned int cpu;
  356. unsigned i;
  357. for (i = 0; i < IPI_MAX; ++i) {
  358. seq_printf(p, "%*s:", prec, ipi_text[i].short_text);
  359. for_each_online_cpu(cpu)
  360. seq_printf(p, " %10lu",
  361. per_cpu(ipi_data, cpu).ipi_count[i]);
  362. seq_printf(p, " %s\n", ipi_text[i].long_text);
  363. }
  364. }
  365. int setup_profiling_timer(unsigned int multiplier)
  366. {
  367. pr_debug("setup_profiling_timer %d\n", multiplier);
  368. return 0;
  369. }
  370. /* TLB flush functions */
  371. struct flush_data {
  372. struct vm_area_struct *vma;
  373. unsigned long addr1;
  374. unsigned long addr2;
  375. };
  376. static void ipi_flush_tlb_all(void *arg)
  377. {
  378. local_flush_tlb_all();
  379. }
  380. void flush_tlb_all(void)
  381. {
  382. on_each_cpu(ipi_flush_tlb_all, NULL, 1);
  383. }
  384. static void ipi_flush_tlb_mm(void *arg)
  385. {
  386. local_flush_tlb_mm(arg);
  387. }
  388. void flush_tlb_mm(struct mm_struct *mm)
  389. {
  390. on_each_cpu(ipi_flush_tlb_mm, mm, 1);
  391. }
  392. static void ipi_flush_tlb_page(void *arg)
  393. {
  394. struct flush_data *fd = arg;
  395. local_flush_tlb_page(fd->vma, fd->addr1);
  396. }
  397. void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
  398. {
  399. struct flush_data fd = {
  400. .vma = vma,
  401. .addr1 = addr,
  402. };
  403. on_each_cpu(ipi_flush_tlb_page, &fd, 1);
  404. }
  405. static void ipi_flush_tlb_range(void *arg)
  406. {
  407. struct flush_data *fd = arg;
  408. local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
  409. }
  410. void flush_tlb_range(struct vm_area_struct *vma,
  411. unsigned long start, unsigned long end)
  412. {
  413. struct flush_data fd = {
  414. .vma = vma,
  415. .addr1 = start,
  416. .addr2 = end,
  417. };
  418. on_each_cpu(ipi_flush_tlb_range, &fd, 1);
  419. }
  420. static void ipi_flush_tlb_kernel_range(void *arg)
  421. {
  422. struct flush_data *fd = arg;
  423. local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
  424. }
  425. void flush_tlb_kernel_range(unsigned long start, unsigned long end)
  426. {
  427. struct flush_data fd = {
  428. .addr1 = start,
  429. .addr2 = end,
  430. };
  431. on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1);
  432. }
  433. /* Cache flush functions */
  434. static void ipi_flush_cache_all(void *arg)
  435. {
  436. local_flush_cache_all();
  437. }
  438. void flush_cache_all(void)
  439. {
  440. on_each_cpu(ipi_flush_cache_all, NULL, 1);
  441. }
  442. static void ipi_flush_cache_page(void *arg)
  443. {
  444. struct flush_data *fd = arg;
  445. local_flush_cache_page(fd->vma, fd->addr1, fd->addr2);
  446. }
  447. void flush_cache_page(struct vm_area_struct *vma,
  448. unsigned long address, unsigned long pfn)
  449. {
  450. struct flush_data fd = {
  451. .vma = vma,
  452. .addr1 = address,
  453. .addr2 = pfn,
  454. };
  455. on_each_cpu(ipi_flush_cache_page, &fd, 1);
  456. }
  457. static void ipi_flush_cache_range(void *arg)
  458. {
  459. struct flush_data *fd = arg;
  460. local_flush_cache_range(fd->vma, fd->addr1, fd->addr2);
  461. }
  462. void flush_cache_range(struct vm_area_struct *vma,
  463. unsigned long start, unsigned long end)
  464. {
  465. struct flush_data fd = {
  466. .vma = vma,
  467. .addr1 = start,
  468. .addr2 = end,
  469. };
  470. on_each_cpu(ipi_flush_cache_range, &fd, 1);
  471. }
  472. static void ipi_flush_icache_range(void *arg)
  473. {
  474. struct flush_data *fd = arg;
  475. local_flush_icache_range(fd->addr1, fd->addr2);
  476. }
  477. void flush_icache_range(unsigned long start, unsigned long end)
  478. {
  479. struct flush_data fd = {
  480. .addr1 = start,
  481. .addr2 = end,
  482. };
  483. on_each_cpu(ipi_flush_icache_range, &fd, 1);
  484. }
  485. EXPORT_SYMBOL(flush_icache_range);
  486. /* ------------------------------------------------------------------------- */
  487. static void ipi_invalidate_dcache_range(void *arg)
  488. {
  489. struct flush_data *fd = arg;
  490. __invalidate_dcache_range(fd->addr1, fd->addr2);
  491. }
  492. static void system_invalidate_dcache_range(unsigned long start,
  493. unsigned long size)
  494. {
  495. struct flush_data fd = {
  496. .addr1 = start,
  497. .addr2 = size,
  498. };
  499. on_each_cpu(ipi_invalidate_dcache_range, &fd, 1);
  500. }
  501. static void ipi_flush_invalidate_dcache_range(void *arg)
  502. {
  503. struct flush_data *fd = arg;
  504. __flush_invalidate_dcache_range(fd->addr1, fd->addr2);
  505. }
  506. static void system_flush_invalidate_dcache_range(unsigned long start,
  507. unsigned long size)
  508. {
  509. struct flush_data fd = {
  510. .addr1 = start,
  511. .addr2 = size,
  512. };
  513. on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1);
  514. }