smp.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759
  1. /*
  2. * linux/arch/arm/kernel/smp.c
  3. *
  4. * Copyright (C) 2002 ARM Limited, All Rights Reserved.
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. */
  10. #include <linux/module.h>
  11. #include <linux/delay.h>
  12. #include <linux/init.h>
  13. #include <linux/spinlock.h>
  14. #include <linux/sched.h>
  15. #include <linux/interrupt.h>
  16. #include <linux/cache.h>
  17. #include <linux/profile.h>
  18. #include <linux/errno.h>
  19. #include <linux/mm.h>
  20. #include <linux/cpu.h>
  21. #include <linux/smp.h>
  22. #include <linux/seq_file.h>
  23. #include <linux/irq.h>
  24. #include <asm/atomic.h>
  25. #include <asm/cacheflush.h>
  26. #include <asm/cpu.h>
  27. #include <asm/mmu_context.h>
  28. #include <asm/pgtable.h>
  29. #include <asm/pgalloc.h>
  30. #include <asm/processor.h>
  31. #include <asm/tlbflush.h>
  32. #include <asm/ptrace.h>
  33. /*
  34. * bitmask of present and online CPUs.
  35. * The present bitmask indicates that the CPU is physically present.
  36. * The online bitmask indicates that the CPU is up and running.
  37. */
  38. cpumask_t cpu_possible_map;
  39. EXPORT_SYMBOL(cpu_possible_map);
  40. cpumask_t cpu_online_map;
  41. EXPORT_SYMBOL(cpu_online_map);
  42. /*
  43. * as from 2.5, kernels no longer have an init_tasks structure
  44. * so we need some other way of telling a new secondary core
  45. * where to place its SVC stack
  46. */
  47. struct secondary_data secondary_data;
  48. /*
  49. * structures for inter-processor calls
  50. * - A collection of single bit ipi messages.
  51. */
  52. struct ipi_data {
  53. spinlock_t lock;
  54. unsigned long ipi_count;
  55. unsigned long bits;
  56. };
  57. static DEFINE_PER_CPU(struct ipi_data, ipi_data) = {
  58. .lock = SPIN_LOCK_UNLOCKED,
  59. };
  60. enum ipi_msg_type {
  61. IPI_TIMER,
  62. IPI_RESCHEDULE,
  63. IPI_CALL_FUNC,
  64. IPI_CPU_STOP,
  65. };
  66. struct smp_call_struct {
  67. void (*func)(void *info);
  68. void *info;
  69. int wait;
  70. cpumask_t pending;
  71. cpumask_t unfinished;
  72. };
  73. static struct smp_call_struct * volatile smp_call_function_data;
  74. static DEFINE_SPINLOCK(smp_call_function_lock);
  75. int __cpuinit __cpu_up(unsigned int cpu)
  76. {
  77. struct cpuinfo_arm *ci = &per_cpu(cpu_data, cpu);
  78. struct task_struct *idle = ci->idle;
  79. pgd_t *pgd;
  80. pmd_t *pmd;
  81. int ret;
  82. /*
  83. * Spawn a new process manually, if not already done.
  84. * Grab a pointer to its task struct so we can mess with it
  85. */
  86. if (!idle) {
  87. idle = fork_idle(cpu);
  88. if (IS_ERR(idle)) {
  89. printk(KERN_ERR "CPU%u: fork() failed\n", cpu);
  90. return PTR_ERR(idle);
  91. }
  92. ci->idle = idle;
  93. }
  94. /*
  95. * Allocate initial page tables to allow the new CPU to
  96. * enable the MMU safely. This essentially means a set
  97. * of our "standard" page tables, with the addition of
  98. * a 1:1 mapping for the physical address of the kernel.
  99. */
  100. pgd = pgd_alloc(&init_mm);
  101. pmd = pmd_offset(pgd, PHYS_OFFSET);
  102. *pmd = __pmd((PHYS_OFFSET & PGDIR_MASK) |
  103. PMD_TYPE_SECT | PMD_SECT_AP_WRITE);
  104. /*
  105. * We need to tell the secondary core where to find
  106. * its stack and the page tables.
  107. */
  108. secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
  109. secondary_data.pgdir = virt_to_phys(pgd);
  110. wmb();
  111. /*
  112. * Now bring the CPU into our world.
  113. */
  114. ret = boot_secondary(cpu, idle);
  115. if (ret == 0) {
  116. unsigned long timeout;
  117. /*
  118. * CPU was successfully started, wait for it
  119. * to come online or time out.
  120. */
  121. timeout = jiffies + HZ;
  122. while (time_before(jiffies, timeout)) {
  123. if (cpu_online(cpu))
  124. break;
  125. udelay(10);
  126. barrier();
  127. }
  128. if (!cpu_online(cpu))
  129. ret = -EIO;
  130. }
  131. secondary_data.stack = NULL;
  132. secondary_data.pgdir = 0;
  133. *pmd_offset(pgd, PHYS_OFFSET) = __pmd(0);
  134. pgd_free(pgd);
  135. if (ret) {
  136. printk(KERN_CRIT "CPU%u: processor failed to boot\n", cpu);
  137. /*
  138. * FIXME: We need to clean up the new idle thread. --rmk
  139. */
  140. }
  141. return ret;
  142. }
  143. #ifdef CONFIG_HOTPLUG_CPU
  144. /*
  145. * __cpu_disable runs on the processor to be shutdown.
  146. */
  147. int __cpuexit __cpu_disable(void)
  148. {
  149. unsigned int cpu = smp_processor_id();
  150. struct task_struct *p;
  151. int ret;
  152. ret = mach_cpu_disable(cpu);
  153. if (ret)
  154. return ret;
  155. /*
  156. * Take this CPU offline. Once we clear this, we can't return,
  157. * and we must not schedule until we're ready to give up the cpu.
  158. */
  159. cpu_clear(cpu, cpu_online_map);
  160. /*
  161. * OK - migrate IRQs away from this CPU
  162. */
  163. migrate_irqs();
  164. /*
  165. * Stop the local timer for this CPU.
  166. */
  167. local_timer_stop(cpu);
  168. /*
  169. * Flush user cache and TLB mappings, and then remove this CPU
  170. * from the vm mask set of all processes.
  171. */
  172. flush_cache_all();
  173. local_flush_tlb_all();
  174. read_lock(&tasklist_lock);
  175. for_each_process(p) {
  176. if (p->mm)
  177. cpu_clear(cpu, p->mm->cpu_vm_mask);
  178. }
  179. read_unlock(&tasklist_lock);
  180. return 0;
  181. }
  182. /*
  183. * called on the thread which is asking for a CPU to be shutdown -
  184. * waits until shutdown has completed, or it is timed out.
  185. */
  186. void __cpuexit __cpu_die(unsigned int cpu)
  187. {
  188. if (!platform_cpu_kill(cpu))
  189. printk("CPU%u: unable to kill\n", cpu);
  190. }
  191. /*
  192. * Called from the idle thread for the CPU which has been shutdown.
  193. *
  194. * Note that we disable IRQs here, but do not re-enable them
  195. * before returning to the caller. This is also the behaviour
  196. * of the other hotplug-cpu capable cores, so presumably coming
  197. * out of idle fixes this.
  198. */
  199. void __cpuexit cpu_die(void)
  200. {
  201. unsigned int cpu = smp_processor_id();
  202. local_irq_disable();
  203. idle_task_exit();
  204. /*
  205. * actual CPU shutdown procedure is at least platform (if not
  206. * CPU) specific
  207. */
  208. platform_cpu_die(cpu);
  209. /*
  210. * Do not return to the idle loop - jump back to the secondary
  211. * cpu initialisation. There's some initialisation which needs
  212. * to be repeated to undo the effects of taking the CPU offline.
  213. */
  214. __asm__("mov sp, %0\n"
  215. " b secondary_start_kernel"
  216. :
  217. : "r" (task_stack_page(current) + THREAD_SIZE - 8));
  218. }
  219. #endif /* CONFIG_HOTPLUG_CPU */
  220. /*
  221. * This is the secondary CPU boot entry. We're using this CPUs
  222. * idle thread stack, but a set of temporary page tables.
  223. */
  224. asmlinkage void __cpuinit secondary_start_kernel(void)
  225. {
  226. struct mm_struct *mm = &init_mm;
  227. unsigned int cpu = smp_processor_id();
  228. printk("CPU%u: Booted secondary processor\n", cpu);
  229. /*
  230. * All kernel threads share the same mm context; grab a
  231. * reference and switch to it.
  232. */
  233. atomic_inc(&mm->mm_users);
  234. atomic_inc(&mm->mm_count);
  235. current->active_mm = mm;
  236. cpu_set(cpu, mm->cpu_vm_mask);
  237. cpu_switch_mm(mm->pgd, mm);
  238. enter_lazy_tlb(mm, current);
  239. local_flush_tlb_all();
  240. cpu_init();
  241. preempt_disable();
  242. /*
  243. * Give the platform a chance to do its own initialisation.
  244. */
  245. platform_secondary_init(cpu);
  246. /*
  247. * Enable local interrupts.
  248. */
  249. local_irq_enable();
  250. local_fiq_enable();
  251. calibrate_delay();
  252. smp_store_cpu_info(cpu);
  253. /*
  254. * OK, now it's safe to let the boot CPU continue
  255. */
  256. cpu_set(cpu, cpu_online_map);
  257. /*
  258. * Setup local timer for this CPU.
  259. */
  260. local_timer_setup(cpu);
  261. /*
  262. * OK, it's off to the idle thread for us
  263. */
  264. cpu_idle();
  265. }
  266. /*
  267. * Called by both boot and secondaries to move global data into
  268. * per-processor storage.
  269. */
  270. void __cpuinit smp_store_cpu_info(unsigned int cpuid)
  271. {
  272. struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid);
  273. cpu_info->loops_per_jiffy = loops_per_jiffy;
  274. }
  275. void __init smp_cpus_done(unsigned int max_cpus)
  276. {
  277. int cpu;
  278. unsigned long bogosum = 0;
  279. for_each_online_cpu(cpu)
  280. bogosum += per_cpu(cpu_data, cpu).loops_per_jiffy;
  281. printk(KERN_INFO "SMP: Total of %d processors activated "
  282. "(%lu.%02lu BogoMIPS).\n",
  283. num_online_cpus(),
  284. bogosum / (500000/HZ),
  285. (bogosum / (5000/HZ)) % 100);
  286. }
  287. void __init smp_prepare_boot_cpu(void)
  288. {
  289. unsigned int cpu = smp_processor_id();
  290. per_cpu(cpu_data, cpu).idle = current;
  291. }
  292. static void send_ipi_message(cpumask_t callmap, enum ipi_msg_type msg)
  293. {
  294. unsigned long flags;
  295. unsigned int cpu;
  296. local_irq_save(flags);
  297. for_each_cpu_mask(cpu, callmap) {
  298. struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
  299. spin_lock(&ipi->lock);
  300. ipi->bits |= 1 << msg;
  301. spin_unlock(&ipi->lock);
  302. }
  303. /*
  304. * Call the platform specific cross-CPU call function.
  305. */
  306. smp_cross_call(callmap);
  307. local_irq_restore(flags);
  308. }
  309. /*
  310. * You must not call this function with disabled interrupts, from a
  311. * hardware interrupt handler, nor from a bottom half handler.
  312. */
  313. static int smp_call_function_on_cpu(void (*func)(void *info), void *info,
  314. int retry, int wait, cpumask_t callmap)
  315. {
  316. struct smp_call_struct data;
  317. unsigned long timeout;
  318. int ret = 0;
  319. data.func = func;
  320. data.info = info;
  321. data.wait = wait;
  322. cpu_clear(smp_processor_id(), callmap);
  323. if (cpus_empty(callmap))
  324. goto out;
  325. data.pending = callmap;
  326. if (wait)
  327. data.unfinished = callmap;
  328. /*
  329. * try to get the mutex on smp_call_function_data
  330. */
  331. spin_lock(&smp_call_function_lock);
  332. smp_call_function_data = &data;
  333. send_ipi_message(callmap, IPI_CALL_FUNC);
  334. timeout = jiffies + HZ;
  335. while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
  336. barrier();
  337. /*
  338. * did we time out?
  339. */
  340. if (!cpus_empty(data.pending)) {
  341. /*
  342. * this may be causing our panic - report it
  343. */
  344. printk(KERN_CRIT
  345. "CPU%u: smp_call_function timeout for %p(%p)\n"
  346. " callmap %lx pending %lx, %swait\n",
  347. smp_processor_id(), func, info, *cpus_addr(callmap),
  348. *cpus_addr(data.pending), wait ? "" : "no ");
  349. /*
  350. * TRACE
  351. */
  352. timeout = jiffies + (5 * HZ);
  353. while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
  354. barrier();
  355. if (cpus_empty(data.pending))
  356. printk(KERN_CRIT " RESOLVED\n");
  357. else
  358. printk(KERN_CRIT " STILL STUCK\n");
  359. }
  360. /*
  361. * whatever happened, we're done with the data, so release it
  362. */
  363. smp_call_function_data = NULL;
  364. spin_unlock(&smp_call_function_lock);
  365. if (!cpus_empty(data.pending)) {
  366. ret = -ETIMEDOUT;
  367. goto out;
  368. }
  369. if (wait)
  370. while (!cpus_empty(data.unfinished))
  371. barrier();
  372. out:
  373. return 0;
  374. }
  375. int smp_call_function(void (*func)(void *info), void *info, int retry,
  376. int wait)
  377. {
  378. return smp_call_function_on_cpu(func, info, retry, wait,
  379. cpu_online_map);
  380. }
  381. EXPORT_SYMBOL_GPL(smp_call_function);
  382. void show_ipi_list(struct seq_file *p)
  383. {
  384. unsigned int cpu;
  385. seq_puts(p, "IPI:");
  386. for_each_present_cpu(cpu)
  387. seq_printf(p, " %10lu", per_cpu(ipi_data, cpu).ipi_count);
  388. seq_putc(p, '\n');
  389. }
  390. void show_local_irqs(struct seq_file *p)
  391. {
  392. unsigned int cpu;
  393. seq_printf(p, "LOC: ");
  394. for_each_present_cpu(cpu)
  395. seq_printf(p, "%10u ", irq_stat[cpu].local_timer_irqs);
  396. seq_putc(p, '\n');
  397. }
  398. static void ipi_timer(void)
  399. {
  400. irq_enter();
  401. profile_tick(CPU_PROFILING);
  402. update_process_times(user_mode(get_irq_regs()));
  403. irq_exit();
  404. }
  405. #ifdef CONFIG_LOCAL_TIMERS
  406. asmlinkage void do_local_timer(struct pt_regs *regs)
  407. {
  408. struct pt_regs *old_regs = set_irq_regs(regs);
  409. int cpu = smp_processor_id();
  410. if (local_timer_ack()) {
  411. irq_stat[cpu].local_timer_irqs++;
  412. ipi_timer();
  413. }
  414. set_irq_regs(old_regs);
  415. }
  416. #endif
  417. /*
  418. * ipi_call_function - handle IPI from smp_call_function()
  419. *
  420. * Note that we copy data out of the cross-call structure and then
  421. * let the caller know that we're here and have done with their data
  422. */
  423. static void ipi_call_function(unsigned int cpu)
  424. {
  425. struct smp_call_struct *data = smp_call_function_data;
  426. void (*func)(void *info) = data->func;
  427. void *info = data->info;
  428. int wait = data->wait;
  429. cpu_clear(cpu, data->pending);
  430. func(info);
  431. if (wait)
  432. cpu_clear(cpu, data->unfinished);
  433. }
  434. static DEFINE_SPINLOCK(stop_lock);
  435. /*
  436. * ipi_cpu_stop - handle IPI from smp_send_stop()
  437. */
  438. static void ipi_cpu_stop(unsigned int cpu)
  439. {
  440. spin_lock(&stop_lock);
  441. printk(KERN_CRIT "CPU%u: stopping\n", cpu);
  442. dump_stack();
  443. spin_unlock(&stop_lock);
  444. cpu_clear(cpu, cpu_online_map);
  445. local_fiq_disable();
  446. local_irq_disable();
  447. while (1)
  448. cpu_relax();
  449. }
  450. /*
  451. * Main handler for inter-processor interrupts
  452. *
  453. * For ARM, the ipimask now only identifies a single
  454. * category of IPI (Bit 1 IPIs have been replaced by a
  455. * different mechanism):
  456. *
  457. * Bit 0 - Inter-processor function call
  458. */
  459. asmlinkage void do_IPI(struct pt_regs *regs)
  460. {
  461. unsigned int cpu = smp_processor_id();
  462. struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
  463. struct pt_regs *old_regs = set_irq_regs(regs);
  464. ipi->ipi_count++;
  465. for (;;) {
  466. unsigned long msgs;
  467. spin_lock(&ipi->lock);
  468. msgs = ipi->bits;
  469. ipi->bits = 0;
  470. spin_unlock(&ipi->lock);
  471. if (!msgs)
  472. break;
  473. do {
  474. unsigned nextmsg;
  475. nextmsg = msgs & -msgs;
  476. msgs &= ~nextmsg;
  477. nextmsg = ffz(~nextmsg);
  478. switch (nextmsg) {
  479. case IPI_TIMER:
  480. ipi_timer();
  481. break;
  482. case IPI_RESCHEDULE:
  483. /*
  484. * nothing more to do - eveything is
  485. * done on the interrupt return path
  486. */
  487. break;
  488. case IPI_CALL_FUNC:
  489. ipi_call_function(cpu);
  490. break;
  491. case IPI_CPU_STOP:
  492. ipi_cpu_stop(cpu);
  493. break;
  494. default:
  495. printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n",
  496. cpu, nextmsg);
  497. break;
  498. }
  499. } while (msgs);
  500. }
  501. set_irq_regs(old_regs);
  502. }
  503. void smp_send_reschedule(int cpu)
  504. {
  505. send_ipi_message(cpumask_of_cpu(cpu), IPI_RESCHEDULE);
  506. }
  507. void smp_send_timer(void)
  508. {
  509. cpumask_t mask = cpu_online_map;
  510. cpu_clear(smp_processor_id(), mask);
  511. send_ipi_message(mask, IPI_TIMER);
  512. }
  513. void smp_send_stop(void)
  514. {
  515. cpumask_t mask = cpu_online_map;
  516. cpu_clear(smp_processor_id(), mask);
  517. send_ipi_message(mask, IPI_CPU_STOP);
  518. }
  519. /*
  520. * not supported here
  521. */
  522. int __init setup_profiling_timer(unsigned int multiplier)
  523. {
  524. return -EINVAL;
  525. }
  526. static int
  527. on_each_cpu_mask(void (*func)(void *), void *info, int retry, int wait,
  528. cpumask_t mask)
  529. {
  530. int ret = 0;
  531. preempt_disable();
  532. ret = smp_call_function_on_cpu(func, info, retry, wait, mask);
  533. if (cpu_isset(smp_processor_id(), mask))
  534. func(info);
  535. preempt_enable();
  536. return ret;
  537. }
  538. /**********************************************************************/
  539. /*
  540. * TLB operations
  541. */
  542. struct tlb_args {
  543. struct vm_area_struct *ta_vma;
  544. unsigned long ta_start;
  545. unsigned long ta_end;
  546. };
  547. static inline void ipi_flush_tlb_all(void *ignored)
  548. {
  549. local_flush_tlb_all();
  550. }
  551. static inline void ipi_flush_tlb_mm(void *arg)
  552. {
  553. struct mm_struct *mm = (struct mm_struct *)arg;
  554. local_flush_tlb_mm(mm);
  555. }
  556. static inline void ipi_flush_tlb_page(void *arg)
  557. {
  558. struct tlb_args *ta = (struct tlb_args *)arg;
  559. local_flush_tlb_page(ta->ta_vma, ta->ta_start);
  560. }
  561. static inline void ipi_flush_tlb_kernel_page(void *arg)
  562. {
  563. struct tlb_args *ta = (struct tlb_args *)arg;
  564. local_flush_tlb_kernel_page(ta->ta_start);
  565. }
  566. static inline void ipi_flush_tlb_range(void *arg)
  567. {
  568. struct tlb_args *ta = (struct tlb_args *)arg;
  569. local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
  570. }
  571. static inline void ipi_flush_tlb_kernel_range(void *arg)
  572. {
  573. struct tlb_args *ta = (struct tlb_args *)arg;
  574. local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
  575. }
  576. void flush_tlb_all(void)
  577. {
  578. on_each_cpu(ipi_flush_tlb_all, NULL, 1, 1);
  579. }
  580. void flush_tlb_mm(struct mm_struct *mm)
  581. {
  582. cpumask_t mask = mm->cpu_vm_mask;
  583. on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, 1, mask);
  584. }
  585. void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
  586. {
  587. cpumask_t mask = vma->vm_mm->cpu_vm_mask;
  588. struct tlb_args ta;
  589. ta.ta_vma = vma;
  590. ta.ta_start = uaddr;
  591. on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, 1, mask);
  592. }
  593. void flush_tlb_kernel_page(unsigned long kaddr)
  594. {
  595. struct tlb_args ta;
  596. ta.ta_start = kaddr;
  597. on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1, 1);
  598. }
  599. void flush_tlb_range(struct vm_area_struct *vma,
  600. unsigned long start, unsigned long end)
  601. {
  602. cpumask_t mask = vma->vm_mm->cpu_vm_mask;
  603. struct tlb_args ta;
  604. ta.ta_vma = vma;
  605. ta.ta_start = start;
  606. ta.ta_end = end;
  607. on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, 1, mask);
  608. }
  609. void flush_tlb_kernel_range(unsigned long start, unsigned long end)
  610. {
  611. struct tlb_args ta;
  612. ta.ta_start = start;
  613. ta.ta_end = end;
  614. on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1, 1);
  615. }