fault_64.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * arch/sparc64/mm/fault.c: Page fault handlers for the 64-bit Sparc.
  4. *
  5. * Copyright (C) 1996, 2008 David S. Miller (davem@davemloft.net)
  6. * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
  7. */
  8. #include <asm/head.h>
  9. #include <linux/string.h>
  10. #include <linux/types.h>
  11. #include <linux/sched.h>
  12. #include <linux/sched/debug.h>
  13. #include <linux/ptrace.h>
  14. #include <linux/mman.h>
  15. #include <linux/signal.h>
  16. #include <linux/mm.h>
  17. #include <linux/extable.h>
  18. #include <linux/init.h>
  19. #include <linux/perf_event.h>
  20. #include <linux/interrupt.h>
  21. #include <linux/kprobes.h>
  22. #include <linux/kdebug.h>
  23. #include <linux/percpu.h>
  24. #include <linux/context_tracking.h>
  25. #include <linux/uaccess.h>
  26. #include <asm/page.h>
  27. #include <asm/openprom.h>
  28. #include <asm/oplib.h>
  29. #include <asm/asi.h>
  30. #include <asm/lsu.h>
  31. #include <asm/sections.h>
  32. #include <asm/mmu_context.h>
  33. #include <asm/setup.h>
  34. int show_unhandled_signals = 1;
  35. static void __kprobes unhandled_fault(unsigned long address,
  36. struct task_struct *tsk,
  37. struct pt_regs *regs)
  38. {
  39. if ((unsigned long) address < PAGE_SIZE) {
  40. printk(KERN_ALERT "Unable to handle kernel NULL "
  41. "pointer dereference\n");
  42. } else {
  43. printk(KERN_ALERT "Unable to handle kernel paging request "
  44. "at virtual address %016lx\n", (unsigned long)address);
  45. }
  46. printk(KERN_ALERT "tsk->{mm,active_mm}->context = %016lx\n",
  47. (tsk->mm ?
  48. CTX_HWBITS(tsk->mm->context) :
  49. CTX_HWBITS(tsk->active_mm->context)));
  50. printk(KERN_ALERT "tsk->{mm,active_mm}->pgd = %016lx\n",
  51. (tsk->mm ? (unsigned long) tsk->mm->pgd :
  52. (unsigned long) tsk->active_mm->pgd));
  53. die_if_kernel("Oops", regs);
  54. }
  55. static void __kprobes bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr)
  56. {
  57. printk(KERN_CRIT "OOPS: Bogus kernel PC [%016lx] in fault handler\n",
  58. regs->tpc);
  59. printk(KERN_CRIT "OOPS: RPC [%016lx]\n", regs->u_regs[15]);
  60. printk("OOPS: RPC <%pS>\n", (void *) regs->u_regs[15]);
  61. printk(KERN_CRIT "OOPS: Fault was to vaddr[%lx]\n", vaddr);
  62. dump_stack();
  63. unhandled_fault(regs->tpc, current, regs);
  64. }
  65. /*
  66. * We now make sure that mmap_lock is held in all paths that call
  67. * this. Additionally, to prevent kswapd from ripping ptes from
  68. * under us, raise interrupts around the time that we look at the
  69. * pte, kswapd will have to wait to get his smp ipi response from
  70. * us. vmtruncate likewise. This saves us having to get pte lock.
  71. */
  72. static unsigned int get_user_insn(unsigned long tpc)
  73. {
  74. pgd_t *pgdp = pgd_offset(current->mm, tpc);
  75. p4d_t *p4dp;
  76. pud_t *pudp;
  77. pmd_t *pmdp;
  78. pte_t *ptep, pte;
  79. unsigned long pa;
  80. u32 insn = 0;
  81. if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp)))
  82. goto out;
  83. p4dp = p4d_offset(pgdp, tpc);
  84. if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp)))
  85. goto out;
  86. pudp = pud_offset(p4dp, tpc);
  87. if (pud_none(*pudp) || unlikely(pud_bad(*pudp)))
  88. goto out;
  89. /* This disables preemption for us as well. */
  90. local_irq_disable();
  91. pmdp = pmd_offset(pudp, tpc);
  92. if (pmd_none(*pmdp) || unlikely(pmd_bad(*pmdp)))
  93. goto out_irq_enable;
  94. #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
  95. if (is_hugetlb_pmd(*pmdp)) {
  96. pa = pmd_pfn(*pmdp) << PAGE_SHIFT;
  97. pa += tpc & ~HPAGE_MASK;
  98. /* Use phys bypass so we don't pollute dtlb/dcache. */
  99. __asm__ __volatile__("lduwa [%1] %2, %0"
  100. : "=r" (insn)
  101. : "r" (pa), "i" (ASI_PHYS_USE_EC));
  102. } else
  103. #endif
  104. {
  105. ptep = pte_offset_map(pmdp, tpc);
  106. pte = *ptep;
  107. if (pte_present(pte)) {
  108. pa = (pte_pfn(pte) << PAGE_SHIFT);
  109. pa += (tpc & ~PAGE_MASK);
  110. /* Use phys bypass so we don't pollute dtlb/dcache. */
  111. __asm__ __volatile__("lduwa [%1] %2, %0"
  112. : "=r" (insn)
  113. : "r" (pa), "i" (ASI_PHYS_USE_EC));
  114. }
  115. pte_unmap(ptep);
  116. }
  117. out_irq_enable:
  118. local_irq_enable();
  119. out:
  120. return insn;
  121. }
  122. static inline void
  123. show_signal_msg(struct pt_regs *regs, int sig, int code,
  124. unsigned long address, struct task_struct *tsk)
  125. {
  126. if (!unhandled_signal(tsk, sig))
  127. return;
  128. if (!printk_ratelimit())
  129. return;
  130. printk("%s%s[%d]: segfault at %lx ip %px (rpc %px) sp %px error %x",
  131. task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
  132. tsk->comm, task_pid_nr(tsk), address,
  133. (void *)regs->tpc, (void *)regs->u_regs[UREG_I7],
  134. (void *)regs->u_regs[UREG_FP], code);
  135. print_vma_addr(KERN_CONT " in ", regs->tpc);
  136. printk(KERN_CONT "\n");
  137. }
  138. static void do_fault_siginfo(int code, int sig, struct pt_regs *regs,
  139. unsigned long fault_addr, unsigned int insn,
  140. int fault_code)
  141. {
  142. unsigned long addr;
  143. if (fault_code & FAULT_CODE_ITLB) {
  144. addr = regs->tpc;
  145. } else {
  146. /* If we were able to probe the faulting instruction, use it
  147. * to compute a precise fault address. Otherwise use the fault
  148. * time provided address which may only have page granularity.
  149. */
  150. if (insn)
  151. addr = compute_effective_address(regs, insn, 0);
  152. else
  153. addr = fault_addr;
  154. }
  155. if (unlikely(show_unhandled_signals))
  156. show_signal_msg(regs, sig, code, addr, current);
  157. force_sig_fault(sig, code, (void __user *) addr, 0);
  158. }
  159. static unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn)
  160. {
  161. if (!insn) {
  162. if (!regs->tpc || (regs->tpc & 0x3))
  163. return 0;
  164. if (regs->tstate & TSTATE_PRIV) {
  165. insn = *(unsigned int *) regs->tpc;
  166. } else {
  167. insn = get_user_insn(regs->tpc);
  168. }
  169. }
  170. return insn;
  171. }
  172. static void __kprobes do_kernel_fault(struct pt_regs *regs, int si_code,
  173. int fault_code, unsigned int insn,
  174. unsigned long address)
  175. {
  176. unsigned char asi = ASI_P;
  177. if ((!insn) && (regs->tstate & TSTATE_PRIV))
  178. goto cannot_handle;
  179. /* If user insn could be read (thus insn is zero), that
  180. * is fine. We will just gun down the process with a signal
  181. * in that case.
  182. */
  183. if (!(fault_code & (FAULT_CODE_WRITE|FAULT_CODE_ITLB)) &&
  184. (insn & 0xc0800000) == 0xc0800000) {
  185. if (insn & 0x2000)
  186. asi = (regs->tstate >> 24);
  187. else
  188. asi = (insn >> 5);
  189. if ((asi & 0xf2) == 0x82) {
  190. if (insn & 0x1000000) {
  191. handle_ldf_stq(insn, regs);
  192. } else {
  193. /* This was a non-faulting load. Just clear the
  194. * destination register(s) and continue with the next
  195. * instruction. -jj
  196. */
  197. handle_ld_nf(insn, regs);
  198. }
  199. return;
  200. }
  201. }
  202. /* Is this in ex_table? */
  203. if (regs->tstate & TSTATE_PRIV) {
  204. const struct exception_table_entry *entry;
  205. entry = search_exception_tables(regs->tpc);
  206. if (entry) {
  207. regs->tpc = entry->fixup;
  208. regs->tnpc = regs->tpc + 4;
  209. return;
  210. }
  211. } else {
  212. /* The si_code was set to make clear whether
  213. * this was a SEGV_MAPERR or SEGV_ACCERR fault.
  214. */
  215. do_fault_siginfo(si_code, SIGSEGV, regs, address, insn, fault_code);
  216. return;
  217. }
  218. cannot_handle:
  219. unhandled_fault (address, current, regs);
  220. }
  221. static void noinline __kprobes bogus_32bit_fault_tpc(struct pt_regs *regs)
  222. {
  223. static int times;
  224. if (times++ < 10)
  225. printk(KERN_ERR "FAULT[%s:%d]: 32-bit process reports "
  226. "64-bit TPC [%lx]\n",
  227. current->comm, current->pid,
  228. regs->tpc);
  229. show_regs(regs);
  230. }
  231. asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
  232. {
  233. enum ctx_state prev_state = exception_enter();
  234. struct mm_struct *mm = current->mm;
  235. struct vm_area_struct *vma;
  236. unsigned int insn = 0;
  237. int si_code, fault_code;
  238. vm_fault_t fault;
  239. unsigned long address, mm_rss;
  240. unsigned int flags = FAULT_FLAG_DEFAULT;
  241. fault_code = get_thread_fault_code();
  242. if (kprobe_page_fault(regs, 0))
  243. goto exit_exception;
  244. si_code = SEGV_MAPERR;
  245. address = current_thread_info()->fault_address;
  246. if ((fault_code & FAULT_CODE_ITLB) &&
  247. (fault_code & FAULT_CODE_DTLB))
  248. BUG();
  249. if (test_thread_flag(TIF_32BIT)) {
  250. if (!(regs->tstate & TSTATE_PRIV)) {
  251. if (unlikely((regs->tpc >> 32) != 0)) {
  252. bogus_32bit_fault_tpc(regs);
  253. goto intr_or_no_mm;
  254. }
  255. }
  256. if (unlikely((address >> 32) != 0))
  257. goto intr_or_no_mm;
  258. }
  259. if (regs->tstate & TSTATE_PRIV) {
  260. unsigned long tpc = regs->tpc;
  261. /* Sanity check the PC. */
  262. if ((tpc >= KERNBASE && tpc < (unsigned long) __init_end) ||
  263. (tpc >= MODULES_VADDR && tpc < MODULES_END)) {
  264. /* Valid, no problems... */
  265. } else {
  266. bad_kernel_pc(regs, address);
  267. goto exit_exception;
  268. }
  269. } else
  270. flags |= FAULT_FLAG_USER;
  271. /*
  272. * If we're in an interrupt or have no user
  273. * context, we must not take the fault..
  274. */
  275. if (faulthandler_disabled() || !mm)
  276. goto intr_or_no_mm;
  277. perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
  278. if (!mmap_read_trylock(mm)) {
  279. if ((regs->tstate & TSTATE_PRIV) &&
  280. !search_exception_tables(regs->tpc)) {
  281. insn = get_fault_insn(regs, insn);
  282. goto handle_kernel_fault;
  283. }
  284. retry:
  285. mmap_read_lock(mm);
  286. }
  287. if (fault_code & FAULT_CODE_BAD_RA)
  288. goto do_sigbus;
  289. vma = find_vma(mm, address);
  290. if (!vma)
  291. goto bad_area;
  292. /* Pure DTLB misses do not tell us whether the fault causing
  293. * load/store/atomic was a write or not, it only says that there
  294. * was no match. So in such a case we (carefully) read the
  295. * instruction to try and figure this out. It's an optimization
  296. * so it's ok if we can't do this.
  297. *
  298. * Special hack, window spill/fill knows the exact fault type.
  299. */
  300. if (((fault_code &
  301. (FAULT_CODE_DTLB | FAULT_CODE_WRITE | FAULT_CODE_WINFIXUP)) == FAULT_CODE_DTLB) &&
  302. (vma->vm_flags & VM_WRITE) != 0) {
  303. insn = get_fault_insn(regs, 0);
  304. if (!insn)
  305. goto continue_fault;
  306. /* All loads, stores and atomics have bits 30 and 31 both set
  307. * in the instruction. Bit 21 is set in all stores, but we
  308. * have to avoid prefetches which also have bit 21 set.
  309. */
  310. if ((insn & 0xc0200000) == 0xc0200000 &&
  311. (insn & 0x01780000) != 0x01680000) {
  312. /* Don't bother updating thread struct value,
  313. * because update_mmu_cache only cares which tlb
  314. * the access came from.
  315. */
  316. fault_code |= FAULT_CODE_WRITE;
  317. }
  318. }
  319. continue_fault:
  320. if (vma->vm_start <= address)
  321. goto good_area;
  322. if (!(vma->vm_flags & VM_GROWSDOWN))
  323. goto bad_area;
  324. if (!(fault_code & FAULT_CODE_WRITE)) {
  325. /* Non-faulting loads shouldn't expand stack. */
  326. insn = get_fault_insn(regs, insn);
  327. if ((insn & 0xc0800000) == 0xc0800000) {
  328. unsigned char asi;
  329. if (insn & 0x2000)
  330. asi = (regs->tstate >> 24);
  331. else
  332. asi = (insn >> 5);
  333. if ((asi & 0xf2) == 0x82)
  334. goto bad_area;
  335. }
  336. }
  337. if (expand_stack(vma, address))
  338. goto bad_area;
  339. /*
  340. * Ok, we have a good vm_area for this memory access, so
  341. * we can handle it..
  342. */
  343. good_area:
  344. si_code = SEGV_ACCERR;
  345. /* If we took a ITLB miss on a non-executable page, catch
  346. * that here.
  347. */
  348. if ((fault_code & FAULT_CODE_ITLB) && !(vma->vm_flags & VM_EXEC)) {
  349. WARN(address != regs->tpc,
  350. "address (%lx) != regs->tpc (%lx)\n", address, regs->tpc);
  351. WARN_ON(regs->tstate & TSTATE_PRIV);
  352. goto bad_area;
  353. }
  354. if (fault_code & FAULT_CODE_WRITE) {
  355. if (!(vma->vm_flags & VM_WRITE))
  356. goto bad_area;
  357. /* Spitfire has an icache which does not snoop
  358. * processor stores. Later processors do...
  359. */
  360. if (tlb_type == spitfire &&
  361. (vma->vm_flags & VM_EXEC) != 0 &&
  362. vma->vm_file != NULL)
  363. set_thread_fault_code(fault_code |
  364. FAULT_CODE_BLKCOMMIT);
  365. flags |= FAULT_FLAG_WRITE;
  366. } else {
  367. /* Allow reads even for write-only mappings */
  368. if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
  369. goto bad_area;
  370. }
  371. fault = handle_mm_fault(vma, address, flags, regs);
  372. if (fault_signal_pending(fault, regs))
  373. goto exit_exception;
  374. if (unlikely(fault & VM_FAULT_ERROR)) {
  375. if (fault & VM_FAULT_OOM)
  376. goto out_of_memory;
  377. else if (fault & VM_FAULT_SIGSEGV)
  378. goto bad_area;
  379. else if (fault & VM_FAULT_SIGBUS)
  380. goto do_sigbus;
  381. BUG();
  382. }
  383. if (flags & FAULT_FLAG_ALLOW_RETRY) {
  384. if (fault & VM_FAULT_RETRY) {
  385. flags |= FAULT_FLAG_TRIED;
  386. /* No need to mmap_read_unlock(mm) as we would
  387. * have already released it in __lock_page_or_retry
  388. * in mm/filemap.c.
  389. */
  390. goto retry;
  391. }
  392. }
  393. mmap_read_unlock(mm);
  394. mm_rss = get_mm_rss(mm);
  395. #if defined(CONFIG_TRANSPARENT_HUGEPAGE)
  396. mm_rss -= (mm->context.thp_pte_count * (HPAGE_SIZE / PAGE_SIZE));
  397. #endif
  398. if (unlikely(mm_rss >
  399. mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit))
  400. tsb_grow(mm, MM_TSB_BASE, mm_rss);
  401. #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
  402. mm_rss = mm->context.hugetlb_pte_count + mm->context.thp_pte_count;
  403. mm_rss *= REAL_HPAGE_PER_HPAGE;
  404. if (unlikely(mm_rss >
  405. mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) {
  406. if (mm->context.tsb_block[MM_TSB_HUGE].tsb)
  407. tsb_grow(mm, MM_TSB_HUGE, mm_rss);
  408. else
  409. hugetlb_setup(regs);
  410. }
  411. #endif
  412. exit_exception:
  413. exception_exit(prev_state);
  414. return;
  415. /*
  416. * Something tried to access memory that isn't in our memory map..
  417. * Fix it, but check if it's kernel or user first..
  418. */
  419. bad_area:
  420. insn = get_fault_insn(regs, insn);
  421. mmap_read_unlock(mm);
  422. handle_kernel_fault:
  423. do_kernel_fault(regs, si_code, fault_code, insn, address);
  424. goto exit_exception;
  425. /*
  426. * We ran out of memory, or some other thing happened to us that made
  427. * us unable to handle the page fault gracefully.
  428. */
  429. out_of_memory:
  430. insn = get_fault_insn(regs, insn);
  431. mmap_read_unlock(mm);
  432. if (!(regs->tstate & TSTATE_PRIV)) {
  433. pagefault_out_of_memory();
  434. goto exit_exception;
  435. }
  436. goto handle_kernel_fault;
  437. intr_or_no_mm:
  438. insn = get_fault_insn(regs, 0);
  439. goto handle_kernel_fault;
  440. do_sigbus:
  441. insn = get_fault_insn(regs, insn);
  442. mmap_read_unlock(mm);
  443. /*
  444. * Send a sigbus, regardless of whether we were in kernel
  445. * or user mode.
  446. */
  447. do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, address, insn, fault_code);
  448. /* Kernel mode? Handle exceptions or die */
  449. if (regs->tstate & TSTATE_PRIV)
  450. goto handle_kernel_fault;
  451. }