fault.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504
  1. /*
  2. * Page fault handler for SH with an MMU.
  3. *
  4. * Copyright (C) 1999 Niibe Yutaka
  5. * Copyright (C) 2003 - 2012 Paul Mundt
  6. *
  7. * Based on linux/arch/i386/mm/fault.c:
  8. * Copyright (C) 1995 Linus Torvalds
  9. *
  10. * This file is subject to the terms and conditions of the GNU General Public
  11. * License. See the file "COPYING" in the main directory of this archive
  12. * for more details.
  13. */
  14. #include <linux/kernel.h>
  15. #include <linux/mm.h>
  16. #include <linux/sched/signal.h>
  17. #include <linux/hardirq.h>
  18. #include <linux/kprobes.h>
  19. #include <linux/perf_event.h>
  20. #include <linux/kdebug.h>
  21. #include <linux/uaccess.h>
  22. #include <asm/io_trapped.h>
  23. #include <asm/mmu_context.h>
  24. #include <asm/tlbflush.h>
  25. #include <asm/traps.h>
  26. static void
  27. force_sig_info_fault(int si_signo, int si_code, unsigned long address)
  28. {
  29. force_sig_fault(si_signo, si_code, (void __user *)address);
  30. }
  31. /*
  32. * This is useful to dump out the page tables associated with
  33. * 'addr' in mm 'mm'.
  34. */
  35. static void show_pte(struct mm_struct *mm, unsigned long addr)
  36. {
  37. pgd_t *pgd;
  38. if (mm) {
  39. pgd = mm->pgd;
  40. } else {
  41. pgd = get_TTB();
  42. if (unlikely(!pgd))
  43. pgd = swapper_pg_dir;
  44. }
  45. pr_alert("pgd = %p\n", pgd);
  46. pgd += pgd_index(addr);
  47. pr_alert("[%08lx] *pgd=%0*llx", addr, (u32)(sizeof(*pgd) * 2),
  48. (u64)pgd_val(*pgd));
  49. do {
  50. p4d_t *p4d;
  51. pud_t *pud;
  52. pmd_t *pmd;
  53. pte_t *pte;
  54. if (pgd_none(*pgd))
  55. break;
  56. if (pgd_bad(*pgd)) {
  57. pr_cont("(bad)");
  58. break;
  59. }
  60. p4d = p4d_offset(pgd, addr);
  61. if (PTRS_PER_P4D != 1)
  62. pr_cont(", *p4d=%0*Lx", (u32)(sizeof(*p4d) * 2),
  63. (u64)p4d_val(*p4d));
  64. if (p4d_none(*p4d))
  65. break;
  66. if (p4d_bad(*p4d)) {
  67. pr_cont("(bad)");
  68. break;
  69. }
  70. pud = pud_offset(p4d, addr);
  71. if (PTRS_PER_PUD != 1)
  72. pr_cont(", *pud=%0*llx", (u32)(sizeof(*pud) * 2),
  73. (u64)pud_val(*pud));
  74. if (pud_none(*pud))
  75. break;
  76. if (pud_bad(*pud)) {
  77. pr_cont("(bad)");
  78. break;
  79. }
  80. pmd = pmd_offset(pud, addr);
  81. if (PTRS_PER_PMD != 1)
  82. pr_cont(", *pmd=%0*llx", (u32)(sizeof(*pmd) * 2),
  83. (u64)pmd_val(*pmd));
  84. if (pmd_none(*pmd))
  85. break;
  86. if (pmd_bad(*pmd)) {
  87. pr_cont("(bad)");
  88. break;
  89. }
  90. /* We must not map this if we have highmem enabled */
  91. if (PageHighMem(pfn_to_page(pmd_val(*pmd) >> PAGE_SHIFT)))
  92. break;
  93. pte = pte_offset_kernel(pmd, addr);
  94. pr_cont(", *pte=%0*llx", (u32)(sizeof(*pte) * 2),
  95. (u64)pte_val(*pte));
  96. } while (0);
  97. pr_cont("\n");
  98. }
  99. static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
  100. {
  101. unsigned index = pgd_index(address);
  102. pgd_t *pgd_k;
  103. p4d_t *p4d, *p4d_k;
  104. pud_t *pud, *pud_k;
  105. pmd_t *pmd, *pmd_k;
  106. pgd += index;
  107. pgd_k = init_mm.pgd + index;
  108. if (!pgd_present(*pgd_k))
  109. return NULL;
  110. p4d = p4d_offset(pgd, address);
  111. p4d_k = p4d_offset(pgd_k, address);
  112. if (!p4d_present(*p4d_k))
  113. return NULL;
  114. pud = pud_offset(p4d, address);
  115. pud_k = pud_offset(p4d_k, address);
  116. if (!pud_present(*pud_k))
  117. return NULL;
  118. if (!pud_present(*pud))
  119. set_pud(pud, *pud_k);
  120. pmd = pmd_offset(pud, address);
  121. pmd_k = pmd_offset(pud_k, address);
  122. if (!pmd_present(*pmd_k))
  123. return NULL;
  124. if (!pmd_present(*pmd))
  125. set_pmd(pmd, *pmd_k);
  126. else {
  127. /*
  128. * The page tables are fully synchronised so there must
  129. * be another reason for the fault. Return NULL here to
  130. * signal that we have not taken care of the fault.
  131. */
  132. BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
  133. return NULL;
  134. }
  135. return pmd_k;
  136. }
  137. #ifdef CONFIG_SH_STORE_QUEUES
  138. #define __FAULT_ADDR_LIMIT P3_ADDR_MAX
  139. #else
  140. #define __FAULT_ADDR_LIMIT VMALLOC_END
  141. #endif
  142. /*
  143. * Handle a fault on the vmalloc or module mapping area
  144. */
  145. static noinline int vmalloc_fault(unsigned long address)
  146. {
  147. pgd_t *pgd_k;
  148. pmd_t *pmd_k;
  149. pte_t *pte_k;
  150. /* Make sure we are in vmalloc/module/P3 area: */
  151. if (!(address >= VMALLOC_START && address < __FAULT_ADDR_LIMIT))
  152. return -1;
  153. /*
  154. * Synchronize this task's top level page-table
  155. * with the 'reference' page table.
  156. *
  157. * Do _not_ use "current" here. We might be inside
  158. * an interrupt in the middle of a task switch..
  159. */
  160. pgd_k = get_TTB();
  161. pmd_k = vmalloc_sync_one(pgd_k, address);
  162. if (!pmd_k)
  163. return -1;
  164. pte_k = pte_offset_kernel(pmd_k, address);
  165. if (!pte_present(*pte_k))
  166. return -1;
  167. return 0;
  168. }
  169. static void
  170. show_fault_oops(struct pt_regs *regs, unsigned long address)
  171. {
  172. if (!oops_may_print())
  173. return;
  174. pr_alert("BUG: unable to handle kernel %s at %08lx\n",
  175. address < PAGE_SIZE ? "NULL pointer dereference"
  176. : "paging request",
  177. address);
  178. pr_alert("PC:");
  179. printk_address(regs->pc, 1);
  180. show_pte(NULL, address);
  181. }
  182. static noinline void
  183. no_context(struct pt_regs *regs, unsigned long error_code,
  184. unsigned long address)
  185. {
  186. /* Are we prepared to handle this kernel fault? */
  187. if (fixup_exception(regs))
  188. return;
  189. if (handle_trapped_io(regs, address))
  190. return;
  191. /*
  192. * Oops. The kernel tried to access some bad page. We'll have to
  193. * terminate things with extreme prejudice.
  194. */
  195. bust_spinlocks(1);
  196. show_fault_oops(regs, address);
  197. die("Oops", regs, error_code);
  198. bust_spinlocks(0);
  199. do_exit(SIGKILL);
  200. }
  201. static void
  202. __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
  203. unsigned long address, int si_code)
  204. {
  205. /* User mode accesses just cause a SIGSEGV */
  206. if (user_mode(regs)) {
  207. /*
  208. * It's possible to have interrupts off here:
  209. */
  210. local_irq_enable();
  211. force_sig_info_fault(SIGSEGV, si_code, address);
  212. return;
  213. }
  214. no_context(regs, error_code, address);
  215. }
  216. static noinline void
  217. bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
  218. unsigned long address)
  219. {
  220. __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
  221. }
  222. static void
  223. __bad_area(struct pt_regs *regs, unsigned long error_code,
  224. unsigned long address, int si_code)
  225. {
  226. struct mm_struct *mm = current->mm;
  227. /*
  228. * Something tried to access memory that isn't in our memory map..
  229. * Fix it, but check if it's kernel or user first..
  230. */
  231. mmap_read_unlock(mm);
  232. __bad_area_nosemaphore(regs, error_code, address, si_code);
  233. }
  234. static noinline void
  235. bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address)
  236. {
  237. __bad_area(regs, error_code, address, SEGV_MAPERR);
  238. }
  239. static noinline void
  240. bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
  241. unsigned long address)
  242. {
  243. __bad_area(regs, error_code, address, SEGV_ACCERR);
  244. }
  245. static void
  246. do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address)
  247. {
  248. struct task_struct *tsk = current;
  249. struct mm_struct *mm = tsk->mm;
  250. mmap_read_unlock(mm);
  251. /* Kernel mode? Handle exceptions or die: */
  252. if (!user_mode(regs))
  253. no_context(regs, error_code, address);
  254. force_sig_info_fault(SIGBUS, BUS_ADRERR, address);
  255. }
  256. static noinline int
  257. mm_fault_error(struct pt_regs *regs, unsigned long error_code,
  258. unsigned long address, vm_fault_t fault)
  259. {
  260. /*
  261. * Pagefault was interrupted by SIGKILL. We have no reason to
  262. * continue pagefault.
  263. */
  264. if (fault_signal_pending(fault, regs)) {
  265. if (!user_mode(regs))
  266. no_context(regs, error_code, address);
  267. return 1;
  268. }
  269. /* Release mmap_lock first if necessary */
  270. if (!(fault & VM_FAULT_RETRY))
  271. mmap_read_unlock(current->mm);
  272. if (!(fault & VM_FAULT_ERROR))
  273. return 0;
  274. if (fault & VM_FAULT_OOM) {
  275. /* Kernel mode? Handle exceptions or die: */
  276. if (!user_mode(regs)) {
  277. no_context(regs, error_code, address);
  278. return 1;
  279. }
  280. /*
  281. * We ran out of memory, call the OOM killer, and return the
  282. * userspace (which will retry the fault, or kill us if we got
  283. * oom-killed):
  284. */
  285. pagefault_out_of_memory();
  286. } else {
  287. if (fault & VM_FAULT_SIGBUS)
  288. do_sigbus(regs, error_code, address);
  289. else if (fault & VM_FAULT_SIGSEGV)
  290. bad_area(regs, error_code, address);
  291. else
  292. BUG();
  293. }
  294. return 1;
  295. }
  296. static inline int access_error(int error_code, struct vm_area_struct *vma)
  297. {
  298. if (error_code & FAULT_CODE_WRITE) {
  299. /* write, present and write, not present: */
  300. if (unlikely(!(vma->vm_flags & VM_WRITE)))
  301. return 1;
  302. return 0;
  303. }
  304. /* ITLB miss on NX page */
  305. if (unlikely((error_code & FAULT_CODE_ITLB) &&
  306. !(vma->vm_flags & VM_EXEC)))
  307. return 1;
  308. /* read, not present: */
  309. if (unlikely(!vma_is_accessible(vma)))
  310. return 1;
  311. return 0;
  312. }
  313. static int fault_in_kernel_space(unsigned long address)
  314. {
  315. return address >= TASK_SIZE;
  316. }
  317. /*
  318. * This routine handles page faults. It determines the address,
  319. * and the problem, and then passes it off to one of the appropriate
  320. * routines.
  321. */
  322. asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
  323. unsigned long error_code,
  324. unsigned long address)
  325. {
  326. unsigned long vec;
  327. struct task_struct *tsk;
  328. struct mm_struct *mm;
  329. struct vm_area_struct * vma;
  330. vm_fault_t fault;
  331. unsigned int flags = FAULT_FLAG_DEFAULT;
  332. tsk = current;
  333. mm = tsk->mm;
  334. vec = lookup_exception_vector();
  335. /*
  336. * We fault-in kernel-space virtual memory on-demand. The
  337. * 'reference' page table is init_mm.pgd.
  338. *
  339. * NOTE! We MUST NOT take any locks for this case. We may
  340. * be in an interrupt or a critical region, and should
  341. * only copy the information from the master page table,
  342. * nothing more.
  343. */
  344. if (unlikely(fault_in_kernel_space(address))) {
  345. if (vmalloc_fault(address) >= 0)
  346. return;
  347. if (kprobe_page_fault(regs, vec))
  348. return;
  349. bad_area_nosemaphore(regs, error_code, address);
  350. return;
  351. }
  352. if (unlikely(kprobe_page_fault(regs, vec)))
  353. return;
  354. /* Only enable interrupts if they were on before the fault */
  355. if ((regs->sr & SR_IMASK) != SR_IMASK)
  356. local_irq_enable();
  357. perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
  358. /*
  359. * If we're in an interrupt, have no user context or are running
  360. * with pagefaults disabled then we must not take the fault:
  361. */
  362. if (unlikely(faulthandler_disabled() || !mm)) {
  363. bad_area_nosemaphore(regs, error_code, address);
  364. return;
  365. }
  366. retry:
  367. mmap_read_lock(mm);
  368. vma = find_vma(mm, address);
  369. if (unlikely(!vma)) {
  370. bad_area(regs, error_code, address);
  371. return;
  372. }
  373. if (likely(vma->vm_start <= address))
  374. goto good_area;
  375. if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
  376. bad_area(regs, error_code, address);
  377. return;
  378. }
  379. if (unlikely(expand_stack(vma, address))) {
  380. bad_area(regs, error_code, address);
  381. return;
  382. }
  383. /*
  384. * Ok, we have a good vm_area for this memory access, so
  385. * we can handle it..
  386. */
  387. good_area:
  388. if (unlikely(access_error(error_code, vma))) {
  389. bad_area_access_error(regs, error_code, address);
  390. return;
  391. }
  392. set_thread_fault_code(error_code);
  393. if (user_mode(regs))
  394. flags |= FAULT_FLAG_USER;
  395. if (error_code & FAULT_CODE_WRITE)
  396. flags |= FAULT_FLAG_WRITE;
  397. /*
  398. * If for any reason at all we couldn't handle the fault,
  399. * make sure we exit gracefully rather than endlessly redo
  400. * the fault.
  401. */
  402. fault = handle_mm_fault(vma, address, flags, regs);
  403. if (unlikely(fault & (VM_FAULT_RETRY | VM_FAULT_ERROR)))
  404. if (mm_fault_error(regs, error_code, address, fault))
  405. return;
  406. if (flags & FAULT_FLAG_ALLOW_RETRY) {
  407. if (fault & VM_FAULT_RETRY) {
  408. flags |= FAULT_FLAG_TRIED;
  409. /*
  410. * No need to mmap_read_unlock(mm) as we would
  411. * have already released it in __lock_page_or_retry
  412. * in mm/filemap.c.
  413. */
  414. goto retry;
  415. }
  416. }
  417. mmap_read_unlock(mm);
  418. }