pgtable.h 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * Copyright (C) 2012 Regents of the University of California
  4. */
  5. #ifndef _ASM_RISCV_PGTABLE_H
  6. #define _ASM_RISCV_PGTABLE_H
  7. #include <linux/mmzone.h>
  8. #include <linux/sizes.h>
  9. #include <asm/pgtable-bits.h>
  10. #ifndef __ASSEMBLY__
  11. /* Page Upper Directory not used in RISC-V */
  12. #include <asm-generic/pgtable-nopud.h>
  13. #include <asm/page.h>
  14. #include <asm/tlbflush.h>
  15. #include <linux/mm_types.h>
  16. #ifdef CONFIG_MMU
  17. #define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
  18. #define VMALLOC_END (PAGE_OFFSET - 1)
  19. #define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE)
  20. #define BPF_JIT_REGION_SIZE (SZ_128M)
  21. #define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE)
  22. #define BPF_JIT_REGION_END (VMALLOC_END)
  23. /*
  24. * Roughly size the vmemmap space to be large enough to fit enough
  25. * struct pages to map half the virtual address space. Then
  26. * position vmemmap directly below the VMALLOC region.
  27. */
  28. #define VMEMMAP_SHIFT \
  29. (CONFIG_VA_BITS - PAGE_SHIFT - 1 + STRUCT_PAGE_MAX_SHIFT)
  30. #define VMEMMAP_SIZE BIT(VMEMMAP_SHIFT)
  31. #define VMEMMAP_END (VMALLOC_START - 1)
  32. #define VMEMMAP_START (VMALLOC_START - VMEMMAP_SIZE)
  33. /*
  34. * Define vmemmap for pfn_to_page & page_to_pfn calls. Needed if kernel
  35. * is configured with CONFIG_SPARSEMEM_VMEMMAP enabled.
  36. */
  37. #define vmemmap ((struct page *)VMEMMAP_START)
  38. #define PCI_IO_SIZE SZ_16M
  39. #define PCI_IO_END VMEMMAP_START
  40. #define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
  41. #define FIXADDR_TOP PCI_IO_START
  42. #ifdef CONFIG_64BIT
  43. #define FIXADDR_SIZE PMD_SIZE
  44. #else
  45. #define FIXADDR_SIZE PGDIR_SIZE
  46. #endif
  47. #define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE)
  48. #endif
  49. #ifdef CONFIG_64BIT
  50. #include <asm/pgtable-64.h>
  51. #else
  52. #include <asm/pgtable-32.h>
  53. #endif /* CONFIG_64BIT */
  54. #ifdef CONFIG_MMU
  55. /* Number of entries in the page global directory */
  56. #define PTRS_PER_PGD (PAGE_SIZE / sizeof(pgd_t))
  57. /* Number of entries in the page table */
  58. #define PTRS_PER_PTE (PAGE_SIZE / sizeof(pte_t))
  59. /* Number of PGD entries that a user-mode program can use */
  60. #define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE)
  61. /* Page protection bits */
  62. #define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_USER | \
  63. _PAGE_SHARE | _PAGE_CACHE | _PAGE_BUF)
  64. #define PAGE_NONE __pgprot(_PAGE_PROT_NONE | _PAGE_CACHE | \
  65. _PAGE_BUF | _PAGE_SHARE | _PAGE_SHARE)
  66. #define PAGE_READ __pgprot(_PAGE_BASE | _PAGE_READ)
  67. #define PAGE_WRITE __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_WRITE)
  68. #define PAGE_EXEC __pgprot(_PAGE_BASE | _PAGE_EXEC)
  69. #define PAGE_READ_EXEC __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC)
  70. #define PAGE_WRITE_EXEC __pgprot(_PAGE_BASE | _PAGE_READ | \
  71. _PAGE_EXEC | _PAGE_WRITE)
  72. #define PAGE_COPY PAGE_READ
  73. #define PAGE_COPY_EXEC PAGE_EXEC
  74. #define PAGE_COPY_READ_EXEC PAGE_READ_EXEC
  75. #define PAGE_SHARED PAGE_WRITE
  76. #define PAGE_SHARED_EXEC PAGE_WRITE_EXEC
  77. #define _PAGE_KERNEL (_PAGE_READ \
  78. | _PAGE_WRITE \
  79. | _PAGE_PRESENT \
  80. | _PAGE_GLOBAL \
  81. | _PAGE_ACCESSED \
  82. | _PAGE_DIRTY \
  83. | _PAGE_CACHE \
  84. | _PAGE_SHARE \
  85. | _PAGE_BUF)
  86. #define PAGE_KERNEL __pgprot(_PAGE_KERNEL)
  87. #define PAGE_KERNEL_READ __pgprot(_PAGE_KERNEL & ~_PAGE_WRITE)
  88. #define PAGE_KERNEL_EXEC __pgprot(_PAGE_KERNEL | _PAGE_EXEC)
  89. #define PAGE_KERNEL_READ_EXEC __pgprot((_PAGE_KERNEL & ~_PAGE_WRITE) \
  90. | _PAGE_EXEC)
  91. #define PAGE_TABLE __pgprot(_PAGE_TABLE)
  92. /*
  93. * The RISC-V ISA doesn't yet specify how to query or modify PMAs, so we can't
  94. * change the properties of memory regions.
  95. */
  96. #define _PAGE_IOREMAP (_PAGE_READ \
  97. | _PAGE_WRITE \
  98. | _PAGE_PRESENT \
  99. | _PAGE_GLOBAL \
  100. | _PAGE_ACCESSED \
  101. | _PAGE_DIRTY \
  102. | _PAGE_SHARE \
  103. | _PAGE_SO)
  104. #define PAGE_KERNEL_IO __pgprot(_PAGE_IOREMAP)
  105. extern pgd_t swapper_pg_dir[];
  106. /* MAP_PRIVATE permissions: xwr (copy-on-write) */
  107. #define __P000 PAGE_NONE
  108. #define __P001 PAGE_READ
  109. #define __P010 PAGE_COPY
  110. #define __P011 PAGE_COPY
  111. #define __P100 PAGE_EXEC
  112. #define __P101 PAGE_READ_EXEC
  113. #define __P110 PAGE_COPY_EXEC
  114. #define __P111 PAGE_COPY_READ_EXEC
  115. /* MAP_SHARED permissions: xwr */
  116. #define __S000 PAGE_NONE
  117. #define __S001 PAGE_READ
  118. #define __S010 PAGE_SHARED
  119. #define __S011 PAGE_SHARED
  120. #define __S100 PAGE_EXEC
  121. #define __S101 PAGE_READ_EXEC
  122. #define __S110 PAGE_SHARED_EXEC
  123. #define __S111 PAGE_SHARED_EXEC
  124. static inline int pmd_present(pmd_t pmd)
  125. {
  126. return (pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROT_NONE));
  127. }
  128. static inline int pmd_none(pmd_t pmd)
  129. {
  130. return (pmd_val(pmd) == 0);
  131. }
  132. static inline int pmd_bad(pmd_t pmd)
  133. {
  134. return !pmd_present(pmd);
  135. }
  136. #define pmd_leaf pmd_leaf
  137. static inline int pmd_leaf(pmd_t pmd)
  138. {
  139. return pmd_present(pmd) &&
  140. (pmd_val(pmd) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC));
  141. }
  142. static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
  143. {
  144. *pmdp = pmd;
  145. }
  146. static inline void pmd_clear(pmd_t *pmdp)
  147. {
  148. set_pmd(pmdp, __pmd(0));
  149. }
  150. static inline pgd_t pfn_pgd(unsigned long pfn, pgprot_t prot)
  151. {
  152. return __pgd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
  153. }
  154. static inline unsigned long _pgd_pfn(pgd_t pgd)
  155. {
  156. return pgd_val(pgd) >> _PAGE_PFN_SHIFT;
  157. }
  158. static inline struct page *pmd_page(pmd_t pmd)
  159. {
  160. return pfn_to_page((pmd_val(pmd) & _PAGE_CHG_MASK) >> _PAGE_PFN_SHIFT);
  161. }
  162. static inline unsigned long pmd_page_vaddr(pmd_t pmd)
  163. {
  164. return (unsigned long)pfn_to_virt((pmd_val(pmd) & _PAGE_CHG_MASK) >> _PAGE_PFN_SHIFT);
  165. }
  166. /* Yields the page frame number (PFN) of a page table entry */
  167. static inline unsigned long pte_pfn(pte_t pte)
  168. {
  169. return ((pte_val(pte) & _PAGE_CHG_MASK) >> _PAGE_PFN_SHIFT);
  170. }
  171. #define pte_page(x) pfn_to_page(pte_pfn(x))
  172. /* Constructs a page table entry */
  173. static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
  174. {
  175. return __pte((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
  176. }
  177. #define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot)
  178. static inline int pte_present(pte_t pte)
  179. {
  180. return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE));
  181. }
  182. static inline int pte_none(pte_t pte)
  183. {
  184. return (pte_val(pte) == 0);
  185. }
  186. static inline int pte_write(pte_t pte)
  187. {
  188. return pte_val(pte) & _PAGE_WRITE;
  189. }
  190. static inline int pte_exec(pte_t pte)
  191. {
  192. return pte_val(pte) & _PAGE_EXEC;
  193. }
  194. static inline int pte_huge(pte_t pte)
  195. {
  196. return pte_present(pte)
  197. && (pte_val(pte) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC));
  198. }
  199. static inline int pte_dirty(pte_t pte)
  200. {
  201. return pte_val(pte) & _PAGE_DIRTY;
  202. }
  203. static inline int pte_young(pte_t pte)
  204. {
  205. return pte_val(pte) & _PAGE_ACCESSED;
  206. }
  207. static inline int pte_special(pte_t pte)
  208. {
  209. return pte_val(pte) & _PAGE_SPECIAL;
  210. }
  211. /* static inline pte_t pte_rdprotect(pte_t pte) */
  212. static inline pte_t pte_wrprotect(pte_t pte)
  213. {
  214. return __pte(pte_val(pte) & ~(_PAGE_WRITE));
  215. }
  216. /* static inline pte_t pte_mkread(pte_t pte) */
  217. static inline pte_t pte_mkwrite(pte_t pte)
  218. {
  219. return __pte(pte_val(pte) | _PAGE_WRITE);
  220. }
  221. /* static inline pte_t pte_mkexec(pte_t pte) */
  222. static inline pte_t pte_mkdirty(pte_t pte)
  223. {
  224. return __pte(pte_val(pte) | _PAGE_DIRTY);
  225. }
  226. static inline pte_t pte_mkclean(pte_t pte)
  227. {
  228. return __pte(pte_val(pte) & ~(_PAGE_DIRTY));
  229. }
  230. static inline pte_t pte_mkyoung(pte_t pte)
  231. {
  232. return __pte(pte_val(pte) | _PAGE_ACCESSED);
  233. }
  234. static inline pte_t pte_mkold(pte_t pte)
  235. {
  236. return __pte(pte_val(pte) & ~(_PAGE_ACCESSED));
  237. }
  238. static inline pte_t pte_mkspecial(pte_t pte)
  239. {
  240. return __pte(pte_val(pte) | _PAGE_SPECIAL);
  241. }
  242. static inline pte_t pte_mkhuge(pte_t pte)
  243. {
  244. return pte;
  245. }
  246. /* Modify page protection bits */
  247. static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
  248. {
  249. return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot));
  250. }
  251. #define pgd_ERROR(e) \
  252. pr_err("%s:%d: bad pgd " PTE_FMT ".\n", __FILE__, __LINE__, pgd_val(e))
  253. /* Commit new configuration to MMU hardware */
  254. static inline void update_mmu_cache(struct vm_area_struct *vma,
  255. unsigned long address, pte_t *ptep)
  256. {
  257. /*
  258. * The kernel assumes that TLBs don't cache invalid entries, but
  259. * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
  260. * cache flush; it is necessary even after writing invalid entries.
  261. * Relying on flush_tlb_fix_spurious_fault would suffice, but
  262. * the extra traps reduce performance. So, eagerly SFENCE.VMA.
  263. */
  264. local_flush_tlb_page(address);
  265. }
  266. #define __HAVE_ARCH_PTE_SAME
  267. static inline int pte_same(pte_t pte_a, pte_t pte_b)
  268. {
  269. return pte_val(pte_a) == pte_val(pte_b);
  270. }
  271. /*
  272. * Certain architectures need to do special things when PTEs within
  273. * a page table are directly modified. Thus, the following hook is
  274. * made available.
  275. */
  276. static inline void set_pte(pte_t *ptep, pte_t pteval)
  277. {
  278. *ptep = pteval;
  279. }
  280. void flush_icache_pte(pte_t pte);
  281. static inline void set_pte_at(struct mm_struct *mm,
  282. unsigned long addr, pte_t *ptep, pte_t pteval)
  283. {
  284. if (pte_present(pteval) && pte_exec(pteval))
  285. flush_icache_pte(pteval);
  286. set_pte(ptep, pteval);
  287. }
  288. static inline void pte_clear(struct mm_struct *mm,
  289. unsigned long addr, pte_t *ptep)
  290. {
  291. set_pte_at(mm, addr, ptep, __pte(0));
  292. }
  293. #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
  294. static inline int ptep_set_access_flags(struct vm_area_struct *vma,
  295. unsigned long address, pte_t *ptep,
  296. pte_t entry, int dirty)
  297. {
  298. if (!pte_same(*ptep, entry))
  299. set_pte_at(vma->vm_mm, address, ptep, entry);
  300. /*
  301. * update_mmu_cache will unconditionally execute, handling both
  302. * the case that the PTE changed and the spurious fault case.
  303. */
  304. return true;
  305. }
  306. #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
  307. static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
  308. unsigned long address, pte_t *ptep)
  309. {
  310. return __pte(atomic_long_xchg((atomic_long_t *)ptep, 0));
  311. }
  312. #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
  313. static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
  314. unsigned long address,
  315. pte_t *ptep)
  316. {
  317. if (!pte_young(*ptep))
  318. return 0;
  319. return test_and_clear_bit(_PAGE_ACCESSED_OFFSET, &pte_val(*ptep));
  320. }
  321. #define __HAVE_ARCH_PTEP_SET_WRPROTECT
  322. static inline void ptep_set_wrprotect(struct mm_struct *mm,
  323. unsigned long address, pte_t *ptep)
  324. {
  325. atomic_long_and(~(unsigned long)_PAGE_WRITE, (atomic_long_t *)ptep);
  326. }
  327. #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
  328. static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
  329. unsigned long address, pte_t *ptep)
  330. {
  331. /*
  332. * This comment is borrowed from x86, but applies equally to RISC-V:
  333. *
  334. * Clearing the accessed bit without a TLB flush
  335. * doesn't cause data corruption. [ It could cause incorrect
  336. * page aging and the (mistaken) reclaim of hot pages, but the
  337. * chance of that should be relatively low. ]
  338. *
  339. * So as a performance optimization don't flush the TLB when
  340. * clearing the accessed bit, it will eventually be flushed by
  341. * a context switch or a VM operation anyway. [ In the rare
  342. * event of it not getting flushed for a long time the delay
  343. * shouldn't really matter because there's no real memory
  344. * pressure for swapout to react to. ]
  345. */
  346. return ptep_test_and_clear_young(vma, address, ptep);
  347. }
  348. #define __HAVE_PHYS_MEM_ACCESS_PROT
  349. struct file;
  350. extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
  351. unsigned long size, pgprot_t vma_prot);
  352. #define pgprot_noncached pgprot_noncached
  353. static inline pgprot_t pgprot_noncached(pgprot_t _prot)
  354. {
  355. unsigned long prot = pgprot_val(_prot);
  356. prot &= ~(_PAGE_CACHE | _PAGE_BUF);
  357. prot |= _PAGE_SO;
  358. return __pgprot(prot);
  359. }
  360. #define pgprot_writecombine pgprot_writecombine
  361. static inline pgprot_t pgprot_writecombine(pgprot_t _prot)
  362. {
  363. unsigned long prot = pgprot_val(_prot);
  364. prot &= ~(_PAGE_CACHE | _PAGE_BUF);
  365. return __pgprot(prot);
  366. }
  367. /*
  368. * Encode and decode a swap entry
  369. *
  370. * Format of swap PTE:
  371. * bit 0: _PAGE_PRESENT (zero)
  372. * bit 1: _PAGE_PROT_NONE (zero)
  373. * bits 2 to 6: swap type
  374. * bits 7 to XLEN-1: swap offset
  375. */
  376. #define __SWP_TYPE_SHIFT 2
  377. #define __SWP_TYPE_BITS 5
  378. #define __SWP_TYPE_MASK ((1UL << __SWP_TYPE_BITS) - 1)
  379. #define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT)
  380. #define MAX_SWAPFILES_CHECK() \
  381. BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS)
  382. #define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK)
  383. #define __swp_offset(x) ((x).val >> __SWP_OFFSET_SHIFT)
  384. #define __swp_entry(type, offset) ((swp_entry_t) \
  385. { ((type) << __SWP_TYPE_SHIFT) | ((offset) << __SWP_OFFSET_SHIFT) })
  386. #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
  387. #define __swp_entry_to_pte(x) ((pte_t) { (x).val })
  388. /*
  389. * In the RV64 Linux scheme, we give the user half of the virtual-address space
  390. * and give the kernel the other (upper) half.
  391. */
  392. #ifdef CONFIG_64BIT
  393. #define KERN_VIRT_START (-(BIT(CONFIG_VA_BITS)) + TASK_SIZE)
  394. #else
  395. #define KERN_VIRT_START FIXADDR_START
  396. #endif
  397. /*
  398. * Task size is 0x4000000000 for RV64 or 0x9fc00000 for RV32.
  399. * Note that PGDIR_SIZE must evenly divide TASK_SIZE.
  400. */
  401. #ifdef CONFIG_64BIT
  402. #define TASK_SIZE_64 (PGDIR_SIZE * PTRS_PER_PGD / 2)
  403. #ifdef CONFIG_COMPAT
  404. #define TASK_SIZE_32 (_AC(0x80000000, UL) - PAGE_SIZE)
  405. #define TASK_SIZE (test_thread_flag(TIF_32BIT) ? \
  406. TASK_SIZE_32 : TASK_SIZE_64)
  407. #else
  408. #define TASK_SIZE TASK_SIZE_64
  409. #endif
  410. #else
  411. #define TASK_SIZE FIXADDR_START
  412. #endif
  413. #else /* CONFIG_MMU */
  414. #define PAGE_SHARED __pgprot(0)
  415. #define PAGE_KERNEL __pgprot(0)
  416. #define swapper_pg_dir NULL
  417. #define TASK_SIZE 0xffffffffUL
  418. #define VMALLOC_START 0
  419. #define VMALLOC_END TASK_SIZE
  420. static inline void __kernel_map_pages(struct page *page, int numpages, int enable) {}
  421. #endif /* !CONFIG_MMU */
  422. #define kern_addr_valid(addr) (1) /* FIXME */
  423. extern void *dtb_early_va;
  424. extern uintptr_t dtb_early_pa;
  425. void setup_bootmem(void);
  426. void paging_init(void);
  427. #define FIRST_USER_ADDRESS 0
  428. /*
  429. * ZERO_PAGE is a global shared page that is always zero,
  430. * used for zero-mapped memory areas, etc.
  431. */
  432. extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
  433. #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
  434. #endif /* !__ASSEMBLY__ */
  435. #endif /* _ASM_RISCV_PGTABLE_H */