hugetlbpage.c 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * IBM System z Huge TLB Page Support for Kernel.
  4. *
  5. * Copyright IBM Corp. 2007,2020
  6. * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
  7. */
  8. #define KMSG_COMPONENT "hugetlb"
  9. #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  10. #include <linux/mm.h>
  11. #include <linux/hugetlb.h>
  12. #include <linux/mman.h>
  13. #include <linux/sched/mm.h>
  14. #include <linux/security.h>
  15. /*
  16. * If the bit selected by single-bit bitmask "a" is set within "x", move
  17. * it to the position indicated by single-bit bitmask "b".
  18. */
  19. #define move_set_bit(x, a, b) (((x) & (a)) >> ilog2(a) << ilog2(b))
  20. static inline unsigned long __pte_to_rste(pte_t pte)
  21. {
  22. unsigned long rste;
  23. /*
  24. * Convert encoding pte bits pmd / pud bits
  25. * lIR.uswrdy.p dy..R...I...wr
  26. * empty 010.000000.0 -> 00..0...1...00
  27. * prot-none, clean, old 111.000000.1 -> 00..1...1...00
  28. * prot-none, clean, young 111.000001.1 -> 01..1...1...00
  29. * prot-none, dirty, old 111.000010.1 -> 10..1...1...00
  30. * prot-none, dirty, young 111.000011.1 -> 11..1...1...00
  31. * read-only, clean, old 111.000100.1 -> 00..1...1...01
  32. * read-only, clean, young 101.000101.1 -> 01..1...0...01
  33. * read-only, dirty, old 111.000110.1 -> 10..1...1...01
  34. * read-only, dirty, young 101.000111.1 -> 11..1...0...01
  35. * read-write, clean, old 111.001100.1 -> 00..1...1...11
  36. * read-write, clean, young 101.001101.1 -> 01..1...0...11
  37. * read-write, dirty, old 110.001110.1 -> 10..0...1...11
  38. * read-write, dirty, young 100.001111.1 -> 11..0...0...11
  39. * HW-bits: R read-only, I invalid
  40. * SW-bits: p present, y young, d dirty, r read, w write, s special,
  41. * u unused, l large
  42. */
  43. if (pte_present(pte)) {
  44. rste = pte_val(pte) & PAGE_MASK;
  45. rste |= move_set_bit(pte_val(pte), _PAGE_READ,
  46. _SEGMENT_ENTRY_READ);
  47. rste |= move_set_bit(pte_val(pte), _PAGE_WRITE,
  48. _SEGMENT_ENTRY_WRITE);
  49. rste |= move_set_bit(pte_val(pte), _PAGE_INVALID,
  50. _SEGMENT_ENTRY_INVALID);
  51. rste |= move_set_bit(pte_val(pte), _PAGE_PROTECT,
  52. _SEGMENT_ENTRY_PROTECT);
  53. rste |= move_set_bit(pte_val(pte), _PAGE_DIRTY,
  54. _SEGMENT_ENTRY_DIRTY);
  55. rste |= move_set_bit(pte_val(pte), _PAGE_YOUNG,
  56. _SEGMENT_ENTRY_YOUNG);
  57. #ifdef CONFIG_MEM_SOFT_DIRTY
  58. rste |= move_set_bit(pte_val(pte), _PAGE_SOFT_DIRTY,
  59. _SEGMENT_ENTRY_SOFT_DIRTY);
  60. #endif
  61. rste |= move_set_bit(pte_val(pte), _PAGE_NOEXEC,
  62. _SEGMENT_ENTRY_NOEXEC);
  63. } else
  64. rste = _SEGMENT_ENTRY_EMPTY;
  65. return rste;
  66. }
  67. static inline pte_t __rste_to_pte(unsigned long rste)
  68. {
  69. int present;
  70. pte_t pte;
  71. if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
  72. present = pud_present(__pud(rste));
  73. else
  74. present = pmd_present(__pmd(rste));
  75. /*
  76. * Convert encoding pmd / pud bits pte bits
  77. * dy..R...I...wr lIR.uswrdy.p
  78. * empty 00..0...1...00 -> 010.000000.0
  79. * prot-none, clean, old 00..1...1...00 -> 111.000000.1
  80. * prot-none, clean, young 01..1...1...00 -> 111.000001.1
  81. * prot-none, dirty, old 10..1...1...00 -> 111.000010.1
  82. * prot-none, dirty, young 11..1...1...00 -> 111.000011.1
  83. * read-only, clean, old 00..1...1...01 -> 111.000100.1
  84. * read-only, clean, young 01..1...0...01 -> 101.000101.1
  85. * read-only, dirty, old 10..1...1...01 -> 111.000110.1
  86. * read-only, dirty, young 11..1...0...01 -> 101.000111.1
  87. * read-write, clean, old 00..1...1...11 -> 111.001100.1
  88. * read-write, clean, young 01..1...0...11 -> 101.001101.1
  89. * read-write, dirty, old 10..0...1...11 -> 110.001110.1
  90. * read-write, dirty, young 11..0...0...11 -> 100.001111.1
  91. * HW-bits: R read-only, I invalid
  92. * SW-bits: p present, y young, d dirty, r read, w write, s special,
  93. * u unused, l large
  94. */
  95. if (present) {
  96. pte_val(pte) = rste & _SEGMENT_ENTRY_ORIGIN_LARGE;
  97. pte_val(pte) |= _PAGE_LARGE | _PAGE_PRESENT;
  98. pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_READ,
  99. _PAGE_READ);
  100. pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_WRITE,
  101. _PAGE_WRITE);
  102. pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_INVALID,
  103. _PAGE_INVALID);
  104. pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_PROTECT,
  105. _PAGE_PROTECT);
  106. pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_DIRTY,
  107. _PAGE_DIRTY);
  108. pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_YOUNG,
  109. _PAGE_YOUNG);
  110. #ifdef CONFIG_MEM_SOFT_DIRTY
  111. pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_SOFT_DIRTY,
  112. _PAGE_SOFT_DIRTY);
  113. #endif
  114. pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_NOEXEC,
  115. _PAGE_NOEXEC);
  116. } else
  117. pte_val(pte) = _PAGE_INVALID;
  118. return pte;
  119. }
  120. static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste)
  121. {
  122. struct page *page;
  123. unsigned long size, paddr;
  124. if (!mm_uses_skeys(mm) ||
  125. rste & _SEGMENT_ENTRY_INVALID)
  126. return;
  127. if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) {
  128. page = pud_page(__pud(rste));
  129. size = PUD_SIZE;
  130. paddr = rste & PUD_MASK;
  131. } else {
  132. page = pmd_page(__pmd(rste));
  133. size = PMD_SIZE;
  134. paddr = rste & PMD_MASK;
  135. }
  136. if (!test_and_set_bit(PG_arch_1, &page->flags))
  137. __storage_key_init_range(paddr, paddr + size - 1);
  138. }
  139. void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
  140. pte_t *ptep, pte_t pte)
  141. {
  142. unsigned long rste;
  143. rste = __pte_to_rste(pte);
  144. if (!MACHINE_HAS_NX)
  145. rste &= ~_SEGMENT_ENTRY_NOEXEC;
  146. /* Set correct table type for 2G hugepages */
  147. if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) {
  148. if (likely(pte_present(pte)))
  149. rste |= _REGION3_ENTRY_LARGE;
  150. rste |= _REGION_ENTRY_TYPE_R3;
  151. } else if (likely(pte_present(pte)))
  152. rste |= _SEGMENT_ENTRY_LARGE;
  153. clear_huge_pte_skeys(mm, rste);
  154. pte_val(*ptep) = rste;
  155. }
  156. pte_t huge_ptep_get(pte_t *ptep)
  157. {
  158. return __rste_to_pte(pte_val(*ptep));
  159. }
  160. pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
  161. unsigned long addr, pte_t *ptep)
  162. {
  163. pte_t pte = huge_ptep_get(ptep);
  164. pmd_t *pmdp = (pmd_t *) ptep;
  165. pud_t *pudp = (pud_t *) ptep;
  166. if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
  167. pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY));
  168. else
  169. pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
  170. return pte;
  171. }
  172. pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
  173. unsigned long addr, unsigned long sz)
  174. {
  175. pgd_t *pgdp;
  176. p4d_t *p4dp;
  177. pud_t *pudp;
  178. pmd_t *pmdp = NULL;
  179. pgdp = pgd_offset(mm, addr);
  180. p4dp = p4d_alloc(mm, pgdp, addr);
  181. if (p4dp) {
  182. pudp = pud_alloc(mm, p4dp, addr);
  183. if (pudp) {
  184. if (sz == PUD_SIZE)
  185. return (pte_t *) pudp;
  186. else if (sz == PMD_SIZE)
  187. pmdp = pmd_alloc(mm, pudp, addr);
  188. }
  189. }
  190. return (pte_t *) pmdp;
  191. }
  192. pte_t *huge_pte_offset(struct mm_struct *mm,
  193. unsigned long addr, unsigned long sz)
  194. {
  195. pgd_t *pgdp;
  196. p4d_t *p4dp;
  197. pud_t *pudp;
  198. pmd_t *pmdp = NULL;
  199. pgdp = pgd_offset(mm, addr);
  200. if (pgd_present(*pgdp)) {
  201. p4dp = p4d_offset(pgdp, addr);
  202. if (p4d_present(*p4dp)) {
  203. pudp = pud_offset(p4dp, addr);
  204. if (pud_present(*pudp)) {
  205. if (pud_large(*pudp))
  206. return (pte_t *) pudp;
  207. pmdp = pmd_offset(pudp, addr);
  208. }
  209. }
  210. }
  211. return (pte_t *) pmdp;
  212. }
  213. int pmd_huge(pmd_t pmd)
  214. {
  215. return pmd_large(pmd);
  216. }
  217. int pud_huge(pud_t pud)
  218. {
  219. return pud_large(pud);
  220. }
  221. struct page *
  222. follow_huge_pud(struct mm_struct *mm, unsigned long address,
  223. pud_t *pud, int flags)
  224. {
  225. if (flags & FOLL_GET)
  226. return NULL;
  227. return pud_page(*pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
  228. }
  229. bool __init arch_hugetlb_valid_size(unsigned long size)
  230. {
  231. if (MACHINE_HAS_EDAT1 && size == PMD_SIZE)
  232. return true;
  233. else if (MACHINE_HAS_EDAT2 && size == PUD_SIZE)
  234. return true;
  235. else
  236. return false;
  237. }
  238. static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
  239. unsigned long addr, unsigned long len,
  240. unsigned long pgoff, unsigned long flags)
  241. {
  242. struct hstate *h = hstate_file(file);
  243. struct vm_unmapped_area_info info;
  244. info.flags = 0;
  245. info.length = len;
  246. info.low_limit = current->mm->mmap_base;
  247. info.high_limit = TASK_SIZE;
  248. info.align_mask = PAGE_MASK & ~huge_page_mask(h);
  249. info.align_offset = 0;
  250. return vm_unmapped_area(&info);
  251. }
  252. static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
  253. unsigned long addr0, unsigned long len,
  254. unsigned long pgoff, unsigned long flags)
  255. {
  256. struct hstate *h = hstate_file(file);
  257. struct vm_unmapped_area_info info;
  258. unsigned long addr;
  259. info.flags = VM_UNMAPPED_AREA_TOPDOWN;
  260. info.length = len;
  261. info.low_limit = max(PAGE_SIZE, mmap_min_addr);
  262. info.high_limit = current->mm->mmap_base;
  263. info.align_mask = PAGE_MASK & ~huge_page_mask(h);
  264. info.align_offset = 0;
  265. addr = vm_unmapped_area(&info);
  266. /*
  267. * A failed mmap() very likely causes application failure,
  268. * so fall back to the bottom-up function here. This scenario
  269. * can happen with large stack limits and large mmap()
  270. * allocations.
  271. */
  272. if (addr & ~PAGE_MASK) {
  273. VM_BUG_ON(addr != -ENOMEM);
  274. info.flags = 0;
  275. info.low_limit = TASK_UNMAPPED_BASE;
  276. info.high_limit = TASK_SIZE;
  277. addr = vm_unmapped_area(&info);
  278. }
  279. return addr;
  280. }
  281. unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
  282. unsigned long len, unsigned long pgoff, unsigned long flags)
  283. {
  284. struct hstate *h = hstate_file(file);
  285. struct mm_struct *mm = current->mm;
  286. struct vm_area_struct *vma;
  287. if (len & ~huge_page_mask(h))
  288. return -EINVAL;
  289. if (len > TASK_SIZE - mmap_min_addr)
  290. return -ENOMEM;
  291. if (flags & MAP_FIXED) {
  292. if (prepare_hugepage_range(file, addr, len))
  293. return -EINVAL;
  294. goto check_asce_limit;
  295. }
  296. if (addr) {
  297. addr = ALIGN(addr, huge_page_size(h));
  298. vma = find_vma(mm, addr);
  299. if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
  300. (!vma || addr + len <= vm_start_gap(vma)))
  301. goto check_asce_limit;
  302. }
  303. if (mm->get_unmapped_area == arch_get_unmapped_area)
  304. addr = hugetlb_get_unmapped_area_bottomup(file, addr, len,
  305. pgoff, flags);
  306. else
  307. addr = hugetlb_get_unmapped_area_topdown(file, addr, len,
  308. pgoff, flags);
  309. if (offset_in_page(addr))
  310. return addr;
  311. check_asce_limit:
  312. return check_asce_limit(mm, addr, len);
  313. }