hugetlbpage.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * SPARC64 Huge TLB page support.
  4. *
  5. * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net)
  6. */
  7. #include <linux/fs.h>
  8. #include <linux/mm.h>
  9. #include <linux/sched/mm.h>
  10. #include <linux/hugetlb.h>
  11. #include <linux/pagemap.h>
  12. #include <linux/sysctl.h>
  13. #include <asm/mman.h>
  14. #include <asm/pgalloc.h>
  15. #include <asm/tlb.h>
  16. #include <asm/tlbflush.h>
  17. #include <asm/cacheflush.h>
  18. #include <asm/mmu_context.h>
  19. /* Slightly simplified from the non-hugepage variant because by
  20. * definition we don't have to worry about any page coloring stuff
  21. */
  22. static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp,
  23. unsigned long addr,
  24. unsigned long len,
  25. unsigned long pgoff,
  26. unsigned long flags)
  27. {
  28. struct hstate *h = hstate_file(filp);
  29. unsigned long task_size = TASK_SIZE;
  30. struct vm_unmapped_area_info info;
  31. if (test_thread_flag(TIF_32BIT))
  32. task_size = STACK_TOP32;
  33. info.flags = 0;
  34. info.length = len;
  35. info.low_limit = TASK_UNMAPPED_BASE;
  36. info.high_limit = min(task_size, VA_EXCLUDE_START);
  37. info.align_mask = PAGE_MASK & ~huge_page_mask(h);
  38. info.align_offset = 0;
  39. addr = vm_unmapped_area(&info);
  40. if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) {
  41. VM_BUG_ON(addr != -ENOMEM);
  42. info.low_limit = VA_EXCLUDE_END;
  43. info.high_limit = task_size;
  44. addr = vm_unmapped_area(&info);
  45. }
  46. return addr;
  47. }
  48. static unsigned long
  49. hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
  50. const unsigned long len,
  51. const unsigned long pgoff,
  52. const unsigned long flags)
  53. {
  54. struct hstate *h = hstate_file(filp);
  55. struct mm_struct *mm = current->mm;
  56. unsigned long addr = addr0;
  57. struct vm_unmapped_area_info info;
  58. /* This should only ever run for 32-bit processes. */
  59. BUG_ON(!test_thread_flag(TIF_32BIT));
  60. info.flags = VM_UNMAPPED_AREA_TOPDOWN;
  61. info.length = len;
  62. info.low_limit = PAGE_SIZE;
  63. info.high_limit = mm->mmap_base;
  64. info.align_mask = PAGE_MASK & ~huge_page_mask(h);
  65. info.align_offset = 0;
  66. addr = vm_unmapped_area(&info);
  67. /*
  68. * A failed mmap() very likely causes application failure,
  69. * so fall back to the bottom-up function here. This scenario
  70. * can happen with large stack limits and large mmap()
  71. * allocations.
  72. */
  73. if (addr & ~PAGE_MASK) {
  74. VM_BUG_ON(addr != -ENOMEM);
  75. info.flags = 0;
  76. info.low_limit = TASK_UNMAPPED_BASE;
  77. info.high_limit = STACK_TOP32;
  78. addr = vm_unmapped_area(&info);
  79. }
  80. return addr;
  81. }
  82. unsigned long
  83. hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
  84. unsigned long len, unsigned long pgoff, unsigned long flags)
  85. {
  86. struct hstate *h = hstate_file(file);
  87. struct mm_struct *mm = current->mm;
  88. struct vm_area_struct *vma;
  89. unsigned long task_size = TASK_SIZE;
  90. if (test_thread_flag(TIF_32BIT))
  91. task_size = STACK_TOP32;
  92. if (len & ~huge_page_mask(h))
  93. return -EINVAL;
  94. if (len > task_size)
  95. return -ENOMEM;
  96. if (flags & MAP_FIXED) {
  97. if (prepare_hugepage_range(file, addr, len))
  98. return -EINVAL;
  99. return addr;
  100. }
  101. if (addr) {
  102. addr = ALIGN(addr, huge_page_size(h));
  103. vma = find_vma(mm, addr);
  104. if (task_size - len >= addr &&
  105. (!vma || addr + len <= vm_start_gap(vma)))
  106. return addr;
  107. }
  108. if (mm->get_unmapped_area == arch_get_unmapped_area)
  109. return hugetlb_get_unmapped_area_bottomup(file, addr, len,
  110. pgoff, flags);
  111. else
  112. return hugetlb_get_unmapped_area_topdown(file, addr, len,
  113. pgoff, flags);
  114. }
  115. static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
  116. {
  117. return entry;
  118. }
  119. static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
  120. {
  121. unsigned long hugepage_size = _PAGE_SZ4MB_4V;
  122. pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V;
  123. switch (shift) {
  124. case HPAGE_16GB_SHIFT:
  125. hugepage_size = _PAGE_SZ16GB_4V;
  126. pte_val(entry) |= _PAGE_PUD_HUGE;
  127. break;
  128. case HPAGE_2GB_SHIFT:
  129. hugepage_size = _PAGE_SZ2GB_4V;
  130. pte_val(entry) |= _PAGE_PMD_HUGE;
  131. break;
  132. case HPAGE_256MB_SHIFT:
  133. hugepage_size = _PAGE_SZ256MB_4V;
  134. pte_val(entry) |= _PAGE_PMD_HUGE;
  135. break;
  136. case HPAGE_SHIFT:
  137. pte_val(entry) |= _PAGE_PMD_HUGE;
  138. break;
  139. case HPAGE_64K_SHIFT:
  140. hugepage_size = _PAGE_SZ64K_4V;
  141. break;
  142. default:
  143. WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift);
  144. }
  145. pte_val(entry) = pte_val(entry) | hugepage_size;
  146. return entry;
  147. }
  148. static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int shift)
  149. {
  150. if (tlb_type == hypervisor)
  151. return sun4v_hugepage_shift_to_tte(entry, shift);
  152. else
  153. return sun4u_hugepage_shift_to_tte(entry, shift);
  154. }
  155. pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
  156. struct page *page, int writeable)
  157. {
  158. unsigned int shift = huge_page_shift(hstate_vma(vma));
  159. pte_t pte;
  160. pte = hugepage_shift_to_tte(entry, shift);
  161. #ifdef CONFIG_SPARC64
  162. /* If this vma has ADI enabled on it, turn on TTE.mcd
  163. */
  164. if (vma->vm_flags & VM_SPARC_ADI)
  165. return pte_mkmcd(pte);
  166. else
  167. return pte_mknotmcd(pte);
  168. #else
  169. return pte;
  170. #endif
  171. }
  172. static unsigned int sun4v_huge_tte_to_shift(pte_t entry)
  173. {
  174. unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4V;
  175. unsigned int shift;
  176. switch (tte_szbits) {
  177. case _PAGE_SZ16GB_4V:
  178. shift = HPAGE_16GB_SHIFT;
  179. break;
  180. case _PAGE_SZ2GB_4V:
  181. shift = HPAGE_2GB_SHIFT;
  182. break;
  183. case _PAGE_SZ256MB_4V:
  184. shift = HPAGE_256MB_SHIFT;
  185. break;
  186. case _PAGE_SZ4MB_4V:
  187. shift = REAL_HPAGE_SHIFT;
  188. break;
  189. case _PAGE_SZ64K_4V:
  190. shift = HPAGE_64K_SHIFT;
  191. break;
  192. default:
  193. shift = PAGE_SHIFT;
  194. break;
  195. }
  196. return shift;
  197. }
  198. static unsigned int sun4u_huge_tte_to_shift(pte_t entry)
  199. {
  200. unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4U;
  201. unsigned int shift;
  202. switch (tte_szbits) {
  203. case _PAGE_SZ256MB_4U:
  204. shift = HPAGE_256MB_SHIFT;
  205. break;
  206. case _PAGE_SZ4MB_4U:
  207. shift = REAL_HPAGE_SHIFT;
  208. break;
  209. case _PAGE_SZ64K_4U:
  210. shift = HPAGE_64K_SHIFT;
  211. break;
  212. default:
  213. shift = PAGE_SHIFT;
  214. break;
  215. }
  216. return shift;
  217. }
  218. static unsigned int huge_tte_to_shift(pte_t entry)
  219. {
  220. unsigned long shift;
  221. if (tlb_type == hypervisor)
  222. shift = sun4v_huge_tte_to_shift(entry);
  223. else
  224. shift = sun4u_huge_tte_to_shift(entry);
  225. if (shift == PAGE_SHIFT)
  226. WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n",
  227. pte_val(entry));
  228. return shift;
  229. }
  230. static unsigned long huge_tte_to_size(pte_t pte)
  231. {
  232. unsigned long size = 1UL << huge_tte_to_shift(pte);
  233. if (size == REAL_HPAGE_SIZE)
  234. size = HPAGE_SIZE;
  235. return size;
  236. }
  237. pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
  238. unsigned long addr, unsigned long sz)
  239. {
  240. pgd_t *pgd;
  241. p4d_t *p4d;
  242. pud_t *pud;
  243. pmd_t *pmd;
  244. pgd = pgd_offset(mm, addr);
  245. p4d = p4d_offset(pgd, addr);
  246. pud = pud_alloc(mm, p4d, addr);
  247. if (!pud)
  248. return NULL;
  249. if (sz >= PUD_SIZE)
  250. return (pte_t *)pud;
  251. pmd = pmd_alloc(mm, pud, addr);
  252. if (!pmd)
  253. return NULL;
  254. if (sz >= PMD_SIZE)
  255. return (pte_t *)pmd;
  256. return pte_alloc_map(mm, pmd, addr);
  257. }
  258. pte_t *huge_pte_offset(struct mm_struct *mm,
  259. unsigned long addr, unsigned long sz)
  260. {
  261. pgd_t *pgd;
  262. p4d_t *p4d;
  263. pud_t *pud;
  264. pmd_t *pmd;
  265. pgd = pgd_offset(mm, addr);
  266. if (pgd_none(*pgd))
  267. return NULL;
  268. p4d = p4d_offset(pgd, addr);
  269. if (p4d_none(*p4d))
  270. return NULL;
  271. pud = pud_offset(p4d, addr);
  272. if (pud_none(*pud))
  273. return NULL;
  274. if (is_hugetlb_pud(*pud))
  275. return (pte_t *)pud;
  276. pmd = pmd_offset(pud, addr);
  277. if (pmd_none(*pmd))
  278. return NULL;
  279. if (is_hugetlb_pmd(*pmd))
  280. return (pte_t *)pmd;
  281. return pte_offset_map(pmd, addr);
  282. }
  283. void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
  284. pte_t *ptep, pte_t entry)
  285. {
  286. unsigned int nptes, orig_shift, shift;
  287. unsigned long i, size;
  288. pte_t orig;
  289. size = huge_tte_to_size(entry);
  290. shift = PAGE_SHIFT;
  291. if (size >= PUD_SIZE)
  292. shift = PUD_SHIFT;
  293. else if (size >= PMD_SIZE)
  294. shift = PMD_SHIFT;
  295. else
  296. shift = PAGE_SHIFT;
  297. nptes = size >> shift;
  298. if (!pte_present(*ptep) && pte_present(entry))
  299. mm->context.hugetlb_pte_count += nptes;
  300. addr &= ~(size - 1);
  301. orig = *ptep;
  302. orig_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig);
  303. for (i = 0; i < nptes; i++)
  304. ptep[i] = __pte(pte_val(entry) + (i << shift));
  305. maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift);
  306. /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
  307. if (size == HPAGE_SIZE)
  308. maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0,
  309. orig_shift);
  310. }
  311. pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
  312. pte_t *ptep)
  313. {
  314. unsigned int i, nptes, orig_shift, shift;
  315. unsigned long size;
  316. pte_t entry;
  317. entry = *ptep;
  318. size = huge_tte_to_size(entry);
  319. shift = PAGE_SHIFT;
  320. if (size >= PUD_SIZE)
  321. shift = PUD_SHIFT;
  322. else if (size >= PMD_SIZE)
  323. shift = PMD_SHIFT;
  324. else
  325. shift = PAGE_SHIFT;
  326. nptes = size >> shift;
  327. orig_shift = pte_none(entry) ? PAGE_SHIFT : huge_tte_to_shift(entry);
  328. if (pte_present(entry))
  329. mm->context.hugetlb_pte_count -= nptes;
  330. addr &= ~(size - 1);
  331. for (i = 0; i < nptes; i++)
  332. ptep[i] = __pte(0UL);
  333. maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift);
  334. /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
  335. if (size == HPAGE_SIZE)
  336. maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0,
  337. orig_shift);
  338. return entry;
  339. }
  340. int pmd_huge(pmd_t pmd)
  341. {
  342. return !pmd_none(pmd) &&
  343. (pmd_val(pmd) & (_PAGE_VALID|_PAGE_PMD_HUGE)) != _PAGE_VALID;
  344. }
  345. int pud_huge(pud_t pud)
  346. {
  347. return !pud_none(pud) &&
  348. (pud_val(pud) & (_PAGE_VALID|_PAGE_PUD_HUGE)) != _PAGE_VALID;
  349. }
  350. static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
  351. unsigned long addr)
  352. {
  353. pgtable_t token = pmd_pgtable(*pmd);
  354. pmd_clear(pmd);
  355. pte_free_tlb(tlb, token, addr);
  356. mm_dec_nr_ptes(tlb->mm);
  357. }
  358. static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
  359. unsigned long addr, unsigned long end,
  360. unsigned long floor, unsigned long ceiling)
  361. {
  362. pmd_t *pmd;
  363. unsigned long next;
  364. unsigned long start;
  365. start = addr;
  366. pmd = pmd_offset(pud, addr);
  367. do {
  368. next = pmd_addr_end(addr, end);
  369. if (pmd_none(*pmd))
  370. continue;
  371. if (is_hugetlb_pmd(*pmd))
  372. pmd_clear(pmd);
  373. else
  374. hugetlb_free_pte_range(tlb, pmd, addr);
  375. } while (pmd++, addr = next, addr != end);
  376. start &= PUD_MASK;
  377. if (start < floor)
  378. return;
  379. if (ceiling) {
  380. ceiling &= PUD_MASK;
  381. if (!ceiling)
  382. return;
  383. }
  384. if (end - 1 > ceiling - 1)
  385. return;
  386. pmd = pmd_offset(pud, start);
  387. pud_clear(pud);
  388. pmd_free_tlb(tlb, pmd, start);
  389. mm_dec_nr_pmds(tlb->mm);
  390. }
  391. static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
  392. unsigned long addr, unsigned long end,
  393. unsigned long floor, unsigned long ceiling)
  394. {
  395. pud_t *pud;
  396. unsigned long next;
  397. unsigned long start;
  398. start = addr;
  399. pud = pud_offset(p4d, addr);
  400. do {
  401. next = pud_addr_end(addr, end);
  402. if (pud_none_or_clear_bad(pud))
  403. continue;
  404. if (is_hugetlb_pud(*pud))
  405. pud_clear(pud);
  406. else
  407. hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
  408. ceiling);
  409. } while (pud++, addr = next, addr != end);
  410. start &= PGDIR_MASK;
  411. if (start < floor)
  412. return;
  413. if (ceiling) {
  414. ceiling &= PGDIR_MASK;
  415. if (!ceiling)
  416. return;
  417. }
  418. if (end - 1 > ceiling - 1)
  419. return;
  420. pud = pud_offset(p4d, start);
  421. p4d_clear(p4d);
  422. pud_free_tlb(tlb, pud, start);
  423. mm_dec_nr_puds(tlb->mm);
  424. }
  425. void hugetlb_free_pgd_range(struct mmu_gather *tlb,
  426. unsigned long addr, unsigned long end,
  427. unsigned long floor, unsigned long ceiling)
  428. {
  429. pgd_t *pgd;
  430. p4d_t *p4d;
  431. unsigned long next;
  432. addr &= PMD_MASK;
  433. if (addr < floor) {
  434. addr += PMD_SIZE;
  435. if (!addr)
  436. return;
  437. }
  438. if (ceiling) {
  439. ceiling &= PMD_MASK;
  440. if (!ceiling)
  441. return;
  442. }
  443. if (end - 1 > ceiling - 1)
  444. end -= PMD_SIZE;
  445. if (addr > end - 1)
  446. return;
  447. pgd = pgd_offset(tlb->mm, addr);
  448. p4d = p4d_offset(pgd, addr);
  449. do {
  450. next = p4d_addr_end(addr, end);
  451. if (p4d_none_or_clear_bad(p4d))
  452. continue;
  453. hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);
  454. } while (p4d++, addr = next, addr != end);
  455. }