hugetlb.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833
  1. /*
  2. * Generic hugetlb support.
  3. * (C) William Irwin, April 2004
  4. */
  5. #include <linux/gfp.h>
  6. #include <linux/list.h>
  7. #include <linux/init.h>
  8. #include <linux/module.h>
  9. #include <linux/mm.h>
  10. #include <linux/sysctl.h>
  11. #include <linux/highmem.h>
  12. #include <linux/nodemask.h>
  13. #include <linux/pagemap.h>
  14. #include <linux/mempolicy.h>
  15. #include <linux/cpuset.h>
  16. #include <linux/mutex.h>
  17. #include <asm/page.h>
  18. #include <asm/pgtable.h>
  19. #include <linux/hugetlb.h>
  20. #include "internal.h"
  21. const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
  22. static unsigned long nr_huge_pages, free_huge_pages, resv_huge_pages;
  23. unsigned long max_huge_pages;
  24. static struct list_head hugepage_freelists[MAX_NUMNODES];
  25. static unsigned int nr_huge_pages_node[MAX_NUMNODES];
  26. static unsigned int free_huge_pages_node[MAX_NUMNODES];
  27. /*
  28. * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
  29. */
  30. static DEFINE_SPINLOCK(hugetlb_lock);
  31. static void clear_huge_page(struct page *page, unsigned long addr)
  32. {
  33. int i;
  34. might_sleep();
  35. for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); i++) {
  36. cond_resched();
  37. clear_user_highpage(page + i, addr);
  38. }
  39. }
  40. static void copy_huge_page(struct page *dst, struct page *src,
  41. unsigned long addr, struct vm_area_struct *vma)
  42. {
  43. int i;
  44. might_sleep();
  45. for (i = 0; i < HPAGE_SIZE/PAGE_SIZE; i++) {
  46. cond_resched();
  47. copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma);
  48. }
  49. }
  50. static void enqueue_huge_page(struct page *page)
  51. {
  52. int nid = page_to_nid(page);
  53. list_add(&page->lru, &hugepage_freelists[nid]);
  54. free_huge_pages++;
  55. free_huge_pages_node[nid]++;
  56. }
  57. static struct page *dequeue_huge_page(struct vm_area_struct *vma,
  58. unsigned long address)
  59. {
  60. int nid = numa_node_id();
  61. struct page *page = NULL;
  62. struct zonelist *zonelist = huge_zonelist(vma, address);
  63. struct zone **z;
  64. for (z = zonelist->zones; *z; z++) {
  65. nid = zone_to_nid(*z);
  66. if (cpuset_zone_allowed_softwall(*z, GFP_HIGHUSER) &&
  67. !list_empty(&hugepage_freelists[nid]))
  68. break;
  69. }
  70. if (*z) {
  71. page = list_entry(hugepage_freelists[nid].next,
  72. struct page, lru);
  73. list_del(&page->lru);
  74. free_huge_pages--;
  75. free_huge_pages_node[nid]--;
  76. }
  77. return page;
  78. }
  79. static void free_huge_page(struct page *page)
  80. {
  81. BUG_ON(page_count(page));
  82. INIT_LIST_HEAD(&page->lru);
  83. spin_lock(&hugetlb_lock);
  84. enqueue_huge_page(page);
  85. spin_unlock(&hugetlb_lock);
  86. }
  87. static int alloc_fresh_huge_page(void)
  88. {
  89. static int nid = 0;
  90. struct page *page;
  91. page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP|__GFP_NOWARN,
  92. HUGETLB_PAGE_ORDER);
  93. nid = next_node(nid, node_online_map);
  94. if (nid == MAX_NUMNODES)
  95. nid = first_node(node_online_map);
  96. if (page) {
  97. set_compound_page_dtor(page, free_huge_page);
  98. spin_lock(&hugetlb_lock);
  99. nr_huge_pages++;
  100. nr_huge_pages_node[page_to_nid(page)]++;
  101. spin_unlock(&hugetlb_lock);
  102. put_page(page); /* free it into the hugepage allocator */
  103. return 1;
  104. }
  105. return 0;
  106. }
  107. static struct page *alloc_huge_page(struct vm_area_struct *vma,
  108. unsigned long addr)
  109. {
  110. struct page *page;
  111. spin_lock(&hugetlb_lock);
  112. if (vma->vm_flags & VM_MAYSHARE)
  113. resv_huge_pages--;
  114. else if (free_huge_pages <= resv_huge_pages)
  115. goto fail;
  116. page = dequeue_huge_page(vma, addr);
  117. if (!page)
  118. goto fail;
  119. spin_unlock(&hugetlb_lock);
  120. set_page_refcounted(page);
  121. return page;
  122. fail:
  123. if (vma->vm_flags & VM_MAYSHARE)
  124. resv_huge_pages++;
  125. spin_unlock(&hugetlb_lock);
  126. return NULL;
  127. }
  128. static int __init hugetlb_init(void)
  129. {
  130. unsigned long i;
  131. if (HPAGE_SHIFT == 0)
  132. return 0;
  133. for (i = 0; i < MAX_NUMNODES; ++i)
  134. INIT_LIST_HEAD(&hugepage_freelists[i]);
  135. for (i = 0; i < max_huge_pages; ++i) {
  136. if (!alloc_fresh_huge_page())
  137. break;
  138. }
  139. max_huge_pages = free_huge_pages = nr_huge_pages = i;
  140. printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages);
  141. return 0;
  142. }
  143. module_init(hugetlb_init);
  144. static int __init hugetlb_setup(char *s)
  145. {
  146. if (sscanf(s, "%lu", &max_huge_pages) <= 0)
  147. max_huge_pages = 0;
  148. return 1;
  149. }
  150. __setup("hugepages=", hugetlb_setup);
  151. #ifdef CONFIG_SYSCTL
  152. static void update_and_free_page(struct page *page)
  153. {
  154. int i;
  155. nr_huge_pages--;
  156. nr_huge_pages_node[page_to_nid(page)]--;
  157. for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) {
  158. page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
  159. 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
  160. 1 << PG_private | 1<< PG_writeback);
  161. }
  162. page[1].lru.next = NULL;
  163. set_page_refcounted(page);
  164. __free_pages(page, HUGETLB_PAGE_ORDER);
  165. }
  166. #ifdef CONFIG_HIGHMEM
  167. static void try_to_free_low(unsigned long count)
  168. {
  169. int i;
  170. for (i = 0; i < MAX_NUMNODES; ++i) {
  171. struct page *page, *next;
  172. list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) {
  173. if (PageHighMem(page))
  174. continue;
  175. list_del(&page->lru);
  176. update_and_free_page(page);
  177. free_huge_pages--;
  178. free_huge_pages_node[page_to_nid(page)]--;
  179. if (count >= nr_huge_pages)
  180. return;
  181. }
  182. }
  183. }
  184. #else
  185. static inline void try_to_free_low(unsigned long count)
  186. {
  187. }
  188. #endif
  189. static unsigned long set_max_huge_pages(unsigned long count)
  190. {
  191. while (count > nr_huge_pages) {
  192. if (!alloc_fresh_huge_page())
  193. return nr_huge_pages;
  194. }
  195. if (count >= nr_huge_pages)
  196. return nr_huge_pages;
  197. spin_lock(&hugetlb_lock);
  198. count = max(count, resv_huge_pages);
  199. try_to_free_low(count);
  200. while (count < nr_huge_pages) {
  201. struct page *page = dequeue_huge_page(NULL, 0);
  202. if (!page)
  203. break;
  204. update_and_free_page(page);
  205. }
  206. spin_unlock(&hugetlb_lock);
  207. return nr_huge_pages;
  208. }
  209. int hugetlb_sysctl_handler(struct ctl_table *table, int write,
  210. struct file *file, void __user *buffer,
  211. size_t *length, loff_t *ppos)
  212. {
  213. proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
  214. max_huge_pages = set_max_huge_pages(max_huge_pages);
  215. return 0;
  216. }
  217. #endif /* CONFIG_SYSCTL */
  218. int hugetlb_report_meminfo(char *buf)
  219. {
  220. return sprintf(buf,
  221. "HugePages_Total: %5lu\n"
  222. "HugePages_Free: %5lu\n"
  223. "HugePages_Rsvd: %5lu\n"
  224. "Hugepagesize: %5lu kB\n",
  225. nr_huge_pages,
  226. free_huge_pages,
  227. resv_huge_pages,
  228. HPAGE_SIZE/1024);
  229. }
  230. int hugetlb_report_node_meminfo(int nid, char *buf)
  231. {
  232. return sprintf(buf,
  233. "Node %d HugePages_Total: %5u\n"
  234. "Node %d HugePages_Free: %5u\n",
  235. nid, nr_huge_pages_node[nid],
  236. nid, free_huge_pages_node[nid]);
  237. }
  238. /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
  239. unsigned long hugetlb_total_pages(void)
  240. {
  241. return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE);
  242. }
  243. /*
  244. * We cannot handle pagefaults against hugetlb pages at all. They cause
  245. * handle_mm_fault() to try to instantiate regular-sized pages in the
  246. * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
  247. * this far.
  248. */
  249. static struct page *hugetlb_nopage(struct vm_area_struct *vma,
  250. unsigned long address, int *unused)
  251. {
  252. BUG();
  253. return NULL;
  254. }
  255. struct vm_operations_struct hugetlb_vm_ops = {
  256. .nopage = hugetlb_nopage,
  257. };
  258. static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
  259. int writable)
  260. {
  261. pte_t entry;
  262. if (writable) {
  263. entry =
  264. pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
  265. } else {
  266. entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));
  267. }
  268. entry = pte_mkyoung(entry);
  269. entry = pte_mkhuge(entry);
  270. return entry;
  271. }
  272. static void set_huge_ptep_writable(struct vm_area_struct *vma,
  273. unsigned long address, pte_t *ptep)
  274. {
  275. pte_t entry;
  276. entry = pte_mkwrite(pte_mkdirty(*ptep));
  277. ptep_set_access_flags(vma, address, ptep, entry, 1);
  278. update_mmu_cache(vma, address, entry);
  279. lazy_mmu_prot_update(entry);
  280. }
  281. int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
  282. struct vm_area_struct *vma)
  283. {
  284. pte_t *src_pte, *dst_pte, entry;
  285. struct page *ptepage;
  286. unsigned long addr;
  287. int cow;
  288. cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
  289. for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {
  290. src_pte = huge_pte_offset(src, addr);
  291. if (!src_pte)
  292. continue;
  293. dst_pte = huge_pte_alloc(dst, addr);
  294. if (!dst_pte)
  295. goto nomem;
  296. spin_lock(&dst->page_table_lock);
  297. spin_lock(&src->page_table_lock);
  298. if (!pte_none(*src_pte)) {
  299. if (cow)
  300. ptep_set_wrprotect(src, addr, src_pte);
  301. entry = *src_pte;
  302. ptepage = pte_page(entry);
  303. get_page(ptepage);
  304. set_huge_pte_at(dst, addr, dst_pte, entry);
  305. }
  306. spin_unlock(&src->page_table_lock);
  307. spin_unlock(&dst->page_table_lock);
  308. }
  309. return 0;
  310. nomem:
  311. return -ENOMEM;
  312. }
  313. void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
  314. unsigned long end)
  315. {
  316. struct mm_struct *mm = vma->vm_mm;
  317. unsigned long address;
  318. pte_t *ptep;
  319. pte_t pte;
  320. struct page *page;
  321. struct page *tmp;
  322. /*
  323. * A page gathering list, protected by per file i_mmap_lock. The
  324. * lock is used to avoid list corruption from multiple unmapping
  325. * of the same page since we are using page->lru.
  326. */
  327. LIST_HEAD(page_list);
  328. WARN_ON(!is_vm_hugetlb_page(vma));
  329. BUG_ON(start & ~HPAGE_MASK);
  330. BUG_ON(end & ~HPAGE_MASK);
  331. spin_lock(&mm->page_table_lock);
  332. for (address = start; address < end; address += HPAGE_SIZE) {
  333. ptep = huge_pte_offset(mm, address);
  334. if (!ptep)
  335. continue;
  336. if (huge_pmd_unshare(mm, &address, ptep))
  337. continue;
  338. pte = huge_ptep_get_and_clear(mm, address, ptep);
  339. if (pte_none(pte))
  340. continue;
  341. page = pte_page(pte);
  342. if (pte_dirty(pte))
  343. set_page_dirty(page);
  344. list_add(&page->lru, &page_list);
  345. }
  346. spin_unlock(&mm->page_table_lock);
  347. flush_tlb_range(vma, start, end);
  348. list_for_each_entry_safe(page, tmp, &page_list, lru) {
  349. list_del(&page->lru);
  350. put_page(page);
  351. }
  352. }
  353. void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
  354. unsigned long end)
  355. {
  356. /*
  357. * It is undesirable to test vma->vm_file as it should be non-null
  358. * for valid hugetlb area. However, vm_file will be NULL in the error
  359. * cleanup path of do_mmap_pgoff. When hugetlbfs ->mmap method fails,
  360. * do_mmap_pgoff() nullifies vma->vm_file before calling this function
  361. * to clean up. Since no pte has actually been setup, it is safe to
  362. * do nothing in this case.
  363. */
  364. if (vma->vm_file) {
  365. spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
  366. __unmap_hugepage_range(vma, start, end);
  367. spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
  368. }
  369. }
  370. static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
  371. unsigned long address, pte_t *ptep, pte_t pte)
  372. {
  373. struct page *old_page, *new_page;
  374. int avoidcopy;
  375. old_page = pte_page(pte);
  376. /* If no-one else is actually using this page, avoid the copy
  377. * and just make the page writable */
  378. avoidcopy = (page_count(old_page) == 1);
  379. if (avoidcopy) {
  380. set_huge_ptep_writable(vma, address, ptep);
  381. return VM_FAULT_MINOR;
  382. }
  383. page_cache_get(old_page);
  384. new_page = alloc_huge_page(vma, address);
  385. if (!new_page) {
  386. page_cache_release(old_page);
  387. return VM_FAULT_OOM;
  388. }
  389. spin_unlock(&mm->page_table_lock);
  390. copy_huge_page(new_page, old_page, address, vma);
  391. spin_lock(&mm->page_table_lock);
  392. ptep = huge_pte_offset(mm, address & HPAGE_MASK);
  393. if (likely(pte_same(*ptep, pte))) {
  394. /* Break COW */
  395. set_huge_pte_at(mm, address, ptep,
  396. make_huge_pte(vma, new_page, 1));
  397. /* Make the old page be freed below */
  398. new_page = old_page;
  399. }
  400. page_cache_release(new_page);
  401. page_cache_release(old_page);
  402. return VM_FAULT_MINOR;
  403. }
  404. int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
  405. unsigned long address, pte_t *ptep, int write_access)
  406. {
  407. int ret = VM_FAULT_SIGBUS;
  408. unsigned long idx;
  409. unsigned long size;
  410. struct page *page;
  411. struct address_space *mapping;
  412. pte_t new_pte;
  413. mapping = vma->vm_file->f_mapping;
  414. idx = ((address - vma->vm_start) >> HPAGE_SHIFT)
  415. + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));
  416. /*
  417. * Use page lock to guard against racing truncation
  418. * before we get page_table_lock.
  419. */
  420. retry:
  421. page = find_lock_page(mapping, idx);
  422. if (!page) {
  423. size = i_size_read(mapping->host) >> HPAGE_SHIFT;
  424. if (idx >= size)
  425. goto out;
  426. if (hugetlb_get_quota(mapping))
  427. goto out;
  428. page = alloc_huge_page(vma, address);
  429. if (!page) {
  430. hugetlb_put_quota(mapping);
  431. ret = VM_FAULT_OOM;
  432. goto out;
  433. }
  434. clear_huge_page(page, address);
  435. if (vma->vm_flags & VM_SHARED) {
  436. int err;
  437. err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
  438. if (err) {
  439. put_page(page);
  440. hugetlb_put_quota(mapping);
  441. if (err == -EEXIST)
  442. goto retry;
  443. goto out;
  444. }
  445. } else
  446. lock_page(page);
  447. }
  448. spin_lock(&mm->page_table_lock);
  449. size = i_size_read(mapping->host) >> HPAGE_SHIFT;
  450. if (idx >= size)
  451. goto backout;
  452. ret = VM_FAULT_MINOR;
  453. if (!pte_none(*ptep))
  454. goto backout;
  455. new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
  456. && (vma->vm_flags & VM_SHARED)));
  457. set_huge_pte_at(mm, address, ptep, new_pte);
  458. if (write_access && !(vma->vm_flags & VM_SHARED)) {
  459. /* Optimization, do the COW without a second fault */
  460. ret = hugetlb_cow(mm, vma, address, ptep, new_pte);
  461. }
  462. spin_unlock(&mm->page_table_lock);
  463. unlock_page(page);
  464. out:
  465. return ret;
  466. backout:
  467. spin_unlock(&mm->page_table_lock);
  468. hugetlb_put_quota(mapping);
  469. unlock_page(page);
  470. put_page(page);
  471. goto out;
  472. }
  473. int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  474. unsigned long address, int write_access)
  475. {
  476. pte_t *ptep;
  477. pte_t entry;
  478. int ret;
  479. static DEFINE_MUTEX(hugetlb_instantiation_mutex);
  480. ptep = huge_pte_alloc(mm, address);
  481. if (!ptep)
  482. return VM_FAULT_OOM;
  483. /*
  484. * Serialize hugepage allocation and instantiation, so that we don't
  485. * get spurious allocation failures if two CPUs race to instantiate
  486. * the same page in the page cache.
  487. */
  488. mutex_lock(&hugetlb_instantiation_mutex);
  489. entry = *ptep;
  490. if (pte_none(entry)) {
  491. ret = hugetlb_no_page(mm, vma, address, ptep, write_access);
  492. mutex_unlock(&hugetlb_instantiation_mutex);
  493. return ret;
  494. }
  495. ret = VM_FAULT_MINOR;
  496. spin_lock(&mm->page_table_lock);
  497. /* Check for a racing update before calling hugetlb_cow */
  498. if (likely(pte_same(entry, *ptep)))
  499. if (write_access && !pte_write(entry))
  500. ret = hugetlb_cow(mm, vma, address, ptep, entry);
  501. spin_unlock(&mm->page_table_lock);
  502. mutex_unlock(&hugetlb_instantiation_mutex);
  503. return ret;
  504. }
  505. int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
  506. struct page **pages, struct vm_area_struct **vmas,
  507. unsigned long *position, int *length, int i)
  508. {
  509. unsigned long pfn_offset;
  510. unsigned long vaddr = *position;
  511. int remainder = *length;
  512. spin_lock(&mm->page_table_lock);
  513. while (vaddr < vma->vm_end && remainder) {
  514. pte_t *pte;
  515. struct page *page;
  516. /*
  517. * Some archs (sparc64, sh*) have multiple pte_ts to
  518. * each hugepage. We have to make * sure we get the
  519. * first, for the page indexing below to work.
  520. */
  521. pte = huge_pte_offset(mm, vaddr & HPAGE_MASK);
  522. if (!pte || pte_none(*pte)) {
  523. int ret;
  524. spin_unlock(&mm->page_table_lock);
  525. ret = hugetlb_fault(mm, vma, vaddr, 0);
  526. spin_lock(&mm->page_table_lock);
  527. if (ret == VM_FAULT_MINOR)
  528. continue;
  529. remainder = 0;
  530. if (!i)
  531. i = -EFAULT;
  532. break;
  533. }
  534. pfn_offset = (vaddr & ~HPAGE_MASK) >> PAGE_SHIFT;
  535. page = pte_page(*pte);
  536. same_page:
  537. if (pages) {
  538. get_page(page);
  539. pages[i] = page + pfn_offset;
  540. }
  541. if (vmas)
  542. vmas[i] = vma;
  543. vaddr += PAGE_SIZE;
  544. ++pfn_offset;
  545. --remainder;
  546. ++i;
  547. if (vaddr < vma->vm_end && remainder &&
  548. pfn_offset < HPAGE_SIZE/PAGE_SIZE) {
  549. /*
  550. * We use pfn_offset to avoid touching the pageframes
  551. * of this compound page.
  552. */
  553. goto same_page;
  554. }
  555. }
  556. spin_unlock(&mm->page_table_lock);
  557. *length = remainder;
  558. *position = vaddr;
  559. return i;
  560. }
  561. void hugetlb_change_protection(struct vm_area_struct *vma,
  562. unsigned long address, unsigned long end, pgprot_t newprot)
  563. {
  564. struct mm_struct *mm = vma->vm_mm;
  565. unsigned long start = address;
  566. pte_t *ptep;
  567. pte_t pte;
  568. BUG_ON(address >= end);
  569. flush_cache_range(vma, address, end);
  570. spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
  571. spin_lock(&mm->page_table_lock);
  572. for (; address < end; address += HPAGE_SIZE) {
  573. ptep = huge_pte_offset(mm, address);
  574. if (!ptep)
  575. continue;
  576. if (huge_pmd_unshare(mm, &address, ptep))
  577. continue;
  578. if (!pte_none(*ptep)) {
  579. pte = huge_ptep_get_and_clear(mm, address, ptep);
  580. pte = pte_mkhuge(pte_modify(pte, newprot));
  581. set_huge_pte_at(mm, address, ptep, pte);
  582. lazy_mmu_prot_update(pte);
  583. }
  584. }
  585. spin_unlock(&mm->page_table_lock);
  586. spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
  587. flush_tlb_range(vma, start, end);
  588. }
  589. struct file_region {
  590. struct list_head link;
  591. long from;
  592. long to;
  593. };
  594. static long region_add(struct list_head *head, long f, long t)
  595. {
  596. struct file_region *rg, *nrg, *trg;
  597. /* Locate the region we are either in or before. */
  598. list_for_each_entry(rg, head, link)
  599. if (f <= rg->to)
  600. break;
  601. /* Round our left edge to the current segment if it encloses us. */
  602. if (f > rg->from)
  603. f = rg->from;
  604. /* Check for and consume any regions we now overlap with. */
  605. nrg = rg;
  606. list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
  607. if (&rg->link == head)
  608. break;
  609. if (rg->from > t)
  610. break;
  611. /* If this area reaches higher then extend our area to
  612. * include it completely. If this is not the first area
  613. * which we intend to reuse, free it. */
  614. if (rg->to > t)
  615. t = rg->to;
  616. if (rg != nrg) {
  617. list_del(&rg->link);
  618. kfree(rg);
  619. }
  620. }
  621. nrg->from = f;
  622. nrg->to = t;
  623. return 0;
  624. }
  625. static long region_chg(struct list_head *head, long f, long t)
  626. {
  627. struct file_region *rg, *nrg;
  628. long chg = 0;
  629. /* Locate the region we are before or in. */
  630. list_for_each_entry(rg, head, link)
  631. if (f <= rg->to)
  632. break;
  633. /* If we are below the current region then a new region is required.
  634. * Subtle, allocate a new region at the position but make it zero
  635. * size such that we can guarentee to record the reservation. */
  636. if (&rg->link == head || t < rg->from) {
  637. nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
  638. if (nrg == 0)
  639. return -ENOMEM;
  640. nrg->from = f;
  641. nrg->to = f;
  642. INIT_LIST_HEAD(&nrg->link);
  643. list_add(&nrg->link, rg->link.prev);
  644. return t - f;
  645. }
  646. /* Round our left edge to the current segment if it encloses us. */
  647. if (f > rg->from)
  648. f = rg->from;
  649. chg = t - f;
  650. /* Check for and consume any regions we now overlap with. */
  651. list_for_each_entry(rg, rg->link.prev, link) {
  652. if (&rg->link == head)
  653. break;
  654. if (rg->from > t)
  655. return chg;
  656. /* We overlap with this area, if it extends futher than
  657. * us then we must extend ourselves. Account for its
  658. * existing reservation. */
  659. if (rg->to > t) {
  660. chg += rg->to - t;
  661. t = rg->to;
  662. }
  663. chg -= rg->to - rg->from;
  664. }
  665. return chg;
  666. }
  667. static long region_truncate(struct list_head *head, long end)
  668. {
  669. struct file_region *rg, *trg;
  670. long chg = 0;
  671. /* Locate the region we are either in or before. */
  672. list_for_each_entry(rg, head, link)
  673. if (end <= rg->to)
  674. break;
  675. if (&rg->link == head)
  676. return 0;
  677. /* If we are in the middle of a region then adjust it. */
  678. if (end > rg->from) {
  679. chg = rg->to - end;
  680. rg->to = end;
  681. rg = list_entry(rg->link.next, typeof(*rg), link);
  682. }
  683. /* Drop any remaining regions. */
  684. list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
  685. if (&rg->link == head)
  686. break;
  687. chg += rg->to - rg->from;
  688. list_del(&rg->link);
  689. kfree(rg);
  690. }
  691. return chg;
  692. }
  693. static int hugetlb_acct_memory(long delta)
  694. {
  695. int ret = -ENOMEM;
  696. spin_lock(&hugetlb_lock);
  697. if ((delta + resv_huge_pages) <= free_huge_pages) {
  698. resv_huge_pages += delta;
  699. ret = 0;
  700. }
  701. spin_unlock(&hugetlb_lock);
  702. return ret;
  703. }
  704. int hugetlb_reserve_pages(struct inode *inode, long from, long to)
  705. {
  706. long ret, chg;
  707. chg = region_chg(&inode->i_mapping->private_list, from, to);
  708. if (chg < 0)
  709. return chg;
  710. ret = hugetlb_acct_memory(chg);
  711. if (ret < 0)
  712. return ret;
  713. region_add(&inode->i_mapping->private_list, from, to);
  714. return 0;
  715. }
  716. void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
  717. {
  718. long chg = region_truncate(&inode->i_mapping->private_list, offset);
  719. hugetlb_acct_memory(freed - chg);
  720. }