vmalloc.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749
  1. /*
  2. * linux/mm/vmalloc.c
  3. *
  4. * Copyright (C) 1993 Linus Torvalds
  5. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  6. * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
  7. * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
  8. * Numa awareness, Christoph Lameter, SGI, June 2005
  9. */
  10. #include <linux/mm.h>
  11. #include <linux/module.h>
  12. #include <linux/highmem.h>
  13. #include <linux/slab.h>
  14. #include <linux/spinlock.h>
  15. #include <linux/interrupt.h>
  16. #include <linux/vmalloc.h>
  17. #include <asm/uaccess.h>
  18. #include <asm/tlbflush.h>
  19. DEFINE_RWLOCK(vmlist_lock);
  20. struct vm_struct *vmlist;
  21. static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
  22. int node);
  23. static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
  24. {
  25. pte_t *pte;
  26. pte = pte_offset_kernel(pmd, addr);
  27. do {
  28. pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
  29. WARN_ON(!pte_none(ptent) && !pte_present(ptent));
  30. } while (pte++, addr += PAGE_SIZE, addr != end);
  31. }
  32. static inline void vunmap_pmd_range(pud_t *pud, unsigned long addr,
  33. unsigned long end)
  34. {
  35. pmd_t *pmd;
  36. unsigned long next;
  37. pmd = pmd_offset(pud, addr);
  38. do {
  39. next = pmd_addr_end(addr, end);
  40. if (pmd_none_or_clear_bad(pmd))
  41. continue;
  42. vunmap_pte_range(pmd, addr, next);
  43. } while (pmd++, addr = next, addr != end);
  44. }
  45. static inline void vunmap_pud_range(pgd_t *pgd, unsigned long addr,
  46. unsigned long end)
  47. {
  48. pud_t *pud;
  49. unsigned long next;
  50. pud = pud_offset(pgd, addr);
  51. do {
  52. next = pud_addr_end(addr, end);
  53. if (pud_none_or_clear_bad(pud))
  54. continue;
  55. vunmap_pmd_range(pud, addr, next);
  56. } while (pud++, addr = next, addr != end);
  57. }
  58. void unmap_vm_area(struct vm_struct *area)
  59. {
  60. pgd_t *pgd;
  61. unsigned long next;
  62. unsigned long addr = (unsigned long) area->addr;
  63. unsigned long end = addr + area->size;
  64. BUG_ON(addr >= end);
  65. pgd = pgd_offset_k(addr);
  66. flush_cache_vunmap(addr, end);
  67. do {
  68. next = pgd_addr_end(addr, end);
  69. if (pgd_none_or_clear_bad(pgd))
  70. continue;
  71. vunmap_pud_range(pgd, addr, next);
  72. } while (pgd++, addr = next, addr != end);
  73. flush_tlb_kernel_range((unsigned long) area->addr, end);
  74. }
  75. static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
  76. unsigned long end, pgprot_t prot, struct page ***pages)
  77. {
  78. pte_t *pte;
  79. pte = pte_alloc_kernel(pmd, addr);
  80. if (!pte)
  81. return -ENOMEM;
  82. do {
  83. struct page *page = **pages;
  84. WARN_ON(!pte_none(*pte));
  85. if (!page)
  86. return -ENOMEM;
  87. set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
  88. (*pages)++;
  89. } while (pte++, addr += PAGE_SIZE, addr != end);
  90. return 0;
  91. }
  92. static inline int vmap_pmd_range(pud_t *pud, unsigned long addr,
  93. unsigned long end, pgprot_t prot, struct page ***pages)
  94. {
  95. pmd_t *pmd;
  96. unsigned long next;
  97. pmd = pmd_alloc(&init_mm, pud, addr);
  98. if (!pmd)
  99. return -ENOMEM;
  100. do {
  101. next = pmd_addr_end(addr, end);
  102. if (vmap_pte_range(pmd, addr, next, prot, pages))
  103. return -ENOMEM;
  104. } while (pmd++, addr = next, addr != end);
  105. return 0;
  106. }
  107. static inline int vmap_pud_range(pgd_t *pgd, unsigned long addr,
  108. unsigned long end, pgprot_t prot, struct page ***pages)
  109. {
  110. pud_t *pud;
  111. unsigned long next;
  112. pud = pud_alloc(&init_mm, pgd, addr);
  113. if (!pud)
  114. return -ENOMEM;
  115. do {
  116. next = pud_addr_end(addr, end);
  117. if (vmap_pmd_range(pud, addr, next, prot, pages))
  118. return -ENOMEM;
  119. } while (pud++, addr = next, addr != end);
  120. return 0;
  121. }
  122. int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
  123. {
  124. pgd_t *pgd;
  125. unsigned long next;
  126. unsigned long addr = (unsigned long) area->addr;
  127. unsigned long end = addr + area->size - PAGE_SIZE;
  128. int err;
  129. BUG_ON(addr >= end);
  130. pgd = pgd_offset_k(addr);
  131. do {
  132. next = pgd_addr_end(addr, end);
  133. err = vmap_pud_range(pgd, addr, next, prot, pages);
  134. if (err)
  135. break;
  136. } while (pgd++, addr = next, addr != end);
  137. flush_cache_vmap((unsigned long) area->addr, end);
  138. return err;
  139. }
  140. static struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long flags,
  141. unsigned long start, unsigned long end,
  142. int node, gfp_t gfp_mask)
  143. {
  144. struct vm_struct **p, *tmp, *area;
  145. unsigned long align = 1;
  146. unsigned long addr;
  147. BUG_ON(in_interrupt());
  148. if (flags & VM_IOREMAP) {
  149. int bit = fls(size);
  150. if (bit > IOREMAP_MAX_ORDER)
  151. bit = IOREMAP_MAX_ORDER;
  152. else if (bit < PAGE_SHIFT)
  153. bit = PAGE_SHIFT;
  154. align = 1ul << bit;
  155. }
  156. addr = ALIGN(start, align);
  157. size = PAGE_ALIGN(size);
  158. if (unlikely(!size))
  159. return NULL;
  160. area = kmalloc_node(sizeof(*area), gfp_mask & GFP_LEVEL_MASK, node);
  161. if (unlikely(!area))
  162. return NULL;
  163. /*
  164. * We always allocate a guard page.
  165. */
  166. size += PAGE_SIZE;
  167. write_lock(&vmlist_lock);
  168. for (p = &vmlist; (tmp = *p) != NULL ;p = &tmp->next) {
  169. if ((unsigned long)tmp->addr < addr) {
  170. if((unsigned long)tmp->addr + tmp->size >= addr)
  171. addr = ALIGN(tmp->size +
  172. (unsigned long)tmp->addr, align);
  173. continue;
  174. }
  175. if ((size + addr) < addr)
  176. goto out;
  177. if (size + addr <= (unsigned long)tmp->addr)
  178. goto found;
  179. addr = ALIGN(tmp->size + (unsigned long)tmp->addr, align);
  180. if (addr > end - size)
  181. goto out;
  182. }
  183. found:
  184. area->next = *p;
  185. *p = area;
  186. area->flags = flags;
  187. area->addr = (void *)addr;
  188. area->size = size;
  189. area->pages = NULL;
  190. area->nr_pages = 0;
  191. area->phys_addr = 0;
  192. write_unlock(&vmlist_lock);
  193. return area;
  194. out:
  195. write_unlock(&vmlist_lock);
  196. kfree(area);
  197. if (printk_ratelimit())
  198. printk(KERN_WARNING "allocation failed: out of vmalloc space - use vmalloc=<size> to increase size.\n");
  199. return NULL;
  200. }
  201. struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
  202. unsigned long start, unsigned long end)
  203. {
  204. return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL);
  205. }
  206. /**
  207. * get_vm_area - reserve a contingous kernel virtual area
  208. * @size: size of the area
  209. * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
  210. *
  211. * Search an area of @size in the kernel virtual mapping area,
  212. * and reserved it for out purposes. Returns the area descriptor
  213. * on success or %NULL on failure.
  214. */
  215. struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
  216. {
  217. return __get_vm_area(size, flags, VMALLOC_START, VMALLOC_END);
  218. }
  219. struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags,
  220. int node, gfp_t gfp_mask)
  221. {
  222. return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node,
  223. gfp_mask);
  224. }
  225. /* Caller must hold vmlist_lock */
  226. static struct vm_struct *__find_vm_area(void *addr)
  227. {
  228. struct vm_struct *tmp;
  229. for (tmp = vmlist; tmp != NULL; tmp = tmp->next) {
  230. if (tmp->addr == addr)
  231. break;
  232. }
  233. return tmp;
  234. }
  235. /* Caller must hold vmlist_lock */
  236. static struct vm_struct *__remove_vm_area(void *addr)
  237. {
  238. struct vm_struct **p, *tmp;
  239. for (p = &vmlist ; (tmp = *p) != NULL ;p = &tmp->next) {
  240. if (tmp->addr == addr)
  241. goto found;
  242. }
  243. return NULL;
  244. found:
  245. unmap_vm_area(tmp);
  246. *p = tmp->next;
  247. /*
  248. * Remove the guard page.
  249. */
  250. tmp->size -= PAGE_SIZE;
  251. return tmp;
  252. }
  253. /**
  254. * remove_vm_area - find and remove a contingous kernel virtual area
  255. * @addr: base address
  256. *
  257. * Search for the kernel VM area starting at @addr, and remove it.
  258. * This function returns the found VM area, but using it is NOT safe
  259. * on SMP machines, except for its size or flags.
  260. */
  261. struct vm_struct *remove_vm_area(void *addr)
  262. {
  263. struct vm_struct *v;
  264. write_lock(&vmlist_lock);
  265. v = __remove_vm_area(addr);
  266. write_unlock(&vmlist_lock);
  267. return v;
  268. }
  269. void __vunmap(void *addr, int deallocate_pages)
  270. {
  271. struct vm_struct *area;
  272. if (!addr)
  273. return;
  274. if ((PAGE_SIZE-1) & (unsigned long)addr) {
  275. printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
  276. WARN_ON(1);
  277. return;
  278. }
  279. area = remove_vm_area(addr);
  280. if (unlikely(!area)) {
  281. printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
  282. addr);
  283. WARN_ON(1);
  284. return;
  285. }
  286. debug_check_no_locks_freed(addr, area->size);
  287. if (deallocate_pages) {
  288. int i;
  289. for (i = 0; i < area->nr_pages; i++) {
  290. BUG_ON(!area->pages[i]);
  291. __free_page(area->pages[i]);
  292. }
  293. if (area->flags & VM_VPAGES)
  294. vfree(area->pages);
  295. else
  296. kfree(area->pages);
  297. }
  298. kfree(area);
  299. return;
  300. }
  301. /**
  302. * vfree - release memory allocated by vmalloc()
  303. * @addr: memory base address
  304. *
  305. * Free the virtually contiguous memory area starting at @addr, as
  306. * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
  307. * NULL, no operation is performed.
  308. *
  309. * Must not be called in interrupt context.
  310. */
  311. void vfree(void *addr)
  312. {
  313. BUG_ON(in_interrupt());
  314. __vunmap(addr, 1);
  315. }
  316. EXPORT_SYMBOL(vfree);
  317. /**
  318. * vunmap - release virtual mapping obtained by vmap()
  319. * @addr: memory base address
  320. *
  321. * Free the virtually contiguous memory area starting at @addr,
  322. * which was created from the page array passed to vmap().
  323. *
  324. * Must not be called in interrupt context.
  325. */
  326. void vunmap(void *addr)
  327. {
  328. BUG_ON(in_interrupt());
  329. __vunmap(addr, 0);
  330. }
  331. EXPORT_SYMBOL(vunmap);
  332. /**
  333. * vmap - map an array of pages into virtually contiguous space
  334. * @pages: array of page pointers
  335. * @count: number of pages to map
  336. * @flags: vm_area->flags
  337. * @prot: page protection for the mapping
  338. *
  339. * Maps @count pages from @pages into contiguous kernel virtual
  340. * space.
  341. */
  342. void *vmap(struct page **pages, unsigned int count,
  343. unsigned long flags, pgprot_t prot)
  344. {
  345. struct vm_struct *area;
  346. if (count > num_physpages)
  347. return NULL;
  348. area = get_vm_area((count << PAGE_SHIFT), flags);
  349. if (!area)
  350. return NULL;
  351. if (map_vm_area(area, prot, &pages)) {
  352. vunmap(area->addr);
  353. return NULL;
  354. }
  355. return area->addr;
  356. }
  357. EXPORT_SYMBOL(vmap);
  358. void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
  359. pgprot_t prot, int node)
  360. {
  361. struct page **pages;
  362. unsigned int nr_pages, array_size, i;
  363. nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT;
  364. array_size = (nr_pages * sizeof(struct page *));
  365. area->nr_pages = nr_pages;
  366. /* Please note that the recursion is strictly bounded. */
  367. if (array_size > PAGE_SIZE) {
  368. pages = __vmalloc_node(array_size, gfp_mask, PAGE_KERNEL, node);
  369. area->flags |= VM_VPAGES;
  370. } else {
  371. pages = kmalloc_node(array_size,
  372. (gfp_mask & ~(__GFP_HIGHMEM | __GFP_ZERO)),
  373. node);
  374. }
  375. area->pages = pages;
  376. if (!area->pages) {
  377. remove_vm_area(area->addr);
  378. kfree(area);
  379. return NULL;
  380. }
  381. memset(area->pages, 0, array_size);
  382. for (i = 0; i < area->nr_pages; i++) {
  383. if (node < 0)
  384. area->pages[i] = alloc_page(gfp_mask);
  385. else
  386. area->pages[i] = alloc_pages_node(node, gfp_mask, 0);
  387. if (unlikely(!area->pages[i])) {
  388. /* Successfully allocated i pages, free them in __vunmap() */
  389. area->nr_pages = i;
  390. goto fail;
  391. }
  392. }
  393. if (map_vm_area(area, prot, &pages))
  394. goto fail;
  395. return area->addr;
  396. fail:
  397. vfree(area->addr);
  398. return NULL;
  399. }
  400. void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
  401. {
  402. return __vmalloc_area_node(area, gfp_mask, prot, -1);
  403. }
  404. /**
  405. * __vmalloc_node - allocate virtually contiguous memory
  406. * @size: allocation size
  407. * @gfp_mask: flags for the page level allocator
  408. * @prot: protection mask for the allocated pages
  409. * @node: node to use for allocation or -1
  410. *
  411. * Allocate enough pages to cover @size from the page level
  412. * allocator with @gfp_mask flags. Map them into contiguous
  413. * kernel virtual space, using a pagetable protection of @prot.
  414. */
  415. static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
  416. int node)
  417. {
  418. struct vm_struct *area;
  419. size = PAGE_ALIGN(size);
  420. if (!size || (size >> PAGE_SHIFT) > num_physpages)
  421. return NULL;
  422. area = get_vm_area_node(size, VM_ALLOC, node, gfp_mask);
  423. if (!area)
  424. return NULL;
  425. return __vmalloc_area_node(area, gfp_mask, prot, node);
  426. }
  427. void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
  428. {
  429. return __vmalloc_node(size, gfp_mask, prot, -1);
  430. }
  431. EXPORT_SYMBOL(__vmalloc);
  432. /**
  433. * vmalloc - allocate virtually contiguous memory
  434. * @size: allocation size
  435. * Allocate enough pages to cover @size from the page level
  436. * allocator and map them into contiguous kernel virtual space.
  437. *
  438. * For tight control over page level allocator and protection flags
  439. * use __vmalloc() instead.
  440. */
  441. void *vmalloc(unsigned long size)
  442. {
  443. return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
  444. }
  445. EXPORT_SYMBOL(vmalloc);
  446. /**
  447. * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
  448. * @size: allocation size
  449. *
  450. * The resulting memory area is zeroed so it can be mapped to userspace
  451. * without leaking data.
  452. */
  453. void *vmalloc_user(unsigned long size)
  454. {
  455. struct vm_struct *area;
  456. void *ret;
  457. ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL);
  458. if (ret) {
  459. write_lock(&vmlist_lock);
  460. area = __find_vm_area(ret);
  461. area->flags |= VM_USERMAP;
  462. write_unlock(&vmlist_lock);
  463. }
  464. return ret;
  465. }
  466. EXPORT_SYMBOL(vmalloc_user);
  467. /**
  468. * vmalloc_node - allocate memory on a specific node
  469. * @size: allocation size
  470. * @node: numa node
  471. *
  472. * Allocate enough pages to cover @size from the page level
  473. * allocator and map them into contiguous kernel virtual space.
  474. *
  475. * For tight control over page level allocator and protection flags
  476. * use __vmalloc() instead.
  477. */
  478. void *vmalloc_node(unsigned long size, int node)
  479. {
  480. return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, node);
  481. }
  482. EXPORT_SYMBOL(vmalloc_node);
  483. #ifndef PAGE_KERNEL_EXEC
  484. # define PAGE_KERNEL_EXEC PAGE_KERNEL
  485. #endif
  486. /**
  487. * vmalloc_exec - allocate virtually contiguous, executable memory
  488. * @size: allocation size
  489. *
  490. * Kernel-internal function to allocate enough pages to cover @size
  491. * the page level allocator and map them into contiguous and
  492. * executable kernel virtual space.
  493. *
  494. * For tight control over page level allocator and protection flags
  495. * use __vmalloc() instead.
  496. */
  497. void *vmalloc_exec(unsigned long size)
  498. {
  499. return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
  500. }
  501. /**
  502. * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
  503. * @size: allocation size
  504. *
  505. * Allocate enough 32bit PA addressable pages to cover @size from the
  506. * page level allocator and map them into contiguous kernel virtual space.
  507. */
  508. void *vmalloc_32(unsigned long size)
  509. {
  510. return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
  511. }
  512. EXPORT_SYMBOL(vmalloc_32);
  513. /**
  514. * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
  515. * @size: allocation size
  516. *
  517. * The resulting memory area is 32bit addressable and zeroed so it can be
  518. * mapped to userspace without leaking data.
  519. */
  520. void *vmalloc_32_user(unsigned long size)
  521. {
  522. struct vm_struct *area;
  523. void *ret;
  524. ret = __vmalloc(size, GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL);
  525. if (ret) {
  526. write_lock(&vmlist_lock);
  527. area = __find_vm_area(ret);
  528. area->flags |= VM_USERMAP;
  529. write_unlock(&vmlist_lock);
  530. }
  531. return ret;
  532. }
  533. EXPORT_SYMBOL(vmalloc_32_user);
  534. long vread(char *buf, char *addr, unsigned long count)
  535. {
  536. struct vm_struct *tmp;
  537. char *vaddr, *buf_start = buf;
  538. unsigned long n;
  539. /* Don't allow overflow */
  540. if ((unsigned long) addr + count < count)
  541. count = -(unsigned long) addr;
  542. read_lock(&vmlist_lock);
  543. for (tmp = vmlist; tmp; tmp = tmp->next) {
  544. vaddr = (char *) tmp->addr;
  545. if (addr >= vaddr + tmp->size - PAGE_SIZE)
  546. continue;
  547. while (addr < vaddr) {
  548. if (count == 0)
  549. goto finished;
  550. *buf = '\0';
  551. buf++;
  552. addr++;
  553. count--;
  554. }
  555. n = vaddr + tmp->size - PAGE_SIZE - addr;
  556. do {
  557. if (count == 0)
  558. goto finished;
  559. *buf = *addr;
  560. buf++;
  561. addr++;
  562. count--;
  563. } while (--n > 0);
  564. }
  565. finished:
  566. read_unlock(&vmlist_lock);
  567. return buf - buf_start;
  568. }
  569. long vwrite(char *buf, char *addr, unsigned long count)
  570. {
  571. struct vm_struct *tmp;
  572. char *vaddr, *buf_start = buf;
  573. unsigned long n;
  574. /* Don't allow overflow */
  575. if ((unsigned long) addr + count < count)
  576. count = -(unsigned long) addr;
  577. read_lock(&vmlist_lock);
  578. for (tmp = vmlist; tmp; tmp = tmp->next) {
  579. vaddr = (char *) tmp->addr;
  580. if (addr >= vaddr + tmp->size - PAGE_SIZE)
  581. continue;
  582. while (addr < vaddr) {
  583. if (count == 0)
  584. goto finished;
  585. buf++;
  586. addr++;
  587. count--;
  588. }
  589. n = vaddr + tmp->size - PAGE_SIZE - addr;
  590. do {
  591. if (count == 0)
  592. goto finished;
  593. *addr = *buf;
  594. buf++;
  595. addr++;
  596. count--;
  597. } while (--n > 0);
  598. }
  599. finished:
  600. read_unlock(&vmlist_lock);
  601. return buf - buf_start;
  602. }
  603. /**
  604. * remap_vmalloc_range - map vmalloc pages to userspace
  605. * @vma: vma to cover (map full range of vma)
  606. * @addr: vmalloc memory
  607. * @pgoff: number of pages into addr before first page to map
  608. * @returns: 0 for success, -Exxx on failure
  609. *
  610. * This function checks that addr is a valid vmalloc'ed area, and
  611. * that it is big enough to cover the vma. Will return failure if
  612. * that criteria isn't met.
  613. *
  614. * Similar to remap_pfn_range() (see mm/memory.c)
  615. */
  616. int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
  617. unsigned long pgoff)
  618. {
  619. struct vm_struct *area;
  620. unsigned long uaddr = vma->vm_start;
  621. unsigned long usize = vma->vm_end - vma->vm_start;
  622. int ret;
  623. if ((PAGE_SIZE-1) & (unsigned long)addr)
  624. return -EINVAL;
  625. read_lock(&vmlist_lock);
  626. area = __find_vm_area(addr);
  627. if (!area)
  628. goto out_einval_locked;
  629. if (!(area->flags & VM_USERMAP))
  630. goto out_einval_locked;
  631. if (usize + (pgoff << PAGE_SHIFT) > area->size - PAGE_SIZE)
  632. goto out_einval_locked;
  633. read_unlock(&vmlist_lock);
  634. addr += pgoff << PAGE_SHIFT;
  635. do {
  636. struct page *page = vmalloc_to_page(addr);
  637. ret = vm_insert_page(vma, uaddr, page);
  638. if (ret)
  639. return ret;
  640. uaddr += PAGE_SIZE;
  641. addr += PAGE_SIZE;
  642. usize -= PAGE_SIZE;
  643. } while (usize > 0);
  644. /* Prevent "things" like memory migration? VM_flags need a cleanup... */
  645. vma->vm_flags |= VM_RESERVED;
  646. return ret;
  647. out_einval_locked:
  648. read_unlock(&vmlist_lock);
  649. return -EINVAL;
  650. }
  651. EXPORT_SYMBOL(remap_vmalloc_range);