iovlock.c 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301
  1. /*
  2. * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
  3. * Portions based on net/core/datagram.c and copyrighted by their authors.
  4. *
  5. * This program is free software; you can redistribute it and/or modify it
  6. * under the terms of the GNU General Public License as published by the Free
  7. * Software Foundation; either version 2 of the License, or (at your option)
  8. * any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful, but WITHOUT
  11. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  13. * more details.
  14. *
  15. * You should have received a copy of the GNU General Public License along with
  16. * this program; if not, write to the Free Software Foundation, Inc., 59
  17. * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18. *
  19. * The full GNU General Public License is included in this distribution in the
  20. * file called COPYING.
  21. */
  22. /*
  23. * This code allows the net stack to make use of a DMA engine for
  24. * skb to iovec copies.
  25. */
  26. #include <linux/dmaengine.h>
  27. #include <linux/pagemap.h>
  28. #include <net/tcp.h> /* for memcpy_toiovec */
  29. #include <asm/io.h>
  30. #include <asm/uaccess.h>
  31. static int num_pages_spanned(struct iovec *iov)
  32. {
  33. return
  34. ((PAGE_ALIGN((unsigned long)iov->iov_base + iov->iov_len) -
  35. ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT);
  36. }
  37. /*
  38. * Pin down all the iovec pages needed for len bytes.
  39. * Return a struct dma_pinned_list to keep track of pages pinned down.
  40. *
  41. * We are allocating a single chunk of memory, and then carving it up into
  42. * 3 sections, the latter 2 whose size depends on the number of iovecs and the
  43. * total number of pages, respectively.
  44. */
  45. struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len)
  46. {
  47. struct dma_pinned_list *local_list;
  48. struct page **pages;
  49. int i;
  50. int ret;
  51. int nr_iovecs = 0;
  52. int iovec_len_used = 0;
  53. int iovec_pages_used = 0;
  54. long err;
  55. /* don't pin down non-user-based iovecs */
  56. if (segment_eq(get_fs(), KERNEL_DS))
  57. return NULL;
  58. /* determine how many iovecs/pages there are, up front */
  59. do {
  60. iovec_len_used += iov[nr_iovecs].iov_len;
  61. iovec_pages_used += num_pages_spanned(&iov[nr_iovecs]);
  62. nr_iovecs++;
  63. } while (iovec_len_used < len);
  64. /* single kmalloc for pinned list, page_list[], and the page arrays */
  65. local_list = kmalloc(sizeof(*local_list)
  66. + (nr_iovecs * sizeof (struct dma_page_list))
  67. + (iovec_pages_used * sizeof (struct page*)), GFP_KERNEL);
  68. if (!local_list) {
  69. err = -ENOMEM;
  70. goto out;
  71. }
  72. /* list of pages starts right after the page list array */
  73. pages = (struct page **) &local_list->page_list[nr_iovecs];
  74. for (i = 0; i < nr_iovecs; i++) {
  75. struct dma_page_list *page_list = &local_list->page_list[i];
  76. len -= iov[i].iov_len;
  77. if (!access_ok(VERIFY_WRITE, iov[i].iov_base, iov[i].iov_len)) {
  78. err = -EFAULT;
  79. goto unpin;
  80. }
  81. page_list->nr_pages = num_pages_spanned(&iov[i]);
  82. page_list->base_address = iov[i].iov_base;
  83. page_list->pages = pages;
  84. pages += page_list->nr_pages;
  85. /* pin pages down */
  86. down_read(&current->mm->mmap_sem);
  87. ret = get_user_pages(
  88. current,
  89. current->mm,
  90. (unsigned long) iov[i].iov_base,
  91. page_list->nr_pages,
  92. 1, /* write */
  93. 0, /* force */
  94. page_list->pages,
  95. NULL);
  96. up_read(&current->mm->mmap_sem);
  97. if (ret != page_list->nr_pages) {
  98. err = -ENOMEM;
  99. goto unpin;
  100. }
  101. local_list->nr_iovecs = i + 1;
  102. }
  103. return local_list;
  104. unpin:
  105. dma_unpin_iovec_pages(local_list);
  106. out:
  107. return ERR_PTR(err);
  108. }
  109. void dma_unpin_iovec_pages(struct dma_pinned_list *pinned_list)
  110. {
  111. int i, j;
  112. if (!pinned_list)
  113. return;
  114. for (i = 0; i < pinned_list->nr_iovecs; i++) {
  115. struct dma_page_list *page_list = &pinned_list->page_list[i];
  116. for (j = 0; j < page_list->nr_pages; j++) {
  117. set_page_dirty_lock(page_list->pages[j]);
  118. page_cache_release(page_list->pages[j]);
  119. }
  120. }
  121. kfree(pinned_list);
  122. }
  123. static dma_cookie_t dma_memcpy_to_kernel_iovec(struct dma_chan *chan, struct
  124. iovec *iov, unsigned char *kdata, size_t len)
  125. {
  126. dma_cookie_t dma_cookie = 0;
  127. while (len > 0) {
  128. if (iov->iov_len) {
  129. int copy = min_t(unsigned int, iov->iov_len, len);
  130. dma_cookie = dma_async_memcpy_buf_to_buf(
  131. chan,
  132. iov->iov_base,
  133. kdata,
  134. copy);
  135. kdata += copy;
  136. len -= copy;
  137. iov->iov_len -= copy;
  138. iov->iov_base += copy;
  139. }
  140. iov++;
  141. }
  142. return dma_cookie;
  143. }
  144. /*
  145. * We have already pinned down the pages we will be using in the iovecs.
  146. * Each entry in iov array has corresponding entry in pinned_list->page_list.
  147. * Using array indexing to keep iov[] and page_list[] in sync.
  148. * Initial elements in iov array's iov->iov_len will be 0 if already copied into
  149. * by another call.
  150. * iov array length remaining guaranteed to be bigger than len.
  151. */
  152. dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
  153. struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len)
  154. {
  155. int iov_byte_offset;
  156. int copy;
  157. dma_cookie_t dma_cookie = 0;
  158. int iovec_idx;
  159. int page_idx;
  160. if (!chan)
  161. return memcpy_toiovec(iov, kdata, len);
  162. /* -> kernel copies (e.g. smbfs) */
  163. if (!pinned_list)
  164. return dma_memcpy_to_kernel_iovec(chan, iov, kdata, len);
  165. iovec_idx = 0;
  166. while (iovec_idx < pinned_list->nr_iovecs) {
  167. struct dma_page_list *page_list;
  168. /* skip already used-up iovecs */
  169. while (!iov[iovec_idx].iov_len)
  170. iovec_idx++;
  171. page_list = &pinned_list->page_list[iovec_idx];
  172. iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK);
  173. page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK)
  174. - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT;
  175. /* break up copies to not cross page boundary */
  176. while (iov[iovec_idx].iov_len) {
  177. copy = min_t(int, PAGE_SIZE - iov_byte_offset, len);
  178. copy = min_t(int, copy, iov[iovec_idx].iov_len);
  179. dma_cookie = dma_async_memcpy_buf_to_pg(chan,
  180. page_list->pages[page_idx],
  181. iov_byte_offset,
  182. kdata,
  183. copy);
  184. len -= copy;
  185. iov[iovec_idx].iov_len -= copy;
  186. iov[iovec_idx].iov_base += copy;
  187. if (!len)
  188. return dma_cookie;
  189. kdata += copy;
  190. iov_byte_offset = 0;
  191. page_idx++;
  192. }
  193. iovec_idx++;
  194. }
  195. /* really bad if we ever run out of iovecs */
  196. BUG();
  197. return -EFAULT;
  198. }
  199. dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
  200. struct dma_pinned_list *pinned_list, struct page *page,
  201. unsigned int offset, size_t len)
  202. {
  203. int iov_byte_offset;
  204. int copy;
  205. dma_cookie_t dma_cookie = 0;
  206. int iovec_idx;
  207. int page_idx;
  208. int err;
  209. /* this needs as-yet-unimplemented buf-to-buff, so punt. */
  210. /* TODO: use dma for this */
  211. if (!chan || !pinned_list) {
  212. u8 *vaddr = kmap(page);
  213. err = memcpy_toiovec(iov, vaddr + offset, len);
  214. kunmap(page);
  215. return err;
  216. }
  217. iovec_idx = 0;
  218. while (iovec_idx < pinned_list->nr_iovecs) {
  219. struct dma_page_list *page_list;
  220. /* skip already used-up iovecs */
  221. while (!iov[iovec_idx].iov_len)
  222. iovec_idx++;
  223. page_list = &pinned_list->page_list[iovec_idx];
  224. iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK);
  225. page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK)
  226. - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT;
  227. /* break up copies to not cross page boundary */
  228. while (iov[iovec_idx].iov_len) {
  229. copy = min_t(int, PAGE_SIZE - iov_byte_offset, len);
  230. copy = min_t(int, copy, iov[iovec_idx].iov_len);
  231. dma_cookie = dma_async_memcpy_pg_to_pg(chan,
  232. page_list->pages[page_idx],
  233. iov_byte_offset,
  234. page,
  235. offset,
  236. copy);
  237. len -= copy;
  238. iov[iovec_idx].iov_len -= copy;
  239. iov[iovec_idx].iov_base += copy;
  240. if (!len)
  241. return dma_cookie;
  242. offset += copy;
  243. iov_byte_offset = 0;
  244. page_idx++;
  245. }
  246. iovec_idx++;
  247. }
  248. /* really bad if we ever run out of iovecs */
  249. BUG();
  250. return -EFAULT;
  251. }