bounce.c 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302
  1. /* bounce buffer handling for block devices
  2. *
  3. * - Split from highmem.c
  4. */
  5. #include <linux/mm.h>
  6. #include <linux/module.h>
  7. #include <linux/swap.h>
  8. #include <linux/bio.h>
  9. #include <linux/pagemap.h>
  10. #include <linux/mempool.h>
  11. #include <linux/blkdev.h>
  12. #include <linux/init.h>
  13. #include <linux/hash.h>
  14. #include <linux/highmem.h>
  15. #include <linux/blktrace_api.h>
  16. #include <asm/tlbflush.h>
  17. #define POOL_SIZE 64
  18. #define ISA_POOL_SIZE 16
  19. static mempool_t *page_pool, *isa_page_pool;
  20. #ifdef CONFIG_HIGHMEM
  21. static __init int init_emergency_pool(void)
  22. {
  23. struct sysinfo i;
  24. si_meminfo(&i);
  25. si_swapinfo(&i);
  26. if (!i.totalhigh)
  27. return 0;
  28. page_pool = mempool_create_page_pool(POOL_SIZE, 0);
  29. BUG_ON(!page_pool);
  30. printk("highmem bounce pool size: %d pages\n", POOL_SIZE);
  31. return 0;
  32. }
  33. __initcall(init_emergency_pool);
  34. /*
  35. * highmem version, map in to vec
  36. */
  37. static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom)
  38. {
  39. unsigned long flags;
  40. unsigned char *vto;
  41. local_irq_save(flags);
  42. vto = kmap_atomic(to->bv_page, KM_BOUNCE_READ);
  43. memcpy(vto + to->bv_offset, vfrom, to->bv_len);
  44. kunmap_atomic(vto, KM_BOUNCE_READ);
  45. local_irq_restore(flags);
  46. }
  47. #else /* CONFIG_HIGHMEM */
  48. #define bounce_copy_vec(to, vfrom) \
  49. memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len)
  50. #endif /* CONFIG_HIGHMEM */
  51. /*
  52. * allocate pages in the DMA region for the ISA pool
  53. */
  54. static void *mempool_alloc_pages_isa(gfp_t gfp_mask, void *data)
  55. {
  56. return mempool_alloc_pages(gfp_mask | GFP_DMA, data);
  57. }
  58. /*
  59. * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA
  60. * as the max address, so check if the pool has already been created.
  61. */
  62. int init_emergency_isa_pool(void)
  63. {
  64. if (isa_page_pool)
  65. return 0;
  66. isa_page_pool = mempool_create(ISA_POOL_SIZE, mempool_alloc_pages_isa,
  67. mempool_free_pages, (void *) 0);
  68. BUG_ON(!isa_page_pool);
  69. printk("isa bounce pool size: %d pages\n", ISA_POOL_SIZE);
  70. return 0;
  71. }
  72. /*
  73. * Simple bounce buffer support for highmem pages. Depending on the
  74. * queue gfp mask set, *to may or may not be a highmem page. kmap it
  75. * always, it will do the Right Thing
  76. */
  77. static void copy_to_high_bio_irq(struct bio *to, struct bio *from)
  78. {
  79. unsigned char *vfrom;
  80. struct bio_vec *tovec, *fromvec;
  81. int i;
  82. __bio_for_each_segment(tovec, to, i, 0) {
  83. fromvec = from->bi_io_vec + i;
  84. /*
  85. * not bounced
  86. */
  87. if (tovec->bv_page == fromvec->bv_page)
  88. continue;
  89. /*
  90. * fromvec->bv_offset and fromvec->bv_len might have been
  91. * modified by the block layer, so use the original copy,
  92. * bounce_copy_vec already uses tovec->bv_len
  93. */
  94. vfrom = page_address(fromvec->bv_page) + tovec->bv_offset;
  95. flush_dcache_page(tovec->bv_page);
  96. bounce_copy_vec(tovec, vfrom);
  97. }
  98. }
  99. static void bounce_end_io(struct bio *bio, mempool_t *pool, int err)
  100. {
  101. struct bio *bio_orig = bio->bi_private;
  102. struct bio_vec *bvec, *org_vec;
  103. int i;
  104. if (test_bit(BIO_EOPNOTSUPP, &bio->bi_flags))
  105. set_bit(BIO_EOPNOTSUPP, &bio_orig->bi_flags);
  106. /*
  107. * free up bounce indirect pages used
  108. */
  109. __bio_for_each_segment(bvec, bio, i, 0) {
  110. org_vec = bio_orig->bi_io_vec + i;
  111. if (bvec->bv_page == org_vec->bv_page)
  112. continue;
  113. dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
  114. mempool_free(bvec->bv_page, pool);
  115. }
  116. bio_endio(bio_orig, bio_orig->bi_size, err);
  117. bio_put(bio);
  118. }
  119. static int bounce_end_io_write(struct bio *bio, unsigned int bytes_done, int err)
  120. {
  121. if (bio->bi_size)
  122. return 1;
  123. bounce_end_io(bio, page_pool, err);
  124. return 0;
  125. }
  126. static int bounce_end_io_write_isa(struct bio *bio, unsigned int bytes_done, int err)
  127. {
  128. if (bio->bi_size)
  129. return 1;
  130. bounce_end_io(bio, isa_page_pool, err);
  131. return 0;
  132. }
  133. static void __bounce_end_io_read(struct bio *bio, mempool_t *pool, int err)
  134. {
  135. struct bio *bio_orig = bio->bi_private;
  136. if (test_bit(BIO_UPTODATE, &bio->bi_flags))
  137. copy_to_high_bio_irq(bio_orig, bio);
  138. bounce_end_io(bio, pool, err);
  139. }
  140. static int bounce_end_io_read(struct bio *bio, unsigned int bytes_done, int err)
  141. {
  142. if (bio->bi_size)
  143. return 1;
  144. __bounce_end_io_read(bio, page_pool, err);
  145. return 0;
  146. }
  147. static int bounce_end_io_read_isa(struct bio *bio, unsigned int bytes_done, int err)
  148. {
  149. if (bio->bi_size)
  150. return 1;
  151. __bounce_end_io_read(bio, isa_page_pool, err);
  152. return 0;
  153. }
  154. static void __blk_queue_bounce(request_queue_t *q, struct bio **bio_orig,
  155. mempool_t *pool)
  156. {
  157. struct page *page;
  158. struct bio *bio = NULL;
  159. int i, rw = bio_data_dir(*bio_orig);
  160. struct bio_vec *to, *from;
  161. bio_for_each_segment(from, *bio_orig, i) {
  162. page = from->bv_page;
  163. /*
  164. * is destination page below bounce pfn?
  165. */
  166. if (page_to_pfn(page) <= q->bounce_pfn)
  167. continue;
  168. /*
  169. * irk, bounce it
  170. */
  171. if (!bio)
  172. bio = bio_alloc(GFP_NOIO, (*bio_orig)->bi_vcnt);
  173. to = bio->bi_io_vec + i;
  174. to->bv_page = mempool_alloc(pool, q->bounce_gfp);
  175. to->bv_len = from->bv_len;
  176. to->bv_offset = from->bv_offset;
  177. inc_zone_page_state(to->bv_page, NR_BOUNCE);
  178. if (rw == WRITE) {
  179. char *vto, *vfrom;
  180. flush_dcache_page(from->bv_page);
  181. vto = page_address(to->bv_page) + to->bv_offset;
  182. vfrom = kmap(from->bv_page) + from->bv_offset;
  183. memcpy(vto, vfrom, to->bv_len);
  184. kunmap(from->bv_page);
  185. }
  186. }
  187. /*
  188. * no pages bounced
  189. */
  190. if (!bio)
  191. return;
  192. blk_add_trace_bio(q, *bio_orig, BLK_TA_BOUNCE);
  193. /*
  194. * at least one page was bounced, fill in possible non-highmem
  195. * pages
  196. */
  197. __bio_for_each_segment(from, *bio_orig, i, 0) {
  198. to = bio_iovec_idx(bio, i);
  199. if (!to->bv_page) {
  200. to->bv_page = from->bv_page;
  201. to->bv_len = from->bv_len;
  202. to->bv_offset = from->bv_offset;
  203. }
  204. }
  205. bio->bi_bdev = (*bio_orig)->bi_bdev;
  206. bio->bi_flags |= (1 << BIO_BOUNCED);
  207. bio->bi_sector = (*bio_orig)->bi_sector;
  208. bio->bi_rw = (*bio_orig)->bi_rw;
  209. bio->bi_vcnt = (*bio_orig)->bi_vcnt;
  210. bio->bi_idx = (*bio_orig)->bi_idx;
  211. bio->bi_size = (*bio_orig)->bi_size;
  212. if (pool == page_pool) {
  213. bio->bi_end_io = bounce_end_io_write;
  214. if (rw == READ)
  215. bio->bi_end_io = bounce_end_io_read;
  216. } else {
  217. bio->bi_end_io = bounce_end_io_write_isa;
  218. if (rw == READ)
  219. bio->bi_end_io = bounce_end_io_read_isa;
  220. }
  221. bio->bi_private = *bio_orig;
  222. *bio_orig = bio;
  223. }
  224. void blk_queue_bounce(request_queue_t *q, struct bio **bio_orig)
  225. {
  226. mempool_t *pool;
  227. /*
  228. * for non-isa bounce case, just check if the bounce pfn is equal
  229. * to or bigger than the highest pfn in the system -- in that case,
  230. * don't waste time iterating over bio segments
  231. */
  232. if (!(q->bounce_gfp & GFP_DMA)) {
  233. if (q->bounce_pfn >= blk_max_pfn)
  234. return;
  235. pool = page_pool;
  236. } else {
  237. BUG_ON(!isa_page_pool);
  238. pool = isa_page_pool;
  239. }
  240. /*
  241. * slow path
  242. */
  243. __blk_queue_bounce(q, bio_orig, pool);
  244. }
  245. EXPORT_SYMBOL(blk_queue_bounce);