bio.h 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * Copyright (C) 2001 Jens Axboe <axboe@suse.de>
  4. */
  5. #ifndef __LINUX_BIO_H
  6. #define __LINUX_BIO_H
  7. #include <linux/highmem.h>
  8. #include <linux/mempool.h>
  9. #include <linux/ioprio.h>
  10. /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */
  11. #include <linux/blk_types.h>
  12. #include <linux/android_kabi.h>
  13. #define BIO_DEBUG
  14. #ifdef BIO_DEBUG
  15. #define BIO_BUG_ON BUG_ON
  16. #else
  17. #define BIO_BUG_ON
  18. #endif
  19. #define BIO_MAX_PAGES 256
  20. #define bio_prio(bio) (bio)->bi_ioprio
  21. #define bio_set_prio(bio, prio) ((bio)->bi_ioprio = prio)
  22. #define bio_iter_iovec(bio, iter) \
  23. bvec_iter_bvec((bio)->bi_io_vec, (iter))
  24. #define bio_iter_page(bio, iter) \
  25. bvec_iter_page((bio)->bi_io_vec, (iter))
  26. #define bio_iter_len(bio, iter) \
  27. bvec_iter_len((bio)->bi_io_vec, (iter))
  28. #define bio_iter_offset(bio, iter) \
  29. bvec_iter_offset((bio)->bi_io_vec, (iter))
  30. #define bio_page(bio) bio_iter_page((bio), (bio)->bi_iter)
  31. #define bio_offset(bio) bio_iter_offset((bio), (bio)->bi_iter)
  32. #define bio_iovec(bio) bio_iter_iovec((bio), (bio)->bi_iter)
  33. #define bvec_iter_sectors(iter) ((iter).bi_size >> 9)
  34. #define bvec_iter_end_sector(iter) ((iter).bi_sector + bvec_iter_sectors((iter)))
  35. #define bio_sectors(bio) bvec_iter_sectors((bio)->bi_iter)
  36. #define bio_end_sector(bio) bvec_iter_end_sector((bio)->bi_iter)
  37. /*
  38. * Return the data direction, READ or WRITE.
  39. */
  40. #define bio_data_dir(bio) \
  41. (op_is_write(bio_op(bio)) ? WRITE : READ)
  42. /*
  43. * Check whether this bio carries any data or not. A NULL bio is allowed.
  44. */
  45. static inline bool bio_has_data(struct bio *bio)
  46. {
  47. if (bio &&
  48. bio->bi_iter.bi_size &&
  49. bio_op(bio) != REQ_OP_DISCARD &&
  50. bio_op(bio) != REQ_OP_SECURE_ERASE &&
  51. bio_op(bio) != REQ_OP_WRITE_ZEROES)
  52. return true;
  53. return false;
  54. }
  55. static inline bool bio_no_advance_iter(const struct bio *bio)
  56. {
  57. return bio_op(bio) == REQ_OP_DISCARD ||
  58. bio_op(bio) == REQ_OP_SECURE_ERASE ||
  59. bio_op(bio) == REQ_OP_WRITE_SAME ||
  60. bio_op(bio) == REQ_OP_WRITE_ZEROES;
  61. }
  62. static inline bool bio_mergeable(struct bio *bio)
  63. {
  64. if (bio->bi_opf & REQ_NOMERGE_FLAGS)
  65. return false;
  66. return true;
  67. }
  68. static inline unsigned int bio_cur_bytes(struct bio *bio)
  69. {
  70. if (bio_has_data(bio))
  71. return bio_iovec(bio).bv_len;
  72. else /* dataless requests such as discard */
  73. return bio->bi_iter.bi_size;
  74. }
  75. static inline void *bio_data(struct bio *bio)
  76. {
  77. if (bio_has_data(bio))
  78. return page_address(bio_page(bio)) + bio_offset(bio);
  79. return NULL;
  80. }
  81. /**
  82. * bio_full - check if the bio is full
  83. * @bio: bio to check
  84. * @len: length of one segment to be added
  85. *
  86. * Return true if @bio is full and one segment with @len bytes can't be
  87. * added to the bio, otherwise return false
  88. */
  89. static inline bool bio_full(struct bio *bio, unsigned len)
  90. {
  91. if (bio->bi_vcnt >= bio->bi_max_vecs)
  92. return true;
  93. if (bio->bi_iter.bi_size > UINT_MAX - len)
  94. return true;
  95. return false;
  96. }
  97. static inline bool bio_next_segment(const struct bio *bio,
  98. struct bvec_iter_all *iter)
  99. {
  100. if (iter->idx >= bio->bi_vcnt)
  101. return false;
  102. bvec_advance(&bio->bi_io_vec[iter->idx], iter);
  103. return true;
  104. }
  105. /*
  106. * drivers should _never_ use the all version - the bio may have been split
  107. * before it got to the driver and the driver won't own all of it
  108. */
  109. #define bio_for_each_segment_all(bvl, bio, iter) \
  110. for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); )
  111. static inline void bio_advance_iter(const struct bio *bio,
  112. struct bvec_iter *iter, unsigned int bytes)
  113. {
  114. iter->bi_sector += bytes >> 9;
  115. if (bio_no_advance_iter(bio))
  116. iter->bi_size -= bytes;
  117. else
  118. bvec_iter_advance(bio->bi_io_vec, iter, bytes);
  119. /* TODO: It is reasonable to complete bio with error here. */
  120. }
  121. #define __bio_for_each_segment(bvl, bio, iter, start) \
  122. for (iter = (start); \
  123. (iter).bi_size && \
  124. ((bvl = bio_iter_iovec((bio), (iter))), 1); \
  125. bio_advance_iter((bio), &(iter), (bvl).bv_len))
  126. #define bio_for_each_segment(bvl, bio, iter) \
  127. __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
  128. #define __bio_for_each_bvec(bvl, bio, iter, start) \
  129. for (iter = (start); \
  130. (iter).bi_size && \
  131. ((bvl = mp_bvec_iter_bvec((bio)->bi_io_vec, (iter))), 1); \
  132. bio_advance_iter((bio), &(iter), (bvl).bv_len))
  133. /* iterate over multi-page bvec */
  134. #define bio_for_each_bvec(bvl, bio, iter) \
  135. __bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter)
  136. /*
  137. * Iterate over all multi-page bvecs. Drivers shouldn't use this version for the
  138. * same reasons as bio_for_each_segment_all().
  139. */
  140. #define bio_for_each_bvec_all(bvl, bio, i) \
  141. for (i = 0, bvl = bio_first_bvec_all(bio); \
  142. i < (bio)->bi_vcnt; i++, bvl++) \
  143. #define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)
  144. static inline unsigned bio_segments(struct bio *bio)
  145. {
  146. unsigned segs = 0;
  147. struct bio_vec bv;
  148. struct bvec_iter iter;
  149. /*
  150. * We special case discard/write same/write zeroes, because they
  151. * interpret bi_size differently:
  152. */
  153. switch (bio_op(bio)) {
  154. case REQ_OP_DISCARD:
  155. case REQ_OP_SECURE_ERASE:
  156. case REQ_OP_WRITE_ZEROES:
  157. return 0;
  158. case REQ_OP_WRITE_SAME:
  159. return 1;
  160. default:
  161. break;
  162. }
  163. bio_for_each_segment(bv, bio, iter)
  164. segs++;
  165. return segs;
  166. }
  167. /*
  168. * get a reference to a bio, so it won't disappear. the intended use is
  169. * something like:
  170. *
  171. * bio_get(bio);
  172. * submit_bio(rw, bio);
  173. * if (bio->bi_flags ...)
  174. * do_something
  175. * bio_put(bio);
  176. *
  177. * without the bio_get(), it could potentially complete I/O before submit_bio
  178. * returns. and then bio would be freed memory when if (bio->bi_flags ...)
  179. * runs
  180. */
  181. static inline void bio_get(struct bio *bio)
  182. {
  183. bio->bi_flags |= (1 << BIO_REFFED);
  184. smp_mb__before_atomic();
  185. atomic_inc(&bio->__bi_cnt);
  186. }
  187. static inline void bio_cnt_set(struct bio *bio, unsigned int count)
  188. {
  189. if (count != 1) {
  190. bio->bi_flags |= (1 << BIO_REFFED);
  191. smp_mb();
  192. }
  193. atomic_set(&bio->__bi_cnt, count);
  194. }
  195. static inline bool bio_flagged(struct bio *bio, unsigned int bit)
  196. {
  197. return (bio->bi_flags & (1U << bit)) != 0;
  198. }
  199. static inline void bio_set_flag(struct bio *bio, unsigned int bit)
  200. {
  201. bio->bi_flags |= (1U << bit);
  202. }
  203. static inline void bio_clear_flag(struct bio *bio, unsigned int bit)
  204. {
  205. bio->bi_flags &= ~(1U << bit);
  206. }
  207. static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv)
  208. {
  209. *bv = mp_bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
  210. }
  211. static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv)
  212. {
  213. struct bvec_iter iter = bio->bi_iter;
  214. int idx;
  215. bio_get_first_bvec(bio, bv);
  216. if (bv->bv_len == bio->bi_iter.bi_size)
  217. return; /* this bio only has a single bvec */
  218. bio_advance_iter(bio, &iter, iter.bi_size);
  219. if (!iter.bi_bvec_done)
  220. idx = iter.bi_idx - 1;
  221. else /* in the middle of bvec */
  222. idx = iter.bi_idx;
  223. *bv = bio->bi_io_vec[idx];
  224. /*
  225. * iter.bi_bvec_done records actual length of the last bvec
  226. * if this bio ends in the middle of one io vector
  227. */
  228. if (iter.bi_bvec_done)
  229. bv->bv_len = iter.bi_bvec_done;
  230. }
  231. static inline struct bio_vec *bio_first_bvec_all(struct bio *bio)
  232. {
  233. WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
  234. return bio->bi_io_vec;
  235. }
  236. static inline struct page *bio_first_page_all(struct bio *bio)
  237. {
  238. return bio_first_bvec_all(bio)->bv_page;
  239. }
  240. static inline struct bio_vec *bio_last_bvec_all(struct bio *bio)
  241. {
  242. WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
  243. return &bio->bi_io_vec[bio->bi_vcnt - 1];
  244. }
  245. enum bip_flags {
  246. BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */
  247. BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */
  248. BIP_CTRL_NOCHECK = 1 << 2, /* disable HBA integrity checking */
  249. BIP_DISK_NOCHECK = 1 << 3, /* disable disk integrity checking */
  250. BIP_IP_CHECKSUM = 1 << 4, /* IP checksum */
  251. };
  252. /*
  253. * bio integrity payload
  254. */
  255. struct bio_integrity_payload {
  256. struct bio *bip_bio; /* parent bio */
  257. struct bvec_iter bip_iter;
  258. unsigned short bip_slab; /* slab the bip came from */
  259. unsigned short bip_vcnt; /* # of integrity bio_vecs */
  260. unsigned short bip_max_vcnt; /* integrity bio_vec slots */
  261. unsigned short bip_flags; /* control flags */
  262. struct bvec_iter bio_iter; /* for rewinding parent bio */
  263. struct work_struct bip_work; /* I/O completion */
  264. struct bio_vec *bip_vec;
  265. ANDROID_KABI_RESERVE(1);
  266. ANDROID_KABI_RESERVE(2);
  267. struct bio_vec bip_inline_vecs[];/* embedded bvec array */
  268. };
  269. #if defined(CONFIG_BLK_DEV_INTEGRITY)
  270. static inline struct bio_integrity_payload *bio_integrity(struct bio *bio)
  271. {
  272. if (bio->bi_opf & REQ_INTEGRITY)
  273. return bio->bi_integrity;
  274. return NULL;
  275. }
  276. static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
  277. {
  278. struct bio_integrity_payload *bip = bio_integrity(bio);
  279. if (bip)
  280. return bip->bip_flags & flag;
  281. return false;
  282. }
  283. static inline sector_t bip_get_seed(struct bio_integrity_payload *bip)
  284. {
  285. return bip->bip_iter.bi_sector;
  286. }
  287. static inline void bip_set_seed(struct bio_integrity_payload *bip,
  288. sector_t seed)
  289. {
  290. bip->bip_iter.bi_sector = seed;
  291. }
  292. #endif /* CONFIG_BLK_DEV_INTEGRITY */
  293. extern void bio_trim(struct bio *bio, int offset, int size);
  294. extern struct bio *bio_split(struct bio *bio, int sectors,
  295. gfp_t gfp, struct bio_set *bs);
  296. /**
  297. * bio_next_split - get next @sectors from a bio, splitting if necessary
  298. * @bio: bio to split
  299. * @sectors: number of sectors to split from the front of @bio
  300. * @gfp: gfp mask
  301. * @bs: bio set to allocate from
  302. *
  303. * Returns a bio representing the next @sectors of @bio - if the bio is smaller
  304. * than @sectors, returns the original bio unchanged.
  305. */
  306. static inline struct bio *bio_next_split(struct bio *bio, int sectors,
  307. gfp_t gfp, struct bio_set *bs)
  308. {
  309. if (sectors >= bio_sectors(bio))
  310. return bio;
  311. return bio_split(bio, sectors, gfp, bs);
  312. }
  313. enum {
  314. BIOSET_NEED_BVECS = BIT(0),
  315. BIOSET_NEED_RESCUER = BIT(1),
  316. };
  317. extern int bioset_init(struct bio_set *, unsigned int, unsigned int, int flags);
  318. extern void bioset_exit(struct bio_set *);
  319. extern int biovec_init_pool(mempool_t *pool, int pool_entries);
  320. extern int bioset_init_from_src(struct bio_set *bs, struct bio_set *src);
  321. extern struct bio *bio_alloc_bioset(gfp_t, unsigned int, struct bio_set *);
  322. extern void bio_put(struct bio *);
  323. extern void __bio_clone_fast(struct bio *, struct bio *);
  324. extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *);
  325. extern struct bio_set fs_bio_set;
  326. static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
  327. {
  328. return bio_alloc_bioset(gfp_mask, nr_iovecs, &fs_bio_set);
  329. }
  330. static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs)
  331. {
  332. return bio_alloc_bioset(gfp_mask, nr_iovecs, NULL);
  333. }
  334. extern blk_qc_t submit_bio(struct bio *);
  335. extern void bio_endio(struct bio *);
  336. static inline void bio_io_error(struct bio *bio)
  337. {
  338. bio->bi_status = BLK_STS_IOERR;
  339. bio_endio(bio);
  340. }
  341. static inline void bio_wouldblock_error(struct bio *bio)
  342. {
  343. bio_set_flag(bio, BIO_QUIET);
  344. bio->bi_status = BLK_STS_AGAIN;
  345. bio_endio(bio);
  346. }
  347. struct request_queue;
  348. extern int submit_bio_wait(struct bio *bio);
  349. extern void bio_advance(struct bio *, unsigned);
  350. extern void bio_init(struct bio *bio, struct bio_vec *table,
  351. unsigned short max_vecs);
  352. extern void bio_uninit(struct bio *);
  353. extern void bio_reset(struct bio *);
  354. void bio_chain(struct bio *, struct bio *);
  355. extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
  356. extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
  357. unsigned int, unsigned int);
  358. bool __bio_try_merge_page(struct bio *bio, struct page *page,
  359. unsigned int len, unsigned int off, bool *same_page);
  360. void __bio_add_page(struct bio *bio, struct page *page,
  361. unsigned int len, unsigned int off);
  362. int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);
  363. void bio_release_pages(struct bio *bio, bool mark_dirty);
  364. extern void bio_set_pages_dirty(struct bio *bio);
  365. extern void bio_check_pages_dirty(struct bio *bio);
  366. extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
  367. struct bio *src, struct bvec_iter *src_iter);
  368. extern void bio_copy_data(struct bio *dst, struct bio *src);
  369. extern void bio_list_copy_data(struct bio *dst, struct bio *src);
  370. extern void bio_free_pages(struct bio *bio);
  371. void zero_fill_bio_iter(struct bio *bio, struct bvec_iter iter);
  372. void bio_truncate(struct bio *bio, unsigned new_size);
  373. void guard_bio_eod(struct bio *bio);
  374. static inline void zero_fill_bio(struct bio *bio)
  375. {
  376. zero_fill_bio_iter(bio, bio->bi_iter);
  377. }
  378. extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);
  379. extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int);
  380. extern unsigned int bvec_nr_vecs(unsigned short idx);
  381. extern const char *bio_devname(struct bio *bio, char *buffer);
  382. #define bio_set_dev(bio, bdev) \
  383. do { \
  384. if ((bio)->bi_disk != (bdev)->bd_disk) \
  385. bio_clear_flag(bio, BIO_THROTTLED);\
  386. (bio)->bi_disk = (bdev)->bd_disk; \
  387. (bio)->bi_partno = (bdev)->bd_partno; \
  388. bio_associate_blkg(bio); \
  389. } while (0)
  390. #define bio_copy_dev(dst, src) \
  391. do { \
  392. (dst)->bi_disk = (src)->bi_disk; \
  393. (dst)->bi_partno = (src)->bi_partno; \
  394. bio_clone_blkg_association(dst, src); \
  395. } while (0)
  396. #define bio_dev(bio) \
  397. disk_devt((bio)->bi_disk)
  398. #ifdef CONFIG_BLK_CGROUP
  399. void bio_associate_blkg(struct bio *bio);
  400. void bio_associate_blkg_from_css(struct bio *bio,
  401. struct cgroup_subsys_state *css);
  402. void bio_clone_blkg_association(struct bio *dst, struct bio *src);
  403. #else /* CONFIG_BLK_CGROUP */
  404. static inline void bio_associate_blkg(struct bio *bio) { }
  405. static inline void bio_associate_blkg_from_css(struct bio *bio,
  406. struct cgroup_subsys_state *css)
  407. { }
  408. static inline void bio_clone_blkg_association(struct bio *dst,
  409. struct bio *src) { }
  410. #endif /* CONFIG_BLK_CGROUP */
  411. #ifdef CONFIG_HIGHMEM
  412. /*
  413. * remember never ever reenable interrupts between a bvec_kmap_irq and
  414. * bvec_kunmap_irq!
  415. */
  416. static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
  417. {
  418. unsigned long addr;
  419. /*
  420. * might not be a highmem page, but the preempt/irq count
  421. * balancing is a lot nicer this way
  422. */
  423. local_irq_save(*flags);
  424. addr = (unsigned long) kmap_atomic(bvec->bv_page);
  425. BUG_ON(addr & ~PAGE_MASK);
  426. return (char *) addr + bvec->bv_offset;
  427. }
  428. static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
  429. {
  430. unsigned long ptr = (unsigned long) buffer & PAGE_MASK;
  431. kunmap_atomic((void *) ptr);
  432. local_irq_restore(*flags);
  433. }
  434. #else
  435. static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
  436. {
  437. return page_address(bvec->bv_page) + bvec->bv_offset;
  438. }
  439. static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
  440. {
  441. *flags = 0;
  442. }
  443. #endif
  444. /*
  445. * BIO list management for use by remapping drivers (e.g. DM or MD) and loop.
  446. *
  447. * A bio_list anchors a singly-linked list of bios chained through the bi_next
  448. * member of the bio. The bio_list also caches the last list member to allow
  449. * fast access to the tail.
  450. */
  451. struct bio_list {
  452. struct bio *head;
  453. struct bio *tail;
  454. };
  455. static inline int bio_list_empty(const struct bio_list *bl)
  456. {
  457. return bl->head == NULL;
  458. }
  459. static inline void bio_list_init(struct bio_list *bl)
  460. {
  461. bl->head = bl->tail = NULL;
  462. }
  463. #define BIO_EMPTY_LIST { NULL, NULL }
  464. #define bio_list_for_each(bio, bl) \
  465. for (bio = (bl)->head; bio; bio = bio->bi_next)
  466. static inline unsigned bio_list_size(const struct bio_list *bl)
  467. {
  468. unsigned sz = 0;
  469. struct bio *bio;
  470. bio_list_for_each(bio, bl)
  471. sz++;
  472. return sz;
  473. }
  474. static inline void bio_list_add(struct bio_list *bl, struct bio *bio)
  475. {
  476. bio->bi_next = NULL;
  477. if (bl->tail)
  478. bl->tail->bi_next = bio;
  479. else
  480. bl->head = bio;
  481. bl->tail = bio;
  482. }
  483. static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)
  484. {
  485. bio->bi_next = bl->head;
  486. bl->head = bio;
  487. if (!bl->tail)
  488. bl->tail = bio;
  489. }
  490. static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
  491. {
  492. if (!bl2->head)
  493. return;
  494. if (bl->tail)
  495. bl->tail->bi_next = bl2->head;
  496. else
  497. bl->head = bl2->head;
  498. bl->tail = bl2->tail;
  499. }
  500. static inline void bio_list_merge_head(struct bio_list *bl,
  501. struct bio_list *bl2)
  502. {
  503. if (!bl2->head)
  504. return;
  505. if (bl->head)
  506. bl2->tail->bi_next = bl->head;
  507. else
  508. bl->tail = bl2->tail;
  509. bl->head = bl2->head;
  510. }
  511. static inline struct bio *bio_list_peek(struct bio_list *bl)
  512. {
  513. return bl->head;
  514. }
  515. static inline struct bio *bio_list_pop(struct bio_list *bl)
  516. {
  517. struct bio *bio = bl->head;
  518. if (bio) {
  519. bl->head = bl->head->bi_next;
  520. if (!bl->head)
  521. bl->tail = NULL;
  522. bio->bi_next = NULL;
  523. }
  524. return bio;
  525. }
  526. static inline struct bio *bio_list_get(struct bio_list *bl)
  527. {
  528. struct bio *bio = bl->head;
  529. bl->head = bl->tail = NULL;
  530. return bio;
  531. }
  532. /*
  533. * Increment chain count for the bio. Make sure the CHAIN flag update
  534. * is visible before the raised count.
  535. */
  536. static inline void bio_inc_remaining(struct bio *bio)
  537. {
  538. bio_set_flag(bio, BIO_CHAIN);
  539. smp_mb__before_atomic();
  540. atomic_inc(&bio->__bi_remaining);
  541. }
  542. /*
  543. * bio_set is used to allow other portions of the IO system to
  544. * allocate their own private memory pools for bio and iovec structures.
  545. * These memory pools in turn all allocate from the bio_slab
  546. * and the bvec_slabs[].
  547. */
  548. #define BIO_POOL_SIZE 2
  549. struct bio_set {
  550. struct kmem_cache *bio_slab;
  551. unsigned int front_pad;
  552. mempool_t bio_pool;
  553. mempool_t bvec_pool;
  554. #if defined(CONFIG_BLK_DEV_INTEGRITY)
  555. mempool_t bio_integrity_pool;
  556. mempool_t bvec_integrity_pool;
  557. #endif
  558. /*
  559. * Deadlock avoidance for stacking block drivers: see comments in
  560. * bio_alloc_bioset() for details
  561. */
  562. spinlock_t rescue_lock;
  563. struct bio_list rescue_list;
  564. struct work_struct rescue_work;
  565. struct workqueue_struct *rescue_workqueue;
  566. ANDROID_KABI_RESERVE(1);
  567. ANDROID_KABI_RESERVE(2);
  568. ANDROID_KABI_RESERVE(3);
  569. ANDROID_KABI_RESERVE(4);
  570. };
  571. struct biovec_slab {
  572. int nr_vecs;
  573. char *name;
  574. struct kmem_cache *slab;
  575. };
  576. static inline bool bioset_initialized(struct bio_set *bs)
  577. {
  578. return bs->bio_slab != NULL;
  579. }
  580. /*
  581. * a small number of entries is fine, not going to be performance critical.
  582. * basically we just need to survive
  583. */
  584. #define BIO_SPLIT_ENTRIES 2
  585. #if defined(CONFIG_BLK_DEV_INTEGRITY)
  586. #define bip_for_each_vec(bvl, bip, iter) \
  587. for_each_bvec(bvl, (bip)->bip_vec, iter, (bip)->bip_iter)
  588. #define bio_for_each_integrity_vec(_bvl, _bio, _iter) \
  589. for_each_bio(_bio) \
  590. bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)
  591. extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
  592. extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
  593. extern bool bio_integrity_prep(struct bio *);
  594. extern void bio_integrity_advance(struct bio *, unsigned int);
  595. extern void bio_integrity_trim(struct bio *);
  596. extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
  597. extern int bioset_integrity_create(struct bio_set *, int);
  598. extern void bioset_integrity_free(struct bio_set *);
  599. extern void bio_integrity_init(void);
  600. #else /* CONFIG_BLK_DEV_INTEGRITY */
  601. static inline void *bio_integrity(struct bio *bio)
  602. {
  603. return NULL;
  604. }
  605. static inline int bioset_integrity_create(struct bio_set *bs, int pool_size)
  606. {
  607. return 0;
  608. }
  609. static inline void bioset_integrity_free (struct bio_set *bs)
  610. {
  611. return;
  612. }
  613. static inline bool bio_integrity_prep(struct bio *bio)
  614. {
  615. return true;
  616. }
  617. static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
  618. gfp_t gfp_mask)
  619. {
  620. return 0;
  621. }
  622. static inline void bio_integrity_advance(struct bio *bio,
  623. unsigned int bytes_done)
  624. {
  625. return;
  626. }
  627. static inline void bio_integrity_trim(struct bio *bio)
  628. {
  629. return;
  630. }
  631. static inline void bio_integrity_init(void)
  632. {
  633. return;
  634. }
  635. static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
  636. {
  637. return false;
  638. }
  639. static inline void *bio_integrity_alloc(struct bio * bio, gfp_t gfp,
  640. unsigned int nr)
  641. {
  642. return ERR_PTR(-EINVAL);
  643. }
  644. static inline int bio_integrity_add_page(struct bio *bio, struct page *page,
  645. unsigned int len, unsigned int offset)
  646. {
  647. return 0;
  648. }
  649. #endif /* CONFIG_BLK_DEV_INTEGRITY */
  650. /*
  651. * Mark a bio as polled. Note that for async polled IO, the caller must
  652. * expect -EWOULDBLOCK if we cannot allocate a request (or other resources).
  653. * We cannot block waiting for requests on polled IO, as those completions
  654. * must be found by the caller. This is different than IRQ driven IO, where
  655. * it's safe to wait for IO to complete.
  656. */
  657. static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb)
  658. {
  659. bio->bi_opf |= REQ_HIPRI;
  660. if (!is_sync_kiocb(kiocb))
  661. bio->bi_opf |= REQ_NOWAIT;
  662. }
  663. #endif /* __LINUX_BIO_H */