blk_types.h 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * Block data types and constants. Directly include this file only to
  4. * break include dependency loop.
  5. */
  6. #ifndef __LINUX_BLK_TYPES_H
  7. #define __LINUX_BLK_TYPES_H
  8. #include <linux/types.h>
  9. #include <linux/bvec.h>
  10. #include <linux/ktime.h>
  11. #include <linux/android_kabi.h>
  12. struct bio_set;
  13. struct bio;
  14. struct bio_integrity_payload;
  15. struct page;
  16. struct io_context;
  17. struct cgroup_subsys_state;
  18. typedef void (bio_end_io_t) (struct bio *);
  19. struct bio_crypt_ctx;
  20. struct block_device {
  21. dev_t bd_dev;
  22. int bd_openers;
  23. struct inode * bd_inode; /* will die */
  24. struct super_block * bd_super;
  25. struct mutex bd_mutex; /* open/close mutex */
  26. void * bd_claiming;
  27. void * bd_holder;
  28. int bd_holders;
  29. bool bd_write_holder;
  30. #ifdef CONFIG_SYSFS
  31. struct list_head bd_holder_disks;
  32. #endif
  33. struct block_device * bd_contains;
  34. u8 bd_partno;
  35. struct hd_struct * bd_part;
  36. /* number of times partitions within this device have been opened. */
  37. unsigned bd_part_count;
  38. spinlock_t bd_size_lock; /* for bd_inode->i_size updates */
  39. struct gendisk * bd_disk;
  40. struct backing_dev_info *bd_bdi;
  41. /* The counter of freeze processes */
  42. int bd_fsfreeze_count;
  43. /* Mutex for freeze */
  44. struct mutex bd_fsfreeze_mutex;
  45. struct super_block *bd_fsfreeze_sb;
  46. ANDROID_KABI_RESERVE(1);
  47. ANDROID_KABI_RESERVE(2);
  48. ANDROID_KABI_RESERVE(3);
  49. ANDROID_KABI_RESERVE(4);
  50. } __randomize_layout;
  51. /*
  52. * Block error status values. See block/blk-core:blk_errors for the details.
  53. * Alpha cannot write a byte atomically, so we need to use 32-bit value.
  54. */
  55. #if defined(CONFIG_ALPHA) && !defined(__alpha_bwx__)
  56. typedef u32 __bitwise blk_status_t;
  57. #else
  58. typedef u8 __bitwise blk_status_t;
  59. #endif
  60. #define BLK_STS_OK 0
  61. #define BLK_STS_NOTSUPP ((__force blk_status_t)1)
  62. #define BLK_STS_TIMEOUT ((__force blk_status_t)2)
  63. #define BLK_STS_NOSPC ((__force blk_status_t)3)
  64. #define BLK_STS_TRANSPORT ((__force blk_status_t)4)
  65. #define BLK_STS_TARGET ((__force blk_status_t)5)
  66. #define BLK_STS_NEXUS ((__force blk_status_t)6)
  67. #define BLK_STS_MEDIUM ((__force blk_status_t)7)
  68. #define BLK_STS_PROTECTION ((__force blk_status_t)8)
  69. #define BLK_STS_RESOURCE ((__force blk_status_t)9)
  70. #define BLK_STS_IOERR ((__force blk_status_t)10)
  71. /* hack for device mapper, don't use elsewhere: */
  72. #define BLK_STS_DM_REQUEUE ((__force blk_status_t)11)
  73. #define BLK_STS_AGAIN ((__force blk_status_t)12)
  74. /*
  75. * BLK_STS_DEV_RESOURCE is returned from the driver to the block layer if
  76. * device related resources are unavailable, but the driver can guarantee
  77. * that the queue will be rerun in the future once resources become
  78. * available again. This is typically the case for device specific
  79. * resources that are consumed for IO. If the driver fails allocating these
  80. * resources, we know that inflight (or pending) IO will free these
  81. * resource upon completion.
  82. *
  83. * This is different from BLK_STS_RESOURCE in that it explicitly references
  84. * a device specific resource. For resources of wider scope, allocation
  85. * failure can happen without having pending IO. This means that we can't
  86. * rely on request completions freeing these resources, as IO may not be in
  87. * flight. Examples of that are kernel memory allocations, DMA mappings, or
  88. * any other system wide resources.
  89. */
  90. #define BLK_STS_DEV_RESOURCE ((__force blk_status_t)13)
  91. /*
  92. * BLK_STS_ZONE_RESOURCE is returned from the driver to the block layer if zone
  93. * related resources are unavailable, but the driver can guarantee the queue
  94. * will be rerun in the future once the resources become available again.
  95. *
  96. * This is different from BLK_STS_DEV_RESOURCE in that it explicitly references
  97. * a zone specific resource and IO to a different zone on the same device could
  98. * still be served. Examples of that are zones that are write-locked, but a read
  99. * to the same zone could be served.
  100. */
  101. #define BLK_STS_ZONE_RESOURCE ((__force blk_status_t)14)
  102. /*
  103. * BLK_STS_ZONE_OPEN_RESOURCE is returned from the driver in the completion
  104. * path if the device returns a status indicating that too many zone resources
  105. * are currently open. The same command should be successful if resubmitted
  106. * after the number of open zones decreases below the device's limits, which is
  107. * reported in the request_queue's max_open_zones.
  108. */
  109. #define BLK_STS_ZONE_OPEN_RESOURCE ((__force blk_status_t)15)
  110. /*
  111. * BLK_STS_ZONE_ACTIVE_RESOURCE is returned from the driver in the completion
  112. * path if the device returns a status indicating that too many zone resources
  113. * are currently active. The same command should be successful if resubmitted
  114. * after the number of active zones decreases below the device's limits, which
  115. * is reported in the request_queue's max_active_zones.
  116. */
  117. #define BLK_STS_ZONE_ACTIVE_RESOURCE ((__force blk_status_t)16)
  118. /**
  119. * blk_path_error - returns true if error may be path related
  120. * @error: status the request was completed with
  121. *
  122. * Description:
  123. * This classifies block error status into non-retryable errors and ones
  124. * that may be successful if retried on a failover path.
  125. *
  126. * Return:
  127. * %false - retrying failover path will not help
  128. * %true - may succeed if retried
  129. */
  130. static inline bool blk_path_error(blk_status_t error)
  131. {
  132. switch (error) {
  133. case BLK_STS_NOTSUPP:
  134. case BLK_STS_NOSPC:
  135. case BLK_STS_TARGET:
  136. case BLK_STS_NEXUS:
  137. case BLK_STS_MEDIUM:
  138. case BLK_STS_PROTECTION:
  139. return false;
  140. }
  141. /* Anything else could be a path failure, so should be retried */
  142. return true;
  143. }
  144. /*
  145. * From most significant bit:
  146. * 1 bit: reserved for other usage, see below
  147. * 12 bits: original size of bio
  148. * 51 bits: issue time of bio
  149. */
  150. #define BIO_ISSUE_RES_BITS 1
  151. #define BIO_ISSUE_SIZE_BITS 12
  152. #define BIO_ISSUE_RES_SHIFT (64 - BIO_ISSUE_RES_BITS)
  153. #define BIO_ISSUE_SIZE_SHIFT (BIO_ISSUE_RES_SHIFT - BIO_ISSUE_SIZE_BITS)
  154. #define BIO_ISSUE_TIME_MASK ((1ULL << BIO_ISSUE_SIZE_SHIFT) - 1)
  155. #define BIO_ISSUE_SIZE_MASK \
  156. (((1ULL << BIO_ISSUE_SIZE_BITS) - 1) << BIO_ISSUE_SIZE_SHIFT)
  157. #define BIO_ISSUE_RES_MASK (~((1ULL << BIO_ISSUE_RES_SHIFT) - 1))
  158. /* Reserved bit for blk-throtl */
  159. #define BIO_ISSUE_THROTL_SKIP_LATENCY (1ULL << 63)
  160. struct bio_issue {
  161. u64 value;
  162. };
  163. static inline u64 __bio_issue_time(u64 time)
  164. {
  165. return time & BIO_ISSUE_TIME_MASK;
  166. }
  167. static inline u64 bio_issue_time(struct bio_issue *issue)
  168. {
  169. return __bio_issue_time(issue->value);
  170. }
  171. static inline sector_t bio_issue_size(struct bio_issue *issue)
  172. {
  173. return ((issue->value & BIO_ISSUE_SIZE_MASK) >> BIO_ISSUE_SIZE_SHIFT);
  174. }
  175. static inline void bio_issue_init(struct bio_issue *issue,
  176. sector_t size)
  177. {
  178. size &= (1ULL << BIO_ISSUE_SIZE_BITS) - 1;
  179. issue->value = ((issue->value & BIO_ISSUE_RES_MASK) |
  180. (ktime_get_ns() & BIO_ISSUE_TIME_MASK) |
  181. ((u64)size << BIO_ISSUE_SIZE_SHIFT));
  182. }
  183. /*
  184. * main unit of I/O for the block layer and lower layers (ie drivers and
  185. * stacking drivers)
  186. */
  187. struct bio {
  188. struct bio *bi_next; /* request queue link */
  189. struct gendisk *bi_disk;
  190. unsigned int bi_opf; /* bottom bits req flags,
  191. * top bits REQ_OP. Use
  192. * accessors.
  193. */
  194. unsigned short bi_flags; /* status, etc and bvec pool number */
  195. unsigned short bi_ioprio;
  196. unsigned short bi_write_hint;
  197. blk_status_t bi_status;
  198. u8 bi_partno;
  199. atomic_t __bi_remaining;
  200. struct bvec_iter bi_iter;
  201. bio_end_io_t *bi_end_io;
  202. void *bi_private;
  203. #ifdef CONFIG_BLK_CGROUP
  204. /*
  205. * Represents the association of the css and request_queue for the bio.
  206. * If a bio goes direct to device, it will not have a blkg as it will
  207. * not have a request_queue associated with it. The reference is put
  208. * on release of the bio.
  209. */
  210. struct blkcg_gq *bi_blkg;
  211. struct bio_issue bi_issue;
  212. #ifdef CONFIG_BLK_CGROUP_IOCOST
  213. u64 bi_iocost_cost;
  214. #endif
  215. #endif
  216. #ifdef CONFIG_BLK_INLINE_ENCRYPTION
  217. struct bio_crypt_ctx *bi_crypt_context;
  218. #if IS_ENABLED(CONFIG_DM_DEFAULT_KEY)
  219. bool bi_skip_dm_default_key;
  220. #endif
  221. #endif
  222. union {
  223. #if defined(CONFIG_BLK_DEV_INTEGRITY)
  224. struct bio_integrity_payload *bi_integrity; /* data integrity */
  225. #endif
  226. };
  227. unsigned short bi_vcnt; /* how many bio_vec's */
  228. /*
  229. * Everything starting with bi_max_vecs will be preserved by bio_reset()
  230. */
  231. unsigned short bi_max_vecs; /* max bvl_vecs we can hold */
  232. atomic_t __bi_cnt; /* pin count */
  233. struct bio_vec *bi_io_vec; /* the actual vec list */
  234. struct bio_set *bi_pool;
  235. ANDROID_KABI_RESERVE(1);
  236. ANDROID_KABI_RESERVE(2);
  237. /*
  238. * We can inline a number of vecs at the end of the bio, to avoid
  239. * double allocations for a small number of bio_vecs. This member
  240. * MUST obviously be kept at the very end of the bio.
  241. */
  242. struct bio_vec bi_inline_vecs[];
  243. };
  244. #define BIO_RESET_BYTES offsetof(struct bio, bi_max_vecs)
  245. /*
  246. * bio flags
  247. */
  248. enum {
  249. BIO_NO_PAGE_REF, /* don't put release vec pages */
  250. BIO_CLONED, /* doesn't own data */
  251. BIO_BOUNCED, /* bio is a bounce bio */
  252. BIO_WORKINGSET, /* contains userspace workingset pages */
  253. BIO_QUIET, /* Make BIO Quiet */
  254. BIO_CHAIN, /* chained bio, ->bi_remaining in effect */
  255. BIO_REFFED, /* bio has elevated ->bi_cnt */
  256. BIO_THROTTLED, /* This bio has already been subjected to
  257. * throttling rules. Don't do it again. */
  258. BIO_TRACE_COMPLETION, /* bio_endio() should trace the final completion
  259. * of this bio. */
  260. BIO_CGROUP_ACCT, /* has been accounted to a cgroup */
  261. BIO_TRACKED, /* set if bio goes through the rq_qos path */
  262. BIO_FLAG_LAST
  263. };
  264. /* See BVEC_POOL_OFFSET below before adding new flags */
  265. /*
  266. * We support 6 different bvec pools, the last one is magic in that it
  267. * is backed by a mempool.
  268. */
  269. #define BVEC_POOL_NR 6
  270. #define BVEC_POOL_MAX (BVEC_POOL_NR - 1)
  271. /*
  272. * Top 3 bits of bio flags indicate the pool the bvecs came from. We add
  273. * 1 to the actual index so that 0 indicates that there are no bvecs to be
  274. * freed.
  275. */
  276. #define BVEC_POOL_BITS (3)
  277. #define BVEC_POOL_OFFSET (16 - BVEC_POOL_BITS)
  278. #define BVEC_POOL_IDX(bio) ((bio)->bi_flags >> BVEC_POOL_OFFSET)
  279. #if (1<< BVEC_POOL_BITS) < (BVEC_POOL_NR+1)
  280. # error "BVEC_POOL_BITS is too small"
  281. #endif
  282. /*
  283. * Flags starting here get preserved by bio_reset() - this includes
  284. * only BVEC_POOL_IDX()
  285. */
  286. #define BIO_RESET_BITS BVEC_POOL_OFFSET
  287. typedef __u32 __bitwise blk_mq_req_flags_t;
  288. /*
  289. * Operations and flags common to the bio and request structures.
  290. * We use 8 bits for encoding the operation, and the remaining 24 for flags.
  291. *
  292. * The least significant bit of the operation number indicates the data
  293. * transfer direction:
  294. *
  295. * - if the least significant bit is set transfers are TO the device
  296. * - if the least significant bit is not set transfers are FROM the device
  297. *
  298. * If a operation does not transfer data the least significant bit has no
  299. * meaning.
  300. */
  301. #define REQ_OP_BITS 8
  302. #define REQ_OP_MASK ((1 << REQ_OP_BITS) - 1)
  303. #define REQ_FLAG_BITS 24
  304. enum req_opf {
  305. /* read sectors from the device */
  306. REQ_OP_READ = 0,
  307. /* write sectors to the device */
  308. REQ_OP_WRITE = 1,
  309. /* flush the volatile write cache */
  310. REQ_OP_FLUSH = 2,
  311. /* discard sectors */
  312. REQ_OP_DISCARD = 3,
  313. /* securely erase sectors */
  314. REQ_OP_SECURE_ERASE = 5,
  315. /* write the same sector many times */
  316. REQ_OP_WRITE_SAME = 7,
  317. /* write the zero filled sector many times */
  318. REQ_OP_WRITE_ZEROES = 9,
  319. /* Open a zone */
  320. REQ_OP_ZONE_OPEN = 10,
  321. /* Close a zone */
  322. REQ_OP_ZONE_CLOSE = 11,
  323. /* Transition a zone to full */
  324. REQ_OP_ZONE_FINISH = 12,
  325. /* write data at the current zone write pointer */
  326. REQ_OP_ZONE_APPEND = 13,
  327. /* reset a zone write pointer */
  328. REQ_OP_ZONE_RESET = 15,
  329. /* reset all the zone present on the device */
  330. REQ_OP_ZONE_RESET_ALL = 17,
  331. /* SCSI passthrough using struct scsi_request */
  332. REQ_OP_SCSI_IN = 32,
  333. REQ_OP_SCSI_OUT = 33,
  334. /* Driver private requests */
  335. REQ_OP_DRV_IN = 34,
  336. REQ_OP_DRV_OUT = 35,
  337. REQ_OP_LAST,
  338. };
  339. enum req_flag_bits {
  340. __REQ_FAILFAST_DEV = /* no driver retries of device errors */
  341. REQ_OP_BITS,
  342. __REQ_FAILFAST_TRANSPORT, /* no driver retries of transport errors */
  343. __REQ_FAILFAST_DRIVER, /* no driver retries of driver errors */
  344. __REQ_SYNC, /* request is sync (sync write or read) */
  345. __REQ_META, /* metadata io request */
  346. __REQ_PRIO, /* boost priority in cfq */
  347. __REQ_NOMERGE, /* don't touch this for merging */
  348. __REQ_IDLE, /* anticipate more IO after this one */
  349. __REQ_INTEGRITY, /* I/O includes block integrity payload */
  350. __REQ_FUA, /* forced unit access */
  351. __REQ_PREFLUSH, /* request for cache flush */
  352. __REQ_RAHEAD, /* read ahead, can fail anytime */
  353. __REQ_BACKGROUND, /* background IO */
  354. __REQ_NOWAIT, /* Don't wait if request will block */
  355. /*
  356. * When a shared kthread needs to issue a bio for a cgroup, doing
  357. * so synchronously can lead to priority inversions as the kthread
  358. * can be trapped waiting for that cgroup. CGROUP_PUNT flag makes
  359. * submit_bio() punt the actual issuing to a dedicated per-blkcg
  360. * work item to avoid such priority inversions.
  361. */
  362. __REQ_CGROUP_PUNT,
  363. /* command specific flags for REQ_OP_WRITE_ZEROES: */
  364. __REQ_NOUNMAP, /* do not free blocks when zeroing */
  365. __REQ_HIPRI,
  366. /* for driver use */
  367. __REQ_DRV,
  368. __REQ_SWAP, /* swapping request. */
  369. __REQ_NR_BITS, /* stops here */
  370. };
  371. #define REQ_FAILFAST_DEV (1ULL << __REQ_FAILFAST_DEV)
  372. #define REQ_FAILFAST_TRANSPORT (1ULL << __REQ_FAILFAST_TRANSPORT)
  373. #define REQ_FAILFAST_DRIVER (1ULL << __REQ_FAILFAST_DRIVER)
  374. #define REQ_SYNC (1ULL << __REQ_SYNC)
  375. #define REQ_META (1ULL << __REQ_META)
  376. #define REQ_PRIO (1ULL << __REQ_PRIO)
  377. #define REQ_NOMERGE (1ULL << __REQ_NOMERGE)
  378. #define REQ_IDLE (1ULL << __REQ_IDLE)
  379. #define REQ_INTEGRITY (1ULL << __REQ_INTEGRITY)
  380. #define REQ_FUA (1ULL << __REQ_FUA)
  381. #define REQ_PREFLUSH (1ULL << __REQ_PREFLUSH)
  382. #define REQ_RAHEAD (1ULL << __REQ_RAHEAD)
  383. #define REQ_BACKGROUND (1ULL << __REQ_BACKGROUND)
  384. #define REQ_NOWAIT (1ULL << __REQ_NOWAIT)
  385. #define REQ_CGROUP_PUNT (1ULL << __REQ_CGROUP_PUNT)
  386. #define REQ_NOUNMAP (1ULL << __REQ_NOUNMAP)
  387. #define REQ_HIPRI (1ULL << __REQ_HIPRI)
  388. #define REQ_DRV (1ULL << __REQ_DRV)
  389. #define REQ_SWAP (1ULL << __REQ_SWAP)
  390. #define REQ_FAILFAST_MASK \
  391. (REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER)
  392. #define REQ_NOMERGE_FLAGS \
  393. (REQ_NOMERGE | REQ_PREFLUSH | REQ_FUA)
  394. enum stat_group {
  395. STAT_READ,
  396. STAT_WRITE,
  397. STAT_DISCARD,
  398. STAT_FLUSH,
  399. NR_STAT_GROUPS
  400. };
  401. #define bio_op(bio) \
  402. ((bio)->bi_opf & REQ_OP_MASK)
  403. #define req_op(req) \
  404. ((req)->cmd_flags & REQ_OP_MASK)
  405. /* obsolete, don't use in new code */
  406. static inline void bio_set_op_attrs(struct bio *bio, unsigned op,
  407. unsigned op_flags)
  408. {
  409. bio->bi_opf = op | op_flags;
  410. }
  411. static inline bool op_is_write(unsigned int op)
  412. {
  413. return (op & 1);
  414. }
  415. /*
  416. * Check if the bio or request is one that needs special treatment in the
  417. * flush state machine.
  418. */
  419. static inline bool op_is_flush(unsigned int op)
  420. {
  421. return op & (REQ_FUA | REQ_PREFLUSH);
  422. }
  423. /*
  424. * Reads are always treated as synchronous, as are requests with the FUA or
  425. * PREFLUSH flag. Other operations may be marked as synchronous using the
  426. * REQ_SYNC flag.
  427. */
  428. static inline bool op_is_sync(unsigned int op)
  429. {
  430. return (op & REQ_OP_MASK) == REQ_OP_READ ||
  431. (op & (REQ_SYNC | REQ_FUA | REQ_PREFLUSH));
  432. }
  433. static inline bool op_is_discard(unsigned int op)
  434. {
  435. return (op & REQ_OP_MASK) == REQ_OP_DISCARD;
  436. }
  437. /*
  438. * Check if a bio or request operation is a zone management operation, with
  439. * the exception of REQ_OP_ZONE_RESET_ALL which is treated as a special case
  440. * due to its different handling in the block layer and device response in
  441. * case of command failure.
  442. */
  443. static inline bool op_is_zone_mgmt(enum req_opf op)
  444. {
  445. switch (op & REQ_OP_MASK) {
  446. case REQ_OP_ZONE_RESET:
  447. case REQ_OP_ZONE_OPEN:
  448. case REQ_OP_ZONE_CLOSE:
  449. case REQ_OP_ZONE_FINISH:
  450. return true;
  451. default:
  452. return false;
  453. }
  454. }
  455. static inline int op_stat_group(unsigned int op)
  456. {
  457. if (op_is_discard(op))
  458. return STAT_DISCARD;
  459. return op_is_write(op);
  460. }
  461. typedef unsigned int blk_qc_t;
  462. #define BLK_QC_T_NONE -1U
  463. #define BLK_QC_T_SHIFT 16
  464. #define BLK_QC_T_INTERNAL (1U << 31)
  465. static inline bool blk_qc_t_valid(blk_qc_t cookie)
  466. {
  467. return cookie != BLK_QC_T_NONE;
  468. }
  469. static inline unsigned int blk_qc_t_to_queue_num(blk_qc_t cookie)
  470. {
  471. return (cookie & ~BLK_QC_T_INTERNAL) >> BLK_QC_T_SHIFT;
  472. }
  473. static inline unsigned int blk_qc_t_to_tag(blk_qc_t cookie)
  474. {
  475. return cookie & ((1u << BLK_QC_T_SHIFT) - 1);
  476. }
  477. static inline bool blk_qc_t_is_internal(blk_qc_t cookie)
  478. {
  479. return (cookie & BLK_QC_T_INTERNAL) != 0;
  480. }
  481. struct blk_rq_stat {
  482. u64 mean;
  483. u64 min;
  484. u64 max;
  485. u32 nr_samples;
  486. u64 batch;
  487. };
  488. #endif /* __LINUX_BLK_TYPES_H */