aio.h 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253
  1. #ifndef __LINUX__AIO_H
  2. #define __LINUX__AIO_H
  3. #include <linux/list.h>
  4. #include <linux/workqueue.h>
  5. #include <linux/aio_abi.h>
  6. #include <linux/uio.h>
  7. #include <asm/atomic.h>
  8. #include <linux/uio.h>
  9. #define AIO_MAXSEGS 4
  10. #define AIO_KIOGRP_NR_ATOMIC 8
  11. struct kioctx;
  12. /* Notes on cancelling a kiocb:
  13. * If a kiocb is cancelled, aio_complete may return 0 to indicate
  14. * that cancel has not yet disposed of the kiocb. All cancel
  15. * operations *must* call aio_put_req to dispose of the kiocb
  16. * to guard against races with the completion code.
  17. */
  18. #define KIOCB_C_CANCELLED 0x01
  19. #define KIOCB_C_COMPLETE 0x02
  20. #define KIOCB_SYNC_KEY (~0U)
  21. /* ki_flags bits */
  22. /*
  23. * This may be used for cancel/retry serialization in the future, but
  24. * for now it's unused and we probably don't want modules to even
  25. * think they can use it.
  26. */
  27. /* #define KIF_LOCKED 0 */
  28. #define KIF_KICKED 1
  29. #define KIF_CANCELLED 2
  30. #define kiocbTryLock(iocb) test_and_set_bit(KIF_LOCKED, &(iocb)->ki_flags)
  31. #define kiocbTryKick(iocb) test_and_set_bit(KIF_KICKED, &(iocb)->ki_flags)
  32. #define kiocbSetLocked(iocb) set_bit(KIF_LOCKED, &(iocb)->ki_flags)
  33. #define kiocbSetKicked(iocb) set_bit(KIF_KICKED, &(iocb)->ki_flags)
  34. #define kiocbSetCancelled(iocb) set_bit(KIF_CANCELLED, &(iocb)->ki_flags)
  35. #define kiocbClearLocked(iocb) clear_bit(KIF_LOCKED, &(iocb)->ki_flags)
  36. #define kiocbClearKicked(iocb) clear_bit(KIF_KICKED, &(iocb)->ki_flags)
  37. #define kiocbClearCancelled(iocb) clear_bit(KIF_CANCELLED, &(iocb)->ki_flags)
  38. #define kiocbIsLocked(iocb) test_bit(KIF_LOCKED, &(iocb)->ki_flags)
  39. #define kiocbIsKicked(iocb) test_bit(KIF_KICKED, &(iocb)->ki_flags)
  40. #define kiocbIsCancelled(iocb) test_bit(KIF_CANCELLED, &(iocb)->ki_flags)
  41. /* is there a better place to document function pointer methods? */
  42. /**
  43. * ki_retry - iocb forward progress callback
  44. * @kiocb: The kiocb struct to advance by performing an operation.
  45. *
  46. * This callback is called when the AIO core wants a given AIO operation
  47. * to make forward progress. The kiocb argument describes the operation
  48. * that is to be performed. As the operation proceeds, perhaps partially,
  49. * ki_retry is expected to update the kiocb with progress made. Typically
  50. * ki_retry is set in the AIO core and it itself calls file_operations
  51. * helpers.
  52. *
  53. * ki_retry's return value determines when the AIO operation is completed
  54. * and an event is generated in the AIO event ring. Except the special
  55. * return values described below, the value that is returned from ki_retry
  56. * is transferred directly into the completion ring as the operation's
  57. * resulting status. Once this has happened ki_retry *MUST NOT* reference
  58. * the kiocb pointer again.
  59. *
  60. * If ki_retry returns -EIOCBQUEUED it has made a promise that aio_complete()
  61. * will be called on the kiocb pointer in the future. The AIO core will
  62. * not ask the method again -- ki_retry must ensure forward progress.
  63. * aio_complete() must be called once and only once in the future, multiple
  64. * calls may result in undefined behaviour.
  65. *
  66. * If ki_retry returns -EIOCBRETRY it has made a promise that kick_iocb()
  67. * will be called on the kiocb pointer in the future. This may happen
  68. * through generic helpers that associate kiocb->ki_wait with a wait
  69. * queue head that ki_retry uses via current->io_wait. It can also happen
  70. * with custom tracking and manual calls to kick_iocb(), though that is
  71. * discouraged. In either case, kick_iocb() must be called once and only
  72. * once. ki_retry must ensure forward progress, the AIO core will wait
  73. * indefinitely for kick_iocb() to be called.
  74. */
  75. struct kiocb {
  76. struct list_head ki_run_list;
  77. long ki_flags;
  78. int ki_users;
  79. unsigned ki_key; /* id of this request */
  80. struct file *ki_filp;
  81. struct kioctx *ki_ctx; /* may be NULL for sync ops */
  82. int (*ki_cancel)(struct kiocb *, struct io_event *);
  83. ssize_t (*ki_retry)(struct kiocb *);
  84. void (*ki_dtor)(struct kiocb *);
  85. union {
  86. void __user *user;
  87. struct task_struct *tsk;
  88. } ki_obj;
  89. __u64 ki_user_data; /* user's data for completion */
  90. wait_queue_t ki_wait;
  91. loff_t ki_pos;
  92. atomic_t ki_bio_count; /* num bio used for this iocb */
  93. void *private;
  94. /* State that we remember to be able to restart/retry */
  95. unsigned short ki_opcode;
  96. size_t ki_nbytes; /* copy of iocb->aio_nbytes */
  97. char __user *ki_buf; /* remaining iocb->aio_buf */
  98. size_t ki_left; /* remaining bytes */
  99. struct iovec ki_inline_vec; /* inline vector */
  100. struct iovec *ki_iovec;
  101. unsigned long ki_nr_segs;
  102. unsigned long ki_cur_seg;
  103. struct list_head ki_list; /* the aio core uses this
  104. * for cancellation */
  105. };
  106. #define is_sync_kiocb(iocb) ((iocb)->ki_key == KIOCB_SYNC_KEY)
  107. #define init_sync_kiocb(x, filp) \
  108. do { \
  109. struct task_struct *tsk = current; \
  110. (x)->ki_flags = 0; \
  111. (x)->ki_users = 1; \
  112. (x)->ki_key = KIOCB_SYNC_KEY; \
  113. (x)->ki_filp = (filp); \
  114. (x)->ki_ctx = NULL; \
  115. (x)->ki_cancel = NULL; \
  116. (x)->ki_retry = NULL; \
  117. (x)->ki_dtor = NULL; \
  118. (x)->ki_obj.tsk = tsk; \
  119. (x)->ki_user_data = 0; \
  120. init_wait((&(x)->ki_wait)); \
  121. } while (0)
  122. #define AIO_RING_MAGIC 0xa10a10a1
  123. #define AIO_RING_COMPAT_FEATURES 1
  124. #define AIO_RING_INCOMPAT_FEATURES 0
  125. struct aio_ring {
  126. unsigned id; /* kernel internal index number */
  127. unsigned nr; /* number of io_events */
  128. unsigned head;
  129. unsigned tail;
  130. unsigned magic;
  131. unsigned compat_features;
  132. unsigned incompat_features;
  133. unsigned header_length; /* size of aio_ring */
  134. struct io_event io_events[0];
  135. }; /* 128 bytes + ring size */
  136. #define aio_ring_avail(info, ring) (((ring)->head + (info)->nr - 1 - (ring)->tail) % (info)->nr)
  137. #define AIO_RING_PAGES 8
  138. struct aio_ring_info {
  139. unsigned long mmap_base;
  140. unsigned long mmap_size;
  141. struct page **ring_pages;
  142. spinlock_t ring_lock;
  143. long nr_pages;
  144. unsigned nr, tail;
  145. struct page *internal_pages[AIO_RING_PAGES];
  146. };
  147. struct kioctx {
  148. atomic_t users;
  149. int dead;
  150. struct mm_struct *mm;
  151. /* This needs improving */
  152. unsigned long user_id;
  153. struct kioctx *next;
  154. wait_queue_head_t wait;
  155. spinlock_t ctx_lock;
  156. int reqs_active;
  157. struct list_head active_reqs; /* used for cancellation */
  158. struct list_head run_list; /* used for kicked reqs */
  159. /* sys_io_setup currently limits this to an unsigned int */
  160. unsigned max_reqs;
  161. struct aio_ring_info ring_info;
  162. struct delayed_work wq;
  163. };
  164. /* prototypes */
  165. extern unsigned aio_max_size;
  166. extern ssize_t FASTCALL(wait_on_sync_kiocb(struct kiocb *iocb));
  167. extern int FASTCALL(aio_put_req(struct kiocb *iocb));
  168. extern void FASTCALL(kick_iocb(struct kiocb *iocb));
  169. extern int FASTCALL(aio_complete(struct kiocb *iocb, long res, long res2));
  170. extern void FASTCALL(__put_ioctx(struct kioctx *ctx));
  171. struct mm_struct;
  172. extern void FASTCALL(exit_aio(struct mm_struct *mm));
  173. extern struct kioctx *lookup_ioctx(unsigned long ctx_id);
  174. extern int FASTCALL(io_submit_one(struct kioctx *ctx,
  175. struct iocb __user *user_iocb, struct iocb *iocb));
  176. /* semi private, but used by the 32bit emulations: */
  177. struct kioctx *lookup_ioctx(unsigned long ctx_id);
  178. int FASTCALL(io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
  179. struct iocb *iocb));
  180. #define get_ioctx(kioctx) do { \
  181. BUG_ON(atomic_read(&(kioctx)->users) <= 0); \
  182. atomic_inc(&(kioctx)->users); \
  183. } while (0)
  184. #define put_ioctx(kioctx) do { \
  185. BUG_ON(atomic_read(&(kioctx)->users) <= 0); \
  186. if (unlikely(atomic_dec_and_test(&(kioctx)->users))) \
  187. __put_ioctx(kioctx); \
  188. } while (0)
  189. #define in_aio() !is_sync_wait(current->io_wait)
  190. /* may be used for debugging */
  191. #define warn_if_async() \
  192. do { \
  193. if (in_aio()) { \
  194. printk(KERN_ERR "%s(%s:%d) called in async context!\n", \
  195. __FUNCTION__, __FILE__, __LINE__); \
  196. dump_stack(); \
  197. } \
  198. } while (0)
  199. #define io_wait_to_kiocb(wait) container_of(wait, struct kiocb, ki_wait)
  200. #include <linux/aio_abi.h>
  201. static inline struct kiocb *list_kiocb(struct list_head *h)
  202. {
  203. return list_entry(h, struct kiocb, ki_list);
  204. }
  205. /* for sysctl: */
  206. extern unsigned long aio_nr;
  207. extern unsigned long aio_max_nr;
  208. #endif /* __LINUX__AIO_H */