aio.c 45 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768
  1. /*
  2. * An async IO implementation for Linux
  3. * Written by Benjamin LaHaise <bcrl@kvack.org>
  4. *
  5. * Implements an efficient asynchronous io interface.
  6. *
  7. * Copyright 2000, 2001, 2002 Red Hat, Inc. All Rights Reserved.
  8. *
  9. * See ../COPYING for licensing terms.
  10. */
  11. #include <linux/kernel.h>
  12. #include <linux/init.h>
  13. #include <linux/errno.h>
  14. #include <linux/time.h>
  15. #include <linux/aio_abi.h>
  16. #include <linux/module.h>
  17. #include <linux/syscalls.h>
  18. #include <linux/uio.h>
  19. #define DEBUG 0
  20. #include <linux/sched.h>
  21. #include <linux/fs.h>
  22. #include <linux/file.h>
  23. #include <linux/mm.h>
  24. #include <linux/mman.h>
  25. #include <linux/slab.h>
  26. #include <linux/timer.h>
  27. #include <linux/aio.h>
  28. #include <linux/highmem.h>
  29. #include <linux/workqueue.h>
  30. #include <linux/security.h>
  31. #include <asm/kmap_types.h>
  32. #include <asm/uaccess.h>
  33. #include <asm/mmu_context.h>
  34. #if DEBUG > 1
  35. #define dprintk printk
  36. #else
  37. #define dprintk(x...) do { ; } while (0)
  38. #endif
  39. /*------ sysctl variables----*/
  40. static DEFINE_SPINLOCK(aio_nr_lock);
  41. unsigned long aio_nr; /* current system wide number of aio requests */
  42. unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */
  43. /*----end sysctl variables---*/
  44. static struct kmem_cache *kiocb_cachep;
  45. static struct kmem_cache *kioctx_cachep;
  46. static struct workqueue_struct *aio_wq;
  47. /* Used for rare fput completion. */
  48. static void aio_fput_routine(struct work_struct *);
  49. static DECLARE_WORK(fput_work, aio_fput_routine);
  50. static DEFINE_SPINLOCK(fput_lock);
  51. static LIST_HEAD(fput_head);
  52. static void aio_kick_handler(struct work_struct *);
  53. static void aio_queue_work(struct kioctx *);
  54. /* aio_setup
  55. * Creates the slab caches used by the aio routines, panic on
  56. * failure as this is done early during the boot sequence.
  57. */
  58. static int __init aio_setup(void)
  59. {
  60. kiocb_cachep = kmem_cache_create("kiocb", sizeof(struct kiocb),
  61. 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
  62. kioctx_cachep = kmem_cache_create("kioctx", sizeof(struct kioctx),
  63. 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
  64. aio_wq = create_workqueue("aio");
  65. pr_debug("aio_setup: sizeof(struct page) = %d\n", (int)sizeof(struct page));
  66. return 0;
  67. }
  68. static void aio_free_ring(struct kioctx *ctx)
  69. {
  70. struct aio_ring_info *info = &ctx->ring_info;
  71. long i;
  72. for (i=0; i<info->nr_pages; i++)
  73. put_page(info->ring_pages[i]);
  74. if (info->mmap_size) {
  75. down_write(&ctx->mm->mmap_sem);
  76. do_munmap(ctx->mm, info->mmap_base, info->mmap_size);
  77. up_write(&ctx->mm->mmap_sem);
  78. }
  79. if (info->ring_pages && info->ring_pages != info->internal_pages)
  80. kfree(info->ring_pages);
  81. info->ring_pages = NULL;
  82. info->nr = 0;
  83. }
  84. static int aio_setup_ring(struct kioctx *ctx)
  85. {
  86. struct aio_ring *ring;
  87. struct aio_ring_info *info = &ctx->ring_info;
  88. unsigned nr_events = ctx->max_reqs;
  89. unsigned long size;
  90. int nr_pages;
  91. /* Compensate for the ring buffer's head/tail overlap entry */
  92. nr_events += 2; /* 1 is required, 2 for good luck */
  93. size = sizeof(struct aio_ring);
  94. size += sizeof(struct io_event) * nr_events;
  95. nr_pages = (size + PAGE_SIZE-1) >> PAGE_SHIFT;
  96. if (nr_pages < 0)
  97. return -EINVAL;
  98. nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring)) / sizeof(struct io_event);
  99. info->nr = 0;
  100. info->ring_pages = info->internal_pages;
  101. if (nr_pages > AIO_RING_PAGES) {
  102. info->ring_pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
  103. if (!info->ring_pages)
  104. return -ENOMEM;
  105. }
  106. info->mmap_size = nr_pages * PAGE_SIZE;
  107. dprintk("attempting mmap of %lu bytes\n", info->mmap_size);
  108. down_write(&ctx->mm->mmap_sem);
  109. info->mmap_base = do_mmap(NULL, 0, info->mmap_size,
  110. PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE,
  111. 0);
  112. if (IS_ERR((void *)info->mmap_base)) {
  113. up_write(&ctx->mm->mmap_sem);
  114. info->mmap_size = 0;
  115. aio_free_ring(ctx);
  116. return -EAGAIN;
  117. }
  118. dprintk("mmap address: 0x%08lx\n", info->mmap_base);
  119. info->nr_pages = get_user_pages(current, ctx->mm,
  120. info->mmap_base, nr_pages,
  121. 1, 0, info->ring_pages, NULL);
  122. up_write(&ctx->mm->mmap_sem);
  123. if (unlikely(info->nr_pages != nr_pages)) {
  124. aio_free_ring(ctx);
  125. return -EAGAIN;
  126. }
  127. ctx->user_id = info->mmap_base;
  128. info->nr = nr_events; /* trusted copy */
  129. ring = kmap_atomic(info->ring_pages[0], KM_USER0);
  130. ring->nr = nr_events; /* user copy */
  131. ring->id = ctx->user_id;
  132. ring->head = ring->tail = 0;
  133. ring->magic = AIO_RING_MAGIC;
  134. ring->compat_features = AIO_RING_COMPAT_FEATURES;
  135. ring->incompat_features = AIO_RING_INCOMPAT_FEATURES;
  136. ring->header_length = sizeof(struct aio_ring);
  137. kunmap_atomic(ring, KM_USER0);
  138. return 0;
  139. }
  140. /* aio_ring_event: returns a pointer to the event at the given index from
  141. * kmap_atomic(, km). Release the pointer with put_aio_ring_event();
  142. */
  143. #define AIO_EVENTS_PER_PAGE (PAGE_SIZE / sizeof(struct io_event))
  144. #define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
  145. #define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)
  146. #define aio_ring_event(info, nr, km) ({ \
  147. unsigned pos = (nr) + AIO_EVENTS_OFFSET; \
  148. struct io_event *__event; \
  149. __event = kmap_atomic( \
  150. (info)->ring_pages[pos / AIO_EVENTS_PER_PAGE], km); \
  151. __event += pos % AIO_EVENTS_PER_PAGE; \
  152. __event; \
  153. })
  154. #define put_aio_ring_event(event, km) do { \
  155. struct io_event *__event = (event); \
  156. (void)__event; \
  157. kunmap_atomic((void *)((unsigned long)__event & PAGE_MASK), km); \
  158. } while(0)
  159. /* ioctx_alloc
  160. * Allocates and initializes an ioctx. Returns an ERR_PTR if it failed.
  161. */
  162. static struct kioctx *ioctx_alloc(unsigned nr_events)
  163. {
  164. struct mm_struct *mm;
  165. struct kioctx *ctx;
  166. /* Prevent overflows */
  167. if ((nr_events > (0x10000000U / sizeof(struct io_event))) ||
  168. (nr_events > (0x10000000U / sizeof(struct kiocb)))) {
  169. pr_debug("ENOMEM: nr_events too high\n");
  170. return ERR_PTR(-EINVAL);
  171. }
  172. if ((unsigned long)nr_events > aio_max_nr)
  173. return ERR_PTR(-EAGAIN);
  174. ctx = kmem_cache_zalloc(kioctx_cachep, GFP_KERNEL);
  175. if (!ctx)
  176. return ERR_PTR(-ENOMEM);
  177. ctx->max_reqs = nr_events;
  178. mm = ctx->mm = current->mm;
  179. atomic_inc(&mm->mm_count);
  180. atomic_set(&ctx->users, 1);
  181. spin_lock_init(&ctx->ctx_lock);
  182. spin_lock_init(&ctx->ring_info.ring_lock);
  183. init_waitqueue_head(&ctx->wait);
  184. INIT_LIST_HEAD(&ctx->active_reqs);
  185. INIT_LIST_HEAD(&ctx->run_list);
  186. INIT_DELAYED_WORK(&ctx->wq, aio_kick_handler);
  187. if (aio_setup_ring(ctx) < 0)
  188. goto out_freectx;
  189. /* limit the number of system wide aios */
  190. spin_lock(&aio_nr_lock);
  191. if (aio_nr + ctx->max_reqs > aio_max_nr ||
  192. aio_nr + ctx->max_reqs < aio_nr)
  193. ctx->max_reqs = 0;
  194. else
  195. aio_nr += ctx->max_reqs;
  196. spin_unlock(&aio_nr_lock);
  197. if (ctx->max_reqs == 0)
  198. goto out_cleanup;
  199. /* now link into global list. kludge. FIXME */
  200. write_lock(&mm->ioctx_list_lock);
  201. ctx->next = mm->ioctx_list;
  202. mm->ioctx_list = ctx;
  203. write_unlock(&mm->ioctx_list_lock);
  204. dprintk("aio: allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
  205. ctx, ctx->user_id, current->mm, ctx->ring_info.nr);
  206. return ctx;
  207. out_cleanup:
  208. __put_ioctx(ctx);
  209. return ERR_PTR(-EAGAIN);
  210. out_freectx:
  211. mmdrop(mm);
  212. kmem_cache_free(kioctx_cachep, ctx);
  213. ctx = ERR_PTR(-ENOMEM);
  214. dprintk("aio: error allocating ioctx %p\n", ctx);
  215. return ctx;
  216. }
  217. /* aio_cancel_all
  218. * Cancels all outstanding aio requests on an aio context. Used
  219. * when the processes owning a context have all exited to encourage
  220. * the rapid destruction of the kioctx.
  221. */
  222. static void aio_cancel_all(struct kioctx *ctx)
  223. {
  224. int (*cancel)(struct kiocb *, struct io_event *);
  225. struct io_event res;
  226. spin_lock_irq(&ctx->ctx_lock);
  227. ctx->dead = 1;
  228. while (!list_empty(&ctx->active_reqs)) {
  229. struct list_head *pos = ctx->active_reqs.next;
  230. struct kiocb *iocb = list_kiocb(pos);
  231. list_del_init(&iocb->ki_list);
  232. cancel = iocb->ki_cancel;
  233. kiocbSetCancelled(iocb);
  234. if (cancel) {
  235. iocb->ki_users++;
  236. spin_unlock_irq(&ctx->ctx_lock);
  237. cancel(iocb, &res);
  238. spin_lock_irq(&ctx->ctx_lock);
  239. }
  240. }
  241. spin_unlock_irq(&ctx->ctx_lock);
  242. }
  243. static void wait_for_all_aios(struct kioctx *ctx)
  244. {
  245. struct task_struct *tsk = current;
  246. DECLARE_WAITQUEUE(wait, tsk);
  247. spin_lock_irq(&ctx->ctx_lock);
  248. if (!ctx->reqs_active)
  249. goto out;
  250. add_wait_queue(&ctx->wait, &wait);
  251. set_task_state(tsk, TASK_UNINTERRUPTIBLE);
  252. while (ctx->reqs_active) {
  253. spin_unlock_irq(&ctx->ctx_lock);
  254. schedule();
  255. set_task_state(tsk, TASK_UNINTERRUPTIBLE);
  256. spin_lock_irq(&ctx->ctx_lock);
  257. }
  258. __set_task_state(tsk, TASK_RUNNING);
  259. remove_wait_queue(&ctx->wait, &wait);
  260. out:
  261. spin_unlock_irq(&ctx->ctx_lock);
  262. }
  263. /* wait_on_sync_kiocb:
  264. * Waits on the given sync kiocb to complete.
  265. */
  266. ssize_t fastcall wait_on_sync_kiocb(struct kiocb *iocb)
  267. {
  268. while (iocb->ki_users) {
  269. set_current_state(TASK_UNINTERRUPTIBLE);
  270. if (!iocb->ki_users)
  271. break;
  272. schedule();
  273. }
  274. __set_current_state(TASK_RUNNING);
  275. return iocb->ki_user_data;
  276. }
  277. /* exit_aio: called when the last user of mm goes away. At this point,
  278. * there is no way for any new requests to be submited or any of the
  279. * io_* syscalls to be called on the context. However, there may be
  280. * outstanding requests which hold references to the context; as they
  281. * go away, they will call put_ioctx and release any pinned memory
  282. * associated with the request (held via struct page * references).
  283. */
  284. void fastcall exit_aio(struct mm_struct *mm)
  285. {
  286. struct kioctx *ctx = mm->ioctx_list;
  287. mm->ioctx_list = NULL;
  288. while (ctx) {
  289. struct kioctx *next = ctx->next;
  290. ctx->next = NULL;
  291. aio_cancel_all(ctx);
  292. wait_for_all_aios(ctx);
  293. /*
  294. * this is an overkill, but ensures we don't leave
  295. * the ctx on the aio_wq
  296. */
  297. flush_workqueue(aio_wq);
  298. if (1 != atomic_read(&ctx->users))
  299. printk(KERN_DEBUG
  300. "exit_aio:ioctx still alive: %d %d %d\n",
  301. atomic_read(&ctx->users), ctx->dead,
  302. ctx->reqs_active);
  303. put_ioctx(ctx);
  304. ctx = next;
  305. }
  306. }
  307. /* __put_ioctx
  308. * Called when the last user of an aio context has gone away,
  309. * and the struct needs to be freed.
  310. */
  311. void fastcall __put_ioctx(struct kioctx *ctx)
  312. {
  313. unsigned nr_events = ctx->max_reqs;
  314. BUG_ON(ctx->reqs_active);
  315. cancel_delayed_work(&ctx->wq);
  316. flush_workqueue(aio_wq);
  317. aio_free_ring(ctx);
  318. mmdrop(ctx->mm);
  319. ctx->mm = NULL;
  320. pr_debug("__put_ioctx: freeing %p\n", ctx);
  321. kmem_cache_free(kioctx_cachep, ctx);
  322. if (nr_events) {
  323. spin_lock(&aio_nr_lock);
  324. BUG_ON(aio_nr - nr_events > aio_nr);
  325. aio_nr -= nr_events;
  326. spin_unlock(&aio_nr_lock);
  327. }
  328. }
  329. /* aio_get_req
  330. * Allocate a slot for an aio request. Increments the users count
  331. * of the kioctx so that the kioctx stays around until all requests are
  332. * complete. Returns NULL if no requests are free.
  333. *
  334. * Returns with kiocb->users set to 2. The io submit code path holds
  335. * an extra reference while submitting the i/o.
  336. * This prevents races between the aio code path referencing the
  337. * req (after submitting it) and aio_complete() freeing the req.
  338. */
  339. static struct kiocb *FASTCALL(__aio_get_req(struct kioctx *ctx));
  340. static struct kiocb fastcall *__aio_get_req(struct kioctx *ctx)
  341. {
  342. struct kiocb *req = NULL;
  343. struct aio_ring *ring;
  344. int okay = 0;
  345. req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL);
  346. if (unlikely(!req))
  347. return NULL;
  348. req->ki_flags = 0;
  349. req->ki_users = 2;
  350. req->ki_key = 0;
  351. req->ki_ctx = ctx;
  352. req->ki_cancel = NULL;
  353. req->ki_retry = NULL;
  354. req->ki_dtor = NULL;
  355. req->private = NULL;
  356. req->ki_iovec = NULL;
  357. INIT_LIST_HEAD(&req->ki_run_list);
  358. /* Check if the completion queue has enough free space to
  359. * accept an event from this io.
  360. */
  361. spin_lock_irq(&ctx->ctx_lock);
  362. ring = kmap_atomic(ctx->ring_info.ring_pages[0], KM_USER0);
  363. if (ctx->reqs_active < aio_ring_avail(&ctx->ring_info, ring)) {
  364. list_add(&req->ki_list, &ctx->active_reqs);
  365. ctx->reqs_active++;
  366. okay = 1;
  367. }
  368. kunmap_atomic(ring, KM_USER0);
  369. spin_unlock_irq(&ctx->ctx_lock);
  370. if (!okay) {
  371. kmem_cache_free(kiocb_cachep, req);
  372. req = NULL;
  373. }
  374. return req;
  375. }
  376. static inline struct kiocb *aio_get_req(struct kioctx *ctx)
  377. {
  378. struct kiocb *req;
  379. /* Handle a potential starvation case -- should be exceedingly rare as
  380. * requests will be stuck on fput_head only if the aio_fput_routine is
  381. * delayed and the requests were the last user of the struct file.
  382. */
  383. req = __aio_get_req(ctx);
  384. if (unlikely(NULL == req)) {
  385. aio_fput_routine(NULL);
  386. req = __aio_get_req(ctx);
  387. }
  388. return req;
  389. }
  390. static inline void really_put_req(struct kioctx *ctx, struct kiocb *req)
  391. {
  392. assert_spin_locked(&ctx->ctx_lock);
  393. if (req->ki_dtor)
  394. req->ki_dtor(req);
  395. if (req->ki_iovec != &req->ki_inline_vec)
  396. kfree(req->ki_iovec);
  397. kmem_cache_free(kiocb_cachep, req);
  398. ctx->reqs_active--;
  399. if (unlikely(!ctx->reqs_active && ctx->dead))
  400. wake_up(&ctx->wait);
  401. }
  402. static void aio_fput_routine(struct work_struct *data)
  403. {
  404. spin_lock_irq(&fput_lock);
  405. while (likely(!list_empty(&fput_head))) {
  406. struct kiocb *req = list_kiocb(fput_head.next);
  407. struct kioctx *ctx = req->ki_ctx;
  408. list_del(&req->ki_list);
  409. spin_unlock_irq(&fput_lock);
  410. /* Complete the fput */
  411. __fput(req->ki_filp);
  412. /* Link the iocb into the context's free list */
  413. spin_lock_irq(&ctx->ctx_lock);
  414. really_put_req(ctx, req);
  415. spin_unlock_irq(&ctx->ctx_lock);
  416. put_ioctx(ctx);
  417. spin_lock_irq(&fput_lock);
  418. }
  419. spin_unlock_irq(&fput_lock);
  420. }
  421. /* __aio_put_req
  422. * Returns true if this put was the last user of the request.
  423. */
  424. static int __aio_put_req(struct kioctx *ctx, struct kiocb *req)
  425. {
  426. dprintk(KERN_DEBUG "aio_put(%p): f_count=%d\n",
  427. req, atomic_read(&req->ki_filp->f_count));
  428. assert_spin_locked(&ctx->ctx_lock);
  429. req->ki_users --;
  430. BUG_ON(req->ki_users < 0);
  431. if (likely(req->ki_users))
  432. return 0;
  433. list_del(&req->ki_list); /* remove from active_reqs */
  434. req->ki_cancel = NULL;
  435. req->ki_retry = NULL;
  436. /* Must be done under the lock to serialise against cancellation.
  437. * Call this aio_fput as it duplicates fput via the fput_work.
  438. */
  439. if (unlikely(atomic_dec_and_test(&req->ki_filp->f_count))) {
  440. get_ioctx(ctx);
  441. spin_lock(&fput_lock);
  442. list_add(&req->ki_list, &fput_head);
  443. spin_unlock(&fput_lock);
  444. queue_work(aio_wq, &fput_work);
  445. } else
  446. really_put_req(ctx, req);
  447. return 1;
  448. }
  449. /* aio_put_req
  450. * Returns true if this put was the last user of the kiocb,
  451. * false if the request is still in use.
  452. */
  453. int fastcall aio_put_req(struct kiocb *req)
  454. {
  455. struct kioctx *ctx = req->ki_ctx;
  456. int ret;
  457. spin_lock_irq(&ctx->ctx_lock);
  458. ret = __aio_put_req(ctx, req);
  459. spin_unlock_irq(&ctx->ctx_lock);
  460. return ret;
  461. }
  462. /* Lookup an ioctx id. ioctx_list is lockless for reads.
  463. * FIXME: this is O(n) and is only suitable for development.
  464. */
  465. struct kioctx *lookup_ioctx(unsigned long ctx_id)
  466. {
  467. struct kioctx *ioctx;
  468. struct mm_struct *mm;
  469. mm = current->mm;
  470. read_lock(&mm->ioctx_list_lock);
  471. for (ioctx = mm->ioctx_list; ioctx; ioctx = ioctx->next)
  472. if (likely(ioctx->user_id == ctx_id && !ioctx->dead)) {
  473. get_ioctx(ioctx);
  474. break;
  475. }
  476. read_unlock(&mm->ioctx_list_lock);
  477. return ioctx;
  478. }
  479. /*
  480. * use_mm
  481. * Makes the calling kernel thread take on the specified
  482. * mm context.
  483. * Called by the retry thread execute retries within the
  484. * iocb issuer's mm context, so that copy_from/to_user
  485. * operations work seamlessly for aio.
  486. * (Note: this routine is intended to be called only
  487. * from a kernel thread context)
  488. */
  489. static void use_mm(struct mm_struct *mm)
  490. {
  491. struct mm_struct *active_mm;
  492. struct task_struct *tsk = current;
  493. task_lock(tsk);
  494. tsk->flags |= PF_BORROWED_MM;
  495. active_mm = tsk->active_mm;
  496. atomic_inc(&mm->mm_count);
  497. tsk->mm = mm;
  498. tsk->active_mm = mm;
  499. /*
  500. * Note that on UML this *requires* PF_BORROWED_MM to be set, otherwise
  501. * it won't work. Update it accordingly if you change it here
  502. */
  503. switch_mm(active_mm, mm, tsk);
  504. task_unlock(tsk);
  505. mmdrop(active_mm);
  506. }
  507. /*
  508. * unuse_mm
  509. * Reverses the effect of use_mm, i.e. releases the
  510. * specified mm context which was earlier taken on
  511. * by the calling kernel thread
  512. * (Note: this routine is intended to be called only
  513. * from a kernel thread context)
  514. */
  515. static void unuse_mm(struct mm_struct *mm)
  516. {
  517. struct task_struct *tsk = current;
  518. task_lock(tsk);
  519. tsk->flags &= ~PF_BORROWED_MM;
  520. tsk->mm = NULL;
  521. /* active_mm is still 'mm' */
  522. enter_lazy_tlb(mm, tsk);
  523. task_unlock(tsk);
  524. }
  525. /*
  526. * Queue up a kiocb to be retried. Assumes that the kiocb
  527. * has already been marked as kicked, and places it on
  528. * the retry run list for the corresponding ioctx, if it
  529. * isn't already queued. Returns 1 if it actually queued
  530. * the kiocb (to tell the caller to activate the work
  531. * queue to process it), or 0, if it found that it was
  532. * already queued.
  533. */
  534. static inline int __queue_kicked_iocb(struct kiocb *iocb)
  535. {
  536. struct kioctx *ctx = iocb->ki_ctx;
  537. assert_spin_locked(&ctx->ctx_lock);
  538. if (list_empty(&iocb->ki_run_list)) {
  539. list_add_tail(&iocb->ki_run_list,
  540. &ctx->run_list);
  541. return 1;
  542. }
  543. return 0;
  544. }
  545. /* aio_run_iocb
  546. * This is the core aio execution routine. It is
  547. * invoked both for initial i/o submission and
  548. * subsequent retries via the aio_kick_handler.
  549. * Expects to be invoked with iocb->ki_ctx->lock
  550. * already held. The lock is released and reacquired
  551. * as needed during processing.
  552. *
  553. * Calls the iocb retry method (already setup for the
  554. * iocb on initial submission) for operation specific
  555. * handling, but takes care of most of common retry
  556. * execution details for a given iocb. The retry method
  557. * needs to be non-blocking as far as possible, to avoid
  558. * holding up other iocbs waiting to be serviced by the
  559. * retry kernel thread.
  560. *
  561. * The trickier parts in this code have to do with
  562. * ensuring that only one retry instance is in progress
  563. * for a given iocb at any time. Providing that guarantee
  564. * simplifies the coding of individual aio operations as
  565. * it avoids various potential races.
  566. */
  567. static ssize_t aio_run_iocb(struct kiocb *iocb)
  568. {
  569. struct kioctx *ctx = iocb->ki_ctx;
  570. ssize_t (*retry)(struct kiocb *);
  571. ssize_t ret;
  572. if (!(retry = iocb->ki_retry)) {
  573. printk("aio_run_iocb: iocb->ki_retry = NULL\n");
  574. return 0;
  575. }
  576. /*
  577. * We don't want the next retry iteration for this
  578. * operation to start until this one has returned and
  579. * updated the iocb state. However, wait_queue functions
  580. * can trigger a kick_iocb from interrupt context in the
  581. * meantime, indicating that data is available for the next
  582. * iteration. We want to remember that and enable the
  583. * next retry iteration _after_ we are through with
  584. * this one.
  585. *
  586. * So, in order to be able to register a "kick", but
  587. * prevent it from being queued now, we clear the kick
  588. * flag, but make the kick code *think* that the iocb is
  589. * still on the run list until we are actually done.
  590. * When we are done with this iteration, we check if
  591. * the iocb was kicked in the meantime and if so, queue
  592. * it up afresh.
  593. */
  594. kiocbClearKicked(iocb);
  595. /*
  596. * This is so that aio_complete knows it doesn't need to
  597. * pull the iocb off the run list (We can't just call
  598. * INIT_LIST_HEAD because we don't want a kick_iocb to
  599. * queue this on the run list yet)
  600. */
  601. iocb->ki_run_list.next = iocb->ki_run_list.prev = NULL;
  602. spin_unlock_irq(&ctx->ctx_lock);
  603. /* Quit retrying if the i/o has been cancelled */
  604. if (kiocbIsCancelled(iocb)) {
  605. ret = -EINTR;
  606. aio_complete(iocb, ret, 0);
  607. /* must not access the iocb after this */
  608. goto out;
  609. }
  610. /*
  611. * Now we are all set to call the retry method in async
  612. * context. By setting this thread's io_wait context
  613. * to point to the wait queue entry inside the currently
  614. * running iocb for the duration of the retry, we ensure
  615. * that async notification wakeups are queued by the
  616. * operation instead of blocking waits, and when notified,
  617. * cause the iocb to be kicked for continuation (through
  618. * the aio_wake_function callback).
  619. */
  620. BUG_ON(current->io_wait != NULL);
  621. current->io_wait = &iocb->ki_wait;
  622. ret = retry(iocb);
  623. current->io_wait = NULL;
  624. if (ret != -EIOCBRETRY && ret != -EIOCBQUEUED) {
  625. BUG_ON(!list_empty(&iocb->ki_wait.task_list));
  626. aio_complete(iocb, ret, 0);
  627. }
  628. out:
  629. spin_lock_irq(&ctx->ctx_lock);
  630. if (-EIOCBRETRY == ret) {
  631. /*
  632. * OK, now that we are done with this iteration
  633. * and know that there is more left to go,
  634. * this is where we let go so that a subsequent
  635. * "kick" can start the next iteration
  636. */
  637. /* will make __queue_kicked_iocb succeed from here on */
  638. INIT_LIST_HEAD(&iocb->ki_run_list);
  639. /* we must queue the next iteration ourselves, if it
  640. * has already been kicked */
  641. if (kiocbIsKicked(iocb)) {
  642. __queue_kicked_iocb(iocb);
  643. /*
  644. * __queue_kicked_iocb will always return 1 here, because
  645. * iocb->ki_run_list is empty at this point so it should
  646. * be safe to unconditionally queue the context into the
  647. * work queue.
  648. */
  649. aio_queue_work(ctx);
  650. }
  651. }
  652. return ret;
  653. }
  654. /*
  655. * __aio_run_iocbs:
  656. * Process all pending retries queued on the ioctx
  657. * run list.
  658. * Assumes it is operating within the aio issuer's mm
  659. * context.
  660. */
  661. static int __aio_run_iocbs(struct kioctx *ctx)
  662. {
  663. struct kiocb *iocb;
  664. struct list_head run_list;
  665. assert_spin_locked(&ctx->ctx_lock);
  666. list_replace_init(&ctx->run_list, &run_list);
  667. while (!list_empty(&run_list)) {
  668. iocb = list_entry(run_list.next, struct kiocb,
  669. ki_run_list);
  670. list_del(&iocb->ki_run_list);
  671. /*
  672. * Hold an extra reference while retrying i/o.
  673. */
  674. iocb->ki_users++; /* grab extra reference */
  675. aio_run_iocb(iocb);
  676. __aio_put_req(ctx, iocb);
  677. }
  678. if (!list_empty(&ctx->run_list))
  679. return 1;
  680. return 0;
  681. }
  682. static void aio_queue_work(struct kioctx * ctx)
  683. {
  684. unsigned long timeout;
  685. /*
  686. * if someone is waiting, get the work started right
  687. * away, otherwise, use a longer delay
  688. */
  689. smp_mb();
  690. if (waitqueue_active(&ctx->wait))
  691. timeout = 1;
  692. else
  693. timeout = HZ/10;
  694. queue_delayed_work(aio_wq, &ctx->wq, timeout);
  695. }
  696. /*
  697. * aio_run_iocbs:
  698. * Process all pending retries queued on the ioctx
  699. * run list.
  700. * Assumes it is operating within the aio issuer's mm
  701. * context.
  702. */
  703. static inline void aio_run_iocbs(struct kioctx *ctx)
  704. {
  705. int requeue;
  706. spin_lock_irq(&ctx->ctx_lock);
  707. requeue = __aio_run_iocbs(ctx);
  708. spin_unlock_irq(&ctx->ctx_lock);
  709. if (requeue)
  710. aio_queue_work(ctx);
  711. }
  712. /*
  713. * just like aio_run_iocbs, but keeps running them until
  714. * the list stays empty
  715. */
  716. static inline void aio_run_all_iocbs(struct kioctx *ctx)
  717. {
  718. spin_lock_irq(&ctx->ctx_lock);
  719. while (__aio_run_iocbs(ctx))
  720. ;
  721. spin_unlock_irq(&ctx->ctx_lock);
  722. }
  723. /*
  724. * aio_kick_handler:
  725. * Work queue handler triggered to process pending
  726. * retries on an ioctx. Takes on the aio issuer's
  727. * mm context before running the iocbs, so that
  728. * copy_xxx_user operates on the issuer's address
  729. * space.
  730. * Run on aiod's context.
  731. */
  732. static void aio_kick_handler(struct work_struct *work)
  733. {
  734. struct kioctx *ctx = container_of(work, struct kioctx, wq.work);
  735. mm_segment_t oldfs = get_fs();
  736. struct mm_struct *mm;
  737. int requeue;
  738. set_fs(USER_DS);
  739. use_mm(ctx->mm);
  740. spin_lock_irq(&ctx->ctx_lock);
  741. requeue =__aio_run_iocbs(ctx);
  742. mm = ctx->mm;
  743. spin_unlock_irq(&ctx->ctx_lock);
  744. unuse_mm(mm);
  745. set_fs(oldfs);
  746. /*
  747. * we're in a worker thread already, don't use queue_delayed_work,
  748. */
  749. if (requeue)
  750. queue_delayed_work(aio_wq, &ctx->wq, 0);
  751. }
  752. /*
  753. * Called by kick_iocb to queue the kiocb for retry
  754. * and if required activate the aio work queue to process
  755. * it
  756. */
  757. static void try_queue_kicked_iocb(struct kiocb *iocb)
  758. {
  759. struct kioctx *ctx = iocb->ki_ctx;
  760. unsigned long flags;
  761. int run = 0;
  762. /* We're supposed to be the only path putting the iocb back on the run
  763. * list. If we find that the iocb is *back* on a wait queue already
  764. * than retry has happened before we could queue the iocb. This also
  765. * means that the retry could have completed and freed our iocb, no
  766. * good. */
  767. BUG_ON((!list_empty(&iocb->ki_wait.task_list)));
  768. spin_lock_irqsave(&ctx->ctx_lock, flags);
  769. /* set this inside the lock so that we can't race with aio_run_iocb()
  770. * testing it and putting the iocb on the run list under the lock */
  771. if (!kiocbTryKick(iocb))
  772. run = __queue_kicked_iocb(iocb);
  773. spin_unlock_irqrestore(&ctx->ctx_lock, flags);
  774. if (run)
  775. aio_queue_work(ctx);
  776. }
  777. /*
  778. * kick_iocb:
  779. * Called typically from a wait queue callback context
  780. * (aio_wake_function) to trigger a retry of the iocb.
  781. * The retry is usually executed by aio workqueue
  782. * threads (See aio_kick_handler).
  783. */
  784. void fastcall kick_iocb(struct kiocb *iocb)
  785. {
  786. /* sync iocbs are easy: they can only ever be executing from a
  787. * single context. */
  788. if (is_sync_kiocb(iocb)) {
  789. kiocbSetKicked(iocb);
  790. wake_up_process(iocb->ki_obj.tsk);
  791. return;
  792. }
  793. try_queue_kicked_iocb(iocb);
  794. }
  795. EXPORT_SYMBOL(kick_iocb);
  796. /* aio_complete
  797. * Called when the io request on the given iocb is complete.
  798. * Returns true if this is the last user of the request. The
  799. * only other user of the request can be the cancellation code.
  800. */
  801. int fastcall aio_complete(struct kiocb *iocb, long res, long res2)
  802. {
  803. struct kioctx *ctx = iocb->ki_ctx;
  804. struct aio_ring_info *info;
  805. struct aio_ring *ring;
  806. struct io_event *event;
  807. unsigned long flags;
  808. unsigned long tail;
  809. int ret;
  810. /*
  811. * Special case handling for sync iocbs:
  812. * - events go directly into the iocb for fast handling
  813. * - the sync task with the iocb in its stack holds the single iocb
  814. * ref, no other paths have a way to get another ref
  815. * - the sync task helpfully left a reference to itself in the iocb
  816. */
  817. if (is_sync_kiocb(iocb)) {
  818. BUG_ON(iocb->ki_users != 1);
  819. iocb->ki_user_data = res;
  820. iocb->ki_users = 0;
  821. wake_up_process(iocb->ki_obj.tsk);
  822. return 1;
  823. }
  824. info = &ctx->ring_info;
  825. /* add a completion event to the ring buffer.
  826. * must be done holding ctx->ctx_lock to prevent
  827. * other code from messing with the tail
  828. * pointer since we might be called from irq
  829. * context.
  830. */
  831. spin_lock_irqsave(&ctx->ctx_lock, flags);
  832. if (iocb->ki_run_list.prev && !list_empty(&iocb->ki_run_list))
  833. list_del_init(&iocb->ki_run_list);
  834. /*
  835. * cancelled requests don't get events, userland was given one
  836. * when the event got cancelled.
  837. */
  838. if (kiocbIsCancelled(iocb))
  839. goto put_rq;
  840. ring = kmap_atomic(info->ring_pages[0], KM_IRQ1);
  841. tail = info->tail;
  842. event = aio_ring_event(info, tail, KM_IRQ0);
  843. if (++tail >= info->nr)
  844. tail = 0;
  845. event->obj = (u64)(unsigned long)iocb->ki_obj.user;
  846. event->data = iocb->ki_user_data;
  847. event->res = res;
  848. event->res2 = res2;
  849. dprintk("aio_complete: %p[%lu]: %p: %p %Lx %lx %lx\n",
  850. ctx, tail, iocb, iocb->ki_obj.user, iocb->ki_user_data,
  851. res, res2);
  852. /* after flagging the request as done, we
  853. * must never even look at it again
  854. */
  855. smp_wmb(); /* make event visible before updating tail */
  856. info->tail = tail;
  857. ring->tail = tail;
  858. put_aio_ring_event(event, KM_IRQ0);
  859. kunmap_atomic(ring, KM_IRQ1);
  860. pr_debug("added to ring %p at [%lu]\n", iocb, tail);
  861. put_rq:
  862. /* everything turned out well, dispose of the aiocb. */
  863. ret = __aio_put_req(ctx, iocb);
  864. if (waitqueue_active(&ctx->wait))
  865. wake_up(&ctx->wait);
  866. spin_unlock_irqrestore(&ctx->ctx_lock, flags);
  867. return ret;
  868. }
  869. /* aio_read_evt
  870. * Pull an event off of the ioctx's event ring. Returns the number of
  871. * events fetched (0 or 1 ;-)
  872. * FIXME: make this use cmpxchg.
  873. * TODO: make the ringbuffer user mmap()able (requires FIXME).
  874. */
  875. static int aio_read_evt(struct kioctx *ioctx, struct io_event *ent)
  876. {
  877. struct aio_ring_info *info = &ioctx->ring_info;
  878. struct aio_ring *ring;
  879. unsigned long head;
  880. int ret = 0;
  881. ring = kmap_atomic(info->ring_pages[0], KM_USER0);
  882. dprintk("in aio_read_evt h%lu t%lu m%lu\n",
  883. (unsigned long)ring->head, (unsigned long)ring->tail,
  884. (unsigned long)ring->nr);
  885. if (ring->head == ring->tail)
  886. goto out;
  887. spin_lock(&info->ring_lock);
  888. head = ring->head % info->nr;
  889. if (head != ring->tail) {
  890. struct io_event *evp = aio_ring_event(info, head, KM_USER1);
  891. *ent = *evp;
  892. head = (head + 1) % info->nr;
  893. smp_mb(); /* finish reading the event before updatng the head */
  894. ring->head = head;
  895. ret = 1;
  896. put_aio_ring_event(evp, KM_USER1);
  897. }
  898. spin_unlock(&info->ring_lock);
  899. out:
  900. kunmap_atomic(ring, KM_USER0);
  901. dprintk("leaving aio_read_evt: %d h%lu t%lu\n", ret,
  902. (unsigned long)ring->head, (unsigned long)ring->tail);
  903. return ret;
  904. }
  905. struct aio_timeout {
  906. struct timer_list timer;
  907. int timed_out;
  908. struct task_struct *p;
  909. };
  910. static void timeout_func(unsigned long data)
  911. {
  912. struct aio_timeout *to = (struct aio_timeout *)data;
  913. to->timed_out = 1;
  914. wake_up_process(to->p);
  915. }
  916. static inline void init_timeout(struct aio_timeout *to)
  917. {
  918. init_timer(&to->timer);
  919. to->timer.data = (unsigned long)to;
  920. to->timer.function = timeout_func;
  921. to->timed_out = 0;
  922. to->p = current;
  923. }
  924. static inline void set_timeout(long start_jiffies, struct aio_timeout *to,
  925. const struct timespec *ts)
  926. {
  927. to->timer.expires = start_jiffies + timespec_to_jiffies(ts);
  928. if (time_after(to->timer.expires, jiffies))
  929. add_timer(&to->timer);
  930. else
  931. to->timed_out = 1;
  932. }
  933. static inline void clear_timeout(struct aio_timeout *to)
  934. {
  935. del_singleshot_timer_sync(&to->timer);
  936. }
  937. static int read_events(struct kioctx *ctx,
  938. long min_nr, long nr,
  939. struct io_event __user *event,
  940. struct timespec __user *timeout)
  941. {
  942. long start_jiffies = jiffies;
  943. struct task_struct *tsk = current;
  944. DECLARE_WAITQUEUE(wait, tsk);
  945. int ret;
  946. int i = 0;
  947. struct io_event ent;
  948. struct aio_timeout to;
  949. int retry = 0;
  950. /* needed to zero any padding within an entry (there shouldn't be
  951. * any, but C is fun!
  952. */
  953. memset(&ent, 0, sizeof(ent));
  954. retry:
  955. ret = 0;
  956. while (likely(i < nr)) {
  957. ret = aio_read_evt(ctx, &ent);
  958. if (unlikely(ret <= 0))
  959. break;
  960. dprintk("read event: %Lx %Lx %Lx %Lx\n",
  961. ent.data, ent.obj, ent.res, ent.res2);
  962. /* Could we split the check in two? */
  963. ret = -EFAULT;
  964. if (unlikely(copy_to_user(event, &ent, sizeof(ent)))) {
  965. dprintk("aio: lost an event due to EFAULT.\n");
  966. break;
  967. }
  968. ret = 0;
  969. /* Good, event copied to userland, update counts. */
  970. event ++;
  971. i ++;
  972. }
  973. if (min_nr <= i)
  974. return i;
  975. if (ret)
  976. return ret;
  977. /* End fast path */
  978. /* racey check, but it gets redone */
  979. if (!retry && unlikely(!list_empty(&ctx->run_list))) {
  980. retry = 1;
  981. aio_run_all_iocbs(ctx);
  982. goto retry;
  983. }
  984. init_timeout(&to);
  985. if (timeout) {
  986. struct timespec ts;
  987. ret = -EFAULT;
  988. if (unlikely(copy_from_user(&ts, timeout, sizeof(ts))))
  989. goto out;
  990. set_timeout(start_jiffies, &to, &ts);
  991. }
  992. while (likely(i < nr)) {
  993. add_wait_queue_exclusive(&ctx->wait, &wait);
  994. do {
  995. set_task_state(tsk, TASK_INTERRUPTIBLE);
  996. ret = aio_read_evt(ctx, &ent);
  997. if (ret)
  998. break;
  999. if (min_nr <= i)
  1000. break;
  1001. ret = 0;
  1002. if (to.timed_out) /* Only check after read evt */
  1003. break;
  1004. schedule();
  1005. if (signal_pending(tsk)) {
  1006. ret = -EINTR;
  1007. break;
  1008. }
  1009. /*ret = aio_read_evt(ctx, &ent);*/
  1010. } while (1) ;
  1011. set_task_state(tsk, TASK_RUNNING);
  1012. remove_wait_queue(&ctx->wait, &wait);
  1013. if (unlikely(ret <= 0))
  1014. break;
  1015. ret = -EFAULT;
  1016. if (unlikely(copy_to_user(event, &ent, sizeof(ent)))) {
  1017. dprintk("aio: lost an event due to EFAULT.\n");
  1018. break;
  1019. }
  1020. /* Good, event copied to userland, update counts. */
  1021. event ++;
  1022. i ++;
  1023. }
  1024. if (timeout)
  1025. clear_timeout(&to);
  1026. out:
  1027. return i ? i : ret;
  1028. }
  1029. /* Take an ioctx and remove it from the list of ioctx's. Protects
  1030. * against races with itself via ->dead.
  1031. */
  1032. static void io_destroy(struct kioctx *ioctx)
  1033. {
  1034. struct mm_struct *mm = current->mm;
  1035. struct kioctx **tmp;
  1036. int was_dead;
  1037. /* delete the entry from the list is someone else hasn't already */
  1038. write_lock(&mm->ioctx_list_lock);
  1039. was_dead = ioctx->dead;
  1040. ioctx->dead = 1;
  1041. for (tmp = &mm->ioctx_list; *tmp && *tmp != ioctx;
  1042. tmp = &(*tmp)->next)
  1043. ;
  1044. if (*tmp)
  1045. *tmp = ioctx->next;
  1046. write_unlock(&mm->ioctx_list_lock);
  1047. dprintk("aio_release(%p)\n", ioctx);
  1048. if (likely(!was_dead))
  1049. put_ioctx(ioctx); /* twice for the list */
  1050. aio_cancel_all(ioctx);
  1051. wait_for_all_aios(ioctx);
  1052. put_ioctx(ioctx); /* once for the lookup */
  1053. }
  1054. /* sys_io_setup:
  1055. * Create an aio_context capable of receiving at least nr_events.
  1056. * ctxp must not point to an aio_context that already exists, and
  1057. * must be initialized to 0 prior to the call. On successful
  1058. * creation of the aio_context, *ctxp is filled in with the resulting
  1059. * handle. May fail with -EINVAL if *ctxp is not initialized,
  1060. * if the specified nr_events exceeds internal limits. May fail
  1061. * with -EAGAIN if the specified nr_events exceeds the user's limit
  1062. * of available events. May fail with -ENOMEM if insufficient kernel
  1063. * resources are available. May fail with -EFAULT if an invalid
  1064. * pointer is passed for ctxp. Will fail with -ENOSYS if not
  1065. * implemented.
  1066. */
  1067. asmlinkage long sys_io_setup(unsigned nr_events, aio_context_t __user *ctxp)
  1068. {
  1069. struct kioctx *ioctx = NULL;
  1070. unsigned long ctx;
  1071. long ret;
  1072. ret = get_user(ctx, ctxp);
  1073. if (unlikely(ret))
  1074. goto out;
  1075. ret = -EINVAL;
  1076. if (unlikely(ctx || nr_events == 0)) {
  1077. pr_debug("EINVAL: io_setup: ctx %lu nr_events %u\n",
  1078. ctx, nr_events);
  1079. goto out;
  1080. }
  1081. ioctx = ioctx_alloc(nr_events);
  1082. ret = PTR_ERR(ioctx);
  1083. if (!IS_ERR(ioctx)) {
  1084. ret = put_user(ioctx->user_id, ctxp);
  1085. if (!ret)
  1086. return 0;
  1087. get_ioctx(ioctx); /* io_destroy() expects us to hold a ref */
  1088. io_destroy(ioctx);
  1089. }
  1090. out:
  1091. return ret;
  1092. }
  1093. /* sys_io_destroy:
  1094. * Destroy the aio_context specified. May cancel any outstanding
  1095. * AIOs and block on completion. Will fail with -ENOSYS if not
  1096. * implemented. May fail with -EFAULT if the context pointed to
  1097. * is invalid.
  1098. */
  1099. asmlinkage long sys_io_destroy(aio_context_t ctx)
  1100. {
  1101. struct kioctx *ioctx = lookup_ioctx(ctx);
  1102. if (likely(NULL != ioctx)) {
  1103. io_destroy(ioctx);
  1104. return 0;
  1105. }
  1106. pr_debug("EINVAL: io_destroy: invalid context id\n");
  1107. return -EINVAL;
  1108. }
  1109. static void aio_advance_iovec(struct kiocb *iocb, ssize_t ret)
  1110. {
  1111. struct iovec *iov = &iocb->ki_iovec[iocb->ki_cur_seg];
  1112. BUG_ON(ret <= 0);
  1113. while (iocb->ki_cur_seg < iocb->ki_nr_segs && ret > 0) {
  1114. ssize_t this = min((ssize_t)iov->iov_len, ret);
  1115. iov->iov_base += this;
  1116. iov->iov_len -= this;
  1117. iocb->ki_left -= this;
  1118. ret -= this;
  1119. if (iov->iov_len == 0) {
  1120. iocb->ki_cur_seg++;
  1121. iov++;
  1122. }
  1123. }
  1124. /* the caller should not have done more io than what fit in
  1125. * the remaining iovecs */
  1126. BUG_ON(ret > 0 && iocb->ki_left == 0);
  1127. }
  1128. static ssize_t aio_rw_vect_retry(struct kiocb *iocb)
  1129. {
  1130. struct file *file = iocb->ki_filp;
  1131. struct address_space *mapping = file->f_mapping;
  1132. struct inode *inode = mapping->host;
  1133. ssize_t (*rw_op)(struct kiocb *, const struct iovec *,
  1134. unsigned long, loff_t);
  1135. ssize_t ret = 0;
  1136. unsigned short opcode;
  1137. if ((iocb->ki_opcode == IOCB_CMD_PREADV) ||
  1138. (iocb->ki_opcode == IOCB_CMD_PREAD)) {
  1139. rw_op = file->f_op->aio_read;
  1140. opcode = IOCB_CMD_PREADV;
  1141. } else {
  1142. rw_op = file->f_op->aio_write;
  1143. opcode = IOCB_CMD_PWRITEV;
  1144. }
  1145. do {
  1146. ret = rw_op(iocb, &iocb->ki_iovec[iocb->ki_cur_seg],
  1147. iocb->ki_nr_segs - iocb->ki_cur_seg,
  1148. iocb->ki_pos);
  1149. if (ret > 0)
  1150. aio_advance_iovec(iocb, ret);
  1151. /* retry all partial writes. retry partial reads as long as its a
  1152. * regular file. */
  1153. } while (ret > 0 && iocb->ki_left > 0 &&
  1154. (opcode == IOCB_CMD_PWRITEV ||
  1155. (!S_ISFIFO(inode->i_mode) && !S_ISSOCK(inode->i_mode))));
  1156. /* This means we must have transferred all that we could */
  1157. /* No need to retry anymore */
  1158. if ((ret == 0) || (iocb->ki_left == 0))
  1159. ret = iocb->ki_nbytes - iocb->ki_left;
  1160. return ret;
  1161. }
  1162. static ssize_t aio_fdsync(struct kiocb *iocb)
  1163. {
  1164. struct file *file = iocb->ki_filp;
  1165. ssize_t ret = -EINVAL;
  1166. if (file->f_op->aio_fsync)
  1167. ret = file->f_op->aio_fsync(iocb, 1);
  1168. return ret;
  1169. }
  1170. static ssize_t aio_fsync(struct kiocb *iocb)
  1171. {
  1172. struct file *file = iocb->ki_filp;
  1173. ssize_t ret = -EINVAL;
  1174. if (file->f_op->aio_fsync)
  1175. ret = file->f_op->aio_fsync(iocb, 0);
  1176. return ret;
  1177. }
  1178. static ssize_t aio_setup_vectored_rw(int type, struct kiocb *kiocb)
  1179. {
  1180. ssize_t ret;
  1181. ret = rw_copy_check_uvector(type, (struct iovec __user *)kiocb->ki_buf,
  1182. kiocb->ki_nbytes, 1,
  1183. &kiocb->ki_inline_vec, &kiocb->ki_iovec);
  1184. if (ret < 0)
  1185. goto out;
  1186. kiocb->ki_nr_segs = kiocb->ki_nbytes;
  1187. kiocb->ki_cur_seg = 0;
  1188. /* ki_nbytes/left now reflect bytes instead of segs */
  1189. kiocb->ki_nbytes = ret;
  1190. kiocb->ki_left = ret;
  1191. ret = 0;
  1192. out:
  1193. return ret;
  1194. }
  1195. static ssize_t aio_setup_single_vector(struct kiocb *kiocb)
  1196. {
  1197. kiocb->ki_iovec = &kiocb->ki_inline_vec;
  1198. kiocb->ki_iovec->iov_base = kiocb->ki_buf;
  1199. kiocb->ki_iovec->iov_len = kiocb->ki_left;
  1200. kiocb->ki_nr_segs = 1;
  1201. kiocb->ki_cur_seg = 0;
  1202. return 0;
  1203. }
  1204. /*
  1205. * aio_setup_iocb:
  1206. * Performs the initial checks and aio retry method
  1207. * setup for the kiocb at the time of io submission.
  1208. */
  1209. static ssize_t aio_setup_iocb(struct kiocb *kiocb)
  1210. {
  1211. struct file *file = kiocb->ki_filp;
  1212. ssize_t ret = 0;
  1213. switch (kiocb->ki_opcode) {
  1214. case IOCB_CMD_PREAD:
  1215. ret = -EBADF;
  1216. if (unlikely(!(file->f_mode & FMODE_READ)))
  1217. break;
  1218. ret = -EFAULT;
  1219. if (unlikely(!access_ok(VERIFY_WRITE, kiocb->ki_buf,
  1220. kiocb->ki_left)))
  1221. break;
  1222. ret = security_file_permission(file, MAY_READ);
  1223. if (unlikely(ret))
  1224. break;
  1225. ret = aio_setup_single_vector(kiocb);
  1226. if (ret)
  1227. break;
  1228. ret = -EINVAL;
  1229. if (file->f_op->aio_read)
  1230. kiocb->ki_retry = aio_rw_vect_retry;
  1231. break;
  1232. case IOCB_CMD_PWRITE:
  1233. ret = -EBADF;
  1234. if (unlikely(!(file->f_mode & FMODE_WRITE)))
  1235. break;
  1236. ret = -EFAULT;
  1237. if (unlikely(!access_ok(VERIFY_READ, kiocb->ki_buf,
  1238. kiocb->ki_left)))
  1239. break;
  1240. ret = security_file_permission(file, MAY_WRITE);
  1241. if (unlikely(ret))
  1242. break;
  1243. ret = aio_setup_single_vector(kiocb);
  1244. if (ret)
  1245. break;
  1246. ret = -EINVAL;
  1247. if (file->f_op->aio_write)
  1248. kiocb->ki_retry = aio_rw_vect_retry;
  1249. break;
  1250. case IOCB_CMD_PREADV:
  1251. ret = -EBADF;
  1252. if (unlikely(!(file->f_mode & FMODE_READ)))
  1253. break;
  1254. ret = security_file_permission(file, MAY_READ);
  1255. if (unlikely(ret))
  1256. break;
  1257. ret = aio_setup_vectored_rw(READ, kiocb);
  1258. if (ret)
  1259. break;
  1260. ret = -EINVAL;
  1261. if (file->f_op->aio_read)
  1262. kiocb->ki_retry = aio_rw_vect_retry;
  1263. break;
  1264. case IOCB_CMD_PWRITEV:
  1265. ret = -EBADF;
  1266. if (unlikely(!(file->f_mode & FMODE_WRITE)))
  1267. break;
  1268. ret = security_file_permission(file, MAY_WRITE);
  1269. if (unlikely(ret))
  1270. break;
  1271. ret = aio_setup_vectored_rw(WRITE, kiocb);
  1272. if (ret)
  1273. break;
  1274. ret = -EINVAL;
  1275. if (file->f_op->aio_write)
  1276. kiocb->ki_retry = aio_rw_vect_retry;
  1277. break;
  1278. case IOCB_CMD_FDSYNC:
  1279. ret = -EINVAL;
  1280. if (file->f_op->aio_fsync)
  1281. kiocb->ki_retry = aio_fdsync;
  1282. break;
  1283. case IOCB_CMD_FSYNC:
  1284. ret = -EINVAL;
  1285. if (file->f_op->aio_fsync)
  1286. kiocb->ki_retry = aio_fsync;
  1287. break;
  1288. default:
  1289. dprintk("EINVAL: io_submit: no operation provided\n");
  1290. ret = -EINVAL;
  1291. }
  1292. if (!kiocb->ki_retry)
  1293. return ret;
  1294. return 0;
  1295. }
  1296. /*
  1297. * aio_wake_function:
  1298. * wait queue callback function for aio notification,
  1299. * Simply triggers a retry of the operation via kick_iocb.
  1300. *
  1301. * This callback is specified in the wait queue entry in
  1302. * a kiocb (current->io_wait points to this wait queue
  1303. * entry when an aio operation executes; it is used
  1304. * instead of a synchronous wait when an i/o blocking
  1305. * condition is encountered during aio).
  1306. *
  1307. * Note:
  1308. * This routine is executed with the wait queue lock held.
  1309. * Since kick_iocb acquires iocb->ctx->ctx_lock, it nests
  1310. * the ioctx lock inside the wait queue lock. This is safe
  1311. * because this callback isn't used for wait queues which
  1312. * are nested inside ioctx lock (i.e. ctx->wait)
  1313. */
  1314. static int aio_wake_function(wait_queue_t *wait, unsigned mode,
  1315. int sync, void *key)
  1316. {
  1317. struct kiocb *iocb = container_of(wait, struct kiocb, ki_wait);
  1318. list_del_init(&wait->task_list);
  1319. kick_iocb(iocb);
  1320. return 1;
  1321. }
  1322. int fastcall io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
  1323. struct iocb *iocb)
  1324. {
  1325. struct kiocb *req;
  1326. struct file *file;
  1327. ssize_t ret;
  1328. /* enforce forwards compatibility on users */
  1329. if (unlikely(iocb->aio_reserved1 || iocb->aio_reserved2 ||
  1330. iocb->aio_reserved3)) {
  1331. pr_debug("EINVAL: io_submit: reserve field set\n");
  1332. return -EINVAL;
  1333. }
  1334. /* prevent overflows */
  1335. if (unlikely(
  1336. (iocb->aio_buf != (unsigned long)iocb->aio_buf) ||
  1337. (iocb->aio_nbytes != (size_t)iocb->aio_nbytes) ||
  1338. ((ssize_t)iocb->aio_nbytes < 0)
  1339. )) {
  1340. pr_debug("EINVAL: io_submit: overflow check\n");
  1341. return -EINVAL;
  1342. }
  1343. file = fget(iocb->aio_fildes);
  1344. if (unlikely(!file))
  1345. return -EBADF;
  1346. req = aio_get_req(ctx); /* returns with 2 references to req */
  1347. if (unlikely(!req)) {
  1348. fput(file);
  1349. return -EAGAIN;
  1350. }
  1351. req->ki_filp = file;
  1352. ret = put_user(req->ki_key, &user_iocb->aio_key);
  1353. if (unlikely(ret)) {
  1354. dprintk("EFAULT: aio_key\n");
  1355. goto out_put_req;
  1356. }
  1357. req->ki_obj.user = user_iocb;
  1358. req->ki_user_data = iocb->aio_data;
  1359. req->ki_pos = iocb->aio_offset;
  1360. req->ki_buf = (char __user *)(unsigned long)iocb->aio_buf;
  1361. req->ki_left = req->ki_nbytes = iocb->aio_nbytes;
  1362. req->ki_opcode = iocb->aio_lio_opcode;
  1363. init_waitqueue_func_entry(&req->ki_wait, aio_wake_function);
  1364. INIT_LIST_HEAD(&req->ki_wait.task_list);
  1365. ret = aio_setup_iocb(req);
  1366. if (ret)
  1367. goto out_put_req;
  1368. spin_lock_irq(&ctx->ctx_lock);
  1369. aio_run_iocb(req);
  1370. if (!list_empty(&ctx->run_list)) {
  1371. /* drain the run list */
  1372. while (__aio_run_iocbs(ctx))
  1373. ;
  1374. }
  1375. spin_unlock_irq(&ctx->ctx_lock);
  1376. aio_put_req(req); /* drop extra ref to req */
  1377. return 0;
  1378. out_put_req:
  1379. aio_put_req(req); /* drop extra ref to req */
  1380. aio_put_req(req); /* drop i/o ref to req */
  1381. return ret;
  1382. }
  1383. /* sys_io_submit:
  1384. * Queue the nr iocbs pointed to by iocbpp for processing. Returns
  1385. * the number of iocbs queued. May return -EINVAL if the aio_context
  1386. * specified by ctx_id is invalid, if nr is < 0, if the iocb at
  1387. * *iocbpp[0] is not properly initialized, if the operation specified
  1388. * is invalid for the file descriptor in the iocb. May fail with
  1389. * -EFAULT if any of the data structures point to invalid data. May
  1390. * fail with -EBADF if the file descriptor specified in the first
  1391. * iocb is invalid. May fail with -EAGAIN if insufficient resources
  1392. * are available to queue any iocbs. Will return 0 if nr is 0. Will
  1393. * fail with -ENOSYS if not implemented.
  1394. */
  1395. asmlinkage long sys_io_submit(aio_context_t ctx_id, long nr,
  1396. struct iocb __user * __user *iocbpp)
  1397. {
  1398. struct kioctx *ctx;
  1399. long ret = 0;
  1400. int i;
  1401. if (unlikely(nr < 0))
  1402. return -EINVAL;
  1403. if (unlikely(!access_ok(VERIFY_READ, iocbpp, (nr*sizeof(*iocbpp)))))
  1404. return -EFAULT;
  1405. ctx = lookup_ioctx(ctx_id);
  1406. if (unlikely(!ctx)) {
  1407. pr_debug("EINVAL: io_submit: invalid context id\n");
  1408. return -EINVAL;
  1409. }
  1410. /*
  1411. * AKPM: should this return a partial result if some of the IOs were
  1412. * successfully submitted?
  1413. */
  1414. for (i=0; i<nr; i++) {
  1415. struct iocb __user *user_iocb;
  1416. struct iocb tmp;
  1417. if (unlikely(__get_user(user_iocb, iocbpp + i))) {
  1418. ret = -EFAULT;
  1419. break;
  1420. }
  1421. if (unlikely(copy_from_user(&tmp, user_iocb, sizeof(tmp)))) {
  1422. ret = -EFAULT;
  1423. break;
  1424. }
  1425. ret = io_submit_one(ctx, user_iocb, &tmp);
  1426. if (ret)
  1427. break;
  1428. }
  1429. put_ioctx(ctx);
  1430. return i ? i : ret;
  1431. }
  1432. /* lookup_kiocb
  1433. * Finds a given iocb for cancellation.
  1434. */
  1435. static struct kiocb *lookup_kiocb(struct kioctx *ctx, struct iocb __user *iocb,
  1436. u32 key)
  1437. {
  1438. struct list_head *pos;
  1439. assert_spin_locked(&ctx->ctx_lock);
  1440. /* TODO: use a hash or array, this sucks. */
  1441. list_for_each(pos, &ctx->active_reqs) {
  1442. struct kiocb *kiocb = list_kiocb(pos);
  1443. if (kiocb->ki_obj.user == iocb && kiocb->ki_key == key)
  1444. return kiocb;
  1445. }
  1446. return NULL;
  1447. }
  1448. /* sys_io_cancel:
  1449. * Attempts to cancel an iocb previously passed to io_submit. If
  1450. * the operation is successfully cancelled, the resulting event is
  1451. * copied into the memory pointed to by result without being placed
  1452. * into the completion queue and 0 is returned. May fail with
  1453. * -EFAULT if any of the data structures pointed to are invalid.
  1454. * May fail with -EINVAL if aio_context specified by ctx_id is
  1455. * invalid. May fail with -EAGAIN if the iocb specified was not
  1456. * cancelled. Will fail with -ENOSYS if not implemented.
  1457. */
  1458. asmlinkage long sys_io_cancel(aio_context_t ctx_id, struct iocb __user *iocb,
  1459. struct io_event __user *result)
  1460. {
  1461. int (*cancel)(struct kiocb *iocb, struct io_event *res);
  1462. struct kioctx *ctx;
  1463. struct kiocb *kiocb;
  1464. u32 key;
  1465. int ret;
  1466. ret = get_user(key, &iocb->aio_key);
  1467. if (unlikely(ret))
  1468. return -EFAULT;
  1469. ctx = lookup_ioctx(ctx_id);
  1470. if (unlikely(!ctx))
  1471. return -EINVAL;
  1472. spin_lock_irq(&ctx->ctx_lock);
  1473. ret = -EAGAIN;
  1474. kiocb = lookup_kiocb(ctx, iocb, key);
  1475. if (kiocb && kiocb->ki_cancel) {
  1476. cancel = kiocb->ki_cancel;
  1477. kiocb->ki_users ++;
  1478. kiocbSetCancelled(kiocb);
  1479. } else
  1480. cancel = NULL;
  1481. spin_unlock_irq(&ctx->ctx_lock);
  1482. if (NULL != cancel) {
  1483. struct io_event tmp;
  1484. pr_debug("calling cancel\n");
  1485. memset(&tmp, 0, sizeof(tmp));
  1486. tmp.obj = (u64)(unsigned long)kiocb->ki_obj.user;
  1487. tmp.data = kiocb->ki_user_data;
  1488. ret = cancel(kiocb, &tmp);
  1489. if (!ret) {
  1490. /* Cancellation succeeded -- copy the result
  1491. * into the user's buffer.
  1492. */
  1493. if (copy_to_user(result, &tmp, sizeof(tmp)))
  1494. ret = -EFAULT;
  1495. }
  1496. } else
  1497. ret = -EINVAL;
  1498. put_ioctx(ctx);
  1499. return ret;
  1500. }
  1501. /* io_getevents:
  1502. * Attempts to read at least min_nr events and up to nr events from
  1503. * the completion queue for the aio_context specified by ctx_id. May
  1504. * fail with -EINVAL if ctx_id is invalid, if min_nr is out of range,
  1505. * if nr is out of range, if when is out of range. May fail with
  1506. * -EFAULT if any of the memory specified to is invalid. May return
  1507. * 0 or < min_nr if no events are available and the timeout specified
  1508. * by when has elapsed, where when == NULL specifies an infinite
  1509. * timeout. Note that the timeout pointed to by when is relative and
  1510. * will be updated if not NULL and the operation blocks. Will fail
  1511. * with -ENOSYS if not implemented.
  1512. */
  1513. asmlinkage long sys_io_getevents(aio_context_t ctx_id,
  1514. long min_nr,
  1515. long nr,
  1516. struct io_event __user *events,
  1517. struct timespec __user *timeout)
  1518. {
  1519. struct kioctx *ioctx = lookup_ioctx(ctx_id);
  1520. long ret = -EINVAL;
  1521. if (likely(ioctx)) {
  1522. if (likely(min_nr <= nr && min_nr >= 0 && nr >= 0))
  1523. ret = read_events(ioctx, min_nr, nr, events, timeout);
  1524. put_ioctx(ioctx);
  1525. }
  1526. return ret;
  1527. }
  1528. __initcall(aio_setup);
  1529. EXPORT_SYMBOL(aio_complete);
  1530. EXPORT_SYMBOL(aio_put_req);
  1531. EXPORT_SYMBOL(wait_on_sync_kiocb);