aio.c 61 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392
  1. /*
  2. * An async IO implementation for Linux
  3. * Written by Benjamin LaHaise <bcrl@kvack.org>
  4. *
  5. * Implements an efficient asynchronous io interface.
  6. *
  7. * Copyright 2000, 2001, 2002 Red Hat, Inc. All Rights Reserved.
  8. * Copyright 2018 Christoph Hellwig.
  9. *
  10. * See ../COPYING for licensing terms.
  11. */
  12. #define pr_fmt(fmt) "%s: " fmt, __func__
  13. #include <linux/kernel.h>
  14. #include <linux/init.h>
  15. #include <linux/errno.h>
  16. #include <linux/time.h>
  17. #include <linux/aio_abi.h>
  18. #include <linux/export.h>
  19. #include <linux/syscalls.h>
  20. #include <linux/backing-dev.h>
  21. #include <linux/refcount.h>
  22. #include <linux/uio.h>
  23. #include <linux/sched/signal.h>
  24. #include <linux/fs.h>
  25. #include <linux/file.h>
  26. #include <linux/mm.h>
  27. #include <linux/mman.h>
  28. #include <linux/percpu.h>
  29. #include <linux/slab.h>
  30. #include <linux/timer.h>
  31. #include <linux/aio.h>
  32. #include <linux/highmem.h>
  33. #include <linux/workqueue.h>
  34. #include <linux/security.h>
  35. #include <linux/eventfd.h>
  36. #include <linux/blkdev.h>
  37. #include <linux/compat.h>
  38. #include <linux/migrate.h>
  39. #include <linux/ramfs.h>
  40. #include <linux/percpu-refcount.h>
  41. #include <linux/mount.h>
  42. #include <linux/pseudo_fs.h>
  43. #include <asm/kmap_types.h>
  44. #include <linux/uaccess.h>
  45. #include <linux/nospec.h>
  46. #include "internal.h"
  47. #define KIOCB_KEY 0
  48. #define AIO_RING_MAGIC 0xa10a10a1
  49. #define AIO_RING_COMPAT_FEATURES 1
  50. #define AIO_RING_INCOMPAT_FEATURES 0
  51. struct aio_ring {
  52. unsigned id; /* kernel internal index number */
  53. unsigned nr; /* number of io_events */
  54. unsigned head; /* Written to by userland or under ring_lock
  55. * mutex by aio_read_events_ring(). */
  56. unsigned tail;
  57. unsigned magic;
  58. unsigned compat_features;
  59. unsigned incompat_features;
  60. unsigned header_length; /* size of aio_ring */
  61. struct io_event io_events[];
  62. }; /* 128 bytes + ring size */
  63. /*
  64. * Plugging is meant to work with larger batches of IOs. If we don't
  65. * have more than the below, then don't bother setting up a plug.
  66. */
  67. #define AIO_PLUG_THRESHOLD 2
  68. #define AIO_RING_PAGES 8
  69. struct kioctx_table {
  70. struct rcu_head rcu;
  71. unsigned nr;
  72. struct kioctx __rcu *table[];
  73. };
  74. struct kioctx_cpu {
  75. unsigned reqs_available;
  76. };
  77. struct ctx_rq_wait {
  78. struct completion comp;
  79. atomic_t count;
  80. };
  81. struct kioctx {
  82. struct percpu_ref users;
  83. atomic_t dead;
  84. struct percpu_ref reqs;
  85. unsigned long user_id;
  86. struct __percpu kioctx_cpu *cpu;
  87. /*
  88. * For percpu reqs_available, number of slots we move to/from global
  89. * counter at a time:
  90. */
  91. unsigned req_batch;
  92. /*
  93. * This is what userspace passed to io_setup(), it's not used for
  94. * anything but counting against the global max_reqs quota.
  95. *
  96. * The real limit is nr_events - 1, which will be larger (see
  97. * aio_setup_ring())
  98. */
  99. unsigned max_reqs;
  100. /* Size of ringbuffer, in units of struct io_event */
  101. unsigned nr_events;
  102. unsigned long mmap_base;
  103. unsigned long mmap_size;
  104. struct page **ring_pages;
  105. long nr_pages;
  106. struct rcu_work free_rwork; /* see free_ioctx() */
  107. /*
  108. * signals when all in-flight requests are done
  109. */
  110. struct ctx_rq_wait *rq_wait;
  111. struct {
  112. /*
  113. * This counts the number of available slots in the ringbuffer,
  114. * so we avoid overflowing it: it's decremented (if positive)
  115. * when allocating a kiocb and incremented when the resulting
  116. * io_event is pulled off the ringbuffer.
  117. *
  118. * We batch accesses to it with a percpu version.
  119. */
  120. atomic_t reqs_available;
  121. } ____cacheline_aligned_in_smp;
  122. struct {
  123. spinlock_t ctx_lock;
  124. struct list_head active_reqs; /* used for cancellation */
  125. } ____cacheline_aligned_in_smp;
  126. struct {
  127. struct mutex ring_lock;
  128. wait_queue_head_t wait;
  129. } ____cacheline_aligned_in_smp;
  130. struct {
  131. unsigned tail;
  132. unsigned completed_events;
  133. spinlock_t completion_lock;
  134. } ____cacheline_aligned_in_smp;
  135. struct page *internal_pages[AIO_RING_PAGES];
  136. struct file *aio_ring_file;
  137. unsigned id;
  138. };
  139. /*
  140. * First field must be the file pointer in all the
  141. * iocb unions! See also 'struct kiocb' in <linux/fs.h>
  142. */
  143. struct fsync_iocb {
  144. struct file *file;
  145. struct work_struct work;
  146. bool datasync;
  147. struct cred *creds;
  148. };
  149. struct poll_iocb {
  150. struct file *file;
  151. struct wait_queue_head *head;
  152. __poll_t events;
  153. bool cancelled;
  154. bool work_scheduled;
  155. bool work_need_resched;
  156. struct wait_queue_entry wait;
  157. struct work_struct work;
  158. };
  159. /*
  160. * NOTE! Each of the iocb union members has the file pointer
  161. * as the first entry in their struct definition. So you can
  162. * access the file pointer through any of the sub-structs,
  163. * or directly as just 'ki_filp' in this struct.
  164. */
  165. struct aio_kiocb {
  166. union {
  167. struct file *ki_filp;
  168. struct kiocb rw;
  169. struct fsync_iocb fsync;
  170. struct poll_iocb poll;
  171. };
  172. struct kioctx *ki_ctx;
  173. kiocb_cancel_fn *ki_cancel;
  174. struct io_event ki_res;
  175. struct list_head ki_list; /* the aio core uses this
  176. * for cancellation */
  177. refcount_t ki_refcnt;
  178. /*
  179. * If the aio_resfd field of the userspace iocb is not zero,
  180. * this is the underlying eventfd context to deliver events to.
  181. */
  182. struct eventfd_ctx *ki_eventfd;
  183. };
  184. /*------ sysctl variables----*/
  185. static DEFINE_SPINLOCK(aio_nr_lock);
  186. unsigned long aio_nr; /* current system wide number of aio requests */
  187. unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */
  188. /*----end sysctl variables---*/
  189. static struct kmem_cache *kiocb_cachep;
  190. static struct kmem_cache *kioctx_cachep;
  191. static struct vfsmount *aio_mnt;
  192. static const struct file_operations aio_ring_fops;
  193. static const struct address_space_operations aio_ctx_aops;
  194. static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages)
  195. {
  196. struct file *file;
  197. struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb);
  198. if (IS_ERR(inode))
  199. return ERR_CAST(inode);
  200. inode->i_mapping->a_ops = &aio_ctx_aops;
  201. inode->i_mapping->private_data = ctx;
  202. inode->i_size = PAGE_SIZE * nr_pages;
  203. file = alloc_file_pseudo(inode, aio_mnt, "[aio]",
  204. O_RDWR, &aio_ring_fops);
  205. if (IS_ERR(file))
  206. iput(inode);
  207. return file;
  208. }
  209. static int aio_init_fs_context(struct fs_context *fc)
  210. {
  211. if (!init_pseudo(fc, AIO_RING_MAGIC))
  212. return -ENOMEM;
  213. fc->s_iflags |= SB_I_NOEXEC;
  214. return 0;
  215. }
  216. /* aio_setup
  217. * Creates the slab caches used by the aio routines, panic on
  218. * failure as this is done early during the boot sequence.
  219. */
  220. static int __init aio_setup(void)
  221. {
  222. static struct file_system_type aio_fs = {
  223. .name = "aio",
  224. .init_fs_context = aio_init_fs_context,
  225. .kill_sb = kill_anon_super,
  226. };
  227. aio_mnt = kern_mount(&aio_fs);
  228. if (IS_ERR(aio_mnt))
  229. panic("Failed to create aio fs mount.");
  230. kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
  231. kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC);
  232. return 0;
  233. }
  234. __initcall(aio_setup);
  235. static void put_aio_ring_file(struct kioctx *ctx)
  236. {
  237. struct file *aio_ring_file = ctx->aio_ring_file;
  238. struct address_space *i_mapping;
  239. if (aio_ring_file) {
  240. truncate_setsize(file_inode(aio_ring_file), 0);
  241. /* Prevent further access to the kioctx from migratepages */
  242. i_mapping = aio_ring_file->f_mapping;
  243. spin_lock(&i_mapping->private_lock);
  244. i_mapping->private_data = NULL;
  245. ctx->aio_ring_file = NULL;
  246. spin_unlock(&i_mapping->private_lock);
  247. fput(aio_ring_file);
  248. }
  249. }
  250. static void aio_free_ring(struct kioctx *ctx)
  251. {
  252. int i;
  253. /* Disconnect the kiotx from the ring file. This prevents future
  254. * accesses to the kioctx from page migration.
  255. */
  256. put_aio_ring_file(ctx);
  257. for (i = 0; i < ctx->nr_pages; i++) {
  258. struct page *page;
  259. pr_debug("pid(%d) [%d] page->count=%d\n", current->pid, i,
  260. page_count(ctx->ring_pages[i]));
  261. page = ctx->ring_pages[i];
  262. if (!page)
  263. continue;
  264. ctx->ring_pages[i] = NULL;
  265. put_page(page);
  266. }
  267. if (ctx->ring_pages && ctx->ring_pages != ctx->internal_pages) {
  268. kfree(ctx->ring_pages);
  269. ctx->ring_pages = NULL;
  270. }
  271. }
  272. static int aio_ring_mremap(struct vm_area_struct *vma)
  273. {
  274. struct file *file = vma->vm_file;
  275. struct mm_struct *mm = vma->vm_mm;
  276. struct kioctx_table *table;
  277. int i, res = -EINVAL;
  278. spin_lock(&mm->ioctx_lock);
  279. rcu_read_lock();
  280. table = rcu_dereference(mm->ioctx_table);
  281. for (i = 0; i < table->nr; i++) {
  282. struct kioctx *ctx;
  283. ctx = rcu_dereference(table->table[i]);
  284. if (ctx && ctx->aio_ring_file == file) {
  285. if (!atomic_read(&ctx->dead)) {
  286. ctx->user_id = ctx->mmap_base = vma->vm_start;
  287. res = 0;
  288. }
  289. break;
  290. }
  291. }
  292. rcu_read_unlock();
  293. spin_unlock(&mm->ioctx_lock);
  294. return res;
  295. }
  296. static const struct vm_operations_struct aio_ring_vm_ops = {
  297. .mremap = aio_ring_mremap,
  298. #if IS_ENABLED(CONFIG_MMU)
  299. .fault = filemap_fault,
  300. .map_pages = filemap_map_pages,
  301. .page_mkwrite = filemap_page_mkwrite,
  302. #endif
  303. };
  304. static int aio_ring_mmap(struct file *file, struct vm_area_struct *vma)
  305. {
  306. vma->vm_flags |= VM_DONTEXPAND;
  307. vma->vm_ops = &aio_ring_vm_ops;
  308. return 0;
  309. }
  310. static const struct file_operations aio_ring_fops = {
  311. .mmap = aio_ring_mmap,
  312. };
  313. #if IS_ENABLED(CONFIG_MIGRATION)
  314. static int aio_migratepage(struct address_space *mapping, struct page *new,
  315. struct page *old, enum migrate_mode mode)
  316. {
  317. struct kioctx *ctx;
  318. unsigned long flags;
  319. pgoff_t idx;
  320. int rc;
  321. /*
  322. * We cannot support the _NO_COPY case here, because copy needs to
  323. * happen under the ctx->completion_lock. That does not work with the
  324. * migration workflow of MIGRATE_SYNC_NO_COPY.
  325. */
  326. if (mode == MIGRATE_SYNC_NO_COPY)
  327. return -EINVAL;
  328. rc = 0;
  329. /* mapping->private_lock here protects against the kioctx teardown. */
  330. spin_lock(&mapping->private_lock);
  331. ctx = mapping->private_data;
  332. if (!ctx) {
  333. rc = -EINVAL;
  334. goto out;
  335. }
  336. /* The ring_lock mutex. The prevents aio_read_events() from writing
  337. * to the ring's head, and prevents page migration from mucking in
  338. * a partially initialized kiotx.
  339. */
  340. if (!mutex_trylock(&ctx->ring_lock)) {
  341. rc = -EAGAIN;
  342. goto out;
  343. }
  344. idx = old->index;
  345. if (idx < (pgoff_t)ctx->nr_pages) {
  346. /* Make sure the old page hasn't already been changed */
  347. if (ctx->ring_pages[idx] != old)
  348. rc = -EAGAIN;
  349. } else
  350. rc = -EINVAL;
  351. if (rc != 0)
  352. goto out_unlock;
  353. /* Writeback must be complete */
  354. BUG_ON(PageWriteback(old));
  355. get_page(new);
  356. rc = migrate_page_move_mapping(mapping, new, old, 1);
  357. if (rc != MIGRATEPAGE_SUCCESS) {
  358. put_page(new);
  359. goto out_unlock;
  360. }
  361. /* Take completion_lock to prevent other writes to the ring buffer
  362. * while the old page is copied to the new. This prevents new
  363. * events from being lost.
  364. */
  365. spin_lock_irqsave(&ctx->completion_lock, flags);
  366. migrate_page_copy(new, old);
  367. BUG_ON(ctx->ring_pages[idx] != old);
  368. ctx->ring_pages[idx] = new;
  369. spin_unlock_irqrestore(&ctx->completion_lock, flags);
  370. /* The old page is no longer accessible. */
  371. put_page(old);
  372. out_unlock:
  373. mutex_unlock(&ctx->ring_lock);
  374. out:
  375. spin_unlock(&mapping->private_lock);
  376. return rc;
  377. }
  378. #endif
  379. static const struct address_space_operations aio_ctx_aops = {
  380. .set_page_dirty = __set_page_dirty_no_writeback,
  381. #if IS_ENABLED(CONFIG_MIGRATION)
  382. .migratepage = aio_migratepage,
  383. #endif
  384. };
  385. static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events)
  386. {
  387. struct aio_ring *ring;
  388. struct mm_struct *mm = current->mm;
  389. unsigned long size, unused;
  390. int nr_pages;
  391. int i;
  392. struct file *file;
  393. /* Compensate for the ring buffer's head/tail overlap entry */
  394. nr_events += 2; /* 1 is required, 2 for good luck */
  395. size = sizeof(struct aio_ring);
  396. size += sizeof(struct io_event) * nr_events;
  397. nr_pages = PFN_UP(size);
  398. if (nr_pages < 0)
  399. return -EINVAL;
  400. file = aio_private_file(ctx, nr_pages);
  401. if (IS_ERR(file)) {
  402. ctx->aio_ring_file = NULL;
  403. return -ENOMEM;
  404. }
  405. ctx->aio_ring_file = file;
  406. nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring))
  407. / sizeof(struct io_event);
  408. ctx->ring_pages = ctx->internal_pages;
  409. if (nr_pages > AIO_RING_PAGES) {
  410. ctx->ring_pages = kcalloc(nr_pages, sizeof(struct page *),
  411. GFP_KERNEL);
  412. if (!ctx->ring_pages) {
  413. put_aio_ring_file(ctx);
  414. return -ENOMEM;
  415. }
  416. }
  417. for (i = 0; i < nr_pages; i++) {
  418. struct page *page;
  419. page = find_or_create_page(file->f_mapping,
  420. i, GFP_HIGHUSER | __GFP_ZERO);
  421. if (!page)
  422. break;
  423. pr_debug("pid(%d) page[%d]->count=%d\n",
  424. current->pid, i, page_count(page));
  425. SetPageUptodate(page);
  426. unlock_page(page);
  427. ctx->ring_pages[i] = page;
  428. }
  429. ctx->nr_pages = i;
  430. if (unlikely(i != nr_pages)) {
  431. aio_free_ring(ctx);
  432. return -ENOMEM;
  433. }
  434. ctx->mmap_size = nr_pages * PAGE_SIZE;
  435. pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size);
  436. if (mmap_write_lock_killable(mm)) {
  437. ctx->mmap_size = 0;
  438. aio_free_ring(ctx);
  439. return -EINTR;
  440. }
  441. ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size,
  442. PROT_READ | PROT_WRITE,
  443. MAP_SHARED, 0, &unused, NULL);
  444. mmap_write_unlock(mm);
  445. if (IS_ERR((void *)ctx->mmap_base)) {
  446. ctx->mmap_size = 0;
  447. aio_free_ring(ctx);
  448. return -ENOMEM;
  449. }
  450. pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base);
  451. ctx->user_id = ctx->mmap_base;
  452. ctx->nr_events = nr_events; /* trusted copy */
  453. ring = kmap_atomic(ctx->ring_pages[0]);
  454. ring->nr = nr_events; /* user copy */
  455. ring->id = ~0U;
  456. ring->head = ring->tail = 0;
  457. ring->magic = AIO_RING_MAGIC;
  458. ring->compat_features = AIO_RING_COMPAT_FEATURES;
  459. ring->incompat_features = AIO_RING_INCOMPAT_FEATURES;
  460. ring->header_length = sizeof(struct aio_ring);
  461. kunmap_atomic(ring);
  462. flush_dcache_page(ctx->ring_pages[0]);
  463. return 0;
  464. }
  465. #define AIO_EVENTS_PER_PAGE (PAGE_SIZE / sizeof(struct io_event))
  466. #define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
  467. #define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)
  468. void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel)
  469. {
  470. struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, rw);
  471. struct kioctx *ctx = req->ki_ctx;
  472. unsigned long flags;
  473. if (WARN_ON_ONCE(!list_empty(&req->ki_list)))
  474. return;
  475. spin_lock_irqsave(&ctx->ctx_lock, flags);
  476. list_add_tail(&req->ki_list, &ctx->active_reqs);
  477. req->ki_cancel = cancel;
  478. spin_unlock_irqrestore(&ctx->ctx_lock, flags);
  479. }
  480. EXPORT_SYMBOL(kiocb_set_cancel_fn);
  481. /*
  482. * free_ioctx() should be RCU delayed to synchronize against the RCU
  483. * protected lookup_ioctx() and also needs process context to call
  484. * aio_free_ring(). Use rcu_work.
  485. */
  486. static void free_ioctx(struct work_struct *work)
  487. {
  488. struct kioctx *ctx = container_of(to_rcu_work(work), struct kioctx,
  489. free_rwork);
  490. pr_debug("freeing %p\n", ctx);
  491. aio_free_ring(ctx);
  492. free_percpu(ctx->cpu);
  493. percpu_ref_exit(&ctx->reqs);
  494. percpu_ref_exit(&ctx->users);
  495. kmem_cache_free(kioctx_cachep, ctx);
  496. }
  497. static void free_ioctx_reqs(struct percpu_ref *ref)
  498. {
  499. struct kioctx *ctx = container_of(ref, struct kioctx, reqs);
  500. /* At this point we know that there are no any in-flight requests */
  501. if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count))
  502. complete(&ctx->rq_wait->comp);
  503. /* Synchronize against RCU protected table->table[] dereferences */
  504. INIT_RCU_WORK(&ctx->free_rwork, free_ioctx);
  505. queue_rcu_work(system_wq, &ctx->free_rwork);
  506. }
  507. /*
  508. * When this function runs, the kioctx has been removed from the "hash table"
  509. * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
  510. * now it's safe to cancel any that need to be.
  511. */
  512. static void free_ioctx_users(struct percpu_ref *ref)
  513. {
  514. struct kioctx *ctx = container_of(ref, struct kioctx, users);
  515. struct aio_kiocb *req;
  516. spin_lock_irq(&ctx->ctx_lock);
  517. while (!list_empty(&ctx->active_reqs)) {
  518. req = list_first_entry(&ctx->active_reqs,
  519. struct aio_kiocb, ki_list);
  520. req->ki_cancel(&req->rw);
  521. list_del_init(&req->ki_list);
  522. }
  523. spin_unlock_irq(&ctx->ctx_lock);
  524. percpu_ref_kill(&ctx->reqs);
  525. percpu_ref_put(&ctx->reqs);
  526. }
  527. static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
  528. {
  529. unsigned i, new_nr;
  530. struct kioctx_table *table, *old;
  531. struct aio_ring *ring;
  532. spin_lock(&mm->ioctx_lock);
  533. table = rcu_dereference_raw(mm->ioctx_table);
  534. while (1) {
  535. if (table)
  536. for (i = 0; i < table->nr; i++)
  537. if (!rcu_access_pointer(table->table[i])) {
  538. ctx->id = i;
  539. rcu_assign_pointer(table->table[i], ctx);
  540. spin_unlock(&mm->ioctx_lock);
  541. /* While kioctx setup is in progress,
  542. * we are protected from page migration
  543. * changes ring_pages by ->ring_lock.
  544. */
  545. ring = kmap_atomic(ctx->ring_pages[0]);
  546. ring->id = ctx->id;
  547. kunmap_atomic(ring);
  548. return 0;
  549. }
  550. new_nr = (table ? table->nr : 1) * 4;
  551. spin_unlock(&mm->ioctx_lock);
  552. table = kzalloc(sizeof(*table) + sizeof(struct kioctx *) *
  553. new_nr, GFP_KERNEL);
  554. if (!table)
  555. return -ENOMEM;
  556. table->nr = new_nr;
  557. spin_lock(&mm->ioctx_lock);
  558. old = rcu_dereference_raw(mm->ioctx_table);
  559. if (!old) {
  560. rcu_assign_pointer(mm->ioctx_table, table);
  561. } else if (table->nr > old->nr) {
  562. memcpy(table->table, old->table,
  563. old->nr * sizeof(struct kioctx *));
  564. rcu_assign_pointer(mm->ioctx_table, table);
  565. kfree_rcu(old, rcu);
  566. } else {
  567. kfree(table);
  568. table = old;
  569. }
  570. }
  571. }
  572. static void aio_nr_sub(unsigned nr)
  573. {
  574. spin_lock(&aio_nr_lock);
  575. if (WARN_ON(aio_nr - nr > aio_nr))
  576. aio_nr = 0;
  577. else
  578. aio_nr -= nr;
  579. spin_unlock(&aio_nr_lock);
  580. }
  581. /* ioctx_alloc
  582. * Allocates and initializes an ioctx. Returns an ERR_PTR if it failed.
  583. */
  584. static struct kioctx *ioctx_alloc(unsigned nr_events)
  585. {
  586. struct mm_struct *mm = current->mm;
  587. struct kioctx *ctx;
  588. int err = -ENOMEM;
  589. /*
  590. * Store the original nr_events -- what userspace passed to io_setup(),
  591. * for counting against the global limit -- before it changes.
  592. */
  593. unsigned int max_reqs = nr_events;
  594. /*
  595. * We keep track of the number of available ringbuffer slots, to prevent
  596. * overflow (reqs_available), and we also use percpu counters for this.
  597. *
  598. * So since up to half the slots might be on other cpu's percpu counters
  599. * and unavailable, double nr_events so userspace sees what they
  600. * expected: additionally, we move req_batch slots to/from percpu
  601. * counters at a time, so make sure that isn't 0:
  602. */
  603. nr_events = max(nr_events, num_possible_cpus() * 4);
  604. nr_events *= 2;
  605. /* Prevent overflows */
  606. if (nr_events > (0x10000000U / sizeof(struct io_event))) {
  607. pr_debug("ENOMEM: nr_events too high\n");
  608. return ERR_PTR(-EINVAL);
  609. }
  610. if (!nr_events || (unsigned long)max_reqs > aio_max_nr)
  611. return ERR_PTR(-EAGAIN);
  612. ctx = kmem_cache_zalloc(kioctx_cachep, GFP_KERNEL);
  613. if (!ctx)
  614. return ERR_PTR(-ENOMEM);
  615. ctx->max_reqs = max_reqs;
  616. spin_lock_init(&ctx->ctx_lock);
  617. spin_lock_init(&ctx->completion_lock);
  618. mutex_init(&ctx->ring_lock);
  619. /* Protect against page migration throughout kiotx setup by keeping
  620. * the ring_lock mutex held until setup is complete. */
  621. mutex_lock(&ctx->ring_lock);
  622. init_waitqueue_head(&ctx->wait);
  623. INIT_LIST_HEAD(&ctx->active_reqs);
  624. if (percpu_ref_init(&ctx->users, free_ioctx_users, 0, GFP_KERNEL))
  625. goto err;
  626. if (percpu_ref_init(&ctx->reqs, free_ioctx_reqs, 0, GFP_KERNEL))
  627. goto err;
  628. ctx->cpu = alloc_percpu(struct kioctx_cpu);
  629. if (!ctx->cpu)
  630. goto err;
  631. err = aio_setup_ring(ctx, nr_events);
  632. if (err < 0)
  633. goto err;
  634. atomic_set(&ctx->reqs_available, ctx->nr_events - 1);
  635. ctx->req_batch = (ctx->nr_events - 1) / (num_possible_cpus() * 4);
  636. if (ctx->req_batch < 1)
  637. ctx->req_batch = 1;
  638. /* limit the number of system wide aios */
  639. spin_lock(&aio_nr_lock);
  640. if (aio_nr + ctx->max_reqs > aio_max_nr ||
  641. aio_nr + ctx->max_reqs < aio_nr) {
  642. spin_unlock(&aio_nr_lock);
  643. err = -EAGAIN;
  644. goto err_ctx;
  645. }
  646. aio_nr += ctx->max_reqs;
  647. spin_unlock(&aio_nr_lock);
  648. percpu_ref_get(&ctx->users); /* io_setup() will drop this ref */
  649. percpu_ref_get(&ctx->reqs); /* free_ioctx_users() will drop this */
  650. err = ioctx_add_table(ctx, mm);
  651. if (err)
  652. goto err_cleanup;
  653. /* Release the ring_lock mutex now that all setup is complete. */
  654. mutex_unlock(&ctx->ring_lock);
  655. pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
  656. ctx, ctx->user_id, mm, ctx->nr_events);
  657. return ctx;
  658. err_cleanup:
  659. aio_nr_sub(ctx->max_reqs);
  660. err_ctx:
  661. atomic_set(&ctx->dead, 1);
  662. if (ctx->mmap_size)
  663. vm_munmap(ctx->mmap_base, ctx->mmap_size);
  664. aio_free_ring(ctx);
  665. err:
  666. mutex_unlock(&ctx->ring_lock);
  667. free_percpu(ctx->cpu);
  668. percpu_ref_exit(&ctx->reqs);
  669. percpu_ref_exit(&ctx->users);
  670. kmem_cache_free(kioctx_cachep, ctx);
  671. pr_debug("error allocating ioctx %d\n", err);
  672. return ERR_PTR(err);
  673. }
  674. /* kill_ioctx
  675. * Cancels all outstanding aio requests on an aio context. Used
  676. * when the processes owning a context have all exited to encourage
  677. * the rapid destruction of the kioctx.
  678. */
  679. static int kill_ioctx(struct mm_struct *mm, struct kioctx *ctx,
  680. struct ctx_rq_wait *wait)
  681. {
  682. struct kioctx_table *table;
  683. spin_lock(&mm->ioctx_lock);
  684. if (atomic_xchg(&ctx->dead, 1)) {
  685. spin_unlock(&mm->ioctx_lock);
  686. return -EINVAL;
  687. }
  688. table = rcu_dereference_raw(mm->ioctx_table);
  689. WARN_ON(ctx != rcu_access_pointer(table->table[ctx->id]));
  690. RCU_INIT_POINTER(table->table[ctx->id], NULL);
  691. spin_unlock(&mm->ioctx_lock);
  692. /* free_ioctx_reqs() will do the necessary RCU synchronization */
  693. wake_up_all(&ctx->wait);
  694. /*
  695. * It'd be more correct to do this in free_ioctx(), after all
  696. * the outstanding kiocbs have finished - but by then io_destroy
  697. * has already returned, so io_setup() could potentially return
  698. * -EAGAIN with no ioctxs actually in use (as far as userspace
  699. * could tell).
  700. */
  701. aio_nr_sub(ctx->max_reqs);
  702. if (ctx->mmap_size)
  703. vm_munmap(ctx->mmap_base, ctx->mmap_size);
  704. ctx->rq_wait = wait;
  705. percpu_ref_kill(&ctx->users);
  706. return 0;
  707. }
  708. /*
  709. * exit_aio: called when the last user of mm goes away. At this point, there is
  710. * no way for any new requests to be submited or any of the io_* syscalls to be
  711. * called on the context.
  712. *
  713. * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on
  714. * them.
  715. */
  716. void exit_aio(struct mm_struct *mm)
  717. {
  718. struct kioctx_table *table = rcu_dereference_raw(mm->ioctx_table);
  719. struct ctx_rq_wait wait;
  720. int i, skipped;
  721. if (!table)
  722. return;
  723. atomic_set(&wait.count, table->nr);
  724. init_completion(&wait.comp);
  725. skipped = 0;
  726. for (i = 0; i < table->nr; ++i) {
  727. struct kioctx *ctx =
  728. rcu_dereference_protected(table->table[i], true);
  729. if (!ctx) {
  730. skipped++;
  731. continue;
  732. }
  733. /*
  734. * We don't need to bother with munmap() here - exit_mmap(mm)
  735. * is coming and it'll unmap everything. And we simply can't,
  736. * this is not necessarily our ->mm.
  737. * Since kill_ioctx() uses non-zero ->mmap_size as indicator
  738. * that it needs to unmap the area, just set it to 0.
  739. */
  740. ctx->mmap_size = 0;
  741. kill_ioctx(mm, ctx, &wait);
  742. }
  743. if (!atomic_sub_and_test(skipped, &wait.count)) {
  744. /* Wait until all IO for the context are done. */
  745. wait_for_completion(&wait.comp);
  746. }
  747. RCU_INIT_POINTER(mm->ioctx_table, NULL);
  748. kfree(table);
  749. }
  750. static void put_reqs_available(struct kioctx *ctx, unsigned nr)
  751. {
  752. struct kioctx_cpu *kcpu;
  753. unsigned long flags;
  754. local_irq_save(flags);
  755. kcpu = this_cpu_ptr(ctx->cpu);
  756. kcpu->reqs_available += nr;
  757. while (kcpu->reqs_available >= ctx->req_batch * 2) {
  758. kcpu->reqs_available -= ctx->req_batch;
  759. atomic_add(ctx->req_batch, &ctx->reqs_available);
  760. }
  761. local_irq_restore(flags);
  762. }
  763. static bool __get_reqs_available(struct kioctx *ctx)
  764. {
  765. struct kioctx_cpu *kcpu;
  766. bool ret = false;
  767. unsigned long flags;
  768. local_irq_save(flags);
  769. kcpu = this_cpu_ptr(ctx->cpu);
  770. if (!kcpu->reqs_available) {
  771. int old, avail = atomic_read(&ctx->reqs_available);
  772. do {
  773. if (avail < ctx->req_batch)
  774. goto out;
  775. old = avail;
  776. avail = atomic_cmpxchg(&ctx->reqs_available,
  777. avail, avail - ctx->req_batch);
  778. } while (avail != old);
  779. kcpu->reqs_available += ctx->req_batch;
  780. }
  781. ret = true;
  782. kcpu->reqs_available--;
  783. out:
  784. local_irq_restore(flags);
  785. return ret;
  786. }
  787. /* refill_reqs_available
  788. * Updates the reqs_available reference counts used for tracking the
  789. * number of free slots in the completion ring. This can be called
  790. * from aio_complete() (to optimistically update reqs_available) or
  791. * from aio_get_req() (the we're out of events case). It must be
  792. * called holding ctx->completion_lock.
  793. */
  794. static void refill_reqs_available(struct kioctx *ctx, unsigned head,
  795. unsigned tail)
  796. {
  797. unsigned events_in_ring, completed;
  798. /* Clamp head since userland can write to it. */
  799. head %= ctx->nr_events;
  800. if (head <= tail)
  801. events_in_ring = tail - head;
  802. else
  803. events_in_ring = ctx->nr_events - (head - tail);
  804. completed = ctx->completed_events;
  805. if (events_in_ring < completed)
  806. completed -= events_in_ring;
  807. else
  808. completed = 0;
  809. if (!completed)
  810. return;
  811. ctx->completed_events -= completed;
  812. put_reqs_available(ctx, completed);
  813. }
  814. /* user_refill_reqs_available
  815. * Called to refill reqs_available when aio_get_req() encounters an
  816. * out of space in the completion ring.
  817. */
  818. static void user_refill_reqs_available(struct kioctx *ctx)
  819. {
  820. spin_lock_irq(&ctx->completion_lock);
  821. if (ctx->completed_events) {
  822. struct aio_ring *ring;
  823. unsigned head;
  824. /* Access of ring->head may race with aio_read_events_ring()
  825. * here, but that's okay since whether we read the old version
  826. * or the new version, and either will be valid. The important
  827. * part is that head cannot pass tail since we prevent
  828. * aio_complete() from updating tail by holding
  829. * ctx->completion_lock. Even if head is invalid, the check
  830. * against ctx->completed_events below will make sure we do the
  831. * safe/right thing.
  832. */
  833. ring = kmap_atomic(ctx->ring_pages[0]);
  834. head = ring->head;
  835. kunmap_atomic(ring);
  836. refill_reqs_available(ctx, head, ctx->tail);
  837. }
  838. spin_unlock_irq(&ctx->completion_lock);
  839. }
  840. static bool get_reqs_available(struct kioctx *ctx)
  841. {
  842. if (__get_reqs_available(ctx))
  843. return true;
  844. user_refill_reqs_available(ctx);
  845. return __get_reqs_available(ctx);
  846. }
  847. /* aio_get_req
  848. * Allocate a slot for an aio request.
  849. * Returns NULL if no requests are free.
  850. *
  851. * The refcount is initialized to 2 - one for the async op completion,
  852. * one for the synchronous code that does this.
  853. */
  854. static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx)
  855. {
  856. struct aio_kiocb *req;
  857. req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL);
  858. if (unlikely(!req))
  859. return NULL;
  860. if (unlikely(!get_reqs_available(ctx))) {
  861. kmem_cache_free(kiocb_cachep, req);
  862. return NULL;
  863. }
  864. percpu_ref_get(&ctx->reqs);
  865. req->ki_ctx = ctx;
  866. INIT_LIST_HEAD(&req->ki_list);
  867. refcount_set(&req->ki_refcnt, 2);
  868. req->ki_eventfd = NULL;
  869. return req;
  870. }
  871. static struct kioctx *lookup_ioctx(unsigned long ctx_id)
  872. {
  873. struct aio_ring __user *ring = (void __user *)ctx_id;
  874. struct mm_struct *mm = current->mm;
  875. struct kioctx *ctx, *ret = NULL;
  876. struct kioctx_table *table;
  877. unsigned id;
  878. if (get_user(id, &ring->id))
  879. return NULL;
  880. rcu_read_lock();
  881. table = rcu_dereference(mm->ioctx_table);
  882. if (!table || id >= table->nr)
  883. goto out;
  884. id = array_index_nospec(id, table->nr);
  885. ctx = rcu_dereference(table->table[id]);
  886. if (ctx && ctx->user_id == ctx_id) {
  887. if (percpu_ref_tryget_live(&ctx->users))
  888. ret = ctx;
  889. }
  890. out:
  891. rcu_read_unlock();
  892. return ret;
  893. }
  894. static inline void iocb_destroy(struct aio_kiocb *iocb)
  895. {
  896. if (iocb->ki_eventfd)
  897. eventfd_ctx_put(iocb->ki_eventfd);
  898. if (iocb->ki_filp)
  899. fput(iocb->ki_filp);
  900. percpu_ref_put(&iocb->ki_ctx->reqs);
  901. kmem_cache_free(kiocb_cachep, iocb);
  902. }
  903. /* aio_complete
  904. * Called when the io request on the given iocb is complete.
  905. */
  906. static void aio_complete(struct aio_kiocb *iocb)
  907. {
  908. struct kioctx *ctx = iocb->ki_ctx;
  909. struct aio_ring *ring;
  910. struct io_event *ev_page, *event;
  911. unsigned tail, pos, head;
  912. unsigned long flags;
  913. /*
  914. * Add a completion event to the ring buffer. Must be done holding
  915. * ctx->completion_lock to prevent other code from messing with the tail
  916. * pointer since we might be called from irq context.
  917. */
  918. spin_lock_irqsave(&ctx->completion_lock, flags);
  919. tail = ctx->tail;
  920. pos = tail + AIO_EVENTS_OFFSET;
  921. if (++tail >= ctx->nr_events)
  922. tail = 0;
  923. ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
  924. event = ev_page + pos % AIO_EVENTS_PER_PAGE;
  925. *event = iocb->ki_res;
  926. kunmap_atomic(ev_page);
  927. flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
  928. pr_debug("%p[%u]: %p: %p %Lx %Lx %Lx\n", ctx, tail, iocb,
  929. (void __user *)(unsigned long)iocb->ki_res.obj,
  930. iocb->ki_res.data, iocb->ki_res.res, iocb->ki_res.res2);
  931. /* after flagging the request as done, we
  932. * must never even look at it again
  933. */
  934. smp_wmb(); /* make event visible before updating tail */
  935. ctx->tail = tail;
  936. ring = kmap_atomic(ctx->ring_pages[0]);
  937. head = ring->head;
  938. ring->tail = tail;
  939. kunmap_atomic(ring);
  940. flush_dcache_page(ctx->ring_pages[0]);
  941. ctx->completed_events++;
  942. if (ctx->completed_events > 1)
  943. refill_reqs_available(ctx, head, tail);
  944. spin_unlock_irqrestore(&ctx->completion_lock, flags);
  945. pr_debug("added to ring %p at [%u]\n", iocb, tail);
  946. /*
  947. * Check if the user asked us to deliver the result through an
  948. * eventfd. The eventfd_signal() function is safe to be called
  949. * from IRQ context.
  950. */
  951. if (iocb->ki_eventfd)
  952. eventfd_signal(iocb->ki_eventfd, 1);
  953. /*
  954. * We have to order our ring_info tail store above and test
  955. * of the wait list below outside the wait lock. This is
  956. * like in wake_up_bit() where clearing a bit has to be
  957. * ordered with the unlocked test.
  958. */
  959. smp_mb();
  960. if (waitqueue_active(&ctx->wait))
  961. wake_up(&ctx->wait);
  962. }
  963. static inline void iocb_put(struct aio_kiocb *iocb)
  964. {
  965. if (refcount_dec_and_test(&iocb->ki_refcnt)) {
  966. aio_complete(iocb);
  967. iocb_destroy(iocb);
  968. }
  969. }
  970. /* aio_read_events_ring
  971. * Pull an event off of the ioctx's event ring. Returns the number of
  972. * events fetched
  973. */
  974. static long aio_read_events_ring(struct kioctx *ctx,
  975. struct io_event __user *event, long nr)
  976. {
  977. struct aio_ring *ring;
  978. unsigned head, tail, pos;
  979. long ret = 0;
  980. int copy_ret;
  981. /*
  982. * The mutex can block and wake us up and that will cause
  983. * wait_event_interruptible_hrtimeout() to schedule without sleeping
  984. * and repeat. This should be rare enough that it doesn't cause
  985. * peformance issues. See the comment in read_events() for more detail.
  986. */
  987. sched_annotate_sleep();
  988. mutex_lock(&ctx->ring_lock);
  989. /* Access to ->ring_pages here is protected by ctx->ring_lock. */
  990. ring = kmap_atomic(ctx->ring_pages[0]);
  991. head = ring->head;
  992. tail = ring->tail;
  993. kunmap_atomic(ring);
  994. /*
  995. * Ensure that once we've read the current tail pointer, that
  996. * we also see the events that were stored up to the tail.
  997. */
  998. smp_rmb();
  999. pr_debug("h%u t%u m%u\n", head, tail, ctx->nr_events);
  1000. if (head == tail)
  1001. goto out;
  1002. head %= ctx->nr_events;
  1003. tail %= ctx->nr_events;
  1004. while (ret < nr) {
  1005. long avail;
  1006. struct io_event *ev;
  1007. struct page *page;
  1008. avail = (head <= tail ? tail : ctx->nr_events) - head;
  1009. if (head == tail)
  1010. break;
  1011. pos = head + AIO_EVENTS_OFFSET;
  1012. page = ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE];
  1013. pos %= AIO_EVENTS_PER_PAGE;
  1014. avail = min(avail, nr - ret);
  1015. avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos);
  1016. ev = kmap(page);
  1017. copy_ret = copy_to_user(event + ret, ev + pos,
  1018. sizeof(*ev) * avail);
  1019. kunmap(page);
  1020. if (unlikely(copy_ret)) {
  1021. ret = -EFAULT;
  1022. goto out;
  1023. }
  1024. ret += avail;
  1025. head += avail;
  1026. head %= ctx->nr_events;
  1027. }
  1028. ring = kmap_atomic(ctx->ring_pages[0]);
  1029. ring->head = head;
  1030. kunmap_atomic(ring);
  1031. flush_dcache_page(ctx->ring_pages[0]);
  1032. pr_debug("%li h%u t%u\n", ret, head, tail);
  1033. out:
  1034. mutex_unlock(&ctx->ring_lock);
  1035. return ret;
  1036. }
  1037. static bool aio_read_events(struct kioctx *ctx, long min_nr, long nr,
  1038. struct io_event __user *event, long *i)
  1039. {
  1040. long ret = aio_read_events_ring(ctx, event + *i, nr - *i);
  1041. if (ret > 0)
  1042. *i += ret;
  1043. if (unlikely(atomic_read(&ctx->dead)))
  1044. ret = -EINVAL;
  1045. if (!*i)
  1046. *i = ret;
  1047. return ret < 0 || *i >= min_nr;
  1048. }
  1049. static long read_events(struct kioctx *ctx, long min_nr, long nr,
  1050. struct io_event __user *event,
  1051. ktime_t until)
  1052. {
  1053. long ret = 0;
  1054. /*
  1055. * Note that aio_read_events() is being called as the conditional - i.e.
  1056. * we're calling it after prepare_to_wait() has set task state to
  1057. * TASK_INTERRUPTIBLE.
  1058. *
  1059. * But aio_read_events() can block, and if it blocks it's going to flip
  1060. * the task state back to TASK_RUNNING.
  1061. *
  1062. * This should be ok, provided it doesn't flip the state back to
  1063. * TASK_RUNNING and return 0 too much - that causes us to spin. That
  1064. * will only happen if the mutex_lock() call blocks, and we then find
  1065. * the ringbuffer empty. So in practice we should be ok, but it's
  1066. * something to be aware of when touching this code.
  1067. */
  1068. if (until == 0)
  1069. aio_read_events(ctx, min_nr, nr, event, &ret);
  1070. else
  1071. wait_event_interruptible_hrtimeout(ctx->wait,
  1072. aio_read_events(ctx, min_nr, nr, event, &ret),
  1073. until);
  1074. return ret;
  1075. }
  1076. /* sys_io_setup:
  1077. * Create an aio_context capable of receiving at least nr_events.
  1078. * ctxp must not point to an aio_context that already exists, and
  1079. * must be initialized to 0 prior to the call. On successful
  1080. * creation of the aio_context, *ctxp is filled in with the resulting
  1081. * handle. May fail with -EINVAL if *ctxp is not initialized,
  1082. * if the specified nr_events exceeds internal limits. May fail
  1083. * with -EAGAIN if the specified nr_events exceeds the user's limit
  1084. * of available events. May fail with -ENOMEM if insufficient kernel
  1085. * resources are available. May fail with -EFAULT if an invalid
  1086. * pointer is passed for ctxp. Will fail with -ENOSYS if not
  1087. * implemented.
  1088. */
  1089. SYSCALL_DEFINE2(io_setup, unsigned, nr_events, aio_context_t __user *, ctxp)
  1090. {
  1091. struct kioctx *ioctx = NULL;
  1092. unsigned long ctx;
  1093. long ret;
  1094. ret = get_user(ctx, ctxp);
  1095. if (unlikely(ret))
  1096. goto out;
  1097. ret = -EINVAL;
  1098. if (unlikely(ctx || nr_events == 0)) {
  1099. pr_debug("EINVAL: ctx %lu nr_events %u\n",
  1100. ctx, nr_events);
  1101. goto out;
  1102. }
  1103. ioctx = ioctx_alloc(nr_events);
  1104. ret = PTR_ERR(ioctx);
  1105. if (!IS_ERR(ioctx)) {
  1106. ret = put_user(ioctx->user_id, ctxp);
  1107. if (ret)
  1108. kill_ioctx(current->mm, ioctx, NULL);
  1109. percpu_ref_put(&ioctx->users);
  1110. }
  1111. out:
  1112. return ret;
  1113. }
  1114. #ifdef CONFIG_COMPAT
  1115. COMPAT_SYSCALL_DEFINE2(io_setup, unsigned, nr_events, u32 __user *, ctx32p)
  1116. {
  1117. struct kioctx *ioctx = NULL;
  1118. unsigned long ctx;
  1119. long ret;
  1120. ret = get_user(ctx, ctx32p);
  1121. if (unlikely(ret))
  1122. goto out;
  1123. ret = -EINVAL;
  1124. if (unlikely(ctx || nr_events == 0)) {
  1125. pr_debug("EINVAL: ctx %lu nr_events %u\n",
  1126. ctx, nr_events);
  1127. goto out;
  1128. }
  1129. ioctx = ioctx_alloc(nr_events);
  1130. ret = PTR_ERR(ioctx);
  1131. if (!IS_ERR(ioctx)) {
  1132. /* truncating is ok because it's a user address */
  1133. ret = put_user((u32)ioctx->user_id, ctx32p);
  1134. if (ret)
  1135. kill_ioctx(current->mm, ioctx, NULL);
  1136. percpu_ref_put(&ioctx->users);
  1137. }
  1138. out:
  1139. return ret;
  1140. }
  1141. #endif
  1142. /* sys_io_destroy:
  1143. * Destroy the aio_context specified. May cancel any outstanding
  1144. * AIOs and block on completion. Will fail with -ENOSYS if not
  1145. * implemented. May fail with -EINVAL if the context pointed to
  1146. * is invalid.
  1147. */
  1148. SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx)
  1149. {
  1150. struct kioctx *ioctx = lookup_ioctx(ctx);
  1151. if (likely(NULL != ioctx)) {
  1152. struct ctx_rq_wait wait;
  1153. int ret;
  1154. init_completion(&wait.comp);
  1155. atomic_set(&wait.count, 1);
  1156. /* Pass requests_done to kill_ioctx() where it can be set
  1157. * in a thread-safe way. If we try to set it here then we have
  1158. * a race condition if two io_destroy() called simultaneously.
  1159. */
  1160. ret = kill_ioctx(current->mm, ioctx, &wait);
  1161. percpu_ref_put(&ioctx->users);
  1162. /* Wait until all IO for the context are done. Otherwise kernel
  1163. * keep using user-space buffers even if user thinks the context
  1164. * is destroyed.
  1165. */
  1166. if (!ret)
  1167. wait_for_completion(&wait.comp);
  1168. return ret;
  1169. }
  1170. pr_debug("EINVAL: invalid context id\n");
  1171. return -EINVAL;
  1172. }
  1173. static void aio_remove_iocb(struct aio_kiocb *iocb)
  1174. {
  1175. struct kioctx *ctx = iocb->ki_ctx;
  1176. unsigned long flags;
  1177. spin_lock_irqsave(&ctx->ctx_lock, flags);
  1178. list_del(&iocb->ki_list);
  1179. spin_unlock_irqrestore(&ctx->ctx_lock, flags);
  1180. }
  1181. static void aio_complete_rw(struct kiocb *kiocb, long res, long res2)
  1182. {
  1183. struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw);
  1184. if (!list_empty_careful(&iocb->ki_list))
  1185. aio_remove_iocb(iocb);
  1186. if (kiocb->ki_flags & IOCB_WRITE) {
  1187. struct inode *inode = file_inode(kiocb->ki_filp);
  1188. /*
  1189. * Tell lockdep we inherited freeze protection from submission
  1190. * thread.
  1191. */
  1192. if (S_ISREG(inode->i_mode))
  1193. __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
  1194. file_end_write(kiocb->ki_filp);
  1195. }
  1196. iocb->ki_res.res = res;
  1197. iocb->ki_res.res2 = res2;
  1198. iocb_put(iocb);
  1199. }
  1200. static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb)
  1201. {
  1202. int ret;
  1203. req->ki_complete = aio_complete_rw;
  1204. req->private = NULL;
  1205. req->ki_pos = iocb->aio_offset;
  1206. req->ki_flags = iocb_flags(req->ki_filp);
  1207. if (iocb->aio_flags & IOCB_FLAG_RESFD)
  1208. req->ki_flags |= IOCB_EVENTFD;
  1209. req->ki_hint = ki_hint_validate(file_write_hint(req->ki_filp));
  1210. if (iocb->aio_flags & IOCB_FLAG_IOPRIO) {
  1211. /*
  1212. * If the IOCB_FLAG_IOPRIO flag of aio_flags is set, then
  1213. * aio_reqprio is interpreted as an I/O scheduling
  1214. * class and priority.
  1215. */
  1216. ret = ioprio_check_cap(iocb->aio_reqprio);
  1217. if (ret) {
  1218. pr_debug("aio ioprio check cap error: %d\n", ret);
  1219. return ret;
  1220. }
  1221. req->ki_ioprio = iocb->aio_reqprio;
  1222. } else
  1223. req->ki_ioprio = get_current_ioprio();
  1224. ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags);
  1225. if (unlikely(ret))
  1226. return ret;
  1227. req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */
  1228. return 0;
  1229. }
  1230. static ssize_t aio_setup_rw(int rw, const struct iocb *iocb,
  1231. struct iovec **iovec, bool vectored, bool compat,
  1232. struct iov_iter *iter)
  1233. {
  1234. void __user *buf = (void __user *)(uintptr_t)iocb->aio_buf;
  1235. size_t len = iocb->aio_nbytes;
  1236. if (!vectored) {
  1237. ssize_t ret = import_single_range(rw, buf, len, *iovec, iter);
  1238. *iovec = NULL;
  1239. return ret;
  1240. }
  1241. return __import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter, compat);
  1242. }
  1243. static inline void aio_rw_done(struct kiocb *req, ssize_t ret)
  1244. {
  1245. switch (ret) {
  1246. case -EIOCBQUEUED:
  1247. break;
  1248. case -ERESTARTSYS:
  1249. case -ERESTARTNOINTR:
  1250. case -ERESTARTNOHAND:
  1251. case -ERESTART_RESTARTBLOCK:
  1252. /*
  1253. * There's no easy way to restart the syscall since other AIO's
  1254. * may be already running. Just fail this IO with EINTR.
  1255. */
  1256. ret = -EINTR;
  1257. fallthrough;
  1258. default:
  1259. req->ki_complete(req, ret, 0);
  1260. }
  1261. }
  1262. static int aio_read(struct kiocb *req, const struct iocb *iocb,
  1263. bool vectored, bool compat)
  1264. {
  1265. struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
  1266. struct iov_iter iter;
  1267. struct file *file;
  1268. int ret;
  1269. ret = aio_prep_rw(req, iocb);
  1270. if (ret)
  1271. return ret;
  1272. file = req->ki_filp;
  1273. if (unlikely(!(file->f_mode & FMODE_READ)))
  1274. return -EBADF;
  1275. ret = -EINVAL;
  1276. if (unlikely(!file->f_op->read_iter))
  1277. return -EINVAL;
  1278. ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter);
  1279. if (ret < 0)
  1280. return ret;
  1281. ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter));
  1282. if (!ret)
  1283. aio_rw_done(req, call_read_iter(file, req, &iter));
  1284. kfree(iovec);
  1285. return ret;
  1286. }
  1287. static int aio_write(struct kiocb *req, const struct iocb *iocb,
  1288. bool vectored, bool compat)
  1289. {
  1290. struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
  1291. struct iov_iter iter;
  1292. struct file *file;
  1293. int ret;
  1294. ret = aio_prep_rw(req, iocb);
  1295. if (ret)
  1296. return ret;
  1297. file = req->ki_filp;
  1298. if (unlikely(!(file->f_mode & FMODE_WRITE)))
  1299. return -EBADF;
  1300. if (unlikely(!file->f_op->write_iter))
  1301. return -EINVAL;
  1302. ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter);
  1303. if (ret < 0)
  1304. return ret;
  1305. ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter));
  1306. if (!ret) {
  1307. /*
  1308. * Open-code file_start_write here to grab freeze protection,
  1309. * which will be released by another thread in
  1310. * aio_complete_rw(). Fool lockdep by telling it the lock got
  1311. * released so that it doesn't complain about the held lock when
  1312. * we return to userspace.
  1313. */
  1314. if (S_ISREG(file_inode(file)->i_mode)) {
  1315. sb_start_write(file_inode(file)->i_sb);
  1316. __sb_writers_release(file_inode(file)->i_sb, SB_FREEZE_WRITE);
  1317. }
  1318. req->ki_flags |= IOCB_WRITE;
  1319. aio_rw_done(req, call_write_iter(file, req, &iter));
  1320. }
  1321. kfree(iovec);
  1322. return ret;
  1323. }
  1324. static void aio_fsync_work(struct work_struct *work)
  1325. {
  1326. struct aio_kiocb *iocb = container_of(work, struct aio_kiocb, fsync.work);
  1327. const struct cred *old_cred = override_creds(iocb->fsync.creds);
  1328. iocb->ki_res.res = vfs_fsync(iocb->fsync.file, iocb->fsync.datasync);
  1329. revert_creds(old_cred);
  1330. put_cred(iocb->fsync.creds);
  1331. iocb_put(iocb);
  1332. }
  1333. static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb,
  1334. bool datasync)
  1335. {
  1336. if (unlikely(iocb->aio_buf || iocb->aio_offset || iocb->aio_nbytes ||
  1337. iocb->aio_rw_flags))
  1338. return -EINVAL;
  1339. if (unlikely(!req->file->f_op->fsync))
  1340. return -EINVAL;
  1341. req->creds = prepare_creds();
  1342. if (!req->creds)
  1343. return -ENOMEM;
  1344. req->datasync = datasync;
  1345. INIT_WORK(&req->work, aio_fsync_work);
  1346. schedule_work(&req->work);
  1347. return 0;
  1348. }
  1349. static void aio_poll_put_work(struct work_struct *work)
  1350. {
  1351. struct poll_iocb *req = container_of(work, struct poll_iocb, work);
  1352. struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);
  1353. iocb_put(iocb);
  1354. }
  1355. /*
  1356. * Safely lock the waitqueue which the request is on, synchronizing with the
  1357. * case where the ->poll() provider decides to free its waitqueue early.
  1358. *
  1359. * Returns true on success, meaning that req->head->lock was locked, req->wait
  1360. * is on req->head, and an RCU read lock was taken. Returns false if the
  1361. * request was already removed from its waitqueue (which might no longer exist).
  1362. */
  1363. static bool poll_iocb_lock_wq(struct poll_iocb *req)
  1364. {
  1365. wait_queue_head_t *head;
  1366. /*
  1367. * While we hold the waitqueue lock and the waitqueue is nonempty,
  1368. * wake_up_pollfree() will wait for us. However, taking the waitqueue
  1369. * lock in the first place can race with the waitqueue being freed.
  1370. *
  1371. * We solve this as eventpoll does: by taking advantage of the fact that
  1372. * all users of wake_up_pollfree() will RCU-delay the actual free. If
  1373. * we enter rcu_read_lock() and see that the pointer to the queue is
  1374. * non-NULL, we can then lock it without the memory being freed out from
  1375. * under us, then check whether the request is still on the queue.
  1376. *
  1377. * Keep holding rcu_read_lock() as long as we hold the queue lock, in
  1378. * case the caller deletes the entry from the queue, leaving it empty.
  1379. * In that case, only RCU prevents the queue memory from being freed.
  1380. */
  1381. rcu_read_lock();
  1382. head = smp_load_acquire(&req->head);
  1383. if (head) {
  1384. spin_lock(&head->lock);
  1385. if (!list_empty(&req->wait.entry))
  1386. return true;
  1387. spin_unlock(&head->lock);
  1388. }
  1389. rcu_read_unlock();
  1390. return false;
  1391. }
  1392. static void poll_iocb_unlock_wq(struct poll_iocb *req)
  1393. {
  1394. spin_unlock(&req->head->lock);
  1395. rcu_read_unlock();
  1396. }
  1397. static void aio_poll_complete_work(struct work_struct *work)
  1398. {
  1399. struct poll_iocb *req = container_of(work, struct poll_iocb, work);
  1400. struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);
  1401. struct poll_table_struct pt = { ._key = req->events };
  1402. struct kioctx *ctx = iocb->ki_ctx;
  1403. __poll_t mask = 0;
  1404. if (!READ_ONCE(req->cancelled))
  1405. mask = vfs_poll(req->file, &pt) & req->events;
  1406. /*
  1407. * Note that ->ki_cancel callers also delete iocb from active_reqs after
  1408. * calling ->ki_cancel. We need the ctx_lock roundtrip here to
  1409. * synchronize with them. In the cancellation case the list_del_init
  1410. * itself is not actually needed, but harmless so we keep it in to
  1411. * avoid further branches in the fast path.
  1412. */
  1413. spin_lock_irq(&ctx->ctx_lock);
  1414. if (poll_iocb_lock_wq(req)) {
  1415. if (!mask && !READ_ONCE(req->cancelled)) {
  1416. /*
  1417. * The request isn't actually ready to be completed yet.
  1418. * Reschedule completion if another wakeup came in.
  1419. */
  1420. if (req->work_need_resched) {
  1421. schedule_work(&req->work);
  1422. req->work_need_resched = false;
  1423. } else {
  1424. req->work_scheduled = false;
  1425. }
  1426. poll_iocb_unlock_wq(req);
  1427. spin_unlock_irq(&ctx->ctx_lock);
  1428. return;
  1429. }
  1430. list_del_init(&req->wait.entry);
  1431. poll_iocb_unlock_wq(req);
  1432. } /* else, POLLFREE has freed the waitqueue, so we must complete */
  1433. list_del_init(&iocb->ki_list);
  1434. iocb->ki_res.res = mangle_poll(mask);
  1435. spin_unlock_irq(&ctx->ctx_lock);
  1436. iocb_put(iocb);
  1437. }
  1438. /* assumes we are called with irqs disabled */
  1439. static int aio_poll_cancel(struct kiocb *iocb)
  1440. {
  1441. struct aio_kiocb *aiocb = container_of(iocb, struct aio_kiocb, rw);
  1442. struct poll_iocb *req = &aiocb->poll;
  1443. if (poll_iocb_lock_wq(req)) {
  1444. WRITE_ONCE(req->cancelled, true);
  1445. if (!req->work_scheduled) {
  1446. schedule_work(&aiocb->poll.work);
  1447. req->work_scheduled = true;
  1448. }
  1449. poll_iocb_unlock_wq(req);
  1450. } /* else, the request was force-cancelled by POLLFREE already */
  1451. return 0;
  1452. }
  1453. static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
  1454. void *key)
  1455. {
  1456. struct poll_iocb *req = container_of(wait, struct poll_iocb, wait);
  1457. struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);
  1458. __poll_t mask = key_to_poll(key);
  1459. unsigned long flags;
  1460. /* for instances that support it check for an event match first: */
  1461. if (mask && !(mask & req->events))
  1462. return 0;
  1463. /*
  1464. * Complete the request inline if possible. This requires that three
  1465. * conditions be met:
  1466. * 1. An event mask must have been passed. If a plain wakeup was done
  1467. * instead, then mask == 0 and we have to call vfs_poll() to get
  1468. * the events, so inline completion isn't possible.
  1469. * 2. The completion work must not have already been scheduled.
  1470. * 3. ctx_lock must not be busy. We have to use trylock because we
  1471. * already hold the waitqueue lock, so this inverts the normal
  1472. * locking order. Use irqsave/irqrestore because not all
  1473. * filesystems (e.g. fuse) call this function with IRQs disabled,
  1474. * yet IRQs have to be disabled before ctx_lock is obtained.
  1475. */
  1476. if (mask && !req->work_scheduled &&
  1477. spin_trylock_irqsave(&iocb->ki_ctx->ctx_lock, flags)) {
  1478. struct kioctx *ctx = iocb->ki_ctx;
  1479. list_del_init(&req->wait.entry);
  1480. list_del(&iocb->ki_list);
  1481. iocb->ki_res.res = mangle_poll(mask);
  1482. if (iocb->ki_eventfd && eventfd_signal_count()) {
  1483. iocb = NULL;
  1484. INIT_WORK(&req->work, aio_poll_put_work);
  1485. schedule_work(&req->work);
  1486. }
  1487. spin_unlock_irqrestore(&ctx->ctx_lock, flags);
  1488. if (iocb)
  1489. iocb_put(iocb);
  1490. } else {
  1491. /*
  1492. * Schedule the completion work if needed. If it was already
  1493. * scheduled, record that another wakeup came in.
  1494. *
  1495. * Don't remove the request from the waitqueue here, as it might
  1496. * not actually be complete yet (we won't know until vfs_poll()
  1497. * is called), and we must not miss any wakeups. POLLFREE is an
  1498. * exception to this; see below.
  1499. */
  1500. if (req->work_scheduled) {
  1501. req->work_need_resched = true;
  1502. } else {
  1503. schedule_work(&req->work);
  1504. req->work_scheduled = true;
  1505. }
  1506. /*
  1507. * If the waitqueue is being freed early but we can't complete
  1508. * the request inline, we have to tear down the request as best
  1509. * we can. That means immediately removing the request from its
  1510. * waitqueue and preventing all further accesses to the
  1511. * waitqueue via the request. We also need to schedule the
  1512. * completion work (done above). Also mark the request as
  1513. * cancelled, to potentially skip an unneeded call to ->poll().
  1514. */
  1515. if (mask & POLLFREE) {
  1516. WRITE_ONCE(req->cancelled, true);
  1517. list_del_init(&req->wait.entry);
  1518. /*
  1519. * Careful: this *must* be the last step, since as soon
  1520. * as req->head is NULL'ed out, the request can be
  1521. * completed and freed, since aio_poll_complete_work()
  1522. * will no longer need to take the waitqueue lock.
  1523. */
  1524. smp_store_release(&req->head, NULL);
  1525. }
  1526. }
  1527. return 1;
  1528. }
  1529. struct aio_poll_table {
  1530. struct poll_table_struct pt;
  1531. struct aio_kiocb *iocb;
  1532. bool queued;
  1533. int error;
  1534. };
  1535. static void
  1536. aio_poll_queue_proc(struct file *file, struct wait_queue_head *head,
  1537. struct poll_table_struct *p)
  1538. {
  1539. struct aio_poll_table *pt = container_of(p, struct aio_poll_table, pt);
  1540. /* multiple wait queues per file are not supported */
  1541. if (unlikely(pt->queued)) {
  1542. pt->error = -EINVAL;
  1543. return;
  1544. }
  1545. pt->queued = true;
  1546. pt->error = 0;
  1547. pt->iocb->poll.head = head;
  1548. add_wait_queue(head, &pt->iocb->poll.wait);
  1549. }
  1550. static int aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb)
  1551. {
  1552. struct kioctx *ctx = aiocb->ki_ctx;
  1553. struct poll_iocb *req = &aiocb->poll;
  1554. struct aio_poll_table apt;
  1555. bool cancel = false;
  1556. __poll_t mask;
  1557. /* reject any unknown events outside the normal event mask. */
  1558. if ((u16)iocb->aio_buf != iocb->aio_buf)
  1559. return -EINVAL;
  1560. /* reject fields that are not defined for poll */
  1561. if (iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags)
  1562. return -EINVAL;
  1563. INIT_WORK(&req->work, aio_poll_complete_work);
  1564. req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP;
  1565. req->head = NULL;
  1566. req->cancelled = false;
  1567. req->work_scheduled = false;
  1568. req->work_need_resched = false;
  1569. apt.pt._qproc = aio_poll_queue_proc;
  1570. apt.pt._key = req->events;
  1571. apt.iocb = aiocb;
  1572. apt.queued = false;
  1573. apt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
  1574. /* initialized the list so that we can do list_empty checks */
  1575. INIT_LIST_HEAD(&req->wait.entry);
  1576. init_waitqueue_func_entry(&req->wait, aio_poll_wake);
  1577. mask = vfs_poll(req->file, &apt.pt) & req->events;
  1578. spin_lock_irq(&ctx->ctx_lock);
  1579. if (likely(apt.queued)) {
  1580. bool on_queue = poll_iocb_lock_wq(req);
  1581. if (!on_queue || req->work_scheduled) {
  1582. /*
  1583. * aio_poll_wake() already either scheduled the async
  1584. * completion work, or completed the request inline.
  1585. */
  1586. if (apt.error) /* unsupported case: multiple queues */
  1587. cancel = true;
  1588. apt.error = 0;
  1589. mask = 0;
  1590. }
  1591. if (mask || apt.error) {
  1592. /* Steal to complete synchronously. */
  1593. list_del_init(&req->wait.entry);
  1594. } else if (cancel) {
  1595. /* Cancel if possible (may be too late though). */
  1596. WRITE_ONCE(req->cancelled, true);
  1597. } else if (on_queue) {
  1598. /*
  1599. * Actually waiting for an event, so add the request to
  1600. * active_reqs so that it can be cancelled if needed.
  1601. */
  1602. list_add_tail(&aiocb->ki_list, &ctx->active_reqs);
  1603. aiocb->ki_cancel = aio_poll_cancel;
  1604. }
  1605. if (on_queue)
  1606. poll_iocb_unlock_wq(req);
  1607. }
  1608. if (mask) { /* no async, we'd stolen it */
  1609. aiocb->ki_res.res = mangle_poll(mask);
  1610. apt.error = 0;
  1611. }
  1612. spin_unlock_irq(&ctx->ctx_lock);
  1613. if (mask)
  1614. iocb_put(aiocb);
  1615. return apt.error;
  1616. }
  1617. static int __io_submit_one(struct kioctx *ctx, const struct iocb *iocb,
  1618. struct iocb __user *user_iocb, struct aio_kiocb *req,
  1619. bool compat)
  1620. {
  1621. req->ki_filp = fget(iocb->aio_fildes);
  1622. if (unlikely(!req->ki_filp))
  1623. return -EBADF;
  1624. if (iocb->aio_flags & IOCB_FLAG_RESFD) {
  1625. struct eventfd_ctx *eventfd;
  1626. /*
  1627. * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an
  1628. * instance of the file* now. The file descriptor must be
  1629. * an eventfd() fd, and will be signaled for each completed
  1630. * event using the eventfd_signal() function.
  1631. */
  1632. eventfd = eventfd_ctx_fdget(iocb->aio_resfd);
  1633. if (IS_ERR(eventfd))
  1634. return PTR_ERR(eventfd);
  1635. req->ki_eventfd = eventfd;
  1636. }
  1637. if (unlikely(put_user(KIOCB_KEY, &user_iocb->aio_key))) {
  1638. pr_debug("EFAULT: aio_key\n");
  1639. return -EFAULT;
  1640. }
  1641. req->ki_res.obj = (u64)(unsigned long)user_iocb;
  1642. req->ki_res.data = iocb->aio_data;
  1643. req->ki_res.res = 0;
  1644. req->ki_res.res2 = 0;
  1645. switch (iocb->aio_lio_opcode) {
  1646. case IOCB_CMD_PREAD:
  1647. return aio_read(&req->rw, iocb, false, compat);
  1648. case IOCB_CMD_PWRITE:
  1649. return aio_write(&req->rw, iocb, false, compat);
  1650. case IOCB_CMD_PREADV:
  1651. return aio_read(&req->rw, iocb, true, compat);
  1652. case IOCB_CMD_PWRITEV:
  1653. return aio_write(&req->rw, iocb, true, compat);
  1654. case IOCB_CMD_FSYNC:
  1655. return aio_fsync(&req->fsync, iocb, false);
  1656. case IOCB_CMD_FDSYNC:
  1657. return aio_fsync(&req->fsync, iocb, true);
  1658. case IOCB_CMD_POLL:
  1659. return aio_poll(req, iocb);
  1660. default:
  1661. pr_debug("invalid aio operation %d\n", iocb->aio_lio_opcode);
  1662. return -EINVAL;
  1663. }
  1664. }
  1665. static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
  1666. bool compat)
  1667. {
  1668. struct aio_kiocb *req;
  1669. struct iocb iocb;
  1670. int err;
  1671. if (unlikely(copy_from_user(&iocb, user_iocb, sizeof(iocb))))
  1672. return -EFAULT;
  1673. /* enforce forwards compatibility on users */
  1674. if (unlikely(iocb.aio_reserved2)) {
  1675. pr_debug("EINVAL: reserve field set\n");
  1676. return -EINVAL;
  1677. }
  1678. /* prevent overflows */
  1679. if (unlikely(
  1680. (iocb.aio_buf != (unsigned long)iocb.aio_buf) ||
  1681. (iocb.aio_nbytes != (size_t)iocb.aio_nbytes) ||
  1682. ((ssize_t)iocb.aio_nbytes < 0)
  1683. )) {
  1684. pr_debug("EINVAL: overflow check\n");
  1685. return -EINVAL;
  1686. }
  1687. req = aio_get_req(ctx);
  1688. if (unlikely(!req))
  1689. return -EAGAIN;
  1690. err = __io_submit_one(ctx, &iocb, user_iocb, req, compat);
  1691. /* Done with the synchronous reference */
  1692. iocb_put(req);
  1693. /*
  1694. * If err is 0, we'd either done aio_complete() ourselves or have
  1695. * arranged for that to be done asynchronously. Anything non-zero
  1696. * means that we need to destroy req ourselves.
  1697. */
  1698. if (unlikely(err)) {
  1699. iocb_destroy(req);
  1700. put_reqs_available(ctx, 1);
  1701. }
  1702. return err;
  1703. }
  1704. /* sys_io_submit:
  1705. * Queue the nr iocbs pointed to by iocbpp for processing. Returns
  1706. * the number of iocbs queued. May return -EINVAL if the aio_context
  1707. * specified by ctx_id is invalid, if nr is < 0, if the iocb at
  1708. * *iocbpp[0] is not properly initialized, if the operation specified
  1709. * is invalid for the file descriptor in the iocb. May fail with
  1710. * -EFAULT if any of the data structures point to invalid data. May
  1711. * fail with -EBADF if the file descriptor specified in the first
  1712. * iocb is invalid. May fail with -EAGAIN if insufficient resources
  1713. * are available to queue any iocbs. Will return 0 if nr is 0. Will
  1714. * fail with -ENOSYS if not implemented.
  1715. */
  1716. SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr,
  1717. struct iocb __user * __user *, iocbpp)
  1718. {
  1719. struct kioctx *ctx;
  1720. long ret = 0;
  1721. int i = 0;
  1722. struct blk_plug plug;
  1723. if (unlikely(nr < 0))
  1724. return -EINVAL;
  1725. ctx = lookup_ioctx(ctx_id);
  1726. if (unlikely(!ctx)) {
  1727. pr_debug("EINVAL: invalid context id\n");
  1728. return -EINVAL;
  1729. }
  1730. if (nr > ctx->nr_events)
  1731. nr = ctx->nr_events;
  1732. if (nr > AIO_PLUG_THRESHOLD)
  1733. blk_start_plug(&plug);
  1734. for (i = 0; i < nr; i++) {
  1735. struct iocb __user *user_iocb;
  1736. if (unlikely(get_user(user_iocb, iocbpp + i))) {
  1737. ret = -EFAULT;
  1738. break;
  1739. }
  1740. ret = io_submit_one(ctx, user_iocb, false);
  1741. if (ret)
  1742. break;
  1743. }
  1744. if (nr > AIO_PLUG_THRESHOLD)
  1745. blk_finish_plug(&plug);
  1746. percpu_ref_put(&ctx->users);
  1747. return i ? i : ret;
  1748. }
  1749. #ifdef CONFIG_COMPAT
  1750. COMPAT_SYSCALL_DEFINE3(io_submit, compat_aio_context_t, ctx_id,
  1751. int, nr, compat_uptr_t __user *, iocbpp)
  1752. {
  1753. struct kioctx *ctx;
  1754. long ret = 0;
  1755. int i = 0;
  1756. struct blk_plug plug;
  1757. if (unlikely(nr < 0))
  1758. return -EINVAL;
  1759. ctx = lookup_ioctx(ctx_id);
  1760. if (unlikely(!ctx)) {
  1761. pr_debug("EINVAL: invalid context id\n");
  1762. return -EINVAL;
  1763. }
  1764. if (nr > ctx->nr_events)
  1765. nr = ctx->nr_events;
  1766. if (nr > AIO_PLUG_THRESHOLD)
  1767. blk_start_plug(&plug);
  1768. for (i = 0; i < nr; i++) {
  1769. compat_uptr_t user_iocb;
  1770. if (unlikely(get_user(user_iocb, iocbpp + i))) {
  1771. ret = -EFAULT;
  1772. break;
  1773. }
  1774. ret = io_submit_one(ctx, compat_ptr(user_iocb), true);
  1775. if (ret)
  1776. break;
  1777. }
  1778. if (nr > AIO_PLUG_THRESHOLD)
  1779. blk_finish_plug(&plug);
  1780. percpu_ref_put(&ctx->users);
  1781. return i ? i : ret;
  1782. }
  1783. #endif
  1784. /* sys_io_cancel:
  1785. * Attempts to cancel an iocb previously passed to io_submit. If
  1786. * the operation is successfully cancelled, the resulting event is
  1787. * copied into the memory pointed to by result without being placed
  1788. * into the completion queue and 0 is returned. May fail with
  1789. * -EFAULT if any of the data structures pointed to are invalid.
  1790. * May fail with -EINVAL if aio_context specified by ctx_id is
  1791. * invalid. May fail with -EAGAIN if the iocb specified was not
  1792. * cancelled. Will fail with -ENOSYS if not implemented.
  1793. */
  1794. SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb,
  1795. struct io_event __user *, result)
  1796. {
  1797. struct kioctx *ctx;
  1798. struct aio_kiocb *kiocb;
  1799. int ret = -EINVAL;
  1800. u32 key;
  1801. u64 obj = (u64)(unsigned long)iocb;
  1802. if (unlikely(get_user(key, &iocb->aio_key)))
  1803. return -EFAULT;
  1804. if (unlikely(key != KIOCB_KEY))
  1805. return -EINVAL;
  1806. ctx = lookup_ioctx(ctx_id);
  1807. if (unlikely(!ctx))
  1808. return -EINVAL;
  1809. spin_lock_irq(&ctx->ctx_lock);
  1810. /* TODO: use a hash or array, this sucks. */
  1811. list_for_each_entry(kiocb, &ctx->active_reqs, ki_list) {
  1812. if (kiocb->ki_res.obj == obj) {
  1813. ret = kiocb->ki_cancel(&kiocb->rw);
  1814. list_del_init(&kiocb->ki_list);
  1815. break;
  1816. }
  1817. }
  1818. spin_unlock_irq(&ctx->ctx_lock);
  1819. if (!ret) {
  1820. /*
  1821. * The result argument is no longer used - the io_event is
  1822. * always delivered via the ring buffer. -EINPROGRESS indicates
  1823. * cancellation is progress:
  1824. */
  1825. ret = -EINPROGRESS;
  1826. }
  1827. percpu_ref_put(&ctx->users);
  1828. return ret;
  1829. }
  1830. static long do_io_getevents(aio_context_t ctx_id,
  1831. long min_nr,
  1832. long nr,
  1833. struct io_event __user *events,
  1834. struct timespec64 *ts)
  1835. {
  1836. ktime_t until = ts ? timespec64_to_ktime(*ts) : KTIME_MAX;
  1837. struct kioctx *ioctx = lookup_ioctx(ctx_id);
  1838. long ret = -EINVAL;
  1839. if (likely(ioctx)) {
  1840. if (likely(min_nr <= nr && min_nr >= 0))
  1841. ret = read_events(ioctx, min_nr, nr, events, until);
  1842. percpu_ref_put(&ioctx->users);
  1843. }
  1844. return ret;
  1845. }
  1846. /* io_getevents:
  1847. * Attempts to read at least min_nr events and up to nr events from
  1848. * the completion queue for the aio_context specified by ctx_id. If
  1849. * it succeeds, the number of read events is returned. May fail with
  1850. * -EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is
  1851. * out of range, if timeout is out of range. May fail with -EFAULT
  1852. * if any of the memory specified is invalid. May return 0 or
  1853. * < min_nr if the timeout specified by timeout has elapsed
  1854. * before sufficient events are available, where timeout == NULL
  1855. * specifies an infinite timeout. Note that the timeout pointed to by
  1856. * timeout is relative. Will fail with -ENOSYS if not implemented.
  1857. */
  1858. #ifdef CONFIG_64BIT
  1859. SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id,
  1860. long, min_nr,
  1861. long, nr,
  1862. struct io_event __user *, events,
  1863. struct __kernel_timespec __user *, timeout)
  1864. {
  1865. struct timespec64 ts;
  1866. int ret;
  1867. if (timeout && unlikely(get_timespec64(&ts, timeout)))
  1868. return -EFAULT;
  1869. ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL);
  1870. if (!ret && signal_pending(current))
  1871. ret = -EINTR;
  1872. return ret;
  1873. }
  1874. #endif
  1875. struct __aio_sigset {
  1876. const sigset_t __user *sigmask;
  1877. size_t sigsetsize;
  1878. };
  1879. SYSCALL_DEFINE6(io_pgetevents,
  1880. aio_context_t, ctx_id,
  1881. long, min_nr,
  1882. long, nr,
  1883. struct io_event __user *, events,
  1884. struct __kernel_timespec __user *, timeout,
  1885. const struct __aio_sigset __user *, usig)
  1886. {
  1887. struct __aio_sigset ksig = { NULL, };
  1888. struct timespec64 ts;
  1889. bool interrupted;
  1890. int ret;
  1891. if (timeout && unlikely(get_timespec64(&ts, timeout)))
  1892. return -EFAULT;
  1893. if (usig && copy_from_user(&ksig, usig, sizeof(ksig)))
  1894. return -EFAULT;
  1895. ret = set_user_sigmask(ksig.sigmask, ksig.sigsetsize);
  1896. if (ret)
  1897. return ret;
  1898. ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL);
  1899. interrupted = signal_pending(current);
  1900. restore_saved_sigmask_unless(interrupted);
  1901. if (interrupted && !ret)
  1902. ret = -ERESTARTNOHAND;
  1903. return ret;
  1904. }
  1905. #if defined(CONFIG_COMPAT_32BIT_TIME) && !defined(CONFIG_64BIT)
  1906. SYSCALL_DEFINE6(io_pgetevents_time32,
  1907. aio_context_t, ctx_id,
  1908. long, min_nr,
  1909. long, nr,
  1910. struct io_event __user *, events,
  1911. struct old_timespec32 __user *, timeout,
  1912. const struct __aio_sigset __user *, usig)
  1913. {
  1914. struct __aio_sigset ksig = { NULL, };
  1915. struct timespec64 ts;
  1916. bool interrupted;
  1917. int ret;
  1918. if (timeout && unlikely(get_old_timespec32(&ts, timeout)))
  1919. return -EFAULT;
  1920. if (usig && copy_from_user(&ksig, usig, sizeof(ksig)))
  1921. return -EFAULT;
  1922. ret = set_user_sigmask(ksig.sigmask, ksig.sigsetsize);
  1923. if (ret)
  1924. return ret;
  1925. ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL);
  1926. interrupted = signal_pending(current);
  1927. restore_saved_sigmask_unless(interrupted);
  1928. if (interrupted && !ret)
  1929. ret = -ERESTARTNOHAND;
  1930. return ret;
  1931. }
  1932. #endif
  1933. #if defined(CONFIG_COMPAT_32BIT_TIME)
  1934. SYSCALL_DEFINE5(io_getevents_time32, __u32, ctx_id,
  1935. __s32, min_nr,
  1936. __s32, nr,
  1937. struct io_event __user *, events,
  1938. struct old_timespec32 __user *, timeout)
  1939. {
  1940. struct timespec64 t;
  1941. int ret;
  1942. if (timeout && get_old_timespec32(&t, timeout))
  1943. return -EFAULT;
  1944. ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL);
  1945. if (!ret && signal_pending(current))
  1946. ret = -EINTR;
  1947. return ret;
  1948. }
  1949. #endif
  1950. #ifdef CONFIG_COMPAT
  1951. struct __compat_aio_sigset {
  1952. compat_uptr_t sigmask;
  1953. compat_size_t sigsetsize;
  1954. };
  1955. #if defined(CONFIG_COMPAT_32BIT_TIME)
  1956. COMPAT_SYSCALL_DEFINE6(io_pgetevents,
  1957. compat_aio_context_t, ctx_id,
  1958. compat_long_t, min_nr,
  1959. compat_long_t, nr,
  1960. struct io_event __user *, events,
  1961. struct old_timespec32 __user *, timeout,
  1962. const struct __compat_aio_sigset __user *, usig)
  1963. {
  1964. struct __compat_aio_sigset ksig = { 0, };
  1965. struct timespec64 t;
  1966. bool interrupted;
  1967. int ret;
  1968. if (timeout && get_old_timespec32(&t, timeout))
  1969. return -EFAULT;
  1970. if (usig && copy_from_user(&ksig, usig, sizeof(ksig)))
  1971. return -EFAULT;
  1972. ret = set_compat_user_sigmask(compat_ptr(ksig.sigmask), ksig.sigsetsize);
  1973. if (ret)
  1974. return ret;
  1975. ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL);
  1976. interrupted = signal_pending(current);
  1977. restore_saved_sigmask_unless(interrupted);
  1978. if (interrupted && !ret)
  1979. ret = -ERESTARTNOHAND;
  1980. return ret;
  1981. }
  1982. #endif
  1983. COMPAT_SYSCALL_DEFINE6(io_pgetevents_time64,
  1984. compat_aio_context_t, ctx_id,
  1985. compat_long_t, min_nr,
  1986. compat_long_t, nr,
  1987. struct io_event __user *, events,
  1988. struct __kernel_timespec __user *, timeout,
  1989. const struct __compat_aio_sigset __user *, usig)
  1990. {
  1991. struct __compat_aio_sigset ksig = { 0, };
  1992. struct timespec64 t;
  1993. bool interrupted;
  1994. int ret;
  1995. if (timeout && get_timespec64(&t, timeout))
  1996. return -EFAULT;
  1997. if (usig && copy_from_user(&ksig, usig, sizeof(ksig)))
  1998. return -EFAULT;
  1999. ret = set_compat_user_sigmask(compat_ptr(ksig.sigmask), ksig.sigsetsize);
  2000. if (ret)
  2001. return ret;
  2002. ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL);
  2003. interrupted = signal_pending(current);
  2004. restore_saved_sigmask_unless(interrupted);
  2005. if (interrupted && !ret)
  2006. ret = -ERESTARTNOHAND;
  2007. return ret;
  2008. }
  2009. #endif