read_write.c 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * linux/fs/read_write.c
  4. *
  5. * Copyright (C) 1991, 1992 Linus Torvalds
  6. */
  7. #include <linux/slab.h>
  8. #include <linux/stat.h>
  9. #include <linux/sched/xacct.h>
  10. #include <linux/fcntl.h>
  11. #include <linux/file.h>
  12. #include <linux/uio.h>
  13. #include <linux/fsnotify.h>
  14. #include <linux/security.h>
  15. #include <linux/export.h>
  16. #include <linux/syscalls.h>
  17. #include <linux/pagemap.h>
  18. #include <linux/splice.h>
  19. #include <linux/compat.h>
  20. #include <linux/mount.h>
  21. #include <linux/fs.h>
  22. #include "internal.h"
  23. #include <linux/uaccess.h>
  24. #include <asm/unistd.h>
  25. const struct file_operations generic_ro_fops = {
  26. .llseek = generic_file_llseek,
  27. .read_iter = generic_file_read_iter,
  28. .mmap = generic_file_readonly_mmap,
  29. .splice_read = generic_file_splice_read,
  30. };
  31. EXPORT_SYMBOL(generic_ro_fops);
  32. static inline bool unsigned_offsets(struct file *file)
  33. {
  34. return file->f_mode & FMODE_UNSIGNED_OFFSET;
  35. }
  36. /**
  37. * vfs_setpos - update the file offset for lseek
  38. * @file: file structure in question
  39. * @offset: file offset to seek to
  40. * @maxsize: maximum file size
  41. *
  42. * This is a low-level filesystem helper for updating the file offset to
  43. * the value specified by @offset if the given offset is valid and it is
  44. * not equal to the current file offset.
  45. *
  46. * Return the specified offset on success and -EINVAL on invalid offset.
  47. */
  48. loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
  49. {
  50. if (offset < 0 && !unsigned_offsets(file))
  51. return -EINVAL;
  52. if (offset > maxsize)
  53. return -EINVAL;
  54. if (offset != file->f_pos) {
  55. file->f_pos = offset;
  56. file->f_version = 0;
  57. }
  58. return offset;
  59. }
  60. EXPORT_SYMBOL(vfs_setpos);
  61. /**
  62. * generic_file_llseek_size - generic llseek implementation for regular files
  63. * @file: file structure to seek on
  64. * @offset: file offset to seek to
  65. * @whence: type of seek
  66. * @size: max size of this file in file system
  67. * @eof: offset used for SEEK_END position
  68. *
  69. * This is a variant of generic_file_llseek that allows passing in a custom
  70. * maximum file size and a custom EOF position, for e.g. hashed directories
  71. *
  72. * Synchronization:
  73. * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
  74. * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
  75. * read/writes behave like SEEK_SET against seeks.
  76. */
  77. loff_t
  78. generic_file_llseek_size(struct file *file, loff_t offset, int whence,
  79. loff_t maxsize, loff_t eof)
  80. {
  81. switch (whence) {
  82. case SEEK_END:
  83. offset += eof;
  84. break;
  85. case SEEK_CUR:
  86. /*
  87. * Here we special-case the lseek(fd, 0, SEEK_CUR)
  88. * position-querying operation. Avoid rewriting the "same"
  89. * f_pos value back to the file because a concurrent read(),
  90. * write() or lseek() might have altered it
  91. */
  92. if (offset == 0)
  93. return file->f_pos;
  94. /*
  95. * f_lock protects against read/modify/write race with other
  96. * SEEK_CURs. Note that parallel writes and reads behave
  97. * like SEEK_SET.
  98. */
  99. spin_lock(&file->f_lock);
  100. offset = vfs_setpos(file, file->f_pos + offset, maxsize);
  101. spin_unlock(&file->f_lock);
  102. return offset;
  103. case SEEK_DATA:
  104. /*
  105. * In the generic case the entire file is data, so as long as
  106. * offset isn't at the end of the file then the offset is data.
  107. */
  108. if ((unsigned long long)offset >= eof)
  109. return -ENXIO;
  110. break;
  111. case SEEK_HOLE:
  112. /*
  113. * There is a virtual hole at the end of the file, so as long as
  114. * offset isn't i_size or larger, return i_size.
  115. */
  116. if ((unsigned long long)offset >= eof)
  117. return -ENXIO;
  118. offset = eof;
  119. break;
  120. }
  121. return vfs_setpos(file, offset, maxsize);
  122. }
  123. EXPORT_SYMBOL(generic_file_llseek_size);
  124. /**
  125. * generic_file_llseek - generic llseek implementation for regular files
  126. * @file: file structure to seek on
  127. * @offset: file offset to seek to
  128. * @whence: type of seek
  129. *
  130. * This is a generic implemenation of ->llseek useable for all normal local
  131. * filesystems. It just updates the file offset to the value specified by
  132. * @offset and @whence.
  133. */
  134. loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
  135. {
  136. struct inode *inode = file->f_mapping->host;
  137. return generic_file_llseek_size(file, offset, whence,
  138. inode->i_sb->s_maxbytes,
  139. i_size_read(inode));
  140. }
  141. EXPORT_SYMBOL(generic_file_llseek);
  142. /**
  143. * fixed_size_llseek - llseek implementation for fixed-sized devices
  144. * @file: file structure to seek on
  145. * @offset: file offset to seek to
  146. * @whence: type of seek
  147. * @size: size of the file
  148. *
  149. */
  150. loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
  151. {
  152. switch (whence) {
  153. case SEEK_SET: case SEEK_CUR: case SEEK_END:
  154. return generic_file_llseek_size(file, offset, whence,
  155. size, size);
  156. default:
  157. return -EINVAL;
  158. }
  159. }
  160. EXPORT_SYMBOL(fixed_size_llseek);
  161. /**
  162. * no_seek_end_llseek - llseek implementation for fixed-sized devices
  163. * @file: file structure to seek on
  164. * @offset: file offset to seek to
  165. * @whence: type of seek
  166. *
  167. */
  168. loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence)
  169. {
  170. switch (whence) {
  171. case SEEK_SET: case SEEK_CUR:
  172. return generic_file_llseek_size(file, offset, whence,
  173. OFFSET_MAX, 0);
  174. default:
  175. return -EINVAL;
  176. }
  177. }
  178. EXPORT_SYMBOL(no_seek_end_llseek);
  179. /**
  180. * no_seek_end_llseek_size - llseek implementation for fixed-sized devices
  181. * @file: file structure to seek on
  182. * @offset: file offset to seek to
  183. * @whence: type of seek
  184. * @size: maximal offset allowed
  185. *
  186. */
  187. loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size)
  188. {
  189. switch (whence) {
  190. case SEEK_SET: case SEEK_CUR:
  191. return generic_file_llseek_size(file, offset, whence,
  192. size, 0);
  193. default:
  194. return -EINVAL;
  195. }
  196. }
  197. EXPORT_SYMBOL(no_seek_end_llseek_size);
  198. /**
  199. * noop_llseek - No Operation Performed llseek implementation
  200. * @file: file structure to seek on
  201. * @offset: file offset to seek to
  202. * @whence: type of seek
  203. *
  204. * This is an implementation of ->llseek useable for the rare special case when
  205. * userspace expects the seek to succeed but the (device) file is actually not
  206. * able to perform the seek. In this case you use noop_llseek() instead of
  207. * falling back to the default implementation of ->llseek.
  208. */
  209. loff_t noop_llseek(struct file *file, loff_t offset, int whence)
  210. {
  211. return file->f_pos;
  212. }
  213. EXPORT_SYMBOL(noop_llseek);
  214. loff_t no_llseek(struct file *file, loff_t offset, int whence)
  215. {
  216. return -ESPIPE;
  217. }
  218. EXPORT_SYMBOL(no_llseek);
  219. loff_t default_llseek(struct file *file, loff_t offset, int whence)
  220. {
  221. struct inode *inode = file_inode(file);
  222. loff_t retval;
  223. inode_lock(inode);
  224. switch (whence) {
  225. case SEEK_END:
  226. offset += i_size_read(inode);
  227. break;
  228. case SEEK_CUR:
  229. if (offset == 0) {
  230. retval = file->f_pos;
  231. goto out;
  232. }
  233. offset += file->f_pos;
  234. break;
  235. case SEEK_DATA:
  236. /*
  237. * In the generic case the entire file is data, so as
  238. * long as offset isn't at the end of the file then the
  239. * offset is data.
  240. */
  241. if (offset >= inode->i_size) {
  242. retval = -ENXIO;
  243. goto out;
  244. }
  245. break;
  246. case SEEK_HOLE:
  247. /*
  248. * There is a virtual hole at the end of the file, so
  249. * as long as offset isn't i_size or larger, return
  250. * i_size.
  251. */
  252. if (offset >= inode->i_size) {
  253. retval = -ENXIO;
  254. goto out;
  255. }
  256. offset = inode->i_size;
  257. break;
  258. }
  259. retval = -EINVAL;
  260. if (offset >= 0 || unsigned_offsets(file)) {
  261. if (offset != file->f_pos) {
  262. file->f_pos = offset;
  263. file->f_version = 0;
  264. }
  265. retval = offset;
  266. }
  267. out:
  268. inode_unlock(inode);
  269. return retval;
  270. }
  271. EXPORT_SYMBOL(default_llseek);
  272. loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
  273. {
  274. loff_t (*fn)(struct file *, loff_t, int);
  275. fn = no_llseek;
  276. if (file->f_mode & FMODE_LSEEK) {
  277. if (file->f_op->llseek)
  278. fn = file->f_op->llseek;
  279. }
  280. return fn(file, offset, whence);
  281. }
  282. EXPORT_SYMBOL(vfs_llseek);
  283. static off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence)
  284. {
  285. off_t retval;
  286. struct fd f = fdget_pos(fd);
  287. if (!f.file)
  288. return -EBADF;
  289. retval = -EINVAL;
  290. if (whence <= SEEK_MAX) {
  291. loff_t res = vfs_llseek(f.file, offset, whence);
  292. retval = res;
  293. if (res != (loff_t)retval)
  294. retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
  295. }
  296. fdput_pos(f);
  297. return retval;
  298. }
  299. SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
  300. {
  301. return ksys_lseek(fd, offset, whence);
  302. }
  303. #ifdef CONFIG_COMPAT
  304. COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
  305. {
  306. return ksys_lseek(fd, offset, whence);
  307. }
  308. #endif
  309. #if !defined(CONFIG_64BIT) || defined(CONFIG_COMPAT) || \
  310. defined(__ARCH_WANT_SYS_LLSEEK)
  311. SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
  312. unsigned long, offset_low, loff_t __user *, result,
  313. unsigned int, whence)
  314. {
  315. int retval;
  316. struct fd f = fdget_pos(fd);
  317. loff_t offset;
  318. if (!f.file)
  319. return -EBADF;
  320. retval = -EINVAL;
  321. if (whence > SEEK_MAX)
  322. goto out_putf;
  323. offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
  324. whence);
  325. retval = (int)offset;
  326. if (offset >= 0) {
  327. retval = -EFAULT;
  328. if (!copy_to_user(result, &offset, sizeof(offset)))
  329. retval = 0;
  330. }
  331. out_putf:
  332. fdput_pos(f);
  333. return retval;
  334. }
  335. #endif
  336. int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
  337. {
  338. struct inode *inode;
  339. int retval = -EINVAL;
  340. inode = file_inode(file);
  341. if (unlikely((ssize_t) count < 0))
  342. return retval;
  343. /*
  344. * ranged mandatory locking does not apply to streams - it makes sense
  345. * only for files where position has a meaning.
  346. */
  347. if (ppos) {
  348. loff_t pos = *ppos;
  349. if (unlikely(pos < 0)) {
  350. if (!unsigned_offsets(file))
  351. return retval;
  352. if (count >= -pos) /* both values are in 0..LLONG_MAX */
  353. return -EOVERFLOW;
  354. } else if (unlikely((loff_t) (pos + count) < 0)) {
  355. if (!unsigned_offsets(file))
  356. return retval;
  357. }
  358. if (unlikely(inode->i_flctx && mandatory_lock(inode))) {
  359. retval = locks_mandatory_area(inode, file, pos, pos + count - 1,
  360. read_write == READ ? F_RDLCK : F_WRLCK);
  361. if (retval < 0)
  362. return retval;
  363. }
  364. }
  365. return security_file_permission(file,
  366. read_write == READ ? MAY_READ : MAY_WRITE);
  367. }
  368. static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
  369. {
  370. struct iovec iov = { .iov_base = buf, .iov_len = len };
  371. struct kiocb kiocb;
  372. struct iov_iter iter;
  373. ssize_t ret;
  374. init_sync_kiocb(&kiocb, filp);
  375. kiocb.ki_pos = (ppos ? *ppos : 0);
  376. iov_iter_init(&iter, READ, &iov, 1, len);
  377. ret = call_read_iter(filp, &kiocb, &iter);
  378. BUG_ON(ret == -EIOCBQUEUED);
  379. if (ppos)
  380. *ppos = kiocb.ki_pos;
  381. return ret;
  382. }
  383. static int warn_unsupported(struct file *file, const char *op)
  384. {
  385. pr_warn_ratelimited(
  386. "kernel %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
  387. op, file, current->pid, current->comm);
  388. return -EINVAL;
  389. }
  390. ssize_t __kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
  391. {
  392. struct kvec iov = {
  393. .iov_base = buf,
  394. .iov_len = min_t(size_t, count, MAX_RW_COUNT),
  395. };
  396. struct kiocb kiocb;
  397. struct iov_iter iter;
  398. ssize_t ret;
  399. if (WARN_ON_ONCE(!(file->f_mode & FMODE_READ)))
  400. return -EINVAL;
  401. if (!(file->f_mode & FMODE_CAN_READ))
  402. return -EINVAL;
  403. /*
  404. * Also fail if ->read_iter and ->read are both wired up as that
  405. * implies very convoluted semantics.
  406. */
  407. if (unlikely(!file->f_op->read_iter || file->f_op->read))
  408. return warn_unsupported(file, "read");
  409. init_sync_kiocb(&kiocb, file);
  410. kiocb.ki_pos = pos ? *pos : 0;
  411. iov_iter_kvec(&iter, READ, &iov, 1, iov.iov_len);
  412. ret = file->f_op->read_iter(&kiocb, &iter);
  413. if (ret > 0) {
  414. if (pos)
  415. *pos = kiocb.ki_pos;
  416. fsnotify_access(file);
  417. add_rchar(current, ret);
  418. }
  419. inc_syscr(current);
  420. return ret;
  421. }
  422. ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
  423. {
  424. ssize_t ret;
  425. ret = rw_verify_area(READ, file, pos, count);
  426. if (ret)
  427. return ret;
  428. return __kernel_read(file, buf, count, pos);
  429. }
  430. EXPORT_SYMBOL_NS(kernel_read, ANDROID_GKI_VFS_EXPORT_ONLY);
  431. ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
  432. {
  433. ssize_t ret;
  434. if (!(file->f_mode & FMODE_READ))
  435. return -EBADF;
  436. if (!(file->f_mode & FMODE_CAN_READ))
  437. return -EINVAL;
  438. if (unlikely(!access_ok(buf, count)))
  439. return -EFAULT;
  440. ret = rw_verify_area(READ, file, pos, count);
  441. if (ret)
  442. return ret;
  443. if (count > MAX_RW_COUNT)
  444. count = MAX_RW_COUNT;
  445. if (file->f_op->read)
  446. ret = file->f_op->read(file, buf, count, pos);
  447. else if (file->f_op->read_iter)
  448. ret = new_sync_read(file, buf, count, pos);
  449. else
  450. ret = -EINVAL;
  451. if (ret > 0) {
  452. fsnotify_access(file);
  453. add_rchar(current, ret);
  454. }
  455. inc_syscr(current);
  456. return ret;
  457. }
  458. static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
  459. {
  460. struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
  461. struct kiocb kiocb;
  462. struct iov_iter iter;
  463. ssize_t ret;
  464. init_sync_kiocb(&kiocb, filp);
  465. kiocb.ki_pos = (ppos ? *ppos : 0);
  466. iov_iter_init(&iter, WRITE, &iov, 1, len);
  467. ret = call_write_iter(filp, &kiocb, &iter);
  468. BUG_ON(ret == -EIOCBQUEUED);
  469. if (ret > 0 && ppos)
  470. *ppos = kiocb.ki_pos;
  471. return ret;
  472. }
  473. /* caller is responsible for file_start_write/file_end_write */
  474. ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos)
  475. {
  476. struct kvec iov = {
  477. .iov_base = (void *)buf,
  478. .iov_len = min_t(size_t, count, MAX_RW_COUNT),
  479. };
  480. struct kiocb kiocb;
  481. struct iov_iter iter;
  482. ssize_t ret;
  483. if (WARN_ON_ONCE(!(file->f_mode & FMODE_WRITE)))
  484. return -EBADF;
  485. if (!(file->f_mode & FMODE_CAN_WRITE))
  486. return -EINVAL;
  487. /*
  488. * Also fail if ->write_iter and ->write are both wired up as that
  489. * implies very convoluted semantics.
  490. */
  491. if (unlikely(!file->f_op->write_iter || file->f_op->write))
  492. return warn_unsupported(file, "write");
  493. init_sync_kiocb(&kiocb, file);
  494. kiocb.ki_pos = pos ? *pos : 0;
  495. iov_iter_kvec(&iter, WRITE, &iov, 1, iov.iov_len);
  496. ret = file->f_op->write_iter(&kiocb, &iter);
  497. if (ret > 0) {
  498. if (pos)
  499. *pos = kiocb.ki_pos;
  500. fsnotify_modify(file);
  501. add_wchar(current, ret);
  502. }
  503. inc_syscw(current);
  504. return ret;
  505. }
  506. /*
  507. * This "EXPORT_SYMBOL_GPL()" is more of a "EXPORT_SYMBOL_DONTUSE()",
  508. * but autofs is one of the few internal kernel users that actually
  509. * wants this _and_ can be built as a module. So we need to export
  510. * this symbol for autofs, even though it really isn't appropriate
  511. * for any other kernel modules.
  512. */
  513. EXPORT_SYMBOL_GPL(__kernel_write);
  514. ssize_t kernel_write(struct file *file, const void *buf, size_t count,
  515. loff_t *pos)
  516. {
  517. ssize_t ret;
  518. ret = rw_verify_area(WRITE, file, pos, count);
  519. if (ret)
  520. return ret;
  521. file_start_write(file);
  522. ret = __kernel_write(file, buf, count, pos);
  523. file_end_write(file);
  524. return ret;
  525. }
  526. EXPORT_SYMBOL_NS(kernel_write, ANDROID_GKI_VFS_EXPORT_ONLY);
  527. ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
  528. {
  529. ssize_t ret;
  530. if (!(file->f_mode & FMODE_WRITE))
  531. return -EBADF;
  532. if (!(file->f_mode & FMODE_CAN_WRITE))
  533. return -EINVAL;
  534. if (unlikely(!access_ok(buf, count)))
  535. return -EFAULT;
  536. ret = rw_verify_area(WRITE, file, pos, count);
  537. if (ret)
  538. return ret;
  539. if (count > MAX_RW_COUNT)
  540. count = MAX_RW_COUNT;
  541. file_start_write(file);
  542. if (file->f_op->write)
  543. ret = file->f_op->write(file, buf, count, pos);
  544. else if (file->f_op->write_iter)
  545. ret = new_sync_write(file, buf, count, pos);
  546. else
  547. ret = -EINVAL;
  548. if (ret > 0) {
  549. fsnotify_modify(file);
  550. add_wchar(current, ret);
  551. }
  552. inc_syscw(current);
  553. file_end_write(file);
  554. return ret;
  555. }
  556. /* file_ppos returns &file->f_pos or NULL if file is stream */
  557. static inline loff_t *file_ppos(struct file *file)
  558. {
  559. return file->f_mode & FMODE_STREAM ? NULL : &file->f_pos;
  560. }
  561. ssize_t ksys_read(unsigned int fd, char __user *buf, size_t count)
  562. {
  563. struct fd f = fdget_pos(fd);
  564. ssize_t ret = -EBADF;
  565. if (f.file) {
  566. loff_t pos, *ppos = file_ppos(f.file);
  567. if (ppos) {
  568. pos = *ppos;
  569. ppos = &pos;
  570. }
  571. ret = vfs_read(f.file, buf, count, ppos);
  572. if (ret >= 0 && ppos)
  573. f.file->f_pos = pos;
  574. fdput_pos(f);
  575. }
  576. return ret;
  577. }
  578. SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
  579. {
  580. return ksys_read(fd, buf, count);
  581. }
  582. ssize_t ksys_write(unsigned int fd, const char __user *buf, size_t count)
  583. {
  584. struct fd f = fdget_pos(fd);
  585. ssize_t ret = -EBADF;
  586. if (f.file) {
  587. loff_t pos, *ppos = file_ppos(f.file);
  588. if (ppos) {
  589. pos = *ppos;
  590. ppos = &pos;
  591. }
  592. ret = vfs_write(f.file, buf, count, ppos);
  593. if (ret >= 0 && ppos)
  594. f.file->f_pos = pos;
  595. fdput_pos(f);
  596. }
  597. return ret;
  598. }
  599. SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
  600. size_t, count)
  601. {
  602. return ksys_write(fd, buf, count);
  603. }
  604. ssize_t ksys_pread64(unsigned int fd, char __user *buf, size_t count,
  605. loff_t pos)
  606. {
  607. struct fd f;
  608. ssize_t ret = -EBADF;
  609. if (pos < 0)
  610. return -EINVAL;
  611. f = fdget(fd);
  612. if (f.file) {
  613. ret = -ESPIPE;
  614. if (f.file->f_mode & FMODE_PREAD)
  615. ret = vfs_read(f.file, buf, count, &pos);
  616. fdput(f);
  617. }
  618. return ret;
  619. }
  620. SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
  621. size_t, count, loff_t, pos)
  622. {
  623. return ksys_pread64(fd, buf, count, pos);
  624. }
  625. ssize_t ksys_pwrite64(unsigned int fd, const char __user *buf,
  626. size_t count, loff_t pos)
  627. {
  628. struct fd f;
  629. ssize_t ret = -EBADF;
  630. if (pos < 0)
  631. return -EINVAL;
  632. f = fdget(fd);
  633. if (f.file) {
  634. ret = -ESPIPE;
  635. if (f.file->f_mode & FMODE_PWRITE)
  636. ret = vfs_write(f.file, buf, count, &pos);
  637. fdput(f);
  638. }
  639. return ret;
  640. }
  641. SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
  642. size_t, count, loff_t, pos)
  643. {
  644. return ksys_pwrite64(fd, buf, count, pos);
  645. }
  646. static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter,
  647. loff_t *ppos, int type, rwf_t flags)
  648. {
  649. struct kiocb kiocb;
  650. ssize_t ret;
  651. init_sync_kiocb(&kiocb, filp);
  652. ret = kiocb_set_rw_flags(&kiocb, flags);
  653. if (ret)
  654. return ret;
  655. kiocb.ki_pos = (ppos ? *ppos : 0);
  656. if (type == READ)
  657. ret = call_read_iter(filp, &kiocb, iter);
  658. else
  659. ret = call_write_iter(filp, &kiocb, iter);
  660. BUG_ON(ret == -EIOCBQUEUED);
  661. if (ppos)
  662. *ppos = kiocb.ki_pos;
  663. return ret;
  664. }
  665. /* Do it by hand, with file-ops */
  666. static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
  667. loff_t *ppos, int type, rwf_t flags)
  668. {
  669. ssize_t ret = 0;
  670. if (flags & ~RWF_HIPRI)
  671. return -EOPNOTSUPP;
  672. while (iov_iter_count(iter)) {
  673. struct iovec iovec = iov_iter_iovec(iter);
  674. ssize_t nr;
  675. if (type == READ) {
  676. nr = filp->f_op->read(filp, iovec.iov_base,
  677. iovec.iov_len, ppos);
  678. } else {
  679. nr = filp->f_op->write(filp, iovec.iov_base,
  680. iovec.iov_len, ppos);
  681. }
  682. if (nr < 0) {
  683. if (!ret)
  684. ret = nr;
  685. break;
  686. }
  687. ret += nr;
  688. if (nr != iovec.iov_len)
  689. break;
  690. iov_iter_advance(iter, nr);
  691. }
  692. return ret;
  693. }
  694. static ssize_t do_iter_read(struct file *file, struct iov_iter *iter,
  695. loff_t *pos, rwf_t flags)
  696. {
  697. size_t tot_len;
  698. ssize_t ret = 0;
  699. if (!(file->f_mode & FMODE_READ))
  700. return -EBADF;
  701. if (!(file->f_mode & FMODE_CAN_READ))
  702. return -EINVAL;
  703. tot_len = iov_iter_count(iter);
  704. if (!tot_len)
  705. goto out;
  706. ret = rw_verify_area(READ, file, pos, tot_len);
  707. if (ret < 0)
  708. return ret;
  709. if (file->f_op->read_iter)
  710. ret = do_iter_readv_writev(file, iter, pos, READ, flags);
  711. else
  712. ret = do_loop_readv_writev(file, iter, pos, READ, flags);
  713. out:
  714. if (ret >= 0)
  715. fsnotify_access(file);
  716. return ret;
  717. }
  718. ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb,
  719. struct iov_iter *iter)
  720. {
  721. size_t tot_len;
  722. ssize_t ret = 0;
  723. if (!file->f_op->read_iter)
  724. return -EINVAL;
  725. if (!(file->f_mode & FMODE_READ))
  726. return -EBADF;
  727. if (!(file->f_mode & FMODE_CAN_READ))
  728. return -EINVAL;
  729. tot_len = iov_iter_count(iter);
  730. if (!tot_len)
  731. goto out;
  732. ret = rw_verify_area(READ, file, &iocb->ki_pos, tot_len);
  733. if (ret < 0)
  734. return ret;
  735. ret = call_read_iter(file, iocb, iter);
  736. out:
  737. if (ret >= 0)
  738. fsnotify_access(file);
  739. return ret;
  740. }
  741. EXPORT_SYMBOL(vfs_iocb_iter_read);
  742. ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
  743. rwf_t flags)
  744. {
  745. if (!file->f_op->read_iter)
  746. return -EINVAL;
  747. return do_iter_read(file, iter, ppos, flags);
  748. }
  749. EXPORT_SYMBOL(vfs_iter_read);
  750. static ssize_t do_iter_write(struct file *file, struct iov_iter *iter,
  751. loff_t *pos, rwf_t flags)
  752. {
  753. size_t tot_len;
  754. ssize_t ret = 0;
  755. if (!(file->f_mode & FMODE_WRITE))
  756. return -EBADF;
  757. if (!(file->f_mode & FMODE_CAN_WRITE))
  758. return -EINVAL;
  759. tot_len = iov_iter_count(iter);
  760. if (!tot_len)
  761. return 0;
  762. ret = rw_verify_area(WRITE, file, pos, tot_len);
  763. if (ret < 0)
  764. return ret;
  765. if (file->f_op->write_iter)
  766. ret = do_iter_readv_writev(file, iter, pos, WRITE, flags);
  767. else
  768. ret = do_loop_readv_writev(file, iter, pos, WRITE, flags);
  769. if (ret > 0)
  770. fsnotify_modify(file);
  771. return ret;
  772. }
  773. ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb,
  774. struct iov_iter *iter)
  775. {
  776. size_t tot_len;
  777. ssize_t ret = 0;
  778. if (!file->f_op->write_iter)
  779. return -EINVAL;
  780. if (!(file->f_mode & FMODE_WRITE))
  781. return -EBADF;
  782. if (!(file->f_mode & FMODE_CAN_WRITE))
  783. return -EINVAL;
  784. tot_len = iov_iter_count(iter);
  785. if (!tot_len)
  786. return 0;
  787. ret = rw_verify_area(WRITE, file, &iocb->ki_pos, tot_len);
  788. if (ret < 0)
  789. return ret;
  790. ret = call_write_iter(file, iocb, iter);
  791. if (ret > 0)
  792. fsnotify_modify(file);
  793. return ret;
  794. }
  795. EXPORT_SYMBOL(vfs_iocb_iter_write);
  796. ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
  797. rwf_t flags)
  798. {
  799. if (!file->f_op->write_iter)
  800. return -EINVAL;
  801. return do_iter_write(file, iter, ppos, flags);
  802. }
  803. EXPORT_SYMBOL(vfs_iter_write);
  804. static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
  805. unsigned long vlen, loff_t *pos, rwf_t flags)
  806. {
  807. struct iovec iovstack[UIO_FASTIOV];
  808. struct iovec *iov = iovstack;
  809. struct iov_iter iter;
  810. ssize_t ret;
  811. ret = import_iovec(READ, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
  812. if (ret >= 0) {
  813. ret = do_iter_read(file, &iter, pos, flags);
  814. kfree(iov);
  815. }
  816. return ret;
  817. }
  818. static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
  819. unsigned long vlen, loff_t *pos, rwf_t flags)
  820. {
  821. struct iovec iovstack[UIO_FASTIOV];
  822. struct iovec *iov = iovstack;
  823. struct iov_iter iter;
  824. ssize_t ret;
  825. ret = import_iovec(WRITE, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
  826. if (ret >= 0) {
  827. file_start_write(file);
  828. ret = do_iter_write(file, &iter, pos, flags);
  829. file_end_write(file);
  830. kfree(iov);
  831. }
  832. return ret;
  833. }
  834. static ssize_t do_readv(unsigned long fd, const struct iovec __user *vec,
  835. unsigned long vlen, rwf_t flags)
  836. {
  837. struct fd f = fdget_pos(fd);
  838. ssize_t ret = -EBADF;
  839. if (f.file) {
  840. loff_t pos, *ppos = file_ppos(f.file);
  841. if (ppos) {
  842. pos = *ppos;
  843. ppos = &pos;
  844. }
  845. ret = vfs_readv(f.file, vec, vlen, ppos, flags);
  846. if (ret >= 0 && ppos)
  847. f.file->f_pos = pos;
  848. fdput_pos(f);
  849. }
  850. if (ret > 0)
  851. add_rchar(current, ret);
  852. inc_syscr(current);
  853. return ret;
  854. }
  855. static ssize_t do_writev(unsigned long fd, const struct iovec __user *vec,
  856. unsigned long vlen, rwf_t flags)
  857. {
  858. struct fd f = fdget_pos(fd);
  859. ssize_t ret = -EBADF;
  860. if (f.file) {
  861. loff_t pos, *ppos = file_ppos(f.file);
  862. if (ppos) {
  863. pos = *ppos;
  864. ppos = &pos;
  865. }
  866. ret = vfs_writev(f.file, vec, vlen, ppos, flags);
  867. if (ret >= 0 && ppos)
  868. f.file->f_pos = pos;
  869. fdput_pos(f);
  870. }
  871. if (ret > 0)
  872. add_wchar(current, ret);
  873. inc_syscw(current);
  874. return ret;
  875. }
  876. static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
  877. {
  878. #define HALF_LONG_BITS (BITS_PER_LONG / 2)
  879. return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
  880. }
  881. static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec,
  882. unsigned long vlen, loff_t pos, rwf_t flags)
  883. {
  884. struct fd f;
  885. ssize_t ret = -EBADF;
  886. if (pos < 0)
  887. return -EINVAL;
  888. f = fdget(fd);
  889. if (f.file) {
  890. ret = -ESPIPE;
  891. if (f.file->f_mode & FMODE_PREAD)
  892. ret = vfs_readv(f.file, vec, vlen, &pos, flags);
  893. fdput(f);
  894. }
  895. if (ret > 0)
  896. add_rchar(current, ret);
  897. inc_syscr(current);
  898. return ret;
  899. }
  900. static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec,
  901. unsigned long vlen, loff_t pos, rwf_t flags)
  902. {
  903. struct fd f;
  904. ssize_t ret = -EBADF;
  905. if (pos < 0)
  906. return -EINVAL;
  907. f = fdget(fd);
  908. if (f.file) {
  909. ret = -ESPIPE;
  910. if (f.file->f_mode & FMODE_PWRITE)
  911. ret = vfs_writev(f.file, vec, vlen, &pos, flags);
  912. fdput(f);
  913. }
  914. if (ret > 0)
  915. add_wchar(current, ret);
  916. inc_syscw(current);
  917. return ret;
  918. }
  919. SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
  920. unsigned long, vlen)
  921. {
  922. return do_readv(fd, vec, vlen, 0);
  923. }
  924. SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
  925. unsigned long, vlen)
  926. {
  927. return do_writev(fd, vec, vlen, 0);
  928. }
  929. SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
  930. unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
  931. {
  932. loff_t pos = pos_from_hilo(pos_h, pos_l);
  933. return do_preadv(fd, vec, vlen, pos, 0);
  934. }
  935. SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec,
  936. unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
  937. rwf_t, flags)
  938. {
  939. loff_t pos = pos_from_hilo(pos_h, pos_l);
  940. if (pos == -1)
  941. return do_readv(fd, vec, vlen, flags);
  942. return do_preadv(fd, vec, vlen, pos, flags);
  943. }
  944. SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
  945. unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
  946. {
  947. loff_t pos = pos_from_hilo(pos_h, pos_l);
  948. return do_pwritev(fd, vec, vlen, pos, 0);
  949. }
  950. SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec,
  951. unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
  952. rwf_t, flags)
  953. {
  954. loff_t pos = pos_from_hilo(pos_h, pos_l);
  955. if (pos == -1)
  956. return do_writev(fd, vec, vlen, flags);
  957. return do_pwritev(fd, vec, vlen, pos, flags);
  958. }
  959. /*
  960. * Various compat syscalls. Note that they all pretend to take a native
  961. * iovec - import_iovec will properly treat those as compat_iovecs based on
  962. * in_compat_syscall().
  963. */
  964. #ifdef CONFIG_COMPAT
  965. #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
  966. COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
  967. const struct iovec __user *, vec,
  968. unsigned long, vlen, loff_t, pos)
  969. {
  970. return do_preadv(fd, vec, vlen, pos, 0);
  971. }
  972. #endif
  973. COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
  974. const struct iovec __user *, vec,
  975. compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
  976. {
  977. loff_t pos = ((loff_t)pos_high << 32) | pos_low;
  978. return do_preadv(fd, vec, vlen, pos, 0);
  979. }
  980. #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2
  981. COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd,
  982. const struct iovec __user *, vec,
  983. unsigned long, vlen, loff_t, pos, rwf_t, flags)
  984. {
  985. if (pos == -1)
  986. return do_readv(fd, vec, vlen, flags);
  987. return do_preadv(fd, vec, vlen, pos, flags);
  988. }
  989. #endif
  990. COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd,
  991. const struct iovec __user *, vec,
  992. compat_ulong_t, vlen, u32, pos_low, u32, pos_high,
  993. rwf_t, flags)
  994. {
  995. loff_t pos = ((loff_t)pos_high << 32) | pos_low;
  996. if (pos == -1)
  997. return do_readv(fd, vec, vlen, flags);
  998. return do_preadv(fd, vec, vlen, pos, flags);
  999. }
  1000. #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
  1001. COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
  1002. const struct iovec __user *, vec,
  1003. unsigned long, vlen, loff_t, pos)
  1004. {
  1005. return do_pwritev(fd, vec, vlen, pos, 0);
  1006. }
  1007. #endif
  1008. COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
  1009. const struct iovec __user *,vec,
  1010. compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
  1011. {
  1012. loff_t pos = ((loff_t)pos_high << 32) | pos_low;
  1013. return do_pwritev(fd, vec, vlen, pos, 0);
  1014. }
  1015. #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2
  1016. COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd,
  1017. const struct iovec __user *, vec,
  1018. unsigned long, vlen, loff_t, pos, rwf_t, flags)
  1019. {
  1020. if (pos == -1)
  1021. return do_writev(fd, vec, vlen, flags);
  1022. return do_pwritev(fd, vec, vlen, pos, flags);
  1023. }
  1024. #endif
  1025. COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd,
  1026. const struct iovec __user *,vec,
  1027. compat_ulong_t, vlen, u32, pos_low, u32, pos_high, rwf_t, flags)
  1028. {
  1029. loff_t pos = ((loff_t)pos_high << 32) | pos_low;
  1030. if (pos == -1)
  1031. return do_writev(fd, vec, vlen, flags);
  1032. return do_pwritev(fd, vec, vlen, pos, flags);
  1033. }
  1034. #endif /* CONFIG_COMPAT */
  1035. static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
  1036. size_t count, loff_t max)
  1037. {
  1038. struct fd in, out;
  1039. struct inode *in_inode, *out_inode;
  1040. loff_t pos;
  1041. loff_t out_pos;
  1042. ssize_t retval;
  1043. int fl;
  1044. /*
  1045. * Get input file, and verify that it is ok..
  1046. */
  1047. retval = -EBADF;
  1048. in = fdget(in_fd);
  1049. if (!in.file)
  1050. goto out;
  1051. if (!(in.file->f_mode & FMODE_READ))
  1052. goto fput_in;
  1053. retval = -ESPIPE;
  1054. if (!ppos) {
  1055. pos = in.file->f_pos;
  1056. } else {
  1057. pos = *ppos;
  1058. if (!(in.file->f_mode & FMODE_PREAD))
  1059. goto fput_in;
  1060. }
  1061. retval = rw_verify_area(READ, in.file, &pos, count);
  1062. if (retval < 0)
  1063. goto fput_in;
  1064. if (count > MAX_RW_COUNT)
  1065. count = MAX_RW_COUNT;
  1066. /*
  1067. * Get output file, and verify that it is ok..
  1068. */
  1069. retval = -EBADF;
  1070. out = fdget(out_fd);
  1071. if (!out.file)
  1072. goto fput_in;
  1073. if (!(out.file->f_mode & FMODE_WRITE))
  1074. goto fput_out;
  1075. in_inode = file_inode(in.file);
  1076. out_inode = file_inode(out.file);
  1077. out_pos = out.file->f_pos;
  1078. retval = rw_verify_area(WRITE, out.file, &out_pos, count);
  1079. if (retval < 0)
  1080. goto fput_out;
  1081. if (!max)
  1082. max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
  1083. if (unlikely(pos + count > max)) {
  1084. retval = -EOVERFLOW;
  1085. if (pos >= max)
  1086. goto fput_out;
  1087. count = max - pos;
  1088. }
  1089. fl = 0;
  1090. #if 0
  1091. /*
  1092. * We need to debate whether we can enable this or not. The
  1093. * man page documents EAGAIN return for the output at least,
  1094. * and the application is arguably buggy if it doesn't expect
  1095. * EAGAIN on a non-blocking file descriptor.
  1096. */
  1097. if (in.file->f_flags & O_NONBLOCK)
  1098. fl = SPLICE_F_NONBLOCK;
  1099. #endif
  1100. file_start_write(out.file);
  1101. retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl);
  1102. file_end_write(out.file);
  1103. if (retval > 0) {
  1104. add_rchar(current, retval);
  1105. add_wchar(current, retval);
  1106. fsnotify_access(in.file);
  1107. fsnotify_modify(out.file);
  1108. out.file->f_pos = out_pos;
  1109. if (ppos)
  1110. *ppos = pos;
  1111. else
  1112. in.file->f_pos = pos;
  1113. }
  1114. inc_syscr(current);
  1115. inc_syscw(current);
  1116. if (pos > max)
  1117. retval = -EOVERFLOW;
  1118. fput_out:
  1119. fdput(out);
  1120. fput_in:
  1121. fdput(in);
  1122. out:
  1123. return retval;
  1124. }
  1125. SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
  1126. {
  1127. loff_t pos;
  1128. off_t off;
  1129. ssize_t ret;
  1130. if (offset) {
  1131. if (unlikely(get_user(off, offset)))
  1132. return -EFAULT;
  1133. pos = off;
  1134. ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
  1135. if (unlikely(put_user(pos, offset)))
  1136. return -EFAULT;
  1137. return ret;
  1138. }
  1139. return do_sendfile(out_fd, in_fd, NULL, count, 0);
  1140. }
  1141. SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
  1142. {
  1143. loff_t pos;
  1144. ssize_t ret;
  1145. if (offset) {
  1146. if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
  1147. return -EFAULT;
  1148. ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
  1149. if (unlikely(put_user(pos, offset)))
  1150. return -EFAULT;
  1151. return ret;
  1152. }
  1153. return do_sendfile(out_fd, in_fd, NULL, count, 0);
  1154. }
  1155. #ifdef CONFIG_COMPAT
  1156. COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
  1157. compat_off_t __user *, offset, compat_size_t, count)
  1158. {
  1159. loff_t pos;
  1160. off_t off;
  1161. ssize_t ret;
  1162. if (offset) {
  1163. if (unlikely(get_user(off, offset)))
  1164. return -EFAULT;
  1165. pos = off;
  1166. ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
  1167. if (unlikely(put_user(pos, offset)))
  1168. return -EFAULT;
  1169. return ret;
  1170. }
  1171. return do_sendfile(out_fd, in_fd, NULL, count, 0);
  1172. }
  1173. COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
  1174. compat_loff_t __user *, offset, compat_size_t, count)
  1175. {
  1176. loff_t pos;
  1177. ssize_t ret;
  1178. if (offset) {
  1179. if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
  1180. return -EFAULT;
  1181. ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
  1182. if (unlikely(put_user(pos, offset)))
  1183. return -EFAULT;
  1184. return ret;
  1185. }
  1186. return do_sendfile(out_fd, in_fd, NULL, count, 0);
  1187. }
  1188. #endif
  1189. /**
  1190. * generic_copy_file_range - copy data between two files
  1191. * @file_in: file structure to read from
  1192. * @pos_in: file offset to read from
  1193. * @file_out: file structure to write data to
  1194. * @pos_out: file offset to write data to
  1195. * @len: amount of data to copy
  1196. * @flags: copy flags
  1197. *
  1198. * This is a generic filesystem helper to copy data from one file to another.
  1199. * It has no constraints on the source or destination file owners - the files
  1200. * can belong to different superblocks and different filesystem types. Short
  1201. * copies are allowed.
  1202. *
  1203. * This should be called from the @file_out filesystem, as per the
  1204. * ->copy_file_range() method.
  1205. *
  1206. * Returns the number of bytes copied or a negative error indicating the
  1207. * failure.
  1208. */
  1209. ssize_t generic_copy_file_range(struct file *file_in, loff_t pos_in,
  1210. struct file *file_out, loff_t pos_out,
  1211. size_t len, unsigned int flags)
  1212. {
  1213. return do_splice_direct(file_in, &pos_in, file_out, &pos_out,
  1214. len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0);
  1215. }
  1216. EXPORT_SYMBOL(generic_copy_file_range);
  1217. static ssize_t do_copy_file_range(struct file *file_in, loff_t pos_in,
  1218. struct file *file_out, loff_t pos_out,
  1219. size_t len, unsigned int flags)
  1220. {
  1221. /*
  1222. * Although we now allow filesystems to handle cross sb copy, passing
  1223. * a file of the wrong filesystem type to filesystem driver can result
  1224. * in an attempt to dereference the wrong type of ->private_data, so
  1225. * avoid doing that until we really have a good reason. NFS defines
  1226. * several different file_system_type structures, but they all end up
  1227. * using the same ->copy_file_range() function pointer.
  1228. */
  1229. if (file_out->f_op->copy_file_range &&
  1230. file_out->f_op->copy_file_range == file_in->f_op->copy_file_range)
  1231. return file_out->f_op->copy_file_range(file_in, pos_in,
  1232. file_out, pos_out,
  1233. len, flags);
  1234. return generic_copy_file_range(file_in, pos_in, file_out, pos_out, len,
  1235. flags);
  1236. }
  1237. /*
  1238. * Performs necessary checks before doing a file copy
  1239. *
  1240. * Can adjust amount of bytes to copy via @req_count argument.
  1241. * Returns appropriate error code that caller should return or
  1242. * zero in case the copy should be allowed.
  1243. */
  1244. static int generic_copy_file_checks(struct file *file_in, loff_t pos_in,
  1245. struct file *file_out, loff_t pos_out,
  1246. size_t *req_count, unsigned int flags)
  1247. {
  1248. struct inode *inode_in = file_inode(file_in);
  1249. struct inode *inode_out = file_inode(file_out);
  1250. uint64_t count = *req_count;
  1251. loff_t size_in;
  1252. int ret;
  1253. ret = generic_file_rw_checks(file_in, file_out);
  1254. if (ret)
  1255. return ret;
  1256. /* Don't touch certain kinds of inodes */
  1257. if (IS_IMMUTABLE(inode_out))
  1258. return -EPERM;
  1259. if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
  1260. return -ETXTBSY;
  1261. /* Ensure offsets don't wrap. */
  1262. if (pos_in + count < pos_in || pos_out + count < pos_out)
  1263. return -EOVERFLOW;
  1264. /* Shorten the copy to EOF */
  1265. size_in = i_size_read(inode_in);
  1266. if (pos_in >= size_in)
  1267. count = 0;
  1268. else
  1269. count = min(count, size_in - (uint64_t)pos_in);
  1270. ret = generic_write_check_limits(file_out, pos_out, &count);
  1271. if (ret)
  1272. return ret;
  1273. /* Don't allow overlapped copying within the same file. */
  1274. if (inode_in == inode_out &&
  1275. pos_out + count > pos_in &&
  1276. pos_out < pos_in + count)
  1277. return -EINVAL;
  1278. *req_count = count;
  1279. return 0;
  1280. }
  1281. /*
  1282. * copy_file_range() differs from regular file read and write in that it
  1283. * specifically allows return partial success. When it does so is up to
  1284. * the copy_file_range method.
  1285. */
  1286. ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in,
  1287. struct file *file_out, loff_t pos_out,
  1288. size_t len, unsigned int flags)
  1289. {
  1290. ssize_t ret;
  1291. if (flags != 0)
  1292. return -EINVAL;
  1293. ret = generic_copy_file_checks(file_in, pos_in, file_out, pos_out, &len,
  1294. flags);
  1295. if (unlikely(ret))
  1296. return ret;
  1297. ret = rw_verify_area(READ, file_in, &pos_in, len);
  1298. if (unlikely(ret))
  1299. return ret;
  1300. ret = rw_verify_area(WRITE, file_out, &pos_out, len);
  1301. if (unlikely(ret))
  1302. return ret;
  1303. if (len == 0)
  1304. return 0;
  1305. file_start_write(file_out);
  1306. /*
  1307. * Try cloning first, this is supported by more file systems, and
  1308. * more efficient if both clone and copy are supported (e.g. NFS).
  1309. */
  1310. if (file_in->f_op->remap_file_range &&
  1311. file_inode(file_in)->i_sb == file_inode(file_out)->i_sb) {
  1312. loff_t cloned;
  1313. cloned = file_in->f_op->remap_file_range(file_in, pos_in,
  1314. file_out, pos_out,
  1315. min_t(loff_t, MAX_RW_COUNT, len),
  1316. REMAP_FILE_CAN_SHORTEN);
  1317. if (cloned > 0) {
  1318. ret = cloned;
  1319. goto done;
  1320. }
  1321. }
  1322. ret = do_copy_file_range(file_in, pos_in, file_out, pos_out, len,
  1323. flags);
  1324. WARN_ON_ONCE(ret == -EOPNOTSUPP);
  1325. done:
  1326. if (ret > 0) {
  1327. fsnotify_access(file_in);
  1328. add_rchar(current, ret);
  1329. fsnotify_modify(file_out);
  1330. add_wchar(current, ret);
  1331. }
  1332. inc_syscr(current);
  1333. inc_syscw(current);
  1334. file_end_write(file_out);
  1335. return ret;
  1336. }
  1337. EXPORT_SYMBOL(vfs_copy_file_range);
  1338. SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in,
  1339. int, fd_out, loff_t __user *, off_out,
  1340. size_t, len, unsigned int, flags)
  1341. {
  1342. loff_t pos_in;
  1343. loff_t pos_out;
  1344. struct fd f_in;
  1345. struct fd f_out;
  1346. ssize_t ret = -EBADF;
  1347. f_in = fdget(fd_in);
  1348. if (!f_in.file)
  1349. goto out2;
  1350. f_out = fdget(fd_out);
  1351. if (!f_out.file)
  1352. goto out1;
  1353. ret = -EFAULT;
  1354. if (off_in) {
  1355. if (copy_from_user(&pos_in, off_in, sizeof(loff_t)))
  1356. goto out;
  1357. } else {
  1358. pos_in = f_in.file->f_pos;
  1359. }
  1360. if (off_out) {
  1361. if (copy_from_user(&pos_out, off_out, sizeof(loff_t)))
  1362. goto out;
  1363. } else {
  1364. pos_out = f_out.file->f_pos;
  1365. }
  1366. ret = vfs_copy_file_range(f_in.file, pos_in, f_out.file, pos_out, len,
  1367. flags);
  1368. if (ret > 0) {
  1369. pos_in += ret;
  1370. pos_out += ret;
  1371. if (off_in) {
  1372. if (copy_to_user(off_in, &pos_in, sizeof(loff_t)))
  1373. ret = -EFAULT;
  1374. } else {
  1375. f_in.file->f_pos = pos_in;
  1376. }
  1377. if (off_out) {
  1378. if (copy_to_user(off_out, &pos_out, sizeof(loff_t)))
  1379. ret = -EFAULT;
  1380. } else {
  1381. f_out.file->f_pos = pos_out;
  1382. }
  1383. }
  1384. out:
  1385. fdput(f_out);
  1386. out1:
  1387. fdput(f_in);
  1388. out2:
  1389. return ret;
  1390. }
  1391. /*
  1392. * Don't operate on ranges the page cache doesn't support, and don't exceed the
  1393. * LFS limits. If pos is under the limit it becomes a short access. If it
  1394. * exceeds the limit we return -EFBIG.
  1395. */
  1396. int generic_write_check_limits(struct file *file, loff_t pos, loff_t *count)
  1397. {
  1398. struct inode *inode = file->f_mapping->host;
  1399. loff_t max_size = inode->i_sb->s_maxbytes;
  1400. loff_t limit = rlimit(RLIMIT_FSIZE);
  1401. if (limit != RLIM_INFINITY) {
  1402. if (pos >= limit) {
  1403. send_sig(SIGXFSZ, current, 0);
  1404. return -EFBIG;
  1405. }
  1406. *count = min(*count, limit - pos);
  1407. }
  1408. if (!(file->f_flags & O_LARGEFILE))
  1409. max_size = MAX_NON_LFS;
  1410. if (unlikely(pos >= max_size))
  1411. return -EFBIG;
  1412. *count = min(*count, max_size - pos);
  1413. return 0;
  1414. }
  1415. /*
  1416. * Performs necessary checks before doing a write
  1417. *
  1418. * Can adjust writing position or amount of bytes to write.
  1419. * Returns appropriate error code that caller should return or
  1420. * zero in case that write should be allowed.
  1421. */
  1422. ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
  1423. {
  1424. struct file *file = iocb->ki_filp;
  1425. struct inode *inode = file->f_mapping->host;
  1426. loff_t count;
  1427. int ret;
  1428. if (IS_SWAPFILE(inode))
  1429. return -ETXTBSY;
  1430. if (!iov_iter_count(from))
  1431. return 0;
  1432. /* FIXME: this is for backwards compatibility with 2.4 */
  1433. if (iocb->ki_flags & IOCB_APPEND)
  1434. iocb->ki_pos = i_size_read(inode);
  1435. if ((iocb->ki_flags & IOCB_NOWAIT) && !(iocb->ki_flags & IOCB_DIRECT))
  1436. return -EINVAL;
  1437. count = iov_iter_count(from);
  1438. ret = generic_write_check_limits(file, iocb->ki_pos, &count);
  1439. if (ret)
  1440. return ret;
  1441. iov_iter_truncate(from, count);
  1442. return iov_iter_count(from);
  1443. }
  1444. EXPORT_SYMBOL(generic_write_checks);
  1445. /*
  1446. * Performs common checks before doing a file copy/clone
  1447. * from @file_in to @file_out.
  1448. */
  1449. int generic_file_rw_checks(struct file *file_in, struct file *file_out)
  1450. {
  1451. struct inode *inode_in = file_inode(file_in);
  1452. struct inode *inode_out = file_inode(file_out);
  1453. /* Don't copy dirs, pipes, sockets... */
  1454. if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
  1455. return -EISDIR;
  1456. if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
  1457. return -EINVAL;
  1458. if (!(file_in->f_mode & FMODE_READ) ||
  1459. !(file_out->f_mode & FMODE_WRITE) ||
  1460. (file_out->f_flags & O_APPEND))
  1461. return -EBADF;
  1462. return 0;
  1463. }