memfd.c 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345
  1. /*
  2. * memfd_create system call and file sealing support
  3. *
  4. * Code was originally included in shmem.c, and broken out to facilitate
  5. * use by hugetlbfs as well as tmpfs.
  6. *
  7. * This file is released under the GPL.
  8. */
  9. #include <linux/fs.h>
  10. #include <linux/vfs.h>
  11. #include <linux/pagemap.h>
  12. #include <linux/file.h>
  13. #include <linux/mm.h>
  14. #include <linux/sched/signal.h>
  15. #include <linux/khugepaged.h>
  16. #include <linux/syscalls.h>
  17. #include <linux/hugetlb.h>
  18. #include <linux/shmem_fs.h>
  19. #include <linux/memfd.h>
  20. #include <uapi/linux/memfd.h>
  21. /*
  22. * We need a tag: a new tag would expand every xa_node by 8 bytes,
  23. * so reuse a tag which we firmly believe is never set or cleared on tmpfs
  24. * or hugetlbfs because they are memory only filesystems.
  25. */
  26. #define MEMFD_TAG_PINNED PAGECACHE_TAG_TOWRITE
  27. #define LAST_SCAN 4 /* about 150ms max */
  28. static void memfd_tag_pins(struct xa_state *xas)
  29. {
  30. struct page *page;
  31. int latency = 0;
  32. int cache_count;
  33. lru_add_drain();
  34. xas_lock_irq(xas);
  35. xas_for_each(xas, page, ULONG_MAX) {
  36. cache_count = 1;
  37. if (!xa_is_value(page) &&
  38. PageTransHuge(page) && !PageHuge(page))
  39. cache_count = HPAGE_PMD_NR;
  40. if (!xa_is_value(page) &&
  41. page_count(page) - total_mapcount(page) != cache_count)
  42. xas_set_mark(xas, MEMFD_TAG_PINNED);
  43. if (cache_count != 1)
  44. xas_set(xas, page->index + cache_count);
  45. latency += cache_count;
  46. if (latency < XA_CHECK_SCHED)
  47. continue;
  48. latency = 0;
  49. xas_pause(xas);
  50. xas_unlock_irq(xas);
  51. cond_resched();
  52. xas_lock_irq(xas);
  53. }
  54. xas_unlock_irq(xas);
  55. }
  56. /*
  57. * Setting SEAL_WRITE requires us to verify there's no pending writer. However,
  58. * via get_user_pages(), drivers might have some pending I/O without any active
  59. * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages
  60. * and see whether it has an elevated ref-count. If so, we tag them and wait for
  61. * them to be dropped.
  62. * The caller must guarantee that no new user will acquire writable references
  63. * to those pages to avoid races.
  64. */
  65. static int memfd_wait_for_pins(struct address_space *mapping)
  66. {
  67. XA_STATE(xas, &mapping->i_pages, 0);
  68. struct page *page;
  69. int error, scan;
  70. memfd_tag_pins(&xas);
  71. error = 0;
  72. for (scan = 0; scan <= LAST_SCAN; scan++) {
  73. int latency = 0;
  74. int cache_count;
  75. if (!xas_marked(&xas, MEMFD_TAG_PINNED))
  76. break;
  77. if (!scan)
  78. lru_add_drain_all();
  79. else if (schedule_timeout_killable((HZ << scan) / 200))
  80. scan = LAST_SCAN;
  81. xas_set(&xas, 0);
  82. xas_lock_irq(&xas);
  83. xas_for_each_marked(&xas, page, ULONG_MAX, MEMFD_TAG_PINNED) {
  84. bool clear = true;
  85. cache_count = 1;
  86. if (!xa_is_value(page) &&
  87. PageTransHuge(page) && !PageHuge(page))
  88. cache_count = HPAGE_PMD_NR;
  89. if (!xa_is_value(page) && cache_count !=
  90. page_count(page) - total_mapcount(page)) {
  91. /*
  92. * On the last scan, we clean up all those tags
  93. * we inserted; but make a note that we still
  94. * found pages pinned.
  95. */
  96. if (scan == LAST_SCAN)
  97. error = -EBUSY;
  98. else
  99. clear = false;
  100. }
  101. if (clear)
  102. xas_clear_mark(&xas, MEMFD_TAG_PINNED);
  103. latency += cache_count;
  104. if (latency < XA_CHECK_SCHED)
  105. continue;
  106. latency = 0;
  107. xas_pause(&xas);
  108. xas_unlock_irq(&xas);
  109. cond_resched();
  110. xas_lock_irq(&xas);
  111. }
  112. xas_unlock_irq(&xas);
  113. }
  114. return error;
  115. }
  116. static unsigned int *memfd_file_seals_ptr(struct file *file)
  117. {
  118. if (shmem_file(file))
  119. return &SHMEM_I(file_inode(file))->seals;
  120. #ifdef CONFIG_HUGETLBFS
  121. if (is_file_hugepages(file))
  122. return &HUGETLBFS_I(file_inode(file))->seals;
  123. #endif
  124. return NULL;
  125. }
  126. #define F_ALL_SEALS (F_SEAL_SEAL | \
  127. F_SEAL_SHRINK | \
  128. F_SEAL_GROW | \
  129. F_SEAL_WRITE | \
  130. F_SEAL_FUTURE_WRITE)
  131. static int memfd_add_seals(struct file *file, unsigned int seals)
  132. {
  133. struct inode *inode = file_inode(file);
  134. unsigned int *file_seals;
  135. int error;
  136. /*
  137. * SEALING
  138. * Sealing allows multiple parties to share a tmpfs or hugetlbfs file
  139. * but restrict access to a specific subset of file operations. Seals
  140. * can only be added, but never removed. This way, mutually untrusted
  141. * parties can share common memory regions with a well-defined policy.
  142. * A malicious peer can thus never perform unwanted operations on a
  143. * shared object.
  144. *
  145. * Seals are only supported on special tmpfs or hugetlbfs files and
  146. * always affect the whole underlying inode. Once a seal is set, it
  147. * may prevent some kinds of access to the file. Currently, the
  148. * following seals are defined:
  149. * SEAL_SEAL: Prevent further seals from being set on this file
  150. * SEAL_SHRINK: Prevent the file from shrinking
  151. * SEAL_GROW: Prevent the file from growing
  152. * SEAL_WRITE: Prevent write access to the file
  153. *
  154. * As we don't require any trust relationship between two parties, we
  155. * must prevent seals from being removed. Therefore, sealing a file
  156. * only adds a given set of seals to the file, it never touches
  157. * existing seals. Furthermore, the "setting seals"-operation can be
  158. * sealed itself, which basically prevents any further seal from being
  159. * added.
  160. *
  161. * Semantics of sealing are only defined on volatile files. Only
  162. * anonymous tmpfs and hugetlbfs files support sealing. More
  163. * importantly, seals are never written to disk. Therefore, there's
  164. * no plan to support it on other file types.
  165. */
  166. if (!(file->f_mode & FMODE_WRITE))
  167. return -EPERM;
  168. if (seals & ~(unsigned int)F_ALL_SEALS)
  169. return -EINVAL;
  170. inode_lock(inode);
  171. file_seals = memfd_file_seals_ptr(file);
  172. if (!file_seals) {
  173. error = -EINVAL;
  174. goto unlock;
  175. }
  176. if (*file_seals & F_SEAL_SEAL) {
  177. error = -EPERM;
  178. goto unlock;
  179. }
  180. if ((seals & F_SEAL_WRITE) && !(*file_seals & F_SEAL_WRITE)) {
  181. error = mapping_deny_writable(file->f_mapping);
  182. if (error)
  183. goto unlock;
  184. error = memfd_wait_for_pins(file->f_mapping);
  185. if (error) {
  186. mapping_allow_writable(file->f_mapping);
  187. goto unlock;
  188. }
  189. }
  190. *file_seals |= seals;
  191. error = 0;
  192. unlock:
  193. inode_unlock(inode);
  194. return error;
  195. }
  196. static int memfd_get_seals(struct file *file)
  197. {
  198. unsigned int *seals = memfd_file_seals_ptr(file);
  199. return seals ? *seals : -EINVAL;
  200. }
  201. long memfd_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
  202. {
  203. long error;
  204. switch (cmd) {
  205. case F_ADD_SEALS:
  206. /* disallow upper 32bit */
  207. if (arg > UINT_MAX)
  208. return -EINVAL;
  209. error = memfd_add_seals(file, arg);
  210. break;
  211. case F_GET_SEALS:
  212. error = memfd_get_seals(file);
  213. break;
  214. default:
  215. error = -EINVAL;
  216. break;
  217. }
  218. return error;
  219. }
  220. #define MFD_NAME_PREFIX "memfd:"
  221. #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1)
  222. #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN)
  223. #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_HUGETLB)
  224. SYSCALL_DEFINE2(memfd_create,
  225. const char __user *, uname,
  226. unsigned int, flags)
  227. {
  228. unsigned int *file_seals;
  229. struct file *file;
  230. int fd, error;
  231. char *name;
  232. long len;
  233. if (!(flags & MFD_HUGETLB)) {
  234. if (flags & ~(unsigned int)MFD_ALL_FLAGS)
  235. return -EINVAL;
  236. } else {
  237. /* Allow huge page size encoding in flags. */
  238. if (flags & ~(unsigned int)(MFD_ALL_FLAGS |
  239. (MFD_HUGE_MASK << MFD_HUGE_SHIFT)))
  240. return -EINVAL;
  241. }
  242. /* length includes terminating zero */
  243. len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1);
  244. if (len <= 0)
  245. return -EFAULT;
  246. if (len > MFD_NAME_MAX_LEN + 1)
  247. return -EINVAL;
  248. name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_KERNEL);
  249. if (!name)
  250. return -ENOMEM;
  251. strcpy(name, MFD_NAME_PREFIX);
  252. if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) {
  253. error = -EFAULT;
  254. goto err_name;
  255. }
  256. /* terminating-zero may have changed after strnlen_user() returned */
  257. if (name[len + MFD_NAME_PREFIX_LEN - 1]) {
  258. error = -EFAULT;
  259. goto err_name;
  260. }
  261. fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0);
  262. if (fd < 0) {
  263. error = fd;
  264. goto err_name;
  265. }
  266. if (flags & MFD_HUGETLB) {
  267. struct user_struct *user = NULL;
  268. file = hugetlb_file_setup(name, 0, VM_NORESERVE, &user,
  269. HUGETLB_ANONHUGE_INODE,
  270. (flags >> MFD_HUGE_SHIFT) &
  271. MFD_HUGE_MASK);
  272. } else
  273. file = shmem_file_setup(name, 0, VM_NORESERVE);
  274. if (IS_ERR(file)) {
  275. error = PTR_ERR(file);
  276. goto err_fd;
  277. }
  278. file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE;
  279. file->f_flags |= O_LARGEFILE;
  280. if (flags & MFD_ALLOW_SEALING) {
  281. file_seals = memfd_file_seals_ptr(file);
  282. *file_seals &= ~F_SEAL_SEAL;
  283. }
  284. fd_install(fd, file);
  285. kfree(name);
  286. return fd;
  287. err_fd:
  288. put_unused_fd(fd);
  289. err_name:
  290. kfree(name);
  291. return error;
  292. }