shm.c 45 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * linux/ipc/shm.c
  4. * Copyright (C) 1992, 1993 Krishna Balasubramanian
  5. * Many improvements/fixes by Bruno Haible.
  6. * Replaced `struct shm_desc' by `struct vm_area_struct', July 1994.
  7. * Fixed the shm swap deallocation (shm_unuse()), August 1998 Andrea Arcangeli.
  8. *
  9. * /proc/sysvipc/shm support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
  10. * BIGMEM support, Andrea Arcangeli <andrea@suse.de>
  11. * SMP thread shm, Jean-Luc Boyard <jean-luc.boyard@siemens.fr>
  12. * HIGHMEM support, Ingo Molnar <mingo@redhat.com>
  13. * Make shmmax, shmall, shmmni sysctl'able, Christoph Rohland <cr@sap.com>
  14. * Shared /dev/zero support, Kanoj Sarcar <kanoj@sgi.com>
  15. * Move the mm functionality over to mm/shmem.c, Christoph Rohland <cr@sap.com>
  16. *
  17. * support for audit of ipc object properties and permission changes
  18. * Dustin Kirkland <dustin.kirkland@us.ibm.com>
  19. *
  20. * namespaces support
  21. * OpenVZ, SWsoft Inc.
  22. * Pavel Emelianov <xemul@openvz.org>
  23. *
  24. * Better ipc lock (kern_ipc_perm.lock) handling
  25. * Davidlohr Bueso <davidlohr.bueso@hp.com>, June 2013.
  26. */
  27. #include <linux/slab.h>
  28. #include <linux/mm.h>
  29. #include <linux/hugetlb.h>
  30. #include <linux/shm.h>
  31. #include <linux/init.h>
  32. #include <linux/file.h>
  33. #include <linux/mman.h>
  34. #include <linux/shmem_fs.h>
  35. #include <linux/security.h>
  36. #include <linux/syscalls.h>
  37. #include <linux/audit.h>
  38. #include <linux/capability.h>
  39. #include <linux/ptrace.h>
  40. #include <linux/seq_file.h>
  41. #include <linux/rwsem.h>
  42. #include <linux/nsproxy.h>
  43. #include <linux/mount.h>
  44. #include <linux/ipc_namespace.h>
  45. #include <linux/rhashtable.h>
  46. #include <linux/uaccess.h>
  47. #include "util.h"
  48. struct shmid_kernel /* private to the kernel */
  49. {
  50. struct kern_ipc_perm shm_perm;
  51. struct file *shm_file;
  52. unsigned long shm_nattch;
  53. unsigned long shm_segsz;
  54. time64_t shm_atim;
  55. time64_t shm_dtim;
  56. time64_t shm_ctim;
  57. struct pid *shm_cprid;
  58. struct pid *shm_lprid;
  59. struct user_struct *mlock_user;
  60. /*
  61. * The task created the shm object, for
  62. * task_lock(shp->shm_creator)
  63. */
  64. struct task_struct *shm_creator;
  65. /*
  66. * List by creator. task_lock(->shm_creator) required for read/write.
  67. * If list_empty(), then the creator is dead already.
  68. */
  69. struct list_head shm_clist;
  70. struct ipc_namespace *ns;
  71. } __randomize_layout;
  72. /* shm_mode upper byte flags */
  73. #define SHM_DEST 01000 /* segment will be destroyed on last detach */
  74. #define SHM_LOCKED 02000 /* segment will not be swapped */
  75. struct shm_file_data {
  76. int id;
  77. struct ipc_namespace *ns;
  78. struct file *file;
  79. const struct vm_operations_struct *vm_ops;
  80. };
  81. #define shm_file_data(file) (*((struct shm_file_data **)&(file)->private_data))
  82. static const struct file_operations shm_file_operations;
  83. static const struct vm_operations_struct shm_vm_ops;
  84. #define shm_ids(ns) ((ns)->ids[IPC_SHM_IDS])
  85. #define shm_unlock(shp) \
  86. ipc_unlock(&(shp)->shm_perm)
  87. static int newseg(struct ipc_namespace *, struct ipc_params *);
  88. static void shm_open(struct vm_area_struct *vma);
  89. static void shm_close(struct vm_area_struct *vma);
  90. static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp);
  91. #ifdef CONFIG_PROC_FS
  92. static int sysvipc_shm_proc_show(struct seq_file *s, void *it);
  93. #endif
  94. void shm_init_ns(struct ipc_namespace *ns)
  95. {
  96. ns->shm_ctlmax = SHMMAX;
  97. ns->shm_ctlall = SHMALL;
  98. ns->shm_ctlmni = SHMMNI;
  99. ns->shm_rmid_forced = 0;
  100. ns->shm_tot = 0;
  101. ipc_init_ids(&shm_ids(ns));
  102. }
  103. /*
  104. * Called with shm_ids.rwsem (writer) and the shp structure locked.
  105. * Only shm_ids.rwsem remains locked on exit.
  106. */
  107. static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
  108. {
  109. struct shmid_kernel *shp;
  110. shp = container_of(ipcp, struct shmid_kernel, shm_perm);
  111. WARN_ON(ns != shp->ns);
  112. if (shp->shm_nattch) {
  113. shp->shm_perm.mode |= SHM_DEST;
  114. /* Do not find it any more */
  115. ipc_set_key_private(&shm_ids(ns), &shp->shm_perm);
  116. shm_unlock(shp);
  117. } else
  118. shm_destroy(ns, shp);
  119. }
  120. #ifdef CONFIG_IPC_NS
  121. void shm_exit_ns(struct ipc_namespace *ns)
  122. {
  123. free_ipcs(ns, &shm_ids(ns), do_shm_rmid);
  124. idr_destroy(&ns->ids[IPC_SHM_IDS].ipcs_idr);
  125. rhashtable_destroy(&ns->ids[IPC_SHM_IDS].key_ht);
  126. }
  127. #endif
  128. static int __init ipc_ns_init(void)
  129. {
  130. shm_init_ns(&init_ipc_ns);
  131. return 0;
  132. }
  133. pure_initcall(ipc_ns_init);
  134. void __init shm_init(void)
  135. {
  136. ipc_init_proc_interface("sysvipc/shm",
  137. #if BITS_PER_LONG <= 32
  138. " key shmid perms size cpid lpid nattch uid gid cuid cgid atime dtime ctime rss swap\n",
  139. #else
  140. " key shmid perms size cpid lpid nattch uid gid cuid cgid atime dtime ctime rss swap\n",
  141. #endif
  142. IPC_SHM_IDS, sysvipc_shm_proc_show);
  143. }
  144. static inline struct shmid_kernel *shm_obtain_object(struct ipc_namespace *ns, int id)
  145. {
  146. struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);
  147. if (IS_ERR(ipcp))
  148. return ERR_CAST(ipcp);
  149. return container_of(ipcp, struct shmid_kernel, shm_perm);
  150. }
  151. static inline struct shmid_kernel *shm_obtain_object_check(struct ipc_namespace *ns, int id)
  152. {
  153. struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&shm_ids(ns), id);
  154. if (IS_ERR(ipcp))
  155. return ERR_CAST(ipcp);
  156. return container_of(ipcp, struct shmid_kernel, shm_perm);
  157. }
  158. /*
  159. * shm_lock_(check_) routines are called in the paths where the rwsem
  160. * is not necessarily held.
  161. */
  162. static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id)
  163. {
  164. struct kern_ipc_perm *ipcp;
  165. rcu_read_lock();
  166. ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);
  167. if (IS_ERR(ipcp))
  168. goto err;
  169. ipc_lock_object(ipcp);
  170. /*
  171. * ipc_rmid() may have already freed the ID while ipc_lock_object()
  172. * was spinning: here verify that the structure is still valid.
  173. * Upon races with RMID, return -EIDRM, thus indicating that
  174. * the ID points to a removed identifier.
  175. */
  176. if (ipc_valid_object(ipcp)) {
  177. /* return a locked ipc object upon success */
  178. return container_of(ipcp, struct shmid_kernel, shm_perm);
  179. }
  180. ipc_unlock_object(ipcp);
  181. ipcp = ERR_PTR(-EIDRM);
  182. err:
  183. rcu_read_unlock();
  184. /*
  185. * Callers of shm_lock() must validate the status of the returned ipc
  186. * object pointer and error out as appropriate.
  187. */
  188. return ERR_CAST(ipcp);
  189. }
  190. static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp)
  191. {
  192. rcu_read_lock();
  193. ipc_lock_object(&ipcp->shm_perm);
  194. }
  195. static void shm_rcu_free(struct rcu_head *head)
  196. {
  197. struct kern_ipc_perm *ptr = container_of(head, struct kern_ipc_perm,
  198. rcu);
  199. struct shmid_kernel *shp = container_of(ptr, struct shmid_kernel,
  200. shm_perm);
  201. security_shm_free(&shp->shm_perm);
  202. kvfree(shp);
  203. }
  204. /*
  205. * It has to be called with shp locked.
  206. * It must be called before ipc_rmid()
  207. */
  208. static inline void shm_clist_rm(struct shmid_kernel *shp)
  209. {
  210. struct task_struct *creator;
  211. /* ensure that shm_creator does not disappear */
  212. rcu_read_lock();
  213. /*
  214. * A concurrent exit_shm may do a list_del_init() as well.
  215. * Just do nothing if exit_shm already did the work
  216. */
  217. if (!list_empty(&shp->shm_clist)) {
  218. /*
  219. * shp->shm_creator is guaranteed to be valid *only*
  220. * if shp->shm_clist is not empty.
  221. */
  222. creator = shp->shm_creator;
  223. task_lock(creator);
  224. /*
  225. * list_del_init() is a nop if the entry was already removed
  226. * from the list.
  227. */
  228. list_del_init(&shp->shm_clist);
  229. task_unlock(creator);
  230. }
  231. rcu_read_unlock();
  232. }
  233. static inline void shm_rmid(struct shmid_kernel *s)
  234. {
  235. shm_clist_rm(s);
  236. ipc_rmid(&shm_ids(s->ns), &s->shm_perm);
  237. }
  238. static int __shm_open(struct vm_area_struct *vma)
  239. {
  240. struct file *file = vma->vm_file;
  241. struct shm_file_data *sfd = shm_file_data(file);
  242. struct shmid_kernel *shp;
  243. shp = shm_lock(sfd->ns, sfd->id);
  244. if (IS_ERR(shp))
  245. return PTR_ERR(shp);
  246. if (shp->shm_file != sfd->file) {
  247. /* ID was reused */
  248. shm_unlock(shp);
  249. return -EINVAL;
  250. }
  251. shp->shm_atim = ktime_get_real_seconds();
  252. ipc_update_pid(&shp->shm_lprid, task_tgid(current));
  253. shp->shm_nattch++;
  254. shm_unlock(shp);
  255. return 0;
  256. }
  257. /* This is called by fork, once for every shm attach. */
  258. static void shm_open(struct vm_area_struct *vma)
  259. {
  260. int err = __shm_open(vma);
  261. /*
  262. * We raced in the idr lookup or with shm_destroy().
  263. * Either way, the ID is busted.
  264. */
  265. WARN_ON_ONCE(err);
  266. }
  267. /*
  268. * shm_destroy - free the struct shmid_kernel
  269. *
  270. * @ns: namespace
  271. * @shp: struct to free
  272. *
  273. * It has to be called with shp and shm_ids.rwsem (writer) locked,
  274. * but returns with shp unlocked and freed.
  275. */
  276. static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
  277. {
  278. struct file *shm_file;
  279. shm_file = shp->shm_file;
  280. shp->shm_file = NULL;
  281. ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;
  282. shm_rmid(shp);
  283. shm_unlock(shp);
  284. if (!is_file_hugepages(shm_file))
  285. shmem_lock(shm_file, 0, shp->mlock_user);
  286. else if (shp->mlock_user)
  287. user_shm_unlock(i_size_read(file_inode(shm_file)),
  288. shp->mlock_user);
  289. fput(shm_file);
  290. ipc_update_pid(&shp->shm_cprid, NULL);
  291. ipc_update_pid(&shp->shm_lprid, NULL);
  292. ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);
  293. }
  294. /*
  295. * shm_may_destroy - identifies whether shm segment should be destroyed now
  296. *
  297. * Returns true if and only if there are no active users of the segment and
  298. * one of the following is true:
  299. *
  300. * 1) shmctl(id, IPC_RMID, NULL) was called for this shp
  301. *
  302. * 2) sysctl kernel.shm_rmid_forced is set to 1.
  303. */
  304. static bool shm_may_destroy(struct shmid_kernel *shp)
  305. {
  306. return (shp->shm_nattch == 0) &&
  307. (shp->ns->shm_rmid_forced ||
  308. (shp->shm_perm.mode & SHM_DEST));
  309. }
  310. /*
  311. * remove the attach descriptor vma.
  312. * free memory for segment if it is marked destroyed.
  313. * The descriptor has already been removed from the current->mm->mmap list
  314. * and will later be kfree()d.
  315. */
  316. static void shm_close(struct vm_area_struct *vma)
  317. {
  318. struct file *file = vma->vm_file;
  319. struct shm_file_data *sfd = shm_file_data(file);
  320. struct shmid_kernel *shp;
  321. struct ipc_namespace *ns = sfd->ns;
  322. down_write(&shm_ids(ns).rwsem);
  323. /* remove from the list of attaches of the shm segment */
  324. shp = shm_lock(ns, sfd->id);
  325. /*
  326. * We raced in the idr lookup or with shm_destroy().
  327. * Either way, the ID is busted.
  328. */
  329. if (WARN_ON_ONCE(IS_ERR(shp)))
  330. goto done; /* no-op */
  331. ipc_update_pid(&shp->shm_lprid, task_tgid(current));
  332. shp->shm_dtim = ktime_get_real_seconds();
  333. shp->shm_nattch--;
  334. if (shm_may_destroy(shp))
  335. shm_destroy(ns, shp);
  336. else
  337. shm_unlock(shp);
  338. done:
  339. up_write(&shm_ids(ns).rwsem);
  340. }
  341. /* Called with ns->shm_ids(ns).rwsem locked */
  342. static int shm_try_destroy_orphaned(int id, void *p, void *data)
  343. {
  344. struct ipc_namespace *ns = data;
  345. struct kern_ipc_perm *ipcp = p;
  346. struct shmid_kernel *shp = container_of(ipcp, struct shmid_kernel, shm_perm);
  347. /*
  348. * We want to destroy segments without users and with already
  349. * exit'ed originating process.
  350. *
  351. * As shp->* are changed under rwsem, it's safe to skip shp locking.
  352. */
  353. if (!list_empty(&shp->shm_clist))
  354. return 0;
  355. if (shm_may_destroy(shp)) {
  356. shm_lock_by_ptr(shp);
  357. shm_destroy(ns, shp);
  358. }
  359. return 0;
  360. }
  361. void shm_destroy_orphaned(struct ipc_namespace *ns)
  362. {
  363. down_write(&shm_ids(ns).rwsem);
  364. if (shm_ids(ns).in_use)
  365. idr_for_each(&shm_ids(ns).ipcs_idr, &shm_try_destroy_orphaned, ns);
  366. up_write(&shm_ids(ns).rwsem);
  367. }
  368. /* Locking assumes this will only be called with task == current */
  369. void exit_shm(struct task_struct *task)
  370. {
  371. for (;;) {
  372. struct shmid_kernel *shp;
  373. struct ipc_namespace *ns;
  374. task_lock(task);
  375. if (list_empty(&task->sysvshm.shm_clist)) {
  376. task_unlock(task);
  377. break;
  378. }
  379. shp = list_first_entry(&task->sysvshm.shm_clist, struct shmid_kernel,
  380. shm_clist);
  381. /*
  382. * 1) Get pointer to the ipc namespace. It is worth to say
  383. * that this pointer is guaranteed to be valid because
  384. * shp lifetime is always shorter than namespace lifetime
  385. * in which shp lives.
  386. * We taken task_lock it means that shp won't be freed.
  387. */
  388. ns = shp->ns;
  389. /*
  390. * 2) If kernel.shm_rmid_forced is not set then only keep track of
  391. * which shmids are orphaned, so that a later set of the sysctl
  392. * can clean them up.
  393. */
  394. if (!ns->shm_rmid_forced)
  395. goto unlink_continue;
  396. /*
  397. * 3) get a reference to the namespace.
  398. * The refcount could be already 0. If it is 0, then
  399. * the shm objects will be free by free_ipc_work().
  400. */
  401. ns = get_ipc_ns_not_zero(ns);
  402. if (!ns) {
  403. unlink_continue:
  404. list_del_init(&shp->shm_clist);
  405. task_unlock(task);
  406. continue;
  407. }
  408. /*
  409. * 4) get a reference to shp.
  410. * This cannot fail: shm_clist_rm() is called before
  411. * ipc_rmid(), thus the refcount cannot be 0.
  412. */
  413. WARN_ON(!ipc_rcu_getref(&shp->shm_perm));
  414. /*
  415. * 5) unlink the shm segment from the list of segments
  416. * created by current.
  417. * This must be done last. After unlinking,
  418. * only the refcounts obtained above prevent IPC_RMID
  419. * from destroying the segment or the namespace.
  420. */
  421. list_del_init(&shp->shm_clist);
  422. task_unlock(task);
  423. /*
  424. * 6) we have all references
  425. * Thus lock & if needed destroy shp.
  426. */
  427. down_write(&shm_ids(ns).rwsem);
  428. shm_lock_by_ptr(shp);
  429. /*
  430. * rcu_read_lock was implicitly taken in shm_lock_by_ptr, it's
  431. * safe to call ipc_rcu_putref here
  432. */
  433. ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);
  434. if (ipc_valid_object(&shp->shm_perm)) {
  435. if (shm_may_destroy(shp))
  436. shm_destroy(ns, shp);
  437. else
  438. shm_unlock(shp);
  439. } else {
  440. /*
  441. * Someone else deleted the shp from namespace
  442. * idr/kht while we have waited.
  443. * Just unlock and continue.
  444. */
  445. shm_unlock(shp);
  446. }
  447. up_write(&shm_ids(ns).rwsem);
  448. put_ipc_ns(ns); /* paired with get_ipc_ns_not_zero */
  449. }
  450. }
  451. static vm_fault_t shm_fault(struct vm_fault *vmf)
  452. {
  453. struct file *file = vmf->vma->vm_file;
  454. struct shm_file_data *sfd = shm_file_data(file);
  455. return sfd->vm_ops->fault(vmf);
  456. }
  457. static int shm_split(struct vm_area_struct *vma, unsigned long addr)
  458. {
  459. struct file *file = vma->vm_file;
  460. struct shm_file_data *sfd = shm_file_data(file);
  461. if (sfd->vm_ops->split)
  462. return sfd->vm_ops->split(vma, addr);
  463. return 0;
  464. }
  465. static unsigned long shm_pagesize(struct vm_area_struct *vma)
  466. {
  467. struct file *file = vma->vm_file;
  468. struct shm_file_data *sfd = shm_file_data(file);
  469. if (sfd->vm_ops->pagesize)
  470. return sfd->vm_ops->pagesize(vma);
  471. return PAGE_SIZE;
  472. }
  473. #ifdef CONFIG_NUMA
  474. static int shm_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
  475. {
  476. struct file *file = vma->vm_file;
  477. struct shm_file_data *sfd = shm_file_data(file);
  478. int err = 0;
  479. if (sfd->vm_ops->set_policy)
  480. err = sfd->vm_ops->set_policy(vma, new);
  481. return err;
  482. }
  483. static struct mempolicy *shm_get_policy(struct vm_area_struct *vma,
  484. unsigned long addr)
  485. {
  486. struct file *file = vma->vm_file;
  487. struct shm_file_data *sfd = shm_file_data(file);
  488. struct mempolicy *pol = NULL;
  489. if (sfd->vm_ops->get_policy)
  490. pol = sfd->vm_ops->get_policy(vma, addr);
  491. else if (vma->vm_policy)
  492. pol = vma->vm_policy;
  493. return pol;
  494. }
  495. #endif
  496. static int shm_mmap(struct file *file, struct vm_area_struct *vma)
  497. {
  498. struct shm_file_data *sfd = shm_file_data(file);
  499. int ret;
  500. /*
  501. * In case of remap_file_pages() emulation, the file can represent an
  502. * IPC ID that was removed, and possibly even reused by another shm
  503. * segment already. Propagate this case as an error to caller.
  504. */
  505. ret = __shm_open(vma);
  506. if (ret)
  507. return ret;
  508. ret = call_mmap(sfd->file, vma);
  509. if (ret) {
  510. shm_close(vma);
  511. return ret;
  512. }
  513. sfd->vm_ops = vma->vm_ops;
  514. #ifdef CONFIG_MMU
  515. WARN_ON(!sfd->vm_ops->fault);
  516. #endif
  517. vma->vm_ops = &shm_vm_ops;
  518. return 0;
  519. }
  520. static int shm_release(struct inode *ino, struct file *file)
  521. {
  522. struct shm_file_data *sfd = shm_file_data(file);
  523. put_ipc_ns(sfd->ns);
  524. fput(sfd->file);
  525. shm_file_data(file) = NULL;
  526. kfree(sfd);
  527. return 0;
  528. }
  529. static int shm_fsync(struct file *file, loff_t start, loff_t end, int datasync)
  530. {
  531. struct shm_file_data *sfd = shm_file_data(file);
  532. if (!sfd->file->f_op->fsync)
  533. return -EINVAL;
  534. return sfd->file->f_op->fsync(sfd->file, start, end, datasync);
  535. }
  536. static long shm_fallocate(struct file *file, int mode, loff_t offset,
  537. loff_t len)
  538. {
  539. struct shm_file_data *sfd = shm_file_data(file);
  540. if (!sfd->file->f_op->fallocate)
  541. return -EOPNOTSUPP;
  542. return sfd->file->f_op->fallocate(file, mode, offset, len);
  543. }
  544. static unsigned long shm_get_unmapped_area(struct file *file,
  545. unsigned long addr, unsigned long len, unsigned long pgoff,
  546. unsigned long flags)
  547. {
  548. struct shm_file_data *sfd = shm_file_data(file);
  549. return sfd->file->f_op->get_unmapped_area(sfd->file, addr, len,
  550. pgoff, flags);
  551. }
  552. static const struct file_operations shm_file_operations = {
  553. .mmap = shm_mmap,
  554. .fsync = shm_fsync,
  555. .release = shm_release,
  556. .get_unmapped_area = shm_get_unmapped_area,
  557. .llseek = noop_llseek,
  558. .fallocate = shm_fallocate,
  559. };
  560. /*
  561. * shm_file_operations_huge is now identical to shm_file_operations,
  562. * but we keep it distinct for the sake of is_file_shm_hugepages().
  563. */
  564. static const struct file_operations shm_file_operations_huge = {
  565. .mmap = shm_mmap,
  566. .fsync = shm_fsync,
  567. .release = shm_release,
  568. .get_unmapped_area = shm_get_unmapped_area,
  569. .llseek = noop_llseek,
  570. .fallocate = shm_fallocate,
  571. };
  572. bool is_file_shm_hugepages(struct file *file)
  573. {
  574. return file->f_op == &shm_file_operations_huge;
  575. }
  576. static const struct vm_operations_struct shm_vm_ops = {
  577. .open = shm_open, /* callback for a new vm-area open */
  578. .close = shm_close, /* callback for when the vm-area is released */
  579. .fault = shm_fault,
  580. .split = shm_split,
  581. .pagesize = shm_pagesize,
  582. #if defined(CONFIG_NUMA)
  583. .set_policy = shm_set_policy,
  584. .get_policy = shm_get_policy,
  585. #endif
  586. };
  587. /**
  588. * newseg - Create a new shared memory segment
  589. * @ns: namespace
  590. * @params: ptr to the structure that contains key, size and shmflg
  591. *
  592. * Called with shm_ids.rwsem held as a writer.
  593. */
  594. static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
  595. {
  596. key_t key = params->key;
  597. int shmflg = params->flg;
  598. size_t size = params->u.size;
  599. int error;
  600. struct shmid_kernel *shp;
  601. size_t numpages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
  602. struct file *file;
  603. char name[13];
  604. vm_flags_t acctflag = 0;
  605. if (size < SHMMIN || size > ns->shm_ctlmax)
  606. return -EINVAL;
  607. if (numpages << PAGE_SHIFT < size)
  608. return -ENOSPC;
  609. if (ns->shm_tot + numpages < ns->shm_tot ||
  610. ns->shm_tot + numpages > ns->shm_ctlall)
  611. return -ENOSPC;
  612. shp = kvmalloc(sizeof(*shp), GFP_KERNEL);
  613. if (unlikely(!shp))
  614. return -ENOMEM;
  615. shp->shm_perm.key = key;
  616. shp->shm_perm.mode = (shmflg & S_IRWXUGO);
  617. shp->mlock_user = NULL;
  618. shp->shm_perm.security = NULL;
  619. error = security_shm_alloc(&shp->shm_perm);
  620. if (error) {
  621. kvfree(shp);
  622. return error;
  623. }
  624. sprintf(name, "SYSV%08x", key);
  625. if (shmflg & SHM_HUGETLB) {
  626. struct hstate *hs;
  627. size_t hugesize;
  628. hs = hstate_sizelog((shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
  629. if (!hs) {
  630. error = -EINVAL;
  631. goto no_file;
  632. }
  633. hugesize = ALIGN(size, huge_page_size(hs));
  634. /* hugetlb_file_setup applies strict accounting */
  635. if (shmflg & SHM_NORESERVE)
  636. acctflag = VM_NORESERVE;
  637. file = hugetlb_file_setup(name, hugesize, acctflag,
  638. &shp->mlock_user, HUGETLB_SHMFS_INODE,
  639. (shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
  640. } else {
  641. /*
  642. * Do not allow no accounting for OVERCOMMIT_NEVER, even
  643. * if it's asked for.
  644. */
  645. if ((shmflg & SHM_NORESERVE) &&
  646. sysctl_overcommit_memory != OVERCOMMIT_NEVER)
  647. acctflag = VM_NORESERVE;
  648. file = shmem_kernel_file_setup(name, size, acctflag);
  649. }
  650. error = PTR_ERR(file);
  651. if (IS_ERR(file))
  652. goto no_file;
  653. shp->shm_cprid = get_pid(task_tgid(current));
  654. shp->shm_lprid = NULL;
  655. shp->shm_atim = shp->shm_dtim = 0;
  656. shp->shm_ctim = ktime_get_real_seconds();
  657. shp->shm_segsz = size;
  658. shp->shm_nattch = 0;
  659. shp->shm_file = file;
  660. shp->shm_creator = current;
  661. /* ipc_addid() locks shp upon success. */
  662. error = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni);
  663. if (error < 0)
  664. goto no_id;
  665. shp->ns = ns;
  666. task_lock(current);
  667. list_add(&shp->shm_clist, &current->sysvshm.shm_clist);
  668. task_unlock(current);
  669. /*
  670. * shmid gets reported as "inode#" in /proc/pid/maps.
  671. * proc-ps tools use this. Changing this will break them.
  672. */
  673. file_inode(file)->i_ino = shp->shm_perm.id;
  674. ns->shm_tot += numpages;
  675. error = shp->shm_perm.id;
  676. ipc_unlock_object(&shp->shm_perm);
  677. rcu_read_unlock();
  678. return error;
  679. no_id:
  680. ipc_update_pid(&shp->shm_cprid, NULL);
  681. ipc_update_pid(&shp->shm_lprid, NULL);
  682. if (is_file_hugepages(file) && shp->mlock_user)
  683. user_shm_unlock(size, shp->mlock_user);
  684. fput(file);
  685. ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);
  686. return error;
  687. no_file:
  688. call_rcu(&shp->shm_perm.rcu, shm_rcu_free);
  689. return error;
  690. }
  691. /*
  692. * Called with shm_ids.rwsem and ipcp locked.
  693. */
  694. static int shm_more_checks(struct kern_ipc_perm *ipcp, struct ipc_params *params)
  695. {
  696. struct shmid_kernel *shp;
  697. shp = container_of(ipcp, struct shmid_kernel, shm_perm);
  698. if (shp->shm_segsz < params->u.size)
  699. return -EINVAL;
  700. return 0;
  701. }
  702. long ksys_shmget(key_t key, size_t size, int shmflg)
  703. {
  704. struct ipc_namespace *ns;
  705. static const struct ipc_ops shm_ops = {
  706. .getnew = newseg,
  707. .associate = security_shm_associate,
  708. .more_checks = shm_more_checks,
  709. };
  710. struct ipc_params shm_params;
  711. ns = current->nsproxy->ipc_ns;
  712. shm_params.key = key;
  713. shm_params.flg = shmflg;
  714. shm_params.u.size = size;
  715. return ipcget(ns, &shm_ids(ns), &shm_ops, &shm_params);
  716. }
  717. SYSCALL_DEFINE3(shmget, key_t, key, size_t, size, int, shmflg)
  718. {
  719. return ksys_shmget(key, size, shmflg);
  720. }
  721. static inline unsigned long copy_shmid_to_user(void __user *buf, struct shmid64_ds *in, int version)
  722. {
  723. switch (version) {
  724. case IPC_64:
  725. return copy_to_user(buf, in, sizeof(*in));
  726. case IPC_OLD:
  727. {
  728. struct shmid_ds out;
  729. memset(&out, 0, sizeof(out));
  730. ipc64_perm_to_ipc_perm(&in->shm_perm, &out.shm_perm);
  731. out.shm_segsz = in->shm_segsz;
  732. out.shm_atime = in->shm_atime;
  733. out.shm_dtime = in->shm_dtime;
  734. out.shm_ctime = in->shm_ctime;
  735. out.shm_cpid = in->shm_cpid;
  736. out.shm_lpid = in->shm_lpid;
  737. out.shm_nattch = in->shm_nattch;
  738. return copy_to_user(buf, &out, sizeof(out));
  739. }
  740. default:
  741. return -EINVAL;
  742. }
  743. }
  744. static inline unsigned long
  745. copy_shmid_from_user(struct shmid64_ds *out, void __user *buf, int version)
  746. {
  747. switch (version) {
  748. case IPC_64:
  749. if (copy_from_user(out, buf, sizeof(*out)))
  750. return -EFAULT;
  751. return 0;
  752. case IPC_OLD:
  753. {
  754. struct shmid_ds tbuf_old;
  755. if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
  756. return -EFAULT;
  757. out->shm_perm.uid = tbuf_old.shm_perm.uid;
  758. out->shm_perm.gid = tbuf_old.shm_perm.gid;
  759. out->shm_perm.mode = tbuf_old.shm_perm.mode;
  760. return 0;
  761. }
  762. default:
  763. return -EINVAL;
  764. }
  765. }
  766. static inline unsigned long copy_shminfo_to_user(void __user *buf, struct shminfo64 *in, int version)
  767. {
  768. switch (version) {
  769. case IPC_64:
  770. return copy_to_user(buf, in, sizeof(*in));
  771. case IPC_OLD:
  772. {
  773. struct shminfo out;
  774. if (in->shmmax > INT_MAX)
  775. out.shmmax = INT_MAX;
  776. else
  777. out.shmmax = (int)in->shmmax;
  778. out.shmmin = in->shmmin;
  779. out.shmmni = in->shmmni;
  780. out.shmseg = in->shmseg;
  781. out.shmall = in->shmall;
  782. return copy_to_user(buf, &out, sizeof(out));
  783. }
  784. default:
  785. return -EINVAL;
  786. }
  787. }
  788. /*
  789. * Calculate and add used RSS and swap pages of a shm.
  790. * Called with shm_ids.rwsem held as a reader
  791. */
  792. static void shm_add_rss_swap(struct shmid_kernel *shp,
  793. unsigned long *rss_add, unsigned long *swp_add)
  794. {
  795. struct inode *inode;
  796. inode = file_inode(shp->shm_file);
  797. if (is_file_hugepages(shp->shm_file)) {
  798. struct address_space *mapping = inode->i_mapping;
  799. struct hstate *h = hstate_file(shp->shm_file);
  800. *rss_add += pages_per_huge_page(h) * mapping->nrpages;
  801. } else {
  802. #ifdef CONFIG_SHMEM
  803. struct shmem_inode_info *info = SHMEM_I(inode);
  804. spin_lock_irq(&info->lock);
  805. *rss_add += inode->i_mapping->nrpages;
  806. *swp_add += info->swapped;
  807. spin_unlock_irq(&info->lock);
  808. #else
  809. *rss_add += inode->i_mapping->nrpages;
  810. #endif
  811. }
  812. }
  813. /*
  814. * Called with shm_ids.rwsem held as a reader
  815. */
  816. static void shm_get_stat(struct ipc_namespace *ns, unsigned long *rss,
  817. unsigned long *swp)
  818. {
  819. int next_id;
  820. int total, in_use;
  821. *rss = 0;
  822. *swp = 0;
  823. in_use = shm_ids(ns).in_use;
  824. for (total = 0, next_id = 0; total < in_use; next_id++) {
  825. struct kern_ipc_perm *ipc;
  826. struct shmid_kernel *shp;
  827. ipc = idr_find(&shm_ids(ns).ipcs_idr, next_id);
  828. if (ipc == NULL)
  829. continue;
  830. shp = container_of(ipc, struct shmid_kernel, shm_perm);
  831. shm_add_rss_swap(shp, rss, swp);
  832. total++;
  833. }
  834. }
  835. /*
  836. * This function handles some shmctl commands which require the rwsem
  837. * to be held in write mode.
  838. * NOTE: no locks must be held, the rwsem is taken inside this function.
  839. */
  840. static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd,
  841. struct shmid64_ds *shmid64)
  842. {
  843. struct kern_ipc_perm *ipcp;
  844. struct shmid_kernel *shp;
  845. int err;
  846. down_write(&shm_ids(ns).rwsem);
  847. rcu_read_lock();
  848. ipcp = ipcctl_obtain_check(ns, &shm_ids(ns), shmid, cmd,
  849. &shmid64->shm_perm, 0);
  850. if (IS_ERR(ipcp)) {
  851. err = PTR_ERR(ipcp);
  852. goto out_unlock1;
  853. }
  854. shp = container_of(ipcp, struct shmid_kernel, shm_perm);
  855. err = security_shm_shmctl(&shp->shm_perm, cmd);
  856. if (err)
  857. goto out_unlock1;
  858. switch (cmd) {
  859. case IPC_RMID:
  860. ipc_lock_object(&shp->shm_perm);
  861. /* do_shm_rmid unlocks the ipc object and rcu */
  862. do_shm_rmid(ns, ipcp);
  863. goto out_up;
  864. case IPC_SET:
  865. ipc_lock_object(&shp->shm_perm);
  866. err = ipc_update_perm(&shmid64->shm_perm, ipcp);
  867. if (err)
  868. goto out_unlock0;
  869. shp->shm_ctim = ktime_get_real_seconds();
  870. break;
  871. default:
  872. err = -EINVAL;
  873. goto out_unlock1;
  874. }
  875. out_unlock0:
  876. ipc_unlock_object(&shp->shm_perm);
  877. out_unlock1:
  878. rcu_read_unlock();
  879. out_up:
  880. up_write(&shm_ids(ns).rwsem);
  881. return err;
  882. }
  883. static int shmctl_ipc_info(struct ipc_namespace *ns,
  884. struct shminfo64 *shminfo)
  885. {
  886. int err = security_shm_shmctl(NULL, IPC_INFO);
  887. if (!err) {
  888. memset(shminfo, 0, sizeof(*shminfo));
  889. shminfo->shmmni = shminfo->shmseg = ns->shm_ctlmni;
  890. shminfo->shmmax = ns->shm_ctlmax;
  891. shminfo->shmall = ns->shm_ctlall;
  892. shminfo->shmmin = SHMMIN;
  893. down_read(&shm_ids(ns).rwsem);
  894. err = ipc_get_maxidx(&shm_ids(ns));
  895. up_read(&shm_ids(ns).rwsem);
  896. if (err < 0)
  897. err = 0;
  898. }
  899. return err;
  900. }
  901. static int shmctl_shm_info(struct ipc_namespace *ns,
  902. struct shm_info *shm_info)
  903. {
  904. int err = security_shm_shmctl(NULL, SHM_INFO);
  905. if (!err) {
  906. memset(shm_info, 0, sizeof(*shm_info));
  907. down_read(&shm_ids(ns).rwsem);
  908. shm_info->used_ids = shm_ids(ns).in_use;
  909. shm_get_stat(ns, &shm_info->shm_rss, &shm_info->shm_swp);
  910. shm_info->shm_tot = ns->shm_tot;
  911. shm_info->swap_attempts = 0;
  912. shm_info->swap_successes = 0;
  913. err = ipc_get_maxidx(&shm_ids(ns));
  914. up_read(&shm_ids(ns).rwsem);
  915. if (err < 0)
  916. err = 0;
  917. }
  918. return err;
  919. }
  920. static int shmctl_stat(struct ipc_namespace *ns, int shmid,
  921. int cmd, struct shmid64_ds *tbuf)
  922. {
  923. struct shmid_kernel *shp;
  924. int err;
  925. memset(tbuf, 0, sizeof(*tbuf));
  926. rcu_read_lock();
  927. if (cmd == SHM_STAT || cmd == SHM_STAT_ANY) {
  928. shp = shm_obtain_object(ns, shmid);
  929. if (IS_ERR(shp)) {
  930. err = PTR_ERR(shp);
  931. goto out_unlock;
  932. }
  933. } else { /* IPC_STAT */
  934. shp = shm_obtain_object_check(ns, shmid);
  935. if (IS_ERR(shp)) {
  936. err = PTR_ERR(shp);
  937. goto out_unlock;
  938. }
  939. }
  940. /*
  941. * Semantically SHM_STAT_ANY ought to be identical to
  942. * that functionality provided by the /proc/sysvipc/
  943. * interface. As such, only audit these calls and
  944. * do not do traditional S_IRUGO permission checks on
  945. * the ipc object.
  946. */
  947. if (cmd == SHM_STAT_ANY)
  948. audit_ipc_obj(&shp->shm_perm);
  949. else {
  950. err = -EACCES;
  951. if (ipcperms(ns, &shp->shm_perm, S_IRUGO))
  952. goto out_unlock;
  953. }
  954. err = security_shm_shmctl(&shp->shm_perm, cmd);
  955. if (err)
  956. goto out_unlock;
  957. ipc_lock_object(&shp->shm_perm);
  958. if (!ipc_valid_object(&shp->shm_perm)) {
  959. ipc_unlock_object(&shp->shm_perm);
  960. err = -EIDRM;
  961. goto out_unlock;
  962. }
  963. kernel_to_ipc64_perm(&shp->shm_perm, &tbuf->shm_perm);
  964. tbuf->shm_segsz = shp->shm_segsz;
  965. tbuf->shm_atime = shp->shm_atim;
  966. tbuf->shm_dtime = shp->shm_dtim;
  967. tbuf->shm_ctime = shp->shm_ctim;
  968. #ifndef CONFIG_64BIT
  969. tbuf->shm_atime_high = shp->shm_atim >> 32;
  970. tbuf->shm_dtime_high = shp->shm_dtim >> 32;
  971. tbuf->shm_ctime_high = shp->shm_ctim >> 32;
  972. #endif
  973. tbuf->shm_cpid = pid_vnr(shp->shm_cprid);
  974. tbuf->shm_lpid = pid_vnr(shp->shm_lprid);
  975. tbuf->shm_nattch = shp->shm_nattch;
  976. if (cmd == IPC_STAT) {
  977. /*
  978. * As defined in SUS:
  979. * Return 0 on success
  980. */
  981. err = 0;
  982. } else {
  983. /*
  984. * SHM_STAT and SHM_STAT_ANY (both Linux specific)
  985. * Return the full id, including the sequence number
  986. */
  987. err = shp->shm_perm.id;
  988. }
  989. ipc_unlock_object(&shp->shm_perm);
  990. out_unlock:
  991. rcu_read_unlock();
  992. return err;
  993. }
  994. static int shmctl_do_lock(struct ipc_namespace *ns, int shmid, int cmd)
  995. {
  996. struct shmid_kernel *shp;
  997. struct file *shm_file;
  998. int err;
  999. rcu_read_lock();
  1000. shp = shm_obtain_object_check(ns, shmid);
  1001. if (IS_ERR(shp)) {
  1002. err = PTR_ERR(shp);
  1003. goto out_unlock1;
  1004. }
  1005. audit_ipc_obj(&(shp->shm_perm));
  1006. err = security_shm_shmctl(&shp->shm_perm, cmd);
  1007. if (err)
  1008. goto out_unlock1;
  1009. ipc_lock_object(&shp->shm_perm);
  1010. /* check if shm_destroy() is tearing down shp */
  1011. if (!ipc_valid_object(&shp->shm_perm)) {
  1012. err = -EIDRM;
  1013. goto out_unlock0;
  1014. }
  1015. if (!ns_capable(ns->user_ns, CAP_IPC_LOCK)) {
  1016. kuid_t euid = current_euid();
  1017. if (!uid_eq(euid, shp->shm_perm.uid) &&
  1018. !uid_eq(euid, shp->shm_perm.cuid)) {
  1019. err = -EPERM;
  1020. goto out_unlock0;
  1021. }
  1022. if (cmd == SHM_LOCK && !rlimit(RLIMIT_MEMLOCK)) {
  1023. err = -EPERM;
  1024. goto out_unlock0;
  1025. }
  1026. }
  1027. shm_file = shp->shm_file;
  1028. if (is_file_hugepages(shm_file))
  1029. goto out_unlock0;
  1030. if (cmd == SHM_LOCK) {
  1031. struct user_struct *user = current_user();
  1032. err = shmem_lock(shm_file, 1, user);
  1033. if (!err && !(shp->shm_perm.mode & SHM_LOCKED)) {
  1034. shp->shm_perm.mode |= SHM_LOCKED;
  1035. shp->mlock_user = user;
  1036. }
  1037. goto out_unlock0;
  1038. }
  1039. /* SHM_UNLOCK */
  1040. if (!(shp->shm_perm.mode & SHM_LOCKED))
  1041. goto out_unlock0;
  1042. shmem_lock(shm_file, 0, shp->mlock_user);
  1043. shp->shm_perm.mode &= ~SHM_LOCKED;
  1044. shp->mlock_user = NULL;
  1045. get_file(shm_file);
  1046. ipc_unlock_object(&shp->shm_perm);
  1047. rcu_read_unlock();
  1048. shmem_unlock_mapping(shm_file->f_mapping);
  1049. fput(shm_file);
  1050. return err;
  1051. out_unlock0:
  1052. ipc_unlock_object(&shp->shm_perm);
  1053. out_unlock1:
  1054. rcu_read_unlock();
  1055. return err;
  1056. }
  1057. static long ksys_shmctl(int shmid, int cmd, struct shmid_ds __user *buf, int version)
  1058. {
  1059. int err;
  1060. struct ipc_namespace *ns;
  1061. struct shmid64_ds sem64;
  1062. if (cmd < 0 || shmid < 0)
  1063. return -EINVAL;
  1064. ns = current->nsproxy->ipc_ns;
  1065. switch (cmd) {
  1066. case IPC_INFO: {
  1067. struct shminfo64 shminfo;
  1068. err = shmctl_ipc_info(ns, &shminfo);
  1069. if (err < 0)
  1070. return err;
  1071. if (copy_shminfo_to_user(buf, &shminfo, version))
  1072. err = -EFAULT;
  1073. return err;
  1074. }
  1075. case SHM_INFO: {
  1076. struct shm_info shm_info;
  1077. err = shmctl_shm_info(ns, &shm_info);
  1078. if (err < 0)
  1079. return err;
  1080. if (copy_to_user(buf, &shm_info, sizeof(shm_info)))
  1081. err = -EFAULT;
  1082. return err;
  1083. }
  1084. case SHM_STAT:
  1085. case SHM_STAT_ANY:
  1086. case IPC_STAT: {
  1087. err = shmctl_stat(ns, shmid, cmd, &sem64);
  1088. if (err < 0)
  1089. return err;
  1090. if (copy_shmid_to_user(buf, &sem64, version))
  1091. err = -EFAULT;
  1092. return err;
  1093. }
  1094. case IPC_SET:
  1095. if (copy_shmid_from_user(&sem64, buf, version))
  1096. return -EFAULT;
  1097. fallthrough;
  1098. case IPC_RMID:
  1099. return shmctl_down(ns, shmid, cmd, &sem64);
  1100. case SHM_LOCK:
  1101. case SHM_UNLOCK:
  1102. return shmctl_do_lock(ns, shmid, cmd);
  1103. default:
  1104. return -EINVAL;
  1105. }
  1106. }
  1107. SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
  1108. {
  1109. return ksys_shmctl(shmid, cmd, buf, IPC_64);
  1110. }
  1111. #ifdef CONFIG_ARCH_WANT_IPC_PARSE_VERSION
  1112. long ksys_old_shmctl(int shmid, int cmd, struct shmid_ds __user *buf)
  1113. {
  1114. int version = ipc_parse_version(&cmd);
  1115. return ksys_shmctl(shmid, cmd, buf, version);
  1116. }
  1117. SYSCALL_DEFINE3(old_shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
  1118. {
  1119. return ksys_old_shmctl(shmid, cmd, buf);
  1120. }
  1121. #endif
  1122. #ifdef CONFIG_COMPAT
  1123. struct compat_shmid_ds {
  1124. struct compat_ipc_perm shm_perm;
  1125. int shm_segsz;
  1126. old_time32_t shm_atime;
  1127. old_time32_t shm_dtime;
  1128. old_time32_t shm_ctime;
  1129. compat_ipc_pid_t shm_cpid;
  1130. compat_ipc_pid_t shm_lpid;
  1131. unsigned short shm_nattch;
  1132. unsigned short shm_unused;
  1133. compat_uptr_t shm_unused2;
  1134. compat_uptr_t shm_unused3;
  1135. };
  1136. struct compat_shminfo64 {
  1137. compat_ulong_t shmmax;
  1138. compat_ulong_t shmmin;
  1139. compat_ulong_t shmmni;
  1140. compat_ulong_t shmseg;
  1141. compat_ulong_t shmall;
  1142. compat_ulong_t __unused1;
  1143. compat_ulong_t __unused2;
  1144. compat_ulong_t __unused3;
  1145. compat_ulong_t __unused4;
  1146. };
  1147. struct compat_shm_info {
  1148. compat_int_t used_ids;
  1149. compat_ulong_t shm_tot, shm_rss, shm_swp;
  1150. compat_ulong_t swap_attempts, swap_successes;
  1151. };
  1152. static int copy_compat_shminfo_to_user(void __user *buf, struct shminfo64 *in,
  1153. int version)
  1154. {
  1155. if (in->shmmax > INT_MAX)
  1156. in->shmmax = INT_MAX;
  1157. if (version == IPC_64) {
  1158. struct compat_shminfo64 info;
  1159. memset(&info, 0, sizeof(info));
  1160. info.shmmax = in->shmmax;
  1161. info.shmmin = in->shmmin;
  1162. info.shmmni = in->shmmni;
  1163. info.shmseg = in->shmseg;
  1164. info.shmall = in->shmall;
  1165. return copy_to_user(buf, &info, sizeof(info));
  1166. } else {
  1167. struct shminfo info;
  1168. memset(&info, 0, sizeof(info));
  1169. info.shmmax = in->shmmax;
  1170. info.shmmin = in->shmmin;
  1171. info.shmmni = in->shmmni;
  1172. info.shmseg = in->shmseg;
  1173. info.shmall = in->shmall;
  1174. return copy_to_user(buf, &info, sizeof(info));
  1175. }
  1176. }
  1177. static int put_compat_shm_info(struct shm_info *ip,
  1178. struct compat_shm_info __user *uip)
  1179. {
  1180. struct compat_shm_info info;
  1181. memset(&info, 0, sizeof(info));
  1182. info.used_ids = ip->used_ids;
  1183. info.shm_tot = ip->shm_tot;
  1184. info.shm_rss = ip->shm_rss;
  1185. info.shm_swp = ip->shm_swp;
  1186. info.swap_attempts = ip->swap_attempts;
  1187. info.swap_successes = ip->swap_successes;
  1188. return copy_to_user(uip, &info, sizeof(info));
  1189. }
  1190. static int copy_compat_shmid_to_user(void __user *buf, struct shmid64_ds *in,
  1191. int version)
  1192. {
  1193. if (version == IPC_64) {
  1194. struct compat_shmid64_ds v;
  1195. memset(&v, 0, sizeof(v));
  1196. to_compat_ipc64_perm(&v.shm_perm, &in->shm_perm);
  1197. v.shm_atime = lower_32_bits(in->shm_atime);
  1198. v.shm_atime_high = upper_32_bits(in->shm_atime);
  1199. v.shm_dtime = lower_32_bits(in->shm_dtime);
  1200. v.shm_dtime_high = upper_32_bits(in->shm_dtime);
  1201. v.shm_ctime = lower_32_bits(in->shm_ctime);
  1202. v.shm_ctime_high = upper_32_bits(in->shm_ctime);
  1203. v.shm_segsz = in->shm_segsz;
  1204. v.shm_nattch = in->shm_nattch;
  1205. v.shm_cpid = in->shm_cpid;
  1206. v.shm_lpid = in->shm_lpid;
  1207. return copy_to_user(buf, &v, sizeof(v));
  1208. } else {
  1209. struct compat_shmid_ds v;
  1210. memset(&v, 0, sizeof(v));
  1211. to_compat_ipc_perm(&v.shm_perm, &in->shm_perm);
  1212. v.shm_perm.key = in->shm_perm.key;
  1213. v.shm_atime = in->shm_atime;
  1214. v.shm_dtime = in->shm_dtime;
  1215. v.shm_ctime = in->shm_ctime;
  1216. v.shm_segsz = in->shm_segsz;
  1217. v.shm_nattch = in->shm_nattch;
  1218. v.shm_cpid = in->shm_cpid;
  1219. v.shm_lpid = in->shm_lpid;
  1220. return copy_to_user(buf, &v, sizeof(v));
  1221. }
  1222. }
  1223. static int copy_compat_shmid_from_user(struct shmid64_ds *out, void __user *buf,
  1224. int version)
  1225. {
  1226. memset(out, 0, sizeof(*out));
  1227. if (version == IPC_64) {
  1228. struct compat_shmid64_ds __user *p = buf;
  1229. return get_compat_ipc64_perm(&out->shm_perm, &p->shm_perm);
  1230. } else {
  1231. struct compat_shmid_ds __user *p = buf;
  1232. return get_compat_ipc_perm(&out->shm_perm, &p->shm_perm);
  1233. }
  1234. }
  1235. static long compat_ksys_shmctl(int shmid, int cmd, void __user *uptr, int version)
  1236. {
  1237. struct ipc_namespace *ns;
  1238. struct shmid64_ds sem64;
  1239. int err;
  1240. ns = current->nsproxy->ipc_ns;
  1241. if (cmd < 0 || shmid < 0)
  1242. return -EINVAL;
  1243. switch (cmd) {
  1244. case IPC_INFO: {
  1245. struct shminfo64 shminfo;
  1246. err = shmctl_ipc_info(ns, &shminfo);
  1247. if (err < 0)
  1248. return err;
  1249. if (copy_compat_shminfo_to_user(uptr, &shminfo, version))
  1250. err = -EFAULT;
  1251. return err;
  1252. }
  1253. case SHM_INFO: {
  1254. struct shm_info shm_info;
  1255. err = shmctl_shm_info(ns, &shm_info);
  1256. if (err < 0)
  1257. return err;
  1258. if (put_compat_shm_info(&shm_info, uptr))
  1259. err = -EFAULT;
  1260. return err;
  1261. }
  1262. case IPC_STAT:
  1263. case SHM_STAT_ANY:
  1264. case SHM_STAT:
  1265. err = shmctl_stat(ns, shmid, cmd, &sem64);
  1266. if (err < 0)
  1267. return err;
  1268. if (copy_compat_shmid_to_user(uptr, &sem64, version))
  1269. err = -EFAULT;
  1270. return err;
  1271. case IPC_SET:
  1272. if (copy_compat_shmid_from_user(&sem64, uptr, version))
  1273. return -EFAULT;
  1274. fallthrough;
  1275. case IPC_RMID:
  1276. return shmctl_down(ns, shmid, cmd, &sem64);
  1277. case SHM_LOCK:
  1278. case SHM_UNLOCK:
  1279. return shmctl_do_lock(ns, shmid, cmd);
  1280. default:
  1281. return -EINVAL;
  1282. }
  1283. return err;
  1284. }
  1285. COMPAT_SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, void __user *, uptr)
  1286. {
  1287. return compat_ksys_shmctl(shmid, cmd, uptr, IPC_64);
  1288. }
  1289. #ifdef CONFIG_ARCH_WANT_COMPAT_IPC_PARSE_VERSION
  1290. long compat_ksys_old_shmctl(int shmid, int cmd, void __user *uptr)
  1291. {
  1292. int version = compat_ipc_parse_version(&cmd);
  1293. return compat_ksys_shmctl(shmid, cmd, uptr, version);
  1294. }
  1295. COMPAT_SYSCALL_DEFINE3(old_shmctl, int, shmid, int, cmd, void __user *, uptr)
  1296. {
  1297. return compat_ksys_old_shmctl(shmid, cmd, uptr);
  1298. }
  1299. #endif
  1300. #endif
  1301. /*
  1302. * Fix shmaddr, allocate descriptor, map shm, add attach descriptor to lists.
  1303. *
  1304. * NOTE! Despite the name, this is NOT a direct system call entrypoint. The
  1305. * "raddr" thing points to kernel space, and there has to be a wrapper around
  1306. * this.
  1307. */
  1308. long do_shmat(int shmid, char __user *shmaddr, int shmflg,
  1309. ulong *raddr, unsigned long shmlba)
  1310. {
  1311. struct shmid_kernel *shp;
  1312. unsigned long addr = (unsigned long)shmaddr;
  1313. unsigned long size;
  1314. struct file *file, *base;
  1315. int err;
  1316. unsigned long flags = MAP_SHARED;
  1317. unsigned long prot;
  1318. int acc_mode;
  1319. struct ipc_namespace *ns;
  1320. struct shm_file_data *sfd;
  1321. int f_flags;
  1322. unsigned long populate = 0;
  1323. err = -EINVAL;
  1324. if (shmid < 0)
  1325. goto out;
  1326. if (addr) {
  1327. if (addr & (shmlba - 1)) {
  1328. if (shmflg & SHM_RND) {
  1329. addr &= ~(shmlba - 1); /* round down */
  1330. /*
  1331. * Ensure that the round-down is non-nil
  1332. * when remapping. This can happen for
  1333. * cases when addr < shmlba.
  1334. */
  1335. if (!addr && (shmflg & SHM_REMAP))
  1336. goto out;
  1337. } else
  1338. #ifndef __ARCH_FORCE_SHMLBA
  1339. if (addr & ~PAGE_MASK)
  1340. #endif
  1341. goto out;
  1342. }
  1343. flags |= MAP_FIXED;
  1344. } else if ((shmflg & SHM_REMAP))
  1345. goto out;
  1346. if (shmflg & SHM_RDONLY) {
  1347. prot = PROT_READ;
  1348. acc_mode = S_IRUGO;
  1349. f_flags = O_RDONLY;
  1350. } else {
  1351. prot = PROT_READ | PROT_WRITE;
  1352. acc_mode = S_IRUGO | S_IWUGO;
  1353. f_flags = O_RDWR;
  1354. }
  1355. if (shmflg & SHM_EXEC) {
  1356. prot |= PROT_EXEC;
  1357. acc_mode |= S_IXUGO;
  1358. }
  1359. /*
  1360. * We cannot rely on the fs check since SYSV IPC does have an
  1361. * additional creator id...
  1362. */
  1363. ns = current->nsproxy->ipc_ns;
  1364. rcu_read_lock();
  1365. shp = shm_obtain_object_check(ns, shmid);
  1366. if (IS_ERR(shp)) {
  1367. err = PTR_ERR(shp);
  1368. goto out_unlock;
  1369. }
  1370. err = -EACCES;
  1371. if (ipcperms(ns, &shp->shm_perm, acc_mode))
  1372. goto out_unlock;
  1373. err = security_shm_shmat(&shp->shm_perm, shmaddr, shmflg);
  1374. if (err)
  1375. goto out_unlock;
  1376. ipc_lock_object(&shp->shm_perm);
  1377. /* check if shm_destroy() is tearing down shp */
  1378. if (!ipc_valid_object(&shp->shm_perm)) {
  1379. ipc_unlock_object(&shp->shm_perm);
  1380. err = -EIDRM;
  1381. goto out_unlock;
  1382. }
  1383. /*
  1384. * We need to take a reference to the real shm file to prevent the
  1385. * pointer from becoming stale in cases where the lifetime of the outer
  1386. * file extends beyond that of the shm segment. It's not usually
  1387. * possible, but it can happen during remap_file_pages() emulation as
  1388. * that unmaps the memory, then does ->mmap() via file reference only.
  1389. * We'll deny the ->mmap() if the shm segment was since removed, but to
  1390. * detect shm ID reuse we need to compare the file pointers.
  1391. */
  1392. base = get_file(shp->shm_file);
  1393. shp->shm_nattch++;
  1394. size = i_size_read(file_inode(base));
  1395. ipc_unlock_object(&shp->shm_perm);
  1396. rcu_read_unlock();
  1397. err = -ENOMEM;
  1398. sfd = kzalloc(sizeof(*sfd), GFP_KERNEL);
  1399. if (!sfd) {
  1400. fput(base);
  1401. goto out_nattch;
  1402. }
  1403. file = alloc_file_clone(base, f_flags,
  1404. is_file_hugepages(base) ?
  1405. &shm_file_operations_huge :
  1406. &shm_file_operations);
  1407. err = PTR_ERR(file);
  1408. if (IS_ERR(file)) {
  1409. kfree(sfd);
  1410. fput(base);
  1411. goto out_nattch;
  1412. }
  1413. sfd->id = shp->shm_perm.id;
  1414. sfd->ns = get_ipc_ns(ns);
  1415. sfd->file = base;
  1416. sfd->vm_ops = NULL;
  1417. file->private_data = sfd;
  1418. err = security_mmap_file(file, prot, flags);
  1419. if (err)
  1420. goto out_fput;
  1421. if (mmap_write_lock_killable(current->mm)) {
  1422. err = -EINTR;
  1423. goto out_fput;
  1424. }
  1425. if (addr && !(shmflg & SHM_REMAP)) {
  1426. err = -EINVAL;
  1427. if (addr + size < addr)
  1428. goto invalid;
  1429. if (find_vma_intersection(current->mm, addr, addr + size))
  1430. goto invalid;
  1431. }
  1432. addr = do_mmap(file, addr, size, prot, flags, 0, &populate, NULL);
  1433. *raddr = addr;
  1434. err = 0;
  1435. if (IS_ERR_VALUE(addr))
  1436. err = (long)addr;
  1437. invalid:
  1438. mmap_write_unlock(current->mm);
  1439. if (populate)
  1440. mm_populate(addr, populate);
  1441. out_fput:
  1442. fput(file);
  1443. out_nattch:
  1444. down_write(&shm_ids(ns).rwsem);
  1445. shp = shm_lock(ns, shmid);
  1446. shp->shm_nattch--;
  1447. if (shm_may_destroy(shp))
  1448. shm_destroy(ns, shp);
  1449. else
  1450. shm_unlock(shp);
  1451. up_write(&shm_ids(ns).rwsem);
  1452. return err;
  1453. out_unlock:
  1454. rcu_read_unlock();
  1455. out:
  1456. return err;
  1457. }
  1458. SYSCALL_DEFINE3(shmat, int, shmid, char __user *, shmaddr, int, shmflg)
  1459. {
  1460. unsigned long ret;
  1461. long err;
  1462. err = do_shmat(shmid, shmaddr, shmflg, &ret, SHMLBA);
  1463. if (err)
  1464. return err;
  1465. force_successful_syscall_return();
  1466. return (long)ret;
  1467. }
  1468. #ifdef CONFIG_COMPAT
  1469. #ifndef COMPAT_SHMLBA
  1470. #define COMPAT_SHMLBA SHMLBA
  1471. #endif
  1472. COMPAT_SYSCALL_DEFINE3(shmat, int, shmid, compat_uptr_t, shmaddr, int, shmflg)
  1473. {
  1474. unsigned long ret;
  1475. long err;
  1476. err = do_shmat(shmid, compat_ptr(shmaddr), shmflg, &ret, COMPAT_SHMLBA);
  1477. if (err)
  1478. return err;
  1479. force_successful_syscall_return();
  1480. return (long)ret;
  1481. }
  1482. #endif
  1483. /*
  1484. * detach and kill segment if marked destroyed.
  1485. * The work is done in shm_close.
  1486. */
  1487. long ksys_shmdt(char __user *shmaddr)
  1488. {
  1489. struct mm_struct *mm = current->mm;
  1490. struct vm_area_struct *vma;
  1491. unsigned long addr = (unsigned long)shmaddr;
  1492. int retval = -EINVAL;
  1493. #ifdef CONFIG_MMU
  1494. loff_t size = 0;
  1495. struct file *file;
  1496. struct vm_area_struct *next;
  1497. #endif
  1498. if (addr & ~PAGE_MASK)
  1499. return retval;
  1500. if (mmap_write_lock_killable(mm))
  1501. return -EINTR;
  1502. /*
  1503. * This function tries to be smart and unmap shm segments that
  1504. * were modified by partial mlock or munmap calls:
  1505. * - It first determines the size of the shm segment that should be
  1506. * unmapped: It searches for a vma that is backed by shm and that
  1507. * started at address shmaddr. It records it's size and then unmaps
  1508. * it.
  1509. * - Then it unmaps all shm vmas that started at shmaddr and that
  1510. * are within the initially determined size and that are from the
  1511. * same shm segment from which we determined the size.
  1512. * Errors from do_munmap are ignored: the function only fails if
  1513. * it's called with invalid parameters or if it's called to unmap
  1514. * a part of a vma. Both calls in this function are for full vmas,
  1515. * the parameters are directly copied from the vma itself and always
  1516. * valid - therefore do_munmap cannot fail. (famous last words?)
  1517. */
  1518. /*
  1519. * If it had been mremap()'d, the starting address would not
  1520. * match the usual checks anyway. So assume all vma's are
  1521. * above the starting address given.
  1522. */
  1523. vma = find_vma(mm, addr);
  1524. #ifdef CONFIG_MMU
  1525. while (vma) {
  1526. next = vma->vm_next;
  1527. /*
  1528. * Check if the starting address would match, i.e. it's
  1529. * a fragment created by mprotect() and/or munmap(), or it
  1530. * otherwise it starts at this address with no hassles.
  1531. */
  1532. if ((vma->vm_ops == &shm_vm_ops) &&
  1533. (vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) {
  1534. /*
  1535. * Record the file of the shm segment being
  1536. * unmapped. With mremap(), someone could place
  1537. * page from another segment but with equal offsets
  1538. * in the range we are unmapping.
  1539. */
  1540. file = vma->vm_file;
  1541. size = i_size_read(file_inode(vma->vm_file));
  1542. do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
  1543. /*
  1544. * We discovered the size of the shm segment, so
  1545. * break out of here and fall through to the next
  1546. * loop that uses the size information to stop
  1547. * searching for matching vma's.
  1548. */
  1549. retval = 0;
  1550. vma = next;
  1551. break;
  1552. }
  1553. vma = next;
  1554. }
  1555. /*
  1556. * We need look no further than the maximum address a fragment
  1557. * could possibly have landed at. Also cast things to loff_t to
  1558. * prevent overflows and make comparisons vs. equal-width types.
  1559. */
  1560. size = PAGE_ALIGN(size);
  1561. while (vma && (loff_t)(vma->vm_end - addr) <= size) {
  1562. next = vma->vm_next;
  1563. /* finding a matching vma now does not alter retval */
  1564. if ((vma->vm_ops == &shm_vm_ops) &&
  1565. ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) &&
  1566. (vma->vm_file == file))
  1567. do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
  1568. vma = next;
  1569. }
  1570. #else /* CONFIG_MMU */
  1571. /* under NOMMU conditions, the exact address to be destroyed must be
  1572. * given
  1573. */
  1574. if (vma && vma->vm_start == addr && vma->vm_ops == &shm_vm_ops) {
  1575. do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
  1576. retval = 0;
  1577. }
  1578. #endif
  1579. mmap_write_unlock(mm);
  1580. return retval;
  1581. }
  1582. SYSCALL_DEFINE1(shmdt, char __user *, shmaddr)
  1583. {
  1584. return ksys_shmdt(shmaddr);
  1585. }
  1586. #ifdef CONFIG_PROC_FS
  1587. static int sysvipc_shm_proc_show(struct seq_file *s, void *it)
  1588. {
  1589. struct pid_namespace *pid_ns = ipc_seq_pid_ns(s);
  1590. struct user_namespace *user_ns = seq_user_ns(s);
  1591. struct kern_ipc_perm *ipcp = it;
  1592. struct shmid_kernel *shp;
  1593. unsigned long rss = 0, swp = 0;
  1594. shp = container_of(ipcp, struct shmid_kernel, shm_perm);
  1595. shm_add_rss_swap(shp, &rss, &swp);
  1596. #if BITS_PER_LONG <= 32
  1597. #define SIZE_SPEC "%10lu"
  1598. #else
  1599. #define SIZE_SPEC "%21lu"
  1600. #endif
  1601. seq_printf(s,
  1602. "%10d %10d %4o " SIZE_SPEC " %5u %5u "
  1603. "%5lu %5u %5u %5u %5u %10llu %10llu %10llu "
  1604. SIZE_SPEC " " SIZE_SPEC "\n",
  1605. shp->shm_perm.key,
  1606. shp->shm_perm.id,
  1607. shp->shm_perm.mode,
  1608. shp->shm_segsz,
  1609. pid_nr_ns(shp->shm_cprid, pid_ns),
  1610. pid_nr_ns(shp->shm_lprid, pid_ns),
  1611. shp->shm_nattch,
  1612. from_kuid_munged(user_ns, shp->shm_perm.uid),
  1613. from_kgid_munged(user_ns, shp->shm_perm.gid),
  1614. from_kuid_munged(user_ns, shp->shm_perm.cuid),
  1615. from_kgid_munged(user_ns, shp->shm_perm.cgid),
  1616. shp->shm_atim,
  1617. shp->shm_dtim,
  1618. shp->shm_ctim,
  1619. rss * PAGE_SIZE,
  1620. swp * PAGE_SIZE);
  1621. return 0;
  1622. }
  1623. #endif