shmem.c 65 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605
  1. /*
  2. * Resizable virtual memory filesystem for Linux.
  3. *
  4. * Copyright (C) 2000 Linus Torvalds.
  5. * 2000 Transmeta Corp.
  6. * 2000-2001 Christoph Rohland
  7. * 2000-2001 SAP AG
  8. * 2002 Red Hat Inc.
  9. * Copyright (C) 2002-2005 Hugh Dickins.
  10. * Copyright (C) 2002-2005 VERITAS Software Corporation.
  11. * Copyright (C) 2004 Andi Kleen, SuSE Labs
  12. *
  13. * Extended attribute support for tmpfs:
  14. * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
  15. * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
  16. *
  17. * This file is released under the GPL.
  18. */
  19. /*
  20. * This virtual memory filesystem is heavily based on the ramfs. It
  21. * extends ramfs by the ability to use swap and honor resource limits
  22. * which makes it a completely usable filesystem.
  23. */
  24. #include <linux/module.h>
  25. #include <linux/init.h>
  26. #include <linux/fs.h>
  27. #include <linux/xattr.h>
  28. #include <linux/generic_acl.h>
  29. #include <linux/mm.h>
  30. #include <linux/mman.h>
  31. #include <linux/file.h>
  32. #include <linux/swap.h>
  33. #include <linux/pagemap.h>
  34. #include <linux/string.h>
  35. #include <linux/slab.h>
  36. #include <linux/backing-dev.h>
  37. #include <linux/shmem_fs.h>
  38. #include <linux/mount.h>
  39. #include <linux/writeback.h>
  40. #include <linux/vfs.h>
  41. #include <linux/blkdev.h>
  42. #include <linux/security.h>
  43. #include <linux/swapops.h>
  44. #include <linux/mempolicy.h>
  45. #include <linux/namei.h>
  46. #include <linux/ctype.h>
  47. #include <linux/migrate.h>
  48. #include <linux/highmem.h>
  49. #include <linux/backing-dev.h>
  50. #include <asm/uaccess.h>
  51. #include <asm/div64.h>
  52. #include <asm/pgtable.h>
  53. /* This magic number is used in glibc for posix shared memory */
  54. #define TMPFS_MAGIC 0x01021994
  55. #define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
  56. #define ENTRIES_PER_PAGEPAGE (ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
  57. #define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
  58. #define SHMEM_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
  59. #define SHMEM_MAX_BYTES ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT)
  60. #define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
  61. /* info->flags needs VM_flags to handle pagein/truncate races efficiently */
  62. #define SHMEM_PAGEIN VM_READ
  63. #define SHMEM_TRUNCATE VM_WRITE
  64. /* Definition to limit shmem_truncate's steps between cond_rescheds */
  65. #define LATENCY_LIMIT 64
  66. /* Pretend that each entry is of this size in directory's i_size */
  67. #define BOGO_DIRENT_SIZE 20
  68. /* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */
  69. enum sgp_type {
  70. SGP_QUICK, /* don't try more than file page cache lookup */
  71. SGP_READ, /* don't exceed i_size, don't allocate page */
  72. SGP_CACHE, /* don't exceed i_size, may allocate page */
  73. SGP_WRITE, /* may exceed i_size, may allocate page */
  74. };
  75. static int shmem_getpage(struct inode *inode, unsigned long idx,
  76. struct page **pagep, enum sgp_type sgp, int *type);
  77. static inline struct page *shmem_dir_alloc(gfp_t gfp_mask)
  78. {
  79. /*
  80. * The above definition of ENTRIES_PER_PAGE, and the use of
  81. * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE:
  82. * might be reconsidered if it ever diverges from PAGE_SIZE.
  83. */
  84. return alloc_pages(gfp_mask, PAGE_CACHE_SHIFT-PAGE_SHIFT);
  85. }
  86. static inline void shmem_dir_free(struct page *page)
  87. {
  88. __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT);
  89. }
  90. static struct page **shmem_dir_map(struct page *page)
  91. {
  92. return (struct page **)kmap_atomic(page, KM_USER0);
  93. }
  94. static inline void shmem_dir_unmap(struct page **dir)
  95. {
  96. kunmap_atomic(dir, KM_USER0);
  97. }
  98. static swp_entry_t *shmem_swp_map(struct page *page)
  99. {
  100. return (swp_entry_t *)kmap_atomic(page, KM_USER1);
  101. }
  102. static inline void shmem_swp_balance_unmap(void)
  103. {
  104. /*
  105. * When passing a pointer to an i_direct entry, to code which
  106. * also handles indirect entries and so will shmem_swp_unmap,
  107. * we must arrange for the preempt count to remain in balance.
  108. * What kmap_atomic of a lowmem page does depends on config
  109. * and architecture, so pretend to kmap_atomic some lowmem page.
  110. */
  111. (void) kmap_atomic(ZERO_PAGE(0), KM_USER1);
  112. }
  113. static inline void shmem_swp_unmap(swp_entry_t *entry)
  114. {
  115. kunmap_atomic(entry, KM_USER1);
  116. }
  117. static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
  118. {
  119. return sb->s_fs_info;
  120. }
  121. /*
  122. * shmem_file_setup pre-accounts the whole fixed size of a VM object,
  123. * for shared memory and for shared anonymous (/dev/zero) mappings
  124. * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
  125. * consistent with the pre-accounting of private mappings ...
  126. */
  127. static inline int shmem_acct_size(unsigned long flags, loff_t size)
  128. {
  129. return (flags & VM_ACCOUNT)?
  130. security_vm_enough_memory(VM_ACCT(size)): 0;
  131. }
  132. static inline void shmem_unacct_size(unsigned long flags, loff_t size)
  133. {
  134. if (flags & VM_ACCOUNT)
  135. vm_unacct_memory(VM_ACCT(size));
  136. }
  137. /*
  138. * ... whereas tmpfs objects are accounted incrementally as
  139. * pages are allocated, in order to allow huge sparse files.
  140. * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
  141. * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
  142. */
  143. static inline int shmem_acct_block(unsigned long flags)
  144. {
  145. return (flags & VM_ACCOUNT)?
  146. 0: security_vm_enough_memory(VM_ACCT(PAGE_CACHE_SIZE));
  147. }
  148. static inline void shmem_unacct_blocks(unsigned long flags, long pages)
  149. {
  150. if (!(flags & VM_ACCOUNT))
  151. vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
  152. }
  153. static const struct super_operations shmem_ops;
  154. static const struct address_space_operations shmem_aops;
  155. static const struct file_operations shmem_file_operations;
  156. static const struct inode_operations shmem_inode_operations;
  157. static const struct inode_operations shmem_dir_inode_operations;
  158. static const struct inode_operations shmem_special_inode_operations;
  159. static struct vm_operations_struct shmem_vm_ops;
  160. static struct backing_dev_info shmem_backing_dev_info __read_mostly = {
  161. .ra_pages = 0, /* No readahead */
  162. .capabilities = BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_WRITEBACK,
  163. .unplug_io_fn = default_unplug_io_fn,
  164. };
  165. static LIST_HEAD(shmem_swaplist);
  166. static DEFINE_SPINLOCK(shmem_swaplist_lock);
  167. static void shmem_free_blocks(struct inode *inode, long pages)
  168. {
  169. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  170. if (sbinfo->max_blocks) {
  171. spin_lock(&sbinfo->stat_lock);
  172. sbinfo->free_blocks += pages;
  173. inode->i_blocks -= pages*BLOCKS_PER_PAGE;
  174. spin_unlock(&sbinfo->stat_lock);
  175. }
  176. }
  177. /*
  178. * shmem_recalc_inode - recalculate the size of an inode
  179. *
  180. * @inode: inode to recalc
  181. *
  182. * We have to calculate the free blocks since the mm can drop
  183. * undirtied hole pages behind our back.
  184. *
  185. * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
  186. * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
  187. *
  188. * It has to be called with the spinlock held.
  189. */
  190. static void shmem_recalc_inode(struct inode *inode)
  191. {
  192. struct shmem_inode_info *info = SHMEM_I(inode);
  193. long freed;
  194. freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
  195. if (freed > 0) {
  196. info->alloced -= freed;
  197. shmem_unacct_blocks(info->flags, freed);
  198. shmem_free_blocks(inode, freed);
  199. }
  200. }
  201. /*
  202. * shmem_swp_entry - find the swap vector position in the info structure
  203. *
  204. * @info: info structure for the inode
  205. * @index: index of the page to find
  206. * @page: optional page to add to the structure. Has to be preset to
  207. * all zeros
  208. *
  209. * If there is no space allocated yet it will return NULL when
  210. * page is NULL, else it will use the page for the needed block,
  211. * setting it to NULL on return to indicate that it has been used.
  212. *
  213. * The swap vector is organized the following way:
  214. *
  215. * There are SHMEM_NR_DIRECT entries directly stored in the
  216. * shmem_inode_info structure. So small files do not need an addional
  217. * allocation.
  218. *
  219. * For pages with index > SHMEM_NR_DIRECT there is the pointer
  220. * i_indirect which points to a page which holds in the first half
  221. * doubly indirect blocks, in the second half triple indirect blocks:
  222. *
  223. * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the
  224. * following layout (for SHMEM_NR_DIRECT == 16):
  225. *
  226. * i_indirect -> dir --> 16-19
  227. * | +-> 20-23
  228. * |
  229. * +-->dir2 --> 24-27
  230. * | +-> 28-31
  231. * | +-> 32-35
  232. * | +-> 36-39
  233. * |
  234. * +-->dir3 --> 40-43
  235. * +-> 44-47
  236. * +-> 48-51
  237. * +-> 52-55
  238. */
  239. static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page)
  240. {
  241. unsigned long offset;
  242. struct page **dir;
  243. struct page *subdir;
  244. if (index < SHMEM_NR_DIRECT) {
  245. shmem_swp_balance_unmap();
  246. return info->i_direct+index;
  247. }
  248. if (!info->i_indirect) {
  249. if (page) {
  250. info->i_indirect = *page;
  251. *page = NULL;
  252. }
  253. return NULL; /* need another page */
  254. }
  255. index -= SHMEM_NR_DIRECT;
  256. offset = index % ENTRIES_PER_PAGE;
  257. index /= ENTRIES_PER_PAGE;
  258. dir = shmem_dir_map(info->i_indirect);
  259. if (index >= ENTRIES_PER_PAGE/2) {
  260. index -= ENTRIES_PER_PAGE/2;
  261. dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE;
  262. index %= ENTRIES_PER_PAGE;
  263. subdir = *dir;
  264. if (!subdir) {
  265. if (page) {
  266. *dir = *page;
  267. *page = NULL;
  268. }
  269. shmem_dir_unmap(dir);
  270. return NULL; /* need another page */
  271. }
  272. shmem_dir_unmap(dir);
  273. dir = shmem_dir_map(subdir);
  274. }
  275. dir += index;
  276. subdir = *dir;
  277. if (!subdir) {
  278. if (!page || !(subdir = *page)) {
  279. shmem_dir_unmap(dir);
  280. return NULL; /* need a page */
  281. }
  282. *dir = subdir;
  283. *page = NULL;
  284. }
  285. shmem_dir_unmap(dir);
  286. return shmem_swp_map(subdir) + offset;
  287. }
  288. static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value)
  289. {
  290. long incdec = value? 1: -1;
  291. entry->val = value;
  292. info->swapped += incdec;
  293. if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) {
  294. struct page *page = kmap_atomic_to_page(entry);
  295. set_page_private(page, page_private(page) + incdec);
  296. }
  297. }
  298. /*
  299. * shmem_swp_alloc - get the position of the swap entry for the page.
  300. * If it does not exist allocate the entry.
  301. *
  302. * @info: info structure for the inode
  303. * @index: index of the page to find
  304. * @sgp: check and recheck i_size? skip allocation?
  305. */
  306. static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp)
  307. {
  308. struct inode *inode = &info->vfs_inode;
  309. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  310. struct page *page = NULL;
  311. swp_entry_t *entry;
  312. if (sgp != SGP_WRITE &&
  313. ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode))
  314. return ERR_PTR(-EINVAL);
  315. while (!(entry = shmem_swp_entry(info, index, &page))) {
  316. if (sgp == SGP_READ)
  317. return shmem_swp_map(ZERO_PAGE(0));
  318. /*
  319. * Test free_blocks against 1 not 0, since we have 1 data
  320. * page (and perhaps indirect index pages) yet to allocate:
  321. * a waste to allocate index if we cannot allocate data.
  322. */
  323. if (sbinfo->max_blocks) {
  324. spin_lock(&sbinfo->stat_lock);
  325. if (sbinfo->free_blocks <= 1) {
  326. spin_unlock(&sbinfo->stat_lock);
  327. return ERR_PTR(-ENOSPC);
  328. }
  329. sbinfo->free_blocks--;
  330. inode->i_blocks += BLOCKS_PER_PAGE;
  331. spin_unlock(&sbinfo->stat_lock);
  332. }
  333. spin_unlock(&info->lock);
  334. page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping) | __GFP_ZERO);
  335. if (page)
  336. set_page_private(page, 0);
  337. spin_lock(&info->lock);
  338. if (!page) {
  339. shmem_free_blocks(inode, 1);
  340. return ERR_PTR(-ENOMEM);
  341. }
  342. if (sgp != SGP_WRITE &&
  343. ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
  344. entry = ERR_PTR(-EINVAL);
  345. break;
  346. }
  347. if (info->next_index <= index)
  348. info->next_index = index + 1;
  349. }
  350. if (page) {
  351. /* another task gave its page, or truncated the file */
  352. shmem_free_blocks(inode, 1);
  353. shmem_dir_free(page);
  354. }
  355. if (info->next_index <= index && !IS_ERR(entry))
  356. info->next_index = index + 1;
  357. return entry;
  358. }
  359. /*
  360. * shmem_free_swp - free some swap entries in a directory
  361. *
  362. * @dir: pointer to the directory
  363. * @edir: pointer after last entry of the directory
  364. * @punch_lock: pointer to spinlock when needed for the holepunch case
  365. */
  366. static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir,
  367. spinlock_t *punch_lock)
  368. {
  369. spinlock_t *punch_unlock = NULL;
  370. swp_entry_t *ptr;
  371. int freed = 0;
  372. for (ptr = dir; ptr < edir; ptr++) {
  373. if (ptr->val) {
  374. if (unlikely(punch_lock)) {
  375. punch_unlock = punch_lock;
  376. punch_lock = NULL;
  377. spin_lock(punch_unlock);
  378. if (!ptr->val)
  379. continue;
  380. }
  381. free_swap_and_cache(*ptr);
  382. *ptr = (swp_entry_t){0};
  383. freed++;
  384. }
  385. }
  386. if (punch_unlock)
  387. spin_unlock(punch_unlock);
  388. return freed;
  389. }
  390. static int shmem_map_and_free_swp(struct page *subdir, int offset,
  391. int limit, struct page ***dir, spinlock_t *punch_lock)
  392. {
  393. swp_entry_t *ptr;
  394. int freed = 0;
  395. ptr = shmem_swp_map(subdir);
  396. for (; offset < limit; offset += LATENCY_LIMIT) {
  397. int size = limit - offset;
  398. if (size > LATENCY_LIMIT)
  399. size = LATENCY_LIMIT;
  400. freed += shmem_free_swp(ptr+offset, ptr+offset+size,
  401. punch_lock);
  402. if (need_resched()) {
  403. shmem_swp_unmap(ptr);
  404. if (*dir) {
  405. shmem_dir_unmap(*dir);
  406. *dir = NULL;
  407. }
  408. cond_resched();
  409. ptr = shmem_swp_map(subdir);
  410. }
  411. }
  412. shmem_swp_unmap(ptr);
  413. return freed;
  414. }
  415. static void shmem_free_pages(struct list_head *next)
  416. {
  417. struct page *page;
  418. int freed = 0;
  419. do {
  420. page = container_of(next, struct page, lru);
  421. next = next->next;
  422. shmem_dir_free(page);
  423. freed++;
  424. if (freed >= LATENCY_LIMIT) {
  425. cond_resched();
  426. freed = 0;
  427. }
  428. } while (next);
  429. }
  430. static void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end)
  431. {
  432. struct shmem_inode_info *info = SHMEM_I(inode);
  433. unsigned long idx;
  434. unsigned long size;
  435. unsigned long limit;
  436. unsigned long stage;
  437. unsigned long diroff;
  438. struct page **dir;
  439. struct page *topdir;
  440. struct page *middir;
  441. struct page *subdir;
  442. swp_entry_t *ptr;
  443. LIST_HEAD(pages_to_free);
  444. long nr_pages_to_free = 0;
  445. long nr_swaps_freed = 0;
  446. int offset;
  447. int freed;
  448. int punch_hole;
  449. spinlock_t *needs_lock;
  450. spinlock_t *punch_lock;
  451. unsigned long upper_limit;
  452. inode->i_ctime = inode->i_mtime = CURRENT_TIME;
  453. idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  454. if (idx >= info->next_index)
  455. return;
  456. spin_lock(&info->lock);
  457. info->flags |= SHMEM_TRUNCATE;
  458. if (likely(end == (loff_t) -1)) {
  459. limit = info->next_index;
  460. upper_limit = SHMEM_MAX_INDEX;
  461. info->next_index = idx;
  462. needs_lock = NULL;
  463. punch_hole = 0;
  464. } else {
  465. if (end + 1 >= inode->i_size) { /* we may free a little more */
  466. limit = (inode->i_size + PAGE_CACHE_SIZE - 1) >>
  467. PAGE_CACHE_SHIFT;
  468. upper_limit = SHMEM_MAX_INDEX;
  469. } else {
  470. limit = (end + 1) >> PAGE_CACHE_SHIFT;
  471. upper_limit = limit;
  472. }
  473. needs_lock = &info->lock;
  474. punch_hole = 1;
  475. }
  476. topdir = info->i_indirect;
  477. if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) {
  478. info->i_indirect = NULL;
  479. nr_pages_to_free++;
  480. list_add(&topdir->lru, &pages_to_free);
  481. }
  482. spin_unlock(&info->lock);
  483. if (info->swapped && idx < SHMEM_NR_DIRECT) {
  484. ptr = info->i_direct;
  485. size = limit;
  486. if (size > SHMEM_NR_DIRECT)
  487. size = SHMEM_NR_DIRECT;
  488. nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size, needs_lock);
  489. }
  490. /*
  491. * If there are no indirect blocks or we are punching a hole
  492. * below indirect blocks, nothing to be done.
  493. */
  494. if (!topdir || limit <= SHMEM_NR_DIRECT)
  495. goto done2;
  496. /*
  497. * The truncation case has already dropped info->lock, and we're safe
  498. * because i_size and next_index have already been lowered, preventing
  499. * access beyond. But in the punch_hole case, we still need to take
  500. * the lock when updating the swap directory, because there might be
  501. * racing accesses by shmem_getpage(SGP_CACHE), shmem_unuse_inode or
  502. * shmem_writepage. However, whenever we find we can remove a whole
  503. * directory page (not at the misaligned start or end of the range),
  504. * we first NULLify its pointer in the level above, and then have no
  505. * need to take the lock when updating its contents: needs_lock and
  506. * punch_lock (either pointing to info->lock or NULL) manage this.
  507. */
  508. upper_limit -= SHMEM_NR_DIRECT;
  509. limit -= SHMEM_NR_DIRECT;
  510. idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0;
  511. offset = idx % ENTRIES_PER_PAGE;
  512. idx -= offset;
  513. dir = shmem_dir_map(topdir);
  514. stage = ENTRIES_PER_PAGEPAGE/2;
  515. if (idx < ENTRIES_PER_PAGEPAGE/2) {
  516. middir = topdir;
  517. diroff = idx/ENTRIES_PER_PAGE;
  518. } else {
  519. dir += ENTRIES_PER_PAGE/2;
  520. dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE;
  521. while (stage <= idx)
  522. stage += ENTRIES_PER_PAGEPAGE;
  523. middir = *dir;
  524. if (*dir) {
  525. diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) %
  526. ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE;
  527. if (!diroff && !offset && upper_limit >= stage) {
  528. if (needs_lock) {
  529. spin_lock(needs_lock);
  530. *dir = NULL;
  531. spin_unlock(needs_lock);
  532. needs_lock = NULL;
  533. } else
  534. *dir = NULL;
  535. nr_pages_to_free++;
  536. list_add(&middir->lru, &pages_to_free);
  537. }
  538. shmem_dir_unmap(dir);
  539. dir = shmem_dir_map(middir);
  540. } else {
  541. diroff = 0;
  542. offset = 0;
  543. idx = stage;
  544. }
  545. }
  546. for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) {
  547. if (unlikely(idx == stage)) {
  548. shmem_dir_unmap(dir);
  549. dir = shmem_dir_map(topdir) +
  550. ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
  551. while (!*dir) {
  552. dir++;
  553. idx += ENTRIES_PER_PAGEPAGE;
  554. if (idx >= limit)
  555. goto done1;
  556. }
  557. stage = idx + ENTRIES_PER_PAGEPAGE;
  558. middir = *dir;
  559. if (punch_hole)
  560. needs_lock = &info->lock;
  561. if (upper_limit >= stage) {
  562. if (needs_lock) {
  563. spin_lock(needs_lock);
  564. *dir = NULL;
  565. spin_unlock(needs_lock);
  566. needs_lock = NULL;
  567. } else
  568. *dir = NULL;
  569. nr_pages_to_free++;
  570. list_add(&middir->lru, &pages_to_free);
  571. }
  572. shmem_dir_unmap(dir);
  573. cond_resched();
  574. dir = shmem_dir_map(middir);
  575. diroff = 0;
  576. }
  577. punch_lock = needs_lock;
  578. subdir = dir[diroff];
  579. if (subdir && !offset && upper_limit-idx >= ENTRIES_PER_PAGE) {
  580. if (needs_lock) {
  581. spin_lock(needs_lock);
  582. dir[diroff] = NULL;
  583. spin_unlock(needs_lock);
  584. punch_lock = NULL;
  585. } else
  586. dir[diroff] = NULL;
  587. nr_pages_to_free++;
  588. list_add(&subdir->lru, &pages_to_free);
  589. }
  590. if (subdir && page_private(subdir) /* has swap entries */) {
  591. size = limit - idx;
  592. if (size > ENTRIES_PER_PAGE)
  593. size = ENTRIES_PER_PAGE;
  594. freed = shmem_map_and_free_swp(subdir,
  595. offset, size, &dir, punch_lock);
  596. if (!dir)
  597. dir = shmem_dir_map(middir);
  598. nr_swaps_freed += freed;
  599. if (offset || punch_lock) {
  600. spin_lock(&info->lock);
  601. set_page_private(subdir,
  602. page_private(subdir) - freed);
  603. spin_unlock(&info->lock);
  604. } else
  605. BUG_ON(page_private(subdir) != freed);
  606. }
  607. offset = 0;
  608. }
  609. done1:
  610. shmem_dir_unmap(dir);
  611. done2:
  612. if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) {
  613. /*
  614. * Call truncate_inode_pages again: racing shmem_unuse_inode
  615. * may have swizzled a page in from swap since vmtruncate or
  616. * generic_delete_inode did it, before we lowered next_index.
  617. * Also, though shmem_getpage checks i_size before adding to
  618. * cache, no recheck after: so fix the narrow window there too.
  619. *
  620. * Recalling truncate_inode_pages_range and unmap_mapping_range
  621. * every time for punch_hole (which never got a chance to clear
  622. * SHMEM_PAGEIN at the start of vmtruncate_range) is expensive,
  623. * yet hardly ever necessary: try to optimize them out later.
  624. */
  625. truncate_inode_pages_range(inode->i_mapping, start, end);
  626. if (punch_hole)
  627. unmap_mapping_range(inode->i_mapping, start,
  628. end - start, 1);
  629. }
  630. spin_lock(&info->lock);
  631. info->flags &= ~SHMEM_TRUNCATE;
  632. info->swapped -= nr_swaps_freed;
  633. if (nr_pages_to_free)
  634. shmem_free_blocks(inode, nr_pages_to_free);
  635. shmem_recalc_inode(inode);
  636. spin_unlock(&info->lock);
  637. /*
  638. * Empty swap vector directory pages to be freed?
  639. */
  640. if (!list_empty(&pages_to_free)) {
  641. pages_to_free.prev->next = NULL;
  642. shmem_free_pages(pages_to_free.next);
  643. }
  644. }
  645. static void shmem_truncate(struct inode *inode)
  646. {
  647. shmem_truncate_range(inode, inode->i_size, (loff_t)-1);
  648. }
  649. static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
  650. {
  651. struct inode *inode = dentry->d_inode;
  652. struct page *page = NULL;
  653. int error;
  654. if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
  655. if (attr->ia_size < inode->i_size) {
  656. /*
  657. * If truncating down to a partial page, then
  658. * if that page is already allocated, hold it
  659. * in memory until the truncation is over, so
  660. * truncate_partial_page cannnot miss it were
  661. * it assigned to swap.
  662. */
  663. if (attr->ia_size & (PAGE_CACHE_SIZE-1)) {
  664. (void) shmem_getpage(inode,
  665. attr->ia_size>>PAGE_CACHE_SHIFT,
  666. &page, SGP_READ, NULL);
  667. }
  668. /*
  669. * Reset SHMEM_PAGEIN flag so that shmem_truncate can
  670. * detect if any pages might have been added to cache
  671. * after truncate_inode_pages. But we needn't bother
  672. * if it's being fully truncated to zero-length: the
  673. * nrpages check is efficient enough in that case.
  674. */
  675. if (attr->ia_size) {
  676. struct shmem_inode_info *info = SHMEM_I(inode);
  677. spin_lock(&info->lock);
  678. info->flags &= ~SHMEM_PAGEIN;
  679. spin_unlock(&info->lock);
  680. }
  681. }
  682. }
  683. error = inode_change_ok(inode, attr);
  684. if (!error)
  685. error = inode_setattr(inode, attr);
  686. #ifdef CONFIG_TMPFS_POSIX_ACL
  687. if (!error && (attr->ia_valid & ATTR_MODE))
  688. error = generic_acl_chmod(inode, &shmem_acl_ops);
  689. #endif
  690. if (page)
  691. page_cache_release(page);
  692. return error;
  693. }
  694. static void shmem_delete_inode(struct inode *inode)
  695. {
  696. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  697. struct shmem_inode_info *info = SHMEM_I(inode);
  698. if (inode->i_op->truncate == shmem_truncate) {
  699. truncate_inode_pages(inode->i_mapping, 0);
  700. shmem_unacct_size(info->flags, inode->i_size);
  701. inode->i_size = 0;
  702. shmem_truncate(inode);
  703. if (!list_empty(&info->swaplist)) {
  704. spin_lock(&shmem_swaplist_lock);
  705. list_del_init(&info->swaplist);
  706. spin_unlock(&shmem_swaplist_lock);
  707. }
  708. }
  709. BUG_ON(inode->i_blocks);
  710. if (sbinfo->max_inodes) {
  711. spin_lock(&sbinfo->stat_lock);
  712. sbinfo->free_inodes++;
  713. spin_unlock(&sbinfo->stat_lock);
  714. }
  715. clear_inode(inode);
  716. }
  717. static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
  718. {
  719. swp_entry_t *ptr;
  720. for (ptr = dir; ptr < edir; ptr++) {
  721. if (ptr->val == entry.val)
  722. return ptr - dir;
  723. }
  724. return -1;
  725. }
  726. static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
  727. {
  728. struct inode *inode;
  729. unsigned long idx;
  730. unsigned long size;
  731. unsigned long limit;
  732. unsigned long stage;
  733. struct page **dir;
  734. struct page *subdir;
  735. swp_entry_t *ptr;
  736. int offset;
  737. idx = 0;
  738. ptr = info->i_direct;
  739. spin_lock(&info->lock);
  740. limit = info->next_index;
  741. size = limit;
  742. if (size > SHMEM_NR_DIRECT)
  743. size = SHMEM_NR_DIRECT;
  744. offset = shmem_find_swp(entry, ptr, ptr+size);
  745. if (offset >= 0) {
  746. shmem_swp_balance_unmap();
  747. goto found;
  748. }
  749. if (!info->i_indirect)
  750. goto lost2;
  751. dir = shmem_dir_map(info->i_indirect);
  752. stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2;
  753. for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
  754. if (unlikely(idx == stage)) {
  755. shmem_dir_unmap(dir-1);
  756. dir = shmem_dir_map(info->i_indirect) +
  757. ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
  758. while (!*dir) {
  759. dir++;
  760. idx += ENTRIES_PER_PAGEPAGE;
  761. if (idx >= limit)
  762. goto lost1;
  763. }
  764. stage = idx + ENTRIES_PER_PAGEPAGE;
  765. subdir = *dir;
  766. shmem_dir_unmap(dir);
  767. dir = shmem_dir_map(subdir);
  768. }
  769. subdir = *dir;
  770. if (subdir && page_private(subdir)) {
  771. ptr = shmem_swp_map(subdir);
  772. size = limit - idx;
  773. if (size > ENTRIES_PER_PAGE)
  774. size = ENTRIES_PER_PAGE;
  775. offset = shmem_find_swp(entry, ptr, ptr+size);
  776. if (offset >= 0) {
  777. shmem_dir_unmap(dir);
  778. goto found;
  779. }
  780. shmem_swp_unmap(ptr);
  781. }
  782. }
  783. lost1:
  784. shmem_dir_unmap(dir-1);
  785. lost2:
  786. spin_unlock(&info->lock);
  787. return 0;
  788. found:
  789. idx += offset;
  790. inode = &info->vfs_inode;
  791. if (move_from_swap_cache(page, idx, inode->i_mapping) == 0) {
  792. info->flags |= SHMEM_PAGEIN;
  793. shmem_swp_set(info, ptr + offset, 0);
  794. }
  795. shmem_swp_unmap(ptr);
  796. spin_unlock(&info->lock);
  797. /*
  798. * Decrement swap count even when the entry is left behind:
  799. * try_to_unuse will skip over mms, then reincrement count.
  800. */
  801. swap_free(entry);
  802. return 1;
  803. }
  804. /*
  805. * shmem_unuse() search for an eventually swapped out shmem page.
  806. */
  807. int shmem_unuse(swp_entry_t entry, struct page *page)
  808. {
  809. struct list_head *p, *next;
  810. struct shmem_inode_info *info;
  811. int found = 0;
  812. spin_lock(&shmem_swaplist_lock);
  813. list_for_each_safe(p, next, &shmem_swaplist) {
  814. info = list_entry(p, struct shmem_inode_info, swaplist);
  815. if (!info->swapped)
  816. list_del_init(&info->swaplist);
  817. else if (shmem_unuse_inode(info, entry, page)) {
  818. /* move head to start search for next from here */
  819. list_move_tail(&shmem_swaplist, &info->swaplist);
  820. found = 1;
  821. break;
  822. }
  823. }
  824. spin_unlock(&shmem_swaplist_lock);
  825. return found;
  826. }
  827. /*
  828. * Move the page from the page cache to the swap cache.
  829. */
  830. static int shmem_writepage(struct page *page, struct writeback_control *wbc)
  831. {
  832. struct shmem_inode_info *info;
  833. swp_entry_t *entry, swap;
  834. struct address_space *mapping;
  835. unsigned long index;
  836. struct inode *inode;
  837. BUG_ON(!PageLocked(page));
  838. BUG_ON(page_mapped(page));
  839. mapping = page->mapping;
  840. index = page->index;
  841. inode = mapping->host;
  842. info = SHMEM_I(inode);
  843. if (info->flags & VM_LOCKED)
  844. goto redirty;
  845. swap = get_swap_page();
  846. if (!swap.val)
  847. goto redirty;
  848. spin_lock(&info->lock);
  849. shmem_recalc_inode(inode);
  850. if (index >= info->next_index) {
  851. BUG_ON(!(info->flags & SHMEM_TRUNCATE));
  852. goto unlock;
  853. }
  854. entry = shmem_swp_entry(info, index, NULL);
  855. BUG_ON(!entry);
  856. BUG_ON(entry->val);
  857. if (move_to_swap_cache(page, swap) == 0) {
  858. shmem_swp_set(info, entry, swap.val);
  859. shmem_swp_unmap(entry);
  860. spin_unlock(&info->lock);
  861. if (list_empty(&info->swaplist)) {
  862. spin_lock(&shmem_swaplist_lock);
  863. /* move instead of add in case we're racing */
  864. list_move_tail(&info->swaplist, &shmem_swaplist);
  865. spin_unlock(&shmem_swaplist_lock);
  866. }
  867. unlock_page(page);
  868. return 0;
  869. }
  870. shmem_swp_unmap(entry);
  871. unlock:
  872. spin_unlock(&info->lock);
  873. swap_free(swap);
  874. redirty:
  875. set_page_dirty(page);
  876. return AOP_WRITEPAGE_ACTIVATE; /* Return with the page locked */
  877. }
  878. #ifdef CONFIG_NUMA
  879. static inline int shmem_parse_mpol(char *value, int *policy, nodemask_t *policy_nodes)
  880. {
  881. char *nodelist = strchr(value, ':');
  882. int err = 1;
  883. if (nodelist) {
  884. /* NUL-terminate policy string */
  885. *nodelist++ = '\0';
  886. if (nodelist_parse(nodelist, *policy_nodes))
  887. goto out;
  888. }
  889. if (!strcmp(value, "default")) {
  890. *policy = MPOL_DEFAULT;
  891. /* Don't allow a nodelist */
  892. if (!nodelist)
  893. err = 0;
  894. } else if (!strcmp(value, "prefer")) {
  895. *policy = MPOL_PREFERRED;
  896. /* Insist on a nodelist of one node only */
  897. if (nodelist) {
  898. char *rest = nodelist;
  899. while (isdigit(*rest))
  900. rest++;
  901. if (!*rest)
  902. err = 0;
  903. }
  904. } else if (!strcmp(value, "bind")) {
  905. *policy = MPOL_BIND;
  906. /* Insist on a nodelist */
  907. if (nodelist)
  908. err = 0;
  909. } else if (!strcmp(value, "interleave")) {
  910. *policy = MPOL_INTERLEAVE;
  911. /* Default to nodes online if no nodelist */
  912. if (!nodelist)
  913. *policy_nodes = node_online_map;
  914. err = 0;
  915. }
  916. out:
  917. /* Restore string for error message */
  918. if (nodelist)
  919. *--nodelist = ':';
  920. return err;
  921. }
  922. static struct page *shmem_swapin_async(struct shared_policy *p,
  923. swp_entry_t entry, unsigned long idx)
  924. {
  925. struct page *page;
  926. struct vm_area_struct pvma;
  927. /* Create a pseudo vma that just contains the policy */
  928. memset(&pvma, 0, sizeof(struct vm_area_struct));
  929. pvma.vm_end = PAGE_SIZE;
  930. pvma.vm_pgoff = idx;
  931. pvma.vm_policy = mpol_shared_policy_lookup(p, idx);
  932. page = read_swap_cache_async(entry, &pvma, 0);
  933. mpol_free(pvma.vm_policy);
  934. return page;
  935. }
  936. struct page *shmem_swapin(struct shmem_inode_info *info, swp_entry_t entry,
  937. unsigned long idx)
  938. {
  939. struct shared_policy *p = &info->policy;
  940. int i, num;
  941. struct page *page;
  942. unsigned long offset;
  943. num = valid_swaphandles(entry, &offset);
  944. for (i = 0; i < num; offset++, i++) {
  945. page = shmem_swapin_async(p,
  946. swp_entry(swp_type(entry), offset), idx);
  947. if (!page)
  948. break;
  949. page_cache_release(page);
  950. }
  951. lru_add_drain(); /* Push any new pages onto the LRU now */
  952. return shmem_swapin_async(p, entry, idx);
  953. }
  954. static struct page *
  955. shmem_alloc_page(gfp_t gfp, struct shmem_inode_info *info,
  956. unsigned long idx)
  957. {
  958. struct vm_area_struct pvma;
  959. struct page *page;
  960. memset(&pvma, 0, sizeof(struct vm_area_struct));
  961. pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
  962. pvma.vm_pgoff = idx;
  963. pvma.vm_end = PAGE_SIZE;
  964. page = alloc_page_vma(gfp | __GFP_ZERO, &pvma, 0);
  965. mpol_free(pvma.vm_policy);
  966. return page;
  967. }
  968. #else
  969. static inline int shmem_parse_mpol(char *value, int *policy, nodemask_t *policy_nodes)
  970. {
  971. return 1;
  972. }
  973. static inline struct page *
  974. shmem_swapin(struct shmem_inode_info *info,swp_entry_t entry,unsigned long idx)
  975. {
  976. swapin_readahead(entry, 0, NULL);
  977. return read_swap_cache_async(entry, NULL, 0);
  978. }
  979. static inline struct page *
  980. shmem_alloc_page(gfp_t gfp,struct shmem_inode_info *info, unsigned long idx)
  981. {
  982. return alloc_page(gfp | __GFP_ZERO);
  983. }
  984. #endif
  985. /*
  986. * shmem_getpage - either get the page from swap or allocate a new one
  987. *
  988. * If we allocate a new one we do not mark it dirty. That's up to the
  989. * vm. If we swap it in we mark it dirty since we also free the swap
  990. * entry since a page cannot live in both the swap and page cache
  991. */
  992. static int shmem_getpage(struct inode *inode, unsigned long idx,
  993. struct page **pagep, enum sgp_type sgp, int *type)
  994. {
  995. struct address_space *mapping = inode->i_mapping;
  996. struct shmem_inode_info *info = SHMEM_I(inode);
  997. struct shmem_sb_info *sbinfo;
  998. struct page *filepage = *pagep;
  999. struct page *swappage;
  1000. swp_entry_t *entry;
  1001. swp_entry_t swap;
  1002. int error;
  1003. if (idx >= SHMEM_MAX_INDEX)
  1004. return -EFBIG;
  1005. /*
  1006. * Normally, filepage is NULL on entry, and either found
  1007. * uptodate immediately, or allocated and zeroed, or read
  1008. * in under swappage, which is then assigned to filepage.
  1009. * But shmem_prepare_write passes in a locked filepage,
  1010. * which may be found not uptodate by other callers too,
  1011. * and may need to be copied from the swappage read in.
  1012. */
  1013. repeat:
  1014. if (!filepage)
  1015. filepage = find_lock_page(mapping, idx);
  1016. if (filepage && PageUptodate(filepage))
  1017. goto done;
  1018. error = 0;
  1019. if (sgp == SGP_QUICK)
  1020. goto failed;
  1021. spin_lock(&info->lock);
  1022. shmem_recalc_inode(inode);
  1023. entry = shmem_swp_alloc(info, idx, sgp);
  1024. if (IS_ERR(entry)) {
  1025. spin_unlock(&info->lock);
  1026. error = PTR_ERR(entry);
  1027. goto failed;
  1028. }
  1029. swap = *entry;
  1030. if (swap.val) {
  1031. /* Look it up and read it in.. */
  1032. swappage = lookup_swap_cache(swap);
  1033. if (!swappage) {
  1034. shmem_swp_unmap(entry);
  1035. /* here we actually do the io */
  1036. if (type && *type == VM_FAULT_MINOR) {
  1037. __count_vm_event(PGMAJFAULT);
  1038. *type = VM_FAULT_MAJOR;
  1039. }
  1040. spin_unlock(&info->lock);
  1041. swappage = shmem_swapin(info, swap, idx);
  1042. if (!swappage) {
  1043. spin_lock(&info->lock);
  1044. entry = shmem_swp_alloc(info, idx, sgp);
  1045. if (IS_ERR(entry))
  1046. error = PTR_ERR(entry);
  1047. else {
  1048. if (entry->val == swap.val)
  1049. error = -ENOMEM;
  1050. shmem_swp_unmap(entry);
  1051. }
  1052. spin_unlock(&info->lock);
  1053. if (error)
  1054. goto failed;
  1055. goto repeat;
  1056. }
  1057. wait_on_page_locked(swappage);
  1058. page_cache_release(swappage);
  1059. goto repeat;
  1060. }
  1061. /* We have to do this with page locked to prevent races */
  1062. if (TestSetPageLocked(swappage)) {
  1063. shmem_swp_unmap(entry);
  1064. spin_unlock(&info->lock);
  1065. wait_on_page_locked(swappage);
  1066. page_cache_release(swappage);
  1067. goto repeat;
  1068. }
  1069. if (PageWriteback(swappage)) {
  1070. shmem_swp_unmap(entry);
  1071. spin_unlock(&info->lock);
  1072. wait_on_page_writeback(swappage);
  1073. unlock_page(swappage);
  1074. page_cache_release(swappage);
  1075. goto repeat;
  1076. }
  1077. if (!PageUptodate(swappage)) {
  1078. shmem_swp_unmap(entry);
  1079. spin_unlock(&info->lock);
  1080. unlock_page(swappage);
  1081. page_cache_release(swappage);
  1082. error = -EIO;
  1083. goto failed;
  1084. }
  1085. if (filepage) {
  1086. shmem_swp_set(info, entry, 0);
  1087. shmem_swp_unmap(entry);
  1088. delete_from_swap_cache(swappage);
  1089. spin_unlock(&info->lock);
  1090. copy_highpage(filepage, swappage);
  1091. unlock_page(swappage);
  1092. page_cache_release(swappage);
  1093. flush_dcache_page(filepage);
  1094. SetPageUptodate(filepage);
  1095. set_page_dirty(filepage);
  1096. swap_free(swap);
  1097. } else if (!(error = move_from_swap_cache(
  1098. swappage, idx, mapping))) {
  1099. info->flags |= SHMEM_PAGEIN;
  1100. shmem_swp_set(info, entry, 0);
  1101. shmem_swp_unmap(entry);
  1102. spin_unlock(&info->lock);
  1103. filepage = swappage;
  1104. swap_free(swap);
  1105. } else {
  1106. shmem_swp_unmap(entry);
  1107. spin_unlock(&info->lock);
  1108. unlock_page(swappage);
  1109. page_cache_release(swappage);
  1110. if (error == -ENOMEM) {
  1111. /* let kswapd refresh zone for GFP_ATOMICs */
  1112. congestion_wait(WRITE, HZ/50);
  1113. }
  1114. goto repeat;
  1115. }
  1116. } else if (sgp == SGP_READ && !filepage) {
  1117. shmem_swp_unmap(entry);
  1118. filepage = find_get_page(mapping, idx);
  1119. if (filepage &&
  1120. (!PageUptodate(filepage) || TestSetPageLocked(filepage))) {
  1121. spin_unlock(&info->lock);
  1122. wait_on_page_locked(filepage);
  1123. page_cache_release(filepage);
  1124. filepage = NULL;
  1125. goto repeat;
  1126. }
  1127. spin_unlock(&info->lock);
  1128. } else {
  1129. shmem_swp_unmap(entry);
  1130. sbinfo = SHMEM_SB(inode->i_sb);
  1131. if (sbinfo->max_blocks) {
  1132. spin_lock(&sbinfo->stat_lock);
  1133. if (sbinfo->free_blocks == 0 ||
  1134. shmem_acct_block(info->flags)) {
  1135. spin_unlock(&sbinfo->stat_lock);
  1136. spin_unlock(&info->lock);
  1137. error = -ENOSPC;
  1138. goto failed;
  1139. }
  1140. sbinfo->free_blocks--;
  1141. inode->i_blocks += BLOCKS_PER_PAGE;
  1142. spin_unlock(&sbinfo->stat_lock);
  1143. } else if (shmem_acct_block(info->flags)) {
  1144. spin_unlock(&info->lock);
  1145. error = -ENOSPC;
  1146. goto failed;
  1147. }
  1148. if (!filepage) {
  1149. spin_unlock(&info->lock);
  1150. filepage = shmem_alloc_page(mapping_gfp_mask(mapping),
  1151. info,
  1152. idx);
  1153. if (!filepage) {
  1154. shmem_unacct_blocks(info->flags, 1);
  1155. shmem_free_blocks(inode, 1);
  1156. error = -ENOMEM;
  1157. goto failed;
  1158. }
  1159. spin_lock(&info->lock);
  1160. entry = shmem_swp_alloc(info, idx, sgp);
  1161. if (IS_ERR(entry))
  1162. error = PTR_ERR(entry);
  1163. else {
  1164. swap = *entry;
  1165. shmem_swp_unmap(entry);
  1166. }
  1167. if (error || swap.val || 0 != add_to_page_cache_lru(
  1168. filepage, mapping, idx, GFP_ATOMIC)) {
  1169. spin_unlock(&info->lock);
  1170. page_cache_release(filepage);
  1171. shmem_unacct_blocks(info->flags, 1);
  1172. shmem_free_blocks(inode, 1);
  1173. filepage = NULL;
  1174. if (error)
  1175. goto failed;
  1176. goto repeat;
  1177. }
  1178. info->flags |= SHMEM_PAGEIN;
  1179. }
  1180. info->alloced++;
  1181. spin_unlock(&info->lock);
  1182. flush_dcache_page(filepage);
  1183. SetPageUptodate(filepage);
  1184. }
  1185. done:
  1186. if (*pagep != filepage) {
  1187. unlock_page(filepage);
  1188. *pagep = filepage;
  1189. }
  1190. return 0;
  1191. failed:
  1192. if (*pagep != filepage) {
  1193. unlock_page(filepage);
  1194. page_cache_release(filepage);
  1195. }
  1196. return error;
  1197. }
  1198. static struct page *shmem_nopage(struct vm_area_struct *vma,
  1199. unsigned long address, int *type)
  1200. {
  1201. struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
  1202. struct page *page = NULL;
  1203. unsigned long idx;
  1204. int error;
  1205. idx = (address - vma->vm_start) >> PAGE_SHIFT;
  1206. idx += vma->vm_pgoff;
  1207. idx >>= PAGE_CACHE_SHIFT - PAGE_SHIFT;
  1208. if (((loff_t) idx << PAGE_CACHE_SHIFT) >= i_size_read(inode))
  1209. return NOPAGE_SIGBUS;
  1210. error = shmem_getpage(inode, idx, &page, SGP_CACHE, type);
  1211. if (error)
  1212. return (error == -ENOMEM)? NOPAGE_OOM: NOPAGE_SIGBUS;
  1213. mark_page_accessed(page);
  1214. return page;
  1215. }
  1216. static int shmem_populate(struct vm_area_struct *vma,
  1217. unsigned long addr, unsigned long len,
  1218. pgprot_t prot, unsigned long pgoff, int nonblock)
  1219. {
  1220. struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
  1221. struct mm_struct *mm = vma->vm_mm;
  1222. enum sgp_type sgp = nonblock? SGP_QUICK: SGP_CACHE;
  1223. unsigned long size;
  1224. size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1225. if (pgoff >= size || pgoff + (len >> PAGE_SHIFT) > size)
  1226. return -EINVAL;
  1227. while ((long) len > 0) {
  1228. struct page *page = NULL;
  1229. int err;
  1230. /*
  1231. * Will need changing if PAGE_CACHE_SIZE != PAGE_SIZE
  1232. */
  1233. err = shmem_getpage(inode, pgoff, &page, sgp, NULL);
  1234. if (err)
  1235. return err;
  1236. /* Page may still be null, but only if nonblock was set. */
  1237. if (page) {
  1238. mark_page_accessed(page);
  1239. err = install_page(mm, vma, addr, page, prot);
  1240. if (err) {
  1241. page_cache_release(page);
  1242. return err;
  1243. }
  1244. } else if (vma->vm_flags & VM_NONLINEAR) {
  1245. /* No page was found just because we can't read it in
  1246. * now (being here implies nonblock != 0), but the page
  1247. * may exist, so set the PTE to fault it in later. */
  1248. err = install_file_pte(mm, vma, addr, pgoff, prot);
  1249. if (err)
  1250. return err;
  1251. }
  1252. len -= PAGE_SIZE;
  1253. addr += PAGE_SIZE;
  1254. pgoff++;
  1255. }
  1256. return 0;
  1257. }
  1258. #ifdef CONFIG_NUMA
  1259. int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
  1260. {
  1261. struct inode *i = vma->vm_file->f_path.dentry->d_inode;
  1262. return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new);
  1263. }
  1264. struct mempolicy *
  1265. shmem_get_policy(struct vm_area_struct *vma, unsigned long addr)
  1266. {
  1267. struct inode *i = vma->vm_file->f_path.dentry->d_inode;
  1268. unsigned long idx;
  1269. idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
  1270. return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx);
  1271. }
  1272. #endif
  1273. int shmem_lock(struct file *file, int lock, struct user_struct *user)
  1274. {
  1275. struct inode *inode = file->f_path.dentry->d_inode;
  1276. struct shmem_inode_info *info = SHMEM_I(inode);
  1277. int retval = -ENOMEM;
  1278. spin_lock(&info->lock);
  1279. if (lock && !(info->flags & VM_LOCKED)) {
  1280. if (!user_shm_lock(inode->i_size, user))
  1281. goto out_nomem;
  1282. info->flags |= VM_LOCKED;
  1283. }
  1284. if (!lock && (info->flags & VM_LOCKED) && user) {
  1285. user_shm_unlock(inode->i_size, user);
  1286. info->flags &= ~VM_LOCKED;
  1287. }
  1288. retval = 0;
  1289. out_nomem:
  1290. spin_unlock(&info->lock);
  1291. return retval;
  1292. }
  1293. static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
  1294. {
  1295. file_accessed(file);
  1296. vma->vm_ops = &shmem_vm_ops;
  1297. return 0;
  1298. }
  1299. static struct inode *
  1300. shmem_get_inode(struct super_block *sb, int mode, dev_t dev)
  1301. {
  1302. struct inode *inode;
  1303. struct shmem_inode_info *info;
  1304. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  1305. if (sbinfo->max_inodes) {
  1306. spin_lock(&sbinfo->stat_lock);
  1307. if (!sbinfo->free_inodes) {
  1308. spin_unlock(&sbinfo->stat_lock);
  1309. return NULL;
  1310. }
  1311. sbinfo->free_inodes--;
  1312. spin_unlock(&sbinfo->stat_lock);
  1313. }
  1314. inode = new_inode(sb);
  1315. if (inode) {
  1316. inode->i_mode = mode;
  1317. inode->i_uid = current->fsuid;
  1318. inode->i_gid = current->fsgid;
  1319. inode->i_blocks = 0;
  1320. inode->i_mapping->a_ops = &shmem_aops;
  1321. inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
  1322. inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
  1323. inode->i_generation = get_seconds();
  1324. info = SHMEM_I(inode);
  1325. memset(info, 0, (char *)inode - (char *)info);
  1326. spin_lock_init(&info->lock);
  1327. INIT_LIST_HEAD(&info->swaplist);
  1328. switch (mode & S_IFMT) {
  1329. default:
  1330. inode->i_op = &shmem_special_inode_operations;
  1331. init_special_inode(inode, mode, dev);
  1332. break;
  1333. case S_IFREG:
  1334. inode->i_op = &shmem_inode_operations;
  1335. inode->i_fop = &shmem_file_operations;
  1336. mpol_shared_policy_init(&info->policy, sbinfo->policy,
  1337. &sbinfo->policy_nodes);
  1338. break;
  1339. case S_IFDIR:
  1340. inc_nlink(inode);
  1341. /* Some things misbehave if size == 0 on a directory */
  1342. inode->i_size = 2 * BOGO_DIRENT_SIZE;
  1343. inode->i_op = &shmem_dir_inode_operations;
  1344. inode->i_fop = &simple_dir_operations;
  1345. break;
  1346. case S_IFLNK:
  1347. /*
  1348. * Must not load anything in the rbtree,
  1349. * mpol_free_shared_policy will not be called.
  1350. */
  1351. mpol_shared_policy_init(&info->policy, MPOL_DEFAULT,
  1352. NULL);
  1353. break;
  1354. }
  1355. } else if (sbinfo->max_inodes) {
  1356. spin_lock(&sbinfo->stat_lock);
  1357. sbinfo->free_inodes++;
  1358. spin_unlock(&sbinfo->stat_lock);
  1359. }
  1360. return inode;
  1361. }
  1362. #ifdef CONFIG_TMPFS
  1363. static const struct inode_operations shmem_symlink_inode_operations;
  1364. static const struct inode_operations shmem_symlink_inline_operations;
  1365. /*
  1366. * Normally tmpfs makes no use of shmem_prepare_write, but it
  1367. * lets a tmpfs file be used read-write below the loop driver.
  1368. */
  1369. static int
  1370. shmem_prepare_write(struct file *file, struct page *page, unsigned offset, unsigned to)
  1371. {
  1372. struct inode *inode = page->mapping->host;
  1373. return shmem_getpage(inode, page->index, &page, SGP_WRITE, NULL);
  1374. }
  1375. static ssize_t
  1376. shmem_file_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
  1377. {
  1378. struct inode *inode = file->f_path.dentry->d_inode;
  1379. loff_t pos;
  1380. unsigned long written;
  1381. ssize_t err;
  1382. if ((ssize_t) count < 0)
  1383. return -EINVAL;
  1384. if (!access_ok(VERIFY_READ, buf, count))
  1385. return -EFAULT;
  1386. mutex_lock(&inode->i_mutex);
  1387. pos = *ppos;
  1388. written = 0;
  1389. err = generic_write_checks(file, &pos, &count, 0);
  1390. if (err || !count)
  1391. goto out;
  1392. err = remove_suid(file->f_path.dentry);
  1393. if (err)
  1394. goto out;
  1395. inode->i_ctime = inode->i_mtime = CURRENT_TIME;
  1396. do {
  1397. struct page *page = NULL;
  1398. unsigned long bytes, index, offset;
  1399. char *kaddr;
  1400. int left;
  1401. offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
  1402. index = pos >> PAGE_CACHE_SHIFT;
  1403. bytes = PAGE_CACHE_SIZE - offset;
  1404. if (bytes > count)
  1405. bytes = count;
  1406. /*
  1407. * We don't hold page lock across copy from user -
  1408. * what would it guard against? - so no deadlock here.
  1409. * But it still may be a good idea to prefault below.
  1410. */
  1411. err = shmem_getpage(inode, index, &page, SGP_WRITE, NULL);
  1412. if (err)
  1413. break;
  1414. left = bytes;
  1415. if (PageHighMem(page)) {
  1416. volatile unsigned char dummy;
  1417. __get_user(dummy, buf);
  1418. __get_user(dummy, buf + bytes - 1);
  1419. kaddr = kmap_atomic(page, KM_USER0);
  1420. left = __copy_from_user_inatomic(kaddr + offset,
  1421. buf, bytes);
  1422. kunmap_atomic(kaddr, KM_USER0);
  1423. }
  1424. if (left) {
  1425. kaddr = kmap(page);
  1426. left = __copy_from_user(kaddr + offset, buf, bytes);
  1427. kunmap(page);
  1428. }
  1429. written += bytes;
  1430. count -= bytes;
  1431. pos += bytes;
  1432. buf += bytes;
  1433. if (pos > inode->i_size)
  1434. i_size_write(inode, pos);
  1435. flush_dcache_page(page);
  1436. set_page_dirty(page);
  1437. mark_page_accessed(page);
  1438. page_cache_release(page);
  1439. if (left) {
  1440. pos -= left;
  1441. written -= left;
  1442. err = -EFAULT;
  1443. break;
  1444. }
  1445. /*
  1446. * Our dirty pages are not counted in nr_dirty,
  1447. * and we do not attempt to balance dirty pages.
  1448. */
  1449. cond_resched();
  1450. } while (count);
  1451. *ppos = pos;
  1452. if (written)
  1453. err = written;
  1454. out:
  1455. mutex_unlock(&inode->i_mutex);
  1456. return err;
  1457. }
  1458. static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
  1459. {
  1460. struct inode *inode = filp->f_path.dentry->d_inode;
  1461. struct address_space *mapping = inode->i_mapping;
  1462. unsigned long index, offset;
  1463. index = *ppos >> PAGE_CACHE_SHIFT;
  1464. offset = *ppos & ~PAGE_CACHE_MASK;
  1465. for (;;) {
  1466. struct page *page = NULL;
  1467. unsigned long end_index, nr, ret;
  1468. loff_t i_size = i_size_read(inode);
  1469. end_index = i_size >> PAGE_CACHE_SHIFT;
  1470. if (index > end_index)
  1471. break;
  1472. if (index == end_index) {
  1473. nr = i_size & ~PAGE_CACHE_MASK;
  1474. if (nr <= offset)
  1475. break;
  1476. }
  1477. desc->error = shmem_getpage(inode, index, &page, SGP_READ, NULL);
  1478. if (desc->error) {
  1479. if (desc->error == -EINVAL)
  1480. desc->error = 0;
  1481. break;
  1482. }
  1483. /*
  1484. * We must evaluate after, since reads (unlike writes)
  1485. * are called without i_mutex protection against truncate
  1486. */
  1487. nr = PAGE_CACHE_SIZE;
  1488. i_size = i_size_read(inode);
  1489. end_index = i_size >> PAGE_CACHE_SHIFT;
  1490. if (index == end_index) {
  1491. nr = i_size & ~PAGE_CACHE_MASK;
  1492. if (nr <= offset) {
  1493. if (page)
  1494. page_cache_release(page);
  1495. break;
  1496. }
  1497. }
  1498. nr -= offset;
  1499. if (page) {
  1500. /*
  1501. * If users can be writing to this page using arbitrary
  1502. * virtual addresses, take care about potential aliasing
  1503. * before reading the page on the kernel side.
  1504. */
  1505. if (mapping_writably_mapped(mapping))
  1506. flush_dcache_page(page);
  1507. /*
  1508. * Mark the page accessed if we read the beginning.
  1509. */
  1510. if (!offset)
  1511. mark_page_accessed(page);
  1512. } else {
  1513. page = ZERO_PAGE(0);
  1514. page_cache_get(page);
  1515. }
  1516. /*
  1517. * Ok, we have the page, and it's up-to-date, so
  1518. * now we can copy it to user space...
  1519. *
  1520. * The actor routine returns how many bytes were actually used..
  1521. * NOTE! This may not be the same as how much of a user buffer
  1522. * we filled up (we may be padding etc), so we can only update
  1523. * "pos" here (the actor routine has to update the user buffer
  1524. * pointers and the remaining count).
  1525. */
  1526. ret = actor(desc, page, offset, nr);
  1527. offset += ret;
  1528. index += offset >> PAGE_CACHE_SHIFT;
  1529. offset &= ~PAGE_CACHE_MASK;
  1530. page_cache_release(page);
  1531. if (ret != nr || !desc->count)
  1532. break;
  1533. cond_resched();
  1534. }
  1535. *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
  1536. file_accessed(filp);
  1537. }
  1538. static ssize_t shmem_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
  1539. {
  1540. read_descriptor_t desc;
  1541. if ((ssize_t) count < 0)
  1542. return -EINVAL;
  1543. if (!access_ok(VERIFY_WRITE, buf, count))
  1544. return -EFAULT;
  1545. if (!count)
  1546. return 0;
  1547. desc.written = 0;
  1548. desc.count = count;
  1549. desc.arg.buf = buf;
  1550. desc.error = 0;
  1551. do_shmem_file_read(filp, ppos, &desc, file_read_actor);
  1552. if (desc.written)
  1553. return desc.written;
  1554. return desc.error;
  1555. }
  1556. static ssize_t shmem_file_sendfile(struct file *in_file, loff_t *ppos,
  1557. size_t count, read_actor_t actor, void *target)
  1558. {
  1559. read_descriptor_t desc;
  1560. if (!count)
  1561. return 0;
  1562. desc.written = 0;
  1563. desc.count = count;
  1564. desc.arg.data = target;
  1565. desc.error = 0;
  1566. do_shmem_file_read(in_file, ppos, &desc, actor);
  1567. if (desc.written)
  1568. return desc.written;
  1569. return desc.error;
  1570. }
  1571. static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
  1572. {
  1573. struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
  1574. buf->f_type = TMPFS_MAGIC;
  1575. buf->f_bsize = PAGE_CACHE_SIZE;
  1576. buf->f_namelen = NAME_MAX;
  1577. spin_lock(&sbinfo->stat_lock);
  1578. if (sbinfo->max_blocks) {
  1579. buf->f_blocks = sbinfo->max_blocks;
  1580. buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
  1581. }
  1582. if (sbinfo->max_inodes) {
  1583. buf->f_files = sbinfo->max_inodes;
  1584. buf->f_ffree = sbinfo->free_inodes;
  1585. }
  1586. /* else leave those fields 0 like simple_statfs */
  1587. spin_unlock(&sbinfo->stat_lock);
  1588. return 0;
  1589. }
  1590. /*
  1591. * File creation. Allocate an inode, and we're done..
  1592. */
  1593. static int
  1594. shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
  1595. {
  1596. struct inode *inode = shmem_get_inode(dir->i_sb, mode, dev);
  1597. int error = -ENOSPC;
  1598. if (inode) {
  1599. error = security_inode_init_security(inode, dir, NULL, NULL,
  1600. NULL);
  1601. if (error) {
  1602. if (error != -EOPNOTSUPP) {
  1603. iput(inode);
  1604. return error;
  1605. }
  1606. }
  1607. error = shmem_acl_init(inode, dir);
  1608. if (error) {
  1609. iput(inode);
  1610. return error;
  1611. }
  1612. if (dir->i_mode & S_ISGID) {
  1613. inode->i_gid = dir->i_gid;
  1614. if (S_ISDIR(mode))
  1615. inode->i_mode |= S_ISGID;
  1616. }
  1617. dir->i_size += BOGO_DIRENT_SIZE;
  1618. dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  1619. d_instantiate(dentry, inode);
  1620. dget(dentry); /* Extra count - pin the dentry in core */
  1621. }
  1622. return error;
  1623. }
  1624. static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
  1625. {
  1626. int error;
  1627. if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
  1628. return error;
  1629. inc_nlink(dir);
  1630. return 0;
  1631. }
  1632. static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
  1633. struct nameidata *nd)
  1634. {
  1635. return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
  1636. }
  1637. /*
  1638. * Link a file..
  1639. */
  1640. static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
  1641. {
  1642. struct inode *inode = old_dentry->d_inode;
  1643. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  1644. /*
  1645. * No ordinary (disk based) filesystem counts links as inodes;
  1646. * but each new link needs a new dentry, pinning lowmem, and
  1647. * tmpfs dentries cannot be pruned until they are unlinked.
  1648. */
  1649. if (sbinfo->max_inodes) {
  1650. spin_lock(&sbinfo->stat_lock);
  1651. if (!sbinfo->free_inodes) {
  1652. spin_unlock(&sbinfo->stat_lock);
  1653. return -ENOSPC;
  1654. }
  1655. sbinfo->free_inodes--;
  1656. spin_unlock(&sbinfo->stat_lock);
  1657. }
  1658. dir->i_size += BOGO_DIRENT_SIZE;
  1659. inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  1660. inc_nlink(inode);
  1661. atomic_inc(&inode->i_count); /* New dentry reference */
  1662. dget(dentry); /* Extra pinning count for the created dentry */
  1663. d_instantiate(dentry, inode);
  1664. return 0;
  1665. }
  1666. static int shmem_unlink(struct inode *dir, struct dentry *dentry)
  1667. {
  1668. struct inode *inode = dentry->d_inode;
  1669. if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) {
  1670. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  1671. if (sbinfo->max_inodes) {
  1672. spin_lock(&sbinfo->stat_lock);
  1673. sbinfo->free_inodes++;
  1674. spin_unlock(&sbinfo->stat_lock);
  1675. }
  1676. }
  1677. dir->i_size -= BOGO_DIRENT_SIZE;
  1678. inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  1679. drop_nlink(inode);
  1680. dput(dentry); /* Undo the count from "create" - this does all the work */
  1681. return 0;
  1682. }
  1683. static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
  1684. {
  1685. if (!simple_empty(dentry))
  1686. return -ENOTEMPTY;
  1687. drop_nlink(dentry->d_inode);
  1688. drop_nlink(dir);
  1689. return shmem_unlink(dir, dentry);
  1690. }
  1691. /*
  1692. * The VFS layer already does all the dentry stuff for rename,
  1693. * we just have to decrement the usage count for the target if
  1694. * it exists so that the VFS layer correctly free's it when it
  1695. * gets overwritten.
  1696. */
  1697. static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
  1698. {
  1699. struct inode *inode = old_dentry->d_inode;
  1700. int they_are_dirs = S_ISDIR(inode->i_mode);
  1701. if (!simple_empty(new_dentry))
  1702. return -ENOTEMPTY;
  1703. if (new_dentry->d_inode) {
  1704. (void) shmem_unlink(new_dir, new_dentry);
  1705. if (they_are_dirs)
  1706. drop_nlink(old_dir);
  1707. } else if (they_are_dirs) {
  1708. drop_nlink(old_dir);
  1709. inc_nlink(new_dir);
  1710. }
  1711. old_dir->i_size -= BOGO_DIRENT_SIZE;
  1712. new_dir->i_size += BOGO_DIRENT_SIZE;
  1713. old_dir->i_ctime = old_dir->i_mtime =
  1714. new_dir->i_ctime = new_dir->i_mtime =
  1715. inode->i_ctime = CURRENT_TIME;
  1716. return 0;
  1717. }
  1718. static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
  1719. {
  1720. int error;
  1721. int len;
  1722. struct inode *inode;
  1723. struct page *page = NULL;
  1724. char *kaddr;
  1725. struct shmem_inode_info *info;
  1726. len = strlen(symname) + 1;
  1727. if (len > PAGE_CACHE_SIZE)
  1728. return -ENAMETOOLONG;
  1729. inode = shmem_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0);
  1730. if (!inode)
  1731. return -ENOSPC;
  1732. error = security_inode_init_security(inode, dir, NULL, NULL,
  1733. NULL);
  1734. if (error) {
  1735. if (error != -EOPNOTSUPP) {
  1736. iput(inode);
  1737. return error;
  1738. }
  1739. error = 0;
  1740. }
  1741. info = SHMEM_I(inode);
  1742. inode->i_size = len-1;
  1743. if (len <= (char *)inode - (char *)info) {
  1744. /* do it inline */
  1745. memcpy(info, symname, len);
  1746. inode->i_op = &shmem_symlink_inline_operations;
  1747. } else {
  1748. error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
  1749. if (error) {
  1750. iput(inode);
  1751. return error;
  1752. }
  1753. inode->i_op = &shmem_symlink_inode_operations;
  1754. kaddr = kmap_atomic(page, KM_USER0);
  1755. memcpy(kaddr, symname, len);
  1756. kunmap_atomic(kaddr, KM_USER0);
  1757. set_page_dirty(page);
  1758. page_cache_release(page);
  1759. }
  1760. if (dir->i_mode & S_ISGID)
  1761. inode->i_gid = dir->i_gid;
  1762. dir->i_size += BOGO_DIRENT_SIZE;
  1763. dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  1764. d_instantiate(dentry, inode);
  1765. dget(dentry);
  1766. return 0;
  1767. }
  1768. static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
  1769. {
  1770. nd_set_link(nd, (char *)SHMEM_I(dentry->d_inode));
  1771. return NULL;
  1772. }
  1773. static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
  1774. {
  1775. struct page *page = NULL;
  1776. int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
  1777. nd_set_link(nd, res ? ERR_PTR(res) : kmap(page));
  1778. return page;
  1779. }
  1780. static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
  1781. {
  1782. if (!IS_ERR(nd_get_link(nd))) {
  1783. struct page *page = cookie;
  1784. kunmap(page);
  1785. mark_page_accessed(page);
  1786. page_cache_release(page);
  1787. }
  1788. }
  1789. static const struct inode_operations shmem_symlink_inline_operations = {
  1790. .readlink = generic_readlink,
  1791. .follow_link = shmem_follow_link_inline,
  1792. };
  1793. static const struct inode_operations shmem_symlink_inode_operations = {
  1794. .truncate = shmem_truncate,
  1795. .readlink = generic_readlink,
  1796. .follow_link = shmem_follow_link,
  1797. .put_link = shmem_put_link,
  1798. };
  1799. #ifdef CONFIG_TMPFS_POSIX_ACL
  1800. /**
  1801. * Superblocks without xattr inode operations will get security.* xattr
  1802. * support from the VFS "for free". As soon as we have any other xattrs
  1803. * like ACLs, we also need to implement the security.* handlers at
  1804. * filesystem level, though.
  1805. */
  1806. static size_t shmem_xattr_security_list(struct inode *inode, char *list,
  1807. size_t list_len, const char *name,
  1808. size_t name_len)
  1809. {
  1810. return security_inode_listsecurity(inode, list, list_len);
  1811. }
  1812. static int shmem_xattr_security_get(struct inode *inode, const char *name,
  1813. void *buffer, size_t size)
  1814. {
  1815. if (strcmp(name, "") == 0)
  1816. return -EINVAL;
  1817. return security_inode_getsecurity(inode, name, buffer, size,
  1818. -EOPNOTSUPP);
  1819. }
  1820. static int shmem_xattr_security_set(struct inode *inode, const char *name,
  1821. const void *value, size_t size, int flags)
  1822. {
  1823. if (strcmp(name, "") == 0)
  1824. return -EINVAL;
  1825. return security_inode_setsecurity(inode, name, value, size, flags);
  1826. }
  1827. static struct xattr_handler shmem_xattr_security_handler = {
  1828. .prefix = XATTR_SECURITY_PREFIX,
  1829. .list = shmem_xattr_security_list,
  1830. .get = shmem_xattr_security_get,
  1831. .set = shmem_xattr_security_set,
  1832. };
  1833. static struct xattr_handler *shmem_xattr_handlers[] = {
  1834. &shmem_xattr_acl_access_handler,
  1835. &shmem_xattr_acl_default_handler,
  1836. &shmem_xattr_security_handler,
  1837. NULL
  1838. };
  1839. #endif
  1840. static struct dentry *shmem_get_parent(struct dentry *child)
  1841. {
  1842. return ERR_PTR(-ESTALE);
  1843. }
  1844. static int shmem_match(struct inode *ino, void *vfh)
  1845. {
  1846. __u32 *fh = vfh;
  1847. __u64 inum = fh[2];
  1848. inum = (inum << 32) | fh[1];
  1849. return ino->i_ino == inum && fh[0] == ino->i_generation;
  1850. }
  1851. static struct dentry *shmem_get_dentry(struct super_block *sb, void *vfh)
  1852. {
  1853. struct dentry *de = NULL;
  1854. struct inode *inode;
  1855. __u32 *fh = vfh;
  1856. __u64 inum = fh[2];
  1857. inum = (inum << 32) | fh[1];
  1858. inode = ilookup5(sb, (unsigned long)(inum+fh[0]), shmem_match, vfh);
  1859. if (inode) {
  1860. de = d_find_alias(inode);
  1861. iput(inode);
  1862. }
  1863. return de? de: ERR_PTR(-ESTALE);
  1864. }
  1865. static struct dentry *shmem_decode_fh(struct super_block *sb, __u32 *fh,
  1866. int len, int type,
  1867. int (*acceptable)(void *context, struct dentry *de),
  1868. void *context)
  1869. {
  1870. if (len < 3)
  1871. return ERR_PTR(-ESTALE);
  1872. return sb->s_export_op->find_exported_dentry(sb, fh, NULL, acceptable,
  1873. context);
  1874. }
  1875. static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
  1876. int connectable)
  1877. {
  1878. struct inode *inode = dentry->d_inode;
  1879. if (*len < 3)
  1880. return 255;
  1881. if (hlist_unhashed(&inode->i_hash)) {
  1882. /* Unfortunately insert_inode_hash is not idempotent,
  1883. * so as we hash inodes here rather than at creation
  1884. * time, we need a lock to ensure we only try
  1885. * to do it once
  1886. */
  1887. static DEFINE_SPINLOCK(lock);
  1888. spin_lock(&lock);
  1889. if (hlist_unhashed(&inode->i_hash))
  1890. __insert_inode_hash(inode,
  1891. inode->i_ino + inode->i_generation);
  1892. spin_unlock(&lock);
  1893. }
  1894. fh[0] = inode->i_generation;
  1895. fh[1] = inode->i_ino;
  1896. fh[2] = ((__u64)inode->i_ino) >> 32;
  1897. *len = 3;
  1898. return 1;
  1899. }
  1900. static struct export_operations shmem_export_ops = {
  1901. .get_parent = shmem_get_parent,
  1902. .get_dentry = shmem_get_dentry,
  1903. .encode_fh = shmem_encode_fh,
  1904. .decode_fh = shmem_decode_fh,
  1905. };
  1906. static int shmem_parse_options(char *options, int *mode, uid_t *uid,
  1907. gid_t *gid, unsigned long *blocks, unsigned long *inodes,
  1908. int *policy, nodemask_t *policy_nodes)
  1909. {
  1910. char *this_char, *value, *rest;
  1911. while (options != NULL) {
  1912. this_char = options;
  1913. for (;;) {
  1914. /*
  1915. * NUL-terminate this option: unfortunately,
  1916. * mount options form a comma-separated list,
  1917. * but mpol's nodelist may also contain commas.
  1918. */
  1919. options = strchr(options, ',');
  1920. if (options == NULL)
  1921. break;
  1922. options++;
  1923. if (!isdigit(*options)) {
  1924. options[-1] = '\0';
  1925. break;
  1926. }
  1927. }
  1928. if (!*this_char)
  1929. continue;
  1930. if ((value = strchr(this_char,'=')) != NULL) {
  1931. *value++ = 0;
  1932. } else {
  1933. printk(KERN_ERR
  1934. "tmpfs: No value for mount option '%s'\n",
  1935. this_char);
  1936. return 1;
  1937. }
  1938. if (!strcmp(this_char,"size")) {
  1939. unsigned long long size;
  1940. size = memparse(value,&rest);
  1941. if (*rest == '%') {
  1942. size <<= PAGE_SHIFT;
  1943. size *= totalram_pages;
  1944. do_div(size, 100);
  1945. rest++;
  1946. }
  1947. if (*rest)
  1948. goto bad_val;
  1949. *blocks = size >> PAGE_CACHE_SHIFT;
  1950. } else if (!strcmp(this_char,"nr_blocks")) {
  1951. *blocks = memparse(value,&rest);
  1952. if (*rest)
  1953. goto bad_val;
  1954. } else if (!strcmp(this_char,"nr_inodes")) {
  1955. *inodes = memparse(value,&rest);
  1956. if (*rest)
  1957. goto bad_val;
  1958. } else if (!strcmp(this_char,"mode")) {
  1959. if (!mode)
  1960. continue;
  1961. *mode = simple_strtoul(value,&rest,8);
  1962. if (*rest)
  1963. goto bad_val;
  1964. } else if (!strcmp(this_char,"uid")) {
  1965. if (!uid)
  1966. continue;
  1967. *uid = simple_strtoul(value,&rest,0);
  1968. if (*rest)
  1969. goto bad_val;
  1970. } else if (!strcmp(this_char,"gid")) {
  1971. if (!gid)
  1972. continue;
  1973. *gid = simple_strtoul(value,&rest,0);
  1974. if (*rest)
  1975. goto bad_val;
  1976. } else if (!strcmp(this_char,"mpol")) {
  1977. if (shmem_parse_mpol(value,policy,policy_nodes))
  1978. goto bad_val;
  1979. } else {
  1980. printk(KERN_ERR "tmpfs: Bad mount option %s\n",
  1981. this_char);
  1982. return 1;
  1983. }
  1984. }
  1985. return 0;
  1986. bad_val:
  1987. printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
  1988. value, this_char);
  1989. return 1;
  1990. }
  1991. static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
  1992. {
  1993. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  1994. unsigned long max_blocks = sbinfo->max_blocks;
  1995. unsigned long max_inodes = sbinfo->max_inodes;
  1996. int policy = sbinfo->policy;
  1997. nodemask_t policy_nodes = sbinfo->policy_nodes;
  1998. unsigned long blocks;
  1999. unsigned long inodes;
  2000. int error = -EINVAL;
  2001. if (shmem_parse_options(data, NULL, NULL, NULL, &max_blocks,
  2002. &max_inodes, &policy, &policy_nodes))
  2003. return error;
  2004. spin_lock(&sbinfo->stat_lock);
  2005. blocks = sbinfo->max_blocks - sbinfo->free_blocks;
  2006. inodes = sbinfo->max_inodes - sbinfo->free_inodes;
  2007. if (max_blocks < blocks)
  2008. goto out;
  2009. if (max_inodes < inodes)
  2010. goto out;
  2011. /*
  2012. * Those tests also disallow limited->unlimited while any are in
  2013. * use, so i_blocks will always be zero when max_blocks is zero;
  2014. * but we must separately disallow unlimited->limited, because
  2015. * in that case we have no record of how much is already in use.
  2016. */
  2017. if (max_blocks && !sbinfo->max_blocks)
  2018. goto out;
  2019. if (max_inodes && !sbinfo->max_inodes)
  2020. goto out;
  2021. error = 0;
  2022. sbinfo->max_blocks = max_blocks;
  2023. sbinfo->free_blocks = max_blocks - blocks;
  2024. sbinfo->max_inodes = max_inodes;
  2025. sbinfo->free_inodes = max_inodes - inodes;
  2026. sbinfo->policy = policy;
  2027. sbinfo->policy_nodes = policy_nodes;
  2028. out:
  2029. spin_unlock(&sbinfo->stat_lock);
  2030. return error;
  2031. }
  2032. #endif
  2033. static void shmem_put_super(struct super_block *sb)
  2034. {
  2035. kfree(sb->s_fs_info);
  2036. sb->s_fs_info = NULL;
  2037. }
  2038. static int shmem_fill_super(struct super_block *sb,
  2039. void *data, int silent)
  2040. {
  2041. struct inode *inode;
  2042. struct dentry *root;
  2043. int mode = S_IRWXUGO | S_ISVTX;
  2044. uid_t uid = current->fsuid;
  2045. gid_t gid = current->fsgid;
  2046. int err = -ENOMEM;
  2047. struct shmem_sb_info *sbinfo;
  2048. unsigned long blocks = 0;
  2049. unsigned long inodes = 0;
  2050. int policy = MPOL_DEFAULT;
  2051. nodemask_t policy_nodes = node_online_map;
  2052. #ifdef CONFIG_TMPFS
  2053. /*
  2054. * Per default we only allow half of the physical ram per
  2055. * tmpfs instance, limiting inodes to one per page of lowmem;
  2056. * but the internal instance is left unlimited.
  2057. */
  2058. if (!(sb->s_flags & MS_NOUSER)) {
  2059. blocks = totalram_pages / 2;
  2060. inodes = totalram_pages - totalhigh_pages;
  2061. if (inodes > blocks)
  2062. inodes = blocks;
  2063. if (shmem_parse_options(data, &mode, &uid, &gid, &blocks,
  2064. &inodes, &policy, &policy_nodes))
  2065. return -EINVAL;
  2066. }
  2067. sb->s_export_op = &shmem_export_ops;
  2068. #else
  2069. sb->s_flags |= MS_NOUSER;
  2070. #endif
  2071. /* Round up to L1_CACHE_BYTES to resist false sharing */
  2072. sbinfo = kmalloc(max((int)sizeof(struct shmem_sb_info),
  2073. L1_CACHE_BYTES), GFP_KERNEL);
  2074. if (!sbinfo)
  2075. return -ENOMEM;
  2076. spin_lock_init(&sbinfo->stat_lock);
  2077. sbinfo->max_blocks = blocks;
  2078. sbinfo->free_blocks = blocks;
  2079. sbinfo->max_inodes = inodes;
  2080. sbinfo->free_inodes = inodes;
  2081. sbinfo->policy = policy;
  2082. sbinfo->policy_nodes = policy_nodes;
  2083. sb->s_fs_info = sbinfo;
  2084. sb->s_maxbytes = SHMEM_MAX_BYTES;
  2085. sb->s_blocksize = PAGE_CACHE_SIZE;
  2086. sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
  2087. sb->s_magic = TMPFS_MAGIC;
  2088. sb->s_op = &shmem_ops;
  2089. sb->s_time_gran = 1;
  2090. #ifdef CONFIG_TMPFS_POSIX_ACL
  2091. sb->s_xattr = shmem_xattr_handlers;
  2092. sb->s_flags |= MS_POSIXACL;
  2093. #endif
  2094. inode = shmem_get_inode(sb, S_IFDIR | mode, 0);
  2095. if (!inode)
  2096. goto failed;
  2097. inode->i_uid = uid;
  2098. inode->i_gid = gid;
  2099. root = d_alloc_root(inode);
  2100. if (!root)
  2101. goto failed_iput;
  2102. sb->s_root = root;
  2103. return 0;
  2104. failed_iput:
  2105. iput(inode);
  2106. failed:
  2107. shmem_put_super(sb);
  2108. return err;
  2109. }
  2110. static struct kmem_cache *shmem_inode_cachep;
  2111. static struct inode *shmem_alloc_inode(struct super_block *sb)
  2112. {
  2113. struct shmem_inode_info *p;
  2114. p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
  2115. if (!p)
  2116. return NULL;
  2117. return &p->vfs_inode;
  2118. }
  2119. static void shmem_destroy_inode(struct inode *inode)
  2120. {
  2121. if ((inode->i_mode & S_IFMT) == S_IFREG) {
  2122. /* only struct inode is valid if it's an inline symlink */
  2123. mpol_free_shared_policy(&SHMEM_I(inode)->policy);
  2124. }
  2125. shmem_acl_destroy_inode(inode);
  2126. kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
  2127. }
  2128. static void init_once(void *foo, struct kmem_cache *cachep,
  2129. unsigned long flags)
  2130. {
  2131. struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
  2132. if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
  2133. SLAB_CTOR_CONSTRUCTOR) {
  2134. inode_init_once(&p->vfs_inode);
  2135. #ifdef CONFIG_TMPFS_POSIX_ACL
  2136. p->i_acl = NULL;
  2137. p->i_default_acl = NULL;
  2138. #endif
  2139. }
  2140. }
  2141. static int init_inodecache(void)
  2142. {
  2143. shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
  2144. sizeof(struct shmem_inode_info),
  2145. 0, 0, init_once, NULL);
  2146. if (shmem_inode_cachep == NULL)
  2147. return -ENOMEM;
  2148. return 0;
  2149. }
  2150. static void destroy_inodecache(void)
  2151. {
  2152. kmem_cache_destroy(shmem_inode_cachep);
  2153. }
  2154. static const struct address_space_operations shmem_aops = {
  2155. .writepage = shmem_writepage,
  2156. .set_page_dirty = __set_page_dirty_no_writeback,
  2157. #ifdef CONFIG_TMPFS
  2158. .prepare_write = shmem_prepare_write,
  2159. .commit_write = simple_commit_write,
  2160. #endif
  2161. .migratepage = migrate_page,
  2162. };
  2163. static const struct file_operations shmem_file_operations = {
  2164. .mmap = shmem_mmap,
  2165. #ifdef CONFIG_TMPFS
  2166. .llseek = generic_file_llseek,
  2167. .read = shmem_file_read,
  2168. .write = shmem_file_write,
  2169. .fsync = simple_sync_file,
  2170. .sendfile = shmem_file_sendfile,
  2171. #endif
  2172. };
  2173. static const struct inode_operations shmem_inode_operations = {
  2174. .truncate = shmem_truncate,
  2175. .setattr = shmem_notify_change,
  2176. .truncate_range = shmem_truncate_range,
  2177. #ifdef CONFIG_TMPFS_POSIX_ACL
  2178. .setxattr = generic_setxattr,
  2179. .getxattr = generic_getxattr,
  2180. .listxattr = generic_listxattr,
  2181. .removexattr = generic_removexattr,
  2182. .permission = shmem_permission,
  2183. #endif
  2184. };
  2185. static const struct inode_operations shmem_dir_inode_operations = {
  2186. #ifdef CONFIG_TMPFS
  2187. .create = shmem_create,
  2188. .lookup = simple_lookup,
  2189. .link = shmem_link,
  2190. .unlink = shmem_unlink,
  2191. .symlink = shmem_symlink,
  2192. .mkdir = shmem_mkdir,
  2193. .rmdir = shmem_rmdir,
  2194. .mknod = shmem_mknod,
  2195. .rename = shmem_rename,
  2196. #endif
  2197. #ifdef CONFIG_TMPFS_POSIX_ACL
  2198. .setattr = shmem_notify_change,
  2199. .setxattr = generic_setxattr,
  2200. .getxattr = generic_getxattr,
  2201. .listxattr = generic_listxattr,
  2202. .removexattr = generic_removexattr,
  2203. .permission = shmem_permission,
  2204. #endif
  2205. };
  2206. static const struct inode_operations shmem_special_inode_operations = {
  2207. #ifdef CONFIG_TMPFS_POSIX_ACL
  2208. .setattr = shmem_notify_change,
  2209. .setxattr = generic_setxattr,
  2210. .getxattr = generic_getxattr,
  2211. .listxattr = generic_listxattr,
  2212. .removexattr = generic_removexattr,
  2213. .permission = shmem_permission,
  2214. #endif
  2215. };
  2216. static const struct super_operations shmem_ops = {
  2217. .alloc_inode = shmem_alloc_inode,
  2218. .destroy_inode = shmem_destroy_inode,
  2219. #ifdef CONFIG_TMPFS
  2220. .statfs = shmem_statfs,
  2221. .remount_fs = shmem_remount_fs,
  2222. #endif
  2223. .delete_inode = shmem_delete_inode,
  2224. .drop_inode = generic_delete_inode,
  2225. .put_super = shmem_put_super,
  2226. };
  2227. static struct vm_operations_struct shmem_vm_ops = {
  2228. .nopage = shmem_nopage,
  2229. .populate = shmem_populate,
  2230. #ifdef CONFIG_NUMA
  2231. .set_policy = shmem_set_policy,
  2232. .get_policy = shmem_get_policy,
  2233. #endif
  2234. };
  2235. static int shmem_get_sb(struct file_system_type *fs_type,
  2236. int flags, const char *dev_name, void *data, struct vfsmount *mnt)
  2237. {
  2238. return get_sb_nodev(fs_type, flags, data, shmem_fill_super, mnt);
  2239. }
  2240. static struct file_system_type tmpfs_fs_type = {
  2241. .owner = THIS_MODULE,
  2242. .name = "tmpfs",
  2243. .get_sb = shmem_get_sb,
  2244. .kill_sb = kill_litter_super,
  2245. };
  2246. static struct vfsmount *shm_mnt;
  2247. static int __init init_tmpfs(void)
  2248. {
  2249. int error;
  2250. error = init_inodecache();
  2251. if (error)
  2252. goto out3;
  2253. error = register_filesystem(&tmpfs_fs_type);
  2254. if (error) {
  2255. printk(KERN_ERR "Could not register tmpfs\n");
  2256. goto out2;
  2257. }
  2258. shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER,
  2259. tmpfs_fs_type.name, NULL);
  2260. if (IS_ERR(shm_mnt)) {
  2261. error = PTR_ERR(shm_mnt);
  2262. printk(KERN_ERR "Could not kern_mount tmpfs\n");
  2263. goto out1;
  2264. }
  2265. return 0;
  2266. out1:
  2267. unregister_filesystem(&tmpfs_fs_type);
  2268. out2:
  2269. destroy_inodecache();
  2270. out3:
  2271. shm_mnt = ERR_PTR(error);
  2272. return error;
  2273. }
  2274. module_init(init_tmpfs)
  2275. /*
  2276. * shmem_file_setup - get an unlinked file living in tmpfs
  2277. *
  2278. * @name: name for dentry (to be seen in /proc/<pid>/maps
  2279. * @size: size to be set for the file
  2280. *
  2281. */
  2282. struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
  2283. {
  2284. int error;
  2285. struct file *file;
  2286. struct inode *inode;
  2287. struct dentry *dentry, *root;
  2288. struct qstr this;
  2289. if (IS_ERR(shm_mnt))
  2290. return (void *)shm_mnt;
  2291. if (size < 0 || size > SHMEM_MAX_BYTES)
  2292. return ERR_PTR(-EINVAL);
  2293. if (shmem_acct_size(flags, size))
  2294. return ERR_PTR(-ENOMEM);
  2295. error = -ENOMEM;
  2296. this.name = name;
  2297. this.len = strlen(name);
  2298. this.hash = 0; /* will go */
  2299. root = shm_mnt->mnt_root;
  2300. dentry = d_alloc(root, &this);
  2301. if (!dentry)
  2302. goto put_memory;
  2303. error = -ENFILE;
  2304. file = get_empty_filp();
  2305. if (!file)
  2306. goto put_dentry;
  2307. error = -ENOSPC;
  2308. inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0);
  2309. if (!inode)
  2310. goto close_file;
  2311. SHMEM_I(inode)->flags = flags & VM_ACCOUNT;
  2312. d_instantiate(dentry, inode);
  2313. inode->i_size = size;
  2314. inode->i_nlink = 0; /* It is unlinked */
  2315. file->f_path.mnt = mntget(shm_mnt);
  2316. file->f_path.dentry = dentry;
  2317. file->f_mapping = inode->i_mapping;
  2318. file->f_op = &shmem_file_operations;
  2319. file->f_mode = FMODE_WRITE | FMODE_READ;
  2320. return file;
  2321. close_file:
  2322. put_filp(file);
  2323. put_dentry:
  2324. dput(dentry);
  2325. put_memory:
  2326. shmem_unacct_size(flags, size);
  2327. return ERR_PTR(error);
  2328. }
  2329. /*
  2330. * shmem_zero_setup - setup a shared anonymous mapping
  2331. *
  2332. * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
  2333. */
  2334. int shmem_zero_setup(struct vm_area_struct *vma)
  2335. {
  2336. struct file *file;
  2337. loff_t size = vma->vm_end - vma->vm_start;
  2338. file = shmem_file_setup("dev/zero", size, vma->vm_flags);
  2339. if (IS_ERR(file))
  2340. return PTR_ERR(file);
  2341. if (vma->vm_file)
  2342. fput(vma->vm_file);
  2343. vma->vm_file = file;
  2344. vma->vm_ops = &shmem_vm_ops;
  2345. return 0;
  2346. }