dcache.c 85 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * fs/dcache.c
  4. *
  5. * Complete reimplementation
  6. * (C) 1997 Thomas Schoebel-Theuer,
  7. * with heavy changes by Linus Torvalds
  8. */
  9. /*
  10. * Notes on the allocation strategy:
  11. *
  12. * The dcache is a master of the icache - whenever a dcache entry
  13. * exists, the inode will always exist. "iput()" is done either when
  14. * the dcache entry is deleted or garbage collected.
  15. */
  16. #include <linux/ratelimit.h>
  17. #include <linux/string.h>
  18. #include <linux/mm.h>
  19. #include <linux/fs.h>
  20. #include <linux/fscrypt.h>
  21. #include <linux/fsnotify.h>
  22. #include <linux/slab.h>
  23. #include <linux/init.h>
  24. #include <linux/hash.h>
  25. #include <linux/cache.h>
  26. #include <linux/export.h>
  27. #include <linux/security.h>
  28. #include <linux/seqlock.h>
  29. #include <linux/memblock.h>
  30. #include <linux/bit_spinlock.h>
  31. #include <linux/rculist_bl.h>
  32. #include <linux/list_lru.h>
  33. #include "internal.h"
  34. #include "mount.h"
  35. /*
  36. * Usage:
  37. * dcache->d_inode->i_lock protects:
  38. * - i_dentry, d_u.d_alias, d_inode of aliases
  39. * dcache_hash_bucket lock protects:
  40. * - the dcache hash table
  41. * s_roots bl list spinlock protects:
  42. * - the s_roots list (see __d_drop)
  43. * dentry->d_sb->s_dentry_lru_lock protects:
  44. * - the dcache lru lists and counters
  45. * d_lock protects:
  46. * - d_flags
  47. * - d_name
  48. * - d_lru
  49. * - d_count
  50. * - d_unhashed()
  51. * - d_parent and d_subdirs
  52. * - childrens' d_child and d_parent
  53. * - d_u.d_alias, d_inode
  54. *
  55. * Ordering:
  56. * dentry->d_inode->i_lock
  57. * dentry->d_lock
  58. * dentry->d_sb->s_dentry_lru_lock
  59. * dcache_hash_bucket lock
  60. * s_roots lock
  61. *
  62. * If there is an ancestor relationship:
  63. * dentry->d_parent->...->d_parent->d_lock
  64. * ...
  65. * dentry->d_parent->d_lock
  66. * dentry->d_lock
  67. *
  68. * If no ancestor relationship:
  69. * arbitrary, since it's serialized on rename_lock
  70. */
  71. int sysctl_vfs_cache_pressure __read_mostly = 100;
  72. EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
  73. __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
  74. EXPORT_SYMBOL(rename_lock);
  75. static struct kmem_cache *dentry_cache __read_mostly;
  76. const struct qstr empty_name = QSTR_INIT("", 0);
  77. EXPORT_SYMBOL(empty_name);
  78. const struct qstr slash_name = QSTR_INIT("/", 1);
  79. EXPORT_SYMBOL(slash_name);
  80. /*
  81. * This is the single most critical data structure when it comes
  82. * to the dcache: the hashtable for lookups. Somebody should try
  83. * to make this good - I've just made it work.
  84. *
  85. * This hash-function tries to avoid losing too many bits of hash
  86. * information, yet avoid using a prime hash-size or similar.
  87. */
  88. static unsigned int d_hash_shift __read_mostly;
  89. static struct hlist_bl_head *dentry_hashtable __read_mostly;
  90. static inline struct hlist_bl_head *d_hash(unsigned int hash)
  91. {
  92. return dentry_hashtable + (hash >> d_hash_shift);
  93. }
  94. #define IN_LOOKUP_SHIFT 10
  95. static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT];
  96. static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent,
  97. unsigned int hash)
  98. {
  99. hash += (unsigned long) parent / L1_CACHE_BYTES;
  100. return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT);
  101. }
  102. /* Statistics gathering. */
  103. struct dentry_stat_t dentry_stat = {
  104. .age_limit = 45,
  105. };
  106. static DEFINE_PER_CPU(long, nr_dentry);
  107. static DEFINE_PER_CPU(long, nr_dentry_unused);
  108. static DEFINE_PER_CPU(long, nr_dentry_negative);
  109. #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
  110. /*
  111. * Here we resort to our own counters instead of using generic per-cpu counters
  112. * for consistency with what the vfs inode code does. We are expected to harvest
  113. * better code and performance by having our own specialized counters.
  114. *
  115. * Please note that the loop is done over all possible CPUs, not over all online
  116. * CPUs. The reason for this is that we don't want to play games with CPUs going
  117. * on and off. If one of them goes off, we will just keep their counters.
  118. *
  119. * glommer: See cffbc8a for details, and if you ever intend to change this,
  120. * please update all vfs counters to match.
  121. */
  122. static long get_nr_dentry(void)
  123. {
  124. int i;
  125. long sum = 0;
  126. for_each_possible_cpu(i)
  127. sum += per_cpu(nr_dentry, i);
  128. return sum < 0 ? 0 : sum;
  129. }
  130. static long get_nr_dentry_unused(void)
  131. {
  132. int i;
  133. long sum = 0;
  134. for_each_possible_cpu(i)
  135. sum += per_cpu(nr_dentry_unused, i);
  136. return sum < 0 ? 0 : sum;
  137. }
  138. static long get_nr_dentry_negative(void)
  139. {
  140. int i;
  141. long sum = 0;
  142. for_each_possible_cpu(i)
  143. sum += per_cpu(nr_dentry_negative, i);
  144. return sum < 0 ? 0 : sum;
  145. }
  146. int proc_nr_dentry(struct ctl_table *table, int write, void *buffer,
  147. size_t *lenp, loff_t *ppos)
  148. {
  149. dentry_stat.nr_dentry = get_nr_dentry();
  150. dentry_stat.nr_unused = get_nr_dentry_unused();
  151. dentry_stat.nr_negative = get_nr_dentry_negative();
  152. return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
  153. }
  154. #endif
  155. /*
  156. * Compare 2 name strings, return 0 if they match, otherwise non-zero.
  157. * The strings are both count bytes long, and count is non-zero.
  158. */
  159. #ifdef CONFIG_DCACHE_WORD_ACCESS
  160. #include <asm/word-at-a-time.h>
  161. /*
  162. * NOTE! 'cs' and 'scount' come from a dentry, so it has a
  163. * aligned allocation for this particular component. We don't
  164. * strictly need the load_unaligned_zeropad() safety, but it
  165. * doesn't hurt either.
  166. *
  167. * In contrast, 'ct' and 'tcount' can be from a pathname, and do
  168. * need the careful unaligned handling.
  169. */
  170. static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
  171. {
  172. unsigned long a,b,mask;
  173. for (;;) {
  174. a = read_word_at_a_time(cs);
  175. b = load_unaligned_zeropad(ct);
  176. if (tcount < sizeof(unsigned long))
  177. break;
  178. if (unlikely(a != b))
  179. return 1;
  180. cs += sizeof(unsigned long);
  181. ct += sizeof(unsigned long);
  182. tcount -= sizeof(unsigned long);
  183. if (!tcount)
  184. return 0;
  185. }
  186. mask = bytemask_from_count(tcount);
  187. return unlikely(!!((a ^ b) & mask));
  188. }
  189. #else
  190. static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
  191. {
  192. do {
  193. if (*cs != *ct)
  194. return 1;
  195. cs++;
  196. ct++;
  197. tcount--;
  198. } while (tcount);
  199. return 0;
  200. }
  201. #endif
  202. static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
  203. {
  204. /*
  205. * Be careful about RCU walk racing with rename:
  206. * use 'READ_ONCE' to fetch the name pointer.
  207. *
  208. * NOTE! Even if a rename will mean that the length
  209. * was not loaded atomically, we don't care. The
  210. * RCU walk will check the sequence count eventually,
  211. * and catch it. And we won't overrun the buffer,
  212. * because we're reading the name pointer atomically,
  213. * and a dentry name is guaranteed to be properly
  214. * terminated with a NUL byte.
  215. *
  216. * End result: even if 'len' is wrong, we'll exit
  217. * early because the data cannot match (there can
  218. * be no NUL in the ct/tcount data)
  219. */
  220. const unsigned char *cs = READ_ONCE(dentry->d_name.name);
  221. return dentry_string_cmp(cs, ct, tcount);
  222. }
  223. struct external_name {
  224. union {
  225. atomic_t count;
  226. struct rcu_head head;
  227. } u;
  228. unsigned char name[];
  229. };
  230. static inline struct external_name *external_name(struct dentry *dentry)
  231. {
  232. return container_of(dentry->d_name.name, struct external_name, name[0]);
  233. }
  234. static void __d_free(struct rcu_head *head)
  235. {
  236. struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
  237. kmem_cache_free(dentry_cache, dentry);
  238. }
  239. static void __d_free_external(struct rcu_head *head)
  240. {
  241. struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
  242. kfree(external_name(dentry));
  243. kmem_cache_free(dentry_cache, dentry);
  244. }
  245. static inline int dname_external(const struct dentry *dentry)
  246. {
  247. return dentry->d_name.name != dentry->d_iname;
  248. }
  249. void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry)
  250. {
  251. spin_lock(&dentry->d_lock);
  252. name->name = dentry->d_name;
  253. if (unlikely(dname_external(dentry))) {
  254. atomic_inc(&external_name(dentry)->u.count);
  255. } else {
  256. memcpy(name->inline_name, dentry->d_iname,
  257. dentry->d_name.len + 1);
  258. name->name.name = name->inline_name;
  259. }
  260. spin_unlock(&dentry->d_lock);
  261. }
  262. EXPORT_SYMBOL(take_dentry_name_snapshot);
  263. void release_dentry_name_snapshot(struct name_snapshot *name)
  264. {
  265. if (unlikely(name->name.name != name->inline_name)) {
  266. struct external_name *p;
  267. p = container_of(name->name.name, struct external_name, name[0]);
  268. if (unlikely(atomic_dec_and_test(&p->u.count)))
  269. kfree_rcu(p, u.head);
  270. }
  271. }
  272. EXPORT_SYMBOL(release_dentry_name_snapshot);
  273. static inline void __d_set_inode_and_type(struct dentry *dentry,
  274. struct inode *inode,
  275. unsigned type_flags)
  276. {
  277. unsigned flags;
  278. dentry->d_inode = inode;
  279. flags = READ_ONCE(dentry->d_flags);
  280. flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
  281. flags |= type_flags;
  282. smp_store_release(&dentry->d_flags, flags);
  283. }
  284. static inline void __d_clear_type_and_inode(struct dentry *dentry)
  285. {
  286. unsigned flags = READ_ONCE(dentry->d_flags);
  287. flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
  288. WRITE_ONCE(dentry->d_flags, flags);
  289. dentry->d_inode = NULL;
  290. if (dentry->d_flags & DCACHE_LRU_LIST)
  291. this_cpu_inc(nr_dentry_negative);
  292. }
  293. static void dentry_free(struct dentry *dentry)
  294. {
  295. WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias));
  296. if (unlikely(dname_external(dentry))) {
  297. struct external_name *p = external_name(dentry);
  298. if (likely(atomic_dec_and_test(&p->u.count))) {
  299. call_rcu(&dentry->d_u.d_rcu, __d_free_external);
  300. return;
  301. }
  302. }
  303. /* if dentry was never visible to RCU, immediate free is OK */
  304. if (dentry->d_flags & DCACHE_NORCU)
  305. __d_free(&dentry->d_u.d_rcu);
  306. else
  307. call_rcu(&dentry->d_u.d_rcu, __d_free);
  308. }
  309. /*
  310. * Release the dentry's inode, using the filesystem
  311. * d_iput() operation if defined.
  312. */
  313. static void dentry_unlink_inode(struct dentry * dentry)
  314. __releases(dentry->d_lock)
  315. __releases(dentry->d_inode->i_lock)
  316. {
  317. struct inode *inode = dentry->d_inode;
  318. raw_write_seqcount_begin(&dentry->d_seq);
  319. __d_clear_type_and_inode(dentry);
  320. hlist_del_init(&dentry->d_u.d_alias);
  321. raw_write_seqcount_end(&dentry->d_seq);
  322. spin_unlock(&dentry->d_lock);
  323. spin_unlock(&inode->i_lock);
  324. if (!inode->i_nlink)
  325. fsnotify_inoderemove(inode);
  326. if (dentry->d_op && dentry->d_op->d_iput)
  327. dentry->d_op->d_iput(dentry, inode);
  328. else
  329. iput(inode);
  330. }
  331. /*
  332. * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry
  333. * is in use - which includes both the "real" per-superblock
  334. * LRU list _and_ the DCACHE_SHRINK_LIST use.
  335. *
  336. * The DCACHE_SHRINK_LIST bit is set whenever the dentry is
  337. * on the shrink list (ie not on the superblock LRU list).
  338. *
  339. * The per-cpu "nr_dentry_unused" counters are updated with
  340. * the DCACHE_LRU_LIST bit.
  341. *
  342. * The per-cpu "nr_dentry_negative" counters are only updated
  343. * when deleted from or added to the per-superblock LRU list, not
  344. * from/to the shrink list. That is to avoid an unneeded dec/inc
  345. * pair when moving from LRU to shrink list in select_collect().
  346. *
  347. * These helper functions make sure we always follow the
  348. * rules. d_lock must be held by the caller.
  349. */
  350. #define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
  351. static void d_lru_add(struct dentry *dentry)
  352. {
  353. D_FLAG_VERIFY(dentry, 0);
  354. dentry->d_flags |= DCACHE_LRU_LIST;
  355. this_cpu_inc(nr_dentry_unused);
  356. if (d_is_negative(dentry))
  357. this_cpu_inc(nr_dentry_negative);
  358. WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
  359. }
  360. static void d_lru_del(struct dentry *dentry)
  361. {
  362. D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
  363. dentry->d_flags &= ~DCACHE_LRU_LIST;
  364. this_cpu_dec(nr_dentry_unused);
  365. if (d_is_negative(dentry))
  366. this_cpu_dec(nr_dentry_negative);
  367. WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
  368. }
  369. static void d_shrink_del(struct dentry *dentry)
  370. {
  371. D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
  372. list_del_init(&dentry->d_lru);
  373. dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
  374. this_cpu_dec(nr_dentry_unused);
  375. }
  376. static void d_shrink_add(struct dentry *dentry, struct list_head *list)
  377. {
  378. D_FLAG_VERIFY(dentry, 0);
  379. list_add(&dentry->d_lru, list);
  380. dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
  381. this_cpu_inc(nr_dentry_unused);
  382. }
  383. /*
  384. * These can only be called under the global LRU lock, ie during the
  385. * callback for freeing the LRU list. "isolate" removes it from the
  386. * LRU lists entirely, while shrink_move moves it to the indicated
  387. * private list.
  388. */
  389. static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry)
  390. {
  391. D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
  392. dentry->d_flags &= ~DCACHE_LRU_LIST;
  393. this_cpu_dec(nr_dentry_unused);
  394. if (d_is_negative(dentry))
  395. this_cpu_dec(nr_dentry_negative);
  396. list_lru_isolate(lru, &dentry->d_lru);
  397. }
  398. static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry,
  399. struct list_head *list)
  400. {
  401. D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
  402. dentry->d_flags |= DCACHE_SHRINK_LIST;
  403. if (d_is_negative(dentry))
  404. this_cpu_dec(nr_dentry_negative);
  405. list_lru_isolate_move(lru, &dentry->d_lru, list);
  406. }
  407. /**
  408. * d_drop - drop a dentry
  409. * @dentry: dentry to drop
  410. *
  411. * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
  412. * be found through a VFS lookup any more. Note that this is different from
  413. * deleting the dentry - d_delete will try to mark the dentry negative if
  414. * possible, giving a successful _negative_ lookup, while d_drop will
  415. * just make the cache lookup fail.
  416. *
  417. * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
  418. * reason (NFS timeouts or autofs deletes).
  419. *
  420. * __d_drop requires dentry->d_lock
  421. * ___d_drop doesn't mark dentry as "unhashed"
  422. * (dentry->d_hash.pprev will be LIST_POISON2, not NULL).
  423. */
  424. static void ___d_drop(struct dentry *dentry)
  425. {
  426. struct hlist_bl_head *b;
  427. /*
  428. * Hashed dentries are normally on the dentry hashtable,
  429. * with the exception of those newly allocated by
  430. * d_obtain_root, which are always IS_ROOT:
  431. */
  432. if (unlikely(IS_ROOT(dentry)))
  433. b = &dentry->d_sb->s_roots;
  434. else
  435. b = d_hash(dentry->d_name.hash);
  436. hlist_bl_lock(b);
  437. __hlist_bl_del(&dentry->d_hash);
  438. hlist_bl_unlock(b);
  439. }
  440. void __d_drop(struct dentry *dentry)
  441. {
  442. if (!d_unhashed(dentry)) {
  443. ___d_drop(dentry);
  444. dentry->d_hash.pprev = NULL;
  445. write_seqcount_invalidate(&dentry->d_seq);
  446. }
  447. }
  448. EXPORT_SYMBOL(__d_drop);
  449. void d_drop(struct dentry *dentry)
  450. {
  451. spin_lock(&dentry->d_lock);
  452. __d_drop(dentry);
  453. spin_unlock(&dentry->d_lock);
  454. }
  455. EXPORT_SYMBOL(d_drop);
  456. static inline void dentry_unlist(struct dentry *dentry, struct dentry *parent)
  457. {
  458. struct dentry *next;
  459. /*
  460. * Inform d_walk() and shrink_dentry_list() that we are no longer
  461. * attached to the dentry tree
  462. */
  463. dentry->d_flags |= DCACHE_DENTRY_KILLED;
  464. if (unlikely(list_empty(&dentry->d_child)))
  465. return;
  466. __list_del_entry(&dentry->d_child);
  467. /*
  468. * Cursors can move around the list of children. While we'd been
  469. * a normal list member, it didn't matter - ->d_child.next would've
  470. * been updated. However, from now on it won't be and for the
  471. * things like d_walk() it might end up with a nasty surprise.
  472. * Normally d_walk() doesn't care about cursors moving around -
  473. * ->d_lock on parent prevents that and since a cursor has no children
  474. * of its own, we get through it without ever unlocking the parent.
  475. * There is one exception, though - if we ascend from a child that
  476. * gets killed as soon as we unlock it, the next sibling is found
  477. * using the value left in its ->d_child.next. And if _that_
  478. * pointed to a cursor, and cursor got moved (e.g. by lseek())
  479. * before d_walk() regains parent->d_lock, we'll end up skipping
  480. * everything the cursor had been moved past.
  481. *
  482. * Solution: make sure that the pointer left behind in ->d_child.next
  483. * points to something that won't be moving around. I.e. skip the
  484. * cursors.
  485. */
  486. while (dentry->d_child.next != &parent->d_subdirs) {
  487. next = list_entry(dentry->d_child.next, struct dentry, d_child);
  488. if (likely(!(next->d_flags & DCACHE_DENTRY_CURSOR)))
  489. break;
  490. dentry->d_child.next = next->d_child.next;
  491. }
  492. }
  493. static void __dentry_kill(struct dentry *dentry)
  494. {
  495. struct dentry *parent = NULL;
  496. bool can_free = true;
  497. if (!IS_ROOT(dentry))
  498. parent = dentry->d_parent;
  499. /*
  500. * The dentry is now unrecoverably dead to the world.
  501. */
  502. lockref_mark_dead(&dentry->d_lockref);
  503. /*
  504. * inform the fs via d_prune that this dentry is about to be
  505. * unhashed and destroyed.
  506. */
  507. if (dentry->d_flags & DCACHE_OP_PRUNE)
  508. dentry->d_op->d_prune(dentry);
  509. if (dentry->d_flags & DCACHE_LRU_LIST) {
  510. if (!(dentry->d_flags & DCACHE_SHRINK_LIST))
  511. d_lru_del(dentry);
  512. }
  513. /* if it was on the hash then remove it */
  514. __d_drop(dentry);
  515. dentry_unlist(dentry, parent);
  516. if (parent)
  517. spin_unlock(&parent->d_lock);
  518. if (dentry->d_inode)
  519. dentry_unlink_inode(dentry);
  520. else
  521. spin_unlock(&dentry->d_lock);
  522. this_cpu_dec(nr_dentry);
  523. if (dentry->d_op && dentry->d_op->d_release)
  524. dentry->d_op->d_release(dentry);
  525. spin_lock(&dentry->d_lock);
  526. if (dentry->d_flags & DCACHE_SHRINK_LIST) {
  527. dentry->d_flags |= DCACHE_MAY_FREE;
  528. can_free = false;
  529. }
  530. spin_unlock(&dentry->d_lock);
  531. if (likely(can_free))
  532. dentry_free(dentry);
  533. cond_resched();
  534. }
  535. static struct dentry *__lock_parent(struct dentry *dentry)
  536. {
  537. struct dentry *parent;
  538. rcu_read_lock();
  539. spin_unlock(&dentry->d_lock);
  540. again:
  541. parent = READ_ONCE(dentry->d_parent);
  542. spin_lock(&parent->d_lock);
  543. /*
  544. * We can't blindly lock dentry until we are sure
  545. * that we won't violate the locking order.
  546. * Any changes of dentry->d_parent must have
  547. * been done with parent->d_lock held, so
  548. * spin_lock() above is enough of a barrier
  549. * for checking if it's still our child.
  550. */
  551. if (unlikely(parent != dentry->d_parent)) {
  552. spin_unlock(&parent->d_lock);
  553. goto again;
  554. }
  555. rcu_read_unlock();
  556. if (parent != dentry)
  557. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  558. else
  559. parent = NULL;
  560. return parent;
  561. }
  562. static inline struct dentry *lock_parent(struct dentry *dentry)
  563. {
  564. struct dentry *parent = dentry->d_parent;
  565. if (IS_ROOT(dentry))
  566. return NULL;
  567. if (likely(spin_trylock(&parent->d_lock)))
  568. return parent;
  569. return __lock_parent(dentry);
  570. }
  571. static inline bool retain_dentry(struct dentry *dentry)
  572. {
  573. WARN_ON(d_in_lookup(dentry));
  574. /* Unreachable? Get rid of it */
  575. if (unlikely(d_unhashed(dentry)))
  576. return false;
  577. if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED))
  578. return false;
  579. if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) {
  580. if (dentry->d_op->d_delete(dentry))
  581. return false;
  582. }
  583. if (unlikely(dentry->d_flags & DCACHE_DONTCACHE))
  584. return false;
  585. /* retain; LRU fodder */
  586. dentry->d_lockref.count--;
  587. if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
  588. d_lru_add(dentry);
  589. else if (unlikely(!(dentry->d_flags & DCACHE_REFERENCED)))
  590. dentry->d_flags |= DCACHE_REFERENCED;
  591. return true;
  592. }
  593. void d_mark_dontcache(struct inode *inode)
  594. {
  595. struct dentry *de;
  596. spin_lock(&inode->i_lock);
  597. hlist_for_each_entry(de, &inode->i_dentry, d_u.d_alias) {
  598. spin_lock(&de->d_lock);
  599. de->d_flags |= DCACHE_DONTCACHE;
  600. spin_unlock(&de->d_lock);
  601. }
  602. inode->i_state |= I_DONTCACHE;
  603. spin_unlock(&inode->i_lock);
  604. }
  605. EXPORT_SYMBOL(d_mark_dontcache);
  606. /*
  607. * Finish off a dentry we've decided to kill.
  608. * dentry->d_lock must be held, returns with it unlocked.
  609. * Returns dentry requiring refcount drop, or NULL if we're done.
  610. */
  611. static struct dentry *dentry_kill(struct dentry *dentry)
  612. __releases(dentry->d_lock)
  613. {
  614. struct inode *inode = dentry->d_inode;
  615. struct dentry *parent = NULL;
  616. if (inode && unlikely(!spin_trylock(&inode->i_lock)))
  617. goto slow_positive;
  618. if (!IS_ROOT(dentry)) {
  619. parent = dentry->d_parent;
  620. if (unlikely(!spin_trylock(&parent->d_lock))) {
  621. parent = __lock_parent(dentry);
  622. if (likely(inode || !dentry->d_inode))
  623. goto got_locks;
  624. /* negative that became positive */
  625. if (parent)
  626. spin_unlock(&parent->d_lock);
  627. inode = dentry->d_inode;
  628. goto slow_positive;
  629. }
  630. }
  631. __dentry_kill(dentry);
  632. return parent;
  633. slow_positive:
  634. spin_unlock(&dentry->d_lock);
  635. spin_lock(&inode->i_lock);
  636. spin_lock(&dentry->d_lock);
  637. parent = lock_parent(dentry);
  638. got_locks:
  639. if (unlikely(dentry->d_lockref.count != 1)) {
  640. dentry->d_lockref.count--;
  641. } else if (likely(!retain_dentry(dentry))) {
  642. __dentry_kill(dentry);
  643. return parent;
  644. }
  645. /* we are keeping it, after all */
  646. if (inode)
  647. spin_unlock(&inode->i_lock);
  648. if (parent)
  649. spin_unlock(&parent->d_lock);
  650. spin_unlock(&dentry->d_lock);
  651. return NULL;
  652. }
  653. /*
  654. * Try to do a lockless dput(), and return whether that was successful.
  655. *
  656. * If unsuccessful, we return false, having already taken the dentry lock.
  657. *
  658. * The caller needs to hold the RCU read lock, so that the dentry is
  659. * guaranteed to stay around even if the refcount goes down to zero!
  660. */
  661. static inline bool fast_dput(struct dentry *dentry)
  662. {
  663. int ret;
  664. unsigned int d_flags;
  665. /*
  666. * If we have a d_op->d_delete() operation, we sould not
  667. * let the dentry count go to zero, so use "put_or_lock".
  668. */
  669. if (unlikely(dentry->d_flags & DCACHE_OP_DELETE))
  670. return lockref_put_or_lock(&dentry->d_lockref);
  671. /*
  672. * .. otherwise, we can try to just decrement the
  673. * lockref optimistically.
  674. */
  675. ret = lockref_put_return(&dentry->d_lockref);
  676. /*
  677. * If the lockref_put_return() failed due to the lock being held
  678. * by somebody else, the fast path has failed. We will need to
  679. * get the lock, and then check the count again.
  680. */
  681. if (unlikely(ret < 0)) {
  682. spin_lock(&dentry->d_lock);
  683. if (dentry->d_lockref.count > 1) {
  684. dentry->d_lockref.count--;
  685. spin_unlock(&dentry->d_lock);
  686. return true;
  687. }
  688. return false;
  689. }
  690. /*
  691. * If we weren't the last ref, we're done.
  692. */
  693. if (ret)
  694. return true;
  695. /*
  696. * Careful, careful. The reference count went down
  697. * to zero, but we don't hold the dentry lock, so
  698. * somebody else could get it again, and do another
  699. * dput(), and we need to not race with that.
  700. *
  701. * However, there is a very special and common case
  702. * where we don't care, because there is nothing to
  703. * do: the dentry is still hashed, it does not have
  704. * a 'delete' op, and it's referenced and already on
  705. * the LRU list.
  706. *
  707. * NOTE! Since we aren't locked, these values are
  708. * not "stable". However, it is sufficient that at
  709. * some point after we dropped the reference the
  710. * dentry was hashed and the flags had the proper
  711. * value. Other dentry users may have re-gotten
  712. * a reference to the dentry and change that, but
  713. * our work is done - we can leave the dentry
  714. * around with a zero refcount.
  715. */
  716. smp_rmb();
  717. d_flags = READ_ONCE(dentry->d_flags);
  718. d_flags &= DCACHE_REFERENCED | DCACHE_LRU_LIST | DCACHE_DISCONNECTED;
  719. /* Nothing to do? Dropping the reference was all we needed? */
  720. if (d_flags == (DCACHE_REFERENCED | DCACHE_LRU_LIST) && !d_unhashed(dentry))
  721. return true;
  722. /*
  723. * Not the fast normal case? Get the lock. We've already decremented
  724. * the refcount, but we'll need to re-check the situation after
  725. * getting the lock.
  726. */
  727. spin_lock(&dentry->d_lock);
  728. /*
  729. * Did somebody else grab a reference to it in the meantime, and
  730. * we're no longer the last user after all? Alternatively, somebody
  731. * else could have killed it and marked it dead. Either way, we
  732. * don't need to do anything else.
  733. */
  734. if (dentry->d_lockref.count) {
  735. spin_unlock(&dentry->d_lock);
  736. return true;
  737. }
  738. /*
  739. * Re-get the reference we optimistically dropped. We hold the
  740. * lock, and we just tested that it was zero, so we can just
  741. * set it to 1.
  742. */
  743. dentry->d_lockref.count = 1;
  744. return false;
  745. }
  746. /*
  747. * This is dput
  748. *
  749. * This is complicated by the fact that we do not want to put
  750. * dentries that are no longer on any hash chain on the unused
  751. * list: we'd much rather just get rid of them immediately.
  752. *
  753. * However, that implies that we have to traverse the dentry
  754. * tree upwards to the parents which might _also_ now be
  755. * scheduled for deletion (it may have been only waiting for
  756. * its last child to go away).
  757. *
  758. * This tail recursion is done by hand as we don't want to depend
  759. * on the compiler to always get this right (gcc generally doesn't).
  760. * Real recursion would eat up our stack space.
  761. */
  762. /*
  763. * dput - release a dentry
  764. * @dentry: dentry to release
  765. *
  766. * Release a dentry. This will drop the usage count and if appropriate
  767. * call the dentry unlink method as well as removing it from the queues and
  768. * releasing its resources. If the parent dentries were scheduled for release
  769. * they too may now get deleted.
  770. */
  771. void dput(struct dentry *dentry)
  772. {
  773. while (dentry) {
  774. might_sleep();
  775. rcu_read_lock();
  776. if (likely(fast_dput(dentry))) {
  777. rcu_read_unlock();
  778. return;
  779. }
  780. /* Slow case: now with the dentry lock held */
  781. rcu_read_unlock();
  782. if (likely(retain_dentry(dentry))) {
  783. spin_unlock(&dentry->d_lock);
  784. return;
  785. }
  786. dentry = dentry_kill(dentry);
  787. }
  788. }
  789. EXPORT_SYMBOL(dput);
  790. static void __dput_to_list(struct dentry *dentry, struct list_head *list)
  791. __must_hold(&dentry->d_lock)
  792. {
  793. if (dentry->d_flags & DCACHE_SHRINK_LIST) {
  794. /* let the owner of the list it's on deal with it */
  795. --dentry->d_lockref.count;
  796. } else {
  797. if (dentry->d_flags & DCACHE_LRU_LIST)
  798. d_lru_del(dentry);
  799. if (!--dentry->d_lockref.count)
  800. d_shrink_add(dentry, list);
  801. }
  802. }
  803. void dput_to_list(struct dentry *dentry, struct list_head *list)
  804. {
  805. rcu_read_lock();
  806. if (likely(fast_dput(dentry))) {
  807. rcu_read_unlock();
  808. return;
  809. }
  810. rcu_read_unlock();
  811. if (!retain_dentry(dentry))
  812. __dput_to_list(dentry, list);
  813. spin_unlock(&dentry->d_lock);
  814. }
  815. /* This must be called with d_lock held */
  816. static inline void __dget_dlock(struct dentry *dentry)
  817. {
  818. dentry->d_lockref.count++;
  819. }
  820. static inline void __dget(struct dentry *dentry)
  821. {
  822. lockref_get(&dentry->d_lockref);
  823. }
  824. struct dentry *dget_parent(struct dentry *dentry)
  825. {
  826. int gotref;
  827. struct dentry *ret;
  828. unsigned seq;
  829. /*
  830. * Do optimistic parent lookup without any
  831. * locking.
  832. */
  833. rcu_read_lock();
  834. seq = raw_seqcount_begin(&dentry->d_seq);
  835. ret = READ_ONCE(dentry->d_parent);
  836. gotref = lockref_get_not_zero(&ret->d_lockref);
  837. rcu_read_unlock();
  838. if (likely(gotref)) {
  839. if (!read_seqcount_retry(&dentry->d_seq, seq))
  840. return ret;
  841. dput(ret);
  842. }
  843. repeat:
  844. /*
  845. * Don't need rcu_dereference because we re-check it was correct under
  846. * the lock.
  847. */
  848. rcu_read_lock();
  849. ret = dentry->d_parent;
  850. spin_lock(&ret->d_lock);
  851. if (unlikely(ret != dentry->d_parent)) {
  852. spin_unlock(&ret->d_lock);
  853. rcu_read_unlock();
  854. goto repeat;
  855. }
  856. rcu_read_unlock();
  857. BUG_ON(!ret->d_lockref.count);
  858. ret->d_lockref.count++;
  859. spin_unlock(&ret->d_lock);
  860. return ret;
  861. }
  862. EXPORT_SYMBOL(dget_parent);
  863. static struct dentry * __d_find_any_alias(struct inode *inode)
  864. {
  865. struct dentry *alias;
  866. if (hlist_empty(&inode->i_dentry))
  867. return NULL;
  868. alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
  869. __dget(alias);
  870. return alias;
  871. }
  872. /**
  873. * d_find_any_alias - find any alias for a given inode
  874. * @inode: inode to find an alias for
  875. *
  876. * If any aliases exist for the given inode, take and return a
  877. * reference for one of them. If no aliases exist, return %NULL.
  878. */
  879. struct dentry *d_find_any_alias(struct inode *inode)
  880. {
  881. struct dentry *de;
  882. spin_lock(&inode->i_lock);
  883. de = __d_find_any_alias(inode);
  884. spin_unlock(&inode->i_lock);
  885. return de;
  886. }
  887. EXPORT_SYMBOL(d_find_any_alias);
  888. /**
  889. * d_find_alias - grab a hashed alias of inode
  890. * @inode: inode in question
  891. *
  892. * If inode has a hashed alias, or is a directory and has any alias,
  893. * acquire the reference to alias and return it. Otherwise return NULL.
  894. * Notice that if inode is a directory there can be only one alias and
  895. * it can be unhashed only if it has no children, or if it is the root
  896. * of a filesystem, or if the directory was renamed and d_revalidate
  897. * was the first vfs operation to notice.
  898. *
  899. * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
  900. * any other hashed alias over that one.
  901. */
  902. static struct dentry *__d_find_alias(struct inode *inode)
  903. {
  904. struct dentry *alias;
  905. if (S_ISDIR(inode->i_mode))
  906. return __d_find_any_alias(inode);
  907. hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
  908. spin_lock(&alias->d_lock);
  909. if (!d_unhashed(alias)) {
  910. __dget_dlock(alias);
  911. spin_unlock(&alias->d_lock);
  912. return alias;
  913. }
  914. spin_unlock(&alias->d_lock);
  915. }
  916. return NULL;
  917. }
  918. struct dentry *d_find_alias(struct inode *inode)
  919. {
  920. struct dentry *de = NULL;
  921. if (!hlist_empty(&inode->i_dentry)) {
  922. spin_lock(&inode->i_lock);
  923. de = __d_find_alias(inode);
  924. spin_unlock(&inode->i_lock);
  925. }
  926. return de;
  927. }
  928. EXPORT_SYMBOL(d_find_alias);
  929. /*
  930. * Try to kill dentries associated with this inode.
  931. * WARNING: you must own a reference to inode.
  932. */
  933. void d_prune_aliases(struct inode *inode)
  934. {
  935. struct dentry *dentry;
  936. restart:
  937. spin_lock(&inode->i_lock);
  938. hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) {
  939. spin_lock(&dentry->d_lock);
  940. if (!dentry->d_lockref.count) {
  941. struct dentry *parent = lock_parent(dentry);
  942. if (likely(!dentry->d_lockref.count)) {
  943. __dentry_kill(dentry);
  944. dput(parent);
  945. goto restart;
  946. }
  947. if (parent)
  948. spin_unlock(&parent->d_lock);
  949. }
  950. spin_unlock(&dentry->d_lock);
  951. }
  952. spin_unlock(&inode->i_lock);
  953. }
  954. EXPORT_SYMBOL(d_prune_aliases);
  955. /*
  956. * Lock a dentry from shrink list.
  957. * Called under rcu_read_lock() and dentry->d_lock; the former
  958. * guarantees that nothing we access will be freed under us.
  959. * Note that dentry is *not* protected from concurrent dentry_kill(),
  960. * d_delete(), etc.
  961. *
  962. * Return false if dentry has been disrupted or grabbed, leaving
  963. * the caller to kick it off-list. Otherwise, return true and have
  964. * that dentry's inode and parent both locked.
  965. */
  966. static bool shrink_lock_dentry(struct dentry *dentry)
  967. {
  968. struct inode *inode;
  969. struct dentry *parent;
  970. if (dentry->d_lockref.count)
  971. return false;
  972. inode = dentry->d_inode;
  973. if (inode && unlikely(!spin_trylock(&inode->i_lock))) {
  974. spin_unlock(&dentry->d_lock);
  975. spin_lock(&inode->i_lock);
  976. spin_lock(&dentry->d_lock);
  977. if (unlikely(dentry->d_lockref.count))
  978. goto out;
  979. /* changed inode means that somebody had grabbed it */
  980. if (unlikely(inode != dentry->d_inode))
  981. goto out;
  982. }
  983. parent = dentry->d_parent;
  984. if (IS_ROOT(dentry) || likely(spin_trylock(&parent->d_lock)))
  985. return true;
  986. spin_unlock(&dentry->d_lock);
  987. spin_lock(&parent->d_lock);
  988. if (unlikely(parent != dentry->d_parent)) {
  989. spin_unlock(&parent->d_lock);
  990. spin_lock(&dentry->d_lock);
  991. goto out;
  992. }
  993. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  994. if (likely(!dentry->d_lockref.count))
  995. return true;
  996. spin_unlock(&parent->d_lock);
  997. out:
  998. if (inode)
  999. spin_unlock(&inode->i_lock);
  1000. return false;
  1001. }
  1002. void shrink_dentry_list(struct list_head *list)
  1003. {
  1004. while (!list_empty(list)) {
  1005. struct dentry *dentry, *parent;
  1006. dentry = list_entry(list->prev, struct dentry, d_lru);
  1007. spin_lock(&dentry->d_lock);
  1008. rcu_read_lock();
  1009. if (!shrink_lock_dentry(dentry)) {
  1010. bool can_free = false;
  1011. rcu_read_unlock();
  1012. d_shrink_del(dentry);
  1013. if (dentry->d_lockref.count < 0)
  1014. can_free = dentry->d_flags & DCACHE_MAY_FREE;
  1015. spin_unlock(&dentry->d_lock);
  1016. if (can_free)
  1017. dentry_free(dentry);
  1018. continue;
  1019. }
  1020. rcu_read_unlock();
  1021. d_shrink_del(dentry);
  1022. parent = dentry->d_parent;
  1023. if (parent != dentry)
  1024. __dput_to_list(parent, list);
  1025. __dentry_kill(dentry);
  1026. }
  1027. }
  1028. static enum lru_status dentry_lru_isolate(struct list_head *item,
  1029. struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
  1030. {
  1031. struct list_head *freeable = arg;
  1032. struct dentry *dentry = container_of(item, struct dentry, d_lru);
  1033. /*
  1034. * we are inverting the lru lock/dentry->d_lock here,
  1035. * so use a trylock. If we fail to get the lock, just skip
  1036. * it
  1037. */
  1038. if (!spin_trylock(&dentry->d_lock))
  1039. return LRU_SKIP;
  1040. /*
  1041. * Referenced dentries are still in use. If they have active
  1042. * counts, just remove them from the LRU. Otherwise give them
  1043. * another pass through the LRU.
  1044. */
  1045. if (dentry->d_lockref.count) {
  1046. d_lru_isolate(lru, dentry);
  1047. spin_unlock(&dentry->d_lock);
  1048. return LRU_REMOVED;
  1049. }
  1050. if (dentry->d_flags & DCACHE_REFERENCED) {
  1051. dentry->d_flags &= ~DCACHE_REFERENCED;
  1052. spin_unlock(&dentry->d_lock);
  1053. /*
  1054. * The list move itself will be made by the common LRU code. At
  1055. * this point, we've dropped the dentry->d_lock but keep the
  1056. * lru lock. This is safe to do, since every list movement is
  1057. * protected by the lru lock even if both locks are held.
  1058. *
  1059. * This is guaranteed by the fact that all LRU management
  1060. * functions are intermediated by the LRU API calls like
  1061. * list_lru_add and list_lru_del. List movement in this file
  1062. * only ever occur through this functions or through callbacks
  1063. * like this one, that are called from the LRU API.
  1064. *
  1065. * The only exceptions to this are functions like
  1066. * shrink_dentry_list, and code that first checks for the
  1067. * DCACHE_SHRINK_LIST flag. Those are guaranteed to be
  1068. * operating only with stack provided lists after they are
  1069. * properly isolated from the main list. It is thus, always a
  1070. * local access.
  1071. */
  1072. return LRU_ROTATE;
  1073. }
  1074. d_lru_shrink_move(lru, dentry, freeable);
  1075. spin_unlock(&dentry->d_lock);
  1076. return LRU_REMOVED;
  1077. }
  1078. /**
  1079. * prune_dcache_sb - shrink the dcache
  1080. * @sb: superblock
  1081. * @sc: shrink control, passed to list_lru_shrink_walk()
  1082. *
  1083. * Attempt to shrink the superblock dcache LRU by @sc->nr_to_scan entries. This
  1084. * is done when we need more memory and called from the superblock shrinker
  1085. * function.
  1086. *
  1087. * This function may fail to free any resources if all the dentries are in
  1088. * use.
  1089. */
  1090. long prune_dcache_sb(struct super_block *sb, struct shrink_control *sc)
  1091. {
  1092. LIST_HEAD(dispose);
  1093. long freed;
  1094. freed = list_lru_shrink_walk(&sb->s_dentry_lru, sc,
  1095. dentry_lru_isolate, &dispose);
  1096. shrink_dentry_list(&dispose);
  1097. return freed;
  1098. }
  1099. static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
  1100. struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
  1101. {
  1102. struct list_head *freeable = arg;
  1103. struct dentry *dentry = container_of(item, struct dentry, d_lru);
  1104. /*
  1105. * we are inverting the lru lock/dentry->d_lock here,
  1106. * so use a trylock. If we fail to get the lock, just skip
  1107. * it
  1108. */
  1109. if (!spin_trylock(&dentry->d_lock))
  1110. return LRU_SKIP;
  1111. d_lru_shrink_move(lru, dentry, freeable);
  1112. spin_unlock(&dentry->d_lock);
  1113. return LRU_REMOVED;
  1114. }
  1115. /**
  1116. * shrink_dcache_sb - shrink dcache for a superblock
  1117. * @sb: superblock
  1118. *
  1119. * Shrink the dcache for the specified super block. This is used to free
  1120. * the dcache before unmounting a file system.
  1121. */
  1122. void shrink_dcache_sb(struct super_block *sb)
  1123. {
  1124. do {
  1125. LIST_HEAD(dispose);
  1126. list_lru_walk(&sb->s_dentry_lru,
  1127. dentry_lru_isolate_shrink, &dispose, 1024);
  1128. shrink_dentry_list(&dispose);
  1129. } while (list_lru_count(&sb->s_dentry_lru) > 0);
  1130. }
  1131. EXPORT_SYMBOL(shrink_dcache_sb);
  1132. /**
  1133. * enum d_walk_ret - action to talke during tree walk
  1134. * @D_WALK_CONTINUE: contrinue walk
  1135. * @D_WALK_QUIT: quit walk
  1136. * @D_WALK_NORETRY: quit when retry is needed
  1137. * @D_WALK_SKIP: skip this dentry and its children
  1138. */
  1139. enum d_walk_ret {
  1140. D_WALK_CONTINUE,
  1141. D_WALK_QUIT,
  1142. D_WALK_NORETRY,
  1143. D_WALK_SKIP,
  1144. };
  1145. /**
  1146. * d_walk - walk the dentry tree
  1147. * @parent: start of walk
  1148. * @data: data passed to @enter() and @finish()
  1149. * @enter: callback when first entering the dentry
  1150. *
  1151. * The @enter() callbacks are called with d_lock held.
  1152. */
  1153. static void d_walk(struct dentry *parent, void *data,
  1154. enum d_walk_ret (*enter)(void *, struct dentry *))
  1155. {
  1156. struct dentry *this_parent;
  1157. struct list_head *next;
  1158. unsigned seq = 0;
  1159. enum d_walk_ret ret;
  1160. bool retry = true;
  1161. again:
  1162. read_seqbegin_or_lock(&rename_lock, &seq);
  1163. this_parent = parent;
  1164. spin_lock(&this_parent->d_lock);
  1165. ret = enter(data, this_parent);
  1166. switch (ret) {
  1167. case D_WALK_CONTINUE:
  1168. break;
  1169. case D_WALK_QUIT:
  1170. case D_WALK_SKIP:
  1171. goto out_unlock;
  1172. case D_WALK_NORETRY:
  1173. retry = false;
  1174. break;
  1175. }
  1176. repeat:
  1177. next = this_parent->d_subdirs.next;
  1178. resume:
  1179. while (next != &this_parent->d_subdirs) {
  1180. struct list_head *tmp = next;
  1181. struct dentry *dentry = list_entry(tmp, struct dentry, d_child);
  1182. next = tmp->next;
  1183. if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR))
  1184. continue;
  1185. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  1186. ret = enter(data, dentry);
  1187. switch (ret) {
  1188. case D_WALK_CONTINUE:
  1189. break;
  1190. case D_WALK_QUIT:
  1191. spin_unlock(&dentry->d_lock);
  1192. goto out_unlock;
  1193. case D_WALK_NORETRY:
  1194. retry = false;
  1195. break;
  1196. case D_WALK_SKIP:
  1197. spin_unlock(&dentry->d_lock);
  1198. continue;
  1199. }
  1200. if (!list_empty(&dentry->d_subdirs)) {
  1201. spin_unlock(&this_parent->d_lock);
  1202. spin_release(&dentry->d_lock.dep_map, _RET_IP_);
  1203. this_parent = dentry;
  1204. spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
  1205. goto repeat;
  1206. }
  1207. spin_unlock(&dentry->d_lock);
  1208. }
  1209. /*
  1210. * All done at this level ... ascend and resume the search.
  1211. */
  1212. rcu_read_lock();
  1213. ascend:
  1214. if (this_parent != parent) {
  1215. struct dentry *child = this_parent;
  1216. this_parent = child->d_parent;
  1217. spin_unlock(&child->d_lock);
  1218. spin_lock(&this_parent->d_lock);
  1219. /* might go back up the wrong parent if we have had a rename. */
  1220. if (need_seqretry(&rename_lock, seq))
  1221. goto rename_retry;
  1222. /* go into the first sibling still alive */
  1223. do {
  1224. next = child->d_child.next;
  1225. if (next == &this_parent->d_subdirs)
  1226. goto ascend;
  1227. child = list_entry(next, struct dentry, d_child);
  1228. } while (unlikely(child->d_flags & DCACHE_DENTRY_KILLED));
  1229. rcu_read_unlock();
  1230. goto resume;
  1231. }
  1232. if (need_seqretry(&rename_lock, seq))
  1233. goto rename_retry;
  1234. rcu_read_unlock();
  1235. out_unlock:
  1236. spin_unlock(&this_parent->d_lock);
  1237. done_seqretry(&rename_lock, seq);
  1238. return;
  1239. rename_retry:
  1240. spin_unlock(&this_parent->d_lock);
  1241. rcu_read_unlock();
  1242. BUG_ON(seq & 1);
  1243. if (!retry)
  1244. return;
  1245. seq = 1;
  1246. goto again;
  1247. }
  1248. struct check_mount {
  1249. struct vfsmount *mnt;
  1250. unsigned int mounted;
  1251. };
  1252. static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry)
  1253. {
  1254. struct check_mount *info = data;
  1255. struct path path = { .mnt = info->mnt, .dentry = dentry };
  1256. if (likely(!d_mountpoint(dentry)))
  1257. return D_WALK_CONTINUE;
  1258. if (__path_is_mountpoint(&path)) {
  1259. info->mounted = 1;
  1260. return D_WALK_QUIT;
  1261. }
  1262. return D_WALK_CONTINUE;
  1263. }
  1264. /**
  1265. * path_has_submounts - check for mounts over a dentry in the
  1266. * current namespace.
  1267. * @parent: path to check.
  1268. *
  1269. * Return true if the parent or its subdirectories contain
  1270. * a mount point in the current namespace.
  1271. */
  1272. int path_has_submounts(const struct path *parent)
  1273. {
  1274. struct check_mount data = { .mnt = parent->mnt, .mounted = 0 };
  1275. read_seqlock_excl(&mount_lock);
  1276. d_walk(parent->dentry, &data, path_check_mount);
  1277. read_sequnlock_excl(&mount_lock);
  1278. return data.mounted;
  1279. }
  1280. EXPORT_SYMBOL(path_has_submounts);
  1281. /*
  1282. * Called by mount code to set a mountpoint and check if the mountpoint is
  1283. * reachable (e.g. NFS can unhash a directory dentry and then the complete
  1284. * subtree can become unreachable).
  1285. *
  1286. * Only one of d_invalidate() and d_set_mounted() must succeed. For
  1287. * this reason take rename_lock and d_lock on dentry and ancestors.
  1288. */
  1289. int d_set_mounted(struct dentry *dentry)
  1290. {
  1291. struct dentry *p;
  1292. int ret = -ENOENT;
  1293. write_seqlock(&rename_lock);
  1294. for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
  1295. /* Need exclusion wrt. d_invalidate() */
  1296. spin_lock(&p->d_lock);
  1297. if (unlikely(d_unhashed(p))) {
  1298. spin_unlock(&p->d_lock);
  1299. goto out;
  1300. }
  1301. spin_unlock(&p->d_lock);
  1302. }
  1303. spin_lock(&dentry->d_lock);
  1304. if (!d_unlinked(dentry)) {
  1305. ret = -EBUSY;
  1306. if (!d_mountpoint(dentry)) {
  1307. dentry->d_flags |= DCACHE_MOUNTED;
  1308. ret = 0;
  1309. }
  1310. }
  1311. spin_unlock(&dentry->d_lock);
  1312. out:
  1313. write_sequnlock(&rename_lock);
  1314. return ret;
  1315. }
  1316. /*
  1317. * Search the dentry child list of the specified parent,
  1318. * and move any unused dentries to the end of the unused
  1319. * list for prune_dcache(). We descend to the next level
  1320. * whenever the d_subdirs list is non-empty and continue
  1321. * searching.
  1322. *
  1323. * It returns zero iff there are no unused children,
  1324. * otherwise it returns the number of children moved to
  1325. * the end of the unused list. This may not be the total
  1326. * number of unused children, because select_parent can
  1327. * drop the lock and return early due to latency
  1328. * constraints.
  1329. */
  1330. struct select_data {
  1331. struct dentry *start;
  1332. union {
  1333. long found;
  1334. struct dentry *victim;
  1335. };
  1336. struct list_head dispose;
  1337. };
  1338. static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
  1339. {
  1340. struct select_data *data = _data;
  1341. enum d_walk_ret ret = D_WALK_CONTINUE;
  1342. if (data->start == dentry)
  1343. goto out;
  1344. if (dentry->d_flags & DCACHE_SHRINK_LIST) {
  1345. data->found++;
  1346. } else {
  1347. if (dentry->d_flags & DCACHE_LRU_LIST)
  1348. d_lru_del(dentry);
  1349. if (!dentry->d_lockref.count) {
  1350. d_shrink_add(dentry, &data->dispose);
  1351. data->found++;
  1352. }
  1353. }
  1354. /*
  1355. * We can return to the caller if we have found some (this
  1356. * ensures forward progress). We'll be coming back to find
  1357. * the rest.
  1358. */
  1359. if (!list_empty(&data->dispose))
  1360. ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
  1361. out:
  1362. return ret;
  1363. }
  1364. static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry)
  1365. {
  1366. struct select_data *data = _data;
  1367. enum d_walk_ret ret = D_WALK_CONTINUE;
  1368. if (data->start == dentry)
  1369. goto out;
  1370. if (dentry->d_flags & DCACHE_SHRINK_LIST) {
  1371. if (!dentry->d_lockref.count) {
  1372. rcu_read_lock();
  1373. data->victim = dentry;
  1374. return D_WALK_QUIT;
  1375. }
  1376. } else {
  1377. if (dentry->d_flags & DCACHE_LRU_LIST)
  1378. d_lru_del(dentry);
  1379. if (!dentry->d_lockref.count)
  1380. d_shrink_add(dentry, &data->dispose);
  1381. }
  1382. /*
  1383. * We can return to the caller if we have found some (this
  1384. * ensures forward progress). We'll be coming back to find
  1385. * the rest.
  1386. */
  1387. if (!list_empty(&data->dispose))
  1388. ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
  1389. out:
  1390. return ret;
  1391. }
  1392. /**
  1393. * shrink_dcache_parent - prune dcache
  1394. * @parent: parent of entries to prune
  1395. *
  1396. * Prune the dcache to remove unused children of the parent dentry.
  1397. */
  1398. void shrink_dcache_parent(struct dentry *parent)
  1399. {
  1400. for (;;) {
  1401. struct select_data data = {.start = parent};
  1402. INIT_LIST_HEAD(&data.dispose);
  1403. d_walk(parent, &data, select_collect);
  1404. if (!list_empty(&data.dispose)) {
  1405. shrink_dentry_list(&data.dispose);
  1406. continue;
  1407. }
  1408. cond_resched();
  1409. if (!data.found)
  1410. break;
  1411. data.victim = NULL;
  1412. d_walk(parent, &data, select_collect2);
  1413. if (data.victim) {
  1414. struct dentry *parent;
  1415. spin_lock(&data.victim->d_lock);
  1416. if (!shrink_lock_dentry(data.victim)) {
  1417. spin_unlock(&data.victim->d_lock);
  1418. rcu_read_unlock();
  1419. } else {
  1420. rcu_read_unlock();
  1421. parent = data.victim->d_parent;
  1422. if (parent != data.victim)
  1423. __dput_to_list(parent, &data.dispose);
  1424. __dentry_kill(data.victim);
  1425. }
  1426. }
  1427. if (!list_empty(&data.dispose))
  1428. shrink_dentry_list(&data.dispose);
  1429. }
  1430. }
  1431. EXPORT_SYMBOL(shrink_dcache_parent);
  1432. static enum d_walk_ret umount_check(void *_data, struct dentry *dentry)
  1433. {
  1434. /* it has busy descendents; complain about those instead */
  1435. if (!list_empty(&dentry->d_subdirs))
  1436. return D_WALK_CONTINUE;
  1437. /* root with refcount 1 is fine */
  1438. if (dentry == _data && dentry->d_lockref.count == 1)
  1439. return D_WALK_CONTINUE;
  1440. printk(KERN_ERR "BUG: Dentry %p{i=%lx,n=%pd} "
  1441. " still in use (%d) [unmount of %s %s]\n",
  1442. dentry,
  1443. dentry->d_inode ?
  1444. dentry->d_inode->i_ino : 0UL,
  1445. dentry,
  1446. dentry->d_lockref.count,
  1447. dentry->d_sb->s_type->name,
  1448. dentry->d_sb->s_id);
  1449. WARN_ON(1);
  1450. return D_WALK_CONTINUE;
  1451. }
  1452. static void do_one_tree(struct dentry *dentry)
  1453. {
  1454. shrink_dcache_parent(dentry);
  1455. d_walk(dentry, dentry, umount_check);
  1456. d_drop(dentry);
  1457. dput(dentry);
  1458. }
  1459. /*
  1460. * destroy the dentries attached to a superblock on unmounting
  1461. */
  1462. void shrink_dcache_for_umount(struct super_block *sb)
  1463. {
  1464. struct dentry *dentry;
  1465. WARN(down_read_trylock(&sb->s_umount), "s_umount should've been locked");
  1466. dentry = sb->s_root;
  1467. sb->s_root = NULL;
  1468. do_one_tree(dentry);
  1469. while (!hlist_bl_empty(&sb->s_roots)) {
  1470. dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash));
  1471. do_one_tree(dentry);
  1472. }
  1473. }
  1474. static enum d_walk_ret find_submount(void *_data, struct dentry *dentry)
  1475. {
  1476. struct dentry **victim = _data;
  1477. if (d_mountpoint(dentry)) {
  1478. __dget_dlock(dentry);
  1479. *victim = dentry;
  1480. return D_WALK_QUIT;
  1481. }
  1482. return D_WALK_CONTINUE;
  1483. }
  1484. /**
  1485. * d_invalidate - detach submounts, prune dcache, and drop
  1486. * @dentry: dentry to invalidate (aka detach, prune and drop)
  1487. */
  1488. void d_invalidate(struct dentry *dentry)
  1489. {
  1490. bool had_submounts = false;
  1491. spin_lock(&dentry->d_lock);
  1492. if (d_unhashed(dentry)) {
  1493. spin_unlock(&dentry->d_lock);
  1494. return;
  1495. }
  1496. __d_drop(dentry);
  1497. spin_unlock(&dentry->d_lock);
  1498. /* Negative dentries can be dropped without further checks */
  1499. if (!dentry->d_inode)
  1500. return;
  1501. shrink_dcache_parent(dentry);
  1502. for (;;) {
  1503. struct dentry *victim = NULL;
  1504. d_walk(dentry, &victim, find_submount);
  1505. if (!victim) {
  1506. if (had_submounts)
  1507. shrink_dcache_parent(dentry);
  1508. return;
  1509. }
  1510. had_submounts = true;
  1511. detach_mounts(victim);
  1512. dput(victim);
  1513. }
  1514. }
  1515. EXPORT_SYMBOL(d_invalidate);
  1516. /**
  1517. * __d_alloc - allocate a dcache entry
  1518. * @sb: filesystem it will belong to
  1519. * @name: qstr of the name
  1520. *
  1521. * Allocates a dentry. It returns %NULL if there is insufficient memory
  1522. * available. On a success the dentry is returned. The name passed in is
  1523. * copied and the copy passed in may be reused after this call.
  1524. */
  1525. static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
  1526. {
  1527. struct dentry *dentry;
  1528. char *dname;
  1529. int err;
  1530. dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL);
  1531. if (!dentry)
  1532. return NULL;
  1533. /*
  1534. * We guarantee that the inline name is always NUL-terminated.
  1535. * This way the memcpy() done by the name switching in rename
  1536. * will still always have a NUL at the end, even if we might
  1537. * be overwriting an internal NUL character
  1538. */
  1539. dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
  1540. if (unlikely(!name)) {
  1541. name = &slash_name;
  1542. dname = dentry->d_iname;
  1543. } else if (name->len > DNAME_INLINE_LEN-1) {
  1544. size_t size = offsetof(struct external_name, name[1]);
  1545. struct external_name *p = kmalloc(size + name->len,
  1546. GFP_KERNEL_ACCOUNT |
  1547. __GFP_RECLAIMABLE);
  1548. if (!p) {
  1549. kmem_cache_free(dentry_cache, dentry);
  1550. return NULL;
  1551. }
  1552. atomic_set(&p->u.count, 1);
  1553. dname = p->name;
  1554. } else {
  1555. dname = dentry->d_iname;
  1556. }
  1557. dentry->d_name.len = name->len;
  1558. dentry->d_name.hash = name->hash;
  1559. memcpy(dname, name->name, name->len);
  1560. dname[name->len] = 0;
  1561. /* Make sure we always see the terminating NUL character */
  1562. smp_store_release(&dentry->d_name.name, dname); /* ^^^ */
  1563. dentry->d_lockref.count = 1;
  1564. dentry->d_flags = 0;
  1565. spin_lock_init(&dentry->d_lock);
  1566. seqcount_spinlock_init(&dentry->d_seq, &dentry->d_lock);
  1567. dentry->d_inode = NULL;
  1568. dentry->d_parent = dentry;
  1569. dentry->d_sb = sb;
  1570. dentry->d_op = NULL;
  1571. dentry->d_fsdata = NULL;
  1572. INIT_HLIST_BL_NODE(&dentry->d_hash);
  1573. INIT_LIST_HEAD(&dentry->d_lru);
  1574. INIT_LIST_HEAD(&dentry->d_subdirs);
  1575. INIT_HLIST_NODE(&dentry->d_u.d_alias);
  1576. INIT_LIST_HEAD(&dentry->d_child);
  1577. d_set_d_op(dentry, dentry->d_sb->s_d_op);
  1578. if (dentry->d_op && dentry->d_op->d_init) {
  1579. err = dentry->d_op->d_init(dentry);
  1580. if (err) {
  1581. if (dname_external(dentry))
  1582. kfree(external_name(dentry));
  1583. kmem_cache_free(dentry_cache, dentry);
  1584. return NULL;
  1585. }
  1586. }
  1587. this_cpu_inc(nr_dentry);
  1588. return dentry;
  1589. }
  1590. /**
  1591. * d_alloc - allocate a dcache entry
  1592. * @parent: parent of entry to allocate
  1593. * @name: qstr of the name
  1594. *
  1595. * Allocates a dentry. It returns %NULL if there is insufficient memory
  1596. * available. On a success the dentry is returned. The name passed in is
  1597. * copied and the copy passed in may be reused after this call.
  1598. */
  1599. struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
  1600. {
  1601. struct dentry *dentry = __d_alloc(parent->d_sb, name);
  1602. if (!dentry)
  1603. return NULL;
  1604. spin_lock(&parent->d_lock);
  1605. /*
  1606. * don't need child lock because it is not subject
  1607. * to concurrency here
  1608. */
  1609. __dget_dlock(parent);
  1610. dentry->d_parent = parent;
  1611. list_add(&dentry->d_child, &parent->d_subdirs);
  1612. spin_unlock(&parent->d_lock);
  1613. return dentry;
  1614. }
  1615. EXPORT_SYMBOL(d_alloc);
  1616. struct dentry *d_alloc_anon(struct super_block *sb)
  1617. {
  1618. return __d_alloc(sb, NULL);
  1619. }
  1620. EXPORT_SYMBOL(d_alloc_anon);
  1621. struct dentry *d_alloc_cursor(struct dentry * parent)
  1622. {
  1623. struct dentry *dentry = d_alloc_anon(parent->d_sb);
  1624. if (dentry) {
  1625. dentry->d_flags |= DCACHE_DENTRY_CURSOR;
  1626. dentry->d_parent = dget(parent);
  1627. }
  1628. return dentry;
  1629. }
  1630. /**
  1631. * d_alloc_pseudo - allocate a dentry (for lookup-less filesystems)
  1632. * @sb: the superblock
  1633. * @name: qstr of the name
  1634. *
  1635. * For a filesystem that just pins its dentries in memory and never
  1636. * performs lookups at all, return an unhashed IS_ROOT dentry.
  1637. * This is used for pipes, sockets et.al. - the stuff that should
  1638. * never be anyone's children or parents. Unlike all other
  1639. * dentries, these will not have RCU delay between dropping the
  1640. * last reference and freeing them.
  1641. *
  1642. * The only user is alloc_file_pseudo() and that's what should
  1643. * be considered a public interface. Don't use directly.
  1644. */
  1645. struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
  1646. {
  1647. struct dentry *dentry = __d_alloc(sb, name);
  1648. if (likely(dentry))
  1649. dentry->d_flags |= DCACHE_NORCU;
  1650. return dentry;
  1651. }
  1652. struct dentry *d_alloc_name(struct dentry *parent, const char *name)
  1653. {
  1654. struct qstr q;
  1655. q.name = name;
  1656. q.hash_len = hashlen_string(parent, name);
  1657. return d_alloc(parent, &q);
  1658. }
  1659. EXPORT_SYMBOL(d_alloc_name);
  1660. void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
  1661. {
  1662. WARN_ON_ONCE(dentry->d_op);
  1663. WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
  1664. DCACHE_OP_COMPARE |
  1665. DCACHE_OP_REVALIDATE |
  1666. DCACHE_OP_WEAK_REVALIDATE |
  1667. DCACHE_OP_DELETE |
  1668. DCACHE_OP_REAL));
  1669. dentry->d_op = op;
  1670. if (!op)
  1671. return;
  1672. if (op->d_hash)
  1673. dentry->d_flags |= DCACHE_OP_HASH;
  1674. if (op->d_compare)
  1675. dentry->d_flags |= DCACHE_OP_COMPARE;
  1676. if (op->d_revalidate)
  1677. dentry->d_flags |= DCACHE_OP_REVALIDATE;
  1678. if (op->d_weak_revalidate)
  1679. dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE;
  1680. if (op->d_delete)
  1681. dentry->d_flags |= DCACHE_OP_DELETE;
  1682. if (op->d_prune)
  1683. dentry->d_flags |= DCACHE_OP_PRUNE;
  1684. if (op->d_real)
  1685. dentry->d_flags |= DCACHE_OP_REAL;
  1686. }
  1687. EXPORT_SYMBOL(d_set_d_op);
  1688. /*
  1689. * d_set_fallthru - Mark a dentry as falling through to a lower layer
  1690. * @dentry - The dentry to mark
  1691. *
  1692. * Mark a dentry as falling through to the lower layer (as set with
  1693. * d_pin_lower()). This flag may be recorded on the medium.
  1694. */
  1695. void d_set_fallthru(struct dentry *dentry)
  1696. {
  1697. spin_lock(&dentry->d_lock);
  1698. dentry->d_flags |= DCACHE_FALLTHRU;
  1699. spin_unlock(&dentry->d_lock);
  1700. }
  1701. EXPORT_SYMBOL(d_set_fallthru);
  1702. static unsigned d_flags_for_inode(struct inode *inode)
  1703. {
  1704. unsigned add_flags = DCACHE_REGULAR_TYPE;
  1705. if (!inode)
  1706. return DCACHE_MISS_TYPE;
  1707. if (S_ISDIR(inode->i_mode)) {
  1708. add_flags = DCACHE_DIRECTORY_TYPE;
  1709. if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) {
  1710. if (unlikely(!inode->i_op->lookup))
  1711. add_flags = DCACHE_AUTODIR_TYPE;
  1712. else
  1713. inode->i_opflags |= IOP_LOOKUP;
  1714. }
  1715. goto type_determined;
  1716. }
  1717. if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) {
  1718. if (unlikely(inode->i_op->get_link)) {
  1719. add_flags = DCACHE_SYMLINK_TYPE;
  1720. goto type_determined;
  1721. }
  1722. inode->i_opflags |= IOP_NOFOLLOW;
  1723. }
  1724. if (unlikely(!S_ISREG(inode->i_mode)))
  1725. add_flags = DCACHE_SPECIAL_TYPE;
  1726. type_determined:
  1727. if (unlikely(IS_AUTOMOUNT(inode)))
  1728. add_flags |= DCACHE_NEED_AUTOMOUNT;
  1729. return add_flags;
  1730. }
  1731. static void __d_instantiate(struct dentry *dentry, struct inode *inode)
  1732. {
  1733. unsigned add_flags = d_flags_for_inode(inode);
  1734. WARN_ON(d_in_lookup(dentry));
  1735. spin_lock(&dentry->d_lock);
  1736. /*
  1737. * Decrement negative dentry count if it was in the LRU list.
  1738. */
  1739. if (dentry->d_flags & DCACHE_LRU_LIST)
  1740. this_cpu_dec(nr_dentry_negative);
  1741. hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
  1742. raw_write_seqcount_begin(&dentry->d_seq);
  1743. __d_set_inode_and_type(dentry, inode, add_flags);
  1744. raw_write_seqcount_end(&dentry->d_seq);
  1745. fsnotify_update_flags(dentry);
  1746. spin_unlock(&dentry->d_lock);
  1747. }
  1748. /**
  1749. * d_instantiate - fill in inode information for a dentry
  1750. * @entry: dentry to complete
  1751. * @inode: inode to attach to this dentry
  1752. *
  1753. * Fill in inode information in the entry.
  1754. *
  1755. * This turns negative dentries into productive full members
  1756. * of society.
  1757. *
  1758. * NOTE! This assumes that the inode count has been incremented
  1759. * (or otherwise set) by the caller to indicate that it is now
  1760. * in use by the dcache.
  1761. */
  1762. void d_instantiate(struct dentry *entry, struct inode * inode)
  1763. {
  1764. BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
  1765. if (inode) {
  1766. security_d_instantiate(entry, inode);
  1767. spin_lock(&inode->i_lock);
  1768. __d_instantiate(entry, inode);
  1769. spin_unlock(&inode->i_lock);
  1770. }
  1771. }
  1772. EXPORT_SYMBOL(d_instantiate);
  1773. /*
  1774. * This should be equivalent to d_instantiate() + unlock_new_inode(),
  1775. * with lockdep-related part of unlock_new_inode() done before
  1776. * anything else. Use that instead of open-coding d_instantiate()/
  1777. * unlock_new_inode() combinations.
  1778. */
  1779. void d_instantiate_new(struct dentry *entry, struct inode *inode)
  1780. {
  1781. BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
  1782. BUG_ON(!inode);
  1783. lockdep_annotate_inode_mutex_key(inode);
  1784. security_d_instantiate(entry, inode);
  1785. spin_lock(&inode->i_lock);
  1786. __d_instantiate(entry, inode);
  1787. WARN_ON(!(inode->i_state & I_NEW));
  1788. inode->i_state &= ~I_NEW & ~I_CREATING;
  1789. smp_mb();
  1790. wake_up_bit(&inode->i_state, __I_NEW);
  1791. spin_unlock(&inode->i_lock);
  1792. }
  1793. EXPORT_SYMBOL(d_instantiate_new);
  1794. struct dentry *d_make_root(struct inode *root_inode)
  1795. {
  1796. struct dentry *res = NULL;
  1797. if (root_inode) {
  1798. res = d_alloc_anon(root_inode->i_sb);
  1799. if (res)
  1800. d_instantiate(res, root_inode);
  1801. else
  1802. iput(root_inode);
  1803. }
  1804. return res;
  1805. }
  1806. EXPORT_SYMBOL(d_make_root);
  1807. static struct dentry *__d_instantiate_anon(struct dentry *dentry,
  1808. struct inode *inode,
  1809. bool disconnected)
  1810. {
  1811. struct dentry *res;
  1812. unsigned add_flags;
  1813. security_d_instantiate(dentry, inode);
  1814. spin_lock(&inode->i_lock);
  1815. res = __d_find_any_alias(inode);
  1816. if (res) {
  1817. spin_unlock(&inode->i_lock);
  1818. dput(dentry);
  1819. goto out_iput;
  1820. }
  1821. /* attach a disconnected dentry */
  1822. add_flags = d_flags_for_inode(inode);
  1823. if (disconnected)
  1824. add_flags |= DCACHE_DISCONNECTED;
  1825. spin_lock(&dentry->d_lock);
  1826. __d_set_inode_and_type(dentry, inode, add_flags);
  1827. hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
  1828. if (!disconnected) {
  1829. hlist_bl_lock(&dentry->d_sb->s_roots);
  1830. hlist_bl_add_head(&dentry->d_hash, &dentry->d_sb->s_roots);
  1831. hlist_bl_unlock(&dentry->d_sb->s_roots);
  1832. }
  1833. spin_unlock(&dentry->d_lock);
  1834. spin_unlock(&inode->i_lock);
  1835. return dentry;
  1836. out_iput:
  1837. iput(inode);
  1838. return res;
  1839. }
  1840. struct dentry *d_instantiate_anon(struct dentry *dentry, struct inode *inode)
  1841. {
  1842. return __d_instantiate_anon(dentry, inode, true);
  1843. }
  1844. EXPORT_SYMBOL(d_instantiate_anon);
  1845. static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected)
  1846. {
  1847. struct dentry *tmp;
  1848. struct dentry *res;
  1849. if (!inode)
  1850. return ERR_PTR(-ESTALE);
  1851. if (IS_ERR(inode))
  1852. return ERR_CAST(inode);
  1853. res = d_find_any_alias(inode);
  1854. if (res)
  1855. goto out_iput;
  1856. tmp = d_alloc_anon(inode->i_sb);
  1857. if (!tmp) {
  1858. res = ERR_PTR(-ENOMEM);
  1859. goto out_iput;
  1860. }
  1861. return __d_instantiate_anon(tmp, inode, disconnected);
  1862. out_iput:
  1863. iput(inode);
  1864. return res;
  1865. }
  1866. /**
  1867. * d_obtain_alias - find or allocate a DISCONNECTED dentry for a given inode
  1868. * @inode: inode to allocate the dentry for
  1869. *
  1870. * Obtain a dentry for an inode resulting from NFS filehandle conversion or
  1871. * similar open by handle operations. The returned dentry may be anonymous,
  1872. * or may have a full name (if the inode was already in the cache).
  1873. *
  1874. * When called on a directory inode, we must ensure that the inode only ever
  1875. * has one dentry. If a dentry is found, that is returned instead of
  1876. * allocating a new one.
  1877. *
  1878. * On successful return, the reference to the inode has been transferred
  1879. * to the dentry. In case of an error the reference on the inode is released.
  1880. * To make it easier to use in export operations a %NULL or IS_ERR inode may
  1881. * be passed in and the error will be propagated to the return value,
  1882. * with a %NULL @inode replaced by ERR_PTR(-ESTALE).
  1883. */
  1884. struct dentry *d_obtain_alias(struct inode *inode)
  1885. {
  1886. return __d_obtain_alias(inode, true);
  1887. }
  1888. EXPORT_SYMBOL_NS(d_obtain_alias, ANDROID_GKI_VFS_EXPORT_ONLY);
  1889. /**
  1890. * d_obtain_root - find or allocate a dentry for a given inode
  1891. * @inode: inode to allocate the dentry for
  1892. *
  1893. * Obtain an IS_ROOT dentry for the root of a filesystem.
  1894. *
  1895. * We must ensure that directory inodes only ever have one dentry. If a
  1896. * dentry is found, that is returned instead of allocating a new one.
  1897. *
  1898. * On successful return, the reference to the inode has been transferred
  1899. * to the dentry. In case of an error the reference on the inode is
  1900. * released. A %NULL or IS_ERR inode may be passed in and will be the
  1901. * error will be propagate to the return value, with a %NULL @inode
  1902. * replaced by ERR_PTR(-ESTALE).
  1903. */
  1904. struct dentry *d_obtain_root(struct inode *inode)
  1905. {
  1906. return __d_obtain_alias(inode, false);
  1907. }
  1908. EXPORT_SYMBOL(d_obtain_root);
  1909. /**
  1910. * d_add_ci - lookup or allocate new dentry with case-exact name
  1911. * @inode: the inode case-insensitive lookup has found
  1912. * @dentry: the negative dentry that was passed to the parent's lookup func
  1913. * @name: the case-exact name to be associated with the returned dentry
  1914. *
  1915. * This is to avoid filling the dcache with case-insensitive names to the
  1916. * same inode, only the actual correct case is stored in the dcache for
  1917. * case-insensitive filesystems.
  1918. *
  1919. * For a case-insensitive lookup match and if the the case-exact dentry
  1920. * already exists in in the dcache, use it and return it.
  1921. *
  1922. * If no entry exists with the exact case name, allocate new dentry with
  1923. * the exact case, and return the spliced entry.
  1924. */
  1925. struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
  1926. struct qstr *name)
  1927. {
  1928. struct dentry *found, *res;
  1929. /*
  1930. * First check if a dentry matching the name already exists,
  1931. * if not go ahead and create it now.
  1932. */
  1933. found = d_hash_and_lookup(dentry->d_parent, name);
  1934. if (found) {
  1935. iput(inode);
  1936. return found;
  1937. }
  1938. if (d_in_lookup(dentry)) {
  1939. found = d_alloc_parallel(dentry->d_parent, name,
  1940. dentry->d_wait);
  1941. if (IS_ERR(found) || !d_in_lookup(found)) {
  1942. iput(inode);
  1943. return found;
  1944. }
  1945. } else {
  1946. found = d_alloc(dentry->d_parent, name);
  1947. if (!found) {
  1948. iput(inode);
  1949. return ERR_PTR(-ENOMEM);
  1950. }
  1951. }
  1952. res = d_splice_alias(inode, found);
  1953. if (res) {
  1954. dput(found);
  1955. return res;
  1956. }
  1957. return found;
  1958. }
  1959. EXPORT_SYMBOL_NS(d_add_ci, ANDROID_GKI_VFS_EXPORT_ONLY);
  1960. static inline bool d_same_name(const struct dentry *dentry,
  1961. const struct dentry *parent,
  1962. const struct qstr *name)
  1963. {
  1964. if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) {
  1965. if (dentry->d_name.len != name->len)
  1966. return false;
  1967. return dentry_cmp(dentry, name->name, name->len) == 0;
  1968. }
  1969. return parent->d_op->d_compare(dentry,
  1970. dentry->d_name.len, dentry->d_name.name,
  1971. name) == 0;
  1972. }
  1973. /**
  1974. * __d_lookup_rcu - search for a dentry (racy, store-free)
  1975. * @parent: parent dentry
  1976. * @name: qstr of name we wish to find
  1977. * @seqp: returns d_seq value at the point where the dentry was found
  1978. * Returns: dentry, or NULL
  1979. *
  1980. * __d_lookup_rcu is the dcache lookup function for rcu-walk name
  1981. * resolution (store-free path walking) design described in
  1982. * Documentation/filesystems/path-lookup.txt.
  1983. *
  1984. * This is not to be used outside core vfs.
  1985. *
  1986. * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
  1987. * held, and rcu_read_lock held. The returned dentry must not be stored into
  1988. * without taking d_lock and checking d_seq sequence count against @seq
  1989. * returned here.
  1990. *
  1991. * A refcount may be taken on the found dentry with the d_rcu_to_refcount
  1992. * function.
  1993. *
  1994. * Alternatively, __d_lookup_rcu may be called again to look up the child of
  1995. * the returned dentry, so long as its parent's seqlock is checked after the
  1996. * child is looked up. Thus, an interlocking stepping of sequence lock checks
  1997. * is formed, giving integrity down the path walk.
  1998. *
  1999. * NOTE! The caller *has* to check the resulting dentry against the sequence
  2000. * number we've returned before using any of the resulting dentry state!
  2001. */
  2002. struct dentry *__d_lookup_rcu(const struct dentry *parent,
  2003. const struct qstr *name,
  2004. unsigned *seqp)
  2005. {
  2006. u64 hashlen = name->hash_len;
  2007. const unsigned char *str = name->name;
  2008. struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen));
  2009. struct hlist_bl_node *node;
  2010. struct dentry *dentry;
  2011. /*
  2012. * Note: There is significant duplication with __d_lookup_rcu which is
  2013. * required to prevent single threaded performance regressions
  2014. * especially on architectures where smp_rmb (in seqcounts) are costly.
  2015. * Keep the two functions in sync.
  2016. */
  2017. /*
  2018. * The hash list is protected using RCU.
  2019. *
  2020. * Carefully use d_seq when comparing a candidate dentry, to avoid
  2021. * races with d_move().
  2022. *
  2023. * It is possible that concurrent renames can mess up our list
  2024. * walk here and result in missing our dentry, resulting in the
  2025. * false-negative result. d_lookup() protects against concurrent
  2026. * renames using rename_lock seqlock.
  2027. *
  2028. * See Documentation/filesystems/path-lookup.txt for more details.
  2029. */
  2030. hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
  2031. unsigned seq;
  2032. seqretry:
  2033. /*
  2034. * The dentry sequence count protects us from concurrent
  2035. * renames, and thus protects parent and name fields.
  2036. *
  2037. * The caller must perform a seqcount check in order
  2038. * to do anything useful with the returned dentry.
  2039. *
  2040. * NOTE! We do a "raw" seqcount_begin here. That means that
  2041. * we don't wait for the sequence count to stabilize if it
  2042. * is in the middle of a sequence change. If we do the slow
  2043. * dentry compare, we will do seqretries until it is stable,
  2044. * and if we end up with a successful lookup, we actually
  2045. * want to exit RCU lookup anyway.
  2046. *
  2047. * Note that raw_seqcount_begin still *does* smp_rmb(), so
  2048. * we are still guaranteed NUL-termination of ->d_name.name.
  2049. */
  2050. seq = raw_seqcount_begin(&dentry->d_seq);
  2051. if (dentry->d_parent != parent)
  2052. continue;
  2053. if (d_unhashed(dentry))
  2054. continue;
  2055. if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) {
  2056. int tlen;
  2057. const char *tname;
  2058. if (dentry->d_name.hash != hashlen_hash(hashlen))
  2059. continue;
  2060. tlen = dentry->d_name.len;
  2061. tname = dentry->d_name.name;
  2062. /* we want a consistent (name,len) pair */
  2063. if (read_seqcount_retry(&dentry->d_seq, seq)) {
  2064. cpu_relax();
  2065. goto seqretry;
  2066. }
  2067. if (parent->d_op->d_compare(dentry,
  2068. tlen, tname, name) != 0)
  2069. continue;
  2070. } else {
  2071. if (dentry->d_name.hash_len != hashlen)
  2072. continue;
  2073. if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0)
  2074. continue;
  2075. }
  2076. *seqp = seq;
  2077. return dentry;
  2078. }
  2079. return NULL;
  2080. }
  2081. /**
  2082. * d_lookup - search for a dentry
  2083. * @parent: parent dentry
  2084. * @name: qstr of name we wish to find
  2085. * Returns: dentry, or NULL
  2086. *
  2087. * d_lookup searches the children of the parent dentry for the name in
  2088. * question. If the dentry is found its reference count is incremented and the
  2089. * dentry is returned. The caller must use dput to free the entry when it has
  2090. * finished using it. %NULL is returned if the dentry does not exist.
  2091. */
  2092. struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
  2093. {
  2094. struct dentry *dentry;
  2095. unsigned seq;
  2096. do {
  2097. seq = read_seqbegin(&rename_lock);
  2098. dentry = __d_lookup(parent, name);
  2099. if (dentry)
  2100. break;
  2101. } while (read_seqretry(&rename_lock, seq));
  2102. return dentry;
  2103. }
  2104. EXPORT_SYMBOL(d_lookup);
  2105. /**
  2106. * __d_lookup - search for a dentry (racy)
  2107. * @parent: parent dentry
  2108. * @name: qstr of name we wish to find
  2109. * Returns: dentry, or NULL
  2110. *
  2111. * __d_lookup is like d_lookup, however it may (rarely) return a
  2112. * false-negative result due to unrelated rename activity.
  2113. *
  2114. * __d_lookup is slightly faster by avoiding rename_lock read seqlock,
  2115. * however it must be used carefully, eg. with a following d_lookup in
  2116. * the case of failure.
  2117. *
  2118. * __d_lookup callers must be commented.
  2119. */
  2120. struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
  2121. {
  2122. unsigned int hash = name->hash;
  2123. struct hlist_bl_head *b = d_hash(hash);
  2124. struct hlist_bl_node *node;
  2125. struct dentry *found = NULL;
  2126. struct dentry *dentry;
  2127. /*
  2128. * Note: There is significant duplication with __d_lookup_rcu which is
  2129. * required to prevent single threaded performance regressions
  2130. * especially on architectures where smp_rmb (in seqcounts) are costly.
  2131. * Keep the two functions in sync.
  2132. */
  2133. /*
  2134. * The hash list is protected using RCU.
  2135. *
  2136. * Take d_lock when comparing a candidate dentry, to avoid races
  2137. * with d_move().
  2138. *
  2139. * It is possible that concurrent renames can mess up our list
  2140. * walk here and result in missing our dentry, resulting in the
  2141. * false-negative result. d_lookup() protects against concurrent
  2142. * renames using rename_lock seqlock.
  2143. *
  2144. * See Documentation/filesystems/path-lookup.txt for more details.
  2145. */
  2146. rcu_read_lock();
  2147. hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
  2148. if (dentry->d_name.hash != hash)
  2149. continue;
  2150. spin_lock(&dentry->d_lock);
  2151. if (dentry->d_parent != parent)
  2152. goto next;
  2153. if (d_unhashed(dentry))
  2154. goto next;
  2155. if (!d_same_name(dentry, parent, name))
  2156. goto next;
  2157. dentry->d_lockref.count++;
  2158. found = dentry;
  2159. spin_unlock(&dentry->d_lock);
  2160. break;
  2161. next:
  2162. spin_unlock(&dentry->d_lock);
  2163. }
  2164. rcu_read_unlock();
  2165. return found;
  2166. }
  2167. /**
  2168. * d_hash_and_lookup - hash the qstr then search for a dentry
  2169. * @dir: Directory to search in
  2170. * @name: qstr of name we wish to find
  2171. *
  2172. * On lookup failure NULL is returned; on bad name - ERR_PTR(-error)
  2173. */
  2174. struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
  2175. {
  2176. /*
  2177. * Check for a fs-specific hash function. Note that we must
  2178. * calculate the standard hash first, as the d_op->d_hash()
  2179. * routine may choose to leave the hash value unchanged.
  2180. */
  2181. name->hash = full_name_hash(dir, name->name, name->len);
  2182. if (dir->d_flags & DCACHE_OP_HASH) {
  2183. int err = dir->d_op->d_hash(dir, name);
  2184. if (unlikely(err < 0))
  2185. return ERR_PTR(err);
  2186. }
  2187. return d_lookup(dir, name);
  2188. }
  2189. EXPORT_SYMBOL(d_hash_and_lookup);
  2190. /*
  2191. * When a file is deleted, we have two options:
  2192. * - turn this dentry into a negative dentry
  2193. * - unhash this dentry and free it.
  2194. *
  2195. * Usually, we want to just turn this into
  2196. * a negative dentry, but if anybody else is
  2197. * currently using the dentry or the inode
  2198. * we can't do that and we fall back on removing
  2199. * it from the hash queues and waiting for
  2200. * it to be deleted later when it has no users
  2201. */
  2202. /**
  2203. * d_delete - delete a dentry
  2204. * @dentry: The dentry to delete
  2205. *
  2206. * Turn the dentry into a negative dentry if possible, otherwise
  2207. * remove it from the hash queues so it can be deleted later
  2208. */
  2209. void d_delete(struct dentry * dentry)
  2210. {
  2211. struct inode *inode = dentry->d_inode;
  2212. spin_lock(&inode->i_lock);
  2213. spin_lock(&dentry->d_lock);
  2214. /*
  2215. * Are we the only user?
  2216. */
  2217. if (dentry->d_lockref.count == 1) {
  2218. dentry->d_flags &= ~DCACHE_CANT_MOUNT;
  2219. dentry_unlink_inode(dentry);
  2220. } else {
  2221. __d_drop(dentry);
  2222. spin_unlock(&dentry->d_lock);
  2223. spin_unlock(&inode->i_lock);
  2224. }
  2225. }
  2226. EXPORT_SYMBOL(d_delete);
  2227. static void __d_rehash(struct dentry *entry)
  2228. {
  2229. struct hlist_bl_head *b = d_hash(entry->d_name.hash);
  2230. hlist_bl_lock(b);
  2231. hlist_bl_add_head_rcu(&entry->d_hash, b);
  2232. hlist_bl_unlock(b);
  2233. }
  2234. /**
  2235. * d_rehash - add an entry back to the hash
  2236. * @entry: dentry to add to the hash
  2237. *
  2238. * Adds a dentry to the hash according to its name.
  2239. */
  2240. void d_rehash(struct dentry * entry)
  2241. {
  2242. spin_lock(&entry->d_lock);
  2243. __d_rehash(entry);
  2244. spin_unlock(&entry->d_lock);
  2245. }
  2246. EXPORT_SYMBOL(d_rehash);
  2247. static inline unsigned start_dir_add(struct inode *dir)
  2248. {
  2249. for (;;) {
  2250. unsigned n = dir->i_dir_seq;
  2251. if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n)
  2252. return n;
  2253. cpu_relax();
  2254. }
  2255. }
  2256. static inline void end_dir_add(struct inode *dir, unsigned n)
  2257. {
  2258. smp_store_release(&dir->i_dir_seq, n + 2);
  2259. }
  2260. static void d_wait_lookup(struct dentry *dentry)
  2261. {
  2262. if (d_in_lookup(dentry)) {
  2263. DECLARE_WAITQUEUE(wait, current);
  2264. add_wait_queue(dentry->d_wait, &wait);
  2265. do {
  2266. set_current_state(TASK_UNINTERRUPTIBLE);
  2267. spin_unlock(&dentry->d_lock);
  2268. schedule();
  2269. spin_lock(&dentry->d_lock);
  2270. } while (d_in_lookup(dentry));
  2271. }
  2272. }
  2273. struct dentry *d_alloc_parallel(struct dentry *parent,
  2274. const struct qstr *name,
  2275. wait_queue_head_t *wq)
  2276. {
  2277. unsigned int hash = name->hash;
  2278. struct hlist_bl_head *b = in_lookup_hash(parent, hash);
  2279. struct hlist_bl_node *node;
  2280. struct dentry *new = d_alloc(parent, name);
  2281. struct dentry *dentry;
  2282. unsigned seq, r_seq, d_seq;
  2283. if (unlikely(!new))
  2284. return ERR_PTR(-ENOMEM);
  2285. retry:
  2286. rcu_read_lock();
  2287. seq = smp_load_acquire(&parent->d_inode->i_dir_seq);
  2288. r_seq = read_seqbegin(&rename_lock);
  2289. dentry = __d_lookup_rcu(parent, name, &d_seq);
  2290. if (unlikely(dentry)) {
  2291. if (!lockref_get_not_dead(&dentry->d_lockref)) {
  2292. rcu_read_unlock();
  2293. goto retry;
  2294. }
  2295. if (read_seqcount_retry(&dentry->d_seq, d_seq)) {
  2296. rcu_read_unlock();
  2297. dput(dentry);
  2298. goto retry;
  2299. }
  2300. rcu_read_unlock();
  2301. dput(new);
  2302. return dentry;
  2303. }
  2304. if (unlikely(read_seqretry(&rename_lock, r_seq))) {
  2305. rcu_read_unlock();
  2306. goto retry;
  2307. }
  2308. if (unlikely(seq & 1)) {
  2309. rcu_read_unlock();
  2310. goto retry;
  2311. }
  2312. hlist_bl_lock(b);
  2313. if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) {
  2314. hlist_bl_unlock(b);
  2315. rcu_read_unlock();
  2316. goto retry;
  2317. }
  2318. /*
  2319. * No changes for the parent since the beginning of d_lookup().
  2320. * Since all removals from the chain happen with hlist_bl_lock(),
  2321. * any potential in-lookup matches are going to stay here until
  2322. * we unlock the chain. All fields are stable in everything
  2323. * we encounter.
  2324. */
  2325. hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) {
  2326. if (dentry->d_name.hash != hash)
  2327. continue;
  2328. if (dentry->d_parent != parent)
  2329. continue;
  2330. if (!d_same_name(dentry, parent, name))
  2331. continue;
  2332. hlist_bl_unlock(b);
  2333. /* now we can try to grab a reference */
  2334. if (!lockref_get_not_dead(&dentry->d_lockref)) {
  2335. rcu_read_unlock();
  2336. goto retry;
  2337. }
  2338. rcu_read_unlock();
  2339. /*
  2340. * somebody is likely to be still doing lookup for it;
  2341. * wait for them to finish
  2342. */
  2343. spin_lock(&dentry->d_lock);
  2344. d_wait_lookup(dentry);
  2345. /*
  2346. * it's not in-lookup anymore; in principle we should repeat
  2347. * everything from dcache lookup, but it's likely to be what
  2348. * d_lookup() would've found anyway. If it is, just return it;
  2349. * otherwise we really have to repeat the whole thing.
  2350. */
  2351. if (unlikely(dentry->d_name.hash != hash))
  2352. goto mismatch;
  2353. if (unlikely(dentry->d_parent != parent))
  2354. goto mismatch;
  2355. if (unlikely(d_unhashed(dentry)))
  2356. goto mismatch;
  2357. if (unlikely(!d_same_name(dentry, parent, name)))
  2358. goto mismatch;
  2359. /* OK, it *is* a hashed match; return it */
  2360. spin_unlock(&dentry->d_lock);
  2361. dput(new);
  2362. return dentry;
  2363. }
  2364. rcu_read_unlock();
  2365. /* we can't take ->d_lock here; it's OK, though. */
  2366. new->d_flags |= DCACHE_PAR_LOOKUP;
  2367. new->d_wait = wq;
  2368. hlist_bl_add_head_rcu(&new->d_u.d_in_lookup_hash, b);
  2369. hlist_bl_unlock(b);
  2370. return new;
  2371. mismatch:
  2372. spin_unlock(&dentry->d_lock);
  2373. dput(dentry);
  2374. goto retry;
  2375. }
  2376. EXPORT_SYMBOL(d_alloc_parallel);
  2377. void __d_lookup_done(struct dentry *dentry)
  2378. {
  2379. struct hlist_bl_head *b = in_lookup_hash(dentry->d_parent,
  2380. dentry->d_name.hash);
  2381. hlist_bl_lock(b);
  2382. dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
  2383. __hlist_bl_del(&dentry->d_u.d_in_lookup_hash);
  2384. wake_up_all(dentry->d_wait);
  2385. dentry->d_wait = NULL;
  2386. hlist_bl_unlock(b);
  2387. INIT_HLIST_NODE(&dentry->d_u.d_alias);
  2388. INIT_LIST_HEAD(&dentry->d_lru);
  2389. }
  2390. EXPORT_SYMBOL(__d_lookup_done);
  2391. /* inode->i_lock held if inode is non-NULL */
  2392. static inline void __d_add(struct dentry *dentry, struct inode *inode)
  2393. {
  2394. struct inode *dir = NULL;
  2395. unsigned n;
  2396. spin_lock(&dentry->d_lock);
  2397. if (unlikely(d_in_lookup(dentry))) {
  2398. dir = dentry->d_parent->d_inode;
  2399. n = start_dir_add(dir);
  2400. __d_lookup_done(dentry);
  2401. }
  2402. if (inode) {
  2403. unsigned add_flags = d_flags_for_inode(inode);
  2404. hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
  2405. raw_write_seqcount_begin(&dentry->d_seq);
  2406. __d_set_inode_and_type(dentry, inode, add_flags);
  2407. raw_write_seqcount_end(&dentry->d_seq);
  2408. fsnotify_update_flags(dentry);
  2409. }
  2410. __d_rehash(dentry);
  2411. if (dir)
  2412. end_dir_add(dir, n);
  2413. spin_unlock(&dentry->d_lock);
  2414. if (inode)
  2415. spin_unlock(&inode->i_lock);
  2416. }
  2417. /**
  2418. * d_add - add dentry to hash queues
  2419. * @entry: dentry to add
  2420. * @inode: The inode to attach to this dentry
  2421. *
  2422. * This adds the entry to the hash queues and initializes @inode.
  2423. * The entry was actually filled in earlier during d_alloc().
  2424. */
  2425. void d_add(struct dentry *entry, struct inode *inode)
  2426. {
  2427. if (inode) {
  2428. security_d_instantiate(entry, inode);
  2429. spin_lock(&inode->i_lock);
  2430. }
  2431. __d_add(entry, inode);
  2432. }
  2433. EXPORT_SYMBOL(d_add);
  2434. /**
  2435. * d_exact_alias - find and hash an exact unhashed alias
  2436. * @entry: dentry to add
  2437. * @inode: The inode to go with this dentry
  2438. *
  2439. * If an unhashed dentry with the same name/parent and desired
  2440. * inode already exists, hash and return it. Otherwise, return
  2441. * NULL.
  2442. *
  2443. * Parent directory should be locked.
  2444. */
  2445. struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode)
  2446. {
  2447. struct dentry *alias;
  2448. unsigned int hash = entry->d_name.hash;
  2449. spin_lock(&inode->i_lock);
  2450. hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
  2451. /*
  2452. * Don't need alias->d_lock here, because aliases with
  2453. * d_parent == entry->d_parent are not subject to name or
  2454. * parent changes, because the parent inode i_mutex is held.
  2455. */
  2456. if (alias->d_name.hash != hash)
  2457. continue;
  2458. if (alias->d_parent != entry->d_parent)
  2459. continue;
  2460. if (!d_same_name(alias, entry->d_parent, &entry->d_name))
  2461. continue;
  2462. spin_lock(&alias->d_lock);
  2463. if (!d_unhashed(alias)) {
  2464. spin_unlock(&alias->d_lock);
  2465. alias = NULL;
  2466. } else {
  2467. __dget_dlock(alias);
  2468. __d_rehash(alias);
  2469. spin_unlock(&alias->d_lock);
  2470. }
  2471. spin_unlock(&inode->i_lock);
  2472. return alias;
  2473. }
  2474. spin_unlock(&inode->i_lock);
  2475. return NULL;
  2476. }
  2477. EXPORT_SYMBOL(d_exact_alias);
  2478. static void swap_names(struct dentry *dentry, struct dentry *target)
  2479. {
  2480. if (unlikely(dname_external(target))) {
  2481. if (unlikely(dname_external(dentry))) {
  2482. /*
  2483. * Both external: swap the pointers
  2484. */
  2485. swap(target->d_name.name, dentry->d_name.name);
  2486. } else {
  2487. /*
  2488. * dentry:internal, target:external. Steal target's
  2489. * storage and make target internal.
  2490. */
  2491. memcpy(target->d_iname, dentry->d_name.name,
  2492. dentry->d_name.len + 1);
  2493. dentry->d_name.name = target->d_name.name;
  2494. target->d_name.name = target->d_iname;
  2495. }
  2496. } else {
  2497. if (unlikely(dname_external(dentry))) {
  2498. /*
  2499. * dentry:external, target:internal. Give dentry's
  2500. * storage to target and make dentry internal
  2501. */
  2502. memcpy(dentry->d_iname, target->d_name.name,
  2503. target->d_name.len + 1);
  2504. target->d_name.name = dentry->d_name.name;
  2505. dentry->d_name.name = dentry->d_iname;
  2506. } else {
  2507. /*
  2508. * Both are internal.
  2509. */
  2510. unsigned int i;
  2511. BUILD_BUG_ON(!IS_ALIGNED(DNAME_INLINE_LEN, sizeof(long)));
  2512. for (i = 0; i < DNAME_INLINE_LEN / sizeof(long); i++) {
  2513. swap(((long *) &dentry->d_iname)[i],
  2514. ((long *) &target->d_iname)[i]);
  2515. }
  2516. }
  2517. }
  2518. swap(dentry->d_name.hash_len, target->d_name.hash_len);
  2519. }
  2520. static void copy_name(struct dentry *dentry, struct dentry *target)
  2521. {
  2522. struct external_name *old_name = NULL;
  2523. if (unlikely(dname_external(dentry)))
  2524. old_name = external_name(dentry);
  2525. if (unlikely(dname_external(target))) {
  2526. atomic_inc(&external_name(target)->u.count);
  2527. dentry->d_name = target->d_name;
  2528. } else {
  2529. memcpy(dentry->d_iname, target->d_name.name,
  2530. target->d_name.len + 1);
  2531. dentry->d_name.name = dentry->d_iname;
  2532. dentry->d_name.hash_len = target->d_name.hash_len;
  2533. }
  2534. if (old_name && likely(atomic_dec_and_test(&old_name->u.count)))
  2535. kfree_rcu(old_name, u.head);
  2536. }
  2537. /*
  2538. * __d_move - move a dentry
  2539. * @dentry: entry to move
  2540. * @target: new dentry
  2541. * @exchange: exchange the two dentries
  2542. *
  2543. * Update the dcache to reflect the move of a file name. Negative
  2544. * dcache entries should not be moved in this way. Caller must hold
  2545. * rename_lock, the i_mutex of the source and target directories,
  2546. * and the sb->s_vfs_rename_mutex if they differ. See lock_rename().
  2547. */
  2548. static void __d_move(struct dentry *dentry, struct dentry *target,
  2549. bool exchange)
  2550. {
  2551. struct dentry *old_parent, *p;
  2552. struct inode *dir = NULL;
  2553. unsigned n;
  2554. WARN_ON(!dentry->d_inode);
  2555. if (WARN_ON(dentry == target))
  2556. return;
  2557. BUG_ON(d_ancestor(target, dentry));
  2558. old_parent = dentry->d_parent;
  2559. p = d_ancestor(old_parent, target);
  2560. if (IS_ROOT(dentry)) {
  2561. BUG_ON(p);
  2562. spin_lock(&target->d_parent->d_lock);
  2563. } else if (!p) {
  2564. /* target is not a descendent of dentry->d_parent */
  2565. spin_lock(&target->d_parent->d_lock);
  2566. spin_lock_nested(&old_parent->d_lock, DENTRY_D_LOCK_NESTED);
  2567. } else {
  2568. BUG_ON(p == dentry);
  2569. spin_lock(&old_parent->d_lock);
  2570. if (p != target)
  2571. spin_lock_nested(&target->d_parent->d_lock,
  2572. DENTRY_D_LOCK_NESTED);
  2573. }
  2574. spin_lock_nested(&dentry->d_lock, 2);
  2575. spin_lock_nested(&target->d_lock, 3);
  2576. if (unlikely(d_in_lookup(target))) {
  2577. dir = target->d_parent->d_inode;
  2578. n = start_dir_add(dir);
  2579. __d_lookup_done(target);
  2580. }
  2581. write_seqcount_begin(&dentry->d_seq);
  2582. write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
  2583. /* unhash both */
  2584. if (!d_unhashed(dentry))
  2585. ___d_drop(dentry);
  2586. if (!d_unhashed(target))
  2587. ___d_drop(target);
  2588. /* ... and switch them in the tree */
  2589. dentry->d_parent = target->d_parent;
  2590. if (!exchange) {
  2591. copy_name(dentry, target);
  2592. target->d_hash.pprev = NULL;
  2593. dentry->d_parent->d_lockref.count++;
  2594. if (dentry != old_parent) /* wasn't IS_ROOT */
  2595. WARN_ON(!--old_parent->d_lockref.count);
  2596. } else {
  2597. target->d_parent = old_parent;
  2598. swap_names(dentry, target);
  2599. list_move(&target->d_child, &target->d_parent->d_subdirs);
  2600. __d_rehash(target);
  2601. fsnotify_update_flags(target);
  2602. }
  2603. list_move(&dentry->d_child, &dentry->d_parent->d_subdirs);
  2604. __d_rehash(dentry);
  2605. fsnotify_update_flags(dentry);
  2606. fscrypt_handle_d_move(dentry);
  2607. write_seqcount_end(&target->d_seq);
  2608. write_seqcount_end(&dentry->d_seq);
  2609. if (dir)
  2610. end_dir_add(dir, n);
  2611. if (dentry->d_parent != old_parent)
  2612. spin_unlock(&dentry->d_parent->d_lock);
  2613. if (dentry != old_parent)
  2614. spin_unlock(&old_parent->d_lock);
  2615. spin_unlock(&target->d_lock);
  2616. spin_unlock(&dentry->d_lock);
  2617. }
  2618. /*
  2619. * d_move - move a dentry
  2620. * @dentry: entry to move
  2621. * @target: new dentry
  2622. *
  2623. * Update the dcache to reflect the move of a file name. Negative
  2624. * dcache entries should not be moved in this way. See the locking
  2625. * requirements for __d_move.
  2626. */
  2627. void d_move(struct dentry *dentry, struct dentry *target)
  2628. {
  2629. write_seqlock(&rename_lock);
  2630. __d_move(dentry, target, false);
  2631. write_sequnlock(&rename_lock);
  2632. }
  2633. EXPORT_SYMBOL(d_move);
  2634. /*
  2635. * d_exchange - exchange two dentries
  2636. * @dentry1: first dentry
  2637. * @dentry2: second dentry
  2638. */
  2639. void d_exchange(struct dentry *dentry1, struct dentry *dentry2)
  2640. {
  2641. write_seqlock(&rename_lock);
  2642. WARN_ON(!dentry1->d_inode);
  2643. WARN_ON(!dentry2->d_inode);
  2644. WARN_ON(IS_ROOT(dentry1));
  2645. WARN_ON(IS_ROOT(dentry2));
  2646. __d_move(dentry1, dentry2, true);
  2647. write_sequnlock(&rename_lock);
  2648. }
  2649. /**
  2650. * d_ancestor - search for an ancestor
  2651. * @p1: ancestor dentry
  2652. * @p2: child dentry
  2653. *
  2654. * Returns the ancestor dentry of p2 which is a child of p1, if p1 is
  2655. * an ancestor of p2, else NULL.
  2656. */
  2657. struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
  2658. {
  2659. struct dentry *p;
  2660. for (p = p2; !IS_ROOT(p); p = p->d_parent) {
  2661. if (p->d_parent == p1)
  2662. return p;
  2663. }
  2664. return NULL;
  2665. }
  2666. /*
  2667. * This helper attempts to cope with remotely renamed directories
  2668. *
  2669. * It assumes that the caller is already holding
  2670. * dentry->d_parent->d_inode->i_mutex, and rename_lock
  2671. *
  2672. * Note: If ever the locking in lock_rename() changes, then please
  2673. * remember to update this too...
  2674. */
  2675. static int __d_unalias(struct inode *inode,
  2676. struct dentry *dentry, struct dentry *alias)
  2677. {
  2678. struct mutex *m1 = NULL;
  2679. struct rw_semaphore *m2 = NULL;
  2680. int ret = -ESTALE;
  2681. /* If alias and dentry share a parent, then no extra locks required */
  2682. if (alias->d_parent == dentry->d_parent)
  2683. goto out_unalias;
  2684. /* See lock_rename() */
  2685. if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
  2686. goto out_err;
  2687. m1 = &dentry->d_sb->s_vfs_rename_mutex;
  2688. if (!inode_trylock_shared(alias->d_parent->d_inode))
  2689. goto out_err;
  2690. m2 = &alias->d_parent->d_inode->i_rwsem;
  2691. out_unalias:
  2692. __d_move(alias, dentry, false);
  2693. ret = 0;
  2694. out_err:
  2695. if (m2)
  2696. up_read(m2);
  2697. if (m1)
  2698. mutex_unlock(m1);
  2699. return ret;
  2700. }
  2701. /**
  2702. * d_splice_alias - splice a disconnected dentry into the tree if one exists
  2703. * @inode: the inode which may have a disconnected dentry
  2704. * @dentry: a negative dentry which we want to point to the inode.
  2705. *
  2706. * If inode is a directory and has an IS_ROOT alias, then d_move that in
  2707. * place of the given dentry and return it, else simply d_add the inode
  2708. * to the dentry and return NULL.
  2709. *
  2710. * If a non-IS_ROOT directory is found, the filesystem is corrupt, and
  2711. * we should error out: directories can't have multiple aliases.
  2712. *
  2713. * This is needed in the lookup routine of any filesystem that is exportable
  2714. * (via knfsd) so that we can build dcache paths to directories effectively.
  2715. *
  2716. * If a dentry was found and moved, then it is returned. Otherwise NULL
  2717. * is returned. This matches the expected return value of ->lookup.
  2718. *
  2719. * Cluster filesystems may call this function with a negative, hashed dentry.
  2720. * In that case, we know that the inode will be a regular file, and also this
  2721. * will only occur during atomic_open. So we need to check for the dentry
  2722. * being already hashed only in the final case.
  2723. */
  2724. struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
  2725. {
  2726. if (IS_ERR(inode))
  2727. return ERR_CAST(inode);
  2728. BUG_ON(!d_unhashed(dentry));
  2729. if (!inode)
  2730. goto out;
  2731. security_d_instantiate(dentry, inode);
  2732. spin_lock(&inode->i_lock);
  2733. if (S_ISDIR(inode->i_mode)) {
  2734. struct dentry *new = __d_find_any_alias(inode);
  2735. if (unlikely(new)) {
  2736. /* The reference to new ensures it remains an alias */
  2737. spin_unlock(&inode->i_lock);
  2738. write_seqlock(&rename_lock);
  2739. if (unlikely(d_ancestor(new, dentry))) {
  2740. write_sequnlock(&rename_lock);
  2741. dput(new);
  2742. new = ERR_PTR(-ELOOP);
  2743. pr_warn_ratelimited(
  2744. "VFS: Lookup of '%s' in %s %s"
  2745. " would have caused loop\n",
  2746. dentry->d_name.name,
  2747. inode->i_sb->s_type->name,
  2748. inode->i_sb->s_id);
  2749. } else if (!IS_ROOT(new)) {
  2750. struct dentry *old_parent = dget(new->d_parent);
  2751. int err = __d_unalias(inode, dentry, new);
  2752. write_sequnlock(&rename_lock);
  2753. if (err) {
  2754. dput(new);
  2755. new = ERR_PTR(err);
  2756. }
  2757. dput(old_parent);
  2758. } else {
  2759. __d_move(new, dentry, false);
  2760. write_sequnlock(&rename_lock);
  2761. }
  2762. iput(inode);
  2763. return new;
  2764. }
  2765. }
  2766. out:
  2767. __d_add(dentry, inode);
  2768. return NULL;
  2769. }
  2770. EXPORT_SYMBOL_NS(d_splice_alias, ANDROID_GKI_VFS_EXPORT_ONLY);
  2771. /*
  2772. * Test whether new_dentry is a subdirectory of old_dentry.
  2773. *
  2774. * Trivially implemented using the dcache structure
  2775. */
  2776. /**
  2777. * is_subdir - is new dentry a subdirectory of old_dentry
  2778. * @new_dentry: new dentry
  2779. * @old_dentry: old dentry
  2780. *
  2781. * Returns true if new_dentry is a subdirectory of the parent (at any depth).
  2782. * Returns false otherwise.
  2783. * Caller must ensure that "new_dentry" is pinned before calling is_subdir()
  2784. */
  2785. bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
  2786. {
  2787. bool result;
  2788. unsigned seq;
  2789. if (new_dentry == old_dentry)
  2790. return true;
  2791. do {
  2792. /* for restarting inner loop in case of seq retry */
  2793. seq = read_seqbegin(&rename_lock);
  2794. /*
  2795. * Need rcu_readlock to protect against the d_parent trashing
  2796. * due to d_move
  2797. */
  2798. rcu_read_lock();
  2799. if (d_ancestor(old_dentry, new_dentry))
  2800. result = true;
  2801. else
  2802. result = false;
  2803. rcu_read_unlock();
  2804. } while (read_seqretry(&rename_lock, seq));
  2805. return result;
  2806. }
  2807. EXPORT_SYMBOL(is_subdir);
  2808. static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
  2809. {
  2810. struct dentry *root = data;
  2811. if (dentry != root) {
  2812. if (d_unhashed(dentry) || !dentry->d_inode)
  2813. return D_WALK_SKIP;
  2814. if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
  2815. dentry->d_flags |= DCACHE_GENOCIDE;
  2816. dentry->d_lockref.count--;
  2817. }
  2818. }
  2819. return D_WALK_CONTINUE;
  2820. }
  2821. void d_genocide(struct dentry *parent)
  2822. {
  2823. d_walk(parent, parent, d_genocide_kill);
  2824. }
  2825. EXPORT_SYMBOL(d_genocide);
  2826. void d_tmpfile(struct dentry *dentry, struct inode *inode)
  2827. {
  2828. inode_dec_link_count(inode);
  2829. BUG_ON(dentry->d_name.name != dentry->d_iname ||
  2830. !hlist_unhashed(&dentry->d_u.d_alias) ||
  2831. !d_unlinked(dentry));
  2832. spin_lock(&dentry->d_parent->d_lock);
  2833. spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
  2834. dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
  2835. (unsigned long long)inode->i_ino);
  2836. spin_unlock(&dentry->d_lock);
  2837. spin_unlock(&dentry->d_parent->d_lock);
  2838. d_instantiate(dentry, inode);
  2839. }
  2840. EXPORT_SYMBOL(d_tmpfile);
  2841. static __initdata unsigned long dhash_entries;
  2842. static int __init set_dhash_entries(char *str)
  2843. {
  2844. if (!str)
  2845. return 0;
  2846. dhash_entries = simple_strtoul(str, &str, 0);
  2847. return 1;
  2848. }
  2849. __setup("dhash_entries=", set_dhash_entries);
  2850. static void __init dcache_init_early(void)
  2851. {
  2852. /* If hashes are distributed across NUMA nodes, defer
  2853. * hash allocation until vmalloc space is available.
  2854. */
  2855. if (hashdist)
  2856. return;
  2857. dentry_hashtable =
  2858. alloc_large_system_hash("Dentry cache",
  2859. sizeof(struct hlist_bl_head),
  2860. dhash_entries,
  2861. 13,
  2862. HASH_EARLY | HASH_ZERO,
  2863. &d_hash_shift,
  2864. NULL,
  2865. 0,
  2866. 0);
  2867. d_hash_shift = 32 - d_hash_shift;
  2868. }
  2869. static void __init dcache_init(void)
  2870. {
  2871. /*
  2872. * A constructor could be added for stable state like the lists,
  2873. * but it is probably not worth it because of the cache nature
  2874. * of the dcache.
  2875. */
  2876. dentry_cache = KMEM_CACHE_USERCOPY(dentry,
  2877. SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD|SLAB_ACCOUNT,
  2878. d_iname);
  2879. /* Hash may have been set up in dcache_init_early */
  2880. if (!hashdist)
  2881. return;
  2882. dentry_hashtable =
  2883. alloc_large_system_hash("Dentry cache",
  2884. sizeof(struct hlist_bl_head),
  2885. dhash_entries,
  2886. 13,
  2887. HASH_ZERO,
  2888. &d_hash_shift,
  2889. NULL,
  2890. 0,
  2891. 0);
  2892. d_hash_shift = 32 - d_hash_shift;
  2893. }
  2894. /* SLAB cache for __getname() consumers */
  2895. struct kmem_cache *names_cachep __read_mostly;
  2896. EXPORT_SYMBOL(names_cachep);
  2897. void __init vfs_caches_init_early(void)
  2898. {
  2899. int i;
  2900. for (i = 0; i < ARRAY_SIZE(in_lookup_hashtable); i++)
  2901. INIT_HLIST_BL_HEAD(&in_lookup_hashtable[i]);
  2902. dcache_init_early();
  2903. inode_init_early();
  2904. }
  2905. void __init vfs_caches_init(void)
  2906. {
  2907. names_cachep = kmem_cache_create_usercopy("names_cache", PATH_MAX, 0,
  2908. SLAB_HWCACHE_ALIGN|SLAB_PANIC, 0, PATH_MAX, NULL);
  2909. dcache_init();
  2910. inode_init();
  2911. files_init();
  2912. files_maxfiles_init();
  2913. mnt_init();
  2914. bdev_cache_init();
  2915. chrdev_init();
  2916. }