xfs_icache.c 42 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2000-2005 Silicon Graphics, Inc.
  4. * All Rights Reserved.
  5. */
  6. #include "xfs.h"
  7. #include "xfs_fs.h"
  8. #include "xfs_shared.h"
  9. #include "xfs_format.h"
  10. #include "xfs_log_format.h"
  11. #include "xfs_trans_resv.h"
  12. #include "xfs_sb.h"
  13. #include "xfs_mount.h"
  14. #include "xfs_inode.h"
  15. #include "xfs_trans.h"
  16. #include "xfs_trans_priv.h"
  17. #include "xfs_inode_item.h"
  18. #include "xfs_quota.h"
  19. #include "xfs_trace.h"
  20. #include "xfs_icache.h"
  21. #include "xfs_bmap_util.h"
  22. #include "xfs_dquot_item.h"
  23. #include "xfs_dquot.h"
  24. #include "xfs_reflink.h"
  25. #include "xfs_ialloc.h"
  26. #include <linux/iversion.h>
  27. /*
  28. * Allocate and initialise an xfs_inode.
  29. */
  30. struct xfs_inode *
  31. xfs_inode_alloc(
  32. struct xfs_mount *mp,
  33. xfs_ino_t ino)
  34. {
  35. struct xfs_inode *ip;
  36. /*
  37. * XXX: If this didn't occur in transactions, we could drop GFP_NOFAIL
  38. * and return NULL here on ENOMEM.
  39. */
  40. ip = kmem_cache_alloc(xfs_inode_zone, GFP_KERNEL | __GFP_NOFAIL);
  41. if (inode_init_always(mp->m_super, VFS_I(ip))) {
  42. kmem_cache_free(xfs_inode_zone, ip);
  43. return NULL;
  44. }
  45. /* VFS doesn't initialise i_mode! */
  46. VFS_I(ip)->i_mode = 0;
  47. XFS_STATS_INC(mp, vn_active);
  48. ASSERT(atomic_read(&ip->i_pincount) == 0);
  49. ASSERT(ip->i_ino == 0);
  50. /* initialise the xfs inode */
  51. ip->i_ino = ino;
  52. ip->i_mount = mp;
  53. memset(&ip->i_imap, 0, sizeof(struct xfs_imap));
  54. ip->i_afp = NULL;
  55. ip->i_cowfp = NULL;
  56. memset(&ip->i_df, 0, sizeof(ip->i_df));
  57. ip->i_flags = 0;
  58. ip->i_delayed_blks = 0;
  59. memset(&ip->i_d, 0, sizeof(ip->i_d));
  60. ip->i_sick = 0;
  61. ip->i_checked = 0;
  62. INIT_WORK(&ip->i_ioend_work, xfs_end_io);
  63. INIT_LIST_HEAD(&ip->i_ioend_list);
  64. spin_lock_init(&ip->i_ioend_lock);
  65. return ip;
  66. }
  67. STATIC void
  68. xfs_inode_free_callback(
  69. struct rcu_head *head)
  70. {
  71. struct inode *inode = container_of(head, struct inode, i_rcu);
  72. struct xfs_inode *ip = XFS_I(inode);
  73. switch (VFS_I(ip)->i_mode & S_IFMT) {
  74. case S_IFREG:
  75. case S_IFDIR:
  76. case S_IFLNK:
  77. xfs_idestroy_fork(&ip->i_df);
  78. break;
  79. }
  80. if (ip->i_afp) {
  81. xfs_idestroy_fork(ip->i_afp);
  82. kmem_cache_free(xfs_ifork_zone, ip->i_afp);
  83. }
  84. if (ip->i_cowfp) {
  85. xfs_idestroy_fork(ip->i_cowfp);
  86. kmem_cache_free(xfs_ifork_zone, ip->i_cowfp);
  87. }
  88. if (ip->i_itemp) {
  89. ASSERT(!test_bit(XFS_LI_IN_AIL,
  90. &ip->i_itemp->ili_item.li_flags));
  91. xfs_inode_item_destroy(ip);
  92. ip->i_itemp = NULL;
  93. }
  94. kmem_cache_free(xfs_inode_zone, ip);
  95. }
  96. static void
  97. __xfs_inode_free(
  98. struct xfs_inode *ip)
  99. {
  100. /* asserts to verify all state is correct here */
  101. ASSERT(atomic_read(&ip->i_pincount) == 0);
  102. ASSERT(!ip->i_itemp || list_empty(&ip->i_itemp->ili_item.li_bio_list));
  103. XFS_STATS_DEC(ip->i_mount, vn_active);
  104. call_rcu(&VFS_I(ip)->i_rcu, xfs_inode_free_callback);
  105. }
  106. void
  107. xfs_inode_free(
  108. struct xfs_inode *ip)
  109. {
  110. ASSERT(!xfs_iflags_test(ip, XFS_IFLUSHING));
  111. /*
  112. * Because we use RCU freeing we need to ensure the inode always
  113. * appears to be reclaimed with an invalid inode number when in the
  114. * free state. The ip->i_flags_lock provides the barrier against lookup
  115. * races.
  116. */
  117. spin_lock(&ip->i_flags_lock);
  118. ip->i_flags = XFS_IRECLAIM;
  119. ip->i_ino = 0;
  120. spin_unlock(&ip->i_flags_lock);
  121. __xfs_inode_free(ip);
  122. }
  123. /*
  124. * Queue background inode reclaim work if there are reclaimable inodes and there
  125. * isn't reclaim work already scheduled or in progress.
  126. */
  127. static void
  128. xfs_reclaim_work_queue(
  129. struct xfs_mount *mp)
  130. {
  131. rcu_read_lock();
  132. if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_RECLAIM_TAG)) {
  133. queue_delayed_work(mp->m_reclaim_workqueue, &mp->m_reclaim_work,
  134. msecs_to_jiffies(xfs_syncd_centisecs / 6 * 10));
  135. }
  136. rcu_read_unlock();
  137. }
  138. static void
  139. xfs_perag_set_reclaim_tag(
  140. struct xfs_perag *pag)
  141. {
  142. struct xfs_mount *mp = pag->pag_mount;
  143. lockdep_assert_held(&pag->pag_ici_lock);
  144. if (pag->pag_ici_reclaimable++)
  145. return;
  146. /* propagate the reclaim tag up into the perag radix tree */
  147. spin_lock(&mp->m_perag_lock);
  148. radix_tree_tag_set(&mp->m_perag_tree, pag->pag_agno,
  149. XFS_ICI_RECLAIM_TAG);
  150. spin_unlock(&mp->m_perag_lock);
  151. /* schedule periodic background inode reclaim */
  152. xfs_reclaim_work_queue(mp);
  153. trace_xfs_perag_set_reclaim(mp, pag->pag_agno, -1, _RET_IP_);
  154. }
  155. static void
  156. xfs_perag_clear_reclaim_tag(
  157. struct xfs_perag *pag)
  158. {
  159. struct xfs_mount *mp = pag->pag_mount;
  160. lockdep_assert_held(&pag->pag_ici_lock);
  161. if (--pag->pag_ici_reclaimable)
  162. return;
  163. /* clear the reclaim tag from the perag radix tree */
  164. spin_lock(&mp->m_perag_lock);
  165. radix_tree_tag_clear(&mp->m_perag_tree, pag->pag_agno,
  166. XFS_ICI_RECLAIM_TAG);
  167. spin_unlock(&mp->m_perag_lock);
  168. trace_xfs_perag_clear_reclaim(mp, pag->pag_agno, -1, _RET_IP_);
  169. }
  170. /*
  171. * We set the inode flag atomically with the radix tree tag.
  172. * Once we get tag lookups on the radix tree, this inode flag
  173. * can go away.
  174. */
  175. void
  176. xfs_inode_set_reclaim_tag(
  177. struct xfs_inode *ip)
  178. {
  179. struct xfs_mount *mp = ip->i_mount;
  180. struct xfs_perag *pag;
  181. pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
  182. spin_lock(&pag->pag_ici_lock);
  183. spin_lock(&ip->i_flags_lock);
  184. radix_tree_tag_set(&pag->pag_ici_root, XFS_INO_TO_AGINO(mp, ip->i_ino),
  185. XFS_ICI_RECLAIM_TAG);
  186. xfs_perag_set_reclaim_tag(pag);
  187. __xfs_iflags_set(ip, XFS_IRECLAIMABLE);
  188. spin_unlock(&ip->i_flags_lock);
  189. spin_unlock(&pag->pag_ici_lock);
  190. xfs_perag_put(pag);
  191. }
  192. STATIC void
  193. xfs_inode_clear_reclaim_tag(
  194. struct xfs_perag *pag,
  195. xfs_ino_t ino)
  196. {
  197. radix_tree_tag_clear(&pag->pag_ici_root,
  198. XFS_INO_TO_AGINO(pag->pag_mount, ino),
  199. XFS_ICI_RECLAIM_TAG);
  200. xfs_perag_clear_reclaim_tag(pag);
  201. }
  202. static void
  203. xfs_inew_wait(
  204. struct xfs_inode *ip)
  205. {
  206. wait_queue_head_t *wq = bit_waitqueue(&ip->i_flags, __XFS_INEW_BIT);
  207. DEFINE_WAIT_BIT(wait, &ip->i_flags, __XFS_INEW_BIT);
  208. do {
  209. prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
  210. if (!xfs_iflags_test(ip, XFS_INEW))
  211. break;
  212. schedule();
  213. } while (true);
  214. finish_wait(wq, &wait.wq_entry);
  215. }
  216. /*
  217. * When we recycle a reclaimable inode, we need to re-initialise the VFS inode
  218. * part of the structure. This is made more complex by the fact we store
  219. * information about the on-disk values in the VFS inode and so we can't just
  220. * overwrite the values unconditionally. Hence we save the parameters we
  221. * need to retain across reinitialisation, and rewrite them into the VFS inode
  222. * after reinitialisation even if it fails.
  223. */
  224. static int
  225. xfs_reinit_inode(
  226. struct xfs_mount *mp,
  227. struct inode *inode)
  228. {
  229. int error;
  230. uint32_t nlink = inode->i_nlink;
  231. uint32_t generation = inode->i_generation;
  232. uint64_t version = inode_peek_iversion(inode);
  233. umode_t mode = inode->i_mode;
  234. dev_t dev = inode->i_rdev;
  235. kuid_t uid = inode->i_uid;
  236. kgid_t gid = inode->i_gid;
  237. error = inode_init_always(mp->m_super, inode);
  238. set_nlink(inode, nlink);
  239. inode->i_generation = generation;
  240. inode_set_iversion_queried(inode, version);
  241. inode->i_mode = mode;
  242. inode->i_rdev = dev;
  243. inode->i_uid = uid;
  244. inode->i_gid = gid;
  245. return error;
  246. }
  247. /*
  248. * If we are allocating a new inode, then check what was returned is
  249. * actually a free, empty inode. If we are not allocating an inode,
  250. * then check we didn't find a free inode.
  251. *
  252. * Returns:
  253. * 0 if the inode free state matches the lookup context
  254. * -ENOENT if the inode is free and we are not allocating
  255. * -EFSCORRUPTED if there is any state mismatch at all
  256. */
  257. static int
  258. xfs_iget_check_free_state(
  259. struct xfs_inode *ip,
  260. int flags)
  261. {
  262. if (flags & XFS_IGET_CREATE) {
  263. /* should be a free inode */
  264. if (VFS_I(ip)->i_mode != 0) {
  265. xfs_warn(ip->i_mount,
  266. "Corruption detected! Free inode 0x%llx not marked free! (mode 0x%x)",
  267. ip->i_ino, VFS_I(ip)->i_mode);
  268. return -EFSCORRUPTED;
  269. }
  270. if (ip->i_d.di_nblocks != 0) {
  271. xfs_warn(ip->i_mount,
  272. "Corruption detected! Free inode 0x%llx has blocks allocated!",
  273. ip->i_ino);
  274. return -EFSCORRUPTED;
  275. }
  276. return 0;
  277. }
  278. /* should be an allocated inode */
  279. if (VFS_I(ip)->i_mode == 0)
  280. return -ENOENT;
  281. return 0;
  282. }
  283. /*
  284. * Check the validity of the inode we just found it the cache
  285. */
  286. static int
  287. xfs_iget_cache_hit(
  288. struct xfs_perag *pag,
  289. struct xfs_inode *ip,
  290. xfs_ino_t ino,
  291. int flags,
  292. int lock_flags) __releases(RCU)
  293. {
  294. struct inode *inode = VFS_I(ip);
  295. struct xfs_mount *mp = ip->i_mount;
  296. int error;
  297. /*
  298. * check for re-use of an inode within an RCU grace period due to the
  299. * radix tree nodes not being updated yet. We monitor for this by
  300. * setting the inode number to zero before freeing the inode structure.
  301. * If the inode has been reallocated and set up, then the inode number
  302. * will not match, so check for that, too.
  303. */
  304. spin_lock(&ip->i_flags_lock);
  305. if (ip->i_ino != ino) {
  306. trace_xfs_iget_skip(ip);
  307. XFS_STATS_INC(mp, xs_ig_frecycle);
  308. error = -EAGAIN;
  309. goto out_error;
  310. }
  311. /*
  312. * If we are racing with another cache hit that is currently
  313. * instantiating this inode or currently recycling it out of
  314. * reclaimabe state, wait for the initialisation to complete
  315. * before continuing.
  316. *
  317. * XXX(hch): eventually we should do something equivalent to
  318. * wait_on_inode to wait for these flags to be cleared
  319. * instead of polling for it.
  320. */
  321. if (ip->i_flags & (XFS_INEW|XFS_IRECLAIM)) {
  322. trace_xfs_iget_skip(ip);
  323. XFS_STATS_INC(mp, xs_ig_frecycle);
  324. error = -EAGAIN;
  325. goto out_error;
  326. }
  327. /*
  328. * Check the inode free state is valid. This also detects lookup
  329. * racing with unlinks.
  330. */
  331. error = xfs_iget_check_free_state(ip, flags);
  332. if (error)
  333. goto out_error;
  334. /*
  335. * If IRECLAIMABLE is set, we've torn down the VFS inode already.
  336. * Need to carefully get it back into useable state.
  337. */
  338. if (ip->i_flags & XFS_IRECLAIMABLE) {
  339. trace_xfs_iget_reclaim(ip);
  340. if (flags & XFS_IGET_INCORE) {
  341. error = -EAGAIN;
  342. goto out_error;
  343. }
  344. /*
  345. * We need to set XFS_IRECLAIM to prevent xfs_reclaim_inode
  346. * from stomping over us while we recycle the inode. We can't
  347. * clear the radix tree reclaimable tag yet as it requires
  348. * pag_ici_lock to be held exclusive.
  349. */
  350. ip->i_flags |= XFS_IRECLAIM;
  351. spin_unlock(&ip->i_flags_lock);
  352. rcu_read_unlock();
  353. ASSERT(!rwsem_is_locked(&inode->i_rwsem));
  354. error = xfs_reinit_inode(mp, inode);
  355. if (error) {
  356. bool wake;
  357. /*
  358. * Re-initializing the inode failed, and we are in deep
  359. * trouble. Try to re-add it to the reclaim list.
  360. */
  361. rcu_read_lock();
  362. spin_lock(&ip->i_flags_lock);
  363. wake = !!__xfs_iflags_test(ip, XFS_INEW);
  364. ip->i_flags &= ~(XFS_INEW | XFS_IRECLAIM);
  365. if (wake)
  366. wake_up_bit(&ip->i_flags, __XFS_INEW_BIT);
  367. ASSERT(ip->i_flags & XFS_IRECLAIMABLE);
  368. trace_xfs_iget_reclaim_fail(ip);
  369. goto out_error;
  370. }
  371. spin_lock(&pag->pag_ici_lock);
  372. spin_lock(&ip->i_flags_lock);
  373. /*
  374. * Clear the per-lifetime state in the inode as we are now
  375. * effectively a new inode and need to return to the initial
  376. * state before reuse occurs.
  377. */
  378. ip->i_flags &= ~XFS_IRECLAIM_RESET_FLAGS;
  379. ip->i_flags |= XFS_INEW;
  380. xfs_inode_clear_reclaim_tag(pag, ip->i_ino);
  381. inode->i_state = I_NEW;
  382. ip->i_sick = 0;
  383. ip->i_checked = 0;
  384. spin_unlock(&ip->i_flags_lock);
  385. spin_unlock(&pag->pag_ici_lock);
  386. } else {
  387. /* If the VFS inode is being torn down, pause and try again. */
  388. if (!igrab(inode)) {
  389. trace_xfs_iget_skip(ip);
  390. error = -EAGAIN;
  391. goto out_error;
  392. }
  393. /* We've got a live one. */
  394. spin_unlock(&ip->i_flags_lock);
  395. rcu_read_unlock();
  396. trace_xfs_iget_hit(ip);
  397. }
  398. if (lock_flags != 0)
  399. xfs_ilock(ip, lock_flags);
  400. if (!(flags & XFS_IGET_INCORE))
  401. xfs_iflags_clear(ip, XFS_ISTALE);
  402. XFS_STATS_INC(mp, xs_ig_found);
  403. return 0;
  404. out_error:
  405. spin_unlock(&ip->i_flags_lock);
  406. rcu_read_unlock();
  407. return error;
  408. }
  409. static int
  410. xfs_iget_cache_miss(
  411. struct xfs_mount *mp,
  412. struct xfs_perag *pag,
  413. xfs_trans_t *tp,
  414. xfs_ino_t ino,
  415. struct xfs_inode **ipp,
  416. int flags,
  417. int lock_flags)
  418. {
  419. struct xfs_inode *ip;
  420. int error;
  421. xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ino);
  422. int iflags;
  423. ip = xfs_inode_alloc(mp, ino);
  424. if (!ip)
  425. return -ENOMEM;
  426. error = xfs_imap(mp, tp, ip->i_ino, &ip->i_imap, flags);
  427. if (error)
  428. goto out_destroy;
  429. /*
  430. * For version 5 superblocks, if we are initialising a new inode and we
  431. * are not utilising the XFS_MOUNT_IKEEP inode cluster mode, we can
  432. * simply build the new inode core with a random generation number.
  433. *
  434. * For version 4 (and older) superblocks, log recovery is dependent on
  435. * the di_flushiter field being initialised from the current on-disk
  436. * value and hence we must also read the inode off disk even when
  437. * initializing new inodes.
  438. */
  439. if (xfs_sb_version_has_v3inode(&mp->m_sb) &&
  440. (flags & XFS_IGET_CREATE) && !(mp->m_flags & XFS_MOUNT_IKEEP)) {
  441. VFS_I(ip)->i_generation = prandom_u32();
  442. } else {
  443. struct xfs_dinode *dip;
  444. struct xfs_buf *bp;
  445. error = xfs_imap_to_bp(mp, tp, &ip->i_imap, &dip, &bp, 0);
  446. if (error)
  447. goto out_destroy;
  448. error = xfs_inode_from_disk(ip, dip);
  449. if (!error)
  450. xfs_buf_set_ref(bp, XFS_INO_REF);
  451. xfs_trans_brelse(tp, bp);
  452. if (error)
  453. goto out_destroy;
  454. }
  455. trace_xfs_iget_miss(ip);
  456. /*
  457. * Check the inode free state is valid. This also detects lookup
  458. * racing with unlinks.
  459. */
  460. error = xfs_iget_check_free_state(ip, flags);
  461. if (error)
  462. goto out_destroy;
  463. /*
  464. * Preload the radix tree so we can insert safely under the
  465. * write spinlock. Note that we cannot sleep inside the preload
  466. * region. Since we can be called from transaction context, don't
  467. * recurse into the file system.
  468. */
  469. if (radix_tree_preload(GFP_NOFS)) {
  470. error = -EAGAIN;
  471. goto out_destroy;
  472. }
  473. /*
  474. * Because the inode hasn't been added to the radix-tree yet it can't
  475. * be found by another thread, so we can do the non-sleeping lock here.
  476. */
  477. if (lock_flags) {
  478. if (!xfs_ilock_nowait(ip, lock_flags))
  479. BUG();
  480. }
  481. /*
  482. * These values must be set before inserting the inode into the radix
  483. * tree as the moment it is inserted a concurrent lookup (allowed by the
  484. * RCU locking mechanism) can find it and that lookup must see that this
  485. * is an inode currently under construction (i.e. that XFS_INEW is set).
  486. * The ip->i_flags_lock that protects the XFS_INEW flag forms the
  487. * memory barrier that ensures this detection works correctly at lookup
  488. * time.
  489. */
  490. iflags = XFS_INEW;
  491. if (flags & XFS_IGET_DONTCACHE)
  492. d_mark_dontcache(VFS_I(ip));
  493. ip->i_udquot = NULL;
  494. ip->i_gdquot = NULL;
  495. ip->i_pdquot = NULL;
  496. xfs_iflags_set(ip, iflags);
  497. /* insert the new inode */
  498. spin_lock(&pag->pag_ici_lock);
  499. error = radix_tree_insert(&pag->pag_ici_root, agino, ip);
  500. if (unlikely(error)) {
  501. WARN_ON(error != -EEXIST);
  502. XFS_STATS_INC(mp, xs_ig_dup);
  503. error = -EAGAIN;
  504. goto out_preload_end;
  505. }
  506. spin_unlock(&pag->pag_ici_lock);
  507. radix_tree_preload_end();
  508. *ipp = ip;
  509. return 0;
  510. out_preload_end:
  511. spin_unlock(&pag->pag_ici_lock);
  512. radix_tree_preload_end();
  513. if (lock_flags)
  514. xfs_iunlock(ip, lock_flags);
  515. out_destroy:
  516. __destroy_inode(VFS_I(ip));
  517. xfs_inode_free(ip);
  518. return error;
  519. }
  520. /*
  521. * Look up an inode by number in the given file system. The inode is looked up
  522. * in the cache held in each AG. If the inode is found in the cache, initialise
  523. * the vfs inode if necessary.
  524. *
  525. * If it is not in core, read it in from the file system's device, add it to the
  526. * cache and initialise the vfs inode.
  527. *
  528. * The inode is locked according to the value of the lock_flags parameter.
  529. * Inode lookup is only done during metadata operations and not as part of the
  530. * data IO path. Hence we only allow locking of the XFS_ILOCK during lookup.
  531. */
  532. int
  533. xfs_iget(
  534. struct xfs_mount *mp,
  535. struct xfs_trans *tp,
  536. xfs_ino_t ino,
  537. uint flags,
  538. uint lock_flags,
  539. struct xfs_inode **ipp)
  540. {
  541. struct xfs_inode *ip;
  542. struct xfs_perag *pag;
  543. xfs_agino_t agino;
  544. int error;
  545. ASSERT((lock_flags & (XFS_IOLOCK_EXCL | XFS_IOLOCK_SHARED)) == 0);
  546. /* reject inode numbers outside existing AGs */
  547. if (!ino || XFS_INO_TO_AGNO(mp, ino) >= mp->m_sb.sb_agcount)
  548. return -EINVAL;
  549. XFS_STATS_INC(mp, xs_ig_attempts);
  550. /* get the perag structure and ensure that it's inode capable */
  551. pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ino));
  552. agino = XFS_INO_TO_AGINO(mp, ino);
  553. again:
  554. error = 0;
  555. rcu_read_lock();
  556. ip = radix_tree_lookup(&pag->pag_ici_root, agino);
  557. if (ip) {
  558. error = xfs_iget_cache_hit(pag, ip, ino, flags, lock_flags);
  559. if (error)
  560. goto out_error_or_again;
  561. } else {
  562. rcu_read_unlock();
  563. if (flags & XFS_IGET_INCORE) {
  564. error = -ENODATA;
  565. goto out_error_or_again;
  566. }
  567. XFS_STATS_INC(mp, xs_ig_missed);
  568. error = xfs_iget_cache_miss(mp, pag, tp, ino, &ip,
  569. flags, lock_flags);
  570. if (error)
  571. goto out_error_or_again;
  572. }
  573. xfs_perag_put(pag);
  574. *ipp = ip;
  575. /*
  576. * If we have a real type for an on-disk inode, we can setup the inode
  577. * now. If it's a new inode being created, xfs_ialloc will handle it.
  578. */
  579. if (xfs_iflags_test(ip, XFS_INEW) && VFS_I(ip)->i_mode != 0)
  580. xfs_setup_existing_inode(ip);
  581. return 0;
  582. out_error_or_again:
  583. if (!(flags & XFS_IGET_INCORE) && error == -EAGAIN) {
  584. delay(1);
  585. goto again;
  586. }
  587. xfs_perag_put(pag);
  588. return error;
  589. }
  590. /*
  591. * "Is this a cached inode that's also allocated?"
  592. *
  593. * Look up an inode by number in the given file system. If the inode is
  594. * in cache and isn't in purgatory, return 1 if the inode is allocated
  595. * and 0 if it is not. For all other cases (not in cache, being torn
  596. * down, etc.), return a negative error code.
  597. *
  598. * The caller has to prevent inode allocation and freeing activity,
  599. * presumably by locking the AGI buffer. This is to ensure that an
  600. * inode cannot transition from allocated to freed until the caller is
  601. * ready to allow that. If the inode is in an intermediate state (new,
  602. * reclaimable, or being reclaimed), -EAGAIN will be returned; if the
  603. * inode is not in the cache, -ENOENT will be returned. The caller must
  604. * deal with these scenarios appropriately.
  605. *
  606. * This is a specialized use case for the online scrubber; if you're
  607. * reading this, you probably want xfs_iget.
  608. */
  609. int
  610. xfs_icache_inode_is_allocated(
  611. struct xfs_mount *mp,
  612. struct xfs_trans *tp,
  613. xfs_ino_t ino,
  614. bool *inuse)
  615. {
  616. struct xfs_inode *ip;
  617. int error;
  618. error = xfs_iget(mp, tp, ino, XFS_IGET_INCORE, 0, &ip);
  619. if (error)
  620. return error;
  621. *inuse = !!(VFS_I(ip)->i_mode);
  622. xfs_irele(ip);
  623. return 0;
  624. }
  625. /*
  626. * The inode lookup is done in batches to keep the amount of lock traffic and
  627. * radix tree lookups to a minimum. The batch size is a trade off between
  628. * lookup reduction and stack usage. This is in the reclaim path, so we can't
  629. * be too greedy.
  630. */
  631. #define XFS_LOOKUP_BATCH 32
  632. /*
  633. * Decide if the given @ip is eligible to be a part of the inode walk, and
  634. * grab it if so. Returns true if it's ready to go or false if we should just
  635. * ignore it.
  636. */
  637. STATIC bool
  638. xfs_inode_walk_ag_grab(
  639. struct xfs_inode *ip,
  640. int flags)
  641. {
  642. struct inode *inode = VFS_I(ip);
  643. bool newinos = !!(flags & XFS_INODE_WALK_INEW_WAIT);
  644. ASSERT(rcu_read_lock_held());
  645. /* Check for stale RCU freed inode */
  646. spin_lock(&ip->i_flags_lock);
  647. if (!ip->i_ino)
  648. goto out_unlock_noent;
  649. /* avoid new or reclaimable inodes. Leave for reclaim code to flush */
  650. if ((!newinos && __xfs_iflags_test(ip, XFS_INEW)) ||
  651. __xfs_iflags_test(ip, XFS_IRECLAIMABLE | XFS_IRECLAIM))
  652. goto out_unlock_noent;
  653. spin_unlock(&ip->i_flags_lock);
  654. /* nothing to sync during shutdown */
  655. if (XFS_FORCED_SHUTDOWN(ip->i_mount))
  656. return false;
  657. /* If we can't grab the inode, it must on it's way to reclaim. */
  658. if (!igrab(inode))
  659. return false;
  660. /* inode is valid */
  661. return true;
  662. out_unlock_noent:
  663. spin_unlock(&ip->i_flags_lock);
  664. return false;
  665. }
  666. /*
  667. * For a given per-AG structure @pag, grab, @execute, and rele all incore
  668. * inodes with the given radix tree @tag.
  669. */
  670. STATIC int
  671. xfs_inode_walk_ag(
  672. struct xfs_perag *pag,
  673. int iter_flags,
  674. int (*execute)(struct xfs_inode *ip, void *args),
  675. void *args,
  676. int tag)
  677. {
  678. struct xfs_mount *mp = pag->pag_mount;
  679. uint32_t first_index;
  680. int last_error = 0;
  681. int skipped;
  682. bool done;
  683. int nr_found;
  684. restart:
  685. done = false;
  686. skipped = 0;
  687. first_index = 0;
  688. nr_found = 0;
  689. do {
  690. struct xfs_inode *batch[XFS_LOOKUP_BATCH];
  691. int error = 0;
  692. int i;
  693. rcu_read_lock();
  694. if (tag == XFS_ICI_NO_TAG)
  695. nr_found = radix_tree_gang_lookup(&pag->pag_ici_root,
  696. (void **)batch, first_index,
  697. XFS_LOOKUP_BATCH);
  698. else
  699. nr_found = radix_tree_gang_lookup_tag(
  700. &pag->pag_ici_root,
  701. (void **) batch, first_index,
  702. XFS_LOOKUP_BATCH, tag);
  703. if (!nr_found) {
  704. rcu_read_unlock();
  705. break;
  706. }
  707. /*
  708. * Grab the inodes before we drop the lock. if we found
  709. * nothing, nr == 0 and the loop will be skipped.
  710. */
  711. for (i = 0; i < nr_found; i++) {
  712. struct xfs_inode *ip = batch[i];
  713. if (done || !xfs_inode_walk_ag_grab(ip, iter_flags))
  714. batch[i] = NULL;
  715. /*
  716. * Update the index for the next lookup. Catch
  717. * overflows into the next AG range which can occur if
  718. * we have inodes in the last block of the AG and we
  719. * are currently pointing to the last inode.
  720. *
  721. * Because we may see inodes that are from the wrong AG
  722. * due to RCU freeing and reallocation, only update the
  723. * index if it lies in this AG. It was a race that lead
  724. * us to see this inode, so another lookup from the
  725. * same index will not find it again.
  726. */
  727. if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag->pag_agno)
  728. continue;
  729. first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1);
  730. if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino))
  731. done = true;
  732. }
  733. /* unlock now we've grabbed the inodes. */
  734. rcu_read_unlock();
  735. for (i = 0; i < nr_found; i++) {
  736. if (!batch[i])
  737. continue;
  738. if ((iter_flags & XFS_INODE_WALK_INEW_WAIT) &&
  739. xfs_iflags_test(batch[i], XFS_INEW))
  740. xfs_inew_wait(batch[i]);
  741. error = execute(batch[i], args);
  742. xfs_irele(batch[i]);
  743. if (error == -EAGAIN) {
  744. skipped++;
  745. continue;
  746. }
  747. if (error && last_error != -EFSCORRUPTED)
  748. last_error = error;
  749. }
  750. /* bail out if the filesystem is corrupted. */
  751. if (error == -EFSCORRUPTED)
  752. break;
  753. cond_resched();
  754. } while (nr_found && !done);
  755. if (skipped) {
  756. delay(1);
  757. goto restart;
  758. }
  759. return last_error;
  760. }
  761. /* Fetch the next (possibly tagged) per-AG structure. */
  762. static inline struct xfs_perag *
  763. xfs_inode_walk_get_perag(
  764. struct xfs_mount *mp,
  765. xfs_agnumber_t agno,
  766. int tag)
  767. {
  768. if (tag == XFS_ICI_NO_TAG)
  769. return xfs_perag_get(mp, agno);
  770. return xfs_perag_get_tag(mp, agno, tag);
  771. }
  772. /*
  773. * Call the @execute function on all incore inodes matching the radix tree
  774. * @tag.
  775. */
  776. int
  777. xfs_inode_walk(
  778. struct xfs_mount *mp,
  779. int iter_flags,
  780. int (*execute)(struct xfs_inode *ip, void *args),
  781. void *args,
  782. int tag)
  783. {
  784. struct xfs_perag *pag;
  785. int error = 0;
  786. int last_error = 0;
  787. xfs_agnumber_t ag;
  788. ag = 0;
  789. while ((pag = xfs_inode_walk_get_perag(mp, ag, tag))) {
  790. ag = pag->pag_agno + 1;
  791. error = xfs_inode_walk_ag(pag, iter_flags, execute, args, tag);
  792. xfs_perag_put(pag);
  793. if (error) {
  794. last_error = error;
  795. if (error == -EFSCORRUPTED)
  796. break;
  797. }
  798. }
  799. return last_error;
  800. }
  801. /*
  802. * Background scanning to trim post-EOF preallocated space. This is queued
  803. * based on the 'speculative_prealloc_lifetime' tunable (5m by default).
  804. */
  805. void
  806. xfs_queue_eofblocks(
  807. struct xfs_mount *mp)
  808. {
  809. rcu_read_lock();
  810. if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_EOFBLOCKS_TAG))
  811. queue_delayed_work(mp->m_eofblocks_workqueue,
  812. &mp->m_eofblocks_work,
  813. msecs_to_jiffies(xfs_eofb_secs * 1000));
  814. rcu_read_unlock();
  815. }
  816. void
  817. xfs_eofblocks_worker(
  818. struct work_struct *work)
  819. {
  820. struct xfs_mount *mp = container_of(to_delayed_work(work),
  821. struct xfs_mount, m_eofblocks_work);
  822. if (!sb_start_write_trylock(mp->m_super))
  823. return;
  824. xfs_icache_free_eofblocks(mp, NULL);
  825. sb_end_write(mp->m_super);
  826. xfs_queue_eofblocks(mp);
  827. }
  828. /*
  829. * Background scanning to trim preallocated CoW space. This is queued
  830. * based on the 'speculative_cow_prealloc_lifetime' tunable (5m by default).
  831. * (We'll just piggyback on the post-EOF prealloc space workqueue.)
  832. */
  833. void
  834. xfs_queue_cowblocks(
  835. struct xfs_mount *mp)
  836. {
  837. rcu_read_lock();
  838. if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_COWBLOCKS_TAG))
  839. queue_delayed_work(mp->m_eofblocks_workqueue,
  840. &mp->m_cowblocks_work,
  841. msecs_to_jiffies(xfs_cowb_secs * 1000));
  842. rcu_read_unlock();
  843. }
  844. void
  845. xfs_cowblocks_worker(
  846. struct work_struct *work)
  847. {
  848. struct xfs_mount *mp = container_of(to_delayed_work(work),
  849. struct xfs_mount, m_cowblocks_work);
  850. if (!sb_start_write_trylock(mp->m_super))
  851. return;
  852. xfs_icache_free_cowblocks(mp, NULL);
  853. sb_end_write(mp->m_super);
  854. xfs_queue_cowblocks(mp);
  855. }
  856. /*
  857. * Grab the inode for reclaim exclusively.
  858. *
  859. * We have found this inode via a lookup under RCU, so the inode may have
  860. * already been freed, or it may be in the process of being recycled by
  861. * xfs_iget(). In both cases, the inode will have XFS_IRECLAIM set. If the inode
  862. * has been fully recycled by the time we get the i_flags_lock, XFS_IRECLAIMABLE
  863. * will not be set. Hence we need to check for both these flag conditions to
  864. * avoid inodes that are no longer reclaim candidates.
  865. *
  866. * Note: checking for other state flags here, under the i_flags_lock or not, is
  867. * racy and should be avoided. Those races should be resolved only after we have
  868. * ensured that we are able to reclaim this inode and the world can see that we
  869. * are going to reclaim it.
  870. *
  871. * Return true if we grabbed it, false otherwise.
  872. */
  873. static bool
  874. xfs_reclaim_inode_grab(
  875. struct xfs_inode *ip)
  876. {
  877. ASSERT(rcu_read_lock_held());
  878. spin_lock(&ip->i_flags_lock);
  879. if (!__xfs_iflags_test(ip, XFS_IRECLAIMABLE) ||
  880. __xfs_iflags_test(ip, XFS_IRECLAIM)) {
  881. /* not a reclaim candidate. */
  882. spin_unlock(&ip->i_flags_lock);
  883. return false;
  884. }
  885. __xfs_iflags_set(ip, XFS_IRECLAIM);
  886. spin_unlock(&ip->i_flags_lock);
  887. return true;
  888. }
  889. /*
  890. * Inode reclaim is non-blocking, so the default action if progress cannot be
  891. * made is to "requeue" the inode for reclaim by unlocking it and clearing the
  892. * XFS_IRECLAIM flag. If we are in a shutdown state, we don't care about
  893. * blocking anymore and hence we can wait for the inode to be able to reclaim
  894. * it.
  895. *
  896. * We do no IO here - if callers require inodes to be cleaned they must push the
  897. * AIL first to trigger writeback of dirty inodes. This enables writeback to be
  898. * done in the background in a non-blocking manner, and enables memory reclaim
  899. * to make progress without blocking.
  900. */
  901. static void
  902. xfs_reclaim_inode(
  903. struct xfs_inode *ip,
  904. struct xfs_perag *pag)
  905. {
  906. xfs_ino_t ino = ip->i_ino; /* for radix_tree_delete */
  907. if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL))
  908. goto out;
  909. if (xfs_iflags_test_and_set(ip, XFS_IFLUSHING))
  910. goto out_iunlock;
  911. if (XFS_FORCED_SHUTDOWN(ip->i_mount)) {
  912. xfs_iunpin_wait(ip);
  913. xfs_iflush_abort(ip);
  914. goto reclaim;
  915. }
  916. if (xfs_ipincount(ip))
  917. goto out_clear_flush;
  918. if (!xfs_inode_clean(ip))
  919. goto out_clear_flush;
  920. xfs_iflags_clear(ip, XFS_IFLUSHING);
  921. reclaim:
  922. /*
  923. * Because we use RCU freeing we need to ensure the inode always appears
  924. * to be reclaimed with an invalid inode number when in the free state.
  925. * We do this as early as possible under the ILOCK so that
  926. * xfs_iflush_cluster() and xfs_ifree_cluster() can be guaranteed to
  927. * detect races with us here. By doing this, we guarantee that once
  928. * xfs_iflush_cluster() or xfs_ifree_cluster() has locked XFS_ILOCK that
  929. * it will see either a valid inode that will serialise correctly, or it
  930. * will see an invalid inode that it can skip.
  931. */
  932. spin_lock(&ip->i_flags_lock);
  933. ip->i_flags = XFS_IRECLAIM;
  934. ip->i_ino = 0;
  935. spin_unlock(&ip->i_flags_lock);
  936. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  937. XFS_STATS_INC(ip->i_mount, xs_ig_reclaims);
  938. /*
  939. * Remove the inode from the per-AG radix tree.
  940. *
  941. * Because radix_tree_delete won't complain even if the item was never
  942. * added to the tree assert that it's been there before to catch
  943. * problems with the inode life time early on.
  944. */
  945. spin_lock(&pag->pag_ici_lock);
  946. if (!radix_tree_delete(&pag->pag_ici_root,
  947. XFS_INO_TO_AGINO(ip->i_mount, ino)))
  948. ASSERT(0);
  949. xfs_perag_clear_reclaim_tag(pag);
  950. spin_unlock(&pag->pag_ici_lock);
  951. /*
  952. * Here we do an (almost) spurious inode lock in order to coordinate
  953. * with inode cache radix tree lookups. This is because the lookup
  954. * can reference the inodes in the cache without taking references.
  955. *
  956. * We make that OK here by ensuring that we wait until the inode is
  957. * unlocked after the lookup before we go ahead and free it.
  958. */
  959. xfs_ilock(ip, XFS_ILOCK_EXCL);
  960. xfs_qm_dqdetach(ip);
  961. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  962. ASSERT(xfs_inode_clean(ip));
  963. __xfs_inode_free(ip);
  964. return;
  965. out_clear_flush:
  966. xfs_iflags_clear(ip, XFS_IFLUSHING);
  967. out_iunlock:
  968. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  969. out:
  970. xfs_iflags_clear(ip, XFS_IRECLAIM);
  971. }
  972. /*
  973. * Walk the AGs and reclaim the inodes in them. Even if the filesystem is
  974. * corrupted, we still want to try to reclaim all the inodes. If we don't,
  975. * then a shut down during filesystem unmount reclaim walk leak all the
  976. * unreclaimed inodes.
  977. *
  978. * Returns non-zero if any AGs or inodes were skipped in the reclaim pass
  979. * so that callers that want to block until all dirty inodes are written back
  980. * and reclaimed can sanely loop.
  981. */
  982. static void
  983. xfs_reclaim_inodes_ag(
  984. struct xfs_mount *mp,
  985. int *nr_to_scan)
  986. {
  987. struct xfs_perag *pag;
  988. xfs_agnumber_t ag = 0;
  989. while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) {
  990. unsigned long first_index = 0;
  991. int done = 0;
  992. int nr_found = 0;
  993. ag = pag->pag_agno + 1;
  994. first_index = READ_ONCE(pag->pag_ici_reclaim_cursor);
  995. do {
  996. struct xfs_inode *batch[XFS_LOOKUP_BATCH];
  997. int i;
  998. rcu_read_lock();
  999. nr_found = radix_tree_gang_lookup_tag(
  1000. &pag->pag_ici_root,
  1001. (void **)batch, first_index,
  1002. XFS_LOOKUP_BATCH,
  1003. XFS_ICI_RECLAIM_TAG);
  1004. if (!nr_found) {
  1005. done = 1;
  1006. rcu_read_unlock();
  1007. break;
  1008. }
  1009. /*
  1010. * Grab the inodes before we drop the lock. if we found
  1011. * nothing, nr == 0 and the loop will be skipped.
  1012. */
  1013. for (i = 0; i < nr_found; i++) {
  1014. struct xfs_inode *ip = batch[i];
  1015. if (done || !xfs_reclaim_inode_grab(ip))
  1016. batch[i] = NULL;
  1017. /*
  1018. * Update the index for the next lookup. Catch
  1019. * overflows into the next AG range which can
  1020. * occur if we have inodes in the last block of
  1021. * the AG and we are currently pointing to the
  1022. * last inode.
  1023. *
  1024. * Because we may see inodes that are from the
  1025. * wrong AG due to RCU freeing and
  1026. * reallocation, only update the index if it
  1027. * lies in this AG. It was a race that lead us
  1028. * to see this inode, so another lookup from
  1029. * the same index will not find it again.
  1030. */
  1031. if (XFS_INO_TO_AGNO(mp, ip->i_ino) !=
  1032. pag->pag_agno)
  1033. continue;
  1034. first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1);
  1035. if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino))
  1036. done = 1;
  1037. }
  1038. /* unlock now we've grabbed the inodes. */
  1039. rcu_read_unlock();
  1040. for (i = 0; i < nr_found; i++) {
  1041. if (batch[i])
  1042. xfs_reclaim_inode(batch[i], pag);
  1043. }
  1044. *nr_to_scan -= XFS_LOOKUP_BATCH;
  1045. cond_resched();
  1046. } while (nr_found && !done && *nr_to_scan > 0);
  1047. if (done)
  1048. first_index = 0;
  1049. WRITE_ONCE(pag->pag_ici_reclaim_cursor, first_index);
  1050. xfs_perag_put(pag);
  1051. }
  1052. }
  1053. void
  1054. xfs_reclaim_inodes(
  1055. struct xfs_mount *mp)
  1056. {
  1057. int nr_to_scan = INT_MAX;
  1058. while (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_RECLAIM_TAG)) {
  1059. xfs_ail_push_all_sync(mp->m_ail);
  1060. xfs_reclaim_inodes_ag(mp, &nr_to_scan);
  1061. }
  1062. }
  1063. /*
  1064. * The shrinker infrastructure determines how many inodes we should scan for
  1065. * reclaim. We want as many clean inodes ready to reclaim as possible, so we
  1066. * push the AIL here. We also want to proactively free up memory if we can to
  1067. * minimise the amount of work memory reclaim has to do so we kick the
  1068. * background reclaim if it isn't already scheduled.
  1069. */
  1070. long
  1071. xfs_reclaim_inodes_nr(
  1072. struct xfs_mount *mp,
  1073. int nr_to_scan)
  1074. {
  1075. /* kick background reclaimer and push the AIL */
  1076. xfs_reclaim_work_queue(mp);
  1077. xfs_ail_push_all(mp->m_ail);
  1078. xfs_reclaim_inodes_ag(mp, &nr_to_scan);
  1079. return 0;
  1080. }
  1081. /*
  1082. * Return the number of reclaimable inodes in the filesystem for
  1083. * the shrinker to determine how much to reclaim.
  1084. */
  1085. int
  1086. xfs_reclaim_inodes_count(
  1087. struct xfs_mount *mp)
  1088. {
  1089. struct xfs_perag *pag;
  1090. xfs_agnumber_t ag = 0;
  1091. int reclaimable = 0;
  1092. while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) {
  1093. ag = pag->pag_agno + 1;
  1094. reclaimable += pag->pag_ici_reclaimable;
  1095. xfs_perag_put(pag);
  1096. }
  1097. return reclaimable;
  1098. }
  1099. STATIC bool
  1100. xfs_inode_match_id(
  1101. struct xfs_inode *ip,
  1102. struct xfs_eofblocks *eofb)
  1103. {
  1104. if ((eofb->eof_flags & XFS_EOF_FLAGS_UID) &&
  1105. !uid_eq(VFS_I(ip)->i_uid, eofb->eof_uid))
  1106. return false;
  1107. if ((eofb->eof_flags & XFS_EOF_FLAGS_GID) &&
  1108. !gid_eq(VFS_I(ip)->i_gid, eofb->eof_gid))
  1109. return false;
  1110. if ((eofb->eof_flags & XFS_EOF_FLAGS_PRID) &&
  1111. ip->i_d.di_projid != eofb->eof_prid)
  1112. return false;
  1113. return true;
  1114. }
  1115. /*
  1116. * A union-based inode filtering algorithm. Process the inode if any of the
  1117. * criteria match. This is for global/internal scans only.
  1118. */
  1119. STATIC bool
  1120. xfs_inode_match_id_union(
  1121. struct xfs_inode *ip,
  1122. struct xfs_eofblocks *eofb)
  1123. {
  1124. if ((eofb->eof_flags & XFS_EOF_FLAGS_UID) &&
  1125. uid_eq(VFS_I(ip)->i_uid, eofb->eof_uid))
  1126. return true;
  1127. if ((eofb->eof_flags & XFS_EOF_FLAGS_GID) &&
  1128. gid_eq(VFS_I(ip)->i_gid, eofb->eof_gid))
  1129. return true;
  1130. if ((eofb->eof_flags & XFS_EOF_FLAGS_PRID) &&
  1131. ip->i_d.di_projid == eofb->eof_prid)
  1132. return true;
  1133. return false;
  1134. }
  1135. /*
  1136. * Is this inode @ip eligible for eof/cow block reclamation, given some
  1137. * filtering parameters @eofb? The inode is eligible if @eofb is null or
  1138. * if the predicate functions match.
  1139. */
  1140. static bool
  1141. xfs_inode_matches_eofb(
  1142. struct xfs_inode *ip,
  1143. struct xfs_eofblocks *eofb)
  1144. {
  1145. bool match;
  1146. if (!eofb)
  1147. return true;
  1148. if (eofb->eof_flags & XFS_EOF_FLAGS_UNION)
  1149. match = xfs_inode_match_id_union(ip, eofb);
  1150. else
  1151. match = xfs_inode_match_id(ip, eofb);
  1152. if (!match)
  1153. return false;
  1154. /* skip the inode if the file size is too small */
  1155. if ((eofb->eof_flags & XFS_EOF_FLAGS_MINFILESIZE) &&
  1156. XFS_ISIZE(ip) < eofb->eof_min_file_size)
  1157. return false;
  1158. return true;
  1159. }
  1160. /*
  1161. * This is a fast pass over the inode cache to try to get reclaim moving on as
  1162. * many inodes as possible in a short period of time. It kicks itself every few
  1163. * seconds, as well as being kicked by the inode cache shrinker when memory
  1164. * goes low.
  1165. */
  1166. void
  1167. xfs_reclaim_worker(
  1168. struct work_struct *work)
  1169. {
  1170. struct xfs_mount *mp = container_of(to_delayed_work(work),
  1171. struct xfs_mount, m_reclaim_work);
  1172. int nr_to_scan = INT_MAX;
  1173. xfs_reclaim_inodes_ag(mp, &nr_to_scan);
  1174. xfs_reclaim_work_queue(mp);
  1175. }
  1176. STATIC int
  1177. xfs_inode_free_eofblocks(
  1178. struct xfs_inode *ip,
  1179. void *args)
  1180. {
  1181. struct xfs_eofblocks *eofb = args;
  1182. bool wait;
  1183. int ret;
  1184. wait = eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC);
  1185. if (!xfs_can_free_eofblocks(ip, false)) {
  1186. /* inode could be preallocated or append-only */
  1187. trace_xfs_inode_free_eofblocks_invalid(ip);
  1188. xfs_inode_clear_eofblocks_tag(ip);
  1189. return 0;
  1190. }
  1191. /*
  1192. * If the mapping is dirty the operation can block and wait for some
  1193. * time. Unless we are waiting, skip it.
  1194. */
  1195. if (!wait && mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY))
  1196. return 0;
  1197. if (!xfs_inode_matches_eofb(ip, eofb))
  1198. return 0;
  1199. /*
  1200. * If the caller is waiting, return -EAGAIN to keep the background
  1201. * scanner moving and revisit the inode in a subsequent pass.
  1202. */
  1203. if (!xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) {
  1204. if (wait)
  1205. return -EAGAIN;
  1206. return 0;
  1207. }
  1208. ret = xfs_free_eofblocks(ip);
  1209. xfs_iunlock(ip, XFS_IOLOCK_EXCL);
  1210. return ret;
  1211. }
  1212. int
  1213. xfs_icache_free_eofblocks(
  1214. struct xfs_mount *mp,
  1215. struct xfs_eofblocks *eofb)
  1216. {
  1217. return xfs_inode_walk(mp, 0, xfs_inode_free_eofblocks, eofb,
  1218. XFS_ICI_EOFBLOCKS_TAG);
  1219. }
  1220. /*
  1221. * Run eofblocks scans on the quotas applicable to the inode. For inodes with
  1222. * multiple quotas, we don't know exactly which quota caused an allocation
  1223. * failure. We make a best effort by including each quota under low free space
  1224. * conditions (less than 1% free space) in the scan.
  1225. */
  1226. static int
  1227. __xfs_inode_free_quota_eofblocks(
  1228. struct xfs_inode *ip,
  1229. int (*execute)(struct xfs_mount *mp,
  1230. struct xfs_eofblocks *eofb))
  1231. {
  1232. int scan = 0;
  1233. struct xfs_eofblocks eofb = {0};
  1234. struct xfs_dquot *dq;
  1235. /*
  1236. * Run a sync scan to increase effectiveness and use the union filter to
  1237. * cover all applicable quotas in a single scan.
  1238. */
  1239. eofb.eof_flags = XFS_EOF_FLAGS_UNION|XFS_EOF_FLAGS_SYNC;
  1240. if (XFS_IS_UQUOTA_ENFORCED(ip->i_mount)) {
  1241. dq = xfs_inode_dquot(ip, XFS_DQTYPE_USER);
  1242. if (dq && xfs_dquot_lowsp(dq)) {
  1243. eofb.eof_uid = VFS_I(ip)->i_uid;
  1244. eofb.eof_flags |= XFS_EOF_FLAGS_UID;
  1245. scan = 1;
  1246. }
  1247. }
  1248. if (XFS_IS_GQUOTA_ENFORCED(ip->i_mount)) {
  1249. dq = xfs_inode_dquot(ip, XFS_DQTYPE_GROUP);
  1250. if (dq && xfs_dquot_lowsp(dq)) {
  1251. eofb.eof_gid = VFS_I(ip)->i_gid;
  1252. eofb.eof_flags |= XFS_EOF_FLAGS_GID;
  1253. scan = 1;
  1254. }
  1255. }
  1256. if (scan)
  1257. execute(ip->i_mount, &eofb);
  1258. return scan;
  1259. }
  1260. int
  1261. xfs_inode_free_quota_eofblocks(
  1262. struct xfs_inode *ip)
  1263. {
  1264. return __xfs_inode_free_quota_eofblocks(ip, xfs_icache_free_eofblocks);
  1265. }
  1266. static inline unsigned long
  1267. xfs_iflag_for_tag(
  1268. int tag)
  1269. {
  1270. switch (tag) {
  1271. case XFS_ICI_EOFBLOCKS_TAG:
  1272. return XFS_IEOFBLOCKS;
  1273. case XFS_ICI_COWBLOCKS_TAG:
  1274. return XFS_ICOWBLOCKS;
  1275. default:
  1276. ASSERT(0);
  1277. return 0;
  1278. }
  1279. }
  1280. static void
  1281. __xfs_inode_set_blocks_tag(
  1282. xfs_inode_t *ip,
  1283. void (*execute)(struct xfs_mount *mp),
  1284. void (*set_tp)(struct xfs_mount *mp, xfs_agnumber_t agno,
  1285. int error, unsigned long caller_ip),
  1286. int tag)
  1287. {
  1288. struct xfs_mount *mp = ip->i_mount;
  1289. struct xfs_perag *pag;
  1290. int tagged;
  1291. /*
  1292. * Don't bother locking the AG and looking up in the radix trees
  1293. * if we already know that we have the tag set.
  1294. */
  1295. if (ip->i_flags & xfs_iflag_for_tag(tag))
  1296. return;
  1297. spin_lock(&ip->i_flags_lock);
  1298. ip->i_flags |= xfs_iflag_for_tag(tag);
  1299. spin_unlock(&ip->i_flags_lock);
  1300. pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
  1301. spin_lock(&pag->pag_ici_lock);
  1302. tagged = radix_tree_tagged(&pag->pag_ici_root, tag);
  1303. radix_tree_tag_set(&pag->pag_ici_root,
  1304. XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), tag);
  1305. if (!tagged) {
  1306. /* propagate the eofblocks tag up into the perag radix tree */
  1307. spin_lock(&ip->i_mount->m_perag_lock);
  1308. radix_tree_tag_set(&ip->i_mount->m_perag_tree,
  1309. XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino),
  1310. tag);
  1311. spin_unlock(&ip->i_mount->m_perag_lock);
  1312. /* kick off background trimming */
  1313. execute(ip->i_mount);
  1314. set_tp(ip->i_mount, pag->pag_agno, -1, _RET_IP_);
  1315. }
  1316. spin_unlock(&pag->pag_ici_lock);
  1317. xfs_perag_put(pag);
  1318. }
  1319. void
  1320. xfs_inode_set_eofblocks_tag(
  1321. xfs_inode_t *ip)
  1322. {
  1323. trace_xfs_inode_set_eofblocks_tag(ip);
  1324. return __xfs_inode_set_blocks_tag(ip, xfs_queue_eofblocks,
  1325. trace_xfs_perag_set_eofblocks,
  1326. XFS_ICI_EOFBLOCKS_TAG);
  1327. }
  1328. static void
  1329. __xfs_inode_clear_blocks_tag(
  1330. xfs_inode_t *ip,
  1331. void (*clear_tp)(struct xfs_mount *mp, xfs_agnumber_t agno,
  1332. int error, unsigned long caller_ip),
  1333. int tag)
  1334. {
  1335. struct xfs_mount *mp = ip->i_mount;
  1336. struct xfs_perag *pag;
  1337. spin_lock(&ip->i_flags_lock);
  1338. ip->i_flags &= ~xfs_iflag_for_tag(tag);
  1339. spin_unlock(&ip->i_flags_lock);
  1340. pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
  1341. spin_lock(&pag->pag_ici_lock);
  1342. radix_tree_tag_clear(&pag->pag_ici_root,
  1343. XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), tag);
  1344. if (!radix_tree_tagged(&pag->pag_ici_root, tag)) {
  1345. /* clear the eofblocks tag from the perag radix tree */
  1346. spin_lock(&ip->i_mount->m_perag_lock);
  1347. radix_tree_tag_clear(&ip->i_mount->m_perag_tree,
  1348. XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino),
  1349. tag);
  1350. spin_unlock(&ip->i_mount->m_perag_lock);
  1351. clear_tp(ip->i_mount, pag->pag_agno, -1, _RET_IP_);
  1352. }
  1353. spin_unlock(&pag->pag_ici_lock);
  1354. xfs_perag_put(pag);
  1355. }
  1356. void
  1357. xfs_inode_clear_eofblocks_tag(
  1358. xfs_inode_t *ip)
  1359. {
  1360. trace_xfs_inode_clear_eofblocks_tag(ip);
  1361. return __xfs_inode_clear_blocks_tag(ip,
  1362. trace_xfs_perag_clear_eofblocks, XFS_ICI_EOFBLOCKS_TAG);
  1363. }
  1364. /*
  1365. * Set ourselves up to free CoW blocks from this file. If it's already clean
  1366. * then we can bail out quickly, but otherwise we must back off if the file
  1367. * is undergoing some kind of write.
  1368. */
  1369. static bool
  1370. xfs_prep_free_cowblocks(
  1371. struct xfs_inode *ip)
  1372. {
  1373. /*
  1374. * Just clear the tag if we have an empty cow fork or none at all. It's
  1375. * possible the inode was fully unshared since it was originally tagged.
  1376. */
  1377. if (!xfs_inode_has_cow_data(ip)) {
  1378. trace_xfs_inode_free_cowblocks_invalid(ip);
  1379. xfs_inode_clear_cowblocks_tag(ip);
  1380. return false;
  1381. }
  1382. /*
  1383. * If the mapping is dirty or under writeback we cannot touch the
  1384. * CoW fork. Leave it alone if we're in the midst of a directio.
  1385. */
  1386. if ((VFS_I(ip)->i_state & I_DIRTY_PAGES) ||
  1387. mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY) ||
  1388. mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_WRITEBACK) ||
  1389. atomic_read(&VFS_I(ip)->i_dio_count))
  1390. return false;
  1391. return true;
  1392. }
  1393. /*
  1394. * Automatic CoW Reservation Freeing
  1395. *
  1396. * These functions automatically garbage collect leftover CoW reservations
  1397. * that were made on behalf of a cowextsize hint when we start to run out
  1398. * of quota or when the reservations sit around for too long. If the file
  1399. * has dirty pages or is undergoing writeback, its CoW reservations will
  1400. * be retained.
  1401. *
  1402. * The actual garbage collection piggybacks off the same code that runs
  1403. * the speculative EOF preallocation garbage collector.
  1404. */
  1405. STATIC int
  1406. xfs_inode_free_cowblocks(
  1407. struct xfs_inode *ip,
  1408. void *args)
  1409. {
  1410. struct xfs_eofblocks *eofb = args;
  1411. int ret = 0;
  1412. if (!xfs_prep_free_cowblocks(ip))
  1413. return 0;
  1414. if (!xfs_inode_matches_eofb(ip, eofb))
  1415. return 0;
  1416. /* Free the CoW blocks */
  1417. xfs_ilock(ip, XFS_IOLOCK_EXCL);
  1418. xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
  1419. /*
  1420. * Check again, nobody else should be able to dirty blocks or change
  1421. * the reflink iflag now that we have the first two locks held.
  1422. */
  1423. if (xfs_prep_free_cowblocks(ip))
  1424. ret = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF, false);
  1425. xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
  1426. xfs_iunlock(ip, XFS_IOLOCK_EXCL);
  1427. return ret;
  1428. }
  1429. int
  1430. xfs_icache_free_cowblocks(
  1431. struct xfs_mount *mp,
  1432. struct xfs_eofblocks *eofb)
  1433. {
  1434. return xfs_inode_walk(mp, 0, xfs_inode_free_cowblocks, eofb,
  1435. XFS_ICI_COWBLOCKS_TAG);
  1436. }
  1437. int
  1438. xfs_inode_free_quota_cowblocks(
  1439. struct xfs_inode *ip)
  1440. {
  1441. return __xfs_inode_free_quota_eofblocks(ip, xfs_icache_free_cowblocks);
  1442. }
  1443. void
  1444. xfs_inode_set_cowblocks_tag(
  1445. xfs_inode_t *ip)
  1446. {
  1447. trace_xfs_inode_set_cowblocks_tag(ip);
  1448. return __xfs_inode_set_blocks_tag(ip, xfs_queue_cowblocks,
  1449. trace_xfs_perag_set_cowblocks,
  1450. XFS_ICI_COWBLOCKS_TAG);
  1451. }
  1452. void
  1453. xfs_inode_clear_cowblocks_tag(
  1454. xfs_inode_t *ip)
  1455. {
  1456. trace_xfs_inode_clear_cowblocks_tag(ip);
  1457. return __xfs_inode_clear_blocks_tag(ip,
  1458. trace_xfs_perag_clear_cowblocks, XFS_ICI_COWBLOCKS_TAG);
  1459. }
  1460. /* Disable post-EOF and CoW block auto-reclamation. */
  1461. void
  1462. xfs_stop_block_reaping(
  1463. struct xfs_mount *mp)
  1464. {
  1465. cancel_delayed_work_sync(&mp->m_eofblocks_work);
  1466. cancel_delayed_work_sync(&mp->m_cowblocks_work);
  1467. }
  1468. /* Enable post-EOF and CoW block auto-reclamation. */
  1469. void
  1470. xfs_start_block_reaping(
  1471. struct xfs_mount *mp)
  1472. {
  1473. xfs_queue_eofblocks(mp);
  1474. xfs_queue_cowblocks(mp);
  1475. }