xfs_iwalk.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright (C) 2019 Oracle. All Rights Reserved.
  4. * Author: Darrick J. Wong <darrick.wong@oracle.com>
  5. */
  6. #include "xfs.h"
  7. #include "xfs_fs.h"
  8. #include "xfs_shared.h"
  9. #include "xfs_format.h"
  10. #include "xfs_log_format.h"
  11. #include "xfs_trans_resv.h"
  12. #include "xfs_mount.h"
  13. #include "xfs_inode.h"
  14. #include "xfs_btree.h"
  15. #include "xfs_ialloc.h"
  16. #include "xfs_ialloc_btree.h"
  17. #include "xfs_iwalk.h"
  18. #include "xfs_error.h"
  19. #include "xfs_trace.h"
  20. #include "xfs_icache.h"
  21. #include "xfs_health.h"
  22. #include "xfs_trans.h"
  23. #include "xfs_pwork.h"
  24. /*
  25. * Walking Inodes in the Filesystem
  26. * ================================
  27. *
  28. * This iterator function walks a subset of filesystem inodes in increasing
  29. * order from @startino until there are no more inodes. For each allocated
  30. * inode it finds, it calls a walk function with the relevant inode number and
  31. * a pointer to caller-provided data. The walk function can return the usual
  32. * negative error code to stop the iteration; 0 to continue the iteration; or
  33. * -ECANCELED to stop the iteration. This return value is returned to the
  34. * caller.
  35. *
  36. * Internally, we allow the walk function to do anything, which means that we
  37. * cannot maintain the inobt cursor or our lock on the AGI buffer. We
  38. * therefore cache the inobt records in kernel memory and only call the walk
  39. * function when our memory buffer is full. @nr_recs is the number of records
  40. * that we've cached, and @sz_recs is the size of our cache.
  41. *
  42. * It is the responsibility of the walk function to ensure it accesses
  43. * allocated inodes, as the inobt records may be stale by the time they are
  44. * acted upon.
  45. */
  46. struct xfs_iwalk_ag {
  47. /* parallel work control data; will be null if single threaded */
  48. struct xfs_pwork pwork;
  49. struct xfs_mount *mp;
  50. struct xfs_trans *tp;
  51. /* Where do we start the traversal? */
  52. xfs_ino_t startino;
  53. /* What was the last inode number we saw when iterating the inobt? */
  54. xfs_ino_t lastino;
  55. /* Array of inobt records we cache. */
  56. struct xfs_inobt_rec_incore *recs;
  57. /* Number of entries allocated for the @recs array. */
  58. unsigned int sz_recs;
  59. /* Number of entries in the @recs array that are in use. */
  60. unsigned int nr_recs;
  61. /* Inode walk function and data pointer. */
  62. xfs_iwalk_fn iwalk_fn;
  63. xfs_inobt_walk_fn inobt_walk_fn;
  64. void *data;
  65. /*
  66. * Make it look like the inodes up to startino are free so that
  67. * bulkstat can start its inode iteration at the correct place without
  68. * needing to special case everywhere.
  69. */
  70. unsigned int trim_start:1;
  71. /* Skip empty inobt records? */
  72. unsigned int skip_empty:1;
  73. };
  74. /*
  75. * Loop over all clusters in a chunk for a given incore inode allocation btree
  76. * record. Do a readahead if there are any allocated inodes in that cluster.
  77. */
  78. STATIC void
  79. xfs_iwalk_ichunk_ra(
  80. struct xfs_mount *mp,
  81. xfs_agnumber_t agno,
  82. struct xfs_inobt_rec_incore *irec)
  83. {
  84. struct xfs_ino_geometry *igeo = M_IGEO(mp);
  85. xfs_agblock_t agbno;
  86. struct blk_plug plug;
  87. int i; /* inode chunk index */
  88. agbno = XFS_AGINO_TO_AGBNO(mp, irec->ir_startino);
  89. blk_start_plug(&plug);
  90. for (i = 0; i < XFS_INODES_PER_CHUNK; i += igeo->inodes_per_cluster) {
  91. xfs_inofree_t imask;
  92. imask = xfs_inobt_maskn(i, igeo->inodes_per_cluster);
  93. if (imask & ~irec->ir_free) {
  94. xfs_btree_reada_bufs(mp, agno, agbno,
  95. igeo->blocks_per_cluster,
  96. &xfs_inode_buf_ops);
  97. }
  98. agbno += igeo->blocks_per_cluster;
  99. }
  100. blk_finish_plug(&plug);
  101. }
  102. /*
  103. * Set the bits in @irec's free mask that correspond to the inodes before
  104. * @agino so that we skip them. This is how we restart an inode walk that was
  105. * interrupted in the middle of an inode record.
  106. */
  107. STATIC void
  108. xfs_iwalk_adjust_start(
  109. xfs_agino_t agino, /* starting inode of chunk */
  110. struct xfs_inobt_rec_incore *irec) /* btree record */
  111. {
  112. int idx; /* index into inode chunk */
  113. int i;
  114. idx = agino - irec->ir_startino;
  115. /*
  116. * We got a right chunk with some left inodes allocated at it. Grab
  117. * the chunk record. Mark all the uninteresting inodes free because
  118. * they're before our start point.
  119. */
  120. for (i = 0; i < idx; i++) {
  121. if (XFS_INOBT_MASK(i) & ~irec->ir_free)
  122. irec->ir_freecount++;
  123. }
  124. irec->ir_free |= xfs_inobt_maskn(0, idx);
  125. }
  126. /* Allocate memory for a walk. */
  127. STATIC int
  128. xfs_iwalk_alloc(
  129. struct xfs_iwalk_ag *iwag)
  130. {
  131. size_t size;
  132. ASSERT(iwag->recs == NULL);
  133. iwag->nr_recs = 0;
  134. /* Allocate a prefetch buffer for inobt records. */
  135. size = iwag->sz_recs * sizeof(struct xfs_inobt_rec_incore);
  136. iwag->recs = kmem_alloc(size, KM_MAYFAIL);
  137. if (iwag->recs == NULL)
  138. return -ENOMEM;
  139. return 0;
  140. }
  141. /* Free memory we allocated for a walk. */
  142. STATIC void
  143. xfs_iwalk_free(
  144. struct xfs_iwalk_ag *iwag)
  145. {
  146. kmem_free(iwag->recs);
  147. iwag->recs = NULL;
  148. }
  149. /* For each inuse inode in each cached inobt record, call our function. */
  150. STATIC int
  151. xfs_iwalk_ag_recs(
  152. struct xfs_iwalk_ag *iwag)
  153. {
  154. struct xfs_mount *mp = iwag->mp;
  155. struct xfs_trans *tp = iwag->tp;
  156. xfs_ino_t ino;
  157. unsigned int i, j;
  158. xfs_agnumber_t agno;
  159. int error;
  160. agno = XFS_INO_TO_AGNO(mp, iwag->startino);
  161. for (i = 0; i < iwag->nr_recs; i++) {
  162. struct xfs_inobt_rec_incore *irec = &iwag->recs[i];
  163. trace_xfs_iwalk_ag_rec(mp, agno, irec);
  164. if (xfs_pwork_want_abort(&iwag->pwork))
  165. return 0;
  166. if (iwag->inobt_walk_fn) {
  167. error = iwag->inobt_walk_fn(mp, tp, agno, irec,
  168. iwag->data);
  169. if (error)
  170. return error;
  171. }
  172. if (!iwag->iwalk_fn)
  173. continue;
  174. for (j = 0; j < XFS_INODES_PER_CHUNK; j++) {
  175. if (xfs_pwork_want_abort(&iwag->pwork))
  176. return 0;
  177. /* Skip if this inode is free */
  178. if (XFS_INOBT_MASK(j) & irec->ir_free)
  179. continue;
  180. /* Otherwise call our function. */
  181. ino = XFS_AGINO_TO_INO(mp, agno, irec->ir_startino + j);
  182. error = iwag->iwalk_fn(mp, tp, ino, iwag->data);
  183. if (error)
  184. return error;
  185. }
  186. }
  187. return 0;
  188. }
  189. /* Delete cursor and let go of AGI. */
  190. static inline void
  191. xfs_iwalk_del_inobt(
  192. struct xfs_trans *tp,
  193. struct xfs_btree_cur **curpp,
  194. struct xfs_buf **agi_bpp,
  195. int error)
  196. {
  197. if (*curpp) {
  198. xfs_btree_del_cursor(*curpp, error);
  199. *curpp = NULL;
  200. }
  201. if (*agi_bpp) {
  202. xfs_trans_brelse(tp, *agi_bpp);
  203. *agi_bpp = NULL;
  204. }
  205. }
  206. /*
  207. * Set ourselves up for walking inobt records starting from a given point in
  208. * the filesystem.
  209. *
  210. * If caller passed in a nonzero start inode number, load the record from the
  211. * inobt and make the record look like all the inodes before agino are free so
  212. * that we skip them, and then move the cursor to the next inobt record. This
  213. * is how we support starting an iwalk in the middle of an inode chunk.
  214. *
  215. * If the caller passed in a start number of zero, move the cursor to the first
  216. * inobt record.
  217. *
  218. * The caller is responsible for cleaning up the cursor and buffer pointer
  219. * regardless of the error status.
  220. */
  221. STATIC int
  222. xfs_iwalk_ag_start(
  223. struct xfs_iwalk_ag *iwag,
  224. xfs_agnumber_t agno,
  225. xfs_agino_t agino,
  226. struct xfs_btree_cur **curpp,
  227. struct xfs_buf **agi_bpp,
  228. int *has_more)
  229. {
  230. struct xfs_mount *mp = iwag->mp;
  231. struct xfs_trans *tp = iwag->tp;
  232. struct xfs_inobt_rec_incore *irec;
  233. int error;
  234. /* Set up a fresh cursor and empty the inobt cache. */
  235. iwag->nr_recs = 0;
  236. error = xfs_inobt_cur(mp, tp, agno, XFS_BTNUM_INO, curpp, agi_bpp);
  237. if (error)
  238. return error;
  239. /* Starting at the beginning of the AG? That's easy! */
  240. if (agino == 0)
  241. return xfs_inobt_lookup(*curpp, 0, XFS_LOOKUP_GE, has_more);
  242. /*
  243. * Otherwise, we have to grab the inobt record where we left off, stuff
  244. * the record into our cache, and then see if there are more records.
  245. * We require a lookup cache of at least two elements so that the
  246. * caller doesn't have to deal with tearing down the cursor to walk the
  247. * records.
  248. */
  249. error = xfs_inobt_lookup(*curpp, agino, XFS_LOOKUP_LE, has_more);
  250. if (error)
  251. return error;
  252. /*
  253. * If the LE lookup at @agino yields no records, jump ahead to the
  254. * inobt cursor increment to see if there are more records to process.
  255. */
  256. if (!*has_more)
  257. goto out_advance;
  258. /* Get the record, should always work */
  259. irec = &iwag->recs[iwag->nr_recs];
  260. error = xfs_inobt_get_rec(*curpp, irec, has_more);
  261. if (error)
  262. return error;
  263. if (XFS_IS_CORRUPT(mp, *has_more != 1))
  264. return -EFSCORRUPTED;
  265. iwag->lastino = XFS_AGINO_TO_INO(mp, agno,
  266. irec->ir_startino + XFS_INODES_PER_CHUNK - 1);
  267. /*
  268. * If the LE lookup yielded an inobt record before the cursor position,
  269. * skip it and see if there's another one after it.
  270. */
  271. if (irec->ir_startino + XFS_INODES_PER_CHUNK <= agino)
  272. goto out_advance;
  273. /*
  274. * If agino fell in the middle of the inode record, make it look like
  275. * the inodes up to agino are free so that we don't return them again.
  276. */
  277. if (iwag->trim_start)
  278. xfs_iwalk_adjust_start(agino, irec);
  279. /*
  280. * The prefetch calculation is supposed to give us a large enough inobt
  281. * record cache that grab_ichunk can stage a partial first record and
  282. * the loop body can cache a record without having to check for cache
  283. * space until after it reads an inobt record.
  284. */
  285. iwag->nr_recs++;
  286. ASSERT(iwag->nr_recs < iwag->sz_recs);
  287. out_advance:
  288. return xfs_btree_increment(*curpp, 0, has_more);
  289. }
  290. /*
  291. * The inobt record cache is full, so preserve the inobt cursor state and
  292. * run callbacks on the cached inobt records. When we're done, restore the
  293. * cursor state to wherever the cursor would have been had the cache not been
  294. * full (and therefore we could've just incremented the cursor) if *@has_more
  295. * is true. On exit, *@has_more will indicate whether or not the caller should
  296. * try for more inode records.
  297. */
  298. STATIC int
  299. xfs_iwalk_run_callbacks(
  300. struct xfs_iwalk_ag *iwag,
  301. xfs_agnumber_t agno,
  302. struct xfs_btree_cur **curpp,
  303. struct xfs_buf **agi_bpp,
  304. int *has_more)
  305. {
  306. struct xfs_mount *mp = iwag->mp;
  307. struct xfs_trans *tp = iwag->tp;
  308. struct xfs_inobt_rec_incore *irec;
  309. xfs_agino_t next_agino;
  310. int error;
  311. next_agino = XFS_INO_TO_AGINO(mp, iwag->lastino) + 1;
  312. ASSERT(iwag->nr_recs > 0);
  313. /* Delete cursor but remember the last record we cached... */
  314. xfs_iwalk_del_inobt(tp, curpp, agi_bpp, 0);
  315. irec = &iwag->recs[iwag->nr_recs - 1];
  316. ASSERT(next_agino == irec->ir_startino + XFS_INODES_PER_CHUNK);
  317. error = xfs_iwalk_ag_recs(iwag);
  318. if (error)
  319. return error;
  320. /* ...empty the cache... */
  321. iwag->nr_recs = 0;
  322. if (!has_more)
  323. return 0;
  324. /* ...and recreate the cursor just past where we left off. */
  325. error = xfs_inobt_cur(mp, tp, agno, XFS_BTNUM_INO, curpp, agi_bpp);
  326. if (error)
  327. return error;
  328. return xfs_inobt_lookup(*curpp, next_agino, XFS_LOOKUP_GE, has_more);
  329. }
  330. /* Walk all inodes in a single AG, from @iwag->startino to the end of the AG. */
  331. STATIC int
  332. xfs_iwalk_ag(
  333. struct xfs_iwalk_ag *iwag)
  334. {
  335. struct xfs_mount *mp = iwag->mp;
  336. struct xfs_trans *tp = iwag->tp;
  337. struct xfs_buf *agi_bp = NULL;
  338. struct xfs_btree_cur *cur = NULL;
  339. xfs_agnumber_t agno;
  340. xfs_agino_t agino;
  341. int has_more;
  342. int error = 0;
  343. /* Set up our cursor at the right place in the inode btree. */
  344. agno = XFS_INO_TO_AGNO(mp, iwag->startino);
  345. agino = XFS_INO_TO_AGINO(mp, iwag->startino);
  346. error = xfs_iwalk_ag_start(iwag, agno, agino, &cur, &agi_bp, &has_more);
  347. while (!error && has_more) {
  348. struct xfs_inobt_rec_incore *irec;
  349. xfs_ino_t rec_fsino;
  350. cond_resched();
  351. if (xfs_pwork_want_abort(&iwag->pwork))
  352. goto out;
  353. /* Fetch the inobt record. */
  354. irec = &iwag->recs[iwag->nr_recs];
  355. error = xfs_inobt_get_rec(cur, irec, &has_more);
  356. if (error || !has_more)
  357. break;
  358. /* Make sure that we always move forward. */
  359. rec_fsino = XFS_AGINO_TO_INO(mp, agno, irec->ir_startino);
  360. if (iwag->lastino != NULLFSINO &&
  361. XFS_IS_CORRUPT(mp, iwag->lastino >= rec_fsino)) {
  362. error = -EFSCORRUPTED;
  363. goto out;
  364. }
  365. iwag->lastino = rec_fsino + XFS_INODES_PER_CHUNK - 1;
  366. /* No allocated inodes in this chunk; skip it. */
  367. if (iwag->skip_empty && irec->ir_freecount == irec->ir_count) {
  368. error = xfs_btree_increment(cur, 0, &has_more);
  369. if (error)
  370. break;
  371. continue;
  372. }
  373. /*
  374. * Start readahead for this inode chunk in anticipation of
  375. * walking the inodes.
  376. */
  377. if (iwag->iwalk_fn)
  378. xfs_iwalk_ichunk_ra(mp, agno, irec);
  379. /*
  380. * If there's space in the buffer for more records, increment
  381. * the btree cursor and grab more.
  382. */
  383. if (++iwag->nr_recs < iwag->sz_recs) {
  384. error = xfs_btree_increment(cur, 0, &has_more);
  385. if (error || !has_more)
  386. break;
  387. continue;
  388. }
  389. /*
  390. * Otherwise, we need to save cursor state and run the callback
  391. * function on the cached records. The run_callbacks function
  392. * is supposed to return a cursor pointing to the record where
  393. * we would be if we had been able to increment like above.
  394. */
  395. ASSERT(has_more);
  396. error = xfs_iwalk_run_callbacks(iwag, agno, &cur, &agi_bp,
  397. &has_more);
  398. }
  399. if (iwag->nr_recs == 0 || error)
  400. goto out;
  401. /* Walk the unprocessed records in the cache. */
  402. error = xfs_iwalk_run_callbacks(iwag, agno, &cur, &agi_bp, &has_more);
  403. out:
  404. xfs_iwalk_del_inobt(tp, &cur, &agi_bp, error);
  405. return error;
  406. }
  407. /*
  408. * We experimentally determined that the reduction in ioctl call overhead
  409. * diminishes when userspace asks for more than 2048 inodes, so we'll cap
  410. * prefetch at this point.
  411. */
  412. #define IWALK_MAX_INODE_PREFETCH (2048U)
  413. /*
  414. * Given the number of inodes to prefetch, set the number of inobt records that
  415. * we cache in memory, which controls the number of inodes we try to read
  416. * ahead. Set the maximum if @inodes == 0.
  417. */
  418. static inline unsigned int
  419. xfs_iwalk_prefetch(
  420. unsigned int inodes)
  421. {
  422. unsigned int inobt_records;
  423. /*
  424. * If the caller didn't tell us the number of inodes they wanted,
  425. * assume the maximum prefetch possible for best performance.
  426. * Otherwise, cap prefetch at that maximum so that we don't start an
  427. * absurd amount of prefetch.
  428. */
  429. if (inodes == 0)
  430. inodes = IWALK_MAX_INODE_PREFETCH;
  431. inodes = min(inodes, IWALK_MAX_INODE_PREFETCH);
  432. /* Round the inode count up to a full chunk. */
  433. inodes = round_up(inodes, XFS_INODES_PER_CHUNK);
  434. /*
  435. * In order to convert the number of inodes to prefetch into an
  436. * estimate of the number of inobt records to cache, we require a
  437. * conversion factor that reflects our expectations of the average
  438. * loading factor of an inode chunk. Based on data gathered, most
  439. * (but not all) filesystems manage to keep the inode chunks totally
  440. * full, so we'll underestimate slightly so that our readahead will
  441. * still deliver the performance we want on aging filesystems:
  442. *
  443. * inobt = inodes / (INODES_PER_CHUNK * (4 / 5));
  444. *
  445. * The funny math is to avoid integer division.
  446. */
  447. inobt_records = (inodes * 5) / (4 * XFS_INODES_PER_CHUNK);
  448. /*
  449. * Allocate enough space to prefetch at least two inobt records so that
  450. * we can cache both the record where the iwalk started and the next
  451. * record. This simplifies the AG inode walk loop setup code.
  452. */
  453. return max(inobt_records, 2U);
  454. }
  455. /*
  456. * Walk all inodes in the filesystem starting from @startino. The @iwalk_fn
  457. * will be called for each allocated inode, being passed the inode's number and
  458. * @data. @max_prefetch controls how many inobt records' worth of inodes we
  459. * try to readahead.
  460. */
  461. int
  462. xfs_iwalk(
  463. struct xfs_mount *mp,
  464. struct xfs_trans *tp,
  465. xfs_ino_t startino,
  466. unsigned int flags,
  467. xfs_iwalk_fn iwalk_fn,
  468. unsigned int inode_records,
  469. void *data)
  470. {
  471. struct xfs_iwalk_ag iwag = {
  472. .mp = mp,
  473. .tp = tp,
  474. .iwalk_fn = iwalk_fn,
  475. .data = data,
  476. .startino = startino,
  477. .sz_recs = xfs_iwalk_prefetch(inode_records),
  478. .trim_start = 1,
  479. .skip_empty = 1,
  480. .pwork = XFS_PWORK_SINGLE_THREADED,
  481. .lastino = NULLFSINO,
  482. };
  483. xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
  484. int error;
  485. ASSERT(agno < mp->m_sb.sb_agcount);
  486. ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL));
  487. error = xfs_iwalk_alloc(&iwag);
  488. if (error)
  489. return error;
  490. for (; agno < mp->m_sb.sb_agcount; agno++) {
  491. error = xfs_iwalk_ag(&iwag);
  492. if (error)
  493. break;
  494. iwag.startino = XFS_AGINO_TO_INO(mp, agno + 1, 0);
  495. if (flags & XFS_INOBT_WALK_SAME_AG)
  496. break;
  497. }
  498. xfs_iwalk_free(&iwag);
  499. return error;
  500. }
  501. /* Run per-thread iwalk work. */
  502. static int
  503. xfs_iwalk_ag_work(
  504. struct xfs_mount *mp,
  505. struct xfs_pwork *pwork)
  506. {
  507. struct xfs_iwalk_ag *iwag;
  508. int error = 0;
  509. iwag = container_of(pwork, struct xfs_iwalk_ag, pwork);
  510. if (xfs_pwork_want_abort(pwork))
  511. goto out;
  512. error = xfs_iwalk_alloc(iwag);
  513. if (error)
  514. goto out;
  515. error = xfs_iwalk_ag(iwag);
  516. xfs_iwalk_free(iwag);
  517. out:
  518. kmem_free(iwag);
  519. return error;
  520. }
  521. /*
  522. * Walk all the inodes in the filesystem using multiple threads to process each
  523. * AG.
  524. */
  525. int
  526. xfs_iwalk_threaded(
  527. struct xfs_mount *mp,
  528. xfs_ino_t startino,
  529. unsigned int flags,
  530. xfs_iwalk_fn iwalk_fn,
  531. unsigned int inode_records,
  532. bool polled,
  533. void *data)
  534. {
  535. struct xfs_pwork_ctl pctl;
  536. xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
  537. unsigned int nr_threads;
  538. int error;
  539. ASSERT(agno < mp->m_sb.sb_agcount);
  540. ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL));
  541. nr_threads = xfs_pwork_guess_datadev_parallelism(mp);
  542. error = xfs_pwork_init(mp, &pctl, xfs_iwalk_ag_work, "xfs_iwalk",
  543. nr_threads);
  544. if (error)
  545. return error;
  546. for (; agno < mp->m_sb.sb_agcount; agno++) {
  547. struct xfs_iwalk_ag *iwag;
  548. if (xfs_pwork_ctl_want_abort(&pctl))
  549. break;
  550. iwag = kmem_zalloc(sizeof(struct xfs_iwalk_ag), 0);
  551. iwag->mp = mp;
  552. iwag->iwalk_fn = iwalk_fn;
  553. iwag->data = data;
  554. iwag->startino = startino;
  555. iwag->sz_recs = xfs_iwalk_prefetch(inode_records);
  556. iwag->lastino = NULLFSINO;
  557. xfs_pwork_queue(&pctl, &iwag->pwork);
  558. startino = XFS_AGINO_TO_INO(mp, agno + 1, 0);
  559. if (flags & XFS_INOBT_WALK_SAME_AG)
  560. break;
  561. }
  562. if (polled)
  563. xfs_pwork_poll(&pctl);
  564. return xfs_pwork_destroy(&pctl);
  565. }
  566. /*
  567. * Allow callers to cache up to a page's worth of inobt records. This reflects
  568. * the existing inumbers prefetching behavior. Since the inobt walk does not
  569. * itself do anything with the inobt records, we can set a fairly high limit
  570. * here.
  571. */
  572. #define MAX_INOBT_WALK_PREFETCH \
  573. (PAGE_SIZE / sizeof(struct xfs_inobt_rec_incore))
  574. /*
  575. * Given the number of records that the user wanted, set the number of inobt
  576. * records that we buffer in memory. Set the maximum if @inobt_records == 0.
  577. */
  578. static inline unsigned int
  579. xfs_inobt_walk_prefetch(
  580. unsigned int inobt_records)
  581. {
  582. /*
  583. * If the caller didn't tell us the number of inobt records they
  584. * wanted, assume the maximum prefetch possible for best performance.
  585. */
  586. if (inobt_records == 0)
  587. inobt_records = MAX_INOBT_WALK_PREFETCH;
  588. /*
  589. * Allocate enough space to prefetch at least two inobt records so that
  590. * we can cache both the record where the iwalk started and the next
  591. * record. This simplifies the AG inode walk loop setup code.
  592. */
  593. inobt_records = max(inobt_records, 2U);
  594. /*
  595. * Cap prefetch at that maximum so that we don't use an absurd amount
  596. * of memory.
  597. */
  598. return min_t(unsigned int, inobt_records, MAX_INOBT_WALK_PREFETCH);
  599. }
  600. /*
  601. * Walk all inode btree records in the filesystem starting from @startino. The
  602. * @inobt_walk_fn will be called for each btree record, being passed the incore
  603. * record and @data. @max_prefetch controls how many inobt records we try to
  604. * cache ahead of time.
  605. */
  606. int
  607. xfs_inobt_walk(
  608. struct xfs_mount *mp,
  609. struct xfs_trans *tp,
  610. xfs_ino_t startino,
  611. unsigned int flags,
  612. xfs_inobt_walk_fn inobt_walk_fn,
  613. unsigned int inobt_records,
  614. void *data)
  615. {
  616. struct xfs_iwalk_ag iwag = {
  617. .mp = mp,
  618. .tp = tp,
  619. .inobt_walk_fn = inobt_walk_fn,
  620. .data = data,
  621. .startino = startino,
  622. .sz_recs = xfs_inobt_walk_prefetch(inobt_records),
  623. .pwork = XFS_PWORK_SINGLE_THREADED,
  624. .lastino = NULLFSINO,
  625. };
  626. xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
  627. int error;
  628. ASSERT(agno < mp->m_sb.sb_agcount);
  629. ASSERT(!(flags & ~XFS_INOBT_WALK_FLAGS_ALL));
  630. error = xfs_iwalk_alloc(&iwag);
  631. if (error)
  632. return error;
  633. for (; agno < mp->m_sb.sb_agcount; agno++) {
  634. error = xfs_iwalk_ag(&iwag);
  635. if (error)
  636. break;
  637. iwag.startino = XFS_AGINO_TO_INO(mp, agno + 1, 0);
  638. if (flags & XFS_INOBT_WALK_SAME_AG)
  639. break;
  640. }
  641. xfs_iwalk_free(&iwag);
  642. return error;
  643. }