xfs_trans_ail.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc.
  4. * Copyright (c) 2008 Dave Chinner
  5. * All Rights Reserved.
  6. */
  7. #include "xfs.h"
  8. #include "xfs_fs.h"
  9. #include "xfs_shared.h"
  10. #include "xfs_format.h"
  11. #include "xfs_log_format.h"
  12. #include "xfs_trans_resv.h"
  13. #include "xfs_mount.h"
  14. #include "xfs_trans.h"
  15. #include "xfs_trans_priv.h"
  16. #include "xfs_trace.h"
  17. #include "xfs_errortag.h"
  18. #include "xfs_error.h"
  19. #include "xfs_log.h"
  20. #ifdef DEBUG
  21. /*
  22. * Check that the list is sorted as it should be.
  23. *
  24. * Called with the ail lock held, but we don't want to assert fail with it
  25. * held otherwise we'll lock everything up and won't be able to debug the
  26. * cause. Hence we sample and check the state under the AIL lock and return if
  27. * everything is fine, otherwise we drop the lock and run the ASSERT checks.
  28. * Asserts may not be fatal, so pick the lock back up and continue onwards.
  29. */
  30. STATIC void
  31. xfs_ail_check(
  32. struct xfs_ail *ailp,
  33. struct xfs_log_item *lip)
  34. __must_hold(&ailp->ail_lock)
  35. {
  36. struct xfs_log_item *prev_lip;
  37. struct xfs_log_item *next_lip;
  38. xfs_lsn_t prev_lsn = NULLCOMMITLSN;
  39. xfs_lsn_t next_lsn = NULLCOMMITLSN;
  40. xfs_lsn_t lsn;
  41. bool in_ail;
  42. if (list_empty(&ailp->ail_head))
  43. return;
  44. /*
  45. * Sample then check the next and previous entries are valid.
  46. */
  47. in_ail = test_bit(XFS_LI_IN_AIL, &lip->li_flags);
  48. prev_lip = list_entry(lip->li_ail.prev, struct xfs_log_item, li_ail);
  49. if (&prev_lip->li_ail != &ailp->ail_head)
  50. prev_lsn = prev_lip->li_lsn;
  51. next_lip = list_entry(lip->li_ail.next, struct xfs_log_item, li_ail);
  52. if (&next_lip->li_ail != &ailp->ail_head)
  53. next_lsn = next_lip->li_lsn;
  54. lsn = lip->li_lsn;
  55. if (in_ail &&
  56. (prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0) &&
  57. (next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0))
  58. return;
  59. spin_unlock(&ailp->ail_lock);
  60. ASSERT(in_ail);
  61. ASSERT(prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0);
  62. ASSERT(next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0);
  63. spin_lock(&ailp->ail_lock);
  64. }
  65. #else /* !DEBUG */
  66. #define xfs_ail_check(a,l)
  67. #endif /* DEBUG */
  68. /*
  69. * Return a pointer to the last item in the AIL. If the AIL is empty, then
  70. * return NULL.
  71. */
  72. static struct xfs_log_item *
  73. xfs_ail_max(
  74. struct xfs_ail *ailp)
  75. {
  76. if (list_empty(&ailp->ail_head))
  77. return NULL;
  78. return list_entry(ailp->ail_head.prev, struct xfs_log_item, li_ail);
  79. }
  80. /*
  81. * Return a pointer to the item which follows the given item in the AIL. If
  82. * the given item is the last item in the list, then return NULL.
  83. */
  84. static struct xfs_log_item *
  85. xfs_ail_next(
  86. struct xfs_ail *ailp,
  87. struct xfs_log_item *lip)
  88. {
  89. if (lip->li_ail.next == &ailp->ail_head)
  90. return NULL;
  91. return list_first_entry(&lip->li_ail, struct xfs_log_item, li_ail);
  92. }
  93. /*
  94. * This is called by the log manager code to determine the LSN of the tail of
  95. * the log. This is exactly the LSN of the first item in the AIL. If the AIL
  96. * is empty, then this function returns 0.
  97. *
  98. * We need the AIL lock in order to get a coherent read of the lsn of the last
  99. * item in the AIL.
  100. */
  101. static xfs_lsn_t
  102. __xfs_ail_min_lsn(
  103. struct xfs_ail *ailp)
  104. {
  105. struct xfs_log_item *lip = xfs_ail_min(ailp);
  106. if (lip)
  107. return lip->li_lsn;
  108. return 0;
  109. }
  110. xfs_lsn_t
  111. xfs_ail_min_lsn(
  112. struct xfs_ail *ailp)
  113. {
  114. xfs_lsn_t lsn;
  115. spin_lock(&ailp->ail_lock);
  116. lsn = __xfs_ail_min_lsn(ailp);
  117. spin_unlock(&ailp->ail_lock);
  118. return lsn;
  119. }
  120. /*
  121. * Return the maximum lsn held in the AIL, or zero if the AIL is empty.
  122. */
  123. static xfs_lsn_t
  124. xfs_ail_max_lsn(
  125. struct xfs_ail *ailp)
  126. {
  127. xfs_lsn_t lsn = 0;
  128. struct xfs_log_item *lip;
  129. spin_lock(&ailp->ail_lock);
  130. lip = xfs_ail_max(ailp);
  131. if (lip)
  132. lsn = lip->li_lsn;
  133. spin_unlock(&ailp->ail_lock);
  134. return lsn;
  135. }
  136. /*
  137. * The cursor keeps track of where our current traversal is up to by tracking
  138. * the next item in the list for us. However, for this to be safe, removing an
  139. * object from the AIL needs to invalidate any cursor that points to it. hence
  140. * the traversal cursor needs to be linked to the struct xfs_ail so that
  141. * deletion can search all the active cursors for invalidation.
  142. */
  143. STATIC void
  144. xfs_trans_ail_cursor_init(
  145. struct xfs_ail *ailp,
  146. struct xfs_ail_cursor *cur)
  147. {
  148. cur->item = NULL;
  149. list_add_tail(&cur->list, &ailp->ail_cursors);
  150. }
  151. /*
  152. * Get the next item in the traversal and advance the cursor. If the cursor
  153. * was invalidated (indicated by a lip of 1), restart the traversal.
  154. */
  155. struct xfs_log_item *
  156. xfs_trans_ail_cursor_next(
  157. struct xfs_ail *ailp,
  158. struct xfs_ail_cursor *cur)
  159. {
  160. struct xfs_log_item *lip = cur->item;
  161. if ((uintptr_t)lip & 1)
  162. lip = xfs_ail_min(ailp);
  163. if (lip)
  164. cur->item = xfs_ail_next(ailp, lip);
  165. return lip;
  166. }
  167. /*
  168. * When the traversal is complete, we need to remove the cursor from the list
  169. * of traversing cursors.
  170. */
  171. void
  172. xfs_trans_ail_cursor_done(
  173. struct xfs_ail_cursor *cur)
  174. {
  175. cur->item = NULL;
  176. list_del_init(&cur->list);
  177. }
  178. /*
  179. * Invalidate any cursor that is pointing to this item. This is called when an
  180. * item is removed from the AIL. Any cursor pointing to this object is now
  181. * invalid and the traversal needs to be terminated so it doesn't reference a
  182. * freed object. We set the low bit of the cursor item pointer so we can
  183. * distinguish between an invalidation and the end of the list when getting the
  184. * next item from the cursor.
  185. */
  186. STATIC void
  187. xfs_trans_ail_cursor_clear(
  188. struct xfs_ail *ailp,
  189. struct xfs_log_item *lip)
  190. {
  191. struct xfs_ail_cursor *cur;
  192. list_for_each_entry(cur, &ailp->ail_cursors, list) {
  193. if (cur->item == lip)
  194. cur->item = (struct xfs_log_item *)
  195. ((uintptr_t)cur->item | 1);
  196. }
  197. }
  198. /*
  199. * Find the first item in the AIL with the given @lsn by searching in ascending
  200. * LSN order and initialise the cursor to point to the next item for a
  201. * ascending traversal. Pass a @lsn of zero to initialise the cursor to the
  202. * first item in the AIL. Returns NULL if the list is empty.
  203. */
  204. struct xfs_log_item *
  205. xfs_trans_ail_cursor_first(
  206. struct xfs_ail *ailp,
  207. struct xfs_ail_cursor *cur,
  208. xfs_lsn_t lsn)
  209. {
  210. struct xfs_log_item *lip;
  211. xfs_trans_ail_cursor_init(ailp, cur);
  212. if (lsn == 0) {
  213. lip = xfs_ail_min(ailp);
  214. goto out;
  215. }
  216. list_for_each_entry(lip, &ailp->ail_head, li_ail) {
  217. if (XFS_LSN_CMP(lip->li_lsn, lsn) >= 0)
  218. goto out;
  219. }
  220. return NULL;
  221. out:
  222. if (lip)
  223. cur->item = xfs_ail_next(ailp, lip);
  224. return lip;
  225. }
  226. static struct xfs_log_item *
  227. __xfs_trans_ail_cursor_last(
  228. struct xfs_ail *ailp,
  229. xfs_lsn_t lsn)
  230. {
  231. struct xfs_log_item *lip;
  232. list_for_each_entry_reverse(lip, &ailp->ail_head, li_ail) {
  233. if (XFS_LSN_CMP(lip->li_lsn, lsn) <= 0)
  234. return lip;
  235. }
  236. return NULL;
  237. }
  238. /*
  239. * Find the last item in the AIL with the given @lsn by searching in descending
  240. * LSN order and initialise the cursor to point to that item. If there is no
  241. * item with the value of @lsn, then it sets the cursor to the last item with an
  242. * LSN lower than @lsn. Returns NULL if the list is empty.
  243. */
  244. struct xfs_log_item *
  245. xfs_trans_ail_cursor_last(
  246. struct xfs_ail *ailp,
  247. struct xfs_ail_cursor *cur,
  248. xfs_lsn_t lsn)
  249. {
  250. xfs_trans_ail_cursor_init(ailp, cur);
  251. cur->item = __xfs_trans_ail_cursor_last(ailp, lsn);
  252. return cur->item;
  253. }
  254. /*
  255. * Splice the log item list into the AIL at the given LSN. We splice to the
  256. * tail of the given LSN to maintain insert order for push traversals. The
  257. * cursor is optional, allowing repeated updates to the same LSN to avoid
  258. * repeated traversals. This should not be called with an empty list.
  259. */
  260. static void
  261. xfs_ail_splice(
  262. struct xfs_ail *ailp,
  263. struct xfs_ail_cursor *cur,
  264. struct list_head *list,
  265. xfs_lsn_t lsn)
  266. {
  267. struct xfs_log_item *lip;
  268. ASSERT(!list_empty(list));
  269. /*
  270. * Use the cursor to determine the insertion point if one is
  271. * provided. If not, or if the one we got is not valid,
  272. * find the place in the AIL where the items belong.
  273. */
  274. lip = cur ? cur->item : NULL;
  275. if (!lip || (uintptr_t)lip & 1)
  276. lip = __xfs_trans_ail_cursor_last(ailp, lsn);
  277. /*
  278. * If a cursor is provided, we know we're processing the AIL
  279. * in lsn order, and future items to be spliced in will
  280. * follow the last one being inserted now. Update the
  281. * cursor to point to that last item, now while we have a
  282. * reliable pointer to it.
  283. */
  284. if (cur)
  285. cur->item = list_entry(list->prev, struct xfs_log_item, li_ail);
  286. /*
  287. * Finally perform the splice. Unless the AIL was empty,
  288. * lip points to the item in the AIL _after_ which the new
  289. * items should go. If lip is null the AIL was empty, so
  290. * the new items go at the head of the AIL.
  291. */
  292. if (lip)
  293. list_splice(list, &lip->li_ail);
  294. else
  295. list_splice(list, &ailp->ail_head);
  296. }
  297. /*
  298. * Delete the given item from the AIL. Return a pointer to the item.
  299. */
  300. static void
  301. xfs_ail_delete(
  302. struct xfs_ail *ailp,
  303. struct xfs_log_item *lip)
  304. {
  305. xfs_ail_check(ailp, lip);
  306. list_del(&lip->li_ail);
  307. xfs_trans_ail_cursor_clear(ailp, lip);
  308. }
  309. /*
  310. * Requeue a failed buffer for writeback.
  311. *
  312. * We clear the log item failed state here as well, but we have to be careful
  313. * about reference counts because the only active reference counts on the buffer
  314. * may be the failed log items. Hence if we clear the log item failed state
  315. * before queuing the buffer for IO we can release all active references to
  316. * the buffer and free it, leading to use after free problems in
  317. * xfs_buf_delwri_queue. It makes no difference to the buffer or log items which
  318. * order we process them in - the buffer is locked, and we own the buffer list
  319. * so nothing on them is going to change while we are performing this action.
  320. *
  321. * Hence we can safely queue the buffer for IO before we clear the failed log
  322. * item state, therefore always having an active reference to the buffer and
  323. * avoiding the transient zero-reference state that leads to use-after-free.
  324. */
  325. static inline int
  326. xfsaild_resubmit_item(
  327. struct xfs_log_item *lip,
  328. struct list_head *buffer_list)
  329. {
  330. struct xfs_buf *bp = lip->li_buf;
  331. if (!xfs_buf_trylock(bp))
  332. return XFS_ITEM_LOCKED;
  333. if (!xfs_buf_delwri_queue(bp, buffer_list)) {
  334. xfs_buf_unlock(bp);
  335. return XFS_ITEM_FLUSHING;
  336. }
  337. /* protected by ail_lock */
  338. list_for_each_entry(lip, &bp->b_li_list, li_bio_list) {
  339. if (bp->b_flags & _XBF_INODES)
  340. clear_bit(XFS_LI_FAILED, &lip->li_flags);
  341. else
  342. xfs_clear_li_failed(lip);
  343. }
  344. xfs_buf_unlock(bp);
  345. return XFS_ITEM_SUCCESS;
  346. }
  347. static inline uint
  348. xfsaild_push_item(
  349. struct xfs_ail *ailp,
  350. struct xfs_log_item *lip)
  351. {
  352. /*
  353. * If log item pinning is enabled, skip the push and track the item as
  354. * pinned. This can help induce head-behind-tail conditions.
  355. */
  356. if (XFS_TEST_ERROR(false, ailp->ail_mount, XFS_ERRTAG_LOG_ITEM_PIN))
  357. return XFS_ITEM_PINNED;
  358. /*
  359. * Consider the item pinned if a push callback is not defined so the
  360. * caller will force the log. This should only happen for intent items
  361. * as they are unpinned once the associated done item is committed to
  362. * the on-disk log.
  363. */
  364. if (!lip->li_ops->iop_push)
  365. return XFS_ITEM_PINNED;
  366. if (test_bit(XFS_LI_FAILED, &lip->li_flags))
  367. return xfsaild_resubmit_item(lip, &ailp->ail_buf_list);
  368. return lip->li_ops->iop_push(lip, &ailp->ail_buf_list);
  369. }
  370. static long
  371. xfsaild_push(
  372. struct xfs_ail *ailp)
  373. {
  374. xfs_mount_t *mp = ailp->ail_mount;
  375. struct xfs_ail_cursor cur;
  376. struct xfs_log_item *lip;
  377. xfs_lsn_t lsn;
  378. xfs_lsn_t target;
  379. long tout;
  380. int stuck = 0;
  381. int flushing = 0;
  382. int count = 0;
  383. /*
  384. * If we encountered pinned items or did not finish writing out all
  385. * buffers the last time we ran, force the log first and wait for it
  386. * before pushing again.
  387. */
  388. if (ailp->ail_log_flush && ailp->ail_last_pushed_lsn == 0 &&
  389. (!list_empty_careful(&ailp->ail_buf_list) ||
  390. xfs_ail_min_lsn(ailp))) {
  391. ailp->ail_log_flush = 0;
  392. XFS_STATS_INC(mp, xs_push_ail_flush);
  393. xfs_log_force(mp, XFS_LOG_SYNC);
  394. }
  395. spin_lock(&ailp->ail_lock);
  396. /* barrier matches the ail_target update in xfs_ail_push() */
  397. smp_rmb();
  398. target = ailp->ail_target;
  399. ailp->ail_target_prev = target;
  400. /* we're done if the AIL is empty or our push has reached the end */
  401. lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->ail_last_pushed_lsn);
  402. if (!lip)
  403. goto out_done;
  404. XFS_STATS_INC(mp, xs_push_ail);
  405. lsn = lip->li_lsn;
  406. while ((XFS_LSN_CMP(lip->li_lsn, target) <= 0)) {
  407. int lock_result;
  408. /*
  409. * Note that iop_push may unlock and reacquire the AIL lock. We
  410. * rely on the AIL cursor implementation to be able to deal with
  411. * the dropped lock.
  412. */
  413. lock_result = xfsaild_push_item(ailp, lip);
  414. switch (lock_result) {
  415. case XFS_ITEM_SUCCESS:
  416. XFS_STATS_INC(mp, xs_push_ail_success);
  417. trace_xfs_ail_push(lip);
  418. ailp->ail_last_pushed_lsn = lsn;
  419. break;
  420. case XFS_ITEM_FLUSHING:
  421. /*
  422. * The item or its backing buffer is already being
  423. * flushed. The typical reason for that is that an
  424. * inode buffer is locked because we already pushed the
  425. * updates to it as part of inode clustering.
  426. *
  427. * We do not want to stop flushing just because lots
  428. * of items are already being flushed, but we need to
  429. * re-try the flushing relatively soon if most of the
  430. * AIL is being flushed.
  431. */
  432. XFS_STATS_INC(mp, xs_push_ail_flushing);
  433. trace_xfs_ail_flushing(lip);
  434. flushing++;
  435. ailp->ail_last_pushed_lsn = lsn;
  436. break;
  437. case XFS_ITEM_PINNED:
  438. XFS_STATS_INC(mp, xs_push_ail_pinned);
  439. trace_xfs_ail_pinned(lip);
  440. stuck++;
  441. ailp->ail_log_flush++;
  442. break;
  443. case XFS_ITEM_LOCKED:
  444. XFS_STATS_INC(mp, xs_push_ail_locked);
  445. trace_xfs_ail_locked(lip);
  446. stuck++;
  447. break;
  448. default:
  449. ASSERT(0);
  450. break;
  451. }
  452. count++;
  453. /*
  454. * Are there too many items we can't do anything with?
  455. *
  456. * If we are skipping too many items because we can't flush
  457. * them or they are already being flushed, we back off and
  458. * given them time to complete whatever operation is being
  459. * done. i.e. remove pressure from the AIL while we can't make
  460. * progress so traversals don't slow down further inserts and
  461. * removals to/from the AIL.
  462. *
  463. * The value of 100 is an arbitrary magic number based on
  464. * observation.
  465. */
  466. if (stuck > 100)
  467. break;
  468. lip = xfs_trans_ail_cursor_next(ailp, &cur);
  469. if (lip == NULL)
  470. break;
  471. lsn = lip->li_lsn;
  472. }
  473. out_done:
  474. xfs_trans_ail_cursor_done(&cur);
  475. spin_unlock(&ailp->ail_lock);
  476. if (xfs_buf_delwri_submit_nowait(&ailp->ail_buf_list))
  477. ailp->ail_log_flush++;
  478. if (!count || XFS_LSN_CMP(lsn, target) >= 0) {
  479. /*
  480. * We reached the target or the AIL is empty, so wait a bit
  481. * longer for I/O to complete and remove pushed items from the
  482. * AIL before we start the next scan from the start of the AIL.
  483. */
  484. tout = 50;
  485. ailp->ail_last_pushed_lsn = 0;
  486. } else if (((stuck + flushing) * 100) / count > 90) {
  487. /*
  488. * Either there is a lot of contention on the AIL or we are
  489. * stuck due to operations in progress. "Stuck" in this case
  490. * is defined as >90% of the items we tried to push were stuck.
  491. *
  492. * Backoff a bit more to allow some I/O to complete before
  493. * restarting from the start of the AIL. This prevents us from
  494. * spinning on the same items, and if they are pinned will all
  495. * the restart to issue a log force to unpin the stuck items.
  496. */
  497. tout = 20;
  498. ailp->ail_last_pushed_lsn = 0;
  499. } else {
  500. /*
  501. * Assume we have more work to do in a short while.
  502. */
  503. tout = 10;
  504. }
  505. return tout;
  506. }
  507. static int
  508. xfsaild(
  509. void *data)
  510. {
  511. struct xfs_ail *ailp = data;
  512. long tout = 0; /* milliseconds */
  513. unsigned int noreclaim_flag;
  514. noreclaim_flag = memalloc_noreclaim_save();
  515. set_freezable();
  516. while (1) {
  517. if (tout && tout <= 20)
  518. set_current_state(TASK_KILLABLE);
  519. else
  520. set_current_state(TASK_INTERRUPTIBLE);
  521. /*
  522. * Check kthread_should_stop() after we set the task state to
  523. * guarantee that we either see the stop bit and exit or the
  524. * task state is reset to runnable such that it's not scheduled
  525. * out indefinitely and detects the stop bit at next iteration.
  526. * A memory barrier is included in above task state set to
  527. * serialize again kthread_stop().
  528. */
  529. if (kthread_should_stop()) {
  530. __set_current_state(TASK_RUNNING);
  531. /*
  532. * The caller forces out the AIL before stopping the
  533. * thread in the common case, which means the delwri
  534. * queue is drained. In the shutdown case, the queue may
  535. * still hold relogged buffers that haven't been
  536. * submitted because they were pinned since added to the
  537. * queue.
  538. *
  539. * Log I/O error processing stales the underlying buffer
  540. * and clears the delwri state, expecting the buf to be
  541. * removed on the next submission attempt. That won't
  542. * happen if we're shutting down, so this is the last
  543. * opportunity to release such buffers from the queue.
  544. */
  545. ASSERT(list_empty(&ailp->ail_buf_list) ||
  546. XFS_FORCED_SHUTDOWN(ailp->ail_mount));
  547. xfs_buf_delwri_cancel(&ailp->ail_buf_list);
  548. break;
  549. }
  550. spin_lock(&ailp->ail_lock);
  551. /*
  552. * Idle if the AIL is empty and we are not racing with a target
  553. * update. We check the AIL after we set the task to a sleep
  554. * state to guarantee that we either catch an ail_target update
  555. * or that a wake_up resets the state to TASK_RUNNING.
  556. * Otherwise, we run the risk of sleeping indefinitely.
  557. *
  558. * The barrier matches the ail_target update in xfs_ail_push().
  559. */
  560. smp_rmb();
  561. if (!xfs_ail_min(ailp) &&
  562. ailp->ail_target == ailp->ail_target_prev &&
  563. list_empty(&ailp->ail_buf_list)) {
  564. spin_unlock(&ailp->ail_lock);
  565. freezable_schedule();
  566. tout = 0;
  567. continue;
  568. }
  569. spin_unlock(&ailp->ail_lock);
  570. if (tout)
  571. freezable_schedule_timeout(msecs_to_jiffies(tout));
  572. __set_current_state(TASK_RUNNING);
  573. try_to_freeze();
  574. tout = xfsaild_push(ailp);
  575. }
  576. memalloc_noreclaim_restore(noreclaim_flag);
  577. return 0;
  578. }
  579. /*
  580. * This routine is called to move the tail of the AIL forward. It does this by
  581. * trying to flush items in the AIL whose lsns are below the given
  582. * threshold_lsn.
  583. *
  584. * The push is run asynchronously in a workqueue, which means the caller needs
  585. * to handle waiting on the async flush for space to become available.
  586. * We don't want to interrupt any push that is in progress, hence we only queue
  587. * work if we set the pushing bit appropriately.
  588. *
  589. * We do this unlocked - we only need to know whether there is anything in the
  590. * AIL at the time we are called. We don't need to access the contents of
  591. * any of the objects, so the lock is not needed.
  592. */
  593. void
  594. xfs_ail_push(
  595. struct xfs_ail *ailp,
  596. xfs_lsn_t threshold_lsn)
  597. {
  598. struct xfs_log_item *lip;
  599. lip = xfs_ail_min(ailp);
  600. if (!lip || XFS_FORCED_SHUTDOWN(ailp->ail_mount) ||
  601. XFS_LSN_CMP(threshold_lsn, ailp->ail_target) <= 0)
  602. return;
  603. /*
  604. * Ensure that the new target is noticed in push code before it clears
  605. * the XFS_AIL_PUSHING_BIT.
  606. */
  607. smp_wmb();
  608. xfs_trans_ail_copy_lsn(ailp, &ailp->ail_target, &threshold_lsn);
  609. smp_wmb();
  610. wake_up_process(ailp->ail_task);
  611. }
  612. /*
  613. * Push out all items in the AIL immediately
  614. */
  615. void
  616. xfs_ail_push_all(
  617. struct xfs_ail *ailp)
  618. {
  619. xfs_lsn_t threshold_lsn = xfs_ail_max_lsn(ailp);
  620. if (threshold_lsn)
  621. xfs_ail_push(ailp, threshold_lsn);
  622. }
  623. /*
  624. * Push out all items in the AIL immediately and wait until the AIL is empty.
  625. */
  626. void
  627. xfs_ail_push_all_sync(
  628. struct xfs_ail *ailp)
  629. {
  630. struct xfs_log_item *lip;
  631. DEFINE_WAIT(wait);
  632. spin_lock(&ailp->ail_lock);
  633. while ((lip = xfs_ail_max(ailp)) != NULL) {
  634. prepare_to_wait(&ailp->ail_empty, &wait, TASK_UNINTERRUPTIBLE);
  635. ailp->ail_target = lip->li_lsn;
  636. wake_up_process(ailp->ail_task);
  637. spin_unlock(&ailp->ail_lock);
  638. schedule();
  639. spin_lock(&ailp->ail_lock);
  640. }
  641. spin_unlock(&ailp->ail_lock);
  642. finish_wait(&ailp->ail_empty, &wait);
  643. }
  644. void
  645. xfs_ail_update_finish(
  646. struct xfs_ail *ailp,
  647. xfs_lsn_t old_lsn) __releases(ailp->ail_lock)
  648. {
  649. struct xfs_mount *mp = ailp->ail_mount;
  650. /* if the tail lsn hasn't changed, don't do updates or wakeups. */
  651. if (!old_lsn || old_lsn == __xfs_ail_min_lsn(ailp)) {
  652. spin_unlock(&ailp->ail_lock);
  653. return;
  654. }
  655. if (!XFS_FORCED_SHUTDOWN(mp))
  656. xlog_assign_tail_lsn_locked(mp);
  657. if (list_empty(&ailp->ail_head))
  658. wake_up_all(&ailp->ail_empty);
  659. spin_unlock(&ailp->ail_lock);
  660. xfs_log_space_wake(mp);
  661. }
  662. /*
  663. * xfs_trans_ail_update - bulk AIL insertion operation.
  664. *
  665. * @xfs_trans_ail_update takes an array of log items that all need to be
  666. * positioned at the same LSN in the AIL. If an item is not in the AIL, it will
  667. * be added. Otherwise, it will be repositioned by removing it and re-adding
  668. * it to the AIL. If we move the first item in the AIL, update the log tail to
  669. * match the new minimum LSN in the AIL.
  670. *
  671. * This function takes the AIL lock once to execute the update operations on
  672. * all the items in the array, and as such should not be called with the AIL
  673. * lock held. As a result, once we have the AIL lock, we need to check each log
  674. * item LSN to confirm it needs to be moved forward in the AIL.
  675. *
  676. * To optimise the insert operation, we delete all the items from the AIL in
  677. * the first pass, moving them into a temporary list, then splice the temporary
  678. * list into the correct position in the AIL. This avoids needing to do an
  679. * insert operation on every item.
  680. *
  681. * This function must be called with the AIL lock held. The lock is dropped
  682. * before returning.
  683. */
  684. void
  685. xfs_trans_ail_update_bulk(
  686. struct xfs_ail *ailp,
  687. struct xfs_ail_cursor *cur,
  688. struct xfs_log_item **log_items,
  689. int nr_items,
  690. xfs_lsn_t lsn) __releases(ailp->ail_lock)
  691. {
  692. struct xfs_log_item *mlip;
  693. xfs_lsn_t tail_lsn = 0;
  694. int i;
  695. LIST_HEAD(tmp);
  696. ASSERT(nr_items > 0); /* Not required, but true. */
  697. mlip = xfs_ail_min(ailp);
  698. for (i = 0; i < nr_items; i++) {
  699. struct xfs_log_item *lip = log_items[i];
  700. if (test_and_set_bit(XFS_LI_IN_AIL, &lip->li_flags)) {
  701. /* check if we really need to move the item */
  702. if (XFS_LSN_CMP(lsn, lip->li_lsn) <= 0)
  703. continue;
  704. trace_xfs_ail_move(lip, lip->li_lsn, lsn);
  705. if (mlip == lip && !tail_lsn)
  706. tail_lsn = lip->li_lsn;
  707. xfs_ail_delete(ailp, lip);
  708. } else {
  709. trace_xfs_ail_insert(lip, 0, lsn);
  710. }
  711. lip->li_lsn = lsn;
  712. list_add(&lip->li_ail, &tmp);
  713. }
  714. if (!list_empty(&tmp))
  715. xfs_ail_splice(ailp, cur, &tmp, lsn);
  716. xfs_ail_update_finish(ailp, tail_lsn);
  717. }
  718. /* Insert a log item into the AIL. */
  719. void
  720. xfs_trans_ail_insert(
  721. struct xfs_ail *ailp,
  722. struct xfs_log_item *lip,
  723. xfs_lsn_t lsn)
  724. {
  725. spin_lock(&ailp->ail_lock);
  726. xfs_trans_ail_update_bulk(ailp, NULL, &lip, 1, lsn);
  727. }
  728. /*
  729. * Delete one log item from the AIL.
  730. *
  731. * If this item was at the tail of the AIL, return the LSN of the log item so
  732. * that we can use it to check if the LSN of the tail of the log has moved
  733. * when finishing up the AIL delete process in xfs_ail_update_finish().
  734. */
  735. xfs_lsn_t
  736. xfs_ail_delete_one(
  737. struct xfs_ail *ailp,
  738. struct xfs_log_item *lip)
  739. {
  740. struct xfs_log_item *mlip = xfs_ail_min(ailp);
  741. xfs_lsn_t lsn = lip->li_lsn;
  742. trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn);
  743. xfs_ail_delete(ailp, lip);
  744. clear_bit(XFS_LI_IN_AIL, &lip->li_flags);
  745. lip->li_lsn = 0;
  746. if (mlip == lip)
  747. return lsn;
  748. return 0;
  749. }
  750. void
  751. xfs_trans_ail_delete(
  752. struct xfs_log_item *lip,
  753. int shutdown_type)
  754. {
  755. struct xfs_ail *ailp = lip->li_ailp;
  756. struct xfs_mount *mp = ailp->ail_mount;
  757. xfs_lsn_t tail_lsn;
  758. spin_lock(&ailp->ail_lock);
  759. if (!test_bit(XFS_LI_IN_AIL, &lip->li_flags)) {
  760. spin_unlock(&ailp->ail_lock);
  761. if (shutdown_type && !XFS_FORCED_SHUTDOWN(mp)) {
  762. xfs_alert_tag(mp, XFS_PTAG_AILDELETE,
  763. "%s: attempting to delete a log item that is not in the AIL",
  764. __func__);
  765. xfs_force_shutdown(mp, shutdown_type);
  766. }
  767. return;
  768. }
  769. /* xfs_ail_update_finish() drops the AIL lock */
  770. xfs_clear_li_failed(lip);
  771. tail_lsn = xfs_ail_delete_one(ailp, lip);
  772. xfs_ail_update_finish(ailp, tail_lsn);
  773. }
  774. int
  775. xfs_trans_ail_init(
  776. xfs_mount_t *mp)
  777. {
  778. struct xfs_ail *ailp;
  779. ailp = kmem_zalloc(sizeof(struct xfs_ail), KM_MAYFAIL);
  780. if (!ailp)
  781. return -ENOMEM;
  782. ailp->ail_mount = mp;
  783. INIT_LIST_HEAD(&ailp->ail_head);
  784. INIT_LIST_HEAD(&ailp->ail_cursors);
  785. spin_lock_init(&ailp->ail_lock);
  786. INIT_LIST_HEAD(&ailp->ail_buf_list);
  787. init_waitqueue_head(&ailp->ail_empty);
  788. ailp->ail_task = kthread_run(xfsaild, ailp, "xfsaild/%s",
  789. ailp->ail_mount->m_super->s_id);
  790. if (IS_ERR(ailp->ail_task))
  791. goto out_free_ailp;
  792. mp->m_ail = ailp;
  793. return 0;
  794. out_free_ailp:
  795. kmem_free(ailp);
  796. return -ENOMEM;
  797. }
  798. void
  799. xfs_trans_ail_destroy(
  800. xfs_mount_t *mp)
  801. {
  802. struct xfs_ail *ailp = mp->m_ail;
  803. kthread_stop(ailp->ail_task);
  804. kmem_free(ailp);
  805. }