xfs_buf_item.c 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2000-2005 Silicon Graphics, Inc.
  4. * All Rights Reserved.
  5. */
  6. #include "xfs.h"
  7. #include "xfs_fs.h"
  8. #include "xfs_shared.h"
  9. #include "xfs_format.h"
  10. #include "xfs_log_format.h"
  11. #include "xfs_trans_resv.h"
  12. #include "xfs_bit.h"
  13. #include "xfs_mount.h"
  14. #include "xfs_trans.h"
  15. #include "xfs_trans_priv.h"
  16. #include "xfs_buf_item.h"
  17. #include "xfs_inode.h"
  18. #include "xfs_inode_item.h"
  19. #include "xfs_quota.h"
  20. #include "xfs_dquot_item.h"
  21. #include "xfs_dquot.h"
  22. #include "xfs_trace.h"
  23. #include "xfs_log.h"
  24. kmem_zone_t *xfs_buf_item_zone;
  25. static inline struct xfs_buf_log_item *BUF_ITEM(struct xfs_log_item *lip)
  26. {
  27. return container_of(lip, struct xfs_buf_log_item, bli_item);
  28. }
  29. /* Is this log iovec plausibly large enough to contain the buffer log format? */
  30. bool
  31. xfs_buf_log_check_iovec(
  32. struct xfs_log_iovec *iovec)
  33. {
  34. struct xfs_buf_log_format *blfp = iovec->i_addr;
  35. char *bmp_end;
  36. char *item_end;
  37. if (offsetof(struct xfs_buf_log_format, blf_data_map) > iovec->i_len)
  38. return false;
  39. item_end = (char *)iovec->i_addr + iovec->i_len;
  40. bmp_end = (char *)&blfp->blf_data_map[blfp->blf_map_size];
  41. return bmp_end <= item_end;
  42. }
  43. static inline int
  44. xfs_buf_log_format_size(
  45. struct xfs_buf_log_format *blfp)
  46. {
  47. return offsetof(struct xfs_buf_log_format, blf_data_map) +
  48. (blfp->blf_map_size * sizeof(blfp->blf_data_map[0]));
  49. }
  50. /*
  51. * This returns the number of log iovecs needed to log the
  52. * given buf log item.
  53. *
  54. * It calculates this as 1 iovec for the buf log format structure
  55. * and 1 for each stretch of non-contiguous chunks to be logged.
  56. * Contiguous chunks are logged in a single iovec.
  57. *
  58. * If the XFS_BLI_STALE flag has been set, then log nothing.
  59. */
  60. STATIC void
  61. xfs_buf_item_size_segment(
  62. struct xfs_buf_log_item *bip,
  63. struct xfs_buf_log_format *blfp,
  64. int *nvecs,
  65. int *nbytes)
  66. {
  67. struct xfs_buf *bp = bip->bli_buf;
  68. int next_bit;
  69. int last_bit;
  70. last_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0);
  71. if (last_bit == -1)
  72. return;
  73. /*
  74. * initial count for a dirty buffer is 2 vectors - the format structure
  75. * and the first dirty region.
  76. */
  77. *nvecs += 2;
  78. *nbytes += xfs_buf_log_format_size(blfp) + XFS_BLF_CHUNK;
  79. while (last_bit != -1) {
  80. /*
  81. * This takes the bit number to start looking from and
  82. * returns the next set bit from there. It returns -1
  83. * if there are no more bits set or the start bit is
  84. * beyond the end of the bitmap.
  85. */
  86. next_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size,
  87. last_bit + 1);
  88. /*
  89. * If we run out of bits, leave the loop,
  90. * else if we find a new set of bits bump the number of vecs,
  91. * else keep scanning the current set of bits.
  92. */
  93. if (next_bit == -1) {
  94. break;
  95. } else if (next_bit != last_bit + 1) {
  96. last_bit = next_bit;
  97. (*nvecs)++;
  98. } else if (xfs_buf_offset(bp, next_bit * XFS_BLF_CHUNK) !=
  99. (xfs_buf_offset(bp, last_bit * XFS_BLF_CHUNK) +
  100. XFS_BLF_CHUNK)) {
  101. last_bit = next_bit;
  102. (*nvecs)++;
  103. } else {
  104. last_bit++;
  105. }
  106. *nbytes += XFS_BLF_CHUNK;
  107. }
  108. }
  109. /*
  110. * This returns the number of log iovecs needed to log the given buf log item.
  111. *
  112. * It calculates this as 1 iovec for the buf log format structure and 1 for each
  113. * stretch of non-contiguous chunks to be logged. Contiguous chunks are logged
  114. * in a single iovec.
  115. *
  116. * Discontiguous buffers need a format structure per region that is being
  117. * logged. This makes the changes in the buffer appear to log recovery as though
  118. * they came from separate buffers, just like would occur if multiple buffers
  119. * were used instead of a single discontiguous buffer. This enables
  120. * discontiguous buffers to be in-memory constructs, completely transparent to
  121. * what ends up on disk.
  122. *
  123. * If the XFS_BLI_STALE flag has been set, then log nothing but the buf log
  124. * format structures.
  125. */
  126. STATIC void
  127. xfs_buf_item_size(
  128. struct xfs_log_item *lip,
  129. int *nvecs,
  130. int *nbytes)
  131. {
  132. struct xfs_buf_log_item *bip = BUF_ITEM(lip);
  133. int i;
  134. ASSERT(atomic_read(&bip->bli_refcount) > 0);
  135. if (bip->bli_flags & XFS_BLI_STALE) {
  136. /*
  137. * The buffer is stale, so all we need to log
  138. * is the buf log format structure with the
  139. * cancel flag in it.
  140. */
  141. trace_xfs_buf_item_size_stale(bip);
  142. ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
  143. *nvecs += bip->bli_format_count;
  144. for (i = 0; i < bip->bli_format_count; i++) {
  145. *nbytes += xfs_buf_log_format_size(&bip->bli_formats[i]);
  146. }
  147. return;
  148. }
  149. ASSERT(bip->bli_flags & XFS_BLI_LOGGED);
  150. if (bip->bli_flags & XFS_BLI_ORDERED) {
  151. /*
  152. * The buffer has been logged just to order it.
  153. * It is not being included in the transaction
  154. * commit, so no vectors are used at all.
  155. */
  156. trace_xfs_buf_item_size_ordered(bip);
  157. *nvecs = XFS_LOG_VEC_ORDERED;
  158. return;
  159. }
  160. /*
  161. * the vector count is based on the number of buffer vectors we have
  162. * dirty bits in. This will only be greater than one when we have a
  163. * compound buffer with more than one segment dirty. Hence for compound
  164. * buffers we need to track which segment the dirty bits correspond to,
  165. * and when we move from one segment to the next increment the vector
  166. * count for the extra buf log format structure that will need to be
  167. * written.
  168. */
  169. for (i = 0; i < bip->bli_format_count; i++) {
  170. xfs_buf_item_size_segment(bip, &bip->bli_formats[i],
  171. nvecs, nbytes);
  172. }
  173. trace_xfs_buf_item_size(bip);
  174. }
  175. static inline void
  176. xfs_buf_item_copy_iovec(
  177. struct xfs_log_vec *lv,
  178. struct xfs_log_iovec **vecp,
  179. struct xfs_buf *bp,
  180. uint offset,
  181. int first_bit,
  182. uint nbits)
  183. {
  184. offset += first_bit * XFS_BLF_CHUNK;
  185. xlog_copy_iovec(lv, vecp, XLOG_REG_TYPE_BCHUNK,
  186. xfs_buf_offset(bp, offset),
  187. nbits * XFS_BLF_CHUNK);
  188. }
  189. static inline bool
  190. xfs_buf_item_straddle(
  191. struct xfs_buf *bp,
  192. uint offset,
  193. int next_bit,
  194. int last_bit)
  195. {
  196. return xfs_buf_offset(bp, offset + (next_bit << XFS_BLF_SHIFT)) !=
  197. (xfs_buf_offset(bp, offset + (last_bit << XFS_BLF_SHIFT)) +
  198. XFS_BLF_CHUNK);
  199. }
  200. static void
  201. xfs_buf_item_format_segment(
  202. struct xfs_buf_log_item *bip,
  203. struct xfs_log_vec *lv,
  204. struct xfs_log_iovec **vecp,
  205. uint offset,
  206. struct xfs_buf_log_format *blfp)
  207. {
  208. struct xfs_buf *bp = bip->bli_buf;
  209. uint base_size;
  210. int first_bit;
  211. int last_bit;
  212. int next_bit;
  213. uint nbits;
  214. /* copy the flags across from the base format item */
  215. blfp->blf_flags = bip->__bli_format.blf_flags;
  216. /*
  217. * Base size is the actual size of the ondisk structure - it reflects
  218. * the actual size of the dirty bitmap rather than the size of the in
  219. * memory structure.
  220. */
  221. base_size = xfs_buf_log_format_size(blfp);
  222. first_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0);
  223. if (!(bip->bli_flags & XFS_BLI_STALE) && first_bit == -1) {
  224. /*
  225. * If the map is not be dirty in the transaction, mark
  226. * the size as zero and do not advance the vector pointer.
  227. */
  228. return;
  229. }
  230. blfp = xlog_copy_iovec(lv, vecp, XLOG_REG_TYPE_BFORMAT, blfp, base_size);
  231. blfp->blf_size = 1;
  232. if (bip->bli_flags & XFS_BLI_STALE) {
  233. /*
  234. * The buffer is stale, so all we need to log
  235. * is the buf log format structure with the
  236. * cancel flag in it.
  237. */
  238. trace_xfs_buf_item_format_stale(bip);
  239. ASSERT(blfp->blf_flags & XFS_BLF_CANCEL);
  240. return;
  241. }
  242. /*
  243. * Fill in an iovec for each set of contiguous chunks.
  244. */
  245. last_bit = first_bit;
  246. nbits = 1;
  247. for (;;) {
  248. /*
  249. * This takes the bit number to start looking from and
  250. * returns the next set bit from there. It returns -1
  251. * if there are no more bits set or the start bit is
  252. * beyond the end of the bitmap.
  253. */
  254. next_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size,
  255. (uint)last_bit + 1);
  256. /*
  257. * If we run out of bits fill in the last iovec and get out of
  258. * the loop. Else if we start a new set of bits then fill in
  259. * the iovec for the series we were looking at and start
  260. * counting the bits in the new one. Else we're still in the
  261. * same set of bits so just keep counting and scanning.
  262. */
  263. if (next_bit == -1) {
  264. xfs_buf_item_copy_iovec(lv, vecp, bp, offset,
  265. first_bit, nbits);
  266. blfp->blf_size++;
  267. break;
  268. } else if (next_bit != last_bit + 1 ||
  269. xfs_buf_item_straddle(bp, offset, next_bit, last_bit)) {
  270. xfs_buf_item_copy_iovec(lv, vecp, bp, offset,
  271. first_bit, nbits);
  272. blfp->blf_size++;
  273. first_bit = next_bit;
  274. last_bit = next_bit;
  275. nbits = 1;
  276. } else {
  277. last_bit++;
  278. nbits++;
  279. }
  280. }
  281. }
  282. /*
  283. * This is called to fill in the vector of log iovecs for the
  284. * given log buf item. It fills the first entry with a buf log
  285. * format structure, and the rest point to contiguous chunks
  286. * within the buffer.
  287. */
  288. STATIC void
  289. xfs_buf_item_format(
  290. struct xfs_log_item *lip,
  291. struct xfs_log_vec *lv)
  292. {
  293. struct xfs_buf_log_item *bip = BUF_ITEM(lip);
  294. struct xfs_buf *bp = bip->bli_buf;
  295. struct xfs_log_iovec *vecp = NULL;
  296. uint offset = 0;
  297. int i;
  298. ASSERT(atomic_read(&bip->bli_refcount) > 0);
  299. ASSERT((bip->bli_flags & XFS_BLI_LOGGED) ||
  300. (bip->bli_flags & XFS_BLI_STALE));
  301. ASSERT((bip->bli_flags & XFS_BLI_STALE) ||
  302. (xfs_blft_from_flags(&bip->__bli_format) > XFS_BLFT_UNKNOWN_BUF
  303. && xfs_blft_from_flags(&bip->__bli_format) < XFS_BLFT_MAX_BUF));
  304. ASSERT(!(bip->bli_flags & XFS_BLI_ORDERED) ||
  305. (bip->bli_flags & XFS_BLI_STALE));
  306. /*
  307. * If it is an inode buffer, transfer the in-memory state to the
  308. * format flags and clear the in-memory state.
  309. *
  310. * For buffer based inode allocation, we do not transfer
  311. * this state if the inode buffer allocation has not yet been committed
  312. * to the log as setting the XFS_BLI_INODE_BUF flag will prevent
  313. * correct replay of the inode allocation.
  314. *
  315. * For icreate item based inode allocation, the buffers aren't written
  316. * to the journal during allocation, and hence we should always tag the
  317. * buffer as an inode buffer so that the correct unlinked list replay
  318. * occurs during recovery.
  319. */
  320. if (bip->bli_flags & XFS_BLI_INODE_BUF) {
  321. if (xfs_sb_version_has_v3inode(&lip->li_mountp->m_sb) ||
  322. !((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) &&
  323. xfs_log_item_in_current_chkpt(lip)))
  324. bip->__bli_format.blf_flags |= XFS_BLF_INODE_BUF;
  325. bip->bli_flags &= ~XFS_BLI_INODE_BUF;
  326. }
  327. for (i = 0; i < bip->bli_format_count; i++) {
  328. xfs_buf_item_format_segment(bip, lv, &vecp, offset,
  329. &bip->bli_formats[i]);
  330. offset += BBTOB(bp->b_maps[i].bm_len);
  331. }
  332. /*
  333. * Check to make sure everything is consistent.
  334. */
  335. trace_xfs_buf_item_format(bip);
  336. }
  337. /*
  338. * This is called to pin the buffer associated with the buf log item in memory
  339. * so it cannot be written out.
  340. *
  341. * We also always take a reference to the buffer log item here so that the bli
  342. * is held while the item is pinned in memory. This means that we can
  343. * unconditionally drop the reference count a transaction holds when the
  344. * transaction is completed.
  345. */
  346. STATIC void
  347. xfs_buf_item_pin(
  348. struct xfs_log_item *lip)
  349. {
  350. struct xfs_buf_log_item *bip = BUF_ITEM(lip);
  351. ASSERT(atomic_read(&bip->bli_refcount) > 0);
  352. ASSERT((bip->bli_flags & XFS_BLI_LOGGED) ||
  353. (bip->bli_flags & XFS_BLI_ORDERED) ||
  354. (bip->bli_flags & XFS_BLI_STALE));
  355. trace_xfs_buf_item_pin(bip);
  356. atomic_inc(&bip->bli_refcount);
  357. atomic_inc(&bip->bli_buf->b_pin_count);
  358. }
  359. /*
  360. * This is called to unpin the buffer associated with the buf log
  361. * item which was previously pinned with a call to xfs_buf_item_pin().
  362. *
  363. * Also drop the reference to the buf item for the current transaction.
  364. * If the XFS_BLI_STALE flag is set and we are the last reference,
  365. * then free up the buf log item and unlock the buffer.
  366. *
  367. * If the remove flag is set we are called from uncommit in the
  368. * forced-shutdown path. If that is true and the reference count on
  369. * the log item is going to drop to zero we need to free the item's
  370. * descriptor in the transaction.
  371. */
  372. STATIC void
  373. xfs_buf_item_unpin(
  374. struct xfs_log_item *lip,
  375. int remove)
  376. {
  377. struct xfs_buf_log_item *bip = BUF_ITEM(lip);
  378. xfs_buf_t *bp = bip->bli_buf;
  379. int stale = bip->bli_flags & XFS_BLI_STALE;
  380. int freed;
  381. ASSERT(bp->b_log_item == bip);
  382. ASSERT(atomic_read(&bip->bli_refcount) > 0);
  383. trace_xfs_buf_item_unpin(bip);
  384. freed = atomic_dec_and_test(&bip->bli_refcount);
  385. if (atomic_dec_and_test(&bp->b_pin_count))
  386. wake_up_all(&bp->b_waiters);
  387. if (freed && stale) {
  388. ASSERT(bip->bli_flags & XFS_BLI_STALE);
  389. ASSERT(xfs_buf_islocked(bp));
  390. ASSERT(bp->b_flags & XBF_STALE);
  391. ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
  392. trace_xfs_buf_item_unpin_stale(bip);
  393. if (remove) {
  394. /*
  395. * If we are in a transaction context, we have to
  396. * remove the log item from the transaction as we are
  397. * about to release our reference to the buffer. If we
  398. * don't, the unlock that occurs later in
  399. * xfs_trans_uncommit() will try to reference the
  400. * buffer which we no longer have a hold on.
  401. */
  402. if (!list_empty(&lip->li_trans))
  403. xfs_trans_del_item(lip);
  404. /*
  405. * Since the transaction no longer refers to the buffer,
  406. * the buffer should no longer refer to the transaction.
  407. */
  408. bp->b_transp = NULL;
  409. }
  410. /*
  411. * If we get called here because of an IO error, we may or may
  412. * not have the item on the AIL. xfs_trans_ail_delete() will
  413. * take care of that situation. xfs_trans_ail_delete() drops
  414. * the AIL lock.
  415. */
  416. if (bip->bli_flags & XFS_BLI_STALE_INODE) {
  417. xfs_buf_item_done(bp);
  418. xfs_buf_inode_iodone(bp);
  419. ASSERT(list_empty(&bp->b_li_list));
  420. } else {
  421. xfs_trans_ail_delete(lip, SHUTDOWN_LOG_IO_ERROR);
  422. xfs_buf_item_relse(bp);
  423. ASSERT(bp->b_log_item == NULL);
  424. }
  425. xfs_buf_relse(bp);
  426. } else if (freed && remove) {
  427. /*
  428. * The buffer must be locked and held by the caller to simulate
  429. * an async I/O failure.
  430. */
  431. xfs_buf_lock(bp);
  432. xfs_buf_hold(bp);
  433. bp->b_flags |= XBF_ASYNC;
  434. xfs_buf_ioend_fail(bp);
  435. }
  436. }
  437. STATIC uint
  438. xfs_buf_item_push(
  439. struct xfs_log_item *lip,
  440. struct list_head *buffer_list)
  441. {
  442. struct xfs_buf_log_item *bip = BUF_ITEM(lip);
  443. struct xfs_buf *bp = bip->bli_buf;
  444. uint rval = XFS_ITEM_SUCCESS;
  445. if (xfs_buf_ispinned(bp))
  446. return XFS_ITEM_PINNED;
  447. if (!xfs_buf_trylock(bp)) {
  448. /*
  449. * If we have just raced with a buffer being pinned and it has
  450. * been marked stale, we could end up stalling until someone else
  451. * issues a log force to unpin the stale buffer. Check for the
  452. * race condition here so xfsaild recognizes the buffer is pinned
  453. * and queues a log force to move it along.
  454. */
  455. if (xfs_buf_ispinned(bp))
  456. return XFS_ITEM_PINNED;
  457. return XFS_ITEM_LOCKED;
  458. }
  459. ASSERT(!(bip->bli_flags & XFS_BLI_STALE));
  460. trace_xfs_buf_item_push(bip);
  461. /* has a previous flush failed due to IO errors? */
  462. if (bp->b_flags & XBF_WRITE_FAIL) {
  463. xfs_buf_alert_ratelimited(bp, "XFS: Failing async write",
  464. "Failing async write on buffer block 0x%llx. Retrying async write.",
  465. (long long)bp->b_bn);
  466. }
  467. if (!xfs_buf_delwri_queue(bp, buffer_list))
  468. rval = XFS_ITEM_FLUSHING;
  469. xfs_buf_unlock(bp);
  470. return rval;
  471. }
  472. /*
  473. * Drop the buffer log item refcount and take appropriate action. This helper
  474. * determines whether the bli must be freed or not, since a decrement to zero
  475. * does not necessarily mean the bli is unused.
  476. *
  477. * Return true if the bli is freed, false otherwise.
  478. */
  479. bool
  480. xfs_buf_item_put(
  481. struct xfs_buf_log_item *bip)
  482. {
  483. struct xfs_log_item *lip = &bip->bli_item;
  484. bool aborted;
  485. bool dirty;
  486. /* drop the bli ref and return if it wasn't the last one */
  487. if (!atomic_dec_and_test(&bip->bli_refcount))
  488. return false;
  489. /*
  490. * We dropped the last ref and must free the item if clean or aborted.
  491. * If the bli is dirty and non-aborted, the buffer was clean in the
  492. * transaction but still awaiting writeback from previous changes. In
  493. * that case, the bli is freed on buffer writeback completion.
  494. */
  495. aborted = test_bit(XFS_LI_ABORTED, &lip->li_flags) ||
  496. XFS_FORCED_SHUTDOWN(lip->li_mountp);
  497. dirty = bip->bli_flags & XFS_BLI_DIRTY;
  498. if (dirty && !aborted)
  499. return false;
  500. /*
  501. * The bli is aborted or clean. An aborted item may be in the AIL
  502. * regardless of dirty state. For example, consider an aborted
  503. * transaction that invalidated a dirty bli and cleared the dirty
  504. * state.
  505. */
  506. if (aborted)
  507. xfs_trans_ail_delete(lip, 0);
  508. xfs_buf_item_relse(bip->bli_buf);
  509. return true;
  510. }
  511. /*
  512. * Release the buffer associated with the buf log item. If there is no dirty
  513. * logged data associated with the buffer recorded in the buf log item, then
  514. * free the buf log item and remove the reference to it in the buffer.
  515. *
  516. * This call ignores the recursion count. It is only called when the buffer
  517. * should REALLY be unlocked, regardless of the recursion count.
  518. *
  519. * We unconditionally drop the transaction's reference to the log item. If the
  520. * item was logged, then another reference was taken when it was pinned, so we
  521. * can safely drop the transaction reference now. This also allows us to avoid
  522. * potential races with the unpin code freeing the bli by not referencing the
  523. * bli after we've dropped the reference count.
  524. *
  525. * If the XFS_BLI_HOLD flag is set in the buf log item, then free the log item
  526. * if necessary but do not unlock the buffer. This is for support of
  527. * xfs_trans_bhold(). Make sure the XFS_BLI_HOLD field is cleared if we don't
  528. * free the item.
  529. */
  530. STATIC void
  531. xfs_buf_item_release(
  532. struct xfs_log_item *lip)
  533. {
  534. struct xfs_buf_log_item *bip = BUF_ITEM(lip);
  535. struct xfs_buf *bp = bip->bli_buf;
  536. bool released;
  537. bool hold = bip->bli_flags & XFS_BLI_HOLD;
  538. bool stale = bip->bli_flags & XFS_BLI_STALE;
  539. #if defined(DEBUG) || defined(XFS_WARN)
  540. bool ordered = bip->bli_flags & XFS_BLI_ORDERED;
  541. bool dirty = bip->bli_flags & XFS_BLI_DIRTY;
  542. bool aborted = test_bit(XFS_LI_ABORTED,
  543. &lip->li_flags);
  544. #endif
  545. trace_xfs_buf_item_release(bip);
  546. /*
  547. * The bli dirty state should match whether the blf has logged segments
  548. * except for ordered buffers, where only the bli should be dirty.
  549. */
  550. ASSERT((!ordered && dirty == xfs_buf_item_dirty_format(bip)) ||
  551. (ordered && dirty && !xfs_buf_item_dirty_format(bip)));
  552. ASSERT(!stale || (bip->__bli_format.blf_flags & XFS_BLF_CANCEL));
  553. /*
  554. * Clear the buffer's association with this transaction and
  555. * per-transaction state from the bli, which has been copied above.
  556. */
  557. bp->b_transp = NULL;
  558. bip->bli_flags &= ~(XFS_BLI_LOGGED | XFS_BLI_HOLD | XFS_BLI_ORDERED);
  559. /*
  560. * Unref the item and unlock the buffer unless held or stale. Stale
  561. * buffers remain locked until final unpin unless the bli is freed by
  562. * the unref call. The latter implies shutdown because buffer
  563. * invalidation dirties the bli and transaction.
  564. */
  565. released = xfs_buf_item_put(bip);
  566. if (hold || (stale && !released))
  567. return;
  568. ASSERT(!stale || aborted);
  569. xfs_buf_relse(bp);
  570. }
  571. STATIC void
  572. xfs_buf_item_committing(
  573. struct xfs_log_item *lip,
  574. xfs_lsn_t commit_lsn)
  575. {
  576. return xfs_buf_item_release(lip);
  577. }
  578. /*
  579. * This is called to find out where the oldest active copy of the
  580. * buf log item in the on disk log resides now that the last log
  581. * write of it completed at the given lsn.
  582. * We always re-log all the dirty data in a buffer, so usually the
  583. * latest copy in the on disk log is the only one that matters. For
  584. * those cases we simply return the given lsn.
  585. *
  586. * The one exception to this is for buffers full of newly allocated
  587. * inodes. These buffers are only relogged with the XFS_BLI_INODE_BUF
  588. * flag set, indicating that only the di_next_unlinked fields from the
  589. * inodes in the buffers will be replayed during recovery. If the
  590. * original newly allocated inode images have not yet been flushed
  591. * when the buffer is so relogged, then we need to make sure that we
  592. * keep the old images in the 'active' portion of the log. We do this
  593. * by returning the original lsn of that transaction here rather than
  594. * the current one.
  595. */
  596. STATIC xfs_lsn_t
  597. xfs_buf_item_committed(
  598. struct xfs_log_item *lip,
  599. xfs_lsn_t lsn)
  600. {
  601. struct xfs_buf_log_item *bip = BUF_ITEM(lip);
  602. trace_xfs_buf_item_committed(bip);
  603. if ((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && lip->li_lsn != 0)
  604. return lip->li_lsn;
  605. return lsn;
  606. }
  607. static const struct xfs_item_ops xfs_buf_item_ops = {
  608. .iop_size = xfs_buf_item_size,
  609. .iop_format = xfs_buf_item_format,
  610. .iop_pin = xfs_buf_item_pin,
  611. .iop_unpin = xfs_buf_item_unpin,
  612. .iop_release = xfs_buf_item_release,
  613. .iop_committing = xfs_buf_item_committing,
  614. .iop_committed = xfs_buf_item_committed,
  615. .iop_push = xfs_buf_item_push,
  616. };
  617. STATIC void
  618. xfs_buf_item_get_format(
  619. struct xfs_buf_log_item *bip,
  620. int count)
  621. {
  622. ASSERT(bip->bli_formats == NULL);
  623. bip->bli_format_count = count;
  624. if (count == 1) {
  625. bip->bli_formats = &bip->__bli_format;
  626. return;
  627. }
  628. bip->bli_formats = kmem_zalloc(count * sizeof(struct xfs_buf_log_format),
  629. 0);
  630. }
  631. STATIC void
  632. xfs_buf_item_free_format(
  633. struct xfs_buf_log_item *bip)
  634. {
  635. if (bip->bli_formats != &bip->__bli_format) {
  636. kmem_free(bip->bli_formats);
  637. bip->bli_formats = NULL;
  638. }
  639. }
  640. /*
  641. * Allocate a new buf log item to go with the given buffer.
  642. * Set the buffer's b_log_item field to point to the new
  643. * buf log item.
  644. */
  645. int
  646. xfs_buf_item_init(
  647. struct xfs_buf *bp,
  648. struct xfs_mount *mp)
  649. {
  650. struct xfs_buf_log_item *bip = bp->b_log_item;
  651. int chunks;
  652. int map_size;
  653. int i;
  654. /*
  655. * Check to see if there is already a buf log item for
  656. * this buffer. If we do already have one, there is
  657. * nothing to do here so return.
  658. */
  659. ASSERT(bp->b_mount == mp);
  660. if (bip) {
  661. ASSERT(bip->bli_item.li_type == XFS_LI_BUF);
  662. ASSERT(!bp->b_transp);
  663. ASSERT(bip->bli_buf == bp);
  664. return 0;
  665. }
  666. bip = kmem_cache_zalloc(xfs_buf_item_zone, GFP_KERNEL | __GFP_NOFAIL);
  667. xfs_log_item_init(mp, &bip->bli_item, XFS_LI_BUF, &xfs_buf_item_ops);
  668. bip->bli_buf = bp;
  669. /*
  670. * chunks is the number of XFS_BLF_CHUNK size pieces the buffer
  671. * can be divided into. Make sure not to truncate any pieces.
  672. * map_size is the size of the bitmap needed to describe the
  673. * chunks of the buffer.
  674. *
  675. * Discontiguous buffer support follows the layout of the underlying
  676. * buffer. This makes the implementation as simple as possible.
  677. */
  678. xfs_buf_item_get_format(bip, bp->b_map_count);
  679. for (i = 0; i < bip->bli_format_count; i++) {
  680. chunks = DIV_ROUND_UP(BBTOB(bp->b_maps[i].bm_len),
  681. XFS_BLF_CHUNK);
  682. map_size = DIV_ROUND_UP(chunks, NBWORD);
  683. if (map_size > XFS_BLF_DATAMAP_SIZE) {
  684. kmem_cache_free(xfs_buf_item_zone, bip);
  685. xfs_err(mp,
  686. "buffer item dirty bitmap (%u uints) too small to reflect %u bytes!",
  687. map_size,
  688. BBTOB(bp->b_maps[i].bm_len));
  689. return -EFSCORRUPTED;
  690. }
  691. bip->bli_formats[i].blf_type = XFS_LI_BUF;
  692. bip->bli_formats[i].blf_blkno = bp->b_maps[i].bm_bn;
  693. bip->bli_formats[i].blf_len = bp->b_maps[i].bm_len;
  694. bip->bli_formats[i].blf_map_size = map_size;
  695. }
  696. bp->b_log_item = bip;
  697. xfs_buf_hold(bp);
  698. return 0;
  699. }
  700. /*
  701. * Mark bytes first through last inclusive as dirty in the buf
  702. * item's bitmap.
  703. */
  704. static void
  705. xfs_buf_item_log_segment(
  706. uint first,
  707. uint last,
  708. uint *map)
  709. {
  710. uint first_bit;
  711. uint last_bit;
  712. uint bits_to_set;
  713. uint bits_set;
  714. uint word_num;
  715. uint *wordp;
  716. uint bit;
  717. uint end_bit;
  718. uint mask;
  719. ASSERT(first < XFS_BLF_DATAMAP_SIZE * XFS_BLF_CHUNK * NBWORD);
  720. ASSERT(last < XFS_BLF_DATAMAP_SIZE * XFS_BLF_CHUNK * NBWORD);
  721. /*
  722. * Convert byte offsets to bit numbers.
  723. */
  724. first_bit = first >> XFS_BLF_SHIFT;
  725. last_bit = last >> XFS_BLF_SHIFT;
  726. /*
  727. * Calculate the total number of bits to be set.
  728. */
  729. bits_to_set = last_bit - first_bit + 1;
  730. /*
  731. * Get a pointer to the first word in the bitmap
  732. * to set a bit in.
  733. */
  734. word_num = first_bit >> BIT_TO_WORD_SHIFT;
  735. wordp = &map[word_num];
  736. /*
  737. * Calculate the starting bit in the first word.
  738. */
  739. bit = first_bit & (uint)(NBWORD - 1);
  740. /*
  741. * First set any bits in the first word of our range.
  742. * If it starts at bit 0 of the word, it will be
  743. * set below rather than here. That is what the variable
  744. * bit tells us. The variable bits_set tracks the number
  745. * of bits that have been set so far. End_bit is the number
  746. * of the last bit to be set in this word plus one.
  747. */
  748. if (bit) {
  749. end_bit = min(bit + bits_to_set, (uint)NBWORD);
  750. mask = ((1U << (end_bit - bit)) - 1) << bit;
  751. *wordp |= mask;
  752. wordp++;
  753. bits_set = end_bit - bit;
  754. } else {
  755. bits_set = 0;
  756. }
  757. /*
  758. * Now set bits a whole word at a time that are between
  759. * first_bit and last_bit.
  760. */
  761. while ((bits_to_set - bits_set) >= NBWORD) {
  762. *wordp = 0xffffffff;
  763. bits_set += NBWORD;
  764. wordp++;
  765. }
  766. /*
  767. * Finally, set any bits left to be set in one last partial word.
  768. */
  769. end_bit = bits_to_set - bits_set;
  770. if (end_bit) {
  771. mask = (1U << end_bit) - 1;
  772. *wordp |= mask;
  773. }
  774. }
  775. /*
  776. * Mark bytes first through last inclusive as dirty in the buf
  777. * item's bitmap.
  778. */
  779. void
  780. xfs_buf_item_log(
  781. struct xfs_buf_log_item *bip,
  782. uint first,
  783. uint last)
  784. {
  785. int i;
  786. uint start;
  787. uint end;
  788. struct xfs_buf *bp = bip->bli_buf;
  789. /*
  790. * walk each buffer segment and mark them dirty appropriately.
  791. */
  792. start = 0;
  793. for (i = 0; i < bip->bli_format_count; i++) {
  794. if (start > last)
  795. break;
  796. end = start + BBTOB(bp->b_maps[i].bm_len) - 1;
  797. /* skip to the map that includes the first byte to log */
  798. if (first > end) {
  799. start += BBTOB(bp->b_maps[i].bm_len);
  800. continue;
  801. }
  802. /*
  803. * Trim the range to this segment and mark it in the bitmap.
  804. * Note that we must convert buffer offsets to segment relative
  805. * offsets (e.g., the first byte of each segment is byte 0 of
  806. * that segment).
  807. */
  808. if (first < start)
  809. first = start;
  810. if (end > last)
  811. end = last;
  812. xfs_buf_item_log_segment(first - start, end - start,
  813. &bip->bli_formats[i].blf_data_map[0]);
  814. start += BBTOB(bp->b_maps[i].bm_len);
  815. }
  816. }
  817. /*
  818. * Return true if the buffer has any ranges logged/dirtied by a transaction,
  819. * false otherwise.
  820. */
  821. bool
  822. xfs_buf_item_dirty_format(
  823. struct xfs_buf_log_item *bip)
  824. {
  825. int i;
  826. for (i = 0; i < bip->bli_format_count; i++) {
  827. if (!xfs_bitmap_empty(bip->bli_formats[i].blf_data_map,
  828. bip->bli_formats[i].blf_map_size))
  829. return true;
  830. }
  831. return false;
  832. }
  833. STATIC void
  834. xfs_buf_item_free(
  835. struct xfs_buf_log_item *bip)
  836. {
  837. xfs_buf_item_free_format(bip);
  838. kmem_free(bip->bli_item.li_lv_shadow);
  839. kmem_cache_free(xfs_buf_item_zone, bip);
  840. }
  841. /*
  842. * xfs_buf_item_relse() is called when the buf log item is no longer needed.
  843. */
  844. void
  845. xfs_buf_item_relse(
  846. xfs_buf_t *bp)
  847. {
  848. struct xfs_buf_log_item *bip = bp->b_log_item;
  849. trace_xfs_buf_item_relse(bp, _RET_IP_);
  850. ASSERT(!test_bit(XFS_LI_IN_AIL, &bip->bli_item.li_flags));
  851. bp->b_log_item = NULL;
  852. xfs_buf_rele(bp);
  853. xfs_buf_item_free(bip);
  854. }
  855. void
  856. xfs_buf_item_done(
  857. struct xfs_buf *bp)
  858. {
  859. /*
  860. * If we are forcibly shutting down, this may well be off the AIL
  861. * already. That's because we simulate the log-committed callbacks to
  862. * unpin these buffers. Or we may never have put this item on AIL
  863. * because of the transaction was aborted forcibly.
  864. * xfs_trans_ail_delete() takes care of these.
  865. *
  866. * Either way, AIL is useless if we're forcing a shutdown.
  867. *
  868. * Note that log recovery writes might have buffer items that are not on
  869. * the AIL even when the file system is not shut down.
  870. */
  871. xfs_trans_ail_delete(&bp->b_log_item->bli_item,
  872. (bp->b_flags & _XBF_LOGRECOVERY) ? 0 :
  873. SHUTDOWN_CORRUPT_INCORE);
  874. xfs_buf_item_relse(bp);
  875. }