123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668 |
- // SPDX-License-Identifier: GPL-2.0
- /*
- * Copyright (c) 2000-2005 Silicon Graphics, Inc.
- * Copyright (c) 2016-2018 Christoph Hellwig.
- * All Rights Reserved.
- */
- #include "xfs.h"
- #include "xfs_shared.h"
- #include "xfs_format.h"
- #include "xfs_log_format.h"
- #include "xfs_trans_resv.h"
- #include "xfs_mount.h"
- #include "xfs_inode.h"
- #include "xfs_trans.h"
- #include "xfs_iomap.h"
- #include "xfs_trace.h"
- #include "xfs_bmap.h"
- #include "xfs_bmap_util.h"
- #include "xfs_reflink.h"
- struct xfs_writepage_ctx {
- struct iomap_writepage_ctx ctx;
- unsigned int data_seq;
- unsigned int cow_seq;
- };
- static inline struct xfs_writepage_ctx *
- XFS_WPC(struct iomap_writepage_ctx *ctx)
- {
- return container_of(ctx, struct xfs_writepage_ctx, ctx);
- }
- /*
- * Fast and loose check if this write could update the on-disk inode size.
- */
- static inline bool xfs_ioend_is_append(struct iomap_ioend *ioend)
- {
- return ioend->io_offset + ioend->io_size >
- XFS_I(ioend->io_inode)->i_d.di_size;
- }
- STATIC int
- xfs_setfilesize_trans_alloc(
- struct iomap_ioend *ioend)
- {
- struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount;
- struct xfs_trans *tp;
- int error;
- error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp);
- if (error)
- return error;
- ioend->io_private = tp;
- /*
- * We may pass freeze protection with a transaction. So tell lockdep
- * we released it.
- */
- __sb_writers_release(ioend->io_inode->i_sb, SB_FREEZE_FS);
- /*
- * We hand off the transaction to the completion thread now, so
- * clear the flag here.
- */
- current_restore_flags_nested(&tp->t_pflags, PF_MEMALLOC_NOFS);
- return 0;
- }
- /*
- * Update on-disk file size now that data has been written to disk.
- */
- STATIC int
- __xfs_setfilesize(
- struct xfs_inode *ip,
- struct xfs_trans *tp,
- xfs_off_t offset,
- size_t size)
- {
- xfs_fsize_t isize;
- xfs_ilock(ip, XFS_ILOCK_EXCL);
- isize = xfs_new_eof(ip, offset + size);
- if (!isize) {
- xfs_iunlock(ip, XFS_ILOCK_EXCL);
- xfs_trans_cancel(tp);
- return 0;
- }
- trace_xfs_setfilesize(ip, offset, size);
- ip->i_d.di_size = isize;
- xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
- xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
- return xfs_trans_commit(tp);
- }
- int
- xfs_setfilesize(
- struct xfs_inode *ip,
- xfs_off_t offset,
- size_t size)
- {
- struct xfs_mount *mp = ip->i_mount;
- struct xfs_trans *tp;
- int error;
- error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp);
- if (error)
- return error;
- return __xfs_setfilesize(ip, tp, offset, size);
- }
- STATIC int
- xfs_setfilesize_ioend(
- struct iomap_ioend *ioend,
- int error)
- {
- struct xfs_inode *ip = XFS_I(ioend->io_inode);
- struct xfs_trans *tp = ioend->io_private;
- /*
- * The transaction may have been allocated in the I/O submission thread,
- * thus we need to mark ourselves as being in a transaction manually.
- * Similarly for freeze protection.
- */
- current_set_flags_nested(&tp->t_pflags, PF_MEMALLOC_NOFS);
- __sb_writers_acquired(VFS_I(ip)->i_sb, SB_FREEZE_FS);
- /* we abort the update if there was an IO error */
- if (error) {
- xfs_trans_cancel(tp);
- return error;
- }
- return __xfs_setfilesize(ip, tp, ioend->io_offset, ioend->io_size);
- }
- /*
- * IO write completion.
- */
- STATIC void
- xfs_end_ioend(
- struct iomap_ioend *ioend)
- {
- struct xfs_inode *ip = XFS_I(ioend->io_inode);
- xfs_off_t offset = ioend->io_offset;
- size_t size = ioend->io_size;
- unsigned int nofs_flag;
- int error;
- /*
- * We can allocate memory here while doing writeback on behalf of
- * memory reclaim. To avoid memory allocation deadlocks set the
- * task-wide nofs context for the following operations.
- */
- nofs_flag = memalloc_nofs_save();
- /*
- * Just clean up the in-memory strutures if the fs has been shut down.
- */
- if (XFS_FORCED_SHUTDOWN(ip->i_mount)) {
- error = -EIO;
- goto done;
- }
- /*
- * Clean up any COW blocks on an I/O error.
- */
- error = blk_status_to_errno(ioend->io_bio->bi_status);
- if (unlikely(error)) {
- if (ioend->io_flags & IOMAP_F_SHARED)
- xfs_reflink_cancel_cow_range(ip, offset, size, true);
- goto done;
- }
- /*
- * Success: commit the COW or unwritten blocks if needed.
- */
- if (ioend->io_flags & IOMAP_F_SHARED)
- error = xfs_reflink_end_cow(ip, offset, size);
- else if (ioend->io_type == IOMAP_UNWRITTEN)
- error = xfs_iomap_write_unwritten(ip, offset, size, false);
- else
- ASSERT(!xfs_ioend_is_append(ioend) || ioend->io_private);
- done:
- if (ioend->io_private)
- error = xfs_setfilesize_ioend(ioend, error);
- iomap_finish_ioends(ioend, error);
- memalloc_nofs_restore(nofs_flag);
- }
- /*
- * If the to be merged ioend has a preallocated transaction for file
- * size updates we need to ensure the ioend it is merged into also
- * has one. If it already has one we can simply cancel the transaction
- * as it is guaranteed to be clean.
- */
- static void
- xfs_ioend_merge_private(
- struct iomap_ioend *ioend,
- struct iomap_ioend *next)
- {
- if (!ioend->io_private) {
- ioend->io_private = next->io_private;
- next->io_private = NULL;
- } else {
- xfs_setfilesize_ioend(next, -ECANCELED);
- }
- }
- /* Finish all pending io completions. */
- void
- xfs_end_io(
- struct work_struct *work)
- {
- struct xfs_inode *ip =
- container_of(work, struct xfs_inode, i_ioend_work);
- struct iomap_ioend *ioend;
- struct list_head tmp;
- unsigned long flags;
- spin_lock_irqsave(&ip->i_ioend_lock, flags);
- list_replace_init(&ip->i_ioend_list, &tmp);
- spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
- iomap_sort_ioends(&tmp);
- while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
- io_list))) {
- list_del_init(&ioend->io_list);
- iomap_ioend_try_merge(ioend, &tmp, xfs_ioend_merge_private);
- xfs_end_ioend(ioend);
- }
- }
- static inline bool xfs_ioend_needs_workqueue(struct iomap_ioend *ioend)
- {
- return ioend->io_private ||
- ioend->io_type == IOMAP_UNWRITTEN ||
- (ioend->io_flags & IOMAP_F_SHARED);
- }
- STATIC void
- xfs_end_bio(
- struct bio *bio)
- {
- struct iomap_ioend *ioend = bio->bi_private;
- struct xfs_inode *ip = XFS_I(ioend->io_inode);
- unsigned long flags;
- ASSERT(xfs_ioend_needs_workqueue(ioend));
- spin_lock_irqsave(&ip->i_ioend_lock, flags);
- if (list_empty(&ip->i_ioend_list))
- WARN_ON_ONCE(!queue_work(ip->i_mount->m_unwritten_workqueue,
- &ip->i_ioend_work));
- list_add_tail(&ioend->io_list, &ip->i_ioend_list);
- spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
- }
- /*
- * Fast revalidation of the cached writeback mapping. Return true if the current
- * mapping is valid, false otherwise.
- */
- static bool
- xfs_imap_valid(
- struct iomap_writepage_ctx *wpc,
- struct xfs_inode *ip,
- loff_t offset)
- {
- if (offset < wpc->iomap.offset ||
- offset >= wpc->iomap.offset + wpc->iomap.length)
- return false;
- /*
- * If this is a COW mapping, it is sufficient to check that the mapping
- * covers the offset. Be careful to check this first because the caller
- * can revalidate a COW mapping without updating the data seqno.
- */
- if (wpc->iomap.flags & IOMAP_F_SHARED)
- return true;
- /*
- * This is not a COW mapping. Check the sequence number of the data fork
- * because concurrent changes could have invalidated the extent. Check
- * the COW fork because concurrent changes since the last time we
- * checked (and found nothing at this offset) could have added
- * overlapping blocks.
- */
- if (XFS_WPC(wpc)->data_seq != READ_ONCE(ip->i_df.if_seq))
- return false;
- if (xfs_inode_has_cow_data(ip) &&
- XFS_WPC(wpc)->cow_seq != READ_ONCE(ip->i_cowfp->if_seq))
- return false;
- return true;
- }
- /*
- * Pass in a dellalloc extent and convert it to real extents, return the real
- * extent that maps offset_fsb in wpc->iomap.
- *
- * The current page is held locked so nothing could have removed the block
- * backing offset_fsb, although it could have moved from the COW to the data
- * fork by another thread.
- */
- static int
- xfs_convert_blocks(
- struct iomap_writepage_ctx *wpc,
- struct xfs_inode *ip,
- int whichfork,
- loff_t offset)
- {
- int error;
- unsigned *seq;
- if (whichfork == XFS_COW_FORK)
- seq = &XFS_WPC(wpc)->cow_seq;
- else
- seq = &XFS_WPC(wpc)->data_seq;
- /*
- * Attempt to allocate whatever delalloc extent currently backs offset
- * and put the result into wpc->iomap. Allocate in a loop because it
- * may take several attempts to allocate real blocks for a contiguous
- * delalloc extent if free space is sufficiently fragmented.
- */
- do {
- error = xfs_bmapi_convert_delalloc(ip, whichfork, offset,
- &wpc->iomap, seq);
- if (error)
- return error;
- } while (wpc->iomap.offset + wpc->iomap.length <= offset);
- return 0;
- }
- static int
- xfs_map_blocks(
- struct iomap_writepage_ctx *wpc,
- struct inode *inode,
- loff_t offset)
- {
- struct xfs_inode *ip = XFS_I(inode);
- struct xfs_mount *mp = ip->i_mount;
- ssize_t count = i_blocksize(inode);
- xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
- xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count);
- xfs_fileoff_t cow_fsb;
- int whichfork;
- struct xfs_bmbt_irec imap;
- struct xfs_iext_cursor icur;
- int retries = 0;
- int error = 0;
- if (XFS_FORCED_SHUTDOWN(mp))
- return -EIO;
- /*
- * COW fork blocks can overlap data fork blocks even if the blocks
- * aren't shared. COW I/O always takes precedent, so we must always
- * check for overlap on reflink inodes unless the mapping is already a
- * COW one, or the COW fork hasn't changed from the last time we looked
- * at it.
- *
- * It's safe to check the COW fork if_seq here without the ILOCK because
- * we've indirectly protected against concurrent updates: writeback has
- * the page locked, which prevents concurrent invalidations by reflink
- * and directio and prevents concurrent buffered writes to the same
- * page. Changes to if_seq always happen under i_lock, which protects
- * against concurrent updates and provides a memory barrier on the way
- * out that ensures that we always see the current value.
- */
- if (xfs_imap_valid(wpc, ip, offset))
- return 0;
- /*
- * If we don't have a valid map, now it's time to get a new one for this
- * offset. This will convert delayed allocations (including COW ones)
- * into real extents. If we return without a valid map, it means we
- * landed in a hole and we skip the block.
- */
- retry:
- cow_fsb = NULLFILEOFF;
- whichfork = XFS_DATA_FORK;
- xfs_ilock(ip, XFS_ILOCK_SHARED);
- ASSERT(ip->i_df.if_format != XFS_DINODE_FMT_BTREE ||
- (ip->i_df.if_flags & XFS_IFEXTENTS));
- /*
- * Check if this is offset is covered by a COW extents, and if yes use
- * it directly instead of looking up anything in the data fork.
- */
- if (xfs_inode_has_cow_data(ip) &&
- xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap))
- cow_fsb = imap.br_startoff;
- if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
- XFS_WPC(wpc)->cow_seq = READ_ONCE(ip->i_cowfp->if_seq);
- xfs_iunlock(ip, XFS_ILOCK_SHARED);
- whichfork = XFS_COW_FORK;
- goto allocate_blocks;
- }
- /*
- * No COW extent overlap. Revalidate now that we may have updated
- * ->cow_seq. If the data mapping is still valid, we're done.
- */
- if (xfs_imap_valid(wpc, ip, offset)) {
- xfs_iunlock(ip, XFS_ILOCK_SHARED);
- return 0;
- }
- /*
- * If we don't have a valid map, now it's time to get a new one for this
- * offset. This will convert delayed allocations (including COW ones)
- * into real extents.
- */
- if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap))
- imap.br_startoff = end_fsb; /* fake a hole past EOF */
- XFS_WPC(wpc)->data_seq = READ_ONCE(ip->i_df.if_seq);
- xfs_iunlock(ip, XFS_ILOCK_SHARED);
- /* landed in a hole or beyond EOF? */
- if (imap.br_startoff > offset_fsb) {
- imap.br_blockcount = imap.br_startoff - offset_fsb;
- imap.br_startoff = offset_fsb;
- imap.br_startblock = HOLESTARTBLOCK;
- imap.br_state = XFS_EXT_NORM;
- }
- /*
- * Truncate to the next COW extent if there is one. This is the only
- * opportunity to do this because we can skip COW fork lookups for the
- * subsequent blocks in the mapping; however, the requirement to treat
- * the COW range separately remains.
- */
- if (cow_fsb != NULLFILEOFF &&
- cow_fsb < imap.br_startoff + imap.br_blockcount)
- imap.br_blockcount = cow_fsb - imap.br_startoff;
- /* got a delalloc extent? */
- if (imap.br_startblock != HOLESTARTBLOCK &&
- isnullstartblock(imap.br_startblock))
- goto allocate_blocks;
- xfs_bmbt_to_iomap(ip, &wpc->iomap, &imap, 0);
- trace_xfs_map_blocks_found(ip, offset, count, whichfork, &imap);
- return 0;
- allocate_blocks:
- error = xfs_convert_blocks(wpc, ip, whichfork, offset);
- if (error) {
- /*
- * If we failed to find the extent in the COW fork we might have
- * raced with a COW to data fork conversion or truncate.
- * Restart the lookup to catch the extent in the data fork for
- * the former case, but prevent additional retries to avoid
- * looping forever for the latter case.
- */
- if (error == -EAGAIN && whichfork == XFS_COW_FORK && !retries++)
- goto retry;
- ASSERT(error != -EAGAIN);
- return error;
- }
- /*
- * Due to merging the return real extent might be larger than the
- * original delalloc one. Trim the return extent to the next COW
- * boundary again to force a re-lookup.
- */
- if (whichfork != XFS_COW_FORK && cow_fsb != NULLFILEOFF) {
- loff_t cow_offset = XFS_FSB_TO_B(mp, cow_fsb);
- if (cow_offset < wpc->iomap.offset + wpc->iomap.length)
- wpc->iomap.length = cow_offset - wpc->iomap.offset;
- }
- ASSERT(wpc->iomap.offset <= offset);
- ASSERT(wpc->iomap.offset + wpc->iomap.length > offset);
- trace_xfs_map_blocks_alloc(ip, offset, count, whichfork, &imap);
- return 0;
- }
- static int
- xfs_prepare_ioend(
- struct iomap_ioend *ioend,
- int status)
- {
- unsigned int nofs_flag;
- /*
- * We can allocate memory here while doing writeback on behalf of
- * memory reclaim. To avoid memory allocation deadlocks set the
- * task-wide nofs context for the following operations.
- */
- nofs_flag = memalloc_nofs_save();
- /* Convert CoW extents to regular */
- if (!status && (ioend->io_flags & IOMAP_F_SHARED)) {
- status = xfs_reflink_convert_cow(XFS_I(ioend->io_inode),
- ioend->io_offset, ioend->io_size);
- }
- /* Reserve log space if we might write beyond the on-disk inode size. */
- if (!status &&
- ((ioend->io_flags & IOMAP_F_SHARED) ||
- ioend->io_type != IOMAP_UNWRITTEN) &&
- xfs_ioend_is_append(ioend) &&
- !ioend->io_private)
- status = xfs_setfilesize_trans_alloc(ioend);
- memalloc_nofs_restore(nofs_flag);
- if (xfs_ioend_needs_workqueue(ioend))
- ioend->io_bio->bi_end_io = xfs_end_bio;
- return status;
- }
- /*
- * If the page has delalloc blocks on it, we need to punch them out before we
- * invalidate the page. If we don't, we leave a stale delalloc mapping on the
- * inode that can trip up a later direct I/O read operation on the same region.
- *
- * We prevent this by truncating away the delalloc regions on the page. Because
- * they are delalloc, we can do this without needing a transaction. Indeed - if
- * we get ENOSPC errors, we have to be able to do this truncation without a
- * transaction as there is no space left for block reservation (typically why we
- * see a ENOSPC in writeback).
- */
- static void
- xfs_discard_page(
- struct page *page,
- loff_t fileoff)
- {
- struct inode *inode = page->mapping->host;
- struct xfs_inode *ip = XFS_I(inode);
- struct xfs_mount *mp = ip->i_mount;
- unsigned int pageoff = offset_in_page(fileoff);
- xfs_fileoff_t start_fsb = XFS_B_TO_FSBT(mp, fileoff);
- xfs_fileoff_t pageoff_fsb = XFS_B_TO_FSBT(mp, pageoff);
- int error;
- if (XFS_FORCED_SHUTDOWN(mp))
- goto out_invalidate;
- xfs_alert_ratelimited(mp,
- "page discard on page "PTR_FMT", inode 0x%llx, offset %llu.",
- page, ip->i_ino, fileoff);
- error = xfs_bmap_punch_delalloc_range(ip, start_fsb,
- i_blocks_per_page(inode, page) - pageoff_fsb);
- if (error && !XFS_FORCED_SHUTDOWN(mp))
- xfs_alert(mp, "page discard unable to remove delalloc mapping.");
- out_invalidate:
- iomap_invalidatepage(page, pageoff, PAGE_SIZE - pageoff);
- }
- static const struct iomap_writeback_ops xfs_writeback_ops = {
- .map_blocks = xfs_map_blocks,
- .prepare_ioend = xfs_prepare_ioend,
- .discard_page = xfs_discard_page,
- };
- STATIC int
- xfs_vm_writepage(
- struct page *page,
- struct writeback_control *wbc)
- {
- struct xfs_writepage_ctx wpc = { };
- return iomap_writepage(page, wbc, &wpc.ctx, &xfs_writeback_ops);
- }
- STATIC int
- xfs_vm_writepages(
- struct address_space *mapping,
- struct writeback_control *wbc)
- {
- struct xfs_writepage_ctx wpc = { };
- xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
- return iomap_writepages(mapping, wbc, &wpc.ctx, &xfs_writeback_ops);
- }
- STATIC int
- xfs_dax_writepages(
- struct address_space *mapping,
- struct writeback_control *wbc)
- {
- struct xfs_inode *ip = XFS_I(mapping->host);
- xfs_iflags_clear(ip, XFS_ITRUNCATED);
- return dax_writeback_mapping_range(mapping,
- xfs_inode_buftarg(ip)->bt_daxdev, wbc);
- }
- STATIC sector_t
- xfs_vm_bmap(
- struct address_space *mapping,
- sector_t block)
- {
- struct xfs_inode *ip = XFS_I(mapping->host);
- trace_xfs_vm_bmap(ip);
- /*
- * The swap code (ab-)uses ->bmap to get a block mapping and then
- * bypasses the file system for actual I/O. We really can't allow
- * that on reflinks inodes, so we have to skip out here. And yes,
- * 0 is the magic code for a bmap error.
- *
- * Since we don't pass back blockdev info, we can't return bmap
- * information for rt files either.
- */
- if (xfs_is_cow_inode(ip) || XFS_IS_REALTIME_INODE(ip))
- return 0;
- return iomap_bmap(mapping, block, &xfs_read_iomap_ops);
- }
- STATIC int
- xfs_vm_readpage(
- struct file *unused,
- struct page *page)
- {
- return iomap_readpage(page, &xfs_read_iomap_ops);
- }
- STATIC void
- xfs_vm_readahead(
- struct readahead_control *rac)
- {
- iomap_readahead(rac, &xfs_read_iomap_ops);
- }
- static int
- xfs_iomap_swapfile_activate(
- struct swap_info_struct *sis,
- struct file *swap_file,
- sector_t *span)
- {
- sis->bdev = xfs_inode_buftarg(XFS_I(file_inode(swap_file)))->bt_bdev;
- return iomap_swapfile_activate(sis, swap_file, span,
- &xfs_read_iomap_ops);
- }
- const struct address_space_operations xfs_address_space_operations = {
- .readpage = xfs_vm_readpage,
- .readahead = xfs_vm_readahead,
- .writepage = xfs_vm_writepage,
- .writepages = xfs_vm_writepages,
- .set_page_dirty = iomap_set_page_dirty,
- .releasepage = iomap_releasepage,
- .invalidatepage = iomap_invalidatepage,
- .bmap = xfs_vm_bmap,
- .direct_IO = noop_direct_IO,
- .migratepage = iomap_migrate_page,
- .is_partially_uptodate = iomap_is_partially_uptodate,
- .error_remove_page = generic_error_remove_page,
- .swap_activate = xfs_iomap_swapfile_activate,
- };
- const struct address_space_operations xfs_dax_aops = {
- .writepages = xfs_dax_writepages,
- .direct_IO = noop_direct_IO,
- .set_page_dirty = noop_set_page_dirty,
- .invalidatepage = noop_invalidatepage,
- .swap_activate = xfs_iomap_swapfile_activate,
- };
|