io.c 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2016 Trond Myklebust
  4. * Copyright (c) 2019 Jeff Layton
  5. *
  6. * I/O and data path helper functionality.
  7. *
  8. * Heavily borrowed from equivalent code in fs/nfs/io.c
  9. */
  10. #include <linux/ceph/ceph_debug.h>
  11. #include <linux/types.h>
  12. #include <linux/kernel.h>
  13. #include <linux/rwsem.h>
  14. #include <linux/fs.h>
  15. #include "super.h"
  16. #include "io.h"
  17. /* Call with exclusively locked inode->i_rwsem */
  18. static void ceph_block_o_direct(struct ceph_inode_info *ci, struct inode *inode)
  19. {
  20. lockdep_assert_held_write(&inode->i_rwsem);
  21. if (READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT) {
  22. spin_lock(&ci->i_ceph_lock);
  23. ci->i_ceph_flags &= ~CEPH_I_ODIRECT;
  24. spin_unlock(&ci->i_ceph_lock);
  25. inode_dio_wait(inode);
  26. }
  27. }
  28. /**
  29. * ceph_start_io_read - declare the file is being used for buffered reads
  30. * @inode: file inode
  31. *
  32. * Declare that a buffered read operation is about to start, and ensure
  33. * that we block all direct I/O.
  34. * On exit, the function ensures that the CEPH_I_ODIRECT flag is unset,
  35. * and holds a shared lock on inode->i_rwsem to ensure that the flag
  36. * cannot be changed.
  37. * In practice, this means that buffered read operations are allowed to
  38. * execute in parallel, thanks to the shared lock, whereas direct I/O
  39. * operations need to wait to grab an exclusive lock in order to set
  40. * CEPH_I_ODIRECT.
  41. * Note that buffered writes and truncates both take a write lock on
  42. * inode->i_rwsem, meaning that those are serialised w.r.t. the reads.
  43. */
  44. void
  45. ceph_start_io_read(struct inode *inode)
  46. {
  47. struct ceph_inode_info *ci = ceph_inode(inode);
  48. /* Be an optimist! */
  49. down_read(&inode->i_rwsem);
  50. if (!(READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT))
  51. return;
  52. up_read(&inode->i_rwsem);
  53. /* Slow path.... */
  54. down_write(&inode->i_rwsem);
  55. ceph_block_o_direct(ci, inode);
  56. downgrade_write(&inode->i_rwsem);
  57. }
  58. /**
  59. * ceph_end_io_read - declare that the buffered read operation is done
  60. * @inode: file inode
  61. *
  62. * Declare that a buffered read operation is done, and release the shared
  63. * lock on inode->i_rwsem.
  64. */
  65. void
  66. ceph_end_io_read(struct inode *inode)
  67. {
  68. up_read(&inode->i_rwsem);
  69. }
  70. /**
  71. * ceph_start_io_write - declare the file is being used for buffered writes
  72. * @inode: file inode
  73. *
  74. * Declare that a buffered write operation is about to start, and ensure
  75. * that we block all direct I/O.
  76. */
  77. void
  78. ceph_start_io_write(struct inode *inode)
  79. {
  80. down_write(&inode->i_rwsem);
  81. ceph_block_o_direct(ceph_inode(inode), inode);
  82. }
  83. /**
  84. * ceph_end_io_write - declare that the buffered write operation is done
  85. * @inode: file inode
  86. *
  87. * Declare that a buffered write operation is done, and release the
  88. * lock on inode->i_rwsem.
  89. */
  90. void
  91. ceph_end_io_write(struct inode *inode)
  92. {
  93. up_write(&inode->i_rwsem);
  94. }
  95. /* Call with exclusively locked inode->i_rwsem */
  96. static void ceph_block_buffered(struct ceph_inode_info *ci, struct inode *inode)
  97. {
  98. lockdep_assert_held_write(&inode->i_rwsem);
  99. if (!(READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT)) {
  100. spin_lock(&ci->i_ceph_lock);
  101. ci->i_ceph_flags |= CEPH_I_ODIRECT;
  102. spin_unlock(&ci->i_ceph_lock);
  103. /* FIXME: unmap_mapping_range? */
  104. filemap_write_and_wait(inode->i_mapping);
  105. }
  106. }
  107. /**
  108. * ceph_end_io_direct - declare the file is being used for direct i/o
  109. * @inode: file inode
  110. *
  111. * Declare that a direct I/O operation is about to start, and ensure
  112. * that we block all buffered I/O.
  113. * On exit, the function ensures that the CEPH_I_ODIRECT flag is set,
  114. * and holds a shared lock on inode->i_rwsem to ensure that the flag
  115. * cannot be changed.
  116. * In practice, this means that direct I/O operations are allowed to
  117. * execute in parallel, thanks to the shared lock, whereas buffered I/O
  118. * operations need to wait to grab an exclusive lock in order to clear
  119. * CEPH_I_ODIRECT.
  120. * Note that buffered writes and truncates both take a write lock on
  121. * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT.
  122. */
  123. void
  124. ceph_start_io_direct(struct inode *inode)
  125. {
  126. struct ceph_inode_info *ci = ceph_inode(inode);
  127. /* Be an optimist! */
  128. down_read(&inode->i_rwsem);
  129. if (READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT)
  130. return;
  131. up_read(&inode->i_rwsem);
  132. /* Slow path.... */
  133. down_write(&inode->i_rwsem);
  134. ceph_block_buffered(ci, inode);
  135. downgrade_write(&inode->i_rwsem);
  136. }
  137. /**
  138. * ceph_end_io_direct - declare that the direct i/o operation is done
  139. * @inode: file inode
  140. *
  141. * Declare that a direct I/O operation is done, and release the shared
  142. * lock on inode->i_rwsem.
  143. */
  144. void
  145. ceph_end_io_direct(struct inode *inode)
  146. {
  147. up_read(&inode->i_rwsem);
  148. }