xfs_buf.c 58 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2000-2006 Silicon Graphics, Inc.
  4. * All Rights Reserved.
  5. */
  6. #include "xfs.h"
  7. #include <linux/backing-dev.h>
  8. #include "xfs_shared.h"
  9. #include "xfs_format.h"
  10. #include "xfs_log_format.h"
  11. #include "xfs_trans_resv.h"
  12. #include "xfs_sb.h"
  13. #include "xfs_mount.h"
  14. #include "xfs_trace.h"
  15. #include "xfs_log.h"
  16. #include "xfs_log_recover.h"
  17. #include "xfs_trans.h"
  18. #include "xfs_buf_item.h"
  19. #include "xfs_errortag.h"
  20. #include "xfs_error.h"
  21. static kmem_zone_t *xfs_buf_zone;
  22. #define xb_to_gfp(flags) \
  23. ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : GFP_NOFS) | __GFP_NOWARN)
  24. /*
  25. * Locking orders
  26. *
  27. * xfs_buf_ioacct_inc:
  28. * xfs_buf_ioacct_dec:
  29. * b_sema (caller holds)
  30. * b_lock
  31. *
  32. * xfs_buf_stale:
  33. * b_sema (caller holds)
  34. * b_lock
  35. * lru_lock
  36. *
  37. * xfs_buf_rele:
  38. * b_lock
  39. * pag_buf_lock
  40. * lru_lock
  41. *
  42. * xfs_buftarg_wait_rele
  43. * lru_lock
  44. * b_lock (trylock due to inversion)
  45. *
  46. * xfs_buftarg_isolate
  47. * lru_lock
  48. * b_lock (trylock due to inversion)
  49. */
  50. static int __xfs_buf_submit(struct xfs_buf *bp, bool wait);
  51. static inline int
  52. xfs_buf_submit(
  53. struct xfs_buf *bp)
  54. {
  55. return __xfs_buf_submit(bp, !(bp->b_flags & XBF_ASYNC));
  56. }
  57. static inline int
  58. xfs_buf_is_vmapped(
  59. struct xfs_buf *bp)
  60. {
  61. /*
  62. * Return true if the buffer is vmapped.
  63. *
  64. * b_addr is null if the buffer is not mapped, but the code is clever
  65. * enough to know it doesn't have to map a single page, so the check has
  66. * to be both for b_addr and bp->b_page_count > 1.
  67. */
  68. return bp->b_addr && bp->b_page_count > 1;
  69. }
  70. static inline int
  71. xfs_buf_vmap_len(
  72. struct xfs_buf *bp)
  73. {
  74. return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
  75. }
  76. /*
  77. * Bump the I/O in flight count on the buftarg if we haven't yet done so for
  78. * this buffer. The count is incremented once per buffer (per hold cycle)
  79. * because the corresponding decrement is deferred to buffer release. Buffers
  80. * can undergo I/O multiple times in a hold-release cycle and per buffer I/O
  81. * tracking adds unnecessary overhead. This is used for sychronization purposes
  82. * with unmount (see xfs_wait_buftarg()), so all we really need is a count of
  83. * in-flight buffers.
  84. *
  85. * Buffers that are never released (e.g., superblock, iclog buffers) must set
  86. * the XBF_NO_IOACCT flag before I/O submission. Otherwise, the buftarg count
  87. * never reaches zero and unmount hangs indefinitely.
  88. */
  89. static inline void
  90. xfs_buf_ioacct_inc(
  91. struct xfs_buf *bp)
  92. {
  93. if (bp->b_flags & XBF_NO_IOACCT)
  94. return;
  95. ASSERT(bp->b_flags & XBF_ASYNC);
  96. spin_lock(&bp->b_lock);
  97. if (!(bp->b_state & XFS_BSTATE_IN_FLIGHT)) {
  98. bp->b_state |= XFS_BSTATE_IN_FLIGHT;
  99. percpu_counter_inc(&bp->b_target->bt_io_count);
  100. }
  101. spin_unlock(&bp->b_lock);
  102. }
  103. /*
  104. * Clear the in-flight state on a buffer about to be released to the LRU or
  105. * freed and unaccount from the buftarg.
  106. */
  107. static inline void
  108. __xfs_buf_ioacct_dec(
  109. struct xfs_buf *bp)
  110. {
  111. lockdep_assert_held(&bp->b_lock);
  112. if (bp->b_state & XFS_BSTATE_IN_FLIGHT) {
  113. bp->b_state &= ~XFS_BSTATE_IN_FLIGHT;
  114. percpu_counter_dec(&bp->b_target->bt_io_count);
  115. }
  116. }
  117. static inline void
  118. xfs_buf_ioacct_dec(
  119. struct xfs_buf *bp)
  120. {
  121. spin_lock(&bp->b_lock);
  122. __xfs_buf_ioacct_dec(bp);
  123. spin_unlock(&bp->b_lock);
  124. }
  125. /*
  126. * When we mark a buffer stale, we remove the buffer from the LRU and clear the
  127. * b_lru_ref count so that the buffer is freed immediately when the buffer
  128. * reference count falls to zero. If the buffer is already on the LRU, we need
  129. * to remove the reference that LRU holds on the buffer.
  130. *
  131. * This prevents build-up of stale buffers on the LRU.
  132. */
  133. void
  134. xfs_buf_stale(
  135. struct xfs_buf *bp)
  136. {
  137. ASSERT(xfs_buf_islocked(bp));
  138. bp->b_flags |= XBF_STALE;
  139. /*
  140. * Clear the delwri status so that a delwri queue walker will not
  141. * flush this buffer to disk now that it is stale. The delwri queue has
  142. * a reference to the buffer, so this is safe to do.
  143. */
  144. bp->b_flags &= ~_XBF_DELWRI_Q;
  145. /*
  146. * Once the buffer is marked stale and unlocked, a subsequent lookup
  147. * could reset b_flags. There is no guarantee that the buffer is
  148. * unaccounted (released to LRU) before that occurs. Drop in-flight
  149. * status now to preserve accounting consistency.
  150. */
  151. spin_lock(&bp->b_lock);
  152. __xfs_buf_ioacct_dec(bp);
  153. atomic_set(&bp->b_lru_ref, 0);
  154. if (!(bp->b_state & XFS_BSTATE_DISPOSE) &&
  155. (list_lru_del(&bp->b_target->bt_lru, &bp->b_lru)))
  156. atomic_dec(&bp->b_hold);
  157. ASSERT(atomic_read(&bp->b_hold) >= 1);
  158. spin_unlock(&bp->b_lock);
  159. }
  160. static int
  161. xfs_buf_get_maps(
  162. struct xfs_buf *bp,
  163. int map_count)
  164. {
  165. ASSERT(bp->b_maps == NULL);
  166. bp->b_map_count = map_count;
  167. if (map_count == 1) {
  168. bp->b_maps = &bp->__b_map;
  169. return 0;
  170. }
  171. bp->b_maps = kmem_zalloc(map_count * sizeof(struct xfs_buf_map),
  172. KM_NOFS);
  173. if (!bp->b_maps)
  174. return -ENOMEM;
  175. return 0;
  176. }
  177. /*
  178. * Frees b_pages if it was allocated.
  179. */
  180. static void
  181. xfs_buf_free_maps(
  182. struct xfs_buf *bp)
  183. {
  184. if (bp->b_maps != &bp->__b_map) {
  185. kmem_free(bp->b_maps);
  186. bp->b_maps = NULL;
  187. }
  188. }
  189. static int
  190. _xfs_buf_alloc(
  191. struct xfs_buftarg *target,
  192. struct xfs_buf_map *map,
  193. int nmaps,
  194. xfs_buf_flags_t flags,
  195. struct xfs_buf **bpp)
  196. {
  197. struct xfs_buf *bp;
  198. int error;
  199. int i;
  200. *bpp = NULL;
  201. bp = kmem_cache_zalloc(xfs_buf_zone, GFP_NOFS | __GFP_NOFAIL);
  202. /*
  203. * We don't want certain flags to appear in b_flags unless they are
  204. * specifically set by later operations on the buffer.
  205. */
  206. flags &= ~(XBF_UNMAPPED | XBF_TRYLOCK | XBF_ASYNC | XBF_READ_AHEAD);
  207. atomic_set(&bp->b_hold, 1);
  208. atomic_set(&bp->b_lru_ref, 1);
  209. init_completion(&bp->b_iowait);
  210. INIT_LIST_HEAD(&bp->b_lru);
  211. INIT_LIST_HEAD(&bp->b_list);
  212. INIT_LIST_HEAD(&bp->b_li_list);
  213. sema_init(&bp->b_sema, 0); /* held, no waiters */
  214. spin_lock_init(&bp->b_lock);
  215. bp->b_target = target;
  216. bp->b_mount = target->bt_mount;
  217. bp->b_flags = flags;
  218. /*
  219. * Set length and io_length to the same value initially.
  220. * I/O routines should use io_length, which will be the same in
  221. * most cases but may be reset (e.g. XFS recovery).
  222. */
  223. error = xfs_buf_get_maps(bp, nmaps);
  224. if (error) {
  225. kmem_cache_free(xfs_buf_zone, bp);
  226. return error;
  227. }
  228. bp->b_bn = map[0].bm_bn;
  229. bp->b_length = 0;
  230. for (i = 0; i < nmaps; i++) {
  231. bp->b_maps[i].bm_bn = map[i].bm_bn;
  232. bp->b_maps[i].bm_len = map[i].bm_len;
  233. bp->b_length += map[i].bm_len;
  234. }
  235. atomic_set(&bp->b_pin_count, 0);
  236. init_waitqueue_head(&bp->b_waiters);
  237. XFS_STATS_INC(bp->b_mount, xb_create);
  238. trace_xfs_buf_init(bp, _RET_IP_);
  239. *bpp = bp;
  240. return 0;
  241. }
  242. /*
  243. * Allocate a page array capable of holding a specified number
  244. * of pages, and point the page buf at it.
  245. */
  246. STATIC int
  247. _xfs_buf_get_pages(
  248. xfs_buf_t *bp,
  249. int page_count)
  250. {
  251. /* Make sure that we have a page list */
  252. if (bp->b_pages == NULL) {
  253. bp->b_page_count = page_count;
  254. if (page_count <= XB_PAGES) {
  255. bp->b_pages = bp->b_page_array;
  256. } else {
  257. bp->b_pages = kmem_alloc(sizeof(struct page *) *
  258. page_count, KM_NOFS);
  259. if (bp->b_pages == NULL)
  260. return -ENOMEM;
  261. }
  262. memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
  263. }
  264. return 0;
  265. }
  266. /*
  267. * Frees b_pages if it was allocated.
  268. */
  269. STATIC void
  270. _xfs_buf_free_pages(
  271. xfs_buf_t *bp)
  272. {
  273. if (bp->b_pages != bp->b_page_array) {
  274. kmem_free(bp->b_pages);
  275. bp->b_pages = NULL;
  276. }
  277. }
  278. /*
  279. * Releases the specified buffer.
  280. *
  281. * The modification state of any associated pages is left unchanged.
  282. * The buffer must not be on any hash - use xfs_buf_rele instead for
  283. * hashed and refcounted buffers
  284. */
  285. static void
  286. xfs_buf_free(
  287. xfs_buf_t *bp)
  288. {
  289. trace_xfs_buf_free(bp, _RET_IP_);
  290. ASSERT(list_empty(&bp->b_lru));
  291. if (bp->b_flags & _XBF_PAGES) {
  292. uint i;
  293. if (xfs_buf_is_vmapped(bp))
  294. vm_unmap_ram(bp->b_addr - bp->b_offset,
  295. bp->b_page_count);
  296. for (i = 0; i < bp->b_page_count; i++) {
  297. struct page *page = bp->b_pages[i];
  298. __free_page(page);
  299. }
  300. if (current->reclaim_state)
  301. current->reclaim_state->reclaimed_slab +=
  302. bp->b_page_count;
  303. } else if (bp->b_flags & _XBF_KMEM)
  304. kmem_free(bp->b_addr);
  305. _xfs_buf_free_pages(bp);
  306. xfs_buf_free_maps(bp);
  307. kmem_cache_free(xfs_buf_zone, bp);
  308. }
  309. /*
  310. * Allocates all the pages for buffer in question and builds it's page list.
  311. */
  312. STATIC int
  313. xfs_buf_allocate_memory(
  314. xfs_buf_t *bp,
  315. uint flags)
  316. {
  317. size_t size;
  318. size_t nbytes, offset;
  319. gfp_t gfp_mask = xb_to_gfp(flags);
  320. unsigned short page_count, i;
  321. xfs_off_t start, end;
  322. int error;
  323. xfs_km_flags_t kmflag_mask = 0;
  324. /*
  325. * assure zeroed buffer for non-read cases.
  326. */
  327. if (!(flags & XBF_READ)) {
  328. kmflag_mask |= KM_ZERO;
  329. gfp_mask |= __GFP_ZERO;
  330. }
  331. /*
  332. * for buffers that are contained within a single page, just allocate
  333. * the memory from the heap - there's no need for the complexity of
  334. * page arrays to keep allocation down to order 0.
  335. */
  336. size = BBTOB(bp->b_length);
  337. if (size < PAGE_SIZE) {
  338. int align_mask = xfs_buftarg_dma_alignment(bp->b_target);
  339. bp->b_addr = kmem_alloc_io(size, align_mask,
  340. KM_NOFS | kmflag_mask);
  341. if (!bp->b_addr) {
  342. /* low memory - use alloc_page loop instead */
  343. goto use_alloc_page;
  344. }
  345. if (((unsigned long)(bp->b_addr + size - 1) & PAGE_MASK) !=
  346. ((unsigned long)bp->b_addr & PAGE_MASK)) {
  347. /* b_addr spans two pages - use alloc_page instead */
  348. kmem_free(bp->b_addr);
  349. bp->b_addr = NULL;
  350. goto use_alloc_page;
  351. }
  352. bp->b_offset = offset_in_page(bp->b_addr);
  353. bp->b_pages = bp->b_page_array;
  354. bp->b_pages[0] = kmem_to_page(bp->b_addr);
  355. bp->b_page_count = 1;
  356. bp->b_flags |= _XBF_KMEM;
  357. return 0;
  358. }
  359. use_alloc_page:
  360. start = BBTOB(bp->b_maps[0].bm_bn) >> PAGE_SHIFT;
  361. end = (BBTOB(bp->b_maps[0].bm_bn + bp->b_length) + PAGE_SIZE - 1)
  362. >> PAGE_SHIFT;
  363. page_count = end - start;
  364. error = _xfs_buf_get_pages(bp, page_count);
  365. if (unlikely(error))
  366. return error;
  367. offset = bp->b_offset;
  368. bp->b_flags |= _XBF_PAGES;
  369. for (i = 0; i < bp->b_page_count; i++) {
  370. struct page *page;
  371. uint retries = 0;
  372. retry:
  373. page = alloc_page(gfp_mask);
  374. if (unlikely(page == NULL)) {
  375. if (flags & XBF_READ_AHEAD) {
  376. bp->b_page_count = i;
  377. error = -ENOMEM;
  378. goto out_free_pages;
  379. }
  380. /*
  381. * This could deadlock.
  382. *
  383. * But until all the XFS lowlevel code is revamped to
  384. * handle buffer allocation failures we can't do much.
  385. */
  386. if (!(++retries % 100))
  387. xfs_err(NULL,
  388. "%s(%u) possible memory allocation deadlock in %s (mode:0x%x)",
  389. current->comm, current->pid,
  390. __func__, gfp_mask);
  391. XFS_STATS_INC(bp->b_mount, xb_page_retries);
  392. congestion_wait(BLK_RW_ASYNC, HZ/50);
  393. goto retry;
  394. }
  395. XFS_STATS_INC(bp->b_mount, xb_page_found);
  396. nbytes = min_t(size_t, size, PAGE_SIZE - offset);
  397. size -= nbytes;
  398. bp->b_pages[i] = page;
  399. offset = 0;
  400. }
  401. return 0;
  402. out_free_pages:
  403. for (i = 0; i < bp->b_page_count; i++)
  404. __free_page(bp->b_pages[i]);
  405. bp->b_flags &= ~_XBF_PAGES;
  406. return error;
  407. }
  408. /*
  409. * Map buffer into kernel address-space if necessary.
  410. */
  411. STATIC int
  412. _xfs_buf_map_pages(
  413. xfs_buf_t *bp,
  414. uint flags)
  415. {
  416. ASSERT(bp->b_flags & _XBF_PAGES);
  417. if (bp->b_page_count == 1) {
  418. /* A single page buffer is always mappable */
  419. bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
  420. } else if (flags & XBF_UNMAPPED) {
  421. bp->b_addr = NULL;
  422. } else {
  423. int retried = 0;
  424. unsigned nofs_flag;
  425. /*
  426. * vm_map_ram() will allocate auxiliary structures (e.g.
  427. * pagetables) with GFP_KERNEL, yet we are likely to be under
  428. * GFP_NOFS context here. Hence we need to tell memory reclaim
  429. * that we are in such a context via PF_MEMALLOC_NOFS to prevent
  430. * memory reclaim re-entering the filesystem here and
  431. * potentially deadlocking.
  432. */
  433. nofs_flag = memalloc_nofs_save();
  434. do {
  435. bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
  436. -1);
  437. if (bp->b_addr)
  438. break;
  439. vm_unmap_aliases();
  440. } while (retried++ <= 1);
  441. memalloc_nofs_restore(nofs_flag);
  442. if (!bp->b_addr)
  443. return -ENOMEM;
  444. bp->b_addr += bp->b_offset;
  445. }
  446. return 0;
  447. }
  448. /*
  449. * Finding and Reading Buffers
  450. */
  451. static int
  452. _xfs_buf_obj_cmp(
  453. struct rhashtable_compare_arg *arg,
  454. const void *obj)
  455. {
  456. const struct xfs_buf_map *map = arg->key;
  457. const struct xfs_buf *bp = obj;
  458. /*
  459. * The key hashing in the lookup path depends on the key being the
  460. * first element of the compare_arg, make sure to assert this.
  461. */
  462. BUILD_BUG_ON(offsetof(struct xfs_buf_map, bm_bn) != 0);
  463. if (bp->b_bn != map->bm_bn)
  464. return 1;
  465. if (unlikely(bp->b_length != map->bm_len)) {
  466. /*
  467. * found a block number match. If the range doesn't
  468. * match, the only way this is allowed is if the buffer
  469. * in the cache is stale and the transaction that made
  470. * it stale has not yet committed. i.e. we are
  471. * reallocating a busy extent. Skip this buffer and
  472. * continue searching for an exact match.
  473. */
  474. ASSERT(bp->b_flags & XBF_STALE);
  475. return 1;
  476. }
  477. return 0;
  478. }
  479. static const struct rhashtable_params xfs_buf_hash_params = {
  480. .min_size = 32, /* empty AGs have minimal footprint */
  481. .nelem_hint = 16,
  482. .key_len = sizeof(xfs_daddr_t),
  483. .key_offset = offsetof(struct xfs_buf, b_bn),
  484. .head_offset = offsetof(struct xfs_buf, b_rhash_head),
  485. .automatic_shrinking = true,
  486. .obj_cmpfn = _xfs_buf_obj_cmp,
  487. };
  488. int
  489. xfs_buf_hash_init(
  490. struct xfs_perag *pag)
  491. {
  492. spin_lock_init(&pag->pag_buf_lock);
  493. return rhashtable_init(&pag->pag_buf_hash, &xfs_buf_hash_params);
  494. }
  495. void
  496. xfs_buf_hash_destroy(
  497. struct xfs_perag *pag)
  498. {
  499. rhashtable_destroy(&pag->pag_buf_hash);
  500. }
  501. /*
  502. * Look up a buffer in the buffer cache and return it referenced and locked
  503. * in @found_bp.
  504. *
  505. * If @new_bp is supplied and we have a lookup miss, insert @new_bp into the
  506. * cache.
  507. *
  508. * If XBF_TRYLOCK is set in @flags, only try to lock the buffer and return
  509. * -EAGAIN if we fail to lock it.
  510. *
  511. * Return values are:
  512. * -EFSCORRUPTED if have been supplied with an invalid address
  513. * -EAGAIN on trylock failure
  514. * -ENOENT if we fail to find a match and @new_bp was NULL
  515. * 0, with @found_bp:
  516. * - @new_bp if we inserted it into the cache
  517. * - the buffer we found and locked.
  518. */
  519. static int
  520. xfs_buf_find(
  521. struct xfs_buftarg *btp,
  522. struct xfs_buf_map *map,
  523. int nmaps,
  524. xfs_buf_flags_t flags,
  525. struct xfs_buf *new_bp,
  526. struct xfs_buf **found_bp)
  527. {
  528. struct xfs_perag *pag;
  529. xfs_buf_t *bp;
  530. struct xfs_buf_map cmap = { .bm_bn = map[0].bm_bn };
  531. xfs_daddr_t eofs;
  532. int i;
  533. *found_bp = NULL;
  534. for (i = 0; i < nmaps; i++)
  535. cmap.bm_len += map[i].bm_len;
  536. /* Check for IOs smaller than the sector size / not sector aligned */
  537. ASSERT(!(BBTOB(cmap.bm_len) < btp->bt_meta_sectorsize));
  538. ASSERT(!(BBTOB(cmap.bm_bn) & (xfs_off_t)btp->bt_meta_sectormask));
  539. /*
  540. * Corrupted block numbers can get through to here, unfortunately, so we
  541. * have to check that the buffer falls within the filesystem bounds.
  542. */
  543. eofs = XFS_FSB_TO_BB(btp->bt_mount, btp->bt_mount->m_sb.sb_dblocks);
  544. if (cmap.bm_bn < 0 || cmap.bm_bn >= eofs) {
  545. xfs_alert(btp->bt_mount,
  546. "%s: daddr 0x%llx out of range, EOFS 0x%llx",
  547. __func__, cmap.bm_bn, eofs);
  548. WARN_ON(1);
  549. return -EFSCORRUPTED;
  550. }
  551. pag = xfs_perag_get(btp->bt_mount,
  552. xfs_daddr_to_agno(btp->bt_mount, cmap.bm_bn));
  553. spin_lock(&pag->pag_buf_lock);
  554. bp = rhashtable_lookup_fast(&pag->pag_buf_hash, &cmap,
  555. xfs_buf_hash_params);
  556. if (bp) {
  557. atomic_inc(&bp->b_hold);
  558. goto found;
  559. }
  560. /* No match found */
  561. if (!new_bp) {
  562. XFS_STATS_INC(btp->bt_mount, xb_miss_locked);
  563. spin_unlock(&pag->pag_buf_lock);
  564. xfs_perag_put(pag);
  565. return -ENOENT;
  566. }
  567. /* the buffer keeps the perag reference until it is freed */
  568. new_bp->b_pag = pag;
  569. rhashtable_insert_fast(&pag->pag_buf_hash, &new_bp->b_rhash_head,
  570. xfs_buf_hash_params);
  571. spin_unlock(&pag->pag_buf_lock);
  572. *found_bp = new_bp;
  573. return 0;
  574. found:
  575. spin_unlock(&pag->pag_buf_lock);
  576. xfs_perag_put(pag);
  577. if (!xfs_buf_trylock(bp)) {
  578. if (flags & XBF_TRYLOCK) {
  579. xfs_buf_rele(bp);
  580. XFS_STATS_INC(btp->bt_mount, xb_busy_locked);
  581. return -EAGAIN;
  582. }
  583. xfs_buf_lock(bp);
  584. XFS_STATS_INC(btp->bt_mount, xb_get_locked_waited);
  585. }
  586. /*
  587. * if the buffer is stale, clear all the external state associated with
  588. * it. We need to keep flags such as how we allocated the buffer memory
  589. * intact here.
  590. */
  591. if (bp->b_flags & XBF_STALE) {
  592. ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
  593. bp->b_flags &= _XBF_KMEM | _XBF_PAGES;
  594. bp->b_ops = NULL;
  595. }
  596. trace_xfs_buf_find(bp, flags, _RET_IP_);
  597. XFS_STATS_INC(btp->bt_mount, xb_get_locked);
  598. *found_bp = bp;
  599. return 0;
  600. }
  601. struct xfs_buf *
  602. xfs_buf_incore(
  603. struct xfs_buftarg *target,
  604. xfs_daddr_t blkno,
  605. size_t numblks,
  606. xfs_buf_flags_t flags)
  607. {
  608. struct xfs_buf *bp;
  609. int error;
  610. DEFINE_SINGLE_BUF_MAP(map, blkno, numblks);
  611. error = xfs_buf_find(target, &map, 1, flags, NULL, &bp);
  612. if (error)
  613. return NULL;
  614. return bp;
  615. }
  616. /*
  617. * Assembles a buffer covering the specified range. The code is optimised for
  618. * cache hits, as metadata intensive workloads will see 3 orders of magnitude
  619. * more hits than misses.
  620. */
  621. int
  622. xfs_buf_get_map(
  623. struct xfs_buftarg *target,
  624. struct xfs_buf_map *map,
  625. int nmaps,
  626. xfs_buf_flags_t flags,
  627. struct xfs_buf **bpp)
  628. {
  629. struct xfs_buf *bp;
  630. struct xfs_buf *new_bp;
  631. int error = 0;
  632. *bpp = NULL;
  633. error = xfs_buf_find(target, map, nmaps, flags, NULL, &bp);
  634. if (!error)
  635. goto found;
  636. if (error != -ENOENT)
  637. return error;
  638. error = _xfs_buf_alloc(target, map, nmaps, flags, &new_bp);
  639. if (error)
  640. return error;
  641. error = xfs_buf_allocate_memory(new_bp, flags);
  642. if (error) {
  643. xfs_buf_free(new_bp);
  644. return error;
  645. }
  646. error = xfs_buf_find(target, map, nmaps, flags, new_bp, &bp);
  647. if (error) {
  648. xfs_buf_free(new_bp);
  649. return error;
  650. }
  651. if (bp != new_bp)
  652. xfs_buf_free(new_bp);
  653. found:
  654. if (!bp->b_addr) {
  655. error = _xfs_buf_map_pages(bp, flags);
  656. if (unlikely(error)) {
  657. xfs_warn_ratelimited(target->bt_mount,
  658. "%s: failed to map %u pages", __func__,
  659. bp->b_page_count);
  660. xfs_buf_relse(bp);
  661. return error;
  662. }
  663. }
  664. /*
  665. * Clear b_error if this is a lookup from a caller that doesn't expect
  666. * valid data to be found in the buffer.
  667. */
  668. if (!(flags & XBF_READ))
  669. xfs_buf_ioerror(bp, 0);
  670. XFS_STATS_INC(target->bt_mount, xb_get);
  671. trace_xfs_buf_get(bp, flags, _RET_IP_);
  672. *bpp = bp;
  673. return 0;
  674. }
  675. int
  676. _xfs_buf_read(
  677. xfs_buf_t *bp,
  678. xfs_buf_flags_t flags)
  679. {
  680. ASSERT(!(flags & XBF_WRITE));
  681. ASSERT(bp->b_maps[0].bm_bn != XFS_BUF_DADDR_NULL);
  682. bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_READ_AHEAD | XBF_DONE);
  683. bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | XBF_READ_AHEAD);
  684. return xfs_buf_submit(bp);
  685. }
  686. /*
  687. * Reverify a buffer found in cache without an attached ->b_ops.
  688. *
  689. * If the caller passed an ops structure and the buffer doesn't have ops
  690. * assigned, set the ops and use it to verify the contents. If verification
  691. * fails, clear XBF_DONE. We assume the buffer has no recorded errors and is
  692. * already in XBF_DONE state on entry.
  693. *
  694. * Under normal operations, every in-core buffer is verified on read I/O
  695. * completion. There are two scenarios that can lead to in-core buffers without
  696. * an assigned ->b_ops. The first is during log recovery of buffers on a V4
  697. * filesystem, though these buffers are purged at the end of recovery. The
  698. * other is online repair, which intentionally reads with a NULL buffer ops to
  699. * run several verifiers across an in-core buffer in order to establish buffer
  700. * type. If repair can't establish that, the buffer will be left in memory
  701. * with NULL buffer ops.
  702. */
  703. int
  704. xfs_buf_reverify(
  705. struct xfs_buf *bp,
  706. const struct xfs_buf_ops *ops)
  707. {
  708. ASSERT(bp->b_flags & XBF_DONE);
  709. ASSERT(bp->b_error == 0);
  710. if (!ops || bp->b_ops)
  711. return 0;
  712. bp->b_ops = ops;
  713. bp->b_ops->verify_read(bp);
  714. if (bp->b_error)
  715. bp->b_flags &= ~XBF_DONE;
  716. return bp->b_error;
  717. }
  718. int
  719. xfs_buf_read_map(
  720. struct xfs_buftarg *target,
  721. struct xfs_buf_map *map,
  722. int nmaps,
  723. xfs_buf_flags_t flags,
  724. struct xfs_buf **bpp,
  725. const struct xfs_buf_ops *ops,
  726. xfs_failaddr_t fa)
  727. {
  728. struct xfs_buf *bp;
  729. int error;
  730. flags |= XBF_READ;
  731. *bpp = NULL;
  732. error = xfs_buf_get_map(target, map, nmaps, flags, &bp);
  733. if (error)
  734. return error;
  735. trace_xfs_buf_read(bp, flags, _RET_IP_);
  736. if (!(bp->b_flags & XBF_DONE)) {
  737. /* Initiate the buffer read and wait. */
  738. XFS_STATS_INC(target->bt_mount, xb_get_read);
  739. bp->b_ops = ops;
  740. error = _xfs_buf_read(bp, flags);
  741. /* Readahead iodone already dropped the buffer, so exit. */
  742. if (flags & XBF_ASYNC)
  743. return 0;
  744. } else {
  745. /* Buffer already read; all we need to do is check it. */
  746. error = xfs_buf_reverify(bp, ops);
  747. /* Readahead already finished; drop the buffer and exit. */
  748. if (flags & XBF_ASYNC) {
  749. xfs_buf_relse(bp);
  750. return 0;
  751. }
  752. /* We do not want read in the flags */
  753. bp->b_flags &= ~XBF_READ;
  754. ASSERT(bp->b_ops != NULL || ops == NULL);
  755. }
  756. /*
  757. * If we've had a read error, then the contents of the buffer are
  758. * invalid and should not be used. To ensure that a followup read tries
  759. * to pull the buffer from disk again, we clear the XBF_DONE flag and
  760. * mark the buffer stale. This ensures that anyone who has a current
  761. * reference to the buffer will interpret it's contents correctly and
  762. * future cache lookups will also treat it as an empty, uninitialised
  763. * buffer.
  764. */
  765. if (error) {
  766. if (!XFS_FORCED_SHUTDOWN(target->bt_mount))
  767. xfs_buf_ioerror_alert(bp, fa);
  768. bp->b_flags &= ~XBF_DONE;
  769. xfs_buf_stale(bp);
  770. xfs_buf_relse(bp);
  771. /* bad CRC means corrupted metadata */
  772. if (error == -EFSBADCRC)
  773. error = -EFSCORRUPTED;
  774. return error;
  775. }
  776. *bpp = bp;
  777. return 0;
  778. }
  779. /*
  780. * If we are not low on memory then do the readahead in a deadlock
  781. * safe manner.
  782. */
  783. void
  784. xfs_buf_readahead_map(
  785. struct xfs_buftarg *target,
  786. struct xfs_buf_map *map,
  787. int nmaps,
  788. const struct xfs_buf_ops *ops)
  789. {
  790. struct xfs_buf *bp;
  791. if (bdi_read_congested(target->bt_bdev->bd_bdi))
  792. return;
  793. xfs_buf_read_map(target, map, nmaps,
  794. XBF_TRYLOCK | XBF_ASYNC | XBF_READ_AHEAD, &bp, ops,
  795. __this_address);
  796. }
  797. /*
  798. * Read an uncached buffer from disk. Allocates and returns a locked
  799. * buffer containing the disk contents or nothing.
  800. */
  801. int
  802. xfs_buf_read_uncached(
  803. struct xfs_buftarg *target,
  804. xfs_daddr_t daddr,
  805. size_t numblks,
  806. int flags,
  807. struct xfs_buf **bpp,
  808. const struct xfs_buf_ops *ops)
  809. {
  810. struct xfs_buf *bp;
  811. int error;
  812. *bpp = NULL;
  813. error = xfs_buf_get_uncached(target, numblks, flags, &bp);
  814. if (error)
  815. return error;
  816. /* set up the buffer for a read IO */
  817. ASSERT(bp->b_map_count == 1);
  818. bp->b_bn = XFS_BUF_DADDR_NULL; /* always null for uncached buffers */
  819. bp->b_maps[0].bm_bn = daddr;
  820. bp->b_flags |= XBF_READ;
  821. bp->b_ops = ops;
  822. xfs_buf_submit(bp);
  823. if (bp->b_error) {
  824. error = bp->b_error;
  825. xfs_buf_relse(bp);
  826. return error;
  827. }
  828. *bpp = bp;
  829. return 0;
  830. }
  831. int
  832. xfs_buf_get_uncached(
  833. struct xfs_buftarg *target,
  834. size_t numblks,
  835. int flags,
  836. struct xfs_buf **bpp)
  837. {
  838. unsigned long page_count;
  839. int error, i;
  840. struct xfs_buf *bp;
  841. DEFINE_SINGLE_BUF_MAP(map, XFS_BUF_DADDR_NULL, numblks);
  842. *bpp = NULL;
  843. /* flags might contain irrelevant bits, pass only what we care about */
  844. error = _xfs_buf_alloc(target, &map, 1, flags & XBF_NO_IOACCT, &bp);
  845. if (error)
  846. goto fail;
  847. page_count = PAGE_ALIGN(numblks << BBSHIFT) >> PAGE_SHIFT;
  848. error = _xfs_buf_get_pages(bp, page_count);
  849. if (error)
  850. goto fail_free_buf;
  851. for (i = 0; i < page_count; i++) {
  852. bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
  853. if (!bp->b_pages[i]) {
  854. error = -ENOMEM;
  855. goto fail_free_mem;
  856. }
  857. }
  858. bp->b_flags |= _XBF_PAGES;
  859. error = _xfs_buf_map_pages(bp, 0);
  860. if (unlikely(error)) {
  861. xfs_warn(target->bt_mount,
  862. "%s: failed to map pages", __func__);
  863. goto fail_free_mem;
  864. }
  865. trace_xfs_buf_get_uncached(bp, _RET_IP_);
  866. *bpp = bp;
  867. return 0;
  868. fail_free_mem:
  869. while (--i >= 0)
  870. __free_page(bp->b_pages[i]);
  871. _xfs_buf_free_pages(bp);
  872. fail_free_buf:
  873. xfs_buf_free_maps(bp);
  874. kmem_cache_free(xfs_buf_zone, bp);
  875. fail:
  876. return error;
  877. }
  878. /*
  879. * Increment reference count on buffer, to hold the buffer concurrently
  880. * with another thread which may release (free) the buffer asynchronously.
  881. * Must hold the buffer already to call this function.
  882. */
  883. void
  884. xfs_buf_hold(
  885. xfs_buf_t *bp)
  886. {
  887. trace_xfs_buf_hold(bp, _RET_IP_);
  888. atomic_inc(&bp->b_hold);
  889. }
  890. /*
  891. * Release a hold on the specified buffer. If the hold count is 1, the buffer is
  892. * placed on LRU or freed (depending on b_lru_ref).
  893. */
  894. void
  895. xfs_buf_rele(
  896. xfs_buf_t *bp)
  897. {
  898. struct xfs_perag *pag = bp->b_pag;
  899. bool release;
  900. bool freebuf = false;
  901. trace_xfs_buf_rele(bp, _RET_IP_);
  902. if (!pag) {
  903. ASSERT(list_empty(&bp->b_lru));
  904. if (atomic_dec_and_test(&bp->b_hold)) {
  905. xfs_buf_ioacct_dec(bp);
  906. xfs_buf_free(bp);
  907. }
  908. return;
  909. }
  910. ASSERT(atomic_read(&bp->b_hold) > 0);
  911. /*
  912. * We grab the b_lock here first to serialise racing xfs_buf_rele()
  913. * calls. The pag_buf_lock being taken on the last reference only
  914. * serialises against racing lookups in xfs_buf_find(). IOWs, the second
  915. * to last reference we drop here is not serialised against the last
  916. * reference until we take bp->b_lock. Hence if we don't grab b_lock
  917. * first, the last "release" reference can win the race to the lock and
  918. * free the buffer before the second-to-last reference is processed,
  919. * leading to a use-after-free scenario.
  920. */
  921. spin_lock(&bp->b_lock);
  922. release = atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock);
  923. if (!release) {
  924. /*
  925. * Drop the in-flight state if the buffer is already on the LRU
  926. * and it holds the only reference. This is racy because we
  927. * haven't acquired the pag lock, but the use of _XBF_IN_FLIGHT
  928. * ensures the decrement occurs only once per-buf.
  929. */
  930. if ((atomic_read(&bp->b_hold) == 1) && !list_empty(&bp->b_lru))
  931. __xfs_buf_ioacct_dec(bp);
  932. goto out_unlock;
  933. }
  934. /* the last reference has been dropped ... */
  935. __xfs_buf_ioacct_dec(bp);
  936. if (!(bp->b_flags & XBF_STALE) && atomic_read(&bp->b_lru_ref)) {
  937. /*
  938. * If the buffer is added to the LRU take a new reference to the
  939. * buffer for the LRU and clear the (now stale) dispose list
  940. * state flag
  941. */
  942. if (list_lru_add(&bp->b_target->bt_lru, &bp->b_lru)) {
  943. bp->b_state &= ~XFS_BSTATE_DISPOSE;
  944. atomic_inc(&bp->b_hold);
  945. }
  946. spin_unlock(&pag->pag_buf_lock);
  947. } else {
  948. /*
  949. * most of the time buffers will already be removed from the
  950. * LRU, so optimise that case by checking for the
  951. * XFS_BSTATE_DISPOSE flag indicating the last list the buffer
  952. * was on was the disposal list
  953. */
  954. if (!(bp->b_state & XFS_BSTATE_DISPOSE)) {
  955. list_lru_del(&bp->b_target->bt_lru, &bp->b_lru);
  956. } else {
  957. ASSERT(list_empty(&bp->b_lru));
  958. }
  959. ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
  960. rhashtable_remove_fast(&pag->pag_buf_hash, &bp->b_rhash_head,
  961. xfs_buf_hash_params);
  962. spin_unlock(&pag->pag_buf_lock);
  963. xfs_perag_put(pag);
  964. freebuf = true;
  965. }
  966. out_unlock:
  967. spin_unlock(&bp->b_lock);
  968. if (freebuf)
  969. xfs_buf_free(bp);
  970. }
  971. /*
  972. * Lock a buffer object, if it is not already locked.
  973. *
  974. * If we come across a stale, pinned, locked buffer, we know that we are
  975. * being asked to lock a buffer that has been reallocated. Because it is
  976. * pinned, we know that the log has not been pushed to disk and hence it
  977. * will still be locked. Rather than continuing to have trylock attempts
  978. * fail until someone else pushes the log, push it ourselves before
  979. * returning. This means that the xfsaild will not get stuck trying
  980. * to push on stale inode buffers.
  981. */
  982. int
  983. xfs_buf_trylock(
  984. struct xfs_buf *bp)
  985. {
  986. int locked;
  987. locked = down_trylock(&bp->b_sema) == 0;
  988. if (locked)
  989. trace_xfs_buf_trylock(bp, _RET_IP_);
  990. else
  991. trace_xfs_buf_trylock_fail(bp, _RET_IP_);
  992. return locked;
  993. }
  994. /*
  995. * Lock a buffer object.
  996. *
  997. * If we come across a stale, pinned, locked buffer, we know that we
  998. * are being asked to lock a buffer that has been reallocated. Because
  999. * it is pinned, we know that the log has not been pushed to disk and
  1000. * hence it will still be locked. Rather than sleeping until someone
  1001. * else pushes the log, push it ourselves before trying to get the lock.
  1002. */
  1003. void
  1004. xfs_buf_lock(
  1005. struct xfs_buf *bp)
  1006. {
  1007. trace_xfs_buf_lock(bp, _RET_IP_);
  1008. if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
  1009. xfs_log_force(bp->b_mount, 0);
  1010. down(&bp->b_sema);
  1011. trace_xfs_buf_lock_done(bp, _RET_IP_);
  1012. }
  1013. void
  1014. xfs_buf_unlock(
  1015. struct xfs_buf *bp)
  1016. {
  1017. ASSERT(xfs_buf_islocked(bp));
  1018. up(&bp->b_sema);
  1019. trace_xfs_buf_unlock(bp, _RET_IP_);
  1020. }
  1021. STATIC void
  1022. xfs_buf_wait_unpin(
  1023. xfs_buf_t *bp)
  1024. {
  1025. DECLARE_WAITQUEUE (wait, current);
  1026. if (atomic_read(&bp->b_pin_count) == 0)
  1027. return;
  1028. add_wait_queue(&bp->b_waiters, &wait);
  1029. for (;;) {
  1030. set_current_state(TASK_UNINTERRUPTIBLE);
  1031. if (atomic_read(&bp->b_pin_count) == 0)
  1032. break;
  1033. io_schedule();
  1034. }
  1035. remove_wait_queue(&bp->b_waiters, &wait);
  1036. set_current_state(TASK_RUNNING);
  1037. }
  1038. static void
  1039. xfs_buf_ioerror_alert_ratelimited(
  1040. struct xfs_buf *bp)
  1041. {
  1042. static unsigned long lasttime;
  1043. static struct xfs_buftarg *lasttarg;
  1044. if (bp->b_target != lasttarg ||
  1045. time_after(jiffies, (lasttime + 5*HZ))) {
  1046. lasttime = jiffies;
  1047. xfs_buf_ioerror_alert(bp, __this_address);
  1048. }
  1049. lasttarg = bp->b_target;
  1050. }
  1051. /*
  1052. * Account for this latest trip around the retry handler, and decide if
  1053. * we've failed enough times to constitute a permanent failure.
  1054. */
  1055. static bool
  1056. xfs_buf_ioerror_permanent(
  1057. struct xfs_buf *bp,
  1058. struct xfs_error_cfg *cfg)
  1059. {
  1060. struct xfs_mount *mp = bp->b_mount;
  1061. if (cfg->max_retries != XFS_ERR_RETRY_FOREVER &&
  1062. ++bp->b_retries > cfg->max_retries)
  1063. return true;
  1064. if (cfg->retry_timeout != XFS_ERR_RETRY_FOREVER &&
  1065. time_after(jiffies, cfg->retry_timeout + bp->b_first_retry_time))
  1066. return true;
  1067. /* At unmount we may treat errors differently */
  1068. if ((mp->m_flags & XFS_MOUNT_UNMOUNTING) && mp->m_fail_unmount)
  1069. return true;
  1070. return false;
  1071. }
  1072. /*
  1073. * On a sync write or shutdown we just want to stale the buffer and let the
  1074. * caller handle the error in bp->b_error appropriately.
  1075. *
  1076. * If the write was asynchronous then no one will be looking for the error. If
  1077. * this is the first failure of this type, clear the error state and write the
  1078. * buffer out again. This means we always retry an async write failure at least
  1079. * once, but we also need to set the buffer up to behave correctly now for
  1080. * repeated failures.
  1081. *
  1082. * If we get repeated async write failures, then we take action according to the
  1083. * error configuration we have been set up to use.
  1084. *
  1085. * Returns true if this function took care of error handling and the caller must
  1086. * not touch the buffer again. Return false if the caller should proceed with
  1087. * normal I/O completion handling.
  1088. */
  1089. static bool
  1090. xfs_buf_ioend_handle_error(
  1091. struct xfs_buf *bp)
  1092. {
  1093. struct xfs_mount *mp = bp->b_mount;
  1094. struct xfs_error_cfg *cfg;
  1095. /*
  1096. * If we've already decided to shutdown the filesystem because of I/O
  1097. * errors, there's no point in giving this a retry.
  1098. */
  1099. if (XFS_FORCED_SHUTDOWN(mp))
  1100. goto out_stale;
  1101. xfs_buf_ioerror_alert_ratelimited(bp);
  1102. /*
  1103. * We're not going to bother about retrying this during recovery.
  1104. * One strike!
  1105. */
  1106. if (bp->b_flags & _XBF_LOGRECOVERY) {
  1107. xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
  1108. return false;
  1109. }
  1110. /*
  1111. * Synchronous writes will have callers process the error.
  1112. */
  1113. if (!(bp->b_flags & XBF_ASYNC))
  1114. goto out_stale;
  1115. trace_xfs_buf_iodone_async(bp, _RET_IP_);
  1116. cfg = xfs_error_get_cfg(mp, XFS_ERR_METADATA, bp->b_error);
  1117. if (bp->b_last_error != bp->b_error ||
  1118. !(bp->b_flags & (XBF_STALE | XBF_WRITE_FAIL))) {
  1119. bp->b_last_error = bp->b_error;
  1120. if (cfg->retry_timeout != XFS_ERR_RETRY_FOREVER &&
  1121. !bp->b_first_retry_time)
  1122. bp->b_first_retry_time = jiffies;
  1123. goto resubmit;
  1124. }
  1125. /*
  1126. * Permanent error - we need to trigger a shutdown if we haven't already
  1127. * to indicate that inconsistency will result from this action.
  1128. */
  1129. if (xfs_buf_ioerror_permanent(bp, cfg)) {
  1130. xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
  1131. goto out_stale;
  1132. }
  1133. /* Still considered a transient error. Caller will schedule retries. */
  1134. if (bp->b_flags & _XBF_INODES)
  1135. xfs_buf_inode_io_fail(bp);
  1136. else if (bp->b_flags & _XBF_DQUOTS)
  1137. xfs_buf_dquot_io_fail(bp);
  1138. else
  1139. ASSERT(list_empty(&bp->b_li_list));
  1140. xfs_buf_ioerror(bp, 0);
  1141. xfs_buf_relse(bp);
  1142. return true;
  1143. resubmit:
  1144. xfs_buf_ioerror(bp, 0);
  1145. bp->b_flags |= (XBF_DONE | XBF_WRITE_FAIL);
  1146. xfs_buf_submit(bp);
  1147. return true;
  1148. out_stale:
  1149. xfs_buf_stale(bp);
  1150. bp->b_flags |= XBF_DONE;
  1151. bp->b_flags &= ~XBF_WRITE;
  1152. trace_xfs_buf_error_relse(bp, _RET_IP_);
  1153. return false;
  1154. }
  1155. static void
  1156. xfs_buf_ioend(
  1157. struct xfs_buf *bp)
  1158. {
  1159. trace_xfs_buf_iodone(bp, _RET_IP_);
  1160. /*
  1161. * Pull in IO completion errors now. We are guaranteed to be running
  1162. * single threaded, so we don't need the lock to read b_io_error.
  1163. */
  1164. if (!bp->b_error && bp->b_io_error)
  1165. xfs_buf_ioerror(bp, bp->b_io_error);
  1166. if (bp->b_flags & XBF_READ) {
  1167. if (!bp->b_error && bp->b_ops)
  1168. bp->b_ops->verify_read(bp);
  1169. if (!bp->b_error)
  1170. bp->b_flags |= XBF_DONE;
  1171. } else {
  1172. if (!bp->b_error) {
  1173. bp->b_flags &= ~XBF_WRITE_FAIL;
  1174. bp->b_flags |= XBF_DONE;
  1175. }
  1176. if (unlikely(bp->b_error) && xfs_buf_ioend_handle_error(bp))
  1177. return;
  1178. /* clear the retry state */
  1179. bp->b_last_error = 0;
  1180. bp->b_retries = 0;
  1181. bp->b_first_retry_time = 0;
  1182. /*
  1183. * Note that for things like remote attribute buffers, there may
  1184. * not be a buffer log item here, so processing the buffer log
  1185. * item must remain optional.
  1186. */
  1187. if (bp->b_log_item)
  1188. xfs_buf_item_done(bp);
  1189. if (bp->b_flags & _XBF_INODES)
  1190. xfs_buf_inode_iodone(bp);
  1191. else if (bp->b_flags & _XBF_DQUOTS)
  1192. xfs_buf_dquot_iodone(bp);
  1193. }
  1194. bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD |
  1195. _XBF_LOGRECOVERY);
  1196. if (bp->b_flags & XBF_ASYNC)
  1197. xfs_buf_relse(bp);
  1198. else
  1199. complete(&bp->b_iowait);
  1200. }
  1201. static void
  1202. xfs_buf_ioend_work(
  1203. struct work_struct *work)
  1204. {
  1205. struct xfs_buf *bp =
  1206. container_of(work, xfs_buf_t, b_ioend_work);
  1207. xfs_buf_ioend(bp);
  1208. }
  1209. static void
  1210. xfs_buf_ioend_async(
  1211. struct xfs_buf *bp)
  1212. {
  1213. INIT_WORK(&bp->b_ioend_work, xfs_buf_ioend_work);
  1214. queue_work(bp->b_mount->m_buf_workqueue, &bp->b_ioend_work);
  1215. }
  1216. void
  1217. __xfs_buf_ioerror(
  1218. xfs_buf_t *bp,
  1219. int error,
  1220. xfs_failaddr_t failaddr)
  1221. {
  1222. ASSERT(error <= 0 && error >= -1000);
  1223. bp->b_error = error;
  1224. trace_xfs_buf_ioerror(bp, error, failaddr);
  1225. }
  1226. void
  1227. xfs_buf_ioerror_alert(
  1228. struct xfs_buf *bp,
  1229. xfs_failaddr_t func)
  1230. {
  1231. xfs_buf_alert_ratelimited(bp, "XFS: metadata IO error",
  1232. "metadata I/O error in \"%pS\" at daddr 0x%llx len %d error %d",
  1233. func, (uint64_t)XFS_BUF_ADDR(bp),
  1234. bp->b_length, -bp->b_error);
  1235. }
  1236. /*
  1237. * To simulate an I/O failure, the buffer must be locked and held with at least
  1238. * three references. The LRU reference is dropped by the stale call. The buf
  1239. * item reference is dropped via ioend processing. The third reference is owned
  1240. * by the caller and is dropped on I/O completion if the buffer is XBF_ASYNC.
  1241. */
  1242. void
  1243. xfs_buf_ioend_fail(
  1244. struct xfs_buf *bp)
  1245. {
  1246. bp->b_flags &= ~XBF_DONE;
  1247. xfs_buf_stale(bp);
  1248. xfs_buf_ioerror(bp, -EIO);
  1249. xfs_buf_ioend(bp);
  1250. }
  1251. int
  1252. xfs_bwrite(
  1253. struct xfs_buf *bp)
  1254. {
  1255. int error;
  1256. ASSERT(xfs_buf_islocked(bp));
  1257. bp->b_flags |= XBF_WRITE;
  1258. bp->b_flags &= ~(XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q |
  1259. XBF_DONE);
  1260. error = xfs_buf_submit(bp);
  1261. if (error)
  1262. xfs_force_shutdown(bp->b_mount, SHUTDOWN_META_IO_ERROR);
  1263. return error;
  1264. }
  1265. static void
  1266. xfs_buf_bio_end_io(
  1267. struct bio *bio)
  1268. {
  1269. struct xfs_buf *bp = (struct xfs_buf *)bio->bi_private;
  1270. if (!bio->bi_status &&
  1271. (bp->b_flags & XBF_WRITE) && (bp->b_flags & XBF_ASYNC) &&
  1272. XFS_TEST_ERROR(false, bp->b_mount, XFS_ERRTAG_BUF_IOERROR))
  1273. bio->bi_status = BLK_STS_IOERR;
  1274. /*
  1275. * don't overwrite existing errors - otherwise we can lose errors on
  1276. * buffers that require multiple bios to complete.
  1277. */
  1278. if (bio->bi_status) {
  1279. int error = blk_status_to_errno(bio->bi_status);
  1280. cmpxchg(&bp->b_io_error, 0, error);
  1281. }
  1282. if (!bp->b_error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
  1283. invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
  1284. if (atomic_dec_and_test(&bp->b_io_remaining) == 1)
  1285. xfs_buf_ioend_async(bp);
  1286. bio_put(bio);
  1287. }
  1288. static void
  1289. xfs_buf_ioapply_map(
  1290. struct xfs_buf *bp,
  1291. int map,
  1292. int *buf_offset,
  1293. int *count,
  1294. int op)
  1295. {
  1296. int page_index;
  1297. int total_nr_pages = bp->b_page_count;
  1298. int nr_pages;
  1299. struct bio *bio;
  1300. sector_t sector = bp->b_maps[map].bm_bn;
  1301. int size;
  1302. int offset;
  1303. /* skip the pages in the buffer before the start offset */
  1304. page_index = 0;
  1305. offset = *buf_offset;
  1306. while (offset >= PAGE_SIZE) {
  1307. page_index++;
  1308. offset -= PAGE_SIZE;
  1309. }
  1310. /*
  1311. * Limit the IO size to the length of the current vector, and update the
  1312. * remaining IO count for the next time around.
  1313. */
  1314. size = min_t(int, BBTOB(bp->b_maps[map].bm_len), *count);
  1315. *count -= size;
  1316. *buf_offset += size;
  1317. next_chunk:
  1318. atomic_inc(&bp->b_io_remaining);
  1319. nr_pages = min(total_nr_pages, BIO_MAX_PAGES);
  1320. bio = bio_alloc(GFP_NOIO, nr_pages);
  1321. bio_set_dev(bio, bp->b_target->bt_bdev);
  1322. bio->bi_iter.bi_sector = sector;
  1323. bio->bi_end_io = xfs_buf_bio_end_io;
  1324. bio->bi_private = bp;
  1325. bio->bi_opf = op;
  1326. for (; size && nr_pages; nr_pages--, page_index++) {
  1327. int rbytes, nbytes = PAGE_SIZE - offset;
  1328. if (nbytes > size)
  1329. nbytes = size;
  1330. rbytes = bio_add_page(bio, bp->b_pages[page_index], nbytes,
  1331. offset);
  1332. if (rbytes < nbytes)
  1333. break;
  1334. offset = 0;
  1335. sector += BTOBB(nbytes);
  1336. size -= nbytes;
  1337. total_nr_pages--;
  1338. }
  1339. if (likely(bio->bi_iter.bi_size)) {
  1340. if (xfs_buf_is_vmapped(bp)) {
  1341. flush_kernel_vmap_range(bp->b_addr,
  1342. xfs_buf_vmap_len(bp));
  1343. }
  1344. submit_bio(bio);
  1345. if (size)
  1346. goto next_chunk;
  1347. } else {
  1348. /*
  1349. * This is guaranteed not to be the last io reference count
  1350. * because the caller (xfs_buf_submit) holds a count itself.
  1351. */
  1352. atomic_dec(&bp->b_io_remaining);
  1353. xfs_buf_ioerror(bp, -EIO);
  1354. bio_put(bio);
  1355. }
  1356. }
  1357. STATIC void
  1358. _xfs_buf_ioapply(
  1359. struct xfs_buf *bp)
  1360. {
  1361. struct blk_plug plug;
  1362. int op;
  1363. int offset;
  1364. int size;
  1365. int i;
  1366. /*
  1367. * Make sure we capture only current IO errors rather than stale errors
  1368. * left over from previous use of the buffer (e.g. failed readahead).
  1369. */
  1370. bp->b_error = 0;
  1371. if (bp->b_flags & XBF_WRITE) {
  1372. op = REQ_OP_WRITE;
  1373. /*
  1374. * Run the write verifier callback function if it exists. If
  1375. * this function fails it will mark the buffer with an error and
  1376. * the IO should not be dispatched.
  1377. */
  1378. if (bp->b_ops) {
  1379. bp->b_ops->verify_write(bp);
  1380. if (bp->b_error) {
  1381. xfs_force_shutdown(bp->b_mount,
  1382. SHUTDOWN_CORRUPT_INCORE);
  1383. return;
  1384. }
  1385. } else if (bp->b_bn != XFS_BUF_DADDR_NULL) {
  1386. struct xfs_mount *mp = bp->b_mount;
  1387. /*
  1388. * non-crc filesystems don't attach verifiers during
  1389. * log recovery, so don't warn for such filesystems.
  1390. */
  1391. if (xfs_sb_version_hascrc(&mp->m_sb)) {
  1392. xfs_warn(mp,
  1393. "%s: no buf ops on daddr 0x%llx len %d",
  1394. __func__, bp->b_bn, bp->b_length);
  1395. xfs_hex_dump(bp->b_addr,
  1396. XFS_CORRUPTION_DUMP_LEN);
  1397. dump_stack();
  1398. }
  1399. }
  1400. } else {
  1401. op = REQ_OP_READ;
  1402. if (bp->b_flags & XBF_READ_AHEAD)
  1403. op |= REQ_RAHEAD;
  1404. }
  1405. /* we only use the buffer cache for meta-data */
  1406. op |= REQ_META;
  1407. /*
  1408. * Walk all the vectors issuing IO on them. Set up the initial offset
  1409. * into the buffer and the desired IO size before we start -
  1410. * _xfs_buf_ioapply_vec() will modify them appropriately for each
  1411. * subsequent call.
  1412. */
  1413. offset = bp->b_offset;
  1414. size = BBTOB(bp->b_length);
  1415. blk_start_plug(&plug);
  1416. for (i = 0; i < bp->b_map_count; i++) {
  1417. xfs_buf_ioapply_map(bp, i, &offset, &size, op);
  1418. if (bp->b_error)
  1419. break;
  1420. if (size <= 0)
  1421. break; /* all done */
  1422. }
  1423. blk_finish_plug(&plug);
  1424. }
  1425. /*
  1426. * Wait for I/O completion of a sync buffer and return the I/O error code.
  1427. */
  1428. static int
  1429. xfs_buf_iowait(
  1430. struct xfs_buf *bp)
  1431. {
  1432. ASSERT(!(bp->b_flags & XBF_ASYNC));
  1433. trace_xfs_buf_iowait(bp, _RET_IP_);
  1434. wait_for_completion(&bp->b_iowait);
  1435. trace_xfs_buf_iowait_done(bp, _RET_IP_);
  1436. return bp->b_error;
  1437. }
  1438. /*
  1439. * Buffer I/O submission path, read or write. Asynchronous submission transfers
  1440. * the buffer lock ownership and the current reference to the IO. It is not
  1441. * safe to reference the buffer after a call to this function unless the caller
  1442. * holds an additional reference itself.
  1443. */
  1444. static int
  1445. __xfs_buf_submit(
  1446. struct xfs_buf *bp,
  1447. bool wait)
  1448. {
  1449. int error = 0;
  1450. trace_xfs_buf_submit(bp, _RET_IP_);
  1451. ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
  1452. /* on shutdown we stale and complete the buffer immediately */
  1453. if (XFS_FORCED_SHUTDOWN(bp->b_mount)) {
  1454. xfs_buf_ioend_fail(bp);
  1455. return -EIO;
  1456. }
  1457. /*
  1458. * Grab a reference so the buffer does not go away underneath us. For
  1459. * async buffers, I/O completion drops the callers reference, which
  1460. * could occur before submission returns.
  1461. */
  1462. xfs_buf_hold(bp);
  1463. if (bp->b_flags & XBF_WRITE)
  1464. xfs_buf_wait_unpin(bp);
  1465. /* clear the internal error state to avoid spurious errors */
  1466. bp->b_io_error = 0;
  1467. /*
  1468. * Set the count to 1 initially, this will stop an I/O completion
  1469. * callout which happens before we have started all the I/O from calling
  1470. * xfs_buf_ioend too early.
  1471. */
  1472. atomic_set(&bp->b_io_remaining, 1);
  1473. if (bp->b_flags & XBF_ASYNC)
  1474. xfs_buf_ioacct_inc(bp);
  1475. _xfs_buf_ioapply(bp);
  1476. /*
  1477. * If _xfs_buf_ioapply failed, we can get back here with only the IO
  1478. * reference we took above. If we drop it to zero, run completion so
  1479. * that we don't return to the caller with completion still pending.
  1480. */
  1481. if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
  1482. if (bp->b_error || !(bp->b_flags & XBF_ASYNC))
  1483. xfs_buf_ioend(bp);
  1484. else
  1485. xfs_buf_ioend_async(bp);
  1486. }
  1487. if (wait)
  1488. error = xfs_buf_iowait(bp);
  1489. /*
  1490. * Release the hold that keeps the buffer referenced for the entire
  1491. * I/O. Note that if the buffer is async, it is not safe to reference
  1492. * after this release.
  1493. */
  1494. xfs_buf_rele(bp);
  1495. return error;
  1496. }
  1497. void *
  1498. xfs_buf_offset(
  1499. struct xfs_buf *bp,
  1500. size_t offset)
  1501. {
  1502. struct page *page;
  1503. if (bp->b_addr)
  1504. return bp->b_addr + offset;
  1505. offset += bp->b_offset;
  1506. page = bp->b_pages[offset >> PAGE_SHIFT];
  1507. return page_address(page) + (offset & (PAGE_SIZE-1));
  1508. }
  1509. void
  1510. xfs_buf_zero(
  1511. struct xfs_buf *bp,
  1512. size_t boff,
  1513. size_t bsize)
  1514. {
  1515. size_t bend;
  1516. bend = boff + bsize;
  1517. while (boff < bend) {
  1518. struct page *page;
  1519. int page_index, page_offset, csize;
  1520. page_index = (boff + bp->b_offset) >> PAGE_SHIFT;
  1521. page_offset = (boff + bp->b_offset) & ~PAGE_MASK;
  1522. page = bp->b_pages[page_index];
  1523. csize = min_t(size_t, PAGE_SIZE - page_offset,
  1524. BBTOB(bp->b_length) - boff);
  1525. ASSERT((csize + page_offset) <= PAGE_SIZE);
  1526. memset(page_address(page) + page_offset, 0, csize);
  1527. boff += csize;
  1528. }
  1529. }
  1530. /*
  1531. * Log a message about and stale a buffer that a caller has decided is corrupt.
  1532. *
  1533. * This function should be called for the kinds of metadata corruption that
  1534. * cannot be detect from a verifier, such as incorrect inter-block relationship
  1535. * data. Do /not/ call this function from a verifier function.
  1536. *
  1537. * The buffer must be XBF_DONE prior to the call. Afterwards, the buffer will
  1538. * be marked stale, but b_error will not be set. The caller is responsible for
  1539. * releasing the buffer or fixing it.
  1540. */
  1541. void
  1542. __xfs_buf_mark_corrupt(
  1543. struct xfs_buf *bp,
  1544. xfs_failaddr_t fa)
  1545. {
  1546. ASSERT(bp->b_flags & XBF_DONE);
  1547. xfs_buf_corruption_error(bp, fa);
  1548. xfs_buf_stale(bp);
  1549. }
  1550. /*
  1551. * Handling of buffer targets (buftargs).
  1552. */
  1553. /*
  1554. * Wait for any bufs with callbacks that have been submitted but have not yet
  1555. * returned. These buffers will have an elevated hold count, so wait on those
  1556. * while freeing all the buffers only held by the LRU.
  1557. */
  1558. static enum lru_status
  1559. xfs_buftarg_wait_rele(
  1560. struct list_head *item,
  1561. struct list_lru_one *lru,
  1562. spinlock_t *lru_lock,
  1563. void *arg)
  1564. {
  1565. struct xfs_buf *bp = container_of(item, struct xfs_buf, b_lru);
  1566. struct list_head *dispose = arg;
  1567. if (atomic_read(&bp->b_hold) > 1) {
  1568. /* need to wait, so skip it this pass */
  1569. trace_xfs_buf_wait_buftarg(bp, _RET_IP_);
  1570. return LRU_SKIP;
  1571. }
  1572. if (!spin_trylock(&bp->b_lock))
  1573. return LRU_SKIP;
  1574. /*
  1575. * clear the LRU reference count so the buffer doesn't get
  1576. * ignored in xfs_buf_rele().
  1577. */
  1578. atomic_set(&bp->b_lru_ref, 0);
  1579. bp->b_state |= XFS_BSTATE_DISPOSE;
  1580. list_lru_isolate_move(lru, item, dispose);
  1581. spin_unlock(&bp->b_lock);
  1582. return LRU_REMOVED;
  1583. }
  1584. void
  1585. xfs_wait_buftarg(
  1586. struct xfs_buftarg *btp)
  1587. {
  1588. LIST_HEAD(dispose);
  1589. int loop = 0;
  1590. bool write_fail = false;
  1591. /*
  1592. * First wait on the buftarg I/O count for all in-flight buffers to be
  1593. * released. This is critical as new buffers do not make the LRU until
  1594. * they are released.
  1595. *
  1596. * Next, flush the buffer workqueue to ensure all completion processing
  1597. * has finished. Just waiting on buffer locks is not sufficient for
  1598. * async IO as the reference count held over IO is not released until
  1599. * after the buffer lock is dropped. Hence we need to ensure here that
  1600. * all reference counts have been dropped before we start walking the
  1601. * LRU list.
  1602. */
  1603. while (percpu_counter_sum(&btp->bt_io_count))
  1604. delay(100);
  1605. flush_workqueue(btp->bt_mount->m_buf_workqueue);
  1606. /* loop until there is nothing left on the lru list. */
  1607. while (list_lru_count(&btp->bt_lru)) {
  1608. list_lru_walk(&btp->bt_lru, xfs_buftarg_wait_rele,
  1609. &dispose, LONG_MAX);
  1610. while (!list_empty(&dispose)) {
  1611. struct xfs_buf *bp;
  1612. bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
  1613. list_del_init(&bp->b_lru);
  1614. if (bp->b_flags & XBF_WRITE_FAIL) {
  1615. write_fail = true;
  1616. xfs_buf_alert_ratelimited(bp,
  1617. "XFS: Corruption Alert",
  1618. "Corruption Alert: Buffer at daddr 0x%llx had permanent write failures!",
  1619. (long long)bp->b_bn);
  1620. }
  1621. xfs_buf_rele(bp);
  1622. }
  1623. if (loop++ != 0)
  1624. delay(100);
  1625. }
  1626. /*
  1627. * If one or more failed buffers were freed, that means dirty metadata
  1628. * was thrown away. This should only ever happen after I/O completion
  1629. * handling has elevated I/O error(s) to permanent failures and shuts
  1630. * down the fs.
  1631. */
  1632. if (write_fail) {
  1633. ASSERT(XFS_FORCED_SHUTDOWN(btp->bt_mount));
  1634. xfs_alert(btp->bt_mount,
  1635. "Please run xfs_repair to determine the extent of the problem.");
  1636. }
  1637. }
  1638. static enum lru_status
  1639. xfs_buftarg_isolate(
  1640. struct list_head *item,
  1641. struct list_lru_one *lru,
  1642. spinlock_t *lru_lock,
  1643. void *arg)
  1644. {
  1645. struct xfs_buf *bp = container_of(item, struct xfs_buf, b_lru);
  1646. struct list_head *dispose = arg;
  1647. /*
  1648. * we are inverting the lru lock/bp->b_lock here, so use a trylock.
  1649. * If we fail to get the lock, just skip it.
  1650. */
  1651. if (!spin_trylock(&bp->b_lock))
  1652. return LRU_SKIP;
  1653. /*
  1654. * Decrement the b_lru_ref count unless the value is already
  1655. * zero. If the value is already zero, we need to reclaim the
  1656. * buffer, otherwise it gets another trip through the LRU.
  1657. */
  1658. if (atomic_add_unless(&bp->b_lru_ref, -1, 0)) {
  1659. spin_unlock(&bp->b_lock);
  1660. return LRU_ROTATE;
  1661. }
  1662. bp->b_state |= XFS_BSTATE_DISPOSE;
  1663. list_lru_isolate_move(lru, item, dispose);
  1664. spin_unlock(&bp->b_lock);
  1665. return LRU_REMOVED;
  1666. }
  1667. static unsigned long
  1668. xfs_buftarg_shrink_scan(
  1669. struct shrinker *shrink,
  1670. struct shrink_control *sc)
  1671. {
  1672. struct xfs_buftarg *btp = container_of(shrink,
  1673. struct xfs_buftarg, bt_shrinker);
  1674. LIST_HEAD(dispose);
  1675. unsigned long freed;
  1676. freed = list_lru_shrink_walk(&btp->bt_lru, sc,
  1677. xfs_buftarg_isolate, &dispose);
  1678. while (!list_empty(&dispose)) {
  1679. struct xfs_buf *bp;
  1680. bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
  1681. list_del_init(&bp->b_lru);
  1682. xfs_buf_rele(bp);
  1683. }
  1684. return freed;
  1685. }
  1686. static unsigned long
  1687. xfs_buftarg_shrink_count(
  1688. struct shrinker *shrink,
  1689. struct shrink_control *sc)
  1690. {
  1691. struct xfs_buftarg *btp = container_of(shrink,
  1692. struct xfs_buftarg, bt_shrinker);
  1693. return list_lru_shrink_count(&btp->bt_lru, sc);
  1694. }
  1695. void
  1696. xfs_free_buftarg(
  1697. struct xfs_buftarg *btp)
  1698. {
  1699. unregister_shrinker(&btp->bt_shrinker);
  1700. ASSERT(percpu_counter_sum(&btp->bt_io_count) == 0);
  1701. percpu_counter_destroy(&btp->bt_io_count);
  1702. list_lru_destroy(&btp->bt_lru);
  1703. xfs_blkdev_issue_flush(btp);
  1704. kmem_free(btp);
  1705. }
  1706. int
  1707. xfs_setsize_buftarg(
  1708. xfs_buftarg_t *btp,
  1709. unsigned int sectorsize)
  1710. {
  1711. /* Set up metadata sector size info */
  1712. btp->bt_meta_sectorsize = sectorsize;
  1713. btp->bt_meta_sectormask = sectorsize - 1;
  1714. if (set_blocksize(btp->bt_bdev, sectorsize)) {
  1715. xfs_warn(btp->bt_mount,
  1716. "Cannot set_blocksize to %u on device %pg",
  1717. sectorsize, btp->bt_bdev);
  1718. return -EINVAL;
  1719. }
  1720. /* Set up device logical sector size mask */
  1721. btp->bt_logical_sectorsize = bdev_logical_block_size(btp->bt_bdev);
  1722. btp->bt_logical_sectormask = bdev_logical_block_size(btp->bt_bdev) - 1;
  1723. return 0;
  1724. }
  1725. /*
  1726. * When allocating the initial buffer target we have not yet
  1727. * read in the superblock, so don't know what sized sectors
  1728. * are being used at this early stage. Play safe.
  1729. */
  1730. STATIC int
  1731. xfs_setsize_buftarg_early(
  1732. xfs_buftarg_t *btp,
  1733. struct block_device *bdev)
  1734. {
  1735. return xfs_setsize_buftarg(btp, bdev_logical_block_size(bdev));
  1736. }
  1737. xfs_buftarg_t *
  1738. xfs_alloc_buftarg(
  1739. struct xfs_mount *mp,
  1740. struct block_device *bdev,
  1741. struct dax_device *dax_dev)
  1742. {
  1743. xfs_buftarg_t *btp;
  1744. btp = kmem_zalloc(sizeof(*btp), KM_NOFS);
  1745. btp->bt_mount = mp;
  1746. btp->bt_dev = bdev->bd_dev;
  1747. btp->bt_bdev = bdev;
  1748. btp->bt_daxdev = dax_dev;
  1749. /*
  1750. * Buffer IO error rate limiting. Limit it to no more than 10 messages
  1751. * per 30 seconds so as to not spam logs too much on repeated errors.
  1752. */
  1753. ratelimit_state_init(&btp->bt_ioerror_rl, 30 * HZ,
  1754. DEFAULT_RATELIMIT_BURST);
  1755. if (xfs_setsize_buftarg_early(btp, bdev))
  1756. goto error_free;
  1757. if (list_lru_init(&btp->bt_lru))
  1758. goto error_free;
  1759. if (percpu_counter_init(&btp->bt_io_count, 0, GFP_KERNEL))
  1760. goto error_lru;
  1761. btp->bt_shrinker.count_objects = xfs_buftarg_shrink_count;
  1762. btp->bt_shrinker.scan_objects = xfs_buftarg_shrink_scan;
  1763. btp->bt_shrinker.seeks = DEFAULT_SEEKS;
  1764. btp->bt_shrinker.flags = SHRINKER_NUMA_AWARE;
  1765. if (register_shrinker(&btp->bt_shrinker))
  1766. goto error_pcpu;
  1767. return btp;
  1768. error_pcpu:
  1769. percpu_counter_destroy(&btp->bt_io_count);
  1770. error_lru:
  1771. list_lru_destroy(&btp->bt_lru);
  1772. error_free:
  1773. kmem_free(btp);
  1774. return NULL;
  1775. }
  1776. /*
  1777. * Cancel a delayed write list.
  1778. *
  1779. * Remove each buffer from the list, clear the delwri queue flag and drop the
  1780. * associated buffer reference.
  1781. */
  1782. void
  1783. xfs_buf_delwri_cancel(
  1784. struct list_head *list)
  1785. {
  1786. struct xfs_buf *bp;
  1787. while (!list_empty(list)) {
  1788. bp = list_first_entry(list, struct xfs_buf, b_list);
  1789. xfs_buf_lock(bp);
  1790. bp->b_flags &= ~_XBF_DELWRI_Q;
  1791. list_del_init(&bp->b_list);
  1792. xfs_buf_relse(bp);
  1793. }
  1794. }
  1795. /*
  1796. * Add a buffer to the delayed write list.
  1797. *
  1798. * This queues a buffer for writeout if it hasn't already been. Note that
  1799. * neither this routine nor the buffer list submission functions perform
  1800. * any internal synchronization. It is expected that the lists are thread-local
  1801. * to the callers.
  1802. *
  1803. * Returns true if we queued up the buffer, or false if it already had
  1804. * been on the buffer list.
  1805. */
  1806. bool
  1807. xfs_buf_delwri_queue(
  1808. struct xfs_buf *bp,
  1809. struct list_head *list)
  1810. {
  1811. ASSERT(xfs_buf_islocked(bp));
  1812. ASSERT(!(bp->b_flags & XBF_READ));
  1813. /*
  1814. * If the buffer is already marked delwri it already is queued up
  1815. * by someone else for imediate writeout. Just ignore it in that
  1816. * case.
  1817. */
  1818. if (bp->b_flags & _XBF_DELWRI_Q) {
  1819. trace_xfs_buf_delwri_queued(bp, _RET_IP_);
  1820. return false;
  1821. }
  1822. trace_xfs_buf_delwri_queue(bp, _RET_IP_);
  1823. /*
  1824. * If a buffer gets written out synchronously or marked stale while it
  1825. * is on a delwri list we lazily remove it. To do this, the other party
  1826. * clears the _XBF_DELWRI_Q flag but otherwise leaves the buffer alone.
  1827. * It remains referenced and on the list. In a rare corner case it
  1828. * might get readded to a delwri list after the synchronous writeout, in
  1829. * which case we need just need to re-add the flag here.
  1830. */
  1831. bp->b_flags |= _XBF_DELWRI_Q;
  1832. if (list_empty(&bp->b_list)) {
  1833. atomic_inc(&bp->b_hold);
  1834. list_add_tail(&bp->b_list, list);
  1835. }
  1836. return true;
  1837. }
  1838. /*
  1839. * Compare function is more complex than it needs to be because
  1840. * the return value is only 32 bits and we are doing comparisons
  1841. * on 64 bit values
  1842. */
  1843. static int
  1844. xfs_buf_cmp(
  1845. void *priv,
  1846. struct list_head *a,
  1847. struct list_head *b)
  1848. {
  1849. struct xfs_buf *ap = container_of(a, struct xfs_buf, b_list);
  1850. struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list);
  1851. xfs_daddr_t diff;
  1852. diff = ap->b_maps[0].bm_bn - bp->b_maps[0].bm_bn;
  1853. if (diff < 0)
  1854. return -1;
  1855. if (diff > 0)
  1856. return 1;
  1857. return 0;
  1858. }
  1859. /*
  1860. * Submit buffers for write. If wait_list is specified, the buffers are
  1861. * submitted using sync I/O and placed on the wait list such that the caller can
  1862. * iowait each buffer. Otherwise async I/O is used and the buffers are released
  1863. * at I/O completion time. In either case, buffers remain locked until I/O
  1864. * completes and the buffer is released from the queue.
  1865. */
  1866. static int
  1867. xfs_buf_delwri_submit_buffers(
  1868. struct list_head *buffer_list,
  1869. struct list_head *wait_list)
  1870. {
  1871. struct xfs_buf *bp, *n;
  1872. int pinned = 0;
  1873. struct blk_plug plug;
  1874. list_sort(NULL, buffer_list, xfs_buf_cmp);
  1875. blk_start_plug(&plug);
  1876. list_for_each_entry_safe(bp, n, buffer_list, b_list) {
  1877. if (!wait_list) {
  1878. if (xfs_buf_ispinned(bp)) {
  1879. pinned++;
  1880. continue;
  1881. }
  1882. if (!xfs_buf_trylock(bp))
  1883. continue;
  1884. } else {
  1885. xfs_buf_lock(bp);
  1886. }
  1887. /*
  1888. * Someone else might have written the buffer synchronously or
  1889. * marked it stale in the meantime. In that case only the
  1890. * _XBF_DELWRI_Q flag got cleared, and we have to drop the
  1891. * reference and remove it from the list here.
  1892. */
  1893. if (!(bp->b_flags & _XBF_DELWRI_Q)) {
  1894. list_del_init(&bp->b_list);
  1895. xfs_buf_relse(bp);
  1896. continue;
  1897. }
  1898. trace_xfs_buf_delwri_split(bp, _RET_IP_);
  1899. /*
  1900. * If we have a wait list, each buffer (and associated delwri
  1901. * queue reference) transfers to it and is submitted
  1902. * synchronously. Otherwise, drop the buffer from the delwri
  1903. * queue and submit async.
  1904. */
  1905. bp->b_flags &= ~_XBF_DELWRI_Q;
  1906. bp->b_flags |= XBF_WRITE;
  1907. if (wait_list) {
  1908. bp->b_flags &= ~XBF_ASYNC;
  1909. list_move_tail(&bp->b_list, wait_list);
  1910. } else {
  1911. bp->b_flags |= XBF_ASYNC;
  1912. list_del_init(&bp->b_list);
  1913. }
  1914. __xfs_buf_submit(bp, false);
  1915. }
  1916. blk_finish_plug(&plug);
  1917. return pinned;
  1918. }
  1919. /*
  1920. * Write out a buffer list asynchronously.
  1921. *
  1922. * This will take the @buffer_list, write all non-locked and non-pinned buffers
  1923. * out and not wait for I/O completion on any of the buffers. This interface
  1924. * is only safely useable for callers that can track I/O completion by higher
  1925. * level means, e.g. AIL pushing as the @buffer_list is consumed in this
  1926. * function.
  1927. *
  1928. * Note: this function will skip buffers it would block on, and in doing so
  1929. * leaves them on @buffer_list so they can be retried on a later pass. As such,
  1930. * it is up to the caller to ensure that the buffer list is fully submitted or
  1931. * cancelled appropriately when they are finished with the list. Failure to
  1932. * cancel or resubmit the list until it is empty will result in leaked buffers
  1933. * at unmount time.
  1934. */
  1935. int
  1936. xfs_buf_delwri_submit_nowait(
  1937. struct list_head *buffer_list)
  1938. {
  1939. return xfs_buf_delwri_submit_buffers(buffer_list, NULL);
  1940. }
  1941. /*
  1942. * Write out a buffer list synchronously.
  1943. *
  1944. * This will take the @buffer_list, write all buffers out and wait for I/O
  1945. * completion on all of the buffers. @buffer_list is consumed by the function,
  1946. * so callers must have some other way of tracking buffers if they require such
  1947. * functionality.
  1948. */
  1949. int
  1950. xfs_buf_delwri_submit(
  1951. struct list_head *buffer_list)
  1952. {
  1953. LIST_HEAD (wait_list);
  1954. int error = 0, error2;
  1955. struct xfs_buf *bp;
  1956. xfs_buf_delwri_submit_buffers(buffer_list, &wait_list);
  1957. /* Wait for IO to complete. */
  1958. while (!list_empty(&wait_list)) {
  1959. bp = list_first_entry(&wait_list, struct xfs_buf, b_list);
  1960. list_del_init(&bp->b_list);
  1961. /*
  1962. * Wait on the locked buffer, check for errors and unlock and
  1963. * release the delwri queue reference.
  1964. */
  1965. error2 = xfs_buf_iowait(bp);
  1966. xfs_buf_relse(bp);
  1967. if (!error)
  1968. error = error2;
  1969. }
  1970. return error;
  1971. }
  1972. /*
  1973. * Push a single buffer on a delwri queue.
  1974. *
  1975. * The purpose of this function is to submit a single buffer of a delwri queue
  1976. * and return with the buffer still on the original queue. The waiting delwri
  1977. * buffer submission infrastructure guarantees transfer of the delwri queue
  1978. * buffer reference to a temporary wait list. We reuse this infrastructure to
  1979. * transfer the buffer back to the original queue.
  1980. *
  1981. * Note the buffer transitions from the queued state, to the submitted and wait
  1982. * listed state and back to the queued state during this call. The buffer
  1983. * locking and queue management logic between _delwri_pushbuf() and
  1984. * _delwri_queue() guarantee that the buffer cannot be queued to another list
  1985. * before returning.
  1986. */
  1987. int
  1988. xfs_buf_delwri_pushbuf(
  1989. struct xfs_buf *bp,
  1990. struct list_head *buffer_list)
  1991. {
  1992. LIST_HEAD (submit_list);
  1993. int error;
  1994. ASSERT(bp->b_flags & _XBF_DELWRI_Q);
  1995. trace_xfs_buf_delwri_pushbuf(bp, _RET_IP_);
  1996. /*
  1997. * Isolate the buffer to a new local list so we can submit it for I/O
  1998. * independently from the rest of the original list.
  1999. */
  2000. xfs_buf_lock(bp);
  2001. list_move(&bp->b_list, &submit_list);
  2002. xfs_buf_unlock(bp);
  2003. /*
  2004. * Delwri submission clears the DELWRI_Q buffer flag and returns with
  2005. * the buffer on the wait list with the original reference. Rather than
  2006. * bounce the buffer from a local wait list back to the original list
  2007. * after I/O completion, reuse the original list as the wait list.
  2008. */
  2009. xfs_buf_delwri_submit_buffers(&submit_list, buffer_list);
  2010. /*
  2011. * The buffer is now locked, under I/O and wait listed on the original
  2012. * delwri queue. Wait for I/O completion, restore the DELWRI_Q flag and
  2013. * return with the buffer unlocked and on the original queue.
  2014. */
  2015. error = xfs_buf_iowait(bp);
  2016. bp->b_flags |= _XBF_DELWRI_Q;
  2017. xfs_buf_unlock(bp);
  2018. return error;
  2019. }
  2020. int __init
  2021. xfs_buf_init(void)
  2022. {
  2023. xfs_buf_zone = kmem_cache_create("xfs_buf", sizeof(struct xfs_buf), 0,
  2024. SLAB_HWCACHE_ALIGN |
  2025. SLAB_RECLAIM_ACCOUNT |
  2026. SLAB_MEM_SPREAD,
  2027. NULL);
  2028. if (!xfs_buf_zone)
  2029. goto out;
  2030. return 0;
  2031. out:
  2032. return -ENOMEM;
  2033. }
  2034. void
  2035. xfs_buf_terminate(void)
  2036. {
  2037. kmem_cache_destroy(xfs_buf_zone);
  2038. }
  2039. void xfs_buf_set_ref(struct xfs_buf *bp, int lru_ref)
  2040. {
  2041. /*
  2042. * Set the lru reference count to 0 based on the error injection tag.
  2043. * This allows userspace to disrupt buffer caching for debug/testing
  2044. * purposes.
  2045. */
  2046. if (XFS_TEST_ERROR(false, bp->b_mount, XFS_ERRTAG_BUF_LRU_REF))
  2047. lru_ref = 0;
  2048. atomic_set(&bp->b_lru_ref, lru_ref);
  2049. }
  2050. /*
  2051. * Verify an on-disk magic value against the magic value specified in the
  2052. * verifier structure. The verifier magic is in disk byte order so the caller is
  2053. * expected to pass the value directly from disk.
  2054. */
  2055. bool
  2056. xfs_verify_magic(
  2057. struct xfs_buf *bp,
  2058. __be32 dmagic)
  2059. {
  2060. struct xfs_mount *mp = bp->b_mount;
  2061. int idx;
  2062. idx = xfs_sb_version_hascrc(&mp->m_sb);
  2063. if (WARN_ON(!bp->b_ops || !bp->b_ops->magic[idx]))
  2064. return false;
  2065. return dmagic == bp->b_ops->magic[idx];
  2066. }
  2067. /*
  2068. * Verify an on-disk magic value against the magic value specified in the
  2069. * verifier structure. The verifier magic is in disk byte order so the caller is
  2070. * expected to pass the value directly from disk.
  2071. */
  2072. bool
  2073. xfs_verify_magic16(
  2074. struct xfs_buf *bp,
  2075. __be16 dmagic)
  2076. {
  2077. struct xfs_mount *mp = bp->b_mount;
  2078. int idx;
  2079. idx = xfs_sb_version_hascrc(&mp->m_sb);
  2080. if (WARN_ON(!bp->b_ops || !bp->b_ops->magic16[idx]))
  2081. return false;
  2082. return dmagic == bp->b_ops->magic16[idx];
  2083. }