raid5.h 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _RAID5_H
  3. #define _RAID5_H
  4. #include <linux/raid/xor.h>
  5. #include <linux/dmaengine.h>
  6. /*
  7. *
  8. * Each stripe contains one buffer per device. Each buffer can be in
  9. * one of a number of states stored in "flags". Changes between
  10. * these states happen *almost* exclusively under the protection of the
  11. * STRIPE_ACTIVE flag. Some very specific changes can happen in bi_end_io, and
  12. * these are not protected by STRIPE_ACTIVE.
  13. *
  14. * The flag bits that are used to represent these states are:
  15. * R5_UPTODATE and R5_LOCKED
  16. *
  17. * State Empty == !UPTODATE, !LOCK
  18. * We have no data, and there is no active request
  19. * State Want == !UPTODATE, LOCK
  20. * A read request is being submitted for this block
  21. * State Dirty == UPTODATE, LOCK
  22. * Some new data is in this buffer, and it is being written out
  23. * State Clean == UPTODATE, !LOCK
  24. * We have valid data which is the same as on disc
  25. *
  26. * The possible state transitions are:
  27. *
  28. * Empty -> Want - on read or write to get old data for parity calc
  29. * Empty -> Dirty - on compute_parity to satisfy write/sync request.
  30. * Empty -> Clean - on compute_block when computing a block for failed drive
  31. * Want -> Empty - on failed read
  32. * Want -> Clean - on successful completion of read request
  33. * Dirty -> Clean - on successful completion of write request
  34. * Dirty -> Clean - on failed write
  35. * Clean -> Dirty - on compute_parity to satisfy write/sync (RECONSTRUCT or RMW)
  36. *
  37. * The Want->Empty, Want->Clean, Dirty->Clean, transitions
  38. * all happen in b_end_io at interrupt time.
  39. * Each sets the Uptodate bit before releasing the Lock bit.
  40. * This leaves one multi-stage transition:
  41. * Want->Dirty->Clean
  42. * This is safe because thinking that a Clean buffer is actually dirty
  43. * will at worst delay some action, and the stripe will be scheduled
  44. * for attention after the transition is complete.
  45. *
  46. * There is one possibility that is not covered by these states. That
  47. * is if one drive has failed and there is a spare being rebuilt. We
  48. * can't distinguish between a clean block that has been generated
  49. * from parity calculations, and a clean block that has been
  50. * successfully written to the spare ( or to parity when resyncing).
  51. * To distinguish these states we have a stripe bit STRIPE_INSYNC that
  52. * is set whenever a write is scheduled to the spare, or to the parity
  53. * disc if there is no spare. A sync request clears this bit, and
  54. * when we find it set with no buffers locked, we know the sync is
  55. * complete.
  56. *
  57. * Buffers for the md device that arrive via make_request are attached
  58. * to the appropriate stripe in one of two lists linked on b_reqnext.
  59. * One list (bh_read) for read requests, one (bh_write) for write.
  60. * There should never be more than one buffer on the two lists
  61. * together, but we are not guaranteed of that so we allow for more.
  62. *
  63. * If a buffer is on the read list when the associated cache buffer is
  64. * Uptodate, the data is copied into the read buffer and it's b_end_io
  65. * routine is called. This may happen in the end_request routine only
  66. * if the buffer has just successfully been read. end_request should
  67. * remove the buffers from the list and then set the Uptodate bit on
  68. * the buffer. Other threads may do this only if they first check
  69. * that the Uptodate bit is set. Once they have checked that they may
  70. * take buffers off the read queue.
  71. *
  72. * When a buffer on the write list is committed for write it is copied
  73. * into the cache buffer, which is then marked dirty, and moved onto a
  74. * third list, the written list (bh_written). Once both the parity
  75. * block and the cached buffer are successfully written, any buffer on
  76. * a written list can be returned with b_end_io.
  77. *
  78. * The write list and read list both act as fifos. The read list,
  79. * write list and written list are protected by the device_lock.
  80. * The device_lock is only for list manipulations and will only be
  81. * held for a very short time. It can be claimed from interrupts.
  82. *
  83. *
  84. * Stripes in the stripe cache can be on one of two lists (or on
  85. * neither). The "inactive_list" contains stripes which are not
  86. * currently being used for any request. They can freely be reused
  87. * for another stripe. The "handle_list" contains stripes that need
  88. * to be handled in some way. Both of these are fifo queues. Each
  89. * stripe is also (potentially) linked to a hash bucket in the hash
  90. * table so that it can be found by sector number. Stripes that are
  91. * not hashed must be on the inactive_list, and will normally be at
  92. * the front. All stripes start life this way.
  93. *
  94. * The inactive_list, handle_list and hash bucket lists are all protected by the
  95. * device_lock.
  96. * - stripes have a reference counter. If count==0, they are on a list.
  97. * - If a stripe might need handling, STRIPE_HANDLE is set.
  98. * - When refcount reaches zero, then if STRIPE_HANDLE it is put on
  99. * handle_list else inactive_list
  100. *
  101. * This, combined with the fact that STRIPE_HANDLE is only ever
  102. * cleared while a stripe has a non-zero count means that if the
  103. * refcount is 0 and STRIPE_HANDLE is set, then it is on the
  104. * handle_list and if recount is 0 and STRIPE_HANDLE is not set, then
  105. * the stripe is on inactive_list.
  106. *
  107. * The possible transitions are:
  108. * activate an unhashed/inactive stripe (get_active_stripe())
  109. * lockdev check-hash unlink-stripe cnt++ clean-stripe hash-stripe unlockdev
  110. * activate a hashed, possibly active stripe (get_active_stripe())
  111. * lockdev check-hash if(!cnt++)unlink-stripe unlockdev
  112. * attach a request to an active stripe (add_stripe_bh())
  113. * lockdev attach-buffer unlockdev
  114. * handle a stripe (handle_stripe())
  115. * setSTRIPE_ACTIVE, clrSTRIPE_HANDLE ...
  116. * (lockdev check-buffers unlockdev) ..
  117. * change-state ..
  118. * record io/ops needed clearSTRIPE_ACTIVE schedule io/ops
  119. * release an active stripe (release_stripe())
  120. * lockdev if (!--cnt) { if STRIPE_HANDLE, add to handle_list else add to inactive-list } unlockdev
  121. *
  122. * The refcount counts each thread that have activated the stripe,
  123. * plus raid5d if it is handling it, plus one for each active request
  124. * on a cached buffer, and plus one if the stripe is undergoing stripe
  125. * operations.
  126. *
  127. * The stripe operations are:
  128. * -copying data between the stripe cache and user application buffers
  129. * -computing blocks to save a disk access, or to recover a missing block
  130. * -updating the parity on a write operation (reconstruct write and
  131. * read-modify-write)
  132. * -checking parity correctness
  133. * -running i/o to disk
  134. * These operations are carried out by raid5_run_ops which uses the async_tx
  135. * api to (optionally) offload operations to dedicated hardware engines.
  136. * When requesting an operation handle_stripe sets the pending bit for the
  137. * operation and increments the count. raid5_run_ops is then run whenever
  138. * the count is non-zero.
  139. * There are some critical dependencies between the operations that prevent some
  140. * from being requested while another is in flight.
  141. * 1/ Parity check operations destroy the in cache version of the parity block,
  142. * so we prevent parity dependent operations like writes and compute_blocks
  143. * from starting while a check is in progress. Some dma engines can perform
  144. * the check without damaging the parity block, in these cases the parity
  145. * block is re-marked up to date (assuming the check was successful) and is
  146. * not re-read from disk.
  147. * 2/ When a write operation is requested we immediately lock the affected
  148. * blocks, and mark them as not up to date. This causes new read requests
  149. * to be held off, as well as parity checks and compute block operations.
  150. * 3/ Once a compute block operation has been requested handle_stripe treats
  151. * that block as if it is up to date. raid5_run_ops guaruntees that any
  152. * operation that is dependent on the compute block result is initiated after
  153. * the compute block completes.
  154. */
  155. /*
  156. * Operations state - intermediate states that are visible outside of
  157. * STRIPE_ACTIVE.
  158. * In general _idle indicates nothing is running, _run indicates a data
  159. * processing operation is active, and _result means the data processing result
  160. * is stable and can be acted upon. For simple operations like biofill and
  161. * compute that only have an _idle and _run state they are indicated with
  162. * sh->state flags (STRIPE_BIOFILL_RUN and STRIPE_COMPUTE_RUN)
  163. */
  164. /**
  165. * enum check_states - handles syncing / repairing a stripe
  166. * @check_state_idle - check operations are quiesced
  167. * @check_state_run - check operation is running
  168. * @check_state_result - set outside lock when check result is valid
  169. * @check_state_compute_run - check failed and we are repairing
  170. * @check_state_compute_result - set outside lock when compute result is valid
  171. */
  172. enum check_states {
  173. check_state_idle = 0,
  174. check_state_run, /* xor parity check */
  175. check_state_run_q, /* q-parity check */
  176. check_state_run_pq, /* pq dual parity check */
  177. check_state_check_result,
  178. check_state_compute_run, /* parity repair */
  179. check_state_compute_result,
  180. };
  181. /**
  182. * enum reconstruct_states - handles writing or expanding a stripe
  183. */
  184. enum reconstruct_states {
  185. reconstruct_state_idle = 0,
  186. reconstruct_state_prexor_drain_run, /* prexor-write */
  187. reconstruct_state_drain_run, /* write */
  188. reconstruct_state_run, /* expand */
  189. reconstruct_state_prexor_drain_result,
  190. reconstruct_state_drain_result,
  191. reconstruct_state_result,
  192. };
  193. #define DEFAULT_STRIPE_SIZE 4096
  194. struct stripe_head {
  195. struct hlist_node hash;
  196. struct list_head lru; /* inactive_list or handle_list */
  197. struct llist_node release_list;
  198. struct r5conf *raid_conf;
  199. short generation; /* increments with every
  200. * reshape */
  201. sector_t sector; /* sector of this row */
  202. short pd_idx; /* parity disk index */
  203. short qd_idx; /* 'Q' disk index for raid6 */
  204. short ddf_layout;/* use DDF ordering to calculate Q */
  205. short hash_lock_index;
  206. unsigned long state; /* state flags */
  207. atomic_t count; /* nr of active thread/requests */
  208. int bm_seq; /* sequence number for bitmap flushes */
  209. int disks; /* disks in stripe */
  210. int overwrite_disks; /* total overwrite disks in stripe,
  211. * this is only checked when stripe
  212. * has STRIPE_BATCH_READY
  213. */
  214. enum check_states check_state;
  215. enum reconstruct_states reconstruct_state;
  216. spinlock_t stripe_lock;
  217. int cpu;
  218. struct r5worker_group *group;
  219. struct stripe_head *batch_head; /* protected by stripe lock */
  220. spinlock_t batch_lock; /* only header's lock is useful */
  221. struct list_head batch_list; /* protected by head's batch lock*/
  222. union {
  223. struct r5l_io_unit *log_io;
  224. struct ppl_io_unit *ppl_io;
  225. };
  226. struct list_head log_list;
  227. sector_t log_start; /* first meta block on the journal */
  228. struct list_head r5c; /* for r5c_cache->stripe_in_journal */
  229. struct page *ppl_page; /* partial parity of this stripe */
  230. /**
  231. * struct stripe_operations
  232. * @target - STRIPE_OP_COMPUTE_BLK target
  233. * @target2 - 2nd compute target in the raid6 case
  234. * @zero_sum_result - P and Q verification flags
  235. * @request - async service request flags for raid_run_ops
  236. */
  237. struct stripe_operations {
  238. int target, target2;
  239. enum sum_check_flags zero_sum_result;
  240. } ops;
  241. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  242. /* These pages will be used by bios in dev[i] */
  243. struct page **pages;
  244. int nr_pages; /* page array size */
  245. int stripes_per_page;
  246. #endif
  247. struct r5dev {
  248. /* rreq and rvec are used for the replacement device when
  249. * writing data to both devices.
  250. */
  251. struct bio req, rreq;
  252. struct bio_vec vec, rvec;
  253. struct page *page, *orig_page;
  254. unsigned int offset; /* offset of the page */
  255. struct bio *toread, *read, *towrite, *written;
  256. sector_t sector; /* sector of this page */
  257. unsigned long flags;
  258. u32 log_checksum;
  259. unsigned short write_hint;
  260. } dev[1]; /* allocated with extra space depending of RAID geometry */
  261. };
  262. /* stripe_head_state - collects and tracks the dynamic state of a stripe_head
  263. * for handle_stripe.
  264. */
  265. struct stripe_head_state {
  266. /* 'syncing' means that we need to read all devices, either
  267. * to check/correct parity, or to reconstruct a missing device.
  268. * 'replacing' means we are replacing one or more drives and
  269. * the source is valid at this point so we don't need to
  270. * read all devices, just the replacement targets.
  271. */
  272. int syncing, expanding, expanded, replacing;
  273. int locked, uptodate, to_read, to_write, failed, written;
  274. int to_fill, compute, req_compute, non_overwrite;
  275. int injournal, just_cached;
  276. int failed_num[2];
  277. int p_failed, q_failed;
  278. int dec_preread_active;
  279. unsigned long ops_request;
  280. struct md_rdev *blocked_rdev;
  281. int handle_bad_blocks;
  282. int log_failed;
  283. int waiting_extra_page;
  284. };
  285. /* Flags for struct r5dev.flags */
  286. enum r5dev_flags {
  287. R5_UPTODATE, /* page contains current data */
  288. R5_LOCKED, /* IO has been submitted on "req" */
  289. R5_DOUBLE_LOCKED,/* Cannot clear R5_LOCKED until 2 writes complete */
  290. R5_OVERWRITE, /* towrite covers whole page */
  291. /* and some that are internal to handle_stripe */
  292. R5_Insync, /* rdev && rdev->in_sync at start */
  293. R5_Wantread, /* want to schedule a read */
  294. R5_Wantwrite,
  295. R5_Overlap, /* There is a pending overlapping request
  296. * on this block */
  297. R5_ReadNoMerge, /* prevent bio from merging in block-layer */
  298. R5_ReadError, /* seen a read error here recently */
  299. R5_ReWrite, /* have tried to over-write the readerror */
  300. R5_Expanded, /* This block now has post-expand data */
  301. R5_Wantcompute, /* compute_block in progress treat as
  302. * uptodate
  303. */
  304. R5_Wantfill, /* dev->toread contains a bio that needs
  305. * filling
  306. */
  307. R5_Wantdrain, /* dev->towrite needs to be drained */
  308. R5_WantFUA, /* Write should be FUA */
  309. R5_SyncIO, /* The IO is sync */
  310. R5_WriteError, /* got a write error - need to record it */
  311. R5_MadeGood, /* A bad block has been fixed by writing to it */
  312. R5_ReadRepl, /* Will/did read from replacement rather than orig */
  313. R5_MadeGoodRepl,/* A bad block on the replacement device has been
  314. * fixed by writing to it */
  315. R5_NeedReplace, /* This device has a replacement which is not
  316. * up-to-date at this stripe. */
  317. R5_WantReplace, /* We need to update the replacement, we have read
  318. * data in, and now is a good time to write it out.
  319. */
  320. R5_Discard, /* Discard the stripe */
  321. R5_SkipCopy, /* Don't copy data from bio to stripe cache */
  322. R5_InJournal, /* data being written is in the journal device.
  323. * if R5_InJournal is set for parity pd_idx, all the
  324. * data and parity being written are in the journal
  325. * device
  326. */
  327. R5_OrigPageUPTDODATE, /* with write back cache, we read old data into
  328. * dev->orig_page for prexor. When this flag is
  329. * set, orig_page contains latest data in the
  330. * raid disk.
  331. */
  332. };
  333. /*
  334. * Stripe state
  335. */
  336. enum {
  337. STRIPE_ACTIVE,
  338. STRIPE_HANDLE,
  339. STRIPE_SYNC_REQUESTED,
  340. STRIPE_SYNCING,
  341. STRIPE_INSYNC,
  342. STRIPE_REPLACED,
  343. STRIPE_PREREAD_ACTIVE,
  344. STRIPE_DELAYED,
  345. STRIPE_DEGRADED,
  346. STRIPE_BIT_DELAY,
  347. STRIPE_EXPANDING,
  348. STRIPE_EXPAND_SOURCE,
  349. STRIPE_EXPAND_READY,
  350. STRIPE_IO_STARTED, /* do not count towards 'bypass_count' */
  351. STRIPE_FULL_WRITE, /* all blocks are set to be overwritten */
  352. STRIPE_BIOFILL_RUN,
  353. STRIPE_COMPUTE_RUN,
  354. STRIPE_ON_UNPLUG_LIST,
  355. STRIPE_DISCARD,
  356. STRIPE_ON_RELEASE_LIST,
  357. STRIPE_BATCH_READY,
  358. STRIPE_BATCH_ERR,
  359. STRIPE_BITMAP_PENDING, /* Being added to bitmap, don't add
  360. * to batch yet.
  361. */
  362. STRIPE_LOG_TRAPPED, /* trapped into log (see raid5-cache.c)
  363. * this bit is used in two scenarios:
  364. *
  365. * 1. write-out phase
  366. * set in first entry of r5l_write_stripe
  367. * clear in second entry of r5l_write_stripe
  368. * used to bypass logic in handle_stripe
  369. *
  370. * 2. caching phase
  371. * set in r5c_try_caching_write()
  372. * clear when journal write is done
  373. * used to initiate r5c_cache_data()
  374. * also used to bypass logic in handle_stripe
  375. */
  376. STRIPE_R5C_CACHING, /* the stripe is in caching phase
  377. * see more detail in the raid5-cache.c
  378. */
  379. STRIPE_R5C_PARTIAL_STRIPE, /* in r5c cache (to-be/being handled or
  380. * in conf->r5c_partial_stripe_list)
  381. */
  382. STRIPE_R5C_FULL_STRIPE, /* in r5c cache (to-be/being handled or
  383. * in conf->r5c_full_stripe_list)
  384. */
  385. STRIPE_R5C_PREFLUSH, /* need to flush journal device */
  386. };
  387. #define STRIPE_EXPAND_SYNC_FLAGS \
  388. ((1 << STRIPE_EXPAND_SOURCE) |\
  389. (1 << STRIPE_EXPAND_READY) |\
  390. (1 << STRIPE_EXPANDING) |\
  391. (1 << STRIPE_SYNC_REQUESTED))
  392. /*
  393. * Operation request flags
  394. */
  395. enum {
  396. STRIPE_OP_BIOFILL,
  397. STRIPE_OP_COMPUTE_BLK,
  398. STRIPE_OP_PREXOR,
  399. STRIPE_OP_BIODRAIN,
  400. STRIPE_OP_RECONSTRUCT,
  401. STRIPE_OP_CHECK,
  402. STRIPE_OP_PARTIAL_PARITY,
  403. };
  404. /*
  405. * RAID parity calculation preferences
  406. */
  407. enum {
  408. PARITY_DISABLE_RMW = 0,
  409. PARITY_ENABLE_RMW,
  410. PARITY_PREFER_RMW,
  411. };
  412. /*
  413. * Pages requested from set_syndrome_sources()
  414. */
  415. enum {
  416. SYNDROME_SRC_ALL,
  417. SYNDROME_SRC_WANT_DRAIN,
  418. SYNDROME_SRC_WRITTEN,
  419. };
  420. /*
  421. * Plugging:
  422. *
  423. * To improve write throughput, we need to delay the handling of some
  424. * stripes until there has been a chance that several write requests
  425. * for the one stripe have all been collected.
  426. * In particular, any write request that would require pre-reading
  427. * is put on a "delayed" queue until there are no stripes currently
  428. * in a pre-read phase. Further, if the "delayed" queue is empty when
  429. * a stripe is put on it then we "plug" the queue and do not process it
  430. * until an unplug call is made. (the unplug_io_fn() is called).
  431. *
  432. * When preread is initiated on a stripe, we set PREREAD_ACTIVE and add
  433. * it to the count of prereading stripes.
  434. * When write is initiated, or the stripe refcnt == 0 (just in case) we
  435. * clear the PREREAD_ACTIVE flag and decrement the count
  436. * Whenever the 'handle' queue is empty and the device is not plugged, we
  437. * move any strips from delayed to handle and clear the DELAYED flag and set
  438. * PREREAD_ACTIVE.
  439. * In stripe_handle, if we find pre-reading is necessary, we do it if
  440. * PREREAD_ACTIVE is set, else we set DELAYED which will send it to the delayed queue.
  441. * HANDLE gets cleared if stripe_handle leaves nothing locked.
  442. */
  443. /* Note: disk_info.rdev can be set to NULL asynchronously by raid5_remove_disk.
  444. * There are three safe ways to access disk_info.rdev.
  445. * 1/ when holding mddev->reconfig_mutex
  446. * 2/ when resync/recovery/reshape is known to be happening - i.e. in code that
  447. * is called as part of performing resync/recovery/reshape.
  448. * 3/ while holding rcu_read_lock(), use rcu_dereference to get the pointer
  449. * and if it is non-NULL, increment rdev->nr_pending before dropping the RCU
  450. * lock.
  451. * When .rdev is set to NULL, the nr_pending count checked again and if
  452. * it has been incremented, the pointer is put back in .rdev.
  453. */
  454. struct disk_info {
  455. struct md_rdev *rdev, *replacement;
  456. struct page *extra_page; /* extra page to use in prexor */
  457. };
  458. /*
  459. * Stripe cache
  460. */
  461. #define NR_STRIPES 256
  462. #if PAGE_SIZE == DEFAULT_STRIPE_SIZE
  463. #define STRIPE_SIZE PAGE_SIZE
  464. #define STRIPE_SHIFT (PAGE_SHIFT - 9)
  465. #define STRIPE_SECTORS (STRIPE_SIZE>>9)
  466. #endif
  467. #define IO_THRESHOLD 1
  468. #define BYPASS_THRESHOLD 1
  469. #define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head))
  470. #define HASH_MASK (NR_HASH - 1)
  471. #define MAX_STRIPE_BATCH 8
  472. /* NOTE NR_STRIPE_HASH_LOCKS must remain below 64.
  473. * This is because we sometimes take all the spinlocks
  474. * and creating that much locking depth can cause
  475. * problems.
  476. */
  477. #define NR_STRIPE_HASH_LOCKS 8
  478. #define STRIPE_HASH_LOCKS_MASK (NR_STRIPE_HASH_LOCKS - 1)
  479. struct r5worker {
  480. struct work_struct work;
  481. struct r5worker_group *group;
  482. struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS];
  483. bool working;
  484. };
  485. struct r5worker_group {
  486. struct list_head handle_list;
  487. struct list_head loprio_list;
  488. struct r5conf *conf;
  489. struct r5worker *workers;
  490. int stripes_cnt;
  491. };
  492. /*
  493. * r5c journal modes of the array: write-back or write-through.
  494. * write-through mode has identical behavior as existing log only
  495. * implementation.
  496. */
  497. enum r5c_journal_mode {
  498. R5C_JOURNAL_MODE_WRITE_THROUGH = 0,
  499. R5C_JOURNAL_MODE_WRITE_BACK = 1,
  500. };
  501. enum r5_cache_state {
  502. R5_INACTIVE_BLOCKED, /* release of inactive stripes blocked,
  503. * waiting for 25% to be free
  504. */
  505. R5_ALLOC_MORE, /* It might help to allocate another
  506. * stripe.
  507. */
  508. R5_DID_ALLOC, /* A stripe was allocated, don't allocate
  509. * more until at least one has been
  510. * released. This avoids flooding
  511. * the cache.
  512. */
  513. R5C_LOG_TIGHT, /* log device space tight, need to
  514. * prioritize stripes at last_checkpoint
  515. */
  516. R5C_LOG_CRITICAL, /* log device is running out of space,
  517. * only process stripes that are already
  518. * occupying the log
  519. */
  520. R5C_EXTRA_PAGE_IN_USE, /* a stripe is using disk_info.extra_page
  521. * for prexor
  522. */
  523. };
  524. #define PENDING_IO_MAX 512
  525. #define PENDING_IO_ONE_FLUSH 128
  526. struct r5pending_data {
  527. struct list_head sibling;
  528. sector_t sector; /* stripe sector */
  529. struct bio_list bios;
  530. };
  531. struct r5conf {
  532. struct hlist_head *stripe_hashtbl;
  533. /* only protect corresponding hash list and inactive_list */
  534. spinlock_t hash_locks[NR_STRIPE_HASH_LOCKS];
  535. struct mddev *mddev;
  536. int chunk_sectors;
  537. int level, algorithm, rmw_level;
  538. int max_degraded;
  539. int raid_disks;
  540. int max_nr_stripes;
  541. int min_nr_stripes;
  542. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  543. unsigned long stripe_size;
  544. unsigned int stripe_shift;
  545. unsigned long stripe_sectors;
  546. #endif
  547. /* reshape_progress is the leading edge of a 'reshape'
  548. * It has value MaxSector when no reshape is happening
  549. * If delta_disks < 0, it is the last sector we started work on,
  550. * else is it the next sector to work on.
  551. */
  552. sector_t reshape_progress;
  553. /* reshape_safe is the trailing edge of a reshape. We know that
  554. * before (or after) this address, all reshape has completed.
  555. */
  556. sector_t reshape_safe;
  557. int previous_raid_disks;
  558. int prev_chunk_sectors;
  559. int prev_algo;
  560. short generation; /* increments with every reshape */
  561. seqcount_spinlock_t gen_lock; /* lock against generation changes */
  562. unsigned long reshape_checkpoint; /* Time we last updated
  563. * metadata */
  564. long long min_offset_diff; /* minimum difference between
  565. * data_offset and
  566. * new_data_offset across all
  567. * devices. May be negative,
  568. * but is closest to zero.
  569. */
  570. struct list_head handle_list; /* stripes needing handling */
  571. struct list_head loprio_list; /* low priority stripes */
  572. struct list_head hold_list; /* preread ready stripes */
  573. struct list_head delayed_list; /* stripes that have plugged requests */
  574. struct list_head bitmap_list; /* stripes delaying awaiting bitmap update */
  575. struct bio *retry_read_aligned; /* currently retrying aligned bios */
  576. unsigned int retry_read_offset; /* sector offset into retry_read_aligned */
  577. struct bio *retry_read_aligned_list; /* aligned bios retry list */
  578. atomic_t preread_active_stripes; /* stripes with scheduled io */
  579. atomic_t active_aligned_reads;
  580. atomic_t pending_full_writes; /* full write backlog */
  581. int bypass_count; /* bypassed prereads */
  582. int bypass_threshold; /* preread nice */
  583. int skip_copy; /* Don't copy data from bio to stripe cache */
  584. struct list_head *last_hold; /* detect hold_list promotions */
  585. atomic_t reshape_stripes; /* stripes with pending writes for reshape */
  586. /* unfortunately we need two cache names as we temporarily have
  587. * two caches.
  588. */
  589. int active_name;
  590. char cache_name[2][32];
  591. struct kmem_cache *slab_cache; /* for allocating stripes */
  592. struct mutex cache_size_mutex; /* Protect changes to cache size */
  593. int seq_flush, seq_write;
  594. int quiesce;
  595. int fullsync; /* set to 1 if a full sync is needed,
  596. * (fresh device added).
  597. * Cleared when a sync completes.
  598. */
  599. int recovery_disabled;
  600. /* per cpu variables */
  601. struct raid5_percpu {
  602. struct page *spare_page; /* Used when checking P/Q in raid6 */
  603. void *scribble; /* space for constructing buffer
  604. * lists and performing address
  605. * conversions
  606. */
  607. int scribble_obj_size;
  608. } __percpu *percpu;
  609. int scribble_disks;
  610. int scribble_sectors;
  611. struct hlist_node node;
  612. /*
  613. * Free stripes pool
  614. */
  615. atomic_t active_stripes;
  616. struct list_head inactive_list[NR_STRIPE_HASH_LOCKS];
  617. atomic_t r5c_cached_full_stripes;
  618. struct list_head r5c_full_stripe_list;
  619. atomic_t r5c_cached_partial_stripes;
  620. struct list_head r5c_partial_stripe_list;
  621. atomic_t r5c_flushing_full_stripes;
  622. atomic_t r5c_flushing_partial_stripes;
  623. atomic_t empty_inactive_list_nr;
  624. struct llist_head released_stripes;
  625. wait_queue_head_t wait_for_quiescent;
  626. wait_queue_head_t wait_for_stripe;
  627. wait_queue_head_t wait_for_overlap;
  628. unsigned long cache_state;
  629. struct shrinker shrinker;
  630. int pool_size; /* number of disks in stripeheads in pool */
  631. spinlock_t device_lock;
  632. struct disk_info *disks;
  633. struct bio_set bio_split;
  634. /* When taking over an array from a different personality, we store
  635. * the new thread here until we fully activate the array.
  636. */
  637. struct md_thread *thread;
  638. struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS];
  639. struct r5worker_group *worker_groups;
  640. int group_cnt;
  641. int worker_cnt_per_group;
  642. struct r5l_log *log;
  643. void *log_private;
  644. spinlock_t pending_bios_lock;
  645. bool batch_bio_dispatch;
  646. struct r5pending_data *pending_data;
  647. struct list_head free_list;
  648. struct list_head pending_list;
  649. int pending_data_cnt;
  650. struct r5pending_data *next_pending_data;
  651. };
  652. #if PAGE_SIZE == DEFAULT_STRIPE_SIZE
  653. #define RAID5_STRIPE_SIZE(conf) STRIPE_SIZE
  654. #define RAID5_STRIPE_SHIFT(conf) STRIPE_SHIFT
  655. #define RAID5_STRIPE_SECTORS(conf) STRIPE_SECTORS
  656. #else
  657. #define RAID5_STRIPE_SIZE(conf) ((conf)->stripe_size)
  658. #define RAID5_STRIPE_SHIFT(conf) ((conf)->stripe_shift)
  659. #define RAID5_STRIPE_SECTORS(conf) ((conf)->stripe_sectors)
  660. #endif
  661. /* bio's attached to a stripe+device for I/O are linked together in bi_sector
  662. * order without overlap. There may be several bio's per stripe+device, and
  663. * a bio could span several devices.
  664. * When walking this list for a particular stripe+device, we must never proceed
  665. * beyond a bio that extends past this device, as the next bio might no longer
  666. * be valid.
  667. * This function is used to determine the 'next' bio in the list, given the
  668. * sector of the current stripe+device
  669. */
  670. static inline struct bio *r5_next_bio(struct r5conf *conf, struct bio *bio, sector_t sector)
  671. {
  672. if (bio_end_sector(bio) < sector + RAID5_STRIPE_SECTORS(conf))
  673. return bio->bi_next;
  674. else
  675. return NULL;
  676. }
  677. /*
  678. * Our supported algorithms
  679. */
  680. #define ALGORITHM_LEFT_ASYMMETRIC 0 /* Rotating Parity N with Data Restart */
  681. #define ALGORITHM_RIGHT_ASYMMETRIC 1 /* Rotating Parity 0 with Data Restart */
  682. #define ALGORITHM_LEFT_SYMMETRIC 2 /* Rotating Parity N with Data Continuation */
  683. #define ALGORITHM_RIGHT_SYMMETRIC 3 /* Rotating Parity 0 with Data Continuation */
  684. /* Define non-rotating (raid4) algorithms. These allow
  685. * conversion of raid4 to raid5.
  686. */
  687. #define ALGORITHM_PARITY_0 4 /* P or P,Q are initial devices */
  688. #define ALGORITHM_PARITY_N 5 /* P or P,Q are final devices. */
  689. /* DDF RAID6 layouts differ from md/raid6 layouts in two ways.
  690. * Firstly, the exact positioning of the parity block is slightly
  691. * different between the 'LEFT_*' modes of md and the "_N_*" modes
  692. * of DDF.
  693. * Secondly, or order of datablocks over which the Q syndrome is computed
  694. * is different.
  695. * Consequently we have different layouts for DDF/raid6 than md/raid6.
  696. * These layouts are from the DDFv1.2 spec.
  697. * Interestingly DDFv1.2-Errata-A does not specify N_CONTINUE but
  698. * leaves RLQ=3 as 'Vendor Specific'
  699. */
  700. #define ALGORITHM_ROTATING_ZERO_RESTART 8 /* DDF PRL=6 RLQ=1 */
  701. #define ALGORITHM_ROTATING_N_RESTART 9 /* DDF PRL=6 RLQ=2 */
  702. #define ALGORITHM_ROTATING_N_CONTINUE 10 /*DDF PRL=6 RLQ=3 */
  703. /* For every RAID5 algorithm we define a RAID6 algorithm
  704. * with exactly the same layout for data and parity, and
  705. * with the Q block always on the last device (N-1).
  706. * This allows trivial conversion from RAID5 to RAID6
  707. */
  708. #define ALGORITHM_LEFT_ASYMMETRIC_6 16
  709. #define ALGORITHM_RIGHT_ASYMMETRIC_6 17
  710. #define ALGORITHM_LEFT_SYMMETRIC_6 18
  711. #define ALGORITHM_RIGHT_SYMMETRIC_6 19
  712. #define ALGORITHM_PARITY_0_6 20
  713. #define ALGORITHM_PARITY_N_6 ALGORITHM_PARITY_N
  714. static inline int algorithm_valid_raid5(int layout)
  715. {
  716. return (layout >= 0) &&
  717. (layout <= 5);
  718. }
  719. static inline int algorithm_valid_raid6(int layout)
  720. {
  721. return (layout >= 0 && layout <= 5)
  722. ||
  723. (layout >= 8 && layout <= 10)
  724. ||
  725. (layout >= 16 && layout <= 20);
  726. }
  727. static inline int algorithm_is_DDF(int layout)
  728. {
  729. return layout >= 8 && layout <= 10;
  730. }
  731. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  732. /*
  733. * Return offset of the corresponding page for r5dev.
  734. */
  735. static inline int raid5_get_page_offset(struct stripe_head *sh, int disk_idx)
  736. {
  737. return (disk_idx % sh->stripes_per_page) * RAID5_STRIPE_SIZE(sh->raid_conf);
  738. }
  739. /*
  740. * Return corresponding page address for r5dev.
  741. */
  742. static inline struct page *
  743. raid5_get_dev_page(struct stripe_head *sh, int disk_idx)
  744. {
  745. return sh->pages[disk_idx / sh->stripes_per_page];
  746. }
  747. #endif
  748. extern void md_raid5_kick_device(struct r5conf *conf);
  749. extern int raid5_set_cache_size(struct mddev *mddev, int size);
  750. extern sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous);
  751. extern void raid5_release_stripe(struct stripe_head *sh);
  752. extern sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
  753. int previous, int *dd_idx,
  754. struct stripe_head *sh);
  755. extern struct stripe_head *
  756. raid5_get_active_stripe(struct r5conf *conf, sector_t sector,
  757. int previous, int noblock, int noquiesce);
  758. extern int raid5_calc_degraded(struct r5conf *conf);
  759. extern int r5c_journal_mode_set(struct mddev *mddev, int journal_mode);
  760. #endif