rcu_segcblist.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * RCU segmented callback lists, function definitions
  4. *
  5. * Copyright IBM Corporation, 2017
  6. *
  7. * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
  8. */
  9. #include <linux/types.h>
  10. #include <linux/kernel.h>
  11. #include <linux/interrupt.h>
  12. #include <linux/rcupdate.h>
  13. #include "rcu_segcblist.h"
  14. /* Initialize simple callback list. */
  15. void rcu_cblist_init(struct rcu_cblist *rclp)
  16. {
  17. rclp->head = NULL;
  18. rclp->tail = &rclp->head;
  19. rclp->len = 0;
  20. }
  21. /*
  22. * Enqueue an rcu_head structure onto the specified callback list.
  23. */
  24. void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp)
  25. {
  26. *rclp->tail = rhp;
  27. rclp->tail = &rhp->next;
  28. WRITE_ONCE(rclp->len, rclp->len + 1);
  29. }
  30. /*
  31. * Flush the second rcu_cblist structure onto the first one, obliterating
  32. * any contents of the first. If rhp is non-NULL, enqueue it as the sole
  33. * element of the second rcu_cblist structure, but ensuring that the second
  34. * rcu_cblist structure, if initially non-empty, always appears non-empty
  35. * throughout the process. If rdp is NULL, the second rcu_cblist structure
  36. * is instead initialized to empty.
  37. */
  38. void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
  39. struct rcu_cblist *srclp,
  40. struct rcu_head *rhp)
  41. {
  42. drclp->head = srclp->head;
  43. if (drclp->head)
  44. drclp->tail = srclp->tail;
  45. else
  46. drclp->tail = &drclp->head;
  47. drclp->len = srclp->len;
  48. if (!rhp) {
  49. rcu_cblist_init(srclp);
  50. } else {
  51. rhp->next = NULL;
  52. srclp->head = rhp;
  53. srclp->tail = &rhp->next;
  54. WRITE_ONCE(srclp->len, 1);
  55. }
  56. }
  57. /*
  58. * Dequeue the oldest rcu_head structure from the specified callback
  59. * list.
  60. */
  61. struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp)
  62. {
  63. struct rcu_head *rhp;
  64. rhp = rclp->head;
  65. if (!rhp)
  66. return NULL;
  67. rclp->len--;
  68. rclp->head = rhp->next;
  69. if (!rclp->head)
  70. rclp->tail = &rclp->head;
  71. return rhp;
  72. }
  73. /* Set the length of an rcu_segcblist structure. */
  74. static void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v)
  75. {
  76. #ifdef CONFIG_RCU_NOCB_CPU
  77. atomic_long_set(&rsclp->len, v);
  78. #else
  79. WRITE_ONCE(rsclp->len, v);
  80. #endif
  81. }
  82. /*
  83. * Increase the numeric length of an rcu_segcblist structure by the
  84. * specified amount, which can be negative. This can cause the ->len
  85. * field to disagree with the actual number of callbacks on the structure.
  86. * This increase is fully ordered with respect to the callers accesses
  87. * both before and after.
  88. */
  89. static void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v)
  90. {
  91. #ifdef CONFIG_RCU_NOCB_CPU
  92. smp_mb__before_atomic(); /* Up to the caller! */
  93. atomic_long_add(v, &rsclp->len);
  94. smp_mb__after_atomic(); /* Up to the caller! */
  95. #else
  96. smp_mb(); /* Up to the caller! */
  97. WRITE_ONCE(rsclp->len, rsclp->len + v);
  98. smp_mb(); /* Up to the caller! */
  99. #endif
  100. }
  101. /*
  102. * Increase the numeric length of an rcu_segcblist structure by one.
  103. * This can cause the ->len field to disagree with the actual number of
  104. * callbacks on the structure. This increase is fully ordered with respect
  105. * to the callers accesses both before and after.
  106. */
  107. void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp)
  108. {
  109. rcu_segcblist_add_len(rsclp, 1);
  110. }
  111. /*
  112. * Exchange the numeric length of the specified rcu_segcblist structure
  113. * with the specified value. This can cause the ->len field to disagree
  114. * with the actual number of callbacks on the structure. This exchange is
  115. * fully ordered with respect to the callers accesses both before and after.
  116. */
  117. static long rcu_segcblist_xchg_len(struct rcu_segcblist *rsclp, long v)
  118. {
  119. #ifdef CONFIG_RCU_NOCB_CPU
  120. return atomic_long_xchg(&rsclp->len, v);
  121. #else
  122. long ret = rsclp->len;
  123. smp_mb(); /* Up to the caller! */
  124. WRITE_ONCE(rsclp->len, v);
  125. smp_mb(); /* Up to the caller! */
  126. return ret;
  127. #endif
  128. }
  129. /*
  130. * Initialize an rcu_segcblist structure.
  131. */
  132. void rcu_segcblist_init(struct rcu_segcblist *rsclp)
  133. {
  134. int i;
  135. BUILD_BUG_ON(RCU_NEXT_TAIL + 1 != ARRAY_SIZE(rsclp->gp_seq));
  136. BUILD_BUG_ON(ARRAY_SIZE(rsclp->tails) != ARRAY_SIZE(rsclp->gp_seq));
  137. rsclp->head = NULL;
  138. for (i = 0; i < RCU_CBLIST_NSEGS; i++)
  139. rsclp->tails[i] = &rsclp->head;
  140. rcu_segcblist_set_len(rsclp, 0);
  141. rsclp->enabled = 1;
  142. }
  143. /*
  144. * Disable the specified rcu_segcblist structure, so that callbacks can
  145. * no longer be posted to it. This structure must be empty.
  146. */
  147. void rcu_segcblist_disable(struct rcu_segcblist *rsclp)
  148. {
  149. WARN_ON_ONCE(!rcu_segcblist_empty(rsclp));
  150. WARN_ON_ONCE(rcu_segcblist_n_cbs(rsclp));
  151. rsclp->enabled = 0;
  152. }
  153. /*
  154. * Mark the specified rcu_segcblist structure as offloaded. This
  155. * structure must be empty.
  156. */
  157. void rcu_segcblist_offload(struct rcu_segcblist *rsclp)
  158. {
  159. rsclp->offloaded = 1;
  160. }
  161. /*
  162. * Does the specified rcu_segcblist structure contain callbacks that
  163. * are ready to be invoked?
  164. */
  165. bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp)
  166. {
  167. return rcu_segcblist_is_enabled(rsclp) &&
  168. &rsclp->head != READ_ONCE(rsclp->tails[RCU_DONE_TAIL]);
  169. }
  170. /*
  171. * Does the specified rcu_segcblist structure contain callbacks that
  172. * are still pending, that is, not yet ready to be invoked?
  173. */
  174. bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp)
  175. {
  176. return rcu_segcblist_is_enabled(rsclp) &&
  177. !rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL);
  178. }
  179. /*
  180. * Return a pointer to the first callback in the specified rcu_segcblist
  181. * structure. This is useful for diagnostics.
  182. */
  183. struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp)
  184. {
  185. if (rcu_segcblist_is_enabled(rsclp))
  186. return rsclp->head;
  187. return NULL;
  188. }
  189. /*
  190. * Return a pointer to the first pending callback in the specified
  191. * rcu_segcblist structure. This is useful just after posting a given
  192. * callback -- if that callback is the first pending callback, then
  193. * you cannot rely on someone else having already started up the required
  194. * grace period.
  195. */
  196. struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp)
  197. {
  198. if (rcu_segcblist_is_enabled(rsclp))
  199. return *rsclp->tails[RCU_DONE_TAIL];
  200. return NULL;
  201. }
  202. /*
  203. * Return false if there are no CBs awaiting grace periods, otherwise,
  204. * return true and store the nearest waited-upon grace period into *lp.
  205. */
  206. bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp)
  207. {
  208. if (!rcu_segcblist_pend_cbs(rsclp))
  209. return false;
  210. *lp = rsclp->gp_seq[RCU_WAIT_TAIL];
  211. return true;
  212. }
  213. /*
  214. * Enqueue the specified callback onto the specified rcu_segcblist
  215. * structure, updating accounting as needed. Note that the ->len
  216. * field may be accessed locklessly, hence the WRITE_ONCE().
  217. * The ->len field is used by rcu_barrier() and friends to determine
  218. * if it must post a callback on this structure, and it is OK
  219. * for rcu_barrier() to sometimes post callbacks needlessly, but
  220. * absolutely not OK for it to ever miss posting a callback.
  221. */
  222. void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
  223. struct rcu_head *rhp)
  224. {
  225. rcu_segcblist_inc_len(rsclp);
  226. smp_mb(); /* Ensure counts are updated before callback is enqueued. */
  227. rhp->next = NULL;
  228. WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rhp);
  229. WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], &rhp->next);
  230. }
  231. /*
  232. * Entrain the specified callback onto the specified rcu_segcblist at
  233. * the end of the last non-empty segment. If the entire rcu_segcblist
  234. * is empty, make no change, but return false.
  235. *
  236. * This is intended for use by rcu_barrier()-like primitives, -not-
  237. * for normal grace-period use. IMPORTANT: The callback you enqueue
  238. * will wait for all prior callbacks, NOT necessarily for a grace
  239. * period. You have been warned.
  240. */
  241. bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
  242. struct rcu_head *rhp)
  243. {
  244. int i;
  245. if (rcu_segcblist_n_cbs(rsclp) == 0)
  246. return false;
  247. rcu_segcblist_inc_len(rsclp);
  248. smp_mb(); /* Ensure counts are updated before callback is entrained. */
  249. rhp->next = NULL;
  250. for (i = RCU_NEXT_TAIL; i > RCU_DONE_TAIL; i--)
  251. if (rsclp->tails[i] != rsclp->tails[i - 1])
  252. break;
  253. WRITE_ONCE(*rsclp->tails[i], rhp);
  254. for (; i <= RCU_NEXT_TAIL; i++)
  255. WRITE_ONCE(rsclp->tails[i], &rhp->next);
  256. return true;
  257. }
  258. /*
  259. * Extract only the counts from the specified rcu_segcblist structure,
  260. * and place them in the specified rcu_cblist structure. This function
  261. * supports both callback orphaning and invocation, hence the separation
  262. * of counts and callbacks. (Callbacks ready for invocation must be
  263. * orphaned and adopted separately from pending callbacks, but counts
  264. * apply to all callbacks. Locking must be used to make sure that
  265. * both orphaned-callbacks lists are consistent.)
  266. */
  267. void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
  268. struct rcu_cblist *rclp)
  269. {
  270. rclp->len = rcu_segcblist_xchg_len(rsclp, 0);
  271. }
  272. /*
  273. * Extract only those callbacks ready to be invoked from the specified
  274. * rcu_segcblist structure and place them in the specified rcu_cblist
  275. * structure.
  276. */
  277. void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
  278. struct rcu_cblist *rclp)
  279. {
  280. int i;
  281. if (!rcu_segcblist_ready_cbs(rsclp))
  282. return; /* Nothing to do. */
  283. *rclp->tail = rsclp->head;
  284. WRITE_ONCE(rsclp->head, *rsclp->tails[RCU_DONE_TAIL]);
  285. WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);
  286. rclp->tail = rsclp->tails[RCU_DONE_TAIL];
  287. for (i = RCU_CBLIST_NSEGS - 1; i >= RCU_DONE_TAIL; i--)
  288. if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL])
  289. WRITE_ONCE(rsclp->tails[i], &rsclp->head);
  290. }
  291. /*
  292. * Extract only those callbacks still pending (not yet ready to be
  293. * invoked) from the specified rcu_segcblist structure and place them in
  294. * the specified rcu_cblist structure. Note that this loses information
  295. * about any callbacks that might have been partway done waiting for
  296. * their grace period. Too bad! They will have to start over.
  297. */
  298. void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
  299. struct rcu_cblist *rclp)
  300. {
  301. int i;
  302. if (!rcu_segcblist_pend_cbs(rsclp))
  303. return; /* Nothing to do. */
  304. *rclp->tail = *rsclp->tails[RCU_DONE_TAIL];
  305. rclp->tail = rsclp->tails[RCU_NEXT_TAIL];
  306. WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);
  307. for (i = RCU_DONE_TAIL + 1; i < RCU_CBLIST_NSEGS; i++)
  308. WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_DONE_TAIL]);
  309. }
  310. /*
  311. * Insert counts from the specified rcu_cblist structure in the
  312. * specified rcu_segcblist structure.
  313. */
  314. void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
  315. struct rcu_cblist *rclp)
  316. {
  317. rcu_segcblist_add_len(rsclp, rclp->len);
  318. rclp->len = 0;
  319. }
  320. /*
  321. * Move callbacks from the specified rcu_cblist to the beginning of the
  322. * done-callbacks segment of the specified rcu_segcblist.
  323. */
  324. void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
  325. struct rcu_cblist *rclp)
  326. {
  327. int i;
  328. if (!rclp->head)
  329. return; /* No callbacks to move. */
  330. *rclp->tail = rsclp->head;
  331. WRITE_ONCE(rsclp->head, rclp->head);
  332. for (i = RCU_DONE_TAIL; i < RCU_CBLIST_NSEGS; i++)
  333. if (&rsclp->head == rsclp->tails[i])
  334. WRITE_ONCE(rsclp->tails[i], rclp->tail);
  335. else
  336. break;
  337. rclp->head = NULL;
  338. rclp->tail = &rclp->head;
  339. }
  340. /*
  341. * Move callbacks from the specified rcu_cblist to the end of the
  342. * new-callbacks segment of the specified rcu_segcblist.
  343. */
  344. void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
  345. struct rcu_cblist *rclp)
  346. {
  347. if (!rclp->head)
  348. return; /* Nothing to do. */
  349. WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head);
  350. WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail);
  351. }
  352. /*
  353. * Advance the callbacks in the specified rcu_segcblist structure based
  354. * on the current value passed in for the grace-period counter.
  355. */
  356. void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)
  357. {
  358. int i, j;
  359. WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp));
  360. if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL))
  361. return;
  362. /*
  363. * Find all callbacks whose ->gp_seq numbers indicate that they
  364. * are ready to invoke, and put them into the RCU_DONE_TAIL segment.
  365. */
  366. for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) {
  367. if (ULONG_CMP_LT(seq, rsclp->gp_seq[i]))
  368. break;
  369. WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]);
  370. }
  371. /* If no callbacks moved, nothing more need be done. */
  372. if (i == RCU_WAIT_TAIL)
  373. return;
  374. /* Clean up tail pointers that might have been misordered above. */
  375. for (j = RCU_WAIT_TAIL; j < i; j++)
  376. WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]);
  377. /*
  378. * Callbacks moved, so clean up the misordered ->tails[] pointers
  379. * that now point into the middle of the list of ready-to-invoke
  380. * callbacks. The overall effect is to copy down the later pointers
  381. * into the gap that was created by the now-ready segments.
  382. */
  383. for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) {
  384. if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL])
  385. break; /* No more callbacks. */
  386. WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]);
  387. rsclp->gp_seq[j] = rsclp->gp_seq[i];
  388. }
  389. }
  390. /*
  391. * "Accelerate" callbacks based on more-accurate grace-period information.
  392. * The reason for this is that RCU does not synchronize the beginnings and
  393. * ends of grace periods, and that callbacks are posted locally. This in
  394. * turn means that the callbacks must be labelled conservatively early
  395. * on, as getting exact information would degrade both performance and
  396. * scalability. When more accurate grace-period information becomes
  397. * available, previously posted callbacks can be "accelerated", marking
  398. * them to complete at the end of the earlier grace period.
  399. *
  400. * This function operates on an rcu_segcblist structure, and also the
  401. * grace-period sequence number seq at which new callbacks would become
  402. * ready to invoke. Returns true if there are callbacks that won't be
  403. * ready to invoke until seq, false otherwise.
  404. */
  405. bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq)
  406. {
  407. int i;
  408. WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp));
  409. if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL))
  410. return false;
  411. /*
  412. * Find the segment preceding the oldest segment of callbacks
  413. * whose ->gp_seq[] completion is at or after that passed in via
  414. * "seq", skipping any empty segments. This oldest segment, along
  415. * with any later segments, can be merged in with any newly arrived
  416. * callbacks in the RCU_NEXT_TAIL segment, and assigned "seq"
  417. * as their ->gp_seq[] grace-period completion sequence number.
  418. */
  419. for (i = RCU_NEXT_READY_TAIL; i > RCU_DONE_TAIL; i--)
  420. if (rsclp->tails[i] != rsclp->tails[i - 1] &&
  421. ULONG_CMP_LT(rsclp->gp_seq[i], seq))
  422. break;
  423. /*
  424. * If all the segments contain callbacks that correspond to
  425. * earlier grace-period sequence numbers than "seq", leave.
  426. * Assuming that the rcu_segcblist structure has enough
  427. * segments in its arrays, this can only happen if some of
  428. * the non-done segments contain callbacks that really are
  429. * ready to invoke. This situation will get straightened
  430. * out by the next call to rcu_segcblist_advance().
  431. *
  432. * Also advance to the oldest segment of callbacks whose
  433. * ->gp_seq[] completion is at or after that passed in via "seq",
  434. * skipping any empty segments.
  435. *
  436. * Note that segment "i" (and any lower-numbered segments
  437. * containing older callbacks) will be unaffected, and their
  438. * grace-period numbers remain unchanged. For example, if i ==
  439. * WAIT_TAIL, then neither WAIT_TAIL nor DONE_TAIL will be touched.
  440. * Instead, the CBs in NEXT_TAIL will be merged with those in
  441. * NEXT_READY_TAIL and the grace-period number of NEXT_READY_TAIL
  442. * would be updated. NEXT_TAIL would then be empty.
  443. */
  444. if (rcu_segcblist_restempty(rsclp, i) || ++i >= RCU_NEXT_TAIL)
  445. return false;
  446. /*
  447. * Merge all later callbacks, including newly arrived callbacks,
  448. * into the segment located by the for-loop above. Assign "seq"
  449. * as the ->gp_seq[] value in order to correctly handle the case
  450. * where there were no pending callbacks in the rcu_segcblist
  451. * structure other than in the RCU_NEXT_TAIL segment.
  452. */
  453. for (; i < RCU_NEXT_TAIL; i++) {
  454. WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]);
  455. rsclp->gp_seq[i] = seq;
  456. }
  457. return true;
  458. }
  459. /*
  460. * Merge the source rcu_segcblist structure into the destination
  461. * rcu_segcblist structure, then initialize the source. Any pending
  462. * callbacks from the source get to start over. It is best to
  463. * advance and accelerate both the destination and the source
  464. * before merging.
  465. */
  466. void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
  467. struct rcu_segcblist *src_rsclp)
  468. {
  469. struct rcu_cblist donecbs;
  470. struct rcu_cblist pendcbs;
  471. rcu_cblist_init(&donecbs);
  472. rcu_cblist_init(&pendcbs);
  473. rcu_segcblist_extract_count(src_rsclp, &donecbs);
  474. rcu_segcblist_extract_done_cbs(src_rsclp, &donecbs);
  475. rcu_segcblist_extract_pend_cbs(src_rsclp, &pendcbs);
  476. rcu_segcblist_insert_count(dst_rsclp, &donecbs);
  477. rcu_segcblist_insert_done_cbs(dst_rsclp, &donecbs);
  478. rcu_segcblist_insert_pend_cbs(dst_rsclp, &pendcbs);
  479. rcu_segcblist_init(src_rsclp);
  480. }