rcupdate.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635
  1. /*
  2. * Read-Copy Update mechanism for mutual exclusion
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License as published by
  6. * the Free Software Foundation; either version 2 of the License, or
  7. * (at your option) any later version.
  8. *
  9. * This program is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write to the Free Software
  16. * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17. *
  18. * Copyright (C) IBM Corporation, 2001
  19. *
  20. * Authors: Dipankar Sarma <dipankar@in.ibm.com>
  21. * Manfred Spraul <manfred@colorfullife.com>
  22. *
  23. * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
  24. * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
  25. * Papers:
  26. * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
  27. * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
  28. *
  29. * For detailed explanation of Read-Copy Update mechanism see -
  30. * http://lse.sourceforge.net/locking/rcupdate.html
  31. *
  32. */
  33. #include <linux/types.h>
  34. #include <linux/kernel.h>
  35. #include <linux/init.h>
  36. #include <linux/spinlock.h>
  37. #include <linux/smp.h>
  38. #include <linux/rcupdate.h>
  39. #include <linux/interrupt.h>
  40. #include <linux/sched.h>
  41. #include <asm/atomic.h>
  42. #include <linux/bitops.h>
  43. #include <linux/module.h>
  44. #include <linux/completion.h>
  45. #include <linux/moduleparam.h>
  46. #include <linux/percpu.h>
  47. #include <linux/notifier.h>
  48. #include <linux/rcupdate.h>
  49. #include <linux/cpu.h>
  50. #include <linux/mutex.h>
  51. /* Definition for rcupdate control block. */
  52. static struct rcu_ctrlblk rcu_ctrlblk = {
  53. .cur = -300,
  54. .completed = -300,
  55. .lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock),
  56. .cpumask = CPU_MASK_NONE,
  57. };
  58. static struct rcu_ctrlblk rcu_bh_ctrlblk = {
  59. .cur = -300,
  60. .completed = -300,
  61. .lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock),
  62. .cpumask = CPU_MASK_NONE,
  63. };
  64. DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L };
  65. DEFINE_PER_CPU(struct rcu_data, rcu_bh_data) = { 0L };
  66. /* Fake initialization required by compiler */
  67. static DEFINE_PER_CPU(struct tasklet_struct, rcu_tasklet) = {NULL};
  68. static int blimit = 10;
  69. static int qhimark = 10000;
  70. static int qlowmark = 100;
  71. static atomic_t rcu_barrier_cpu_count;
  72. static DEFINE_MUTEX(rcu_barrier_mutex);
  73. static struct completion rcu_barrier_completion;
  74. #ifdef CONFIG_SMP
  75. static void force_quiescent_state(struct rcu_data *rdp,
  76. struct rcu_ctrlblk *rcp)
  77. {
  78. int cpu;
  79. cpumask_t cpumask;
  80. set_need_resched();
  81. if (unlikely(!rcp->signaled)) {
  82. rcp->signaled = 1;
  83. /*
  84. * Don't send IPI to itself. With irqs disabled,
  85. * rdp->cpu is the current cpu.
  86. */
  87. cpumask = rcp->cpumask;
  88. cpu_clear(rdp->cpu, cpumask);
  89. for_each_cpu_mask(cpu, cpumask)
  90. smp_send_reschedule(cpu);
  91. }
  92. }
  93. #else
  94. static inline void force_quiescent_state(struct rcu_data *rdp,
  95. struct rcu_ctrlblk *rcp)
  96. {
  97. set_need_resched();
  98. }
  99. #endif
  100. /**
  101. * call_rcu - Queue an RCU callback for invocation after a grace period.
  102. * @head: structure to be used for queueing the RCU updates.
  103. * @func: actual update function to be invoked after the grace period
  104. *
  105. * The update function will be invoked some time after a full grace
  106. * period elapses, in other words after all currently executing RCU
  107. * read-side critical sections have completed. RCU read-side critical
  108. * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
  109. * and may be nested.
  110. */
  111. void fastcall call_rcu(struct rcu_head *head,
  112. void (*func)(struct rcu_head *rcu))
  113. {
  114. unsigned long flags;
  115. struct rcu_data *rdp;
  116. head->func = func;
  117. head->next = NULL;
  118. local_irq_save(flags);
  119. rdp = &__get_cpu_var(rcu_data);
  120. *rdp->nxttail = head;
  121. rdp->nxttail = &head->next;
  122. if (unlikely(++rdp->qlen > qhimark)) {
  123. rdp->blimit = INT_MAX;
  124. force_quiescent_state(rdp, &rcu_ctrlblk);
  125. }
  126. local_irq_restore(flags);
  127. }
  128. /**
  129. * call_rcu_bh - Queue an RCU for invocation after a quicker grace period.
  130. * @head: structure to be used for queueing the RCU updates.
  131. * @func: actual update function to be invoked after the grace period
  132. *
  133. * The update function will be invoked some time after a full grace
  134. * period elapses, in other words after all currently executing RCU
  135. * read-side critical sections have completed. call_rcu_bh() assumes
  136. * that the read-side critical sections end on completion of a softirq
  137. * handler. This means that read-side critical sections in process
  138. * context must not be interrupted by softirqs. This interface is to be
  139. * used when most of the read-side critical sections are in softirq context.
  140. * RCU read-side critical sections are delimited by rcu_read_lock() and
  141. * rcu_read_unlock(), * if in interrupt context or rcu_read_lock_bh()
  142. * and rcu_read_unlock_bh(), if in process context. These may be nested.
  143. */
  144. void fastcall call_rcu_bh(struct rcu_head *head,
  145. void (*func)(struct rcu_head *rcu))
  146. {
  147. unsigned long flags;
  148. struct rcu_data *rdp;
  149. head->func = func;
  150. head->next = NULL;
  151. local_irq_save(flags);
  152. rdp = &__get_cpu_var(rcu_bh_data);
  153. *rdp->nxttail = head;
  154. rdp->nxttail = &head->next;
  155. if (unlikely(++rdp->qlen > qhimark)) {
  156. rdp->blimit = INT_MAX;
  157. force_quiescent_state(rdp, &rcu_bh_ctrlblk);
  158. }
  159. local_irq_restore(flags);
  160. }
  161. /*
  162. * Return the number of RCU batches processed thus far. Useful
  163. * for debug and statistics.
  164. */
  165. long rcu_batches_completed(void)
  166. {
  167. return rcu_ctrlblk.completed;
  168. }
  169. /*
  170. * Return the number of RCU batches processed thus far. Useful
  171. * for debug and statistics.
  172. */
  173. long rcu_batches_completed_bh(void)
  174. {
  175. return rcu_bh_ctrlblk.completed;
  176. }
  177. static void rcu_barrier_callback(struct rcu_head *notused)
  178. {
  179. if (atomic_dec_and_test(&rcu_barrier_cpu_count))
  180. complete(&rcu_barrier_completion);
  181. }
  182. /*
  183. * Called with preemption disabled, and from cross-cpu IRQ context.
  184. */
  185. static void rcu_barrier_func(void *notused)
  186. {
  187. int cpu = smp_processor_id();
  188. struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
  189. struct rcu_head *head;
  190. head = &rdp->barrier;
  191. atomic_inc(&rcu_barrier_cpu_count);
  192. call_rcu(head, rcu_barrier_callback);
  193. }
  194. /**
  195. * rcu_barrier - Wait until all the in-flight RCUs are complete.
  196. */
  197. void rcu_barrier(void)
  198. {
  199. BUG_ON(in_interrupt());
  200. /* Take cpucontrol mutex to protect against CPU hotplug */
  201. mutex_lock(&rcu_barrier_mutex);
  202. init_completion(&rcu_barrier_completion);
  203. atomic_set(&rcu_barrier_cpu_count, 0);
  204. on_each_cpu(rcu_barrier_func, NULL, 0, 1);
  205. wait_for_completion(&rcu_barrier_completion);
  206. mutex_unlock(&rcu_barrier_mutex);
  207. }
  208. EXPORT_SYMBOL_GPL(rcu_barrier);
  209. /*
  210. * Invoke the completed RCU callbacks. They are expected to be in
  211. * a per-cpu list.
  212. */
  213. static void rcu_do_batch(struct rcu_data *rdp)
  214. {
  215. struct rcu_head *next, *list;
  216. int count = 0;
  217. list = rdp->donelist;
  218. while (list) {
  219. next = list->next;
  220. prefetch(next);
  221. list->func(list);
  222. list = next;
  223. if (++count >= rdp->blimit)
  224. break;
  225. }
  226. rdp->donelist = list;
  227. local_irq_disable();
  228. rdp->qlen -= count;
  229. local_irq_enable();
  230. if (rdp->blimit == INT_MAX && rdp->qlen <= qlowmark)
  231. rdp->blimit = blimit;
  232. if (!rdp->donelist)
  233. rdp->donetail = &rdp->donelist;
  234. else
  235. tasklet_schedule(&per_cpu(rcu_tasklet, rdp->cpu));
  236. }
  237. /*
  238. * Grace period handling:
  239. * The grace period handling consists out of two steps:
  240. * - A new grace period is started.
  241. * This is done by rcu_start_batch. The start is not broadcasted to
  242. * all cpus, they must pick this up by comparing rcp->cur with
  243. * rdp->quiescbatch. All cpus are recorded in the
  244. * rcu_ctrlblk.cpumask bitmap.
  245. * - All cpus must go through a quiescent state.
  246. * Since the start of the grace period is not broadcasted, at least two
  247. * calls to rcu_check_quiescent_state are required:
  248. * The first call just notices that a new grace period is running. The
  249. * following calls check if there was a quiescent state since the beginning
  250. * of the grace period. If so, it updates rcu_ctrlblk.cpumask. If
  251. * the bitmap is empty, then the grace period is completed.
  252. * rcu_check_quiescent_state calls rcu_start_batch(0) to start the next grace
  253. * period (if necessary).
  254. */
  255. /*
  256. * Register a new batch of callbacks, and start it up if there is currently no
  257. * active batch and the batch to be registered has not already occurred.
  258. * Caller must hold rcu_ctrlblk.lock.
  259. */
  260. static void rcu_start_batch(struct rcu_ctrlblk *rcp)
  261. {
  262. if (rcp->next_pending &&
  263. rcp->completed == rcp->cur) {
  264. rcp->next_pending = 0;
  265. /*
  266. * next_pending == 0 must be visible in
  267. * __rcu_process_callbacks() before it can see new value of cur.
  268. */
  269. smp_wmb();
  270. rcp->cur++;
  271. /*
  272. * Accessing nohz_cpu_mask before incrementing rcp->cur needs a
  273. * Barrier Otherwise it can cause tickless idle CPUs to be
  274. * included in rcp->cpumask, which will extend graceperiods
  275. * unnecessarily.
  276. */
  277. smp_mb();
  278. cpus_andnot(rcp->cpumask, cpu_online_map, nohz_cpu_mask);
  279. rcp->signaled = 0;
  280. }
  281. }
  282. /*
  283. * cpu went through a quiescent state since the beginning of the grace period.
  284. * Clear it from the cpu mask and complete the grace period if it was the last
  285. * cpu. Start another grace period if someone has further entries pending
  286. */
  287. static void cpu_quiet(int cpu, struct rcu_ctrlblk *rcp)
  288. {
  289. cpu_clear(cpu, rcp->cpumask);
  290. if (cpus_empty(rcp->cpumask)) {
  291. /* batch completed ! */
  292. rcp->completed = rcp->cur;
  293. rcu_start_batch(rcp);
  294. }
  295. }
  296. /*
  297. * Check if the cpu has gone through a quiescent state (say context
  298. * switch). If so and if it already hasn't done so in this RCU
  299. * quiescent cycle, then indicate that it has done so.
  300. */
  301. static void rcu_check_quiescent_state(struct rcu_ctrlblk *rcp,
  302. struct rcu_data *rdp)
  303. {
  304. if (rdp->quiescbatch != rcp->cur) {
  305. /* start new grace period: */
  306. rdp->qs_pending = 1;
  307. rdp->passed_quiesc = 0;
  308. rdp->quiescbatch = rcp->cur;
  309. return;
  310. }
  311. /* Grace period already completed for this cpu?
  312. * qs_pending is checked instead of the actual bitmap to avoid
  313. * cacheline trashing.
  314. */
  315. if (!rdp->qs_pending)
  316. return;
  317. /*
  318. * Was there a quiescent state since the beginning of the grace
  319. * period? If no, then exit and wait for the next call.
  320. */
  321. if (!rdp->passed_quiesc)
  322. return;
  323. rdp->qs_pending = 0;
  324. spin_lock(&rcp->lock);
  325. /*
  326. * rdp->quiescbatch/rcp->cur and the cpu bitmap can come out of sync
  327. * during cpu startup. Ignore the quiescent state.
  328. */
  329. if (likely(rdp->quiescbatch == rcp->cur))
  330. cpu_quiet(rdp->cpu, rcp);
  331. spin_unlock(&rcp->lock);
  332. }
  333. #ifdef CONFIG_HOTPLUG_CPU
  334. /* warning! helper for rcu_offline_cpu. do not use elsewhere without reviewing
  335. * locking requirements, the list it's pulling from has to belong to a cpu
  336. * which is dead and hence not processing interrupts.
  337. */
  338. static void rcu_move_batch(struct rcu_data *this_rdp, struct rcu_head *list,
  339. struct rcu_head **tail)
  340. {
  341. local_irq_disable();
  342. *this_rdp->nxttail = list;
  343. if (list)
  344. this_rdp->nxttail = tail;
  345. local_irq_enable();
  346. }
  347. static void __rcu_offline_cpu(struct rcu_data *this_rdp,
  348. struct rcu_ctrlblk *rcp, struct rcu_data *rdp)
  349. {
  350. /* if the cpu going offline owns the grace period
  351. * we can block indefinitely waiting for it, so flush
  352. * it here
  353. */
  354. spin_lock_bh(&rcp->lock);
  355. if (rcp->cur != rcp->completed)
  356. cpu_quiet(rdp->cpu, rcp);
  357. spin_unlock_bh(&rcp->lock);
  358. rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail);
  359. rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail);
  360. rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail);
  361. }
  362. static void rcu_offline_cpu(int cpu)
  363. {
  364. struct rcu_data *this_rdp = &get_cpu_var(rcu_data);
  365. struct rcu_data *this_bh_rdp = &get_cpu_var(rcu_bh_data);
  366. __rcu_offline_cpu(this_rdp, &rcu_ctrlblk,
  367. &per_cpu(rcu_data, cpu));
  368. __rcu_offline_cpu(this_bh_rdp, &rcu_bh_ctrlblk,
  369. &per_cpu(rcu_bh_data, cpu));
  370. put_cpu_var(rcu_data);
  371. put_cpu_var(rcu_bh_data);
  372. tasklet_kill_immediate(&per_cpu(rcu_tasklet, cpu), cpu);
  373. }
  374. #else
  375. static void rcu_offline_cpu(int cpu)
  376. {
  377. }
  378. #endif
  379. /*
  380. * This does the RCU processing work from tasklet context.
  381. */
  382. static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp,
  383. struct rcu_data *rdp)
  384. {
  385. if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) {
  386. *rdp->donetail = rdp->curlist;
  387. rdp->donetail = rdp->curtail;
  388. rdp->curlist = NULL;
  389. rdp->curtail = &rdp->curlist;
  390. }
  391. if (rdp->nxtlist && !rdp->curlist) {
  392. local_irq_disable();
  393. rdp->curlist = rdp->nxtlist;
  394. rdp->curtail = rdp->nxttail;
  395. rdp->nxtlist = NULL;
  396. rdp->nxttail = &rdp->nxtlist;
  397. local_irq_enable();
  398. /*
  399. * start the next batch of callbacks
  400. */
  401. /* determine batch number */
  402. rdp->batch = rcp->cur + 1;
  403. /* see the comment and corresponding wmb() in
  404. * the rcu_start_batch()
  405. */
  406. smp_rmb();
  407. if (!rcp->next_pending) {
  408. /* and start it/schedule start if it's a new batch */
  409. spin_lock(&rcp->lock);
  410. rcp->next_pending = 1;
  411. rcu_start_batch(rcp);
  412. spin_unlock(&rcp->lock);
  413. }
  414. }
  415. rcu_check_quiescent_state(rcp, rdp);
  416. if (rdp->donelist)
  417. rcu_do_batch(rdp);
  418. }
  419. static void rcu_process_callbacks(unsigned long unused)
  420. {
  421. __rcu_process_callbacks(&rcu_ctrlblk, &__get_cpu_var(rcu_data));
  422. __rcu_process_callbacks(&rcu_bh_ctrlblk, &__get_cpu_var(rcu_bh_data));
  423. }
  424. static int __rcu_pending(struct rcu_ctrlblk *rcp, struct rcu_data *rdp)
  425. {
  426. /* This cpu has pending rcu entries and the grace period
  427. * for them has completed.
  428. */
  429. if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch))
  430. return 1;
  431. /* This cpu has no pending entries, but there are new entries */
  432. if (!rdp->curlist && rdp->nxtlist)
  433. return 1;
  434. /* This cpu has finished callbacks to invoke */
  435. if (rdp->donelist)
  436. return 1;
  437. /* The rcu core waits for a quiescent state from the cpu */
  438. if (rdp->quiescbatch != rcp->cur || rdp->qs_pending)
  439. return 1;
  440. /* nothing to do */
  441. return 0;
  442. }
  443. /*
  444. * Check to see if there is any immediate RCU-related work to be done
  445. * by the current CPU, returning 1 if so. This function is part of the
  446. * RCU implementation; it is -not- an exported member of the RCU API.
  447. */
  448. int rcu_pending(int cpu)
  449. {
  450. return __rcu_pending(&rcu_ctrlblk, &per_cpu(rcu_data, cpu)) ||
  451. __rcu_pending(&rcu_bh_ctrlblk, &per_cpu(rcu_bh_data, cpu));
  452. }
  453. /*
  454. * Check to see if any future RCU-related work will need to be done
  455. * by the current CPU, even if none need be done immediately, returning
  456. * 1 if so. This function is part of the RCU implementation; it is -not-
  457. * an exported member of the RCU API.
  458. */
  459. int rcu_needs_cpu(int cpu)
  460. {
  461. struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
  462. struct rcu_data *rdp_bh = &per_cpu(rcu_bh_data, cpu);
  463. return (!!rdp->curlist || !!rdp_bh->curlist || rcu_pending(cpu));
  464. }
  465. void rcu_check_callbacks(int cpu, int user)
  466. {
  467. if (user ||
  468. (idle_cpu(cpu) && !in_softirq() &&
  469. hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
  470. rcu_qsctr_inc(cpu);
  471. rcu_bh_qsctr_inc(cpu);
  472. } else if (!in_softirq())
  473. rcu_bh_qsctr_inc(cpu);
  474. tasklet_schedule(&per_cpu(rcu_tasklet, cpu));
  475. }
  476. static void rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp,
  477. struct rcu_data *rdp)
  478. {
  479. memset(rdp, 0, sizeof(*rdp));
  480. rdp->curtail = &rdp->curlist;
  481. rdp->nxttail = &rdp->nxtlist;
  482. rdp->donetail = &rdp->donelist;
  483. rdp->quiescbatch = rcp->completed;
  484. rdp->qs_pending = 0;
  485. rdp->cpu = cpu;
  486. rdp->blimit = blimit;
  487. }
  488. static void __devinit rcu_online_cpu(int cpu)
  489. {
  490. struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
  491. struct rcu_data *bh_rdp = &per_cpu(rcu_bh_data, cpu);
  492. rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp);
  493. rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp);
  494. tasklet_init(&per_cpu(rcu_tasklet, cpu), rcu_process_callbacks, 0UL);
  495. }
  496. static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
  497. unsigned long action, void *hcpu)
  498. {
  499. long cpu = (long)hcpu;
  500. switch (action) {
  501. case CPU_UP_PREPARE:
  502. rcu_online_cpu(cpu);
  503. break;
  504. case CPU_DEAD:
  505. rcu_offline_cpu(cpu);
  506. break;
  507. default:
  508. break;
  509. }
  510. return NOTIFY_OK;
  511. }
  512. static struct notifier_block __cpuinitdata rcu_nb = {
  513. .notifier_call = rcu_cpu_notify,
  514. };
  515. /*
  516. * Initializes rcu mechanism. Assumed to be called early.
  517. * That is before local timer(SMP) or jiffie timer (uniproc) is setup.
  518. * Note that rcu_qsctr and friends are implicitly
  519. * initialized due to the choice of ``0'' for RCU_CTR_INVALID.
  520. */
  521. void __init rcu_init(void)
  522. {
  523. rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE,
  524. (void *)(long)smp_processor_id());
  525. /* Register notifier for non-boot CPUs */
  526. register_cpu_notifier(&rcu_nb);
  527. }
  528. struct rcu_synchronize {
  529. struct rcu_head head;
  530. struct completion completion;
  531. };
  532. /* Because of FASTCALL declaration of complete, we use this wrapper */
  533. static void wakeme_after_rcu(struct rcu_head *head)
  534. {
  535. struct rcu_synchronize *rcu;
  536. rcu = container_of(head, struct rcu_synchronize, head);
  537. complete(&rcu->completion);
  538. }
  539. /**
  540. * synchronize_rcu - wait until a grace period has elapsed.
  541. *
  542. * Control will return to the caller some time after a full grace
  543. * period has elapsed, in other words after all currently executing RCU
  544. * read-side critical sections have completed. RCU read-side critical
  545. * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
  546. * and may be nested.
  547. *
  548. * If your read-side code is not protected by rcu_read_lock(), do -not-
  549. * use synchronize_rcu().
  550. */
  551. void synchronize_rcu(void)
  552. {
  553. struct rcu_synchronize rcu;
  554. init_completion(&rcu.completion);
  555. /* Will wake me after RCU finished */
  556. call_rcu(&rcu.head, wakeme_after_rcu);
  557. /* Wait for it */
  558. wait_for_completion(&rcu.completion);
  559. }
  560. module_param(blimit, int, 0);
  561. module_param(qhimark, int, 0);
  562. module_param(qlowmark, int, 0);
  563. EXPORT_SYMBOL_GPL(rcu_batches_completed);
  564. EXPORT_SYMBOL_GPL(rcu_batches_completed_bh);
  565. EXPORT_SYMBOL_GPL(call_rcu);
  566. EXPORT_SYMBOL_GPL(call_rcu_bh);
  567. EXPORT_SYMBOL_GPL(synchronize_rcu);