rcutorture.c 27 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004
  1. /*
  2. * Read-Copy Update module-based torture test facility
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License as published by
  6. * the Free Software Foundation; either version 2 of the License, or
  7. * (at your option) any later version.
  8. *
  9. * This program is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write to the Free Software
  16. * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17. *
  18. * Copyright (C) IBM Corporation, 2005, 2006
  19. *
  20. * Authors: Paul E. McKenney <paulmck@us.ibm.com>
  21. * Josh Triplett <josh@freedesktop.org>
  22. *
  23. * See also: Documentation/RCU/torture.txt
  24. */
  25. #include <linux/types.h>
  26. #include <linux/kernel.h>
  27. #include <linux/init.h>
  28. #include <linux/module.h>
  29. #include <linux/kthread.h>
  30. #include <linux/err.h>
  31. #include <linux/spinlock.h>
  32. #include <linux/smp.h>
  33. #include <linux/rcupdate.h>
  34. #include <linux/interrupt.h>
  35. #include <linux/sched.h>
  36. #include <asm/atomic.h>
  37. #include <linux/bitops.h>
  38. #include <linux/module.h>
  39. #include <linux/completion.h>
  40. #include <linux/moduleparam.h>
  41. #include <linux/percpu.h>
  42. #include <linux/notifier.h>
  43. #include <linux/cpu.h>
  44. #include <linux/random.h>
  45. #include <linux/delay.h>
  46. #include <linux/byteorder/swabb.h>
  47. #include <linux/stat.h>
  48. #include <linux/srcu.h>
  49. MODULE_LICENSE("GPL");
  50. MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and "
  51. "Josh Triplett <josh@freedesktop.org>");
  52. static int nreaders = -1; /* # reader threads, defaults to 2*ncpus */
  53. static int nfakewriters = 4; /* # fake writer threads */
  54. static int stat_interval; /* Interval between stats, in seconds. */
  55. /* Defaults to "only at end of test". */
  56. static int verbose; /* Print more debug info. */
  57. static int test_no_idle_hz; /* Test RCU's support for tickless idle CPUs. */
  58. static int shuffle_interval = 5; /* Interval between shuffles (in sec)*/
  59. static char *torture_type = "rcu"; /* What RCU implementation to torture. */
  60. module_param(nreaders, int, 0444);
  61. MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
  62. module_param(nfakewriters, int, 0444);
  63. MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");
  64. module_param(stat_interval, int, 0444);
  65. MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
  66. module_param(verbose, bool, 0444);
  67. MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
  68. module_param(test_no_idle_hz, bool, 0444);
  69. MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
  70. module_param(shuffle_interval, int, 0444);
  71. MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
  72. module_param(torture_type, charp, 0444);
  73. MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
  74. #define TORTURE_FLAG "-torture:"
  75. #define PRINTK_STRING(s) \
  76. do { printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
  77. #define VERBOSE_PRINTK_STRING(s) \
  78. do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
  79. #define VERBOSE_PRINTK_ERRSTRING(s) \
  80. do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0)
  81. static char printk_buf[4096];
  82. static int nrealreaders;
  83. static struct task_struct *writer_task;
  84. static struct task_struct **fakewriter_tasks;
  85. static struct task_struct **reader_tasks;
  86. static struct task_struct *stats_task;
  87. static struct task_struct *shuffler_task;
  88. #define RCU_TORTURE_PIPE_LEN 10
  89. struct rcu_torture {
  90. struct rcu_head rtort_rcu;
  91. int rtort_pipe_count;
  92. struct list_head rtort_free;
  93. int rtort_mbtest;
  94. };
  95. static int fullstop = 0; /* stop generating callbacks at test end. */
  96. static LIST_HEAD(rcu_torture_freelist);
  97. static struct rcu_torture *rcu_torture_current = NULL;
  98. static long rcu_torture_current_version = 0;
  99. static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
  100. static DEFINE_SPINLOCK(rcu_torture_lock);
  101. static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
  102. { 0 };
  103. static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =
  104. { 0 };
  105. static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
  106. static atomic_t n_rcu_torture_alloc;
  107. static atomic_t n_rcu_torture_alloc_fail;
  108. static atomic_t n_rcu_torture_free;
  109. static atomic_t n_rcu_torture_mberror;
  110. static atomic_t n_rcu_torture_error;
  111. static struct list_head rcu_torture_removed;
  112. /*
  113. * Allocate an element from the rcu_tortures pool.
  114. */
  115. static struct rcu_torture *
  116. rcu_torture_alloc(void)
  117. {
  118. struct list_head *p;
  119. spin_lock_bh(&rcu_torture_lock);
  120. if (list_empty(&rcu_torture_freelist)) {
  121. atomic_inc(&n_rcu_torture_alloc_fail);
  122. spin_unlock_bh(&rcu_torture_lock);
  123. return NULL;
  124. }
  125. atomic_inc(&n_rcu_torture_alloc);
  126. p = rcu_torture_freelist.next;
  127. list_del_init(p);
  128. spin_unlock_bh(&rcu_torture_lock);
  129. return container_of(p, struct rcu_torture, rtort_free);
  130. }
  131. /*
  132. * Free an element to the rcu_tortures pool.
  133. */
  134. static void
  135. rcu_torture_free(struct rcu_torture *p)
  136. {
  137. atomic_inc(&n_rcu_torture_free);
  138. spin_lock_bh(&rcu_torture_lock);
  139. list_add_tail(&p->rtort_free, &rcu_torture_freelist);
  140. spin_unlock_bh(&rcu_torture_lock);
  141. }
  142. struct rcu_random_state {
  143. unsigned long rrs_state;
  144. long rrs_count;
  145. };
  146. #define RCU_RANDOM_MULT 39916801 /* prime */
  147. #define RCU_RANDOM_ADD 479001701 /* prime */
  148. #define RCU_RANDOM_REFRESH 10000
  149. #define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }
  150. /*
  151. * Crude but fast random-number generator. Uses a linear congruential
  152. * generator, with occasional help from get_random_bytes().
  153. */
  154. static unsigned long
  155. rcu_random(struct rcu_random_state *rrsp)
  156. {
  157. long refresh;
  158. if (--rrsp->rrs_count < 0) {
  159. get_random_bytes(&refresh, sizeof(refresh));
  160. rrsp->rrs_state += refresh;
  161. rrsp->rrs_count = RCU_RANDOM_REFRESH;
  162. }
  163. rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;
  164. return swahw32(rrsp->rrs_state);
  165. }
  166. /*
  167. * Operations vector for selecting different types of tests.
  168. */
  169. struct rcu_torture_ops {
  170. void (*init)(void);
  171. void (*cleanup)(void);
  172. int (*readlock)(void);
  173. void (*readdelay)(struct rcu_random_state *rrsp);
  174. void (*readunlock)(int idx);
  175. int (*completed)(void);
  176. void (*deferredfree)(struct rcu_torture *p);
  177. void (*sync)(void);
  178. int (*stats)(char *page);
  179. char *name;
  180. };
  181. static struct rcu_torture_ops *cur_ops = NULL;
  182. /*
  183. * Definitions for rcu torture testing.
  184. */
  185. static int rcu_torture_read_lock(void) __acquires(RCU)
  186. {
  187. rcu_read_lock();
  188. return 0;
  189. }
  190. static void rcu_read_delay(struct rcu_random_state *rrsp)
  191. {
  192. long delay;
  193. const long longdelay = 200;
  194. /* We want there to be long-running readers, but not all the time. */
  195. delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay);
  196. if (!delay)
  197. udelay(longdelay);
  198. }
  199. static void rcu_torture_read_unlock(int idx) __releases(RCU)
  200. {
  201. rcu_read_unlock();
  202. }
  203. static int rcu_torture_completed(void)
  204. {
  205. return rcu_batches_completed();
  206. }
  207. static void
  208. rcu_torture_cb(struct rcu_head *p)
  209. {
  210. int i;
  211. struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
  212. if (fullstop) {
  213. /* Test is ending, just drop callbacks on the floor. */
  214. /* The next initialization will pick up the pieces. */
  215. return;
  216. }
  217. i = rp->rtort_pipe_count;
  218. if (i > RCU_TORTURE_PIPE_LEN)
  219. i = RCU_TORTURE_PIPE_LEN;
  220. atomic_inc(&rcu_torture_wcount[i]);
  221. if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
  222. rp->rtort_mbtest = 0;
  223. rcu_torture_free(rp);
  224. } else
  225. cur_ops->deferredfree(rp);
  226. }
  227. static void rcu_torture_deferred_free(struct rcu_torture *p)
  228. {
  229. call_rcu(&p->rtort_rcu, rcu_torture_cb);
  230. }
  231. static struct rcu_torture_ops rcu_ops = {
  232. .init = NULL,
  233. .cleanup = NULL,
  234. .readlock = rcu_torture_read_lock,
  235. .readdelay = rcu_read_delay,
  236. .readunlock = rcu_torture_read_unlock,
  237. .completed = rcu_torture_completed,
  238. .deferredfree = rcu_torture_deferred_free,
  239. .sync = synchronize_rcu,
  240. .stats = NULL,
  241. .name = "rcu"
  242. };
  243. static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
  244. {
  245. int i;
  246. struct rcu_torture *rp;
  247. struct rcu_torture *rp1;
  248. cur_ops->sync();
  249. list_add(&p->rtort_free, &rcu_torture_removed);
  250. list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
  251. i = rp->rtort_pipe_count;
  252. if (i > RCU_TORTURE_PIPE_LEN)
  253. i = RCU_TORTURE_PIPE_LEN;
  254. atomic_inc(&rcu_torture_wcount[i]);
  255. if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
  256. rp->rtort_mbtest = 0;
  257. list_del(&rp->rtort_free);
  258. rcu_torture_free(rp);
  259. }
  260. }
  261. }
  262. static void rcu_sync_torture_init(void)
  263. {
  264. INIT_LIST_HEAD(&rcu_torture_removed);
  265. }
  266. static struct rcu_torture_ops rcu_sync_ops = {
  267. .init = rcu_sync_torture_init,
  268. .cleanup = NULL,
  269. .readlock = rcu_torture_read_lock,
  270. .readdelay = rcu_read_delay,
  271. .readunlock = rcu_torture_read_unlock,
  272. .completed = rcu_torture_completed,
  273. .deferredfree = rcu_sync_torture_deferred_free,
  274. .sync = synchronize_rcu,
  275. .stats = NULL,
  276. .name = "rcu_sync"
  277. };
  278. /*
  279. * Definitions for rcu_bh torture testing.
  280. */
  281. static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
  282. {
  283. rcu_read_lock_bh();
  284. return 0;
  285. }
  286. static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
  287. {
  288. rcu_read_unlock_bh();
  289. }
  290. static int rcu_bh_torture_completed(void)
  291. {
  292. return rcu_batches_completed_bh();
  293. }
  294. static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
  295. {
  296. call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
  297. }
  298. struct rcu_bh_torture_synchronize {
  299. struct rcu_head head;
  300. struct completion completion;
  301. };
  302. static void rcu_bh_torture_wakeme_after_cb(struct rcu_head *head)
  303. {
  304. struct rcu_bh_torture_synchronize *rcu;
  305. rcu = container_of(head, struct rcu_bh_torture_synchronize, head);
  306. complete(&rcu->completion);
  307. }
  308. static void rcu_bh_torture_synchronize(void)
  309. {
  310. struct rcu_bh_torture_synchronize rcu;
  311. init_completion(&rcu.completion);
  312. call_rcu_bh(&rcu.head, rcu_bh_torture_wakeme_after_cb);
  313. wait_for_completion(&rcu.completion);
  314. }
  315. static struct rcu_torture_ops rcu_bh_ops = {
  316. .init = NULL,
  317. .cleanup = NULL,
  318. .readlock = rcu_bh_torture_read_lock,
  319. .readdelay = rcu_read_delay, /* just reuse rcu's version. */
  320. .readunlock = rcu_bh_torture_read_unlock,
  321. .completed = rcu_bh_torture_completed,
  322. .deferredfree = rcu_bh_torture_deferred_free,
  323. .sync = rcu_bh_torture_synchronize,
  324. .stats = NULL,
  325. .name = "rcu_bh"
  326. };
  327. static struct rcu_torture_ops rcu_bh_sync_ops = {
  328. .init = rcu_sync_torture_init,
  329. .cleanup = NULL,
  330. .readlock = rcu_bh_torture_read_lock,
  331. .readdelay = rcu_read_delay, /* just reuse rcu's version. */
  332. .readunlock = rcu_bh_torture_read_unlock,
  333. .completed = rcu_bh_torture_completed,
  334. .deferredfree = rcu_sync_torture_deferred_free,
  335. .sync = rcu_bh_torture_synchronize,
  336. .stats = NULL,
  337. .name = "rcu_bh_sync"
  338. };
  339. /*
  340. * Definitions for srcu torture testing.
  341. */
  342. static struct srcu_struct srcu_ctl;
  343. static void srcu_torture_init(void)
  344. {
  345. init_srcu_struct(&srcu_ctl);
  346. rcu_sync_torture_init();
  347. }
  348. static void srcu_torture_cleanup(void)
  349. {
  350. synchronize_srcu(&srcu_ctl);
  351. cleanup_srcu_struct(&srcu_ctl);
  352. }
  353. static int srcu_torture_read_lock(void) __acquires(&srcu_ctl)
  354. {
  355. return srcu_read_lock(&srcu_ctl);
  356. }
  357. static void srcu_read_delay(struct rcu_random_state *rrsp)
  358. {
  359. long delay;
  360. const long uspertick = 1000000 / HZ;
  361. const long longdelay = 10;
  362. /* We want there to be long-running readers, but not all the time. */
  363. delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick);
  364. if (!delay)
  365. schedule_timeout_interruptible(longdelay);
  366. }
  367. static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl)
  368. {
  369. srcu_read_unlock(&srcu_ctl, idx);
  370. }
  371. static int srcu_torture_completed(void)
  372. {
  373. return srcu_batches_completed(&srcu_ctl);
  374. }
  375. static void srcu_torture_synchronize(void)
  376. {
  377. synchronize_srcu(&srcu_ctl);
  378. }
  379. static int srcu_torture_stats(char *page)
  380. {
  381. int cnt = 0;
  382. int cpu;
  383. int idx = srcu_ctl.completed & 0x1;
  384. cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):",
  385. torture_type, TORTURE_FLAG, idx);
  386. for_each_possible_cpu(cpu) {
  387. cnt += sprintf(&page[cnt], " %d(%d,%d)", cpu,
  388. per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx],
  389. per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]);
  390. }
  391. cnt += sprintf(&page[cnt], "\n");
  392. return cnt;
  393. }
  394. static struct rcu_torture_ops srcu_ops = {
  395. .init = srcu_torture_init,
  396. .cleanup = srcu_torture_cleanup,
  397. .readlock = srcu_torture_read_lock,
  398. .readdelay = srcu_read_delay,
  399. .readunlock = srcu_torture_read_unlock,
  400. .completed = srcu_torture_completed,
  401. .deferredfree = rcu_sync_torture_deferred_free,
  402. .sync = srcu_torture_synchronize,
  403. .stats = srcu_torture_stats,
  404. .name = "srcu"
  405. };
  406. /*
  407. * Definitions for sched torture testing.
  408. */
  409. static int sched_torture_read_lock(void)
  410. {
  411. preempt_disable();
  412. return 0;
  413. }
  414. static void sched_torture_read_unlock(int idx)
  415. {
  416. preempt_enable();
  417. }
  418. static int sched_torture_completed(void)
  419. {
  420. return 0;
  421. }
  422. static void sched_torture_synchronize(void)
  423. {
  424. synchronize_sched();
  425. }
  426. static struct rcu_torture_ops sched_ops = {
  427. .init = rcu_sync_torture_init,
  428. .cleanup = NULL,
  429. .readlock = sched_torture_read_lock,
  430. .readdelay = rcu_read_delay, /* just reuse rcu's version. */
  431. .readunlock = sched_torture_read_unlock,
  432. .completed = sched_torture_completed,
  433. .deferredfree = rcu_sync_torture_deferred_free,
  434. .sync = sched_torture_synchronize,
  435. .stats = NULL,
  436. .name = "sched"
  437. };
  438. static struct rcu_torture_ops *torture_ops[] =
  439. { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops, &srcu_ops,
  440. &sched_ops, NULL };
  441. /*
  442. * RCU torture writer kthread. Repeatedly substitutes a new structure
  443. * for that pointed to by rcu_torture_current, freeing the old structure
  444. * after a series of grace periods (the "pipeline").
  445. */
  446. static int
  447. rcu_torture_writer(void *arg)
  448. {
  449. int i;
  450. long oldbatch = rcu_batches_completed();
  451. struct rcu_torture *rp;
  452. struct rcu_torture *old_rp;
  453. static DEFINE_RCU_RANDOM(rand);
  454. VERBOSE_PRINTK_STRING("rcu_torture_writer task started");
  455. set_user_nice(current, 19);
  456. current->flags |= PF_NOFREEZE;
  457. do {
  458. schedule_timeout_uninterruptible(1);
  459. if ((rp = rcu_torture_alloc()) == NULL)
  460. continue;
  461. rp->rtort_pipe_count = 0;
  462. udelay(rcu_random(&rand) & 0x3ff);
  463. old_rp = rcu_torture_current;
  464. rp->rtort_mbtest = 1;
  465. rcu_assign_pointer(rcu_torture_current, rp);
  466. smp_wmb();
  467. if (old_rp != NULL) {
  468. i = old_rp->rtort_pipe_count;
  469. if (i > RCU_TORTURE_PIPE_LEN)
  470. i = RCU_TORTURE_PIPE_LEN;
  471. atomic_inc(&rcu_torture_wcount[i]);
  472. old_rp->rtort_pipe_count++;
  473. cur_ops->deferredfree(old_rp);
  474. }
  475. rcu_torture_current_version++;
  476. oldbatch = cur_ops->completed();
  477. } while (!kthread_should_stop() && !fullstop);
  478. VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
  479. while (!kthread_should_stop())
  480. schedule_timeout_uninterruptible(1);
  481. return 0;
  482. }
  483. /*
  484. * RCU torture fake writer kthread. Repeatedly calls sync, with a random
  485. * delay between calls.
  486. */
  487. static int
  488. rcu_torture_fakewriter(void *arg)
  489. {
  490. DEFINE_RCU_RANDOM(rand);
  491. VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task started");
  492. set_user_nice(current, 19);
  493. current->flags |= PF_NOFREEZE;
  494. do {
  495. schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
  496. udelay(rcu_random(&rand) & 0x3ff);
  497. cur_ops->sync();
  498. } while (!kthread_should_stop() && !fullstop);
  499. VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping");
  500. while (!kthread_should_stop())
  501. schedule_timeout_uninterruptible(1);
  502. return 0;
  503. }
  504. /*
  505. * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current,
  506. * incrementing the corresponding element of the pipeline array. The
  507. * counter in the element should never be greater than 1, otherwise, the
  508. * RCU implementation is broken.
  509. */
  510. static int
  511. rcu_torture_reader(void *arg)
  512. {
  513. int completed;
  514. int idx;
  515. DEFINE_RCU_RANDOM(rand);
  516. struct rcu_torture *p;
  517. int pipe_count;
  518. VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
  519. set_user_nice(current, 19);
  520. current->flags |= PF_NOFREEZE;
  521. do {
  522. idx = cur_ops->readlock();
  523. completed = cur_ops->completed();
  524. p = rcu_dereference(rcu_torture_current);
  525. if (p == NULL) {
  526. /* Wait for rcu_torture_writer to get underway */
  527. cur_ops->readunlock(idx);
  528. schedule_timeout_interruptible(HZ);
  529. continue;
  530. }
  531. if (p->rtort_mbtest == 0)
  532. atomic_inc(&n_rcu_torture_mberror);
  533. cur_ops->readdelay(&rand);
  534. preempt_disable();
  535. pipe_count = p->rtort_pipe_count;
  536. if (pipe_count > RCU_TORTURE_PIPE_LEN) {
  537. /* Should not happen, but... */
  538. pipe_count = RCU_TORTURE_PIPE_LEN;
  539. }
  540. ++__get_cpu_var(rcu_torture_count)[pipe_count];
  541. completed = cur_ops->completed() - completed;
  542. if (completed > RCU_TORTURE_PIPE_LEN) {
  543. /* Should not happen, but... */
  544. completed = RCU_TORTURE_PIPE_LEN;
  545. }
  546. ++__get_cpu_var(rcu_torture_batch)[completed];
  547. preempt_enable();
  548. cur_ops->readunlock(idx);
  549. schedule();
  550. } while (!kthread_should_stop() && !fullstop);
  551. VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
  552. while (!kthread_should_stop())
  553. schedule_timeout_uninterruptible(1);
  554. return 0;
  555. }
  556. /*
  557. * Create an RCU-torture statistics message in the specified buffer.
  558. */
  559. static int
  560. rcu_torture_printk(char *page)
  561. {
  562. int cnt = 0;
  563. int cpu;
  564. int i;
  565. long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
  566. long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
  567. for_each_possible_cpu(cpu) {
  568. for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
  569. pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
  570. batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
  571. }
  572. }
  573. for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
  574. if (pipesummary[i] != 0)
  575. break;
  576. }
  577. cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
  578. cnt += sprintf(&page[cnt],
  579. "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d "
  580. "rtmbe: %d",
  581. rcu_torture_current,
  582. rcu_torture_current_version,
  583. list_empty(&rcu_torture_freelist),
  584. atomic_read(&n_rcu_torture_alloc),
  585. atomic_read(&n_rcu_torture_alloc_fail),
  586. atomic_read(&n_rcu_torture_free),
  587. atomic_read(&n_rcu_torture_mberror));
  588. if (atomic_read(&n_rcu_torture_mberror) != 0)
  589. cnt += sprintf(&page[cnt], " !!!");
  590. cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
  591. if (i > 1) {
  592. cnt += sprintf(&page[cnt], "!!! ");
  593. atomic_inc(&n_rcu_torture_error);
  594. }
  595. cnt += sprintf(&page[cnt], "Reader Pipe: ");
  596. for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
  597. cnt += sprintf(&page[cnt], " %ld", pipesummary[i]);
  598. cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
  599. cnt += sprintf(&page[cnt], "Reader Batch: ");
  600. for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
  601. cnt += sprintf(&page[cnt], " %ld", batchsummary[i]);
  602. cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
  603. cnt += sprintf(&page[cnt], "Free-Block Circulation: ");
  604. for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
  605. cnt += sprintf(&page[cnt], " %d",
  606. atomic_read(&rcu_torture_wcount[i]));
  607. }
  608. cnt += sprintf(&page[cnt], "\n");
  609. if (cur_ops->stats != NULL)
  610. cnt += cur_ops->stats(&page[cnt]);
  611. return cnt;
  612. }
  613. /*
  614. * Print torture statistics. Caller must ensure that there is only
  615. * one call to this function at a given time!!! This is normally
  616. * accomplished by relying on the module system to only have one copy
  617. * of the module loaded, and then by giving the rcu_torture_stats
  618. * kthread full control (or the init/cleanup functions when rcu_torture_stats
  619. * thread is not running).
  620. */
  621. static void
  622. rcu_torture_stats_print(void)
  623. {
  624. int cnt;
  625. cnt = rcu_torture_printk(printk_buf);
  626. printk(KERN_ALERT "%s", printk_buf);
  627. }
  628. /*
  629. * Periodically prints torture statistics, if periodic statistics printing
  630. * was specified via the stat_interval module parameter.
  631. *
  632. * No need to worry about fullstop here, since this one doesn't reference
  633. * volatile state or register callbacks.
  634. */
  635. static int
  636. rcu_torture_stats(void *arg)
  637. {
  638. VERBOSE_PRINTK_STRING("rcu_torture_stats task started");
  639. do {
  640. schedule_timeout_interruptible(stat_interval * HZ);
  641. rcu_torture_stats_print();
  642. } while (!kthread_should_stop());
  643. VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
  644. return 0;
  645. }
  646. static int rcu_idle_cpu; /* Force all torture tasks off this CPU */
  647. /* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case
  648. * is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs.
  649. */
  650. static void rcu_torture_shuffle_tasks(void)
  651. {
  652. cpumask_t tmp_mask = CPU_MASK_ALL;
  653. int i;
  654. lock_cpu_hotplug();
  655. /* No point in shuffling if there is only one online CPU (ex: UP) */
  656. if (num_online_cpus() == 1) {
  657. unlock_cpu_hotplug();
  658. return;
  659. }
  660. if (rcu_idle_cpu != -1)
  661. cpu_clear(rcu_idle_cpu, tmp_mask);
  662. set_cpus_allowed(current, tmp_mask);
  663. if (reader_tasks != NULL) {
  664. for (i = 0; i < nrealreaders; i++)
  665. if (reader_tasks[i])
  666. set_cpus_allowed(reader_tasks[i], tmp_mask);
  667. }
  668. if (fakewriter_tasks != NULL) {
  669. for (i = 0; i < nfakewriters; i++)
  670. if (fakewriter_tasks[i])
  671. set_cpus_allowed(fakewriter_tasks[i], tmp_mask);
  672. }
  673. if (writer_task)
  674. set_cpus_allowed(writer_task, tmp_mask);
  675. if (stats_task)
  676. set_cpus_allowed(stats_task, tmp_mask);
  677. if (rcu_idle_cpu == -1)
  678. rcu_idle_cpu = num_online_cpus() - 1;
  679. else
  680. rcu_idle_cpu--;
  681. unlock_cpu_hotplug();
  682. }
  683. /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
  684. * system to become idle at a time and cut off its timer ticks. This is meant
  685. * to test the support for such tickless idle CPU in RCU.
  686. */
  687. static int
  688. rcu_torture_shuffle(void *arg)
  689. {
  690. VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started");
  691. do {
  692. schedule_timeout_interruptible(shuffle_interval * HZ);
  693. rcu_torture_shuffle_tasks();
  694. } while (!kthread_should_stop());
  695. VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
  696. return 0;
  697. }
  698. static inline void
  699. rcu_torture_print_module_parms(char *tag)
  700. {
  701. printk(KERN_ALERT "%s" TORTURE_FLAG
  702. "--- %s: nreaders=%d nfakewriters=%d "
  703. "stat_interval=%d verbose=%d test_no_idle_hz=%d "
  704. "shuffle_interval = %d\n",
  705. torture_type, tag, nrealreaders, nfakewriters,
  706. stat_interval, verbose, test_no_idle_hz, shuffle_interval);
  707. }
  708. static void
  709. rcu_torture_cleanup(void)
  710. {
  711. int i;
  712. fullstop = 1;
  713. if (shuffler_task != NULL) {
  714. VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
  715. kthread_stop(shuffler_task);
  716. }
  717. shuffler_task = NULL;
  718. if (writer_task != NULL) {
  719. VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task");
  720. kthread_stop(writer_task);
  721. }
  722. writer_task = NULL;
  723. if (reader_tasks != NULL) {
  724. for (i = 0; i < nrealreaders; i++) {
  725. if (reader_tasks[i] != NULL) {
  726. VERBOSE_PRINTK_STRING(
  727. "Stopping rcu_torture_reader task");
  728. kthread_stop(reader_tasks[i]);
  729. }
  730. reader_tasks[i] = NULL;
  731. }
  732. kfree(reader_tasks);
  733. reader_tasks = NULL;
  734. }
  735. rcu_torture_current = NULL;
  736. if (fakewriter_tasks != NULL) {
  737. for (i = 0; i < nfakewriters; i++) {
  738. if (fakewriter_tasks[i] != NULL) {
  739. VERBOSE_PRINTK_STRING(
  740. "Stopping rcu_torture_fakewriter task");
  741. kthread_stop(fakewriter_tasks[i]);
  742. }
  743. fakewriter_tasks[i] = NULL;
  744. }
  745. kfree(fakewriter_tasks);
  746. fakewriter_tasks = NULL;
  747. }
  748. if (stats_task != NULL) {
  749. VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task");
  750. kthread_stop(stats_task);
  751. }
  752. stats_task = NULL;
  753. /* Wait for all RCU callbacks to fire. */
  754. rcu_barrier();
  755. rcu_torture_stats_print(); /* -After- the stats thread is stopped! */
  756. if (cur_ops->cleanup != NULL)
  757. cur_ops->cleanup();
  758. if (atomic_read(&n_rcu_torture_error))
  759. rcu_torture_print_module_parms("End of test: FAILURE");
  760. else
  761. rcu_torture_print_module_parms("End of test: SUCCESS");
  762. }
  763. static int
  764. rcu_torture_init(void)
  765. {
  766. int i;
  767. int cpu;
  768. int firsterr = 0;
  769. /* Process args and tell the world that the torturer is on the job. */
  770. for (i = 0; cur_ops = torture_ops[i], cur_ops != NULL; i++) {
  771. cur_ops = torture_ops[i];
  772. if (strcmp(torture_type, cur_ops->name) == 0) {
  773. break;
  774. }
  775. }
  776. if (cur_ops == NULL) {
  777. printk(KERN_ALERT "rcutorture: invalid torture type: \"%s\"\n",
  778. torture_type);
  779. return (-EINVAL);
  780. }
  781. if (cur_ops->init != NULL)
  782. cur_ops->init(); /* no "goto unwind" prior to this point!!! */
  783. if (nreaders >= 0)
  784. nrealreaders = nreaders;
  785. else
  786. nrealreaders = 2 * num_online_cpus();
  787. rcu_torture_print_module_parms("Start of test");
  788. fullstop = 0;
  789. /* Set up the freelist. */
  790. INIT_LIST_HEAD(&rcu_torture_freelist);
  791. for (i = 0; i < sizeof(rcu_tortures) / sizeof(rcu_tortures[0]); i++) {
  792. rcu_tortures[i].rtort_mbtest = 0;
  793. list_add_tail(&rcu_tortures[i].rtort_free,
  794. &rcu_torture_freelist);
  795. }
  796. /* Initialize the statistics so that each run gets its own numbers. */
  797. rcu_torture_current = NULL;
  798. rcu_torture_current_version = 0;
  799. atomic_set(&n_rcu_torture_alloc, 0);
  800. atomic_set(&n_rcu_torture_alloc_fail, 0);
  801. atomic_set(&n_rcu_torture_free, 0);
  802. atomic_set(&n_rcu_torture_mberror, 0);
  803. atomic_set(&n_rcu_torture_error, 0);
  804. for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
  805. atomic_set(&rcu_torture_wcount[i], 0);
  806. for_each_possible_cpu(cpu) {
  807. for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
  808. per_cpu(rcu_torture_count, cpu)[i] = 0;
  809. per_cpu(rcu_torture_batch, cpu)[i] = 0;
  810. }
  811. }
  812. /* Start up the kthreads. */
  813. VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task");
  814. writer_task = kthread_run(rcu_torture_writer, NULL,
  815. "rcu_torture_writer");
  816. if (IS_ERR(writer_task)) {
  817. firsterr = PTR_ERR(writer_task);
  818. VERBOSE_PRINTK_ERRSTRING("Failed to create writer");
  819. writer_task = NULL;
  820. goto unwind;
  821. }
  822. fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]),
  823. GFP_KERNEL);
  824. if (fakewriter_tasks == NULL) {
  825. VERBOSE_PRINTK_ERRSTRING("out of memory");
  826. firsterr = -ENOMEM;
  827. goto unwind;
  828. }
  829. for (i = 0; i < nfakewriters; i++) {
  830. VERBOSE_PRINTK_STRING("Creating rcu_torture_fakewriter task");
  831. fakewriter_tasks[i] = kthread_run(rcu_torture_fakewriter, NULL,
  832. "rcu_torture_fakewriter");
  833. if (IS_ERR(fakewriter_tasks[i])) {
  834. firsterr = PTR_ERR(fakewriter_tasks[i]);
  835. VERBOSE_PRINTK_ERRSTRING("Failed to create fakewriter");
  836. fakewriter_tasks[i] = NULL;
  837. goto unwind;
  838. }
  839. }
  840. reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]),
  841. GFP_KERNEL);
  842. if (reader_tasks == NULL) {
  843. VERBOSE_PRINTK_ERRSTRING("out of memory");
  844. firsterr = -ENOMEM;
  845. goto unwind;
  846. }
  847. for (i = 0; i < nrealreaders; i++) {
  848. VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task");
  849. reader_tasks[i] = kthread_run(rcu_torture_reader, NULL,
  850. "rcu_torture_reader");
  851. if (IS_ERR(reader_tasks[i])) {
  852. firsterr = PTR_ERR(reader_tasks[i]);
  853. VERBOSE_PRINTK_ERRSTRING("Failed to create reader");
  854. reader_tasks[i] = NULL;
  855. goto unwind;
  856. }
  857. }
  858. if (stat_interval > 0) {
  859. VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task");
  860. stats_task = kthread_run(rcu_torture_stats, NULL,
  861. "rcu_torture_stats");
  862. if (IS_ERR(stats_task)) {
  863. firsterr = PTR_ERR(stats_task);
  864. VERBOSE_PRINTK_ERRSTRING("Failed to create stats");
  865. stats_task = NULL;
  866. goto unwind;
  867. }
  868. }
  869. if (test_no_idle_hz) {
  870. rcu_idle_cpu = num_online_cpus() - 1;
  871. /* Create the shuffler thread */
  872. shuffler_task = kthread_run(rcu_torture_shuffle, NULL,
  873. "rcu_torture_shuffle");
  874. if (IS_ERR(shuffler_task)) {
  875. firsterr = PTR_ERR(shuffler_task);
  876. VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler");
  877. shuffler_task = NULL;
  878. goto unwind;
  879. }
  880. }
  881. return 0;
  882. unwind:
  883. rcu_torture_cleanup();
  884. return firsterr;
  885. }
  886. module_init(rcu_torture_init);
  887. module_exit(rcu_torture_cleanup);