rcuscale.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * Read-Copy Update module-based scalability-test facility
  4. *
  5. * Copyright (C) IBM Corporation, 2015
  6. *
  7. * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
  8. */
  9. #define pr_fmt(fmt) fmt
  10. #include <linux/types.h>
  11. #include <linux/kernel.h>
  12. #include <linux/init.h>
  13. #include <linux/mm.h>
  14. #include <linux/module.h>
  15. #include <linux/kthread.h>
  16. #include <linux/err.h>
  17. #include <linux/spinlock.h>
  18. #include <linux/smp.h>
  19. #include <linux/rcupdate.h>
  20. #include <linux/interrupt.h>
  21. #include <linux/sched.h>
  22. #include <uapi/linux/sched/types.h>
  23. #include <linux/atomic.h>
  24. #include <linux/bitops.h>
  25. #include <linux/completion.h>
  26. #include <linux/moduleparam.h>
  27. #include <linux/percpu.h>
  28. #include <linux/notifier.h>
  29. #include <linux/reboot.h>
  30. #include <linux/freezer.h>
  31. #include <linux/cpu.h>
  32. #include <linux/delay.h>
  33. #include <linux/stat.h>
  34. #include <linux/srcu.h>
  35. #include <linux/slab.h>
  36. #include <asm/byteorder.h>
  37. #include <linux/torture.h>
  38. #include <linux/vmalloc.h>
  39. #include "rcu.h"
  40. MODULE_LICENSE("GPL");
  41. MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
  42. #define SCALE_FLAG "-scale:"
  43. #define SCALEOUT_STRING(s) \
  44. pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s)
  45. #define VERBOSE_SCALEOUT_STRING(s) \
  46. do { if (verbose) pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s); } while (0)
  47. #define VERBOSE_SCALEOUT_ERRSTRING(s) \
  48. do { if (verbose) pr_alert("%s" SCALE_FLAG "!!! %s\n", scale_type, s); } while (0)
  49. /*
  50. * The intended use cases for the nreaders and nwriters module parameters
  51. * are as follows:
  52. *
  53. * 1. Specify only the nr_cpus kernel boot parameter. This will
  54. * set both nreaders and nwriters to the value specified by
  55. * nr_cpus for a mixed reader/writer test.
  56. *
  57. * 2. Specify the nr_cpus kernel boot parameter, but set
  58. * rcuscale.nreaders to zero. This will set nwriters to the
  59. * value specified by nr_cpus for an update-only test.
  60. *
  61. * 3. Specify the nr_cpus kernel boot parameter, but set
  62. * rcuscale.nwriters to zero. This will set nreaders to the
  63. * value specified by nr_cpus for a read-only test.
  64. *
  65. * Various other use cases may of course be specified.
  66. *
  67. * Note that this test's readers are intended only as a test load for
  68. * the writers. The reader scalability statistics will be overly
  69. * pessimistic due to the per-critical-section interrupt disabling,
  70. * test-end checks, and the pair of calls through pointers.
  71. */
  72. #ifdef MODULE
  73. # define RCUSCALE_SHUTDOWN 0
  74. #else
  75. # define RCUSCALE_SHUTDOWN 1
  76. #endif
  77. torture_param(bool, gp_async, false, "Use asynchronous GP wait primitives");
  78. torture_param(int, gp_async_max, 1000, "Max # outstanding waits per reader");
  79. torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
  80. torture_param(int, holdoff, 10, "Holdoff time before test start (s)");
  81. torture_param(int, nreaders, -1, "Number of RCU reader threads");
  82. torture_param(int, nwriters, -1, "Number of RCU updater threads");
  83. torture_param(bool, shutdown, RCUSCALE_SHUTDOWN,
  84. "Shutdown at end of scalability tests.");
  85. torture_param(int, verbose, 1, "Enable verbose debugging printk()s");
  86. torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable");
  87. torture_param(int, kfree_rcu_test, 0, "Do we run a kfree_rcu() scale test?");
  88. torture_param(int, kfree_mult, 1, "Multiple of kfree_obj size to allocate.");
  89. static char *scale_type = "rcu";
  90. module_param(scale_type, charp, 0444);
  91. MODULE_PARM_DESC(scale_type, "Type of RCU to scalability-test (rcu, srcu, ...)");
  92. static int nrealreaders;
  93. static int nrealwriters;
  94. static struct task_struct **writer_tasks;
  95. static struct task_struct **reader_tasks;
  96. static struct task_struct *shutdown_task;
  97. static u64 **writer_durations;
  98. static int *writer_n_durations;
  99. static atomic_t n_rcu_scale_reader_started;
  100. static atomic_t n_rcu_scale_writer_started;
  101. static atomic_t n_rcu_scale_writer_finished;
  102. static wait_queue_head_t shutdown_wq;
  103. static u64 t_rcu_scale_writer_started;
  104. static u64 t_rcu_scale_writer_finished;
  105. static unsigned long b_rcu_gp_test_started;
  106. static unsigned long b_rcu_gp_test_finished;
  107. static DEFINE_PER_CPU(atomic_t, n_async_inflight);
  108. #define MAX_MEAS 10000
  109. #define MIN_MEAS 100
  110. /*
  111. * Operations vector for selecting different types of tests.
  112. */
  113. struct rcu_scale_ops {
  114. int ptype;
  115. void (*init)(void);
  116. void (*cleanup)(void);
  117. int (*readlock)(void);
  118. void (*readunlock)(int idx);
  119. unsigned long (*get_gp_seq)(void);
  120. unsigned long (*gp_diff)(unsigned long new, unsigned long old);
  121. unsigned long (*exp_completed)(void);
  122. void (*async)(struct rcu_head *head, rcu_callback_t func);
  123. void (*gp_barrier)(void);
  124. void (*sync)(void);
  125. void (*exp_sync)(void);
  126. const char *name;
  127. };
  128. static struct rcu_scale_ops *cur_ops;
  129. /*
  130. * Definitions for rcu scalability testing.
  131. */
  132. static int rcu_scale_read_lock(void) __acquires(RCU)
  133. {
  134. rcu_read_lock();
  135. return 0;
  136. }
  137. static void rcu_scale_read_unlock(int idx) __releases(RCU)
  138. {
  139. rcu_read_unlock();
  140. }
  141. static unsigned long __maybe_unused rcu_no_completed(void)
  142. {
  143. return 0;
  144. }
  145. static void rcu_sync_scale_init(void)
  146. {
  147. }
  148. static struct rcu_scale_ops rcu_ops = {
  149. .ptype = RCU_FLAVOR,
  150. .init = rcu_sync_scale_init,
  151. .readlock = rcu_scale_read_lock,
  152. .readunlock = rcu_scale_read_unlock,
  153. .get_gp_seq = rcu_get_gp_seq,
  154. .gp_diff = rcu_seq_diff,
  155. .exp_completed = rcu_exp_batches_completed,
  156. .async = call_rcu,
  157. .gp_barrier = rcu_barrier,
  158. .sync = synchronize_rcu,
  159. .exp_sync = synchronize_rcu_expedited,
  160. .name = "rcu"
  161. };
  162. /*
  163. * Definitions for srcu scalability testing.
  164. */
  165. DEFINE_STATIC_SRCU(srcu_ctl_scale);
  166. static struct srcu_struct *srcu_ctlp = &srcu_ctl_scale;
  167. static int srcu_scale_read_lock(void) __acquires(srcu_ctlp)
  168. {
  169. return srcu_read_lock(srcu_ctlp);
  170. }
  171. static void srcu_scale_read_unlock(int idx) __releases(srcu_ctlp)
  172. {
  173. srcu_read_unlock(srcu_ctlp, idx);
  174. }
  175. static unsigned long srcu_scale_completed(void)
  176. {
  177. return srcu_batches_completed(srcu_ctlp);
  178. }
  179. static void srcu_call_rcu(struct rcu_head *head, rcu_callback_t func)
  180. {
  181. call_srcu(srcu_ctlp, head, func);
  182. }
  183. static void srcu_rcu_barrier(void)
  184. {
  185. srcu_barrier(srcu_ctlp);
  186. }
  187. static void srcu_scale_synchronize(void)
  188. {
  189. synchronize_srcu(srcu_ctlp);
  190. }
  191. static void srcu_scale_synchronize_expedited(void)
  192. {
  193. synchronize_srcu_expedited(srcu_ctlp);
  194. }
  195. static struct rcu_scale_ops srcu_ops = {
  196. .ptype = SRCU_FLAVOR,
  197. .init = rcu_sync_scale_init,
  198. .readlock = srcu_scale_read_lock,
  199. .readunlock = srcu_scale_read_unlock,
  200. .get_gp_seq = srcu_scale_completed,
  201. .gp_diff = rcu_seq_diff,
  202. .exp_completed = srcu_scale_completed,
  203. .async = srcu_call_rcu,
  204. .gp_barrier = srcu_rcu_barrier,
  205. .sync = srcu_scale_synchronize,
  206. .exp_sync = srcu_scale_synchronize_expedited,
  207. .name = "srcu"
  208. };
  209. static struct srcu_struct srcud;
  210. static void srcu_sync_scale_init(void)
  211. {
  212. srcu_ctlp = &srcud;
  213. init_srcu_struct(srcu_ctlp);
  214. }
  215. static void srcu_sync_scale_cleanup(void)
  216. {
  217. cleanup_srcu_struct(srcu_ctlp);
  218. }
  219. static struct rcu_scale_ops srcud_ops = {
  220. .ptype = SRCU_FLAVOR,
  221. .init = srcu_sync_scale_init,
  222. .cleanup = srcu_sync_scale_cleanup,
  223. .readlock = srcu_scale_read_lock,
  224. .readunlock = srcu_scale_read_unlock,
  225. .get_gp_seq = srcu_scale_completed,
  226. .gp_diff = rcu_seq_diff,
  227. .exp_completed = srcu_scale_completed,
  228. .async = srcu_call_rcu,
  229. .gp_barrier = srcu_rcu_barrier,
  230. .sync = srcu_scale_synchronize,
  231. .exp_sync = srcu_scale_synchronize_expedited,
  232. .name = "srcud"
  233. };
  234. /*
  235. * Definitions for RCU-tasks scalability testing.
  236. */
  237. static int tasks_scale_read_lock(void)
  238. {
  239. return 0;
  240. }
  241. static void tasks_scale_read_unlock(int idx)
  242. {
  243. }
  244. static struct rcu_scale_ops tasks_ops = {
  245. .ptype = RCU_TASKS_FLAVOR,
  246. .init = rcu_sync_scale_init,
  247. .readlock = tasks_scale_read_lock,
  248. .readunlock = tasks_scale_read_unlock,
  249. .get_gp_seq = rcu_no_completed,
  250. .gp_diff = rcu_seq_diff,
  251. .async = call_rcu_tasks,
  252. .gp_barrier = rcu_barrier_tasks,
  253. .sync = synchronize_rcu_tasks,
  254. .exp_sync = synchronize_rcu_tasks,
  255. .name = "tasks"
  256. };
  257. static unsigned long rcuscale_seq_diff(unsigned long new, unsigned long old)
  258. {
  259. if (!cur_ops->gp_diff)
  260. return new - old;
  261. return cur_ops->gp_diff(new, old);
  262. }
  263. /*
  264. * If scalability tests complete, wait for shutdown to commence.
  265. */
  266. static void rcu_scale_wait_shutdown(void)
  267. {
  268. cond_resched_tasks_rcu_qs();
  269. if (atomic_read(&n_rcu_scale_writer_finished) < nrealwriters)
  270. return;
  271. while (!torture_must_stop())
  272. schedule_timeout_uninterruptible(1);
  273. }
  274. /*
  275. * RCU scalability reader kthread. Repeatedly does empty RCU read-side
  276. * critical section, minimizing update-side interference. However, the
  277. * point of this test is not to evaluate reader scalability, but instead
  278. * to serve as a test load for update-side scalability testing.
  279. */
  280. static int
  281. rcu_scale_reader(void *arg)
  282. {
  283. unsigned long flags;
  284. int idx;
  285. long me = (long)arg;
  286. VERBOSE_SCALEOUT_STRING("rcu_scale_reader task started");
  287. set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
  288. set_user_nice(current, MAX_NICE);
  289. atomic_inc(&n_rcu_scale_reader_started);
  290. do {
  291. local_irq_save(flags);
  292. idx = cur_ops->readlock();
  293. cur_ops->readunlock(idx);
  294. local_irq_restore(flags);
  295. rcu_scale_wait_shutdown();
  296. } while (!torture_must_stop());
  297. torture_kthread_stopping("rcu_scale_reader");
  298. return 0;
  299. }
  300. /*
  301. * Callback function for asynchronous grace periods from rcu_scale_writer().
  302. */
  303. static void rcu_scale_async_cb(struct rcu_head *rhp)
  304. {
  305. atomic_dec(this_cpu_ptr(&n_async_inflight));
  306. kfree(rhp);
  307. }
  308. /*
  309. * RCU scale writer kthread. Repeatedly does a grace period.
  310. */
  311. static int
  312. rcu_scale_writer(void *arg)
  313. {
  314. int i = 0;
  315. int i_max;
  316. long me = (long)arg;
  317. struct rcu_head *rhp = NULL;
  318. bool started = false, done = false, alldone = false;
  319. u64 t;
  320. u64 *wdp;
  321. u64 *wdpp = writer_durations[me];
  322. VERBOSE_SCALEOUT_STRING("rcu_scale_writer task started");
  323. WARN_ON(!wdpp);
  324. set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
  325. sched_set_fifo_low(current);
  326. if (holdoff)
  327. schedule_timeout_uninterruptible(holdoff * HZ);
  328. /*
  329. * Wait until rcu_end_inkernel_boot() is called for normal GP tests
  330. * so that RCU is not always expedited for normal GP tests.
  331. * The system_state test is approximate, but works well in practice.
  332. */
  333. while (!gp_exp && system_state != SYSTEM_RUNNING)
  334. schedule_timeout_uninterruptible(1);
  335. t = ktime_get_mono_fast_ns();
  336. if (atomic_inc_return(&n_rcu_scale_writer_started) >= nrealwriters) {
  337. t_rcu_scale_writer_started = t;
  338. if (gp_exp) {
  339. b_rcu_gp_test_started =
  340. cur_ops->exp_completed() / 2;
  341. } else {
  342. b_rcu_gp_test_started = cur_ops->get_gp_seq();
  343. }
  344. }
  345. do {
  346. if (writer_holdoff)
  347. udelay(writer_holdoff);
  348. wdp = &wdpp[i];
  349. *wdp = ktime_get_mono_fast_ns();
  350. if (gp_async) {
  351. retry:
  352. if (!rhp)
  353. rhp = kmalloc(sizeof(*rhp), GFP_KERNEL);
  354. if (rhp && atomic_read(this_cpu_ptr(&n_async_inflight)) < gp_async_max) {
  355. atomic_inc(this_cpu_ptr(&n_async_inflight));
  356. cur_ops->async(rhp, rcu_scale_async_cb);
  357. rhp = NULL;
  358. } else if (!kthread_should_stop()) {
  359. cur_ops->gp_barrier();
  360. goto retry;
  361. } else {
  362. kfree(rhp); /* Because we are stopping. */
  363. }
  364. } else if (gp_exp) {
  365. cur_ops->exp_sync();
  366. } else {
  367. cur_ops->sync();
  368. }
  369. t = ktime_get_mono_fast_ns();
  370. *wdp = t - *wdp;
  371. i_max = i;
  372. if (!started &&
  373. atomic_read(&n_rcu_scale_writer_started) >= nrealwriters)
  374. started = true;
  375. if (!done && i >= MIN_MEAS) {
  376. done = true;
  377. sched_set_normal(current, 0);
  378. pr_alert("%s%s rcu_scale_writer %ld has %d measurements\n",
  379. scale_type, SCALE_FLAG, me, MIN_MEAS);
  380. if (atomic_inc_return(&n_rcu_scale_writer_finished) >=
  381. nrealwriters) {
  382. schedule_timeout_interruptible(10);
  383. rcu_ftrace_dump(DUMP_ALL);
  384. SCALEOUT_STRING("Test complete");
  385. t_rcu_scale_writer_finished = t;
  386. if (gp_exp) {
  387. b_rcu_gp_test_finished =
  388. cur_ops->exp_completed() / 2;
  389. } else {
  390. b_rcu_gp_test_finished =
  391. cur_ops->get_gp_seq();
  392. }
  393. if (shutdown) {
  394. smp_mb(); /* Assign before wake. */
  395. wake_up(&shutdown_wq);
  396. }
  397. }
  398. }
  399. if (done && !alldone &&
  400. atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters)
  401. alldone = true;
  402. if (started && !alldone && i < MAX_MEAS - 1)
  403. i++;
  404. rcu_scale_wait_shutdown();
  405. } while (!torture_must_stop());
  406. if (gp_async) {
  407. cur_ops->gp_barrier();
  408. }
  409. writer_n_durations[me] = i_max;
  410. torture_kthread_stopping("rcu_scale_writer");
  411. return 0;
  412. }
  413. static void
  414. rcu_scale_print_module_parms(struct rcu_scale_ops *cur_ops, const char *tag)
  415. {
  416. pr_alert("%s" SCALE_FLAG
  417. "--- %s: nreaders=%d nwriters=%d verbose=%d shutdown=%d\n",
  418. scale_type, tag, nrealreaders, nrealwriters, verbose, shutdown);
  419. }
  420. static void
  421. rcu_scale_cleanup(void)
  422. {
  423. int i;
  424. int j;
  425. int ngps = 0;
  426. u64 *wdp;
  427. u64 *wdpp;
  428. /*
  429. * Would like warning at start, but everything is expedited
  430. * during the mid-boot phase, so have to wait till the end.
  431. */
  432. if (rcu_gp_is_expedited() && !rcu_gp_is_normal() && !gp_exp)
  433. VERBOSE_SCALEOUT_ERRSTRING("All grace periods expedited, no normal ones to measure!");
  434. if (rcu_gp_is_normal() && gp_exp)
  435. VERBOSE_SCALEOUT_ERRSTRING("All grace periods normal, no expedited ones to measure!");
  436. if (gp_exp && gp_async)
  437. VERBOSE_SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!");
  438. if (torture_cleanup_begin())
  439. return;
  440. if (!cur_ops) {
  441. torture_cleanup_end();
  442. return;
  443. }
  444. if (reader_tasks) {
  445. for (i = 0; i < nrealreaders; i++)
  446. torture_stop_kthread(rcu_scale_reader,
  447. reader_tasks[i]);
  448. kfree(reader_tasks);
  449. }
  450. if (writer_tasks) {
  451. for (i = 0; i < nrealwriters; i++) {
  452. torture_stop_kthread(rcu_scale_writer,
  453. writer_tasks[i]);
  454. if (!writer_n_durations)
  455. continue;
  456. j = writer_n_durations[i];
  457. pr_alert("%s%s writer %d gps: %d\n",
  458. scale_type, SCALE_FLAG, i, j);
  459. ngps += j;
  460. }
  461. pr_alert("%s%s start: %llu end: %llu duration: %llu gps: %d batches: %ld\n",
  462. scale_type, SCALE_FLAG,
  463. t_rcu_scale_writer_started, t_rcu_scale_writer_finished,
  464. t_rcu_scale_writer_finished -
  465. t_rcu_scale_writer_started,
  466. ngps,
  467. rcuscale_seq_diff(b_rcu_gp_test_finished,
  468. b_rcu_gp_test_started));
  469. for (i = 0; i < nrealwriters; i++) {
  470. if (!writer_durations)
  471. break;
  472. if (!writer_n_durations)
  473. continue;
  474. wdpp = writer_durations[i];
  475. if (!wdpp)
  476. continue;
  477. for (j = 0; j <= writer_n_durations[i]; j++) {
  478. wdp = &wdpp[j];
  479. pr_alert("%s%s %4d writer-duration: %5d %llu\n",
  480. scale_type, SCALE_FLAG,
  481. i, j, *wdp);
  482. if (j % 100 == 0)
  483. schedule_timeout_uninterruptible(1);
  484. }
  485. kfree(writer_durations[i]);
  486. }
  487. kfree(writer_tasks);
  488. kfree(writer_durations);
  489. kfree(writer_n_durations);
  490. }
  491. /* Do torture-type-specific cleanup operations. */
  492. if (cur_ops->cleanup != NULL)
  493. cur_ops->cleanup();
  494. torture_cleanup_end();
  495. }
  496. /*
  497. * Return the number if non-negative. If -1, the number of CPUs.
  498. * If less than -1, that much less than the number of CPUs, but
  499. * at least one.
  500. */
  501. static int compute_real(int n)
  502. {
  503. int nr;
  504. if (n >= 0) {
  505. nr = n;
  506. } else {
  507. nr = num_online_cpus() + 1 + n;
  508. if (nr <= 0)
  509. nr = 1;
  510. }
  511. return nr;
  512. }
  513. /*
  514. * RCU scalability shutdown kthread. Just waits to be awakened, then shuts
  515. * down system.
  516. */
  517. static int
  518. rcu_scale_shutdown(void *arg)
  519. {
  520. wait_event(shutdown_wq,
  521. atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters);
  522. smp_mb(); /* Wake before output. */
  523. rcu_scale_cleanup();
  524. kernel_power_off();
  525. return -EINVAL;
  526. }
  527. /*
  528. * kfree_rcu() scalability tests: Start a kfree_rcu() loop on all CPUs for number
  529. * of iterations and measure total time and number of GP for all iterations to complete.
  530. */
  531. torture_param(int, kfree_nthreads, -1, "Number of threads running loops of kfree_rcu().");
  532. torture_param(int, kfree_alloc_num, 8000, "Number of allocations and frees done in an iteration.");
  533. torture_param(int, kfree_loops, 10, "Number of loops doing kfree_alloc_num allocations and frees.");
  534. static struct task_struct **kfree_reader_tasks;
  535. static int kfree_nrealthreads;
  536. static atomic_t n_kfree_scale_thread_started;
  537. static atomic_t n_kfree_scale_thread_ended;
  538. struct kfree_obj {
  539. char kfree_obj[8];
  540. struct rcu_head rh;
  541. };
  542. static int
  543. kfree_scale_thread(void *arg)
  544. {
  545. int i, loop = 0;
  546. long me = (long)arg;
  547. struct kfree_obj *alloc_ptr;
  548. u64 start_time, end_time;
  549. long long mem_begin, mem_during = 0;
  550. VERBOSE_SCALEOUT_STRING("kfree_scale_thread task started");
  551. set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
  552. set_user_nice(current, MAX_NICE);
  553. start_time = ktime_get_mono_fast_ns();
  554. if (atomic_inc_return(&n_kfree_scale_thread_started) >= kfree_nrealthreads) {
  555. if (gp_exp)
  556. b_rcu_gp_test_started = cur_ops->exp_completed() / 2;
  557. else
  558. b_rcu_gp_test_started = cur_ops->get_gp_seq();
  559. }
  560. do {
  561. if (!mem_during) {
  562. mem_during = mem_begin = si_mem_available();
  563. } else if (loop % (kfree_loops / 4) == 0) {
  564. mem_during = (mem_during + si_mem_available()) / 2;
  565. }
  566. for (i = 0; i < kfree_alloc_num; i++) {
  567. alloc_ptr = kmalloc(kfree_mult * sizeof(struct kfree_obj), GFP_KERNEL);
  568. if (!alloc_ptr)
  569. return -ENOMEM;
  570. kfree_rcu(alloc_ptr, rh);
  571. }
  572. cond_resched();
  573. } while (!torture_must_stop() && ++loop < kfree_loops);
  574. if (atomic_inc_return(&n_kfree_scale_thread_ended) >= kfree_nrealthreads) {
  575. end_time = ktime_get_mono_fast_ns();
  576. if (gp_exp)
  577. b_rcu_gp_test_finished = cur_ops->exp_completed() / 2;
  578. else
  579. b_rcu_gp_test_finished = cur_ops->get_gp_seq();
  580. pr_alert("Total time taken by all kfree'ers: %llu ns, loops: %d, batches: %ld, memory footprint: %lldMB\n",
  581. (unsigned long long)(end_time - start_time), kfree_loops,
  582. rcuscale_seq_diff(b_rcu_gp_test_finished, b_rcu_gp_test_started),
  583. (mem_begin - mem_during) >> (20 - PAGE_SHIFT));
  584. if (shutdown) {
  585. smp_mb(); /* Assign before wake. */
  586. wake_up(&shutdown_wq);
  587. }
  588. }
  589. torture_kthread_stopping("kfree_scale_thread");
  590. return 0;
  591. }
  592. static void
  593. kfree_scale_cleanup(void)
  594. {
  595. int i;
  596. if (torture_cleanup_begin())
  597. return;
  598. if (kfree_reader_tasks) {
  599. for (i = 0; i < kfree_nrealthreads; i++)
  600. torture_stop_kthread(kfree_scale_thread,
  601. kfree_reader_tasks[i]);
  602. kfree(kfree_reader_tasks);
  603. }
  604. torture_cleanup_end();
  605. }
  606. /*
  607. * shutdown kthread. Just waits to be awakened, then shuts down system.
  608. */
  609. static int
  610. kfree_scale_shutdown(void *arg)
  611. {
  612. wait_event(shutdown_wq,
  613. atomic_read(&n_kfree_scale_thread_ended) >= kfree_nrealthreads);
  614. smp_mb(); /* Wake before output. */
  615. kfree_scale_cleanup();
  616. kernel_power_off();
  617. return -EINVAL;
  618. }
  619. static int __init
  620. kfree_scale_init(void)
  621. {
  622. long i;
  623. int firsterr = 0;
  624. kfree_nrealthreads = compute_real(kfree_nthreads);
  625. /* Start up the kthreads. */
  626. if (shutdown) {
  627. init_waitqueue_head(&shutdown_wq);
  628. firsterr = torture_create_kthread(kfree_scale_shutdown, NULL,
  629. shutdown_task);
  630. if (firsterr)
  631. goto unwind;
  632. schedule_timeout_uninterruptible(1);
  633. }
  634. pr_alert("kfree object size=%zu\n", kfree_mult * sizeof(struct kfree_obj));
  635. kfree_reader_tasks = kcalloc(kfree_nrealthreads, sizeof(kfree_reader_tasks[0]),
  636. GFP_KERNEL);
  637. if (kfree_reader_tasks == NULL) {
  638. firsterr = -ENOMEM;
  639. goto unwind;
  640. }
  641. for (i = 0; i < kfree_nrealthreads; i++) {
  642. firsterr = torture_create_kthread(kfree_scale_thread, (void *)i,
  643. kfree_reader_tasks[i]);
  644. if (firsterr)
  645. goto unwind;
  646. }
  647. while (atomic_read(&n_kfree_scale_thread_started) < kfree_nrealthreads)
  648. schedule_timeout_uninterruptible(1);
  649. torture_init_end();
  650. return 0;
  651. unwind:
  652. torture_init_end();
  653. kfree_scale_cleanup();
  654. return firsterr;
  655. }
  656. static int __init
  657. rcu_scale_init(void)
  658. {
  659. long i;
  660. int firsterr = 0;
  661. static struct rcu_scale_ops *scale_ops[] = {
  662. &rcu_ops, &srcu_ops, &srcud_ops, &tasks_ops,
  663. };
  664. if (!torture_init_begin(scale_type, verbose))
  665. return -EBUSY;
  666. /* Process args and announce that the scalability'er is on the job. */
  667. for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
  668. cur_ops = scale_ops[i];
  669. if (strcmp(scale_type, cur_ops->name) == 0)
  670. break;
  671. }
  672. if (i == ARRAY_SIZE(scale_ops)) {
  673. pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
  674. pr_alert("rcu-scale types:");
  675. for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
  676. pr_cont(" %s", scale_ops[i]->name);
  677. pr_cont("\n");
  678. WARN_ON(!IS_MODULE(CONFIG_RCU_SCALE_TEST));
  679. firsterr = -EINVAL;
  680. cur_ops = NULL;
  681. goto unwind;
  682. }
  683. if (cur_ops->init)
  684. cur_ops->init();
  685. if (kfree_rcu_test)
  686. return kfree_scale_init();
  687. nrealwriters = compute_real(nwriters);
  688. nrealreaders = compute_real(nreaders);
  689. atomic_set(&n_rcu_scale_reader_started, 0);
  690. atomic_set(&n_rcu_scale_writer_started, 0);
  691. atomic_set(&n_rcu_scale_writer_finished, 0);
  692. rcu_scale_print_module_parms(cur_ops, "Start of test");
  693. /* Start up the kthreads. */
  694. if (shutdown) {
  695. init_waitqueue_head(&shutdown_wq);
  696. firsterr = torture_create_kthread(rcu_scale_shutdown, NULL,
  697. shutdown_task);
  698. if (firsterr)
  699. goto unwind;
  700. schedule_timeout_uninterruptible(1);
  701. }
  702. reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]),
  703. GFP_KERNEL);
  704. if (reader_tasks == NULL) {
  705. VERBOSE_SCALEOUT_ERRSTRING("out of memory");
  706. firsterr = -ENOMEM;
  707. goto unwind;
  708. }
  709. for (i = 0; i < nrealreaders; i++) {
  710. firsterr = torture_create_kthread(rcu_scale_reader, (void *)i,
  711. reader_tasks[i]);
  712. if (firsterr)
  713. goto unwind;
  714. }
  715. while (atomic_read(&n_rcu_scale_reader_started) < nrealreaders)
  716. schedule_timeout_uninterruptible(1);
  717. writer_tasks = kcalloc(nrealwriters, sizeof(reader_tasks[0]),
  718. GFP_KERNEL);
  719. writer_durations = kcalloc(nrealwriters, sizeof(*writer_durations),
  720. GFP_KERNEL);
  721. writer_n_durations =
  722. kcalloc(nrealwriters, sizeof(*writer_n_durations),
  723. GFP_KERNEL);
  724. if (!writer_tasks || !writer_durations || !writer_n_durations) {
  725. VERBOSE_SCALEOUT_ERRSTRING("out of memory");
  726. firsterr = -ENOMEM;
  727. goto unwind;
  728. }
  729. for (i = 0; i < nrealwriters; i++) {
  730. writer_durations[i] =
  731. kcalloc(MAX_MEAS, sizeof(*writer_durations[i]),
  732. GFP_KERNEL);
  733. if (!writer_durations[i]) {
  734. firsterr = -ENOMEM;
  735. goto unwind;
  736. }
  737. firsterr = torture_create_kthread(rcu_scale_writer, (void *)i,
  738. writer_tasks[i]);
  739. if (firsterr)
  740. goto unwind;
  741. }
  742. torture_init_end();
  743. return 0;
  744. unwind:
  745. torture_init_end();
  746. rcu_scale_cleanup();
  747. return firsterr;
  748. }
  749. module_init(rcu_scale_init);
  750. module_exit(rcu_scale_cleanup);