refscale.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719
  1. // SPDX-License-Identifier: GPL-2.0+
  2. //
  3. // Scalability test comparing RCU vs other mechanisms
  4. // for acquiring references on objects.
  5. //
  6. // Copyright (C) Google, 2020.
  7. //
  8. // Author: Joel Fernandes <joel@joelfernandes.org>
  9. #define pr_fmt(fmt) fmt
  10. #include <linux/atomic.h>
  11. #include <linux/bitops.h>
  12. #include <linux/completion.h>
  13. #include <linux/cpu.h>
  14. #include <linux/delay.h>
  15. #include <linux/err.h>
  16. #include <linux/init.h>
  17. #include <linux/interrupt.h>
  18. #include <linux/kthread.h>
  19. #include <linux/kernel.h>
  20. #include <linux/mm.h>
  21. #include <linux/module.h>
  22. #include <linux/moduleparam.h>
  23. #include <linux/notifier.h>
  24. #include <linux/percpu.h>
  25. #include <linux/rcupdate.h>
  26. #include <linux/rcupdate_trace.h>
  27. #include <linux/reboot.h>
  28. #include <linux/sched.h>
  29. #include <linux/spinlock.h>
  30. #include <linux/smp.h>
  31. #include <linux/stat.h>
  32. #include <linux/srcu.h>
  33. #include <linux/slab.h>
  34. #include <linux/torture.h>
  35. #include <linux/types.h>
  36. #include "rcu.h"
  37. #define SCALE_FLAG "-ref-scale: "
  38. #define SCALEOUT(s, x...) \
  39. pr_alert("%s" SCALE_FLAG s, scale_type, ## x)
  40. #define VERBOSE_SCALEOUT(s, x...) \
  41. do { if (verbose) pr_alert("%s" SCALE_FLAG s, scale_type, ## x); } while (0)
  42. #define VERBOSE_SCALEOUT_ERRSTRING(s, x...) \
  43. do { if (verbose) pr_alert("%s" SCALE_FLAG "!!! " s, scale_type, ## x); } while (0)
  44. MODULE_LICENSE("GPL");
  45. MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");
  46. static char *scale_type = "rcu";
  47. module_param(scale_type, charp, 0444);
  48. MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock.");
  49. torture_param(int, verbose, 0, "Enable verbose debugging printk()s");
  50. // Wait until there are multiple CPUs before starting test.
  51. torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
  52. "Holdoff time before test start (s)");
  53. // Number of loops per experiment, all readers execute operations concurrently.
  54. torture_param(long, loops, 10000, "Number of loops per experiment.");
  55. // Number of readers, with -1 defaulting to about 75% of the CPUs.
  56. torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
  57. // Number of runs.
  58. torture_param(int, nruns, 30, "Number of experiments to run.");
  59. // Reader delay in nanoseconds, 0 for no delay.
  60. torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
  61. #ifdef MODULE
  62. # define REFSCALE_SHUTDOWN 0
  63. #else
  64. # define REFSCALE_SHUTDOWN 1
  65. #endif
  66. torture_param(bool, shutdown, REFSCALE_SHUTDOWN,
  67. "Shutdown at end of scalability tests.");
  68. struct reader_task {
  69. struct task_struct *task;
  70. int start_reader;
  71. wait_queue_head_t wq;
  72. u64 last_duration_ns;
  73. };
  74. static struct task_struct *shutdown_task;
  75. static wait_queue_head_t shutdown_wq;
  76. static struct task_struct *main_task;
  77. static wait_queue_head_t main_wq;
  78. static int shutdown_start;
  79. static struct reader_task *reader_tasks;
  80. // Number of readers that are part of the current experiment.
  81. static atomic_t nreaders_exp;
  82. // Use to wait for all threads to start.
  83. static atomic_t n_init;
  84. static atomic_t n_started;
  85. static atomic_t n_warmedup;
  86. static atomic_t n_cooleddown;
  87. // Track which experiment is currently running.
  88. static int exp_idx;
  89. // Operations vector for selecting different types of tests.
  90. struct ref_scale_ops {
  91. void (*init)(void);
  92. void (*cleanup)(void);
  93. void (*readsection)(const int nloops);
  94. void (*delaysection)(const int nloops, const int udl, const int ndl);
  95. const char *name;
  96. };
  97. static struct ref_scale_ops *cur_ops;
  98. static void un_delay(const int udl, const int ndl)
  99. {
  100. if (udl)
  101. udelay(udl);
  102. if (ndl)
  103. ndelay(ndl);
  104. }
  105. static void ref_rcu_read_section(const int nloops)
  106. {
  107. int i;
  108. for (i = nloops; i >= 0; i--) {
  109. rcu_read_lock();
  110. rcu_read_unlock();
  111. }
  112. }
  113. static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl)
  114. {
  115. int i;
  116. for (i = nloops; i >= 0; i--) {
  117. rcu_read_lock();
  118. un_delay(udl, ndl);
  119. rcu_read_unlock();
  120. }
  121. }
  122. static void rcu_sync_scale_init(void)
  123. {
  124. }
  125. static struct ref_scale_ops rcu_ops = {
  126. .init = rcu_sync_scale_init,
  127. .readsection = ref_rcu_read_section,
  128. .delaysection = ref_rcu_delay_section,
  129. .name = "rcu"
  130. };
  131. // Definitions for SRCU ref scale testing.
  132. DEFINE_STATIC_SRCU(srcu_refctl_scale);
  133. static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale;
  134. static void srcu_ref_scale_read_section(const int nloops)
  135. {
  136. int i;
  137. int idx;
  138. for (i = nloops; i >= 0; i--) {
  139. idx = srcu_read_lock(srcu_ctlp);
  140. srcu_read_unlock(srcu_ctlp, idx);
  141. }
  142. }
  143. static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
  144. {
  145. int i;
  146. int idx;
  147. for (i = nloops; i >= 0; i--) {
  148. idx = srcu_read_lock(srcu_ctlp);
  149. un_delay(udl, ndl);
  150. srcu_read_unlock(srcu_ctlp, idx);
  151. }
  152. }
  153. static struct ref_scale_ops srcu_ops = {
  154. .init = rcu_sync_scale_init,
  155. .readsection = srcu_ref_scale_read_section,
  156. .delaysection = srcu_ref_scale_delay_section,
  157. .name = "srcu"
  158. };
  159. // Definitions for RCU Tasks ref scale testing: Empty read markers.
  160. // These definitions also work for RCU Rude readers.
  161. static void rcu_tasks_ref_scale_read_section(const int nloops)
  162. {
  163. int i;
  164. for (i = nloops; i >= 0; i--)
  165. continue;
  166. }
  167. static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
  168. {
  169. int i;
  170. for (i = nloops; i >= 0; i--)
  171. un_delay(udl, ndl);
  172. }
  173. static struct ref_scale_ops rcu_tasks_ops = {
  174. .init = rcu_sync_scale_init,
  175. .readsection = rcu_tasks_ref_scale_read_section,
  176. .delaysection = rcu_tasks_ref_scale_delay_section,
  177. .name = "rcu-tasks"
  178. };
  179. // Definitions for RCU Tasks Trace ref scale testing.
  180. static void rcu_trace_ref_scale_read_section(const int nloops)
  181. {
  182. int i;
  183. for (i = nloops; i >= 0; i--) {
  184. rcu_read_lock_trace();
  185. rcu_read_unlock_trace();
  186. }
  187. }
  188. static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
  189. {
  190. int i;
  191. for (i = nloops; i >= 0; i--) {
  192. rcu_read_lock_trace();
  193. un_delay(udl, ndl);
  194. rcu_read_unlock_trace();
  195. }
  196. }
  197. static struct ref_scale_ops rcu_trace_ops = {
  198. .init = rcu_sync_scale_init,
  199. .readsection = rcu_trace_ref_scale_read_section,
  200. .delaysection = rcu_trace_ref_scale_delay_section,
  201. .name = "rcu-trace"
  202. };
  203. // Definitions for reference count
  204. static atomic_t refcnt;
  205. static void ref_refcnt_section(const int nloops)
  206. {
  207. int i;
  208. for (i = nloops; i >= 0; i--) {
  209. atomic_inc(&refcnt);
  210. atomic_dec(&refcnt);
  211. }
  212. }
  213. static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl)
  214. {
  215. int i;
  216. for (i = nloops; i >= 0; i--) {
  217. atomic_inc(&refcnt);
  218. un_delay(udl, ndl);
  219. atomic_dec(&refcnt);
  220. }
  221. }
  222. static struct ref_scale_ops refcnt_ops = {
  223. .init = rcu_sync_scale_init,
  224. .readsection = ref_refcnt_section,
  225. .delaysection = ref_refcnt_delay_section,
  226. .name = "refcnt"
  227. };
  228. // Definitions for rwlock
  229. static rwlock_t test_rwlock;
  230. static void ref_rwlock_init(void)
  231. {
  232. rwlock_init(&test_rwlock);
  233. }
  234. static void ref_rwlock_section(const int nloops)
  235. {
  236. int i;
  237. for (i = nloops; i >= 0; i--) {
  238. read_lock(&test_rwlock);
  239. read_unlock(&test_rwlock);
  240. }
  241. }
  242. static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl)
  243. {
  244. int i;
  245. for (i = nloops; i >= 0; i--) {
  246. read_lock(&test_rwlock);
  247. un_delay(udl, ndl);
  248. read_unlock(&test_rwlock);
  249. }
  250. }
  251. static struct ref_scale_ops rwlock_ops = {
  252. .init = ref_rwlock_init,
  253. .readsection = ref_rwlock_section,
  254. .delaysection = ref_rwlock_delay_section,
  255. .name = "rwlock"
  256. };
  257. // Definitions for rwsem
  258. static struct rw_semaphore test_rwsem;
  259. static void ref_rwsem_init(void)
  260. {
  261. init_rwsem(&test_rwsem);
  262. }
  263. static void ref_rwsem_section(const int nloops)
  264. {
  265. int i;
  266. for (i = nloops; i >= 0; i--) {
  267. down_read(&test_rwsem);
  268. up_read(&test_rwsem);
  269. }
  270. }
  271. static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl)
  272. {
  273. int i;
  274. for (i = nloops; i >= 0; i--) {
  275. down_read(&test_rwsem);
  276. un_delay(udl, ndl);
  277. up_read(&test_rwsem);
  278. }
  279. }
  280. static struct ref_scale_ops rwsem_ops = {
  281. .init = ref_rwsem_init,
  282. .readsection = ref_rwsem_section,
  283. .delaysection = ref_rwsem_delay_section,
  284. .name = "rwsem"
  285. };
  286. static void rcu_scale_one_reader(void)
  287. {
  288. if (readdelay <= 0)
  289. cur_ops->readsection(loops);
  290. else
  291. cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);
  292. }
  293. // Reader kthread. Repeatedly does empty RCU read-side
  294. // critical section, minimizing update-side interference.
  295. static int
  296. ref_scale_reader(void *arg)
  297. {
  298. unsigned long flags;
  299. long me = (long)arg;
  300. struct reader_task *rt = &(reader_tasks[me]);
  301. u64 start;
  302. s64 duration;
  303. VERBOSE_SCALEOUT("ref_scale_reader %ld: task started", me);
  304. set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
  305. set_user_nice(current, MAX_NICE);
  306. atomic_inc(&n_init);
  307. if (holdoff)
  308. schedule_timeout_interruptible(holdoff * HZ);
  309. repeat:
  310. VERBOSE_SCALEOUT("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, smp_processor_id());
  311. // Wait for signal that this reader can start.
  312. wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||
  313. torture_must_stop());
  314. if (torture_must_stop())
  315. goto end;
  316. // Make sure that the CPU is affinitized appropriately during testing.
  317. WARN_ON_ONCE(smp_processor_id() != me);
  318. WRITE_ONCE(rt->start_reader, 0);
  319. if (!atomic_dec_return(&n_started))
  320. while (atomic_read_acquire(&n_started))
  321. cpu_relax();
  322. VERBOSE_SCALEOUT("ref_scale_reader %ld: experiment %d started", me, exp_idx);
  323. // To reduce noise, do an initial cache-warming invocation, check
  324. // in, and then keep warming until everyone has checked in.
  325. rcu_scale_one_reader();
  326. if (!atomic_dec_return(&n_warmedup))
  327. while (atomic_read_acquire(&n_warmedup))
  328. rcu_scale_one_reader();
  329. // Also keep interrupts disabled. This also has the effect
  330. // of preventing entries into slow path for rcu_read_unlock().
  331. local_irq_save(flags);
  332. start = ktime_get_mono_fast_ns();
  333. rcu_scale_one_reader();
  334. duration = ktime_get_mono_fast_ns() - start;
  335. local_irq_restore(flags);
  336. rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;
  337. // To reduce runtime-skew noise, do maintain-load invocations until
  338. // everyone is done.
  339. if (!atomic_dec_return(&n_cooleddown))
  340. while (atomic_read_acquire(&n_cooleddown))
  341. rcu_scale_one_reader();
  342. if (atomic_dec_and_test(&nreaders_exp))
  343. wake_up(&main_wq);
  344. VERBOSE_SCALEOUT("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)",
  345. me, exp_idx, atomic_read(&nreaders_exp));
  346. if (!torture_must_stop())
  347. goto repeat;
  348. end:
  349. torture_kthread_stopping("ref_scale_reader");
  350. return 0;
  351. }
  352. static void reset_readers(void)
  353. {
  354. int i;
  355. struct reader_task *rt;
  356. for (i = 0; i < nreaders; i++) {
  357. rt = &(reader_tasks[i]);
  358. rt->last_duration_ns = 0;
  359. }
  360. }
  361. // Print the results of each reader and return the sum of all their durations.
  362. static u64 process_durations(int n)
  363. {
  364. int i;
  365. struct reader_task *rt;
  366. char buf1[64];
  367. char *buf;
  368. u64 sum = 0;
  369. buf = kmalloc(128 + nreaders * 32, GFP_KERNEL);
  370. if (!buf)
  371. return 0;
  372. buf[0] = 0;
  373. sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",
  374. exp_idx);
  375. for (i = 0; i < n && !torture_must_stop(); i++) {
  376. rt = &(reader_tasks[i]);
  377. sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns);
  378. if (i % 5 == 0)
  379. strcat(buf, "\n");
  380. strcat(buf, buf1);
  381. sum += rt->last_duration_ns;
  382. }
  383. strcat(buf, "\n");
  384. SCALEOUT("%s\n", buf);
  385. kfree(buf);
  386. return sum;
  387. }
  388. // The main_func is the main orchestrator, it performs a bunch of
  389. // experiments. For every experiment, it orders all the readers
  390. // involved to start and waits for them to finish the experiment. It
  391. // then reads their timestamps and starts the next experiment. Each
  392. // experiment progresses from 1 concurrent reader to N of them at which
  393. // point all the timestamps are printed.
  394. static int main_func(void *arg)
  395. {
  396. bool errexit = false;
  397. int exp, r;
  398. char buf1[64];
  399. char *buf;
  400. u64 *result_avg;
  401. set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids));
  402. set_user_nice(current, MAX_NICE);
  403. VERBOSE_SCALEOUT("main_func task started");
  404. result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL);
  405. buf = kzalloc(64 + nruns * 32, GFP_KERNEL);
  406. if (!result_avg || !buf) {
  407. VERBOSE_SCALEOUT_ERRSTRING("out of memory");
  408. errexit = true;
  409. }
  410. if (holdoff)
  411. schedule_timeout_interruptible(holdoff * HZ);
  412. // Wait for all threads to start.
  413. atomic_inc(&n_init);
  414. while (atomic_read(&n_init) < nreaders + 1)
  415. schedule_timeout_uninterruptible(1);
  416. // Start exp readers up per experiment
  417. for (exp = 0; exp < nruns && !torture_must_stop(); exp++) {
  418. if (errexit)
  419. break;
  420. if (torture_must_stop())
  421. goto end;
  422. reset_readers();
  423. atomic_set(&nreaders_exp, nreaders);
  424. atomic_set(&n_started, nreaders);
  425. atomic_set(&n_warmedup, nreaders);
  426. atomic_set(&n_cooleddown, nreaders);
  427. exp_idx = exp;
  428. for (r = 0; r < nreaders; r++) {
  429. smp_store_release(&reader_tasks[r].start_reader, 1);
  430. wake_up(&reader_tasks[r].wq);
  431. }
  432. VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers",
  433. nreaders);
  434. wait_event(main_wq,
  435. !atomic_read(&nreaders_exp) || torture_must_stop());
  436. VERBOSE_SCALEOUT("main_func: experiment ended");
  437. if (torture_must_stop())
  438. goto end;
  439. result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops);
  440. }
  441. // Print the average of all experiments
  442. SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");
  443. if (!errexit) {
  444. buf[0] = 0;
  445. strcat(buf, "\n");
  446. strcat(buf, "Runs\tTime(ns)\n");
  447. }
  448. for (exp = 0; exp < nruns; exp++) {
  449. u64 avg;
  450. u32 rem;
  451. if (errexit)
  452. break;
  453. avg = div_u64_rem(result_avg[exp], 1000, &rem);
  454. sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem);
  455. strcat(buf, buf1);
  456. }
  457. if (!errexit)
  458. SCALEOUT("%s", buf);
  459. // This will shutdown everything including us.
  460. if (shutdown) {
  461. shutdown_start = 1;
  462. wake_up(&shutdown_wq);
  463. }
  464. // Wait for torture to stop us
  465. while (!torture_must_stop())
  466. schedule_timeout_uninterruptible(1);
  467. end:
  468. torture_kthread_stopping("main_func");
  469. kfree(result_avg);
  470. kfree(buf);
  471. return 0;
  472. }
  473. static void
  474. ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag)
  475. {
  476. pr_alert("%s" SCALE_FLAG
  477. "--- %s: verbose=%d shutdown=%d holdoff=%d loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag,
  478. verbose, shutdown, holdoff, loops, nreaders, nruns, readdelay);
  479. }
  480. static void
  481. ref_scale_cleanup(void)
  482. {
  483. int i;
  484. if (torture_cleanup_begin())
  485. return;
  486. if (!cur_ops) {
  487. torture_cleanup_end();
  488. return;
  489. }
  490. if (reader_tasks) {
  491. for (i = 0; i < nreaders; i++)
  492. torture_stop_kthread("ref_scale_reader",
  493. reader_tasks[i].task);
  494. }
  495. kfree(reader_tasks);
  496. torture_stop_kthread("main_task", main_task);
  497. kfree(main_task);
  498. // Do scale-type-specific cleanup operations.
  499. if (cur_ops->cleanup != NULL)
  500. cur_ops->cleanup();
  501. torture_cleanup_end();
  502. }
  503. // Shutdown kthread. Just waits to be awakened, then shuts down system.
  504. static int
  505. ref_scale_shutdown(void *arg)
  506. {
  507. wait_event(shutdown_wq, shutdown_start);
  508. smp_mb(); // Wake before output.
  509. ref_scale_cleanup();
  510. kernel_power_off();
  511. return -EINVAL;
  512. }
  513. static int __init
  514. ref_scale_init(void)
  515. {
  516. long i;
  517. int firsterr = 0;
  518. static struct ref_scale_ops *scale_ops[] = {
  519. &rcu_ops, &srcu_ops, &rcu_trace_ops, &rcu_tasks_ops,
  520. &refcnt_ops, &rwlock_ops, &rwsem_ops,
  521. };
  522. if (!torture_init_begin(scale_type, verbose))
  523. return -EBUSY;
  524. for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
  525. cur_ops = scale_ops[i];
  526. if (strcmp(scale_type, cur_ops->name) == 0)
  527. break;
  528. }
  529. if (i == ARRAY_SIZE(scale_ops)) {
  530. pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
  531. pr_alert("rcu-scale types:");
  532. for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
  533. pr_cont(" %s", scale_ops[i]->name);
  534. pr_cont("\n");
  535. WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST));
  536. firsterr = -EINVAL;
  537. cur_ops = NULL;
  538. goto unwind;
  539. }
  540. if (cur_ops->init)
  541. cur_ops->init();
  542. ref_scale_print_module_parms(cur_ops, "Start of test");
  543. // Shutdown task
  544. if (shutdown) {
  545. init_waitqueue_head(&shutdown_wq);
  546. firsterr = torture_create_kthread(ref_scale_shutdown, NULL,
  547. shutdown_task);
  548. if (firsterr)
  549. goto unwind;
  550. schedule_timeout_uninterruptible(1);
  551. }
  552. // Reader tasks (default to ~75% of online CPUs).
  553. if (nreaders < 0)
  554. nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2);
  555. reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]),
  556. GFP_KERNEL);
  557. if (!reader_tasks) {
  558. VERBOSE_SCALEOUT_ERRSTRING("out of memory");
  559. firsterr = -ENOMEM;
  560. goto unwind;
  561. }
  562. VERBOSE_SCALEOUT("Starting %d reader threads\n", nreaders);
  563. for (i = 0; i < nreaders; i++) {
  564. firsterr = torture_create_kthread(ref_scale_reader, (void *)i,
  565. reader_tasks[i].task);
  566. if (firsterr)
  567. goto unwind;
  568. init_waitqueue_head(&(reader_tasks[i].wq));
  569. }
  570. // Main Task
  571. init_waitqueue_head(&main_wq);
  572. firsterr = torture_create_kthread(main_func, NULL, main_task);
  573. if (firsterr)
  574. goto unwind;
  575. torture_init_end();
  576. return 0;
  577. unwind:
  578. torture_init_end();
  579. ref_scale_cleanup();
  580. return firsterr;
  581. }
  582. module_init(ref_scale_init);
  583. module_exit(ref_scale_cleanup);