scftorture.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575
  1. // SPDX-License-Identifier: GPL-2.0+
  2. //
  3. // Torture test for smp_call_function() and friends.
  4. //
  5. // Copyright (C) Facebook, 2020.
  6. //
  7. // Author: Paul E. McKenney <paulmck@kernel.org>
  8. #define pr_fmt(fmt) fmt
  9. #include <linux/atomic.h>
  10. #include <linux/bitops.h>
  11. #include <linux/completion.h>
  12. #include <linux/cpu.h>
  13. #include <linux/delay.h>
  14. #include <linux/err.h>
  15. #include <linux/init.h>
  16. #include <linux/interrupt.h>
  17. #include <linux/kthread.h>
  18. #include <linux/kernel.h>
  19. #include <linux/mm.h>
  20. #include <linux/module.h>
  21. #include <linux/moduleparam.h>
  22. #include <linux/notifier.h>
  23. #include <linux/percpu.h>
  24. #include <linux/rcupdate.h>
  25. #include <linux/rcupdate_trace.h>
  26. #include <linux/reboot.h>
  27. #include <linux/sched.h>
  28. #include <linux/spinlock.h>
  29. #include <linux/smp.h>
  30. #include <linux/stat.h>
  31. #include <linux/srcu.h>
  32. #include <linux/slab.h>
  33. #include <linux/torture.h>
  34. #include <linux/types.h>
  35. #define SCFTORT_STRING "scftorture"
  36. #define SCFTORT_FLAG SCFTORT_STRING ": "
  37. #define SCFTORTOUT(s, x...) \
  38. pr_alert(SCFTORT_FLAG s, ## x)
  39. #define VERBOSE_SCFTORTOUT(s, x...) \
  40. do { if (verbose) pr_alert(SCFTORT_FLAG s, ## x); } while (0)
  41. #define VERBOSE_SCFTORTOUT_ERRSTRING(s, x...) \
  42. do { if (verbose) pr_alert(SCFTORT_FLAG "!!! " s, ## x); } while (0)
  43. MODULE_LICENSE("GPL");
  44. MODULE_AUTHOR("Paul E. McKenney <paulmck@kernel.org>");
  45. // Wait until there are multiple CPUs before starting test.
  46. torture_param(int, holdoff, IS_BUILTIN(CONFIG_SCF_TORTURE_TEST) ? 10 : 0,
  47. "Holdoff time before test start (s)");
  48. torture_param(int, longwait, 0, "Include ridiculously long waits? (seconds)");
  49. torture_param(int, nthreads, -1, "# threads, defaults to -1 for all CPUs.");
  50. torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
  51. torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (s), 0=disable");
  52. torture_param(int, shutdown_secs, 0, "Shutdown time (ms), <= zero to disable.");
  53. torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s.");
  54. torture_param(int, stutter_cpus, 5, "Number of jiffies to change CPUs under test, 0=disable");
  55. torture_param(bool, use_cpus_read_lock, 0, "Use cpus_read_lock() to exclude CPU hotplug.");
  56. torture_param(int, verbose, 0, "Enable verbose debugging printk()s");
  57. torture_param(int, weight_single, -1, "Testing weight for single-CPU no-wait operations.");
  58. torture_param(int, weight_single_wait, -1, "Testing weight for single-CPU operations.");
  59. torture_param(int, weight_many, -1, "Testing weight for multi-CPU no-wait operations.");
  60. torture_param(int, weight_many_wait, -1, "Testing weight for multi-CPU operations.");
  61. torture_param(int, weight_all, -1, "Testing weight for all-CPU no-wait operations.");
  62. torture_param(int, weight_all_wait, -1, "Testing weight for all-CPU operations.");
  63. char *torture_type = "";
  64. #ifdef MODULE
  65. # define SCFTORT_SHUTDOWN 0
  66. #else
  67. # define SCFTORT_SHUTDOWN 1
  68. #endif
  69. torture_param(bool, shutdown, SCFTORT_SHUTDOWN, "Shutdown at end of torture test.");
  70. struct scf_statistics {
  71. struct task_struct *task;
  72. int cpu;
  73. long long n_single;
  74. long long n_single_ofl;
  75. long long n_single_wait;
  76. long long n_single_wait_ofl;
  77. long long n_many;
  78. long long n_many_wait;
  79. long long n_all;
  80. long long n_all_wait;
  81. };
  82. static struct scf_statistics *scf_stats_p;
  83. static struct task_struct *scf_torture_stats_task;
  84. static DEFINE_PER_CPU(long long, scf_invoked_count);
  85. // Data for random primitive selection
  86. #define SCF_PRIM_SINGLE 0
  87. #define SCF_PRIM_MANY 1
  88. #define SCF_PRIM_ALL 2
  89. #define SCF_NPRIMS (2 * 3) // Need wait and no-wait versions of each.
  90. static char *scf_prim_name[] = {
  91. "smp_call_function_single",
  92. "smp_call_function_many",
  93. "smp_call_function",
  94. };
  95. struct scf_selector {
  96. unsigned long scfs_weight;
  97. int scfs_prim;
  98. bool scfs_wait;
  99. };
  100. static struct scf_selector scf_sel_array[SCF_NPRIMS];
  101. static int scf_sel_array_len;
  102. static unsigned long scf_sel_totweight;
  103. // Communicate between caller and handler.
  104. struct scf_check {
  105. bool scfc_in;
  106. bool scfc_out;
  107. int scfc_cpu; // -1 for not _single().
  108. bool scfc_wait;
  109. };
  110. // Use to wait for all threads to start.
  111. static atomic_t n_started;
  112. static atomic_t n_errs;
  113. static atomic_t n_mb_in_errs;
  114. static atomic_t n_mb_out_errs;
  115. static atomic_t n_alloc_errs;
  116. static bool scfdone;
  117. static char *bangstr = "";
  118. static DEFINE_TORTURE_RANDOM_PERCPU(scf_torture_rand);
  119. // Print torture statistics. Caller must ensure serialization.
  120. static void scf_torture_stats_print(void)
  121. {
  122. int cpu;
  123. int i;
  124. long long invoked_count = 0;
  125. bool isdone = READ_ONCE(scfdone);
  126. struct scf_statistics scfs = {};
  127. for_each_possible_cpu(cpu)
  128. invoked_count += data_race(per_cpu(scf_invoked_count, cpu));
  129. for (i = 0; i < nthreads; i++) {
  130. scfs.n_single += scf_stats_p[i].n_single;
  131. scfs.n_single_ofl += scf_stats_p[i].n_single_ofl;
  132. scfs.n_single_wait += scf_stats_p[i].n_single_wait;
  133. scfs.n_single_wait_ofl += scf_stats_p[i].n_single_wait_ofl;
  134. scfs.n_many += scf_stats_p[i].n_many;
  135. scfs.n_many_wait += scf_stats_p[i].n_many_wait;
  136. scfs.n_all += scf_stats_p[i].n_all;
  137. scfs.n_all_wait += scf_stats_p[i].n_all_wait;
  138. }
  139. if (atomic_read(&n_errs) || atomic_read(&n_mb_in_errs) ||
  140. atomic_read(&n_mb_out_errs) || atomic_read(&n_alloc_errs))
  141. bangstr = "!!! ";
  142. pr_alert("%s %sscf_invoked_count %s: %lld single: %lld/%lld single_ofl: %lld/%lld many: %lld/%lld all: %lld/%lld ",
  143. SCFTORT_FLAG, bangstr, isdone ? "VER" : "ver", invoked_count,
  144. scfs.n_single, scfs.n_single_wait, scfs.n_single_ofl, scfs.n_single_wait_ofl,
  145. scfs.n_many, scfs.n_many_wait, scfs.n_all, scfs.n_all_wait);
  146. torture_onoff_stats();
  147. pr_cont("ste: %d stnmie: %d stnmoe: %d staf: %d\n", atomic_read(&n_errs),
  148. atomic_read(&n_mb_in_errs), atomic_read(&n_mb_out_errs),
  149. atomic_read(&n_alloc_errs));
  150. }
  151. // Periodically prints torture statistics, if periodic statistics printing
  152. // was specified via the stat_interval module parameter.
  153. static int
  154. scf_torture_stats(void *arg)
  155. {
  156. VERBOSE_TOROUT_STRING("scf_torture_stats task started");
  157. do {
  158. schedule_timeout_interruptible(stat_interval * HZ);
  159. scf_torture_stats_print();
  160. torture_shutdown_absorb("scf_torture_stats");
  161. } while (!torture_must_stop());
  162. torture_kthread_stopping("scf_torture_stats");
  163. return 0;
  164. }
  165. // Add a primitive to the scf_sel_array[].
  166. static void scf_sel_add(unsigned long weight, int prim, bool wait)
  167. {
  168. struct scf_selector *scfsp = &scf_sel_array[scf_sel_array_len];
  169. // If no weight, if array would overflow, if computing three-place
  170. // percentages would overflow, or if the scf_prim_name[] array would
  171. // overflow, don't bother. In the last three two cases, complain.
  172. if (!weight ||
  173. WARN_ON_ONCE(scf_sel_array_len >= ARRAY_SIZE(scf_sel_array)) ||
  174. WARN_ON_ONCE(0 - 100000 * weight <= 100000 * scf_sel_totweight) ||
  175. WARN_ON_ONCE(prim >= ARRAY_SIZE(scf_prim_name)))
  176. return;
  177. scf_sel_totweight += weight;
  178. scfsp->scfs_weight = scf_sel_totweight;
  179. scfsp->scfs_prim = prim;
  180. scfsp->scfs_wait = wait;
  181. scf_sel_array_len++;
  182. }
  183. // Dump out weighting percentages for scf_prim_name[] array.
  184. static void scf_sel_dump(void)
  185. {
  186. int i;
  187. unsigned long oldw = 0;
  188. struct scf_selector *scfsp;
  189. unsigned long w;
  190. for (i = 0; i < scf_sel_array_len; i++) {
  191. scfsp = &scf_sel_array[i];
  192. w = (scfsp->scfs_weight - oldw) * 100000 / scf_sel_totweight;
  193. pr_info("%s: %3lu.%03lu %s(%s)\n", __func__, w / 1000, w % 1000,
  194. scf_prim_name[scfsp->scfs_prim],
  195. scfsp->scfs_wait ? "wait" : "nowait");
  196. oldw = scfsp->scfs_weight;
  197. }
  198. }
  199. // Randomly pick a primitive and wait/nowait, based on weightings.
  200. static struct scf_selector *scf_sel_rand(struct torture_random_state *trsp)
  201. {
  202. int i;
  203. unsigned long w = torture_random(trsp) % (scf_sel_totweight + 1);
  204. for (i = 0; i < scf_sel_array_len; i++)
  205. if (scf_sel_array[i].scfs_weight >= w)
  206. return &scf_sel_array[i];
  207. WARN_ON_ONCE(1);
  208. return &scf_sel_array[0];
  209. }
  210. // Update statistics and occasionally burn up mass quantities of CPU time,
  211. // if told to do so via scftorture.longwait. Otherwise, occasionally burn
  212. // a little bit.
  213. static void scf_handler(void *scfc_in)
  214. {
  215. int i;
  216. int j;
  217. unsigned long r = torture_random(this_cpu_ptr(&scf_torture_rand));
  218. struct scf_check *scfcp = scfc_in;
  219. if (likely(scfcp)) {
  220. WRITE_ONCE(scfcp->scfc_out, false); // For multiple receivers.
  221. if (WARN_ON_ONCE(unlikely(!READ_ONCE(scfcp->scfc_in))))
  222. atomic_inc(&n_mb_in_errs);
  223. }
  224. this_cpu_inc(scf_invoked_count);
  225. if (longwait <= 0) {
  226. if (!(r & 0xffc0))
  227. udelay(r & 0x3f);
  228. goto out;
  229. }
  230. if (r & 0xfff)
  231. goto out;
  232. r = (r >> 12);
  233. if (longwait <= 0) {
  234. udelay((r & 0xff) + 1);
  235. goto out;
  236. }
  237. r = r % longwait + 1;
  238. for (i = 0; i < r; i++) {
  239. for (j = 0; j < 1000; j++) {
  240. udelay(1000);
  241. cpu_relax();
  242. }
  243. }
  244. out:
  245. if (unlikely(!scfcp))
  246. return;
  247. if (scfcp->scfc_wait)
  248. WRITE_ONCE(scfcp->scfc_out, true);
  249. else
  250. kfree(scfcp);
  251. }
  252. // As above, but check for correct CPU.
  253. static void scf_handler_1(void *scfc_in)
  254. {
  255. struct scf_check *scfcp = scfc_in;
  256. if (likely(scfcp) && WARN_ONCE(smp_processor_id() != scfcp->scfc_cpu, "%s: Wanted CPU %d got CPU %d\n", __func__, scfcp->scfc_cpu, smp_processor_id())) {
  257. atomic_inc(&n_errs);
  258. }
  259. scf_handler(scfcp);
  260. }
  261. // Randomly do an smp_call_function*() invocation.
  262. static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_random_state *trsp)
  263. {
  264. uintptr_t cpu;
  265. int ret = 0;
  266. struct scf_check *scfcp = NULL;
  267. struct scf_selector *scfsp = scf_sel_rand(trsp);
  268. if (use_cpus_read_lock)
  269. cpus_read_lock();
  270. else
  271. preempt_disable();
  272. if (scfsp->scfs_prim == SCF_PRIM_SINGLE || scfsp->scfs_wait) {
  273. scfcp = kmalloc(sizeof(*scfcp), GFP_ATOMIC);
  274. if (WARN_ON_ONCE(!scfcp)) {
  275. atomic_inc(&n_alloc_errs);
  276. } else {
  277. scfcp->scfc_cpu = -1;
  278. scfcp->scfc_wait = scfsp->scfs_wait;
  279. scfcp->scfc_out = false;
  280. }
  281. }
  282. switch (scfsp->scfs_prim) {
  283. case SCF_PRIM_SINGLE:
  284. cpu = torture_random(trsp) % nr_cpu_ids;
  285. if (scfsp->scfs_wait)
  286. scfp->n_single_wait++;
  287. else
  288. scfp->n_single++;
  289. if (scfcp) {
  290. scfcp->scfc_cpu = cpu;
  291. barrier(); // Prevent race-reduction compiler optimizations.
  292. scfcp->scfc_in = true;
  293. }
  294. ret = smp_call_function_single(cpu, scf_handler_1, (void *)scfcp, scfsp->scfs_wait);
  295. if (ret) {
  296. if (scfsp->scfs_wait)
  297. scfp->n_single_wait_ofl++;
  298. else
  299. scfp->n_single_ofl++;
  300. kfree(scfcp);
  301. scfcp = NULL;
  302. }
  303. break;
  304. case SCF_PRIM_MANY:
  305. if (scfsp->scfs_wait)
  306. scfp->n_many_wait++;
  307. else
  308. scfp->n_many++;
  309. if (scfcp) {
  310. barrier(); // Prevent race-reduction compiler optimizations.
  311. scfcp->scfc_in = true;
  312. }
  313. smp_call_function_many(cpu_online_mask, scf_handler, scfcp, scfsp->scfs_wait);
  314. break;
  315. case SCF_PRIM_ALL:
  316. if (scfsp->scfs_wait)
  317. scfp->n_all_wait++;
  318. else
  319. scfp->n_all++;
  320. if (scfcp) {
  321. barrier(); // Prevent race-reduction compiler optimizations.
  322. scfcp->scfc_in = true;
  323. }
  324. smp_call_function(scf_handler, scfcp, scfsp->scfs_wait);
  325. break;
  326. default:
  327. WARN_ON_ONCE(1);
  328. if (scfcp)
  329. scfcp->scfc_out = true;
  330. }
  331. if (scfcp && scfsp->scfs_wait) {
  332. if (WARN_ON_ONCE((num_online_cpus() > 1 || scfsp->scfs_prim == SCF_PRIM_SINGLE) &&
  333. !scfcp->scfc_out))
  334. atomic_inc(&n_mb_out_errs); // Leak rather than trash!
  335. else
  336. kfree(scfcp);
  337. barrier(); // Prevent race-reduction compiler optimizations.
  338. }
  339. if (use_cpus_read_lock)
  340. cpus_read_unlock();
  341. else
  342. preempt_enable();
  343. if (!(torture_random(trsp) & 0xfff))
  344. schedule_timeout_uninterruptible(1);
  345. }
  346. // SCF test kthread. Repeatedly does calls to members of the
  347. // smp_call_function() family of functions.
  348. static int scftorture_invoker(void *arg)
  349. {
  350. int cpu;
  351. DEFINE_TORTURE_RANDOM(rand);
  352. struct scf_statistics *scfp = (struct scf_statistics *)arg;
  353. bool was_offline = false;
  354. VERBOSE_SCFTORTOUT("scftorture_invoker %d: task started", scfp->cpu);
  355. cpu = scfp->cpu % nr_cpu_ids;
  356. set_cpus_allowed_ptr(current, cpumask_of(cpu));
  357. set_user_nice(current, MAX_NICE);
  358. if (holdoff)
  359. schedule_timeout_interruptible(holdoff * HZ);
  360. VERBOSE_SCFTORTOUT("scftorture_invoker %d: Waiting for all SCF torturers from cpu %d", scfp->cpu, smp_processor_id());
  361. // Make sure that the CPU is affinitized appropriately during testing.
  362. WARN_ON_ONCE(smp_processor_id() != scfp->cpu);
  363. if (!atomic_dec_return(&n_started))
  364. while (atomic_read_acquire(&n_started)) {
  365. if (torture_must_stop()) {
  366. VERBOSE_SCFTORTOUT("scftorture_invoker %d ended before starting", scfp->cpu);
  367. goto end;
  368. }
  369. schedule_timeout_uninterruptible(1);
  370. }
  371. VERBOSE_SCFTORTOUT("scftorture_invoker %d started", scfp->cpu);
  372. do {
  373. scftorture_invoke_one(scfp, &rand);
  374. while (cpu_is_offline(cpu) && !torture_must_stop()) {
  375. schedule_timeout_interruptible(HZ / 5);
  376. was_offline = true;
  377. }
  378. if (was_offline) {
  379. set_cpus_allowed_ptr(current, cpumask_of(cpu));
  380. was_offline = false;
  381. }
  382. cond_resched();
  383. } while (!torture_must_stop());
  384. VERBOSE_SCFTORTOUT("scftorture_invoker %d ended", scfp->cpu);
  385. end:
  386. torture_kthread_stopping("scftorture_invoker");
  387. return 0;
  388. }
  389. static void
  390. scftorture_print_module_parms(const char *tag)
  391. {
  392. pr_alert(SCFTORT_FLAG
  393. "--- %s: verbose=%d holdoff=%d longwait=%d nthreads=%d onoff_holdoff=%d onoff_interval=%d shutdown_secs=%d stat_interval=%d stutter_cpus=%d use_cpus_read_lock=%d, weight_single=%d, weight_single_wait=%d, weight_many=%d, weight_many_wait=%d, weight_all=%d, weight_all_wait=%d\n", tag,
  394. verbose, holdoff, longwait, nthreads, onoff_holdoff, onoff_interval, shutdown, stat_interval, stutter_cpus, use_cpus_read_lock, weight_single, weight_single_wait, weight_many, weight_many_wait, weight_all, weight_all_wait);
  395. }
  396. static void scf_cleanup_handler(void *unused)
  397. {
  398. }
  399. static void scf_torture_cleanup(void)
  400. {
  401. int i;
  402. if (torture_cleanup_begin())
  403. return;
  404. WRITE_ONCE(scfdone, true);
  405. if (nthreads)
  406. for (i = 0; i < nthreads; i++)
  407. torture_stop_kthread("scftorture_invoker", scf_stats_p[i].task);
  408. else
  409. goto end;
  410. smp_call_function(scf_cleanup_handler, NULL, 0);
  411. torture_stop_kthread(scf_torture_stats, scf_torture_stats_task);
  412. scf_torture_stats_print(); // -After- the stats thread is stopped!
  413. kfree(scf_stats_p); // -After- the last stats print has completed!
  414. scf_stats_p = NULL;
  415. if (atomic_read(&n_errs) || atomic_read(&n_mb_in_errs) || atomic_read(&n_mb_out_errs))
  416. scftorture_print_module_parms("End of test: FAILURE");
  417. else if (torture_onoff_failures())
  418. scftorture_print_module_parms("End of test: LOCK_HOTPLUG");
  419. else
  420. scftorture_print_module_parms("End of test: SUCCESS");
  421. end:
  422. torture_cleanup_end();
  423. }
  424. static int __init scf_torture_init(void)
  425. {
  426. long i;
  427. int firsterr = 0;
  428. unsigned long weight_single1 = weight_single;
  429. unsigned long weight_single_wait1 = weight_single_wait;
  430. unsigned long weight_many1 = weight_many;
  431. unsigned long weight_many_wait1 = weight_many_wait;
  432. unsigned long weight_all1 = weight_all;
  433. unsigned long weight_all_wait1 = weight_all_wait;
  434. if (!torture_init_begin(SCFTORT_STRING, verbose))
  435. return -EBUSY;
  436. scftorture_print_module_parms("Start of test");
  437. if (weight_single == -1 && weight_single_wait == -1 &&
  438. weight_many == -1 && weight_many_wait == -1 &&
  439. weight_all == -1 && weight_all_wait == -1) {
  440. weight_single1 = 2 * nr_cpu_ids;
  441. weight_single_wait1 = 2 * nr_cpu_ids;
  442. weight_many1 = 2;
  443. weight_many_wait1 = 2;
  444. weight_all1 = 1;
  445. weight_all_wait1 = 1;
  446. } else {
  447. if (weight_single == -1)
  448. weight_single1 = 0;
  449. if (weight_single_wait == -1)
  450. weight_single_wait1 = 0;
  451. if (weight_many == -1)
  452. weight_many1 = 0;
  453. if (weight_many_wait == -1)
  454. weight_many_wait1 = 0;
  455. if (weight_all == -1)
  456. weight_all1 = 0;
  457. if (weight_all_wait == -1)
  458. weight_all_wait1 = 0;
  459. }
  460. if (weight_single1 == 0 && weight_single_wait1 == 0 &&
  461. weight_many1 == 0 && weight_many_wait1 == 0 &&
  462. weight_all1 == 0 && weight_all_wait1 == 0) {
  463. VERBOSE_SCFTORTOUT_ERRSTRING("all zero weights makes no sense");
  464. firsterr = -EINVAL;
  465. goto unwind;
  466. }
  467. scf_sel_add(weight_single1, SCF_PRIM_SINGLE, false);
  468. scf_sel_add(weight_single_wait1, SCF_PRIM_SINGLE, true);
  469. scf_sel_add(weight_many1, SCF_PRIM_MANY, false);
  470. scf_sel_add(weight_many_wait1, SCF_PRIM_MANY, true);
  471. scf_sel_add(weight_all1, SCF_PRIM_ALL, false);
  472. scf_sel_add(weight_all_wait1, SCF_PRIM_ALL, true);
  473. scf_sel_dump();
  474. if (onoff_interval > 0) {
  475. firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, NULL);
  476. if (firsterr)
  477. goto unwind;
  478. }
  479. if (shutdown_secs > 0) {
  480. firsterr = torture_shutdown_init(shutdown_secs, scf_torture_cleanup);
  481. if (firsterr)
  482. goto unwind;
  483. }
  484. // Worker tasks invoking smp_call_function().
  485. if (nthreads < 0)
  486. nthreads = num_online_cpus();
  487. scf_stats_p = kcalloc(nthreads, sizeof(scf_stats_p[0]), GFP_KERNEL);
  488. if (!scf_stats_p) {
  489. VERBOSE_SCFTORTOUT_ERRSTRING("out of memory");
  490. firsterr = -ENOMEM;
  491. goto unwind;
  492. }
  493. VERBOSE_SCFTORTOUT("Starting %d smp_call_function() threads\n", nthreads);
  494. atomic_set(&n_started, nthreads);
  495. for (i = 0; i < nthreads; i++) {
  496. scf_stats_p[i].cpu = i;
  497. firsterr = torture_create_kthread(scftorture_invoker, (void *)&scf_stats_p[i],
  498. scf_stats_p[i].task);
  499. if (firsterr)
  500. goto unwind;
  501. }
  502. if (stat_interval > 0) {
  503. firsterr = torture_create_kthread(scf_torture_stats, NULL, scf_torture_stats_task);
  504. if (firsterr)
  505. goto unwind;
  506. }
  507. torture_init_end();
  508. return 0;
  509. unwind:
  510. torture_init_end();
  511. scf_torture_cleanup();
  512. return firsterr;
  513. }
  514. module_init(scf_torture_init);
  515. module_exit(scf_torture_cleanup);