test_vmalloc.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Test module for stress and analyze performance of vmalloc allocator.
  4. * (C) 2018 Uladzislau Rezki (Sony) <urezki@gmail.com>
  5. */
  6. #include <linux/init.h>
  7. #include <linux/kernel.h>
  8. #include <linux/module.h>
  9. #include <linux/vmalloc.h>
  10. #include <linux/random.h>
  11. #include <linux/kthread.h>
  12. #include <linux/moduleparam.h>
  13. #include <linux/completion.h>
  14. #include <linux/delay.h>
  15. #include <linux/rwsem.h>
  16. #include <linux/mm.h>
  17. #include <linux/rcupdate.h>
  18. #include <linux/slab.h>
  19. #define __param(type, name, init, msg) \
  20. static type name = init; \
  21. module_param(name, type, 0444); \
  22. MODULE_PARM_DESC(name, msg) \
  23. __param(bool, single_cpu_test, false,
  24. "Use single first online CPU to run tests");
  25. __param(bool, sequential_test_order, false,
  26. "Use sequential stress tests order");
  27. __param(int, test_repeat_count, 1,
  28. "Set test repeat counter");
  29. __param(int, test_loop_count, 1000000,
  30. "Set test loop counter");
  31. __param(int, run_test_mask, INT_MAX,
  32. "Set tests specified in the mask.\n\n"
  33. "\t\tid: 1, name: fix_size_alloc_test\n"
  34. "\t\tid: 2, name: full_fit_alloc_test\n"
  35. "\t\tid: 4, name: long_busy_list_alloc_test\n"
  36. "\t\tid: 8, name: random_size_alloc_test\n"
  37. "\t\tid: 16, name: fix_align_alloc_test\n"
  38. "\t\tid: 32, name: random_size_align_alloc_test\n"
  39. "\t\tid: 64, name: align_shift_alloc_test\n"
  40. "\t\tid: 128, name: pcpu_alloc_test\n"
  41. "\t\tid: 256, name: kvfree_rcu_1_arg_vmalloc_test\n"
  42. "\t\tid: 512, name: kvfree_rcu_2_arg_vmalloc_test\n"
  43. "\t\tid: 1024, name: kvfree_rcu_1_arg_slab_test\n"
  44. "\t\tid: 2048, name: kvfree_rcu_2_arg_slab_test\n"
  45. /* Add a new test case description here. */
  46. );
  47. /*
  48. * Depends on single_cpu_test parameter. If it is true, then
  49. * use first online CPU to trigger a test on, otherwise go with
  50. * all online CPUs.
  51. */
  52. static cpumask_t cpus_run_test_mask = CPU_MASK_NONE;
  53. /*
  54. * Read write semaphore for synchronization of setup
  55. * phase that is done in main thread and workers.
  56. */
  57. static DECLARE_RWSEM(prepare_for_test_rwsem);
  58. /*
  59. * Completion tracking for worker threads.
  60. */
  61. static DECLARE_COMPLETION(test_all_done_comp);
  62. static atomic_t test_n_undone = ATOMIC_INIT(0);
  63. static inline void
  64. test_report_one_done(void)
  65. {
  66. if (atomic_dec_and_test(&test_n_undone))
  67. complete(&test_all_done_comp);
  68. }
  69. static int random_size_align_alloc_test(void)
  70. {
  71. unsigned long size, align, rnd;
  72. void *ptr;
  73. int i;
  74. for (i = 0; i < test_loop_count; i++) {
  75. get_random_bytes(&rnd, sizeof(rnd));
  76. /*
  77. * Maximum 1024 pages, if PAGE_SIZE is 4096.
  78. */
  79. align = 1 << (rnd % 23);
  80. /*
  81. * Maximum 10 pages.
  82. */
  83. size = ((rnd % 10) + 1) * PAGE_SIZE;
  84. ptr = __vmalloc_node(size, align, GFP_KERNEL | __GFP_ZERO, 0,
  85. __builtin_return_address(0));
  86. if (!ptr)
  87. return -1;
  88. vfree(ptr);
  89. }
  90. return 0;
  91. }
  92. /*
  93. * This test case is supposed to be failed.
  94. */
  95. static int align_shift_alloc_test(void)
  96. {
  97. unsigned long align;
  98. void *ptr;
  99. int i;
  100. for (i = 0; i < BITS_PER_LONG; i++) {
  101. align = ((unsigned long) 1) << i;
  102. ptr = __vmalloc_node(PAGE_SIZE, align, GFP_KERNEL|__GFP_ZERO, 0,
  103. __builtin_return_address(0));
  104. if (!ptr)
  105. return -1;
  106. vfree(ptr);
  107. }
  108. return 0;
  109. }
  110. static int fix_align_alloc_test(void)
  111. {
  112. void *ptr;
  113. int i;
  114. for (i = 0; i < test_loop_count; i++) {
  115. ptr = __vmalloc_node(5 * PAGE_SIZE, THREAD_ALIGN << 1,
  116. GFP_KERNEL | __GFP_ZERO, 0,
  117. __builtin_return_address(0));
  118. if (!ptr)
  119. return -1;
  120. vfree(ptr);
  121. }
  122. return 0;
  123. }
  124. static int random_size_alloc_test(void)
  125. {
  126. unsigned int n;
  127. void *p;
  128. int i;
  129. for (i = 0; i < test_loop_count; i++) {
  130. get_random_bytes(&n, sizeof(i));
  131. n = (n % 100) + 1;
  132. p = vmalloc(n * PAGE_SIZE);
  133. if (!p)
  134. return -1;
  135. *((__u8 *)p) = 1;
  136. vfree(p);
  137. }
  138. return 0;
  139. }
  140. static int long_busy_list_alloc_test(void)
  141. {
  142. void *ptr_1, *ptr_2;
  143. void **ptr;
  144. int rv = -1;
  145. int i;
  146. ptr = vmalloc(sizeof(void *) * 15000);
  147. if (!ptr)
  148. return rv;
  149. for (i = 0; i < 15000; i++)
  150. ptr[i] = vmalloc(1 * PAGE_SIZE);
  151. for (i = 0; i < test_loop_count; i++) {
  152. ptr_1 = vmalloc(100 * PAGE_SIZE);
  153. if (!ptr_1)
  154. goto leave;
  155. ptr_2 = vmalloc(1 * PAGE_SIZE);
  156. if (!ptr_2) {
  157. vfree(ptr_1);
  158. goto leave;
  159. }
  160. *((__u8 *)ptr_1) = 0;
  161. *((__u8 *)ptr_2) = 1;
  162. vfree(ptr_1);
  163. vfree(ptr_2);
  164. }
  165. /* Success */
  166. rv = 0;
  167. leave:
  168. for (i = 0; i < 15000; i++)
  169. vfree(ptr[i]);
  170. vfree(ptr);
  171. return rv;
  172. }
  173. static int full_fit_alloc_test(void)
  174. {
  175. void **ptr, **junk_ptr, *tmp;
  176. int junk_length;
  177. int rv = -1;
  178. int i;
  179. junk_length = fls(num_online_cpus());
  180. junk_length *= (32 * 1024 * 1024 / PAGE_SIZE);
  181. ptr = vmalloc(sizeof(void *) * junk_length);
  182. if (!ptr)
  183. return rv;
  184. junk_ptr = vmalloc(sizeof(void *) * junk_length);
  185. if (!junk_ptr) {
  186. vfree(ptr);
  187. return rv;
  188. }
  189. for (i = 0; i < junk_length; i++) {
  190. ptr[i] = vmalloc(1 * PAGE_SIZE);
  191. junk_ptr[i] = vmalloc(1 * PAGE_SIZE);
  192. }
  193. for (i = 0; i < junk_length; i++)
  194. vfree(junk_ptr[i]);
  195. for (i = 0; i < test_loop_count; i++) {
  196. tmp = vmalloc(1 * PAGE_SIZE);
  197. if (!tmp)
  198. goto error;
  199. *((__u8 *)tmp) = 1;
  200. vfree(tmp);
  201. }
  202. /* Success */
  203. rv = 0;
  204. error:
  205. for (i = 0; i < junk_length; i++)
  206. vfree(ptr[i]);
  207. vfree(ptr);
  208. vfree(junk_ptr);
  209. return rv;
  210. }
  211. static int fix_size_alloc_test(void)
  212. {
  213. void *ptr;
  214. int i;
  215. for (i = 0; i < test_loop_count; i++) {
  216. ptr = vmalloc(3 * PAGE_SIZE);
  217. if (!ptr)
  218. return -1;
  219. *((__u8 *)ptr) = 0;
  220. vfree(ptr);
  221. }
  222. return 0;
  223. }
  224. static int
  225. pcpu_alloc_test(void)
  226. {
  227. int rv = 0;
  228. #ifndef CONFIG_NEED_PER_CPU_KM
  229. void __percpu **pcpu;
  230. size_t size, align;
  231. int i;
  232. pcpu = vmalloc(sizeof(void __percpu *) * 35000);
  233. if (!pcpu)
  234. return -1;
  235. for (i = 0; i < 35000; i++) {
  236. unsigned int r;
  237. get_random_bytes(&r, sizeof(i));
  238. size = (r % (PAGE_SIZE / 4)) + 1;
  239. /*
  240. * Maximum PAGE_SIZE
  241. */
  242. get_random_bytes(&r, sizeof(i));
  243. align = 1 << ((i % 11) + 1);
  244. pcpu[i] = __alloc_percpu(size, align);
  245. if (!pcpu[i])
  246. rv = -1;
  247. }
  248. for (i = 0; i < 35000; i++)
  249. free_percpu(pcpu[i]);
  250. vfree(pcpu);
  251. #endif
  252. return rv;
  253. }
  254. struct test_kvfree_rcu {
  255. struct rcu_head rcu;
  256. unsigned char array[20];
  257. };
  258. static int
  259. kvfree_rcu_1_arg_vmalloc_test(void)
  260. {
  261. struct test_kvfree_rcu *p;
  262. int i;
  263. for (i = 0; i < test_loop_count; i++) {
  264. p = vmalloc(1 * PAGE_SIZE);
  265. if (!p)
  266. return -1;
  267. p->array[0] = 'a';
  268. kvfree_rcu(p);
  269. }
  270. return 0;
  271. }
  272. static int
  273. kvfree_rcu_2_arg_vmalloc_test(void)
  274. {
  275. struct test_kvfree_rcu *p;
  276. int i;
  277. for (i = 0; i < test_loop_count; i++) {
  278. p = vmalloc(1 * PAGE_SIZE);
  279. if (!p)
  280. return -1;
  281. p->array[0] = 'a';
  282. kvfree_rcu(p, rcu);
  283. }
  284. return 0;
  285. }
  286. static int
  287. kvfree_rcu_1_arg_slab_test(void)
  288. {
  289. struct test_kvfree_rcu *p;
  290. int i;
  291. for (i = 0; i < test_loop_count; i++) {
  292. p = kmalloc(sizeof(*p), GFP_KERNEL);
  293. if (!p)
  294. return -1;
  295. p->array[0] = 'a';
  296. kvfree_rcu(p);
  297. }
  298. return 0;
  299. }
  300. static int
  301. kvfree_rcu_2_arg_slab_test(void)
  302. {
  303. struct test_kvfree_rcu *p;
  304. int i;
  305. for (i = 0; i < test_loop_count; i++) {
  306. p = kmalloc(sizeof(*p), GFP_KERNEL);
  307. if (!p)
  308. return -1;
  309. p->array[0] = 'a';
  310. kvfree_rcu(p, rcu);
  311. }
  312. return 0;
  313. }
  314. struct test_case_desc {
  315. const char *test_name;
  316. int (*test_func)(void);
  317. };
  318. static struct test_case_desc test_case_array[] = {
  319. { "fix_size_alloc_test", fix_size_alloc_test },
  320. { "full_fit_alloc_test", full_fit_alloc_test },
  321. { "long_busy_list_alloc_test", long_busy_list_alloc_test },
  322. { "random_size_alloc_test", random_size_alloc_test },
  323. { "fix_align_alloc_test", fix_align_alloc_test },
  324. { "random_size_align_alloc_test", random_size_align_alloc_test },
  325. { "align_shift_alloc_test", align_shift_alloc_test },
  326. { "pcpu_alloc_test", pcpu_alloc_test },
  327. { "kvfree_rcu_1_arg_vmalloc_test", kvfree_rcu_1_arg_vmalloc_test },
  328. { "kvfree_rcu_2_arg_vmalloc_test", kvfree_rcu_2_arg_vmalloc_test },
  329. { "kvfree_rcu_1_arg_slab_test", kvfree_rcu_1_arg_slab_test },
  330. { "kvfree_rcu_2_arg_slab_test", kvfree_rcu_2_arg_slab_test },
  331. /* Add a new test case here. */
  332. };
  333. struct test_case_data {
  334. int test_failed;
  335. int test_passed;
  336. u64 time;
  337. };
  338. /* Split it to get rid of: WARNING: line over 80 characters */
  339. static struct test_case_data
  340. per_cpu_test_data[NR_CPUS][ARRAY_SIZE(test_case_array)];
  341. static struct test_driver {
  342. struct task_struct *task;
  343. unsigned long start;
  344. unsigned long stop;
  345. int cpu;
  346. } per_cpu_test_driver[NR_CPUS];
  347. static void shuffle_array(int *arr, int n)
  348. {
  349. unsigned int rnd;
  350. int i, j, x;
  351. for (i = n - 1; i > 0; i--) {
  352. get_random_bytes(&rnd, sizeof(rnd));
  353. /* Cut the range. */
  354. j = rnd % i;
  355. /* Swap indexes. */
  356. x = arr[i];
  357. arr[i] = arr[j];
  358. arr[j] = x;
  359. }
  360. }
  361. static int test_func(void *private)
  362. {
  363. struct test_driver *t = private;
  364. int random_array[ARRAY_SIZE(test_case_array)];
  365. int index, i, j;
  366. ktime_t kt;
  367. u64 delta;
  368. if (set_cpus_allowed_ptr(current, cpumask_of(t->cpu)) < 0)
  369. pr_err("Failed to set affinity to %d CPU\n", t->cpu);
  370. for (i = 0; i < ARRAY_SIZE(test_case_array); i++)
  371. random_array[i] = i;
  372. if (!sequential_test_order)
  373. shuffle_array(random_array, ARRAY_SIZE(test_case_array));
  374. /*
  375. * Block until initialization is done.
  376. */
  377. down_read(&prepare_for_test_rwsem);
  378. t->start = get_cycles();
  379. for (i = 0; i < ARRAY_SIZE(test_case_array); i++) {
  380. index = random_array[i];
  381. /*
  382. * Skip tests if run_test_mask has been specified.
  383. */
  384. if (!((run_test_mask & (1 << index)) >> index))
  385. continue;
  386. kt = ktime_get();
  387. for (j = 0; j < test_repeat_count; j++) {
  388. if (!test_case_array[index].test_func())
  389. per_cpu_test_data[t->cpu][index].test_passed++;
  390. else
  391. per_cpu_test_data[t->cpu][index].test_failed++;
  392. }
  393. /*
  394. * Take an average time that test took.
  395. */
  396. delta = (u64) ktime_us_delta(ktime_get(), kt);
  397. do_div(delta, (u32) test_repeat_count);
  398. per_cpu_test_data[t->cpu][index].time = delta;
  399. }
  400. t->stop = get_cycles();
  401. up_read(&prepare_for_test_rwsem);
  402. test_report_one_done();
  403. /*
  404. * Wait for the kthread_stop() call.
  405. */
  406. while (!kthread_should_stop())
  407. msleep(10);
  408. return 0;
  409. }
  410. static void
  411. init_test_configurtion(void)
  412. {
  413. /*
  414. * Reset all data of all CPUs.
  415. */
  416. memset(per_cpu_test_data, 0, sizeof(per_cpu_test_data));
  417. if (single_cpu_test)
  418. cpumask_set_cpu(cpumask_first(cpu_online_mask),
  419. &cpus_run_test_mask);
  420. else
  421. cpumask_and(&cpus_run_test_mask, cpu_online_mask,
  422. cpu_online_mask);
  423. if (test_repeat_count <= 0)
  424. test_repeat_count = 1;
  425. if (test_loop_count <= 0)
  426. test_loop_count = 1;
  427. }
  428. static void do_concurrent_test(void)
  429. {
  430. int cpu, ret;
  431. /*
  432. * Set some basic configurations plus sanity check.
  433. */
  434. init_test_configurtion();
  435. /*
  436. * Put on hold all workers.
  437. */
  438. down_write(&prepare_for_test_rwsem);
  439. for_each_cpu(cpu, &cpus_run_test_mask) {
  440. struct test_driver *t = &per_cpu_test_driver[cpu];
  441. t->cpu = cpu;
  442. t->task = kthread_run(test_func, t, "vmalloc_test/%d", cpu);
  443. if (!IS_ERR(t->task))
  444. /* Success. */
  445. atomic_inc(&test_n_undone);
  446. else
  447. pr_err("Failed to start kthread for %d CPU\n", cpu);
  448. }
  449. /*
  450. * Now let the workers do their job.
  451. */
  452. up_write(&prepare_for_test_rwsem);
  453. /*
  454. * Sleep quiet until all workers are done with 1 second
  455. * interval. Since the test can take a lot of time we
  456. * can run into a stack trace of the hung task. That is
  457. * why we go with completion_timeout and HZ value.
  458. */
  459. do {
  460. ret = wait_for_completion_timeout(&test_all_done_comp, HZ);
  461. } while (!ret);
  462. for_each_cpu(cpu, &cpus_run_test_mask) {
  463. struct test_driver *t = &per_cpu_test_driver[cpu];
  464. int i;
  465. if (!IS_ERR(t->task))
  466. kthread_stop(t->task);
  467. for (i = 0; i < ARRAY_SIZE(test_case_array); i++) {
  468. if (!((run_test_mask & (1 << i)) >> i))
  469. continue;
  470. pr_info(
  471. "Summary: %s passed: %d failed: %d repeat: %d loops: %d avg: %llu usec\n",
  472. test_case_array[i].test_name,
  473. per_cpu_test_data[cpu][i].test_passed,
  474. per_cpu_test_data[cpu][i].test_failed,
  475. test_repeat_count, test_loop_count,
  476. per_cpu_test_data[cpu][i].time);
  477. }
  478. pr_info("All test took CPU%d=%lu cycles\n",
  479. cpu, t->stop - t->start);
  480. }
  481. }
  482. static int vmalloc_test_init(void)
  483. {
  484. do_concurrent_test();
  485. return -EAGAIN; /* Fail will directly unload the module */
  486. }
  487. static void vmalloc_test_exit(void)
  488. {
  489. }
  490. module_init(vmalloc_test_init)
  491. module_exit(vmalloc_test_exit)
  492. MODULE_LICENSE("GPL");
  493. MODULE_AUTHOR("Uladzislau Rezki");
  494. MODULE_DESCRIPTION("vmalloc test module");