ring_buffer_benchmark.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * ring buffer tester and benchmark
  4. *
  5. * Copyright (C) 2009 Steven Rostedt <srostedt@redhat.com>
  6. */
  7. #include <linux/ring_buffer.h>
  8. #include <linux/completion.h>
  9. #include <linux/kthread.h>
  10. #include <uapi/linux/sched/types.h>
  11. #include <linux/module.h>
  12. #include <linux/ktime.h>
  13. #include <asm/local.h>
  14. struct rb_page {
  15. u64 ts;
  16. local_t commit;
  17. char data[4080];
  18. };
  19. /* run time and sleep time in seconds */
  20. #define RUN_TIME 10ULL
  21. #define SLEEP_TIME 10
  22. /* number of events for writer to wake up the reader */
  23. static int wakeup_interval = 100;
  24. static int reader_finish;
  25. static DECLARE_COMPLETION(read_start);
  26. static DECLARE_COMPLETION(read_done);
  27. static struct trace_buffer *buffer;
  28. static struct task_struct *producer;
  29. static struct task_struct *consumer;
  30. static unsigned long read;
  31. static unsigned int disable_reader;
  32. module_param(disable_reader, uint, 0644);
  33. MODULE_PARM_DESC(disable_reader, "only run producer");
  34. static unsigned int write_iteration = 50;
  35. module_param(write_iteration, uint, 0644);
  36. MODULE_PARM_DESC(write_iteration, "# of writes between timestamp readings");
  37. static int producer_nice = MAX_NICE;
  38. static int consumer_nice = MAX_NICE;
  39. static int producer_fifo;
  40. static int consumer_fifo;
  41. module_param(producer_nice, int, 0644);
  42. MODULE_PARM_DESC(producer_nice, "nice prio for producer");
  43. module_param(consumer_nice, int, 0644);
  44. MODULE_PARM_DESC(consumer_nice, "nice prio for consumer");
  45. module_param(producer_fifo, int, 0644);
  46. MODULE_PARM_DESC(producer_fifo, "use fifo for producer: 0 - disabled, 1 - low prio, 2 - fifo");
  47. module_param(consumer_fifo, int, 0644);
  48. MODULE_PARM_DESC(consumer_fifo, "use fifo for consumer: 0 - disabled, 1 - low prio, 2 - fifo");
  49. static int read_events;
  50. static int test_error;
  51. #define TEST_ERROR() \
  52. do { \
  53. if (!test_error) { \
  54. test_error = 1; \
  55. WARN_ON(1); \
  56. } \
  57. } while (0)
  58. enum event_status {
  59. EVENT_FOUND,
  60. EVENT_DROPPED,
  61. };
  62. static bool break_test(void)
  63. {
  64. return test_error || kthread_should_stop();
  65. }
  66. static enum event_status read_event(int cpu)
  67. {
  68. struct ring_buffer_event *event;
  69. int *entry;
  70. u64 ts;
  71. event = ring_buffer_consume(buffer, cpu, &ts, NULL);
  72. if (!event)
  73. return EVENT_DROPPED;
  74. entry = ring_buffer_event_data(event);
  75. if (*entry != cpu) {
  76. TEST_ERROR();
  77. return EVENT_DROPPED;
  78. }
  79. read++;
  80. return EVENT_FOUND;
  81. }
  82. static enum event_status read_page(int cpu)
  83. {
  84. struct ring_buffer_event *event;
  85. struct rb_page *rpage;
  86. unsigned long commit;
  87. void *bpage;
  88. int *entry;
  89. int ret;
  90. int inc;
  91. int i;
  92. bpage = ring_buffer_alloc_read_page(buffer, cpu);
  93. if (IS_ERR(bpage))
  94. return EVENT_DROPPED;
  95. ret = ring_buffer_read_page(buffer, &bpage, PAGE_SIZE, cpu, 1);
  96. if (ret >= 0) {
  97. rpage = bpage;
  98. /* The commit may have missed event flags set, clear them */
  99. commit = local_read(&rpage->commit) & 0xfffff;
  100. for (i = 0; i < commit && !test_error ; i += inc) {
  101. if (i >= (PAGE_SIZE - offsetof(struct rb_page, data))) {
  102. TEST_ERROR();
  103. break;
  104. }
  105. inc = -1;
  106. event = (void *)&rpage->data[i];
  107. switch (event->type_len) {
  108. case RINGBUF_TYPE_PADDING:
  109. /* failed writes may be discarded events */
  110. if (!event->time_delta)
  111. TEST_ERROR();
  112. inc = event->array[0] + 4;
  113. break;
  114. case RINGBUF_TYPE_TIME_EXTEND:
  115. inc = 8;
  116. break;
  117. case 0:
  118. entry = ring_buffer_event_data(event);
  119. if (*entry != cpu) {
  120. TEST_ERROR();
  121. break;
  122. }
  123. read++;
  124. if (!event->array[0]) {
  125. TEST_ERROR();
  126. break;
  127. }
  128. inc = event->array[0] + 4;
  129. break;
  130. default:
  131. entry = ring_buffer_event_data(event);
  132. if (*entry != cpu) {
  133. TEST_ERROR();
  134. break;
  135. }
  136. read++;
  137. inc = ((event->type_len + 1) * 4);
  138. }
  139. if (test_error)
  140. break;
  141. if (inc <= 0) {
  142. TEST_ERROR();
  143. break;
  144. }
  145. }
  146. }
  147. ring_buffer_free_read_page(buffer, cpu, bpage);
  148. if (ret < 0)
  149. return EVENT_DROPPED;
  150. return EVENT_FOUND;
  151. }
  152. static void ring_buffer_consumer(void)
  153. {
  154. /* toggle between reading pages and events */
  155. read_events ^= 1;
  156. read = 0;
  157. /*
  158. * Continue running until the producer specifically asks to stop
  159. * and is ready for the completion.
  160. */
  161. while (!READ_ONCE(reader_finish)) {
  162. int found = 1;
  163. while (found && !test_error) {
  164. int cpu;
  165. found = 0;
  166. for_each_online_cpu(cpu) {
  167. enum event_status stat;
  168. if (read_events)
  169. stat = read_event(cpu);
  170. else
  171. stat = read_page(cpu);
  172. if (test_error)
  173. break;
  174. if (stat == EVENT_FOUND)
  175. found = 1;
  176. }
  177. }
  178. /* Wait till the producer wakes us up when there is more data
  179. * available or when the producer wants us to finish reading.
  180. */
  181. set_current_state(TASK_INTERRUPTIBLE);
  182. if (reader_finish)
  183. break;
  184. schedule();
  185. }
  186. __set_current_state(TASK_RUNNING);
  187. reader_finish = 0;
  188. complete(&read_done);
  189. }
  190. static void ring_buffer_producer(void)
  191. {
  192. ktime_t start_time, end_time, timeout;
  193. unsigned long long time;
  194. unsigned long long entries;
  195. unsigned long long overruns;
  196. unsigned long missed = 0;
  197. unsigned long hit = 0;
  198. unsigned long avg;
  199. int cnt = 0;
  200. /*
  201. * Hammer the buffer for 10 secs (this may
  202. * make the system stall)
  203. */
  204. trace_printk("Starting ring buffer hammer\n");
  205. start_time = ktime_get();
  206. timeout = ktime_add_ns(start_time, RUN_TIME * NSEC_PER_SEC);
  207. do {
  208. struct ring_buffer_event *event;
  209. int *entry;
  210. int i;
  211. for (i = 0; i < write_iteration; i++) {
  212. event = ring_buffer_lock_reserve(buffer, 10);
  213. if (!event) {
  214. missed++;
  215. } else {
  216. hit++;
  217. entry = ring_buffer_event_data(event);
  218. *entry = smp_processor_id();
  219. ring_buffer_unlock_commit(buffer, event);
  220. }
  221. }
  222. end_time = ktime_get();
  223. cnt++;
  224. if (consumer && !(cnt % wakeup_interval))
  225. wake_up_process(consumer);
  226. #ifndef CONFIG_PREEMPTION
  227. /*
  228. * If we are a non preempt kernel, the 10 seconds run will
  229. * stop everything while it runs. Instead, we will call
  230. * cond_resched and also add any time that was lost by a
  231. * reschedule.
  232. *
  233. * Do a cond resched at the same frequency we would wake up
  234. * the reader.
  235. */
  236. if (cnt % wakeup_interval)
  237. cond_resched();
  238. #endif
  239. } while (ktime_before(end_time, timeout) && !break_test());
  240. trace_printk("End ring buffer hammer\n");
  241. if (consumer) {
  242. /* Init both completions here to avoid races */
  243. init_completion(&read_start);
  244. init_completion(&read_done);
  245. /* the completions must be visible before the finish var */
  246. smp_wmb();
  247. reader_finish = 1;
  248. wake_up_process(consumer);
  249. wait_for_completion(&read_done);
  250. }
  251. time = ktime_us_delta(end_time, start_time);
  252. entries = ring_buffer_entries(buffer);
  253. overruns = ring_buffer_overruns(buffer);
  254. if (test_error)
  255. trace_printk("ERROR!\n");
  256. if (!disable_reader) {
  257. if (consumer_fifo)
  258. trace_printk("Running Consumer at SCHED_FIFO %s\n",
  259. consumer_fifo == 1 ? "low" : "high");
  260. else
  261. trace_printk("Running Consumer at nice: %d\n",
  262. consumer_nice);
  263. }
  264. if (producer_fifo)
  265. trace_printk("Running Producer at SCHED_FIFO %s\n",
  266. producer_fifo == 1 ? "low" : "high");
  267. else
  268. trace_printk("Running Producer at nice: %d\n",
  269. producer_nice);
  270. /* Let the user know that the test is running at low priority */
  271. if (!producer_fifo && !consumer_fifo &&
  272. producer_nice == MAX_NICE && consumer_nice == MAX_NICE)
  273. trace_printk("WARNING!!! This test is running at lowest priority.\n");
  274. trace_printk("Time: %lld (usecs)\n", time);
  275. trace_printk("Overruns: %lld\n", overruns);
  276. if (disable_reader)
  277. trace_printk("Read: (reader disabled)\n");
  278. else
  279. trace_printk("Read: %ld (by %s)\n", read,
  280. read_events ? "events" : "pages");
  281. trace_printk("Entries: %lld\n", entries);
  282. trace_printk("Total: %lld\n", entries + overruns + read);
  283. trace_printk("Missed: %ld\n", missed);
  284. trace_printk("Hit: %ld\n", hit);
  285. /* Convert time from usecs to millisecs */
  286. do_div(time, USEC_PER_MSEC);
  287. if (time)
  288. hit /= (long)time;
  289. else
  290. trace_printk("TIME IS ZERO??\n");
  291. trace_printk("Entries per millisec: %ld\n", hit);
  292. if (hit) {
  293. /* Calculate the average time in nanosecs */
  294. avg = NSEC_PER_MSEC / hit;
  295. trace_printk("%ld ns per entry\n", avg);
  296. }
  297. if (missed) {
  298. if (time)
  299. missed /= (long)time;
  300. trace_printk("Total iterations per millisec: %ld\n",
  301. hit + missed);
  302. /* it is possible that hit + missed will overflow and be zero */
  303. if (!(hit + missed)) {
  304. trace_printk("hit + missed overflowed and totalled zero!\n");
  305. hit--; /* make it non zero */
  306. }
  307. /* Calculate the average time in nanosecs */
  308. avg = NSEC_PER_MSEC / (hit + missed);
  309. trace_printk("%ld ns per entry\n", avg);
  310. }
  311. }
  312. static void wait_to_die(void)
  313. {
  314. set_current_state(TASK_INTERRUPTIBLE);
  315. while (!kthread_should_stop()) {
  316. schedule();
  317. set_current_state(TASK_INTERRUPTIBLE);
  318. }
  319. __set_current_state(TASK_RUNNING);
  320. }
  321. static int ring_buffer_consumer_thread(void *arg)
  322. {
  323. while (!break_test()) {
  324. complete(&read_start);
  325. ring_buffer_consumer();
  326. set_current_state(TASK_INTERRUPTIBLE);
  327. if (break_test())
  328. break;
  329. schedule();
  330. }
  331. __set_current_state(TASK_RUNNING);
  332. if (!kthread_should_stop())
  333. wait_to_die();
  334. return 0;
  335. }
  336. static int ring_buffer_producer_thread(void *arg)
  337. {
  338. while (!break_test()) {
  339. ring_buffer_reset(buffer);
  340. if (consumer) {
  341. wake_up_process(consumer);
  342. wait_for_completion(&read_start);
  343. }
  344. ring_buffer_producer();
  345. if (break_test())
  346. goto out_kill;
  347. trace_printk("Sleeping for 10 secs\n");
  348. set_current_state(TASK_INTERRUPTIBLE);
  349. if (break_test())
  350. goto out_kill;
  351. schedule_timeout(HZ * SLEEP_TIME);
  352. }
  353. out_kill:
  354. __set_current_state(TASK_RUNNING);
  355. if (!kthread_should_stop())
  356. wait_to_die();
  357. return 0;
  358. }
  359. static int __init ring_buffer_benchmark_init(void)
  360. {
  361. int ret;
  362. /* make a one meg buffer in overwite mode */
  363. buffer = ring_buffer_alloc(1000000, RB_FL_OVERWRITE);
  364. if (!buffer)
  365. return -ENOMEM;
  366. if (!disable_reader) {
  367. consumer = kthread_create(ring_buffer_consumer_thread,
  368. NULL, "rb_consumer");
  369. ret = PTR_ERR(consumer);
  370. if (IS_ERR(consumer))
  371. goto out_fail;
  372. }
  373. producer = kthread_run(ring_buffer_producer_thread,
  374. NULL, "rb_producer");
  375. ret = PTR_ERR(producer);
  376. if (IS_ERR(producer))
  377. goto out_kill;
  378. /*
  379. * Run them as low-prio background tasks by default:
  380. */
  381. if (!disable_reader) {
  382. if (consumer_fifo >= 2)
  383. sched_set_fifo(consumer);
  384. else if (consumer_fifo == 1)
  385. sched_set_fifo_low(consumer);
  386. else
  387. set_user_nice(consumer, consumer_nice);
  388. }
  389. if (producer_fifo >= 2)
  390. sched_set_fifo(producer);
  391. else if (producer_fifo == 1)
  392. sched_set_fifo_low(producer);
  393. else
  394. set_user_nice(producer, producer_nice);
  395. return 0;
  396. out_kill:
  397. if (consumer)
  398. kthread_stop(consumer);
  399. out_fail:
  400. ring_buffer_free(buffer);
  401. return ret;
  402. }
  403. static void __exit ring_buffer_benchmark_exit(void)
  404. {
  405. kthread_stop(producer);
  406. if (consumer)
  407. kthread_stop(consumer);
  408. ring_buffer_free(buffer);
  409. }
  410. module_init(ring_buffer_benchmark_init);
  411. module_exit(ring_buffer_benchmark_exit);
  412. MODULE_AUTHOR("Steven Rostedt");
  413. MODULE_DESCRIPTION("ring_buffer_benchmark");
  414. MODULE_LICENSE("GPL");