map_perf_test_user.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /* Copyright (c) 2016 Facebook
  3. */
  4. #define _GNU_SOURCE
  5. #include <sched.h>
  6. #include <stdio.h>
  7. #include <sys/types.h>
  8. #include <asm/unistd.h>
  9. #include <unistd.h>
  10. #include <assert.h>
  11. #include <sys/wait.h>
  12. #include <stdlib.h>
  13. #include <signal.h>
  14. #include <string.h>
  15. #include <time.h>
  16. #include <sys/resource.h>
  17. #include <arpa/inet.h>
  18. #include <errno.h>
  19. #include <bpf/bpf.h>
  20. #include <bpf/libbpf.h>
  21. #define TEST_BIT(t) (1U << (t))
  22. #define MAX_NR_CPUS 1024
  23. static __u64 time_get_ns(void)
  24. {
  25. struct timespec ts;
  26. clock_gettime(CLOCK_MONOTONIC, &ts);
  27. return ts.tv_sec * 1000000000ull + ts.tv_nsec;
  28. }
  29. enum test_type {
  30. HASH_PREALLOC,
  31. PERCPU_HASH_PREALLOC,
  32. HASH_KMALLOC,
  33. PERCPU_HASH_KMALLOC,
  34. LRU_HASH_PREALLOC,
  35. NOCOMMON_LRU_HASH_PREALLOC,
  36. LPM_KMALLOC,
  37. HASH_LOOKUP,
  38. ARRAY_LOOKUP,
  39. INNER_LRU_HASH_PREALLOC,
  40. LRU_HASH_LOOKUP,
  41. NR_TESTS,
  42. };
  43. const char *test_map_names[NR_TESTS] = {
  44. [HASH_PREALLOC] = "hash_map",
  45. [PERCPU_HASH_PREALLOC] = "percpu_hash_map",
  46. [HASH_KMALLOC] = "hash_map_alloc",
  47. [PERCPU_HASH_KMALLOC] = "percpu_hash_map_alloc",
  48. [LRU_HASH_PREALLOC] = "lru_hash_map",
  49. [NOCOMMON_LRU_HASH_PREALLOC] = "nocommon_lru_hash_map",
  50. [LPM_KMALLOC] = "lpm_trie_map_alloc",
  51. [HASH_LOOKUP] = "hash_map",
  52. [ARRAY_LOOKUP] = "array_map",
  53. [INNER_LRU_HASH_PREALLOC] = "inner_lru_hash_map",
  54. [LRU_HASH_LOOKUP] = "lru_hash_lookup_map",
  55. };
  56. enum map_idx {
  57. array_of_lru_hashs_idx,
  58. hash_map_alloc_idx,
  59. lru_hash_lookup_idx,
  60. NR_IDXES,
  61. };
  62. static int map_fd[NR_IDXES];
  63. static int test_flags = ~0;
  64. static uint32_t num_map_entries;
  65. static uint32_t inner_lru_hash_size;
  66. static int lru_hash_lookup_test_entries = 32;
  67. static uint32_t max_cnt = 1000000;
  68. static int check_test_flags(enum test_type t)
  69. {
  70. return test_flags & TEST_BIT(t);
  71. }
  72. static void test_hash_prealloc(int cpu)
  73. {
  74. __u64 start_time;
  75. int i;
  76. start_time = time_get_ns();
  77. for (i = 0; i < max_cnt; i++)
  78. syscall(__NR_getuid);
  79. printf("%d:hash_map_perf pre-alloc %lld events per sec\n",
  80. cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
  81. }
  82. static int pre_test_lru_hash_lookup(int tasks)
  83. {
  84. int fd = map_fd[lru_hash_lookup_idx];
  85. uint32_t key;
  86. long val = 1;
  87. int ret;
  88. if (num_map_entries > lru_hash_lookup_test_entries)
  89. lru_hash_lookup_test_entries = num_map_entries;
  90. /* Populate the lru_hash_map for LRU_HASH_LOOKUP perf test.
  91. *
  92. * It is fine that the user requests for a map with
  93. * num_map_entries < 32 and some of the later lru hash lookup
  94. * may return not found. For LRU map, we are not interested
  95. * in such small map performance.
  96. */
  97. for (key = 0; key < lru_hash_lookup_test_entries; key++) {
  98. ret = bpf_map_update_elem(fd, &key, &val, BPF_NOEXIST);
  99. if (ret)
  100. return ret;
  101. }
  102. return 0;
  103. }
  104. static void do_test_lru(enum test_type test, int cpu)
  105. {
  106. static int inner_lru_map_fds[MAX_NR_CPUS];
  107. struct sockaddr_in6 in6 = { .sin6_family = AF_INET6 };
  108. const char *test_name;
  109. __u64 start_time;
  110. int i, ret;
  111. if (test == INNER_LRU_HASH_PREALLOC && cpu) {
  112. /* If CPU is not 0, create inner_lru hash map and insert the fd
  113. * value into the array_of_lru_hash map. In case of CPU 0,
  114. * 'inner_lru_hash_map' was statically inserted on the map init
  115. */
  116. int outer_fd = map_fd[array_of_lru_hashs_idx];
  117. unsigned int mycpu, mynode;
  118. assert(cpu < MAX_NR_CPUS);
  119. ret = syscall(__NR_getcpu, &mycpu, &mynode, NULL);
  120. assert(!ret);
  121. inner_lru_map_fds[cpu] =
  122. bpf_create_map_node(BPF_MAP_TYPE_LRU_HASH,
  123. test_map_names[INNER_LRU_HASH_PREALLOC],
  124. sizeof(uint32_t),
  125. sizeof(long),
  126. inner_lru_hash_size, 0,
  127. mynode);
  128. if (inner_lru_map_fds[cpu] == -1) {
  129. printf("cannot create BPF_MAP_TYPE_LRU_HASH %s(%d)\n",
  130. strerror(errno), errno);
  131. exit(1);
  132. }
  133. ret = bpf_map_update_elem(outer_fd, &cpu,
  134. &inner_lru_map_fds[cpu],
  135. BPF_ANY);
  136. if (ret) {
  137. printf("cannot update ARRAY_OF_LRU_HASHS with key:%u. %s(%d)\n",
  138. cpu, strerror(errno), errno);
  139. exit(1);
  140. }
  141. }
  142. in6.sin6_addr.s6_addr16[0] = 0xdead;
  143. in6.sin6_addr.s6_addr16[1] = 0xbeef;
  144. if (test == LRU_HASH_PREALLOC) {
  145. test_name = "lru_hash_map_perf";
  146. in6.sin6_addr.s6_addr16[2] = 0;
  147. } else if (test == NOCOMMON_LRU_HASH_PREALLOC) {
  148. test_name = "nocommon_lru_hash_map_perf";
  149. in6.sin6_addr.s6_addr16[2] = 1;
  150. } else if (test == INNER_LRU_HASH_PREALLOC) {
  151. test_name = "inner_lru_hash_map_perf";
  152. in6.sin6_addr.s6_addr16[2] = 2;
  153. } else if (test == LRU_HASH_LOOKUP) {
  154. test_name = "lru_hash_lookup_perf";
  155. in6.sin6_addr.s6_addr16[2] = 3;
  156. in6.sin6_addr.s6_addr32[3] = 0;
  157. } else {
  158. assert(0);
  159. }
  160. start_time = time_get_ns();
  161. for (i = 0; i < max_cnt; i++) {
  162. ret = connect(-1, (const struct sockaddr *)&in6, sizeof(in6));
  163. assert(ret == -1 && errno == EBADF);
  164. if (in6.sin6_addr.s6_addr32[3] <
  165. lru_hash_lookup_test_entries - 32)
  166. in6.sin6_addr.s6_addr32[3] += 32;
  167. else
  168. in6.sin6_addr.s6_addr32[3] = 0;
  169. }
  170. printf("%d:%s pre-alloc %lld events per sec\n",
  171. cpu, test_name,
  172. max_cnt * 1000000000ll / (time_get_ns() - start_time));
  173. }
  174. static void test_lru_hash_prealloc(int cpu)
  175. {
  176. do_test_lru(LRU_HASH_PREALLOC, cpu);
  177. }
  178. static void test_nocommon_lru_hash_prealloc(int cpu)
  179. {
  180. do_test_lru(NOCOMMON_LRU_HASH_PREALLOC, cpu);
  181. }
  182. static void test_inner_lru_hash_prealloc(int cpu)
  183. {
  184. do_test_lru(INNER_LRU_HASH_PREALLOC, cpu);
  185. }
  186. static void test_lru_hash_lookup(int cpu)
  187. {
  188. do_test_lru(LRU_HASH_LOOKUP, cpu);
  189. }
  190. static void test_percpu_hash_prealloc(int cpu)
  191. {
  192. __u64 start_time;
  193. int i;
  194. start_time = time_get_ns();
  195. for (i = 0; i < max_cnt; i++)
  196. syscall(__NR_geteuid);
  197. printf("%d:percpu_hash_map_perf pre-alloc %lld events per sec\n",
  198. cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
  199. }
  200. static void test_hash_kmalloc(int cpu)
  201. {
  202. __u64 start_time;
  203. int i;
  204. start_time = time_get_ns();
  205. for (i = 0; i < max_cnt; i++)
  206. syscall(__NR_getgid);
  207. printf("%d:hash_map_perf kmalloc %lld events per sec\n",
  208. cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
  209. }
  210. static void test_percpu_hash_kmalloc(int cpu)
  211. {
  212. __u64 start_time;
  213. int i;
  214. start_time = time_get_ns();
  215. for (i = 0; i < max_cnt; i++)
  216. syscall(__NR_getegid);
  217. printf("%d:percpu_hash_map_perf kmalloc %lld events per sec\n",
  218. cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
  219. }
  220. static void test_lpm_kmalloc(int cpu)
  221. {
  222. __u64 start_time;
  223. int i;
  224. start_time = time_get_ns();
  225. for (i = 0; i < max_cnt; i++)
  226. syscall(__NR_gettid);
  227. printf("%d:lpm_perf kmalloc %lld events per sec\n",
  228. cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
  229. }
  230. static void test_hash_lookup(int cpu)
  231. {
  232. __u64 start_time;
  233. int i;
  234. start_time = time_get_ns();
  235. for (i = 0; i < max_cnt; i++)
  236. syscall(__NR_getpgid, 0);
  237. printf("%d:hash_lookup %lld lookups per sec\n",
  238. cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time));
  239. }
  240. static void test_array_lookup(int cpu)
  241. {
  242. __u64 start_time;
  243. int i;
  244. start_time = time_get_ns();
  245. for (i = 0; i < max_cnt; i++)
  246. syscall(__NR_getppid, 0);
  247. printf("%d:array_lookup %lld lookups per sec\n",
  248. cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time));
  249. }
  250. typedef int (*pre_test_func)(int tasks);
  251. const pre_test_func pre_test_funcs[] = {
  252. [LRU_HASH_LOOKUP] = pre_test_lru_hash_lookup,
  253. };
  254. typedef void (*test_func)(int cpu);
  255. const test_func test_funcs[] = {
  256. [HASH_PREALLOC] = test_hash_prealloc,
  257. [PERCPU_HASH_PREALLOC] = test_percpu_hash_prealloc,
  258. [HASH_KMALLOC] = test_hash_kmalloc,
  259. [PERCPU_HASH_KMALLOC] = test_percpu_hash_kmalloc,
  260. [LRU_HASH_PREALLOC] = test_lru_hash_prealloc,
  261. [NOCOMMON_LRU_HASH_PREALLOC] = test_nocommon_lru_hash_prealloc,
  262. [LPM_KMALLOC] = test_lpm_kmalloc,
  263. [HASH_LOOKUP] = test_hash_lookup,
  264. [ARRAY_LOOKUP] = test_array_lookup,
  265. [INNER_LRU_HASH_PREALLOC] = test_inner_lru_hash_prealloc,
  266. [LRU_HASH_LOOKUP] = test_lru_hash_lookup,
  267. };
  268. static int pre_test(int tasks)
  269. {
  270. int i;
  271. for (i = 0; i < NR_TESTS; i++) {
  272. if (pre_test_funcs[i] && check_test_flags(i)) {
  273. int ret = pre_test_funcs[i](tasks);
  274. if (ret)
  275. return ret;
  276. }
  277. }
  278. return 0;
  279. }
  280. static void loop(int cpu)
  281. {
  282. cpu_set_t cpuset;
  283. int i;
  284. CPU_ZERO(&cpuset);
  285. CPU_SET(cpu, &cpuset);
  286. sched_setaffinity(0, sizeof(cpuset), &cpuset);
  287. for (i = 0; i < NR_TESTS; i++) {
  288. if (check_test_flags(i))
  289. test_funcs[i](cpu);
  290. }
  291. }
  292. static void run_perf_test(int tasks)
  293. {
  294. pid_t pid[tasks];
  295. int i;
  296. assert(!pre_test(tasks));
  297. for (i = 0; i < tasks; i++) {
  298. pid[i] = fork();
  299. if (pid[i] == 0) {
  300. loop(i);
  301. exit(0);
  302. } else if (pid[i] == -1) {
  303. printf("couldn't spawn #%d process\n", i);
  304. exit(1);
  305. }
  306. }
  307. for (i = 0; i < tasks; i++) {
  308. int status;
  309. assert(waitpid(pid[i], &status, 0) == pid[i]);
  310. assert(status == 0);
  311. }
  312. }
  313. static void fill_lpm_trie(void)
  314. {
  315. struct bpf_lpm_trie_key *key;
  316. unsigned long value = 0;
  317. unsigned int i;
  318. int r;
  319. key = alloca(sizeof(*key) + 4);
  320. key->prefixlen = 32;
  321. for (i = 0; i < 512; ++i) {
  322. key->prefixlen = rand() % 33;
  323. key->data[0] = rand() & 0xff;
  324. key->data[1] = rand() & 0xff;
  325. key->data[2] = rand() & 0xff;
  326. key->data[3] = rand() & 0xff;
  327. r = bpf_map_update_elem(map_fd[hash_map_alloc_idx],
  328. key, &value, 0);
  329. assert(!r);
  330. }
  331. key->prefixlen = 32;
  332. key->data[0] = 192;
  333. key->data[1] = 168;
  334. key->data[2] = 0;
  335. key->data[3] = 1;
  336. value = 128;
  337. r = bpf_map_update_elem(map_fd[hash_map_alloc_idx], key, &value, 0);
  338. assert(!r);
  339. }
  340. static void fixup_map(struct bpf_object *obj)
  341. {
  342. struct bpf_map *map;
  343. int i;
  344. bpf_object__for_each_map(map, obj) {
  345. const char *name = bpf_map__name(map);
  346. /* Only change the max_entries for the enabled test(s) */
  347. for (i = 0; i < NR_TESTS; i++) {
  348. if (!strcmp(test_map_names[i], name) &&
  349. (check_test_flags(i))) {
  350. bpf_map__resize(map, num_map_entries);
  351. continue;
  352. }
  353. }
  354. }
  355. inner_lru_hash_size = num_map_entries;
  356. }
  357. int main(int argc, char **argv)
  358. {
  359. struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
  360. int nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
  361. struct bpf_link *links[8];
  362. struct bpf_program *prog;
  363. struct bpf_object *obj;
  364. struct bpf_map *map;
  365. char filename[256];
  366. int i = 0;
  367. if (setrlimit(RLIMIT_MEMLOCK, &r)) {
  368. perror("setrlimit(RLIMIT_MEMLOCK)");
  369. return 1;
  370. }
  371. if (argc > 1)
  372. test_flags = atoi(argv[1]) ? : test_flags;
  373. if (argc > 2)
  374. nr_cpus = atoi(argv[2]) ? : nr_cpus;
  375. if (argc > 3)
  376. num_map_entries = atoi(argv[3]);
  377. if (argc > 4)
  378. max_cnt = atoi(argv[4]);
  379. snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
  380. obj = bpf_object__open_file(filename, NULL);
  381. if (libbpf_get_error(obj)) {
  382. fprintf(stderr, "ERROR: opening BPF object file failed\n");
  383. return 0;
  384. }
  385. map = bpf_object__find_map_by_name(obj, "inner_lru_hash_map");
  386. if (libbpf_get_error(map)) {
  387. fprintf(stderr, "ERROR: finding a map in obj file failed\n");
  388. goto cleanup;
  389. }
  390. inner_lru_hash_size = bpf_map__max_entries(map);
  391. if (!inner_lru_hash_size) {
  392. fprintf(stderr, "ERROR: failed to get map attribute\n");
  393. goto cleanup;
  394. }
  395. /* resize BPF map prior to loading */
  396. if (num_map_entries > 0)
  397. fixup_map(obj);
  398. /* load BPF program */
  399. if (bpf_object__load(obj)) {
  400. fprintf(stderr, "ERROR: loading BPF object file failed\n");
  401. goto cleanup;
  402. }
  403. map_fd[0] = bpf_object__find_map_fd_by_name(obj, "array_of_lru_hashs");
  404. map_fd[1] = bpf_object__find_map_fd_by_name(obj, "hash_map_alloc");
  405. map_fd[2] = bpf_object__find_map_fd_by_name(obj, "lru_hash_lookup_map");
  406. if (map_fd[0] < 0 || map_fd[1] < 0 || map_fd[2] < 0) {
  407. fprintf(stderr, "ERROR: finding a map in obj file failed\n");
  408. goto cleanup;
  409. }
  410. bpf_object__for_each_program(prog, obj) {
  411. links[i] = bpf_program__attach(prog);
  412. if (libbpf_get_error(links[i])) {
  413. fprintf(stderr, "ERROR: bpf_program__attach failed\n");
  414. links[i] = NULL;
  415. goto cleanup;
  416. }
  417. i++;
  418. }
  419. fill_lpm_trie();
  420. run_perf_test(nr_cpus);
  421. cleanup:
  422. for (i--; i >= 0; i--)
  423. bpf_link__destroy(links[i]);
  424. bpf_object__close(obj);
  425. return 0;
  426. }