xskmap.c 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* XSKMAP used for AF_XDP sockets
  3. * Copyright(c) 2018 Intel Corporation.
  4. */
  5. #include <linux/bpf.h>
  6. #include <linux/capability.h>
  7. #include <net/xdp_sock.h>
  8. #include <linux/slab.h>
  9. #include <linux/sched.h>
  10. #include "xsk.h"
  11. int xsk_map_inc(struct xsk_map *map)
  12. {
  13. bpf_map_inc(&map->map);
  14. return 0;
  15. }
  16. void xsk_map_put(struct xsk_map *map)
  17. {
  18. bpf_map_put(&map->map);
  19. }
  20. static struct xsk_map_node *xsk_map_node_alloc(struct xsk_map *map,
  21. struct xdp_sock **map_entry)
  22. {
  23. struct xsk_map_node *node;
  24. int err;
  25. node = kzalloc(sizeof(*node), GFP_ATOMIC | __GFP_NOWARN);
  26. if (!node)
  27. return ERR_PTR(-ENOMEM);
  28. err = xsk_map_inc(map);
  29. if (err) {
  30. kfree(node);
  31. return ERR_PTR(err);
  32. }
  33. node->map = map;
  34. node->map_entry = map_entry;
  35. return node;
  36. }
  37. static void xsk_map_node_free(struct xsk_map_node *node)
  38. {
  39. xsk_map_put(node->map);
  40. kfree(node);
  41. }
  42. static void xsk_map_sock_add(struct xdp_sock *xs, struct xsk_map_node *node)
  43. {
  44. spin_lock_bh(&xs->map_list_lock);
  45. list_add_tail(&node->node, &xs->map_list);
  46. spin_unlock_bh(&xs->map_list_lock);
  47. }
  48. static void xsk_map_sock_delete(struct xdp_sock *xs,
  49. struct xdp_sock **map_entry)
  50. {
  51. struct xsk_map_node *n, *tmp;
  52. spin_lock_bh(&xs->map_list_lock);
  53. list_for_each_entry_safe(n, tmp, &xs->map_list, node) {
  54. if (map_entry == n->map_entry) {
  55. list_del(&n->node);
  56. xsk_map_node_free(n);
  57. }
  58. }
  59. spin_unlock_bh(&xs->map_list_lock);
  60. }
  61. static struct bpf_map *xsk_map_alloc(union bpf_attr *attr)
  62. {
  63. struct bpf_map_memory mem;
  64. int err, numa_node;
  65. struct xsk_map *m;
  66. u64 size;
  67. if (!capable(CAP_NET_ADMIN))
  68. return ERR_PTR(-EPERM);
  69. if (attr->max_entries == 0 || attr->key_size != 4 ||
  70. attr->value_size != 4 ||
  71. attr->map_flags & ~(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY))
  72. return ERR_PTR(-EINVAL);
  73. numa_node = bpf_map_attr_numa_node(attr);
  74. size = struct_size(m, xsk_map, attr->max_entries);
  75. err = bpf_map_charge_init(&mem, size);
  76. if (err < 0)
  77. return ERR_PTR(err);
  78. m = bpf_map_area_alloc(size, numa_node);
  79. if (!m) {
  80. bpf_map_charge_finish(&mem);
  81. return ERR_PTR(-ENOMEM);
  82. }
  83. bpf_map_init_from_attr(&m->map, attr);
  84. bpf_map_charge_move(&m->map.memory, &mem);
  85. spin_lock_init(&m->lock);
  86. return &m->map;
  87. }
  88. static void xsk_map_free(struct bpf_map *map)
  89. {
  90. struct xsk_map *m = container_of(map, struct xsk_map, map);
  91. bpf_clear_redirect_map(map);
  92. synchronize_net();
  93. bpf_map_area_free(m);
  94. }
  95. static int xsk_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
  96. {
  97. struct xsk_map *m = container_of(map, struct xsk_map, map);
  98. u32 index = key ? *(u32 *)key : U32_MAX;
  99. u32 *next = next_key;
  100. if (index >= m->map.max_entries) {
  101. *next = 0;
  102. return 0;
  103. }
  104. if (index == m->map.max_entries - 1)
  105. return -ENOENT;
  106. *next = index + 1;
  107. return 0;
  108. }
  109. static int xsk_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
  110. {
  111. const int ret = BPF_REG_0, mp = BPF_REG_1, index = BPF_REG_2;
  112. struct bpf_insn *insn = insn_buf;
  113. *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
  114. *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5);
  115. *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(sizeof(struct xsk_sock *)));
  116. *insn++ = BPF_ALU64_IMM(BPF_ADD, mp, offsetof(struct xsk_map, xsk_map));
  117. *insn++ = BPF_ALU64_REG(BPF_ADD, ret, mp);
  118. *insn++ = BPF_LDX_MEM(BPF_SIZEOF(struct xsk_sock *), ret, ret, 0);
  119. *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
  120. *insn++ = BPF_MOV64_IMM(ret, 0);
  121. return insn - insn_buf;
  122. }
  123. static void *xsk_map_lookup_elem(struct bpf_map *map, void *key)
  124. {
  125. WARN_ON_ONCE(!rcu_read_lock_held());
  126. return __xsk_map_lookup_elem(map, *(u32 *)key);
  127. }
  128. static void *xsk_map_lookup_elem_sys_only(struct bpf_map *map, void *key)
  129. {
  130. return ERR_PTR(-EOPNOTSUPP);
  131. }
  132. static int xsk_map_update_elem(struct bpf_map *map, void *key, void *value,
  133. u64 map_flags)
  134. {
  135. struct xsk_map *m = container_of(map, struct xsk_map, map);
  136. struct xdp_sock *xs, *old_xs, **map_entry;
  137. u32 i = *(u32 *)key, fd = *(u32 *)value;
  138. struct xsk_map_node *node;
  139. struct socket *sock;
  140. int err;
  141. if (unlikely(map_flags > BPF_EXIST))
  142. return -EINVAL;
  143. if (unlikely(i >= m->map.max_entries))
  144. return -E2BIG;
  145. sock = sockfd_lookup(fd, &err);
  146. if (!sock)
  147. return err;
  148. if (sock->sk->sk_family != PF_XDP) {
  149. sockfd_put(sock);
  150. return -EOPNOTSUPP;
  151. }
  152. xs = (struct xdp_sock *)sock->sk;
  153. map_entry = &m->xsk_map[i];
  154. node = xsk_map_node_alloc(m, map_entry);
  155. if (IS_ERR(node)) {
  156. sockfd_put(sock);
  157. return PTR_ERR(node);
  158. }
  159. spin_lock_bh(&m->lock);
  160. old_xs = READ_ONCE(*map_entry);
  161. if (old_xs == xs) {
  162. err = 0;
  163. goto out;
  164. } else if (old_xs && map_flags == BPF_NOEXIST) {
  165. err = -EEXIST;
  166. goto out;
  167. } else if (!old_xs && map_flags == BPF_EXIST) {
  168. err = -ENOENT;
  169. goto out;
  170. }
  171. xsk_map_sock_add(xs, node);
  172. WRITE_ONCE(*map_entry, xs);
  173. if (old_xs)
  174. xsk_map_sock_delete(old_xs, map_entry);
  175. spin_unlock_bh(&m->lock);
  176. sockfd_put(sock);
  177. return 0;
  178. out:
  179. spin_unlock_bh(&m->lock);
  180. sockfd_put(sock);
  181. xsk_map_node_free(node);
  182. return err;
  183. }
  184. static int xsk_map_delete_elem(struct bpf_map *map, void *key)
  185. {
  186. struct xsk_map *m = container_of(map, struct xsk_map, map);
  187. struct xdp_sock *old_xs, **map_entry;
  188. int k = *(u32 *)key;
  189. if (k >= map->max_entries)
  190. return -EINVAL;
  191. spin_lock_bh(&m->lock);
  192. map_entry = &m->xsk_map[k];
  193. old_xs = xchg(map_entry, NULL);
  194. if (old_xs)
  195. xsk_map_sock_delete(old_xs, map_entry);
  196. spin_unlock_bh(&m->lock);
  197. return 0;
  198. }
  199. void xsk_map_try_sock_delete(struct xsk_map *map, struct xdp_sock *xs,
  200. struct xdp_sock **map_entry)
  201. {
  202. spin_lock_bh(&map->lock);
  203. if (READ_ONCE(*map_entry) == xs) {
  204. WRITE_ONCE(*map_entry, NULL);
  205. xsk_map_sock_delete(xs, map_entry);
  206. }
  207. spin_unlock_bh(&map->lock);
  208. }
  209. static bool xsk_map_meta_equal(const struct bpf_map *meta0,
  210. const struct bpf_map *meta1)
  211. {
  212. return meta0->max_entries == meta1->max_entries &&
  213. bpf_map_meta_equal(meta0, meta1);
  214. }
  215. static int xsk_map_btf_id;
  216. const struct bpf_map_ops xsk_map_ops = {
  217. .map_meta_equal = xsk_map_meta_equal,
  218. .map_alloc = xsk_map_alloc,
  219. .map_free = xsk_map_free,
  220. .map_get_next_key = xsk_map_get_next_key,
  221. .map_lookup_elem = xsk_map_lookup_elem,
  222. .map_gen_lookup = xsk_map_gen_lookup,
  223. .map_lookup_elem_sys_only = xsk_map_lookup_elem_sys_only,
  224. .map_update_elem = xsk_map_update_elem,
  225. .map_delete_elem = xsk_map_delete_elem,
  226. .map_check_btf = map_check_no_btf,
  227. .map_btf_name = "xsk_map",
  228. .map_btf_id = &xsk_map_btf_id,
  229. };