arraymap.c 34 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
  3. * Copyright (c) 2016,2017 Facebook
  4. */
  5. #include <linux/bpf.h>
  6. #include <linux/btf.h>
  7. #include <linux/err.h>
  8. #include <linux/slab.h>
  9. #include <linux/mm.h>
  10. #include <linux/filter.h>
  11. #include <linux/perf_event.h>
  12. #include <uapi/linux/btf.h>
  13. #include <linux/rcupdate_trace.h>
  14. #include "map_in_map.h"
  15. #define ARRAY_CREATE_FLAG_MASK \
  16. (BPF_F_NUMA_NODE | BPF_F_MMAPABLE | BPF_F_ACCESS_MASK | \
  17. BPF_F_PRESERVE_ELEMS | BPF_F_INNER_MAP)
  18. static void bpf_array_free_percpu(struct bpf_array *array)
  19. {
  20. int i;
  21. for (i = 0; i < array->map.max_entries; i++) {
  22. free_percpu(array->pptrs[i]);
  23. cond_resched();
  24. }
  25. }
  26. static int bpf_array_alloc_percpu(struct bpf_array *array)
  27. {
  28. void __percpu *ptr;
  29. int i;
  30. for (i = 0; i < array->map.max_entries; i++) {
  31. ptr = __alloc_percpu_gfp(array->elem_size, 8,
  32. GFP_USER | __GFP_NOWARN);
  33. if (!ptr) {
  34. bpf_array_free_percpu(array);
  35. return -ENOMEM;
  36. }
  37. array->pptrs[i] = ptr;
  38. cond_resched();
  39. }
  40. return 0;
  41. }
  42. /* Called from syscall */
  43. int array_map_alloc_check(union bpf_attr *attr)
  44. {
  45. bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
  46. int numa_node = bpf_map_attr_numa_node(attr);
  47. /* check sanity of attributes */
  48. if (attr->max_entries == 0 || attr->key_size != 4 ||
  49. attr->value_size == 0 ||
  50. attr->map_flags & ~ARRAY_CREATE_FLAG_MASK ||
  51. !bpf_map_flags_access_ok(attr->map_flags) ||
  52. (percpu && numa_node != NUMA_NO_NODE))
  53. return -EINVAL;
  54. if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
  55. attr->map_flags & (BPF_F_MMAPABLE | BPF_F_INNER_MAP))
  56. return -EINVAL;
  57. if (attr->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY &&
  58. attr->map_flags & BPF_F_PRESERVE_ELEMS)
  59. return -EINVAL;
  60. if (attr->value_size > KMALLOC_MAX_SIZE)
  61. /* if value_size is bigger, the user space won't be able to
  62. * access the elements.
  63. */
  64. return -E2BIG;
  65. return 0;
  66. }
  67. static struct bpf_map *array_map_alloc(union bpf_attr *attr)
  68. {
  69. bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
  70. int ret, numa_node = bpf_map_attr_numa_node(attr);
  71. u32 elem_size, index_mask, max_entries;
  72. bool bypass_spec_v1 = bpf_bypass_spec_v1();
  73. u64 cost, array_size, mask64;
  74. struct bpf_map_memory mem;
  75. struct bpf_array *array;
  76. elem_size = round_up(attr->value_size, 8);
  77. max_entries = attr->max_entries;
  78. /* On 32 bit archs roundup_pow_of_two() with max_entries that has
  79. * upper most bit set in u32 space is undefined behavior due to
  80. * resulting 1U << 32, so do it manually here in u64 space.
  81. */
  82. mask64 = fls_long(max_entries - 1);
  83. mask64 = 1ULL << mask64;
  84. mask64 -= 1;
  85. index_mask = mask64;
  86. if (!bypass_spec_v1) {
  87. /* round up array size to nearest power of 2,
  88. * since cpu will speculate within index_mask limits
  89. */
  90. max_entries = index_mask + 1;
  91. /* Check for overflows. */
  92. if (max_entries < attr->max_entries)
  93. return ERR_PTR(-E2BIG);
  94. }
  95. array_size = sizeof(*array);
  96. if (percpu) {
  97. array_size += (u64) max_entries * sizeof(void *);
  98. } else {
  99. /* rely on vmalloc() to return page-aligned memory and
  100. * ensure array->value is exactly page-aligned
  101. */
  102. if (attr->map_flags & BPF_F_MMAPABLE) {
  103. array_size = PAGE_ALIGN(array_size);
  104. array_size += PAGE_ALIGN((u64) max_entries * elem_size);
  105. } else {
  106. array_size += (u64) max_entries * elem_size;
  107. }
  108. }
  109. /* make sure there is no u32 overflow later in round_up() */
  110. cost = array_size;
  111. if (percpu)
  112. cost += (u64)attr->max_entries * elem_size * num_possible_cpus();
  113. ret = bpf_map_charge_init(&mem, cost);
  114. if (ret < 0)
  115. return ERR_PTR(ret);
  116. /* allocate all map elements and zero-initialize them */
  117. if (attr->map_flags & BPF_F_MMAPABLE) {
  118. void *data;
  119. /* kmalloc'ed memory can't be mmap'ed, use explicit vmalloc */
  120. data = bpf_map_area_mmapable_alloc(array_size, numa_node);
  121. if (!data) {
  122. bpf_map_charge_finish(&mem);
  123. return ERR_PTR(-ENOMEM);
  124. }
  125. array = data + PAGE_ALIGN(sizeof(struct bpf_array))
  126. - offsetof(struct bpf_array, value);
  127. } else {
  128. array = bpf_map_area_alloc(array_size, numa_node);
  129. }
  130. if (!array) {
  131. bpf_map_charge_finish(&mem);
  132. return ERR_PTR(-ENOMEM);
  133. }
  134. array->index_mask = index_mask;
  135. array->map.bypass_spec_v1 = bypass_spec_v1;
  136. /* copy mandatory map attributes */
  137. bpf_map_init_from_attr(&array->map, attr);
  138. bpf_map_charge_move(&array->map.memory, &mem);
  139. array->elem_size = elem_size;
  140. if (percpu && bpf_array_alloc_percpu(array)) {
  141. bpf_map_charge_finish(&array->map.memory);
  142. bpf_map_area_free(array);
  143. return ERR_PTR(-ENOMEM);
  144. }
  145. return &array->map;
  146. }
  147. /* Called from syscall or from eBPF program */
  148. static void *array_map_lookup_elem(struct bpf_map *map, void *key)
  149. {
  150. struct bpf_array *array = container_of(map, struct bpf_array, map);
  151. u32 index = *(u32 *)key;
  152. if (unlikely(index >= array->map.max_entries))
  153. return NULL;
  154. return array->value + array->elem_size * (index & array->index_mask);
  155. }
  156. static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm,
  157. u32 off)
  158. {
  159. struct bpf_array *array = container_of(map, struct bpf_array, map);
  160. if (map->max_entries != 1)
  161. return -ENOTSUPP;
  162. if (off >= map->value_size)
  163. return -EINVAL;
  164. *imm = (unsigned long)array->value;
  165. return 0;
  166. }
  167. static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm,
  168. u32 *off)
  169. {
  170. struct bpf_array *array = container_of(map, struct bpf_array, map);
  171. u64 base = (unsigned long)array->value;
  172. u64 range = array->elem_size;
  173. if (map->max_entries != 1)
  174. return -ENOTSUPP;
  175. if (imm < base || imm >= base + range)
  176. return -ENOENT;
  177. *off = imm - base;
  178. return 0;
  179. }
  180. /* emit BPF instructions equivalent to C code of array_map_lookup_elem() */
  181. static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
  182. {
  183. struct bpf_array *array = container_of(map, struct bpf_array, map);
  184. struct bpf_insn *insn = insn_buf;
  185. u32 elem_size = round_up(map->value_size, 8);
  186. const int ret = BPF_REG_0;
  187. const int map_ptr = BPF_REG_1;
  188. const int index = BPF_REG_2;
  189. if (map->map_flags & BPF_F_INNER_MAP)
  190. return -EOPNOTSUPP;
  191. *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
  192. *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
  193. if (!map->bypass_spec_v1) {
  194. *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4);
  195. *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);
  196. } else {
  197. *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3);
  198. }
  199. if (is_power_of_2(elem_size)) {
  200. *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size));
  201. } else {
  202. *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size);
  203. }
  204. *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr);
  205. *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
  206. *insn++ = BPF_MOV64_IMM(ret, 0);
  207. return insn - insn_buf;
  208. }
  209. /* Called from eBPF program */
  210. static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key)
  211. {
  212. struct bpf_array *array = container_of(map, struct bpf_array, map);
  213. u32 index = *(u32 *)key;
  214. if (unlikely(index >= array->map.max_entries))
  215. return NULL;
  216. return this_cpu_ptr(array->pptrs[index & array->index_mask]);
  217. }
  218. int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)
  219. {
  220. struct bpf_array *array = container_of(map, struct bpf_array, map);
  221. u32 index = *(u32 *)key;
  222. void __percpu *pptr;
  223. int cpu, off = 0;
  224. u32 size;
  225. if (unlikely(index >= array->map.max_entries))
  226. return -ENOENT;
  227. /* per_cpu areas are zero-filled and bpf programs can only
  228. * access 'value_size' of them, so copying rounded areas
  229. * will not leak any kernel data
  230. */
  231. size = round_up(map->value_size, 8);
  232. rcu_read_lock();
  233. pptr = array->pptrs[index & array->index_mask];
  234. for_each_possible_cpu(cpu) {
  235. bpf_long_memcpy(value + off, per_cpu_ptr(pptr, cpu), size);
  236. off += size;
  237. }
  238. rcu_read_unlock();
  239. return 0;
  240. }
  241. /* Called from syscall */
  242. static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
  243. {
  244. struct bpf_array *array = container_of(map, struct bpf_array, map);
  245. u32 index = key ? *(u32 *)key : U32_MAX;
  246. u32 *next = (u32 *)next_key;
  247. if (index >= array->map.max_entries) {
  248. *next = 0;
  249. return 0;
  250. }
  251. if (index == array->map.max_entries - 1)
  252. return -ENOENT;
  253. *next = index + 1;
  254. return 0;
  255. }
  256. /* Called from syscall or from eBPF program */
  257. static int array_map_update_elem(struct bpf_map *map, void *key, void *value,
  258. u64 map_flags)
  259. {
  260. struct bpf_array *array = container_of(map, struct bpf_array, map);
  261. u32 index = *(u32 *)key;
  262. char *val;
  263. if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST))
  264. /* unknown flags */
  265. return -EINVAL;
  266. if (unlikely(index >= array->map.max_entries))
  267. /* all elements were pre-allocated, cannot insert a new one */
  268. return -E2BIG;
  269. if (unlikely(map_flags & BPF_NOEXIST))
  270. /* all elements already exist */
  271. return -EEXIST;
  272. if (unlikely((map_flags & BPF_F_LOCK) &&
  273. !map_value_has_spin_lock(map)))
  274. return -EINVAL;
  275. if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
  276. memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]),
  277. value, map->value_size);
  278. } else {
  279. val = array->value +
  280. array->elem_size * (index & array->index_mask);
  281. if (map_flags & BPF_F_LOCK)
  282. copy_map_value_locked(map, val, value, false);
  283. else
  284. copy_map_value(map, val, value);
  285. }
  286. return 0;
  287. }
  288. int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
  289. u64 map_flags)
  290. {
  291. struct bpf_array *array = container_of(map, struct bpf_array, map);
  292. u32 index = *(u32 *)key;
  293. void __percpu *pptr;
  294. int cpu, off = 0;
  295. u32 size;
  296. if (unlikely(map_flags > BPF_EXIST))
  297. /* unknown flags */
  298. return -EINVAL;
  299. if (unlikely(index >= array->map.max_entries))
  300. /* all elements were pre-allocated, cannot insert a new one */
  301. return -E2BIG;
  302. if (unlikely(map_flags == BPF_NOEXIST))
  303. /* all elements already exist */
  304. return -EEXIST;
  305. /* the user space will provide round_up(value_size, 8) bytes that
  306. * will be copied into per-cpu area. bpf programs can only access
  307. * value_size of it. During lookup the same extra bytes will be
  308. * returned or zeros which were zero-filled by percpu_alloc,
  309. * so no kernel data leaks possible
  310. */
  311. size = round_up(map->value_size, 8);
  312. rcu_read_lock();
  313. pptr = array->pptrs[index & array->index_mask];
  314. for_each_possible_cpu(cpu) {
  315. bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off, size);
  316. off += size;
  317. }
  318. rcu_read_unlock();
  319. return 0;
  320. }
  321. /* Called from syscall or from eBPF program */
  322. static int array_map_delete_elem(struct bpf_map *map, void *key)
  323. {
  324. return -EINVAL;
  325. }
  326. static void *array_map_vmalloc_addr(struct bpf_array *array)
  327. {
  328. return (void *)round_down((unsigned long)array, PAGE_SIZE);
  329. }
  330. /* Called when map->refcnt goes to zero, either from workqueue or from syscall */
  331. static void array_map_free(struct bpf_map *map)
  332. {
  333. struct bpf_array *array = container_of(map, struct bpf_array, map);
  334. if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
  335. bpf_array_free_percpu(array);
  336. if (array->map.map_flags & BPF_F_MMAPABLE)
  337. bpf_map_area_free(array_map_vmalloc_addr(array));
  338. else
  339. bpf_map_area_free(array);
  340. }
  341. static void array_map_seq_show_elem(struct bpf_map *map, void *key,
  342. struct seq_file *m)
  343. {
  344. void *value;
  345. rcu_read_lock();
  346. value = array_map_lookup_elem(map, key);
  347. if (!value) {
  348. rcu_read_unlock();
  349. return;
  350. }
  351. if (map->btf_key_type_id)
  352. seq_printf(m, "%u: ", *(u32 *)key);
  353. btf_type_seq_show(map->btf, map->btf_value_type_id, value, m);
  354. seq_puts(m, "\n");
  355. rcu_read_unlock();
  356. }
  357. static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key,
  358. struct seq_file *m)
  359. {
  360. struct bpf_array *array = container_of(map, struct bpf_array, map);
  361. u32 index = *(u32 *)key;
  362. void __percpu *pptr;
  363. int cpu;
  364. rcu_read_lock();
  365. seq_printf(m, "%u: {\n", *(u32 *)key);
  366. pptr = array->pptrs[index & array->index_mask];
  367. for_each_possible_cpu(cpu) {
  368. seq_printf(m, "\tcpu%d: ", cpu);
  369. btf_type_seq_show(map->btf, map->btf_value_type_id,
  370. per_cpu_ptr(pptr, cpu), m);
  371. seq_puts(m, "\n");
  372. }
  373. seq_puts(m, "}\n");
  374. rcu_read_unlock();
  375. }
  376. static int array_map_check_btf(const struct bpf_map *map,
  377. const struct btf *btf,
  378. const struct btf_type *key_type,
  379. const struct btf_type *value_type)
  380. {
  381. u32 int_data;
  382. /* One exception for keyless BTF: .bss/.data/.rodata map */
  383. if (btf_type_is_void(key_type)) {
  384. if (map->map_type != BPF_MAP_TYPE_ARRAY ||
  385. map->max_entries != 1)
  386. return -EINVAL;
  387. if (BTF_INFO_KIND(value_type->info) != BTF_KIND_DATASEC)
  388. return -EINVAL;
  389. return 0;
  390. }
  391. if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT)
  392. return -EINVAL;
  393. int_data = *(u32 *)(key_type + 1);
  394. /* bpf array can only take a u32 key. This check makes sure
  395. * that the btf matches the attr used during map_create.
  396. */
  397. if (BTF_INT_BITS(int_data) != 32 || BTF_INT_OFFSET(int_data))
  398. return -EINVAL;
  399. return 0;
  400. }
  401. static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
  402. {
  403. struct bpf_array *array = container_of(map, struct bpf_array, map);
  404. pgoff_t pgoff = PAGE_ALIGN(sizeof(*array)) >> PAGE_SHIFT;
  405. if (!(map->map_flags & BPF_F_MMAPABLE))
  406. return -EINVAL;
  407. if (vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) >
  408. PAGE_ALIGN((u64)array->map.max_entries * array->elem_size))
  409. return -EINVAL;
  410. return remap_vmalloc_range(vma, array_map_vmalloc_addr(array),
  411. vma->vm_pgoff + pgoff);
  412. }
  413. static bool array_map_meta_equal(const struct bpf_map *meta0,
  414. const struct bpf_map *meta1)
  415. {
  416. if (!bpf_map_meta_equal(meta0, meta1))
  417. return false;
  418. return meta0->map_flags & BPF_F_INNER_MAP ? true :
  419. meta0->max_entries == meta1->max_entries;
  420. }
  421. struct bpf_iter_seq_array_map_info {
  422. struct bpf_map *map;
  423. void *percpu_value_buf;
  424. u32 index;
  425. };
  426. static void *bpf_array_map_seq_start(struct seq_file *seq, loff_t *pos)
  427. {
  428. struct bpf_iter_seq_array_map_info *info = seq->private;
  429. struct bpf_map *map = info->map;
  430. struct bpf_array *array;
  431. u32 index;
  432. if (info->index >= map->max_entries)
  433. return NULL;
  434. if (*pos == 0)
  435. ++*pos;
  436. array = container_of(map, struct bpf_array, map);
  437. index = info->index & array->index_mask;
  438. if (info->percpu_value_buf)
  439. return array->pptrs[index];
  440. return array->value + array->elem_size * index;
  441. }
  442. static void *bpf_array_map_seq_next(struct seq_file *seq, void *v, loff_t *pos)
  443. {
  444. struct bpf_iter_seq_array_map_info *info = seq->private;
  445. struct bpf_map *map = info->map;
  446. struct bpf_array *array;
  447. u32 index;
  448. ++*pos;
  449. ++info->index;
  450. if (info->index >= map->max_entries)
  451. return NULL;
  452. array = container_of(map, struct bpf_array, map);
  453. index = info->index & array->index_mask;
  454. if (info->percpu_value_buf)
  455. return array->pptrs[index];
  456. return array->value + array->elem_size * index;
  457. }
  458. static int __bpf_array_map_seq_show(struct seq_file *seq, void *v)
  459. {
  460. struct bpf_iter_seq_array_map_info *info = seq->private;
  461. struct bpf_iter__bpf_map_elem ctx = {};
  462. struct bpf_map *map = info->map;
  463. struct bpf_iter_meta meta;
  464. struct bpf_prog *prog;
  465. int off = 0, cpu = 0;
  466. void __percpu **pptr;
  467. u32 size;
  468. meta.seq = seq;
  469. prog = bpf_iter_get_info(&meta, v == NULL);
  470. if (!prog)
  471. return 0;
  472. ctx.meta = &meta;
  473. ctx.map = info->map;
  474. if (v) {
  475. ctx.key = &info->index;
  476. if (!info->percpu_value_buf) {
  477. ctx.value = v;
  478. } else {
  479. pptr = v;
  480. size = round_up(map->value_size, 8);
  481. for_each_possible_cpu(cpu) {
  482. bpf_long_memcpy(info->percpu_value_buf + off,
  483. per_cpu_ptr(pptr, cpu),
  484. size);
  485. off += size;
  486. }
  487. ctx.value = info->percpu_value_buf;
  488. }
  489. }
  490. return bpf_iter_run_prog(prog, &ctx);
  491. }
  492. static int bpf_array_map_seq_show(struct seq_file *seq, void *v)
  493. {
  494. return __bpf_array_map_seq_show(seq, v);
  495. }
  496. static void bpf_array_map_seq_stop(struct seq_file *seq, void *v)
  497. {
  498. if (!v)
  499. (void)__bpf_array_map_seq_show(seq, NULL);
  500. }
  501. static int bpf_iter_init_array_map(void *priv_data,
  502. struct bpf_iter_aux_info *aux)
  503. {
  504. struct bpf_iter_seq_array_map_info *seq_info = priv_data;
  505. struct bpf_map *map = aux->map;
  506. void *value_buf;
  507. u32 buf_size;
  508. if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
  509. buf_size = round_up(map->value_size, 8) * num_possible_cpus();
  510. value_buf = kmalloc(buf_size, GFP_USER | __GFP_NOWARN);
  511. if (!value_buf)
  512. return -ENOMEM;
  513. seq_info->percpu_value_buf = value_buf;
  514. }
  515. seq_info->map = map;
  516. return 0;
  517. }
  518. static void bpf_iter_fini_array_map(void *priv_data)
  519. {
  520. struct bpf_iter_seq_array_map_info *seq_info = priv_data;
  521. kfree(seq_info->percpu_value_buf);
  522. }
  523. static const struct seq_operations bpf_array_map_seq_ops = {
  524. .start = bpf_array_map_seq_start,
  525. .next = bpf_array_map_seq_next,
  526. .stop = bpf_array_map_seq_stop,
  527. .show = bpf_array_map_seq_show,
  528. };
  529. static const struct bpf_iter_seq_info iter_seq_info = {
  530. .seq_ops = &bpf_array_map_seq_ops,
  531. .init_seq_private = bpf_iter_init_array_map,
  532. .fini_seq_private = bpf_iter_fini_array_map,
  533. .seq_priv_size = sizeof(struct bpf_iter_seq_array_map_info),
  534. };
  535. static int array_map_btf_id;
  536. const struct bpf_map_ops array_map_ops = {
  537. .map_meta_equal = array_map_meta_equal,
  538. .map_alloc_check = array_map_alloc_check,
  539. .map_alloc = array_map_alloc,
  540. .map_free = array_map_free,
  541. .map_get_next_key = array_map_get_next_key,
  542. .map_lookup_elem = array_map_lookup_elem,
  543. .map_update_elem = array_map_update_elem,
  544. .map_delete_elem = array_map_delete_elem,
  545. .map_gen_lookup = array_map_gen_lookup,
  546. .map_direct_value_addr = array_map_direct_value_addr,
  547. .map_direct_value_meta = array_map_direct_value_meta,
  548. .map_mmap = array_map_mmap,
  549. .map_seq_show_elem = array_map_seq_show_elem,
  550. .map_check_btf = array_map_check_btf,
  551. .map_lookup_batch = generic_map_lookup_batch,
  552. .map_update_batch = generic_map_update_batch,
  553. .map_btf_name = "bpf_array",
  554. .map_btf_id = &array_map_btf_id,
  555. .iter_seq_info = &iter_seq_info,
  556. };
  557. static int percpu_array_map_btf_id;
  558. const struct bpf_map_ops percpu_array_map_ops = {
  559. .map_meta_equal = bpf_map_meta_equal,
  560. .map_alloc_check = array_map_alloc_check,
  561. .map_alloc = array_map_alloc,
  562. .map_free = array_map_free,
  563. .map_get_next_key = array_map_get_next_key,
  564. .map_lookup_elem = percpu_array_map_lookup_elem,
  565. .map_update_elem = array_map_update_elem,
  566. .map_delete_elem = array_map_delete_elem,
  567. .map_seq_show_elem = percpu_array_map_seq_show_elem,
  568. .map_check_btf = array_map_check_btf,
  569. .map_btf_name = "bpf_array",
  570. .map_btf_id = &percpu_array_map_btf_id,
  571. .iter_seq_info = &iter_seq_info,
  572. };
  573. static int fd_array_map_alloc_check(union bpf_attr *attr)
  574. {
  575. /* only file descriptors can be stored in this type of map */
  576. if (attr->value_size != sizeof(u32))
  577. return -EINVAL;
  578. /* Program read-only/write-only not supported for special maps yet. */
  579. if (attr->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG))
  580. return -EINVAL;
  581. return array_map_alloc_check(attr);
  582. }
  583. static void fd_array_map_free(struct bpf_map *map)
  584. {
  585. struct bpf_array *array = container_of(map, struct bpf_array, map);
  586. int i;
  587. /* make sure it's empty */
  588. for (i = 0; i < array->map.max_entries; i++)
  589. BUG_ON(array->ptrs[i] != NULL);
  590. bpf_map_area_free(array);
  591. }
  592. static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key)
  593. {
  594. return ERR_PTR(-EOPNOTSUPP);
  595. }
  596. /* only called from syscall */
  597. int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value)
  598. {
  599. void **elem, *ptr;
  600. int ret = 0;
  601. if (!map->ops->map_fd_sys_lookup_elem)
  602. return -ENOTSUPP;
  603. rcu_read_lock();
  604. elem = array_map_lookup_elem(map, key);
  605. if (elem && (ptr = READ_ONCE(*elem)))
  606. *value = map->ops->map_fd_sys_lookup_elem(ptr);
  607. else
  608. ret = -ENOENT;
  609. rcu_read_unlock();
  610. return ret;
  611. }
  612. /* only called from syscall */
  613. int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
  614. void *key, void *value, u64 map_flags)
  615. {
  616. struct bpf_array *array = container_of(map, struct bpf_array, map);
  617. void *new_ptr, *old_ptr;
  618. u32 index = *(u32 *)key, ufd;
  619. if (map_flags != BPF_ANY)
  620. return -EINVAL;
  621. if (index >= array->map.max_entries)
  622. return -E2BIG;
  623. ufd = *(u32 *)value;
  624. new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd);
  625. if (IS_ERR(new_ptr))
  626. return PTR_ERR(new_ptr);
  627. if (map->ops->map_poke_run) {
  628. mutex_lock(&array->aux->poke_mutex);
  629. old_ptr = xchg(array->ptrs + index, new_ptr);
  630. map->ops->map_poke_run(map, index, old_ptr, new_ptr);
  631. mutex_unlock(&array->aux->poke_mutex);
  632. } else {
  633. old_ptr = xchg(array->ptrs + index, new_ptr);
  634. }
  635. if (old_ptr)
  636. map->ops->map_fd_put_ptr(old_ptr);
  637. return 0;
  638. }
  639. static int fd_array_map_delete_elem(struct bpf_map *map, void *key)
  640. {
  641. struct bpf_array *array = container_of(map, struct bpf_array, map);
  642. void *old_ptr;
  643. u32 index = *(u32 *)key;
  644. if (index >= array->map.max_entries)
  645. return -E2BIG;
  646. if (map->ops->map_poke_run) {
  647. mutex_lock(&array->aux->poke_mutex);
  648. old_ptr = xchg(array->ptrs + index, NULL);
  649. map->ops->map_poke_run(map, index, old_ptr, NULL);
  650. mutex_unlock(&array->aux->poke_mutex);
  651. } else {
  652. old_ptr = xchg(array->ptrs + index, NULL);
  653. }
  654. if (old_ptr) {
  655. map->ops->map_fd_put_ptr(old_ptr);
  656. return 0;
  657. } else {
  658. return -ENOENT;
  659. }
  660. }
  661. static void *prog_fd_array_get_ptr(struct bpf_map *map,
  662. struct file *map_file, int fd)
  663. {
  664. struct bpf_array *array = container_of(map, struct bpf_array, map);
  665. struct bpf_prog *prog = bpf_prog_get(fd);
  666. if (IS_ERR(prog))
  667. return prog;
  668. if (!bpf_prog_array_compatible(array, prog)) {
  669. bpf_prog_put(prog);
  670. return ERR_PTR(-EINVAL);
  671. }
  672. return prog;
  673. }
  674. static void prog_fd_array_put_ptr(void *ptr)
  675. {
  676. bpf_prog_put(ptr);
  677. }
  678. static u32 prog_fd_array_sys_lookup_elem(void *ptr)
  679. {
  680. return ((struct bpf_prog *)ptr)->aux->id;
  681. }
  682. /* decrement refcnt of all bpf_progs that are stored in this map */
  683. static void bpf_fd_array_map_clear(struct bpf_map *map)
  684. {
  685. struct bpf_array *array = container_of(map, struct bpf_array, map);
  686. int i;
  687. for (i = 0; i < array->map.max_entries; i++)
  688. fd_array_map_delete_elem(map, &i);
  689. }
  690. static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key,
  691. struct seq_file *m)
  692. {
  693. void **elem, *ptr;
  694. u32 prog_id;
  695. rcu_read_lock();
  696. elem = array_map_lookup_elem(map, key);
  697. if (elem) {
  698. ptr = READ_ONCE(*elem);
  699. if (ptr) {
  700. seq_printf(m, "%u: ", *(u32 *)key);
  701. prog_id = prog_fd_array_sys_lookup_elem(ptr);
  702. btf_type_seq_show(map->btf, map->btf_value_type_id,
  703. &prog_id, m);
  704. seq_puts(m, "\n");
  705. }
  706. }
  707. rcu_read_unlock();
  708. }
  709. struct prog_poke_elem {
  710. struct list_head list;
  711. struct bpf_prog_aux *aux;
  712. };
  713. static int prog_array_map_poke_track(struct bpf_map *map,
  714. struct bpf_prog_aux *prog_aux)
  715. {
  716. struct prog_poke_elem *elem;
  717. struct bpf_array_aux *aux;
  718. int ret = 0;
  719. aux = container_of(map, struct bpf_array, map)->aux;
  720. mutex_lock(&aux->poke_mutex);
  721. list_for_each_entry(elem, &aux->poke_progs, list) {
  722. if (elem->aux == prog_aux)
  723. goto out;
  724. }
  725. elem = kmalloc(sizeof(*elem), GFP_KERNEL);
  726. if (!elem) {
  727. ret = -ENOMEM;
  728. goto out;
  729. }
  730. INIT_LIST_HEAD(&elem->list);
  731. /* We must track the program's aux info at this point in time
  732. * since the program pointer itself may not be stable yet, see
  733. * also comment in prog_array_map_poke_run().
  734. */
  735. elem->aux = prog_aux;
  736. list_add_tail(&elem->list, &aux->poke_progs);
  737. out:
  738. mutex_unlock(&aux->poke_mutex);
  739. return ret;
  740. }
  741. static void prog_array_map_poke_untrack(struct bpf_map *map,
  742. struct bpf_prog_aux *prog_aux)
  743. {
  744. struct prog_poke_elem *elem, *tmp;
  745. struct bpf_array_aux *aux;
  746. aux = container_of(map, struct bpf_array, map)->aux;
  747. mutex_lock(&aux->poke_mutex);
  748. list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) {
  749. if (elem->aux == prog_aux) {
  750. list_del_init(&elem->list);
  751. kfree(elem);
  752. break;
  753. }
  754. }
  755. mutex_unlock(&aux->poke_mutex);
  756. }
  757. static void prog_array_map_poke_run(struct bpf_map *map, u32 key,
  758. struct bpf_prog *old,
  759. struct bpf_prog *new)
  760. {
  761. u8 *old_addr, *new_addr, *old_bypass_addr;
  762. struct prog_poke_elem *elem;
  763. struct bpf_array_aux *aux;
  764. aux = container_of(map, struct bpf_array, map)->aux;
  765. WARN_ON_ONCE(!mutex_is_locked(&aux->poke_mutex));
  766. list_for_each_entry(elem, &aux->poke_progs, list) {
  767. struct bpf_jit_poke_descriptor *poke;
  768. int i, ret;
  769. for (i = 0; i < elem->aux->size_poke_tab; i++) {
  770. poke = &elem->aux->poke_tab[i];
  771. /* Few things to be aware of:
  772. *
  773. * 1) We can only ever access aux in this context, but
  774. * not aux->prog since it might not be stable yet and
  775. * there could be danger of use after free otherwise.
  776. * 2) Initially when we start tracking aux, the program
  777. * is not JITed yet and also does not have a kallsyms
  778. * entry. We skip these as poke->tailcall_target_stable
  779. * is not active yet. The JIT will do the final fixup
  780. * before setting it stable. The various
  781. * poke->tailcall_target_stable are successively
  782. * activated, so tail call updates can arrive from here
  783. * while JIT is still finishing its final fixup for
  784. * non-activated poke entries.
  785. * 3) On program teardown, the program's kallsym entry gets
  786. * removed out of RCU callback, but we can only untrack
  787. * from sleepable context, therefore bpf_arch_text_poke()
  788. * might not see that this is in BPF text section and
  789. * bails out with -EINVAL. As these are unreachable since
  790. * RCU grace period already passed, we simply skip them.
  791. * 4) Also programs reaching refcount of zero while patching
  792. * is in progress is okay since we're protected under
  793. * poke_mutex and untrack the programs before the JIT
  794. * buffer is freed. When we're still in the middle of
  795. * patching and suddenly kallsyms entry of the program
  796. * gets evicted, we just skip the rest which is fine due
  797. * to point 3).
  798. * 5) Any other error happening below from bpf_arch_text_poke()
  799. * is a unexpected bug.
  800. */
  801. if (!READ_ONCE(poke->tailcall_target_stable))
  802. continue;
  803. if (poke->reason != BPF_POKE_REASON_TAIL_CALL)
  804. continue;
  805. if (poke->tail_call.map != map ||
  806. poke->tail_call.key != key)
  807. continue;
  808. old_bypass_addr = old ? NULL : poke->bypass_addr;
  809. old_addr = old ? (u8 *)old->bpf_func + poke->adj_off : NULL;
  810. new_addr = new ? (u8 *)new->bpf_func + poke->adj_off : NULL;
  811. if (new) {
  812. ret = bpf_arch_text_poke(poke->tailcall_target,
  813. BPF_MOD_JUMP,
  814. old_addr, new_addr);
  815. BUG_ON(ret < 0 && ret != -EINVAL);
  816. if (!old) {
  817. ret = bpf_arch_text_poke(poke->tailcall_bypass,
  818. BPF_MOD_JUMP,
  819. poke->bypass_addr,
  820. NULL);
  821. BUG_ON(ret < 0 && ret != -EINVAL);
  822. }
  823. } else {
  824. ret = bpf_arch_text_poke(poke->tailcall_bypass,
  825. BPF_MOD_JUMP,
  826. old_bypass_addr,
  827. poke->bypass_addr);
  828. BUG_ON(ret < 0 && ret != -EINVAL);
  829. /* let other CPUs finish the execution of program
  830. * so that it will not possible to expose them
  831. * to invalid nop, stack unwind, nop state
  832. */
  833. if (!ret)
  834. synchronize_rcu();
  835. ret = bpf_arch_text_poke(poke->tailcall_target,
  836. BPF_MOD_JUMP,
  837. old_addr, NULL);
  838. BUG_ON(ret < 0 && ret != -EINVAL);
  839. }
  840. }
  841. }
  842. }
  843. static void prog_array_map_clear_deferred(struct work_struct *work)
  844. {
  845. struct bpf_map *map = container_of(work, struct bpf_array_aux,
  846. work)->map;
  847. bpf_fd_array_map_clear(map);
  848. bpf_map_put(map);
  849. }
  850. static void prog_array_map_clear(struct bpf_map *map)
  851. {
  852. struct bpf_array_aux *aux = container_of(map, struct bpf_array,
  853. map)->aux;
  854. bpf_map_inc(map);
  855. schedule_work(&aux->work);
  856. }
  857. static struct bpf_map *prog_array_map_alloc(union bpf_attr *attr)
  858. {
  859. struct bpf_array_aux *aux;
  860. struct bpf_map *map;
  861. aux = kzalloc(sizeof(*aux), GFP_KERNEL);
  862. if (!aux)
  863. return ERR_PTR(-ENOMEM);
  864. INIT_WORK(&aux->work, prog_array_map_clear_deferred);
  865. INIT_LIST_HEAD(&aux->poke_progs);
  866. mutex_init(&aux->poke_mutex);
  867. spin_lock_init(&aux->owner.lock);
  868. map = array_map_alloc(attr);
  869. if (IS_ERR(map)) {
  870. kfree(aux);
  871. return map;
  872. }
  873. container_of(map, struct bpf_array, map)->aux = aux;
  874. aux->map = map;
  875. return map;
  876. }
  877. static void prog_array_map_free(struct bpf_map *map)
  878. {
  879. struct prog_poke_elem *elem, *tmp;
  880. struct bpf_array_aux *aux;
  881. aux = container_of(map, struct bpf_array, map)->aux;
  882. list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) {
  883. list_del_init(&elem->list);
  884. kfree(elem);
  885. }
  886. kfree(aux);
  887. fd_array_map_free(map);
  888. }
  889. /* prog_array->aux->{type,jited} is a runtime binding.
  890. * Doing static check alone in the verifier is not enough.
  891. * Thus, prog_array_map cannot be used as an inner_map
  892. * and map_meta_equal is not implemented.
  893. */
  894. static int prog_array_map_btf_id;
  895. const struct bpf_map_ops prog_array_map_ops = {
  896. .map_alloc_check = fd_array_map_alloc_check,
  897. .map_alloc = prog_array_map_alloc,
  898. .map_free = prog_array_map_free,
  899. .map_poke_track = prog_array_map_poke_track,
  900. .map_poke_untrack = prog_array_map_poke_untrack,
  901. .map_poke_run = prog_array_map_poke_run,
  902. .map_get_next_key = array_map_get_next_key,
  903. .map_lookup_elem = fd_array_map_lookup_elem,
  904. .map_delete_elem = fd_array_map_delete_elem,
  905. .map_fd_get_ptr = prog_fd_array_get_ptr,
  906. .map_fd_put_ptr = prog_fd_array_put_ptr,
  907. .map_fd_sys_lookup_elem = prog_fd_array_sys_lookup_elem,
  908. .map_release_uref = prog_array_map_clear,
  909. .map_seq_show_elem = prog_array_map_seq_show_elem,
  910. .map_btf_name = "bpf_array",
  911. .map_btf_id = &prog_array_map_btf_id,
  912. };
  913. static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,
  914. struct file *map_file)
  915. {
  916. struct bpf_event_entry *ee;
  917. ee = kzalloc(sizeof(*ee), GFP_ATOMIC);
  918. if (ee) {
  919. ee->event = perf_file->private_data;
  920. ee->perf_file = perf_file;
  921. ee->map_file = map_file;
  922. }
  923. return ee;
  924. }
  925. static void __bpf_event_entry_free(struct rcu_head *rcu)
  926. {
  927. struct bpf_event_entry *ee;
  928. ee = container_of(rcu, struct bpf_event_entry, rcu);
  929. fput(ee->perf_file);
  930. kfree(ee);
  931. }
  932. static void bpf_event_entry_free_rcu(struct bpf_event_entry *ee)
  933. {
  934. call_rcu(&ee->rcu, __bpf_event_entry_free);
  935. }
  936. static void *perf_event_fd_array_get_ptr(struct bpf_map *map,
  937. struct file *map_file, int fd)
  938. {
  939. struct bpf_event_entry *ee;
  940. struct perf_event *event;
  941. struct file *perf_file;
  942. u64 value;
  943. perf_file = perf_event_get(fd);
  944. if (IS_ERR(perf_file))
  945. return perf_file;
  946. ee = ERR_PTR(-EOPNOTSUPP);
  947. event = perf_file->private_data;
  948. if (perf_event_read_local(event, &value, NULL, NULL) == -EOPNOTSUPP)
  949. goto err_out;
  950. ee = bpf_event_entry_gen(perf_file, map_file);
  951. if (ee)
  952. return ee;
  953. ee = ERR_PTR(-ENOMEM);
  954. err_out:
  955. fput(perf_file);
  956. return ee;
  957. }
  958. static void perf_event_fd_array_put_ptr(void *ptr)
  959. {
  960. bpf_event_entry_free_rcu(ptr);
  961. }
  962. static void perf_event_fd_array_release(struct bpf_map *map,
  963. struct file *map_file)
  964. {
  965. struct bpf_array *array = container_of(map, struct bpf_array, map);
  966. struct bpf_event_entry *ee;
  967. int i;
  968. if (map->map_flags & BPF_F_PRESERVE_ELEMS)
  969. return;
  970. rcu_read_lock();
  971. for (i = 0; i < array->map.max_entries; i++) {
  972. ee = READ_ONCE(array->ptrs[i]);
  973. if (ee && ee->map_file == map_file)
  974. fd_array_map_delete_elem(map, &i);
  975. }
  976. rcu_read_unlock();
  977. }
  978. static void perf_event_fd_array_map_free(struct bpf_map *map)
  979. {
  980. if (map->map_flags & BPF_F_PRESERVE_ELEMS)
  981. bpf_fd_array_map_clear(map);
  982. fd_array_map_free(map);
  983. }
  984. static int perf_event_array_map_btf_id;
  985. const struct bpf_map_ops perf_event_array_map_ops = {
  986. .map_meta_equal = bpf_map_meta_equal,
  987. .map_alloc_check = fd_array_map_alloc_check,
  988. .map_alloc = array_map_alloc,
  989. .map_free = perf_event_fd_array_map_free,
  990. .map_get_next_key = array_map_get_next_key,
  991. .map_lookup_elem = fd_array_map_lookup_elem,
  992. .map_delete_elem = fd_array_map_delete_elem,
  993. .map_fd_get_ptr = perf_event_fd_array_get_ptr,
  994. .map_fd_put_ptr = perf_event_fd_array_put_ptr,
  995. .map_release = perf_event_fd_array_release,
  996. .map_check_btf = map_check_no_btf,
  997. .map_btf_name = "bpf_array",
  998. .map_btf_id = &perf_event_array_map_btf_id,
  999. };
  1000. #ifdef CONFIG_CGROUPS
  1001. static void *cgroup_fd_array_get_ptr(struct bpf_map *map,
  1002. struct file *map_file /* not used */,
  1003. int fd)
  1004. {
  1005. return cgroup_get_from_fd(fd);
  1006. }
  1007. static void cgroup_fd_array_put_ptr(void *ptr)
  1008. {
  1009. /* cgroup_put free cgrp after a rcu grace period */
  1010. cgroup_put(ptr);
  1011. }
  1012. static void cgroup_fd_array_free(struct bpf_map *map)
  1013. {
  1014. bpf_fd_array_map_clear(map);
  1015. fd_array_map_free(map);
  1016. }
  1017. static int cgroup_array_map_btf_id;
  1018. const struct bpf_map_ops cgroup_array_map_ops = {
  1019. .map_meta_equal = bpf_map_meta_equal,
  1020. .map_alloc_check = fd_array_map_alloc_check,
  1021. .map_alloc = array_map_alloc,
  1022. .map_free = cgroup_fd_array_free,
  1023. .map_get_next_key = array_map_get_next_key,
  1024. .map_lookup_elem = fd_array_map_lookup_elem,
  1025. .map_delete_elem = fd_array_map_delete_elem,
  1026. .map_fd_get_ptr = cgroup_fd_array_get_ptr,
  1027. .map_fd_put_ptr = cgroup_fd_array_put_ptr,
  1028. .map_check_btf = map_check_no_btf,
  1029. .map_btf_name = "bpf_array",
  1030. .map_btf_id = &cgroup_array_map_btf_id,
  1031. };
  1032. #endif
  1033. static struct bpf_map *array_of_map_alloc(union bpf_attr *attr)
  1034. {
  1035. struct bpf_map *map, *inner_map_meta;
  1036. inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd);
  1037. if (IS_ERR(inner_map_meta))
  1038. return inner_map_meta;
  1039. map = array_map_alloc(attr);
  1040. if (IS_ERR(map)) {
  1041. bpf_map_meta_free(inner_map_meta);
  1042. return map;
  1043. }
  1044. map->inner_map_meta = inner_map_meta;
  1045. return map;
  1046. }
  1047. static void array_of_map_free(struct bpf_map *map)
  1048. {
  1049. /* map->inner_map_meta is only accessed by syscall which
  1050. * is protected by fdget/fdput.
  1051. */
  1052. bpf_map_meta_free(map->inner_map_meta);
  1053. bpf_fd_array_map_clear(map);
  1054. fd_array_map_free(map);
  1055. }
  1056. static void *array_of_map_lookup_elem(struct bpf_map *map, void *key)
  1057. {
  1058. struct bpf_map **inner_map = array_map_lookup_elem(map, key);
  1059. if (!inner_map)
  1060. return NULL;
  1061. return READ_ONCE(*inner_map);
  1062. }
  1063. static int array_of_map_gen_lookup(struct bpf_map *map,
  1064. struct bpf_insn *insn_buf)
  1065. {
  1066. struct bpf_array *array = container_of(map, struct bpf_array, map);
  1067. u32 elem_size = round_up(map->value_size, 8);
  1068. struct bpf_insn *insn = insn_buf;
  1069. const int ret = BPF_REG_0;
  1070. const int map_ptr = BPF_REG_1;
  1071. const int index = BPF_REG_2;
  1072. *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
  1073. *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
  1074. if (!map->bypass_spec_v1) {
  1075. *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6);
  1076. *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);
  1077. } else {
  1078. *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5);
  1079. }
  1080. if (is_power_of_2(elem_size))
  1081. *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size));
  1082. else
  1083. *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size);
  1084. *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr);
  1085. *insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0);
  1086. *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);
  1087. *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
  1088. *insn++ = BPF_MOV64_IMM(ret, 0);
  1089. return insn - insn_buf;
  1090. }
  1091. static int array_of_maps_map_btf_id;
  1092. const struct bpf_map_ops array_of_maps_map_ops = {
  1093. .map_alloc_check = fd_array_map_alloc_check,
  1094. .map_alloc = array_of_map_alloc,
  1095. .map_free = array_of_map_free,
  1096. .map_get_next_key = array_map_get_next_key,
  1097. .map_lookup_elem = array_of_map_lookup_elem,
  1098. .map_delete_elem = fd_array_map_delete_elem,
  1099. .map_fd_get_ptr = bpf_map_fd_get_ptr,
  1100. .map_fd_put_ptr = bpf_map_fd_put_ptr,
  1101. .map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem,
  1102. .map_gen_lookup = array_of_map_gen_lookup,
  1103. .map_check_btf = map_check_no_btf,
  1104. .map_btf_name = "bpf_array",
  1105. .map_btf_id = &array_of_maps_map_btf_id,
  1106. };