bpf_local_storage.h 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * Copyright (c) 2019 Facebook
  4. * Copyright 2020 Google LLC.
  5. */
  6. #ifndef _BPF_LOCAL_STORAGE_H
  7. #define _BPF_LOCAL_STORAGE_H
  8. #include <linux/bpf.h>
  9. #include <linux/rculist.h>
  10. #include <linux/list.h>
  11. #include <linux/hash.h>
  12. #include <linux/types.h>
  13. #include <uapi/linux/btf.h>
  14. #define BPF_LOCAL_STORAGE_CACHE_SIZE 16
  15. struct bpf_local_storage_map_bucket {
  16. struct hlist_head list;
  17. raw_spinlock_t lock;
  18. };
  19. /* Thp map is not the primary owner of a bpf_local_storage_elem.
  20. * Instead, the container object (eg. sk->sk_bpf_storage) is.
  21. *
  22. * The map (bpf_local_storage_map) is for two purposes
  23. * 1. Define the size of the "local storage". It is
  24. * the map's value_size.
  25. *
  26. * 2. Maintain a list to keep track of all elems such
  27. * that they can be cleaned up during the map destruction.
  28. *
  29. * When a bpf local storage is being looked up for a
  30. * particular object, the "bpf_map" pointer is actually used
  31. * as the "key" to search in the list of elem in
  32. * the respective bpf_local_storage owned by the object.
  33. *
  34. * e.g. sk->sk_bpf_storage is the mini-map with the "bpf_map" pointer
  35. * as the searching key.
  36. */
  37. struct bpf_local_storage_map {
  38. struct bpf_map map;
  39. /* Lookup elem does not require accessing the map.
  40. *
  41. * Updating/Deleting requires a bucket lock to
  42. * link/unlink the elem from the map. Having
  43. * multiple buckets to improve contention.
  44. */
  45. struct bpf_local_storage_map_bucket *buckets;
  46. u32 bucket_log;
  47. u16 elem_size;
  48. u16 cache_idx;
  49. };
  50. struct bpf_local_storage_data {
  51. /* smap is used as the searching key when looking up
  52. * from the object's bpf_local_storage.
  53. *
  54. * Put it in the same cacheline as the data to minimize
  55. * the number of cachelines access during the cache hit case.
  56. */
  57. struct bpf_local_storage_map __rcu *smap;
  58. u8 data[] __aligned(8);
  59. };
  60. /* Linked to bpf_local_storage and bpf_local_storage_map */
  61. struct bpf_local_storage_elem {
  62. struct hlist_node map_node; /* Linked to bpf_local_storage_map */
  63. struct hlist_node snode; /* Linked to bpf_local_storage */
  64. struct bpf_local_storage __rcu *local_storage;
  65. struct rcu_head rcu;
  66. /* 8 bytes hole */
  67. /* The data is stored in aother cacheline to minimize
  68. * the number of cachelines access during a cache hit.
  69. */
  70. struct bpf_local_storage_data sdata ____cacheline_aligned;
  71. };
  72. struct bpf_local_storage {
  73. struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE];
  74. struct hlist_head list; /* List of bpf_local_storage_elem */
  75. void *owner; /* The object that owns the above "list" of
  76. * bpf_local_storage_elem.
  77. */
  78. struct rcu_head rcu;
  79. raw_spinlock_t lock; /* Protect adding/removing from the "list" */
  80. };
  81. /* U16_MAX is much more than enough for sk local storage
  82. * considering a tcp_sock is ~2k.
  83. */
  84. #define BPF_LOCAL_STORAGE_MAX_VALUE_SIZE \
  85. min_t(u32, \
  86. (KMALLOC_MAX_SIZE - MAX_BPF_STACK - \
  87. sizeof(struct bpf_local_storage_elem)), \
  88. (U16_MAX - sizeof(struct bpf_local_storage_elem)))
  89. #define SELEM(_SDATA) \
  90. container_of((_SDATA), struct bpf_local_storage_elem, sdata)
  91. #define SDATA(_SELEM) (&(_SELEM)->sdata)
  92. #define BPF_LOCAL_STORAGE_CACHE_SIZE 16
  93. struct bpf_local_storage_cache {
  94. spinlock_t idx_lock;
  95. u64 idx_usage_counts[BPF_LOCAL_STORAGE_CACHE_SIZE];
  96. };
  97. #define DEFINE_BPF_STORAGE_CACHE(name) \
  98. static struct bpf_local_storage_cache name = { \
  99. .idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock), \
  100. }
  101. u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache);
  102. void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache,
  103. u16 idx);
  104. /* Helper functions for bpf_local_storage */
  105. int bpf_local_storage_map_alloc_check(union bpf_attr *attr);
  106. struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr);
  107. struct bpf_local_storage_data *
  108. bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
  109. struct bpf_local_storage_map *smap,
  110. bool cacheit_lockit);
  111. void bpf_local_storage_map_free(struct bpf_local_storage_map *smap);
  112. int bpf_local_storage_map_check_btf(const struct bpf_map *map,
  113. const struct btf *btf,
  114. const struct btf_type *key_type,
  115. const struct btf_type *value_type);
  116. void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
  117. struct bpf_local_storage_elem *selem);
  118. bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
  119. struct bpf_local_storage_elem *selem,
  120. bool uncharge_omem);
  121. void bpf_selem_unlink(struct bpf_local_storage_elem *selem);
  122. void bpf_selem_link_map(struct bpf_local_storage_map *smap,
  123. struct bpf_local_storage_elem *selem);
  124. void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem);
  125. struct bpf_local_storage_elem *
  126. bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
  127. bool charge_mem);
  128. int
  129. bpf_local_storage_alloc(void *owner,
  130. struct bpf_local_storage_map *smap,
  131. struct bpf_local_storage_elem *first_selem);
  132. struct bpf_local_storage_data *
  133. bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
  134. void *value, u64 map_flags);
  135. #endif /* _BPF_LOCAL_STORAGE_H */