bpf_lru_list.h 2.1 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /* Copyright (c) 2016 Facebook
  3. */
  4. #ifndef __BPF_LRU_LIST_H_
  5. #define __BPF_LRU_LIST_H_
  6. #include <linux/list.h>
  7. #include <linux/spinlock_types.h>
  8. #define NR_BPF_LRU_LIST_T (3)
  9. #define NR_BPF_LRU_LIST_COUNT (2)
  10. #define NR_BPF_LRU_LOCAL_LIST_T (2)
  11. #define BPF_LOCAL_LIST_T_OFFSET NR_BPF_LRU_LIST_T
  12. enum bpf_lru_list_type {
  13. BPF_LRU_LIST_T_ACTIVE,
  14. BPF_LRU_LIST_T_INACTIVE,
  15. BPF_LRU_LIST_T_FREE,
  16. BPF_LRU_LOCAL_LIST_T_FREE,
  17. BPF_LRU_LOCAL_LIST_T_PENDING,
  18. };
  19. struct bpf_lru_node {
  20. struct list_head list;
  21. u16 cpu;
  22. u8 type;
  23. u8 ref;
  24. };
  25. struct bpf_lru_list {
  26. struct list_head lists[NR_BPF_LRU_LIST_T];
  27. unsigned int counts[NR_BPF_LRU_LIST_COUNT];
  28. /* The next inactive list rotation starts from here */
  29. struct list_head *next_inactive_rotation;
  30. raw_spinlock_t lock ____cacheline_aligned_in_smp;
  31. };
  32. struct bpf_lru_locallist {
  33. struct list_head lists[NR_BPF_LRU_LOCAL_LIST_T];
  34. u16 next_steal;
  35. raw_spinlock_t lock;
  36. };
  37. struct bpf_common_lru {
  38. struct bpf_lru_list lru_list;
  39. struct bpf_lru_locallist __percpu *local_list;
  40. };
  41. typedef bool (*del_from_htab_func)(void *arg, struct bpf_lru_node *node);
  42. struct bpf_lru {
  43. union {
  44. struct bpf_common_lru common_lru;
  45. struct bpf_lru_list __percpu *percpu_lru;
  46. };
  47. del_from_htab_func del_from_htab;
  48. void *del_arg;
  49. unsigned int hash_offset;
  50. unsigned int nr_scans;
  51. bool percpu;
  52. };
  53. static inline void bpf_lru_node_set_ref(struct bpf_lru_node *node)
  54. {
  55. /* ref is an approximation on access frequency. It does not
  56. * have to be very accurate. Hence, no protection is used.
  57. */
  58. if (!node->ref)
  59. node->ref = 1;
  60. }
  61. int bpf_lru_init(struct bpf_lru *lru, bool percpu, u32 hash_offset,
  62. del_from_htab_func del_from_htab, void *delete_arg);
  63. void bpf_lru_populate(struct bpf_lru *lru, void *buf, u32 node_offset,
  64. u32 elem_size, u32 nr_elems);
  65. void bpf_lru_destroy(struct bpf_lru *lru);
  66. struct bpf_lru_node *bpf_lru_pop_free(struct bpf_lru *lru, u32 hash);
  67. void bpf_lru_push_free(struct bpf_lru *lru, struct bpf_lru_node *node);
  68. void bpf_lru_promote(struct bpf_lru *lru, struct bpf_lru_node *node);
  69. #endif