dispatcher.c 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /* Copyright(c) 2019 Intel Corporation. */
  3. #include <linux/hash.h>
  4. #include <linux/bpf.h>
  5. #include <linux/filter.h>
  6. /* The BPF dispatcher is a multiway branch code generator. The
  7. * dispatcher is a mechanism to avoid the performance penalty of an
  8. * indirect call, which is expensive when retpolines are enabled. A
  9. * dispatch client registers a BPF program into the dispatcher, and if
  10. * there is available room in the dispatcher a direct call to the BPF
  11. * program will be generated. All calls to the BPF programs called via
  12. * the dispatcher will then be a direct call, instead of an
  13. * indirect. The dispatcher hijacks a trampoline function it via the
  14. * __fentry__ of the trampoline. The trampoline function has the
  15. * following signature:
  16. *
  17. * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi,
  18. * unsigned int (*bpf_func)(const void *,
  19. * const struct bpf_insn *));
  20. */
  21. static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog(
  22. struct bpf_dispatcher *d, struct bpf_prog *prog)
  23. {
  24. int i;
  25. for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
  26. if (prog == d->progs[i].prog)
  27. return &d->progs[i];
  28. }
  29. return NULL;
  30. }
  31. static struct bpf_dispatcher_prog *bpf_dispatcher_find_free(
  32. struct bpf_dispatcher *d)
  33. {
  34. return bpf_dispatcher_find_prog(d, NULL);
  35. }
  36. static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d,
  37. struct bpf_prog *prog)
  38. {
  39. struct bpf_dispatcher_prog *entry;
  40. if (!prog)
  41. return false;
  42. entry = bpf_dispatcher_find_prog(d, prog);
  43. if (entry) {
  44. refcount_inc(&entry->users);
  45. return false;
  46. }
  47. entry = bpf_dispatcher_find_free(d);
  48. if (!entry)
  49. return false;
  50. bpf_prog_inc(prog);
  51. entry->prog = prog;
  52. refcount_set(&entry->users, 1);
  53. d->num_progs++;
  54. return true;
  55. }
  56. static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,
  57. struct bpf_prog *prog)
  58. {
  59. struct bpf_dispatcher_prog *entry;
  60. if (!prog)
  61. return false;
  62. entry = bpf_dispatcher_find_prog(d, prog);
  63. if (!entry)
  64. return false;
  65. if (refcount_dec_and_test(&entry->users)) {
  66. entry->prog = NULL;
  67. bpf_prog_put(prog);
  68. d->num_progs--;
  69. return true;
  70. }
  71. return false;
  72. }
  73. int __weak arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs)
  74. {
  75. return -ENOTSUPP;
  76. }
  77. static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image)
  78. {
  79. s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0];
  80. int i;
  81. for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
  82. if (d->progs[i].prog)
  83. *ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;
  84. }
  85. return arch_prepare_bpf_dispatcher(image, &ips[0], d->num_progs);
  86. }
  87. static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)
  88. {
  89. void *old, *new;
  90. u32 noff;
  91. int err;
  92. if (!prev_num_progs) {
  93. old = NULL;
  94. noff = 0;
  95. } else {
  96. old = d->image + d->image_off;
  97. noff = d->image_off ^ (PAGE_SIZE / 2);
  98. }
  99. new = d->num_progs ? d->image + noff : NULL;
  100. if (new) {
  101. if (bpf_dispatcher_prepare(d, new))
  102. return;
  103. }
  104. err = bpf_arch_text_poke(d->func, BPF_MOD_JUMP, old, new);
  105. if (err || !new)
  106. return;
  107. d->image_off = noff;
  108. }
  109. void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
  110. struct bpf_prog *to)
  111. {
  112. bool changed = false;
  113. int prev_num_progs;
  114. if (from == to)
  115. return;
  116. mutex_lock(&d->mutex);
  117. if (!d->image) {
  118. d->image = bpf_jit_alloc_exec_page();
  119. if (!d->image)
  120. goto out;
  121. bpf_image_ksym_add(d->image, &d->ksym);
  122. }
  123. prev_num_progs = d->num_progs;
  124. changed |= bpf_dispatcher_remove_prog(d, from);
  125. changed |= bpf_dispatcher_add_prog(d, to);
  126. if (!changed)
  127. goto out;
  128. bpf_dispatcher_update(d, prev_num_progs);
  129. out:
  130. mutex_unlock(&d->mutex);
  131. }