hbm_kern.h 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217
  1. /* SPDX-License-Identifier: GPL-2.0
  2. *
  3. * Copyright (c) 2019 Facebook
  4. *
  5. * This program is free software; you can redistribute it and/or
  6. * modify it under the terms of version 2 of the GNU General Public
  7. * License as published by the Free Software Foundation.
  8. *
  9. * Include file for sample Host Bandwidth Manager (HBM) BPF programs
  10. */
  11. #define KBUILD_MODNAME "foo"
  12. #include <stddef.h>
  13. #include <stdbool.h>
  14. #include <uapi/linux/bpf.h>
  15. #include <uapi/linux/if_ether.h>
  16. #include <uapi/linux/if_packet.h>
  17. #include <uapi/linux/ip.h>
  18. #include <uapi/linux/ipv6.h>
  19. #include <uapi/linux/in.h>
  20. #include <uapi/linux/tcp.h>
  21. #include <uapi/linux/filter.h>
  22. #include <uapi/linux/pkt_cls.h>
  23. #include <net/ipv6.h>
  24. #include <net/inet_ecn.h>
  25. #include <bpf/bpf_endian.h>
  26. #include <bpf/bpf_helpers.h>
  27. #include "hbm.h"
  28. #define DROP_PKT 0
  29. #define ALLOW_PKT 1
  30. #define TCP_ECN_OK 1
  31. #define CWR 2
  32. #ifndef HBM_DEBUG // Define HBM_DEBUG to enable debugging
  33. #undef bpf_printk
  34. #define bpf_printk(fmt, ...)
  35. #endif
  36. #define INITIAL_CREDIT_PACKETS 100
  37. #define MAX_BYTES_PER_PACKET 1500
  38. #define MARK_THRESH (40 * MAX_BYTES_PER_PACKET)
  39. #define DROP_THRESH (80 * 5 * MAX_BYTES_PER_PACKET)
  40. #define LARGE_PKT_DROP_THRESH (DROP_THRESH - (15 * MAX_BYTES_PER_PACKET))
  41. #define MARK_REGION_SIZE (LARGE_PKT_DROP_THRESH - MARK_THRESH)
  42. #define LARGE_PKT_THRESH 120
  43. #define MAX_CREDIT (100 * MAX_BYTES_PER_PACKET)
  44. #define INIT_CREDIT (INITIAL_CREDIT_PACKETS * MAX_BYTES_PER_PACKET)
  45. // Time base accounting for fq's EDT
  46. #define BURST_SIZE_NS 100000 // 100us
  47. #define MARK_THRESH_NS 50000 // 50us
  48. #define DROP_THRESH_NS 500000 // 500us
  49. // Reserve 20us of queuing for small packets (less than 120 bytes)
  50. #define LARGE_PKT_DROP_THRESH_NS (DROP_THRESH_NS - 20000)
  51. #define MARK_REGION_SIZE_NS (LARGE_PKT_DROP_THRESH_NS - MARK_THRESH_NS)
  52. // rate in bytes per ns << 20
  53. #define CREDIT_PER_NS(delta, rate) ((((u64)(delta)) * (rate)) >> 20)
  54. #define BYTES_PER_NS(delta, rate) ((((u64)(delta)) * (rate)) >> 20)
  55. #define BYTES_TO_NS(bytes, rate) div64_u64(((u64)(bytes)) << 20, (u64)(rate))
  56. struct {
  57. __uint(type, BPF_MAP_TYPE_CGROUP_STORAGE);
  58. __type(key, struct bpf_cgroup_storage_key);
  59. __type(value, struct hbm_vqueue);
  60. } queue_state SEC(".maps");
  61. struct {
  62. __uint(type, BPF_MAP_TYPE_ARRAY);
  63. __uint(max_entries, 1);
  64. __type(key, u32);
  65. __type(value, struct hvm_queue_stats);
  66. } queue_stats SEC(".maps");
  67. struct hbm_pkt_info {
  68. int cwnd;
  69. int rtt;
  70. int packets_out;
  71. bool is_ip;
  72. bool is_tcp;
  73. short ecn;
  74. };
  75. static int get_tcp_info(struct __sk_buff *skb, struct hbm_pkt_info *pkti)
  76. {
  77. struct bpf_sock *sk;
  78. struct bpf_tcp_sock *tp;
  79. sk = skb->sk;
  80. if (sk) {
  81. sk = bpf_sk_fullsock(sk);
  82. if (sk) {
  83. if (sk->protocol == IPPROTO_TCP) {
  84. tp = bpf_tcp_sock(sk);
  85. if (tp) {
  86. pkti->cwnd = tp->snd_cwnd;
  87. pkti->rtt = tp->srtt_us >> 3;
  88. pkti->packets_out = tp->packets_out;
  89. return 0;
  90. }
  91. }
  92. }
  93. }
  94. pkti->cwnd = 0;
  95. pkti->rtt = 0;
  96. pkti->packets_out = 0;
  97. return 1;
  98. }
  99. static void hbm_get_pkt_info(struct __sk_buff *skb,
  100. struct hbm_pkt_info *pkti)
  101. {
  102. struct iphdr iph;
  103. struct ipv6hdr *ip6h;
  104. pkti->cwnd = 0;
  105. pkti->rtt = 0;
  106. bpf_skb_load_bytes(skb, 0, &iph, 12);
  107. if (iph.version == 6) {
  108. ip6h = (struct ipv6hdr *)&iph;
  109. pkti->is_ip = true;
  110. pkti->is_tcp = (ip6h->nexthdr == 6);
  111. pkti->ecn = (ip6h->flow_lbl[0] >> 4) & INET_ECN_MASK;
  112. } else if (iph.version == 4) {
  113. pkti->is_ip = true;
  114. pkti->is_tcp = (iph.protocol == 6);
  115. pkti->ecn = iph.tos & INET_ECN_MASK;
  116. } else {
  117. pkti->is_ip = false;
  118. pkti->is_tcp = false;
  119. pkti->ecn = 0;
  120. }
  121. if (pkti->is_tcp)
  122. get_tcp_info(skb, pkti);
  123. }
  124. static __always_inline void hbm_init_vqueue(struct hbm_vqueue *qdp, int rate)
  125. {
  126. bpf_printk("Initializing queue_state, rate:%d\n", rate * 128);
  127. qdp->lasttime = bpf_ktime_get_ns();
  128. qdp->credit = INIT_CREDIT;
  129. qdp->rate = rate * 128;
  130. }
  131. static __always_inline void hbm_init_edt_vqueue(struct hbm_vqueue *qdp,
  132. int rate)
  133. {
  134. unsigned long long curtime;
  135. curtime = bpf_ktime_get_ns();
  136. bpf_printk("Initializing queue_state, rate:%d\n", rate * 128);
  137. qdp->lasttime = curtime - BURST_SIZE_NS; // support initial burst
  138. qdp->credit = 0; // not used
  139. qdp->rate = rate * 128;
  140. }
  141. static __always_inline void hbm_update_stats(struct hbm_queue_stats *qsp,
  142. int len,
  143. unsigned long long curtime,
  144. bool congestion_flag,
  145. bool drop_flag,
  146. bool cwr_flag,
  147. bool ecn_ce_flag,
  148. struct hbm_pkt_info *pkti,
  149. int credit)
  150. {
  151. int rv = ALLOW_PKT;
  152. if (qsp != NULL) {
  153. // Following is needed for work conserving
  154. __sync_add_and_fetch(&(qsp->bytes_total), len);
  155. if (qsp->stats) {
  156. // Optionally update statistics
  157. if (qsp->firstPacketTime == 0)
  158. qsp->firstPacketTime = curtime;
  159. qsp->lastPacketTime = curtime;
  160. __sync_add_and_fetch(&(qsp->pkts_total), 1);
  161. if (congestion_flag) {
  162. __sync_add_and_fetch(&(qsp->pkts_marked), 1);
  163. __sync_add_and_fetch(&(qsp->bytes_marked), len);
  164. }
  165. if (drop_flag) {
  166. __sync_add_and_fetch(&(qsp->pkts_dropped), 1);
  167. __sync_add_and_fetch(&(qsp->bytes_dropped),
  168. len);
  169. }
  170. if (ecn_ce_flag)
  171. __sync_add_and_fetch(&(qsp->pkts_ecn_ce), 1);
  172. if (pkti->cwnd) {
  173. __sync_add_and_fetch(&(qsp->sum_cwnd),
  174. pkti->cwnd);
  175. __sync_add_and_fetch(&(qsp->sum_cwnd_cnt), 1);
  176. }
  177. if (pkti->rtt)
  178. __sync_add_and_fetch(&(qsp->sum_rtt),
  179. pkti->rtt);
  180. __sync_add_and_fetch(&(qsp->sum_credit), credit);
  181. if (drop_flag)
  182. rv = DROP_PKT;
  183. if (cwr_flag)
  184. rv |= 2;
  185. if (rv == DROP_PKT)
  186. __sync_add_and_fetch(&(qsp->returnValCount[0]),
  187. 1);
  188. else if (rv == ALLOW_PKT)
  189. __sync_add_and_fetch(&(qsp->returnValCount[1]),
  190. 1);
  191. else if (rv == 2)
  192. __sync_add_and_fetch(&(qsp->returnValCount[2]),
  193. 1);
  194. else if (rv == 3)
  195. __sync_add_and_fetch(&(qsp->returnValCount[3]),
  196. 1);
  197. }
  198. }
  199. }