dim.h 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336
  1. /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
  2. /* Copyright (c) 2019 Mellanox Technologies. */
  3. #ifndef DIM_H
  4. #define DIM_H
  5. #include <linux/bits.h>
  6. #include <linux/kernel.h>
  7. #include <linux/module.h>
  8. #include <linux/types.h>
  9. #include <linux/workqueue.h>
  10. /*
  11. * Number of events between DIM iterations.
  12. * Causes a moderation of the algorithm run.
  13. */
  14. #define DIM_NEVENTS 64
  15. /*
  16. * Is a difference between values justifies taking an action.
  17. * We consider 10% difference as significant.
  18. */
  19. #define IS_SIGNIFICANT_DIFF(val, ref) \
  20. (((100UL * abs((val) - (ref))) / (ref)) > 10)
  21. /*
  22. * Calculate the gap between two values.
  23. * Take wrap-around and variable size into consideration.
  24. */
  25. #define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) \
  26. & (BIT_ULL(bits) - 1))
  27. /**
  28. * struct dim_cq_moder - Structure for CQ moderation values.
  29. * Used for communications between DIM and its consumer.
  30. *
  31. * @usec: CQ timer suggestion (by DIM)
  32. * @pkts: CQ packet counter suggestion (by DIM)
  33. * @comps: Completion counter
  34. * @cq_period_mode: CQ period count mode (from CQE/EQE)
  35. */
  36. struct dim_cq_moder {
  37. u16 usec;
  38. u16 pkts;
  39. u16 comps;
  40. u8 cq_period_mode;
  41. };
  42. /**
  43. * struct dim_sample - Structure for DIM sample data.
  44. * Used for communications between DIM and its consumer.
  45. *
  46. * @time: Sample timestamp
  47. * @pkt_ctr: Number of packets
  48. * @byte_ctr: Number of bytes
  49. * @event_ctr: Number of events
  50. * @comp_ctr: Current completion counter
  51. */
  52. struct dim_sample {
  53. ktime_t time;
  54. u32 pkt_ctr;
  55. u32 byte_ctr;
  56. u16 event_ctr;
  57. u32 comp_ctr;
  58. };
  59. /**
  60. * struct dim_stats - Structure for DIM stats.
  61. * Used for holding current measured rates.
  62. *
  63. * @ppms: Packets per msec
  64. * @bpms: Bytes per msec
  65. * @epms: Events per msec
  66. * @cpms: Completions per msec
  67. * @cpe_ratio: Ratio of completions to events
  68. */
  69. struct dim_stats {
  70. int ppms; /* packets per msec */
  71. int bpms; /* bytes per msec */
  72. int epms; /* events per msec */
  73. int cpms; /* completions per msec */
  74. int cpe_ratio; /* ratio of completions to events */
  75. };
  76. /**
  77. * struct dim - Main structure for dynamic interrupt moderation (DIM).
  78. * Used for holding all information about a specific DIM instance.
  79. *
  80. * @state: Algorithm state (see below)
  81. * @prev_stats: Measured rates from previous iteration (for comparison)
  82. * @start_sample: Sampled data at start of current iteration
  83. * @measuring_sample: A &dim_sample that is used to update the current events
  84. * @work: Work to perform on action required
  85. * @priv: A pointer to the struct that points to dim
  86. * @profile_ix: Current moderation profile
  87. * @mode: CQ period count mode
  88. * @tune_state: Algorithm tuning state (see below)
  89. * @steps_right: Number of steps taken towards higher moderation
  90. * @steps_left: Number of steps taken towards lower moderation
  91. * @tired: Parking depth counter
  92. */
  93. struct dim {
  94. u8 state;
  95. struct dim_stats prev_stats;
  96. struct dim_sample start_sample;
  97. struct dim_sample measuring_sample;
  98. struct work_struct work;
  99. void *priv;
  100. u8 profile_ix;
  101. u8 mode;
  102. u8 tune_state;
  103. u8 steps_right;
  104. u8 steps_left;
  105. u8 tired;
  106. };
  107. /**
  108. * enum dim_cq_period_mode - Modes for CQ period count
  109. *
  110. * @DIM_CQ_PERIOD_MODE_START_FROM_EQE: Start counting from EQE
  111. * @DIM_CQ_PERIOD_MODE_START_FROM_CQE: Start counting from CQE (implies timer reset)
  112. * @DIM_CQ_PERIOD_NUM_MODES: Number of modes
  113. */
  114. enum dim_cq_period_mode {
  115. DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0,
  116. DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1,
  117. DIM_CQ_PERIOD_NUM_MODES
  118. };
  119. /**
  120. * enum dim_state - DIM algorithm states
  121. *
  122. * These will determine if the algorithm is in a valid state to start an iteration.
  123. *
  124. * @DIM_START_MEASURE: This is the first iteration (also after applying a new profile)
  125. * @DIM_MEASURE_IN_PROGRESS: Algorithm is already in progress - check if
  126. * need to perform an action
  127. * @DIM_APPLY_NEW_PROFILE: DIM consumer is currently applying a profile - no need to measure
  128. */
  129. enum dim_state {
  130. DIM_START_MEASURE,
  131. DIM_MEASURE_IN_PROGRESS,
  132. DIM_APPLY_NEW_PROFILE,
  133. };
  134. /**
  135. * enum dim_tune_state - DIM algorithm tune states
  136. *
  137. * These will determine which action the algorithm should perform.
  138. *
  139. * @DIM_PARKING_ON_TOP: Algorithm found a local top point - exit on significant difference
  140. * @DIM_PARKING_TIRED: Algorithm found a deep top point - don't exit if tired > 0
  141. * @DIM_GOING_RIGHT: Algorithm is currently trying higher moderation levels
  142. * @DIM_GOING_LEFT: Algorithm is currently trying lower moderation levels
  143. */
  144. enum dim_tune_state {
  145. DIM_PARKING_ON_TOP,
  146. DIM_PARKING_TIRED,
  147. DIM_GOING_RIGHT,
  148. DIM_GOING_LEFT,
  149. };
  150. /**
  151. * enum dim_stats_state - DIM algorithm statistics states
  152. *
  153. * These will determine the verdict of current iteration.
  154. *
  155. * @DIM_STATS_WORSE: Current iteration shows worse performance than before
  156. * @DIM_STATS_SAME: Current iteration shows same performance than before
  157. * @DIM_STATS_BETTER: Current iteration shows better performance than before
  158. */
  159. enum dim_stats_state {
  160. DIM_STATS_WORSE,
  161. DIM_STATS_SAME,
  162. DIM_STATS_BETTER,
  163. };
  164. /**
  165. * enum dim_step_result - DIM algorithm step results
  166. *
  167. * These describe the result of a step.
  168. *
  169. * @DIM_STEPPED: Performed a regular step
  170. * @DIM_TOO_TIRED: Same kind of step was done multiple times - should go to
  171. * tired parking
  172. * @DIM_ON_EDGE: Stepped to the most left/right profile
  173. */
  174. enum dim_step_result {
  175. DIM_STEPPED,
  176. DIM_TOO_TIRED,
  177. DIM_ON_EDGE,
  178. };
  179. /**
  180. * dim_on_top - check if current state is a good place to stop (top location)
  181. * @dim: DIM context
  182. *
  183. * Check if current profile is a good place to park at.
  184. * This will result in reducing the DIM checks frequency as we assume we
  185. * shouldn't probably change profiles, unless traffic pattern wasn't changed.
  186. */
  187. bool dim_on_top(struct dim *dim);
  188. /**
  189. * dim_turn - change profile altering direction
  190. * @dim: DIM context
  191. *
  192. * Go left if we were going right and vice-versa.
  193. * Do nothing if currently parking.
  194. */
  195. void dim_turn(struct dim *dim);
  196. /**
  197. * dim_park_on_top - enter a parking state on a top location
  198. * @dim: DIM context
  199. *
  200. * Enter parking state.
  201. * Clear all movement history.
  202. */
  203. void dim_park_on_top(struct dim *dim);
  204. /**
  205. * dim_park_tired - enter a tired parking state
  206. * @dim: DIM context
  207. *
  208. * Enter parking state.
  209. * Clear all movement history and cause DIM checks frequency to reduce.
  210. */
  211. void dim_park_tired(struct dim *dim);
  212. /**
  213. * dim_calc_stats - calculate the difference between two samples
  214. * @start: start sample
  215. * @end: end sample
  216. * @curr_stats: delta between samples
  217. *
  218. * Calculate the delta between two samples (in data rates).
  219. * Takes into consideration counter wrap-around.
  220. */
  221. void dim_calc_stats(struct dim_sample *start, struct dim_sample *end,
  222. struct dim_stats *curr_stats);
  223. /**
  224. * dim_update_sample - set a sample's fields with given values
  225. * @event_ctr: number of events to set
  226. * @packets: number of packets to set
  227. * @bytes: number of bytes to set
  228. * @s: DIM sample
  229. */
  230. static inline void
  231. dim_update_sample(u16 event_ctr, u64 packets, u64 bytes, struct dim_sample *s)
  232. {
  233. s->time = ktime_get();
  234. s->pkt_ctr = packets;
  235. s->byte_ctr = bytes;
  236. s->event_ctr = event_ctr;
  237. }
  238. /**
  239. * dim_update_sample_with_comps - set a sample's fields with given
  240. * values including the completion parameter
  241. * @event_ctr: number of events to set
  242. * @packets: number of packets to set
  243. * @bytes: number of bytes to set
  244. * @comps: number of completions to set
  245. * @s: DIM sample
  246. */
  247. static inline void
  248. dim_update_sample_with_comps(u16 event_ctr, u64 packets, u64 bytes, u64 comps,
  249. struct dim_sample *s)
  250. {
  251. dim_update_sample(event_ctr, packets, bytes, s);
  252. s->comp_ctr = comps;
  253. }
  254. /* Net DIM */
  255. /**
  256. * net_dim_get_rx_moderation - provide a CQ moderation object for the given RX profile
  257. * @cq_period_mode: CQ period mode
  258. * @ix: Profile index
  259. */
  260. struct dim_cq_moder net_dim_get_rx_moderation(u8 cq_period_mode, int ix);
  261. /**
  262. * net_dim_get_def_rx_moderation - provide the default RX moderation
  263. * @cq_period_mode: CQ period mode
  264. */
  265. struct dim_cq_moder net_dim_get_def_rx_moderation(u8 cq_period_mode);
  266. /**
  267. * net_dim_get_tx_moderation - provide a CQ moderation object for the given TX profile
  268. * @cq_period_mode: CQ period mode
  269. * @ix: Profile index
  270. */
  271. struct dim_cq_moder net_dim_get_tx_moderation(u8 cq_period_mode, int ix);
  272. /**
  273. * net_dim_get_def_tx_moderation - provide the default TX moderation
  274. * @cq_period_mode: CQ period mode
  275. */
  276. struct dim_cq_moder net_dim_get_def_tx_moderation(u8 cq_period_mode);
  277. /**
  278. * net_dim - main DIM algorithm entry point
  279. * @dim: DIM instance information
  280. * @end_sample: Current data measurement
  281. *
  282. * Called by the consumer.
  283. * This is the main logic of the algorithm, where data is processed in order
  284. * to decide on next required action.
  285. */
  286. void net_dim(struct dim *dim, struct dim_sample end_sample);
  287. /* RDMA DIM */
  288. /*
  289. * RDMA DIM profile:
  290. * profile size must be of RDMA_DIM_PARAMS_NUM_PROFILES.
  291. */
  292. #define RDMA_DIM_PARAMS_NUM_PROFILES 9
  293. #define RDMA_DIM_START_PROFILE 0
  294. /**
  295. * rdma_dim - Runs the adaptive moderation.
  296. * @dim: The moderation struct.
  297. * @completions: The number of completions collected in this round.
  298. *
  299. * Each call to rdma_dim takes the latest amount of completions that
  300. * have been collected and counts them as a new event.
  301. * Once enough events have been collected the algorithm decides a new
  302. * moderation level.
  303. */
  304. void rdma_dim(struct dim *dim, u64 completions);
  305. #endif /* DIM_H */