net_dim.c 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246
  1. // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2. /*
  3. * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
  4. */
  5. #include <linux/dim.h>
  6. /*
  7. * Net DIM profiles:
  8. * There are different set of profiles for each CQ period mode.
  9. * There are different set of profiles for RX/TX CQs.
  10. * Each profile size must be of NET_DIM_PARAMS_NUM_PROFILES
  11. */
  12. #define NET_DIM_PARAMS_NUM_PROFILES 5
  13. #define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
  14. #define NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE 128
  15. #define NET_DIM_DEF_PROFILE_CQE 1
  16. #define NET_DIM_DEF_PROFILE_EQE 1
  17. #define NET_DIM_RX_EQE_PROFILES { \
  18. {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  19. {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  20. {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  21. {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  22. {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  23. }
  24. #define NET_DIM_RX_CQE_PROFILES { \
  25. {2, 256}, \
  26. {8, 128}, \
  27. {16, 64}, \
  28. {32, 64}, \
  29. {64, 64} \
  30. }
  31. #define NET_DIM_TX_EQE_PROFILES { \
  32. {1, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
  33. {8, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
  34. {32, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
  35. {64, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
  36. {128, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE} \
  37. }
  38. #define NET_DIM_TX_CQE_PROFILES { \
  39. {5, 128}, \
  40. {8, 64}, \
  41. {16, 32}, \
  42. {32, 32}, \
  43. {64, 32} \
  44. }
  45. static const struct dim_cq_moder
  46. rx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
  47. NET_DIM_RX_EQE_PROFILES,
  48. NET_DIM_RX_CQE_PROFILES,
  49. };
  50. static const struct dim_cq_moder
  51. tx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
  52. NET_DIM_TX_EQE_PROFILES,
  53. NET_DIM_TX_CQE_PROFILES,
  54. };
  55. struct dim_cq_moder
  56. net_dim_get_rx_moderation(u8 cq_period_mode, int ix)
  57. {
  58. struct dim_cq_moder cq_moder = rx_profile[cq_period_mode][ix];
  59. cq_moder.cq_period_mode = cq_period_mode;
  60. return cq_moder;
  61. }
  62. EXPORT_SYMBOL(net_dim_get_rx_moderation);
  63. struct dim_cq_moder
  64. net_dim_get_def_rx_moderation(u8 cq_period_mode)
  65. {
  66. u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
  67. NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
  68. return net_dim_get_rx_moderation(cq_period_mode, profile_ix);
  69. }
  70. EXPORT_SYMBOL(net_dim_get_def_rx_moderation);
  71. struct dim_cq_moder
  72. net_dim_get_tx_moderation(u8 cq_period_mode, int ix)
  73. {
  74. struct dim_cq_moder cq_moder = tx_profile[cq_period_mode][ix];
  75. cq_moder.cq_period_mode = cq_period_mode;
  76. return cq_moder;
  77. }
  78. EXPORT_SYMBOL(net_dim_get_tx_moderation);
  79. struct dim_cq_moder
  80. net_dim_get_def_tx_moderation(u8 cq_period_mode)
  81. {
  82. u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
  83. NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
  84. return net_dim_get_tx_moderation(cq_period_mode, profile_ix);
  85. }
  86. EXPORT_SYMBOL(net_dim_get_def_tx_moderation);
  87. static int net_dim_step(struct dim *dim)
  88. {
  89. if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2))
  90. return DIM_TOO_TIRED;
  91. switch (dim->tune_state) {
  92. case DIM_PARKING_ON_TOP:
  93. case DIM_PARKING_TIRED:
  94. break;
  95. case DIM_GOING_RIGHT:
  96. if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1))
  97. return DIM_ON_EDGE;
  98. dim->profile_ix++;
  99. dim->steps_right++;
  100. break;
  101. case DIM_GOING_LEFT:
  102. if (dim->profile_ix == 0)
  103. return DIM_ON_EDGE;
  104. dim->profile_ix--;
  105. dim->steps_left++;
  106. break;
  107. }
  108. dim->tired++;
  109. return DIM_STEPPED;
  110. }
  111. static void net_dim_exit_parking(struct dim *dim)
  112. {
  113. dim->tune_state = dim->profile_ix ? DIM_GOING_LEFT : DIM_GOING_RIGHT;
  114. net_dim_step(dim);
  115. }
  116. static int net_dim_stats_compare(struct dim_stats *curr,
  117. struct dim_stats *prev)
  118. {
  119. if (!prev->bpms)
  120. return curr->bpms ? DIM_STATS_BETTER : DIM_STATS_SAME;
  121. if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))
  122. return (curr->bpms > prev->bpms) ? DIM_STATS_BETTER :
  123. DIM_STATS_WORSE;
  124. if (!prev->ppms)
  125. return curr->ppms ? DIM_STATS_BETTER :
  126. DIM_STATS_SAME;
  127. if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))
  128. return (curr->ppms > prev->ppms) ? DIM_STATS_BETTER :
  129. DIM_STATS_WORSE;
  130. if (!prev->epms)
  131. return DIM_STATS_SAME;
  132. if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))
  133. return (curr->epms < prev->epms) ? DIM_STATS_BETTER :
  134. DIM_STATS_WORSE;
  135. return DIM_STATS_SAME;
  136. }
  137. static bool net_dim_decision(struct dim_stats *curr_stats, struct dim *dim)
  138. {
  139. int prev_state = dim->tune_state;
  140. int prev_ix = dim->profile_ix;
  141. int stats_res;
  142. int step_res;
  143. switch (dim->tune_state) {
  144. case DIM_PARKING_ON_TOP:
  145. stats_res = net_dim_stats_compare(curr_stats,
  146. &dim->prev_stats);
  147. if (stats_res != DIM_STATS_SAME)
  148. net_dim_exit_parking(dim);
  149. break;
  150. case DIM_PARKING_TIRED:
  151. dim->tired--;
  152. if (!dim->tired)
  153. net_dim_exit_parking(dim);
  154. break;
  155. case DIM_GOING_RIGHT:
  156. case DIM_GOING_LEFT:
  157. stats_res = net_dim_stats_compare(curr_stats,
  158. &dim->prev_stats);
  159. if (stats_res != DIM_STATS_BETTER)
  160. dim_turn(dim);
  161. if (dim_on_top(dim)) {
  162. dim_park_on_top(dim);
  163. break;
  164. }
  165. step_res = net_dim_step(dim);
  166. switch (step_res) {
  167. case DIM_ON_EDGE:
  168. dim_park_on_top(dim);
  169. break;
  170. case DIM_TOO_TIRED:
  171. dim_park_tired(dim);
  172. break;
  173. }
  174. break;
  175. }
  176. if (prev_state != DIM_PARKING_ON_TOP ||
  177. dim->tune_state != DIM_PARKING_ON_TOP)
  178. dim->prev_stats = *curr_stats;
  179. return dim->profile_ix != prev_ix;
  180. }
  181. void net_dim(struct dim *dim, struct dim_sample end_sample)
  182. {
  183. struct dim_stats curr_stats;
  184. u16 nevents;
  185. switch (dim->state) {
  186. case DIM_MEASURE_IN_PROGRESS:
  187. nevents = BIT_GAP(BITS_PER_TYPE(u16),
  188. end_sample.event_ctr,
  189. dim->start_sample.event_ctr);
  190. if (nevents < DIM_NEVENTS)
  191. break;
  192. dim_calc_stats(&dim->start_sample, &end_sample, &curr_stats);
  193. if (net_dim_decision(&curr_stats, dim)) {
  194. dim->state = DIM_APPLY_NEW_PROFILE;
  195. schedule_work(&dim->work);
  196. break;
  197. }
  198. /* fall through */
  199. case DIM_START_MEASURE:
  200. dim_update_sample(end_sample.event_ctr, end_sample.pkt_ctr,
  201. end_sample.byte_ctr, &dim->start_sample);
  202. dim->state = DIM_MEASURE_IN_PROGRESS;
  203. break;
  204. case DIM_APPLY_NEW_PROFILE:
  205. break;
  206. }
  207. }
  208. EXPORT_SYMBOL(net_dim);