Commit | Line | Data |
---|---|---|
0e58983d TG |
1 | /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ |
2 | /* Copyright (c) 2019 Mellanox Technologies. */ | |
3 | ||
4 | #ifndef DIM_H | |
5 | #define DIM_H | |
6 | ||
7 | #include <linux/module.h> | |
8 | ||
4f75da36 TG |
9 | /** |
10 | * Number of events between DIM iterations. | |
11 | * Causes a moderation of the algorithm run. | |
12 | */ | |
449986ea | 13 | #define DIM_NEVENTS 64 |
0e58983d | 14 | |
4f75da36 TG |
15 | /** |
16 | * Is a difference between values justifies taking an action. | |
17 | * We consider 10% difference as significant. | |
18 | */ | |
0e58983d TG |
19 | #define IS_SIGNIFICANT_DIFF(val, ref) \ |
20 | (((100UL * abs((val) - (ref))) / (ref)) > 10) | |
4f75da36 TG |
21 | |
22 | /** | |
23 | * Calculate the gap between two values. | |
24 | * Take wrap-around and variable size into consideration. | |
25 | */ | |
0e58983d | 26 | #define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) \ |
4f75da36 | 27 | & (BIT_ULL(bits) - 1)) |
0e58983d | 28 | |
4f75da36 TG |
29 | /** |
30 | * Structure for CQ moderation values. | |
31 | * Used for communications between DIM and its consumer. | |
32 | * | |
33 | * @usec: CQ timer suggestion (by DIM) | |
34 | * @pkts: CQ packet counter suggestion (by DIM) | |
35 | * @cq_period_mode: CQ priod count mode (from CQE/EQE) | |
36 | */ | |
8960b389 | 37 | struct dim_cq_moder { |
0e58983d TG |
38 | u16 usec; |
39 | u16 pkts; | |
398c2b05 | 40 | u16 comps; |
0e58983d TG |
41 | u8 cq_period_mode; |
42 | }; | |
43 | ||
4f75da36 TG |
44 | /** |
45 | * Structure for DIM sample data. | |
46 | * Used for communications between DIM and its consumer. | |
47 | * | |
48 | * @time: Sample timestamp | |
49 | * @pkt_ctr: Number of packets | |
50 | * @byte_ctr: Number of bytes | |
51 | * @event_ctr: Number of events | |
52 | */ | |
8960b389 | 53 | struct dim_sample { |
0e58983d TG |
54 | ktime_t time; |
55 | u32 pkt_ctr; | |
56 | u32 byte_ctr; | |
57 | u16 event_ctr; | |
398c2b05 | 58 | u32 comp_ctr; |
0e58983d TG |
59 | }; |
60 | ||
4f75da36 TG |
61 | /** |
62 | * Structure for DIM stats. | |
63 | * Used for holding current measured rates. | |
64 | * | |
65 | * @ppms: Packets per msec | |
66 | * @bpms: Bytes per msec | |
67 | * @epms: Events per msec | |
68 | */ | |
449986ea | 69 | struct dim_stats { |
398c2b05 YF |
70 | int ppms; /* packets per msec */ |
71 | int bpms; /* bytes per msec */ | |
72 | int epms; /* events per msec */ | |
73 | int cpms; /* completions per msec */ | |
74 | int cpe_ratio; /* ratio of completions to events */ | |
0e58983d TG |
75 | }; |
76 | ||
4f75da36 TG |
77 | /** |
78 | * Main structure for dynamic interrupt moderation (DIM). | |
79 | * Used for holding all information about a specific DIM instance. | |
80 | * | |
81 | * @state: Algorithm state (see below) | |
82 | * @prev_stats: Measured rates from previous iteration (for comparison) | |
83 | * @start_sample: Sampled data at start of current iteration | |
84 | * @work: Work to perform on action required | |
f4915455 | 85 | * @priv: A pointer to the struct that points to dim |
4f75da36 TG |
86 | * @profile_ix: Current moderation profile |
87 | * @mode: CQ period count mode | |
88 | * @tune_state: Algorithm tuning state (see below) | |
89 | * @steps_right: Number of steps taken towards higher moderation | |
90 | * @steps_left: Number of steps taken towards lower moderation | |
91 | * @tired: Parking depth counter | |
92 | */ | |
93 | struct dim { | |
0e58983d | 94 | u8 state; |
449986ea | 95 | struct dim_stats prev_stats; |
8960b389 | 96 | struct dim_sample start_sample; |
398c2b05 | 97 | struct dim_sample measuring_sample; |
0e58983d | 98 | struct work_struct work; |
f4915455 | 99 | void *priv; |
0e58983d TG |
100 | u8 profile_ix; |
101 | u8 mode; | |
102 | u8 tune_state; | |
103 | u8 steps_right; | |
104 | u8 steps_left; | |
105 | u8 tired; | |
106 | }; | |
107 | ||
4f75da36 TG |
108 | /** |
109 | * enum dim_cq_period_mode | |
110 | * | |
111 | * These are the modes for CQ period count. | |
112 | * | |
113 | * @DIM_CQ_PERIOD_MODE_START_FROM_EQE: Start counting from EQE | |
114 | * @DIM_CQ_PERIOD_MODE_START_FROM_CQE: Start counting from CQE (implies timer reset) | |
115 | * @DIM_CQ_PERIOD_NUM_MODES: Number of modes | |
116 | */ | |
0e58983d | 117 | enum { |
c002bd52 TG |
118 | DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0, |
119 | DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1, | |
120 | DIM_CQ_PERIOD_NUM_MODES | |
0e58983d TG |
121 | }; |
122 | ||
4f75da36 TG |
123 | /** |
124 | * enum dim_state | |
125 | * | |
126 | * These are the DIM algorithm states. | |
127 | * These will determine if the algorithm is in a valid state to start an iteration. | |
128 | * | |
129 | * @DIM_START_MEASURE: This is the first iteration (also after applying a new profile) | |
130 | * @DIM_MEASURE_IN_PROGRESS: Algorithm is already in progress - check if | |
131 | * need to perform an action | |
132 | * @DIM_APPLY_NEW_PROFILE: DIM consumer is currently applying a profile - no need to measure | |
133 | */ | |
0e58983d | 134 | enum { |
c002bd52 TG |
135 | DIM_START_MEASURE, |
136 | DIM_MEASURE_IN_PROGRESS, | |
137 | DIM_APPLY_NEW_PROFILE, | |
0e58983d TG |
138 | }; |
139 | ||
4f75da36 TG |
140 | /** |
141 | * enum dim_tune_state | |
142 | * | |
143 | * These are the DIM algorithm tune states. | |
144 | * These will determine which action the algorithm should perform. | |
145 | * | |
146 | * @DIM_PARKING_ON_TOP: Algorithm found a local top point - exit on significant difference | |
147 | * @DIM_PARKING_TIRED: Algorithm found a deep top point - don't exit if tired > 0 | |
148 | * @DIM_GOING_RIGHT: Algorithm is currently trying higher moderation levels | |
149 | * @DIM_GOING_LEFT: Algorithm is currently trying lower moderation levels | |
150 | */ | |
0e58983d | 151 | enum { |
449986ea TG |
152 | DIM_PARKING_ON_TOP, |
153 | DIM_PARKING_TIRED, | |
154 | DIM_GOING_RIGHT, | |
155 | DIM_GOING_LEFT, | |
0e58983d TG |
156 | }; |
157 | ||
4f75da36 TG |
158 | /** |
159 | * enum dim_stats_state | |
160 | * | |
161 | * These are the DIM algorithm statistics states. | |
162 | * These will determine the verdict of current iteration. | |
163 | * | |
164 | * @DIM_STATS_WORSE: Current iteration shows worse performance than before | |
165 | * @DIM_STATS_WORSE: Current iteration shows same performance than before | |
166 | * @DIM_STATS_WORSE: Current iteration shows better performance than before | |
167 | */ | |
0e58983d | 168 | enum { |
449986ea TG |
169 | DIM_STATS_WORSE, |
170 | DIM_STATS_SAME, | |
171 | DIM_STATS_BETTER, | |
0e58983d TG |
172 | }; |
173 | ||
4f75da36 TG |
174 | /** |
175 | * enum dim_step_result | |
176 | * | |
177 | * These are the DIM algorithm step results. | |
178 | * These describe the result of a step. | |
179 | * | |
180 | * @DIM_STEPPED: Performed a regular step | |
181 | * @DIM_TOO_TIRED: Same kind of step was done multiple times - should go to | |
182 | * tired parking | |
183 | * @DIM_ON_EDGE: Stepped to the most left/right profile | |
184 | */ | |
0e58983d | 185 | enum { |
449986ea TG |
186 | DIM_STEPPED, |
187 | DIM_TOO_TIRED, | |
188 | DIM_ON_EDGE, | |
0e58983d TG |
189 | }; |
190 | ||
4f75da36 TG |
191 | /** |
192 | * dim_on_top - check if current state is a good place to stop (top location) | |
193 | * @dim: DIM context | |
194 | * | |
195 | * Check if current profile is a good place to park at. | |
196 | * This will result in reducing the DIM checks frequency as we assume we | |
197 | * shouldn't probably change profiles, unless traffic pattern wasn't changed. | |
198 | */ | |
199 | bool dim_on_top(struct dim *dim); | |
0e58983d | 200 | |
4f75da36 TG |
201 | /** |
202 | * dim_turn - change profile alterning direction | |
203 | * @dim: DIM context | |
204 | * | |
205 | * Go left if we were going right and vice-versa. | |
206 | * Do nothing if currently parking. | |
207 | */ | |
208 | void dim_turn(struct dim *dim); | |
0e58983d | 209 | |
4f75da36 TG |
210 | /** |
211 | * dim_park_on_top - enter a parking state on a top location | |
212 | * @dim: DIM context | |
213 | * | |
214 | * Enter parking state. | |
215 | * Clear all movement history. | |
216 | */ | |
217 | void dim_park_on_top(struct dim *dim); | |
0e58983d | 218 | |
4f75da36 TG |
219 | /** |
220 | * dim_park_tired - enter a tired parking state | |
221 | * @dim: DIM context | |
222 | * | |
223 | * Enter parking state. | |
224 | * Clear all movement history and cause DIM checks frequency to reduce. | |
225 | */ | |
226 | void dim_park_tired(struct dim *dim); | |
227 | ||
228 | /** | |
229 | * dim_calc_stats - calculate the difference between two samples | |
230 | * @start: start sample | |
231 | * @end: end sample | |
232 | * @curr_stats: delta between samples | |
233 | * | |
234 | * Calculate the delta between two samples (in data rates). | |
235 | * Takes into consideration counter wrap-around. | |
236 | */ | |
237 | void dim_calc_stats(struct dim_sample *start, struct dim_sample *end, | |
238 | struct dim_stats *curr_stats); | |
0e58983d | 239 | |
4f75da36 TG |
240 | /** |
241 | * dim_update_sample - set a sample's fields with give values | |
242 | * @event_ctr: number of events to set | |
243 | * @packets: number of packets to set | |
244 | * @bytes: number of bytes to set | |
245 | * @s: DIM sample | |
246 | */ | |
0e58983d | 247 | static inline void |
8960b389 | 248 | dim_update_sample(u16 event_ctr, u64 packets, u64 bytes, struct dim_sample *s) |
0e58983d TG |
249 | { |
250 | s->time = ktime_get(); | |
251 | s->pkt_ctr = packets; | |
252 | s->byte_ctr = bytes; | |
253 | s->event_ctr = event_ctr; | |
254 | } | |
255 | ||
398c2b05 YF |
256 | /** |
257 | * dim_update_sample_with_comps - set a sample's fields with given | |
258 | * values including the completion parameter | |
259 | * @event_ctr: number of events to set | |
260 | * @packets: number of packets to set | |
261 | * @bytes: number of bytes to set | |
262 | * @comps: number of completions to set | |
263 | * @s: DIM sample | |
264 | */ | |
265 | static inline void | |
266 | dim_update_sample_with_comps(u16 event_ctr, u64 packets, u64 bytes, u64 comps, | |
267 | struct dim_sample *s) | |
268 | { | |
269 | dim_update_sample(event_ctr, packets, bytes, s); | |
270 | s->comp_ctr = comps; | |
271 | } | |
272 | ||
4f75da36 TG |
273 | /* Net DIM */ |
274 | ||
4f75da36 TG |
275 | /** |
276 | * net_dim_get_rx_moderation - provide a CQ moderation object for the given RX profile | |
277 | * @cq_period_mode: CQ period mode | |
278 | * @ix: Profile index | |
279 | */ | |
280 | struct dim_cq_moder net_dim_get_rx_moderation(u8 cq_period_mode, int ix); | |
281 | ||
282 | /** | |
283 | * net_dim_get_def_rx_moderation - provide the default RX moderation | |
284 | * @cq_period_mode: CQ period mode | |
285 | */ | |
286 | struct dim_cq_moder net_dim_get_def_rx_moderation(u8 cq_period_mode); | |
287 | ||
288 | /** | |
289 | * net_dim_get_tx_moderation - provide a CQ moderation object for the given TX profile | |
290 | * @cq_period_mode: CQ period mode | |
291 | * @ix: Profile index | |
292 | */ | |
293 | struct dim_cq_moder net_dim_get_tx_moderation(u8 cq_period_mode, int ix); | |
294 | ||
295 | /** | |
296 | * net_dim_get_def_tx_moderation - provide the default TX moderation | |
297 | * @cq_period_mode: CQ period mode | |
298 | */ | |
299 | struct dim_cq_moder net_dim_get_def_tx_moderation(u8 cq_period_mode); | |
300 | ||
301 | /** | |
302 | * net_dim - main DIM algorithm entry point | |
303 | * @dim: DIM instance information | |
304 | * @end_sample: Current data measurement | |
305 | * | |
306 | * Called by the consumer. | |
307 | * This is the main logic of the algorithm, where data is processed in order to decide on next | |
308 | * required action. | |
309 | */ | |
310 | void net_dim(struct dim *dim, struct dim_sample end_sample); | |
311 | ||
f4915455 YF |
312 | /* RDMA DIM */ |
313 | ||
314 | /* | |
315 | * RDMA DIM profile: | |
316 | * profile size must be of RDMA_DIM_PARAMS_NUM_PROFILES. | |
317 | */ | |
318 | #define RDMA_DIM_PARAMS_NUM_PROFILES 9 | |
319 | #define RDMA_DIM_START_PROFILE 0 | |
320 | ||
f4915455 YF |
321 | /** |
322 | * rdma_dim - Runs the adaptive moderation. | |
323 | * @dim: The moderation struct. | |
324 | * @completions: The number of completions collected in this round. | |
325 | * | |
326 | * Each call to rdma_dim takes the latest amount of completions that | |
327 | * have been collected and counts them as a new event. | |
328 | * Once enough events have been collected the algorithm decides a new | |
329 | * moderation level. | |
330 | */ | |
331 | void rdma_dim(struct dim *dim, u64 completions); | |
332 | ||
0e58983d | 333 | #endif /* DIM_H */ |