Commit | Line | Data |
---|---|---|
96de2506 JK |
1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2017-2018 Netronome Systems, Inc. */ | |
43f84b72 PJV |
3 | |
4 | #include <linux/hash.h> | |
5 | #include <linux/hashtable.h> | |
6 | #include <linux/jhash.h> | |
467322e2 | 7 | #include <linux/math64.h> |
43f84b72 PJV |
8 | #include <linux/vmalloc.h> |
9 | #include <net/pkt_cls.h> | |
10 | ||
11 | #include "cmsg.h" | |
e236e484 | 12 | #include "conntrack.h" |
43f84b72 PJV |
13 | #include "main.h" |
14 | #include "../nfp_app.h" | |
15 | ||
16 | struct nfp_mask_id_table { | |
17 | struct hlist_node link; | |
18 | u32 hash_key; | |
19 | u32 ref_cnt; | |
20 | u8 mask_id; | |
21 | }; | |
22 | ||
c01d0efa PJV |
23 | struct nfp_fl_flow_table_cmp_arg { |
24 | struct net_device *netdev; | |
25 | unsigned long cookie; | |
c01d0efa PJV |
26 | }; |
27 | ||
cf4172d5 JH |
28 | struct nfp_fl_stats_ctx_to_flow { |
29 | struct rhash_head ht_node; | |
30 | u32 stats_cxt; | |
31 | struct nfp_fl_payload *flow; | |
32 | }; | |
33 | ||
34 | static const struct rhashtable_params stats_ctx_table_params = { | |
35 | .key_offset = offsetof(struct nfp_fl_stats_ctx_to_flow, stats_cxt), | |
36 | .head_offset = offsetof(struct nfp_fl_stats_ctx_to_flow, ht_node), | |
37 | .key_len = sizeof(u32), | |
38 | }; | |
39 | ||
abfcdc1d PJV |
40 | static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id) |
41 | { | |
42 | struct nfp_flower_priv *priv = app->priv; | |
43 | struct circ_buf *ring; | |
44 | ||
45 | ring = &priv->stats_ids.free_list; | |
7ffd9041 YZ |
46 | /* Check if buffer is full, stats_ring_size must be power of 2 */ |
47 | if (!CIRC_SPACE(ring->head, ring->tail, priv->stats_ring_size)) | |
abfcdc1d PJV |
48 | return -ENOBUFS; |
49 | ||
7ffd9041 YZ |
50 | /* Each increment of head represents size of NFP_FL_STATS_ELEM_RS */ |
51 | memcpy(&ring->buf[ring->head * NFP_FL_STATS_ELEM_RS], | |
52 | &stats_context_id, NFP_FL_STATS_ELEM_RS); | |
53 | ring->head = (ring->head + 1) & (priv->stats_ring_size - 1); | |
abfcdc1d PJV |
54 | |
55 | return 0; | |
56 | } | |
57 | ||
58 | static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id) | |
59 | { | |
60 | struct nfp_flower_priv *priv = app->priv; | |
61 | u32 freed_stats_id, temp_stats_id; | |
62 | struct circ_buf *ring; | |
63 | ||
64 | ring = &priv->stats_ids.free_list; | |
12ecf615 | 65 | freed_stats_id = priv->stats_ring_size; |
abfcdc1d PJV |
66 | /* Check for unallocated entries first. */ |
67 | if (priv->stats_ids.init_unalloc > 0) { | |
467322e2 PJV |
68 | *stats_context_id = |
69 | FIELD_PREP(NFP_FL_STAT_ID_STAT, | |
70 | priv->stats_ids.init_unalloc - 1) | | |
71 | FIELD_PREP(NFP_FL_STAT_ID_MU_NUM, | |
72 | priv->active_mem_unit); | |
39f14c00 JH |
73 | |
74 | if (++priv->active_mem_unit == priv->total_mem_units) { | |
75 | priv->stats_ids.init_unalloc--; | |
76 | priv->active_mem_unit = 0; | |
77 | } | |
78 | ||
abfcdc1d PJV |
79 | return 0; |
80 | } | |
81 | ||
82 | /* Check if buffer is empty. */ | |
83 | if (ring->head == ring->tail) { | |
84 | *stats_context_id = freed_stats_id; | |
85 | return -ENOENT; | |
86 | } | |
87 | ||
7ffd9041 YZ |
88 | /* Each increment of tail represents size of NFP_FL_STATS_ELEM_RS */ |
89 | memcpy(&temp_stats_id, &ring->buf[ring->tail * NFP_FL_STATS_ELEM_RS], | |
90 | NFP_FL_STATS_ELEM_RS); | |
abfcdc1d | 91 | *stats_context_id = temp_stats_id; |
7ffd9041 YZ |
92 | memcpy(&ring->buf[ring->tail * NFP_FL_STATS_ELEM_RS], &freed_stats_id, |
93 | NFP_FL_STATS_ELEM_RS); | |
94 | /* stats_ring_size must be power of 2 */ | |
95 | ring->tail = (ring->tail + 1) & (priv->stats_ring_size - 1); | |
abfcdc1d PJV |
96 | |
97 | return 0; | |
98 | } | |
99 | ||
43f84b72 PJV |
100 | /* Must be called with either RTNL or rcu_read_lock */ |
101 | struct nfp_fl_payload * | |
54a4a034 | 102 | nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie, |
d4b69bad | 103 | struct net_device *netdev) |
43f84b72 | 104 | { |
c01d0efa | 105 | struct nfp_fl_flow_table_cmp_arg flower_cmp_arg; |
43f84b72 | 106 | struct nfp_flower_priv *priv = app->priv; |
43f84b72 | 107 | |
c01d0efa PJV |
108 | flower_cmp_arg.netdev = netdev; |
109 | flower_cmp_arg.cookie = tc_flower_cookie; | |
43f84b72 | 110 | |
c01d0efa PJV |
111 | return rhashtable_lookup_fast(&priv->flow_table, &flower_cmp_arg, |
112 | nfp_flower_table_params); | |
43f84b72 PJV |
113 | } |
114 | ||
abfcdc1d PJV |
115 | void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb) |
116 | { | |
79ede4ae | 117 | unsigned int msg_len = nfp_flower_cmsg_get_data_len(skb); |
7fade107 PJV |
118 | struct nfp_flower_priv *priv = app->priv; |
119 | struct nfp_fl_stats_frame *stats; | |
abfcdc1d | 120 | unsigned char *msg; |
7fade107 | 121 | u32 ctx_id; |
abfcdc1d PJV |
122 | int i; |
123 | ||
124 | msg = nfp_flower_cmsg_get_data(skb); | |
125 | ||
7fade107 PJV |
126 | spin_lock(&priv->stats_lock); |
127 | for (i = 0; i < msg_len / sizeof(*stats); i++) { | |
128 | stats = (struct nfp_fl_stats_frame *)msg + i; | |
129 | ctx_id = be32_to_cpu(stats->stats_con_id); | |
130 | priv->stats[ctx_id].pkts += be32_to_cpu(stats->pkt_count); | |
131 | priv->stats[ctx_id].bytes += be64_to_cpu(stats->byte_count); | |
132 | priv->stats[ctx_id].used = jiffies; | |
133 | } | |
134 | spin_unlock(&priv->stats_lock); | |
abfcdc1d PJV |
135 | } |
136 | ||
43f84b72 PJV |
137 | static int nfp_release_mask_id(struct nfp_app *app, u8 mask_id) |
138 | { | |
139 | struct nfp_flower_priv *priv = app->priv; | |
140 | struct circ_buf *ring; | |
43f84b72 PJV |
141 | |
142 | ring = &priv->mask_ids.mask_id_free_list; | |
7ffd9041 YZ |
143 | /* Checking if buffer is full, |
144 | * NFP_FLOWER_MASK_ENTRY_RS must be power of 2 | |
145 | */ | |
43f84b72 PJV |
146 | if (CIRC_SPACE(ring->head, ring->tail, NFP_FLOWER_MASK_ENTRY_RS) == 0) |
147 | return -ENOBUFS; | |
148 | ||
7ffd9041 YZ |
149 | /* Each increment of head represents size of |
150 | * NFP_FLOWER_MASK_ELEMENT_RS | |
151 | */ | |
152 | memcpy(&ring->buf[ring->head * NFP_FLOWER_MASK_ELEMENT_RS], &mask_id, | |
153 | NFP_FLOWER_MASK_ELEMENT_RS); | |
154 | ring->head = (ring->head + 1) & (NFP_FLOWER_MASK_ENTRY_RS - 1); | |
43f84b72 | 155 | |
51bef926 | 156 | priv->mask_ids.last_used[mask_id] = ktime_get(); |
43f84b72 PJV |
157 | |
158 | return 0; | |
159 | } | |
160 | ||
161 | static int nfp_mask_alloc(struct nfp_app *app, u8 *mask_id) | |
162 | { | |
163 | struct nfp_flower_priv *priv = app->priv; | |
51bef926 | 164 | ktime_t reuse_timeout; |
43f84b72 PJV |
165 | struct circ_buf *ring; |
166 | u8 temp_id, freed_id; | |
167 | ||
168 | ring = &priv->mask_ids.mask_id_free_list; | |
169 | freed_id = NFP_FLOWER_MASK_ENTRY_RS - 1; | |
170 | /* Checking for unallocated entries first. */ | |
171 | if (priv->mask_ids.init_unallocated > 0) { | |
172 | *mask_id = priv->mask_ids.init_unallocated; | |
173 | priv->mask_ids.init_unallocated--; | |
174 | return 0; | |
175 | } | |
176 | ||
177 | /* Checking if buffer is empty. */ | |
178 | if (ring->head == ring->tail) | |
179 | goto err_not_found; | |
180 | ||
7ffd9041 YZ |
181 | /* Each increment of tail represents size of |
182 | * NFP_FLOWER_MASK_ELEMENT_RS | |
183 | */ | |
184 | memcpy(&temp_id, &ring->buf[ring->tail * NFP_FLOWER_MASK_ELEMENT_RS], | |
185 | NFP_FLOWER_MASK_ELEMENT_RS); | |
43f84b72 PJV |
186 | *mask_id = temp_id; |
187 | ||
51bef926 AB |
188 | reuse_timeout = ktime_add_ns(priv->mask_ids.last_used[*mask_id], |
189 | NFP_FL_MASK_REUSE_TIME_NS); | |
43f84b72 | 190 | |
51bef926 | 191 | if (ktime_before(ktime_get(), reuse_timeout)) |
43f84b72 PJV |
192 | goto err_not_found; |
193 | ||
7ffd9041 YZ |
194 | memcpy(&ring->buf[ring->tail * NFP_FLOWER_MASK_ELEMENT_RS], &freed_id, |
195 | NFP_FLOWER_MASK_ELEMENT_RS); | |
196 | /* NFP_FLOWER_MASK_ENTRY_RS must be power of 2 */ | |
197 | ring->tail = (ring->tail + 1) & (NFP_FLOWER_MASK_ENTRY_RS - 1); | |
43f84b72 PJV |
198 | |
199 | return 0; | |
200 | ||
201 | err_not_found: | |
202 | *mask_id = freed_id; | |
203 | return -ENOENT; | |
204 | } | |
205 | ||
206 | static int | |
207 | nfp_add_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) | |
208 | { | |
209 | struct nfp_flower_priv *priv = app->priv; | |
210 | struct nfp_mask_id_table *mask_entry; | |
211 | unsigned long hash_key; | |
212 | u8 mask_id; | |
213 | ||
214 | if (nfp_mask_alloc(app, &mask_id)) | |
215 | return -ENOENT; | |
216 | ||
217 | mask_entry = kmalloc(sizeof(*mask_entry), GFP_KERNEL); | |
218 | if (!mask_entry) { | |
219 | nfp_release_mask_id(app, mask_id); | |
220 | return -ENOMEM; | |
221 | } | |
222 | ||
223 | INIT_HLIST_NODE(&mask_entry->link); | |
224 | mask_entry->mask_id = mask_id; | |
225 | hash_key = jhash(mask_data, mask_len, priv->mask_id_seed); | |
226 | mask_entry->hash_key = hash_key; | |
227 | mask_entry->ref_cnt = 1; | |
228 | hash_add(priv->mask_table, &mask_entry->link, hash_key); | |
229 | ||
230 | return mask_id; | |
231 | } | |
232 | ||
233 | static struct nfp_mask_id_table * | |
234 | nfp_search_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) | |
235 | { | |
236 | struct nfp_flower_priv *priv = app->priv; | |
237 | struct nfp_mask_id_table *mask_entry; | |
238 | unsigned long hash_key; | |
239 | ||
240 | hash_key = jhash(mask_data, mask_len, priv->mask_id_seed); | |
241 | ||
242 | hash_for_each_possible(priv->mask_table, mask_entry, link, hash_key) | |
243 | if (mask_entry->hash_key == hash_key) | |
244 | return mask_entry; | |
245 | ||
246 | return NULL; | |
247 | } | |
248 | ||
249 | static int | |
250 | nfp_find_in_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) | |
251 | { | |
252 | struct nfp_mask_id_table *mask_entry; | |
253 | ||
254 | mask_entry = nfp_search_mask_table(app, mask_data, mask_len); | |
255 | if (!mask_entry) | |
256 | return -ENOENT; | |
257 | ||
258 | mask_entry->ref_cnt++; | |
259 | ||
260 | /* Casting u8 to int for later use. */ | |
261 | return mask_entry->mask_id; | |
262 | } | |
263 | ||
264 | static bool | |
265 | nfp_check_mask_add(struct nfp_app *app, char *mask_data, u32 mask_len, | |
266 | u8 *meta_flags, u8 *mask_id) | |
267 | { | |
268 | int id; | |
269 | ||
270 | id = nfp_find_in_mask_table(app, mask_data, mask_len); | |
271 | if (id < 0) { | |
272 | id = nfp_add_mask_table(app, mask_data, mask_len); | |
273 | if (id < 0) | |
274 | return false; | |
6c3ab204 | 275 | *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; |
43f84b72 PJV |
276 | } |
277 | *mask_id = id; | |
278 | ||
279 | return true; | |
280 | } | |
281 | ||
282 | static bool | |
283 | nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len, | |
284 | u8 *meta_flags, u8 *mask_id) | |
285 | { | |
286 | struct nfp_mask_id_table *mask_entry; | |
287 | ||
288 | mask_entry = nfp_search_mask_table(app, mask_data, mask_len); | |
289 | if (!mask_entry) | |
290 | return false; | |
291 | ||
292 | *mask_id = mask_entry->mask_id; | |
293 | mask_entry->ref_cnt--; | |
294 | if (!mask_entry->ref_cnt) { | |
295 | hash_del(&mask_entry->link); | |
296 | nfp_release_mask_id(app, *mask_id); | |
297 | kfree(mask_entry); | |
298 | if (meta_flags) | |
6c3ab204 | 299 | *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; |
43f84b72 PJV |
300 | } |
301 | ||
302 | return true; | |
303 | } | |
304 | ||
4b15fb18 | 305 | int nfp_compile_flow_metadata(struct nfp_app *app, u32 cookie, |
54a4a034 | 306 | struct nfp_fl_payload *nfp_flow, |
bef6e97d PJV |
307 | struct net_device *netdev, |
308 | struct netlink_ext_ack *extack) | |
43f84b72 | 309 | { |
cf4172d5 | 310 | struct nfp_fl_stats_ctx_to_flow *ctx_entry; |
43f84b72 PJV |
311 | struct nfp_flower_priv *priv = app->priv; |
312 | struct nfp_fl_payload *check_entry; | |
313 | u8 new_mask_id; | |
abfcdc1d | 314 | u32 stats_cxt; |
cf4172d5 | 315 | int err; |
abfcdc1d | 316 | |
cf4172d5 | 317 | err = nfp_get_stats_entry(app, &stats_cxt); |
bef6e97d PJV |
318 | if (err) { |
319 | NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot allocate new stats context"); | |
cf4172d5 | 320 | return err; |
bef6e97d | 321 | } |
abfcdc1d PJV |
322 | |
323 | nfp_flow->meta.host_ctx_id = cpu_to_be32(stats_cxt); | |
4b15fb18 | 324 | nfp_flow->meta.host_cookie = cpu_to_be64(cookie); |
4f63fde3 | 325 | nfp_flow->ingress_dev = netdev; |
43f84b72 | 326 | |
cf4172d5 JH |
327 | ctx_entry = kzalloc(sizeof(*ctx_entry), GFP_KERNEL); |
328 | if (!ctx_entry) { | |
329 | err = -ENOMEM; | |
330 | goto err_release_stats; | |
331 | } | |
332 | ||
333 | ctx_entry->stats_cxt = stats_cxt; | |
334 | ctx_entry->flow = nfp_flow; | |
335 | ||
336 | if (rhashtable_insert_fast(&priv->stats_ctx_table, &ctx_entry->ht_node, | |
337 | stats_ctx_table_params)) { | |
338 | err = -ENOMEM; | |
339 | goto err_free_ctx_entry; | |
340 | } | |
341 | ||
9bacb93b | 342 | /* Do not allocate a mask-id for pre_tun_rules. These flows are used to |
d8ce0275 LP |
343 | * configure the pre_tun table and are never actually send to the |
344 | * firmware as an add-flow message. This causes the mask-id allocation | |
345 | * on the firmware to get out of sync if allocated here. | |
346 | */ | |
43f84b72 | 347 | new_mask_id = 0; |
d8ce0275 LP |
348 | if (!nfp_flow->pre_tun_rule.dev && |
349 | !nfp_check_mask_add(app, nfp_flow->mask_data, | |
43f84b72 | 350 | nfp_flow->meta.mask_len, |
abfcdc1d | 351 | &nfp_flow->meta.flags, &new_mask_id)) { |
bef6e97d | 352 | NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot allocate a new mask id"); |
cf4172d5 JH |
353 | err = -ENOENT; |
354 | goto err_remove_rhash; | |
abfcdc1d | 355 | } |
43f84b72 PJV |
356 | |
357 | nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); | |
358 | priv->flower_version++; | |
359 | ||
360 | /* Update flow payload with mask ids. */ | |
361 | nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id; | |
7fade107 PJV |
362 | priv->stats[stats_cxt].pkts = 0; |
363 | priv->stats[stats_cxt].bytes = 0; | |
364 | priv->stats[stats_cxt].used = jiffies; | |
43f84b72 | 365 | |
4b15fb18 | 366 | check_entry = nfp_flower_search_fl_table(app, cookie, netdev); |
43f84b72 | 367 | if (check_entry) { |
bef6e97d | 368 | NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot offload duplicate flow entry"); |
cf4172d5 JH |
369 | err = -EEXIST; |
370 | goto err_remove_mask; | |
43f84b72 PJV |
371 | } |
372 | ||
373 | return 0; | |
cf4172d5 JH |
374 | |
375 | err_remove_mask: | |
d8ce0275 LP |
376 | if (!nfp_flow->pre_tun_rule.dev) |
377 | nfp_check_mask_remove(app, nfp_flow->mask_data, | |
378 | nfp_flow->meta.mask_len, | |
379 | NULL, &new_mask_id); | |
cf4172d5 JH |
380 | err_remove_rhash: |
381 | WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table, | |
382 | &ctx_entry->ht_node, | |
383 | stats_ctx_table_params)); | |
384 | err_free_ctx_entry: | |
385 | kfree(ctx_entry); | |
386 | err_release_stats: | |
387 | nfp_release_stats_entry(app, stats_cxt); | |
388 | ||
389 | return err; | |
43f84b72 PJV |
390 | } |
391 | ||
8af56f40 JH |
392 | void __nfp_modify_flow_metadata(struct nfp_flower_priv *priv, |
393 | struct nfp_fl_payload *nfp_flow) | |
394 | { | |
395 | nfp_flow->meta.flags &= ~NFP_FL_META_FLAG_MANAGE_MASK; | |
396 | nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); | |
397 | priv->flower_version++; | |
398 | } | |
399 | ||
43f84b72 PJV |
400 | int nfp_modify_flow_metadata(struct nfp_app *app, |
401 | struct nfp_fl_payload *nfp_flow) | |
402 | { | |
cf4172d5 | 403 | struct nfp_fl_stats_ctx_to_flow *ctx_entry; |
43f84b72 PJV |
404 | struct nfp_flower_priv *priv = app->priv; |
405 | u8 new_mask_id = 0; | |
abfcdc1d | 406 | u32 temp_ctx_id; |
43f84b72 | 407 | |
8af56f40 JH |
408 | __nfp_modify_flow_metadata(priv, nfp_flow); |
409 | ||
d8ce0275 LP |
410 | if (!nfp_flow->pre_tun_rule.dev) |
411 | nfp_check_mask_remove(app, nfp_flow->mask_data, | |
412 | nfp_flow->meta.mask_len, &nfp_flow->meta.flags, | |
413 | &new_mask_id); | |
43f84b72 | 414 | |
43f84b72 PJV |
415 | /* Update flow payload with mask ids. */ |
416 | nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id; | |
417 | ||
cf4172d5 | 418 | /* Release the stats ctx id and ctx to flow table entry. */ |
abfcdc1d PJV |
419 | temp_ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id); |
420 | ||
cf4172d5 JH |
421 | ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &temp_ctx_id, |
422 | stats_ctx_table_params); | |
423 | if (!ctx_entry) | |
424 | return -ENOENT; | |
425 | ||
426 | WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table, | |
427 | &ctx_entry->ht_node, | |
428 | stats_ctx_table_params)); | |
429 | kfree(ctx_entry); | |
430 | ||
abfcdc1d | 431 | return nfp_release_stats_entry(app, temp_ctx_id); |
43f84b72 PJV |
432 | } |
433 | ||
cf4172d5 JH |
434 | struct nfp_fl_payload * |
435 | nfp_flower_get_fl_payload_from_ctx(struct nfp_app *app, u32 ctx_id) | |
436 | { | |
437 | struct nfp_fl_stats_ctx_to_flow *ctx_entry; | |
438 | struct nfp_flower_priv *priv = app->priv; | |
439 | ||
440 | ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &ctx_id, | |
441 | stats_ctx_table_params); | |
442 | if (!ctx_entry) | |
443 | return NULL; | |
444 | ||
445 | return ctx_entry->flow; | |
446 | } | |
447 | ||
c01d0efa PJV |
448 | static int nfp_fl_obj_cmpfn(struct rhashtable_compare_arg *arg, |
449 | const void *obj) | |
450 | { | |
451 | const struct nfp_fl_flow_table_cmp_arg *cmp_arg = arg->key; | |
452 | const struct nfp_fl_payload *flow_entry = obj; | |
453 | ||
d4b69bad | 454 | if (flow_entry->ingress_dev == cmp_arg->netdev) |
c01d0efa PJV |
455 | return flow_entry->tc_flower_cookie != cmp_arg->cookie; |
456 | ||
457 | return 1; | |
458 | } | |
459 | ||
460 | static u32 nfp_fl_obj_hashfn(const void *data, u32 len, u32 seed) | |
461 | { | |
462 | const struct nfp_fl_payload *flower_entry = data; | |
463 | ||
464 | return jhash2((u32 *)&flower_entry->tc_flower_cookie, | |
465 | sizeof(flower_entry->tc_flower_cookie) / sizeof(u32), | |
466 | seed); | |
467 | } | |
468 | ||
469 | static u32 nfp_fl_key_hashfn(const void *data, u32 len, u32 seed) | |
470 | { | |
471 | const struct nfp_fl_flow_table_cmp_arg *cmp_arg = data; | |
472 | ||
473 | return jhash2((u32 *)&cmp_arg->cookie, | |
474 | sizeof(cmp_arg->cookie) / sizeof(u32), seed); | |
475 | } | |
476 | ||
477 | const struct rhashtable_params nfp_flower_table_params = { | |
478 | .head_offset = offsetof(struct nfp_fl_payload, fl_node), | |
479 | .hashfn = nfp_fl_key_hashfn, | |
480 | .obj_cmpfn = nfp_fl_obj_cmpfn, | |
481 | .obj_hashfn = nfp_fl_obj_hashfn, | |
482 | .automatic_shrinking = true, | |
483 | }; | |
484 | ||
2ea538db YZ |
485 | const struct rhashtable_params merge_table_params = { |
486 | .key_offset = offsetof(struct nfp_merge_info, parent_ctx), | |
487 | .head_offset = offsetof(struct nfp_merge_info, ht_node), | |
488 | .key_len = sizeof(u64), | |
489 | }; | |
490 | ||
e236e484 LP |
491 | const struct rhashtable_params nfp_zone_table_params = { |
492 | .head_offset = offsetof(struct nfp_fl_ct_zone_entry, hash_node), | |
493 | .key_len = sizeof(u16), | |
494 | .key_offset = offsetof(struct nfp_fl_ct_zone_entry, zone), | |
495 | .automatic_shrinking = false, | |
496 | }; | |
497 | ||
fa81d6d2 LP |
498 | const struct rhashtable_params nfp_ct_map_params = { |
499 | .head_offset = offsetof(struct nfp_fl_ct_map_entry, hash_node), | |
500 | .key_len = sizeof(unsigned long), | |
501 | .key_offset = offsetof(struct nfp_fl_ct_map_entry, cookie), | |
502 | .automatic_shrinking = true, | |
503 | }; | |
504 | ||
29c69134 LP |
505 | const struct rhashtable_params neigh_table_params = { |
506 | .key_offset = offsetof(struct nfp_neigh_entry, neigh_cookie), | |
507 | .head_offset = offsetof(struct nfp_neigh_entry, ht_node), | |
508 | .key_len = sizeof(unsigned long), | |
509 | }; | |
510 | ||
467322e2 PJV |
511 | int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count, |
512 | unsigned int host_num_mems) | |
43f84b72 PJV |
513 | { |
514 | struct nfp_flower_priv *priv = app->priv; | |
467322e2 | 515 | int err, stats_size; |
43f84b72 PJV |
516 | |
517 | hash_init(priv->mask_table); | |
c01d0efa PJV |
518 | |
519 | err = rhashtable_init(&priv->flow_table, &nfp_flower_table_params); | |
520 | if (err) | |
521 | return err; | |
522 | ||
cf4172d5 JH |
523 | err = rhashtable_init(&priv->stats_ctx_table, &stats_ctx_table_params); |
524 | if (err) | |
525 | goto err_free_flow_table; | |
526 | ||
2ea538db YZ |
527 | err = rhashtable_init(&priv->merge_table, &merge_table_params); |
528 | if (err) | |
529 | goto err_free_stats_ctx_table; | |
530 | ||
14690995 YL |
531 | mutex_init(&priv->nfp_fl_lock); |
532 | ||
e236e484 LP |
533 | err = rhashtable_init(&priv->ct_zone_table, &nfp_zone_table_params); |
534 | if (err) | |
535 | goto err_free_merge_table; | |
536 | ||
fa81d6d2 LP |
537 | err = rhashtable_init(&priv->ct_map_table, &nfp_ct_map_params); |
538 | if (err) | |
539 | goto err_free_ct_zone_table; | |
540 | ||
29c69134 LP |
541 | err = rhashtable_init(&priv->neigh_table, &neigh_table_params); |
542 | if (err) | |
543 | goto err_free_ct_map_table; | |
544 | ||
545 | INIT_LIST_HEAD(&priv->predt_list); | |
546 | ||
43f84b72 PJV |
547 | get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed)); |
548 | ||
549 | /* Init ring buffer and unallocated mask_ids. */ | |
550 | priv->mask_ids.mask_id_free_list.buf = | |
551 | kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, | |
552 | NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL); | |
553 | if (!priv->mask_ids.mask_id_free_list.buf) | |
29c69134 | 554 | goto err_free_neigh_table; |
43f84b72 PJV |
555 | |
556 | priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1; | |
557 | ||
558 | /* Init timestamps for mask id*/ | |
559 | priv->mask_ids.last_used = | |
560 | kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, | |
561 | sizeof(*priv->mask_ids.last_used), GFP_KERNEL); | |
562 | if (!priv->mask_ids.last_used) | |
563 | goto err_free_mask_id; | |
564 | ||
abfcdc1d PJV |
565 | /* Init ring buffer and unallocated stats_ids. */ |
566 | priv->stats_ids.free_list.buf = | |
42bc47b3 | 567 | vmalloc(array_size(NFP_FL_STATS_ELEM_RS, |
12ecf615 | 568 | priv->stats_ring_size)); |
abfcdc1d PJV |
569 | if (!priv->stats_ids.free_list.buf) |
570 | goto err_free_last_used; | |
571 | ||
467322e2 | 572 | priv->stats_ids.init_unalloc = div_u64(host_ctx_count, host_num_mems); |
abfcdc1d | 573 | |
467322e2 PJV |
574 | stats_size = FIELD_PREP(NFP_FL_STAT_ID_STAT, host_ctx_count) | |
575 | FIELD_PREP(NFP_FL_STAT_ID_MU_NUM, host_num_mems - 1); | |
576 | priv->stats = kvmalloc_array(stats_size, sizeof(struct nfp_fl_stats), | |
577 | GFP_KERNEL); | |
7fade107 PJV |
578 | if (!priv->stats) |
579 | goto err_free_ring_buf; | |
580 | ||
581 | spin_lock_init(&priv->stats_lock); | |
29c69134 | 582 | spin_lock_init(&priv->predt_lock); |
7fade107 | 583 | |
43f84b72 PJV |
584 | return 0; |
585 | ||
7fade107 PJV |
586 | err_free_ring_buf: |
587 | vfree(priv->stats_ids.free_list.buf); | |
abfcdc1d | 588 | err_free_last_used: |
88f0f09b | 589 | kfree(priv->mask_ids.last_used); |
43f84b72 PJV |
590 | err_free_mask_id: |
591 | kfree(priv->mask_ids.mask_id_free_list.buf); | |
29c69134 LP |
592 | err_free_neigh_table: |
593 | rhashtable_destroy(&priv->neigh_table); | |
fa81d6d2 LP |
594 | err_free_ct_map_table: |
595 | rhashtable_destroy(&priv->ct_map_table); | |
e236e484 LP |
596 | err_free_ct_zone_table: |
597 | rhashtable_destroy(&priv->ct_zone_table); | |
2ea538db YZ |
598 | err_free_merge_table: |
599 | rhashtable_destroy(&priv->merge_table); | |
cf4172d5 JH |
600 | err_free_stats_ctx_table: |
601 | rhashtable_destroy(&priv->stats_ctx_table); | |
c01d0efa PJV |
602 | err_free_flow_table: |
603 | rhashtable_destroy(&priv->flow_table); | |
43f84b72 PJV |
604 | return -ENOMEM; |
605 | } | |
606 | ||
072c089c LP |
607 | static void nfp_zone_table_entry_destroy(struct nfp_fl_ct_zone_entry *zt) |
608 | { | |
609 | if (!zt) | |
610 | return; | |
611 | ||
612 | if (!list_empty(&zt->pre_ct_list)) { | |
fa81d6d2 | 613 | struct rhashtable *m_table = &zt->priv->ct_map_table; |
072c089c | 614 | struct nfp_fl_ct_flow_entry *entry, *tmp; |
fa81d6d2 | 615 | struct nfp_fl_ct_map_entry *map; |
072c089c LP |
616 | |
617 | WARN_ONCE(1, "pre_ct_list not empty as expected, cleaning up\n"); | |
618 | list_for_each_entry_safe(entry, tmp, &zt->pre_ct_list, | |
619 | list_node) { | |
fa81d6d2 LP |
620 | map = rhashtable_lookup_fast(m_table, |
621 | &entry->cookie, | |
622 | nfp_ct_map_params); | |
623 | WARN_ON_ONCE(rhashtable_remove_fast(m_table, | |
624 | &map->hash_node, | |
625 | nfp_ct_map_params)); | |
072c089c | 626 | nfp_fl_ct_clean_flow_entry(entry); |
fa81d6d2 | 627 | kfree(map); |
072c089c LP |
628 | } |
629 | } | |
630 | ||
631 | if (!list_empty(&zt->post_ct_list)) { | |
fa81d6d2 | 632 | struct rhashtable *m_table = &zt->priv->ct_map_table; |
072c089c | 633 | struct nfp_fl_ct_flow_entry *entry, *tmp; |
fa81d6d2 | 634 | struct nfp_fl_ct_map_entry *map; |
072c089c LP |
635 | |
636 | WARN_ONCE(1, "post_ct_list not empty as expected, cleaning up\n"); | |
637 | list_for_each_entry_safe(entry, tmp, &zt->post_ct_list, | |
638 | list_node) { | |
fa81d6d2 LP |
639 | map = rhashtable_lookup_fast(m_table, |
640 | &entry->cookie, | |
641 | nfp_ct_map_params); | |
642 | WARN_ON_ONCE(rhashtable_remove_fast(m_table, | |
643 | &map->hash_node, | |
644 | nfp_ct_map_params)); | |
072c089c | 645 | nfp_fl_ct_clean_flow_entry(entry); |
fa81d6d2 | 646 | kfree(map); |
072c089c LP |
647 | } |
648 | } | |
f7ae12e2 | 649 | |
95255017 LP |
650 | if (zt->nft) { |
651 | nf_flow_table_offload_del_cb(zt->nft, | |
652 | nfp_fl_ct_handle_nft_flow, | |
653 | zt); | |
654 | zt->nft = NULL; | |
655 | } | |
656 | ||
657 | if (!list_empty(&zt->nft_flows_list)) { | |
658 | struct rhashtable *m_table = &zt->priv->ct_map_table; | |
659 | struct nfp_fl_ct_flow_entry *entry, *tmp; | |
660 | struct nfp_fl_ct_map_entry *map; | |
661 | ||
662 | WARN_ONCE(1, "nft_flows_list not empty as expected, cleaning up\n"); | |
663 | list_for_each_entry_safe(entry, tmp, &zt->nft_flows_list, | |
664 | list_node) { | |
665 | map = rhashtable_lookup_fast(m_table, | |
666 | &entry->cookie, | |
667 | nfp_ct_map_params); | |
668 | WARN_ON_ONCE(rhashtable_remove_fast(m_table, | |
669 | &map->hash_node, | |
670 | nfp_ct_map_params)); | |
671 | nfp_fl_ct_clean_flow_entry(entry); | |
672 | kfree(map); | |
673 | } | |
674 | } | |
675 | ||
f7ae12e2 LP |
676 | rhashtable_free_and_destroy(&zt->tc_merge_tb, |
677 | nfp_check_rhashtable_empty, NULL); | |
b5e30c61 LP |
678 | rhashtable_free_and_destroy(&zt->nft_merge_tb, |
679 | nfp_check_rhashtable_empty, NULL); | |
f7ae12e2 | 680 | |
072c089c LP |
681 | kfree(zt); |
682 | } | |
683 | ||
e236e484 LP |
684 | static void nfp_free_zone_table_entry(void *ptr, void *arg) |
685 | { | |
bd0fe7f9 LP |
686 | struct nfp_fl_ct_zone_entry *zt = ptr; |
687 | ||
072c089c | 688 | nfp_zone_table_entry_destroy(zt); |
e236e484 LP |
689 | } |
690 | ||
fa81d6d2 LP |
691 | static void nfp_free_map_table_entry(void *ptr, void *arg) |
692 | { | |
693 | struct nfp_fl_ct_map_entry *map = ptr; | |
694 | ||
695 | if (!map) | |
696 | return; | |
697 | ||
698 | kfree(map); | |
699 | } | |
700 | ||
43f84b72 PJV |
701 | void nfp_flower_metadata_cleanup(struct nfp_app *app) |
702 | { | |
703 | struct nfp_flower_priv *priv = app->priv; | |
704 | ||
705 | if (!priv) | |
706 | return; | |
707 | ||
c01d0efa PJV |
708 | rhashtable_free_and_destroy(&priv->flow_table, |
709 | nfp_check_rhashtable_empty, NULL); | |
cf4172d5 JH |
710 | rhashtable_free_and_destroy(&priv->stats_ctx_table, |
711 | nfp_check_rhashtable_empty, NULL); | |
2ea538db YZ |
712 | rhashtable_free_and_destroy(&priv->merge_table, |
713 | nfp_check_rhashtable_empty, NULL); | |
e236e484 LP |
714 | rhashtable_free_and_destroy(&priv->ct_zone_table, |
715 | nfp_free_zone_table_entry, NULL); | |
072c089c | 716 | nfp_zone_table_entry_destroy(priv->ct_zone_wc); |
fa81d6d2 LP |
717 | |
718 | rhashtable_free_and_destroy(&priv->ct_map_table, | |
719 | nfp_free_map_table_entry, NULL); | |
29c69134 LP |
720 | rhashtable_free_and_destroy(&priv->neigh_table, |
721 | nfp_check_rhashtable_empty, NULL); | |
7fade107 | 722 | kvfree(priv->stats); |
43f84b72 PJV |
723 | kfree(priv->mask_ids.mask_id_free_list.buf); |
724 | kfree(priv->mask_ids.last_used); | |
abfcdc1d | 725 | vfree(priv->stats_ids.free_list.buf); |
43f84b72 | 726 | } |