projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
treewide: Use fallthrough pseudo-keyword
[linux-block.git] / drivers / net / ethernet / sfc / mcdi_filters.c
CommitLineData
2c6c1e3c
EC		 1// SPDX-License-Identifier: GPL-2.0-only
2/****************************************************************************
3 * Driver for Solarflare network controllers and boards
4 * Copyright 2005-2018 Solarflare Communications Inc.
5 * Copyright 2019-2020 Xilinx Inc.
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 as published
9 * by the Free Software Foundation, incorporated herein by reference.
10 */
11
6c77065b
AM		 12#include "mcdi_filters.h"
13#include "mcdi.h"
14#include "nic.h"
15#include "rx_common.h"
16
17/* The maximum size of a shared RSS context */
18/* TODO: this should really be from the mcdi protocol export */
19#define EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE 64UL
20
21#define EFX_EF10_FILTER_ID_INVALID 0xffff
22
23/* An arbitrary search limit for the software hash table */
24#define EFX_EF10_FILTER_SEARCH_LIMIT 200
25
26static struct efx_filter_spec *
27efx_mcdi_filter_entry_spec(const struct efx_mcdi_filter_table *table,
28 unsigned int filter_idx)
29{
30 return (struct efx_filter_spec *)(table->entry[filter_idx].spec &
31 ~EFX_EF10_FILTER_FLAGS);
32}
33
34static unsigned int
35efx_mcdi_filter_entry_flags(const struct efx_mcdi_filter_table *table,
36 unsigned int filter_idx)
37{
38 return table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAGS;
39}
40
41static u32 efx_mcdi_filter_get_unsafe_id(u32 filter_id)
42{
43 WARN_ON_ONCE(filter_id == EFX_EF10_FILTER_ID_INVALID);
44 return filter_id & (EFX_MCDI_FILTER_TBL_ROWS - 1);
45}
46
47static unsigned int efx_mcdi_filter_get_unsafe_pri(u32 filter_id)
48{
49 return filter_id / (EFX_MCDI_FILTER_TBL_ROWS * 2);
50}
51
52static u32 efx_mcdi_filter_make_filter_id(unsigned int pri, u16 idx)
53{
54 return pri * EFX_MCDI_FILTER_TBL_ROWS * 2 + idx;
55}
56
57/*
58 * Decide whether a filter should be exclusive or else should allow
59 * delivery to additional recipients. Currently we decide that
60 * filters for specific local unicast MAC and IP addresses are
61 * exclusive.
62 */
63static bool efx_mcdi_filter_is_exclusive(const struct efx_filter_spec *spec)
64{
65 if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC &&
66 !is_multicast_ether_addr(spec->loc_mac))
67 return true;
68
69 if ((spec->match_flags &
70 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) ==
71 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) {
72 if (spec->ether_type == htons(ETH_P_IP) &&
73 !ipv4_is_multicast(spec->loc_host[0]))
74 return true;
75 if (spec->ether_type == htons(ETH_P_IPV6) &&
76 ((const u8 *)spec->loc_host)[0] != 0xff)
77 return true;
78 }
79
80 return false;
81}
82
83static void
84efx_mcdi_filter_set_entry(struct efx_mcdi_filter_table *table,
85 unsigned int filter_idx,
86 const struct efx_filter_spec *spec,
87 unsigned int flags)
88{
89 table->entry[filter_idx].spec = (unsigned long)spec | flags;
90}
91
92static void
93efx_mcdi_filter_push_prep_set_match_fields(struct efx_nic *efx,
94 const struct efx_filter_spec *spec,
95 efx_dword_t *inbuf)
96{
97 enum efx_encap_type encap_type = efx_filter_get_encap_type(spec);
98 u32 match_fields = 0, uc_match, mc_match;
99
100 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
101 efx_mcdi_filter_is_exclusive(spec) ?
102 MC_CMD_FILTER_OP_IN_OP_INSERT :
103 MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE);
104
105 /*
106 * Convert match flags and values. Unlike almost
107 * everything else in MCDI, these fields are in
108 * network byte order.
109 */
110#define COPY_VALUE(value, mcdi_field) \
111 do { \
112 match_fields |= \
113 1 << MC_CMD_FILTER_OP_IN_MATCH_ ## \
114 mcdi_field ## _LBN; \
115 BUILD_BUG_ON( \
116 MC_CMD_FILTER_OP_IN_ ## mcdi_field ## _LEN < \
117 sizeof(value)); \
118 memcpy(MCDI_PTR(inbuf, FILTER_OP_IN_ ## mcdi_field), \
119 &value, sizeof(value)); \
120 } while (0)
121#define COPY_FIELD(gen_flag, gen_field, mcdi_field) \
122 if (spec->match_flags & EFX_FILTER_MATCH_ ## gen_flag) { \
123 COPY_VALUE(spec->gen_field, mcdi_field); \
124 }
125 /*
126 * Handle encap filters first. They will always be mismatch
127 * (unknown UC or MC) filters
128 */
129 if (encap_type) {
130 /*
131 * ether_type and outer_ip_proto need to be variables
132 * because COPY_VALUE wants to memcpy them
133 */
134 __be16 ether_type =
135 htons(encap_type & EFX_ENCAP_FLAG_IPV6 ?
136 ETH_P_IPV6 : ETH_P_IP);
137 u8 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE;
138 u8 outer_ip_proto;
139
140 switch (encap_type & EFX_ENCAP_TYPES_MASK) {
141 case EFX_ENCAP_TYPE_VXLAN:
142 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN;
df561f66 143 fallthrough;
6c77065b
AM		 144 case EFX_ENCAP_TYPE_GENEVE:
145 COPY_VALUE(ether_type, ETHER_TYPE);
146 outer_ip_proto = IPPROTO_UDP;
147 COPY_VALUE(outer_ip_proto, IP_PROTO);
148 /*
149 * We always need to set the type field, even
150 * though we're not matching on the TNI.
151 */
152 MCDI_POPULATE_DWORD_1(inbuf,
153 FILTER_OP_EXT_IN_VNI_OR_VSID,
154 FILTER_OP_EXT_IN_VNI_TYPE,
155 vni_type);
156 break;
157 case EFX_ENCAP_TYPE_NVGRE:
158 COPY_VALUE(ether_type, ETHER_TYPE);
159 outer_ip_proto = IPPROTO_GRE;
160 COPY_VALUE(outer_ip_proto, IP_PROTO);
161 break;
162 default:
163 WARN_ON(1);
164 }
165
166 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN;
167 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN;
168 } else {
169 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN;
170 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN;
171 }
172
173 if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC_IG)
174 match_fields |=
175 is_multicast_ether_addr(spec->loc_mac) ?
176 1 << mc_match :
177 1 << uc_match;
178 COPY_FIELD(REM_HOST, rem_host, SRC_IP);
179 COPY_FIELD(LOC_HOST, loc_host, DST_IP);
180 COPY_FIELD(REM_MAC, rem_mac, SRC_MAC);
181 COPY_FIELD(REM_PORT, rem_port, SRC_PORT);
182 COPY_FIELD(LOC_MAC, loc_mac, DST_MAC);
183 COPY_FIELD(LOC_PORT, loc_port, DST_PORT);
184 COPY_FIELD(ETHER_TYPE, ether_type, ETHER_TYPE);
185 COPY_FIELD(INNER_VID, inner_vid, INNER_VLAN);
186 COPY_FIELD(OUTER_VID, outer_vid, OUTER_VLAN);
187 COPY_FIELD(IP_PROTO, ip_proto, IP_PROTO);
188#undef COPY_FIELD
189#undef COPY_VALUE
190 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_MATCH_FIELDS,
191 match_fields);
192}
193
194static void efx_mcdi_filter_push_prep(struct efx_nic *efx,
195 const struct efx_filter_spec *spec,
196 efx_dword_t *inbuf, u64 handle,
197 struct efx_rss_context *ctx,
198 bool replacing)
199{
6c77065b
AM		 200 u32 flags = spec->flags;
201
202 memset(inbuf, 0, MC_CMD_FILTER_OP_EXT_IN_LEN);
203
204 /* If RSS filter, caller better have given us an RSS context */
205 if (flags & EFX_FILTER_FLAG_RX_RSS) {
206 /*
207 * We don't have the ability to return an error, so we'll just
208 * log a warning and disable RSS for the filter.
209 */
210 if (WARN_ON_ONCE(!ctx))
211 flags &= ~EFX_FILTER_FLAG_RX_RSS;
212 else if (WARN_ON_ONCE(ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID))
213 flags &= ~EFX_FILTER_FLAG_RX_RSS;
214 }
215
216 if (replacing) {
217 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
218 MC_CMD_FILTER_OP_IN_OP_REPLACE);
219 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, handle);
220 } else {
221 efx_mcdi_filter_push_prep_set_match_fields(efx, spec, inbuf);
222 }
223
dfcabb07 224 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, efx->vport_id);
6c77065b
AM		 225 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_DEST,
226 spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ?
227 MC_CMD_FILTER_OP_IN_RX_DEST_DROP :
228 MC_CMD_FILTER_OP_IN_RX_DEST_HOST);
229 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DOMAIN, 0);
230 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DEST,
231 MC_CMD_FILTER_OP_IN_TX_DEST_DEFAULT);
232 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_QUEUE,
233 spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ?
234 0 : spec->dmaq_id);
235 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_MODE,
236 (flags & EFX_FILTER_FLAG_RX_RSS) ?
237 MC_CMD_FILTER_OP_IN_RX_MODE_RSS :
238 MC_CMD_FILTER_OP_IN_RX_MODE_SIMPLE);
239 if (flags & EFX_FILTER_FLAG_RX_RSS)
240 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_CONTEXT, ctx->context_id);
241}
242
243static int efx_mcdi_filter_push(struct efx_nic *efx,
244 const struct efx_filter_spec *spec, u64 *handle,
245 struct efx_rss_context *ctx, bool replacing)
246{
247 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
248 MCDI_DECLARE_BUF(outbuf, MC_CMD_FILTER_OP_EXT_OUT_LEN);
249 size_t outlen;
250 int rc;
251
252 efx_mcdi_filter_push_prep(efx, spec, inbuf, *handle, ctx, replacing);
253 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf),
254 outbuf, sizeof(outbuf), &outlen);
255 if (rc && spec->priority != EFX_FILTER_PRI_HINT)
256 efx_mcdi_display_error(efx, MC_CMD_FILTER_OP, sizeof(inbuf),
257 outbuf, outlen, rc);
258 if (rc == 0)
259 *handle = MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE);
260 if (rc == -ENOSPC)
261 rc = -EBUSY; /* to match efx_farch_filter_insert() */
262 return rc;
263}
264
265static u32 efx_mcdi_filter_mcdi_flags_from_spec(const struct efx_filter_spec *spec)
266{
267 enum efx_encap_type encap_type = efx_filter_get_encap_type(spec);
268 unsigned int match_flags = spec->match_flags;
269 unsigned int uc_match, mc_match;
270 u32 mcdi_flags = 0;
271
272#define MAP_FILTER_TO_MCDI_FLAG(gen_flag, mcdi_field, encap) { \
273 unsigned int old_match_flags = match_flags; \
274 match_flags &= ~EFX_FILTER_MATCH_ ## gen_flag; \
275 if (match_flags != old_match_flags) \
276 mcdi_flags |= \
277 (1 << ((encap) ? \
278 MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_ ## \
279 mcdi_field ## _LBN : \
280 MC_CMD_FILTER_OP_EXT_IN_MATCH_ ##\
281 mcdi_field ## _LBN)); \
282 }
283 /* inner or outer based on encap type */
284 MAP_FILTER_TO_MCDI_FLAG(REM_HOST, SRC_IP, encap_type);
285 MAP_FILTER_TO_MCDI_FLAG(LOC_HOST, DST_IP, encap_type);
286 MAP_FILTER_TO_MCDI_FLAG(REM_MAC, SRC_MAC, encap_type);
287 MAP_FILTER_TO_MCDI_FLAG(REM_PORT, SRC_PORT, encap_type);
288 MAP_FILTER_TO_MCDI_FLAG(LOC_MAC, DST_MAC, encap_type);
289 MAP_FILTER_TO_MCDI_FLAG(LOC_PORT, DST_PORT, encap_type);
290 MAP_FILTER_TO_MCDI_FLAG(ETHER_TYPE, ETHER_TYPE, encap_type);
291 MAP_FILTER_TO_MCDI_FLAG(IP_PROTO, IP_PROTO, encap_type);
292 /* always outer */
293 MAP_FILTER_TO_MCDI_FLAG(INNER_VID, INNER_VLAN, false);
294 MAP_FILTER_TO_MCDI_FLAG(OUTER_VID, OUTER_VLAN, false);
295#undef MAP_FILTER_TO_MCDI_FLAG
296
297 /* special handling for encap type, and mismatch */
298 if (encap_type) {
299 match_flags &= ~EFX_FILTER_MATCH_ENCAP_TYPE;
300 mcdi_flags |=
301 (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN);
302 mcdi_flags |= (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN);
303
304 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN;
305 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN;
306 } else {
307 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN;
308 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN;
309 }
310
311 if (match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) {
312 match_flags &= ~EFX_FILTER_MATCH_LOC_MAC_IG;
313 mcdi_flags |=
314 is_multicast_ether_addr(spec->loc_mac) ?
315 1 << mc_match :
316 1 << uc_match;
317 }
318
319 /* Did we map them all? */
320 WARN_ON_ONCE(match_flags);
321
322 return mcdi_flags;
323}
324
325static int efx_mcdi_filter_pri(struct efx_mcdi_filter_table *table,
326 const struct efx_filter_spec *spec)
327{
328 u32 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec);
329 unsigned int match_pri;
330
331 for (match_pri = 0;
332 match_pri < table->rx_match_count;
333 match_pri++)
334 if (table->rx_match_mcdi_flags[match_pri] == mcdi_flags)
335 return match_pri;
336
337 return -EPROTONOSUPPORT;
338}
339
340static s32 efx_mcdi_filter_insert_locked(struct efx_nic *efx,
341 struct efx_filter_spec *spec,
342 bool replace_equal)
343{
344 DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
6c77065b
AM		 345 struct efx_mcdi_filter_table *table;
346 struct efx_filter_spec *saved_spec;
347 struct efx_rss_context *ctx = NULL;
348 unsigned int match_pri, hash;
349 unsigned int priv_flags;
350 bool rss_locked = false;
351 bool replacing = false;
352 unsigned int depth, i;
353 int ins_index = -1;
354 DEFINE_WAIT(wait);
355 bool is_mc_recip;
356 s32 rc;
357
358 WARN_ON(!rwsem_is_locked(&efx->filter_sem));
359 table = efx->filter_state;
360 down_write(&table->lock);
361
362 /* For now, only support RX filters */
363 if ((spec->flags & (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)) !=
364 EFX_FILTER_FLAG_RX) {
365 rc = -EINVAL;
366 goto out_unlock;
367 }
368
369 rc = efx_mcdi_filter_pri(table, spec);
370 if (rc < 0)
371 goto out_unlock;
372 match_pri = rc;
373
374 hash = efx_filter_spec_hash(spec);
375 is_mc_recip = efx_filter_is_mc_recipient(spec);
376 if (is_mc_recip)
377 bitmap_zero(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
378
379 if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {
380 mutex_lock(&efx->rss_lock);
381 rss_locked = true;
382 if (spec->rss_context)
383 ctx = efx_find_rss_context_entry(efx, spec->rss_context);
384 else
385 ctx = &efx->rss_context;
386 if (!ctx) {
387 rc = -ENOENT;
388 goto out_unlock;
389 }
390 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
391 rc = -EOPNOTSUPP;
392 goto out_unlock;
393 }
394 }
395
396 /* Find any existing filters with the same match tuple or
397 * else a free slot to insert at.
398 */
399 for (depth = 1; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
400 i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1);
401 saved_spec = efx_mcdi_filter_entry_spec(table, i);
402
403 if (!saved_spec) {
404 if (ins_index < 0)
405 ins_index = i;
406 } else if (efx_filter_spec_equal(spec, saved_spec)) {
407 if (spec->priority < saved_spec->priority &&
408 spec->priority != EFX_FILTER_PRI_AUTO) {
409 rc = -EPERM;
410 goto out_unlock;
411 }
412 if (!is_mc_recip) {
413 /* This is the only one */
414 if (spec->priority ==
415 saved_spec->priority &&
416 !replace_equal) {
417 rc = -EEXIST;
418 goto out_unlock;
419 }
420 ins_index = i;
421 break;
422 } else if (spec->priority >
423 saved_spec->priority ||
424 (spec->priority ==
425 saved_spec->priority &&
426 replace_equal)) {
427 if (ins_index < 0)
428 ins_index = i;
429 else
430 __set_bit(depth, mc_rem_map);
431 }
432 }
433 }
434
435 /* Once we reach the maximum search depth, use the first suitable
436 * slot, or return -EBUSY if there was none
437 */
438 if (ins_index < 0) {
439 rc = -EBUSY;
440 goto out_unlock;
441 }
442
443 /* Create a software table entry if necessary. */
444 saved_spec = efx_mcdi_filter_entry_spec(table, ins_index);
445 if (saved_spec) {
446 if (spec->priority == EFX_FILTER_PRI_AUTO &&
447 saved_spec->priority >= EFX_FILTER_PRI_AUTO) {
448 /* Just make sure it won't be removed */
449 if (saved_spec->priority > EFX_FILTER_PRI_AUTO)
450 saved_spec->flags |= EFX_FILTER_FLAG_RX_OVER_AUTO;
451 table->entry[ins_index].spec &=
452 ~EFX_EF10_FILTER_FLAG_AUTO_OLD;
453 rc = ins_index;
454 goto out_unlock;
455 }
456 replacing = true;
457 priv_flags = efx_mcdi_filter_entry_flags(table, ins_index);
458 } else {
459 saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC);
460 if (!saved_spec) {
461 rc = -ENOMEM;
462 goto out_unlock;
463 }
464 *saved_spec = *spec;
465 priv_flags = 0;
466 }
467 efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags);
468
469 /* Actually insert the filter on the HW */
470 rc = efx_mcdi_filter_push(efx, spec, &table->entry[ins_index].handle,
471 ctx, replacing);
472
e4fe938c 473 if (rc == -EINVAL && efx->must_realloc_vis)
6c77065b
AM		 474 /* The MC rebooted under us, causing it to reject our filter
475 * insertion as pointing to an invalid VI (spec->dmaq_id).
476 */
477 rc = -EAGAIN;
478
479 /* Finalise the software table entry */
480 if (rc == 0) {
481 if (replacing) {
482 /* Update the fields that may differ */
483 if (saved_spec->priority == EFX_FILTER_PRI_AUTO)
484 saved_spec->flags |=
485 EFX_FILTER_FLAG_RX_OVER_AUTO;
486 saved_spec->priority = spec->priority;
487 saved_spec->flags &= EFX_FILTER_FLAG_RX_OVER_AUTO;
488 saved_spec->flags |= spec->flags;
489 saved_spec->rss_context = spec->rss_context;
490 saved_spec->dmaq_id = spec->dmaq_id;
491 }
492 } else if (!replacing) {
493 kfree(saved_spec);
494 saved_spec = NULL;
495 } else {
496 /* We failed to replace, so the old filter is still present.
497 * Roll back the software table to reflect this. In fact the
498 * efx_mcdi_filter_set_entry() call below will do the right
499 * thing, so nothing extra is needed here.
500 */
501 }
502 efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags);
503
504 /* Remove and finalise entries for lower-priority multicast
505 * recipients
506 */
507 if (is_mc_recip) {
508 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
509 unsigned int depth, i;
510
511 memset(inbuf, 0, sizeof(inbuf));
512
513 for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
514 if (!test_bit(depth, mc_rem_map))
515 continue;
516
517 i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1);
518 saved_spec = efx_mcdi_filter_entry_spec(table, i);
519 priv_flags = efx_mcdi_filter_entry_flags(table, i);
520
521 if (rc == 0) {
522 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
523 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
524 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
525 table->entry[i].handle);
526 rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP,
527 inbuf, sizeof(inbuf),
528 NULL, 0, NULL);
529 }
530
531 if (rc == 0) {
532 kfree(saved_spec);
533 saved_spec = NULL;
534 priv_flags = 0;
535 }
536 efx_mcdi_filter_set_entry(table, i, saved_spec,
537 priv_flags);
538 }
539 }
540
541 /* If successful, return the inserted filter ID */
542 if (rc == 0)
543 rc = efx_mcdi_filter_make_filter_id(match_pri, ins_index);
544
545out_unlock:
546 if (rss_locked)
547 mutex_unlock(&efx->rss_lock);
548 up_write(&table->lock);
549 return rc;
550}
551
552s32 efx_mcdi_filter_insert(struct efx_nic *efx, struct efx_filter_spec *spec,
553 bool replace_equal)
554{
555 s32 ret;
556
557 down_read(&efx->filter_sem);
558 ret = efx_mcdi_filter_insert_locked(efx, spec, replace_equal);
559 up_read(&efx->filter_sem);
560
561 return ret;
562}
563
564/*
565 * Remove a filter.
566 * If !by_index, remove by ID
567 * If by_index, remove by index
568 * Filter ID may come from userland and must be range-checked.
569 * Caller must hold efx->filter_sem for read, and efx->filter_state->lock
570 * for write.
571 */
572static int efx_mcdi_filter_remove_internal(struct efx_nic *efx,
573 unsigned int priority_mask,
574 u32 filter_id, bool by_index)
575{
576 unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id);
577 struct efx_mcdi_filter_table *table = efx->filter_state;
578 MCDI_DECLARE_BUF(inbuf,
579 MC_CMD_FILTER_OP_IN_HANDLE_OFST +
580 MC_CMD_FILTER_OP_IN_HANDLE_LEN);
581 struct efx_filter_spec *spec;
582 DEFINE_WAIT(wait);
583 int rc;
584
585 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
586 if (!spec ||
587 (!by_index &&
588 efx_mcdi_filter_pri(table, spec) !=
589 efx_mcdi_filter_get_unsafe_pri(filter_id)))
590 return -ENOENT;
591
592 if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO &&
593 priority_mask == (1U << EFX_FILTER_PRI_AUTO)) {
594 /* Just remove flags */
595 spec->flags &= ~EFX_FILTER_FLAG_RX_OVER_AUTO;
596 table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_AUTO_OLD;
597 return 0;
598 }
599
600 if (!(priority_mask & (1U << spec->priority)))
601 return -ENOENT;
602
603 if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) {
604 /* Reset to an automatic filter */
605
606 struct efx_filter_spec new_spec = *spec;
607
608 new_spec.priority = EFX_FILTER_PRI_AUTO;
609 new_spec.flags = (EFX_FILTER_FLAG_RX |
610 (efx_rss_active(&efx->rss_context) ?
611 EFX_FILTER_FLAG_RX_RSS : 0));
612 new_spec.dmaq_id = 0;
613 new_spec.rss_context = 0;
614 rc = efx_mcdi_filter_push(efx, &new_spec,
615 &table->entry[filter_idx].handle,
616 &efx->rss_context,
617 true);
618
619 if (rc == 0)
620 *spec = new_spec;
621 } else {
622 /* Really remove the filter */
623
624 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
625 efx_mcdi_filter_is_exclusive(spec) ?
626 MC_CMD_FILTER_OP_IN_OP_REMOVE :
627 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
628 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
629 table->entry[filter_idx].handle);
630 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP,
631 inbuf, sizeof(inbuf), NULL, 0, NULL);
632
633 if ((rc == 0) || (rc == -ENOENT)) {
634 /* Filter removed OK or didn't actually exist */
635 kfree(spec);
636 efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0);
637 } else {
638 efx_mcdi_display_error(efx, MC_CMD_FILTER_OP,
639 MC_CMD_FILTER_OP_EXT_IN_LEN,
640 NULL, 0, rc);
641 }
642 }
643
644 return rc;
645}
646
647/* Remove filters that weren't renewed. */
648static void efx_mcdi_filter_remove_old(struct efx_nic *efx)
649{
650 struct efx_mcdi_filter_table *table = efx->filter_state;
651 int remove_failed = 0;
652 int remove_noent = 0;
653 int rc;
654 int i;
655
656 down_write(&table->lock);
657 for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) {
658 if (READ_ONCE(table->entry[i].spec) &
659 EFX_EF10_FILTER_FLAG_AUTO_OLD) {
660 rc = efx_mcdi_filter_remove_internal(efx,
661 1U << EFX_FILTER_PRI_AUTO, i, true);
662 if (rc == -ENOENT)
663 remove_noent++;
664 else if (rc)
665 remove_failed++;
666 }
667 }
668 up_write(&table->lock);
669
670 if (remove_failed)
671 netif_info(efx, drv, efx->net_dev,
672 "%s: failed to remove %d filters\n",
673 __func__, remove_failed);
674 if (remove_noent)
675 netif_info(efx, drv, efx->net_dev,
676 "%s: failed to remove %d non-existent filters\n",
677 __func__, remove_noent);
678}
679
680int efx_mcdi_filter_remove_safe(struct efx_nic *efx,
681 enum efx_filter_priority priority,
682 u32 filter_id)
683{
684 struct efx_mcdi_filter_table *table;
685 int rc;
686
687 down_read(&efx->filter_sem);
688 table = efx->filter_state;
689 down_write(&table->lock);
690 rc = efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id,
691 false);
692 up_write(&table->lock);
693 up_read(&efx->filter_sem);
694 return rc;
695}
696
697/* Caller must hold efx->filter_sem for read */
698static void efx_mcdi_filter_remove_unsafe(struct efx_nic *efx,
699 enum efx_filter_priority priority,
700 u32 filter_id)
701{
702 struct efx_mcdi_filter_table *table = efx->filter_state;
703
704 if (filter_id == EFX_EF10_FILTER_ID_INVALID)
705 return;
706
707 down_write(&table->lock);
708 efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id,
709 true);
710 up_write(&table->lock);
711}
712
713int efx_mcdi_filter_get_safe(struct efx_nic *efx,
714 enum efx_filter_priority priority,
715 u32 filter_id, struct efx_filter_spec *spec)
716{
717 unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id);
718 const struct efx_filter_spec *saved_spec;
719 struct efx_mcdi_filter_table *table;
720 int rc;
721
722 down_read(&efx->filter_sem);
723 table = efx->filter_state;
724 down_read(&table->lock);
725 saved_spec = efx_mcdi_filter_entry_spec(table, filter_idx);
726 if (saved_spec && saved_spec->priority == priority &&
727 efx_mcdi_filter_pri(table, saved_spec) ==
728 efx_mcdi_filter_get_unsafe_pri(filter_id)) {
729 *spec = *saved_spec;
730 rc = 0;
731 } else {
732 rc = -ENOENT;
733 }
734 up_read(&table->lock);
735 up_read(&efx->filter_sem);
736 return rc;
737}
738
739static int efx_mcdi_filter_insert_addr_list(struct efx_nic *efx,
740 struct efx_mcdi_filter_vlan *vlan,
741 bool multicast, bool rollback)
742{
743 struct efx_mcdi_filter_table *table = efx->filter_state;
744 struct efx_mcdi_dev_addr *addr_list;
745 enum efx_filter_flags filter_flags;
746 struct efx_filter_spec spec;
747 u8 baddr[ETH_ALEN];
748 unsigned int i, j;
749 int addr_count;
750 u16 *ids;
751 int rc;
752
753 if (multicast) {
754 addr_list = table->dev_mc_list;
755 addr_count = table->dev_mc_count;
756 ids = vlan->mc;
757 } else {
758 addr_list = table->dev_uc_list;
759 addr_count = table->dev_uc_count;
760 ids = vlan->uc;
761 }
762
763 filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0;
764
765 /* Insert/renew filters */
766 for (i = 0; i < addr_count; i++) {
767 EFX_WARN_ON_PARANOID(ids[i] != EFX_EF10_FILTER_ID_INVALID);
768 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
769 efx_filter_set_eth_local(&spec, vlan->vid, addr_list[i].addr);
770 rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
771 if (rc < 0) {
772 if (rollback) {
773 netif_info(efx, drv, efx->net_dev,
774 "efx_mcdi_filter_insert failed rc=%d\n",
775 rc);
776 /* Fall back to promiscuous */
777 for (j = 0; j < i; j++) {
778 efx_mcdi_filter_remove_unsafe(
779 efx, EFX_FILTER_PRI_AUTO,
780 ids[j]);
781 ids[j] = EFX_EF10_FILTER_ID_INVALID;
782 }
783 return rc;
784 } else {
785 /* keep invalid ID, and carry on */
786 }
787 } else {
788 ids[i] = efx_mcdi_filter_get_unsafe_id(rc);
789 }
790 }
791
792 if (multicast && rollback) {
793 /* Also need an Ethernet broadcast filter */
794 EFX_WARN_ON_PARANOID(vlan->default_filters[EFX_EF10_BCAST] !=
795 EFX_EF10_FILTER_ID_INVALID);
796 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
797 eth_broadcast_addr(baddr);
798 efx_filter_set_eth_local(&spec, vlan->vid, baddr);
799 rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
800 if (rc < 0) {
801 netif_warn(efx, drv, efx->net_dev,
802 "Broadcast filter insert failed rc=%d\n", rc);
803 /* Fall back to promiscuous */
804 for (j = 0; j < i; j++) {
805 efx_mcdi_filter_remove_unsafe(
806 efx, EFX_FILTER_PRI_AUTO,
807 ids[j]);
808 ids[j] = EFX_EF10_FILTER_ID_INVALID;
809 }
810 return rc;
811 } else {
812 vlan->default_filters[EFX_EF10_BCAST] =
813 efx_mcdi_filter_get_unsafe_id(rc);
814 }
815 }
816
817 return 0;
818}
819
820static int efx_mcdi_filter_insert_def(struct efx_nic *efx,
821 struct efx_mcdi_filter_vlan *vlan,
822 enum efx_encap_type encap_type,
823 bool multicast, bool rollback)
824{
fd14e5fd 825 struct efx_mcdi_filter_table *table = efx->filter_state;
6c77065b
AM		 826 enum efx_filter_flags filter_flags;
827 struct efx_filter_spec spec;
828 u8 baddr[ETH_ALEN];
829 int rc;
830 u16 *id;
831
832 filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0;
833
834 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
835
836 if (multicast)
837 efx_filter_set_mc_def(&spec);
838 else
839 efx_filter_set_uc_def(&spec);
840
841 if (encap_type) {
6d9b5dcd 842 if (efx_has_cap(efx, VXLAN_NVGRE))
6c77065b
AM		 843 efx_filter_set_encap_type(&spec, encap_type);
844 else
845 /*
846 * don't insert encap filters on non-supporting
847 * platforms. ID will be left as INVALID.
848 */
849 return 0;
850 }
851
852 if (vlan->vid != EFX_FILTER_VID_UNSPEC)
853 efx_filter_set_eth_local(&spec, vlan->vid, NULL);
854
855 rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
856 if (rc < 0) {
857 const char *um = multicast ? "Multicast" : "Unicast";
858 const char *encap_name = "";
859 const char *encap_ipv = "";
860
861 if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
862 EFX_ENCAP_TYPE_VXLAN)
863 encap_name = "VXLAN ";
864 else if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
865 EFX_ENCAP_TYPE_NVGRE)
866 encap_name = "NVGRE ";
867 else if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
868 EFX_ENCAP_TYPE_GENEVE)
869 encap_name = "GENEVE ";
870 if (encap_type & EFX_ENCAP_FLAG_IPV6)
871 encap_ipv = "IPv6 ";
872 else if (encap_type)
873 encap_ipv = "IPv4 ";
874
875 /*
876 * unprivileged functions can't insert mismatch filters
877 * for encapsulated or unicast traffic, so downgrade
878 * those warnings to debug.
879 */
880 netif_cond_dbg(efx, drv, efx->net_dev,
881 rc == -EPERM && (encap_type || !multicast), warn,
882 "%s%s%s mismatch filter insert failed rc=%d\n",
883 encap_name, encap_ipv, um, rc);
884 } else if (multicast) {
885 /* mapping from encap types to default filter IDs (multicast) */
886 static enum efx_mcdi_filter_default_filters map[] = {
887 [EFX_ENCAP_TYPE_NONE] = EFX_EF10_MCDEF,
888 [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_MCDEF,
889 [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_MCDEF,
890 [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_MCDEF,
891 [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] =
892 EFX_EF10_VXLAN6_MCDEF,
893 [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] =
894 EFX_EF10_NVGRE6_MCDEF,
895 [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] =
896 EFX_EF10_GENEVE6_MCDEF,
897 };
898
899 /* quick bounds check (BCAST result impossible) */
900 BUILD_BUG_ON(EFX_EF10_BCAST != 0);
901 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) {
902 WARN_ON(1);
903 return -EINVAL;
904 }
905 /* then follow map */
906 id = &vlan->default_filters[map[encap_type]];
907
908 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID);
909 *id = efx_mcdi_filter_get_unsafe_id(rc);
fd14e5fd 910 if (!table->mc_chaining && !encap_type) {
6c77065b
AM		 911 /* Also need an Ethernet broadcast filter */
912 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO,
913 filter_flags, 0);
914 eth_broadcast_addr(baddr);
915 efx_filter_set_eth_local(&spec, vlan->vid, baddr);
916 rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
917 if (rc < 0) {
918 netif_warn(efx, drv, efx->net_dev,
919 "Broadcast filter insert failed rc=%d\n",
920 rc);
921 if (rollback) {
922 /* Roll back the mc_def filter */
923 efx_mcdi_filter_remove_unsafe(
924 efx, EFX_FILTER_PRI_AUTO,
925 *id);
926 *id = EFX_EF10_FILTER_ID_INVALID;
927 return rc;
928 }
929 } else {
930 EFX_WARN_ON_PARANOID(
931 vlan->default_filters[EFX_EF10_BCAST] !=
932 EFX_EF10_FILTER_ID_INVALID);
933 vlan->default_filters[EFX_EF10_BCAST] =
934 efx_mcdi_filter_get_unsafe_id(rc);
935 }
936 }
937 rc = 0;
938 } else {
939 /* mapping from encap types to default filter IDs (unicast) */
940 static enum efx_mcdi_filter_default_filters map[] = {
941 [EFX_ENCAP_TYPE_NONE] = EFX_EF10_UCDEF,
942 [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_UCDEF,
943 [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_UCDEF,
944 [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_UCDEF,
945 [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] =
946 EFX_EF10_VXLAN6_UCDEF,
947 [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] =
948 EFX_EF10_NVGRE6_UCDEF,
949 [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] =
950 EFX_EF10_GENEVE6_UCDEF,
951 };
952
953 /* quick bounds check (BCAST result impossible) */
954 BUILD_BUG_ON(EFX_EF10_BCAST != 0);
955 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) {
956 WARN_ON(1);
957 return -EINVAL;
958 }
959 /* then follow map */
960 id = &vlan->default_filters[map[encap_type]];
961 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID);
962 *id = rc;
963 rc = 0;
964 }
965 return rc;
966}
967
968/*
969 * Caller must hold efx->filter_sem for read if race against
970 * efx_mcdi_filter_table_remove() is possible
971 */
972static void efx_mcdi_filter_vlan_sync_rx_mode(struct efx_nic *efx,
973 struct efx_mcdi_filter_vlan *vlan)
974{
975 struct efx_mcdi_filter_table *table = efx->filter_state;
6c77065b
AM		 976
977 /*
978 * Do not install unspecified VID if VLAN filtering is enabled.
979 * Do not install all specified VIDs if VLAN filtering is disabled.
980 */
981 if ((vlan->vid == EFX_FILTER_VID_UNSPEC) == table->vlan_filter)
982 return;
983
984 /* Insert/renew unicast filters */
985 if (table->uc_promisc) {
986 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NONE,
987 false, false);
988 efx_mcdi_filter_insert_addr_list(efx, vlan, false, false);
989 } else {
990 /*
991 * If any of the filters failed to insert, fall back to
992 * promiscuous mode - add in the uc_def filter. But keep
993 * our individual unicast filters.
994 */
995 if (efx_mcdi_filter_insert_addr_list(efx, vlan, false, false))
996 efx_mcdi_filter_insert_def(efx, vlan,
997 EFX_ENCAP_TYPE_NONE,
998 false, false);
999 }
1000 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN,
1001 false, false);
1002 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN |
1003 EFX_ENCAP_FLAG_IPV6,
1004 false, false);
1005 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE,
1006 false, false);
1007 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE |
1008 EFX_ENCAP_FLAG_IPV6,
1009 false, false);
1010 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE,
1011 false, false);
1012 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE |
1013 EFX_ENCAP_FLAG_IPV6,
1014 false, false);
1015
1016 /*
1017 * Insert/renew multicast filters
1018 *
1019 * If changing promiscuous state with cascaded multicast filters, remove
1020 * old filters first, so that packets are dropped rather than duplicated
1021 */
fd14e5fd 1022 if (table->mc_chaining && table->mc_promisc_last != table->mc_promisc)
6c77065b
AM		 1023 efx_mcdi_filter_remove_old(efx);
1024 if (table->mc_promisc) {
fd14e5fd 1025 if (table->mc_chaining) {
6c77065b
AM		 1026 /*
1027 * If we failed to insert promiscuous filters, rollback
1028 * and fall back to individual multicast filters
1029 */
1030 if (efx_mcdi_filter_insert_def(efx, vlan,
1031 EFX_ENCAP_TYPE_NONE,
1032 true, true)) {
1033 /* Changing promisc state, so remove old filters */
1034 efx_mcdi_filter_remove_old(efx);
1035 efx_mcdi_filter_insert_addr_list(efx, vlan,
1036 true, false);
1037 }
1038 } else {
1039 /*
1040 * If we failed to insert promiscuous filters, don't
1041 * rollback. Regardless, also insert the mc_list,
1042 * unless it's incomplete due to overflow
1043 */
1044 efx_mcdi_filter_insert_def(efx, vlan,
1045 EFX_ENCAP_TYPE_NONE,
1046 true, false);
1047 if (!table->mc_overflow)
1048 efx_mcdi_filter_insert_addr_list(efx, vlan,
1049 true, false);
1050 }
1051 } else {
1052 /*
1053 * If any filters failed to insert, rollback and fall back to
1054 * promiscuous mode - mc_def filter and maybe broadcast. If
1055 * that fails, roll back again and insert as many of our
1056 * individual multicast filters as we can.
1057 */
1058 if (efx_mcdi_filter_insert_addr_list(efx, vlan, true, true)) {
1059 /* Changing promisc state, so remove old filters */
fd14e5fd 1060 if (table->mc_chaining)
6c77065b
AM		 1061 efx_mcdi_filter_remove_old(efx);
1062 if (efx_mcdi_filter_insert_def(efx, vlan,
1063 EFX_ENCAP_TYPE_NONE,
1064 true, true))
1065 efx_mcdi_filter_insert_addr_list(efx, vlan,
1066 true, false);
1067 }
1068 }
1069 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN,
1070 true, false);
1071 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN |
1072 EFX_ENCAP_FLAG_IPV6,
1073 true, false);
1074 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE,
1075 true, false);
1076 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE |
1077 EFX_ENCAP_FLAG_IPV6,
1078 true, false);
1079 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE,
1080 true, false);
1081 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE |
1082 EFX_ENCAP_FLAG_IPV6,
1083 true, false);
1084}
1085
1086int efx_mcdi_filter_clear_rx(struct efx_nic *efx,
1087 enum efx_filter_priority priority)
1088{
1089 struct efx_mcdi_filter_table *table;
1090 unsigned int priority_mask;
1091 unsigned int i;
1092 int rc;
1093
1094 priority_mask = (((1U << (priority + 1)) - 1) &
1095 ~(1U << EFX_FILTER_PRI_AUTO));
1096
1097 down_read(&efx->filter_sem);
1098 table = efx->filter_state;
1099 down_write(&table->lock);
1100 for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) {
1101 rc = efx_mcdi_filter_remove_internal(efx, priority_mask,
1102 i, true);
1103 if (rc && rc != -ENOENT)
1104 break;
1105 rc = 0;
1106 }
1107
1108 up_write(&table->lock);
1109 up_read(&efx->filter_sem);
1110 return rc;
1111}
1112
1113u32 efx_mcdi_filter_count_rx_used(struct efx_nic *efx,
1114 enum efx_filter_priority priority)
1115{
1116 struct efx_mcdi_filter_table *table;
1117 unsigned int filter_idx;
1118 s32 count = 0;
1119
1120 down_read(&efx->filter_sem);
1121 table = efx->filter_state;
1122 down_read(&table->lock);
1123 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1124 if (table->entry[filter_idx].spec &&
1125 efx_mcdi_filter_entry_spec(table, filter_idx)->priority ==
1126 priority)
1127 ++count;
1128 }
1129 up_read(&table->lock);
1130 up_read(&efx->filter_sem);
1131 return count;
1132}
1133
1134u32 efx_mcdi_filter_get_rx_id_limit(struct efx_nic *efx)
1135{
1136 struct efx_mcdi_filter_table *table = efx->filter_state;
1137
1138 return table->rx_match_count * EFX_MCDI_FILTER_TBL_ROWS * 2;
1139}
1140
1141s32 efx_mcdi_filter_get_rx_ids(struct efx_nic *efx,
1142 enum efx_filter_priority priority,
1143 u32 *buf, u32 size)
1144{
1145 struct efx_mcdi_filter_table *table;
1146 struct efx_filter_spec *spec;
1147 unsigned int filter_idx;
1148 s32 count = 0;
1149
1150 down_read(&efx->filter_sem);
1151 table = efx->filter_state;
1152 down_read(&table->lock);
1153
1154 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1155 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1156 if (spec && spec->priority == priority) {
1157 if (count == size) {
1158 count = -EMSGSIZE;
1159 break;
1160 }
1161 buf[count++] =
1162 efx_mcdi_filter_make_filter_id(
1163 efx_mcdi_filter_pri(table, spec),
1164 filter_idx);
1165 }
1166 }
1167 up_read(&table->lock);
1168 up_read(&efx->filter_sem);
1169 return count;
1170}
1171
1172static int efx_mcdi_filter_match_flags_from_mcdi(bool encap, u32 mcdi_flags)
1173{
1174 int match_flags = 0;
1175
1176#define MAP_FLAG(gen_flag, mcdi_field) do { \
1177 u32 old_mcdi_flags = mcdi_flags; \
1178 mcdi_flags &= ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ ## \
1179 mcdi_field ## _LBN); \
1180 if (mcdi_flags != old_mcdi_flags) \
1181 match_flags |= EFX_FILTER_MATCH_ ## gen_flag; \
1182 } while (0)
1183
1184 if (encap) {
1185 /* encap filters must specify encap type */
1186 match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
1187 /* and imply ethertype and ip proto */
1188 mcdi_flags &=
1189 ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN);
1190 mcdi_flags &=
1191 ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN);
1192 /* VLAN tags refer to the outer packet */
1193 MAP_FLAG(INNER_VID, INNER_VLAN);
1194 MAP_FLAG(OUTER_VID, OUTER_VLAN);
1195 /* everything else refers to the inner packet */
1196 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_UCAST_DST);
1197 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_MCAST_DST);
1198 MAP_FLAG(REM_HOST, IFRM_SRC_IP);
1199 MAP_FLAG(LOC_HOST, IFRM_DST_IP);
1200 MAP_FLAG(REM_MAC, IFRM_SRC_MAC);
1201 MAP_FLAG(REM_PORT, IFRM_SRC_PORT);
1202 MAP_FLAG(LOC_MAC, IFRM_DST_MAC);
1203 MAP_FLAG(LOC_PORT, IFRM_DST_PORT);
1204 MAP_FLAG(ETHER_TYPE, IFRM_ETHER_TYPE);
1205 MAP_FLAG(IP_PROTO, IFRM_IP_PROTO);
1206 } else {
1207 MAP_FLAG(LOC_MAC_IG, UNKNOWN_UCAST_DST);
1208 MAP_FLAG(LOC_MAC_IG, UNKNOWN_MCAST_DST);
1209 MAP_FLAG(REM_HOST, SRC_IP);
1210 MAP_FLAG(LOC_HOST, DST_IP);
1211 MAP_FLAG(REM_MAC, SRC_MAC);
1212 MAP_FLAG(REM_PORT, SRC_PORT);
1213 MAP_FLAG(LOC_MAC, DST_MAC);
1214 MAP_FLAG(LOC_PORT, DST_PORT);
1215 MAP_FLAG(ETHER_TYPE, ETHER_TYPE);
1216 MAP_FLAG(INNER_VID, INNER_VLAN);
1217 MAP_FLAG(OUTER_VID, OUTER_VLAN);
1218 MAP_FLAG(IP_PROTO, IP_PROTO);
1219 }
1220#undef MAP_FLAG
1221
1222 /* Did we map them all? */
1223 if (mcdi_flags)
1224 return -EINVAL;
1225
1226 return match_flags;
1227}
1228
1229bool efx_mcdi_filter_match_supported(struct efx_mcdi_filter_table *table,
1230 bool encap,
1231 enum efx_filter_match_flags match_flags)
1232{
1233 unsigned int match_pri;
1234 int mf;
1235
1236 for (match_pri = 0;
1237 match_pri < table->rx_match_count;
1238 match_pri++) {
1239 mf = efx_mcdi_filter_match_flags_from_mcdi(encap,
1240 table->rx_match_mcdi_flags[match_pri]);
1241 if (mf == match_flags)
1242 return true;
1243 }
1244
1245 return false;
1246}
1247
1248static int
1249efx_mcdi_filter_table_probe_matches(struct efx_nic *efx,
1250 struct efx_mcdi_filter_table *table,
1251 bool encap)
1252{
1253 MCDI_DECLARE_BUF(inbuf, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN);
1254 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX);
1255 unsigned int pd_match_pri, pd_match_count;
1256 size_t outlen;
1257 int rc;
1258
1259 /* Find out which RX filter types are supported, and their priorities */
1260 MCDI_SET_DWORD(inbuf, GET_PARSER_DISP_INFO_IN_OP,
1261 encap ?
1262 MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_ENCAP_RX_MATCHES :
1263 MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES);
1264 rc = efx_mcdi_rpc(efx, MC_CMD_GET_PARSER_DISP_INFO,
1265 inbuf, sizeof(inbuf), outbuf, sizeof(outbuf),
1266 &outlen);
1267 if (rc)
1268 return rc;
1269
1270 pd_match_count = MCDI_VAR_ARRAY_LEN(
1271 outlen, GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES);
1272
1273 for (pd_match_pri = 0; pd_match_pri < pd_match_count; pd_match_pri++) {
1274 u32 mcdi_flags =
1275 MCDI_ARRAY_DWORD(
1276 outbuf,
1277 GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES,
1278 pd_match_pri);
1279 rc = efx_mcdi_filter_match_flags_from_mcdi(encap, mcdi_flags);
1280 if (rc < 0) {
1281 netif_dbg(efx, probe, efx->net_dev,
1282 "%s: fw flags %#x pri %u not supported in driver\n",
1283 __func__, mcdi_flags, pd_match_pri);
1284 } else {
1285 netif_dbg(efx, probe, efx->net_dev,
1286 "%s: fw flags %#x pri %u supported as driver flags %#x pri %u\n",
1287 __func__, mcdi_flags, pd_match_pri,
1288 rc, table->rx_match_count);
1289 table->rx_match_mcdi_flags[table->rx_match_count] = mcdi_flags;
1290 table->rx_match_count++;
1291 }
1292 }
1293
1294 return 0;
1295}
1296
fd14e5fd 1297int efx_mcdi_filter_table_probe(struct efx_nic *efx, bool multicast_chaining)
6c77065b 1298{
6c77065b
AM		 1299 struct net_device *net_dev = efx->net_dev;
1300 struct efx_mcdi_filter_table *table;
6c77065b
AM		 1301 int rc;
1302
1303 if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1304 return -EINVAL;
1305
1306 if (efx->filter_state) /* already probed */
1307 return 0;
1308
1309 table = kzalloc(sizeof(*table), GFP_KERNEL);
1310 if (!table)
1311 return -ENOMEM;
1312
fd14e5fd 1313 table->mc_chaining = multicast_chaining;
6c77065b
AM		 1314 table->rx_match_count = 0;
1315 rc = efx_mcdi_filter_table_probe_matches(efx, table, false);
1316 if (rc)
1317 goto fail;
6d9b5dcd 1318 if (efx_has_cap(efx, VXLAN_NVGRE))
6c77065b
AM		 1319 rc = efx_mcdi_filter_table_probe_matches(efx, table, true);
1320 if (rc)
1321 goto fail;
1322 if ((efx_supported_features(efx) & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1323 !(efx_mcdi_filter_match_supported(table, false,
1324 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC)) &&
1325 efx_mcdi_filter_match_supported(table, false,
1326 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC_IG)))) {
1327 netif_info(efx, probe, net_dev,
1328 "VLAN filters are not supported in this firmware variant\n");
1329 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
1330 efx->fixed_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
1331 net_dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
1332 }
1333
1334 table->entry = vzalloc(array_size(EFX_MCDI_FILTER_TBL_ROWS,
1335 sizeof(*table->entry)));
1336 if (!table->entry) {
1337 rc = -ENOMEM;
1338 goto fail;
1339 }
1340
1341 table->mc_promisc_last = false;
1342 table->vlan_filter =
1343 !!(efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);
1344 INIT_LIST_HEAD(&table->vlan_list);
1345 init_rwsem(&table->lock);
1346
1347 efx->filter_state = table;
1348
6c77065b 1349 return 0;
6c77065b
AM		 1350fail:
1351 kfree(table);
1352 return rc;
1353}
1354
e4fe938c
EC		 1355void efx_mcdi_filter_table_reset_mc_allocations(struct efx_nic *efx)
1356{
1357 struct efx_mcdi_filter_table *table = efx->filter_state;
1358
1359 if (table) {
1360 table->must_restore_filters = true;
1361 table->must_restore_rss_contexts = true;
1362 }
1363}
1364
6c77065b
AM		 1365/*
1366 * Caller must hold efx->filter_sem for read if race against
1367 * efx_mcdi_filter_table_remove() is possible
1368 */
1369void efx_mcdi_filter_table_restore(struct efx_nic *efx)
1370{
1371 struct efx_mcdi_filter_table *table = efx->filter_state;
6c77065b
AM		 1372 unsigned int invalid_filters = 0, failed = 0;
1373 struct efx_mcdi_filter_vlan *vlan;
1374 struct efx_filter_spec *spec;
1375 struct efx_rss_context *ctx;
1376 unsigned int filter_idx;
1377 u32 mcdi_flags;
1378 int match_pri;
1379 int rc, i;
1380
1381 WARN_ON(!rwsem_is_locked(&efx->filter_sem));
1382
6545be82 1383 if (!table || !table->must_restore_filters)
6c77065b
AM		 1384 return;
1385
1386 down_write(&table->lock);
1387 mutex_lock(&efx->rss_lock);
1388
1389 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1390 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1391 if (!spec)
1392 continue;
1393
1394 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec);
1395 match_pri = 0;
1396 while (match_pri < table->rx_match_count &&
1397 table->rx_match_mcdi_flags[match_pri] != mcdi_flags)
1398 ++match_pri;
1399 if (match_pri >= table->rx_match_count) {
1400 invalid_filters++;
1401 goto not_restored;
1402 }
1403 if (spec->rss_context)
1404 ctx = efx_find_rss_context_entry(efx, spec->rss_context);
1405 else
1406 ctx = &efx->rss_context;
1407 if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {
1408 if (!ctx) {
1409 netif_warn(efx, drv, efx->net_dev,
1410 "Warning: unable to restore a filter with nonexistent RSS context %u.\n",
1411 spec->rss_context);
1412 invalid_filters++;
1413 goto not_restored;
1414 }
1415 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
1416 netif_warn(efx, drv, efx->net_dev,
1417 "Warning: unable to restore a filter with RSS context %u as it was not created.\n",
1418 spec->rss_context);
1419 invalid_filters++;
1420 goto not_restored;
1421 }
1422 }
1423
1424 rc = efx_mcdi_filter_push(efx, spec,
1425 &table->entry[filter_idx].handle,
1426 ctx, false);
1427 if (rc)
1428 failed++;
1429
1430 if (rc) {
1431not_restored:
1432 list_for_each_entry(vlan, &table->vlan_list, list)
1433 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; ++i)
1434 if (vlan->default_filters[i] == filter_idx)
1435 vlan->default_filters[i] =
1436 EFX_EF10_FILTER_ID_INVALID;
1437
1438 kfree(spec);
1439 efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0);
1440 }
1441 }
1442
1443 mutex_unlock(&efx->rss_lock);
1444 up_write(&table->lock);
1445
1446 /*
1447 * This can happen validly if the MC's capabilities have changed, so
1448 * is not an error.
1449 */
1450 if (invalid_filters)
1451 netif_dbg(efx, drv, efx->net_dev,
1452 "Did not restore %u filters that are now unsupported.\n",
1453 invalid_filters);
1454
1455 if (failed)
1456 netif_err(efx, hw, efx->net_dev,
1457 "unable to restore %u filters\n", failed);
1458 else
e4fe938c 1459 table->must_restore_filters = false;
6c77065b
AM		 1460}
1461
965470ee 1462void efx_mcdi_filter_table_down(struct efx_nic *efx)
6c77065b
AM		 1463{
1464 struct efx_mcdi_filter_table *table = efx->filter_state;
1465 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
1466 struct efx_filter_spec *spec;
1467 unsigned int filter_idx;
1468 int rc;
1469
6c77065b
AM		 1470 if (!table)
1471 return;
1472
965470ee
EC		 1473 efx_mcdi_filter_cleanup_vlans(efx);
1474
6c77065b
AM		 1475 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1476 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1477 if (!spec)
1478 continue;
1479
1480 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
1481 efx_mcdi_filter_is_exclusive(spec) ?
1482 MC_CMD_FILTER_OP_IN_OP_REMOVE :
1483 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
1484 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
1485 table->entry[filter_idx].handle);
1486 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf,
1487 sizeof(inbuf), NULL, 0, NULL);
1488 if (rc)
1489 netif_info(efx, drv, efx->net_dev,
1490 "%s: filter %04x remove failed\n",
1491 __func__, filter_idx);
1492 kfree(spec);
1493 }
965470ee
EC		 1494}
1495
1496void efx_mcdi_filter_table_remove(struct efx_nic *efx)
1497{
1498 struct efx_mcdi_filter_table *table = efx->filter_state;
1499
1500 efx_mcdi_filter_table_down(efx);
1501
1502 efx->filter_state = NULL;
1503 /*
1504 * If we were called without locking, then it's not safe to free
1505 * the table as others might be using it. So we just WARN, leak
1506 * the memory, and potentially get an inconsistent filter table
1507 * state.
1508 * This should never actually happen.
1509 */
1510 if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1511 return;
1512
1513 if (!table)
1514 return;
6c77065b
AM		 1515
1516 vfree(table->entry);
1517 kfree(table);
1518}
1519
1520static void efx_mcdi_filter_mark_one_old(struct efx_nic *efx, uint16_t *id)
1521{
1522 struct efx_mcdi_filter_table *table = efx->filter_state;
1523 unsigned int filter_idx;
1524
1525 efx_rwsem_assert_write_locked(&table->lock);
1526
1527 if (*id != EFX_EF10_FILTER_ID_INVALID) {
1528 filter_idx = efx_mcdi_filter_get_unsafe_id(*id);
1529 if (!table->entry[filter_idx].spec)
1530 netif_dbg(efx, drv, efx->net_dev,
1531 "marked null spec old %04x:%04x\n", *id,
1532 filter_idx);
1533 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_AUTO_OLD;
1534 *id = EFX_EF10_FILTER_ID_INVALID;
1535 }
1536}
1537
1538/* Mark old per-VLAN filters that may need to be removed */
1539static void _efx_mcdi_filter_vlan_mark_old(struct efx_nic *efx,
1540 struct efx_mcdi_filter_vlan *vlan)
1541{
1542 struct efx_mcdi_filter_table *table = efx->filter_state;
1543 unsigned int i;
1544
1545 for (i = 0; i < table->dev_uc_count; i++)
1546 efx_mcdi_filter_mark_one_old(efx, &vlan->uc[i]);
1547 for (i = 0; i < table->dev_mc_count; i++)
1548 efx_mcdi_filter_mark_one_old(efx, &vlan->mc[i]);
1549 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
1550 efx_mcdi_filter_mark_one_old(efx, &vlan->default_filters[i]);
1551}
1552
1553/*
1554 * Mark old filters that may need to be removed.
1555 * Caller must hold efx->filter_sem for read if race against
1556 * efx_mcdi_filter_table_remove() is possible
1557 */
1558static void efx_mcdi_filter_mark_old(struct efx_nic *efx)
1559{
1560 struct efx_mcdi_filter_table *table = efx->filter_state;
1561 struct efx_mcdi_filter_vlan *vlan;
1562
1563 down_write(&table->lock);
1564 list_for_each_entry(vlan, &table->vlan_list, list)
1565 _efx_mcdi_filter_vlan_mark_old(efx, vlan);
1566 up_write(&table->lock);
1567}
1568
1569int efx_mcdi_filter_add_vlan(struct efx_nic *efx, u16 vid)
1570{
1571 struct efx_mcdi_filter_table *table = efx->filter_state;
1572 struct efx_mcdi_filter_vlan *vlan;
1573 unsigned int i;
1574
1575 if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1576 return -EINVAL;
1577
1578 vlan = efx_mcdi_filter_find_vlan(efx, vid);
1579 if (WARN_ON(vlan)) {
1580 netif_err(efx, drv, efx->net_dev,
1581 "VLAN %u already added\n", vid);
1582 return -EALREADY;
1583 }
1584
1585 vlan = kzalloc(sizeof(*vlan), GFP_KERNEL);
1586 if (!vlan)
1587 return -ENOMEM;
1588
1589 vlan->vid = vid;
1590
1591 for (i = 0; i < ARRAY_SIZE(vlan->uc); i++)
1592 vlan->uc[i] = EFX_EF10_FILTER_ID_INVALID;
1593 for (i = 0; i < ARRAY_SIZE(vlan->mc); i++)
1594 vlan->mc[i] = EFX_EF10_FILTER_ID_INVALID;
1595 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
1596 vlan->default_filters[i] = EFX_EF10_FILTER_ID_INVALID;
1597
1598 list_add_tail(&vlan->list, &table->vlan_list);
1599
1600 if (efx_dev_registered(efx))
1601 efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan);
1602
1603 return 0;
1604}
1605
1606static void efx_mcdi_filter_del_vlan_internal(struct efx_nic *efx,
1607 struct efx_mcdi_filter_vlan *vlan)
1608{
1609 unsigned int i;
1610
1611 /* See comment in efx_mcdi_filter_table_remove() */
1612 if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1613 return;
1614
1615 list_del(&vlan->list);
1616
1617 for (i = 0; i < ARRAY_SIZE(vlan->uc); i++)
1618 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
1619 vlan->uc[i]);
1620 for (i = 0; i < ARRAY_SIZE(vlan->mc); i++)
1621 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
1622 vlan->mc[i]);
1623 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
1624 if (vlan->default_filters[i] != EFX_EF10_FILTER_ID_INVALID)
1625 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
1626 vlan->default_filters[i]);
1627
1628 kfree(vlan);
1629}
1630
1631void efx_mcdi_filter_del_vlan(struct efx_nic *efx, u16 vid)
1632{
1633 struct efx_mcdi_filter_vlan *vlan;
1634
1635 /* See comment in efx_mcdi_filter_table_remove() */
1636 if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1637 return;
1638
1639 vlan = efx_mcdi_filter_find_vlan(efx, vid);
1640 if (!vlan) {
1641 netif_err(efx, drv, efx->net_dev,
1642 "VLAN %u not found in filter state\n", vid);
1643 return;
1644 }
1645
1646 efx_mcdi_filter_del_vlan_internal(efx, vlan);
1647}
1648
1649struct efx_mcdi_filter_vlan *efx_mcdi_filter_find_vlan(struct efx_nic *efx,
1650 u16 vid)
1651{
1652 struct efx_mcdi_filter_table *table = efx->filter_state;
1653 struct efx_mcdi_filter_vlan *vlan;
1654
1655 WARN_ON(!rwsem_is_locked(&efx->filter_sem));
1656
1657 list_for_each_entry(vlan, &table->vlan_list, list) {
1658 if (vlan->vid == vid)
1659 return vlan;
1660 }
1661
1662 return NULL;
1663}
1664
1665void efx_mcdi_filter_cleanup_vlans(struct efx_nic *efx)
1666{
1667 struct efx_mcdi_filter_table *table = efx->filter_state;
1668 struct efx_mcdi_filter_vlan *vlan, *next_vlan;
1669
1670 /* See comment in efx_mcdi_filter_table_remove() */
1671 if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1672 return;
1673
1674 if (!table)
1675 return;
1676
1677 list_for_each_entry_safe(vlan, next_vlan, &table->vlan_list, list)
1678 efx_mcdi_filter_del_vlan_internal(efx, vlan);
1679}
1680
1681static void efx_mcdi_filter_uc_addr_list(struct efx_nic *efx)
1682{
1683 struct efx_mcdi_filter_table *table = efx->filter_state;
1684 struct net_device *net_dev = efx->net_dev;
1685 struct netdev_hw_addr *uc;
1686 unsigned int i;
1687
1688 table->uc_promisc = !!(net_dev->flags & IFF_PROMISC);
1689 ether_addr_copy(table->dev_uc_list[0].addr, net_dev->dev_addr);
1690 i = 1;
1691 netdev_for_each_uc_addr(uc, net_dev) {
1692 if (i >= EFX_EF10_FILTER_DEV_UC_MAX) {
1693 table->uc_promisc = true;
1694 break;
1695 }
1696 ether_addr_copy(table->dev_uc_list[i].addr, uc->addr);
1697 i++;
1698 }
1699
1700 table->dev_uc_count = i;
1701}
1702
1703static void efx_mcdi_filter_mc_addr_list(struct efx_nic *efx)
1704{
1705 struct efx_mcdi_filter_table *table = efx->filter_state;
1706 struct net_device *net_dev = efx->net_dev;
1707 struct netdev_hw_addr *mc;
1708 unsigned int i;
1709
1710 table->mc_overflow = false;
1711 table->mc_promisc = !!(net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI));
1712
1713 i = 0;
1714 netdev_for_each_mc_addr(mc, net_dev) {
1715 if (i >= EFX_EF10_FILTER_DEV_MC_MAX) {
1716 table->mc_promisc = true;
1717 table->mc_overflow = true;
1718 break;
1719 }
1720 ether_addr_copy(table->dev_mc_list[i].addr, mc->addr);
1721 i++;
1722 }
1723
1724 table->dev_mc_count = i;
1725}
1726
1727/*
1728 * Caller must hold efx->filter_sem for read if race against
1729 * efx_mcdi_filter_table_remove() is possible
1730 */
1731void efx_mcdi_filter_sync_rx_mode(struct efx_nic *efx)
1732{
1733 struct efx_mcdi_filter_table *table = efx->filter_state;
1734 struct net_device *net_dev = efx->net_dev;
1735 struct efx_mcdi_filter_vlan *vlan;
1736 bool vlan_filter;
1737
1738 if (!efx_dev_registered(efx))
1739 return;
1740
1741 if (!table)
1742 return;
1743
1744 efx_mcdi_filter_mark_old(efx);
1745
1746 /*
1747 * Copy/convert the address lists; add the primary station
1748 * address and broadcast address
1749 */
1750 netif_addr_lock_bh(net_dev);
1751 efx_mcdi_filter_uc_addr_list(efx);
1752 efx_mcdi_filter_mc_addr_list(efx);
1753 netif_addr_unlock_bh(net_dev);
1754
1755 /*
1756 * If VLAN filtering changes, all old filters are finally removed.
1757 * Do it in advance to avoid conflicts for unicast untagged and
1758 * VLAN 0 tagged filters.
1759 */
1760 vlan_filter = !!(net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);
1761 if (table->vlan_filter != vlan_filter) {
1762 table->vlan_filter = vlan_filter;
1763 efx_mcdi_filter_remove_old(efx);
1764 }
1765
1766 list_for_each_entry(vlan, &table->vlan_list, list)
1767 efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan);
1768
1769 efx_mcdi_filter_remove_old(efx);
1770 table->mc_promisc_last = table->mc_promisc;
1771}
1772
1773#ifdef CONFIG_RFS_ACCEL
1774
1775bool efx_mcdi_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
1776 unsigned int filter_idx)
1777{
1778 struct efx_filter_spec *spec, saved_spec;
1779 struct efx_mcdi_filter_table *table;
1780 struct efx_arfs_rule *rule = NULL;
1781 bool ret = true, force = false;
1782 u16 arfs_id;
1783
1784 down_read(&efx->filter_sem);
1785 table = efx->filter_state;
1786 down_write(&table->lock);
1787 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1788
1789 if (!spec || spec->priority != EFX_FILTER_PRI_HINT)
1790 goto out_unlock;
1791
1792 spin_lock_bh(&efx->rps_hash_lock);
1793 if (!efx->rps_hash_table) {
1794 /* In the absence of the table, we always return 0 to ARFS. */
1795 arfs_id = 0;
1796 } else {
1797 rule = efx_rps_hash_find(efx, spec);
1798 if (!rule)
1799 /* ARFS table doesn't know of this filter, so remove it */
1800 goto expire;
1801 arfs_id = rule->arfs_id;
1802 ret = efx_rps_check_rule(rule, filter_idx, &force);
1803 if (force)
1804 goto expire;
1805 if (!ret) {
1806 spin_unlock_bh(&efx->rps_hash_lock);
1807 goto out_unlock;
1808 }
1809 }
1810 if (!rps_may_expire_flow(efx->net_dev, spec->dmaq_id, flow_id, arfs_id))
1811 ret = false;
1812 else if (rule)
1813 rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;
1814expire:
1815 saved_spec = *spec; /* remove operation will kfree spec */
1816 spin_unlock_bh(&efx->rps_hash_lock);
1817 /*
1818 * At this point (since we dropped the lock), another thread might queue
1819 * up a fresh insertion request (but the actual insertion will be held
1820 * up by our possession of the filter table lock). In that case, it
1821 * will set rule->filter_id to EFX_ARFS_FILTER_ID_PENDING, meaning that
1822 * the rule is not removed by efx_rps_hash_del() below.
1823 */
1824 if (ret)
1825 ret = efx_mcdi_filter_remove_internal(efx, 1U << spec->priority,
1826 filter_idx, true) == 0;
1827 /*
1828 * While we can't safely dereference rule (we dropped the lock), we can
1829 * still test it for NULL.
1830 */
1831 if (ret && rule) {
1832 /* Expiring, so remove entry from ARFS table */
1833 spin_lock_bh(&efx->rps_hash_lock);
1834 efx_rps_hash_del(efx, &saved_spec);
1835 spin_unlock_bh(&efx->rps_hash_lock);
1836 }
1837out_unlock:
1838 up_write(&table->lock);
1839 up_read(&efx->filter_sem);
1840 return ret;
1841}
1842
1843#endif /* CONFIG_RFS_ACCEL */
1844
1845#define RSS_MODE_HASH_ADDRS (1 << RSS_MODE_HASH_SRC_ADDR_LBN |\
1846 1 << RSS_MODE_HASH_DST_ADDR_LBN)
1847#define RSS_MODE_HASH_PORTS (1 << RSS_MODE_HASH_SRC_PORT_LBN |\
1848 1 << RSS_MODE_HASH_DST_PORT_LBN)
1849#define RSS_CONTEXT_FLAGS_DEFAULT (1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_LBN |\
1850 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV4_EN_LBN |\
1851 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV6_EN_LBN |\
1852 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV6_EN_LBN |\
1853 (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV4_RSS_MODE_LBN |\
1854 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN |\
1855 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV4_RSS_MODE_LBN |\
1856 (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV6_RSS_MODE_LBN |\
1857 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN |\
1858 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV6_RSS_MODE_LBN)
1859
1860int efx_mcdi_get_rss_context_flags(struct efx_nic *efx, u32 context, u32 *flags)
1861{
1862 /*
1863 * Firmware had a bug (sfc bug 61952) where it would not actually
1864 * fill in the flags field in the response to MC_CMD_RSS_CONTEXT_GET_FLAGS.
1865 * This meant that it would always contain whatever was previously
1866 * in the MCDI buffer. Fortunately, all firmware versions with
1867 * this bug have the same default flags value for a newly-allocated
1868 * RSS context, and the only time we want to get the flags is just
1869 * after allocating. Moreover, the response has a 32-bit hole
1870 * where the context ID would be in the request, so we can use an
1871 * overlength buffer in the request and pre-fill the flags field
1872 * with what we believe the default to be. Thus if the firmware
1873 * has the bug, it will leave our pre-filled value in the flags
1874 * field of the response, and we will get the right answer.
1875 *
1876 * However, this does mean that this function should NOT be used if
1877 * the RSS context flags might not be their defaults - it is ONLY
1878 * reliably correct for a newly-allocated RSS context.
1879 */
1880 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN);
1881 MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN);
1882 size_t outlen;
1883 int rc;
1884
1885 /* Check we have a hole for the context ID */
1886 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_LEN != MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_FLAGS_OFST);
1887 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_IN_RSS_CONTEXT_ID, context);
1888 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS,
1889 RSS_CONTEXT_FLAGS_DEFAULT);
1890 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_FLAGS, inbuf,
1891 sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);
1892 if (rc == 0) {
1893 if (outlen < MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN)
1894 rc = -EIO;
1895 else
1896 *flags = MCDI_DWORD(outbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS);
1897 }
1898 return rc;
1899}
1900
1901/*
1902 * Attempt to enable 4-tuple UDP hashing on the specified RSS context.
1903 * If we fail, we just leave the RSS context at its default hash settings,
1904 * which is safe but may slightly reduce performance.
1905 * Defaults are 4-tuple for TCP and 2-tuple for UDP and other-IP, so we
1906 * just need to set the UDP ports flags (for both IP versions).
1907 */
1908void efx_mcdi_set_rss_context_flags(struct efx_nic *efx,
1909 struct efx_rss_context *ctx)
1910{
1911 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN);
1912 u32 flags;
1913
1914 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN != 0);
1915
1916 if (efx_mcdi_get_rss_context_flags(efx, ctx->context_id, &flags) != 0)
1917 return;
1918 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
1919 ctx->context_id);
1920 flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN;
1921 flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN;
1922 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, flags);
1923 if (!efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_FLAGS, inbuf, sizeof(inbuf),
1924 NULL, 0, NULL))
1925 /* Succeeded, so UDP 4-tuple is now enabled */
1926 ctx->rx_hash_udp_4tuple = true;
1927}
1928
1929static int efx_mcdi_filter_alloc_rss_context(struct efx_nic *efx, bool exclusive,
1930 struct efx_rss_context *ctx,
1931 unsigned *context_size)
1932{
1933 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN);
1934 MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
6c77065b
AM		 1935 size_t outlen;
1936 int rc;
1937 u32 alloc_type = exclusive ?
1938 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE :
1939 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
1940 unsigned rss_spread = exclusive ?
1941 efx->rss_spread :
1942 min(rounddown_pow_of_two(efx->rss_spread),
1943 EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE);
1944
1945 if (!exclusive && rss_spread == 1) {
1946 ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
1947 if (context_size)
1948 *context_size = 1;
1949 return 0;
1950 }
1951
6d9b5dcd 1952 if (efx_has_cap(efx, RX_RSS_LIMITED))
6c77065b
AM		 1953 return -EOPNOTSUPP;
1954
1955 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
dfcabb07 1956 efx->vport_id);
6c77065b
AM		 1957 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_TYPE, alloc_type);
1958 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, rss_spread);
1959
1960 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_ALLOC, inbuf, sizeof(inbuf),
1961 outbuf, sizeof(outbuf), &outlen);
1962 if (rc != 0)
1963 return rc;
1964
1965 if (outlen < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)
1966 return -EIO;
1967
1968 ctx->context_id = MCDI_DWORD(outbuf, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
1969
1970 if (context_size)
1971 *context_size = rss_spread;
1972
6d9b5dcd 1973 if (efx_has_cap(efx, ADDITIONAL_RSS_MODES))
6c77065b
AM		 1974 efx_mcdi_set_rss_context_flags(efx, ctx);
1975
1976 return 0;
1977}
1978
1979static int efx_mcdi_filter_free_rss_context(struct efx_nic *efx, u32 context)
1980{
1981 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_FREE_IN_LEN);
1982
1983 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID,
1984 context);
1985 return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_FREE, inbuf, sizeof(inbuf),
1986 NULL, 0, NULL);
1987}
1988
1989static int efx_mcdi_filter_populate_rss_table(struct efx_nic *efx, u32 context,
1990 const u32 *rx_indir_table, const u8 *key)
1991{
1992 MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN);
1993 MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN);
1994 int i, rc;
1995
1996 MCDI_SET_DWORD(tablebuf, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
1997 context);
1998 BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) !=
1999 MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN);
2000
2001 /* This iterates over the length of efx->rss_context.rx_indir_table, but
2002 * copies bytes from rx_indir_table. That's because the latter is a
2003 * pointer rather than an array, but should have the same length.
2004 * The efx->rss_context.rx_hash_key loop below is similar.
2005 */
2006 for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_indir_table); ++i)
2007 MCDI_PTR(tablebuf,
2008 RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE)[i] =
2009 (u8) rx_indir_table[i];
2010
2011 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_TABLE, tablebuf,
2012 sizeof(tablebuf), NULL, 0, NULL);
2013 if (rc != 0)
2014 return rc;
2015
2016 MCDI_SET_DWORD(keybuf, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
2017 context);
2018 BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_hash_key) !=
2019 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
2020 for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_hash_key); ++i)
2021 MCDI_PTR(keybuf, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY)[i] = key[i];
2022
2023 return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_KEY, keybuf,
2024 sizeof(keybuf), NULL, 0, NULL);
2025}
2026
2027void efx_mcdi_rx_free_indir_table(struct efx_nic *efx)
2028{
2029 int rc;
2030
2031 if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID) {
2032 rc = efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id);
2033 WARN_ON(rc != 0);
2034 }
2035 efx->rss_context.context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
2036}
2037
2038static int efx_mcdi_filter_rx_push_shared_rss_config(struct efx_nic *efx,
2039 unsigned *context_size)
2040{
dbf2c669 2041 struct efx_mcdi_filter_table *table = efx->filter_state;
6c77065b
AM		 2042 int rc = efx_mcdi_filter_alloc_rss_context(efx, false, &efx->rss_context,
2043 context_size);
2044
2045 if (rc != 0)
2046 return rc;
2047
dbf2c669 2048 table->rx_rss_context_exclusive = false;
6c77065b
AM		 2049 efx_set_default_rx_indir_table(efx, &efx->rss_context);
2050 return 0;
2051}
2052
2053static int efx_mcdi_filter_rx_push_exclusive_rss_config(struct efx_nic *efx,
2054 const u32 *rx_indir_table,
2055 const u8 *key)
2056{
dbf2c669 2057 struct efx_mcdi_filter_table *table = efx->filter_state;
6c77065b 2058 u32 old_rx_rss_context = efx->rss_context.context_id;
6c77065b
AM		 2059 int rc;
2060
2061 if (efx->rss_context.context_id == EFX_MCDI_RSS_CONTEXT_INVALID ||
dbf2c669 2062 !table->rx_rss_context_exclusive) {
6c77065b
AM		 2063 rc = efx_mcdi_filter_alloc_rss_context(efx, true, &efx->rss_context,
2064 NULL);
2065 if (rc == -EOPNOTSUPP)
2066 return rc;
2067 else if (rc != 0)
2068 goto fail1;
2069 }
2070
2071 rc = efx_mcdi_filter_populate_rss_table(efx, efx->rss_context.context_id,
2072 rx_indir_table, key);
2073 if (rc != 0)
2074 goto fail2;
2075
2076 if (efx->rss_context.context_id != old_rx_rss_context &&
2077 old_rx_rss_context != EFX_MCDI_RSS_CONTEXT_INVALID)
2078 WARN_ON(efx_mcdi_filter_free_rss_context(efx, old_rx_rss_context) != 0);
dbf2c669 2079 table->rx_rss_context_exclusive = true;
6c77065b
AM		 2080 if (rx_indir_table != efx->rss_context.rx_indir_table)
2081 memcpy(efx->rss_context.rx_indir_table, rx_indir_table,
2082 sizeof(efx->rss_context.rx_indir_table));
2083 if (key != efx->rss_context.rx_hash_key)
2084 memcpy(efx->rss_context.rx_hash_key, key,
2085 efx->type->rx_hash_key_size);
2086
2087 return 0;
2088
2089fail2:
2090 if (old_rx_rss_context != efx->rss_context.context_id) {
2091 WARN_ON(efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id) != 0);
2092 efx->rss_context.context_id = old_rx_rss_context;
2093 }
2094fail1:
2095 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
2096 return rc;
2097}
2098
2099int efx_mcdi_rx_push_rss_context_config(struct efx_nic *efx,
2100 struct efx_rss_context *ctx,
2101 const u32 *rx_indir_table,
2102 const u8 *key)
2103{
2104 int rc;
2105
2106 WARN_ON(!mutex_is_locked(&efx->rss_lock));
2107
2108 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
2109 rc = efx_mcdi_filter_alloc_rss_context(efx, true, ctx, NULL);
2110 if (rc)
2111 return rc;
2112 }
2113
2114 if (!rx_indir_table) /* Delete this context */
2115 return efx_mcdi_filter_free_rss_context(efx, ctx->context_id);
2116
2117 rc = efx_mcdi_filter_populate_rss_table(efx, ctx->context_id,
2118 rx_indir_table, key);
2119 if (rc)
2120 return rc;
2121
2122 memcpy(ctx->rx_indir_table, rx_indir_table,
2123 sizeof(efx->rss_context.rx_indir_table));
2124 memcpy(ctx->rx_hash_key, key, efx->type->rx_hash_key_size);
2125
2126 return 0;
2127}
2128
2129int efx_mcdi_rx_pull_rss_context_config(struct efx_nic *efx,
2130 struct efx_rss_context *ctx)
2131{
2132 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN);
2133 MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN);
2134 MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN);
2135 size_t outlen;
2136 int rc, i;
2137
2138 WARN_ON(!mutex_is_locked(&efx->rss_lock));
2139
2140 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN !=
2141 MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN);
2142
2143 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID)
2144 return -ENOENT;
2145
2146 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_TABLE_IN_RSS_CONTEXT_ID,
2147 ctx->context_id);
2148 BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_indir_table) !=
2149 MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE_LEN);
2150 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_TABLE, inbuf, sizeof(inbuf),
2151 tablebuf, sizeof(tablebuf), &outlen);
2152 if (rc != 0)
2153 return rc;
2154
2155 if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN))
2156 return -EIO;
2157
2158 for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++)
2159 ctx->rx_indir_table[i] = MCDI_PTR(tablebuf,
2160 RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE)[i];
2161
2162 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_KEY_IN_RSS_CONTEXT_ID,
2163 ctx->context_id);
2164 BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_hash_key) !=
2165 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
2166 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_KEY, inbuf, sizeof(inbuf),
2167 keybuf, sizeof(keybuf), &outlen);
2168 if (rc != 0)
2169 return rc;
2170
2171 if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN))
2172 return -EIO;
2173
2174 for (i = 0; i < ARRAY_SIZE(ctx->rx_hash_key); ++i)
2175 ctx->rx_hash_key[i] = MCDI_PTR(
2176 keybuf, RSS_CONTEXT_GET_KEY_OUT_TOEPLITZ_KEY)[i];
2177
2178 return 0;
2179}
2180
2181int efx_mcdi_rx_pull_rss_config(struct efx_nic *efx)
2182{
2183 int rc;
2184
2185 mutex_lock(&efx->rss_lock);
2186 rc = efx_mcdi_rx_pull_rss_context_config(efx, &efx->rss_context);
2187 mutex_unlock(&efx->rss_lock);
2188 return rc;
2189}
2190
2191void efx_mcdi_rx_restore_rss_contexts(struct efx_nic *efx)
2192{
e4fe938c 2193 struct efx_mcdi_filter_table *table = efx->filter_state;
6c77065b
AM		 2194 struct efx_rss_context *ctx;
2195 int rc;
2196
2197 WARN_ON(!mutex_is_locked(&efx->rss_lock));
2198
e4fe938c 2199 if (!table->must_restore_rss_contexts)
6c77065b
AM		 2200 return;
2201
2202 list_for_each_entry(ctx, &efx->rss_context.list, list) {
2203 /* previous NIC RSS context is gone */
2204 ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
2205 /* so try to allocate a new one */
2206 rc = efx_mcdi_rx_push_rss_context_config(efx, ctx,
2207 ctx->rx_indir_table,
2208 ctx->rx_hash_key);
2209 if (rc)
2210 netif_warn(efx, probe, efx->net_dev,
2211 "failed to restore RSS context %u, rc=%d"
2212 "; RSS filters may fail to be applied\n",
2213 ctx->user_id, rc);
2214 }
e4fe938c 2215 table->must_restore_rss_contexts = false;
6c77065b
AM		 2216}
2217
2218int efx_mcdi_pf_rx_push_rss_config(struct efx_nic *efx, bool user,
2219 const u32 *rx_indir_table,
2220 const u8 *key)
2221{
2222 int rc;
2223
2224 if (efx->rss_spread == 1)
2225 return 0;
2226
2227 if (!key)
2228 key = efx->rss_context.rx_hash_key;
2229
2230 rc = efx_mcdi_filter_rx_push_exclusive_rss_config(efx, rx_indir_table, key);
2231
2232 if (rc == -ENOBUFS && !user) {
2233 unsigned context_size;
2234 bool mismatch = false;
2235 size_t i;
2236
2237 for (i = 0;
2238 i < ARRAY_SIZE(efx->rss_context.rx_indir_table) && !mismatch;
2239 i++)
2240 mismatch = rx_indir_table[i] !=
2241 ethtool_rxfh_indir_default(i, efx->rss_spread);
2242
2243 rc = efx_mcdi_filter_rx_push_shared_rss_config(efx, &context_size);
2244 if (rc == 0) {
2245 if (context_size != efx->rss_spread)
2246 netif_warn(efx, probe, efx->net_dev,
2247 "Could not allocate an exclusive RSS"
2248 " context; allocated a shared one of"
2249 " different size."
2250 " Wanted %u, got %u.\n",
2251 efx->rss_spread, context_size);
2252 else if (mismatch)
2253 netif_warn(efx, probe, efx->net_dev,
2254 "Could not allocate an exclusive RSS"
2255 " context; allocated a shared one but"
2256 " could not apply custom"
2257 " indirection.\n");
2258 else
2259 netif_info(efx, probe, efx->net_dev,
2260 "Could not allocate an exclusive RSS"
2261 " context; allocated a shared one.\n");
2262 }
2263 }
2264 return rc;
2265}
2266
2267int efx_mcdi_vf_rx_push_rss_config(struct efx_nic *efx, bool user,
2268 const u32 *rx_indir_table
2269 __attribute__ ((unused)),
2270 const u8 *key
2271 __attribute__ ((unused)))
2272{
2273 if (user)
2274 return -EOPNOTSUPP;
2275 if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID)
2276 return 0;
2277 return efx_mcdi_filter_rx_push_shared_rss_config(efx, NULL);
2278}
b6d02dd2
EC		 2279
2280int efx_mcdi_push_default_indir_table(struct efx_nic *efx,
2281 unsigned int rss_spread)
2282{
2283 int rc = 0;
2284
2285 if (efx->rss_spread == rss_spread)
2286 return 0;
2287
2288 efx->rss_spread = rss_spread;
2289 if (!efx->filter_state)
2290 return 0;
2291
2292 efx_mcdi_rx_free_indir_table(efx);
2293 if (rss_spread > 1) {
2294 efx_set_default_rx_indir_table(efx, &efx->rss_context);
2295 rc = efx->type->rx_push_rss_config(efx, false,
2296 efx->rss_context.rx_indir_table, NULL);
2297 }
2298 return rc;
2299}
Linux 6.x block layer and io_uring tree(s)