drivers/net/ethernet/sfc/mcdi_filters.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /****************************************************************************
   3  * Driver for Solarflare network controllers and boards
   4  * Copyright 2005-2018 Solarflare Communications Inc.
   5  * Copyright 2019-2020 Xilinx Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 as published
   9  * by the Free Software Foundation, incorporated herein by reference.
  10  */
  11
  12 #include "mcdi_filters.h"
  13 #include "mcdi.h"
  14 #include "nic.h"
  15 #include "rx_common.h"
  16
  17 /* The maximum size of a shared RSS context */
  18 /* TODO: this should really be from the mcdi protocol export */
  19 #define EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE 64UL
  20
  21 #define EFX_EF10_FILTER_ID_INVALID 0xffff
  22
  23 /* An arbitrary search limit for the software hash table */
  24 #define EFX_EF10_FILTER_SEARCH_LIMIT 200
  25
  26 static struct efx_filter_spec *
  27 efx_mcdi_filter_entry_spec(const struct efx_mcdi_filter_table *table,
  28                            unsigned int filter_idx)
  29 {
  30         return (struct efx_filter_spec *)(table->entry[filter_idx].spec &
  31                                           ~EFX_EF10_FILTER_FLAGS);
  32 }
  33
  34 static unsigned int
  35 efx_mcdi_filter_entry_flags(const struct efx_mcdi_filter_table *table,
  36                            unsigned int filter_idx)
  37 {
  38         return table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAGS;
  39 }
  40
  41 static u32 efx_mcdi_filter_get_unsafe_id(u32 filter_id)
  42 {
  43         WARN_ON_ONCE(filter_id == EFX_EF10_FILTER_ID_INVALID);
  44         return filter_id & (EFX_MCDI_FILTER_TBL_ROWS - 1);
  45 }
  46
  47 static unsigned int efx_mcdi_filter_get_unsafe_pri(u32 filter_id)
  48 {
  49         return filter_id / (EFX_MCDI_FILTER_TBL_ROWS * 2);
  50 }
  51
  52 static u32 efx_mcdi_filter_make_filter_id(unsigned int pri, u16 idx)
  53 {
  54         return pri * EFX_MCDI_FILTER_TBL_ROWS * 2 + idx;
  55 }
  56
  57 /*
  58  * Decide whether a filter should be exclusive or else should allow
  59  * delivery to additional recipients.  Currently we decide that
  60  * filters for specific local unicast MAC and IP addresses are
  61  * exclusive.
  62  */
  63 static bool efx_mcdi_filter_is_exclusive(const struct efx_filter_spec *spec)
  64 {
  65         if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC &&
  66             !is_multicast_ether_addr(spec->loc_mac))
  67                 return true;
  68
  69         if ((spec->match_flags &
  70              (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) ==
  71             (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) {
  72                 if (spec->ether_type == htons(ETH_P_IP) &&
  73                     !ipv4_is_multicast(spec->loc_host[0]))
  74                         return true;
  75                 if (spec->ether_type == htons(ETH_P_IPV6) &&
  76                     ((const u8 *)spec->loc_host)[0] != 0xff)
  77                         return true;
  78         }
  79
  80         return false;
  81 }
  82
  83 static void
  84 efx_mcdi_filter_set_entry(struct efx_mcdi_filter_table *table,
  85                           unsigned int filter_idx,
  86                           const struct efx_filter_spec *spec,
  87                           unsigned int flags)
  88 {
  89         table->entry[filter_idx].spec = (unsigned long)spec | flags;
  90 }
  91
  92 static void
  93 efx_mcdi_filter_push_prep_set_match_fields(struct efx_nic *efx,
  94                                            const struct efx_filter_spec *spec,
  95                                            efx_dword_t *inbuf)
  96 {
  97         enum efx_encap_type encap_type = efx_filter_get_encap_type(spec);
  98         u32 match_fields = 0, uc_match, mc_match;
  99
 100         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
 101                        efx_mcdi_filter_is_exclusive(spec) ?
 102                        MC_CMD_FILTER_OP_IN_OP_INSERT :
 103                        MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE);
 104
 105         /*
 106          * Convert match flags and values.  Unlike almost
 107          * everything else in MCDI, these fields are in
 108          * network byte order.
 109          */
 110 #define COPY_VALUE(value, mcdi_field)                                        \
 111         do {                                                         \
 112                 match_fields |=                                      \
 113                         1 << MC_CMD_FILTER_OP_IN_MATCH_ ##           \
 114                         mcdi_field ## _LBN;                          \
 115                 BUILD_BUG_ON(                                        \
 116                         MC_CMD_FILTER_OP_IN_ ## mcdi_field ## _LEN < \
 117                         sizeof(value));                              \
 118                 memcpy(MCDI_PTR(inbuf, FILTER_OP_IN_ ## mcdi_field), \
 119                        &value, sizeof(value));                       \
 120         } while (0)
 121 #define COPY_FIELD(gen_flag, gen_field, mcdi_field)                          \
 122         if (spec->match_flags & EFX_FILTER_MATCH_ ## gen_flag) {     \
 123                 COPY_VALUE(spec->gen_field, mcdi_field);             \
 124         }
 125         /*
 126          * Handle encap filters first.  They will always be mismatch
 127          * (unknown UC or MC) filters
 128          */
 129         if (encap_type) {
 130                 /*
 131                  * ether_type and outer_ip_proto need to be variables
 132                  * because COPY_VALUE wants to memcpy them
 133                  */
 134                 __be16 ether_type =
 135                         htons(encap_type & EFX_ENCAP_FLAG_IPV6 ?
 136                               ETH_P_IPV6 : ETH_P_IP);
 137                 u8 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE;
 138                 u8 outer_ip_proto;
 139
 140                 switch (encap_type & EFX_ENCAP_TYPES_MASK) {
 141                 case EFX_ENCAP_TYPE_VXLAN:
 142                         vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN;
 143                         fallthrough;
 144                 case EFX_ENCAP_TYPE_GENEVE:
 145                         COPY_VALUE(ether_type, ETHER_TYPE);
 146                         outer_ip_proto = IPPROTO_UDP;
 147                         COPY_VALUE(outer_ip_proto, IP_PROTO);
 148                         /*
 149                          * We always need to set the type field, even
 150                          * though we're not matching on the TNI.
 151                          */
 152                         MCDI_POPULATE_DWORD_1(inbuf,
 153                                 FILTER_OP_EXT_IN_VNI_OR_VSID,
 154                                 FILTER_OP_EXT_IN_VNI_TYPE,
 155                                 vni_type);
 156                         break;
 157                 case EFX_ENCAP_TYPE_NVGRE:
 158                         COPY_VALUE(ether_type, ETHER_TYPE);
 159                         outer_ip_proto = IPPROTO_GRE;
 160                         COPY_VALUE(outer_ip_proto, IP_PROTO);
 161                         break;
 162                 default:
 163                         WARN_ON(1);
 164                 }
 165
 166                 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN;
 167                 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN;
 168         } else {
 169                 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN;
 170                 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN;
 171         }
 172
 173         if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC_IG)
 174                 match_fields |=
 175                         is_multicast_ether_addr(spec->loc_mac) ?
 176                         1 << mc_match :
 177                         1 << uc_match;
 178         COPY_FIELD(REM_HOST, rem_host, SRC_IP);
 179         COPY_FIELD(LOC_HOST, loc_host, DST_IP);
 180         COPY_FIELD(REM_MAC, rem_mac, SRC_MAC);
 181         COPY_FIELD(REM_PORT, rem_port, SRC_PORT);
 182         COPY_FIELD(LOC_MAC, loc_mac, DST_MAC);
 183         COPY_FIELD(LOC_PORT, loc_port, DST_PORT);
 184         COPY_FIELD(ETHER_TYPE, ether_type, ETHER_TYPE);
 185         COPY_FIELD(INNER_VID, inner_vid, INNER_VLAN);
 186         COPY_FIELD(OUTER_VID, outer_vid, OUTER_VLAN);
 187         COPY_FIELD(IP_PROTO, ip_proto, IP_PROTO);
 188 #undef COPY_FIELD
 189 #undef COPY_VALUE
 190         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_MATCH_FIELDS,
 191                        match_fields);
 192 }
 193
 194 static void efx_mcdi_filter_push_prep(struct efx_nic *efx,
 195                                       const struct efx_filter_spec *spec,
 196                                       efx_dword_t *inbuf, u64 handle,
 197                                       struct efx_rss_context *ctx,
 198                                       bool replacing)
 199 {
 200         u32 flags = spec->flags;
 201
 202         memset(inbuf, 0, MC_CMD_FILTER_OP_EXT_IN_LEN);
 203
 204         /* If RSS filter, caller better have given us an RSS context */
 205         if (flags & EFX_FILTER_FLAG_RX_RSS) {
 206                 /*
 207                  * We don't have the ability to return an error, so we'll just
 208                  * log a warning and disable RSS for the filter.
 209                  */
 210                 if (WARN_ON_ONCE(!ctx))
 211                         flags &= ~EFX_FILTER_FLAG_RX_RSS;
 212                 else if (WARN_ON_ONCE(ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID))
 213                         flags &= ~EFX_FILTER_FLAG_RX_RSS;
 214         }
 215
 216         if (replacing) {
 217                 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
 218                                MC_CMD_FILTER_OP_IN_OP_REPLACE);
 219                 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, handle);
 220         } else {
 221                 efx_mcdi_filter_push_prep_set_match_fields(efx, spec, inbuf);
 222         }
 223
 224         if (flags & EFX_FILTER_FLAG_VPORT_ID)
 225                 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, spec->vport_id);
 226         else
 227                 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, efx->vport_id);
 228         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_DEST,
 229                        spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ?
 230                        MC_CMD_FILTER_OP_IN_RX_DEST_DROP :
 231                        MC_CMD_FILTER_OP_IN_RX_DEST_HOST);
 232         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DOMAIN, 0);
 233         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DEST,
 234                        MC_CMD_FILTER_OP_IN_TX_DEST_DEFAULT);
 235         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_QUEUE,
 236                        spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ?
 237                        0 : spec->dmaq_id);
 238         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_MODE,
 239                        (flags & EFX_FILTER_FLAG_RX_RSS) ?
 240                        MC_CMD_FILTER_OP_IN_RX_MODE_RSS :
 241                        MC_CMD_FILTER_OP_IN_RX_MODE_SIMPLE);
 242         if (flags & EFX_FILTER_FLAG_RX_RSS)
 243                 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_CONTEXT, ctx->context_id);
 244 }
 245
 246 static int efx_mcdi_filter_push(struct efx_nic *efx,
 247                                 const struct efx_filter_spec *spec, u64 *handle,
 248                                 struct efx_rss_context *ctx, bool replacing)
 249 {
 250         MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
 251         MCDI_DECLARE_BUF(outbuf, MC_CMD_FILTER_OP_EXT_OUT_LEN);
 252         size_t outlen;
 253         int rc;
 254
 255         efx_mcdi_filter_push_prep(efx, spec, inbuf, *handle, ctx, replacing);
 256         rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf),
 257                                 outbuf, sizeof(outbuf), &outlen);
 258         if (rc && spec->priority != EFX_FILTER_PRI_HINT)
 259                 efx_mcdi_display_error(efx, MC_CMD_FILTER_OP, sizeof(inbuf),
 260                                        outbuf, outlen, rc);
 261         if (rc == 0)
 262                 *handle = MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE);
 263         if (rc == -ENOSPC)
 264                 rc = -EBUSY; /* to match efx_farch_filter_insert() */
 265         return rc;
 266 }
 267
 268 static u32 efx_mcdi_filter_mcdi_flags_from_spec(const struct efx_filter_spec *spec)
 269 {
 270         enum efx_encap_type encap_type = efx_filter_get_encap_type(spec);
 271         unsigned int match_flags = spec->match_flags;
 272         unsigned int uc_match, mc_match;
 273         u32 mcdi_flags = 0;
 274
 275 #define MAP_FILTER_TO_MCDI_FLAG(gen_flag, mcdi_field, encap) {          \
 276                 unsigned int  old_match_flags = match_flags;            \
 277                 match_flags &= ~EFX_FILTER_MATCH_ ## gen_flag;          \
 278                 if (match_flags != old_match_flags)                     \
 279                         mcdi_flags |=                                   \
 280                                 (1 << ((encap) ?                        \
 281                                        MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_ ## \
 282                                        mcdi_field ## _LBN :             \
 283                                        MC_CMD_FILTER_OP_EXT_IN_MATCH_ ##\
 284                                        mcdi_field ## _LBN));            \
 285         }
 286         /* inner or outer based on encap type */
 287         MAP_FILTER_TO_MCDI_FLAG(REM_HOST, SRC_IP, encap_type);
 288         MAP_FILTER_TO_MCDI_FLAG(LOC_HOST, DST_IP, encap_type);
 289         MAP_FILTER_TO_MCDI_FLAG(REM_MAC, SRC_MAC, encap_type);
 290         MAP_FILTER_TO_MCDI_FLAG(REM_PORT, SRC_PORT, encap_type);
 291         MAP_FILTER_TO_MCDI_FLAG(LOC_MAC, DST_MAC, encap_type);
 292         MAP_FILTER_TO_MCDI_FLAG(LOC_PORT, DST_PORT, encap_type);
 293         MAP_FILTER_TO_MCDI_FLAG(ETHER_TYPE, ETHER_TYPE, encap_type);
 294         MAP_FILTER_TO_MCDI_FLAG(IP_PROTO, IP_PROTO, encap_type);
 295         /* always outer */
 296         MAP_FILTER_TO_MCDI_FLAG(INNER_VID, INNER_VLAN, false);
 297         MAP_FILTER_TO_MCDI_FLAG(OUTER_VID, OUTER_VLAN, false);
 298 #undef MAP_FILTER_TO_MCDI_FLAG
 299
 300         /* special handling for encap type, and mismatch */
 301         if (encap_type) {
 302                 match_flags &= ~EFX_FILTER_MATCH_ENCAP_TYPE;
 303                 mcdi_flags |=
 304                         (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN);
 305                 mcdi_flags |= (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN);
 306
 307                 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN;
 308                 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN;
 309         } else {
 310                 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN;
 311                 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN;
 312         }
 313
 314         if (match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) {
 315                 match_flags &= ~EFX_FILTER_MATCH_LOC_MAC_IG;
 316                 mcdi_flags |=
 317                         is_multicast_ether_addr(spec->loc_mac) ?
 318                         1 << mc_match :
 319                         1 << uc_match;
 320         }
 321
 322         /* Did we map them all? */
 323         WARN_ON_ONCE(match_flags);
 324
 325         return mcdi_flags;
 326 }
 327
 328 static int efx_mcdi_filter_pri(struct efx_mcdi_filter_table *table,
 329                                const struct efx_filter_spec *spec)
 330 {
 331         u32 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec);
 332         unsigned int match_pri;
 333
 334         for (match_pri = 0;
 335              match_pri < table->rx_match_count;
 336              match_pri++)
 337                 if (table->rx_match_mcdi_flags[match_pri] == mcdi_flags)
 338                         return match_pri;
 339
 340         return -EPROTONOSUPPORT;
 341 }
 342
 343 static s32 efx_mcdi_filter_insert_locked(struct efx_nic *efx,
 344                                          struct efx_filter_spec *spec,
 345                                          bool replace_equal)
 346 {
 347         DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
 348         struct efx_mcdi_filter_table *table;
 349         struct efx_filter_spec *saved_spec;
 350         struct efx_rss_context *ctx = NULL;
 351         unsigned int match_pri, hash;
 352         unsigned int priv_flags;
 353         bool rss_locked = false;
 354         bool replacing = false;
 355         unsigned int depth, i;
 356         int ins_index = -1;
 357         DEFINE_WAIT(wait);
 358         bool is_mc_recip;
 359         s32 rc;
 360
 361         WARN_ON(!rwsem_is_locked(&efx->filter_sem));
 362         table = efx->filter_state;
 363         down_write(&table->lock);
 364
 365         /* For now, only support RX filters */
 366         if ((spec->flags & (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)) !=
 367             EFX_FILTER_FLAG_RX) {
 368                 rc = -EINVAL;
 369                 goto out_unlock;
 370         }
 371
 372         rc = efx_mcdi_filter_pri(table, spec);
 373         if (rc < 0)
 374                 goto out_unlock;
 375         match_pri = rc;
 376
 377         hash = efx_filter_spec_hash(spec);
 378         is_mc_recip = efx_filter_is_mc_recipient(spec);
 379         if (is_mc_recip)
 380                 bitmap_zero(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
 381
 382         if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {
 383                 mutex_lock(&efx->rss_lock);
 384                 rss_locked = true;
 385                 if (spec->rss_context)
 386                         ctx = efx_find_rss_context_entry(efx, spec->rss_context);
 387                 else
 388                         ctx = &efx->rss_context;
 389                 if (!ctx) {
 390                         rc = -ENOENT;
 391                         goto out_unlock;
 392                 }
 393                 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
 394                         rc = -EOPNOTSUPP;
 395                         goto out_unlock;
 396                 }
 397         }
 398
 399         /* Find any existing filters with the same match tuple or
 400          * else a free slot to insert at.
 401          */
 402         for (depth = 1; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
 403                 i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1);
 404                 saved_spec = efx_mcdi_filter_entry_spec(table, i);
 405
 406                 if (!saved_spec) {
 407                         if (ins_index < 0)
 408                                 ins_index = i;
 409                 } else if (efx_filter_spec_equal(spec, saved_spec)) {
 410                         if (spec->priority < saved_spec->priority &&
 411                             spec->priority != EFX_FILTER_PRI_AUTO) {
 412                                 rc = -EPERM;
 413                                 goto out_unlock;
 414                         }
 415                         if (!is_mc_recip) {
 416                                 /* This is the only one */
 417                                 if (spec->priority ==
 418                                     saved_spec->priority &&
 419                                     !replace_equal) {
 420                                         rc = -EEXIST;
 421                                         goto out_unlock;
 422                                 }
 423                                 ins_index = i;
 424                                 break;
 425                         } else if (spec->priority >
 426                                    saved_spec->priority ||
 427                                    (spec->priority ==
 428                                     saved_spec->priority &&
 429                                     replace_equal)) {
 430                                 if (ins_index < 0)
 431                                         ins_index = i;
 432                                 else
 433                                         __set_bit(depth, mc_rem_map);
 434                         }
 435                 }
 436         }
 437
 438         /* Once we reach the maximum search depth, use the first suitable
 439          * slot, or return -EBUSY if there was none
 440          */
 441         if (ins_index < 0) {
 442                 rc = -EBUSY;
 443                 goto out_unlock;
 444         }
 445
 446         /* Create a software table entry if necessary. */
 447         saved_spec = efx_mcdi_filter_entry_spec(table, ins_index);
 448         if (saved_spec) {
 449                 if (spec->priority == EFX_FILTER_PRI_AUTO &&
 450                     saved_spec->priority >= EFX_FILTER_PRI_AUTO) {
 451                         /* Just make sure it won't be removed */
 452                         if (saved_spec->priority > EFX_FILTER_PRI_AUTO)
 453                                 saved_spec->flags |= EFX_FILTER_FLAG_RX_OVER_AUTO;
 454                         table->entry[ins_index].spec &=
 455                                 ~EFX_EF10_FILTER_FLAG_AUTO_OLD;
 456                         rc = ins_index;
 457                         goto out_unlock;
 458                 }
 459                 replacing = true;
 460                 priv_flags = efx_mcdi_filter_entry_flags(table, ins_index);
 461         } else {
 462                 saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC);
 463                 if (!saved_spec) {
 464                         rc = -ENOMEM;
 465                         goto out_unlock;
 466                 }
 467                 *saved_spec = *spec;
 468                 priv_flags = 0;
 469         }
 470         efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags);
 471
 472         /* Actually insert the filter on the HW */
 473         rc = efx_mcdi_filter_push(efx, spec, &table->entry[ins_index].handle,
 474                                   ctx, replacing);
 475
 476         if (rc == -EINVAL && efx->must_realloc_vis)
 477                 /* The MC rebooted under us, causing it to reject our filter
 478                  * insertion as pointing to an invalid VI (spec->dmaq_id).
 479                  */
 480                 rc = -EAGAIN;
 481
 482         /* Finalise the software table entry */
 483         if (rc == 0) {
 484                 if (replacing) {
 485                         /* Update the fields that may differ */
 486                         if (saved_spec->priority == EFX_FILTER_PRI_AUTO)
 487                                 saved_spec->flags |=
 488                                         EFX_FILTER_FLAG_RX_OVER_AUTO;
 489                         saved_spec->priority = spec->priority;
 490                         saved_spec->flags &= EFX_FILTER_FLAG_RX_OVER_AUTO;
 491                         saved_spec->flags |= spec->flags;
 492                         saved_spec->rss_context = spec->rss_context;
 493                         saved_spec->dmaq_id = spec->dmaq_id;
 494                         saved_spec->vport_id = spec->vport_id;
 495                 }
 496         } else if (!replacing) {
 497                 kfree(saved_spec);
 498                 saved_spec = NULL;
 499         } else {
 500                 /* We failed to replace, so the old filter is still present.
 501                  * Roll back the software table to reflect this.  In fact the
 502                  * efx_mcdi_filter_set_entry() call below will do the right
 503                  * thing, so nothing extra is needed here.
 504                  */
 505         }
 506         efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags);
 507
 508         /* Remove and finalise entries for lower-priority multicast
 509          * recipients
 510          */
 511         if (is_mc_recip) {
 512                 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
 513                 unsigned int depth, i;
 514
 515                 memset(inbuf, 0, sizeof(inbuf));
 516
 517                 for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
 518                         if (!test_bit(depth, mc_rem_map))
 519                                 continue;
 520
 521                         i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1);
 522                         saved_spec = efx_mcdi_filter_entry_spec(table, i);
 523                         priv_flags = efx_mcdi_filter_entry_flags(table, i);
 524
 525                         if (rc == 0) {
 526                                 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
 527                                                MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
 528                                 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
 529                                                table->entry[i].handle);
 530                                 rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP,
 531                                                   inbuf, sizeof(inbuf),
 532                                                   NULL, 0, NULL);
 533                         }
 534
 535                         if (rc == 0) {
 536                                 kfree(saved_spec);
 537                                 saved_spec = NULL;
 538                                 priv_flags = 0;
 539                         }
 540                         efx_mcdi_filter_set_entry(table, i, saved_spec,
 541                                                   priv_flags);
 542                 }
 543         }
 544
 545         /* If successful, return the inserted filter ID */
 546         if (rc == 0)
 547                 rc = efx_mcdi_filter_make_filter_id(match_pri, ins_index);
 548
 549 out_unlock:
 550         if (rss_locked)
 551                 mutex_unlock(&efx->rss_lock);
 552         up_write(&table->lock);
 553         return rc;
 554 }
 555
 556 s32 efx_mcdi_filter_insert(struct efx_nic *efx, struct efx_filter_spec *spec,
 557                            bool replace_equal)
 558 {
 559         s32 ret;
 560
 561         down_read(&efx->filter_sem);
 562         ret = efx_mcdi_filter_insert_locked(efx, spec, replace_equal);
 563         up_read(&efx->filter_sem);
 564
 565         return ret;
 566 }
 567
 568 /*
 569  * Remove a filter.
 570  * If !by_index, remove by ID
 571  * If by_index, remove by index
 572  * Filter ID may come from userland and must be range-checked.
 573  * Caller must hold efx->filter_sem for read, and efx->filter_state->lock
 574  * for write.
 575  */
 576 static int efx_mcdi_filter_remove_internal(struct efx_nic *efx,
 577                                            unsigned int priority_mask,
 578                                            u32 filter_id, bool by_index)
 579 {
 580         unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id);
 581         struct efx_mcdi_filter_table *table = efx->filter_state;
 582         MCDI_DECLARE_BUF(inbuf,
 583                          MC_CMD_FILTER_OP_IN_HANDLE_OFST +
 584                          MC_CMD_FILTER_OP_IN_HANDLE_LEN);
 585         struct efx_filter_spec *spec;
 586         DEFINE_WAIT(wait);
 587         int rc;
 588
 589         spec = efx_mcdi_filter_entry_spec(table, filter_idx);
 590         if (!spec ||
 591             (!by_index &&
 592              efx_mcdi_filter_pri(table, spec) !=
 593              efx_mcdi_filter_get_unsafe_pri(filter_id)))
 594                 return -ENOENT;
 595
 596         if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO &&
 597             priority_mask == (1U << EFX_FILTER_PRI_AUTO)) {
 598                 /* Just remove flags */
 599                 spec->flags &= ~EFX_FILTER_FLAG_RX_OVER_AUTO;
 600                 table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_AUTO_OLD;
 601                 return 0;
 602         }
 603
 604         if (!(priority_mask & (1U << spec->priority)))
 605                 return -ENOENT;
 606
 607         if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) {
 608                 /* Reset to an automatic filter */
 609
 610                 struct efx_filter_spec new_spec = *spec;
 611
 612                 new_spec.priority = EFX_FILTER_PRI_AUTO;
 613                 new_spec.flags = (EFX_FILTER_FLAG_RX |
 614                                   (efx_rss_active(&efx->rss_context) ?
 615                                    EFX_FILTER_FLAG_RX_RSS : 0));
 616                 new_spec.dmaq_id = 0;
 617                 new_spec.rss_context = 0;
 618                 rc = efx_mcdi_filter_push(efx, &new_spec,
 619                                           &table->entry[filter_idx].handle,
 620                                           &efx->rss_context,
 621                                           true);
 622
 623                 if (rc == 0)
 624                         *spec = new_spec;
 625         } else {
 626                 /* Really remove the filter */
 627
 628                 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
 629                                efx_mcdi_filter_is_exclusive(spec) ?
 630                                MC_CMD_FILTER_OP_IN_OP_REMOVE :
 631                                MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
 632                 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
 633                                table->entry[filter_idx].handle);
 634                 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP,
 635                                         inbuf, sizeof(inbuf), NULL, 0, NULL);
 636
 637                 if ((rc == 0) || (rc == -ENOENT)) {
 638                         /* Filter removed OK or didn't actually exist */
 639                         kfree(spec);
 640                         efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0);
 641                 } else {
 642                         efx_mcdi_display_error(efx, MC_CMD_FILTER_OP,
 643                                                MC_CMD_FILTER_OP_EXT_IN_LEN,
 644                                                NULL, 0, rc);
 645                 }
 646         }
 647
 648         return rc;
 649 }
 650
 651 /* Remove filters that weren't renewed. */
 652 static void efx_mcdi_filter_remove_old(struct efx_nic *efx)
 653 {
 654         struct efx_mcdi_filter_table *table = efx->filter_state;
 655         int remove_failed = 0;
 656         int remove_noent = 0;
 657         int rc;
 658         int i;
 659
 660         down_write(&table->lock);
 661         for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) {
 662                 if (READ_ONCE(table->entry[i].spec) &
 663                     EFX_EF10_FILTER_FLAG_AUTO_OLD) {
 664                         rc = efx_mcdi_filter_remove_internal(efx,
 665                                         1U << EFX_FILTER_PRI_AUTO, i, true);
 666                         if (rc == -ENOENT)
 667                                 remove_noent++;
 668                         else if (rc)
 669                                 remove_failed++;
 670                 }
 671         }
 672         up_write(&table->lock);
 673
 674         if (remove_failed)
 675                 netif_info(efx, drv, efx->net_dev,
 676                            "%s: failed to remove %d filters\n",
 677                            __func__, remove_failed);
 678         if (remove_noent)
 679                 netif_info(efx, drv, efx->net_dev,
 680                            "%s: failed to remove %d non-existent filters\n",
 681                            __func__, remove_noent);
 682 }
 683
 684 int efx_mcdi_filter_remove_safe(struct efx_nic *efx,
 685                                 enum efx_filter_priority priority,
 686                                 u32 filter_id)
 687 {
 688         struct efx_mcdi_filter_table *table;
 689         int rc;
 690
 691         down_read(&efx->filter_sem);
 692         table = efx->filter_state;
 693         down_write(&table->lock);
 694         rc = efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id,
 695                                              false);
 696         up_write(&table->lock);
 697         up_read(&efx->filter_sem);
 698         return rc;
 699 }
 700
 701 /* Caller must hold efx->filter_sem for read */
 702 static void efx_mcdi_filter_remove_unsafe(struct efx_nic *efx,
 703                                           enum efx_filter_priority priority,
 704                                           u32 filter_id)
 705 {
 706         struct efx_mcdi_filter_table *table = efx->filter_state;
 707
 708         if (filter_id == EFX_EF10_FILTER_ID_INVALID)
 709                 return;
 710
 711         down_write(&table->lock);
 712         efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id,
 713                                         true);
 714         up_write(&table->lock);
 715 }
 716
 717 int efx_mcdi_filter_get_safe(struct efx_nic *efx,
 718                              enum efx_filter_priority priority,
 719                              u32 filter_id, struct efx_filter_spec *spec)
 720 {
 721         unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id);
 722         const struct efx_filter_spec *saved_spec;
 723         struct efx_mcdi_filter_table *table;
 724         int rc;
 725
 726         down_read(&efx->filter_sem);
 727         table = efx->filter_state;
 728         down_read(&table->lock);
 729         saved_spec = efx_mcdi_filter_entry_spec(table, filter_idx);
 730         if (saved_spec && saved_spec->priority == priority &&
 731             efx_mcdi_filter_pri(table, saved_spec) ==
 732             efx_mcdi_filter_get_unsafe_pri(filter_id)) {
 733                 *spec = *saved_spec;
 734                 rc = 0;
 735         } else {
 736                 rc = -ENOENT;
 737         }
 738         up_read(&table->lock);
 739         up_read(&efx->filter_sem);
 740         return rc;
 741 }
 742
 743 static int efx_mcdi_filter_insert_addr_list(struct efx_nic *efx,
 744                                             struct efx_mcdi_filter_vlan *vlan,
 745                                             bool multicast, bool rollback)
 746 {
 747         struct efx_mcdi_filter_table *table = efx->filter_state;
 748         struct efx_mcdi_dev_addr *addr_list;
 749         enum efx_filter_flags filter_flags;
 750         struct efx_filter_spec spec;
 751         u8 baddr[ETH_ALEN];
 752         unsigned int i, j;
 753         int addr_count;
 754         u16 *ids;
 755         int rc;
 756
 757         if (multicast) {
 758                 addr_list = table->dev_mc_list;
 759                 addr_count = table->dev_mc_count;
 760                 ids = vlan->mc;
 761         } else {
 762                 addr_list = table->dev_uc_list;
 763                 addr_count = table->dev_uc_count;
 764                 ids = vlan->uc;
 765         }
 766
 767         filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0;
 768
 769         /* Insert/renew filters */
 770         for (i = 0; i < addr_count; i++) {
 771                 EFX_WARN_ON_PARANOID(ids[i] != EFX_EF10_FILTER_ID_INVALID);
 772                 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
 773                 efx_filter_set_eth_local(&spec, vlan->vid, addr_list[i].addr);
 774                 rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
 775                 if (rc < 0) {
 776                         if (rollback) {
 777                                 netif_info(efx, drv, efx->net_dev,
 778                                            "efx_mcdi_filter_insert failed rc=%d\n",
 779                                            rc);
 780                                 /* Fall back to promiscuous */
 781                                 for (j = 0; j < i; j++) {
 782                                         efx_mcdi_filter_remove_unsafe(
 783                                                 efx, EFX_FILTER_PRI_AUTO,
 784                                                 ids[j]);
 785                                         ids[j] = EFX_EF10_FILTER_ID_INVALID;
 786                                 }
 787                                 return rc;
 788                         } else {
 789                                 /* keep invalid ID, and carry on */
 790                         }
 791                 } else {
 792                         ids[i] = efx_mcdi_filter_get_unsafe_id(rc);
 793                 }
 794         }
 795
 796         if (multicast && rollback) {
 797                 /* Also need an Ethernet broadcast filter */
 798                 EFX_WARN_ON_PARANOID(vlan->default_filters[EFX_EF10_BCAST] !=
 799                                      EFX_EF10_FILTER_ID_INVALID);
 800                 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
 801                 eth_broadcast_addr(baddr);
 802                 efx_filter_set_eth_local(&spec, vlan->vid, baddr);
 803                 rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
 804                 if (rc < 0) {
 805                         netif_warn(efx, drv, efx->net_dev,
 806                                    "Broadcast filter insert failed rc=%d\n", rc);
 807                         /* Fall back to promiscuous */
 808                         for (j = 0; j < i; j++) {
 809                                 efx_mcdi_filter_remove_unsafe(
 810                                         efx, EFX_FILTER_PRI_AUTO,
 811                                         ids[j]);
 812                                 ids[j] = EFX_EF10_FILTER_ID_INVALID;
 813                         }
 814                         return rc;
 815                 } else {
 816                         vlan->default_filters[EFX_EF10_BCAST] =
 817                                 efx_mcdi_filter_get_unsafe_id(rc);
 818                 }
 819         }
 820
 821         return 0;
 822 }
 823
 824 static int efx_mcdi_filter_insert_def(struct efx_nic *efx,
 825                                       struct efx_mcdi_filter_vlan *vlan,
 826                                       enum efx_encap_type encap_type,
 827                                       bool multicast, bool rollback)
 828 {
 829         struct efx_mcdi_filter_table *table = efx->filter_state;
 830         enum efx_filter_flags filter_flags;
 831         struct efx_filter_spec spec;
 832         u8 baddr[ETH_ALEN];
 833         int rc;
 834         u16 *id;
 835
 836         filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0;
 837
 838         efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
 839
 840         if (multicast)
 841                 efx_filter_set_mc_def(&spec);
 842         else
 843                 efx_filter_set_uc_def(&spec);
 844
 845         if (encap_type) {
 846                 if (efx_has_cap(efx, VXLAN_NVGRE))
 847                         efx_filter_set_encap_type(&spec, encap_type);
 848                 else
 849                         /*
 850                          * don't insert encap filters on non-supporting
 851                          * platforms. ID will be left as INVALID.
 852                          */
 853                         return 0;
 854         }
 855
 856         if (vlan->vid != EFX_FILTER_VID_UNSPEC)
 857                 efx_filter_set_eth_local(&spec, vlan->vid, NULL);
 858
 859         rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
 860         if (rc < 0) {
 861                 const char *um = multicast ? "Multicast" : "Unicast";
 862                 const char *encap_name = "";
 863                 const char *encap_ipv = "";
 864
 865                 if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
 866                     EFX_ENCAP_TYPE_VXLAN)
 867                         encap_name = "VXLAN ";
 868                 else if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
 869                          EFX_ENCAP_TYPE_NVGRE)
 870                         encap_name = "NVGRE ";
 871                 else if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
 872                          EFX_ENCAP_TYPE_GENEVE)
 873                         encap_name = "GENEVE ";
 874                 if (encap_type & EFX_ENCAP_FLAG_IPV6)
 875                         encap_ipv = "IPv6 ";
 876                 else if (encap_type)
 877                         encap_ipv = "IPv4 ";
 878
 879                 /*
 880                  * unprivileged functions can't insert mismatch filters
 881                  * for encapsulated or unicast traffic, so downgrade
 882                  * those warnings to debug.
 883                  */
 884                 netif_cond_dbg(efx, drv, efx->net_dev,
 885                                rc == -EPERM && (encap_type || !multicast), warn,
 886                                "%s%s%s mismatch filter insert failed rc=%d\n",
 887                                encap_name, encap_ipv, um, rc);
 888         } else if (multicast) {
 889                 /* mapping from encap types to default filter IDs (multicast) */
 890                 static enum efx_mcdi_filter_default_filters map[] = {
 891                         [EFX_ENCAP_TYPE_NONE] = EFX_EF10_MCDEF,
 892                         [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_MCDEF,
 893                         [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_MCDEF,
 894                         [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_MCDEF,
 895                         [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] =
 896                                 EFX_EF10_VXLAN6_MCDEF,
 897                         [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] =
 898                                 EFX_EF10_NVGRE6_MCDEF,
 899                         [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] =
 900                                 EFX_EF10_GENEVE6_MCDEF,
 901                 };
 902
 903                 /* quick bounds check (BCAST result impossible) */
 904                 BUILD_BUG_ON(EFX_EF10_BCAST != 0);
 905                 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) {
 906                         WARN_ON(1);
 907                         return -EINVAL;
 908                 }
 909                 /* then follow map */
 910                 id = &vlan->default_filters[map[encap_type]];
 911
 912                 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID);
 913                 *id = efx_mcdi_filter_get_unsafe_id(rc);
 914                 if (!table->mc_chaining && !encap_type) {
 915                         /* Also need an Ethernet broadcast filter */
 916                         efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO,
 917                                            filter_flags, 0);
 918                         eth_broadcast_addr(baddr);
 919                         efx_filter_set_eth_local(&spec, vlan->vid, baddr);
 920                         rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
 921                         if (rc < 0) {
 922                                 netif_warn(efx, drv, efx->net_dev,
 923                                            "Broadcast filter insert failed rc=%d\n",
 924                                            rc);
 925                                 if (rollback) {
 926                                         /* Roll back the mc_def filter */
 927                                         efx_mcdi_filter_remove_unsafe(
 928                                                         efx, EFX_FILTER_PRI_AUTO,
 929                                                         *id);
 930                                         *id = EFX_EF10_FILTER_ID_INVALID;
 931                                         return rc;
 932                                 }
 933                         } else {
 934                                 EFX_WARN_ON_PARANOID(
 935                                         vlan->default_filters[EFX_EF10_BCAST] !=
 936                                         EFX_EF10_FILTER_ID_INVALID);
 937                                 vlan->default_filters[EFX_EF10_BCAST] =
 938                                         efx_mcdi_filter_get_unsafe_id(rc);
 939                         }
 940                 }
 941                 rc = 0;
 942         } else {
 943                 /* mapping from encap types to default filter IDs (unicast) */
 944                 static enum efx_mcdi_filter_default_filters map[] = {
 945                         [EFX_ENCAP_TYPE_NONE] = EFX_EF10_UCDEF,
 946                         [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_UCDEF,
 947                         [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_UCDEF,
 948                         [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_UCDEF,
 949                         [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] =
 950                                 EFX_EF10_VXLAN6_UCDEF,
 951                         [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] =
 952                                 EFX_EF10_NVGRE6_UCDEF,
 953                         [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] =
 954                                 EFX_EF10_GENEVE6_UCDEF,
 955                 };
 956
 957                 /* quick bounds check (BCAST result impossible) */
 958                 BUILD_BUG_ON(EFX_EF10_BCAST != 0);
 959                 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) {
 960                         WARN_ON(1);
 961                         return -EINVAL;
 962                 }
 963                 /* then follow map */
 964                 id = &vlan->default_filters[map[encap_type]];
 965                 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID);
 966                 *id = rc;
 967                 rc = 0;
 968         }
 969         return rc;
 970 }
 971
 972 /*
 973  * Caller must hold efx->filter_sem for read if race against
 974  * efx_mcdi_filter_table_remove() is possible
 975  */
 976 static void efx_mcdi_filter_vlan_sync_rx_mode(struct efx_nic *efx,
 977                                               struct efx_mcdi_filter_vlan *vlan)
 978 {
 979         struct efx_mcdi_filter_table *table = efx->filter_state;
 980
 981         /*
 982          * Do not install unspecified VID if VLAN filtering is enabled.
 983          * Do not install all specified VIDs if VLAN filtering is disabled.
 984          */
 985         if ((vlan->vid == EFX_FILTER_VID_UNSPEC) == table->vlan_filter)
 986                 return;
 987
 988         /* Insert/renew unicast filters */
 989         if (table->uc_promisc) {
 990                 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NONE,
 991                                            false, false);
 992                 efx_mcdi_filter_insert_addr_list(efx, vlan, false, false);
 993         } else {
 994                 /*
 995                  * If any of the filters failed to insert, fall back to
 996                  * promiscuous mode - add in the uc_def filter.  But keep
 997                  * our individual unicast filters.
 998                  */
 999                 if (efx_mcdi_filter_insert_addr_list(efx, vlan, false, false))
1000                         efx_mcdi_filter_insert_def(efx, vlan,
1001                                                    EFX_ENCAP_TYPE_NONE,
1002                                                    false, false);
1003         }
1004         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN,
1005                                    false, false);
1006         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN |
1007                                               EFX_ENCAP_FLAG_IPV6,
1008                                    false, false);
1009         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE,
1010                                    false, false);
1011         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE |
1012                                               EFX_ENCAP_FLAG_IPV6,
1013                                    false, false);
1014         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE,
1015                                    false, false);
1016         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE |
1017                                               EFX_ENCAP_FLAG_IPV6,
1018                                    false, false);
1019
1020         /*
1021          * Insert/renew multicast filters
1022          *
1023          * If changing promiscuous state with cascaded multicast filters, remove
1024          * old filters first, so that packets are dropped rather than duplicated
1025          */
1026         if (table->mc_chaining && table->mc_promisc_last != table->mc_promisc)
1027                 efx_mcdi_filter_remove_old(efx);
1028         if (table->mc_promisc) {
1029                 if (table->mc_chaining) {
1030                         /*
1031                          * If we failed to insert promiscuous filters, rollback
1032                          * and fall back to individual multicast filters
1033                          */
1034                         if (efx_mcdi_filter_insert_def(efx, vlan,
1035                                                        EFX_ENCAP_TYPE_NONE,
1036                                                        true, true)) {
1037                                 /* Changing promisc state, so remove old filters */
1038                                 efx_mcdi_filter_remove_old(efx);
1039                                 efx_mcdi_filter_insert_addr_list(efx, vlan,
1040                                                                  true, false);
1041                         }
1042                 } else {
1043                         /*
1044                          * If we failed to insert promiscuous filters, don't
1045                          * rollback.  Regardless, also insert the mc_list,
1046                          * unless it's incomplete due to overflow
1047                          */
1048                         efx_mcdi_filter_insert_def(efx, vlan,
1049                                                    EFX_ENCAP_TYPE_NONE,
1050                                                    true, false);
1051                         if (!table->mc_overflow)
1052                                 efx_mcdi_filter_insert_addr_list(efx, vlan,
1053                                                                  true, false);
1054                 }
1055         } else {
1056                 /*
1057                  * If any filters failed to insert, rollback and fall back to
1058                  * promiscuous mode - mc_def filter and maybe broadcast.  If
1059                  * that fails, roll back again and insert as many of our
1060                  * individual multicast filters as we can.
1061                  */
1062                 if (efx_mcdi_filter_insert_addr_list(efx, vlan, true, true)) {
1063                         /* Changing promisc state, so remove old filters */
1064                         if (table->mc_chaining)
1065                                 efx_mcdi_filter_remove_old(efx);
1066                         if (efx_mcdi_filter_insert_def(efx, vlan,
1067                                                        EFX_ENCAP_TYPE_NONE,
1068                                                        true, true))
1069                                 efx_mcdi_filter_insert_addr_list(efx, vlan,
1070                                                                  true, false);
1071                 }
1072         }
1073         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN,
1074                                    true, false);
1075         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN |
1076                                               EFX_ENCAP_FLAG_IPV6,
1077                                    true, false);
1078         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE,
1079                                    true, false);
1080         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE |
1081                                               EFX_ENCAP_FLAG_IPV6,
1082                                    true, false);
1083         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE,
1084                                    true, false);
1085         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE |
1086                                               EFX_ENCAP_FLAG_IPV6,
1087                                    true, false);
1088 }
1089
1090 int efx_mcdi_filter_clear_rx(struct efx_nic *efx,
1091                              enum efx_filter_priority priority)
1092 {
1093         struct efx_mcdi_filter_table *table;
1094         unsigned int priority_mask;
1095         unsigned int i;
1096         int rc;
1097
1098         priority_mask = (((1U << (priority + 1)) - 1) &
1099                          ~(1U << EFX_FILTER_PRI_AUTO));
1100
1101         down_read(&efx->filter_sem);
1102         table = efx->filter_state;
1103         down_write(&table->lock);
1104         for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) {
1105                 rc = efx_mcdi_filter_remove_internal(efx, priority_mask,
1106                                                      i, true);
1107                 if (rc && rc != -ENOENT)
1108                         break;
1109                 rc = 0;
1110         }
1111
1112         up_write(&table->lock);
1113         up_read(&efx->filter_sem);
1114         return rc;
1115 }
1116
1117 u32 efx_mcdi_filter_count_rx_used(struct efx_nic *efx,
1118                                  enum efx_filter_priority priority)
1119 {
1120         struct efx_mcdi_filter_table *table;
1121         unsigned int filter_idx;
1122         s32 count = 0;
1123
1124         down_read(&efx->filter_sem);
1125         table = efx->filter_state;
1126         down_read(&table->lock);
1127         for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1128                 if (table->entry[filter_idx].spec &&
1129                     efx_mcdi_filter_entry_spec(table, filter_idx)->priority ==
1130                     priority)
1131                         ++count;
1132         }
1133         up_read(&table->lock);
1134         up_read(&efx->filter_sem);
1135         return count;
1136 }
1137
1138 u32 efx_mcdi_filter_get_rx_id_limit(struct efx_nic *efx)
1139 {
1140         struct efx_mcdi_filter_table *table = efx->filter_state;
1141
1142         return table->rx_match_count * EFX_MCDI_FILTER_TBL_ROWS * 2;
1143 }
1144
1145 s32 efx_mcdi_filter_get_rx_ids(struct efx_nic *efx,
1146                                enum efx_filter_priority priority,
1147                                u32 *buf, u32 size)
1148 {
1149         struct efx_mcdi_filter_table *table;
1150         struct efx_filter_spec *spec;
1151         unsigned int filter_idx;
1152         s32 count = 0;
1153
1154         down_read(&efx->filter_sem);
1155         table = efx->filter_state;
1156         down_read(&table->lock);
1157
1158         for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1159                 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1160                 if (spec && spec->priority == priority) {
1161                         if (count == size) {
1162                                 count = -EMSGSIZE;
1163                                 break;
1164                         }
1165                         buf[count++] =
1166                                 efx_mcdi_filter_make_filter_id(
1167                                         efx_mcdi_filter_pri(table, spec),
1168                                         filter_idx);
1169                 }
1170         }
1171         up_read(&table->lock);
1172         up_read(&efx->filter_sem);
1173         return count;
1174 }
1175
1176 static int efx_mcdi_filter_match_flags_from_mcdi(bool encap, u32 mcdi_flags)
1177 {
1178         int match_flags = 0;
1179
1180 #define MAP_FLAG(gen_flag, mcdi_field) do {                             \
1181                 u32 old_mcdi_flags = mcdi_flags;                        \
1182                 mcdi_flags &= ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ ##  \
1183                                      mcdi_field ## _LBN);               \
1184                 if (mcdi_flags != old_mcdi_flags)                       \
1185                         match_flags |= EFX_FILTER_MATCH_ ## gen_flag;   \
1186         } while (0)
1187
1188         if (encap) {
1189                 /* encap filters must specify encap type */
1190                 match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
1191                 /* and imply ethertype and ip proto */
1192                 mcdi_flags &=
1193                         ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN);
1194                 mcdi_flags &=
1195                         ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN);
1196                 /* VLAN tags refer to the outer packet */
1197                 MAP_FLAG(INNER_VID, INNER_VLAN);
1198                 MAP_FLAG(OUTER_VID, OUTER_VLAN);
1199                 /* everything else refers to the inner packet */
1200                 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_UCAST_DST);
1201                 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_MCAST_DST);
1202                 MAP_FLAG(REM_HOST, IFRM_SRC_IP);
1203                 MAP_FLAG(LOC_HOST, IFRM_DST_IP);
1204                 MAP_FLAG(REM_MAC, IFRM_SRC_MAC);
1205                 MAP_FLAG(REM_PORT, IFRM_SRC_PORT);
1206                 MAP_FLAG(LOC_MAC, IFRM_DST_MAC);
1207                 MAP_FLAG(LOC_PORT, IFRM_DST_PORT);
1208                 MAP_FLAG(ETHER_TYPE, IFRM_ETHER_TYPE);
1209                 MAP_FLAG(IP_PROTO, IFRM_IP_PROTO);
1210         } else {
1211                 MAP_FLAG(LOC_MAC_IG, UNKNOWN_UCAST_DST);
1212                 MAP_FLAG(LOC_MAC_IG, UNKNOWN_MCAST_DST);
1213                 MAP_FLAG(REM_HOST, SRC_IP);
1214                 MAP_FLAG(LOC_HOST, DST_IP);
1215                 MAP_FLAG(REM_MAC, SRC_MAC);
1216                 MAP_FLAG(REM_PORT, SRC_PORT);
1217                 MAP_FLAG(LOC_MAC, DST_MAC);
1218                 MAP_FLAG(LOC_PORT, DST_PORT);
1219                 MAP_FLAG(ETHER_TYPE, ETHER_TYPE);
1220                 MAP_FLAG(INNER_VID, INNER_VLAN);
1221                 MAP_FLAG(OUTER_VID, OUTER_VLAN);
1222                 MAP_FLAG(IP_PROTO, IP_PROTO);
1223         }
1224 #undef MAP_FLAG
1225
1226         /* Did we map them all? */
1227         if (mcdi_flags)
1228                 return -EINVAL;
1229
1230         return match_flags;
1231 }
1232
1233 bool efx_mcdi_filter_match_supported(struct efx_mcdi_filter_table *table,
1234                                      bool encap,
1235                                      enum efx_filter_match_flags match_flags)
1236 {
1237         unsigned int match_pri;
1238         int mf;
1239
1240         for (match_pri = 0;
1241              match_pri < table->rx_match_count;
1242              match_pri++) {
1243                 mf = efx_mcdi_filter_match_flags_from_mcdi(encap,
1244                                 table->rx_match_mcdi_flags[match_pri]);
1245                 if (mf == match_flags)
1246                         return true;
1247         }
1248
1249         return false;
1250 }
1251
1252 static int
1253 efx_mcdi_filter_table_probe_matches(struct efx_nic *efx,
1254                                     struct efx_mcdi_filter_table *table,
1255                                     bool encap)
1256 {
1257         MCDI_DECLARE_BUF(inbuf, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN);
1258         MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX);
1259         unsigned int pd_match_pri, pd_match_count;
1260         size_t outlen;
1261         int rc;
1262
1263         /* Find out which RX filter types are supported, and their priorities */
1264         MCDI_SET_DWORD(inbuf, GET_PARSER_DISP_INFO_IN_OP,
1265                        encap ?
1266                        MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_ENCAP_RX_MATCHES :
1267                        MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES);
1268         rc = efx_mcdi_rpc(efx, MC_CMD_GET_PARSER_DISP_INFO,
1269                           inbuf, sizeof(inbuf), outbuf, sizeof(outbuf),
1270                           &outlen);
1271         if (rc)
1272                 return rc;
1273
1274         pd_match_count = MCDI_VAR_ARRAY_LEN(
1275                 outlen, GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES);
1276
1277         for (pd_match_pri = 0; pd_match_pri < pd_match_count; pd_match_pri++) {
1278                 u32 mcdi_flags =
1279                         MCDI_ARRAY_DWORD(
1280                                 outbuf,
1281                                 GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES,
1282                                 pd_match_pri);
1283                 rc = efx_mcdi_filter_match_flags_from_mcdi(encap, mcdi_flags);
1284                 if (rc < 0) {
1285                         netif_dbg(efx, probe, efx->net_dev,
1286                                   "%s: fw flags %#x pri %u not supported in driver\n",
1287                                   __func__, mcdi_flags, pd_match_pri);
1288                 } else {
1289                         netif_dbg(efx, probe, efx->net_dev,
1290                                   "%s: fw flags %#x pri %u supported as driver flags %#x pri %u\n",
1291                                   __func__, mcdi_flags, pd_match_pri,
1292                                   rc, table->rx_match_count);
1293                         table->rx_match_mcdi_flags[table->rx_match_count] = mcdi_flags;
1294                         table->rx_match_count++;
1295                 }
1296         }
1297
1298         return 0;
1299 }
1300
1301 int efx_mcdi_filter_table_probe(struct efx_nic *efx, bool multicast_chaining)
1302 {
1303         struct net_device *net_dev = efx->net_dev;
1304         struct efx_mcdi_filter_table *table;
1305         int rc;
1306
1307         if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1308                 return -EINVAL;
1309
1310         if (efx->filter_state) /* already probed */
1311                 return 0;
1312
1313         table = kzalloc(sizeof(*table), GFP_KERNEL);
1314         if (!table)
1315                 return -ENOMEM;
1316
1317         table->mc_chaining = multicast_chaining;
1318         table->rx_match_count = 0;
1319         rc = efx_mcdi_filter_table_probe_matches(efx, table, false);
1320         if (rc)
1321                 goto fail;
1322         if (efx_has_cap(efx, VXLAN_NVGRE))
1323                 rc = efx_mcdi_filter_table_probe_matches(efx, table, true);
1324         if (rc)
1325                 goto fail;
1326         if ((efx_supported_features(efx) & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1327             !(efx_mcdi_filter_match_supported(table, false,
1328                 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC)) &&
1329               efx_mcdi_filter_match_supported(table, false,
1330                 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC_IG)))) {
1331                 netif_info(efx, probe, net_dev,
1332                            "VLAN filters are not supported in this firmware variant\n");
1333                 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
1334                 efx->fixed_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
1335                 net_dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
1336         }
1337
1338         table->entry = vzalloc(array_size(EFX_MCDI_FILTER_TBL_ROWS,
1339                                           sizeof(*table->entry)));
1340         if (!table->entry) {
1341                 rc = -ENOMEM;
1342                 goto fail;
1343         }
1344
1345         table->mc_promisc_last = false;
1346         table->vlan_filter =
1347                 !!(efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);
1348         INIT_LIST_HEAD(&table->vlan_list);
1349         init_rwsem(&table->lock);
1350
1351         efx->filter_state = table;
1352
1353         return 0;
1354 fail:
1355         kfree(table);
1356         return rc;
1357 }
1358
1359 void efx_mcdi_filter_table_reset_mc_allocations(struct efx_nic *efx)
1360 {
1361         struct efx_mcdi_filter_table *table = efx->filter_state;
1362
1363         if (table) {
1364                 table->must_restore_filters = true;
1365                 table->must_restore_rss_contexts = true;
1366         }
1367 }
1368
1369 /*
1370  * Caller must hold efx->filter_sem for read if race against
1371  * efx_mcdi_filter_table_remove() is possible
1372  */
1373 void efx_mcdi_filter_table_restore(struct efx_nic *efx)
1374 {
1375         struct efx_mcdi_filter_table *table = efx->filter_state;
1376         unsigned int invalid_filters = 0, failed = 0;
1377         struct efx_mcdi_filter_vlan *vlan;
1378         struct efx_filter_spec *spec;
1379         struct efx_rss_context *ctx;
1380         unsigned int filter_idx;
1381         u32 mcdi_flags;
1382         int match_pri;
1383         int rc, i;
1384
1385         WARN_ON(!rwsem_is_locked(&efx->filter_sem));
1386
1387         if (!table || !table->must_restore_filters)
1388                 return;
1389
1390         down_write(&table->lock);
1391         mutex_lock(&efx->rss_lock);
1392
1393         for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1394                 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1395                 if (!spec)
1396                         continue;
1397
1398                 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec);
1399                 match_pri = 0;
1400                 while (match_pri < table->rx_match_count &&
1401                        table->rx_match_mcdi_flags[match_pri] != mcdi_flags)
1402                         ++match_pri;
1403                 if (match_pri >= table->rx_match_count) {
1404                         invalid_filters++;
1405                         goto not_restored;
1406                 }
1407                 if (spec->rss_context)
1408                         ctx = efx_find_rss_context_entry(efx, spec->rss_context);
1409                 else
1410                         ctx = &efx->rss_context;
1411                 if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {
1412                         if (!ctx) {
1413                                 netif_warn(efx, drv, efx->net_dev,
1414                                            "Warning: unable to restore a filter with nonexistent RSS context %u.\n",
1415                                            spec->rss_context);
1416                                 invalid_filters++;
1417                                 goto not_restored;
1418                         }
1419                         if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
1420                                 netif_warn(efx, drv, efx->net_dev,
1421                                            "Warning: unable to restore a filter with RSS context %u as it was not created.\n",
1422                                            spec->rss_context);
1423                                 invalid_filters++;
1424                                 goto not_restored;
1425                         }
1426                 }
1427
1428                 rc = efx_mcdi_filter_push(efx, spec,
1429                                           &table->entry[filter_idx].handle,
1430                                           ctx, false);
1431                 if (rc)
1432                         failed++;
1433
1434                 if (rc) {
1435 not_restored:
1436                         list_for_each_entry(vlan, &table->vlan_list, list)
1437                                 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; ++i)
1438                                         if (vlan->default_filters[i] == filter_idx)
1439                                                 vlan->default_filters[i] =
1440                                                         EFX_EF10_FILTER_ID_INVALID;
1441
1442                         kfree(spec);
1443                         efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0);
1444                 }
1445         }
1446
1447         mutex_unlock(&efx->rss_lock);
1448         up_write(&table->lock);
1449
1450         /*
1451          * This can happen validly if the MC's capabilities have changed, so
1452          * is not an error.
1453          */
1454         if (invalid_filters)
1455                 netif_dbg(efx, drv, efx->net_dev,
1456                           "Did not restore %u filters that are now unsupported.\n",
1457                           invalid_filters);
1458
1459         if (failed)
1460                 netif_err(efx, hw, efx->net_dev,
1461                           "unable to restore %u filters\n", failed);
1462         else
1463                 table->must_restore_filters = false;
1464 }
1465
1466 void efx_mcdi_filter_table_down(struct efx_nic *efx)
1467 {
1468         struct efx_mcdi_filter_table *table = efx->filter_state;
1469         MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
1470         struct efx_filter_spec *spec;
1471         unsigned int filter_idx;
1472         int rc;
1473
1474         if (!table)
1475                 return;
1476
1477         efx_mcdi_filter_cleanup_vlans(efx);
1478
1479         for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1480                 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1481                 if (!spec)
1482                         continue;
1483
1484                 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
1485                                efx_mcdi_filter_is_exclusive(spec) ?
1486                                MC_CMD_FILTER_OP_IN_OP_REMOVE :
1487                                MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
1488                 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
1489                                table->entry[filter_idx].handle);
1490                 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf,
1491                                         sizeof(inbuf), NULL, 0, NULL);
1492                 if (rc)
1493                         netif_info(efx, drv, efx->net_dev,
1494                                    "%s: filter %04x remove failed\n",
1495                                    __func__, filter_idx);
1496                 kfree(spec);
1497         }
1498 }
1499
1500 void efx_mcdi_filter_table_remove(struct efx_nic *efx)
1501 {
1502         struct efx_mcdi_filter_table *table = efx->filter_state;
1503
1504         efx_mcdi_filter_table_down(efx);
1505
1506         efx->filter_state = NULL;
1507         /*
1508          * If we were called without locking, then it's not safe to free
1509          * the table as others might be using it.  So we just WARN, leak
1510          * the memory, and potentially get an inconsistent filter table
1511          * state.
1512          * This should never actually happen.
1513          */
1514         if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1515                 return;
1516
1517         if (!table)
1518                 return;
1519
1520         vfree(table->entry);
1521         kfree(table);
1522 }
1523
1524 static void efx_mcdi_filter_mark_one_old(struct efx_nic *efx, uint16_t *id)
1525 {
1526         struct efx_mcdi_filter_table *table = efx->filter_state;
1527         unsigned int filter_idx;
1528
1529         efx_rwsem_assert_write_locked(&table->lock);
1530
1531         if (*id != EFX_EF10_FILTER_ID_INVALID) {
1532                 filter_idx = efx_mcdi_filter_get_unsafe_id(*id);
1533                 if (!table->entry[filter_idx].spec)
1534                         netif_dbg(efx, drv, efx->net_dev,
1535                                   "marked null spec old %04x:%04x\n", *id,
1536                                   filter_idx);
1537                 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_AUTO_OLD;
1538                 *id = EFX_EF10_FILTER_ID_INVALID;
1539         }
1540 }
1541
1542 /* Mark old per-VLAN filters that may need to be removed */
1543 static void _efx_mcdi_filter_vlan_mark_old(struct efx_nic *efx,
1544                                            struct efx_mcdi_filter_vlan *vlan)
1545 {
1546         struct efx_mcdi_filter_table *table = efx->filter_state;
1547         unsigned int i;
1548
1549         for (i = 0; i < table->dev_uc_count; i++)
1550                 efx_mcdi_filter_mark_one_old(efx, &vlan->uc[i]);
1551         for (i = 0; i < table->dev_mc_count; i++)
1552                 efx_mcdi_filter_mark_one_old(efx, &vlan->mc[i]);
1553         for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
1554                 efx_mcdi_filter_mark_one_old(efx, &vlan->default_filters[i]);
1555 }
1556
1557 /*
1558  * Mark old filters that may need to be removed.
1559  * Caller must hold efx->filter_sem for read if race against
1560  * efx_mcdi_filter_table_remove() is possible
1561  */
1562 static void efx_mcdi_filter_mark_old(struct efx_nic *efx)
1563 {
1564         struct efx_mcdi_filter_table *table = efx->filter_state;
1565         struct efx_mcdi_filter_vlan *vlan;
1566
1567         down_write(&table->lock);
1568         list_for_each_entry(vlan, &table->vlan_list, list)
1569                 _efx_mcdi_filter_vlan_mark_old(efx, vlan);
1570         up_write(&table->lock);
1571 }
1572
1573 int efx_mcdi_filter_add_vlan(struct efx_nic *efx, u16 vid)
1574 {
1575         struct efx_mcdi_filter_table *table = efx->filter_state;
1576         struct efx_mcdi_filter_vlan *vlan;
1577         unsigned int i;
1578
1579         if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1580                 return -EINVAL;
1581
1582         vlan = efx_mcdi_filter_find_vlan(efx, vid);
1583         if (WARN_ON(vlan)) {
1584                 netif_err(efx, drv, efx->net_dev,
1585                           "VLAN %u already added\n", vid);
1586                 return -EALREADY;
1587         }
1588
1589         vlan = kzalloc(sizeof(*vlan), GFP_KERNEL);
1590         if (!vlan)
1591                 return -ENOMEM;
1592
1593         vlan->vid = vid;
1594
1595         for (i = 0; i < ARRAY_SIZE(vlan->uc); i++)
1596                 vlan->uc[i] = EFX_EF10_FILTER_ID_INVALID;
1597         for (i = 0; i < ARRAY_SIZE(vlan->mc); i++)
1598                 vlan->mc[i] = EFX_EF10_FILTER_ID_INVALID;
1599         for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
1600                 vlan->default_filters[i] = EFX_EF10_FILTER_ID_INVALID;
1601
1602         list_add_tail(&vlan->list, &table->vlan_list);
1603
1604         if (efx_dev_registered(efx))
1605                 efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan);
1606
1607         return 0;
1608 }
1609
1610 static void efx_mcdi_filter_del_vlan_internal(struct efx_nic *efx,
1611                                               struct efx_mcdi_filter_vlan *vlan)
1612 {
1613         unsigned int i;
1614
1615         /* See comment in efx_mcdi_filter_table_remove() */
1616         if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1617                 return;
1618
1619         list_del(&vlan->list);
1620
1621         for (i = 0; i < ARRAY_SIZE(vlan->uc); i++)
1622                 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
1623                                               vlan->uc[i]);
1624         for (i = 0; i < ARRAY_SIZE(vlan->mc); i++)
1625                 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
1626                                               vlan->mc[i]);
1627         for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
1628                 if (vlan->default_filters[i] != EFX_EF10_FILTER_ID_INVALID)
1629                         efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
1630                                                       vlan->default_filters[i]);
1631
1632         kfree(vlan);
1633 }
1634
1635 void efx_mcdi_filter_del_vlan(struct efx_nic *efx, u16 vid)
1636 {
1637         struct efx_mcdi_filter_vlan *vlan;
1638
1639         /* See comment in efx_mcdi_filter_table_remove() */
1640         if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1641                 return;
1642
1643         vlan = efx_mcdi_filter_find_vlan(efx, vid);
1644         if (!vlan) {
1645                 netif_err(efx, drv, efx->net_dev,
1646                           "VLAN %u not found in filter state\n", vid);
1647                 return;
1648         }
1649
1650         efx_mcdi_filter_del_vlan_internal(efx, vlan);
1651 }
1652
1653 struct efx_mcdi_filter_vlan *efx_mcdi_filter_find_vlan(struct efx_nic *efx,
1654                                                        u16 vid)
1655 {
1656         struct efx_mcdi_filter_table *table = efx->filter_state;
1657         struct efx_mcdi_filter_vlan *vlan;
1658
1659         WARN_ON(!rwsem_is_locked(&efx->filter_sem));
1660
1661         list_for_each_entry(vlan, &table->vlan_list, list) {
1662                 if (vlan->vid == vid)
1663                         return vlan;
1664         }
1665
1666         return NULL;
1667 }
1668
1669 void efx_mcdi_filter_cleanup_vlans(struct efx_nic *efx)
1670 {
1671         struct efx_mcdi_filter_table *table = efx->filter_state;
1672         struct efx_mcdi_filter_vlan *vlan, *next_vlan;
1673
1674         /* See comment in efx_mcdi_filter_table_remove() */
1675         if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1676                 return;
1677
1678         if (!table)
1679                 return;
1680
1681         list_for_each_entry_safe(vlan, next_vlan, &table->vlan_list, list)
1682                 efx_mcdi_filter_del_vlan_internal(efx, vlan);
1683 }
1684
1685 static void efx_mcdi_filter_uc_addr_list(struct efx_nic *efx)
1686 {
1687         struct efx_mcdi_filter_table *table = efx->filter_state;
1688         struct net_device *net_dev = efx->net_dev;
1689         struct netdev_hw_addr *uc;
1690         unsigned int i;
1691
1692         table->uc_promisc = !!(net_dev->flags & IFF_PROMISC);
1693         ether_addr_copy(table->dev_uc_list[0].addr, net_dev->dev_addr);
1694         i = 1;
1695         netdev_for_each_uc_addr(uc, net_dev) {
1696                 if (i >= EFX_EF10_FILTER_DEV_UC_MAX) {
1697                         table->uc_promisc = true;
1698                         break;
1699                 }
1700                 ether_addr_copy(table->dev_uc_list[i].addr, uc->addr);
1701                 i++;
1702         }
1703
1704         table->dev_uc_count = i;
1705 }
1706
1707 static void efx_mcdi_filter_mc_addr_list(struct efx_nic *efx)
1708 {
1709         struct efx_mcdi_filter_table *table = efx->filter_state;
1710         struct net_device *net_dev = efx->net_dev;
1711         struct netdev_hw_addr *mc;
1712         unsigned int i;
1713
1714         table->mc_overflow = false;
1715         table->mc_promisc = !!(net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI));
1716
1717         i = 0;
1718         netdev_for_each_mc_addr(mc, net_dev) {
1719                 if (i >= EFX_EF10_FILTER_DEV_MC_MAX) {
1720                         table->mc_promisc = true;
1721                         table->mc_overflow = true;
1722                         break;
1723                 }
1724                 ether_addr_copy(table->dev_mc_list[i].addr, mc->addr);
1725                 i++;
1726         }
1727
1728         table->dev_mc_count = i;
1729 }
1730
1731 /*
1732  * Caller must hold efx->filter_sem for read if race against
1733  * efx_mcdi_filter_table_remove() is possible
1734  */
1735 void efx_mcdi_filter_sync_rx_mode(struct efx_nic *efx)
1736 {
1737         struct efx_mcdi_filter_table *table = efx->filter_state;
1738         struct net_device *net_dev = efx->net_dev;
1739         struct efx_mcdi_filter_vlan *vlan;
1740         bool vlan_filter;
1741
1742         if (!efx_dev_registered(efx))
1743                 return;
1744
1745         if (!table)
1746                 return;
1747
1748         efx_mcdi_filter_mark_old(efx);
1749
1750         /*
1751          * Copy/convert the address lists; add the primary station
1752          * address and broadcast address
1753          */
1754         netif_addr_lock_bh(net_dev);
1755         efx_mcdi_filter_uc_addr_list(efx);
1756         efx_mcdi_filter_mc_addr_list(efx);
1757         netif_addr_unlock_bh(net_dev);
1758
1759         /*
1760          * If VLAN filtering changes, all old filters are finally removed.
1761          * Do it in advance to avoid conflicts for unicast untagged and
1762          * VLAN 0 tagged filters.
1763          */
1764         vlan_filter = !!(net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);
1765         if (table->vlan_filter != vlan_filter) {
1766                 table->vlan_filter = vlan_filter;
1767                 efx_mcdi_filter_remove_old(efx);
1768         }
1769
1770         list_for_each_entry(vlan, &table->vlan_list, list)
1771                 efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan);
1772
1773         efx_mcdi_filter_remove_old(efx);
1774         table->mc_promisc_last = table->mc_promisc;
1775 }
1776
1777 #ifdef CONFIG_RFS_ACCEL
1778
1779 bool efx_mcdi_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
1780                                     unsigned int filter_idx)
1781 {
1782         struct efx_filter_spec *spec, saved_spec;
1783         struct efx_mcdi_filter_table *table;
1784         struct efx_arfs_rule *rule = NULL;
1785         bool ret = true, force = false;
1786         u16 arfs_id;
1787
1788         down_read(&efx->filter_sem);
1789         table = efx->filter_state;
1790         down_write(&table->lock);
1791         spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1792
1793         if (!spec || spec->priority != EFX_FILTER_PRI_HINT)
1794                 goto out_unlock;
1795
1796         spin_lock_bh(&efx->rps_hash_lock);
1797         if (!efx->rps_hash_table) {
1798                 /* In the absence of the table, we always return 0 to ARFS. */
1799                 arfs_id = 0;
1800         } else {
1801                 rule = efx_rps_hash_find(efx, spec);
1802                 if (!rule)
1803                         /* ARFS table doesn't know of this filter, so remove it */
1804                         goto expire;
1805                 arfs_id = rule->arfs_id;
1806                 ret = efx_rps_check_rule(rule, filter_idx, &force);
1807                 if (force)
1808                         goto expire;
1809                 if (!ret) {
1810                         spin_unlock_bh(&efx->rps_hash_lock);
1811                         goto out_unlock;
1812                 }
1813         }
1814         if (!rps_may_expire_flow(efx->net_dev, spec->dmaq_id, flow_id, arfs_id))
1815                 ret = false;
1816         else if (rule)
1817                 rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;
1818 expire:
1819         saved_spec = *spec; /* remove operation will kfree spec */
1820         spin_unlock_bh(&efx->rps_hash_lock);
1821         /*
1822          * At this point (since we dropped the lock), another thread might queue
1823          * up a fresh insertion request (but the actual insertion will be held
1824          * up by our possession of the filter table lock).  In that case, it
1825          * will set rule->filter_id to EFX_ARFS_FILTER_ID_PENDING, meaning that
1826          * the rule is not removed by efx_rps_hash_del() below.
1827          */
1828         if (ret)
1829                 ret = efx_mcdi_filter_remove_internal(efx, 1U << spec->priority,
1830                                                       filter_idx, true) == 0;
1831         /*
1832          * While we can't safely dereference rule (we dropped the lock), we can
1833          * still test it for NULL.
1834          */
1835         if (ret && rule) {
1836                 /* Expiring, so remove entry from ARFS table */
1837                 spin_lock_bh(&efx->rps_hash_lock);
1838                 efx_rps_hash_del(efx, &saved_spec);
1839                 spin_unlock_bh(&efx->rps_hash_lock);
1840         }
1841 out_unlock:
1842         up_write(&table->lock);
1843         up_read(&efx->filter_sem);
1844         return ret;
1845 }
1846
1847 #endif /* CONFIG_RFS_ACCEL */
1848
1849 #define RSS_MODE_HASH_ADDRS     (1 << RSS_MODE_HASH_SRC_ADDR_LBN |\
1850                                  1 << RSS_MODE_HASH_DST_ADDR_LBN)
1851 #define RSS_MODE_HASH_PORTS     (1 << RSS_MODE_HASH_SRC_PORT_LBN |\
1852                                  1 << RSS_MODE_HASH_DST_PORT_LBN)
1853 #define RSS_CONTEXT_FLAGS_DEFAULT       (1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_LBN |\
1854                                          1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV4_EN_LBN |\
1855                                          1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV6_EN_LBN |\
1856                                          1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV6_EN_LBN |\
1857                                          (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV4_RSS_MODE_LBN |\
1858                                          RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN |\
1859                                          RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV4_RSS_MODE_LBN |\
1860                                          (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV6_RSS_MODE_LBN |\
1861                                          RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN |\
1862                                          RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV6_RSS_MODE_LBN)
1863
1864 int efx_mcdi_get_rss_context_flags(struct efx_nic *efx, u32 context, u32 *flags)
1865 {
1866         /*
1867          * Firmware had a bug (sfc bug 61952) where it would not actually
1868          * fill in the flags field in the response to MC_CMD_RSS_CONTEXT_GET_FLAGS.
1869          * This meant that it would always contain whatever was previously
1870          * in the MCDI buffer.  Fortunately, all firmware versions with
1871          * this bug have the same default flags value for a newly-allocated
1872          * RSS context, and the only time we want to get the flags is just
1873          * after allocating.  Moreover, the response has a 32-bit hole
1874          * where the context ID would be in the request, so we can use an
1875          * overlength buffer in the request and pre-fill the flags field
1876          * with what we believe the default to be.  Thus if the firmware
1877          * has the bug, it will leave our pre-filled value in the flags
1878          * field of the response, and we will get the right answer.
1879          *
1880          * However, this does mean that this function should NOT be used if
1881          * the RSS context flags might not be their defaults - it is ONLY
1882          * reliably correct for a newly-allocated RSS context.
1883          */
1884         MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN);
1885         MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN);
1886         size_t outlen;
1887         int rc;
1888
1889         /* Check we have a hole for the context ID */
1890         BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_LEN != MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_FLAGS_OFST);
1891         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_IN_RSS_CONTEXT_ID, context);
1892         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS,
1893                        RSS_CONTEXT_FLAGS_DEFAULT);
1894         rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_FLAGS, inbuf,
1895                           sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);
1896         if (rc == 0) {
1897                 if (outlen < MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN)
1898                         rc = -EIO;
1899                 else
1900                         *flags = MCDI_DWORD(outbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS);
1901         }
1902         return rc;
1903 }
1904
1905 /*
1906  * Attempt to enable 4-tuple UDP hashing on the specified RSS context.
1907  * If we fail, we just leave the RSS context at its default hash settings,
1908  * which is safe but may slightly reduce performance.
1909  * Defaults are 4-tuple for TCP and 2-tuple for UDP and other-IP, so we
1910  * just need to set the UDP ports flags (for both IP versions).
1911  */
1912 void efx_mcdi_set_rss_context_flags(struct efx_nic *efx,
1913                                     struct efx_rss_context *ctx)
1914 {
1915         MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN);
1916         u32 flags;
1917
1918         BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN != 0);
1919
1920         if (efx_mcdi_get_rss_context_flags(efx, ctx->context_id, &flags) != 0)
1921                 return;
1922         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
1923                        ctx->context_id);
1924         flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN;
1925         flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN;
1926         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, flags);
1927         if (!efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_FLAGS, inbuf, sizeof(inbuf),
1928                           NULL, 0, NULL))
1929                 /* Succeeded, so UDP 4-tuple is now enabled */
1930                 ctx->rx_hash_udp_4tuple = true;
1931 }
1932
1933 static int efx_mcdi_filter_alloc_rss_context(struct efx_nic *efx, bool exclusive,
1934                                              struct efx_rss_context *ctx,
1935                                              unsigned *context_size)
1936 {
1937         MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN);
1938         MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
1939         size_t outlen;
1940         int rc;
1941         u32 alloc_type = exclusive ?
1942                                 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE :
1943                                 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
1944         unsigned rss_spread = exclusive ?
1945                                 efx->rss_spread :
1946                                 min(rounddown_pow_of_two(efx->rss_spread),
1947                                     EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE);
1948
1949         if (!exclusive && rss_spread == 1) {
1950                 ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
1951                 if (context_size)
1952                         *context_size = 1;
1953                 return 0;
1954         }
1955
1956         if (efx_has_cap(efx, RX_RSS_LIMITED))
1957                 return -EOPNOTSUPP;
1958
1959         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
1960                        efx->vport_id);
1961         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_TYPE, alloc_type);
1962         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, rss_spread);
1963
1964         rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_ALLOC, inbuf, sizeof(inbuf),
1965                 outbuf, sizeof(outbuf), &outlen);
1966         if (rc != 0)
1967                 return rc;
1968
1969         if (outlen < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)
1970                 return -EIO;
1971
1972         ctx->context_id = MCDI_DWORD(outbuf, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
1973
1974         if (context_size)
1975                 *context_size = rss_spread;
1976
1977         if (efx_has_cap(efx, ADDITIONAL_RSS_MODES))
1978                 efx_mcdi_set_rss_context_flags(efx, ctx);
1979
1980         return 0;
1981 }
1982
1983 static int efx_mcdi_filter_free_rss_context(struct efx_nic *efx, u32 context)
1984 {
1985         MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_FREE_IN_LEN);
1986
1987         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID,
1988                        context);
1989         return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_FREE, inbuf, sizeof(inbuf),
1990                             NULL, 0, NULL);
1991 }
1992
1993 static int efx_mcdi_filter_populate_rss_table(struct efx_nic *efx, u32 context,
1994                                        const u32 *rx_indir_table, const u8 *key)
1995 {
1996         MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN);
1997         MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN);
1998         int i, rc;
1999
2000         MCDI_SET_DWORD(tablebuf, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
2001                        context);
2002         BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) !=
2003                      MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN);
2004
2005         /* This iterates over the length of efx->rss_context.rx_indir_table, but
2006          * copies bytes from rx_indir_table.  That's because the latter is a
2007          * pointer rather than an array, but should have the same length.
2008          * The efx->rss_context.rx_hash_key loop below is similar.
2009          */
2010         for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_indir_table); ++i)
2011                 MCDI_PTR(tablebuf,
2012                          RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE)[i] =
2013                                 (u8) rx_indir_table[i];
2014
2015         rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_TABLE, tablebuf,
2016                           sizeof(tablebuf), NULL, 0, NULL);
2017         if (rc != 0)
2018                 return rc;
2019
2020         MCDI_SET_DWORD(keybuf, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
2021                        context);
2022         BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_hash_key) !=
2023                      MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
2024         for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_hash_key); ++i)
2025                 MCDI_PTR(keybuf, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY)[i] = key[i];
2026
2027         return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_KEY, keybuf,
2028                             sizeof(keybuf), NULL, 0, NULL);
2029 }
2030
2031 void efx_mcdi_rx_free_indir_table(struct efx_nic *efx)
2032 {
2033         int rc;
2034
2035         if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID) {
2036                 rc = efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id);
2037                 WARN_ON(rc != 0);
2038         }
2039         efx->rss_context.context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
2040 }
2041
2042 static int efx_mcdi_filter_rx_push_shared_rss_config(struct efx_nic *efx,
2043                                               unsigned *context_size)
2044 {
2045         struct efx_mcdi_filter_table *table = efx->filter_state;
2046         int rc = efx_mcdi_filter_alloc_rss_context(efx, false, &efx->rss_context,
2047                                             context_size);
2048
2049         if (rc != 0)
2050                 return rc;
2051
2052         table->rx_rss_context_exclusive = false;
2053         efx_set_default_rx_indir_table(efx, &efx->rss_context);
2054         return 0;
2055 }
2056
2057 static int efx_mcdi_filter_rx_push_exclusive_rss_config(struct efx_nic *efx,
2058                                                  const u32 *rx_indir_table,
2059                                                  const u8 *key)
2060 {
2061         struct efx_mcdi_filter_table *table = efx->filter_state;
2062         u32 old_rx_rss_context = efx->rss_context.context_id;
2063         int rc;
2064
2065         if (efx->rss_context.context_id == EFX_MCDI_RSS_CONTEXT_INVALID ||
2066             !table->rx_rss_context_exclusive) {
2067                 rc = efx_mcdi_filter_alloc_rss_context(efx, true, &efx->rss_context,
2068                                                 NULL);
2069                 if (rc == -EOPNOTSUPP)
2070                         return rc;
2071                 else if (rc != 0)
2072                         goto fail1;
2073         }
2074
2075         rc = efx_mcdi_filter_populate_rss_table(efx, efx->rss_context.context_id,
2076                                          rx_indir_table, key);
2077         if (rc != 0)
2078                 goto fail2;
2079
2080         if (efx->rss_context.context_id != old_rx_rss_context &&
2081             old_rx_rss_context != EFX_MCDI_RSS_CONTEXT_INVALID)
2082                 WARN_ON(efx_mcdi_filter_free_rss_context(efx, old_rx_rss_context) != 0);
2083         table->rx_rss_context_exclusive = true;
2084         if (rx_indir_table != efx->rss_context.rx_indir_table)
2085                 memcpy(efx->rss_context.rx_indir_table, rx_indir_table,
2086                        sizeof(efx->rss_context.rx_indir_table));
2087         if (key != efx->rss_context.rx_hash_key)
2088                 memcpy(efx->rss_context.rx_hash_key, key,
2089                        efx->type->rx_hash_key_size);
2090
2091         return 0;
2092
2093 fail2:
2094         if (old_rx_rss_context != efx->rss_context.context_id) {
2095                 WARN_ON(efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id) != 0);
2096                 efx->rss_context.context_id = old_rx_rss_context;
2097         }
2098 fail1:
2099         netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
2100         return rc;
2101 }
2102
2103 int efx_mcdi_rx_push_rss_context_config(struct efx_nic *efx,
2104                                         struct efx_rss_context *ctx,
2105                                         const u32 *rx_indir_table,
2106                                         const u8 *key)
2107 {
2108         int rc;
2109
2110         WARN_ON(!mutex_is_locked(&efx->rss_lock));
2111
2112         if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
2113                 rc = efx_mcdi_filter_alloc_rss_context(efx, true, ctx, NULL);
2114                 if (rc)
2115                         return rc;
2116         }
2117
2118         if (!rx_indir_table) /* Delete this context */
2119                 return efx_mcdi_filter_free_rss_context(efx, ctx->context_id);
2120
2121         rc = efx_mcdi_filter_populate_rss_table(efx, ctx->context_id,
2122                                          rx_indir_table, key);
2123         if (rc)
2124                 return rc;
2125
2126         memcpy(ctx->rx_indir_table, rx_indir_table,
2127                sizeof(efx->rss_context.rx_indir_table));
2128         memcpy(ctx->rx_hash_key, key, efx->type->rx_hash_key_size);
2129
2130         return 0;
2131 }
2132
2133 int efx_mcdi_rx_pull_rss_context_config(struct efx_nic *efx,
2134                                         struct efx_rss_context *ctx)
2135 {
2136         MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN);
2137         MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN);
2138         MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN);
2139         size_t outlen;
2140         int rc, i;
2141
2142         WARN_ON(!mutex_is_locked(&efx->rss_lock));
2143
2144         BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN !=
2145                      MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN);
2146
2147         if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID)
2148                 return -ENOENT;
2149
2150         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_TABLE_IN_RSS_CONTEXT_ID,
2151                        ctx->context_id);
2152         BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_indir_table) !=
2153                      MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE_LEN);
2154         rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_TABLE, inbuf, sizeof(inbuf),
2155                           tablebuf, sizeof(tablebuf), &outlen);
2156         if (rc != 0)
2157                 return rc;
2158
2159         if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN))
2160                 return -EIO;
2161
2162         for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++)
2163                 ctx->rx_indir_table[i] = MCDI_PTR(tablebuf,
2164                                 RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE)[i];
2165
2166         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_KEY_IN_RSS_CONTEXT_ID,
2167                        ctx->context_id);
2168         BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_hash_key) !=
2169                      MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
2170         rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_KEY, inbuf, sizeof(inbuf),
2171                           keybuf, sizeof(keybuf), &outlen);
2172         if (rc != 0)
2173                 return rc;
2174
2175         if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN))
2176                 return -EIO;
2177
2178         for (i = 0; i < ARRAY_SIZE(ctx->rx_hash_key); ++i)
2179                 ctx->rx_hash_key[i] = MCDI_PTR(
2180                                 keybuf, RSS_CONTEXT_GET_KEY_OUT_TOEPLITZ_KEY)[i];
2181
2182         return 0;
2183 }
2184
2185 int efx_mcdi_rx_pull_rss_config(struct efx_nic *efx)
2186 {
2187         int rc;
2188
2189         mutex_lock(&efx->rss_lock);
2190         rc = efx_mcdi_rx_pull_rss_context_config(efx, &efx->rss_context);
2191         mutex_unlock(&efx->rss_lock);
2192         return rc;
2193 }
2194
2195 void efx_mcdi_rx_restore_rss_contexts(struct efx_nic *efx)
2196 {
2197         struct efx_mcdi_filter_table *table = efx->filter_state;
2198         struct efx_rss_context *ctx;
2199         int rc;
2200
2201         WARN_ON(!mutex_is_locked(&efx->rss_lock));
2202
2203         if (!table->must_restore_rss_contexts)
2204                 return;
2205
2206         list_for_each_entry(ctx, &efx->rss_context.list, list) {
2207                 /* previous NIC RSS context is gone */
2208                 ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
2209                 /* so try to allocate a new one */
2210                 rc = efx_mcdi_rx_push_rss_context_config(efx, ctx,
2211                                                          ctx->rx_indir_table,
2212                                                          ctx->rx_hash_key);
2213                 if (rc)
2214                         netif_warn(efx, probe, efx->net_dev,
2215                                    "failed to restore RSS context %u, rc=%d"
2216                                    "; RSS filters may fail to be applied\n",
2217                                    ctx->user_id, rc);
2218         }
2219         table->must_restore_rss_contexts = false;
2220 }
2221
2222 int efx_mcdi_pf_rx_push_rss_config(struct efx_nic *efx, bool user,
2223                                    const u32 *rx_indir_table,
2224                                    const u8 *key)
2225 {
2226         int rc;
2227
2228         if (efx->rss_spread == 1)
2229                 return 0;
2230
2231         if (!key)
2232                 key = efx->rss_context.rx_hash_key;
2233
2234         rc = efx_mcdi_filter_rx_push_exclusive_rss_config(efx, rx_indir_table, key);
2235
2236         if (rc == -ENOBUFS && !user) {
2237                 unsigned context_size;
2238                 bool mismatch = false;
2239                 size_t i;
2240
2241                 for (i = 0;
2242                      i < ARRAY_SIZE(efx->rss_context.rx_indir_table) && !mismatch;
2243                      i++)
2244                         mismatch = rx_indir_table[i] !=
2245                                 ethtool_rxfh_indir_default(i, efx->rss_spread);
2246
2247                 rc = efx_mcdi_filter_rx_push_shared_rss_config(efx, &context_size);
2248                 if (rc == 0) {
2249                         if (context_size != efx->rss_spread)
2250                                 netif_warn(efx, probe, efx->net_dev,
2251                                            "Could not allocate an exclusive RSS"
2252                                            " context; allocated a shared one of"
2253                                            " different size."
2254                                            " Wanted %u, got %u.\n",
2255                                            efx->rss_spread, context_size);
2256                         else if (mismatch)
2257                                 netif_warn(efx, probe, efx->net_dev,
2258                                            "Could not allocate an exclusive RSS"
2259                                            " context; allocated a shared one but"
2260                                            " could not apply custom"
2261                                            " indirection.\n");
2262                         else
2263                                 netif_info(efx, probe, efx->net_dev,
2264                                            "Could not allocate an exclusive RSS"
2265                                            " context; allocated a shared one.\n");
2266                 }
2267         }
2268         return rc;
2269 }
2270
2271 int efx_mcdi_vf_rx_push_rss_config(struct efx_nic *efx, bool user,
2272                                    const u32 *rx_indir_table
2273                                    __attribute__ ((unused)),
2274                                    const u8 *key
2275                                    __attribute__ ((unused)))
2276 {
2277         if (user)
2278                 return -EOPNOTSUPP;
2279         if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID)
2280                 return 0;
2281         return efx_mcdi_filter_rx_push_shared_rss_config(efx, NULL);
2282 }
2283
2284 int efx_mcdi_push_default_indir_table(struct efx_nic *efx,
2285                                       unsigned int rss_spread)
2286 {
2287         int rc = 0;
2288
2289         if (efx->rss_spread == rss_spread)
2290                 return 0;
2291
2292         efx->rss_spread = rss_spread;
2293         if (!efx->filter_state)
2294                 return 0;
2295
2296         efx_mcdi_rx_free_indir_table(efx);
2297         if (rss_spread > 1) {
2298                 efx_set_default_rx_indir_table(efx, &efx->rss_context);
2299                 rc = efx->type->rx_push_rss_config(efx, false,
2300                                    efx->rss_context.rx_indir_table, NULL);
2301         }
2302         return rc;
2303 }