projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
mlxsw: spectrum_router: Extract mlxsw_sp_fi_is_gateway()
[linux-2.6-block.git] / drivers / net / ethernet / mellanox / mlxsw / spectrum_router.c
CommitLineData
464dce18
IS		 1/*
2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
6ddb7426 3 * Copyright (c) 2016-2017 Mellanox Technologies. All rights reserved.
464dce18
IS		 4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
c723c735 6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
6ddb7426 7 * Copyright (c) 2017 Petr Machata <petrm@mellanox.com>
464dce18
IS		 8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions are met:
11 *
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. Neither the names of the copyright holders nor the names of its
18 * contributors may be used to endorse or promote products derived from
19 * this software without specific prior written permission.
20 *
21 * Alternatively, this software may be distributed under the terms of the
22 * GNU General Public License ("GPL") version 2 as published by the Free
23 * Software Foundation.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
26 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
29 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37
38#include <linux/kernel.h>
39#include <linux/types.h>
5e9c16cc
JP		 40#include <linux/rhashtable.h>
41#include <linux/bitops.h>
42#include <linux/in6.h>
c723c735 43#include <linux/notifier.h>
df6dd79b 44#include <linux/inetdevice.h>
9db032bb 45#include <linux/netdevice.h>
03ea01e9 46#include <linux/if_bridge.h>
b5f3e0d4 47#include <linux/socket.h>
428b851f 48#include <linux/route.h>
c723c735 49#include <net/netevent.h>
6cf3c971
JP		 50#include <net/neighbour.h>
51#include <net/arp.h>
b45f64d1 52#include <net/ip_fib.h>
583419fd 53#include <net/ip6_fib.h>
5d7bfd14 54#include <net/fib_rules.h>
6ddb7426 55#include <net/ip_tunnels.h>
57837885 56#include <net/l3mdev.h>
5ea1237f 57#include <net/addrconf.h>
d5eb89cf
AS		 58#include <net/ndisc.h>
59#include <net/ipv6.h>
04b1d4e5 60#include <net/fib_notifier.h>
464dce18
IS		 61
62#include "spectrum.h"
63#include "core.h"
64#include "reg.h"
e0c0afd8
AS		 65#include "spectrum_cnt.h"
66#include "spectrum_dpipe.h"
38ebc0f4 67#include "spectrum_ipip.h"
e0c0afd8 68#include "spectrum_router.h"
464dce18 69
9011b677
IS		 70struct mlxsw_sp_vr;
71struct mlxsw_sp_lpm_tree;
e4f3c1c1 72struct mlxsw_sp_rif_ops;
9011b677
IS		 73
74struct mlxsw_sp_router {
75 struct mlxsw_sp *mlxsw_sp;
5f9efffb 76 struct mlxsw_sp_rif **rifs;
9011b677
IS		 77 struct mlxsw_sp_vr *vrs;
78 struct rhashtable neigh_ht;
79 struct rhashtable nexthop_group_ht;
80 struct rhashtable nexthop_ht;
81 struct {
82 struct mlxsw_sp_lpm_tree *trees;
83 unsigned int tree_count;
84 } lpm;
85 struct {
86 struct delayed_work dw;
87 unsigned long interval; /* ms */
88 } neighs_update;
89 struct delayed_work nexthop_probe_dw;
90#define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
91 struct list_head nexthop_neighs_list;
92 bool aborted;
7e39d115 93 struct notifier_block fib_nb;
e4f3c1c1 94 const struct mlxsw_sp_rif_ops **rif_ops_arr;
38ebc0f4 95 const struct mlxsw_sp_ipip_ops **ipip_ops_arr;
9011b677
IS		 96};
97
4724ba56
IS		 98struct mlxsw_sp_rif {
99 struct list_head nexthop_list;
100 struct list_head neigh_list;
101 struct net_device *dev;
a1107487 102 struct mlxsw_sp_fid *fid;
4724ba56
IS		 103 unsigned char addr[ETH_ALEN];
104 int mtu;
bf95233e 105 u16 rif_index;
6913229e 106 u16 vr_id;
e4f3c1c1
IS		 107 const struct mlxsw_sp_rif_ops *ops;
108 struct mlxsw_sp *mlxsw_sp;
109
e0c0afd8
AS		 110 unsigned int counter_ingress;
111 bool counter_ingress_valid;
112 unsigned int counter_egress;
113 bool counter_egress_valid;
4724ba56
IS		 114};
115
e4f3c1c1
IS		 116struct mlxsw_sp_rif_params {
117 struct net_device *dev;
118 union {
119 u16 system_port;
120 u16 lag_id;
121 };
122 u16 vid;
123 bool lag;
124};
125
4d93ceeb
IS		 126struct mlxsw_sp_rif_subport {
127 struct mlxsw_sp_rif common;
128 union {
129 u16 system_port;
130 u16 lag_id;
131 };
132 u16 vid;
133 bool lag;
134};
135
6ddb7426
PM		 136struct mlxsw_sp_rif_ipip_lb {
137 struct mlxsw_sp_rif common;
138 struct mlxsw_sp_rif_ipip_lb_config lb_config;
139 u16 ul_vr_id; /* Reserved for Spectrum-2. */
140};
141
142struct mlxsw_sp_rif_params_ipip_lb {
143 struct mlxsw_sp_rif_params common;
144 struct mlxsw_sp_rif_ipip_lb_config lb_config;
145};
146
e4f3c1c1
IS		 147struct mlxsw_sp_rif_ops {
148 enum mlxsw_sp_rif_type type;
149 size_t rif_size;
150
151 void (*setup)(struct mlxsw_sp_rif *rif,
152 const struct mlxsw_sp_rif_params *params);
153 int (*configure)(struct mlxsw_sp_rif *rif);
154 void (*deconfigure)(struct mlxsw_sp_rif *rif);
155 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
156};
157
e0c0afd8
AS		 158static unsigned int *
159mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
160 enum mlxsw_sp_rif_counter_dir dir)
161{
162 switch (dir) {
163 case MLXSW_SP_RIF_COUNTER_EGRESS:
164 return &rif->counter_egress;
165 case MLXSW_SP_RIF_COUNTER_INGRESS:
166 return &rif->counter_ingress;
167 }
168 return NULL;
169}
170
171static bool
172mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
173 enum mlxsw_sp_rif_counter_dir dir)
174{
175 switch (dir) {
176 case MLXSW_SP_RIF_COUNTER_EGRESS:
177 return rif->counter_egress_valid;
178 case MLXSW_SP_RIF_COUNTER_INGRESS:
179 return rif->counter_ingress_valid;
180 }
181 return false;
182}
183
184static void
185mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
186 enum mlxsw_sp_rif_counter_dir dir,
187 bool valid)
188{
189 switch (dir) {
190 case MLXSW_SP_RIF_COUNTER_EGRESS:
191 rif->counter_egress_valid = valid;
192 break;
193 case MLXSW_SP_RIF_COUNTER_INGRESS:
194 rif->counter_ingress_valid = valid;
195 break;
196 }
197}
198
199static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
200 unsigned int counter_index, bool enable,
201 enum mlxsw_sp_rif_counter_dir dir)
202{
203 char ritr_pl[MLXSW_REG_RITR_LEN];
204 bool is_egress = false;
205 int err;
206
207 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
208 is_egress = true;
209 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
210 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
211 if (err)
212 return err;
213
214 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
215 is_egress);
216 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
217}
218
219int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
220 struct mlxsw_sp_rif *rif,
221 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
222{
223 char ricnt_pl[MLXSW_REG_RICNT_LEN];
224 unsigned int *p_counter_index;
225 bool valid;
226 int err;
227
228 valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
229 if (!valid)
230 return -EINVAL;
231
232 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
233 if (!p_counter_index)
234 return -EINVAL;
235 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
236 MLXSW_REG_RICNT_OPCODE_NOP);
237 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
238 if (err)
239 return err;
240 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
241 return 0;
242}
243
244static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
245 unsigned int counter_index)
246{
247 char ricnt_pl[MLXSW_REG_RICNT_LEN];
248
249 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
250 MLXSW_REG_RICNT_OPCODE_CLEAR);
251 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
252}
253
254int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
255 struct mlxsw_sp_rif *rif,
256 enum mlxsw_sp_rif_counter_dir dir)
257{
258 unsigned int *p_counter_index;
259 int err;
260
261 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
262 if (!p_counter_index)
263 return -EINVAL;
264 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
265 p_counter_index);
266 if (err)
267 return err;
268
269 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
270 if (err)
271 goto err_counter_clear;
272
273 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
274 *p_counter_index, true, dir);
275 if (err)
276 goto err_counter_edit;
277 mlxsw_sp_rif_counter_valid_set(rif, dir, true);
278 return 0;
279
280err_counter_edit:
281err_counter_clear:
282 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
283 *p_counter_index);
284 return err;
285}
286
287void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
288 struct mlxsw_sp_rif *rif,
289 enum mlxsw_sp_rif_counter_dir dir)
290{
291 unsigned int *p_counter_index;
292
6b1206bb
AS		 293 if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
294 return;
295
e0c0afd8
AS		 296 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
297 if (WARN_ON(!p_counter_index))
298 return;
299 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
300 *p_counter_index, false, dir);
301 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
302 *p_counter_index);
303 mlxsw_sp_rif_counter_valid_set(rif, dir, false);
304}
305
e4f3c1c1
IS		 306static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
307{
308 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
309 struct devlink *devlink;
310
311 devlink = priv_to_devlink(mlxsw_sp->core);
312 if (!devlink_dpipe_table_counter_enabled(devlink,
313 MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
314 return;
315 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
316}
317
318static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
319{
320 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
321
322 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
323}
324
4724ba56
IS		 325static struct mlxsw_sp_rif *
326mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
327 const struct net_device *dev);
328
7dcc18ad 329#define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
9011b677
IS		 330
331struct mlxsw_sp_prefix_usage {
332 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
333};
334
53342023
JP		 335#define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
336 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
337
338static bool
339mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
340 struct mlxsw_sp_prefix_usage *prefix_usage2)
341{
342 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
343}
344
6b75c480
JP		 345static bool
346mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
347{
348 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
349
350 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
351}
352
353static void
354mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
355 struct mlxsw_sp_prefix_usage *prefix_usage2)
356{
357 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
358}
359
5e9c16cc
JP		 360static void
361mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
362 unsigned char prefix_len)
363{
364 set_bit(prefix_len, prefix_usage->b);
365}
366
367static void
368mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
369 unsigned char prefix_len)
370{
371 clear_bit(prefix_len, prefix_usage->b);
372}
373
374struct mlxsw_sp_fib_key {
375 unsigned char addr[sizeof(struct in6_addr)];
376 unsigned char prefix_len;
377};
378
61c503f9
JP		 379enum mlxsw_sp_fib_entry_type {
380 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
381 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
382 MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
383};
384
a7ff87ac 385struct mlxsw_sp_nexthop_group;
9011b677 386struct mlxsw_sp_fib;
a7ff87ac 387
9aecce1c
IS		 388struct mlxsw_sp_fib_node {
389 struct list_head entry_list;
b45f64d1 390 struct list_head list;
9aecce1c 391 struct rhash_head ht_node;
76610ebb 392 struct mlxsw_sp_fib *fib;
5e9c16cc 393 struct mlxsw_sp_fib_key key;
9aecce1c
IS		 394};
395
9aecce1c
IS		 396struct mlxsw_sp_fib_entry {
397 struct list_head list;
398 struct mlxsw_sp_fib_node *fib_node;
61c503f9 399 enum mlxsw_sp_fib_entry_type type;
a7ff87ac
JP		 400 struct list_head nexthop_group_node;
401 struct mlxsw_sp_nexthop_group *nh_group;
5e9c16cc
JP		 402};
403
4f1c7f1f
IS		 404struct mlxsw_sp_fib4_entry {
405 struct mlxsw_sp_fib_entry common;
406 u32 tb_id;
407 u32 prio;
408 u8 tos;
409 u8 type;
410};
411
428b851f
IS		 412struct mlxsw_sp_fib6_entry {
413 struct mlxsw_sp_fib_entry common;
414 struct list_head rt6_list;
415 unsigned int nrt6;
416};
417
418struct mlxsw_sp_rt6 {
419 struct list_head list;
420 struct rt6_info *rt;
421};
422
9011b677
IS		 423struct mlxsw_sp_lpm_tree {
424 u8 id; /* tree ID */
425 unsigned int ref_count;
426 enum mlxsw_sp_l3proto proto;
427 struct mlxsw_sp_prefix_usage prefix_usage;
428};
429
5e9c16cc
JP		 430struct mlxsw_sp_fib {
431 struct rhashtable ht;
9aecce1c 432 struct list_head node_list;
76610ebb
IS		 433 struct mlxsw_sp_vr *vr;
434 struct mlxsw_sp_lpm_tree *lpm_tree;
5e9c16cc
JP		 435 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
436 struct mlxsw_sp_prefix_usage prefix_usage;
76610ebb 437 enum mlxsw_sp_l3proto proto;
5e9c16cc
JP		 438};
439
9011b677
IS		 440struct mlxsw_sp_vr {
441 u16 id; /* virtual router ID */
442 u32 tb_id; /* kernel fib table id */
443 unsigned int rif_count;
444 struct mlxsw_sp_fib *fib4;
a3d9bc50 445 struct mlxsw_sp_fib *fib6;
9011b677
IS		 446};
447
9aecce1c 448static const struct rhashtable_params mlxsw_sp_fib_ht_params;
5e9c16cc 449
76610ebb
IS		 450static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
451 enum mlxsw_sp_l3proto proto)
5e9c16cc
JP		 452{
453 struct mlxsw_sp_fib *fib;
454 int err;
455
456 fib = kzalloc(sizeof(*fib), GFP_KERNEL);
457 if (!fib)
458 return ERR_PTR(-ENOMEM);
459 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
460 if (err)
461 goto err_rhashtable_init;
9aecce1c 462 INIT_LIST_HEAD(&fib->node_list);
76610ebb
IS		 463 fib->proto = proto;
464 fib->vr = vr;
5e9c16cc
JP		 465 return fib;
466
467err_rhashtable_init:
468 kfree(fib);
469 return ERR_PTR(err);
470}
471
472static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
473{
9aecce1c 474 WARN_ON(!list_empty(&fib->node_list));
76610ebb 475 WARN_ON(fib->lpm_tree);
5e9c16cc
JP		 476 rhashtable_destroy(&fib->ht);
477 kfree(fib);
478}
479
53342023 480static struct mlxsw_sp_lpm_tree *
382dbb40 481mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
53342023
JP		 482{
483 static struct mlxsw_sp_lpm_tree *lpm_tree;
484 int i;
485
9011b677
IS		 486 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
487 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
382dbb40
IS		 488 if (lpm_tree->ref_count == 0)
489 return lpm_tree;
53342023
JP		 490 }
491 return NULL;
492}
493
494static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
495 struct mlxsw_sp_lpm_tree *lpm_tree)
496{
497 char ralta_pl[MLXSW_REG_RALTA_LEN];
498
1a9234e6
IS		 499 mlxsw_reg_ralta_pack(ralta_pl, true,
500 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
501 lpm_tree->id);
53342023
JP		 502 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
503}
504
cc702670
IS		 505static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
506 struct mlxsw_sp_lpm_tree *lpm_tree)
53342023
JP		 507{
508 char ralta_pl[MLXSW_REG_RALTA_LEN];
509
1a9234e6
IS		 510 mlxsw_reg_ralta_pack(ralta_pl, false,
511 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
512 lpm_tree->id);
cc702670 513 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
53342023
JP		 514}
515
516static int
517mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
518 struct mlxsw_sp_prefix_usage *prefix_usage,
519 struct mlxsw_sp_lpm_tree *lpm_tree)
520{
521 char ralst_pl[MLXSW_REG_RALST_LEN];
522 u8 root_bin = 0;
523 u8 prefix;
524 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
525
526 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
527 root_bin = prefix;
528
529 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
530 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
531 if (prefix == 0)
532 continue;
533 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
534 MLXSW_REG_RALST_BIN_NO_CHILD);
535 last_prefix = prefix;
536 }
537 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
538}
539
540static struct mlxsw_sp_lpm_tree *
541mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
542 struct mlxsw_sp_prefix_usage *prefix_usage,
382dbb40 543 enum mlxsw_sp_l3proto proto)
53342023
JP		 544{
545 struct mlxsw_sp_lpm_tree *lpm_tree;
546 int err;
547
382dbb40 548 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
53342023
JP		 549 if (!lpm_tree)
550 return ERR_PTR(-EBUSY);
551 lpm_tree->proto = proto;
552 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
553 if (err)
554 return ERR_PTR(err);
555
556 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
557 lpm_tree);
558 if (err)
559 goto err_left_struct_set;
2083d367
JP		 560 memcpy(&lpm_tree->prefix_usage, prefix_usage,
561 sizeof(lpm_tree->prefix_usage));
53342023
JP		 562 return lpm_tree;
563
564err_left_struct_set:
565 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
566 return ERR_PTR(err);
567}
568
cc702670
IS		 569static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
570 struct mlxsw_sp_lpm_tree *lpm_tree)
53342023 571{
cc702670 572 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
53342023
JP		 573}
574
575static struct mlxsw_sp_lpm_tree *
576mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
577 struct mlxsw_sp_prefix_usage *prefix_usage,
382dbb40 578 enum mlxsw_sp_l3proto proto)
53342023
JP		 579{
580 struct mlxsw_sp_lpm_tree *lpm_tree;
581 int i;
582
9011b677
IS		 583 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
584 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
8b99becd
JP		 585 if (lpm_tree->ref_count != 0 &&
586 lpm_tree->proto == proto &&
53342023
JP		 587 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
588 prefix_usage))
fc922bb0 589 return lpm_tree;
53342023 590 }
fc922bb0
IS		 591 return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
592}
53342023 593
fc922bb0
IS		 594static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
595{
53342023 596 lpm_tree->ref_count++;
53342023
JP		 597}
598
cc702670
IS		 599static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
600 struct mlxsw_sp_lpm_tree *lpm_tree)
53342023
JP		 601{
602 if (--lpm_tree->ref_count == 0)
cc702670 603 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
53342023
JP		 604}
605
d7a60306 606#define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
8494ab06
IS		 607
608static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
53342023
JP		 609{
610 struct mlxsw_sp_lpm_tree *lpm_tree;
8494ab06 611 u64 max_trees;
53342023
JP		 612 int i;
613
8494ab06
IS		 614 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
615 return -EIO;
616
617 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
9011b677
IS		 618 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
619 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
8494ab06
IS		 620 sizeof(struct mlxsw_sp_lpm_tree),
621 GFP_KERNEL);
9011b677 622 if (!mlxsw_sp->router->lpm.trees)
8494ab06
IS		 623 return -ENOMEM;
624
9011b677
IS		 625 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
626 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
53342023
JP		 627 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
628 }
8494ab06
IS		 629
630 return 0;
631}
632
633static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
634{
9011b677 635 kfree(mlxsw_sp->router->lpm.trees);
53342023
JP		 636}
637
76610ebb
IS		 638static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
639{
a3d9bc50 640 return !!vr->fib4 || !!vr->fib6;
76610ebb
IS		 641}
642
6b75c480
JP		 643static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
644{
645 struct mlxsw_sp_vr *vr;
646 int i;
647
c1a38311 648 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
9011b677 649 vr = &mlxsw_sp->router->vrs[i];
76610ebb 650 if (!mlxsw_sp_vr_is_used(vr))
6b75c480
JP		 651 return vr;
652 }
653 return NULL;
654}
655
656static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
0adb214b 657 const struct mlxsw_sp_fib *fib, u8 tree_id)
6b75c480
JP		 658{
659 char raltb_pl[MLXSW_REG_RALTB_LEN];
660
76610ebb
IS		 661 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
662 (enum mlxsw_reg_ralxx_protocol) fib->proto,
0adb214b 663 tree_id);
6b75c480
JP		 664 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
665}
666
667static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
76610ebb 668 const struct mlxsw_sp_fib *fib)
6b75c480
JP		 669{
670 char raltb_pl[MLXSW_REG_RALTB_LEN];
671
672 /* Bind to tree 0 which is default */
76610ebb
IS		 673 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
674 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
6b75c480
JP		 675 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
676}
677
678static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
679{
680 /* For our purpose, squash main and local table into one */
681 if (tb_id == RT_TABLE_LOCAL)
682 tb_id = RT_TABLE_MAIN;
683 return tb_id;
684}
685
686static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
76610ebb 687 u32 tb_id)
6b75c480
JP		 688{
689 struct mlxsw_sp_vr *vr;
690 int i;
691
692 tb_id = mlxsw_sp_fix_tb_id(tb_id);
9497c042 693
c1a38311 694 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
9011b677 695 vr = &mlxsw_sp->router->vrs[i];
76610ebb 696 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
6b75c480
JP		 697 return vr;
698 }
699 return NULL;
700}
701
76610ebb
IS		 702static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
703 enum mlxsw_sp_l3proto proto)
704{
705 switch (proto) {
706 case MLXSW_SP_L3_PROTO_IPV4:
707 return vr->fib4;
708 case MLXSW_SP_L3_PROTO_IPV6:
a3d9bc50 709 return vr->fib6;
76610ebb
IS		 710 }
711 return NULL;
712}
713
6b75c480 714static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
76610ebb 715 u32 tb_id)
6b75c480 716{
6b75c480 717 struct mlxsw_sp_vr *vr;
a3d9bc50 718 int err;
6b75c480
JP		 719
720 vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
721 if (!vr)
722 return ERR_PTR(-EBUSY);
76610ebb
IS		 723 vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
724 if (IS_ERR(vr->fib4))
725 return ERR_CAST(vr->fib4);
a3d9bc50
IS		 726 vr->fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6);
727 if (IS_ERR(vr->fib6)) {
728 err = PTR_ERR(vr->fib6);
729 goto err_fib6_create;
730 }
6b75c480 731 vr->tb_id = tb_id;
6b75c480 732 return vr;
a3d9bc50
IS		 733
734err_fib6_create:
735 mlxsw_sp_fib_destroy(vr->fib4);
736 vr->fib4 = NULL;
737 return ERR_PTR(err);
6b75c480
JP		 738}
739
76610ebb 740static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
6b75c480 741{
a3d9bc50
IS		 742 mlxsw_sp_fib_destroy(vr->fib6);
743 vr->fib6 = NULL;
76610ebb
IS		 744 mlxsw_sp_fib_destroy(vr->fib4);
745 vr->fib4 = NULL;
6b75c480
JP		 746}
747
76610ebb 748static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
6b75c480
JP		 749{
750 struct mlxsw_sp_vr *vr;
6b75c480
JP		 751
752 tb_id = mlxsw_sp_fix_tb_id(tb_id);
76610ebb
IS		 753 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
754 if (!vr)
755 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
6b75c480
JP		 756 return vr;
757}
758
76610ebb 759static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
6b75c480 760{
a3d9bc50
IS		 761 if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
762 list_empty(&vr->fib6->node_list))
76610ebb 763 mlxsw_sp_vr_destroy(vr);
6b75c480
JP		 764}
765
fc922bb0
IS		 766static bool
767mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
768 enum mlxsw_sp_l3proto proto, u8 tree_id)
769{
770 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
771
772 if (!mlxsw_sp_vr_is_used(vr))
773 return false;
774 if (fib->lpm_tree && fib->lpm_tree->id == tree_id)
775 return true;
776 return false;
777}
778
779static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
780 struct mlxsw_sp_fib *fib,
781 struct mlxsw_sp_lpm_tree *new_tree)
782{
783 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
784 int err;
785
786 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
787 if (err)
788 return err;
789 fib->lpm_tree = new_tree;
790 mlxsw_sp_lpm_tree_hold(new_tree);
791 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
792 return 0;
793}
794
795static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
796 struct mlxsw_sp_fib *fib,
797 struct mlxsw_sp_lpm_tree *new_tree)
798{
799 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
800 enum mlxsw_sp_l3proto proto = fib->proto;
801 u8 old_id, new_id = new_tree->id;
802 struct mlxsw_sp_vr *vr;
803 int i, err;
804
805 if (!old_tree)
806 goto no_replace;
807 old_id = old_tree->id;
808
809 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
810 vr = &mlxsw_sp->router->vrs[i];
811 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
812 continue;
813 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
814 mlxsw_sp_vr_fib(vr, proto),
815 new_tree);
816 if (err)
817 goto err_tree_replace;
818 }
819
820 return 0;
821
822err_tree_replace:
823 for (i--; i >= 0; i--) {
824 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
825 continue;
826 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
827 mlxsw_sp_vr_fib(vr, proto),
828 old_tree);
829 }
830 return err;
831
832no_replace:
833 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
834 if (err)
835 return err;
836 fib->lpm_tree = new_tree;
837 mlxsw_sp_lpm_tree_hold(new_tree);
838 return 0;
839}
840
841static void
842mlxsw_sp_vrs_prefixes(struct mlxsw_sp *mlxsw_sp,
843 enum mlxsw_sp_l3proto proto,
844 struct mlxsw_sp_prefix_usage *req_prefix_usage)
845{
846 int i;
847
848 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
849 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
850 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
851 unsigned char prefix;
852
853 if (!mlxsw_sp_vr_is_used(vr))
854 continue;
855 mlxsw_sp_prefix_usage_for_each(prefix, &fib->prefix_usage)
856 mlxsw_sp_prefix_usage_set(req_prefix_usage, prefix);
857 }
858}
859
9497c042 860static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
6b75c480
JP		 861{
862 struct mlxsw_sp_vr *vr;
c1a38311 863 u64 max_vrs;
6b75c480
JP		 864 int i;
865
c1a38311 866 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
9497c042
NF		 867 return -EIO;
868
c1a38311 869 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
9011b677
IS		 870 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
871 GFP_KERNEL);
872 if (!mlxsw_sp->router->vrs)
9497c042
NF		 873 return -ENOMEM;
874
c1a38311 875 for (i = 0; i < max_vrs; i++) {
9011b677 876 vr = &mlxsw_sp->router->vrs[i];
6b75c480
JP		 877 vr->id = i;
878 }
9497c042
NF		 879
880 return 0;
881}
882
ac571de9
IS		 883static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
884
9497c042
NF		 885static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
886{
3057224e
IS		 887 /* At this stage we're guaranteed not to have new incoming
888 * FIB notifications and the work queue is free from FIBs
889 * sitting on top of mlxsw netdevs. However, we can still
890 * have other FIBs queued. Flush the queue before flushing
891 * the device's tables. No need for locks, as we're the only
892 * writer.
893 */
894 mlxsw_core_flush_owq();
ac571de9 895 mlxsw_sp_router_fib_flush(mlxsw_sp);
9011b677 896 kfree(mlxsw_sp->router->vrs);
6b75c480
JP		 897}
898
6ddb7426
PM		 899static struct net_device *
900__mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
901{
902 struct ip_tunnel *tun = netdev_priv(ol_dev);
903 struct net *net = dev_net(ol_dev);
904
905 return __dev_get_by_index(net, tun->parms.link);
906}
907
908static u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
909{
910 struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
911
912 if (d)
913 return l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
914 else
915 return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN;
916}
917
6cf3c971 918struct mlxsw_sp_neigh_key {
33b1341c 919 struct neighbour *n;
6cf3c971
JP		 920};
921
922struct mlxsw_sp_neigh_entry {
9665b745 923 struct list_head rif_list_node;
6cf3c971
JP		 924 struct rhash_head ht_node;
925 struct mlxsw_sp_neigh_key key;
926 u16 rif;
5c8802f1 927 bool connected;
a6bf9e93 928 unsigned char ha[ETH_ALEN];
a7ff87ac
JP		 929 struct list_head nexthop_list; /* list of nexthops using
930 * this neigh entry
931 */
b2157149 932 struct list_head nexthop_neighs_list_node;
7cfcbc75
AS		 933 unsigned int counter_index;
934 bool counter_valid;
6cf3c971
JP		 935};
936
937static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
938 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
939 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
940 .key_len = sizeof(struct mlxsw_sp_neigh_key),
941};
942
f17cc84d
AS		 943struct mlxsw_sp_neigh_entry *
944mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
945 struct mlxsw_sp_neigh_entry *neigh_entry)
946{
947 if (!neigh_entry) {
948 if (list_empty(&rif->neigh_list))
949 return NULL;
950 else
951 return list_first_entry(&rif->neigh_list,
952 typeof(*neigh_entry),
953 rif_list_node);
954 }
955 if (neigh_entry->rif_list_node.next == &rif->neigh_list)
956 return NULL;
957 return list_next_entry(neigh_entry, rif_list_node);
958}
959
960int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
961{
962 return neigh_entry->key.n->tbl->family;
963}
964
965unsigned char *
966mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
967{
968 return neigh_entry->ha;
969}
970
971u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
972{
973 struct neighbour *n;
974
975 n = neigh_entry->key.n;
976 return ntohl(*((__be32 *) n->primary_key));
977}
978
0250768c
AS		 979struct in6_addr *
980mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
981{
982 struct neighbour *n;
983
984 n = neigh_entry->key.n;
985 return (struct in6_addr *) &n->primary_key;
986}
987
7cfcbc75
AS		 988int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
989 struct mlxsw_sp_neigh_entry *neigh_entry,
990 u64 *p_counter)
991{
992 if (!neigh_entry->counter_valid)
993 return -EINVAL;
994
995 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
996 p_counter, NULL);
997}
998
6cf3c971 999static struct mlxsw_sp_neigh_entry *
5c8802f1
IS		 1000mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
1001 u16 rif)
6cf3c971
JP		 1002{
1003 struct mlxsw_sp_neigh_entry *neigh_entry;
1004
5c8802f1 1005 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
6cf3c971
JP		 1006 if (!neigh_entry)
1007 return NULL;
5c8802f1 1008
33b1341c 1009 neigh_entry->key.n = n;
6cf3c971 1010 neigh_entry->rif = rif;
a7ff87ac 1011 INIT_LIST_HEAD(&neigh_entry->nexthop_list);
5c8802f1 1012
6cf3c971
JP		 1013 return neigh_entry;
1014}
1015
5c8802f1 1016static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
6cf3c971
JP		 1017{
1018 kfree(neigh_entry);
1019}
1020
5c8802f1
IS		 1021static int
1022mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
1023 struct mlxsw_sp_neigh_entry *neigh_entry)
6cf3c971 1024{
9011b677 1025 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
5c8802f1
IS		 1026 &neigh_entry->ht_node,
1027 mlxsw_sp_neigh_ht_params);
1028}
6cf3c971 1029
5c8802f1
IS		 1030static void
1031mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
1032 struct mlxsw_sp_neigh_entry *neigh_entry)
1033{
9011b677 1034 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
5c8802f1
IS		 1035 &neigh_entry->ht_node,
1036 mlxsw_sp_neigh_ht_params);
6cf3c971
JP		 1037}
1038
7cfcbc75 1039static bool
1ed5574c
AS		 1040mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
1041 struct mlxsw_sp_neigh_entry *neigh_entry)
7cfcbc75
AS		 1042{
1043 struct devlink *devlink;
1ed5574c
AS		 1044 const char *table_name;
1045
1046 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
1047 case AF_INET:
1048 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
1049 break;
1050 case AF_INET6:
1051 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
1052 break;
1053 default:
1054 WARN_ON(1);
1055 return false;
1056 }
7cfcbc75
AS		 1057
1058 devlink = priv_to_devlink(mlxsw_sp->core);
1ed5574c 1059 return devlink_dpipe_table_counter_enabled(devlink, table_name);
7cfcbc75
AS		 1060}
1061
1062static void
1063mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
1064 struct mlxsw_sp_neigh_entry *neigh_entry)
1065{
1ed5574c 1066 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
7cfcbc75
AS		 1067 return;
1068
1069 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
1070 return;
1071
1072 neigh_entry->counter_valid = true;
1073}
1074
1075static void
1076mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
1077 struct mlxsw_sp_neigh_entry *neigh_entry)
1078{
1079 if (!neigh_entry->counter_valid)
1080 return;
1081 mlxsw_sp_flow_counter_free(mlxsw_sp,
1082 neigh_entry->counter_index);
1083 neigh_entry->counter_valid = false;
1084}
1085
5c8802f1
IS		 1086static struct mlxsw_sp_neigh_entry *
1087mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
6cf3c971 1088{
6cf3c971 1089 struct mlxsw_sp_neigh_entry *neigh_entry;
bf95233e 1090 struct mlxsw_sp_rif *rif;
6cf3c971
JP		 1091 int err;
1092
bf95233e
AS		 1093 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
1094 if (!rif)
5c8802f1 1095 return ERR_PTR(-EINVAL);
6cf3c971 1096
bf95233e 1097 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
6cf3c971 1098 if (!neigh_entry)
5c8802f1
IS		 1099 return ERR_PTR(-ENOMEM);
1100
6cf3c971
JP		 1101 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
1102 if (err)
1103 goto err_neigh_entry_insert;
5c8802f1 1104
7cfcbc75 1105 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
bf95233e 1106 list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
9665b745 1107
5c8802f1 1108 return neigh_entry;
6cf3c971
JP		 1109
1110err_neigh_entry_insert:
5c8802f1
IS		 1111 mlxsw_sp_neigh_entry_free(neigh_entry);
1112 return ERR_PTR(err);
6cf3c971
JP		 1113}
1114
5c8802f1
IS		 1115static void
1116mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1117 struct mlxsw_sp_neigh_entry *neigh_entry)
6cf3c971 1118{
9665b745 1119 list_del(&neigh_entry->rif_list_node);
7cfcbc75 1120 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
5c8802f1
IS		 1121 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
1122 mlxsw_sp_neigh_entry_free(neigh_entry);
1123}
6cf3c971 1124
5c8802f1
IS		 1125static struct mlxsw_sp_neigh_entry *
1126mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1127{
1128 struct mlxsw_sp_neigh_key key;
6cf3c971 1129
5c8802f1 1130 key.n = n;
9011b677 1131 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
5c8802f1 1132 &key, mlxsw_sp_neigh_ht_params);
6cf3c971
JP		 1133}
1134
c723c735
YG		 1135static void
1136mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
1137{
a6c9b5d1 1138 unsigned long interval;
c723c735 1139
b5f3e0d4 1140#if IS_ENABLED(CONFIG_IPV6)
a6c9b5d1
AS		 1141 interval = min_t(unsigned long,
1142 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
1143 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
b5f3e0d4
IS		 1144#else
1145 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
1146#endif
9011b677 1147 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
c723c735
YG		 1148}
1149
1150static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1151 char *rauhtd_pl,
1152 int ent_index)
1153{
1154 struct net_device *dev;
1155 struct neighbour *n;
1156 __be32 dipn;
1157 u32 dip;
1158 u16 rif;
1159
1160 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
1161
5f9efffb 1162 if (!mlxsw_sp->router->rifs[rif]) {
c723c735
YG		 1163 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1164 return;
1165 }
1166
1167 dipn = htonl(dip);
5f9efffb 1168 dev = mlxsw_sp->router->rifs[rif]->dev;
c723c735
YG		 1169 n = neigh_lookup(&arp_tbl, &dipn, dev);
1170 if (!n) {
1171 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
1172 &dip);
1173 return;
1174 }
1175
1176 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
1177 neigh_event_send(n, NULL);
1178 neigh_release(n);
1179}
1180
df9a21f1 1181#if IS_ENABLED(CONFIG_IPV6)
60f040ca
AS		 1182static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1183 char *rauhtd_pl,
1184 int rec_index)
1185{
1186 struct net_device *dev;
1187 struct neighbour *n;
1188 struct in6_addr dip;
1189 u16 rif;
1190
1191 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
1192 (char *) &dip);
1193
1194 if (!mlxsw_sp->router->rifs[rif]) {
1195 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1196 return;
1197 }
1198
1199 dev = mlxsw_sp->router->rifs[rif]->dev;
1200 n = neigh_lookup(&nd_tbl, &dip, dev);
1201 if (!n) {
1202 netdev_err(dev, "Failed to find matching neighbour for IP=%pI6c\n",
1203 &dip);
1204 return;
1205 }
1206
1207 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
1208 neigh_event_send(n, NULL);
1209 neigh_release(n);
1210}
b5f3e0d4
IS		 1211#else
1212static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1213 char *rauhtd_pl,
1214 int rec_index)
1215{
1216}
1217#endif
60f040ca 1218
c723c735
YG		 1219static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1220 char *rauhtd_pl,
1221 int rec_index)
1222{
1223 u8 num_entries;
1224 int i;
1225
1226 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1227 rec_index);
1228 /* Hardware starts counting at 0, so add 1. */
1229 num_entries++;
1230
1231 /* Each record consists of several neighbour entries. */
1232 for (i = 0; i < num_entries; i++) {
1233 int ent_index;
1234
1235 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
1236 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
1237 ent_index);
1238 }
1239
1240}
1241
60f040ca
AS		 1242static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1243 char *rauhtd_pl,
1244 int rec_index)
1245{
1246 /* One record contains one entry. */
1247 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
1248 rec_index);
1249}
1250
c723c735
YG		 1251static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
1252 char *rauhtd_pl, int rec_index)
1253{
1254 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
1255 case MLXSW_REG_RAUHTD_TYPE_IPV4:
1256 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
1257 rec_index);
1258 break;
1259 case MLXSW_REG_RAUHTD_TYPE_IPV6:
60f040ca
AS		 1260 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
1261 rec_index);
c723c735
YG		 1262 break;
1263 }
1264}
1265
42cdb338
AS		 1266static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1267{
1268 u8 num_rec, last_rec_index, num_entries;
1269
1270 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1271 last_rec_index = num_rec - 1;
1272
1273 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1274 return false;
1275 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1276 MLXSW_REG_RAUHTD_TYPE_IPV6)
1277 return true;
1278
1279 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1280 last_rec_index);
1281 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1282 return true;
1283 return false;
1284}
1285
60f040ca
AS		 1286static int
1287__mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
1288 char *rauhtd_pl,
1289 enum mlxsw_reg_rauhtd_type type)
c723c735 1290{
60f040ca
AS		 1291 int i, num_rec;
1292 int err;
c723c735
YG		 1293
1294 /* Make sure the neighbour's netdev isn't removed in the
1295 * process.
1296 */
1297 rtnl_lock();
1298 do {
60f040ca 1299 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
c723c735
YG		 1300 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1301 rauhtd_pl);
1302 if (err) {
1303 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1304 break;
1305 }
1306 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1307 for (i = 0; i < num_rec; i++)
1308 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1309 i);
42cdb338 1310 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
c723c735
YG		 1311 rtnl_unlock();
1312
60f040ca
AS		 1313 return err;
1314}
1315
1316static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1317{
1318 enum mlxsw_reg_rauhtd_type type;
1319 char *rauhtd_pl;
1320 int err;
1321
1322 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1323 if (!rauhtd_pl)
1324 return -ENOMEM;
1325
1326 type = MLXSW_REG_RAUHTD_TYPE_IPV4;
1327 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1328 if (err)
1329 goto out;
1330
1331 type = MLXSW_REG_RAUHTD_TYPE_IPV6;
1332 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1333out:
c723c735 1334 kfree(rauhtd_pl);
b2157149
YG		 1335 return err;
1336}
1337
1338static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1339{
1340 struct mlxsw_sp_neigh_entry *neigh_entry;
1341
1342 /* Take RTNL mutex here to prevent lists from changes */
1343 rtnl_lock();
9011b677 1344 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
8a0b7275 1345 nexthop_neighs_list_node)
b2157149
YG		 1346 /* If this neigh have nexthops, make the kernel think this neigh
1347 * is active regardless of the traffic.
1348 */
8a0b7275 1349 neigh_event_send(neigh_entry->key.n, NULL);
b2157149
YG		 1350 rtnl_unlock();
1351}
1352
1353static void
1354mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1355{
9011b677 1356 unsigned long interval = mlxsw_sp->router->neighs_update.interval;
b2157149 1357
9011b677 1358 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
b2157149
YG		 1359 msecs_to_jiffies(interval));
1360}
1361
1362static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1363{
9011b677 1364 struct mlxsw_sp_router *router;
b2157149
YG		 1365 int err;
1366
9011b677
IS		 1367 router = container_of(work, struct mlxsw_sp_router,
1368 neighs_update.dw.work);
1369 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
b2157149 1370 if (err)
9011b677 1371 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
b2157149 1372
9011b677 1373 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
b2157149 1374
9011b677 1375 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
c723c735
YG		 1376}
1377
0b2361d9
YG		 1378static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1379{
1380 struct mlxsw_sp_neigh_entry *neigh_entry;
9011b677 1381 struct mlxsw_sp_router *router;
0b2361d9 1382
9011b677
IS		 1383 router = container_of(work, struct mlxsw_sp_router,
1384 nexthop_probe_dw.work);
0b2361d9
YG		 1385 /* Iterate over nexthop neighbours, find those who are unresolved and
1386 * send arp on them. This solves the chicken-egg problem when
1387 * the nexthop wouldn't get offloaded until the neighbor is resolved
1388 * but it wouldn't get resolved ever in case traffic is flowing in HW
1389 * using different nexthop.
1390 *
1391 * Take RTNL mutex here to prevent lists from changes.
1392 */
1393 rtnl_lock();
9011b677 1394 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
8a0b7275 1395 nexthop_neighs_list_node)
01b1aa35 1396 if (!neigh_entry->connected)
33b1341c 1397 neigh_event_send(neigh_entry->key.n, NULL);
0b2361d9
YG		 1398 rtnl_unlock();
1399
9011b677 1400 mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
0b2361d9
YG		 1401 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1402}
1403
a7ff87ac
JP		 1404static void
1405mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1406 struct mlxsw_sp_neigh_entry *neigh_entry,
1407 bool removing);
1408
5c8802f1
IS		 1409static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1410{
1411 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1412 MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1413}
1414
1415static void
1416mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1417 struct mlxsw_sp_neigh_entry *neigh_entry,
1418 enum mlxsw_reg_rauht_op op)
a6bf9e93 1419{
33b1341c 1420 struct neighbour *n = neigh_entry->key.n;
5c8802f1 1421 u32 dip = ntohl(*((__be32 *) n->primary_key));
a6bf9e93 1422 char rauht_pl[MLXSW_REG_RAUHT_LEN];
5c8802f1
IS		 1423
1424 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1425 dip);
7cfcbc75
AS		 1426 if (neigh_entry->counter_valid)
1427 mlxsw_reg_rauht_pack_counter(rauht_pl,
1428 neigh_entry->counter_index);
5c8802f1
IS		 1429 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1430}
1431
d5eb89cf
AS		 1432static void
1433mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
1434 struct mlxsw_sp_neigh_entry *neigh_entry,
1435 enum mlxsw_reg_rauht_op op)
1436{
1437 struct neighbour *n = neigh_entry->key.n;
1438 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1439 const char *dip = n->primary_key;
1440
1441 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1442 dip);
7cfcbc75
AS		 1443 if (neigh_entry->counter_valid)
1444 mlxsw_reg_rauht_pack_counter(rauht_pl,
1445 neigh_entry->counter_index);
d5eb89cf
AS		 1446 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1447}
1448
1d1056d8 1449bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
d5eb89cf 1450{
1d1056d8
AS		 1451 struct neighbour *n = neigh_entry->key.n;
1452
d5eb89cf
AS		 1453 /* Packets with a link-local destination address are trapped
1454 * after LPM lookup and never reach the neighbour table, so
1455 * there is no need to program such neighbours to the device.
1456 */
1457 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
1458 IPV6_ADDR_LINKLOCAL)
1459 return true;
1460 return false;
1461}
1462
5c8802f1
IS		 1463static void
1464mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1465 struct mlxsw_sp_neigh_entry *neigh_entry,
1466 bool adding)
1467{
1468 if (!adding && !neigh_entry->connected)
1469 return;
1470 neigh_entry->connected = adding;
b5f3e0d4 1471 if (neigh_entry->key.n->tbl->family == AF_INET) {
5c8802f1
IS		 1472 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1473 mlxsw_sp_rauht_op(adding));
b5f3e0d4 1474 } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
1d1056d8 1475 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
d5eb89cf
AS		 1476 return;
1477 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
1478 mlxsw_sp_rauht_op(adding));
1479 } else {
5c8802f1 1480 WARN_ON_ONCE(1);
d5eb89cf 1481 }
5c8802f1
IS		 1482}
1483
a481d713
AS		 1484void
1485mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
1486 struct mlxsw_sp_neigh_entry *neigh_entry,
1487 bool adding)
1488{
1489 if (adding)
1490 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1491 else
1492 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1493 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
1494}
1495
5c8802f1
IS		 1496struct mlxsw_sp_neigh_event_work {
1497 struct work_struct work;
1498 struct mlxsw_sp *mlxsw_sp;
1499 struct neighbour *n;
1500};
1501
1502static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1503{
1504 struct mlxsw_sp_neigh_event_work *neigh_work =
1505 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1506 struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1507 struct mlxsw_sp_neigh_entry *neigh_entry;
1508 struct neighbour *n = neigh_work->n;
1509 unsigned char ha[ETH_ALEN];
a6bf9e93 1510 bool entry_connected;
93a87e5e 1511 u8 nud_state, dead;
a6bf9e93 1512
5c8802f1
IS		 1513 /* If these parameters are changed after we release the lock,
1514 * then we are guaranteed to receive another event letting us
1515 * know about it.
1516 */
a6bf9e93 1517 read_lock_bh(&n->lock);
5c8802f1 1518 memcpy(ha, n->ha, ETH_ALEN);
a6bf9e93 1519 nud_state = n->nud_state;
93a87e5e 1520 dead = n->dead;
a6bf9e93
YG		 1521 read_unlock_bh(&n->lock);
1522
5c8802f1 1523 rtnl_lock();
93a87e5e 1524 entry_connected = nud_state & NUD_VALID && !dead;
5c8802f1
IS		 1525 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1526 if (!entry_connected && !neigh_entry)
1527 goto out;
1528 if (!neigh_entry) {
1529 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1530 if (IS_ERR(neigh_entry))
1531 goto out;
a6bf9e93
YG		 1532 }
1533
5c8802f1
IS		 1534 memcpy(neigh_entry->ha, ha, ETH_ALEN);
1535 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1536 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1537
1538 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1539 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1540
1541out:
1542 rtnl_unlock();
a6bf9e93 1543 neigh_release(n);
5c8802f1 1544 kfree(neigh_work);
a6bf9e93
YG		 1545}
1546
e7322638
JP		 1547int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1548 unsigned long event, void *ptr)
c723c735 1549{
5c8802f1 1550 struct mlxsw_sp_neigh_event_work *neigh_work;
c723c735
YG		 1551 struct mlxsw_sp_port *mlxsw_sp_port;
1552 struct mlxsw_sp *mlxsw_sp;
1553 unsigned long interval;
1554 struct neigh_parms *p;
a6bf9e93 1555 struct neighbour *n;
c723c735
YG		 1556
1557 switch (event) {
1558 case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1559 p = ptr;
1560
1561 /* We don't care about changes in the default table. */
b5f3e0d4
IS		 1562 if (!p->dev || (p->tbl->family != AF_INET &&
1563 p->tbl->family != AF_INET6))
c723c735
YG		 1564 return NOTIFY_DONE;
1565
1566 /* We are in atomic context and can't take RTNL mutex,
1567 * so use RCU variant to walk the device chain.
1568 */
1569 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1570 if (!mlxsw_sp_port)
1571 return NOTIFY_DONE;
1572
1573 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1574 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
9011b677 1575 mlxsw_sp->router->neighs_update.interval = interval;
c723c735
YG		 1576
1577 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1578 break;
a6bf9e93
YG		 1579 case NETEVENT_NEIGH_UPDATE:
1580 n = ptr;
a6bf9e93 1581
b5f3e0d4 1582 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
a6bf9e93
YG		 1583 return NOTIFY_DONE;
1584
5c8802f1 1585 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
a6bf9e93
YG		 1586 if (!mlxsw_sp_port)
1587 return NOTIFY_DONE;
1588
5c8802f1
IS		 1589 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1590 if (!neigh_work) {
a6bf9e93 1591 mlxsw_sp_port_dev_put(mlxsw_sp_port);
5c8802f1 1592 return NOTIFY_BAD;
a6bf9e93 1593 }
5c8802f1
IS		 1594
1595 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1596 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1597 neigh_work->n = n;
a6bf9e93
YG		 1598
1599 /* Take a reference to ensure the neighbour won't be
1600 * destructed until we drop the reference in delayed
1601 * work.
1602 */
1603 neigh_clone(n);
5c8802f1
IS		 1604 mlxsw_core_schedule_work(&neigh_work->work);
1605 mlxsw_sp_port_dev_put(mlxsw_sp_port);
a6bf9e93 1606 break;
c723c735
YG		 1607 }
1608
1609 return NOTIFY_DONE;
1610}
1611
6cf3c971
JP		 1612static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1613{
c723c735
YG		 1614 int err;
1615
9011b677 1616 err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
c723c735
YG		 1617 &mlxsw_sp_neigh_ht_params);
1618 if (err)
1619 return err;
1620
1621 /* Initialize the polling interval according to the default
1622 * table.
1623 */
1624 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1625
0b2361d9 1626 /* Create the delayed works for the activity_update */
9011b677 1627 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
c723c735 1628 mlxsw_sp_router_neighs_update_work);
9011b677 1629 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
0b2361d9 1630 mlxsw_sp_router_probe_unresolved_nexthops);
9011b677
IS		 1631 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1632 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
c723c735 1633 return 0;
6cf3c971
JP		 1634}
1635
1636static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1637{
9011b677
IS		 1638 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
1639 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
1640 rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
6cf3c971
JP		 1641}
1642
9665b745 1643static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
bf95233e 1644 struct mlxsw_sp_rif *rif)
9665b745
IS		 1645{
1646 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1647
bf95233e 1648 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
4a3c67a6
IS		 1649 rif_list_node) {
1650 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
9665b745 1651 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
4a3c67a6 1652 }
9665b745
IS		 1653}
1654
c53b8e1b
IS		 1655struct mlxsw_sp_nexthop_key {
1656 struct fib_nh *fib_nh;
1657};
1658
a7ff87ac
JP		 1659struct mlxsw_sp_nexthop {
1660 struct list_head neigh_list_node; /* member of neigh entry list */
9665b745 1661 struct list_head rif_list_node;
a7ff87ac
JP		 1662 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1663 * this belongs to
1664 */
c53b8e1b
IS		 1665 struct rhash_head ht_node;
1666 struct mlxsw_sp_nexthop_key key;
58adf2c4 1667 unsigned char gw_addr[sizeof(struct in6_addr)];
e6f3b379 1668 int ifindex;
bf95233e 1669 struct mlxsw_sp_rif *rif;
a7ff87ac
JP		 1670 u8 should_offload:1, /* set indicates this neigh is connected and
1671 * should be put to KVD linear area of this group.
1672 */
1673 offloaded:1, /* set in case the neigh is actually put into
1674 * KVD linear area of this group.
1675 */
1676 update:1; /* set indicates that MAC of this neigh should be
1677 * updated in HW
1678 */
1679 struct mlxsw_sp_neigh_entry *neigh_entry;
1680};
1681
1682struct mlxsw_sp_nexthop_group {
ba31d366 1683 void *priv;
e9ad5e7d 1684 struct rhash_head ht_node;
a7ff87ac 1685 struct list_head fib_list; /* list of fib entries that use this group */
58adf2c4 1686 struct neigh_table *neigh_tbl;
b3e8d1eb
IS		 1687 u8 adj_index_valid:1,
1688 gateway:1; /* routes using the group use a gateway */
a7ff87ac
JP		 1689 u32 adj_index;
1690 u16 ecmp_size;
1691 u16 count;
1692 struct mlxsw_sp_nexthop nexthops[0];
bf95233e 1693#define nh_rif nexthops[0].rif
a7ff87ac
JP		 1694};
1695
ba31d366
AS		 1696static struct fib_info *
1697mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp)
1698{
1699 return nh_grp->priv;
1700}
1701
1702struct mlxsw_sp_nexthop_group_cmp_arg {
e6f3b379
AS		 1703 enum mlxsw_sp_l3proto proto;
1704 union {
1705 struct fib_info *fi;
1706 struct mlxsw_sp_fib6_entry *fib6_entry;
1707 };
ba31d366
AS		 1708};
1709
e6f3b379
AS		 1710static bool
1711mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
1712 const struct in6_addr *gw, int ifindex)
1713{
1714 int i;
1715
1716 for (i = 0; i < nh_grp->count; i++) {
1717 const struct mlxsw_sp_nexthop *nh;
1718
1719 nh = &nh_grp->nexthops[i];
1720 if (nh->ifindex == ifindex &&
1721 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
1722 return true;
1723 }
1724
1725 return false;
1726}
1727
1728static bool
1729mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
1730 const struct mlxsw_sp_fib6_entry *fib6_entry)
1731{
1732 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
1733
1734 if (nh_grp->count != fib6_entry->nrt6)
1735 return false;
1736
1737 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
1738 struct in6_addr *gw;
1739 int ifindex;
1740
1741 ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex;
1742 gw = &mlxsw_sp_rt6->rt->rt6i_gateway;
1743 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex))
1744 return false;
1745 }
1746
1747 return true;
1748}
1749
ba31d366
AS		 1750static int
1751mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
1752{
1753 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
1754 const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
1755
e6f3b379
AS		 1756 switch (cmp_arg->proto) {
1757 case MLXSW_SP_L3_PROTO_IPV4:
1758 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp);
1759 case MLXSW_SP_L3_PROTO_IPV6:
1760 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
1761 cmp_arg->fib6_entry);
1762 default:
1763 WARN_ON(1);
1764 return 1;
1765 }
1766}
1767
1768static int
1769mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp)
1770{
1771 return nh_grp->neigh_tbl->family;
ba31d366
AS		 1772}
1773
1774static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
1775{
1776 const struct mlxsw_sp_nexthop_group *nh_grp = data;
e6f3b379
AS		 1777 const struct mlxsw_sp_nexthop *nh;
1778 struct fib_info *fi;
1779 unsigned int val;
1780 int i;
ba31d366 1781
e6f3b379
AS		 1782 switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
1783 case AF_INET:
1784 fi = mlxsw_sp_nexthop4_group_fi(nh_grp);
1785 return jhash(&fi, sizeof(fi), seed);
1786 case AF_INET6:
1787 val = nh_grp->count;
1788 for (i = 0; i < nh_grp->count; i++) {
1789 nh = &nh_grp->nexthops[i];
1790 val ^= nh->ifindex;
1791 }
1792 return jhash(&val, sizeof(val), seed);
1793 default:
1794 WARN_ON(1);
1795 return 0;
1796 }
1797}
1798
1799static u32
1800mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
1801{
1802 unsigned int val = fib6_entry->nrt6;
1803 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
1804 struct net_device *dev;
1805
1806 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
1807 dev = mlxsw_sp_rt6->rt->dst.dev;
1808 val ^= dev->ifindex;
1809 }
1810
1811 return jhash(&val, sizeof(val), seed);
ba31d366
AS		 1812}
1813
1814static u32
1815mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
1816{
1817 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
1818
e6f3b379
AS		 1819 switch (cmp_arg->proto) {
1820 case MLXSW_SP_L3_PROTO_IPV4:
1821 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
1822 case MLXSW_SP_L3_PROTO_IPV6:
1823 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
1824 default:
1825 WARN_ON(1);
1826 return 0;
1827 }
ba31d366
AS		 1828}
1829
e9ad5e7d 1830static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
e9ad5e7d 1831 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
ba31d366
AS		 1832 .hashfn = mlxsw_sp_nexthop_group_hash,
1833 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj,
1834 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp,
e9ad5e7d
IS		 1835};
1836
1837static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1838 struct mlxsw_sp_nexthop_group *nh_grp)
1839{
e6f3b379
AS		 1840 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
1841 !nh_grp->gateway)
1842 return 0;
1843
9011b677 1844 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
e9ad5e7d
IS		 1845 &nh_grp->ht_node,
1846 mlxsw_sp_nexthop_group_ht_params);
1847}
1848
1849static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1850 struct mlxsw_sp_nexthop_group *nh_grp)
1851{
e6f3b379
AS		 1852 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
1853 !nh_grp->gateway)
1854 return;
1855
9011b677 1856 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
e9ad5e7d
IS		 1857 &nh_grp->ht_node,
1858 mlxsw_sp_nexthop_group_ht_params);
1859}
1860
1861static struct mlxsw_sp_nexthop_group *
ba31d366
AS		 1862mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
1863 struct fib_info *fi)
e9ad5e7d 1864{
ba31d366
AS		 1865 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
1866
e6f3b379 1867 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4;
ba31d366
AS		 1868 cmp_arg.fi = fi;
1869 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
1870 &cmp_arg,
e9ad5e7d
IS		 1871 mlxsw_sp_nexthop_group_ht_params);
1872}
1873
e6f3b379
AS		 1874static struct mlxsw_sp_nexthop_group *
1875mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
1876 struct mlxsw_sp_fib6_entry *fib6_entry)
1877{
1878 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
1879
1880 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6;
1881 cmp_arg.fib6_entry = fib6_entry;
1882 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
1883 &cmp_arg,
1884 mlxsw_sp_nexthop_group_ht_params);
1885}
1886
c53b8e1b
IS		 1887static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1888 .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1889 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1890 .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1891};
1892
1893static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1894 struct mlxsw_sp_nexthop *nh)
1895{
9011b677 1896 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
c53b8e1b
IS		 1897 &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1898}
1899
1900static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1901 struct mlxsw_sp_nexthop *nh)
1902{
9011b677 1903 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
c53b8e1b
IS		 1904 mlxsw_sp_nexthop_ht_params);
1905}
1906
ad178c8e
IS		 1907static struct mlxsw_sp_nexthop *
1908mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1909 struct mlxsw_sp_nexthop_key key)
1910{
9011b677 1911 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
ad178c8e
IS		 1912 mlxsw_sp_nexthop_ht_params);
1913}
1914
a7ff87ac 1915static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
76610ebb 1916 const struct mlxsw_sp_fib *fib,
a7ff87ac
JP		 1917 u32 adj_index, u16 ecmp_size,
1918 u32 new_adj_index,
1919 u16 new_ecmp_size)
1920{
1921 char raleu_pl[MLXSW_REG_RALEU_LEN];
1922
1a9234e6 1923 mlxsw_reg_raleu_pack(raleu_pl,
76610ebb
IS		 1924 (enum mlxsw_reg_ralxx_protocol) fib->proto,
1925 fib->vr->id, adj_index, ecmp_size, new_adj_index,
1a9234e6 1926 new_ecmp_size);
a7ff87ac
JP		 1927 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1928}
1929
1930static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1931 struct mlxsw_sp_nexthop_group *nh_grp,
1932 u32 old_adj_index, u16 old_ecmp_size)
1933{
1934 struct mlxsw_sp_fib_entry *fib_entry;
76610ebb 1935 struct mlxsw_sp_fib *fib = NULL;
a7ff87ac
JP		 1936 int err;
1937
1938 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
76610ebb 1939 if (fib == fib_entry->fib_node->fib)
a7ff87ac 1940 continue;
76610ebb
IS		 1941 fib = fib_entry->fib_node->fib;
1942 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
a7ff87ac
JP		 1943 old_adj_index,
1944 old_ecmp_size,
1945 nh_grp->adj_index,
1946 nh_grp->ecmp_size);
1947 if (err)
1948 return err;
1949 }
1950 return 0;
1951}
1952
1953static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1954 struct mlxsw_sp_nexthop *nh)
1955{
1956 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1957 char ratr_pl[MLXSW_REG_RATR_LEN];
1958
1959 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
89e41982
PM		 1960 true, MLXSW_REG_RATR_TYPE_ETHERNET,
1961 adj_index, neigh_entry->rif);
a7ff87ac
JP		 1962 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1963 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1964}
1965
1966static int
1967mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
a59b7e02
IS		 1968 struct mlxsw_sp_nexthop_group *nh_grp,
1969 bool reallocate)
a7ff87ac
JP		 1970{
1971 u32 adj_index = nh_grp->adj_index; /* base */
1972 struct mlxsw_sp_nexthop *nh;
1973 int i;
1974 int err;
1975
1976 for (i = 0; i < nh_grp->count; i++) {
1977 nh = &nh_grp->nexthops[i];
1978
1979 if (!nh->should_offload) {
1980 nh->offloaded = 0;
1981 continue;
1982 }
1983
a59b7e02 1984 if (nh->update || reallocate) {
a7ff87ac
JP		 1985 err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1986 adj_index, nh);
1987 if (err)
1988 return err;
1989 nh->update = 0;
1990 nh->offloaded = 1;
1991 }
1992 adj_index++;
1993 }
1994 return 0;
1995}
1996
1997static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1998 struct mlxsw_sp_fib_entry *fib_entry);
1999
1819ae3d
IS		 2000static bool
2001mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2002 const struct mlxsw_sp_fib_entry *fib_entry);
2003
a7ff87ac
JP		 2004static int
2005mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
2006 struct mlxsw_sp_nexthop_group *nh_grp)
2007{
2008 struct mlxsw_sp_fib_entry *fib_entry;
2009 int err;
2010
2011 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1819ae3d
IS		 2012 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2013 fib_entry))
2014 continue;
a7ff87ac
JP		 2015 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2016 if (err)
2017 return err;
2018 }
2019 return 0;
2020}
2021
77d964e6
IS		 2022static void
2023mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2024 enum mlxsw_reg_ralue_op op, int err);
2025
2026static void
2027mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp)
2028{
2029 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
2030 struct mlxsw_sp_fib_entry *fib_entry;
2031
2032 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2033 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2034 fib_entry))
2035 continue;
2036 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2037 }
2038}
2039
a7ff87ac
JP		 2040static void
2041mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
2042 struct mlxsw_sp_nexthop_group *nh_grp)
2043{
2044 struct mlxsw_sp_nexthop *nh;
2045 bool offload_change = false;
2046 u32 adj_index;
2047 u16 ecmp_size = 0;
2048 bool old_adj_index_valid;
2049 u32 old_adj_index;
2050 u16 old_ecmp_size;
a7ff87ac
JP		 2051 int i;
2052 int err;
2053
b3e8d1eb
IS		 2054 if (!nh_grp->gateway) {
2055 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2056 return;
2057 }
2058
a7ff87ac
JP		 2059 for (i = 0; i < nh_grp->count; i++) {
2060 nh = &nh_grp->nexthops[i];
2061
56b8a9ed 2062 if (nh->should_offload != nh->offloaded) {
a7ff87ac
JP		 2063 offload_change = true;
2064 if (nh->should_offload)
2065 nh->update = 1;
2066 }
2067 if (nh->should_offload)
2068 ecmp_size++;
2069 }
2070 if (!offload_change) {
2071 /* Nothing was added or removed, so no need to reallocate. Just
2072 * update MAC on existing adjacency indexes.
2073 */
a59b7e02
IS		 2074 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
2075 false);
a7ff87ac
JP		 2076 if (err) {
2077 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2078 goto set_trap;
2079 }
2080 return;
2081 }
2082 if (!ecmp_size)
2083 /* No neigh of this group is connected so we just set
2084 * the trap and let everthing flow through kernel.
2085 */
2086 goto set_trap;
2087
13124443
AS		 2088 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
2089 if (err) {
a7ff87ac
JP		 2090 /* We ran out of KVD linear space, just set the
2091 * trap and let everything flow through kernel.
2092 */
2093 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
2094 goto set_trap;
2095 }
a7ff87ac
JP		 2096 old_adj_index_valid = nh_grp->adj_index_valid;
2097 old_adj_index = nh_grp->adj_index;
2098 old_ecmp_size = nh_grp->ecmp_size;
2099 nh_grp->adj_index_valid = 1;
2100 nh_grp->adj_index = adj_index;
2101 nh_grp->ecmp_size = ecmp_size;
a59b7e02 2102 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
a7ff87ac
JP		 2103 if (err) {
2104 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2105 goto set_trap;
2106 }
2107
2108 if (!old_adj_index_valid) {
2109 /* The trap was set for fib entries, so we have to call
2110 * fib entry update to unset it and use adjacency index.
2111 */
2112 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2113 if (err) {
2114 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
2115 goto set_trap;
2116 }
2117 return;
2118 }
2119
2120 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
2121 old_adj_index, old_ecmp_size);
2122 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
2123 if (err) {
2124 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
2125 goto set_trap;
2126 }
77d964e6
IS		 2127
2128 /* Offload state within the group changed, so update the flags. */
2129 mlxsw_sp_nexthop_fib_entries_refresh(nh_grp);
2130
a7ff87ac
JP		 2131 return;
2132
2133set_trap:
2134 old_adj_index_valid = nh_grp->adj_index_valid;
2135 nh_grp->adj_index_valid = 0;
2136 for (i = 0; i < nh_grp->count; i++) {
2137 nh = &nh_grp->nexthops[i];
2138 nh->offloaded = 0;
2139 }
2140 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2141 if (err)
2142 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
2143 if (old_adj_index_valid)
2144 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
2145}
2146
2147static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
2148 bool removing)
2149{
213666a3 2150 if (!removing)
a7ff87ac 2151 nh->should_offload = 1;
213666a3 2152 else if (nh->offloaded)
a7ff87ac
JP		 2153 nh->should_offload = 0;
2154 nh->update = 1;
2155}
2156
2157static void
2158mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2159 struct mlxsw_sp_neigh_entry *neigh_entry,
2160 bool removing)
2161{
2162 struct mlxsw_sp_nexthop *nh;
2163
a7ff87ac
JP		 2164 list_for_each_entry(nh, &neigh_entry->nexthop_list,
2165 neigh_list_node) {
2166 __mlxsw_sp_nexthop_neigh_update(nh, removing);
2167 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2168 }
a7ff87ac
JP		 2169}
2170
9665b745 2171static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
bf95233e 2172 struct mlxsw_sp_rif *rif)
9665b745 2173{
bf95233e 2174 if (nh->rif)
9665b745
IS		 2175 return;
2176
bf95233e
AS		 2177 nh->rif = rif;
2178 list_add(&nh->rif_list_node, &rif->nexthop_list);
9665b745
IS		 2179}
2180
2181static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
2182{
bf95233e 2183 if (!nh->rif)
9665b745
IS		 2184 return;
2185
2186 list_del(&nh->rif_list_node);
bf95233e 2187 nh->rif = NULL;
9665b745
IS		 2188}
2189
a8c97014
IS		 2190static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
2191 struct mlxsw_sp_nexthop *nh)
a7ff87ac
JP		 2192{
2193 struct mlxsw_sp_neigh_entry *neigh_entry;
a7ff87ac 2194 struct neighbour *n;
93a87e5e 2195 u8 nud_state, dead;
c53b8e1b
IS		 2196 int err;
2197
ad178c8e 2198 if (!nh->nh_grp->gateway || nh->neigh_entry)
b8399a1e
IS		 2199 return 0;
2200
33b1341c 2201 /* Take a reference of neigh here ensuring that neigh would
8de3c178 2202 * not be destructed before the nexthop entry is finished.
33b1341c 2203 * The reference is taken either in neigh_lookup() or
fd76d910 2204 * in neigh_create() in case n is not found.
33b1341c 2205 */
58adf2c4 2206 n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
33b1341c 2207 if (!n) {
58adf2c4
IS		 2208 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
2209 nh->rif->dev);
a8c97014
IS		 2210 if (IS_ERR(n))
2211 return PTR_ERR(n);
a7ff87ac 2212 neigh_event_send(n, NULL);
33b1341c
JP		 2213 }
2214 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2215 if (!neigh_entry) {
5c8802f1
IS		 2216 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2217 if (IS_ERR(neigh_entry)) {
c53b8e1b
IS		 2218 err = -EINVAL;
2219 goto err_neigh_entry_create;
5c8802f1 2220 }
a7ff87ac 2221 }
b2157149
YG		 2222
2223 /* If that is the first nexthop connected to that neigh, add to
2224 * nexthop_neighs_list
2225 */
2226 if (list_empty(&neigh_entry->nexthop_list))
2227 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
9011b677 2228 &mlxsw_sp->router->nexthop_neighs_list);
b2157149 2229
a7ff87ac
JP		 2230 nh->neigh_entry = neigh_entry;
2231 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
2232 read_lock_bh(&n->lock);
2233 nud_state = n->nud_state;
93a87e5e 2234 dead = n->dead;
a7ff87ac 2235 read_unlock_bh(&n->lock);
93a87e5e 2236 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
a7ff87ac
JP		 2237
2238 return 0;
c53b8e1b
IS		 2239
2240err_neigh_entry_create:
2241 neigh_release(n);
c53b8e1b 2242 return err;
a7ff87ac
JP		 2243}
2244
a8c97014
IS		 2245static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
2246 struct mlxsw_sp_nexthop *nh)
a7ff87ac
JP		 2247{
2248 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
a8c97014 2249 struct neighbour *n;
a7ff87ac 2250
b8399a1e 2251 if (!neigh_entry)
a8c97014
IS		 2252 return;
2253 n = neigh_entry->key.n;
b8399a1e 2254
58312125 2255 __mlxsw_sp_nexthop_neigh_update(nh, true);
a7ff87ac 2256 list_del(&nh->neigh_list_node);
e58be79e 2257 nh->neigh_entry = NULL;
b2157149
YG		 2258
2259 /* If that is the last nexthop connected to that neigh, remove from
2260 * nexthop_neighs_list
2261 */
e58be79e
IS		 2262 if (list_empty(&neigh_entry->nexthop_list))
2263 list_del(&neigh_entry->nexthop_neighs_list_node);
b2157149 2264
5c8802f1
IS		 2265 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2266 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2267
2268 neigh_release(n);
a8c97014 2269}
c53b8e1b 2270
6ddb7426
PM		 2271static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2272 const struct net_device *dev,
2273 enum mlxsw_sp_ipip_type *p_type)
2274{
2275 struct mlxsw_sp_router *router = mlxsw_sp->router;
2276 const struct mlxsw_sp_ipip_ops *ipip_ops;
2277 enum mlxsw_sp_ipip_type ipipt;
2278
2279 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
2280 ipip_ops = router->ipip_ops_arr[ipipt];
2281 if (dev->type == ipip_ops->dev_type) {
2282 if (p_type)
2283 *p_type = ipipt;
2284 return true;
2285 }
2286 }
2287 return false;
2288}
2289
0e6ea2a4
IS		 2290static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
2291 struct mlxsw_sp_nexthop_group *nh_grp,
2292 struct mlxsw_sp_nexthop *nh,
2293 struct fib_nh *fib_nh)
a8c97014
IS		 2294{
2295 struct net_device *dev = fib_nh->nh_dev;
df6dd79b 2296 struct in_device *in_dev;
bf95233e 2297 struct mlxsw_sp_rif *rif;
a8c97014
IS		 2298 int err;
2299
2300 nh->nh_grp = nh_grp;
2301 nh->key.fib_nh = fib_nh;
58adf2c4 2302 memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
a8c97014
IS		 2303 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
2304 if (err)
2305 return err;
2306
97989ee0
IS		 2307 if (!dev)
2308 return 0;
2309
df6dd79b
IS		 2310 in_dev = __in_dev_get_rtnl(dev);
2311 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
2312 fib_nh->nh_flags & RTNH_F_LINKDOWN)
2313 return 0;
2314
bf95233e
AS		 2315 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
2316 if (!rif)
a8c97014 2317 return 0;
bf95233e 2318 mlxsw_sp_nexthop_rif_init(nh, rif);
a8c97014
IS		 2319
2320 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
2321 if (err)
2322 goto err_nexthop_neigh_init;
2323
2324 return 0;
2325
2326err_nexthop_neigh_init:
a4e75b76 2327 mlxsw_sp_nexthop_rif_fini(nh);
a8c97014
IS		 2328 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2329 return err;
2330}
2331
0e6ea2a4
IS		 2332static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
2333 struct mlxsw_sp_nexthop *nh)
a8c97014
IS		 2334{
2335 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
9665b745 2336 mlxsw_sp_nexthop_rif_fini(nh);
c53b8e1b 2337 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
a7ff87ac
JP		 2338}
2339
0e6ea2a4
IS		 2340static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
2341 unsigned long event, struct fib_nh *fib_nh)
ad178c8e
IS		 2342{
2343 struct mlxsw_sp_nexthop_key key;
2344 struct mlxsw_sp_nexthop *nh;
bf95233e 2345 struct mlxsw_sp_rif *rif;
ad178c8e 2346
9011b677 2347 if (mlxsw_sp->router->aborted)
ad178c8e
IS		 2348 return;
2349
2350 key.fib_nh = fib_nh;
2351 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
2352 if (WARN_ON_ONCE(!nh))
2353 return;
2354
bf95233e
AS		 2355 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
2356 if (!rif)
ad178c8e
IS		 2357 return;
2358
2359 switch (event) {
2360 case FIB_EVENT_NH_ADD:
bf95233e 2361 mlxsw_sp_nexthop_rif_init(nh, rif);
ad178c8e
IS		 2362 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
2363 break;
2364 case FIB_EVENT_NH_DEL:
2365 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
9665b745 2366 mlxsw_sp_nexthop_rif_fini(nh);
ad178c8e
IS		 2367 break;
2368 }
2369
2370 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2371}
2372
9665b745 2373static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
bf95233e 2374 struct mlxsw_sp_rif *rif)
9665b745
IS		 2375{
2376 struct mlxsw_sp_nexthop *nh, *tmp;
2377
bf95233e 2378 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
9665b745
IS		 2379 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2380 mlxsw_sp_nexthop_rif_fini(nh);
2381 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2382 }
2383}
2384
9b01451a
PM		 2385static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
2386 const struct fib_info *fi)
2387{
2388 return fi->fib_nh->nh_scope == RT_SCOPE_LINK;
2389}
2390
a7ff87ac 2391static struct mlxsw_sp_nexthop_group *
0e6ea2a4 2392mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
a7ff87ac
JP		 2393{
2394 struct mlxsw_sp_nexthop_group *nh_grp;
2395 struct mlxsw_sp_nexthop *nh;
2396 struct fib_nh *fib_nh;
2397 size_t alloc_size;
2398 int i;
2399 int err;
2400
2401 alloc_size = sizeof(*nh_grp) +
2402 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
2403 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
2404 if (!nh_grp)
2405 return ERR_PTR(-ENOMEM);
ba31d366 2406 nh_grp->priv = fi;
a7ff87ac 2407 INIT_LIST_HEAD(&nh_grp->fib_list);
58adf2c4
IS		 2408 nh_grp->neigh_tbl = &arp_tbl;
2409
9b01451a 2410 nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi);
a7ff87ac 2411 nh_grp->count = fi->fib_nhs;
7387dbbc 2412 fib_info_hold(fi);
a7ff87ac
JP		 2413 for (i = 0; i < nh_grp->count; i++) {
2414 nh = &nh_grp->nexthops[i];
2415 fib_nh = &fi->fib_nh[i];
0e6ea2a4 2416 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
a7ff87ac 2417 if (err)
0e6ea2a4 2418 goto err_nexthop4_init;
a7ff87ac 2419 }
e9ad5e7d
IS		 2420 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
2421 if (err)
2422 goto err_nexthop_group_insert;
a7ff87ac
JP		 2423 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2424 return nh_grp;
2425
e9ad5e7d 2426err_nexthop_group_insert:
0e6ea2a4 2427err_nexthop4_init:
df6dd79b
IS		 2428 for (i--; i >= 0; i--) {
2429 nh = &nh_grp->nexthops[i];
0e6ea2a4 2430 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
df6dd79b 2431 }
ba31d366 2432 fib_info_put(fi);
a7ff87ac
JP		 2433 kfree(nh_grp);
2434 return ERR_PTR(err);
2435}
2436
2437static void
0e6ea2a4
IS		 2438mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
2439 struct mlxsw_sp_nexthop_group *nh_grp)
a7ff87ac
JP		 2440{
2441 struct mlxsw_sp_nexthop *nh;
2442 int i;
2443
e9ad5e7d 2444 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
a7ff87ac
JP		 2445 for (i = 0; i < nh_grp->count; i++) {
2446 nh = &nh_grp->nexthops[i];
0e6ea2a4 2447 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
a7ff87ac 2448 }
58312125
IS		 2449 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2450 WARN_ON_ONCE(nh_grp->adj_index_valid);
ba31d366 2451 fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp));
a7ff87ac
JP		 2452 kfree(nh_grp);
2453}
2454
0e6ea2a4
IS		 2455static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
2456 struct mlxsw_sp_fib_entry *fib_entry,
2457 struct fib_info *fi)
a7ff87ac
JP		 2458{
2459 struct mlxsw_sp_nexthop_group *nh_grp;
2460
ba31d366 2461 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
a7ff87ac 2462 if (!nh_grp) {
0e6ea2a4 2463 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
a7ff87ac
JP		 2464 if (IS_ERR(nh_grp))
2465 return PTR_ERR(nh_grp);
2466 }
2467 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
2468 fib_entry->nh_group = nh_grp;
2469 return 0;
2470}
2471
0e6ea2a4
IS		 2472static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
2473 struct mlxsw_sp_fib_entry *fib_entry)
a7ff87ac
JP		 2474{
2475 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2476
2477 list_del(&fib_entry->nexthop_group_node);
2478 if (!list_empty(&nh_grp->fib_list))
2479 return;
0e6ea2a4 2480 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
a7ff87ac
JP		 2481}
2482
4f1c7f1f
IS		 2483static bool
2484mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2485{
2486 struct mlxsw_sp_fib4_entry *fib4_entry;
2487
2488 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
2489 common);
2490 return !fib4_entry->tos;
2491}
2492
013b20f9
IS		 2493static bool
2494mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2495{
2496 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
2497
4f1c7f1f
IS		 2498 switch (fib_entry->fib_node->fib->proto) {
2499 case MLXSW_SP_L3_PROTO_IPV4:
2500 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
2501 return false;
2502 break;
2503 case MLXSW_SP_L3_PROTO_IPV6:
2504 break;
2505 }
9aecce1c 2506
013b20f9
IS		 2507 switch (fib_entry->type) {
2508 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2509 return !!nh_group->adj_index_valid;
2510 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
70ad3506 2511 return !!nh_group->nh_rif;
013b20f9
IS		 2512 default:
2513 return false;
2514 }
2515}
2516
428b851f
IS		 2517static struct mlxsw_sp_nexthop *
2518mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
2519 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
2520{
2521 int i;
2522
2523 for (i = 0; i < nh_grp->count; i++) {
2524 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2525 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2526
2527 if (nh->rif && nh->rif->dev == rt->dst.dev &&
2528 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
2529 &rt->rt6i_gateway))
2530 return nh;
2531 continue;
2532 }
2533
2534 return NULL;
2535}
2536
3984d1a8
IS		 2537static void
2538mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2539{
2540 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2541 int i;
2542
2543 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
2544 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
2545 return;
2546 }
2547
2548 for (i = 0; i < nh_grp->count; i++) {
2549 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2550
2551 if (nh->offloaded)
2552 nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
2553 else
2554 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
2555 }
2556}
2557
2558static void
2559mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2560{
2561 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2562 int i;
2563
2564 for (i = 0; i < nh_grp->count; i++) {
2565 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2566
2567 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
2568 }
2569}
2570
428b851f
IS		 2571static void
2572mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2573{
2574 struct mlxsw_sp_fib6_entry *fib6_entry;
2575 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2576
2577 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
2578 common);
2579
2580 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
2581 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
fe400799 2582 list)->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
428b851f
IS		 2583 return;
2584 }
2585
2586 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2587 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2588 struct mlxsw_sp_nexthop *nh;
2589
2590 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
2591 if (nh && nh->offloaded)
fe400799 2592 mlxsw_sp_rt6->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
428b851f 2593 else
fe400799 2594 mlxsw_sp_rt6->rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
428b851f
IS		 2595 }
2596}
2597
2598static void
2599mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2600{
2601 struct mlxsw_sp_fib6_entry *fib6_entry;
2602 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2603
2604 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
2605 common);
2606 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2607 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2608
fe400799 2609 rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
428b851f
IS		 2610 }
2611}
2612
013b20f9
IS		 2613static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2614{
76610ebb 2615 switch (fib_entry->fib_node->fib->proto) {
013b20f9 2616 case MLXSW_SP_L3_PROTO_IPV4:
3984d1a8 2617 mlxsw_sp_fib4_entry_offload_set(fib_entry);
013b20f9
IS		 2618 break;
2619 case MLXSW_SP_L3_PROTO_IPV6:
428b851f
IS		 2620 mlxsw_sp_fib6_entry_offload_set(fib_entry);
2621 break;
013b20f9
IS		 2622 }
2623}
2624
2625static void
2626mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2627{
76610ebb 2628 switch (fib_entry->fib_node->fib->proto) {
013b20f9 2629 case MLXSW_SP_L3_PROTO_IPV4:
3984d1a8 2630 mlxsw_sp_fib4_entry_offload_unset(fib_entry);
013b20f9
IS		 2631 break;
2632 case MLXSW_SP_L3_PROTO_IPV6:
428b851f
IS		 2633 mlxsw_sp_fib6_entry_offload_unset(fib_entry);
2634 break;
013b20f9 2635 }
013b20f9
IS		 2636}
2637
2638static void
2639mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2640 enum mlxsw_reg_ralue_op op, int err)
2641{
2642 switch (op) {
2643 case MLXSW_REG_RALUE_OP_WRITE_DELETE:
013b20f9
IS		 2644 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
2645 case MLXSW_REG_RALUE_OP_WRITE_WRITE:
2646 if (err)
2647 return;
1353ee70 2648 if (mlxsw_sp_fib_entry_should_offload(fib_entry))
013b20f9 2649 mlxsw_sp_fib_entry_offload_set(fib_entry);
1353ee70 2650 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry))
013b20f9
IS		 2651 mlxsw_sp_fib_entry_offload_unset(fib_entry);
2652 return;
2653 default:
2654 return;
2655 }
2656}
2657
9dbf4d76
IS		 2658static void
2659mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
2660 const struct mlxsw_sp_fib_entry *fib_entry,
2661 enum mlxsw_reg_ralue_op op)
a7ff87ac 2662{
76610ebb 2663 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
9dbf4d76
IS		 2664 enum mlxsw_reg_ralxx_protocol proto;
2665 u32 *p_dip;
2666
2667 proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
2668
2669 switch (fib->proto) {
2670 case MLXSW_SP_L3_PROTO_IPV4:
2671 p_dip = (u32 *) fib_entry->fib_node->key.addr;
2672 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
2673 fib_entry->fib_node->key.prefix_len,
2674 *p_dip);
2675 break;
2676 case MLXSW_SP_L3_PROTO_IPV6:
2677 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
2678 fib_entry->fib_node->key.prefix_len,
2679 fib_entry->fib_node->key.addr);
2680 break;
2681 }
2682}
2683
2684static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
2685 struct mlxsw_sp_fib_entry *fib_entry,
2686 enum mlxsw_reg_ralue_op op)
2687{
2688 char ralue_pl[MLXSW_REG_RALUE_LEN];
a7ff87ac
JP		 2689 enum mlxsw_reg_ralue_trap_action trap_action;
2690 u16 trap_id = 0;
2691 u32 adjacency_index = 0;
2692 u16 ecmp_size = 0;
2693
2694 /* In case the nexthop group adjacency index is valid, use it
2695 * with provided ECMP size. Otherwise, setup trap and pass
2696 * traffic to kernel.
2697 */
4b411477 2698 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
a7ff87ac
JP		 2699 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2700 adjacency_index = fib_entry->nh_group->adj_index;
2701 ecmp_size = fib_entry->nh_group->ecmp_size;
2702 } else {
2703 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2704 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2705 }
2706
9dbf4d76 2707 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
a7ff87ac
JP		 2708 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
2709 adjacency_index, ecmp_size);
2710 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2711}
2712
9dbf4d76
IS		 2713static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
2714 struct mlxsw_sp_fib_entry *fib_entry,
2715 enum mlxsw_reg_ralue_op op)
61c503f9 2716{
bf95233e 2717 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
70ad3506 2718 enum mlxsw_reg_ralue_trap_action trap_action;
61c503f9 2719 char ralue_pl[MLXSW_REG_RALUE_LEN];
70ad3506 2720 u16 trap_id = 0;
bf95233e 2721 u16 rif_index = 0;
70ad3506
IS		 2722
2723 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2724 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
bf95233e 2725 rif_index = rif->rif_index;
70ad3506
IS		 2726 } else {
2727 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2728 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2729 }
61c503f9 2730
9dbf4d76 2731 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
bf95233e
AS		 2732 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
2733 rif_index);
61c503f9
JP		 2734 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2735}
2736
9dbf4d76
IS		 2737static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
2738 struct mlxsw_sp_fib_entry *fib_entry,
2739 enum mlxsw_reg_ralue_op op)
61c503f9
JP		 2740{
2741 char ralue_pl[MLXSW_REG_RALUE_LEN];
61c503f9 2742
9dbf4d76 2743 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
61c503f9
JP		 2744 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2745 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2746}
2747
9dbf4d76
IS		 2748static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2749 struct mlxsw_sp_fib_entry *fib_entry,
2750 enum mlxsw_reg_ralue_op op)
61c503f9
JP		 2751{
2752 switch (fib_entry->type) {
2753 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
9dbf4d76 2754 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
61c503f9 2755 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
9dbf4d76 2756 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
61c503f9 2757 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
9dbf4d76 2758 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
61c503f9
JP		 2759 }
2760 return -EINVAL;
2761}
2762
2763static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2764 struct mlxsw_sp_fib_entry *fib_entry,
2765 enum mlxsw_reg_ralue_op op)
2766{
9dbf4d76 2767 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
013b20f9 2768
013b20f9 2769 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
9dbf4d76 2770
013b20f9 2771 return err;
61c503f9
JP		 2772}
2773
2774static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
2775 struct mlxsw_sp_fib_entry *fib_entry)
2776{
7146da31
JP		 2777 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2778 MLXSW_REG_RALUE_OP_WRITE_WRITE);
61c503f9
JP		 2779}
2780
2781static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
2782 struct mlxsw_sp_fib_entry *fib_entry)
2783{
2784 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2785 MLXSW_REG_RALUE_OP_WRITE_DELETE);
2786}
2787
61c503f9 2788static int
013b20f9
IS		 2789mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
2790 const struct fib_entry_notifier_info *fen_info,
2791 struct mlxsw_sp_fib_entry *fib_entry)
61c503f9 2792{
b45f64d1 2793 struct fib_info *fi = fen_info->fi;
61c503f9 2794
97989ee0
IS		 2795 switch (fen_info->type) {
2796 case RTN_BROADCAST: /* fall through */
2797 case RTN_LOCAL:
61c503f9
JP		 2798 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2799 return 0;
97989ee0
IS		 2800 case RTN_UNREACHABLE: /* fall through */
2801 case RTN_BLACKHOLE: /* fall through */
2802 case RTN_PROHIBIT:
2803 /* Packets hitting these routes need to be trapped, but
2804 * can do so with a lower priority than packets directed
2805 * at the host, so use action type local instead of trap.
2806 */
61c503f9 2807 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
97989ee0
IS		 2808 return 0;
2809 case RTN_UNICAST:
9b01451a 2810 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi))
97989ee0 2811 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
9b01451a
PM		 2812 else
2813 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
97989ee0
IS		 2814 return 0;
2815 default:
2816 return -EINVAL;
2817 }
a7ff87ac
JP		 2818}
2819
4f1c7f1f 2820static struct mlxsw_sp_fib4_entry *
9aecce1c
IS		 2821mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
2822 struct mlxsw_sp_fib_node *fib_node,
2823 const struct fib_entry_notifier_info *fen_info)
61c503f9 2824{
4f1c7f1f 2825 struct mlxsw_sp_fib4_entry *fib4_entry;
61c503f9 2826 struct mlxsw_sp_fib_entry *fib_entry;
61c503f9
JP		 2827 int err;
2828
4f1c7f1f
IS		 2829 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
2830 if (!fib4_entry)
2831 return ERR_PTR(-ENOMEM);
2832 fib_entry = &fib4_entry->common;
61c503f9 2833
013b20f9 2834 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
61c503f9 2835 if (err)
013b20f9 2836 goto err_fib4_entry_type_set;
61c503f9 2837
0e6ea2a4 2838 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
b8399a1e 2839 if (err)
0e6ea2a4 2840 goto err_nexthop4_group_get;
b8399a1e 2841
4f1c7f1f
IS		 2842 fib4_entry->prio = fen_info->fi->fib_priority;
2843 fib4_entry->tb_id = fen_info->tb_id;
2844 fib4_entry->type = fen_info->type;
2845 fib4_entry->tos = fen_info->tos;
9aecce1c
IS		 2846
2847 fib_entry->fib_node = fib_node;
2848
4f1c7f1f 2849 return fib4_entry;
5b004412 2850
0e6ea2a4 2851err_nexthop4_group_get:
013b20f9 2852err_fib4_entry_type_set:
4f1c7f1f 2853 kfree(fib4_entry);
5b004412
JP		 2854 return ERR_PTR(err);
2855}
2856
9aecce1c 2857static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
4f1c7f1f 2858 struct mlxsw_sp_fib4_entry *fib4_entry)
9aecce1c 2859{
0e6ea2a4 2860 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
4f1c7f1f 2861 kfree(fib4_entry);
9aecce1c
IS		 2862}
2863
2864static struct mlxsw_sp_fib_node *
160e22aa
IS		 2865mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2866 size_t addr_len, unsigned char prefix_len);
9aecce1c 2867
4f1c7f1f 2868static struct mlxsw_sp_fib4_entry *
9aecce1c
IS		 2869mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
2870 const struct fib_entry_notifier_info *fen_info)
5b004412 2871{
4f1c7f1f 2872 struct mlxsw_sp_fib4_entry *fib4_entry;
9aecce1c 2873 struct mlxsw_sp_fib_node *fib_node;
160e22aa
IS		 2874 struct mlxsw_sp_fib *fib;
2875 struct mlxsw_sp_vr *vr;
2876
2877 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
2878 if (!vr)
2879 return NULL;
2880 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
5b004412 2881
160e22aa
IS		 2882 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2883 sizeof(fen_info->dst),
2884 fen_info->dst_len);
2885 if (!fib_node)
9aecce1c
IS		 2886 return NULL;
2887
4f1c7f1f
IS		 2888 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
2889 if (fib4_entry->tb_id == fen_info->tb_id &&
2890 fib4_entry->tos == fen_info->tos &&
2891 fib4_entry->type == fen_info->type &&
ba31d366
AS		 2892 mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) ==
2893 fen_info->fi) {
4f1c7f1f 2894 return fib4_entry;
9aecce1c
IS		 2895 }
2896 }
2897
2898 return NULL;
2899}
2900
2901static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
2902 .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
2903 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
2904 .key_len = sizeof(struct mlxsw_sp_fib_key),
2905 .automatic_shrinking = true,
2906};
2907
2908static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
2909 struct mlxsw_sp_fib_node *fib_node)
2910{
2911 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
2912 mlxsw_sp_fib_ht_params);
2913}
2914
2915static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
2916 struct mlxsw_sp_fib_node *fib_node)
2917{
2918 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
2919 mlxsw_sp_fib_ht_params);
2920}
2921
2922static struct mlxsw_sp_fib_node *
2923mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2924 size_t addr_len, unsigned char prefix_len)
2925{
2926 struct mlxsw_sp_fib_key key;
2927
2928 memset(&key, 0, sizeof(key));
2929 memcpy(key.addr, addr, addr_len);
2930 key.prefix_len = prefix_len;
2931 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
2932}
2933
2934static struct mlxsw_sp_fib_node *
76610ebb 2935mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
9aecce1c
IS		 2936 size_t addr_len, unsigned char prefix_len)
2937{
2938 struct mlxsw_sp_fib_node *fib_node;
2939
2940 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
2941 if (!fib_node)
5b004412
JP		 2942 return NULL;
2943
9aecce1c 2944 INIT_LIST_HEAD(&fib_node->entry_list);
76610ebb 2945 list_add(&fib_node->list, &fib->node_list);
9aecce1c
IS		 2946 memcpy(fib_node->key.addr, addr, addr_len);
2947 fib_node->key.prefix_len = prefix_len;
9aecce1c
IS		 2948
2949 return fib_node;
2950}
2951
2952static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2953{
9aecce1c
IS		 2954 list_del(&fib_node->list);
2955 WARN_ON(!list_empty(&fib_node->entry_list));
2956 kfree(fib_node);
2957}
2958
2959static bool
2960mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2961 const struct mlxsw_sp_fib_entry *fib_entry)
2962{
2963 return list_first_entry(&fib_node->entry_list,
2964 struct mlxsw_sp_fib_entry, list) == fib_entry;
2965}
2966
fc922bb0
IS		 2967static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
2968 struct mlxsw_sp_fib *fib,
2969 struct mlxsw_sp_fib_node *fib_node)
2970{
2971 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
2972 struct mlxsw_sp_lpm_tree *lpm_tree;
2973 int err;
2974
2975 /* Since the tree is shared between all virtual routers we must
2976 * make sure it contains all the required prefix lengths. This
2977 * can be computed by either adding the new prefix length to the
2978 * existing prefix usage of a bound tree, or by aggregating the
2979 * prefix lengths across all virtual routers and adding the new
2980 * one as well.
2981 */
2982 if (fib->lpm_tree)
2983 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
2984 &fib->lpm_tree->prefix_usage);
2985 else
2986 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
2987 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2988
2989 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2990 fib->proto);
2991 if (IS_ERR(lpm_tree))
2992 return PTR_ERR(lpm_tree);
2993
2994 if (fib->lpm_tree && fib->lpm_tree->id == lpm_tree->id)
2995 return 0;
2996
2997 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
2998 if (err)
2999 return err;
3000
3001 return 0;
3002}
3003
3004static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
3005 struct mlxsw_sp_fib *fib)
3006{
3007 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
3008 struct mlxsw_sp_lpm_tree *lpm_tree;
3009
3010 /* Aggregate prefix lengths across all virtual routers to make
3011 * sure we only have used prefix lengths in the LPM tree.
3012 */
3013 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
3014 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
3015 fib->proto);
3016 if (IS_ERR(lpm_tree))
3017 goto err_tree_get;
3018 mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
3019
3020err_tree_get:
3021 if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage))
3022 return;
3023 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
3024 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
3025 fib->lpm_tree = NULL;
3026}
3027
9aecce1c
IS		 3028static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
3029{
3030 unsigned char prefix_len = fib_node->key.prefix_len;
76610ebb 3031 struct mlxsw_sp_fib *fib = fib_node->fib;
9aecce1c
IS		 3032
3033 if (fib->prefix_ref_count[prefix_len]++ == 0)
3034 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
3035}
3036
3037static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
3038{
3039 unsigned char prefix_len = fib_node->key.prefix_len;
76610ebb 3040 struct mlxsw_sp_fib *fib = fib_node->fib;
9aecce1c
IS		 3041
3042 if (--fib->prefix_ref_count[prefix_len] == 0)
3043 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
5b004412
JP		 3044}
3045
76610ebb
IS		 3046static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
3047 struct mlxsw_sp_fib_node *fib_node,
3048 struct mlxsw_sp_fib *fib)
3049{
76610ebb
IS		 3050 int err;
3051
3052 err = mlxsw_sp_fib_node_insert(fib, fib_node);
3053 if (err)
3054 return err;
3055 fib_node->fib = fib;
3056
fc922bb0
IS		 3057 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib, fib_node);
3058 if (err)
3059 goto err_fib_lpm_tree_link;
76610ebb
IS		 3060
3061 mlxsw_sp_fib_node_prefix_inc(fib_node);
3062
3063 return 0;
3064
fc922bb0 3065err_fib_lpm_tree_link:
76610ebb
IS		 3066 fib_node->fib = NULL;
3067 mlxsw_sp_fib_node_remove(fib, fib_node);
3068 return err;
3069}
3070
3071static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
3072 struct mlxsw_sp_fib_node *fib_node)
3073{
76610ebb
IS		 3074 struct mlxsw_sp_fib *fib = fib_node->fib;
3075
3076 mlxsw_sp_fib_node_prefix_dec(fib_node);
fc922bb0 3077 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib);
76610ebb
IS		 3078 fib_node->fib = NULL;
3079 mlxsw_sp_fib_node_remove(fib, fib_node);
3080}
3081
9aecce1c 3082static struct mlxsw_sp_fib_node *
731ea1ca
IS		 3083mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
3084 size_t addr_len, unsigned char prefix_len,
3085 enum mlxsw_sp_l3proto proto)
5b004412 3086{
9aecce1c 3087 struct mlxsw_sp_fib_node *fib_node;
76610ebb 3088 struct mlxsw_sp_fib *fib;
9aecce1c
IS		 3089 struct mlxsw_sp_vr *vr;
3090 int err;
3091
731ea1ca 3092 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id);
9aecce1c
IS		 3093 if (IS_ERR(vr))
3094 return ERR_CAST(vr);
731ea1ca 3095 fib = mlxsw_sp_vr_fib(vr, proto);
9aecce1c 3096
731ea1ca 3097 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
9aecce1c
IS		 3098 if (fib_node)
3099 return fib_node;
5b004412 3100
731ea1ca 3101 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
9aecce1c
IS		 3102 if (!fib_node) {
3103 err = -ENOMEM;
3104 goto err_fib_node_create;
5b004412 3105 }
9aecce1c 3106
76610ebb
IS		 3107 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
3108 if (err)
3109 goto err_fib_node_init;
3110
9aecce1c
IS		 3111 return fib_node;
3112
76610ebb
IS		 3113err_fib_node_init:
3114 mlxsw_sp_fib_node_destroy(fib_node);
9aecce1c 3115err_fib_node_create:
76610ebb 3116 mlxsw_sp_vr_put(vr);
9aecce1c 3117 return ERR_PTR(err);
5b004412
JP		 3118}
3119
731ea1ca
IS		 3120static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
3121 struct mlxsw_sp_fib_node *fib_node)
5b004412 3122{
76610ebb 3123 struct mlxsw_sp_vr *vr = fib_node->fib->vr;
5b004412 3124
9aecce1c
IS		 3125 if (!list_empty(&fib_node->entry_list))
3126 return;
76610ebb 3127 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
9aecce1c 3128 mlxsw_sp_fib_node_destroy(fib_node);
76610ebb 3129 mlxsw_sp_vr_put(vr);
61c503f9
JP		 3130}
3131
4f1c7f1f 3132static struct mlxsw_sp_fib4_entry *
9aecce1c 3133mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
4f1c7f1f 3134 const struct mlxsw_sp_fib4_entry *new4_entry)
61c503f9 3135{
4f1c7f1f 3136 struct mlxsw_sp_fib4_entry *fib4_entry;
9aecce1c 3137
4f1c7f1f
IS		 3138 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3139 if (fib4_entry->tb_id > new4_entry->tb_id)
9aecce1c 3140 continue;
4f1c7f1f 3141 if (fib4_entry->tb_id != new4_entry->tb_id)
9aecce1c 3142 break;
4f1c7f1f 3143 if (fib4_entry->tos > new4_entry->tos)
9aecce1c 3144 continue;
4f1c7f1f
IS		 3145 if (fib4_entry->prio >= new4_entry->prio ||
3146 fib4_entry->tos < new4_entry->tos)
3147 return fib4_entry;
9aecce1c
IS		 3148 }
3149
3150 return NULL;
3151}
3152
4f1c7f1f
IS		 3153static int
3154mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry,
3155 struct mlxsw_sp_fib4_entry *new4_entry)
4283bce5
IS		 3156{
3157 struct mlxsw_sp_fib_node *fib_node;
3158
4f1c7f1f 3159 if (WARN_ON(!fib4_entry))
4283bce5
IS		 3160 return -EINVAL;
3161
4f1c7f1f
IS		 3162 fib_node = fib4_entry->common.fib_node;
3163 list_for_each_entry_from(fib4_entry, &fib_node->entry_list,
3164 common.list) {
3165 if (fib4_entry->tb_id != new4_entry->tb_id ||
3166 fib4_entry->tos != new4_entry->tos ||
3167 fib4_entry->prio != new4_entry->prio)
4283bce5
IS		 3168 break;
3169 }
3170
4f1c7f1f 3171 list_add_tail(&new4_entry->common.list, &fib4_entry->common.list);
4283bce5
IS		 3172 return 0;
3173}
3174
9aecce1c 3175static int
9efbee6f 3176mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry,
599cf8f9 3177 bool replace, bool append)
9aecce1c 3178{
9efbee6f 3179 struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node;
4f1c7f1f 3180 struct mlxsw_sp_fib4_entry *fib4_entry;
9aecce1c 3181
4f1c7f1f 3182 fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry);
9aecce1c 3183
4283bce5 3184 if (append)
4f1c7f1f
IS		 3185 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry);
3186 if (replace && WARN_ON(!fib4_entry))
599cf8f9 3187 return -EINVAL;
4283bce5 3188
599cf8f9
IS		 3189 /* Insert new entry before replaced one, so that we can later
3190 * remove the second.
3191 */
4f1c7f1f
IS		 3192 if (fib4_entry) {
3193 list_add_tail(&new4_entry->common.list,
3194 &fib4_entry->common.list);
9aecce1c 3195 } else {
4f1c7f1f 3196 struct mlxsw_sp_fib4_entry *last;
9aecce1c 3197
4f1c7f1f
IS		 3198 list_for_each_entry(last, &fib_node->entry_list, common.list) {
3199 if (new4_entry->tb_id > last->tb_id)
9aecce1c 3200 break;
4f1c7f1f 3201 fib4_entry = last;
9aecce1c
IS		 3202 }
3203
4f1c7f1f
IS		 3204 if (fib4_entry)
3205 list_add(&new4_entry->common.list,
3206 &fib4_entry->common.list);
9aecce1c 3207 else
4f1c7f1f
IS		 3208 list_add(&new4_entry->common.list,
3209 &fib_node->entry_list);
9aecce1c
IS		 3210 }
3211
3212 return 0;
3213}
3214
3215static void
4f1c7f1f 3216mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry)
9aecce1c 3217{
4f1c7f1f 3218 list_del(&fib4_entry->common.list);
9aecce1c
IS		 3219}
3220
80c238f9
IS		 3221static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp,
3222 struct mlxsw_sp_fib_entry *fib_entry)
9aecce1c 3223{
9efbee6f
IS		 3224 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3225
9aecce1c
IS		 3226 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3227 return 0;
3228
3229 /* To prevent packet loss, overwrite the previously offloaded
3230 * entry.
3231 */
3232 if (!list_is_singular(&fib_node->entry_list)) {
3233 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3234 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3235
3236 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
3237 }
3238
3239 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3240}
3241
80c238f9
IS		 3242static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp,
3243 struct mlxsw_sp_fib_entry *fib_entry)
9aecce1c 3244{
9efbee6f
IS		 3245 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3246
9aecce1c
IS		 3247 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3248 return;
3249
3250 /* Promote the next entry by overwriting the deleted entry */
3251 if (!list_is_singular(&fib_node->entry_list)) {
3252 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3253 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3254
3255 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
3256 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
3257 return;
3258 }
3259
3260 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
3261}
3262
3263static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
4f1c7f1f 3264 struct mlxsw_sp_fib4_entry *fib4_entry,
599cf8f9 3265 bool replace, bool append)
9aecce1c 3266{
9aecce1c
IS		 3267 int err;
3268
9efbee6f 3269 err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append);
9aecce1c
IS		 3270 if (err)
3271 return err;
3272
80c238f9 3273 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common);
9aecce1c 3274 if (err)
80c238f9 3275 goto err_fib_node_entry_add;
9aecce1c 3276
9aecce1c
IS		 3277 return 0;
3278
80c238f9 3279err_fib_node_entry_add:
4f1c7f1f 3280 mlxsw_sp_fib4_node_list_remove(fib4_entry);
9aecce1c
IS		 3281 return err;
3282}
3283
3284static void
3285mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
4f1c7f1f 3286 struct mlxsw_sp_fib4_entry *fib4_entry)
9aecce1c 3287{
80c238f9 3288 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common);
4f1c7f1f 3289 mlxsw_sp_fib4_node_list_remove(fib4_entry);
9aecce1c
IS		 3290}
3291
599cf8f9 3292static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
4f1c7f1f 3293 struct mlxsw_sp_fib4_entry *fib4_entry,
599cf8f9
IS		 3294 bool replace)
3295{
4f1c7f1f
IS		 3296 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
3297 struct mlxsw_sp_fib4_entry *replaced;
599cf8f9
IS		 3298
3299 if (!replace)
3300 return;
3301
3302 /* We inserted the new entry before replaced one */
4f1c7f1f 3303 replaced = list_next_entry(fib4_entry, common.list);
599cf8f9
IS		 3304
3305 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
3306 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
731ea1ca 3307 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
599cf8f9
IS		 3308}
3309
9aecce1c
IS		 3310static int
3311mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
4283bce5 3312 const struct fib_entry_notifier_info *fen_info,
599cf8f9 3313 bool replace, bool append)
9aecce1c 3314{
4f1c7f1f 3315 struct mlxsw_sp_fib4_entry *fib4_entry;
9aecce1c 3316 struct mlxsw_sp_fib_node *fib_node;
61c503f9
JP		 3317 int err;
3318
9011b677 3319 if (mlxsw_sp->router->aborted)
b45f64d1
JP		 3320 return 0;
3321
731ea1ca
IS		 3322 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
3323 &fen_info->dst, sizeof(fen_info->dst),
3324 fen_info->dst_len,
3325 MLXSW_SP_L3_PROTO_IPV4);
9aecce1c
IS		 3326 if (IS_ERR(fib_node)) {
3327 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
3328 return PTR_ERR(fib_node);
b45f64d1 3329 }
61c503f9 3330
4f1c7f1f
IS		 3331 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
3332 if (IS_ERR(fib4_entry)) {
9aecce1c 3333 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
4f1c7f1f 3334 err = PTR_ERR(fib4_entry);
9aecce1c
IS		 3335 goto err_fib4_entry_create;
3336 }
5b004412 3337
4f1c7f1f 3338 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace,
599cf8f9 3339 append);
b45f64d1 3340 if (err) {
9aecce1c
IS		 3341 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
3342 goto err_fib4_node_entry_link;
b45f64d1 3343 }
9aecce1c 3344
4f1c7f1f 3345 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace);
599cf8f9 3346
61c503f9
JP		 3347 return 0;
3348
9aecce1c 3349err_fib4_node_entry_link:
4f1c7f1f 3350 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
9aecce1c 3351err_fib4_entry_create:
731ea1ca 3352 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
61c503f9
JP		 3353 return err;
3354}
3355
37956d78
JP		 3356static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
3357 struct fib_entry_notifier_info *fen_info)
61c503f9 3358{
4f1c7f1f 3359 struct mlxsw_sp_fib4_entry *fib4_entry;
9aecce1c 3360 struct mlxsw_sp_fib_node *fib_node;
61c503f9 3361
9011b677 3362 if (mlxsw_sp->router->aborted)
37956d78 3363 return;
b45f64d1 3364
4f1c7f1f
IS		 3365 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
3366 if (WARN_ON(!fib4_entry))
37956d78 3367 return;
4f1c7f1f 3368 fib_node = fib4_entry->common.fib_node;
5b004412 3369
4f1c7f1f
IS		 3370 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
3371 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
731ea1ca 3372 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
61c503f9 3373}
b45f64d1 3374
428b851f
IS		 3375static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt)
3376{
3377 /* Packets with link-local destination IP arriving to the router
3378 * are trapped to the CPU, so no need to program specific routes
3379 * for them.
3380 */
3381 if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL)
3382 return true;
3383
3384 /* Multicast routes aren't supported, so ignore them. Neighbour
3385 * Discovery packets are specifically trapped.
3386 */
3387 if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST)
3388 return true;
3389
3390 /* Cloned routes are irrelevant in the forwarding path. */
3391 if (rt->rt6i_flags & RTF_CACHE)
3392 return true;
3393
3394 return false;
3395}
3396
3397static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt)
3398{
3399 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3400
3401 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
3402 if (!mlxsw_sp_rt6)
3403 return ERR_PTR(-ENOMEM);
3404
3405 /* In case of route replace, replaced route is deleted with
3406 * no notification. Take reference to prevent accessing freed
3407 * memory.
3408 */
3409 mlxsw_sp_rt6->rt = rt;
3410 rt6_hold(rt);
3411
3412 return mlxsw_sp_rt6;
3413}
3414
3415#if IS_ENABLED(CONFIG_IPV6)
3416static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3417{
3418 rt6_release(rt);
3419}
3420#else
3421static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3422{
3423}
3424#endif
3425
3426static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
3427{
3428 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
3429 kfree(mlxsw_sp_rt6);
3430}
3431
3432static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt)
3433{
3434 /* RTF_CACHE routes are ignored */
3435 return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
3436}
3437
3438static struct rt6_info *
3439mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
3440{
3441 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3442 list)->rt;
3443}
3444
3445static struct mlxsw_sp_fib6_entry *
3446mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
0a7fd1ac 3447 const struct rt6_info *nrt, bool replace)
428b851f
IS		 3448{
3449 struct mlxsw_sp_fib6_entry *fib6_entry;
3450
0a7fd1ac 3451 if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace)
428b851f
IS		 3452 return NULL;
3453
3454 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3455 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3456
3457 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
3458 * virtual router.
3459 */
3460 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
3461 continue;
3462 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
3463 break;
3464 if (rt->rt6i_metric < nrt->rt6i_metric)
3465 continue;
3466 if (rt->rt6i_metric == nrt->rt6i_metric &&
3467 mlxsw_sp_fib6_rt_can_mp(rt))
3468 return fib6_entry;
3469 if (rt->rt6i_metric > nrt->rt6i_metric)
3470 break;
3471 }
3472
3473 return NULL;
3474}
3475
3476static struct mlxsw_sp_rt6 *
3477mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
3478 const struct rt6_info *rt)
3479{
3480 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3481
3482 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3483 if (mlxsw_sp_rt6->rt == rt)
3484 return mlxsw_sp_rt6;
3485 }
3486
3487 return NULL;
3488}
3489
3490static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
3491 struct mlxsw_sp_nexthop_group *nh_grp,
3492 struct mlxsw_sp_nexthop *nh,
3493 const struct rt6_info *rt)
3494{
3495 struct net_device *dev = rt->dst.dev;
3496 struct mlxsw_sp_rif *rif;
3497 int err;
3498
3499 nh->nh_grp = nh_grp;
3500 memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr));
3501
3502 if (!dev)
3503 return 0;
e6f3b379 3504 nh->ifindex = dev->ifindex;
428b851f
IS		 3505
3506 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3507 if (!rif)
3508 return 0;
3509 mlxsw_sp_nexthop_rif_init(nh, rif);
3510
3511 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
3512 if (err)
3513 goto err_nexthop_neigh_init;
3514
3515 return 0;
3516
3517err_nexthop_neigh_init:
3518 mlxsw_sp_nexthop_rif_fini(nh);
3519 return err;
3520}
3521
3522static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
3523 struct mlxsw_sp_nexthop *nh)
3524{
3525 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
3526 mlxsw_sp_nexthop_rif_fini(nh);
3527}
3528
3529static struct mlxsw_sp_nexthop_group *
3530mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
3531 struct mlxsw_sp_fib6_entry *fib6_entry)
3532{
3533 struct mlxsw_sp_nexthop_group *nh_grp;
3534 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3535 struct mlxsw_sp_nexthop *nh;
3536 size_t alloc_size;
3537 int i = 0;
3538 int err;
3539
3540 alloc_size = sizeof(*nh_grp) +
3541 fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop);
3542 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
3543 if (!nh_grp)
3544 return ERR_PTR(-ENOMEM);
3545 INIT_LIST_HEAD(&nh_grp->fib_list);
3546#if IS_ENABLED(CONFIG_IPV6)
3547 nh_grp->neigh_tbl = &nd_tbl;
3548#endif
3549 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
3550 struct mlxsw_sp_rt6, list);
3551 nh_grp->gateway = !!(mlxsw_sp_rt6->rt->rt6i_flags & RTF_GATEWAY);
3552 nh_grp->count = fib6_entry->nrt6;
3553 for (i = 0; i < nh_grp->count; i++) {
3554 struct rt6_info *rt = mlxsw_sp_rt6->rt;
3555
3556 nh = &nh_grp->nexthops[i];
3557 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
3558 if (err)
3559 goto err_nexthop6_init;
3560 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
3561 }
e6f3b379
AS		 3562
3563 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
3564 if (err)
3565 goto err_nexthop_group_insert;
3566
428b851f
IS		 3567 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3568 return nh_grp;
3569
e6f3b379 3570err_nexthop_group_insert:
428b851f
IS		 3571err_nexthop6_init:
3572 for (i--; i >= 0; i--) {
3573 nh = &nh_grp->nexthops[i];
3574 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
3575 }
3576 kfree(nh_grp);
3577 return ERR_PTR(err);
3578}
3579
3580static void
3581mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
3582 struct mlxsw_sp_nexthop_group *nh_grp)
3583{
3584 struct mlxsw_sp_nexthop *nh;
3585 int i = nh_grp->count;
3586
e6f3b379 3587 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
428b851f
IS		 3588 for (i--; i >= 0; i--) {
3589 nh = &nh_grp->nexthops[i];
3590 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
3591 }
3592 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3593 WARN_ON(nh_grp->adj_index_valid);
3594 kfree(nh_grp);
3595}
3596
3597static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
3598 struct mlxsw_sp_fib6_entry *fib6_entry)
3599{
3600 struct mlxsw_sp_nexthop_group *nh_grp;
3601
e6f3b379
AS		 3602 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
3603 if (!nh_grp) {
3604 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
3605 if (IS_ERR(nh_grp))
3606 return PTR_ERR(nh_grp);
3607 }
428b851f
IS		 3608
3609 list_add_tail(&fib6_entry->common.nexthop_group_node,
3610 &nh_grp->fib_list);
3611 fib6_entry->common.nh_group = nh_grp;
3612
3613 return 0;
3614}
3615
3616static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
3617 struct mlxsw_sp_fib_entry *fib_entry)
3618{
3619 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3620
3621 list_del(&fib_entry->nexthop_group_node);
3622 if (!list_empty(&nh_grp->fib_list))
3623 return;
3624 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
3625}
3626
3627static int
3628mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
3629 struct mlxsw_sp_fib6_entry *fib6_entry)
3630{
3631 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
3632 int err;
3633
3634 fib6_entry->common.nh_group = NULL;
3635 list_del(&fib6_entry->common.nexthop_group_node);
3636
3637 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
3638 if (err)
3639 goto err_nexthop6_group_get;
3640
3641 /* In case this entry is offloaded, then the adjacency index
3642 * currently associated with it in the device's table is that
3643 * of the old group. Start using the new one instead.
3644 */
3645 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
3646 if (err)
3647 goto err_fib_node_entry_add;
3648
3649 if (list_empty(&old_nh_grp->fib_list))
3650 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
3651
3652 return 0;
3653
3654err_fib_node_entry_add:
3655 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
3656err_nexthop6_group_get:
3657 list_add_tail(&fib6_entry->common.nexthop_group_node,
3658 &old_nh_grp->fib_list);
3659 fib6_entry->common.nh_group = old_nh_grp;
3660 return err;
3661}
3662
3663static int
3664mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
3665 struct mlxsw_sp_fib6_entry *fib6_entry,
3666 struct rt6_info *rt)
3667{
3668 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3669 int err;
3670
3671 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
3672 if (IS_ERR(mlxsw_sp_rt6))
3673 return PTR_ERR(mlxsw_sp_rt6);
3674
3675 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
3676 fib6_entry->nrt6++;
3677
3678 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
3679 if (err)
3680 goto err_nexthop6_group_update;
3681
3682 return 0;
3683
3684err_nexthop6_group_update:
3685 fib6_entry->nrt6--;
3686 list_del(&mlxsw_sp_rt6->list);
3687 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3688 return err;
3689}
3690
3691static void
3692mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
3693 struct mlxsw_sp_fib6_entry *fib6_entry,
3694 struct rt6_info *rt)
3695{
3696 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3697
3698 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt);
3699 if (WARN_ON(!mlxsw_sp_rt6))
3700 return;
3701
3702 fib6_entry->nrt6--;
3703 list_del(&mlxsw_sp_rt6->list);
3704 mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
3705 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3706}
3707
3708static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp_fib_entry *fib_entry,
3709 const struct rt6_info *rt)
3710{
3711 /* Packets hitting RTF_REJECT routes need to be discarded by the
3712 * stack. We can rely on their destination device not having a
3713 * RIF (it's the loopback device) and can thus use action type
3714 * local, which will cause them to be trapped with a lower
3715 * priority than packets that need to be locally received.
3716 */
d3b6d377 3717 if (rt->rt6i_flags & (RTF_LOCAL | RTF_ANYCAST))
428b851f
IS		 3718 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
3719 else if (rt->rt6i_flags & RTF_REJECT)
3720 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3721 else if (rt->rt6i_flags & RTF_GATEWAY)
3722 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
3723 else
3724 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3725}
3726
3727static void
3728mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
3729{
3730 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
3731
3732 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
3733 list) {
3734 fib6_entry->nrt6--;
3735 list_del(&mlxsw_sp_rt6->list);
3736 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3737 }
3738}
3739
3740static struct mlxsw_sp_fib6_entry *
3741mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
3742 struct mlxsw_sp_fib_node *fib_node,
3743 struct rt6_info *rt)
3744{
3745 struct mlxsw_sp_fib6_entry *fib6_entry;
3746 struct mlxsw_sp_fib_entry *fib_entry;
3747 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3748 int err;
3749
3750 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
3751 if (!fib6_entry)
3752 return ERR_PTR(-ENOMEM);
3753 fib_entry = &fib6_entry->common;
3754
3755 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
3756 if (IS_ERR(mlxsw_sp_rt6)) {
3757 err = PTR_ERR(mlxsw_sp_rt6);
3758 goto err_rt6_create;
3759 }
3760
3761 mlxsw_sp_fib6_entry_type_set(fib_entry, mlxsw_sp_rt6->rt);
3762
3763 INIT_LIST_HEAD(&fib6_entry->rt6_list);
3764 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
3765 fib6_entry->nrt6 = 1;
3766 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
3767 if (err)
3768 goto err_nexthop6_group_get;
3769
3770 fib_entry->fib_node = fib_node;
3771
3772 return fib6_entry;
3773
3774err_nexthop6_group_get:
3775 list_del(&mlxsw_sp_rt6->list);
3776 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3777err_rt6_create:
3778 kfree(fib6_entry);
3779 return ERR_PTR(err);
3780}
3781
3782static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
3783 struct mlxsw_sp_fib6_entry *fib6_entry)
3784{
3785 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
3786 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
3787 WARN_ON(fib6_entry->nrt6);
3788 kfree(fib6_entry);
3789}
3790
3791static struct mlxsw_sp_fib6_entry *
3792mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
0a7fd1ac 3793 const struct rt6_info *nrt, bool replace)
428b851f 3794{
0a7fd1ac 3795 struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL;
428b851f
IS		 3796
3797 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3798 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3799
3800 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
3801 continue;
3802 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
3803 break;
0a7fd1ac
IS		 3804 if (replace && rt->rt6i_metric == nrt->rt6i_metric) {
3805 if (mlxsw_sp_fib6_rt_can_mp(rt) ==
3806 mlxsw_sp_fib6_rt_can_mp(nrt))
3807 return fib6_entry;
3808 if (mlxsw_sp_fib6_rt_can_mp(nrt))
3809 fallback = fallback ?: fib6_entry;
3810 }
428b851f 3811 if (rt->rt6i_metric > nrt->rt6i_metric)
0a7fd1ac 3812 return fallback ?: fib6_entry;
428b851f
IS		 3813 }
3814
0a7fd1ac 3815 return fallback;
428b851f
IS		 3816}
3817
3818static int
0a7fd1ac
IS		 3819mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
3820 bool replace)
428b851f
IS		 3821{
3822 struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
3823 struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
3824 struct mlxsw_sp_fib6_entry *fib6_entry;
3825
0a7fd1ac
IS		 3826 fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
3827
3828 if (replace && WARN_ON(!fib6_entry))
3829 return -EINVAL;
428b851f
IS		 3830
3831 if (fib6_entry) {
3832 list_add_tail(&new6_entry->common.list,
3833 &fib6_entry->common.list);
3834 } else {
3835 struct mlxsw_sp_fib6_entry *last;
3836
3837 list_for_each_entry(last, &fib_node->entry_list, common.list) {
3838 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last);
3839
3840 if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id)
3841 break;
3842 fib6_entry = last;
3843 }
3844
3845 if (fib6_entry)
3846 list_add(&new6_entry->common.list,
3847 &fib6_entry->common.list);
3848 else
3849 list_add(&new6_entry->common.list,
3850 &fib_node->entry_list);
3851 }
3852
3853 return 0;
3854}
3855
3856static void
3857mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry)
3858{
3859 list_del(&fib6_entry->common.list);
3860}
3861
3862static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp,
0a7fd1ac
IS		 3863 struct mlxsw_sp_fib6_entry *fib6_entry,
3864 bool replace)
428b851f
IS		 3865{
3866 int err;
3867
0a7fd1ac 3868 err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace);
428b851f
IS		 3869 if (err)
3870 return err;
3871
3872 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
3873 if (err)
3874 goto err_fib_node_entry_add;
3875
3876 return 0;
3877
3878err_fib_node_entry_add:
3879 mlxsw_sp_fib6_node_list_remove(fib6_entry);
3880 return err;
3881}
3882
3883static void
3884mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
3885 struct mlxsw_sp_fib6_entry *fib6_entry)
3886{
3887 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common);
3888 mlxsw_sp_fib6_node_list_remove(fib6_entry);
3889}
3890
3891static struct mlxsw_sp_fib6_entry *
3892mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
3893 const struct rt6_info *rt)
3894{
3895 struct mlxsw_sp_fib6_entry *fib6_entry;
3896 struct mlxsw_sp_fib_node *fib_node;
3897 struct mlxsw_sp_fib *fib;
3898 struct mlxsw_sp_vr *vr;
3899
3900 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id);
3901 if (!vr)
3902 return NULL;
3903 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
3904
3905 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr,
3906 sizeof(rt->rt6i_dst.addr),
3907 rt->rt6i_dst.plen);
3908 if (!fib_node)
3909 return NULL;
3910
3911 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3912 struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3913
3914 if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id &&
3915 rt->rt6i_metric == iter_rt->rt6i_metric &&
3916 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
3917 return fib6_entry;
3918 }
3919
3920 return NULL;
3921}
3922
0a7fd1ac
IS		 3923static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
3924 struct mlxsw_sp_fib6_entry *fib6_entry,
3925 bool replace)
3926{
3927 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
3928 struct mlxsw_sp_fib6_entry *replaced;
3929
3930 if (!replace)
3931 return;
3932
3933 replaced = list_next_entry(fib6_entry, common.list);
3934
3935 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced);
3936 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced);
3937 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3938}
3939
428b851f 3940static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
0a7fd1ac 3941 struct rt6_info *rt, bool replace)
428b851f
IS		 3942{
3943 struct mlxsw_sp_fib6_entry *fib6_entry;
3944 struct mlxsw_sp_fib_node *fib_node;
3945 int err;
3946
3947 if (mlxsw_sp->router->aborted)
3948 return 0;
3949
f36f5ac6
IS		 3950 if (rt->rt6i_src.plen)
3951 return -EINVAL;
3952
428b851f
IS		 3953 if (mlxsw_sp_fib6_rt_should_ignore(rt))
3954 return 0;
3955
3956 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id,
3957 &rt->rt6i_dst.addr,
3958 sizeof(rt->rt6i_dst.addr),
3959 rt->rt6i_dst.plen,
3960 MLXSW_SP_L3_PROTO_IPV6);
3961 if (IS_ERR(fib_node))
3962 return PTR_ERR(fib_node);
3963
3964 /* Before creating a new entry, try to append route to an existing
3965 * multipath entry.
3966 */
0a7fd1ac 3967 fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace);
428b851f
IS		 3968 if (fib6_entry) {
3969 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt);
3970 if (err)
3971 goto err_fib6_entry_nexthop_add;
3972 return 0;
3973 }
3974
3975 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt);
3976 if (IS_ERR(fib6_entry)) {
3977 err = PTR_ERR(fib6_entry);
3978 goto err_fib6_entry_create;
3979 }
3980
0a7fd1ac 3981 err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace);
428b851f
IS		 3982 if (err)
3983 goto err_fib6_node_entry_link;
3984
0a7fd1ac
IS		 3985 mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace);
3986
428b851f
IS		 3987 return 0;
3988
3989err_fib6_node_entry_link:
3990 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
3991err_fib6_entry_create:
3992err_fib6_entry_nexthop_add:
3993 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3994 return err;
3995}
3996
3997static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
3998 struct rt6_info *rt)
3999{
4000 struct mlxsw_sp_fib6_entry *fib6_entry;
4001 struct mlxsw_sp_fib_node *fib_node;
4002
4003 if (mlxsw_sp->router->aborted)
4004 return;
4005
4006 if (mlxsw_sp_fib6_rt_should_ignore(rt))
4007 return;
4008
4009 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
4010 if (WARN_ON(!fib6_entry))
4011 return;
4012
4013 /* If route is part of a multipath entry, but not the last one
4014 * removed, then only reduce its nexthop group.
4015 */
4016 if (!list_is_singular(&fib6_entry->rt6_list)) {
4017 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt);
4018 return;
4019 }
4020
4021 fib_node = fib6_entry->common.fib_node;
4022
4023 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4024 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4025 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4026}
4027
bc65a8a4
IS		 4028static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
4029 enum mlxsw_reg_ralxx_protocol proto,
4030 u8 tree_id)
b45f64d1
JP		 4031{
4032 char ralta_pl[MLXSW_REG_RALTA_LEN];
4033 char ralst_pl[MLXSW_REG_RALST_LEN];
b5d90e6d 4034 int i, err;
b45f64d1 4035
bc65a8a4 4036 mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id);
b45f64d1
JP		 4037 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
4038 if (err)
4039 return err;
4040
bc65a8a4 4041 mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id);
b45f64d1
JP		 4042 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
4043 if (err)
4044 return err;
4045
b5d90e6d 4046 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
9011b677 4047 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
b5d90e6d
IS		 4048 char raltb_pl[MLXSW_REG_RALTB_LEN];
4049 char ralue_pl[MLXSW_REG_RALUE_LEN];
b45f64d1 4050
bc65a8a4 4051 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id);
b5d90e6d
IS		 4052 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
4053 raltb_pl);
4054 if (err)
4055 return err;
4056
bc65a8a4
IS		 4057 mlxsw_reg_ralue_pack(ralue_pl, proto,
4058 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0);
b5d90e6d
IS		 4059 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
4060 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
4061 ralue_pl);
4062 if (err)
4063 return err;
4064 }
4065
4066 return 0;
b45f64d1
JP		 4067}
4068
bc65a8a4
IS		 4069static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
4070{
4071 enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4;
4072 int err;
4073
4074 err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4075 MLXSW_SP_LPM_TREE_MIN);
4076 if (err)
4077 return err;
4078
4079 proto = MLXSW_REG_RALXX_PROTOCOL_IPV6;
4080 return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4081 MLXSW_SP_LPM_TREE_MIN + 1);
4082}
4083
9aecce1c
IS		 4084static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
4085 struct mlxsw_sp_fib_node *fib_node)
4086{
4f1c7f1f 4087 struct mlxsw_sp_fib4_entry *fib4_entry, *tmp;
9aecce1c 4088
4f1c7f1f
IS		 4089 list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list,
4090 common.list) {
4091 bool do_break = &tmp->common.list == &fib_node->entry_list;
9aecce1c 4092
4f1c7f1f
IS		 4093 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
4094 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
731ea1ca 4095 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
9aecce1c
IS		 4096 /* Break when entry list is empty and node was freed.
4097 * Otherwise, we'll access freed memory in the next
4098 * iteration.
4099 */
4100 if (do_break)
4101 break;
4102 }
4103}
4104
428b851f
IS		 4105static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
4106 struct mlxsw_sp_fib_node *fib_node)
4107{
4108 struct mlxsw_sp_fib6_entry *fib6_entry, *tmp;
4109
4110 list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list,
4111 common.list) {
4112 bool do_break = &tmp->common.list == &fib_node->entry_list;
4113
4114 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4115 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4116 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4117 if (do_break)
4118 break;
4119 }
4120}
4121
9aecce1c
IS		 4122static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
4123 struct mlxsw_sp_fib_node *fib_node)
4124{
76610ebb 4125 switch (fib_node->fib->proto) {
9aecce1c
IS		 4126 case MLXSW_SP_L3_PROTO_IPV4:
4127 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
4128 break;
4129 case MLXSW_SP_L3_PROTO_IPV6:
428b851f 4130 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
9aecce1c
IS		 4131 break;
4132 }
4133}
4134
76610ebb
IS		 4135static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
4136 struct mlxsw_sp_vr *vr,
4137 enum mlxsw_sp_l3proto proto)
b45f64d1 4138{
76610ebb 4139 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
9aecce1c 4140 struct mlxsw_sp_fib_node *fib_node, *tmp;
76610ebb
IS		 4141
4142 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
4143 bool do_break = &tmp->list == &fib->node_list;
4144
4145 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
4146 if (do_break)
4147 break;
4148 }
4149}
4150
4151static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
4152{
b45f64d1 4153 int i;
b45f64d1 4154
c1a38311 4155 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
9011b677 4156 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
ac571de9 4157
76610ebb 4158 if (!mlxsw_sp_vr_is_used(vr))
b45f64d1 4159 continue;
76610ebb 4160 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
a3d9bc50
IS		 4161
4162 /* If virtual router was only used for IPv4, then it's no
4163 * longer used.
4164 */
4165 if (!mlxsw_sp_vr_is_used(vr))
4166 continue;
4167 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
b45f64d1 4168 }
ac571de9
IS		 4169}
4170
bc65a8a4 4171static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
ac571de9
IS		 4172{
4173 int err;
4174
9011b677 4175 if (mlxsw_sp->router->aborted)
d331d303
IS		 4176 return;
4177 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
ac571de9 4178 mlxsw_sp_router_fib_flush(mlxsw_sp);
9011b677 4179 mlxsw_sp->router->aborted = true;
b45f64d1
JP		 4180 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
4181 if (err)
4182 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
4183}
4184
3057224e 4185struct mlxsw_sp_fib_event_work {
a0e4761d 4186 struct work_struct work;
ad178c8e 4187 union {
428b851f 4188 struct fib6_entry_notifier_info fen6_info;
ad178c8e 4189 struct fib_entry_notifier_info fen_info;
5d7bfd14 4190 struct fib_rule_notifier_info fr_info;
ad178c8e
IS		 4191 struct fib_nh_notifier_info fnh_info;
4192 };
3057224e
IS		 4193 struct mlxsw_sp *mlxsw_sp;
4194 unsigned long event;
4195};
4196
66a5763a 4197static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
b45f64d1 4198{
3057224e 4199 struct mlxsw_sp_fib_event_work *fib_work =
a0e4761d 4200 container_of(work, struct mlxsw_sp_fib_event_work, work);
3057224e 4201 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
5d7bfd14 4202 struct fib_rule *rule;
599cf8f9 4203 bool replace, append;
b45f64d1
JP		 4204 int err;
4205
3057224e
IS		 4206 /* Protect internal structures from changes */
4207 rtnl_lock();
4208 switch (fib_work->event) {
599cf8f9 4209 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4283bce5 4210 case FIB_EVENT_ENTRY_APPEND: /* fall through */
b45f64d1 4211 case FIB_EVENT_ENTRY_ADD:
599cf8f9 4212 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4283bce5
IS		 4213 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
4214 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
599cf8f9 4215 replace, append);
b45f64d1 4216 if (err)
bc65a8a4 4217 mlxsw_sp_router_fib_abort(mlxsw_sp);
3057224e 4218 fib_info_put(fib_work->fen_info.fi);
b45f64d1
JP		 4219 break;
4220 case FIB_EVENT_ENTRY_DEL:
3057224e
IS		 4221 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
4222 fib_info_put(fib_work->fen_info.fi);
b45f64d1
JP		 4223 break;
4224 case FIB_EVENT_RULE_ADD: /* fall through */
4225 case FIB_EVENT_RULE_DEL:
5d7bfd14 4226 rule = fib_work->fr_info.rule;
c7f6e665 4227 if (!fib4_rule_default(rule) && !rule->l3mdev)
bc65a8a4 4228 mlxsw_sp_router_fib_abort(mlxsw_sp);
5d7bfd14 4229 fib_rule_put(rule);
b45f64d1 4230 break;
ad178c8e
IS		 4231 case FIB_EVENT_NH_ADD: /* fall through */
4232 case FIB_EVENT_NH_DEL:
0e6ea2a4
IS		 4233 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
4234 fib_work->fnh_info.fib_nh);
ad178c8e
IS		 4235 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
4236 break;
b45f64d1 4237 }
3057224e
IS		 4238 rtnl_unlock();
4239 kfree(fib_work);
4240}
4241
66a5763a
IS		 4242static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
4243{
583419fd
IS		 4244 struct mlxsw_sp_fib_event_work *fib_work =
4245 container_of(work, struct mlxsw_sp_fib_event_work, work);
4246 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4247 struct fib_rule *rule;
0a7fd1ac 4248 bool replace;
428b851f 4249 int err;
583419fd
IS		 4250
4251 rtnl_lock();
4252 switch (fib_work->event) {
0a7fd1ac 4253 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
428b851f 4254 case FIB_EVENT_ENTRY_ADD:
0a7fd1ac 4255 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
428b851f 4256 err = mlxsw_sp_router_fib6_add(mlxsw_sp,
0a7fd1ac 4257 fib_work->fen6_info.rt, replace);
428b851f
IS		 4258 if (err)
4259 mlxsw_sp_router_fib_abort(mlxsw_sp);
4260 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4261 break;
4262 case FIB_EVENT_ENTRY_DEL:
4263 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
4264 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4265 break;
583419fd
IS		 4266 case FIB_EVENT_RULE_ADD: /* fall through */
4267 case FIB_EVENT_RULE_DEL:
4268 rule = fib_work->fr_info.rule;
4269 if (!fib6_rule_default(rule) && !rule->l3mdev)
4270 mlxsw_sp_router_fib_abort(mlxsw_sp);
4271 fib_rule_put(rule);
4272 break;
4273 }
4274 rtnl_unlock();
4275 kfree(fib_work);
66a5763a
IS		 4276}
4277
4278static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
4279 struct fib_notifier_info *info)
4280{
4281 switch (fib_work->event) {
4282 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4283 case FIB_EVENT_ENTRY_APPEND: /* fall through */
4284 case FIB_EVENT_ENTRY_ADD: /* fall through */
4285 case FIB_EVENT_ENTRY_DEL:
4286 memcpy(&fib_work->fen_info, info, sizeof(fib_work->fen_info));
4287 /* Take referece on fib_info to prevent it from being
4288 * freed while work is queued. Release it afterwards.
4289 */
4290 fib_info_hold(fib_work->fen_info.fi);
4291 break;
4292 case FIB_EVENT_RULE_ADD: /* fall through */
4293 case FIB_EVENT_RULE_DEL:
4294 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4295 fib_rule_get(fib_work->fr_info.rule);
4296 break;
4297 case FIB_EVENT_NH_ADD: /* fall through */
4298 case FIB_EVENT_NH_DEL:
4299 memcpy(&fib_work->fnh_info, info, sizeof(fib_work->fnh_info));
4300 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
4301 break;
4302 }
4303}
4304
4305static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
4306 struct fib_notifier_info *info)
4307{
583419fd 4308 switch (fib_work->event) {
0a7fd1ac 4309 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
428b851f
IS		 4310 case FIB_EVENT_ENTRY_ADD: /* fall through */
4311 case FIB_EVENT_ENTRY_DEL:
4312 memcpy(&fib_work->fen6_info, info, sizeof(fib_work->fen6_info));
4313 rt6_hold(fib_work->fen6_info.rt);
4314 break;
583419fd
IS		 4315 case FIB_EVENT_RULE_ADD: /* fall through */
4316 case FIB_EVENT_RULE_DEL:
4317 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4318 fib_rule_get(fib_work->fr_info.rule);
4319 break;
4320 }
66a5763a
IS		 4321}
4322
3057224e
IS		 4323/* Called with rcu_read_lock() */
4324static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
4325 unsigned long event, void *ptr)
4326{
3057224e
IS		 4327 struct mlxsw_sp_fib_event_work *fib_work;
4328 struct fib_notifier_info *info = ptr;
7e39d115 4329 struct mlxsw_sp_router *router;
3057224e 4330
65e65ec1 4331 if (!net_eq(info->net, &init_net))
3057224e
IS		 4332 return NOTIFY_DONE;
4333
4334 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
4335 if (WARN_ON(!fib_work))
4336 return NOTIFY_BAD;
4337
7e39d115
IS		 4338 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
4339 fib_work->mlxsw_sp = router->mlxsw_sp;
3057224e
IS		 4340 fib_work->event = event;
4341
66a5763a
IS		 4342 switch (info->family) {
4343 case AF_INET:
4344 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
4345 mlxsw_sp_router_fib4_event(fib_work, info);
3057224e 4346 break;
66a5763a
IS		 4347 case AF_INET6:
4348 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
4349 mlxsw_sp_router_fib6_event(fib_work, info);
ad178c8e 4350 break;
3057224e
IS		 4351 }
4352
a0e4761d 4353 mlxsw_core_schedule_work(&fib_work->work);
3057224e 4354
b45f64d1
JP		 4355 return NOTIFY_DONE;
4356}
4357
4724ba56
IS		 4358static struct mlxsw_sp_rif *
4359mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
4360 const struct net_device *dev)
4361{
4362 int i;
4363
4364 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
5f9efffb
IS		 4365 if (mlxsw_sp->router->rifs[i] &&
4366 mlxsw_sp->router->rifs[i]->dev == dev)
4367 return mlxsw_sp->router->rifs[i];
4724ba56
IS		 4368
4369 return NULL;
4370}
4371
4372static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
4373{
4374 char ritr_pl[MLXSW_REG_RITR_LEN];
4375 int err;
4376
4377 mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
4378 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4379 if (WARN_ON_ONCE(err))
4380 return err;
4381
4382 mlxsw_reg_ritr_enable_set(ritr_pl, false);
4383 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4384}
4385
4386static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
bf95233e 4387 struct mlxsw_sp_rif *rif)
4724ba56 4388{
bf95233e
AS		 4389 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
4390 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
4391 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
4724ba56
IS		 4392}
4393
5ea1237f
AS		 4394static bool
4395mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
4396 unsigned long event)
4724ba56 4397{
5ea1237f
AS		 4398 struct inet6_dev *inet6_dev;
4399 bool addr_list_empty = true;
4400 struct in_device *idev;
4401
4724ba56
IS		 4402 switch (event) {
4403 case NETDEV_UP:
f1b1f273 4404 return rif == NULL;
4724ba56 4405 case NETDEV_DOWN:
5ea1237f
AS		 4406 idev = __in_dev_get_rtnl(dev);
4407 if (idev && idev->ifa_list)
4408 addr_list_empty = false;
4409
4410 inet6_dev = __in6_dev_get(dev);
4411 if (addr_list_empty && inet6_dev &&
4412 !list_empty(&inet6_dev->addr_list))
4413 addr_list_empty = false;
4414
4415 if (rif && addr_list_empty &&
bf95233e 4416 !netif_is_l3_slave(rif->dev))
4724ba56
IS		 4417 return true;
4418 /* It is possible we already removed the RIF ourselves
4419 * if it was assigned to a netdev that is now a bridge
4420 * or LAG slave.
4421 */
4422 return false;
4423 }
4424
4425 return false;
4426}
4427
e4f3c1c1
IS		 4428static enum mlxsw_sp_rif_type
4429mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
4430 const struct net_device *dev)
4431{
4432 enum mlxsw_sp_fid_type type;
4433
6ddb7426
PM		 4434 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
4435 return MLXSW_SP_RIF_TYPE_IPIP_LB;
4436
4437 /* Otherwise RIF type is derived from the type of the underlying FID. */
e4f3c1c1
IS		 4438 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
4439 type = MLXSW_SP_FID_TYPE_8021Q;
4440 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
4441 type = MLXSW_SP_FID_TYPE_8021Q;
4442 else if (netif_is_bridge_master(dev))
4443 type = MLXSW_SP_FID_TYPE_8021D;
4444 else
4445 type = MLXSW_SP_FID_TYPE_RFID;
4446
4447 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
4448}
4449
de5ed99e 4450static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
4724ba56
IS		 4451{
4452 int i;
4453
de5ed99e
IS		 4454 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
4455 if (!mlxsw_sp->router->rifs[i]) {
4456 *p_rif_index = i;
4457 return 0;
4458 }
4459 }
4724ba56 4460
de5ed99e 4461 return -ENOBUFS;
4724ba56
IS		 4462}
4463
e4f3c1c1
IS		 4464static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
4465 u16 vr_id,
4466 struct net_device *l3_dev)
4724ba56 4467{
bf95233e 4468 struct mlxsw_sp_rif *rif;
4724ba56 4469
e4f3c1c1 4470 rif = kzalloc(rif_size, GFP_KERNEL);
bf95233e 4471 if (!rif)
4724ba56
IS		 4472 return NULL;
4473
bf95233e
AS		 4474 INIT_LIST_HEAD(&rif->nexthop_list);
4475 INIT_LIST_HEAD(&rif->neigh_list);
4476 ether_addr_copy(rif->addr, l3_dev->dev_addr);
4477 rif->mtu = l3_dev->mtu;
4478 rif->vr_id = vr_id;
4479 rif->dev = l3_dev;
4480 rif->rif_index = rif_index;
4724ba56 4481
bf95233e 4482 return rif;
4724ba56
IS		 4483}
4484
5f9efffb
IS		 4485struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
4486 u16 rif_index)
4487{
4488 return mlxsw_sp->router->rifs[rif_index];
4489}
4490
fd1b9d41
AS		 4491u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
4492{
4493 return rif->rif_index;
4494}
4495
4496int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
4497{
4498 return rif->dev->ifindex;
4499}
4500
4724ba56 4501static struct mlxsw_sp_rif *
e4f3c1c1
IS		 4502mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
4503 const struct mlxsw_sp_rif_params *params)
4724ba56 4504{
e4f3c1c1
IS		 4505 u32 tb_id = l3mdev_fib_table(params->dev);
4506 const struct mlxsw_sp_rif_ops *ops;
010cadf9 4507 struct mlxsw_sp_fid *fid = NULL;
e4f3c1c1 4508 enum mlxsw_sp_rif_type type;
bf95233e 4509 struct mlxsw_sp_rif *rif;
a1107487
IS		 4510 struct mlxsw_sp_vr *vr;
4511 u16 rif_index;
4724ba56
IS		 4512 int err;
4513
e4f3c1c1
IS		 4514 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
4515 ops = mlxsw_sp->router->rif_ops_arr[type];
4516
c9ec53f0
IS		 4517 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
4518 if (IS_ERR(vr))
4519 return ERR_CAST(vr);
4520
de5ed99e
IS		 4521 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
4522 if (err)
4523 goto err_rif_index_alloc;
4724ba56 4524
e4f3c1c1 4525 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
a13a594d
IS		 4526 if (!rif) {
4527 err = -ENOMEM;
4528 goto err_rif_alloc;
4529 }
e4f3c1c1
IS		 4530 rif->mlxsw_sp = mlxsw_sp;
4531 rif->ops = ops;
a13a594d 4532
010cadf9
PM		 4533 if (ops->fid_get) {
4534 fid = ops->fid_get(rif);
4535 if (IS_ERR(fid)) {
4536 err = PTR_ERR(fid);
4537 goto err_fid_get;
4538 }
4539 rif->fid = fid;
4d93ceeb
IS		 4540 }
4541
e4f3c1c1
IS		 4542 if (ops->setup)
4543 ops->setup(rif, params);
4544
4545 err = ops->configure(rif);
4724ba56 4546 if (err)
e4f3c1c1 4547 goto err_configure;
4724ba56 4548
e4f3c1c1 4549 mlxsw_sp_rif_counters_alloc(rif);
5f9efffb 4550 mlxsw_sp->router->rifs[rif_index] = rif;
6913229e 4551 vr->rif_count++;
4724ba56 4552
bf95233e 4553 return rif;
4724ba56 4554
e4f3c1c1 4555err_configure:
010cadf9
PM		 4556 if (fid)
4557 mlxsw_sp_fid_put(fid);
a1107487 4558err_fid_get:
e4f3c1c1
IS		 4559 kfree(rif);
4560err_rif_alloc:
de5ed99e 4561err_rif_index_alloc:
c9ec53f0 4562 mlxsw_sp_vr_put(vr);
4724ba56
IS		 4563 return ERR_PTR(err);
4564}
4565
e4f3c1c1 4566void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
4724ba56 4567{
e4f3c1c1
IS		 4568 const struct mlxsw_sp_rif_ops *ops = rif->ops;
4569 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
a1107487 4570 struct mlxsw_sp_fid *fid = rif->fid;
e4f3c1c1 4571 struct mlxsw_sp_vr *vr;
4724ba56 4572
bf95233e 4573 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
e4f3c1c1 4574 vr = &mlxsw_sp->router->vrs[rif->vr_id];
e0c0afd8 4575
6913229e 4576 vr->rif_count--;
e4f3c1c1 4577 mlxsw_sp->router->rifs[rif->rif_index] = NULL;
e4f3c1c1 4578 mlxsw_sp_rif_counters_free(rif);
e4f3c1c1 4579 ops->deconfigure(rif);
010cadf9
PM		 4580 if (fid)
4581 /* Loopback RIFs are not associated with a FID. */
4582 mlxsw_sp_fid_put(fid);
e4f3c1c1 4583 kfree(rif);
c9ec53f0 4584 mlxsw_sp_vr_put(vr);
4724ba56
IS		 4585}
4586
e4f3c1c1
IS		 4587static void
4588mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
4589 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
4590{
4591 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
4592
4593 params->vid = mlxsw_sp_port_vlan->vid;
4594 params->lag = mlxsw_sp_port->lagged;
4595 if (params->lag)
4596 params->lag_id = mlxsw_sp_port->lag_id;
4597 else
4598 params->system_port = mlxsw_sp_port->local_port;
4599}
4600
7cbecf24 4601static int
a1107487 4602mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
7cbecf24 4603 struct net_device *l3_dev)
4724ba56 4604{
7cbecf24 4605 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
1b8f09a0 4606 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
7cbecf24 4607 u16 vid = mlxsw_sp_port_vlan->vid;
bf95233e 4608 struct mlxsw_sp_rif *rif;
a1107487 4609 struct mlxsw_sp_fid *fid;
03ea01e9 4610 int err;
4724ba56 4611
1b8f09a0 4612 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
bf95233e 4613 if (!rif) {
e4f3c1c1
IS		 4614 struct mlxsw_sp_rif_params params = {
4615 .dev = l3_dev,
4616 };
4617
4618 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
4619 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
bf95233e
AS		 4620 if (IS_ERR(rif))
4621 return PTR_ERR(rif);
4724ba56
IS		 4622 }
4623
a1107487 4624 /* FID was already created, just take a reference */
e4f3c1c1 4625 fid = rif->ops->fid_get(rif);
a1107487
IS		 4626 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
4627 if (err)
4628 goto err_fid_port_vid_map;
4629
7cbecf24 4630 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
03ea01e9
IS		 4631 if (err)
4632 goto err_port_vid_learning_set;
4633
7cbecf24 4634 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
03ea01e9
IS		 4635 BR_STATE_FORWARDING);
4636 if (err)
4637 goto err_port_vid_stp_set;
4638
a1107487 4639 mlxsw_sp_port_vlan->fid = fid;
4724ba56 4640
4724ba56 4641 return 0;
03ea01e9
IS		 4642
4643err_port_vid_stp_set:
7cbecf24 4644 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
03ea01e9 4645err_port_vid_learning_set:
a1107487
IS		 4646 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
4647err_fid_port_vid_map:
4648 mlxsw_sp_fid_put(fid);
03ea01e9 4649 return err;
4724ba56
IS		 4650}
4651
a1107487
IS		 4652void
4653mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
4724ba56 4654{
ce95e154 4655 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
7cbecf24 4656 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
ce95e154 4657 u16 vid = mlxsw_sp_port_vlan->vid;
ce95e154 4658
a1107487
IS		 4659 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
4660 return;
4aafc368 4661
a1107487 4662 mlxsw_sp_port_vlan->fid = NULL;
7cbecf24
IS		 4663 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
4664 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
a1107487
IS		 4665 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
4666 /* If router port holds the last reference on the rFID, then the
4667 * associated Sub-port RIF will be destroyed.
4668 */
4669 mlxsw_sp_fid_put(fid);
4724ba56
IS		 4670}
4671
7cbecf24
IS		 4672static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
4673 struct net_device *port_dev,
4674 unsigned long event, u16 vid)
4724ba56
IS		 4675{
4676 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
ce95e154 4677 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
4724ba56 4678
ce95e154 4679 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
7cbecf24
IS		 4680 if (WARN_ON(!mlxsw_sp_port_vlan))
4681 return -EINVAL;
4724ba56
IS		 4682
4683 switch (event) {
4684 case NETDEV_UP:
a1107487 4685 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
7cbecf24 4686 l3_dev);
4724ba56 4687 case NETDEV_DOWN:
a1107487 4688 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
4724ba56
IS		 4689 break;
4690 }
4691
4692 return 0;
4693}
4694
4695static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
4696 unsigned long event)
4697{
2b94e58d
JP		 4698 if (netif_is_bridge_port(port_dev) ||
4699 netif_is_lag_port(port_dev) ||
4700 netif_is_ovs_port(port_dev))
4724ba56
IS		 4701 return 0;
4702
7cbecf24 4703 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
4724ba56
IS		 4704}
4705
4706static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
4707 struct net_device *lag_dev,
4708 unsigned long event, u16 vid)
4709{
4710 struct net_device *port_dev;
4711 struct list_head *iter;
4712 int err;
4713
4714 netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
4715 if (mlxsw_sp_port_dev_check(port_dev)) {
7cbecf24
IS		 4716 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
4717 port_dev,
4718 event, vid);
4724ba56
IS		 4719 if (err)
4720 return err;
4721 }
4722 }
4723
4724 return 0;
4725}
4726
4727static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
4728 unsigned long event)
4729{
4730 if (netif_is_bridge_port(lag_dev))
4731 return 0;
4732
4733 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
4734}
4735
4724ba56 4736static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
4724ba56
IS		 4737 unsigned long event)
4738{
4739 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
e4f3c1c1
IS		 4740 struct mlxsw_sp_rif_params params = {
4741 .dev = l3_dev,
4742 };
a1107487 4743 struct mlxsw_sp_rif *rif;
4724ba56
IS		 4744
4745 switch (event) {
4746 case NETDEV_UP:
e4f3c1c1
IS		 4747 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
4748 if (IS_ERR(rif))
4749 return PTR_ERR(rif);
4750 break;
4724ba56 4751 case NETDEV_DOWN:
a1107487 4752 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
e4f3c1c1 4753 mlxsw_sp_rif_destroy(rif);
4724ba56
IS		 4754 break;
4755 }
4756
4757 return 0;
4758}
4759
4760static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
4761 unsigned long event)
4762{
4763 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
4724ba56
IS		 4764 u16 vid = vlan_dev_vlan_id(vlan_dev);
4765
6b27c8ad
IS		 4766 if (netif_is_bridge_port(vlan_dev))
4767 return 0;
4768
4724ba56 4769 if (mlxsw_sp_port_dev_check(real_dev))
7cbecf24
IS		 4770 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
4771 event, vid);
4724ba56
IS		 4772 else if (netif_is_lag_master(real_dev))
4773 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
4774 vid);
c57529e1 4775 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
a1107487 4776 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
4724ba56
IS		 4777
4778 return 0;
4779}
4780
b1e45526
IS		 4781static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
4782 unsigned long event)
4783{
4784 if (mlxsw_sp_port_dev_check(dev))
4785 return mlxsw_sp_inetaddr_port_event(dev, event);
4786 else if (netif_is_lag_master(dev))
4787 return mlxsw_sp_inetaddr_lag_event(dev, event);
4788 else if (netif_is_bridge_master(dev))
a1107487 4789 return mlxsw_sp_inetaddr_bridge_event(dev, event);
b1e45526
IS		 4790 else if (is_vlan_dev(dev))
4791 return mlxsw_sp_inetaddr_vlan_event(dev, event);
4792 else
4793 return 0;
4794}
4795
4724ba56
IS		 4796int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
4797 unsigned long event, void *ptr)
4798{
4799 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
4800 struct net_device *dev = ifa->ifa_dev->dev;
4801 struct mlxsw_sp *mlxsw_sp;
bf95233e 4802 struct mlxsw_sp_rif *rif;
4724ba56
IS		 4803 int err = 0;
4804
4805 mlxsw_sp = mlxsw_sp_lower_get(dev);
4806 if (!mlxsw_sp)
4807 goto out;
4808
bf95233e 4809 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5ea1237f 4810 if (!mlxsw_sp_rif_should_config(rif, dev, event))
4724ba56
IS		 4811 goto out;
4812
b1e45526 4813 err = __mlxsw_sp_inetaddr_event(dev, event);
4724ba56
IS		 4814out:
4815 return notifier_from_errno(err);
4816}
4817
5ea1237f
AS		 4818struct mlxsw_sp_inet6addr_event_work {
4819 struct work_struct work;
4820 struct net_device *dev;
4821 unsigned long event;
4822};
4823
4824static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
4825{
4826 struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
4827 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
4828 struct net_device *dev = inet6addr_work->dev;
4829 unsigned long event = inet6addr_work->event;
4830 struct mlxsw_sp *mlxsw_sp;
4831 struct mlxsw_sp_rif *rif;
4832
4833 rtnl_lock();
4834 mlxsw_sp = mlxsw_sp_lower_get(dev);
4835 if (!mlxsw_sp)
4836 goto out;
4837
4838 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4839 if (!mlxsw_sp_rif_should_config(rif, dev, event))
4840 goto out;
4841
4842 __mlxsw_sp_inetaddr_event(dev, event);
4843out:
4844 rtnl_unlock();
4845 dev_put(dev);
4846 kfree(inet6addr_work);
4847}
4848
4849/* Called with rcu_read_lock() */
4850int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
4851 unsigned long event, void *ptr)
4852{
4853 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
4854 struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
4855 struct net_device *dev = if6->idev->dev;
4856
4857 if (!mlxsw_sp_port_dev_lower_find_rcu(dev))
4858 return NOTIFY_DONE;
4859
4860 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
4861 if (!inet6addr_work)
4862 return NOTIFY_BAD;
4863
4864 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
4865 inet6addr_work->dev = dev;
4866 inet6addr_work->event = event;
4867 dev_hold(dev);
4868 mlxsw_core_schedule_work(&inet6addr_work->work);
4869
4870 return NOTIFY_DONE;
4871}
4872
bf95233e 4873static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
4724ba56
IS		 4874 const char *mac, int mtu)
4875{
4876 char ritr_pl[MLXSW_REG_RITR_LEN];
4877 int err;
4878
bf95233e 4879 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
4724ba56
IS		 4880 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4881 if (err)
4882 return err;
4883
4884 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
4885 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
4886 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
4887 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4888}
4889
4890int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
4891{
4892 struct mlxsw_sp *mlxsw_sp;
bf95233e 4893 struct mlxsw_sp_rif *rif;
a1107487 4894 u16 fid_index;
4724ba56
IS		 4895 int err;
4896
4897 mlxsw_sp = mlxsw_sp_lower_get(dev);
4898 if (!mlxsw_sp)
4899 return 0;
4900
bf95233e
AS		 4901 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4902 if (!rif)
4724ba56 4903 return 0;
a1107487 4904 fid_index = mlxsw_sp_fid_index(rif->fid);
4724ba56 4905
a1107487 4906 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
4724ba56
IS		 4907 if (err)
4908 return err;
4909
bf95233e
AS		 4910 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
4911 dev->mtu);
4724ba56
IS		 4912 if (err)
4913 goto err_rif_edit;
4914
a1107487 4915 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
4724ba56
IS		 4916 if (err)
4917 goto err_rif_fdb_op;
4918
bf95233e
AS		 4919 ether_addr_copy(rif->addr, dev->dev_addr);
4920 rif->mtu = dev->mtu;
4724ba56 4921
bf95233e 4922 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
4724ba56
IS		 4923
4924 return 0;
4925
4926err_rif_fdb_op:
bf95233e 4927 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
4724ba56 4928err_rif_edit:
a1107487 4929 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
4724ba56
IS		 4930 return err;
4931}
4932
b1e45526
IS		 4933static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
4934 struct net_device *l3_dev)
7179eb5a 4935{
b1e45526 4936 struct mlxsw_sp_rif *rif;
7179eb5a 4937
b1e45526
IS		 4938 /* If netdev is already associated with a RIF, then we need to
4939 * destroy it and create a new one with the new virtual router ID.
7179eb5a 4940 */
b1e45526
IS		 4941 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4942 if (rif)
4943 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
7179eb5a 4944
b1e45526 4945 return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
7179eb5a
IS		 4946}
4947
b1e45526
IS		 4948static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
4949 struct net_device *l3_dev)
7179eb5a 4950{
b1e45526 4951 struct mlxsw_sp_rif *rif;
7179eb5a 4952
b1e45526
IS		 4953 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4954 if (!rif)
7179eb5a 4955 return;
b1e45526 4956 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
7179eb5a
IS		 4957}
4958
b1e45526
IS		 4959int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
4960 struct netdev_notifier_changeupper_info *info)
3d70e458 4961{
b1e45526
IS		 4962 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
4963 int err = 0;
3d70e458 4964
b1e45526
IS		 4965 if (!mlxsw_sp)
4966 return 0;
3d70e458 4967
b1e45526
IS		 4968 switch (event) {
4969 case NETDEV_PRECHANGEUPPER:
4970 return 0;
4971 case NETDEV_CHANGEUPPER:
4972 if (info->linking)
4973 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
4974 else
4975 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
4976 break;
4977 }
3d70e458 4978
b1e45526 4979 return err;
3d70e458
IS		 4980}
4981
e4f3c1c1
IS		 4982static struct mlxsw_sp_rif_subport *
4983mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
a1107487 4984{
e4f3c1c1
IS		 4985 return container_of(rif, struct mlxsw_sp_rif_subport, common);
4986}
4987
4988static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
4989 const struct mlxsw_sp_rif_params *params)
4990{
4991 struct mlxsw_sp_rif_subport *rif_subport;
4992
4993 rif_subport = mlxsw_sp_rif_subport_rif(rif);
4994 rif_subport->vid = params->vid;
4995 rif_subport->lag = params->lag;
4996 if (params->lag)
4997 rif_subport->lag_id = params->lag_id;
a1107487 4998 else
e4f3c1c1
IS		 4999 rif_subport->system_port = params->system_port;
5000}
5001
5002static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
5003{
5004 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5005 struct mlxsw_sp_rif_subport *rif_subport;
5006 char ritr_pl[MLXSW_REG_RITR_LEN];
5007
5008 rif_subport = mlxsw_sp_rif_subport_rif(rif);
5009 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
9571e828
PM		 5010 rif->rif_index, rif->vr_id, rif->dev->mtu);
5011 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
e4f3c1c1
IS		 5012 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
5013 rif_subport->lag ? rif_subport->lag_id :
5014 rif_subport->system_port,
5015 rif_subport->vid);
5016
5017 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5018}
5019
5020static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
5021{
010cadf9
PM		 5022 int err;
5023
5024 err = mlxsw_sp_rif_subport_op(rif, true);
5025 if (err)
5026 return err;
5027
5028 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5029 mlxsw_sp_fid_index(rif->fid), true);
5030 if (err)
5031 goto err_rif_fdb_op;
5032
5033 mlxsw_sp_fid_rif_set(rif->fid, rif);
5034 return 0;
5035
5036err_rif_fdb_op:
5037 mlxsw_sp_rif_subport_op(rif, false);
5038 return err;
a1107487
IS		 5039}
5040
e4f3c1c1
IS		 5041static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
5042{
010cadf9
PM		 5043 struct mlxsw_sp_fid *fid = rif->fid;
5044
5045 mlxsw_sp_fid_rif_set(fid, NULL);
5046 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5047 mlxsw_sp_fid_index(fid), false);
e4f3c1c1
IS		 5048 mlxsw_sp_rif_subport_op(rif, false);
5049}
5050
5051static struct mlxsw_sp_fid *
5052mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
5053{
5054 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
5055}
5056
5057static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
5058 .type = MLXSW_SP_RIF_TYPE_SUBPORT,
5059 .rif_size = sizeof(struct mlxsw_sp_rif_subport),
5060 .setup = mlxsw_sp_rif_subport_setup,
5061 .configure = mlxsw_sp_rif_subport_configure,
5062 .deconfigure = mlxsw_sp_rif_subport_deconfigure,
5063 .fid_get = mlxsw_sp_rif_subport_fid_get,
5064};
5065
5066static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
5067 enum mlxsw_reg_ritr_if_type type,
5068 u16 vid_fid, bool enable)
5069{
5070 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5071 char ritr_pl[MLXSW_REG_RITR_LEN];
5072
5073 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
9571e828
PM		 5074 rif->dev->mtu);
5075 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
e4f3c1c1
IS		 5076 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
5077
5078 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5079}
5080
5081static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
5082{
5083 return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
5084}
5085
5086static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
5087{
5088 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5089 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5090 int err;
5091
5092 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
5093 if (err)
5094 return err;
5095
0d284818
IS		 5096 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5097 mlxsw_sp_router_port(mlxsw_sp), true);
5098 if (err)
5099 goto err_fid_mc_flood_set;
5100
e4f3c1c1
IS		 5101 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5102 mlxsw_sp_router_port(mlxsw_sp), true);
5103 if (err)
5104 goto err_fid_bc_flood_set;
5105
010cadf9
PM		 5106 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5107 mlxsw_sp_fid_index(rif->fid), true);
5108 if (err)
5109 goto err_rif_fdb_op;
5110
5111 mlxsw_sp_fid_rif_set(rif->fid, rif);
e4f3c1c1
IS		 5112 return 0;
5113
010cadf9
PM		 5114err_rif_fdb_op:
5115 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5116 mlxsw_sp_router_port(mlxsw_sp), false);
e4f3c1c1 5117err_fid_bc_flood_set:
0d284818
IS		 5118 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5119 mlxsw_sp_router_port(mlxsw_sp), false);
5120err_fid_mc_flood_set:
e4f3c1c1
IS		 5121 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5122 return err;
5123}
5124
5125static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
5126{
e4f3c1c1 5127 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
010cadf9
PM		 5128 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5129 struct mlxsw_sp_fid *fid = rif->fid;
e4f3c1c1 5130
010cadf9
PM		 5131 mlxsw_sp_fid_rif_set(fid, NULL);
5132 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5133 mlxsw_sp_fid_index(fid), false);
e4f3c1c1
IS		 5134 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5135 mlxsw_sp_router_port(mlxsw_sp), false);
0d284818
IS		 5136 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5137 mlxsw_sp_router_port(mlxsw_sp), false);
e4f3c1c1
IS		 5138 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5139}
5140
5141static struct mlxsw_sp_fid *
5142mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
5143{
5144 u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
5145
5146 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
5147}
5148
5149static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
5150 .type = MLXSW_SP_RIF_TYPE_VLAN,
5151 .rif_size = sizeof(struct mlxsw_sp_rif),
5152 .configure = mlxsw_sp_rif_vlan_configure,
5153 .deconfigure = mlxsw_sp_rif_vlan_deconfigure,
5154 .fid_get = mlxsw_sp_rif_vlan_fid_get,
5155};
5156
5157static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
5158{
5159 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5160 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5161 int err;
5162
5163 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
5164 true);
5165 if (err)
5166 return err;
5167
0d284818
IS		 5168 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5169 mlxsw_sp_router_port(mlxsw_sp), true);
5170 if (err)
5171 goto err_fid_mc_flood_set;
5172
e4f3c1c1
IS		 5173 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5174 mlxsw_sp_router_port(mlxsw_sp), true);
5175 if (err)
5176 goto err_fid_bc_flood_set;
5177
010cadf9
PM		 5178 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5179 mlxsw_sp_fid_index(rif->fid), true);
5180 if (err)
5181 goto err_rif_fdb_op;
5182
5183 mlxsw_sp_fid_rif_set(rif->fid, rif);
e4f3c1c1
IS		 5184 return 0;
5185
010cadf9
PM		 5186err_rif_fdb_op:
5187 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5188 mlxsw_sp_router_port(mlxsw_sp), false);
e4f3c1c1 5189err_fid_bc_flood_set:
0d284818
IS		 5190 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5191 mlxsw_sp_router_port(mlxsw_sp), false);
5192err_fid_mc_flood_set:
e4f3c1c1
IS		 5193 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5194 return err;
5195}
5196
5197static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
5198{
e4f3c1c1 5199 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
010cadf9
PM		 5200 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5201 struct mlxsw_sp_fid *fid = rif->fid;
e4f3c1c1 5202
010cadf9
PM		 5203 mlxsw_sp_fid_rif_set(fid, NULL);
5204 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5205 mlxsw_sp_fid_index(fid), false);
e4f3c1c1
IS		 5206 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5207 mlxsw_sp_router_port(mlxsw_sp), false);
0d284818
IS		 5208 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5209 mlxsw_sp_router_port(mlxsw_sp), false);
e4f3c1c1
IS		 5210 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5211}
5212
5213static struct mlxsw_sp_fid *
5214mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
5215{
5216 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
5217}
5218
5219static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
5220 .type = MLXSW_SP_RIF_TYPE_FID,
5221 .rif_size = sizeof(struct mlxsw_sp_rif),
5222 .configure = mlxsw_sp_rif_fid_configure,
5223 .deconfigure = mlxsw_sp_rif_fid_deconfigure,
5224 .fid_get = mlxsw_sp_rif_fid_fid_get,
5225};
5226
6ddb7426
PM		 5227static struct mlxsw_sp_rif_ipip_lb *
5228mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
5229{
5230 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
5231}
5232
5233static void
5234mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
5235 const struct mlxsw_sp_rif_params *params)
5236{
5237 struct mlxsw_sp_rif_params_ipip_lb *params_lb;
5238 struct mlxsw_sp_rif_ipip_lb *rif_lb;
5239
5240 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
5241 common);
5242 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
5243 rif_lb->lb_config = params_lb->lb_config;
5244}
5245
5246static int
5247mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif,
5248 struct mlxsw_sp_vr *ul_vr, bool enable)
5249{
5250 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
5251 struct mlxsw_sp_rif *rif = &lb_rif->common;
5252 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5253 char ritr_pl[MLXSW_REG_RITR_LEN];
5254 u32 saddr4;
5255
5256 switch (lb_cf.ul_protocol) {
5257 case MLXSW_SP_L3_PROTO_IPV4:
5258 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
5259 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
5260 rif->rif_index, rif->vr_id, rif->dev->mtu);
5261 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
5262 MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
5263 ul_vr->id, saddr4, lb_cf.okey);
5264 break;
5265
5266 case MLXSW_SP_L3_PROTO_IPV6:
5267 return -EAFNOSUPPORT;
5268 }
5269
5270 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5271}
5272
5273static int
5274mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
5275{
5276 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5277 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
5278 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5279 struct mlxsw_sp_vr *ul_vr;
5280 int err;
5281
5282 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id);
5283 if (IS_ERR(ul_vr))
5284 return PTR_ERR(ul_vr);
5285
5286 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true);
5287 if (err)
5288 goto err_loopback_op;
5289
5290 lb_rif->ul_vr_id = ul_vr->id;
5291 ++ul_vr->rif_count;
5292 return 0;
5293
5294err_loopback_op:
5295 mlxsw_sp_vr_put(ul_vr);
5296 return err;
5297}
5298
5299static void mlxsw_sp_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
5300{
5301 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5302 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5303 struct mlxsw_sp_vr *ul_vr;
5304
5305 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
5306 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, false);
5307
5308 --ul_vr->rif_count;
5309 mlxsw_sp_vr_put(ul_vr);
5310}
5311
5312static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_ipip_lb_ops = {
5313 .type = MLXSW_SP_RIF_TYPE_IPIP_LB,
5314 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb),
5315 .setup = mlxsw_sp_rif_ipip_lb_setup,
5316 .configure = mlxsw_sp_rif_ipip_lb_configure,
5317 .deconfigure = mlxsw_sp_rif_ipip_lb_deconfigure,
5318};
5319
e4f3c1c1
IS		 5320static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
5321 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops,
5322 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_ops,
5323 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops,
6ddb7426 5324 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp_rif_ipip_lb_ops,
e4f3c1c1
IS		 5325};
5326
348b8fc3
IS		 5327static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
5328{
5329 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5330
5331 mlxsw_sp->router->rifs = kcalloc(max_rifs,
5332 sizeof(struct mlxsw_sp_rif *),
5333 GFP_KERNEL);
5334 if (!mlxsw_sp->router->rifs)
5335 return -ENOMEM;
e4f3c1c1
IS		 5336
5337 mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
5338
348b8fc3
IS		 5339 return 0;
5340}
5341
5342static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
5343{
5344 int i;
5345
5346 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
5347 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
5348
5349 kfree(mlxsw_sp->router->rifs);
5350}
5351
38ebc0f4
PM		 5352static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
5353{
5354 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
5355 return 0;
5356}
5357
5358static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
5359{
5360}
5361
c3852ef7
IS		 5362static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
5363{
7e39d115 5364 struct mlxsw_sp_router *router;
c3852ef7
IS		 5365
5366 /* Flush pending FIB notifications and then flush the device's
5367 * table before requesting another dump. The FIB notification
5368 * block is unregistered, so no need to take RTNL.
5369 */
5370 mlxsw_core_flush_owq();
7e39d115
IS		 5371 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
5372 mlxsw_sp_router_fib_flush(router->mlxsw_sp);
c3852ef7
IS		 5373}
5374
4724ba56
IS		 5375static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5376{
5377 char rgcr_pl[MLXSW_REG_RGCR_LEN];
5378 u64 max_rifs;
5379 int err;
5380
5381 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
5382 return -EIO;
4724ba56 5383 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
4724ba56 5384
e29237e7 5385 mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
4724ba56
IS		 5386 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
5387 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5388 if (err)
348b8fc3 5389 return err;
4724ba56 5390 return 0;
4724ba56
IS		 5391}
5392
5393static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
5394{
5395 char rgcr_pl[MLXSW_REG_RGCR_LEN];
4724ba56 5396
e29237e7 5397 mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
4724ba56 5398 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
4724ba56
IS		 5399}
5400
b45f64d1
JP		 5401int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5402{
9011b677 5403 struct mlxsw_sp_router *router;
b45f64d1
JP		 5404 int err;
5405
9011b677
IS		 5406 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
5407 if (!router)
5408 return -ENOMEM;
5409 mlxsw_sp->router = router;
5410 router->mlxsw_sp = mlxsw_sp;
5411
5412 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
b45f64d1
JP		 5413 err = __mlxsw_sp_router_init(mlxsw_sp);
5414 if (err)
9011b677 5415 goto err_router_init;
b45f64d1 5416
348b8fc3
IS		 5417 err = mlxsw_sp_rifs_init(mlxsw_sp);
5418 if (err)
5419 goto err_rifs_init;
5420
38ebc0f4
PM		 5421 err = mlxsw_sp_ipips_init(mlxsw_sp);
5422 if (err)
5423 goto err_ipips_init;
5424
9011b677 5425 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
c53b8e1b
IS		 5426 &mlxsw_sp_nexthop_ht_params);
5427 if (err)
5428 goto err_nexthop_ht_init;
5429
9011b677 5430 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
e9ad5e7d
IS		 5431 &mlxsw_sp_nexthop_group_ht_params);
5432 if (err)
5433 goto err_nexthop_group_ht_init;
5434
8494ab06
IS		 5435 err = mlxsw_sp_lpm_init(mlxsw_sp);
5436 if (err)
5437 goto err_lpm_init;
5438
b45f64d1
JP		 5439 err = mlxsw_sp_vrs_init(mlxsw_sp);
5440 if (err)
5441 goto err_vrs_init;
5442
8c9583a8 5443 err = mlxsw_sp_neigh_init(mlxsw_sp);
b45f64d1
JP		 5444 if (err)
5445 goto err_neigh_init;
5446
7e39d115
IS		 5447 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
5448 err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
c3852ef7
IS		 5449 mlxsw_sp_router_fib_dump_flush);
5450 if (err)
5451 goto err_register_fib_notifier;
5452
b45f64d1
JP		 5453 return 0;
5454
c3852ef7
IS		 5455err_register_fib_notifier:
5456 mlxsw_sp_neigh_fini(mlxsw_sp);
b45f64d1
JP		 5457err_neigh_init:
5458 mlxsw_sp_vrs_fini(mlxsw_sp);
5459err_vrs_init:
8494ab06
IS		 5460 mlxsw_sp_lpm_fini(mlxsw_sp);
5461err_lpm_init:
9011b677 5462 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
e9ad5e7d 5463err_nexthop_group_ht_init:
9011b677 5464 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
c53b8e1b 5465err_nexthop_ht_init:
38ebc0f4
PM		 5466 mlxsw_sp_ipips_fini(mlxsw_sp);
5467err_ipips_init:
348b8fc3
IS		 5468 mlxsw_sp_rifs_fini(mlxsw_sp);
5469err_rifs_init:
b45f64d1 5470 __mlxsw_sp_router_fini(mlxsw_sp);
9011b677
IS		 5471err_router_init:
5472 kfree(mlxsw_sp->router);
b45f64d1
JP		 5473 return err;
5474}
5475
5476void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
5477{
7e39d115 5478 unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
b45f64d1
JP		 5479 mlxsw_sp_neigh_fini(mlxsw_sp);
5480 mlxsw_sp_vrs_fini(mlxsw_sp);
8494ab06 5481 mlxsw_sp_lpm_fini(mlxsw_sp);
9011b677
IS		 5482 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
5483 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
38ebc0f4 5484 mlxsw_sp_ipips_fini(mlxsw_sp);
348b8fc3 5485 mlxsw_sp_rifs_fini(mlxsw_sp);
b45f64d1 5486 __mlxsw_sp_router_fini(mlxsw_sp);
9011b677 5487 kfree(mlxsw_sp->router);
b45f64d1 5488}
Linux 6.x block layer and io_uring tree(s)