projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: sched: don't take rtnl lock during flow_action setup
[linux-block.git] / net / sched / cls_api.c
CommitLineData
2874c5fd 1// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4
LT		 2/*
3 * net/sched/cls_api.c Packet classifier API.
4 *
1da177e4
LT		 5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
6 *
7 * Changes:
8 *
9 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
1da177e4
LT		 10 */
11
1da177e4
LT		 12#include <linux/module.h>
13#include <linux/types.h>
14#include <linux/kernel.h>
1da177e4 15#include <linux/string.h>
1da177e4 16#include <linux/errno.h>
33a48927 17#include <linux/err.h>
1da177e4 18#include <linux/skbuff.h>
1da177e4
LT		 19#include <linux/init.h>
20#include <linux/kmod.h>
5a0e3ad6 21#include <linux/slab.h>
48617387 22#include <linux/idr.h>
7f76fa36 23#include <linux/rhashtable.h>
59eb87cb 24#include <linux/jhash.h>
b854272b
DL		 25#include <net/net_namespace.h>
26#include <net/sock.h>
dc5fc579 27#include <net/netlink.h>
1da177e4
LT		 28#include <net/pkt_sched.h>
29#include <net/pkt_cls.h>
e3ab786b 30#include <net/tc_act/tc_pedit.h>
3a7b6861
PNA		 31#include <net/tc_act/tc_mirred.h>
32#include <net/tc_act/tc_vlan.h>
33#include <net/tc_act/tc_tunnel_key.h>
34#include <net/tc_act/tc_csum.h>
35#include <net/tc_act/tc_gact.h>
8c8cfc6e 36#include <net/tc_act/tc_police.h>
a7a7be60 37#include <net/tc_act/tc_sample.h>
3a7b6861 38#include <net/tc_act/tc_skbedit.h>
b57dc7c1 39#include <net/tc_act/tc_ct.h>
6749d590 40#include <net/tc_act/tc_mpls.h>
4e481908 41#include <net/flow_offload.h>
1da177e4 42
e331473f
DC		 43extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
44
1da177e4 45/* The list of all installed classifier types */
36272874 46static LIST_HEAD(tcf_proto_base);
1da177e4
LT		 47
48/* Protects list of registered TC modules. It is pure SMP lock. */
49static DEFINE_RWLOCK(cls_mod_lock);
50
59eb87cb
JH		 51static u32 destroy_obj_hashfn(const struct tcf_proto *tp)
52{
53 return jhash_3words(tp->chain->index, tp->prio,
54 (__force __u32)tp->protocol, 0);
55}
56
57static void tcf_proto_signal_destroying(struct tcf_chain *chain,
58 struct tcf_proto *tp)
59{
60 struct tcf_block *block = chain->block;
61
62 mutex_lock(&block->proto_destroy_lock);
63 hash_add_rcu(block->proto_destroy_ht, &tp->destroy_ht_node,
64 destroy_obj_hashfn(tp));
65 mutex_unlock(&block->proto_destroy_lock);
66}
67
68static bool tcf_proto_cmp(const struct tcf_proto *tp1,
69 const struct tcf_proto *tp2)
70{
71 return tp1->chain->index == tp2->chain->index &&
72 tp1->prio == tp2->prio &&
73 tp1->protocol == tp2->protocol;
74}
75
76static bool tcf_proto_exists_destroying(struct tcf_chain *chain,
77 struct tcf_proto *tp)
78{
79 u32 hash = destroy_obj_hashfn(tp);
80 struct tcf_proto *iter;
81 bool found = false;
82
83 rcu_read_lock();
84 hash_for_each_possible_rcu(chain->block->proto_destroy_ht, iter,
85 destroy_ht_node, hash) {
86 if (tcf_proto_cmp(tp, iter)) {
87 found = true;
88 break;
89 }
90 }
91 rcu_read_unlock();
92
93 return found;
94}
95
96static void
97tcf_proto_signal_destroyed(struct tcf_chain *chain, struct tcf_proto *tp)
98{
99 struct tcf_block *block = chain->block;
100
101 mutex_lock(&block->proto_destroy_lock);
102 if (hash_hashed(&tp->destroy_ht_node))
103 hash_del_rcu(&tp->destroy_ht_node);
104 mutex_unlock(&block->proto_destroy_lock);
105}
106
1da177e4
LT		 107/* Find classifier type by string name */
108
f34e8bff 109static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
1da177e4 110{
dcd76081 111 const struct tcf_proto_ops *t, *res = NULL;
1da177e4
LT		 112
113 if (kind) {
114 read_lock(&cls_mod_lock);
36272874 115 list_for_each_entry(t, &tcf_proto_base, head) {
33a48927 116 if (strcmp(kind, t->kind) == 0) {
dcd76081
ED		 117 if (try_module_get(t->owner))
118 res = t;
1da177e4
LT		 119 break;
120 }
121 }
122 read_unlock(&cls_mod_lock);
123 }
dcd76081 124 return res;
1da177e4
LT		 125}
126
f34e8bff 127static const struct tcf_proto_ops *
12db03b6
VB		 128tcf_proto_lookup_ops(const char *kind, bool rtnl_held,
129 struct netlink_ext_ack *extack)
f34e8bff
JP		 130{
131 const struct tcf_proto_ops *ops;
132
133 ops = __tcf_proto_lookup_ops(kind);
134 if (ops)
135 return ops;
136#ifdef CONFIG_MODULES
12db03b6
VB		 137 if (rtnl_held)
138 rtnl_unlock();
f34e8bff 139 request_module("cls_%s", kind);
12db03b6
VB		 140 if (rtnl_held)
141 rtnl_lock();
f34e8bff
JP		 142 ops = __tcf_proto_lookup_ops(kind);
143 /* We dropped the RTNL semaphore in order to perform
144 * the module load. So, even if we succeeded in loading
145 * the module we have to replay the request. We indicate
146 * this using -EAGAIN.
147 */
148 if (ops) {
149 module_put(ops->owner);
150 return ERR_PTR(-EAGAIN);
151 }
152#endif
153 NL_SET_ERR_MSG(extack, "TC classifier not found");
154 return ERR_PTR(-ENOENT);
155}
156
1da177e4
LT		 157/* Register(unregister) new classifier type */
158
159int register_tcf_proto_ops(struct tcf_proto_ops *ops)
160{
36272874 161 struct tcf_proto_ops *t;
1da177e4
LT		 162 int rc = -EEXIST;
163
164 write_lock(&cls_mod_lock);
36272874 165 list_for_each_entry(t, &tcf_proto_base, head)
1da177e4
LT		 166 if (!strcmp(ops->kind, t->kind))
167 goto out;
168
36272874 169 list_add_tail(&ops->head, &tcf_proto_base);
1da177e4
LT		 170 rc = 0;
171out:
172 write_unlock(&cls_mod_lock);
173 return rc;
174}
aa767bfe 175EXPORT_SYMBOL(register_tcf_proto_ops);
1da177e4 176
7aa0045d
CW		 177static struct workqueue_struct *tc_filter_wq;
178
1da177e4
LT		 179int unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
180{
36272874 181 struct tcf_proto_ops *t;
1da177e4
LT		 182 int rc = -ENOENT;
183
c78e1746
DB		 184 /* Wait for outstanding call_rcu()s, if any, from a
185 * tcf_proto_ops's destroy() handler.
186 */
187 rcu_barrier();
7aa0045d 188 flush_workqueue(tc_filter_wq);
c78e1746 189
1da177e4 190 write_lock(&cls_mod_lock);
dcd76081
ED		 191 list_for_each_entry(t, &tcf_proto_base, head) {
192 if (t == ops) {
193 list_del(&t->head);
194 rc = 0;
1da177e4 195 break;
dcd76081
ED		 196 }
197 }
1da177e4
LT		 198 write_unlock(&cls_mod_lock);
199 return rc;
200}
aa767bfe 201EXPORT_SYMBOL(unregister_tcf_proto_ops);
1da177e4 202
aaa908ff 203bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
7aa0045d 204{
aaa908ff
CW		 205 INIT_RCU_WORK(rwork, func);
206 return queue_rcu_work(tc_filter_wq, rwork);
7aa0045d
CW		 207}
208EXPORT_SYMBOL(tcf_queue_work);
209
1da177e4
LT		 210/* Select new prio value from the range, managed by kernel. */
211
aa767bfe 212static inline u32 tcf_auto_prio(struct tcf_proto *tp)
1da177e4 213{
aa767bfe 214 u32 first = TC_H_MAKE(0xC0000000U, 0U);
1da177e4
LT		 215
216 if (tp)
cc7ec456 217 first = tp->prio - 1;
1da177e4 218
7961973a 219 return TC_H_MAJ(first);
1da177e4
LT		 220}
221
6f96c3c6
CW		 222static bool tcf_proto_check_kind(struct nlattr *kind, char *name)
223{
224 if (kind)
225 return nla_strlcpy(name, kind, IFNAMSIZ) >= IFNAMSIZ;
226 memset(name, 0, IFNAMSIZ);
227 return false;
228}
229
470502de
VB		 230static bool tcf_proto_is_unlocked(const char *kind)
231{
232 const struct tcf_proto_ops *ops;
233 bool ret;
234
6f96c3c6
CW		 235 if (strlen(kind) == 0)
236 return false;
237
470502de
VB		 238 ops = tcf_proto_lookup_ops(kind, false, NULL);
239 /* On error return false to take rtnl lock. Proto lookup/create
240 * functions will perform lookup again and properly handle errors.
241 */
242 if (IS_ERR(ops))
243 return false;
244
245 ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED);
246 module_put(ops->owner);
247 return ret;
248}
249
33a48927 250static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
c35a4acc 251 u32 prio, struct tcf_chain *chain,
12db03b6 252 bool rtnl_held,
c35a4acc 253 struct netlink_ext_ack *extack)
33a48927
JP		 254{
255 struct tcf_proto *tp;
256 int err;
257
258 tp = kzalloc(sizeof(*tp), GFP_KERNEL);
259 if (!tp)
260 return ERR_PTR(-ENOBUFS);
261
12db03b6 262 tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack);
f34e8bff
JP		 263 if (IS_ERR(tp->ops)) {
264 err = PTR_ERR(tp->ops);
d68d75fd 265 goto errout;
33a48927
JP		 266 }
267 tp->classify = tp->ops->classify;
268 tp->protocol = protocol;
269 tp->prio = prio;
5bc17018 270 tp->chain = chain;
8b64678e 271 spin_lock_init(&tp->lock);
4dbfa766 272 refcount_set(&tp->refcnt, 1);
33a48927
JP		 273
274 err = tp->ops->init(tp);
275 if (err) {
276 module_put(tp->ops->owner);
277 goto errout;
278 }
279 return tp;
280
281errout:
282 kfree(tp);
283 return ERR_PTR(err);
284}
285
4dbfa766
VB		 286static void tcf_proto_get(struct tcf_proto *tp)
287{
288 refcount_inc(&tp->refcnt);
289}
290
291static void tcf_chain_put(struct tcf_chain *chain);
292
12db03b6 293static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held,
59eb87cb 294 bool sig_destroy, struct netlink_ext_ack *extack)
cf1facda 295{
12db03b6 296 tp->ops->destroy(tp, rtnl_held, extack);
59eb87cb
JH		 297 if (sig_destroy)
298 tcf_proto_signal_destroyed(tp->chain, tp);
4dbfa766 299 tcf_chain_put(tp->chain);
763dbf63
WC		 300 module_put(tp->ops->owner);
301 kfree_rcu(tp, rcu);
cf1facda
JP		 302}
303
12db03b6 304static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held,
4dbfa766
VB		 305 struct netlink_ext_ack *extack)
306{
307 if (refcount_dec_and_test(&tp->refcnt))
59eb87cb 308 tcf_proto_destroy(tp, rtnl_held, true, extack);
4dbfa766
VB		 309}
310
a5b72a08 311static bool tcf_proto_check_delete(struct tcf_proto *tp)
8b64678e 312{
a5b72a08
DC		 313 if (tp->ops->delete_empty)
314 return tp->ops->delete_empty(tp);
8b64678e 315
a5b72a08 316 tp->deleting = true;
8b64678e
VB		 317 return tp->deleting;
318}
319
320static void tcf_proto_mark_delete(struct tcf_proto *tp)
321{
322 spin_lock(&tp->lock);
323 tp->deleting = true;
324 spin_unlock(&tp->lock);
325}
326
327static bool tcf_proto_is_deleting(struct tcf_proto *tp)
328{
329 bool deleting;
330
331 spin_lock(&tp->lock);
332 deleting = tp->deleting;
333 spin_unlock(&tp->lock);
334
335 return deleting;
336}
337
c266f64d
VB		 338#define ASSERT_BLOCK_LOCKED(block) \
339 lockdep_assert_held(&(block)->lock)
340
a9b19443
JP		 341struct tcf_filter_chain_list_item {
342 struct list_head list;
343 tcf_chain_head_change_t *chain_head_change;
344 void *chain_head_change_priv;
345};
346
5bc17018
JP		 347static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
348 u32 chain_index)
2190d1d0 349{
5bc17018
JP		 350 struct tcf_chain *chain;
351
c266f64d
VB		 352 ASSERT_BLOCK_LOCKED(block);
353
5bc17018
JP		 354 chain = kzalloc(sizeof(*chain), GFP_KERNEL);
355 if (!chain)
356 return NULL;
357 list_add_tail(&chain->list, &block->chain_list);
ed76f5ed 358 mutex_init(&chain->filter_chain_lock);
5bc17018
JP		 359 chain->block = block;
360 chain->index = chain_index;
e2ef7544 361 chain->refcnt = 1;
f71e0ca4
JP		 362 if (!chain->index)
363 block->chain0.chain = chain;
5bc17018 364 return chain;
2190d1d0
JP		 365}
366
a9b19443
JP		 367static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
368 struct tcf_proto *tp_head)
369{
370 if (item->chain_head_change)
371 item->chain_head_change(tp_head, item->chain_head_change_priv);
372}
f71e0ca4
JP		 373
374static void tcf_chain0_head_change(struct tcf_chain *chain,
375 struct tcf_proto *tp_head)
c7eb7d72 376{
a9b19443 377 struct tcf_filter_chain_list_item *item;
f71e0ca4 378 struct tcf_block *block = chain->block;
a9b19443 379
f71e0ca4
JP		 380 if (chain->index)
381 return;
165f0135
VB		 382
383 mutex_lock(&block->lock);
f71e0ca4 384 list_for_each_entry(item, &block->chain0.filter_chain_list, list)
a9b19443 385 tcf_chain_head_change_item(item, tp_head);
165f0135 386 mutex_unlock(&block->lock);
c7eb7d72
JP		 387}
388
c266f64d
VB		 389/* Returns true if block can be safely freed. */
390
391static bool tcf_chain_detach(struct tcf_chain *chain)
f93e1cdc 392{
efbf7897
CW		 393 struct tcf_block *block = chain->block;
394
c266f64d
VB		 395 ASSERT_BLOCK_LOCKED(block);
396
e2ef7544 397 list_del(&chain->list);
f71e0ca4
JP		 398 if (!chain->index)
399 block->chain0.chain = NULL;
c266f64d
VB		 400
401 if (list_empty(&block->chain_list) &&
402 refcount_read(&block->refcnt) == 0)
403 return true;
404
405 return false;
406}
407
408static void tcf_block_destroy(struct tcf_block *block)
409{
410 mutex_destroy(&block->lock);
59eb87cb 411 mutex_destroy(&block->proto_destroy_lock);
c266f64d
VB		 412 kfree_rcu(block, rcu);
413}
414
415static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
416{
417 struct tcf_block *block = chain->block;
418
ed76f5ed 419 mutex_destroy(&chain->filter_chain_lock);
ee3bbfe8 420 kfree_rcu(chain, rcu);
c266f64d
VB		 421 if (free_block)
422 tcf_block_destroy(block);
e2ef7544 423}
744a4cf6 424
e2ef7544
CW		 425static void tcf_chain_hold(struct tcf_chain *chain)
426{
c266f64d
VB		 427 ASSERT_BLOCK_LOCKED(chain->block);
428
e2ef7544 429 ++chain->refcnt;
2190d1d0
JP		 430}
431
3d32f4c5 432static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
1f3ed383 433{
c266f64d
VB		 434 ASSERT_BLOCK_LOCKED(chain->block);
435
1f3ed383 436 /* In case all the references are action references, this
3d32f4c5 437 * chain should not be shown to the user.
1f3ed383
JP		 438 */
439 return chain->refcnt == chain->action_refcnt;
440}
441
32a4f5ec
JP		 442static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
443 u32 chain_index)
5bc17018
JP		 444{
445 struct tcf_chain *chain;
446
c266f64d
VB		 447 ASSERT_BLOCK_LOCKED(block);
448
5bc17018 449 list_for_each_entry(chain, &block->chain_list, list) {
32a4f5ec 450 if (chain->index == chain_index)
e2ef7544 451 return chain;
32a4f5ec
JP		 452 }
453 return NULL;
454}
455
456static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
457 u32 seq, u16 flags, int event, bool unicast);
458
53681407
JP		 459static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
460 u32 chain_index, bool create,
461 bool by_act)
32a4f5ec 462{
c266f64d
VB		 463 struct tcf_chain *chain = NULL;
464 bool is_first_reference;
32a4f5ec 465
c266f64d
VB		 466 mutex_lock(&block->lock);
467 chain = tcf_chain_lookup(block, chain_index);
32a4f5ec
JP		 468 if (chain) {
469 tcf_chain_hold(chain);
53681407
JP		 470 } else {
471 if (!create)
c266f64d 472 goto errout;
53681407
JP		 473 chain = tcf_chain_create(block, chain_index);
474 if (!chain)
c266f64d 475 goto errout;
5bc17018 476 }
80532384 477
53681407
JP		 478 if (by_act)
479 ++chain->action_refcnt;
c266f64d
VB		 480 is_first_reference = chain->refcnt - chain->action_refcnt == 1;
481 mutex_unlock(&block->lock);
53681407
JP		 482
483 /* Send notification only in case we got the first
484 * non-action reference. Until then, the chain acts only as
485 * a placeholder for actions pointing to it and user ought
486 * not know about them.
487 */
c266f64d 488 if (is_first_reference && !by_act)
53681407
JP		 489 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
490 RTM_NEWCHAIN, false);
491
32a4f5ec 492 return chain;
c266f64d
VB		 493
494errout:
495 mutex_unlock(&block->lock);
496 return chain;
5bc17018 497}
53681407 498
290b1c8b
JP		 499static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
500 bool create)
53681407
JP		 501{
502 return __tcf_chain_get(block, chain_index, create, false);
503}
5bc17018 504
1f3ed383
JP		 505struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
506{
53681407 507 return __tcf_chain_get(block, chain_index, true, true);
1f3ed383
JP		 508}
509EXPORT_SYMBOL(tcf_chain_get_by_act);
510
a5654820
VB		 511static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
512 void *tmplt_priv);
513static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
514 void *tmplt_priv, u32 chain_index,
515 struct tcf_block *block, struct sk_buff *oskb,
516 u32 seq, u16 flags, bool unicast);
9f407f17 517
91052fa1
VB		 518static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
519 bool explicitly_created)
5bc17018 520{
c266f64d 521 struct tcf_block *block = chain->block;
a5654820 522 const struct tcf_proto_ops *tmplt_ops;
b62989fc 523 bool free_block = false;
c266f64d 524 unsigned int refcnt;
a5654820 525 void *tmplt_priv;
c266f64d
VB		 526
527 mutex_lock(&block->lock);
91052fa1
VB		 528 if (explicitly_created) {
529 if (!chain->explicitly_created) {
530 mutex_unlock(&block->lock);
531 return;
532 }
533 chain->explicitly_created = false;
534 }
535
53681407
JP		 536 if (by_act)
537 chain->action_refcnt--;
c266f64d
VB		 538
539 /* tc_chain_notify_delete can't be called while holding block lock.
540 * However, when block is unlocked chain can be changed concurrently, so
541 * save these to temporary variables.
542 */
543 refcnt = --chain->refcnt;
a5654820
VB		 544 tmplt_ops = chain->tmplt_ops;
545 tmplt_priv = chain->tmplt_priv;
53681407
JP		 546
547 /* The last dropped non-action reference will trigger notification. */
b62989fc
VB		 548 if (refcnt - chain->action_refcnt == 0 && !by_act) {
549 tc_chain_notify_delete(tmplt_ops, tmplt_priv, chain->index,
a5654820 550 block, NULL, 0, 0, false);
726d0612
VB		 551 /* Last reference to chain, no need to lock. */
552 chain->flushing = false;
553 }
53681407 554
b62989fc
VB		 555 if (refcnt == 0)
556 free_block = tcf_chain_detach(chain);
557 mutex_unlock(&block->lock);
558
c266f64d 559 if (refcnt == 0) {
a5654820 560 tc_chain_tmplt_del(tmplt_ops, tmplt_priv);
c266f64d 561 tcf_chain_destroy(chain, free_block);
32a4f5ec 562 }
5bc17018 563}
53681407 564
290b1c8b 565static void tcf_chain_put(struct tcf_chain *chain)
53681407 566{
91052fa1 567 __tcf_chain_put(chain, false, false);
53681407 568}
5bc17018 569
1f3ed383
JP		 570void tcf_chain_put_by_act(struct tcf_chain *chain)
571{
91052fa1 572 __tcf_chain_put(chain, true, false);
1f3ed383
JP		 573}
574EXPORT_SYMBOL(tcf_chain_put_by_act);
575
32a4f5ec
JP		 576static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
577{
91052fa1 578 __tcf_chain_put(chain, false, true);
32a4f5ec
JP		 579}
580
12db03b6 581static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held)
290b1c8b 582{
4dbfa766 583 struct tcf_proto *tp, *tp_next;
290b1c8b 584
ed76f5ed
VB		 585 mutex_lock(&chain->filter_chain_lock);
586 tp = tcf_chain_dereference(chain->filter_chain, chain);
59eb87cb
JH		 587 while (tp) {
588 tp_next = rcu_dereference_protected(tp->next, 1);
589 tcf_proto_signal_destroying(chain, tp);
590 tp = tp_next;
591 }
592 tp = tcf_chain_dereference(chain->filter_chain, chain);
4dbfa766 593 RCU_INIT_POINTER(chain->filter_chain, NULL);
290b1c8b 594 tcf_chain0_head_change(chain, NULL);
726d0612 595 chain->flushing = true;
ed76f5ed
VB		 596 mutex_unlock(&chain->filter_chain_lock);
597
290b1c8b 598 while (tp) {
4dbfa766 599 tp_next = rcu_dereference_protected(tp->next, 1);
12db03b6 600 tcf_proto_put(tp, rtnl_held, NULL);
4dbfa766 601 tp = tp_next;
290b1c8b
JP		 602 }
603}
604
59094b1e
PNA		 605static int tcf_block_setup(struct tcf_block *block,
606 struct flow_block_offload *bo);
607
25a443f7
JH		 608static void tc_indr_block_cmd(struct net_device *dev, struct tcf_block *block,
609 flow_indr_block_bind_cb_t *cb, void *cb_priv,
610 enum flow_block_command command, bool ingress)
7f76fa36 611{
955bcb6e 612 struct flow_block_offload bo = {
7f76fa36 613 .command = command,
25a443f7
JH		 614 .binder_type = ingress ?
615 FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS :
616 FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS,
242453c2 617 .net = dev_net(dev),
618 .block_shared = tcf_block_non_null_shared(block),
7f76fa36 619 };
59094b1e 620 INIT_LIST_HEAD(&bo.cb_list);
7f76fa36 621
242453c2 622 if (!block)
7f76fa36
JH		 623 return;
624
242453c2 625 bo.block = &block->flow_block;
14bfb13f 626
4f8116c8 627 down_write(&block->cb_lock);
242453c2 628 cb(dev, cb_priv, TC_SETUP_BLOCK, &bo);
629
630 tcf_block_setup(block, &bo);
4f8116c8 631 up_write(&block->cb_lock);
7f76fa36
JH		 632}
633
25a443f7 634static struct tcf_block *tc_dev_block(struct net_device *dev, bool ingress)
7f76fa36 635{
4e481908 636 const struct Qdisc_class_ops *cops;
25a443f7 637 const struct Qdisc_ops *ops;
4e481908 638 struct Qdisc *qdisc;
7f76fa36 639
4e481908 640 if (!dev_ingress_queue(dev))
641 return NULL;
7f76fa36 642
4e481908 643 qdisc = dev_ingress_queue(dev)->qdisc_sleeping;
644 if (!qdisc)
645 return NULL;
7f76fa36 646
25a443f7
JH		 647 ops = qdisc->ops;
648 if (!ops)
649 return NULL;
650
651 if (!ingress && !strcmp("ingress", ops->id))
652 return NULL;
653
654 cops = ops->cl_ops;
4e481908 655 if (!cops)
656 return NULL;
7f76fa36 657
4e481908 658 if (!cops->tcf_block)
659 return NULL;
7f76fa36 660
25a443f7
JH		 661 return cops->tcf_block(qdisc,
662 ingress ? TC_H_MIN_INGRESS : TC_H_MIN_EGRESS,
663 NULL);
7f76fa36 664}
7f76fa36 665
25a443f7
JH		 666static void tc_indr_block_get_and_cmd(struct net_device *dev,
667 flow_indr_block_bind_cb_t *cb,
668 void *cb_priv,
669 enum flow_block_command command)
e4da9102 670{
25a443f7
JH		 671 struct tcf_block *block;
672
673 block = tc_dev_block(dev, true);
674 tc_indr_block_cmd(dev, block, cb, cb_priv, command, true);
e4da9102 675
25a443f7
JH		 676 block = tc_dev_block(dev, false);
677 tc_indr_block_cmd(dev, block, cb, cb_priv, command, false);
e4da9102 678}
679
4e481908 680static void tc_indr_block_call(struct tcf_block *block,
681 struct net_device *dev,
7f76fa36 682 struct tcf_block_ext_info *ei,
9c0e189e 683 enum flow_block_command command,
7f76fa36
JH		 684 struct netlink_ext_ack *extack)
685{
955bcb6e 686 struct flow_block_offload bo = {
7f76fa36
JH		 687 .command = command,
688 .binder_type = ei->binder_type,
da3eeb90 689 .net = dev_net(dev),
14bfb13f 690 .block = &block->flow_block,
955bcb6e 691 .block_shared = tcf_block_shared(block),
7f76fa36
JH		 692 .extack = extack,
693 };
59094b1e 694 INIT_LIST_HEAD(&bo.cb_list);
7f76fa36 695
1150ab0f 696 flow_indr_block_call(dev, &bo, command);
59094b1e 697 tcf_block_setup(block, &bo);
7f76fa36
JH		 698}
699
caa72601
JP		 700static bool tcf_block_offload_in_use(struct tcf_block *block)
701{
97394bef 702 return atomic_read(&block->offloadcnt);
caa72601
JP		 703}
704
705static int tcf_block_offload_cmd(struct tcf_block *block,
706 struct net_device *dev,
707 struct tcf_block_ext_info *ei,
9c0e189e 708 enum flow_block_command command,
60513bd8 709 struct netlink_ext_ack *extack)
8c4083b3 710{
955bcb6e 711 struct flow_block_offload bo = {};
59094b1e 712 int err;
8c4083b3 713
da3eeb90 714 bo.net = dev_net(dev);
8c4083b3
JP		 715 bo.command = command;
716 bo.binder_type = ei->binder_type;
14bfb13f 717 bo.block = &block->flow_block;
955bcb6e 718 bo.block_shared = tcf_block_shared(block);
60513bd8 719 bo.extack = extack;
59094b1e
PNA		 720 INIT_LIST_HEAD(&bo.cb_list);
721
722 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
723 if (err < 0)
724 return err;
725
726 return tcf_block_setup(block, &bo);
8c4083b3
JP		 727}
728
caa72601 729static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
60513bd8
JH		 730 struct tcf_block_ext_info *ei,
731 struct netlink_ext_ack *extack)
8c4083b3 732{
caa72601
JP		 733 struct net_device *dev = q->dev_queue->dev;
734 int err;
735
4f8116c8 736 down_write(&block->cb_lock);
caa72601
JP		 737 if (!dev->netdev_ops->ndo_setup_tc)
738 goto no_offload_dev_inc;
739
740 /* If tc offload feature is disabled and the block we try to bind
741 * to already has some offloaded filters, forbid to bind.
742 */
60513bd8
JH		 743 if (!tc_can_offload(dev) && tcf_block_offload_in_use(block)) {
744 NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
4f8116c8
VB		 745 err = -EOPNOTSUPP;
746 goto err_unlock;
60513bd8 747 }
caa72601 748
9c0e189e 749 err = tcf_block_offload_cmd(block, dev, ei, FLOW_BLOCK_BIND, extack);
caa72601
JP		 750 if (err == -EOPNOTSUPP)
751 goto no_offload_dev_inc;
7f76fa36 752 if (err)
4f8116c8 753 goto err_unlock;
7f76fa36 754
9c0e189e 755 tc_indr_block_call(block, dev, ei, FLOW_BLOCK_BIND, extack);
4f8116c8 756 up_write(&block->cb_lock);
7f76fa36 757 return 0;
caa72601
JP		 758
759no_offload_dev_inc:
4f8116c8
VB		 760 if (tcf_block_offload_in_use(block)) {
761 err = -EOPNOTSUPP;
762 goto err_unlock;
763 }
764 err = 0;
caa72601 765 block->nooffloaddevcnt++;
9c0e189e 766 tc_indr_block_call(block, dev, ei, FLOW_BLOCK_BIND, extack);
4f8116c8
VB		 767err_unlock:
768 up_write(&block->cb_lock);
769 return err;
8c4083b3
JP		 770}
771
772static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
773 struct tcf_block_ext_info *ei)
774{
caa72601
JP		 775 struct net_device *dev = q->dev_queue->dev;
776 int err;
777
4f8116c8 778 down_write(&block->cb_lock);
9c0e189e 779 tc_indr_block_call(block, dev, ei, FLOW_BLOCK_UNBIND, NULL);
7f76fa36 780
caa72601
JP		 781 if (!dev->netdev_ops->ndo_setup_tc)
782 goto no_offload_dev_dec;
9c0e189e 783 err = tcf_block_offload_cmd(block, dev, ei, FLOW_BLOCK_UNBIND, NULL);
caa72601
JP		 784 if (err == -EOPNOTSUPP)
785 goto no_offload_dev_dec;
4f8116c8 786 up_write(&block->cb_lock);
caa72601
JP		 787 return;
788
789no_offload_dev_dec:
790 WARN_ON(block->nooffloaddevcnt-- == 0);
4f8116c8 791 up_write(&block->cb_lock);
8c4083b3
JP		 792}
793
a9b19443 794static int
f71e0ca4
JP		 795tcf_chain0_head_change_cb_add(struct tcf_block *block,
796 struct tcf_block_ext_info *ei,
797 struct netlink_ext_ack *extack)
a9b19443
JP		 798{
799 struct tcf_filter_chain_list_item *item;
165f0135 800 struct tcf_chain *chain0;
a9b19443
JP		 801
802 item = kmalloc(sizeof(*item), GFP_KERNEL);
803 if (!item) {
804 NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed");
805 return -ENOMEM;
806 }
807 item->chain_head_change = ei->chain_head_change;
808 item->chain_head_change_priv = ei->chain_head_change_priv;
165f0135
VB		 809
810 mutex_lock(&block->lock);
811 chain0 = block->chain0.chain;
ed76f5ed
VB		 812 if (chain0)
813 tcf_chain_hold(chain0);
814 else
815 list_add(&item->list, &block->chain0.filter_chain_list);
165f0135
VB		 816 mutex_unlock(&block->lock);
817
ed76f5ed
VB		 818 if (chain0) {
819 struct tcf_proto *tp_head;
820
821 mutex_lock(&chain0->filter_chain_lock);
822
823 tp_head = tcf_chain_dereference(chain0->filter_chain, chain0);
824 if (tp_head)
825 tcf_chain_head_change_item(item, tp_head);
826
827 mutex_lock(&block->lock);
828 list_add(&item->list, &block->chain0.filter_chain_list);
829 mutex_unlock(&block->lock);
830
831 mutex_unlock(&chain0->filter_chain_lock);
832 tcf_chain_put(chain0);
833 }
834
a9b19443
JP		 835 return 0;
836}
837
838static void
f71e0ca4
JP		 839tcf_chain0_head_change_cb_del(struct tcf_block *block,
840 struct tcf_block_ext_info *ei)
a9b19443
JP		 841{
842 struct tcf_filter_chain_list_item *item;
843
165f0135 844 mutex_lock(&block->lock);
f71e0ca4 845 list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
a9b19443
JP		 846 if ((!ei->chain_head_change && !ei->chain_head_change_priv) ||
847 (item->chain_head_change == ei->chain_head_change &&
848 item->chain_head_change_priv == ei->chain_head_change_priv)) {
165f0135 849 if (block->chain0.chain)
f71e0ca4 850 tcf_chain_head_change_item(item, NULL);
a9b19443 851 list_del(&item->list);
165f0135
VB		 852 mutex_unlock(&block->lock);
853
a9b19443
JP		 854 kfree(item);
855 return;
856 }
857 }
165f0135 858 mutex_unlock(&block->lock);
a9b19443
JP		 859 WARN_ON(1);
860}
861
48617387 862struct tcf_net {
ab281629 863 spinlock_t idr_lock; /* Protects idr */
48617387
JP		 864 struct idr idr;
865};
866
867static unsigned int tcf_net_id;
868
869static int tcf_block_insert(struct tcf_block *block, struct net *net,
bb047ddd 870 struct netlink_ext_ack *extack)
a9b19443 871{
48617387 872 struct tcf_net *tn = net_generic(net, tcf_net_id);
ab281629
VB		 873 int err;
874
875 idr_preload(GFP_KERNEL);
876 spin_lock(&tn->idr_lock);
877 err = idr_alloc_u32(&tn->idr, block, &block->index, block->index,
878 GFP_NOWAIT);
879 spin_unlock(&tn->idr_lock);
880 idr_preload_end();
48617387 881
ab281629 882 return err;
a9b19443
JP		 883}
884
48617387
JP		 885static void tcf_block_remove(struct tcf_block *block, struct net *net)
886{
887 struct tcf_net *tn = net_generic(net, tcf_net_id);
888
ab281629 889 spin_lock(&tn->idr_lock);
9c160941 890 idr_remove(&tn->idr, block->index);
ab281629 891 spin_unlock(&tn->idr_lock);
48617387
JP		 892}
893
894static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
bb047ddd 895 u32 block_index,
48617387 896 struct netlink_ext_ack *extack)
6529eaba 897{
48617387 898 struct tcf_block *block;
6529eaba 899
48617387 900 block = kzalloc(sizeof(*block), GFP_KERNEL);
8d1a77f9
AA		 901 if (!block) {
902 NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
48617387 903 return ERR_PTR(-ENOMEM);
8d1a77f9 904 }
c266f64d 905 mutex_init(&block->lock);
59eb87cb 906 mutex_init(&block->proto_destroy_lock);
4f8116c8 907 init_rwsem(&block->cb_lock);
14bfb13f 908 flow_block_init(&block->flow_block);
5bc17018 909 INIT_LIST_HEAD(&block->chain_list);
f36fe1c4 910 INIT_LIST_HEAD(&block->owner_list);
f71e0ca4 911 INIT_LIST_HEAD(&block->chain0.filter_chain_list);
acb67442 912
cfebd7e2 913 refcount_set(&block->refcnt, 1);
48617387 914 block->net = net;
bb047ddd
JP		 915 block->index = block_index;
916
917 /* Don't store q pointer for blocks which are shared */
918 if (!tcf_block_shared(block))
919 block->q = q;
48617387 920 return block;
48617387
JP		 921}
922
923static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
924{
925 struct tcf_net *tn = net_generic(net, tcf_net_id);
926
322d884b 927 return idr_find(&tn->idr, block_index);
48617387
JP		 928}
929
0607e439
VB		 930static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
931{
932 struct tcf_block *block;
933
934 rcu_read_lock();
935 block = tcf_block_lookup(net, block_index);
936 if (block && !refcount_inc_not_zero(&block->refcnt))
937 block = NULL;
938 rcu_read_unlock();
939
940 return block;
941}
942
bbf73830
VB		 943static struct tcf_chain *
944__tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
f0023436 945{
bbf73830
VB		 946 mutex_lock(&block->lock);
947 if (chain)
948 chain = list_is_last(&chain->list, &block->chain_list) ?
949 NULL : list_next_entry(chain, list);
950 else
951 chain = list_first_entry_or_null(&block->chain_list,
952 struct tcf_chain, list);
f0023436 953
bbf73830
VB		 954 /* skip all action-only chains */
955 while (chain && tcf_chain_held_by_acts_only(chain))
956 chain = list_is_last(&chain->list, &block->chain_list) ?
957 NULL : list_next_entry(chain, list);
958
959 if (chain)
f0023436 960 tcf_chain_hold(chain);
bbf73830 961 mutex_unlock(&block->lock);
f0023436 962
bbf73830 963 return chain;
f0023436
VB		 964}
965
bbf73830
VB		 966/* Function to be used by all clients that want to iterate over all chains on
967 * block. It properly obtains block->lock and takes reference to chain before
968 * returning it. Users of this function must be tolerant to concurrent chain
969 * insertion/deletion or ensure that no concurrent chain modification is
970 * possible. Note that all netlink dump callbacks cannot guarantee to provide
971 * consistent dump because rtnl lock is released each time skb is filled with
972 * data and sent to user-space.
973 */
974
975struct tcf_chain *
976tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
f0023436 977{
bbf73830 978 struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain);
f0023436 979
bbf73830 980 if (chain)
f0023436 981 tcf_chain_put(chain);
bbf73830
VB		 982
983 return chain_next;
984}
985EXPORT_SYMBOL(tcf_get_next_chain);
986
fe2923af
VB		 987static struct tcf_proto *
988__tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
989{
8b64678e
VB		 990 u32 prio = 0;
991
fe2923af
VB		 992 ASSERT_RTNL();
993 mutex_lock(&chain->filter_chain_lock);
994
8b64678e 995 if (!tp) {
fe2923af 996 tp = tcf_chain_dereference(chain->filter_chain, chain);
8b64678e
VB		 997 } else if (tcf_proto_is_deleting(tp)) {
998 /* 'deleting' flag is set and chain->filter_chain_lock was
999 * unlocked, which means next pointer could be invalid. Restart
1000 * search.
1001 */
1002 prio = tp->prio + 1;
1003 tp = tcf_chain_dereference(chain->filter_chain, chain);
1004
1005 for (; tp; tp = tcf_chain_dereference(tp->next, chain))
1006 if (!tp->deleting && tp->prio >= prio)
1007 break;
1008 } else {
fe2923af 1009 tp = tcf_chain_dereference(tp->next, chain);
8b64678e 1010 }
fe2923af
VB		 1011
1012 if (tp)
1013 tcf_proto_get(tp);
1014
1015 mutex_unlock(&chain->filter_chain_lock);
1016
1017 return tp;
1018}
1019
1020/* Function to be used by all clients that want to iterate over all tp's on
1021 * chain. Users of this function must be tolerant to concurrent tp
1022 * insertion/deletion or ensure that no concurrent chain modification is
1023 * possible. Note that all netlink dump callbacks cannot guarantee to provide
1024 * consistent dump because rtnl lock is released each time skb is filled with
1025 * data and sent to user-space.
1026 */
1027
1028struct tcf_proto *
12db03b6
VB		 1029tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp,
1030 bool rtnl_held)
fe2923af
VB		 1031{
1032 struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp);
1033
1034 if (tp)
12db03b6 1035 tcf_proto_put(tp, rtnl_held, NULL);
fe2923af
VB		 1036
1037 return tp_next;
1038}
1039EXPORT_SYMBOL(tcf_get_next_proto);
1040
12db03b6 1041static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held)
bbf73830
VB		 1042{
1043 struct tcf_chain *chain;
1044
1045 /* Last reference to block. At this point chains cannot be added or
1046 * removed concurrently.
1047 */
1048 for (chain = tcf_get_next_chain(block, NULL);
1049 chain;
1050 chain = tcf_get_next_chain(block, chain)) {
1051 tcf_chain_put_explicitly_created(chain);
12db03b6 1052 tcf_chain_flush(chain, rtnl_held);
f0023436
VB		 1053 }
1054}
1055
18d3eefb
VB		 1056/* Lookup Qdisc and increments its reference counter.
1057 * Set parent, if necessary.
1058 */
1059
1060static int __tcf_qdisc_find(struct net *net, struct Qdisc **q,
1061 u32 *parent, int ifindex, bool rtnl_held,
1062 struct netlink_ext_ack *extack)
1063{
1064 const struct Qdisc_class_ops *cops;
1065 struct net_device *dev;
1066 int err = 0;
1067
1068 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1069 return 0;
1070
1071 rcu_read_lock();
1072
1073 /* Find link */
1074 dev = dev_get_by_index_rcu(net, ifindex);
1075 if (!dev) {
1076 rcu_read_unlock();
1077 return -ENODEV;
1078 }
1079
1080 /* Find qdisc */
1081 if (!*parent) {
1082 *q = dev->qdisc;
1083 *parent = (*q)->handle;
1084 } else {
1085 *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
1086 if (!*q) {
1087 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1088 err = -EINVAL;
1089 goto errout_rcu;
1090 }
1091 }
1092
1093 *q = qdisc_refcount_inc_nz(*q);
1094 if (!*q) {
1095 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1096 err = -EINVAL;
1097 goto errout_rcu;
1098 }
1099
1100 /* Is it classful? */
1101 cops = (*q)->ops->cl_ops;
1102 if (!cops) {
1103 NL_SET_ERR_MSG(extack, "Qdisc not classful");
1104 err = -EINVAL;
1105 goto errout_qdisc;
1106 }
1107
1108 if (!cops->tcf_block) {
1109 NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
1110 err = -EOPNOTSUPP;
1111 goto errout_qdisc;
1112 }
1113
1114errout_rcu:
1115 /* At this point we know that qdisc is not noop_qdisc,
1116 * which means that qdisc holds a reference to net_device
1117 * and we hold a reference to qdisc, so it is safe to release
1118 * rcu read lock.
1119 */
1120 rcu_read_unlock();
1121 return err;
1122
1123errout_qdisc:
1124 rcu_read_unlock();
1125
1126 if (rtnl_held)
1127 qdisc_put(*q);
1128 else
1129 qdisc_put_unlocked(*q);
1130 *q = NULL;
1131
1132 return err;
1133}
1134
1135static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl,
1136 int ifindex, struct netlink_ext_ack *extack)
1137{
1138 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1139 return 0;
1140
1141 /* Do we search for filter, attached to class? */
1142 if (TC_H_MIN(parent)) {
1143 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1144
1145 *cl = cops->find(q, parent);
1146 if (*cl == 0) {
1147 NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
1148 return -ENOENT;
1149 }
1150 }
1151
1152 return 0;
1153}
1154
1155static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q,
1156 unsigned long cl, int ifindex,
1157 u32 block_index,
1158 struct netlink_ext_ack *extack)
1159{
1160 struct tcf_block *block;
1161
1162 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
1163 block = tcf_block_refcnt_get(net, block_index);
1164 if (!block) {
1165 NL_SET_ERR_MSG(extack, "Block of given index was not found");
1166 return ERR_PTR(-EINVAL);
1167 }
1168 } else {
1169 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1170
1171 block = cops->tcf_block(q, cl, extack);
1172 if (!block)
1173 return ERR_PTR(-EINVAL);
1174
1175 if (tcf_block_shared(block)) {
1176 NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
1177 return ERR_PTR(-EOPNOTSUPP);
1178 }
1179
1180 /* Always take reference to block in order to support execution
1181 * of rules update path of cls API without rtnl lock. Caller
1182 * must release block when it is finished using it. 'if' block
1183 * of this conditional obtain reference to block by calling
1184 * tcf_block_refcnt_get().
1185 */
1186 refcount_inc(&block->refcnt);
1187 }
1188
1189 return block;
1190}
1191
0607e439 1192static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
12db03b6 1193 struct tcf_block_ext_info *ei, bool rtnl_held)
0607e439 1194{
c266f64d 1195 if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) {
0607e439
VB		 1196 /* Flushing/putting all chains will cause the block to be
1197 * deallocated when last chain is freed. However, if chain_list
1198 * is empty, block has to be manually deallocated. After block
1199 * reference counter reached 0, it is no longer possible to
1200 * increment it or add new chains to block.
1201 */
1202 bool free_block = list_empty(&block->chain_list);
1203
c266f64d 1204 mutex_unlock(&block->lock);
0607e439
VB		 1205 if (tcf_block_shared(block))
1206 tcf_block_remove(block, block->net);
0607e439
VB		 1207
1208 if (q)
1209 tcf_block_offload_unbind(block, q, ei);
1210
1211 if (free_block)
c266f64d 1212 tcf_block_destroy(block);
0607e439 1213 else
12db03b6 1214 tcf_block_flush_all_chains(block, rtnl_held);
0607e439
VB		 1215 } else if (q) {
1216 tcf_block_offload_unbind(block, q, ei);
1217 }
1218}
1219
12db03b6 1220static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held)
0607e439 1221{
12db03b6 1222 __tcf_block_put(block, NULL, NULL, rtnl_held);
0607e439
VB		 1223}
1224
c431f89b
VB		 1225/* Find tcf block.
1226 * Set q, parent, cl when appropriate.
1227 */
1228
1229static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
1230 u32 *parent, unsigned long *cl,
1231 int ifindex, u32 block_index,
1232 struct netlink_ext_ack *extack)
1233{
1234 struct tcf_block *block;
e368fdb6 1235 int err = 0;
c431f89b 1236
18d3eefb 1237 ASSERT_RTNL();
e368fdb6 1238
18d3eefb
VB		 1239 err = __tcf_qdisc_find(net, q, parent, ifindex, true, extack);
1240 if (err)
1241 goto errout;
c431f89b 1242
18d3eefb
VB		 1243 err = __tcf_qdisc_cl_find(*q, *parent, cl, ifindex, extack);
1244 if (err)
1245 goto errout_qdisc;
787ce6d0 1246
18d3eefb 1247 block = __tcf_block_find(net, *q, *cl, ifindex, block_index, extack);
af736bf0
DC		 1248 if (IS_ERR(block)) {
1249 err = PTR_ERR(block);
18d3eefb 1250 goto errout_qdisc;
af736bf0 1251 }
c431f89b
VB		 1252
1253 return block;
e368fdb6 1254
e368fdb6 1255errout_qdisc:
18d3eefb 1256 if (*q)
e368fdb6 1257 qdisc_put(*q);
18d3eefb
VB		 1258errout:
1259 *q = NULL;
e368fdb6
VB		 1260 return ERR_PTR(err);
1261}
1262
12db03b6
VB		 1263static void tcf_block_release(struct Qdisc *q, struct tcf_block *block,
1264 bool rtnl_held)
e368fdb6 1265{
787ce6d0 1266 if (!IS_ERR_OR_NULL(block))
12db03b6 1267 tcf_block_refcnt_put(block, rtnl_held);
787ce6d0 1268
470502de
VB		 1269 if (q) {
1270 if (rtnl_held)
1271 qdisc_put(q);
1272 else
1273 qdisc_put_unlocked(q);
1274 }
c431f89b
VB		 1275}
1276
f36fe1c4
JP		 1277struct tcf_block_owner_item {
1278 struct list_head list;
1279 struct Qdisc *q;
32f8c409 1280 enum flow_block_binder_type binder_type;
f36fe1c4
JP		 1281};
1282
1283static void
1284tcf_block_owner_netif_keep_dst(struct tcf_block *block,
1285 struct Qdisc *q,
32f8c409 1286 enum flow_block_binder_type binder_type)
f36fe1c4
JP		 1287{
1288 if (block->keep_dst &&
32f8c409
PNA		 1289 binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
1290 binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
f36fe1c4
JP		 1291 netif_keep_dst(qdisc_dev(q));
1292}
1293
1294void tcf_block_netif_keep_dst(struct tcf_block *block)
1295{
1296 struct tcf_block_owner_item *item;
1297
1298 block->keep_dst = true;
1299 list_for_each_entry(item, &block->owner_list, list)
1300 tcf_block_owner_netif_keep_dst(block, item->q,
1301 item->binder_type);
1302}
1303EXPORT_SYMBOL(tcf_block_netif_keep_dst);
1304
1305static int tcf_block_owner_add(struct tcf_block *block,
1306 struct Qdisc *q,
32f8c409 1307 enum flow_block_binder_type binder_type)
f36fe1c4
JP		 1308{
1309 struct tcf_block_owner_item *item;
1310
1311 item = kmalloc(sizeof(*item), GFP_KERNEL);
1312 if (!item)
1313 return -ENOMEM;
1314 item->q = q;
1315 item->binder_type = binder_type;
1316 list_add(&item->list, &block->owner_list);
1317 return 0;
1318}
1319
1320static void tcf_block_owner_del(struct tcf_block *block,
1321 struct Qdisc *q,
32f8c409 1322 enum flow_block_binder_type binder_type)
f36fe1c4
JP		 1323{
1324 struct tcf_block_owner_item *item;
1325
1326 list_for_each_entry(item, &block->owner_list, list) {
1327 if (item->q == q && item->binder_type == binder_type) {
1328 list_del(&item->list);
1329 kfree(item);
1330 return;
1331 }
1332 }
1333 WARN_ON(1);
1334}
1335
48617387
JP		 1336int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
1337 struct tcf_block_ext_info *ei,
1338 struct netlink_ext_ack *extack)
1339{
1340 struct net *net = qdisc_net(q);
1341 struct tcf_block *block = NULL;
48617387
JP		 1342 int err;
1343
787ce6d0 1344 if (ei->block_index)
48617387 1345 /* block_index not 0 means the shared block is requested */
787ce6d0 1346 block = tcf_block_refcnt_get(net, ei->block_index);
48617387
JP		 1347
1348 if (!block) {
bb047ddd 1349 block = tcf_block_create(net, q, ei->block_index, extack);
48617387
JP		 1350 if (IS_ERR(block))
1351 return PTR_ERR(block);
bb047ddd
JP		 1352 if (tcf_block_shared(block)) {
1353 err = tcf_block_insert(block, net, extack);
48617387
JP		 1354 if (err)
1355 goto err_block_insert;
1356 }
1357 }
1358
f36fe1c4
JP		 1359 err = tcf_block_owner_add(block, q, ei->binder_type);
1360 if (err)
1361 goto err_block_owner_add;
1362
1363 tcf_block_owner_netif_keep_dst(block, q, ei->binder_type);
1364
f71e0ca4 1365 err = tcf_chain0_head_change_cb_add(block, ei, extack);
a9b19443 1366 if (err)
f71e0ca4 1367 goto err_chain0_head_change_cb_add;
caa72601 1368
60513bd8 1369 err = tcf_block_offload_bind(block, q, ei, extack);
caa72601
JP		 1370 if (err)
1371 goto err_block_offload_bind;
1372
6529eaba
JP		 1373 *p_block = block;
1374 return 0;
2190d1d0 1375
caa72601 1376err_block_offload_bind:
f71e0ca4
JP		 1377 tcf_chain0_head_change_cb_del(block, ei);
1378err_chain0_head_change_cb_add:
f36fe1c4
JP		 1379 tcf_block_owner_del(block, q, ei->binder_type);
1380err_block_owner_add:
48617387 1381err_block_insert:
12db03b6 1382 tcf_block_refcnt_put(block, true);
2190d1d0 1383 return err;
6529eaba 1384}
8c4083b3
JP		 1385EXPORT_SYMBOL(tcf_block_get_ext);
1386
c7eb7d72
JP		 1387static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
1388{
1389 struct tcf_proto __rcu **p_filter_chain = priv;
1390
1391 rcu_assign_pointer(*p_filter_chain, tp_head);
1392}
1393
8c4083b3 1394int tcf_block_get(struct tcf_block **p_block,
8d1a77f9
AA		 1395 struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
1396 struct netlink_ext_ack *extack)
8c4083b3 1397{
c7eb7d72
JP		 1398 struct tcf_block_ext_info ei = {
1399 .chain_head_change = tcf_chain_head_change_dflt,
1400 .chain_head_change_priv = p_filter_chain,
1401 };
8c4083b3 1402
c7eb7d72 1403 WARN_ON(!p_filter_chain);
8d1a77f9 1404 return tcf_block_get_ext(p_block, q, &ei, extack);
8c4083b3 1405}
6529eaba
JP		 1406EXPORT_SYMBOL(tcf_block_get);
1407
7aa0045d 1408/* XXX: Standalone actions are not allowed to jump to any chain, and bound
a60b3f51 1409 * actions should be all removed after flushing.
7aa0045d 1410 */
c7eb7d72 1411void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
e1ea2f98 1412 struct tcf_block_ext_info *ei)
7aa0045d 1413{
c30abd5e
DM		 1414 if (!block)
1415 return;
f71e0ca4 1416 tcf_chain0_head_change_cb_del(block, ei);
f36fe1c4 1417 tcf_block_owner_del(block, q, ei->binder_type);
a60b3f51 1418
12db03b6 1419 __tcf_block_put(block, q, ei, true);
6529eaba 1420}
8c4083b3
JP		 1421EXPORT_SYMBOL(tcf_block_put_ext);
1422
1423void tcf_block_put(struct tcf_block *block)
1424{
1425 struct tcf_block_ext_info ei = {0, };
1426
4853f128
JP		 1427 if (!block)
1428 return;
c7eb7d72 1429 tcf_block_put_ext(block, block->q, &ei);
8c4083b3 1430}
e1ea2f98 1431
6529eaba 1432EXPORT_SYMBOL(tcf_block_put);
cf1facda 1433
32636742 1434static int
a7323311 1435tcf_block_playback_offloads(struct tcf_block *block, flow_setup_cb_t *cb,
32636742
JH		 1436 void *cb_priv, bool add, bool offload_in_use,
1437 struct netlink_ext_ack *extack)
1438{
bbf73830 1439 struct tcf_chain *chain, *chain_prev;
fe2923af 1440 struct tcf_proto *tp, *tp_prev;
32636742
JH		 1441 int err;
1442
4f8116c8
VB		 1443 lockdep_assert_held(&block->cb_lock);
1444
bbf73830
VB		 1445 for (chain = __tcf_get_next_chain(block, NULL);
1446 chain;
1447 chain_prev = chain,
1448 chain = __tcf_get_next_chain(block, chain),
1449 tcf_chain_put(chain_prev)) {
fe2923af
VB		 1450 for (tp = __tcf_get_next_proto(chain, NULL); tp;
1451 tp_prev = tp,
1452 tp = __tcf_get_next_proto(chain, tp),
12db03b6 1453 tcf_proto_put(tp_prev, true, NULL)) {
32636742
JH		 1454 if (tp->ops->reoffload) {
1455 err = tp->ops->reoffload(tp, add, cb, cb_priv,
1456 extack);
1457 if (err && add)
1458 goto err_playback_remove;
1459 } else if (add && offload_in_use) {
1460 err = -EOPNOTSUPP;
1461 NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support");
1462 goto err_playback_remove;
1463 }
1464 }
1465 }
1466
1467 return 0;
1468
1469err_playback_remove:
12db03b6 1470 tcf_proto_put(tp, true, NULL);
bbf73830 1471 tcf_chain_put(chain);
32636742
JH		 1472 tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use,
1473 extack);
1474 return err;
1475}
1476
59094b1e
PNA		 1477static int tcf_block_bind(struct tcf_block *block,
1478 struct flow_block_offload *bo)
1479{
1480 struct flow_block_cb *block_cb, *next;
1481 int err, i = 0;
1482
4f8116c8
VB		 1483 lockdep_assert_held(&block->cb_lock);
1484
59094b1e
PNA		 1485 list_for_each_entry(block_cb, &bo->cb_list, list) {
1486 err = tcf_block_playback_offloads(block, block_cb->cb,
1487 block_cb->cb_priv, true,
1488 tcf_block_offload_in_use(block),
1489 bo->extack);
1490 if (err)
1491 goto err_unroll;
c9f14470
VB		 1492 if (!bo->unlocked_driver_cb)
1493 block->lockeddevcnt++;
59094b1e
PNA		 1494
1495 i++;
1496 }
14bfb13f 1497 list_splice(&bo->cb_list, &block->flow_block.cb_list);
59094b1e
PNA		 1498
1499 return 0;
1500
1501err_unroll:
1502 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1503 if (i-- > 0) {
1504 list_del(&block_cb->list);
1505 tcf_block_playback_offloads(block, block_cb->cb,
1506 block_cb->cb_priv, false,
1507 tcf_block_offload_in_use(block),
1508 NULL);
c9f14470
VB		 1509 if (!bo->unlocked_driver_cb)
1510 block->lockeddevcnt--;
59094b1e
PNA		 1511 }
1512 flow_block_cb_free(block_cb);
1513 }
1514
1515 return err;
1516}
1517
1518static void tcf_block_unbind(struct tcf_block *block,
1519 struct flow_block_offload *bo)
1520{
1521 struct flow_block_cb *block_cb, *next;
1522
4f8116c8
VB		 1523 lockdep_assert_held(&block->cb_lock);
1524
59094b1e
PNA		 1525 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1526 tcf_block_playback_offloads(block, block_cb->cb,
1527 block_cb->cb_priv, false,
1528 tcf_block_offload_in_use(block),
1529 NULL);
1530 list_del(&block_cb->list);
1531 flow_block_cb_free(block_cb);
c9f14470
VB		 1532 if (!bo->unlocked_driver_cb)
1533 block->lockeddevcnt--;
59094b1e
PNA		 1534 }
1535}
1536
1537static int tcf_block_setup(struct tcf_block *block,
1538 struct flow_block_offload *bo)
1539{
1540 int err;
1541
1542 switch (bo->command) {
1543 case FLOW_BLOCK_BIND:
1544 err = tcf_block_bind(block, bo);
1545 break;
1546 case FLOW_BLOCK_UNBIND:
1547 err = 0;
1548 tcf_block_unbind(block, bo);
1549 break;
1550 default:
1551 WARN_ON_ONCE(1);
1552 err = -EOPNOTSUPP;
1553 }
1554
1555 return err;
1556}
1557
87d83093
JP		 1558/* Main classifier routine: scans classifier chain attached
1559 * to this qdisc, (optionally) tests for protocol and asks
1560 * specific classifiers.
1561 */
1562int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
1563 struct tcf_result *res, bool compat_mode)
1564{
87d83093
JP		 1565#ifdef CONFIG_NET_CLS_ACT
1566 const int max_reclassify_loop = 4;
ee538dce
JP		 1567 const struct tcf_proto *orig_tp = tp;
1568 const struct tcf_proto *first_tp;
87d83093
JP		 1569 int limit = 0;
1570
1571reclassify:
1572#endif
1573 for (; tp; tp = rcu_dereference_bh(tp->next)) {
cd0c4e70 1574 __be16 protocol = tc_skb_protocol(skb);
87d83093
JP		 1575 int err;
1576
1577 if (tp->protocol != protocol &&
1578 tp->protocol != htons(ETH_P_ALL))
1579 continue;
1580
1581 err = tp->classify(skb, tp, res);
1582#ifdef CONFIG_NET_CLS_ACT
db50514f 1583 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
ee538dce 1584 first_tp = orig_tp;
87d83093 1585 goto reset;
db50514f 1586 } else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
ee538dce 1587 first_tp = res->goto_tp;
95a7233c
PB		 1588
1589#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
1590 {
1591 struct tc_skb_ext *ext;
1592
1593 ext = skb_ext_add(skb, TC_SKB_EXT);
1594 if (WARN_ON_ONCE(!ext))
1595 return TC_ACT_SHOT;
1596
1597 ext->chain = err & TC_ACT_EXT_VAL_MASK;
1598 }
1599#endif
db50514f
JP		 1600 goto reset;
1601 }
87d83093
JP		 1602#endif
1603 if (err >= 0)
1604 return err;
1605 }
1606
1607 return TC_ACT_UNSPEC; /* signal: continue lookup */
1608#ifdef CONFIG_NET_CLS_ACT
1609reset:
1610 if (unlikely(limit++ >= max_reclassify_loop)) {
9d3aaff3
JP		 1611 net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
1612 tp->chain->block->index,
1613 tp->prio & 0xffff,
87d83093
JP		 1614 ntohs(tp->protocol));
1615 return TC_ACT_SHOT;
1616 }
1617
ee538dce 1618 tp = first_tp;
87d83093
JP		 1619 goto reclassify;
1620#endif
1621}
1622EXPORT_SYMBOL(tcf_classify);
1623
2190d1d0
JP		 1624struct tcf_chain_info {
1625 struct tcf_proto __rcu **pprev;
1626 struct tcf_proto __rcu *next;
1627};
1628
ed76f5ed
VB		 1629static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain,
1630 struct tcf_chain_info *chain_info)
2190d1d0 1631{
ed76f5ed 1632 return tcf_chain_dereference(*chain_info->pprev, chain);
2190d1d0
JP		 1633}
1634
726d0612
VB		 1635static int tcf_chain_tp_insert(struct tcf_chain *chain,
1636 struct tcf_chain_info *chain_info,
1637 struct tcf_proto *tp)
2190d1d0 1638{
726d0612
VB		 1639 if (chain->flushing)
1640 return -EAGAIN;
1641
c7eb7d72 1642 if (*chain_info->pprev == chain->filter_chain)
f71e0ca4 1643 tcf_chain0_head_change(chain, tp);
4dbfa766 1644 tcf_proto_get(tp);
ed76f5ed 1645 RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
2190d1d0 1646 rcu_assign_pointer(*chain_info->pprev, tp);
726d0612
VB		 1647
1648 return 0;
2190d1d0
JP		 1649}
1650
1651static void tcf_chain_tp_remove(struct tcf_chain *chain,
1652 struct tcf_chain_info *chain_info,
1653 struct tcf_proto *tp)
1654{
ed76f5ed 1655 struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain);
2190d1d0 1656
8b64678e 1657 tcf_proto_mark_delete(tp);
c7eb7d72 1658 if (tp == chain->filter_chain)
f71e0ca4 1659 tcf_chain0_head_change(chain, next);
2190d1d0
JP		 1660 RCU_INIT_POINTER(*chain_info->pprev, next);
1661}
1662
8b64678e
VB		 1663static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1664 struct tcf_chain_info *chain_info,
1665 u32 protocol, u32 prio,
1666 bool prio_allocate);
1667
1668/* Try to insert new proto.
1669 * If proto with specified priority already exists, free new proto
1670 * and return existing one.
1671 */
1672
1673static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain,
1674 struct tcf_proto *tp_new,
12db03b6
VB		 1675 u32 protocol, u32 prio,
1676 bool rtnl_held)
8b64678e
VB		 1677{
1678 struct tcf_chain_info chain_info;
1679 struct tcf_proto *tp;
726d0612 1680 int err = 0;
8b64678e
VB		 1681
1682 mutex_lock(&chain->filter_chain_lock);
1683
59eb87cb
JH		 1684 if (tcf_proto_exists_destroying(chain, tp_new)) {
1685 mutex_unlock(&chain->filter_chain_lock);
1686 tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1687 return ERR_PTR(-EAGAIN);
1688 }
1689
8b64678e
VB		 1690 tp = tcf_chain_tp_find(chain, &chain_info,
1691 protocol, prio, false);
1692 if (!tp)
726d0612 1693 err = tcf_chain_tp_insert(chain, &chain_info, tp_new);
8b64678e
VB		 1694 mutex_unlock(&chain->filter_chain_lock);
1695
1696 if (tp) {
59eb87cb 1697 tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
8b64678e 1698 tp_new = tp;
726d0612 1699 } else if (err) {
59eb87cb 1700 tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
726d0612 1701 tp_new = ERR_PTR(err);
8b64678e
VB		 1702 }
1703
1704 return tp_new;
1705}
1706
1707static void tcf_chain_tp_delete_empty(struct tcf_chain *chain,
12db03b6 1708 struct tcf_proto *tp, bool rtnl_held,
8b64678e
VB		 1709 struct netlink_ext_ack *extack)
1710{
1711 struct tcf_chain_info chain_info;
1712 struct tcf_proto *tp_iter;
1713 struct tcf_proto **pprev;
1714 struct tcf_proto *next;
1715
1716 mutex_lock(&chain->filter_chain_lock);
1717
1718 /* Atomically find and remove tp from chain. */
1719 for (pprev = &chain->filter_chain;
1720 (tp_iter = tcf_chain_dereference(*pprev, chain));
1721 pprev = &tp_iter->next) {
1722 if (tp_iter == tp) {
1723 chain_info.pprev = pprev;
1724 chain_info.next = tp_iter->next;
1725 WARN_ON(tp_iter->deleting);
1726 break;
1727 }
1728 }
1729 /* Verify that tp still exists and no new filters were inserted
1730 * concurrently.
1731 * Mark tp for deletion if it is empty.
1732 */
a5b72a08 1733 if (!tp_iter || !tcf_proto_check_delete(tp)) {
8b64678e
VB		 1734 mutex_unlock(&chain->filter_chain_lock);
1735 return;
1736 }
1737
59eb87cb 1738 tcf_proto_signal_destroying(chain, tp);
8b64678e
VB		 1739 next = tcf_chain_dereference(chain_info.next, chain);
1740 if (tp == chain->filter_chain)
1741 tcf_chain0_head_change(chain, next);
1742 RCU_INIT_POINTER(*chain_info.pprev, next);
1743 mutex_unlock(&chain->filter_chain_lock);
1744
12db03b6 1745 tcf_proto_put(tp, rtnl_held, extack);
8b64678e
VB		 1746}
1747
2190d1d0
JP		 1748static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1749 struct tcf_chain_info *chain_info,
1750 u32 protocol, u32 prio,
1751 bool prio_allocate)
1752{
1753 struct tcf_proto **pprev;
1754 struct tcf_proto *tp;
1755
1756 /* Check the chain for existence of proto-tcf with this priority */
1757 for (pprev = &chain->filter_chain;
ed76f5ed
VB		 1758 (tp = tcf_chain_dereference(*pprev, chain));
1759 pprev = &tp->next) {
2190d1d0
JP		 1760 if (tp->prio >= prio) {
1761 if (tp->prio == prio) {
1762 if (prio_allocate ||
1763 (tp->protocol != protocol && protocol))
1764 return ERR_PTR(-EINVAL);
1765 } else {
1766 tp = NULL;
1767 }
1768 break;
1769 }
1770 }
1771 chain_info->pprev = pprev;
4dbfa766
VB		 1772 if (tp) {
1773 chain_info->next = tp->next;
1774 tcf_proto_get(tp);
1775 } else {
1776 chain_info->next = NULL;
1777 }
2190d1d0
JP		 1778 return tp;
1779}
1780
7120371c 1781static int tcf_fill_node(struct net *net, struct sk_buff *skb,
7960d1da
JP		 1782 struct tcf_proto *tp, struct tcf_block *block,
1783 struct Qdisc *q, u32 parent, void *fh,
12db03b6
VB		 1784 u32 portid, u32 seq, u16 flags, int event,
1785 bool rtnl_held)
7120371c
WC		 1786{
1787 struct tcmsg *tcm;
1788 struct nlmsghdr *nlh;
1789 unsigned char *b = skb_tail_pointer(skb);
1790
1791 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1792 if (!nlh)
1793 goto out_nlmsg_trim;
1794 tcm = nlmsg_data(nlh);
1795 tcm->tcm_family = AF_UNSPEC;
1796 tcm->tcm__pad1 = 0;
1797 tcm->tcm__pad2 = 0;
7960d1da
JP		 1798 if (q) {
1799 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1800 tcm->tcm_parent = parent;
1801 } else {
1802 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
1803 tcm->tcm_block_index = block->index;
1804 }
7120371c
WC		 1805 tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
1806 if (nla_put_string(skb, TCA_KIND, tp->ops->kind))
1807 goto nla_put_failure;
1808 if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
1809 goto nla_put_failure;
1810 if (!fh) {
1811 tcm->tcm_handle = 0;
1812 } else {
12db03b6
VB		 1813 if (tp->ops->dump &&
1814 tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0)
7120371c
WC		 1815 goto nla_put_failure;
1816 }
1817 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1818 return skb->len;
1819
1820out_nlmsg_trim:
1821nla_put_failure:
1822 nlmsg_trim(skb, b);
1823 return -1;
1824}
1825
1826static int tfilter_notify(struct net *net, struct sk_buff *oskb,
1827 struct nlmsghdr *n, struct tcf_proto *tp,
7960d1da 1828 struct tcf_block *block, struct Qdisc *q,
12db03b6
VB		 1829 u32 parent, void *fh, int event, bool unicast,
1830 bool rtnl_held)
7120371c
WC		 1831{
1832 struct sk_buff *skb;
1833 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
5b5f99b1 1834 int err = 0;
7120371c
WC		 1835
1836 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1837 if (!skb)
1838 return -ENOBUFS;
1839
7960d1da 1840 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
12db03b6
VB		 1841 n->nlmsg_seq, n->nlmsg_flags, event,
1842 rtnl_held) <= 0) {
7120371c
WC		 1843 kfree_skb(skb);
1844 return -EINVAL;
1845 }
1846
1847 if (unicast)
5b5f99b1
ZW		 1848 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1849 else
1850 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1851 n->nlmsg_flags & NLM_F_ECHO);
7120371c 1852
5b5f99b1
ZW		 1853 if (err > 0)
1854 err = 0;
1855 return err;
7120371c
WC		 1856}
1857
1858static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
1859 struct nlmsghdr *n, struct tcf_proto *tp,
7960d1da 1860 struct tcf_block *block, struct Qdisc *q,
c35a4acc 1861 u32 parent, void *fh, bool unicast, bool *last,
12db03b6 1862 bool rtnl_held, struct netlink_ext_ack *extack)
7120371c
WC		 1863{
1864 struct sk_buff *skb;
1865 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1866 int err;
1867
1868 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1869 if (!skb)
1870 return -ENOBUFS;
1871
7960d1da 1872 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
12db03b6
VB		 1873 n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER,
1874 rtnl_held) <= 0) {
c35a4acc 1875 NL_SET_ERR_MSG(extack, "Failed to build del event notification");
7120371c
WC		 1876 kfree_skb(skb);
1877 return -EINVAL;
1878 }
1879
12db03b6 1880 err = tp->ops->delete(tp, fh, last, rtnl_held, extack);
7120371c
WC		 1881 if (err) {
1882 kfree_skb(skb);
1883 return err;
1884 }
1885
1886 if (unicast)
5b5f99b1
ZW		 1887 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1888 else
1889 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1890 n->nlmsg_flags & NLM_F_ECHO);
c35a4acc
AA		 1891 if (err < 0)
1892 NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");
5b5f99b1
ZW		 1893
1894 if (err > 0)
1895 err = 0;
c35a4acc 1896 return err;
7120371c
WC		 1897}
1898
1899static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
7960d1da
JP		 1900 struct tcf_block *block, struct Qdisc *q,
1901 u32 parent, struct nlmsghdr *n,
12db03b6
VB		 1902 struct tcf_chain *chain, int event,
1903 bool rtnl_held)
7120371c
WC		 1904{
1905 struct tcf_proto *tp;
1906
12db03b6
VB		 1907 for (tp = tcf_get_next_proto(chain, NULL, rtnl_held);
1908 tp; tp = tcf_get_next_proto(chain, tp, rtnl_held))
7960d1da 1909 tfilter_notify(net, oskb, n, tp, block,
12db03b6 1910 q, parent, NULL, event, false, rtnl_held);
7120371c
WC		 1911}
1912
7d5509fa
VB		 1913static void tfilter_put(struct tcf_proto *tp, void *fh)
1914{
1915 if (tp->ops->put && fh)
1916 tp->ops->put(tp, fh);
1917}
1918
c431f89b 1919static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
c21ef3e3 1920 struct netlink_ext_ack *extack)
1da177e4 1921{
3b1e0a65 1922 struct net *net = sock_net(skb->sk);
add93b61 1923 struct nlattr *tca[TCA_MAX + 1];
6f96c3c6 1924 char name[IFNAMSIZ];
1da177e4
LT		 1925 struct tcmsg *t;
1926 u32 protocol;
1927 u32 prio;
9d36d9e5 1928 bool prio_allocate;
1da177e4 1929 u32 parent;
5bc17018 1930 u32 chain_index;
7960d1da 1931 struct Qdisc *q = NULL;
2190d1d0 1932 struct tcf_chain_info chain_info;
5bc17018 1933 struct tcf_chain *chain = NULL;
6529eaba 1934 struct tcf_block *block;
1da177e4 1935 struct tcf_proto *tp;
1da177e4 1936 unsigned long cl;
8113c095 1937 void *fh;
1da177e4 1938 int err;
628185cf 1939 int tp_created;
470502de 1940 bool rtnl_held = false;
1da177e4 1941
c431f89b 1942 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
dfc47ef8 1943 return -EPERM;
de179c8c 1944
1da177e4 1945replay:
628185cf
DB		 1946 tp_created = 0;
1947
8cb08174
JB		 1948 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
1949 rtm_tca_policy, extack);
de179c8c
H		 1950 if (err < 0)
1951 return err;
1952
942b8165 1953 t = nlmsg_data(n);
1da177e4
LT		 1954 protocol = TC_H_MIN(t->tcm_info);
1955 prio = TC_H_MAJ(t->tcm_info);
9d36d9e5 1956 prio_allocate = false;
1da177e4 1957 parent = t->tcm_parent;
4dbfa766 1958 tp = NULL;
1da177e4 1959 cl = 0;
470502de 1960 block = NULL;
1da177e4
LT		 1961
1962 if (prio == 0) {
c431f89b
VB		 1963 /* If no priority is provided by the user,
1964 * we allocate one.
1965 */
1966 if (n->nlmsg_flags & NLM_F_CREATE) {
1967 prio = TC_H_MAKE(0x80000000U, 0U);
1968 prio_allocate = true;
1969 } else {
c35a4acc 1970 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
1da177e4 1971 return -ENOENT;
ea7f8277 1972 }
1da177e4
LT		 1973 }
1974
1975 /* Find head of filter chain. */
1976
470502de
VB		 1977 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
1978 if (err)
1979 return err;
1980
6f96c3c6
CW		 1981 if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
1982 NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
1983 err = -EINVAL;
1984 goto errout;
1985 }
1986
470502de
VB		 1987 /* Take rtnl mutex if rtnl_held was set to true on previous iteration,
1988 * block is shared (no qdisc found), qdisc is not unlocked, classifier
1989 * type is not specified, classifier is not unlocked.
1990 */
1991 if (rtnl_held ||
1992 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
6f96c3c6 1993 !tcf_proto_is_unlocked(name)) {
470502de
VB		 1994 rtnl_held = true;
1995 rtnl_lock();
1996 }
1997
1998 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
1999 if (err)
2000 goto errout;
2001
2002 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2003 extack);
c431f89b
VB		 2004 if (IS_ERR(block)) {
2005 err = PTR_ERR(block);
2006 goto errout;
6bb16e7a 2007 }
5bc17018
JP		 2008
2009 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2010 if (chain_index > TC_ACT_EXT_VAL_MASK) {
c35a4acc 2011 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
5bc17018
JP		 2012 err = -EINVAL;
2013 goto errout;
2014 }
c431f89b 2015 chain = tcf_chain_get(block, chain_index, true);
5bc17018 2016 if (!chain) {
d5ed72a5 2017 NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
c431f89b 2018 err = -ENOMEM;
ea7f8277
DB		 2019 goto errout;
2020 }
1da177e4 2021
ed76f5ed 2022 mutex_lock(&chain->filter_chain_lock);
2190d1d0
JP		 2023 tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2024 prio, prio_allocate);
2025 if (IS_ERR(tp)) {
c35a4acc 2026 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2190d1d0 2027 err = PTR_ERR(tp);
ed76f5ed 2028 goto errout_locked;
1da177e4
LT		 2029 }
2030
2031 if (tp == NULL) {
8b64678e
VB		 2032 struct tcf_proto *tp_new = NULL;
2033
726d0612
VB		 2034 if (chain->flushing) {
2035 err = -EAGAIN;
2036 goto errout_locked;
2037 }
2038
1da177e4
LT		 2039 /* Proto-tcf does not exist, create new one */
2040
6bb16e7a 2041 if (tca[TCA_KIND] == NULL || !protocol) {
c35a4acc 2042 NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
6bb16e7a 2043 err = -EINVAL;
ed76f5ed 2044 goto errout_locked;
6bb16e7a 2045 }
1da177e4 2046
c431f89b 2047 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
c35a4acc 2048 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
6bb16e7a 2049 err = -ENOENT;
ed76f5ed 2050 goto errout_locked;
6bb16e7a 2051 }
1da177e4 2052
9d36d9e5 2053 if (prio_allocate)
ed76f5ed
VB		 2054 prio = tcf_auto_prio(tcf_chain_tp_prev(chain,
2055 &chain_info));
1da177e4 2056
ed76f5ed 2057 mutex_unlock(&chain->filter_chain_lock);
36d79af7
ED		 2058 tp_new = tcf_proto_create(name, protocol, prio, chain,
2059 rtnl_held, extack);
8b64678e
VB		 2060 if (IS_ERR(tp_new)) {
2061 err = PTR_ERR(tp_new);
726d0612 2062 goto errout_tp;
1da177e4 2063 }
ed76f5ed 2064
12186be7 2065 tp_created = 1;
12db03b6
VB		 2066 tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio,
2067 rtnl_held);
726d0612
VB		 2068 if (IS_ERR(tp)) {
2069 err = PTR_ERR(tp);
2070 goto errout_tp;
2071 }
ed76f5ed
VB		 2072 } else {
2073 mutex_unlock(&chain->filter_chain_lock);
6bb16e7a 2074 }
1da177e4 2075
8b64678e
VB		 2076 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2077 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2078 err = -EINVAL;
2079 goto errout;
2080 }
2081
1da177e4
LT		 2082 fh = tp->ops->get(tp, t->tcm_handle);
2083
8113c095 2084 if (!fh) {
c431f89b 2085 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
c35a4acc 2086 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
6bb16e7a 2087 err = -ENOENT;
1da177e4 2088 goto errout;
6bb16e7a 2089 }
c431f89b 2090 } else if (n->nlmsg_flags & NLM_F_EXCL) {
7d5509fa 2091 tfilter_put(tp, fh);
c431f89b
VB		 2092 NL_SET_ERR_MSG(extack, "Filter already exists");
2093 err = -EEXIST;
2094 goto errout;
1da177e4
LT		 2095 }
2096
9f407f17
JP		 2097 if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
2098 NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
2099 err = -EINVAL;
2100 goto errout;
2101 }
2102
2f7ef2f8 2103 err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
7306db38 2104 n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE,
12db03b6 2105 rtnl_held, extack);
7d5509fa 2106 if (err == 0) {
7960d1da 2107 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
12db03b6 2108 RTM_NEWTFILTER, false, rtnl_held);
7d5509fa 2109 tfilter_put(tp, fh);
503d81d4
VB		 2110 /* q pointer is NULL for shared blocks */
2111 if (q)
2112 q->flags &= ~TCQ_F_CAN_BYPASS;
7d5509fa 2113 }
1da177e4
LT		 2114
2115errout:
8b64678e 2116 if (err && tp_created)
12db03b6 2117 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL);
726d0612 2118errout_tp:
4dbfa766
VB		 2119 if (chain) {
2120 if (tp && !IS_ERR(tp))
12db03b6 2121 tcf_proto_put(tp, rtnl_held, NULL);
4dbfa766
VB		 2122 if (!tp_created)
2123 tcf_chain_put(chain);
2124 }
12db03b6 2125 tcf_block_release(q, block, rtnl_held);
470502de
VB		 2126
2127 if (rtnl_held)
2128 rtnl_unlock();
2129
2130 if (err == -EAGAIN) {
2131 /* Take rtnl lock in case EAGAIN is caused by concurrent flush
2132 * of target chain.
2133 */
2134 rtnl_held = true;
1da177e4
LT		 2135 /* Replay the request. */
2136 goto replay;
470502de 2137 }
1da177e4 2138 return err;
ed76f5ed
VB		 2139
2140errout_locked:
2141 mutex_unlock(&chain->filter_chain_lock);
2142 goto errout;
1da177e4
LT		 2143}
2144
c431f89b
VB		 2145static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2146 struct netlink_ext_ack *extack)
2147{
2148 struct net *net = sock_net(skb->sk);
2149 struct nlattr *tca[TCA_MAX + 1];
6f96c3c6 2150 char name[IFNAMSIZ];
c431f89b
VB		 2151 struct tcmsg *t;
2152 u32 protocol;
2153 u32 prio;
2154 u32 parent;
2155 u32 chain_index;
2156 struct Qdisc *q = NULL;
2157 struct tcf_chain_info chain_info;
2158 struct tcf_chain *chain = NULL;
470502de 2159 struct tcf_block *block = NULL;
c431f89b
VB		 2160 struct tcf_proto *tp = NULL;
2161 unsigned long cl = 0;
2162 void *fh = NULL;
2163 int err;
470502de 2164 bool rtnl_held = false;
c431f89b
VB		 2165
2166 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2167 return -EPERM;
2168
8cb08174
JB		 2169 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2170 rtm_tca_policy, extack);
c431f89b
VB		 2171 if (err < 0)
2172 return err;
2173
2174 t = nlmsg_data(n);
2175 protocol = TC_H_MIN(t->tcm_info);
2176 prio = TC_H_MAJ(t->tcm_info);
2177 parent = t->tcm_parent;
2178
2179 if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) {
2180 NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set");
2181 return -ENOENT;
2182 }
2183
2184 /* Find head of filter chain. */
2185
470502de
VB		 2186 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2187 if (err)
2188 return err;
2189
6f96c3c6
CW		 2190 if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2191 NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2192 err = -EINVAL;
2193 goto errout;
2194 }
470502de
VB		 2195 /* Take rtnl mutex if flushing whole chain, block is shared (no qdisc
2196 * found), qdisc is not unlocked, classifier type is not specified,
2197 * classifier is not unlocked.
2198 */
2199 if (!prio ||
2200 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
6f96c3c6 2201 !tcf_proto_is_unlocked(name)) {
470502de
VB		 2202 rtnl_held = true;
2203 rtnl_lock();
2204 }
2205
2206 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2207 if (err)
2208 goto errout;
2209
2210 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2211 extack);
c431f89b
VB		 2212 if (IS_ERR(block)) {
2213 err = PTR_ERR(block);
2214 goto errout;
2215 }
2216
2217 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2218 if (chain_index > TC_ACT_EXT_VAL_MASK) {
2219 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2220 err = -EINVAL;
2221 goto errout;
2222 }
2223 chain = tcf_chain_get(block, chain_index, false);
2224 if (!chain) {
5ca8a25c
JP		 2225 /* User requested flush on non-existent chain. Nothing to do,
2226 * so just return success.
2227 */
2228 if (prio == 0) {
2229 err = 0;
2230 goto errout;
2231 }
c431f89b 2232 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
b7b4247d 2233 err = -ENOENT;
c431f89b
VB		 2234 goto errout;
2235 }
2236
2237 if (prio == 0) {
2238 tfilter_notify_chain(net, skb, block, q, parent, n,
12db03b6
VB		 2239 chain, RTM_DELTFILTER, rtnl_held);
2240 tcf_chain_flush(chain, rtnl_held);
c431f89b
VB		 2241 err = 0;
2242 goto errout;
2243 }
2244
ed76f5ed 2245 mutex_lock(&chain->filter_chain_lock);
c431f89b
VB		 2246 tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2247 prio, false);
2248 if (!tp || IS_ERR(tp)) {
2249 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
0e399035 2250 err = tp ? PTR_ERR(tp) : -ENOENT;
ed76f5ed 2251 goto errout_locked;
c431f89b
VB		 2252 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2253 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2254 err = -EINVAL;
ed76f5ed
VB		 2255 goto errout_locked;
2256 } else if (t->tcm_handle == 0) {
59eb87cb 2257 tcf_proto_signal_destroying(chain, tp);
ed76f5ed
VB		 2258 tcf_chain_tp_remove(chain, &chain_info, tp);
2259 mutex_unlock(&chain->filter_chain_lock);
2260
12db03b6 2261 tcf_proto_put(tp, rtnl_held, NULL);
ed76f5ed 2262 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
12db03b6 2263 RTM_DELTFILTER, false, rtnl_held);
ed76f5ed 2264 err = 0;
c431f89b
VB		 2265 goto errout;
2266 }
ed76f5ed 2267 mutex_unlock(&chain->filter_chain_lock);
c431f89b
VB		 2268
2269 fh = tp->ops->get(tp, t->tcm_handle);
2270
2271 if (!fh) {
ed76f5ed
VB		 2272 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2273 err = -ENOENT;
c431f89b
VB		 2274 } else {
2275 bool last;
2276
2277 err = tfilter_del_notify(net, skb, n, tp, block,
2278 q, parent, fh, false, &last,
12db03b6
VB		 2279 rtnl_held, extack);
2280
c431f89b
VB		 2281 if (err)
2282 goto errout;
8b64678e 2283 if (last)
12db03b6 2284 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack);
c431f89b
VB		 2285 }
2286
2287errout:
4dbfa766
VB		 2288 if (chain) {
2289 if (tp && !IS_ERR(tp))
12db03b6 2290 tcf_proto_put(tp, rtnl_held, NULL);
c431f89b 2291 tcf_chain_put(chain);
4dbfa766 2292 }
12db03b6 2293 tcf_block_release(q, block, rtnl_held);
470502de
VB		 2294
2295 if (rtnl_held)
2296 rtnl_unlock();
2297
c431f89b 2298 return err;
ed76f5ed
VB		 2299
2300errout_locked:
2301 mutex_unlock(&chain->filter_chain_lock);
2302 goto errout;
c431f89b
VB		 2303}
2304
2305static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2306 struct netlink_ext_ack *extack)
2307{
2308 struct net *net = sock_net(skb->sk);
2309 struct nlattr *tca[TCA_MAX + 1];
6f96c3c6 2310 char name[IFNAMSIZ];
c431f89b
VB		 2311 struct tcmsg *t;
2312 u32 protocol;
2313 u32 prio;
2314 u32 parent;
2315 u32 chain_index;
2316 struct Qdisc *q = NULL;
2317 struct tcf_chain_info chain_info;
2318 struct tcf_chain *chain = NULL;
470502de 2319 struct tcf_block *block = NULL;
c431f89b
VB		 2320 struct tcf_proto *tp = NULL;
2321 unsigned long cl = 0;
2322 void *fh = NULL;
2323 int err;
470502de 2324 bool rtnl_held = false;
c431f89b 2325
8cb08174
JB		 2326 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2327 rtm_tca_policy, extack);
c431f89b
VB		 2328 if (err < 0)
2329 return err;
2330
2331 t = nlmsg_data(n);
2332 protocol = TC_H_MIN(t->tcm_info);
2333 prio = TC_H_MAJ(t->tcm_info);
2334 parent = t->tcm_parent;
2335
2336 if (prio == 0) {
2337 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
2338 return -ENOENT;
2339 }
2340
2341 /* Find head of filter chain. */
2342
470502de
VB		 2343 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2344 if (err)
2345 return err;
2346
6f96c3c6
CW		 2347 if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2348 NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2349 err = -EINVAL;
2350 goto errout;
2351 }
470502de
VB		 2352 /* Take rtnl mutex if block is shared (no qdisc found), qdisc is not
2353 * unlocked, classifier type is not specified, classifier is not
2354 * unlocked.
2355 */
2356 if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
6f96c3c6 2357 !tcf_proto_is_unlocked(name)) {
470502de
VB		 2358 rtnl_held = true;
2359 rtnl_lock();
2360 }
2361
2362 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2363 if (err)
2364 goto errout;
2365
2366 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2367 extack);
c431f89b
VB		 2368 if (IS_ERR(block)) {
2369 err = PTR_ERR(block);
2370 goto errout;
2371 }
2372
2373 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2374 if (chain_index > TC_ACT_EXT_VAL_MASK) {
2375 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2376 err = -EINVAL;
2377 goto errout;
2378 }
2379 chain = tcf_chain_get(block, chain_index, false);
2380 if (!chain) {
2381 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2382 err = -EINVAL;
2383 goto errout;
2384 }
2385
ed76f5ed 2386 mutex_lock(&chain->filter_chain_lock);
c431f89b
VB		 2387 tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2388 prio, false);
ed76f5ed 2389 mutex_unlock(&chain->filter_chain_lock);
c431f89b
VB		 2390 if (!tp || IS_ERR(tp)) {
2391 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
0e399035 2392 err = tp ? PTR_ERR(tp) : -ENOENT;
c431f89b
VB		 2393 goto errout;
2394 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2395 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2396 err = -EINVAL;
2397 goto errout;
2398 }
2399
2400 fh = tp->ops->get(tp, t->tcm_handle);
2401
2402 if (!fh) {
2403 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2404 err = -ENOENT;
2405 } else {
2406 err = tfilter_notify(net, skb, n, tp, block, q, parent,
12db03b6 2407 fh, RTM_NEWTFILTER, true, rtnl_held);
c431f89b
VB		 2408 if (err < 0)
2409 NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
2410 }
2411
7d5509fa 2412 tfilter_put(tp, fh);
c431f89b 2413errout:
4dbfa766
VB		 2414 if (chain) {
2415 if (tp && !IS_ERR(tp))
12db03b6 2416 tcf_proto_put(tp, rtnl_held, NULL);
c431f89b 2417 tcf_chain_put(chain);
4dbfa766 2418 }
12db03b6 2419 tcf_block_release(q, block, rtnl_held);
470502de
VB		 2420
2421 if (rtnl_held)
2422 rtnl_unlock();
2423
c431f89b
VB		 2424 return err;
2425}
2426
aa767bfe 2427struct tcf_dump_args {
1da177e4
LT		 2428 struct tcf_walker w;
2429 struct sk_buff *skb;
2430 struct netlink_callback *cb;
7960d1da 2431 struct tcf_block *block;
a10fa201
JP		 2432 struct Qdisc *q;
2433 u32 parent;
1da177e4
LT		 2434};
2435
8113c095 2436static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
1da177e4 2437{
aa767bfe 2438 struct tcf_dump_args *a = (void *)arg;
832d1d5b 2439 struct net *net = sock_net(a->skb->sk);
1da177e4 2440
7960d1da 2441 return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
a10fa201 2442 n, NETLINK_CB(a->cb->skb).portid,
5a7a5555 2443 a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
12db03b6 2444 RTM_NEWTFILTER, true);
1da177e4
LT		 2445}
2446
a10fa201
JP		 2447static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
2448 struct sk_buff *skb, struct netlink_callback *cb,
acb31fae
JP		 2449 long index_start, long *p_index)
2450{
2451 struct net *net = sock_net(skb->sk);
7960d1da 2452 struct tcf_block *block = chain->block;
acb31fae 2453 struct tcmsg *tcm = nlmsg_data(cb->nlh);
fe2923af 2454 struct tcf_proto *tp, *tp_prev;
acb31fae 2455 struct tcf_dump_args arg;
acb31fae 2456
fe2923af
VB		 2457 for (tp = __tcf_get_next_proto(chain, NULL);
2458 tp;
2459 tp_prev = tp,
2460 tp = __tcf_get_next_proto(chain, tp),
12db03b6 2461 tcf_proto_put(tp_prev, true, NULL),
fe2923af 2462 (*p_index)++) {
acb31fae
JP		 2463 if (*p_index < index_start)
2464 continue;
2465 if (TC_H_MAJ(tcm->tcm_info) &&
2466 TC_H_MAJ(tcm->tcm_info) != tp->prio)
2467 continue;
2468 if (TC_H_MIN(tcm->tcm_info) &&
2469 TC_H_MIN(tcm->tcm_info) != tp->protocol)
2470 continue;
2471 if (*p_index > index_start)
2472 memset(&cb->args[1], 0,
2473 sizeof(cb->args) - sizeof(cb->args[0]));
2474 if (cb->args[1] == 0) {
53189183 2475 if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
acb31fae
JP		 2476 NETLINK_CB(cb->skb).portid,
2477 cb->nlh->nlmsg_seq, NLM_F_MULTI,
12db03b6 2478 RTM_NEWTFILTER, true) <= 0)
fe2923af 2479 goto errout;
acb31fae
JP		 2480 cb->args[1] = 1;
2481 }
2482 if (!tp->ops->walk)
2483 continue;
2484 arg.w.fn = tcf_node_dump;
2485 arg.skb = skb;
2486 arg.cb = cb;
7960d1da 2487 arg.block = block;
a10fa201
JP		 2488 arg.q = q;
2489 arg.parent = parent;
acb31fae
JP		 2490 arg.w.stop = 0;
2491 arg.w.skip = cb->args[1] - 1;
2492 arg.w.count = 0;
01683a14 2493 arg.w.cookie = cb->args[2];
12db03b6 2494 tp->ops->walk(tp, &arg.w, true);
01683a14 2495 cb->args[2] = arg.w.cookie;
acb31fae
JP		 2496 cb->args[1] = arg.w.count + 1;
2497 if (arg.w.stop)
fe2923af 2498 goto errout;
acb31fae 2499 }
5bc17018 2500 return true;
fe2923af
VB		 2501
2502errout:
12db03b6 2503 tcf_proto_put(tp, true, NULL);
fe2923af 2504 return false;
acb31fae
JP		 2505}
2506
bd27a875 2507/* called with RTNL */
1da177e4
LT		 2508static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
2509{
bbf73830 2510 struct tcf_chain *chain, *chain_prev;
3b1e0a65 2511 struct net *net = sock_net(skb->sk);
5bc17018 2512 struct nlattr *tca[TCA_MAX + 1];
7960d1da 2513 struct Qdisc *q = NULL;
6529eaba 2514 struct tcf_block *block;
942b8165 2515 struct tcmsg *tcm = nlmsg_data(cb->nlh);
acb31fae
JP		 2516 long index_start;
2517 long index;
a10fa201 2518 u32 parent;
5bc17018 2519 int err;
1da177e4 2520
573ce260 2521 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
1da177e4 2522 return skb->len;
5bc17018 2523
8cb08174
JB		 2524 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
2525 NULL, cb->extack);
5bc17018
JP		 2526 if (err)
2527 return err;
2528
7960d1da 2529 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
787ce6d0 2530 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
7960d1da
JP		 2531 if (!block)
2532 goto out;
d680b352
JP		 2533 /* If we work with block index, q is NULL and parent value
2534 * will never be used in the following code. The check
2535 * in tcf_fill_node prevents it. However, compiler does not
2536 * see that far, so set parent to zero to silence the warning
2537 * about parent being uninitialized.
2538 */
2539 parent = 0;
a10fa201 2540 } else {
7960d1da
JP		 2541 const struct Qdisc_class_ops *cops;
2542 struct net_device *dev;
2543 unsigned long cl = 0;
2544
2545 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2546 if (!dev)
2547 return skb->len;
2548
2549 parent = tcm->tcm_parent;
2550 if (!parent) {
2551 q = dev->qdisc;
2552 parent = q->handle;
2553 } else {
2554 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2555 }
2556 if (!q)
2557 goto out;
2558 cops = q->ops->cl_ops;
2559 if (!cops)
143976ce 2560 goto out;
7960d1da
JP		 2561 if (!cops->tcf_block)
2562 goto out;
2563 if (TC_H_MIN(tcm->tcm_parent)) {
2564 cl = cops->find(q, tcm->tcm_parent);
2565 if (cl == 0)
2566 goto out;
2567 }
2568 block = cops->tcf_block(q, cl, NULL);
2569 if (!block)
2570 goto out;
2571 if (tcf_block_shared(block))
2572 q = NULL;
1da177e4 2573 }
1da177e4 2574
acb31fae
JP		 2575 index_start = cb->args[0];
2576 index = 0;
5bc17018 2577
bbf73830
VB		 2578 for (chain = __tcf_get_next_chain(block, NULL);
2579 chain;
2580 chain_prev = chain,
2581 chain = __tcf_get_next_chain(block, chain),
2582 tcf_chain_put(chain_prev)) {
5bc17018
JP		 2583 if (tca[TCA_CHAIN] &&
2584 nla_get_u32(tca[TCA_CHAIN]) != chain->index)
2585 continue;
a10fa201 2586 if (!tcf_chain_dump(chain, q, parent, skb, cb,
5ae437ad 2587 index_start, &index)) {
bbf73830 2588 tcf_chain_put(chain);
5ae437ad 2589 err = -EMSGSIZE;
5bc17018 2590 break;
5ae437ad 2591 }
5bc17018
JP		 2592 }
2593
787ce6d0 2594 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
12db03b6 2595 tcf_block_refcnt_put(block, true);
acb31fae 2596 cb->args[0] = index;
1da177e4 2597
1da177e4 2598out:
5ae437ad
RK		 2599 /* If we did no progress, the error (EMSGSIZE) is real */
2600 if (skb->len == 0 && err)
2601 return err;
1da177e4
LT		 2602 return skb->len;
2603}
2604
a5654820
VB		 2605static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops,
2606 void *tmplt_priv, u32 chain_index,
2607 struct net *net, struct sk_buff *skb,
2608 struct tcf_block *block,
32a4f5ec
JP		 2609 u32 portid, u32 seq, u16 flags, int event)
2610{
2611 unsigned char *b = skb_tail_pointer(skb);
9f407f17 2612 const struct tcf_proto_ops *ops;
32a4f5ec
JP		 2613 struct nlmsghdr *nlh;
2614 struct tcmsg *tcm;
9f407f17
JP		 2615 void *priv;
2616
a5654820
VB		 2617 ops = tmplt_ops;
2618 priv = tmplt_priv;
32a4f5ec
JP		 2619
2620 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
2621 if (!nlh)
2622 goto out_nlmsg_trim;
2623 tcm = nlmsg_data(nlh);
2624 tcm->tcm_family = AF_UNSPEC;
2625 tcm->tcm__pad1 = 0;
2626 tcm->tcm__pad2 = 0;
2627 tcm->tcm_handle = 0;
2628 if (block->q) {
2629 tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex;
2630 tcm->tcm_parent = block->q->handle;
2631 } else {
2632 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2633 tcm->tcm_block_index = block->index;
2634 }
2635
a5654820 2636 if (nla_put_u32(skb, TCA_CHAIN, chain_index))
32a4f5ec
JP		 2637 goto nla_put_failure;
2638
9f407f17
JP		 2639 if (ops) {
2640 if (nla_put_string(skb, TCA_KIND, ops->kind))
2641 goto nla_put_failure;
2642 if (ops->tmplt_dump(skb, net, priv) < 0)
2643 goto nla_put_failure;
2644 }
2645
32a4f5ec
JP		 2646 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2647 return skb->len;
2648
2649out_nlmsg_trim:
2650nla_put_failure:
2651 nlmsg_trim(skb, b);
2652 return -EMSGSIZE;
2653}
2654
2655static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
2656 u32 seq, u16 flags, int event, bool unicast)
2657{
2658 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2659 struct tcf_block *block = chain->block;
2660 struct net *net = block->net;
2661 struct sk_buff *skb;
5b5f99b1 2662 int err = 0;
32a4f5ec
JP		 2663
2664 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2665 if (!skb)
2666 return -ENOBUFS;
2667
a5654820
VB		 2668 if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
2669 chain->index, net, skb, block, portid,
32a4f5ec
JP		 2670 seq, flags, event) <= 0) {
2671 kfree_skb(skb);
2672 return -EINVAL;
2673 }
2674
2675 if (unicast)
5b5f99b1
ZW		 2676 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2677 else
2678 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
2679 flags & NLM_F_ECHO);
32a4f5ec 2680
5b5f99b1
ZW		 2681 if (err > 0)
2682 err = 0;
2683 return err;
32a4f5ec
JP		 2684}
2685
a5654820
VB		 2686static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
2687 void *tmplt_priv, u32 chain_index,
2688 struct tcf_block *block, struct sk_buff *oskb,
2689 u32 seq, u16 flags, bool unicast)
2690{
2691 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2692 struct net *net = block->net;
2693 struct sk_buff *skb;
2694
2695 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2696 if (!skb)
2697 return -ENOBUFS;
2698
2699 if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb,
2700 block, portid, seq, flags, RTM_DELCHAIN) <= 0) {
2701 kfree_skb(skb);
2702 return -EINVAL;
2703 }
2704
2705 if (unicast)
2706 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2707
2708 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
2709}
2710
9f407f17
JP		 2711static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
2712 struct nlattr **tca,
2713 struct netlink_ext_ack *extack)
2714{
2715 const struct tcf_proto_ops *ops;
2dd5616e 2716 char name[IFNAMSIZ];
9f407f17
JP		 2717 void *tmplt_priv;
2718
2719 /* If kind is not set, user did not specify template. */
2720 if (!tca[TCA_KIND])
2721 return 0;
2722
2dd5616e
ED		 2723 if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2724 NL_SET_ERR_MSG(extack, "Specified TC chain template name too long");
2725 return -EINVAL;
2726 }
2727
2728 ops = tcf_proto_lookup_ops(name, true, extack);
9f407f17
JP		 2729 if (IS_ERR(ops))
2730 return PTR_ERR(ops);
2731 if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) {
2732 NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier");
2733 return -EOPNOTSUPP;
2734 }
2735
2736 tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
2737 if (IS_ERR(tmplt_priv)) {
2738 module_put(ops->owner);
2739 return PTR_ERR(tmplt_priv);
2740 }
2741 chain->tmplt_ops = ops;
2742 chain->tmplt_priv = tmplt_priv;
2743 return 0;
2744}
2745
a5654820
VB		 2746static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
2747 void *tmplt_priv)
9f407f17 2748{
9f407f17 2749 /* If template ops are set, no work to do for us. */
a5654820 2750 if (!tmplt_ops)
9f407f17
JP		 2751 return;
2752
a5654820
VB		 2753 tmplt_ops->tmplt_destroy(tmplt_priv);
2754 module_put(tmplt_ops->owner);
9f407f17
JP		 2755}
2756
32a4f5ec
JP		 2757/* Add/delete/get a chain */
2758
2759static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
2760 struct netlink_ext_ack *extack)
2761{
2762 struct net *net = sock_net(skb->sk);
2763 struct nlattr *tca[TCA_MAX + 1];
2764 struct tcmsg *t;
2765 u32 parent;
2766 u32 chain_index;
2767 struct Qdisc *q = NULL;
2768 struct tcf_chain *chain = NULL;
2769 struct tcf_block *block;
2770 unsigned long cl;
2771 int err;
2772
2773 if (n->nlmsg_type != RTM_GETCHAIN &&
2774 !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2775 return -EPERM;
2776
2777replay:
8cb08174
JB		 2778 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2779 rtm_tca_policy, extack);
32a4f5ec
JP		 2780 if (err < 0)
2781 return err;
2782
2783 t = nlmsg_data(n);
2784 parent = t->tcm_parent;
2785 cl = 0;
2786
2787 block = tcf_block_find(net, &q, &parent, &cl,
2788 t->tcm_ifindex, t->tcm_block_index, extack);
2789 if (IS_ERR(block))
2790 return PTR_ERR(block);
2791
2792 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2793 if (chain_index > TC_ACT_EXT_VAL_MASK) {
2794 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
e368fdb6
VB		 2795 err = -EINVAL;
2796 goto errout_block;
32a4f5ec 2797 }
2cbfab07
VB		 2798
2799 mutex_lock(&block->lock);
32a4f5ec
JP		 2800 chain = tcf_chain_lookup(block, chain_index);
2801 if (n->nlmsg_type == RTM_NEWCHAIN) {
2802 if (chain) {
3d32f4c5 2803 if (tcf_chain_held_by_acts_only(chain)) {
1f3ed383 2804 /* The chain exists only because there is
3d32f4c5 2805 * some action referencing it.
1f3ed383
JP		 2806 */
2807 tcf_chain_hold(chain);
2808 } else {
2809 NL_SET_ERR_MSG(extack, "Filter chain already exists");
e368fdb6 2810 err = -EEXIST;
2cbfab07 2811 goto errout_block_locked;
1f3ed383
JP		 2812 }
2813 } else {
2814 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2815 NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
e368fdb6 2816 err = -ENOENT;
2cbfab07 2817 goto errout_block_locked;
1f3ed383
JP		 2818 }
2819 chain = tcf_chain_create(block, chain_index);
2820 if (!chain) {
2821 NL_SET_ERR_MSG(extack, "Failed to create filter chain");
e368fdb6 2822 err = -ENOMEM;
2cbfab07 2823 goto errout_block_locked;
1f3ed383 2824 }
32a4f5ec
JP		 2825 }
2826 } else {
3d32f4c5 2827 if (!chain || tcf_chain_held_by_acts_only(chain)) {
32a4f5ec 2828 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
e368fdb6 2829 err = -EINVAL;
2cbfab07 2830 goto errout_block_locked;
32a4f5ec
JP		 2831 }
2832 tcf_chain_hold(chain);
2833 }
2834
2cbfab07
VB		 2835 if (n->nlmsg_type == RTM_NEWCHAIN) {
2836 /* Modifying chain requires holding parent block lock. In case
2837 * the chain was successfully added, take a reference to the
2838 * chain. This ensures that an empty chain does not disappear at
2839 * the end of this function.
2840 */
2841 tcf_chain_hold(chain);
2842 chain->explicitly_created = true;
2843 }
2844 mutex_unlock(&block->lock);
2845
32a4f5ec
JP		 2846 switch (n->nlmsg_type) {
2847 case RTM_NEWCHAIN:
9f407f17 2848 err = tc_chain_tmplt_add(chain, net, tca, extack);
2cbfab07
VB		 2849 if (err) {
2850 tcf_chain_put_explicitly_created(chain);
9f407f17 2851 goto errout;
2cbfab07
VB		 2852 }
2853
32a4f5ec
JP		 2854 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
2855 RTM_NEWCHAIN, false);
2856 break;
2857 case RTM_DELCHAIN:
f5b9bac7 2858 tfilter_notify_chain(net, skb, block, q, parent, n,
12db03b6 2859 chain, RTM_DELTFILTER, true);
32a4f5ec 2860 /* Flush the chain first as the user requested chain removal. */
12db03b6 2861 tcf_chain_flush(chain, true);
32a4f5ec
JP		 2862 /* In case the chain was successfully deleted, put a reference
2863 * to the chain previously taken during addition.
2864 */
2865 tcf_chain_put_explicitly_created(chain);
2866 break;
2867 case RTM_GETCHAIN:
32a4f5ec
JP		 2868 err = tc_chain_notify(chain, skb, n->nlmsg_seq,
2869 n->nlmsg_seq, n->nlmsg_type, true);
2870 if (err < 0)
2871 NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
2872 break;
2873 default:
2874 err = -EOPNOTSUPP;
2875 NL_SET_ERR_MSG(extack, "Unsupported message type");
2876 goto errout;
2877 }
2878
2879errout:
2880 tcf_chain_put(chain);
e368fdb6 2881errout_block:
12db03b6 2882 tcf_block_release(q, block, true);
32a4f5ec
JP		 2883 if (err == -EAGAIN)
2884 /* Replay the request. */
2885 goto replay;
2886 return err;
2cbfab07
VB		 2887
2888errout_block_locked:
2889 mutex_unlock(&block->lock);
2890 goto errout_block;
32a4f5ec
JP		 2891}
2892
2893/* called with RTNL */
2894static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
2895{
2896 struct net *net = sock_net(skb->sk);
2897 struct nlattr *tca[TCA_MAX + 1];
2898 struct Qdisc *q = NULL;
2899 struct tcf_block *block;
32a4f5ec 2900 struct tcmsg *tcm = nlmsg_data(cb->nlh);
ace4a267 2901 struct tcf_chain *chain;
32a4f5ec
JP		 2902 long index_start;
2903 long index;
2904 u32 parent;
2905 int err;
2906
2907 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2908 return skb->len;
2909
8cb08174
JB		 2910 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
2911 rtm_tca_policy, cb->extack);
32a4f5ec
JP		 2912 if (err)
2913 return err;
2914
2915 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
787ce6d0 2916 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
32a4f5ec
JP		 2917 if (!block)
2918 goto out;
2919 /* If we work with block index, q is NULL and parent value
2920 * will never be used in the following code. The check
2921 * in tcf_fill_node prevents it. However, compiler does not
2922 * see that far, so set parent to zero to silence the warning
2923 * about parent being uninitialized.
2924 */
2925 parent = 0;
2926 } else {
2927 const struct Qdisc_class_ops *cops;
2928 struct net_device *dev;
2929 unsigned long cl = 0;
2930
2931 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2932 if (!dev)
2933 return skb->len;
2934
2935 parent = tcm->tcm_parent;
2936 if (!parent) {
2937 q = dev->qdisc;
2938 parent = q->handle;
2939 } else {
2940 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2941 }
2942 if (!q)
2943 goto out;
2944 cops = q->ops->cl_ops;
2945 if (!cops)
2946 goto out;
2947 if (!cops->tcf_block)
2948 goto out;
2949 if (TC_H_MIN(tcm->tcm_parent)) {
2950 cl = cops->find(q, tcm->tcm_parent);
2951 if (cl == 0)
2952 goto out;
2953 }
2954 block = cops->tcf_block(q, cl, NULL);
2955 if (!block)
2956 goto out;
2957 if (tcf_block_shared(block))
2958 q = NULL;
2959 }
2960
2961 index_start = cb->args[0];
2962 index = 0;
2963
ace4a267
VB		 2964 mutex_lock(&block->lock);
2965 list_for_each_entry(chain, &block->chain_list, list) {
32a4f5ec
JP		 2966 if ((tca[TCA_CHAIN] &&
2967 nla_get_u32(tca[TCA_CHAIN]) != chain->index))
2968 continue;
2969 if (index < index_start) {
2970 index++;
2971 continue;
2972 }
ace4a267
VB		 2973 if (tcf_chain_held_by_acts_only(chain))
2974 continue;
a5654820
VB		 2975 err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
2976 chain->index, net, skb, block,
32a4f5ec
JP		 2977 NETLINK_CB(cb->skb).portid,
2978 cb->nlh->nlmsg_seq, NLM_F_MULTI,
2979 RTM_NEWCHAIN);
ace4a267 2980 if (err <= 0)
32a4f5ec
JP		 2981 break;
2982 index++;
2983 }
ace4a267 2984 mutex_unlock(&block->lock);
32a4f5ec 2985
787ce6d0 2986 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
12db03b6 2987 tcf_block_refcnt_put(block, true);
32a4f5ec
JP		 2988 cb->args[0] = index;
2989
2990out:
2991 /* If we did no progress, the error (EMSGSIZE) is real */
2992 if (skb->len == 0 && err)
2993 return err;
2994 return skb->len;
2995}
2996
18d0264f 2997void tcf_exts_destroy(struct tcf_exts *exts)
1da177e4
LT		 2998{
2999#ifdef CONFIG_NET_CLS_ACT
3d66b89c
ED		 3000 if (exts->actions) {
3001 tcf_action_destroy(exts->actions, TCA_ACT_UNBIND);
3002 kfree(exts->actions);
3003 }
22dc13c8 3004 exts->nr_actions = 0;
1da177e4
LT		 3005#endif
3006}
aa767bfe 3007EXPORT_SYMBOL(tcf_exts_destroy);
1da177e4 3008
c1b52739 3009int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
50a56190 3010 struct nlattr *rate_tlv, struct tcf_exts *exts, bool ovr,
ec6743a1 3011 bool rtnl_held, struct netlink_ext_ack *extack)
1da177e4 3012{
1da177e4
LT		 3013#ifdef CONFIG_NET_CLS_ACT
3014 {
1da177e4 3015 struct tc_action *act;
d04e6990 3016 size_t attr_size = 0;
1da177e4 3017
5da57f42 3018 if (exts->police && tb[exts->police]) {
9fb9f251
JP		 3019 act = tcf_action_init_1(net, tp, tb[exts->police],
3020 rate_tlv, "police", ovr,
ec6743a1
VB		 3021 TCA_ACT_BIND, rtnl_held,
3022 extack);
ab27cfb8
PM		 3023 if (IS_ERR(act))
3024 return PTR_ERR(act);
1da177e4 3025
33be6271 3026 act->type = exts->type = TCA_OLD_COMPAT;
22dc13c8
WC		 3027 exts->actions[0] = act;
3028 exts->nr_actions = 1;
5da57f42 3029 } else if (exts->action && tb[exts->action]) {
90b73b77 3030 int err;
22dc13c8 3031
9fb9f251
JP		 3032 err = tcf_action_init(net, tp, tb[exts->action],
3033 rate_tlv, NULL, ovr, TCA_ACT_BIND,
ec6743a1
VB		 3034 exts->actions, &attr_size,
3035 rtnl_held, extack);
90b73b77 3036 if (err < 0)
33be6271 3037 return err;
90b73b77 3038 exts->nr_actions = err;
1da177e4
LT		 3039 }
3040 }
1da177e4 3041#else
5da57f42 3042 if ((exts->action && tb[exts->action]) ||
50a56190
AA		 3043 (exts->police && tb[exts->police])) {
3044 NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
1da177e4 3045 return -EOPNOTSUPP;
50a56190 3046 }
1da177e4
LT		 3047#endif
3048
3049 return 0;
3050}
aa767bfe 3051EXPORT_SYMBOL(tcf_exts_validate);
1da177e4 3052
9b0d4446 3053void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
1da177e4
LT		 3054{
3055#ifdef CONFIG_NET_CLS_ACT
22dc13c8
WC		 3056 struct tcf_exts old = *dst;
3057
9b0d4446 3058 *dst = *src;
22dc13c8 3059 tcf_exts_destroy(&old);
1da177e4
LT		 3060#endif
3061}
aa767bfe 3062EXPORT_SYMBOL(tcf_exts_change);
1da177e4 3063
22dc13c8
WC		 3064#ifdef CONFIG_NET_CLS_ACT
3065static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
3066{
3067 if (exts->nr_actions == 0)
3068 return NULL;
3069 else
3070 return exts->actions[0];
3071}
3072#endif
33be6271 3073
5da57f42 3074int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
1da177e4
LT		 3075{
3076#ifdef CONFIG_NET_CLS_ACT
9cc63db5
CW		 3077 struct nlattr *nest;
3078
978dfd8d 3079 if (exts->action && tcf_exts_has_actions(exts)) {
1da177e4
LT		 3080 /*
3081 * again for backward compatible mode - we want
3082 * to work with both old and new modes of entering
3083 * tc data even if iproute2 was newer - jhs
3084 */
33be6271 3085 if (exts->type != TCA_OLD_COMPAT) {
ae0be8de 3086 nest = nla_nest_start_noflag(skb, exts->action);
4b3550ef
PM		 3087 if (nest == NULL)
3088 goto nla_put_failure;
22dc13c8 3089
90b73b77 3090 if (tcf_action_dump(skb, exts->actions, 0, 0) < 0)
add93b61 3091 goto nla_put_failure;
4b3550ef 3092 nla_nest_end(skb, nest);
5da57f42 3093 } else if (exts->police) {
33be6271 3094 struct tc_action *act = tcf_exts_first_act(exts);
ae0be8de 3095 nest = nla_nest_start_noflag(skb, exts->police);
63acd680 3096 if (nest == NULL || !act)
4b3550ef 3097 goto nla_put_failure;
33be6271 3098 if (tcf_action_dump_old(skb, act, 0, 0) < 0)
add93b61 3099 goto nla_put_failure;
4b3550ef 3100 nla_nest_end(skb, nest);
1da177e4
LT		 3101 }
3102 }
1da177e4 3103 return 0;
9cc63db5
CW		 3104
3105nla_put_failure:
3106 nla_nest_cancel(skb, nest);
1da177e4 3107 return -1;
9cc63db5
CW		 3108#else
3109 return 0;
3110#endif
1da177e4 3111}
aa767bfe 3112EXPORT_SYMBOL(tcf_exts_dump);
1da177e4 3113
aa767bfe 3114
5da57f42 3115int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
1da177e4
LT		 3116{
3117#ifdef CONFIG_NET_CLS_ACT
33be6271 3118 struct tc_action *a = tcf_exts_first_act(exts);
b057df24 3119 if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0)
33be6271 3120 return -1;
1da177e4
LT		 3121#endif
3122 return 0;
1da177e4 3123}
aa767bfe 3124EXPORT_SYMBOL(tcf_exts_dump_stats);
1da177e4 3125
40119211
VB		 3126static void tcf_block_offload_inc(struct tcf_block *block, u32 *flags)
3127{
3128 if (*flags & TCA_CLS_FLAGS_IN_HW)
3129 return;
3130 *flags |= TCA_CLS_FLAGS_IN_HW;
3131 atomic_inc(&block->offloadcnt);
3132}
3133
3134static void tcf_block_offload_dec(struct tcf_block *block, u32 *flags)
3135{
3136 if (!(*flags & TCA_CLS_FLAGS_IN_HW))
3137 return;
3138 *flags &= ~TCA_CLS_FLAGS_IN_HW;
3139 atomic_dec(&block->offloadcnt);
3140}
3141
3142static void tc_cls_offload_cnt_update(struct tcf_block *block,
3143 struct tcf_proto *tp, u32 *cnt,
3144 u32 *flags, u32 diff, bool add)
3145{
3146 lockdep_assert_held(&block->cb_lock);
3147
3148 spin_lock(&tp->lock);
3149 if (add) {
3150 if (!*cnt)
3151 tcf_block_offload_inc(block, flags);
3152 *cnt += diff;
3153 } else {
3154 *cnt -= diff;
3155 if (!*cnt)
3156 tcf_block_offload_dec(block, flags);
3157 }
3158 spin_unlock(&tp->lock);
3159}
3160
3161static void
3162tc_cls_offload_cnt_reset(struct tcf_block *block, struct tcf_proto *tp,
3163 u32 *cnt, u32 *flags)
3164{
3165 lockdep_assert_held(&block->cb_lock);
3166
3167 spin_lock(&tp->lock);
3168 tcf_block_offload_dec(block, flags);
3169 *cnt = 0;
3170 spin_unlock(&tp->lock);
3171}
3172
3173static int
3174__tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3175 void *type_data, bool err_stop)
717503b9 3176{
955bcb6e 3177 struct flow_block_cb *block_cb;
aeb3fecd
CW		 3178 int ok_count = 0;
3179 int err;
3180
14bfb13f 3181 list_for_each_entry(block_cb, &block->flow_block.cb_list, list) {
aeb3fecd
CW		 3182 err = block_cb->cb(type, type_data, block_cb->cb_priv);
3183 if (err) {
40119211
VB		 3184 if (err_stop)
3185 return err;
aeb3fecd
CW		 3186 } else {
3187 ok_count++;
3188 }
3189 }
40119211
VB		 3190 return ok_count;
3191}
3192
3193int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3194 void *type_data, bool err_stop, bool rtnl_held)
3195{
11bd634d 3196 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
40119211
VB		 3197 int ok_count;
3198
11bd634d
VB		 3199retry:
3200 if (take_rtnl)
3201 rtnl_lock();
40119211 3202 down_read(&block->cb_lock);
11bd634d
VB		 3203 /* Need to obtain rtnl lock if block is bound to devs that require it.
3204 * In block bind code cb_lock is obtained while holding rtnl, so we must
3205 * obtain the locks in same order here.
3206 */
3207 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3208 up_read(&block->cb_lock);
3209 take_rtnl = true;
3210 goto retry;
3211 }
3212
40119211 3213 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
11bd634d 3214
4f8116c8 3215 up_read(&block->cb_lock);
11bd634d
VB		 3216 if (take_rtnl)
3217 rtnl_unlock();
aeb3fecd 3218 return ok_count;
717503b9
JP		 3219}
3220EXPORT_SYMBOL(tc_setup_cb_call);
b3f55bdd 3221
40119211
VB		 3222/* Non-destructive filter add. If filter that wasn't already in hardware is
3223 * successfully offloaded, increment block offloads counter. On failure,
3224 * previously offloaded filter is considered to be intact and offloads counter
3225 * is not decremented.
3226 */
3227
3228int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp,
3229 enum tc_setup_type type, void *type_data, bool err_stop,
3230 u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3231{
11bd634d 3232 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
40119211
VB		 3233 int ok_count;
3234
11bd634d
VB		 3235retry:
3236 if (take_rtnl)
3237 rtnl_lock();
40119211 3238 down_read(&block->cb_lock);
11bd634d
VB		 3239 /* Need to obtain rtnl lock if block is bound to devs that require it.
3240 * In block bind code cb_lock is obtained while holding rtnl, so we must
3241 * obtain the locks in same order here.
3242 */
3243 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3244 up_read(&block->cb_lock);
3245 take_rtnl = true;
3246 goto retry;
3247 }
3248
40119211
VB		 3249 /* Make sure all netdevs sharing this block are offload-capable. */
3250 if (block->nooffloaddevcnt && err_stop) {
3251 ok_count = -EOPNOTSUPP;
3252 goto err_unlock;
3253 }
3254
3255 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
a449a3e7
VB		 3256 if (ok_count < 0)
3257 goto err_unlock;
3258
3259 if (tp->ops->hw_add)
3260 tp->ops->hw_add(tp, type_data);
40119211
VB		 3261 if (ok_count > 0)
3262 tc_cls_offload_cnt_update(block, tp, in_hw_count, flags,
3263 ok_count, true);
3264err_unlock:
3265 up_read(&block->cb_lock);
11bd634d
VB		 3266 if (take_rtnl)
3267 rtnl_unlock();
40119211
VB		 3268 return ok_count < 0 ? ok_count : 0;
3269}
3270EXPORT_SYMBOL(tc_setup_cb_add);
3271
3272/* Destructive filter replace. If filter that wasn't already in hardware is
3273 * successfully offloaded, increment block offload counter. On failure,
3274 * previously offloaded filter is considered to be destroyed and offload counter
3275 * is decremented.
3276 */
3277
3278int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp,
3279 enum tc_setup_type type, void *type_data, bool err_stop,
3280 u32 *old_flags, unsigned int *old_in_hw_count,
3281 u32 *new_flags, unsigned int *new_in_hw_count,
3282 bool rtnl_held)
3283{
11bd634d 3284 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
40119211
VB		 3285 int ok_count;
3286
11bd634d
VB		 3287retry:
3288 if (take_rtnl)
3289 rtnl_lock();
40119211 3290 down_read(&block->cb_lock);
11bd634d
VB		 3291 /* Need to obtain rtnl lock if block is bound to devs that require it.
3292 * In block bind code cb_lock is obtained while holding rtnl, so we must
3293 * obtain the locks in same order here.
3294 */
3295 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3296 up_read(&block->cb_lock);
3297 take_rtnl = true;
3298 goto retry;
3299 }
3300
40119211
VB		 3301 /* Make sure all netdevs sharing this block are offload-capable. */
3302 if (block->nooffloaddevcnt && err_stop) {
3303 ok_count = -EOPNOTSUPP;
3304 goto err_unlock;
3305 }
3306
3307 tc_cls_offload_cnt_reset(block, tp, old_in_hw_count, old_flags);
a449a3e7
VB		 3308 if (tp->ops->hw_del)
3309 tp->ops->hw_del(tp, type_data);
40119211
VB		 3310
3311 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
a449a3e7
VB		 3312 if (ok_count < 0)
3313 goto err_unlock;
3314
3315 if (tp->ops->hw_add)
3316 tp->ops->hw_add(tp, type_data);
40119211 3317 if (ok_count > 0)
a449a3e7
VB		 3318 tc_cls_offload_cnt_update(block, tp, new_in_hw_count,
3319 new_flags, ok_count, true);
40119211
VB		 3320err_unlock:
3321 up_read(&block->cb_lock);
11bd634d
VB		 3322 if (take_rtnl)
3323 rtnl_unlock();
40119211
VB		 3324 return ok_count < 0 ? ok_count : 0;
3325}
3326EXPORT_SYMBOL(tc_setup_cb_replace);
3327
3328/* Destroy filter and decrement block offload counter, if filter was previously
3329 * offloaded.
3330 */
3331
3332int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp,
3333 enum tc_setup_type type, void *type_data, bool err_stop,
3334 u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3335{
11bd634d 3336 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
40119211
VB		 3337 int ok_count;
3338
11bd634d
VB		 3339retry:
3340 if (take_rtnl)
3341 rtnl_lock();
40119211 3342 down_read(&block->cb_lock);
11bd634d
VB		 3343 /* Need to obtain rtnl lock if block is bound to devs that require it.
3344 * In block bind code cb_lock is obtained while holding rtnl, so we must
3345 * obtain the locks in same order here.
3346 */
3347 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3348 up_read(&block->cb_lock);
3349 take_rtnl = true;
3350 goto retry;
3351 }
3352
40119211
VB		 3353 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3354
3355 tc_cls_offload_cnt_reset(block, tp, in_hw_count, flags);
a449a3e7
VB		 3356 if (tp->ops->hw_del)
3357 tp->ops->hw_del(tp, type_data);
3358
40119211 3359 up_read(&block->cb_lock);
11bd634d
VB		 3360 if (take_rtnl)
3361 rtnl_unlock();
40119211
VB		 3362 return ok_count < 0 ? ok_count : 0;
3363}
3364EXPORT_SYMBOL(tc_setup_cb_destroy);
3365
3366int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp,
3367 bool add, flow_setup_cb_t *cb,
3368 enum tc_setup_type type, void *type_data,
3369 void *cb_priv, u32 *flags, unsigned int *in_hw_count)
3370{
3371 int err = cb(type, type_data, cb_priv);
3372
3373 if (err) {
3374 if (add && tc_skip_sw(*flags))
3375 return err;
3376 } else {
3377 tc_cls_offload_cnt_update(block, tp, in_hw_count, flags, 1,
3378 add);
3379 }
3380
3381 return 0;
3382}
3383EXPORT_SYMBOL(tc_setup_cb_reoffload);
3384
5a6ff4b1
VB		 3385void tc_cleanup_flow_action(struct flow_action *flow_action)
3386{
3387 struct flow_action_entry *entry;
3388 int i;
3389
1158958a
VB		 3390 flow_action_for_each(i, entry, flow_action)
3391 if (entry->destructor)
3392 entry->destructor(entry->destructor_priv);
5a6ff4b1
VB		 3393}
3394EXPORT_SYMBOL(tc_cleanup_flow_action);
3395
1158958a
VB		 3396static void tcf_mirred_get_dev(struct flow_action_entry *entry,
3397 const struct tc_action *act)
3398{
470d5060
VB		 3399#ifdef CONFIG_NET_CLS_ACT
3400 entry->dev = act->ops->get_dev(act, &entry->destructor);
1158958a
VB		 3401 if (!entry->dev)
3402 return;
1158958a 3403 entry->destructor_priv = entry->dev;
470d5060 3404#endif
1158958a
VB		 3405}
3406
3407static void tcf_tunnel_encap_put_tunnel(void *priv)
3408{
3409 struct ip_tunnel_info *tunnel = priv;
3410
3411 kfree(tunnel);
3412}
3413
3414static int tcf_tunnel_encap_get_tunnel(struct flow_action_entry *entry,
3415 const struct tc_action *act)
3416{
3417 entry->tunnel = tcf_tunnel_info_copy(act);
3418 if (!entry->tunnel)
3419 return -ENOMEM;
3420 entry->destructor = tcf_tunnel_encap_put_tunnel;
3421 entry->destructor_priv = entry->tunnel;
3422 return 0;
3423}
3424
4a5da47d
VB		 3425static void tcf_sample_get_group(struct flow_action_entry *entry,
3426 const struct tc_action *act)
3427{
3428#ifdef CONFIG_NET_CLS_ACT
3429 entry->sample.psample_group =
3430 act->ops->get_psample_group(act, &entry->destructor);
3431 entry->destructor_priv = entry->sample.psample_group;
3432#endif
3433}
3434
3a7b6861 3435int tc_setup_flow_action(struct flow_action *flow_action,
b15e7a6e 3436 const struct tcf_exts *exts)
3a7b6861 3437{
7a472814 3438 struct tc_action *act;
9838b20a 3439 int i, j, k, err = 0;
3a7b6861
PNA		 3440
3441 if (!exts)
3442 return 0;
3443
3444 j = 0;
3445 tcf_exts_for_each_action(i, act, exts) {
3446 struct flow_action_entry *entry;
3447
3448 entry = &flow_action->entries[j];
7a472814 3449 spin_lock_bh(&act->tcfa_lock);
3a7b6861
PNA		 3450 if (is_tcf_gact_ok(act)) {
3451 entry->id = FLOW_ACTION_ACCEPT;
3452 } else if (is_tcf_gact_shot(act)) {
3453 entry->id = FLOW_ACTION_DROP;
3454 } else if (is_tcf_gact_trap(act)) {
3455 entry->id = FLOW_ACTION_TRAP;
3456 } else if (is_tcf_gact_goto_chain(act)) {
3457 entry->id = FLOW_ACTION_GOTO;
3458 entry->chain_index = tcf_gact_goto_chain_index(act);
3459 } else if (is_tcf_mirred_egress_redirect(act)) {
3460 entry->id = FLOW_ACTION_REDIRECT;
1158958a 3461 tcf_mirred_get_dev(entry, act);
3a7b6861
PNA		 3462 } else if (is_tcf_mirred_egress_mirror(act)) {
3463 entry->id = FLOW_ACTION_MIRRED;
1158958a 3464 tcf_mirred_get_dev(entry, act);
48e584ac
JH		 3465 } else if (is_tcf_mirred_ingress_redirect(act)) {
3466 entry->id = FLOW_ACTION_REDIRECT_INGRESS;
1158958a 3467 tcf_mirred_get_dev(entry, act);
48e584ac
JH		 3468 } else if (is_tcf_mirred_ingress_mirror(act)) {
3469 entry->id = FLOW_ACTION_MIRRED_INGRESS;
1158958a 3470 tcf_mirred_get_dev(entry, act);
3a7b6861
PNA		 3471 } else if (is_tcf_vlan(act)) {
3472 switch (tcf_vlan_action(act)) {
3473 case TCA_VLAN_ACT_PUSH:
3474 entry->id = FLOW_ACTION_VLAN_PUSH;
3475 entry->vlan.vid = tcf_vlan_push_vid(act);
3476 entry->vlan.proto = tcf_vlan_push_proto(act);
3477 entry->vlan.prio = tcf_vlan_push_prio(act);
3478 break;
3479 case TCA_VLAN_ACT_POP:
3480 entry->id = FLOW_ACTION_VLAN_POP;
3481 break;
3482 case TCA_VLAN_ACT_MODIFY:
3483 entry->id = FLOW_ACTION_VLAN_MANGLE;
3484 entry->vlan.vid = tcf_vlan_push_vid(act);
3485 entry->vlan.proto = tcf_vlan_push_proto(act);
3486 entry->vlan.prio = tcf_vlan_push_prio(act);
3487 break;
3488 default:
9838b20a 3489 err = -EOPNOTSUPP;
7a472814 3490 goto err_out_locked;
3a7b6861
PNA		 3491 }
3492 } else if (is_tcf_tunnel_set(act)) {
3493 entry->id = FLOW_ACTION_TUNNEL_ENCAP;
1158958a
VB		 3494 err = tcf_tunnel_encap_get_tunnel(entry, act);
3495 if (err)
7a472814 3496 goto err_out_locked;
3a7b6861
PNA		 3497 } else if (is_tcf_tunnel_release(act)) {
3498 entry->id = FLOW_ACTION_TUNNEL_DECAP;
3a7b6861
PNA		 3499 } else if (is_tcf_pedit(act)) {
3500 for (k = 0; k < tcf_pedit_nkeys(act); k++) {
3501 switch (tcf_pedit_cmd(act, k)) {
3502 case TCA_PEDIT_KEY_EX_CMD_SET:
3503 entry->id = FLOW_ACTION_MANGLE;
3504 break;
3505 case TCA_PEDIT_KEY_EX_CMD_ADD:
3506 entry->id = FLOW_ACTION_ADD;
3507 break;
3508 default:
9838b20a 3509 err = -EOPNOTSUPP;
7a472814 3510 goto err_out_locked;
3a7b6861
PNA		 3511 }
3512 entry->mangle.htype = tcf_pedit_htype(act, k);
3513 entry->mangle.mask = tcf_pedit_mask(act, k);
3514 entry->mangle.val = tcf_pedit_val(act, k);
3515 entry->mangle.offset = tcf_pedit_offset(act, k);
3516 entry = &flow_action->entries[++j];
3517 }
3518 } else if (is_tcf_csum(act)) {
3519 entry->id = FLOW_ACTION_CSUM;
3520 entry->csum_flags = tcf_csum_update_flags(act);
3521 } else if (is_tcf_skbedit_mark(act)) {
3522 entry->id = FLOW_ACTION_MARK;
3523 entry->mark = tcf_skbedit_mark(act);
a7a7be60
PJV		 3524 } else if (is_tcf_sample(act)) {
3525 entry->id = FLOW_ACTION_SAMPLE;
a7a7be60
PJV		 3526 entry->sample.trunc_size = tcf_sample_trunc_size(act);
3527 entry->sample.truncate = tcf_sample_truncate(act);
3528 entry->sample.rate = tcf_sample_rate(act);
4a5da47d 3529 tcf_sample_get_group(entry, act);
8c8cfc6e
PJV		 3530 } else if (is_tcf_police(act)) {
3531 entry->id = FLOW_ACTION_POLICE;
3532 entry->police.burst = tcf_police_tcfp_burst(act);
3533 entry->police.rate_bytes_ps =
3534 tcf_police_rate_bytes_ps(act);
b57dc7c1
PB		 3535 } else if (is_tcf_ct(act)) {
3536 entry->id = FLOW_ACTION_CT;
3537 entry->ct.action = tcf_ct_action(act);
3538 entry->ct.zone = tcf_ct_zone(act);
6749d590
JH		 3539 } else if (is_tcf_mpls(act)) {
3540 switch (tcf_mpls_action(act)) {
3541 case TCA_MPLS_ACT_PUSH:
3542 entry->id = FLOW_ACTION_MPLS_PUSH;
3543 entry->mpls_push.proto = tcf_mpls_proto(act);
3544 entry->mpls_push.label = tcf_mpls_label(act);
3545 entry->mpls_push.tc = tcf_mpls_tc(act);
3546 entry->mpls_push.bos = tcf_mpls_bos(act);
3547 entry->mpls_push.ttl = tcf_mpls_ttl(act);
3548 break;
3549 case TCA_MPLS_ACT_POP:
3550 entry->id = FLOW_ACTION_MPLS_POP;
3551 entry->mpls_pop.proto = tcf_mpls_proto(act);
3552 break;
3553 case TCA_MPLS_ACT_MODIFY:
3554 entry->id = FLOW_ACTION_MPLS_MANGLE;
3555 entry->mpls_mangle.label = tcf_mpls_label(act);
3556 entry->mpls_mangle.tc = tcf_mpls_tc(act);
3557 entry->mpls_mangle.bos = tcf_mpls_bos(act);
3558 entry->mpls_mangle.ttl = tcf_mpls_ttl(act);
3559 break;
3560 default:
7a472814 3561 goto err_out_locked;
6749d590 3562 }
fb1b775a
JH		 3563 } else if (is_tcf_skbedit_ptype(act)) {
3564 entry->id = FLOW_ACTION_PTYPE;
3565 entry->ptype = tcf_skbedit_ptype(act);
3a7b6861 3566 } else {
9838b20a 3567 err = -EOPNOTSUPP;
7a472814 3568 goto err_out_locked;
3a7b6861 3569 }
7a472814 3570 spin_unlock_bh(&act->tcfa_lock);
3a7b6861
PNA		 3571
3572 if (!is_tcf_pedit(act))
3573 j++;
3574 }
9838b20a 3575
3a7b6861 3576err_out:
5a6ff4b1
VB		 3577 if (err)
3578 tc_cleanup_flow_action(flow_action);
3579
9838b20a 3580 return err;
7a472814
VB		 3581err_out_locked:
3582 spin_unlock_bh(&act->tcfa_lock);
3583 goto err_out;
3a7b6861
PNA		 3584}
3585EXPORT_SYMBOL(tc_setup_flow_action);
3586
e3ab786b
PNA		 3587unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
3588{
3589 unsigned int num_acts = 0;
3590 struct tc_action *act;
3591 int i;
3592
3593 tcf_exts_for_each_action(i, act, exts) {
3594 if (is_tcf_pedit(act))
3595 num_acts += tcf_pedit_nkeys(act);
3596 else
3597 num_acts++;
3598 }
3599 return num_acts;
3600}
3601EXPORT_SYMBOL(tcf_exts_num_actions);
3602
48617387
JP		 3603static __net_init int tcf_net_init(struct net *net)
3604{
3605 struct tcf_net *tn = net_generic(net, tcf_net_id);
3606
ab281629 3607 spin_lock_init(&tn->idr_lock);
48617387
JP		 3608 idr_init(&tn->idr);
3609 return 0;
3610}
3611
3612static void __net_exit tcf_net_exit(struct net *net)
3613{
3614 struct tcf_net *tn = net_generic(net, tcf_net_id);
3615
3616 idr_destroy(&tn->idr);
3617}
3618
3619static struct pernet_operations tcf_net_ops = {
3620 .init = tcf_net_init,
3621 .exit = tcf_net_exit,
3622 .id = &tcf_net_id,
3623 .size = sizeof(struct tcf_net),
3624};
3625
25a443f7
JH		 3626static struct flow_indr_block_entry block_entry = {
3627 .cb = tc_indr_block_get_and_cmd,
3628 .list = LIST_HEAD_INIT(block_entry.list),
1150ab0f 3629};
3630
1da177e4
LT		 3631static int __init tc_filter_init(void)
3632{
48617387
JP		 3633 int err;
3634
7aa0045d
CW		 3635 tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0);
3636 if (!tc_filter_wq)
3637 return -ENOMEM;
3638
48617387
JP		 3639 err = register_pernet_subsys(&tcf_net_ops);
3640 if (err)
3641 goto err_register_pernet_subsys;
3642
25a443f7 3643 flow_indr_add_block_cb(&block_entry);
1150ab0f 3644
470502de
VB		 3645 rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL,
3646 RTNL_FLAG_DOIT_UNLOCKED);
3647 rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL,
3648 RTNL_FLAG_DOIT_UNLOCKED);
c431f89b 3649 rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter,
470502de 3650 tc_dump_tfilter, RTNL_FLAG_DOIT_UNLOCKED);
32a4f5ec
JP		 3651 rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0);
3652 rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0);
3653 rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain,
3654 tc_dump_chain, 0);
1da177e4 3655
1da177e4 3656 return 0;
48617387
JP		 3657
3658err_register_pernet_subsys:
3659 destroy_workqueue(tc_filter_wq);
3660 return err;
1da177e4
LT		 3661}
3662
3663subsys_initcall(tc_filter_init);
Linux 6.x block layer and io_uring tree(s)