projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
qlge: fix some indentation issues
[linux-block.git] / net / sched / cls_flower.c
CommitLineData
77b9900e
JP		 1/*
2 * net/sched/cls_flower.c Flower classifier
3 *
4 * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 */
11
12#include <linux/kernel.h>
13#include <linux/init.h>
14#include <linux/module.h>
15#include <linux/rhashtable.h>
d9363774 16#include <linux/workqueue.h>
77b9900e
JP		 17
18#include <linux/if_ether.h>
19#include <linux/in6.h>
20#include <linux/ip.h>
a577d8f7 21#include <linux/mpls.h>
77b9900e
JP		 22
23#include <net/sch_generic.h>
24#include <net/pkt_cls.h>
25#include <net/ip.h>
26#include <net/flow_dissector.h>
0a6e7778 27#include <net/geneve.h>
77b9900e 28
bc3103f1
AV		 29#include <net/dst.h>
30#include <net/dst_metadata.h>
31
77b9900e
JP		 32struct fl_flow_key {
33 int indev_ifindex;
42aecaa9 34 struct flow_dissector_key_control control;
bc3103f1 35 struct flow_dissector_key_control enc_control;
77b9900e
JP		 36 struct flow_dissector_key_basic basic;
37 struct flow_dissector_key_eth_addrs eth;
9399ae9a 38 struct flow_dissector_key_vlan vlan;
d64efd09 39 struct flow_dissector_key_vlan cvlan;
77b9900e 40 union {
c3f83241 41 struct flow_dissector_key_ipv4_addrs ipv4;
77b9900e
JP		 42 struct flow_dissector_key_ipv6_addrs ipv6;
43 };
44 struct flow_dissector_key_ports tp;
7b684884 45 struct flow_dissector_key_icmp icmp;
99d31326 46 struct flow_dissector_key_arp arp;
bc3103f1
AV		 47 struct flow_dissector_key_keyid enc_key_id;
48 union {
49 struct flow_dissector_key_ipv4_addrs enc_ipv4;
50 struct flow_dissector_key_ipv6_addrs enc_ipv6;
51 };
f4d997fd 52 struct flow_dissector_key_ports enc_tp;
a577d8f7 53 struct flow_dissector_key_mpls mpls;
fdfc7dd6 54 struct flow_dissector_key_tcp tcp;
4d80cc0a 55 struct flow_dissector_key_ip ip;
0e2c17b6 56 struct flow_dissector_key_ip enc_ip;
0a6e7778 57 struct flow_dissector_key_enc_opts enc_opts;
5c72299f
AN		 58 struct flow_dissector_key_ports tp_min;
59 struct flow_dissector_key_ports tp_max;
77b9900e
JP		 60} __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */
61
62struct fl_flow_mask_range {
63 unsigned short int start;
64 unsigned short int end;
65};
66
67struct fl_flow_mask {
68 struct fl_flow_key key;
69 struct fl_flow_mask_range range;
5c72299f 70 u32 flags;
05cd271f
PB		 71 struct rhash_head ht_node;
72 struct rhashtable ht;
73 struct rhashtable_params filter_ht_params;
74 struct flow_dissector dissector;
75 struct list_head filters;
44a5cd43 76 struct rcu_work rwork;
05cd271f 77 struct list_head list;
77b9900e
JP		 78};
79
b95ec7eb
JP		 80struct fl_flow_tmplt {
81 struct fl_flow_key dummy_key;
82 struct fl_flow_key mask;
83 struct flow_dissector dissector;
84 struct tcf_chain *chain;
85};
86
77b9900e
JP		 87struct cls_fl_head {
88 struct rhashtable ht;
05cd271f 89 struct list_head masks;
aaa908ff 90 struct rcu_work rwork;
c15ab236 91 struct idr handle_idr;
77b9900e
JP		 92};
93
94struct cls_fl_filter {
05cd271f 95 struct fl_flow_mask *mask;
77b9900e
JP		 96 struct rhash_head ht_node;
97 struct fl_flow_key mkey;
98 struct tcf_exts exts;
99 struct tcf_result res;
100 struct fl_flow_key key;
101 struct list_head list;
102 u32 handle;
e69985c6 103 u32 flags;
86c55361 104 u32 in_hw_count;
aaa908ff 105 struct rcu_work rwork;
7091d8c7 106 struct net_device *hw_dev;
77b9900e
JP		 107};
108
05cd271f
PB		 109static const struct rhashtable_params mask_ht_params = {
110 .key_offset = offsetof(struct fl_flow_mask, key),
111 .key_len = sizeof(struct fl_flow_key),
112 .head_offset = offsetof(struct fl_flow_mask, ht_node),
113 .automatic_shrinking = true,
114};
115
77b9900e
JP		 116static unsigned short int fl_mask_range(const struct fl_flow_mask *mask)
117{
118 return mask->range.end - mask->range.start;
119}
120
121static void fl_mask_update_range(struct fl_flow_mask *mask)
122{
123 const u8 *bytes = (const u8 *) &mask->key;
124 size_t size = sizeof(mask->key);
05cd271f 125 size_t i, first = 0, last;
77b9900e 126
05cd271f
PB		 127 for (i = 0; i < size; i++) {
128 if (bytes[i]) {
129 first = i;
130 break;
131 }
132 }
133 last = first;
134 for (i = size - 1; i != first; i--) {
77b9900e 135 if (bytes[i]) {
77b9900e 136 last = i;
05cd271f 137 break;
77b9900e
JP		 138 }
139 }
140 mask->range.start = rounddown(first, sizeof(long));
141 mask->range.end = roundup(last + 1, sizeof(long));
142}
143
144static void *fl_key_get_start(struct fl_flow_key *key,
145 const struct fl_flow_mask *mask)
146{
147 return (u8 *) key + mask->range.start;
148}
149
150static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key,
151 struct fl_flow_mask *mask)
152{
153 const long *lkey = fl_key_get_start(key, mask);
154 const long *lmask = fl_key_get_start(&mask->key, mask);
155 long *lmkey = fl_key_get_start(mkey, mask);
156 int i;
157
158 for (i = 0; i < fl_mask_range(mask); i += sizeof(long))
159 *lmkey++ = *lkey++ & *lmask++;
160}
161
b95ec7eb
JP		 162static bool fl_mask_fits_tmplt(struct fl_flow_tmplt *tmplt,
163 struct fl_flow_mask *mask)
164{
165 const long *lmask = fl_key_get_start(&mask->key, mask);
166 const long *ltmplt;
167 int i;
168
169 if (!tmplt)
170 return true;
171 ltmplt = fl_key_get_start(&tmplt->mask, mask);
172 for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) {
173 if (~*ltmplt++ & *lmask++)
174 return false;
175 }
176 return true;
177}
178
77b9900e
JP		 179static void fl_clear_masked_range(struct fl_flow_key *key,
180 struct fl_flow_mask *mask)
181{
182 memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask));
183}
184
5c72299f
AN		 185static bool fl_range_port_dst_cmp(struct cls_fl_filter *filter,
186 struct fl_flow_key *key,
187 struct fl_flow_key *mkey)
188{
189 __be16 min_mask, max_mask, min_val, max_val;
190
191 min_mask = htons(filter->mask->key.tp_min.dst);
192 max_mask = htons(filter->mask->key.tp_max.dst);
193 min_val = htons(filter->key.tp_min.dst);
194 max_val = htons(filter->key.tp_max.dst);
195
196 if (min_mask && max_mask) {
197 if (htons(key->tp.dst) < min_val ||
198 htons(key->tp.dst) > max_val)
199 return false;
200
201 /* skb does not have min and max values */
202 mkey->tp_min.dst = filter->mkey.tp_min.dst;
203 mkey->tp_max.dst = filter->mkey.tp_max.dst;
204 }
205 return true;
206}
207
208static bool fl_range_port_src_cmp(struct cls_fl_filter *filter,
209 struct fl_flow_key *key,
210 struct fl_flow_key *mkey)
211{
212 __be16 min_mask, max_mask, min_val, max_val;
213
214 min_mask = htons(filter->mask->key.tp_min.src);
215 max_mask = htons(filter->mask->key.tp_max.src);
216 min_val = htons(filter->key.tp_min.src);
217 max_val = htons(filter->key.tp_max.src);
218
219 if (min_mask && max_mask) {
220 if (htons(key->tp.src) < min_val ||
221 htons(key->tp.src) > max_val)
222 return false;
223
224 /* skb does not have min and max values */
225 mkey->tp_min.src = filter->mkey.tp_min.src;
226 mkey->tp_max.src = filter->mkey.tp_max.src;
227 }
228 return true;
229}
230
231static struct cls_fl_filter *__fl_lookup(struct fl_flow_mask *mask,
232 struct fl_flow_key *mkey)
a3308d8f 233{
05cd271f
PB		 234 return rhashtable_lookup_fast(&mask->ht, fl_key_get_start(mkey, mask),
235 mask->filter_ht_params);
a3308d8f
PB		 236}
237
5c72299f
AN		 238static struct cls_fl_filter *fl_lookup_range(struct fl_flow_mask *mask,
239 struct fl_flow_key *mkey,
240 struct fl_flow_key *key)
241{
242 struct cls_fl_filter *filter, *f;
243
244 list_for_each_entry_rcu(filter, &mask->filters, list) {
245 if (!fl_range_port_dst_cmp(filter, key, mkey))
246 continue;
247
248 if (!fl_range_port_src_cmp(filter, key, mkey))
249 continue;
250
251 f = __fl_lookup(mask, mkey);
252 if (f)
253 return f;
254 }
255 return NULL;
256}
257
258static struct cls_fl_filter *fl_lookup(struct fl_flow_mask *mask,
259 struct fl_flow_key *mkey,
260 struct fl_flow_key *key)
261{
262 if ((mask->flags & TCA_FLOWER_MASK_FLAGS_RANGE))
263 return fl_lookup_range(mask, mkey, key);
264
265 return __fl_lookup(mask, mkey);
266}
267
77b9900e
JP		 268static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
269 struct tcf_result *res)
270{
271 struct cls_fl_head *head = rcu_dereference_bh(tp->root);
272 struct cls_fl_filter *f;
05cd271f 273 struct fl_flow_mask *mask;
77b9900e
JP		 274 struct fl_flow_key skb_key;
275 struct fl_flow_key skb_mkey;
276
05cd271f
PB		 277 list_for_each_entry_rcu(mask, &head->masks, list) {
278 fl_clear_masked_range(&skb_key, mask);
bc3103f1 279
05cd271f
PB		 280 skb_key.indev_ifindex = skb->skb_iif;
281 /* skb_flow_dissect() does not set n_proto in case an unknown
282 * protocol, so do it rather here.
283 */
284 skb_key.basic.n_proto = skb->protocol;
285 skb_flow_dissect_tunnel_info(skb, &mask->dissector, &skb_key);
286 skb_flow_dissect(skb, &mask->dissector, &skb_key, 0);
77b9900e 287
05cd271f 288 fl_set_masked_key(&skb_mkey, &skb_key, mask);
77b9900e 289
5c72299f 290 f = fl_lookup(mask, &skb_mkey, &skb_key);
05cd271f
PB		 291 if (f && !tc_skip_sw(f->flags)) {
292 *res = f->res;
293 return tcf_exts_exec(skb, &f->exts, res);
294 }
77b9900e
JP		 295 }
296 return -1;
297}
298
299static int fl_init(struct tcf_proto *tp)
300{
301 struct cls_fl_head *head;
302
303 head = kzalloc(sizeof(*head), GFP_KERNEL);
304 if (!head)
305 return -ENOBUFS;
306
05cd271f 307 INIT_LIST_HEAD_RCU(&head->masks);
77b9900e 308 rcu_assign_pointer(tp->root, head);
c15ab236 309 idr_init(&head->handle_idr);
77b9900e 310
05cd271f
PB		 311 return rhashtable_init(&head->ht, &mask_ht_params);
312}
313
44a5cd43
PA		 314static void fl_mask_free(struct fl_flow_mask *mask)
315{
316 rhashtable_destroy(&mask->ht);
317 kfree(mask);
318}
319
320static void fl_mask_free_work(struct work_struct *work)
321{
322 struct fl_flow_mask *mask = container_of(to_rcu_work(work),
323 struct fl_flow_mask, rwork);
324
325 fl_mask_free(mask);
326}
327
05cd271f
PB		 328static bool fl_mask_put(struct cls_fl_head *head, struct fl_flow_mask *mask,
329 bool async)
330{
331 if (!list_empty(&mask->filters))
332 return false;
333
334 rhashtable_remove_fast(&head->ht, &mask->ht_node, mask_ht_params);
05cd271f
PB		 335 list_del_rcu(&mask->list);
336 if (async)
44a5cd43 337 tcf_queue_work(&mask->rwork, fl_mask_free_work);
05cd271f 338 else
44a5cd43 339 fl_mask_free(mask);
05cd271f
PB		 340
341 return true;
77b9900e
JP		 342}
343
0dadc117
CW		 344static void __fl_destroy_filter(struct cls_fl_filter *f)
345{
346 tcf_exts_destroy(&f->exts);
347 tcf_exts_put_net(&f->exts);
348 kfree(f);
349}
350
0552c8af 351static void fl_destroy_filter_work(struct work_struct *work)
77b9900e 352{
aaa908ff
CW		 353 struct cls_fl_filter *f = container_of(to_rcu_work(work),
354 struct cls_fl_filter, rwork);
77b9900e 355
0552c8af 356 rtnl_lock();
0dadc117 357 __fl_destroy_filter(f);
0552c8af
CW		 358 rtnl_unlock();
359}
360
1b0f8037
JK		 361static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f,
362 struct netlink_ext_ack *extack)
5b33f488 363{
de4784ca 364 struct tc_cls_flower_offload cls_flower = {};
208c0f4b 365 struct tcf_block *block = tp->chain->block;
5b33f488 366
1b0f8037 367 tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack);
de4784ca
JP		 368 cls_flower.command = TC_CLSFLOWER_DESTROY;
369 cls_flower.cookie = (unsigned long) f;
5b33f488 370
aeb3fecd 371 tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false);
caa72601 372 tcf_block_offload_dec(block, &f->flags);
5b33f488
AV		 373}
374
e8eb36cd 375static int fl_hw_replace_filter(struct tcf_proto *tp,
41002038
QM		 376 struct cls_fl_filter *f,
377 struct netlink_ext_ack *extack)
5b33f488 378{
de4784ca 379 struct tc_cls_flower_offload cls_flower = {};
208c0f4b 380 struct tcf_block *block = tp->chain->block;
717503b9 381 bool skip_sw = tc_skip_sw(f->flags);
e8eb36cd 382 int err;
5b33f488 383
e3ab786b 384 cls_flower.rule = flow_rule_alloc(tcf_exts_num_actions(&f->exts));
8f256622
PNA		 385 if (!cls_flower.rule)
386 return -ENOMEM;
387
ea205940 388 tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack);
de4784ca
JP		 389 cls_flower.command = TC_CLSFLOWER_REPLACE;
390 cls_flower.cookie = (unsigned long) f;
8f256622
PNA		 391 cls_flower.rule->match.dissector = &f->mask->dissector;
392 cls_flower.rule->match.mask = &f->mask->key;
393 cls_flower.rule->match.key = &f->mkey;
384c181e 394 cls_flower.classid = f->res.classid;
5b33f488 395
3a7b6861
PNA		 396 err = tc_setup_flow_action(&cls_flower.rule->action, &f->exts);
397 if (err) {
398 kfree(cls_flower.rule);
399 return err;
400 }
401
aeb3fecd 402 err = tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, skip_sw);
8f256622
PNA		 403 kfree(cls_flower.rule);
404
717503b9 405 if (err < 0) {
1b0f8037 406 fl_hw_destroy_filter(tp, f, NULL);
e8eb36cd 407 return err;
717503b9 408 } else if (err > 0) {
31533cba 409 f->in_hw_count = err;
caa72601 410 tcf_block_offload_inc(block, &f->flags);
717503b9
JP		 411 }
412
413 if (skip_sw && !(f->flags & TCA_CLS_FLAGS_IN_HW))
414 return -EINVAL;
415
e8eb36cd 416 return 0;
5b33f488
AV		 417}
418
10cbc684
AV		 419static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f)
420{
de4784ca 421 struct tc_cls_flower_offload cls_flower = {};
208c0f4b 422 struct tcf_block *block = tp->chain->block;
10cbc684 423
ea205940 424 tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, NULL);
de4784ca
JP		 425 cls_flower.command = TC_CLSFLOWER_STATS;
426 cls_flower.cookie = (unsigned long) f;
384c181e 427 cls_flower.classid = f->res.classid;
10cbc684 428
aeb3fecd 429 tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false);
3b1903ef
PNA		 430
431 tcf_exts_stats_update(&f->exts, cls_flower.stats.bytes,
432 cls_flower.stats.pkts,
433 cls_flower.stats.lastused);
10cbc684
AV		 434}
435
05cd271f 436static bool __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f,
1b0f8037 437 struct netlink_ext_ack *extack)
13fa876e 438{
c15ab236 439 struct cls_fl_head *head = rtnl_dereference(tp->root);
05cd271f
PB		 440 bool async = tcf_exts_get_net(&f->exts);
441 bool last;
c15ab236 442
9c160941 443 idr_remove(&head->handle_idr, f->handle);
13fa876e 444 list_del_rcu(&f->list);
05cd271f 445 last = fl_mask_put(head, f->mask, async);
79685219 446 if (!tc_skip_hw(f->flags))
1b0f8037 447 fl_hw_destroy_filter(tp, f, extack);
13fa876e 448 tcf_unbind_filter(tp, &f->res);
05cd271f 449 if (async)
aaa908ff 450 tcf_queue_work(&f->rwork, fl_destroy_filter_work);
0dadc117
CW		 451 else
452 __fl_destroy_filter(f);
05cd271f
PB		 453
454 return last;
13fa876e
RD		 455}
456
d9363774
DB		 457static void fl_destroy_sleepable(struct work_struct *work)
458{
aaa908ff
CW		 459 struct cls_fl_head *head = container_of(to_rcu_work(work),
460 struct cls_fl_head,
461 rwork);
de9dc650
PB		 462
463 rhashtable_destroy(&head->ht);
d9363774
DB		 464 kfree(head);
465 module_put(THIS_MODULE);
466}
467
12db03b6
VB		 468static void fl_destroy(struct tcf_proto *tp, bool rtnl_held,
469 struct netlink_ext_ack *extack)
77b9900e
JP		 470{
471 struct cls_fl_head *head = rtnl_dereference(tp->root);
05cd271f 472 struct fl_flow_mask *mask, *next_mask;
77b9900e
JP		 473 struct cls_fl_filter *f, *next;
474
05cd271f
PB		 475 list_for_each_entry_safe(mask, next_mask, &head->masks, list) {
476 list_for_each_entry_safe(f, next, &mask->filters, list) {
477 if (__fl_delete(tp, f, extack))
478 break;
479 }
480 }
c15ab236 481 idr_destroy(&head->handle_idr);
d9363774
DB		 482
483 __module_get(THIS_MODULE);
aaa908ff 484 tcf_queue_work(&head->rwork, fl_destroy_sleepable);
77b9900e
JP		 485}
486
8113c095 487static void *fl_get(struct tcf_proto *tp, u32 handle)
77b9900e
JP		 488{
489 struct cls_fl_head *head = rtnl_dereference(tp->root);
77b9900e 490
322d884b 491 return idr_find(&head->handle_idr, handle);
77b9900e
JP		 492}
493
494static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = {
495 [TCA_FLOWER_UNSPEC] = { .type = NLA_UNSPEC },
496 [TCA_FLOWER_CLASSID] = { .type = NLA_U32 },
497 [TCA_FLOWER_INDEV] = { .type = NLA_STRING,
498 .len = IFNAMSIZ },
499 [TCA_FLOWER_KEY_ETH_DST] = { .len = ETH_ALEN },
500 [TCA_FLOWER_KEY_ETH_DST_MASK] = { .len = ETH_ALEN },
501 [TCA_FLOWER_KEY_ETH_SRC] = { .len = ETH_ALEN },
502 [TCA_FLOWER_KEY_ETH_SRC_MASK] = { .len = ETH_ALEN },
503 [TCA_FLOWER_KEY_ETH_TYPE] = { .type = NLA_U16 },
504 [TCA_FLOWER_KEY_IP_PROTO] = { .type = NLA_U8 },
505 [TCA_FLOWER_KEY_IPV4_SRC] = { .type = NLA_U32 },
506 [TCA_FLOWER_KEY_IPV4_SRC_MASK] = { .type = NLA_U32 },
507 [TCA_FLOWER_KEY_IPV4_DST] = { .type = NLA_U32 },
508 [TCA_FLOWER_KEY_IPV4_DST_MASK] = { .type = NLA_U32 },
509 [TCA_FLOWER_KEY_IPV6_SRC] = { .len = sizeof(struct in6_addr) },
510 [TCA_FLOWER_KEY_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
511 [TCA_FLOWER_KEY_IPV6_DST] = { .len = sizeof(struct in6_addr) },
512 [TCA_FLOWER_KEY_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
513 [TCA_FLOWER_KEY_TCP_SRC] = { .type = NLA_U16 },
514 [TCA_FLOWER_KEY_TCP_DST] = { .type = NLA_U16 },
b175c3a4
JHS		 515 [TCA_FLOWER_KEY_UDP_SRC] = { .type = NLA_U16 },
516 [TCA_FLOWER_KEY_UDP_DST] = { .type = NLA_U16 },
9399ae9a
HHZ		 517 [TCA_FLOWER_KEY_VLAN_ID] = { .type = NLA_U16 },
518 [TCA_FLOWER_KEY_VLAN_PRIO] = { .type = NLA_U8 },
519 [TCA_FLOWER_KEY_VLAN_ETH_TYPE] = { .type = NLA_U16 },
bc3103f1
AV		 520 [TCA_FLOWER_KEY_ENC_KEY_ID] = { .type = NLA_U32 },
521 [TCA_FLOWER_KEY_ENC_IPV4_SRC] = { .type = NLA_U32 },
522 [TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 },
523 [TCA_FLOWER_KEY_ENC_IPV4_DST] = { .type = NLA_U32 },
524 [TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 },
525 [TCA_FLOWER_KEY_ENC_IPV6_SRC] = { .len = sizeof(struct in6_addr) },
526 [TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
527 [TCA_FLOWER_KEY_ENC_IPV6_DST] = { .len = sizeof(struct in6_addr) },
528 [TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
aa72d708
OG		 529 [TCA_FLOWER_KEY_TCP_SRC_MASK] = { .type = NLA_U16 },
530 [TCA_FLOWER_KEY_TCP_DST_MASK] = { .type = NLA_U16 },
531 [TCA_FLOWER_KEY_UDP_SRC_MASK] = { .type = NLA_U16 },
532 [TCA_FLOWER_KEY_UDP_DST_MASK] = { .type = NLA_U16 },
5976c5f4
SH		 533 [TCA_FLOWER_KEY_SCTP_SRC_MASK] = { .type = NLA_U16 },
534 [TCA_FLOWER_KEY_SCTP_DST_MASK] = { .type = NLA_U16 },
535 [TCA_FLOWER_KEY_SCTP_SRC] = { .type = NLA_U16 },
536 [TCA_FLOWER_KEY_SCTP_DST] = { .type = NLA_U16 },
f4d997fd
HHZ		 537 [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT] = { .type = NLA_U16 },
538 [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK] = { .type = NLA_U16 },
539 [TCA_FLOWER_KEY_ENC_UDP_DST_PORT] = { .type = NLA_U16 },
540 [TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK] = { .type = NLA_U16 },
faa3ffce
OG		 541 [TCA_FLOWER_KEY_FLAGS] = { .type = NLA_U32 },
542 [TCA_FLOWER_KEY_FLAGS_MASK] = { .type = NLA_U32 },
7b684884
SH		 543 [TCA_FLOWER_KEY_ICMPV4_TYPE] = { .type = NLA_U8 },
544 [TCA_FLOWER_KEY_ICMPV4_TYPE_MASK] = { .type = NLA_U8 },
545 [TCA_FLOWER_KEY_ICMPV4_CODE] = { .type = NLA_U8 },
546 [TCA_FLOWER_KEY_ICMPV4_CODE_MASK] = { .type = NLA_U8 },
547 [TCA_FLOWER_KEY_ICMPV6_TYPE] = { .type = NLA_U8 },
548 [TCA_FLOWER_KEY_ICMPV6_TYPE_MASK] = { .type = NLA_U8 },
549 [TCA_FLOWER_KEY_ICMPV6_CODE] = { .type = NLA_U8 },
550 [TCA_FLOWER_KEY_ICMPV6_CODE_MASK] = { .type = NLA_U8 },
99d31326
SH		 551 [TCA_FLOWER_KEY_ARP_SIP] = { .type = NLA_U32 },
552 [TCA_FLOWER_KEY_ARP_SIP_MASK] = { .type = NLA_U32 },
553 [TCA_FLOWER_KEY_ARP_TIP] = { .type = NLA_U32 },
554 [TCA_FLOWER_KEY_ARP_TIP_MASK] = { .type = NLA_U32 },
555 [TCA_FLOWER_KEY_ARP_OP] = { .type = NLA_U8 },
556 [TCA_FLOWER_KEY_ARP_OP_MASK] = { .type = NLA_U8 },
557 [TCA_FLOWER_KEY_ARP_SHA] = { .len = ETH_ALEN },
558 [TCA_FLOWER_KEY_ARP_SHA_MASK] = { .len = ETH_ALEN },
559 [TCA_FLOWER_KEY_ARP_THA] = { .len = ETH_ALEN },
560 [TCA_FLOWER_KEY_ARP_THA_MASK] = { .len = ETH_ALEN },
a577d8f7
BL		 561 [TCA_FLOWER_KEY_MPLS_TTL] = { .type = NLA_U8 },
562 [TCA_FLOWER_KEY_MPLS_BOS] = { .type = NLA_U8 },
563 [TCA_FLOWER_KEY_MPLS_TC] = { .type = NLA_U8 },
564 [TCA_FLOWER_KEY_MPLS_LABEL] = { .type = NLA_U32 },
fdfc7dd6
JP		 565 [TCA_FLOWER_KEY_TCP_FLAGS] = { .type = NLA_U16 },
566 [TCA_FLOWER_KEY_TCP_FLAGS_MASK] = { .type = NLA_U16 },
4d80cc0a
OG		 567 [TCA_FLOWER_KEY_IP_TOS] = { .type = NLA_U8 },
568 [TCA_FLOWER_KEY_IP_TOS_MASK] = { .type = NLA_U8 },
569 [TCA_FLOWER_KEY_IP_TTL] = { .type = NLA_U8 },
570 [TCA_FLOWER_KEY_IP_TTL_MASK] = { .type = NLA_U8 },
d64efd09
JL		 571 [TCA_FLOWER_KEY_CVLAN_ID] = { .type = NLA_U16 },
572 [TCA_FLOWER_KEY_CVLAN_PRIO] = { .type = NLA_U8 },
573 [TCA_FLOWER_KEY_CVLAN_ETH_TYPE] = { .type = NLA_U16 },
0e2c17b6
OG		 574 [TCA_FLOWER_KEY_ENC_IP_TOS] = { .type = NLA_U8 },
575 [TCA_FLOWER_KEY_ENC_IP_TOS_MASK] = { .type = NLA_U8 },
576 [TCA_FLOWER_KEY_ENC_IP_TTL] = { .type = NLA_U8 },
577 [TCA_FLOWER_KEY_ENC_IP_TTL_MASK] = { .type = NLA_U8 },
0a6e7778
PJV		 578 [TCA_FLOWER_KEY_ENC_OPTS] = { .type = NLA_NESTED },
579 [TCA_FLOWER_KEY_ENC_OPTS_MASK] = { .type = NLA_NESTED },
580};
581
582static const struct nla_policy
583enc_opts_policy[TCA_FLOWER_KEY_ENC_OPTS_MAX + 1] = {
584 [TCA_FLOWER_KEY_ENC_OPTS_GENEVE] = { .type = NLA_NESTED },
585};
586
587static const struct nla_policy
588geneve_opt_policy[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1] = {
589 [TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] = { .type = NLA_U16 },
590 [TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] = { .type = NLA_U8 },
591 [TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA] = { .type = NLA_BINARY,
592 .len = 128 },
77b9900e
JP		 593};
594
595static void fl_set_key_val(struct nlattr **tb,
596 void *val, int val_type,
597 void *mask, int mask_type, int len)
598{
599 if (!tb[val_type])
600 return;
601 memcpy(val, nla_data(tb[val_type]), len);
602 if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type])
603 memset(mask, 0xff, len);
604 else
605 memcpy(mask, nla_data(tb[mask_type]), len);
606}
607
5c72299f
AN		 608static int fl_set_key_port_range(struct nlattr **tb, struct fl_flow_key *key,
609 struct fl_flow_key *mask)
610{
611 fl_set_key_val(tb, &key->tp_min.dst,
612 TCA_FLOWER_KEY_PORT_DST_MIN, &mask->tp_min.dst,
613 TCA_FLOWER_UNSPEC, sizeof(key->tp_min.dst));
614 fl_set_key_val(tb, &key->tp_max.dst,
615 TCA_FLOWER_KEY_PORT_DST_MAX, &mask->tp_max.dst,
616 TCA_FLOWER_UNSPEC, sizeof(key->tp_max.dst));
617 fl_set_key_val(tb, &key->tp_min.src,
618 TCA_FLOWER_KEY_PORT_SRC_MIN, &mask->tp_min.src,
619 TCA_FLOWER_UNSPEC, sizeof(key->tp_min.src));
620 fl_set_key_val(tb, &key->tp_max.src,
621 TCA_FLOWER_KEY_PORT_SRC_MAX, &mask->tp_max.src,
622 TCA_FLOWER_UNSPEC, sizeof(key->tp_max.src));
623
624 if ((mask->tp_min.dst && mask->tp_max.dst &&
625 htons(key->tp_max.dst) <= htons(key->tp_min.dst)) ||
626 (mask->tp_min.src && mask->tp_max.src &&
627 htons(key->tp_max.src) <= htons(key->tp_min.src)))
628 return -EINVAL;
629
630 return 0;
631}
632
1a7fca63
BL		 633static int fl_set_key_mpls(struct nlattr **tb,
634 struct flow_dissector_key_mpls *key_val,
635 struct flow_dissector_key_mpls *key_mask)
a577d8f7
BL		 636{
637 if (tb[TCA_FLOWER_KEY_MPLS_TTL]) {
638 key_val->mpls_ttl = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TTL]);
639 key_mask->mpls_ttl = MPLS_TTL_MASK;
640 }
641 if (tb[TCA_FLOWER_KEY_MPLS_BOS]) {
1a7fca63
BL		 642 u8 bos = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_BOS]);
643
644 if (bos & ~MPLS_BOS_MASK)
645 return -EINVAL;
646 key_val->mpls_bos = bos;
a577d8f7
BL		 647 key_mask->mpls_bos = MPLS_BOS_MASK;
648 }
649 if (tb[TCA_FLOWER_KEY_MPLS_TC]) {
1a7fca63
BL		 650 u8 tc = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TC]);
651
652 if (tc & ~MPLS_TC_MASK)
653 return -EINVAL;
654 key_val->mpls_tc = tc;
a577d8f7
BL		 655 key_mask->mpls_tc = MPLS_TC_MASK;
656 }
657 if (tb[TCA_FLOWER_KEY_MPLS_LABEL]) {
1a7fca63
BL		 658 u32 label = nla_get_u32(tb[TCA_FLOWER_KEY_MPLS_LABEL]);
659
660 if (label & ~MPLS_LABEL_MASK)
661 return -EINVAL;
662 key_val->mpls_label = label;
a577d8f7
BL		 663 key_mask->mpls_label = MPLS_LABEL_MASK;
664 }
1a7fca63 665 return 0;
a577d8f7
BL		 666}
667
9399ae9a 668static void fl_set_key_vlan(struct nlattr **tb,
aaab0834 669 __be16 ethertype,
d64efd09 670 int vlan_id_key, int vlan_prio_key,
9399ae9a
HHZ		 671 struct flow_dissector_key_vlan *key_val,
672 struct flow_dissector_key_vlan *key_mask)
673{
674#define VLAN_PRIORITY_MASK 0x7
675
d64efd09 676 if (tb[vlan_id_key]) {
9399ae9a 677 key_val->vlan_id =
d64efd09 678 nla_get_u16(tb[vlan_id_key]) & VLAN_VID_MASK;
9399ae9a
HHZ		 679 key_mask->vlan_id = VLAN_VID_MASK;
680 }
d64efd09 681 if (tb[vlan_prio_key]) {
9399ae9a 682 key_val->vlan_priority =
d64efd09 683 nla_get_u8(tb[vlan_prio_key]) &
9399ae9a
HHZ		 684 VLAN_PRIORITY_MASK;
685 key_mask->vlan_priority = VLAN_PRIORITY_MASK;
686 }
aaab0834
JL		 687 key_val->vlan_tpid = ethertype;
688 key_mask->vlan_tpid = cpu_to_be16(~0);
9399ae9a
HHZ		 689}
690
faa3ffce
OG		 691static void fl_set_key_flag(u32 flower_key, u32 flower_mask,
692 u32 *dissector_key, u32 *dissector_mask,
693 u32 flower_flag_bit, u32 dissector_flag_bit)
694{
695 if (flower_mask & flower_flag_bit) {
696 *dissector_mask |= dissector_flag_bit;
697 if (flower_key & flower_flag_bit)
698 *dissector_key |= dissector_flag_bit;
699 }
700}
701
d9724772
OG		 702static int fl_set_key_flags(struct nlattr **tb,
703 u32 *flags_key, u32 *flags_mask)
faa3ffce
OG		 704{
705 u32 key, mask;
706
d9724772
OG		 707 /* mask is mandatory for flags */
708 if (!tb[TCA_FLOWER_KEY_FLAGS_MASK])
709 return -EINVAL;
faa3ffce
OG		 710
711 key = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS]));
d9724772 712 mask = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS_MASK]));
faa3ffce
OG		 713
714 *flags_key = 0;
715 *flags_mask = 0;
716
717 fl_set_key_flag(key, mask, flags_key, flags_mask,
718 TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
459d153d
PJV		 719 fl_set_key_flag(key, mask, flags_key, flags_mask,
720 TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST,
721 FLOW_DIS_FIRST_FRAG);
d9724772
OG		 722
723 return 0;
faa3ffce
OG		 724}
725
0e2c17b6 726static void fl_set_key_ip(struct nlattr **tb, bool encap,
4d80cc0a
OG		 727 struct flow_dissector_key_ip *key,
728 struct flow_dissector_key_ip *mask)
729{
0e2c17b6
OG		 730 int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS;
731 int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL;
732 int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK;
733 int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK;
4d80cc0a 734
0e2c17b6
OG		 735 fl_set_key_val(tb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos));
736 fl_set_key_val(tb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl));
4d80cc0a
OG		 737}
738
0a6e7778
PJV		 739static int fl_set_geneve_opt(const struct nlattr *nla, struct fl_flow_key *key,
740 int depth, int option_len,
741 struct netlink_ext_ack *extack)
742{
743 struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1];
744 struct nlattr *class = NULL, *type = NULL, *data = NULL;
745 struct geneve_opt *opt;
746 int err, data_len = 0;
747
748 if (option_len > sizeof(struct geneve_opt))
749 data_len = option_len - sizeof(struct geneve_opt);
750
751 opt = (struct geneve_opt *)&key->enc_opts.data[key->enc_opts.len];
752 memset(opt, 0xff, option_len);
753 opt->length = data_len / 4;
754 opt->r1 = 0;
755 opt->r2 = 0;
756 opt->r3 = 0;
757
758 /* If no mask has been prodived we assume an exact match. */
759 if (!depth)
760 return sizeof(struct geneve_opt) + data_len;
761
762 if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_GENEVE) {
763 NL_SET_ERR_MSG(extack, "Non-geneve option type for mask");
764 return -EINVAL;
765 }
766
767 err = nla_parse_nested(tb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX,
768 nla, geneve_opt_policy, extack);
769 if (err < 0)
770 return err;
771
772 /* We are not allowed to omit any of CLASS, TYPE or DATA
773 * fields from the key.
774 */
775 if (!option_len &&
776 (!tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] ||
777 !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] ||
778 !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA])) {
779 NL_SET_ERR_MSG(extack, "Missing tunnel key geneve option class, type or data");
780 return -EINVAL;
781 }
782
783 /* Omitting any of CLASS, TYPE or DATA fields is allowed
784 * for the mask.
785 */
786 if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]) {
787 int new_len = key->enc_opts.len;
788
789 data = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA];
790 data_len = nla_len(data);
791 if (data_len < 4) {
792 NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is less than 4 bytes long");
793 return -ERANGE;
794 }
795 if (data_len % 4) {
796 NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is not a multiple of 4 bytes long");
797 return -ERANGE;
798 }
799
800 new_len += sizeof(struct geneve_opt) + data_len;
801 BUILD_BUG_ON(FLOW_DIS_TUN_OPTS_MAX != IP_TUNNEL_OPTS_MAX);
802 if (new_len > FLOW_DIS_TUN_OPTS_MAX) {
803 NL_SET_ERR_MSG(extack, "Tunnel options exceeds max size");
804 return -ERANGE;
805 }
806 opt->length = data_len / 4;
807 memcpy(opt->opt_data, nla_data(data), data_len);
808 }
809
810 if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]) {
811 class = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS];
812 opt->opt_class = nla_get_be16(class);
813 }
814
815 if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]) {
816 type = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE];
817 opt->type = nla_get_u8(type);
818 }
819
820 return sizeof(struct geneve_opt) + data_len;
821}
822
823static int fl_set_enc_opt(struct nlattr **tb, struct fl_flow_key *key,
824 struct fl_flow_key *mask,
825 struct netlink_ext_ack *extack)
826{
827 const struct nlattr *nla_enc_key, *nla_opt_key, *nla_opt_msk = NULL;
63c82997
JK		 828 int err, option_len, key_depth, msk_depth = 0;
829
830 err = nla_validate_nested(tb[TCA_FLOWER_KEY_ENC_OPTS],
831 TCA_FLOWER_KEY_ENC_OPTS_MAX,
832 enc_opts_policy, extack);
833 if (err)
834 return err;
0a6e7778
PJV		 835
836 nla_enc_key = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS]);
837
838 if (tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]) {
63c82997
JK		 839 err = nla_validate_nested(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK],
840 TCA_FLOWER_KEY_ENC_OPTS_MAX,
841 enc_opts_policy, extack);
842 if (err)
843 return err;
844
0a6e7778
PJV		 845 nla_opt_msk = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]);
846 msk_depth = nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]);
847 }
848
849 nla_for_each_attr(nla_opt_key, nla_enc_key,
850 nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS]), key_depth) {
851 switch (nla_type(nla_opt_key)) {
852 case TCA_FLOWER_KEY_ENC_OPTS_GENEVE:
853 option_len = 0;
854 key->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT;
855 option_len = fl_set_geneve_opt(nla_opt_key, key,
856 key_depth, option_len,
857 extack);
858 if (option_len < 0)
859 return option_len;
860
861 key->enc_opts.len += option_len;
862 /* At the same time we need to parse through the mask
863 * in order to verify exact and mask attribute lengths.
864 */
865 mask->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT;
866 option_len = fl_set_geneve_opt(nla_opt_msk, mask,
867 msk_depth, option_len,
868 extack);
869 if (option_len < 0)
870 return option_len;
871
872 mask->enc_opts.len += option_len;
873 if (key->enc_opts.len != mask->enc_opts.len) {
874 NL_SET_ERR_MSG(extack, "Key and mask miss aligned");
875 return -EINVAL;
876 }
877
878 if (msk_depth)
879 nla_opt_msk = nla_next(nla_opt_msk, &msk_depth);
880 break;
881 default:
882 NL_SET_ERR_MSG(extack, "Unknown tunnel option type");
883 return -EINVAL;
884 }
885 }
886
887 return 0;
888}
889
77b9900e 890static int fl_set_key(struct net *net, struct nlattr **tb,
1057c55f
AA		 891 struct fl_flow_key *key, struct fl_flow_key *mask,
892 struct netlink_ext_ack *extack)
77b9900e 893{
9399ae9a 894 __be16 ethertype;
d9724772 895 int ret = 0;
dd3aa3b5 896#ifdef CONFIG_NET_CLS_IND
77b9900e 897 if (tb[TCA_FLOWER_INDEV]) {
1057c55f 898 int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV], extack);
77b9900e
JP		 899 if (err < 0)
900 return err;
901 key->indev_ifindex = err;
902 mask->indev_ifindex = 0xffffffff;
903 }
dd3aa3b5 904#endif
77b9900e
JP		 905
906 fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
907 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
908 sizeof(key->eth.dst));
909 fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
910 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
911 sizeof(key->eth.src));
66530bdf 912
0b498a52 913 if (tb[TCA_FLOWER_KEY_ETH_TYPE]) {
9399ae9a
HHZ		 914 ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]);
915
aaab0834 916 if (eth_type_vlan(ethertype)) {
d64efd09
JL		 917 fl_set_key_vlan(tb, ethertype, TCA_FLOWER_KEY_VLAN_ID,
918 TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan,
919 &mask->vlan);
920
5e9a0fe4
JL		 921 if (tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]) {
922 ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]);
923 if (eth_type_vlan(ethertype)) {
924 fl_set_key_vlan(tb, ethertype,
925 TCA_FLOWER_KEY_CVLAN_ID,
926 TCA_FLOWER_KEY_CVLAN_PRIO,
927 &key->cvlan, &mask->cvlan);
928 fl_set_key_val(tb, &key->basic.n_proto,
929 TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
930 &mask->basic.n_proto,
931 TCA_FLOWER_UNSPEC,
932 sizeof(key->basic.n_proto));
933 } else {
934 key->basic.n_proto = ethertype;
935 mask->basic.n_proto = cpu_to_be16(~0);
936 }
d64efd09 937 }
0b498a52
AB		 938 } else {
939 key->basic.n_proto = ethertype;
940 mask->basic.n_proto = cpu_to_be16(~0);
941 }
9399ae9a 942 }
66530bdf 943
77b9900e
JP		 944 if (key->basic.n_proto == htons(ETH_P_IP) ||
945 key->basic.n_proto == htons(ETH_P_IPV6)) {
946 fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
947 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
948 sizeof(key->basic.ip_proto));
0e2c17b6 949 fl_set_key_ip(tb, false, &key->ip, &mask->ip);
77b9900e 950 }
66530bdf
JHS		 951
952 if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) {
953 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
970bfcd0 954 mask->control.addr_type = ~0;
77b9900e
JP		 955 fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
956 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
957 sizeof(key->ipv4.src));
958 fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
959 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
960 sizeof(key->ipv4.dst));
66530bdf
JHS		 961 } else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) {
962 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
970bfcd0 963 mask->control.addr_type = ~0;
77b9900e
JP		 964 fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
965 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
966 sizeof(key->ipv6.src));
967 fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
968 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
969 sizeof(key->ipv6.dst));
970 }
66530bdf 971
77b9900e
JP		 972 if (key->basic.ip_proto == IPPROTO_TCP) {
973 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
aa72d708 974 &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
77b9900e
JP		 975 sizeof(key->tp.src));
976 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
aa72d708 977 &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
77b9900e 978 sizeof(key->tp.dst));
fdfc7dd6
JP		 979 fl_set_key_val(tb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS,
980 &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK,
981 sizeof(key->tcp.flags));
77b9900e
JP		 982 } else if (key->basic.ip_proto == IPPROTO_UDP) {
983 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
aa72d708 984 &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
77b9900e
JP		 985 sizeof(key->tp.src));
986 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
aa72d708 987 &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
77b9900e 988 sizeof(key->tp.dst));
5976c5f4
SH		 989 } else if (key->basic.ip_proto == IPPROTO_SCTP) {
990 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
991 &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
992 sizeof(key->tp.src));
993 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
994 &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
995 sizeof(key->tp.dst));
7b684884
SH		 996 } else if (key->basic.n_proto == htons(ETH_P_IP) &&
997 key->basic.ip_proto == IPPROTO_ICMP) {
998 fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV4_TYPE,
999 &mask->icmp.type,
1000 TCA_FLOWER_KEY_ICMPV4_TYPE_MASK,
1001 sizeof(key->icmp.type));
1002 fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV4_CODE,
1003 &mask->icmp.code,
1004 TCA_FLOWER_KEY_ICMPV4_CODE_MASK,
1005 sizeof(key->icmp.code));
1006 } else if (key->basic.n_proto == htons(ETH_P_IPV6) &&
1007 key->basic.ip_proto == IPPROTO_ICMPV6) {
1008 fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV6_TYPE,
1009 &mask->icmp.type,
1010 TCA_FLOWER_KEY_ICMPV6_TYPE_MASK,
1011 sizeof(key->icmp.type));
040587af 1012 fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV6_CODE,
7b684884 1013 &mask->icmp.code,
040587af 1014 TCA_FLOWER_KEY_ICMPV6_CODE_MASK,
7b684884 1015 sizeof(key->icmp.code));
a577d8f7
BL		 1016 } else if (key->basic.n_proto == htons(ETH_P_MPLS_UC) ||
1017 key->basic.n_proto == htons(ETH_P_MPLS_MC)) {
1a7fca63
BL		 1018 ret = fl_set_key_mpls(tb, &key->mpls, &mask->mpls);
1019 if (ret)
1020 return ret;
99d31326
SH		 1021 } else if (key->basic.n_proto == htons(ETH_P_ARP) ||
1022 key->basic.n_proto == htons(ETH_P_RARP)) {
1023 fl_set_key_val(tb, &key->arp.sip, TCA_FLOWER_KEY_ARP_SIP,
1024 &mask->arp.sip, TCA_FLOWER_KEY_ARP_SIP_MASK,
1025 sizeof(key->arp.sip));
1026 fl_set_key_val(tb, &key->arp.tip, TCA_FLOWER_KEY_ARP_TIP,
1027 &mask->arp.tip, TCA_FLOWER_KEY_ARP_TIP_MASK,
1028 sizeof(key->arp.tip));
1029 fl_set_key_val(tb, &key->arp.op, TCA_FLOWER_KEY_ARP_OP,
1030 &mask->arp.op, TCA_FLOWER_KEY_ARP_OP_MASK,
1031 sizeof(key->arp.op));
1032 fl_set_key_val(tb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA,
1033 mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK,
1034 sizeof(key->arp.sha));
1035 fl_set_key_val(tb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA,
1036 mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK,
1037 sizeof(key->arp.tha));
77b9900e
JP		 1038 }
1039
5c72299f
AN		 1040 if (key->basic.ip_proto == IPPROTO_TCP ||
1041 key->basic.ip_proto == IPPROTO_UDP ||
1042 key->basic.ip_proto == IPPROTO_SCTP) {
1043 ret = fl_set_key_port_range(tb, key, mask);
1044 if (ret)
1045 return ret;
1046 }
1047
bc3103f1
AV		 1048 if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] ||
1049 tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) {
1050 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
970bfcd0 1051 mask->enc_control.addr_type = ~0;
bc3103f1
AV		 1052 fl_set_key_val(tb, &key->enc_ipv4.src,
1053 TCA_FLOWER_KEY_ENC_IPV4_SRC,
1054 &mask->enc_ipv4.src,
1055 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
1056 sizeof(key->enc_ipv4.src));
1057 fl_set_key_val(tb, &key->enc_ipv4.dst,
1058 TCA_FLOWER_KEY_ENC_IPV4_DST,
1059 &mask->enc_ipv4.dst,
1060 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
1061 sizeof(key->enc_ipv4.dst));
1062 }
1063
1064 if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] ||
1065 tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) {
1066 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
970bfcd0 1067 mask->enc_control.addr_type = ~0;
bc3103f1
AV		 1068 fl_set_key_val(tb, &key->enc_ipv6.src,
1069 TCA_FLOWER_KEY_ENC_IPV6_SRC,
1070 &mask->enc_ipv6.src,
1071 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
1072 sizeof(key->enc_ipv6.src));
1073 fl_set_key_val(tb, &key->enc_ipv6.dst,
1074 TCA_FLOWER_KEY_ENC_IPV6_DST,
1075 &mask->enc_ipv6.dst,
1076 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
1077 sizeof(key->enc_ipv6.dst));
1078 }
1079
1080 fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID,
eb523f42 1081 &mask->enc_key_id.keyid, TCA_FLOWER_UNSPEC,
bc3103f1
AV		 1082 sizeof(key->enc_key_id.keyid));
1083
f4d997fd
HHZ		 1084 fl_set_key_val(tb, &key->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
1085 &mask->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
1086 sizeof(key->enc_tp.src));
1087
1088 fl_set_key_val(tb, &key->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
1089 &mask->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
1090 sizeof(key->enc_tp.dst));
1091
0e2c17b6
OG		 1092 fl_set_key_ip(tb, true, &key->enc_ip, &mask->enc_ip);
1093
0a6e7778
PJV		 1094 if (tb[TCA_FLOWER_KEY_ENC_OPTS]) {
1095 ret = fl_set_enc_opt(tb, key, mask, extack);
1096 if (ret)
1097 return ret;
1098 }
1099
d9724772
OG		 1100 if (tb[TCA_FLOWER_KEY_FLAGS])
1101 ret = fl_set_key_flags(tb, &key->control.flags, &mask->control.flags);
faa3ffce 1102
d9724772 1103 return ret;
77b9900e
JP		 1104}
1105
05cd271f
PB		 1106static void fl_mask_copy(struct fl_flow_mask *dst,
1107 struct fl_flow_mask *src)
77b9900e 1108{
05cd271f
PB		 1109 const void *psrc = fl_key_get_start(&src->key, src);
1110 void *pdst = fl_key_get_start(&dst->key, src);
77b9900e 1111
05cd271f
PB		 1112 memcpy(pdst, psrc, fl_mask_range(src));
1113 dst->range = src->range;
77b9900e
JP		 1114}
1115
1116static const struct rhashtable_params fl_ht_params = {
1117 .key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */
1118 .head_offset = offsetof(struct cls_fl_filter, ht_node),
1119 .automatic_shrinking = true,
1120};
1121
05cd271f 1122static int fl_init_mask_hashtable(struct fl_flow_mask *mask)
77b9900e 1123{
05cd271f
PB		 1124 mask->filter_ht_params = fl_ht_params;
1125 mask->filter_ht_params.key_len = fl_mask_range(mask);
1126 mask->filter_ht_params.key_offset += mask->range.start;
77b9900e 1127
05cd271f 1128 return rhashtable_init(&mask->ht, &mask->filter_ht_params);
77b9900e
JP		 1129}
1130
1131#define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member)
cb205a81 1132#define FL_KEY_MEMBER_SIZE(member) FIELD_SIZEOF(struct fl_flow_key, member)
77b9900e 1133
339ba878
HHZ		 1134#define FL_KEY_IS_MASKED(mask, member) \
1135 memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member), \
1136 0, FL_KEY_MEMBER_SIZE(member)) \
77b9900e
JP		 1137
1138#define FL_KEY_SET(keys, cnt, id, member) \
1139 do { \
1140 keys[cnt].key_id = id; \
1141 keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member); \
1142 cnt++; \
1143 } while(0);
1144
339ba878 1145#define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member) \
77b9900e 1146 do { \
339ba878 1147 if (FL_KEY_IS_MASKED(mask, member)) \
77b9900e
JP		 1148 FL_KEY_SET(keys, cnt, id, member); \
1149 } while(0);
1150
33fb5cba
JP		 1151static void fl_init_dissector(struct flow_dissector *dissector,
1152 struct fl_flow_key *mask)
77b9900e
JP		 1153{
1154 struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX];
1155 size_t cnt = 0;
1156
42aecaa9 1157 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control);
77b9900e 1158 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic);
33fb5cba 1159 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
339ba878 1160 FLOW_DISSECTOR_KEY_ETH_ADDRS, eth);
33fb5cba 1161 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
339ba878 1162 FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
33fb5cba 1163 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
339ba878 1164 FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
5c72299f
AN		 1165 if (FL_KEY_IS_MASKED(mask, tp) ||
1166 FL_KEY_IS_MASKED(mask, tp_min) || FL_KEY_IS_MASKED(mask, tp_max))
1167 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_PORTS, tp);
33fb5cba 1168 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
4d80cc0a 1169 FLOW_DISSECTOR_KEY_IP, ip);
33fb5cba 1170 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
fdfc7dd6 1171 FLOW_DISSECTOR_KEY_TCP, tcp);
33fb5cba 1172 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
7b684884 1173 FLOW_DISSECTOR_KEY_ICMP, icmp);
33fb5cba 1174 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
99d31326 1175 FLOW_DISSECTOR_KEY_ARP, arp);
33fb5cba 1176 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
a577d8f7 1177 FLOW_DISSECTOR_KEY_MPLS, mpls);
33fb5cba 1178 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
9399ae9a 1179 FLOW_DISSECTOR_KEY_VLAN, vlan);
33fb5cba 1180 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
d64efd09 1181 FLOW_DISSECTOR_KEY_CVLAN, cvlan);
33fb5cba 1182 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
519d1052 1183 FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
33fb5cba 1184 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
519d1052 1185 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4);
33fb5cba 1186 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
519d1052 1187 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6);
33fb5cba
JP		 1188 if (FL_KEY_IS_MASKED(mask, enc_ipv4) ||
1189 FL_KEY_IS_MASKED(mask, enc_ipv6))
519d1052
HHZ		 1190 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL,
1191 enc_control);
33fb5cba 1192 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
f4d997fd 1193 FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp);
33fb5cba 1194 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
0e2c17b6 1195 FLOW_DISSECTOR_KEY_ENC_IP, enc_ip);
0a6e7778
PJV		 1196 FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1197 FLOW_DISSECTOR_KEY_ENC_OPTS, enc_opts);
77b9900e 1198
33fb5cba 1199 skb_flow_dissector_init(dissector, keys, cnt);
05cd271f
PB		 1200}
1201
1202static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head,
1203 struct fl_flow_mask *mask)
1204{
1205 struct fl_flow_mask *newmask;
1206 int err;
1207
1208 newmask = kzalloc(sizeof(*newmask), GFP_KERNEL);
1209 if (!newmask)
1210 return ERR_PTR(-ENOMEM);
1211
1212 fl_mask_copy(newmask, mask);
1213
5c72299f
AN		 1214 if ((newmask->key.tp_min.dst && newmask->key.tp_max.dst) ||
1215 (newmask->key.tp_min.src && newmask->key.tp_max.src))
1216 newmask->flags |= TCA_FLOWER_MASK_FLAGS_RANGE;
1217
05cd271f
PB		 1218 err = fl_init_mask_hashtable(newmask);
1219 if (err)
1220 goto errout_free;
1221
33fb5cba 1222 fl_init_dissector(&newmask->dissector, &newmask->key);
05cd271f
PB		 1223
1224 INIT_LIST_HEAD_RCU(&newmask->filters);
1225
1226 err = rhashtable_insert_fast(&head->ht, &newmask->ht_node,
1227 mask_ht_params);
1228 if (err)
1229 goto errout_destroy;
1230
1231 list_add_tail_rcu(&newmask->list, &head->masks);
1232
1233 return newmask;
1234
1235errout_destroy:
1236 rhashtable_destroy(&newmask->ht);
1237errout_free:
1238 kfree(newmask);
1239
1240 return ERR_PTR(err);
77b9900e
JP		 1241}
1242
1243static int fl_check_assign_mask(struct cls_fl_head *head,
05cd271f
PB		 1244 struct cls_fl_filter *fnew,
1245 struct cls_fl_filter *fold,
77b9900e
JP		 1246 struct fl_flow_mask *mask)
1247{
05cd271f 1248 struct fl_flow_mask *newmask;
77b9900e 1249
05cd271f
PB		 1250 fnew->mask = rhashtable_lookup_fast(&head->ht, mask, mask_ht_params);
1251 if (!fnew->mask) {
1252 if (fold)
77b9900e 1253 return -EINVAL;
77b9900e 1254
05cd271f
PB		 1255 newmask = fl_create_new_mask(head, mask);
1256 if (IS_ERR(newmask))
1257 return PTR_ERR(newmask);
77b9900e 1258
05cd271f 1259 fnew->mask = newmask;
f6521c58 1260 } else if (fold && fold->mask != fnew->mask) {
05cd271f
PB		 1261 return -EINVAL;
1262 }
77b9900e
JP		 1263
1264 return 0;
1265}
1266
1267static int fl_set_parms(struct net *net, struct tcf_proto *tp,
1268 struct cls_fl_filter *f, struct fl_flow_mask *mask,
1269 unsigned long base, struct nlattr **tb,
50a56190 1270 struct nlattr *est, bool ovr,
b95ec7eb 1271 struct fl_flow_tmplt *tmplt,
50a56190 1272 struct netlink_ext_ack *extack)
77b9900e 1273{
77b9900e
JP		 1274 int err;
1275
ec6743a1
VB		 1276 err = tcf_exts_validate(net, tp, tb, est, &f->exts, ovr, true,
1277 extack);
77b9900e
JP		 1278 if (err < 0)
1279 return err;
1280
1281 if (tb[TCA_FLOWER_CLASSID]) {
1282 f->res.classid = nla_get_u32(tb[TCA_FLOWER_CLASSID]);
1283 tcf_bind_filter(tp, &f->res, base);
1284 }
1285
1057c55f 1286 err = fl_set_key(net, tb, &f->key, &mask->key, extack);
77b9900e 1287 if (err)
45507529 1288 return err;
77b9900e
JP		 1289
1290 fl_mask_update_range(mask);
1291 fl_set_masked_key(&f->mkey, &f->key, mask);
1292
b95ec7eb
JP		 1293 if (!fl_mask_fits_tmplt(tmplt, mask)) {
1294 NL_SET_ERR_MSG_MOD(extack, "Mask does not fit the template");
1295 return -EINVAL;
1296 }
1297
77b9900e 1298 return 0;
77b9900e
JP		 1299}
1300
77b9900e
JP		 1301static int fl_change(struct net *net, struct sk_buff *in_skb,
1302 struct tcf_proto *tp, unsigned long base,
1303 u32 handle, struct nlattr **tca,
12db03b6
VB		 1304 void **arg, bool ovr, bool rtnl_held,
1305 struct netlink_ext_ack *extack)
77b9900e
JP		 1306{
1307 struct cls_fl_head *head = rtnl_dereference(tp->root);
8113c095 1308 struct cls_fl_filter *fold = *arg;
77b9900e 1309 struct cls_fl_filter *fnew;
2cddd201 1310 struct fl_flow_mask *mask;
39b7b6a6 1311 struct nlattr **tb;
77b9900e
JP		 1312 int err;
1313
1314 if (!tca[TCA_OPTIONS])
1315 return -EINVAL;
1316
2cddd201
IV		 1317 mask = kzalloc(sizeof(struct fl_flow_mask), GFP_KERNEL);
1318 if (!mask)
39b7b6a6
AB		 1319 return -ENOBUFS;
1320
2cddd201
IV		 1321 tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL);
1322 if (!tb) {
1323 err = -ENOBUFS;
1324 goto errout_mask_alloc;
1325 }
1326
fceb6435
JB		 1327 err = nla_parse_nested(tb, TCA_FLOWER_MAX, tca[TCA_OPTIONS],
1328 fl_policy, NULL);
77b9900e 1329 if (err < 0)
39b7b6a6 1330 goto errout_tb;
77b9900e 1331
39b7b6a6
AB		 1332 if (fold && handle && fold->handle != handle) {
1333 err = -EINVAL;
1334 goto errout_tb;
1335 }
77b9900e
JP		 1336
1337 fnew = kzalloc(sizeof(*fnew), GFP_KERNEL);
39b7b6a6
AB		 1338 if (!fnew) {
1339 err = -ENOBUFS;
1340 goto errout_tb;
1341 }
77b9900e 1342
b9a24bb7
WC		 1343 err = tcf_exts_init(&fnew->exts, TCA_FLOWER_ACT, 0);
1344 if (err < 0)
1345 goto errout;
77b9900e
JP		 1346
1347 if (!handle) {
85bd0438
MW		 1348 handle = 1;
1349 err = idr_alloc_u32(&head->handle_idr, fnew, &handle,
1350 INT_MAX, GFP_KERNEL);
1351 } else if (!fold) {
1352 /* user specifies a handle and it doesn't exist */
1353 err = idr_alloc_u32(&head->handle_idr, fnew, &handle,
1354 handle, GFP_KERNEL);
77b9900e 1355 }
85bd0438
MW		 1356 if (err)
1357 goto errout;
1358 fnew->handle = handle;
77b9900e 1359
e69985c6
AV		 1360 if (tb[TCA_FLOWER_FLAGS]) {
1361 fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]);
1362
1363 if (!tc_flags_valid(fnew->flags)) {
1364 err = -EINVAL;
fe2502e4 1365 goto errout_idr;
e69985c6
AV		 1366 }
1367 }
5b33f488 1368
2cddd201 1369 err = fl_set_parms(net, tp, fnew, mask, base, tb, tca[TCA_RATE], ovr,
b95ec7eb 1370 tp->chain->tmplt_priv, extack);
77b9900e 1371 if (err)
fe2502e4 1372 goto errout_idr;
77b9900e 1373
2cddd201 1374 err = fl_check_assign_mask(head, fnew, fold, mask);
77b9900e 1375 if (err)
fe2502e4 1376 goto errout_idr;
77b9900e 1377
4cc1feeb 1378 if (!fold && __fl_lookup(fnew->mask, &fnew->mkey)) {
35cc3cef
OG		 1379 err = -EEXIST;
1380 goto errout_mask;
e69985c6 1381 }
5b33f488 1382
35cc3cef
OG		 1383 err = rhashtable_insert_fast(&fnew->mask->ht, &fnew->ht_node,
1384 fnew->mask->filter_ht_params);
1385 if (err)
1386 goto errout_mask;
1387
79685219 1388 if (!tc_skip_hw(fnew->flags)) {
05cd271f 1389 err = fl_hw_replace_filter(tp, fnew, extack);
79685219 1390 if (err)
c1f7e029 1391 goto errout_mask_ht;
79685219 1392 }
5b33f488 1393
55593960
OG		 1394 if (!tc_in_hw(fnew->flags))
1395 fnew->flags |= TCA_CLS_FLAGS_NOT_IN_HW;
1396
5b33f488 1397 if (fold) {
599d2570
RD		 1398 rhashtable_remove_fast(&fold->mask->ht,
1399 &fold->ht_node,
1400 fold->mask->filter_ht_params);
79685219 1401 if (!tc_skip_hw(fold->flags))
1b0f8037 1402 fl_hw_destroy_filter(tp, fold, NULL);
5b33f488 1403 }
77b9900e 1404
8113c095 1405 *arg = fnew;
77b9900e
JP		 1406
1407 if (fold) {
234a4624 1408 idr_replace(&head->handle_idr, fnew, fnew->handle);
ff3532f2 1409 list_replace_rcu(&fold->list, &fnew->list);
77b9900e 1410 tcf_unbind_filter(tp, &fold->res);
0dadc117 1411 tcf_exts_get_net(&fold->exts);
aaa908ff 1412 tcf_queue_work(&fold->rwork, fl_destroy_filter_work);
77b9900e 1413 } else {
05cd271f 1414 list_add_tail_rcu(&fnew->list, &fnew->mask->filters);
77b9900e
JP		 1415 }
1416
39b7b6a6 1417 kfree(tb);
2cddd201 1418 kfree(mask);
77b9900e
JP		 1419 return 0;
1420
c1f7e029
PM		 1421errout_mask_ht:
1422 rhashtable_remove_fast(&fnew->mask->ht, &fnew->ht_node,
1423 fnew->mask->filter_ht_params);
1424
05cd271f
PB		 1425errout_mask:
1426 fl_mask_put(head, fnew->mask, false);
1427
fe2502e4 1428errout_idr:
8258d2da 1429 if (!fold)
9c160941 1430 idr_remove(&head->handle_idr, fnew->handle);
77b9900e 1431errout:
b9a24bb7 1432 tcf_exts_destroy(&fnew->exts);
77b9900e 1433 kfree(fnew);
39b7b6a6
AB		 1434errout_tb:
1435 kfree(tb);
2cddd201
IV		 1436errout_mask_alloc:
1437 kfree(mask);
77b9900e
JP		 1438 return err;
1439}
1440
571acf21 1441static int fl_delete(struct tcf_proto *tp, void *arg, bool *last,
12db03b6 1442 bool rtnl_held, struct netlink_ext_ack *extack)
77b9900e
JP		 1443{
1444 struct cls_fl_head *head = rtnl_dereference(tp->root);
8113c095 1445 struct cls_fl_filter *f = arg;
77b9900e 1446
35cc3cef
OG		 1447 rhashtable_remove_fast(&f->mask->ht, &f->ht_node,
1448 f->mask->filter_ht_params);
1b0f8037 1449 __fl_delete(tp, f, extack);
05cd271f 1450 *last = list_empty(&head->masks);
77b9900e
JP		 1451 return 0;
1452}
1453
12db03b6
VB		 1454static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg,
1455 bool rtnl_held)
77b9900e
JP		 1456{
1457 struct cls_fl_head *head = rtnl_dereference(tp->root);
1458 struct cls_fl_filter *f;
05cd271f 1459
01683a14
VB		 1460 arg->count = arg->skip;
1461
1462 while ((f = idr_get_next_ul(&head->handle_idr,
1463 &arg->cookie)) != NULL) {
1464 if (arg->fn(tp, f, arg) < 0) {
1465 arg->stop = 1;
1466 break;
05cd271f 1467 }
01683a14
VB		 1468 arg->cookie = f->handle + 1;
1469 arg->count++;
77b9900e
JP		 1470 }
1471}
1472
31533cba
JH		 1473static int fl_reoffload(struct tcf_proto *tp, bool add, tc_setup_cb_t *cb,
1474 void *cb_priv, struct netlink_ext_ack *extack)
1475{
1476 struct cls_fl_head *head = rtnl_dereference(tp->root);
1477 struct tc_cls_flower_offload cls_flower = {};
1478 struct tcf_block *block = tp->chain->block;
1479 struct fl_flow_mask *mask;
1480 struct cls_fl_filter *f;
1481 int err;
1482
1483 list_for_each_entry(mask, &head->masks, list) {
1484 list_for_each_entry(f, &mask->filters, list) {
1485 if (tc_skip_hw(f->flags))
1486 continue;
1487
e3ab786b
PNA		 1488 cls_flower.rule =
1489 flow_rule_alloc(tcf_exts_num_actions(&f->exts));
8f256622
PNA		 1490 if (!cls_flower.rule)
1491 return -ENOMEM;
1492
31533cba
JH		 1493 tc_cls_common_offload_init(&cls_flower.common, tp,
1494 f->flags, extack);
1495 cls_flower.command = add ?
1496 TC_CLSFLOWER_REPLACE : TC_CLSFLOWER_DESTROY;
1497 cls_flower.cookie = (unsigned long)f;
8f256622
PNA		 1498 cls_flower.rule->match.dissector = &mask->dissector;
1499 cls_flower.rule->match.mask = &mask->key;
1500 cls_flower.rule->match.key = &f->mkey;
3a7b6861
PNA		 1501
1502 err = tc_setup_flow_action(&cls_flower.rule->action,
1503 &f->exts);
1504 if (err) {
1505 kfree(cls_flower.rule);
1506 return err;
1507 }
1508
31533cba
JH		 1509 cls_flower.classid = f->res.classid;
1510
1511 err = cb(TC_SETUP_CLSFLOWER, &cls_flower, cb_priv);
8f256622
PNA		 1512 kfree(cls_flower.rule);
1513
31533cba
JH		 1514 if (err) {
1515 if (add && tc_skip_sw(f->flags))
1516 return err;
1517 continue;
1518 }
1519
1520 tc_cls_offload_cnt_update(block, &f->in_hw_count,
1521 &f->flags, add);
1522 }
1523 }
1524
1525 return 0;
1526}
1527
8f256622
PNA		 1528static int fl_hw_create_tmplt(struct tcf_chain *chain,
1529 struct fl_flow_tmplt *tmplt)
34738452
JP		 1530{
1531 struct tc_cls_flower_offload cls_flower = {};
1532 struct tcf_block *block = chain->block;
34738452 1533
e3ab786b 1534 cls_flower.rule = flow_rule_alloc(0);
8f256622
PNA		 1535 if (!cls_flower.rule)
1536 return -ENOMEM;
1537
34738452
JP		 1538 cls_flower.common.chain_index = chain->index;
1539 cls_flower.command = TC_CLSFLOWER_TMPLT_CREATE;
1540 cls_flower.cookie = (unsigned long) tmplt;
8f256622
PNA		 1541 cls_flower.rule->match.dissector = &tmplt->dissector;
1542 cls_flower.rule->match.mask = &tmplt->mask;
1543 cls_flower.rule->match.key = &tmplt->dummy_key;
34738452
JP		 1544
1545 /* We don't care if driver (any of them) fails to handle this
1546 * call. It serves just as a hint for it.
1547 */
aeb3fecd 1548 tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false);
8f256622
PNA		 1549 kfree(cls_flower.rule);
1550
1551 return 0;
34738452
JP		 1552}
1553
1554static void fl_hw_destroy_tmplt(struct tcf_chain *chain,
1555 struct fl_flow_tmplt *tmplt)
1556{
1557 struct tc_cls_flower_offload cls_flower = {};
1558 struct tcf_block *block = chain->block;
1559
1560 cls_flower.common.chain_index = chain->index;
1561 cls_flower.command = TC_CLSFLOWER_TMPLT_DESTROY;
1562 cls_flower.cookie = (unsigned long) tmplt;
1563
aeb3fecd 1564 tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false);
34738452
JP		 1565}
1566
b95ec7eb
JP		 1567static void *fl_tmplt_create(struct net *net, struct tcf_chain *chain,
1568 struct nlattr **tca,
1569 struct netlink_ext_ack *extack)
1570{
1571 struct fl_flow_tmplt *tmplt;
1572 struct nlattr **tb;
1573 int err;
1574
1575 if (!tca[TCA_OPTIONS])
1576 return ERR_PTR(-EINVAL);
1577
1578 tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL);
1579 if (!tb)
1580 return ERR_PTR(-ENOBUFS);
1581 err = nla_parse_nested(tb, TCA_FLOWER_MAX, tca[TCA_OPTIONS],
1582 fl_policy, NULL);
1583 if (err)
1584 goto errout_tb;
1585
1586 tmplt = kzalloc(sizeof(*tmplt), GFP_KERNEL);
1cbc36a5
DC		 1587 if (!tmplt) {
1588 err = -ENOMEM;
b95ec7eb 1589 goto errout_tb;
1cbc36a5 1590 }
b95ec7eb
JP		 1591 tmplt->chain = chain;
1592 err = fl_set_key(net, tb, &tmplt->dummy_key, &tmplt->mask, extack);
1593 if (err)
1594 goto errout_tmplt;
b95ec7eb
JP		 1595
1596 fl_init_dissector(&tmplt->dissector, &tmplt->mask);
1597
8f256622
PNA		 1598 err = fl_hw_create_tmplt(chain, tmplt);
1599 if (err)
1600 goto errout_tmplt;
34738452 1601
8f256622 1602 kfree(tb);
b95ec7eb
JP		 1603 return tmplt;
1604
1605errout_tmplt:
1606 kfree(tmplt);
1607errout_tb:
1608 kfree(tb);
1609 return ERR_PTR(err);
1610}
1611
ec3ed293
VB		 1612static void fl_tmplt_destroy(void *tmplt_priv)
1613{
1614 struct fl_flow_tmplt *tmplt = tmplt_priv;
1615
95278dda
CW		 1616 fl_hw_destroy_tmplt(tmplt->chain, tmplt);
1617 kfree(tmplt);
ec3ed293
VB		 1618}
1619
77b9900e
JP		 1620static int fl_dump_key_val(struct sk_buff *skb,
1621 void *val, int val_type,
1622 void *mask, int mask_type, int len)
1623{
1624 int err;
1625
1626 if (!memchr_inv(mask, 0, len))
1627 return 0;
1628 err = nla_put(skb, val_type, len, val);
1629 if (err)
1630 return err;
1631 if (mask_type != TCA_FLOWER_UNSPEC) {
1632 err = nla_put(skb, mask_type, len, mask);
1633 if (err)
1634 return err;
1635 }
1636 return 0;
1637}
1638
5c72299f
AN		 1639static int fl_dump_key_port_range(struct sk_buff *skb, struct fl_flow_key *key,
1640 struct fl_flow_key *mask)
1641{
1642 if (fl_dump_key_val(skb, &key->tp_min.dst, TCA_FLOWER_KEY_PORT_DST_MIN,
1643 &mask->tp_min.dst, TCA_FLOWER_UNSPEC,
1644 sizeof(key->tp_min.dst)) ||
1645 fl_dump_key_val(skb, &key->tp_max.dst, TCA_FLOWER_KEY_PORT_DST_MAX,
1646 &mask->tp_max.dst, TCA_FLOWER_UNSPEC,
1647 sizeof(key->tp_max.dst)) ||
1648 fl_dump_key_val(skb, &key->tp_min.src, TCA_FLOWER_KEY_PORT_SRC_MIN,
1649 &mask->tp_min.src, TCA_FLOWER_UNSPEC,
1650 sizeof(key->tp_min.src)) ||
1651 fl_dump_key_val(skb, &key->tp_max.src, TCA_FLOWER_KEY_PORT_SRC_MAX,
1652 &mask->tp_max.src, TCA_FLOWER_UNSPEC,
1653 sizeof(key->tp_max.src)))
1654 return -1;
1655
1656 return 0;
1657}
1658
a577d8f7
BL		 1659static int fl_dump_key_mpls(struct sk_buff *skb,
1660 struct flow_dissector_key_mpls *mpls_key,
1661 struct flow_dissector_key_mpls *mpls_mask)
1662{
1663 int err;
1664
1665 if (!memchr_inv(mpls_mask, 0, sizeof(*mpls_mask)))
1666 return 0;
1667 if (mpls_mask->mpls_ttl) {
1668 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TTL,
1669 mpls_key->mpls_ttl);
1670 if (err)
1671 return err;
1672 }
1673 if (mpls_mask->mpls_tc) {
1674 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TC,
1675 mpls_key->mpls_tc);
1676 if (err)
1677 return err;
1678 }
1679 if (mpls_mask->mpls_label) {
1680 err = nla_put_u32(skb, TCA_FLOWER_KEY_MPLS_LABEL,
1681 mpls_key->mpls_label);
1682 if (err)
1683 return err;
1684 }
1685 if (mpls_mask->mpls_bos) {
1686 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_BOS,
1687 mpls_key->mpls_bos);
1688 if (err)
1689 return err;
1690 }
1691 return 0;
1692}
1693
0e2c17b6 1694static int fl_dump_key_ip(struct sk_buff *skb, bool encap,
4d80cc0a
OG		 1695 struct flow_dissector_key_ip *key,
1696 struct flow_dissector_key_ip *mask)
1697{
0e2c17b6
OG		 1698 int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS;
1699 int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL;
1700 int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK;
1701 int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK;
1702
1703 if (fl_dump_key_val(skb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos)) ||
1704 fl_dump_key_val(skb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl)))
4d80cc0a
OG		 1705 return -1;
1706
1707 return 0;
1708}
1709
9399ae9a 1710static int fl_dump_key_vlan(struct sk_buff *skb,
d64efd09 1711 int vlan_id_key, int vlan_prio_key,
9399ae9a
HHZ		 1712 struct flow_dissector_key_vlan *vlan_key,
1713 struct flow_dissector_key_vlan *vlan_mask)
1714{
1715 int err;
1716
1717 if (!memchr_inv(vlan_mask, 0, sizeof(*vlan_mask)))
1718 return 0;
1719 if (vlan_mask->vlan_id) {
d64efd09 1720 err = nla_put_u16(skb, vlan_id_key,
9399ae9a
HHZ		 1721 vlan_key->vlan_id);
1722 if (err)
1723 return err;
1724 }
1725 if (vlan_mask->vlan_priority) {
d64efd09 1726 err = nla_put_u8(skb, vlan_prio_key,
9399ae9a
HHZ		 1727 vlan_key->vlan_priority);
1728 if (err)
1729 return err;
1730 }
1731 return 0;
1732}
1733
faa3ffce
OG		 1734static void fl_get_key_flag(u32 dissector_key, u32 dissector_mask,
1735 u32 *flower_key, u32 *flower_mask,
1736 u32 flower_flag_bit, u32 dissector_flag_bit)
1737{
1738 if (dissector_mask & dissector_flag_bit) {
1739 *flower_mask |= flower_flag_bit;
1740 if (dissector_key & dissector_flag_bit)
1741 *flower_key |= flower_flag_bit;
1742 }
1743}
1744
1745static int fl_dump_key_flags(struct sk_buff *skb, u32 flags_key, u32 flags_mask)
1746{
1747 u32 key, mask;
1748 __be32 _key, _mask;
1749 int err;
1750
1751 if (!memchr_inv(&flags_mask, 0, sizeof(flags_mask)))
1752 return 0;
1753
1754 key = 0;
1755 mask = 0;
1756
1757 fl_get_key_flag(flags_key, flags_mask, &key, &mask,
1758 TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
459d153d
PJV		 1759 fl_get_key_flag(flags_key, flags_mask, &key, &mask,
1760 TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST,
1761 FLOW_DIS_FIRST_FRAG);
faa3ffce
OG		 1762
1763 _key = cpu_to_be32(key);
1764 _mask = cpu_to_be32(mask);
1765
1766 err = nla_put(skb, TCA_FLOWER_KEY_FLAGS, 4, &_key);
1767 if (err)
1768 return err;
1769
1770 return nla_put(skb, TCA_FLOWER_KEY_FLAGS_MASK, 4, &_mask);
1771}
1772
0a6e7778
PJV		 1773static int fl_dump_key_geneve_opt(struct sk_buff *skb,
1774 struct flow_dissector_key_enc_opts *enc_opts)
1775{
1776 struct geneve_opt *opt;
1777 struct nlattr *nest;
1778 int opt_off = 0;
1779
1780 nest = nla_nest_start(skb, TCA_FLOWER_KEY_ENC_OPTS_GENEVE);
1781 if (!nest)
1782 goto nla_put_failure;
1783
1784 while (enc_opts->len > opt_off) {
1785 opt = (struct geneve_opt *)&enc_opts->data[opt_off];
1786
1787 if (nla_put_be16(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS,
1788 opt->opt_class))
1789 goto nla_put_failure;
1790 if (nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE,
1791 opt->type))
1792 goto nla_put_failure;
1793 if (nla_put(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA,
1794 opt->length * 4, opt->opt_data))
1795 goto nla_put_failure;
1796
1797 opt_off += sizeof(struct geneve_opt) + opt->length * 4;
1798 }
1799 nla_nest_end(skb, nest);
1800 return 0;
1801
1802nla_put_failure:
1803 nla_nest_cancel(skb, nest);
1804 return -EMSGSIZE;
1805}
1806
1807static int fl_dump_key_options(struct sk_buff *skb, int enc_opt_type,
1808 struct flow_dissector_key_enc_opts *enc_opts)
1809{
1810 struct nlattr *nest;
1811 int err;
1812
1813 if (!enc_opts->len)
1814 return 0;
1815
1816 nest = nla_nest_start(skb, enc_opt_type);
1817 if (!nest)
1818 goto nla_put_failure;
1819
1820 switch (enc_opts->dst_opt_type) {
1821 case TUNNEL_GENEVE_OPT:
1822 err = fl_dump_key_geneve_opt(skb, enc_opts);
1823 if (err)
1824 goto nla_put_failure;
1825 break;
1826 default:
1827 goto nla_put_failure;
1828 }
1829 nla_nest_end(skb, nest);
1830 return 0;
1831
1832nla_put_failure:
1833 nla_nest_cancel(skb, nest);
1834 return -EMSGSIZE;
1835}
1836
1837static int fl_dump_key_enc_opt(struct sk_buff *skb,
1838 struct flow_dissector_key_enc_opts *key_opts,
1839 struct flow_dissector_key_enc_opts *msk_opts)
1840{
1841 int err;
1842
1843 err = fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS, key_opts);
1844 if (err)
1845 return err;
1846
1847 return fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS_MASK, msk_opts);
1848}
1849
f5749081
JP		 1850static int fl_dump_key(struct sk_buff *skb, struct net *net,
1851 struct fl_flow_key *key, struct fl_flow_key *mask)
77b9900e 1852{
77b9900e
JP		 1853 if (mask->indev_ifindex) {
1854 struct net_device *dev;
1855
1856 dev = __dev_get_by_index(net, key->indev_ifindex);
1857 if (dev && nla_put_string(skb, TCA_FLOWER_INDEV, dev->name))
1858 goto nla_put_failure;
1859 }
1860
1861 if (fl_dump_key_val(skb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
1862 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
1863 sizeof(key->eth.dst)) ||
1864 fl_dump_key_val(skb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
1865 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
1866 sizeof(key->eth.src)) ||
1867 fl_dump_key_val(skb, &key->basic.n_proto, TCA_FLOWER_KEY_ETH_TYPE,
1868 &mask->basic.n_proto, TCA_FLOWER_UNSPEC,
1869 sizeof(key->basic.n_proto)))
1870 goto nla_put_failure;
9399ae9a 1871
a577d8f7
BL		 1872 if (fl_dump_key_mpls(skb, &key->mpls, &mask->mpls))
1873 goto nla_put_failure;
1874
d64efd09
JL		 1875 if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_VLAN_ID,
1876 TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan, &mask->vlan))
9399ae9a
HHZ		 1877 goto nla_put_failure;
1878
d64efd09
JL		 1879 if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_CVLAN_ID,
1880 TCA_FLOWER_KEY_CVLAN_PRIO,
1881 &key->cvlan, &mask->cvlan) ||
1882 (mask->cvlan.vlan_tpid &&
158abbf1
JL		 1883 nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE,
1884 key->cvlan.vlan_tpid)))
d3069512
JL		 1885 goto nla_put_failure;
1886
5e9a0fe4
JL		 1887 if (mask->basic.n_proto) {
1888 if (mask->cvlan.vlan_tpid) {
1889 if (nla_put_be16(skb, TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
1890 key->basic.n_proto))
1891 goto nla_put_failure;
1892 } else if (mask->vlan.vlan_tpid) {
1893 if (nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE,
1894 key->basic.n_proto))
1895 goto nla_put_failure;
1896 }
d64efd09
JL		 1897 }
1898
77b9900e
JP		 1899 if ((key->basic.n_proto == htons(ETH_P_IP) ||
1900 key->basic.n_proto == htons(ETH_P_IPV6)) &&
4d80cc0a 1901 (fl_dump_key_val(skb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
77b9900e 1902 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
4d80cc0a 1903 sizeof(key->basic.ip_proto)) ||
0e2c17b6 1904 fl_dump_key_ip(skb, false, &key->ip, &mask->ip)))
77b9900e
JP		 1905 goto nla_put_failure;
1906
c3f83241 1907 if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
77b9900e
JP		 1908 (fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
1909 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
1910 sizeof(key->ipv4.src)) ||
1911 fl_dump_key_val(skb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
1912 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
1913 sizeof(key->ipv4.dst))))
1914 goto nla_put_failure;
c3f83241 1915 else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
77b9900e
JP		 1916 (fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
1917 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
1918 sizeof(key->ipv6.src)) ||
1919 fl_dump_key_val(skb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
1920 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
1921 sizeof(key->ipv6.dst))))
1922 goto nla_put_failure;
1923
1924 if (key->basic.ip_proto == IPPROTO_TCP &&
1925 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
aa72d708 1926 &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
77b9900e
JP		 1927 sizeof(key->tp.src)) ||
1928 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
aa72d708 1929 &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
fdfc7dd6
JP		 1930 sizeof(key->tp.dst)) ||
1931 fl_dump_key_val(skb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS,
1932 &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK,
1933 sizeof(key->tcp.flags))))
77b9900e
JP		 1934 goto nla_put_failure;
1935 else if (key->basic.ip_proto == IPPROTO_UDP &&
1936 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
aa72d708 1937 &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
77b9900e
JP		 1938 sizeof(key->tp.src)) ||
1939 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
aa72d708 1940 &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
5976c5f4
SH		 1941 sizeof(key->tp.dst))))
1942 goto nla_put_failure;
1943 else if (key->basic.ip_proto == IPPROTO_SCTP &&
1944 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
1945 &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
1946 sizeof(key->tp.src)) ||
1947 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
1948 &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
77b9900e
JP		 1949 sizeof(key->tp.dst))))
1950 goto nla_put_failure;
7b684884
SH		 1951 else if (key->basic.n_proto == htons(ETH_P_IP) &&
1952 key->basic.ip_proto == IPPROTO_ICMP &&
1953 (fl_dump_key_val(skb, &key->icmp.type,
1954 TCA_FLOWER_KEY_ICMPV4_TYPE, &mask->icmp.type,
1955 TCA_FLOWER_KEY_ICMPV4_TYPE_MASK,
1956 sizeof(key->icmp.type)) ||
1957 fl_dump_key_val(skb, &key->icmp.code,
1958 TCA_FLOWER_KEY_ICMPV4_CODE, &mask->icmp.code,
1959 TCA_FLOWER_KEY_ICMPV4_CODE_MASK,
1960 sizeof(key->icmp.code))))
1961 goto nla_put_failure;
1962 else if (key->basic.n_proto == htons(ETH_P_IPV6) &&
1963 key->basic.ip_proto == IPPROTO_ICMPV6 &&
1964 (fl_dump_key_val(skb, &key->icmp.type,
1965 TCA_FLOWER_KEY_ICMPV6_TYPE, &mask->icmp.type,
1966 TCA_FLOWER_KEY_ICMPV6_TYPE_MASK,
1967 sizeof(key->icmp.type)) ||
1968 fl_dump_key_val(skb, &key->icmp.code,
1969 TCA_FLOWER_KEY_ICMPV6_CODE, &mask->icmp.code,
1970 TCA_FLOWER_KEY_ICMPV6_CODE_MASK,
1971 sizeof(key->icmp.code))))
1972 goto nla_put_failure;
99d31326
SH		 1973 else if ((key->basic.n_proto == htons(ETH_P_ARP) ||
1974 key->basic.n_proto == htons(ETH_P_RARP)) &&
1975 (fl_dump_key_val(skb, &key->arp.sip,
1976 TCA_FLOWER_KEY_ARP_SIP, &mask->arp.sip,
1977 TCA_FLOWER_KEY_ARP_SIP_MASK,
1978 sizeof(key->arp.sip)) ||
1979 fl_dump_key_val(skb, &key->arp.tip,
1980 TCA_FLOWER_KEY_ARP_TIP, &mask->arp.tip,
1981 TCA_FLOWER_KEY_ARP_TIP_MASK,
1982 sizeof(key->arp.tip)) ||
1983 fl_dump_key_val(skb, &key->arp.op,
1984 TCA_FLOWER_KEY_ARP_OP, &mask->arp.op,
1985 TCA_FLOWER_KEY_ARP_OP_MASK,
1986 sizeof(key->arp.op)) ||
1987 fl_dump_key_val(skb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA,
1988 mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK,
1989 sizeof(key->arp.sha)) ||
1990 fl_dump_key_val(skb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA,
1991 mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK,
1992 sizeof(key->arp.tha))))
1993 goto nla_put_failure;
77b9900e 1994
5c72299f
AN		 1995 if ((key->basic.ip_proto == IPPROTO_TCP ||
1996 key->basic.ip_proto == IPPROTO_UDP ||
1997 key->basic.ip_proto == IPPROTO_SCTP) &&
1998 fl_dump_key_port_range(skb, key, mask))
1999 goto nla_put_failure;
2000
bc3103f1
AV		 2001 if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
2002 (fl_dump_key_val(skb, &key->enc_ipv4.src,
2003 TCA_FLOWER_KEY_ENC_IPV4_SRC, &mask->enc_ipv4.src,
2004 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
2005 sizeof(key->enc_ipv4.src)) ||
2006 fl_dump_key_val(skb, &key->enc_ipv4.dst,
2007 TCA_FLOWER_KEY_ENC_IPV4_DST, &mask->enc_ipv4.dst,
2008 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
2009 sizeof(key->enc_ipv4.dst))))
2010 goto nla_put_failure;
2011 else if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
2012 (fl_dump_key_val(skb, &key->enc_ipv6.src,
2013 TCA_FLOWER_KEY_ENC_IPV6_SRC, &mask->enc_ipv6.src,
2014 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
2015 sizeof(key->enc_ipv6.src)) ||
2016 fl_dump_key_val(skb, &key->enc_ipv6.dst,
2017 TCA_FLOWER_KEY_ENC_IPV6_DST,
2018 &mask->enc_ipv6.dst,
2019 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
2020 sizeof(key->enc_ipv6.dst))))
2021 goto nla_put_failure;
2022
2023 if (fl_dump_key_val(skb, &key->enc_key_id, TCA_FLOWER_KEY_ENC_KEY_ID,
eb523f42 2024 &mask->enc_key_id, TCA_FLOWER_UNSPEC,
f4d997fd
HHZ		 2025 sizeof(key->enc_key_id)) ||
2026 fl_dump_key_val(skb, &key->enc_tp.src,
2027 TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
2028 &mask->enc_tp.src,
2029 TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
2030 sizeof(key->enc_tp.src)) ||
2031 fl_dump_key_val(skb, &key->enc_tp.dst,
2032 TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
2033 &mask->enc_tp.dst,
2034 TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
0e2c17b6 2035 sizeof(key->enc_tp.dst)) ||
0a6e7778
PJV		 2036 fl_dump_key_ip(skb, true, &key->enc_ip, &mask->enc_ip) ||
2037 fl_dump_key_enc_opt(skb, &key->enc_opts, &mask->enc_opts))
bc3103f1
AV		 2038 goto nla_put_failure;
2039
faa3ffce
OG		 2040 if (fl_dump_key_flags(skb, key->control.flags, mask->control.flags))
2041 goto nla_put_failure;
2042
f5749081
JP		 2043 return 0;
2044
2045nla_put_failure:
2046 return -EMSGSIZE;
2047}
2048
2049static int fl_dump(struct net *net, struct tcf_proto *tp, void *fh,
12db03b6 2050 struct sk_buff *skb, struct tcmsg *t, bool rtnl_held)
f5749081
JP		 2051{
2052 struct cls_fl_filter *f = fh;
2053 struct nlattr *nest;
2054 struct fl_flow_key *key, *mask;
2055
2056 if (!f)
2057 return skb->len;
2058
2059 t->tcm_handle = f->handle;
2060
2061 nest = nla_nest_start(skb, TCA_OPTIONS);
2062 if (!nest)
2063 goto nla_put_failure;
2064
2065 if (f->res.classid &&
2066 nla_put_u32(skb, TCA_FLOWER_CLASSID, f->res.classid))
2067 goto nla_put_failure;
2068
2069 key = &f->key;
2070 mask = &f->mask->key;
2071
2072 if (fl_dump_key(skb, net, key, mask))
2073 goto nla_put_failure;
2074
2075 if (!tc_skip_hw(f->flags))
2076 fl_hw_update_stats(tp, f);
2077
749e6720
OG		 2078 if (f->flags && nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags))
2079 goto nla_put_failure;
e69985c6 2080
86c55361
VB		 2081 if (nla_put_u32(skb, TCA_FLOWER_IN_HW_COUNT, f->in_hw_count))
2082 goto nla_put_failure;
2083
77b9900e
JP		 2084 if (tcf_exts_dump(skb, &f->exts))
2085 goto nla_put_failure;
2086
2087 nla_nest_end(skb, nest);
2088
2089 if (tcf_exts_dump_stats(skb, &f->exts) < 0)
2090 goto nla_put_failure;
2091
2092 return skb->len;
2093
2094nla_put_failure:
2095 nla_nest_cancel(skb, nest);
2096 return -1;
2097}
2098
b95ec7eb
JP		 2099static int fl_tmplt_dump(struct sk_buff *skb, struct net *net, void *tmplt_priv)
2100{
2101 struct fl_flow_tmplt *tmplt = tmplt_priv;
2102 struct fl_flow_key *key, *mask;
2103 struct nlattr *nest;
2104
2105 nest = nla_nest_start(skb, TCA_OPTIONS);
2106 if (!nest)
2107 goto nla_put_failure;
2108
2109 key = &tmplt->dummy_key;
2110 mask = &tmplt->mask;
2111
2112 if (fl_dump_key(skb, net, key, mask))
2113 goto nla_put_failure;
2114
2115 nla_nest_end(skb, nest);
2116
2117 return skb->len;
2118
2119nla_put_failure:
2120 nla_nest_cancel(skb, nest);
2121 return -EMSGSIZE;
2122}
2123
07d79fc7
CW		 2124static void fl_bind_class(void *fh, u32 classid, unsigned long cl)
2125{
2126 struct cls_fl_filter *f = fh;
2127
2128 if (f && f->res.classid == classid)
2129 f->res.class = cl;
2130}
2131
77b9900e
JP		 2132static struct tcf_proto_ops cls_fl_ops __read_mostly = {
2133 .kind = "flower",
2134 .classify = fl_classify,
2135 .init = fl_init,
2136 .destroy = fl_destroy,
2137 .get = fl_get,
2138 .change = fl_change,
2139 .delete = fl_delete,
2140 .walk = fl_walk,
31533cba 2141 .reoffload = fl_reoffload,
77b9900e 2142 .dump = fl_dump,
07d79fc7 2143 .bind_class = fl_bind_class,
b95ec7eb
JP		 2144 .tmplt_create = fl_tmplt_create,
2145 .tmplt_destroy = fl_tmplt_destroy,
2146 .tmplt_dump = fl_tmplt_dump,
77b9900e
JP		 2147 .owner = THIS_MODULE,
2148};
2149
2150static int __init cls_fl_init(void)
2151{
2152 return register_tcf_proto_ops(&cls_fl_ops);
2153}
2154
2155static void __exit cls_fl_exit(void)
2156{
2157 unregister_tcf_proto_ops(&cls_fl_ops);
2158}
2159
2160module_init(cls_fl_init);
2161module_exit(cls_fl_exit);
2162
2163MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>");
2164MODULE_DESCRIPTION("Flower classifier");
2165MODULE_LICENSE("GPL v2");
Linux 6.x block layer and io_uring tree(s)