projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
netfilter: nf_tables: do not reduce read-only expressions
[linux-block.git] / net / netfilter / nft_payload.c
CommitLineData
d2912cb1 1// SPDX-License-Identifier: GPL-2.0-only
96518518 2/*
ef1f7df9 3 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
18140969 4 * Copyright (c) 2016 Pablo Neira Ayuso <pablo@netfilter.org>
96518518 5 *
96518518
PM		 6 * Development of this code funded by Astaro AG (http://www.astaro.com/)
7 */
8
9#include <linux/kernel.h>
8cfd23e6 10#include <linux/if_vlan.h>
96518518
PM		 11#include <linux/init.h>
12#include <linux/module.h>
13#include <linux/netlink.h>
14#include <linux/netfilter.h>
15#include <linux/netfilter/nf_tables.h>
16#include <net/netfilter/nf_tables_core.h>
17#include <net/netfilter/nf_tables.h>
c9626a2c 18#include <net/netfilter/nf_tables_offload.h>
18140969
PNA		 19/* For layer 4 checksum field offset. */
20#include <linux/tcp.h>
21#include <linux/udp.h>
22#include <linux/icmpv6.h>
c9626a2c
PNA		 23#include <linux/ip.h>
24#include <linux/ipv6.h>
346e320c 25#include <net/sctp/checksum.h>
96518518 26
8dfd8b09
PNA		 27static bool nft_payload_rebuild_vlan_hdr(const struct sk_buff *skb, int mac_off,
28 struct vlan_ethhdr *veth)
29{
30 if (skb_copy_bits(skb, mac_off, veth, ETH_HLEN))
31 return false;
32
33 veth->h_vlan_proto = skb->vlan_proto;
34 veth->h_vlan_TCI = htons(skb_vlan_tag_get(skb));
35 veth->h_vlan_encapsulated_proto = skb->protocol;
36
37 return true;
38}
39
8cfd23e6
FW		 40/* add vlan header into the user buffer for if tag was removed by offloads */
41static bool
42nft_payload_copy_vlan(u32 *d, const struct sk_buff *skb, u8 offset, u8 len)
43{
44 int mac_off = skb_mac_header(skb) - skb->data;
f41f72d0 45 u8 *vlanh, *dst_u8 = (u8 *) d;
8cfd23e6 46 struct vlan_ethhdr veth;
f6ae9f12
PNA		 47 u8 vlan_hlen = 0;
48
49 if ((skb->protocol == htons(ETH_P_8021AD) ||
50 skb->protocol == htons(ETH_P_8021Q)) &&
51 offset >= VLAN_ETH_HLEN && offset < VLAN_ETH_HLEN + VLAN_HLEN)
52 vlan_hlen += VLAN_HLEN;
8cfd23e6
FW		 53
54 vlanh = (u8 *) &veth;
f6ae9f12 55 if (offset < VLAN_ETH_HLEN + vlan_hlen) {
f41f72d0 56 u8 ethlen = len;
8cfd23e6 57
f6ae9f12
PNA		 58 if (vlan_hlen &&
59 skb_copy_bits(skb, mac_off, &veth, VLAN_ETH_HLEN) < 0)
60 return false;
61 else if (!nft_payload_rebuild_vlan_hdr(skb, mac_off, &veth))
8cfd23e6
FW		 62 return false;
63
f6ae9f12
PNA		 64 if (offset + len > VLAN_ETH_HLEN + vlan_hlen)
65 ethlen -= offset + len - VLAN_ETH_HLEN + vlan_hlen;
8cfd23e6 66
f6ae9f12 67 memcpy(dst_u8, vlanh + offset - vlan_hlen, ethlen);
8cfd23e6
FW		 68
69 len -= ethlen;
70 if (len == 0)
71 return true;
72
73 dst_u8 += ethlen;
f6ae9f12 74 offset = ETH_HLEN + vlan_hlen;
f41f72d0 75 } else {
f6ae9f12 76 offset -= VLAN_HLEN + vlan_hlen;
8cfd23e6
FW		 77 }
78
8cfd23e6
FW		 79 return skb_copy_bits(skb, offset + mac_off, dst_u8, len) == 0;
80}
81
c46b38dc
PNA		 82static int __nft_payload_inner_offset(struct nft_pktinfo *pkt)
83{
84 unsigned int thoff = nft_thoff(pkt);
85
a9e8503d 86 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
c46b38dc
PNA		 87 return -1;
88
89 switch (pkt->tprot) {
90 case IPPROTO_UDP:
91 pkt->inneroff = thoff + sizeof(struct udphdr);
92 break;
93 case IPPROTO_TCP: {
94 struct tcphdr *th, _tcph;
95
96 th = skb_header_pointer(pkt->skb, thoff, sizeof(_tcph), &_tcph);
97 if (!th)
98 return -1;
99
100 pkt->inneroff = thoff + __tcp_hdrlen(th);
101 }
102 break;
103 default:
104 return -1;
105 }
106
107 pkt->flags |= NFT_PKTINFO_INNER;
108
109 return 0;
110}
111
112static int nft_payload_inner_offset(const struct nft_pktinfo *pkt)
113{
114 if (!(pkt->flags & NFT_PKTINFO_INNER) &&
115 __nft_payload_inner_offset((struct nft_pktinfo *)pkt) < 0)
116 return -1;
117
118 return pkt->inneroff;
119}
120
10870dd8
FW		 121void nft_payload_eval(const struct nft_expr *expr,
122 struct nft_regs *regs,
123 const struct nft_pktinfo *pkt)
96518518
PM		 124{
125 const struct nft_payload *priv = nft_expr_priv(expr);
126 const struct sk_buff *skb = pkt->skb;
49499c3e 127 u32 *dest = &regs->data[priv->dreg];
96518518
PM		 128 int offset;
129
1e105e6a
FW		 130 if (priv->len % NFT_REG32_SIZE)
131 dest[priv->len / NFT_REG32_SIZE] = 0;
132
96518518
PM		 133 switch (priv->base) {
134 case NFT_PAYLOAD_LL_HEADER:
135 if (!skb_mac_header_was_set(skb))
136 goto err;
8cfd23e6
FW		 137
138 if (skb_vlan_tag_present(skb)) {
139 if (!nft_payload_copy_vlan(dest, skb,
140 priv->offset, priv->len))
141 goto err;
142 return;
143 }
96518518
PM		 144 offset = skb_mac_header(skb) - skb->data;
145 break;
146 case NFT_PAYLOAD_NETWORK_HEADER:
147 offset = skb_network_offset(skb);
148 break;
149 case NFT_PAYLOAD_TRANSPORT_HEADER:
a9e8503d 150 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
a20877b5 151 goto err;
2d7b4ace 152 offset = nft_thoff(pkt);
96518518 153 break;
c46b38dc
PNA		 154 case NFT_PAYLOAD_INNER_HEADER:
155 offset = nft_payload_inner_offset(pkt);
156 if (offset < 0)
157 goto err;
158 break;
96518518 159 default:
8801d791
PNA		 160 WARN_ON_ONCE(1);
161 goto err;
96518518
PM		 162 }
163 offset += priv->offset;
164
fad136ea 165 if (skb_copy_bits(skb, offset, dest, priv->len) < 0)
96518518
PM		 166 goto err;
167 return;
168err:
a55e22e9 169 regs->verdict.code = NFT_BREAK;
96518518
PM		 170}
171
172static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = {
7ec3f7b4
PM		 173 [NFTA_PAYLOAD_SREG] = { .type = NLA_U32 },
174 [NFTA_PAYLOAD_DREG] = { .type = NLA_U32 },
175 [NFTA_PAYLOAD_BASE] = { .type = NLA_U32 },
176 [NFTA_PAYLOAD_OFFSET] = { .type = NLA_U32 },
177 [NFTA_PAYLOAD_LEN] = { .type = NLA_U32 },
178 [NFTA_PAYLOAD_CSUM_TYPE] = { .type = NLA_U32 },
179 [NFTA_PAYLOAD_CSUM_OFFSET] = { .type = NLA_U32 },
9d6effb2 180 [NFTA_PAYLOAD_CSUM_FLAGS] = { .type = NLA_U32 },
96518518
PM		 181};
182
183static int nft_payload_init(const struct nft_ctx *ctx,
184 const struct nft_expr *expr,
185 const struct nlattr * const tb[])
186{
187 struct nft_payload *priv = nft_expr_priv(expr);
96518518 188
c29b72e0 189 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
96518518
PM		 190 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
191 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
96518518 192
345023b0
PNA		 193 return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG],
194 &priv->dreg, NULL, NFT_DATA_VALUE,
195 priv->len);
96518518
PM		 196}
197
198static int nft_payload_dump(struct sk_buff *skb, const struct nft_expr *expr)
199{
200 const struct nft_payload *priv = nft_expr_priv(expr);
201
b1c96ed3 202 if (nft_dump_register(skb, NFTA_PAYLOAD_DREG, priv->dreg) ||
96518518
PM		 203 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
204 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
205 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)))
206 goto nla_put_failure;
207 return 0;
208
209nla_put_failure:
210 return -1;
211}
212
a7c176bf
PNA		 213static bool nft_payload_reduce(struct nft_regs_track *track,
214 const struct nft_expr *expr)
215{
216 const struct nft_payload *priv = nft_expr_priv(expr);
217 const struct nft_payload *payload;
218
219 if (!track->regs[priv->dreg].selector ||
220 track->regs[priv->dreg].selector->ops != expr->ops) {
221 track->regs[priv->dreg].selector = expr;
222 track->regs[priv->dreg].bitwise = NULL;
223 return false;
224 }
225
226 payload = nft_expr_priv(track->regs[priv->dreg].selector);
227 if (priv->base != payload->base ||
228 priv->offset != payload->offset ||
229 priv->len != payload->len) {
230 track->regs[priv->dreg].selector = expr;
231 track->regs[priv->dreg].bitwise = NULL;
232 return false;
233 }
234
235 if (!track->regs[priv->dreg].bitwise)
236 return true;
237
be5650f8 238 return nft_expr_reduce_bitwise(track, expr);
a7c176bf
PNA		 239}
240
a5d45bc0
PNA		 241static bool nft_payload_offload_mask(struct nft_offload_reg *reg,
242 u32 priv_len, u32 field_len)
243{
244 unsigned int remainder, delta, k;
245 struct nft_data mask = {};
246 __be32 remainder_mask;
247
248 if (priv_len == field_len) {
249 memset(&reg->mask, 0xff, priv_len);
250 return true;
251 } else if (priv_len > field_len) {
252 return false;
253 }
254
255 memset(&mask, 0xff, field_len);
256 remainder = priv_len % sizeof(u32);
257 if (remainder) {
258 k = priv_len / sizeof(u32);
259 delta = field_len - priv_len;
260 remainder_mask = htonl(~((1 << (delta * BITS_PER_BYTE)) - 1));
261 mask.data[k] = (__force u32)remainder_mask;
262 }
263
264 memcpy(&reg->mask, &mask, field_len);
265
266 return true;
267}
268
c9626a2c
PNA		 269static int nft_payload_offload_ll(struct nft_offload_ctx *ctx,
270 struct nft_flow_rule *flow,
271 const struct nft_payload *priv)
272{
273 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
274
275 switch (priv->offset) {
276 case offsetof(struct ethhdr, h_source):
a5d45bc0 277 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN))
a69a85da 278 return -EOPNOTSUPP;
279
c9626a2c
PNA		 280 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
281 src, ETH_ALEN, reg);
282 break;
283 case offsetof(struct ethhdr, h_dest):
a5d45bc0 284 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN))
a69a85da 285 return -EOPNOTSUPP;
286
c9626a2c
PNA		 287 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
288 dst, ETH_ALEN, reg);
289 break;
a82055af 290 case offsetof(struct ethhdr, h_proto):
a5d45bc0 291 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
a82055af
PNA		 292 return -EOPNOTSUPP;
293
294 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic,
295 n_proto, sizeof(__be16), reg);
296 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
297 break;
298 case offsetof(struct vlan_ethhdr, h_vlan_TCI):
a5d45bc0 299 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
a82055af
PNA		 300 return -EOPNOTSUPP;
301
ff4d90a8
PNA		 302 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_VLAN, vlan,
303 vlan_tci, sizeof(__be16), reg,
304 NFT_OFFLOAD_F_NETWORK2HOST);
a82055af
PNA		 305 break;
306 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto):
a5d45bc0 307 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
a82055af
PNA		 308 return -EOPNOTSUPP;
309
310 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_VLAN, vlan,
311 vlan_tpid, sizeof(__be16), reg);
89d8fd44
PNA		 312 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
313 break;
314 case offsetof(struct vlan_ethhdr, h_vlan_TCI) + sizeof(struct vlan_hdr):
a5d45bc0 315 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
89d8fd44
PNA		 316 return -EOPNOTSUPP;
317
ff4d90a8
PNA		 318 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_CVLAN, cvlan,
319 vlan_tci, sizeof(__be16), reg,
320 NFT_OFFLOAD_F_NETWORK2HOST);
89d8fd44
PNA		 321 break;
322 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto) +
323 sizeof(struct vlan_hdr):
a5d45bc0 324 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
89d8fd44
PNA		 325 return -EOPNOTSUPP;
326
14c20643 327 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_CVLAN, cvlan,
89d8fd44 328 vlan_tpid, sizeof(__be16), reg);
14c20643 329 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
a82055af 330 break;
a69a85da 331 default:
332 return -EOPNOTSUPP;
c9626a2c
PNA		 333 }
334
335 return 0;
336}
337
338static int nft_payload_offload_ip(struct nft_offload_ctx *ctx,
339 struct nft_flow_rule *flow,
340 const struct nft_payload *priv)
341{
342 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
343
344 switch (priv->offset) {
345 case offsetof(struct iphdr, saddr):
a5d45bc0
PNA		 346 if (!nft_payload_offload_mask(reg, priv->len,
347 sizeof(struct in_addr)))
a69a85da 348 return -EOPNOTSUPP;
349
c9626a2c
PNA		 350 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, src,
351 sizeof(struct in_addr), reg);
3c78e9e0 352 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
c9626a2c
PNA		 353 break;
354 case offsetof(struct iphdr, daddr):
a5d45bc0
PNA		 355 if (!nft_payload_offload_mask(reg, priv->len,
356 sizeof(struct in_addr)))
a69a85da 357 return -EOPNOTSUPP;
358
c9626a2c
PNA		 359 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, dst,
360 sizeof(struct in_addr), reg);
3c78e9e0 361 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
c9626a2c
PNA		 362 break;
363 case offsetof(struct iphdr, protocol):
a5d45bc0 364 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8)))
a69a85da 365 return -EOPNOTSUPP;
366
c9626a2c
PNA		 367 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
368 sizeof(__u8), reg);
369 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
370 break;
371 default:
372 return -EOPNOTSUPP;
373 }
374
375 return 0;
376}
377
378static int nft_payload_offload_ip6(struct nft_offload_ctx *ctx,
379 struct nft_flow_rule *flow,
380 const struct nft_payload *priv)
381{
382 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
383
384 switch (priv->offset) {
385 case offsetof(struct ipv6hdr, saddr):
a5d45bc0
PNA		 386 if (!nft_payload_offload_mask(reg, priv->len,
387 sizeof(struct in6_addr)))
a69a85da 388 return -EOPNOTSUPP;
389
c9626a2c
PNA		 390 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, src,
391 sizeof(struct in6_addr), reg);
3c78e9e0 392 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
c9626a2c
PNA		 393 break;
394 case offsetof(struct ipv6hdr, daddr):
a5d45bc0
PNA		 395 if (!nft_payload_offload_mask(reg, priv->len,
396 sizeof(struct in6_addr)))
a69a85da 397 return -EOPNOTSUPP;
398
c9626a2c
PNA		 399 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, dst,
400 sizeof(struct in6_addr), reg);
3c78e9e0 401 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
c9626a2c
PNA		 402 break;
403 case offsetof(struct ipv6hdr, nexthdr):
a5d45bc0 404 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8)))
a69a85da 405 return -EOPNOTSUPP;
406
c9626a2c
PNA		 407 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
408 sizeof(__u8), reg);
409 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
410 break;
411 default:
412 return -EOPNOTSUPP;
413 }
414
415 return 0;
416}
417
418static int nft_payload_offload_nh(struct nft_offload_ctx *ctx,
419 struct nft_flow_rule *flow,
420 const struct nft_payload *priv)
421{
422 int err;
423
424 switch (ctx->dep.l3num) {
425 case htons(ETH_P_IP):
426 err = nft_payload_offload_ip(ctx, flow, priv);
427 break;
428 case htons(ETH_P_IPV6):
429 err = nft_payload_offload_ip6(ctx, flow, priv);
430 break;
431 default:
432 return -EOPNOTSUPP;
433 }
434
435 return err;
436}
437
438static int nft_payload_offload_tcp(struct nft_offload_ctx *ctx,
439 struct nft_flow_rule *flow,
440 const struct nft_payload *priv)
441{
442 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
443
444 switch (priv->offset) {
445 case offsetof(struct tcphdr, source):
a5d45bc0 446 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
a69a85da 447 return -EOPNOTSUPP;
448
c9626a2c
PNA		 449 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
450 sizeof(__be16), reg);
451 break;
452 case offsetof(struct tcphdr, dest):
a5d45bc0 453 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
a69a85da 454 return -EOPNOTSUPP;
455
c9626a2c
PNA		 456 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
457 sizeof(__be16), reg);
458 break;
459 default:
460 return -EOPNOTSUPP;
461 }
462
463 return 0;
464}
465
466static int nft_payload_offload_udp(struct nft_offload_ctx *ctx,
467 struct nft_flow_rule *flow,
468 const struct nft_payload *priv)
469{
470 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
471
472 switch (priv->offset) {
473 case offsetof(struct udphdr, source):
a5d45bc0 474 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
a69a85da 475 return -EOPNOTSUPP;
476
c9626a2c
PNA		 477 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
478 sizeof(__be16), reg);
479 break;
480 case offsetof(struct udphdr, dest):
a5d45bc0 481 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
a69a85da 482 return -EOPNOTSUPP;
483
c9626a2c
PNA		 484 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
485 sizeof(__be16), reg);
486 break;
487 default:
488 return -EOPNOTSUPP;
489 }
490
491 return 0;
492}
493
494static int nft_payload_offload_th(struct nft_offload_ctx *ctx,
495 struct nft_flow_rule *flow,
496 const struct nft_payload *priv)
497{
498 int err;
499
500 switch (ctx->dep.protonum) {
501 case IPPROTO_TCP:
502 err = nft_payload_offload_tcp(ctx, flow, priv);
503 break;
504 case IPPROTO_UDP:
505 err = nft_payload_offload_udp(ctx, flow, priv);
506 break;
507 default:
508 return -EOPNOTSUPP;
509 }
510
511 return err;
512}
513
514static int nft_payload_offload(struct nft_offload_ctx *ctx,
515 struct nft_flow_rule *flow,
516 const struct nft_expr *expr)
517{
518 const struct nft_payload *priv = nft_expr_priv(expr);
519 int err;
520
521 switch (priv->base) {
522 case NFT_PAYLOAD_LL_HEADER:
523 err = nft_payload_offload_ll(ctx, flow, priv);
524 break;
525 case NFT_PAYLOAD_NETWORK_HEADER:
526 err = nft_payload_offload_nh(ctx, flow, priv);
527 break;
528 case NFT_PAYLOAD_TRANSPORT_HEADER:
529 err = nft_payload_offload_th(ctx, flow, priv);
530 break;
531 default:
532 err = -EOPNOTSUPP;
533 break;
534 }
535 return err;
536}
537
ef1f7df9
PM		 538static const struct nft_expr_ops nft_payload_ops = {
539 .type = &nft_payload_type,
96518518 540 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
96518518
PM		 541 .eval = nft_payload_eval,
542 .init = nft_payload_init,
543 .dump = nft_payload_dump,
a7c176bf 544 .reduce = nft_payload_reduce,
c9626a2c 545 .offload = nft_payload_offload,
ef1f7df9
PM		 546};
547
c29b72e0
PM		 548const struct nft_expr_ops nft_payload_fast_ops = {
549 .type = &nft_payload_type,
550 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
551 .eval = nft_payload_eval,
552 .init = nft_payload_init,
553 .dump = nft_payload_dump,
a7c176bf 554 .reduce = nft_payload_reduce,
c9626a2c 555 .offload = nft_payload_offload,
c29b72e0
PM		 556};
557
18140969
PNA		 558static inline void nft_csum_replace(__sum16 *sum, __wsum fsum, __wsum tsum)
559{
560 *sum = csum_fold(csum_add(csum_sub(~csum_unfold(*sum), fsum), tsum));
561 if (*sum == 0)
562 *sum = CSUM_MANGLED_0;
563}
564
565static bool nft_payload_udp_checksum(struct sk_buff *skb, unsigned int thoff)
566{
567 struct udphdr *uh, _uh;
568
569 uh = skb_header_pointer(skb, thoff, sizeof(_uh), &_uh);
570 if (!uh)
571 return false;
572
5fd02ebe 573 return (__force bool)uh->check;
18140969
PNA		 574}
575
576static int nft_payload_l4csum_offset(const struct nft_pktinfo *pkt,
577 struct sk_buff *skb,
578 unsigned int *l4csum_offset)
579{
4e1860a3
PNA		 580 if (pkt->fragoff)
581 return -1;
582
18140969
PNA		 583 switch (pkt->tprot) {
584 case IPPROTO_TCP:
585 *l4csum_offset = offsetof(struct tcphdr, check);
586 break;
587 case IPPROTO_UDP:
2d7b4ace 588 if (!nft_payload_udp_checksum(skb, nft_thoff(pkt)))
18140969 589 return -1;
954d8297 590 fallthrough;
18140969
PNA		 591 case IPPROTO_UDPLITE:
592 *l4csum_offset = offsetof(struct udphdr, check);
593 break;
594 case IPPROTO_ICMPV6:
595 *l4csum_offset = offsetof(struct icmp6hdr, icmp6_cksum);
596 break;
597 default:
598 return -1;
599 }
600
2d7b4ace 601 *l4csum_offset += nft_thoff(pkt);
18140969
PNA		 602 return 0;
603}
604
346e320c
DC		 605static int nft_payload_csum_sctp(struct sk_buff *skb, int offset)
606{
607 struct sctphdr *sh;
608
609 if (skb_ensure_writable(skb, offset + sizeof(*sh)))
610 return -1;
611
612 sh = (struct sctphdr *)(skb->data + offset);
613 sh->checksum = sctp_compute_cksum(skb, offset);
614 skb->ip_summed = CHECKSUM_UNNECESSARY;
615 return 0;
616}
617
18140969
PNA		 618static int nft_payload_l4csum_update(const struct nft_pktinfo *pkt,
619 struct sk_buff *skb,
620 __wsum fsum, __wsum tsum)
621{
622 int l4csum_offset;
623 __sum16 sum;
624
625 /* If we cannot determine layer 4 checksum offset or this packet doesn't
626 * require layer 4 checksum recalculation, skip this packet.
627 */
628 if (nft_payload_l4csum_offset(pkt, skb, &l4csum_offset) < 0)
629 return 0;
630
631 if (skb_copy_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
632 return -1;
633
634 /* Checksum mangling for an arbitrary amount of bytes, based on
635 * inet_proto_csum_replace*() functions.
636 */
637 if (skb->ip_summed != CHECKSUM_PARTIAL) {
638 nft_csum_replace(&sum, fsum, tsum);
639 if (skb->ip_summed == CHECKSUM_COMPLETE) {
640 skb->csum = ~csum_add(csum_sub(~(skb->csum), fsum),
641 tsum);
642 }
643 } else {
644 sum = ~csum_fold(csum_add(csum_sub(csum_unfold(sum), fsum),
645 tsum));
646 }
647
7418ee4c 648 if (skb_ensure_writable(skb, l4csum_offset + sizeof(sum)) ||
18140969
PNA		 649 skb_store_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
650 return -1;
651
652 return 0;
653}
654
053d20f5
PNA		 655static int nft_payload_csum_inet(struct sk_buff *skb, const u32 *src,
656 __wsum fsum, __wsum tsum, int csum_offset)
657{
658 __sum16 sum;
659
660 if (skb_copy_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
661 return -1;
662
663 nft_csum_replace(&sum, fsum, tsum);
7418ee4c 664 if (skb_ensure_writable(skb, csum_offset + sizeof(sum)) ||
053d20f5
PNA		 665 skb_store_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
666 return -1;
667
668 return 0;
669}
670
7ec3f7b4
PM		 671static void nft_payload_set_eval(const struct nft_expr *expr,
672 struct nft_regs *regs,
673 const struct nft_pktinfo *pkt)
674{
675 const struct nft_payload_set *priv = nft_expr_priv(expr);
676 struct sk_buff *skb = pkt->skb;
677 const u32 *src = &regs->data[priv->sreg];
678 int offset, csum_offset;
679 __wsum fsum, tsum;
7ec3f7b4
PM		 680
681 switch (priv->base) {
682 case NFT_PAYLOAD_LL_HEADER:
683 if (!skb_mac_header_was_set(skb))
684 goto err;
685 offset = skb_mac_header(skb) - skb->data;
686 break;
687 case NFT_PAYLOAD_NETWORK_HEADER:
688 offset = skb_network_offset(skb);
689 break;
690 case NFT_PAYLOAD_TRANSPORT_HEADER:
a9e8503d 691 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
a20877b5 692 goto err;
2d7b4ace 693 offset = nft_thoff(pkt);
7ec3f7b4 694 break;
c46b38dc
PNA		 695 case NFT_PAYLOAD_INNER_HEADER:
696 offset = nft_payload_inner_offset(pkt);
697 if (offset < 0)
698 goto err;
699 break;
7ec3f7b4 700 default:
8801d791
PNA		 701 WARN_ON_ONCE(1);
702 goto err;
7ec3f7b4
PM		 703 }
704
705 csum_offset = offset + priv->csum_offset;
706 offset += priv->offset;
707
053d20f5 708 if ((priv->csum_type == NFT_PAYLOAD_CSUM_INET || priv->csum_flags) &&
c46b38dc
PNA		 709 ((priv->base != NFT_PAYLOAD_TRANSPORT_HEADER &&
710 priv->base != NFT_PAYLOAD_INNER_HEADER) ||
7ec3f7b4 711 skb->ip_summed != CHECKSUM_PARTIAL)) {
7ec3f7b4
PM		 712 fsum = skb_checksum(skb, offset, priv->len, 0);
713 tsum = csum_partial(src, priv->len, 0);
7ec3f7b4 714
053d20f5
PNA		 715 if (priv->csum_type == NFT_PAYLOAD_CSUM_INET &&
716 nft_payload_csum_inet(skb, src, fsum, tsum, csum_offset))
7ec3f7b4 717 goto err;
18140969
PNA		 718
719 if (priv->csum_flags &&
720 nft_payload_l4csum_update(pkt, skb, fsum, tsum) < 0)
721 goto err;
7ec3f7b4
PM		 722 }
723
7418ee4c 724 if (skb_ensure_writable(skb, max(offset + priv->len, 0)) ||
7ec3f7b4
PM		 725 skb_store_bits(skb, offset, src, priv->len) < 0)
726 goto err;
727
346e320c
DC		 728 if (priv->csum_type == NFT_PAYLOAD_CSUM_SCTP &&
729 pkt->tprot == IPPROTO_SCTP &&
730 skb->ip_summed != CHECKSUM_PARTIAL) {
a9e8503d
FW		 731 if (pkt->fragoff == 0 &&
732 nft_payload_csum_sctp(skb, nft_thoff(pkt)))
346e320c
DC		 733 goto err;
734 }
735
7ec3f7b4
PM		 736 return;
737err:
738 regs->verdict.code = NFT_BREAK;
739}
740
741static int nft_payload_set_init(const struct nft_ctx *ctx,
742 const struct nft_expr *expr,
743 const struct nlattr * const tb[])
744{
745 struct nft_payload_set *priv = nft_expr_priv(expr);
746
747 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
748 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
749 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
7ec3f7b4
PM		 750
751 if (tb[NFTA_PAYLOAD_CSUM_TYPE])
752 priv->csum_type =
753 ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_TYPE]));
754 if (tb[NFTA_PAYLOAD_CSUM_OFFSET])
755 priv->csum_offset =
756 ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_OFFSET]));
18140969
PNA		 757 if (tb[NFTA_PAYLOAD_CSUM_FLAGS]) {
758 u32 flags;
759
760 flags = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_FLAGS]));
761 if (flags & ~NFT_PAYLOAD_L4CSUM_PSEUDOHDR)
762 return -EINVAL;
763
764 priv->csum_flags = flags;
765 }
7ec3f7b4
PM		 766
767 switch (priv->csum_type) {
768 case NFT_PAYLOAD_CSUM_NONE:
769 case NFT_PAYLOAD_CSUM_INET:
770 break;
346e320c
DC		 771 case NFT_PAYLOAD_CSUM_SCTP:
772 if (priv->base != NFT_PAYLOAD_TRANSPORT_HEADER)
773 return -EINVAL;
774
775 if (priv->csum_offset != offsetof(struct sctphdr, checksum))
776 return -EINVAL;
777 break;
7ec3f7b4
PM		 778 default:
779 return -EOPNOTSUPP;
780 }
781
4f16d25c
PNA		 782 return nft_parse_register_load(tb[NFTA_PAYLOAD_SREG], &priv->sreg,
783 priv->len);
7ec3f7b4
PM		 784}
785
786static int nft_payload_set_dump(struct sk_buff *skb, const struct nft_expr *expr)
787{
788 const struct nft_payload_set *priv = nft_expr_priv(expr);
789
790 if (nft_dump_register(skb, NFTA_PAYLOAD_SREG, priv->sreg) ||
791 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
792 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
793 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)) ||
794 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_TYPE, htonl(priv->csum_type)) ||
795 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_OFFSET,
18140969
PNA		 796 htonl(priv->csum_offset)) ||
797 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_FLAGS, htonl(priv->csum_flags)))
7ec3f7b4
PM		 798 goto nla_put_failure;
799 return 0;
800
801nla_put_failure:
802 return -1;
803}
804
cc003c7e
PNA		 805static bool nft_payload_set_reduce(struct nft_regs_track *track,
806 const struct nft_expr *expr)
807{
808 int i;
809
810 for (i = 0; i < NFT_REG32_NUM; i++) {
811 if (!track->regs[i].selector)
812 continue;
813
814 if (track->regs[i].selector->ops != &nft_payload_ops &&
815 track->regs[i].selector->ops != &nft_payload_fast_ops)
816 continue;
817
818 track->regs[i].selector = NULL;
819 track->regs[i].bitwise = NULL;
820 }
821
822 return false;
823}
824
7ec3f7b4
PM		 825static const struct nft_expr_ops nft_payload_set_ops = {
826 .type = &nft_payload_type,
827 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload_set)),
828 .eval = nft_payload_set_eval,
829 .init = nft_payload_set_init,
830 .dump = nft_payload_set_dump,
cc003c7e 831 .reduce = nft_payload_set_reduce,
7ec3f7b4
PM		 832};
833
0ca743a5
PNA		 834static const struct nft_expr_ops *
835nft_payload_select_ops(const struct nft_ctx *ctx,
836 const struct nlattr * const tb[])
c29b72e0
PM		 837{
838 enum nft_payload_bases base;
839 unsigned int offset, len;
840
7ec3f7b4 841 if (tb[NFTA_PAYLOAD_BASE] == NULL ||
c29b72e0
PM		 842 tb[NFTA_PAYLOAD_OFFSET] == NULL ||
843 tb[NFTA_PAYLOAD_LEN] == NULL)
844 return ERR_PTR(-EINVAL);
845
846 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
847 switch (base) {
848 case NFT_PAYLOAD_LL_HEADER:
849 case NFT_PAYLOAD_NETWORK_HEADER:
850 case NFT_PAYLOAD_TRANSPORT_HEADER:
c46b38dc 851 case NFT_PAYLOAD_INNER_HEADER:
c29b72e0
PM		 852 break;
853 default:
854 return ERR_PTR(-EOPNOTSUPP);
855 }
856
7ec3f7b4
PM		 857 if (tb[NFTA_PAYLOAD_SREG] != NULL) {
858 if (tb[NFTA_PAYLOAD_DREG] != NULL)
859 return ERR_PTR(-EINVAL);
860 return &nft_payload_set_ops;
861 }
862
863 if (tb[NFTA_PAYLOAD_DREG] == NULL)
864 return ERR_PTR(-EINVAL);
865
c29b72e0 866 offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
45d9bcda 867 len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
c29b72e0 868
f627ed91 869 if (len <= 4 && is_power_of_2(len) && IS_ALIGNED(offset, len) &&
c46b38dc 870 base != NFT_PAYLOAD_LL_HEADER && base != NFT_PAYLOAD_INNER_HEADER)
c29b72e0
PM		 871 return &nft_payload_fast_ops;
872 else
873 return &nft_payload_ops;
874}
875
4e24877e 876struct nft_expr_type nft_payload_type __read_mostly = {
ef1f7df9 877 .name = "payload",
c29b72e0 878 .select_ops = nft_payload_select_ops,
96518518
PM		 879 .policy = nft_payload_policy,
880 .maxattr = NFTA_PAYLOAD_MAX,
ef1f7df9 881 .owner = THIS_MODULE,
96518518 882};
Linux 6.x block layer and io_uring tree(s)