[linux-2.6-block.git] / samples / bpf / xdp_redirect_cpu.bpf.c

/*  XDP redirect to CPUs via cpumap (BPF_MAP_TYPE_CPUMAP)
 *
 *  GPLv2, Copyright(c) 2017 Jesper Dangaard Brouer, Red Hat, Inc.
 */
#include "vmlinux.h"
#include "xdp_sample.bpf.h"
#include "xdp_sample_shared.h"
#include "hash_func01.h"

/* Special map type that can XDP_REDIRECT frames to another CPU */
struct {
	__uint(type, BPF_MAP_TYPE_CPUMAP);
	__uint(key_size, sizeof(u32));
	__uint(value_size, sizeof(struct bpf_cpumap_val));
} cpu_map SEC(".maps");

/* Set of maps controlling available CPU, and for iterating through
 * selectable redirect CPUs.
 */
struct {
	__uint(type, BPF_MAP_TYPE_ARRAY);
	__type(key, u32);
	__type(value, u32);
} cpus_available SEC(".maps");

struct {
	__uint(type, BPF_MAP_TYPE_ARRAY);
	__type(key, u32);
	__type(value, u32);
	__uint(max_entries, 1);
} cpus_count SEC(".maps");

struct {
	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
	__type(key, u32);
	__type(value, u32);
	__uint(max_entries, 1);
} cpus_iterator SEC(".maps");

struct {
	__uint(type, BPF_MAP_TYPE_DEVMAP);
	__uint(key_size, sizeof(int));
	__uint(value_size, sizeof(struct bpf_devmap_val));
	__uint(max_entries, 1);
} tx_port SEC(".maps");

char tx_mac_addr[ETH_ALEN];

/* Helper parse functions */

static __always_inline
bool parse_eth(struct ethhdr *eth, void *data_end,
	       u16 *eth_proto, u64 *l3_offset)
{
	u16 eth_type;
	u64 offset;

	offset = sizeof(*eth);
	if ((void *)eth + offset > data_end)
		return false;

	eth_type = eth->h_proto;

	/* Skip non 802.3 Ethertypes */
	if (__builtin_expect(bpf_ntohs(eth_type) < ETH_P_802_3_MIN, 0))
		return false;

	/* Handle VLAN tagged packet */
	if (eth_type == bpf_htons(ETH_P_8021Q) ||
	    eth_type == bpf_htons(ETH_P_8021AD)) {
		struct vlan_hdr *vlan_hdr;

		vlan_hdr = (void *)eth + offset;
		offset += sizeof(*vlan_hdr);
		if ((void *)eth + offset > data_end)
			return false;
		eth_type = vlan_hdr->h_vlan_encapsulated_proto;
	}
	/* Handle double VLAN tagged packet */
	if (eth_type == bpf_htons(ETH_P_8021Q) ||
	    eth_type == bpf_htons(ETH_P_8021AD)) {
		struct vlan_hdr *vlan_hdr;

		vlan_hdr = (void *)eth + offset;
		offset += sizeof(*vlan_hdr);
		if ((void *)eth + offset > data_end)
			return false;
		eth_type = vlan_hdr->h_vlan_encapsulated_proto;
	}

	*eth_proto = bpf_ntohs(eth_type);
	*l3_offset = offset;
	return true;
}

static __always_inline
u16 get_dest_port_ipv4_udp(struct xdp_md *ctx, u64 nh_off)
{
	void *data_end = (void *)(long)ctx->data_end;
	void *data     = (void *)(long)ctx->data;
	struct iphdr *iph = data + nh_off;
	struct udphdr *udph;

	if (iph + 1 > data_end)
		return 0;
	if (!(iph->protocol == IPPROTO_UDP))
		return 0;

	udph = (void *)(iph + 1);
	if (udph + 1 > data_end)
		return 0;

	return bpf_ntohs(udph->dest);
}

static __always_inline
int get_proto_ipv4(struct xdp_md *ctx, u64 nh_off)
{
	void *data_end = (void *)(long)ctx->data_end;
	void *data     = (void *)(long)ctx->data;
	struct iphdr *iph = data + nh_off;

	if (iph + 1 > data_end)
		return 0;
	return iph->protocol;
}

static __always_inline
int get_proto_ipv6(struct xdp_md *ctx, u64 nh_off)
{
	void *data_end = (void *)(long)ctx->data_end;
	void *data     = (void *)(long)ctx->data;
	struct ipv6hdr *ip6h = data + nh_off;

	if (ip6h + 1 > data_end)
		return 0;
	return ip6h->nexthdr;
}

SEC("xdp")
int  xdp_prognum0_no_touch(struct xdp_md *ctx)
{
	u32 key = bpf_get_smp_processor_id();
	struct datarec *rec;
	u32 *cpu_selected;
	u32 cpu_dest = 0;
	u32 key0 = 0;

	/* Only use first entry in cpus_available */
	cpu_selected = bpf_map_lookup_elem(&cpus_available, &key0);
	if (!cpu_selected)
		return XDP_ABORTED;
	cpu_dest = *cpu_selected;

	rec = bpf_map_lookup_elem(&rx_cnt, &key);
	if (!rec)
		return XDP_PASS;
	NO_TEAR_INC(rec->processed);

	if (cpu_dest >= nr_cpus) {
		NO_TEAR_INC(rec->issue);
		return XDP_ABORTED;
	}
	return bpf_redirect_map(&cpu_map, cpu_dest, 0);
}

SEC("xdp")
int  xdp_prognum1_touch_data(struct xdp_md *ctx)
{
	void *data_end = (void *)(long)ctx->data_end;
	void *data     = (void *)(long)ctx->data;
	u32 key = bpf_get_smp_processor_id();
	struct ethhdr *eth = data;
	struct datarec *rec;
	u32 *cpu_selected;
	u32 cpu_dest = 0;
	u32 key0 = 0;
	u16 eth_type;

	/* Only use first entry in cpus_available */
	cpu_selected = bpf_map_lookup_elem(&cpus_available, &key0);
	if (!cpu_selected)
		return XDP_ABORTED;
	cpu_dest = *cpu_selected;

	/* Validate packet length is minimum Eth header size */
	if (eth + 1 > data_end)
		return XDP_ABORTED;

	rec = bpf_map_lookup_elem(&rx_cnt, &key);
	if (!rec)
		return XDP_PASS;
	NO_TEAR_INC(rec->processed);

	/* Read packet data, and use it (drop non 802.3 Ethertypes) */
	eth_type = eth->h_proto;
	if (bpf_ntohs(eth_type) < ETH_P_802_3_MIN) {
		NO_TEAR_INC(rec->dropped);
		return XDP_DROP;
	}

	if (cpu_dest >= nr_cpus) {
		NO_TEAR_INC(rec->issue);
		return XDP_ABORTED;
	}
	return bpf_redirect_map(&cpu_map, cpu_dest, 0);
}

SEC("xdp")
int  xdp_prognum2_round_robin(struct xdp_md *ctx)
{
	void *data_end = (void *)(long)ctx->data_end;
	void *data     = (void *)(long)ctx->data;
	u32 key = bpf_get_smp_processor_id();
	struct datarec *rec;
	u32 cpu_dest = 0;
	u32 key0 = 0;

	u32 *cpu_selected;
	u32 *cpu_iterator;
	u32 *cpu_max;
	u32 cpu_idx;

	cpu_max = bpf_map_lookup_elem(&cpus_count, &key0);
	if (!cpu_max)
		return XDP_ABORTED;

	cpu_iterator = bpf_map_lookup_elem(&cpus_iterator, &key0);
	if (!cpu_iterator)
		return XDP_ABORTED;
	cpu_idx = *cpu_iterator;

	*cpu_iterator += 1;
	if (*cpu_iterator == *cpu_max)
		*cpu_iterator = 0;

	cpu_selected = bpf_map_lookup_elem(&cpus_available, &cpu_idx);
	if (!cpu_selected)
		return XDP_ABORTED;
	cpu_dest = *cpu_selected;

	rec = bpf_map_lookup_elem(&rx_cnt, &key);
	if (!rec)
		return XDP_PASS;
	NO_TEAR_INC(rec->processed);

	if (cpu_dest >= nr_cpus) {
		NO_TEAR_INC(rec->issue);
		return XDP_ABORTED;
	}
	return bpf_redirect_map(&cpu_map, cpu_dest, 0);
}

SEC("xdp")
int  xdp_prognum3_proto_separate(struct xdp_md *ctx)
{
	void *data_end = (void *)(long)ctx->data_end;
	void *data     = (void *)(long)ctx->data;
	u32 key = bpf_get_smp_processor_id();
	struct ethhdr *eth = data;
	u8 ip_proto = IPPROTO_UDP;
	struct datarec *rec;
	u16 eth_proto = 0;
	u64 l3_offset = 0;
	u32 cpu_dest = 0;
	u32 *cpu_lookup;
	u32 cpu_idx = 0;

	rec = bpf_map_lookup_elem(&rx_cnt, &key);
	if (!rec)
		return XDP_PASS;
	NO_TEAR_INC(rec->processed);

	if (!(parse_eth(eth, data_end, &eth_proto, &l3_offset)))
		return XDP_PASS; /* Just skip */

	/* Extract L4 protocol */
	switch (eth_proto) {
	case ETH_P_IP:
		ip_proto = get_proto_ipv4(ctx, l3_offset);
		break;
	case ETH_P_IPV6:
		ip_proto = get_proto_ipv6(ctx, l3_offset);
		break;
	case ETH_P_ARP:
		cpu_idx = 0; /* ARP packet handled on separate CPU */
		break;
	default:
		cpu_idx = 0;
	}

	/* Choose CPU based on L4 protocol */
	switch (ip_proto) {
	case IPPROTO_ICMP:
	case IPPROTO_ICMPV6:
		cpu_idx = 2;
		break;
	case IPPROTO_TCP:
		cpu_idx = 0;
		break;
	case IPPROTO_UDP:
		cpu_idx = 1;
		break;
	default:
		cpu_idx = 0;
	}

	cpu_lookup = bpf_map_lookup_elem(&cpus_available, &cpu_idx);
	if (!cpu_lookup)
		return XDP_ABORTED;
	cpu_dest = *cpu_lookup;

	if (cpu_dest >= nr_cpus) {
		NO_TEAR_INC(rec->issue);
		return XDP_ABORTED;
	}
	return bpf_redirect_map(&cpu_map, cpu_dest, 0);
}

SEC("xdp")
int  xdp_prognum4_ddos_filter_pktgen(struct xdp_md *ctx)
{
	void *data_end = (void *)(long)ctx->data_end;
	void *data     = (void *)(long)ctx->data;
	u32 key = bpf_get_smp_processor_id();
	struct ethhdr *eth = data;
	u8 ip_proto = IPPROTO_UDP;
	struct datarec *rec;
	u16 eth_proto = 0;
	u64 l3_offset = 0;
	u32 cpu_dest = 0;
	u32 *cpu_lookup;
	u32 cpu_idx = 0;
	u16 dest_port;

	rec = bpf_map_lookup_elem(&rx_cnt, &key);
	if (!rec)
		return XDP_PASS;
	NO_TEAR_INC(rec->processed);

	if (!(parse_eth(eth, data_end, &eth_proto, &l3_offset)))
		return XDP_PASS; /* Just skip */

	/* Extract L4 protocol */
	switch (eth_proto) {
	case ETH_P_IP:
		ip_proto = get_proto_ipv4(ctx, l3_offset);
		break;
	case ETH_P_IPV6:
		ip_proto = get_proto_ipv6(ctx, l3_offset);
		break;
	case ETH_P_ARP:
		cpu_idx = 0; /* ARP packet handled on separate CPU */
		break;
	default:
		cpu_idx = 0;
	}

	/* Choose CPU based on L4 protocol */
	switch (ip_proto) {
	case IPPROTO_ICMP:
	case IPPROTO_ICMPV6:
		cpu_idx = 2;
		break;
	case IPPROTO_TCP:
		cpu_idx = 0;
		break;
	case IPPROTO_UDP:
		cpu_idx = 1;
		/* DDoS filter UDP port 9 (pktgen) */
		dest_port = get_dest_port_ipv4_udp(ctx, l3_offset);
		if (dest_port == 9) {
			NO_TEAR_INC(rec->dropped);
			return XDP_DROP;
		}
		break;
	default:
		cpu_idx = 0;
	}

	cpu_lookup = bpf_map_lookup_elem(&cpus_available, &cpu_idx);
	if (!cpu_lookup)
		return XDP_ABORTED;
	cpu_dest = *cpu_lookup;

	if (cpu_dest >= nr_cpus) {
		NO_TEAR_INC(rec->issue);
		return XDP_ABORTED;
	}
	return bpf_redirect_map(&cpu_map, cpu_dest, 0);
}

/* Hashing initval */
#define INITVAL 15485863

static __always_inline
u32 get_ipv4_hash_ip_pair(struct xdp_md *ctx, u64 nh_off)
{
	void *data_end = (void *)(long)ctx->data_end;
	void *data     = (void *)(long)ctx->data;
	struct iphdr *iph = data + nh_off;
	u32 cpu_hash;

	if (iph + 1 > data_end)
		return 0;

	cpu_hash = iph->saddr + iph->daddr;
	cpu_hash = SuperFastHash((char *)&cpu_hash, 4, INITVAL + iph->protocol);

	return cpu_hash;
}

static __always_inline
u32 get_ipv6_hash_ip_pair(struct xdp_md *ctx, u64 nh_off)
{
	void *data_end = (void *)(long)ctx->data_end;
	void *data     = (void *)(long)ctx->data;
	struct ipv6hdr *ip6h = data + nh_off;
	u32 cpu_hash;

	if (ip6h + 1 > data_end)
		return 0;

	cpu_hash  = ip6h->saddr.in6_u.u6_addr32[0] + ip6h->daddr.in6_u.u6_addr32[0];
	cpu_hash += ip6h->saddr.in6_u.u6_addr32[1] + ip6h->daddr.in6_u.u6_addr32[1];
	cpu_hash += ip6h->saddr.in6_u.u6_addr32[2] + ip6h->daddr.in6_u.u6_addr32[2];
	cpu_hash += ip6h->saddr.in6_u.u6_addr32[3] + ip6h->daddr.in6_u.u6_addr32[3];
	cpu_hash = SuperFastHash((char *)&cpu_hash, 4, INITVAL + ip6h->nexthdr);

	return cpu_hash;
}

/* Load-Balance traffic based on hashing IP-addrs + L4-proto.  The
 * hashing scheme is symmetric, meaning swapping IP src/dest still hit
 * same CPU.
 */
SEC("xdp")
int  xdp_prognum5_lb_hash_ip_pairs(struct xdp_md *ctx)
{
	void *data_end = (void *)(long)ctx->data_end;
	void *data     = (void *)(long)ctx->data;
	u32 key = bpf_get_smp_processor_id();
	struct ethhdr *eth = data;
	struct datarec *rec;
	u16 eth_proto = 0;
	u64 l3_offset = 0;
	u32 cpu_dest = 0;
	u32 cpu_idx = 0;
	u32 *cpu_lookup;
	u32 key0 = 0;
	u32 *cpu_max;
	u32 cpu_hash;

	rec = bpf_map_lookup_elem(&rx_cnt, &key);
	if (!rec)
		return XDP_PASS;
	NO_TEAR_INC(rec->processed);

	cpu_max = bpf_map_lookup_elem(&cpus_count, &key0);
	if (!cpu_max)
		return XDP_ABORTED;

	if (!(parse_eth(eth, data_end, &eth_proto, &l3_offset)))
		return XDP_PASS; /* Just skip */

	/* Hash for IPv4 and IPv6 */
	switch (eth_proto) {
	case ETH_P_IP:
		cpu_hash = get_ipv4_hash_ip_pair(ctx, l3_offset);
		break;
	case ETH_P_IPV6:
		cpu_hash = get_ipv6_hash_ip_pair(ctx, l3_offset);
		break;
	case ETH_P_ARP: /* ARP packet handled on CPU idx 0 */
	default:
		cpu_hash = 0;
	}

	/* Choose CPU based on hash */
	cpu_idx = cpu_hash % *cpu_max;

	cpu_lookup = bpf_map_lookup_elem(&cpus_available, &cpu_idx);
	if (!cpu_lookup)
		return XDP_ABORTED;
	cpu_dest = *cpu_lookup;

	if (cpu_dest >= nr_cpus) {
		NO_TEAR_INC(rec->issue);
		return XDP_ABORTED;
	}
	return bpf_redirect_map(&cpu_map, cpu_dest, 0);
}

SEC("xdp/cpumap")
int xdp_redirect_cpu_devmap(struct xdp_md *ctx)
{
	void *data_end = (void *)(long)ctx->data_end;
	void *data = (void *)(long)ctx->data;
	struct ethhdr *eth = data;
	u64 nh_off;

	nh_off = sizeof(*eth);
	if (data + nh_off > data_end)
		return XDP_DROP;

	swap_src_dst_mac(data);
	return bpf_redirect_map(&tx_port, 0, 0);
}

SEC("xdp/cpumap")
int xdp_redirect_cpu_pass(struct xdp_md *ctx)
{
	return XDP_PASS;
}

SEC("xdp/cpumap")
int xdp_redirect_cpu_drop(struct xdp_md *ctx)
{
	return XDP_DROP;
}

SEC("xdp/devmap")
int xdp_redirect_egress_prog(struct xdp_md *ctx)
{
	void *data_end = (void *)(long)ctx->data_end;
	void *data = (void *)(long)ctx->data;
	struct ethhdr *eth = data;
	u64 nh_off;

	nh_off = sizeof(*eth);
	if (data + nh_off > data_end)
		return XDP_DROP;

	__builtin_memcpy(eth->h_source, (const char *)tx_mac_addr, ETH_ALEN);

	return XDP_PASS;
}

char _license[] SEC("license") = "GPL";
Commit	Line	Data
fad3917e JDB	1	/* XDP redirect to CPUs via cpumap (BPF_MAP_TYPE_CPUMAP)
	2	*
	3	* GPLv2, Copyright(c) 2017 Jesper Dangaard Brouer, Red Hat, Inc.
	4	*/
79ccf452 KKD	5	#include "vmlinux.h"
	6	#include "xdp_sample.bpf.h"
	7	#include "xdp_sample_shared.h"
1bca4e6b	8	#include "hash_func01.h"
fad3917e	9
fad3917e	10	/* Special map type that can XDP_REDIRECT frames to another CPU */
451d1dc8 DL	11	struct {
	12	__uint(type, BPF_MAP_TYPE_CPUMAP);
	13	__uint(key_size, sizeof(u32));
ce4dade7	14	__uint(value_size, sizeof(struct bpf_cpumap_val));
451d1dc8	15	} cpu_map SEC(".maps");
fad3917e	16
fad3917e JDB	17	/* Set of maps controlling available CPU, and for iterating through
	18	* selectable redirect CPUs.
	19	*/
451d1dc8 DL	20	struct {
	21	__uint(type, BPF_MAP_TYPE_ARRAY);
	22	__type(key, u32);
	23	__type(value, u32);
451d1dc8	24	} cpus_available SEC(".maps");
79ccf452	25
451d1dc8 DL	26	struct {
	27	__uint(type, BPF_MAP_TYPE_ARRAY);
	28	__type(key, u32);
	29	__type(value, u32);
	30	__uint(max_entries, 1);
	31	} cpus_count SEC(".maps");
79ccf452	32
451d1dc8 DL	33	struct {
	34	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
	35	__type(key, u32);
	36	__type(value, u32);
	37	__uint(max_entries, 1);
	38	} cpus_iterator SEC(".maps");
fad3917e	39
451d1dc8	40	struct {
79ccf452 KKD	41	__uint(type, BPF_MAP_TYPE_DEVMAP);
	42	__uint(key_size, sizeof(int));
	43	__uint(value_size, sizeof(struct bpf_devmap_val));
451d1dc8	44	__uint(max_entries, 1);
79ccf452	45	} tx_port SEC(".maps");
fad3917e	46
79ccf452	47	char tx_mac_addr[ETH_ALEN];
fad3917e	48
79ccf452	49	/* Helper parse functions */
fad3917e JDB	50
	51	static __always_inline
	52	bool parse_eth(struct ethhdr eth, void data_end,
	53	u16 eth_proto, u64 l3_offset)
	54	{
	55	u16 eth_type;
	56	u64 offset;
	57
	58	offset = sizeof(*eth);
	59	if ((void *)eth + offset > data_end)
	60	return false;
	61
	62	eth_type = eth->h_proto;
	63
	64	/* Skip non 802.3 Ethertypes */
79ccf452	65	if (__builtin_expect(bpf_ntohs(eth_type) < ETH_P_802_3_MIN, 0))
fad3917e JDB	66	return false;
	67
	68	/* Handle VLAN tagged packet */
79ccf452 KKD	69	if (eth_type == bpf_htons(ETH_P_8021Q) \|\|
79ccf452 KKD	70	eth_type == bpf_htons(ETH_P_8021AD)) {
fad3917e JDB	71	struct vlan_hdr *vlan_hdr;
	72
	73	vlan_hdr = (void *)eth + offset;
	74	offset += sizeof(*vlan_hdr);
	75	if ((void *)eth + offset > data_end)
	76	return false;
	77	eth_type = vlan_hdr->h_vlan_encapsulated_proto;
	78	}
d23b27c0	79	/* Handle double VLAN tagged packet */
79ccf452 KKD	80	if (eth_type == bpf_htons(ETH_P_8021Q) \|\|
79ccf452 KKD	81	eth_type == bpf_htons(ETH_P_8021AD)) {
d23b27c0 JDB	82	struct vlan_hdr *vlan_hdr;
	83
	84	vlan_hdr = (void *)eth + offset;
	85	offset += sizeof(*vlan_hdr);
	86	if ((void *)eth + offset > data_end)
	87	return false;
	88	eth_type = vlan_hdr->h_vlan_encapsulated_proto;
	89	}
fad3917e	90
79ccf452	91	*eth_proto = bpf_ntohs(eth_type);
fad3917e JDB	92	*l3_offset = offset;
	93	return true;
	94	}
	95
	96	static __always_inline
	97	u16 get_dest_port_ipv4_udp(struct xdp_md *ctx, u64 nh_off)
	98	{
	99	void data_end = (void )(long)ctx->data_end;
	100	void data = (void )(long)ctx->data;
	101	struct iphdr *iph = data + nh_off;
	102	struct udphdr *udph;
fad3917e JDB	103
	104	if (iph + 1 > data_end)
	105	return 0;
	106	if (!(iph->protocol == IPPROTO_UDP))
	107	return 0;
	108
	109	udph = (void *)(iph + 1);
	110	if (udph + 1 > data_end)
	111	return 0;
	112
ac55b3f0	113	return bpf_ntohs(udph->dest);
fad3917e JDB	114	}
	115
	116	static __always_inline
	117	int get_proto_ipv4(struct xdp_md *ctx, u64 nh_off)
	118	{
	119	void data_end = (void )(long)ctx->data_end;
	120	void data = (void )(long)ctx->data;
	121	struct iphdr *iph = data + nh_off;
	122
	123	if (iph + 1 > data_end)
	124	return 0;
	125	return iph->protocol;
	126	}
	127
	128	static __always_inline
	129	int get_proto_ipv6(struct xdp_md *ctx, u64 nh_off)
	130	{
	131	void data_end = (void )(long)ctx->data_end;
	132	void data = (void )(long)ctx->data;
	133	struct ipv6hdr *ip6h = data + nh_off;
	134
	135	if (ip6h + 1 > data_end)
	136	return 0;
	137	return ip6h->nexthdr;
	138	}
	139
79ccf452	140	SEC("xdp")
fad3917e JDB	141	int xdp_prognum0_no_touch(struct xdp_md *ctx)
fad3917e JDB	142	{
79ccf452	143	u32 key = bpf_get_smp_processor_id();
fad3917e JDB	144	struct datarec *rec;
fad3917e JDB	145	u32 *cpu_selected;
79ccf452 KKD	146	u32 cpu_dest = 0;
79ccf452 KKD	147	u32 key0 = 0;
fad3917e JDB	148
fad3917e JDB	149	/* Only use first entry in cpus_available */
79ccf452	150	cpu_selected = bpf_map_lookup_elem(&cpus_available, &key0);
fad3917e JDB	151	if (!cpu_selected)
	152	return XDP_ABORTED;
	153	cpu_dest = *cpu_selected;
	154
fad3917e JDB	155	rec = bpf_map_lookup_elem(&rx_cnt, &key);
fad3917e JDB	156	if (!rec)
79ccf452 KKD	157	return XDP_PASS;
79ccf452 KKD	158	NO_TEAR_INC(rec->processed);
fad3917e	159
79ccf452 KKD	160	if (cpu_dest >= nr_cpus) {
79ccf452 KKD	161	NO_TEAR_INC(rec->issue);
fad3917e JDB	162	return XDP_ABORTED;
fad3917e JDB	163	}
fad3917e JDB	164	return bpf_redirect_map(&cpu_map, cpu_dest, 0);
	165	}
	166
79ccf452	167	SEC("xdp")
fad3917e JDB	168	int xdp_prognum1_touch_data(struct xdp_md *ctx)
	169	{
	170	void data_end = (void )(long)ctx->data_end;
	171	void data = (void )(long)ctx->data;
79ccf452	172	u32 key = bpf_get_smp_processor_id();
fad3917e JDB	173	struct ethhdr *eth = data;
	174	struct datarec *rec;
	175	u32 *cpu_selected;
79ccf452 KKD	176	u32 cpu_dest = 0;
79ccf452 KKD	177	u32 key0 = 0;
fad3917e	178	u16 eth_type;
fad3917e JDB	179
fad3917e JDB	180	/* Only use first entry in cpus_available */
79ccf452	181	cpu_selected = bpf_map_lookup_elem(&cpus_available, &key0);
fad3917e JDB	182	if (!cpu_selected)
	183	return XDP_ABORTED;
	184	cpu_dest = *cpu_selected;
	185
	186	/* Validate packet length is minimum Eth header size */
	187	if (eth + 1 > data_end)
	188	return XDP_ABORTED;
	189
fad3917e JDB	190	rec = bpf_map_lookup_elem(&rx_cnt, &key);
fad3917e JDB	191	if (!rec)
79ccf452 KKD	192	return XDP_PASS;
79ccf452 KKD	193	NO_TEAR_INC(rec->processed);
fad3917e JDB	194
	195	/* Read packet data, and use it (drop non 802.3 Ethertypes) */
	196	eth_type = eth->h_proto;
79ccf452 KKD	197	if (bpf_ntohs(eth_type) < ETH_P_802_3_MIN) {
79ccf452 KKD	198	NO_TEAR_INC(rec->dropped);
fad3917e JDB	199	return XDP_DROP;
	200	}
	201
79ccf452 KKD	202	if (cpu_dest >= nr_cpus) {
79ccf452 KKD	203	NO_TEAR_INC(rec->issue);
fad3917e JDB	204	return XDP_ABORTED;
fad3917e JDB	205	}
fad3917e JDB	206	return bpf_redirect_map(&cpu_map, cpu_dest, 0);
	207	}
	208
79ccf452	209	SEC("xdp")
fad3917e JDB	210	int xdp_prognum2_round_robin(struct xdp_md *ctx)
	211	{
	212	void data_end = (void )(long)ctx->data_end;
	213	void data = (void )(long)ctx->data;
79ccf452	214	u32 key = bpf_get_smp_processor_id();
fad3917e	215	struct datarec *rec;
79ccf452	216	u32 cpu_dest = 0;
fad3917e JDB	217	u32 key0 = 0;
	218
	219	u32 *cpu_selected;
	220	u32 *cpu_iterator;
	221	u32 *cpu_max;
	222	u32 cpu_idx;
	223
	224	cpu_max = bpf_map_lookup_elem(&cpus_count, &key0);
	225	if (!cpu_max)
	226	return XDP_ABORTED;
	227
	228	cpu_iterator = bpf_map_lookup_elem(&cpus_iterator, &key0);
	229	if (!cpu_iterator)
	230	return XDP_ABORTED;
	231	cpu_idx = *cpu_iterator;
	232
	233	*cpu_iterator += 1;
	234	if (cpu_iterator == cpu_max)
	235	*cpu_iterator = 0;
	236
	237	cpu_selected = bpf_map_lookup_elem(&cpus_available, &cpu_idx);
	238	if (!cpu_selected)
	239	return XDP_ABORTED;
	240	cpu_dest = *cpu_selected;
	241
79ccf452	242	rec = bpf_map_lookup_elem(&rx_cnt, &key);
fad3917e	243	if (!rec)
79ccf452 KKD	244	return XDP_PASS;
79ccf452 KKD	245	NO_TEAR_INC(rec->processed);
fad3917e	246
79ccf452 KKD	247	if (cpu_dest >= nr_cpus) {
79ccf452 KKD	248	NO_TEAR_INC(rec->issue);
fad3917e JDB	249	return XDP_ABORTED;
fad3917e JDB	250	}
fad3917e JDB	251	return bpf_redirect_map(&cpu_map, cpu_dest, 0);
	252	}
	253
79ccf452	254	SEC("xdp")
fad3917e JDB	255	int xdp_prognum3_proto_separate(struct xdp_md *ctx)
	256	{
	257	void data_end = (void )(long)ctx->data_end;
	258	void data = (void )(long)ctx->data;
79ccf452	259	u32 key = bpf_get_smp_processor_id();
fad3917e JDB	260	struct ethhdr *eth = data;
	261	u8 ip_proto = IPPROTO_UDP;
	262	struct datarec *rec;
	263	u16 eth_proto = 0;
	264	u64 l3_offset = 0;
	265	u32 cpu_dest = 0;
fad3917e	266	u32 *cpu_lookup;
79ccf452	267	u32 cpu_idx = 0;
fad3917e	268
fad3917e JDB	269	rec = bpf_map_lookup_elem(&rx_cnt, &key);
fad3917e JDB	270	if (!rec)
79ccf452 KKD	271	return XDP_PASS;
79ccf452 KKD	272	NO_TEAR_INC(rec->processed);
fad3917e JDB	273
	274	if (!(parse_eth(eth, data_end, &eth_proto, &l3_offset)))
	275	return XDP_PASS; /* Just skip */
	276
	277	/* Extract L4 protocol */
	278	switch (eth_proto) {
	279	case ETH_P_IP:
	280	ip_proto = get_proto_ipv4(ctx, l3_offset);
	281	break;
	282	case ETH_P_IPV6:
	283	ip_proto = get_proto_ipv6(ctx, l3_offset);
	284	break;
	285	case ETH_P_ARP:
	286	cpu_idx = 0; /* ARP packet handled on separate CPU */
	287	break;
	288	default:
	289	cpu_idx = 0;
	290	}
	291
	292	/* Choose CPU based on L4 protocol */
	293	switch (ip_proto) {
	294	case IPPROTO_ICMP:
	295	case IPPROTO_ICMPV6:
	296	cpu_idx = 2;
	297	break;
	298	case IPPROTO_TCP:
	299	cpu_idx = 0;
	300	break;
	301	case IPPROTO_UDP:
	302	cpu_idx = 1;
	303	break;
	304	default:
	305	cpu_idx = 0;
	306	}
	307
	308	cpu_lookup = bpf_map_lookup_elem(&cpus_available, &cpu_idx);
	309	if (!cpu_lookup)
	310	return XDP_ABORTED;
	311	cpu_dest = *cpu_lookup;
	312
79ccf452 KKD	313	if (cpu_dest >= nr_cpus) {
79ccf452 KKD	314	NO_TEAR_INC(rec->issue);
fad3917e JDB	315	return XDP_ABORTED;
fad3917e JDB	316	}
fad3917e JDB	317	return bpf_redirect_map(&cpu_map, cpu_dest, 0);
	318	}
	319
79ccf452	320	SEC("xdp")
fad3917e JDB	321	int xdp_prognum4_ddos_filter_pktgen(struct xdp_md *ctx)
	322	{
	323	void data_end = (void )(long)ctx->data_end;
	324	void data = (void )(long)ctx->data;
79ccf452	325	u32 key = bpf_get_smp_processor_id();
fad3917e JDB	326	struct ethhdr *eth = data;
	327	u8 ip_proto = IPPROTO_UDP;
	328	struct datarec *rec;
	329	u16 eth_proto = 0;
	330	u64 l3_offset = 0;
	331	u32 cpu_dest = 0;
79ccf452	332	u32 *cpu_lookup;
fad3917e JDB	333	u32 cpu_idx = 0;
fad3917e JDB	334	u16 dest_port;
fad3917e	335
fad3917e JDB	336	rec = bpf_map_lookup_elem(&rx_cnt, &key);
fad3917e JDB	337	if (!rec)
79ccf452 KKD	338	return XDP_PASS;
79ccf452 KKD	339	NO_TEAR_INC(rec->processed);
fad3917e JDB	340
	341	if (!(parse_eth(eth, data_end, &eth_proto, &l3_offset)))
	342	return XDP_PASS; /* Just skip */
	343
	344	/* Extract L4 protocol */
	345	switch (eth_proto) {
	346	case ETH_P_IP:
	347	ip_proto = get_proto_ipv4(ctx, l3_offset);
	348	break;
	349	case ETH_P_IPV6:
	350	ip_proto = get_proto_ipv6(ctx, l3_offset);
	351	break;
	352	case ETH_P_ARP:
	353	cpu_idx = 0; /* ARP packet handled on separate CPU */
	354	break;
	355	default:
	356	cpu_idx = 0;
	357	}
	358
	359	/* Choose CPU based on L4 protocol */
	360	switch (ip_proto) {
	361	case IPPROTO_ICMP:
	362	case IPPROTO_ICMPV6:
	363	cpu_idx = 2;
	364	break;
	365	case IPPROTO_TCP:
	366	cpu_idx = 0;
	367	break;
	368	case IPPROTO_UDP:
	369	cpu_idx = 1;
	370	/* DDoS filter UDP port 9 (pktgen) */
	371	dest_port = get_dest_port_ipv4_udp(ctx, l3_offset);
	372	if (dest_port == 9) {
79ccf452	373	NO_TEAR_INC(rec->dropped);
fad3917e JDB	374	return XDP_DROP;
	375	}
	376	break;
	377	default:
	378	cpu_idx = 0;
	379	}
	380
	381	cpu_lookup = bpf_map_lookup_elem(&cpus_available, &cpu_idx);
	382	if (!cpu_lookup)
	383	return XDP_ABORTED;
	384	cpu_dest = *cpu_lookup;
	385
79ccf452 KKD	386	if (cpu_dest >= nr_cpus) {
79ccf452 KKD	387	NO_TEAR_INC(rec->issue);
fad3917e JDB	388	return XDP_ABORTED;
fad3917e JDB	389	}
fad3917e JDB	390	return bpf_redirect_map(&cpu_map, cpu_dest, 0);
	391	}
	392
1bca4e6b JDB	393	/* Hashing initval */
	394	#define INITVAL 15485863
	395
	396	static __always_inline
	397	u32 get_ipv4_hash_ip_pair(struct xdp_md *ctx, u64 nh_off)
	398	{
	399	void data_end = (void )(long)ctx->data_end;
	400	void data = (void )(long)ctx->data;
	401	struct iphdr *iph = data + nh_off;
	402	u32 cpu_hash;
	403
	404	if (iph + 1 > data_end)
	405	return 0;
	406
	407	cpu_hash = iph->saddr + iph->daddr;
	408	cpu_hash = SuperFastHash((char *)&cpu_hash, 4, INITVAL + iph->protocol);
	409
	410	return cpu_hash;
	411	}
	412
	413	static __always_inline
	414	u32 get_ipv6_hash_ip_pair(struct xdp_md *ctx, u64 nh_off)
	415	{
	416	void data_end = (void )(long)ctx->data_end;
	417	void data = (void )(long)ctx->data;
	418	struct ipv6hdr *ip6h = data + nh_off;
	419	u32 cpu_hash;
	420
	421	if (ip6h + 1 > data_end)
	422	return 0;
	423
79ccf452 KKD	424	cpu_hash = ip6h->saddr.in6_u.u6_addr32[0] + ip6h->daddr.in6_u.u6_addr32[0];
	425	cpu_hash += ip6h->saddr.in6_u.u6_addr32[1] + ip6h->daddr.in6_u.u6_addr32[1];
	426	cpu_hash += ip6h->saddr.in6_u.u6_addr32[2] + ip6h->daddr.in6_u.u6_addr32[2];
	427	cpu_hash += ip6h->saddr.in6_u.u6_addr32[3] + ip6h->daddr.in6_u.u6_addr32[3];
1bca4e6b JDB	428	cpu_hash = SuperFastHash((char *)&cpu_hash, 4, INITVAL + ip6h->nexthdr);
	429
	430	return cpu_hash;
	431	}
	432
	433	/* Load-Balance traffic based on hashing IP-addrs + L4-proto. The
	434	* hashing scheme is symmetric, meaning swapping IP src/dest still hit
	435	* same CPU.
	436	*/
79ccf452	437	SEC("xdp")
1bca4e6b JDB	438	int xdp_prognum5_lb_hash_ip_pairs(struct xdp_md *ctx)
	439	{
	440	void data_end = (void )(long)ctx->data_end;
	441	void data = (void )(long)ctx->data;
79ccf452	442	u32 key = bpf_get_smp_processor_id();
1bca4e6b	443	struct ethhdr *eth = data;
1bca4e6b JDB	444	struct datarec *rec;
	445	u16 eth_proto = 0;
	446	u64 l3_offset = 0;
	447	u32 cpu_dest = 0;
	448	u32 cpu_idx = 0;
	449	u32 *cpu_lookup;
79ccf452	450	u32 key0 = 0;
1bca4e6b JDB	451	u32 *cpu_max;
1bca4e6b JDB	452	u32 cpu_hash;
1bca4e6b	453
1bca4e6b JDB	454	rec = bpf_map_lookup_elem(&rx_cnt, &key);
1bca4e6b JDB	455	if (!rec)
79ccf452 KKD	456	return XDP_PASS;
79ccf452 KKD	457	NO_TEAR_INC(rec->processed);
1bca4e6b	458
79ccf452	459	cpu_max = bpf_map_lookup_elem(&cpus_count, &key0);
1bca4e6b JDB	460	if (!cpu_max)
	461	return XDP_ABORTED;
	462
	463	if (!(parse_eth(eth, data_end, &eth_proto, &l3_offset)))
	464	return XDP_PASS; /* Just skip */
	465
	466	/* Hash for IPv4 and IPv6 */
	467	switch (eth_proto) {
	468	case ETH_P_IP:
	469	cpu_hash = get_ipv4_hash_ip_pair(ctx, l3_offset);
	470	break;
	471	case ETH_P_IPV6:
	472	cpu_hash = get_ipv6_hash_ip_pair(ctx, l3_offset);
	473	break;
	474	case ETH_P_ARP: /* ARP packet handled on CPU idx 0 */
	475	default:
	476	cpu_hash = 0;
	477	}
	478
	479	/* Choose CPU based on hash */
	480	cpu_idx = cpu_hash % *cpu_max;
	481
	482	cpu_lookup = bpf_map_lookup_elem(&cpus_available, &cpu_idx);
	483	if (!cpu_lookup)
	484	return XDP_ABORTED;
	485	cpu_dest = *cpu_lookup;
	486
79ccf452 KKD	487	if (cpu_dest >= nr_cpus) {
79ccf452 KKD	488	NO_TEAR_INC(rec->issue);
1bca4e6b JDB	489	return XDP_ABORTED;
1bca4e6b JDB	490	}
1bca4e6b JDB	491	return bpf_redirect_map(&cpu_map, cpu_dest, 0);
1bca4e6b JDB	492	}
fad3917e	493
8bab5322	494	SEC("xdp/cpumap")
79ccf452	495	int xdp_redirect_cpu_devmap(struct xdp_md *ctx)
fad3917e	496	{
79ccf452 KKD	497	void data_end = (void )(long)ctx->data_end;
	498	void data = (void )(long)ctx->data;
	499	struct ethhdr *eth = data;
	500	u64 nh_off;
fad3917e	501
79ccf452 KKD	502	nh_off = sizeof(*eth);
	503	if (data + nh_off > data_end)
	504	return XDP_DROP;
fad3917e	505
79ccf452 KKD	506	swap_src_dst_mac(data);
79ccf452 KKD	507	return bpf_redirect_map(&tx_port, 0, 0);
fad3917e JDB	508	}
fad3917e JDB	509
8bab5322	510	SEC("xdp/cpumap")
79ccf452	511	int xdp_redirect_cpu_pass(struct xdp_md *ctx)
fad3917e	512	{
79ccf452	513	return XDP_PASS;
fad3917e JDB	514	}
fad3917e JDB	515
8bab5322	516	SEC("xdp/cpumap")
79ccf452	517	int xdp_redirect_cpu_drop(struct xdp_md *ctx)
fad3917e	518	{
79ccf452	519	return XDP_DROP;
fad3917e JDB	520	}
fad3917e JDB	521
8bab5322	522	SEC("xdp/devmap")
79ccf452	523	int xdp_redirect_egress_prog(struct xdp_md *ctx)
fad3917e	524	{
79ccf452 KKD	525	void data_end = (void )(long)ctx->data_end;
	526	void data = (void )(long)ctx->data;
	527	struct ethhdr *eth = data;
	528	u64 nh_off;
fad3917e	529
79ccf452 KKD	530	nh_off = sizeof(*eth);
	531	if (data + nh_off > data_end)
	532	return XDP_DROP;
fad3917e	533
79ccf452	534	__builtin_memcpy(eth->h_source, (const char *)tx_mac_addr, ETH_ALEN);
fad3917e	535
79ccf452	536	return XDP_PASS;
fad3917e JDB	537	}
fad3917e JDB	538
79ccf452	539	char _license[] SEC("license") = "GPL";