projects / linux-2.6-block.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Merge tag 'pm-6.16-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm
[linux-2.6-block.git] / drivers / net / netdevsim / netdev.c
... / ...
CommitLineData
1/*
2 * Copyright (C) 2017 Netronome Systems, Inc.
3 *
4 * This software is licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this
6 * source tree.
7 *
8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
14 */
15
16#include <linux/debugfs.h>
17#include <linux/etherdevice.h>
18#include <linux/ethtool_netlink.h>
19#include <linux/kernel.h>
20#include <linux/module.h>
21#include <linux/netdevice.h>
22#include <linux/slab.h>
23#include <net/netdev_queues.h>
24#include <net/netdev_rx_queue.h>
25#include <net/page_pool/helpers.h>
26#include <net/netlink.h>
27#include <net/net_shaper.h>
28#include <net/netdev_lock.h>
29#include <net/pkt_cls.h>
30#include <net/rtnetlink.h>
31#include <net/udp_tunnel.h>
32#include <net/busy_poll.h>
33
34#include "netdevsim.h"
35
36MODULE_IMPORT_NS("NETDEV_INTERNAL");
37
38#define NSIM_RING_SIZE 256
39
40static int nsim_napi_rx(struct nsim_rq *rq, struct sk_buff *skb)
41{
42 if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) {
43 dev_kfree_skb_any(skb);
44 return NET_RX_DROP;
45 }
46
47 skb_queue_tail(&rq->skb_queue, skb);
48 return NET_RX_SUCCESS;
49}
50
51static int nsim_forward_skb(struct net_device *dev, struct sk_buff *skb,
52 struct nsim_rq *rq)
53{
54 return __dev_forward_skb(dev, skb) ?: nsim_napi_rx(rq, skb);
55}
56
57static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev)
58{
59 struct netdevsim *ns = netdev_priv(dev);
60 struct net_device *peer_dev;
61 unsigned int len = skb->len;
62 struct netdevsim *peer_ns;
63 struct netdev_config *cfg;
64 struct nsim_rq *rq;
65 int rxq;
66
67 rcu_read_lock();
68 if (!nsim_ipsec_tx(ns, skb))
69 goto out_drop_free;
70
71 peer_ns = rcu_dereference(ns->peer);
72 if (!peer_ns)
73 goto out_drop_free;
74
75 peer_dev = peer_ns->netdev;
76 rxq = skb_get_queue_mapping(skb);
77 if (rxq >= peer_dev->num_rx_queues)
78 rxq = rxq % peer_dev->num_rx_queues;
79 rq = peer_ns->rq[rxq];
80
81 cfg = peer_dev->cfg;
82 if (skb_is_nonlinear(skb) &&
83 (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED ||
84 (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED &&
85 cfg->hds_thresh > len)))
86 skb_linearize(skb);
87
88 skb_tx_timestamp(skb);
89 if (unlikely(nsim_forward_skb(peer_dev, skb, rq) == NET_RX_DROP))
90 goto out_drop_cnt;
91
92 if (!hrtimer_active(&rq->napi_timer))
93 hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL);
94
95 rcu_read_unlock();
96 u64_stats_update_begin(&ns->syncp);
97 ns->tx_packets++;
98 ns->tx_bytes += len;
99 u64_stats_update_end(&ns->syncp);
100 return NETDEV_TX_OK;
101
102out_drop_free:
103 dev_kfree_skb(skb);
104out_drop_cnt:
105 rcu_read_unlock();
106 u64_stats_update_begin(&ns->syncp);
107 ns->tx_dropped++;
108 u64_stats_update_end(&ns->syncp);
109 return NETDEV_TX_OK;
110}
111
112static void nsim_set_rx_mode(struct net_device *dev)
113{
114}
115
116static int nsim_change_mtu(struct net_device *dev, int new_mtu)
117{
118 struct netdevsim *ns = netdev_priv(dev);
119
120 if (ns->xdp.prog && !ns->xdp.prog->aux->xdp_has_frags &&
121 new_mtu > NSIM_XDP_MAX_MTU)
122 return -EBUSY;
123
124 WRITE_ONCE(dev->mtu, new_mtu);
125
126 return 0;
127}
128
129static void
130nsim_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
131{
132 struct netdevsim *ns = netdev_priv(dev);
133 unsigned int start;
134
135 do {
136 start = u64_stats_fetch_begin(&ns->syncp);
137 stats->tx_bytes = ns->tx_bytes;
138 stats->tx_packets = ns->tx_packets;
139 stats->tx_dropped = ns->tx_dropped;
140 } while (u64_stats_fetch_retry(&ns->syncp, start));
141}
142
143static int
144nsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
145{
146 return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv);
147}
148
149static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
150{
151 struct netdevsim *ns = netdev_priv(dev);
152 struct nsim_dev *nsim_dev = ns->nsim_dev;
153
154 /* Only refuse multicast addresses, zero address can mean unset/any. */
155 if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac))
156 return -EINVAL;
157 memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN);
158
159 return 0;
160}
161
162static int nsim_set_vf_vlan(struct net_device *dev, int vf,
163 u16 vlan, u8 qos, __be16 vlan_proto)
164{
165 struct netdevsim *ns = netdev_priv(dev);
166 struct nsim_dev *nsim_dev = ns->nsim_dev;
167
168 if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7)
169 return -EINVAL;
170
171 nsim_dev->vfconfigs[vf].vlan = vlan;
172 nsim_dev->vfconfigs[vf].qos = qos;
173 nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto;
174
175 return 0;
176}
177
178static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max)
179{
180 struct netdevsim *ns = netdev_priv(dev);
181 struct nsim_dev *nsim_dev = ns->nsim_dev;
182
183 if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) {
184 pr_err("Not supported in switchdev mode. Please use devlink API.\n");
185 return -EOPNOTSUPP;
186 }
187
188 if (vf >= nsim_dev_get_vfs(nsim_dev))
189 return -EINVAL;
190
191 nsim_dev->vfconfigs[vf].min_tx_rate = min;
192 nsim_dev->vfconfigs[vf].max_tx_rate = max;
193
194 return 0;
195}
196
197static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val)
198{
199 struct netdevsim *ns = netdev_priv(dev);
200 struct nsim_dev *nsim_dev = ns->nsim_dev;
201
202 if (vf >= nsim_dev_get_vfs(nsim_dev))
203 return -EINVAL;
204 nsim_dev->vfconfigs[vf].spoofchk_enabled = val;
205
206 return 0;
207}
208
209static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val)
210{
211 struct netdevsim *ns = netdev_priv(dev);
212 struct nsim_dev *nsim_dev = ns->nsim_dev;
213
214 if (vf >= nsim_dev_get_vfs(nsim_dev))
215 return -EINVAL;
216 nsim_dev->vfconfigs[vf].rss_query_enabled = val;
217
218 return 0;
219}
220
221static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val)
222{
223 struct netdevsim *ns = netdev_priv(dev);
224 struct nsim_dev *nsim_dev = ns->nsim_dev;
225
226 if (vf >= nsim_dev_get_vfs(nsim_dev))
227 return -EINVAL;
228 nsim_dev->vfconfigs[vf].trusted = val;
229
230 return 0;
231}
232
233static int
234nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi)
235{
236 struct netdevsim *ns = netdev_priv(dev);
237 struct nsim_dev *nsim_dev = ns->nsim_dev;
238
239 if (vf >= nsim_dev_get_vfs(nsim_dev))
240 return -EINVAL;
241
242 ivi->vf = vf;
243 ivi->linkstate = nsim_dev->vfconfigs[vf].link_state;
244 ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate;
245 ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate;
246 ivi->vlan = nsim_dev->vfconfigs[vf].vlan;
247 ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto;
248 ivi->qos = nsim_dev->vfconfigs[vf].qos;
249 memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN);
250 ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled;
251 ivi->trusted = nsim_dev->vfconfigs[vf].trusted;
252 ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled;
253
254 return 0;
255}
256
257static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state)
258{
259 struct netdevsim *ns = netdev_priv(dev);
260 struct nsim_dev *nsim_dev = ns->nsim_dev;
261
262 if (vf >= nsim_dev_get_vfs(nsim_dev))
263 return -EINVAL;
264
265 switch (state) {
266 case IFLA_VF_LINK_STATE_AUTO:
267 case IFLA_VF_LINK_STATE_ENABLE:
268 case IFLA_VF_LINK_STATE_DISABLE:
269 break;
270 default:
271 return -EINVAL;
272 }
273
274 nsim_dev->vfconfigs[vf].link_state = state;
275
276 return 0;
277}
278
279static void nsim_taprio_stats(struct tc_taprio_qopt_stats *stats)
280{
281 stats->window_drops = 0;
282 stats->tx_overruns = 0;
283}
284
285static int nsim_setup_tc_taprio(struct net_device *dev,
286 struct tc_taprio_qopt_offload *offload)
287{
288 int err = 0;
289
290 switch (offload->cmd) {
291 case TAPRIO_CMD_REPLACE:
292 case TAPRIO_CMD_DESTROY:
293 break;
294 case TAPRIO_CMD_STATS:
295 nsim_taprio_stats(&offload->stats);
296 break;
297 default:
298 err = -EOPNOTSUPP;
299 }
300
301 return err;
302}
303
304static LIST_HEAD(nsim_block_cb_list);
305
306static int
307nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data)
308{
309 struct netdevsim *ns = netdev_priv(dev);
310
311 switch (type) {
312 case TC_SETUP_QDISC_TAPRIO:
313 return nsim_setup_tc_taprio(dev, type_data);
314 case TC_SETUP_BLOCK:
315 return flow_block_cb_setup_simple(type_data,
316 &nsim_block_cb_list,
317 nsim_setup_tc_block_cb,
318 ns, ns, true);
319 default:
320 return -EOPNOTSUPP;
321 }
322}
323
324static int
325nsim_set_features(struct net_device *dev, netdev_features_t features)
326{
327 struct netdevsim *ns = netdev_priv(dev);
328
329 if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC))
330 return nsim_bpf_disable_tc(ns);
331
332 return 0;
333}
334
335static int nsim_get_iflink(const struct net_device *dev)
336{
337 struct netdevsim *nsim, *peer;
338 int iflink;
339
340 nsim = netdev_priv(dev);
341
342 rcu_read_lock();
343 peer = rcu_dereference(nsim->peer);
344 iflink = peer ? READ_ONCE(peer->netdev->ifindex) :
345 READ_ONCE(dev->ifindex);
346 rcu_read_unlock();
347
348 return iflink;
349}
350
351static int nsim_rcv(struct nsim_rq *rq, int budget)
352{
353 struct sk_buff *skb;
354 int i;
355
356 for (i = 0; i < budget; i++) {
357 if (skb_queue_empty(&rq->skb_queue))
358 break;
359
360 skb = skb_dequeue(&rq->skb_queue);
361 skb_mark_napi_id(skb, &rq->napi);
362 netif_receive_skb(skb);
363 }
364
365 return i;
366}
367
368static int nsim_poll(struct napi_struct *napi, int budget)
369{
370 struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi);
371 int done;
372
373 done = nsim_rcv(rq, budget);
374 if (done < budget)
375 napi_complete_done(napi, done);
376
377 return done;
378}
379
380static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi)
381{
382 struct page_pool_params params = {
383 .order = 0,
384 .pool_size = NSIM_RING_SIZE,
385 .nid = NUMA_NO_NODE,
386 .dev = &napi->dev->dev,
387 .napi = napi,
388 .dma_dir = DMA_BIDIRECTIONAL,
389 .netdev = napi->dev,
390 };
391 struct page_pool *pool;
392
393 pool = page_pool_create(&params);
394 if (IS_ERR(pool))
395 return PTR_ERR(pool);
396
397 *p = pool;
398 return 0;
399}
400
401static int nsim_init_napi(struct netdevsim *ns)
402{
403 struct net_device *dev = ns->netdev;
404 struct nsim_rq *rq;
405 int err, i;
406
407 for (i = 0; i < dev->num_rx_queues; i++) {
408 rq = ns->rq[i];
409
410 netif_napi_add_config_locked(dev, &rq->napi, nsim_poll, i);
411 }
412
413 for (i = 0; i < dev->num_rx_queues; i++) {
414 rq = ns->rq[i];
415
416 err = nsim_create_page_pool(&rq->page_pool, &rq->napi);
417 if (err)
418 goto err_pp_destroy;
419 }
420
421 return 0;
422
423err_pp_destroy:
424 while (i--) {
425 page_pool_destroy(ns->rq[i]->page_pool);
426 ns->rq[i]->page_pool = NULL;
427 }
428
429 for (i = 0; i < dev->num_rx_queues; i++)
430 __netif_napi_del_locked(&ns->rq[i]->napi);
431
432 return err;
433}
434
435static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer)
436{
437 struct nsim_rq *rq;
438
439 rq = container_of(timer, struct nsim_rq, napi_timer);
440 napi_schedule(&rq->napi);
441
442 return HRTIMER_NORESTART;
443}
444
445static void nsim_rq_timer_init(struct nsim_rq *rq)
446{
447 hrtimer_setup(&rq->napi_timer, nsim_napi_schedule, CLOCK_MONOTONIC,
448 HRTIMER_MODE_REL);
449}
450
451static void nsim_enable_napi(struct netdevsim *ns)
452{
453 struct net_device *dev = ns->netdev;
454 int i;
455
456 for (i = 0; i < dev->num_rx_queues; i++) {
457 struct nsim_rq *rq = ns->rq[i];
458
459 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi);
460 napi_enable_locked(&rq->napi);
461 }
462}
463
464static int nsim_open(struct net_device *dev)
465{
466 struct netdevsim *ns = netdev_priv(dev);
467 int err;
468
469 netdev_assert_locked(dev);
470
471 err = nsim_init_napi(ns);
472 if (err)
473 return err;
474
475 nsim_enable_napi(ns);
476
477 return 0;
478}
479
480static void nsim_del_napi(struct netdevsim *ns)
481{
482 struct net_device *dev = ns->netdev;
483 int i;
484
485 for (i = 0; i < dev->num_rx_queues; i++) {
486 struct nsim_rq *rq = ns->rq[i];
487
488 napi_disable_locked(&rq->napi);
489 __netif_napi_del_locked(&rq->napi);
490 }
491 synchronize_net();
492
493 for (i = 0; i < dev->num_rx_queues; i++) {
494 page_pool_destroy(ns->rq[i]->page_pool);
495 ns->rq[i]->page_pool = NULL;
496 }
497}
498
499static int nsim_stop(struct net_device *dev)
500{
501 struct netdevsim *ns = netdev_priv(dev);
502 struct netdevsim *peer;
503
504 netdev_assert_locked(dev);
505
506 netif_carrier_off(dev);
507 peer = rtnl_dereference(ns->peer);
508 if (peer)
509 netif_carrier_off(peer->netdev);
510
511 nsim_del_napi(ns);
512
513 return 0;
514}
515
516static int nsim_shaper_set(struct net_shaper_binding *binding,
517 const struct net_shaper *shaper,
518 struct netlink_ext_ack *extack)
519{
520 return 0;
521}
522
523static int nsim_shaper_del(struct net_shaper_binding *binding,
524 const struct net_shaper_handle *handle,
525 struct netlink_ext_ack *extack)
526{
527 return 0;
528}
529
530static int nsim_shaper_group(struct net_shaper_binding *binding,
531 int leaves_count,
532 const struct net_shaper *leaves,
533 const struct net_shaper *root,
534 struct netlink_ext_ack *extack)
535{
536 return 0;
537}
538
539static void nsim_shaper_cap(struct net_shaper_binding *binding,
540 enum net_shaper_scope scope,
541 unsigned long *flags)
542{
543 *flags = ULONG_MAX;
544}
545
546static const struct net_shaper_ops nsim_shaper_ops = {
547 .set = nsim_shaper_set,
548 .delete = nsim_shaper_del,
549 .group = nsim_shaper_group,
550 .capabilities = nsim_shaper_cap,
551};
552
553static const struct net_device_ops nsim_netdev_ops = {
554 .ndo_start_xmit = nsim_start_xmit,
555 .ndo_set_rx_mode = nsim_set_rx_mode,
556 .ndo_set_mac_address = eth_mac_addr,
557 .ndo_validate_addr = eth_validate_addr,
558 .ndo_change_mtu = nsim_change_mtu,
559 .ndo_get_stats64 = nsim_get_stats64,
560 .ndo_set_vf_mac = nsim_set_vf_mac,
561 .ndo_set_vf_vlan = nsim_set_vf_vlan,
562 .ndo_set_vf_rate = nsim_set_vf_rate,
563 .ndo_set_vf_spoofchk = nsim_set_vf_spoofchk,
564 .ndo_set_vf_trust = nsim_set_vf_trust,
565 .ndo_get_vf_config = nsim_get_vf_config,
566 .ndo_set_vf_link_state = nsim_set_vf_link_state,
567 .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en,
568 .ndo_setup_tc = nsim_setup_tc,
569 .ndo_set_features = nsim_set_features,
570 .ndo_get_iflink = nsim_get_iflink,
571 .ndo_bpf = nsim_bpf,
572 .ndo_open = nsim_open,
573 .ndo_stop = nsim_stop,
574 .net_shaper_ops = &nsim_shaper_ops,
575};
576
577static const struct net_device_ops nsim_vf_netdev_ops = {
578 .ndo_start_xmit = nsim_start_xmit,
579 .ndo_set_rx_mode = nsim_set_rx_mode,
580 .ndo_set_mac_address = eth_mac_addr,
581 .ndo_validate_addr = eth_validate_addr,
582 .ndo_change_mtu = nsim_change_mtu,
583 .ndo_get_stats64 = nsim_get_stats64,
584 .ndo_setup_tc = nsim_setup_tc,
585 .ndo_set_features = nsim_set_features,
586};
587
588/* We don't have true per-queue stats, yet, so do some random fakery here.
589 * Only report stuff for queue 0.
590 */
591static void nsim_get_queue_stats_rx(struct net_device *dev, int idx,
592 struct netdev_queue_stats_rx *stats)
593{
594 struct rtnl_link_stats64 rtstats = {};
595
596 if (!idx)
597 nsim_get_stats64(dev, &rtstats);
598
599 stats->packets = rtstats.rx_packets - !!rtstats.rx_packets;
600 stats->bytes = rtstats.rx_bytes;
601}
602
603static void nsim_get_queue_stats_tx(struct net_device *dev, int idx,
604 struct netdev_queue_stats_tx *stats)
605{
606 struct rtnl_link_stats64 rtstats = {};
607
608 if (!idx)
609 nsim_get_stats64(dev, &rtstats);
610
611 stats->packets = rtstats.tx_packets - !!rtstats.tx_packets;
612 stats->bytes = rtstats.tx_bytes;
613}
614
615static void nsim_get_base_stats(struct net_device *dev,
616 struct netdev_queue_stats_rx *rx,
617 struct netdev_queue_stats_tx *tx)
618{
619 struct rtnl_link_stats64 rtstats = {};
620
621 nsim_get_stats64(dev, &rtstats);
622
623 rx->packets = !!rtstats.rx_packets;
624 rx->bytes = 0;
625 tx->packets = !!rtstats.tx_packets;
626 tx->bytes = 0;
627}
628
629static const struct netdev_stat_ops nsim_stat_ops = {
630 .get_queue_stats_tx = nsim_get_queue_stats_tx,
631 .get_queue_stats_rx = nsim_get_queue_stats_rx,
632 .get_base_stats = nsim_get_base_stats,
633};
634
635static struct nsim_rq *nsim_queue_alloc(void)
636{
637 struct nsim_rq *rq;
638
639 rq = kzalloc(sizeof(*rq), GFP_KERNEL_ACCOUNT);
640 if (!rq)
641 return NULL;
642
643 skb_queue_head_init(&rq->skb_queue);
644 nsim_rq_timer_init(rq);
645 return rq;
646}
647
648static void nsim_queue_free(struct nsim_rq *rq)
649{
650 hrtimer_cancel(&rq->napi_timer);
651 skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE);
652 kfree(rq);
653}
654
655/* Queue reset mode is controlled by ns->rq_reset_mode.
656 * - normal - new NAPI new pool (old NAPI enabled when new added)
657 * - mode 1 - allocate new pool (NAPI is only disabled / enabled)
658 * - mode 2 - new NAPI new pool (old NAPI removed before new added)
659 * - mode 3 - new NAPI new pool (old NAPI disabled when new added)
660 */
661struct nsim_queue_mem {
662 struct nsim_rq *rq;
663 struct page_pool *pp;
664};
665
666static int
667nsim_queue_mem_alloc(struct net_device *dev, void *per_queue_mem, int idx)
668{
669 struct nsim_queue_mem *qmem = per_queue_mem;
670 struct netdevsim *ns = netdev_priv(dev);
671 int err;
672
673 if (ns->rq_reset_mode > 3)
674 return -EINVAL;
675
676 if (ns->rq_reset_mode == 1) {
677 if (!netif_running(ns->netdev))
678 return -ENETDOWN;
679 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi);
680 }
681
682 qmem->rq = nsim_queue_alloc();
683 if (!qmem->rq)
684 return -ENOMEM;
685
686 err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi);
687 if (err)
688 goto err_free;
689
690 if (!ns->rq_reset_mode)
691 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll,
692 idx);
693
694 return 0;
695
696err_free:
697 nsim_queue_free(qmem->rq);
698 return err;
699}
700
701static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem)
702{
703 struct nsim_queue_mem *qmem = per_queue_mem;
704 struct netdevsim *ns = netdev_priv(dev);
705
706 page_pool_destroy(qmem->pp);
707 if (qmem->rq) {
708 if (!ns->rq_reset_mode)
709 netif_napi_del_locked(&qmem->rq->napi);
710 page_pool_destroy(qmem->rq->page_pool);
711 nsim_queue_free(qmem->rq);
712 }
713}
714
715static int
716nsim_queue_start(struct net_device *dev, void *per_queue_mem, int idx)
717{
718 struct nsim_queue_mem *qmem = per_queue_mem;
719 struct netdevsim *ns = netdev_priv(dev);
720
721 netdev_assert_locked(dev);
722
723 if (ns->rq_reset_mode == 1) {
724 ns->rq[idx]->page_pool = qmem->pp;
725 napi_enable_locked(&ns->rq[idx]->napi);
726 return 0;
727 }
728
729 /* netif_napi_add()/_del() should normally be called from alloc/free,
730 * here we want to test various call orders.
731 */
732 if (ns->rq_reset_mode == 2) {
733 netif_napi_del_locked(&ns->rq[idx]->napi);
734 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll,
735 idx);
736 } else if (ns->rq_reset_mode == 3) {
737 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll,
738 idx);
739 netif_napi_del_locked(&ns->rq[idx]->napi);
740 }
741
742 ns->rq[idx] = qmem->rq;
743 napi_enable_locked(&ns->rq[idx]->napi);
744
745 return 0;
746}
747
748static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx)
749{
750 struct nsim_queue_mem *qmem = per_queue_mem;
751 struct netdevsim *ns = netdev_priv(dev);
752
753 netdev_assert_locked(dev);
754
755 napi_disable_locked(&ns->rq[idx]->napi);
756
757 if (ns->rq_reset_mode == 1) {
758 qmem->pp = ns->rq[idx]->page_pool;
759 page_pool_disable_direct_recycling(qmem->pp);
760 } else {
761 qmem->rq = ns->rq[idx];
762 }
763
764 return 0;
765}
766
767static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = {
768 .ndo_queue_mem_size = sizeof(struct nsim_queue_mem),
769 .ndo_queue_mem_alloc = nsim_queue_mem_alloc,
770 .ndo_queue_mem_free = nsim_queue_mem_free,
771 .ndo_queue_start = nsim_queue_start,
772 .ndo_queue_stop = nsim_queue_stop,
773};
774
775static ssize_t
776nsim_qreset_write(struct file *file, const char __user *data,
777 size_t count, loff_t *ppos)
778{
779 struct netdevsim *ns = file->private_data;
780 unsigned int queue, mode;
781 char buf[32];
782 ssize_t ret;
783
784 if (count >= sizeof(buf))
785 return -EINVAL;
786 if (copy_from_user(buf, data, count))
787 return -EFAULT;
788 buf[count] = '\0';
789
790 ret = sscanf(buf, "%u %u", &queue, &mode);
791 if (ret != 2)
792 return -EINVAL;
793
794 netdev_lock(ns->netdev);
795 if (queue >= ns->netdev->real_num_rx_queues) {
796 ret = -EINVAL;
797 goto exit_unlock;
798 }
799
800 ns->rq_reset_mode = mode;
801 ret = netdev_rx_queue_restart(ns->netdev, queue);
802 ns->rq_reset_mode = 0;
803 if (ret)
804 goto exit_unlock;
805
806 ret = count;
807exit_unlock:
808 netdev_unlock(ns->netdev);
809 return ret;
810}
811
812static const struct file_operations nsim_qreset_fops = {
813 .open = simple_open,
814 .write = nsim_qreset_write,
815 .owner = THIS_MODULE,
816};
817
818static ssize_t
819nsim_pp_hold_read(struct file *file, char __user *data,
820 size_t count, loff_t *ppos)
821{
822 struct netdevsim *ns = file->private_data;
823 char buf[3] = "n\n";
824
825 if (ns->page)
826 buf[0] = 'y';
827
828 return simple_read_from_buffer(data, count, ppos, buf, 2);
829}
830
831static ssize_t
832nsim_pp_hold_write(struct file *file, const char __user *data,
833 size_t count, loff_t *ppos)
834{
835 struct netdevsim *ns = file->private_data;
836 ssize_t ret;
837 bool val;
838
839 ret = kstrtobool_from_user(data, count, &val);
840 if (ret)
841 return ret;
842
843 rtnl_lock();
844 ret = count;
845 if (val == !!ns->page)
846 goto exit;
847
848 if (!netif_running(ns->netdev) && val) {
849 ret = -ENETDOWN;
850 } else if (val) {
851 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool);
852 if (!ns->page)
853 ret = -ENOMEM;
854 } else {
855 page_pool_put_full_page(ns->page->pp, ns->page, false);
856 ns->page = NULL;
857 }
858
859exit:
860 rtnl_unlock();
861 return ret;
862}
863
864static const struct file_operations nsim_pp_hold_fops = {
865 .open = simple_open,
866 .read = nsim_pp_hold_read,
867 .write = nsim_pp_hold_write,
868 .llseek = generic_file_llseek,
869 .owner = THIS_MODULE,
870};
871
872static void nsim_setup(struct net_device *dev)
873{
874 ether_setup(dev);
875 eth_hw_addr_random(dev);
876
877 dev->tx_queue_len = 0;
878 dev->flags &= ~IFF_MULTICAST;
879 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE |
880 IFF_NO_QUEUE;
881 dev->features |= NETIF_F_HIGHDMA |
882 NETIF_F_SG |
883 NETIF_F_FRAGLIST |
884 NETIF_F_HW_CSUM |
885 NETIF_F_LRO |
886 NETIF_F_TSO;
887 dev->hw_features |= NETIF_F_HW_TC |
888 NETIF_F_SG |
889 NETIF_F_FRAGLIST |
890 NETIF_F_HW_CSUM |
891 NETIF_F_LRO |
892 NETIF_F_TSO;
893 dev->max_mtu = ETH_MAX_MTU;
894 dev->xdp_features = NETDEV_XDP_ACT_HW_OFFLOAD;
895}
896
897static int nsim_queue_init(struct netdevsim *ns)
898{
899 struct net_device *dev = ns->netdev;
900 int i;
901
902 ns->rq = kcalloc(dev->num_rx_queues, sizeof(*ns->rq),
903 GFP_KERNEL_ACCOUNT);
904 if (!ns->rq)
905 return -ENOMEM;
906
907 for (i = 0; i < dev->num_rx_queues; i++) {
908 ns->rq[i] = nsim_queue_alloc();
909 if (!ns->rq[i])
910 goto err_free_prev;
911 }
912
913 return 0;
914
915err_free_prev:
916 while (i--)
917 kfree(ns->rq[i]);
918 kfree(ns->rq);
919 return -ENOMEM;
920}
921
922static void nsim_queue_uninit(struct netdevsim *ns)
923{
924 struct net_device *dev = ns->netdev;
925 int i;
926
927 for (i = 0; i < dev->num_rx_queues; i++)
928 nsim_queue_free(ns->rq[i]);
929
930 kfree(ns->rq);
931 ns->rq = NULL;
932}
933
934static int nsim_init_netdevsim(struct netdevsim *ns)
935{
936 struct mock_phc *phc;
937 int err;
938
939 phc = mock_phc_create(&ns->nsim_bus_dev->dev);
940 if (IS_ERR(phc))
941 return PTR_ERR(phc);
942
943 ns->phc = phc;
944 ns->netdev->netdev_ops = &nsim_netdev_ops;
945 ns->netdev->stat_ops = &nsim_stat_ops;
946 ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops;
947 netdev_lockdep_set_classes(ns->netdev);
948
949 err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev);
950 if (err)
951 goto err_phc_destroy;
952
953 rtnl_lock();
954 err = nsim_queue_init(ns);
955 if (err)
956 goto err_utn_destroy;
957
958 err = nsim_bpf_init(ns);
959 if (err)
960 goto err_rq_destroy;
961
962 nsim_macsec_init(ns);
963 nsim_ipsec_init(ns);
964
965 err = register_netdevice(ns->netdev);
966 if (err)
967 goto err_ipsec_teardown;
968 rtnl_unlock();
969
970 if (IS_ENABLED(CONFIG_DEBUG_NET)) {
971 ns->nb.notifier_call = netdev_debug_event;
972 if (register_netdevice_notifier_dev_net(ns->netdev, &ns->nb,
973 &ns->nn))
974 ns->nb.notifier_call = NULL;
975 }
976
977 return 0;
978
979err_ipsec_teardown:
980 nsim_ipsec_teardown(ns);
981 nsim_macsec_teardown(ns);
982 nsim_bpf_uninit(ns);
983err_rq_destroy:
984 nsim_queue_uninit(ns);
985err_utn_destroy:
986 rtnl_unlock();
987 nsim_udp_tunnels_info_destroy(ns->netdev);
988err_phc_destroy:
989 mock_phc_destroy(ns->phc);
990 return err;
991}
992
993static int nsim_init_netdevsim_vf(struct netdevsim *ns)
994{
995 int err;
996
997 ns->netdev->netdev_ops = &nsim_vf_netdev_ops;
998 rtnl_lock();
999 err = register_netdevice(ns->netdev);
1000 rtnl_unlock();
1001 return err;
1002}
1003
1004static void nsim_exit_netdevsim(struct netdevsim *ns)
1005{
1006 nsim_udp_tunnels_info_destroy(ns->netdev);
1007 mock_phc_destroy(ns->phc);
1008}
1009
1010struct netdevsim *
1011nsim_create(struct nsim_dev *nsim_dev, struct nsim_dev_port *nsim_dev_port)
1012{
1013 struct net_device *dev;
1014 struct netdevsim *ns;
1015 int err;
1016
1017 dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup,
1018 nsim_dev->nsim_bus_dev->num_queues);
1019 if (!dev)
1020 return ERR_PTR(-ENOMEM);
1021
1022 dev_net_set(dev, nsim_dev_net(nsim_dev));
1023 ns = netdev_priv(dev);
1024 ns->netdev = dev;
1025 u64_stats_init(&ns->syncp);
1026 ns->nsim_dev = nsim_dev;
1027 ns->nsim_dev_port = nsim_dev_port;
1028 ns->nsim_bus_dev = nsim_dev->nsim_bus_dev;
1029 SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev);
1030 SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port);
1031 nsim_ethtool_init(ns);
1032 if (nsim_dev_port_is_pf(nsim_dev_port))
1033 err = nsim_init_netdevsim(ns);
1034 else
1035 err = nsim_init_netdevsim_vf(ns);
1036 if (err)
1037 goto err_free_netdev;
1038
1039 ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir,
1040 ns, &nsim_pp_hold_fops);
1041 ns->qr_dfs = debugfs_create_file("queue_reset", 0200,
1042 nsim_dev_port->ddir, ns,
1043 &nsim_qreset_fops);
1044
1045 return ns;
1046
1047err_free_netdev:
1048 free_netdev(dev);
1049 return ERR_PTR(err);
1050}
1051
1052void nsim_destroy(struct netdevsim *ns)
1053{
1054 struct net_device *dev = ns->netdev;
1055 struct netdevsim *peer;
1056
1057 debugfs_remove(ns->qr_dfs);
1058 debugfs_remove(ns->pp_dfs);
1059
1060 if (ns->nb.notifier_call)
1061 unregister_netdevice_notifier_dev_net(ns->netdev, &ns->nb,
1062 &ns->nn);
1063
1064 rtnl_lock();
1065 peer = rtnl_dereference(ns->peer);
1066 if (peer)
1067 RCU_INIT_POINTER(peer->peer, NULL);
1068 RCU_INIT_POINTER(ns->peer, NULL);
1069 unregister_netdevice(dev);
1070 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) {
1071 nsim_macsec_teardown(ns);
1072 nsim_ipsec_teardown(ns);
1073 nsim_bpf_uninit(ns);
1074 nsim_queue_uninit(ns);
1075 }
1076 rtnl_unlock();
1077 if (nsim_dev_port_is_pf(ns->nsim_dev_port))
1078 nsim_exit_netdevsim(ns);
1079
1080 /* Put this intentionally late to exercise the orphaning path */
1081 if (ns->page) {
1082 page_pool_put_full_page(ns->page->pp, ns->page, false);
1083 ns->page = NULL;
1084 }
1085
1086 free_netdev(dev);
1087}
1088
1089bool netdev_is_nsim(struct net_device *dev)
1090{
1091 return dev->netdev_ops == &nsim_netdev_ops;
1092}
1093
1094static int nsim_validate(struct nlattr *tb[], struct nlattr *data[],
1095 struct netlink_ext_ack *extack)
1096{
1097 NL_SET_ERR_MSG_MOD(extack,
1098 "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device");
1099 return -EOPNOTSUPP;
1100}
1101
1102static struct rtnl_link_ops nsim_link_ops __read_mostly = {
1103 .kind = DRV_NAME,
1104 .validate = nsim_validate,
1105};
1106
1107static int __init nsim_module_init(void)
1108{
1109 int err;
1110
1111 err = nsim_dev_init();
1112 if (err)
1113 return err;
1114
1115 err = nsim_bus_init();
1116 if (err)
1117 goto err_dev_exit;
1118
1119 err = rtnl_link_register(&nsim_link_ops);
1120 if (err)
1121 goto err_bus_exit;
1122
1123 return 0;
1124
1125err_bus_exit:
1126 nsim_bus_exit();
1127err_dev_exit:
1128 nsim_dev_exit();
1129 return err;
1130}
1131
1132static void __exit nsim_module_exit(void)
1133{
1134 rtnl_link_unregister(&nsim_link_ops);
1135 nsim_bus_exit();
1136 nsim_dev_exit();
1137}
1138
1139module_init(nsim_module_init);
1140module_exit(nsim_module_exit);
1141MODULE_LICENSE("GPL");
1142MODULE_DESCRIPTION("Simulated networking device for testing");
1143MODULE_ALIAS_RTNL_LINK(DRV_NAME);
Linux 6.x block layer and io_uring tree(s)