drivers/net/netdevsim/netdev.c

   1 /*
   2  * Copyright (C) 2017 Netronome Systems, Inc.
   3  *
   4  * This software is licensed under the GNU General License Version 2,
   5  * June 1991 as shown in the file COPYING in the top-level directory of this
   6  * source tree.
   7  *
   8  * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
   9  * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
  10  * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  11  * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
  12  * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
  13  * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
  14  */
  15
  16 #include <linux/debugfs.h>
  17 #include <linux/etherdevice.h>
  18 #include <linux/ethtool_netlink.h>
  19 #include <linux/kernel.h>
  20 #include <linux/module.h>
  21 #include <linux/netdevice.h>
  22 #include <linux/slab.h>
  23 #include <net/netdev_queues.h>
  24 #include <net/netdev_rx_queue.h>
  25 #include <net/page_pool/helpers.h>
  26 #include <net/netlink.h>
  27 #include <net/net_shaper.h>
  28 #include <net/netdev_lock.h>
  29 #include <net/pkt_cls.h>
  30 #include <net/rtnetlink.h>
  31 #include <net/udp_tunnel.h>
  32 #include <net/busy_poll.h>
  33
  34 #include "netdevsim.h"
  35
  36 MODULE_IMPORT_NS("NETDEV_INTERNAL");
  37
  38 #define NSIM_RING_SIZE          256
  39
  40 static int nsim_napi_rx(struct nsim_rq *rq, struct sk_buff *skb)
  41 {
  42         if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) {
  43                 dev_kfree_skb_any(skb);
  44                 return NET_RX_DROP;
  45         }
  46
  47         skb_queue_tail(&rq->skb_queue, skb);
  48         return NET_RX_SUCCESS;
  49 }
  50
  51 static int nsim_forward_skb(struct net_device *dev, struct sk_buff *skb,
  52                             struct nsim_rq *rq)
  53 {
  54         return __dev_forward_skb(dev, skb) ?: nsim_napi_rx(rq, skb);
  55 }
  56
  57 static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev)
  58 {
  59         struct netdevsim *ns = netdev_priv(dev);
  60         struct net_device *peer_dev;
  61         unsigned int len = skb->len;
  62         struct netdevsim *peer_ns;
  63         struct netdev_config *cfg;
  64         struct nsim_rq *rq;
  65         int rxq;
  66
  67         rcu_read_lock();
  68         if (!nsim_ipsec_tx(ns, skb))
  69                 goto out_drop_free;
  70
  71         peer_ns = rcu_dereference(ns->peer);
  72         if (!peer_ns)
  73                 goto out_drop_free;
  74
  75         peer_dev = peer_ns->netdev;
  76         rxq = skb_get_queue_mapping(skb);
  77         if (rxq >= peer_dev->num_rx_queues)
  78                 rxq = rxq % peer_dev->num_rx_queues;
  79         rq = peer_ns->rq[rxq];
  80
  81         cfg = peer_dev->cfg;
  82         if (skb_is_nonlinear(skb) &&
  83             (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED ||
  84              (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED &&
  85               cfg->hds_thresh > len)))
  86                 skb_linearize(skb);
  87
  88         skb_tx_timestamp(skb);
  89         if (unlikely(nsim_forward_skb(peer_dev, skb, rq) == NET_RX_DROP))
  90                 goto out_drop_cnt;
  91
  92         if (!hrtimer_active(&rq->napi_timer))
  93                 hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL);
  94
  95         rcu_read_unlock();
  96         u64_stats_update_begin(&ns->syncp);
  97         ns->tx_packets++;
  98         ns->tx_bytes += len;
  99         u64_stats_update_end(&ns->syncp);
 100         return NETDEV_TX_OK;
 101
 102 out_drop_free:
 103         dev_kfree_skb(skb);
 104 out_drop_cnt:
 105         rcu_read_unlock();
 106         u64_stats_update_begin(&ns->syncp);
 107         ns->tx_dropped++;
 108         u64_stats_update_end(&ns->syncp);
 109         return NETDEV_TX_OK;
 110 }
 111
 112 static void nsim_set_rx_mode(struct net_device *dev)
 113 {
 114 }
 115
 116 static int nsim_change_mtu(struct net_device *dev, int new_mtu)
 117 {
 118         struct netdevsim *ns = netdev_priv(dev);
 119
 120         if (ns->xdp.prog && !ns->xdp.prog->aux->xdp_has_frags &&
 121             new_mtu > NSIM_XDP_MAX_MTU)
 122                 return -EBUSY;
 123
 124         WRITE_ONCE(dev->mtu, new_mtu);
 125
 126         return 0;
 127 }
 128
 129 static void
 130 nsim_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
 131 {
 132         struct netdevsim *ns = netdev_priv(dev);
 133         unsigned int start;
 134
 135         do {
 136                 start = u64_stats_fetch_begin(&ns->syncp);
 137                 stats->tx_bytes = ns->tx_bytes;
 138                 stats->tx_packets = ns->tx_packets;
 139                 stats->tx_dropped = ns->tx_dropped;
 140         } while (u64_stats_fetch_retry(&ns->syncp, start));
 141 }
 142
 143 static int
 144 nsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
 145 {
 146         return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv);
 147 }
 148
 149 static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
 150 {
 151         struct netdevsim *ns = netdev_priv(dev);
 152         struct nsim_dev *nsim_dev = ns->nsim_dev;
 153
 154         /* Only refuse multicast addresses, zero address can mean unset/any. */
 155         if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac))
 156                 return -EINVAL;
 157         memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN);
 158
 159         return 0;
 160 }
 161
 162 static int nsim_set_vf_vlan(struct net_device *dev, int vf,
 163                             u16 vlan, u8 qos, __be16 vlan_proto)
 164 {
 165         struct netdevsim *ns = netdev_priv(dev);
 166         struct nsim_dev *nsim_dev = ns->nsim_dev;
 167
 168         if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7)
 169                 return -EINVAL;
 170
 171         nsim_dev->vfconfigs[vf].vlan = vlan;
 172         nsim_dev->vfconfigs[vf].qos = qos;
 173         nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto;
 174
 175         return 0;
 176 }
 177
 178 static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max)
 179 {
 180         struct netdevsim *ns = netdev_priv(dev);
 181         struct nsim_dev *nsim_dev = ns->nsim_dev;
 182
 183         if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) {
 184                 pr_err("Not supported in switchdev mode. Please use devlink API.\n");
 185                 return -EOPNOTSUPP;
 186         }
 187
 188         if (vf >= nsim_dev_get_vfs(nsim_dev))
 189                 return -EINVAL;
 190
 191         nsim_dev->vfconfigs[vf].min_tx_rate = min;
 192         nsim_dev->vfconfigs[vf].max_tx_rate = max;
 193
 194         return 0;
 195 }
 196
 197 static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val)
 198 {
 199         struct netdevsim *ns = netdev_priv(dev);
 200         struct nsim_dev *nsim_dev = ns->nsim_dev;
 201
 202         if (vf >= nsim_dev_get_vfs(nsim_dev))
 203                 return -EINVAL;
 204         nsim_dev->vfconfigs[vf].spoofchk_enabled = val;
 205
 206         return 0;
 207 }
 208
 209 static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val)
 210 {
 211         struct netdevsim *ns = netdev_priv(dev);
 212         struct nsim_dev *nsim_dev = ns->nsim_dev;
 213
 214         if (vf >= nsim_dev_get_vfs(nsim_dev))
 215                 return -EINVAL;
 216         nsim_dev->vfconfigs[vf].rss_query_enabled = val;
 217
 218         return 0;
 219 }
 220
 221 static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val)
 222 {
 223         struct netdevsim *ns = netdev_priv(dev);
 224         struct nsim_dev *nsim_dev = ns->nsim_dev;
 225
 226         if (vf >= nsim_dev_get_vfs(nsim_dev))
 227                 return -EINVAL;
 228         nsim_dev->vfconfigs[vf].trusted = val;
 229
 230         return 0;
 231 }
 232
 233 static int
 234 nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi)
 235 {
 236         struct netdevsim *ns = netdev_priv(dev);
 237         struct nsim_dev *nsim_dev = ns->nsim_dev;
 238
 239         if (vf >= nsim_dev_get_vfs(nsim_dev))
 240                 return -EINVAL;
 241
 242         ivi->vf = vf;
 243         ivi->linkstate = nsim_dev->vfconfigs[vf].link_state;
 244         ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate;
 245         ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate;
 246         ivi->vlan = nsim_dev->vfconfigs[vf].vlan;
 247         ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto;
 248         ivi->qos = nsim_dev->vfconfigs[vf].qos;
 249         memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN);
 250         ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled;
 251         ivi->trusted = nsim_dev->vfconfigs[vf].trusted;
 252         ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled;
 253
 254         return 0;
 255 }
 256
 257 static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state)
 258 {
 259         struct netdevsim *ns = netdev_priv(dev);
 260         struct nsim_dev *nsim_dev = ns->nsim_dev;
 261
 262         if (vf >= nsim_dev_get_vfs(nsim_dev))
 263                 return -EINVAL;
 264
 265         switch (state) {
 266         case IFLA_VF_LINK_STATE_AUTO:
 267         case IFLA_VF_LINK_STATE_ENABLE:
 268         case IFLA_VF_LINK_STATE_DISABLE:
 269                 break;
 270         default:
 271                 return -EINVAL;
 272         }
 273
 274         nsim_dev->vfconfigs[vf].link_state = state;
 275
 276         return 0;
 277 }
 278
 279 static void nsim_taprio_stats(struct tc_taprio_qopt_stats *stats)
 280 {
 281         stats->window_drops = 0;
 282         stats->tx_overruns = 0;
 283 }
 284
 285 static int nsim_setup_tc_taprio(struct net_device *dev,
 286                                 struct tc_taprio_qopt_offload *offload)
 287 {
 288         int err = 0;
 289
 290         switch (offload->cmd) {
 291         case TAPRIO_CMD_REPLACE:
 292         case TAPRIO_CMD_DESTROY:
 293                 break;
 294         case TAPRIO_CMD_STATS:
 295                 nsim_taprio_stats(&offload->stats);
 296                 break;
 297         default:
 298                 err = -EOPNOTSUPP;
 299         }
 300
 301         return err;
 302 }
 303
 304 static LIST_HEAD(nsim_block_cb_list);
 305
 306 static int
 307 nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data)
 308 {
 309         struct netdevsim *ns = netdev_priv(dev);
 310
 311         switch (type) {
 312         case TC_SETUP_QDISC_TAPRIO:
 313                 return nsim_setup_tc_taprio(dev, type_data);
 314         case TC_SETUP_BLOCK:
 315                 return flow_block_cb_setup_simple(type_data,
 316                                                   &nsim_block_cb_list,
 317                                                   nsim_setup_tc_block_cb,
 318                                                   ns, ns, true);
 319         default:
 320                 return -EOPNOTSUPP;
 321         }
 322 }
 323
 324 static int
 325 nsim_set_features(struct net_device *dev, netdev_features_t features)
 326 {
 327         struct netdevsim *ns = netdev_priv(dev);
 328
 329         if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC))
 330                 return nsim_bpf_disable_tc(ns);
 331
 332         return 0;
 333 }
 334
 335 static int nsim_get_iflink(const struct net_device *dev)
 336 {
 337         struct netdevsim *nsim, *peer;
 338         int iflink;
 339
 340         nsim = netdev_priv(dev);
 341
 342         rcu_read_lock();
 343         peer = rcu_dereference(nsim->peer);
 344         iflink = peer ? READ_ONCE(peer->netdev->ifindex) :
 345                         READ_ONCE(dev->ifindex);
 346         rcu_read_unlock();
 347
 348         return iflink;
 349 }
 350
 351 static int nsim_rcv(struct nsim_rq *rq, int budget)
 352 {
 353         struct sk_buff *skb;
 354         int i;
 355
 356         for (i = 0; i < budget; i++) {
 357                 if (skb_queue_empty(&rq->skb_queue))
 358                         break;
 359
 360                 skb = skb_dequeue(&rq->skb_queue);
 361                 skb_mark_napi_id(skb, &rq->napi);
 362                 netif_receive_skb(skb);
 363         }
 364
 365         return i;
 366 }
 367
 368 static int nsim_poll(struct napi_struct *napi, int budget)
 369 {
 370         struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi);
 371         int done;
 372
 373         done = nsim_rcv(rq, budget);
 374         if (done < budget)
 375                 napi_complete_done(napi, done);
 376
 377         return done;
 378 }
 379
 380 static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi)
 381 {
 382         struct page_pool_params params = {
 383                 .order = 0,
 384                 .pool_size = NSIM_RING_SIZE,
 385                 .nid = NUMA_NO_NODE,
 386                 .dev = &napi->dev->dev,
 387                 .napi = napi,
 388                 .dma_dir = DMA_BIDIRECTIONAL,
 389                 .netdev = napi->dev,
 390         };
 391         struct page_pool *pool;
 392
 393         pool = page_pool_create(&params);
 394         if (IS_ERR(pool))
 395                 return PTR_ERR(pool);
 396
 397         *p = pool;
 398         return 0;
 399 }
 400
 401 static int nsim_init_napi(struct netdevsim *ns)
 402 {
 403         struct net_device *dev = ns->netdev;
 404         struct nsim_rq *rq;
 405         int err, i;
 406
 407         for (i = 0; i < dev->num_rx_queues; i++) {
 408                 rq = ns->rq[i];
 409
 410                 netif_napi_add_config_locked(dev, &rq->napi, nsim_poll, i);
 411         }
 412
 413         for (i = 0; i < dev->num_rx_queues; i++) {
 414                 rq = ns->rq[i];
 415
 416                 err = nsim_create_page_pool(&rq->page_pool, &rq->napi);
 417                 if (err)
 418                         goto err_pp_destroy;
 419         }
 420
 421         return 0;
 422
 423 err_pp_destroy:
 424         while (i--) {
 425                 page_pool_destroy(ns->rq[i]->page_pool);
 426                 ns->rq[i]->page_pool = NULL;
 427         }
 428
 429         for (i = 0; i < dev->num_rx_queues; i++)
 430                 __netif_napi_del_locked(&ns->rq[i]->napi);
 431
 432         return err;
 433 }
 434
 435 static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer)
 436 {
 437         struct nsim_rq *rq;
 438
 439         rq = container_of(timer, struct nsim_rq, napi_timer);
 440         napi_schedule(&rq->napi);
 441
 442         return HRTIMER_NORESTART;
 443 }
 444
 445 static void nsim_rq_timer_init(struct nsim_rq *rq)
 446 {
 447         hrtimer_setup(&rq->napi_timer, nsim_napi_schedule, CLOCK_MONOTONIC,
 448                       HRTIMER_MODE_REL);
 449 }
 450
 451 static void nsim_enable_napi(struct netdevsim *ns)
 452 {
 453         struct net_device *dev = ns->netdev;
 454         int i;
 455
 456         for (i = 0; i < dev->num_rx_queues; i++) {
 457                 struct nsim_rq *rq = ns->rq[i];
 458
 459                 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi);
 460                 napi_enable_locked(&rq->napi);
 461         }
 462 }
 463
 464 static int nsim_open(struct net_device *dev)
 465 {
 466         struct netdevsim *ns = netdev_priv(dev);
 467         int err;
 468
 469         netdev_assert_locked(dev);
 470
 471         err = nsim_init_napi(ns);
 472         if (err)
 473                 return err;
 474
 475         nsim_enable_napi(ns);
 476
 477         return 0;
 478 }
 479
 480 static void nsim_del_napi(struct netdevsim *ns)
 481 {
 482         struct net_device *dev = ns->netdev;
 483         int i;
 484
 485         for (i = 0; i < dev->num_rx_queues; i++) {
 486                 struct nsim_rq *rq = ns->rq[i];
 487
 488                 napi_disable_locked(&rq->napi);
 489                 __netif_napi_del_locked(&rq->napi);
 490         }
 491         synchronize_net();
 492
 493         for (i = 0; i < dev->num_rx_queues; i++) {
 494                 page_pool_destroy(ns->rq[i]->page_pool);
 495                 ns->rq[i]->page_pool = NULL;
 496         }
 497 }
 498
 499 static int nsim_stop(struct net_device *dev)
 500 {
 501         struct netdevsim *ns = netdev_priv(dev);
 502         struct netdevsim *peer;
 503
 504         netdev_assert_locked(dev);
 505
 506         netif_carrier_off(dev);
 507         peer = rtnl_dereference(ns->peer);
 508         if (peer)
 509                 netif_carrier_off(peer->netdev);
 510
 511         nsim_del_napi(ns);
 512
 513         return 0;
 514 }
 515
 516 static int nsim_shaper_set(struct net_shaper_binding *binding,
 517                            const struct net_shaper *shaper,
 518                            struct netlink_ext_ack *extack)
 519 {
 520         return 0;
 521 }
 522
 523 static int nsim_shaper_del(struct net_shaper_binding *binding,
 524                            const struct net_shaper_handle *handle,
 525                            struct netlink_ext_ack *extack)
 526 {
 527         return 0;
 528 }
 529
 530 static int nsim_shaper_group(struct net_shaper_binding *binding,
 531                              int leaves_count,
 532                              const struct net_shaper *leaves,
 533                              const struct net_shaper *root,
 534                              struct netlink_ext_ack *extack)
 535 {
 536         return 0;
 537 }
 538
 539 static void nsim_shaper_cap(struct net_shaper_binding *binding,
 540                             enum net_shaper_scope scope,
 541                             unsigned long *flags)
 542 {
 543         *flags = ULONG_MAX;
 544 }
 545
 546 static const struct net_shaper_ops nsim_shaper_ops = {
 547         .set                    = nsim_shaper_set,
 548         .delete                 = nsim_shaper_del,
 549         .group                  = nsim_shaper_group,
 550         .capabilities           = nsim_shaper_cap,
 551 };
 552
 553 static const struct net_device_ops nsim_netdev_ops = {
 554         .ndo_start_xmit         = nsim_start_xmit,
 555         .ndo_set_rx_mode        = nsim_set_rx_mode,
 556         .ndo_set_mac_address    = eth_mac_addr,
 557         .ndo_validate_addr      = eth_validate_addr,
 558         .ndo_change_mtu         = nsim_change_mtu,
 559         .ndo_get_stats64        = nsim_get_stats64,
 560         .ndo_set_vf_mac         = nsim_set_vf_mac,
 561         .ndo_set_vf_vlan        = nsim_set_vf_vlan,
 562         .ndo_set_vf_rate        = nsim_set_vf_rate,
 563         .ndo_set_vf_spoofchk    = nsim_set_vf_spoofchk,
 564         .ndo_set_vf_trust       = nsim_set_vf_trust,
 565         .ndo_get_vf_config      = nsim_get_vf_config,
 566         .ndo_set_vf_link_state  = nsim_set_vf_link_state,
 567         .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en,
 568         .ndo_setup_tc           = nsim_setup_tc,
 569         .ndo_set_features       = nsim_set_features,
 570         .ndo_get_iflink         = nsim_get_iflink,
 571         .ndo_bpf                = nsim_bpf,
 572         .ndo_open               = nsim_open,
 573         .ndo_stop               = nsim_stop,
 574         .net_shaper_ops         = &nsim_shaper_ops,
 575 };
 576
 577 static const struct net_device_ops nsim_vf_netdev_ops = {
 578         .ndo_start_xmit         = nsim_start_xmit,
 579         .ndo_set_rx_mode        = nsim_set_rx_mode,
 580         .ndo_set_mac_address    = eth_mac_addr,
 581         .ndo_validate_addr      = eth_validate_addr,
 582         .ndo_change_mtu         = nsim_change_mtu,
 583         .ndo_get_stats64        = nsim_get_stats64,
 584         .ndo_setup_tc           = nsim_setup_tc,
 585         .ndo_set_features       = nsim_set_features,
 586 };
 587
 588 /* We don't have true per-queue stats, yet, so do some random fakery here.
 589  * Only report stuff for queue 0.
 590  */
 591 static void nsim_get_queue_stats_rx(struct net_device *dev, int idx,
 592                                     struct netdev_queue_stats_rx *stats)
 593 {
 594         struct rtnl_link_stats64 rtstats = {};
 595
 596         if (!idx)
 597                 nsim_get_stats64(dev, &rtstats);
 598
 599         stats->packets = rtstats.rx_packets - !!rtstats.rx_packets;
 600         stats->bytes = rtstats.rx_bytes;
 601 }
 602
 603 static void nsim_get_queue_stats_tx(struct net_device *dev, int idx,
 604                                     struct netdev_queue_stats_tx *stats)
 605 {
 606         struct rtnl_link_stats64 rtstats = {};
 607
 608         if (!idx)
 609                 nsim_get_stats64(dev, &rtstats);
 610
 611         stats->packets = rtstats.tx_packets - !!rtstats.tx_packets;
 612         stats->bytes = rtstats.tx_bytes;
 613 }
 614
 615 static void nsim_get_base_stats(struct net_device *dev,
 616                                 struct netdev_queue_stats_rx *rx,
 617                                 struct netdev_queue_stats_tx *tx)
 618 {
 619         struct rtnl_link_stats64 rtstats = {};
 620
 621         nsim_get_stats64(dev, &rtstats);
 622
 623         rx->packets = !!rtstats.rx_packets;
 624         rx->bytes = 0;
 625         tx->packets = !!rtstats.tx_packets;
 626         tx->bytes = 0;
 627 }
 628
 629 static const struct netdev_stat_ops nsim_stat_ops = {
 630         .get_queue_stats_tx     = nsim_get_queue_stats_tx,
 631         .get_queue_stats_rx     = nsim_get_queue_stats_rx,
 632         .get_base_stats         = nsim_get_base_stats,
 633 };
 634
 635 static struct nsim_rq *nsim_queue_alloc(void)
 636 {
 637         struct nsim_rq *rq;
 638
 639         rq = kzalloc(sizeof(*rq), GFP_KERNEL_ACCOUNT);
 640         if (!rq)
 641                 return NULL;
 642
 643         skb_queue_head_init(&rq->skb_queue);
 644         nsim_rq_timer_init(rq);
 645         return rq;
 646 }
 647
 648 static void nsim_queue_free(struct nsim_rq *rq)
 649 {
 650         hrtimer_cancel(&rq->napi_timer);
 651         skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE);
 652         kfree(rq);
 653 }
 654
 655 /* Queue reset mode is controlled by ns->rq_reset_mode.
 656  * - normal - new NAPI new pool (old NAPI enabled when new added)
 657  * - mode 1 - allocate new pool (NAPI is only disabled / enabled)
 658  * - mode 2 - new NAPI new pool (old NAPI removed before new added)
 659  * - mode 3 - new NAPI new pool (old NAPI disabled when new added)
 660  */
 661 struct nsim_queue_mem {
 662         struct nsim_rq *rq;
 663         struct page_pool *pp;
 664 };
 665
 666 static int
 667 nsim_queue_mem_alloc(struct net_device *dev, void *per_queue_mem, int idx)
 668 {
 669         struct nsim_queue_mem *qmem = per_queue_mem;
 670         struct netdevsim *ns = netdev_priv(dev);
 671         int err;
 672
 673         if (ns->rq_reset_mode > 3)
 674                 return -EINVAL;
 675
 676         if (ns->rq_reset_mode == 1) {
 677                 if (!netif_running(ns->netdev))
 678                         return -ENETDOWN;
 679                 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi);
 680         }
 681
 682         qmem->rq = nsim_queue_alloc();
 683         if (!qmem->rq)
 684                 return -ENOMEM;
 685
 686         err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi);
 687         if (err)
 688                 goto err_free;
 689
 690         if (!ns->rq_reset_mode)
 691                 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll,
 692                                              idx);
 693
 694         return 0;
 695
 696 err_free:
 697         nsim_queue_free(qmem->rq);
 698         return err;
 699 }
 700
 701 static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem)
 702 {
 703         struct nsim_queue_mem *qmem = per_queue_mem;
 704         struct netdevsim *ns = netdev_priv(dev);
 705
 706         page_pool_destroy(qmem->pp);
 707         if (qmem->rq) {
 708                 if (!ns->rq_reset_mode)
 709                         netif_napi_del_locked(&qmem->rq->napi);
 710                 page_pool_destroy(qmem->rq->page_pool);
 711                 nsim_queue_free(qmem->rq);
 712         }
 713 }
 714
 715 static int
 716 nsim_queue_start(struct net_device *dev, void *per_queue_mem, int idx)
 717 {
 718         struct nsim_queue_mem *qmem = per_queue_mem;
 719         struct netdevsim *ns = netdev_priv(dev);
 720
 721         netdev_assert_locked(dev);
 722
 723         if (ns->rq_reset_mode == 1) {
 724                 ns->rq[idx]->page_pool = qmem->pp;
 725                 napi_enable_locked(&ns->rq[idx]->napi);
 726                 return 0;
 727         }
 728
 729         /* netif_napi_add()/_del() should normally be called from alloc/free,
 730          * here we want to test various call orders.
 731          */
 732         if (ns->rq_reset_mode == 2) {
 733                 netif_napi_del_locked(&ns->rq[idx]->napi);
 734                 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll,
 735                                              idx);
 736         } else if (ns->rq_reset_mode == 3) {
 737                 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll,
 738                                              idx);
 739                 netif_napi_del_locked(&ns->rq[idx]->napi);
 740         }
 741
 742         ns->rq[idx] = qmem->rq;
 743         napi_enable_locked(&ns->rq[idx]->napi);
 744
 745         return 0;
 746 }
 747
 748 static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx)
 749 {
 750         struct nsim_queue_mem *qmem = per_queue_mem;
 751         struct netdevsim *ns = netdev_priv(dev);
 752
 753         netdev_assert_locked(dev);
 754
 755         napi_disable_locked(&ns->rq[idx]->napi);
 756
 757         if (ns->rq_reset_mode == 1) {
 758                 qmem->pp = ns->rq[idx]->page_pool;
 759                 page_pool_disable_direct_recycling(qmem->pp);
 760         } else {
 761                 qmem->rq = ns->rq[idx];
 762         }
 763
 764         return 0;
 765 }
 766
 767 static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = {
 768         .ndo_queue_mem_size     = sizeof(struct nsim_queue_mem),
 769         .ndo_queue_mem_alloc    = nsim_queue_mem_alloc,
 770         .ndo_queue_mem_free     = nsim_queue_mem_free,
 771         .ndo_queue_start        = nsim_queue_start,
 772         .ndo_queue_stop         = nsim_queue_stop,
 773 };
 774
 775 static ssize_t
 776 nsim_qreset_write(struct file *file, const char __user *data,
 777                   size_t count, loff_t *ppos)
 778 {
 779         struct netdevsim *ns = file->private_data;
 780         unsigned int queue, mode;
 781         char buf[32];
 782         ssize_t ret;
 783
 784         if (count >= sizeof(buf))
 785                 return -EINVAL;
 786         if (copy_from_user(buf, data, count))
 787                 return -EFAULT;
 788         buf[count] = '\0';
 789
 790         ret = sscanf(buf, "%u %u", &queue, &mode);
 791         if (ret != 2)
 792                 return -EINVAL;
 793
 794         netdev_lock(ns->netdev);
 795         if (queue >= ns->netdev->real_num_rx_queues) {
 796                 ret = -EINVAL;
 797                 goto exit_unlock;
 798         }
 799
 800         ns->rq_reset_mode = mode;
 801         ret = netdev_rx_queue_restart(ns->netdev, queue);
 802         ns->rq_reset_mode = 0;
 803         if (ret)
 804                 goto exit_unlock;
 805
 806         ret = count;
 807 exit_unlock:
 808         netdev_unlock(ns->netdev);
 809         return ret;
 810 }
 811
 812 static const struct file_operations nsim_qreset_fops = {
 813         .open = simple_open,
 814         .write = nsim_qreset_write,
 815         .owner = THIS_MODULE,
 816 };
 817
 818 static ssize_t
 819 nsim_pp_hold_read(struct file *file, char __user *data,
 820                   size_t count, loff_t *ppos)
 821 {
 822         struct netdevsim *ns = file->private_data;
 823         char buf[3] = "n\n";
 824
 825         if (ns->page)
 826                 buf[0] = 'y';
 827
 828         return simple_read_from_buffer(data, count, ppos, buf, 2);
 829 }
 830
 831 static ssize_t
 832 nsim_pp_hold_write(struct file *file, const char __user *data,
 833                    size_t count, loff_t *ppos)
 834 {
 835         struct netdevsim *ns = file->private_data;
 836         ssize_t ret;
 837         bool val;
 838
 839         ret = kstrtobool_from_user(data, count, &val);
 840         if (ret)
 841                 return ret;
 842
 843         rtnl_lock();
 844         ret = count;
 845         if (val == !!ns->page)
 846                 goto exit;
 847
 848         if (!netif_running(ns->netdev) && val) {
 849                 ret = -ENETDOWN;
 850         } else if (val) {
 851                 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool);
 852                 if (!ns->page)
 853                         ret = -ENOMEM;
 854         } else {
 855                 page_pool_put_full_page(ns->page->pp, ns->page, false);
 856                 ns->page = NULL;
 857         }
 858
 859 exit:
 860         rtnl_unlock();
 861         return ret;
 862 }
 863
 864 static const struct file_operations nsim_pp_hold_fops = {
 865         .open = simple_open,
 866         .read = nsim_pp_hold_read,
 867         .write = nsim_pp_hold_write,
 868         .llseek = generic_file_llseek,
 869         .owner = THIS_MODULE,
 870 };
 871
 872 static void nsim_setup(struct net_device *dev)
 873 {
 874         ether_setup(dev);
 875         eth_hw_addr_random(dev);
 876
 877         dev->tx_queue_len = 0;
 878         dev->flags &= ~IFF_MULTICAST;
 879         dev->priv_flags |= IFF_LIVE_ADDR_CHANGE |
 880                            IFF_NO_QUEUE;
 881         dev->features |= NETIF_F_HIGHDMA |
 882                          NETIF_F_SG |
 883                          NETIF_F_FRAGLIST |
 884                          NETIF_F_HW_CSUM |
 885                          NETIF_F_LRO |
 886                          NETIF_F_TSO;
 887         dev->hw_features |= NETIF_F_HW_TC |
 888                             NETIF_F_SG |
 889                             NETIF_F_FRAGLIST |
 890                             NETIF_F_HW_CSUM |
 891                             NETIF_F_LRO |
 892                             NETIF_F_TSO;
 893         dev->max_mtu = ETH_MAX_MTU;
 894         dev->xdp_features = NETDEV_XDP_ACT_HW_OFFLOAD;
 895 }
 896
 897 static int nsim_queue_init(struct netdevsim *ns)
 898 {
 899         struct net_device *dev = ns->netdev;
 900         int i;
 901
 902         ns->rq = kcalloc(dev->num_rx_queues, sizeof(*ns->rq),
 903                          GFP_KERNEL_ACCOUNT);
 904         if (!ns->rq)
 905                 return -ENOMEM;
 906
 907         for (i = 0; i < dev->num_rx_queues; i++) {
 908                 ns->rq[i] = nsim_queue_alloc();
 909                 if (!ns->rq[i])
 910                         goto err_free_prev;
 911         }
 912
 913         return 0;
 914
 915 err_free_prev:
 916         while (i--)
 917                 kfree(ns->rq[i]);
 918         kfree(ns->rq);
 919         return -ENOMEM;
 920 }
 921
 922 static void nsim_queue_uninit(struct netdevsim *ns)
 923 {
 924         struct net_device *dev = ns->netdev;
 925         int i;
 926
 927         for (i = 0; i < dev->num_rx_queues; i++)
 928                 nsim_queue_free(ns->rq[i]);
 929
 930         kfree(ns->rq);
 931         ns->rq = NULL;
 932 }
 933
 934 static int nsim_init_netdevsim(struct netdevsim *ns)
 935 {
 936         struct mock_phc *phc;
 937         int err;
 938
 939         phc = mock_phc_create(&ns->nsim_bus_dev->dev);
 940         if (IS_ERR(phc))
 941                 return PTR_ERR(phc);
 942
 943         ns->phc = phc;
 944         ns->netdev->netdev_ops = &nsim_netdev_ops;
 945         ns->netdev->stat_ops = &nsim_stat_ops;
 946         ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops;
 947         netdev_lockdep_set_classes(ns->netdev);
 948
 949         err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev);
 950         if (err)
 951                 goto err_phc_destroy;
 952
 953         rtnl_lock();
 954         err = nsim_queue_init(ns);
 955         if (err)
 956                 goto err_utn_destroy;
 957
 958         err = nsim_bpf_init(ns);
 959         if (err)
 960                 goto err_rq_destroy;
 961
 962         nsim_macsec_init(ns);
 963         nsim_ipsec_init(ns);
 964
 965         err = register_netdevice(ns->netdev);
 966         if (err)
 967                 goto err_ipsec_teardown;
 968         rtnl_unlock();
 969
 970         if (IS_ENABLED(CONFIG_DEBUG_NET)) {
 971                 ns->nb.notifier_call = netdev_debug_event;
 972                 if (register_netdevice_notifier_dev_net(ns->netdev, &ns->nb,
 973                                                         &ns->nn))
 974                         ns->nb.notifier_call = NULL;
 975         }
 976
 977         return 0;
 978
 979 err_ipsec_teardown:
 980         nsim_ipsec_teardown(ns);
 981         nsim_macsec_teardown(ns);
 982         nsim_bpf_uninit(ns);
 983 err_rq_destroy:
 984         nsim_queue_uninit(ns);
 985 err_utn_destroy:
 986         rtnl_unlock();
 987         nsim_udp_tunnels_info_destroy(ns->netdev);
 988 err_phc_destroy:
 989         mock_phc_destroy(ns->phc);
 990         return err;
 991 }
 992
 993 static int nsim_init_netdevsim_vf(struct netdevsim *ns)
 994 {
 995         int err;
 996
 997         ns->netdev->netdev_ops = &nsim_vf_netdev_ops;
 998         rtnl_lock();
 999         err = register_netdevice(ns->netdev);
1000         rtnl_unlock();
1001         return err;
1002 }
1003
1004 static void nsim_exit_netdevsim(struct netdevsim *ns)
1005 {
1006         nsim_udp_tunnels_info_destroy(ns->netdev);
1007         mock_phc_destroy(ns->phc);
1008 }
1009
1010 struct netdevsim *
1011 nsim_create(struct nsim_dev *nsim_dev, struct nsim_dev_port *nsim_dev_port)
1012 {
1013         struct net_device *dev;
1014         struct netdevsim *ns;
1015         int err;
1016
1017         dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup,
1018                               nsim_dev->nsim_bus_dev->num_queues);
1019         if (!dev)
1020                 return ERR_PTR(-ENOMEM);
1021
1022         dev_net_set(dev, nsim_dev_net(nsim_dev));
1023         ns = netdev_priv(dev);
1024         ns->netdev = dev;
1025         u64_stats_init(&ns->syncp);
1026         ns->nsim_dev = nsim_dev;
1027         ns->nsim_dev_port = nsim_dev_port;
1028         ns->nsim_bus_dev = nsim_dev->nsim_bus_dev;
1029         SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev);
1030         SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port);
1031         nsim_ethtool_init(ns);
1032         if (nsim_dev_port_is_pf(nsim_dev_port))
1033                 err = nsim_init_netdevsim(ns);
1034         else
1035                 err = nsim_init_netdevsim_vf(ns);
1036         if (err)
1037                 goto err_free_netdev;
1038
1039         ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir,
1040                                          ns, &nsim_pp_hold_fops);
1041         ns->qr_dfs = debugfs_create_file("queue_reset", 0200,
1042                                          nsim_dev_port->ddir, ns,
1043                                          &nsim_qreset_fops);
1044
1045         return ns;
1046
1047 err_free_netdev:
1048         free_netdev(dev);
1049         return ERR_PTR(err);
1050 }
1051
1052 void nsim_destroy(struct netdevsim *ns)
1053 {
1054         struct net_device *dev = ns->netdev;
1055         struct netdevsim *peer;
1056
1057         debugfs_remove(ns->qr_dfs);
1058         debugfs_remove(ns->pp_dfs);
1059
1060         if (ns->nb.notifier_call)
1061                 unregister_netdevice_notifier_dev_net(ns->netdev, &ns->nb,
1062                                                       &ns->nn);
1063
1064         rtnl_lock();
1065         peer = rtnl_dereference(ns->peer);
1066         if (peer)
1067                 RCU_INIT_POINTER(peer->peer, NULL);
1068         RCU_INIT_POINTER(ns->peer, NULL);
1069         unregister_netdevice(dev);
1070         if (nsim_dev_port_is_pf(ns->nsim_dev_port)) {
1071                 nsim_macsec_teardown(ns);
1072                 nsim_ipsec_teardown(ns);
1073                 nsim_bpf_uninit(ns);
1074                 nsim_queue_uninit(ns);
1075         }
1076         rtnl_unlock();
1077         if (nsim_dev_port_is_pf(ns->nsim_dev_port))
1078                 nsim_exit_netdevsim(ns);
1079
1080         /* Put this intentionally late to exercise the orphaning path */
1081         if (ns->page) {
1082                 page_pool_put_full_page(ns->page->pp, ns->page, false);
1083                 ns->page = NULL;
1084         }
1085
1086         free_netdev(dev);
1087 }
1088
1089 bool netdev_is_nsim(struct net_device *dev)
1090 {
1091         return dev->netdev_ops == &nsim_netdev_ops;
1092 }
1093
1094 static int nsim_validate(struct nlattr *tb[], struct nlattr *data[],
1095                          struct netlink_ext_ack *extack)
1096 {
1097         NL_SET_ERR_MSG_MOD(extack,
1098                            "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device");
1099         return -EOPNOTSUPP;
1100 }
1101
1102 static struct rtnl_link_ops nsim_link_ops __read_mostly = {
1103         .kind           = DRV_NAME,
1104         .validate       = nsim_validate,
1105 };
1106
1107 static int __init nsim_module_init(void)
1108 {
1109         int err;
1110
1111         err = nsim_dev_init();
1112         if (err)
1113                 return err;
1114
1115         err = nsim_bus_init();
1116         if (err)
1117                 goto err_dev_exit;
1118
1119         err = rtnl_link_register(&nsim_link_ops);
1120         if (err)
1121                 goto err_bus_exit;
1122
1123         return 0;
1124
1125 err_bus_exit:
1126         nsim_bus_exit();
1127 err_dev_exit:
1128         nsim_dev_exit();
1129         return err;
1130 }
1131
1132 static void __exit nsim_module_exit(void)
1133 {
1134         rtnl_link_unregister(&nsim_link_ops);
1135         nsim_bus_exit();
1136         nsim_dev_exit();
1137 }
1138
1139 module_init(nsim_module_init);
1140 module_exit(nsim_module_exit);
1141 MODULE_LICENSE("GPL");
1142 MODULE_DESCRIPTION("Simulated networking device for testing");
1143 MODULE_ALIAS_RTNL_LINK(DRV_NAME);