net/sched/sch_multiq.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2008, Intel Corporation.
   4  *
   5  * Author: Alexander Duyck <alexander.h.duyck@intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/slab.h>
  10 #include <linux/types.h>
  11 #include <linux/kernel.h>
  12 #include <linux/string.h>
  13 #include <linux/errno.h>
  14 #include <linux/skbuff.h>
  15 #include <net/netlink.h>
  16 #include <net/pkt_sched.h>
  17 #include <net/pkt_cls.h>
  18
  19 struct multiq_sched_data {
  20         u16 bands;
  21         u16 max_bands;
  22         u16 curband;
  23         struct tcf_proto __rcu *filter_list;
  24         struct tcf_block *block;
  25         struct Qdisc **queues;
  26 };
  27
  28
  29 static struct Qdisc *
  30 multiq_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
  31 {
  32         struct multiq_sched_data *q = qdisc_priv(sch);
  33         u32 band;
  34         struct tcf_result res;
  35         struct tcf_proto *fl = rcu_dereference_bh(q->filter_list);
  36         int err;
  37
  38         *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
  39         err = tcf_classify(skb, fl, &res, false);
  40 #ifdef CONFIG_NET_CLS_ACT
  41         switch (err) {
  42         case TC_ACT_STOLEN:
  43         case TC_ACT_QUEUED:
  44         case TC_ACT_TRAP:
  45                 *qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
  46                 /* fall through */
  47         case TC_ACT_SHOT:
  48                 return NULL;
  49         }
  50 #endif
  51         band = skb_get_queue_mapping(skb);
  52
  53         if (band >= q->bands)
  54                 return q->queues[0];
  55
  56         return q->queues[band];
  57 }
  58
  59 static int
  60 multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
  61                struct sk_buff **to_free)
  62 {
  63         struct Qdisc *qdisc;
  64         int ret;
  65
  66         qdisc = multiq_classify(skb, sch, &ret);
  67 #ifdef CONFIG_NET_CLS_ACT
  68         if (qdisc == NULL) {
  69
  70                 if (ret & __NET_XMIT_BYPASS)
  71                         qdisc_qstats_drop(sch);
  72                 __qdisc_drop(skb, to_free);
  73                 return ret;
  74         }
  75 #endif
  76
  77         ret = qdisc_enqueue(skb, qdisc, to_free);
  78         if (ret == NET_XMIT_SUCCESS) {
  79                 sch->q.qlen++;
  80                 return NET_XMIT_SUCCESS;
  81         }
  82         if (net_xmit_drop_count(ret))
  83                 qdisc_qstats_drop(sch);
  84         return ret;
  85 }
  86
  87 static struct sk_buff *multiq_dequeue(struct Qdisc *sch)
  88 {
  89         struct multiq_sched_data *q = qdisc_priv(sch);
  90         struct Qdisc *qdisc;
  91         struct sk_buff *skb;
  92         int band;
  93
  94         for (band = 0; band < q->bands; band++) {
  95                 /* cycle through bands to ensure fairness */
  96                 q->curband++;
  97                 if (q->curband >= q->bands)
  98                         q->curband = 0;
  99
 100                 /* Check that target subqueue is available before
 101                  * pulling an skb to avoid head-of-line blocking.
 102                  */
 103                 if (!netif_xmit_stopped(
 104                     netdev_get_tx_queue(qdisc_dev(sch), q->curband))) {
 105                         qdisc = q->queues[q->curband];
 106                         skb = qdisc->dequeue(qdisc);
 107                         if (skb) {
 108                                 qdisc_bstats_update(sch, skb);
 109                                 sch->q.qlen--;
 110                                 return skb;
 111                         }
 112                 }
 113         }
 114         return NULL;
 115
 116 }
 117
 118 static struct sk_buff *multiq_peek(struct Qdisc *sch)
 119 {
 120         struct multiq_sched_data *q = qdisc_priv(sch);
 121         unsigned int curband = q->curband;
 122         struct Qdisc *qdisc;
 123         struct sk_buff *skb;
 124         int band;
 125
 126         for (band = 0; band < q->bands; band++) {
 127                 /* cycle through bands to ensure fairness */
 128                 curband++;
 129                 if (curband >= q->bands)
 130                         curband = 0;
 131
 132                 /* Check that target subqueue is available before
 133                  * pulling an skb to avoid head-of-line blocking.
 134                  */
 135                 if (!netif_xmit_stopped(
 136                     netdev_get_tx_queue(qdisc_dev(sch), curband))) {
 137                         qdisc = q->queues[curband];
 138                         skb = qdisc->ops->peek(qdisc);
 139                         if (skb)
 140                                 return skb;
 141                 }
 142         }
 143         return NULL;
 144
 145 }
 146
 147 static void
 148 multiq_reset(struct Qdisc *sch)
 149 {
 150         u16 band;
 151         struct multiq_sched_data *q = qdisc_priv(sch);
 152
 153         for (band = 0; band < q->bands; band++)
 154                 qdisc_reset(q->queues[band]);
 155         sch->q.qlen = 0;
 156         q->curband = 0;
 157 }
 158
 159 static void
 160 multiq_destroy(struct Qdisc *sch)
 161 {
 162         int band;
 163         struct multiq_sched_data *q = qdisc_priv(sch);
 164
 165         tcf_block_put(q->block);
 166         for (band = 0; band < q->bands; band++)
 167                 qdisc_put(q->queues[band]);
 168
 169         kfree(q->queues);
 170 }
 171
 172 static int multiq_tune(struct Qdisc *sch, struct nlattr *opt,
 173                        struct netlink_ext_ack *extack)
 174 {
 175         struct multiq_sched_data *q = qdisc_priv(sch);
 176         struct tc_multiq_qopt *qopt;
 177         int i;
 178
 179         if (!netif_is_multiqueue(qdisc_dev(sch)))
 180                 return -EOPNOTSUPP;
 181         if (nla_len(opt) < sizeof(*qopt))
 182                 return -EINVAL;
 183
 184         qopt = nla_data(opt);
 185
 186         qopt->bands = qdisc_dev(sch)->real_num_tx_queues;
 187
 188         sch_tree_lock(sch);
 189         q->bands = qopt->bands;
 190         for (i = q->bands; i < q->max_bands; i++) {
 191                 if (q->queues[i] != &noop_qdisc) {
 192                         struct Qdisc *child = q->queues[i];
 193
 194                         q->queues[i] = &noop_qdisc;
 195                         qdisc_tree_flush_backlog(child);
 196                         qdisc_put(child);
 197                 }
 198         }
 199
 200         sch_tree_unlock(sch);
 201
 202         for (i = 0; i < q->bands; i++) {
 203                 if (q->queues[i] == &noop_qdisc) {
 204                         struct Qdisc *child, *old;
 205                         child = qdisc_create_dflt(sch->dev_queue,
 206                                                   &pfifo_qdisc_ops,
 207                                                   TC_H_MAKE(sch->handle,
 208                                                             i + 1), extack);
 209                         if (child) {
 210                                 sch_tree_lock(sch);
 211                                 old = q->queues[i];
 212                                 q->queues[i] = child;
 213                                 if (child != &noop_qdisc)
 214                                         qdisc_hash_add(child, true);
 215
 216                                 if (old != &noop_qdisc) {
 217                                         qdisc_tree_flush_backlog(old);
 218                                         qdisc_put(old);
 219                                 }
 220                                 sch_tree_unlock(sch);
 221                         }
 222                 }
 223         }
 224         return 0;
 225 }
 226
 227 static int multiq_init(struct Qdisc *sch, struct nlattr *opt,
 228                        struct netlink_ext_ack *extack)
 229 {
 230         struct multiq_sched_data *q = qdisc_priv(sch);
 231         int i, err;
 232
 233         q->queues = NULL;
 234
 235         if (!opt)
 236                 return -EINVAL;
 237
 238         err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
 239         if (err)
 240                 return err;
 241
 242         q->max_bands = qdisc_dev(sch)->num_tx_queues;
 243
 244         q->queues = kcalloc(q->max_bands, sizeof(struct Qdisc *), GFP_KERNEL);
 245         if (!q->queues)
 246                 return -ENOBUFS;
 247         for (i = 0; i < q->max_bands; i++)
 248                 q->queues[i] = &noop_qdisc;
 249
 250         return multiq_tune(sch, opt, extack);
 251 }
 252
 253 static int multiq_dump(struct Qdisc *sch, struct sk_buff *skb)
 254 {
 255         struct multiq_sched_data *q = qdisc_priv(sch);
 256         unsigned char *b = skb_tail_pointer(skb);
 257         struct tc_multiq_qopt opt;
 258
 259         opt.bands = q->bands;
 260         opt.max_bands = q->max_bands;
 261
 262         if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
 263                 goto nla_put_failure;
 264
 265         return skb->len;
 266
 267 nla_put_failure:
 268         nlmsg_trim(skb, b);
 269         return -1;
 270 }
 271
 272 static int multiq_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
 273                         struct Qdisc **old, struct netlink_ext_ack *extack)
 274 {
 275         struct multiq_sched_data *q = qdisc_priv(sch);
 276         unsigned long band = arg - 1;
 277
 278         if (new == NULL)
 279                 new = &noop_qdisc;
 280
 281         *old = qdisc_replace(sch, new, &q->queues[band]);
 282         return 0;
 283 }
 284
 285 static struct Qdisc *
 286 multiq_leaf(struct Qdisc *sch, unsigned long arg)
 287 {
 288         struct multiq_sched_data *q = qdisc_priv(sch);
 289         unsigned long band = arg - 1;
 290
 291         return q->queues[band];
 292 }
 293
 294 static unsigned long multiq_find(struct Qdisc *sch, u32 classid)
 295 {
 296         struct multiq_sched_data *q = qdisc_priv(sch);
 297         unsigned long band = TC_H_MIN(classid);
 298
 299         if (band - 1 >= q->bands)
 300                 return 0;
 301         return band;
 302 }
 303
 304 static unsigned long multiq_bind(struct Qdisc *sch, unsigned long parent,
 305                                  u32 classid)
 306 {
 307         return multiq_find(sch, classid);
 308 }
 309
 310
 311 static void multiq_unbind(struct Qdisc *q, unsigned long cl)
 312 {
 313 }
 314
 315 static int multiq_dump_class(struct Qdisc *sch, unsigned long cl,
 316                              struct sk_buff *skb, struct tcmsg *tcm)
 317 {
 318         struct multiq_sched_data *q = qdisc_priv(sch);
 319
 320         tcm->tcm_handle |= TC_H_MIN(cl);
 321         tcm->tcm_info = q->queues[cl - 1]->handle;
 322         return 0;
 323 }
 324
 325 static int multiq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
 326                                  struct gnet_dump *d)
 327 {
 328         struct multiq_sched_data *q = qdisc_priv(sch);
 329         struct Qdisc *cl_q;
 330
 331         cl_q = q->queues[cl - 1];
 332         if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch),
 333                                   d, NULL, &cl_q->bstats) < 0 ||
 334             qdisc_qstats_copy(d, cl_q) < 0)
 335                 return -1;
 336
 337         return 0;
 338 }
 339
 340 static void multiq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 341 {
 342         struct multiq_sched_data *q = qdisc_priv(sch);
 343         int band;
 344
 345         if (arg->stop)
 346                 return;
 347
 348         for (band = 0; band < q->bands; band++) {
 349                 if (arg->count < arg->skip) {
 350                         arg->count++;
 351                         continue;
 352                 }
 353                 if (arg->fn(sch, band + 1, arg) < 0) {
 354                         arg->stop = 1;
 355                         break;
 356                 }
 357                 arg->count++;
 358         }
 359 }
 360
 361 static struct tcf_block *multiq_tcf_block(struct Qdisc *sch, unsigned long cl,
 362                                           struct netlink_ext_ack *extack)
 363 {
 364         struct multiq_sched_data *q = qdisc_priv(sch);
 365
 366         if (cl)
 367                 return NULL;
 368         return q->block;
 369 }
 370
 371 static const struct Qdisc_class_ops multiq_class_ops = {
 372         .graft          =       multiq_graft,
 373         .leaf           =       multiq_leaf,
 374         .find           =       multiq_find,
 375         .walk           =       multiq_walk,
 376         .tcf_block      =       multiq_tcf_block,
 377         .bind_tcf       =       multiq_bind,
 378         .unbind_tcf     =       multiq_unbind,
 379         .dump           =       multiq_dump_class,
 380         .dump_stats     =       multiq_dump_class_stats,
 381 };
 382
 383 static struct Qdisc_ops multiq_qdisc_ops __read_mostly = {
 384         .next           =       NULL,
 385         .cl_ops         =       &multiq_class_ops,
 386         .id             =       "multiq",
 387         .priv_size      =       sizeof(struct multiq_sched_data),
 388         .enqueue        =       multiq_enqueue,
 389         .dequeue        =       multiq_dequeue,
 390         .peek           =       multiq_peek,
 391         .init           =       multiq_init,
 392         .reset          =       multiq_reset,
 393         .destroy        =       multiq_destroy,
 394         .change         =       multiq_tune,
 395         .dump           =       multiq_dump,
 396         .owner          =       THIS_MODULE,
 397 };
 398
 399 static int __init multiq_module_init(void)
 400 {
 401         return register_qdisc(&multiq_qdisc_ops);
 402 }
 403
 404 static void __exit multiq_module_exit(void)
 405 {
 406         unregister_qdisc(&multiq_qdisc_ops);
 407 }
 408
 409 module_init(multiq_module_init)
 410 module_exit(multiq_module_exit)
 411
 412 MODULE_LICENSE("GPL");