drivers/block/null_blk/main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and
   4  * Shaohua Li <shli@fb.com>
   5  */
   6 #include <linux/module.h>
   7
   8 #include <linux/moduleparam.h>
   9 #include <linux/sched.h>
  10 #include <linux/fs.h>
  11 #include <linux/init.h>
  12 #include "null_blk.h"
  13
  14 #undef pr_fmt
  15 #define pr_fmt(fmt)     "null_blk: " fmt
  16
  17 #define FREE_BATCH              16
  18
  19 #define TICKS_PER_SEC           50ULL
  20 #define TIMER_INTERVAL          (NSEC_PER_SEC / TICKS_PER_SEC)
  21
  22 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
  23 static DECLARE_FAULT_ATTR(null_timeout_attr);
  24 static DECLARE_FAULT_ATTR(null_requeue_attr);
  25 static DECLARE_FAULT_ATTR(null_init_hctx_attr);
  26 #endif
  27
  28 static inline u64 mb_per_tick(int mbps)
  29 {
  30         return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
  31 }
  32
  33 /*
  34  * Status flags for nullb_device.
  35  *
  36  * CONFIGURED:  Device has been configured and turned on. Cannot reconfigure.
  37  * UP:          Device is currently on and visible in userspace.
  38  * THROTTLED:   Device is being throttled.
  39  * CACHE:       Device is using a write-back cache.
  40  */
  41 enum nullb_device_flags {
  42         NULLB_DEV_FL_CONFIGURED = 0,
  43         NULLB_DEV_FL_UP         = 1,
  44         NULLB_DEV_FL_THROTTLED  = 2,
  45         NULLB_DEV_FL_CACHE      = 3,
  46 };
  47
  48 #define MAP_SZ          ((PAGE_SIZE >> SECTOR_SHIFT) + 2)
  49 /*
  50  * nullb_page is a page in memory for nullb devices.
  51  *
  52  * @page:       The page holding the data.
  53  * @bitmap:     The bitmap represents which sector in the page has data.
  54  *              Each bit represents one block size. For example, sector 8
  55  *              will use the 7th bit
  56  * The highest 2 bits of bitmap are for special purpose. LOCK means the cache
  57  * page is being flushing to storage. FREE means the cache page is freed and
  58  * should be skipped from flushing to storage. Please see
  59  * null_make_cache_space
  60  */
  61 struct nullb_page {
  62         struct page *page;
  63         DECLARE_BITMAP(bitmap, MAP_SZ);
  64 };
  65 #define NULLB_PAGE_LOCK (MAP_SZ - 1)
  66 #define NULLB_PAGE_FREE (MAP_SZ - 2)
  67
  68 static LIST_HEAD(nullb_list);
  69 static struct mutex lock;
  70 static int null_major;
  71 static DEFINE_IDA(nullb_indexes);
  72 static struct blk_mq_tag_set tag_set;
  73
  74 enum {
  75         NULL_IRQ_NONE           = 0,
  76         NULL_IRQ_SOFTIRQ        = 1,
  77         NULL_IRQ_TIMER          = 2,
  78 };
  79
  80 static bool g_virt_boundary = false;
  81 module_param_named(virt_boundary, g_virt_boundary, bool, 0444);
  82 MODULE_PARM_DESC(virt_boundary, "Require a virtual boundary for the device. Default: False");
  83
  84 static int g_no_sched;
  85 module_param_named(no_sched, g_no_sched, int, 0444);
  86 MODULE_PARM_DESC(no_sched, "No io scheduler");
  87
  88 static int g_submit_queues = 1;
  89 module_param_named(submit_queues, g_submit_queues, int, 0444);
  90 MODULE_PARM_DESC(submit_queues, "Number of submission queues");
  91
  92 static int g_poll_queues = 1;
  93 module_param_named(poll_queues, g_poll_queues, int, 0444);
  94 MODULE_PARM_DESC(poll_queues, "Number of IOPOLL submission queues");
  95
  96 static int g_home_node = NUMA_NO_NODE;
  97 module_param_named(home_node, g_home_node, int, 0444);
  98 MODULE_PARM_DESC(home_node, "Home node for the device");
  99
 100 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
 101 /*
 102  * For more details about fault injection, please refer to
 103  * Documentation/fault-injection/fault-injection.rst.
 104  */
 105 static char g_timeout_str[80];
 106 module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), 0444);
 107 MODULE_PARM_DESC(timeout, "Fault injection. timeout=<interval>,<probability>,<space>,<times>");
 108
 109 static char g_requeue_str[80];
 110 module_param_string(requeue, g_requeue_str, sizeof(g_requeue_str), 0444);
 111 MODULE_PARM_DESC(requeue, "Fault injection. requeue=<interval>,<probability>,<space>,<times>");
 112
 113 static char g_init_hctx_str[80];
 114 module_param_string(init_hctx, g_init_hctx_str, sizeof(g_init_hctx_str), 0444);
 115 MODULE_PARM_DESC(init_hctx, "Fault injection to fail hctx init. init_hctx=<interval>,<probability>,<space>,<times>");
 116 #endif
 117
 118 static int g_queue_mode = NULL_Q_MQ;
 119
 120 static int null_param_store_val(const char *str, int *val, int min, int max)
 121 {
 122         int ret, new_val;
 123
 124         ret = kstrtoint(str, 10, &new_val);
 125         if (ret)
 126                 return -EINVAL;
 127
 128         if (new_val < min || new_val > max)
 129                 return -EINVAL;
 130
 131         *val = new_val;
 132         return 0;
 133 }
 134
 135 static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
 136 {
 137         return null_param_store_val(str, &g_queue_mode, NULL_Q_BIO, NULL_Q_MQ);
 138 }
 139
 140 static const struct kernel_param_ops null_queue_mode_param_ops = {
 141         .set    = null_set_queue_mode,
 142         .get    = param_get_int,
 143 };
 144
 145 device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, 0444);
 146 MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
 147
 148 static int g_gb = 250;
 149 module_param_named(gb, g_gb, int, 0444);
 150 MODULE_PARM_DESC(gb, "Size in GB");
 151
 152 static int g_bs = 512;
 153 module_param_named(bs, g_bs, int, 0444);
 154 MODULE_PARM_DESC(bs, "Block size (in bytes)");
 155
 156 static int g_max_sectors;
 157 module_param_named(max_sectors, g_max_sectors, int, 0444);
 158 MODULE_PARM_DESC(max_sectors, "Maximum size of a command (in 512B sectors)");
 159
 160 static unsigned int nr_devices = 1;
 161 module_param(nr_devices, uint, 0444);
 162 MODULE_PARM_DESC(nr_devices, "Number of devices to register");
 163
 164 static bool g_blocking;
 165 module_param_named(blocking, g_blocking, bool, 0444);
 166 MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
 167
 168 static bool shared_tags;
 169 module_param(shared_tags, bool, 0444);
 170 MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
 171
 172 static bool g_shared_tag_bitmap;
 173 module_param_named(shared_tag_bitmap, g_shared_tag_bitmap, bool, 0444);
 174 MODULE_PARM_DESC(shared_tag_bitmap, "Use shared tag bitmap for all submission queues for blk-mq");
 175
 176 static int g_irqmode = NULL_IRQ_SOFTIRQ;
 177
 178 static int null_set_irqmode(const char *str, const struct kernel_param *kp)
 179 {
 180         return null_param_store_val(str, &g_irqmode, NULL_IRQ_NONE,
 181                                         NULL_IRQ_TIMER);
 182 }
 183
 184 static const struct kernel_param_ops null_irqmode_param_ops = {
 185         .set    = null_set_irqmode,
 186         .get    = param_get_int,
 187 };
 188
 189 device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, 0444);
 190 MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
 191
 192 static unsigned long g_completion_nsec = 10000;
 193 module_param_named(completion_nsec, g_completion_nsec, ulong, 0444);
 194 MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
 195
 196 static int g_hw_queue_depth = 64;
 197 module_param_named(hw_queue_depth, g_hw_queue_depth, int, 0444);
 198 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
 199
 200 static bool g_use_per_node_hctx;
 201 module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, 0444);
 202 MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
 203
 204 static bool g_memory_backed;
 205 module_param_named(memory_backed, g_memory_backed, bool, 0444);
 206 MODULE_PARM_DESC(memory_backed, "Create a memory-backed block device. Default: false");
 207
 208 static bool g_discard;
 209 module_param_named(discard, g_discard, bool, 0444);
 210 MODULE_PARM_DESC(discard, "Support discard operations (requires memory-backed null_blk device). Default: false");
 211
 212 static unsigned long g_cache_size;
 213 module_param_named(cache_size, g_cache_size, ulong, 0444);
 214 MODULE_PARM_DESC(mbps, "Cache size in MiB for memory-backed device. Default: 0 (none)");
 215
 216 static unsigned int g_mbps;
 217 module_param_named(mbps, g_mbps, uint, 0444);
 218 MODULE_PARM_DESC(mbps, "Limit maximum bandwidth (in MiB/s). Default: 0 (no limit)");
 219
 220 static bool g_zoned;
 221 module_param_named(zoned, g_zoned, bool, S_IRUGO);
 222 MODULE_PARM_DESC(zoned, "Make device as a host-managed zoned block device. Default: false");
 223
 224 static unsigned long g_zone_size = 256;
 225 module_param_named(zone_size, g_zone_size, ulong, S_IRUGO);
 226 MODULE_PARM_DESC(zone_size, "Zone size in MB when block device is zoned. Must be power-of-two: Default: 256");
 227
 228 static unsigned long g_zone_capacity;
 229 module_param_named(zone_capacity, g_zone_capacity, ulong, 0444);
 230 MODULE_PARM_DESC(zone_capacity, "Zone capacity in MB when block device is zoned. Can be less than or equal to zone size. Default: Zone size");
 231
 232 static unsigned int g_zone_nr_conv;
 233 module_param_named(zone_nr_conv, g_zone_nr_conv, uint, 0444);
 234 MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones when block device is zoned. Default: 0");
 235
 236 static unsigned int g_zone_max_open;
 237 module_param_named(zone_max_open, g_zone_max_open, uint, 0444);
 238 MODULE_PARM_DESC(zone_max_open, "Maximum number of open zones when block device is zoned. Default: 0 (no limit)");
 239
 240 static unsigned int g_zone_max_active;
 241 module_param_named(zone_max_active, g_zone_max_active, uint, 0444);
 242 MODULE_PARM_DESC(zone_max_active, "Maximum number of active zones when block device is zoned. Default: 0 (no limit)");
 243
 244 static struct nullb_device *null_alloc_dev(void);
 245 static void null_free_dev(struct nullb_device *dev);
 246 static void null_del_dev(struct nullb *nullb);
 247 static int null_add_dev(struct nullb_device *dev);
 248 static struct nullb *null_find_dev_by_name(const char *name);
 249 static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
 250
 251 static inline struct nullb_device *to_nullb_device(struct config_item *item)
 252 {
 253         return item ? container_of(item, struct nullb_device, item) : NULL;
 254 }
 255
 256 static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
 257 {
 258         return snprintf(page, PAGE_SIZE, "%u\n", val);
 259 }
 260
 261 static inline ssize_t nullb_device_ulong_attr_show(unsigned long val,
 262         char *page)
 263 {
 264         return snprintf(page, PAGE_SIZE, "%lu\n", val);
 265 }
 266
 267 static inline ssize_t nullb_device_bool_attr_show(bool val, char *page)
 268 {
 269         return snprintf(page, PAGE_SIZE, "%u\n", val);
 270 }
 271
 272 static ssize_t nullb_device_uint_attr_store(unsigned int *val,
 273         const char *page, size_t count)
 274 {
 275         unsigned int tmp;
 276         int result;
 277
 278         result = kstrtouint(page, 0, &tmp);
 279         if (result < 0)
 280                 return result;
 281
 282         *val = tmp;
 283         return count;
 284 }
 285
 286 static ssize_t nullb_device_ulong_attr_store(unsigned long *val,
 287         const char *page, size_t count)
 288 {
 289         int result;
 290         unsigned long tmp;
 291
 292         result = kstrtoul(page, 0, &tmp);
 293         if (result < 0)
 294                 return result;
 295
 296         *val = tmp;
 297         return count;
 298 }
 299
 300 static ssize_t nullb_device_bool_attr_store(bool *val, const char *page,
 301         size_t count)
 302 {
 303         bool tmp;
 304         int result;
 305
 306         result = kstrtobool(page,  &tmp);
 307         if (result < 0)
 308                 return result;
 309
 310         *val = tmp;
 311         return count;
 312 }
 313
 314 /* The following macro should only be used with TYPE = {uint, ulong, bool}. */
 315 #define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY)                            \
 316 static ssize_t                                                          \
 317 nullb_device_##NAME##_show(struct config_item *item, char *page)        \
 318 {                                                                       \
 319         return nullb_device_##TYPE##_attr_show(                         \
 320                                 to_nullb_device(item)->NAME, page);     \
 321 }                                                                       \
 322 static ssize_t                                                          \
 323 nullb_device_##NAME##_store(struct config_item *item, const char *page, \
 324                             size_t count)                               \
 325 {                                                                       \
 326         int (*apply_fn)(struct nullb_device *dev, TYPE new_value) = APPLY;\
 327         struct nullb_device *dev = to_nullb_device(item);               \
 328         TYPE new_value = 0;                                             \
 329         int ret;                                                        \
 330                                                                         \
 331         ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\
 332         if (ret < 0)                                                    \
 333                 return ret;                                             \
 334         if (apply_fn)                                                   \
 335                 ret = apply_fn(dev, new_value);                         \
 336         else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags))        \
 337                 ret = -EBUSY;                                           \
 338         if (ret < 0)                                                    \
 339                 return ret;                                             \
 340         dev->NAME = new_value;                                          \
 341         return count;                                                   \
 342 }                                                                       \
 343 CONFIGFS_ATTR(nullb_device_, NAME);
 344
 345 static int nullb_update_nr_hw_queues(struct nullb_device *dev,
 346                                      unsigned int submit_queues,
 347                                      unsigned int poll_queues)
 348
 349 {
 350         struct blk_mq_tag_set *set;
 351         int ret, nr_hw_queues;
 352
 353         if (!dev->nullb)
 354                 return 0;
 355
 356         /*
 357          * Make sure at least one submit queue exists.
 358          */
 359         if (!submit_queues)
 360                 return -EINVAL;
 361
 362         /*
 363          * Make sure that null_init_hctx() does not access nullb->queues[] past
 364          * the end of that array.
 365          */
 366         if (submit_queues > nr_cpu_ids || poll_queues > g_poll_queues)
 367                 return -EINVAL;
 368
 369         /*
 370          * Keep previous and new queue numbers in nullb_device for reference in
 371          * the call back function null_map_queues().
 372          */
 373         dev->prev_submit_queues = dev->submit_queues;
 374         dev->prev_poll_queues = dev->poll_queues;
 375         dev->submit_queues = submit_queues;
 376         dev->poll_queues = poll_queues;
 377
 378         set = dev->nullb->tag_set;
 379         nr_hw_queues = submit_queues + poll_queues;
 380         blk_mq_update_nr_hw_queues(set, nr_hw_queues);
 381         ret = set->nr_hw_queues == nr_hw_queues ? 0 : -ENOMEM;
 382
 383         if (ret) {
 384                 /* on error, revert the queue numbers */
 385                 dev->submit_queues = dev->prev_submit_queues;
 386                 dev->poll_queues = dev->prev_poll_queues;
 387         }
 388
 389         return ret;
 390 }
 391
 392 static int nullb_apply_submit_queues(struct nullb_device *dev,
 393                                      unsigned int submit_queues)
 394 {
 395         return nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues);
 396 }
 397
 398 static int nullb_apply_poll_queues(struct nullb_device *dev,
 399                                    unsigned int poll_queues)
 400 {
 401         return nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues);
 402 }
 403
 404 NULLB_DEVICE_ATTR(size, ulong, NULL);
 405 NULLB_DEVICE_ATTR(completion_nsec, ulong, NULL);
 406 NULLB_DEVICE_ATTR(submit_queues, uint, nullb_apply_submit_queues);
 407 NULLB_DEVICE_ATTR(poll_queues, uint, nullb_apply_poll_queues);
 408 NULLB_DEVICE_ATTR(home_node, uint, NULL);
 409 NULLB_DEVICE_ATTR(queue_mode, uint, NULL);
 410 NULLB_DEVICE_ATTR(blocksize, uint, NULL);
 411 NULLB_DEVICE_ATTR(max_sectors, uint, NULL);
 412 NULLB_DEVICE_ATTR(irqmode, uint, NULL);
 413 NULLB_DEVICE_ATTR(hw_queue_depth, uint, NULL);
 414 NULLB_DEVICE_ATTR(index, uint, NULL);
 415 NULLB_DEVICE_ATTR(blocking, bool, NULL);
 416 NULLB_DEVICE_ATTR(use_per_node_hctx, bool, NULL);
 417 NULLB_DEVICE_ATTR(memory_backed, bool, NULL);
 418 NULLB_DEVICE_ATTR(discard, bool, NULL);
 419 NULLB_DEVICE_ATTR(mbps, uint, NULL);
 420 NULLB_DEVICE_ATTR(cache_size, ulong, NULL);
 421 NULLB_DEVICE_ATTR(zoned, bool, NULL);
 422 NULLB_DEVICE_ATTR(zone_size, ulong, NULL);
 423 NULLB_DEVICE_ATTR(zone_capacity, ulong, NULL);
 424 NULLB_DEVICE_ATTR(zone_nr_conv, uint, NULL);
 425 NULLB_DEVICE_ATTR(zone_max_open, uint, NULL);
 426 NULLB_DEVICE_ATTR(zone_max_active, uint, NULL);
 427 NULLB_DEVICE_ATTR(virt_boundary, bool, NULL);
 428
 429 static ssize_t nullb_device_power_show(struct config_item *item, char *page)
 430 {
 431         return nullb_device_bool_attr_show(to_nullb_device(item)->power, page);
 432 }
 433
 434 static ssize_t nullb_device_power_store(struct config_item *item,
 435                                      const char *page, size_t count)
 436 {
 437         struct nullb_device *dev = to_nullb_device(item);
 438         bool newp = false;
 439         ssize_t ret;
 440
 441         ret = nullb_device_bool_attr_store(&newp, page, count);
 442         if (ret < 0)
 443                 return ret;
 444
 445         if (!dev->power && newp) {
 446                 if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags))
 447                         return count;
 448                 ret = null_add_dev(dev);
 449                 if (ret) {
 450                         clear_bit(NULLB_DEV_FL_UP, &dev->flags);
 451                         return ret;
 452                 }
 453
 454                 set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
 455                 dev->power = newp;
 456         } else if (dev->power && !newp) {
 457                 if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
 458                         mutex_lock(&lock);
 459                         dev->power = newp;
 460                         null_del_dev(dev->nullb);
 461                         mutex_unlock(&lock);
 462                 }
 463                 clear_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
 464         }
 465
 466         return count;
 467 }
 468
 469 CONFIGFS_ATTR(nullb_device_, power);
 470
 471 static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page)
 472 {
 473         struct nullb_device *t_dev = to_nullb_device(item);
 474
 475         return badblocks_show(&t_dev->badblocks, page, 0);
 476 }
 477
 478 static ssize_t nullb_device_badblocks_store(struct config_item *item,
 479                                      const char *page, size_t count)
 480 {
 481         struct nullb_device *t_dev = to_nullb_device(item);
 482         char *orig, *buf, *tmp;
 483         u64 start, end;
 484         int ret;
 485
 486         orig = kstrndup(page, count, GFP_KERNEL);
 487         if (!orig)
 488                 return -ENOMEM;
 489
 490         buf = strstrip(orig);
 491
 492         ret = -EINVAL;
 493         if (buf[0] != '+' && buf[0] != '-')
 494                 goto out;
 495         tmp = strchr(&buf[1], '-');
 496         if (!tmp)
 497                 goto out;
 498         *tmp = '\0';
 499         ret = kstrtoull(buf + 1, 0, &start);
 500         if (ret)
 501                 goto out;
 502         ret = kstrtoull(tmp + 1, 0, &end);
 503         if (ret)
 504                 goto out;
 505         ret = -EINVAL;
 506         if (start > end)
 507                 goto out;
 508         /* enable badblocks */
 509         cmpxchg(&t_dev->badblocks.shift, -1, 0);
 510         if (buf[0] == '+')
 511                 ret = badblocks_set(&t_dev->badblocks, start,
 512                         end - start + 1, 1);
 513         else
 514                 ret = badblocks_clear(&t_dev->badblocks, start,
 515                         end - start + 1);
 516         if (ret == 0)
 517                 ret = count;
 518 out:
 519         kfree(orig);
 520         return ret;
 521 }
 522 CONFIGFS_ATTR(nullb_device_, badblocks);
 523
 524 static struct configfs_attribute *nullb_device_attrs[] = {
 525         &nullb_device_attr_size,
 526         &nullb_device_attr_completion_nsec,
 527         &nullb_device_attr_submit_queues,
 528         &nullb_device_attr_poll_queues,
 529         &nullb_device_attr_home_node,
 530         &nullb_device_attr_queue_mode,
 531         &nullb_device_attr_blocksize,
 532         &nullb_device_attr_max_sectors,
 533         &nullb_device_attr_irqmode,
 534         &nullb_device_attr_hw_queue_depth,
 535         &nullb_device_attr_index,
 536         &nullb_device_attr_blocking,
 537         &nullb_device_attr_use_per_node_hctx,
 538         &nullb_device_attr_power,
 539         &nullb_device_attr_memory_backed,
 540         &nullb_device_attr_discard,
 541         &nullb_device_attr_mbps,
 542         &nullb_device_attr_cache_size,
 543         &nullb_device_attr_badblocks,
 544         &nullb_device_attr_zoned,
 545         &nullb_device_attr_zone_size,
 546         &nullb_device_attr_zone_capacity,
 547         &nullb_device_attr_zone_nr_conv,
 548         &nullb_device_attr_zone_max_open,
 549         &nullb_device_attr_zone_max_active,
 550         &nullb_device_attr_virt_boundary,
 551         NULL,
 552 };
 553
 554 static void nullb_device_release(struct config_item *item)
 555 {
 556         struct nullb_device *dev = to_nullb_device(item);
 557
 558         null_free_device_storage(dev, false);
 559         null_free_dev(dev);
 560 }
 561
 562 static struct configfs_item_operations nullb_device_ops = {
 563         .release        = nullb_device_release,
 564 };
 565
 566 static const struct config_item_type nullb_device_type = {
 567         .ct_item_ops    = &nullb_device_ops,
 568         .ct_attrs       = nullb_device_attrs,
 569         .ct_owner       = THIS_MODULE,
 570 };
 571
 572 static struct
 573 config_item *nullb_group_make_item(struct config_group *group, const char *name)
 574 {
 575         struct nullb_device *dev;
 576
 577         if (null_find_dev_by_name(name))
 578                 return ERR_PTR(-EEXIST);
 579
 580         dev = null_alloc_dev();
 581         if (!dev)
 582                 return ERR_PTR(-ENOMEM);
 583
 584         config_item_init_type_name(&dev->item, name, &nullb_device_type);
 585
 586         return &dev->item;
 587 }
 588
 589 static void
 590 nullb_group_drop_item(struct config_group *group, struct config_item *item)
 591 {
 592         struct nullb_device *dev = to_nullb_device(item);
 593
 594         if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
 595                 mutex_lock(&lock);
 596                 dev->power = false;
 597                 null_del_dev(dev->nullb);
 598                 mutex_unlock(&lock);
 599         }
 600
 601         config_item_put(item);
 602 }
 603
 604 static ssize_t memb_group_features_show(struct config_item *item, char *page)
 605 {
 606         return snprintf(page, PAGE_SIZE,
 607                         "memory_backed,discard,bandwidth,cache,badblocks,zoned,zone_size,zone_capacity,zone_nr_conv,zone_max_open,zone_max_active,blocksize,max_sectors,virt_boundary\n");
 608 }
 609
 610 CONFIGFS_ATTR_RO(memb_group_, features);
 611
 612 static struct configfs_attribute *nullb_group_attrs[] = {
 613         &memb_group_attr_features,
 614         NULL,
 615 };
 616
 617 static struct configfs_group_operations nullb_group_ops = {
 618         .make_item      = nullb_group_make_item,
 619         .drop_item      = nullb_group_drop_item,
 620 };
 621
 622 static const struct config_item_type nullb_group_type = {
 623         .ct_group_ops   = &nullb_group_ops,
 624         .ct_attrs       = nullb_group_attrs,
 625         .ct_owner       = THIS_MODULE,
 626 };
 627
 628 static struct configfs_subsystem nullb_subsys = {
 629         .su_group = {
 630                 .cg_item = {
 631                         .ci_namebuf = "nullb",
 632                         .ci_type = &nullb_group_type,
 633                 },
 634         },
 635 };
 636
 637 static inline int null_cache_active(struct nullb *nullb)
 638 {
 639         return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
 640 }
 641
 642 static struct nullb_device *null_alloc_dev(void)
 643 {
 644         struct nullb_device *dev;
 645
 646         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 647         if (!dev)
 648                 return NULL;
 649         INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
 650         INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
 651         if (badblocks_init(&dev->badblocks, 0)) {
 652                 kfree(dev);
 653                 return NULL;
 654         }
 655
 656         dev->size = g_gb * 1024;
 657         dev->completion_nsec = g_completion_nsec;
 658         dev->submit_queues = g_submit_queues;
 659         dev->prev_submit_queues = g_submit_queues;
 660         dev->poll_queues = g_poll_queues;
 661         dev->prev_poll_queues = g_poll_queues;
 662         dev->home_node = g_home_node;
 663         dev->queue_mode = g_queue_mode;
 664         dev->blocksize = g_bs;
 665         dev->max_sectors = g_max_sectors;
 666         dev->irqmode = g_irqmode;
 667         dev->hw_queue_depth = g_hw_queue_depth;
 668         dev->blocking = g_blocking;
 669         dev->memory_backed = g_memory_backed;
 670         dev->discard = g_discard;
 671         dev->cache_size = g_cache_size;
 672         dev->mbps = g_mbps;
 673         dev->use_per_node_hctx = g_use_per_node_hctx;
 674         dev->zoned = g_zoned;
 675         dev->zone_size = g_zone_size;
 676         dev->zone_capacity = g_zone_capacity;
 677         dev->zone_nr_conv = g_zone_nr_conv;
 678         dev->zone_max_open = g_zone_max_open;
 679         dev->zone_max_active = g_zone_max_active;
 680         dev->virt_boundary = g_virt_boundary;
 681         return dev;
 682 }
 683
 684 static void null_free_dev(struct nullb_device *dev)
 685 {
 686         if (!dev)
 687                 return;
 688
 689         null_free_zoned_dev(dev);
 690         badblocks_exit(&dev->badblocks);
 691         kfree(dev);
 692 }
 693
 694 static void put_tag(struct nullb_queue *nq, unsigned int tag)
 695 {
 696         clear_bit_unlock(tag, nq->tag_map);
 697
 698         if (waitqueue_active(&nq->wait))
 699                 wake_up(&nq->wait);
 700 }
 701
 702 static unsigned int get_tag(struct nullb_queue *nq)
 703 {
 704         unsigned int tag;
 705
 706         do {
 707                 tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
 708                 if (tag >= nq->queue_depth)
 709                         return -1U;
 710         } while (test_and_set_bit_lock(tag, nq->tag_map));
 711
 712         return tag;
 713 }
 714
 715 static void free_cmd(struct nullb_cmd *cmd)
 716 {
 717         put_tag(cmd->nq, cmd->tag);
 718 }
 719
 720 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer);
 721
 722 static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
 723 {
 724         struct nullb_cmd *cmd;
 725         unsigned int tag;
 726
 727         tag = get_tag(nq);
 728         if (tag != -1U) {
 729                 cmd = &nq->cmds[tag];
 730                 cmd->tag = tag;
 731                 cmd->error = BLK_STS_OK;
 732                 cmd->nq = nq;
 733                 if (nq->dev->irqmode == NULL_IRQ_TIMER) {
 734                         hrtimer_init(&cmd->timer, CLOCK_MONOTONIC,
 735                                      HRTIMER_MODE_REL);
 736                         cmd->timer.function = null_cmd_timer_expired;
 737                 }
 738                 return cmd;
 739         }
 740
 741         return NULL;
 742 }
 743
 744 static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, struct bio *bio)
 745 {
 746         struct nullb_cmd *cmd;
 747         DEFINE_WAIT(wait);
 748
 749         do {
 750                 /*
 751                  * This avoids multiple return statements, multiple calls to
 752                  * __alloc_cmd() and a fast path call to prepare_to_wait().
 753                  */
 754                 cmd = __alloc_cmd(nq);
 755                 if (cmd) {
 756                         cmd->bio = bio;
 757                         return cmd;
 758                 }
 759                 prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
 760                 io_schedule();
 761                 finish_wait(&nq->wait, &wait);
 762         } while (1);
 763 }
 764
 765 static void end_cmd(struct nullb_cmd *cmd)
 766 {
 767         int queue_mode = cmd->nq->dev->queue_mode;
 768
 769         switch (queue_mode)  {
 770         case NULL_Q_MQ:
 771                 blk_mq_end_request(cmd->rq, cmd->error);
 772                 return;
 773         case NULL_Q_BIO:
 774                 cmd->bio->bi_status = cmd->error;
 775                 bio_endio(cmd->bio);
 776                 break;
 777         }
 778
 779         free_cmd(cmd);
 780 }
 781
 782 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
 783 {
 784         end_cmd(container_of(timer, struct nullb_cmd, timer));
 785
 786         return HRTIMER_NORESTART;
 787 }
 788
 789 static void null_cmd_end_timer(struct nullb_cmd *cmd)
 790 {
 791         ktime_t kt = cmd->nq->dev->completion_nsec;
 792
 793         hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL);
 794 }
 795
 796 static void null_complete_rq(struct request *rq)
 797 {
 798         end_cmd(blk_mq_rq_to_pdu(rq));
 799 }
 800
 801 static struct nullb_page *null_alloc_page(void)
 802 {
 803         struct nullb_page *t_page;
 804
 805         t_page = kmalloc(sizeof(struct nullb_page), GFP_NOIO);
 806         if (!t_page)
 807                 return NULL;
 808
 809         t_page->page = alloc_pages(GFP_NOIO, 0);
 810         if (!t_page->page) {
 811                 kfree(t_page);
 812                 return NULL;
 813         }
 814
 815         memset(t_page->bitmap, 0, sizeof(t_page->bitmap));
 816         return t_page;
 817 }
 818
 819 static void null_free_page(struct nullb_page *t_page)
 820 {
 821         __set_bit(NULLB_PAGE_FREE, t_page->bitmap);
 822         if (test_bit(NULLB_PAGE_LOCK, t_page->bitmap))
 823                 return;
 824         __free_page(t_page->page);
 825         kfree(t_page);
 826 }
 827
 828 static bool null_page_empty(struct nullb_page *page)
 829 {
 830         int size = MAP_SZ - 2;
 831
 832         return find_first_bit(page->bitmap, size) == size;
 833 }
 834
 835 static void null_free_sector(struct nullb *nullb, sector_t sector,
 836         bool is_cache)
 837 {
 838         unsigned int sector_bit;
 839         u64 idx;
 840         struct nullb_page *t_page, *ret;
 841         struct radix_tree_root *root;
 842
 843         root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 844         idx = sector >> PAGE_SECTORS_SHIFT;
 845         sector_bit = (sector & SECTOR_MASK);
 846
 847         t_page = radix_tree_lookup(root, idx);
 848         if (t_page) {
 849                 __clear_bit(sector_bit, t_page->bitmap);
 850
 851                 if (null_page_empty(t_page)) {
 852                         ret = radix_tree_delete_item(root, idx, t_page);
 853                         WARN_ON(ret != t_page);
 854                         null_free_page(ret);
 855                         if (is_cache)
 856                                 nullb->dev->curr_cache -= PAGE_SIZE;
 857                 }
 858         }
 859 }
 860
 861 static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,
 862         struct nullb_page *t_page, bool is_cache)
 863 {
 864         struct radix_tree_root *root;
 865
 866         root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 867
 868         if (radix_tree_insert(root, idx, t_page)) {
 869                 null_free_page(t_page);
 870                 t_page = radix_tree_lookup(root, idx);
 871                 WARN_ON(!t_page || t_page->page->index != idx);
 872         } else if (is_cache)
 873                 nullb->dev->curr_cache += PAGE_SIZE;
 874
 875         return t_page;
 876 }
 877
 878 static void null_free_device_storage(struct nullb_device *dev, bool is_cache)
 879 {
 880         unsigned long pos = 0;
 881         int nr_pages;
 882         struct nullb_page *ret, *t_pages[FREE_BATCH];
 883         struct radix_tree_root *root;
 884
 885         root = is_cache ? &dev->cache : &dev->data;
 886
 887         do {
 888                 int i;
 889
 890                 nr_pages = radix_tree_gang_lookup(root,
 891                                 (void **)t_pages, pos, FREE_BATCH);
 892
 893                 for (i = 0; i < nr_pages; i++) {
 894                         pos = t_pages[i]->page->index;
 895                         ret = radix_tree_delete_item(root, pos, t_pages[i]);
 896                         WARN_ON(ret != t_pages[i]);
 897                         null_free_page(ret);
 898                 }
 899
 900                 pos++;
 901         } while (nr_pages == FREE_BATCH);
 902
 903         if (is_cache)
 904                 dev->curr_cache = 0;
 905 }
 906
 907 static struct nullb_page *__null_lookup_page(struct nullb *nullb,
 908         sector_t sector, bool for_write, bool is_cache)
 909 {
 910         unsigned int sector_bit;
 911         u64 idx;
 912         struct nullb_page *t_page;
 913         struct radix_tree_root *root;
 914
 915         idx = sector >> PAGE_SECTORS_SHIFT;
 916         sector_bit = (sector & SECTOR_MASK);
 917
 918         root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 919         t_page = radix_tree_lookup(root, idx);
 920         WARN_ON(t_page && t_page->page->index != idx);
 921
 922         if (t_page && (for_write || test_bit(sector_bit, t_page->bitmap)))
 923                 return t_page;
 924
 925         return NULL;
 926 }
 927
 928 static struct nullb_page *null_lookup_page(struct nullb *nullb,
 929         sector_t sector, bool for_write, bool ignore_cache)
 930 {
 931         struct nullb_page *page = NULL;
 932
 933         if (!ignore_cache)
 934                 page = __null_lookup_page(nullb, sector, for_write, true);
 935         if (page)
 936                 return page;
 937         return __null_lookup_page(nullb, sector, for_write, false);
 938 }
 939
 940 static struct nullb_page *null_insert_page(struct nullb *nullb,
 941                                            sector_t sector, bool ignore_cache)
 942         __releases(&nullb->lock)
 943         __acquires(&nullb->lock)
 944 {
 945         u64 idx;
 946         struct nullb_page *t_page;
 947
 948         t_page = null_lookup_page(nullb, sector, true, ignore_cache);
 949         if (t_page)
 950                 return t_page;
 951
 952         spin_unlock_irq(&nullb->lock);
 953
 954         t_page = null_alloc_page();
 955         if (!t_page)
 956                 goto out_lock;
 957
 958         if (radix_tree_preload(GFP_NOIO))
 959                 goto out_freepage;
 960
 961         spin_lock_irq(&nullb->lock);
 962         idx = sector >> PAGE_SECTORS_SHIFT;
 963         t_page->page->index = idx;
 964         t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache);
 965         radix_tree_preload_end();
 966
 967         return t_page;
 968 out_freepage:
 969         null_free_page(t_page);
 970 out_lock:
 971         spin_lock_irq(&nullb->lock);
 972         return null_lookup_page(nullb, sector, true, ignore_cache);
 973 }
 974
 975 static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
 976 {
 977         int i;
 978         unsigned int offset;
 979         u64 idx;
 980         struct nullb_page *t_page, *ret;
 981         void *dst, *src;
 982
 983         idx = c_page->page->index;
 984
 985         t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true);
 986
 987         __clear_bit(NULLB_PAGE_LOCK, c_page->bitmap);
 988         if (test_bit(NULLB_PAGE_FREE, c_page->bitmap)) {
 989                 null_free_page(c_page);
 990                 if (t_page && null_page_empty(t_page)) {
 991                         ret = radix_tree_delete_item(&nullb->dev->data,
 992                                 idx, t_page);
 993                         null_free_page(t_page);
 994                 }
 995                 return 0;
 996         }
 997
 998         if (!t_page)
 999                 return -ENOMEM;
1000
1001         src = kmap_atomic(c_page->page);
1002         dst = kmap_atomic(t_page->page);
1003
1004         for (i = 0; i < PAGE_SECTORS;
1005                         i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
1006                 if (test_bit(i, c_page->bitmap)) {
1007                         offset = (i << SECTOR_SHIFT);
1008                         memcpy(dst + offset, src + offset,
1009                                 nullb->dev->blocksize);
1010                         __set_bit(i, t_page->bitmap);
1011                 }
1012         }
1013
1014         kunmap_atomic(dst);
1015         kunmap_atomic(src);
1016
1017         ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
1018         null_free_page(ret);
1019         nullb->dev->curr_cache -= PAGE_SIZE;
1020
1021         return 0;
1022 }
1023
1024 static int null_make_cache_space(struct nullb *nullb, unsigned long n)
1025 {
1026         int i, err, nr_pages;
1027         struct nullb_page *c_pages[FREE_BATCH];
1028         unsigned long flushed = 0, one_round;
1029
1030 again:
1031         if ((nullb->dev->cache_size * 1024 * 1024) >
1032              nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)
1033                 return 0;
1034
1035         nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,
1036                         (void **)c_pages, nullb->cache_flush_pos, FREE_BATCH);
1037         /*
1038          * nullb_flush_cache_page could unlock before using the c_pages. To
1039          * avoid race, we don't allow page free
1040          */
1041         for (i = 0; i < nr_pages; i++) {
1042                 nullb->cache_flush_pos = c_pages[i]->page->index;
1043                 /*
1044                  * We found the page which is being flushed to disk by other
1045                  * threads
1046                  */
1047                 if (test_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap))
1048                         c_pages[i] = NULL;
1049                 else
1050                         __set_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap);
1051         }
1052
1053         one_round = 0;
1054         for (i = 0; i < nr_pages; i++) {
1055                 if (c_pages[i] == NULL)
1056                         continue;
1057                 err = null_flush_cache_page(nullb, c_pages[i]);
1058                 if (err)
1059                         return err;
1060                 one_round++;
1061         }
1062         flushed += one_round << PAGE_SHIFT;
1063
1064         if (n > flushed) {
1065                 if (nr_pages == 0)
1066                         nullb->cache_flush_pos = 0;
1067                 if (one_round == 0) {
1068                         /* give other threads a chance */
1069                         spin_unlock_irq(&nullb->lock);
1070                         spin_lock_irq(&nullb->lock);
1071                 }
1072                 goto again;
1073         }
1074         return 0;
1075 }
1076
1077 static int copy_to_nullb(struct nullb *nullb, struct page *source,
1078         unsigned int off, sector_t sector, size_t n, bool is_fua)
1079 {
1080         size_t temp, count = 0;
1081         unsigned int offset;
1082         struct nullb_page *t_page;
1083         void *dst, *src;
1084
1085         while (count < n) {
1086                 temp = min_t(size_t, nullb->dev->blocksize, n - count);
1087
1088                 if (null_cache_active(nullb) && !is_fua)
1089                         null_make_cache_space(nullb, PAGE_SIZE);
1090
1091                 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
1092                 t_page = null_insert_page(nullb, sector,
1093                         !null_cache_active(nullb) || is_fua);
1094                 if (!t_page)
1095                         return -ENOSPC;
1096
1097                 src = kmap_atomic(source);
1098                 dst = kmap_atomic(t_page->page);
1099                 memcpy(dst + offset, src + off + count, temp);
1100                 kunmap_atomic(dst);
1101                 kunmap_atomic(src);
1102
1103                 __set_bit(sector & SECTOR_MASK, t_page->bitmap);
1104
1105                 if (is_fua)
1106                         null_free_sector(nullb, sector, true);
1107
1108                 count += temp;
1109                 sector += temp >> SECTOR_SHIFT;
1110         }
1111         return 0;
1112 }
1113
1114 static int copy_from_nullb(struct nullb *nullb, struct page *dest,
1115         unsigned int off, sector_t sector, size_t n)
1116 {
1117         size_t temp, count = 0;
1118         unsigned int offset;
1119         struct nullb_page *t_page;
1120         void *dst, *src;
1121
1122         while (count < n) {
1123                 temp = min_t(size_t, nullb->dev->blocksize, n - count);
1124
1125                 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
1126                 t_page = null_lookup_page(nullb, sector, false,
1127                         !null_cache_active(nullb));
1128
1129                 dst = kmap_atomic(dest);
1130                 if (!t_page) {
1131                         memset(dst + off + count, 0, temp);
1132                         goto next;
1133                 }
1134                 src = kmap_atomic(t_page->page);
1135                 memcpy(dst + off + count, src + offset, temp);
1136                 kunmap_atomic(src);
1137 next:
1138                 kunmap_atomic(dst);
1139
1140                 count += temp;
1141                 sector += temp >> SECTOR_SHIFT;
1142         }
1143         return 0;
1144 }
1145
1146 static void nullb_fill_pattern(struct nullb *nullb, struct page *page,
1147                                unsigned int len, unsigned int off)
1148 {
1149         void *dst;
1150
1151         dst = kmap_atomic(page);
1152         memset(dst + off, 0xFF, len);
1153         kunmap_atomic(dst);
1154 }
1155
1156 blk_status_t null_handle_discard(struct nullb_device *dev,
1157                                  sector_t sector, sector_t nr_sectors)
1158 {
1159         struct nullb *nullb = dev->nullb;
1160         size_t n = nr_sectors << SECTOR_SHIFT;
1161         size_t temp;
1162
1163         spin_lock_irq(&nullb->lock);
1164         while (n > 0) {
1165                 temp = min_t(size_t, n, dev->blocksize);
1166                 null_free_sector(nullb, sector, false);
1167                 if (null_cache_active(nullb))
1168                         null_free_sector(nullb, sector, true);
1169                 sector += temp >> SECTOR_SHIFT;
1170                 n -= temp;
1171         }
1172         spin_unlock_irq(&nullb->lock);
1173
1174         return BLK_STS_OK;
1175 }
1176
1177 static int null_handle_flush(struct nullb *nullb)
1178 {
1179         int err;
1180
1181         if (!null_cache_active(nullb))
1182                 return 0;
1183
1184         spin_lock_irq(&nullb->lock);
1185         while (true) {
1186                 err = null_make_cache_space(nullb,
1187                         nullb->dev->cache_size * 1024 * 1024);
1188                 if (err || nullb->dev->curr_cache == 0)
1189                         break;
1190         }
1191
1192         WARN_ON(!radix_tree_empty(&nullb->dev->cache));
1193         spin_unlock_irq(&nullb->lock);
1194         return err;
1195 }
1196
1197 static int null_transfer(struct nullb *nullb, struct page *page,
1198         unsigned int len, unsigned int off, bool is_write, sector_t sector,
1199         bool is_fua)
1200 {
1201         struct nullb_device *dev = nullb->dev;
1202         unsigned int valid_len = len;
1203         int err = 0;
1204
1205         if (!is_write) {
1206                 if (dev->zoned)
1207                         valid_len = null_zone_valid_read_len(nullb,
1208                                 sector, len);
1209
1210                 if (valid_len) {
1211                         err = copy_from_nullb(nullb, page, off,
1212                                 sector, valid_len);
1213                         off += valid_len;
1214                         len -= valid_len;
1215                 }
1216
1217                 if (len)
1218                         nullb_fill_pattern(nullb, page, len, off);
1219                 flush_dcache_page(page);
1220         } else {
1221                 flush_dcache_page(page);
1222                 err = copy_to_nullb(nullb, page, off, sector, len, is_fua);
1223         }
1224
1225         return err;
1226 }
1227
1228 static int null_handle_rq(struct nullb_cmd *cmd)
1229 {
1230         struct request *rq = cmd->rq;
1231         struct nullb *nullb = cmd->nq->dev->nullb;
1232         int err;
1233         unsigned int len;
1234         sector_t sector = blk_rq_pos(rq);
1235         struct req_iterator iter;
1236         struct bio_vec bvec;
1237
1238         spin_lock_irq(&nullb->lock);
1239         rq_for_each_segment(bvec, rq, iter) {
1240                 len = bvec.bv_len;
1241                 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1242                                      op_is_write(req_op(rq)), sector,
1243                                      rq->cmd_flags & REQ_FUA);
1244                 if (err) {
1245                         spin_unlock_irq(&nullb->lock);
1246                         return err;
1247                 }
1248                 sector += len >> SECTOR_SHIFT;
1249         }
1250         spin_unlock_irq(&nullb->lock);
1251
1252         return 0;
1253 }
1254
1255 static int null_handle_bio(struct nullb_cmd *cmd)
1256 {
1257         struct bio *bio = cmd->bio;
1258         struct nullb *nullb = cmd->nq->dev->nullb;
1259         int err;
1260         unsigned int len;
1261         sector_t sector = bio->bi_iter.bi_sector;
1262         struct bio_vec bvec;
1263         struct bvec_iter iter;
1264
1265         spin_lock_irq(&nullb->lock);
1266         bio_for_each_segment(bvec, bio, iter) {
1267                 len = bvec.bv_len;
1268                 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1269                                      op_is_write(bio_op(bio)), sector,
1270                                      bio->bi_opf & REQ_FUA);
1271                 if (err) {
1272                         spin_unlock_irq(&nullb->lock);
1273                         return err;
1274                 }
1275                 sector += len >> SECTOR_SHIFT;
1276         }
1277         spin_unlock_irq(&nullb->lock);
1278         return 0;
1279 }
1280
1281 static void null_stop_queue(struct nullb *nullb)
1282 {
1283         struct request_queue *q = nullb->q;
1284
1285         if (nullb->dev->queue_mode == NULL_Q_MQ)
1286                 blk_mq_stop_hw_queues(q);
1287 }
1288
1289 static void null_restart_queue_async(struct nullb *nullb)
1290 {
1291         struct request_queue *q = nullb->q;
1292
1293         if (nullb->dev->queue_mode == NULL_Q_MQ)
1294                 blk_mq_start_stopped_hw_queues(q, true);
1295 }
1296
1297 static inline blk_status_t null_handle_throttled(struct nullb_cmd *cmd)
1298 {
1299         struct nullb_device *dev = cmd->nq->dev;
1300         struct nullb *nullb = dev->nullb;
1301         blk_status_t sts = BLK_STS_OK;
1302         struct request *rq = cmd->rq;
1303
1304         if (!hrtimer_active(&nullb->bw_timer))
1305                 hrtimer_restart(&nullb->bw_timer);
1306
1307         if (atomic_long_sub_return(blk_rq_bytes(rq), &nullb->cur_bytes) < 0) {
1308                 null_stop_queue(nullb);
1309                 /* race with timer */
1310                 if (atomic_long_read(&nullb->cur_bytes) > 0)
1311                         null_restart_queue_async(nullb);
1312                 /* requeue request */
1313                 sts = BLK_STS_DEV_RESOURCE;
1314         }
1315         return sts;
1316 }
1317
1318 static inline blk_status_t null_handle_badblocks(struct nullb_cmd *cmd,
1319                                                  sector_t sector,
1320                                                  sector_t nr_sectors)
1321 {
1322         struct badblocks *bb = &cmd->nq->dev->badblocks;
1323         sector_t first_bad;
1324         int bad_sectors;
1325
1326         if (badblocks_check(bb, sector, nr_sectors, &first_bad, &bad_sectors))
1327                 return BLK_STS_IOERR;
1328
1329         return BLK_STS_OK;
1330 }
1331
1332 static inline blk_status_t null_handle_memory_backed(struct nullb_cmd *cmd,
1333                                                      enum req_op op,
1334                                                      sector_t sector,
1335                                                      sector_t nr_sectors)
1336 {
1337         struct nullb_device *dev = cmd->nq->dev;
1338         int err;
1339
1340         if (op == REQ_OP_DISCARD)
1341                 return null_handle_discard(dev, sector, nr_sectors);
1342
1343         if (dev->queue_mode == NULL_Q_BIO)
1344                 err = null_handle_bio(cmd);
1345         else
1346                 err = null_handle_rq(cmd);
1347
1348         return errno_to_blk_status(err);
1349 }
1350
1351 static void nullb_zero_read_cmd_buffer(struct nullb_cmd *cmd)
1352 {
1353         struct nullb_device *dev = cmd->nq->dev;
1354         struct bio *bio;
1355
1356         if (dev->memory_backed)
1357                 return;
1358
1359         if (dev->queue_mode == NULL_Q_BIO && bio_op(cmd->bio) == REQ_OP_READ) {
1360                 zero_fill_bio(cmd->bio);
1361         } else if (req_op(cmd->rq) == REQ_OP_READ) {
1362                 __rq_for_each_bio(bio, cmd->rq)
1363                         zero_fill_bio(bio);
1364         }
1365 }
1366
1367 static inline void nullb_complete_cmd(struct nullb_cmd *cmd)
1368 {
1369         /*
1370          * Since root privileges are required to configure the null_blk
1371          * driver, it is fine that this driver does not initialize the
1372          * data buffers of read commands. Zero-initialize these buffers
1373          * anyway if KMSAN is enabled to prevent that KMSAN complains
1374          * about null_blk not initializing read data buffers.
1375          */
1376         if (IS_ENABLED(CONFIG_KMSAN))
1377                 nullb_zero_read_cmd_buffer(cmd);
1378
1379         /* Complete IO by inline, softirq or timer */
1380         switch (cmd->nq->dev->irqmode) {
1381         case NULL_IRQ_SOFTIRQ:
1382                 switch (cmd->nq->dev->queue_mode) {
1383                 case NULL_Q_MQ:
1384                         if (likely(!blk_should_fake_timeout(cmd->rq->q)))
1385                                 blk_mq_complete_request(cmd->rq);
1386                         break;
1387                 case NULL_Q_BIO:
1388                         /*
1389                          * XXX: no proper submitting cpu information available.
1390                          */
1391                         end_cmd(cmd);
1392                         break;
1393                 }
1394                 break;
1395         case NULL_IRQ_NONE:
1396                 end_cmd(cmd);
1397                 break;
1398         case NULL_IRQ_TIMER:
1399                 null_cmd_end_timer(cmd);
1400                 break;
1401         }
1402 }
1403
1404 blk_status_t null_process_cmd(struct nullb_cmd *cmd, enum req_op op,
1405                               sector_t sector, unsigned int nr_sectors)
1406 {
1407         struct nullb_device *dev = cmd->nq->dev;
1408         blk_status_t ret;
1409
1410         if (dev->badblocks.shift != -1) {
1411                 ret = null_handle_badblocks(cmd, sector, nr_sectors);
1412                 if (ret != BLK_STS_OK)
1413                         return ret;
1414         }
1415
1416         if (dev->memory_backed)
1417                 return null_handle_memory_backed(cmd, op, sector, nr_sectors);
1418
1419         return BLK_STS_OK;
1420 }
1421
1422 static blk_status_t null_handle_cmd(struct nullb_cmd *cmd, sector_t sector,
1423                                     sector_t nr_sectors, enum req_op op)
1424 {
1425         struct nullb_device *dev = cmd->nq->dev;
1426         struct nullb *nullb = dev->nullb;
1427         blk_status_t sts;
1428
1429         if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) {
1430                 sts = null_handle_throttled(cmd);
1431                 if (sts != BLK_STS_OK)
1432                         return sts;
1433         }
1434
1435         if (op == REQ_OP_FLUSH) {
1436                 cmd->error = errno_to_blk_status(null_handle_flush(nullb));
1437                 goto out;
1438         }
1439
1440         if (dev->zoned)
1441                 sts = null_process_zoned_cmd(cmd, op, sector, nr_sectors);
1442         else
1443                 sts = null_process_cmd(cmd, op, sector, nr_sectors);
1444
1445         /* Do not overwrite errors (e.g. timeout errors) */
1446         if (cmd->error == BLK_STS_OK)
1447                 cmd->error = sts;
1448
1449 out:
1450         nullb_complete_cmd(cmd);
1451         return BLK_STS_OK;
1452 }
1453
1454 static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
1455 {
1456         struct nullb *nullb = container_of(timer, struct nullb, bw_timer);
1457         ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1458         unsigned int mbps = nullb->dev->mbps;
1459
1460         if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps))
1461                 return HRTIMER_NORESTART;
1462
1463         atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps));
1464         null_restart_queue_async(nullb);
1465
1466         hrtimer_forward_now(&nullb->bw_timer, timer_interval);
1467
1468         return HRTIMER_RESTART;
1469 }
1470
1471 static void nullb_setup_bwtimer(struct nullb *nullb)
1472 {
1473         ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1474
1475         hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1476         nullb->bw_timer.function = nullb_bwtimer_fn;
1477         atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps));
1478         hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL);
1479 }
1480
1481 static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
1482 {
1483         int index = 0;
1484
1485         if (nullb->nr_queues != 1)
1486                 index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);
1487
1488         return &nullb->queues[index];
1489 }
1490
1491 static void null_submit_bio(struct bio *bio)
1492 {
1493         sector_t sector = bio->bi_iter.bi_sector;
1494         sector_t nr_sectors = bio_sectors(bio);
1495         struct nullb *nullb = bio->bi_bdev->bd_disk->private_data;
1496         struct nullb_queue *nq = nullb_to_queue(nullb);
1497
1498         null_handle_cmd(alloc_cmd(nq, bio), sector, nr_sectors, bio_op(bio));
1499 }
1500
1501 static bool should_timeout_request(struct request *rq)
1502 {
1503 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1504         if (g_timeout_str[0])
1505                 return should_fail(&null_timeout_attr, 1);
1506 #endif
1507         return false;
1508 }
1509
1510 static bool should_requeue_request(struct request *rq)
1511 {
1512 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1513         if (g_requeue_str[0])
1514                 return should_fail(&null_requeue_attr, 1);
1515 #endif
1516         return false;
1517 }
1518
1519 static int null_map_queues(struct blk_mq_tag_set *set)
1520 {
1521         struct nullb *nullb = set->driver_data;
1522         int i, qoff;
1523         unsigned int submit_queues = g_submit_queues;
1524         unsigned int poll_queues = g_poll_queues;
1525
1526         if (nullb) {
1527                 struct nullb_device *dev = nullb->dev;
1528
1529                 /*
1530                  * Refer nr_hw_queues of the tag set to check if the expected
1531                  * number of hardware queues are prepared. If block layer failed
1532                  * to prepare them, use previous numbers of submit queues and
1533                  * poll queues to map queues.
1534                  */
1535                 if (set->nr_hw_queues ==
1536                     dev->submit_queues + dev->poll_queues) {
1537                         submit_queues = dev->submit_queues;
1538                         poll_queues = dev->poll_queues;
1539                 } else if (set->nr_hw_queues ==
1540                            dev->prev_submit_queues + dev->prev_poll_queues) {
1541                         submit_queues = dev->prev_submit_queues;
1542                         poll_queues = dev->prev_poll_queues;
1543                 } else {
1544                         pr_warn("tag set has unexpected nr_hw_queues: %d\n",
1545                                 set->nr_hw_queues);
1546                         return -EINVAL;
1547                 }
1548         }
1549
1550         for (i = 0, qoff = 0; i < set->nr_maps; i++) {
1551                 struct blk_mq_queue_map *map = &set->map[i];
1552
1553                 switch (i) {
1554                 case HCTX_TYPE_DEFAULT:
1555                         map->nr_queues = submit_queues;
1556                         break;
1557                 case HCTX_TYPE_READ:
1558                         map->nr_queues = 0;
1559                         continue;
1560                 case HCTX_TYPE_POLL:
1561                         map->nr_queues = poll_queues;
1562                         break;
1563                 }
1564                 map->queue_offset = qoff;
1565                 qoff += map->nr_queues;
1566                 blk_mq_map_queues(map);
1567         }
1568
1569         return 0;
1570 }
1571
1572 static int null_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
1573 {
1574         struct nullb_queue *nq = hctx->driver_data;
1575         LIST_HEAD(list);
1576         int nr = 0;
1577
1578         spin_lock(&nq->poll_lock);
1579         list_splice_init(&nq->poll_list, &list);
1580         spin_unlock(&nq->poll_lock);
1581
1582         while (!list_empty(&list)) {
1583                 struct nullb_cmd *cmd;
1584                 struct request *req;
1585
1586                 req = list_first_entry(&list, struct request, queuelist);
1587                 list_del_init(&req->queuelist);
1588                 cmd = blk_mq_rq_to_pdu(req);
1589                 cmd->error = null_process_cmd(cmd, req_op(req), blk_rq_pos(req),
1590                                                 blk_rq_sectors(req));
1591                 if (!blk_mq_add_to_batch(req, iob, (__force int) cmd->error,
1592                                         blk_mq_end_request_batch))
1593                         end_cmd(cmd);
1594                 nr++;
1595         }
1596
1597         return nr;
1598 }
1599
1600 static enum blk_eh_timer_return null_timeout_rq(struct request *rq)
1601 {
1602         struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1603         struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1604
1605         pr_info("rq %p timed out\n", rq);
1606
1607         if (hctx->type == HCTX_TYPE_POLL) {
1608                 struct nullb_queue *nq = hctx->driver_data;
1609
1610                 spin_lock(&nq->poll_lock);
1611                 list_del_init(&rq->queuelist);
1612                 spin_unlock(&nq->poll_lock);
1613         }
1614
1615         /*
1616          * If the device is marked as blocking (i.e. memory backed or zoned
1617          * device), the submission path may be blocked waiting for resources
1618          * and cause real timeouts. For these real timeouts, the submission
1619          * path will complete the request using blk_mq_complete_request().
1620          * Only fake timeouts need to execute blk_mq_complete_request() here.
1621          */
1622         cmd->error = BLK_STS_TIMEOUT;
1623         if (cmd->fake_timeout || hctx->type == HCTX_TYPE_POLL)
1624                 blk_mq_complete_request(rq);
1625         return BLK_EH_DONE;
1626 }
1627
1628 static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
1629                          const struct blk_mq_queue_data *bd)
1630 {
1631         struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
1632         struct nullb_queue *nq = hctx->driver_data;
1633         sector_t nr_sectors = blk_rq_sectors(bd->rq);
1634         sector_t sector = blk_rq_pos(bd->rq);
1635         const bool is_poll = hctx->type == HCTX_TYPE_POLL;
1636
1637         might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
1638
1639         if (!is_poll && nq->dev->irqmode == NULL_IRQ_TIMER) {
1640                 hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1641                 cmd->timer.function = null_cmd_timer_expired;
1642         }
1643         cmd->rq = bd->rq;
1644         cmd->error = BLK_STS_OK;
1645         cmd->nq = nq;
1646         cmd->fake_timeout = should_timeout_request(bd->rq);
1647
1648         blk_mq_start_request(bd->rq);
1649
1650         if (should_requeue_request(bd->rq)) {
1651                 /*
1652                  * Alternate between hitting the core BUSY path, and the
1653                  * driver driven requeue path
1654                  */
1655                 nq->requeue_selection++;
1656                 if (nq->requeue_selection & 1)
1657                         return BLK_STS_RESOURCE;
1658                 else {
1659                         blk_mq_requeue_request(bd->rq, true);
1660                         return BLK_STS_OK;
1661                 }
1662         }
1663
1664         if (is_poll) {
1665                 spin_lock(&nq->poll_lock);
1666                 list_add_tail(&bd->rq->queuelist, &nq->poll_list);
1667                 spin_unlock(&nq->poll_lock);
1668                 return BLK_STS_OK;
1669         }
1670         if (cmd->fake_timeout)
1671                 return BLK_STS_OK;
1672
1673         return null_handle_cmd(cmd, sector, nr_sectors, req_op(bd->rq));
1674 }
1675
1676 static void cleanup_queue(struct nullb_queue *nq)
1677 {
1678         bitmap_free(nq->tag_map);
1679         kfree(nq->cmds);
1680 }
1681
1682 static void cleanup_queues(struct nullb *nullb)
1683 {
1684         int i;
1685
1686         for (i = 0; i < nullb->nr_queues; i++)
1687                 cleanup_queue(&nullb->queues[i]);
1688
1689         kfree(nullb->queues);
1690 }
1691
1692 static void null_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
1693 {
1694         struct nullb_queue *nq = hctx->driver_data;
1695         struct nullb *nullb = nq->dev->nullb;
1696
1697         nullb->nr_queues--;
1698 }
1699
1700 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
1701 {
1702         init_waitqueue_head(&nq->wait);
1703         nq->queue_depth = nullb->queue_depth;
1704         nq->dev = nullb->dev;
1705         INIT_LIST_HEAD(&nq->poll_list);
1706         spin_lock_init(&nq->poll_lock);
1707 }
1708
1709 static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
1710                           unsigned int hctx_idx)
1711 {
1712         struct nullb *nullb = hctx->queue->queuedata;
1713         struct nullb_queue *nq;
1714
1715 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1716         if (g_init_hctx_str[0] && should_fail(&null_init_hctx_attr, 1))
1717                 return -EFAULT;
1718 #endif
1719
1720         nq = &nullb->queues[hctx_idx];
1721         hctx->driver_data = nq;
1722         null_init_queue(nullb, nq);
1723         nullb->nr_queues++;
1724
1725         return 0;
1726 }
1727
1728 static const struct blk_mq_ops null_mq_ops = {
1729         .queue_rq       = null_queue_rq,
1730         .complete       = null_complete_rq,
1731         .timeout        = null_timeout_rq,
1732         .poll           = null_poll,
1733         .map_queues     = null_map_queues,
1734         .init_hctx      = null_init_hctx,
1735         .exit_hctx      = null_exit_hctx,
1736 };
1737
1738 static void null_del_dev(struct nullb *nullb)
1739 {
1740         struct nullb_device *dev;
1741
1742         if (!nullb)
1743                 return;
1744
1745         dev = nullb->dev;
1746
1747         ida_simple_remove(&nullb_indexes, nullb->index);
1748
1749         list_del_init(&nullb->list);
1750
1751         del_gendisk(nullb->disk);
1752
1753         if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
1754                 hrtimer_cancel(&nullb->bw_timer);
1755                 atomic_long_set(&nullb->cur_bytes, LONG_MAX);
1756                 null_restart_queue_async(nullb);
1757         }
1758
1759         put_disk(nullb->disk);
1760         if (dev->queue_mode == NULL_Q_MQ &&
1761             nullb->tag_set == &nullb->__tag_set)
1762                 blk_mq_free_tag_set(nullb->tag_set);
1763         cleanup_queues(nullb);
1764         if (null_cache_active(nullb))
1765                 null_free_device_storage(nullb->dev, true);
1766         kfree(nullb);
1767         dev->nullb = NULL;
1768 }
1769
1770 static void null_config_discard(struct nullb *nullb)
1771 {
1772         if (nullb->dev->discard == false)
1773                 return;
1774
1775         if (!nullb->dev->memory_backed) {
1776                 nullb->dev->discard = false;
1777                 pr_info("discard option is ignored without memory backing\n");
1778                 return;
1779         }
1780
1781         if (nullb->dev->zoned) {
1782                 nullb->dev->discard = false;
1783                 pr_info("discard option is ignored in zoned mode\n");
1784                 return;
1785         }
1786
1787         nullb->q->limits.discard_granularity = nullb->dev->blocksize;
1788         blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9);
1789 }
1790
1791 static const struct block_device_operations null_bio_ops = {
1792         .owner          = THIS_MODULE,
1793         .submit_bio     = null_submit_bio,
1794         .report_zones   = null_report_zones,
1795 };
1796
1797 static const struct block_device_operations null_rq_ops = {
1798         .owner          = THIS_MODULE,
1799         .report_zones   = null_report_zones,
1800 };
1801
1802 static int setup_commands(struct nullb_queue *nq)
1803 {
1804         struct nullb_cmd *cmd;
1805         int i;
1806
1807         nq->cmds = kcalloc(nq->queue_depth, sizeof(*cmd), GFP_KERNEL);
1808         if (!nq->cmds)
1809                 return -ENOMEM;
1810
1811         nq->tag_map = bitmap_zalloc(nq->queue_depth, GFP_KERNEL);
1812         if (!nq->tag_map) {
1813                 kfree(nq->cmds);
1814                 return -ENOMEM;
1815         }
1816
1817         for (i = 0; i < nq->queue_depth; i++) {
1818                 cmd = &nq->cmds[i];
1819                 cmd->tag = -1U;
1820         }
1821
1822         return 0;
1823 }
1824
1825 static int setup_queues(struct nullb *nullb)
1826 {
1827         int nqueues = nr_cpu_ids;
1828
1829         if (g_poll_queues)
1830                 nqueues += g_poll_queues;
1831
1832         nullb->queues = kcalloc(nqueues, sizeof(struct nullb_queue),
1833                                 GFP_KERNEL);
1834         if (!nullb->queues)
1835                 return -ENOMEM;
1836
1837         nullb->queue_depth = nullb->dev->hw_queue_depth;
1838         return 0;
1839 }
1840
1841 static int init_driver_queues(struct nullb *nullb)
1842 {
1843         struct nullb_queue *nq;
1844         int i, ret = 0;
1845
1846         for (i = 0; i < nullb->dev->submit_queues; i++) {
1847                 nq = &nullb->queues[i];
1848
1849                 null_init_queue(nullb, nq);
1850
1851                 ret = setup_commands(nq);
1852                 if (ret)
1853                         return ret;
1854                 nullb->nr_queues++;
1855         }
1856         return 0;
1857 }
1858
1859 static int null_gendisk_register(struct nullb *nullb)
1860 {
1861         sector_t size = ((sector_t)nullb->dev->size * SZ_1M) >> SECTOR_SHIFT;
1862         struct gendisk *disk = nullb->disk;
1863
1864         set_capacity(disk, size);
1865
1866         disk->major             = null_major;
1867         disk->first_minor       = nullb->index;
1868         disk->minors            = 1;
1869         if (queue_is_mq(nullb->q))
1870                 disk->fops              = &null_rq_ops;
1871         else
1872                 disk->fops              = &null_bio_ops;
1873         disk->private_data      = nullb;
1874         strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
1875
1876         if (nullb->dev->zoned) {
1877                 int ret = null_register_zoned_dev(nullb);
1878
1879                 if (ret)
1880                         return ret;
1881         }
1882
1883         return add_disk(disk);
1884 }
1885
1886 static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set)
1887 {
1888         int poll_queues;
1889
1890         set->ops = &null_mq_ops;
1891         set->nr_hw_queues = nullb ? nullb->dev->submit_queues :
1892                                                 g_submit_queues;
1893         poll_queues = nullb ? nullb->dev->poll_queues : g_poll_queues;
1894         if (poll_queues)
1895                 set->nr_hw_queues += poll_queues;
1896         set->queue_depth = nullb ? nullb->dev->hw_queue_depth :
1897                                                 g_hw_queue_depth;
1898         set->numa_node = nullb ? nullb->dev->home_node : g_home_node;
1899         set->cmd_size   = sizeof(struct nullb_cmd);
1900         set->flags = BLK_MQ_F_SHOULD_MERGE;
1901         if (g_no_sched)
1902                 set->flags |= BLK_MQ_F_NO_SCHED;
1903         if (g_shared_tag_bitmap)
1904                 set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;
1905         set->driver_data = nullb;
1906         if (poll_queues)
1907                 set->nr_maps = 3;
1908         else
1909                 set->nr_maps = 1;
1910
1911         if ((nullb && nullb->dev->blocking) || g_blocking)
1912                 set->flags |= BLK_MQ_F_BLOCKING;
1913
1914         return blk_mq_alloc_tag_set(set);
1915 }
1916
1917 static int null_validate_conf(struct nullb_device *dev)
1918 {
1919         dev->blocksize = round_down(dev->blocksize, 512);
1920         dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
1921
1922         if (dev->queue_mode == NULL_Q_MQ && dev->use_per_node_hctx) {
1923                 if (dev->submit_queues != nr_online_nodes)
1924                         dev->submit_queues = nr_online_nodes;
1925         } else if (dev->submit_queues > nr_cpu_ids)
1926                 dev->submit_queues = nr_cpu_ids;
1927         else if (dev->submit_queues == 0)
1928                 dev->submit_queues = 1;
1929         dev->prev_submit_queues = dev->submit_queues;
1930
1931         if (dev->poll_queues > g_poll_queues)
1932                 dev->poll_queues = g_poll_queues;
1933         dev->prev_poll_queues = dev->poll_queues;
1934
1935         dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ);
1936         dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER);
1937
1938         /* Do memory allocation, so set blocking */
1939         if (dev->memory_backed)
1940                 dev->blocking = true;
1941         else /* cache is meaningless */
1942                 dev->cache_size = 0;
1943         dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024,
1944                                                 dev->cache_size);
1945         dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps);
1946         /* can not stop a queue */
1947         if (dev->queue_mode == NULL_Q_BIO)
1948                 dev->mbps = 0;
1949
1950         if (dev->zoned &&
1951             (!dev->zone_size || !is_power_of_2(dev->zone_size))) {
1952                 pr_err("zone_size must be power-of-two\n");
1953                 return -EINVAL;
1954         }
1955
1956         return 0;
1957 }
1958
1959 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1960 static bool __null_setup_fault(struct fault_attr *attr, char *str)
1961 {
1962         if (!str[0])
1963                 return true;
1964
1965         if (!setup_fault_attr(attr, str))
1966                 return false;
1967
1968         attr->verbose = 0;
1969         return true;
1970 }
1971 #endif
1972
1973 static bool null_setup_fault(void)
1974 {
1975 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1976         if (!__null_setup_fault(&null_timeout_attr, g_timeout_str))
1977                 return false;
1978         if (!__null_setup_fault(&null_requeue_attr, g_requeue_str))
1979                 return false;
1980         if (!__null_setup_fault(&null_init_hctx_attr, g_init_hctx_str))
1981                 return false;
1982 #endif
1983         return true;
1984 }
1985
1986 static int null_add_dev(struct nullb_device *dev)
1987 {
1988         struct nullb *nullb;
1989         int rv;
1990
1991         rv = null_validate_conf(dev);
1992         if (rv)
1993                 return rv;
1994
1995         nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
1996         if (!nullb) {
1997                 rv = -ENOMEM;
1998                 goto out;
1999         }
2000         nullb->dev = dev;
2001         dev->nullb = nullb;
2002
2003         spin_lock_init(&nullb->lock);
2004
2005         rv = setup_queues(nullb);
2006         if (rv)
2007                 goto out_free_nullb;
2008
2009         if (dev->queue_mode == NULL_Q_MQ) {
2010                 if (shared_tags) {
2011                         nullb->tag_set = &tag_set;
2012                         rv = 0;
2013                 } else {
2014                         nullb->tag_set = &nullb->__tag_set;
2015                         rv = null_init_tag_set(nullb, nullb->tag_set);
2016                 }
2017
2018                 if (rv)
2019                         goto out_cleanup_queues;
2020
2021                 if (!null_setup_fault())
2022                         goto out_cleanup_tags;
2023
2024                 nullb->tag_set->timeout = 5 * HZ;
2025                 nullb->disk = blk_mq_alloc_disk(nullb->tag_set, nullb);
2026                 if (IS_ERR(nullb->disk)) {
2027                         rv = PTR_ERR(nullb->disk);
2028                         goto out_cleanup_tags;
2029                 }
2030                 nullb->q = nullb->disk->queue;
2031         } else if (dev->queue_mode == NULL_Q_BIO) {
2032                 rv = -ENOMEM;
2033                 nullb->disk = blk_alloc_disk(nullb->dev->home_node);
2034                 if (!nullb->disk)
2035                         goto out_cleanup_queues;
2036
2037                 nullb->q = nullb->disk->queue;
2038                 rv = init_driver_queues(nullb);
2039                 if (rv)
2040                         goto out_cleanup_disk;
2041         }
2042
2043         if (dev->mbps) {
2044                 set_bit(NULLB_DEV_FL_THROTTLED, &dev->flags);
2045                 nullb_setup_bwtimer(nullb);
2046         }
2047
2048         if (dev->cache_size > 0) {
2049                 set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
2050                 blk_queue_write_cache(nullb->q, true, true);
2051         }
2052
2053         if (dev->zoned) {
2054                 rv = null_init_zoned_dev(dev, nullb->q);
2055                 if (rv)
2056                         goto out_cleanup_disk;
2057         }
2058
2059         nullb->q->queuedata = nullb;
2060         blk_queue_flag_set(QUEUE_FLAG_NONROT, nullb->q);
2061         blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, nullb->q);
2062
2063         mutex_lock(&lock);
2064         nullb->index = ida_simple_get(&nullb_indexes, 0, 0, GFP_KERNEL);
2065         dev->index = nullb->index;
2066         mutex_unlock(&lock);
2067
2068         blk_queue_logical_block_size(nullb->q, dev->blocksize);
2069         blk_queue_physical_block_size(nullb->q, dev->blocksize);
2070         if (!dev->max_sectors)
2071                 dev->max_sectors = queue_max_hw_sectors(nullb->q);
2072         dev->max_sectors = min_t(unsigned int, dev->max_sectors,
2073                                  BLK_DEF_MAX_SECTORS);
2074         blk_queue_max_hw_sectors(nullb->q, dev->max_sectors);
2075
2076         if (dev->virt_boundary)
2077                 blk_queue_virt_boundary(nullb->q, PAGE_SIZE - 1);
2078
2079         null_config_discard(nullb);
2080
2081         if (config_item_name(&dev->item)) {
2082                 /* Use configfs dir name as the device name */
2083                 snprintf(nullb->disk_name, sizeof(nullb->disk_name),
2084                          "%s", config_item_name(&dev->item));
2085         } else {
2086                 sprintf(nullb->disk_name, "nullb%d", nullb->index);
2087         }
2088
2089         rv = null_gendisk_register(nullb);
2090         if (rv)
2091                 goto out_cleanup_zone;
2092
2093         mutex_lock(&lock);
2094         list_add_tail(&nullb->list, &nullb_list);
2095         mutex_unlock(&lock);
2096
2097         pr_info("disk %s created\n", nullb->disk_name);
2098
2099         return 0;
2100 out_cleanup_zone:
2101         null_free_zoned_dev(dev);
2102 out_cleanup_disk:
2103         put_disk(nullb->disk);
2104 out_cleanup_tags:
2105         if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set)
2106                 blk_mq_free_tag_set(nullb->tag_set);
2107 out_cleanup_queues:
2108         cleanup_queues(nullb);
2109 out_free_nullb:
2110         kfree(nullb);
2111         dev->nullb = NULL;
2112 out:
2113         return rv;
2114 }
2115
2116 static struct nullb *null_find_dev_by_name(const char *name)
2117 {
2118         struct nullb *nullb = NULL, *nb;
2119
2120         mutex_lock(&lock);
2121         list_for_each_entry(nb, &nullb_list, list) {
2122                 if (strcmp(nb->disk_name, name) == 0) {
2123                         nullb = nb;
2124                         break;
2125                 }
2126         }
2127         mutex_unlock(&lock);
2128
2129         return nullb;
2130 }
2131
2132 static int null_create_dev(void)
2133 {
2134         struct nullb_device *dev;
2135         int ret;
2136
2137         dev = null_alloc_dev();
2138         if (!dev)
2139                 return -ENOMEM;
2140
2141         ret = null_add_dev(dev);
2142         if (ret) {
2143                 null_free_dev(dev);
2144                 return ret;
2145         }
2146
2147         return 0;
2148 }
2149
2150 static void null_destroy_dev(struct nullb *nullb)
2151 {
2152         struct nullb_device *dev = nullb->dev;
2153
2154         null_del_dev(nullb);
2155         null_free_dev(dev);
2156 }
2157
2158 static int __init null_init(void)
2159 {
2160         int ret = 0;
2161         unsigned int i;
2162         struct nullb *nullb;
2163
2164         if (g_bs > PAGE_SIZE) {
2165                 pr_warn("invalid block size\n");
2166                 pr_warn("defaults block size to %lu\n", PAGE_SIZE);
2167                 g_bs = PAGE_SIZE;
2168         }
2169
2170         if (g_max_sectors > BLK_DEF_MAX_SECTORS) {
2171                 pr_warn("invalid max sectors\n");
2172                 pr_warn("defaults max sectors to %u\n", BLK_DEF_MAX_SECTORS);
2173                 g_max_sectors = BLK_DEF_MAX_SECTORS;
2174         }
2175
2176         if (g_home_node != NUMA_NO_NODE && g_home_node >= nr_online_nodes) {
2177                 pr_err("invalid home_node value\n");
2178                 g_home_node = NUMA_NO_NODE;
2179         }
2180
2181         if (g_queue_mode == NULL_Q_RQ) {
2182                 pr_err("legacy IO path is no longer available\n");
2183                 return -EINVAL;
2184         }
2185
2186         if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) {
2187                 if (g_submit_queues != nr_online_nodes) {
2188                         pr_warn("submit_queues param is set to %u.\n",
2189                                 nr_online_nodes);
2190                         g_submit_queues = nr_online_nodes;
2191                 }
2192         } else if (g_submit_queues > nr_cpu_ids) {
2193                 g_submit_queues = nr_cpu_ids;
2194         } else if (g_submit_queues <= 0) {
2195                 g_submit_queues = 1;
2196         }
2197
2198         if (g_queue_mode == NULL_Q_MQ && shared_tags) {
2199                 ret = null_init_tag_set(NULL, &tag_set);
2200                 if (ret)
2201                         return ret;
2202         }
2203
2204         config_group_init(&nullb_subsys.su_group);
2205         mutex_init(&nullb_subsys.su_mutex);
2206
2207         ret = configfs_register_subsystem(&nullb_subsys);
2208         if (ret)
2209                 goto err_tagset;
2210
2211         mutex_init(&lock);
2212
2213         null_major = register_blkdev(0, "nullb");
2214         if (null_major < 0) {
2215                 ret = null_major;
2216                 goto err_conf;
2217         }
2218
2219         for (i = 0; i < nr_devices; i++) {
2220                 ret = null_create_dev();
2221                 if (ret)
2222                         goto err_dev;
2223         }
2224
2225         pr_info("module loaded\n");
2226         return 0;
2227
2228 err_dev:
2229         while (!list_empty(&nullb_list)) {
2230                 nullb = list_entry(nullb_list.next, struct nullb, list);
2231                 null_destroy_dev(nullb);
2232         }
2233         unregister_blkdev(null_major, "nullb");
2234 err_conf:
2235         configfs_unregister_subsystem(&nullb_subsys);
2236 err_tagset:
2237         if (g_queue_mode == NULL_Q_MQ && shared_tags)
2238                 blk_mq_free_tag_set(&tag_set);
2239         return ret;
2240 }
2241
2242 static void __exit null_exit(void)
2243 {
2244         struct nullb *nullb;
2245
2246         configfs_unregister_subsystem(&nullb_subsys);
2247
2248         unregister_blkdev(null_major, "nullb");
2249
2250         mutex_lock(&lock);
2251         while (!list_empty(&nullb_list)) {
2252                 nullb = list_entry(nullb_list.next, struct nullb, list);
2253                 null_destroy_dev(nullb);
2254         }
2255         mutex_unlock(&lock);
2256
2257         if (g_queue_mode == NULL_Q_MQ && shared_tags)
2258                 blk_mq_free_tag_set(&tag_set);
2259 }
2260
2261 module_init(null_init);
2262 module_exit(null_exit);
2263
2264 MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>");
2265 MODULE_LICENSE("GPL");