drivers/block/null_blk/main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and
   4  * Shaohua Li <shli@fb.com>
   5  */
   6 #include <linux/module.h>
   7
   8 #include <linux/moduleparam.h>
   9 #include <linux/sched.h>
  10 #include <linux/fs.h>
  11 #include <linux/init.h>
  12 #include "null_blk.h"
  13
  14 #undef pr_fmt
  15 #define pr_fmt(fmt)     "null_blk: " fmt
  16
  17 #define FREE_BATCH              16
  18
  19 #define TICKS_PER_SEC           50ULL
  20 #define TIMER_INTERVAL          (NSEC_PER_SEC / TICKS_PER_SEC)
  21
  22 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
  23 static DECLARE_FAULT_ATTR(null_timeout_attr);
  24 static DECLARE_FAULT_ATTR(null_requeue_attr);
  25 static DECLARE_FAULT_ATTR(null_init_hctx_attr);
  26 #endif
  27
  28 static inline u64 mb_per_tick(int mbps)
  29 {
  30         return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
  31 }
  32
  33 /*
  34  * Status flags for nullb_device.
  35  *
  36  * CONFIGURED:  Device has been configured and turned on. Cannot reconfigure.
  37  * UP:          Device is currently on and visible in userspace.
  38  * THROTTLED:   Device is being throttled.
  39  * CACHE:       Device is using a write-back cache.
  40  */
  41 enum nullb_device_flags {
  42         NULLB_DEV_FL_CONFIGURED = 0,
  43         NULLB_DEV_FL_UP         = 1,
  44         NULLB_DEV_FL_THROTTLED  = 2,
  45         NULLB_DEV_FL_CACHE      = 3,
  46 };
  47
  48 #define MAP_SZ          ((PAGE_SIZE >> SECTOR_SHIFT) + 2)
  49 /*
  50  * nullb_page is a page in memory for nullb devices.
  51  *
  52  * @page:       The page holding the data.
  53  * @bitmap:     The bitmap represents which sector in the page has data.
  54  *              Each bit represents one block size. For example, sector 8
  55  *              will use the 7th bit
  56  * The highest 2 bits of bitmap are for special purpose. LOCK means the cache
  57  * page is being flushing to storage. FREE means the cache page is freed and
  58  * should be skipped from flushing to storage. Please see
  59  * null_make_cache_space
  60  */
  61 struct nullb_page {
  62         struct page *page;
  63         DECLARE_BITMAP(bitmap, MAP_SZ);
  64 };
  65 #define NULLB_PAGE_LOCK (MAP_SZ - 1)
  66 #define NULLB_PAGE_FREE (MAP_SZ - 2)
  67
  68 static LIST_HEAD(nullb_list);
  69 static struct mutex lock;
  70 static int null_major;
  71 static DEFINE_IDA(nullb_indexes);
  72 static struct blk_mq_tag_set tag_set;
  73
  74 enum {
  75         NULL_IRQ_NONE           = 0,
  76         NULL_IRQ_SOFTIRQ        = 1,
  77         NULL_IRQ_TIMER          = 2,
  78 };
  79
  80 static bool g_virt_boundary = false;
  81 module_param_named(virt_boundary, g_virt_boundary, bool, 0444);
  82 MODULE_PARM_DESC(virt_boundary, "Require a virtual boundary for the device. Default: False");
  83
  84 static int g_no_sched;
  85 module_param_named(no_sched, g_no_sched, int, 0444);
  86 MODULE_PARM_DESC(no_sched, "No io scheduler");
  87
  88 static int g_submit_queues = 1;
  89 module_param_named(submit_queues, g_submit_queues, int, 0444);
  90 MODULE_PARM_DESC(submit_queues, "Number of submission queues");
  91
  92 static int g_poll_queues = 1;
  93 module_param_named(poll_queues, g_poll_queues, int, 0444);
  94 MODULE_PARM_DESC(poll_queues, "Number of IOPOLL submission queues");
  95
  96 static int g_home_node = NUMA_NO_NODE;
  97 module_param_named(home_node, g_home_node, int, 0444);
  98 MODULE_PARM_DESC(home_node, "Home node for the device");
  99
 100 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
 101 /*
 102  * For more details about fault injection, please refer to
 103  * Documentation/fault-injection/fault-injection.rst.
 104  */
 105 static char g_timeout_str[80];
 106 module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), 0444);
 107 MODULE_PARM_DESC(timeout, "Fault injection. timeout=<interval>,<probability>,<space>,<times>");
 108
 109 static char g_requeue_str[80];
 110 module_param_string(requeue, g_requeue_str, sizeof(g_requeue_str), 0444);
 111 MODULE_PARM_DESC(requeue, "Fault injection. requeue=<interval>,<probability>,<space>,<times>");
 112
 113 static char g_init_hctx_str[80];
 114 module_param_string(init_hctx, g_init_hctx_str, sizeof(g_init_hctx_str), 0444);
 115 MODULE_PARM_DESC(init_hctx, "Fault injection to fail hctx init. init_hctx=<interval>,<probability>,<space>,<times>");
 116 #endif
 117
 118 /*
 119  * Historic queue modes.
 120  *
 121  * These days nothing but NULL_Q_MQ is actually supported, but we keep it the
 122  * enum for error reporting.
 123  */
 124 enum {
 125         NULL_Q_BIO      = 0,
 126         NULL_Q_RQ       = 1,
 127         NULL_Q_MQ       = 2,
 128 };
 129
 130 static int g_queue_mode = NULL_Q_MQ;
 131
 132 static int null_param_store_val(const char *str, int *val, int min, int max)
 133 {
 134         int ret, new_val;
 135
 136         ret = kstrtoint(str, 10, &new_val);
 137         if (ret)
 138                 return -EINVAL;
 139
 140         if (new_val < min || new_val > max)
 141                 return -EINVAL;
 142
 143         *val = new_val;
 144         return 0;
 145 }
 146
 147 static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
 148 {
 149         return null_param_store_val(str, &g_queue_mode, NULL_Q_BIO, NULL_Q_MQ);
 150 }
 151
 152 static const struct kernel_param_ops null_queue_mode_param_ops = {
 153         .set    = null_set_queue_mode,
 154         .get    = param_get_int,
 155 };
 156
 157 device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, 0444);
 158 MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
 159
 160 static int g_gb = 250;
 161 module_param_named(gb, g_gb, int, 0444);
 162 MODULE_PARM_DESC(gb, "Size in GB");
 163
 164 static int g_bs = 512;
 165 module_param_named(bs, g_bs, int, 0444);
 166 MODULE_PARM_DESC(bs, "Block size (in bytes)");
 167
 168 static int g_max_sectors;
 169 module_param_named(max_sectors, g_max_sectors, int, 0444);
 170 MODULE_PARM_DESC(max_sectors, "Maximum size of a command (in 512B sectors)");
 171
 172 static unsigned int nr_devices = 1;
 173 module_param(nr_devices, uint, 0444);
 174 MODULE_PARM_DESC(nr_devices, "Number of devices to register");
 175
 176 static bool g_blocking;
 177 module_param_named(blocking, g_blocking, bool, 0444);
 178 MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
 179
 180 static bool g_shared_tags;
 181 module_param_named(shared_tags, g_shared_tags, bool, 0444);
 182 MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
 183
 184 static bool g_shared_tag_bitmap;
 185 module_param_named(shared_tag_bitmap, g_shared_tag_bitmap, bool, 0444);
 186 MODULE_PARM_DESC(shared_tag_bitmap, "Use shared tag bitmap for all submission queues for blk-mq");
 187
 188 static int g_irqmode = NULL_IRQ_SOFTIRQ;
 189
 190 static int null_set_irqmode(const char *str, const struct kernel_param *kp)
 191 {
 192         return null_param_store_val(str, &g_irqmode, NULL_IRQ_NONE,
 193                                         NULL_IRQ_TIMER);
 194 }
 195
 196 static const struct kernel_param_ops null_irqmode_param_ops = {
 197         .set    = null_set_irqmode,
 198         .get    = param_get_int,
 199 };
 200
 201 device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, 0444);
 202 MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
 203
 204 static unsigned long g_completion_nsec = 10000;
 205 module_param_named(completion_nsec, g_completion_nsec, ulong, 0444);
 206 MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
 207
 208 static int g_hw_queue_depth = 64;
 209 module_param_named(hw_queue_depth, g_hw_queue_depth, int, 0444);
 210 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
 211
 212 static bool g_use_per_node_hctx;
 213 module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, 0444);
 214 MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
 215
 216 static bool g_memory_backed;
 217 module_param_named(memory_backed, g_memory_backed, bool, 0444);
 218 MODULE_PARM_DESC(memory_backed, "Create a memory-backed block device. Default: false");
 219
 220 static bool g_discard;
 221 module_param_named(discard, g_discard, bool, 0444);
 222 MODULE_PARM_DESC(discard, "Support discard operations (requires memory-backed null_blk device). Default: false");
 223
 224 static unsigned long g_cache_size;
 225 module_param_named(cache_size, g_cache_size, ulong, 0444);
 226 MODULE_PARM_DESC(mbps, "Cache size in MiB for memory-backed device. Default: 0 (none)");
 227
 228 static bool g_fua = true;
 229 module_param_named(fua, g_fua, bool, 0444);
 230 MODULE_PARM_DESC(zoned, "Enable/disable FUA support when cache_size is used. Default: true");
 231
 232 static unsigned int g_mbps;
 233 module_param_named(mbps, g_mbps, uint, 0444);
 234 MODULE_PARM_DESC(mbps, "Limit maximum bandwidth (in MiB/s). Default: 0 (no limit)");
 235
 236 static bool g_zoned;
 237 module_param_named(zoned, g_zoned, bool, S_IRUGO);
 238 MODULE_PARM_DESC(zoned, "Make device as a host-managed zoned block device. Default: false");
 239
 240 static unsigned long g_zone_size = 256;
 241 module_param_named(zone_size, g_zone_size, ulong, S_IRUGO);
 242 MODULE_PARM_DESC(zone_size, "Zone size in MB when block device is zoned. Must be power-of-two: Default: 256");
 243
 244 static unsigned long g_zone_capacity;
 245 module_param_named(zone_capacity, g_zone_capacity, ulong, 0444);
 246 MODULE_PARM_DESC(zone_capacity, "Zone capacity in MB when block device is zoned. Can be less than or equal to zone size. Default: Zone size");
 247
 248 static unsigned int g_zone_nr_conv;
 249 module_param_named(zone_nr_conv, g_zone_nr_conv, uint, 0444);
 250 MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones when block device is zoned. Default: 0");
 251
 252 static unsigned int g_zone_max_open;
 253 module_param_named(zone_max_open, g_zone_max_open, uint, 0444);
 254 MODULE_PARM_DESC(zone_max_open, "Maximum number of open zones when block device is zoned. Default: 0 (no limit)");
 255
 256 static unsigned int g_zone_max_active;
 257 module_param_named(zone_max_active, g_zone_max_active, uint, 0444);
 258 MODULE_PARM_DESC(zone_max_active, "Maximum number of active zones when block device is zoned. Default: 0 (no limit)");
 259
 260 static int g_zone_append_max_sectors = INT_MAX;
 261 module_param_named(zone_append_max_sectors, g_zone_append_max_sectors, int, 0444);
 262 MODULE_PARM_DESC(zone_append_max_sectors,
 263                  "Maximum size of a zone append command (in 512B sectors). Specify 0 for zone append emulation");
 264
 265 static struct nullb_device *null_alloc_dev(void);
 266 static void null_free_dev(struct nullb_device *dev);
 267 static void null_del_dev(struct nullb *nullb);
 268 static int null_add_dev(struct nullb_device *dev);
 269 static struct nullb *null_find_dev_by_name(const char *name);
 270 static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
 271
 272 static inline struct nullb_device *to_nullb_device(struct config_item *item)
 273 {
 274         return item ? container_of(to_config_group(item), struct nullb_device, group) : NULL;
 275 }
 276
 277 static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
 278 {
 279         return snprintf(page, PAGE_SIZE, "%u\n", val);
 280 }
 281
 282 static inline ssize_t nullb_device_ulong_attr_show(unsigned long val,
 283         char *page)
 284 {
 285         return snprintf(page, PAGE_SIZE, "%lu\n", val);
 286 }
 287
 288 static inline ssize_t nullb_device_bool_attr_show(bool val, char *page)
 289 {
 290         return snprintf(page, PAGE_SIZE, "%u\n", val);
 291 }
 292
 293 static ssize_t nullb_device_uint_attr_store(unsigned int *val,
 294         const char *page, size_t count)
 295 {
 296         unsigned int tmp;
 297         int result;
 298
 299         result = kstrtouint(page, 0, &tmp);
 300         if (result < 0)
 301                 return result;
 302
 303         *val = tmp;
 304         return count;
 305 }
 306
 307 static ssize_t nullb_device_ulong_attr_store(unsigned long *val,
 308         const char *page, size_t count)
 309 {
 310         int result;
 311         unsigned long tmp;
 312
 313         result = kstrtoul(page, 0, &tmp);
 314         if (result < 0)
 315                 return result;
 316
 317         *val = tmp;
 318         return count;
 319 }
 320
 321 static ssize_t nullb_device_bool_attr_store(bool *val, const char *page,
 322         size_t count)
 323 {
 324         bool tmp;
 325         int result;
 326
 327         result = kstrtobool(page,  &tmp);
 328         if (result < 0)
 329                 return result;
 330
 331         *val = tmp;
 332         return count;
 333 }
 334
 335 /* The following macro should only be used with TYPE = {uint, ulong, bool}. */
 336 #define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY)                            \
 337 static ssize_t                                                          \
 338 nullb_device_##NAME##_show(struct config_item *item, char *page)        \
 339 {                                                                       \
 340         return nullb_device_##TYPE##_attr_show(                         \
 341                                 to_nullb_device(item)->NAME, page);     \
 342 }                                                                       \
 343 static ssize_t                                                          \
 344 nullb_device_##NAME##_store(struct config_item *item, const char *page, \
 345                             size_t count)                               \
 346 {                                                                       \
 347         int (*apply_fn)(struct nullb_device *dev, TYPE new_value) = APPLY;\
 348         struct nullb_device *dev = to_nullb_device(item);               \
 349         TYPE new_value = 0;                                             \
 350         int ret;                                                        \
 351                                                                         \
 352         ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\
 353         if (ret < 0)                                                    \
 354                 return ret;                                             \
 355         if (apply_fn)                                                   \
 356                 ret = apply_fn(dev, new_value);                         \
 357         else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags))        \
 358                 ret = -EBUSY;                                           \
 359         if (ret < 0)                                                    \
 360                 return ret;                                             \
 361         dev->NAME = new_value;                                          \
 362         return count;                                                   \
 363 }                                                                       \
 364 CONFIGFS_ATTR(nullb_device_, NAME);
 365
 366 static int nullb_update_nr_hw_queues(struct nullb_device *dev,
 367                                      unsigned int submit_queues,
 368                                      unsigned int poll_queues)
 369
 370 {
 371         struct blk_mq_tag_set *set;
 372         int ret, nr_hw_queues;
 373
 374         if (!dev->nullb)
 375                 return 0;
 376
 377         /*
 378          * Make sure at least one submit queue exists.
 379          */
 380         if (!submit_queues)
 381                 return -EINVAL;
 382
 383         /*
 384          * Make sure that null_init_hctx() does not access nullb->queues[] past
 385          * the end of that array.
 386          */
 387         if (submit_queues > nr_cpu_ids || poll_queues > g_poll_queues)
 388                 return -EINVAL;
 389
 390         /*
 391          * Keep previous and new queue numbers in nullb_device for reference in
 392          * the call back function null_map_queues().
 393          */
 394         dev->prev_submit_queues = dev->submit_queues;
 395         dev->prev_poll_queues = dev->poll_queues;
 396         dev->submit_queues = submit_queues;
 397         dev->poll_queues = poll_queues;
 398
 399         set = dev->nullb->tag_set;
 400         nr_hw_queues = submit_queues + poll_queues;
 401         blk_mq_update_nr_hw_queues(set, nr_hw_queues);
 402         ret = set->nr_hw_queues == nr_hw_queues ? 0 : -ENOMEM;
 403
 404         if (ret) {
 405                 /* on error, revert the queue numbers */
 406                 dev->submit_queues = dev->prev_submit_queues;
 407                 dev->poll_queues = dev->prev_poll_queues;
 408         }
 409
 410         return ret;
 411 }
 412
 413 static int nullb_apply_submit_queues(struct nullb_device *dev,
 414                                      unsigned int submit_queues)
 415 {
 416         return nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues);
 417 }
 418
 419 static int nullb_apply_poll_queues(struct nullb_device *dev,
 420                                    unsigned int poll_queues)
 421 {
 422         return nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues);
 423 }
 424
 425 NULLB_DEVICE_ATTR(size, ulong, NULL);
 426 NULLB_DEVICE_ATTR(completion_nsec, ulong, NULL);
 427 NULLB_DEVICE_ATTR(submit_queues, uint, nullb_apply_submit_queues);
 428 NULLB_DEVICE_ATTR(poll_queues, uint, nullb_apply_poll_queues);
 429 NULLB_DEVICE_ATTR(home_node, uint, NULL);
 430 NULLB_DEVICE_ATTR(queue_mode, uint, NULL);
 431 NULLB_DEVICE_ATTR(blocksize, uint, NULL);
 432 NULLB_DEVICE_ATTR(max_sectors, uint, NULL);
 433 NULLB_DEVICE_ATTR(irqmode, uint, NULL);
 434 NULLB_DEVICE_ATTR(hw_queue_depth, uint, NULL);
 435 NULLB_DEVICE_ATTR(index, uint, NULL);
 436 NULLB_DEVICE_ATTR(blocking, bool, NULL);
 437 NULLB_DEVICE_ATTR(use_per_node_hctx, bool, NULL);
 438 NULLB_DEVICE_ATTR(memory_backed, bool, NULL);
 439 NULLB_DEVICE_ATTR(discard, bool, NULL);
 440 NULLB_DEVICE_ATTR(mbps, uint, NULL);
 441 NULLB_DEVICE_ATTR(cache_size, ulong, NULL);
 442 NULLB_DEVICE_ATTR(zoned, bool, NULL);
 443 NULLB_DEVICE_ATTR(zone_size, ulong, NULL);
 444 NULLB_DEVICE_ATTR(zone_capacity, ulong, NULL);
 445 NULLB_DEVICE_ATTR(zone_nr_conv, uint, NULL);
 446 NULLB_DEVICE_ATTR(zone_max_open, uint, NULL);
 447 NULLB_DEVICE_ATTR(zone_max_active, uint, NULL);
 448 NULLB_DEVICE_ATTR(zone_append_max_sectors, uint, NULL);
 449 NULLB_DEVICE_ATTR(virt_boundary, bool, NULL);
 450 NULLB_DEVICE_ATTR(no_sched, bool, NULL);
 451 NULLB_DEVICE_ATTR(shared_tags, bool, NULL);
 452 NULLB_DEVICE_ATTR(shared_tag_bitmap, bool, NULL);
 453 NULLB_DEVICE_ATTR(fua, bool, NULL);
 454
 455 static ssize_t nullb_device_power_show(struct config_item *item, char *page)
 456 {
 457         return nullb_device_bool_attr_show(to_nullb_device(item)->power, page);
 458 }
 459
 460 static ssize_t nullb_device_power_store(struct config_item *item,
 461                                      const char *page, size_t count)
 462 {
 463         struct nullb_device *dev = to_nullb_device(item);
 464         bool newp = false;
 465         ssize_t ret;
 466
 467         ret = nullb_device_bool_attr_store(&newp, page, count);
 468         if (ret < 0)
 469                 return ret;
 470
 471         if (!dev->power && newp) {
 472                 if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags))
 473                         return count;
 474                 ret = null_add_dev(dev);
 475                 if (ret) {
 476                         clear_bit(NULLB_DEV_FL_UP, &dev->flags);
 477                         return ret;
 478                 }
 479
 480                 set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
 481                 dev->power = newp;
 482         } else if (dev->power && !newp) {
 483                 if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
 484                         mutex_lock(&lock);
 485                         dev->power = newp;
 486                         null_del_dev(dev->nullb);
 487                         mutex_unlock(&lock);
 488                 }
 489                 clear_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
 490         }
 491
 492         return count;
 493 }
 494
 495 CONFIGFS_ATTR(nullb_device_, power);
 496
 497 static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page)
 498 {
 499         struct nullb_device *t_dev = to_nullb_device(item);
 500
 501         return badblocks_show(&t_dev->badblocks, page, 0);
 502 }
 503
 504 static ssize_t nullb_device_badblocks_store(struct config_item *item,
 505                                      const char *page, size_t count)
 506 {
 507         struct nullb_device *t_dev = to_nullb_device(item);
 508         char *orig, *buf, *tmp;
 509         u64 start, end;
 510         int ret;
 511
 512         orig = kstrndup(page, count, GFP_KERNEL);
 513         if (!orig)
 514                 return -ENOMEM;
 515
 516         buf = strstrip(orig);
 517
 518         ret = -EINVAL;
 519         if (buf[0] != '+' && buf[0] != '-')
 520                 goto out;
 521         tmp = strchr(&buf[1], '-');
 522         if (!tmp)
 523                 goto out;
 524         *tmp = '\0';
 525         ret = kstrtoull(buf + 1, 0, &start);
 526         if (ret)
 527                 goto out;
 528         ret = kstrtoull(tmp + 1, 0, &end);
 529         if (ret)
 530                 goto out;
 531         ret = -EINVAL;
 532         if (start > end)
 533                 goto out;
 534         /* enable badblocks */
 535         cmpxchg(&t_dev->badblocks.shift, -1, 0);
 536         if (buf[0] == '+')
 537                 ret = badblocks_set(&t_dev->badblocks, start,
 538                         end - start + 1, 1);
 539         else
 540                 ret = badblocks_clear(&t_dev->badblocks, start,
 541                         end - start + 1);
 542         if (ret == 0)
 543                 ret = count;
 544 out:
 545         kfree(orig);
 546         return ret;
 547 }
 548 CONFIGFS_ATTR(nullb_device_, badblocks);
 549
 550 static ssize_t nullb_device_zone_readonly_store(struct config_item *item,
 551                                                 const char *page, size_t count)
 552 {
 553         struct nullb_device *dev = to_nullb_device(item);
 554
 555         return zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY);
 556 }
 557 CONFIGFS_ATTR_WO(nullb_device_, zone_readonly);
 558
 559 static ssize_t nullb_device_zone_offline_store(struct config_item *item,
 560                                                const char *page, size_t count)
 561 {
 562         struct nullb_device *dev = to_nullb_device(item);
 563
 564         return zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE);
 565 }
 566 CONFIGFS_ATTR_WO(nullb_device_, zone_offline);
 567
 568 static struct configfs_attribute *nullb_device_attrs[] = {
 569         &nullb_device_attr_size,
 570         &nullb_device_attr_completion_nsec,
 571         &nullb_device_attr_submit_queues,
 572         &nullb_device_attr_poll_queues,
 573         &nullb_device_attr_home_node,
 574         &nullb_device_attr_queue_mode,
 575         &nullb_device_attr_blocksize,
 576         &nullb_device_attr_max_sectors,
 577         &nullb_device_attr_irqmode,
 578         &nullb_device_attr_hw_queue_depth,
 579         &nullb_device_attr_index,
 580         &nullb_device_attr_blocking,
 581         &nullb_device_attr_use_per_node_hctx,
 582         &nullb_device_attr_power,
 583         &nullb_device_attr_memory_backed,
 584         &nullb_device_attr_discard,
 585         &nullb_device_attr_mbps,
 586         &nullb_device_attr_cache_size,
 587         &nullb_device_attr_badblocks,
 588         &nullb_device_attr_zoned,
 589         &nullb_device_attr_zone_size,
 590         &nullb_device_attr_zone_capacity,
 591         &nullb_device_attr_zone_nr_conv,
 592         &nullb_device_attr_zone_max_open,
 593         &nullb_device_attr_zone_max_active,
 594         &nullb_device_attr_zone_append_max_sectors,
 595         &nullb_device_attr_zone_readonly,
 596         &nullb_device_attr_zone_offline,
 597         &nullb_device_attr_virt_boundary,
 598         &nullb_device_attr_no_sched,
 599         &nullb_device_attr_shared_tags,
 600         &nullb_device_attr_shared_tag_bitmap,
 601         &nullb_device_attr_fua,
 602         NULL,
 603 };
 604
 605 static void nullb_device_release(struct config_item *item)
 606 {
 607         struct nullb_device *dev = to_nullb_device(item);
 608
 609         null_free_device_storage(dev, false);
 610         null_free_dev(dev);
 611 }
 612
 613 static struct configfs_item_operations nullb_device_ops = {
 614         .release        = nullb_device_release,
 615 };
 616
 617 static const struct config_item_type nullb_device_type = {
 618         .ct_item_ops    = &nullb_device_ops,
 619         .ct_attrs       = nullb_device_attrs,
 620         .ct_owner       = THIS_MODULE,
 621 };
 622
 623 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
 624
 625 static void nullb_add_fault_config(struct nullb_device *dev)
 626 {
 627         fault_config_init(&dev->timeout_config, "timeout_inject");
 628         fault_config_init(&dev->requeue_config, "requeue_inject");
 629         fault_config_init(&dev->init_hctx_fault_config, "init_hctx_fault_inject");
 630
 631         configfs_add_default_group(&dev->timeout_config.group, &dev->group);
 632         configfs_add_default_group(&dev->requeue_config.group, &dev->group);
 633         configfs_add_default_group(&dev->init_hctx_fault_config.group, &dev->group);
 634 }
 635
 636 #else
 637
 638 static void nullb_add_fault_config(struct nullb_device *dev)
 639 {
 640 }
 641
 642 #endif
 643
 644 static struct
 645 config_group *nullb_group_make_group(struct config_group *group, const char *name)
 646 {
 647         struct nullb_device *dev;
 648
 649         if (null_find_dev_by_name(name))
 650                 return ERR_PTR(-EEXIST);
 651
 652         dev = null_alloc_dev();
 653         if (!dev)
 654                 return ERR_PTR(-ENOMEM);
 655
 656         config_group_init_type_name(&dev->group, name, &nullb_device_type);
 657         nullb_add_fault_config(dev);
 658
 659         return &dev->group;
 660 }
 661
 662 static void
 663 nullb_group_drop_item(struct config_group *group, struct config_item *item)
 664 {
 665         struct nullb_device *dev = to_nullb_device(item);
 666
 667         if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
 668                 mutex_lock(&lock);
 669                 dev->power = false;
 670                 null_del_dev(dev->nullb);
 671                 mutex_unlock(&lock);
 672         }
 673
 674         config_item_put(item);
 675 }
 676
 677 static ssize_t memb_group_features_show(struct config_item *item, char *page)
 678 {
 679         return snprintf(page, PAGE_SIZE,
 680                         "badblocks,blocking,blocksize,cache_size,fua,"
 681                         "completion_nsec,discard,home_node,hw_queue_depth,"
 682                         "irqmode,max_sectors,mbps,memory_backed,no_sched,"
 683                         "poll_queues,power,queue_mode,shared_tag_bitmap,"
 684                         "shared_tags,size,submit_queues,use_per_node_hctx,"
 685                         "virt_boundary,zoned,zone_capacity,zone_max_active,"
 686                         "zone_max_open,zone_nr_conv,zone_offline,zone_readonly,"
 687                         "zone_size,zone_append_max_sectors\n");
 688 }
 689
 690 CONFIGFS_ATTR_RO(memb_group_, features);
 691
 692 static struct configfs_attribute *nullb_group_attrs[] = {
 693         &memb_group_attr_features,
 694         NULL,
 695 };
 696
 697 static struct configfs_group_operations nullb_group_ops = {
 698         .make_group     = nullb_group_make_group,
 699         .drop_item      = nullb_group_drop_item,
 700 };
 701
 702 static const struct config_item_type nullb_group_type = {
 703         .ct_group_ops   = &nullb_group_ops,
 704         .ct_attrs       = nullb_group_attrs,
 705         .ct_owner       = THIS_MODULE,
 706 };
 707
 708 static struct configfs_subsystem nullb_subsys = {
 709         .su_group = {
 710                 .cg_item = {
 711                         .ci_namebuf = "nullb",
 712                         .ci_type = &nullb_group_type,
 713                 },
 714         },
 715 };
 716
 717 static inline int null_cache_active(struct nullb *nullb)
 718 {
 719         return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
 720 }
 721
 722 static struct nullb_device *null_alloc_dev(void)
 723 {
 724         struct nullb_device *dev;
 725
 726         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 727         if (!dev)
 728                 return NULL;
 729
 730 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
 731         dev->timeout_config.attr = null_timeout_attr;
 732         dev->requeue_config.attr = null_requeue_attr;
 733         dev->init_hctx_fault_config.attr = null_init_hctx_attr;
 734 #endif
 735
 736         INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
 737         INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
 738         if (badblocks_init(&dev->badblocks, 0)) {
 739                 kfree(dev);
 740                 return NULL;
 741         }
 742
 743         dev->size = g_gb * 1024;
 744         dev->completion_nsec = g_completion_nsec;
 745         dev->submit_queues = g_submit_queues;
 746         dev->prev_submit_queues = g_submit_queues;
 747         dev->poll_queues = g_poll_queues;
 748         dev->prev_poll_queues = g_poll_queues;
 749         dev->home_node = g_home_node;
 750         dev->queue_mode = g_queue_mode;
 751         dev->blocksize = g_bs;
 752         dev->max_sectors = g_max_sectors;
 753         dev->irqmode = g_irqmode;
 754         dev->hw_queue_depth = g_hw_queue_depth;
 755         dev->blocking = g_blocking;
 756         dev->memory_backed = g_memory_backed;
 757         dev->discard = g_discard;
 758         dev->cache_size = g_cache_size;
 759         dev->mbps = g_mbps;
 760         dev->use_per_node_hctx = g_use_per_node_hctx;
 761         dev->zoned = g_zoned;
 762         dev->zone_size = g_zone_size;
 763         dev->zone_capacity = g_zone_capacity;
 764         dev->zone_nr_conv = g_zone_nr_conv;
 765         dev->zone_max_open = g_zone_max_open;
 766         dev->zone_max_active = g_zone_max_active;
 767         dev->zone_append_max_sectors = g_zone_append_max_sectors;
 768         dev->virt_boundary = g_virt_boundary;
 769         dev->no_sched = g_no_sched;
 770         dev->shared_tags = g_shared_tags;
 771         dev->shared_tag_bitmap = g_shared_tag_bitmap;
 772         dev->fua = g_fua;
 773
 774         return dev;
 775 }
 776
 777 static void null_free_dev(struct nullb_device *dev)
 778 {
 779         if (!dev)
 780                 return;
 781
 782         null_free_zoned_dev(dev);
 783         badblocks_exit(&dev->badblocks);
 784         kfree(dev);
 785 }
 786
 787 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
 788 {
 789         struct nullb_cmd *cmd = container_of(timer, struct nullb_cmd, timer);
 790
 791         blk_mq_end_request(blk_mq_rq_from_pdu(cmd), cmd->error);
 792         return HRTIMER_NORESTART;
 793 }
 794
 795 static void null_cmd_end_timer(struct nullb_cmd *cmd)
 796 {
 797         ktime_t kt = cmd->nq->dev->completion_nsec;
 798
 799         hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL);
 800 }
 801
 802 static void null_complete_rq(struct request *rq)
 803 {
 804         struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
 805
 806         blk_mq_end_request(rq, cmd->error);
 807 }
 808
 809 static struct nullb_page *null_alloc_page(void)
 810 {
 811         struct nullb_page *t_page;
 812
 813         t_page = kmalloc(sizeof(struct nullb_page), GFP_NOIO);
 814         if (!t_page)
 815                 return NULL;
 816
 817         t_page->page = alloc_pages(GFP_NOIO, 0);
 818         if (!t_page->page) {
 819                 kfree(t_page);
 820                 return NULL;
 821         }
 822
 823         memset(t_page->bitmap, 0, sizeof(t_page->bitmap));
 824         return t_page;
 825 }
 826
 827 static void null_free_page(struct nullb_page *t_page)
 828 {
 829         __set_bit(NULLB_PAGE_FREE, t_page->bitmap);
 830         if (test_bit(NULLB_PAGE_LOCK, t_page->bitmap))
 831                 return;
 832         __free_page(t_page->page);
 833         kfree(t_page);
 834 }
 835
 836 static bool null_page_empty(struct nullb_page *page)
 837 {
 838         int size = MAP_SZ - 2;
 839
 840         return find_first_bit(page->bitmap, size) == size;
 841 }
 842
 843 static void null_free_sector(struct nullb *nullb, sector_t sector,
 844         bool is_cache)
 845 {
 846         unsigned int sector_bit;
 847         u64 idx;
 848         struct nullb_page *t_page, *ret;
 849         struct radix_tree_root *root;
 850
 851         root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 852         idx = sector >> PAGE_SECTORS_SHIFT;
 853         sector_bit = (sector & SECTOR_MASK);
 854
 855         t_page = radix_tree_lookup(root, idx);
 856         if (t_page) {
 857                 __clear_bit(sector_bit, t_page->bitmap);
 858
 859                 if (null_page_empty(t_page)) {
 860                         ret = radix_tree_delete_item(root, idx, t_page);
 861                         WARN_ON(ret != t_page);
 862                         null_free_page(ret);
 863                         if (is_cache)
 864                                 nullb->dev->curr_cache -= PAGE_SIZE;
 865                 }
 866         }
 867 }
 868
 869 static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,
 870         struct nullb_page *t_page, bool is_cache)
 871 {
 872         struct radix_tree_root *root;
 873
 874         root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 875
 876         if (radix_tree_insert(root, idx, t_page)) {
 877                 null_free_page(t_page);
 878                 t_page = radix_tree_lookup(root, idx);
 879                 WARN_ON(!t_page || t_page->page->index != idx);
 880         } else if (is_cache)
 881                 nullb->dev->curr_cache += PAGE_SIZE;
 882
 883         return t_page;
 884 }
 885
 886 static void null_free_device_storage(struct nullb_device *dev, bool is_cache)
 887 {
 888         unsigned long pos = 0;
 889         int nr_pages;
 890         struct nullb_page *ret, *t_pages[FREE_BATCH];
 891         struct radix_tree_root *root;
 892
 893         root = is_cache ? &dev->cache : &dev->data;
 894
 895         do {
 896                 int i;
 897
 898                 nr_pages = radix_tree_gang_lookup(root,
 899                                 (void **)t_pages, pos, FREE_BATCH);
 900
 901                 for (i = 0; i < nr_pages; i++) {
 902                         pos = t_pages[i]->page->index;
 903                         ret = radix_tree_delete_item(root, pos, t_pages[i]);
 904                         WARN_ON(ret != t_pages[i]);
 905                         null_free_page(ret);
 906                 }
 907
 908                 pos++;
 909         } while (nr_pages == FREE_BATCH);
 910
 911         if (is_cache)
 912                 dev->curr_cache = 0;
 913 }
 914
 915 static struct nullb_page *__null_lookup_page(struct nullb *nullb,
 916         sector_t sector, bool for_write, bool is_cache)
 917 {
 918         unsigned int sector_bit;
 919         u64 idx;
 920         struct nullb_page *t_page;
 921         struct radix_tree_root *root;
 922
 923         idx = sector >> PAGE_SECTORS_SHIFT;
 924         sector_bit = (sector & SECTOR_MASK);
 925
 926         root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 927         t_page = radix_tree_lookup(root, idx);
 928         WARN_ON(t_page && t_page->page->index != idx);
 929
 930         if (t_page && (for_write || test_bit(sector_bit, t_page->bitmap)))
 931                 return t_page;
 932
 933         return NULL;
 934 }
 935
 936 static struct nullb_page *null_lookup_page(struct nullb *nullb,
 937         sector_t sector, bool for_write, bool ignore_cache)
 938 {
 939         struct nullb_page *page = NULL;
 940
 941         if (!ignore_cache)
 942                 page = __null_lookup_page(nullb, sector, for_write, true);
 943         if (page)
 944                 return page;
 945         return __null_lookup_page(nullb, sector, for_write, false);
 946 }
 947
 948 static struct nullb_page *null_insert_page(struct nullb *nullb,
 949                                            sector_t sector, bool ignore_cache)
 950         __releases(&nullb->lock)
 951         __acquires(&nullb->lock)
 952 {
 953         u64 idx;
 954         struct nullb_page *t_page;
 955
 956         t_page = null_lookup_page(nullb, sector, true, ignore_cache);
 957         if (t_page)
 958                 return t_page;
 959
 960         spin_unlock_irq(&nullb->lock);
 961
 962         t_page = null_alloc_page();
 963         if (!t_page)
 964                 goto out_lock;
 965
 966         if (radix_tree_preload(GFP_NOIO))
 967                 goto out_freepage;
 968
 969         spin_lock_irq(&nullb->lock);
 970         idx = sector >> PAGE_SECTORS_SHIFT;
 971         t_page->page->index = idx;
 972         t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache);
 973         radix_tree_preload_end();
 974
 975         return t_page;
 976 out_freepage:
 977         null_free_page(t_page);
 978 out_lock:
 979         spin_lock_irq(&nullb->lock);
 980         return null_lookup_page(nullb, sector, true, ignore_cache);
 981 }
 982
 983 static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
 984 {
 985         int i;
 986         unsigned int offset;
 987         u64 idx;
 988         struct nullb_page *t_page, *ret;
 989         void *dst, *src;
 990
 991         idx = c_page->page->index;
 992
 993         t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true);
 994
 995         __clear_bit(NULLB_PAGE_LOCK, c_page->bitmap);
 996         if (test_bit(NULLB_PAGE_FREE, c_page->bitmap)) {
 997                 null_free_page(c_page);
 998                 if (t_page && null_page_empty(t_page)) {
 999                         ret = radix_tree_delete_item(&nullb->dev->data,
1000                                 idx, t_page);
1001                         null_free_page(t_page);
1002                 }
1003                 return 0;
1004         }
1005
1006         if (!t_page)
1007                 return -ENOMEM;
1008
1009         src = kmap_local_page(c_page->page);
1010         dst = kmap_local_page(t_page->page);
1011
1012         for (i = 0; i < PAGE_SECTORS;
1013                         i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
1014                 if (test_bit(i, c_page->bitmap)) {
1015                         offset = (i << SECTOR_SHIFT);
1016                         memcpy(dst + offset, src + offset,
1017                                 nullb->dev->blocksize);
1018                         __set_bit(i, t_page->bitmap);
1019                 }
1020         }
1021
1022         kunmap_local(dst);
1023         kunmap_local(src);
1024
1025         ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
1026         null_free_page(ret);
1027         nullb->dev->curr_cache -= PAGE_SIZE;
1028
1029         return 0;
1030 }
1031
1032 static int null_make_cache_space(struct nullb *nullb, unsigned long n)
1033 {
1034         int i, err, nr_pages;
1035         struct nullb_page *c_pages[FREE_BATCH];
1036         unsigned long flushed = 0, one_round;
1037
1038 again:
1039         if ((nullb->dev->cache_size * 1024 * 1024) >
1040              nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)
1041                 return 0;
1042
1043         nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,
1044                         (void **)c_pages, nullb->cache_flush_pos, FREE_BATCH);
1045         /*
1046          * nullb_flush_cache_page could unlock before using the c_pages. To
1047          * avoid race, we don't allow page free
1048          */
1049         for (i = 0; i < nr_pages; i++) {
1050                 nullb->cache_flush_pos = c_pages[i]->page->index;
1051                 /*
1052                  * We found the page which is being flushed to disk by other
1053                  * threads
1054                  */
1055                 if (test_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap))
1056                         c_pages[i] = NULL;
1057                 else
1058                         __set_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap);
1059         }
1060
1061         one_round = 0;
1062         for (i = 0; i < nr_pages; i++) {
1063                 if (c_pages[i] == NULL)
1064                         continue;
1065                 err = null_flush_cache_page(nullb, c_pages[i]);
1066                 if (err)
1067                         return err;
1068                 one_round++;
1069         }
1070         flushed += one_round << PAGE_SHIFT;
1071
1072         if (n > flushed) {
1073                 if (nr_pages == 0)
1074                         nullb->cache_flush_pos = 0;
1075                 if (one_round == 0) {
1076                         /* give other threads a chance */
1077                         spin_unlock_irq(&nullb->lock);
1078                         spin_lock_irq(&nullb->lock);
1079                 }
1080                 goto again;
1081         }
1082         return 0;
1083 }
1084
1085 static int copy_to_nullb(struct nullb *nullb, struct page *source,
1086         unsigned int off, sector_t sector, size_t n, bool is_fua)
1087 {
1088         size_t temp, count = 0;
1089         unsigned int offset;
1090         struct nullb_page *t_page;
1091
1092         while (count < n) {
1093                 temp = min_t(size_t, nullb->dev->blocksize, n - count);
1094
1095                 if (null_cache_active(nullb) && !is_fua)
1096                         null_make_cache_space(nullb, PAGE_SIZE);
1097
1098                 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
1099                 t_page = null_insert_page(nullb, sector,
1100                         !null_cache_active(nullb) || is_fua);
1101                 if (!t_page)
1102                         return -ENOSPC;
1103
1104                 memcpy_page(t_page->page, offset, source, off + count, temp);
1105
1106                 __set_bit(sector & SECTOR_MASK, t_page->bitmap);
1107
1108                 if (is_fua)
1109                         null_free_sector(nullb, sector, true);
1110
1111                 count += temp;
1112                 sector += temp >> SECTOR_SHIFT;
1113         }
1114         return 0;
1115 }
1116
1117 static int copy_from_nullb(struct nullb *nullb, struct page *dest,
1118         unsigned int off, sector_t sector, size_t n)
1119 {
1120         size_t temp, count = 0;
1121         unsigned int offset;
1122         struct nullb_page *t_page;
1123
1124         while (count < n) {
1125                 temp = min_t(size_t, nullb->dev->blocksize, n - count);
1126
1127                 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
1128                 t_page = null_lookup_page(nullb, sector, false,
1129                         !null_cache_active(nullb));
1130
1131                 if (t_page)
1132                         memcpy_page(dest, off + count, t_page->page, offset,
1133                                     temp);
1134                 else
1135                         zero_user(dest, off + count, temp);
1136
1137                 count += temp;
1138                 sector += temp >> SECTOR_SHIFT;
1139         }
1140         return 0;
1141 }
1142
1143 static void nullb_fill_pattern(struct nullb *nullb, struct page *page,
1144                                unsigned int len, unsigned int off)
1145 {
1146         memset_page(page, off, 0xff, len);
1147 }
1148
1149 blk_status_t null_handle_discard(struct nullb_device *dev,
1150                                  sector_t sector, sector_t nr_sectors)
1151 {
1152         struct nullb *nullb = dev->nullb;
1153         size_t n = nr_sectors << SECTOR_SHIFT;
1154         size_t temp;
1155
1156         spin_lock_irq(&nullb->lock);
1157         while (n > 0) {
1158                 temp = min_t(size_t, n, dev->blocksize);
1159                 null_free_sector(nullb, sector, false);
1160                 if (null_cache_active(nullb))
1161                         null_free_sector(nullb, sector, true);
1162                 sector += temp >> SECTOR_SHIFT;
1163                 n -= temp;
1164         }
1165         spin_unlock_irq(&nullb->lock);
1166
1167         return BLK_STS_OK;
1168 }
1169
1170 static blk_status_t null_handle_flush(struct nullb *nullb)
1171 {
1172         int err;
1173
1174         if (!null_cache_active(nullb))
1175                 return 0;
1176
1177         spin_lock_irq(&nullb->lock);
1178         while (true) {
1179                 err = null_make_cache_space(nullb,
1180                         nullb->dev->cache_size * 1024 * 1024);
1181                 if (err || nullb->dev->curr_cache == 0)
1182                         break;
1183         }
1184
1185         WARN_ON(!radix_tree_empty(&nullb->dev->cache));
1186         spin_unlock_irq(&nullb->lock);
1187         return errno_to_blk_status(err);
1188 }
1189
1190 static int null_transfer(struct nullb *nullb, struct page *page,
1191         unsigned int len, unsigned int off, bool is_write, sector_t sector,
1192         bool is_fua)
1193 {
1194         struct nullb_device *dev = nullb->dev;
1195         unsigned int valid_len = len;
1196         int err = 0;
1197
1198         if (!is_write) {
1199                 if (dev->zoned)
1200                         valid_len = null_zone_valid_read_len(nullb,
1201                                 sector, len);
1202
1203                 if (valid_len) {
1204                         err = copy_from_nullb(nullb, page, off,
1205                                 sector, valid_len);
1206                         off += valid_len;
1207                         len -= valid_len;
1208                 }
1209
1210                 if (len)
1211                         nullb_fill_pattern(nullb, page, len, off);
1212                 flush_dcache_page(page);
1213         } else {
1214                 flush_dcache_page(page);
1215                 err = copy_to_nullb(nullb, page, off, sector, len, is_fua);
1216         }
1217
1218         return err;
1219 }
1220
1221 static int null_handle_rq(struct nullb_cmd *cmd)
1222 {
1223         struct request *rq = blk_mq_rq_from_pdu(cmd);
1224         struct nullb *nullb = cmd->nq->dev->nullb;
1225         int err = 0;
1226         unsigned int len;
1227         sector_t sector = blk_rq_pos(rq);
1228         struct req_iterator iter;
1229         struct bio_vec bvec;
1230
1231         spin_lock_irq(&nullb->lock);
1232         rq_for_each_segment(bvec, rq, iter) {
1233                 len = bvec.bv_len;
1234                 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1235                                      op_is_write(req_op(rq)), sector,
1236                                      rq->cmd_flags & REQ_FUA);
1237                 if (err)
1238                         break;
1239                 sector += len >> SECTOR_SHIFT;
1240         }
1241         spin_unlock_irq(&nullb->lock);
1242
1243         return errno_to_blk_status(err);
1244 }
1245
1246 static inline blk_status_t null_handle_throttled(struct nullb_cmd *cmd)
1247 {
1248         struct nullb_device *dev = cmd->nq->dev;
1249         struct nullb *nullb = dev->nullb;
1250         blk_status_t sts = BLK_STS_OK;
1251         struct request *rq = blk_mq_rq_from_pdu(cmd);
1252
1253         if (!hrtimer_active(&nullb->bw_timer))
1254                 hrtimer_restart(&nullb->bw_timer);
1255
1256         if (atomic_long_sub_return(blk_rq_bytes(rq), &nullb->cur_bytes) < 0) {
1257                 blk_mq_stop_hw_queues(nullb->q);
1258                 /* race with timer */
1259                 if (atomic_long_read(&nullb->cur_bytes) > 0)
1260                         blk_mq_start_stopped_hw_queues(nullb->q, true);
1261                 /* requeue request */
1262                 sts = BLK_STS_DEV_RESOURCE;
1263         }
1264         return sts;
1265 }
1266
1267 static inline blk_status_t null_handle_badblocks(struct nullb_cmd *cmd,
1268                                                  sector_t sector,
1269                                                  sector_t nr_sectors)
1270 {
1271         struct badblocks *bb = &cmd->nq->dev->badblocks;
1272         sector_t first_bad;
1273         int bad_sectors;
1274
1275         if (badblocks_check(bb, sector, nr_sectors, &first_bad, &bad_sectors))
1276                 return BLK_STS_IOERR;
1277
1278         return BLK_STS_OK;
1279 }
1280
1281 static inline blk_status_t null_handle_memory_backed(struct nullb_cmd *cmd,
1282                                                      enum req_op op,
1283                                                      sector_t sector,
1284                                                      sector_t nr_sectors)
1285 {
1286         struct nullb_device *dev = cmd->nq->dev;
1287
1288         if (op == REQ_OP_DISCARD)
1289                 return null_handle_discard(dev, sector, nr_sectors);
1290
1291         return null_handle_rq(cmd);
1292 }
1293
1294 static void nullb_zero_read_cmd_buffer(struct nullb_cmd *cmd)
1295 {
1296         struct request *rq = blk_mq_rq_from_pdu(cmd);
1297         struct nullb_device *dev = cmd->nq->dev;
1298         struct bio *bio;
1299
1300         if (!dev->memory_backed && req_op(rq) == REQ_OP_READ) {
1301                 __rq_for_each_bio(bio, rq)
1302                         zero_fill_bio(bio);
1303         }
1304 }
1305
1306 static inline void nullb_complete_cmd(struct nullb_cmd *cmd)
1307 {
1308         struct request *rq = blk_mq_rq_from_pdu(cmd);
1309
1310         /*
1311          * Since root privileges are required to configure the null_blk
1312          * driver, it is fine that this driver does not initialize the
1313          * data buffers of read commands. Zero-initialize these buffers
1314          * anyway if KMSAN is enabled to prevent that KMSAN complains
1315          * about null_blk not initializing read data buffers.
1316          */
1317         if (IS_ENABLED(CONFIG_KMSAN))
1318                 nullb_zero_read_cmd_buffer(cmd);
1319
1320         /* Complete IO by inline, softirq or timer */
1321         switch (cmd->nq->dev->irqmode) {
1322         case NULL_IRQ_SOFTIRQ:
1323                 blk_mq_complete_request(rq);
1324                 break;
1325         case NULL_IRQ_NONE:
1326                 blk_mq_end_request(rq, cmd->error);
1327                 break;
1328         case NULL_IRQ_TIMER:
1329                 null_cmd_end_timer(cmd);
1330                 break;
1331         }
1332 }
1333
1334 blk_status_t null_process_cmd(struct nullb_cmd *cmd, enum req_op op,
1335                               sector_t sector, unsigned int nr_sectors)
1336 {
1337         struct nullb_device *dev = cmd->nq->dev;
1338         blk_status_t ret;
1339
1340         if (dev->badblocks.shift != -1) {
1341                 ret = null_handle_badblocks(cmd, sector, nr_sectors);
1342                 if (ret != BLK_STS_OK)
1343                         return ret;
1344         }
1345
1346         if (dev->memory_backed)
1347                 return null_handle_memory_backed(cmd, op, sector, nr_sectors);
1348
1349         return BLK_STS_OK;
1350 }
1351
1352 static void null_handle_cmd(struct nullb_cmd *cmd, sector_t sector,
1353                             sector_t nr_sectors, enum req_op op)
1354 {
1355         struct nullb_device *dev = cmd->nq->dev;
1356         struct nullb *nullb = dev->nullb;
1357         blk_status_t sts;
1358
1359         if (op == REQ_OP_FLUSH) {
1360                 cmd->error = null_handle_flush(nullb);
1361                 goto out;
1362         }
1363
1364         if (dev->zoned)
1365                 sts = null_process_zoned_cmd(cmd, op, sector, nr_sectors);
1366         else
1367                 sts = null_process_cmd(cmd, op, sector, nr_sectors);
1368
1369         /* Do not overwrite errors (e.g. timeout errors) */
1370         if (cmd->error == BLK_STS_OK)
1371                 cmd->error = sts;
1372
1373 out:
1374         nullb_complete_cmd(cmd);
1375 }
1376
1377 static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
1378 {
1379         struct nullb *nullb = container_of(timer, struct nullb, bw_timer);
1380         ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1381         unsigned int mbps = nullb->dev->mbps;
1382
1383         if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps))
1384                 return HRTIMER_NORESTART;
1385
1386         atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps));
1387         blk_mq_start_stopped_hw_queues(nullb->q, true);
1388
1389         hrtimer_forward_now(&nullb->bw_timer, timer_interval);
1390
1391         return HRTIMER_RESTART;
1392 }
1393
1394 static void nullb_setup_bwtimer(struct nullb *nullb)
1395 {
1396         ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1397
1398         hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1399         nullb->bw_timer.function = nullb_bwtimer_fn;
1400         atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps));
1401         hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL);
1402 }
1403
1404 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1405
1406 static bool should_timeout_request(struct request *rq)
1407 {
1408         struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1409         struct nullb_device *dev = cmd->nq->dev;
1410
1411         return should_fail(&dev->timeout_config.attr, 1);
1412 }
1413
1414 static bool should_requeue_request(struct request *rq)
1415 {
1416         struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1417         struct nullb_device *dev = cmd->nq->dev;
1418
1419         return should_fail(&dev->requeue_config.attr, 1);
1420 }
1421
1422 static bool should_init_hctx_fail(struct nullb_device *dev)
1423 {
1424         return should_fail(&dev->init_hctx_fault_config.attr, 1);
1425 }
1426
1427 #else
1428
1429 static bool should_timeout_request(struct request *rq)
1430 {
1431         return false;
1432 }
1433
1434 static bool should_requeue_request(struct request *rq)
1435 {
1436         return false;
1437 }
1438
1439 static bool should_init_hctx_fail(struct nullb_device *dev)
1440 {
1441         return false;
1442 }
1443
1444 #endif
1445
1446 static void null_map_queues(struct blk_mq_tag_set *set)
1447 {
1448         struct nullb *nullb = set->driver_data;
1449         int i, qoff;
1450         unsigned int submit_queues = g_submit_queues;
1451         unsigned int poll_queues = g_poll_queues;
1452
1453         if (nullb) {
1454                 struct nullb_device *dev = nullb->dev;
1455
1456                 /*
1457                  * Refer nr_hw_queues of the tag set to check if the expected
1458                  * number of hardware queues are prepared. If block layer failed
1459                  * to prepare them, use previous numbers of submit queues and
1460                  * poll queues to map queues.
1461                  */
1462                 if (set->nr_hw_queues ==
1463                     dev->submit_queues + dev->poll_queues) {
1464                         submit_queues = dev->submit_queues;
1465                         poll_queues = dev->poll_queues;
1466                 } else if (set->nr_hw_queues ==
1467                            dev->prev_submit_queues + dev->prev_poll_queues) {
1468                         submit_queues = dev->prev_submit_queues;
1469                         poll_queues = dev->prev_poll_queues;
1470                 } else {
1471                         pr_warn("tag set has unexpected nr_hw_queues: %d\n",
1472                                 set->nr_hw_queues);
1473                         WARN_ON_ONCE(true);
1474                         submit_queues = 1;
1475                         poll_queues = 0;
1476                 }
1477         }
1478
1479         for (i = 0, qoff = 0; i < set->nr_maps; i++) {
1480                 struct blk_mq_queue_map *map = &set->map[i];
1481
1482                 switch (i) {
1483                 case HCTX_TYPE_DEFAULT:
1484                         map->nr_queues = submit_queues;
1485                         break;
1486                 case HCTX_TYPE_READ:
1487                         map->nr_queues = 0;
1488                         continue;
1489                 case HCTX_TYPE_POLL:
1490                         map->nr_queues = poll_queues;
1491                         break;
1492                 }
1493                 map->queue_offset = qoff;
1494                 qoff += map->nr_queues;
1495                 blk_mq_map_queues(map);
1496         }
1497 }
1498
1499 static int null_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
1500 {
1501         struct nullb_queue *nq = hctx->driver_data;
1502         LIST_HEAD(list);
1503         int nr = 0;
1504         struct request *rq;
1505
1506         spin_lock(&nq->poll_lock);
1507         list_splice_init(&nq->poll_list, &list);
1508         list_for_each_entry(rq, &list, queuelist)
1509                 blk_mq_set_request_complete(rq);
1510         spin_unlock(&nq->poll_lock);
1511
1512         while (!list_empty(&list)) {
1513                 struct nullb_cmd *cmd;
1514                 struct request *req;
1515
1516                 req = list_first_entry(&list, struct request, queuelist);
1517                 list_del_init(&req->queuelist);
1518                 cmd = blk_mq_rq_to_pdu(req);
1519                 cmd->error = null_process_cmd(cmd, req_op(req), blk_rq_pos(req),
1520                                                 blk_rq_sectors(req));
1521                 if (!blk_mq_add_to_batch(req, iob, (__force int) cmd->error,
1522                                         blk_mq_end_request_batch))
1523                         blk_mq_end_request(req, cmd->error);
1524                 nr++;
1525         }
1526
1527         return nr;
1528 }
1529
1530 static enum blk_eh_timer_return null_timeout_rq(struct request *rq)
1531 {
1532         struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1533         struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1534
1535         if (hctx->type == HCTX_TYPE_POLL) {
1536                 struct nullb_queue *nq = hctx->driver_data;
1537
1538                 spin_lock(&nq->poll_lock);
1539                 /* The request may have completed meanwhile. */
1540                 if (blk_mq_request_completed(rq)) {
1541                         spin_unlock(&nq->poll_lock);
1542                         return BLK_EH_DONE;
1543                 }
1544                 list_del_init(&rq->queuelist);
1545                 spin_unlock(&nq->poll_lock);
1546         }
1547
1548         pr_info("rq %p timed out\n", rq);
1549
1550         /*
1551          * If the device is marked as blocking (i.e. memory backed or zoned
1552          * device), the submission path may be blocked waiting for resources
1553          * and cause real timeouts. For these real timeouts, the submission
1554          * path will complete the request using blk_mq_complete_request().
1555          * Only fake timeouts need to execute blk_mq_complete_request() here.
1556          */
1557         cmd->error = BLK_STS_TIMEOUT;
1558         if (cmd->fake_timeout || hctx->type == HCTX_TYPE_POLL)
1559                 blk_mq_complete_request(rq);
1560         return BLK_EH_DONE;
1561 }
1562
1563 static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
1564                                   const struct blk_mq_queue_data *bd)
1565 {
1566         struct request *rq = bd->rq;
1567         struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1568         struct nullb_queue *nq = hctx->driver_data;
1569         sector_t nr_sectors = blk_rq_sectors(rq);
1570         sector_t sector = blk_rq_pos(rq);
1571         const bool is_poll = hctx->type == HCTX_TYPE_POLL;
1572
1573         might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
1574
1575         if (!is_poll && nq->dev->irqmode == NULL_IRQ_TIMER) {
1576                 hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1577                 cmd->timer.function = null_cmd_timer_expired;
1578         }
1579         cmd->error = BLK_STS_OK;
1580         cmd->nq = nq;
1581         cmd->fake_timeout = should_timeout_request(rq) ||
1582                 blk_should_fake_timeout(rq->q);
1583
1584         if (should_requeue_request(rq)) {
1585                 /*
1586                  * Alternate between hitting the core BUSY path, and the
1587                  * driver driven requeue path
1588                  */
1589                 nq->requeue_selection++;
1590                 if (nq->requeue_selection & 1)
1591                         return BLK_STS_RESOURCE;
1592                 blk_mq_requeue_request(rq, true);
1593                 return BLK_STS_OK;
1594         }
1595
1596         if (test_bit(NULLB_DEV_FL_THROTTLED, &nq->dev->flags)) {
1597                 blk_status_t sts = null_handle_throttled(cmd);
1598
1599                 if (sts != BLK_STS_OK)
1600                         return sts;
1601         }
1602
1603         blk_mq_start_request(rq);
1604
1605         if (is_poll) {
1606                 spin_lock(&nq->poll_lock);
1607                 list_add_tail(&rq->queuelist, &nq->poll_list);
1608                 spin_unlock(&nq->poll_lock);
1609                 return BLK_STS_OK;
1610         }
1611         if (cmd->fake_timeout)
1612                 return BLK_STS_OK;
1613
1614         null_handle_cmd(cmd, sector, nr_sectors, req_op(rq));
1615         return BLK_STS_OK;
1616 }
1617
1618 static void null_queue_rqs(struct request **rqlist)
1619 {
1620         struct request *requeue_list = NULL;
1621         struct request **requeue_lastp = &requeue_list;
1622         struct blk_mq_queue_data bd = { };
1623         blk_status_t ret;
1624
1625         do {
1626                 struct request *rq = rq_list_pop(rqlist);
1627
1628                 bd.rq = rq;
1629                 ret = null_queue_rq(rq->mq_hctx, &bd);
1630                 if (ret != BLK_STS_OK)
1631                         rq_list_add_tail(&requeue_lastp, rq);
1632         } while (!rq_list_empty(*rqlist));
1633
1634         *rqlist = requeue_list;
1635 }
1636
1637 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
1638 {
1639         nq->dev = nullb->dev;
1640         INIT_LIST_HEAD(&nq->poll_list);
1641         spin_lock_init(&nq->poll_lock);
1642 }
1643
1644 static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
1645                           unsigned int hctx_idx)
1646 {
1647         struct nullb *nullb = hctx->queue->queuedata;
1648         struct nullb_queue *nq;
1649
1650         if (should_init_hctx_fail(nullb->dev))
1651                 return -EFAULT;
1652
1653         nq = &nullb->queues[hctx_idx];
1654         hctx->driver_data = nq;
1655         null_init_queue(nullb, nq);
1656
1657         return 0;
1658 }
1659
1660 static const struct blk_mq_ops null_mq_ops = {
1661         .queue_rq       = null_queue_rq,
1662         .queue_rqs      = null_queue_rqs,
1663         .complete       = null_complete_rq,
1664         .timeout        = null_timeout_rq,
1665         .poll           = null_poll,
1666         .map_queues     = null_map_queues,
1667         .init_hctx      = null_init_hctx,
1668 };
1669
1670 static void null_del_dev(struct nullb *nullb)
1671 {
1672         struct nullb_device *dev;
1673
1674         if (!nullb)
1675                 return;
1676
1677         dev = nullb->dev;
1678
1679         ida_free(&nullb_indexes, nullb->index);
1680
1681         list_del_init(&nullb->list);
1682
1683         del_gendisk(nullb->disk);
1684
1685         if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
1686                 hrtimer_cancel(&nullb->bw_timer);
1687                 atomic_long_set(&nullb->cur_bytes, LONG_MAX);
1688                 blk_mq_start_stopped_hw_queues(nullb->q, true);
1689         }
1690
1691         put_disk(nullb->disk);
1692         if (nullb->tag_set == &nullb->__tag_set)
1693                 blk_mq_free_tag_set(nullb->tag_set);
1694         kfree(nullb->queues);
1695         if (null_cache_active(nullb))
1696                 null_free_device_storage(nullb->dev, true);
1697         kfree(nullb);
1698         dev->nullb = NULL;
1699 }
1700
1701 static void null_config_discard(struct nullb *nullb, struct queue_limits *lim)
1702 {
1703         if (nullb->dev->discard == false)
1704                 return;
1705
1706         if (!nullb->dev->memory_backed) {
1707                 nullb->dev->discard = false;
1708                 pr_info("discard option is ignored without memory backing\n");
1709                 return;
1710         }
1711
1712         if (nullb->dev->zoned) {
1713                 nullb->dev->discard = false;
1714                 pr_info("discard option is ignored in zoned mode\n");
1715                 return;
1716         }
1717
1718         lim->max_hw_discard_sectors = UINT_MAX >> 9;
1719 }
1720
1721 static const struct block_device_operations null_ops = {
1722         .owner          = THIS_MODULE,
1723         .report_zones   = null_report_zones,
1724 };
1725
1726 static int setup_queues(struct nullb *nullb)
1727 {
1728         int nqueues = nr_cpu_ids;
1729
1730         if (g_poll_queues)
1731                 nqueues += g_poll_queues;
1732
1733         nullb->queues = kcalloc(nqueues, sizeof(struct nullb_queue),
1734                                 GFP_KERNEL);
1735         if (!nullb->queues)
1736                 return -ENOMEM;
1737
1738         return 0;
1739 }
1740
1741 static int null_init_tag_set(struct blk_mq_tag_set *set, int poll_queues)
1742 {
1743         set->ops = &null_mq_ops;
1744         set->cmd_size = sizeof(struct nullb_cmd);
1745         set->timeout = 5 * HZ;
1746         set->nr_maps = 1;
1747         if (poll_queues) {
1748                 set->nr_hw_queues += poll_queues;
1749                 set->nr_maps += 2;
1750         }
1751         return blk_mq_alloc_tag_set(set);
1752 }
1753
1754 static int null_init_global_tag_set(void)
1755 {
1756         int error;
1757
1758         if (tag_set.ops)
1759                 return 0;
1760
1761         tag_set.nr_hw_queues = g_submit_queues;
1762         tag_set.queue_depth = g_hw_queue_depth;
1763         tag_set.numa_node = g_home_node;
1764         tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
1765         if (g_no_sched)
1766                 tag_set.flags |= BLK_MQ_F_NO_SCHED;
1767         if (g_shared_tag_bitmap)
1768                 tag_set.flags |= BLK_MQ_F_TAG_HCTX_SHARED;
1769         if (g_blocking)
1770                 tag_set.flags |= BLK_MQ_F_BLOCKING;
1771
1772         error = null_init_tag_set(&tag_set, g_poll_queues);
1773         if (error)
1774                 tag_set.ops = NULL;
1775         return error;
1776 }
1777
1778 static int null_setup_tagset(struct nullb *nullb)
1779 {
1780         if (nullb->dev->shared_tags) {
1781                 nullb->tag_set = &tag_set;
1782                 return null_init_global_tag_set();
1783         }
1784
1785         nullb->tag_set = &nullb->__tag_set;
1786         nullb->tag_set->driver_data = nullb;
1787         nullb->tag_set->nr_hw_queues = nullb->dev->submit_queues;
1788         nullb->tag_set->queue_depth = nullb->dev->hw_queue_depth;
1789         nullb->tag_set->numa_node = nullb->dev->home_node;
1790         nullb->tag_set->flags = BLK_MQ_F_SHOULD_MERGE;
1791         if (nullb->dev->no_sched)
1792                 nullb->tag_set->flags |= BLK_MQ_F_NO_SCHED;
1793         if (nullb->dev->shared_tag_bitmap)
1794                 nullb->tag_set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;
1795         if (nullb->dev->blocking)
1796                 nullb->tag_set->flags |= BLK_MQ_F_BLOCKING;
1797         return null_init_tag_set(nullb->tag_set, nullb->dev->poll_queues);
1798 }
1799
1800 static int null_validate_conf(struct nullb_device *dev)
1801 {
1802         if (dev->queue_mode == NULL_Q_RQ) {
1803                 pr_err("legacy IO path is no longer available\n");
1804                 return -EINVAL;
1805         }
1806         if (dev->queue_mode == NULL_Q_BIO) {
1807                 pr_err("BIO-based IO path is no longer available, using blk-mq instead.\n");
1808                 dev->queue_mode = NULL_Q_MQ;
1809         }
1810
1811         dev->blocksize = round_down(dev->blocksize, 512);
1812         dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
1813
1814         if (dev->use_per_node_hctx) {
1815                 if (dev->submit_queues != nr_online_nodes)
1816                         dev->submit_queues = nr_online_nodes;
1817         } else if (dev->submit_queues > nr_cpu_ids)
1818                 dev->submit_queues = nr_cpu_ids;
1819         else if (dev->submit_queues == 0)
1820                 dev->submit_queues = 1;
1821         dev->prev_submit_queues = dev->submit_queues;
1822
1823         if (dev->poll_queues > g_poll_queues)
1824                 dev->poll_queues = g_poll_queues;
1825         dev->prev_poll_queues = dev->poll_queues;
1826         dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER);
1827
1828         /* Do memory allocation, so set blocking */
1829         if (dev->memory_backed)
1830                 dev->blocking = true;
1831         else /* cache is meaningless */
1832                 dev->cache_size = 0;
1833         dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024,
1834                                                 dev->cache_size);
1835         dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps);
1836
1837         if (dev->zoned &&
1838             (!dev->zone_size || !is_power_of_2(dev->zone_size))) {
1839                 pr_err("zone_size must be power-of-two\n");
1840                 return -EINVAL;
1841         }
1842
1843         return 0;
1844 }
1845
1846 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1847 static bool __null_setup_fault(struct fault_attr *attr, char *str)
1848 {
1849         if (!str[0])
1850                 return true;
1851
1852         if (!setup_fault_attr(attr, str))
1853                 return false;
1854
1855         attr->verbose = 0;
1856         return true;
1857 }
1858 #endif
1859
1860 static bool null_setup_fault(void)
1861 {
1862 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1863         if (!__null_setup_fault(&null_timeout_attr, g_timeout_str))
1864                 return false;
1865         if (!__null_setup_fault(&null_requeue_attr, g_requeue_str))
1866                 return false;
1867         if (!__null_setup_fault(&null_init_hctx_attr, g_init_hctx_str))
1868                 return false;
1869 #endif
1870         return true;
1871 }
1872
1873 static int null_add_dev(struct nullb_device *dev)
1874 {
1875         struct queue_limits lim = {
1876                 .logical_block_size     = dev->blocksize,
1877                 .physical_block_size    = dev->blocksize,
1878                 .max_hw_sectors         = dev->max_sectors,
1879         };
1880
1881         struct nullb *nullb;
1882         int rv;
1883
1884         rv = null_validate_conf(dev);
1885         if (rv)
1886                 return rv;
1887
1888         nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
1889         if (!nullb) {
1890                 rv = -ENOMEM;
1891                 goto out;
1892         }
1893         nullb->dev = dev;
1894         dev->nullb = nullb;
1895
1896         spin_lock_init(&nullb->lock);
1897
1898         rv = setup_queues(nullb);
1899         if (rv)
1900                 goto out_free_nullb;
1901
1902         rv = null_setup_tagset(nullb);
1903         if (rv)
1904                 goto out_cleanup_queues;
1905
1906         if (dev->virt_boundary)
1907                 lim.virt_boundary_mask = PAGE_SIZE - 1;
1908         null_config_discard(nullb, &lim);
1909         if (dev->zoned) {
1910                 rv = null_init_zoned_dev(dev, &lim);
1911                 if (rv)
1912                         goto out_cleanup_tags;
1913         }
1914
1915         nullb->disk = blk_mq_alloc_disk(nullb->tag_set, &lim, nullb);
1916         if (IS_ERR(nullb->disk)) {
1917                 rv = PTR_ERR(nullb->disk);
1918                 goto out_cleanup_zone;
1919         }
1920         nullb->q = nullb->disk->queue;
1921
1922         if (dev->mbps) {
1923                 set_bit(NULLB_DEV_FL_THROTTLED, &dev->flags);
1924                 nullb_setup_bwtimer(nullb);
1925         }
1926
1927         if (dev->cache_size > 0) {
1928                 set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
1929                 blk_queue_write_cache(nullb->q, true, dev->fua);
1930         }
1931
1932         nullb->q->queuedata = nullb;
1933         blk_queue_flag_set(QUEUE_FLAG_NONROT, nullb->q);
1934
1935         mutex_lock(&lock);
1936         rv = ida_alloc(&nullb_indexes, GFP_KERNEL);
1937         if (rv < 0) {
1938                 mutex_unlock(&lock);
1939                 goto out_cleanup_disk;
1940         }
1941         nullb->index = rv;
1942         dev->index = rv;
1943         mutex_unlock(&lock);
1944
1945         if (config_item_name(&dev->group.cg_item)) {
1946                 /* Use configfs dir name as the device name */
1947                 snprintf(nullb->disk_name, sizeof(nullb->disk_name),
1948                          "%s", config_item_name(&dev->group.cg_item));
1949         } else {
1950                 sprintf(nullb->disk_name, "nullb%d", nullb->index);
1951         }
1952
1953         set_capacity(nullb->disk,
1954                 ((sector_t)nullb->dev->size * SZ_1M) >> SECTOR_SHIFT);
1955         nullb->disk->major = null_major;
1956         nullb->disk->first_minor = nullb->index;
1957         nullb->disk->minors = 1;
1958         nullb->disk->fops = &null_ops;
1959         nullb->disk->private_data = nullb;
1960         strscpy_pad(nullb->disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
1961
1962         if (nullb->dev->zoned) {
1963                 rv = null_register_zoned_dev(nullb);
1964                 if (rv)
1965                         goto out_ida_free;
1966         }
1967
1968         rv = add_disk(nullb->disk);
1969         if (rv)
1970                 goto out_ida_free;
1971
1972         mutex_lock(&lock);
1973         list_add_tail(&nullb->list, &nullb_list);
1974         mutex_unlock(&lock);
1975
1976         pr_info("disk %s created\n", nullb->disk_name);
1977
1978         return 0;
1979
1980 out_ida_free:
1981         ida_free(&nullb_indexes, nullb->index);
1982 out_cleanup_zone:
1983         null_free_zoned_dev(dev);
1984 out_cleanup_disk:
1985         put_disk(nullb->disk);
1986 out_cleanup_tags:
1987         if (nullb->tag_set == &nullb->__tag_set)
1988                 blk_mq_free_tag_set(nullb->tag_set);
1989 out_cleanup_queues:
1990         kfree(nullb->queues);
1991 out_free_nullb:
1992         kfree(nullb);
1993         dev->nullb = NULL;
1994 out:
1995         return rv;
1996 }
1997
1998 static struct nullb *null_find_dev_by_name(const char *name)
1999 {
2000         struct nullb *nullb = NULL, *nb;
2001
2002         mutex_lock(&lock);
2003         list_for_each_entry(nb, &nullb_list, list) {
2004                 if (strcmp(nb->disk_name, name) == 0) {
2005                         nullb = nb;
2006                         break;
2007                 }
2008         }
2009         mutex_unlock(&lock);
2010
2011         return nullb;
2012 }
2013
2014 static int null_create_dev(void)
2015 {
2016         struct nullb_device *dev;
2017         int ret;
2018
2019         dev = null_alloc_dev();
2020         if (!dev)
2021                 return -ENOMEM;
2022
2023         ret = null_add_dev(dev);
2024         if (ret) {
2025                 null_free_dev(dev);
2026                 return ret;
2027         }
2028
2029         return 0;
2030 }
2031
2032 static void null_destroy_dev(struct nullb *nullb)
2033 {
2034         struct nullb_device *dev = nullb->dev;
2035
2036         null_del_dev(nullb);
2037         null_free_device_storage(dev, false);
2038         null_free_dev(dev);
2039 }
2040
2041 static int __init null_init(void)
2042 {
2043         int ret = 0;
2044         unsigned int i;
2045         struct nullb *nullb;
2046
2047         if (g_bs > PAGE_SIZE) {
2048                 pr_warn("invalid block size\n");
2049                 pr_warn("defaults block size to %lu\n", PAGE_SIZE);
2050                 g_bs = PAGE_SIZE;
2051         }
2052
2053         if (g_home_node != NUMA_NO_NODE && g_home_node >= nr_online_nodes) {
2054                 pr_err("invalid home_node value\n");
2055                 g_home_node = NUMA_NO_NODE;
2056         }
2057
2058         if (!null_setup_fault())
2059                 return -EINVAL;
2060
2061         if (g_queue_mode == NULL_Q_RQ) {
2062                 pr_err("legacy IO path is no longer available\n");
2063                 return -EINVAL;
2064         }
2065
2066         if (g_use_per_node_hctx) {
2067                 if (g_submit_queues != nr_online_nodes) {
2068                         pr_warn("submit_queues param is set to %u.\n",
2069                                 nr_online_nodes);
2070                         g_submit_queues = nr_online_nodes;
2071                 }
2072         } else if (g_submit_queues > nr_cpu_ids) {
2073                 g_submit_queues = nr_cpu_ids;
2074         } else if (g_submit_queues <= 0) {
2075                 g_submit_queues = 1;
2076         }
2077
2078         config_group_init(&nullb_subsys.su_group);
2079         mutex_init(&nullb_subsys.su_mutex);
2080
2081         ret = configfs_register_subsystem(&nullb_subsys);
2082         if (ret)
2083                 return ret;
2084
2085         mutex_init(&lock);
2086
2087         null_major = register_blkdev(0, "nullb");
2088         if (null_major < 0) {
2089                 ret = null_major;
2090                 goto err_conf;
2091         }
2092
2093         for (i = 0; i < nr_devices; i++) {
2094                 ret = null_create_dev();
2095                 if (ret)
2096                         goto err_dev;
2097         }
2098
2099         pr_info("module loaded\n");
2100         return 0;
2101
2102 err_dev:
2103         while (!list_empty(&nullb_list)) {
2104                 nullb = list_entry(nullb_list.next, struct nullb, list);
2105                 null_destroy_dev(nullb);
2106         }
2107         unregister_blkdev(null_major, "nullb");
2108 err_conf:
2109         configfs_unregister_subsystem(&nullb_subsys);
2110         return ret;
2111 }
2112
2113 static void __exit null_exit(void)
2114 {
2115         struct nullb *nullb;
2116
2117         configfs_unregister_subsystem(&nullb_subsys);
2118
2119         unregister_blkdev(null_major, "nullb");
2120
2121         mutex_lock(&lock);
2122         while (!list_empty(&nullb_list)) {
2123                 nullb = list_entry(nullb_list.next, struct nullb, list);
2124                 null_destroy_dev(nullb);
2125         }
2126         mutex_unlock(&lock);
2127
2128         if (tag_set.ops)
2129                 blk_mq_free_tag_set(&tag_set);
2130 }
2131
2132 module_init(null_init);
2133 module_exit(null_exit);
2134
2135 MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>");
2136 MODULE_LICENSE("GPL");