block/blk-sysfs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Functions related to sysfs handling
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/slab.h>
   7 #include <linux/module.h>
   8 #include <linux/bio.h>
   9 #include <linux/blkdev.h>
  10 #include <linux/backing-dev.h>
  11 #include <linux/blktrace_api.h>
  12 #include <linux/debugfs.h>
  13
  14 #include "blk.h"
  15 #include "blk-mq.h"
  16 #include "blk-mq-debugfs.h"
  17 #include "blk-mq-sched.h"
  18 #include "blk-rq-qos.h"
  19 #include "blk-wbt.h"
  20 #include "blk-cgroup.h"
  21 #include "blk-throttle.h"
  22
  23 struct queue_sysfs_entry {
  24         struct attribute attr;
  25         ssize_t (*show)(struct gendisk *disk, char *page);
  26         ssize_t (*show_limit)(struct gendisk *disk, char *page);
  27
  28         ssize_t (*store)(struct gendisk *disk, const char *page, size_t count);
  29         int (*store_limit)(struct gendisk *disk, const char *page,
  30                         size_t count, struct queue_limits *lim);
  31 };
  32
  33 static ssize_t
  34 queue_var_show(unsigned long var, char *page)
  35 {
  36         return sysfs_emit(page, "%lu\n", var);
  37 }
  38
  39 static ssize_t
  40 queue_var_store(unsigned long *var, const char *page, size_t count)
  41 {
  42         int err;
  43         unsigned long v;
  44
  45         err = kstrtoul(page, 10, &v);
  46         if (err || v > UINT_MAX)
  47                 return -EINVAL;
  48
  49         *var = v;
  50
  51         return count;
  52 }
  53
  54 static ssize_t queue_requests_show(struct gendisk *disk, char *page)
  55 {
  56         ssize_t ret;
  57
  58         mutex_lock(&disk->queue->elevator_lock);
  59         ret = queue_var_show(disk->queue->nr_requests, page);
  60         mutex_unlock(&disk->queue->elevator_lock);
  61         return ret;
  62 }
  63
  64 static ssize_t
  65 queue_requests_store(struct gendisk *disk, const char *page, size_t count)
  66 {
  67         unsigned long nr;
  68         int ret, err;
  69         unsigned int memflags;
  70         struct request_queue *q = disk->queue;
  71
  72         if (!queue_is_mq(q))
  73                 return -EINVAL;
  74
  75         ret = queue_var_store(&nr, page, count);
  76         if (ret < 0)
  77                 return ret;
  78
  79         memflags = blk_mq_freeze_queue(q);
  80         mutex_lock(&q->elevator_lock);
  81         if (nr < BLKDEV_MIN_RQ)
  82                 nr = BLKDEV_MIN_RQ;
  83
  84         err = blk_mq_update_nr_requests(disk->queue, nr);
  85         if (err)
  86                 ret = err;
  87         mutex_unlock(&q->elevator_lock);
  88         blk_mq_unfreeze_queue(q, memflags);
  89         return ret;
  90 }
  91
  92 static ssize_t queue_ra_show(struct gendisk *disk, char *page)
  93 {
  94         ssize_t ret;
  95
  96         mutex_lock(&disk->queue->limits_lock);
  97         ret = queue_var_show(disk->bdi->ra_pages << (PAGE_SHIFT - 10), page);
  98         mutex_unlock(&disk->queue->limits_lock);
  99
 100         return ret;
 101 }
 102
 103 static ssize_t
 104 queue_ra_store(struct gendisk *disk, const char *page, size_t count)
 105 {
 106         unsigned long ra_kb;
 107         ssize_t ret;
 108         unsigned int memflags;
 109         struct request_queue *q = disk->queue;
 110
 111         ret = queue_var_store(&ra_kb, page, count);
 112         if (ret < 0)
 113                 return ret;
 114         /*
 115          * ->ra_pages is protected by ->limits_lock because it is usually
 116          * calculated from the queue limits by queue_limits_commit_update.
 117          */
 118         mutex_lock(&q->limits_lock);
 119         memflags = blk_mq_freeze_queue(q);
 120         disk->bdi->ra_pages = ra_kb >> (PAGE_SHIFT - 10);
 121         mutex_unlock(&q->limits_lock);
 122         blk_mq_unfreeze_queue(q, memflags);
 123
 124         return ret;
 125 }
 126
 127 #define QUEUE_SYSFS_LIMIT_SHOW(_field)                                  \
 128 static ssize_t queue_##_field##_show(struct gendisk *disk, char *page)  \
 129 {                                                                       \
 130         return queue_var_show(disk->queue->limits._field, page);        \
 131 }
 132
 133 QUEUE_SYSFS_LIMIT_SHOW(max_segments)
 134 QUEUE_SYSFS_LIMIT_SHOW(max_discard_segments)
 135 QUEUE_SYSFS_LIMIT_SHOW(max_integrity_segments)
 136 QUEUE_SYSFS_LIMIT_SHOW(max_segment_size)
 137 QUEUE_SYSFS_LIMIT_SHOW(max_write_streams)
 138 QUEUE_SYSFS_LIMIT_SHOW(write_stream_granularity)
 139 QUEUE_SYSFS_LIMIT_SHOW(logical_block_size)
 140 QUEUE_SYSFS_LIMIT_SHOW(physical_block_size)
 141 QUEUE_SYSFS_LIMIT_SHOW(chunk_sectors)
 142 QUEUE_SYSFS_LIMIT_SHOW(io_min)
 143 QUEUE_SYSFS_LIMIT_SHOW(io_opt)
 144 QUEUE_SYSFS_LIMIT_SHOW(discard_granularity)
 145 QUEUE_SYSFS_LIMIT_SHOW(zone_write_granularity)
 146 QUEUE_SYSFS_LIMIT_SHOW(virt_boundary_mask)
 147 QUEUE_SYSFS_LIMIT_SHOW(dma_alignment)
 148 QUEUE_SYSFS_LIMIT_SHOW(max_open_zones)
 149 QUEUE_SYSFS_LIMIT_SHOW(max_active_zones)
 150 QUEUE_SYSFS_LIMIT_SHOW(atomic_write_unit_min)
 151 QUEUE_SYSFS_LIMIT_SHOW(atomic_write_unit_max)
 152
 153 #define QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(_field)                 \
 154 static ssize_t queue_##_field##_show(struct gendisk *disk, char *page)  \
 155 {                                                                       \
 156         return sysfs_emit(page, "%llu\n",                               \
 157                 (unsigned long long)disk->queue->limits._field <<       \
 158                         SECTOR_SHIFT);                                  \
 159 }
 160
 161 QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_discard_sectors)
 162 QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_hw_discard_sectors)
 163 QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_write_zeroes_sectors)
 164 QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(atomic_write_max_sectors)
 165 QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(atomic_write_boundary_sectors)
 166 QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_zone_append_sectors)
 167
 168 #define QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(_field)                    \
 169 static ssize_t queue_##_field##_show(struct gendisk *disk, char *page)  \
 170 {                                                                       \
 171         return queue_var_show(disk->queue->limits._field >> 1, page);   \
 172 }
 173
 174 QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(max_sectors)
 175 QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(max_hw_sectors)
 176
 177 #define QUEUE_SYSFS_SHOW_CONST(_name, _val)                             \
 178 static ssize_t queue_##_name##_show(struct gendisk *disk, char *page)   \
 179 {                                                                       \
 180         return sysfs_emit(page, "%d\n", _val);                          \
 181 }
 182
 183 /* deprecated fields */
 184 QUEUE_SYSFS_SHOW_CONST(discard_zeroes_data, 0)
 185 QUEUE_SYSFS_SHOW_CONST(write_same_max, 0)
 186 QUEUE_SYSFS_SHOW_CONST(poll_delay, -1)
 187
 188 static int queue_max_discard_sectors_store(struct gendisk *disk,
 189                 const char *page, size_t count, struct queue_limits *lim)
 190 {
 191         unsigned long max_discard_bytes;
 192         ssize_t ret;
 193
 194         ret = queue_var_store(&max_discard_bytes, page, count);
 195         if (ret < 0)
 196                 return ret;
 197
 198         if (max_discard_bytes & (disk->queue->limits.discard_granularity - 1))
 199                 return -EINVAL;
 200
 201         if ((max_discard_bytes >> SECTOR_SHIFT) > UINT_MAX)
 202                 return -EINVAL;
 203
 204         lim->max_user_discard_sectors = max_discard_bytes >> SECTOR_SHIFT;
 205         return 0;
 206 }
 207
 208 static int
 209 queue_max_sectors_store(struct gendisk *disk, const char *page, size_t count,
 210                 struct queue_limits *lim)
 211 {
 212         unsigned long max_sectors_kb;
 213         ssize_t ret;
 214
 215         ret = queue_var_store(&max_sectors_kb, page, count);
 216         if (ret < 0)
 217                 return ret;
 218
 219         lim->max_user_sectors = max_sectors_kb << 1;
 220         return 0;
 221 }
 222
 223 static ssize_t queue_feature_store(struct gendisk *disk, const char *page,
 224                 size_t count, struct queue_limits *lim, blk_features_t feature)
 225 {
 226         unsigned long val;
 227         ssize_t ret;
 228
 229         ret = queue_var_store(&val, page, count);
 230         if (ret < 0)
 231                 return ret;
 232
 233         if (val)
 234                 lim->features |= feature;
 235         else
 236                 lim->features &= ~feature;
 237         return 0;
 238 }
 239
 240 #define QUEUE_SYSFS_FEATURE(_name, _feature)                            \
 241 static ssize_t queue_##_name##_show(struct gendisk *disk, char *page)   \
 242 {                                                                       \
 243         return sysfs_emit(page, "%u\n",                                 \
 244                 !!(disk->queue->limits.features & _feature));           \
 245 }                                                                       \
 246 static int queue_##_name##_store(struct gendisk *disk,                  \
 247                 const char *page, size_t count, struct queue_limits *lim) \
 248 {                                                                       \
 249         return queue_feature_store(disk, page, count, lim, _feature);   \
 250 }
 251
 252 QUEUE_SYSFS_FEATURE(rotational, BLK_FEAT_ROTATIONAL)
 253 QUEUE_SYSFS_FEATURE(add_random, BLK_FEAT_ADD_RANDOM)
 254 QUEUE_SYSFS_FEATURE(iostats, BLK_FEAT_IO_STAT)
 255 QUEUE_SYSFS_FEATURE(stable_writes, BLK_FEAT_STABLE_WRITES);
 256
 257 #define QUEUE_SYSFS_FEATURE_SHOW(_name, _feature)                       \
 258 static ssize_t queue_##_name##_show(struct gendisk *disk, char *page)   \
 259 {                                                                       \
 260         return sysfs_emit(page, "%u\n",                                 \
 261                 !!(disk->queue->limits.features & _feature));           \
 262 }
 263
 264 QUEUE_SYSFS_FEATURE_SHOW(fua, BLK_FEAT_FUA);
 265 QUEUE_SYSFS_FEATURE_SHOW(dax, BLK_FEAT_DAX);
 266
 267 static ssize_t queue_poll_show(struct gendisk *disk, char *page)
 268 {
 269         if (queue_is_mq(disk->queue))
 270                 return sysfs_emit(page, "%u\n", blk_mq_can_poll(disk->queue));
 271
 272         return sysfs_emit(page, "%u\n",
 273                         !!(disk->queue->limits.features & BLK_FEAT_POLL));
 274 }
 275
 276 static ssize_t queue_zoned_show(struct gendisk *disk, char *page)
 277 {
 278         if (blk_queue_is_zoned(disk->queue))
 279                 return sysfs_emit(page, "host-managed\n");
 280         return sysfs_emit(page, "none\n");
 281 }
 282
 283 static ssize_t queue_nr_zones_show(struct gendisk *disk, char *page)
 284 {
 285         return queue_var_show(disk_nr_zones(disk), page);
 286 }
 287
 288 static ssize_t queue_iostats_passthrough_show(struct gendisk *disk, char *page)
 289 {
 290         return queue_var_show(!!blk_queue_passthrough_stat(disk->queue), page);
 291 }
 292
 293 static int queue_iostats_passthrough_store(struct gendisk *disk,
 294                 const char *page, size_t count, struct queue_limits *lim)
 295 {
 296         unsigned long ios;
 297         ssize_t ret;
 298
 299         ret = queue_var_store(&ios, page, count);
 300         if (ret < 0)
 301                 return ret;
 302
 303         if (ios)
 304                 lim->flags |= BLK_FLAG_IOSTATS_PASSTHROUGH;
 305         else
 306                 lim->flags &= ~BLK_FLAG_IOSTATS_PASSTHROUGH;
 307         return 0;
 308 }
 309
 310 static ssize_t queue_nomerges_show(struct gendisk *disk, char *page)
 311 {
 312         return queue_var_show((blk_queue_nomerges(disk->queue) << 1) |
 313                                blk_queue_noxmerges(disk->queue), page);
 314 }
 315
 316 static ssize_t queue_nomerges_store(struct gendisk *disk, const char *page,
 317                                     size_t count)
 318 {
 319         unsigned long nm;
 320         unsigned int memflags;
 321         struct request_queue *q = disk->queue;
 322         ssize_t ret = queue_var_store(&nm, page, count);
 323
 324         if (ret < 0)
 325                 return ret;
 326
 327         memflags = blk_mq_freeze_queue(q);
 328         blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, q);
 329         blk_queue_flag_clear(QUEUE_FLAG_NOXMERGES, q);
 330         if (nm == 2)
 331                 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
 332         else if (nm)
 333                 blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
 334         blk_mq_unfreeze_queue(q, memflags);
 335
 336         return ret;
 337 }
 338
 339 static ssize_t queue_rq_affinity_show(struct gendisk *disk, char *page)
 340 {
 341         bool set = test_bit(QUEUE_FLAG_SAME_COMP, &disk->queue->queue_flags);
 342         bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &disk->queue->queue_flags);
 343
 344         return queue_var_show(set << force, page);
 345 }
 346
 347 static ssize_t
 348 queue_rq_affinity_store(struct gendisk *disk, const char *page, size_t count)
 349 {
 350         ssize_t ret = -EINVAL;
 351 #ifdef CONFIG_SMP
 352         struct request_queue *q = disk->queue;
 353         unsigned long val;
 354         unsigned int memflags;
 355
 356         ret = queue_var_store(&val, page, count);
 357         if (ret < 0)
 358                 return ret;
 359
 360         /*
 361          * Here we update two queue flags each using atomic bitops, although
 362          * updating two flags isn't atomic it should be harmless as those flags
 363          * are accessed individually using atomic test_bit operation. So we
 364          * don't grab any lock while updating these flags.
 365          */
 366         memflags = blk_mq_freeze_queue(q);
 367         if (val == 2) {
 368                 blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
 369                 blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);
 370         } else if (val == 1) {
 371                 blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
 372                 blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
 373         } else if (val == 0) {
 374                 blk_queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);
 375                 blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
 376         }
 377         blk_mq_unfreeze_queue(q, memflags);
 378 #endif
 379         return ret;
 380 }
 381
 382 static ssize_t queue_poll_delay_store(struct gendisk *disk, const char *page,
 383                                 size_t count)
 384 {
 385         return count;
 386 }
 387
 388 static ssize_t queue_poll_store(struct gendisk *disk, const char *page,
 389                                 size_t count)
 390 {
 391         unsigned int memflags;
 392         ssize_t ret = count;
 393         struct request_queue *q = disk->queue;
 394
 395         memflags = blk_mq_freeze_queue(q);
 396         if (!(q->limits.features & BLK_FEAT_POLL)) {
 397                 ret = -EINVAL;
 398                 goto out;
 399         }
 400
 401         pr_info_ratelimited("writes to the poll attribute are ignored.\n");
 402         pr_info_ratelimited("please use driver specific parameters instead.\n");
 403 out:
 404         blk_mq_unfreeze_queue(q, memflags);
 405         return ret;
 406 }
 407
 408 static ssize_t queue_io_timeout_show(struct gendisk *disk, char *page)
 409 {
 410         return sysfs_emit(page, "%u\n",
 411                         jiffies_to_msecs(READ_ONCE(disk->queue->rq_timeout)));
 412 }
 413
 414 static ssize_t queue_io_timeout_store(struct gendisk *disk, const char *page,
 415                                   size_t count)
 416 {
 417         unsigned int val, memflags;
 418         int err;
 419         struct request_queue *q = disk->queue;
 420
 421         err = kstrtou32(page, 10, &val);
 422         if (err || val == 0)
 423                 return -EINVAL;
 424
 425         memflags = blk_mq_freeze_queue(q);
 426         blk_queue_rq_timeout(q, msecs_to_jiffies(val));
 427         blk_mq_unfreeze_queue(q, memflags);
 428
 429         return count;
 430 }
 431
 432 static ssize_t queue_wc_show(struct gendisk *disk, char *page)
 433 {
 434         if (blk_queue_write_cache(disk->queue))
 435                 return sysfs_emit(page, "write back\n");
 436         return sysfs_emit(page, "write through\n");
 437 }
 438
 439 static int queue_wc_store(struct gendisk *disk, const char *page,
 440                 size_t count, struct queue_limits *lim)
 441 {
 442         bool disable;
 443
 444         if (!strncmp(page, "write back", 10)) {
 445                 disable = false;
 446         } else if (!strncmp(page, "write through", 13) ||
 447                    !strncmp(page, "none", 4)) {
 448                 disable = true;
 449         } else {
 450                 return -EINVAL;
 451         }
 452
 453         if (disable)
 454                 lim->flags |= BLK_FLAG_WRITE_CACHE_DISABLED;
 455         else
 456                 lim->flags &= ~BLK_FLAG_WRITE_CACHE_DISABLED;
 457         return 0;
 458 }
 459
 460 #define QUEUE_RO_ENTRY(_prefix, _name)                  \
 461 static struct queue_sysfs_entry _prefix##_entry = {     \
 462         .attr   = { .name = _name, .mode = 0444 },      \
 463         .show   = _prefix##_show,                       \
 464 };
 465
 466 #define QUEUE_RW_ENTRY(_prefix, _name)                  \
 467 static struct queue_sysfs_entry _prefix##_entry = {     \
 468         .attr   = { .name = _name, .mode = 0644 },      \
 469         .show   = _prefix##_show,                       \
 470         .store  = _prefix##_store,                      \
 471 };
 472
 473 #define QUEUE_LIM_RO_ENTRY(_prefix, _name)                      \
 474 static struct queue_sysfs_entry _prefix##_entry = {     \
 475         .attr           = { .name = _name, .mode = 0444 },      \
 476         .show_limit     = _prefix##_show,                       \
 477 }
 478
 479 #define QUEUE_LIM_RW_ENTRY(_prefix, _name)                      \
 480 static struct queue_sysfs_entry _prefix##_entry = {     \
 481         .attr           = { .name = _name, .mode = 0644 },      \
 482         .show_limit     = _prefix##_show,                       \
 483         .store_limit    = _prefix##_store,                      \
 484 }
 485
 486 QUEUE_RW_ENTRY(queue_requests, "nr_requests");
 487 QUEUE_RW_ENTRY(queue_ra, "read_ahead_kb");
 488 QUEUE_LIM_RW_ENTRY(queue_max_sectors, "max_sectors_kb");
 489 QUEUE_LIM_RO_ENTRY(queue_max_hw_sectors, "max_hw_sectors_kb");
 490 QUEUE_LIM_RO_ENTRY(queue_max_segments, "max_segments");
 491 QUEUE_LIM_RO_ENTRY(queue_max_integrity_segments, "max_integrity_segments");
 492 QUEUE_LIM_RO_ENTRY(queue_max_segment_size, "max_segment_size");
 493 QUEUE_LIM_RO_ENTRY(queue_max_write_streams, "max_write_streams");
 494 QUEUE_LIM_RO_ENTRY(queue_write_stream_granularity, "write_stream_granularity");
 495 QUEUE_RW_ENTRY(elv_iosched, "scheduler");
 496
 497 QUEUE_LIM_RO_ENTRY(queue_logical_block_size, "logical_block_size");
 498 QUEUE_LIM_RO_ENTRY(queue_physical_block_size, "physical_block_size");
 499 QUEUE_LIM_RO_ENTRY(queue_chunk_sectors, "chunk_sectors");
 500 QUEUE_LIM_RO_ENTRY(queue_io_min, "minimum_io_size");
 501 QUEUE_LIM_RO_ENTRY(queue_io_opt, "optimal_io_size");
 502
 503 QUEUE_LIM_RO_ENTRY(queue_max_discard_segments, "max_discard_segments");
 504 QUEUE_LIM_RO_ENTRY(queue_discard_granularity, "discard_granularity");
 505 QUEUE_LIM_RO_ENTRY(queue_max_hw_discard_sectors, "discard_max_hw_bytes");
 506 QUEUE_LIM_RW_ENTRY(queue_max_discard_sectors, "discard_max_bytes");
 507 QUEUE_RO_ENTRY(queue_discard_zeroes_data, "discard_zeroes_data");
 508
 509 QUEUE_LIM_RO_ENTRY(queue_atomic_write_max_sectors, "atomic_write_max_bytes");
 510 QUEUE_LIM_RO_ENTRY(queue_atomic_write_boundary_sectors,
 511                 "atomic_write_boundary_bytes");
 512 QUEUE_LIM_RO_ENTRY(queue_atomic_write_unit_max, "atomic_write_unit_max_bytes");
 513 QUEUE_LIM_RO_ENTRY(queue_atomic_write_unit_min, "atomic_write_unit_min_bytes");
 514
 515 QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes");
 516 QUEUE_LIM_RO_ENTRY(queue_max_write_zeroes_sectors, "write_zeroes_max_bytes");
 517 QUEUE_LIM_RO_ENTRY(queue_max_zone_append_sectors, "zone_append_max_bytes");
 518 QUEUE_LIM_RO_ENTRY(queue_zone_write_granularity, "zone_write_granularity");
 519
 520 QUEUE_LIM_RO_ENTRY(queue_zoned, "zoned");
 521 QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones");
 522 QUEUE_LIM_RO_ENTRY(queue_max_open_zones, "max_open_zones");
 523 QUEUE_LIM_RO_ENTRY(queue_max_active_zones, "max_active_zones");
 524
 525 QUEUE_RW_ENTRY(queue_nomerges, "nomerges");
 526 QUEUE_LIM_RW_ENTRY(queue_iostats_passthrough, "iostats_passthrough");
 527 QUEUE_RW_ENTRY(queue_rq_affinity, "rq_affinity");
 528 QUEUE_RW_ENTRY(queue_poll, "io_poll");
 529 QUEUE_RW_ENTRY(queue_poll_delay, "io_poll_delay");
 530 QUEUE_LIM_RW_ENTRY(queue_wc, "write_cache");
 531 QUEUE_LIM_RO_ENTRY(queue_fua, "fua");
 532 QUEUE_LIM_RO_ENTRY(queue_dax, "dax");
 533 QUEUE_RW_ENTRY(queue_io_timeout, "io_timeout");
 534 QUEUE_LIM_RO_ENTRY(queue_virt_boundary_mask, "virt_boundary_mask");
 535 QUEUE_LIM_RO_ENTRY(queue_dma_alignment, "dma_alignment");
 536
 537 /* legacy alias for logical_block_size: */
 538 static struct queue_sysfs_entry queue_hw_sector_size_entry = {
 539         .attr           = {.name = "hw_sector_size", .mode = 0444 },
 540         .show_limit     = queue_logical_block_size_show,
 541 };
 542
 543 QUEUE_LIM_RW_ENTRY(queue_rotational, "rotational");
 544 QUEUE_LIM_RW_ENTRY(queue_iostats, "iostats");
 545 QUEUE_LIM_RW_ENTRY(queue_add_random, "add_random");
 546 QUEUE_LIM_RW_ENTRY(queue_stable_writes, "stable_writes");
 547
 548 #ifdef CONFIG_BLK_WBT
 549 static ssize_t queue_var_store64(s64 *var, const char *page)
 550 {
 551         int err;
 552         s64 v;
 553
 554         err = kstrtos64(page, 10, &v);
 555         if (err < 0)
 556                 return err;
 557
 558         *var = v;
 559         return 0;
 560 }
 561
 562 static ssize_t queue_wb_lat_show(struct gendisk *disk, char *page)
 563 {
 564         ssize_t ret;
 565         struct request_queue *q = disk->queue;
 566
 567         mutex_lock(&disk->rqos_state_mutex);
 568         if (!wbt_rq_qos(q)) {
 569                 ret = -EINVAL;
 570                 goto out;
 571         }
 572
 573         if (wbt_disabled(q)) {
 574                 ret = sysfs_emit(page, "0\n");
 575                 goto out;
 576         }
 577
 578         ret = sysfs_emit(page, "%llu\n", div_u64(wbt_get_min_lat(q), 1000));
 579 out:
 580         mutex_unlock(&disk->rqos_state_mutex);
 581         return ret;
 582 }
 583
 584 static ssize_t queue_wb_lat_store(struct gendisk *disk, const char *page,
 585                                   size_t count)
 586 {
 587         struct request_queue *q = disk->queue;
 588         struct rq_qos *rqos;
 589         ssize_t ret;
 590         s64 val;
 591         unsigned int memflags;
 592
 593         ret = queue_var_store64(&val, page);
 594         if (ret < 0)
 595                 return ret;
 596         if (val < -1)
 597                 return -EINVAL;
 598
 599         memflags = blk_mq_freeze_queue(q);
 600
 601         rqos = wbt_rq_qos(q);
 602         if (!rqos) {
 603                 ret = wbt_init(disk);
 604                 if (ret)
 605                         goto out;
 606         }
 607
 608         ret = count;
 609         if (val == -1)
 610                 val = wbt_default_latency_nsec(q);
 611         else if (val >= 0)
 612                 val *= 1000ULL;
 613
 614         if (wbt_get_min_lat(q) == val)
 615                 goto out;
 616
 617         /*
 618          * Ensure that the queue is idled, in case the latency update
 619          * ends up either enabling or disabling wbt completely. We can't
 620          * have IO inflight if that happens.
 621          */
 622         blk_mq_quiesce_queue(q);
 623
 624         mutex_lock(&disk->rqos_state_mutex);
 625         wbt_set_min_lat(q, val);
 626         mutex_unlock(&disk->rqos_state_mutex);
 627
 628         blk_mq_unquiesce_queue(q);
 629 out:
 630         blk_mq_unfreeze_queue(q, memflags);
 631
 632         return ret;
 633 }
 634
 635 QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec");
 636 #endif
 637
 638 /* Common attributes for bio-based and request-based queues. */
 639 static struct attribute *queue_attrs[] = {
 640         /*
 641          * Attributes which are protected with q->limits_lock.
 642          */
 643         &queue_max_hw_sectors_entry.attr,
 644         &queue_max_sectors_entry.attr,
 645         &queue_max_segments_entry.attr,
 646         &queue_max_discard_segments_entry.attr,
 647         &queue_max_integrity_segments_entry.attr,
 648         &queue_max_segment_size_entry.attr,
 649         &queue_max_write_streams_entry.attr,
 650         &queue_write_stream_granularity_entry.attr,
 651         &queue_hw_sector_size_entry.attr,
 652         &queue_logical_block_size_entry.attr,
 653         &queue_physical_block_size_entry.attr,
 654         &queue_chunk_sectors_entry.attr,
 655         &queue_io_min_entry.attr,
 656         &queue_io_opt_entry.attr,
 657         &queue_discard_granularity_entry.attr,
 658         &queue_max_discard_sectors_entry.attr,
 659         &queue_max_hw_discard_sectors_entry.attr,
 660         &queue_atomic_write_max_sectors_entry.attr,
 661         &queue_atomic_write_boundary_sectors_entry.attr,
 662         &queue_atomic_write_unit_min_entry.attr,
 663         &queue_atomic_write_unit_max_entry.attr,
 664         &queue_max_write_zeroes_sectors_entry.attr,
 665         &queue_max_zone_append_sectors_entry.attr,
 666         &queue_zone_write_granularity_entry.attr,
 667         &queue_rotational_entry.attr,
 668         &queue_zoned_entry.attr,
 669         &queue_max_open_zones_entry.attr,
 670         &queue_max_active_zones_entry.attr,
 671         &queue_iostats_passthrough_entry.attr,
 672         &queue_iostats_entry.attr,
 673         &queue_stable_writes_entry.attr,
 674         &queue_add_random_entry.attr,
 675         &queue_wc_entry.attr,
 676         &queue_fua_entry.attr,
 677         &queue_dax_entry.attr,
 678         &queue_virt_boundary_mask_entry.attr,
 679         &queue_dma_alignment_entry.attr,
 680         &queue_ra_entry.attr,
 681
 682         /*
 683          * Attributes which don't require locking.
 684          */
 685         &queue_discard_zeroes_data_entry.attr,
 686         &queue_write_same_max_entry.attr,
 687         &queue_nr_zones_entry.attr,
 688         &queue_nomerges_entry.attr,
 689         &queue_poll_entry.attr,
 690         &queue_poll_delay_entry.attr,
 691
 692         NULL,
 693 };
 694
 695 /* Request-based queue attributes that are not relevant for bio-based queues. */
 696 static struct attribute *blk_mq_queue_attrs[] = {
 697         /*
 698          * Attributes which require some form of locking other than
 699          * q->sysfs_lock.
 700          */
 701         &elv_iosched_entry.attr,
 702         &queue_requests_entry.attr,
 703 #ifdef CONFIG_BLK_WBT
 704         &queue_wb_lat_entry.attr,
 705 #endif
 706         /*
 707          * Attributes which don't require locking.
 708          */
 709         &queue_rq_affinity_entry.attr,
 710         &queue_io_timeout_entry.attr,
 711
 712         NULL,
 713 };
 714
 715 static umode_t queue_attr_visible(struct kobject *kobj, struct attribute *attr,
 716                                 int n)
 717 {
 718         struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
 719         struct request_queue *q = disk->queue;
 720
 721         if ((attr == &queue_max_open_zones_entry.attr ||
 722              attr == &queue_max_active_zones_entry.attr) &&
 723             !blk_queue_is_zoned(q))
 724                 return 0;
 725
 726         return attr->mode;
 727 }
 728
 729 static umode_t blk_mq_queue_attr_visible(struct kobject *kobj,
 730                                          struct attribute *attr, int n)
 731 {
 732         struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
 733         struct request_queue *q = disk->queue;
 734
 735         if (!queue_is_mq(q))
 736                 return 0;
 737
 738         if (attr == &queue_io_timeout_entry.attr && !q->mq_ops->timeout)
 739                 return 0;
 740
 741         return attr->mode;
 742 }
 743
 744 static struct attribute_group queue_attr_group = {
 745         .attrs = queue_attrs,
 746         .is_visible = queue_attr_visible,
 747 };
 748
 749 static struct attribute_group blk_mq_queue_attr_group = {
 750         .attrs = blk_mq_queue_attrs,
 751         .is_visible = blk_mq_queue_attr_visible,
 752 };
 753
 754 #define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)
 755
 756 static ssize_t
 757 queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
 758 {
 759         struct queue_sysfs_entry *entry = to_queue(attr);
 760         struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
 761
 762         if (!entry->show && !entry->show_limit)
 763                 return -EIO;
 764
 765         if (entry->show_limit) {
 766                 ssize_t res;
 767
 768                 mutex_lock(&disk->queue->limits_lock);
 769                 res = entry->show_limit(disk, page);
 770                 mutex_unlock(&disk->queue->limits_lock);
 771                 return res;
 772         }
 773
 774         return entry->show(disk, page);
 775 }
 776
 777 static ssize_t
 778 queue_attr_store(struct kobject *kobj, struct attribute *attr,
 779                     const char *page, size_t length)
 780 {
 781         struct queue_sysfs_entry *entry = to_queue(attr);
 782         struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
 783         struct request_queue *q = disk->queue;
 784
 785         if (!entry->store_limit && !entry->store)
 786                 return -EIO;
 787
 788         if (entry->store_limit) {
 789                 ssize_t res;
 790
 791                 struct queue_limits lim = queue_limits_start_update(q);
 792
 793                 res = entry->store_limit(disk, page, length, &lim);
 794                 if (res < 0) {
 795                         queue_limits_cancel_update(q);
 796                         return res;
 797                 }
 798
 799                 res = queue_limits_commit_update_frozen(q, &lim);
 800                 if (res)
 801                         return res;
 802                 return length;
 803         }
 804
 805         return entry->store(disk, page, length);
 806 }
 807
 808 static const struct sysfs_ops queue_sysfs_ops = {
 809         .show   = queue_attr_show,
 810         .store  = queue_attr_store,
 811 };
 812
 813 static const struct attribute_group *blk_queue_attr_groups[] = {
 814         &queue_attr_group,
 815         &blk_mq_queue_attr_group,
 816         NULL
 817 };
 818
 819 static void blk_queue_release(struct kobject *kobj)
 820 {
 821         /* nothing to do here, all data is associated with the parent gendisk */
 822 }
 823
 824 static const struct kobj_type blk_queue_ktype = {
 825         .default_groups = blk_queue_attr_groups,
 826         .sysfs_ops      = &queue_sysfs_ops,
 827         .release        = blk_queue_release,
 828 };
 829
 830 static void blk_debugfs_remove(struct gendisk *disk)
 831 {
 832         struct request_queue *q = disk->queue;
 833
 834         mutex_lock(&q->debugfs_mutex);
 835         blk_trace_shutdown(q);
 836         debugfs_remove_recursive(q->debugfs_dir);
 837         q->debugfs_dir = NULL;
 838         q->sched_debugfs_dir = NULL;
 839         q->rqos_debugfs_dir = NULL;
 840         mutex_unlock(&q->debugfs_mutex);
 841 }
 842
 843 /**
 844  * blk_register_queue - register a block layer queue with sysfs
 845  * @disk: Disk of which the request queue should be registered with sysfs.
 846  */
 847 int blk_register_queue(struct gendisk *disk)
 848 {
 849         struct request_queue *q = disk->queue;
 850         int ret;
 851
 852         kobject_init(&disk->queue_kobj, &blk_queue_ktype);
 853         ret = kobject_add(&disk->queue_kobj, &disk_to_dev(disk)->kobj, "queue");
 854         if (ret < 0)
 855                 goto out_put_queue_kobj;
 856
 857         if (queue_is_mq(q)) {
 858                 ret = blk_mq_sysfs_register(disk);
 859                 if (ret)
 860                         goto out_put_queue_kobj;
 861         }
 862         mutex_lock(&q->sysfs_lock);
 863
 864         mutex_lock(&q->debugfs_mutex);
 865         q->debugfs_dir = debugfs_create_dir(disk->disk_name, blk_debugfs_root);
 866         if (queue_is_mq(q))
 867                 blk_mq_debugfs_register(q);
 868         mutex_unlock(&q->debugfs_mutex);
 869
 870         ret = disk_register_independent_access_ranges(disk);
 871         if (ret)
 872                 goto out_debugfs_remove;
 873
 874         ret = blk_crypto_sysfs_register(disk);
 875         if (ret)
 876                 goto out_unregister_ia_ranges;
 877
 878         if (queue_is_mq(q))
 879                 elevator_set_default(q);
 880         wbt_enable_default(disk);
 881
 882         blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
 883
 884         /* Now everything is ready and send out KOBJ_ADD uevent */
 885         kobject_uevent(&disk->queue_kobj, KOBJ_ADD);
 886         if (q->elevator)
 887                 kobject_uevent(&q->elevator->kobj, KOBJ_ADD);
 888         mutex_unlock(&q->sysfs_lock);
 889
 890         /*
 891          * SCSI probing may synchronously create and destroy a lot of
 892          * request_queues for non-existent devices.  Shutting down a fully
 893          * functional queue takes measureable wallclock time as RCU grace
 894          * periods are involved.  To avoid excessive latency in these
 895          * cases, a request_queue starts out in a degraded mode which is
 896          * faster to shut down and is made fully functional here as
 897          * request_queues for non-existent devices never get registered.
 898          */
 899         blk_queue_flag_set(QUEUE_FLAG_INIT_DONE, q);
 900         percpu_ref_switch_to_percpu(&q->q_usage_counter);
 901
 902         return ret;
 903
 904 out_unregister_ia_ranges:
 905         disk_unregister_independent_access_ranges(disk);
 906 out_debugfs_remove:
 907         blk_debugfs_remove(disk);
 908         mutex_unlock(&q->sysfs_lock);
 909         if (queue_is_mq(q))
 910                 blk_mq_sysfs_unregister(disk);
 911 out_put_queue_kobj:
 912         kobject_put(&disk->queue_kobj);
 913         return ret;
 914 }
 915
 916 /**
 917  * blk_unregister_queue - counterpart of blk_register_queue()
 918  * @disk: Disk of which the request queue should be unregistered from sysfs.
 919  *
 920  * Note: the caller is responsible for guaranteeing that this function is called
 921  * after blk_register_queue() has finished.
 922  */
 923 void blk_unregister_queue(struct gendisk *disk)
 924 {
 925         struct request_queue *q = disk->queue;
 926
 927         if (WARN_ON(!q))
 928                 return;
 929
 930         /* Return early if disk->queue was never registered. */
 931         if (!blk_queue_registered(q))
 932                 return;
 933
 934         /*
 935          * Since sysfs_remove_dir() prevents adding new directory entries
 936          * before removal of existing entries starts, protect against
 937          * concurrent elv_iosched_store() calls.
 938          */
 939         mutex_lock(&q->sysfs_lock);
 940         blk_queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
 941         mutex_unlock(&q->sysfs_lock);
 942
 943         /*
 944          * Remove the sysfs attributes before unregistering the queue data
 945          * structures that can be modified through sysfs.
 946          */
 947         if (queue_is_mq(q))
 948                 blk_mq_sysfs_unregister(disk);
 949         blk_crypto_sysfs_unregister(disk);
 950
 951         mutex_lock(&q->sysfs_lock);
 952         disk_unregister_independent_access_ranges(disk);
 953         mutex_unlock(&q->sysfs_lock);
 954
 955         /* Now that we've deleted all child objects, we can delete the queue. */
 956         kobject_uevent(&disk->queue_kobj, KOBJ_REMOVE);
 957         kobject_del(&disk->queue_kobj);
 958
 959         if (queue_is_mq(q))
 960                 elevator_set_none(q);
 961
 962         blk_debugfs_remove(disk);
 963 }