projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
pktcdvd: stop setting q->queuedata
[linux-2.6-block.git] / drivers / block / ublk_drv.c
CommitLineData
71f28f31
ML		 1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Userspace block device - block device which IO is handled from userspace
4 *
5 * Take full use of io_uring passthrough command for communicating with
6 * ublk userspace daemon(ublksrvd) for handling basic IO request.
7 *
8 * Copyright 2022 Ming Lei <ming.lei@redhat.com>
9 *
10 * (part of code stolen from loop.c)
11 */
12#include <linux/module.h>
13#include <linux/moduleparam.h>
14#include <linux/sched.h>
15#include <linux/fs.h>
16#include <linux/pagemap.h>
17#include <linux/file.h>
18#include <linux/stat.h>
19#include <linux/errno.h>
20#include <linux/major.h>
21#include <linux/wait.h>
22#include <linux/blkdev.h>
23#include <linux/init.h>
24#include <linux/swap.h>
25#include <linux/slab.h>
26#include <linux/compat.h>
27#include <linux/mutex.h>
28#include <linux/writeback.h>
29#include <linux/completion.h>
30#include <linux/highmem.h>
31#include <linux/sysfs.h>
32#include <linux/miscdevice.h>
33#include <linux/falloc.h>
34#include <linux/uio.h>
35#include <linux/ioprio.h>
36#include <linux/sched/mm.h>
37#include <linux/uaccess.h>
38#include <linux/cdev.h>
b66509b8 39#include <linux/io_uring/cmd.h>
71f28f31
ML		 40#include <linux/blk-mq.h>
41#include <linux/delay.h>
42#include <linux/mm.h>
43#include <asm/page.h>
0edb3696 44#include <linux/task_work.h>
4093cb5a 45#include <linux/namei.h>
82840669 46#include <linux/kref.h>
71f28f31
ML		 47#include <uapi/linux/ublk_cmd.h>
48
49#define UBLK_MINORS (1U << MINORBITS)
50
6d8c5afc 51/* All UBLK_F_* have to be included into UBLK_F_ALL */
c86019ff
Z		 52#define UBLK_F_ALL (UBLK_F_SUPPORT_ZERO_COPY \
53 | UBLK_F_URING_CMD_COMP_IN_TASK \
77a440e2 54 | UBLK_F_NEED_GET_DATA \
a0d41dc1 55 | UBLK_F_USER_RECOVERY \
4093cb5a 56 | UBLK_F_USER_RECOVERY_REISSUE \
2d786e66 57 | UBLK_F_UNPRIVILEGED_DEV \
1172d5b8 58 | UBLK_F_CMD_IOCTL_ENCODE \
29802d7c
AH		 59 | UBLK_F_USER_COPY \
60 | UBLK_F_ZONED)
6d8c5afc 61
0aa73170 62/* All UBLK_PARAM_TYPE_* should be included here */
29802d7c
AH		 63#define UBLK_PARAM_TYPE_ALL \
64 (UBLK_PARAM_TYPE_BASIC | UBLK_PARAM_TYPE_DISCARD | \
65 UBLK_PARAM_TYPE_DEVT | UBLK_PARAM_TYPE_ZONED)
0aa73170 66
0edb3696 67struct ublk_rq_data {
7d4a9317 68 struct llist_node node;
82840669
ML		 69
70 struct kref ref;
29802d7c
AH		 71 __u64 sector;
72 __u32 operation;
73 __u32 nr_zones;
0edb3696
ML		 74};
75
71f28f31 76struct ublk_uring_cmd_pdu {
3ab6e94c 77 struct ublk_queue *ubq;
216c8f5e 78 u16 tag;
71f28f31
ML		 79};
80
81/*
82 * io command is active: sqe cmd is received, and its cqe isn't done
83 *
84 * If the flag is set, the io command is owned by ublk driver, and waited
85 * for incoming blk-mq request from the ublk block device.
86 *
87 * If the flag is cleared, the io command will be completed, and owned by
88 * ublk server.
89 */
90#define UBLK_IO_FLAG_ACTIVE 0x01
91
92/*
93 * IO command is completed via cqe, and it is being handled by ublksrv, and
94 * not committed yet
95 *
96 * Basically exclusively with UBLK_IO_FLAG_ACTIVE, so can be served for
97 * cross verification
98 */
99#define UBLK_IO_FLAG_OWNED_BY_SRV 0x02
100
101/*
102 * IO command is aborted, so this flag is set in case of
103 * !UBLK_IO_FLAG_ACTIVE.
104 *
105 * After this flag is observed, any pending or new incoming request
106 * associated with this io command will be failed immediately
107 */
108#define UBLK_IO_FLAG_ABORTED 0x04
109
c86019ff
Z		 110/*
111 * UBLK_IO_FLAG_NEED_GET_DATA is set because IO command requires
112 * get data buffer address from ublksrv.
113 *
114 * Then, bio data could be copied into this data buffer for a WRITE request
115 * after the IO command is issued again and UBLK_IO_FLAG_NEED_GET_DATA is unset.
116 */
117#define UBLK_IO_FLAG_NEED_GET_DATA 0x08
118
85248d67
ML		 119/* atomic RW with ubq->cancel_lock */
120#define UBLK_IO_FLAG_CANCELED 0x80000000
121
71f28f31
ML		 122struct ublk_io {
123 /* userspace buffer address from io cmd */
124 __u64 addr;
125 unsigned int flags;
126 int res;
127
128 struct io_uring_cmd *cmd;
129};
130
131struct ublk_queue {
132 int q_id;
133 int q_depth;
134
0edb3696 135 unsigned long flags;
71f28f31
ML		 136 struct task_struct *ubq_daemon;
137 char *io_cmd_buf;
138
3ab6e94c
ML		 139 struct llist_head io_cmds;
140
71f28f31
ML		 141 unsigned long io_addr; /* mapped vm address */
142 unsigned int max_io_sz;
bbae8d1f 143 bool force_abort;
c0b79b0f 144 bool timeout;
216c8f5e 145 bool canceling;
71f28f31 146 unsigned short nr_io_ready; /* how many ios setup */
85248d67 147 spinlock_t cancel_lock;
71f28f31 148 struct ublk_device *dev;
72495b5a 149 struct ublk_io ios[];
71f28f31
ML		 150};
151
71f28f31
ML		 152struct ublk_device {
153 struct gendisk *ub_disk;
71f28f31
ML		 154
155 char *__queues;
156
29baef78 157 unsigned int queue_size;
71f28f31
ML		 158 struct ublksrv_ctrl_dev_info dev_info;
159
160 struct blk_mq_tag_set tag_set;
161
162 struct cdev cdev;
163 struct device cdev_dev;
164
8d9fdb60
DC		 165#define UB_STATE_OPEN 0
166#define UB_STATE_USED 1
0abe39de 167#define UB_STATE_DELETED 2
fa362045 168 unsigned long state;
71f28f31
ML		 169 int ub_number;
170
171 struct mutex mutex;
172
28dde8c0 173 spinlock_t lock;
71f28f31
ML		 174 struct mm_struct *mm;
175
0aa73170
ML		 176 struct ublk_params params;
177
71f28f31
ML		 178 struct completion completion;
179 unsigned int nr_queues_ready;
73a166d9 180 unsigned int nr_privileged_daemon;
71f28f31 181
bbae8d1f 182 struct work_struct quiesce_work;
71f28f31
ML		 183 struct work_struct stop_work;
184};
185
0aa73170
ML		 186/* header of ublk_params */
187struct ublk_params_header {
188 __u32 len;
189 __u32 types;
190};
191
216c8f5e
ML		 192static bool ublk_abort_requests(struct ublk_device *ub, struct ublk_queue *ubq);
193
29802d7c
AH		 194static inline unsigned int ublk_req_build_flags(struct request *req);
195static inline struct ublksrv_io_desc *ublk_get_iod(struct ublk_queue *ubq,
196 int tag);
9d4ed6d4
AH		 197static inline bool ublk_dev_is_user_copy(const struct ublk_device *ub)
198{
199 return ub->dev_info.flags & UBLK_F_USER_COPY;
200}
201
29802d7c
AH		 202static inline bool ublk_dev_is_zoned(const struct ublk_device *ub)
203{
204 return ub->dev_info.flags & UBLK_F_ZONED;
205}
206
207static inline bool ublk_queue_is_zoned(struct ublk_queue *ubq)
208{
209 return ubq->flags & UBLK_F_ZONED;
210}
211
212#ifdef CONFIG_BLK_DEV_ZONED
213
214static int ublk_get_nr_zones(const struct ublk_device *ub)
215{
216 const struct ublk_param_basic *p = &ub->params.basic;
217
218 /* Zone size is a power of 2 */
219 return p->dev_sectors >> ilog2(p->chunk_sectors);
220}
221
222static int ublk_revalidate_disk_zones(struct ublk_device *ub)
223{
224 return blk_revalidate_disk_zones(ub->ub_disk, NULL);
225}
226
227static int ublk_dev_param_zoned_validate(const struct ublk_device *ub)
228{
229 const struct ublk_param_zoned *p = &ub->params.zoned;
230 int nr_zones;
231
232 if (!ublk_dev_is_zoned(ub))
233 return -EINVAL;
234
235 if (!p->max_zone_append_sectors)
236 return -EINVAL;
237
238 nr_zones = ublk_get_nr_zones(ub);
239
240 if (p->max_active_zones > nr_zones)
241 return -EINVAL;
242
243 if (p->max_open_zones > nr_zones)
244 return -EINVAL;
245
246 return 0;
247}
248
494ea040 249static void ublk_dev_param_zoned_apply(struct ublk_device *ub)
29802d7c 250{
851e0629 251 blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, ub->ub_disk->queue);
29802d7c
AH		 252 blk_queue_required_elevator_features(ub->ub_disk->queue,
253 ELEVATOR_F_ZBD_SEQ_WRITE);
29802d7c 254 ub->ub_disk->nr_zones = ublk_get_nr_zones(ub);
29802d7c
AH		 255}
256
257/* Based on virtblk_alloc_report_buffer */
258static void *ublk_alloc_report_buffer(struct ublk_device *ublk,
259 unsigned int nr_zones, size_t *buflen)
260{
261 struct request_queue *q = ublk->ub_disk->queue;
262 size_t bufsize;
263 void *buf;
264
265 nr_zones = min_t(unsigned int, nr_zones,
266 ublk->ub_disk->nr_zones);
267
268 bufsize = nr_zones * sizeof(struct blk_zone);
269 bufsize =
270 min_t(size_t, bufsize, queue_max_hw_sectors(q) << SECTOR_SHIFT);
271
272 while (bufsize >= sizeof(struct blk_zone)) {
273 buf = kvmalloc(bufsize, GFP_KERNEL | __GFP_NORETRY);
274 if (buf) {
275 *buflen = bufsize;
276 return buf;
277 }
278 bufsize >>= 1;
279 }
280
281 *buflen = 0;
282 return NULL;
283}
284
285static int ublk_report_zones(struct gendisk *disk, sector_t sector,
286 unsigned int nr_zones, report_zones_cb cb, void *data)
287{
288 struct ublk_device *ub = disk->private_data;
289 unsigned int zone_size_sectors = disk->queue->limits.chunk_sectors;
290 unsigned int first_zone = sector >> ilog2(zone_size_sectors);
291 unsigned int done_zones = 0;
292 unsigned int max_zones_per_request;
293 int ret;
294 struct blk_zone *buffer;
295 size_t buffer_length;
296
297 nr_zones = min_t(unsigned int, ub->ub_disk->nr_zones - first_zone,
298 nr_zones);
299
300 buffer = ublk_alloc_report_buffer(ub, nr_zones, &buffer_length);
301 if (!buffer)
302 return -ENOMEM;
303
304 max_zones_per_request = buffer_length / sizeof(struct blk_zone);
305
306 while (done_zones < nr_zones) {
307 unsigned int remaining_zones = nr_zones - done_zones;
308 unsigned int zones_in_request =
309 min_t(unsigned int, remaining_zones, max_zones_per_request);
310 struct request *req;
311 struct ublk_rq_data *pdu;
312 blk_status_t status;
313
314 memset(buffer, 0, buffer_length);
315
316 req = blk_mq_alloc_request(disk->queue, REQ_OP_DRV_IN, 0);
317 if (IS_ERR(req)) {
318 ret = PTR_ERR(req);
319 goto out;
320 }
321
322 pdu = blk_mq_rq_to_pdu(req);
323 pdu->operation = UBLK_IO_OP_REPORT_ZONES;
324 pdu->sector = sector;
325 pdu->nr_zones = zones_in_request;
326
327 ret = blk_rq_map_kern(disk->queue, req, buffer, buffer_length,
328 GFP_KERNEL);
329 if (ret) {
330 blk_mq_free_request(req);
331 goto out;
332 }
333
334 status = blk_execute_rq(req, 0);
335 ret = blk_status_to_errno(status);
336 blk_mq_free_request(req);
337 if (ret)
338 goto out;
339
340 for (unsigned int i = 0; i < zones_in_request; i++) {
341 struct blk_zone *zone = buffer + i;
342
343 /* A zero length zone means no more zones in this response */
344 if (!zone->len)
345 break;
346
347 ret = cb(zone, i, data);
348 if (ret)
349 goto out;
350
351 done_zones++;
352 sector += zone_size_sectors;
353
354 }
355 }
356
357 ret = done_zones;
358
359out:
360 kvfree(buffer);
361 return ret;
362}
363
364static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq,
365 struct request *req)
366{
367 struct ublksrv_io_desc *iod = ublk_get_iod(ubq, req->tag);
368 struct ublk_io *io = &ubq->ios[req->tag];
369 struct ublk_rq_data *pdu = blk_mq_rq_to_pdu(req);
370 u32 ublk_op;
371
372 switch (req_op(req)) {
373 case REQ_OP_ZONE_OPEN:
374 ublk_op = UBLK_IO_OP_ZONE_OPEN;
375 break;
376 case REQ_OP_ZONE_CLOSE:
377 ublk_op = UBLK_IO_OP_ZONE_CLOSE;
378 break;
379 case REQ_OP_ZONE_FINISH:
380 ublk_op = UBLK_IO_OP_ZONE_FINISH;
381 break;
382 case REQ_OP_ZONE_RESET:
383 ublk_op = UBLK_IO_OP_ZONE_RESET;
384 break;
385 case REQ_OP_ZONE_APPEND:
386 ublk_op = UBLK_IO_OP_ZONE_APPEND;
387 break;
851e0629
ML		 388 case REQ_OP_ZONE_RESET_ALL:
389 ublk_op = UBLK_IO_OP_ZONE_RESET_ALL;
390 break;
29802d7c
AH		 391 case REQ_OP_DRV_IN:
392 ublk_op = pdu->operation;
393 switch (ublk_op) {
394 case UBLK_IO_OP_REPORT_ZONES:
395 iod->op_flags = ublk_op | ublk_req_build_flags(req);
396 iod->nr_zones = pdu->nr_zones;
397 iod->start_sector = pdu->sector;
398 return BLK_STS_OK;
399 default:
400 return BLK_STS_IOERR;
401 }
29802d7c 402 case REQ_OP_DRV_OUT:
851e0629 403 /* We do not support drv_out */
29802d7c
AH		 404 return BLK_STS_NOTSUPP;
405 default:
406 return BLK_STS_IOERR;
407 }
408
409 iod->op_flags = ublk_op | ublk_req_build_flags(req);
410 iod->nr_sectors = blk_rq_sectors(req);
411 iod->start_sector = blk_rq_pos(req);
412 iod->addr = io->addr;
413
414 return BLK_STS_OK;
415}
416
417#else
418
419#define ublk_report_zones (NULL)
420
421static int ublk_dev_param_zoned_validate(const struct ublk_device *ub)
422{
423 return -EOPNOTSUPP;
424}
425
494ea040 426static void ublk_dev_param_zoned_apply(struct ublk_device *ub)
29802d7c 427{
29802d7c
AH		 428}
429
430static int ublk_revalidate_disk_zones(struct ublk_device *ub)
431{
432 return 0;
433}
434
435static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq,
436 struct request *req)
437{
c8659bbb 438 return BLK_STS_NOTSUPP;
29802d7c
AH		 439}
440
441#endif
442
82840669
ML		 443static inline void __ublk_complete_rq(struct request *req);
444static void ublk_complete_rq(struct kref *ref);
445
71f28f31 446static dev_t ublk_chr_devt;
2eefd399
IO		 447static const struct class ublk_chr_class = {
448 .name = "ublk-char",
449};
71f28f31
ML		 450
451static DEFINE_IDR(ublk_index_idr);
452static DEFINE_SPINLOCK(ublk_idr_lock);
453static wait_queue_head_t ublk_idr_wq; /* wait until one idr is freed */
454
455static DEFINE_MUTEX(ublk_ctl_mutex);
456
403ebc87
ML		 457/*
458 * Max ublk devices allowed to add
459 *
460 * It can be extended to one per-user limit in future or even controlled
461 * by cgroup.
462 */
56974c07 463#define UBLK_MAX_UBLKS UBLK_MINORS
403ebc87
ML		 464static unsigned int ublks_max = 64;
465static unsigned int ublks_added; /* protected by ublk_ctl_mutex */
466
71f28f31
ML		 467static struct miscdevice ublk_misc;
468
62fe99ce
ML		 469static inline unsigned ublk_pos_to_hwq(loff_t pos)
470{
471 return ((pos - UBLKSRV_IO_BUF_OFFSET) >> UBLK_QID_OFF) &
472 UBLK_QID_BITS_MASK;
473}
474
475static inline unsigned ublk_pos_to_buf_off(loff_t pos)
476{
477 return (pos - UBLKSRV_IO_BUF_OFFSET) & UBLK_IO_BUF_BITS_MASK;
478}
479
480static inline unsigned ublk_pos_to_tag(loff_t pos)
481{
482 return ((pos - UBLKSRV_IO_BUF_OFFSET) >> UBLK_TAG_OFF) &
483 UBLK_TAG_BITS_MASK;
484}
485
0aa73170
ML		 486static void ublk_dev_param_basic_apply(struct ublk_device *ub)
487{
488 struct request_queue *q = ub->ub_disk->queue;
489 const struct ublk_param_basic *p = &ub->params.basic;
490
0aa73170
ML		 491 blk_queue_write_cache(q, p->attrs & UBLK_ATTR_VOLATILE_CACHE,
492 p->attrs & UBLK_ATTR_FUA);
493 if (p->attrs & UBLK_ATTR_ROTATIONAL)
494 blk_queue_flag_clear(QUEUE_FLAG_NONROT, q);
495 else
496 blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
497
0aa73170
ML		 498 if (p->attrs & UBLK_ATTR_READ_ONLY)
499 set_disk_ro(ub->ub_disk, true);
500
501 set_capacity(ub->ub_disk, p->dev_sectors);
502}
503
0aa73170
ML		 504static int ublk_validate_params(const struct ublk_device *ub)
505{
506 /* basic param is the only one which must be set */
507 if (ub->params.types & UBLK_PARAM_TYPE_BASIC) {
508 const struct ublk_param_basic *p = &ub->params.basic;
509
1d166527 510 if (p->logical_bs_shift > PAGE_SHIFT || p->logical_bs_shift < 9)
0aa73170
ML		 511 return -EINVAL;
512
513 if (p->logical_bs_shift > p->physical_bs_shift)
514 return -EINVAL;
515
4bf9cbf3 516 if (p->max_sectors > (ub->dev_info.max_io_buf_bytes >> 9))
0aa73170 517 return -EINVAL;
29802d7c
AH		 518
519 if (ublk_dev_is_zoned(ub) && !p->chunk_sectors)
520 return -EINVAL;
0aa73170
ML		 521 } else
522 return -EINVAL;
523
524 if (ub->params.types & UBLK_PARAM_TYPE_DISCARD) {
525 const struct ublk_param_discard *p = &ub->params.discard;
526
527 /* So far, only support single segment discard */
528 if (p->max_discard_sectors && p->max_discard_segments != 1)
529 return -EINVAL;
530
531 if (!p->discard_granularity)
532 return -EINVAL;
533 }
534
abb864d3
ML		 535 /* dev_t is read-only */
536 if (ub->params.types & UBLK_PARAM_TYPE_DEVT)
537 return -EINVAL;
538
29802d7c
AH		 539 if (ub->params.types & UBLK_PARAM_TYPE_ZONED)
540 return ublk_dev_param_zoned_validate(ub);
541 else if (ublk_dev_is_zoned(ub))
542 return -EINVAL;
543
0aa73170
ML		 544 return 0;
545}
546
494ea040 547static void ublk_apply_params(struct ublk_device *ub)
0aa73170 548{
0aa73170
ML		 549 ublk_dev_param_basic_apply(ub);
550
29802d7c 551 if (ub->params.types & UBLK_PARAM_TYPE_ZONED)
494ea040 552 ublk_dev_param_zoned_apply(ub);
0aa73170
ML		 553}
554
1172d5b8 555static inline bool ublk_support_user_copy(const struct ublk_queue *ubq)
0edb3696 556{
1172d5b8
ML		 557 return ubq->flags & UBLK_F_USER_COPY;
558}
559
82840669
ML		 560static inline bool ublk_need_req_ref(const struct ublk_queue *ubq)
561{
1172d5b8
ML		 562 /*
563 * read()/write() is involved in user copy, so request reference
564 * has to be grabbed
565 */
566 return ublk_support_user_copy(ubq);
82840669
ML		 567}
568
569static inline void ublk_init_req_ref(const struct ublk_queue *ubq,
570 struct request *req)
571{
572 if (ublk_need_req_ref(ubq)) {
573 struct ublk_rq_data *data = blk_mq_rq_to_pdu(req);
574
575 kref_init(&data->ref);
576 }
577}
578
579static inline bool ublk_get_req_ref(const struct ublk_queue *ubq,
580 struct request *req)
581{
582 if (ublk_need_req_ref(ubq)) {
583 struct ublk_rq_data *data = blk_mq_rq_to_pdu(req);
584
585 return kref_get_unless_zero(&data->ref);
586 }
587
588 return true;
589}
590
591static inline void ublk_put_req_ref(const struct ublk_queue *ubq,
592 struct request *req)
593{
594 if (ublk_need_req_ref(ubq)) {
595 struct ublk_rq_data *data = blk_mq_rq_to_pdu(req);
596
597 kref_put(&data->ref, ublk_complete_rq);
598 } else {
599 __ublk_complete_rq(req);
600 }
0edb3696
ML		 601}
602
c86019ff
Z		 603static inline bool ublk_need_get_data(const struct ublk_queue *ubq)
604{
96cf2f54 605 return ubq->flags & UBLK_F_NEED_GET_DATA;
c86019ff
Z		 606}
607
71f28f31
ML		 608static struct ublk_device *ublk_get_device(struct ublk_device *ub)
609{
610 if (kobject_get_unless_zero(&ub->cdev_dev.kobj))
611 return ub;
612 return NULL;
613}
614
615static void ublk_put_device(struct ublk_device *ub)
616{
617 put_device(&ub->cdev_dev);
618}
619
620static inline struct ublk_queue *ublk_get_queue(struct ublk_device *dev,
621 int qid)
622{
623 return (struct ublk_queue *)&(dev->__queues[qid * dev->queue_size]);
624}
625
626static inline bool ublk_rq_has_data(const struct request *rq)
627{
731e208d 628 return bio_has_data(rq->bio);
71f28f31
ML		 629}
630
631static inline struct ublksrv_io_desc *ublk_get_iod(struct ublk_queue *ubq,
632 int tag)
633{
634 return (struct ublksrv_io_desc *)
635 &(ubq->io_cmd_buf[tag * sizeof(struct ublksrv_io_desc)]);
636}
637
638static inline char *ublk_queue_cmd_buf(struct ublk_device *ub, int q_id)
639{
640 return ublk_get_queue(ub, q_id)->io_cmd_buf;
641}
642
643static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub, int q_id)
644{
645 struct ublk_queue *ubq = ublk_get_queue(ub, q_id);
646
647 return round_up(ubq->q_depth * sizeof(struct ublksrv_io_desc),
648 PAGE_SIZE);
649}
650
a0d41dc1
Z		 651static inline bool ublk_queue_can_use_recovery_reissue(
652 struct ublk_queue *ubq)
653{
96cf2f54
ML		 654 return (ubq->flags & UBLK_F_USER_RECOVERY) &&
655 (ubq->flags & UBLK_F_USER_RECOVERY_REISSUE);
a0d41dc1
Z		 656}
657
77a440e2
Z		 658static inline bool ublk_queue_can_use_recovery(
659 struct ublk_queue *ubq)
660{
96cf2f54 661 return ubq->flags & UBLK_F_USER_RECOVERY;
77a440e2
Z		 662}
663
664static inline bool ublk_can_use_recovery(struct ublk_device *ub)
665{
96cf2f54 666 return ub->dev_info.flags & UBLK_F_USER_RECOVERY;
77a440e2
Z		 667}
668
6d9e6dfd
CH		 669static void ublk_free_disk(struct gendisk *disk)
670{
671 struct ublk_device *ub = disk->private_data;
672
673 clear_bit(UB_STATE_USED, &ub->state);
674 put_device(&ub->cdev_dev);
675}
676
48a90519
ML		 677static void ublk_store_owner_uid_gid(unsigned int *owner_uid,
678 unsigned int *owner_gid)
679{
680 kuid_t uid;
681 kgid_t gid;
682
683 current_uid_gid(&uid, &gid);
684
685 *owner_uid = from_kuid(&init_user_ns, uid);
686 *owner_gid = from_kgid(&init_user_ns, gid);
687}
688
05bdb996 689static int ublk_open(struct gendisk *disk, blk_mode_t mode)
48a90519 690{
d32e2bf8 691 struct ublk_device *ub = disk->private_data;
48a90519
ML		 692
693 if (capable(CAP_SYS_ADMIN))
694 return 0;
695
696 /*
697 * If it is one unprivileged device, only owner can open
698 * the disk. Otherwise it could be one trap made by one
699 * evil user who grants this disk's privileges to other
700 * users deliberately.
701 *
702 * This way is reasonable too given anyone can create
703 * unprivileged device, and no need other's grant.
704 */
705 if (ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV) {
706 unsigned int curr_uid, curr_gid;
707
708 ublk_store_owner_uid_gid(&curr_uid, &curr_gid);
709
710 if (curr_uid != ub->dev_info.owner_uid || curr_gid !=
711 ub->dev_info.owner_gid)
712 return -EPERM;
713 }
714
715 return 0;
716}
717
71f28f31
ML		 718static const struct block_device_operations ub_fops = {
719 .owner = THIS_MODULE,
48a90519 720 .open = ublk_open,
6d9e6dfd 721 .free_disk = ublk_free_disk,
29802d7c 722 .report_zones = ublk_report_zones,
71f28f31
ML		 723};
724
725#define UBLK_MAX_PIN_PAGES 32
726
71f28f31
ML		 727struct ublk_io_iter {
728 struct page *pages[UBLK_MAX_PIN_PAGES];
71f28f31
ML		 729 struct bio *bio;
730 struct bvec_iter iter;
731};
732
981f95a5
ML		 733/* return how many pages are copied */
734static void ublk_copy_io_pages(struct ublk_io_iter *data,
735 size_t total, size_t pg_off, int dir)
71f28f31 736{
71f28f31
ML		 737 unsigned done = 0;
738 unsigned pg_idx = 0;
739
740 while (done < total) {
741 struct bio_vec bv = bio_iter_iovec(data->bio, data->iter);
981f95a5
ML		 742 unsigned int bytes = min3(bv.bv_len, (unsigned)total - done,
743 (unsigned)(PAGE_SIZE - pg_off));
71f28f31
ML		 744 void *bv_buf = bvec_kmap_local(&bv);
745 void *pg_buf = kmap_local_page(data->pages[pg_idx]);
746
981f95a5
ML		 747 if (dir == ITER_DEST)
748 memcpy(pg_buf + pg_off, bv_buf, bytes);
71f28f31 749 else
981f95a5 750 memcpy(bv_buf, pg_buf + pg_off, bytes);
71f28f31
ML		 751
752 kunmap_local(pg_buf);
753 kunmap_local(bv_buf);
754
755 /* advance page array */
981f95a5
ML		 756 pg_off += bytes;
757 if (pg_off == PAGE_SIZE) {
71f28f31 758 pg_idx += 1;
981f95a5 759 pg_off = 0;
71f28f31
ML		 760 }
761
762 done += bytes;
763
764 /* advance bio */
765 bio_advance_iter_single(data->bio, &data->iter, bytes);
766 if (!data->iter.bi_size) {
767 data->bio = data->bio->bi_next;
768 if (data->bio == NULL)
769 break;
770 data->iter = data->bio->bi_iter;
771 }
772 }
71f28f31 773}
71f28f31 774
38f2dd34
ML		 775static bool ublk_advance_io_iter(const struct request *req,
776 struct ublk_io_iter *iter, unsigned int offset)
777{
778 struct bio *bio = req->bio;
779
780 for_each_bio(bio) {
781 if (bio->bi_iter.bi_size > offset) {
782 iter->bio = bio;
783 iter->iter = bio->bi_iter;
784 bio_advance_iter(iter->bio, &iter->iter, offset);
785 return true;
786 }
787 offset -= bio->bi_iter.bi_size;
788 }
789 return false;
71f28f31
ML		 790}
791
981f95a5
ML		 792/*
793 * Copy data between request pages and io_iter, and 'offset'
794 * is the start point of linear offset of request.
795 */
796static size_t ublk_copy_user_pages(const struct request *req,
38f2dd34 797 unsigned offset, struct iov_iter *uiter, int dir)
71f28f31 798{
38f2dd34 799 struct ublk_io_iter iter;
981f95a5
ML		 800 size_t done = 0;
801
38f2dd34
ML		 802 if (!ublk_advance_io_iter(req, &iter, offset))
803 return 0;
804
981f95a5
ML		 805 while (iov_iter_count(uiter) && iter.bio) {
806 unsigned nr_pages;
b8b637d7
ML		 807 ssize_t len;
808 size_t off;
981f95a5
ML		 809 int i;
810
811 len = iov_iter_get_pages2(uiter, iter.pages,
812 iov_iter_count(uiter),
813 UBLK_MAX_PIN_PAGES, &off);
814 if (len <= 0)
815 return done;
816
817 ublk_copy_io_pages(&iter, len, off, dir);
818 nr_pages = DIV_ROUND_UP(len + off, PAGE_SIZE);
819 for (i = 0; i < nr_pages; i++) {
820 if (dir == ITER_DEST)
71f28f31
ML		 821 set_page_dirty(iter.pages[i]);
822 put_page(iter.pages[i]);
823 }
981f95a5 824 done += len;
71f28f31
ML		 825 }
826
827 return done;
828}
829
2f3af723
ML		 830static inline bool ublk_need_map_req(const struct request *req)
831{
832 return ublk_rq_has_data(req) && req_op(req) == REQ_OP_WRITE;
833}
834
835static inline bool ublk_need_unmap_req(const struct request *req)
836{
29802d7c
AH		 837 return ublk_rq_has_data(req) &&
838 (req_op(req) == REQ_OP_READ || req_op(req) == REQ_OP_DRV_IN);
2f3af723
ML		 839}
840
71f28f31
ML		 841static int ublk_map_io(const struct ublk_queue *ubq, const struct request *req,
842 struct ublk_io *io)
843{
844 const unsigned int rq_bytes = blk_rq_bytes(req);
23ef8220 845
1172d5b8
ML		 846 if (ublk_support_user_copy(ubq))
847 return rq_bytes;
848
71f28f31
ML		 849 /*
850 * no zero copy, we delay copy WRITE request data into ublksrv
851 * context and the big benefit is that pinning pages in current
852 * context is pretty fast, see ublk_pin_user_pages
853 */
2f3af723 854 if (ublk_need_map_req(req)) {
981f95a5 855 struct iov_iter iter;
981f95a5 856 const int dir = ITER_DEST;
71f28f31 857
9fd7874c 858 import_ubuf(dir, u64_to_user_ptr(io->addr), rq_bytes, &iter);
38f2dd34 859 return ublk_copy_user_pages(req, 0, &iter, dir);
71f28f31
ML		 860 }
861 return rq_bytes;
862}
863
864static int ublk_unmap_io(const struct ublk_queue *ubq,
865 const struct request *req,
866 struct ublk_io *io)
867{
868 const unsigned int rq_bytes = blk_rq_bytes(req);
869
1172d5b8
ML		 870 if (ublk_support_user_copy(ubq))
871 return rq_bytes;
872
2f3af723 873 if (ublk_need_unmap_req(req)) {
981f95a5 874 struct iov_iter iter;
981f95a5 875 const int dir = ITER_SOURCE;
71f28f31
ML		 876
877 WARN_ON_ONCE(io->res > rq_bytes);
878
9fd7874c 879 import_ubuf(dir, u64_to_user_ptr(io->addr), io->res, &iter);
38f2dd34 880 return ublk_copy_user_pages(req, 0, &iter, dir);
71f28f31
ML		 881 }
882 return rq_bytes;
883}
884
885static inline unsigned int ublk_req_build_flags(struct request *req)
886{
887 unsigned flags = 0;
888
889 if (req->cmd_flags & REQ_FAILFAST_DEV)
890 flags |= UBLK_IO_F_FAILFAST_DEV;
891
892 if (req->cmd_flags & REQ_FAILFAST_TRANSPORT)
893 flags |= UBLK_IO_F_FAILFAST_TRANSPORT;
894
895 if (req->cmd_flags & REQ_FAILFAST_DRIVER)
896 flags |= UBLK_IO_F_FAILFAST_DRIVER;
897
898 if (req->cmd_flags & REQ_META)
899 flags |= UBLK_IO_F_META;
900
71f28f31
ML		 901 if (req->cmd_flags & REQ_FUA)
902 flags |= UBLK_IO_F_FUA;
903
71f28f31
ML		 904 if (req->cmd_flags & REQ_NOUNMAP)
905 flags |= UBLK_IO_F_NOUNMAP;
906
907 if (req->cmd_flags & REQ_SWAP)
908 flags |= UBLK_IO_F_SWAP;
909
910 return flags;
911}
912
f2450f8a 913static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req)
71f28f31
ML		 914{
915 struct ublksrv_io_desc *iod = ublk_get_iod(ubq, req->tag);
916 struct ublk_io *io = &ubq->ios[req->tag];
29802d7c 917 enum req_op op = req_op(req);
71f28f31
ML		 918 u32 ublk_op;
919
29802d7c
AH		 920 if (!ublk_queue_is_zoned(ubq) &&
921 (op_is_zone_mgmt(op) || op == REQ_OP_ZONE_APPEND))
c8659bbb 922 return BLK_STS_IOERR;
29802d7c 923
71f28f31
ML		 924 switch (req_op(req)) {
925 case REQ_OP_READ:
926 ublk_op = UBLK_IO_OP_READ;
927 break;
928 case REQ_OP_WRITE:
929 ublk_op = UBLK_IO_OP_WRITE;
930 break;
931 case REQ_OP_FLUSH:
932 ublk_op = UBLK_IO_OP_FLUSH;
933 break;
934 case REQ_OP_DISCARD:
935 ublk_op = UBLK_IO_OP_DISCARD;
936 break;
937 case REQ_OP_WRITE_ZEROES:
938 ublk_op = UBLK_IO_OP_WRITE_ZEROES;
939 break;
940 default:
29802d7c
AH		 941 if (ublk_queue_is_zoned(ubq))
942 return ublk_setup_iod_zoned(ubq, req);
71f28f31
ML		 943 return BLK_STS_IOERR;
944 }
945
946 /* need to translate since kernel may change */
947 iod->op_flags = ublk_op | ublk_req_build_flags(req);
948 iod->nr_sectors = blk_rq_sectors(req);
949 iod->start_sector = blk_rq_pos(req);
950 iod->addr = io->addr;
951
952 return BLK_STS_OK;
953}
954
955static inline struct ublk_uring_cmd_pdu *ublk_get_uring_cmd_pdu(
956 struct io_uring_cmd *ioucmd)
957{
958 return (struct ublk_uring_cmd_pdu *)&ioucmd->pdu;
959}
960
966120b5 961static inline bool ubq_daemon_is_dying(struct ublk_queue *ubq)
71f28f31
ML		 962{
963 return ubq->ubq_daemon->flags & PF_EXITING;
964}
965
966/* todo: handle partial completion */
82840669 967static inline void __ublk_complete_rq(struct request *req)
71f28f31
ML		 968{
969 struct ublk_queue *ubq = req->mq_hctx->driver_data;
970 struct ublk_io *io = &ubq->ios[req->tag];
971 unsigned int unmapped_bytes;
903f8aee 972 blk_status_t res = BLK_STS_OK;
71f28f31 973
82840669
ML		 974 /* called from ublk_abort_queue() code path */
975 if (io->flags & UBLK_IO_FLAG_ABORTED) {
976 res = BLK_STS_IOERR;
977 goto exit;
978 }
979
71f28f31
ML		 980 /* failed read IO if nothing is read */
981 if (!io->res && req_op(req) == REQ_OP_READ)
982 io->res = -EIO;
983
984 if (io->res < 0) {
903f8aee
ML		 985 res = errno_to_blk_status(io->res);
986 goto exit;
71f28f31
ML		 987 }
988
989 /*
b352389e 990 * FLUSH, DISCARD or WRITE_ZEROES usually won't return bytes returned, so end them
71f28f31
ML		 991 * directly.
992 *
993 * Both the two needn't unmap.
994 */
29802d7c
AH		 995 if (req_op(req) != REQ_OP_READ && req_op(req) != REQ_OP_WRITE &&
996 req_op(req) != REQ_OP_DRV_IN)
903f8aee 997 goto exit;
71f28f31
ML		 998
999 /* for READ request, writing data in iod->addr to rq buffers */
1000 unmapped_bytes = ublk_unmap_io(ubq, req, io);
1001
1002 /*
1003 * Extremely impossible since we got data filled in just before
1004 *
1005 * Re-read simply for this unlikely case.
1006 */
1007 if (unlikely(unmapped_bytes < io->res))
1008 io->res = unmapped_bytes;
1009
1010 if (blk_update_request(req, BLK_STS_OK, io->res))
1011 blk_mq_requeue_request(req, true);
1012 else
1013 __blk_mq_end_request(req, BLK_STS_OK);
903f8aee
ML		 1014
1015 return;
1016exit:
1017 blk_mq_end_request(req, res);
71f28f31
ML		 1018}
1019
82840669
ML		 1020static void ublk_complete_rq(struct kref *ref)
1021{
1022 struct ublk_rq_data *data = container_of(ref, struct ublk_rq_data,
1023 ref);
1024 struct request *req = blk_mq_rq_from_pdu(data);
1025
1026 __ublk_complete_rq(req);
1027}
1028
71f28f31 1029/*
bb241747
Z		 1030 * Since __ublk_rq_task_work always fails requests immediately during
1031 * exiting, __ublk_fail_req() is only called from abort context during
1032 * exiting. So lock is unnecessary.
71f28f31
ML		 1033 *
1034 * Also aborting may not be started yet, keep in mind that one failed
1035 * request may be issued by block layer again.
1036 */
a0d41dc1
Z		 1037static void __ublk_fail_req(struct ublk_queue *ubq, struct ublk_io *io,
1038 struct request *req)
71f28f31
ML		 1039{
1040 WARN_ON_ONCE(io->flags & UBLK_IO_FLAG_ACTIVE);
1041
b4e1353f
ML		 1042 if (ublk_queue_can_use_recovery_reissue(ubq))
1043 blk_mq_requeue_request(req, false);
1044 else
1045 ublk_put_req_ref(ubq, req);
71f28f31
ML		 1046}
1047
9d2789ac
JA		 1048static void ubq_complete_io_cmd(struct ublk_io *io, int res,
1049 unsigned issue_flags)
c86019ff
Z		 1050{
1051 /* mark this cmd owned by ublksrv */
1052 io->flags |= UBLK_IO_FLAG_OWNED_BY_SRV;
1053
1054 /*
1055 * clear ACTIVE since we are done with this sqe/cmd slot
1056 * We can only accept io cmd in case of being not active.
1057 */
1058 io->flags &= ~UBLK_IO_FLAG_ACTIVE;
1059
1060 /* tell ublksrv one io request is coming */
9d2789ac 1061 io_uring_cmd_done(io->cmd, res, 0, issue_flags);
c86019ff
Z		 1062}
1063
71f28f31
ML		 1064#define UBLK_REQUEUE_DELAY_MS 3
1065
42cf5fc5
Z		 1066static inline void __ublk_abort_rq(struct ublk_queue *ubq,
1067 struct request *rq)
1068{
1069 /* We cannot process this rq so just requeue it. */
1070 if (ublk_queue_can_use_recovery(ubq))
1071 blk_mq_requeue_request(rq, false);
1072 else
1073 blk_mq_end_request(rq, BLK_STS_IOERR);
42cf5fc5
Z		 1074}
1075
9d2789ac
JA		 1076static inline void __ublk_rq_task_work(struct request *req,
1077 unsigned issue_flags)
71f28f31 1078{
71f28f31
ML		 1079 struct ublk_queue *ubq = req->mq_hctx->driver_data;
1080 int tag = req->tag;
1081 struct ublk_io *io = &ubq->ios[tag];
71f28f31
ML		 1082 unsigned int mapped_bytes;
1083
1084 pr_devel("%s: complete: op %d, qid %d tag %d io_flags %x addr %llx\n",
1085 __func__, io->cmd->cmd_op, ubq->q_id, req->tag, io->flags,
1086 ublk_get_iod(ubq, req->tag)->addr);
1087
ae3f7193
Z		 1088 /*
1089 * Task is exiting if either:
1090 *
1091 * (1) current != ubq_daemon.
1092 * io_uring_cmd_complete_in_task() tries to run task_work
1093 * in a workqueue if ubq_daemon(cmd's task) is PF_EXITING.
1094 *
1095 * (2) current->flags & PF_EXITING.
1096 */
1097 if (unlikely(current != ubq->ubq_daemon || current->flags & PF_EXITING)) {
42cf5fc5 1098 __ublk_abort_rq(ubq, req);
71f28f31
ML		 1099 return;
1100 }
1101
2f3af723 1102 if (ublk_need_get_data(ubq) && ublk_need_map_req(req)) {
c86019ff
Z		 1103 /*
1104 * We have not handled UBLK_IO_NEED_GET_DATA command yet,
1105 * so immepdately pass UBLK_IO_RES_NEED_GET_DATA to ublksrv
1106 * and notify it.
1107 */
1108 if (!(io->flags & UBLK_IO_FLAG_NEED_GET_DATA)) {
1109 io->flags |= UBLK_IO_FLAG_NEED_GET_DATA;
1110 pr_devel("%s: need get data. op %d, qid %d tag %d io_flags %x\n",
1111 __func__, io->cmd->cmd_op, ubq->q_id,
1112 req->tag, io->flags);
9d2789ac 1113 ubq_complete_io_cmd(io, UBLK_IO_RES_NEED_GET_DATA, issue_flags);
c86019ff
Z		 1114 return;
1115 }
1116 /*
1117 * We have handled UBLK_IO_NEED_GET_DATA command,
1118 * so clear UBLK_IO_FLAG_NEED_GET_DATA now and just
1119 * do the copy work.
1120 */
1121 io->flags &= ~UBLK_IO_FLAG_NEED_GET_DATA;
92cb6e2e
Z		 1122 /* update iod->addr because ublksrv may have passed a new io buffer */
1123 ublk_get_iod(ubq, req->tag)->addr = io->addr;
1124 pr_devel("%s: update iod->addr: op %d, qid %d tag %d io_flags %x addr %llx\n",
1125 __func__, io->cmd->cmd_op, ubq->q_id, req->tag, io->flags,
1126 ublk_get_iod(ubq, req->tag)->addr);
c86019ff
Z		 1127 }
1128
71f28f31
ML		 1129 mapped_bytes = ublk_map_io(ubq, req, io);
1130
1131 /* partially mapped, update io descriptor */
1132 if (unlikely(mapped_bytes != blk_rq_bytes(req))) {
1133 /*
1134 * Nothing mapped, retry until we succeed.
1135 *
1136 * We may never succeed in mapping any bytes here because
1137 * of OOM. TODO: reserve one buffer with single page pinned
1138 * for providing forward progress guarantee.
1139 */
1140 if (unlikely(!mapped_bytes)) {
1141 blk_mq_requeue_request(req, false);
1142 blk_mq_delay_kick_requeue_list(req->q,
1143 UBLK_REQUEUE_DELAY_MS);
1144 return;
1145 }
1146
1147 ublk_get_iod(ubq, req->tag)->nr_sectors =
1148 mapped_bytes >> 9;
1149 }
1150
82840669 1151 ublk_init_req_ref(ubq, req);
9d2789ac 1152 ubq_complete_io_cmd(io, UBLK_IO_RES_OK, issue_flags);
71f28f31
ML		 1153}
1154
9d2789ac
JA		 1155static inline void ublk_forward_io_cmds(struct ublk_queue *ubq,
1156 unsigned issue_flags)
7d4a9317
ML		 1157{
1158 struct llist_node *io_cmds = llist_del_all(&ubq->io_cmds);
1159 struct ublk_rq_data *data, *tmp;
1160
1161 io_cmds = llist_reverse_order(io_cmds);
1162 llist_for_each_entry_safe(data, tmp, io_cmds, node)
9d2789ac 1163 __ublk_rq_task_work(blk_mq_rq_from_pdu(data), issue_flags);
7d4a9317
ML		 1164}
1165
9d2789ac 1166static void ublk_rq_task_work_cb(struct io_uring_cmd *cmd, unsigned issue_flags)
0edb3696
ML		 1167{
1168 struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
3ab6e94c 1169 struct ublk_queue *ubq = pdu->ubq;
0edb3696 1170
9d2789ac 1171 ublk_forward_io_cmds(ubq, issue_flags);
0edb3696
ML		 1172}
1173
7d4a9317 1174static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq)
3ab6e94c 1175{
7d4a9317 1176 struct ublk_rq_data *data = blk_mq_rq_to_pdu(rq);
3ab6e94c 1177
b4e1353f
ML		 1178 if (llist_add(&data->node, &ubq->io_cmds)) {
1179 struct ublk_io *io = &ubq->ios[rq->tag];
7d4a9317 1180
216c8f5e 1181 io_uring_cmd_complete_in_task(io->cmd, ublk_rq_task_work_cb);
3ab6e94c
ML		 1182 }
1183}
1184
c0b79b0f
ML		 1185static enum blk_eh_timer_return ublk_timeout(struct request *rq)
1186{
1187 struct ublk_queue *ubq = rq->mq_hctx->driver_data;
216c8f5e
ML		 1188 unsigned int nr_inflight = 0;
1189 int i;
c0b79b0f
ML		 1190
1191 if (ubq->flags & UBLK_F_UNPRIVILEGED_DEV) {
1192 if (!ubq->timeout) {
1193 send_sig(SIGKILL, ubq->ubq_daemon, 0);
1194 ubq->timeout = true;
1195 }
1196
1197 return BLK_EH_DONE;
1198 }
1199
216c8f5e
ML		 1200 if (!ubq_daemon_is_dying(ubq))
1201 return BLK_EH_RESET_TIMER;
1202
1203 for (i = 0; i < ubq->q_depth; i++) {
1204 struct ublk_io *io = &ubq->ios[i];
1205
1206 if (!(io->flags & UBLK_IO_FLAG_ACTIVE))
1207 nr_inflight++;
1208 }
1209
1210 /* cancelable uring_cmd can't help us if all commands are in-flight */
1211 if (nr_inflight == ubq->q_depth) {
1212 struct ublk_device *ub = ubq->dev;
1213
1214 if (ublk_abort_requests(ub, ubq)) {
1215 if (ublk_can_use_recovery(ub))
1216 schedule_work(&ub->quiesce_work);
1217 else
1218 schedule_work(&ub->stop_work);
1219 }
1220 return BLK_EH_DONE;
1221 }
1222
c0b79b0f
ML		 1223 return BLK_EH_RESET_TIMER;
1224}
1225
71f28f31
ML		 1226static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx,
1227 const struct blk_mq_queue_data *bd)
1228{
1229 struct ublk_queue *ubq = hctx->driver_data;
1230 struct request *rq = bd->rq;
71f28f31
ML		 1231 blk_status_t res;
1232
1233 /* fill iod to slot in io cmd buffer */
1234 res = ublk_setup_iod(ubq, rq);
1235 if (unlikely(res != BLK_STS_OK))
1236 return BLK_STS_IOERR;
3ab6e94c 1237
bbae8d1f
Z		 1238 /* With recovery feature enabled, force_abort is set in
1239 * ublk_stop_dev() before calling del_gendisk(). We have to
1240 * abort all requeued and new rqs here to let del_gendisk()
1241 * move on. Besides, we cannot not call io_uring_cmd_complete_in_task()
1242 * to avoid UAF on io_uring ctx.
1243 *
1244 * Note: force_abort is guaranteed to be seen because it is set
1245 * before request queue is unqiuesced.
1246 */
1247 if (ublk_queue_can_use_recovery(ubq) && unlikely(ubq->force_abort))
1248 return BLK_STS_IOERR;
71f28f31 1249
216c8f5e 1250 if (unlikely(ubq->canceling)) {
42cf5fc5
Z		 1251 __ublk_abort_rq(ubq, rq);
1252 return BLK_STS_OK;
71f28f31
ML		 1253 }
1254
b4e1353f 1255 blk_mq_start_request(bd->rq);
7d4a9317 1256 ublk_queue_cmd(ubq, rq);
0edb3696 1257
71f28f31
ML		 1258 return BLK_STS_OK;
1259}
1260
71f28f31
ML		 1261static int ublk_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
1262 unsigned int hctx_idx)
1263{
cebbe577 1264 struct ublk_device *ub = driver_data;
71f28f31
ML		 1265 struct ublk_queue *ubq = ublk_get_queue(ub, hctx->queue_num);
1266
1267 hctx->driver_data = ubq;
1268 return 0;
1269}
1270
1271static const struct blk_mq_ops ublk_mq_ops = {
1272 .queue_rq = ublk_queue_rq,
1273 .init_hctx = ublk_init_hctx,
c0b79b0f 1274 .timeout = ublk_timeout,
71f28f31
ML		 1275};
1276
1277static int ublk_ch_open(struct inode *inode, struct file *filp)
1278{
1279 struct ublk_device *ub = container_of(inode->i_cdev,
1280 struct ublk_device, cdev);
1281
fa362045
CH		 1282 if (test_and_set_bit(UB_STATE_OPEN, &ub->state))
1283 return -EBUSY;
1284 filp->private_data = ub;
1285 return 0;
71f28f31
ML		 1286}
1287
1288static int ublk_ch_release(struct inode *inode, struct file *filp)
1289{
1290 struct ublk_device *ub = filp->private_data;
1291
fa362045 1292 clear_bit(UB_STATE_OPEN, &ub->state);
71f28f31
ML		 1293 return 0;
1294}
1295
1296/* map pre-allocated per-queue cmd buffer to ublksrv daemon */
1297static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma)
1298{
1299 struct ublk_device *ub = filp->private_data;
1300 size_t sz = vma->vm_end - vma->vm_start;
1301 unsigned max_sz = UBLK_MAX_QUEUE_DEPTH * sizeof(struct ublksrv_io_desc);
1302 unsigned long pfn, end, phys_off = vma->vm_pgoff << PAGE_SHIFT;
1303 int q_id, ret = 0;
1304
28dde8c0 1305 spin_lock(&ub->lock);
71f28f31
ML		 1306 if (!ub->mm)
1307 ub->mm = current->mm;
1308 if (current->mm != ub->mm)
1309 ret = -EINVAL;
28dde8c0 1310 spin_unlock(&ub->lock);
71f28f31
ML		 1311
1312 if (ret)
1313 return ret;
1314
1315 if (vma->vm_flags & VM_WRITE)
1316 return -EPERM;
1317
1318 end = UBLKSRV_CMD_BUF_OFFSET + ub->dev_info.nr_hw_queues * max_sz;
1319 if (phys_off < UBLKSRV_CMD_BUF_OFFSET || phys_off >= end)
1320 return -EINVAL;
1321
1322 q_id = (phys_off - UBLKSRV_CMD_BUF_OFFSET) / max_sz;
1323 pr_devel("%s: qid %d, pid %d, addr %lx pg_off %lx sz %lu\n",
1324 __func__, q_id, current->pid, vma->vm_start,
1325 phys_off, (unsigned long)sz);
1326
1327 if (sz != ublk_queue_cmd_buf_size(ub, q_id))
1328 return -EINVAL;
1329
1330 pfn = virt_to_phys(ublk_queue_cmd_buf(ub, q_id)) >> PAGE_SHIFT;
1331 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
1332}
1333
1334static void ublk_commit_completion(struct ublk_device *ub,
fd9b8547 1335 const struct ublksrv_io_cmd *ub_cmd)
71f28f31
ML		 1336{
1337 u32 qid = ub_cmd->q_id, tag = ub_cmd->tag;
1338 struct ublk_queue *ubq = ublk_get_queue(ub, qid);
1339 struct ublk_io *io = &ubq->ios[tag];
1340 struct request *req;
1341
1342 /* now this cmd slot is owned by nbd driver */
1343 io->flags &= ~UBLK_IO_FLAG_OWNED_BY_SRV;
1344 io->res = ub_cmd->result;
1345
1346 /* find the io request and complete */
1347 req = blk_mq_tag_to_rq(ub->tag_set.tags[qid], tag);
e24721e4
ML		 1348 if (WARN_ON_ONCE(unlikely(!req)))
1349 return;
71f28f31 1350
29802d7c
AH		 1351 if (req_op(req) == REQ_OP_ZONE_APPEND)
1352 req->__sector = ub_cmd->zone_append_lba;
1353
e24721e4 1354 if (likely(!blk_should_fake_timeout(req->q)))
82840669 1355 ublk_put_req_ref(ubq, req);
71f28f31
ML		 1356}
1357
1358/*
216c8f5e
ML		 1359 * Called from ubq_daemon context via cancel fn, meantime quiesce ublk
1360 * blk-mq queue, so we are called exclusively with blk-mq and ubq_daemon
1361 * context, so everything is serialized.
71f28f31
ML		 1362 */
1363static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq)
1364{
1365 int i;
1366
71f28f31
ML		 1367 for (i = 0; i < ubq->q_depth; i++) {
1368 struct ublk_io *io = &ubq->ios[i];
1369
1370 if (!(io->flags & UBLK_IO_FLAG_ACTIVE)) {
1371 struct request *rq;
1372
1373 /*
1374 * Either we fail the request or ublk_rq_task_work_fn
1375 * will do it
1376 */
1377 rq = blk_mq_tag_to_rq(ub->tag_set.tags[ubq->q_id], i);
b4e1353f
ML		 1378 if (rq && blk_mq_request_started(rq)) {
1379 io->flags |= UBLK_IO_FLAG_ABORTED;
a0d41dc1 1380 __ublk_fail_req(ubq, io, rq);
b4e1353f 1381 }
71f28f31
ML		 1382 }
1383 }
71f28f31
ML		 1384}
1385
216c8f5e 1386static bool ublk_abort_requests(struct ublk_device *ub, struct ublk_queue *ubq)
71f28f31 1387{
bd23f6c2 1388 struct gendisk *disk;
216c8f5e
ML		 1389
1390 spin_lock(&ubq->cancel_lock);
1391 if (ubq->canceling) {
1392 spin_unlock(&ubq->cancel_lock);
1393 return false;
1394 }
1395 ubq->canceling = true;
1396 spin_unlock(&ubq->cancel_lock);
71f28f31 1397
bd23f6c2
ML		 1398 spin_lock(&ub->lock);
1399 disk = ub->ub_disk;
1400 if (disk)
1401 get_device(disk_to_dev(disk));
1402 spin_unlock(&ub->lock);
1403
1404 /* Our disk has been dead */
1405 if (!disk)
1406 return false;
1407
1408 /* Now we are serialized with ublk_queue_rq() */
1409 blk_mq_quiesce_queue(disk->queue);
216c8f5e
ML		 1410 /* abort queue is for making forward progress */
1411 ublk_abort_queue(ub, ubq);
bd23f6c2
ML		 1412 blk_mq_unquiesce_queue(disk->queue);
1413 put_device(disk_to_dev(disk));
1414
1415 return true;
1416}
1417
216c8f5e
ML		 1418static void ublk_cancel_cmd(struct ublk_queue *ubq, struct ublk_io *io,
1419 unsigned int issue_flags)
bd23f6c2 1420{
216c8f5e 1421 bool done;
bd23f6c2 1422
216c8f5e
ML		 1423 if (!(io->flags & UBLK_IO_FLAG_ACTIVE))
1424 return;
bd23f6c2 1425
216c8f5e
ML		 1426 spin_lock(&ubq->cancel_lock);
1427 done = !!(io->flags & UBLK_IO_FLAG_CANCELED);
1428 if (!done)
1429 io->flags |= UBLK_IO_FLAG_CANCELED;
1430 spin_unlock(&ubq->cancel_lock);
1431
1432 if (!done)
1433 io_uring_cmd_done(io->cmd, UBLK_IO_RES_ABORT, 0, issue_flags);
1434}
1435
1436/*
1437 * The ublk char device won't be closed when calling cancel fn, so both
1438 * ublk device and queue are guaranteed to be live
1439 */
1440static void ublk_uring_cmd_cancel_fn(struct io_uring_cmd *cmd,
1441 unsigned int issue_flags)
1442{
1443 struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
1444 struct ublk_queue *ubq = pdu->ubq;
1445 struct task_struct *task;
1446 struct ublk_device *ub;
1447 bool need_schedule;
1448 struct ublk_io *io;
bd23f6c2 1449
216c8f5e 1450 if (WARN_ON_ONCE(!ubq))
bd23f6c2
ML		 1451 return;
1452
216c8f5e
ML		 1453 if (WARN_ON_ONCE(pdu->tag >= ubq->q_depth))
1454 return;
71f28f31 1455
216c8f5e
ML		 1456 task = io_uring_cmd_get_task(cmd);
1457 if (WARN_ON_ONCE(task && task != ubq->ubq_daemon))
1458 return;
1459
1460 ub = ubq->dev;
1461 need_schedule = ublk_abort_requests(ub, ubq);
1462
1463 io = &ubq->ios[pdu->tag];
1464 WARN_ON_ONCE(io->cmd != cmd);
b4e1353f 1465 ublk_cancel_cmd(ubq, io, issue_flags);
216c8f5e
ML		 1466
1467 if (need_schedule) {
1468 if (ublk_can_use_recovery(ub))
1469 schedule_work(&ub->quiesce_work);
1470 else
1471 schedule_work(&ub->stop_work);
1472 }
71f28f31
ML		 1473}
1474
a8ce5f52
ML		 1475static inline bool ublk_queue_ready(struct ublk_queue *ubq)
1476{
1477 return ubq->nr_io_ready == ubq->q_depth;
1478}
1479
71f28f31
ML		 1480static void ublk_cancel_queue(struct ublk_queue *ubq)
1481{
1482 int i;
1483
216c8f5e
ML		 1484 for (i = 0; i < ubq->q_depth; i++)
1485 ublk_cancel_cmd(ubq, &ubq->ios[i], IO_URING_F_UNLOCKED);
71f28f31
ML		 1486}
1487
1488/* Cancel all pending commands, must be called after del_gendisk() returns */
1489static void ublk_cancel_dev(struct ublk_device *ub)
1490{
1491 int i;
1492
1493 for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1494 ublk_cancel_queue(ublk_get_queue(ub, i));
1495}
1496
bbae8d1f
Z		 1497static bool ublk_check_inflight_rq(struct request *rq, void *data)
1498{
1499 bool *idle = data;
1500
1501 if (blk_mq_request_started(rq)) {
1502 *idle = false;
1503 return false;
1504 }
1505 return true;
1506}
1507
1508static void ublk_wait_tagset_rqs_idle(struct ublk_device *ub)
1509{
1510 bool idle;
1511
1512 WARN_ON_ONCE(!blk_queue_quiesced(ub->ub_disk->queue));
1513 while (true) {
1514 idle = true;
1515 blk_mq_tagset_busy_iter(&ub->tag_set,
1516 ublk_check_inflight_rq, &idle);
1517 if (idle)
1518 break;
1519 msleep(UBLK_REQUEUE_DELAY_MS);
1520 }
1521}
1522
1523static void __ublk_quiesce_dev(struct ublk_device *ub)
71f28f31 1524{
bbae8d1f
Z		 1525 pr_devel("%s: quiesce ub: dev_id %d state %s\n",
1526 __func__, ub->dev_info.dev_id,
1527 ub->dev_info.state == UBLK_S_DEV_LIVE ?
1528 "LIVE" : "QUIESCED");
1529 blk_mq_quiesce_queue(ub->ub_disk->queue);
1530 ublk_wait_tagset_rqs_idle(ub);
1531 ub->dev_info.state = UBLK_S_DEV_QUIESCED;
bbae8d1f
Z		 1532}
1533
1534static void ublk_quiesce_work_fn(struct work_struct *work)
1535{
1536 struct ublk_device *ub =
1537 container_of(work, struct ublk_device, quiesce_work);
1538
71f28f31 1539 mutex_lock(&ub->mutex);
6d9e6dfd 1540 if (ub->dev_info.state != UBLK_S_DEV_LIVE)
71f28f31 1541 goto unlock;
bbae8d1f
Z		 1542 __ublk_quiesce_dev(ub);
1543 unlock:
1544 mutex_unlock(&ub->mutex);
85248d67 1545 ublk_cancel_dev(ub);
bbae8d1f 1546}
71f28f31 1547
bbae8d1f
Z		 1548static void ublk_unquiesce_dev(struct ublk_device *ub)
1549{
1550 int i;
1551
1552 pr_devel("%s: unquiesce ub: dev_id %d state %s\n",
1553 __func__, ub->dev_info.dev_id,
1554 ub->dev_info.state == UBLK_S_DEV_LIVE ?
1555 "LIVE" : "QUIESCED");
1556 /* quiesce_work has run. We let requeued rqs be aborted
1557 * before running fallback_wq. "force_abort" must be seen
1558 * after request queue is unqiuesced. Then del_gendisk()
1559 * can move on.
1560 */
1561 for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1562 ublk_get_queue(ub, i)->force_abort = true;
1563
1564 blk_mq_unquiesce_queue(ub->ub_disk->queue);
1565 /* We may have requeued some rqs in ublk_quiesce_queue() */
1566 blk_mq_kick_requeue_list(ub->ub_disk->queue);
1567}
1568
1569static void ublk_stop_dev(struct ublk_device *ub)
1570{
bd23f6c2
ML		 1571 struct gendisk *disk;
1572
bbae8d1f
Z		 1573 mutex_lock(&ub->mutex);
1574 if (ub->dev_info.state == UBLK_S_DEV_DEAD)
1575 goto unlock;
1576 if (ublk_can_use_recovery(ub)) {
1577 if (ub->dev_info.state == UBLK_S_DEV_LIVE)
1578 __ublk_quiesce_dev(ub);
1579 ublk_unquiesce_dev(ub);
1580 }
71f28f31 1581 del_gendisk(ub->ub_disk);
bd23f6c2
ML		 1582
1583 /* Sync with ublk_abort_queue() by holding the lock */
1584 spin_lock(&ub->lock);
1585 disk = ub->ub_disk;
71f28f31
ML		 1586 ub->dev_info.state = UBLK_S_DEV_DEAD;
1587 ub->dev_info.ublksrv_pid = -1;
6d9e6dfd 1588 ub->ub_disk = NULL;
bd23f6c2
ML		 1589 spin_unlock(&ub->lock);
1590 put_disk(disk);
71f28f31
ML		 1591 unlock:
1592 mutex_unlock(&ub->mutex);
85248d67 1593 ublk_cancel_dev(ub);
71f28f31
ML		 1594}
1595
71f28f31
ML		 1596/* device can only be started after all IOs are ready */
1597static void ublk_mark_io_ready(struct ublk_device *ub, struct ublk_queue *ubq)
1598{
1599 mutex_lock(&ub->mutex);
1600 ubq->nr_io_ready++;
1601 if (ublk_queue_ready(ubq)) {
1602 ubq->ubq_daemon = current;
1603 get_task_struct(ubq->ubq_daemon);
1604 ub->nr_queues_ready++;
73a166d9
ML		 1605
1606 if (capable(CAP_SYS_ADMIN))
1607 ub->nr_privileged_daemon++;
71f28f31
ML		 1608 }
1609 if (ub->nr_queues_ready == ub->dev_info.nr_hw_queues)
1610 complete_all(&ub->completion);
1611 mutex_unlock(&ub->mutex);
1612}
1613
c86019ff 1614static void ublk_handle_need_get_data(struct ublk_device *ub, int q_id,
fee32f31 1615 int tag)
c86019ff
Z		 1616{
1617 struct ublk_queue *ubq = ublk_get_queue(ub, q_id);
1618 struct request *req = blk_mq_tag_to_rq(ub->tag_set.tags[q_id], tag);
1619
7d4a9317 1620 ublk_queue_cmd(ubq, req);
c86019ff
Z		 1621}
1622
2d786e66
ML		 1623static inline int ublk_check_cmd_op(u32 cmd_op)
1624{
1625 u32 ioc_type = _IOC_TYPE(cmd_op);
1626
e485bd9e 1627 if (!IS_ENABLED(CONFIG_BLKDEV_UBLK_LEGACY_OPCODES) && ioc_type != 'u')
2d786e66
ML		 1628 return -EOPNOTSUPP;
1629
1630 if (ioc_type != 'u' && ioc_type != 0)
1631 return -EOPNOTSUPP;
1632
1633 return 0;
1634}
1635
f236a214
ML		 1636static inline void ublk_fill_io_cmd(struct ublk_io *io,
1637 struct io_uring_cmd *cmd, unsigned long buf_addr)
1638{
1639 io->cmd = cmd;
1640 io->flags |= UBLK_IO_FLAG_ACTIVE;
1641 io->addr = buf_addr;
1642}
1643
216c8f5e
ML		 1644static inline void ublk_prep_cancel(struct io_uring_cmd *cmd,
1645 unsigned int issue_flags,
1646 struct ublk_queue *ubq, unsigned int tag)
1647{
1648 struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
1649
1650 /*
1651 * Safe to refer to @ubq since ublk_queue won't be died until its
1652 * commands are completed
1653 */
1654 pdu->ubq = ubq;
1655 pdu->tag = tag;
1656 io_uring_cmd_mark_cancelable(cmd, issue_flags);
1657}
1658
8c68ae3b
JA		 1659static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,
1660 unsigned int issue_flags,
03e5cb7b 1661 const struct ublksrv_io_cmd *ub_cmd)
71f28f31 1662{
71f28f31
ML		 1663 struct ublk_device *ub = cmd->file->private_data;
1664 struct ublk_queue *ubq;
1665 struct ublk_io *io;
1666 u32 cmd_op = cmd->cmd_op;
1667 unsigned tag = ub_cmd->tag;
1668 int ret = -EINVAL;
2f1e07dd 1669 struct request *req;
71f28f31
ML		 1670
1671 pr_devel("%s: received: cmd op %d queue %d tag %d result %d\n",
1672 __func__, cmd->cmd_op, ub_cmd->q_id, tag,
1673 ub_cmd->result);
1674
71f28f31
ML		 1675 if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues)
1676 goto out;
1677
1678 ubq = ublk_get_queue(ub, ub_cmd->q_id);
1679 if (!ubq || ub_cmd->q_id != ubq->q_id)
1680 goto out;
1681
1682 if (ubq->ubq_daemon && ubq->ubq_daemon != current)
1683 goto out;
1684
1685 if (tag >= ubq->q_depth)
1686 goto out;
1687
1688 io = &ubq->ios[tag];
1689
1690 /* there is pending io cmd, something must be wrong */
1691 if (io->flags & UBLK_IO_FLAG_ACTIVE) {
1692 ret = -EBUSY;
1693 goto out;
1694 }
1695
c86019ff
Z		 1696 /*
1697 * ensure that the user issues UBLK_IO_NEED_GET_DATA
1698 * iff the driver have set the UBLK_IO_FLAG_NEED_GET_DATA.
1699 */
1700 if ((!!(io->flags & UBLK_IO_FLAG_NEED_GET_DATA))
2d786e66
ML		 1701 ^ (_IOC_NR(cmd_op) == UBLK_IO_NEED_GET_DATA))
1702 goto out;
1703
1704 ret = ublk_check_cmd_op(cmd_op);
1705 if (ret)
c86019ff
Z		 1706 goto out;
1707
7c75661c 1708 ret = -EINVAL;
2d786e66 1709 switch (_IOC_NR(cmd_op)) {
71f28f31
ML		 1710 case UBLK_IO_FETCH_REQ:
1711 /* UBLK_IO_FETCH_REQ is only allowed before queue is setup */
1712 if (ublk_queue_ready(ubq)) {
1713 ret = -EBUSY;
1714 goto out;
1715 }
1716 /*
1717 * The io is being handled by server, so COMMIT_RQ is expected
1718 * instead of FETCH_REQ
1719 */
1720 if (io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)
1721 goto out;
71f28f31 1722
1172d5b8
ML		 1723 if (!ublk_support_user_copy(ubq)) {
1724 /*
1725 * FETCH_RQ has to provide IO buffer if NEED GET
1726 * DATA is not enabled
1727 */
1728 if (!ub_cmd->addr && !ublk_need_get_data(ubq))
1729 goto out;
1a6e88b9
AH		 1730 } else if (ub_cmd->addr) {
1731 /* User copy requires addr to be unset */
1732 ret = -EINVAL;
1733 goto out;
1172d5b8 1734 }
71f28f31 1735
f236a214 1736 ublk_fill_io_cmd(io, cmd, ub_cmd->addr);
71f28f31
ML		 1737 ublk_mark_io_ready(ub, ubq);
1738 break;
1739 case UBLK_IO_COMMIT_AND_FETCH_REQ:
2f1e07dd 1740 req = blk_mq_tag_to_rq(ub->tag_set.tags[ub_cmd->q_id], tag);
1172d5b8 1741
71f28f31
ML		 1742 if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV))
1743 goto out;
1172d5b8
ML		 1744
1745 if (!ublk_support_user_copy(ubq)) {
1746 /*
1747 * COMMIT_AND_FETCH_REQ has to provide IO buffer if
1748 * NEED GET DATA is not enabled or it is Read IO.
1749 */
1750 if (!ub_cmd->addr && (!ublk_need_get_data(ubq) ||
1751 req_op(req) == REQ_OP_READ))
1752 goto out;
29802d7c
AH		 1753 } else if (req_op(req) != REQ_OP_ZONE_APPEND && ub_cmd->addr) {
1754 /*
1755 * User copy requires addr to be unset when command is
1756 * not zone append
1757 */
1a6e88b9
AH		 1758 ret = -EINVAL;
1759 goto out;
1172d5b8 1760 }
1a6e88b9 1761
f236a214 1762 ublk_fill_io_cmd(io, cmd, ub_cmd->addr);
71f28f31
ML		 1763 ublk_commit_completion(ub, ub_cmd);
1764 break;
c86019ff
Z		 1765 case UBLK_IO_NEED_GET_DATA:
1766 if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV))
1767 goto out;
f236a214 1768 ublk_fill_io_cmd(io, cmd, ub_cmd->addr);
fee32f31 1769 ublk_handle_need_get_data(ub, ub_cmd->q_id, ub_cmd->tag);
c86019ff 1770 break;
71f28f31
ML		 1771 default:
1772 goto out;
1773 }
216c8f5e 1774 ublk_prep_cancel(cmd, issue_flags, ubq, tag);
71f28f31
ML		 1775 return -EIOCBQUEUED;
1776
1777 out:
9d2789ac 1778 io_uring_cmd_done(cmd, ret, 0, issue_flags);
71f28f31
ML		 1779 pr_devel("%s: complete: cmd op %d, tag %d ret %x io_flags %x\n",
1780 __func__, cmd_op, tag, ret, io->flags);
1781 return -EIOCBQUEUED;
1782}
1783
62fe99ce
ML		 1784static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub,
1785 struct ublk_queue *ubq, int tag, size_t offset)
1786{
1787 struct request *req;
1788
1789 if (!ublk_need_req_ref(ubq))
1790 return NULL;
1791
1792 req = blk_mq_tag_to_rq(ub->tag_set.tags[ubq->q_id], tag);
1793 if (!req)
1794 return NULL;
1795
1796 if (!ublk_get_req_ref(ubq, req))
1797 return NULL;
1798
1799 if (unlikely(!blk_mq_request_started(req) || req->tag != tag))
1800 goto fail_put;
1801
1802 if (!ublk_rq_has_data(req))
1803 goto fail_put;
1804
1805 if (offset > blk_rq_bytes(req))
1806 goto fail_put;
1807
1808 return req;
1809fail_put:
1810 ublk_put_req_ref(ubq, req);
1811 return NULL;
1812}
1813
3421c7f6
ML		 1814static inline int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd,
1815 unsigned int issue_flags)
8c68ae3b 1816{
8c68ae3b
JA		 1817 /*
1818 * Not necessary for async retry, but let's keep it simple and always
1819 * copy the values to avoid any potential reuse.
1820 */
03e5cb7b
LT		 1821 const struct ublksrv_io_cmd *ub_src = io_uring_sqe_cmd(cmd->sqe);
1822 const struct ublksrv_io_cmd ub_cmd = {
1823 .q_id = READ_ONCE(ub_src->q_id),
1824 .tag = READ_ONCE(ub_src->tag),
1825 .result = READ_ONCE(ub_src->result),
1826 .addr = READ_ONCE(ub_src->addr)
1827 };
8c68ae3b 1828
3421c7f6
ML		 1829 WARN_ON_ONCE(issue_flags & IO_URING_F_UNLOCKED);
1830
8c68ae3b
JA		 1831 return __ublk_ch_uring_cmd(cmd, issue_flags, &ub_cmd);
1832}
1833
3421c7f6
ML		 1834static void ublk_ch_uring_cmd_cb(struct io_uring_cmd *cmd,
1835 unsigned int issue_flags)
1836{
1837 ublk_ch_uring_cmd_local(cmd, issue_flags);
1838}
1839
1840static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
1841{
216c8f5e
ML		 1842 if (unlikely(issue_flags & IO_URING_F_CANCEL)) {
1843 ublk_uring_cmd_cancel_fn(cmd, issue_flags);
1844 return 0;
1845 }
1846
3421c7f6
ML		 1847 /* well-implemented server won't run into unlocked */
1848 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) {
1849 io_uring_cmd_complete_in_task(cmd, ublk_ch_uring_cmd_cb);
1850 return -EIOCBQUEUED;
1851 }
1852
1853 return ublk_ch_uring_cmd_local(cmd, issue_flags);
1854}
1855
62fe99ce
ML		 1856static inline bool ublk_check_ubuf_dir(const struct request *req,
1857 int ubuf_dir)
1858{
1859 /* copy ubuf to request pages */
29802d7c
AH		 1860 if ((req_op(req) == REQ_OP_READ || req_op(req) == REQ_OP_DRV_IN) &&
1861 ubuf_dir == ITER_SOURCE)
62fe99ce
ML		 1862 return true;
1863
1864 /* copy request pages to ubuf */
29802d7c
AH		 1865 if ((req_op(req) == REQ_OP_WRITE ||
1866 req_op(req) == REQ_OP_ZONE_APPEND) &&
1867 ubuf_dir == ITER_DEST)
62fe99ce
ML		 1868 return true;
1869
1870 return false;
1871}
1872
1873static struct request *ublk_check_and_get_req(struct kiocb *iocb,
1874 struct iov_iter *iter, size_t *off, int dir)
1875{
1876 struct ublk_device *ub = iocb->ki_filp->private_data;
1877 struct ublk_queue *ubq;
1878 struct request *req;
1879 size_t buf_off;
1880 u16 tag, q_id;
1881
1882 if (!ub)
1883 return ERR_PTR(-EACCES);
1884
1885 if (!user_backed_iter(iter))
1886 return ERR_PTR(-EACCES);
1887
1888 if (ub->dev_info.state == UBLK_S_DEV_DEAD)
1889 return ERR_PTR(-EACCES);
1890
1891 tag = ublk_pos_to_tag(iocb->ki_pos);
1892 q_id = ublk_pos_to_hwq(iocb->ki_pos);
1893 buf_off = ublk_pos_to_buf_off(iocb->ki_pos);
1894
1895 if (q_id >= ub->dev_info.nr_hw_queues)
1896 return ERR_PTR(-EINVAL);
1897
1898 ubq = ublk_get_queue(ub, q_id);
1899 if (!ubq)
1900 return ERR_PTR(-EINVAL);
1901
1902 if (tag >= ubq->q_depth)
1903 return ERR_PTR(-EINVAL);
1904
1905 req = __ublk_check_and_get_req(ub, ubq, tag, buf_off);
1906 if (!req)
1907 return ERR_PTR(-EINVAL);
1908
1909 if (!req->mq_hctx || !req->mq_hctx->driver_data)
1910 goto fail;
1911
1912 if (!ublk_check_ubuf_dir(req, dir))
1913 goto fail;
1914
1915 *off = buf_off;
1916 return req;
1917fail:
1918 ublk_put_req_ref(ubq, req);
1919 return ERR_PTR(-EACCES);
1920}
1921
1922static ssize_t ublk_ch_read_iter(struct kiocb *iocb, struct iov_iter *to)
1923{
1924 struct ublk_queue *ubq;
1925 struct request *req;
1926 size_t buf_off;
1927 size_t ret;
1928
1929 req = ublk_check_and_get_req(iocb, to, &buf_off, ITER_DEST);
1930 if (IS_ERR(req))
1931 return PTR_ERR(req);
1932
1933 ret = ublk_copy_user_pages(req, buf_off, to, ITER_DEST);
1934 ubq = req->mq_hctx->driver_data;
1935 ublk_put_req_ref(ubq, req);
1936
1937 return ret;
1938}
1939
1940static ssize_t ublk_ch_write_iter(struct kiocb *iocb, struct iov_iter *from)
1941{
1942 struct ublk_queue *ubq;
1943 struct request *req;
1944 size_t buf_off;
1945 size_t ret;
1946
1947 req = ublk_check_and_get_req(iocb, from, &buf_off, ITER_SOURCE);
1948 if (IS_ERR(req))
1949 return PTR_ERR(req);
1950
1951 ret = ublk_copy_user_pages(req, buf_off, from, ITER_SOURCE);
1952 ubq = req->mq_hctx->driver_data;
1953 ublk_put_req_ref(ubq, req);
1954
1955 return ret;
1956}
1957
71f28f31
ML		 1958static const struct file_operations ublk_ch_fops = {
1959 .owner = THIS_MODULE,
1960 .open = ublk_ch_open,
1961 .release = ublk_ch_release,
1962 .llseek = no_llseek,
62fe99ce
ML		 1963 .read_iter = ublk_ch_read_iter,
1964 .write_iter = ublk_ch_write_iter,
71f28f31
ML		 1965 .uring_cmd = ublk_ch_uring_cmd,
1966 .mmap = ublk_ch_mmap,
1967};
1968
1969static void ublk_deinit_queue(struct ublk_device *ub, int q_id)
1970{
1971 int size = ublk_queue_cmd_buf_size(ub, q_id);
1972 struct ublk_queue *ubq = ublk_get_queue(ub, q_id);
1973
1974 if (ubq->ubq_daemon)
1975 put_task_struct(ubq->ubq_daemon);
1976 if (ubq->io_cmd_buf)
1977 free_pages((unsigned long)ubq->io_cmd_buf, get_order(size));
1978}
1979
1980static int ublk_init_queue(struct ublk_device *ub, int q_id)
1981{
1982 struct ublk_queue *ubq = ublk_get_queue(ub, q_id);
1983 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO;
1984 void *ptr;
1985 int size;
1986
85248d67 1987 spin_lock_init(&ubq->cancel_lock);
6d8c5afc 1988 ubq->flags = ub->dev_info.flags;
71f28f31
ML		 1989 ubq->q_id = q_id;
1990 ubq->q_depth = ub->dev_info.queue_depth;
1991 size = ublk_queue_cmd_buf_size(ub, q_id);
1992
1993 ptr = (void *) __get_free_pages(gfp_flags, get_order(size));
1994 if (!ptr)
1995 return -ENOMEM;
1996
1997 ubq->io_cmd_buf = ptr;
1998 ubq->dev = ub;
1999 return 0;
2000}
2001
2002static void ublk_deinit_queues(struct ublk_device *ub)
2003{
2004 int nr_queues = ub->dev_info.nr_hw_queues;
2005 int i;
2006
2007 if (!ub->__queues)
2008 return;
2009
2010 for (i = 0; i < nr_queues; i++)
2011 ublk_deinit_queue(ub, i);
2012 kfree(ub->__queues);
2013}
2014
2015static int ublk_init_queues(struct ublk_device *ub)
2016{
2017 int nr_queues = ub->dev_info.nr_hw_queues;
2018 int depth = ub->dev_info.queue_depth;
2019 int ubq_size = sizeof(struct ublk_queue) + depth * sizeof(struct ublk_io);
2020 int i, ret = -ENOMEM;
2021
2022 ub->queue_size = ubq_size;
2023 ub->__queues = kcalloc(nr_queues, ubq_size, GFP_KERNEL);
2024 if (!ub->__queues)
2025 return ret;
2026
2027 for (i = 0; i < nr_queues; i++) {
2028 if (ublk_init_queue(ub, i))
2029 goto fail;
2030 }
2031
2032 init_completion(&ub->completion);
2033 return 0;
2034
2035 fail:
2036 ublk_deinit_queues(ub);
2037 return ret;
2038}
2039
fa9482e0 2040static int ublk_alloc_dev_number(struct ublk_device *ub, int idx)
71f28f31
ML		 2041{
2042 int i = idx;
2043 int err;
2044
2045 spin_lock(&ublk_idr_lock);
2046 /* allocate id, if @id >= 0, we're requesting that specific id */
2047 if (i >= 0) {
2048 err = idr_alloc(&ublk_index_idr, ub, i, i + 1, GFP_NOWAIT);
2049 if (err == -ENOSPC)
2050 err = -EEXIST;
2051 } else {
56974c07
MC		 2052 err = idr_alloc(&ublk_index_idr, ub, 0, UBLK_MAX_UBLKS,
2053 GFP_NOWAIT);
71f28f31
ML		 2054 }
2055 spin_unlock(&ublk_idr_lock);
2056
2057 if (err >= 0)
2058 ub->ub_number = err;
2059
2060 return err;
2061}
2062
fa9482e0 2063static void ublk_free_dev_number(struct ublk_device *ub)
71f28f31
ML		 2064{
2065 spin_lock(&ublk_idr_lock);
2066 idr_remove(&ublk_index_idr, ub->ub_number);
2067 wake_up_all(&ublk_idr_wq);
2068 spin_unlock(&ublk_idr_lock);
71f28f31
ML		 2069}
2070
2071static void ublk_cdev_rel(struct device *dev)
2072{
2073 struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev);
2074
71f28f31 2075 blk_mq_free_tag_set(&ub->tag_set);
71f28f31 2076 ublk_deinit_queues(ub);
fa9482e0
CH		 2077 ublk_free_dev_number(ub);
2078 mutex_destroy(&ub->mutex);
2079 kfree(ub);
71f28f31
ML		 2080}
2081
2082static int ublk_add_chdev(struct ublk_device *ub)
2083{
2084 struct device *dev = &ub->cdev_dev;
2085 int minor = ub->ub_number;
2086 int ret;
2087
2088 dev->parent = ublk_misc.this_device;
2089 dev->devt = MKDEV(MAJOR(ublk_chr_devt), minor);
2eefd399 2090 dev->class = &ublk_chr_class;
71f28f31
ML		 2091 dev->release = ublk_cdev_rel;
2092 device_initialize(dev);
2093
2094 ret = dev_set_name(dev, "ublkc%d", minor);
2095 if (ret)
2096 goto fail;
2097
2098 cdev_init(&ub->cdev, &ublk_ch_fops);
2099 ret = cdev_device_add(&ub->cdev, dev);
2100 if (ret)
2101 goto fail;
403ebc87
ML		 2102
2103 ublks_added++;
71f28f31
ML		 2104 return 0;
2105 fail:
2106 put_device(dev);
2107 return ret;
2108}
2109
2110static void ublk_stop_work_fn(struct work_struct *work)
2111{
2112 struct ublk_device *ub =
2113 container_of(work, struct ublk_device, stop_work);
2114
2115 ublk_stop_dev(ub);
2116}
2117
4bf9cbf3 2118/* align max io buffer size with PAGE_SIZE */
6d9e6dfd 2119static void ublk_align_max_io_size(struct ublk_device *ub)
71f28f31 2120{
4bf9cbf3 2121 unsigned int max_io_bytes = ub->dev_info.max_io_buf_bytes;
71f28f31 2122
4bf9cbf3
ML		 2123 ub->dev_info.max_io_buf_bytes =
2124 round_down(max_io_bytes, PAGE_SIZE);
71f28f31
ML		 2125}
2126
fa9482e0 2127static int ublk_add_tag_set(struct ublk_device *ub)
71f28f31 2128{
71f28f31
ML		 2129 ub->tag_set.ops = &ublk_mq_ops;
2130 ub->tag_set.nr_hw_queues = ub->dev_info.nr_hw_queues;
2131 ub->tag_set.queue_depth = ub->dev_info.queue_depth;
2132 ub->tag_set.numa_node = NUMA_NO_NODE;
0edb3696 2133 ub->tag_set.cmd_size = sizeof(struct ublk_rq_data);
71f28f31
ML		 2134 ub->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
2135 ub->tag_set.driver_data = ub;
fa9482e0 2136 return blk_mq_alloc_tag_set(&ub->tag_set);
71f28f31
ML		 2137}
2138
2139static void ublk_remove(struct ublk_device *ub)
2140{
34d8f2be
CH		 2141 ublk_stop_dev(ub);
2142 cancel_work_sync(&ub->stop_work);
bbae8d1f 2143 cancel_work_sync(&ub->quiesce_work);
71f28f31
ML		 2144 cdev_device_del(&ub->cdev, &ub->cdev_dev);
2145 put_device(&ub->cdev_dev);
403ebc87 2146 ublks_added--;
71f28f31
ML		 2147}
2148
2149static struct ublk_device *ublk_get_device_from_id(int idx)
2150{
2151 struct ublk_device *ub = NULL;
2152
2153 if (idx < 0)
2154 return NULL;
2155
2156 spin_lock(&ublk_idr_lock);
2157 ub = idr_find(&ublk_index_idr, idx);
2158 if (ub)
2159 ub = ublk_get_device(ub);
2160 spin_unlock(&ublk_idr_lock);
2161
2162 return ub;
2163}
2164
bfbcef03 2165static int ublk_ctrl_start_dev(struct ublk_device *ub, struct io_uring_cmd *cmd)
71f28f31 2166{
fd9b8547 2167 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
494ea040 2168 const struct ublk_param_basic *p = &ub->params.basic;
71f28f31 2169 int ublksrv_pid = (int)header->data[0];
494ea040
CH		 2170 struct queue_limits lim = {
2171 .logical_block_size = 1 << p->logical_bs_shift,
2172 .physical_block_size = 1 << p->physical_bs_shift,
2173 .io_min = 1 << p->io_min_shift,
2174 .io_opt = 1 << p->io_opt_shift,
2175 .max_hw_sectors = p->max_sectors,
2176 .chunk_sectors = p->chunk_sectors,
2177 .virt_boundary_mask = p->virt_boundary_mask,
2178
2179 };
6d9e6dfd 2180 struct gendisk *disk;
34d8f2be 2181 int ret = -EINVAL;
71f28f31
ML		 2182
2183 if (ublksrv_pid <= 0)
34d8f2be 2184 return -EINVAL;
494ea040
CH		 2185 if (!(ub->params.types & UBLK_PARAM_TYPE_BASIC))
2186 return -EINVAL;
2187
2188 if (ub->params.types & UBLK_PARAM_TYPE_DISCARD) {
2189 const struct ublk_param_discard *pd = &ub->params.discard;
2190
2191 lim.discard_alignment = pd->discard_alignment;
2192 lim.discard_granularity = pd->discard_granularity;
2193 lim.max_hw_discard_sectors = pd->max_discard_sectors;
2194 lim.max_write_zeroes_sectors = pd->max_write_zeroes_sectors;
2195 lim.max_discard_segments = pd->max_discard_segments;
2196 }
2197
2198 if (ub->params.types & UBLK_PARAM_TYPE_ZONED) {
2199 const struct ublk_param_zoned *p = &ub->params.zoned;
2200
2201 if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED))
2202 return -EOPNOTSUPP;
2203
2204 lim.zoned = true;
2205 lim.max_active_zones = p->max_active_zones;
2206 lim.max_open_zones = p->max_open_zones;
2207 lim.max_zone_append_sectors = p->max_zone_append_sectors;
2208 }
34d8f2be 2209
53e7d08f
ML		 2210 if (wait_for_completion_interruptible(&ub->completion) != 0)
2211 return -EINTR;
71f28f31 2212
71f28f31 2213 mutex_lock(&ub->mutex);
6d9e6dfd
CH		 2214 if (ub->dev_info.state == UBLK_S_DEV_LIVE ||
2215 test_bit(UB_STATE_USED, &ub->state)) {
71f28f31 2216 ret = -EEXIST;
34d8f2be 2217 goto out_unlock;
71f28f31 2218 }
71f28f31 2219
494ea040 2220 disk = blk_mq_alloc_disk(&ub->tag_set, &lim, NULL);
6d9e6dfd
CH		 2221 if (IS_ERR(disk)) {
2222 ret = PTR_ERR(disk);
2223 goto out_unlock;
34d8f2be 2224 }
6d9e6dfd
CH		 2225 sprintf(disk->disk_name, "ublkb%d", ub->ub_number);
2226 disk->fops = &ub_fops;
2227 disk->private_data = ub;
2228
34d8f2be 2229 ub->dev_info.ublksrv_pid = ublksrv_pid;
6d9e6dfd 2230 ub->ub_disk = disk;
0aa73170 2231
494ea040 2232 ublk_apply_params(ub);
0aa73170 2233
73a166d9
ML		 2234 /* don't probe partitions if any one ubq daemon is un-trusted */
2235 if (ub->nr_privileged_daemon != ub->nr_queues_ready)
2236 set_bit(GD_SUPPRESS_PART_SCAN, &disk->state);
2237
6d9e6dfd 2238 get_device(&ub->cdev_dev);
4985e7b2 2239 ub->dev_info.state = UBLK_S_DEV_LIVE;
29802d7c
AH		 2240
2241 if (ublk_dev_is_zoned(ub)) {
2242 ret = ublk_revalidate_disk_zones(ub);
2243 if (ret)
2244 goto out_put_cdev;
2245 }
2246
6d9e6dfd 2247 ret = add_disk(disk);
29802d7c
AH		 2248 if (ret)
2249 goto out_put_cdev;
2250
2251 set_bit(UB_STATE_USED, &ub->state);
2252
2253out_put_cdev:
6d9e6dfd 2254 if (ret) {
4985e7b2 2255 ub->dev_info.state = UBLK_S_DEV_DEAD;
93d71ec8 2256 ublk_put_device(ub);
6d9e6dfd 2257 }
0aa73170
ML		 2258 if (ret)
2259 put_disk(disk);
34d8f2be
CH		 2260out_unlock:
2261 mutex_unlock(&ub->mutex);
71f28f31
ML		 2262 return ret;
2263}
2264
bfbcef03
ML		 2265static int ublk_ctrl_get_queue_affinity(struct ublk_device *ub,
2266 struct io_uring_cmd *cmd)
71f28f31 2267{
fd9b8547 2268 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
71f28f31 2269 void __user *argp = (void __user *)(unsigned long)header->addr;
c50061f0 2270 cpumask_var_t cpumask;
71f28f31
ML		 2271 unsigned long queue;
2272 unsigned int retlen;
c50061f0 2273 unsigned int i;
bfbcef03
ML		 2274 int ret;
2275
34d8f2be
CH		 2276 if (header->len * BITS_PER_BYTE < nr_cpu_ids)
2277 return -EINVAL;
2278 if (header->len & (sizeof(unsigned long)-1))
2279 return -EINVAL;
2280 if (!header->addr)
2281 return -EINVAL;
71f28f31 2282
71f28f31
ML		 2283 queue = header->data[0];
2284 if (queue >= ub->dev_info.nr_hw_queues)
bfbcef03 2285 return -EINVAL;
71f28f31 2286
c50061f0 2287 if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
bfbcef03 2288 return -ENOMEM;
c50061f0
CH		 2289
2290 for_each_possible_cpu(i) {
2291 if (ub->tag_set.map[HCTX_TYPE_DEFAULT].mq_map[i] == queue)
2292 cpumask_set_cpu(i, cpumask);
71f28f31 2293 }
c50061f0
CH		 2294
2295 ret = -EFAULT;
2296 retlen = min_t(unsigned short, header->len, cpumask_size());
2297 if (copy_to_user(argp, cpumask, retlen))
2298 goto out_free_cpumask;
2299 if (retlen != header->len &&
2300 clear_user(argp + retlen, header->len - retlen))
2301 goto out_free_cpumask;
2302
71f28f31 2303 ret = 0;
c50061f0
CH		 2304out_free_cpumask:
2305 free_cpumask_var(cpumask);
71f28f31
ML		 2306 return ret;
2307}
2308
34d8f2be 2309static inline void ublk_dump_dev_info(struct ublksrv_ctrl_dev_info *info)
71f28f31 2310{
34d8f2be 2311 pr_devel("%s: dev id %d flags %llx\n", __func__,
6d8c5afc 2312 info->dev_id, info->flags);
4bf9cbf3
ML		 2313 pr_devel("\t nr_hw_queues %d queue_depth %d\n",
2314 info->nr_hw_queues, info->queue_depth);
34d8f2be
CH		 2315}
2316
2317static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd)
2318{
fd9b8547 2319 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
34d8f2be
CH		 2320 void __user *argp = (void __user *)(unsigned long)header->addr;
2321 struct ublksrv_ctrl_dev_info info;
71f28f31 2322 struct ublk_device *ub;
34d8f2be
CH		 2323 int ret = -EINVAL;
2324
2325 if (header->len < sizeof(info) || !header->addr)
2326 return -EINVAL;
2327 if (header->queue_id != (u16)-1) {
2328 pr_warn("%s: queue_id is wrong %x\n",
2329 __func__, header->queue_id);
2330 return -EINVAL;
2331 }
4093cb5a 2332
34d8f2be
CH		 2333 if (copy_from_user(&info, argp, sizeof(info)))
2334 return -EFAULT;
4093cb5a
ML		 2335
2336 if (capable(CAP_SYS_ADMIN))
2337 info.flags &= ~UBLK_F_UNPRIVILEGED_DEV;
2338 else if (!(info.flags & UBLK_F_UNPRIVILEGED_DEV))
2339 return -EPERM;
2340
c0b79b0f
ML		 2341 /*
2342 * unprivileged device can't be trusted, but RECOVERY and
2343 * RECOVERY_REISSUE still may hang error handling, so can't
2344 * support recovery features for unprivileged ublk now
2345 *
2346 * TODO: provide forward progress for RECOVERY handler, so that
2347 * unprivileged device can benefit from it
2348 */
2349 if (info.flags & UBLK_F_UNPRIVILEGED_DEV)
2350 info.flags &= ~(UBLK_F_USER_RECOVERY_REISSUE |
2351 UBLK_F_USER_RECOVERY);
2352
4093cb5a 2353 /* the created device is always owned by current user */
48a90519 2354 ublk_store_owner_uid_gid(&info.owner_uid, &info.owner_gid);
4093cb5a 2355
34d8f2be
CH		 2356 if (header->dev_id != info.dev_id) {
2357 pr_warn("%s: dev id not match %u %u\n",
2358 __func__, header->dev_id, info.dev_id);
2359 return -EINVAL;
2360 }
71f28f31 2361
56974c07
MC		 2362 if (header->dev_id != U32_MAX && header->dev_id >= UBLK_MAX_UBLKS) {
2363 pr_warn("%s: dev id is too large. Max supported is %d\n",
2364 __func__, UBLK_MAX_UBLKS - 1);
2365 return -EINVAL;
2366 }
2367
4093cb5a
ML		 2368 ublk_dump_dev_info(&info);
2369
71f28f31
ML		 2370 ret = mutex_lock_killable(&ublk_ctl_mutex);
2371 if (ret)
2372 return ret;
2373
403ebc87
ML		 2374 ret = -EACCES;
2375 if (ublks_added >= ublks_max)
2376 goto out_unlock;
2377
cfee7e4d
CH		 2378 ret = -ENOMEM;
2379 ub = kzalloc(sizeof(*ub), GFP_KERNEL);
2380 if (!ub)
2381 goto out_unlock;
fa9482e0 2382 mutex_init(&ub->mutex);
28dde8c0 2383 spin_lock_init(&ub->lock);
bbae8d1f 2384 INIT_WORK(&ub->quiesce_work, ublk_quiesce_work_fn);
fa9482e0 2385 INIT_WORK(&ub->stop_work, ublk_stop_work_fn);
cfee7e4d 2386
fa9482e0
CH		 2387 ret = ublk_alloc_dev_number(ub, header->dev_id);
2388 if (ret < 0)
2389 goto out_free_ub;
71f28f31 2390
34d8f2be 2391 memcpy(&ub->dev_info, &info, sizeof(info));
71f28f31 2392
34d8f2be
CH		 2393 /* update device id */
2394 ub->dev_info.dev_id = ub->ub_number;
2395
6d8c5afc
ML		 2396 /*
2397 * 64bit flags will be copied back to userspace as feature
2398 * negotiation result, so have to clear flags which driver
2399 * doesn't support yet, then userspace can get correct flags
2400 * (features) to handle.
2401 */
2402 ub->dev_info.flags &= UBLK_F_ALL;
2403
29dc5d06
ML		 2404 ub->dev_info.flags |= UBLK_F_CMD_IOCTL_ENCODE |
2405 UBLK_F_URING_CMD_COMP_IN_TASK;
224e858f 2406
1172d5b8 2407 /* GET_DATA isn't needed any more with USER_COPY */
9d4ed6d4 2408 if (ublk_dev_is_user_copy(ub))
1172d5b8 2409 ub->dev_info.flags &= ~UBLK_F_NEED_GET_DATA;
2d786e66 2410
29802d7c
AH		 2411 /* Zoned storage support requires user copy feature */
2412 if (ublk_dev_is_zoned(ub) &&
2413 (!IS_ENABLED(CONFIG_BLK_DEV_ZONED) || !ublk_dev_is_user_copy(ub))) {
2414 ret = -EINVAL;
2415 goto out_free_dev_number;
2416 }
2417
fa9482e0 2418 /* We are not ready to support zero copy */
6d8c5afc 2419 ub->dev_info.flags &= ~UBLK_F_SUPPORT_ZERO_COPY;
fa9482e0 2420
fa9482e0
CH		 2421 ub->dev_info.nr_hw_queues = min_t(unsigned int,
2422 ub->dev_info.nr_hw_queues, nr_cpu_ids);
2423 ublk_align_max_io_size(ub);
2424
2425 ret = ublk_init_queues(ub);
34d8f2be 2426 if (ret)
fa9482e0 2427 goto out_free_dev_number;
34d8f2be 2428
fa9482e0
CH		 2429 ret = ublk_add_tag_set(ub);
2430 if (ret)
2431 goto out_deinit_queues;
2432
2433 ret = -EFAULT;
2434 if (copy_to_user(argp, &ub->dev_info, sizeof(info)))
2435 goto out_free_tag_set;
2436
2437 /*
2438 * Add the char dev so that ublksrv daemon can be setup.
2439 * ublk_add_chdev() will cleanup everything if it fails.
2440 */
2441 ret = ublk_add_chdev(ub);
2442 goto out_unlock;
2443
2444out_free_tag_set:
2445 blk_mq_free_tag_set(&ub->tag_set);
2446out_deinit_queues:
2447 ublk_deinit_queues(ub);
2448out_free_dev_number:
2449 ublk_free_dev_number(ub);
2450out_free_ub:
2451 mutex_destroy(&ub->mutex);
2452 kfree(ub);
34d8f2be 2453out_unlock:
71f28f31 2454 mutex_unlock(&ublk_ctl_mutex);
71f28f31
ML		 2455 return ret;
2456}
2457
2458static inline bool ublk_idr_freed(int id)
2459{
2460 void *ptr;
2461
2462 spin_lock(&ublk_idr_lock);
2463 ptr = idr_find(&ublk_index_idr, id);
2464 spin_unlock(&ublk_idr_lock);
2465
2466 return ptr == NULL;
2467}
2468
bfbcef03 2469static int ublk_ctrl_del_dev(struct ublk_device **p_ub)
71f28f31 2470{
bfbcef03
ML		 2471 struct ublk_device *ub = *p_ub;
2472 int idx = ub->ub_number;
71f28f31
ML		 2473 int ret;
2474
2475 ret = mutex_lock_killable(&ublk_ctl_mutex);
2476 if (ret)
2477 return ret;
2478
0abe39de 2479 if (!test_bit(UB_STATE_DELETED, &ub->state)) {
71f28f31 2480 ublk_remove(ub);
0abe39de 2481 set_bit(UB_STATE_DELETED, &ub->state);
71f28f31
ML		 2482 }
2483
bfbcef03
ML		 2484 /* Mark the reference as consumed */
2485 *p_ub = NULL;
2486 ublk_put_device(ub);
0abe39de 2487 mutex_unlock(&ublk_ctl_mutex);
71f28f31
ML		 2488
2489 /*
2490 * Wait until the idr is removed, then it can be reused after
2491 * DEL_DEV command is returned.
0abe39de
ML		 2492 *
2493 * If we returns because of user interrupt, future delete command
2494 * may come:
2495 *
2496 * - the device number isn't freed, this device won't or needn't
2497 * be deleted again, since UB_STATE_DELETED is set, and device
2498 * will be released after the last reference is dropped
2499 *
2500 * - the device number is freed already, we will not find this
2501 * device via ublk_get_device_from_id()
71f28f31 2502 */
3e9dce80
ML		 2503 if (wait_event_interruptible(ublk_idr_wq, ublk_idr_freed(idx)))
2504 return -EINTR;
0abe39de 2505 return 0;
71f28f31
ML		 2506}
2507
71f28f31
ML		 2508static inline void ublk_ctrl_cmd_dump(struct io_uring_cmd *cmd)
2509{
fd9b8547 2510 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
71f28f31
ML		 2511
2512 pr_devel("%s: cmd_op %x, dev id %d qid %d data %llx buf %llx len %u\n",
2513 __func__, cmd->cmd_op, header->dev_id, header->queue_id,
2514 header->data[0], header->addr, header->len);
2515}
2516
bfbcef03 2517static int ublk_ctrl_stop_dev(struct ublk_device *ub)
71f28f31 2518{
34d8f2be
CH		 2519 ublk_stop_dev(ub);
2520 cancel_work_sync(&ub->stop_work);
bbae8d1f 2521 cancel_work_sync(&ub->quiesce_work);
71f28f31
ML		 2522
2523 return 0;
2524}
2525
bfbcef03
ML		 2526static int ublk_ctrl_get_dev_info(struct ublk_device *ub,
2527 struct io_uring_cmd *cmd)
71f28f31 2528{
fd9b8547 2529 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
71f28f31 2530 void __user *argp = (void __user *)(unsigned long)header->addr;
34d8f2be
CH		 2531
2532 if (header->len < sizeof(struct ublksrv_ctrl_dev_info) || !header->addr)
2533 return -EINVAL;
2534
34d8f2be 2535 if (copy_to_user(argp, &ub->dev_info, sizeof(ub->dev_info)))
bfbcef03 2536 return -EFAULT;
34d8f2be 2537
bfbcef03 2538 return 0;
34d8f2be
CH		 2539}
2540
abb864d3
ML		 2541/* TYPE_DEVT is readonly, so fill it up before returning to userspace */
2542static void ublk_ctrl_fill_params_devt(struct ublk_device *ub)
2543{
2544 ub->params.devt.char_major = MAJOR(ub->cdev_dev.devt);
2545 ub->params.devt.char_minor = MINOR(ub->cdev_dev.devt);
2546
2547 if (ub->ub_disk) {
2548 ub->params.devt.disk_major = MAJOR(disk_devt(ub->ub_disk));
2549 ub->params.devt.disk_minor = MINOR(disk_devt(ub->ub_disk));
2550 } else {
2551 ub->params.devt.disk_major = 0;
2552 ub->params.devt.disk_minor = 0;
2553 }
2554 ub->params.types |= UBLK_PARAM_TYPE_DEVT;
34d8f2be
CH		 2555}
2556
bfbcef03
ML		 2557static int ublk_ctrl_get_params(struct ublk_device *ub,
2558 struct io_uring_cmd *cmd)
0aa73170 2559{
fd9b8547 2560 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
0aa73170
ML		 2561 void __user *argp = (void __user *)(unsigned long)header->addr;
2562 struct ublk_params_header ph;
0aa73170
ML		 2563 int ret;
2564
2565 if (header->len <= sizeof(ph) || !header->addr)
2566 return -EINVAL;
2567
2568 if (copy_from_user(&ph, argp, sizeof(ph)))
2569 return -EFAULT;
2570
2571 if (ph.len > header->len || !ph.len)
2572 return -EINVAL;
2573
2574 if (ph.len > sizeof(struct ublk_params))
2575 ph.len = sizeof(struct ublk_params);
2576
0aa73170 2577 mutex_lock(&ub->mutex);
abb864d3 2578 ublk_ctrl_fill_params_devt(ub);
0aa73170
ML		 2579 if (copy_to_user(argp, &ub->params, ph.len))
2580 ret = -EFAULT;
2581 else
2582 ret = 0;
2583 mutex_unlock(&ub->mutex);
2584
0aa73170
ML		 2585 return ret;
2586}
2587
bfbcef03
ML		 2588static int ublk_ctrl_set_params(struct ublk_device *ub,
2589 struct io_uring_cmd *cmd)
0aa73170 2590{
fd9b8547 2591 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
0aa73170
ML		 2592 void __user *argp = (void __user *)(unsigned long)header->addr;
2593 struct ublk_params_header ph;
0aa73170
ML		 2594 int ret = -EFAULT;
2595
2596 if (header->len <= sizeof(ph) || !header->addr)
2597 return -EINVAL;
2598
2599 if (copy_from_user(&ph, argp, sizeof(ph)))
2600 return -EFAULT;
2601
2602 if (ph.len > header->len || !ph.len || !ph.types)
2603 return -EINVAL;
2604
2605 if (ph.len > sizeof(struct ublk_params))
2606 ph.len = sizeof(struct ublk_params);
2607
0aa73170
ML		 2608 /* parameters can only be changed when device isn't live */
2609 mutex_lock(&ub->mutex);
2610 if (ub->dev_info.state == UBLK_S_DEV_LIVE) {
2611 ret = -EACCES;
2612 } else if (copy_from_user(&ub->params, argp, ph.len)) {
2613 ret = -EFAULT;
2614 } else {
2615 /* clear all we don't support yet */
2616 ub->params.types &= UBLK_PARAM_TYPE_ALL;
2617 ret = ublk_validate_params(ub);
1d166527
ML		 2618 if (ret)
2619 ub->params.types = 0;
0aa73170
ML		 2620 }
2621 mutex_unlock(&ub->mutex);
0aa73170
ML		 2622
2623 return ret;
2624}
2625
c732a852
Z		 2626static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq)
2627{
2628 int i;
2629
2630 WARN_ON_ONCE(!(ubq->ubq_daemon && ubq_daemon_is_dying(ubq)));
85248d67 2631
c732a852 2632 /* All old ioucmds have to be completed */
85248d67 2633 ubq->nr_io_ready = 0;
c732a852
Z		 2634 /* old daemon is PF_EXITING, put it now */
2635 put_task_struct(ubq->ubq_daemon);
2636 /* We have to reset it to NULL, otherwise ub won't accept new FETCH_REQ */
2637 ubq->ubq_daemon = NULL;
c0b79b0f 2638 ubq->timeout = false;
216c8f5e 2639 ubq->canceling = false;
c732a852
Z		 2640
2641 for (i = 0; i < ubq->q_depth; i++) {
2642 struct ublk_io *io = &ubq->ios[i];
2643
2644 /* forget everything now and be ready for new FETCH_REQ */
2645 io->flags = 0;
2646 io->cmd = NULL;
2647 io->addr = 0;
2648 }
2649}
2650
bfbcef03
ML		 2651static int ublk_ctrl_start_recovery(struct ublk_device *ub,
2652 struct io_uring_cmd *cmd)
c732a852 2653{
fd9b8547 2654 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
c732a852
Z		 2655 int ret = -EINVAL;
2656 int i;
2657
c732a852
Z		 2658 mutex_lock(&ub->mutex);
2659 if (!ublk_can_use_recovery(ub))
2660 goto out_unlock;
2661 /*
2662 * START_RECOVERY is only allowd after:
2663 *
2664 * (1) UB_STATE_OPEN is not set, which means the dying process is exited
2665 * and related io_uring ctx is freed so file struct of /dev/ublkcX is
2666 * released.
2667 *
2668 * (2) UBLK_S_DEV_QUIESCED is set, which means the quiesce_work:
2669 * (a)has quiesced request queue
2670 * (b)has requeued every inflight rqs whose io_flags is ACTIVE
2671 * (c)has requeued/aborted every inflight rqs whose io_flags is NOT ACTIVE
2672 * (d)has completed/camceled all ioucmds owned by ther dying process
2673 */
2674 if (test_bit(UB_STATE_OPEN, &ub->state) ||
2675 ub->dev_info.state != UBLK_S_DEV_QUIESCED) {
2676 ret = -EBUSY;
2677 goto out_unlock;
2678 }
2679 pr_devel("%s: start recovery for dev id %d.\n", __func__, header->dev_id);
2680 for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
2681 ublk_queue_reinit(ub, ublk_get_queue(ub, i));
2682 /* set to NULL, otherwise new ubq_daemon cannot mmap the io_cmd_buf */
2683 ub->mm = NULL;
2684 ub->nr_queues_ready = 0;
73a166d9 2685 ub->nr_privileged_daemon = 0;
c732a852
Z		 2686 init_completion(&ub->completion);
2687 ret = 0;
2688 out_unlock:
2689 mutex_unlock(&ub->mutex);
c732a852
Z		 2690 return ret;
2691}
2692
bfbcef03
ML		 2693static int ublk_ctrl_end_recovery(struct ublk_device *ub,
2694 struct io_uring_cmd *cmd)
c732a852 2695{
fd9b8547 2696 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
c732a852 2697 int ublksrv_pid = (int)header->data[0];
c732a852
Z		 2698 int ret = -EINVAL;
2699
c732a852
Z		 2700 pr_devel("%s: Waiting for new ubq_daemons(nr: %d) are ready, dev id %d...\n",
2701 __func__, ub->dev_info.nr_hw_queues, header->dev_id);
2702 /* wait until new ubq_daemon sending all FETCH_REQ */
0c0cbd4e
ML		 2703 if (wait_for_completion_interruptible(&ub->completion))
2704 return -EINTR;
2705
c732a852
Z		 2706 pr_devel("%s: All new ubq_daemons(nr: %d) are ready, dev id %d\n",
2707 __func__, ub->dev_info.nr_hw_queues, header->dev_id);
2708
2709 mutex_lock(&ub->mutex);
2710 if (!ublk_can_use_recovery(ub))
2711 goto out_unlock;
2712
2713 if (ub->dev_info.state != UBLK_S_DEV_QUIESCED) {
2714 ret = -EBUSY;
2715 goto out_unlock;
2716 }
2717 ub->dev_info.ublksrv_pid = ublksrv_pid;
2718 pr_devel("%s: new ublksrv_pid %d, dev id %d\n",
2719 __func__, ublksrv_pid, header->dev_id);
2720 blk_mq_unquiesce_queue(ub->ub_disk->queue);
2721 pr_devel("%s: queue unquiesced, dev id %d.\n",
2722 __func__, header->dev_id);
2723 blk_mq_kick_requeue_list(ub->ub_disk->queue);
2724 ub->dev_info.state = UBLK_S_DEV_LIVE;
c732a852
Z		 2725 ret = 0;
2726 out_unlock:
2727 mutex_unlock(&ub->mutex);
c732a852
Z		 2728 return ret;
2729}
2730
b5bbc52f
ML		 2731static int ublk_ctrl_get_features(struct io_uring_cmd *cmd)
2732{
2733 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
2734 void __user *argp = (void __user *)(unsigned long)header->addr;
2735 u64 features = UBLK_F_ALL & ~UBLK_F_SUPPORT_ZERO_COPY;
2736
2737 if (header->len != UBLK_FEATURES_LEN || !header->addr)
2738 return -EINVAL;
2739
2740 if (copy_to_user(argp, &features, UBLK_FEATURES_LEN))
2741 return -EFAULT;
2742
2743 return 0;
2744}
2745
4093cb5a
ML		 2746/*
2747 * All control commands are sent via /dev/ublk-control, so we have to check
2748 * the destination device's permission
2749 */
2750static int ublk_char_dev_permission(struct ublk_device *ub,
2751 const char *dev_path, int mask)
2752{
2753 int err;
2754 struct path path;
2755 struct kstat stat;
2756
2757 err = kern_path(dev_path, LOOKUP_FOLLOW, &path);
2758 if (err)
2759 return err;
2760
2761 err = vfs_getattr(&path, &stat, STATX_TYPE, AT_STATX_SYNC_AS_STAT);
2762 if (err)
2763 goto exit;
2764
2765 err = -EPERM;
2766 if (stat.rdev != ub->cdev_dev.devt || !S_ISCHR(stat.mode))
2767 goto exit;
2768
5b0ed596 2769 err = inode_permission(&nop_mnt_idmap,
4093cb5a
ML		 2770 d_backing_inode(path.dentry), mask);
2771exit:
2772 path_put(&path);
2773 return err;
2774}
2775
2776static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub,
2777 struct io_uring_cmd *cmd)
2778{
fd9b8547 2779 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)io_uring_sqe_cmd(cmd->sqe);
4093cb5a
ML		 2780 bool unprivileged = ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV;
2781 void __user *argp = (void __user *)(unsigned long)header->addr;
2782 char *dev_path = NULL;
2783 int ret = 0;
2784 int mask;
2785
2786 if (!unprivileged) {
2787 if (!capable(CAP_SYS_ADMIN))
2788 return -EPERM;
2789 /*
2790 * The new added command of UBLK_CMD_GET_DEV_INFO2 includes
2791 * char_dev_path in payload too, since userspace may not
2792 * know if the specified device is created as unprivileged
2793 * mode.
2794 */
2d786e66 2795 if (_IOC_NR(cmd->cmd_op) != UBLK_CMD_GET_DEV_INFO2)
4093cb5a
ML		 2796 return 0;
2797 }
2798
2799 /*
2800 * User has to provide the char device path for unprivileged ublk
2801 *
2802 * header->addr always points to the dev path buffer, and
2803 * header->dev_path_len records length of dev path buffer.
2804 */
2805 if (!header->dev_path_len || header->dev_path_len > PATH_MAX)
2806 return -EINVAL;
2807
2808 if (header->len < header->dev_path_len)
2809 return -EINVAL;
2810
66a6a5d0
RJ		 2811 dev_path = memdup_user_nul(argp, header->dev_path_len);
2812 if (IS_ERR(dev_path))
2813 return PTR_ERR(dev_path);
4093cb5a
ML		 2814
2815 ret = -EINVAL;
2d786e66 2816 switch (_IOC_NR(cmd->cmd_op)) {
4093cb5a
ML		 2817 case UBLK_CMD_GET_DEV_INFO:
2818 case UBLK_CMD_GET_DEV_INFO2:
2819 case UBLK_CMD_GET_QUEUE_AFFINITY:
2820 case UBLK_CMD_GET_PARAMS:
b5bbc52f 2821 case (_IOC_NR(UBLK_U_CMD_GET_FEATURES)):
4093cb5a
ML		 2822 mask = MAY_READ;
2823 break;
2824 case UBLK_CMD_START_DEV:
2825 case UBLK_CMD_STOP_DEV:
2826 case UBLK_CMD_ADD_DEV:
2827 case UBLK_CMD_DEL_DEV:
2828 case UBLK_CMD_SET_PARAMS:
2829 case UBLK_CMD_START_USER_RECOVERY:
2830 case UBLK_CMD_END_USER_RECOVERY:
2831 mask = MAY_READ | MAY_WRITE;
2832 break;
2833 default:
2834 goto exit;
2835 }
2836
2837 ret = ublk_char_dev_permission(ub, dev_path, mask);
2838 if (!ret) {
2839 header->len -= header->dev_path_len;
2840 header->addr += header->dev_path_len;
2841 }
2842 pr_devel("%s: dev id %d cmd_op %x uid %d gid %d path %s ret %d\n",
2843 __func__, ub->ub_number, cmd->cmd_op,
2844 ub->dev_info.owner_uid, ub->dev_info.owner_gid,
2845 dev_path, ret);
2846exit:
2847 kfree(dev_path);
c732a852
Z		 2848 return ret;
2849}
2850
34d8f2be
CH		 2851static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,
2852 unsigned int issue_flags)
2853{
fd9b8547 2854 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
bfbcef03 2855 struct ublk_device *ub = NULL;
2d786e66 2856 u32 cmd_op = cmd->cmd_op;
71f28f31
ML		 2857 int ret = -EINVAL;
2858
fa8e442e
ML		 2859 if (issue_flags & IO_URING_F_NONBLOCK)
2860 return -EAGAIN;
2861
71f28f31
ML		 2862 ublk_ctrl_cmd_dump(cmd);
2863
2864 if (!(issue_flags & IO_URING_F_SQE128))
2865 goto out;
2866
2d786e66
ML		 2867 ret = ublk_check_cmd_op(cmd_op);
2868 if (ret)
2869 goto out;
2870
b5bbc52f
ML		 2871 if (cmd_op == UBLK_U_CMD_GET_FEATURES) {
2872 ret = ublk_ctrl_get_features(cmd);
2873 goto out;
2874 }
2875
2d786e66 2876 if (_IOC_NR(cmd_op) != UBLK_CMD_ADD_DEV) {
bfbcef03
ML		 2877 ret = -ENODEV;
2878 ub = ublk_get_device_from_id(header->dev_id);
2879 if (!ub)
2880 goto out;
4093cb5a
ML		 2881
2882 ret = ublk_ctrl_uring_cmd_permission(ub, cmd);
2d786e66
ML		 2883 if (ret)
2884 goto put_dev;
bfbcef03
ML		 2885 }
2886
2d786e66 2887 switch (_IOC_NR(cmd_op)) {
71f28f31 2888 case UBLK_CMD_START_DEV:
bfbcef03 2889 ret = ublk_ctrl_start_dev(ub, cmd);
71f28f31
ML		 2890 break;
2891 case UBLK_CMD_STOP_DEV:
bfbcef03 2892 ret = ublk_ctrl_stop_dev(ub);
71f28f31
ML		 2893 break;
2894 case UBLK_CMD_GET_DEV_INFO:
4093cb5a 2895 case UBLK_CMD_GET_DEV_INFO2:
bfbcef03 2896 ret = ublk_ctrl_get_dev_info(ub, cmd);
71f28f31
ML		 2897 break;
2898 case UBLK_CMD_ADD_DEV:
34d8f2be 2899 ret = ublk_ctrl_add_dev(cmd);
71f28f31
ML		 2900 break;
2901 case UBLK_CMD_DEL_DEV:
bfbcef03 2902 ret = ublk_ctrl_del_dev(&ub);
71f28f31
ML		 2903 break;
2904 case UBLK_CMD_GET_QUEUE_AFFINITY:
bfbcef03 2905 ret = ublk_ctrl_get_queue_affinity(ub, cmd);
71f28f31 2906 break;
0aa73170 2907 case UBLK_CMD_GET_PARAMS:
bfbcef03 2908 ret = ublk_ctrl_get_params(ub, cmd);
0aa73170
ML		 2909 break;
2910 case UBLK_CMD_SET_PARAMS:
bfbcef03 2911 ret = ublk_ctrl_set_params(ub, cmd);
0aa73170 2912 break;
c732a852 2913 case UBLK_CMD_START_USER_RECOVERY:
bfbcef03 2914 ret = ublk_ctrl_start_recovery(ub, cmd);
c732a852
Z		 2915 break;
2916 case UBLK_CMD_END_USER_RECOVERY:
bfbcef03 2917 ret = ublk_ctrl_end_recovery(ub, cmd);
c732a852 2918 break;
71f28f31 2919 default:
bfbcef03 2920 ret = -ENOTSUPP;
71f28f31 2921 break;
6b1439d2 2922 }
4093cb5a
ML		 2923
2924 put_dev:
bfbcef03
ML		 2925 if (ub)
2926 ublk_put_device(ub);
71f28f31 2927 out:
9d2789ac 2928 io_uring_cmd_done(cmd, ret, 0, issue_flags);
71f28f31
ML		 2929 pr_devel("%s: cmd done ret %d cmd_op %x, dev id %d qid %d\n",
2930 __func__, ret, cmd->cmd_op, header->dev_id, header->queue_id);
2931 return -EIOCBQUEUED;
2932}
2933
2934static const struct file_operations ublk_ctl_fops = {
2935 .open = nonseekable_open,
2936 .uring_cmd = ublk_ctrl_uring_cmd,
2937 .owner = THIS_MODULE,
2938 .llseek = noop_llseek,
2939};
2940
2941static struct miscdevice ublk_misc = {
2942 .minor = MISC_DYNAMIC_MINOR,
2943 .name = "ublk-control",
2944 .fops = &ublk_ctl_fops,
2945};
2946
2947static int __init ublk_init(void)
2948{
2949 int ret;
2950
62fe99ce
ML		 2951 BUILD_BUG_ON((u64)UBLKSRV_IO_BUF_OFFSET +
2952 UBLKSRV_IO_BUF_TOTAL_SIZE < UBLKSRV_IO_BUF_OFFSET);
2953
71f28f31
ML		 2954 init_waitqueue_head(&ublk_idr_wq);
2955
2956 ret = misc_register(&ublk_misc);
2957 if (ret)
2958 return ret;
2959
2960 ret = alloc_chrdev_region(&ublk_chr_devt, 0, UBLK_MINORS, "ublk-char");
2961 if (ret)
2962 goto unregister_mis;
2963
2eefd399
IO		 2964 ret = class_register(&ublk_chr_class);
2965 if (ret)
71f28f31 2966 goto free_chrdev_region;
2eefd399 2967
71f28f31
ML		 2968 return 0;
2969
2970free_chrdev_region:
2971 unregister_chrdev_region(ublk_chr_devt, UBLK_MINORS);
2972unregister_mis:
2973 misc_deregister(&ublk_misc);
2974 return ret;
2975}
2976
2977static void __exit ublk_exit(void)
2978{
2979 struct ublk_device *ub;
2980 int id;
2981
71f28f31
ML		 2982 idr_for_each_entry(&ublk_index_idr, ub, id)
2983 ublk_remove(ub);
2984
2eefd399 2985 class_unregister(&ublk_chr_class);
8e4ff684
ML		 2986 misc_deregister(&ublk_misc);
2987
71f28f31
ML		 2988 idr_destroy(&ublk_index_idr);
2989 unregister_chrdev_region(ublk_chr_devt, UBLK_MINORS);
2990}
2991
2992module_init(ublk_init);
2993module_exit(ublk_exit);
2994
6eba24ae
MC		 2995static int ublk_set_max_ublks(const char *buf, const struct kernel_param *kp)
2996{
2997 return param_set_uint_minmax(buf, kp, 0, UBLK_MAX_UBLKS);
2998}
2999
3000static int ublk_get_max_ublks(char *buf, const struct kernel_param *kp)
3001{
3002 return sysfs_emit(buf, "%u\n", ublks_max);
3003}
3004
3005static const struct kernel_param_ops ublk_max_ublks_ops = {
3006 .set = ublk_set_max_ublks,
3007 .get = ublk_get_max_ublks,
3008};
3009
3010module_param_cb(ublks_max, &ublk_max_ublks_ops, &ublks_max, 0644);
403ebc87
ML		 3011MODULE_PARM_DESC(ublks_max, "max number of ublk devices allowed to add(default: 64)");
3012
71f28f31
ML		 3013MODULE_AUTHOR("Ming Lei <ming.lei@redhat.com>");
3014MODULE_LICENSE("GPL");
Linux 6.x block layer and io_uring tree(s)