projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ASoC: Merge up v6.6-rc7
[linux-block.git] / drivers / block / rbd.c
CommitLineData
e2a58ee5 1
602adf40
YS		 2/*
3 rbd.c -- Export ceph rados objects as a Linux block device
4
5
6 based on drivers/block/osdblk.c:
7
8 Copyright 2009 Red Hat, Inc.
9
10 This program is free software; you can redistribute it and/or modify
11 it under the terms of the GNU General Public License as published by
12 the Free Software Foundation.
13
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with this program; see the file COPYING. If not, write to
21 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
22
23
24
dfc5606d 25 For usage instructions, please refer to:
602adf40 26
dfc5606d 27 Documentation/ABI/testing/sysfs-bus-rbd
602adf40
YS		 28
29 */
30
31#include <linux/ceph/libceph.h>
32#include <linux/ceph/osd_client.h>
33#include <linux/ceph/mon_client.h>
ed95b21a 34#include <linux/ceph/cls_lock_client.h>
43df3d35 35#include <linux/ceph/striper.h>
602adf40 36#include <linux/ceph/decode.h>
82995cc6 37#include <linux/fs_parser.h>
30d1cff8 38#include <linux/bsearch.h>
602adf40
YS		 39
40#include <linux/kernel.h>
41#include <linux/device.h>
42#include <linux/module.h>
7ad18afa 43#include <linux/blk-mq.h>
602adf40
YS		 44#include <linux/fs.h>
45#include <linux/blkdev.h>
1c2a9dfe 46#include <linux/slab.h>
f8a22fc2 47#include <linux/idr.h>
bc1ecc65 48#include <linux/workqueue.h>
602adf40
YS		 49
50#include "rbd_types.h"
51
aafb230e
AE		 52#define RBD_DEBUG /* Activate rbd_assert() calls */
53
a2acd00e
AE		 54/*
55 * Increment the given counter and return its updated value.
56 * If the counter is already 0 it will not be incremented.
57 * If the counter is already at its maximum value returns
58 * -EINVAL without updating it.
59 */
60static int atomic_inc_return_safe(atomic_t *v)
61{
62 unsigned int counter;
63
bfc18e38 64 counter = (unsigned int)atomic_fetch_add_unless(v, 1, 0);
a2acd00e
AE		 65 if (counter <= (unsigned int)INT_MAX)
66 return (int)counter;
67
68 atomic_dec(v);
69
70 return -EINVAL;
71}
72
73/* Decrement the counter. Return the resulting value, or -EINVAL */
74static int atomic_dec_return_safe(atomic_t *v)
75{
76 int counter;
77
78 counter = atomic_dec_return(v);
79 if (counter >= 0)
80 return counter;
81
82 atomic_inc(v);
83
84 return -EINVAL;
85}
86
f0f8cef5 87#define RBD_DRV_NAME "rbd"
602adf40 88
7e513d43
ID		 89#define RBD_MINORS_PER_MAJOR 256
90#define RBD_SINGLE_MAJOR_PART_SHIFT 4
602adf40 91
6d69bb53
ID		 92#define RBD_MAX_PARENT_CHAIN_LEN 16
93
d4b125e9
AE		 94#define RBD_SNAP_DEV_NAME_PREFIX "snap_"
95#define RBD_MAX_SNAP_NAME_LEN \
96 (NAME_MAX - (sizeof (RBD_SNAP_DEV_NAME_PREFIX) - 1))
97
35d489f9 98#define RBD_MAX_SNAP_COUNT 510 /* allows max snapc to fit in 4KB */
602adf40
YS		 99
100#define RBD_SNAP_HEAD_NAME "-"
101
9682fc6d
AE		 102#define BAD_SNAP_INDEX U32_MAX /* invalid index into snap array */
103
9e15b77d
AE		 104/* This allows a single page to hold an image name sent by OSD */
105#define RBD_IMAGE_NAME_LEN_MAX (PAGE_SIZE - sizeof (__le32) - 1)
1e130199 106#define RBD_IMAGE_ID_LEN_MAX 64
9e15b77d 107
1e130199 108#define RBD_OBJ_PREFIX_LEN_MAX 64
589d30e0 109
ed95b21a 110#define RBD_NOTIFY_TIMEOUT 5 /* seconds */
99d16943
ID		 111#define RBD_RETRY_DELAY msecs_to_jiffies(1000)
112
d889140c
AE		 113/* Feature bits */
114
8767b293
ID		 115#define RBD_FEATURE_LAYERING (1ULL<<0)
116#define RBD_FEATURE_STRIPINGV2 (1ULL<<1)
117#define RBD_FEATURE_EXCLUSIVE_LOCK (1ULL<<2)
22e8bd51
ID		 118#define RBD_FEATURE_OBJECT_MAP (1ULL<<3)
119#define RBD_FEATURE_FAST_DIFF (1ULL<<4)
b9f6d447 120#define RBD_FEATURE_DEEP_FLATTEN (1ULL<<5)
8767b293 121#define RBD_FEATURE_DATA_POOL (1ULL<<7)
e573427a 122#define RBD_FEATURE_OPERATIONS (1ULL<<8)
8767b293 123
ed95b21a
ID		 124#define RBD_FEATURES_ALL (RBD_FEATURE_LAYERING | \
125 RBD_FEATURE_STRIPINGV2 | \
7e97332e 126 RBD_FEATURE_EXCLUSIVE_LOCK | \
22e8bd51
ID		 127 RBD_FEATURE_OBJECT_MAP | \
128 RBD_FEATURE_FAST_DIFF | \
b9f6d447 129 RBD_FEATURE_DEEP_FLATTEN | \
e573427a
ID		 130 RBD_FEATURE_DATA_POOL | \
131 RBD_FEATURE_OPERATIONS)
d889140c
AE		 132
133/* Features supported by this (client software) implementation. */
134
770eba6e 135#define RBD_FEATURES_SUPPORTED (RBD_FEATURES_ALL)
d889140c 136
81a89793
AE		 137/*
138 * An RBD device name will be "rbd#", where the "rbd" comes from
139 * RBD_DRV_NAME above, and # is a unique integer identifier.
81a89793 140 */
602adf40
YS		 141#define DEV_NAME_LEN 32
142
143/*
144 * block device image metadata (in-memory version)
145 */
146struct rbd_image_header {
f35a4dee 147 /* These six fields never change for a given rbd image */
849b4260 148 char *object_prefix;
602adf40 149 __u8 obj_order;
f35a4dee
AE		 150 u64 stripe_unit;
151 u64 stripe_count;
7e97332e 152 s64 data_pool_id;
f35a4dee 153 u64 features; /* Might be changeable someday? */
602adf40 154
f84344f3
AE		 155 /* The remaining fields need to be updated occasionally */
156 u64 image_size;
157 struct ceph_snap_context *snapc;
f35a4dee
AE		 158 char *snap_names; /* format 1 only */
159 u64 *snap_sizes; /* format 1 only */
59c2be1e
YS		 160};
161
0d7dbfce
AE		 162/*
163 * An rbd image specification.
164 *
165 * The tuple (pool_id, image_id, snap_id) is sufficient to uniquely
c66c6e0c
AE		 166 * identify an image. Each rbd_dev structure includes a pointer to
167 * an rbd_spec structure that encapsulates this identity.
168 *
169 * Each of the id's in an rbd_spec has an associated name. For a
170 * user-mapped image, the names are supplied and the id's associated
171 * with them are looked up. For a layered image, a parent image is
172 * defined by the tuple, and the names are looked up.
173 *
174 * An rbd_dev structure contains a parent_spec pointer which is
175 * non-null if the image it represents is a child in a layered
176 * image. This pointer will refer to the rbd_spec structure used
177 * by the parent rbd_dev for its own identity (i.e., the structure
178 * is shared between the parent and child).
179 *
180 * Since these structures are populated once, during the discovery
181 * phase of image construction, they are effectively immutable so
182 * we make no effort to synchronize access to them.
183 *
184 * Note that code herein does not assume the image name is known (it
185 * could be a null pointer).
0d7dbfce
AE		 186 */
187struct rbd_spec {
188 u64 pool_id;
ecb4dc22 189 const char *pool_name;
b26c047b 190 const char *pool_ns; /* NULL if default, never "" */
0d7dbfce 191
ecb4dc22
AE		 192 const char *image_id;
193 const char *image_name;
0d7dbfce
AE		 194
195 u64 snap_id;
ecb4dc22 196 const char *snap_name;
0d7dbfce
AE		 197
198 struct kref kref;
199};
200
602adf40 201/*
f0f8cef5 202 * an instance of the client. multiple devices may share an rbd client.
602adf40
YS		 203 */
204struct rbd_client {
205 struct ceph_client *client;
206 struct kref kref;
207 struct list_head node;
208};
209
0192ce2e
ID		 210struct pending_result {
211 int result; /* first nonzero result */
212 int num_pending;
213};
214
bf0d5f50 215struct rbd_img_request;
bf0d5f50 216
9969ebc5 217enum obj_request_type {
a1fbb5e7 218 OBJ_REQUEST_NODATA = 1,
5359a17d 219 OBJ_REQUEST_BIO, /* pointer into provided bio (list) */
7e07efb1 220 OBJ_REQUEST_BVECS, /* pointer into provided bio_vec array */
afb97888 221 OBJ_REQUEST_OWN_BVECS, /* private bio_vec array, doesn't own pages */
9969ebc5 222};
bf0d5f50 223
6d2940c8 224enum obj_operation_type {
a1fbb5e7 225 OBJ_OP_READ = 1,
6d2940c8 226 OBJ_OP_WRITE,
90e98c52 227 OBJ_OP_DISCARD,
6484cbe9 228 OBJ_OP_ZEROOUT,
6d2940c8
GZ		 229};
230
0ad5d953
ID		 231#define RBD_OBJ_FLAG_DELETION (1U << 0)
232#define RBD_OBJ_FLAG_COPYUP_ENABLED (1U << 1)
793333a3 233#define RBD_OBJ_FLAG_COPYUP_ZEROS (1U << 2)
22e8bd51
ID		 234#define RBD_OBJ_FLAG_MAY_EXIST (1U << 3)
235#define RBD_OBJ_FLAG_NOOP_FOR_NONEXISTENT (1U << 4)
0ad5d953 236
a9b67e69 237enum rbd_obj_read_state {
85b5e6d1
ID		 238 RBD_OBJ_READ_START = 1,
239 RBD_OBJ_READ_OBJECT,
a9b67e69
ID		 240 RBD_OBJ_READ_PARENT,
241};
242
3da691bf
ID		 243/*
244 * Writes go through the following state machine to deal with
245 * layering:
246 *
89a59c1c
ID		 247 * . . . . . RBD_OBJ_WRITE_GUARD. . . . . . . . . . . . . .
248 * . | .
249 * . v .
250 * . RBD_OBJ_WRITE_READ_FROM_PARENT. . . .
251 * . | . .
252 * . v v (deep-copyup .
253 * (image . RBD_OBJ_WRITE_COPYUP_EMPTY_SNAPC . not needed) .
254 * flattened) v | . .
255 * . v . .
256 * . . . .RBD_OBJ_WRITE_COPYUP_OPS. . . . . (copyup .
257 * | not needed) v
258 * v .
259 * done . . . . . . . . . . . . . . . . . .
260 * ^
261 * |
262 * RBD_OBJ_WRITE_FLAT
3da691bf
ID		 263 *
264 * Writes start in RBD_OBJ_WRITE_GUARD or _FLAT, depending on whether
89a59c1c
ID		 265 * assert_exists guard is needed or not (in some cases it's not needed
266 * even if there is a parent).
3da691bf
ID		 267 */
268enum rbd_obj_write_state {
85b5e6d1 269 RBD_OBJ_WRITE_START = 1,
22e8bd51 270 RBD_OBJ_WRITE_PRE_OBJECT_MAP,
85b5e6d1 271 RBD_OBJ_WRITE_OBJECT,
793333a3
ID		 272 __RBD_OBJ_WRITE_COPYUP,
273 RBD_OBJ_WRITE_COPYUP,
22e8bd51 274 RBD_OBJ_WRITE_POST_OBJECT_MAP,
793333a3
ID		 275};
276
277enum rbd_obj_copyup_state {
278 RBD_OBJ_COPYUP_START = 1,
279 RBD_OBJ_COPYUP_READ_PARENT,
22e8bd51
ID		 280 __RBD_OBJ_COPYUP_OBJECT_MAPS,
281 RBD_OBJ_COPYUP_OBJECT_MAPS,
793333a3
ID		 282 __RBD_OBJ_COPYUP_WRITE_OBJECT,
283 RBD_OBJ_COPYUP_WRITE_OBJECT,
926f9b3f
AE		 284};
285
bf0d5f50 286struct rbd_obj_request {
43df3d35 287 struct ceph_object_extent ex;
0ad5d953 288 unsigned int flags; /* RBD_OBJ_FLAG_* */
c5b5ef6c 289 union {
a9b67e69 290 enum rbd_obj_read_state read_state; /* for reads */
3da691bf 291 enum rbd_obj_write_state write_state; /* for writes */
c5b5ef6c 292 };
bf0d5f50 293
51c3509e 294 struct rbd_img_request *img_request;
86bd7998
ID		 295 struct ceph_file_extent *img_extents;
296 u32 num_img_extents;
bf0d5f50 297
788e2df3 298 union {
5359a17d 299 struct ceph_bio_iter bio_pos;
788e2df3 300 struct {
7e07efb1
ID		 301 struct ceph_bvec_iter bvec_pos;
302 u32 bvec_count;
afb97888 303 u32 bvec_idx;
788e2df3
AE		 304 };
305 };
793333a3
ID		 306
307 enum rbd_obj_copyup_state copyup_state;
7e07efb1
ID		 308 struct bio_vec *copyup_bvecs;
309 u32 copyup_bvec_count;
bf0d5f50 310
bcbab1db 311 struct list_head osd_reqs; /* w/ r_private_item */
bf0d5f50 312
85b5e6d1 313 struct mutex state_mutex;
793333a3 314 struct pending_result pending;
bf0d5f50
AE		 315 struct kref kref;
316};
317
0c425248 318enum img_req_flags {
9849e986 319 IMG_REQ_CHILD, /* initiator: block = 0, child image = 1 */
d0b2e944 320 IMG_REQ_LAYERED, /* ENOENT handling: normal = 0, layered = 1 */
0c425248
AE		 321};
322
0192ce2e
ID		 323enum rbd_img_state {
324 RBD_IMG_START = 1,
637cd060 325 RBD_IMG_EXCLUSIVE_LOCK,
0192ce2e
ID		 326 __RBD_IMG_OBJECT_REQUESTS,
327 RBD_IMG_OBJECT_REQUESTS,
328};
329
bf0d5f50 330struct rbd_img_request {
bf0d5f50 331 struct rbd_device *rbd_dev;
9bb0248d 332 enum obj_operation_type op_type;
ecc633ca 333 enum obj_request_type data_type;
0c425248 334 unsigned long flags;
0192ce2e 335 enum rbd_img_state state;
bf0d5f50 336 union {
9849e986 337 u64 snap_id; /* for reads */
bf0d5f50 338 struct ceph_snap_context *snapc; /* for writes */
9849e986 339 };
59e542c8 340 struct rbd_obj_request *obj_request; /* obj req initiator */
bf0d5f50 341
e1fddc8f 342 struct list_head lock_item;
43df3d35 343 struct list_head object_extents; /* obj_req.ex structs */
bf0d5f50 344
0192ce2e
ID		 345 struct mutex state_mutex;
346 struct pending_result pending;
347 struct work_struct work;
348 int work_result;
bf0d5f50
AE		 349};
350
351#define for_each_obj_request(ireq, oreq) \
43df3d35 352 list_for_each_entry(oreq, &(ireq)->object_extents, ex.oe_item)
bf0d5f50 353#define for_each_obj_request_safe(ireq, oreq, n) \
43df3d35 354 list_for_each_entry_safe(oreq, n, &(ireq)->object_extents, ex.oe_item)
bf0d5f50 355
99d16943
ID		 356enum rbd_watch_state {
357 RBD_WATCH_STATE_UNREGISTERED,
358 RBD_WATCH_STATE_REGISTERED,
359 RBD_WATCH_STATE_ERROR,
360};
361
ed95b21a
ID		 362enum rbd_lock_state {
363 RBD_LOCK_STATE_UNLOCKED,
364 RBD_LOCK_STATE_LOCKED,
365 RBD_LOCK_STATE_RELEASING,
366};
367
368/* WatchNotify::ClientId */
369struct rbd_client_id {
370 u64 gid;
371 u64 handle;
372};
373
f84344f3 374struct rbd_mapping {
99c1f08f 375 u64 size;
f84344f3
AE		 376};
377
602adf40
YS		 378/*
379 * a single device
380 */
381struct rbd_device {
de71a297 382 int dev_id; /* blkdev unique id */
602adf40
YS		 383
384 int major; /* blkdev assigned major */
dd82fff1 385 int minor;
602adf40 386 struct gendisk *disk; /* blkdev's gendisk and rq */
602adf40 387
a30b71b9 388 u32 image_format; /* Either 1 or 2 */
602adf40
YS		 389 struct rbd_client *rbd_client;
390
391 char name[DEV_NAME_LEN]; /* blkdev name, e.g. rbd3 */
392
b82d167b 393 spinlock_t lock; /* queue, flags, open_count */
602adf40
YS		 394
395 struct rbd_image_header header;
b82d167b 396 unsigned long flags; /* possibly lock protected */
0d7dbfce 397 struct rbd_spec *spec;
d147543d 398 struct rbd_options *opts;
0d6d1e9c 399 char *config_info; /* add{,_single_major} string */
602adf40 400
c41d13a3 401 struct ceph_object_id header_oid;
922dab61 402 struct ceph_object_locator header_oloc;
971f839a 403
1643dfa4 404 struct ceph_file_layout layout; /* used for all rbd requests */
0903e875 405
99d16943
ID		 406 struct mutex watch_mutex;
407 enum rbd_watch_state watch_state;
922dab61 408 struct ceph_osd_linger_request *watch_handle;
99d16943
ID		 409 u64 watch_cookie;
410 struct delayed_work watch_dwork;
59c2be1e 411
ed95b21a
ID		 412 struct rw_semaphore lock_rwsem;
413 enum rbd_lock_state lock_state;
cbbfb0ff 414 char lock_cookie[32];
ed95b21a
ID		 415 struct rbd_client_id owner_cid;
416 struct work_struct acquired_lock_work;
417 struct work_struct released_lock_work;
418 struct delayed_work lock_dwork;
419 struct work_struct unlock_work;
e1fddc8f 420 spinlock_t lock_lists_lock;
637cd060 421 struct list_head acquiring_list;
e1fddc8f 422 struct list_head running_list;
637cd060
ID		 423 struct completion acquire_wait;
424 int acquire_err;
e1fddc8f 425 struct completion releasing_wait;
ed95b21a 426
22e8bd51
ID		 427 spinlock_t object_map_lock;
428 u8 *object_map;
429 u64 object_map_size; /* in objects */
430 u64 object_map_flags;
ed95b21a 431
1643dfa4 432 struct workqueue_struct *task_wq;
59c2be1e 433
86b00e0d
AE		 434 struct rbd_spec *parent_spec;
435 u64 parent_overlap;
a2acd00e 436 atomic_t parent_ref;
2f82ee54 437 struct rbd_device *parent;
86b00e0d 438
7ad18afa
CH		 439 /* Block layer tags. */
440 struct blk_mq_tag_set tag_set;
441
c666601a
JD		 442 /* protects updating the header */
443 struct rw_semaphore header_rwsem;
f84344f3
AE		 444
445 struct rbd_mapping mapping;
602adf40
YS		 446
447 struct list_head node;
dfc5606d 448
dfc5606d
YS		 449 /* sysfs related */
450 struct device dev;
b82d167b 451 unsigned long open_count; /* protected by lock */
dfc5606d
YS		 452};
453
b82d167b 454/*
87c0fded
ID		 455 * Flag bits for rbd_dev->flags:
456 * - REMOVING (which is coupled with rbd_dev->open_count) is protected
457 * by rbd_dev->lock
b82d167b 458 */
6d292906 459enum rbd_dev_flags {
686238b7 460 RBD_DEV_FLAG_EXISTS, /* rbd_dev_device_setup() ran */
b82d167b 461 RBD_DEV_FLAG_REMOVING, /* this mapping is being removed */
39258aa2 462 RBD_DEV_FLAG_READONLY, /* -o ro or snapshot */
6d292906
AE		 463};
464
cfbf6377 465static DEFINE_MUTEX(client_mutex); /* Serialize client creation */
e124a82f 466
602adf40 467static LIST_HEAD(rbd_dev_list); /* devices */
e124a82f
AE		 468static DEFINE_SPINLOCK(rbd_dev_list_lock);
469
432b8587
AE		 470static LIST_HEAD(rbd_client_list); /* clients */
471static DEFINE_SPINLOCK(rbd_client_list_lock);
602adf40 472
78c2a44a
AE		 473/* Slab caches for frequently-allocated structures */
474
1c2a9dfe 475static struct kmem_cache *rbd_img_request_cache;
868311b1 476static struct kmem_cache *rbd_obj_request_cache;
1c2a9dfe 477
9b60e70b 478static int rbd_major;
f8a22fc2
ID		 479static DEFINE_IDA(rbd_dev_id_ida);
480
f5ee37bd
ID		 481static struct workqueue_struct *rbd_wq;
482
89a59c1c
ID		 483static struct ceph_snap_context rbd_empty_snapc = {
484 .nref = REFCOUNT_INIT(1),
485};
486
9b60e70b 487/*
3cfa3b16 488 * single-major requires >= 0.75 version of userspace rbd utility.
9b60e70b 489 */
3cfa3b16 490static bool single_major = true;
5657a819 491module_param(single_major, bool, 0444);
3cfa3b16 492MODULE_PARM_DESC(single_major, "Use a single major number for all rbd devices (default: true)");
9b60e70b 493
75cff725
GKH		 494static ssize_t add_store(const struct bus_type *bus, const char *buf, size_t count);
495static ssize_t remove_store(const struct bus_type *bus, const char *buf,
7e9586ba 496 size_t count);
75cff725 497static ssize_t add_single_major_store(const struct bus_type *bus, const char *buf,
7e9586ba 498 size_t count);
75cff725 499static ssize_t remove_single_major_store(const struct bus_type *bus, const char *buf,
7e9586ba 500 size_t count);
6d69bb53 501static int rbd_dev_image_probe(struct rbd_device *rbd_dev, int depth);
f0f8cef5 502
9b60e70b
ID		 503static int rbd_dev_id_to_minor(int dev_id)
504{
7e513d43 505 return dev_id << RBD_SINGLE_MAJOR_PART_SHIFT;
9b60e70b
ID		 506}
507
508static int minor_to_rbd_dev_id(int minor)
509{
7e513d43 510 return minor >> RBD_SINGLE_MAJOR_PART_SHIFT;
9b60e70b
ID		 511}
512
39258aa2
ID		 513static bool rbd_is_ro(struct rbd_device *rbd_dev)
514{
515 return test_bit(RBD_DEV_FLAG_READONLY, &rbd_dev->flags);
516}
517
f3c0e459
ID		 518static bool rbd_is_snap(struct rbd_device *rbd_dev)
519{
520 return rbd_dev->spec->snap_id != CEPH_NOSNAP;
521}
522
ed95b21a
ID		 523static bool __rbd_is_lock_owner(struct rbd_device *rbd_dev)
524{
637cd060
ID		 525 lockdep_assert_held(&rbd_dev->lock_rwsem);
526
ed95b21a
ID		 527 return rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED ||
528 rbd_dev->lock_state == RBD_LOCK_STATE_RELEASING;
529}
530
531static bool rbd_is_lock_owner(struct rbd_device *rbd_dev)
532{
533 bool is_lock_owner;
534
535 down_read(&rbd_dev->lock_rwsem);
536 is_lock_owner = __rbd_is_lock_owner(rbd_dev);
537 up_read(&rbd_dev->lock_rwsem);
538 return is_lock_owner;
539}
540
75cff725 541static ssize_t supported_features_show(const struct bus_type *bus, char *buf)
8767b293
ID		 542{
543 return sprintf(buf, "0x%llx\n", RBD_FEATURES_SUPPORTED);
544}
545
7e9586ba
GKH		 546static BUS_ATTR_WO(add);
547static BUS_ATTR_WO(remove);
548static BUS_ATTR_WO(add_single_major);
549static BUS_ATTR_WO(remove_single_major);
550static BUS_ATTR_RO(supported_features);
b15a21dd
GKH		 551
552static struct attribute *rbd_bus_attrs[] = {
553 &bus_attr_add.attr,
554 &bus_attr_remove.attr,
9b60e70b
ID		 555 &bus_attr_add_single_major.attr,
556 &bus_attr_remove_single_major.attr,
8767b293 557 &bus_attr_supported_features.attr,
b15a21dd 558 NULL,
f0f8cef5 559};
92c76dc0
ID		 560
561static umode_t rbd_bus_is_visible(struct kobject *kobj,
562 struct attribute *attr, int index)
563{
9b60e70b
ID		 564 if (!single_major &&
565 (attr == &bus_attr_add_single_major.attr ||
566 attr == &bus_attr_remove_single_major.attr))
567 return 0;
568
92c76dc0
ID		 569 return attr->mode;
570}
571
572static const struct attribute_group rbd_bus_group = {
573 .attrs = rbd_bus_attrs,
574 .is_visible = rbd_bus_is_visible,
575};
576__ATTRIBUTE_GROUPS(rbd_bus);
f0f8cef5
AE		 577
578static struct bus_type rbd_bus_type = {
579 .name = "rbd",
b15a21dd 580 .bus_groups = rbd_bus_groups,
f0f8cef5
AE		 581};
582
583static void rbd_root_dev_release(struct device *dev)
584{
585}
586
587static struct device rbd_root_dev = {
588 .init_name = "rbd",
589 .release = rbd_root_dev_release,
590};
591
06ecc6cb
AE		 592static __printf(2, 3)
593void rbd_warn(struct rbd_device *rbd_dev, const char *fmt, ...)
594{
595 struct va_format vaf;
596 va_list args;
597
598 va_start(args, fmt);
599 vaf.fmt = fmt;
600 vaf.va = &args;
601
602 if (!rbd_dev)
603 printk(KERN_WARNING "%s: %pV\n", RBD_DRV_NAME, &vaf);
604 else if (rbd_dev->disk)
605 printk(KERN_WARNING "%s: %s: %pV\n",
606 RBD_DRV_NAME, rbd_dev->disk->disk_name, &vaf);
607 else if (rbd_dev->spec && rbd_dev->spec->image_name)
608 printk(KERN_WARNING "%s: image %s: %pV\n",
609 RBD_DRV_NAME, rbd_dev->spec->image_name, &vaf);
610 else if (rbd_dev->spec && rbd_dev->spec->image_id)
611 printk(KERN_WARNING "%s: id %s: %pV\n",
612 RBD_DRV_NAME, rbd_dev->spec->image_id, &vaf);
613 else /* punt */
614 printk(KERN_WARNING "%s: rbd_dev %p: %pV\n",
615 RBD_DRV_NAME, rbd_dev, &vaf);
616 va_end(args);
617}
618
aafb230e
AE		 619#ifdef RBD_DEBUG
620#define rbd_assert(expr) \
621 if (unlikely(!(expr))) { \
622 printk(KERN_ERR "\nAssertion failure in %s() " \
623 "at line %d:\n\n" \
624 "\trbd_assert(%s);\n\n", \
625 __func__, __LINE__, #expr); \
626 BUG(); \
627 }
628#else /* !RBD_DEBUG */
629# define rbd_assert(expr) ((void) 0)
630#endif /* !RBD_DEBUG */
dfc5606d 631
05a46afd 632static void rbd_dev_remove_parent(struct rbd_device *rbd_dev);
8b3e1a56 633
cc4a38bd 634static int rbd_dev_refresh(struct rbd_device *rbd_dev);
510a7330
ID		 635static int rbd_dev_v2_header_onetime(struct rbd_device *rbd_dev,
636 struct rbd_image_header *header);
54cac61f
AE		 637static const char *rbd_dev_v2_snap_name(struct rbd_device *rbd_dev,
638 u64 snap_id);
2ad3d716
AE		 639static int _rbd_dev_v2_snap_size(struct rbd_device *rbd_dev, u64 snap_id,
640 u8 *order, u64 *snap_size);
22e8bd51 641static int rbd_dev_v2_get_flags(struct rbd_device *rbd_dev);
59c2be1e 642
54ab3b24 643static void rbd_obj_handle_request(struct rbd_obj_request *obj_req, int result);
0192ce2e
ID		 644static void rbd_img_handle_request(struct rbd_img_request *img_req, int result);
645
646/*
647 * Return true if nothing else is pending.
648 */
649static bool pending_result_dec(struct pending_result *pending, int *result)
650{
651 rbd_assert(pending->num_pending > 0);
652
653 if (*result && !pending->result)
654 pending->result = *result;
655 if (--pending->num_pending)
656 return false;
657
658 *result = pending->result;
659 return true;
660}
59c2be1e 661
05bdb996 662static int rbd_open(struct gendisk *disk, blk_mode_t mode)
602adf40 663{
d32e2bf8 664 struct rbd_device *rbd_dev = disk->private_data;
b82d167b 665 bool removing = false;
602adf40 666
a14ea269 667 spin_lock_irq(&rbd_dev->lock);
b82d167b
AE		 668 if (test_bit(RBD_DEV_FLAG_REMOVING, &rbd_dev->flags))
669 removing = true;
670 else
671 rbd_dev->open_count++;
a14ea269 672 spin_unlock_irq(&rbd_dev->lock);
b82d167b
AE		 673 if (removing)
674 return -ENOENT;
675
c3e946ce 676 (void) get_device(&rbd_dev->dev);
340c7a2b 677
602adf40
YS		 678 return 0;
679}
680
ae220766 681static void rbd_release(struct gendisk *disk)
dfc5606d
YS		 682{
683 struct rbd_device *rbd_dev = disk->private_data;
b82d167b
AE		 684 unsigned long open_count_before;
685
a14ea269 686 spin_lock_irq(&rbd_dev->lock);
b82d167b 687 open_count_before = rbd_dev->open_count--;
a14ea269 688 spin_unlock_irq(&rbd_dev->lock);
b82d167b 689 rbd_assert(open_count_before > 0);
dfc5606d 690
c3e946ce 691 put_device(&rbd_dev->dev);
dfc5606d
YS		 692}
693
602adf40
YS		 694static const struct block_device_operations rbd_bd_ops = {
695 .owner = THIS_MODULE,
696 .open = rbd_open,
dfc5606d 697 .release = rbd_release,
602adf40
YS		 698};
699
700/*
7262cfca 701 * Initialize an rbd client instance. Success or not, this function
cfbf6377 702 * consumes ceph_opts. Caller holds client_mutex.
602adf40 703 */
f8c38929 704static struct rbd_client *rbd_client_create(struct ceph_options *ceph_opts)
602adf40
YS		 705{
706 struct rbd_client *rbdc;
707 int ret = -ENOMEM;
708
37206ee5 709 dout("%s:\n", __func__);
602adf40
YS		 710 rbdc = kmalloc(sizeof(struct rbd_client), GFP_KERNEL);
711 if (!rbdc)
712 goto out_opt;
713
714 kref_init(&rbdc->kref);
715 INIT_LIST_HEAD(&rbdc->node);
716
74da4a0f 717 rbdc->client = ceph_create_client(ceph_opts, rbdc);
602adf40 718 if (IS_ERR(rbdc->client))
08f75463 719 goto out_rbdc;
43ae4701 720 ceph_opts = NULL; /* Now rbdc->client is responsible for ceph_opts */
602adf40
YS		 721
722 ret = ceph_open_session(rbdc->client);
723 if (ret < 0)
08f75463 724 goto out_client;
602adf40 725
432b8587 726 spin_lock(&rbd_client_list_lock);
602adf40 727 list_add_tail(&rbdc->node, &rbd_client_list);
432b8587 728 spin_unlock(&rbd_client_list_lock);
602adf40 729
37206ee5 730 dout("%s: rbdc %p\n", __func__, rbdc);
bc534d86 731
602adf40 732 return rbdc;
08f75463 733out_client:
602adf40 734 ceph_destroy_client(rbdc->client);
08f75463 735out_rbdc:
602adf40
YS		 736 kfree(rbdc);
737out_opt:
43ae4701
AE		 738 if (ceph_opts)
739 ceph_destroy_options(ceph_opts);
37206ee5
AE		 740 dout("%s: error %d\n", __func__, ret);
741
28f259b7 742 return ERR_PTR(ret);
602adf40
YS		 743}
744
2f82ee54
AE		 745static struct rbd_client *__rbd_get_client(struct rbd_client *rbdc)
746{
747 kref_get(&rbdc->kref);
748
749 return rbdc;
750}
751
602adf40 752/*
1f7ba331
AE		 753 * Find a ceph client with specific addr and configuration. If
754 * found, bump its reference count.
602adf40 755 */
1f7ba331 756static struct rbd_client *rbd_client_find(struct ceph_options *ceph_opts)
602adf40 757{
3302ffd4 758 struct rbd_client *rbdc = NULL, *iter;
602adf40 759
43ae4701 760 if (ceph_opts->flags & CEPH_OPT_NOSHARE)
602adf40
YS		 761 return NULL;
762
1f7ba331 763 spin_lock(&rbd_client_list_lock);
3302ffd4
JK		 764 list_for_each_entry(iter, &rbd_client_list, node) {
765 if (!ceph_compare_options(ceph_opts, iter->client)) {
766 __rbd_get_client(iter);
2f82ee54 767
3302ffd4 768 rbdc = iter;
1f7ba331
AE		 769 break;
770 }
771 }
772 spin_unlock(&rbd_client_list_lock);
773
3302ffd4 774 return rbdc;
602adf40
YS		 775}
776
59c2be1e 777/*
210c104c 778 * (Per device) rbd map options
59c2be1e
YS		 779 */
780enum {
b5584180 781 Opt_queue_depth,
0c93e1b7 782 Opt_alloc_size,
34f55d0b 783 Opt_lock_timeout,
59c2be1e 784 /* int args above */
b26c047b 785 Opt_pool_ns,
dc1dad8e 786 Opt_compression_hint,
59c2be1e 787 /* string args above */
cc0538b6
AE		 788 Opt_read_only,
789 Opt_read_write,
80de1912 790 Opt_lock_on_read,
e010dd0a 791 Opt_exclusive,
d9360540 792 Opt_notrim,
59c2be1e
YS		 793};
794
dc1dad8e
ID		 795enum {
796 Opt_compression_hint_none,
797 Opt_compression_hint_compressible,
798 Opt_compression_hint_incompressible,
799};
800
801static const struct constant_table rbd_param_compression_hint[] = {
802 {"none", Opt_compression_hint_none},
803 {"compressible", Opt_compression_hint_compressible},
804 {"incompressible", Opt_compression_hint_incompressible},
805 {}
806};
807
d7167b14 808static const struct fs_parameter_spec rbd_parameters[] = {
82995cc6 809 fsparam_u32 ("alloc_size", Opt_alloc_size),
dc1dad8e
ID		 810 fsparam_enum ("compression_hint", Opt_compression_hint,
811 rbd_param_compression_hint),
82995cc6
DH		 812 fsparam_flag ("exclusive", Opt_exclusive),
813 fsparam_flag ("lock_on_read", Opt_lock_on_read),
814 fsparam_u32 ("lock_timeout", Opt_lock_timeout),
815 fsparam_flag ("notrim", Opt_notrim),
816 fsparam_string ("_pool_ns", Opt_pool_ns),
817 fsparam_u32 ("queue_depth", Opt_queue_depth),
818 fsparam_flag ("read_only", Opt_read_only),
819 fsparam_flag ("read_write", Opt_read_write),
820 fsparam_flag ("ro", Opt_read_only),
821 fsparam_flag ("rw", Opt_read_write),
822 {}
823};
824
98571b5a 825struct rbd_options {
b5584180 826 int queue_depth;
0c93e1b7 827 int alloc_size;
34f55d0b 828 unsigned long lock_timeout;
98571b5a 829 bool read_only;
80de1912 830 bool lock_on_read;
e010dd0a 831 bool exclusive;
d9360540 832 bool trim;
dc1dad8e
ID		 833
834 u32 alloc_hint_flags; /* CEPH_OSD_OP_ALLOC_HINT_FLAG_* */
98571b5a
AE		 835};
836
d2a27964 837#define RBD_QUEUE_DEPTH_DEFAULT BLKDEV_DEFAULT_RQ
0c93e1b7 838#define RBD_ALLOC_SIZE_DEFAULT (64 * 1024)
34f55d0b 839#define RBD_LOCK_TIMEOUT_DEFAULT 0 /* no timeout */
98571b5a 840#define RBD_READ_ONLY_DEFAULT false
80de1912 841#define RBD_LOCK_ON_READ_DEFAULT false
e010dd0a 842#define RBD_EXCLUSIVE_DEFAULT false
d9360540 843#define RBD_TRIM_DEFAULT true
98571b5a 844
82995cc6 845struct rbd_parse_opts_ctx {
c300156b 846 struct rbd_spec *spec;
82995cc6 847 struct ceph_options *copts;
c300156b
ID		 848 struct rbd_options *opts;
849};
850
6d2940c8
GZ		 851static char* obj_op_name(enum obj_operation_type op_type)
852{
853 switch (op_type) {
854 case OBJ_OP_READ:
855 return "read";
856 case OBJ_OP_WRITE:
857 return "write";
90e98c52
GZ		 858 case OBJ_OP_DISCARD:
859 return "discard";
6484cbe9
ID		 860 case OBJ_OP_ZEROOUT:
861 return "zeroout";
6d2940c8
GZ		 862 default:
863 return "???";
864 }
865}
866
602adf40
YS		 867/*
868 * Destroy ceph client
d23a4b3f 869 *
432b8587 870 * Caller must hold rbd_client_list_lock.
602adf40
YS		 871 */
872static void rbd_client_release(struct kref *kref)
873{
874 struct rbd_client *rbdc = container_of(kref, struct rbd_client, kref);
875
37206ee5 876 dout("%s: rbdc %p\n", __func__, rbdc);
cd9d9f5d 877 spin_lock(&rbd_client_list_lock);
602adf40 878 list_del(&rbdc->node);
cd9d9f5d 879 spin_unlock(&rbd_client_list_lock);
602adf40
YS		 880
881 ceph_destroy_client(rbdc->client);
882 kfree(rbdc);
883}
884
885/*
886 * Drop reference to ceph client node. If it's not referenced anymore, release
887 * it.
888 */
9d3997fd 889static void rbd_put_client(struct rbd_client *rbdc)
602adf40 890{
c53d5893
AE		 891 if (rbdc)
892 kref_put(&rbdc->kref, rbd_client_release);
602adf40
YS		 893}
894
5feb0d8d
ID		 895/*
896 * Get a ceph client with specific addr and configuration, if one does
897 * not exist create it. Either way, ceph_opts is consumed by this
898 * function.
899 */
900static struct rbd_client *rbd_get_client(struct ceph_options *ceph_opts)
901{
902 struct rbd_client *rbdc;
dd435855 903 int ret;
5feb0d8d 904
a32e4143 905 mutex_lock(&client_mutex);
5feb0d8d 906 rbdc = rbd_client_find(ceph_opts);
dd435855 907 if (rbdc) {
5feb0d8d 908 ceph_destroy_options(ceph_opts);
dd435855
ID		 909
910 /*
911 * Using an existing client. Make sure ->pg_pools is up to
912 * date before we look up the pool id in do_rbd_add().
913 */
9d4a227f
ID		 914 ret = ceph_wait_for_latest_osdmap(rbdc->client,
915 rbdc->client->options->mount_timeout);
dd435855
ID		 916 if (ret) {
917 rbd_warn(NULL, "failed to get latest osdmap: %d", ret);
918 rbd_put_client(rbdc);
919 rbdc = ERR_PTR(ret);
920 }
921 } else {
5feb0d8d 922 rbdc = rbd_client_create(ceph_opts);
dd435855 923 }
5feb0d8d
ID		 924 mutex_unlock(&client_mutex);
925
926 return rbdc;
927}
928
a30b71b9
AE		 929static bool rbd_image_format_valid(u32 image_format)
930{
931 return image_format == 1 || image_format == 2;
932}
933
8e94af8e
AE		 934static bool rbd_dev_ondisk_valid(struct rbd_image_header_ondisk *ondisk)
935{
103a150f
AE		 936 size_t size;
937 u32 snap_count;
938
939 /* The header has to start with the magic rbd header text */
940 if (memcmp(&ondisk->text, RBD_HEADER_TEXT, sizeof (RBD_HEADER_TEXT)))
941 return false;
942
db2388b6
AE		 943 /* The bio layer requires at least sector-sized I/O */
944
945 if (ondisk->options.order < SECTOR_SHIFT)
946 return false;
947
948 /* If we use u64 in a few spots we may be able to loosen this */
949
950 if (ondisk->options.order > 8 * sizeof (int) - 1)
951 return false;
952
103a150f
AE		 953 /*
954 * The size of a snapshot header has to fit in a size_t, and
955 * that limits the number of snapshots.
956 */
957 snap_count = le32_to_cpu(ondisk->snap_count);
958 size = SIZE_MAX - sizeof (struct ceph_snap_context);
959 if (snap_count > size / sizeof (__le64))
960 return false;
961
962 /*
963 * Not only that, but the size of the entire the snapshot
964 * header must also be representable in a size_t.
965 */
966 size -= snap_count * sizeof (__le64);
967 if ((u64) size < le64_to_cpu(ondisk->snap_names_len))
968 return false;
969
970 return true;
8e94af8e
AE		 971}
972
5bc3fb17
ID		 973/*
974 * returns the size of an object in the image
975 */
976static u32 rbd_obj_bytes(struct rbd_image_header *header)
977{
978 return 1U << header->obj_order;
979}
980
263423f8
ID		 981static void rbd_init_layout(struct rbd_device *rbd_dev)
982{
983 if (rbd_dev->header.stripe_unit == 0 ||
984 rbd_dev->header.stripe_count == 0) {
985 rbd_dev->header.stripe_unit = rbd_obj_bytes(&rbd_dev->header);
986 rbd_dev->header.stripe_count = 1;
987 }
988
989 rbd_dev->layout.stripe_unit = rbd_dev->header.stripe_unit;
990 rbd_dev->layout.stripe_count = rbd_dev->header.stripe_count;
991 rbd_dev->layout.object_size = rbd_obj_bytes(&rbd_dev->header);
7e97332e
ID		 992 rbd_dev->layout.pool_id = rbd_dev->header.data_pool_id == CEPH_NOPOOL ?
993 rbd_dev->spec->pool_id : rbd_dev->header.data_pool_id;
263423f8
ID		 994 RCU_INIT_POINTER(rbd_dev->layout.pool_ns, NULL);
995}
996
510a7330
ID		 997static void rbd_image_header_cleanup(struct rbd_image_header *header)
998{
999 kfree(header->object_prefix);
1000 ceph_put_snap_context(header->snapc);
1001 kfree(header->snap_sizes);
1002 kfree(header->snap_names);
1003
1004 memset(header, 0, sizeof(*header));
1005}
1006
602adf40 1007/*
bb23e37a
AE		 1008 * Fill an rbd image header with information from the given format 1
1009 * on-disk header.
602adf40 1010 */
510a7330
ID		 1011static int rbd_header_from_disk(struct rbd_image_header *header,
1012 struct rbd_image_header_ondisk *ondisk,
1013 bool first_time)
602adf40 1014{
bb23e37a
AE		 1015 struct ceph_snap_context *snapc;
1016 char *object_prefix = NULL;
1017 char *snap_names = NULL;
1018 u64 *snap_sizes = NULL;
ccece235 1019 u32 snap_count;
bb23e37a 1020 int ret = -ENOMEM;
621901d6 1021 u32 i;
602adf40 1022
bb23e37a 1023 /* Allocate this now to avoid having to handle failure below */
6a52325f 1024
bb23e37a 1025 if (first_time) {
848d796c
ID		 1026 object_prefix = kstrndup(ondisk->object_prefix,
1027 sizeof(ondisk->object_prefix),
1028 GFP_KERNEL);
bb23e37a
AE		 1029 if (!object_prefix)
1030 return -ENOMEM;
bb23e37a 1031 }
00f1f36f 1032
bb23e37a 1033 /* Allocate the snapshot context and fill it in */
00f1f36f 1034
bb23e37a
AE		 1035 snap_count = le32_to_cpu(ondisk->snap_count);
1036 snapc = ceph_create_snap_context(snap_count, GFP_KERNEL);
1037 if (!snapc)
1038 goto out_err;
1039 snapc->seq = le64_to_cpu(ondisk->snap_seq);
602adf40 1040 if (snap_count) {
bb23e37a 1041 struct rbd_image_snap_ondisk *snaps;
f785cc1d
AE		 1042 u64 snap_names_len = le64_to_cpu(ondisk->snap_names_len);
1043
bb23e37a 1044 /* We'll keep a copy of the snapshot names... */
621901d6 1045
bb23e37a
AE		 1046 if (snap_names_len > (u64)SIZE_MAX)
1047 goto out_2big;
1048 snap_names = kmalloc(snap_names_len, GFP_KERNEL);
1049 if (!snap_names)
6a52325f
AE		 1050 goto out_err;
1051
bb23e37a 1052 /* ...as well as the array of their sizes. */
88a25a5f
ME		 1053 snap_sizes = kmalloc_array(snap_count,
1054 sizeof(*header->snap_sizes),
1055 GFP_KERNEL);
bb23e37a 1056 if (!snap_sizes)
6a52325f 1057 goto out_err;
bb23e37a 1058
f785cc1d 1059 /*
bb23e37a
AE		 1060 * Copy the names, and fill in each snapshot's id
1061 * and size.
1062 *
99a41ebc 1063 * Note that rbd_dev_v1_header_info() guarantees the
bb23e37a 1064 * ondisk buffer we're working with has
f785cc1d
AE		 1065 * snap_names_len bytes beyond the end of the
1066 * snapshot id array, this memcpy() is safe.
1067 */
bb23e37a
AE		 1068 memcpy(snap_names, &ondisk->snaps[snap_count], snap_names_len);
1069 snaps = ondisk->snaps;
1070 for (i = 0; i < snap_count; i++) {
1071 snapc->snaps[i] = le64_to_cpu(snaps[i].id);
1072 snap_sizes[i] = le64_to_cpu(snaps[i].image_size);
1073 }
602adf40 1074 }
6a52325f 1075
bb23e37a 1076 /* We won't fail any more, fill in the header */
621901d6 1077
bb23e37a
AE		 1078 if (first_time) {
1079 header->object_prefix = object_prefix;
1080 header->obj_order = ondisk->options.order;
602adf40 1081 }
849b4260 1082
bb23e37a 1083 /* The remaining fields always get updated (when we refresh) */
621901d6 1084
f84344f3 1085 header->image_size = le64_to_cpu(ondisk->image_size);
bb23e37a
AE		 1086 header->snapc = snapc;
1087 header->snap_names = snap_names;
1088 header->snap_sizes = snap_sizes;
468521c1 1089
602adf40 1090 return 0;
bb23e37a
AE		 1091out_2big:
1092 ret = -EIO;
6a52325f 1093out_err:
bb23e37a
AE		 1094 kfree(snap_sizes);
1095 kfree(snap_names);
1096 ceph_put_snap_context(snapc);
1097 kfree(object_prefix);
ccece235 1098
bb23e37a 1099 return ret;
602adf40
YS		 1100}
1101
9682fc6d
AE		 1102static const char *_rbd_dev_v1_snap_name(struct rbd_device *rbd_dev, u32 which)
1103{
1104 const char *snap_name;
1105
1106 rbd_assert(which < rbd_dev->header.snapc->num_snaps);
1107
1108 /* Skip over names until we find the one we are looking for */
1109
1110 snap_name = rbd_dev->header.snap_names;
1111 while (which--)
1112 snap_name += strlen(snap_name) + 1;
1113
1114 return kstrdup(snap_name, GFP_KERNEL);
1115}
1116
30d1cff8
AE		 1117/*
1118 * Snapshot id comparison function for use with qsort()/bsearch().
1119 * Note that result is for snapshots in *descending* order.
1120 */
1121static int snapid_compare_reverse(const void *s1, const void *s2)
1122{
1123 u64 snap_id1 = *(u64 *)s1;
1124 u64 snap_id2 = *(u64 *)s2;
1125
1126 if (snap_id1 < snap_id2)
1127 return 1;
1128 return snap_id1 == snap_id2 ? 0 : -1;
1129}
1130
1131/*
1132 * Search a snapshot context to see if the given snapshot id is
1133 * present.
1134 *
1135 * Returns the position of the snapshot id in the array if it's found,
1136 * or BAD_SNAP_INDEX otherwise.
1137 *
1138 * Note: The snapshot array is in kept sorted (by the osd) in
1139 * reverse order, highest snapshot id first.
1140 */
9682fc6d
AE		 1141static u32 rbd_dev_snap_index(struct rbd_device *rbd_dev, u64 snap_id)
1142{
1143 struct ceph_snap_context *snapc = rbd_dev->header.snapc;
30d1cff8 1144 u64 *found;
9682fc6d 1145
30d1cff8
AE		 1146 found = bsearch(&snap_id, &snapc->snaps, snapc->num_snaps,
1147 sizeof (snap_id), snapid_compare_reverse);
9682fc6d 1148
30d1cff8 1149 return found ? (u32)(found - &snapc->snaps[0]) : BAD_SNAP_INDEX;
9682fc6d
AE		 1150}
1151
2ad3d716
AE		 1152static const char *rbd_dev_v1_snap_name(struct rbd_device *rbd_dev,
1153 u64 snap_id)
9e15b77d 1154{
54cac61f 1155 u32 which;
da6a6b63 1156 const char *snap_name;
9e15b77d 1157
54cac61f
AE		 1158 which = rbd_dev_snap_index(rbd_dev, snap_id);
1159 if (which == BAD_SNAP_INDEX)
da6a6b63 1160 return ERR_PTR(-ENOENT);
54cac61f 1161
da6a6b63
JD		 1162 snap_name = _rbd_dev_v1_snap_name(rbd_dev, which);
1163 return snap_name ? snap_name : ERR_PTR(-ENOMEM);
54cac61f
AE		 1164}
1165
1166static const char *rbd_snap_name(struct rbd_device *rbd_dev, u64 snap_id)
1167{
9e15b77d
AE		 1168 if (snap_id == CEPH_NOSNAP)
1169 return RBD_SNAP_HEAD_NAME;
1170
54cac61f
AE		 1171 rbd_assert(rbd_image_format_valid(rbd_dev->image_format));
1172 if (rbd_dev->image_format == 1)
1173 return rbd_dev_v1_snap_name(rbd_dev, snap_id);
9e15b77d 1174
54cac61f 1175 return rbd_dev_v2_snap_name(rbd_dev, snap_id);
9e15b77d
AE		 1176}
1177
2ad3d716
AE		 1178static int rbd_snap_size(struct rbd_device *rbd_dev, u64 snap_id,
1179 u64 *snap_size)
602adf40 1180{
2ad3d716
AE		 1181 rbd_assert(rbd_image_format_valid(rbd_dev->image_format));
1182 if (snap_id == CEPH_NOSNAP) {
1183 *snap_size = rbd_dev->header.image_size;
1184 } else if (rbd_dev->image_format == 1) {
1185 u32 which;
602adf40 1186
2ad3d716
AE		 1187 which = rbd_dev_snap_index(rbd_dev, snap_id);
1188 if (which == BAD_SNAP_INDEX)
1189 return -ENOENT;
e86924a8 1190
2ad3d716
AE		 1191 *snap_size = rbd_dev->header.snap_sizes[which];
1192 } else {
1193 u64 size = 0;
1194 int ret;
1195
1196 ret = _rbd_dev_v2_snap_size(rbd_dev, snap_id, NULL, &size);
1197 if (ret)
1198 return ret;
1199
1200 *snap_size = size;
1201 }
1202 return 0;
602adf40
YS		 1203}
1204
2ad3d716
AE		 1205static int rbd_dev_mapping_set(struct rbd_device *rbd_dev)
1206{
8f4b7d98 1207 u64 snap_id = rbd_dev->spec->snap_id;
2ad3d716 1208 u64 size = 0;
2ad3d716
AE		 1209 int ret;
1210
2ad3d716 1211 ret = rbd_snap_size(rbd_dev, snap_id, &size);
2ad3d716
AE		 1212 if (ret)
1213 return ret;
1214
1215 rbd_dev->mapping.size = size;
8b0241f8 1216 return 0;
602adf40
YS		 1217}
1218
d1cf5788
AE		 1219static void rbd_dev_mapping_clear(struct rbd_device *rbd_dev)
1220{
1221 rbd_dev->mapping.size = 0;
200a6a8b
AE		 1222}
1223
5359a17d 1224static void zero_bios(struct ceph_bio_iter *bio_pos, u32 off, u32 bytes)
b9434c5b 1225{
5359a17d 1226 struct ceph_bio_iter it = *bio_pos;
b9434c5b 1227
5359a17d
ID		 1228 ceph_bio_iter_advance(&it, off);
1229 ceph_bio_iter_advance_step(&it, bytes, ({
732022b8 1230 memzero_bvec(&bv);
5359a17d 1231 }));
b9434c5b
AE		 1232}
1233
7e07efb1 1234static void zero_bvecs(struct ceph_bvec_iter *bvec_pos, u32 off, u32 bytes)
602adf40 1235{
7e07efb1 1236 struct ceph_bvec_iter it = *bvec_pos;
602adf40 1237
7e07efb1
ID		 1238 ceph_bvec_iter_advance(&it, off);
1239 ceph_bvec_iter_advance_step(&it, bytes, ({
732022b8 1240 memzero_bvec(&bv);
7e07efb1 1241 }));
f7760dad
AE		 1242}
1243
1244/*
3da691bf 1245 * Zero a range in @obj_req data buffer defined by a bio (list) or
afb97888 1246 * (private) bio_vec array.
f7760dad 1247 *
3da691bf 1248 * @off is relative to the start of the data buffer.
926f9b3f 1249 */
3da691bf
ID		 1250static void rbd_obj_zero_range(struct rbd_obj_request *obj_req, u32 off,
1251 u32 bytes)
926f9b3f 1252{
54ab3b24
ID		 1253 dout("%s %p data buf %u~%u\n", __func__, obj_req, off, bytes);
1254
ecc633ca 1255 switch (obj_req->img_request->data_type) {
3da691bf
ID		 1256 case OBJ_REQUEST_BIO:
1257 zero_bios(&obj_req->bio_pos, off, bytes);
1258 break;
1259 case OBJ_REQUEST_BVECS:
afb97888 1260 case OBJ_REQUEST_OWN_BVECS:
3da691bf
ID		 1261 zero_bvecs(&obj_req->bvec_pos, off, bytes);
1262 break;
1263 default:
16809372 1264 BUG();
6365d33a
AE		 1265 }
1266}
1267
bf0d5f50
AE		 1268static void rbd_obj_request_destroy(struct kref *kref);
1269static void rbd_obj_request_put(struct rbd_obj_request *obj_request)
1270{
1271 rbd_assert(obj_request != NULL);
37206ee5 1272 dout("%s: obj %p (was %d)\n", __func__, obj_request,
2c935bc5 1273 kref_read(&obj_request->kref));
bf0d5f50
AE		 1274 kref_put(&obj_request->kref, rbd_obj_request_destroy);
1275}
1276
bf0d5f50
AE		 1277static inline void rbd_img_obj_request_add(struct rbd_img_request *img_request,
1278 struct rbd_obj_request *obj_request)
1279{
25dcf954
AE		 1280 rbd_assert(obj_request->img_request == NULL);
1281
b155e86c 1282 /* Image request now owns object's original reference */
bf0d5f50 1283 obj_request->img_request = img_request;
15961b44 1284 dout("%s: img %p obj %p\n", __func__, img_request, obj_request);
bf0d5f50
AE		 1285}
1286
1287static inline void rbd_img_obj_request_del(struct rbd_img_request *img_request,
1288 struct rbd_obj_request *obj_request)
1289{
15961b44 1290 dout("%s: img %p obj %p\n", __func__, img_request, obj_request);
43df3d35 1291 list_del(&obj_request->ex.oe_item);
bf0d5f50 1292 rbd_assert(obj_request->img_request == img_request);
bf0d5f50
AE		 1293 rbd_obj_request_put(obj_request);
1294}
1295
a086a1b8 1296static void rbd_osd_submit(struct ceph_osd_request *osd_req)
bf0d5f50 1297{
a086a1b8 1298 struct rbd_obj_request *obj_req = osd_req->r_priv;
980917fc 1299
a086a1b8
ID		 1300 dout("%s osd_req %p for obj_req %p objno %llu %llu~%llu\n",
1301 __func__, osd_req, obj_req, obj_req->ex.oe_objno,
1302 obj_req->ex.oe_off, obj_req->ex.oe_len);
a8af0d68 1303 ceph_osdc_start_request(osd_req->r_osdc, osd_req);
bf0d5f50
AE		 1304}
1305
0c425248
AE		 1306/*
1307 * The default/initial value for all image request flags is 0. Each
1308 * is conditionally set to 1 at image request initialization time
1309 * and currently never change thereafter.
1310 */
d0b2e944
AE		 1311static void img_request_layered_set(struct rbd_img_request *img_request)
1312{
1313 set_bit(IMG_REQ_LAYERED, &img_request->flags);
d0b2e944
AE		 1314}
1315
1316static bool img_request_layered_test(struct rbd_img_request *img_request)
1317{
d0b2e944
AE		 1318 return test_bit(IMG_REQ_LAYERED, &img_request->flags) != 0;
1319}
1320
3da691bf 1321static bool rbd_obj_is_entire(struct rbd_obj_request *obj_req)
6e2a4505 1322{
3da691bf 1323 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
b9434c5b 1324
43df3d35
ID		 1325 return !obj_req->ex.oe_off &&
1326 obj_req->ex.oe_len == rbd_dev->layout.object_size;
6e2a4505
AE		 1327}
1328
3da691bf 1329static bool rbd_obj_is_tail(struct rbd_obj_request *obj_req)
bf0d5f50 1330{
3da691bf 1331 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
bf0d5f50 1332
43df3d35 1333 return obj_req->ex.oe_off + obj_req->ex.oe_len ==
3da691bf 1334 rbd_dev->layout.object_size;
0dcc685e
ID		 1335}
1336
13488d53
ID		 1337/*
1338 * Must be called after rbd_obj_calc_img_extents().
1339 */
09fe05c5 1340static void rbd_obj_set_copyup_enabled(struct rbd_obj_request *obj_req)
13488d53 1341{
870611e4
ID		 1342 rbd_assert(obj_req->img_request->snapc);
1343
09fe05c5
ID		 1344 if (obj_req->img_request->op_type == OBJ_OP_DISCARD) {
1345 dout("%s %p objno %llu discard\n", __func__, obj_req,
1346 obj_req->ex.oe_objno);
1347 return;
1348 }
13488d53 1349
09fe05c5
ID		 1350 if (!obj_req->num_img_extents) {
1351 dout("%s %p objno %llu not overlapping\n", __func__, obj_req,
1352 obj_req->ex.oe_objno);
1353 return;
1354 }
1355
1356 if (rbd_obj_is_entire(obj_req) &&
1357 !obj_req->img_request->snapc->num_snaps) {
1358 dout("%s %p objno %llu entire\n", __func__, obj_req,
1359 obj_req->ex.oe_objno);
1360 return;
1361 }
1362
1363 obj_req->flags |= RBD_OBJ_FLAG_COPYUP_ENABLED;
13488d53
ID		 1364}
1365
86bd7998 1366static u64 rbd_obj_img_extents_bytes(struct rbd_obj_request *obj_req)
bf0d5f50 1367{
86bd7998
ID		 1368 return ceph_file_extents_bytes(obj_req->img_extents,
1369 obj_req->num_img_extents);
bf0d5f50
AE		 1370}
1371
3da691bf 1372static bool rbd_img_is_write(struct rbd_img_request *img_req)
bf0d5f50 1373{
9bb0248d 1374 switch (img_req->op_type) {
3da691bf
ID		 1375 case OBJ_OP_READ:
1376 return false;
1377 case OBJ_OP_WRITE:
1378 case OBJ_OP_DISCARD:
6484cbe9 1379 case OBJ_OP_ZEROOUT:
3da691bf
ID		 1380 return true;
1381 default:
c6244b3b 1382 BUG();
3da691bf 1383 }
90e98c52
GZ		 1384}
1385
85e084fe 1386static void rbd_osd_req_callback(struct ceph_osd_request *osd_req)
bf0d5f50 1387{
3da691bf 1388 struct rbd_obj_request *obj_req = osd_req->r_priv;
54ab3b24 1389 int result;
bf0d5f50 1390
3da691bf
ID		 1391 dout("%s osd_req %p result %d for obj_req %p\n", __func__, osd_req,
1392 osd_req->r_result, obj_req);
bf0d5f50 1393
54ab3b24
ID		 1394 /*
1395 * Writes aren't allowed to return a data payload. In some
1396 * guarded write cases (e.g. stat + zero on an empty object)
1397 * a stat response makes it through, but we don't care.
1398 */
1399 if (osd_req->r_result > 0 && rbd_img_is_write(obj_req->img_request))
1400 result = 0;
3da691bf 1401 else
54ab3b24 1402 result = osd_req->r_result;
bf0d5f50 1403
54ab3b24 1404 rbd_obj_handle_request(obj_req, result);
bf0d5f50
AE		 1405}
1406
bcbab1db 1407static void rbd_osd_format_read(struct ceph_osd_request *osd_req)
430c28c3 1408{
bcbab1db 1409 struct rbd_obj_request *obj_request = osd_req->r_priv;
22d2cfdf
ID		 1410 struct rbd_device *rbd_dev = obj_request->img_request->rbd_dev;
1411 struct ceph_options *opt = rbd_dev->rbd_client->client->options;
430c28c3 1412
22d2cfdf 1413 osd_req->r_flags = CEPH_OSD_FLAG_READ | opt->read_from_replica;
7c84883a 1414 osd_req->r_snapid = obj_request->img_request->snap_id;
9d4df01f
AE		 1415}
1416
bcbab1db 1417static void rbd_osd_format_write(struct ceph_osd_request *osd_req)
9d4df01f 1418{
bcbab1db 1419 struct rbd_obj_request *obj_request = osd_req->r_priv;
9d4df01f 1420
a162b308 1421 osd_req->r_flags = CEPH_OSD_FLAG_WRITE;
fac02ddf 1422 ktime_get_real_ts64(&osd_req->r_mtime);
43df3d35 1423 osd_req->r_data_offset = obj_request->ex.oe_off;
430c28c3
AE		 1424}
1425
bc81207e 1426static struct ceph_osd_request *
bcbab1db
ID		 1427__rbd_obj_add_osd_request(struct rbd_obj_request *obj_req,
1428 struct ceph_snap_context *snapc, int num_ops)
bc81207e 1429{
e28eded5 1430 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
bc81207e
ID		 1431 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
1432 struct ceph_osd_request *req;
a90bb0c1
ID		 1433 const char *name_format = rbd_dev->image_format == 1 ?
1434 RBD_V1_DATA_FORMAT : RBD_V2_DATA_FORMAT;
bcbab1db 1435 int ret;
bc81207e 1436
e28eded5 1437 req = ceph_osdc_alloc_request(osdc, snapc, num_ops, false, GFP_NOIO);
bc81207e 1438 if (!req)
bcbab1db 1439 return ERR_PTR(-ENOMEM);
bc81207e 1440
bcbab1db 1441 list_add_tail(&req->r_private_item, &obj_req->osd_reqs);
bc81207e 1442 req->r_callback = rbd_osd_req_callback;
a162b308 1443 req->r_priv = obj_req;
bc81207e 1444
b26c047b
ID		 1445 /*
1446 * Data objects may be stored in a separate pool, but always in
1447 * the same namespace in that pool as the header in its pool.
1448 */
1449 ceph_oloc_copy(&req->r_base_oloc, &rbd_dev->header_oloc);
bc81207e 1450 req->r_base_oloc.pool = rbd_dev->layout.pool_id;
b26c047b 1451
bcbab1db
ID		 1452 ret = ceph_oid_aprintf(&req->r_base_oid, GFP_NOIO, name_format,
1453 rbd_dev->header.object_prefix,
1454 obj_req->ex.oe_objno);
1455 if (ret)
1456 return ERR_PTR(ret);
bc81207e 1457
bc81207e 1458 return req;
bc81207e
ID		 1459}
1460
e28eded5 1461static struct ceph_osd_request *
bcbab1db 1462rbd_obj_add_osd_request(struct rbd_obj_request *obj_req, int num_ops)
bf0d5f50 1463{
870611e4 1464 rbd_assert(obj_req->img_request->snapc);
bcbab1db
ID		 1465 return __rbd_obj_add_osd_request(obj_req, obj_req->img_request->snapc,
1466 num_ops);
bf0d5f50
AE		 1467}
1468
ecc633ca 1469static struct rbd_obj_request *rbd_obj_request_create(void)
bf0d5f50
AE		 1470{
1471 struct rbd_obj_request *obj_request;
bf0d5f50 1472
5a60e876 1473 obj_request = kmem_cache_zalloc(rbd_obj_request_cache, GFP_NOIO);
6c696d85 1474 if (!obj_request)
f907ad55 1475 return NULL;
f907ad55 1476
43df3d35 1477 ceph_object_extent_init(&obj_request->ex);
bcbab1db 1478 INIT_LIST_HEAD(&obj_request->osd_reqs);
85b5e6d1 1479 mutex_init(&obj_request->state_mutex);
bf0d5f50
AE		 1480 kref_init(&obj_request->kref);
1481
67e2b652 1482 dout("%s %p\n", __func__, obj_request);
bf0d5f50
AE		 1483 return obj_request;
1484}
1485
1486static void rbd_obj_request_destroy(struct kref *kref)
1487{
1488 struct rbd_obj_request *obj_request;
bcbab1db 1489 struct ceph_osd_request *osd_req;
7e07efb1 1490 u32 i;
bf0d5f50
AE		 1491
1492 obj_request = container_of(kref, struct rbd_obj_request, kref);
1493
37206ee5
AE		 1494 dout("%s: obj %p\n", __func__, obj_request);
1495
bcbab1db
ID		 1496 while (!list_empty(&obj_request->osd_reqs)) {
1497 osd_req = list_first_entry(&obj_request->osd_reqs,
1498 struct ceph_osd_request, r_private_item);
1499 list_del_init(&osd_req->r_private_item);
1500 ceph_osdc_put_request(osd_req);
1501 }
bf0d5f50 1502
ecc633ca 1503 switch (obj_request->img_request->data_type) {
9969ebc5 1504 case OBJ_REQUEST_NODATA:
bf0d5f50 1505 case OBJ_REQUEST_BIO:
7e07efb1 1506 case OBJ_REQUEST_BVECS:
5359a17d 1507 break; /* Nothing to do */
afb97888
ID		 1508 case OBJ_REQUEST_OWN_BVECS:
1509 kfree(obj_request->bvec_pos.bvecs);
788e2df3 1510 break;
7e07efb1 1511 default:
16809372 1512 BUG();
bf0d5f50
AE		 1513 }
1514
86bd7998 1515 kfree(obj_request->img_extents);
7e07efb1
ID		 1516 if (obj_request->copyup_bvecs) {
1517 for (i = 0; i < obj_request->copyup_bvec_count; i++) {
1518 if (obj_request->copyup_bvecs[i].bv_page)
1519 __free_page(obj_request->copyup_bvecs[i].bv_page);
1520 }
1521 kfree(obj_request->copyup_bvecs);
bf0d5f50
AE		 1522 }
1523
868311b1 1524 kmem_cache_free(rbd_obj_request_cache, obj_request);
bf0d5f50
AE		 1525}
1526
fb65d228
AE		 1527/* It's OK to call this for a device with no parent */
1528
1529static void rbd_spec_put(struct rbd_spec *spec);
1530static void rbd_dev_unparent(struct rbd_device *rbd_dev)
1531{
1532 rbd_dev_remove_parent(rbd_dev);
1533 rbd_spec_put(rbd_dev->parent_spec);
1534 rbd_dev->parent_spec = NULL;
1535 rbd_dev->parent_overlap = 0;
1536}
1537
a2acd00e
AE		 1538/*
1539 * Parent image reference counting is used to determine when an
1540 * image's parent fields can be safely torn down--after there are no
1541 * more in-flight requests to the parent image. When the last
1542 * reference is dropped, cleaning them up is safe.
1543 */
1544static void rbd_dev_parent_put(struct rbd_device *rbd_dev)
1545{
1546 int counter;
1547
1548 if (!rbd_dev->parent_spec)
1549 return;
1550
1551 counter = atomic_dec_return_safe(&rbd_dev->parent_ref);
1552 if (counter > 0)
1553 return;
1554
1555 /* Last reference; clean up parent data structures */
1556
1557 if (!counter)
1558 rbd_dev_unparent(rbd_dev);
1559 else
9584d508 1560 rbd_warn(rbd_dev, "parent reference underflow");
a2acd00e
AE		 1561}
1562
1563/*
1564 * If an image has a non-zero parent overlap, get a reference to its
1565 * parent.
1566 *
1567 * Returns true if the rbd device has a parent with a non-zero
1568 * overlap and a reference for it was successfully taken, or
1569 * false otherwise.
1570 */
1571static bool rbd_dev_parent_get(struct rbd_device *rbd_dev)
1572{
ae43e9d0 1573 int counter = 0;
a2acd00e
AE		 1574
1575 if (!rbd_dev->parent_spec)
1576 return false;
1577
ae43e9d0
ID		 1578 if (rbd_dev->parent_overlap)
1579 counter = atomic_inc_return_safe(&rbd_dev->parent_ref);
a2acd00e
AE		 1580
1581 if (counter < 0)
9584d508 1582 rbd_warn(rbd_dev, "parent reference overflow");
a2acd00e 1583
ae43e9d0 1584 return counter > 0;
a2acd00e
AE		 1585}
1586
59e542c8
ID		 1587static void rbd_img_request_init(struct rbd_img_request *img_request,
1588 struct rbd_device *rbd_dev,
1589 enum obj_operation_type op_type)
bf0d5f50 1590{
59e542c8 1591 memset(img_request, 0, sizeof(*img_request));
bf0d5f50 1592
bf0d5f50 1593 img_request->rbd_dev = rbd_dev;
9bb0248d 1594 img_request->op_type = op_type;
a0c5895b 1595
e1fddc8f 1596 INIT_LIST_HEAD(&img_request->lock_item);
43df3d35 1597 INIT_LIST_HEAD(&img_request->object_extents);
0192ce2e 1598 mutex_init(&img_request->state_mutex);
bf0d5f50
AE		 1599}
1600
870611e4
ID		 1601/*
1602 * Only snap_id is captured here, for reads. For writes, snapshot
1603 * context is captured in rbd_img_object_requests() after exclusive
1604 * lock is ensured to be held.
1605 */
a52cc685
ID		 1606static void rbd_img_capture_header(struct rbd_img_request *img_req)
1607{
1608 struct rbd_device *rbd_dev = img_req->rbd_dev;
1609
1610 lockdep_assert_held(&rbd_dev->header_rwsem);
1611
870611e4 1612 if (!rbd_img_is_write(img_req))
a52cc685
ID		 1613 img_req->snap_id = rbd_dev->spec->snap_id;
1614
1615 if (rbd_dev_parent_get(rbd_dev))
1616 img_request_layered_set(img_req);
1617}
1618
679a97d2 1619static void rbd_img_request_destroy(struct rbd_img_request *img_request)
bf0d5f50 1620{
bf0d5f50
AE		 1621 struct rbd_obj_request *obj_request;
1622 struct rbd_obj_request *next_obj_request;
1623
37206ee5
AE		 1624 dout("%s: img %p\n", __func__, img_request);
1625
e1fddc8f 1626 WARN_ON(!list_empty(&img_request->lock_item));
bf0d5f50
AE		 1627 for_each_obj_request_safe(img_request, obj_request, next_obj_request)
1628 rbd_img_obj_request_del(img_request, obj_request);
1629
78b42a87 1630 if (img_request_layered_test(img_request))
a2acd00e 1631 rbd_dev_parent_put(img_request->rbd_dev);
a2acd00e 1632
9bb0248d 1633 if (rbd_img_is_write(img_request))
812164f8 1634 ceph_put_snap_context(img_request->snapc);
bf0d5f50 1635
59e542c8
ID		 1636 if (test_bit(IMG_REQ_CHILD, &img_request->flags))
1637 kmem_cache_free(rbd_img_request_cache, img_request);
bf0d5f50
AE		 1638}
1639
22e8bd51
ID		 1640#define BITS_PER_OBJ 2
1641#define OBJS_PER_BYTE (BITS_PER_BYTE / BITS_PER_OBJ)
1642#define OBJ_MASK ((1 << BITS_PER_OBJ) - 1)
e93f3152 1643
22e8bd51
ID		 1644static void __rbd_object_map_index(struct rbd_device *rbd_dev, u64 objno,
1645 u64 *index, u8 *shift)
1646{
1647 u32 off;
e93f3152 1648
22e8bd51
ID		 1649 rbd_assert(objno < rbd_dev->object_map_size);
1650 *index = div_u64_rem(objno, OBJS_PER_BYTE, &off);
1651 *shift = (OBJS_PER_BYTE - off - 1) * BITS_PER_OBJ;
1652}
e93f3152 1653
22e8bd51
ID		 1654static u8 __rbd_object_map_get(struct rbd_device *rbd_dev, u64 objno)
1655{
1656 u64 index;
1657 u8 shift;
e93f3152 1658
22e8bd51
ID		 1659 lockdep_assert_held(&rbd_dev->object_map_lock);
1660 __rbd_object_map_index(rbd_dev, objno, &index, &shift);
1661 return (rbd_dev->object_map[index] >> shift) & OBJ_MASK;
e93f3152
AE		 1662}
1663
22e8bd51 1664static void __rbd_object_map_set(struct rbd_device *rbd_dev, u64 objno, u8 val)
e93f3152 1665{
22e8bd51
ID		 1666 u64 index;
1667 u8 shift;
1668 u8 *p;
e93f3152 1669
22e8bd51
ID		 1670 lockdep_assert_held(&rbd_dev->object_map_lock);
1671 rbd_assert(!(val & ~OBJ_MASK));
e93f3152 1672
22e8bd51
ID		 1673 __rbd_object_map_index(rbd_dev, objno, &index, &shift);
1674 p = &rbd_dev->object_map[index];
1675 *p = (*p & ~(OBJ_MASK << shift)) | (val << shift);
e93f3152
AE		 1676}
1677
22e8bd51 1678static u8 rbd_object_map_get(struct rbd_device *rbd_dev, u64 objno)
1217857f 1679{
22e8bd51
ID		 1680 u8 state;
1681
1682 spin_lock(&rbd_dev->object_map_lock);
1683 state = __rbd_object_map_get(rbd_dev, objno);
1684 spin_unlock(&rbd_dev->object_map_lock);
1685 return state;
3da691bf 1686}
1217857f 1687
22e8bd51 1688static bool use_object_map(struct rbd_device *rbd_dev)
3da691bf 1689{
3fe69921
ID		 1690 /*
1691 * An image mapped read-only can't use the object map -- it isn't
1692 * loaded because the header lock isn't acquired. Someone else can
1693 * write to the image and update the object map behind our back.
1694 *
1695 * A snapshot can't be written to, so using the object map is always
1696 * safe.
1697 */
1698 if (!rbd_is_snap(rbd_dev) && rbd_is_ro(rbd_dev))
1699 return false;
1700
22e8bd51
ID		 1701 return ((rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP) &&
1702 !(rbd_dev->object_map_flags & RBD_FLAG_OBJECT_MAP_INVALID));
3da691bf
ID		 1703}
1704
22e8bd51 1705static bool rbd_object_map_may_exist(struct rbd_device *rbd_dev, u64 objno)
3da691bf 1706{
22e8bd51 1707 u8 state;
8b3e1a56 1708
22e8bd51
ID		 1709 /* fall back to default logic if object map is disabled or invalid */
1710 if (!use_object_map(rbd_dev))
1711 return true;
3da691bf 1712
22e8bd51
ID		 1713 state = rbd_object_map_get(rbd_dev, objno);
1714 return state != OBJECT_NONEXISTENT;
1217857f
AE		 1715}
1716
22e8bd51
ID		 1717static void rbd_object_map_name(struct rbd_device *rbd_dev, u64 snap_id,
1718 struct ceph_object_id *oid)
13488d53 1719{
22e8bd51
ID		 1720 if (snap_id == CEPH_NOSNAP)
1721 ceph_oid_printf(oid, "%s%s", RBD_OBJECT_MAP_PREFIX,
1722 rbd_dev->spec->image_id);
1723 else
1724 ceph_oid_printf(oid, "%s%s.%016llx", RBD_OBJECT_MAP_PREFIX,
1725 rbd_dev->spec->image_id, snap_id);
13488d53
ID		 1726}
1727
22e8bd51 1728static int rbd_object_map_lock(struct rbd_device *rbd_dev)
2169238d 1729{
22e8bd51
ID		 1730 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
1731 CEPH_DEFINE_OID_ONSTACK(oid);
1732 u8 lock_type;
1733 char *lock_tag;
1734 struct ceph_locker *lockers;
1735 u32 num_lockers;
1736 bool broke_lock = false;
1737 int ret;
2169238d 1738
22e8bd51 1739 rbd_object_map_name(rbd_dev, CEPH_NOSNAP, &oid);
2169238d 1740
22e8bd51
ID		 1741again:
1742 ret = ceph_cls_lock(osdc, &oid, &rbd_dev->header_oloc, RBD_LOCK_NAME,
1743 CEPH_CLS_LOCK_EXCLUSIVE, "", "", "", 0);
1744 if (ret != -EBUSY || broke_lock) {
1745 if (ret == -EEXIST)
1746 ret = 0; /* already locked by myself */
1747 if (ret)
1748 rbd_warn(rbd_dev, "failed to lock object map: %d", ret);
1749 return ret;
1750 }
2169238d 1751
22e8bd51
ID		 1752 ret = ceph_cls_lock_info(osdc, &oid, &rbd_dev->header_oloc,
1753 RBD_LOCK_NAME, &lock_type, &lock_tag,
1754 &lockers, &num_lockers);
1755 if (ret) {
1756 if (ret == -ENOENT)
1757 goto again;
3da691bf 1758
22e8bd51 1759 rbd_warn(rbd_dev, "failed to get object map lockers: %d", ret);
86bd7998 1760 return ret;
22e8bd51 1761 }
86bd7998 1762
22e8bd51
ID		 1763 kfree(lock_tag);
1764 if (num_lockers == 0)
1765 goto again;
2169238d 1766
22e8bd51
ID		 1767 rbd_warn(rbd_dev, "breaking object map lock owned by %s%llu",
1768 ENTITY_NAME(lockers[0].id.name));
2169238d 1769
22e8bd51
ID		 1770 ret = ceph_cls_break_lock(osdc, &oid, &rbd_dev->header_oloc,
1771 RBD_LOCK_NAME, lockers[0].id.cookie,
1772 &lockers[0].id.name);
1773 ceph_free_lockers(lockers, num_lockers);
1774 if (ret) {
1775 if (ret == -ENOENT)
1776 goto again;
13488d53 1777
22e8bd51
ID		 1778 rbd_warn(rbd_dev, "failed to break object map lock: %d", ret);
1779 return ret;
3da691bf
ID		 1780 }
1781
22e8bd51
ID		 1782 broke_lock = true;
1783 goto again;
2169238d
AE		 1784}
1785
22e8bd51 1786static void rbd_object_map_unlock(struct rbd_device *rbd_dev)
6484cbe9 1787{
22e8bd51
ID		 1788 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
1789 CEPH_DEFINE_OID_ONSTACK(oid);
1790 int ret;
1791
1792 rbd_object_map_name(rbd_dev, CEPH_NOSNAP, &oid);
1793
1794 ret = ceph_cls_unlock(osdc, &oid, &rbd_dev->header_oloc, RBD_LOCK_NAME,
1795 "");
1796 if (ret && ret != -ENOENT)
1797 rbd_warn(rbd_dev, "failed to unlock object map: %d", ret);
6484cbe9
ID		 1798}
1799
22e8bd51 1800static int decode_object_map_header(void **p, void *end, u64 *object_map_size)
6484cbe9 1801{
22e8bd51
ID		 1802 u8 struct_v;
1803 u32 struct_len;
1804 u32 header_len;
1805 void *header_end;
6484cbe9
ID		 1806 int ret;
1807
22e8bd51
ID		 1808 ceph_decode_32_safe(p, end, header_len, e_inval);
1809 header_end = *p + header_len;
0c93e1b7 1810
22e8bd51
ID		 1811 ret = ceph_start_decoding(p, end, 1, "BitVector header", &struct_v,
1812 &struct_len);
6484cbe9
ID		 1813 if (ret)
1814 return ret;
1815
22e8bd51 1816 ceph_decode_64_safe(p, end, *object_map_size, e_inval);
6484cbe9 1817
22e8bd51 1818 *p = header_end;
6484cbe9 1819 return 0;
22e8bd51
ID		 1820
1821e_inval:
1822 return -EINVAL;
6484cbe9
ID		 1823}
1824
22e8bd51 1825static int __rbd_object_map_load(struct rbd_device *rbd_dev)
13488d53 1826{
22e8bd51
ID		 1827 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
1828 CEPH_DEFINE_OID_ONSTACK(oid);
1829 struct page **pages;
1830 void *p, *end;
1831 size_t reply_len;
1832 u64 num_objects;
1833 u64 object_map_bytes;
1834 u64 object_map_size;
1835 int num_pages;
1836 int ret;
13488d53 1837
22e8bd51 1838 rbd_assert(!rbd_dev->object_map && !rbd_dev->object_map_size);
13488d53 1839
22e8bd51
ID		 1840 num_objects = ceph_get_num_objects(&rbd_dev->layout,
1841 rbd_dev->mapping.size);
1842 object_map_bytes = DIV_ROUND_UP_ULL(num_objects * BITS_PER_OBJ,
1843 BITS_PER_BYTE);
1844 num_pages = calc_pages_for(0, object_map_bytes) + 1;
1845 pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL);
1846 if (IS_ERR(pages))
1847 return PTR_ERR(pages);
13488d53 1848
22e8bd51
ID		 1849 reply_len = num_pages * PAGE_SIZE;
1850 rbd_object_map_name(rbd_dev, rbd_dev->spec->snap_id, &oid);
1851 ret = ceph_osdc_call(osdc, &oid, &rbd_dev->header_oloc,
1852 "rbd", "object_map_load", CEPH_OSD_FLAG_READ,
1853 NULL, 0, pages, &reply_len);
1854 if (ret)
1855 goto out;
3b434a2a 1856
22e8bd51
ID		 1857 p = page_address(pages[0]);
1858 end = p + min(reply_len, (size_t)PAGE_SIZE);
1859 ret = decode_object_map_header(&p, end, &object_map_size);
1860 if (ret)
1861 goto out;
1862
1863 if (object_map_size != num_objects) {
1864 rbd_warn(rbd_dev, "object map size mismatch: %llu vs %llu",
1865 object_map_size, num_objects);
1866 ret = -EINVAL;
1867 goto out;
3b434a2a
JD		 1868 }
1869
22e8bd51
ID		 1870 if (offset_in_page(p) + object_map_bytes > reply_len) {
1871 ret = -EINVAL;
1872 goto out;
1873 }
1874
1875 rbd_dev->object_map = kvmalloc(object_map_bytes, GFP_KERNEL);
1876 if (!rbd_dev->object_map) {
1877 ret = -ENOMEM;
1878 goto out;
1879 }
1880
1881 rbd_dev->object_map_size = object_map_size;
1882 ceph_copy_from_page_vector(pages, rbd_dev->object_map,
1883 offset_in_page(p), object_map_bytes);
1884
1885out:
1886 ceph_release_page_vector(pages, num_pages);
1887 return ret;
1888}
3da691bf 1889
22e8bd51
ID		 1890static void rbd_object_map_free(struct rbd_device *rbd_dev)
1891{
1892 kvfree(rbd_dev->object_map);
1893 rbd_dev->object_map = NULL;
1894 rbd_dev->object_map_size = 0;
3b434a2a
JD		 1895}
1896
22e8bd51 1897static int rbd_object_map_load(struct rbd_device *rbd_dev)
bf0d5f50 1898{
3da691bf 1899 int ret;
37206ee5 1900
22e8bd51 1901 ret = __rbd_object_map_load(rbd_dev);
86bd7998
ID		 1902 if (ret)
1903 return ret;
f1a4739f 1904
22e8bd51
ID		 1905 ret = rbd_dev_v2_get_flags(rbd_dev);
1906 if (ret) {
1907 rbd_object_map_free(rbd_dev);
1908 return ret;
1909 }
1910
1911 if (rbd_dev->object_map_flags & RBD_FLAG_OBJECT_MAP_INVALID)
1912 rbd_warn(rbd_dev, "object map is invalid");
1913
1914 return 0;
1915}
1916
1917static int rbd_object_map_open(struct rbd_device *rbd_dev)
1918{
1919 int ret;
1920
1921 ret = rbd_object_map_lock(rbd_dev);
1922 if (ret)
1923 return ret;
1924
1925 ret = rbd_object_map_load(rbd_dev);
1926 if (ret) {
1927 rbd_object_map_unlock(rbd_dev);
1928 return ret;
1929 }
1930
1931 return 0;
1932}
1933
1934static void rbd_object_map_close(struct rbd_device *rbd_dev)
1935{
1936 rbd_object_map_free(rbd_dev);
1937 rbd_object_map_unlock(rbd_dev);
1938}
1939
1940/*
1941 * This function needs snap_id (or more precisely just something to
1942 * distinguish between HEAD and snapshot object maps), new_state and
1943 * current_state that were passed to rbd_object_map_update().
1944 *
1945 * To avoid allocating and stashing a context we piggyback on the OSD
1946 * request. A HEAD update has two ops (assert_locked). For new_state
1947 * and current_state we decode our own object_map_update op, encoded in
1948 * rbd_cls_object_map_update().
1949 */
1950static int rbd_object_map_update_finish(struct rbd_obj_request *obj_req,
1951 struct ceph_osd_request *osd_req)
1952{
1953 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
1954 struct ceph_osd_data *osd_data;
1955 u64 objno;
3f649ab7 1956 u8 state, new_state, current_state;
22e8bd51
ID		 1957 bool has_current_state;
1958 void *p;
1959
1960 if (osd_req->r_result)
1961 return osd_req->r_result;
1962
1963 /*
1964 * Nothing to do for a snapshot object map.
1965 */
1966 if (osd_req->r_num_ops == 1)
1967 return 0;
1968
1969 /*
1970 * Update in-memory HEAD object map.
1971 */
1972 rbd_assert(osd_req->r_num_ops == 2);
1973 osd_data = osd_req_op_data(osd_req, 1, cls, request_data);
1974 rbd_assert(osd_data->type == CEPH_OSD_DATA_TYPE_PAGES);
1975
1976 p = page_address(osd_data->pages[0]);
1977 objno = ceph_decode_64(&p);
1978 rbd_assert(objno == obj_req->ex.oe_objno);
1979 rbd_assert(ceph_decode_64(&p) == objno + 1);
1980 new_state = ceph_decode_8(&p);
1981 has_current_state = ceph_decode_8(&p);
1982 if (has_current_state)
1983 current_state = ceph_decode_8(&p);
1984
1985 spin_lock(&rbd_dev->object_map_lock);
1986 state = __rbd_object_map_get(rbd_dev, objno);
1987 if (!has_current_state || current_state == state ||
1988 (current_state == OBJECT_EXISTS && state == OBJECT_EXISTS_CLEAN))
1989 __rbd_object_map_set(rbd_dev, objno, new_state);
1990 spin_unlock(&rbd_dev->object_map_lock);
1991
1992 return 0;
1993}
1994
1995static void rbd_object_map_callback(struct ceph_osd_request *osd_req)
1996{
1997 struct rbd_obj_request *obj_req = osd_req->r_priv;
1998 int result;
1999
2000 dout("%s osd_req %p result %d for obj_req %p\n", __func__, osd_req,
2001 osd_req->r_result, obj_req);
2002
2003 result = rbd_object_map_update_finish(obj_req, osd_req);
2004 rbd_obj_handle_request(obj_req, result);
2005}
2006
2007static bool update_needed(struct rbd_device *rbd_dev, u64 objno, u8 new_state)
2008{
2009 u8 state = rbd_object_map_get(rbd_dev, objno);
bf0d5f50 2010
22e8bd51
ID		 2011 if (state == new_state ||
2012 (new_state == OBJECT_PENDING && state == OBJECT_NONEXISTENT) ||
2013 (new_state == OBJECT_NONEXISTENT && state != OBJECT_PENDING))
2014 return false;
2015
2016 return true;
2017}
2018
2019static int rbd_cls_object_map_update(struct ceph_osd_request *req,
2020 int which, u64 objno, u8 new_state,
2021 const u8 *current_state)
2022{
2023 struct page **pages;
2024 void *p, *start;
2025 int ret;
2026
2027 ret = osd_req_op_cls_init(req, which, "rbd", "object_map_update");
2028 if (ret)
2029 return ret;
2030
2031 pages = ceph_alloc_page_vector(1, GFP_NOIO);
2032 if (IS_ERR(pages))
2033 return PTR_ERR(pages);
2034
2035 p = start = page_address(pages[0]);
2036 ceph_encode_64(&p, objno);
2037 ceph_encode_64(&p, objno + 1);
2038 ceph_encode_8(&p, new_state);
2039 if (current_state) {
2040 ceph_encode_8(&p, 1);
2041 ceph_encode_8(&p, *current_state);
2042 } else {
2043 ceph_encode_8(&p, 0);
2044 }
2045
2046 osd_req_op_cls_request_data_pages(req, which, pages, p - start, 0,
2047 false, true);
2048 return 0;
2049}
2050
2051/*
2052 * Return:
2053 * 0 - object map update sent
2054 * 1 - object map update isn't needed
2055 * <0 - error
2056 */
2057static int rbd_object_map_update(struct rbd_obj_request *obj_req, u64 snap_id,
2058 u8 new_state, const u8 *current_state)
2059{
2060 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
2061 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
2062 struct ceph_osd_request *req;
2063 int num_ops = 1;
2064 int which = 0;
2065 int ret;
2066
2067 if (snap_id == CEPH_NOSNAP) {
2068 if (!update_needed(rbd_dev, obj_req->ex.oe_objno, new_state))
2069 return 1;
2070
2071 num_ops++; /* assert_locked */
2072 }
2073
2074 req = ceph_osdc_alloc_request(osdc, NULL, num_ops, false, GFP_NOIO);
2075 if (!req)
2076 return -ENOMEM;
2077
2078 list_add_tail(&req->r_private_item, &obj_req->osd_reqs);
2079 req->r_callback = rbd_object_map_callback;
2080 req->r_priv = obj_req;
2081
2082 rbd_object_map_name(rbd_dev, snap_id, &req->r_base_oid);
2083 ceph_oloc_copy(&req->r_base_oloc, &rbd_dev->header_oloc);
2084 req->r_flags = CEPH_OSD_FLAG_WRITE;
2085 ktime_get_real_ts64(&req->r_mtime);
2086
2087 if (snap_id == CEPH_NOSNAP) {
2088 /*
2089 * Protect against possible race conditions during lock
2090 * ownership transitions.
2091 */
2092 ret = ceph_cls_assert_locked(req, which++, RBD_LOCK_NAME,
2093 CEPH_CLS_LOCK_EXCLUSIVE, "", "");
3da691bf
ID		 2094 if (ret)
2095 return ret;
22e8bd51
ID		 2096 }
2097
2098 ret = rbd_cls_object_map_update(req, which, obj_req->ex.oe_objno,
2099 new_state, current_state);
2100 if (ret)
2101 return ret;
2102
2103 ret = ceph_osdc_alloc_messages(req, GFP_NOIO);
2104 if (ret)
2105 return ret;
13488d53 2106
a8af0d68 2107 ceph_osdc_start_request(osdc, req);
22e8bd51
ID		 2108 return 0;
2109}
2110
86bd7998
ID		 2111static void prune_extents(struct ceph_file_extent *img_extents,
2112 u32 *num_img_extents, u64 overlap)
e93f3152 2113{
86bd7998 2114 u32 cnt = *num_img_extents;
e93f3152 2115
86bd7998
ID		 2116 /* drop extents completely beyond the overlap */
2117 while (cnt && img_extents[cnt - 1].fe_off >= overlap)
2118 cnt--;
e93f3152 2119
86bd7998
ID		 2120 if (cnt) {
2121 struct ceph_file_extent *ex = &img_extents[cnt - 1];
e93f3152 2122
86bd7998
ID		 2123 /* trim final overlapping extent */
2124 if (ex->fe_off + ex->fe_len > overlap)
2125 ex->fe_len = overlap - ex->fe_off;
2126 }
e93f3152 2127
86bd7998 2128 *num_img_extents = cnt;
e93f3152
AE		 2129}
2130
86bd7998
ID		 2131/*
2132 * Determine the byte range(s) covered by either just the object extent
2133 * or the entire object in the parent image.
2134 */
2135static int rbd_obj_calc_img_extents(struct rbd_obj_request *obj_req,
2136 bool entire)
e93f3152 2137{
86bd7998
ID		 2138 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
2139 int ret;
e93f3152 2140
86bd7998
ID		 2141 if (!rbd_dev->parent_overlap)
2142 return 0;
e93f3152 2143
86bd7998
ID		 2144 ret = ceph_extent_to_file(&rbd_dev->layout, obj_req->ex.oe_objno,
2145 entire ? 0 : obj_req->ex.oe_off,
2146 entire ? rbd_dev->layout.object_size :
2147 obj_req->ex.oe_len,
2148 &obj_req->img_extents,
2149 &obj_req->num_img_extents);
2150 if (ret)
2151 return ret;
e93f3152 2152
86bd7998
ID		 2153 prune_extents(obj_req->img_extents, &obj_req->num_img_extents,
2154 rbd_dev->parent_overlap);
2155 return 0;
e93f3152
AE		 2156}
2157
bcbab1db 2158static void rbd_osd_setup_data(struct ceph_osd_request *osd_req, int which)
1217857f 2159{
bcbab1db
ID		 2160 struct rbd_obj_request *obj_req = osd_req->r_priv;
2161
ecc633ca 2162 switch (obj_req->img_request->data_type) {
3da691bf 2163 case OBJ_REQUEST_BIO:
bcbab1db 2164 osd_req_op_extent_osd_data_bio(osd_req, which,
3da691bf 2165 &obj_req->bio_pos,
43df3d35 2166 obj_req->ex.oe_len);
3da691bf
ID		 2167 break;
2168 case OBJ_REQUEST_BVECS:
afb97888 2169 case OBJ_REQUEST_OWN_BVECS:
3da691bf 2170 rbd_assert(obj_req->bvec_pos.iter.bi_size ==
43df3d35 2171 obj_req->ex.oe_len);
afb97888 2172 rbd_assert(obj_req->bvec_idx == obj_req->bvec_count);
bcbab1db 2173 osd_req_op_extent_osd_data_bvec_pos(osd_req, which,
3da691bf
ID		 2174 &obj_req->bvec_pos);
2175 break;
2176 default:
16809372 2177 BUG();
1217857f 2178 }
3da691bf 2179}
1217857f 2180
bcbab1db 2181static int rbd_osd_setup_stat(struct ceph_osd_request *osd_req, int which)
3da691bf
ID		 2182{
2183 struct page **pages;
8b3e1a56 2184
3da691bf
ID		 2185 /*
2186 * The response data for a STAT call consists of:
2187 * le64 length;
2188 * struct {
2189 * le32 tv_sec;
2190 * le32 tv_nsec;
2191 * } mtime;
2192 */
2193 pages = ceph_alloc_page_vector(1, GFP_NOIO);
2194 if (IS_ERR(pages))
2195 return PTR_ERR(pages);
2196
bcbab1db
ID		 2197 osd_req_op_init(osd_req, which, CEPH_OSD_OP_STAT, 0);
2198 osd_req_op_raw_data_in_pages(osd_req, which, pages,
3da691bf
ID		 2199 8 + sizeof(struct ceph_timespec),
2200 0, false, true);
2201 return 0;
1217857f
AE		 2202}
2203
b5ae8cbc
ID		 2204static int rbd_osd_setup_copyup(struct ceph_osd_request *osd_req, int which,
2205 u32 bytes)
2206{
2207 struct rbd_obj_request *obj_req = osd_req->r_priv;
2208 int ret;
2209
2210 ret = osd_req_op_cls_init(osd_req, which, "rbd", "copyup");
2211 if (ret)
2212 return ret;
2213
2214 osd_req_op_cls_request_data_bvecs(osd_req, which, obj_req->copyup_bvecs,
2215 obj_req->copyup_bvec_count, bytes);
2216 return 0;
2217}
2218
ea9b743c
ID		 2219static int rbd_obj_init_read(struct rbd_obj_request *obj_req)
2220{
2221 obj_req->read_state = RBD_OBJ_READ_START;
2222 return 0;
2223}
2224
bcbab1db
ID		 2225static void __rbd_osd_setup_write_ops(struct ceph_osd_request *osd_req,
2226 int which)
2169238d 2227{
bcbab1db 2228 struct rbd_obj_request *obj_req = osd_req->r_priv;
3da691bf
ID		 2229 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
2230 u16 opcode;
2169238d 2231
8b5bec5c
ID		 2232 if (!use_object_map(rbd_dev) ||
2233 !(obj_req->flags & RBD_OBJ_FLAG_MAY_EXIST)) {
2234 osd_req_op_alloc_hint_init(osd_req, which++,
2235 rbd_dev->layout.object_size,
d3798acc 2236 rbd_dev->layout.object_size,
dc1dad8e 2237 rbd_dev->opts->alloc_hint_flags);
8b5bec5c 2238 }
2169238d 2239
3da691bf
ID		 2240 if (rbd_obj_is_entire(obj_req))
2241 opcode = CEPH_OSD_OP_WRITEFULL;
2242 else
2243 opcode = CEPH_OSD_OP_WRITE;
2169238d 2244
bcbab1db 2245 osd_req_op_extent_init(osd_req, which, opcode,
43df3d35 2246 obj_req->ex.oe_off, obj_req->ex.oe_len, 0, 0);
bcbab1db 2247 rbd_osd_setup_data(osd_req, which);
3da691bf 2248}
2169238d 2249
ea9b743c 2250static int rbd_obj_init_write(struct rbd_obj_request *obj_req)
3da691bf 2251{
3da691bf
ID		 2252 int ret;
2253
86bd7998
ID		 2254 /* reverse map the entire object onto the parent */
2255 ret = rbd_obj_calc_img_extents(obj_req, true);
2256 if (ret)
2257 return ret;
2258
85b5e6d1 2259 obj_req->write_state = RBD_OBJ_WRITE_START;
3da691bf 2260 return 0;
2169238d
AE		 2261}
2262
6484cbe9
ID		 2263static u16 truncate_or_zero_opcode(struct rbd_obj_request *obj_req)
2264{
2265 return rbd_obj_is_tail(obj_req) ? CEPH_OSD_OP_TRUNCATE :
2266 CEPH_OSD_OP_ZERO;
2267}
2268
27bbd911
ID		 2269static void __rbd_osd_setup_discard_ops(struct ceph_osd_request *osd_req,
2270 int which)
2271{
2272 struct rbd_obj_request *obj_req = osd_req->r_priv;
2273
2274 if (rbd_obj_is_entire(obj_req) && !obj_req->num_img_extents) {
2275 rbd_assert(obj_req->flags & RBD_OBJ_FLAG_DELETION);
2276 osd_req_op_init(osd_req, which, CEPH_OSD_OP_DELETE, 0);
13488d53 2277 } else {
27bbd911
ID		 2278 osd_req_op_extent_init(osd_req, which,
2279 truncate_or_zero_opcode(obj_req),
2280 obj_req->ex.oe_off, obj_req->ex.oe_len,
2281 0, 0);
2282 }
2283}
2284
ea9b743c 2285static int rbd_obj_init_discard(struct rbd_obj_request *obj_req)
6484cbe9 2286{
0c93e1b7 2287 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
27bbd911 2288 u64 off, next_off;
6484cbe9
ID		 2289 int ret;
2290
0c93e1b7
ID		 2291 /*
2292 * Align the range to alloc_size boundary and punt on discards
2293 * that are too small to free up any space.
2294 *
2295 * alloc_size == object_size && is_tail() is a special case for
2296 * filestore with filestore_punch_hole = false, needed to allow
2297 * truncate (in addition to delete).
2298 */
2299 if (rbd_dev->opts->alloc_size != rbd_dev->layout.object_size ||
2300 !rbd_obj_is_tail(obj_req)) {
27bbd911
ID		 2301 off = round_up(obj_req->ex.oe_off, rbd_dev->opts->alloc_size);
2302 next_off = round_down(obj_req->ex.oe_off + obj_req->ex.oe_len,
2303 rbd_dev->opts->alloc_size);
0c93e1b7
ID		 2304 if (off >= next_off)
2305 return 1;
27bbd911
ID		 2306
2307 dout("%s %p %llu~%llu -> %llu~%llu\n", __func__,
2308 obj_req, obj_req->ex.oe_off, obj_req->ex.oe_len,
2309 off, next_off - off);
2310 obj_req->ex.oe_off = off;
2311 obj_req->ex.oe_len = next_off - off;
0c93e1b7
ID		 2312 }
2313
6484cbe9
ID		 2314 /* reverse map the entire object onto the parent */
2315 ret = rbd_obj_calc_img_extents(obj_req, true);
2316 if (ret)
2317 return ret;
2318
22e8bd51 2319 obj_req->flags |= RBD_OBJ_FLAG_NOOP_FOR_NONEXISTENT;
0ad5d953
ID		 2320 if (rbd_obj_is_entire(obj_req) && !obj_req->num_img_extents)
2321 obj_req->flags |= RBD_OBJ_FLAG_DELETION;
2322
85b5e6d1 2323 obj_req->write_state = RBD_OBJ_WRITE_START;
6484cbe9
ID		 2324 return 0;
2325}
2326
bcbab1db
ID		 2327static void __rbd_osd_setup_zeroout_ops(struct ceph_osd_request *osd_req,
2328 int which)
3da691bf 2329{
bcbab1db 2330 struct rbd_obj_request *obj_req = osd_req->r_priv;
3b434a2a
JD		 2331 u16 opcode;
2332
3da691bf 2333 if (rbd_obj_is_entire(obj_req)) {
86bd7998 2334 if (obj_req->num_img_extents) {
0ad5d953 2335 if (!(obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED))
bcbab1db 2336 osd_req_op_init(osd_req, which++,
9b17eb2c 2337 CEPH_OSD_OP_CREATE, 0);
3b434a2a
JD		 2338 opcode = CEPH_OSD_OP_TRUNCATE;
2339 } else {
0ad5d953 2340 rbd_assert(obj_req->flags & RBD_OBJ_FLAG_DELETION);
bcbab1db 2341 osd_req_op_init(osd_req, which++,
3da691bf
ID		 2342 CEPH_OSD_OP_DELETE, 0);
2343 opcode = 0;
3b434a2a 2344 }
3b434a2a 2345 } else {
6484cbe9 2346 opcode = truncate_or_zero_opcode(obj_req);
3b434a2a
JD		 2347 }
2348
3da691bf 2349 if (opcode)
bcbab1db 2350 osd_req_op_extent_init(osd_req, which, opcode,
43df3d35 2351 obj_req->ex.oe_off, obj_req->ex.oe_len,
3da691bf 2352 0, 0);
3b434a2a
JD		 2353}
2354
ea9b743c 2355static int rbd_obj_init_zeroout(struct rbd_obj_request *obj_req)
bf0d5f50 2356{
3da691bf 2357 int ret;
37206ee5 2358
86bd7998
ID		 2359 /* reverse map the entire object onto the parent */
2360 ret = rbd_obj_calc_img_extents(obj_req, true);
2361 if (ret)
2362 return ret;
f1a4739f 2363
0ad5d953 2364 if (!obj_req->num_img_extents) {
22e8bd51 2365 obj_req->flags |= RBD_OBJ_FLAG_NOOP_FOR_NONEXISTENT;
0ad5d953
ID		 2366 if (rbd_obj_is_entire(obj_req))
2367 obj_req->flags |= RBD_OBJ_FLAG_DELETION;
3da691bf 2368 }
3b434a2a 2369
a086a1b8 2370 obj_req->write_state = RBD_OBJ_WRITE_START;
3da691bf
ID		 2371 return 0;
2372}
9d4df01f 2373
a086a1b8
ID		 2374static int count_write_ops(struct rbd_obj_request *obj_req)
2375{
8b5bec5c
ID		 2376 struct rbd_img_request *img_req = obj_req->img_request;
2377
2378 switch (img_req->op_type) {
a086a1b8 2379 case OBJ_OP_WRITE:
8b5bec5c
ID		 2380 if (!use_object_map(img_req->rbd_dev) ||
2381 !(obj_req->flags & RBD_OBJ_FLAG_MAY_EXIST))
2382 return 2; /* setallochint + write/writefull */
2383
2384 return 1; /* write/writefull */
a086a1b8
ID		 2385 case OBJ_OP_DISCARD:
2386 return 1; /* delete/truncate/zero */
2387 case OBJ_OP_ZEROOUT:
2388 if (rbd_obj_is_entire(obj_req) && obj_req->num_img_extents &&
2389 !(obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED))
2390 return 2; /* create + truncate */
bf0d5f50 2391
a086a1b8
ID		 2392 return 1; /* delete/truncate/zero */
2393 default:
2394 BUG();
3da691bf 2395 }
a086a1b8 2396}
3b434a2a 2397
a086a1b8
ID		 2398static void rbd_osd_setup_write_ops(struct ceph_osd_request *osd_req,
2399 int which)
2400{
2401 struct rbd_obj_request *obj_req = osd_req->r_priv;
2402
2403 switch (obj_req->img_request->op_type) {
2404 case OBJ_OP_WRITE:
2405 __rbd_osd_setup_write_ops(osd_req, which);
2406 break;
2407 case OBJ_OP_DISCARD:
2408 __rbd_osd_setup_discard_ops(osd_req, which);
2409 break;
2410 case OBJ_OP_ZEROOUT:
2411 __rbd_osd_setup_zeroout_ops(osd_req, which);
2412 break;
2413 default:
2414 BUG();
2415 }
3da691bf 2416}
9d4df01f 2417
3da691bf 2418/*
a086a1b8
ID		 2419 * Prune the list of object requests (adjust offset and/or length, drop
2420 * redundant requests). Prepare object request state machines and image
2421 * request state machine for execution.
3da691bf
ID		 2422 */
2423static int __rbd_img_fill_request(struct rbd_img_request *img_req)
2424{
0c93e1b7 2425 struct rbd_obj_request *obj_req, *next_obj_req;
3da691bf 2426 int ret;
430c28c3 2427
0c93e1b7 2428 for_each_obj_request_safe(img_req, obj_req, next_obj_req) {
9bb0248d 2429 switch (img_req->op_type) {
3da691bf 2430 case OBJ_OP_READ:
ea9b743c 2431 ret = rbd_obj_init_read(obj_req);
3da691bf
ID		 2432 break;
2433 case OBJ_OP_WRITE:
ea9b743c 2434 ret = rbd_obj_init_write(obj_req);
3da691bf
ID		 2435 break;
2436 case OBJ_OP_DISCARD:
ea9b743c 2437 ret = rbd_obj_init_discard(obj_req);
3da691bf 2438 break;
6484cbe9 2439 case OBJ_OP_ZEROOUT:
ea9b743c 2440 ret = rbd_obj_init_zeroout(obj_req);
6484cbe9 2441 break;
3da691bf 2442 default:
16809372 2443 BUG();
3da691bf 2444 }
0c93e1b7 2445 if (ret < 0)
3da691bf 2446 return ret;
0c93e1b7 2447 if (ret > 0) {
0c93e1b7
ID		 2448 rbd_img_obj_request_del(img_req, obj_req);
2449 continue;
2450 }
bf0d5f50
AE		 2451 }
2452
0192ce2e 2453 img_req->state = RBD_IMG_START;
bf0d5f50 2454 return 0;
3da691bf 2455}
bf0d5f50 2456
5a237819
ID		 2457union rbd_img_fill_iter {
2458 struct ceph_bio_iter bio_iter;
2459 struct ceph_bvec_iter bvec_iter;
2460};
bf0d5f50 2461
5a237819
ID		 2462struct rbd_img_fill_ctx {
2463 enum obj_request_type pos_type;
2464 union rbd_img_fill_iter *pos;
2465 union rbd_img_fill_iter iter;
2466 ceph_object_extent_fn_t set_pos_fn;
afb97888
ID		 2467 ceph_object_extent_fn_t count_fn;
2468 ceph_object_extent_fn_t copy_fn;
5a237819 2469};
bf0d5f50 2470
5a237819 2471static struct ceph_object_extent *alloc_object_extent(void *arg)
0eefd470 2472{
5a237819
ID		 2473 struct rbd_img_request *img_req = arg;
2474 struct rbd_obj_request *obj_req;
0eefd470 2475
5a237819
ID		 2476 obj_req = rbd_obj_request_create();
2477 if (!obj_req)
2478 return NULL;
2761713d 2479
5a237819
ID		 2480 rbd_img_obj_request_add(img_req, obj_req);
2481 return &obj_req->ex;
2482}
0eefd470 2483
afb97888
ID		 2484/*
2485 * While su != os && sc == 1 is technically not fancy (it's the same
2486 * layout as su == os && sc == 1), we can't use the nocopy path for it
2487 * because ->set_pos_fn() should be called only once per object.
2488 * ceph_file_to_extents() invokes action_fn once per stripe unit, so
2489 * treat su != os && sc == 1 as fancy.
2490 */
2491static bool rbd_layout_is_fancy(struct ceph_file_layout *l)
2492{
2493 return l->stripe_unit != l->object_size;
2494}
0eefd470 2495
afb97888
ID		 2496static int rbd_img_fill_request_nocopy(struct rbd_img_request *img_req,
2497 struct ceph_file_extent *img_extents,
2498 u32 num_img_extents,
2499 struct rbd_img_fill_ctx *fctx)
2500{
2501 u32 i;
2502 int ret;
2503
2504 img_req->data_type = fctx->pos_type;
0eefd470
AE		 2505
2506 /*
afb97888
ID		 2507 * Create object requests and set each object request's starting
2508 * position in the provided bio (list) or bio_vec array.
0eefd470 2509 */
afb97888
ID		 2510 fctx->iter = *fctx->pos;
2511 for (i = 0; i < num_img_extents; i++) {
2512 ret = ceph_file_to_extents(&img_req->rbd_dev->layout,
2513 img_extents[i].fe_off,
2514 img_extents[i].fe_len,
2515 &img_req->object_extents,
2516 alloc_object_extent, img_req,
2517 fctx->set_pos_fn, &fctx->iter);
2518 if (ret)
2519 return ret;
2520 }
0eefd470 2521
afb97888 2522 return __rbd_img_fill_request(img_req);
0eefd470
AE		 2523}
2524
5a237819
ID		 2525/*
2526 * Map a list of image extents to a list of object extents, create the
2527 * corresponding object requests (normally each to a different object,
2528 * but not always) and add them to @img_req. For each object request,
afb97888 2529 * set up its data descriptor to point to the corresponding chunk(s) of
5a237819
ID		 2530 * @fctx->pos data buffer.
2531 *
afb97888
ID		 2532 * Because ceph_file_to_extents() will merge adjacent object extents
2533 * together, each object request's data descriptor may point to multiple
2534 * different chunks of @fctx->pos data buffer.
2535 *
5a237819
ID		 2536 * @fctx->pos data buffer is assumed to be large enough.
2537 */
2538static int rbd_img_fill_request(struct rbd_img_request *img_req,
2539 struct ceph_file_extent *img_extents,
2540 u32 num_img_extents,
2541 struct rbd_img_fill_ctx *fctx)
3d7efd18 2542{
afb97888
ID		 2543 struct rbd_device *rbd_dev = img_req->rbd_dev;
2544 struct rbd_obj_request *obj_req;
5a237819
ID		 2545 u32 i;
2546 int ret;
2547
afb97888
ID		 2548 if (fctx->pos_type == OBJ_REQUEST_NODATA ||
2549 !rbd_layout_is_fancy(&rbd_dev->layout))
2550 return rbd_img_fill_request_nocopy(img_req, img_extents,
2551 num_img_extents, fctx);
3d7efd18 2552
afb97888 2553 img_req->data_type = OBJ_REQUEST_OWN_BVECS;
0eefd470 2554
bbea1c1a 2555 /*
afb97888
ID		 2556 * Create object requests and determine ->bvec_count for each object
2557 * request. Note that ->bvec_count sum over all object requests may
2558 * be greater than the number of bio_vecs in the provided bio (list)
2559 * or bio_vec array because when mapped, those bio_vecs can straddle
2560 * stripe unit boundaries.
bbea1c1a 2561 */
5a237819
ID		 2562 fctx->iter = *fctx->pos;
2563 for (i = 0; i < num_img_extents; i++) {
afb97888 2564 ret = ceph_file_to_extents(&rbd_dev->layout,
5a237819
ID		 2565 img_extents[i].fe_off,
2566 img_extents[i].fe_len,
2567 &img_req->object_extents,
2568 alloc_object_extent, img_req,
afb97888
ID		 2569 fctx->count_fn, &fctx->iter);
2570 if (ret)
2571 return ret;
bbea1c1a 2572 }
0eefd470 2573
afb97888
ID		 2574 for_each_obj_request(img_req, obj_req) {
2575 obj_req->bvec_pos.bvecs = kmalloc_array(obj_req->bvec_count,
2576 sizeof(*obj_req->bvec_pos.bvecs),
2577 GFP_NOIO);
2578 if (!obj_req->bvec_pos.bvecs)
2579 return -ENOMEM;
2580 }
0eefd470 2581
8785b1d4 2582 /*
afb97888
ID		 2583 * Fill in each object request's private bio_vec array, splitting and
2584 * rearranging the provided bio_vecs in stripe unit chunks as needed.
8785b1d4 2585 */
afb97888
ID		 2586 fctx->iter = *fctx->pos;
2587 for (i = 0; i < num_img_extents; i++) {
2588 ret = ceph_iterate_extents(&rbd_dev->layout,
2589 img_extents[i].fe_off,
2590 img_extents[i].fe_len,
2591 &img_req->object_extents,
2592 fctx->copy_fn, &fctx->iter);
5a237819
ID		 2593 if (ret)
2594 return ret;
2595 }
3d7efd18 2596
5a237819
ID		 2597 return __rbd_img_fill_request(img_req);
2598}
2599
2600static int rbd_img_fill_nodata(struct rbd_img_request *img_req,
2601 u64 off, u64 len)
2602{
2603 struct ceph_file_extent ex = { off, len };
a55e601b 2604 union rbd_img_fill_iter dummy = {};
5a237819
ID		 2605 struct rbd_img_fill_ctx fctx = {
2606 .pos_type = OBJ_REQUEST_NODATA,
2607 .pos = &dummy,
2608 };
2609
2610 return rbd_img_fill_request(img_req, &ex, 1, &fctx);
2611}
2612
2613static void set_bio_pos(struct ceph_object_extent *ex, u32 bytes, void *arg)
2614{
2615 struct rbd_obj_request *obj_req =
2616 container_of(ex, struct rbd_obj_request, ex);
2617 struct ceph_bio_iter *it = arg;
3d7efd18 2618
5a237819
ID		 2619 dout("%s objno %llu bytes %u\n", __func__, ex->oe_objno, bytes);
2620 obj_req->bio_pos = *it;
2621 ceph_bio_iter_advance(it, bytes);
2622}
3d7efd18 2623
afb97888
ID		 2624static void count_bio_bvecs(struct ceph_object_extent *ex, u32 bytes, void *arg)
2625{
2626 struct rbd_obj_request *obj_req =
2627 container_of(ex, struct rbd_obj_request, ex);
2628 struct ceph_bio_iter *it = arg;
0eefd470 2629
afb97888
ID		 2630 dout("%s objno %llu bytes %u\n", __func__, ex->oe_objno, bytes);
2631 ceph_bio_iter_advance_step(it, bytes, ({
2632 obj_req->bvec_count++;
2633 }));
0eefd470 2634
afb97888 2635}
0eefd470 2636
afb97888
ID		 2637static void copy_bio_bvecs(struct ceph_object_extent *ex, u32 bytes, void *arg)
2638{
2639 struct rbd_obj_request *obj_req =
2640 container_of(ex, struct rbd_obj_request, ex);
2641 struct ceph_bio_iter *it = arg;
0eefd470 2642
afb97888
ID		 2643 dout("%s objno %llu bytes %u\n", __func__, ex->oe_objno, bytes);
2644 ceph_bio_iter_advance_step(it, bytes, ({
2645 obj_req->bvec_pos.bvecs[obj_req->bvec_idx++] = bv;
2646 obj_req->bvec_pos.iter.bi_size += bv.bv_len;
2647 }));
3d7efd18
AE		 2648}
2649
5a237819
ID		 2650static int __rbd_img_fill_from_bio(struct rbd_img_request *img_req,
2651 struct ceph_file_extent *img_extents,
2652 u32 num_img_extents,
2653 struct ceph_bio_iter *bio_pos)
2654{
2655 struct rbd_img_fill_ctx fctx = {
2656 .pos_type = OBJ_REQUEST_BIO,
2657 .pos = (union rbd_img_fill_iter *)bio_pos,
2658 .set_pos_fn = set_bio_pos,
afb97888
ID		 2659 .count_fn = count_bio_bvecs,
2660 .copy_fn = copy_bio_bvecs,
5a237819 2661 };
3d7efd18 2662
5a237819
ID		 2663 return rbd_img_fill_request(img_req, img_extents, num_img_extents,
2664 &fctx);
2665}
3d7efd18 2666
5a237819
ID		 2667static int rbd_img_fill_from_bio(struct rbd_img_request *img_req,
2668 u64 off, u64 len, struct bio *bio)
2669{
2670 struct ceph_file_extent ex = { off, len };
2671 struct ceph_bio_iter it = { .bio = bio, .iter = bio->bi_iter };
3d7efd18 2672
5a237819
ID		 2673 return __rbd_img_fill_from_bio(img_req, &ex, 1, &it);
2674}
a9e8ba2c 2675
5a237819
ID		 2676static void set_bvec_pos(struct ceph_object_extent *ex, u32 bytes, void *arg)
2677{
2678 struct rbd_obj_request *obj_req =
2679 container_of(ex, struct rbd_obj_request, ex);
2680 struct ceph_bvec_iter *it = arg;
3d7efd18 2681
5a237819
ID		 2682 obj_req->bvec_pos = *it;
2683 ceph_bvec_iter_shorten(&obj_req->bvec_pos, bytes);
2684 ceph_bvec_iter_advance(it, bytes);
2685}
3d7efd18 2686
afb97888
ID		 2687static void count_bvecs(struct ceph_object_extent *ex, u32 bytes, void *arg)
2688{
2689 struct rbd_obj_request *obj_req =
2690 container_of(ex, struct rbd_obj_request, ex);
2691 struct ceph_bvec_iter *it = arg;
058aa991 2692
afb97888
ID		 2693 ceph_bvec_iter_advance_step(it, bytes, ({
2694 obj_req->bvec_count++;
2695 }));
2696}
058aa991 2697
afb97888
ID		 2698static void copy_bvecs(struct ceph_object_extent *ex, u32 bytes, void *arg)
2699{
2700 struct rbd_obj_request *obj_req =
2701 container_of(ex, struct rbd_obj_request, ex);
2702 struct ceph_bvec_iter *it = arg;
3d7efd18 2703
afb97888
ID		 2704 ceph_bvec_iter_advance_step(it, bytes, ({
2705 obj_req->bvec_pos.bvecs[obj_req->bvec_idx++] = bv;
2706 obj_req->bvec_pos.iter.bi_size += bv.bv_len;
2707 }));
3d7efd18
AE		 2708}
2709
5a237819
ID		 2710static int __rbd_img_fill_from_bvecs(struct rbd_img_request *img_req,
2711 struct ceph_file_extent *img_extents,
2712 u32 num_img_extents,
2713 struct ceph_bvec_iter *bvec_pos)
c5b5ef6c 2714{
5a237819
ID		 2715 struct rbd_img_fill_ctx fctx = {
2716 .pos_type = OBJ_REQUEST_BVECS,
2717 .pos = (union rbd_img_fill_iter *)bvec_pos,
2718 .set_pos_fn = set_bvec_pos,
afb97888
ID		 2719 .count_fn = count_bvecs,
2720 .copy_fn = copy_bvecs,
5a237819 2721 };
c5b5ef6c 2722
5a237819
ID		 2723 return rbd_img_fill_request(img_req, img_extents, num_img_extents,
2724 &fctx);
2725}
c5b5ef6c 2726
5a237819
ID		 2727static int rbd_img_fill_from_bvecs(struct rbd_img_request *img_req,
2728 struct ceph_file_extent *img_extents,
2729 u32 num_img_extents,
2730 struct bio_vec *bvecs)
2731{
2732 struct ceph_bvec_iter it = {
2733 .bvecs = bvecs,
2734 .iter = { .bi_size = ceph_file_extents_bytes(img_extents,
2735 num_img_extents) },
2736 };
c5b5ef6c 2737
5a237819
ID		 2738 return __rbd_img_fill_from_bvecs(img_req, img_extents, num_img_extents,
2739 &it);
2740}
c5b5ef6c 2741
0192ce2e 2742static void rbd_img_handle_request_work(struct work_struct *work)
bf0d5f50 2743{
0192ce2e
ID		 2744 struct rbd_img_request *img_req =
2745 container_of(work, struct rbd_img_request, work);
c5b5ef6c 2746
0192ce2e
ID		 2747 rbd_img_handle_request(img_req, img_req->work_result);
2748}
c2e82414 2749
0192ce2e
ID		 2750static void rbd_img_schedule(struct rbd_img_request *img_req, int result)
2751{
2752 INIT_WORK(&img_req->work, rbd_img_handle_request_work);
2753 img_req->work_result = result;
2754 queue_work(rbd_wq, &img_req->work);
c5b5ef6c 2755}
c2e82414 2756
22e8bd51
ID		 2757static bool rbd_obj_may_exist(struct rbd_obj_request *obj_req)
2758{
2759 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
2760
2761 if (rbd_object_map_may_exist(rbd_dev, obj_req->ex.oe_objno)) {
2762 obj_req->flags |= RBD_OBJ_FLAG_MAY_EXIST;
2763 return true;
2764 }
2765
2766 dout("%s %p objno %llu assuming dne\n", __func__, obj_req,
2767 obj_req->ex.oe_objno);
2768 return false;
2769}
2770
85b5e6d1
ID		 2771static int rbd_obj_read_object(struct rbd_obj_request *obj_req)
2772{
a086a1b8
ID		 2773 struct ceph_osd_request *osd_req;
2774 int ret;
2775
2776 osd_req = __rbd_obj_add_osd_request(obj_req, NULL, 1);
2777 if (IS_ERR(osd_req))
2778 return PTR_ERR(osd_req);
2779
2780 osd_req_op_extent_init(osd_req, 0, CEPH_OSD_OP_READ,
2781 obj_req->ex.oe_off, obj_req->ex.oe_len, 0, 0);
2782 rbd_osd_setup_data(osd_req, 0);
2783 rbd_osd_format_read(osd_req);
2784
2785 ret = ceph_osdc_alloc_messages(osd_req, GFP_NOIO);
2786 if (ret)
2787 return ret;
2788
2789 rbd_osd_submit(osd_req);
85b5e6d1 2790 return 0;
c5b5ef6c
AE		 2791}
2792
86bd7998 2793static int rbd_obj_read_from_parent(struct rbd_obj_request *obj_req)
c5b5ef6c 2794{
3da691bf 2795 struct rbd_img_request *img_req = obj_req->img_request;
a52cc685 2796 struct rbd_device *parent = img_req->rbd_dev->parent;
3da691bf 2797 struct rbd_img_request *child_img_req;
c5b5ef6c
AE		 2798 int ret;
2799
59e542c8 2800 child_img_req = kmem_cache_alloc(rbd_img_request_cache, GFP_NOIO);
3da691bf 2801 if (!child_img_req)
710214e3
ID		 2802 return -ENOMEM;
2803
59e542c8 2804 rbd_img_request_init(child_img_req, parent, OBJ_OP_READ);
e93aca0a
ID		 2805 __set_bit(IMG_REQ_CHILD, &child_img_req->flags);
2806 child_img_req->obj_request = obj_req;
a90bb0c1 2807
a52cc685
ID		 2808 down_read(&parent->header_rwsem);
2809 rbd_img_capture_header(child_img_req);
2810 up_read(&parent->header_rwsem);
2811
21ed05a8
ID		 2812 dout("%s child_img_req %p for obj_req %p\n", __func__, child_img_req,
2813 obj_req);
2814
3da691bf 2815 if (!rbd_img_is_write(img_req)) {
ecc633ca 2816 switch (img_req->data_type) {
3da691bf 2817 case OBJ_REQUEST_BIO:
5a237819
ID		 2818 ret = __rbd_img_fill_from_bio(child_img_req,
2819 obj_req->img_extents,
2820 obj_req->num_img_extents,
2821 &obj_req->bio_pos);
3da691bf
ID		 2822 break;
2823 case OBJ_REQUEST_BVECS:
afb97888 2824 case OBJ_REQUEST_OWN_BVECS:
5a237819
ID		 2825 ret = __rbd_img_fill_from_bvecs(child_img_req,
2826 obj_req->img_extents,
2827 obj_req->num_img_extents,
2828 &obj_req->bvec_pos);
3da691bf
ID		 2829 break;
2830 default:
d342a15b 2831 BUG();
3da691bf
ID		 2832 }
2833 } else {
5a237819
ID		 2834 ret = rbd_img_fill_from_bvecs(child_img_req,
2835 obj_req->img_extents,
2836 obj_req->num_img_extents,
2837 obj_req->copyup_bvecs);
3da691bf
ID		 2838 }
2839 if (ret) {
679a97d2 2840 rbd_img_request_destroy(child_img_req);
3da691bf
ID		 2841 return ret;
2842 }
2843
0192ce2e
ID		 2844 /* avoid parent chain recursion */
2845 rbd_img_schedule(child_img_req, 0);
3da691bf
ID		 2846 return 0;
2847}
2848
85b5e6d1 2849static bool rbd_obj_advance_read(struct rbd_obj_request *obj_req, int *result)
3da691bf
ID		 2850{
2851 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
2852 int ret;
2853
22e8bd51 2854again:
a9b67e69 2855 switch (obj_req->read_state) {
85b5e6d1
ID		 2856 case RBD_OBJ_READ_START:
2857 rbd_assert(!*result);
2858
22e8bd51
ID		 2859 if (!rbd_obj_may_exist(obj_req)) {
2860 *result = -ENOENT;
2861 obj_req->read_state = RBD_OBJ_READ_OBJECT;
2862 goto again;
2863 }
2864
85b5e6d1 2865 ret = rbd_obj_read_object(obj_req);
3da691bf 2866 if (ret) {
85b5e6d1 2867 *result = ret;
3da691bf
ID		 2868 return true;
2869 }
85b5e6d1
ID		 2870 obj_req->read_state = RBD_OBJ_READ_OBJECT;
2871 return false;
a9b67e69
ID		 2872 case RBD_OBJ_READ_OBJECT:
2873 if (*result == -ENOENT && rbd_dev->parent_overlap) {
2874 /* reverse map this object extent onto the parent */
2875 ret = rbd_obj_calc_img_extents(obj_req, false);
86bd7998 2876 if (ret) {
54ab3b24 2877 *result = ret;
86bd7998
ID		 2878 return true;
2879 }
a9b67e69
ID		 2880 if (obj_req->num_img_extents) {
2881 ret = rbd_obj_read_from_parent(obj_req);
2882 if (ret) {
2883 *result = ret;
2884 return true;
2885 }
2886 obj_req->read_state = RBD_OBJ_READ_PARENT;
2887 return false;
2888 }
86bd7998 2889 }
710214e3 2890
a9b67e69
ID		 2891 /*
2892 * -ENOENT means a hole in the image -- zero-fill the entire
2893 * length of the request. A short read also implies zero-fill
2894 * to the end of the request.
2895 */
2896 if (*result == -ENOENT) {
2897 rbd_obj_zero_range(obj_req, 0, obj_req->ex.oe_len);
2898 *result = 0;
2899 } else if (*result >= 0) {
2900 if (*result < obj_req->ex.oe_len)
2901 rbd_obj_zero_range(obj_req, *result,
2902 obj_req->ex.oe_len - *result);
2903 else
2904 rbd_assert(*result == obj_req->ex.oe_len);
2905 *result = 0;
2906 }
2907 return true;
2908 case RBD_OBJ_READ_PARENT:
d435c9a7
ID		 2909 /*
2910 * The parent image is read only up to the overlap -- zero-fill
2911 * from the overlap to the end of the request.
2912 */
2913 if (!*result) {
2914 u32 obj_overlap = rbd_obj_img_extents_bytes(obj_req);
2915
2916 if (obj_overlap < obj_req->ex.oe_len)
2917 rbd_obj_zero_range(obj_req, obj_overlap,
2918 obj_req->ex.oe_len - obj_overlap);
2919 }
a9b67e69
ID		 2920 return true;
2921 default:
2922 BUG();
710214e3 2923 }
3da691bf 2924}
c5b5ef6c 2925
22e8bd51
ID		 2926static bool rbd_obj_write_is_noop(struct rbd_obj_request *obj_req)
2927{
2928 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
2929
2930 if (rbd_object_map_may_exist(rbd_dev, obj_req->ex.oe_objno))
2931 obj_req->flags |= RBD_OBJ_FLAG_MAY_EXIST;
2932
2933 if (!(obj_req->flags & RBD_OBJ_FLAG_MAY_EXIST) &&
2934 (obj_req->flags & RBD_OBJ_FLAG_NOOP_FOR_NONEXISTENT)) {
2935 dout("%s %p noop for nonexistent\n", __func__, obj_req);
2936 return true;
2937 }
2938
2939 return false;
2940}
2941
2942/*
2943 * Return:
2944 * 0 - object map update sent
2945 * 1 - object map update isn't needed
2946 * <0 - error
2947 */
2948static int rbd_obj_write_pre_object_map(struct rbd_obj_request *obj_req)
2949{
2950 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
2951 u8 new_state;
2952
2953 if (!(rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP))
2954 return 1;
2955
2956 if (obj_req->flags & RBD_OBJ_FLAG_DELETION)
2957 new_state = OBJECT_PENDING;
2958 else
2959 new_state = OBJECT_EXISTS;
2960
2961 return rbd_object_map_update(obj_req, CEPH_NOSNAP, new_state, NULL);
2962}
2963
85b5e6d1
ID		 2964static int rbd_obj_write_object(struct rbd_obj_request *obj_req)
2965{
a086a1b8
ID		 2966 struct ceph_osd_request *osd_req;
2967 int num_ops = count_write_ops(obj_req);
2968 int which = 0;
2969 int ret;
710214e3 2970
a086a1b8
ID		 2971 if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED)
2972 num_ops++; /* stat */
2973
2974 osd_req = rbd_obj_add_osd_request(obj_req, num_ops);
2975 if (IS_ERR(osd_req))
2976 return PTR_ERR(osd_req);
2977
2978 if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED) {
2979 ret = rbd_osd_setup_stat(osd_req, which++);
2980 if (ret)
2981 return ret;
710214e3 2982 }
c5b5ef6c 2983
a086a1b8
ID		 2984 rbd_osd_setup_write_ops(osd_req, which);
2985 rbd_osd_format_write(osd_req);
2986
2987 ret = ceph_osdc_alloc_messages(osd_req, GFP_NOIO);
2988 if (ret)
2989 return ret;
2990
2991 rbd_osd_submit(osd_req);
85b5e6d1 2992 return 0;
3da691bf 2993}
c5b5ef6c 2994
3da691bf
ID		 2995/*
2996 * copyup_bvecs pages are never highmem pages
2997 */
2998static bool is_zero_bvecs(struct bio_vec *bvecs, u32 bytes)
2999{
3000 struct ceph_bvec_iter it = {
3001 .bvecs = bvecs,
3002 .iter = { .bi_size = bytes },
3003 };
c5b5ef6c 3004
3da691bf 3005 ceph_bvec_iter_advance_step(&it, bytes, ({
cf58b537 3006 if (memchr_inv(bvec_virt(&bv), 0, bv.bv_len))
3da691bf
ID		 3007 return false;
3008 }));
3009 return true;
c5b5ef6c
AE		 3010}
3011
3a482501
ID		 3012#define MODS_ONLY U32_MAX
3013
793333a3
ID		 3014static int rbd_obj_copyup_empty_snapc(struct rbd_obj_request *obj_req,
3015 u32 bytes)
b454e36d 3016{
bcbab1db 3017 struct ceph_osd_request *osd_req;
fe943d50 3018 int ret;
70d045f6 3019
3da691bf 3020 dout("%s obj_req %p bytes %u\n", __func__, obj_req, bytes);
89a59c1c 3021 rbd_assert(bytes > 0 && bytes != MODS_ONLY);
70d045f6 3022
bcbab1db
ID		 3023 osd_req = __rbd_obj_add_osd_request(obj_req, &rbd_empty_snapc, 1);
3024 if (IS_ERR(osd_req))
3025 return PTR_ERR(osd_req);
b454e36d 3026
b5ae8cbc 3027 ret = rbd_osd_setup_copyup(osd_req, 0, bytes);
fe943d50
CX		 3028 if (ret)
3029 return ret;
3030
bcbab1db 3031 rbd_osd_format_write(osd_req);
3da691bf 3032
bcbab1db 3033 ret = ceph_osdc_alloc_messages(osd_req, GFP_NOIO);
89a59c1c
ID		 3034 if (ret)
3035 return ret;
3036
a086a1b8 3037 rbd_osd_submit(osd_req);
89a59c1c
ID		 3038 return 0;
3039}
3040
793333a3
ID		 3041static int rbd_obj_copyup_current_snapc(struct rbd_obj_request *obj_req,
3042 u32 bytes)
b454e36d 3043{
bcbab1db 3044 struct ceph_osd_request *osd_req;
a086a1b8
ID		 3045 int num_ops = count_write_ops(obj_req);
3046 int which = 0;
fe943d50 3047 int ret;
70d045f6 3048
3da691bf 3049 dout("%s obj_req %p bytes %u\n", __func__, obj_req, bytes);
70d045f6 3050
a086a1b8
ID		 3051 if (bytes != MODS_ONLY)
3052 num_ops++; /* copyup */
13488d53 3053
a086a1b8 3054 osd_req = rbd_obj_add_osd_request(obj_req, num_ops);
bcbab1db
ID		 3055 if (IS_ERR(osd_req))
3056 return PTR_ERR(osd_req);
b454e36d 3057
3a482501 3058 if (bytes != MODS_ONLY) {
b5ae8cbc 3059 ret = rbd_osd_setup_copyup(osd_req, which++, bytes);
3a482501
ID		 3060 if (ret)
3061 return ret;
3da691bf 3062 }
3da691bf 3063
a086a1b8
ID		 3064 rbd_osd_setup_write_ops(osd_req, which);
3065 rbd_osd_format_write(osd_req);
70d045f6 3066
bcbab1db 3067 ret = ceph_osdc_alloc_messages(osd_req, GFP_NOIO);
26f887e0
ID		 3068 if (ret)
3069 return ret;
3070
a086a1b8 3071 rbd_osd_submit(osd_req);
3da691bf 3072 return 0;
70d045f6
ID		 3073}
3074
7e07efb1 3075static int setup_copyup_bvecs(struct rbd_obj_request *obj_req, u64 obj_overlap)
70d045f6 3076{
7e07efb1 3077 u32 i;
b454e36d 3078
7e07efb1
ID		 3079 rbd_assert(!obj_req->copyup_bvecs);
3080 obj_req->copyup_bvec_count = calc_pages_for(0, obj_overlap);
3081 obj_req->copyup_bvecs = kcalloc(obj_req->copyup_bvec_count,
3082 sizeof(*obj_req->copyup_bvecs),
3083 GFP_NOIO);
3084 if (!obj_req->copyup_bvecs)
3085 return -ENOMEM;
b454e36d 3086
7e07efb1
ID		 3087 for (i = 0; i < obj_req->copyup_bvec_count; i++) {
3088 unsigned int len = min(obj_overlap, (u64)PAGE_SIZE);
7df2af0b 3089 struct page *page = alloc_page(GFP_NOIO);
7e07efb1 3090
7df2af0b 3091 if (!page)
7e07efb1 3092 return -ENOMEM;
3d7efd18 3093
7df2af0b 3094 bvec_set_page(&obj_req->copyup_bvecs[i], page, len, 0);
7e07efb1
ID		 3095 obj_overlap -= len;
3096 }
b454e36d 3097
7e07efb1
ID		 3098 rbd_assert(!obj_overlap);
3099 return 0;
b454e36d
AE		 3100}
3101
0ad5d953
ID		 3102/*
3103 * The target object doesn't exist. Read the data for the entire
3104 * target object up to the overlap point (if any) from the parent,
3105 * so we can use it for a copyup.
3106 */
793333a3 3107static int rbd_obj_copyup_read_parent(struct rbd_obj_request *obj_req)
bf0d5f50 3108{
3da691bf 3109 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
3da691bf 3110 int ret;
bf0d5f50 3111
86bd7998
ID		 3112 rbd_assert(obj_req->num_img_extents);
3113 prune_extents(obj_req->img_extents, &obj_req->num_img_extents,
3114 rbd_dev->parent_overlap);
3115 if (!obj_req->num_img_extents) {
3da691bf
ID		 3116 /*
3117 * The overlap has become 0 (most likely because the
3a482501
ID		 3118 * image has been flattened). Re-submit the original write
3119 * request -- pass MODS_ONLY since the copyup isn't needed
3120 * anymore.
3da691bf 3121 */
793333a3 3122 return rbd_obj_copyup_current_snapc(obj_req, MODS_ONLY);
bf0d5f50
AE		 3123 }
3124
86bd7998 3125 ret = setup_copyup_bvecs(obj_req, rbd_obj_img_extents_bytes(obj_req));
3da691bf
ID		 3126 if (ret)
3127 return ret;
3128
86bd7998 3129 return rbd_obj_read_from_parent(obj_req);
bf0d5f50 3130}
8b3e1a56 3131
22e8bd51
ID		 3132static void rbd_obj_copyup_object_maps(struct rbd_obj_request *obj_req)
3133{
3134 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
3135 struct ceph_snap_context *snapc = obj_req->img_request->snapc;
3136 u8 new_state;
3137 u32 i;
3138 int ret;
3139
3140 rbd_assert(!obj_req->pending.result && !obj_req->pending.num_pending);
3141
3142 if (!(rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP))
3143 return;
3144
3145 if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ZEROS)
3146 return;
3147
3148 for (i = 0; i < snapc->num_snaps; i++) {
3149 if ((rbd_dev->header.features & RBD_FEATURE_FAST_DIFF) &&
3150 i + 1 < snapc->num_snaps)
3151 new_state = OBJECT_EXISTS_CLEAN;
3152 else
3153 new_state = OBJECT_EXISTS;
3154
3155 ret = rbd_object_map_update(obj_req, snapc->snaps[i],
3156 new_state, NULL);
3157 if (ret < 0) {
3158 obj_req->pending.result = ret;
3159 return;
3160 }
3161
3162 rbd_assert(!ret);
3163 obj_req->pending.num_pending++;
3164 }
3165}
3166
793333a3
ID		 3167static void rbd_obj_copyup_write_object(struct rbd_obj_request *obj_req)
3168{
3169 u32 bytes = rbd_obj_img_extents_bytes(obj_req);
3170 int ret;
3171
3172 rbd_assert(!obj_req->pending.result && !obj_req->pending.num_pending);
3173
3174 /*
3175 * Only send non-zero copyup data to save some I/O and network
3176 * bandwidth -- zero copyup data is equivalent to the object not
3177 * existing.
3178 */
3179 if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ZEROS)
3180 bytes = 0;
3181
3182 if (obj_req->img_request->snapc->num_snaps && bytes > 0) {
3183 /*
3184 * Send a copyup request with an empty snapshot context to
3185 * deep-copyup the object through all existing snapshots.
3186 * A second request with the current snapshot context will be
3187 * sent for the actual modification.
3188 */
3189 ret = rbd_obj_copyup_empty_snapc(obj_req, bytes);
3190 if (ret) {
3191 obj_req->pending.result = ret;
3192 return;
3193 }
3194
3195 obj_req->pending.num_pending++;
3196 bytes = MODS_ONLY;
3197 }
3198
3199 ret = rbd_obj_copyup_current_snapc(obj_req, bytes);
3200 if (ret) {
3201 obj_req->pending.result = ret;
3202 return;
3203 }
3204
3205 obj_req->pending.num_pending++;
3206}
3207
3208static bool rbd_obj_advance_copyup(struct rbd_obj_request *obj_req, int *result)
3209{
22e8bd51 3210 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
793333a3
ID		 3211 int ret;
3212
3213again:
3214 switch (obj_req->copyup_state) {
3215 case RBD_OBJ_COPYUP_START:
3216 rbd_assert(!*result);
3217
3218 ret = rbd_obj_copyup_read_parent(obj_req);
3219 if (ret) {
3220 *result = ret;
3221 return true;
3222 }
3223 if (obj_req->num_img_extents)
3224 obj_req->copyup_state = RBD_OBJ_COPYUP_READ_PARENT;
3225 else
3226 obj_req->copyup_state = RBD_OBJ_COPYUP_WRITE_OBJECT;
3227 return false;
3228 case RBD_OBJ_COPYUP_READ_PARENT:
3229 if (*result)
3230 return true;
3231
3232 if (is_zero_bvecs(obj_req->copyup_bvecs,
3233 rbd_obj_img_extents_bytes(obj_req))) {
3234 dout("%s %p detected zeros\n", __func__, obj_req);
3235 obj_req->flags |= RBD_OBJ_FLAG_COPYUP_ZEROS;
3236 }
3237
22e8bd51
ID		 3238 rbd_obj_copyup_object_maps(obj_req);
3239 if (!obj_req->pending.num_pending) {
3240 *result = obj_req->pending.result;
3241 obj_req->copyup_state = RBD_OBJ_COPYUP_OBJECT_MAPS;
3242 goto again;
3243 }
3244 obj_req->copyup_state = __RBD_OBJ_COPYUP_OBJECT_MAPS;
3245 return false;
3246 case __RBD_OBJ_COPYUP_OBJECT_MAPS:
3247 if (!pending_result_dec(&obj_req->pending, result))
3248 return false;
df561f66 3249 fallthrough;
22e8bd51
ID		 3250 case RBD_OBJ_COPYUP_OBJECT_MAPS:
3251 if (*result) {
3252 rbd_warn(rbd_dev, "snap object map update failed: %d",
3253 *result);
3254 return true;
3255 }
3256
793333a3
ID		 3257 rbd_obj_copyup_write_object(obj_req);
3258 if (!obj_req->pending.num_pending) {
3259 *result = obj_req->pending.result;
3260 obj_req->copyup_state = RBD_OBJ_COPYUP_WRITE_OBJECT;
3261 goto again;
3262 }
3263 obj_req->copyup_state = __RBD_OBJ_COPYUP_WRITE_OBJECT;
3264 return false;
3265 case __RBD_OBJ_COPYUP_WRITE_OBJECT:
3266 if (!pending_result_dec(&obj_req->pending, result))
3267 return false;
df561f66 3268 fallthrough;
793333a3
ID		 3269 case RBD_OBJ_COPYUP_WRITE_OBJECT:
3270 return true;
3271 default:
3272 BUG();
3273 }
3274}
3275
22e8bd51
ID		 3276/*
3277 * Return:
3278 * 0 - object map update sent
3279 * 1 - object map update isn't needed
3280 * <0 - error
3281 */
3282static int rbd_obj_write_post_object_map(struct rbd_obj_request *obj_req)
3283{
3284 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
3285 u8 current_state = OBJECT_PENDING;
3286
3287 if (!(rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP))
3288 return 1;
3289
3290 if (!(obj_req->flags & RBD_OBJ_FLAG_DELETION))
3291 return 1;
3292
3293 return rbd_object_map_update(obj_req, CEPH_NOSNAP, OBJECT_NONEXISTENT,
3294 &current_state);
3295}
3296
85b5e6d1 3297static bool rbd_obj_advance_write(struct rbd_obj_request *obj_req, int *result)
8b3e1a56 3298{
793333a3 3299 struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
3da691bf 3300 int ret;
8b3e1a56 3301
793333a3 3302again:
3da691bf 3303 switch (obj_req->write_state) {
85b5e6d1
ID		 3304 case RBD_OBJ_WRITE_START:
3305 rbd_assert(!*result);
3306
09fe05c5 3307 rbd_obj_set_copyup_enabled(obj_req);
22e8bd51
ID		 3308 if (rbd_obj_write_is_noop(obj_req))
3309 return true;
3310
3311 ret = rbd_obj_write_pre_object_map(obj_req);
3312 if (ret < 0) {
3313 *result = ret;
3314 return true;
3315 }
3316 obj_req->write_state = RBD_OBJ_WRITE_PRE_OBJECT_MAP;
3317 if (ret > 0)
3318 goto again;
3319 return false;
3320 case RBD_OBJ_WRITE_PRE_OBJECT_MAP:
3321 if (*result) {
3322 rbd_warn(rbd_dev, "pre object map update failed: %d",
3323 *result);
3324 return true;
3325 }
85b5e6d1
ID		 3326 ret = rbd_obj_write_object(obj_req);
3327 if (ret) {
3328 *result = ret;
3329 return true;
3330 }
3331 obj_req->write_state = RBD_OBJ_WRITE_OBJECT;
3332 return false;
0ad5d953 3333 case RBD_OBJ_WRITE_OBJECT:
54ab3b24 3334 if (*result == -ENOENT) {
0ad5d953 3335 if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED) {
793333a3
ID		 3336 *result = 0;
3337 obj_req->copyup_state = RBD_OBJ_COPYUP_START;
3338 obj_req->write_state = __RBD_OBJ_WRITE_COPYUP;
3339 goto again;
0ad5d953 3340 }
3da691bf 3341 /*
0ad5d953
ID		 3342 * On a non-existent object:
3343 * delete - -ENOENT, truncate/zero - 0
3da691bf 3344 */
0ad5d953
ID		 3345 if (obj_req->flags & RBD_OBJ_FLAG_DELETION)
3346 *result = 0;
3da691bf 3347 }
a9b67e69 3348 if (*result)
3a482501 3349 return true;
8b3e1a56 3350
793333a3
ID		 3351 obj_req->write_state = RBD_OBJ_WRITE_COPYUP;
3352 goto again;
3353 case __RBD_OBJ_WRITE_COPYUP:
3354 if (!rbd_obj_advance_copyup(obj_req, result))
3355 return false;
df561f66 3356 fallthrough;
793333a3 3357 case RBD_OBJ_WRITE_COPYUP:
22e8bd51 3358 if (*result) {
793333a3 3359 rbd_warn(rbd_dev, "copyup failed: %d", *result);
22e8bd51
ID		 3360 return true;
3361 }
3362 ret = rbd_obj_write_post_object_map(obj_req);
3363 if (ret < 0) {
3364 *result = ret;
3365 return true;
3366 }
3367 obj_req->write_state = RBD_OBJ_WRITE_POST_OBJECT_MAP;
3368 if (ret > 0)
3369 goto again;
3370 return false;
3371 case RBD_OBJ_WRITE_POST_OBJECT_MAP:
3372 if (*result)
3373 rbd_warn(rbd_dev, "post object map update failed: %d",
3374 *result);
793333a3 3375 return true;
3da691bf 3376 default:
c6244b3b 3377 BUG();
3da691bf
ID		 3378 }
3379}
02c74fba 3380
3da691bf 3381/*
0ad5d953 3382 * Return true if @obj_req is completed.
3da691bf 3383 */
54ab3b24
ID		 3384static bool __rbd_obj_handle_request(struct rbd_obj_request *obj_req,
3385 int *result)
3da691bf 3386{
0ad5d953 3387 struct rbd_img_request *img_req = obj_req->img_request;
0192ce2e 3388 struct rbd_device *rbd_dev = img_req->rbd_dev;
0ad5d953
ID		 3389 bool done;
3390
85b5e6d1 3391 mutex_lock(&obj_req->state_mutex);
0ad5d953 3392 if (!rbd_img_is_write(img_req))
85b5e6d1 3393 done = rbd_obj_advance_read(obj_req, result);
0ad5d953 3394 else
85b5e6d1
ID		 3395 done = rbd_obj_advance_write(obj_req, result);
3396 mutex_unlock(&obj_req->state_mutex);
0ad5d953 3397
0192ce2e
ID		 3398 if (done && *result) {
3399 rbd_assert(*result < 0);
3400 rbd_warn(rbd_dev, "%s at objno %llu %llu~%llu result %d",
3401 obj_op_name(img_req->op_type), obj_req->ex.oe_objno,
3402 obj_req->ex.oe_off, obj_req->ex.oe_len, *result);
3403 }
0ad5d953 3404 return done;
3da691bf 3405}
02c74fba 3406
0192ce2e
ID		 3407/*
3408 * This is open-coded in rbd_img_handle_request() to avoid parent chain
3409 * recursion.
3410 */
3411static void rbd_obj_handle_request(struct rbd_obj_request *obj_req, int result)
3412{
3413 if (__rbd_obj_handle_request(obj_req, &result))
3414 rbd_img_handle_request(obj_req->img_request, result);
3415}
3416
e1fddc8f
ID		 3417static bool need_exclusive_lock(struct rbd_img_request *img_req)
3418{
3419 struct rbd_device *rbd_dev = img_req->rbd_dev;
3420
3421 if (!(rbd_dev->header.features & RBD_FEATURE_EXCLUSIVE_LOCK))
3422 return false;
3423
3fe69921 3424 if (rbd_is_ro(rbd_dev))
e1fddc8f
ID		 3425 return false;
3426
3427 rbd_assert(!test_bit(IMG_REQ_CHILD, &img_req->flags));
22e8bd51
ID		 3428 if (rbd_dev->opts->lock_on_read ||
3429 (rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP))
e1fddc8f
ID		 3430 return true;
3431
3432 return rbd_img_is_write(img_req);
3433}
3434
637cd060 3435static bool rbd_lock_add_request(struct rbd_img_request *img_req)
e1fddc8f
ID		 3436{
3437 struct rbd_device *rbd_dev = img_req->rbd_dev;
637cd060 3438 bool locked;
e1fddc8f
ID		 3439
3440 lockdep_assert_held(&rbd_dev->lock_rwsem);
637cd060 3441 locked = rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED;
e1fddc8f
ID		 3442 spin_lock(&rbd_dev->lock_lists_lock);
3443 rbd_assert(list_empty(&img_req->lock_item));
637cd060
ID		 3444 if (!locked)
3445 list_add_tail(&img_req->lock_item, &rbd_dev->acquiring_list);
3446 else
3447 list_add_tail(&img_req->lock_item, &rbd_dev->running_list);
e1fddc8f 3448 spin_unlock(&rbd_dev->lock_lists_lock);
637cd060 3449 return locked;
e1fddc8f
ID		 3450}
3451
3452static void rbd_lock_del_request(struct rbd_img_request *img_req)
3453{
3454 struct rbd_device *rbd_dev = img_req->rbd_dev;
3455 bool need_wakeup;
3456
3457 lockdep_assert_held(&rbd_dev->lock_rwsem);
3458 spin_lock(&rbd_dev->lock_lists_lock);
3459 rbd_assert(!list_empty(&img_req->lock_item));
3460 list_del_init(&img_req->lock_item);
3461 need_wakeup = (rbd_dev->lock_state == RBD_LOCK_STATE_RELEASING &&
3462 list_empty(&rbd_dev->running_list));
3463 spin_unlock(&rbd_dev->lock_lists_lock);
3464 if (need_wakeup)
3465 complete(&rbd_dev->releasing_wait);
3466}
3467
637cd060
ID		 3468static int rbd_img_exclusive_lock(struct rbd_img_request *img_req)
3469{
3470 struct rbd_device *rbd_dev = img_req->rbd_dev;
3471
3472 if (!need_exclusive_lock(img_req))
3473 return 1;
3474
3475 if (rbd_lock_add_request(img_req))
3476 return 1;
3477
3478 if (rbd_dev->opts->exclusive) {
3479 WARN_ON(1); /* lock got released? */
3480 return -EROFS;
3481 }
3482
3483 /*
3484 * Note the use of mod_delayed_work() in rbd_acquire_lock()
3485 * and cancel_delayed_work() in wake_lock_waiters().
3486 */
3487 dout("%s rbd_dev %p queueing lock_dwork\n", __func__, rbd_dev);
3488 queue_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0);
3489 return 0;
3490}
3491
0192ce2e 3492static void rbd_img_object_requests(struct rbd_img_request *img_req)
7114edac 3493{
870611e4 3494 struct rbd_device *rbd_dev = img_req->rbd_dev;
0192ce2e 3495 struct rbd_obj_request *obj_req;
7114edac 3496
0192ce2e 3497 rbd_assert(!img_req->pending.result && !img_req->pending.num_pending);
870611e4
ID		 3498 rbd_assert(!need_exclusive_lock(img_req) ||
3499 __rbd_is_lock_owner(rbd_dev));
3500
3501 if (rbd_img_is_write(img_req)) {
3502 rbd_assert(!img_req->snapc);
3503 down_read(&rbd_dev->header_rwsem);
3504 img_req->snapc = ceph_get_snap_context(rbd_dev->header.snapc);
3505 up_read(&rbd_dev->header_rwsem);
3506 }
0192ce2e
ID		 3507
3508 for_each_obj_request(img_req, obj_req) {
3509 int result = 0;
a9e8ba2c 3510
0192ce2e
ID		 3511 if (__rbd_obj_handle_request(obj_req, &result)) {
3512 if (result) {
3513 img_req->pending.result = result;
3514 return;
3515 }
3516 } else {
3517 img_req->pending.num_pending++;
3518 }
3519 }
8b3e1a56
AE		 3520}
3521
0192ce2e 3522static bool rbd_img_advance(struct rbd_img_request *img_req, int *result)
8b3e1a56 3523{
3da691bf 3524 int ret;
8b3e1a56 3525
0192ce2e
ID		 3526again:
3527 switch (img_req->state) {
3528 case RBD_IMG_START:
3529 rbd_assert(!*result);
8b3e1a56 3530
637cd060
ID		 3531 ret = rbd_img_exclusive_lock(img_req);
3532 if (ret < 0) {
3533 *result = ret;
3da691bf
ID		 3534 return true;
3535 }
637cd060
ID		 3536 img_req->state = RBD_IMG_EXCLUSIVE_LOCK;
3537 if (ret > 0)
3538 goto again;
3da691bf 3539 return false;
637cd060
ID		 3540 case RBD_IMG_EXCLUSIVE_LOCK:
3541 if (*result)
89a59c1c
ID		 3542 return true;
3543
0192ce2e
ID		 3544 rbd_img_object_requests(img_req);
3545 if (!img_req->pending.num_pending) {
3546 *result = img_req->pending.result;
3547 img_req->state = RBD_IMG_OBJECT_REQUESTS;
3548 goto again;
3da691bf 3549 }
0192ce2e 3550 img_req->state = __RBD_IMG_OBJECT_REQUESTS;
3da691bf 3551 return false;
0192ce2e
ID		 3552 case __RBD_IMG_OBJECT_REQUESTS:
3553 if (!pending_result_dec(&img_req->pending, result))
3554 return false;
df561f66 3555 fallthrough;
0192ce2e
ID		 3556 case RBD_IMG_OBJECT_REQUESTS:
3557 return true;
3da691bf 3558 default:
c6244b3b 3559 BUG();
3da691bf
ID		 3560 }
3561}
02c74fba 3562
3da691bf 3563/*
0192ce2e 3564 * Return true if @img_req is completed.
3da691bf 3565 */
0192ce2e
ID		 3566static bool __rbd_img_handle_request(struct rbd_img_request *img_req,
3567 int *result)
7114edac 3568{
0192ce2e
ID		 3569 struct rbd_device *rbd_dev = img_req->rbd_dev;
3570 bool done;
7114edac 3571
e1fddc8f
ID		 3572 if (need_exclusive_lock(img_req)) {
3573 down_read(&rbd_dev->lock_rwsem);
3574 mutex_lock(&img_req->state_mutex);
3575 done = rbd_img_advance(img_req, result);
3576 if (done)
3577 rbd_lock_del_request(img_req);
3578 mutex_unlock(&img_req->state_mutex);
3579 up_read(&rbd_dev->lock_rwsem);
3580 } else {
3581 mutex_lock(&img_req->state_mutex);
3582 done = rbd_img_advance(img_req, result);
3583 mutex_unlock(&img_req->state_mutex);
02c74fba 3584 }
a9e8ba2c 3585
0192ce2e
ID		 3586 if (done && *result) {
3587 rbd_assert(*result < 0);
3588 rbd_warn(rbd_dev, "%s%s result %d",
3589 test_bit(IMG_REQ_CHILD, &img_req->flags) ? "child " : "",
3590 obj_op_name(img_req->op_type), *result);
7114edac 3591 }
0192ce2e 3592 return done;
7114edac 3593}
a9e8ba2c 3594
0192ce2e 3595static void rbd_img_handle_request(struct rbd_img_request *img_req, int result)
3da691bf 3596{
7114edac 3597again:
0192ce2e 3598 if (!__rbd_img_handle_request(img_req, &result))
7114edac 3599 return;
8b3e1a56 3600
7114edac 3601 if (test_bit(IMG_REQ_CHILD, &img_req->flags)) {
0192ce2e
ID		 3602 struct rbd_obj_request *obj_req = img_req->obj_request;
3603
679a97d2 3604 rbd_img_request_destroy(img_req);
0192ce2e
ID		 3605 if (__rbd_obj_handle_request(obj_req, &result)) {
3606 img_req = obj_req->img_request;
3607 goto again;
3608 }
3609 } else {
59e542c8 3610 struct request *rq = blk_mq_rq_from_pdu(img_req);
0192ce2e 3611
679a97d2 3612 rbd_img_request_destroy(img_req);
0192ce2e 3613 blk_mq_end_request(rq, errno_to_blk_status(result));
7114edac 3614 }
8b3e1a56 3615}
bf0d5f50 3616
ed95b21a 3617static const struct rbd_client_id rbd_empty_cid;
b8d70035 3618
ed95b21a
ID		 3619static bool rbd_cid_equal(const struct rbd_client_id *lhs,
3620 const struct rbd_client_id *rhs)
3621{
3622 return lhs->gid == rhs->gid && lhs->handle == rhs->handle;
3623}
3624
3625static struct rbd_client_id rbd_get_cid(struct rbd_device *rbd_dev)
3626{
3627 struct rbd_client_id cid;
3628
3629 mutex_lock(&rbd_dev->watch_mutex);
3630 cid.gid = ceph_client_gid(rbd_dev->rbd_client->client);
3631 cid.handle = rbd_dev->watch_cookie;
3632 mutex_unlock(&rbd_dev->watch_mutex);
3633 return cid;
3634}
3635
3636/*
3637 * lock_rwsem must be held for write
3638 */
3639static void rbd_set_owner_cid(struct rbd_device *rbd_dev,
3640 const struct rbd_client_id *cid)
3641{
3642 dout("%s rbd_dev %p %llu-%llu -> %llu-%llu\n", __func__, rbd_dev,
3643 rbd_dev->owner_cid.gid, rbd_dev->owner_cid.handle,
3644 cid->gid, cid->handle);
3645 rbd_dev->owner_cid = *cid; /* struct */
3646}
3647
3648static void format_lock_cookie(struct rbd_device *rbd_dev, char *buf)
3649{
3650 mutex_lock(&rbd_dev->watch_mutex);
3651 sprintf(buf, "%s %llu", RBD_LOCK_COOKIE_PREFIX, rbd_dev->watch_cookie);
3652 mutex_unlock(&rbd_dev->watch_mutex);
3653}
3654
edd8ca80
FM		 3655static void __rbd_lock(struct rbd_device *rbd_dev, const char *cookie)
3656{
3657 struct rbd_client_id cid = rbd_get_cid(rbd_dev);
3658
a2b1da09 3659 rbd_dev->lock_state = RBD_LOCK_STATE_LOCKED;
edd8ca80
FM		 3660 strcpy(rbd_dev->lock_cookie, cookie);
3661 rbd_set_owner_cid(rbd_dev, &cid);
3662 queue_work(rbd_dev->task_wq, &rbd_dev->acquired_lock_work);
3663}
3664
ed95b21a
ID		 3665/*
3666 * lock_rwsem must be held for write
3667 */
3668static int rbd_lock(struct rbd_device *rbd_dev)
b8d70035 3669{
922dab61 3670 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
ed95b21a 3671 char cookie[32];
e627db08 3672 int ret;
b8d70035 3673
cbbfb0ff
ID		 3674 WARN_ON(__rbd_is_lock_owner(rbd_dev) ||
3675 rbd_dev->lock_cookie[0] != '\0');
52bb1f9b 3676
ed95b21a
ID		 3677 format_lock_cookie(rbd_dev, cookie);
3678 ret = ceph_cls_lock(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc,
3679 RBD_LOCK_NAME, CEPH_CLS_LOCK_EXCLUSIVE, cookie,
3680 RBD_LOCK_TAG, "", 0);
9d01e07f 3681 if (ret && ret != -EEXIST)
ed95b21a 3682 return ret;
b8d70035 3683
edd8ca80 3684 __rbd_lock(rbd_dev, cookie);
ed95b21a 3685 return 0;
b8d70035
AE		 3686}
3687
ed95b21a
ID		 3688/*
3689 * lock_rwsem must be held for write
3690 */
bbead745 3691static void rbd_unlock(struct rbd_device *rbd_dev)
bb040aa0 3692{
922dab61 3693 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
bb040aa0
ID		 3694 int ret;
3695
cbbfb0ff
ID		 3696 WARN_ON(!__rbd_is_lock_owner(rbd_dev) ||
3697 rbd_dev->lock_cookie[0] == '\0');
bb040aa0 3698
ed95b21a 3699 ret = ceph_cls_unlock(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc,
cbbfb0ff 3700 RBD_LOCK_NAME, rbd_dev->lock_cookie);
bbead745 3701 if (ret && ret != -ENOENT)
637cd060 3702 rbd_warn(rbd_dev, "failed to unlock header: %d", ret);
bb040aa0 3703
bbead745
ID		 3704 /* treat errors as the image is unlocked */
3705 rbd_dev->lock_state = RBD_LOCK_STATE_UNLOCKED;
cbbfb0ff 3706 rbd_dev->lock_cookie[0] = '\0';
ed95b21a
ID		 3707 rbd_set_owner_cid(rbd_dev, &rbd_empty_cid);
3708 queue_work(rbd_dev->task_wq, &rbd_dev->released_lock_work);
bb040aa0
ID		 3709}
3710
ed95b21a
ID		 3711static int __rbd_notify_op_lock(struct rbd_device *rbd_dev,
3712 enum rbd_notify_op notify_op,
3713 struct page ***preply_pages,
3714 size_t *preply_len)
9969ebc5
AE		 3715{
3716 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
ed95b21a 3717 struct rbd_client_id cid = rbd_get_cid(rbd_dev);
08a79102
KS		 3718 char buf[4 + 8 + 8 + CEPH_ENCODING_START_BLK_LEN];
3719 int buf_size = sizeof(buf);
ed95b21a 3720 void *p = buf;
9969ebc5 3721
ed95b21a 3722 dout("%s rbd_dev %p notify_op %d\n", __func__, rbd_dev, notify_op);
9969ebc5 3723
ed95b21a
ID		 3724 /* encode *LockPayload NotifyMessage (op + ClientId) */
3725 ceph_start_encoding(&p, 2, 1, buf_size - CEPH_ENCODING_START_BLK_LEN);
3726 ceph_encode_32(&p, notify_op);
3727 ceph_encode_64(&p, cid.gid);
3728 ceph_encode_64(&p, cid.handle);
8eb87565 3729
ed95b21a
ID		 3730 return ceph_osdc_notify(osdc, &rbd_dev->header_oid,
3731 &rbd_dev->header_oloc, buf, buf_size,
3732 RBD_NOTIFY_TIMEOUT, preply_pages, preply_len);
b30a01f2
ID		 3733}
3734
ed95b21a
ID		 3735static void rbd_notify_op_lock(struct rbd_device *rbd_dev,
3736 enum rbd_notify_op notify_op)
b30a01f2 3737{
8ae0299a 3738 __rbd_notify_op_lock(rbd_dev, notify_op, NULL, NULL);
ed95b21a 3739}
b30a01f2 3740
ed95b21a
ID		 3741static void rbd_notify_acquired_lock(struct work_struct *work)
3742{
3743 struct rbd_device *rbd_dev = container_of(work, struct rbd_device,
3744 acquired_lock_work);
76756a51 3745
ed95b21a 3746 rbd_notify_op_lock(rbd_dev, RBD_NOTIFY_OP_ACQUIRED_LOCK);
c525f036
ID		 3747}
3748
ed95b21a 3749static void rbd_notify_released_lock(struct work_struct *work)
c525f036 3750{
ed95b21a
ID		 3751 struct rbd_device *rbd_dev = container_of(work, struct rbd_device,
3752 released_lock_work);
811c6688 3753
ed95b21a 3754 rbd_notify_op_lock(rbd_dev, RBD_NOTIFY_OP_RELEASED_LOCK);
fca27065
ID		 3755}
3756
ed95b21a 3757static int rbd_request_lock(struct rbd_device *rbd_dev)
36be9a76 3758{
ed95b21a
ID		 3759 struct page **reply_pages;
3760 size_t reply_len;
3761 bool lock_owner_responded = false;
36be9a76
AE		 3762 int ret;
3763
ed95b21a 3764 dout("%s rbd_dev %p\n", __func__, rbd_dev);
36be9a76 3765
ed95b21a
ID		 3766 ret = __rbd_notify_op_lock(rbd_dev, RBD_NOTIFY_OP_REQUEST_LOCK,
3767 &reply_pages, &reply_len);
3768 if (ret && ret != -ETIMEDOUT) {
3769 rbd_warn(rbd_dev, "failed to request lock: %d", ret);
36be9a76 3770 goto out;
ed95b21a 3771 }
36be9a76 3772
ed95b21a
ID		 3773 if (reply_len > 0 && reply_len <= PAGE_SIZE) {
3774 void *p = page_address(reply_pages[0]);
3775 void *const end = p + reply_len;
3776 u32 n;
36be9a76 3777
ed95b21a
ID		 3778 ceph_decode_32_safe(&p, end, n, e_inval); /* num_acks */
3779 while (n--) {
3780 u8 struct_v;
3781 u32 len;
36be9a76 3782
ed95b21a
ID		 3783 ceph_decode_need(&p, end, 8 + 8, e_inval);
3784 p += 8 + 8; /* skip gid and cookie */
04017e29 3785
ed95b21a
ID		 3786 ceph_decode_32_safe(&p, end, len, e_inval);
3787 if (!len)
3788 continue;
3789
3790 if (lock_owner_responded) {
3791 rbd_warn(rbd_dev,
3792 "duplicate lock owners detected");
3793 ret = -EIO;
3794 goto out;
3795 }
3796
3797 lock_owner_responded = true;
3798 ret = ceph_start_decoding(&p, end, 1, "ResponseMessage",
3799 &struct_v, &len);
3800 if (ret) {
3801 rbd_warn(rbd_dev,
3802 "failed to decode ResponseMessage: %d",
3803 ret);
3804 goto e_inval;
3805 }
3806
3807 ret = ceph_decode_32(&p);
3808 }
3809 }
3810
3811 if (!lock_owner_responded) {
3812 rbd_warn(rbd_dev, "no lock owners detected");
3813 ret = -ETIMEDOUT;
3814 }
3815
3816out:
3817 ceph_release_page_vector(reply_pages, calc_pages_for(0, reply_len));
3818 return ret;
3819
3820e_inval:
3821 ret = -EINVAL;
3822 goto out;
3823}
3824
637cd060
ID		 3825/*
3826 * Either image request state machine(s) or rbd_add_acquire_lock()
3827 * (i.e. "rbd map").
3828 */
3829static void wake_lock_waiters(struct rbd_device *rbd_dev, int result)
ed95b21a 3830{
637cd060
ID		 3831 struct rbd_img_request *img_req;
3832
3833 dout("%s rbd_dev %p result %d\n", __func__, rbd_dev, result);
d9b9c893 3834 lockdep_assert_held_write(&rbd_dev->lock_rwsem);
ed95b21a
ID		 3835
3836 cancel_delayed_work(&rbd_dev->lock_dwork);
637cd060
ID		 3837 if (!completion_done(&rbd_dev->acquire_wait)) {
3838 rbd_assert(list_empty(&rbd_dev->acquiring_list) &&
3839 list_empty(&rbd_dev->running_list));
3840 rbd_dev->acquire_err = result;
3841 complete_all(&rbd_dev->acquire_wait);
3842 return;
3843 }
3844
3845 list_for_each_entry(img_req, &rbd_dev->acquiring_list, lock_item) {
3846 mutex_lock(&img_req->state_mutex);
3847 rbd_assert(img_req->state == RBD_IMG_EXCLUSIVE_LOCK);
3848 rbd_img_schedule(img_req, result);
3849 mutex_unlock(&img_req->state_mutex);
3850 }
3851
3852 list_splice_tail_init(&rbd_dev->acquiring_list, &rbd_dev->running_list);
ed95b21a
ID		 3853}
3854
58815900
ID		 3855static bool locker_equal(const struct ceph_locker *lhs,
3856 const struct ceph_locker *rhs)
3857{
3858 return lhs->id.name.type == rhs->id.name.type &&
3859 lhs->id.name.num == rhs->id.name.num &&
3860 !strcmp(lhs->id.cookie, rhs->id.cookie) &&
3861 ceph_addr_equal_no_type(&lhs->info.addr, &rhs->info.addr);
3862}
3863
f38cb9d9
ID		 3864static void free_locker(struct ceph_locker *locker)
3865{
3866 if (locker)
3867 ceph_free_lockers(locker, 1);
3868}
3869
3870static struct ceph_locker *get_lock_owner_info(struct rbd_device *rbd_dev)
ed95b21a
ID		 3871{
3872 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
f38cb9d9
ID		 3873 struct ceph_locker *lockers;
3874 u32 num_lockers;
ed95b21a
ID		 3875 u8 lock_type;
3876 char *lock_tag;
8ff2c64c 3877 u64 handle;
ed95b21a
ID		 3878 int ret;
3879
ed95b21a
ID		 3880 ret = ceph_cls_lock_info(osdc, &rbd_dev->header_oid,
3881 &rbd_dev->header_oloc, RBD_LOCK_NAME,
f38cb9d9
ID		 3882 &lock_type, &lock_tag, &lockers, &num_lockers);
3883 if (ret) {
9d01e07f 3884 rbd_warn(rbd_dev, "failed to get header lockers: %d", ret);
f38cb9d9
ID		 3885 return ERR_PTR(ret);
3886 }
ed95b21a 3887
f38cb9d9 3888 if (num_lockers == 0) {
ed95b21a 3889 dout("%s rbd_dev %p no lockers detected\n", __func__, rbd_dev);
f38cb9d9 3890 lockers = NULL;
ed95b21a
ID		 3891 goto out;
3892 }
3893
3894 if (strcmp(lock_tag, RBD_LOCK_TAG)) {
3895 rbd_warn(rbd_dev, "locked by external mechanism, tag %s",
3896 lock_tag);
f38cb9d9 3897 goto err_busy;
ed95b21a
ID		 3898 }
3899
8ff2c64c
ID		 3900 if (lock_type != CEPH_CLS_LOCK_EXCLUSIVE) {
3901 rbd_warn(rbd_dev, "incompatible lock type detected");
f38cb9d9 3902 goto err_busy;
ed95b21a
ID		 3903 }
3904
f38cb9d9 3905 WARN_ON(num_lockers != 1);
8ff2c64c
ID		 3906 ret = sscanf(lockers[0].id.cookie, RBD_LOCK_COOKIE_PREFIX " %llu",
3907 &handle);
3908 if (ret != 1) {
ed95b21a 3909 rbd_warn(rbd_dev, "locked by external mechanism, cookie %s",
f38cb9d9
ID		 3910 lockers[0].id.cookie);
3911 goto err_busy;
ed95b21a 3912 }
8ff2c64c
ID		 3913 if (ceph_addr_is_blank(&lockers[0].info.addr)) {
3914 rbd_warn(rbd_dev, "locker has a blank address");
3915 goto err_busy;
3916 }
3917
3918 dout("%s rbd_dev %p got locker %s%llu@%pISpc/%u handle %llu\n",
3919 __func__, rbd_dev, ENTITY_NAME(lockers[0].id.name),
3920 &lockers[0].info.addr.in_addr,
3921 le32_to_cpu(lockers[0].info.addr.nonce), handle);
ed95b21a
ID		 3922
3923out:
3924 kfree(lock_tag);
f38cb9d9
ID		 3925 return lockers;
3926
3927err_busy:
3928 kfree(lock_tag);
3929 ceph_free_lockers(lockers, num_lockers);
3930 return ERR_PTR(-EBUSY);
ed95b21a
ID		 3931}
3932
3933static int find_watcher(struct rbd_device *rbd_dev,
3934 const struct ceph_locker *locker)
3935{
3936 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
3937 struct ceph_watch_item *watchers;
3938 u32 num_watchers;
3939 u64 cookie;
3940 int i;
3941 int ret;
3942
3943 ret = ceph_osdc_list_watchers(osdc, &rbd_dev->header_oid,
3944 &rbd_dev->header_oloc, &watchers,
3945 &num_watchers);
9d01e07f
ID		 3946 if (ret) {
3947 rbd_warn(rbd_dev, "failed to get watchers: %d", ret);
ed95b21a 3948 return ret;
9d01e07f 3949 }
ed95b21a
ID		 3950
3951 sscanf(locker->id.cookie, RBD_LOCK_COOKIE_PREFIX " %llu", &cookie);
3952 for (i = 0; i < num_watchers; i++) {
313771e8
ID		 3953 /*
3954 * Ignore addr->type while comparing. This mimics
3955 * entity_addr_t::get_legacy_str() + strcmp().
3956 */
3957 if (ceph_addr_equal_no_type(&watchers[i].addr,
3958 &locker->info.addr) &&
ed95b21a
ID		 3959 watchers[i].cookie == cookie) {
3960 struct rbd_client_id cid = {
3961 .gid = le64_to_cpu(watchers[i].name.num),
3962 .handle = cookie,
3963 };
3964
3965 dout("%s rbd_dev %p found cid %llu-%llu\n", __func__,
3966 rbd_dev, cid.gid, cid.handle);
3967 rbd_set_owner_cid(rbd_dev, &cid);
3968 ret = 1;
3969 goto out;
3970 }
3971 }
3972
3973 dout("%s rbd_dev %p no watchers\n", __func__, rbd_dev);
3974 ret = 0;
3975out:
3976 kfree(watchers);
3977 return ret;
3978}
3979
3980/*
3981 * lock_rwsem must be held for write
3982 */
3983static int rbd_try_lock(struct rbd_device *rbd_dev)
3984{
3985 struct ceph_client *client = rbd_dev->rbd_client->client;
58815900 3986 struct ceph_locker *locker, *refreshed_locker;
ed95b21a
ID		 3987 int ret;
3988
3989 for (;;) {
58815900 3990 locker = refreshed_locker = NULL;
f38cb9d9 3991
ed95b21a 3992 ret = rbd_lock(rbd_dev);
9d01e07f
ID		 3993 if (!ret)
3994 goto out;
3995 if (ret != -EBUSY) {
3996 rbd_warn(rbd_dev, "failed to lock header: %d", ret);
f38cb9d9 3997 goto out;
9d01e07f 3998 }
ed95b21a
ID		 3999
4000 /* determine if the current lock holder is still alive */
f38cb9d9
ID		 4001 locker = get_lock_owner_info(rbd_dev);
4002 if (IS_ERR(locker)) {
4003 ret = PTR_ERR(locker);
4004 locker = NULL;
4005 goto out;
4006 }
4007 if (!locker)
ed95b21a
ID		 4008 goto again;
4009
f38cb9d9 4010 ret = find_watcher(rbd_dev, locker);
637cd060
ID		 4011 if (ret)
4012 goto out; /* request lock or error */
ed95b21a 4013
58815900
ID		 4014 refreshed_locker = get_lock_owner_info(rbd_dev);
4015 if (IS_ERR(refreshed_locker)) {
4016 ret = PTR_ERR(refreshed_locker);
4017 refreshed_locker = NULL;
4018 goto out;
4019 }
4020 if (!refreshed_locker ||
4021 !locker_equal(locker, refreshed_locker))
4022 goto again;
4023
22e8bd51 4024 rbd_warn(rbd_dev, "breaking header lock owned by %s%llu",
f38cb9d9 4025 ENTITY_NAME(locker->id.name));
ed95b21a 4026
0b98acd6 4027 ret = ceph_monc_blocklist_add(&client->monc,
f38cb9d9 4028 &locker->info.addr);
ed95b21a 4029 if (ret) {
f38cb9d9
ID		 4030 rbd_warn(rbd_dev, "failed to blocklist %s%llu: %d",
4031 ENTITY_NAME(locker->id.name), ret);
ed95b21a
ID		 4032 goto out;
4033 }
4034
4035 ret = ceph_cls_break_lock(&client->osdc, &rbd_dev->header_oid,
4036 &rbd_dev->header_oloc, RBD_LOCK_NAME,
f38cb9d9
ID		 4037 locker->id.cookie, &locker->id.name);
4038 if (ret && ret != -ENOENT) {
4039 rbd_warn(rbd_dev, "failed to break header lock: %d",
4040 ret);
ed95b21a 4041 goto out;
f38cb9d9 4042 }
ed95b21a
ID		 4043
4044again:
58815900 4045 free_locker(refreshed_locker);
f38cb9d9 4046 free_locker(locker);
ed95b21a
ID		 4047 }
4048
4049out:
58815900 4050 free_locker(refreshed_locker);
f38cb9d9 4051 free_locker(locker);
ed95b21a
ID		 4052 return ret;
4053}
4054
22e8bd51
ID		 4055static int rbd_post_acquire_action(struct rbd_device *rbd_dev)
4056{
4057 int ret;
4058
870611e4
ID		 4059 ret = rbd_dev_refresh(rbd_dev);
4060 if (ret)
4061 return ret;
4062
22e8bd51
ID		 4063 if (rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP) {
4064 ret = rbd_object_map_open(rbd_dev);
4065 if (ret)
4066 return ret;
4067 }
4068
4069 return 0;
4070}
4071
ed95b21a 4072/*
637cd060
ID		 4073 * Return:
4074 * 0 - lock acquired
4075 * 1 - caller should call rbd_request_lock()
4076 * <0 - error
ed95b21a 4077 */
637cd060 4078static int rbd_try_acquire_lock(struct rbd_device *rbd_dev)
ed95b21a 4079{
637cd060 4080 int ret;
ed95b21a
ID		 4081
4082 down_read(&rbd_dev->lock_rwsem);
4083 dout("%s rbd_dev %p read lock_state %d\n", __func__, rbd_dev,
4084 rbd_dev->lock_state);
4085 if (__rbd_is_lock_owner(rbd_dev)) {
ed95b21a 4086 up_read(&rbd_dev->lock_rwsem);
637cd060 4087 return 0;
ed95b21a
ID		 4088 }
4089
4090 up_read(&rbd_dev->lock_rwsem);
4091 down_write(&rbd_dev->lock_rwsem);
4092 dout("%s rbd_dev %p write lock_state %d\n", __func__, rbd_dev,
4093 rbd_dev->lock_state);
637cd060
ID		 4094 if (__rbd_is_lock_owner(rbd_dev)) {
4095 up_write(&rbd_dev->lock_rwsem);
4096 return 0;
ed95b21a
ID		 4097 }
4098
637cd060
ID		 4099 ret = rbd_try_lock(rbd_dev);
4100 if (ret < 0) {
9d01e07f
ID		 4101 rbd_warn(rbd_dev, "failed to acquire lock: %d", ret);
4102 goto out;
637cd060
ID		 4103 }
4104 if (ret > 0) {
4105 up_write(&rbd_dev->lock_rwsem);
4106 return ret;
4107 }
4108
4109 rbd_assert(rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED);
4110 rbd_assert(list_empty(&rbd_dev->running_list));
4111
22e8bd51
ID		 4112 ret = rbd_post_acquire_action(rbd_dev);
4113 if (ret) {
4114 rbd_warn(rbd_dev, "post-acquire action failed: %d", ret);
4115 /*
4116 * Can't stay in RBD_LOCK_STATE_LOCKED because
4117 * rbd_lock_add_request() would let the request through,
4118 * assuming that e.g. object map is locked and loaded.
4119 */
4120 rbd_unlock(rbd_dev);
ed95b21a
ID		 4121 }
4122
637cd060
ID		 4123out:
4124 wake_lock_waiters(rbd_dev, ret);
ed95b21a 4125 up_write(&rbd_dev->lock_rwsem);
637cd060 4126 return ret;
ed95b21a
ID		 4127}
4128
4129static void rbd_acquire_lock(struct work_struct *work)
4130{
4131 struct rbd_device *rbd_dev = container_of(to_delayed_work(work),
4132 struct rbd_device, lock_dwork);
637cd060 4133 int ret;
ed95b21a
ID		 4134
4135 dout("%s rbd_dev %p\n", __func__, rbd_dev);
4136again:
637cd060
ID		 4137 ret = rbd_try_acquire_lock(rbd_dev);
4138 if (ret <= 0) {
4139 dout("%s rbd_dev %p ret %d - done\n", __func__, rbd_dev, ret);
ed95b21a
ID		 4140 return;
4141 }
4142
4143 ret = rbd_request_lock(rbd_dev);
4144 if (ret == -ETIMEDOUT) {
4145 goto again; /* treat this as a dead client */
e010dd0a
ID		 4146 } else if (ret == -EROFS) {
4147 rbd_warn(rbd_dev, "peer will not release lock");
637cd060
ID		 4148 down_write(&rbd_dev->lock_rwsem);
4149 wake_lock_waiters(rbd_dev, ret);
4150 up_write(&rbd_dev->lock_rwsem);
ed95b21a
ID		 4151 } else if (ret < 0) {
4152 rbd_warn(rbd_dev, "error requesting lock: %d", ret);
4153 mod_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork,
4154 RBD_RETRY_DELAY);
4155 } else {
4156 /*
4157 * lock owner acked, but resend if we don't see them
4158 * release the lock
4159 */
6b0a8774 4160 dout("%s rbd_dev %p requeuing lock_dwork\n", __func__,
ed95b21a
ID		 4161 rbd_dev);
4162 mod_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork,
4163 msecs_to_jiffies(2 * RBD_NOTIFY_TIMEOUT * MSEC_PER_SEC));
4164 }
4165}
4166
a2b1da09 4167static bool rbd_quiesce_lock(struct rbd_device *rbd_dev)
ed95b21a 4168{
a2b1da09 4169 dout("%s rbd_dev %p\n", __func__, rbd_dev);
d9b9c893 4170 lockdep_assert_held_write(&rbd_dev->lock_rwsem);
a2b1da09 4171
ed95b21a
ID		 4172 if (rbd_dev->lock_state != RBD_LOCK_STATE_LOCKED)
4173 return false;
4174
52bb1f9b 4175 /*
ed95b21a 4176 * Ensure that all in-flight IO is flushed.
52bb1f9b 4177 */
e1fddc8f
ID		 4178 rbd_dev->lock_state = RBD_LOCK_STATE_RELEASING;
4179 rbd_assert(!completion_done(&rbd_dev->releasing_wait));
ed9eb710
ID		 4180 if (list_empty(&rbd_dev->running_list))
4181 return true;
4182
4183 up_write(&rbd_dev->lock_rwsem);
4184 wait_for_completion(&rbd_dev->releasing_wait);
ed95b21a
ID		 4185
4186 down_write(&rbd_dev->lock_rwsem);
ed95b21a
ID		 4187 if (rbd_dev->lock_state != RBD_LOCK_STATE_RELEASING)
4188 return false;
4189
e1fddc8f 4190 rbd_assert(list_empty(&rbd_dev->running_list));
a2b1da09
ID		 4191 return true;
4192}
4193
22e8bd51
ID		 4194static void rbd_pre_release_action(struct rbd_device *rbd_dev)
4195{
4196 if (rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP)
4197 rbd_object_map_close(rbd_dev);
4198}
4199
e1fddc8f
ID		 4200static void __rbd_release_lock(struct rbd_device *rbd_dev)
4201{
4202 rbd_assert(list_empty(&rbd_dev->running_list));
4203
22e8bd51 4204 rbd_pre_release_action(rbd_dev);
bbead745 4205 rbd_unlock(rbd_dev);
e1fddc8f
ID		 4206}
4207
a2b1da09
ID		 4208/*
4209 * lock_rwsem must be held for write
4210 */
4211static void rbd_release_lock(struct rbd_device *rbd_dev)
4212{
4213 if (!rbd_quiesce_lock(rbd_dev))
4214 return;
4215
e1fddc8f 4216 __rbd_release_lock(rbd_dev);
a2b1da09 4217
bbead745
ID		 4218 /*
4219 * Give others a chance to grab the lock - we would re-acquire
637cd060
ID		 4220 * almost immediately if we got new IO while draining the running
4221 * list otherwise. We need to ack our own notifications, so this
4222 * lock_dwork will be requeued from rbd_handle_released_lock() by
4223 * way of maybe_kick_acquire().
bbead745
ID		 4224 */
4225 cancel_delayed_work(&rbd_dev->lock_dwork);
ed95b21a
ID		 4226}
4227
4228static void rbd_release_lock_work(struct work_struct *work)
4229{
4230 struct rbd_device *rbd_dev = container_of(work, struct rbd_device,
4231 unlock_work);
4232
4233 down_write(&rbd_dev->lock_rwsem);
4234 rbd_release_lock(rbd_dev);
4235 up_write(&rbd_dev->lock_rwsem);
4236}
4237
637cd060
ID		 4238static void maybe_kick_acquire(struct rbd_device *rbd_dev)
4239{
4240 bool have_requests;
4241
4242 dout("%s rbd_dev %p\n", __func__, rbd_dev);
4243 if (__rbd_is_lock_owner(rbd_dev))
4244 return;
4245
4246 spin_lock(&rbd_dev->lock_lists_lock);
4247 have_requests = !list_empty(&rbd_dev->acquiring_list);
4248 spin_unlock(&rbd_dev->lock_lists_lock);
4249 if (have_requests || delayed_work_pending(&rbd_dev->lock_dwork)) {
4250 dout("%s rbd_dev %p kicking lock_dwork\n", __func__, rbd_dev);
4251 mod_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0);
4252 }
4253}
4254
ed95b21a
ID		 4255static void rbd_handle_acquired_lock(struct rbd_device *rbd_dev, u8 struct_v,
4256 void **p)
4257{
4258 struct rbd_client_id cid = { 0 };
4259
4260 if (struct_v >= 2) {
4261 cid.gid = ceph_decode_64(p);
4262 cid.handle = ceph_decode_64(p);
4263 }
4264
4265 dout("%s rbd_dev %p cid %llu-%llu\n", __func__, rbd_dev, cid.gid,
4266 cid.handle);
4267 if (!rbd_cid_equal(&cid, &rbd_empty_cid)) {
4268 down_write(&rbd_dev->lock_rwsem);
4269 if (rbd_cid_equal(&cid, &rbd_dev->owner_cid)) {
8798d070
ID		 4270 dout("%s rbd_dev %p cid %llu-%llu == owner_cid\n",
4271 __func__, rbd_dev, cid.gid, cid.handle);
4272 } else {
4273 rbd_set_owner_cid(rbd_dev, &cid);
ed95b21a 4274 }
ed95b21a
ID		 4275 downgrade_write(&rbd_dev->lock_rwsem);
4276 } else {
4277 down_read(&rbd_dev->lock_rwsem);
4278 }
4279
637cd060 4280 maybe_kick_acquire(rbd_dev);
ed95b21a
ID		 4281 up_read(&rbd_dev->lock_rwsem);
4282}
4283
4284static void rbd_handle_released_lock(struct rbd_device *rbd_dev, u8 struct_v,
4285 void **p)
4286{
4287 struct rbd_client_id cid = { 0 };
4288
4289 if (struct_v >= 2) {
4290 cid.gid = ceph_decode_64(p);
4291 cid.handle = ceph_decode_64(p);
4292 }
4293
4294 dout("%s rbd_dev %p cid %llu-%llu\n", __func__, rbd_dev, cid.gid,
4295 cid.handle);
4296 if (!rbd_cid_equal(&cid, &rbd_empty_cid)) {
4297 down_write(&rbd_dev->lock_rwsem);
4298 if (!rbd_cid_equal(&cid, &rbd_dev->owner_cid)) {
8798d070 4299 dout("%s rbd_dev %p cid %llu-%llu != owner_cid %llu-%llu\n",
ed95b21a
ID		 4300 __func__, rbd_dev, cid.gid, cid.handle,
4301 rbd_dev->owner_cid.gid, rbd_dev->owner_cid.handle);
8798d070
ID		 4302 } else {
4303 rbd_set_owner_cid(rbd_dev, &rbd_empty_cid);
ed95b21a 4304 }
ed95b21a
ID		 4305 downgrade_write(&rbd_dev->lock_rwsem);
4306 } else {
4307 down_read(&rbd_dev->lock_rwsem);
4308 }
4309
637cd060 4310 maybe_kick_acquire(rbd_dev);
ed95b21a
ID		 4311 up_read(&rbd_dev->lock_rwsem);
4312}
4313
3b77faa0
ID		 4314/*
4315 * Returns result for ResponseMessage to be encoded (<= 0), or 1 if no
4316 * ResponseMessage is needed.
4317 */
4318static int rbd_handle_request_lock(struct rbd_device *rbd_dev, u8 struct_v,
4319 void **p)
ed95b21a
ID		 4320{
4321 struct rbd_client_id my_cid = rbd_get_cid(rbd_dev);
4322 struct rbd_client_id cid = { 0 };
3b77faa0 4323 int result = 1;
ed95b21a
ID		 4324
4325 if (struct_v >= 2) {
4326 cid.gid = ceph_decode_64(p);
4327 cid.handle = ceph_decode_64(p);
4328 }
4329
4330 dout("%s rbd_dev %p cid %llu-%llu\n", __func__, rbd_dev, cid.gid,
4331 cid.handle);
4332 if (rbd_cid_equal(&cid, &my_cid))
3b77faa0 4333 return result;
ed95b21a
ID		 4334
4335 down_read(&rbd_dev->lock_rwsem);
3b77faa0
ID		 4336 if (__rbd_is_lock_owner(rbd_dev)) {
4337 if (rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED &&
4338 rbd_cid_equal(&rbd_dev->owner_cid, &rbd_empty_cid))
4339 goto out_unlock;
4340
4341 /*
4342 * encode ResponseMessage(0) so the peer can detect
4343 * a missing owner
4344 */
4345 result = 0;
4346
4347 if (rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED) {
e010dd0a
ID		 4348 if (!rbd_dev->opts->exclusive) {
4349 dout("%s rbd_dev %p queueing unlock_work\n",
4350 __func__, rbd_dev);
4351 queue_work(rbd_dev->task_wq,
4352 &rbd_dev->unlock_work);
4353 } else {
4354 /* refuse to release the lock */
4355 result = -EROFS;
4356 }
ed95b21a
ID		 4357 }
4358 }
3b77faa0
ID		 4359
4360out_unlock:
ed95b21a 4361 up_read(&rbd_dev->lock_rwsem);
3b77faa0 4362 return result;
ed95b21a
ID		 4363}
4364
4365static void __rbd_acknowledge_notify(struct rbd_device *rbd_dev,
4366 u64 notify_id, u64 cookie, s32 *result)
4367{
4368 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
08a79102
KS		 4369 char buf[4 + CEPH_ENCODING_START_BLK_LEN];
4370 int buf_size = sizeof(buf);
ed95b21a
ID		 4371 int ret;
4372
4373 if (result) {
4374 void *p = buf;
4375
4376 /* encode ResponseMessage */
4377 ceph_start_encoding(&p, 1, 1,
4378 buf_size - CEPH_ENCODING_START_BLK_LEN);
4379 ceph_encode_32(&p, *result);
4380 } else {
4381 buf_size = 0;
4382 }
b8d70035 4383
922dab61
ID		 4384 ret = ceph_osdc_notify_ack(osdc, &rbd_dev->header_oid,
4385 &rbd_dev->header_oloc, notify_id, cookie,
ed95b21a 4386 buf, buf_size);
52bb1f9b 4387 if (ret)
ed95b21a
ID		 4388 rbd_warn(rbd_dev, "acknowledge_notify failed: %d", ret);
4389}
4390
4391static void rbd_acknowledge_notify(struct rbd_device *rbd_dev, u64 notify_id,
4392 u64 cookie)
4393{
4394 dout("%s rbd_dev %p\n", __func__, rbd_dev);
4395 __rbd_acknowledge_notify(rbd_dev, notify_id, cookie, NULL);
4396}
4397
4398static void rbd_acknowledge_notify_result(struct rbd_device *rbd_dev,
4399 u64 notify_id, u64 cookie, s32 result)
4400{
4401 dout("%s rbd_dev %p result %d\n", __func__, rbd_dev, result);
4402 __rbd_acknowledge_notify(rbd_dev, notify_id, cookie, &result);
4403}
4404
4405static void rbd_watch_cb(void *arg, u64 notify_id, u64 cookie,
4406 u64 notifier_id, void *data, size_t data_len)
4407{
4408 struct rbd_device *rbd_dev = arg;
4409 void *p = data;
4410 void *const end = p + data_len;
d4c2269b 4411 u8 struct_v = 0;
ed95b21a
ID		 4412 u32 len;
4413 u32 notify_op;
4414 int ret;
4415
4416 dout("%s rbd_dev %p cookie %llu notify_id %llu data_len %zu\n",
4417 __func__, rbd_dev, cookie, notify_id, data_len);
4418 if (data_len) {
4419 ret = ceph_start_decoding(&p, end, 1, "NotifyMessage",
4420 &struct_v, &len);
4421 if (ret) {
4422 rbd_warn(rbd_dev, "failed to decode NotifyMessage: %d",
4423 ret);
4424 return;
4425 }
4426
4427 notify_op = ceph_decode_32(&p);
4428 } else {
4429 /* legacy notification for header updates */
4430 notify_op = RBD_NOTIFY_OP_HEADER_UPDATE;
4431 len = 0;
4432 }
4433
4434 dout("%s rbd_dev %p notify_op %u\n", __func__, rbd_dev, notify_op);
4435 switch (notify_op) {
4436 case RBD_NOTIFY_OP_ACQUIRED_LOCK:
4437 rbd_handle_acquired_lock(rbd_dev, struct_v, &p);
4438 rbd_acknowledge_notify(rbd_dev, notify_id, cookie);
4439 break;
4440 case RBD_NOTIFY_OP_RELEASED_LOCK:
4441 rbd_handle_released_lock(rbd_dev, struct_v, &p);
4442 rbd_acknowledge_notify(rbd_dev, notify_id, cookie);
4443 break;
4444 case RBD_NOTIFY_OP_REQUEST_LOCK:
3b77faa0
ID		 4445 ret = rbd_handle_request_lock(rbd_dev, struct_v, &p);
4446 if (ret <= 0)
ed95b21a 4447 rbd_acknowledge_notify_result(rbd_dev, notify_id,
3b77faa0 4448 cookie, ret);
ed95b21a
ID		 4449 else
4450 rbd_acknowledge_notify(rbd_dev, notify_id, cookie);
4451 break;
4452 case RBD_NOTIFY_OP_HEADER_UPDATE:
4453 ret = rbd_dev_refresh(rbd_dev);
4454 if (ret)
4455 rbd_warn(rbd_dev, "refresh failed: %d", ret);
4456
4457 rbd_acknowledge_notify(rbd_dev, notify_id, cookie);
4458 break;
4459 default:
4460 if (rbd_is_lock_owner(rbd_dev))
4461 rbd_acknowledge_notify_result(rbd_dev, notify_id,
4462 cookie, -EOPNOTSUPP);
4463 else
4464 rbd_acknowledge_notify(rbd_dev, notify_id, cookie);
4465 break;
4466 }
b8d70035
AE		 4467}
4468
99d16943
ID		 4469static void __rbd_unregister_watch(struct rbd_device *rbd_dev);
4470
922dab61 4471static void rbd_watch_errcb(void *arg, u64 cookie, int err)
bb040aa0 4472{
922dab61 4473 struct rbd_device *rbd_dev = arg;
bb040aa0 4474
922dab61 4475 rbd_warn(rbd_dev, "encountered watch error: %d", err);
bb040aa0 4476
ed95b21a
ID		 4477 down_write(&rbd_dev->lock_rwsem);
4478 rbd_set_owner_cid(rbd_dev, &rbd_empty_cid);
4479 up_write(&rbd_dev->lock_rwsem);
4480
99d16943
ID		 4481 mutex_lock(&rbd_dev->watch_mutex);
4482 if (rbd_dev->watch_state == RBD_WATCH_STATE_REGISTERED) {
4483 __rbd_unregister_watch(rbd_dev);
4484 rbd_dev->watch_state = RBD_WATCH_STATE_ERROR;
bb040aa0 4485
99d16943 4486 queue_delayed_work(rbd_dev->task_wq, &rbd_dev->watch_dwork, 0);
bb040aa0 4487 }
99d16943 4488 mutex_unlock(&rbd_dev->watch_mutex);
bb040aa0
ID		 4489}
4490
9969ebc5 4491/*
99d16943 4492 * watch_mutex must be locked
9969ebc5 4493 */
99d16943 4494static int __rbd_register_watch(struct rbd_device *rbd_dev)
9969ebc5
AE		 4495{
4496 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
922dab61 4497 struct ceph_osd_linger_request *handle;
9969ebc5 4498
922dab61 4499 rbd_assert(!rbd_dev->watch_handle);
99d16943 4500 dout("%s rbd_dev %p\n", __func__, rbd_dev);
9969ebc5 4501
922dab61
ID		 4502 handle = ceph_osdc_watch(osdc, &rbd_dev->header_oid,
4503 &rbd_dev->header_oloc, rbd_watch_cb,
4504 rbd_watch_errcb, rbd_dev);
4505 if (IS_ERR(handle))
4506 return PTR_ERR(handle);
8eb87565 4507
922dab61 4508 rbd_dev->watch_handle = handle;
b30a01f2 4509 return 0;
b30a01f2
ID		 4510}
4511
99d16943
ID		 4512/*
4513 * watch_mutex must be locked
4514 */
4515static void __rbd_unregister_watch(struct rbd_device *rbd_dev)
b30a01f2 4516{
922dab61
ID		 4517 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
4518 int ret;
b30a01f2 4519
99d16943
ID		 4520 rbd_assert(rbd_dev->watch_handle);
4521 dout("%s rbd_dev %p\n", __func__, rbd_dev);
b30a01f2 4522
922dab61
ID		 4523 ret = ceph_osdc_unwatch(osdc, rbd_dev->watch_handle);
4524 if (ret)
4525 rbd_warn(rbd_dev, "failed to unwatch: %d", ret);
76756a51 4526
922dab61 4527 rbd_dev->watch_handle = NULL;
c525f036
ID		 4528}
4529
99d16943
ID		 4530static int rbd_register_watch(struct rbd_device *rbd_dev)
4531{
4532 int ret;
4533
4534 mutex_lock(&rbd_dev->watch_mutex);
4535 rbd_assert(rbd_dev->watch_state == RBD_WATCH_STATE_UNREGISTERED);
4536 ret = __rbd_register_watch(rbd_dev);
4537 if (ret)
4538 goto out;
4539
4540 rbd_dev->watch_state = RBD_WATCH_STATE_REGISTERED;
4541 rbd_dev->watch_cookie = rbd_dev->watch_handle->linger_id;
4542
4543out:
4544 mutex_unlock(&rbd_dev->watch_mutex);
4545 return ret;
4546}
4547
4548static void cancel_tasks_sync(struct rbd_device *rbd_dev)
c525f036 4549{
99d16943
ID		 4550 dout("%s rbd_dev %p\n", __func__, rbd_dev);
4551
ed95b21a
ID		 4552 cancel_work_sync(&rbd_dev->acquired_lock_work);
4553 cancel_work_sync(&rbd_dev->released_lock_work);
4554 cancel_delayed_work_sync(&rbd_dev->lock_dwork);
4555 cancel_work_sync(&rbd_dev->unlock_work);
99d16943
ID		 4556}
4557
0e4e1de5
ID		 4558/*
4559 * header_rwsem must not be held to avoid a deadlock with
4560 * rbd_dev_refresh() when flushing notifies.
4561 */
99d16943
ID		 4562static void rbd_unregister_watch(struct rbd_device *rbd_dev)
4563{
4564 cancel_tasks_sync(rbd_dev);
4565
4566 mutex_lock(&rbd_dev->watch_mutex);
4567 if (rbd_dev->watch_state == RBD_WATCH_STATE_REGISTERED)
4568 __rbd_unregister_watch(rbd_dev);
4569 rbd_dev->watch_state = RBD_WATCH_STATE_UNREGISTERED;
4570 mutex_unlock(&rbd_dev->watch_mutex);
811c6688 4571
23edca86 4572 cancel_delayed_work_sync(&rbd_dev->watch_dwork);
811c6688 4573 ceph_osdc_flush_notifies(&rbd_dev->rbd_client->client->osdc);
fca27065
ID		 4574}
4575
14bb211d
ID		 4576/*
4577 * lock_rwsem must be held for write
4578 */
4579static void rbd_reacquire_lock(struct rbd_device *rbd_dev)
4580{
4581 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
4582 char cookie[32];
4583 int ret;
4584
a2b1da09
ID		 4585 if (!rbd_quiesce_lock(rbd_dev))
4586 return;
14bb211d
ID		 4587
4588 format_lock_cookie(rbd_dev, cookie);
4589 ret = ceph_cls_set_cookie(osdc, &rbd_dev->header_oid,
4590 &rbd_dev->header_oloc, RBD_LOCK_NAME,
4591 CEPH_CLS_LOCK_EXCLUSIVE, rbd_dev->lock_cookie,
4592 RBD_LOCK_TAG, cookie);
4593 if (ret) {
4594 if (ret != -EOPNOTSUPP)
4595 rbd_warn(rbd_dev, "failed to update lock cookie: %d",
4596 ret);
4597
4598 /*
4599 * Lock cookie cannot be updated on older OSDs, so do
4600 * a manual release and queue an acquire.
4601 */
e1fddc8f 4602 __rbd_release_lock(rbd_dev);
a2b1da09 4603 queue_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0);
14bb211d 4604 } else {
edd8ca80 4605 __rbd_lock(rbd_dev, cookie);
637cd060 4606 wake_lock_waiters(rbd_dev, 0);
14bb211d
ID		 4607 }
4608}
4609
99d16943
ID		 4610static void rbd_reregister_watch(struct work_struct *work)
4611{
4612 struct rbd_device *rbd_dev = container_of(to_delayed_work(work),
4613 struct rbd_device, watch_dwork);
4614 int ret;
4615
4616 dout("%s rbd_dev %p\n", __func__, rbd_dev);
4617
4618 mutex_lock(&rbd_dev->watch_mutex);
87c0fded
ID		 4619 if (rbd_dev->watch_state != RBD_WATCH_STATE_ERROR) {
4620 mutex_unlock(&rbd_dev->watch_mutex);
14bb211d 4621 return;
87c0fded 4622 }
99d16943
ID		 4623
4624 ret = __rbd_register_watch(rbd_dev);
4625 if (ret) {
4626 rbd_warn(rbd_dev, "failed to reregister watch: %d", ret);
0b98acd6 4627 if (ret != -EBLOCKLISTED && ret != -ENOENT) {
99d16943
ID		 4628 queue_delayed_work(rbd_dev->task_wq,
4629 &rbd_dev->watch_dwork,
4630 RBD_RETRY_DELAY);
637cd060
ID		 4631 mutex_unlock(&rbd_dev->watch_mutex);
4632 return;
87c0fded 4633 }
637cd060 4634
87c0fded 4635 mutex_unlock(&rbd_dev->watch_mutex);
637cd060
ID		 4636 down_write(&rbd_dev->lock_rwsem);
4637 wake_lock_waiters(rbd_dev, ret);
4638 up_write(&rbd_dev->lock_rwsem);
14bb211d 4639 return;
99d16943
ID		 4640 }
4641
4642 rbd_dev->watch_state = RBD_WATCH_STATE_REGISTERED;
4643 rbd_dev->watch_cookie = rbd_dev->watch_handle->linger_id;
4644 mutex_unlock(&rbd_dev->watch_mutex);
4645
14bb211d
ID		 4646 down_write(&rbd_dev->lock_rwsem);
4647 if (rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED)
4648 rbd_reacquire_lock(rbd_dev);
4649 up_write(&rbd_dev->lock_rwsem);
4650
99d16943
ID		 4651 ret = rbd_dev_refresh(rbd_dev);
4652 if (ret)
f6870cc9 4653 rbd_warn(rbd_dev, "reregistration refresh failed: %d", ret);
99d16943
ID		 4654}
4655
36be9a76 4656/*
f40eb349
AE		 4657 * Synchronous osd object method call. Returns the number of bytes
4658 * returned in the outbound buffer, or a negative error code.
36be9a76
AE		 4659 */
4660static int rbd_obj_method_sync(struct rbd_device *rbd_dev,
ecd4a68a
ID		 4661 struct ceph_object_id *oid,
4662 struct ceph_object_locator *oloc,
36be9a76 4663 const char *method_name,
4157976b 4664 const void *outbound,
36be9a76 4665 size_t outbound_size,
4157976b 4666 void *inbound,
e2a58ee5 4667 size_t inbound_size)
36be9a76 4668{
ecd4a68a
ID		 4669 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
4670 struct page *req_page = NULL;
4671 struct page *reply_page;
36be9a76
AE		 4672 int ret;
4673
4674 /*
6010a451
AE		 4675 * Method calls are ultimately read operations. The result
4676 * should placed into the inbound buffer provided. They
4677 * also supply outbound data--parameters for the object
4678 * method. Currently if this is present it will be a
4679 * snapshot id.
36be9a76 4680 */
ecd4a68a
ID		 4681 if (outbound) {
4682 if (outbound_size > PAGE_SIZE)
4683 return -E2BIG;
36be9a76 4684
ecd4a68a
ID		 4685 req_page = alloc_page(GFP_KERNEL);
4686 if (!req_page)
4687 return -ENOMEM;
04017e29 4688
ecd4a68a 4689 memcpy(page_address(req_page), outbound, outbound_size);
04017e29 4690 }
36be9a76 4691
ecd4a68a
ID		 4692 reply_page = alloc_page(GFP_KERNEL);
4693 if (!reply_page) {
4694 if (req_page)
4695 __free_page(req_page);
4696 return -ENOMEM;
4697 }
57385b51 4698
ecd4a68a
ID		 4699 ret = ceph_osdc_call(osdc, oid, oloc, RBD_DRV_NAME, method_name,
4700 CEPH_OSD_FLAG_READ, req_page, outbound_size,
68ada915 4701 &reply_page, &inbound_size);
ecd4a68a
ID		 4702 if (!ret) {
4703 memcpy(inbound, page_address(reply_page), inbound_size);
4704 ret = inbound_size;
4705 }
36be9a76 4706
ecd4a68a
ID		 4707 if (req_page)
4708 __free_page(req_page);
4709 __free_page(reply_page);
36be9a76
AE		 4710 return ret;
4711}
4712
7ad18afa 4713static void rbd_queue_workfn(struct work_struct *work)
bf0d5f50 4714{
59e542c8
ID		 4715 struct rbd_img_request *img_request =
4716 container_of(work, struct rbd_img_request, work);
4717 struct rbd_device *rbd_dev = img_request->rbd_dev;
4718 enum obj_operation_type op_type = img_request->op_type;
4719 struct request *rq = blk_mq_rq_from_pdu(img_request);
bc1ecc65
ID		 4720 u64 offset = (u64)blk_rq_pos(rq) << SECTOR_SHIFT;
4721 u64 length = blk_rq_bytes(rq);
4e752f0a 4722 u64 mapping_size;
bf0d5f50
AE		 4723 int result;
4724
bc1ecc65 4725 /* Ignore/skip any zero-length requests */
bc1ecc65
ID		 4726 if (!length) {
4727 dout("%s: zero-length request\n", __func__);
4728 result = 0;
59e542c8 4729 goto err_img_request;
bc1ecc65 4730 }
4dda41d3 4731
7ad18afa
CH		 4732 blk_mq_start_request(rq);
4733
4e752f0a
JD		 4734 down_read(&rbd_dev->header_rwsem);
4735 mapping_size = rbd_dev->mapping.size;
a52cc685 4736 rbd_img_capture_header(img_request);
4e752f0a
JD		 4737 up_read(&rbd_dev->header_rwsem);
4738
4739 if (offset + length > mapping_size) {
bc1ecc65 4740 rbd_warn(rbd_dev, "beyond EOD (%llu~%llu > %llu)", offset,
4e752f0a 4741 length, mapping_size);
bc1ecc65 4742 result = -EIO;
a52cc685 4743 goto err_img_request;
bc1ecc65 4744 }
bf0d5f50 4745
21ed05a8
ID		 4746 dout("%s rbd_dev %p img_req %p %s %llu~%llu\n", __func__, rbd_dev,
4747 img_request, obj_op_name(op_type), offset, length);
4748
6484cbe9 4749 if (op_type == OBJ_OP_DISCARD || op_type == OBJ_OP_ZEROOUT)
5a237819 4750 result = rbd_img_fill_nodata(img_request, offset, length);
90e98c52 4751 else
5a237819
ID		 4752 result = rbd_img_fill_from_bio(img_request, offset, length,
4753 rq->bio);
0192ce2e 4754 if (result)
bc1ecc65 4755 goto err_img_request;
bf0d5f50 4756
e1fddc8f 4757 rbd_img_handle_request(img_request, 0);
bc1ecc65 4758 return;
bf0d5f50 4759
bc1ecc65 4760err_img_request:
679a97d2 4761 rbd_img_request_destroy(img_request);
bc1ecc65
ID		 4762 if (result)
4763 rbd_warn(rbd_dev, "%s %llx at %llx result %d",
6d2940c8 4764 obj_op_name(op_type), length, offset, result);
2a842aca 4765 blk_mq_end_request(rq, errno_to_blk_status(result));
bc1ecc65 4766}
bf0d5f50 4767
fc17b653 4768static blk_status_t rbd_queue_rq(struct blk_mq_hw_ctx *hctx,
7ad18afa 4769 const struct blk_mq_queue_data *bd)
bc1ecc65 4770{
59e542c8
ID		 4771 struct rbd_device *rbd_dev = hctx->queue->queuedata;
4772 struct rbd_img_request *img_req = blk_mq_rq_to_pdu(bd->rq);
4773 enum obj_operation_type op_type;
bf0d5f50 4774
59e542c8
ID		 4775 switch (req_op(bd->rq)) {
4776 case REQ_OP_DISCARD:
4777 op_type = OBJ_OP_DISCARD;
4778 break;
4779 case REQ_OP_WRITE_ZEROES:
4780 op_type = OBJ_OP_ZEROOUT;
4781 break;
4782 case REQ_OP_WRITE:
4783 op_type = OBJ_OP_WRITE;
4784 break;
4785 case REQ_OP_READ:
4786 op_type = OBJ_OP_READ;
4787 break;
4788 default:
4789 rbd_warn(rbd_dev, "unknown req_op %d", req_op(bd->rq));
4790 return BLK_STS_IOERR;
4791 }
4792
4793 rbd_img_request_init(img_req, rbd_dev, op_type);
4794
4795 if (rbd_img_is_write(img_req)) {
4796 if (rbd_is_ro(rbd_dev)) {
4797 rbd_warn(rbd_dev, "%s on read-only mapping",
4798 obj_op_name(img_req->op_type));
4799 return BLK_STS_IOERR;
4800 }
4801 rbd_assert(!rbd_is_snap(rbd_dev));
4802 }
4803
4804 INIT_WORK(&img_req->work, rbd_queue_workfn);
4805 queue_work(rbd_wq, &img_req->work);
fc17b653 4806 return BLK_STS_OK;
bf0d5f50
AE		 4807}
4808
602adf40
YS		 4809static void rbd_free_disk(struct rbd_device *rbd_dev)
4810{
8b9ab626 4811 put_disk(rbd_dev->disk);
5769ed0c 4812 blk_mq_free_tag_set(&rbd_dev->tag_set);
a0cab924 4813 rbd_dev->disk = NULL;
602adf40
YS		 4814}
4815
788e2df3 4816static int rbd_obj_read_sync(struct rbd_device *rbd_dev,
fe5478e0
ID		 4817 struct ceph_object_id *oid,
4818 struct ceph_object_locator *oloc,
4819 void *buf, int buf_len)
788e2df3
AE		 4820
4821{
fe5478e0
ID		 4822 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
4823 struct ceph_osd_request *req;
4824 struct page **pages;
4825 int num_pages = calc_pages_for(0, buf_len);
788e2df3
AE		 4826 int ret;
4827
fe5478e0
ID		 4828 req = ceph_osdc_alloc_request(osdc, NULL, 1, false, GFP_KERNEL);
4829 if (!req)
4830 return -ENOMEM;
788e2df3 4831
fe5478e0
ID		 4832 ceph_oid_copy(&req->r_base_oid, oid);
4833 ceph_oloc_copy(&req->r_base_oloc, oloc);
4834 req->r_flags = CEPH_OSD_FLAG_READ;
430c28c3 4835
fe5478e0
ID		 4836 pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL);
4837 if (IS_ERR(pages)) {
4838 ret = PTR_ERR(pages);
4839 goto out_req;
4840 }
1ceae7ef 4841
fe5478e0
ID		 4842 osd_req_op_extent_init(req, 0, CEPH_OSD_OP_READ, 0, buf_len, 0, 0);
4843 osd_req_op_extent_osd_data_pages(req, 0, pages, buf_len, 0, false,
4844 true);
4845
26f887e0
ID		 4846 ret = ceph_osdc_alloc_messages(req, GFP_KERNEL);
4847 if (ret)
4848 goto out_req;
4849
a8af0d68 4850 ceph_osdc_start_request(osdc, req);
fe5478e0
ID		 4851 ret = ceph_osdc_wait_request(osdc, req);
4852 if (ret >= 0)
4853 ceph_copy_from_page_vector(pages, buf, 0, ret);
788e2df3 4854
fe5478e0
ID		 4855out_req:
4856 ceph_osdc_put_request(req);
788e2df3
AE		 4857 return ret;
4858}
4859
602adf40 4860/*
662518b1
AE		 4861 * Read the complete header for the given rbd device. On successful
4862 * return, the rbd_dev->header field will contain up-to-date
4863 * information about the image.
602adf40 4864 */
510a7330
ID		 4865static int rbd_dev_v1_header_info(struct rbd_device *rbd_dev,
4866 struct rbd_image_header *header,
4867 bool first_time)
602adf40 4868{
4156d998 4869 struct rbd_image_header_ondisk *ondisk = NULL;
50f7c4c9 4870 u32 snap_count = 0;
4156d998
AE		 4871 u64 names_size = 0;
4872 u32 want_count;
4873 int ret;
602adf40 4874
00f1f36f 4875 /*
4156d998
AE		 4876 * The complete header will include an array of its 64-bit
4877 * snapshot ids, followed by the names of those snapshots as
4878 * a contiguous block of NUL-terminated strings. Note that
4879 * the number of snapshots could change by the time we read
4880 * it in, in which case we re-read it.
00f1f36f 4881 */
4156d998
AE		 4882 do {
4883 size_t size;
4884
4885 kfree(ondisk);
4886
4887 size = sizeof (*ondisk);
4888 size += snap_count * sizeof (struct rbd_image_snap_ondisk);
4889 size += names_size;
4890 ondisk = kmalloc(size, GFP_KERNEL);
4891 if (!ondisk)
662518b1 4892 return -ENOMEM;
4156d998 4893
fe5478e0
ID		 4894 ret = rbd_obj_read_sync(rbd_dev, &rbd_dev->header_oid,
4895 &rbd_dev->header_oloc, ondisk, size);
4156d998 4896 if (ret < 0)
662518b1 4897 goto out;
c0cd10db 4898 if ((size_t)ret < size) {
4156d998 4899 ret = -ENXIO;
06ecc6cb
AE		 4900 rbd_warn(rbd_dev, "short header read (want %zd got %d)",
4901 size, ret);
662518b1 4902 goto out;
4156d998
AE		 4903 }
4904 if (!rbd_dev_ondisk_valid(ondisk)) {
4905 ret = -ENXIO;
06ecc6cb 4906 rbd_warn(rbd_dev, "invalid header");
662518b1 4907 goto out;
81e759fb 4908 }
602adf40 4909
4156d998
AE		 4910 names_size = le64_to_cpu(ondisk->snap_names_len);
4911 want_count = snap_count;
4912 snap_count = le32_to_cpu(ondisk->snap_count);
4913 } while (snap_count != want_count);
00f1f36f 4914
510a7330 4915 ret = rbd_header_from_disk(header, ondisk, first_time);
662518b1 4916out:
4156d998
AE		 4917 kfree(ondisk);
4918
4919 return ret;
602adf40
YS		 4920}
4921
9875201e
JD		 4922static void rbd_dev_update_size(struct rbd_device *rbd_dev)
4923{
4924 sector_t size;
9875201e
JD		 4925
4926 /*
811c6688
ID		 4927 * If EXISTS is not set, rbd_dev->disk may be NULL, so don't
4928 * try to update its size. If REMOVING is set, updating size
4929 * is just useless work since the device can't be opened.
9875201e 4930 */
811c6688
ID		 4931 if (test_bit(RBD_DEV_FLAG_EXISTS, &rbd_dev->flags) &&
4932 !test_bit(RBD_DEV_FLAG_REMOVING, &rbd_dev->flags)) {
9875201e
JD		 4933 size = (sector_t)rbd_dev->mapping.size / SECTOR_SIZE;
4934 dout("setting size to %llu sectors", (unsigned long long)size);
e864e49a 4935 set_capacity_and_notify(rbd_dev->disk, size);
9875201e
JD		 4936 }
4937}
4938
f363b089 4939static const struct blk_mq_ops rbd_mq_ops = {
7ad18afa 4940 .queue_rq = rbd_queue_rq,
7ad18afa
CH		 4941};
4942
602adf40
YS		 4943static int rbd_init_disk(struct rbd_device *rbd_dev)
4944{
4945 struct gendisk *disk;
4946 struct request_queue *q;
420efbdf
ID		 4947 unsigned int objset_bytes =
4948 rbd_dev->layout.object_size * rbd_dev->layout.stripe_count;
7ad18afa 4949 int err;
602adf40 4950
7ad18afa
CH		 4951 memset(&rbd_dev->tag_set, 0, sizeof(rbd_dev->tag_set));
4952 rbd_dev->tag_set.ops = &rbd_mq_ops;
b5584180 4953 rbd_dev->tag_set.queue_depth = rbd_dev->opts->queue_depth;
7ad18afa 4954 rbd_dev->tag_set.numa_node = NUMA_NO_NODE;
56d18f62 4955 rbd_dev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
f9b6b98d 4956 rbd_dev->tag_set.nr_hw_queues = num_present_cpus();
59e542c8 4957 rbd_dev->tag_set.cmd_size = sizeof(struct rbd_img_request);
7ad18afa
CH		 4958
4959 err = blk_mq_alloc_tag_set(&rbd_dev->tag_set);
4960 if (err)
195b1956 4961 return err;
029bcbd8 4962
195b1956
CH		 4963 disk = blk_mq_alloc_disk(&rbd_dev->tag_set, rbd_dev);
4964 if (IS_ERR(disk)) {
4965 err = PTR_ERR(disk);
7ad18afa
CH		 4966 goto out_tag_set;
4967 }
195b1956
CH		 4968 q = disk->queue;
4969
4970 snprintf(disk->disk_name, sizeof(disk->disk_name), RBD_DRV_NAME "%d",
4971 rbd_dev->dev_id);
4972 disk->major = rbd_dev->major;
4973 disk->first_minor = rbd_dev->minor;
1ebe2e5f 4974 if (single_major)
195b1956 4975 disk->minors = (1 << RBD_SINGLE_MAJOR_PART_SHIFT);
1ebe2e5f 4976 else
195b1956 4977 disk->minors = RBD_MINORS_PER_MAJOR;
195b1956 4978 disk->fops = &rbd_bd_ops;
0077a500 4979 disk->private_data = rbd_dev;
7ad18afa 4980
8b904b5b 4981 blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
d8a2c89c 4982 /* QUEUE_FLAG_ADD_RANDOM is off by default for blk-mq */
593a9e7b 4983
420efbdf 4984 blk_queue_max_hw_sectors(q, objset_bytes >> SECTOR_SHIFT);
0d9fde4f 4985 q->limits.max_sectors = queue_max_hw_sectors(q);
21acdf45 4986 blk_queue_max_segments(q, USHRT_MAX);
24f1df60 4987 blk_queue_max_segment_size(q, UINT_MAX);
16d80c54
ID		 4988 blk_queue_io_min(q, rbd_dev->opts->alloc_size);
4989 blk_queue_io_opt(q, rbd_dev->opts->alloc_size);
029bcbd8 4990
d9360540 4991 if (rbd_dev->opts->trim) {
16d80c54 4992 q->limits.discard_granularity = rbd_dev->opts->alloc_size;
d9360540
ID		 4993 blk_queue_max_discard_sectors(q, objset_bytes >> SECTOR_SHIFT);
4994 blk_queue_max_write_zeroes_sectors(q, objset_bytes >> SECTOR_SHIFT);
4995 }
90e98c52 4996
bae818ee 4997 if (!ceph_test_opt(rbd_dev->rbd_client->client, NOCRC))
1cb039f3 4998 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
bae818ee 4999
602adf40 5000 rbd_dev->disk = disk;
602adf40 5001
602adf40 5002 return 0;
7ad18afa
CH		 5003out_tag_set:
5004 blk_mq_free_tag_set(&rbd_dev->tag_set);
7ad18afa 5005 return err;
602adf40
YS		 5006}
5007
dfc5606d
YS		 5008/*
5009 sysfs
5010*/
5011
593a9e7b
AE		 5012static struct rbd_device *dev_to_rbd_dev(struct device *dev)
5013{
5014 return container_of(dev, struct rbd_device, dev);
5015}
5016
dfc5606d
YS		 5017static ssize_t rbd_size_show(struct device *dev,
5018 struct device_attribute *attr, char *buf)
5019{
593a9e7b 5020 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
a51aa0c0 5021
fc71d833
AE		 5022 return sprintf(buf, "%llu\n",
5023 (unsigned long long)rbd_dev->mapping.size);
dfc5606d
YS		 5024}
5025
34b13184
AE		 5026static ssize_t rbd_features_show(struct device *dev,
5027 struct device_attribute *attr, char *buf)
5028{
5029 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
5030
fa58bcad 5031 return sprintf(buf, "0x%016llx\n", rbd_dev->header.features);
34b13184
AE		 5032}
5033
dfc5606d
YS		 5034static ssize_t rbd_major_show(struct device *dev,
5035 struct device_attribute *attr, char *buf)
5036{
593a9e7b 5037 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
602adf40 5038
fc71d833
AE		 5039 if (rbd_dev->major)
5040 return sprintf(buf, "%d\n", rbd_dev->major);
5041
5042 return sprintf(buf, "(none)\n");
dd82fff1
ID		 5043}
5044
5045static ssize_t rbd_minor_show(struct device *dev,
5046 struct device_attribute *attr, char *buf)
5047{
5048 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
fc71d833 5049
dd82fff1 5050 return sprintf(buf, "%d\n", rbd_dev->minor);
dfc5606d
YS		 5051}
5052
005a07bf
ID		 5053static ssize_t rbd_client_addr_show(struct device *dev,
5054 struct device_attribute *attr, char *buf)
5055{
5056 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
5057 struct ceph_entity_addr *client_addr =
5058 ceph_client_addr(rbd_dev->rbd_client->client);
5059
5060 return sprintf(buf, "%pISpc/%u\n", &client_addr->in_addr,
5061 le32_to_cpu(client_addr->nonce));
5062}
5063
dfc5606d
YS		 5064static ssize_t rbd_client_id_show(struct device *dev,
5065 struct device_attribute *attr, char *buf)
602adf40 5066{
593a9e7b 5067 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
dfc5606d 5068
1dbb4399 5069 return sprintf(buf, "client%lld\n",
033268a5 5070 ceph_client_gid(rbd_dev->rbd_client->client));
602adf40
YS		 5071}
5072
267fb90b
MC		 5073static ssize_t rbd_cluster_fsid_show(struct device *dev,
5074 struct device_attribute *attr, char *buf)
5075{
5076 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
5077
5078 return sprintf(buf, "%pU\n", &rbd_dev->rbd_client->client->fsid);
5079}
5080
0d6d1e9c
MC		 5081static ssize_t rbd_config_info_show(struct device *dev,
5082 struct device_attribute *attr, char *buf)
5083{
5084 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
5085
f44d04e6
ID		 5086 if (!capable(CAP_SYS_ADMIN))
5087 return -EPERM;
5088
0d6d1e9c 5089 return sprintf(buf, "%s\n", rbd_dev->config_info);
602adf40
YS		 5090}
5091
dfc5606d
YS		 5092static ssize_t rbd_pool_show(struct device *dev,
5093 struct device_attribute *attr, char *buf)
602adf40 5094{
593a9e7b 5095 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
dfc5606d 5096
0d7dbfce 5097 return sprintf(buf, "%s\n", rbd_dev->spec->pool_name);
dfc5606d
YS		 5098}
5099
9bb2f334
AE		 5100static ssize_t rbd_pool_id_show(struct device *dev,
5101 struct device_attribute *attr, char *buf)
5102{
5103 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
5104
0d7dbfce 5105 return sprintf(buf, "%llu\n",
fc71d833 5106 (unsigned long long) rbd_dev->spec->pool_id);
9bb2f334
AE		 5107}
5108
b26c047b
ID		 5109static ssize_t rbd_pool_ns_show(struct device *dev,
5110 struct device_attribute *attr, char *buf)
5111{
5112 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
5113
5114 return sprintf(buf, "%s\n", rbd_dev->spec->pool_ns ?: "");
5115}
5116
dfc5606d
YS		 5117static ssize_t rbd_name_show(struct device *dev,
5118 struct device_attribute *attr, char *buf)
5119{
593a9e7b 5120 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
dfc5606d 5121
a92ffdf8
AE		 5122 if (rbd_dev->spec->image_name)
5123 return sprintf(buf, "%s\n", rbd_dev->spec->image_name);
5124
5125 return sprintf(buf, "(unknown)\n");
dfc5606d
YS		 5126}
5127
589d30e0
AE		 5128static ssize_t rbd_image_id_show(struct device *dev,
5129 struct device_attribute *attr, char *buf)
5130{
5131 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
5132
0d7dbfce 5133 return sprintf(buf, "%s\n", rbd_dev->spec->image_id);
589d30e0
AE		 5134}
5135
34b13184
AE		 5136/*
5137 * Shows the name of the currently-mapped snapshot (or
5138 * RBD_SNAP_HEAD_NAME for the base image).
5139 */
dfc5606d
YS		 5140static ssize_t rbd_snap_show(struct device *dev,
5141 struct device_attribute *attr,
5142 char *buf)
5143{
593a9e7b 5144 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
dfc5606d 5145
0d7dbfce 5146 return sprintf(buf, "%s\n", rbd_dev->spec->snap_name);
dfc5606d
YS		 5147}
5148
92a58671
MC		 5149static ssize_t rbd_snap_id_show(struct device *dev,
5150 struct device_attribute *attr, char *buf)
5151{
5152 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
5153
5154 return sprintf(buf, "%llu\n", rbd_dev->spec->snap_id);
5155}
5156
86b00e0d 5157/*
ff96128f
ID		 5158 * For a v2 image, shows the chain of parent images, separated by empty
5159 * lines. For v1 images or if there is no parent, shows "(no parent
5160 * image)".
86b00e0d
AE		 5161 */
5162static ssize_t rbd_parent_show(struct device *dev,
ff96128f
ID		 5163 struct device_attribute *attr,
5164 char *buf)
86b00e0d
AE		 5165{
5166 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
ff96128f 5167 ssize_t count = 0;
86b00e0d 5168
ff96128f 5169 if (!rbd_dev->parent)
86b00e0d
AE		 5170 return sprintf(buf, "(no parent image)\n");
5171
ff96128f
ID		 5172 for ( ; rbd_dev->parent; rbd_dev = rbd_dev->parent) {
5173 struct rbd_spec *spec = rbd_dev->parent_spec;
5174
5175 count += sprintf(&buf[count], "%s"
5176 "pool_id %llu\npool_name %s\n"
e92c0eaf 5177 "pool_ns %s\n"
ff96128f
ID		 5178 "image_id %s\nimage_name %s\n"
5179 "snap_id %llu\nsnap_name %s\n"
5180 "overlap %llu\n",
5181 !count ? "" : "\n", /* first? */
5182 spec->pool_id, spec->pool_name,
e92c0eaf 5183 spec->pool_ns ?: "",
ff96128f
ID		 5184 spec->image_id, spec->image_name ?: "(unknown)",
5185 spec->snap_id, spec->snap_name,
5186 rbd_dev->parent_overlap);
5187 }
5188
5189 return count;
86b00e0d
AE		 5190}
5191
dfc5606d
YS		 5192static ssize_t rbd_image_refresh(struct device *dev,
5193 struct device_attribute *attr,
5194 const char *buf,
5195 size_t size)
5196{
593a9e7b 5197 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
b813623a 5198 int ret;
602adf40 5199
f44d04e6
ID		 5200 if (!capable(CAP_SYS_ADMIN))
5201 return -EPERM;
5202
cc4a38bd 5203 ret = rbd_dev_refresh(rbd_dev);
e627db08 5204 if (ret)
52bb1f9b 5205 return ret;
b813623a 5206
52bb1f9b 5207 return size;
dfc5606d 5208}
602adf40 5209
5657a819
JP		 5210static DEVICE_ATTR(size, 0444, rbd_size_show, NULL);
5211static DEVICE_ATTR(features, 0444, rbd_features_show, NULL);
5212static DEVICE_ATTR(major, 0444, rbd_major_show, NULL);
5213static DEVICE_ATTR(minor, 0444, rbd_minor_show, NULL);
5214static DEVICE_ATTR(client_addr, 0444, rbd_client_addr_show, NULL);
5215static DEVICE_ATTR(client_id, 0444, rbd_client_id_show, NULL);
5216static DEVICE_ATTR(cluster_fsid, 0444, rbd_cluster_fsid_show, NULL);
5217static DEVICE_ATTR(config_info, 0400, rbd_config_info_show, NULL);
5218static DEVICE_ATTR(pool, 0444, rbd_pool_show, NULL);
5219static DEVICE_ATTR(pool_id, 0444, rbd_pool_id_show, NULL);
b26c047b 5220static DEVICE_ATTR(pool_ns, 0444, rbd_pool_ns_show, NULL);
5657a819
JP		 5221static DEVICE_ATTR(name, 0444, rbd_name_show, NULL);
5222static DEVICE_ATTR(image_id, 0444, rbd_image_id_show, NULL);
5223static DEVICE_ATTR(refresh, 0200, NULL, rbd_image_refresh);
5224static DEVICE_ATTR(current_snap, 0444, rbd_snap_show, NULL);
5225static DEVICE_ATTR(snap_id, 0444, rbd_snap_id_show, NULL);
5226static DEVICE_ATTR(parent, 0444, rbd_parent_show, NULL);
dfc5606d
YS		 5227
5228static struct attribute *rbd_attrs[] = {
5229 &dev_attr_size.attr,
34b13184 5230 &dev_attr_features.attr,
dfc5606d 5231 &dev_attr_major.attr,
dd82fff1 5232 &dev_attr_minor.attr,
005a07bf 5233 &dev_attr_client_addr.attr,
dfc5606d 5234 &dev_attr_client_id.attr,
267fb90b 5235 &dev_attr_cluster_fsid.attr,
0d6d1e9c 5236 &dev_attr_config_info.attr,
dfc5606d 5237 &dev_attr_pool.attr,
9bb2f334 5238 &dev_attr_pool_id.attr,
b26c047b 5239 &dev_attr_pool_ns.attr,
dfc5606d 5240 &dev_attr_name.attr,
589d30e0 5241 &dev_attr_image_id.attr,
dfc5606d 5242 &dev_attr_current_snap.attr,
92a58671 5243 &dev_attr_snap_id.attr,
86b00e0d 5244 &dev_attr_parent.attr,
dfc5606d 5245 &dev_attr_refresh.attr,
dfc5606d
YS		 5246 NULL
5247};
5248
5249static struct attribute_group rbd_attr_group = {
5250 .attrs = rbd_attrs,
5251};
5252
5253static const struct attribute_group *rbd_attr_groups[] = {
5254 &rbd_attr_group,
5255 NULL
5256};
5257
6cac4695 5258static void rbd_dev_release(struct device *dev);
dfc5606d 5259
b9942bc9 5260static const struct device_type rbd_device_type = {
dfc5606d
YS		 5261 .name = "rbd",
5262 .groups = rbd_attr_groups,
6cac4695 5263 .release = rbd_dev_release,
dfc5606d
YS		 5264};
5265
8b8fb99c
AE		 5266static struct rbd_spec *rbd_spec_get(struct rbd_spec *spec)
5267{
5268 kref_get(&spec->kref);
5269
5270 return spec;
5271}
5272
5273static void rbd_spec_free(struct kref *kref);
5274static void rbd_spec_put(struct rbd_spec *spec)
5275{
5276 if (spec)
5277 kref_put(&spec->kref, rbd_spec_free);
5278}
5279
5280static struct rbd_spec *rbd_spec_alloc(void)
5281{
5282 struct rbd_spec *spec;
5283
5284 spec = kzalloc(sizeof (*spec), GFP_KERNEL);
5285 if (!spec)
5286 return NULL;
04077599
ID		 5287
5288 spec->pool_id = CEPH_NOPOOL;
5289 spec->snap_id = CEPH_NOSNAP;
8b8fb99c
AE		 5290 kref_init(&spec->kref);
5291
8b8fb99c
AE		 5292 return spec;
5293}
5294
5295static void rbd_spec_free(struct kref *kref)
5296{
5297 struct rbd_spec *spec = container_of(kref, struct rbd_spec, kref);
5298
5299 kfree(spec->pool_name);
b26c047b 5300 kfree(spec->pool_ns);
8b8fb99c
AE		 5301 kfree(spec->image_id);
5302 kfree(spec->image_name);
5303 kfree(spec->snap_name);
5304 kfree(spec);
5305}
5306
1643dfa4 5307static void rbd_dev_free(struct rbd_device *rbd_dev)
dd5ac32d 5308{
99d16943 5309 WARN_ON(rbd_dev->watch_state != RBD_WATCH_STATE_UNREGISTERED);
ed95b21a 5310 WARN_ON(rbd_dev->lock_state != RBD_LOCK_STATE_UNLOCKED);
dd5ac32d 5311
c41d13a3 5312 ceph_oid_destroy(&rbd_dev->header_oid);
6b6dddbe 5313 ceph_oloc_destroy(&rbd_dev->header_oloc);
0d6d1e9c 5314 kfree(rbd_dev->config_info);
c41d13a3 5315
dd5ac32d
ID		 5316 rbd_put_client(rbd_dev->rbd_client);
5317 rbd_spec_put(rbd_dev->spec);
5318 kfree(rbd_dev->opts);
5319 kfree(rbd_dev);
1643dfa4
ID		 5320}
5321
5322static void rbd_dev_release(struct device *dev)
5323{
5324 struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
5325 bool need_put = !!rbd_dev->opts;
5326
5327 if (need_put) {
5328 destroy_workqueue(rbd_dev->task_wq);
5329 ida_simple_remove(&rbd_dev_id_ida, rbd_dev->dev_id);
5330 }
5331
5332 rbd_dev_free(rbd_dev);
dd5ac32d
ID		 5333
5334 /*
5335 * This is racy, but way better than putting module outside of
5336 * the release callback. The race window is pretty small, so
5337 * doing something similar to dm (dm-builtin.c) is overkill.
5338 */
5339 if (need_put)
5340 module_put(THIS_MODULE);
5341}
5342
f7c4d9b1 5343static struct rbd_device *__rbd_dev_create(struct rbd_spec *spec)
c53d5893
AE		 5344{
5345 struct rbd_device *rbd_dev;
5346
1643dfa4 5347 rbd_dev = kzalloc(sizeof(*rbd_dev), GFP_KERNEL);
c53d5893
AE		 5348 if (!rbd_dev)
5349 return NULL;
5350
5351 spin_lock_init(&rbd_dev->lock);
5352 INIT_LIST_HEAD(&rbd_dev->node);
c53d5893
AE		 5353 init_rwsem(&rbd_dev->header_rwsem);
5354
7e97332e 5355 rbd_dev->header.data_pool_id = CEPH_NOPOOL;
c41d13a3 5356 ceph_oid_init(&rbd_dev->header_oid);
431a02cd 5357 rbd_dev->header_oloc.pool = spec->pool_id;
b26c047b
ID		 5358 if (spec->pool_ns) {
5359 WARN_ON(!*spec->pool_ns);
5360 rbd_dev->header_oloc.pool_ns =
5361 ceph_find_or_create_string(spec->pool_ns,
5362 strlen(spec->pool_ns));
5363 }
c41d13a3 5364
99d16943
ID		 5365 mutex_init(&rbd_dev->watch_mutex);
5366 rbd_dev->watch_state = RBD_WATCH_STATE_UNREGISTERED;
5367 INIT_DELAYED_WORK(&rbd_dev->watch_dwork, rbd_reregister_watch);
5368
ed95b21a
ID		 5369 init_rwsem(&rbd_dev->lock_rwsem);
5370 rbd_dev->lock_state = RBD_LOCK_STATE_UNLOCKED;
5371 INIT_WORK(&rbd_dev->acquired_lock_work, rbd_notify_acquired_lock);
5372 INIT_WORK(&rbd_dev->released_lock_work, rbd_notify_released_lock);
5373 INIT_DELAYED_WORK(&rbd_dev->lock_dwork, rbd_acquire_lock);
5374 INIT_WORK(&rbd_dev->unlock_work, rbd_release_lock_work);
e1fddc8f 5375 spin_lock_init(&rbd_dev->lock_lists_lock);
637cd060 5376 INIT_LIST_HEAD(&rbd_dev->acquiring_list);
e1fddc8f 5377 INIT_LIST_HEAD(&rbd_dev->running_list);
637cd060 5378 init_completion(&rbd_dev->acquire_wait);
e1fddc8f 5379 init_completion(&rbd_dev->releasing_wait);
ed95b21a 5380
22e8bd51 5381 spin_lock_init(&rbd_dev->object_map_lock);
ed95b21a 5382
dd5ac32d
ID		 5383 rbd_dev->dev.bus = &rbd_bus_type;
5384 rbd_dev->dev.type = &rbd_device_type;
5385 rbd_dev->dev.parent = &rbd_root_dev;
dd5ac32d
ID		 5386 device_initialize(&rbd_dev->dev);
5387
1643dfa4
ID		 5388 return rbd_dev;
5389}
5390
5391/*
5392 * Create a mapping rbd_dev.
5393 */
5394static struct rbd_device *rbd_dev_create(struct rbd_client *rbdc,
5395 struct rbd_spec *spec,
5396 struct rbd_options *opts)
5397{
5398 struct rbd_device *rbd_dev;
5399
f7c4d9b1 5400 rbd_dev = __rbd_dev_create(spec);
1643dfa4
ID		 5401 if (!rbd_dev)
5402 return NULL;
5403
1643dfa4
ID		 5404 /* get an id and fill in device name */
5405 rbd_dev->dev_id = ida_simple_get(&rbd_dev_id_ida, 0,
5406 minor_to_rbd_dev_id(1 << MINORBITS),
5407 GFP_KERNEL);
5408 if (rbd_dev->dev_id < 0)
5409 goto fail_rbd_dev;
5410
5411 sprintf(rbd_dev->name, RBD_DRV_NAME "%d", rbd_dev->dev_id);
5412 rbd_dev->task_wq = alloc_ordered_workqueue("%s-tasks", WQ_MEM_RECLAIM,
5413 rbd_dev->name);
5414 if (!rbd_dev->task_wq)
5415 goto fail_dev_id;
dd5ac32d 5416
1643dfa4
ID		 5417 /* we have a ref from do_rbd_add() */
5418 __module_get(THIS_MODULE);
dd5ac32d 5419
f7c4d9b1
ID		 5420 rbd_dev->rbd_client = rbdc;
5421 rbd_dev->spec = spec;
5422 rbd_dev->opts = opts;
5423
1643dfa4 5424 dout("%s rbd_dev %p dev_id %d\n", __func__, rbd_dev, rbd_dev->dev_id);
c53d5893 5425 return rbd_dev;
1643dfa4
ID		 5426
5427fail_dev_id:
5428 ida_simple_remove(&rbd_dev_id_ida, rbd_dev->dev_id);
5429fail_rbd_dev:
5430 rbd_dev_free(rbd_dev);
5431 return NULL;
c53d5893
AE		 5432}
5433
5434static void rbd_dev_destroy(struct rbd_device *rbd_dev)
5435{
dd5ac32d
ID		 5436 if (rbd_dev)
5437 put_device(&rbd_dev->dev);
c53d5893
AE		 5438}
5439
9d475de5
AE		 5440/*
5441 * Get the size and object order for an image snapshot, or if
5442 * snap_id is CEPH_NOSNAP, gets this information for the base
5443 * image.
5444 */
5445static int _rbd_dev_v2_snap_size(struct rbd_device *rbd_dev, u64 snap_id,
5446 u8 *order, u64 *snap_size)
5447{
5448 __le64 snapid = cpu_to_le64(snap_id);
5449 int ret;
5450 struct {
5451 u8 order;
5452 __le64 size;
5453 } __attribute__ ((packed)) size_buf = { 0 };
5454
ecd4a68a
ID		 5455 ret = rbd_obj_method_sync(rbd_dev, &rbd_dev->header_oid,
5456 &rbd_dev->header_oloc, "get_size",
5457 &snapid, sizeof(snapid),
5458 &size_buf, sizeof(size_buf));
36be9a76 5459 dout("%s: rbd_obj_method_sync returned %d\n", __func__, ret);
9d475de5
AE		 5460 if (ret < 0)
5461 return ret;
57385b51
AE		 5462 if (ret < sizeof (size_buf))
5463 return -ERANGE;
9d475de5 5464
c3545579 5465 if (order) {
c86f86e9 5466 *order = size_buf.order;
c3545579
JD		 5467 dout(" order %u", (unsigned int)*order);
5468 }
9d475de5
AE		 5469 *snap_size = le64_to_cpu(size_buf.size);
5470
c3545579
JD		 5471 dout(" snap_id 0x%016llx snap_size = %llu\n",
5472 (unsigned long long)snap_id,
57385b51 5473 (unsigned long long)*snap_size);
9d475de5
AE		 5474
5475 return 0;
5476}
5477
510a7330
ID		 5478static int rbd_dev_v2_object_prefix(struct rbd_device *rbd_dev,
5479 char **pobject_prefix)
1e130199 5480{
5435d206 5481 size_t size;
1e130199 5482 void *reply_buf;
510a7330 5483 char *object_prefix;
1e130199
AE		 5484 int ret;
5485 void *p;
5486
5435d206
DY		 5487 /* Response will be an encoded string, which includes a length */
5488 size = sizeof(__le32) + RBD_OBJ_PREFIX_LEN_MAX;
5489 reply_buf = kzalloc(size, GFP_KERNEL);
1e130199
AE		 5490 if (!reply_buf)
5491 return -ENOMEM;
5492
ecd4a68a
ID		 5493 ret = rbd_obj_method_sync(rbd_dev, &rbd_dev->header_oid,
5494 &rbd_dev->header_oloc, "get_object_prefix",
5435d206 5495 NULL, 0, reply_buf, size);
36be9a76 5496 dout("%s: rbd_obj_method_sync returned %d\n", __func__, ret);
1e130199
AE		 5497 if (ret < 0)
5498 goto out;
5499
5500 p = reply_buf;
510a7330
ID		 5501 object_prefix = ceph_extract_encoded_string(&p, p + ret, NULL,
5502 GFP_NOIO);
5503 if (IS_ERR(object_prefix)) {
5504 ret = PTR_ERR(object_prefix);
5505 goto out;
5506 }
57385b51 5507 ret = 0;
1e130199 5508
510a7330
ID		 5509 *pobject_prefix = object_prefix;
5510 dout(" object_prefix = %s\n", object_prefix);
1e130199
AE		 5511out:
5512 kfree(reply_buf);
5513
5514 return ret;
5515}
5516
b1b5402a 5517static int _rbd_dev_v2_snap_features(struct rbd_device *rbd_dev, u64 snap_id,
196e2d6d 5518 bool read_only, u64 *snap_features)
b1b5402a 5519{
196e2d6d
ID		 5520 struct {
5521 __le64 snap_id;
5522 u8 read_only;
5523 } features_in;
b1b5402a
AE		 5524 struct {
5525 __le64 features;
5526 __le64 incompat;
4157976b 5527 } __attribute__ ((packed)) features_buf = { 0 };
d3767f0f 5528 u64 unsup;
b1b5402a
AE		 5529 int ret;
5530
196e2d6d
ID		 5531 features_in.snap_id = cpu_to_le64(snap_id);
5532 features_in.read_only = read_only;
5533
ecd4a68a
ID		 5534 ret = rbd_obj_method_sync(rbd_dev, &rbd_dev->header_oid,
5535 &rbd_dev->header_oloc, "get_features",
196e2d6d 5536 &features_in, sizeof(features_in),
ecd4a68a 5537 &features_buf, sizeof(features_buf));
36be9a76 5538 dout("%s: rbd_obj_method_sync returned %d\n", __func__, ret);
b1b5402a
AE		 5539 if (ret < 0)
5540 return ret;
57385b51
AE		 5541 if (ret < sizeof (features_buf))
5542 return -ERANGE;
d889140c 5543
d3767f0f
ID		 5544 unsup = le64_to_cpu(features_buf.incompat) & ~RBD_FEATURES_SUPPORTED;
5545 if (unsup) {
5546 rbd_warn(rbd_dev, "image uses unsupported features: 0x%llx",
5547 unsup);
b8f5c6ed 5548 return -ENXIO;
d3767f0f 5549 }
d889140c 5550
b1b5402a
AE		 5551 *snap_features = le64_to_cpu(features_buf.features);
5552
5553 dout(" snap_id 0x%016llx features = 0x%016llx incompat = 0x%016llx\n",
57385b51
AE		 5554 (unsigned long long)snap_id,
5555 (unsigned long long)*snap_features,
5556 (unsigned long long)le64_to_cpu(features_buf.incompat));
b1b5402a
AE		 5557
5558 return 0;
5559}
5560
22e8bd51
ID		 5561/*
5562 * These are generic image flags, but since they are used only for
5563 * object map, store them in rbd_dev->object_map_flags.
5564 *
5565 * For the same reason, this function is called only on object map
5566 * (re)load and not on header refresh.
5567 */
5568static int rbd_dev_v2_get_flags(struct rbd_device *rbd_dev)
5569{
5570 __le64 snapid = cpu_to_le64(rbd_dev->spec->snap_id);
5571 __le64 flags;
5572 int ret;
5573
5574 ret = rbd_obj_method_sync(rbd_dev, &rbd_dev->header_oid,
5575 &rbd_dev->header_oloc, "get_flags",
5576 &snapid, sizeof(snapid),
5577 &flags, sizeof(flags));
5578 if (ret < 0)
5579 return ret;
5580 if (ret < sizeof(flags))
5581 return -EBADMSG;
5582
5583 rbd_dev->object_map_flags = le64_to_cpu(flags);
5584 return 0;
5585}
5586
eb3b2d6b
ID		 5587struct parent_image_info {
5588 u64 pool_id;
e92c0eaf 5589 const char *pool_ns;
eb3b2d6b
ID		 5590 const char *image_id;
5591 u64 snap_id;
5592
e92c0eaf 5593 bool has_overlap;
eb3b2d6b
ID		 5594 u64 overlap;
5595};
5596
c1031177
ID		 5597static void rbd_parent_info_cleanup(struct parent_image_info *pii)
5598{
5599 kfree(pii->pool_ns);
5600 kfree(pii->image_id);
5601
5602 memset(pii, 0, sizeof(*pii));
5603}
5604
e92c0eaf
ID		 5605/*
5606 * The caller is responsible for @pii.
5607 */
5608static int decode_parent_image_spec(void **p, void *end,
5609 struct parent_image_info *pii)
5610{
5611 u8 struct_v;
5612 u32 struct_len;
5613 int ret;
5614
5615 ret = ceph_start_decoding(p, end, 1, "ParentImageSpec",
5616 &struct_v, &struct_len);
5617 if (ret)
5618 return ret;
5619
5620 ceph_decode_64_safe(p, end, pii->pool_id, e_inval);
5621 pii->pool_ns = ceph_extract_encoded_string(p, end, NULL, GFP_KERNEL);
5622 if (IS_ERR(pii->pool_ns)) {
5623 ret = PTR_ERR(pii->pool_ns);
5624 pii->pool_ns = NULL;
5625 return ret;
5626 }
5627 pii->image_id = ceph_extract_encoded_string(p, end, NULL, GFP_KERNEL);
5628 if (IS_ERR(pii->image_id)) {
5629 ret = PTR_ERR(pii->image_id);
5630 pii->image_id = NULL;
5631 return ret;
5632 }
5633 ceph_decode_64_safe(p, end, pii->snap_id, e_inval);
5634 return 0;
5635
5636e_inval:
5637 return -EINVAL;
5638}
5639
5640static int __get_parent_info(struct rbd_device *rbd_dev,
5641 struct page *req_page,
5642 struct page *reply_page,
5643 struct parent_image_info *pii)
5644{
5645 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
5646 size_t reply_len = PAGE_SIZE;
5647 void *p, *end;
5648 int ret;
5649
5650 ret = ceph_osdc_call(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc,
5651 "rbd", "parent_get", CEPH_OSD_FLAG_READ,
68ada915 5652 req_page, sizeof(u64), &reply_page, &reply_len);
e92c0eaf
ID		 5653 if (ret)
5654 return ret == -EOPNOTSUPP ? 1 : ret;
5655
5656 p = page_address(reply_page);
5657 end = p + reply_len;
5658 ret = decode_parent_image_spec(&p, end, pii);
5659 if (ret)
5660 return ret;
5661
5662 ret = ceph_osdc_call(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc,
5663 "rbd", "parent_overlap_get", CEPH_OSD_FLAG_READ,
68ada915 5664 req_page, sizeof(u64), &reply_page, &reply_len);
e92c0eaf
ID		 5665 if (ret)
5666 return ret;
5667
5668 p = page_address(reply_page);
5669 end = p + reply_len;
5670 ceph_decode_8_safe(&p, end, pii->has_overlap, e_inval);
5671 if (pii->has_overlap)
5672 ceph_decode_64_safe(&p, end, pii->overlap, e_inval);
5673
c1031177
ID		 5674 dout("%s pool_id %llu pool_ns %s image_id %s snap_id %llu has_overlap %d overlap %llu\n",
5675 __func__, pii->pool_id, pii->pool_ns, pii->image_id, pii->snap_id,
5676 pii->has_overlap, pii->overlap);
e92c0eaf
ID		 5677 return 0;
5678
5679e_inval:
5680 return -EINVAL;
5681}
5682
eb3b2d6b
ID		 5683/*
5684 * The caller is responsible for @pii.
5685 */
5686static int __get_parent_info_legacy(struct rbd_device *rbd_dev,
5687 struct page *req_page,
5688 struct page *reply_page,
5689 struct parent_image_info *pii)
5690{
5691 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
5692 size_t reply_len = PAGE_SIZE;
5693 void *p, *end;
5694 int ret;
5695
5696 ret = ceph_osdc_call(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc,
5697 "rbd", "get_parent", CEPH_OSD_FLAG_READ,
68ada915 5698 req_page, sizeof(u64), &reply_page, &reply_len);
eb3b2d6b
ID		 5699 if (ret)
5700 return ret;
5701
5702 p = page_address(reply_page);
5703 end = p + reply_len;
5704 ceph_decode_64_safe(&p, end, pii->pool_id, e_inval);
5705 pii->image_id = ceph_extract_encoded_string(&p, end, NULL, GFP_KERNEL);
5706 if (IS_ERR(pii->image_id)) {
5707 ret = PTR_ERR(pii->image_id);
5708 pii->image_id = NULL;
5709 return ret;
5710 }
5711 ceph_decode_64_safe(&p, end, pii->snap_id, e_inval);
e92c0eaf 5712 pii->has_overlap = true;
eb3b2d6b
ID		 5713 ceph_decode_64_safe(&p, end, pii->overlap, e_inval);
5714
c1031177
ID		 5715 dout("%s pool_id %llu pool_ns %s image_id %s snap_id %llu has_overlap %d overlap %llu\n",
5716 __func__, pii->pool_id, pii->pool_ns, pii->image_id, pii->snap_id,
5717 pii->has_overlap, pii->overlap);
eb3b2d6b
ID		 5718 return 0;
5719
5720e_inval:
5721 return -EINVAL;
5722}
5723
c1031177
ID		 5724static int rbd_dev_v2_parent_info(struct rbd_device *rbd_dev,
5725 struct parent_image_info *pii)
eb3b2d6b
ID		 5726{
5727 struct page *req_page, *reply_page;
5728 void *p;
5729 int ret;
5730
5731 req_page = alloc_page(GFP_KERNEL);
5732 if (!req_page)
5733 return -ENOMEM;
5734
5735 reply_page = alloc_page(GFP_KERNEL);
5736 if (!reply_page) {
5737 __free_page(req_page);
5738 return -ENOMEM;
5739 }
5740
5741 p = page_address(req_page);
5742 ceph_encode_64(&p, rbd_dev->spec->snap_id);
e92c0eaf
ID		 5743 ret = __get_parent_info(rbd_dev, req_page, reply_page, pii);
5744 if (ret > 0)
5745 ret = __get_parent_info_legacy(rbd_dev, req_page, reply_page,
5746 pii);
eb3b2d6b
ID		 5747
5748 __free_page(req_page);
5749 __free_page(reply_page);
5750 return ret;
5751}
5752
c1031177 5753static int rbd_dev_setup_parent(struct rbd_device *rbd_dev)
86b00e0d
AE		 5754{
5755 struct rbd_spec *parent_spec;
eb3b2d6b 5756 struct parent_image_info pii = { 0 };
86b00e0d
AE		 5757 int ret;
5758
5759 parent_spec = rbd_spec_alloc();
5760 if (!parent_spec)
5761 return -ENOMEM;
5762
c1031177 5763 ret = rbd_dev_v2_parent_info(rbd_dev, &pii);
eb3b2d6b 5764 if (ret)
86b00e0d 5765 goto out_err;
86b00e0d 5766
c1031177 5767 if (pii.pool_id == CEPH_NOPOOL || !pii.has_overlap)
86b00e0d
AE		 5768 goto out; /* No parent? No problem. */
5769
0903e875
AE		 5770 /* The ceph file layout needs to fit pool id in 32 bits */
5771
5772 ret = -EIO;
eb3b2d6b 5773 if (pii.pool_id > (u64)U32_MAX) {
9584d508 5774 rbd_warn(NULL, "parent pool id too large (%llu > %u)",
eb3b2d6b 5775 (unsigned long long)pii.pool_id, U32_MAX);
86b00e0d
AE		 5776 goto out_err;
5777 }
86b00e0d 5778
3b5cf2a2 5779 /*
c1031177
ID		 5780 * The parent won't change except when the clone is flattened,
5781 * so we only need to record the parent image spec once.
3b5cf2a2 5782 */
c1031177
ID		 5783 parent_spec->pool_id = pii.pool_id;
5784 if (pii.pool_ns && *pii.pool_ns) {
5785 parent_spec->pool_ns = pii.pool_ns;
5786 pii.pool_ns = NULL;
3b5cf2a2 5787 }
c1031177
ID		 5788 parent_spec->image_id = pii.image_id;
5789 pii.image_id = NULL;
5790 parent_spec->snap_id = pii.snap_id;
5791
5792 rbd_assert(!rbd_dev->parent_spec);
5793 rbd_dev->parent_spec = parent_spec;
5794 parent_spec = NULL; /* rbd_dev now owns this */
3b5cf2a2
AE		 5795
5796 /*
c1031177
ID		 5797 * Record the parent overlap. If it's zero, issue a warning as
5798 * we will proceed as if there is no parent.
3b5cf2a2 5799 */
c1031177
ID		 5800 if (!pii.overlap)
5801 rbd_warn(rbd_dev, "clone is standalone (overlap 0)");
eb3b2d6b 5802 rbd_dev->parent_overlap = pii.overlap;
cf32bd9c 5803
86b00e0d
AE		 5804out:
5805 ret = 0;
5806out_err:
c1031177 5807 rbd_parent_info_cleanup(&pii);
86b00e0d 5808 rbd_spec_put(parent_spec);
86b00e0d
AE		 5809 return ret;
5810}
5811
510a7330
ID		 5812static int rbd_dev_v2_striping_info(struct rbd_device *rbd_dev,
5813 u64 *stripe_unit, u64 *stripe_count)
cc070d59
AE		 5814{
5815 struct {
5816 __le64 stripe_unit;
5817 __le64 stripe_count;
5818 } __attribute__ ((packed)) striping_info_buf = { 0 };
5819 size_t size = sizeof (striping_info_buf);
cc070d59
AE		 5820 int ret;
5821
ecd4a68a
ID		 5822 ret = rbd_obj_method_sync(rbd_dev, &rbd_dev->header_oid,
5823 &rbd_dev->header_oloc, "get_stripe_unit_count",
5824 NULL, 0, &striping_info_buf, size);
cc070d59
AE		 5825 dout("%s: rbd_obj_method_sync returned %d\n", __func__, ret);
5826 if (ret < 0)
5827 return ret;
5828 if (ret < size)
5829 return -ERANGE;
5830
510a7330
ID		 5831 *stripe_unit = le64_to_cpu(striping_info_buf.stripe_unit);
5832 *stripe_count = le64_to_cpu(striping_info_buf.stripe_count);
5833 dout(" stripe_unit = %llu stripe_count = %llu\n", *stripe_unit,
5834 *stripe_count);
5835
cc070d59
AE		 5836 return 0;
5837}
5838
510a7330 5839static int rbd_dev_v2_data_pool(struct rbd_device *rbd_dev, s64 *data_pool_id)
7e97332e 5840{
510a7330 5841 __le64 data_pool_buf;
7e97332e
ID		 5842 int ret;
5843
5844 ret = rbd_obj_method_sync(rbd_dev, &rbd_dev->header_oid,
5845 &rbd_dev->header_oloc, "get_data_pool",
510a7330
ID		 5846 NULL, 0, &data_pool_buf,
5847 sizeof(data_pool_buf));
5848 dout("%s: rbd_obj_method_sync returned %d\n", __func__, ret);
7e97332e
ID		 5849 if (ret < 0)
5850 return ret;
510a7330 5851 if (ret < sizeof(data_pool_buf))
7e97332e
ID		 5852 return -EBADMSG;
5853
510a7330
ID		 5854 *data_pool_id = le64_to_cpu(data_pool_buf);
5855 dout(" data_pool_id = %lld\n", *data_pool_id);
5856 WARN_ON(*data_pool_id == CEPH_NOPOOL);
5857
7e97332e
ID		 5858 return 0;
5859}
5860
9e15b77d
AE		 5861static char *rbd_dev_image_name(struct rbd_device *rbd_dev)
5862{
ecd4a68a 5863 CEPH_DEFINE_OID_ONSTACK(oid);
9e15b77d
AE		 5864 size_t image_id_size;
5865 char *image_id;
5866 void *p;
5867 void *end;
5868 size_t size;
5869 void *reply_buf = NULL;
5870 size_t len = 0;
5871 char *image_name = NULL;
5872 int ret;
5873
5874 rbd_assert(!rbd_dev->spec->image_name);
5875
69e7a02f
AE		 5876 len = strlen(rbd_dev->spec->image_id);
5877 image_id_size = sizeof (__le32) + len;
9e15b77d
AE		 5878 image_id = kmalloc(image_id_size, GFP_KERNEL);
5879 if (!image_id)
5880 return NULL;
5881
5882 p = image_id;
4157976b 5883 end = image_id + image_id_size;
57385b51 5884 ceph_encode_string(&p, end, rbd_dev->spec->image_id, (u32)len);
9e15b77d
AE		 5885
5886 size = sizeof (__le32) + RBD_IMAGE_NAME_LEN_MAX;
5887 reply_buf = kmalloc(size, GFP_KERNEL);
5888 if (!reply_buf)
5889 goto out;
5890
ecd4a68a
ID		 5891 ceph_oid_printf(&oid, "%s", RBD_DIRECTORY);
5892 ret = rbd_obj_method_sync(rbd_dev, &oid, &rbd_dev->header_oloc,
5893 "dir_get_name", image_id, image_id_size,
5894 reply_buf, size);
9e15b77d
AE		 5895 if (ret < 0)
5896 goto out;
5897 p = reply_buf;
f40eb349
AE		 5898 end = reply_buf + ret;
5899
9e15b77d
AE		 5900 image_name = ceph_extract_encoded_string(&p, end, &len, GFP_KERNEL);
5901 if (IS_ERR(image_name))
5902 image_name = NULL;
5903 else
5904 dout("%s: name is %s len is %zd\n", __func__, image_name, len);
5905out:
5906 kfree(reply_buf);
5907 kfree(image_id);
5908
5909 return image_name;
5910}
5911
2ad3d716
AE		 5912static u64 rbd_v1_snap_id_by_name(struct rbd_device *rbd_dev, const char *name)
5913{
5914 struct ceph_snap_context *snapc = rbd_dev->header.snapc;
5915 const char *snap_name;
5916 u32 which = 0;
5917
5918 /* Skip over names until we find the one we are looking for */
5919
5920 snap_name = rbd_dev->header.snap_names;
5921 while (which < snapc->num_snaps) {
5922 if (!strcmp(name, snap_name))
5923 return snapc->snaps[which];
5924 snap_name += strlen(snap_name) + 1;
5925 which++;
5926 }
5927 return CEPH_NOSNAP;
5928}
5929
5930static u64 rbd_v2_snap_id_by_name(struct rbd_device *rbd_dev, const char *name)
5931{
5932 struct ceph_snap_context *snapc = rbd_dev->header.snapc;
5933 u32 which;
5934 bool found = false;
5935 u64 snap_id;
5936
5937 for (which = 0; !found && which < snapc->num_snaps; which++) {
5938 const char *snap_name;
5939
5940 snap_id = snapc->snaps[which];
5941 snap_name = rbd_dev_v2_snap_name(rbd_dev, snap_id);
efadc98a
JD		 5942 if (IS_ERR(snap_name)) {
5943 /* ignore no-longer existing snapshots */
5944 if (PTR_ERR(snap_name) == -ENOENT)
5945 continue;
5946 else
5947 break;
5948 }
2ad3d716
AE		 5949 found = !strcmp(name, snap_name);
5950 kfree(snap_name);
5951 }
5952 return found ? snap_id : CEPH_NOSNAP;
5953}
5954
5955/*
5956 * Assumes name is never RBD_SNAP_HEAD_NAME; returns CEPH_NOSNAP if
5957 * no snapshot by that name is found, or if an error occurs.
5958 */
5959static u64 rbd_snap_id_by_name(struct rbd_device *rbd_dev, const char *name)
5960{
5961 if (rbd_dev->image_format == 1)
5962 return rbd_v1_snap_id_by_name(rbd_dev, name);
5963
5964 return rbd_v2_snap_id_by_name(rbd_dev, name);
5965}
5966
9e15b77d 5967/*
04077599
ID		 5968 * An image being mapped will have everything but the snap id.
5969 */
5970static int rbd_spec_fill_snap_id(struct rbd_device *rbd_dev)
5971{
5972 struct rbd_spec *spec = rbd_dev->spec;
5973
5974 rbd_assert(spec->pool_id != CEPH_NOPOOL && spec->pool_name);
5975 rbd_assert(spec->image_id && spec->image_name);
5976 rbd_assert(spec->snap_name);
5977
5978 if (strcmp(spec->snap_name, RBD_SNAP_HEAD_NAME)) {
5979 u64 snap_id;
5980
5981 snap_id = rbd_snap_id_by_name(rbd_dev, spec->snap_name);
5982 if (snap_id == CEPH_NOSNAP)
5983 return -ENOENT;
5984
5985 spec->snap_id = snap_id;
5986 } else {
5987 spec->snap_id = CEPH_NOSNAP;
5988 }
5989
5990 return 0;
5991}
5992
5993/*
5994 * A parent image will have all ids but none of the names.
e1d4213f 5995 *
04077599
ID		 5996 * All names in an rbd spec are dynamically allocated. It's OK if we
5997 * can't figure out the name for an image id.
9e15b77d 5998 */
04077599 5999static int rbd_spec_fill_names(struct rbd_device *rbd_dev)
9e15b77d 6000{
2e9f7f1c
AE		 6001 struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
6002 struct rbd_spec *spec = rbd_dev->spec;
6003 const char *pool_name;
6004 const char *image_name;
6005 const char *snap_name;
9e15b77d
AE		 6006 int ret;
6007
04077599
ID		 6008 rbd_assert(spec->pool_id != CEPH_NOPOOL);
6009 rbd_assert(spec->image_id);
6010 rbd_assert(spec->snap_id != CEPH_NOSNAP);
9e15b77d 6011
2e9f7f1c 6012 /* Get the pool name; we have to make our own copy of this */
9e15b77d 6013
2e9f7f1c
AE		 6014 pool_name = ceph_pg_pool_name_by_id(osdc->osdmap, spec->pool_id);
6015 if (!pool_name) {
6016 rbd_warn(rbd_dev, "no pool with id %llu", spec->pool_id);
935dc89f
AE		 6017 return -EIO;
6018 }
2e9f7f1c
AE		 6019 pool_name = kstrdup(pool_name, GFP_KERNEL);
6020 if (!pool_name)
9e15b77d
AE		 6021 return -ENOMEM;
6022
6023 /* Fetch the image name; tolerate failure here */
6024
2e9f7f1c
AE		 6025 image_name = rbd_dev_image_name(rbd_dev);
6026 if (!image_name)
06ecc6cb 6027 rbd_warn(rbd_dev, "unable to get image name");
9e15b77d 6028
04077599 6029 /* Fetch the snapshot name */
9e15b77d 6030
2e9f7f1c 6031 snap_name = rbd_snap_name(rbd_dev, spec->snap_id);
da6a6b63
JD		 6032 if (IS_ERR(snap_name)) {
6033 ret = PTR_ERR(snap_name);
9e15b77d 6034 goto out_err;
2e9f7f1c
AE		 6035 }
6036
6037 spec->pool_name = pool_name;
6038 spec->image_name = image_name;
6039 spec->snap_name = snap_name;
9e15b77d
AE		 6040
6041 return 0;
04077599 6042
9e15b77d 6043out_err:
2e9f7f1c
AE		 6044 kfree(image_name);
6045 kfree(pool_name);
9e15b77d
AE		 6046 return ret;
6047}
6048
510a7330
ID		 6049static int rbd_dev_v2_snap_context(struct rbd_device *rbd_dev,
6050 struct ceph_snap_context **psnapc)
35d489f9
AE		 6051{
6052 size_t size;
6053 int ret;
6054 void *reply_buf;
6055 void *p;
6056 void *end;
6057 u64 seq;
6058 u32 snap_count;
6059 struct ceph_snap_context *snapc;
6060 u32 i;
6061
6062 /*
6063 * We'll need room for the seq value (maximum snapshot id),
6064 * snapshot count, and array of that many snapshot ids.
6065 * For now we have a fixed upper limit on the number we're
6066 * prepared to receive.
6067 */
6068 size = sizeof (__le64) + sizeof (__le32) +
6069 RBD_MAX_SNAP_COUNT * sizeof (__le64);
6070 reply_buf = kzalloc(size, GFP_KERNEL);
6071 if (!reply_buf)
6072 return -ENOMEM;
6073
ecd4a68a
ID		 6074 ret = rbd_obj_method_sync(rbd_dev, &rbd_dev->header_oid,
6075 &rbd_dev->header_oloc, "get_snapcontext",
6076 NULL, 0, reply_buf, size);
36be9a76 6077 dout("%s: rbd_obj_method_sync returned %d\n", __func__, ret);
35d489f9
AE		 6078 if (ret < 0)
6079 goto out;
6080
35d489f9 6081 p = reply_buf;
57385b51
AE		 6082 end = reply_buf + ret;
6083 ret = -ERANGE;
35d489f9
AE		 6084 ceph_decode_64_safe(&p, end, seq, out);
6085 ceph_decode_32_safe(&p, end, snap_count, out);
6086
6087 /*
6088 * Make sure the reported number of snapshot ids wouldn't go
6089 * beyond the end of our buffer. But before checking that,
6090 * make sure the computed size of the snapshot context we
6091 * allocate is representable in a size_t.
6092 */
6093 if (snap_count > (SIZE_MAX - sizeof (struct ceph_snap_context))
6094 / sizeof (u64)) {
6095 ret = -EINVAL;
6096 goto out;
6097 }
6098 if (!ceph_has_room(&p, end, snap_count * sizeof (__le64)))
6099 goto out;
468521c1 6100 ret = 0;
35d489f9 6101
812164f8 6102 snapc = ceph_create_snap_context(snap_count, GFP_KERNEL);
35d489f9
AE		 6103 if (!snapc) {
6104 ret = -ENOMEM;
6105 goto out;
6106 }
35d489f9 6107 snapc->seq = seq;
35d489f9
AE		 6108 for (i = 0; i < snap_count; i++)
6109 snapc->snaps[i] = ceph_decode_64(&p);
6110
510a7330 6111 *psnapc = snapc;
35d489f9 6112 dout(" snap context seq = %llu, snap_count = %u\n",
57385b51 6113 (unsigned long long)seq, (unsigned int)snap_count);
35d489f9
AE		 6114out:
6115 kfree(reply_buf);
6116
57385b51 6117 return ret;
35d489f9
AE		 6118}
6119
54cac61f
AE		 6120static const char *rbd_dev_v2_snap_name(struct rbd_device *rbd_dev,
6121 u64 snap_id)
b8b1e2db
AE		 6122{
6123 size_t size;
6124 void *reply_buf;
54cac61f 6125 __le64 snapid;
b8b1e2db
AE		 6126 int ret;
6127 void *p;
6128 void *end;
b8b1e2db
AE		 6129 char *snap_name;
6130
6131 size = sizeof (__le32) + RBD_MAX_SNAP_NAME_LEN;
6132 reply_buf = kmalloc(size, GFP_KERNEL);
6133 if (!reply_buf)
6134 return ERR_PTR(-ENOMEM);
6135
54cac61f 6136 snapid = cpu_to_le64(snap_id);
ecd4a68a
ID		 6137 ret = rbd_obj_method_sync(rbd_dev, &rbd_dev->header_oid,
6138 &rbd_dev->header_oloc, "get_snapshot_name",
6139 &snapid, sizeof(snapid), reply_buf, size);
36be9a76 6140 dout("%s: rbd_obj_method_sync returned %d\n", __func__, ret);
f40eb349
AE		 6141 if (ret < 0) {
6142 snap_name = ERR_PTR(ret);
b8b1e2db 6143 goto out;
f40eb349 6144 }
b8b1e2db
AE		 6145
6146 p = reply_buf;
f40eb349 6147 end = reply_buf + ret;
e5c35534 6148 snap_name = ceph_extract_encoded_string(&p, end, NULL, GFP_KERNEL);
f40eb349 6149 if (IS_ERR(snap_name))
b8b1e2db 6150 goto out;
b8b1e2db 6151
f40eb349 6152 dout(" snap_id 0x%016llx snap_name = %s\n",
54cac61f 6153 (unsigned long long)snap_id, snap_name);
b8b1e2db
AE		 6154out:
6155 kfree(reply_buf);
6156
f40eb349 6157 return snap_name;
b8b1e2db
AE		 6158}
6159
510a7330
ID		 6160static int rbd_dev_v2_header_info(struct rbd_device *rbd_dev,
6161 struct rbd_image_header *header,
6162 bool first_time)
117973fb
AE		 6163{
6164 int ret;
117973fb 6165
510a7330
ID		 6166 ret = _rbd_dev_v2_snap_size(rbd_dev, CEPH_NOSNAP,
6167 first_time ? &header->obj_order : NULL,
6168 &header->image_size);
1617e40c 6169 if (ret)
cfbf6377 6170 return ret;
1617e40c 6171
2df3fac7 6172 if (first_time) {
510a7330 6173 ret = rbd_dev_v2_header_onetime(rbd_dev, header);
2df3fac7 6174 if (ret)
cfbf6377 6175 return ret;
2df3fac7
AE		 6176 }
6177
510a7330
ID		 6178 ret = rbd_dev_v2_snap_context(rbd_dev, &header->snapc);
6179 if (ret)
6180 return ret;
117973fb 6181
510a7330 6182 return 0;
117973fb
AE		 6183}
6184
510a7330
ID		 6185static int rbd_dev_header_info(struct rbd_device *rbd_dev,
6186 struct rbd_image_header *header,
6187 bool first_time)
a720ae09
ID		 6188{
6189 rbd_assert(rbd_image_format_valid(rbd_dev->image_format));
510a7330 6190 rbd_assert(!header->object_prefix && !header->snapc);
a720ae09
ID		 6191
6192 if (rbd_dev->image_format == 1)
510a7330 6193 return rbd_dev_v1_header_info(rbd_dev, header, first_time);
a720ae09 6194
510a7330 6195 return rbd_dev_v2_header_info(rbd_dev, header, first_time);
a720ae09
ID		 6196}
6197
e28fff26
AE		 6198/*
6199 * Skips over white space at *buf, and updates *buf to point to the
6200 * first found non-space character (if any). Returns the length of
593a9e7b
AE		 6201 * the token (string of non-white space characters) found. Note
6202 * that *buf must be terminated with '\0'.
e28fff26
AE		 6203 */
6204static inline size_t next_token(const char **buf)
6205{
6206 /*
6207 * These are the characters that produce nonzero for
6208 * isspace() in the "C" and "POSIX" locales.
6209 */
435a120a 6210 static const char spaces[] = " \f\n\r\t\v";
e28fff26
AE		 6211
6212 *buf += strspn(*buf, spaces); /* Find start of token */
6213
6214 return strcspn(*buf, spaces); /* Return token length */
6215}
6216
ea3352f4
AE		 6217/*
6218 * Finds the next token in *buf, dynamically allocates a buffer big
6219 * enough to hold a copy of it, and copies the token into the new
6220 * buffer. The copy is guaranteed to be terminated with '\0'. Note
6221 * that a duplicate buffer is created even for a zero-length token.
6222 *
6223 * Returns a pointer to the newly-allocated duplicate, or a null
6224 * pointer if memory for the duplicate was not available. If
6225 * the lenp argument is a non-null pointer, the length of the token
6226 * (not including the '\0') is returned in *lenp.
6227 *
6228 * If successful, the *buf pointer will be updated to point beyond
6229 * the end of the found token.
6230 *
6231 * Note: uses GFP_KERNEL for allocation.
6232 */
6233static inline char *dup_token(const char **buf, size_t *lenp)
6234{
6235 char *dup;
6236 size_t len;
6237
6238 len = next_token(buf);
4caf35f9 6239 dup = kmemdup(*buf, len + 1, GFP_KERNEL);
ea3352f4
AE		 6240 if (!dup)
6241 return NULL;
ea3352f4
AE		 6242 *(dup + len) = '\0';
6243 *buf += len;
6244
6245 if (lenp)
6246 *lenp = len;
6247
6248 return dup;
6249}
6250
82995cc6
DH		 6251static int rbd_parse_param(struct fs_parameter *param,
6252 struct rbd_parse_opts_ctx *pctx)
6253{
6254 struct rbd_options *opt = pctx->opts;
6255 struct fs_parse_result result;
3fbb8d55 6256 struct p_log log = {.prefix = "rbd"};
82995cc6
DH		 6257 int token, ret;
6258
6259 ret = ceph_parse_param(param, pctx->copts, NULL);
6260 if (ret != -ENOPARAM)
6261 return ret;
6262
d7167b14 6263 token = __fs_parse(&log, rbd_parameters, param, &result);
82995cc6
DH		 6264 dout("%s fs_parse '%s' token %d\n", __func__, param->key, token);
6265 if (token < 0) {
2c3f3dc3
AV		 6266 if (token == -ENOPARAM)
6267 return inval_plog(&log, "Unknown parameter '%s'",
6268 param->key);
82995cc6
DH		 6269 return token;
6270 }
6271
6272 switch (token) {
6273 case Opt_queue_depth:
6274 if (result.uint_32 < 1)
6275 goto out_of_range;
6276 opt->queue_depth = result.uint_32;
6277 break;
6278 case Opt_alloc_size:
6279 if (result.uint_32 < SECTOR_SIZE)
6280 goto out_of_range;
2c3f3dc3
AV		 6281 if (!is_power_of_2(result.uint_32))
6282 return inval_plog(&log, "alloc_size must be a power of 2");
82995cc6
DH		 6283 opt->alloc_size = result.uint_32;
6284 break;
6285 case Opt_lock_timeout:
6286 /* 0 is "wait forever" (i.e. infinite timeout) */
6287 if (result.uint_32 > INT_MAX / 1000)
6288 goto out_of_range;
6289 opt->lock_timeout = msecs_to_jiffies(result.uint_32 * 1000);
6290 break;
6291 case Opt_pool_ns:
6292 kfree(pctx->spec->pool_ns);
6293 pctx->spec->pool_ns = param->string;
6294 param->string = NULL;
6295 break;
dc1dad8e
ID		 6296 case Opt_compression_hint:
6297 switch (result.uint_32) {
6298 case Opt_compression_hint_none:
6299 opt->alloc_hint_flags &=
6300 ~(CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE |
6301 CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE);
6302 break;
6303 case Opt_compression_hint_compressible:
6304 opt->alloc_hint_flags |=
6305 CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE;
6306 opt->alloc_hint_flags &=
6307 ~CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE;
6308 break;
6309 case Opt_compression_hint_incompressible:
6310 opt->alloc_hint_flags |=
6311 CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE;
6312 opt->alloc_hint_flags &=
6313 ~CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE;
6314 break;
6315 default:
6316 BUG();
6317 }
6318 break;
82995cc6
DH		 6319 case Opt_read_only:
6320 opt->read_only = true;
6321 break;
6322 case Opt_read_write:
6323 opt->read_only = false;
6324 break;
6325 case Opt_lock_on_read:
6326 opt->lock_on_read = true;
6327 break;
6328 case Opt_exclusive:
6329 opt->exclusive = true;
6330 break;
6331 case Opt_notrim:
6332 opt->trim = false;
6333 break;
6334 default:
6335 BUG();
6336 }
6337
6338 return 0;
6339
6340out_of_range:
2c3f3dc3 6341 return inval_plog(&log, "%s out of range", param->key);
82995cc6
DH		 6342}
6343
6344/*
6345 * This duplicates most of generic_parse_monolithic(), untying it from
6346 * fs_context and skipping standard superblock and security options.
6347 */
6348static int rbd_parse_options(char *options, struct rbd_parse_opts_ctx *pctx)
6349{
6350 char *key;
6351 int ret = 0;
6352
6353 dout("%s '%s'\n", __func__, options);
6354 while ((key = strsep(&options, ",")) != NULL) {
6355 if (*key) {
6356 struct fs_parameter param = {
6357 .key = key,
0f89589a 6358 .type = fs_value_is_flag,
82995cc6
DH		 6359 };
6360 char *value = strchr(key, '=');
6361 size_t v_len = 0;
6362
6363 if (value) {
6364 if (value == key)
6365 continue;
6366 *value++ = 0;
6367 v_len = strlen(value);
82995cc6
DH		 6368 param.string = kmemdup_nul(value, v_len,
6369 GFP_KERNEL);
6370 if (!param.string)
6371 return -ENOMEM;
0f89589a 6372 param.type = fs_value_is_string;
82995cc6
DH		 6373 }
6374 param.size = v_len;
6375
6376 ret = rbd_parse_param(&param, pctx);
6377 kfree(param.string);
6378 if (ret)
6379 break;
6380 }
6381 }
6382
6383 return ret;
6384}
6385
a725f65e 6386/*
859c31df
AE		 6387 * Parse the options provided for an "rbd add" (i.e., rbd image
6388 * mapping) request. These arrive via a write to /sys/bus/rbd/add,
6389 * and the data written is passed here via a NUL-terminated buffer.
6390 * Returns 0 if successful or an error code otherwise.
d22f76e7 6391 *
859c31df
AE		 6392 * The information extracted from these options is recorded in
6393 * the other parameters which return dynamically-allocated
6394 * structures:
6395 * ceph_opts
6396 * The address of a pointer that will refer to a ceph options
6397 * structure. Caller must release the returned pointer using
6398 * ceph_destroy_options() when it is no longer needed.
6399 * rbd_opts
6400 * Address of an rbd options pointer. Fully initialized by
6401 * this function; caller must release with kfree().
6402 * spec
6403 * Address of an rbd image specification pointer. Fully
6404 * initialized by this function based on parsed options.
6405 * Caller must release with rbd_spec_put().
6406 *
6407 * The options passed take this form:
6408 * <mon_addrs> <options> <pool_name> <image_name> [<snap_id>]
6409 * where:
6410 * <mon_addrs>
6411 * A comma-separated list of one or more monitor addresses.
6412 * A monitor address is an ip address, optionally followed
6413 * by a port number (separated by a colon).
6414 * I.e.: ip1[:port1][,ip2[:port2]...]
6415 * <options>
6416 * A comma-separated list of ceph and/or rbd options.
6417 * <pool_name>
6418 * The name of the rados pool containing the rbd image.
6419 * <image_name>
6420 * The name of the image in that pool to map.
6421 * <snap_id>
6422 * An optional snapshot id. If provided, the mapping will
6423 * present data from the image at the time that snapshot was
6424 * created. The image head is used if no snapshot id is
6425 * provided. Snapshot mappings are always read-only.
a725f65e 6426 */
859c31df 6427static int rbd_add_parse_args(const char *buf,
dc79b113 6428 struct ceph_options **ceph_opts,
859c31df
AE		 6429 struct rbd_options **opts,
6430 struct rbd_spec **rbd_spec)
e28fff26 6431{
d22f76e7 6432 size_t len;
859c31df 6433 char *options;
0ddebc0c 6434 const char *mon_addrs;
ecb4dc22 6435 char *snap_name;
0ddebc0c 6436 size_t mon_addrs_size;
82995cc6 6437 struct rbd_parse_opts_ctx pctx = { 0 };
dc79b113 6438 int ret;
e28fff26
AE		 6439
6440 /* The first four tokens are required */
6441
7ef3214a 6442 len = next_token(&buf);
4fb5d671
AE		 6443 if (!len) {
6444 rbd_warn(NULL, "no monitor address(es) provided");
6445 return -EINVAL;
6446 }
0ddebc0c 6447 mon_addrs = buf;
82995cc6 6448 mon_addrs_size = len;
7ef3214a 6449 buf += len;
a725f65e 6450
dc79b113 6451 ret = -EINVAL;
f28e565a
AE		 6452 options = dup_token(&buf, NULL);
6453 if (!options)
dc79b113 6454 return -ENOMEM;
4fb5d671
AE		 6455 if (!*options) {
6456 rbd_warn(NULL, "no options provided");
6457 goto out_err;
6458 }
e28fff26 6459
c300156b
ID		 6460 pctx.spec = rbd_spec_alloc();
6461 if (!pctx.spec)
f28e565a 6462 goto out_mem;
859c31df 6463
c300156b
ID		 6464 pctx.spec->pool_name = dup_token(&buf, NULL);
6465 if (!pctx.spec->pool_name)
859c31df 6466 goto out_mem;
c300156b 6467 if (!*pctx.spec->pool_name) {
4fb5d671
AE		 6468 rbd_warn(NULL, "no pool name provided");
6469 goto out_err;
6470 }
e28fff26 6471
c300156b
ID		 6472 pctx.spec->image_name = dup_token(&buf, NULL);
6473 if (!pctx.spec->image_name)
f28e565a 6474 goto out_mem;
c300156b 6475 if (!*pctx.spec->image_name) {
4fb5d671
AE		 6476 rbd_warn(NULL, "no image name provided");
6477 goto out_err;
6478 }
d4b125e9 6479
f28e565a
AE		 6480 /*
6481 * Snapshot name is optional; default is to use "-"
6482 * (indicating the head/no snapshot).
6483 */
3feeb894 6484 len = next_token(&buf);
820a5f3e 6485 if (!len) {
3feeb894
AE		 6486 buf = RBD_SNAP_HEAD_NAME; /* No snapshot supplied */
6487 len = sizeof (RBD_SNAP_HEAD_NAME) - 1;
f28e565a 6488 } else if (len > RBD_MAX_SNAP_NAME_LEN) {
dc79b113 6489 ret = -ENAMETOOLONG;
f28e565a 6490 goto out_err;
849b4260 6491 }
ecb4dc22
AE		 6492 snap_name = kmemdup(buf, len + 1, GFP_KERNEL);
6493 if (!snap_name)
f28e565a 6494 goto out_mem;
ecb4dc22 6495 *(snap_name + len) = '\0';
c300156b 6496 pctx.spec->snap_name = snap_name;
e5c35534 6497
82995cc6
DH		 6498 pctx.copts = ceph_alloc_options();
6499 if (!pctx.copts)
6500 goto out_mem;
6501
0ddebc0c 6502 /* Initialize all rbd options to the defaults */
e28fff26 6503
c300156b
ID		 6504 pctx.opts = kzalloc(sizeof(*pctx.opts), GFP_KERNEL);
6505 if (!pctx.opts)
4e9afeba
AE		 6506 goto out_mem;
6507
c300156b
ID		 6508 pctx.opts->read_only = RBD_READ_ONLY_DEFAULT;
6509 pctx.opts->queue_depth = RBD_QUEUE_DEPTH_DEFAULT;
0c93e1b7 6510 pctx.opts->alloc_size = RBD_ALLOC_SIZE_DEFAULT;
c300156b
ID		 6511 pctx.opts->lock_timeout = RBD_LOCK_TIMEOUT_DEFAULT;
6512 pctx.opts->lock_on_read = RBD_LOCK_ON_READ_DEFAULT;
6513 pctx.opts->exclusive = RBD_EXCLUSIVE_DEFAULT;
6514 pctx.opts->trim = RBD_TRIM_DEFAULT;
d22f76e7 6515
2d7c86a8
VS		 6516 ret = ceph_parse_mon_ips(mon_addrs, mon_addrs_size, pctx.copts, NULL,
6517 ',');
82995cc6 6518 if (ret)
dc79b113 6519 goto out_err;
859c31df 6520
82995cc6
DH		 6521 ret = rbd_parse_options(options, &pctx);
6522 if (ret)
6523 goto out_err;
6524
6525 *ceph_opts = pctx.copts;
c300156b
ID		 6526 *opts = pctx.opts;
6527 *rbd_spec = pctx.spec;
82995cc6 6528 kfree(options);
dc79b113 6529 return 0;
82995cc6 6530
f28e565a 6531out_mem:
dc79b113 6532 ret = -ENOMEM;
d22f76e7 6533out_err:
c300156b 6534 kfree(pctx.opts);
82995cc6 6535 ceph_destroy_options(pctx.copts);
c300156b 6536 rbd_spec_put(pctx.spec);
f28e565a 6537 kfree(options);
dc79b113 6538 return ret;
a725f65e
AE		 6539}
6540
e010dd0a
ID		 6541static void rbd_dev_image_unlock(struct rbd_device *rbd_dev)
6542{
6543 down_write(&rbd_dev->lock_rwsem);
6544 if (__rbd_is_lock_owner(rbd_dev))
e1fddc8f 6545 __rbd_release_lock(rbd_dev);
e010dd0a
ID		 6546 up_write(&rbd_dev->lock_rwsem);
6547}
6548
637cd060
ID		 6549/*
6550 * If the wait is interrupted, an error is returned even if the lock
6551 * was successfully acquired. rbd_dev_image_unlock() will release it
6552 * if needed.
6553 */
e010dd0a
ID		 6554static int rbd_add_acquire_lock(struct rbd_device *rbd_dev)
6555{
637cd060 6556 long ret;
2f18d466 6557
e010dd0a 6558 if (!(rbd_dev->header.features & RBD_FEATURE_EXCLUSIVE_LOCK)) {
637cd060
ID		 6559 if (!rbd_dev->opts->exclusive && !rbd_dev->opts->lock_on_read)
6560 return 0;
6561
e010dd0a
ID		 6562 rbd_warn(rbd_dev, "exclusive-lock feature is not enabled");
6563 return -EINVAL;
6564 }
6565
3fe69921 6566 if (rbd_is_ro(rbd_dev))
637cd060
ID		 6567 return 0;
6568
6569 rbd_assert(!rbd_is_lock_owner(rbd_dev));
6570 queue_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0);
6571 ret = wait_for_completion_killable_timeout(&rbd_dev->acquire_wait,
6572 ceph_timeout_jiffies(rbd_dev->opts->lock_timeout));
25e6be21 6573 if (ret > 0) {
637cd060 6574 ret = rbd_dev->acquire_err;
25e6be21
DY		 6575 } else {
6576 cancel_delayed_work_sync(&rbd_dev->lock_dwork);
6577 if (!ret)
6578 ret = -ETIMEDOUT;
637cd060 6579
9d01e07f 6580 rbd_warn(rbd_dev, "failed to acquire lock: %ld", ret);
e010dd0a 6581 }
9d01e07f
ID		 6582 if (ret)
6583 return ret;
e010dd0a 6584
637cd060
ID		 6585 /*
6586 * The lock may have been released by now, unless automatic lock
6587 * transitions are disabled.
6588 */
6589 rbd_assert(!rbd_dev->opts->exclusive || rbd_is_lock_owner(rbd_dev));
e010dd0a
ID		 6590 return 0;
6591}
6592
589d30e0
AE		 6593/*
6594 * An rbd format 2 image has a unique identifier, distinct from the
6595 * name given to it by the user. Internally, that identifier is
6596 * what's used to specify the names of objects related to the image.
6597 *
6598 * A special "rbd id" object is used to map an rbd image name to its
6599 * id. If that object doesn't exist, then there is no v2 rbd image
6600 * with the supplied name.
6601 *
6602 * This function will record the given rbd_dev's image_id field if
6603 * it can be determined, and in that case will return 0. If any
6604 * errors occur a negative errno will be returned and the rbd_dev's
6605 * image_id field will be unchanged (and should be NULL).
6606 */
6607static int rbd_dev_image_id(struct rbd_device *rbd_dev)
6608{
6609 int ret;
6610 size_t size;
ecd4a68a 6611 CEPH_DEFINE_OID_ONSTACK(oid);
589d30e0 6612 void *response;
c0fba368 6613 char *image_id;
2f82ee54 6614
2c0d0a10
AE		 6615 /*
6616 * When probing a parent image, the image id is already
6617 * known (and the image name likely is not). There's no
c0fba368
AE		 6618 * need to fetch the image id again in this case. We
6619 * do still need to set the image format though.
2c0d0a10 6620 */
c0fba368
AE		 6621 if (rbd_dev->spec->image_id) {
6622 rbd_dev->image_format = *rbd_dev->spec->image_id ? 2 : 1;
6623
2c0d0a10 6624 return 0;
c0fba368 6625 }
2c0d0a10 6626
589d30e0
AE		 6627 /*
6628 * First, see if the format 2 image id file exists, and if
6629 * so, get the image's persistent id from it.
6630 */
ecd4a68a
ID		 6631 ret = ceph_oid_aprintf(&oid, GFP_KERNEL, "%s%s", RBD_ID_PREFIX,
6632 rbd_dev->spec->image_name);
6633 if (ret)
6634 return ret;
6635
6636 dout("rbd id object name is %s\n", oid.name);
589d30e0
AE		 6637
6638 /* Response will be an encoded string, which includes a length */
589d30e0
AE		 6639 size = sizeof (__le32) + RBD_IMAGE_ID_LEN_MAX;
6640 response = kzalloc(size, GFP_NOIO);
6641 if (!response) {
6642 ret = -ENOMEM;
6643 goto out;
6644 }
6645
c0fba368
AE		 6646 /* If it doesn't exist we'll assume it's a format 1 image */
6647
ecd4a68a
ID		 6648 ret = rbd_obj_method_sync(rbd_dev, &oid, &rbd_dev->header_oloc,
6649 "get_id", NULL, 0,
5435d206 6650 response, size);
36be9a76 6651 dout("%s: rbd_obj_method_sync returned %d\n", __func__, ret);
c0fba368
AE		 6652 if (ret == -ENOENT) {
6653 image_id = kstrdup("", GFP_KERNEL);
6654 ret = image_id ? 0 : -ENOMEM;
6655 if (!ret)
6656 rbd_dev->image_format = 1;
7dd440c9 6657 } else if (ret >= 0) {
c0fba368
AE		 6658 void *p = response;
6659
6660 image_id = ceph_extract_encoded_string(&p, p + ret,
979ed480 6661 NULL, GFP_NOIO);
461f758a 6662 ret = PTR_ERR_OR_ZERO(image_id);
c0fba368
AE		 6663 if (!ret)
6664 rbd_dev->image_format = 2;
c0fba368
AE		 6665 }
6666
6667 if (!ret) {
6668 rbd_dev->spec->image_id = image_id;
6669 dout("image_id is %s\n", image_id);
589d30e0
AE		 6670 }
6671out:
6672 kfree(response);
ecd4a68a 6673 ceph_oid_destroy(&oid);
589d30e0
AE		 6674 return ret;
6675}
6676
3abef3b3
AE		 6677/*
6678 * Undo whatever state changes are made by v1 or v2 header info
6679 * call.
6680 */
6fd48b3b
AE		 6681static void rbd_dev_unprobe(struct rbd_device *rbd_dev)
6682{
e69b8d41 6683 rbd_dev_parent_put(rbd_dev);
22e8bd51 6684 rbd_object_map_free(rbd_dev);
da5ef6be 6685 rbd_dev_mapping_clear(rbd_dev);
6fd48b3b
AE		 6686
6687 /* Free dynamic fields from the header, then zero it out */
6688
510a7330 6689 rbd_image_header_cleanup(&rbd_dev->header);
6fd48b3b
AE		 6690}
6691
510a7330
ID		 6692static int rbd_dev_v2_header_onetime(struct rbd_device *rbd_dev,
6693 struct rbd_image_header *header)
a30b71b9
AE		 6694{
6695 int ret;
a30b71b9 6696
510a7330 6697 ret = rbd_dev_v2_object_prefix(rbd_dev, &header->object_prefix);
57385b51 6698 if (ret)
510a7330 6699 return ret;
b1b5402a 6700
2df3fac7
AE		 6701 /*
6702 * Get the and check features for the image. Currently the
6703 * features are assumed to never change.
6704 */
510a7330
ID		 6705 ret = _rbd_dev_v2_snap_features(rbd_dev, CEPH_NOSNAP,
6706 rbd_is_ro(rbd_dev), &header->features);
57385b51 6707 if (ret)
510a7330 6708 return ret;
35d489f9 6709
cc070d59
AE		 6710 /* If the image supports fancy striping, get its parameters */
6711
510a7330
ID		 6712 if (header->features & RBD_FEATURE_STRIPINGV2) {
6713 ret = rbd_dev_v2_striping_info(rbd_dev, &header->stripe_unit,
6714 &header->stripe_count);
6715 if (ret)
6716 return ret;
cc070d59 6717 }
a30b71b9 6718
510a7330
ID		 6719 if (header->features & RBD_FEATURE_DATA_POOL) {
6720 ret = rbd_dev_v2_data_pool(rbd_dev, &header->data_pool_id);
7e97332e 6721 if (ret)
510a7330 6722 return ret;
7e97332e
ID		 6723 }
6724
35152979 6725 return 0;
a30b71b9
AE		 6726}
6727
6d69bb53
ID		 6728/*
6729 * @depth is rbd_dev_image_probe() -> rbd_dev_probe_parent() ->
6730 * rbd_dev_image_probe() recursion depth, which means it's also the
6731 * length of the already discovered part of the parent chain.
6732 */
6733static int rbd_dev_probe_parent(struct rbd_device *rbd_dev, int depth)
83a06263 6734{
2f82ee54 6735 struct rbd_device *parent = NULL;
124afba2
AE		 6736 int ret;
6737
6738 if (!rbd_dev->parent_spec)
6739 return 0;
124afba2 6740
6d69bb53
ID		 6741 if (++depth > RBD_MAX_PARENT_CHAIN_LEN) {
6742 pr_info("parent chain is too long (%d)\n", depth);
6743 ret = -EINVAL;
6744 goto out_err;
6745 }
6746
f7c4d9b1 6747 parent = __rbd_dev_create(rbd_dev->parent_spec);
1f2c6651
ID		 6748 if (!parent) {
6749 ret = -ENOMEM;
124afba2 6750 goto out_err;
1f2c6651
ID		 6751 }
6752
6753 /*
6754 * Images related by parent/child relationships always share
6755 * rbd_client and spec/parent_spec, so bump their refcounts.
6756 */
f7c4d9b1
ID		 6757 parent->rbd_client = __rbd_get_client(rbd_dev->rbd_client);
6758 parent->spec = rbd_spec_get(rbd_dev->parent_spec);
124afba2 6759
39258aa2
ID		 6760 __set_bit(RBD_DEV_FLAG_READONLY, &parent->flags);
6761
6d69bb53 6762 ret = rbd_dev_image_probe(parent, depth);
124afba2
AE		 6763 if (ret < 0)
6764 goto out_err;
1f2c6651 6765
124afba2 6766 rbd_dev->parent = parent;
a2acd00e 6767 atomic_set(&rbd_dev->parent_ref, 1);
124afba2 6768 return 0;
1f2c6651 6769
124afba2 6770out_err:
1f2c6651 6771 rbd_dev_unparent(rbd_dev);
1761b229 6772 rbd_dev_destroy(parent);
124afba2
AE		 6773 return ret;
6774}
6775
5769ed0c
ID		 6776static void rbd_dev_device_release(struct rbd_device *rbd_dev)
6777{
6778 clear_bit(RBD_DEV_FLAG_EXISTS, &rbd_dev->flags);
5769ed0c
ID		 6779 rbd_free_disk(rbd_dev);
6780 if (!single_major)
6781 unregister_blkdev(rbd_dev->major, rbd_dev->name);
6782}
6783
811c6688
ID		 6784/*
6785 * rbd_dev->header_rwsem must be locked for write and will be unlocked
6786 * upon return.
6787 */
200a6a8b 6788static int rbd_dev_device_setup(struct rbd_device *rbd_dev)
124afba2 6789{
83a06263 6790 int ret;
d1cf5788 6791
9b60e70b 6792 /* Record our major and minor device numbers. */
83a06263 6793
9b60e70b
ID		 6794 if (!single_major) {
6795 ret = register_blkdev(0, rbd_dev->name);
6796 if (ret < 0)
1643dfa4 6797 goto err_out_unlock;
9b60e70b
ID		 6798
6799 rbd_dev->major = ret;
6800 rbd_dev->minor = 0;
6801 } else {
6802 rbd_dev->major = rbd_major;
6803 rbd_dev->minor = rbd_dev_id_to_minor(rbd_dev->dev_id);
6804 }
83a06263
AE		 6805
6806 /* Set up the blkdev mapping. */
6807
6808 ret = rbd_init_disk(rbd_dev);
6809 if (ret)
6810 goto err_out_blkdev;
6811
f35a4dee 6812 set_capacity(rbd_dev->disk, rbd_dev->mapping.size / SECTOR_SIZE);
39258aa2 6813 set_disk_ro(rbd_dev->disk, rbd_is_ro(rbd_dev));
f35a4dee 6814
5769ed0c 6815 ret = dev_set_name(&rbd_dev->dev, "%d", rbd_dev->dev_id);
f35a4dee 6816 if (ret)
da5ef6be 6817 goto err_out_disk;
83a06263 6818
129b79d4 6819 set_bit(RBD_DEV_FLAG_EXISTS, &rbd_dev->flags);
811c6688 6820 up_write(&rbd_dev->header_rwsem);
5769ed0c 6821 return 0;
2f82ee54 6822
83a06263
AE		 6823err_out_disk:
6824 rbd_free_disk(rbd_dev);
6825err_out_blkdev:
9b60e70b
ID		 6826 if (!single_major)
6827 unregister_blkdev(rbd_dev->major, rbd_dev->name);
811c6688
ID		 6828err_out_unlock:
6829 up_write(&rbd_dev->header_rwsem);
83a06263
AE		 6830 return ret;
6831}
6832
332bb12d
AE		 6833static int rbd_dev_header_name(struct rbd_device *rbd_dev)
6834{
6835 struct rbd_spec *spec = rbd_dev->spec;
c41d13a3 6836 int ret;
332bb12d
AE		 6837
6838 /* Record the header object name for this rbd image. */
6839
6840 rbd_assert(rbd_image_format_valid(rbd_dev->image_format));
332bb12d 6841 if (rbd_dev->image_format == 1)
c41d13a3
ID		 6842 ret = ceph_oid_aprintf(&rbd_dev->header_oid, GFP_KERNEL, "%s%s",
6843 spec->image_name, RBD_SUFFIX);
332bb12d 6844 else
c41d13a3
ID		 6845 ret = ceph_oid_aprintf(&rbd_dev->header_oid, GFP_KERNEL, "%s%s",
6846 RBD_HEADER_PREFIX, spec->image_id);
332bb12d 6847
c41d13a3 6848 return ret;
332bb12d
AE		 6849}
6850
b9ef2b88
ID		 6851static void rbd_print_dne(struct rbd_device *rbd_dev, bool is_snap)
6852{
6853 if (!is_snap) {
6854 pr_info("image %s/%s%s%s does not exist\n",
6855 rbd_dev->spec->pool_name,
6856 rbd_dev->spec->pool_ns ?: "",
6857 rbd_dev->spec->pool_ns ? "/" : "",
6858 rbd_dev->spec->image_name);
6859 } else {
6860 pr_info("snap %s/%s%s%s@%s does not exist\n",
6861 rbd_dev->spec->pool_name,
6862 rbd_dev->spec->pool_ns ?: "",
6863 rbd_dev->spec->pool_ns ? "/" : "",
6864 rbd_dev->spec->image_name,
6865 rbd_dev->spec->snap_name);
6866 }
6867}
6868
200a6a8b
AE		 6869static void rbd_dev_image_release(struct rbd_device *rbd_dev)
6870{
b8776051 6871 if (!rbd_is_ro(rbd_dev))
fd22aef8 6872 rbd_unregister_watch(rbd_dev);
952c48b0
ID		 6873
6874 rbd_dev_unprobe(rbd_dev);
6fd48b3b
AE		 6875 rbd_dev->image_format = 0;
6876 kfree(rbd_dev->spec->image_id);
6877 rbd_dev->spec->image_id = NULL;
200a6a8b
AE		 6878}
6879
a30b71b9
AE		 6880/*
6881 * Probe for the existence of the header object for the given rbd
1f3ef788
AE		 6882 * device. If this image is the one being mapped (i.e., not a
6883 * parent), initiate a watch on its header object before using that
6884 * object to get detailed information about the rbd image.
0e4e1de5
ID		 6885 *
6886 * On success, returns with header_rwsem held for write if called
6887 * with @depth == 0.
a30b71b9 6888 */
6d69bb53 6889static int rbd_dev_image_probe(struct rbd_device *rbd_dev, int depth)
a30b71b9 6890{
b9ef2b88 6891 bool need_watch = !rbd_is_ro(rbd_dev);
a30b71b9
AE		 6892 int ret;
6893
6894 /*
3abef3b3
AE		 6895 * Get the id from the image id object. Unless there's an
6896 * error, rbd_dev->spec->image_id will be filled in with
6897 * a dynamically-allocated string, and rbd_dev->image_format
6898 * will be set to either 1 or 2.
a30b71b9
AE		 6899 */
6900 ret = rbd_dev_image_id(rbd_dev);
6901 if (ret)
c0fba368 6902 return ret;
c0fba368 6903
332bb12d
AE		 6904 ret = rbd_dev_header_name(rbd_dev);
6905 if (ret)
6906 goto err_out_format;
6907
b9ef2b88 6908 if (need_watch) {
99d16943 6909 ret = rbd_register_watch(rbd_dev);
1fe48023
ID		 6910 if (ret) {
6911 if (ret == -ENOENT)
b9ef2b88 6912 rbd_print_dne(rbd_dev, false);
c41d13a3 6913 goto err_out_format;
1fe48023 6914 }
1f3ef788 6915 }
b644de2b 6916
0e4e1de5
ID		 6917 if (!depth)
6918 down_write(&rbd_dev->header_rwsem);
6919
510a7330 6920 ret = rbd_dev_header_info(rbd_dev, &rbd_dev->header, true);
b9ef2b88
ID		 6921 if (ret) {
6922 if (ret == -ENOENT && !need_watch)
6923 rbd_print_dne(rbd_dev, false);
952c48b0 6924 goto err_out_probe;
b9ef2b88 6925 }
83a06263 6926
510a7330
ID		 6927 rbd_init_layout(rbd_dev);
6928
04077599
ID		 6929 /*
6930 * If this image is the one being mapped, we have pool name and
6931 * id, image name and id, and snap name - need to fill snap id.
6932 * Otherwise this is a parent image, identified by pool, image
6933 * and snap ids - need to fill in names for those ids.
6934 */
6d69bb53 6935 if (!depth)
04077599
ID		 6936 ret = rbd_spec_fill_snap_id(rbd_dev);
6937 else
6938 ret = rbd_spec_fill_names(rbd_dev);
1fe48023
ID		 6939 if (ret) {
6940 if (ret == -ENOENT)
b9ef2b88 6941 rbd_print_dne(rbd_dev, true);
33dca39f 6942 goto err_out_probe;
1fe48023 6943 }
9bb81c9b 6944
da5ef6be
ID		 6945 ret = rbd_dev_mapping_set(rbd_dev);
6946 if (ret)
6947 goto err_out_probe;
6948
f3c0e459 6949 if (rbd_is_snap(rbd_dev) &&
22e8bd51
ID		 6950 (rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP)) {
6951 ret = rbd_object_map_load(rbd_dev);
6952 if (ret)
6953 goto err_out_probe;
6954 }
6955
e8f59b59 6956 if (rbd_dev->header.features & RBD_FEATURE_LAYERING) {
c1031177 6957 ret = rbd_dev_setup_parent(rbd_dev);
e8f59b59
ID		 6958 if (ret)
6959 goto err_out_probe;
e8f59b59
ID		 6960 }
6961
6d69bb53 6962 ret = rbd_dev_probe_parent(rbd_dev, depth);
30d60ba2
AE		 6963 if (ret)
6964 goto err_out_probe;
6965
6966 dout("discovered format %u image, header name is %s\n",
c41d13a3 6967 rbd_dev->image_format, rbd_dev->header_oid.name);
30d60ba2 6968 return 0;
e8f59b59 6969
6fd48b3b 6970err_out_probe:
0e4e1de5
ID		 6971 if (!depth)
6972 up_write(&rbd_dev->header_rwsem);
b9ef2b88 6973 if (need_watch)
99d16943 6974 rbd_unregister_watch(rbd_dev);
952c48b0 6975 rbd_dev_unprobe(rbd_dev);
332bb12d
AE		 6976err_out_format:
6977 rbd_dev->image_format = 0;
5655c4d9
AE		 6978 kfree(rbd_dev->spec->image_id);
6979 rbd_dev->spec->image_id = NULL;
a30b71b9
AE		 6980 return ret;
6981}
6982
510a7330
ID		 6983static void rbd_dev_update_header(struct rbd_device *rbd_dev,
6984 struct rbd_image_header *header)
6985{
6986 rbd_assert(rbd_image_format_valid(rbd_dev->image_format));
6987 rbd_assert(rbd_dev->header.object_prefix); /* !first_time */
6988
0b207d02
ID		 6989 if (rbd_dev->header.image_size != header->image_size) {
6990 rbd_dev->header.image_size = header->image_size;
6991
6992 if (!rbd_is_snap(rbd_dev)) {
6993 rbd_dev->mapping.size = header->image_size;
6994 rbd_dev_update_size(rbd_dev);
6995 }
6996 }
510a7330
ID		 6997
6998 ceph_put_snap_context(rbd_dev->header.snapc);
6999 rbd_dev->header.snapc = header->snapc;
7000 header->snapc = NULL;
7001
7002 if (rbd_dev->image_format == 1) {
7003 kfree(rbd_dev->header.snap_names);
7004 rbd_dev->header.snap_names = header->snap_names;
7005 header->snap_names = NULL;
7006
7007 kfree(rbd_dev->header.snap_sizes);
7008 rbd_dev->header.snap_sizes = header->snap_sizes;
7009 header->snap_sizes = NULL;
7010 }
7011}
7012
c1031177
ID		 7013static void rbd_dev_update_parent(struct rbd_device *rbd_dev,
7014 struct parent_image_info *pii)
7015{
7016 if (pii->pool_id == CEPH_NOPOOL || !pii->has_overlap) {
7017 /*
7018 * Either the parent never existed, or we have
7019 * record of it but the image got flattened so it no
7020 * longer has a parent. When the parent of a
7021 * layered image disappears we immediately set the
7022 * overlap to 0. The effect of this is that all new
7023 * requests will be treated as if the image had no
7024 * parent.
7025 *
7026 * If !pii.has_overlap, the parent image spec is not
7027 * applicable. It's there to avoid duplication in each
7028 * snapshot record.
7029 */
7030 if (rbd_dev->parent_overlap) {
7031 rbd_dev->parent_overlap = 0;
7032 rbd_dev_parent_put(rbd_dev);
7033 pr_info("%s: clone has been flattened\n",
7034 rbd_dev->disk->disk_name);
7035 }
7036 } else {
7037 rbd_assert(rbd_dev->parent_spec);
7038
7039 /*
7040 * Update the parent overlap. If it became zero, issue
7041 * a warning as we will proceed as if there is no parent.
7042 */
7043 if (!pii->overlap && rbd_dev->parent_overlap)
7044 rbd_warn(rbd_dev,
7045 "clone has become standalone (overlap 0)");
7046 rbd_dev->parent_overlap = pii->overlap;
7047 }
7048}
7049
0b035401
ID		 7050static int rbd_dev_refresh(struct rbd_device *rbd_dev)
7051{
510a7330 7052 struct rbd_image_header header = { 0 };
c1031177 7053 struct parent_image_info pii = { 0 };
0b035401
ID		 7054 int ret;
7055
0b207d02 7056 dout("%s rbd_dev %p\n", __func__, rbd_dev);
0b035401 7057
510a7330 7058 ret = rbd_dev_header_info(rbd_dev, &header, false);
0b035401
ID		 7059 if (ret)
7060 goto out;
7061
7062 /*
7063 * If there is a parent, see if it has disappeared due to the
7064 * mapped image getting flattened.
7065 */
7066 if (rbd_dev->parent) {
c1031177 7067 ret = rbd_dev_v2_parent_info(rbd_dev, &pii);
0b035401
ID		 7068 if (ret)
7069 goto out;
7070 }
7071
0b207d02 7072 down_write(&rbd_dev->header_rwsem);
510a7330 7073 rbd_dev_update_header(rbd_dev, &header);
c1031177
ID		 7074 if (rbd_dev->parent)
7075 rbd_dev_update_parent(rbd_dev, &pii);
0b035401 7076 up_write(&rbd_dev->header_rwsem);
0b035401 7077
0b207d02 7078out:
c1031177 7079 rbd_parent_info_cleanup(&pii);
510a7330 7080 rbd_image_header_cleanup(&header);
0b035401
ID		 7081 return ret;
7082}
7083
75cff725 7084static ssize_t do_rbd_add(const char *buf, size_t count)
602adf40 7085{
cb8627c7 7086 struct rbd_device *rbd_dev = NULL;
dc79b113 7087 struct ceph_options *ceph_opts = NULL;
4e9afeba 7088 struct rbd_options *rbd_opts = NULL;
859c31df 7089 struct rbd_spec *spec = NULL;
9d3997fd 7090 struct rbd_client *rbdc;
b51c83c2 7091 int rc;
602adf40 7092
f44d04e6
ID		 7093 if (!capable(CAP_SYS_ADMIN))
7094 return -EPERM;
7095
602adf40
YS		 7096 if (!try_module_get(THIS_MODULE))
7097 return -ENODEV;
7098
602adf40 7099 /* parse add command */
859c31df 7100 rc = rbd_add_parse_args(buf, &ceph_opts, &rbd_opts, &spec);
dc79b113 7101 if (rc < 0)
dd5ac32d 7102 goto out;
78cea76e 7103
9d3997fd
AE		 7104 rbdc = rbd_get_client(ceph_opts);
7105 if (IS_ERR(rbdc)) {
7106 rc = PTR_ERR(rbdc);
0ddebc0c 7107 goto err_out_args;
9d3997fd 7108 }
602adf40 7109
602adf40 7110 /* pick the pool */
dd435855 7111 rc = ceph_pg_poolid_by_name(rbdc->client->osdc.osdmap, spec->pool_name);
1fe48023
ID		 7112 if (rc < 0) {
7113 if (rc == -ENOENT)
7114 pr_info("pool %s does not exist\n", spec->pool_name);
602adf40 7115 goto err_out_client;
1fe48023 7116 }
c0cd10db 7117 spec->pool_id = (u64)rc;
859c31df 7118
d147543d 7119 rbd_dev = rbd_dev_create(rbdc, spec, rbd_opts);
b51c83c2
ID		 7120 if (!rbd_dev) {
7121 rc = -ENOMEM;
bd4ba655 7122 goto err_out_client;
b51c83c2 7123 }
c53d5893
AE		 7124 rbdc = NULL; /* rbd_dev now owns this */
7125 spec = NULL; /* rbd_dev now owns this */
d147543d 7126 rbd_opts = NULL; /* rbd_dev now owns this */
602adf40 7127
39258aa2
ID		 7128 /* if we are mapping a snapshot it will be a read-only mapping */
7129 if (rbd_dev->opts->read_only ||
7130 strcmp(rbd_dev->spec->snap_name, RBD_SNAP_HEAD_NAME))
7131 __set_bit(RBD_DEV_FLAG_READONLY, &rbd_dev->flags);
7132
0d6d1e9c
MC		 7133 rbd_dev->config_info = kstrdup(buf, GFP_KERNEL);
7134 if (!rbd_dev->config_info) {
7135 rc = -ENOMEM;
7136 goto err_out_rbd_dev;
7137 }
7138
6d69bb53 7139 rc = rbd_dev_image_probe(rbd_dev, 0);
0e4e1de5 7140 if (rc < 0)
c53d5893 7141 goto err_out_rbd_dev;
05fd6f6f 7142
0c93e1b7
ID		 7143 if (rbd_dev->opts->alloc_size > rbd_dev->layout.object_size) {
7144 rbd_warn(rbd_dev, "alloc_size adjusted to %u",
7145 rbd_dev->layout.object_size);
7146 rbd_dev->opts->alloc_size = rbd_dev->layout.object_size;
7147 }
7148
b536f69a 7149 rc = rbd_dev_device_setup(rbd_dev);
fd22aef8 7150 if (rc)
8b679ec5 7151 goto err_out_image_probe;
3abef3b3 7152
637cd060
ID		 7153 rc = rbd_add_acquire_lock(rbd_dev);
7154 if (rc)
7155 goto err_out_image_lock;
3abef3b3 7156
5769ed0c
ID		 7157 /* Everything's ready. Announce the disk to the world. */
7158
7159 rc = device_add(&rbd_dev->dev);
7160 if (rc)
e010dd0a 7161 goto err_out_image_lock;
5769ed0c 7162
27c97abc
LC		 7163 rc = device_add_disk(&rbd_dev->dev, rbd_dev->disk, NULL);
7164 if (rc)
7165 goto err_out_cleanup_disk;
5769ed0c
ID		 7166
7167 spin_lock(&rbd_dev_list_lock);
7168 list_add_tail(&rbd_dev->node, &rbd_dev_list);
7169 spin_unlock(&rbd_dev_list_lock);
7170
7171 pr_info("%s: capacity %llu features 0x%llx\n", rbd_dev->disk->disk_name,
7172 (unsigned long long)get_capacity(rbd_dev->disk) << SECTOR_SHIFT,
7173 rbd_dev->header.features);
dd5ac32d
ID		 7174 rc = count;
7175out:
7176 module_put(THIS_MODULE);
7177 return rc;
b536f69a 7178
27c97abc
LC		 7179err_out_cleanup_disk:
7180 rbd_free_disk(rbd_dev);
e010dd0a
ID		 7181err_out_image_lock:
7182 rbd_dev_image_unlock(rbd_dev);
5769ed0c 7183 rbd_dev_device_release(rbd_dev);
8b679ec5
ID		 7184err_out_image_probe:
7185 rbd_dev_image_release(rbd_dev);
c53d5893
AE		 7186err_out_rbd_dev:
7187 rbd_dev_destroy(rbd_dev);
bd4ba655 7188err_out_client:
9d3997fd 7189 rbd_put_client(rbdc);
0ddebc0c 7190err_out_args:
859c31df 7191 rbd_spec_put(spec);
d147543d 7192 kfree(rbd_opts);
dd5ac32d 7193 goto out;
602adf40
YS		 7194}
7195
75cff725 7196static ssize_t add_store(const struct bus_type *bus, const char *buf, size_t count)
9b60e70b
ID		 7197{
7198 if (single_major)
7199 return -EINVAL;
7200
75cff725 7201 return do_rbd_add(buf, count);
9b60e70b
ID		 7202}
7203
75cff725 7204static ssize_t add_single_major_store(const struct bus_type *bus, const char *buf,
7e9586ba 7205 size_t count)
9b60e70b 7206{
75cff725 7207 return do_rbd_add(buf, count);
9b60e70b
ID		 7208}
7209
05a46afd
AE		 7210static void rbd_dev_remove_parent(struct rbd_device *rbd_dev)
7211{
ad945fc1 7212 while (rbd_dev->parent) {
05a46afd
AE		 7213 struct rbd_device *first = rbd_dev;
7214 struct rbd_device *second = first->parent;
7215 struct rbd_device *third;
7216
7217 /*
7218 * Follow to the parent with no grandparent and
7219 * remove it.
7220 */
7221 while (second && (third = second->parent)) {
7222 first = second;
7223 second = third;
7224 }
ad945fc1 7225 rbd_assert(second);
8ad42cd0 7226 rbd_dev_image_release(second);
8b679ec5 7227 rbd_dev_destroy(second);
ad945fc1
AE		 7228 first->parent = NULL;
7229 first->parent_overlap = 0;
7230
7231 rbd_assert(first->parent_spec);
05a46afd
AE		 7232 rbd_spec_put(first->parent_spec);
7233 first->parent_spec = NULL;
05a46afd
AE		 7234 }
7235}
7236
75cff725 7237static ssize_t do_rbd_remove(const char *buf, size_t count)
602adf40
YS		 7238{
7239 struct rbd_device *rbd_dev = NULL;
751cc0e3 7240 int dev_id;
0276dca6 7241 char opt_buf[6];
0276dca6 7242 bool force = false;
0d8189e1 7243 int ret;
602adf40 7244
f44d04e6
ID		 7245 if (!capable(CAP_SYS_ADMIN))
7246 return -EPERM;
7247
0276dca6
MC		 7248 dev_id = -1;
7249 opt_buf[0] = '\0';
7250 sscanf(buf, "%d %5s", &dev_id, opt_buf);
7251 if (dev_id < 0) {
7252 pr_err("dev_id out of range\n");
602adf40 7253 return -EINVAL;
0276dca6
MC		 7254 }
7255 if (opt_buf[0] != '\0') {
7256 if (!strcmp(opt_buf, "force")) {
7257 force = true;
7258 } else {
7259 pr_err("bad remove option at '%s'\n", opt_buf);
7260 return -EINVAL;
7261 }
7262 }
602adf40 7263
751cc0e3
AE		 7264 ret = -ENOENT;
7265 spin_lock(&rbd_dev_list_lock);
cd59cdef 7266 list_for_each_entry(rbd_dev, &rbd_dev_list, node) {
751cc0e3
AE		 7267 if (rbd_dev->dev_id == dev_id) {
7268 ret = 0;
7269 break;
7270 }
42382b70 7271 }
751cc0e3
AE		 7272 if (!ret) {
7273 spin_lock_irq(&rbd_dev->lock);
0276dca6 7274 if (rbd_dev->open_count && !force)
751cc0e3 7275 ret = -EBUSY;
85f5a4d6
ID		 7276 else if (test_and_set_bit(RBD_DEV_FLAG_REMOVING,
7277 &rbd_dev->flags))
7278 ret = -EINPROGRESS;
751cc0e3
AE		 7279 spin_unlock_irq(&rbd_dev->lock);
7280 }
7281 spin_unlock(&rbd_dev_list_lock);
85f5a4d6 7282 if (ret)
1ba0f1e7 7283 return ret;
751cc0e3 7284
0276dca6
MC		 7285 if (force) {
7286 /*
7287 * Prevent new IO from being queued and wait for existing
7288 * IO to complete/fail.
7289 */
7290 blk_mq_freeze_queue(rbd_dev->disk->queue);
7a5428dc 7291 blk_mark_disk_dead(rbd_dev->disk);
0276dca6
MC		 7292 }
7293
5769ed0c
ID		 7294 del_gendisk(rbd_dev->disk);
7295 spin_lock(&rbd_dev_list_lock);
7296 list_del_init(&rbd_dev->node);
7297 spin_unlock(&rbd_dev_list_lock);
7298 device_del(&rbd_dev->dev);
fca27065 7299
e010dd0a 7300 rbd_dev_image_unlock(rbd_dev);
dd5ac32d 7301 rbd_dev_device_release(rbd_dev);
8ad42cd0 7302 rbd_dev_image_release(rbd_dev);
8b679ec5 7303 rbd_dev_destroy(rbd_dev);
1ba0f1e7 7304 return count;
602adf40
YS		 7305}
7306
75cff725 7307static ssize_t remove_store(const struct bus_type *bus, const char *buf, size_t count)
9b60e70b
ID		 7308{
7309 if (single_major)
7310 return -EINVAL;
7311
75cff725 7312 return do_rbd_remove(buf, count);
9b60e70b
ID		 7313}
7314
75cff725 7315static ssize_t remove_single_major_store(const struct bus_type *bus, const char *buf,
7e9586ba 7316 size_t count)
9b60e70b 7317{
75cff725 7318 return do_rbd_remove(buf, count);
9b60e70b
ID		 7319}
7320
602adf40
YS		 7321/*
7322 * create control files in sysfs
dfc5606d 7323 * /sys/bus/rbd/...
602adf40 7324 */
7d8dc534 7325static int __init rbd_sysfs_init(void)
602adf40 7326{
dfc5606d 7327 int ret;
602adf40 7328
fed4c143 7329 ret = device_register(&rbd_root_dev);
7f21735f
YY		 7330 if (ret < 0) {
7331 put_device(&rbd_root_dev);
dfc5606d 7332 return ret;
7f21735f 7333 }
602adf40 7334
fed4c143
AE		 7335 ret = bus_register(&rbd_bus_type);
7336 if (ret < 0)
7337 device_unregister(&rbd_root_dev);
602adf40 7338
602adf40
YS		 7339 return ret;
7340}
7341
7d8dc534 7342static void __exit rbd_sysfs_cleanup(void)
602adf40 7343{
dfc5606d 7344 bus_unregister(&rbd_bus_type);
fed4c143 7345 device_unregister(&rbd_root_dev);
602adf40
YS		 7346}
7347
7d8dc534 7348static int __init rbd_slab_init(void)
1c2a9dfe
AE		 7349{
7350 rbd_assert(!rbd_img_request_cache);
03d94406 7351 rbd_img_request_cache = KMEM_CACHE(rbd_img_request, 0);
868311b1
AE		 7352 if (!rbd_img_request_cache)
7353 return -ENOMEM;
7354
7355 rbd_assert(!rbd_obj_request_cache);
03d94406 7356 rbd_obj_request_cache = KMEM_CACHE(rbd_obj_request, 0);
78c2a44a
AE		 7357 if (!rbd_obj_request_cache)
7358 goto out_err;
7359
6c696d85 7360 return 0;
1c2a9dfe 7361
6c696d85 7362out_err:
868311b1
AE		 7363 kmem_cache_destroy(rbd_img_request_cache);
7364 rbd_img_request_cache = NULL;
1c2a9dfe
AE		 7365 return -ENOMEM;
7366}
7367
7368static void rbd_slab_exit(void)
7369{
868311b1
AE		 7370 rbd_assert(rbd_obj_request_cache);
7371 kmem_cache_destroy(rbd_obj_request_cache);
7372 rbd_obj_request_cache = NULL;
7373
1c2a9dfe
AE		 7374 rbd_assert(rbd_img_request_cache);
7375 kmem_cache_destroy(rbd_img_request_cache);
7376 rbd_img_request_cache = NULL;
7377}
7378
cc344fa1 7379static int __init rbd_init(void)
602adf40
YS		 7380{
7381 int rc;
7382
1e32d34c
AE		 7383 if (!libceph_compatible(NULL)) {
7384 rbd_warn(NULL, "libceph incompatibility (quitting)");
1e32d34c
AE		 7385 return -EINVAL;
7386 }
e1b4d96d 7387
1c2a9dfe 7388 rc = rbd_slab_init();
602adf40
YS		 7389 if (rc)
7390 return rc;
e1b4d96d 7391
f5ee37bd
ID		 7392 /*
7393 * The number of active work items is limited by the number of
f77303bd 7394 * rbd devices * queue depth, so leave @max_active at default.
f5ee37bd
ID		 7395 */
7396 rbd_wq = alloc_workqueue(RBD_DRV_NAME, WQ_MEM_RECLAIM, 0);
7397 if (!rbd_wq) {
7398 rc = -ENOMEM;
7399 goto err_out_slab;
7400 }
7401
9b60e70b
ID		 7402 if (single_major) {
7403 rbd_major = register_blkdev(0, RBD_DRV_NAME);
7404 if (rbd_major < 0) {
7405 rc = rbd_major;
f5ee37bd 7406 goto err_out_wq;
9b60e70b
ID		 7407 }
7408 }
7409
1c2a9dfe
AE		 7410 rc = rbd_sysfs_init();
7411 if (rc)
9b60e70b
ID		 7412 goto err_out_blkdev;
7413
7414 if (single_major)
7415 pr_info("loaded (major %d)\n", rbd_major);
7416 else
7417 pr_info("loaded\n");
1c2a9dfe 7418
e1b4d96d
ID		 7419 return 0;
7420
9b60e70b
ID		 7421err_out_blkdev:
7422 if (single_major)
7423 unregister_blkdev(rbd_major, RBD_DRV_NAME);
f5ee37bd
ID		 7424err_out_wq:
7425 destroy_workqueue(rbd_wq);
e1b4d96d
ID		 7426err_out_slab:
7427 rbd_slab_exit();
1c2a9dfe 7428 return rc;
602adf40
YS		 7429}
7430
cc344fa1 7431static void __exit rbd_exit(void)
602adf40 7432{
ffe312cf 7433 ida_destroy(&rbd_dev_id_ida);
602adf40 7434 rbd_sysfs_cleanup();
9b60e70b
ID		 7435 if (single_major)
7436 unregister_blkdev(rbd_major, RBD_DRV_NAME);
f5ee37bd 7437 destroy_workqueue(rbd_wq);
1c2a9dfe 7438 rbd_slab_exit();
602adf40
YS		 7439}
7440
7441module_init(rbd_init);
7442module_exit(rbd_exit);
7443
d552c619 7444MODULE_AUTHOR("Alex Elder <elder@inktank.com>");
602adf40
YS		 7445MODULE_AUTHOR("Sage Weil <sage@newdream.net>");
7446MODULE_AUTHOR("Yehuda Sadeh <yehuda@hq.newdream.net>");
602adf40
YS		 7447/* following authorship retained from original osdblk.c */
7448MODULE_AUTHOR("Jeff Garzik <jeff@garzik.org>");
7449
90da258b 7450MODULE_DESCRIPTION("RADOS Block Device (RBD) driver");
602adf40 7451MODULE_LICENSE("GPL");
Linux 6.x block layer and io_uring tree(s)