Commit | Line | Data |
---|---|---|
2025cf9e | 1 | // SPDX-License-Identifier: GPL-2.0-only |
047fc8a1 RZ |
2 | /* |
3 | * NVDIMM Block Window Driver | |
4 | * Copyright (c) 2014, Intel Corporation. | |
047fc8a1 RZ |
5 | */ |
6 | ||
7 | #include <linux/blkdev.h> | |
8 | #include <linux/fs.h> | |
9 | #include <linux/genhd.h> | |
10 | #include <linux/module.h> | |
11 | #include <linux/moduleparam.h> | |
12 | #include <linux/nd.h> | |
13 | #include <linux/sizes.h> | |
14 | #include "nd.h" | |
15 | ||
9d90725d DW |
16 | static u32 nsblk_meta_size(struct nd_namespace_blk *nsblk) |
17 | { | |
18 | return nsblk->lbasize - ((nsblk->lbasize >= 4096) ? 4096 : 512); | |
19 | } | |
047fc8a1 | 20 | |
9d90725d | 21 | static u32 nsblk_internal_lbasize(struct nd_namespace_blk *nsblk) |
fcae6957 | 22 | { |
9d90725d DW |
23 | return roundup(nsblk->lbasize, INT_LBASIZE_ALIGNMENT); |
24 | } | |
25 | ||
26 | static u32 nsblk_sector_size(struct nd_namespace_blk *nsblk) | |
27 | { | |
28 | return nsblk->lbasize - nsblk_meta_size(nsblk); | |
fcae6957 VV |
29 | } |
30 | ||
047fc8a1 RZ |
31 | static resource_size_t to_dev_offset(struct nd_namespace_blk *nsblk, |
32 | resource_size_t ns_offset, unsigned int len) | |
33 | { | |
34 | int i; | |
35 | ||
36 | for (i = 0; i < nsblk->num_resources; i++) { | |
37 | if (ns_offset < resource_size(nsblk->res[i])) { | |
38 | if (ns_offset + len > resource_size(nsblk->res[i])) { | |
39 | dev_WARN_ONCE(&nsblk->common.dev, 1, | |
40 | "illegal request\n"); | |
41 | return SIZE_MAX; | |
42 | } | |
43 | return nsblk->res[i]->start + ns_offset; | |
44 | } | |
45 | ns_offset -= resource_size(nsblk->res[i]); | |
46 | } | |
47 | ||
48 | dev_WARN_ONCE(&nsblk->common.dev, 1, "request out of range\n"); | |
49 | return SIZE_MAX; | |
50 | } | |
51 | ||
9d90725d DW |
52 | static struct nd_blk_region *to_ndbr(struct nd_namespace_blk *nsblk) |
53 | { | |
54 | struct nd_region *nd_region; | |
55 | struct device *parent; | |
56 | ||
57 | parent = nsblk->common.dev.parent; | |
58 | nd_region = container_of(parent, struct nd_region, dev); | |
59 | return container_of(nd_region, struct nd_blk_region, nd_region); | |
60 | } | |
61 | ||
fcae6957 | 62 | #ifdef CONFIG_BLK_DEV_INTEGRITY |
9d90725d DW |
63 | static int nd_blk_rw_integrity(struct nd_namespace_blk *nsblk, |
64 | struct bio_integrity_payload *bip, u64 lba, int rw) | |
fcae6957 | 65 | { |
9d90725d DW |
66 | struct nd_blk_region *ndbr = to_ndbr(nsblk); |
67 | unsigned int len = nsblk_meta_size(nsblk); | |
fcae6957 | 68 | resource_size_t dev_offset, ns_offset; |
9d90725d | 69 | u32 internal_lbasize, sector_size; |
fcae6957 VV |
70 | int err = 0; |
71 | ||
9d90725d DW |
72 | internal_lbasize = nsblk_internal_lbasize(nsblk); |
73 | sector_size = nsblk_sector_size(nsblk); | |
74 | ns_offset = lba * internal_lbasize + sector_size; | |
fcae6957 VV |
75 | dev_offset = to_dev_offset(nsblk, ns_offset, len); |
76 | if (dev_offset == SIZE_MAX) | |
77 | return -EIO; | |
78 | ||
79 | while (len) { | |
80 | unsigned int cur_len; | |
81 | struct bio_vec bv; | |
82 | void *iobuf; | |
83 | ||
84 | bv = bvec_iter_bvec(bip->bip_vec, bip->bip_iter); | |
85 | /* | |
86 | * The 'bv' obtained from bvec_iter_bvec has its .bv_len and | |
87 | * .bv_offset already adjusted for iter->bi_bvec_done, and we | |
88 | * can use those directly | |
89 | */ | |
90 | ||
91 | cur_len = min(len, bv.bv_len); | |
92 | iobuf = kmap_atomic(bv.bv_page); | |
93 | err = ndbr->do_io(ndbr, dev_offset, iobuf + bv.bv_offset, | |
94 | cur_len, rw); | |
95 | kunmap_atomic(iobuf); | |
96 | if (err) | |
97 | return err; | |
98 | ||
99 | len -= cur_len; | |
100 | dev_offset += cur_len; | |
b1fb2c52 DM |
101 | if (!bvec_iter_advance(bip->bip_vec, &bip->bip_iter, cur_len)) |
102 | return -EIO; | |
fcae6957 VV |
103 | } |
104 | ||
105 | return err; | |
106 | } | |
107 | ||
108 | #else /* CONFIG_BLK_DEV_INTEGRITY */ | |
9d90725d DW |
109 | static int nd_blk_rw_integrity(struct nd_namespace_blk *nsblk, |
110 | struct bio_integrity_payload *bip, u64 lba, int rw) | |
fcae6957 VV |
111 | { |
112 | return 0; | |
113 | } | |
114 | #endif | |
115 | ||
9d90725d DW |
116 | static int nsblk_do_bvec(struct nd_namespace_blk *nsblk, |
117 | struct bio_integrity_payload *bip, struct page *page, | |
118 | unsigned int len, unsigned int off, int rw, sector_t sector) | |
fcae6957 | 119 | { |
9d90725d | 120 | struct nd_blk_region *ndbr = to_ndbr(nsblk); |
fcae6957 | 121 | resource_size_t dev_offset, ns_offset; |
9d90725d | 122 | u32 internal_lbasize, sector_size; |
fcae6957 VV |
123 | int err = 0; |
124 | void *iobuf; | |
125 | u64 lba; | |
126 | ||
9d90725d DW |
127 | internal_lbasize = nsblk_internal_lbasize(nsblk); |
128 | sector_size = nsblk_sector_size(nsblk); | |
fcae6957 VV |
129 | while (len) { |
130 | unsigned int cur_len; | |
131 | ||
132 | /* | |
133 | * If we don't have an integrity payload, we don't have to | |
134 | * split the bvec into sectors, as this would cause unnecessary | |
135 | * Block Window setup/move steps. the do_io routine is capable | |
136 | * of handling len <= PAGE_SIZE. | |
137 | */ | |
9d90725d | 138 | cur_len = bip ? min(len, sector_size) : len; |
fcae6957 | 139 | |
9d90725d DW |
140 | lba = div_u64(sector << SECTOR_SHIFT, sector_size); |
141 | ns_offset = lba * internal_lbasize; | |
142 | dev_offset = to_dev_offset(nsblk, ns_offset, cur_len); | |
fcae6957 VV |
143 | if (dev_offset == SIZE_MAX) |
144 | return -EIO; | |
145 | ||
146 | iobuf = kmap_atomic(page); | |
147 | err = ndbr->do_io(ndbr, dev_offset, iobuf + off, cur_len, rw); | |
148 | kunmap_atomic(iobuf); | |
149 | if (err) | |
150 | return err; | |
151 | ||
152 | if (bip) { | |
9d90725d | 153 | err = nd_blk_rw_integrity(nsblk, bip, lba, rw); |
fcae6957 VV |
154 | if (err) |
155 | return err; | |
156 | } | |
157 | len -= cur_len; | |
158 | off += cur_len; | |
9d90725d | 159 | sector += sector_size >> SECTOR_SHIFT; |
fcae6957 VV |
160 | } |
161 | ||
162 | return err; | |
163 | } | |
164 | ||
c62b37d9 | 165 | static blk_qc_t nd_blk_submit_bio(struct bio *bio) |
047fc8a1 | 166 | { |
fcae6957 | 167 | struct bio_integrity_payload *bip; |
309dca30 | 168 | struct nd_namespace_blk *nsblk = bio->bi_bdev->bd_disk->private_data; |
047fc8a1 | 169 | struct bvec_iter iter; |
f0dc089c | 170 | unsigned long start; |
047fc8a1 RZ |
171 | struct bio_vec bvec; |
172 | int err = 0, rw; | |
f0dc089c | 173 | bool do_acct; |
047fc8a1 | 174 | |
e23947bd DM |
175 | if (!bio_integrity_prep(bio)) |
176 | return BLK_QC_T_NONE; | |
fcae6957 VV |
177 | |
178 | bip = bio_integrity(bio); | |
047fc8a1 | 179 | rw = bio_data_dir(bio); |
309dca30 | 180 | do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue); |
0fd92f89 CH |
181 | if (do_acct) |
182 | start = bio_start_io_acct(bio); | |
047fc8a1 RZ |
183 | bio_for_each_segment(bvec, bio, iter) { |
184 | unsigned int len = bvec.bv_len; | |
047fc8a1 RZ |
185 | |
186 | BUG_ON(len > PAGE_SIZE); | |
9d90725d DW |
187 | err = nsblk_do_bvec(nsblk, bip, bvec.bv_page, len, |
188 | bvec.bv_offset, rw, iter.bi_sector); | |
fcae6957 | 189 | if (err) { |
9d90725d | 190 | dev_dbg(&nsblk->common.dev, |
fcae6957 VV |
191 | "io error in %s sector %lld, len %d,\n", |
192 | (rw == READ) ? "READ" : "WRITE", | |
193 | (unsigned long long) iter.bi_sector, len); | |
4e4cbee9 | 194 | bio->bi_status = errno_to_blk_status(err); |
f0dc089c | 195 | break; |
047fc8a1 | 196 | } |
047fc8a1 | 197 | } |
f0dc089c | 198 | if (do_acct) |
0fd92f89 | 199 | bio_end_io_acct(bio, start); |
047fc8a1 | 200 | |
4246a0b6 | 201 | bio_endio(bio); |
dece1635 | 202 | return BLK_QC_T_NONE; |
047fc8a1 RZ |
203 | } |
204 | ||
9d90725d | 205 | static int nsblk_rw_bytes(struct nd_namespace_common *ndns, |
3ae3d67b VV |
206 | resource_size_t offset, void *iobuf, size_t n, int rw, |
207 | unsigned long flags) | |
047fc8a1 | 208 | { |
9d90725d DW |
209 | struct nd_namespace_blk *nsblk = to_nd_namespace_blk(&ndns->dev); |
210 | struct nd_blk_region *ndbr = to_ndbr(nsblk); | |
047fc8a1 RZ |
211 | resource_size_t dev_offset; |
212 | ||
213 | dev_offset = to_dev_offset(nsblk, offset, n); | |
214 | ||
9d90725d | 215 | if (unlikely(offset + n > nsblk->size)) { |
047fc8a1 RZ |
216 | dev_WARN_ONCE(&ndns->dev, 1, "request out of range\n"); |
217 | return -EFAULT; | |
218 | } | |
219 | ||
220 | if (dev_offset == SIZE_MAX) | |
221 | return -EIO; | |
222 | ||
223 | return ndbr->do_io(ndbr, dev_offset, iobuf, n, rw); | |
224 | } | |
225 | ||
226 | static const struct block_device_operations nd_blk_fops = { | |
227 | .owner = THIS_MODULE, | |
c62b37d9 | 228 | .submit_bio = nd_blk_submit_bio, |
047fc8a1 RZ |
229 | }; |
230 | ||
d29cee12 DW |
231 | static void nd_blk_release_queue(void *q) |
232 | { | |
233 | blk_cleanup_queue(q); | |
234 | } | |
235 | ||
236 | static void nd_blk_release_disk(void *disk) | |
237 | { | |
238 | del_gendisk(disk); | |
239 | put_disk(disk); | |
240 | } | |
241 | ||
9d90725d | 242 | static int nsblk_attach_disk(struct nd_namespace_blk *nsblk) |
047fc8a1 | 243 | { |
9d90725d | 244 | struct device *dev = &nsblk->common.dev; |
fcae6957 | 245 | resource_size_t available_disk_size; |
d29cee12 | 246 | struct request_queue *q; |
047fc8a1 | 247 | struct gendisk *disk; |
fcae6957 VV |
248 | u64 internal_nlba; |
249 | ||
9d90725d DW |
250 | internal_nlba = div_u64(nsblk->size, nsblk_internal_lbasize(nsblk)); |
251 | available_disk_size = internal_nlba * nsblk_sector_size(nsblk); | |
047fc8a1 | 252 | |
c62b37d9 | 253 | q = blk_alloc_queue(NUMA_NO_NODE); |
d29cee12 DW |
254 | if (!q) |
255 | return -ENOMEM; | |
f02716db | 256 | if (devm_add_action_or_reset(dev, nd_blk_release_queue, q)) |
047fc8a1 RZ |
257 | return -ENOMEM; |
258 | ||
d29cee12 | 259 | blk_queue_max_hw_sectors(q, UINT_MAX); |
9d90725d | 260 | blk_queue_logical_block_size(q, nsblk_sector_size(nsblk)); |
8b904b5b | 261 | blk_queue_flag_set(QUEUE_FLAG_NONROT, q); |
047fc8a1 | 262 | |
d29cee12 DW |
263 | disk = alloc_disk(0); |
264 | if (!disk) | |
265 | return -ENOMEM; | |
047fc8a1 | 266 | |
047fc8a1 RZ |
267 | disk->first_minor = 0; |
268 | disk->fops = &nd_blk_fops; | |
d29cee12 | 269 | disk->queue = q; |
047fc8a1 | 270 | disk->flags = GENHD_FL_EXT_DEVT; |
daa28975 | 271 | disk->private_data = nsblk; |
9d90725d | 272 | nvdimm_namespace_disk_name(&nsblk->common, disk->disk_name); |
047fc8a1 | 273 | |
f02716db DW |
274 | if (devm_add_action_or_reset(dev, nd_blk_release_disk, disk)) |
275 | return -ENOMEM; | |
276 | ||
9d90725d DW |
277 | if (nsblk_meta_size(nsblk)) { |
278 | int rc = nd_integrity_init(disk, nsblk_meta_size(nsblk)); | |
fcae6957 | 279 | |
d29cee12 | 280 | if (rc) |
fcae6957 | 281 | return rc; |
fcae6957 VV |
282 | } |
283 | ||
284 | set_capacity(disk, available_disk_size >> SECTOR_SHIFT); | |
fef912bf | 285 | device_add_disk(dev, disk, NULL); |
32f61d67 | 286 | nvdimm_check_and_set_ro(disk); |
047fc8a1 RZ |
287 | return 0; |
288 | } | |
289 | ||
290 | static int nd_blk_probe(struct device *dev) | |
291 | { | |
292 | struct nd_namespace_common *ndns; | |
fcae6957 | 293 | struct nd_namespace_blk *nsblk; |
047fc8a1 RZ |
294 | |
295 | ndns = nvdimm_namespace_common_probe(dev); | |
296 | if (IS_ERR(ndns)) | |
297 | return PTR_ERR(ndns); | |
298 | ||
fcae6957 | 299 | nsblk = to_nd_namespace_blk(&ndns->dev); |
9d90725d DW |
300 | nsblk->size = nvdimm_namespace_capacity(ndns); |
301 | dev_set_drvdata(dev, nsblk); | |
302 | ||
303 | ndns->rw_bytes = nsblk_rw_bytes; | |
047fc8a1 | 304 | if (is_nd_btt(dev)) |
d29cee12 | 305 | return nvdimm_namespace_attach_btt(ndns); |
200c79da | 306 | else if (nd_btt_probe(dev, ndns) == 0) { |
047fc8a1 | 307 | /* we'll come back as btt-blk */ |
d29cee12 | 308 | return -ENXIO; |
047fc8a1 | 309 | } else |
9d90725d | 310 | return nsblk_attach_disk(nsblk); |
047fc8a1 RZ |
311 | } |
312 | ||
313 | static int nd_blk_remove(struct device *dev) | |
314 | { | |
047fc8a1 | 315 | if (is_nd_btt(dev)) |
298f2bc5 | 316 | nvdimm_namespace_detach_btt(to_nd_btt(dev)); |
047fc8a1 RZ |
317 | return 0; |
318 | } | |
319 | ||
320 | static struct nd_device_driver nd_blk_driver = { | |
321 | .probe = nd_blk_probe, | |
322 | .remove = nd_blk_remove, | |
323 | .drv = { | |
324 | .name = "nd_blk", | |
325 | }, | |
326 | .type = ND_DRIVER_NAMESPACE_BLK, | |
327 | }; | |
328 | ||
329 | static int __init nd_blk_init(void) | |
330 | { | |
ec56151d | 331 | return nd_driver_register(&nd_blk_driver); |
047fc8a1 RZ |
332 | } |
333 | ||
334 | static void __exit nd_blk_exit(void) | |
335 | { | |
336 | driver_unregister(&nd_blk_driver.drv); | |
047fc8a1 RZ |
337 | } |
338 | ||
339 | MODULE_AUTHOR("Ross Zwisler <ross.zwisler@linux.intel.com>"); | |
340 | MODULE_LICENSE("GPL v2"); | |
341 | MODULE_ALIAS_ND_DEVICE(ND_DEVICE_NAMESPACE_BLK); | |
342 | module_init(nd_blk_init); | |
343 | module_exit(nd_blk_exit); |