Commit | Line | Data |
---|---|---|
4f28f523 JA |
1 | /* |
2 | * TODO | |
3 | * | |
4 | * - Proper ioctls | |
5 | * - Get rid of device list? | |
6 | */ | |
7 | #include <linux/kernel.h> | |
8 | #include <linux/module.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/poll.h> | |
11 | #include <linux/slab.h> | |
12 | #include <linux/idr.h> | |
13 | #include <linux/file.h> | |
14 | #include <linux/miscdevice.h> | |
15 | #include <linux/cdev.h> | |
16 | #include <linux/bio.h> | |
17 | #include <linux/blkdev.h> | |
18 | ||
19 | #include "kcompat.h" | |
20 | #include "binject.h" | |
21 | ||
22 | static LIST_HEAD(b_dev_list); | |
23 | static DEFINE_SPINLOCK(b_dev_lock); | |
24 | static DEFINE_IDR(b_minor_idr); | |
25 | static struct kmem_cache *b_slab; | |
26 | static struct class *b_class; | |
27 | static int b_major; | |
28 | ||
29 | #define B_MAX_DEVS 64 | |
30 | ||
53354c46 JA |
31 | struct b_dev_cpu { |
32 | spinlock_t lock; | |
33 | struct list_head done_list; | |
53354c46 JA |
34 | }; |
35 | ||
4f28f523 JA |
36 | struct b_dev { |
37 | struct list_head device_list; | |
53354c46 JA |
38 | struct list_head reaped_done; |
39 | spinlock_t done_lock; | |
4f28f523 | 40 | atomic_t in_flight; |
4f28f523 JA |
41 | wait_queue_head_t wq_done; |
42 | struct block_device *bdev; | |
4f28f523 JA |
43 | atomic_t ref; |
44 | struct file *file; | |
45 | struct device *dev; | |
46 | int minor; | |
53354c46 | 47 | struct b_dev_cpu __percpu *cpu_queue; |
4f28f523 JA |
48 | struct rcu_head rcu_free; |
49 | }; | |
50 | ||
51 | struct b_cmd { | |
52 | struct list_head list; | |
53 | struct b_dev *bd; | |
54 | struct bio *bio; | |
55 | struct b_user_cmd cmd; | |
56 | u64 issue_time; | |
57 | }; | |
58 | ||
59 | static const unsigned long uc_flag_map[__B_FLAG_NR] = { | |
60 | B_REQ_SYNC, | |
61 | B_REQ_UNPLUG, | |
62 | B_REQ_NOIDLE, | |
63 | B_REQ_HARDBARRIER, | |
64 | B_REQ_META, | |
65 | B_REQ_RAHEAD, | |
66 | B_REQ_FAILFAST_DEV, | |
67 | B_REQ_FAILFAST_TRANSPORT, | |
68 | B_REQ_FAILFAST_DRIVER | |
69 | }; | |
70 | ||
71 | struct uc_map { | |
72 | int type; | |
73 | unsigned int data_transfer : 1; | |
74 | unsigned int todevice : 1; | |
75 | unsigned int map_zero : 1; | |
41c1852f | 76 | unsigned long rw_flags; |
4f28f523 JA |
77 | }; |
78 | ||
79 | static const struct uc_map uc_map[B_TYPE_NR] = { | |
80 | { | |
81 | .type = B_TYPE_READ, | |
82 | .data_transfer = 1, | |
83 | .todevice = 0, | |
84 | .map_zero = 0, | |
85 | }, | |
86 | { | |
87 | .type = B_TYPE_WRITE, | |
88 | .data_transfer = 1, | |
89 | .todevice = 1, | |
90 | .map_zero = 0, | |
41c1852f | 91 | .rw_flags = B_REQ_WRITE, |
4f28f523 JA |
92 | }, |
93 | { | |
94 | .type = B_TYPE_DISCARD, | |
95 | .data_transfer = 0, | |
96 | .todevice = 0, | |
97 | .map_zero = 0, | |
41c1852f | 98 | .rw_flags = B_REQ_DISCARD | B_REQ_WRITE, |
4f28f523 JA |
99 | }, |
100 | { | |
101 | .type = B_TYPE_READVOID, | |
102 | .data_transfer = 1, | |
103 | .todevice = 0, | |
104 | .map_zero = 1, | |
105 | }, | |
106 | { | |
107 | .type = B_TYPE_WRITEZERO, | |
108 | .data_transfer = 1, | |
109 | .todevice = 1, | |
110 | .map_zero = 1, | |
41c1852f JA |
111 | .rw_flags = B_REQ_WRITE, |
112 | }, | |
113 | { | |
114 | .type = B_TYPE_READBARRIER, | |
115 | .data_transfer = 1, | |
116 | .todevice = 0, | |
117 | .map_zero = 0, | |
118 | .rw_flags = B_REQ_HARDBARRIER, | |
119 | }, | |
120 | { | |
121 | .type = B_TYPE_WRITEBARRIER, | |
122 | .data_transfer = 1, | |
123 | .todevice = 1, | |
124 | .map_zero = 0, | |
125 | .rw_flags = B_REQ_HARDBARRIER | B_REQ_FLUSH | B_REQ_WRITE, | |
4f28f523 JA |
126 | } |
127 | }; | |
128 | ||
129 | static void b_dev_complete_commands(struct b_dev *bd); | |
130 | ||
131 | static void b_dev_remove_lookup(struct b_dev *bd) | |
132 | { | |
133 | if (!list_empty(&bd->device_list)) { | |
134 | list_del_init(&bd->device_list); | |
135 | idr_remove(&b_minor_idr, bd->minor); | |
136 | } | |
137 | } | |
138 | ||
139 | static void bd_rcu_free(struct rcu_head *head) | |
140 | { | |
53354c46 JA |
141 | struct b_dev *bd = container_of(head, struct b_dev, rcu_free); |
142 | ||
143 | free_percpu(bd->cpu_queue); | |
144 | kfree(bd); | |
4f28f523 JA |
145 | } |
146 | ||
147 | static void b_dev_put(struct b_dev *bd) | |
148 | { | |
149 | if (!atomic_dec_and_test(&bd->ref)) | |
150 | return; | |
151 | ||
152 | spin_lock(&b_dev_lock); | |
153 | b_dev_remove_lookup(bd); | |
154 | spin_unlock(&b_dev_lock); | |
155 | ||
156 | b_dev_complete_commands(bd); | |
157 | ||
158 | device_destroy(b_class, MKDEV(b_major, bd->minor)); | |
159 | fput(bd->file); | |
160 | module_put(THIS_MODULE); | |
161 | ||
162 | call_rcu(&bd->rcu_free, bd_rcu_free); | |
163 | } | |
164 | ||
165 | static struct b_cmd *get_free_command(struct b_dev *bd) | |
166 | { | |
167 | struct b_cmd *bc; | |
168 | ||
169 | bc = kmem_cache_alloc(b_slab, GFP_KERNEL); | |
170 | if (bc) { | |
4f28f523 JA |
171 | bc->bd = bd; |
172 | return bc; | |
173 | } | |
174 | ||
175 | return ERR_PTR(-ENOMEM); | |
176 | } | |
177 | ||
178 | static struct b_cmd *get_completed_command(struct b_dev *bd) | |
179 | { | |
180 | struct b_cmd *bc = NULL; | |
53354c46 | 181 | int cpu, spliced = 0; |
4f28f523 | 182 | |
53354c46 JA |
183 | spin_lock(&bd->done_lock); |
184 | if (!list_empty(&bd->reaped_done)) { | |
185 | ret_one: | |
186 | bc = list_entry(bd->reaped_done.next, struct b_cmd, list); | |
146229b6 | 187 | list_del_init(&bc->list); |
4f28f523 | 188 | } |
53354c46 JA |
189 | spin_unlock(&bd->done_lock); |
190 | ||
191 | if (bc) | |
192 | return bc; | |
193 | ||
194 | spin_lock(&bd->done_lock); | |
195 | for_each_possible_cpu(cpu) { | |
196 | struct b_dev_cpu *bdc = per_cpu_ptr(bd->cpu_queue, cpu); | |
197 | ||
198 | spin_lock_irq(&bdc->lock); | |
199 | if (!list_empty(&bdc->done_list)) { | |
200 | list_splice_init(&bdc->done_list, &bd->reaped_done); | |
53354c46 JA |
201 | spliced++; |
202 | } | |
203 | spin_unlock_irq(&bdc->lock); | |
204 | } | |
205 | ||
206 | if (spliced) | |
207 | goto ret_one; | |
208 | ||
209 | spin_unlock(&bd->done_lock); | |
210 | return NULL; | |
211 | } | |
212 | ||
08ceb814 | 213 | static int bd_pending_done(struct b_dev *bd) |
53354c46 | 214 | { |
08ceb814 | 215 | int cpu; |
53354c46 | 216 | |
53354c46 JA |
217 | for_each_possible_cpu(cpu) { |
218 | struct b_dev_cpu *bdc = per_cpu_ptr(bd->cpu_queue, cpu); | |
219 | ||
08ceb814 JA |
220 | if (!list_empty_careful(&bdc->done_list)) |
221 | return 1; | |
53354c46 JA |
222 | } |
223 | ||
08ceb814 | 224 | return 0; |
4f28f523 JA |
225 | } |
226 | ||
227 | static struct b_cmd *get_done_command(struct b_dev *bd, int block) | |
228 | { | |
229 | struct b_cmd *bc; | |
230 | int ret; | |
231 | ||
232 | do { | |
233 | bc = get_completed_command(bd); | |
234 | if (bc) | |
235 | break; | |
236 | ||
237 | if (!block) | |
238 | break; | |
239 | ||
08ceb814 | 240 | ret = wait_event_interruptible(bd->wq_done, bd_pending_done(bd)); |
4f28f523 JA |
241 | if (ret) { |
242 | bc = ERR_PTR(-ERESTARTSYS); | |
243 | break; | |
244 | } | |
245 | } while (1); | |
246 | ||
247 | return bc; | |
248 | } | |
249 | ||
250 | static void bc_put_bio_pages(struct bio *bio) | |
251 | { | |
252 | struct bio_vec *bv; | |
253 | unsigned int i; | |
254 | ||
255 | __bio_for_each_segment(bv, bio, i, 0) { | |
256 | if (bv->bv_page != ZERO_PAGE(0)) | |
257 | __free_page(bv->bv_page); | |
258 | } | |
259 | } | |
260 | ||
261 | static void complete_and_free_bio(struct b_cmd *bc) | |
262 | { | |
263 | if (bc->bio) { | |
264 | const struct uc_map *ucm = &uc_map[bc->cmd.type]; | |
265 | ||
266 | if (ucm->data_transfer) { | |
267 | if (!ucm->map_zero) | |
268 | bio_unmap_user(bc->bio); | |
269 | else | |
270 | bc_put_bio_pages(bc->bio); | |
271 | } | |
272 | bio_put(bc->bio); | |
273 | bc->bio = NULL; | |
274 | } | |
275 | } | |
276 | ||
277 | static void b_dev_complete_commands(struct b_dev *bd) | |
278 | { | |
279 | struct b_cmd *bc; | |
280 | ||
281 | wait_event(bd->wq_done, !atomic_read(&bd->in_flight)); | |
282 | ||
283 | while ((bc = get_completed_command(bd)) != NULL) | |
284 | complete_and_free_bio(bc); | |
285 | } | |
286 | ||
287 | static int b_dev_validate_command(struct b_user_cmd *buc) | |
288 | { | |
2e59579d JA |
289 | int i; |
290 | ||
4f28f523 JA |
291 | if (!binject_buc_check_magic(buc)) |
292 | return -EINVAL; | |
293 | ||
2e59579d JA |
294 | for (i = 0; i < B_TYPE_NR; i++) { |
295 | const struct uc_map *ucm = &uc_map[i]; | |
296 | ||
297 | if (ucm->type != buc->type) | |
298 | continue; | |
299 | if (ucm->data_transfer && !buc->len) | |
300 | break; | |
301 | ||
302 | return 0; | |
4f28f523 | 303 | } |
2e59579d JA |
304 | |
305 | return -EINVAL; | |
4f28f523 JA |
306 | } |
307 | ||
308 | static void b_cmd_endio(struct bio *bio, int error) | |
309 | { | |
310 | struct b_cmd *bc = bio->bi_private; | |
311 | struct b_dev *bd = bc->bd; | |
53354c46 | 312 | struct b_dev_cpu *bdc; |
4f28f523 JA |
313 | unsigned long flags; |
314 | unsigned long now; | |
315 | ||
316 | now = ktime_to_ns(ktime_get()); | |
317 | bc->cmd.nsec = now - bc->issue_time; | |
318 | bc->cmd.error = error; | |
319 | ||
53354c46 JA |
320 | local_irq_save(flags); |
321 | bdc = per_cpu_ptr(bd->cpu_queue, smp_processor_id()); | |
322 | ||
323 | spin_lock(&bdc->lock); | |
324 | list_add_tail(&bc->list, &bdc->done_list); | |
53354c46 | 325 | spin_unlock_irqrestore(&bdc->lock, flags); |
4f28f523 JA |
326 | |
327 | atomic_dec(&bd->in_flight); | |
328 | ||
329 | wake_up(&bd->wq_done); | |
330 | } | |
331 | ||
332 | #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18) | |
333 | static int bio_cmd_endio(struct bio *bio, unsigned int bytes, int err) | |
334 | { | |
335 | if (bio->bi_size) | |
336 | return 1; | |
337 | ||
338 | b_cmd_endio(bio, err); | |
339 | return 0; | |
340 | } | |
341 | #else | |
342 | static void bio_cmd_endio(struct bio *bio, int err) | |
343 | { | |
344 | b_cmd_endio(bio, err); | |
345 | } | |
346 | #endif | |
347 | ||
41c1852f JA |
348 | #define len_to_pages(len) ((len + PAGE_SIZE - 1) / PAGE_SIZE) |
349 | ||
4f28f523 JA |
350 | static int zero_map_bio(struct request_queue *q, struct bio *bio, |
351 | const struct uc_map *ucm, unsigned int len) | |
352 | { | |
353 | unsigned int i, nr_pages, this_len, ret, err; | |
354 | struct page *page; | |
355 | ||
41c1852f | 356 | nr_pages = len_to_pages(len); |
4f28f523 JA |
357 | for (i = 0; i < nr_pages; i++) { |
358 | if (ucm->todevice) | |
359 | page = ZERO_PAGE(0); | |
360 | else { | |
361 | page = alloc_page(GFP_KERNEL); | |
362 | if (!page) { | |
363 | err = -ENOMEM; | |
364 | goto oom; | |
365 | } | |
366 | } | |
367 | ||
368 | this_len = PAGE_SIZE; | |
369 | if (this_len > len) | |
370 | this_len = len; | |
371 | ||
41c1852f JA |
372 | ret = bio_add_pc_page(q, bio, page, this_len, 0); |
373 | if (ret < this_len) { | |
4f28f523 JA |
374 | err = -E2BIG; |
375 | goto oom; | |
376 | } | |
377 | } | |
378 | return 0; | |
379 | oom: | |
380 | bc_put_bio_pages(bio); | |
381 | return err; | |
382 | } | |
383 | ||
384 | static void map_uc_to_bio_flags(struct bio *bio, struct b_user_cmd *uc) | |
385 | { | |
386 | unsigned int i; | |
387 | ||
388 | for (i = 0; i < 8 * sizeof(uc->flags); i++) { | |
389 | unsigned long mask; | |
390 | ||
391 | if (uc->flags & (1UL << i)) | |
392 | bio->bi_rw |= uc_flag_map[i]; | |
393 | ||
394 | mask = ~((1UL << i) - 1); | |
395 | if (!(mask & uc->flags)) | |
396 | break; | |
397 | } | |
398 | } | |
399 | ||
400 | static struct bio *map_uc_to_bio(struct b_dev *bd, struct b_user_cmd *uc) | |
401 | { | |
402 | struct request_queue *q = bdev_get_queue(bd->bdev); | |
403 | const struct uc_map *ucm = &uc_map[uc->type]; | |
404 | struct bio *bio; | |
405 | ||
406 | if (ucm->data_transfer && !ucm->map_zero) { | |
407 | bio = binject_map_bio(q, bd->bdev, uc->buf, uc->len, | |
408 | !ucm->todevice, GFP_KERNEL); | |
409 | } else { | |
41c1852f | 410 | bio = bio_alloc(GFP_KERNEL, len_to_pages(uc->len)); |
4f28f523 JA |
411 | if (bio) { |
412 | bio->bi_bdev = bd->bdev; | |
4f28f523 JA |
413 | if (ucm->map_zero && uc->len) { |
414 | int err; | |
415 | ||
416 | err = zero_map_bio(q, bio, ucm, uc->len); | |
417 | if (err) { | |
418 | bio_put(bio); | |
419 | bio = ERR_PTR(err); | |
420 | } | |
421 | } else | |
422 | bio->bi_size = uc->len; | |
423 | } | |
424 | } | |
425 | ||
426 | if (!bio) | |
427 | bio = ERR_PTR(-ENOMEM); | |
428 | else if (!IS_ERR(bio)) { | |
429 | map_uc_to_bio_flags(bio, uc); | |
430 | bio->bi_sector = uc->offset / binject_get_bs(q); | |
41c1852f | 431 | bio->bi_rw |= ucm->rw_flags; |
4f28f523 JA |
432 | } |
433 | ||
434 | return bio; | |
435 | } | |
436 | ||
437 | static int b_dev_add_command(struct b_dev *bd, struct b_cmd *bc) | |
438 | { | |
439 | struct b_user_cmd *uc = &bc->cmd; | |
440 | struct bio *bio; | |
441 | ||
442 | bio = map_uc_to_bio(bd, uc); | |
443 | if (IS_ERR(bio)) | |
444 | return PTR_ERR(bio); | |
445 | ||
446 | bio_get(bio); | |
447 | bc->bio = bio; | |
448 | ||
449 | bio->bi_end_io = bio_cmd_endio; | |
450 | bio->bi_private = bc; | |
451 | ||
452 | bc->issue_time = ktime_to_ns(ktime_get()); | |
453 | ||
454 | atomic_inc(&bd->in_flight); | |
455 | submit_bio(bio->bi_rw, bio); | |
456 | return 0; | |
457 | } | |
458 | ||
459 | static void b_dev_free_command(struct b_dev *bd, struct b_cmd *bc) | |
460 | { | |
146229b6 | 461 | BUG_ON(!list_empty(&bc->list)); |
4f28f523 JA |
462 | kmem_cache_free(b_slab, bc); |
463 | } | |
464 | ||
465 | /* | |
466 | * We are always writable, as we have an infinite queue depth | |
467 | */ | |
468 | static unsigned int b_dev_poll(struct file *file, poll_table *wait) | |
469 | { | |
470 | struct b_dev *bd = file->private_data; | |
471 | unsigned int mask = POLLOUT; | |
472 | ||
473 | poll_wait(file, &bd->wq_done, wait); | |
474 | ||
08ceb814 JA |
475 | if (bd_pending_done(bd)) |
476 | mask |= POLLIN | POLLRDNORM; | |
4f28f523 JA |
477 | |
478 | return mask; | |
479 | } | |
480 | ||
481 | static int b_dev_release(struct inode *inode, struct file *file) | |
482 | { | |
483 | struct b_dev *bd = file->private_data; | |
484 | ||
485 | b_dev_put(bd); | |
486 | return 0; | |
487 | } | |
488 | ||
489 | static struct b_dev *b_dev_lookup(int minor) | |
490 | { | |
491 | struct b_dev *bd; | |
492 | ||
493 | rcu_read_lock(); | |
41c1852f | 494 | |
4f28f523 | 495 | bd = idr_find(&b_minor_idr, minor); |
41c1852f JA |
496 | if (bd && !atomic_inc_not_zero(&bd->ref)) |
497 | bd = NULL; | |
4f28f523 | 498 | |
41c1852f | 499 | rcu_read_unlock(); |
4f28f523 JA |
500 | return bd; |
501 | } | |
502 | ||
503 | static int b_dev_open(struct inode *inode, struct file *file) | |
504 | { | |
505 | struct b_dev *bd; | |
506 | ||
507 | bd = b_dev_lookup(iminor(inode)); | |
508 | if (!bd) | |
509 | return -ENODEV; | |
510 | ||
511 | file->private_data = bd; | |
512 | return 0; | |
513 | } | |
514 | ||
515 | static ssize_t b_dev_write(struct file *file, const char __user *buf, | |
516 | size_t count, loff_t *ppos) | |
517 | { | |
518 | struct b_dev *bd = file->private_data; | |
41c1852f | 519 | struct b_cmd *bc = NULL; |
4f28f523 JA |
520 | unsigned int total; |
521 | ssize_t done = 0; | |
522 | int err = 0; | |
523 | ||
524 | if (count % sizeof(struct b_user_cmd)) | |
525 | return -EINVAL; | |
526 | ||
527 | total = count / sizeof(struct b_user_cmd); | |
528 | while (total) { | |
4f28f523 JA |
529 | bc = get_free_command(bd); |
530 | if (IS_ERR(bc)) { | |
531 | err = PTR_ERR(bc); | |
41c1852f | 532 | bc = NULL; |
4f28f523 JA |
533 | break; |
534 | } | |
535 | ||
536 | if (copy_from_user(&bc->cmd, buf, sizeof(struct b_user_cmd))) { | |
537 | err = -EFAULT; | |
538 | break; | |
539 | } | |
540 | ||
541 | err = b_dev_validate_command(&bc->cmd); | |
41c1852f | 542 | if (err) |
4f28f523 | 543 | break; |
4f28f523 JA |
544 | |
545 | err = b_dev_add_command(bd, bc); | |
41c1852f | 546 | if (err) |
4f28f523 | 547 | break; |
4f28f523 JA |
548 | |
549 | done += sizeof(struct b_user_cmd); | |
550 | buf += sizeof(struct b_user_cmd); | |
551 | total--; | |
a8527ea7 | 552 | bc = NULL; |
4f28f523 JA |
553 | } |
554 | ||
41c1852f JA |
555 | if (bc) |
556 | b_dev_free_command(bd, bc); | |
557 | ||
4f28f523 JA |
558 | *ppos = done; |
559 | if (!done) | |
560 | done = err; | |
561 | ||
562 | return done; | |
563 | } | |
564 | ||
565 | static ssize_t b_dev_read(struct file *file, char __user *buf, size_t count, | |
566 | loff_t *ppos) | |
567 | { | |
568 | struct b_dev *bd = file->private_data; | |
569 | unsigned int total; | |
570 | ssize_t done = 0; | |
571 | int err = 0; | |
572 | ||
573 | if (count % sizeof(struct b_user_cmd)) | |
574 | return -EINVAL; | |
575 | ||
576 | total = count / sizeof(struct b_user_cmd); | |
577 | while (total) { | |
578 | struct b_cmd *bc; | |
579 | ||
580 | bc = get_done_command(bd, !(file->f_flags & O_NONBLOCK)); | |
581 | if (IS_ERR(bc)) { | |
582 | err = PTR_ERR(bc); | |
583 | break; | |
584 | } | |
585 | ||
586 | complete_and_free_bio(bc); | |
587 | ||
588 | if (copy_to_user(buf, &bc->cmd, sizeof(bc->cmd))) | |
589 | err = -EFAULT; | |
590 | ||
591 | b_dev_free_command(bd, bc); | |
592 | ||
593 | if (err) | |
594 | break; | |
595 | ||
596 | done += sizeof(struct b_user_cmd); | |
597 | buf += sizeof(struct b_user_cmd); | |
598 | total--; | |
599 | } | |
600 | ||
601 | *ppos = done; | |
602 | if (!done) | |
603 | done = err; | |
604 | ||
605 | return done; | |
606 | } | |
607 | ||
608 | static const struct file_operations b_dev_fops = { | |
609 | .open = b_dev_open, | |
610 | .release = b_dev_release, | |
611 | .read = b_dev_read, | |
612 | .write = b_dev_write, | |
613 | .poll = b_dev_poll, | |
614 | .owner = THIS_MODULE, | |
615 | }; | |
616 | ||
617 | static int b_del_dev(struct b_ioctl_cmd *bic) | |
618 | { | |
619 | struct b_dev *bd; | |
620 | ||
621 | bd = b_dev_lookup(bic->minor); | |
622 | if (bd) { | |
623 | spin_lock(&b_dev_lock); | |
624 | b_dev_remove_lookup(bd); | |
625 | spin_unlock(&b_dev_lock); | |
626 | ||
627 | /* | |
628 | * Our lookup grabbed a reference, drop two | |
629 | */ | |
630 | b_dev_put(bd); | |
631 | b_dev_put(bd); | |
632 | return 0; | |
633 | } | |
634 | ||
635 | return -ENODEV; | |
636 | } | |
637 | ||
638 | static int b_add_dev(struct b_ioctl_cmd *bic) | |
639 | { | |
640 | struct inode *inode; | |
641 | struct file *file; | |
642 | struct b_dev *bd; | |
53354c46 | 643 | int ret, cpu; |
4f28f523 JA |
644 | |
645 | file = fget(bic->fd); | |
646 | if (!file) | |
647 | return -EBADF; | |
648 | ||
649 | __module_get(THIS_MODULE); | |
650 | ||
651 | inode = file->f_mapping->host; | |
652 | if (!S_ISBLK(inode->i_mode)) { | |
653 | ret = -EINVAL; | |
654 | goto out_put; | |
655 | } | |
656 | ||
657 | ret = idr_pre_get(&b_minor_idr, GFP_KERNEL); | |
658 | if (!ret) { | |
659 | ret = -ENOMEM; | |
660 | goto out_put; | |
661 | } | |
662 | ||
663 | bd = kzalloc(sizeof(*bd), GFP_KERNEL); | |
664 | if (!bd) { | |
665 | ret = -ENOMEM; | |
666 | goto out_put; | |
667 | } | |
668 | ||
53354c46 JA |
669 | bd->cpu_queue = alloc_percpu(struct b_dev_cpu); |
670 | if (!bd->cpu_queue) { | |
671 | kfree(bd); | |
672 | ret = -ENOMEM; | |
673 | goto out_put; | |
674 | } | |
675 | ||
676 | for_each_possible_cpu(cpu) { | |
677 | struct b_dev_cpu *bdc; | |
678 | ||
679 | bdc = per_cpu_ptr(bd->cpu_queue, cpu); | |
680 | INIT_LIST_HEAD(&bdc->done_list); | |
53354c46 JA |
681 | spin_lock_init(&bdc->lock); |
682 | } | |
683 | ||
4f28f523 | 684 | atomic_set(&bd->ref, 1); |
53354c46 JA |
685 | spin_lock_init(&bd->done_lock); |
686 | INIT_LIST_HEAD(&bd->reaped_done); | |
4f28f523 JA |
687 | init_waitqueue_head(&bd->wq_done); |
688 | bd->file = file; | |
689 | bd->bdev = inode->i_bdev;; | |
690 | ||
691 | spin_lock(&b_dev_lock); | |
692 | ||
693 | ret = idr_get_new(&b_minor_idr, bd, &bd->minor); | |
694 | if (ret < 0) | |
695 | goto out_unlock; | |
696 | ||
697 | if (bd->minor >= B_MAX_DEVS) | |
698 | goto out_idr; | |
699 | ||
700 | spin_unlock(&b_dev_lock); | |
701 | ||
702 | INIT_LIST_HEAD(&bd->device_list); | |
703 | bd->dev = binject_device_create(b_class, NULL, | |
704 | MKDEV(b_major, bd->minor), bd, "binject%d", bd->minor); | |
705 | ||
706 | spin_lock(&b_dev_lock); | |
707 | ||
708 | if (IS_ERR(bd->dev)) | |
709 | goto out_idr; | |
710 | ||
711 | list_add_tail(&bd->device_list, &b_dev_list); | |
5b7b80bb | 712 | bic->minor = bd->minor; |
4f28f523 JA |
713 | spin_unlock(&b_dev_lock); |
714 | return 0; | |
715 | out_idr: | |
716 | idr_remove(&b_minor_idr, bd->minor); | |
717 | out_unlock: | |
718 | spin_unlock(&b_dev_lock); | |
53354c46 | 719 | free_percpu(bd->cpu_queue); |
4f28f523 JA |
720 | kfree(bd); |
721 | out_put: | |
722 | fput(file); | |
723 | module_put(THIS_MODULE); | |
724 | return ret; | |
725 | } | |
726 | ||
727 | static long b_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg) | |
728 | { | |
729 | int __user *uarg = (int __user *) arg; | |
730 | struct b_ioctl_cmd bic; | |
5b7b80bb | 731 | int ret = -ENOTTY; |
4f28f523 JA |
732 | |
733 | if (copy_from_user(&bic, uarg, sizeof(bic))) | |
734 | return -EFAULT; | |
735 | ||
736 | switch (cmd) { | |
737 | case 0: | |
5b7b80bb JA |
738 | ret = b_add_dev(&bic); |
739 | if (!ret && copy_to_user(uarg, &bic, sizeof(bic))) { | |
740 | b_del_dev(&bic); | |
741 | ret = -EFAULT; | |
742 | } | |
743 | break; | |
4f28f523 | 744 | case 1: |
5b7b80bb JA |
745 | ret = b_del_dev(&bic); |
746 | break; | |
4f28f523 JA |
747 | default: |
748 | break; | |
749 | } | |
750 | ||
751 | return -ENOTTY; | |
752 | } | |
753 | ||
754 | static const struct file_operations b_misc_fops = { | |
755 | .unlocked_ioctl = b_misc_ioctl, | |
756 | .owner = THIS_MODULE, | |
757 | }; | |
758 | ||
759 | static struct miscdevice b_misc_dev = { | |
760 | .minor = MISC_DYNAMIC_MINOR, | |
761 | .name = "binject-ctl", | |
762 | .fops = &b_misc_fops, | |
763 | }; | |
764 | ||
765 | static void __exit b_exit(void) | |
766 | { | |
767 | synchronize_rcu(); | |
768 | kmem_cache_destroy(b_slab); | |
769 | class_destroy(b_class); | |
770 | misc_deregister(&b_misc_dev); | |
771 | } | |
772 | ||
fefac214 | 773 | static void __b_cmd_init_once(struct b_cmd *bc) |
146229b6 | 774 | { |
146229b6 JA |
775 | INIT_LIST_HEAD(&bc->list); |
776 | } | |
777 | ||
fefac214 JA |
778 | #ifdef KCOMPAT_OLD_SLAB |
779 | static void b_cmd_init_once(void *data, kmem_cache_t *slab, unsigned long flags) | |
780 | { | |
781 | if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) == | |
782 | SLAB_CTOR_CONSTRUCTOR) | |
783 | __b_cmd_init_once(data); | |
784 | } | |
785 | #else | |
786 | static void b_cmd_init_once(void *data) | |
787 | { | |
788 | __b_cmd_init_once(data); | |
789 | } | |
790 | #endif | |
791 | ||
4f28f523 JA |
792 | static int __init b_init(void) |
793 | { | |
794 | int ret; | |
795 | ||
146229b6 JA |
796 | b_slab = binject_create_slab("binject", sizeof(struct b_cmd), |
797 | SLAB_HWCACHE_ALIGN, b_cmd_init_once); | |
4f28f523 JA |
798 | if (!b_slab) { |
799 | printk(KERN_ERR "binject: failed to create cmd slab\n"); | |
800 | return -ENOMEM; | |
801 | } | |
802 | ||
803 | ret = misc_register(&b_misc_dev); | |
804 | if (ret < 0) | |
805 | goto fail_misc; | |
806 | ||
807 | b_major = register_chrdev(0, "binject", &b_dev_fops); | |
808 | if (b_major < 0) | |
809 | goto fail_chr; | |
810 | ||
811 | b_class = class_create(THIS_MODULE, "binject"); | |
812 | if (IS_ERR(b_class)) | |
813 | goto fail_class; | |
814 | ||
815 | return 0; | |
816 | fail_class: | |
817 | unregister_chrdev(b_major, "binject"); | |
818 | fail_chr: | |
819 | misc_deregister(&b_misc_dev); | |
820 | fail_misc: | |
821 | kmem_cache_destroy(b_slab); | |
822 | return ret; | |
823 | } | |
824 | ||
825 | module_init(b_init); | |
826 | module_exit(b_exit); | |
827 | ||
828 | MODULE_LICENSE("GPL"); | |
829 | MODULE_AUTHOR("Jens Axboe <jaxboe@fusionio.com>"); |