#define B_MAX_DEVS 64
+struct b_dev_cpu {
+ spinlock_t lock;
+ struct list_head done_list;
+};
+
struct b_dev {
struct list_head device_list;
- struct list_head done_list;
+ struct list_head reaped_done;
+ spinlock_t done_lock;
atomic_t in_flight;
- unsigned int done_cmds;
wait_queue_head_t wq_done;
struct block_device *bdev;
- spinlock_t lock;
atomic_t ref;
struct file *file;
struct device *dev;
int minor;
+ struct b_dev_cpu __percpu *cpu_queue;
struct rcu_head rcu_free;
};
static void bd_rcu_free(struct rcu_head *head)
{
- kfree(container_of(head, struct b_dev, rcu_free));
+ struct b_dev *bd = container_of(head, struct b_dev, rcu_free);
+
+ free_percpu(bd->cpu_queue);
+ kfree(bd);
}
static void b_dev_put(struct b_dev *bd)
bc = kmem_cache_alloc(b_slab, GFP_KERNEL);
if (bc) {
- memset(bc, 0, sizeof(*bc));
- INIT_LIST_HEAD(&bc->list);
bc->bd = bd;
return bc;
}
static struct b_cmd *get_completed_command(struct b_dev *bd)
{
struct b_cmd *bc = NULL;
+ int cpu, spliced = 0;
- spin_lock_irq(&bd->lock);
- if (!list_empty(&bd->done_list)) {
- bc = list_entry(bd->done_list.next, struct b_cmd, list);
- bd->done_cmds--;
- list_del(&bc->list);
+ spin_lock(&bd->done_lock);
+ if (!list_empty(&bd->reaped_done)) {
+ret_one:
+ bc = list_entry(bd->reaped_done.next, struct b_cmd, list);
+ list_del_init(&bc->list);
}
- spin_unlock_irq(&bd->lock);
- return bc;
+ spin_unlock(&bd->done_lock);
+
+ if (bc)
+ return bc;
+
+ spin_lock(&bd->done_lock);
+ for_each_possible_cpu(cpu) {
+ struct b_dev_cpu *bdc = per_cpu_ptr(bd->cpu_queue, cpu);
+
+ spin_lock_irq(&bdc->lock);
+ if (!list_empty(&bdc->done_list)) {
+ list_splice_init(&bdc->done_list, &bd->reaped_done);
+ spliced++;
+ }
+ spin_unlock_irq(&bdc->lock);
+ }
+
+ if (spliced)
+ goto ret_one;
+
+ spin_unlock(&bd->done_lock);
+ return NULL;
+}
+
+static int bd_pending_done(struct b_dev *bd)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ struct b_dev_cpu *bdc = per_cpu_ptr(bd->cpu_queue, cpu);
+
+ if (!list_empty_careful(&bdc->done_list))
+ return 1;
+ }
+
+ return 0;
}
static struct b_cmd *get_done_command(struct b_dev *bd, int block)
if (!block)
break;
- ret = wait_event_interruptible(bd->wq_done, bd->done_cmds);
+ ret = wait_event_interruptible(bd->wq_done, bd_pending_done(bd));
if (ret) {
bc = ERR_PTR(-ERESTARTSYS);
break;
static int b_dev_validate_command(struct b_user_cmd *buc)
{
+ int i;
+
if (!binject_buc_check_magic(buc))
return -EINVAL;
- switch (buc->type) {
- case B_TYPE_WRITE:
- case B_TYPE_READ:
- case B_TYPE_DISCARD:
- case B_TYPE_READVOID:
- case B_TYPE_WRITEZERO:
- if (buc->len)
- return 0;
- return -EINVAL;
- default:
- return -EINVAL;
+ for (i = 0; i < B_TYPE_NR; i++) {
+ const struct uc_map *ucm = &uc_map[i];
+
+ if (ucm->type != buc->type)
+ continue;
+ if (ucm->data_transfer && !buc->len)
+ break;
+
+ return 0;
}
+
+ return -EINVAL;
}
static void b_cmd_endio(struct bio *bio, int error)
{
struct b_cmd *bc = bio->bi_private;
struct b_dev *bd = bc->bd;
+ struct b_dev_cpu *bdc;
unsigned long flags;
unsigned long now;
bc->cmd.nsec = now - bc->issue_time;
bc->cmd.error = error;
- spin_lock_irqsave(&bd->lock, flags);
- list_add_tail(&bc->list, &bd->done_list);
- bd->done_cmds++;
- spin_unlock_irqrestore(&bd->lock, flags);
+ local_irq_save(flags);
+ bdc = per_cpu_ptr(bd->cpu_queue, smp_processor_id());
+
+ spin_lock(&bdc->lock);
+ list_add_tail(&bc->list, &bdc->done_list);
+ spin_unlock_irqrestore(&bdc->lock, flags);
atomic_dec(&bd->in_flight);
- wake_up(&bd->wq_done);
+ smp_mb();
+ if (waitqueue_active(&bd->wq_done))
+ wake_up(&bd->wq_done);
}
#if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
static void b_dev_free_command(struct b_dev *bd, struct b_cmd *bc)
{
+ BUG_ON(!list_empty(&bc->list));
kmem_cache_free(b_slab, bc);
}
poll_wait(file, &bd->wq_done, wait);
- spin_lock_irq(&bd->lock);
- if (!list_empty(&bd->done_list))
+ if (bd_pending_done(bd))
mask |= POLLIN | POLLRDNORM;
- spin_unlock_irq(&bd->lock);
return mask;
}
struct inode *inode;
struct file *file;
struct b_dev *bd;
- int ret;
+ int ret, cpu;
file = fget(bic->fd);
if (!file)
goto out_put;
}
+ bd->cpu_queue = alloc_percpu(struct b_dev_cpu);
+ if (!bd->cpu_queue) {
+ kfree(bd);
+ ret = -ENOMEM;
+ goto out_put;
+ }
+
+ for_each_possible_cpu(cpu) {
+ struct b_dev_cpu *bdc;
+
+ bdc = per_cpu_ptr(bd->cpu_queue, cpu);
+ INIT_LIST_HEAD(&bdc->done_list);
+ spin_lock_init(&bdc->lock);
+ }
+
atomic_set(&bd->ref, 1);
- spin_lock_init(&bd->lock);
- INIT_LIST_HEAD(&bd->done_list);
+ spin_lock_init(&bd->done_lock);
+ INIT_LIST_HEAD(&bd->reaped_done);
init_waitqueue_head(&bd->wq_done);
bd->file = file;
bd->bdev = inode->i_bdev;;
idr_remove(&b_minor_idr, bd->minor);
out_unlock:
spin_unlock(&b_dev_lock);
+ free_percpu(bd->cpu_queue);
kfree(bd);
out_put:
fput(file);
return -EFAULT;
switch (cmd) {
- case 0:
+ case B_IOCTL_ADD:
ret = b_add_dev(&bic);
if (!ret && copy_to_user(uarg, &bic, sizeof(bic))) {
b_del_dev(&bic);
ret = -EFAULT;
}
break;
- case 1:
+ case B_IOCTL_DEL:
ret = b_del_dev(&bic);
break;
default:
break;
}
- return -ENOTTY;
+ return ret;
}
static const struct file_operations b_misc_fops = {
misc_deregister(&b_misc_dev);
}
+static void __b_cmd_init_once(struct b_cmd *bc)
+{
+ INIT_LIST_HEAD(&bc->list);
+}
+
+#ifdef KCOMPAT_OLD_SLAB
+static void b_cmd_init_once(void *data, kmem_cache_t *slab, unsigned long flags)
+{
+ if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
+ SLAB_CTOR_CONSTRUCTOR)
+ __b_cmd_init_once(data);
+}
+#else
+static void b_cmd_init_once(void *data)
+{
+ __b_cmd_init_once(data);
+}
+#endif
+
static int __init b_init(void)
{
int ret;
- b_slab = binject_create_slab("binject", sizeof(struct b_cmd));
+ b_slab = binject_create_slab("binject", sizeof(struct b_cmd),
+ SLAB_HWCACHE_ALIGN, b_cmd_init_once);
if (!b_slab) {
printk(KERN_ERR "binject: failed to create cmd slab\n");
return -ENOMEM;
projects / binject.git / blobdiff