[linux-2.6-block.git] / block / blk-mq-sysfs.c

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>

#include <linux/blk-mq.h>
#include "blk-mq.h"
#include "blk-mq-tag.h"

static void blk_mq_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj);

	free_percpu(ctxs->queue_ctx);
	kfree(ctxs);
}

static void blk_mq_ctx_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj);

	/* ctx->ctxs won't be released until all ctx are freed */
	kobject_put(&ctx->ctxs->kobj);
}

static void blk_mq_hw_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
						  kobj);
	free_cpumask_var(hctx->cpumask);
	kfree(hctx->ctxs);
	kfree(hctx);
}

struct blk_mq_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_ctx *, char *);
	ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t);
};

struct blk_mq_hw_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
	ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t);
};

static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr,
				 char *page)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(ctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr,
				  const char *page, size_t length)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(ctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
				    struct attribute *attr, char *page)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(hctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
				     struct attribute *attr, const char *page,
				     size_t length)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(hctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
					    char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_tags);
}

static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
						     char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
}

static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	unsigned int i, first = 1;
	ssize_t ret = 0;

	for_each_cpu(i, hctx->cpumask) {
		if (first)
			ret += sprintf(ret + page, "%u", i);
		else
			ret += sprintf(ret + page, ", %u", i);

		first = 0;
	}

	ret += sprintf(ret + page, "\n");
	return ret;
}

static struct attribute *default_ctx_attrs[] = {
	NULL,
};

static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
	.attr = {.name = "nr_tags", .mode = 0444 },
	.show = blk_mq_hw_sysfs_nr_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
	.attr = {.name = "nr_reserved_tags", .mode = 0444 },
	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
	.attr = {.name = "cpu_list", .mode = 0444 },
	.show = blk_mq_hw_sysfs_cpus_show,
};

static struct attribute *default_hw_ctx_attrs[] = {
	&blk_mq_hw_sysfs_nr_tags.attr,
	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
	&blk_mq_hw_sysfs_cpus.attr,
	NULL,
};

static const struct sysfs_ops blk_mq_sysfs_ops = {
	.show	= blk_mq_sysfs_show,
	.store	= blk_mq_sysfs_store,
};

static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	.show	= blk_mq_hw_sysfs_show,
	.store	= blk_mq_hw_sysfs_store,
};

static struct kobj_type blk_mq_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.release	= blk_mq_sysfs_release,
};

static struct kobj_type blk_mq_ctx_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.default_attrs	= default_ctx_attrs,
	.release	= blk_mq_ctx_sysfs_release,
};

static struct kobj_type blk_mq_hw_ktype = {
	.sysfs_ops	= &blk_mq_hw_sysfs_ops,
	.default_attrs	= default_hw_ctx_attrs,
	.release	= blk_mq_hw_sysfs_release,
};

static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_ctx *ctx;
	int i;

	if (!hctx->nr_ctx)
		return;

	hctx_for_each_ctx(hctx, ctx, i)
		kobject_del(&ctx->kobj);

	kobject_del(&hctx->kobj);
}

static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	struct blk_mq_ctx *ctx;
	int i, ret;

	if (!hctx->nr_ctx)
		return 0;

	ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num);
	if (ret)
		return ret;

	hctx_for_each_ctx(hctx, ctx, i) {
		ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
		if (ret)
			break;
	}

	return ret;
}

void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	lockdep_assert_held(&q->sysfs_lock);

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
	kobject_del(q->mq_kobj);
	kobject_put(&dev->kobj);

	q->mq_sysfs_init_done = false;
}

void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
{
	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
}

void blk_mq_sysfs_deinit(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);
		kobject_put(&ctx->kobj);
	}
	kobject_put(q->mq_kobj);
}

void blk_mq_sysfs_init(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	kobject_init(q->mq_kobj, &blk_mq_ktype);

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);

		kobject_get(q->mq_kobj);
		kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
	}
}

int __blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int ret, i;

	WARN_ON_ONCE(!q->kobj.parent);
	lockdep_assert_held(&q->sysfs_lock);

	ret = kobject_add(q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
	if (ret < 0)
		goto out;

	kobject_uevent(q->mq_kobj, KOBJ_ADD);

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			goto unreg;
	}

	q->mq_sysfs_init_done = true;

out:
	return ret;

unreg:
	while (--i >= 0)
		blk_mq_unregister_hctx(q->queue_hw_ctx[i]);

	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
	kobject_del(q->mq_kobj);
	kobject_put(&dev->kobj);
	return ret;
}

int blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
	int ret;

	mutex_lock(&q->sysfs_lock);
	ret = __blk_mq_register_dev(dev, q);
	mutex_unlock(&q->sysfs_lock);

	return ret;
}

void blk_mq_sysfs_unregister(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	mutex_lock(&q->sysfs_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

unlock:
	mutex_unlock(&q->sysfs_lock);
}

int blk_mq_sysfs_register(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i, ret = 0;

	mutex_lock(&q->sysfs_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			break;
	}

unlock:
	mutex_unlock(&q->sysfs_lock);

	return ret;
}
Commit	Line	Data
320ae51f JA	1	#include <linux/kernel.h>
	2	#include <linux/module.h>
	3	#include <linux/backing-dev.h>
	4	#include <linux/bio.h>
	5	#include <linux/blkdev.h>
	6	#include <linux/mm.h>
	7	#include <linux/init.h>
	8	#include <linux/slab.h>
	9	#include <linux/workqueue.h>
	10	#include <linux/smp.h>
	11
	12	#include <linux/blk-mq.h>
	13	#include "blk-mq.h"
	14	#include "blk-mq-tag.h"
	15
	16	static void blk_mq_sysfs_release(struct kobject *kobj)
	17	{
1db4909e ML	18	struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj);
	19
	20	free_percpu(ctxs->queue_ctx);
	21	kfree(ctxs);
	22	}
	23
	24	static void blk_mq_ctx_sysfs_release(struct kobject *kobj)
	25	{
	26	struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	27
	28	/* ctx->ctxs won't be released until all ctx are freed */
	29	kobject_put(&ctx->ctxs->kobj);
320ae51f JA	30	}
320ae51f JA	31
6c8b232e ML	32	static void blk_mq_hw_sysfs_release(struct kobject *kobj)
	33	{
	34	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
	35	kobj);
01388df3	36	free_cpumask_var(hctx->cpumask);
6c8b232e ML	37	kfree(hctx->ctxs);
	38	kfree(hctx);
	39	}
	40
320ae51f JA	41	struct blk_mq_ctx_sysfs_entry {
	42	struct attribute attr;
	43	ssize_t (show)(struct blk_mq_ctx , char *);
	44	ssize_t (store)(struct blk_mq_ctx , const char *, size_t);
	45	};
	46
	47	struct blk_mq_hw_ctx_sysfs_entry {
	48	struct attribute attr;
	49	ssize_t (show)(struct blk_mq_hw_ctx , char *);
	50	ssize_t (store)(struct blk_mq_hw_ctx , const char *, size_t);
	51	};
	52
	53	static ssize_t blk_mq_sysfs_show(struct kobject kobj, struct attribute attr,
	54	char *page)
	55	{
	56	struct blk_mq_ctx_sysfs_entry *entry;
	57	struct blk_mq_ctx *ctx;
	58	struct request_queue *q;
	59	ssize_t res;
	60
	61	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	62	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	63	q = ctx->queue;
	64
	65	if (!entry->show)
	66	return -EIO;
	67
	68	res = -ENOENT;
	69	mutex_lock(&q->sysfs_lock);
	70	if (!blk_queue_dying(q))
	71	res = entry->show(ctx, page);
	72	mutex_unlock(&q->sysfs_lock);
	73	return res;
	74	}
	75
	76	static ssize_t blk_mq_sysfs_store(struct kobject kobj, struct attribute attr,
	77	const char *page, size_t length)
	78	{
	79	struct blk_mq_ctx_sysfs_entry *entry;
	80	struct blk_mq_ctx *ctx;
	81	struct request_queue *q;
	82	ssize_t res;
	83
	84	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	85	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	86	q = ctx->queue;
	87
	88	if (!entry->store)
	89	return -EIO;
	90
	91	res = -ENOENT;
	92	mutex_lock(&q->sysfs_lock);
	93	if (!blk_queue_dying(q))
	94	res = entry->store(ctx, page, length);
	95	mutex_unlock(&q->sysfs_lock);
	96	return res;
	97	}
	98
	99	static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
	100	struct attribute attr, char page)
	101	{
	102	struct blk_mq_hw_ctx_sysfs_entry *entry;
	103	struct blk_mq_hw_ctx *hctx;
	104	struct request_queue *q;
105	ssize_t res;
106
107	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
108	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
109	q = hctx->queue;
110
111	if (!entry->show)
112	return -EIO;
113
114	res = -ENOENT;
115	mutex_lock(&q->sysfs_lock);
116	if (!blk_queue_dying(q))
117	res = entry->show(hctx, page);
118	mutex_unlock(&q->sysfs_lock);
119	return res;
120	}
121
122	static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
123	struct attribute attr, const char page,
124	size_t length)
125	{
126	struct blk_mq_hw_ctx_sysfs_entry *entry;
127	struct blk_mq_hw_ctx *hctx;
128	struct request_queue *q;
129	ssize_t res;
130
131	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
132	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
133	q = hctx->queue;
134
135	if (!entry->store)
136	return -EIO;
137
138	res = -ENOENT;
139	mutex_lock(&q->sysfs_lock);
140	if (!blk_queue_dying(q))
141	res = entry->store(hctx, page, length);
142	mutex_unlock(&q->sysfs_lock);
143	return res;
144	}
145
d96b37c0 OS	146	static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	147	char *page)
bd166ef1	148	{
d96b37c0	149	return sprintf(page, "%u\n", hctx->tags->nr_tags);
bd166ef1 JA	150	}
bd166ef1 JA	151
d96b37c0 OS	152	static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	153	char *page)
320ae51f	154	{
d96b37c0	155	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
320ae51f JA	156	}
320ae51f JA	157
676141e4 JA	158	static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx hctx, char page)
676141e4 JA	159	{
cb2da43e	160	unsigned int i, first = 1;
676141e4 JA	161	ssize_t ret = 0;
676141e4 JA	162
cb2da43e	163	for_each_cpu(i, hctx->cpumask) {
676141e4 JA	164	if (first)
	165	ret += sprintf(ret + page, "%u", i);
	166	else
	167	ret += sprintf(ret + page, ", %u", i);
	168
	169	first = 0;
	170	}
	171
676141e4 JA	172	ret += sprintf(ret + page, "\n");
	173	return ret;
	174	}
	175
320ae51f	176	static struct attribute *default_ctx_attrs[] = {
320ae51f JA	177	NULL,
	178	};
	179
d96b37c0	180	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
5657a819	181	.attr = {.name = "nr_tags", .mode = 0444 },
d96b37c0 OS	182	.show = blk_mq_hw_sysfs_nr_tags_show,
	183	};
	184	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
5657a819	185	.attr = {.name = "nr_reserved_tags", .mode = 0444 },
d96b37c0 OS	186	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
d96b37c0 OS	187	};
676141e4	188	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
5657a819	189	.attr = {.name = "cpu_list", .mode = 0444 },
676141e4 JA	190	.show = blk_mq_hw_sysfs_cpus_show,
676141e4 JA	191	};
320ae51f JA	192
320ae51f JA	193	static struct attribute *default_hw_ctx_attrs[] = {
d96b37c0 OS	194	&blk_mq_hw_sysfs_nr_tags.attr,
d96b37c0 OS	195	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
676141e4	196	&blk_mq_hw_sysfs_cpus.attr,
320ae51f JA	197	NULL,
	198	};
	199
	200	static const struct sysfs_ops blk_mq_sysfs_ops = {
	201	.show = blk_mq_sysfs_show,
	202	.store = blk_mq_sysfs_store,
	203	};
	204
	205	static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	206	.show = blk_mq_hw_sysfs_show,
	207	.store = blk_mq_hw_sysfs_store,
	208	};
	209
	210	static struct kobj_type blk_mq_ktype = {
	211	.sysfs_ops = &blk_mq_sysfs_ops,
	212	.release = blk_mq_sysfs_release,
	213	};
	214
	215	static struct kobj_type blk_mq_ctx_ktype = {
	216	.sysfs_ops = &blk_mq_sysfs_ops,
	217	.default_attrs = default_ctx_attrs,
1db4909e	218	.release = blk_mq_ctx_sysfs_release,
320ae51f JA	219	};
	220
	221	static struct kobj_type blk_mq_hw_ktype = {
	222	.sysfs_ops = &blk_mq_hw_sysfs_ops,
	223	.default_attrs = default_hw_ctx_attrs,
6c8b232e	224	.release = blk_mq_hw_sysfs_release,
320ae51f JA	225	};
320ae51f JA	226
ee3c5db0	227	static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	228	{
	229	struct blk_mq_ctx *ctx;
	230	int i;
	231
4593fdbe	232	if (!hctx->nr_ctx)
67aec14c JA	233	return;
	234
	235	hctx_for_each_ctx(hctx, ctx, i)
	236	kobject_del(&ctx->kobj);
	237
	238	kobject_del(&hctx->kobj);
	239	}
	240
ee3c5db0	241	static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	242	{
	243	struct request_queue *q = hctx->queue;
	244	struct blk_mq_ctx *ctx;
	245	int i, ret;
	246
4593fdbe	247	if (!hctx->nr_ctx)
67aec14c JA	248	return 0;
67aec14c JA	249
1db4909e	250	ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num);
67aec14c JA	251	if (ret)
	252	return ret;
	253
	254	hctx_for_each_ctx(hctx, ctx, i) {
	255	ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
	256	if (ret)
	257	break;
	258	}
	259
	260	return ret;
	261	}
	262
667257e8	263	void blk_mq_unregister_dev(struct device dev, struct request_queue q)
320ae51f	264	{
85157366	265	struct blk_mq_hw_ctx *hctx;
7ea5fe31	266	int i;
85157366	267
2d0364c8 BVA	268	lockdep_assert_held(&q->sysfs_lock);
2d0364c8 BVA	269
6c8b232e	270	queue_for_each_hw_ctx(q, hctx, i)
67aec14c JA	271	blk_mq_unregister_hctx(hctx);
67aec14c JA	272
1db4909e ML	273	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
1db4909e ML	274	kobject_del(q->mq_kobj);
b21d5b30	275	kobject_put(&dev->kobj);
4593fdbe AM	276
4593fdbe AM	277	q->mq_sysfs_init_done = false;
c0f3fd2b JA	278	}
c0f3fd2b JA	279
868f2f0b KB	280	void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
	281	{
	282	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
	283	}
	284
7ea5fe31 ML	285	void blk_mq_sysfs_deinit(struct request_queue *q)
	286	{
	287	struct blk_mq_ctx *ctx;
	288	int cpu;
	289
	290	for_each_possible_cpu(cpu) {
	291	ctx = per_cpu_ptr(q->queue_ctx, cpu);
	292	kobject_put(&ctx->kobj);
	293	}
1db4909e	294	kobject_put(q->mq_kobj);
7ea5fe31 ML	295	}
7ea5fe31 ML	296
737f98cf	297	void blk_mq_sysfs_init(struct request_queue *q)
67aec14c	298	{
67aec14c	299	struct blk_mq_ctx *ctx;
897bb0c7	300	int cpu;
67aec14c	301
1db4909e	302	kobject_init(q->mq_kobj, &blk_mq_ktype);
67aec14c	303
897bb0c7 TG	304	for_each_possible_cpu(cpu) {
897bb0c7 TG	305	ctx = per_cpu_ptr(q->queue_ctx, cpu);
1db4909e ML	306
1db4909e ML	307	kobject_get(q->mq_kobj);
06a41a99	308	kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
897bb0c7	309	}
67aec14c JA	310	}
67aec14c JA	311
2d0364c8	312	int __blk_mq_register_dev(struct device dev, struct request_queue q)
320ae51f	313	{
320ae51f	314	struct blk_mq_hw_ctx *hctx;
67aec14c	315	int ret, i;
320ae51f	316
2d0364c8 BVA	317	WARN_ON_ONCE(!q->kobj.parent);
2d0364c8 BVA	318	lockdep_assert_held(&q->sysfs_lock);
4593fdbe	319
1db4909e	320	ret = kobject_add(q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
320ae51f	321	if (ret < 0)
4593fdbe	322	goto out;
320ae51f	323
1db4909e	324	kobject_uevent(q->mq_kobj, KOBJ_ADD);
320ae51f JA	325
320ae51f JA	326	queue_for_each_hw_ctx(q, hctx, i) {
67aec14c	327	ret = blk_mq_register_hctx(hctx);
320ae51f	328	if (ret)
f05d1ba7	329	goto unreg;
320ae51f JA	330	}
320ae51f JA	331
f05d1ba7	332	q->mq_sysfs_init_done = true;
2d0364c8	333
4593fdbe	334	out:
2d0364c8	335	return ret;
f05d1ba7 BVA	336
	337	unreg:
	338	while (--i >= 0)
	339	blk_mq_unregister_hctx(q->queue_hw_ctx[i]);
	340
1db4909e ML	341	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
1db4909e ML	342	kobject_del(q->mq_kobj);
f05d1ba7 BVA	343	kobject_put(&dev->kobj);
f05d1ba7 BVA	344	return ret;
2d0364c8 BVA	345	}
	346
	347	int blk_mq_register_dev(struct device dev, struct request_queue q)
	348	{
	349	int ret;
	350
	351	mutex_lock(&q->sysfs_lock);
	352	ret = __blk_mq_register_dev(dev, q);
	353	mutex_unlock(&q->sysfs_lock);
320ae51f	354
4593fdbe	355	return ret;
320ae51f	356	}
67aec14c JA	357
	358	void blk_mq_sysfs_unregister(struct request_queue *q)
	359	{
	360	struct blk_mq_hw_ctx *hctx;
	361	int i;
	362
2d0364c8	363	mutex_lock(&q->sysfs_lock);
4593fdbe	364	if (!q->mq_sysfs_init_done)
2d0364c8	365	goto unlock;
4593fdbe	366
67aec14c JA	367	queue_for_each_hw_ctx(q, hctx, i)
67aec14c JA	368	blk_mq_unregister_hctx(hctx);
2d0364c8 BVA	369
	370	unlock:
	371	mutex_unlock(&q->sysfs_lock);
67aec14c JA	372	}
	373
	374	int blk_mq_sysfs_register(struct request_queue *q)
	375	{
	376	struct blk_mq_hw_ctx *hctx;
	377	int i, ret = 0;
	378
2d0364c8	379	mutex_lock(&q->sysfs_lock);
4593fdbe	380	if (!q->mq_sysfs_init_done)
2d0364c8	381	goto unlock;
4593fdbe	382
67aec14c JA	383	queue_for_each_hw_ctx(q, hctx, i) {
	384	ret = blk_mq_register_hctx(hctx);
	385	if (ret)
	386	break;
	387	}
	388
2d0364c8 BVA	389	unlock:
	390	mutex_unlock(&q->sysfs_lock);
	391
67aec14c JA	392	return ret;
67aec14c JA	393	}