[linux-block.git] / block / blk-mq-sysfs.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>

#include "blk.h"
#include "blk-mq.h"

static void blk_mq_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj);

	free_percpu(ctxs->queue_ctx);
	kfree(ctxs);
}

static void blk_mq_ctx_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj);

	/* ctx->ctxs won't be released until all ctx are freed */
	kobject_put(&ctx->ctxs->kobj);
}

static void blk_mq_hw_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
						  kobj);

	blk_free_flush_queue(hctx->fq);
	sbitmap_free(&hctx->ctx_map);
	free_cpumask_var(hctx->cpumask);
	kfree(hctx->ctxs);
	kfree(hctx);
}

struct blk_mq_hw_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
};

static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
				    struct attribute *attr, char *page)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->show)
		return -EIO;

	mutex_lock(&q->sysfs_lock);
	res = entry->show(hctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
					    char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_tags);
}

static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
						     char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
}

static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	const size_t size = PAGE_SIZE - 1;
	unsigned int i, first = 1;
	int ret = 0, pos = 0;

	for_each_cpu(i, hctx->cpumask) {
		if (first)
			ret = snprintf(pos + page, size - pos, "%u", i);
		else
			ret = snprintf(pos + page, size - pos, ", %u", i);

		if (ret >= size - pos)
			break;

		first = 0;
		pos += ret;
	}

	ret = snprintf(pos + page, size + 1 - pos, "\n");
	return pos + ret;
}

static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
	.attr = {.name = "nr_tags", .mode = 0444 },
	.show = blk_mq_hw_sysfs_nr_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
	.attr = {.name = "nr_reserved_tags", .mode = 0444 },
	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
	.attr = {.name = "cpu_list", .mode = 0444 },
	.show = blk_mq_hw_sysfs_cpus_show,
};

static struct attribute *default_hw_ctx_attrs[] = {
	&blk_mq_hw_sysfs_nr_tags.attr,
	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
	&blk_mq_hw_sysfs_cpus.attr,
	NULL,
};
ATTRIBUTE_GROUPS(default_hw_ctx);

static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	.show	= blk_mq_hw_sysfs_show,
};

static const struct kobj_type blk_mq_ktype = {
	.release	= blk_mq_sysfs_release,
};

static const struct kobj_type blk_mq_ctx_ktype = {
	.release	= blk_mq_ctx_sysfs_release,
};

static const struct kobj_type blk_mq_hw_ktype = {
	.sysfs_ops	= &blk_mq_hw_sysfs_ops,
	.default_groups = default_hw_ctx_groups,
	.release	= blk_mq_hw_sysfs_release,
};

static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_ctx *ctx;
	int i;

	if (!hctx->nr_ctx)
		return;

	hctx_for_each_ctx(hctx, ctx, i)
		kobject_del(&ctx->kobj);

	kobject_del(&hctx->kobj);
}

static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	struct blk_mq_ctx *ctx;
	int i, j, ret;

	if (!hctx->nr_ctx)
		return 0;

	ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num);
	if (ret)
		return ret;

	hctx_for_each_ctx(hctx, ctx, i) {
		ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
		if (ret)
			goto out;
	}

	return 0;
out:
	hctx_for_each_ctx(hctx, ctx, j) {
		if (j < i)
			kobject_del(&ctx->kobj);
	}
	kobject_del(&hctx->kobj);
	return ret;
}

void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
{
	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
}

void blk_mq_sysfs_deinit(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);
		kobject_put(&ctx->kobj);
	}
	kobject_put(q->mq_kobj);
}

void blk_mq_sysfs_init(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	kobject_init(q->mq_kobj, &blk_mq_ktype);

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);

		kobject_get(q->mq_kobj);
		kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
	}
}

int blk_mq_sysfs_register(struct gendisk *disk)
{
	struct request_queue *q = disk->queue;
	struct blk_mq_hw_ctx *hctx;
	unsigned long i, j;
	int ret;

	lockdep_assert_held(&q->sysfs_dir_lock);

	ret = kobject_add(q->mq_kobj, &disk_to_dev(disk)->kobj, "mq");
	if (ret < 0)
		goto out;

	kobject_uevent(q->mq_kobj, KOBJ_ADD);

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			goto unreg;
	}

	q->mq_sysfs_init_done = true;

out:
	return ret;

unreg:
	queue_for_each_hw_ctx(q, hctx, j) {
		if (j < i)
			blk_mq_unregister_hctx(hctx);
	}

	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
	kobject_del(q->mq_kobj);
	return ret;
}

void blk_mq_sysfs_unregister(struct gendisk *disk)
{
	struct request_queue *q = disk->queue;
	struct blk_mq_hw_ctx *hctx;
	unsigned long i;

	lockdep_assert_held(&q->sysfs_dir_lock);

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
	kobject_del(q->mq_kobj);

	q->mq_sysfs_init_done = false;
}

void blk_mq_sysfs_unregister_hctxs(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned long i;

	mutex_lock(&q->sysfs_dir_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

unlock:
	mutex_unlock(&q->sysfs_dir_lock);
}

int blk_mq_sysfs_register_hctxs(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned long i;
	int ret = 0;

	mutex_lock(&q->sysfs_dir_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			break;
	}

unlock:
	mutex_unlock(&q->sysfs_dir_lock);

	return ret;
}
Commit	Line	Data
3dcf60bc	1	// SPDX-License-Identifier: GPL-2.0
320ae51f JA	2	#include <linux/kernel.h>
	3	#include <linux/module.h>
	4	#include <linux/backing-dev.h>
	5	#include <linux/bio.h>
	6	#include <linux/blkdev.h>
	7	#include <linux/mm.h>
	8	#include <linux/init.h>
	9	#include <linux/slab.h>
	10	#include <linux/workqueue.h>
	11	#include <linux/smp.h>
	12
c7e2d94b	13	#include "blk.h"
320ae51f	14	#include "blk-mq.h"
320ae51f JA	15
	16	static void blk_mq_sysfs_release(struct kobject *kobj)
	17	{
1db4909e ML	18	struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj);
	19
	20	free_percpu(ctxs->queue_ctx);
	21	kfree(ctxs);
	22	}
	23
	24	static void blk_mq_ctx_sysfs_release(struct kobject *kobj)
	25	{
	26	struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	27
	28	/* ctx->ctxs won't be released until all ctx are freed */
	29	kobject_put(&ctx->ctxs->kobj);
320ae51f JA	30	}
320ae51f JA	31
6c8b232e ML	32	static void blk_mq_hw_sysfs_release(struct kobject *kobj)
	33	{
	34	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
	35	kobj);
c7e2d94b	36
c7e2d94b ML	37	blk_free_flush_queue(hctx->fq);
c7e2d94b ML	38	sbitmap_free(&hctx->ctx_map);
01388df3	39	free_cpumask_var(hctx->cpumask);
6c8b232e ML	40	kfree(hctx->ctxs);
	41	kfree(hctx);
	42	}
	43
320ae51f JA	44	struct blk_mq_hw_ctx_sysfs_entry {
	45	struct attribute attr;
	46	ssize_t (show)(struct blk_mq_hw_ctx , char *);
320ae51f JA	47	};
320ae51f JA	48
320ae51f JA	49	static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
	50	struct attribute attr, char page)
	51	{
	52	struct blk_mq_hw_ctx_sysfs_entry *entry;
	53	struct blk_mq_hw_ctx *hctx;
	54	struct request_queue *q;
	55	ssize_t res;
	56
	57	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	58	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	59	q = hctx->queue;
	60
	61	if (!entry->show)
	62	return -EIO;
	63
320ae51f	64	mutex_lock(&q->sysfs_lock);
bae85c15	65	res = entry->show(hctx, page);
320ae51f JA	66	mutex_unlock(&q->sysfs_lock);
	67	return res;
	68	}
	69
d96b37c0 OS	70	static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	71	char *page)
bd166ef1	72	{
d96b37c0	73	return sprintf(page, "%u\n", hctx->tags->nr_tags);
bd166ef1 JA	74	}
bd166ef1 JA	75
d96b37c0 OS	76	static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	77	char *page)
320ae51f	78	{
d96b37c0	79	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
320ae51f JA	80	}
320ae51f JA	81
676141e4 JA	82	static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx hctx, char page)
676141e4 JA	83	{
8962842c	84	const size_t size = PAGE_SIZE - 1;
cb2da43e	85	unsigned int i, first = 1;
8962842c	86	int ret = 0, pos = 0;
676141e4	87
cb2da43e	88	for_each_cpu(i, hctx->cpumask) {
676141e4	89	if (first)
8962842c	90	ret = snprintf(pos + page, size - pos, "%u", i);
676141e4	91	else
8962842c ML	92	ret = snprintf(pos + page, size - pos, ", %u", i);
	93
	94	if (ret >= size - pos)
	95	break;
676141e4 JA	96
676141e4 JA	97	first = 0;
8962842c	98	pos += ret;
676141e4 JA	99	}
676141e4 JA	100
d2c9be89	101	ret = snprintf(pos + page, size + 1 - pos, "\n");
8962842c	102	return pos + ret;
676141e4 JA	103	}
676141e4 JA	104
d96b37c0	105	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
5657a819	106	.attr = {.name = "nr_tags", .mode = 0444 },
d96b37c0 OS	107	.show = blk_mq_hw_sysfs_nr_tags_show,
	108	};
	109	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
5657a819	110	.attr = {.name = "nr_reserved_tags", .mode = 0444 },
d96b37c0 OS	111	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
d96b37c0 OS	112	};
676141e4	113	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
5657a819	114	.attr = {.name = "cpu_list", .mode = 0444 },
676141e4 JA	115	.show = blk_mq_hw_sysfs_cpus_show,
676141e4 JA	116	};
320ae51f JA	117
320ae51f JA	118	static struct attribute *default_hw_ctx_attrs[] = {
d96b37c0 OS	119	&blk_mq_hw_sysfs_nr_tags.attr,
d96b37c0 OS	120	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
676141e4	121	&blk_mq_hw_sysfs_cpus.attr,
320ae51f JA	122	NULL,
320ae51f JA	123	};
800f5aa1	124	ATTRIBUTE_GROUPS(default_hw_ctx);
320ae51f	125
320ae51f JA	126	static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
320ae51f JA	127	.show = blk_mq_hw_sysfs_show,
320ae51f JA	128	};
320ae51f JA	129
5f622417	130	static const struct kobj_type blk_mq_ktype = {
320ae51f JA	131	.release = blk_mq_sysfs_release,
	132	};
	133
5f622417	134	static const struct kobj_type blk_mq_ctx_ktype = {
1db4909e	135	.release = blk_mq_ctx_sysfs_release,
320ae51f JA	136	};
320ae51f JA	137
5f622417	138	static const struct kobj_type blk_mq_hw_ktype = {
320ae51f	139	.sysfs_ops = &blk_mq_hw_sysfs_ops,
800f5aa1	140	.default_groups = default_hw_ctx_groups,
6c8b232e	141	.release = blk_mq_hw_sysfs_release,
320ae51f JA	142	};
320ae51f JA	143
ee3c5db0	144	static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	145	{
	146	struct blk_mq_ctx *ctx;
	147	int i;
	148
4593fdbe	149	if (!hctx->nr_ctx)
67aec14c JA	150	return;
	151
	152	hctx_for_each_ctx(hctx, ctx, i)
	153	kobject_del(&ctx->kobj);
	154
	155	kobject_del(&hctx->kobj);
	156	}
	157
ee3c5db0	158	static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	159	{
	160	struct request_queue *q = hctx->queue;
	161	struct blk_mq_ctx *ctx;
4b7a21c5	162	int i, j, ret;
67aec14c	163
4593fdbe	164	if (!hctx->nr_ctx)
67aec14c JA	165	return 0;
67aec14c JA	166
1db4909e	167	ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num);
67aec14c JA	168	if (ret)
	169	return ret;
	170
	171	hctx_for_each_ctx(hctx, ctx, i) {
	172	ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
	173	if (ret)
4b7a21c5	174	goto out;
67aec14c JA	175	}
67aec14c JA	176
4b7a21c5 YB	177	return 0;
	178	out:
	179	hctx_for_each_ctx(hctx, ctx, j) {
	180	if (j < i)
	181	kobject_del(&ctx->kobj);
	182	}
	183	kobject_del(&hctx->kobj);
67aec14c JA	184	return ret;
	185	}
	186
868f2f0b KB	187	void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
	188	{
	189	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
	190	}
	191
7ea5fe31 ML	192	void blk_mq_sysfs_deinit(struct request_queue *q)
	193	{
	194	struct blk_mq_ctx *ctx;
	195	int cpu;
	196
	197	for_each_possible_cpu(cpu) {
	198	ctx = per_cpu_ptr(q->queue_ctx, cpu);
	199	kobject_put(&ctx->kobj);
	200	}
1db4909e	201	kobject_put(q->mq_kobj);
7ea5fe31 ML	202	}
7ea5fe31 ML	203
737f98cf	204	void blk_mq_sysfs_init(struct request_queue *q)
67aec14c	205	{
67aec14c	206	struct blk_mq_ctx *ctx;
897bb0c7	207	int cpu;
67aec14c	208
1db4909e	209	kobject_init(q->mq_kobj, &blk_mq_ktype);
67aec14c	210
897bb0c7 TG	211	for_each_possible_cpu(cpu) {
897bb0c7 TG	212	ctx = per_cpu_ptr(q->queue_ctx, cpu);
1db4909e ML	213
1db4909e ML	214	kobject_get(q->mq_kobj);
06a41a99	215	kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
897bb0c7	216	}
67aec14c JA	217	}
67aec14c JA	218
8682b92e	219	int blk_mq_sysfs_register(struct gendisk *disk)
320ae51f	220	{
8682b92e	221	struct request_queue *q = disk->queue;
320ae51f	222	struct blk_mq_hw_ctx *hctx;
4f481208 ML	223	unsigned long i, j;
4f481208 ML	224	int ret;
320ae51f	225
cecf5d87	226	lockdep_assert_held(&q->sysfs_dir_lock);
4593fdbe	227
8682b92e	228	ret = kobject_add(q->mq_kobj, &disk_to_dev(disk)->kobj, "mq");
320ae51f	229	if (ret < 0)
4593fdbe	230	goto out;
320ae51f	231
1db4909e	232	kobject_uevent(q->mq_kobj, KOBJ_ADD);
320ae51f JA	233
320ae51f JA	234	queue_for_each_hw_ctx(q, hctx, i) {
67aec14c	235	ret = blk_mq_register_hctx(hctx);
320ae51f	236	if (ret)
f05d1ba7	237	goto unreg;
320ae51f JA	238	}
320ae51f JA	239
f05d1ba7	240	q->mq_sysfs_init_done = true;
2d0364c8	241
4593fdbe	242	out:
2d0364c8	243	return ret;
f05d1ba7 BVA	244
f05d1ba7 BVA	245	unreg:
4f481208 ML	246	queue_for_each_hw_ctx(q, hctx, j) {
	247	if (j < i)
	248	blk_mq_unregister_hctx(hctx);
	249	}
f05d1ba7	250
1db4909e ML	251	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
1db4909e ML	252	kobject_del(q->mq_kobj);
f05d1ba7	253	return ret;
2d0364c8 BVA	254	}
2d0364c8 BVA	255
8682b92e CH	256	void blk_mq_sysfs_unregister(struct gendisk *disk)
	257	{
	258	struct request_queue *q = disk->queue;
	259	struct blk_mq_hw_ctx *hctx;
	260	unsigned long i;
	261
	262	lockdep_assert_held(&q->sysfs_dir_lock);
	263
	264	queue_for_each_hw_ctx(q, hctx, i)
	265	blk_mq_unregister_hctx(hctx);
	266
	267	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
	268	kobject_del(q->mq_kobj);
	269
	270	q->mq_sysfs_init_done = false;
	271	}
	272
eaa870f9	273	void blk_mq_sysfs_unregister_hctxs(struct request_queue *q)
67aec14c JA	274	{
67aec14c JA	275	struct blk_mq_hw_ctx *hctx;
4f481208	276	unsigned long i;
67aec14c	277
cecf5d87	278	mutex_lock(&q->sysfs_dir_lock);
4593fdbe	279	if (!q->mq_sysfs_init_done)
2d0364c8	280	goto unlock;
4593fdbe	281
67aec14c JA	282	queue_for_each_hw_ctx(q, hctx, i)
67aec14c JA	283	blk_mq_unregister_hctx(hctx);
2d0364c8 BVA	284
2d0364c8 BVA	285	unlock:
cecf5d87	286	mutex_unlock(&q->sysfs_dir_lock);
67aec14c JA	287	}
67aec14c JA	288
eaa870f9	289	int blk_mq_sysfs_register_hctxs(struct request_queue *q)
67aec14c JA	290	{
67aec14c JA	291	struct blk_mq_hw_ctx *hctx;
4f481208 ML	292	unsigned long i;
4f481208 ML	293	int ret = 0;
67aec14c	294
cecf5d87	295	mutex_lock(&q->sysfs_dir_lock);
4593fdbe	296	if (!q->mq_sysfs_init_done)
2d0364c8	297	goto unlock;
4593fdbe	298
67aec14c JA	299	queue_for_each_hw_ctx(q, hctx, i) {
	300	ret = blk_mq_register_hctx(hctx);
	301	if (ret)
	302	break;
	303	}
	304
2d0364c8	305	unlock:
cecf5d87	306	mutex_unlock(&q->sysfs_dir_lock);
2d0364c8	307
67aec14c JA	308	return ret;
67aec14c JA	309	}