[linux-2.6-block.git] / fs / io-wq.h

#ifndef INTERNAL_IO_WQ_H
#define INTERNAL_IO_WQ_H

struct io_wq;

enum {
	IO_WQ_WORK_CANCEL	= 1,
	IO_WQ_WORK_HASHED	= 4,
	IO_WQ_WORK_UNBOUND	= 32,
	IO_WQ_WORK_NO_CANCEL	= 256,
	IO_WQ_WORK_CONCURRENT	= 512,

	IO_WQ_HASH_SHIFT	= 24,	/* upper 8 bits are used for hash key */
};

enum io_wq_cancel {
	IO_WQ_CANCEL_OK,	/* cancelled before started */
	IO_WQ_CANCEL_RUNNING,	/* found, running, and attempted cancelled */
	IO_WQ_CANCEL_NOTFOUND,	/* work not found */
};

struct io_wq_work_node {
	struct io_wq_work_node *next;
};

struct io_wq_work_list {
	struct io_wq_work_node *first;
	struct io_wq_work_node *last;
};

static inline void wq_list_add_after(struct io_wq_work_node *node,
				     struct io_wq_work_node *pos,
				     struct io_wq_work_list *list)
{
	struct io_wq_work_node *next = pos->next;

	pos->next = node;
	node->next = next;
	if (!next)
		list->last = node;
}

static inline void wq_list_add_tail(struct io_wq_work_node *node,
				    struct io_wq_work_list *list)
{
	if (!list->first) {
		list->last = node;
		WRITE_ONCE(list->first, node);
	} else {
		list->last->next = node;
		list->last = node;
	}
}

static inline void wq_list_cut(struct io_wq_work_list *list,
			       struct io_wq_work_node *last,
			       struct io_wq_work_node *prev)
{
	/* first in the list, if prev==NULL */
	if (!prev)
		WRITE_ONCE(list->first, last->next);
	else
		prev->next = last->next;

	if (last == list->last)
		list->last = prev;
	last->next = NULL;
}

static inline void wq_list_del(struct io_wq_work_list *list,
			       struct io_wq_work_node *node,
			       struct io_wq_work_node *prev)
{
	wq_list_cut(list, node, prev);
}

#define wq_list_for_each(pos, prv, head)			\
	for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)

#define wq_list_empty(list)	(READ_ONCE((list)->first) == NULL)
#define INIT_WQ_LIST(list)	do {				\
	(list)->first = NULL;					\
	(list)->last = NULL;					\
} while (0)

struct io_wq_work {
	struct io_wq_work_node list;
	void (*func)(struct io_wq_work **);
	struct files_struct *files;
	struct mm_struct *mm;
	const struct cred *creds;
	struct fs_struct *fs;
	unsigned flags;
	pid_t task_pid;
};

#define INIT_IO_WORK(work, _func)				\
	do {							\
		*(work) = (struct io_wq_work){ .func = _func };	\
	} while (0)						\

static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
{
	if (!work->list.next)
		return NULL;

	return container_of(work->list.next, struct io_wq_work, list);
}

typedef void (free_work_fn)(struct io_wq_work *);

struct io_wq_data {
	struct user_struct *user;

	free_work_fn *free_work;
};

struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data);
bool io_wq_get(struct io_wq *wq, struct io_wq_data *data);
void io_wq_destroy(struct io_wq *wq);

void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work);
void io_wq_hash_work(struct io_wq_work *work, void *val);

static inline bool io_wq_is_hashed(struct io_wq_work *work)
{
	return work->flags & IO_WQ_WORK_HASHED;
}

void io_wq_cancel_all(struct io_wq *wq);
enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork);
enum io_wq_cancel io_wq_cancel_pid(struct io_wq *wq, pid_t pid);

typedef bool (work_cancel_fn)(struct io_wq_work *, void *);

enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
					void *data);

#if defined(CONFIG_IO_WQ)
extern void io_wq_worker_sleeping(struct task_struct *);
extern void io_wq_worker_running(struct task_struct *);
#else
static inline void io_wq_worker_sleeping(struct task_struct *tsk)
{
}
static inline void io_wq_worker_running(struct task_struct *tsk)
{
}
#endif

static inline bool io_wq_current_is_worker(void)
{
	return in_task() && (current->flags & PF_IO_WORKER);
}
#endif
Commit	Line	Data
771b53d0 JA	1	#ifndef INTERNAL_IO_WQ_H
	2	#define INTERNAL_IO_WQ_H
	3
	4	struct io_wq;
	5
	6	enum {
	7	IO_WQ_WORK_CANCEL = 1,
771b53d0	8	IO_WQ_WORK_HASHED = 4,
c5def4ab	9	IO_WQ_WORK_UNBOUND = 32,
0c9d5ccd	10	IO_WQ_WORK_NO_CANCEL = 256,
895e2ca0	11	IO_WQ_WORK_CONCURRENT = 512,
771b53d0 JA	12
	13	IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */
	14	};
	15
	16	enum io_wq_cancel {
	17	IO_WQ_CANCEL_OK, /* cancelled before started */
	18	IO_WQ_CANCEL_RUNNING, /* found, running, and attempted cancelled */
	19	IO_WQ_CANCEL_NOTFOUND, /* work not found */
	20	};
	21
6206f0e1 JA	22	struct io_wq_work_node {
	23	struct io_wq_work_node *next;
	24	};
	25
	26	struct io_wq_work_list {
	27	struct io_wq_work_node *first;
	28	struct io_wq_work_node *last;
	29	};
	30
86f3cd1b PB	31	static inline void wq_list_add_after(struct io_wq_work_node *node,
	32	struct io_wq_work_node *pos,
	33	struct io_wq_work_list *list)
	34	{
	35	struct io_wq_work_node *next = pos->next;
	36
	37	pos->next = node;
	38	node->next = next;
	39	if (!next)
	40	list->last = node;
	41	}
	42
6206f0e1 JA	43	static inline void wq_list_add_tail(struct io_wq_work_node *node,
	44	struct io_wq_work_list *list)
	45	{
	46	if (!list->first) {
e995d512 JA	47	list->last = node;
e995d512 JA	48	WRITE_ONCE(list->first, node);
6206f0e1 JA	49	} else {
	50	list->last->next = node;
	51	list->last = node;
	52	}
	53	}
	54
86f3cd1b PB	55	static inline void wq_list_cut(struct io_wq_work_list *list,
86f3cd1b PB	56	struct io_wq_work_node *last,
6206f0e1 JA	57	struct io_wq_work_node *prev)
6206f0e1 JA	58	{
86f3cd1b PB	59	/* first in the list, if prev==NULL */
	60	if (!prev)
	61	WRITE_ONCE(list->first, last->next);
	62	else
	63	prev->next = last->next;
	64
	65	if (last == list->last)
6206f0e1	66	list->last = prev;
86f3cd1b PB	67	last->next = NULL;
	68	}
	69
	70	static inline void wq_list_del(struct io_wq_work_list *list,
	71	struct io_wq_work_node *node,
	72	struct io_wq_work_node *prev)
	73	{
	74	wq_list_cut(list, node, prev);
6206f0e1 JA	75	}
	76
	77	#define wq_list_for_each(pos, prv, head) \
	78	for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)
	79
e995d512	80	#define wq_list_empty(list) (READ_ONCE((list)->first) == NULL)
6206f0e1 JA	81	#define INIT_WQ_LIST(list) do { \
	82	(list)->first = NULL; \
	83	(list)->last = NULL; \
	84	} while (0)
	85
771b53d0	86	struct io_wq_work {
18a542ff	87	struct io_wq_work_node list;
771b53d0	88	void (func)(struct io_wq_work *);
fcb323cc	89	struct files_struct *files;
cccf0ee8 JA	90	struct mm_struct *mm;
cccf0ee8 JA	91	const struct cred *creds;
9392a27d	92	struct fs_struct *fs;
6206f0e1	93	unsigned flags;
36282881	94	pid_t task_pid;
771b53d0 JA	95	};
771b53d0 JA	96
2d141dd2 JA	97	#define INIT_IO_WORK(work, _func) \
	98	do { \
	99	*(work) = (struct io_wq_work){ .func = _func }; \
	100	} while (0) \
771b53d0	101
86f3cd1b PB	102	static inline struct io_wq_work wq_next_work(struct io_wq_work work)
	103	{
	104	if (!work->list.next)
	105	return NULL;
	106
	107	return container_of(work->list.next, struct io_wq_work, list);
	108	}
	109
e9fd9396	110	typedef void (free_work_fn)(struct io_wq_work *);
7d723065	111
576a347b	112	struct io_wq_data {
576a347b JA	113	struct user_struct *user;
576a347b JA	114
e9fd9396	115	free_work_fn *free_work;
576a347b JA	116	};
	117
	118	struct io_wq io_wq_create(unsigned bounded, struct io_wq_data data);
eba6f5a3	119	bool io_wq_get(struct io_wq wq, struct io_wq_data data);
771b53d0 JA	120	void io_wq_destroy(struct io_wq *wq);
	121
	122	void io_wq_enqueue(struct io_wq wq, struct io_wq_work work);
8766dd51 PB	123	void io_wq_hash_work(struct io_wq_work work, void val);
	124
	125	static inline bool io_wq_is_hashed(struct io_wq_work *work)
	126	{
	127	return work->flags & IO_WQ_WORK_HASHED;
	128	}
771b53d0 JA	129
	130	void io_wq_cancel_all(struct io_wq *wq);
	131	enum io_wq_cancel io_wq_cancel_work(struct io_wq wq, struct io_wq_work cwork);
36282881	132	enum io_wq_cancel io_wq_cancel_pid(struct io_wq *wq, pid_t pid);
771b53d0	133
62755e35 JA	134	typedef bool (work_cancel_fn)(struct io_wq_work , void );
	135
	136	enum io_wq_cancel io_wq_cancel_cb(struct io_wq wq, work_cancel_fn cancel,
	137	void *data);
	138
771b53d0 JA	139	#if defined(CONFIG_IO_WQ)
	140	extern void io_wq_worker_sleeping(struct task_struct *);
	141	extern void io_wq_worker_running(struct task_struct *);
	142	#else
	143	static inline void io_wq_worker_sleeping(struct task_struct *tsk)
	144	{
	145	}
	146	static inline void io_wq_worker_running(struct task_struct *tsk)
	147	{
	148	}
525b305d	149	#endif
771b53d0	150
525b305d JA	151	static inline bool io_wq_current_is_worker(void)
	152	{
	153	return in_task() && (current->flags & PF_IO_WORKER);
	154	}
	155	#endif