Commit | Line | Data |
---|---|---|
771b53d0 JA |
1 | #ifndef INTERNAL_IO_WQ_H |
2 | #define INTERNAL_IO_WQ_H | |
3 | ||
4 | struct io_wq; | |
5 | ||
6 | enum { | |
7 | IO_WQ_WORK_CANCEL = 1, | |
e883a79d PB |
8 | IO_WQ_WORK_HASHED = 2, |
9 | IO_WQ_WORK_UNBOUND = 4, | |
10 | IO_WQ_WORK_NO_CANCEL = 8, | |
11 | IO_WQ_WORK_CONCURRENT = 16, | |
771b53d0 | 12 | |
0f203765 JA |
13 | IO_WQ_WORK_FILES = 32, |
14 | IO_WQ_WORK_FS = 64, | |
15 | IO_WQ_WORK_MM = 128, | |
16 | IO_WQ_WORK_CREDS = 256, | |
17 | IO_WQ_WORK_BLKCG = 512, | |
18 | ||
771b53d0 JA |
19 | IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */ |
20 | }; | |
21 | ||
22 | enum io_wq_cancel { | |
23 | IO_WQ_CANCEL_OK, /* cancelled before started */ | |
24 | IO_WQ_CANCEL_RUNNING, /* found, running, and attempted cancelled */ | |
25 | IO_WQ_CANCEL_NOTFOUND, /* work not found */ | |
26 | }; | |
27 | ||
6206f0e1 JA |
28 | struct io_wq_work_node { |
29 | struct io_wq_work_node *next; | |
30 | }; | |
31 | ||
32 | struct io_wq_work_list { | |
33 | struct io_wq_work_node *first; | |
34 | struct io_wq_work_node *last; | |
35 | }; | |
36 | ||
86f3cd1b PB |
37 | static inline void wq_list_add_after(struct io_wq_work_node *node, |
38 | struct io_wq_work_node *pos, | |
39 | struct io_wq_work_list *list) | |
40 | { | |
41 | struct io_wq_work_node *next = pos->next; | |
42 | ||
43 | pos->next = node; | |
44 | node->next = next; | |
45 | if (!next) | |
46 | list->last = node; | |
47 | } | |
48 | ||
6206f0e1 JA |
49 | static inline void wq_list_add_tail(struct io_wq_work_node *node, |
50 | struct io_wq_work_list *list) | |
51 | { | |
52 | if (!list->first) { | |
e995d512 JA |
53 | list->last = node; |
54 | WRITE_ONCE(list->first, node); | |
6206f0e1 JA |
55 | } else { |
56 | list->last->next = node; | |
57 | list->last = node; | |
58 | } | |
59 | } | |
60 | ||
86f3cd1b PB |
61 | static inline void wq_list_cut(struct io_wq_work_list *list, |
62 | struct io_wq_work_node *last, | |
6206f0e1 JA |
63 | struct io_wq_work_node *prev) |
64 | { | |
86f3cd1b PB |
65 | /* first in the list, if prev==NULL */ |
66 | if (!prev) | |
67 | WRITE_ONCE(list->first, last->next); | |
68 | else | |
69 | prev->next = last->next; | |
70 | ||
71 | if (last == list->last) | |
6206f0e1 | 72 | list->last = prev; |
86f3cd1b PB |
73 | last->next = NULL; |
74 | } | |
75 | ||
76 | static inline void wq_list_del(struct io_wq_work_list *list, | |
77 | struct io_wq_work_node *node, | |
78 | struct io_wq_work_node *prev) | |
79 | { | |
80 | wq_list_cut(list, node, prev); | |
6206f0e1 JA |
81 | } |
82 | ||
83 | #define wq_list_for_each(pos, prv, head) \ | |
84 | for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next) | |
85 | ||
e995d512 | 86 | #define wq_list_empty(list) (READ_ONCE((list)->first) == NULL) |
6206f0e1 JA |
87 | #define INIT_WQ_LIST(list) do { \ |
88 | (list)->first = NULL; \ | |
89 | (list)->last = NULL; \ | |
90 | } while (0) | |
91 | ||
771b53d0 | 92 | struct io_wq_work { |
18a542ff | 93 | struct io_wq_work_node list; |
fcb323cc | 94 | struct files_struct *files; |
cccf0ee8 | 95 | struct mm_struct *mm; |
91d8f519 DZ |
96 | #ifdef CONFIG_BLK_CGROUP |
97 | struct cgroup_subsys_state *blkcg_css; | |
98 | #endif | |
cccf0ee8 | 99 | const struct cred *creds; |
9b828492 | 100 | struct nsproxy *nsproxy; |
9392a27d | 101 | struct fs_struct *fs; |
57f1a649 | 102 | unsigned long fsize; |
6206f0e1 | 103 | unsigned flags; |
771b53d0 JA |
104 | }; |
105 | ||
86f3cd1b PB |
106 | static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) |
107 | { | |
108 | if (!work->list.next) | |
109 | return NULL; | |
110 | ||
111 | return container_of(work->list.next, struct io_wq_work, list); | |
112 | } | |
113 | ||
e9fd9396 | 114 | typedef void (free_work_fn)(struct io_wq_work *); |
f4db7182 | 115 | typedef struct io_wq_work *(io_wq_work_fn)(struct io_wq_work *); |
7d723065 | 116 | |
576a347b | 117 | struct io_wq_data { |
576a347b JA |
118 | struct user_struct *user; |
119 | ||
f5fa38c5 | 120 | io_wq_work_fn *do_work; |
e9fd9396 | 121 | free_work_fn *free_work; |
576a347b JA |
122 | }; |
123 | ||
124 | struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data); | |
eba6f5a3 | 125 | bool io_wq_get(struct io_wq *wq, struct io_wq_data *data); |
771b53d0 JA |
126 | void io_wq_destroy(struct io_wq *wq); |
127 | ||
128 | void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); | |
8766dd51 PB |
129 | void io_wq_hash_work(struct io_wq_work *work, void *val); |
130 | ||
131 | static inline bool io_wq_is_hashed(struct io_wq_work *work) | |
132 | { | |
133 | return work->flags & IO_WQ_WORK_HASHED; | |
134 | } | |
771b53d0 JA |
135 | |
136 | void io_wq_cancel_all(struct io_wq *wq); | |
137 | enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork); | |
138 | ||
62755e35 JA |
139 | typedef bool (work_cancel_fn)(struct io_wq_work *, void *); |
140 | ||
141 | enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, | |
4f26bda1 | 142 | void *data, bool cancel_all); |
62755e35 | 143 | |
aa96bf8a JA |
144 | struct task_struct *io_wq_get_task(struct io_wq *wq); |
145 | ||
771b53d0 JA |
146 | #if defined(CONFIG_IO_WQ) |
147 | extern void io_wq_worker_sleeping(struct task_struct *); | |
148 | extern void io_wq_worker_running(struct task_struct *); | |
149 | #else | |
150 | static inline void io_wq_worker_sleeping(struct task_struct *tsk) | |
151 | { | |
152 | } | |
153 | static inline void io_wq_worker_running(struct task_struct *tsk) | |
154 | { | |
155 | } | |
525b305d | 156 | #endif |
771b53d0 | 157 | |
525b305d JA |
158 | static inline bool io_wq_current_is_worker(void) |
159 | { | |
160 | return in_task() && (current->flags & PF_IO_WORKER); | |
161 | } | |
162 | #endif |