Commit | Line | Data |
---|---|---|
86db1e29 JA |
1 | /* |
2 | * Functions related to io context handling | |
3 | */ | |
4 | #include <linux/kernel.h> | |
5 | #include <linux/module.h> | |
6 | #include <linux/init.h> | |
7 | #include <linux/bio.h> | |
8 | #include <linux/blkdev.h> | |
9 | #include <linux/bootmem.h> /* for max_pfn/max_low_pfn */ | |
5a0e3ad6 | 10 | #include <linux/slab.h> |
86db1e29 JA |
11 | |
12 | #include "blk.h" | |
13 | ||
14 | /* | |
15 | * For io context allocations | |
16 | */ | |
17 | static struct kmem_cache *iocontext_cachep; | |
18 | ||
19 | static void cfq_dtor(struct io_context *ioc) | |
20 | { | |
ffc4e759 JA |
21 | if (!hlist_empty(&ioc->cic_list)) { |
22 | struct cfq_io_context *cic; | |
23 | ||
e2bd9678 | 24 | cic = hlist_entry(ioc->cic_list.first, struct cfq_io_context, |
ffc4e759 JA |
25 | cic_list); |
26 | cic->dtor(ioc); | |
27 | } | |
86db1e29 JA |
28 | } |
29 | ||
42ec57a8 TH |
30 | /** |
31 | * put_io_context - put a reference of io_context | |
32 | * @ioc: io_context to put | |
33 | * | |
34 | * Decrement reference count of @ioc and release it if the count reaches | |
35 | * zero. | |
86db1e29 | 36 | */ |
42ec57a8 | 37 | void put_io_context(struct io_context *ioc) |
86db1e29 JA |
38 | { |
39 | if (ioc == NULL) | |
42ec57a8 | 40 | return; |
86db1e29 | 41 | |
42ec57a8 | 42 | BUG_ON(atomic_long_read(&ioc->refcount) <= 0); |
86db1e29 | 43 | |
42ec57a8 TH |
44 | if (!atomic_long_dec_and_test(&ioc->refcount)) |
45 | return; | |
86db1e29 | 46 | |
42ec57a8 TH |
47 | rcu_read_lock(); |
48 | cfq_dtor(ioc); | |
49 | rcu_read_unlock(); | |
50 | ||
51 | kmem_cache_free(iocontext_cachep, ioc); | |
86db1e29 JA |
52 | } |
53 | EXPORT_SYMBOL(put_io_context); | |
54 | ||
55 | static void cfq_exit(struct io_context *ioc) | |
56 | { | |
86db1e29 | 57 | rcu_read_lock(); |
86db1e29 | 58 | |
ffc4e759 JA |
59 | if (!hlist_empty(&ioc->cic_list)) { |
60 | struct cfq_io_context *cic; | |
61 | ||
e2bd9678 | 62 | cic = hlist_entry(ioc->cic_list.first, struct cfq_io_context, |
ffc4e759 JA |
63 | cic_list); |
64 | cic->exit(ioc); | |
65 | } | |
66 | rcu_read_unlock(); | |
86db1e29 JA |
67 | } |
68 | ||
27667c99 | 69 | /* Called by the exiting task */ |
b69f2292 | 70 | void exit_io_context(struct task_struct *task) |
86db1e29 JA |
71 | { |
72 | struct io_context *ioc; | |
73 | ||
b69f2292 LR |
74 | task_lock(task); |
75 | ioc = task->io_context; | |
76 | task->io_context = NULL; | |
77 | task_unlock(task); | |
86db1e29 | 78 | |
27667c99 | 79 | if (atomic_dec_and_test(&ioc->nr_tasks)) |
86db1e29 JA |
80 | cfq_exit(ioc); |
81 | ||
61cc74fb | 82 | put_io_context(ioc); |
86db1e29 JA |
83 | } |
84 | ||
85 | struct io_context *alloc_io_context(gfp_t gfp_flags, int node) | |
86 | { | |
df415656 | 87 | struct io_context *ioc; |
86db1e29 | 88 | |
42ec57a8 TH |
89 | ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO, |
90 | node); | |
91 | if (unlikely(!ioc)) | |
92 | return NULL; | |
93 | ||
94 | /* initialize */ | |
95 | atomic_long_set(&ioc->refcount, 1); | |
96 | atomic_set(&ioc->nr_tasks, 1); | |
97 | spin_lock_init(&ioc->lock); | |
98 | INIT_RADIX_TREE(&ioc->radix_root, GFP_ATOMIC | __GFP_HIGH); | |
99 | INIT_HLIST_HEAD(&ioc->cic_list); | |
86db1e29 | 100 | |
df415656 | 101 | return ioc; |
86db1e29 JA |
102 | } |
103 | ||
42ec57a8 TH |
104 | /** |
105 | * current_io_context - get io_context of %current | |
106 | * @gfp_flags: allocation flags, used if allocation is necessary | |
107 | * @node: allocation node, used if allocation is necessary | |
86db1e29 | 108 | * |
42ec57a8 TH |
109 | * Return io_context of %current. If it doesn't exist, it is created with |
110 | * @gfp_flags and @node. The returned io_context does NOT have its | |
111 | * reference count incremented. Because io_context is exited only on task | |
112 | * exit, %current can be sure that the returned io_context is valid and | |
113 | * alive as long as it is executing. | |
86db1e29 JA |
114 | */ |
115 | struct io_context *current_io_context(gfp_t gfp_flags, int node) | |
116 | { | |
117 | struct task_struct *tsk = current; | |
118 | struct io_context *ret; | |
119 | ||
120 | ret = tsk->io_context; | |
121 | if (likely(ret)) | |
122 | return ret; | |
123 | ||
124 | ret = alloc_io_context(gfp_flags, node); | |
125 | if (ret) { | |
126 | /* make sure set_task_ioprio() sees the settings above */ | |
127 | smp_wmb(); | |
128 | tsk->io_context = ret; | |
129 | } | |
130 | ||
131 | return ret; | |
132 | } | |
133 | ||
134 | /* | |
135 | * If the current task has no IO context then create one and initialise it. | |
136 | * If it does have a context, take a ref on it. | |
137 | * | |
138 | * This is always called in the context of the task which submitted the I/O. | |
139 | */ | |
140 | struct io_context *get_io_context(gfp_t gfp_flags, int node) | |
141 | { | |
df415656 | 142 | struct io_context *ioc = NULL; |
86db1e29 JA |
143 | |
144 | /* | |
145 | * Check for unlikely race with exiting task. ioc ref count is | |
146 | * zero when ioc is being detached. | |
147 | */ | |
148 | do { | |
df415656 PB |
149 | ioc = current_io_context(gfp_flags, node); |
150 | if (unlikely(!ioc)) | |
86db1e29 | 151 | break; |
df415656 | 152 | } while (!atomic_long_inc_not_zero(&ioc->refcount)); |
86db1e29 | 153 | |
df415656 | 154 | return ioc; |
86db1e29 JA |
155 | } |
156 | EXPORT_SYMBOL(get_io_context); | |
157 | ||
13341598 | 158 | static int __init blk_ioc_init(void) |
86db1e29 JA |
159 | { |
160 | iocontext_cachep = kmem_cache_create("blkdev_ioc", | |
161 | sizeof(struct io_context), 0, SLAB_PANIC, NULL); | |
162 | return 0; | |
163 | } | |
164 | subsys_initcall(blk_ioc_init); |