Commit | Line | Data |
---|---|---|
cafe5635 KO |
1 | /* |
2 | * Moving/copying garbage collector | |
3 | * | |
4 | * Copyright 2012 Google, Inc. | |
5 | */ | |
6 | ||
7 | #include "bcache.h" | |
8 | #include "btree.h" | |
9 | #include "debug.h" | |
10 | #include "request.h" | |
11 | ||
c37511b8 KO |
12 | #include <trace/events/bcache.h> |
13 | ||
cafe5635 | 14 | struct moving_io { |
220bb38c | 15 | struct closure cl; |
cafe5635 | 16 | struct keybuf_key *w; |
220bb38c | 17 | struct data_insert_op op; |
cafe5635 KO |
18 | struct bbio bio; |
19 | }; | |
20 | ||
21 | static bool moving_pred(struct keybuf *buf, struct bkey *k) | |
22 | { | |
23 | struct cache_set *c = container_of(buf, struct cache_set, | |
24 | moving_gc_keys); | |
25 | unsigned i; | |
26 | ||
27 | for (i = 0; i < KEY_PTRS(k); i++) { | |
28 | struct cache *ca = PTR_CACHE(c, k, i); | |
29 | struct bucket *g = PTR_BUCKET(c, k, i); | |
30 | ||
31 | if (GC_SECTORS_USED(g) < ca->gc_move_threshold) | |
32 | return true; | |
33 | } | |
34 | ||
35 | return false; | |
36 | } | |
37 | ||
38 | /* Moving GC - IO loop */ | |
39 | ||
40 | static void moving_io_destructor(struct closure *cl) | |
41 | { | |
220bb38c | 42 | struct moving_io *io = container_of(cl, struct moving_io, cl); |
cafe5635 KO |
43 | kfree(io); |
44 | } | |
45 | ||
46 | static void write_moving_finish(struct closure *cl) | |
47 | { | |
220bb38c | 48 | struct moving_io *io = container_of(cl, struct moving_io, cl); |
cafe5635 | 49 | struct bio *bio = &io->bio.bio; |
8e51e414 KO |
50 | struct bio_vec *bv; |
51 | int i; | |
cafe5635 | 52 | |
8e51e414 | 53 | bio_for_each_segment_all(bv, bio, i) |
cafe5635 KO |
54 | __free_page(bv->bv_page); |
55 | ||
220bb38c | 56 | if (io->op.replace_collision) |
c37511b8 | 57 | trace_bcache_gc_copy_collision(&io->w->key); |
cafe5635 | 58 | |
220bb38c | 59 | bch_keybuf_del(&io->op.c->moving_gc_keys, io->w); |
cafe5635 | 60 | |
220bb38c | 61 | up(&io->op.c->moving_in_flight); |
cafe5635 KO |
62 | |
63 | closure_return_with_destructor(cl, moving_io_destructor); | |
64 | } | |
65 | ||
66 | static void read_moving_endio(struct bio *bio, int error) | |
67 | { | |
68 | struct moving_io *io = container_of(bio->bi_private, | |
220bb38c | 69 | struct moving_io, cl); |
cafe5635 KO |
70 | |
71 | if (error) | |
220bb38c | 72 | io->op.error = error; |
cafe5635 | 73 | |
220bb38c | 74 | bch_bbio_endio(io->op.c, bio, error, "reading data to move"); |
cafe5635 KO |
75 | } |
76 | ||
77 | static void moving_init(struct moving_io *io) | |
78 | { | |
79 | struct bio *bio = &io->bio.bio; | |
80 | ||
81 | bio_init(bio); | |
82 | bio_get(bio); | |
83 | bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0)); | |
84 | ||
4f024f37 | 85 | bio->bi_iter.bi_size = KEY_SIZE(&io->w->key) << 9; |
cafe5635 KO |
86 | bio->bi_max_vecs = DIV_ROUND_UP(KEY_SIZE(&io->w->key), |
87 | PAGE_SECTORS); | |
220bb38c | 88 | bio->bi_private = &io->cl; |
cafe5635 | 89 | bio->bi_io_vec = bio->bi_inline_vecs; |
169ef1cf | 90 | bch_bio_map(bio, NULL); |
cafe5635 KO |
91 | } |
92 | ||
93 | static void write_moving(struct closure *cl) | |
94 | { | |
220bb38c KO |
95 | struct moving_io *io = container_of(cl, struct moving_io, cl); |
96 | struct data_insert_op *op = &io->op; | |
cafe5635 | 97 | |
220bb38c | 98 | if (!op->error) { |
cafe5635 KO |
99 | moving_init(io); |
100 | ||
4f024f37 | 101 | io->bio.bio.bi_iter.bi_sector = KEY_START(&io->w->key); |
220bb38c KO |
102 | op->write_prio = 1; |
103 | op->bio = &io->bio.bio; | |
cafe5635 | 104 | |
220bb38c KO |
105 | op->writeback = KEY_DIRTY(&io->w->key); |
106 | op->csum = KEY_CSUM(&io->w->key); | |
cafe5635 | 107 | |
220bb38c KO |
108 | bkey_copy(&op->replace_key, &io->w->key); |
109 | op->replace = true; | |
cafe5635 | 110 | |
220bb38c | 111 | closure_call(&op->cl, bch_data_insert, NULL, cl); |
cafe5635 KO |
112 | } |
113 | ||
72a44517 | 114 | continue_at(cl, write_moving_finish, system_wq); |
cafe5635 KO |
115 | } |
116 | ||
117 | static void read_moving_submit(struct closure *cl) | |
118 | { | |
220bb38c | 119 | struct moving_io *io = container_of(cl, struct moving_io, cl); |
cafe5635 KO |
120 | struct bio *bio = &io->bio.bio; |
121 | ||
220bb38c | 122 | bch_submit_bbio(bio, io->op.c, &io->w->key, 0); |
cafe5635 | 123 | |
72a44517 | 124 | continue_at(cl, write_moving, system_wq); |
cafe5635 KO |
125 | } |
126 | ||
72a44517 | 127 | static void read_moving(struct cache_set *c) |
cafe5635 | 128 | { |
cafe5635 KO |
129 | struct keybuf_key *w; |
130 | struct moving_io *io; | |
131 | struct bio *bio; | |
72a44517 KO |
132 | struct closure cl; |
133 | ||
134 | closure_init_stack(&cl); | |
cafe5635 KO |
135 | |
136 | /* XXX: if we error, background writeback could stall indefinitely */ | |
137 | ||
138 | while (!test_bit(CACHE_SET_STOPPING, &c->flags)) { | |
72c27061 KO |
139 | w = bch_keybuf_next_rescan(c, &c->moving_gc_keys, |
140 | &MAX_KEY, moving_pred); | |
cafe5635 KO |
141 | if (!w) |
142 | break; | |
143 | ||
144 | io = kzalloc(sizeof(struct moving_io) + sizeof(struct bio_vec) | |
145 | * DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS), | |
146 | GFP_KERNEL); | |
147 | if (!io) | |
148 | goto err; | |
149 | ||
150 | w->private = io; | |
151 | io->w = w; | |
220bb38c KO |
152 | io->op.inode = KEY_INODE(&w->key); |
153 | io->op.c = c; | |
cafe5635 KO |
154 | |
155 | moving_init(io); | |
156 | bio = &io->bio.bio; | |
157 | ||
158 | bio->bi_rw = READ; | |
159 | bio->bi_end_io = read_moving_endio; | |
160 | ||
8e51e414 | 161 | if (bio_alloc_pages(bio, GFP_KERNEL)) |
cafe5635 KO |
162 | goto err; |
163 | ||
c37511b8 | 164 | trace_bcache_gc_copy(&w->key); |
cafe5635 | 165 | |
72a44517 | 166 | down(&c->moving_in_flight); |
220bb38c | 167 | closure_call(&io->cl, read_moving_submit, NULL, &cl); |
cafe5635 KO |
168 | } |
169 | ||
170 | if (0) { | |
171 | err: if (!IS_ERR_OR_NULL(w->private)) | |
172 | kfree(w->private); | |
173 | ||
174 | bch_keybuf_del(&c->moving_gc_keys, w); | |
175 | } | |
176 | ||
72a44517 | 177 | closure_sync(&cl); |
cafe5635 KO |
178 | } |
179 | ||
b1a67b0f KO |
180 | static bool bucket_cmp(struct bucket *l, struct bucket *r) |
181 | { | |
182 | return GC_SECTORS_USED(l) < GC_SECTORS_USED(r); | |
183 | } | |
184 | ||
185 | static unsigned bucket_heap_top(struct cache *ca) | |
186 | { | |
187 | return GC_SECTORS_USED(heap_peek(&ca->heap)); | |
188 | } | |
189 | ||
72a44517 | 190 | void bch_moving_gc(struct cache_set *c) |
cafe5635 | 191 | { |
cafe5635 KO |
192 | struct cache *ca; |
193 | struct bucket *b; | |
194 | unsigned i; | |
195 | ||
cafe5635 | 196 | if (!c->copy_gc_enabled) |
72a44517 | 197 | return; |
cafe5635 KO |
198 | |
199 | mutex_lock(&c->bucket_lock); | |
200 | ||
201 | for_each_cache(ca, c, i) { | |
202 | unsigned sectors_to_move = 0; | |
203 | unsigned reserve_sectors = ca->sb.bucket_size * | |
204 | min(fifo_used(&ca->free), ca->free.size / 2); | |
205 | ||
206 | ca->heap.used = 0; | |
207 | ||
208 | for_each_bucket(b, ca) { | |
209 | if (!GC_SECTORS_USED(b)) | |
210 | continue; | |
211 | ||
212 | if (!heap_full(&ca->heap)) { | |
213 | sectors_to_move += GC_SECTORS_USED(b); | |
214 | heap_add(&ca->heap, b, bucket_cmp); | |
215 | } else if (bucket_cmp(b, heap_peek(&ca->heap))) { | |
b1a67b0f | 216 | sectors_to_move -= bucket_heap_top(ca); |
cafe5635 KO |
217 | sectors_to_move += GC_SECTORS_USED(b); |
218 | ||
219 | ca->heap.data[0] = b; | |
220 | heap_sift(&ca->heap, 0, bucket_cmp); | |
221 | } | |
222 | } | |
223 | ||
224 | while (sectors_to_move > reserve_sectors) { | |
225 | heap_pop(&ca->heap, b, bucket_cmp); | |
226 | sectors_to_move -= GC_SECTORS_USED(b); | |
227 | } | |
228 | ||
b1a67b0f | 229 | ca->gc_move_threshold = bucket_heap_top(ca); |
cafe5635 KO |
230 | |
231 | pr_debug("threshold %u", ca->gc_move_threshold); | |
232 | } | |
233 | ||
234 | mutex_unlock(&c->bucket_lock); | |
235 | ||
236 | c->moving_gc_keys.last_scanned = ZERO_KEY; | |
237 | ||
72a44517 | 238 | read_moving(c); |
cafe5635 KO |
239 | } |
240 | ||
241 | void bch_moving_init_cache_set(struct cache_set *c) | |
242 | { | |
72c27061 | 243 | bch_keybuf_init(&c->moving_gc_keys); |
72a44517 | 244 | sema_init(&c->moving_in_flight, 64); |
cafe5635 | 245 | } |