Commit | Line | Data |
---|---|---|
b3b94faa DT |
1 | /* |
2 | * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. | |
3a8a9a10 | 3 | * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. |
b3b94faa DT |
4 | * |
5 | * This copyrighted material is made available to anyone wishing to use, | |
6 | * modify, copy, or redistribute it subject to the terms and conditions | |
e9fc2aa0 | 7 | * of the GNU General Public License version 2. |
b3b94faa DT |
8 | */ |
9 | ||
10 | #include <linux/sched.h> | |
11 | #include <linux/slab.h> | |
12 | #include <linux/spinlock.h> | |
13 | #include <linux/completion.h> | |
14 | #include <linux/buffer_head.h> | |
71b86f56 | 15 | #include <linux/crc32.h> |
5c676f6d | 16 | #include <linux/gfs2_ondisk.h> |
f45b7ddd | 17 | #include <linux/bio.h> |
7d308590 | 18 | #include <linux/lm_interface.h> |
b3b94faa DT |
19 | |
20 | #include "gfs2.h" | |
5c676f6d | 21 | #include "incore.h" |
b3b94faa DT |
22 | #include "bmap.h" |
23 | #include "dir.h" | |
b3b94faa DT |
24 | #include "glock.h" |
25 | #include "glops.h" | |
26 | #include "inode.h" | |
27 | #include "log.h" | |
28 | #include "meta_io.h" | |
29 | #include "quota.h" | |
30 | #include "recovery.h" | |
31 | #include "rgrp.h" | |
32 | #include "super.h" | |
33 | #include "trans.h" | |
5c676f6d | 34 | #include "util.h" |
b3b94faa | 35 | |
cd915493 | 36 | static const u32 gfs2_old_fs_formats[] = { |
75d3b817 SW |
37 | 0 |
38 | }; | |
39 | ||
cd915493 | 40 | static const u32 gfs2_old_multihost_formats[] = { |
75d3b817 SW |
41 | 0 |
42 | }; | |
43 | ||
b3b94faa DT |
44 | /** |
45 | * gfs2_tune_init - Fill a gfs2_tune structure with default values | |
46 | * @gt: tune | |
47 | * | |
48 | */ | |
49 | ||
50 | void gfs2_tune_init(struct gfs2_tune *gt) | |
51 | { | |
52 | spin_lock_init(>->gt_spin); | |
53 | ||
54 | gt->gt_ilimit = 100; | |
55 | gt->gt_ilimit_tries = 3; | |
56 | gt->gt_ilimit_min = 1; | |
57 | gt->gt_demote_secs = 300; | |
58 | gt->gt_incore_log_blocks = 1024; | |
59 | gt->gt_log_flush_secs = 60; | |
60 | gt->gt_jindex_refresh_secs = 60; | |
61 | gt->gt_scand_secs = 15; | |
62 | gt->gt_recoverd_secs = 60; | |
63 | gt->gt_logd_secs = 1; | |
64 | gt->gt_quotad_secs = 5; | |
b3b94faa DT |
65 | gt->gt_quota_simul_sync = 64; |
66 | gt->gt_quota_warn_period = 10; | |
67 | gt->gt_quota_scale_num = 1; | |
68 | gt->gt_quota_scale_den = 1; | |
69 | gt->gt_quota_cache_secs = 300; | |
70 | gt->gt_quota_quantum = 60; | |
71 | gt->gt_atime_quantum = 3600; | |
72 | gt->gt_new_files_jdata = 0; | |
73 | gt->gt_new_files_directio = 0; | |
b3b94faa DT |
74 | gt->gt_max_readahead = 1 << 18; |
75 | gt->gt_lockdump_size = 131072; | |
76 | gt->gt_stall_secs = 600; | |
77 | gt->gt_complain_secs = 10; | |
78 | gt->gt_reclaim_limit = 5000; | |
b3b94faa DT |
79 | gt->gt_statfs_quantum = 30; |
80 | gt->gt_statfs_slow = 0; | |
81 | } | |
82 | ||
83 | /** | |
84 | * gfs2_check_sb - Check superblock | |
85 | * @sdp: the filesystem | |
86 | * @sb: The superblock | |
87 | * @silent: Don't print a message if the check fails | |
88 | * | |
89 | * Checks the version code of the FS is one that we understand how to | |
90 | * read and that the sizes of the various on-disk structures have not | |
91 | * changed. | |
92 | */ | |
93 | ||
f50dfaf7 | 94 | int gfs2_check_sb(struct gfs2_sbd *sdp, struct gfs2_sb_host *sb, int silent) |
b3b94faa DT |
95 | { |
96 | unsigned int x; | |
97 | ||
bb8d8a6f SW |
98 | if (sb->sb_magic != GFS2_MAGIC || |
99 | sb->sb_type != GFS2_METATYPE_SB) { | |
b3b94faa | 100 | if (!silent) |
d92a8d48 | 101 | printk(KERN_WARNING "GFS2: not a GFS2 filesystem\n"); |
b3b94faa DT |
102 | return -EINVAL; |
103 | } | |
104 | ||
105 | /* If format numbers match exactly, we're done. */ | |
106 | ||
107 | if (sb->sb_fs_format == GFS2_FORMAT_FS && | |
108 | sb->sb_multihost_format == GFS2_FORMAT_MULTI) | |
109 | return 0; | |
110 | ||
111 | if (sb->sb_fs_format != GFS2_FORMAT_FS) { | |
112 | for (x = 0; gfs2_old_fs_formats[x]; x++) | |
113 | if (gfs2_old_fs_formats[x] == sb->sb_fs_format) | |
114 | break; | |
115 | ||
116 | if (!gfs2_old_fs_formats[x]) { | |
568f4c96 SW |
117 | printk(KERN_WARNING |
118 | "GFS2: code version (%u, %u) is incompatible " | |
b3b94faa DT |
119 | "with ondisk format (%u, %u)\n", |
120 | GFS2_FORMAT_FS, GFS2_FORMAT_MULTI, | |
121 | sb->sb_fs_format, sb->sb_multihost_format); | |
568f4c96 SW |
122 | printk(KERN_WARNING |
123 | "GFS2: I don't know how to upgrade this FS\n"); | |
b3b94faa DT |
124 | return -EINVAL; |
125 | } | |
126 | } | |
127 | ||
128 | if (sb->sb_multihost_format != GFS2_FORMAT_MULTI) { | |
129 | for (x = 0; gfs2_old_multihost_formats[x]; x++) | |
568f4c96 SW |
130 | if (gfs2_old_multihost_formats[x] == |
131 | sb->sb_multihost_format) | |
b3b94faa DT |
132 | break; |
133 | ||
134 | if (!gfs2_old_multihost_formats[x]) { | |
568f4c96 SW |
135 | printk(KERN_WARNING |
136 | "GFS2: code version (%u, %u) is incompatible " | |
b3b94faa DT |
137 | "with ondisk format (%u, %u)\n", |
138 | GFS2_FORMAT_FS, GFS2_FORMAT_MULTI, | |
139 | sb->sb_fs_format, sb->sb_multihost_format); | |
568f4c96 SW |
140 | printk(KERN_WARNING |
141 | "GFS2: I don't know how to upgrade this FS\n"); | |
b3b94faa DT |
142 | return -EINVAL; |
143 | } | |
144 | } | |
145 | ||
146 | if (!sdp->sd_args.ar_upgrade) { | |
568f4c96 SW |
147 | printk(KERN_WARNING |
148 | "GFS2: code version (%u, %u) is incompatible " | |
b3b94faa DT |
149 | "with ondisk format (%u, %u)\n", |
150 | GFS2_FORMAT_FS, GFS2_FORMAT_MULTI, | |
151 | sb->sb_fs_format, sb->sb_multihost_format); | |
568f4c96 SW |
152 | printk(KERN_INFO |
153 | "GFS2: Use the \"upgrade\" mount option to upgrade " | |
b3b94faa | 154 | "the FS\n"); |
d92a8d48 | 155 | printk(KERN_INFO "GFS2: See the manual for more details\n"); |
b3b94faa DT |
156 | return -EINVAL; |
157 | } | |
158 | ||
159 | return 0; | |
160 | } | |
161 | ||
f45b7ddd | 162 | |
6712ecf8 | 163 | static void end_bio_io_page(struct bio *bio, int error) |
f45b7ddd SW |
164 | { |
165 | struct page *page = bio->bi_private; | |
f45b7ddd SW |
166 | |
167 | if (!error) | |
168 | SetPageUptodate(page); | |
2b98a54f SW |
169 | else |
170 | printk(KERN_WARNING "gfs2: error %d reading superblock\n", error); | |
f45b7ddd SW |
171 | unlock_page(page); |
172 | return 0; | |
173 | } | |
174 | ||
bb8d8a6f SW |
175 | static void gfs2_sb_in(struct gfs2_sb_host *sb, const void *buf) |
176 | { | |
177 | const struct gfs2_sb *str = buf; | |
178 | ||
179 | sb->sb_magic = be32_to_cpu(str->sb_header.mh_magic); | |
180 | sb->sb_type = be32_to_cpu(str->sb_header.mh_type); | |
181 | sb->sb_format = be32_to_cpu(str->sb_header.mh_format); | |
182 | sb->sb_fs_format = be32_to_cpu(str->sb_fs_format); | |
183 | sb->sb_multihost_format = be32_to_cpu(str->sb_multihost_format); | |
184 | sb->sb_bsize = be32_to_cpu(str->sb_bsize); | |
185 | sb->sb_bsize_shift = be32_to_cpu(str->sb_bsize_shift); | |
186 | sb->sb_master_dir.no_addr = be64_to_cpu(str->sb_master_dir.no_addr); | |
187 | sb->sb_master_dir.no_formal_ino = be64_to_cpu(str->sb_master_dir.no_formal_ino); | |
188 | sb->sb_root_dir.no_addr = be64_to_cpu(str->sb_root_dir.no_addr); | |
189 | sb->sb_root_dir.no_formal_ino = be64_to_cpu(str->sb_root_dir.no_formal_ino); | |
190 | ||
191 | memcpy(sb->sb_lockproto, str->sb_lockproto, GFS2_LOCKNAME_LEN); | |
192 | memcpy(sb->sb_locktable, str->sb_locktable, GFS2_LOCKNAME_LEN); | |
193 | } | |
194 | ||
aac1a3c7 SW |
195 | /** |
196 | * gfs2_read_super - Read the gfs2 super block from disk | |
bb8d8a6f | 197 | * @sdp: The GFS2 super block |
aac1a3c7 | 198 | * @sector: The location of the super block |
bb8d8a6f | 199 | * @error: The error code to return |
aac1a3c7 SW |
200 | * |
201 | * This uses the bio functions to read the super block from disk | |
202 | * because we want to be 100% sure that we never read cached data. | |
203 | * A super block is read twice only during each GFS2 mount and is | |
204 | * never written to by the filesystem. The first time its read no | |
205 | * locks are held, and the only details which are looked at are those | |
206 | * relating to the locking protocol. Once locking is up and working, | |
207 | * the sb is read again under the lock to establish the location of | |
208 | * the master directory (contains pointers to journals etc) and the | |
209 | * root directory. | |
210 | * | |
bb8d8a6f | 211 | * Returns: 0 on success or error |
aac1a3c7 SW |
212 | */ |
213 | ||
bb8d8a6f | 214 | int gfs2_read_super(struct gfs2_sbd *sdp, sector_t sector) |
f45b7ddd | 215 | { |
bb8d8a6f SW |
216 | struct super_block *sb = sdp->sd_vfs; |
217 | struct gfs2_sb *p; | |
f45b7ddd SW |
218 | struct page *page; |
219 | struct bio *bio; | |
220 | ||
221 | page = alloc_page(GFP_KERNEL); | |
222 | if (unlikely(!page)) | |
bb8d8a6f | 223 | return -ENOBUFS; |
f45b7ddd SW |
224 | |
225 | ClearPageUptodate(page); | |
226 | ClearPageDirty(page); | |
227 | lock_page(page); | |
228 | ||
229 | bio = bio_alloc(GFP_KERNEL, 1); | |
230 | if (unlikely(!bio)) { | |
231 | __free_page(page); | |
bb8d8a6f | 232 | return -ENOBUFS; |
f45b7ddd SW |
233 | } |
234 | ||
0da3585e | 235 | bio->bi_sector = sector * (sb->s_blocksize >> 9); |
f45b7ddd SW |
236 | bio->bi_bdev = sb->s_bdev; |
237 | bio_add_page(bio, page, PAGE_SIZE, 0); | |
238 | ||
239 | bio->bi_end_io = end_bio_io_page; | |
240 | bio->bi_private = page; | |
3cf1e7be | 241 | submit_bio(READ_SYNC | (1 << BIO_RW_META), bio); |
f45b7ddd SW |
242 | wait_on_page_locked(page); |
243 | bio_put(bio); | |
244 | if (!PageUptodate(page)) { | |
245 | __free_page(page); | |
bb8d8a6f | 246 | return -EIO; |
f45b7ddd | 247 | } |
bb8d8a6f SW |
248 | p = kmap(page); |
249 | gfs2_sb_in(&sdp->sd_sb, p); | |
250 | kunmap(page); | |
251 | __free_page(page); | |
252 | return 0; | |
f45b7ddd SW |
253 | } |
254 | ||
b3b94faa DT |
255 | /** |
256 | * gfs2_read_sb - Read super block | |
257 | * @sdp: The GFS2 superblock | |
258 | * @gl: the glock for the superblock (assumed to be held) | |
259 | * @silent: Don't print message if mount fails | |
260 | * | |
261 | */ | |
262 | ||
263 | int gfs2_read_sb(struct gfs2_sbd *sdp, struct gfs2_glock *gl, int silent) | |
264 | { | |
cd915493 SW |
265 | u32 hash_blocks, ind_blocks, leaf_blocks; |
266 | u32 tmp_blocks; | |
b3b94faa DT |
267 | unsigned int x; |
268 | int error; | |
269 | ||
bb8d8a6f SW |
270 | error = gfs2_read_super(sdp, GFS2_SB_ADDR >> sdp->sd_fsb2bb_shift); |
271 | if (error) { | |
b3b94faa DT |
272 | if (!silent) |
273 | fs_err(sdp, "can't read superblock\n"); | |
bb8d8a6f | 274 | return error; |
b3b94faa | 275 | } |
b3b94faa DT |
276 | |
277 | error = gfs2_check_sb(sdp, &sdp->sd_sb, silent); | |
278 | if (error) | |
279 | return error; | |
280 | ||
281 | sdp->sd_fsb2bb_shift = sdp->sd_sb.sb_bsize_shift - | |
282 | GFS2_BASIC_BLOCK_SHIFT; | |
283 | sdp->sd_fsb2bb = 1 << sdp->sd_fsb2bb_shift; | |
284 | sdp->sd_diptrs = (sdp->sd_sb.sb_bsize - | |
cd915493 | 285 | sizeof(struct gfs2_dinode)) / sizeof(u64); |
b3b94faa | 286 | sdp->sd_inptrs = (sdp->sd_sb.sb_bsize - |
cd915493 | 287 | sizeof(struct gfs2_meta_header)) / sizeof(u64); |
b3b94faa DT |
288 | sdp->sd_jbsize = sdp->sd_sb.sb_bsize - sizeof(struct gfs2_meta_header); |
289 | sdp->sd_hash_bsize = sdp->sd_sb.sb_bsize / 2; | |
290 | sdp->sd_hash_bsize_shift = sdp->sd_sb.sb_bsize_shift - 1; | |
cd915493 | 291 | sdp->sd_hash_ptrs = sdp->sd_hash_bsize / sizeof(u64); |
b3b94faa DT |
292 | sdp->sd_qc_per_block = (sdp->sd_sb.sb_bsize - |
293 | sizeof(struct gfs2_meta_header)) / | |
f45b7ddd | 294 | sizeof(struct gfs2_quota_change); |
b3b94faa DT |
295 | |
296 | /* Compute maximum reservation required to add a entry to a directory */ | |
297 | ||
cd915493 | 298 | hash_blocks = DIV_ROUND_UP(sizeof(u64) * (1 << GFS2_DIR_MAX_DEPTH), |
b3b94faa DT |
299 | sdp->sd_jbsize); |
300 | ||
301 | ind_blocks = 0; | |
302 | for (tmp_blocks = hash_blocks; tmp_blocks > sdp->sd_diptrs;) { | |
5c676f6d | 303 | tmp_blocks = DIV_ROUND_UP(tmp_blocks, sdp->sd_inptrs); |
b3b94faa DT |
304 | ind_blocks += tmp_blocks; |
305 | } | |
306 | ||
307 | leaf_blocks = 2 + GFS2_DIR_MAX_DEPTH; | |
308 | ||
309 | sdp->sd_max_dirres = hash_blocks + ind_blocks + leaf_blocks; | |
310 | ||
311 | sdp->sd_heightsize[0] = sdp->sd_sb.sb_bsize - | |
312 | sizeof(struct gfs2_dinode); | |
313 | sdp->sd_heightsize[1] = sdp->sd_sb.sb_bsize * sdp->sd_diptrs; | |
314 | for (x = 2;; x++) { | |
cd915493 SW |
315 | u64 space, d; |
316 | u32 m; | |
b3b94faa DT |
317 | |
318 | space = sdp->sd_heightsize[x - 1] * sdp->sd_inptrs; | |
319 | d = space; | |
320 | m = do_div(d, sdp->sd_inptrs); | |
321 | ||
322 | if (d != sdp->sd_heightsize[x - 1] || m) | |
323 | break; | |
324 | sdp->sd_heightsize[x] = space; | |
325 | } | |
326 | sdp->sd_max_height = x; | |
327 | gfs2_assert(sdp, sdp->sd_max_height <= GFS2_MAX_META_HEIGHT); | |
328 | ||
329 | sdp->sd_jheightsize[0] = sdp->sd_sb.sb_bsize - | |
330 | sizeof(struct gfs2_dinode); | |
331 | sdp->sd_jheightsize[1] = sdp->sd_jbsize * sdp->sd_diptrs; | |
332 | for (x = 2;; x++) { | |
cd915493 SW |
333 | u64 space, d; |
334 | u32 m; | |
b3b94faa DT |
335 | |
336 | space = sdp->sd_jheightsize[x - 1] * sdp->sd_inptrs; | |
337 | d = space; | |
338 | m = do_div(d, sdp->sd_inptrs); | |
339 | ||
340 | if (d != sdp->sd_jheightsize[x - 1] || m) | |
341 | break; | |
342 | sdp->sd_jheightsize[x] = space; | |
343 | } | |
344 | sdp->sd_max_jheight = x; | |
345 | gfs2_assert(sdp, sdp->sd_max_jheight <= GFS2_MAX_META_HEIGHT); | |
346 | ||
347 | return 0; | |
348 | } | |
349 | ||
b3b94faa DT |
350 | /** |
351 | * gfs2_jindex_hold - Grab a lock on the jindex | |
352 | * @sdp: The GFS2 superblock | |
353 | * @ji_gh: the holder for the jindex glock | |
354 | * | |
355 | * This is very similar to the gfs2_rindex_hold() function, except that | |
356 | * in general we hold the jindex lock for longer periods of time and | |
357 | * we grab it far less frequently (in general) then the rgrp lock. | |
358 | * | |
359 | * Returns: errno | |
360 | */ | |
361 | ||
362 | int gfs2_jindex_hold(struct gfs2_sbd *sdp, struct gfs2_holder *ji_gh) | |
363 | { | |
feaa7bba | 364 | struct gfs2_inode *dip = GFS2_I(sdp->sd_jindex); |
b3b94faa DT |
365 | struct qstr name; |
366 | char buf[20]; | |
367 | struct gfs2_jdesc *jd; | |
368 | int error; | |
369 | ||
370 | name.name = buf; | |
371 | ||
f55ab26a | 372 | mutex_lock(&sdp->sd_jindex_mutex); |
b3b94faa DT |
373 | |
374 | for (;;) { | |
1c0f4872 | 375 | error = gfs2_glock_nq_init(dip->i_gl, LM_ST_SHARED, 0, ji_gh); |
b3b94faa DT |
376 | if (error) |
377 | break; | |
378 | ||
379 | name.len = sprintf(buf, "journal%u", sdp->sd_journals); | |
c752666c | 380 | name.hash = gfs2_disk_hash(name.name, name.len); |
b3b94faa | 381 | |
dbb7cae2 | 382 | error = gfs2_dir_check(sdp->sd_jindex, &name, NULL); |
b3b94faa DT |
383 | if (error == -ENOENT) { |
384 | error = 0; | |
385 | break; | |
386 | } | |
387 | ||
388 | gfs2_glock_dq_uninit(ji_gh); | |
389 | ||
390 | if (error) | |
391 | break; | |
392 | ||
393 | error = -ENOMEM; | |
394 | jd = kzalloc(sizeof(struct gfs2_jdesc), GFP_KERNEL); | |
395 | if (!jd) | |
396 | break; | |
397 | ||
c752666c SW |
398 | jd->jd_inode = gfs2_lookupi(sdp->sd_jindex, &name, 1, NULL); |
399 | if (!jd->jd_inode || IS_ERR(jd->jd_inode)) { | |
400 | if (!jd->jd_inode) | |
401 | error = -ENOENT; | |
402 | else | |
403 | error = PTR_ERR(jd->jd_inode); | |
b3b94faa DT |
404 | kfree(jd); |
405 | break; | |
406 | } | |
407 | ||
408 | spin_lock(&sdp->sd_jindex_spin); | |
409 | jd->jd_jid = sdp->sd_journals++; | |
410 | list_add_tail(&jd->jd_list, &sdp->sd_jindex_list); | |
411 | spin_unlock(&sdp->sd_jindex_spin); | |
412 | } | |
413 | ||
f55ab26a | 414 | mutex_unlock(&sdp->sd_jindex_mutex); |
b3b94faa DT |
415 | |
416 | return error; | |
417 | } | |
418 | ||
419 | /** | |
420 | * gfs2_jindex_free - Clear all the journal index information | |
421 | * @sdp: The GFS2 superblock | |
422 | * | |
423 | */ | |
424 | ||
425 | void gfs2_jindex_free(struct gfs2_sbd *sdp) | |
426 | { | |
427 | struct list_head list; | |
428 | struct gfs2_jdesc *jd; | |
429 | ||
430 | spin_lock(&sdp->sd_jindex_spin); | |
431 | list_add(&list, &sdp->sd_jindex_list); | |
432 | list_del_init(&sdp->sd_jindex_list); | |
433 | sdp->sd_journals = 0; | |
434 | spin_unlock(&sdp->sd_jindex_spin); | |
435 | ||
436 | while (!list_empty(&list)) { | |
437 | jd = list_entry(list.next, struct gfs2_jdesc, jd_list); | |
438 | list_del(&jd->jd_list); | |
7359a19c | 439 | iput(jd->jd_inode); |
b3b94faa DT |
440 | kfree(jd); |
441 | } | |
442 | } | |
443 | ||
444 | static struct gfs2_jdesc *jdesc_find_i(struct list_head *head, unsigned int jid) | |
445 | { | |
446 | struct gfs2_jdesc *jd; | |
447 | int found = 0; | |
448 | ||
449 | list_for_each_entry(jd, head, jd_list) { | |
450 | if (jd->jd_jid == jid) { | |
451 | found = 1; | |
452 | break; | |
453 | } | |
454 | } | |
455 | ||
456 | if (!found) | |
457 | jd = NULL; | |
458 | ||
459 | return jd; | |
460 | } | |
461 | ||
462 | struct gfs2_jdesc *gfs2_jdesc_find(struct gfs2_sbd *sdp, unsigned int jid) | |
463 | { | |
464 | struct gfs2_jdesc *jd; | |
465 | ||
466 | spin_lock(&sdp->sd_jindex_spin); | |
467 | jd = jdesc_find_i(&sdp->sd_jindex_list, jid); | |
468 | spin_unlock(&sdp->sd_jindex_spin); | |
469 | ||
470 | return jd; | |
471 | } | |
472 | ||
473 | void gfs2_jdesc_make_dirty(struct gfs2_sbd *sdp, unsigned int jid) | |
474 | { | |
475 | struct gfs2_jdesc *jd; | |
476 | ||
477 | spin_lock(&sdp->sd_jindex_spin); | |
478 | jd = jdesc_find_i(&sdp->sd_jindex_list, jid); | |
479 | if (jd) | |
480 | jd->jd_dirty = 1; | |
481 | spin_unlock(&sdp->sd_jindex_spin); | |
482 | } | |
483 | ||
484 | struct gfs2_jdesc *gfs2_jdesc_find_dirty(struct gfs2_sbd *sdp) | |
485 | { | |
486 | struct gfs2_jdesc *jd; | |
487 | int found = 0; | |
488 | ||
489 | spin_lock(&sdp->sd_jindex_spin); | |
490 | ||
491 | list_for_each_entry(jd, &sdp->sd_jindex_list, jd_list) { | |
492 | if (jd->jd_dirty) { | |
493 | jd->jd_dirty = 0; | |
494 | found = 1; | |
495 | break; | |
496 | } | |
497 | } | |
498 | spin_unlock(&sdp->sd_jindex_spin); | |
499 | ||
500 | if (!found) | |
501 | jd = NULL; | |
502 | ||
503 | return jd; | |
504 | } | |
505 | ||
506 | int gfs2_jdesc_check(struct gfs2_jdesc *jd) | |
507 | { | |
feaa7bba SW |
508 | struct gfs2_inode *ip = GFS2_I(jd->jd_inode); |
509 | struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); | |
b3b94faa DT |
510 | int ar; |
511 | int error; | |
512 | ||
feaa7bba | 513 | if (ip->i_di.di_size < (8 << 20) || ip->i_di.di_size > (1 << 30) || |
b3b94faa DT |
514 | (ip->i_di.di_size & (sdp->sd_sb.sb_bsize - 1))) { |
515 | gfs2_consist_inode(ip); | |
516 | return -EIO; | |
517 | } | |
518 | jd->jd_blocks = ip->i_di.di_size >> sdp->sd_sb.sb_bsize_shift; | |
519 | ||
feaa7bba | 520 | error = gfs2_write_alloc_required(ip, 0, ip->i_di.di_size, &ar); |
b3b94faa DT |
521 | if (!error && ar) { |
522 | gfs2_consist_inode(ip); | |
523 | error = -EIO; | |
524 | } | |
525 | ||
526 | return error; | |
527 | } | |
528 | ||
b3b94faa DT |
529 | /** |
530 | * gfs2_make_fs_rw - Turn a Read-Only FS into a Read-Write one | |
531 | * @sdp: the filesystem | |
532 | * | |
533 | * Returns: errno | |
534 | */ | |
535 | ||
536 | int gfs2_make_fs_rw(struct gfs2_sbd *sdp) | |
537 | { | |
feaa7bba | 538 | struct gfs2_inode *ip = GFS2_I(sdp->sd_jdesc->jd_inode); |
5c676f6d | 539 | struct gfs2_glock *j_gl = ip->i_gl; |
b3b94faa | 540 | struct gfs2_holder t_gh; |
55167622 | 541 | struct gfs2_log_header_host head; |
b3b94faa DT |
542 | int error; |
543 | ||
1c0f4872 | 544 | error = gfs2_glock_nq_init(sdp->sd_trans_gl, LM_ST_SHARED, 0, &t_gh); |
b3b94faa DT |
545 | if (error) |
546 | return error; | |
547 | ||
5c676f6d | 548 | gfs2_meta_cache_flush(ip); |
1a14d3a6 | 549 | j_gl->gl_ops->go_inval(j_gl, DIO_METADATA); |
b3b94faa DT |
550 | |
551 | error = gfs2_find_jhead(sdp->sd_jdesc, &head); | |
552 | if (error) | |
553 | goto fail; | |
554 | ||
555 | if (!(head.lh_flags & GFS2_LOG_HEAD_UNMOUNT)) { | |
556 | gfs2_consist(sdp); | |
557 | error = -EIO; | |
558 | goto fail; | |
559 | } | |
560 | ||
561 | /* Initialize some head of the log stuff */ | |
562 | sdp->sd_log_sequence = head.lh_sequence + 1; | |
563 | gfs2_log_pointers_init(sdp, head.lh_blkno); | |
564 | ||
b3b94faa DT |
565 | error = gfs2_quota_init(sdp); |
566 | if (error) | |
a91ea69f | 567 | goto fail; |
b3b94faa DT |
568 | |
569 | set_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); | |
570 | ||
571 | gfs2_glock_dq_uninit(&t_gh); | |
572 | ||
573 | return 0; | |
574 | ||
a91ea69f | 575 | fail: |
b3b94faa DT |
576 | t_gh.gh_flags |= GL_NOCACHE; |
577 | gfs2_glock_dq_uninit(&t_gh); | |
578 | ||
579 | return error; | |
580 | } | |
581 | ||
582 | /** | |
583 | * gfs2_make_fs_ro - Turn a Read-Write FS into a Read-Only one | |
584 | * @sdp: the filesystem | |
585 | * | |
586 | * Returns: errno | |
587 | */ | |
588 | ||
589 | int gfs2_make_fs_ro(struct gfs2_sbd *sdp) | |
590 | { | |
591 | struct gfs2_holder t_gh; | |
592 | int error; | |
593 | ||
b3b94faa DT |
594 | gfs2_quota_sync(sdp); |
595 | gfs2_statfs_sync(sdp); | |
596 | ||
1c0f4872 SW |
597 | error = gfs2_glock_nq_init(sdp->sd_trans_gl, LM_ST_SHARED, GL_NOCACHE, |
598 | &t_gh); | |
b3b94faa DT |
599 | if (error && !test_bit(SDF_SHUTDOWN, &sdp->sd_flags)) |
600 | return error; | |
601 | ||
602 | gfs2_meta_syncfs(sdp); | |
603 | gfs2_log_shutdown(sdp); | |
604 | ||
605 | clear_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); | |
606 | ||
607 | if (t_gh.gh_gl) | |
608 | gfs2_glock_dq_uninit(&t_gh); | |
609 | ||
b3b94faa DT |
610 | gfs2_quota_cleanup(sdp); |
611 | ||
612 | return error; | |
613 | } | |
614 | ||
bb8d8a6f SW |
615 | static void gfs2_statfs_change_in(struct gfs2_statfs_change_host *sc, const void *buf) |
616 | { | |
617 | const struct gfs2_statfs_change *str = buf; | |
618 | ||
619 | sc->sc_total = be64_to_cpu(str->sc_total); | |
620 | sc->sc_free = be64_to_cpu(str->sc_free); | |
621 | sc->sc_dinodes = be64_to_cpu(str->sc_dinodes); | |
622 | } | |
623 | ||
624 | static void gfs2_statfs_change_out(const struct gfs2_statfs_change_host *sc, void *buf) | |
625 | { | |
626 | struct gfs2_statfs_change *str = buf; | |
627 | ||
628 | str->sc_total = cpu_to_be64(sc->sc_total); | |
629 | str->sc_free = cpu_to_be64(sc->sc_free); | |
630 | str->sc_dinodes = cpu_to_be64(sc->sc_dinodes); | |
631 | } | |
632 | ||
b3b94faa DT |
633 | int gfs2_statfs_init(struct gfs2_sbd *sdp) |
634 | { | |
feaa7bba | 635 | struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode); |
bd209cc0 | 636 | struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master; |
feaa7bba | 637 | struct gfs2_inode *l_ip = GFS2_I(sdp->sd_sc_inode); |
bd209cc0 | 638 | struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local; |
b3b94faa DT |
639 | struct buffer_head *m_bh, *l_bh; |
640 | struct gfs2_holder gh; | |
641 | int error; | |
642 | ||
643 | error = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE, GL_NOCACHE, | |
644 | &gh); | |
645 | if (error) | |
646 | return error; | |
647 | ||
648 | error = gfs2_meta_inode_buffer(m_ip, &m_bh); | |
649 | if (error) | |
650 | goto out; | |
651 | ||
652 | if (sdp->sd_args.ar_spectator) { | |
653 | spin_lock(&sdp->sd_statfs_spin); | |
654 | gfs2_statfs_change_in(m_sc, m_bh->b_data + | |
655 | sizeof(struct gfs2_dinode)); | |
656 | spin_unlock(&sdp->sd_statfs_spin); | |
657 | } else { | |
658 | error = gfs2_meta_inode_buffer(l_ip, &l_bh); | |
659 | if (error) | |
660 | goto out_m_bh; | |
661 | ||
662 | spin_lock(&sdp->sd_statfs_spin); | |
663 | gfs2_statfs_change_in(m_sc, m_bh->b_data + | |
664 | sizeof(struct gfs2_dinode)); | |
665 | gfs2_statfs_change_in(l_sc, l_bh->b_data + | |
666 | sizeof(struct gfs2_dinode)); | |
667 | spin_unlock(&sdp->sd_statfs_spin); | |
668 | ||
669 | brelse(l_bh); | |
670 | } | |
671 | ||
a91ea69f | 672 | out_m_bh: |
b3b94faa | 673 | brelse(m_bh); |
a91ea69f | 674 | out: |
b3b94faa | 675 | gfs2_glock_dq_uninit(&gh); |
b3b94faa DT |
676 | return 0; |
677 | } | |
678 | ||
cd915493 SW |
679 | void gfs2_statfs_change(struct gfs2_sbd *sdp, s64 total, s64 free, |
680 | s64 dinodes) | |
b3b94faa | 681 | { |
feaa7bba | 682 | struct gfs2_inode *l_ip = GFS2_I(sdp->sd_sc_inode); |
bd209cc0 | 683 | struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local; |
b3b94faa DT |
684 | struct buffer_head *l_bh; |
685 | int error; | |
686 | ||
687 | error = gfs2_meta_inode_buffer(l_ip, &l_bh); | |
688 | if (error) | |
689 | return; | |
690 | ||
f55ab26a | 691 | mutex_lock(&sdp->sd_statfs_mutex); |
d4e9c4c3 | 692 | gfs2_trans_add_bh(l_ip->i_gl, l_bh, 1); |
f55ab26a | 693 | mutex_unlock(&sdp->sd_statfs_mutex); |
b3b94faa DT |
694 | |
695 | spin_lock(&sdp->sd_statfs_spin); | |
696 | l_sc->sc_total += total; | |
697 | l_sc->sc_free += free; | |
698 | l_sc->sc_dinodes += dinodes; | |
907b9bce | 699 | gfs2_statfs_change_out(l_sc, l_bh->b_data + sizeof(struct gfs2_dinode)); |
b3b94faa DT |
700 | spin_unlock(&sdp->sd_statfs_spin); |
701 | ||
702 | brelse(l_bh); | |
703 | } | |
704 | ||
705 | int gfs2_statfs_sync(struct gfs2_sbd *sdp) | |
706 | { | |
feaa7bba SW |
707 | struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode); |
708 | struct gfs2_inode *l_ip = GFS2_I(sdp->sd_sc_inode); | |
bd209cc0 AV |
709 | struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master; |
710 | struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local; | |
b3b94faa DT |
711 | struct gfs2_holder gh; |
712 | struct buffer_head *m_bh, *l_bh; | |
713 | int error; | |
714 | ||
715 | error = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE, GL_NOCACHE, | |
716 | &gh); | |
717 | if (error) | |
718 | return error; | |
719 | ||
720 | error = gfs2_meta_inode_buffer(m_ip, &m_bh); | |
721 | if (error) | |
722 | goto out; | |
723 | ||
724 | spin_lock(&sdp->sd_statfs_spin); | |
725 | gfs2_statfs_change_in(m_sc, m_bh->b_data + | |
907b9bce | 726 | sizeof(struct gfs2_dinode)); |
b3b94faa DT |
727 | if (!l_sc->sc_total && !l_sc->sc_free && !l_sc->sc_dinodes) { |
728 | spin_unlock(&sdp->sd_statfs_spin); | |
729 | goto out_bh; | |
730 | } | |
731 | spin_unlock(&sdp->sd_statfs_spin); | |
732 | ||
733 | error = gfs2_meta_inode_buffer(l_ip, &l_bh); | |
734 | if (error) | |
735 | goto out_bh; | |
736 | ||
737 | error = gfs2_trans_begin(sdp, 2 * RES_DINODE, 0); | |
738 | if (error) | |
739 | goto out_bh2; | |
740 | ||
f55ab26a | 741 | mutex_lock(&sdp->sd_statfs_mutex); |
d4e9c4c3 | 742 | gfs2_trans_add_bh(l_ip->i_gl, l_bh, 1); |
f55ab26a | 743 | mutex_unlock(&sdp->sd_statfs_mutex); |
b3b94faa DT |
744 | |
745 | spin_lock(&sdp->sd_statfs_spin); | |
746 | m_sc->sc_total += l_sc->sc_total; | |
747 | m_sc->sc_free += l_sc->sc_free; | |
748 | m_sc->sc_dinodes += l_sc->sc_dinodes; | |
749 | memset(l_sc, 0, sizeof(struct gfs2_statfs_change)); | |
750 | memset(l_bh->b_data + sizeof(struct gfs2_dinode), | |
751 | 0, sizeof(struct gfs2_statfs_change)); | |
752 | spin_unlock(&sdp->sd_statfs_spin); | |
753 | ||
d4e9c4c3 | 754 | gfs2_trans_add_bh(m_ip->i_gl, m_bh, 1); |
b3b94faa DT |
755 | gfs2_statfs_change_out(m_sc, m_bh->b_data + sizeof(struct gfs2_dinode)); |
756 | ||
757 | gfs2_trans_end(sdp); | |
758 | ||
a91ea69f | 759 | out_bh2: |
b3b94faa | 760 | brelse(l_bh); |
a91ea69f | 761 | out_bh: |
b3b94faa | 762 | brelse(m_bh); |
a91ea69f | 763 | out: |
b3b94faa | 764 | gfs2_glock_dq_uninit(&gh); |
b3b94faa DT |
765 | return error; |
766 | } | |
767 | ||
768 | /** | |
769 | * gfs2_statfs_i - Do a statfs | |
770 | * @sdp: the filesystem | |
771 | * @sg: the sg structure | |
772 | * | |
773 | * Returns: errno | |
774 | */ | |
775 | ||
bd209cc0 | 776 | int gfs2_statfs_i(struct gfs2_sbd *sdp, struct gfs2_statfs_change_host *sc) |
b3b94faa | 777 | { |
bd209cc0 AV |
778 | struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master; |
779 | struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local; | |
b3b94faa DT |
780 | |
781 | spin_lock(&sdp->sd_statfs_spin); | |
782 | ||
783 | *sc = *m_sc; | |
784 | sc->sc_total += l_sc->sc_total; | |
785 | sc->sc_free += l_sc->sc_free; | |
786 | sc->sc_dinodes += l_sc->sc_dinodes; | |
787 | ||
788 | spin_unlock(&sdp->sd_statfs_spin); | |
789 | ||
790 | if (sc->sc_free < 0) | |
791 | sc->sc_free = 0; | |
792 | if (sc->sc_free > sc->sc_total) | |
793 | sc->sc_free = sc->sc_total; | |
794 | if (sc->sc_dinodes < 0) | |
795 | sc->sc_dinodes = 0; | |
796 | ||
797 | return 0; | |
798 | } | |
799 | ||
800 | /** | |
801 | * statfs_fill - fill in the sg for a given RG | |
802 | * @rgd: the RG | |
803 | * @sc: the sc structure | |
804 | * | |
805 | * Returns: 0 on success, -ESTALE if the LVB is invalid | |
806 | */ | |
807 | ||
808 | static int statfs_slow_fill(struct gfs2_rgrpd *rgd, | |
bd209cc0 | 809 | struct gfs2_statfs_change_host *sc) |
b3b94faa DT |
810 | { |
811 | gfs2_rgrp_verify(rgd); | |
bb8d8a6f | 812 | sc->sc_total += rgd->rd_data; |
b3b94faa DT |
813 | sc->sc_free += rgd->rd_rg.rg_free; |
814 | sc->sc_dinodes += rgd->rd_rg.rg_dinodes; | |
815 | return 0; | |
816 | } | |
817 | ||
818 | /** | |
819 | * gfs2_statfs_slow - Stat a filesystem using asynchronous locking | |
820 | * @sdp: the filesystem | |
821 | * @sc: the sc info that will be returned | |
822 | * | |
823 | * Any error (other than a signal) will cause this routine to fall back | |
824 | * to the synchronous version. | |
825 | * | |
826 | * FIXME: This really shouldn't busy wait like this. | |
827 | * | |
828 | * Returns: errno | |
829 | */ | |
830 | ||
bd209cc0 | 831 | int gfs2_statfs_slow(struct gfs2_sbd *sdp, struct gfs2_statfs_change_host *sc) |
b3b94faa DT |
832 | { |
833 | struct gfs2_holder ri_gh; | |
834 | struct gfs2_rgrpd *rgd_next; | |
835 | struct gfs2_holder *gha, *gh; | |
836 | unsigned int slots = 64; | |
837 | unsigned int x; | |
838 | int done; | |
839 | int error = 0, err; | |
840 | ||
bd209cc0 | 841 | memset(sc, 0, sizeof(struct gfs2_statfs_change_host)); |
b3b94faa DT |
842 | gha = kcalloc(slots, sizeof(struct gfs2_holder), GFP_KERNEL); |
843 | if (!gha) | |
844 | return -ENOMEM; | |
845 | ||
846 | error = gfs2_rindex_hold(sdp, &ri_gh); | |
847 | if (error) | |
848 | goto out; | |
849 | ||
850 | rgd_next = gfs2_rgrpd_get_first(sdp); | |
851 | ||
852 | for (;;) { | |
853 | done = 1; | |
854 | ||
855 | for (x = 0; x < slots; x++) { | |
856 | gh = gha + x; | |
857 | ||
858 | if (gh->gh_gl && gfs2_glock_poll(gh)) { | |
859 | err = gfs2_glock_wait(gh); | |
860 | if (err) { | |
861 | gfs2_holder_uninit(gh); | |
862 | error = err; | |
863 | } else { | |
864 | if (!error) | |
5c676f6d SW |
865 | error = statfs_slow_fill( |
866 | gh->gh_gl->gl_object, sc); | |
b3b94faa DT |
867 | gfs2_glock_dq_uninit(gh); |
868 | } | |
869 | } | |
870 | ||
871 | if (gh->gh_gl) | |
872 | done = 0; | |
873 | else if (rgd_next && !error) { | |
874 | error = gfs2_glock_nq_init(rgd_next->rd_gl, | |
875 | LM_ST_SHARED, | |
876 | GL_ASYNC, | |
877 | gh); | |
878 | rgd_next = gfs2_rgrpd_get_next(rgd_next); | |
879 | done = 0; | |
880 | } | |
881 | ||
882 | if (signal_pending(current)) | |
883 | error = -ERESTARTSYS; | |
884 | } | |
885 | ||
886 | if (done) | |
887 | break; | |
888 | ||
889 | yield(); | |
890 | } | |
891 | ||
892 | gfs2_glock_dq_uninit(&ri_gh); | |
893 | ||
a91ea69f | 894 | out: |
b3b94faa | 895 | kfree(gha); |
b3b94faa DT |
896 | return error; |
897 | } | |
898 | ||
899 | struct lfcc { | |
900 | struct list_head list; | |
901 | struct gfs2_holder gh; | |
902 | }; | |
903 | ||
904 | /** | |
905 | * gfs2_lock_fs_check_clean - Stop all writes to the FS and check that all | |
906 | * journals are clean | |
907 | * @sdp: the file system | |
908 | * @state: the state to put the transaction lock into | |
909 | * @t_gh: the hold on the transaction lock | |
910 | * | |
911 | * Returns: errno | |
912 | */ | |
913 | ||
08bc2dbc AB |
914 | static int gfs2_lock_fs_check_clean(struct gfs2_sbd *sdp, |
915 | struct gfs2_holder *t_gh) | |
b3b94faa | 916 | { |
5c676f6d | 917 | struct gfs2_inode *ip; |
b3b94faa DT |
918 | struct gfs2_holder ji_gh; |
919 | struct gfs2_jdesc *jd; | |
920 | struct lfcc *lfcc; | |
921 | LIST_HEAD(list); | |
55167622 | 922 | struct gfs2_log_header_host lh; |
b3b94faa DT |
923 | int error; |
924 | ||
925 | error = gfs2_jindex_hold(sdp, &ji_gh); | |
926 | if (error) | |
927 | return error; | |
928 | ||
929 | list_for_each_entry(jd, &sdp->sd_jindex_list, jd_list) { | |
930 | lfcc = kmalloc(sizeof(struct lfcc), GFP_KERNEL); | |
931 | if (!lfcc) { | |
932 | error = -ENOMEM; | |
933 | goto out; | |
934 | } | |
feaa7bba SW |
935 | ip = GFS2_I(jd->jd_inode); |
936 | error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &lfcc->gh); | |
b3b94faa DT |
937 | if (error) { |
938 | kfree(lfcc); | |
939 | goto out; | |
940 | } | |
941 | list_add(&lfcc->list, &list); | |
942 | } | |
943 | ||
944 | error = gfs2_glock_nq_init(sdp->sd_trans_gl, LM_ST_DEFERRED, | |
579b78a4 | 945 | LM_FLAG_PRIORITY | GL_NOCACHE, |
b3b94faa DT |
946 | t_gh); |
947 | ||
948 | list_for_each_entry(jd, &sdp->sd_jindex_list, jd_list) { | |
949 | error = gfs2_jdesc_check(jd); | |
950 | if (error) | |
951 | break; | |
952 | error = gfs2_find_jhead(jd, &lh); | |
953 | if (error) | |
954 | break; | |
955 | if (!(lh.lh_flags & GFS2_LOG_HEAD_UNMOUNT)) { | |
956 | error = -EBUSY; | |
957 | break; | |
958 | } | |
959 | } | |
960 | ||
961 | if (error) | |
962 | gfs2_glock_dq_uninit(t_gh); | |
963 | ||
a91ea69f | 964 | out: |
b3b94faa DT |
965 | while (!list_empty(&list)) { |
966 | lfcc = list_entry(list.next, struct lfcc, list); | |
967 | list_del(&lfcc->list); | |
968 | gfs2_glock_dq_uninit(&lfcc->gh); | |
969 | kfree(lfcc); | |
970 | } | |
971 | gfs2_glock_dq_uninit(&ji_gh); | |
b3b94faa DT |
972 | return error; |
973 | } | |
974 | ||
975 | /** | |
976 | * gfs2_freeze_fs - freezes the file system | |
977 | * @sdp: the file system | |
978 | * | |
979 | * This function flushes data and meta data for all machines by | |
980 | * aquiring the transaction log exclusively. All journals are | |
981 | * ensured to be in a clean state as well. | |
982 | * | |
983 | * Returns: errno | |
984 | */ | |
985 | ||
986 | int gfs2_freeze_fs(struct gfs2_sbd *sdp) | |
987 | { | |
988 | int error = 0; | |
989 | ||
f55ab26a | 990 | mutex_lock(&sdp->sd_freeze_lock); |
b3b94faa DT |
991 | |
992 | if (!sdp->sd_freeze_count++) { | |
993 | error = gfs2_lock_fs_check_clean(sdp, &sdp->sd_freeze_gh); | |
994 | if (error) | |
995 | sdp->sd_freeze_count--; | |
996 | } | |
997 | ||
f55ab26a | 998 | mutex_unlock(&sdp->sd_freeze_lock); |
b3b94faa DT |
999 | |
1000 | return error; | |
1001 | } | |
1002 | ||
1003 | /** | |
1004 | * gfs2_unfreeze_fs - unfreezes the file system | |
1005 | * @sdp: the file system | |
1006 | * | |
1007 | * This function allows the file system to proceed by unlocking | |
1008 | * the exclusively held transaction lock. Other GFS2 nodes are | |
1009 | * now free to acquire the lock shared and go on with their lives. | |
1010 | * | |
1011 | */ | |
1012 | ||
1013 | void gfs2_unfreeze_fs(struct gfs2_sbd *sdp) | |
1014 | { | |
f55ab26a | 1015 | mutex_lock(&sdp->sd_freeze_lock); |
b3b94faa DT |
1016 | |
1017 | if (sdp->sd_freeze_count && !--sdp->sd_freeze_count) | |
1018 | gfs2_glock_dq_uninit(&sdp->sd_freeze_gh); | |
1019 | ||
f55ab26a | 1020 | mutex_unlock(&sdp->sd_freeze_lock); |
b3b94faa DT |
1021 | } |
1022 |