| 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | |
| 3 | #include "misc.h" |
| 4 | #include "ctree.h" |
| 5 | #include "block-rsv.h" |
| 6 | #include "space-info.h" |
| 7 | #include "transaction.h" |
| 8 | |
| 9 | static u64 block_rsv_release_bytes(struct btrfs_fs_info *fs_info, |
| 10 | struct btrfs_block_rsv *block_rsv, |
| 11 | struct btrfs_block_rsv *dest, u64 num_bytes, |
| 12 | u64 *qgroup_to_release_ret) |
| 13 | { |
| 14 | struct btrfs_space_info *space_info = block_rsv->space_info; |
| 15 | u64 qgroup_to_release = 0; |
| 16 | u64 ret; |
| 17 | |
| 18 | spin_lock(&block_rsv->lock); |
| 19 | if (num_bytes == (u64)-1) { |
| 20 | num_bytes = block_rsv->size; |
| 21 | qgroup_to_release = block_rsv->qgroup_rsv_size; |
| 22 | } |
| 23 | block_rsv->size -= num_bytes; |
| 24 | if (block_rsv->reserved >= block_rsv->size) { |
| 25 | num_bytes = block_rsv->reserved - block_rsv->size; |
| 26 | block_rsv->reserved = block_rsv->size; |
| 27 | block_rsv->full = 1; |
| 28 | } else { |
| 29 | num_bytes = 0; |
| 30 | } |
| 31 | if (block_rsv->qgroup_rsv_reserved >= block_rsv->qgroup_rsv_size) { |
| 32 | qgroup_to_release = block_rsv->qgroup_rsv_reserved - |
| 33 | block_rsv->qgroup_rsv_size; |
| 34 | block_rsv->qgroup_rsv_reserved = block_rsv->qgroup_rsv_size; |
| 35 | } else { |
| 36 | qgroup_to_release = 0; |
| 37 | } |
| 38 | spin_unlock(&block_rsv->lock); |
| 39 | |
| 40 | ret = num_bytes; |
| 41 | if (num_bytes > 0) { |
| 42 | if (dest) { |
| 43 | spin_lock(&dest->lock); |
| 44 | if (!dest->full) { |
| 45 | u64 bytes_to_add; |
| 46 | |
| 47 | bytes_to_add = dest->size - dest->reserved; |
| 48 | bytes_to_add = min(num_bytes, bytes_to_add); |
| 49 | dest->reserved += bytes_to_add; |
| 50 | if (dest->reserved >= dest->size) |
| 51 | dest->full = 1; |
| 52 | num_bytes -= bytes_to_add; |
| 53 | } |
| 54 | spin_unlock(&dest->lock); |
| 55 | } |
| 56 | if (num_bytes) |
| 57 | btrfs_space_info_add_old_bytes(fs_info, space_info, |
| 58 | num_bytes); |
| 59 | } |
| 60 | if (qgroup_to_release_ret) |
| 61 | *qgroup_to_release_ret = qgroup_to_release; |
| 62 | return ret; |
| 63 | } |
| 64 | |
| 65 | int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src, |
| 66 | struct btrfs_block_rsv *dst, u64 num_bytes, |
| 67 | bool update_size) |
| 68 | { |
| 69 | int ret; |
| 70 | |
| 71 | ret = btrfs_block_rsv_use_bytes(src, num_bytes); |
| 72 | if (ret) |
| 73 | return ret; |
| 74 | |
| 75 | btrfs_block_rsv_add_bytes(dst, num_bytes, update_size); |
| 76 | return 0; |
| 77 | } |
| 78 | |
| 79 | void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type) |
| 80 | { |
| 81 | memset(rsv, 0, sizeof(*rsv)); |
| 82 | spin_lock_init(&rsv->lock); |
| 83 | rsv->type = type; |
| 84 | } |
| 85 | |
| 86 | void btrfs_init_metadata_block_rsv(struct btrfs_fs_info *fs_info, |
| 87 | struct btrfs_block_rsv *rsv, |
| 88 | unsigned short type) |
| 89 | { |
| 90 | btrfs_init_block_rsv(rsv, type); |
| 91 | rsv->space_info = btrfs_find_space_info(fs_info, |
| 92 | BTRFS_BLOCK_GROUP_METADATA); |
| 93 | } |
| 94 | |
| 95 | struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_fs_info *fs_info, |
| 96 | unsigned short type) |
| 97 | { |
| 98 | struct btrfs_block_rsv *block_rsv; |
| 99 | |
| 100 | block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS); |
| 101 | if (!block_rsv) |
| 102 | return NULL; |
| 103 | |
| 104 | btrfs_init_metadata_block_rsv(fs_info, block_rsv, type); |
| 105 | return block_rsv; |
| 106 | } |
| 107 | |
| 108 | void btrfs_free_block_rsv(struct btrfs_fs_info *fs_info, |
| 109 | struct btrfs_block_rsv *rsv) |
| 110 | { |
| 111 | if (!rsv) |
| 112 | return; |
| 113 | btrfs_block_rsv_release(fs_info, rsv, (u64)-1); |
| 114 | kfree(rsv); |
| 115 | } |
| 116 | |
| 117 | int btrfs_block_rsv_add(struct btrfs_root *root, |
| 118 | struct btrfs_block_rsv *block_rsv, u64 num_bytes, |
| 119 | enum btrfs_reserve_flush_enum flush) |
| 120 | { |
| 121 | int ret; |
| 122 | |
| 123 | if (num_bytes == 0) |
| 124 | return 0; |
| 125 | |
| 126 | ret = btrfs_reserve_metadata_bytes(root, block_rsv, num_bytes, flush); |
| 127 | if (!ret) |
| 128 | btrfs_block_rsv_add_bytes(block_rsv, num_bytes, true); |
| 129 | |
| 130 | return ret; |
| 131 | } |
| 132 | |
| 133 | int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_factor) |
| 134 | { |
| 135 | u64 num_bytes = 0; |
| 136 | int ret = -ENOSPC; |
| 137 | |
| 138 | if (!block_rsv) |
| 139 | return 0; |
| 140 | |
| 141 | spin_lock(&block_rsv->lock); |
| 142 | num_bytes = div_factor(block_rsv->size, min_factor); |
| 143 | if (block_rsv->reserved >= num_bytes) |
| 144 | ret = 0; |
| 145 | spin_unlock(&block_rsv->lock); |
| 146 | |
| 147 | return ret; |
| 148 | } |
| 149 | |
| 150 | int btrfs_block_rsv_refill(struct btrfs_root *root, |
| 151 | struct btrfs_block_rsv *block_rsv, u64 min_reserved, |
| 152 | enum btrfs_reserve_flush_enum flush) |
| 153 | { |
| 154 | u64 num_bytes = 0; |
| 155 | int ret = -ENOSPC; |
| 156 | |
| 157 | if (!block_rsv) |
| 158 | return 0; |
| 159 | |
| 160 | spin_lock(&block_rsv->lock); |
| 161 | num_bytes = min_reserved; |
| 162 | if (block_rsv->reserved >= num_bytes) |
| 163 | ret = 0; |
| 164 | else |
| 165 | num_bytes -= block_rsv->reserved; |
| 166 | spin_unlock(&block_rsv->lock); |
| 167 | |
| 168 | if (!ret) |
| 169 | return 0; |
| 170 | |
| 171 | ret = btrfs_reserve_metadata_bytes(root, block_rsv, num_bytes, flush); |
| 172 | if (!ret) { |
| 173 | btrfs_block_rsv_add_bytes(block_rsv, num_bytes, false); |
| 174 | return 0; |
| 175 | } |
| 176 | |
| 177 | return ret; |
| 178 | } |
| 179 | |
| 180 | u64 __btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, |
| 181 | struct btrfs_block_rsv *block_rsv, |
| 182 | u64 num_bytes, u64 *qgroup_to_release) |
| 183 | { |
| 184 | struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; |
| 185 | struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv; |
| 186 | struct btrfs_block_rsv *target = NULL; |
| 187 | |
| 188 | /* |
| 189 | * If we are the delayed_rsv then push to the global rsv, otherwise dump |
| 190 | * into the delayed rsv if it is not full. |
| 191 | */ |
| 192 | if (block_rsv == delayed_rsv) |
| 193 | target = global_rsv; |
| 194 | else if (block_rsv != global_rsv && !delayed_rsv->full) |
| 195 | target = delayed_rsv; |
| 196 | |
| 197 | if (target && block_rsv->space_info != target->space_info) |
| 198 | target = NULL; |
| 199 | |
| 200 | return block_rsv_release_bytes(fs_info, block_rsv, target, num_bytes, |
| 201 | qgroup_to_release); |
| 202 | } |
| 203 | |
| 204 | int btrfs_block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv, u64 num_bytes) |
| 205 | { |
| 206 | int ret = -ENOSPC; |
| 207 | |
| 208 | spin_lock(&block_rsv->lock); |
| 209 | if (block_rsv->reserved >= num_bytes) { |
| 210 | block_rsv->reserved -= num_bytes; |
| 211 | if (block_rsv->reserved < block_rsv->size) |
| 212 | block_rsv->full = 0; |
| 213 | ret = 0; |
| 214 | } |
| 215 | spin_unlock(&block_rsv->lock); |
| 216 | return ret; |
| 217 | } |
| 218 | |
| 219 | void btrfs_block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv, |
| 220 | u64 num_bytes, bool update_size) |
| 221 | { |
| 222 | spin_lock(&block_rsv->lock); |
| 223 | block_rsv->reserved += num_bytes; |
| 224 | if (update_size) |
| 225 | block_rsv->size += num_bytes; |
| 226 | else if (block_rsv->reserved >= block_rsv->size) |
| 227 | block_rsv->full = 1; |
| 228 | spin_unlock(&block_rsv->lock); |
| 229 | } |
| 230 | |
| 231 | int btrfs_cond_migrate_bytes(struct btrfs_fs_info *fs_info, |
| 232 | struct btrfs_block_rsv *dest, u64 num_bytes, |
| 233 | int min_factor) |
| 234 | { |
| 235 | struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; |
| 236 | u64 min_bytes; |
| 237 | |
| 238 | if (global_rsv->space_info != dest->space_info) |
| 239 | return -ENOSPC; |
| 240 | |
| 241 | spin_lock(&global_rsv->lock); |
| 242 | min_bytes = div_factor(global_rsv->size, min_factor); |
| 243 | if (global_rsv->reserved < min_bytes + num_bytes) { |
| 244 | spin_unlock(&global_rsv->lock); |
| 245 | return -ENOSPC; |
| 246 | } |
| 247 | global_rsv->reserved -= num_bytes; |
| 248 | if (global_rsv->reserved < global_rsv->size) |
| 249 | global_rsv->full = 0; |
| 250 | spin_unlock(&global_rsv->lock); |
| 251 | |
| 252 | btrfs_block_rsv_add_bytes(dest, num_bytes, true); |
| 253 | return 0; |
| 254 | } |
| 255 | |
| 256 | void btrfs_update_global_block_rsv(struct btrfs_fs_info *fs_info) |
| 257 | { |
| 258 | struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv; |
| 259 | struct btrfs_space_info *sinfo = block_rsv->space_info; |
| 260 | u64 num_bytes; |
| 261 | |
| 262 | /* |
| 263 | * The global block rsv is based on the size of the extent tree, the |
| 264 | * checksum tree and the root tree. If the fs is empty we want to set |
| 265 | * it to a minimal amount for safety. |
| 266 | */ |
| 267 | num_bytes = btrfs_root_used(&fs_info->extent_root->root_item) + |
| 268 | btrfs_root_used(&fs_info->csum_root->root_item) + |
| 269 | btrfs_root_used(&fs_info->tree_root->root_item); |
| 270 | num_bytes = max_t(u64, num_bytes, SZ_16M); |
| 271 | |
| 272 | spin_lock(&sinfo->lock); |
| 273 | spin_lock(&block_rsv->lock); |
| 274 | |
| 275 | block_rsv->size = min_t(u64, num_bytes, SZ_512M); |
| 276 | |
| 277 | if (block_rsv->reserved < block_rsv->size) { |
| 278 | num_bytes = btrfs_space_info_used(sinfo, true); |
| 279 | if (sinfo->total_bytes > num_bytes) { |
| 280 | num_bytes = sinfo->total_bytes - num_bytes; |
| 281 | num_bytes = min(num_bytes, |
| 282 | block_rsv->size - block_rsv->reserved); |
| 283 | block_rsv->reserved += num_bytes; |
| 284 | btrfs_space_info_update_bytes_may_use(fs_info, sinfo, |
| 285 | num_bytes); |
| 286 | } |
| 287 | } else if (block_rsv->reserved > block_rsv->size) { |
| 288 | num_bytes = block_rsv->reserved - block_rsv->size; |
| 289 | btrfs_space_info_update_bytes_may_use(fs_info, sinfo, |
| 290 | -num_bytes); |
| 291 | block_rsv->reserved = block_rsv->size; |
| 292 | } |
| 293 | |
| 294 | if (block_rsv->reserved == block_rsv->size) |
| 295 | block_rsv->full = 1; |
| 296 | else |
| 297 | block_rsv->full = 0; |
| 298 | |
| 299 | spin_unlock(&block_rsv->lock); |
| 300 | spin_unlock(&sinfo->lock); |
| 301 | } |
| 302 | |
| 303 | void btrfs_init_global_block_rsv(struct btrfs_fs_info *fs_info) |
| 304 | { |
| 305 | struct btrfs_space_info *space_info; |
| 306 | |
| 307 | space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); |
| 308 | fs_info->chunk_block_rsv.space_info = space_info; |
| 309 | |
| 310 | space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); |
| 311 | fs_info->global_block_rsv.space_info = space_info; |
| 312 | fs_info->trans_block_rsv.space_info = space_info; |
| 313 | fs_info->empty_block_rsv.space_info = space_info; |
| 314 | fs_info->delayed_block_rsv.space_info = space_info; |
| 315 | fs_info->delayed_refs_rsv.space_info = space_info; |
| 316 | |
| 317 | fs_info->extent_root->block_rsv = &fs_info->delayed_refs_rsv; |
| 318 | fs_info->csum_root->block_rsv = &fs_info->delayed_refs_rsv; |
| 319 | fs_info->dev_root->block_rsv = &fs_info->global_block_rsv; |
| 320 | fs_info->tree_root->block_rsv = &fs_info->global_block_rsv; |
| 321 | if (fs_info->quota_root) |
| 322 | fs_info->quota_root->block_rsv = &fs_info->global_block_rsv; |
| 323 | fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv; |
| 324 | |
| 325 | btrfs_update_global_block_rsv(fs_info); |
| 326 | } |
| 327 | |
| 328 | void btrfs_release_global_block_rsv(struct btrfs_fs_info *fs_info) |
| 329 | { |
| 330 | btrfs_block_rsv_release(fs_info, &fs_info->global_block_rsv, (u64)-1); |
| 331 | WARN_ON(fs_info->trans_block_rsv.size > 0); |
| 332 | WARN_ON(fs_info->trans_block_rsv.reserved > 0); |
| 333 | WARN_ON(fs_info->chunk_block_rsv.size > 0); |
| 334 | WARN_ON(fs_info->chunk_block_rsv.reserved > 0); |
| 335 | WARN_ON(fs_info->delayed_block_rsv.size > 0); |
| 336 | WARN_ON(fs_info->delayed_block_rsv.reserved > 0); |
| 337 | WARN_ON(fs_info->delayed_refs_rsv.reserved > 0); |
| 338 | WARN_ON(fs_info->delayed_refs_rsv.size > 0); |
| 339 | } |
| 340 | |
| 341 | static struct btrfs_block_rsv *get_block_rsv( |
| 342 | const struct btrfs_trans_handle *trans, |
| 343 | const struct btrfs_root *root) |
| 344 | { |
| 345 | struct btrfs_fs_info *fs_info = root->fs_info; |
| 346 | struct btrfs_block_rsv *block_rsv = NULL; |
| 347 | |
| 348 | if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || |
| 349 | (root == fs_info->csum_root && trans->adding_csums) || |
| 350 | (root == fs_info->uuid_root)) |
| 351 | block_rsv = trans->block_rsv; |
| 352 | |
| 353 | if (!block_rsv) |
| 354 | block_rsv = root->block_rsv; |
| 355 | |
| 356 | if (!block_rsv) |
| 357 | block_rsv = &fs_info->empty_block_rsv; |
| 358 | |
| 359 | return block_rsv; |
| 360 | } |
| 361 | |
| 362 | struct btrfs_block_rsv *btrfs_use_block_rsv(struct btrfs_trans_handle *trans, |
| 363 | struct btrfs_root *root, |
| 364 | u32 blocksize) |
| 365 | { |
| 366 | struct btrfs_fs_info *fs_info = root->fs_info; |
| 367 | struct btrfs_block_rsv *block_rsv; |
| 368 | struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; |
| 369 | int ret; |
| 370 | bool global_updated = false; |
| 371 | |
| 372 | block_rsv = get_block_rsv(trans, root); |
| 373 | |
| 374 | if (unlikely(block_rsv->size == 0)) |
| 375 | goto try_reserve; |
| 376 | again: |
| 377 | ret = btrfs_block_rsv_use_bytes(block_rsv, blocksize); |
| 378 | if (!ret) |
| 379 | return block_rsv; |
| 380 | |
| 381 | if (block_rsv->failfast) |
| 382 | return ERR_PTR(ret); |
| 383 | |
| 384 | if (block_rsv->type == BTRFS_BLOCK_RSV_GLOBAL && !global_updated) { |
| 385 | global_updated = true; |
| 386 | btrfs_update_global_block_rsv(fs_info); |
| 387 | goto again; |
| 388 | } |
| 389 | |
| 390 | /* |
| 391 | * The global reserve still exists to save us from ourselves, so don't |
| 392 | * warn_on if we are short on our delayed refs reserve. |
| 393 | */ |
| 394 | if (block_rsv->type != BTRFS_BLOCK_RSV_DELREFS && |
| 395 | btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { |
| 396 | static DEFINE_RATELIMIT_STATE(_rs, |
| 397 | DEFAULT_RATELIMIT_INTERVAL * 10, |
| 398 | /*DEFAULT_RATELIMIT_BURST*/ 1); |
| 399 | if (__ratelimit(&_rs)) |
| 400 | WARN(1, KERN_DEBUG |
| 401 | "BTRFS: block rsv returned %d\n", ret); |
| 402 | } |
| 403 | try_reserve: |
| 404 | ret = btrfs_reserve_metadata_bytes(root, block_rsv, blocksize, |
| 405 | BTRFS_RESERVE_NO_FLUSH); |
| 406 | if (!ret) |
| 407 | return block_rsv; |
| 408 | /* |
| 409 | * If we couldn't reserve metadata bytes try and use some from |
| 410 | * the global reserve if its space type is the same as the global |
| 411 | * reservation. |
| 412 | */ |
| 413 | if (block_rsv->type != BTRFS_BLOCK_RSV_GLOBAL && |
| 414 | block_rsv->space_info == global_rsv->space_info) { |
| 415 | ret = btrfs_block_rsv_use_bytes(global_rsv, blocksize); |
| 416 | if (!ret) |
| 417 | return global_rsv; |
| 418 | } |
| 419 | return ERR_PTR(ret); |
| 420 | } |