drivers/block/drbd/drbd_actlog.c

   1 /*
   2    drbd_actlog.c
   3
   4    This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
   5
   6    Copyright (C) 2003-2008, LINBIT Information Technologies GmbH.
   7    Copyright (C) 2003-2008, Philipp Reisner <philipp.reisner@linbit.com>.
   8    Copyright (C) 2003-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
   9
  10    drbd is free software; you can redistribute it and/or modify
  11    it under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 2, or (at your option)
  13    any later version.
  14
  15    drbd is distributed in the hope that it will be useful,
  16    but WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18    GNU General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with drbd; see the file COPYING.  If not, write to
  22    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
  23
  24  */
  25
  26 #include <linux/slab.h>
  27 #include <linux/crc32c.h>
  28 #include <linux/drbd.h>
  29 #include <linux/drbd_limits.h>
  30 #include <linux/dynamic_debug.h>
  31 #include "drbd_int.h"
  32 #include "drbd_wrappers.h"
  33
  34
  35 enum al_transaction_types {
  36         AL_TR_UPDATE = 0,
  37         AL_TR_INITIALIZED = 0xffff
  38 };
  39 /* all fields on disc in big endian */
  40 struct __packed al_transaction_on_disk {
  41         /* don't we all like magic */
  42         __be32  magic;
  43
  44         /* to identify the most recent transaction block
  45          * in the on disk ring buffer */
  46         __be32  tr_number;
  47
  48         /* checksum on the full 4k block, with this field set to 0. */
  49         __be32  crc32c;
  50
  51         /* type of transaction, special transaction types like:
  52          * purge-all, set-all-idle, set-all-active, ... to-be-defined
  53          * see also enum al_transaction_types */
  54         __be16  transaction_type;
  55
  56         /* we currently allow only a few thousand extents,
  57          * so 16bit will be enough for the slot number. */
  58
  59         /* how many updates in this transaction */
  60         __be16  n_updates;
  61
  62         /* maximum slot number, "al-extents" in drbd.conf speak.
  63          * Having this in each transaction should make reconfiguration
  64          * of that parameter easier. */
  65         __be16  context_size;
  66
  67         /* slot number the context starts with */
  68         __be16  context_start_slot_nr;
  69
  70         /* Some reserved bytes.  Expected usage is a 64bit counter of
  71          * sectors-written since device creation, and other data generation tag
  72          * supporting usage */
  73         __be32  __reserved[4];
  74
  75         /* --- 36 byte used --- */
  76
  77         /* Reserve space for up to AL_UPDATES_PER_TRANSACTION changes
  78          * in one transaction, then use the remaining byte in the 4k block for
  79          * context information.  "Flexible" number of updates per transaction
  80          * does not help, as we have to account for the case when all update
  81          * slots are used anyways, so it would only complicate code without
  82          * additional benefit.
  83          */
  84         __be16  update_slot_nr[AL_UPDATES_PER_TRANSACTION];
  85
  86         /* but the extent number is 32bit, which at an extent size of 4 MiB
  87          * allows to cover device sizes of up to 2**54 Byte (16 PiB) */
  88         __be32  update_extent_nr[AL_UPDATES_PER_TRANSACTION];
  89
  90         /* --- 420 bytes used (36 + 64*6) --- */
  91
  92         /* 4096 - 420 = 3676 = 919 * 4 */
  93         __be32  context[AL_CONTEXT_PER_TRANSACTION];
  94 };
  95
  96 struct update_odbm_work {
  97         struct drbd_work w;
  98         unsigned int enr;
  99 };
 100
 101 struct update_al_work {
 102         struct drbd_work w;
 103         struct completion event;
 104         int err;
 105 };
 106
 107 static int al_write_transaction(struct drbd_conf *mdev, bool delegate);
 108
 109 void *drbd_md_get_buffer(struct drbd_conf *mdev)
 110 {
 111         int r;
 112
 113         wait_event(mdev->misc_wait,
 114                    (r = atomic_cmpxchg(&mdev->md_io_in_use, 0, 1)) == 0 ||
 115                    mdev->state.disk <= D_FAILED);
 116
 117         return r ? NULL : page_address(mdev->md_io_page);
 118 }
 119
 120 void drbd_md_put_buffer(struct drbd_conf *mdev)
 121 {
 122         if (atomic_dec_and_test(&mdev->md_io_in_use))
 123                 wake_up(&mdev->misc_wait);
 124 }
 125
 126 void wait_until_done_or_force_detached(struct drbd_conf *mdev, struct drbd_backing_dev *bdev,
 127                                      unsigned int *done)
 128 {
 129         long dt;
 130
 131         rcu_read_lock();
 132         dt = rcu_dereference(bdev->disk_conf)->disk_timeout;
 133         rcu_read_unlock();
 134         dt = dt * HZ / 10;
 135         if (dt == 0)
 136                 dt = MAX_SCHEDULE_TIMEOUT;
 137
 138         dt = wait_event_timeout(mdev->misc_wait,
 139                         *done || test_bit(FORCE_DETACH, &mdev->flags), dt);
 140         if (dt == 0) {
 141                 dev_err(DEV, "meta-data IO operation timed out\n");
 142                 drbd_chk_io_error(mdev, 1, DRBD_FORCE_DETACH);
 143         }
 144 }
 145
 146 static int _drbd_md_sync_page_io(struct drbd_conf *mdev,
 147                                  struct drbd_backing_dev *bdev,
 148                                  struct page *page, sector_t sector,
 149                                  int rw, int size)
 150 {
 151         struct bio *bio;
 152         int err;
 153
 154         mdev->md_io.done = 0;
 155         mdev->md_io.error = -ENODEV;
 156
 157         if ((rw & WRITE) && !test_bit(MD_NO_FUA, &mdev->flags))
 158                 rw |= REQ_FUA | REQ_FLUSH;
 159         rw |= REQ_SYNC;
 160
 161         bio = bio_alloc_drbd(GFP_NOIO);
 162         bio->bi_bdev = bdev->md_bdev;
 163         bio->bi_sector = sector;
 164         err = -EIO;
 165         if (bio_add_page(bio, page, size, 0) != size)
 166                 goto out;
 167         bio->bi_private = &mdev->md_io;
 168         bio->bi_end_io = drbd_md_io_complete;
 169         bio->bi_rw = rw;
 170
 171         if (!(rw & WRITE) && mdev->state.disk == D_DISKLESS && mdev->ldev == NULL)
 172                 /* special case, drbd_md_read() during drbd_adm_attach(): no get_ldev */
 173                 ;
 174         else if (!get_ldev_if_state(mdev, D_ATTACHING)) {
 175                 /* Corresponding put_ldev in drbd_md_io_complete() */
 176                 dev_err(DEV, "ASSERT FAILED: get_ldev_if_state() == 1 in _drbd_md_sync_page_io()\n");
 177                 err = -ENODEV;
 178                 goto out;
 179         }
 180
 181         bio_get(bio); /* one bio_put() is in the completion handler */
 182         atomic_inc(&mdev->md_io_in_use); /* drbd_md_put_buffer() is in the completion handler */
 183         if (drbd_insert_fault(mdev, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD))
 184                 bio_endio(bio, -EIO);
 185         else
 186                 submit_bio(rw, bio);
 187         wait_until_done_or_force_detached(mdev, bdev, &mdev->md_io.done);
 188         if (bio_flagged(bio, BIO_UPTODATE))
 189                 err = mdev->md_io.error;
 190
 191  out:
 192         bio_put(bio);
 193         return err;
 194 }
 195
 196 int drbd_md_sync_page_io(struct drbd_conf *mdev, struct drbd_backing_dev *bdev,
 197                          sector_t sector, int rw)
 198 {
 199         int err;
 200         struct page *iop = mdev->md_io_page;
 201
 202         D_ASSERT(atomic_read(&mdev->md_io_in_use) == 1);
 203
 204         BUG_ON(!bdev->md_bdev);
 205
 206         dev_dbg(DEV, "meta_data io: %s [%d]:%s(,%llus,%s) %pS\n",
 207              current->comm, current->pid, __func__,
 208              (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ",
 209              (void*)_RET_IP_ );
 210
 211         if (sector < drbd_md_first_sector(bdev) ||
 212             sector + 7 > drbd_md_last_sector(bdev))
 213                 dev_alert(DEV, "%s [%d]:%s(,%llus,%s) out of range md access!\n",
 214                      current->comm, current->pid, __func__,
 215                      (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ");
 216
 217         /* we do all our meta data IO in aligned 4k blocks. */
 218         err = _drbd_md_sync_page_io(mdev, bdev, iop, sector, rw, 4096);
 219         if (err) {
 220                 dev_err(DEV, "drbd_md_sync_page_io(,%llus,%s) failed with error %d\n",
 221                     (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ", err);
 222         }
 223         return err;
 224 }
 225
 226 static struct lc_element *_al_get(struct drbd_conf *mdev, unsigned int enr)
 227 {
 228         struct lc_element *al_ext;
 229         struct lc_element *tmp;
 230         int wake;
 231
 232         spin_lock_irq(&mdev->al_lock);
 233         tmp = lc_find(mdev->resync, enr/AL_EXT_PER_BM_SECT);
 234         if (unlikely(tmp != NULL)) {
 235                 struct bm_extent  *bm_ext = lc_entry(tmp, struct bm_extent, lce);
 236                 if (test_bit(BME_NO_WRITES, &bm_ext->flags)) {
 237                         wake = !test_and_set_bit(BME_PRIORITY, &bm_ext->flags);
 238                         spin_unlock_irq(&mdev->al_lock);
 239                         if (wake)
 240                                 wake_up(&mdev->al_wait);
 241                         return NULL;
 242                 }
 243         }
 244         al_ext = lc_get(mdev->act_log, enr);
 245         spin_unlock_irq(&mdev->al_lock);
 246         return al_ext;
 247 }
 248
 249 /*
 250  * @delegate:   delegate activity log I/O to the worker thread
 251  */
 252 void drbd_al_begin_io(struct drbd_conf *mdev, struct drbd_interval *i, bool delegate)
 253 {
 254         /* for bios crossing activity log extent boundaries,
 255          * we may need to activate two extents in one go */
 256         unsigned first = i->sector >> (AL_EXTENT_SHIFT-9);
 257         unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
 258         unsigned enr;
 259         bool locked = false;
 260
 261         /* When called through generic_make_request(), we must delegate
 262          * activity log I/O to the worker thread: a further request
 263          * submitted via generic_make_request() within the same task
 264          * would be queued on current->bio_list, and would only start
 265          * after this function returns (see generic_make_request()).
 266          *
 267          * However, if we *are* the worker, we must not delegate to ourselves.
 268          */
 269
 270         if (delegate)
 271                 BUG_ON(current == mdev->tconn->worker.task);
 272
 273         D_ASSERT(first <= last);
 274         D_ASSERT(atomic_read(&mdev->local_cnt) > 0);
 275
 276         for (enr = first; enr <= last; enr++)
 277                 wait_event(mdev->al_wait, _al_get(mdev, enr) != NULL);
 278
 279         /* Serialize multiple transactions.
 280          * This uses test_and_set_bit, memory barrier is implicit.
 281          */
 282         wait_event(mdev->al_wait,
 283                         mdev->act_log->pending_changes == 0 ||
 284                         (locked = lc_try_lock_for_transaction(mdev->act_log)));
 285
 286         if (locked) {
 287                 /* Double check: it may have been committed by someone else,
 288                  * while we have been waiting for the lock. */
 289                 if (mdev->act_log->pending_changes) {
 290                         bool write_al_updates;
 291
 292                         rcu_read_lock();
 293                         write_al_updates = rcu_dereference(mdev->ldev->disk_conf)->al_updates;
 294                         rcu_read_unlock();
 295
 296                         if (write_al_updates) {
 297                                 al_write_transaction(mdev, delegate);
 298                                 mdev->al_writ_cnt++;
 299                         }
 300
 301                         spin_lock_irq(&mdev->al_lock);
 302                         /* FIXME
 303                         if (err)
 304                                 we need an "lc_cancel" here;
 305                         */
 306                         lc_committed(mdev->act_log);
 307                         spin_unlock_irq(&mdev->al_lock);
 308                 }
 309                 lc_unlock(mdev->act_log);
 310                 wake_up(&mdev->al_wait);
 311         }
 312 }
 313
 314 void drbd_al_complete_io(struct drbd_conf *mdev, struct drbd_interval *i)
 315 {
 316         /* for bios crossing activity log extent boundaries,
 317          * we may need to activate two extents in one go */
 318         unsigned first = i->sector >> (AL_EXTENT_SHIFT-9);
 319         unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
 320         unsigned enr;
 321         struct lc_element *extent;
 322         unsigned long flags;
 323
 324         D_ASSERT(first <= last);
 325         spin_lock_irqsave(&mdev->al_lock, flags);
 326
 327         for (enr = first; enr <= last; enr++) {
 328                 extent = lc_find(mdev->act_log, enr);
 329                 if (!extent) {
 330                         dev_err(DEV, "al_complete_io() called on inactive extent %u\n", enr);
 331                         continue;
 332                 }
 333                 lc_put(mdev->act_log, extent);
 334         }
 335         spin_unlock_irqrestore(&mdev->al_lock, flags);
 336         wake_up(&mdev->al_wait);
 337 }
 338
 339 #if (PAGE_SHIFT + 3) < (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT)
 340 /* Currently BM_BLOCK_SHIFT, BM_EXT_SHIFT and AL_EXTENT_SHIFT
 341  * are still coupled, or assume too much about their relation.
 342  * Code below will not work if this is violated.
 343  * Will be cleaned up with some followup patch.
 344  */
 345 # error FIXME
 346 #endif
 347
 348 static unsigned int al_extent_to_bm_page(unsigned int al_enr)
 349 {
 350         return al_enr >>
 351                 /* bit to page */
 352                 ((PAGE_SHIFT + 3) -
 353                 /* al extent number to bit */
 354                  (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT));
 355 }
 356
 357 static unsigned int rs_extent_to_bm_page(unsigned int rs_enr)
 358 {
 359         return rs_enr >>
 360                 /* bit to page */
 361                 ((PAGE_SHIFT + 3) -
 362                 /* resync extent number to bit */
 363                  (BM_EXT_SHIFT - BM_BLOCK_SHIFT));
 364 }
 365
 366 static sector_t al_tr_number_to_on_disk_sector(struct drbd_conf *mdev)
 367 {
 368         const unsigned int stripes = mdev->ldev->md.al_stripes;
 369         const unsigned int stripe_size_4kB = mdev->ldev->md.al_stripe_size_4k;
 370
 371         /* transaction number, modulo on-disk ring buffer wrap around */
 372         unsigned int t = mdev->al_tr_number % (mdev->ldev->md.al_size_4k);
 373
 374         /* ... to aligned 4k on disk block */
 375         t = ((t % stripes) * stripe_size_4kB) + t/stripes;
 376
 377         /* ... to 512 byte sector in activity log */
 378         t *= 8;
 379
 380         /* ... plus offset to the on disk position */
 381         return mdev->ldev->md.md_offset + mdev->ldev->md.al_offset + t;
 382 }
 383
 384 static int
 385 _al_write_transaction(struct drbd_conf *mdev)
 386 {
 387         struct al_transaction_on_disk *buffer;
 388         struct lc_element *e;
 389         sector_t sector;
 390         int i, mx;
 391         unsigned extent_nr;
 392         unsigned crc = 0;
 393         int err = 0;
 394
 395         if (!get_ldev(mdev)) {
 396                 dev_err(DEV, "disk is %s, cannot start al transaction\n",
 397                         drbd_disk_str(mdev->state.disk));
 398                 return -EIO;
 399         }
 400
 401         /* The bitmap write may have failed, causing a state change. */
 402         if (mdev->state.disk < D_INCONSISTENT) {
 403                 dev_err(DEV,
 404                         "disk is %s, cannot write al transaction\n",
 405                         drbd_disk_str(mdev->state.disk));
 406                 put_ldev(mdev);
 407                 return -EIO;
 408         }
 409
 410         buffer = drbd_md_get_buffer(mdev); /* protects md_io_buffer, al_tr_cycle, ... */
 411         if (!buffer) {
 412                 dev_err(DEV, "disk failed while waiting for md_io buffer\n");
 413                 put_ldev(mdev);
 414                 return -ENODEV;
 415         }
 416
 417         memset(buffer, 0, sizeof(*buffer));
 418         buffer->magic = cpu_to_be32(DRBD_AL_MAGIC);
 419         buffer->tr_number = cpu_to_be32(mdev->al_tr_number);
 420
 421         i = 0;
 422
 423         /* Even though no one can start to change this list
 424          * once we set the LC_LOCKED -- from drbd_al_begin_io(),
 425          * lc_try_lock_for_transaction() --, someone may still
 426          * be in the process of changing it. */
 427         spin_lock_irq(&mdev->al_lock);
 428         list_for_each_entry(e, &mdev->act_log->to_be_changed, list) {
 429                 if (i == AL_UPDATES_PER_TRANSACTION) {
 430                         i++;
 431                         break;
 432                 }
 433                 buffer->update_slot_nr[i] = cpu_to_be16(e->lc_index);
 434                 buffer->update_extent_nr[i] = cpu_to_be32(e->lc_new_number);
 435                 if (e->lc_number != LC_FREE)
 436                         drbd_bm_mark_for_writeout(mdev,
 437                                         al_extent_to_bm_page(e->lc_number));
 438                 i++;
 439         }
 440         spin_unlock_irq(&mdev->al_lock);
 441         BUG_ON(i > AL_UPDATES_PER_TRANSACTION);
 442
 443         buffer->n_updates = cpu_to_be16(i);
 444         for ( ; i < AL_UPDATES_PER_TRANSACTION; i++) {
 445                 buffer->update_slot_nr[i] = cpu_to_be16(-1);
 446                 buffer->update_extent_nr[i] = cpu_to_be32(LC_FREE);
 447         }
 448
 449         buffer->context_size = cpu_to_be16(mdev->act_log->nr_elements);
 450         buffer->context_start_slot_nr = cpu_to_be16(mdev->al_tr_cycle);
 451
 452         mx = min_t(int, AL_CONTEXT_PER_TRANSACTION,
 453                    mdev->act_log->nr_elements - mdev->al_tr_cycle);
 454         for (i = 0; i < mx; i++) {
 455                 unsigned idx = mdev->al_tr_cycle + i;
 456                 extent_nr = lc_element_by_index(mdev->act_log, idx)->lc_number;
 457                 buffer->context[i] = cpu_to_be32(extent_nr);
 458         }
 459         for (; i < AL_CONTEXT_PER_TRANSACTION; i++)
 460                 buffer->context[i] = cpu_to_be32(LC_FREE);
 461
 462         mdev->al_tr_cycle += AL_CONTEXT_PER_TRANSACTION;
 463         if (mdev->al_tr_cycle >= mdev->act_log->nr_elements)
 464                 mdev->al_tr_cycle = 0;
 465
 466         sector = al_tr_number_to_on_disk_sector(mdev);
 467
 468         crc = crc32c(0, buffer, 4096);
 469         buffer->crc32c = cpu_to_be32(crc);
 470
 471         /* normal execution path goes through all three branches */
 472         if (drbd_bm_write_hinted(mdev))
 473                 err = -EIO;
 474                 /* drbd_chk_io_error done already */
 475         else if (drbd_md_sync_page_io(mdev, mdev->ldev, sector, WRITE)) {
 476                 err = -EIO;
 477                 drbd_chk_io_error(mdev, 1, DRBD_META_IO_ERROR);
 478         } else {
 479                 mdev->al_tr_number++;
 480         }
 481
 482         drbd_md_put_buffer(mdev);
 483         put_ldev(mdev);
 484
 485         return err;
 486 }
 487
 488
 489 static int w_al_write_transaction(struct drbd_work *w, int unused)
 490 {
 491         struct update_al_work *aw = container_of(w, struct update_al_work, w);
 492         struct drbd_conf *mdev = w->mdev;
 493         int err;
 494
 495         err = _al_write_transaction(mdev);
 496         aw->err = err;
 497         complete(&aw->event);
 498
 499         return err != -EIO ? err : 0;
 500 }
 501
 502 /* Calls from worker context (see w_restart_disk_io()) need to write the
 503    transaction directly. Others came through generic_make_request(),
 504    those need to delegate it to the worker. */
 505 static int al_write_transaction(struct drbd_conf *mdev, bool delegate)
 506 {
 507         if (delegate) {
 508                 struct update_al_work al_work;
 509                 init_completion(&al_work.event);
 510                 al_work.w.cb = w_al_write_transaction;
 511                 al_work.w.mdev = mdev;
 512                 drbd_queue_work_front(&mdev->tconn->sender_work, &al_work.w);
 513                 wait_for_completion(&al_work.event);
 514                 return al_work.err;
 515         } else
 516                 return _al_write_transaction(mdev);
 517 }
 518
 519 static int _try_lc_del(struct drbd_conf *mdev, struct lc_element *al_ext)
 520 {
 521         int rv;
 522
 523         spin_lock_irq(&mdev->al_lock);
 524         rv = (al_ext->refcnt == 0);
 525         if (likely(rv))
 526                 lc_del(mdev->act_log, al_ext);
 527         spin_unlock_irq(&mdev->al_lock);
 528
 529         return rv;
 530 }
 531
 532 /**
 533  * drbd_al_shrink() - Removes all active extents form the activity log
 534  * @mdev:       DRBD device.
 535  *
 536  * Removes all active extents form the activity log, waiting until
 537  * the reference count of each entry dropped to 0 first, of course.
 538  *
 539  * You need to lock mdev->act_log with lc_try_lock() / lc_unlock()
 540  */
 541 void drbd_al_shrink(struct drbd_conf *mdev)
 542 {
 543         struct lc_element *al_ext;
 544         int i;
 545
 546         D_ASSERT(test_bit(__LC_LOCKED, &mdev->act_log->flags));
 547
 548         for (i = 0; i < mdev->act_log->nr_elements; i++) {
 549                 al_ext = lc_element_by_index(mdev->act_log, i);
 550                 if (al_ext->lc_number == LC_FREE)
 551                         continue;
 552                 wait_event(mdev->al_wait, _try_lc_del(mdev, al_ext));
 553         }
 554
 555         wake_up(&mdev->al_wait);
 556 }
 557
 558 static int w_update_odbm(struct drbd_work *w, int unused)
 559 {
 560         struct update_odbm_work *udw = container_of(w, struct update_odbm_work, w);
 561         struct drbd_conf *mdev = w->mdev;
 562         struct sib_info sib = { .sib_reason = SIB_SYNC_PROGRESS, };
 563
 564         if (!get_ldev(mdev)) {
 565                 if (__ratelimit(&drbd_ratelimit_state))
 566                         dev_warn(DEV, "Can not update on disk bitmap, local IO disabled.\n");
 567                 kfree(udw);
 568                 return 0;
 569         }
 570
 571         drbd_bm_write_page(mdev, rs_extent_to_bm_page(udw->enr));
 572         put_ldev(mdev);
 573
 574         kfree(udw);
 575
 576         if (drbd_bm_total_weight(mdev) <= mdev->rs_failed) {
 577                 switch (mdev->state.conn) {
 578                 case C_SYNC_SOURCE:  case C_SYNC_TARGET:
 579                 case C_PAUSED_SYNC_S: case C_PAUSED_SYNC_T:
 580                         drbd_resync_finished(mdev);
 581                 default:
 582                         /* nothing to do */
 583                         break;
 584                 }
 585         }
 586         drbd_bcast_event(mdev, &sib);
 587
 588         return 0;
 589 }
 590
 591
 592 /* ATTENTION. The AL's extents are 4MB each, while the extents in the
 593  * resync LRU-cache are 16MB each.
 594  * The caller of this function has to hold an get_ldev() reference.
 595  *
 596  * TODO will be obsoleted once we have a caching lru of the on disk bitmap
 597  */
 598 static void drbd_try_clear_on_disk_bm(struct drbd_conf *mdev, sector_t sector,
 599                                       int count, int success)
 600 {
 601         struct lc_element *e;
 602         struct update_odbm_work *udw;
 603
 604         unsigned int enr;
 605
 606         D_ASSERT(atomic_read(&mdev->local_cnt));
 607
 608         /* I simply assume that a sector/size pair never crosses
 609          * a 16 MB extent border. (Currently this is true...) */
 610         enr = BM_SECT_TO_EXT(sector);
 611
 612         e = lc_get(mdev->resync, enr);
 613         if (e) {
 614                 struct bm_extent *ext = lc_entry(e, struct bm_extent, lce);
 615                 if (ext->lce.lc_number == enr) {
 616                         if (success)
 617                                 ext->rs_left -= count;
 618                         else
 619                                 ext->rs_failed += count;
 620                         if (ext->rs_left < ext->rs_failed) {
 621                                 dev_warn(DEV, "BAD! sector=%llus enr=%u rs_left=%d "
 622                                     "rs_failed=%d count=%d cstate=%s\n",
 623                                      (unsigned long long)sector,
 624                                      ext->lce.lc_number, ext->rs_left,
 625                                      ext->rs_failed, count,
 626                                      drbd_conn_str(mdev->state.conn));
 627
 628                                 /* We don't expect to be able to clear more bits
 629                                  * than have been set when we originally counted
 630                                  * the set bits to cache that value in ext->rs_left.
 631                                  * Whatever the reason (disconnect during resync,
 632                                  * delayed local completion of an application write),
 633                                  * try to fix it up by recounting here. */
 634                                 ext->rs_left = drbd_bm_e_weight(mdev, enr);
 635                         }
 636                 } else {
 637                         /* Normally this element should be in the cache,
 638                          * since drbd_rs_begin_io() pulled it already in.
 639                          *
 640                          * But maybe an application write finished, and we set
 641                          * something outside the resync lru_cache in sync.
 642                          */
 643                         int rs_left = drbd_bm_e_weight(mdev, enr);
 644                         if (ext->flags != 0) {
 645                                 dev_warn(DEV, "changing resync lce: %d[%u;%02lx]"
 646                                      " -> %d[%u;00]\n",
 647                                      ext->lce.lc_number, ext->rs_left,
 648                                      ext->flags, enr, rs_left);
 649                                 ext->flags = 0;
 650                         }
 651                         if (ext->rs_failed) {
 652                                 dev_warn(DEV, "Kicking resync_lru element enr=%u "
 653                                      "out with rs_failed=%d\n",
 654                                      ext->lce.lc_number, ext->rs_failed);
 655                         }
 656                         ext->rs_left = rs_left;
 657                         ext->rs_failed = success ? 0 : count;
 658                         /* we don't keep a persistent log of the resync lru,
 659                          * we can commit any change right away. */
 660                         lc_committed(mdev->resync);
 661                 }
 662                 lc_put(mdev->resync, &ext->lce);
 663                 /* no race, we are within the al_lock! */
 664
 665                 if (ext->rs_left == ext->rs_failed) {
 666                         ext->rs_failed = 0;
 667
 668                         udw = kmalloc(sizeof(*udw), GFP_ATOMIC);
 669                         if (udw) {
 670                                 udw->enr = ext->lce.lc_number;
 671                                 udw->w.cb = w_update_odbm;
 672                                 udw->w.mdev = mdev;
 673                                 drbd_queue_work_front(&mdev->tconn->sender_work, &udw->w);
 674                         } else {
 675                                 dev_warn(DEV, "Could not kmalloc an udw\n");
 676                         }
 677                 }
 678         } else {
 679                 dev_err(DEV, "lc_get() failed! locked=%d/%d flags=%lu\n",
 680                     mdev->resync_locked,
 681                     mdev->resync->nr_elements,
 682                     mdev->resync->flags);
 683         }
 684 }
 685
 686 void drbd_advance_rs_marks(struct drbd_conf *mdev, unsigned long still_to_go)
 687 {
 688         unsigned long now = jiffies;
 689         unsigned long last = mdev->rs_mark_time[mdev->rs_last_mark];
 690         int next = (mdev->rs_last_mark + 1) % DRBD_SYNC_MARKS;
 691         if (time_after_eq(now, last + DRBD_SYNC_MARK_STEP)) {
 692                 if (mdev->rs_mark_left[mdev->rs_last_mark] != still_to_go &&
 693                     mdev->state.conn != C_PAUSED_SYNC_T &&
 694                     mdev->state.conn != C_PAUSED_SYNC_S) {
 695                         mdev->rs_mark_time[next] = now;
 696                         mdev->rs_mark_left[next] = still_to_go;
 697                         mdev->rs_last_mark = next;
 698                 }
 699         }
 700 }
 701
 702 /* clear the bit corresponding to the piece of storage in question:
 703  * size byte of data starting from sector.  Only clear a bits of the affected
 704  * one ore more _aligned_ BM_BLOCK_SIZE blocks.
 705  *
 706  * called by worker on C_SYNC_TARGET and receiver on SyncSource.
 707  *
 708  */
 709 void __drbd_set_in_sync(struct drbd_conf *mdev, sector_t sector, int size,
 710                        const char *file, const unsigned int line)
 711 {
 712         /* Is called from worker and receiver context _only_ */
 713         unsigned long sbnr, ebnr, lbnr;
 714         unsigned long count = 0;
 715         sector_t esector, nr_sectors;
 716         int wake_up = 0;
 717         unsigned long flags;
 718
 719         if (size <= 0 || !IS_ALIGNED(size, 512) || size > DRBD_MAX_BIO_SIZE) {
 720                 dev_err(DEV, "drbd_set_in_sync: sector=%llus size=%d nonsense!\n",
 721                                 (unsigned long long)sector, size);
 722                 return;
 723         }
 724
 725         if (!get_ldev(mdev))
 726                 return; /* no disk, no metadata, no bitmap to clear bits in */
 727
 728         nr_sectors = drbd_get_capacity(mdev->this_bdev);
 729         esector = sector + (size >> 9) - 1;
 730
 731         if (!expect(sector < nr_sectors))
 732                 goto out;
 733         if (!expect(esector < nr_sectors))
 734                 esector = nr_sectors - 1;
 735
 736         lbnr = BM_SECT_TO_BIT(nr_sectors-1);
 737
 738         /* we clear it (in sync).
 739          * round up start sector, round down end sector.  we make sure we only
 740          * clear full, aligned, BM_BLOCK_SIZE (4K) blocks */
 741         if (unlikely(esector < BM_SECT_PER_BIT-1))
 742                 goto out;
 743         if (unlikely(esector == (nr_sectors-1)))
 744                 ebnr = lbnr;
 745         else
 746                 ebnr = BM_SECT_TO_BIT(esector - (BM_SECT_PER_BIT-1));
 747         sbnr = BM_SECT_TO_BIT(sector + BM_SECT_PER_BIT-1);
 748
 749         if (sbnr > ebnr)
 750                 goto out;
 751
 752         /*
 753          * ok, (capacity & 7) != 0 sometimes, but who cares...
 754          * we count rs_{total,left} in bits, not sectors.
 755          */
 756         count = drbd_bm_clear_bits(mdev, sbnr, ebnr);
 757         if (count) {
 758                 drbd_advance_rs_marks(mdev, drbd_bm_total_weight(mdev));
 759                 spin_lock_irqsave(&mdev->al_lock, flags);
 760                 drbd_try_clear_on_disk_bm(mdev, sector, count, true);
 761                 spin_unlock_irqrestore(&mdev->al_lock, flags);
 762
 763                 /* just wake_up unconditional now, various lc_chaged(),
 764                  * lc_put() in drbd_try_clear_on_disk_bm(). */
 765                 wake_up = 1;
 766         }
 767 out:
 768         put_ldev(mdev);
 769         if (wake_up)
 770                 wake_up(&mdev->al_wait);
 771 }
 772
 773 /*
 774  * this is intended to set one request worth of data out of sync.
 775  * affects at least 1 bit,
 776  * and at most 1+DRBD_MAX_BIO_SIZE/BM_BLOCK_SIZE bits.
 777  *
 778  * called by tl_clear and drbd_send_dblock (==drbd_make_request).
 779  * so this can be _any_ process.
 780  */
 781 int __drbd_set_out_of_sync(struct drbd_conf *mdev, sector_t sector, int size,
 782                             const char *file, const unsigned int line)
 783 {
 784         unsigned long sbnr, ebnr, flags;
 785         sector_t esector, nr_sectors;
 786         unsigned int enr, count = 0;
 787         struct lc_element *e;
 788
 789         /* this should be an empty REQ_FLUSH */
 790         if (size == 0)
 791                 return 0;
 792
 793         if (size < 0 || !IS_ALIGNED(size, 512) || size > DRBD_MAX_BIO_SIZE) {
 794                 dev_err(DEV, "sector: %llus, size: %d\n",
 795                         (unsigned long long)sector, size);
 796                 return 0;
 797         }
 798
 799         if (!get_ldev(mdev))
 800                 return 0; /* no disk, no metadata, no bitmap to set bits in */
 801
 802         nr_sectors = drbd_get_capacity(mdev->this_bdev);
 803         esector = sector + (size >> 9) - 1;
 804
 805         if (!expect(sector < nr_sectors))
 806                 goto out;
 807         if (!expect(esector < nr_sectors))
 808                 esector = nr_sectors - 1;
 809
 810         /* we set it out of sync,
 811          * we do not need to round anything here */
 812         sbnr = BM_SECT_TO_BIT(sector);
 813         ebnr = BM_SECT_TO_BIT(esector);
 814
 815         /* ok, (capacity & 7) != 0 sometimes, but who cares...
 816          * we count rs_{total,left} in bits, not sectors.  */
 817         spin_lock_irqsave(&mdev->al_lock, flags);
 818         count = drbd_bm_set_bits(mdev, sbnr, ebnr);
 819
 820         enr = BM_SECT_TO_EXT(sector);
 821         e = lc_find(mdev->resync, enr);
 822         if (e)
 823                 lc_entry(e, struct bm_extent, lce)->rs_left += count;
 824         spin_unlock_irqrestore(&mdev->al_lock, flags);
 825
 826 out:
 827         put_ldev(mdev);
 828
 829         return count;
 830 }
 831
 832 static
 833 struct bm_extent *_bme_get(struct drbd_conf *mdev, unsigned int enr)
 834 {
 835         struct lc_element *e;
 836         struct bm_extent *bm_ext;
 837         int wakeup = 0;
 838         unsigned long rs_flags;
 839
 840         spin_lock_irq(&mdev->al_lock);
 841         if (mdev->resync_locked > mdev->resync->nr_elements/2) {
 842                 spin_unlock_irq(&mdev->al_lock);
 843                 return NULL;
 844         }
 845         e = lc_get(mdev->resync, enr);
 846         bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
 847         if (bm_ext) {
 848                 if (bm_ext->lce.lc_number != enr) {
 849                         bm_ext->rs_left = drbd_bm_e_weight(mdev, enr);
 850                         bm_ext->rs_failed = 0;
 851                         lc_committed(mdev->resync);
 852                         wakeup = 1;
 853                 }
 854                 if (bm_ext->lce.refcnt == 1)
 855                         mdev->resync_locked++;
 856                 set_bit(BME_NO_WRITES, &bm_ext->flags);
 857         }
 858         rs_flags = mdev->resync->flags;
 859         spin_unlock_irq(&mdev->al_lock);
 860         if (wakeup)
 861                 wake_up(&mdev->al_wait);
 862
 863         if (!bm_ext) {
 864                 if (rs_flags & LC_STARVING)
 865                         dev_warn(DEV, "Have to wait for element"
 866                              " (resync LRU too small?)\n");
 867                 BUG_ON(rs_flags & LC_LOCKED);
 868         }
 869
 870         return bm_ext;
 871 }
 872
 873 static int _is_in_al(struct drbd_conf *mdev, unsigned int enr)
 874 {
 875         int rv;
 876
 877         spin_lock_irq(&mdev->al_lock);
 878         rv = lc_is_used(mdev->act_log, enr);
 879         spin_unlock_irq(&mdev->al_lock);
 880
 881         return rv;
 882 }
 883
 884 /**
 885  * drbd_rs_begin_io() - Gets an extent in the resync LRU cache and sets it to BME_LOCKED
 886  * @mdev:       DRBD device.
 887  * @sector:     The sector number.
 888  *
 889  * This functions sleeps on al_wait. Returns 0 on success, -EINTR if interrupted.
 890  */
 891 int drbd_rs_begin_io(struct drbd_conf *mdev, sector_t sector)
 892 {
 893         unsigned int enr = BM_SECT_TO_EXT(sector);
 894         struct bm_extent *bm_ext;
 895         int i, sig;
 896         int sa = 200; /* Step aside 200 times, then grab the extent and let app-IO wait.
 897                          200 times -> 20 seconds. */
 898
 899 retry:
 900         sig = wait_event_interruptible(mdev->al_wait,
 901                         (bm_ext = _bme_get(mdev, enr)));
 902         if (sig)
 903                 return -EINTR;
 904
 905         if (test_bit(BME_LOCKED, &bm_ext->flags))
 906                 return 0;
 907
 908         for (i = 0; i < AL_EXT_PER_BM_SECT; i++) {
 909                 sig = wait_event_interruptible(mdev->al_wait,
 910                                                !_is_in_al(mdev, enr * AL_EXT_PER_BM_SECT + i) ||
 911                                                test_bit(BME_PRIORITY, &bm_ext->flags));
 912
 913                 if (sig || (test_bit(BME_PRIORITY, &bm_ext->flags) && sa)) {
 914                         spin_lock_irq(&mdev->al_lock);
 915                         if (lc_put(mdev->resync, &bm_ext->lce) == 0) {
 916                                 bm_ext->flags = 0; /* clears BME_NO_WRITES and eventually BME_PRIORITY */
 917                                 mdev->resync_locked--;
 918                                 wake_up(&mdev->al_wait);
 919                         }
 920                         spin_unlock_irq(&mdev->al_lock);
 921                         if (sig)
 922                                 return -EINTR;
 923                         if (schedule_timeout_interruptible(HZ/10))
 924                                 return -EINTR;
 925                         if (sa && --sa == 0)
 926                                 dev_warn(DEV,"drbd_rs_begin_io() stepped aside for 20sec."
 927                                          "Resync stalled?\n");
 928                         goto retry;
 929                 }
 930         }
 931         set_bit(BME_LOCKED, &bm_ext->flags);
 932         return 0;
 933 }
 934
 935 /**
 936  * drbd_try_rs_begin_io() - Gets an extent in the resync LRU cache, does not sleep
 937  * @mdev:       DRBD device.
 938  * @sector:     The sector number.
 939  *
 940  * Gets an extent in the resync LRU cache, sets it to BME_NO_WRITES, then
 941  * tries to set it to BME_LOCKED. Returns 0 upon success, and -EAGAIN
 942  * if there is still application IO going on in this area.
 943  */
 944 int drbd_try_rs_begin_io(struct drbd_conf *mdev, sector_t sector)
 945 {
 946         unsigned int enr = BM_SECT_TO_EXT(sector);
 947         const unsigned int al_enr = enr*AL_EXT_PER_BM_SECT;
 948         struct lc_element *e;
 949         struct bm_extent *bm_ext;
 950         int i;
 951
 952         spin_lock_irq(&mdev->al_lock);
 953         if (mdev->resync_wenr != LC_FREE && mdev->resync_wenr != enr) {
 954                 /* in case you have very heavy scattered io, it may
 955                  * stall the syncer undefined if we give up the ref count
 956                  * when we try again and requeue.
 957                  *
 958                  * if we don't give up the refcount, but the next time
 959                  * we are scheduled this extent has been "synced" by new
 960                  * application writes, we'd miss the lc_put on the
 961                  * extent we keep the refcount on.
 962                  * so we remembered which extent we had to try again, and
 963                  * if the next requested one is something else, we do
 964                  * the lc_put here...
 965                  * we also have to wake_up
 966                  */
 967                 e = lc_find(mdev->resync, mdev->resync_wenr);
 968                 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
 969                 if (bm_ext) {
 970                         D_ASSERT(!test_bit(BME_LOCKED, &bm_ext->flags));
 971                         D_ASSERT(test_bit(BME_NO_WRITES, &bm_ext->flags));
 972                         clear_bit(BME_NO_WRITES, &bm_ext->flags);
 973                         mdev->resync_wenr = LC_FREE;
 974                         if (lc_put(mdev->resync, &bm_ext->lce) == 0)
 975                                 mdev->resync_locked--;
 976                         wake_up(&mdev->al_wait);
 977                 } else {
 978                         dev_alert(DEV, "LOGIC BUG\n");
 979                 }
 980         }
 981         /* TRY. */
 982         e = lc_try_get(mdev->resync, enr);
 983         bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
 984         if (bm_ext) {
 985                 if (test_bit(BME_LOCKED, &bm_ext->flags))
 986                         goto proceed;
 987                 if (!test_and_set_bit(BME_NO_WRITES, &bm_ext->flags)) {
 988                         mdev->resync_locked++;
 989                 } else {
 990                         /* we did set the BME_NO_WRITES,
 991                          * but then could not set BME_LOCKED,
 992                          * so we tried again.
 993                          * drop the extra reference. */
 994                         bm_ext->lce.refcnt--;
 995                         D_ASSERT(bm_ext->lce.refcnt > 0);
 996                 }
 997                 goto check_al;
 998         } else {
 999                 /* do we rather want to try later? */
1000                 if (mdev->resync_locked > mdev->resync->nr_elements-3)
1001                         goto try_again;
1002                 /* Do or do not. There is no try. -- Yoda */
1003                 e = lc_get(mdev->resync, enr);
1004                 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
1005                 if (!bm_ext) {
1006                         const unsigned long rs_flags = mdev->resync->flags;
1007                         if (rs_flags & LC_STARVING)
1008                                 dev_warn(DEV, "Have to wait for element"
1009                                      " (resync LRU too small?)\n");
1010                         BUG_ON(rs_flags & LC_LOCKED);
1011                         goto try_again;
1012                 }
1013                 if (bm_ext->lce.lc_number != enr) {
1014                         bm_ext->rs_left = drbd_bm_e_weight(mdev, enr);
1015                         bm_ext->rs_failed = 0;
1016                         lc_committed(mdev->resync);
1017                         wake_up(&mdev->al_wait);
1018                         D_ASSERT(test_bit(BME_LOCKED, &bm_ext->flags) == 0);
1019                 }
1020                 set_bit(BME_NO_WRITES, &bm_ext->flags);
1021                 D_ASSERT(bm_ext->lce.refcnt == 1);
1022                 mdev->resync_locked++;
1023                 goto check_al;
1024         }
1025 check_al:
1026         for (i = 0; i < AL_EXT_PER_BM_SECT; i++) {
1027                 if (lc_is_used(mdev->act_log, al_enr+i))
1028                         goto try_again;
1029         }
1030         set_bit(BME_LOCKED, &bm_ext->flags);
1031 proceed:
1032         mdev->resync_wenr = LC_FREE;
1033         spin_unlock_irq(&mdev->al_lock);
1034         return 0;
1035
1036 try_again:
1037         if (bm_ext)
1038                 mdev->resync_wenr = enr;
1039         spin_unlock_irq(&mdev->al_lock);
1040         return -EAGAIN;
1041 }
1042
1043 void drbd_rs_complete_io(struct drbd_conf *mdev, sector_t sector)
1044 {
1045         unsigned int enr = BM_SECT_TO_EXT(sector);
1046         struct lc_element *e;
1047         struct bm_extent *bm_ext;
1048         unsigned long flags;
1049
1050         spin_lock_irqsave(&mdev->al_lock, flags);
1051         e = lc_find(mdev->resync, enr);
1052         bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
1053         if (!bm_ext) {
1054                 spin_unlock_irqrestore(&mdev->al_lock, flags);
1055                 if (__ratelimit(&drbd_ratelimit_state))
1056                         dev_err(DEV, "drbd_rs_complete_io() called, but extent not found\n");
1057                 return;
1058         }
1059
1060         if (bm_ext->lce.refcnt == 0) {
1061                 spin_unlock_irqrestore(&mdev->al_lock, flags);
1062                 dev_err(DEV, "drbd_rs_complete_io(,%llu [=%u]) called, "
1063                     "but refcnt is 0!?\n",
1064                     (unsigned long long)sector, enr);
1065                 return;
1066         }
1067
1068         if (lc_put(mdev->resync, &bm_ext->lce) == 0) {
1069                 bm_ext->flags = 0; /* clear BME_LOCKED, BME_NO_WRITES and BME_PRIORITY */
1070                 mdev->resync_locked--;
1071                 wake_up(&mdev->al_wait);
1072         }
1073
1074         spin_unlock_irqrestore(&mdev->al_lock, flags);
1075 }
1076
1077 /**
1078  * drbd_rs_cancel_all() - Removes all extents from the resync LRU (even BME_LOCKED)
1079  * @mdev:       DRBD device.
1080  */
1081 void drbd_rs_cancel_all(struct drbd_conf *mdev)
1082 {
1083         spin_lock_irq(&mdev->al_lock);
1084
1085         if (get_ldev_if_state(mdev, D_FAILED)) { /* Makes sure ->resync is there. */
1086                 lc_reset(mdev->resync);
1087                 put_ldev(mdev);
1088         }
1089         mdev->resync_locked = 0;
1090         mdev->resync_wenr = LC_FREE;
1091         spin_unlock_irq(&mdev->al_lock);
1092         wake_up(&mdev->al_wait);
1093 }
1094
1095 /**
1096  * drbd_rs_del_all() - Gracefully remove all extents from the resync LRU
1097  * @mdev:       DRBD device.
1098  *
1099  * Returns 0 upon success, -EAGAIN if at least one reference count was
1100  * not zero.
1101  */
1102 int drbd_rs_del_all(struct drbd_conf *mdev)
1103 {
1104         struct lc_element *e;
1105         struct bm_extent *bm_ext;
1106         int i;
1107
1108         spin_lock_irq(&mdev->al_lock);
1109
1110         if (get_ldev_if_state(mdev, D_FAILED)) {
1111                 /* ok, ->resync is there. */
1112                 for (i = 0; i < mdev->resync->nr_elements; i++) {
1113                         e = lc_element_by_index(mdev->resync, i);
1114                         bm_ext = lc_entry(e, struct bm_extent, lce);
1115                         if (bm_ext->lce.lc_number == LC_FREE)
1116                                 continue;
1117                         if (bm_ext->lce.lc_number == mdev->resync_wenr) {
1118                                 dev_info(DEV, "dropping %u in drbd_rs_del_all, apparently"
1119                                      " got 'synced' by application io\n",
1120                                      mdev->resync_wenr);
1121                                 D_ASSERT(!test_bit(BME_LOCKED, &bm_ext->flags));
1122                                 D_ASSERT(test_bit(BME_NO_WRITES, &bm_ext->flags));
1123                                 clear_bit(BME_NO_WRITES, &bm_ext->flags);
1124                                 mdev->resync_wenr = LC_FREE;
1125                                 lc_put(mdev->resync, &bm_ext->lce);
1126                         }
1127                         if (bm_ext->lce.refcnt != 0) {
1128                                 dev_info(DEV, "Retrying drbd_rs_del_all() later. "
1129                                      "refcnt=%d\n", bm_ext->lce.refcnt);
1130                                 put_ldev(mdev);
1131                                 spin_unlock_irq(&mdev->al_lock);
1132                                 return -EAGAIN;
1133                         }
1134                         D_ASSERT(!test_bit(BME_LOCKED, &bm_ext->flags));
1135                         D_ASSERT(!test_bit(BME_NO_WRITES, &bm_ext->flags));
1136                         lc_del(mdev->resync, &bm_ext->lce);
1137                 }
1138                 D_ASSERT(mdev->resync->used == 0);
1139                 put_ldev(mdev);
1140         }
1141         spin_unlock_irq(&mdev->al_lock);
1142         wake_up(&mdev->al_wait);
1143
1144         return 0;
1145 }
1146
1147 /**
1148  * drbd_rs_failed_io() - Record information on a failure to resync the specified blocks
1149  * @mdev:       DRBD device.
1150  * @sector:     The sector number.
1151  * @size:       Size of failed IO operation, in byte.
1152  */
1153 void drbd_rs_failed_io(struct drbd_conf *mdev, sector_t sector, int size)
1154 {
1155         /* Is called from worker and receiver context _only_ */
1156         unsigned long sbnr, ebnr, lbnr;
1157         unsigned long count;
1158         sector_t esector, nr_sectors;
1159         int wake_up = 0;
1160
1161         if (size <= 0 || !IS_ALIGNED(size, 512) || size > DRBD_MAX_BIO_SIZE) {
1162                 dev_err(DEV, "drbd_rs_failed_io: sector=%llus size=%d nonsense!\n",
1163                                 (unsigned long long)sector, size);
1164                 return;
1165         }
1166         nr_sectors = drbd_get_capacity(mdev->this_bdev);
1167         esector = sector + (size >> 9) - 1;
1168
1169         if (!expect(sector < nr_sectors))
1170                 return;
1171         if (!expect(esector < nr_sectors))
1172                 esector = nr_sectors - 1;
1173
1174         lbnr = BM_SECT_TO_BIT(nr_sectors-1);
1175
1176         /*
1177          * round up start sector, round down end sector.  we make sure we only
1178          * handle full, aligned, BM_BLOCK_SIZE (4K) blocks */
1179         if (unlikely(esector < BM_SECT_PER_BIT-1))
1180                 return;
1181         if (unlikely(esector == (nr_sectors-1)))
1182                 ebnr = lbnr;
1183         else
1184                 ebnr = BM_SECT_TO_BIT(esector - (BM_SECT_PER_BIT-1));
1185         sbnr = BM_SECT_TO_BIT(sector + BM_SECT_PER_BIT-1);
1186
1187         if (sbnr > ebnr)
1188                 return;
1189
1190         /*
1191          * ok, (capacity & 7) != 0 sometimes, but who cares...
1192          * we count rs_{total,left} in bits, not sectors.
1193          */
1194         spin_lock_irq(&mdev->al_lock);
1195         count = drbd_bm_count_bits(mdev, sbnr, ebnr);
1196         if (count) {
1197                 mdev->rs_failed += count;
1198
1199                 if (get_ldev(mdev)) {
1200                         drbd_try_clear_on_disk_bm(mdev, sector, count, false);
1201                         put_ldev(mdev);
1202                 }
1203
1204                 /* just wake_up unconditional now, various lc_chaged(),
1205                  * lc_put() in drbd_try_clear_on_disk_bm(). */
1206                 wake_up = 1;
1207         }
1208         spin_unlock_irq(&mdev->al_lock);
1209         if (wake_up)
1210                 wake_up(&mdev->al_wait);
1211 }