drivers/block/drbd/drbd_state.c

   1 /*
   2    drbd_state.c
   3
   4    This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
   5
   6    Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
   7    Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
   8    Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
   9
  10    Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
  11    from Logicworks, Inc. for making SDP replication support possible.
  12
  13    drbd is free software; you can redistribute it and/or modify
  14    it under the terms of the GNU General Public License as published by
  15    the Free Software Foundation; either version 2, or (at your option)
  16    any later version.
  17
  18    drbd is distributed in the hope that it will be useful,
  19    but WITHOUT ANY WARRANTY; without even the implied warranty of
  20    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21    GNU General Public License for more details.
  22
  23    You should have received a copy of the GNU General Public License
  24    along with drbd; see the file COPYING.  If not, write to
  25    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
  26  */
  27
  28 #include <linux/drbd_limits.h>
  29 #include "drbd_int.h"
  30 #include "drbd_req.h"
  31
  32 struct after_state_chg_work {
  33         struct drbd_work w;
  34         union drbd_state os;
  35         union drbd_state ns;
  36         enum chg_state_flags flags;
  37         struct completion *done;
  38 };
  39
  40
  41 extern void _tl_restart(struct drbd_conf *mdev, enum drbd_req_event what);
  42 int drbd_send_state_req(struct drbd_conf *, union drbd_state, union drbd_state);
  43 static int w_after_state_ch(struct drbd_work *w, int unused);
  44 static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
  45                            union drbd_state ns, enum chg_state_flags flags);
  46 static void after_conn_state_ch(struct drbd_tconn *tconn, union drbd_state os,
  47                                 union drbd_state ns, enum chg_state_flags flags);
  48 static enum drbd_state_rv is_valid_state(struct drbd_conf *, union drbd_state);
  49 static enum drbd_state_rv is_valid_soft_transition(union drbd_state, union drbd_state);
  50 static union drbd_state sanitize_state(struct drbd_conf *mdev, union drbd_state os,
  51                                        union drbd_state ns, const char **warn_sync_abort);
  52
  53 /**
  54  * cl_wide_st_chg() - true if the state change is a cluster wide one
  55  * @mdev:       DRBD device.
  56  * @os:         old (current) state.
  57  * @ns:         new (wanted) state.
  58  */
  59 static int cl_wide_st_chg(struct drbd_conf *mdev,
  60                           union drbd_state os, union drbd_state ns)
  61 {
  62         return (os.conn >= C_CONNECTED && ns.conn >= C_CONNECTED &&
  63                  ((os.role != R_PRIMARY && ns.role == R_PRIMARY) ||
  64                   (os.conn != C_STARTING_SYNC_T && ns.conn == C_STARTING_SYNC_T) ||
  65                   (os.conn != C_STARTING_SYNC_S && ns.conn == C_STARTING_SYNC_S) ||
  66                   (os.disk != D_DISKLESS && ns.disk == D_DISKLESS))) ||
  67                 (os.conn >= C_CONNECTED && ns.conn == C_DISCONNECTING) ||
  68                 (os.conn == C_CONNECTED && ns.conn == C_VERIFY_S);
  69 }
  70
  71 enum drbd_state_rv
  72 drbd_change_state(struct drbd_conf *mdev, enum chg_state_flags f,
  73                   union drbd_state mask, union drbd_state val)
  74 {
  75         unsigned long flags;
  76         union drbd_state os, ns;
  77         enum drbd_state_rv rv;
  78
  79         spin_lock_irqsave(&mdev->tconn->req_lock, flags);
  80         os = mdev->state;
  81         ns.i = (os.i & ~mask.i) | val.i;
  82         rv = _drbd_set_state(mdev, ns, f, NULL);
  83         ns = mdev->state;
  84         spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);
  85
  86         return rv;
  87 }
  88
  89 /**
  90  * drbd_force_state() - Impose a change which happens outside our control on our state
  91  * @mdev:       DRBD device.
  92  * @mask:       mask of state bits to change.
  93  * @val:        value of new state bits.
  94  */
  95 void drbd_force_state(struct drbd_conf *mdev,
  96         union drbd_state mask, union drbd_state val)
  97 {
  98         drbd_change_state(mdev, CS_HARD, mask, val);
  99 }
 100
 101 static enum drbd_state_rv
 102 _req_st_cond(struct drbd_conf *mdev, union drbd_state mask,
 103              union drbd_state val)
 104 {
 105         union drbd_state os, ns;
 106         unsigned long flags;
 107         enum drbd_state_rv rv;
 108
 109         if (test_and_clear_bit(CL_ST_CHG_SUCCESS, &mdev->flags))
 110                 return SS_CW_SUCCESS;
 111
 112         if (test_and_clear_bit(CL_ST_CHG_FAIL, &mdev->flags))
 113                 return SS_CW_FAILED_BY_PEER;
 114
 115         rv = 0;
 116         spin_lock_irqsave(&mdev->tconn->req_lock, flags);
 117         os = mdev->state;
 118         ns.i = (os.i & ~mask.i) | val.i;
 119         ns = sanitize_state(mdev, os, ns, NULL);
 120
 121         if (!cl_wide_st_chg(mdev, os, ns))
 122                 rv = SS_CW_NO_NEED;
 123         if (!rv) {
 124                 rv = is_valid_state(mdev, ns);
 125                 if (rv == SS_SUCCESS) {
 126                         rv = is_valid_soft_transition(os, ns);
 127                         if (rv == SS_SUCCESS)
 128                                 rv = SS_UNKNOWN_ERROR; /* cont waiting, otherwise fail. */
 129                 }
 130         }
 131         spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);
 132
 133         return rv;
 134 }
 135
 136 /**
 137  * drbd_req_state() - Perform an eventually cluster wide state change
 138  * @mdev:       DRBD device.
 139  * @mask:       mask of state bits to change.
 140  * @val:        value of new state bits.
 141  * @f:          flags
 142  *
 143  * Should not be called directly, use drbd_request_state() or
 144  * _drbd_request_state().
 145  */
 146 static enum drbd_state_rv
 147 drbd_req_state(struct drbd_conf *mdev, union drbd_state mask,
 148                union drbd_state val, enum chg_state_flags f)
 149 {
 150         struct completion done;
 151         unsigned long flags;
 152         union drbd_state os, ns;
 153         enum drbd_state_rv rv;
 154
 155         init_completion(&done);
 156
 157         if (f & CS_SERIALIZE)
 158                 mutex_lock(&mdev->state_mutex);
 159
 160         spin_lock_irqsave(&mdev->tconn->req_lock, flags);
 161         os = mdev->state;
 162         ns.i = (os.i & ~mask.i) | val.i;
 163
 164         ns = sanitize_state(mdev, os, ns, NULL);
 165
 166         if (cl_wide_st_chg(mdev, os, ns)) {
 167                 rv = is_valid_state(mdev, ns);
 168                 if (rv == SS_SUCCESS)
 169                         rv = is_valid_soft_transition(os, ns);
 170                 spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);
 171
 172                 if (rv < SS_SUCCESS) {
 173                         if (f & CS_VERBOSE)
 174                                 print_st_err(mdev, os, ns, rv);
 175                         goto abort;
 176                 }
 177
 178                 drbd_state_lock(mdev);
 179                 if (!drbd_send_state_req(mdev, mask, val)) {
 180                         drbd_state_unlock(mdev);
 181                         rv = SS_CW_FAILED_BY_PEER;
 182                         if (f & CS_VERBOSE)
 183                                 print_st_err(mdev, os, ns, rv);
 184                         goto abort;
 185                 }
 186
 187                 wait_event(mdev->state_wait,
 188                         (rv = _req_st_cond(mdev, mask, val)));
 189
 190                 if (rv < SS_SUCCESS) {
 191                         drbd_state_unlock(mdev);
 192                         if (f & CS_VERBOSE)
 193                                 print_st_err(mdev, os, ns, rv);
 194                         goto abort;
 195                 }
 196                 spin_lock_irqsave(&mdev->tconn->req_lock, flags);
 197                 os = mdev->state;
 198                 ns.i = (os.i & ~mask.i) | val.i;
 199                 rv = _drbd_set_state(mdev, ns, f, &done);
 200                 drbd_state_unlock(mdev);
 201         } else {
 202                 rv = _drbd_set_state(mdev, ns, f, &done);
 203         }
 204
 205         spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);
 206
 207         if (f & CS_WAIT_COMPLETE && rv == SS_SUCCESS) {
 208                 D_ASSERT(current != mdev->tconn->worker.task);
 209                 wait_for_completion(&done);
 210         }
 211
 212 abort:
 213         if (f & CS_SERIALIZE)
 214                 mutex_unlock(&mdev->state_mutex);
 215
 216         return rv;
 217 }
 218
 219 /**
 220  * _drbd_request_state() - Request a state change (with flags)
 221  * @mdev:       DRBD device.
 222  * @mask:       mask of state bits to change.
 223  * @val:        value of new state bits.
 224  * @f:          flags
 225  *
 226  * Cousin of drbd_request_state(), useful with the CS_WAIT_COMPLETE
 227  * flag, or when logging of failed state change requests is not desired.
 228  */
 229 enum drbd_state_rv
 230 _drbd_request_state(struct drbd_conf *mdev, union drbd_state mask,
 231                     union drbd_state val, enum chg_state_flags f)
 232 {
 233         enum drbd_state_rv rv;
 234
 235         wait_event(mdev->state_wait,
 236                    (rv = drbd_req_state(mdev, mask, val, f)) != SS_IN_TRANSIENT_STATE);
 237
 238         return rv;
 239 }
 240
 241 static void print_st(struct drbd_conf *mdev, char *name, union drbd_state ns)
 242 {
 243         dev_err(DEV, " %s = { cs:%s ro:%s/%s ds:%s/%s %c%c%c%c%c%c }\n",
 244             name,
 245             drbd_conn_str(ns.conn),
 246             drbd_role_str(ns.role),
 247             drbd_role_str(ns.peer),
 248             drbd_disk_str(ns.disk),
 249             drbd_disk_str(ns.pdsk),
 250             is_susp(ns) ? 's' : 'r',
 251             ns.aftr_isp ? 'a' : '-',
 252             ns.peer_isp ? 'p' : '-',
 253             ns.user_isp ? 'u' : '-',
 254             ns.susp_fen ? 'F' : '-',
 255             ns.susp_nod ? 'N' : '-'
 256             );
 257 }
 258
 259 void print_st_err(struct drbd_conf *mdev, union drbd_state os,
 260                   union drbd_state ns, enum drbd_state_rv err)
 261 {
 262         if (err == SS_IN_TRANSIENT_STATE)
 263                 return;
 264         dev_err(DEV, "State change failed: %s\n", drbd_set_st_err_str(err));
 265         print_st(mdev, " state", os);
 266         print_st(mdev, "wanted", ns);
 267 }
 268
 269
 270 /**
 271  * is_valid_state() - Returns an SS_ error code if ns is not valid
 272  * @mdev:       DRBD device.
 273  * @ns:         State to consider.
 274  */
 275 static enum drbd_state_rv
 276 is_valid_state(struct drbd_conf *mdev, union drbd_state ns)
 277 {
 278         /* See drbd_state_sw_errors in drbd_strings.c */
 279
 280         enum drbd_fencing_p fp;
 281         enum drbd_state_rv rv = SS_SUCCESS;
 282
 283         fp = FP_DONT_CARE;
 284         if (get_ldev(mdev)) {
 285                 fp = mdev->ldev->dc.fencing;
 286                 put_ldev(mdev);
 287         }
 288
 289         if (get_net_conf(mdev->tconn)) {
 290                 if (!mdev->tconn->net_conf->two_primaries &&
 291                     ns.role == R_PRIMARY && ns.peer == R_PRIMARY)
 292                         rv = SS_TWO_PRIMARIES;
 293                 put_net_conf(mdev->tconn);
 294         }
 295
 296         if (rv <= 0)
 297                 /* already found a reason to abort */;
 298         else if (ns.role == R_SECONDARY && mdev->open_cnt)
 299                 rv = SS_DEVICE_IN_USE;
 300
 301         else if (ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.disk < D_UP_TO_DATE)
 302                 rv = SS_NO_UP_TO_DATE_DISK;
 303
 304         else if (fp >= FP_RESOURCE &&
 305                  ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.pdsk >= D_UNKNOWN)
 306                 rv = SS_PRIMARY_NOP;
 307
 308         else if (ns.role == R_PRIMARY && ns.disk <= D_INCONSISTENT && ns.pdsk <= D_INCONSISTENT)
 309                 rv = SS_NO_UP_TO_DATE_DISK;
 310
 311         else if (ns.conn > C_CONNECTED && ns.disk < D_INCONSISTENT)
 312                 rv = SS_NO_LOCAL_DISK;
 313
 314         else if (ns.conn > C_CONNECTED && ns.pdsk < D_INCONSISTENT)
 315                 rv = SS_NO_REMOTE_DISK;
 316
 317         else if (ns.conn > C_CONNECTED && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE)
 318                 rv = SS_NO_UP_TO_DATE_DISK;
 319
 320         else if ((ns.conn == C_CONNECTED ||
 321                   ns.conn == C_WF_BITMAP_S ||
 322                   ns.conn == C_SYNC_SOURCE ||
 323                   ns.conn == C_PAUSED_SYNC_S) &&
 324                   ns.disk == D_OUTDATED)
 325                 rv = SS_CONNECTED_OUTDATES;
 326
 327         else if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) &&
 328                  (mdev->sync_conf.verify_alg[0] == 0))
 329                 rv = SS_NO_VERIFY_ALG;
 330
 331         else if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) &&
 332                   mdev->tconn->agreed_pro_version < 88)
 333                 rv = SS_NOT_SUPPORTED;
 334
 335         else if (ns.conn >= C_CONNECTED && ns.pdsk == D_UNKNOWN)
 336                 rv = SS_CONNECTED_OUTDATES;
 337
 338         return rv;
 339 }
 340
 341 /**
 342  * is_valid_soft_transition() - Returns an SS_ error code if the state transition is not possible
 343  * @mdev:       DRBD device.
 344  * @ns:         new state.
 345  * @os:         old state.
 346  */
 347 static enum drbd_state_rv
 348 is_valid_soft_transition(union drbd_state os, union drbd_state ns)
 349 {
 350         enum drbd_state_rv rv = SS_SUCCESS;
 351
 352         if ((ns.conn == C_STARTING_SYNC_T || ns.conn == C_STARTING_SYNC_S) &&
 353             os.conn > C_CONNECTED)
 354                 rv = SS_RESYNC_RUNNING;
 355
 356         if (ns.conn == C_DISCONNECTING && os.conn == C_STANDALONE)
 357                 rv = SS_ALREADY_STANDALONE;
 358
 359         if (ns.disk > D_ATTACHING && os.disk == D_DISKLESS)
 360                 rv = SS_IS_DISKLESS;
 361
 362         if (ns.conn == C_WF_CONNECTION && os.conn < C_UNCONNECTED)
 363                 rv = SS_NO_NET_CONFIG;
 364
 365         if (ns.disk == D_OUTDATED && os.disk < D_OUTDATED && os.disk != D_ATTACHING)
 366                 rv = SS_LOWER_THAN_OUTDATED;
 367
 368         if (ns.conn == C_DISCONNECTING && os.conn == C_UNCONNECTED)
 369                 rv = SS_IN_TRANSIENT_STATE;
 370
 371         if (ns.conn == os.conn && ns.conn == C_WF_REPORT_PARAMS)
 372                 rv = SS_IN_TRANSIENT_STATE;
 373
 374         if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && os.conn < C_CONNECTED)
 375                 rv = SS_NEED_CONNECTION;
 376
 377         if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) &&
 378             ns.conn != os.conn && os.conn > C_CONNECTED)
 379                 rv = SS_RESYNC_RUNNING;
 380
 381         if ((ns.conn == C_STARTING_SYNC_S || ns.conn == C_STARTING_SYNC_T) &&
 382             os.conn < C_CONNECTED)
 383                 rv = SS_NEED_CONNECTION;
 384
 385         if ((ns.conn == C_SYNC_TARGET || ns.conn == C_SYNC_SOURCE)
 386             && os.conn < C_WF_REPORT_PARAMS)
 387                 rv = SS_NEED_CONNECTION; /* No NetworkFailure -> SyncTarget etc... */
 388
 389         return rv;
 390 }
 391
 392 /**
 393  * sanitize_state() - Resolves implicitly necessary additional changes to a state transition
 394  * @mdev:       DRBD device.
 395  * @os:         old state.
 396  * @ns:         new state.
 397  * @warn_sync_abort:
 398  *
 399  * When we loose connection, we have to set the state of the peers disk (pdsk)
 400  * to D_UNKNOWN. This rule and many more along those lines are in this function.
 401  */
 402 static union drbd_state sanitize_state(struct drbd_conf *mdev, union drbd_state os,
 403                                        union drbd_state ns, const char **warn_sync_abort)
 404 {
 405         enum drbd_fencing_p fp;
 406         enum drbd_disk_state disk_min, disk_max, pdsk_min, pdsk_max;
 407
 408         fp = FP_DONT_CARE;
 409         if (get_ldev(mdev)) {
 410                 fp = mdev->ldev->dc.fencing;
 411                 put_ldev(mdev);
 412         }
 413
 414         /* Disallow Network errors to configure a device's network part */
 415         if ((ns.conn >= C_TIMEOUT && ns.conn <= C_TEAR_DOWN) &&
 416             os.conn <= C_DISCONNECTING)
 417                 ns.conn = os.conn;
 418
 419         /* After a network error (+C_TEAR_DOWN) only C_UNCONNECTED or C_DISCONNECTING can follow.
 420          * If you try to go into some Sync* state, that shall fail (elsewhere). */
 421         if (os.conn >= C_TIMEOUT && os.conn <= C_TEAR_DOWN &&
 422             ns.conn != C_UNCONNECTED && ns.conn != C_DISCONNECTING && ns.conn <= C_TEAR_DOWN)
 423                 ns.conn = os.conn;
 424
 425         /* we cannot fail (again) if we already detached */
 426         if (ns.disk == D_FAILED && os.disk == D_DISKLESS)
 427                 ns.disk = D_DISKLESS;
 428
 429         /* if we are only D_ATTACHING yet,
 430          * we can (and should) go directly to D_DISKLESS. */
 431         if (ns.disk == D_FAILED && os.disk == D_ATTACHING)
 432                 ns.disk = D_DISKLESS;
 433
 434         /* After C_DISCONNECTING only C_STANDALONE may follow */
 435         if (os.conn == C_DISCONNECTING && ns.conn != C_STANDALONE)
 436                 ns.conn = os.conn;
 437
 438         if (ns.conn < C_CONNECTED) {
 439                 ns.peer_isp = 0;
 440                 ns.peer = R_UNKNOWN;
 441                 if (ns.pdsk > D_UNKNOWN || ns.pdsk < D_INCONSISTENT)
 442                         ns.pdsk = D_UNKNOWN;
 443         }
 444
 445         /* Clear the aftr_isp when becoming unconfigured */
 446         if (ns.conn == C_STANDALONE && ns.disk == D_DISKLESS && ns.role == R_SECONDARY)
 447                 ns.aftr_isp = 0;
 448
 449         /* Abort resync if a disk fails/detaches */
 450         if (os.conn > C_CONNECTED && ns.conn > C_CONNECTED &&
 451             (ns.disk <= D_FAILED || ns.pdsk <= D_FAILED)) {
 452                 if (warn_sync_abort)
 453                         *warn_sync_abort =
 454                                 os.conn == C_VERIFY_S || os.conn == C_VERIFY_T ?
 455                                 "Online-verify" : "Resync";
 456                 ns.conn = C_CONNECTED;
 457         }
 458
 459         /* Connection breaks down before we finished "Negotiating" */
 460         if (ns.conn < C_CONNECTED && ns.disk == D_NEGOTIATING &&
 461             get_ldev_if_state(mdev, D_NEGOTIATING)) {
 462                 if (mdev->ed_uuid == mdev->ldev->md.uuid[UI_CURRENT]) {
 463                         ns.disk = mdev->new_state_tmp.disk;
 464                         ns.pdsk = mdev->new_state_tmp.pdsk;
 465                 } else {
 466                         dev_alert(DEV, "Connection lost while negotiating, no data!\n");
 467                         ns.disk = D_DISKLESS;
 468                         ns.pdsk = D_UNKNOWN;
 469                 }
 470                 put_ldev(mdev);
 471         }
 472
 473         /* D_CONSISTENT and D_OUTDATED vanish when we get connected */
 474         if (ns.conn >= C_CONNECTED && ns.conn < C_AHEAD) {
 475                 if (ns.disk == D_CONSISTENT || ns.disk == D_OUTDATED)
 476                         ns.disk = D_UP_TO_DATE;
 477                 if (ns.pdsk == D_CONSISTENT || ns.pdsk == D_OUTDATED)
 478                         ns.pdsk = D_UP_TO_DATE;
 479         }
 480
 481         /* Implications of the connection stat on the disk states */
 482         disk_min = D_DISKLESS;
 483         disk_max = D_UP_TO_DATE;
 484         pdsk_min = D_INCONSISTENT;
 485         pdsk_max = D_UNKNOWN;
 486         switch ((enum drbd_conns)ns.conn) {
 487         case C_WF_BITMAP_T:
 488         case C_PAUSED_SYNC_T:
 489         case C_STARTING_SYNC_T:
 490         case C_WF_SYNC_UUID:
 491         case C_BEHIND:
 492                 disk_min = D_INCONSISTENT;
 493                 disk_max = D_OUTDATED;
 494                 pdsk_min = D_UP_TO_DATE;
 495                 pdsk_max = D_UP_TO_DATE;
 496                 break;
 497         case C_VERIFY_S:
 498         case C_VERIFY_T:
 499                 disk_min = D_UP_TO_DATE;
 500                 disk_max = D_UP_TO_DATE;
 501                 pdsk_min = D_UP_TO_DATE;
 502                 pdsk_max = D_UP_TO_DATE;
 503                 break;
 504         case C_CONNECTED:
 505                 disk_min = D_DISKLESS;
 506                 disk_max = D_UP_TO_DATE;
 507                 pdsk_min = D_DISKLESS;
 508                 pdsk_max = D_UP_TO_DATE;
 509                 break;
 510         case C_WF_BITMAP_S:
 511         case C_PAUSED_SYNC_S:
 512         case C_STARTING_SYNC_S:
 513         case C_AHEAD:
 514                 disk_min = D_UP_TO_DATE;
 515                 disk_max = D_UP_TO_DATE;
 516                 pdsk_min = D_INCONSISTENT;
 517                 pdsk_max = D_CONSISTENT; /* D_OUTDATED would be nice. But explicit outdate necessary*/
 518                 break;
 519         case C_SYNC_TARGET:
 520                 disk_min = D_INCONSISTENT;
 521                 disk_max = D_INCONSISTENT;
 522                 pdsk_min = D_UP_TO_DATE;
 523                 pdsk_max = D_UP_TO_DATE;
 524                 break;
 525         case C_SYNC_SOURCE:
 526                 disk_min = D_UP_TO_DATE;
 527                 disk_max = D_UP_TO_DATE;
 528                 pdsk_min = D_INCONSISTENT;
 529                 pdsk_max = D_INCONSISTENT;
 530                 break;
 531         case C_STANDALONE:
 532         case C_DISCONNECTING:
 533         case C_UNCONNECTED:
 534         case C_TIMEOUT:
 535         case C_BROKEN_PIPE:
 536         case C_NETWORK_FAILURE:
 537         case C_PROTOCOL_ERROR:
 538         case C_TEAR_DOWN:
 539         case C_WF_CONNECTION:
 540         case C_WF_REPORT_PARAMS:
 541         case C_MASK:
 542                 break;
 543         }
 544         if (ns.disk > disk_max)
 545                 ns.disk = disk_max;
 546
 547         if (ns.disk < disk_min) {
 548                 dev_warn(DEV, "Implicitly set disk from %s to %s\n",
 549                          drbd_disk_str(ns.disk), drbd_disk_str(disk_min));
 550                 ns.disk = disk_min;
 551         }
 552         if (ns.pdsk > pdsk_max)
 553                 ns.pdsk = pdsk_max;
 554
 555         if (ns.pdsk < pdsk_min) {
 556                 dev_warn(DEV, "Implicitly set pdsk from %s to %s\n",
 557                          drbd_disk_str(ns.pdsk), drbd_disk_str(pdsk_min));
 558                 ns.pdsk = pdsk_min;
 559         }
 560
 561         if (fp == FP_STONITH &&
 562             (ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.pdsk > D_OUTDATED) &&
 563             !(os.role == R_PRIMARY && os.conn < C_CONNECTED && os.pdsk > D_OUTDATED))
 564                 ns.susp_fen = 1; /* Suspend IO while fence-peer handler runs (peer lost) */
 565
 566         if (mdev->sync_conf.on_no_data == OND_SUSPEND_IO &&
 567             (ns.role == R_PRIMARY && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE) &&
 568             !(os.role == R_PRIMARY && os.disk < D_UP_TO_DATE && os.pdsk < D_UP_TO_DATE))
 569                 ns.susp_nod = 1; /* Suspend IO while no data available (no accessible data available) */
 570
 571         if (ns.aftr_isp || ns.peer_isp || ns.user_isp) {
 572                 if (ns.conn == C_SYNC_SOURCE)
 573                         ns.conn = C_PAUSED_SYNC_S;
 574                 if (ns.conn == C_SYNC_TARGET)
 575                         ns.conn = C_PAUSED_SYNC_T;
 576         } else {
 577                 if (ns.conn == C_PAUSED_SYNC_S)
 578                         ns.conn = C_SYNC_SOURCE;
 579                 if (ns.conn == C_PAUSED_SYNC_T)
 580                         ns.conn = C_SYNC_TARGET;
 581         }
 582
 583         return ns;
 584 }
 585
 586 void drbd_resume_al(struct drbd_conf *mdev)
 587 {
 588         if (test_and_clear_bit(AL_SUSPENDED, &mdev->flags))
 589                 dev_info(DEV, "Resumed AL updates\n");
 590 }
 591
 592 /* helper for __drbd_set_state */
 593 static void set_ov_position(struct drbd_conf *mdev, enum drbd_conns cs)
 594 {
 595         if (mdev->tconn->agreed_pro_version < 90)
 596                 mdev->ov_start_sector = 0;
 597         mdev->rs_total = drbd_bm_bits(mdev);
 598         mdev->ov_position = 0;
 599         if (cs == C_VERIFY_T) {
 600                 /* starting online verify from an arbitrary position
 601                  * does not fit well into the existing protocol.
 602                  * on C_VERIFY_T, we initialize ov_left and friends
 603                  * implicitly in receive_DataRequest once the
 604                  * first P_OV_REQUEST is received */
 605                 mdev->ov_start_sector = ~(sector_t)0;
 606         } else {
 607                 unsigned long bit = BM_SECT_TO_BIT(mdev->ov_start_sector);
 608                 if (bit >= mdev->rs_total) {
 609                         mdev->ov_start_sector =
 610                                 BM_BIT_TO_SECT(mdev->rs_total - 1);
 611                         mdev->rs_total = 1;
 612                 } else
 613                         mdev->rs_total -= bit;
 614                 mdev->ov_position = mdev->ov_start_sector;
 615         }
 616         mdev->ov_left = mdev->rs_total;
 617 }
 618
 619 /**
 620  * __drbd_set_state() - Set a new DRBD state
 621  * @mdev:       DRBD device.
 622  * @ns:         new state.
 623  * @flags:      Flags
 624  * @done:       Optional completion, that will get completed after the after_state_ch() finished
 625  *
 626  * Caller needs to hold req_lock, and global_state_lock. Do not call directly.
 627  */
 628 enum drbd_state_rv
 629 __drbd_set_state(struct drbd_conf *mdev, union drbd_state ns,
 630                  enum chg_state_flags flags, struct completion *done)
 631 {
 632         union drbd_state os;
 633         enum drbd_state_rv rv = SS_SUCCESS;
 634         const char *warn_sync_abort = NULL;
 635         struct after_state_chg_work *ascw;
 636
 637         os = mdev->state;
 638
 639         ns = sanitize_state(mdev, os, ns, &warn_sync_abort);
 640
 641         if (ns.i == os.i)
 642                 return SS_NOTHING_TO_DO;
 643
 644         if (!(flags & CS_HARD)) {
 645                 /*  pre-state-change checks ; only look at ns  */
 646                 /* See drbd_state_sw_errors in drbd_strings.c */
 647
 648                 rv = is_valid_state(mdev, ns);
 649                 if (rv < SS_SUCCESS) {
 650                         /* If the old state was illegal as well, then let
 651                            this happen...*/
 652
 653                         if (is_valid_state(mdev, os) == rv)
 654                                 rv = is_valid_soft_transition(os, ns);
 655                 } else
 656                         rv = is_valid_soft_transition(os, ns);
 657         }
 658
 659         if (rv < SS_SUCCESS) {
 660                 if (flags & CS_VERBOSE)
 661                         print_st_err(mdev, os, ns, rv);
 662                 return rv;
 663         }
 664
 665         if (warn_sync_abort)
 666                 dev_warn(DEV, "%s aborted.\n", warn_sync_abort);
 667
 668         {
 669         char *pbp, pb[300];
 670         pbp = pb;
 671         *pbp = 0;
 672         if (ns.role != os.role)
 673                 pbp += sprintf(pbp, "role( %s -> %s ) ",
 674                                drbd_role_str(os.role),
 675                                drbd_role_str(ns.role));
 676         if (ns.peer != os.peer)
 677                 pbp += sprintf(pbp, "peer( %s -> %s ) ",
 678                                drbd_role_str(os.peer),
 679                                drbd_role_str(ns.peer));
 680         if (ns.conn != os.conn)
 681                 pbp += sprintf(pbp, "conn( %s -> %s ) ",
 682                                drbd_conn_str(os.conn),
 683                                drbd_conn_str(ns.conn));
 684         if (ns.disk != os.disk)
 685                 pbp += sprintf(pbp, "disk( %s -> %s ) ",
 686                                drbd_disk_str(os.disk),
 687                                drbd_disk_str(ns.disk));
 688         if (ns.pdsk != os.pdsk)
 689                 pbp += sprintf(pbp, "pdsk( %s -> %s ) ",
 690                                drbd_disk_str(os.pdsk),
 691                                drbd_disk_str(ns.pdsk));
 692         if (is_susp(ns) != is_susp(os))
 693                 pbp += sprintf(pbp, "susp( %d -> %d ) ",
 694                                is_susp(os),
 695                                is_susp(ns));
 696         if (ns.aftr_isp != os.aftr_isp)
 697                 pbp += sprintf(pbp, "aftr_isp( %d -> %d ) ",
 698                                os.aftr_isp,
 699                                ns.aftr_isp);
 700         if (ns.peer_isp != os.peer_isp)
 701                 pbp += sprintf(pbp, "peer_isp( %d -> %d ) ",
 702                                os.peer_isp,
 703                                ns.peer_isp);
 704         if (ns.user_isp != os.user_isp)
 705                 pbp += sprintf(pbp, "user_isp( %d -> %d ) ",
 706                                os.user_isp,
 707                                ns.user_isp);
 708         dev_info(DEV, "%s\n", pb);
 709         }
 710
 711         /* solve the race between becoming unconfigured,
 712          * worker doing the cleanup, and
 713          * admin reconfiguring us:
 714          * on (re)configure, first set CONFIG_PENDING,
 715          * then wait for a potentially exiting worker,
 716          * start the worker, and schedule one no_op.
 717          * then proceed with configuration.
 718          */
 719         if (ns.disk == D_DISKLESS &&
 720             ns.conn == C_STANDALONE &&
 721             ns.role == R_SECONDARY &&
 722             !test_and_set_bit(CONFIG_PENDING, &mdev->flags))
 723                 set_bit(DEVICE_DYING, &mdev->flags);
 724
 725         /* if we are going -> D_FAILED or D_DISKLESS, grab one extra reference
 726          * on the ldev here, to be sure the transition -> D_DISKLESS resp.
 727          * drbd_ldev_destroy() won't happen before our corresponding
 728          * after_state_ch works run, where we put_ldev again. */
 729         if ((os.disk != D_FAILED && ns.disk == D_FAILED) ||
 730             (os.disk != D_DISKLESS && ns.disk == D_DISKLESS))
 731                 atomic_inc(&mdev->local_cnt);
 732
 733         mdev->state = ns;
 734
 735         if (os.disk == D_ATTACHING && ns.disk >= D_NEGOTIATING)
 736                 drbd_print_uuids(mdev, "attached to UUIDs");
 737
 738         wake_up(&mdev->misc_wait);
 739         wake_up(&mdev->state_wait);
 740
 741         /* aborted verify run. log the last position */
 742         if ((os.conn == C_VERIFY_S || os.conn == C_VERIFY_T) &&
 743             ns.conn < C_CONNECTED) {
 744                 mdev->ov_start_sector =
 745                         BM_BIT_TO_SECT(drbd_bm_bits(mdev) - mdev->ov_left);
 746                 dev_info(DEV, "Online Verify reached sector %llu\n",
 747                         (unsigned long long)mdev->ov_start_sector);
 748         }
 749
 750         if ((os.conn == C_PAUSED_SYNC_T || os.conn == C_PAUSED_SYNC_S) &&
 751             (ns.conn == C_SYNC_TARGET  || ns.conn == C_SYNC_SOURCE)) {
 752                 dev_info(DEV, "Syncer continues.\n");
 753                 mdev->rs_paused += (long)jiffies
 754                                   -(long)mdev->rs_mark_time[mdev->rs_last_mark];
 755                 if (ns.conn == C_SYNC_TARGET)
 756                         mod_timer(&mdev->resync_timer, jiffies);
 757         }
 758
 759         if ((os.conn == C_SYNC_TARGET  || os.conn == C_SYNC_SOURCE) &&
 760             (ns.conn == C_PAUSED_SYNC_T || ns.conn == C_PAUSED_SYNC_S)) {
 761                 dev_info(DEV, "Resync suspended\n");
 762                 mdev->rs_mark_time[mdev->rs_last_mark] = jiffies;
 763         }
 764
 765         if (os.conn == C_CONNECTED &&
 766             (ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T)) {
 767                 unsigned long now = jiffies;
 768                 int i;
 769
 770                 set_ov_position(mdev, ns.conn);
 771                 mdev->rs_start = now;
 772                 mdev->rs_last_events = 0;
 773                 mdev->rs_last_sect_ev = 0;
 774                 mdev->ov_last_oos_size = 0;
 775                 mdev->ov_last_oos_start = 0;
 776
 777                 for (i = 0; i < DRBD_SYNC_MARKS; i++) {
 778                         mdev->rs_mark_left[i] = mdev->ov_left;
 779                         mdev->rs_mark_time[i] = now;
 780                 }
 781
 782                 drbd_rs_controller_reset(mdev);
 783
 784                 if (ns.conn == C_VERIFY_S) {
 785                         dev_info(DEV, "Starting Online Verify from sector %llu\n",
 786                                         (unsigned long long)mdev->ov_position);
 787                         mod_timer(&mdev->resync_timer, jiffies);
 788                 }
 789         }
 790
 791         if (get_ldev(mdev)) {
 792                 u32 mdf = mdev->ldev->md.flags & ~(MDF_CONSISTENT|MDF_PRIMARY_IND|
 793                                                  MDF_CONNECTED_IND|MDF_WAS_UP_TO_DATE|
 794                                                  MDF_PEER_OUT_DATED|MDF_CRASHED_PRIMARY);
 795
 796                 if (test_bit(CRASHED_PRIMARY, &mdev->flags))
 797                         mdf |= MDF_CRASHED_PRIMARY;
 798                 if (mdev->state.role == R_PRIMARY ||
 799                     (mdev->state.pdsk < D_INCONSISTENT && mdev->state.peer == R_PRIMARY))
 800                         mdf |= MDF_PRIMARY_IND;
 801                 if (mdev->state.conn > C_WF_REPORT_PARAMS)
 802                         mdf |= MDF_CONNECTED_IND;
 803                 if (mdev->state.disk > D_INCONSISTENT)
 804                         mdf |= MDF_CONSISTENT;
 805                 if (mdev->state.disk > D_OUTDATED)
 806                         mdf |= MDF_WAS_UP_TO_DATE;
 807                 if (mdev->state.pdsk <= D_OUTDATED && mdev->state.pdsk >= D_INCONSISTENT)
 808                         mdf |= MDF_PEER_OUT_DATED;
 809                 if (mdf != mdev->ldev->md.flags) {
 810                         mdev->ldev->md.flags = mdf;
 811                         drbd_md_mark_dirty(mdev);
 812                 }
 813                 if (os.disk < D_CONSISTENT && ns.disk >= D_CONSISTENT)
 814                         drbd_set_ed_uuid(mdev, mdev->ldev->md.uuid[UI_CURRENT]);
 815                 put_ldev(mdev);
 816         }
 817
 818         /* Peer was forced D_UP_TO_DATE & R_PRIMARY, consider to resync */
 819         if (os.disk == D_INCONSISTENT && os.pdsk == D_INCONSISTENT &&
 820             os.peer == R_SECONDARY && ns.peer == R_PRIMARY)
 821                 set_bit(CONSIDER_RESYNC, &mdev->flags);
 822
 823         /* Receiver should clean up itself */
 824         if (os.conn != C_DISCONNECTING && ns.conn == C_DISCONNECTING)
 825                 drbd_thread_stop_nowait(&mdev->tconn->receiver);
 826
 827         /* Now the receiver finished cleaning up itself, it should die */
 828         if (os.conn != C_STANDALONE && ns.conn == C_STANDALONE)
 829                 drbd_thread_stop_nowait(&mdev->tconn->receiver);
 830
 831         /* Upon network failure, we need to restart the receiver. */
 832         if (os.conn > C_TEAR_DOWN &&
 833             ns.conn <= C_TEAR_DOWN && ns.conn >= C_TIMEOUT)
 834                 drbd_thread_restart_nowait(&mdev->tconn->receiver);
 835
 836         /* Resume AL writing if we get a connection */
 837         if (os.conn < C_CONNECTED && ns.conn >= C_CONNECTED)
 838                 drbd_resume_al(mdev);
 839
 840         ascw = kmalloc(sizeof(*ascw), GFP_ATOMIC);
 841         if (ascw) {
 842                 ascw->os = os;
 843                 ascw->ns = ns;
 844                 ascw->flags = flags;
 845                 ascw->w.cb = w_after_state_ch;
 846                 ascw->w.mdev = mdev;
 847                 ascw->done = done;
 848                 drbd_queue_work(&mdev->tconn->data.work, &ascw->w);
 849         } else {
 850                 dev_warn(DEV, "Could not kmalloc an ascw\n");
 851         }
 852
 853         return rv;
 854 }
 855
 856 static int w_after_state_ch(struct drbd_work *w, int unused)
 857 {
 858         struct after_state_chg_work *ascw =
 859                 container_of(w, struct after_state_chg_work, w);
 860         struct drbd_conf *mdev = w->mdev;
 861
 862         after_state_ch(mdev, ascw->os, ascw->ns, ascw->flags);
 863         if (ascw->flags & CS_WAIT_COMPLETE) {
 864                 D_ASSERT(ascw->done != NULL);
 865                 complete(ascw->done);
 866         }
 867         kfree(ascw);
 868
 869         return 1;
 870 }
 871
 872 static void abw_start_sync(struct drbd_conf *mdev, int rv)
 873 {
 874         if (rv) {
 875                 dev_err(DEV, "Writing the bitmap failed not starting resync.\n");
 876                 _drbd_request_state(mdev, NS(conn, C_CONNECTED), CS_VERBOSE);
 877                 return;
 878         }
 879
 880         switch (mdev->state.conn) {
 881         case C_STARTING_SYNC_T:
 882                 _drbd_request_state(mdev, NS(conn, C_WF_SYNC_UUID), CS_VERBOSE);
 883                 break;
 884         case C_STARTING_SYNC_S:
 885                 drbd_start_resync(mdev, C_SYNC_SOURCE);
 886                 break;
 887         }
 888 }
 889
 890 int drbd_bitmap_io_from_worker(struct drbd_conf *mdev,
 891                 int (*io_fn)(struct drbd_conf *),
 892                 char *why, enum bm_flag flags)
 893 {
 894         int rv;
 895
 896         D_ASSERT(current == mdev->tconn->worker.task);
 897
 898         /* open coded non-blocking drbd_suspend_io(mdev); */
 899         set_bit(SUSPEND_IO, &mdev->flags);
 900
 901         drbd_bm_lock(mdev, why, flags);
 902         rv = io_fn(mdev);
 903         drbd_bm_unlock(mdev);
 904
 905         drbd_resume_io(mdev);
 906
 907         return rv;
 908 }
 909
 910 /**
 911  * after_state_ch() - Perform after state change actions that may sleep
 912  * @mdev:       DRBD device.
 913  * @os:         old state.
 914  * @ns:         new state.
 915  * @flags:      Flags
 916  */
 917 static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
 918                            union drbd_state ns, enum chg_state_flags flags)
 919 {
 920         enum drbd_fencing_p fp;
 921         enum drbd_req_event what = NOTHING;
 922         union drbd_state nsm = (union drbd_state){ .i = -1 };
 923
 924         if (os.conn != C_CONNECTED && ns.conn == C_CONNECTED) {
 925                 clear_bit(CRASHED_PRIMARY, &mdev->flags);
 926                 if (mdev->p_uuid)
 927                         mdev->p_uuid[UI_FLAGS] &= ~((u64)2);
 928         }
 929
 930         fp = FP_DONT_CARE;
 931         if (get_ldev(mdev)) {
 932                 fp = mdev->ldev->dc.fencing;
 933                 put_ldev(mdev);
 934         }
 935
 936         /* Inform userspace about the change... */
 937         drbd_bcast_state(mdev, ns);
 938
 939         if (!(os.role == R_PRIMARY && os.disk < D_UP_TO_DATE && os.pdsk < D_UP_TO_DATE) &&
 940             (ns.role == R_PRIMARY && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE))
 941                 drbd_khelper(mdev, "pri-on-incon-degr");
 942
 943         /* Here we have the actions that are performed after a
 944            state change. This function might sleep */
 945
 946         nsm.i = -1;
 947         if (ns.susp_nod) {
 948                 if (os.conn < C_CONNECTED && ns.conn >= C_CONNECTED)
 949                         what = RESEND;
 950
 951                 if (os.disk == D_ATTACHING && ns.disk > D_ATTACHING)
 952                         what = RESTART_FROZEN_DISK_IO;
 953
 954                 if (what != NOTHING)
 955                         nsm.susp_nod = 0;
 956         }
 957
 958         if (ns.susp_fen) {
 959                 /* case1: The outdate peer handler is successful: */
 960                 if (os.pdsk > D_OUTDATED  && ns.pdsk <= D_OUTDATED) {
 961                         tl_clear(mdev);
 962                         if (test_bit(NEW_CUR_UUID, &mdev->flags)) {
 963                                 drbd_uuid_new_current(mdev);
 964                                 clear_bit(NEW_CUR_UUID, &mdev->flags);
 965                         }
 966                         spin_lock_irq(&mdev->tconn->req_lock);
 967                         _drbd_set_state(_NS(mdev, susp_fen, 0), CS_VERBOSE, NULL);
 968                         spin_unlock_irq(&mdev->tconn->req_lock);
 969                 }
 970                 /* case2: The connection was established again: */
 971                 if (os.conn < C_CONNECTED && ns.conn >= C_CONNECTED) {
 972                         clear_bit(NEW_CUR_UUID, &mdev->flags);
 973                         what = RESEND;
 974                         nsm.susp_fen = 0;
 975                 }
 976         }
 977
 978         if (what != NOTHING) {
 979                 spin_lock_irq(&mdev->tconn->req_lock);
 980                 _tl_restart(mdev, what);
 981                 nsm.i &= mdev->state.i;
 982                 _drbd_set_state(mdev, nsm, CS_VERBOSE, NULL);
 983                 spin_unlock_irq(&mdev->tconn->req_lock);
 984         }
 985
 986         /* Became sync source.  With protocol >= 96, we still need to send out
 987          * the sync uuid now. Need to do that before any drbd_send_state, or
 988          * the other side may go "paused sync" before receiving the sync uuids,
 989          * which is unexpected. */
 990         if ((os.conn != C_SYNC_SOURCE && os.conn != C_PAUSED_SYNC_S) &&
 991             (ns.conn == C_SYNC_SOURCE || ns.conn == C_PAUSED_SYNC_S) &&
 992             mdev->tconn->agreed_pro_version >= 96 && get_ldev(mdev)) {
 993                 drbd_gen_and_send_sync_uuid(mdev);
 994                 put_ldev(mdev);
 995         }
 996
 997         /* Do not change the order of the if above and the two below... */
 998         if (os.pdsk == D_DISKLESS && ns.pdsk > D_DISKLESS) {      /* attach on the peer */
 999                 drbd_send_uuids(mdev);
1000                 drbd_send_state(mdev);
1001         }
1002         /* No point in queuing send_bitmap if we don't have a connection
1003          * anymore, so check also the _current_ state, not only the new state
1004          * at the time this work was queued. */
1005         if (os.conn != C_WF_BITMAP_S && ns.conn == C_WF_BITMAP_S &&
1006             mdev->state.conn == C_WF_BITMAP_S)
1007                 drbd_queue_bitmap_io(mdev, &drbd_send_bitmap, NULL,
1008                                 "send_bitmap (WFBitMapS)",
1009                                 BM_LOCKED_TEST_ALLOWED);
1010
1011         /* Lost contact to peer's copy of the data */
1012         if ((os.pdsk >= D_INCONSISTENT &&
1013              os.pdsk != D_UNKNOWN &&
1014              os.pdsk != D_OUTDATED)
1015         &&  (ns.pdsk < D_INCONSISTENT ||
1016              ns.pdsk == D_UNKNOWN ||
1017              ns.pdsk == D_OUTDATED)) {
1018                 if (get_ldev(mdev)) {
1019                         if ((ns.role == R_PRIMARY || ns.peer == R_PRIMARY) &&
1020                             mdev->ldev->md.uuid[UI_BITMAP] == 0 && ns.disk >= D_UP_TO_DATE) {
1021                                 if (is_susp(mdev->state)) {
1022                                         set_bit(NEW_CUR_UUID, &mdev->flags);
1023                                 } else {
1024                                         drbd_uuid_new_current(mdev);
1025                                         drbd_send_uuids(mdev);
1026                                 }
1027                         }
1028                         put_ldev(mdev);
1029                 }
1030         }
1031
1032         if (ns.pdsk < D_INCONSISTENT && get_ldev(mdev)) {
1033                 if (ns.peer == R_PRIMARY && mdev->ldev->md.uuid[UI_BITMAP] == 0) {
1034                         drbd_uuid_new_current(mdev);
1035                         drbd_send_uuids(mdev);
1036                 }
1037
1038                 /* D_DISKLESS Peer becomes secondary */
1039                 if (os.peer == R_PRIMARY && ns.peer == R_SECONDARY)
1040                         /* We may still be Primary ourselves.
1041                          * No harm done if the bitmap still changes,
1042                          * redirtied pages will follow later. */
1043                         drbd_bitmap_io_from_worker(mdev, &drbd_bm_write,
1044                                 "demote diskless peer", BM_LOCKED_SET_ALLOWED);
1045                 put_ldev(mdev);
1046         }
1047
1048         /* Write out all changed bits on demote.
1049          * Though, no need to da that just yet
1050          * if there is a resync going on still */
1051         if (os.role == R_PRIMARY && ns.role == R_SECONDARY &&
1052                 mdev->state.conn <= C_CONNECTED && get_ldev(mdev)) {
1053                 /* No changes to the bitmap expected this time, so assert that,
1054                  * even though no harm was done if it did change. */
1055                 drbd_bitmap_io_from_worker(mdev, &drbd_bm_write,
1056                                 "demote", BM_LOCKED_TEST_ALLOWED);
1057                 put_ldev(mdev);
1058         }
1059
1060         /* Last part of the attaching process ... */
1061         if (ns.conn >= C_CONNECTED &&
1062             os.disk == D_ATTACHING && ns.disk == D_NEGOTIATING) {
1063                 drbd_send_sizes(mdev, 0, 0);  /* to start sync... */
1064                 drbd_send_uuids(mdev);
1065                 drbd_send_state(mdev);
1066         }
1067
1068         /* We want to pause/continue resync, tell peer. */
1069         if (ns.conn >= C_CONNECTED &&
1070              ((os.aftr_isp != ns.aftr_isp) ||
1071               (os.user_isp != ns.user_isp)))
1072                 drbd_send_state(mdev);
1073
1074         /* In case one of the isp bits got set, suspend other devices. */
1075         if ((!os.aftr_isp && !os.peer_isp && !os.user_isp) &&
1076             (ns.aftr_isp || ns.peer_isp || ns.user_isp))
1077                 suspend_other_sg(mdev);
1078
1079         /* Make sure the peer gets informed about eventual state
1080            changes (ISP bits) while we were in WFReportParams. */
1081         if (os.conn == C_WF_REPORT_PARAMS && ns.conn >= C_CONNECTED)
1082                 drbd_send_state(mdev);
1083
1084         if (os.conn != C_AHEAD && ns.conn == C_AHEAD)
1085                 drbd_send_state(mdev);
1086
1087         /* We are in the progress to start a full sync... */
1088         if ((os.conn != C_STARTING_SYNC_T && ns.conn == C_STARTING_SYNC_T) ||
1089             (os.conn != C_STARTING_SYNC_S && ns.conn == C_STARTING_SYNC_S))
1090                 /* no other bitmap changes expected during this phase */
1091                 drbd_queue_bitmap_io(mdev,
1092                         &drbd_bmio_set_n_write, &abw_start_sync,
1093                         "set_n_write from StartingSync", BM_LOCKED_TEST_ALLOWED);
1094
1095         /* We are invalidating our self... */
1096         if (os.conn < C_CONNECTED && ns.conn < C_CONNECTED &&
1097             os.disk > D_INCONSISTENT && ns.disk == D_INCONSISTENT)
1098                 /* other bitmap operation expected during this phase */
1099                 drbd_queue_bitmap_io(mdev, &drbd_bmio_set_n_write, NULL,
1100                         "set_n_write from invalidate", BM_LOCKED_MASK);
1101
1102         /* first half of local IO error, failure to attach,
1103          * or administrative detach */
1104         if (os.disk != D_FAILED && ns.disk == D_FAILED) {
1105                 enum drbd_io_error_p eh;
1106                 int was_io_error;
1107                 /* corresponding get_ldev was in __drbd_set_state, to serialize
1108                  * our cleanup here with the transition to D_DISKLESS,
1109                  * so it is safe to dreference ldev here. */
1110                 eh = mdev->ldev->dc.on_io_error;
1111                 was_io_error = test_and_clear_bit(WAS_IO_ERROR, &mdev->flags);
1112
1113                 /* current state still has to be D_FAILED,
1114                  * there is only one way out: to D_DISKLESS,
1115                  * and that may only happen after our put_ldev below. */
1116                 if (mdev->state.disk != D_FAILED)
1117                         dev_err(DEV,
1118                                 "ASSERT FAILED: disk is %s during detach\n",
1119                                 drbd_disk_str(mdev->state.disk));
1120
1121                 if (drbd_send_state(mdev))
1122                         dev_warn(DEV, "Notified peer that I am detaching my disk\n");
1123                 else
1124                         dev_err(DEV, "Sending state for detaching disk failed\n");
1125
1126                 drbd_rs_cancel_all(mdev);
1127
1128                 /* In case we want to get something to stable storage still,
1129                  * this may be the last chance.
1130                  * Following put_ldev may transition to D_DISKLESS. */
1131                 drbd_md_sync(mdev);
1132                 put_ldev(mdev);
1133
1134                 if (was_io_error && eh == EP_CALL_HELPER)
1135                         drbd_khelper(mdev, "local-io-error");
1136         }
1137
1138         /* second half of local IO error, failure to attach,
1139          * or administrative detach,
1140          * after local_cnt references have reached zero again */
1141         if (os.disk != D_DISKLESS && ns.disk == D_DISKLESS) {
1142                 /* We must still be diskless,
1143                  * re-attach has to be serialized with this! */
1144                 if (mdev->state.disk != D_DISKLESS)
1145                         dev_err(DEV,
1146                                 "ASSERT FAILED: disk is %s while going diskless\n",
1147                                 drbd_disk_str(mdev->state.disk));
1148
1149                 mdev->rs_total = 0;
1150                 mdev->rs_failed = 0;
1151                 atomic_set(&mdev->rs_pending_cnt, 0);
1152
1153                 if (drbd_send_state(mdev))
1154                         dev_warn(DEV, "Notified peer that I'm now diskless.\n");
1155                 /* corresponding get_ldev in __drbd_set_state
1156                  * this may finally trigger drbd_ldev_destroy. */
1157                 put_ldev(mdev);
1158         }
1159
1160         /* Notify peer that I had a local IO error, and did not detached.. */
1161         if (os.disk == D_UP_TO_DATE && ns.disk == D_INCONSISTENT)
1162                 drbd_send_state(mdev);
1163
1164         /* Disks got bigger while they were detached */
1165         if (ns.disk > D_NEGOTIATING && ns.pdsk > D_NEGOTIATING &&
1166             test_and_clear_bit(RESYNC_AFTER_NEG, &mdev->flags)) {
1167                 if (ns.conn == C_CONNECTED)
1168                         resync_after_online_grow(mdev);
1169         }
1170
1171         /* A resync finished or aborted, wake paused devices... */
1172         if ((os.conn > C_CONNECTED && ns.conn <= C_CONNECTED) ||
1173             (os.peer_isp && !ns.peer_isp) ||
1174             (os.user_isp && !ns.user_isp))
1175                 resume_next_sg(mdev);
1176
1177         /* sync target done with resync.  Explicitly notify peer, even though
1178          * it should (at least for non-empty resyncs) already know itself. */
1179         if (os.disk < D_UP_TO_DATE && os.conn >= C_SYNC_SOURCE && ns.conn == C_CONNECTED)
1180                 drbd_send_state(mdev);
1181
1182         /* This triggers bitmap writeout of potentially still unwritten pages
1183          * if the resync finished cleanly, or aborted because of peer disk
1184          * failure, or because of connection loss.
1185          * For resync aborted because of local disk failure, we cannot do
1186          * any bitmap writeout anymore.
1187          * No harm done if some bits change during this phase.
1188          */
1189         if (os.conn > C_CONNECTED && ns.conn <= C_CONNECTED && get_ldev(mdev)) {
1190                 drbd_queue_bitmap_io(mdev, &drbd_bm_write, NULL,
1191                         "write from resync_finished", BM_LOCKED_SET_ALLOWED);
1192                 put_ldev(mdev);
1193         }
1194
1195         if (ns.disk == D_DISKLESS &&
1196             ns.conn == C_STANDALONE &&
1197             ns.role == R_SECONDARY) {
1198                 if (os.aftr_isp != ns.aftr_isp)
1199                         resume_next_sg(mdev);
1200         }
1201
1202         after_conn_state_ch(mdev->tconn, os, ns, flags);
1203         drbd_md_sync(mdev);
1204 }
1205
1206 static void after_conn_state_ch(struct drbd_tconn *tconn, union drbd_state os,
1207                                 union drbd_state ns, enum chg_state_flags flags)
1208 {
1209         /* Upon network configuration, we need to start the receiver */
1210         if (os.conn == C_STANDALONE && ns.conn == C_UNCONNECTED)
1211                 drbd_thread_start(&tconn->receiver);
1212
1213         if (ns.disk == D_DISKLESS &&
1214             ns.conn == C_STANDALONE &&
1215             ns.role == R_SECONDARY) {
1216                 /* if (test_bit(DEVICE_DYING, &mdev->flags)) TODO: DEVICE_DYING functionality */
1217                 drbd_thread_stop_nowait(&tconn->worker);
1218         }
1219 }