projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drbd: Converted drbdd() from mdev to tconn
[linux-2.6-block.git] / drivers / block / drbd / drbd_main.c
CommitLineData
b411b363
PR		 1/*
2 drbd.c
3
4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
5
6 Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
7 Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
8 Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
9
10 Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
11 from Logicworks, Inc. for making SDP replication support possible.
12
13 drbd is free software; you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation; either version 2, or (at your option)
16 any later version.
17
18 drbd is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
22
23 You should have received a copy of the GNU General Public License
24 along with drbd; see the file COPYING. If not, write to
25 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
26
27 */
28
b411b363 29#include <linux/module.h>
b411b363
PR		 30#include <linux/drbd.h>
31#include <asm/uaccess.h>
32#include <asm/types.h>
33#include <net/sock.h>
34#include <linux/ctype.h>
2a48fc0a 35#include <linux/mutex.h>
b411b363
PR		 36#include <linux/fs.h>
37#include <linux/file.h>
38#include <linux/proc_fs.h>
39#include <linux/init.h>
40#include <linux/mm.h>
41#include <linux/memcontrol.h>
42#include <linux/mm_inline.h>
43#include <linux/slab.h>
44#include <linux/random.h>
45#include <linux/reboot.h>
46#include <linux/notifier.h>
47#include <linux/kthread.h>
48
49#define __KERNEL_SYSCALLS__
50#include <linux/unistd.h>
51#include <linux/vmalloc.h>
52
53#include <linux/drbd_limits.h>
54#include "drbd_int.h"
b411b363
PR		 55#include "drbd_req.h" /* only for _req_mod in tl_release and tl_clear */
56
57#include "drbd_vli.h"
58
2a48fc0a 59static DEFINE_MUTEX(drbd_main_mutex);
b411b363
PR		 60int drbdd_init(struct drbd_thread *);
61int drbd_worker(struct drbd_thread *);
62int drbd_asender(struct drbd_thread *);
63
64int drbd_init(void);
65static int drbd_open(struct block_device *bdev, fmode_t mode);
66static int drbd_release(struct gendisk *gd, fmode_t mode);
b411b363
PR		 67static int w_md_sync(struct drbd_conf *mdev, struct drbd_work *w, int unused);
68static void md_sync_timer_fn(unsigned long data);
69static int w_bitmap_io(struct drbd_conf *mdev, struct drbd_work *w, int unused);
e9e6f3ec 70static int w_go_diskless(struct drbd_conf *mdev, struct drbd_work *w, int unused);
b411b363 71
b411b363
PR		 72MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
73 "Lars Ellenberg <lars@linbit.com>");
74MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION);
75MODULE_VERSION(REL_VERSION);
76MODULE_LICENSE("GPL");
2b8a90b5
PR		 77MODULE_PARM_DESC(minor_count, "Maximum number of drbd devices ("
78 __stringify(DRBD_MINOR_COUNT_MIN) "-" __stringify(DRBD_MINOR_COUNT_MAX) ")");
b411b363
PR		 79MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR);
80
81#include <linux/moduleparam.h>
82/* allow_open_on_secondary */
83MODULE_PARM_DESC(allow_oos, "DONT USE!");
84/* thanks to these macros, if compiled into the kernel (not-module),
85 * this becomes the boot parameter drbd.minor_count */
86module_param(minor_count, uint, 0444);
87module_param(disable_sendpage, bool, 0644);
88module_param(allow_oos, bool, 0);
89module_param(cn_idx, uint, 0444);
90module_param(proc_details, int, 0644);
91
92#ifdef CONFIG_DRBD_FAULT_INJECTION
93int enable_faults;
94int fault_rate;
95static int fault_count;
96int fault_devs;
97/* bitmap of enabled faults */
98module_param(enable_faults, int, 0664);
99/* fault rate % value - applies to all enabled faults */
100module_param(fault_rate, int, 0664);
101/* count of faults inserted */
102module_param(fault_count, int, 0664);
103/* bitmap of devices to insert faults on */
104module_param(fault_devs, int, 0644);
105#endif
106
107/* module parameter, defined */
2b8a90b5 108unsigned int minor_count = DRBD_MINOR_COUNT_DEF;
b411b363
PR		 109int disable_sendpage;
110int allow_oos;
111unsigned int cn_idx = CN_IDX_DRBD;
112int proc_details; /* Detail level in proc drbd*/
113
114/* Module parameter for setting the user mode helper program
115 * to run. Default is /sbin/drbdadm */
116char usermode_helper[80] = "/sbin/drbdadm";
117
118module_param_string(usermode_helper, usermode_helper, sizeof(usermode_helper), 0644);
119
120/* in 2.6.x, our device mapping and config info contains our virtual gendisks
121 * as member "struct gendisk *vdisk;"
122 */
123struct drbd_conf **minor_table;
2111438b 124struct list_head drbd_tconns; /* list of struct drbd_tconn */
b411b363
PR		 125
126struct kmem_cache *drbd_request_cache;
6c852bec 127struct kmem_cache *drbd_ee_cache; /* peer requests */
b411b363
PR		 128struct kmem_cache *drbd_bm_ext_cache; /* bitmap extents */
129struct kmem_cache *drbd_al_ext_cache; /* activity log extents */
130mempool_t *drbd_request_mempool;
131mempool_t *drbd_ee_mempool;
132
133/* I do not use a standard mempool, because:
134 1) I want to hand out the pre-allocated objects first.
135 2) I want to be able to interrupt sleeping allocation with a signal.
136 Note: This is a single linked list, the next pointer is the private
137 member of struct page.
138 */
139struct page *drbd_pp_pool;
140spinlock_t drbd_pp_lock;
141int drbd_pp_vacant;
142wait_queue_head_t drbd_pp_wait;
143
144DEFINE_RATELIMIT_STATE(drbd_ratelimit_state, 5 * HZ, 5);
145
7d4e9d09 146static const struct block_device_operations drbd_ops = {
b411b363
PR		 147 .owner = THIS_MODULE,
148 .open = drbd_open,
149 .release = drbd_release,
150};
151
152#define ARRY_SIZE(A) (sizeof(A)/sizeof(A[0]))
153
154#ifdef __CHECKER__
155/* When checking with sparse, and this is an inline function, sparse will
156 give tons of false positives. When this is a real functions sparse works.
157 */
158int _get_ldev_if_state(struct drbd_conf *mdev, enum drbd_disk_state mins)
159{
160 int io_allowed;
161
162 atomic_inc(&mdev->local_cnt);
163 io_allowed = (mdev->state.disk >= mins);
164 if (!io_allowed) {
165 if (atomic_dec_and_test(&mdev->local_cnt))
166 wake_up(&mdev->misc_wait);
167 }
168 return io_allowed;
169}
170
171#endif
172
173/**
174 * DOC: The transfer log
175 *
176 * The transfer log is a single linked list of &struct drbd_tl_epoch objects.
87eeee41 177 * mdev->tconn->newest_tle points to the head, mdev->tconn->oldest_tle points to the tail
b411b363
PR		 178 * of the list. There is always at least one &struct drbd_tl_epoch object.
179 *
180 * Each &struct drbd_tl_epoch has a circular double linked list of requests
181 * attached.
182 */
183static int tl_init(struct drbd_conf *mdev)
184{
185 struct drbd_tl_epoch *b;
186
187 /* during device minor initialization, we may well use GFP_KERNEL */
188 b = kmalloc(sizeof(struct drbd_tl_epoch), GFP_KERNEL);
189 if (!b)
190 return 0;
191 INIT_LIST_HEAD(&b->requests);
192 INIT_LIST_HEAD(&b->w.list);
193 b->next = NULL;
194 b->br_number = 4711;
7e602c0a 195 b->n_writes = 0;
b411b363
PR		 196 b->w.cb = NULL; /* if this is != NULL, we need to dec_ap_pending in tl_clear */
197
87eeee41
PR		 198 mdev->tconn->oldest_tle = b;
199 mdev->tconn->newest_tle = b;
200 INIT_LIST_HEAD(&mdev->tconn->out_of_sequence_requests);
b411b363 201
b411b363
PR		 202 return 1;
203}
204
205static void tl_cleanup(struct drbd_conf *mdev)
206{
87eeee41
PR		 207 D_ASSERT(mdev->tconn->oldest_tle == mdev->tconn->newest_tle);
208 D_ASSERT(list_empty(&mdev->tconn->out_of_sequence_requests));
209 kfree(mdev->tconn->oldest_tle);
210 mdev->tconn->oldest_tle = NULL;
211 kfree(mdev->tconn->unused_spare_tle);
212 mdev->tconn->unused_spare_tle = NULL;
d628769b
AG		 213}
214
b411b363
PR		 215/**
216 * _tl_add_barrier() - Adds a barrier to the transfer log
217 * @mdev: DRBD device.
218 * @new: Barrier to be added before the current head of the TL.
219 *
220 * The caller must hold the req_lock.
221 */
222void _tl_add_barrier(struct drbd_conf *mdev, struct drbd_tl_epoch *new)
223{
224 struct drbd_tl_epoch *newest_before;
225
226 INIT_LIST_HEAD(&new->requests);
227 INIT_LIST_HEAD(&new->w.list);
228 new->w.cb = NULL; /* if this is != NULL, we need to dec_ap_pending in tl_clear */
229 new->next = NULL;
7e602c0a 230 new->n_writes = 0;
b411b363 231
87eeee41 232 newest_before = mdev->tconn->newest_tle;
b411b363
PR		 233 /* never send a barrier number == 0, because that is special-cased
234 * when using TCQ for our write ordering code */
235 new->br_number = (newest_before->br_number+1) ?: 1;
87eeee41
PR		 236 if (mdev->tconn->newest_tle != new) {
237 mdev->tconn->newest_tle->next = new;
238 mdev->tconn->newest_tle = new;
b411b363
PR		 239 }
240}
241
242/**
243 * tl_release() - Free or recycle the oldest &struct drbd_tl_epoch object of the TL
244 * @mdev: DRBD device.
245 * @barrier_nr: Expected identifier of the DRBD write barrier packet.
246 * @set_size: Expected number of requests before that barrier.
247 *
248 * In case the passed barrier_nr or set_size does not match the oldest
249 * &struct drbd_tl_epoch objects this function will cause a termination
250 * of the connection.
251 */
252void tl_release(struct drbd_conf *mdev, unsigned int barrier_nr,
253 unsigned int set_size)
254{
255 struct drbd_tl_epoch *b, *nob; /* next old barrier */
256 struct list_head *le, *tle;
257 struct drbd_request *r;
258
87eeee41 259 spin_lock_irq(&mdev->tconn->req_lock);
b411b363 260
87eeee41 261 b = mdev->tconn->oldest_tle;
b411b363
PR		 262
263 /* first some paranoia code */
264 if (b == NULL) {
265 dev_err(DEV, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
266 barrier_nr);
267 goto bail;
268 }
269 if (b->br_number != barrier_nr) {
270 dev_err(DEV, "BAD! BarrierAck #%u received, expected #%u!\n",
271 barrier_nr, b->br_number);
272 goto bail;
273 }
7e602c0a
PR		 274 if (b->n_writes != set_size) {
275 dev_err(DEV, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
276 barrier_nr, set_size, b->n_writes);
b411b363
PR		 277 goto bail;
278 }
279
280 /* Clean up list of requests processed during current epoch */
281 list_for_each_safe(le, tle, &b->requests) {
282 r = list_entry(le, struct drbd_request, tl_requests);
8554df1c 283 _req_mod(r, BARRIER_ACKED);
b411b363
PR		 284 }
285 /* There could be requests on the list waiting for completion
286 of the write to the local disk. To avoid corruptions of
287 slab's data structures we have to remove the lists head.
288
289 Also there could have been a barrier ack out of sequence, overtaking
290 the write acks - which would be a bug and violating write ordering.
291 To not deadlock in case we lose connection while such requests are
292 still pending, we need some way to find them for the
8554df1c 293 _req_mode(CONNECTION_LOST_WHILE_PENDING).
b411b363
PR		 294
295 These have been list_move'd to the out_of_sequence_requests list in
8554df1c 296 _req_mod(, BARRIER_ACKED) above.
b411b363
PR		 297 */
298 list_del_init(&b->requests);
299
300 nob = b->next;
301 if (test_and_clear_bit(CREATE_BARRIER, &mdev->flags)) {
302 _tl_add_barrier(mdev, b);
303 if (nob)
87eeee41 304 mdev->tconn->oldest_tle = nob;
b411b363 305 /* if nob == NULL b was the only barrier, and becomes the new
87eeee41 306 barrier. Therefore mdev->tconn->oldest_tle points already to b */
b411b363
PR		 307 } else {
308 D_ASSERT(nob != NULL);
87eeee41 309 mdev->tconn->oldest_tle = nob;
b411b363
PR		 310 kfree(b);
311 }
312
87eeee41 313 spin_unlock_irq(&mdev->tconn->req_lock);
b411b363
PR		 314 dec_ap_pending(mdev);
315
316 return;
317
318bail:
87eeee41 319 spin_unlock_irq(&mdev->tconn->req_lock);
b411b363
PR		 320 drbd_force_state(mdev, NS(conn, C_PROTOCOL_ERROR));
321}
322
617049aa 323
b411b363 324/**
11b58e73 325 * _tl_restart() - Walks the transfer log, and applies an action to all requests
b411b363 326 * @mdev: DRBD device.
11b58e73 327 * @what: The action/event to perform with all request objects
b411b363 328 *
8554df1c
AG		 329 * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
330 * RESTART_FROZEN_DISK_IO.
b411b363 331 */
b8907339 332void _tl_restart(struct drbd_conf *mdev, enum drbd_req_event what)
b411b363 333{
11b58e73 334 struct drbd_tl_epoch *b, *tmp, **pn;
b9b98716 335 struct list_head *le, *tle, carry_reads;
11b58e73
PR		 336 struct drbd_request *req;
337 int rv, n_writes, n_reads;
b411b363 338
87eeee41
PR		 339 b = mdev->tconn->oldest_tle;
340 pn = &mdev->tconn->oldest_tle;
b411b363 341 while (b) {
11b58e73
PR		 342 n_writes = 0;
343 n_reads = 0;
b9b98716 344 INIT_LIST_HEAD(&carry_reads);
b411b363 345 list_for_each_safe(le, tle, &b->requests) {
11b58e73
PR		 346 req = list_entry(le, struct drbd_request, tl_requests);
347 rv = _req_mod(req, what);
348
349 n_writes += (rv & MR_WRITE) >> MR_WRITE_SHIFT;
350 n_reads += (rv & MR_READ) >> MR_READ_SHIFT;
b411b363
PR		 351 }
352 tmp = b->next;
353
b9b98716 354 if (n_writes) {
8554df1c 355 if (what == RESEND) {
11b58e73
PR		 356 b->n_writes = n_writes;
357 if (b->w.cb == NULL) {
358 b->w.cb = w_send_barrier;
359 inc_ap_pending(mdev);
360 set_bit(CREATE_BARRIER, &mdev->flags);
361 }
362
e42325a5 363 drbd_queue_work(&mdev->tconn->data.work, &b->w);
11b58e73
PR		 364 }
365 pn = &b->next;
366 } else {
b9b98716
PR		 367 if (n_reads)
368 list_add(&carry_reads, &b->requests);
11b58e73
PR		 369 /* there could still be requests on that ring list,
370 * in case local io is still pending */
371 list_del(&b->requests);
372
373 /* dec_ap_pending corresponding to queue_barrier.
374 * the newest barrier may not have been queued yet,
375 * in which case w.cb is still NULL. */
376 if (b->w.cb != NULL)
377 dec_ap_pending(mdev);
378
87eeee41 379 if (b == mdev->tconn->newest_tle) {
11b58e73
PR		 380 /* recycle, but reinit! */
381 D_ASSERT(tmp == NULL);
382 INIT_LIST_HEAD(&b->requests);
b9b98716 383 list_splice(&carry_reads, &b->requests);
11b58e73
PR		 384 INIT_LIST_HEAD(&b->w.list);
385 b->w.cb = NULL;
386 b->br_number = net_random();
387 b->n_writes = 0;
388
389 *pn = b;
390 break;
391 }
392 *pn = tmp;
393 kfree(b);
b411b363 394 }
b411b363 395 b = tmp;
b9b98716 396 list_splice(&carry_reads, &b->requests);
b411b363 397 }
11b58e73
PR		 398}
399
b411b363
PR		 400
401/**
402 * tl_clear() - Clears all requests and &struct drbd_tl_epoch objects out of the TL
403 * @mdev: DRBD device.
404 *
405 * This is called after the connection to the peer was lost. The storage covered
406 * by the requests on the transfer gets marked as our of sync. Called from the
407 * receiver thread and the worker thread.
408 */
409void tl_clear(struct drbd_conf *mdev)
410{
b411b363
PR		 411 struct list_head *le, *tle;
412 struct drbd_request *r;
b411b363 413
87eeee41 414 spin_lock_irq(&mdev->tconn->req_lock);
b411b363 415
8554df1c 416 _tl_restart(mdev, CONNECTION_LOST_WHILE_PENDING);
b411b363
PR		 417
418 /* we expect this list to be empty. */
87eeee41 419 D_ASSERT(list_empty(&mdev->tconn->out_of_sequence_requests));
b411b363
PR		 420
421 /* but just in case, clean it up anyways! */
87eeee41 422 list_for_each_safe(le, tle, &mdev->tconn->out_of_sequence_requests) {
b411b363
PR		 423 r = list_entry(le, struct drbd_request, tl_requests);
424 /* It would be nice to complete outside of spinlock.
425 * But this is easier for now. */
8554df1c 426 _req_mod(r, CONNECTION_LOST_WHILE_PENDING);
b411b363
PR		 427 }
428
429 /* ensure bit indicating barrier is required is clear */
430 clear_bit(CREATE_BARRIER, &mdev->flags);
431
87eeee41 432 spin_unlock_irq(&mdev->tconn->req_lock);
b411b363
PR		 433}
434
11b58e73
PR		 435void tl_restart(struct drbd_conf *mdev, enum drbd_req_event what)
436{
87eeee41 437 spin_lock_irq(&mdev->tconn->req_lock);
11b58e73 438 _tl_restart(mdev, what);
87eeee41 439 spin_unlock_irq(&mdev->tconn->req_lock);
b411b363
PR		 440}
441
b411b363
PR		 442static int drbd_thread_setup(void *arg)
443{
444 struct drbd_thread *thi = (struct drbd_thread *) arg;
445 struct drbd_conf *mdev = thi->mdev;
446 unsigned long flags;
447 int retval;
448
449restart:
450 retval = thi->function(thi);
451
452 spin_lock_irqsave(&thi->t_lock, flags);
453
e77a0a5c 454 /* if the receiver has been "EXITING", the last thing it did
b411b363
PR		 455 * was set the conn state to "StandAlone",
456 * if now a re-connect request comes in, conn state goes C_UNCONNECTED,
457 * and receiver thread will be "started".
e77a0a5c 458 * drbd_thread_start needs to set "RESTARTING" in that case.
b411b363 459 * t_state check and assignment needs to be within the same spinlock,
e77a0a5c
AG		 460 * so either thread_start sees EXITING, and can remap to RESTARTING,
461 * or thread_start see NONE, and can proceed as normal.
b411b363
PR		 462 */
463
e77a0a5c 464 if (thi->t_state == RESTARTING) {
bed879ae 465 dev_info(DEV, "Restarting %s thread\n", thi->name);
e77a0a5c 466 thi->t_state = RUNNING;
b411b363
PR		 467 spin_unlock_irqrestore(&thi->t_lock, flags);
468 goto restart;
469 }
470
471 thi->task = NULL;
e77a0a5c 472 thi->t_state = NONE;
b411b363
PR		 473 smp_mb();
474 complete(&thi->stop);
475 spin_unlock_irqrestore(&thi->t_lock, flags);
476
477 dev_info(DEV, "Terminating %s\n", current->comm);
478
479 /* Release mod reference taken when thread was started */
480 module_put(THIS_MODULE);
481 return retval;
482}
483
484static void drbd_thread_init(struct drbd_conf *mdev, struct drbd_thread *thi,
bed879ae 485 int (*func) (struct drbd_thread *), char *name)
b411b363
PR		 486{
487 spin_lock_init(&thi->t_lock);
488 thi->task = NULL;
e77a0a5c 489 thi->t_state = NONE;
b411b363
PR		 490 thi->function = func;
491 thi->mdev = mdev;
bed879ae 492 strncpy(thi->name, name, ARRAY_SIZE(thi->name));
b411b363
PR		 493}
494
495int drbd_thread_start(struct drbd_thread *thi)
496{
497 struct drbd_conf *mdev = thi->mdev;
498 struct task_struct *nt;
499 unsigned long flags;
500
b411b363
PR		 501 /* is used from state engine doing drbd_thread_stop_nowait,
502 * while holding the req lock irqsave */
503 spin_lock_irqsave(&thi->t_lock, flags);
504
505 switch (thi->t_state) {
e77a0a5c 506 case NONE:
b411b363 507 dev_info(DEV, "Starting %s thread (from %s [%d])\n",
bed879ae 508 thi->name, current->comm, current->pid);
b411b363
PR		 509
510 /* Get ref on module for thread - this is released when thread exits */
511 if (!try_module_get(THIS_MODULE)) {
512 dev_err(DEV, "Failed to get module reference in drbd_thread_start\n");
513 spin_unlock_irqrestore(&thi->t_lock, flags);
81e84650 514 return false;
b411b363
PR		 515 }
516
517 init_completion(&thi->stop);
518 D_ASSERT(thi->task == NULL);
519 thi->reset_cpu_mask = 1;
e77a0a5c 520 thi->t_state = RUNNING;
b411b363
PR		 521 spin_unlock_irqrestore(&thi->t_lock, flags);
522 flush_signals(current); /* otherw. may get -ERESTARTNOINTR */
523
524 nt = kthread_create(drbd_thread_setup, (void *) thi,
bed879ae 525 "drbd%d_%s", mdev_to_minor(mdev), thi->name);
b411b363
PR		 526
527 if (IS_ERR(nt)) {
528 dev_err(DEV, "Couldn't start thread\n");
529
530 module_put(THIS_MODULE);
81e84650 531 return false;
b411b363
PR		 532 }
533 spin_lock_irqsave(&thi->t_lock, flags);
534 thi->task = nt;
e77a0a5c 535 thi->t_state = RUNNING;
b411b363
PR		 536 spin_unlock_irqrestore(&thi->t_lock, flags);
537 wake_up_process(nt);
538 break;
e77a0a5c
AG		 539 case EXITING:
540 thi->t_state = RESTARTING;
b411b363 541 dev_info(DEV, "Restarting %s thread (from %s [%d])\n",
bed879ae 542 thi->name, current->comm, current->pid);
b411b363 543 /* fall through */
e77a0a5c
AG		 544 case RUNNING:
545 case RESTARTING:
b411b363
PR		 546 default:
547 spin_unlock_irqrestore(&thi->t_lock, flags);
548 break;
549 }
550
81e84650 551 return true;
b411b363
PR		 552}
553
554
555void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait)
556{
557 unsigned long flags;
558
e77a0a5c 559 enum drbd_thread_state ns = restart ? RESTARTING : EXITING;
b411b363
PR		 560
561 /* may be called from state engine, holding the req lock irqsave */
562 spin_lock_irqsave(&thi->t_lock, flags);
563
e77a0a5c 564 if (thi->t_state == NONE) {
b411b363
PR		 565 spin_unlock_irqrestore(&thi->t_lock, flags);
566 if (restart)
567 drbd_thread_start(thi);
568 return;
569 }
570
571 if (thi->t_state != ns) {
572 if (thi->task == NULL) {
573 spin_unlock_irqrestore(&thi->t_lock, flags);
574 return;
575 }
576
577 thi->t_state = ns;
578 smp_mb();
579 init_completion(&thi->stop);
580 if (thi->task != current)
581 force_sig(DRBD_SIGKILL, thi->task);
582
583 }
584
585 spin_unlock_irqrestore(&thi->t_lock, flags);
586
587 if (wait)
588 wait_for_completion(&thi->stop);
589}
590
bed879ae
PR		 591static struct drbd_thread *drbd_task_to_thread(struct drbd_conf *mdev, struct task_struct *task)
592{
593 struct drbd_tconn *tconn = mdev->tconn;
594 struct drbd_thread *thi =
595 task == tconn->receiver.task ? &tconn->receiver :
596 task == tconn->asender.task ? &tconn->asender :
597 task == tconn->worker.task ? &tconn->worker : NULL;
598
599 return thi;
600}
601
602char *drbd_task_to_thread_name(struct drbd_conf *mdev, struct task_struct *task)
603{
604 struct drbd_thread *thi = drbd_task_to_thread(mdev, task);
605 return thi ? thi->name : task->comm;
606}
607
b411b363 608#ifdef CONFIG_SMP
80822284
PR		 609static int conn_lowest_minor(struct drbd_tconn *tconn)
610{
611 int minor = 0;
612 idr_get_next(&tconn->volumes, &minor);
613 return minor;
614}
b411b363
PR		 615/**
616 * drbd_calc_cpu_mask() - Generate CPU masks, spread over all CPUs
617 * @mdev: DRBD device.
618 *
619 * Forces all threads of a device onto the same CPU. This is beneficial for
620 * DRBD's performance. May be overwritten by user's configuration.
621 */
80822284 622void drbd_calc_cpu_mask(struct drbd_tconn *tconn)
b411b363
PR		 623{
624 int ord, cpu;
625
626 /* user override. */
80822284 627 if (cpumask_weight(tconn->cpu_mask))
b411b363
PR		 628 return;
629
80822284 630 ord = conn_lowest_minor(tconn) % cpumask_weight(cpu_online_mask);
b411b363
PR		 631 for_each_online_cpu(cpu) {
632 if (ord-- == 0) {
80822284 633 cpumask_set_cpu(cpu, tconn->cpu_mask);
b411b363
PR		 634 return;
635 }
636 }
637 /* should not be reached */
80822284 638 cpumask_setall(tconn->cpu_mask);
b411b363
PR		 639}
640
641/**
642 * drbd_thread_current_set_cpu() - modifies the cpu mask of the _current_ thread
643 * @mdev: DRBD device.
bc31fe33 644 * @thi: drbd_thread object
b411b363
PR		 645 *
646 * call in the "main loop" of _all_ threads, no need for any mutex, current won't die
647 * prematurely.
648 */
80822284 649void drbd_thread_current_set_cpu(struct drbd_thread *thi)
b411b363
PR		 650{
651 struct task_struct *p = current;
bed879ae 652
b411b363
PR		 653 if (!thi->reset_cpu_mask)
654 return;
655 thi->reset_cpu_mask = 0;
80822284 656 set_cpus_allowed_ptr(p, thi->mdev->tconn->cpu_mask);
b411b363
PR		 657}
658#endif
659
d38e787e 660static void prepare_header80(struct p_header80 *h, enum drbd_packet cmd, int size)
fd340c12
PR		 661{
662 h->magic = cpu_to_be32(DRBD_MAGIC);
663 h->command = cpu_to_be16(cmd);
664 h->length = cpu_to_be16(size);
665}
666
d38e787e 667static void prepare_header95(struct p_header95 *h, enum drbd_packet cmd, int size)
fd340c12
PR		 668{
669 h->magic = cpu_to_be16(DRBD_MAGIC_BIG);
670 h->command = cpu_to_be16(cmd);
671 h->length = cpu_to_be32(size);
672}
673
d38e787e
PR		 674static void _prepare_header(struct drbd_tconn *tconn, int vnr, struct p_header *h,
675 enum drbd_packet cmd, int size)
676{
677 if (tconn->agreed_pro_version >= 100 || size > DRBD_MAX_SIZE_H80_PACKET)
678 prepare_header95(&h->h95, cmd, size);
679 else
680 prepare_header80(&h->h80, cmd, size);
681}
682
fd340c12 683static void prepare_header(struct drbd_conf *mdev, struct p_header *h,
d8763023 684 enum drbd_packet cmd, int size)
fd340c12 685{
d38e787e 686 _prepare_header(mdev->tconn, mdev->vnr, h, cmd, size);
fd340c12
PR		 687}
688
b411b363 689/* the appropriate socket mutex must be held already */
d38e787e 690int _conn_send_cmd(struct drbd_tconn *tconn, int vnr, struct socket *sock,
d8763023
AG		 691 enum drbd_packet cmd, struct p_header *h, size_t size,
692 unsigned msg_flags)
b411b363
PR		 693{
694 int sent, ok;
695
d38e787e 696 _prepare_header(tconn, vnr, h, cmd, size - sizeof(struct p_header));
b411b363 697
d38e787e 698 sent = drbd_send(tconn, sock, h, size, msg_flags);
b411b363
PR		 699
700 ok = (sent == size);
0ddc5549 701 if (!ok && !signal_pending(current))
d38e787e
PR		 702 conn_warn(tconn, "short sent %s size=%d sent=%d\n",
703 cmdname(cmd), (int)size, sent);
b411b363
PR		 704 return ok;
705}
706
707/* don't pass the socket. we may only look at it
708 * when we hold the appropriate socket mutex.
709 */
710int drbd_send_cmd(struct drbd_conf *mdev, int use_data_socket,
d8763023 711 enum drbd_packet cmd, struct p_header *h, size_t size)
b411b363
PR		 712{
713 int ok = 0;
714 struct socket *sock;
715
716 if (use_data_socket) {
e42325a5
PR		 717 mutex_lock(&mdev->tconn->data.mutex);
718 sock = mdev->tconn->data.socket;
b411b363 719 } else {
e42325a5
PR		 720 mutex_lock(&mdev->tconn->meta.mutex);
721 sock = mdev->tconn->meta.socket;
b411b363
PR		 722 }
723
724 /* drbd_disconnect() could have called drbd_free_sock()
725 * while we were waiting in down()... */
726 if (likely(sock != NULL))
727 ok = _drbd_send_cmd(mdev, sock, cmd, h, size, 0);
728
729 if (use_data_socket)
e42325a5 730 mutex_unlock(&mdev->tconn->data.mutex);
b411b363 731 else
e42325a5 732 mutex_unlock(&mdev->tconn->meta.mutex);
b411b363
PR		 733 return ok;
734}
735
61120870 736int conn_send_cmd2(struct drbd_tconn *tconn, enum drbd_packet cmd, char *data,
b411b363
PR		 737 size_t size)
738{
61120870 739 struct p_header80 h;
b411b363
PR		 740 int ok;
741
61120870 742 prepare_header80(&h, cmd, size);
b411b363 743
61120870 744 if (!drbd_get_data_sock(tconn))
b411b363
PR		 745 return 0;
746
b411b363 747 ok = (sizeof(h) ==
61120870 748 drbd_send(tconn, tconn->data.socket, &h, sizeof(h), 0));
b411b363 749 ok = ok && (size ==
61120870 750 drbd_send(tconn, tconn->data.socket, data, size, 0));
b411b363 751
61120870 752 drbd_put_data_sock(tconn);
b411b363
PR		 753
754 return ok;
755}
756
757int drbd_send_sync_param(struct drbd_conf *mdev, struct syncer_conf *sc)
758{
8e26f9cc 759 struct p_rs_param_95 *p;
b411b363
PR		 760 struct socket *sock;
761 int size, rv;
31890f4a 762 const int apv = mdev->tconn->agreed_pro_version;
b411b363
PR		 763
764 size = apv <= 87 ? sizeof(struct p_rs_param)
765 : apv == 88 ? sizeof(struct p_rs_param)
766 + strlen(mdev->sync_conf.verify_alg) + 1
8e26f9cc
PR		 767 : apv <= 94 ? sizeof(struct p_rs_param_89)
768 : /* apv >= 95 */ sizeof(struct p_rs_param_95);
b411b363
PR		 769
770 /* used from admin command context and receiver/worker context.
771 * to avoid kmalloc, grab the socket right here,
772 * then use the pre-allocated sbuf there */
e42325a5
PR		 773 mutex_lock(&mdev->tconn->data.mutex);
774 sock = mdev->tconn->data.socket;
b411b363
PR		 775
776 if (likely(sock != NULL)) {
d8763023
AG		 777 enum drbd_packet cmd =
778 apv >= 89 ? P_SYNC_PARAM89 : P_SYNC_PARAM;
b411b363 779
e42325a5 780 p = &mdev->tconn->data.sbuf.rs_param_95;
b411b363
PR		 781
782 /* initialize verify_alg and csums_alg */
783 memset(p->verify_alg, 0, 2 * SHARED_SECRET_MAX);
784
785 p->rate = cpu_to_be32(sc->rate);
8e26f9cc
PR		 786 p->c_plan_ahead = cpu_to_be32(sc->c_plan_ahead);
787 p->c_delay_target = cpu_to_be32(sc->c_delay_target);
788 p->c_fill_target = cpu_to_be32(sc->c_fill_target);
789 p->c_max_rate = cpu_to_be32(sc->c_max_rate);
b411b363
PR		 790
791 if (apv >= 88)
792 strcpy(p->verify_alg, mdev->sync_conf.verify_alg);
793 if (apv >= 89)
794 strcpy(p->csums_alg, mdev->sync_conf.csums_alg);
795
796 rv = _drbd_send_cmd(mdev, sock, cmd, &p->head, size, 0);
797 } else
798 rv = 0; /* not ok */
799
e42325a5 800 mutex_unlock(&mdev->tconn->data.mutex);
b411b363
PR		 801
802 return rv;
803}
804
dc8228d1 805int drbd_send_protocol(struct drbd_tconn *tconn)
b411b363
PR		 806{
807 struct p_protocol *p;
cf14c2e9 808 int size, cf, rv;
b411b363
PR		 809
810 size = sizeof(struct p_protocol);
811
dc8228d1
PR		 812 if (tconn->agreed_pro_version >= 87)
813 size += strlen(tconn->net_conf->integrity_alg) + 1;
b411b363
PR		 814
815 /* we must not recurse into our own queue,
816 * as that is blocked during handshake */
817 p = kmalloc(size, GFP_NOIO);
818 if (p == NULL)
819 return 0;
820
dc8228d1
PR		 821 p->protocol = cpu_to_be32(tconn->net_conf->wire_protocol);
822 p->after_sb_0p = cpu_to_be32(tconn->net_conf->after_sb_0p);
823 p->after_sb_1p = cpu_to_be32(tconn->net_conf->after_sb_1p);
824 p->after_sb_2p = cpu_to_be32(tconn->net_conf->after_sb_2p);
825 p->two_primaries = cpu_to_be32(tconn->net_conf->two_primaries);
b411b363 826
cf14c2e9 827 cf = 0;
dc8228d1 828 if (tconn->net_conf->want_lose)
cf14c2e9 829 cf |= CF_WANT_LOSE;
dc8228d1
PR		 830 if (tconn->net_conf->dry_run) {
831 if (tconn->agreed_pro_version >= 92)
cf14c2e9
PR		 832 cf |= CF_DRY_RUN;
833 else {
dc8228d1 834 conn_err(tconn, "--dry-run is not supported by peer");
7ac314c8 835 kfree(p);
148efa16 836 return -1;
cf14c2e9
PR		 837 }
838 }
839 p->conn_flags = cpu_to_be32(cf);
840
dc8228d1
PR		 841 if (tconn->agreed_pro_version >= 87)
842 strcpy(p->integrity_alg, tconn->net_conf->integrity_alg);
b411b363 843
dc8228d1 844 rv = conn_send_cmd2(tconn, P_PROTOCOL, p->head.payload, size - sizeof(struct p_header));
b411b363
PR		 845 kfree(p);
846 return rv;
847}
848
849int _drbd_send_uuids(struct drbd_conf *mdev, u64 uuid_flags)
850{
851 struct p_uuids p;
852 int i;
853
854 if (!get_ldev_if_state(mdev, D_NEGOTIATING))
855 return 1;
856
857 for (i = UI_CURRENT; i < UI_SIZE; i++)
858 p.uuid[i] = mdev->ldev ? cpu_to_be64(mdev->ldev->md.uuid[i]) : 0;
859
860 mdev->comm_bm_set = drbd_bm_total_weight(mdev);
861 p.uuid[UI_SIZE] = cpu_to_be64(mdev->comm_bm_set);
89e58e75 862 uuid_flags |= mdev->tconn->net_conf->want_lose ? 1 : 0;
b411b363
PR		 863 uuid_flags |= test_bit(CRASHED_PRIMARY, &mdev->flags) ? 2 : 0;
864 uuid_flags |= mdev->new_state_tmp.disk == D_INCONSISTENT ? 4 : 0;
865 p.uuid[UI_FLAGS] = cpu_to_be64(uuid_flags);
866
867 put_ldev(mdev);
868
c012949a 869 return drbd_send_cmd(mdev, USE_DATA_SOCKET, P_UUIDS, &p.head, sizeof(p));
b411b363
PR		 870}
871
872int drbd_send_uuids(struct drbd_conf *mdev)
873{
874 return _drbd_send_uuids(mdev, 0);
875}
876
877int drbd_send_uuids_skip_initial_sync(struct drbd_conf *mdev)
878{
879 return _drbd_send_uuids(mdev, 8);
880}
881
62b0da3a
LE		 882void drbd_print_uuids(struct drbd_conf *mdev, const char *text)
883{
884 if (get_ldev_if_state(mdev, D_NEGOTIATING)) {
885 u64 *uuid = mdev->ldev->md.uuid;
886 dev_info(DEV, "%s %016llX:%016llX:%016llX:%016llX\n",
887 text,
888 (unsigned long long)uuid[UI_CURRENT],
889 (unsigned long long)uuid[UI_BITMAP],
890 (unsigned long long)uuid[UI_HISTORY_START],
891 (unsigned long long)uuid[UI_HISTORY_END]);
892 put_ldev(mdev);
893 } else {
894 dev_info(DEV, "%s effective data uuid: %016llX\n",
895 text,
896 (unsigned long long)mdev->ed_uuid);
897 }
898}
899
5a22db89 900int drbd_gen_and_send_sync_uuid(struct drbd_conf *mdev)
b411b363
PR		 901{
902 struct p_rs_uuid p;
5a22db89
LE		 903 u64 uuid;
904
905 D_ASSERT(mdev->state.disk == D_UP_TO_DATE);
b411b363 906
4a23f264 907 uuid = mdev->ldev->md.uuid[UI_BITMAP] + UUID_NEW_BM_OFFSET;
5a22db89 908 drbd_uuid_set(mdev, UI_BITMAP, uuid);
62b0da3a 909 drbd_print_uuids(mdev, "updated sync UUID");
5a22db89
LE		 910 drbd_md_sync(mdev);
911 p.uuid = cpu_to_be64(uuid);
b411b363 912
c012949a 913 return drbd_send_cmd(mdev, USE_DATA_SOCKET, P_SYNC_UUID, &p.head, sizeof(p));
b411b363
PR		 914}
915
e89b591c 916int drbd_send_sizes(struct drbd_conf *mdev, int trigger_reply, enum dds_flags flags)
b411b363
PR		 917{
918 struct p_sizes p;
919 sector_t d_size, u_size;
99432fcc 920 int q_order_type, max_bio_size;
b411b363
PR		 921 int ok;
922
923 if (get_ldev_if_state(mdev, D_NEGOTIATING)) {
924 D_ASSERT(mdev->ldev->backing_bdev);
925 d_size = drbd_get_max_capacity(mdev->ldev);
926 u_size = mdev->ldev->dc.disk_size;
927 q_order_type = drbd_queue_order_type(mdev);
99432fcc
PR		 928 max_bio_size = queue_max_hw_sectors(mdev->ldev->backing_bdev->bd_disk->queue) << 9;
929 max_bio_size = min_t(int, max_bio_size, DRBD_MAX_BIO_SIZE);
b411b363
PR		 930 put_ldev(mdev);
931 } else {
932 d_size = 0;
933 u_size = 0;
934 q_order_type = QUEUE_ORDERED_NONE;
99432fcc 935 max_bio_size = DRBD_MAX_BIO_SIZE; /* ... multiple BIOs per peer_request */
b411b363
PR		 936 }
937
938 p.d_size = cpu_to_be64(d_size);
939 p.u_size = cpu_to_be64(u_size);
940 p.c_size = cpu_to_be64(trigger_reply ? 0 : drbd_get_capacity(mdev->this_bdev));
99432fcc 941 p.max_bio_size = cpu_to_be32(max_bio_size);
e89b591c
PR		 942 p.queue_order_type = cpu_to_be16(q_order_type);
943 p.dds_flags = cpu_to_be16(flags);
b411b363 944
c012949a 945 ok = drbd_send_cmd(mdev, USE_DATA_SOCKET, P_SIZES, &p.head, sizeof(p));
b411b363
PR		 946 return ok;
947}
948
949/**
950 * drbd_send_state() - Sends the drbd state to the peer
951 * @mdev: DRBD device.
952 */
953int drbd_send_state(struct drbd_conf *mdev)
954{
955 struct socket *sock;
956 struct p_state p;
957 int ok = 0;
958
e42325a5 959 mutex_lock(&mdev->tconn->data.mutex);
b411b363
PR		 960
961 p.state = cpu_to_be32(mdev->state.i); /* Within the send mutex */
e42325a5 962 sock = mdev->tconn->data.socket;
b411b363
PR		 963
964 if (likely(sock != NULL)) {
c012949a 965 ok = _drbd_send_cmd(mdev, sock, P_STATE, &p.head, sizeof(p), 0);
b411b363
PR		 966 }
967
e42325a5 968 mutex_unlock(&mdev->tconn->data.mutex);
b411b363 969
b411b363
PR		 970 return ok;
971}
972
973int drbd_send_state_req(struct drbd_conf *mdev,
974 union drbd_state mask, union drbd_state val)
975{
976 struct p_req_state p;
977
978 p.mask = cpu_to_be32(mask.i);
979 p.val = cpu_to_be32(val.i);
980
c012949a 981 return drbd_send_cmd(mdev, USE_DATA_SOCKET, P_STATE_CHG_REQ, &p.head, sizeof(p));
b411b363
PR		 982}
983
bf885f8a 984int drbd_send_sr_reply(struct drbd_conf *mdev, enum drbd_state_rv retcode)
b411b363
PR		 985{
986 struct p_req_state_reply p;
987
988 p.retcode = cpu_to_be32(retcode);
989
c012949a 990 return drbd_send_cmd(mdev, USE_META_SOCKET, P_STATE_CHG_REPLY, &p.head, sizeof(p));
b411b363
PR		 991}
992
993int fill_bitmap_rle_bits(struct drbd_conf *mdev,
994 struct p_compressed_bm *p,
995 struct bm_xfer_ctx *c)
996{
997 struct bitstream bs;
998 unsigned long plain_bits;
999 unsigned long tmp;
1000 unsigned long rl;
1001 unsigned len;
1002 unsigned toggle;
1003 int bits;
1004
1005 /* may we use this feature? */
1006 if ((mdev->sync_conf.use_rle == 0) ||
31890f4a 1007 (mdev->tconn->agreed_pro_version < 90))
b411b363
PR		 1008 return 0;
1009
1010 if (c->bit_offset >= c->bm_bits)
1011 return 0; /* nothing to do. */
1012
1013 /* use at most thus many bytes */
1014 bitstream_init(&bs, p->code, BM_PACKET_VLI_BYTES_MAX, 0);
1015 memset(p->code, 0, BM_PACKET_VLI_BYTES_MAX);
1016 /* plain bits covered in this code string */
1017 plain_bits = 0;
1018
1019 /* p->encoding & 0x80 stores whether the first run length is set.
1020 * bit offset is implicit.
1021 * start with toggle == 2 to be able to tell the first iteration */
1022 toggle = 2;
1023
1024 /* see how much plain bits we can stuff into one packet
1025 * using RLE and VLI. */
1026 do {
1027 tmp = (toggle == 0) ? _drbd_bm_find_next_zero(mdev, c->bit_offset)
1028 : _drbd_bm_find_next(mdev, c->bit_offset);
1029 if (tmp == -1UL)
1030 tmp = c->bm_bits;
1031 rl = tmp - c->bit_offset;
1032
1033 if (toggle == 2) { /* first iteration */
1034 if (rl == 0) {
1035 /* the first checked bit was set,
1036 * store start value, */
1037 DCBP_set_start(p, 1);
1038 /* but skip encoding of zero run length */
1039 toggle = !toggle;
1040 continue;
1041 }
1042 DCBP_set_start(p, 0);
1043 }
1044
1045 /* paranoia: catch zero runlength.
1046 * can only happen if bitmap is modified while we scan it. */
1047 if (rl == 0) {
1048 dev_err(DEV, "unexpected zero runlength while encoding bitmap "
1049 "t:%u bo:%lu\n", toggle, c->bit_offset);
1050 return -1;
1051 }
1052
1053 bits = vli_encode_bits(&bs, rl);
1054 if (bits == -ENOBUFS) /* buffer full */
1055 break;
1056 if (bits <= 0) {
1057 dev_err(DEV, "error while encoding bitmap: %d\n", bits);
1058 return 0;
1059 }
1060
1061 toggle = !toggle;
1062 plain_bits += rl;
1063 c->bit_offset = tmp;
1064 } while (c->bit_offset < c->bm_bits);
1065
1066 len = bs.cur.b - p->code + !!bs.cur.bit;
1067
1068 if (plain_bits < (len << 3)) {
1069 /* incompressible with this method.
1070 * we need to rewind both word and bit position. */
1071 c->bit_offset -= plain_bits;
1072 bm_xfer_ctx_bit_to_word_offset(c);
1073 c->bit_offset = c->word_offset * BITS_PER_LONG;
1074 return 0;
1075 }
1076
1077 /* RLE + VLI was able to compress it just fine.
1078 * update c->word_offset. */
1079 bm_xfer_ctx_bit_to_word_offset(c);
1080
1081 /* store pad_bits */
1082 DCBP_set_pad_bits(p, (8 - bs.cur.bit) & 0x7);
1083
1084 return len;
1085}
1086
f70af118
AG		 1087/**
1088 * send_bitmap_rle_or_plain
1089 *
1090 * Return 0 when done, 1 when another iteration is needed, and a negative error
1091 * code upon failure.
1092 */
1093static int
b411b363 1094send_bitmap_rle_or_plain(struct drbd_conf *mdev,
c012949a 1095 struct p_header *h, struct bm_xfer_ctx *c)
b411b363
PR		 1096{
1097 struct p_compressed_bm *p = (void*)h;
1098 unsigned long num_words;
1099 int len;
1100 int ok;
1101
1102 len = fill_bitmap_rle_bits(mdev, p, c);
1103
1104 if (len < 0)
f70af118 1105 return -EIO;
b411b363
PR		 1106
1107 if (len) {
1108 DCBP_set_code(p, RLE_VLI_Bits);
e42325a5 1109 ok = _drbd_send_cmd(mdev, mdev->tconn->data.socket, P_COMPRESSED_BITMAP, h,
b411b363
PR		 1110 sizeof(*p) + len, 0);
1111
1112 c->packets[0]++;
1113 c->bytes[0] += sizeof(*p) + len;
1114
1115 if (c->bit_offset >= c->bm_bits)
1116 len = 0; /* DONE */
1117 } else {
1118 /* was not compressible.
1119 * send a buffer full of plain text bits instead. */
1120 num_words = min_t(size_t, BM_PACKET_WORDS, c->bm_words - c->word_offset);
1121 len = num_words * sizeof(long);
1122 if (len)
1123 drbd_bm_get_lel(mdev, c->word_offset, num_words, (unsigned long*)h->payload);
e42325a5 1124 ok = _drbd_send_cmd(mdev, mdev->tconn->data.socket, P_BITMAP,
0b70a13d 1125 h, sizeof(struct p_header80) + len, 0);
b411b363
PR		 1126 c->word_offset += num_words;
1127 c->bit_offset = c->word_offset * BITS_PER_LONG;
1128
1129 c->packets[1]++;
0b70a13d 1130 c->bytes[1] += sizeof(struct p_header80) + len;
b411b363
PR		 1131
1132 if (c->bit_offset > c->bm_bits)
1133 c->bit_offset = c->bm_bits;
1134 }
f70af118
AG		 1135 if (ok) {
1136 if (len == 0) {
1137 INFO_bm_xfer_stats(mdev, "send", c);
1138 return 0;
1139 } else
1140 return 1;
1141 }
1142 return -EIO;
b411b363
PR		 1143}
1144
1145/* See the comment at receive_bitmap() */
1146int _drbd_send_bitmap(struct drbd_conf *mdev)
1147{
1148 struct bm_xfer_ctx c;
c012949a 1149 struct p_header *p;
f70af118 1150 int err;
b411b363 1151
841ce241
AG		 1152 if (!expect(mdev->bitmap))
1153 return false;
b411b363
PR		 1154
1155 /* maybe we should use some per thread scratch page,
1156 * and allocate that during initial device creation? */
c012949a 1157 p = (struct p_header *) __get_free_page(GFP_NOIO);
b411b363
PR		 1158 if (!p) {
1159 dev_err(DEV, "failed to allocate one page buffer in %s\n", __func__);
81e84650 1160 return false;
b411b363
PR		 1161 }
1162
1163 if (get_ldev(mdev)) {
1164 if (drbd_md_test_flag(mdev->ldev, MDF_FULL_SYNC)) {
1165 dev_info(DEV, "Writing the whole bitmap, MDF_FullSync was set.\n");
1166 drbd_bm_set_all(mdev);
1167 if (drbd_bm_write(mdev)) {
1168 /* write_bm did fail! Leave full sync flag set in Meta P_DATA
1169 * but otherwise process as per normal - need to tell other
1170 * side that a full resync is required! */
1171 dev_err(DEV, "Failed to write bitmap to disk!\n");
1172 } else {
1173 drbd_md_clear_flag(mdev, MDF_FULL_SYNC);
1174 drbd_md_sync(mdev);
1175 }
1176 }
1177 put_ldev(mdev);
1178 }
1179
1180 c = (struct bm_xfer_ctx) {
1181 .bm_bits = drbd_bm_bits(mdev),
1182 .bm_words = drbd_bm_words(mdev),
1183 };
1184
1185 do {
f70af118
AG		 1186 err = send_bitmap_rle_or_plain(mdev, p, &c);
1187 } while (err > 0);
b411b363
PR		 1188
1189 free_page((unsigned long) p);
f70af118 1190 return err == 0;
b411b363
PR		 1191}
1192
1193int drbd_send_bitmap(struct drbd_conf *mdev)
1194{
1195 int err;
1196
61120870 1197 if (!drbd_get_data_sock(mdev->tconn))
b411b363
PR		 1198 return -1;
1199 err = !_drbd_send_bitmap(mdev);
61120870 1200 drbd_put_data_sock(mdev->tconn);
b411b363
PR		 1201 return err;
1202}
1203
1204int drbd_send_b_ack(struct drbd_conf *mdev, u32 barrier_nr, u32 set_size)
1205{
1206 int ok;
1207 struct p_barrier_ack p;
1208
1209 p.barrier = barrier_nr;
1210 p.set_size = cpu_to_be32(set_size);
1211
1212 if (mdev->state.conn < C_CONNECTED)
81e84650 1213 return false;
c012949a 1214 ok = drbd_send_cmd(mdev, USE_META_SOCKET, P_BARRIER_ACK, &p.head, sizeof(p));
b411b363
PR		 1215 return ok;
1216}
1217
1218/**
1219 * _drbd_send_ack() - Sends an ack packet
1220 * @mdev: DRBD device.
1221 * @cmd: Packet command code.
1222 * @sector: sector, needs to be in big endian byte order
1223 * @blksize: size in byte, needs to be in big endian byte order
1224 * @block_id: Id, big endian byte order
1225 */
d8763023
AG		 1226static int _drbd_send_ack(struct drbd_conf *mdev, enum drbd_packet cmd,
1227 u64 sector, u32 blksize, u64 block_id)
b411b363
PR		 1228{
1229 int ok;
1230 struct p_block_ack p;
1231
1232 p.sector = sector;
1233 p.block_id = block_id;
1234 p.blksize = blksize;
1235 p.seq_num = cpu_to_be32(atomic_add_return(1, &mdev->packet_seq));
1236
e42325a5 1237 if (!mdev->tconn->meta.socket || mdev->state.conn < C_CONNECTED)
81e84650 1238 return false;
c012949a 1239 ok = drbd_send_cmd(mdev, USE_META_SOCKET, cmd, &p.head, sizeof(p));
b411b363
PR		 1240 return ok;
1241}
1242
2b2bf214
LE		 1243/* dp->sector and dp->block_id already/still in network byte order,
1244 * data_size is payload size according to dp->head,
1245 * and may need to be corrected for digest size. */
d8763023 1246int drbd_send_ack_dp(struct drbd_conf *mdev, enum drbd_packet cmd,
2b2bf214 1247 struct p_data *dp, int data_size)
b411b363 1248{
a0638456
PR		 1249 data_size -= (mdev->tconn->agreed_pro_version >= 87 && mdev->tconn->integrity_r_tfm) ?
1250 crypto_hash_digestsize(mdev->tconn->integrity_r_tfm) : 0;
b411b363
PR		 1251 return _drbd_send_ack(mdev, cmd, dp->sector, cpu_to_be32(data_size),
1252 dp->block_id);
1253}
1254
d8763023 1255int drbd_send_ack_rp(struct drbd_conf *mdev, enum drbd_packet cmd,
b411b363
PR		 1256 struct p_block_req *rp)
1257{
1258 return _drbd_send_ack(mdev, cmd, rp->sector, rp->blksize, rp->block_id);
1259}
1260
1261/**
1262 * drbd_send_ack() - Sends an ack packet
db830c46
AG		 1263 * @mdev: DRBD device
1264 * @cmd: packet command code
1265 * @peer_req: peer request
b411b363 1266 */
d8763023 1267int drbd_send_ack(struct drbd_conf *mdev, enum drbd_packet cmd,
db830c46 1268 struct drbd_peer_request *peer_req)
b411b363
PR		 1269{
1270 return _drbd_send_ack(mdev, cmd,
db830c46
AG		 1271 cpu_to_be64(peer_req->i.sector),
1272 cpu_to_be32(peer_req->i.size),
1273 peer_req->block_id);
b411b363
PR		 1274}
1275
1276/* This function misuses the block_id field to signal if the blocks
1277 * are is sync or not. */
d8763023 1278int drbd_send_ack_ex(struct drbd_conf *mdev, enum drbd_packet cmd,
b411b363
PR		 1279 sector_t sector, int blksize, u64 block_id)
1280{
1281 return _drbd_send_ack(mdev, cmd,
1282 cpu_to_be64(sector),
1283 cpu_to_be32(blksize),
1284 cpu_to_be64(block_id));
1285}
1286
1287int drbd_send_drequest(struct drbd_conf *mdev, int cmd,
1288 sector_t sector, int size, u64 block_id)
1289{
1290 int ok;
1291 struct p_block_req p;
1292
1293 p.sector = cpu_to_be64(sector);
1294 p.block_id = block_id;
1295 p.blksize = cpu_to_be32(size);
1296
c012949a 1297 ok = drbd_send_cmd(mdev, USE_DATA_SOCKET, cmd, &p.head, sizeof(p));
b411b363
PR		 1298 return ok;
1299}
1300
d8763023
AG		 1301int drbd_send_drequest_csum(struct drbd_conf *mdev, sector_t sector, int size,
1302 void *digest, int digest_size, enum drbd_packet cmd)
b411b363
PR		 1303{
1304 int ok;
1305 struct p_block_req p;
1306
fd340c12 1307 prepare_header(mdev, &p.head, cmd, sizeof(p) - sizeof(struct p_header) + digest_size);
b411b363 1308 p.sector = cpu_to_be64(sector);
9a8e7753 1309 p.block_id = ID_SYNCER /* unused */;
b411b363
PR		 1310 p.blksize = cpu_to_be32(size);
1311
e42325a5 1312 mutex_lock(&mdev->tconn->data.mutex);
b411b363 1313
bedbd2a5
PR		 1314 ok = (sizeof(p) == drbd_send(mdev->tconn, mdev->tconn->data.socket, &p, sizeof(p), 0));
1315 ok = ok && (digest_size == drbd_send(mdev->tconn, mdev->tconn->data.socket, digest, digest_size, 0));
b411b363 1316
e42325a5 1317 mutex_unlock(&mdev->tconn->data.mutex);
b411b363
PR		 1318
1319 return ok;
1320}
1321
1322int drbd_send_ov_request(struct drbd_conf *mdev, sector_t sector, int size)
1323{
1324 int ok;
1325 struct p_block_req p;
1326
1327 p.sector = cpu_to_be64(sector);
9a8e7753 1328 p.block_id = ID_SYNCER /* unused */;
b411b363
PR		 1329 p.blksize = cpu_to_be32(size);
1330
c012949a 1331 ok = drbd_send_cmd(mdev, USE_DATA_SOCKET, P_OV_REQUEST, &p.head, sizeof(p));
b411b363
PR		 1332 return ok;
1333}
1334
1335/* called on sndtimeo
81e84650
AG		 1336 * returns false if we should retry,
1337 * true if we think connection is dead
b411b363 1338 */
1a7ba646 1339static int we_should_drop_the_connection(struct drbd_tconn *tconn, struct socket *sock)
b411b363
PR		 1340{
1341 int drop_it;
1342 /* long elapsed = (long)(jiffies - mdev->last_received); */
1343
1a7ba646
PR		 1344 drop_it = tconn->meta.socket == sock
1345 || !tconn->asender.task
1346 || get_t_state(&tconn->asender) != RUNNING
1347 || tconn->volume0->state.conn < C_CONNECTED;
b411b363
PR		 1348
1349 if (drop_it)
81e84650 1350 return true;
b411b363 1351
1a7ba646 1352 drop_it = !--tconn->ko_count;
b411b363 1353 if (!drop_it) {
1a7ba646
PR		 1354 conn_err(tconn, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
1355 current->comm, current->pid, tconn->ko_count);
1356 request_ping(tconn);
b411b363
PR		 1357 }
1358
1359 return drop_it; /* && (mdev->state == R_PRIMARY) */;
1360}
1361
1a7ba646 1362static void drbd_update_congested(struct drbd_tconn *tconn)
9e204cdd 1363{
1a7ba646 1364 struct sock *sk = tconn->data.socket->sk;
9e204cdd 1365 if (sk->sk_wmem_queued > sk->sk_sndbuf * 4 / 5)
1a7ba646 1366 set_bit(NET_CONGESTED, &tconn->flags);
9e204cdd
AG		 1367}
1368
b411b363
PR		 1369/* The idea of sendpage seems to be to put some kind of reference
1370 * to the page into the skb, and to hand it over to the NIC. In
1371 * this process get_page() gets called.
1372 *
1373 * As soon as the page was really sent over the network put_page()
1374 * gets called by some part of the network layer. [ NIC driver? ]
1375 *
1376 * [ get_page() / put_page() increment/decrement the count. If count
1377 * reaches 0 the page will be freed. ]
1378 *
1379 * This works nicely with pages from FSs.
1380 * But this means that in protocol A we might signal IO completion too early!
1381 *
1382 * In order not to corrupt data during a resync we must make sure
1383 * that we do not reuse our own buffer pages (EEs) to early, therefore
1384 * we have the net_ee list.
1385 *
1386 * XFS seems to have problems, still, it submits pages with page_count == 0!
1387 * As a workaround, we disable sendpage on pages
1388 * with page_count == 0 or PageSlab.
1389 */
1390static int _drbd_no_send_page(struct drbd_conf *mdev, struct page *page,
ba11ad9a 1391 int offset, size_t size, unsigned msg_flags)
b411b363 1392{
bedbd2a5 1393 int sent = drbd_send(mdev->tconn, mdev->tconn->data.socket, kmap(page) + offset, size, msg_flags);
b411b363
PR		 1394 kunmap(page);
1395 if (sent == size)
1396 mdev->send_cnt += size>>9;
1397 return sent == size;
1398}
1399
1400static int _drbd_send_page(struct drbd_conf *mdev, struct page *page,
ba11ad9a 1401 int offset, size_t size, unsigned msg_flags)
b411b363
PR		 1402{
1403 mm_segment_t oldfs = get_fs();
1404 int sent, ok;
1405 int len = size;
1406
1407 /* e.g. XFS meta- & log-data is in slab pages, which have a
1408 * page_count of 0 and/or have PageSlab() set.
1409 * we cannot use send_page for those, as that does get_page();
1410 * put_page(); and would cause either a VM_BUG directly, or
1411 * __page_cache_release a page that would actually still be referenced
1412 * by someone, leading to some obscure delayed Oops somewhere else. */
1413 if (disable_sendpage || (page_count(page) < 1) || PageSlab(page))
ba11ad9a 1414 return _drbd_no_send_page(mdev, page, offset, size, msg_flags);
b411b363 1415
ba11ad9a 1416 msg_flags |= MSG_NOSIGNAL;
1a7ba646 1417 drbd_update_congested(mdev->tconn);
b411b363
PR		 1418 set_fs(KERNEL_DS);
1419 do {
e42325a5 1420 sent = mdev->tconn->data.socket->ops->sendpage(mdev->tconn->data.socket, page,
b411b363 1421 offset, len,
ba11ad9a 1422 msg_flags);
b411b363 1423 if (sent == -EAGAIN) {
1a7ba646 1424 if (we_should_drop_the_connection(mdev->tconn,
e42325a5 1425 mdev->tconn->data.socket))
b411b363
PR		 1426 break;
1427 else
1428 continue;
1429 }
1430 if (sent <= 0) {
1431 dev_warn(DEV, "%s: size=%d len=%d sent=%d\n",
1432 __func__, (int)size, len, sent);
1433 break;
1434 }
1435 len -= sent;
1436 offset += sent;
1437 } while (len > 0 /* THINK && mdev->cstate >= C_CONNECTED*/);
1438 set_fs(oldfs);
01a311a5 1439 clear_bit(NET_CONGESTED, &mdev->tconn->flags);
b411b363
PR		 1440
1441 ok = (len == 0);
1442 if (likely(ok))
1443 mdev->send_cnt += size>>9;
1444 return ok;
1445}
1446
1447static int _drbd_send_bio(struct drbd_conf *mdev, struct bio *bio)
1448{
1449 struct bio_vec *bvec;
1450 int i;
ba11ad9a 1451 /* hint all but last page with MSG_MORE */
b411b363
PR		 1452 __bio_for_each_segment(bvec, bio, i, 0) {
1453 if (!_drbd_no_send_page(mdev, bvec->bv_page,
ba11ad9a
LE		 1454 bvec->bv_offset, bvec->bv_len,
1455 i == bio->bi_vcnt -1 ? 0 : MSG_MORE))
b411b363
PR		 1456 return 0;
1457 }
1458 return 1;
1459}
1460
1461static int _drbd_send_zc_bio(struct drbd_conf *mdev, struct bio *bio)
1462{
1463 struct bio_vec *bvec;
1464 int i;
ba11ad9a 1465 /* hint all but last page with MSG_MORE */
b411b363
PR		 1466 __bio_for_each_segment(bvec, bio, i, 0) {
1467 if (!_drbd_send_page(mdev, bvec->bv_page,
ba11ad9a
LE		 1468 bvec->bv_offset, bvec->bv_len,
1469 i == bio->bi_vcnt -1 ? 0 : MSG_MORE))
b411b363
PR		 1470 return 0;
1471 }
b411b363
PR		 1472 return 1;
1473}
1474
db830c46
AG		 1475static int _drbd_send_zc_ee(struct drbd_conf *mdev,
1476 struct drbd_peer_request *peer_req)
45bb912b 1477{
db830c46
AG		 1478 struct page *page = peer_req->pages;
1479 unsigned len = peer_req->i.size;
1480
ba11ad9a 1481 /* hint all but last page with MSG_MORE */
45bb912b
LE		 1482 page_chain_for_each(page) {
1483 unsigned l = min_t(unsigned, len, PAGE_SIZE);
ba11ad9a
LE		 1484 if (!_drbd_send_page(mdev, page, 0, l,
1485 page_chain_next(page) ? MSG_MORE : 0))
45bb912b
LE		 1486 return 0;
1487 len -= l;
1488 }
1489 return 1;
1490}
1491
76d2e7ec
PR		 1492static u32 bio_flags_to_wire(struct drbd_conf *mdev, unsigned long bi_rw)
1493{
31890f4a 1494 if (mdev->tconn->agreed_pro_version >= 95)
76d2e7ec 1495 return (bi_rw & REQ_SYNC ? DP_RW_SYNC : 0) |
76d2e7ec
PR		 1496 (bi_rw & REQ_FUA ? DP_FUA : 0) |
1497 (bi_rw & REQ_FLUSH ? DP_FLUSH : 0) |
1498 (bi_rw & REQ_DISCARD ? DP_DISCARD : 0);
1499 else
721a9602 1500 return bi_rw & REQ_SYNC ? DP_RW_SYNC : 0;
76d2e7ec
PR		 1501}
1502
b411b363
PR		 1503/* Used to send write requests
1504 * R_PRIMARY -> Peer (P_DATA)
1505 */
1506int drbd_send_dblock(struct drbd_conf *mdev, struct drbd_request *req)
1507{
1508 int ok = 1;
1509 struct p_data p;
1510 unsigned int dp_flags = 0;
1511 void *dgb;
1512 int dgs;
1513
61120870 1514 if (!drbd_get_data_sock(mdev->tconn))
b411b363
PR		 1515 return 0;
1516
a0638456
PR		 1517 dgs = (mdev->tconn->agreed_pro_version >= 87 && mdev->tconn->integrity_w_tfm) ?
1518 crypto_hash_digestsize(mdev->tconn->integrity_w_tfm) : 0;
b411b363 1519
fd340c12 1520 prepare_header(mdev, &p.head, P_DATA, sizeof(p) - sizeof(struct p_header) + dgs + req->i.size);
ace652ac 1521 p.sector = cpu_to_be64(req->i.sector);
b411b363 1522 p.block_id = (unsigned long)req;
fd340c12 1523 p.seq_num = cpu_to_be32(req->seq_num = atomic_add_return(1, &mdev->packet_seq));
b411b363 1524
76d2e7ec
PR		 1525 dp_flags = bio_flags_to_wire(mdev, req->master_bio->bi_rw);
1526
b411b363
PR		 1527 if (mdev->state.conn >= C_SYNC_SOURCE &&
1528 mdev->state.conn <= C_PAUSED_SYNC_T)
1529 dp_flags |= DP_MAY_SET_IN_SYNC;
1530
1531 p.dp_flags = cpu_to_be32(dp_flags);
b411b363
PR		 1532 set_bit(UNPLUG_REMOTE, &mdev->flags);
1533 ok = (sizeof(p) ==
bedbd2a5 1534 drbd_send(mdev->tconn, mdev->tconn->data.socket, &p, sizeof(p), dgs ? MSG_MORE : 0));
b411b363 1535 if (ok && dgs) {
a0638456
PR		 1536 dgb = mdev->tconn->int_dig_out;
1537 drbd_csum_bio(mdev, mdev->tconn->integrity_w_tfm, req->master_bio, dgb);
bedbd2a5 1538 ok = dgs == drbd_send(mdev->tconn, mdev->tconn->data.socket, dgb, dgs, 0);
b411b363
PR		 1539 }
1540 if (ok) {
470be44a
LE		 1541 /* For protocol A, we have to memcpy the payload into
1542 * socket buffers, as we may complete right away
1543 * as soon as we handed it over to tcp, at which point the data
1544 * pages may become invalid.
1545 *
1546 * For data-integrity enabled, we copy it as well, so we can be
1547 * sure that even if the bio pages may still be modified, it
1548 * won't change the data on the wire, thus if the digest checks
1549 * out ok after sending on this side, but does not fit on the
1550 * receiving side, we sure have detected corruption elsewhere.
1551 */
89e58e75 1552 if (mdev->tconn->net_conf->wire_protocol == DRBD_PROT_A || dgs)
b411b363
PR		 1553 ok = _drbd_send_bio(mdev, req->master_bio);
1554 else
1555 ok = _drbd_send_zc_bio(mdev, req->master_bio);
470be44a
LE		 1556
1557 /* double check digest, sometimes buffers have been modified in flight. */
1558 if (dgs > 0 && dgs <= 64) {
24c4830c 1559 /* 64 byte, 512 bit, is the largest digest size
470be44a
LE		 1560 * currently supported in kernel crypto. */
1561 unsigned char digest[64];
a0638456
PR		 1562 drbd_csum_bio(mdev, mdev->tconn->integrity_w_tfm, req->master_bio, digest);
1563 if (memcmp(mdev->tconn->int_dig_out, digest, dgs)) {
470be44a
LE		 1564 dev_warn(DEV,
1565 "Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
ace652ac 1566 (unsigned long long)req->i.sector, req->i.size);
470be44a
LE		 1567 }
1568 } /* else if (dgs > 64) {
1569 ... Be noisy about digest too large ...
1570 } */
b411b363
PR		 1571 }
1572
61120870 1573 drbd_put_data_sock(mdev->tconn);
bd26bfc5 1574
b411b363
PR		 1575 return ok;
1576}
1577
1578/* answer packet, used to send data back for read requests:
1579 * Peer -> (diskless) R_PRIMARY (P_DATA_REPLY)
1580 * C_SYNC_SOURCE -> C_SYNC_TARGET (P_RS_DATA_REPLY)
1581 */
d8763023 1582int drbd_send_block(struct drbd_conf *mdev, enum drbd_packet cmd,
db830c46 1583 struct drbd_peer_request *peer_req)
b411b363
PR		 1584{
1585 int ok;
1586 struct p_data p;
1587 void *dgb;
1588 int dgs;
1589
a0638456
PR		 1590 dgs = (mdev->tconn->agreed_pro_version >= 87 && mdev->tconn->integrity_w_tfm) ?
1591 crypto_hash_digestsize(mdev->tconn->integrity_w_tfm) : 0;
b411b363 1592
db830c46
AG		 1593 prepare_header(mdev, &p.head, cmd, sizeof(p) -
1594 sizeof(struct p_header80) +
1595 dgs + peer_req->i.size);
1596 p.sector = cpu_to_be64(peer_req->i.sector);
1597 p.block_id = peer_req->block_id;
cc378270 1598 p.seq_num = 0; /* unused */
b411b363
PR		 1599
1600 /* Only called by our kernel thread.
1601 * This one may be interrupted by DRBD_SIG and/or DRBD_SIGKILL
1602 * in response to admin command or module unload.
1603 */
61120870 1604 if (!drbd_get_data_sock(mdev->tconn))
b411b363
PR		 1605 return 0;
1606
bedbd2a5 1607 ok = sizeof(p) == drbd_send(mdev->tconn, mdev->tconn->data.socket, &p, sizeof(p), dgs ? MSG_MORE : 0);
b411b363 1608 if (ok && dgs) {
a0638456 1609 dgb = mdev->tconn->int_dig_out;
db830c46 1610 drbd_csum_ee(mdev, mdev->tconn->integrity_w_tfm, peer_req, dgb);
bedbd2a5 1611 ok = dgs == drbd_send(mdev->tconn, mdev->tconn->data.socket, dgb, dgs, 0);
b411b363
PR		 1612 }
1613 if (ok)
db830c46 1614 ok = _drbd_send_zc_ee(mdev, peer_req);
b411b363 1615
61120870 1616 drbd_put_data_sock(mdev->tconn);
bd26bfc5 1617
b411b363
PR		 1618 return ok;
1619}
1620
73a01a18
PR		 1621int drbd_send_oos(struct drbd_conf *mdev, struct drbd_request *req)
1622{
1623 struct p_block_desc p;
1624
ace652ac
AG		 1625 p.sector = cpu_to_be64(req->i.sector);
1626 p.blksize = cpu_to_be32(req->i.size);
73a01a18
PR		 1627
1628 return drbd_send_cmd(mdev, USE_DATA_SOCKET, P_OUT_OF_SYNC, &p.head, sizeof(p));
1629}
1630
b411b363
PR		 1631/*
1632 drbd_send distinguishes two cases:
1633
1634 Packets sent via the data socket "sock"
1635 and packets sent via the meta data socket "msock"
1636
1637 sock msock
1638 -----------------+-------------------------+------------------------------
1639 timeout conf.timeout / 2 conf.timeout / 2
1640 timeout action send a ping via msock Abort communication
1641 and close all sockets
1642*/
1643
1644/*
1645 * you must have down()ed the appropriate [m]sock_mutex elsewhere!
1646 */
bedbd2a5 1647int drbd_send(struct drbd_tconn *tconn, struct socket *sock,
b411b363
PR		 1648 void *buf, size_t size, unsigned msg_flags)
1649{
1650 struct kvec iov;
1651 struct msghdr msg;
1652 int rv, sent = 0;
1653
1654 if (!sock)
1655 return -1000;
1656
1657 /* THINK if (signal_pending) return ... ? */
1658
1659 iov.iov_base = buf;
1660 iov.iov_len = size;
1661
1662 msg.msg_name = NULL;
1663 msg.msg_namelen = 0;
1664 msg.msg_control = NULL;
1665 msg.msg_controllen = 0;
1666 msg.msg_flags = msg_flags | MSG_NOSIGNAL;
1667
bedbd2a5
PR		 1668 if (sock == tconn->data.socket) {
1669 tconn->ko_count = tconn->net_conf->ko_count;
1670 drbd_update_congested(tconn);
b411b363
PR		 1671 }
1672 do {
1673 /* STRANGE
1674 * tcp_sendmsg does _not_ use its size parameter at all ?
1675 *
1676 * -EAGAIN on timeout, -EINTR on signal.
1677 */
1678/* THINK
1679 * do we need to block DRBD_SIG if sock == &meta.socket ??
1680 * otherwise wake_asender() might interrupt some send_*Ack !
1681 */
1682 rv = kernel_sendmsg(sock, &msg, &iov, 1, size);
1683 if (rv == -EAGAIN) {
bedbd2a5 1684 if (we_should_drop_the_connection(tconn, sock))
b411b363
PR		 1685 break;
1686 else
1687 continue;
1688 }
b411b363
PR		 1689 if (rv == -EINTR) {
1690 flush_signals(current);
1691 rv = 0;
1692 }
1693 if (rv < 0)
1694 break;
1695 sent += rv;
1696 iov.iov_base += rv;
1697 iov.iov_len -= rv;
1698 } while (sent < size);
1699
bedbd2a5
PR		 1700 if (sock == tconn->data.socket)
1701 clear_bit(NET_CONGESTED, &tconn->flags);
b411b363
PR		 1702
1703 if (rv <= 0) {
1704 if (rv != -EAGAIN) {
bedbd2a5
PR		 1705 conn_err(tconn, "%s_sendmsg returned %d\n",
1706 sock == tconn->meta.socket ? "msock" : "sock",
1707 rv);
1708 drbd_force_state(tconn->volume0, NS(conn, C_BROKEN_PIPE));
b411b363 1709 } else
bedbd2a5 1710 drbd_force_state(tconn->volume0, NS(conn, C_TIMEOUT));
b411b363
PR		 1711 }
1712
1713 return sent;
1714}
1715
1716static int drbd_open(struct block_device *bdev, fmode_t mode)
1717{
1718 struct drbd_conf *mdev = bdev->bd_disk->private_data;
1719 unsigned long flags;
1720 int rv = 0;
1721
2a48fc0a 1722 mutex_lock(&drbd_main_mutex);
87eeee41 1723 spin_lock_irqsave(&mdev->tconn->req_lock, flags);
b411b363
PR		 1724 /* to have a stable mdev->state.role
1725 * and no race with updating open_cnt */
1726
1727 if (mdev->state.role != R_PRIMARY) {
1728 if (mode & FMODE_WRITE)
1729 rv = -EROFS;
1730 else if (!allow_oos)
1731 rv = -EMEDIUMTYPE;
1732 }
1733
1734 if (!rv)
1735 mdev->open_cnt++;
87eeee41 1736 spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);
2a48fc0a 1737 mutex_unlock(&drbd_main_mutex);
b411b363
PR		 1738
1739 return rv;
1740}
1741
1742static int drbd_release(struct gendisk *gd, fmode_t mode)
1743{
1744 struct drbd_conf *mdev = gd->private_data;
2a48fc0a 1745 mutex_lock(&drbd_main_mutex);
b411b363 1746 mdev->open_cnt--;
2a48fc0a 1747 mutex_unlock(&drbd_main_mutex);
b411b363
PR		 1748 return 0;
1749}
1750
b411b363
PR		 1751static void drbd_set_defaults(struct drbd_conf *mdev)
1752{
85f4cc17
PR		 1753 /* This way we get a compile error when sync_conf grows,
1754 and we forgot to initialize it here */
1755 mdev->sync_conf = (struct syncer_conf) {
1756 /* .rate = */ DRBD_RATE_DEF,
1757 /* .after = */ DRBD_AFTER_DEF,
1758 /* .al_extents = */ DRBD_AL_EXTENTS_DEF,
85f4cc17
PR		 1759 /* .verify_alg = */ {}, 0,
1760 /* .cpu_mask = */ {}, 0,
1761 /* .csums_alg = */ {}, 0,
e756414f 1762 /* .use_rle = */ 0,
9a31d716
PR		 1763 /* .on_no_data = */ DRBD_ON_NO_DATA_DEF,
1764 /* .c_plan_ahead = */ DRBD_C_PLAN_AHEAD_DEF,
1765 /* .c_delay_target = */ DRBD_C_DELAY_TARGET_DEF,
1766 /* .c_fill_target = */ DRBD_C_FILL_TARGET_DEF,
0f0601f4
LE		 1767 /* .c_max_rate = */ DRBD_C_MAX_RATE_DEF,
1768 /* .c_min_rate = */ DRBD_C_MIN_RATE_DEF
85f4cc17
PR		 1769 };
1770
1771 /* Have to use that way, because the layout differs between
1772 big endian and little endian */
b411b363
PR		 1773 mdev->state = (union drbd_state) {
1774 { .role = R_SECONDARY,
1775 .peer = R_UNKNOWN,
1776 .conn = C_STANDALONE,
1777 .disk = D_DISKLESS,
1778 .pdsk = D_UNKNOWN,
fb22c402
PR		 1779 .susp = 0,
1780 .susp_nod = 0,
1781 .susp_fen = 0
b411b363
PR		 1782 } };
1783}
1784
1785void drbd_init_set_defaults(struct drbd_conf *mdev)
1786{
1787 /* the memset(,0,) did most of this.
1788 * note: only assignments, no allocation in here */
1789
1790 drbd_set_defaults(mdev);
1791
b411b363
PR		 1792 atomic_set(&mdev->ap_bio_cnt, 0);
1793 atomic_set(&mdev->ap_pending_cnt, 0);
1794 atomic_set(&mdev->rs_pending_cnt, 0);
1795 atomic_set(&mdev->unacked_cnt, 0);
1796 atomic_set(&mdev->local_cnt, 0);
b411b363 1797 atomic_set(&mdev->pp_in_use, 0);
435f0740 1798 atomic_set(&mdev->pp_in_use_by_net, 0);
778f271d 1799 atomic_set(&mdev->rs_sect_in, 0);
0f0601f4 1800 atomic_set(&mdev->rs_sect_ev, 0);
759fbdfb 1801 atomic_set(&mdev->ap_in_flight, 0);
b411b363
PR		 1802
1803 mutex_init(&mdev->md_io_mutex);
e42325a5
PR		 1804 mutex_init(&mdev->tconn->data.mutex);
1805 mutex_init(&mdev->tconn->meta.mutex);
1806 sema_init(&mdev->tconn->data.work.s, 0);
1807 sema_init(&mdev->tconn->meta.work.s, 0);
b411b363
PR		 1808 mutex_init(&mdev->state_mutex);
1809
e42325a5
PR		 1810 spin_lock_init(&mdev->tconn->data.work.q_lock);
1811 spin_lock_init(&mdev->tconn->meta.work.q_lock);
b411b363
PR		 1812
1813 spin_lock_init(&mdev->al_lock);
87eeee41 1814 spin_lock_init(&mdev->tconn->req_lock);
b411b363
PR		 1815 spin_lock_init(&mdev->peer_seq_lock);
1816 spin_lock_init(&mdev->epoch_lock);
1817
1818 INIT_LIST_HEAD(&mdev->active_ee);
1819 INIT_LIST_HEAD(&mdev->sync_ee);
1820 INIT_LIST_HEAD(&mdev->done_ee);
1821 INIT_LIST_HEAD(&mdev->read_ee);
1822 INIT_LIST_HEAD(&mdev->net_ee);
1823 INIT_LIST_HEAD(&mdev->resync_reads);
e42325a5
PR		 1824 INIT_LIST_HEAD(&mdev->tconn->data.work.q);
1825 INIT_LIST_HEAD(&mdev->tconn->meta.work.q);
b411b363
PR		 1826 INIT_LIST_HEAD(&mdev->resync_work.list);
1827 INIT_LIST_HEAD(&mdev->unplug_work.list);
e9e6f3ec 1828 INIT_LIST_HEAD(&mdev->go_diskless.list);
b411b363 1829 INIT_LIST_HEAD(&mdev->md_sync_work.list);
c4752ef1 1830 INIT_LIST_HEAD(&mdev->start_resync_work.list);
b411b363 1831 INIT_LIST_HEAD(&mdev->bm_io_work.w.list);
0ced55a3 1832
794abb75 1833 mdev->resync_work.cb = w_resync_timer;
b411b363 1834 mdev->unplug_work.cb = w_send_write_hint;
e9e6f3ec 1835 mdev->go_diskless.cb = w_go_diskless;
b411b363
PR		 1836 mdev->md_sync_work.cb = w_md_sync;
1837 mdev->bm_io_work.w.cb = w_bitmap_io;
370a43e7 1838 mdev->start_resync_work.cb = w_start_resync;
b411b363
PR		 1839 init_timer(&mdev->resync_timer);
1840 init_timer(&mdev->md_sync_timer);
370a43e7 1841 init_timer(&mdev->start_resync_timer);
7fde2be9 1842 init_timer(&mdev->request_timer);
b411b363
PR		 1843 mdev->resync_timer.function = resync_timer_fn;
1844 mdev->resync_timer.data = (unsigned long) mdev;
1845 mdev->md_sync_timer.function = md_sync_timer_fn;
1846 mdev->md_sync_timer.data = (unsigned long) mdev;
370a43e7
PR		 1847 mdev->start_resync_timer.function = start_resync_timer_fn;
1848 mdev->start_resync_timer.data = (unsigned long) mdev;
7fde2be9
PR		 1849 mdev->request_timer.function = request_timer_fn;
1850 mdev->request_timer.data = (unsigned long) mdev;
b411b363
PR		 1851
1852 init_waitqueue_head(&mdev->misc_wait);
1853 init_waitqueue_head(&mdev->state_wait);
1854 init_waitqueue_head(&mdev->ee_wait);
1855 init_waitqueue_head(&mdev->al_wait);
1856 init_waitqueue_head(&mdev->seq_wait);
1857
bed879ae
PR		 1858 drbd_thread_init(mdev, &mdev->tconn->receiver, drbdd_init, "receiver");
1859 drbd_thread_init(mdev, &mdev->tconn->worker, drbd_worker, "worker");
1860 drbd_thread_init(mdev, &mdev->tconn->asender, drbd_asender, "asender");
b411b363 1861
fd340c12 1862 /* mdev->tconn->agreed_pro_version gets initialized in drbd_connect() */
2451fc3b 1863 mdev->write_ordering = WO_bdev_flush;
b411b363 1864 mdev->resync_wenr = LC_FREE;
99432fcc
PR		 1865 mdev->peer_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
1866 mdev->local_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
b411b363
PR		 1867}
1868
1869void drbd_mdev_cleanup(struct drbd_conf *mdev)
1870{
1d7734a0 1871 int i;
e6b3ea83 1872 if (mdev->tconn->receiver.t_state != NONE)
b411b363 1873 dev_err(DEV, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
e6b3ea83 1874 mdev->tconn->receiver.t_state);
b411b363
PR		 1875
1876 /* no need to lock it, I'm the only thread alive */
1877 if (atomic_read(&mdev->current_epoch->epoch_size) != 0)
1878 dev_err(DEV, "epoch_size:%d\n", atomic_read(&mdev->current_epoch->epoch_size));
1879 mdev->al_writ_cnt =
1880 mdev->bm_writ_cnt =
1881 mdev->read_cnt =
1882 mdev->recv_cnt =
1883 mdev->send_cnt =
1884 mdev->writ_cnt =
1885 mdev->p_size =
1886 mdev->rs_start =
1887 mdev->rs_total =
1d7734a0
LE		 1888 mdev->rs_failed = 0;
1889 mdev->rs_last_events = 0;
0f0601f4 1890 mdev->rs_last_sect_ev = 0;
1d7734a0
LE		 1891 for (i = 0; i < DRBD_SYNC_MARKS; i++) {
1892 mdev->rs_mark_left[i] = 0;
1893 mdev->rs_mark_time[i] = 0;
1894 }
89e58e75 1895 D_ASSERT(mdev->tconn->net_conf == NULL);
b411b363
PR		 1896
1897 drbd_set_my_capacity(mdev, 0);
1898 if (mdev->bitmap) {
1899 /* maybe never allocated. */
02d9a94b 1900 drbd_bm_resize(mdev, 0, 1);
b411b363
PR		 1901 drbd_bm_cleanup(mdev);
1902 }
1903
1904 drbd_free_resources(mdev);
0778286a 1905 clear_bit(AL_SUSPENDED, &mdev->flags);
b411b363
PR		 1906
1907 /*
1908 * currently we drbd_init_ee only on module load, so
1909 * we may do drbd_release_ee only on module unload!
1910 */
1911 D_ASSERT(list_empty(&mdev->active_ee));
1912 D_ASSERT(list_empty(&mdev->sync_ee));
1913 D_ASSERT(list_empty(&mdev->done_ee));
1914 D_ASSERT(list_empty(&mdev->read_ee));
1915 D_ASSERT(list_empty(&mdev->net_ee));
1916 D_ASSERT(list_empty(&mdev->resync_reads));
e42325a5
PR		 1917 D_ASSERT(list_empty(&mdev->tconn->data.work.q));
1918 D_ASSERT(list_empty(&mdev->tconn->meta.work.q));
b411b363
PR		 1919 D_ASSERT(list_empty(&mdev->resync_work.list));
1920 D_ASSERT(list_empty(&mdev->unplug_work.list));
e9e6f3ec 1921 D_ASSERT(list_empty(&mdev->go_diskless.list));
2265b473
LE		 1922
1923 drbd_set_defaults(mdev);
b411b363
PR		 1924}
1925
1926
1927static void drbd_destroy_mempools(void)
1928{
1929 struct page *page;
1930
1931 while (drbd_pp_pool) {
1932 page = drbd_pp_pool;
1933 drbd_pp_pool = (struct page *)page_private(page);
1934 __free_page(page);
1935 drbd_pp_vacant--;
1936 }
1937
1938 /* D_ASSERT(atomic_read(&drbd_pp_vacant)==0); */
1939
1940 if (drbd_ee_mempool)
1941 mempool_destroy(drbd_ee_mempool);
1942 if (drbd_request_mempool)
1943 mempool_destroy(drbd_request_mempool);
1944 if (drbd_ee_cache)
1945 kmem_cache_destroy(drbd_ee_cache);
1946 if (drbd_request_cache)
1947 kmem_cache_destroy(drbd_request_cache);
1948 if (drbd_bm_ext_cache)
1949 kmem_cache_destroy(drbd_bm_ext_cache);
1950 if (drbd_al_ext_cache)
1951 kmem_cache_destroy(drbd_al_ext_cache);
1952
1953 drbd_ee_mempool = NULL;
1954 drbd_request_mempool = NULL;
1955 drbd_ee_cache = NULL;
1956 drbd_request_cache = NULL;
1957 drbd_bm_ext_cache = NULL;
1958 drbd_al_ext_cache = NULL;
1959
1960 return;
1961}
1962
1963static int drbd_create_mempools(void)
1964{
1965 struct page *page;
1816a2b4 1966 const int number = (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * minor_count;
b411b363
PR		 1967 int i;
1968
1969 /* prepare our caches and mempools */
1970 drbd_request_mempool = NULL;
1971 drbd_ee_cache = NULL;
1972 drbd_request_cache = NULL;
1973 drbd_bm_ext_cache = NULL;
1974 drbd_al_ext_cache = NULL;
1975 drbd_pp_pool = NULL;
1976
1977 /* caches */
1978 drbd_request_cache = kmem_cache_create(
1979 "drbd_req", sizeof(struct drbd_request), 0, 0, NULL);
1980 if (drbd_request_cache == NULL)
1981 goto Enomem;
1982
1983 drbd_ee_cache = kmem_cache_create(
f6ffca9f 1984 "drbd_ee", sizeof(struct drbd_peer_request), 0, 0, NULL);
b411b363
PR		 1985 if (drbd_ee_cache == NULL)
1986 goto Enomem;
1987
1988 drbd_bm_ext_cache = kmem_cache_create(
1989 "drbd_bm", sizeof(struct bm_extent), 0, 0, NULL);
1990 if (drbd_bm_ext_cache == NULL)
1991 goto Enomem;
1992
1993 drbd_al_ext_cache = kmem_cache_create(
1994 "drbd_al", sizeof(struct lc_element), 0, 0, NULL);
1995 if (drbd_al_ext_cache == NULL)
1996 goto Enomem;
1997
1998 /* mempools */
1999 drbd_request_mempool = mempool_create(number,
2000 mempool_alloc_slab, mempool_free_slab, drbd_request_cache);
2001 if (drbd_request_mempool == NULL)
2002 goto Enomem;
2003
2004 drbd_ee_mempool = mempool_create(number,
2005 mempool_alloc_slab, mempool_free_slab, drbd_ee_cache);
2027ae1f 2006 if (drbd_ee_mempool == NULL)
b411b363
PR		 2007 goto Enomem;
2008
2009 /* drbd's page pool */
2010 spin_lock_init(&drbd_pp_lock);
2011
2012 for (i = 0; i < number; i++) {
2013 page = alloc_page(GFP_HIGHUSER);
2014 if (!page)
2015 goto Enomem;
2016 set_page_private(page, (unsigned long)drbd_pp_pool);
2017 drbd_pp_pool = page;
2018 }
2019 drbd_pp_vacant = number;
2020
2021 return 0;
2022
2023Enomem:
2024 drbd_destroy_mempools(); /* in case we allocated some */
2025 return -ENOMEM;
2026}
2027
2028static int drbd_notify_sys(struct notifier_block *this, unsigned long code,
2029 void *unused)
2030{
2031 /* just so we have it. you never know what interesting things we
2032 * might want to do here some day...
2033 */
2034
2035 return NOTIFY_DONE;
2036}
2037
2038static struct notifier_block drbd_notifier = {
2039 .notifier_call = drbd_notify_sys,
2040};
2041
2042static void drbd_release_ee_lists(struct drbd_conf *mdev)
2043{
2044 int rr;
2045
2046 rr = drbd_release_ee(mdev, &mdev->active_ee);
2047 if (rr)
2048 dev_err(DEV, "%d EEs in active list found!\n", rr);
2049
2050 rr = drbd_release_ee(mdev, &mdev->sync_ee);
2051 if (rr)
2052 dev_err(DEV, "%d EEs in sync list found!\n", rr);
2053
2054 rr = drbd_release_ee(mdev, &mdev->read_ee);
2055 if (rr)
2056 dev_err(DEV, "%d EEs in read list found!\n", rr);
2057
2058 rr = drbd_release_ee(mdev, &mdev->done_ee);
2059 if (rr)
2060 dev_err(DEV, "%d EEs in done list found!\n", rr);
2061
2062 rr = drbd_release_ee(mdev, &mdev->net_ee);
2063 if (rr)
2064 dev_err(DEV, "%d EEs in net list found!\n", rr);
2065}
2066
2067/* caution. no locking.
2068 * currently only used from module cleanup code. */
2069static void drbd_delete_device(unsigned int minor)
2070{
2071 struct drbd_conf *mdev = minor_to_mdev(minor);
2072
2073 if (!mdev)
2074 return;
2075
2076 /* paranoia asserts */
70dc65e1 2077 D_ASSERT(mdev->open_cnt == 0);
e42325a5 2078 D_ASSERT(list_empty(&mdev->tconn->data.work.q));
b411b363
PR		 2079 /* end paranoia asserts */
2080
2081 del_gendisk(mdev->vdisk);
2082
2083 /* cleanup stuff that may have been allocated during
2084 * device (re-)configuration or state changes */
2085
2086 if (mdev->this_bdev)
2087 bdput(mdev->this_bdev);
2088
2089 drbd_free_resources(mdev);
2111438b 2090 drbd_free_tconn(mdev->tconn);
b411b363
PR		 2091
2092 drbd_release_ee_lists(mdev);
2093
b411b363
PR		 2094 lc_destroy(mdev->act_log);
2095 lc_destroy(mdev->resync);
2096
2097 kfree(mdev->p_uuid);
2098 /* mdev->p_uuid = NULL; */
2099
b411b363
PR		 2100 /* cleanup the rest that has been
2101 * allocated from drbd_new_device
2102 * and actually free the mdev itself */
2103 drbd_free_mdev(mdev);
2104}
2105
2106static void drbd_cleanup(void)
2107{
2108 unsigned int i;
2109
2110 unregister_reboot_notifier(&drbd_notifier);
2111
17a93f30
LE		 2112 /* first remove proc,
2113 * drbdsetup uses it's presence to detect
2114 * whether DRBD is loaded.
2115 * If we would get stuck in proc removal,
2116 * but have netlink already deregistered,
2117 * some drbdsetup commands may wait forever
2118 * for an answer.
2119 */
2120 if (drbd_proc)
2121 remove_proc_entry("drbd", NULL);
2122
b411b363
PR		 2123 drbd_nl_cleanup();
2124
2125 if (minor_table) {
b411b363
PR		 2126 i = minor_count;
2127 while (i--)
2128 drbd_delete_device(i);
2129 drbd_destroy_mempools();
2130 }
2131
2132 kfree(minor_table);
2133
2134 unregister_blkdev(DRBD_MAJOR, "drbd");
2135
2136 printk(KERN_INFO "drbd: module cleanup done.\n");
2137}
2138
2139/**
2140 * drbd_congested() - Callback for pdflush
2141 * @congested_data: User data
2142 * @bdi_bits: Bits pdflush is currently interested in
2143 *
2144 * Returns 1<<BDI_async_congested and/or 1<<BDI_sync_congested if we are congested.
2145 */
2146static int drbd_congested(void *congested_data, int bdi_bits)
2147{
2148 struct drbd_conf *mdev = congested_data;
2149 struct request_queue *q;
2150 char reason = '-';
2151 int r = 0;
2152
1b881ef7 2153 if (!may_inc_ap_bio(mdev)) {
b411b363
PR		 2154 /* DRBD has frozen IO */
2155 r = bdi_bits;
2156 reason = 'd';
2157 goto out;
2158 }
2159
2160 if (get_ldev(mdev)) {
2161 q = bdev_get_queue(mdev->ldev->backing_bdev);
2162 r = bdi_congested(&q->backing_dev_info, bdi_bits);
2163 put_ldev(mdev);
2164 if (r)
2165 reason = 'b';
2166 }
2167
01a311a5 2168 if (bdi_bits & (1 << BDI_async_congested) && test_bit(NET_CONGESTED, &mdev->tconn->flags)) {
b411b363
PR		 2169 r |= (1 << BDI_async_congested);
2170 reason = reason == 'b' ? 'a' : 'n';
2171 }
2172
2173out:
2174 mdev->congestion_reason = reason;
2175 return r;
2176}
2177
2111438b
PR		 2178struct drbd_tconn *drbd_new_tconn(char *name)
2179{
2180 struct drbd_tconn *tconn;
2181
2182 tconn = kzalloc(sizeof(struct drbd_tconn), GFP_KERNEL);
2183 if (!tconn)
2184 return NULL;
2185
2186 tconn->name = kstrdup(name, GFP_KERNEL);
2187 if (!tconn->name)
2188 goto fail;
2189
b2fb6dbe
PR		 2190 atomic_set(&tconn->net_cnt, 0);
2191 init_waitqueue_head(&tconn->net_cnt_wait);
062e879c 2192 idr_init(&tconn->volumes);
b2fb6dbe 2193
2111438b
PR		 2194 write_lock_irq(&global_state_lock);
2195 list_add(&tconn->all_tconn, &drbd_tconns);
2196 write_unlock_irq(&global_state_lock);
2197
2198 return tconn;
2199
2200fail:
2201 kfree(tconn->name);
2202 kfree(tconn);
2203
2204 return NULL;
2205}
2206
2207void drbd_free_tconn(struct drbd_tconn *tconn)
2208{
2209 write_lock_irq(&global_state_lock);
2210 list_del(&tconn->all_tconn);
2211 write_unlock_irq(&global_state_lock);
062e879c 2212 idr_destroy(&tconn->volumes);
2111438b
PR		 2213
2214 kfree(tconn->name);
b42a70ad
PR		 2215 kfree(tconn->int_dig_out);
2216 kfree(tconn->int_dig_in);
2217 kfree(tconn->int_dig_vv);
2111438b
PR		 2218 kfree(tconn);
2219}
2220
b411b363
PR		 2221struct drbd_conf *drbd_new_device(unsigned int minor)
2222{
2223 struct drbd_conf *mdev;
2224 struct gendisk *disk;
2225 struct request_queue *q;
60ae4966 2226 char conn_name[9]; /* drbd1234N */
062e879c 2227 int vnr;
b411b363
PR		 2228
2229 /* GFP_KERNEL, we are outside of all write-out paths */
2230 mdev = kzalloc(sizeof(struct drbd_conf), GFP_KERNEL);
2231 if (!mdev)
2232 return NULL;
60ae4966
PR		 2233 sprintf(conn_name, "drbd%d", minor);
2234 mdev->tconn = drbd_new_tconn(conn_name);
2111438b
PR		 2235 if (!mdev->tconn)
2236 goto out_no_tconn;
062e879c
PR		 2237 if (!idr_pre_get(&mdev->tconn->volumes, GFP_KERNEL))
2238 goto out_no_cpumask;
2239 if (idr_get_new(&mdev->tconn->volumes, mdev, &vnr))
2240 goto out_no_cpumask;
2241 if (vnr != 0) {
2242 dev_err(DEV, "vnr = %d\n", vnr);
2243 goto out_no_cpumask;
2244 }
80822284 2245 if (!zalloc_cpumask_var(&mdev->tconn->cpu_mask, GFP_KERNEL))
b411b363
PR		 2246 goto out_no_cpumask;
2247
2111438b 2248 mdev->tconn->volume0 = mdev;
b411b363
PR		 2249 mdev->minor = minor;
2250
2251 drbd_init_set_defaults(mdev);
2252
2253 q = blk_alloc_queue(GFP_KERNEL);
2254 if (!q)
2255 goto out_no_q;
2256 mdev->rq_queue = q;
2257 q->queuedata = mdev;
b411b363
PR		 2258
2259 disk = alloc_disk(1);
2260 if (!disk)
2261 goto out_no_disk;
2262 mdev->vdisk = disk;
2263
81e84650 2264 set_disk_ro(disk, true);
b411b363
PR		 2265
2266 disk->queue = q;
2267 disk->major = DRBD_MAJOR;
2268 disk->first_minor = minor;
2269 disk->fops = &drbd_ops;
2270 sprintf(disk->disk_name, "drbd%d", minor);
2271 disk->private_data = mdev;
2272
2273 mdev->this_bdev = bdget(MKDEV(DRBD_MAJOR, minor));
2274 /* we have no partitions. we contain only ourselves. */
2275 mdev->this_bdev->bd_contains = mdev->this_bdev;
2276
2277 q->backing_dev_info.congested_fn = drbd_congested;
2278 q->backing_dev_info.congested_data = mdev;
2279
2f58dcfc 2280 blk_queue_make_request(q, drbd_make_request);
99432fcc
PR		 2281 /* Setting the max_hw_sectors to an odd value of 8kibyte here
2282 This triggers a max_bio_size message upon first attach or connect */
2283 blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8);
b411b363
PR		 2284 blk_queue_bounce_limit(q, BLK_BOUNCE_ANY);
2285 blk_queue_merge_bvec(q, drbd_merge_bvec);
87eeee41 2286 q->queue_lock = &mdev->tconn->req_lock; /* needed since we use */
b411b363
PR		 2287
2288 mdev->md_io_page = alloc_page(GFP_KERNEL);
2289 if (!mdev->md_io_page)
2290 goto out_no_io_page;
2291
2292 if (drbd_bm_init(mdev))
2293 goto out_no_bitmap;
2294 /* no need to lock access, we are still initializing this minor device. */
2295 if (!tl_init(mdev))
2296 goto out_no_tl;
dac1389c 2297 mdev->read_requests = RB_ROOT;
de696716 2298 mdev->write_requests = RB_ROOT;
b411b363 2299
b411b363
PR		 2300 mdev->current_epoch = kzalloc(sizeof(struct drbd_epoch), GFP_KERNEL);
2301 if (!mdev->current_epoch)
2302 goto out_no_epoch;
2303
2304 INIT_LIST_HEAD(&mdev->current_epoch->list);
2305 mdev->epochs = 1;
2306
2307 return mdev;
2308
2309/* out_whatever_else:
2310 kfree(mdev->current_epoch); */
2311out_no_epoch:
b411b363
PR		 2312 tl_cleanup(mdev);
2313out_no_tl:
2314 drbd_bm_cleanup(mdev);
2315out_no_bitmap:
2316 __free_page(mdev->md_io_page);
2317out_no_io_page:
2318 put_disk(disk);
2319out_no_disk:
2320 blk_cleanup_queue(q);
2321out_no_q:
80822284 2322 free_cpumask_var(mdev->tconn->cpu_mask);
b411b363 2323out_no_cpumask:
2111438b
PR		 2324 drbd_free_tconn(mdev->tconn);
2325out_no_tconn:
b411b363
PR		 2326 kfree(mdev);
2327 return NULL;
2328}
2329
2330/* counterpart of drbd_new_device.
2331 * last part of drbd_delete_device. */
2332void drbd_free_mdev(struct drbd_conf *mdev)
2333{
2334 kfree(mdev->current_epoch);
b411b363
PR		 2335 tl_cleanup(mdev);
2336 if (mdev->bitmap) /* should no longer be there. */
2337 drbd_bm_cleanup(mdev);
2338 __free_page(mdev->md_io_page);
2339 put_disk(mdev->vdisk);
2340 blk_cleanup_queue(mdev->rq_queue);
b411b363
PR		 2341 kfree(mdev);
2342}
2343
2344
2345int __init drbd_init(void)
2346{
2347 int err;
2348
fd340c12
PR		 2349 BUILD_BUG_ON(sizeof(struct p_header80) != sizeof(struct p_header95));
2350 BUILD_BUG_ON(sizeof(struct p_handshake) != 80);
b411b363 2351
2b8a90b5 2352 if (minor_count < DRBD_MINOR_COUNT_MIN || minor_count > DRBD_MINOR_COUNT_MAX) {
b411b363
PR		 2353 printk(KERN_ERR
2354 "drbd: invalid minor_count (%d)\n", minor_count);
2355#ifdef MODULE
2356 return -EINVAL;
2357#else
2358 minor_count = 8;
2359#endif
2360 }
2361
2362 err = drbd_nl_init();
2363 if (err)
2364 return err;
2365
2366 err = register_blkdev(DRBD_MAJOR, "drbd");
2367 if (err) {
2368 printk(KERN_ERR
2369 "drbd: unable to register block device major %d\n",
2370 DRBD_MAJOR);
2371 return err;
2372 }
2373
2374 register_reboot_notifier(&drbd_notifier);
2375
2376 /*
2377 * allocate all necessary structs
2378 */
2379 err = -ENOMEM;
2380
2381 init_waitqueue_head(&drbd_pp_wait);
2382
2383 drbd_proc = NULL; /* play safe for drbd_cleanup */
2384 minor_table = kzalloc(sizeof(struct drbd_conf *)*minor_count,
2385 GFP_KERNEL);
2386 if (!minor_table)
2387 goto Enomem;
2388
2389 err = drbd_create_mempools();
2390 if (err)
2391 goto Enomem;
2392
8c484ee4 2393 drbd_proc = proc_create_data("drbd", S_IFREG | S_IRUGO , NULL, &drbd_proc_fops, NULL);
b411b363
PR		 2394 if (!drbd_proc) {
2395 printk(KERN_ERR "drbd: unable to register proc file\n");
2396 goto Enomem;
2397 }
2398
2399 rwlock_init(&global_state_lock);
2111438b 2400 INIT_LIST_HEAD(&drbd_tconns);
b411b363
PR		 2401
2402 printk(KERN_INFO "drbd: initialized. "
2403 "Version: " REL_VERSION " (api:%d/proto:%d-%d)\n",
2404 API_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX);
2405 printk(KERN_INFO "drbd: %s\n", drbd_buildtag());
2406 printk(KERN_INFO "drbd: registered as block device major %d\n",
2407 DRBD_MAJOR);
2408 printk(KERN_INFO "drbd: minor_table @ 0x%p\n", minor_table);
2409
2410 return 0; /* Success! */
2411
2412Enomem:
2413 drbd_cleanup();
2414 if (err == -ENOMEM)
2415 /* currently always the case */
2416 printk(KERN_ERR "drbd: ran out of memory\n");
2417 else
2418 printk(KERN_ERR "drbd: initialization failure\n");
2419 return err;
2420}
2421
2422void drbd_free_bc(struct drbd_backing_dev *ldev)
2423{
2424 if (ldev == NULL)
2425 return;
2426
e525fd89
TH		 2427 blkdev_put(ldev->backing_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
2428 blkdev_put(ldev->md_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
b411b363
PR		 2429
2430 kfree(ldev);
2431}
2432
2433void drbd_free_sock(struct drbd_conf *mdev)
2434{
e42325a5
PR		 2435 if (mdev->tconn->data.socket) {
2436 mutex_lock(&mdev->tconn->data.mutex);
2437 kernel_sock_shutdown(mdev->tconn->data.socket, SHUT_RDWR);
2438 sock_release(mdev->tconn->data.socket);
2439 mdev->tconn->data.socket = NULL;
2440 mutex_unlock(&mdev->tconn->data.mutex);
b411b363 2441 }
e42325a5
PR		 2442 if (mdev->tconn->meta.socket) {
2443 mutex_lock(&mdev->tconn->meta.mutex);
2444 kernel_sock_shutdown(mdev->tconn->meta.socket, SHUT_RDWR);
2445 sock_release(mdev->tconn->meta.socket);
2446 mdev->tconn->meta.socket = NULL;
2447 mutex_unlock(&mdev->tconn->meta.mutex);
b411b363
PR		 2448 }
2449}
2450
2451
2452void drbd_free_resources(struct drbd_conf *mdev)
2453{
2454 crypto_free_hash(mdev->csums_tfm);
2455 mdev->csums_tfm = NULL;
2456 crypto_free_hash(mdev->verify_tfm);
2457 mdev->verify_tfm = NULL;
a0638456
PR		 2458 crypto_free_hash(mdev->tconn->cram_hmac_tfm);
2459 mdev->tconn->cram_hmac_tfm = NULL;
2460 crypto_free_hash(mdev->tconn->integrity_w_tfm);
2461 mdev->tconn->integrity_w_tfm = NULL;
2462 crypto_free_hash(mdev->tconn->integrity_r_tfm);
2463 mdev->tconn->integrity_r_tfm = NULL;
b411b363
PR		 2464
2465 drbd_free_sock(mdev);
2466
2467 __no_warn(local,
2468 drbd_free_bc(mdev->ldev);
2469 mdev->ldev = NULL;);
2470}
2471
2472/* meta data management */
2473
2474struct meta_data_on_disk {
2475 u64 la_size; /* last agreed size. */
2476 u64 uuid[UI_SIZE]; /* UUIDs. */
2477 u64 device_uuid;
2478 u64 reserved_u64_1;
2479 u32 flags; /* MDF */
2480 u32 magic;
2481 u32 md_size_sect;
2482 u32 al_offset; /* offset to this block */
2483 u32 al_nr_extents; /* important for restoring the AL */
2484 /* `-- act_log->nr_elements <-- sync_conf.al_extents */
2485 u32 bm_offset; /* offset to the bitmap, from here */
2486 u32 bm_bytes_per_bit; /* BM_BLOCK_SIZE */
99432fcc
PR		 2487 u32 la_peer_max_bio_size; /* last peer max_bio_size */
2488 u32 reserved_u32[3];
b411b363
PR		 2489
2490} __packed;
2491
2492/**
2493 * drbd_md_sync() - Writes the meta data super block if the MD_DIRTY flag bit is set
2494 * @mdev: DRBD device.
2495 */
2496void drbd_md_sync(struct drbd_conf *mdev)
2497{
2498 struct meta_data_on_disk *buffer;
2499 sector_t sector;
2500 int i;
2501
ee15b038
LE		 2502 del_timer(&mdev->md_sync_timer);
2503 /* timer may be rearmed by drbd_md_mark_dirty() now. */
b411b363
PR		 2504 if (!test_and_clear_bit(MD_DIRTY, &mdev->flags))
2505 return;
b411b363
PR		 2506
2507 /* We use here D_FAILED and not D_ATTACHING because we try to write
2508 * metadata even if we detach due to a disk failure! */
2509 if (!get_ldev_if_state(mdev, D_FAILED))
2510 return;
2511
b411b363
PR		 2512 mutex_lock(&mdev->md_io_mutex);
2513 buffer = (struct meta_data_on_disk *)page_address(mdev->md_io_page);
2514 memset(buffer, 0, 512);
2515
2516 buffer->la_size = cpu_to_be64(drbd_get_capacity(mdev->this_bdev));
2517 for (i = UI_CURRENT; i < UI_SIZE; i++)
2518 buffer->uuid[i] = cpu_to_be64(mdev->ldev->md.uuid[i]);
2519 buffer->flags = cpu_to_be32(mdev->ldev->md.flags);
2520 buffer->magic = cpu_to_be32(DRBD_MD_MAGIC);
2521
2522 buffer->md_size_sect = cpu_to_be32(mdev->ldev->md.md_size_sect);
2523 buffer->al_offset = cpu_to_be32(mdev->ldev->md.al_offset);
2524 buffer->al_nr_extents = cpu_to_be32(mdev->act_log->nr_elements);
2525 buffer->bm_bytes_per_bit = cpu_to_be32(BM_BLOCK_SIZE);
2526 buffer->device_uuid = cpu_to_be64(mdev->ldev->md.device_uuid);
2527
2528 buffer->bm_offset = cpu_to_be32(mdev->ldev->md.bm_offset);
99432fcc 2529 buffer->la_peer_max_bio_size = cpu_to_be32(mdev->peer_max_bio_size);
b411b363
PR		 2530
2531 D_ASSERT(drbd_md_ss__(mdev, mdev->ldev) == mdev->ldev->md.md_offset);
2532 sector = mdev->ldev->md.md_offset;
2533
3f3a9b84 2534 if (!drbd_md_sync_page_io(mdev, mdev->ldev, sector, WRITE)) {
b411b363
PR		 2535 /* this was a try anyways ... */
2536 dev_err(DEV, "meta data update failed!\n");
81e84650 2537 drbd_chk_io_error(mdev, 1, true);
b411b363
PR		 2538 }
2539
2540 /* Update mdev->ldev->md.la_size_sect,
2541 * since we updated it on metadata. */
2542 mdev->ldev->md.la_size_sect = drbd_get_capacity(mdev->this_bdev);
2543
2544 mutex_unlock(&mdev->md_io_mutex);
2545 put_ldev(mdev);
2546}
2547
2548/**
2549 * drbd_md_read() - Reads in the meta data super block
2550 * @mdev: DRBD device.
2551 * @bdev: Device from which the meta data should be read in.
2552 *
116676ca 2553 * Return 0 (NO_ERROR) on success, and an enum drbd_ret_code in case
b411b363
PR		 2554 * something goes wrong. Currently only: ERR_IO_MD_DISK, ERR_MD_INVALID.
2555 */
2556int drbd_md_read(struct drbd_conf *mdev, struct drbd_backing_dev *bdev)
2557{
2558 struct meta_data_on_disk *buffer;
2559 int i, rv = NO_ERROR;
2560
2561 if (!get_ldev_if_state(mdev, D_ATTACHING))
2562 return ERR_IO_MD_DISK;
2563
b411b363
PR		 2564 mutex_lock(&mdev->md_io_mutex);
2565 buffer = (struct meta_data_on_disk *)page_address(mdev->md_io_page);
2566
2567 if (!drbd_md_sync_page_io(mdev, bdev, bdev->md.md_offset, READ)) {
25985edc 2568 /* NOTE: can't do normal error processing here as this is
b411b363
PR		 2569 called BEFORE disk is attached */
2570 dev_err(DEV, "Error while reading metadata.\n");
2571 rv = ERR_IO_MD_DISK;
2572 goto err;
2573 }
2574
e7fad8af 2575 if (buffer->magic != cpu_to_be32(DRBD_MD_MAGIC)) {
b411b363
PR		 2576 dev_err(DEV, "Error while reading metadata, magic not found.\n");
2577 rv = ERR_MD_INVALID;
2578 goto err;
2579 }
2580 if (be32_to_cpu(buffer->al_offset) != bdev->md.al_offset) {
2581 dev_err(DEV, "unexpected al_offset: %d (expected %d)\n",
2582 be32_to_cpu(buffer->al_offset), bdev->md.al_offset);
2583 rv = ERR_MD_INVALID;
2584 goto err;
2585 }
2586 if (be32_to_cpu(buffer->bm_offset) != bdev->md.bm_offset) {
2587 dev_err(DEV, "unexpected bm_offset: %d (expected %d)\n",
2588 be32_to_cpu(buffer->bm_offset), bdev->md.bm_offset);
2589 rv = ERR_MD_INVALID;
2590 goto err;
2591 }
2592 if (be32_to_cpu(buffer->md_size_sect) != bdev->md.md_size_sect) {
2593 dev_err(DEV, "unexpected md_size: %u (expected %u)\n",
2594 be32_to_cpu(buffer->md_size_sect), bdev->md.md_size_sect);
2595 rv = ERR_MD_INVALID;
2596 goto err;
2597 }
2598
2599 if (be32_to_cpu(buffer->bm_bytes_per_bit) != BM_BLOCK_SIZE) {
2600 dev_err(DEV, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
2601 be32_to_cpu(buffer->bm_bytes_per_bit), BM_BLOCK_SIZE);
2602 rv = ERR_MD_INVALID;
2603 goto err;
2604 }
2605
2606 bdev->md.la_size_sect = be64_to_cpu(buffer->la_size);
2607 for (i = UI_CURRENT; i < UI_SIZE; i++)
2608 bdev->md.uuid[i] = be64_to_cpu(buffer->uuid[i]);
2609 bdev->md.flags = be32_to_cpu(buffer->flags);
2610 mdev->sync_conf.al_extents = be32_to_cpu(buffer->al_nr_extents);
2611 bdev->md.device_uuid = be64_to_cpu(buffer->device_uuid);
2612
87eeee41 2613 spin_lock_irq(&mdev->tconn->req_lock);
99432fcc
PR		 2614 if (mdev->state.conn < C_CONNECTED) {
2615 int peer;
2616 peer = be32_to_cpu(buffer->la_peer_max_bio_size);
2617 peer = max_t(int, peer, DRBD_MAX_BIO_SIZE_SAFE);
2618 mdev->peer_max_bio_size = peer;
2619 }
87eeee41 2620 spin_unlock_irq(&mdev->tconn->req_lock);
99432fcc 2621
b411b363
PR		 2622 if (mdev->sync_conf.al_extents < 7)
2623 mdev->sync_conf.al_extents = 127;
2624
2625 err:
2626 mutex_unlock(&mdev->md_io_mutex);
2627 put_ldev(mdev);
2628
2629 return rv;
2630}
2631
2632/**
2633 * drbd_md_mark_dirty() - Mark meta data super block as dirty
2634 * @mdev: DRBD device.
2635 *
2636 * Call this function if you change anything that should be written to
2637 * the meta-data super block. This function sets MD_DIRTY, and starts a
2638 * timer that ensures that within five seconds you have to call drbd_md_sync().
2639 */
ca0e6098 2640#ifdef DEBUG
ee15b038
LE		 2641void drbd_md_mark_dirty_(struct drbd_conf *mdev, unsigned int line, const char *func)
2642{
2643 if (!test_and_set_bit(MD_DIRTY, &mdev->flags)) {
2644 mod_timer(&mdev->md_sync_timer, jiffies + HZ);
2645 mdev->last_md_mark_dirty.line = line;
2646 mdev->last_md_mark_dirty.func = func;
2647 }
2648}
2649#else
b411b363
PR		 2650void drbd_md_mark_dirty(struct drbd_conf *mdev)
2651{
ee15b038 2652 if (!test_and_set_bit(MD_DIRTY, &mdev->flags))
ca0e6098 2653 mod_timer(&mdev->md_sync_timer, jiffies + 5*HZ);
b411b363 2654}
ee15b038 2655#endif
b411b363
PR		 2656
2657static void drbd_uuid_move_history(struct drbd_conf *mdev) __must_hold(local)
2658{
2659 int i;
2660
62b0da3a 2661 for (i = UI_HISTORY_START; i < UI_HISTORY_END; i++)
b411b363 2662 mdev->ldev->md.uuid[i+1] = mdev->ldev->md.uuid[i];
b411b363
PR		 2663}
2664
2665void _drbd_uuid_set(struct drbd_conf *mdev, int idx, u64 val) __must_hold(local)
2666{
2667 if (idx == UI_CURRENT) {
2668 if (mdev->state.role == R_PRIMARY)
2669 val |= 1;
2670 else
2671 val &= ~((u64)1);
2672
2673 drbd_set_ed_uuid(mdev, val);
2674 }
2675
2676 mdev->ldev->md.uuid[idx] = val;
b411b363
PR		 2677 drbd_md_mark_dirty(mdev);
2678}
2679
2680
2681void drbd_uuid_set(struct drbd_conf *mdev, int idx, u64 val) __must_hold(local)
2682{
2683 if (mdev->ldev->md.uuid[idx]) {
2684 drbd_uuid_move_history(mdev);
2685 mdev->ldev->md.uuid[UI_HISTORY_START] = mdev->ldev->md.uuid[idx];
b411b363
PR		 2686 }
2687 _drbd_uuid_set(mdev, idx, val);
2688}
2689
2690/**
2691 * drbd_uuid_new_current() - Creates a new current UUID
2692 * @mdev: DRBD device.
2693 *
2694 * Creates a new current UUID, and rotates the old current UUID into
2695 * the bitmap slot. Causes an incremental resync upon next connect.
2696 */
2697void drbd_uuid_new_current(struct drbd_conf *mdev) __must_hold(local)
2698{
2699 u64 val;
62b0da3a
LE		 2700 unsigned long long bm_uuid = mdev->ldev->md.uuid[UI_BITMAP];
2701
2702 if (bm_uuid)
2703 dev_warn(DEV, "bm UUID was already set: %llX\n", bm_uuid);
b411b363 2704
b411b363 2705 mdev->ldev->md.uuid[UI_BITMAP] = mdev->ldev->md.uuid[UI_CURRENT];
b411b363
PR		 2706
2707 get_random_bytes(&val, sizeof(u64));
2708 _drbd_uuid_set(mdev, UI_CURRENT, val);
62b0da3a 2709 drbd_print_uuids(mdev, "new current UUID");
aaa8e2b3
LE		 2710 /* get it to stable storage _now_ */
2711 drbd_md_sync(mdev);
b411b363
PR		 2712}
2713
2714void drbd_uuid_set_bm(struct drbd_conf *mdev, u64 val) __must_hold(local)
2715{
2716 if (mdev->ldev->md.uuid[UI_BITMAP] == 0 && val == 0)
2717 return;
2718
2719 if (val == 0) {
2720 drbd_uuid_move_history(mdev);
2721 mdev->ldev->md.uuid[UI_HISTORY_START] = mdev->ldev->md.uuid[UI_BITMAP];
2722 mdev->ldev->md.uuid[UI_BITMAP] = 0;
b411b363 2723 } else {
62b0da3a
LE		 2724 unsigned long long bm_uuid = mdev->ldev->md.uuid[UI_BITMAP];
2725 if (bm_uuid)
2726 dev_warn(DEV, "bm UUID was already set: %llX\n", bm_uuid);
b411b363 2727
62b0da3a 2728 mdev->ldev->md.uuid[UI_BITMAP] = val & ~((u64)1);
b411b363
PR		 2729 }
2730 drbd_md_mark_dirty(mdev);
2731}
2732
2733/**
2734 * drbd_bmio_set_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
2735 * @mdev: DRBD device.
2736 *
2737 * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
2738 */
2739int drbd_bmio_set_n_write(struct drbd_conf *mdev)
2740{
2741 int rv = -EIO;
2742
2743 if (get_ldev_if_state(mdev, D_ATTACHING)) {
2744 drbd_md_set_flag(mdev, MDF_FULL_SYNC);
2745 drbd_md_sync(mdev);
2746 drbd_bm_set_all(mdev);
2747
2748 rv = drbd_bm_write(mdev);
2749
2750 if (!rv) {
2751 drbd_md_clear_flag(mdev, MDF_FULL_SYNC);
2752 drbd_md_sync(mdev);
2753 }
2754
2755 put_ldev(mdev);
2756 }
2757
2758 return rv;
2759}
2760
2761/**
2762 * drbd_bmio_clear_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
2763 * @mdev: DRBD device.
2764 *
2765 * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
2766 */
2767int drbd_bmio_clear_n_write(struct drbd_conf *mdev)
2768{
2769 int rv = -EIO;
2770
0778286a 2771 drbd_resume_al(mdev);
b411b363
PR		 2772 if (get_ldev_if_state(mdev, D_ATTACHING)) {
2773 drbd_bm_clear_all(mdev);
2774 rv = drbd_bm_write(mdev);
2775 put_ldev(mdev);
2776 }
2777
2778 return rv;
2779}
2780
2781static int w_bitmap_io(struct drbd_conf *mdev, struct drbd_work *w, int unused)
2782{
2783 struct bm_io_work *work = container_of(w, struct bm_io_work, w);
02851e9f 2784 int rv = -EIO;
b411b363
PR		 2785
2786 D_ASSERT(atomic_read(&mdev->ap_bio_cnt) == 0);
2787
02851e9f 2788 if (get_ldev(mdev)) {
20ceb2b2 2789 drbd_bm_lock(mdev, work->why, work->flags);
02851e9f
LE		 2790 rv = work->io_fn(mdev);
2791 drbd_bm_unlock(mdev);
2792 put_ldev(mdev);
2793 }
b411b363
PR		 2794
2795 clear_bit(BITMAP_IO, &mdev->flags);
127b3178 2796 smp_mb__after_clear_bit();
b411b363
PR		 2797 wake_up(&mdev->misc_wait);
2798
2799 if (work->done)
2800 work->done(mdev, rv);
2801
2802 clear_bit(BITMAP_IO_QUEUED, &mdev->flags);
2803 work->why = NULL;
20ceb2b2 2804 work->flags = 0;
b411b363
PR		 2805
2806 return 1;
2807}
2808
82f59cc6
LE		 2809void drbd_ldev_destroy(struct drbd_conf *mdev)
2810{
2811 lc_destroy(mdev->resync);
2812 mdev->resync = NULL;
2813 lc_destroy(mdev->act_log);
2814 mdev->act_log = NULL;
2815 __no_warn(local,
2816 drbd_free_bc(mdev->ldev);
2817 mdev->ldev = NULL;);
2818
2819 if (mdev->md_io_tmpp) {
2820 __free_page(mdev->md_io_tmpp);
2821 mdev->md_io_tmpp = NULL;
2822 }
2823 clear_bit(GO_DISKLESS, &mdev->flags);
2824}
2825
e9e6f3ec
LE		 2826static int w_go_diskless(struct drbd_conf *mdev, struct drbd_work *w, int unused)
2827{
2828 D_ASSERT(mdev->state.disk == D_FAILED);
9d282875
LE		 2829 /* we cannot assert local_cnt == 0 here, as get_ldev_if_state will
2830 * inc/dec it frequently. Once we are D_DISKLESS, no one will touch
82f59cc6
LE		 2831 * the protected members anymore, though, so once put_ldev reaches zero
2832 * again, it will be safe to free them. */
e9e6f3ec 2833 drbd_force_state(mdev, NS(disk, D_DISKLESS));
e9e6f3ec
LE		 2834 return 1;
2835}
2836
2837void drbd_go_diskless(struct drbd_conf *mdev)
2838{
2839 D_ASSERT(mdev->state.disk == D_FAILED);
2840 if (!test_and_set_bit(GO_DISKLESS, &mdev->flags))
e42325a5 2841 drbd_queue_work(&mdev->tconn->data.work, &mdev->go_diskless);
e9e6f3ec
LE		 2842}
2843
b411b363
PR		 2844/**
2845 * drbd_queue_bitmap_io() - Queues an IO operation on the whole bitmap
2846 * @mdev: DRBD device.
2847 * @io_fn: IO callback to be called when bitmap IO is possible
2848 * @done: callback to be called after the bitmap IO was performed
2849 * @why: Descriptive text of the reason for doing the IO
2850 *
2851 * While IO on the bitmap happens we freeze application IO thus we ensure
2852 * that drbd_set_out_of_sync() can not be called. This function MAY ONLY be
2853 * called from worker context. It MUST NOT be used while a previous such
2854 * work is still pending!
2855 */
2856void drbd_queue_bitmap_io(struct drbd_conf *mdev,
2857 int (*io_fn)(struct drbd_conf *),
2858 void (*done)(struct drbd_conf *, int),
20ceb2b2 2859 char *why, enum bm_flag flags)
b411b363 2860{
e6b3ea83 2861 D_ASSERT(current == mdev->tconn->worker.task);
b411b363
PR		 2862
2863 D_ASSERT(!test_bit(BITMAP_IO_QUEUED, &mdev->flags));
2864 D_ASSERT(!test_bit(BITMAP_IO, &mdev->flags));
2865 D_ASSERT(list_empty(&mdev->bm_io_work.w.list));
2866 if (mdev->bm_io_work.why)
2867 dev_err(DEV, "FIXME going to queue '%s' but '%s' still pending?\n",
2868 why, mdev->bm_io_work.why);
2869
2870 mdev->bm_io_work.io_fn = io_fn;
2871 mdev->bm_io_work.done = done;
2872 mdev->bm_io_work.why = why;
20ceb2b2 2873 mdev->bm_io_work.flags = flags;
b411b363 2874
87eeee41 2875 spin_lock_irq(&mdev->tconn->req_lock);
b411b363
PR		 2876 set_bit(BITMAP_IO, &mdev->flags);
2877 if (atomic_read(&mdev->ap_bio_cnt) == 0) {
127b3178 2878 if (!test_and_set_bit(BITMAP_IO_QUEUED, &mdev->flags))
e42325a5 2879 drbd_queue_work(&mdev->tconn->data.work, &mdev->bm_io_work.w);
b411b363 2880 }
87eeee41 2881 spin_unlock_irq(&mdev->tconn->req_lock);
b411b363
PR		 2882}
2883
2884/**
2885 * drbd_bitmap_io() - Does an IO operation on the whole bitmap
2886 * @mdev: DRBD device.
2887 * @io_fn: IO callback to be called when bitmap IO is possible
2888 * @why: Descriptive text of the reason for doing the IO
2889 *
2890 * freezes application IO while that the actual IO operations runs. This
2891 * functions MAY NOT be called from worker context.
2892 */
20ceb2b2
LE		 2893int drbd_bitmap_io(struct drbd_conf *mdev, int (*io_fn)(struct drbd_conf *),
2894 char *why, enum bm_flag flags)
b411b363
PR		 2895{
2896 int rv;
2897
e6b3ea83 2898 D_ASSERT(current != mdev->tconn->worker.task);
b411b363 2899
20ceb2b2
LE		 2900 if ((flags & BM_LOCKED_SET_ALLOWED) == 0)
2901 drbd_suspend_io(mdev);
b411b363 2902
20ceb2b2 2903 drbd_bm_lock(mdev, why, flags);
b411b363
PR		 2904 rv = io_fn(mdev);
2905 drbd_bm_unlock(mdev);
2906
20ceb2b2
LE		 2907 if ((flags & BM_LOCKED_SET_ALLOWED) == 0)
2908 drbd_resume_io(mdev);
b411b363
PR		 2909
2910 return rv;
2911}
2912
2913void drbd_md_set_flag(struct drbd_conf *mdev, int flag) __must_hold(local)
2914{
2915 if ((mdev->ldev->md.flags & flag) != flag) {
2916 drbd_md_mark_dirty(mdev);
2917 mdev->ldev->md.flags |= flag;
2918 }
2919}
2920
2921void drbd_md_clear_flag(struct drbd_conf *mdev, int flag) __must_hold(local)
2922{
2923 if ((mdev->ldev->md.flags & flag) != 0) {
2924 drbd_md_mark_dirty(mdev);
2925 mdev->ldev->md.flags &= ~flag;
2926 }
2927}
2928int drbd_md_test_flag(struct drbd_backing_dev *bdev, int flag)
2929{
2930 return (bdev->md.flags & flag) != 0;
2931}
2932
2933static void md_sync_timer_fn(unsigned long data)
2934{
2935 struct drbd_conf *mdev = (struct drbd_conf *) data;
2936
e42325a5 2937 drbd_queue_work_front(&mdev->tconn->data.work, &mdev->md_sync_work);
b411b363
PR		 2938}
2939
2940static int w_md_sync(struct drbd_conf *mdev, struct drbd_work *w, int unused)
2941{
2942 dev_warn(DEV, "md_sync_timer expired! Worker calls drbd_md_sync().\n");
ee15b038
LE		 2943#ifdef DEBUG
2944 dev_warn(DEV, "last md_mark_dirty: %s:%u\n",
2945 mdev->last_md_mark_dirty.func, mdev->last_md_mark_dirty.line);
2946#endif
b411b363 2947 drbd_md_sync(mdev);
b411b363
PR		 2948 return 1;
2949}
2950
d8763023 2951const char *cmdname(enum drbd_packet cmd)
f2ad9063
AG		 2952{
2953 /* THINK may need to become several global tables
2954 * when we want to support more than
2955 * one PRO_VERSION */
2956 static const char *cmdnames[] = {
2957 [P_DATA] = "Data",
2958 [P_DATA_REPLY] = "DataReply",
2959 [P_RS_DATA_REPLY] = "RSDataReply",
2960 [P_BARRIER] = "Barrier",
2961 [P_BITMAP] = "ReportBitMap",
2962 [P_BECOME_SYNC_TARGET] = "BecomeSyncTarget",
2963 [P_BECOME_SYNC_SOURCE] = "BecomeSyncSource",
2964 [P_UNPLUG_REMOTE] = "UnplugRemote",
2965 [P_DATA_REQUEST] = "DataRequest",
2966 [P_RS_DATA_REQUEST] = "RSDataRequest",
2967 [P_SYNC_PARAM] = "SyncParam",
2968 [P_SYNC_PARAM89] = "SyncParam89",
2969 [P_PROTOCOL] = "ReportProtocol",
2970 [P_UUIDS] = "ReportUUIDs",
2971 [P_SIZES] = "ReportSizes",
2972 [P_STATE] = "ReportState",
2973 [P_SYNC_UUID] = "ReportSyncUUID",
2974 [P_AUTH_CHALLENGE] = "AuthChallenge",
2975 [P_AUTH_RESPONSE] = "AuthResponse",
2976 [P_PING] = "Ping",
2977 [P_PING_ACK] = "PingAck",
2978 [P_RECV_ACK] = "RecvAck",
2979 [P_WRITE_ACK] = "WriteAck",
2980 [P_RS_WRITE_ACK] = "RSWriteAck",
2981 [P_DISCARD_ACK] = "DiscardAck",
2982 [P_NEG_ACK] = "NegAck",
2983 [P_NEG_DREPLY] = "NegDReply",
2984 [P_NEG_RS_DREPLY] = "NegRSDReply",
2985 [P_BARRIER_ACK] = "BarrierAck",
2986 [P_STATE_CHG_REQ] = "StateChgRequest",
2987 [P_STATE_CHG_REPLY] = "StateChgReply",
2988 [P_OV_REQUEST] = "OVRequest",
2989 [P_OV_REPLY] = "OVReply",
2990 [P_OV_RESULT] = "OVResult",
2991 [P_CSUM_RS_REQUEST] = "CsumRSRequest",
2992 [P_RS_IS_IN_SYNC] = "CsumRSIsInSync",
2993 [P_COMPRESSED_BITMAP] = "CBitmap",
2994 [P_DELAY_PROBE] = "DelayProbe",
2995 [P_OUT_OF_SYNC] = "OutOfSync",
2996 [P_MAX_CMD] = NULL,
2997 };
2998
2999 if (cmd == P_HAND_SHAKE_M)
3000 return "HandShakeM";
3001 if (cmd == P_HAND_SHAKE_S)
3002 return "HandShakeS";
3003 if (cmd == P_HAND_SHAKE)
3004 return "HandShake";
3005 if (cmd >= P_MAX_CMD)
3006 return "Unknown";
3007 return cmdnames[cmd];
3008}
3009
b411b363
PR		 3010#ifdef CONFIG_DRBD_FAULT_INJECTION
3011/* Fault insertion support including random number generator shamelessly
3012 * stolen from kernel/rcutorture.c */
3013struct fault_random_state {
3014 unsigned long state;
3015 unsigned long count;
3016};
3017
3018#define FAULT_RANDOM_MULT 39916801 /* prime */
3019#define FAULT_RANDOM_ADD 479001701 /* prime */
3020#define FAULT_RANDOM_REFRESH 10000
3021
3022/*
3023 * Crude but fast random-number generator. Uses a linear congruential
3024 * generator, with occasional help from get_random_bytes().
3025 */
3026static unsigned long
3027_drbd_fault_random(struct fault_random_state *rsp)
3028{
3029 long refresh;
3030
49829ea7 3031 if (!rsp->count--) {
b411b363
PR		 3032 get_random_bytes(&refresh, sizeof(refresh));
3033 rsp->state += refresh;
3034 rsp->count = FAULT_RANDOM_REFRESH;
3035 }
3036 rsp->state = rsp->state * FAULT_RANDOM_MULT + FAULT_RANDOM_ADD;
3037 return swahw32(rsp->state);
3038}
3039
3040static char *
3041_drbd_fault_str(unsigned int type) {
3042 static char *_faults[] = {
3043 [DRBD_FAULT_MD_WR] = "Meta-data write",
3044 [DRBD_FAULT_MD_RD] = "Meta-data read",
3045 [DRBD_FAULT_RS_WR] = "Resync write",
3046 [DRBD_FAULT_RS_RD] = "Resync read",
3047 [DRBD_FAULT_DT_WR] = "Data write",
3048 [DRBD_FAULT_DT_RD] = "Data read",
3049 [DRBD_FAULT_DT_RA] = "Data read ahead",
3050 [DRBD_FAULT_BM_ALLOC] = "BM allocation",
6b4388ac
PR		 3051 [DRBD_FAULT_AL_EE] = "EE allocation",
3052 [DRBD_FAULT_RECEIVE] = "receive data corruption",
b411b363
PR		 3053 };
3054
3055 return (type < DRBD_FAULT_MAX) ? _faults[type] : "**Unknown**";
3056}
3057
3058unsigned int
3059_drbd_insert_fault(struct drbd_conf *mdev, unsigned int type)
3060{
3061 static struct fault_random_state rrs = {0, 0};
3062
3063 unsigned int ret = (
3064 (fault_devs == 0 ||
3065 ((1 << mdev_to_minor(mdev)) & fault_devs) != 0) &&
3066 (((_drbd_fault_random(&rrs) % 100) + 1) <= fault_rate));
3067
3068 if (ret) {
3069 fault_count++;
3070
7383506c 3071 if (__ratelimit(&drbd_ratelimit_state))
b411b363
PR		 3072 dev_warn(DEV, "***Simulating %s failure\n",
3073 _drbd_fault_str(type));
3074 }
3075
3076 return ret;
3077}
3078#endif
3079
3080const char *drbd_buildtag(void)
3081{
3082 /* DRBD built from external sources has here a reference to the
3083 git hash of the source code. */
3084
3085 static char buildtag[38] = "\0uilt-in";
3086
3087 if (buildtag[0] == 0) {
3088#ifdef CONFIG_MODULES
3089 if (THIS_MODULE != NULL)
3090 sprintf(buildtag, "srcversion: %-24s", THIS_MODULE->srcversion);
3091 else
3092#endif
3093 buildtag[0] = 'b';
3094 }
3095
3096 return buildtag;
3097}
3098
3099module_init(drbd_init)
3100module_exit(drbd_cleanup)
3101
b411b363
PR		 3102EXPORT_SYMBOL(drbd_conn_str);
3103EXPORT_SYMBOL(drbd_role_str);
3104EXPORT_SYMBOL(drbd_disk_str);
3105EXPORT_SYMBOL(drbd_set_st_err_str);
Linux 6.x block layer and io_uring tree(s)