projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
libceph: new sparse_read op, support sparse reads on msgr2 crc codepath
[linux-2.6-block.git] / net / ceph / messenger_v2.c
CommitLineData
cd1a677c
ID		 1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Ceph msgr2 protocol implementation
4 *
5 * Copyright (C) 2020 Ilya Dryomov <idryomov@gmail.com>
6 */
7
8#include <linux/ceph/ceph_debug.h>
9
10#include <crypto/aead.h>
11#include <crypto/algapi.h> /* for crypto_memneq() */
12#include <crypto/hash.h>
be695ee2 13#include <crypto/sha2.h>
cd1a677c
ID		 14#include <linux/bvec.h>
15#include <linux/crc32c.h>
16#include <linux/net.h>
17#include <linux/scatterlist.h>
18#include <linux/socket.h>
19#include <linux/sched/mm.h>
20#include <net/sock.h>
21#include <net/tcp.h>
22
23#include <linux/ceph/ceph_features.h>
24#include <linux/ceph/decode.h>
25#include <linux/ceph/libceph.h>
26#include <linux/ceph/messenger.h>
27
28#include "crypto.h" /* for CEPH_KEY_LEN and CEPH_MAX_CON_SECRET_LEN */
29
30#define FRAME_TAG_HELLO 1
31#define FRAME_TAG_AUTH_REQUEST 2
32#define FRAME_TAG_AUTH_BAD_METHOD 3
33#define FRAME_TAG_AUTH_REPLY_MORE 4
34#define FRAME_TAG_AUTH_REQUEST_MORE 5
35#define FRAME_TAG_AUTH_DONE 6
36#define FRAME_TAG_AUTH_SIGNATURE 7
37#define FRAME_TAG_CLIENT_IDENT 8
38#define FRAME_TAG_SERVER_IDENT 9
39#define FRAME_TAG_IDENT_MISSING_FEATURES 10
40#define FRAME_TAG_SESSION_RECONNECT 11
41#define FRAME_TAG_SESSION_RESET 12
42#define FRAME_TAG_SESSION_RETRY 13
43#define FRAME_TAG_SESSION_RETRY_GLOBAL 14
44#define FRAME_TAG_SESSION_RECONNECT_OK 15
45#define FRAME_TAG_WAIT 16
46#define FRAME_TAG_MESSAGE 17
47#define FRAME_TAG_KEEPALIVE2 18
48#define FRAME_TAG_KEEPALIVE2_ACK 19
49#define FRAME_TAG_ACK 20
50
51#define FRAME_LATE_STATUS_ABORTED 0x1
52#define FRAME_LATE_STATUS_COMPLETE 0xe
53#define FRAME_LATE_STATUS_ABORTED_MASK 0xf
54
ec3bc567
JL		 55#define IN_S_HANDLE_PREAMBLE 1
56#define IN_S_HANDLE_CONTROL 2
57#define IN_S_HANDLE_CONTROL_REMAINDER 3
58#define IN_S_PREPARE_READ_DATA 4
59#define IN_S_PREPARE_READ_DATA_CONT 5
60#define IN_S_PREPARE_READ_ENC_PAGE 6
61#define IN_S_PREPARE_SPARSE_DATA 7
62#define IN_S_PREPARE_SPARSE_DATA_CONT 8
63#define IN_S_HANDLE_EPILOGUE 9
64#define IN_S_FINISH_SKIP 10
cd1a677c
ID		 65
66#define OUT_S_QUEUE_DATA 1
67#define OUT_S_QUEUE_DATA_CONT 2
68#define OUT_S_QUEUE_ENC_PAGE 3
69#define OUT_S_QUEUE_ZEROS 4
70#define OUT_S_FINISH_MESSAGE 5
71#define OUT_S_GET_NEXT 6
72
73#define CTRL_BODY(p) ((void *)(p) + CEPH_PREAMBLE_LEN)
74#define FRONT_PAD(p) ((void *)(p) + CEPH_EPILOGUE_SECURE_LEN)
75#define MIDDLE_PAD(p) (FRONT_PAD(p) + CEPH_GCM_BLOCK_LEN)
76#define DATA_PAD(p) (MIDDLE_PAD(p) + CEPH_GCM_BLOCK_LEN)
77
78#define CEPH_MSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL)
79
80static int do_recvmsg(struct socket *sock, struct iov_iter *it)
81{
82 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
83 int ret;
84
85 msg.msg_iter = *it;
86 while (iov_iter_count(it)) {
87 ret = sock_recvmsg(sock, &msg, msg.msg_flags);
88 if (ret <= 0) {
89 if (ret == -EAGAIN)
90 ret = 0;
91 return ret;
92 }
93
94 iov_iter_advance(it, ret);
95 }
96
97 WARN_ON(msg_data_left(&msg));
98 return 1;
99}
100
101/*
102 * Read as much as possible.
103 *
104 * Return:
105 * 1 - done, nothing (else) to read
106 * 0 - socket is empty, need to wait
107 * <0 - error
108 */
109static int ceph_tcp_recv(struct ceph_connection *con)
110{
111 int ret;
112
113 dout("%s con %p %s %zu\n", __func__, con,
114 iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need",
115 iov_iter_count(&con->v2.in_iter));
116 ret = do_recvmsg(con->sock, &con->v2.in_iter);
117 dout("%s con %p ret %d left %zu\n", __func__, con, ret,
118 iov_iter_count(&con->v2.in_iter));
119 return ret;
120}
121
5da4d7b8
DH		 122static int do_sendmsg(struct socket *sock, struct iov_iter *it)
123{
124 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
125 int ret;
126
127 msg.msg_iter = *it;
128 while (iov_iter_count(it)) {
129 ret = sock_sendmsg(sock, &msg);
130 if (ret <= 0) {
131 if (ret == -EAGAIN)
132 ret = 0;
133 return ret;
134 }
135
136 iov_iter_advance(it, ret);
137 }
138
139 WARN_ON(msg_data_left(&msg));
140 return 1;
141}
142
143static int do_try_sendpage(struct socket *sock, struct iov_iter *it)
144{
145 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
146 struct bio_vec bv;
147 int ret;
148
149 if (WARN_ON(!iov_iter_is_bvec(it)))
150 return -EINVAL;
151
152 while (iov_iter_count(it)) {
153 /* iov_iter_iovec() for ITER_BVEC */
154 bvec_set_page(&bv, it->bvec->bv_page,
155 min(iov_iter_count(it),
156 it->bvec->bv_len - it->iov_offset),
157 it->bvec->bv_offset + it->iov_offset);
158
159 /*
160 * MSG_SPLICE_PAGES cannot properly handle pages with
161 * page_count == 0, we need to fall back to sendmsg if
162 * that's the case.
163 *
164 * Same goes for slab pages: skb_can_coalesce() allows
165 * coalescing neighboring slab objects into a single frag
166 * which triggers one of hardened usercopy checks.
167 */
168 if (sendpage_ok(bv.bv_page))
169 msg.msg_flags |= MSG_SPLICE_PAGES;
170 else
171 msg.msg_flags &= ~MSG_SPLICE_PAGES;
172
173 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bv, 1, bv.bv_len);
174 ret = sock_sendmsg(sock, &msg);
175 if (ret <= 0) {
176 if (ret == -EAGAIN)
177 ret = 0;
178 return ret;
179 }
180
181 iov_iter_advance(it, ret);
182 }
183
184 return 1;
185}
186
cd1a677c
ID		 187/*
188 * Write as much as possible. The socket is expected to be corked,
fa094cca 189 * so we don't bother with MSG_MORE here.
cd1a677c
ID		 190 *
191 * Return:
5da4d7b8 192 * 1 - done, nothing (else) to write
cd1a677c
ID		 193 * 0 - socket is full, need to wait
194 * <0 - error
195 */
196static int ceph_tcp_send(struct ceph_connection *con)
197{
198 int ret;
199
200 dout("%s con %p have %zu try_sendpage %d\n", __func__, con,
201 iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage);
5da4d7b8
DH		 202 if (con->v2.out_iter_sendpage)
203 ret = do_try_sendpage(con->sock, &con->v2.out_iter);
204 else
205 ret = do_sendmsg(con->sock, &con->v2.out_iter);
cd1a677c
ID		 206 dout("%s con %p ret %d left %zu\n", __func__, con, ret,
207 iov_iter_count(&con->v2.out_iter));
208 return ret;
209}
210
211static void add_in_kvec(struct ceph_connection *con, void *buf, int len)
212{
213 BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs));
214 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
215
216 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf;
217 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len;
218 con->v2.in_kvec_cnt++;
219
220 con->v2.in_iter.nr_segs++;
221 con->v2.in_iter.count += len;
222}
223
224static void reset_in_kvecs(struct ceph_connection *con)
225{
226 WARN_ON(iov_iter_count(&con->v2.in_iter));
227
228 con->v2.in_kvec_cnt = 0;
de4eda9d 229 iov_iter_kvec(&con->v2.in_iter, ITER_DEST, con->v2.in_kvecs, 0, 0);
cd1a677c
ID		 230}
231
232static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv)
233{
234 WARN_ON(iov_iter_count(&con->v2.in_iter));
235
236 con->v2.in_bvec = *bv;
de4eda9d 237 iov_iter_bvec(&con->v2.in_iter, ITER_DEST, &con->v2.in_bvec, 1, bv->bv_len);
cd1a677c
ID		 238}
239
240static void set_in_skip(struct ceph_connection *con, int len)
241{
242 WARN_ON(iov_iter_count(&con->v2.in_iter));
243
244 dout("%s con %p len %d\n", __func__, con, len);
de4eda9d 245 iov_iter_discard(&con->v2.in_iter, ITER_DEST, len);
cd1a677c
ID		 246}
247
248static void add_out_kvec(struct ceph_connection *con, void *buf, int len)
249{
250 BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs));
251 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
252 WARN_ON(con->v2.out_zero);
253
254 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf;
255 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len;
256 con->v2.out_kvec_cnt++;
257
258 con->v2.out_iter.nr_segs++;
259 con->v2.out_iter.count += len;
260}
261
262static void reset_out_kvecs(struct ceph_connection *con)
263{
264 WARN_ON(iov_iter_count(&con->v2.out_iter));
265 WARN_ON(con->v2.out_zero);
266
267 con->v2.out_kvec_cnt = 0;
268
de4eda9d 269 iov_iter_kvec(&con->v2.out_iter, ITER_SOURCE, con->v2.out_kvecs, 0, 0);
cd1a677c
ID		 270 con->v2.out_iter_sendpage = false;
271}
272
273static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv,
274 bool zerocopy)
275{
276 WARN_ON(iov_iter_count(&con->v2.out_iter));
277 WARN_ON(con->v2.out_zero);
278
279 con->v2.out_bvec = *bv;
280 con->v2.out_iter_sendpage = zerocopy;
de4eda9d 281 iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,
cd1a677c
ID		 282 con->v2.out_bvec.bv_len);
283}
284
285static void set_out_bvec_zero(struct ceph_connection *con)
286{
287 WARN_ON(iov_iter_count(&con->v2.out_iter));
288 WARN_ON(!con->v2.out_zero);
289
1eb9cd15
CH		 290 bvec_set_page(&con->v2.out_bvec, ceph_zero_page,
291 min(con->v2.out_zero, (int)PAGE_SIZE), 0);
cd1a677c 292 con->v2.out_iter_sendpage = true;
de4eda9d 293 iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,
cd1a677c
ID		 294 con->v2.out_bvec.bv_len);
295}
296
297static void out_zero_add(struct ceph_connection *con, int len)
298{
299 dout("%s con %p len %d\n", __func__, con, len);
300 con->v2.out_zero += len;
301}
302
303static void *alloc_conn_buf(struct ceph_connection *con, int len)
304{
305 void *buf;
306
307 dout("%s con %p len %d\n", __func__, con, len);
308
309 if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs)))
310 return NULL;
311
a421ef30 312 buf = kvmalloc(len, GFP_NOIO);
cd1a677c
ID		 313 if (!buf)
314 return NULL;
315
316 con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf;
317 return buf;
318}
319
320static void free_conn_bufs(struct ceph_connection *con)
321{
322 while (con->v2.conn_buf_cnt)
323 kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]);
324}
325
326static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len)
327{
328 BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs));
329
330 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf;
331 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len;
332 con->v2.in_sign_kvec_cnt++;
333}
334
335static void clear_in_sign_kvecs(struct ceph_connection *con)
336{
337 con->v2.in_sign_kvec_cnt = 0;
338}
339
340static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len)
341{
342 BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs));
343
344 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf;
345 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len;
346 con->v2.out_sign_kvec_cnt++;
347}
348
349static void clear_out_sign_kvecs(struct ceph_connection *con)
350{
351 con->v2.out_sign_kvec_cnt = 0;
352}
353
354static bool con_secure(struct ceph_connection *con)
355{
356 return con->v2.con_mode == CEPH_CON_MODE_SECURE;
357}
358
359static int front_len(const struct ceph_msg *msg)
360{
361 return le32_to_cpu(msg->hdr.front_len);
362}
363
364static int middle_len(const struct ceph_msg *msg)
365{
366 return le32_to_cpu(msg->hdr.middle_len);
367}
368
369static int data_len(const struct ceph_msg *msg)
370{
371 return le32_to_cpu(msg->hdr.data_len);
372}
373
374static bool need_padding(int len)
375{
376 return !IS_ALIGNED(len, CEPH_GCM_BLOCK_LEN);
377}
378
379static int padded_len(int len)
380{
381 return ALIGN(len, CEPH_GCM_BLOCK_LEN);
382}
383
384static int padding_len(int len)
385{
386 return padded_len(len) - len;
387}
388
389/* preamble + control segment */
390static int head_onwire_len(int ctrl_len, bool secure)
391{
392 int head_len;
393 int rem_len;
394
a282a2f1
ID		 395 BUG_ON(ctrl_len < 0 || ctrl_len > CEPH_MSG_MAX_CONTROL_LEN);
396
cd1a677c
ID		 397 if (secure) {
398 head_len = CEPH_PREAMBLE_SECURE_LEN;
399 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
400 rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
401 head_len += padded_len(rem_len) + CEPH_GCM_TAG_LEN;
402 }
403 } else {
404 head_len = CEPH_PREAMBLE_PLAIN_LEN;
405 if (ctrl_len)
406 head_len += ctrl_len + CEPH_CRC_LEN;
407 }
408 return head_len;
409}
410
411/* front, middle and data segments + epilogue */
412static int __tail_onwire_len(int front_len, int middle_len, int data_len,
413 bool secure)
414{
a282a2f1
ID		 415 BUG_ON(front_len < 0 || front_len > CEPH_MSG_MAX_FRONT_LEN ||
416 middle_len < 0 || middle_len > CEPH_MSG_MAX_MIDDLE_LEN ||
417 data_len < 0 || data_len > CEPH_MSG_MAX_DATA_LEN);
418
cd1a677c
ID		 419 if (!front_len && !middle_len && !data_len)
420 return 0;
421
422 if (!secure)
423 return front_len + middle_len + data_len +
424 CEPH_EPILOGUE_PLAIN_LEN;
425
426 return padded_len(front_len) + padded_len(middle_len) +
427 padded_len(data_len) + CEPH_EPILOGUE_SECURE_LEN;
428}
429
430static int tail_onwire_len(const struct ceph_msg *msg, bool secure)
431{
432 return __tail_onwire_len(front_len(msg), middle_len(msg),
433 data_len(msg), secure);
434}
435
436/* head_onwire_len(sizeof(struct ceph_msg_header2), false) */
437#define MESSAGE_HEAD_PLAIN_LEN (CEPH_PREAMBLE_PLAIN_LEN + \
438 sizeof(struct ceph_msg_header2) + \
439 CEPH_CRC_LEN)
440
441static const int frame_aligns[] = {
442 sizeof(void *),
443 sizeof(void *),
444 sizeof(void *),
445 PAGE_SIZE
446};
447
448/*
449 * Discards trailing empty segments, unless there is just one segment.
450 * A frame always has at least one (possibly empty) segment.
451 */
452static int calc_segment_count(const int *lens, int len_cnt)
453{
454 int i;
455
456 for (i = len_cnt - 1; i >= 0; i--) {
457 if (lens[i])
458 return i + 1;
459 }
460
461 return 1;
462}
463
464static void init_frame_desc(struct ceph_frame_desc *desc, int tag,
465 const int *lens, int len_cnt)
466{
467 int i;
468
469 memset(desc, 0, sizeof(*desc));
470
471 desc->fd_tag = tag;
472 desc->fd_seg_cnt = calc_segment_count(lens, len_cnt);
473 BUG_ON(desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT);
474 for (i = 0; i < desc->fd_seg_cnt; i++) {
475 desc->fd_lens[i] = lens[i];
476 desc->fd_aligns[i] = frame_aligns[i];
477 }
478}
479
480/*
481 * Preamble crc covers everything up to itself (28 bytes) and
482 * is calculated and verified irrespective of the connection mode
483 * (i.e. even if the frame is encrypted).
484 */
485static void encode_preamble(const struct ceph_frame_desc *desc, void *p)
486{
487 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
488 void *start = p;
489 int i;
490
491 memset(p, 0, CEPH_PREAMBLE_LEN);
492
493 ceph_encode_8(&p, desc->fd_tag);
494 ceph_encode_8(&p, desc->fd_seg_cnt);
495 for (i = 0; i < desc->fd_seg_cnt; i++) {
496 ceph_encode_32(&p, desc->fd_lens[i]);
497 ceph_encode_16(&p, desc->fd_aligns[i]);
498 }
499
500 put_unaligned_le32(crc32c(0, start, crcp - start), crcp);
501}
502
503static int decode_preamble(void *p, struct ceph_frame_desc *desc)
504{
505 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
506 u32 crc, expected_crc;
507 int i;
508
509 crc = crc32c(0, p, crcp - p);
510 expected_crc = get_unaligned_le32(crcp);
511 if (crc != expected_crc) {
512 pr_err("bad preamble crc, calculated %u, expected %u\n",
513 crc, expected_crc);
514 return -EBADMSG;
515 }
516
517 memset(desc, 0, sizeof(*desc));
518
519 desc->fd_tag = ceph_decode_8(&p);
520 desc->fd_seg_cnt = ceph_decode_8(&p);
521 if (desc->fd_seg_cnt < 1 ||
522 desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT) {
523 pr_err("bad segment count %d\n", desc->fd_seg_cnt);
524 return -EINVAL;
525 }
526 for (i = 0; i < desc->fd_seg_cnt; i++) {
527 desc->fd_lens[i] = ceph_decode_32(&p);
528 desc->fd_aligns[i] = ceph_decode_16(&p);
529 }
530
a282a2f1
ID		 531 if (desc->fd_lens[0] < 0 ||
532 desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) {
533 pr_err("bad control segment length %d\n", desc->fd_lens[0]);
cd1a677c
ID		 534 return -EINVAL;
535 }
a282a2f1
ID		 536 if (desc->fd_lens[1] < 0 ||
537 desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) {
538 pr_err("bad front segment length %d\n", desc->fd_lens[1]);
cd1a677c
ID		 539 return -EINVAL;
540 }
a282a2f1
ID		 541 if (desc->fd_lens[2] < 0 ||
542 desc->fd_lens[2] > CEPH_MSG_MAX_MIDDLE_LEN) {
543 pr_err("bad middle segment length %d\n", desc->fd_lens[2]);
cd1a677c
ID		 544 return -EINVAL;
545 }
a282a2f1
ID		 546 if (desc->fd_lens[3] < 0 ||
547 desc->fd_lens[3] > CEPH_MSG_MAX_DATA_LEN) {
548 pr_err("bad data segment length %d\n", desc->fd_lens[3]);
cd1a677c
ID		 549 return -EINVAL;
550 }
a282a2f1
ID		 551
552 /*
553 * This would fire for FRAME_TAG_WAIT (it has one empty
554 * segment), but we should never get it as client.
555 */
556 if (!desc->fd_lens[desc->fd_seg_cnt - 1]) {
557 pr_err("last segment empty, segment count %d\n",
558 desc->fd_seg_cnt);
cd1a677c
ID		 559 return -EINVAL;
560 }
561
562 return 0;
563}
564
565static void encode_epilogue_plain(struct ceph_connection *con, bool aborted)
566{
567 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
568 FRAME_LATE_STATUS_COMPLETE;
569 cpu_to_le32s(&con->v2.out_epil.front_crc);
570 cpu_to_le32s(&con->v2.out_epil.middle_crc);
571 cpu_to_le32s(&con->v2.out_epil.data_crc);
572}
573
574static void encode_epilogue_secure(struct ceph_connection *con, bool aborted)
575{
576 memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil));
577 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
578 FRAME_LATE_STATUS_COMPLETE;
579}
580
581static int decode_epilogue(void *p, u32 *front_crc, u32 *middle_crc,
582 u32 *data_crc)
583{
584 u8 late_status;
585
586 late_status = ceph_decode_8(&p);
587 if ((late_status & FRAME_LATE_STATUS_ABORTED_MASK) !=
588 FRAME_LATE_STATUS_COMPLETE) {
589 /* we should never get an aborted message as client */
590 pr_err("bad late_status 0x%x\n", late_status);
591 return -EINVAL;
592 }
593
594 if (front_crc && middle_crc && data_crc) {
595 *front_crc = ceph_decode_32(&p);
596 *middle_crc = ceph_decode_32(&p);
597 *data_crc = ceph_decode_32(&p);
598 }
599
600 return 0;
601}
602
603static void fill_header(struct ceph_msg_header *hdr,
604 const struct ceph_msg_header2 *hdr2,
605 int front_len, int middle_len, int data_len,
606 const struct ceph_entity_name *peer_name)
607{
608 hdr->seq = hdr2->seq;
609 hdr->tid = hdr2->tid;
610 hdr->type = hdr2->type;
611 hdr->priority = hdr2->priority;
612 hdr->version = hdr2->version;
613 hdr->front_len = cpu_to_le32(front_len);
614 hdr->middle_len = cpu_to_le32(middle_len);
615 hdr->data_len = cpu_to_le32(data_len);
616 hdr->data_off = hdr2->data_off;
617 hdr->src = *peer_name;
618 hdr->compat_version = hdr2->compat_version;
619 hdr->reserved = 0;
620 hdr->crc = 0;
621}
622
623static void fill_header2(struct ceph_msg_header2 *hdr2,
624 const struct ceph_msg_header *hdr, u64 ack_seq)
625{
626 hdr2->seq = hdr->seq;
627 hdr2->tid = hdr->tid;
628 hdr2->type = hdr->type;
629 hdr2->priority = hdr->priority;
630 hdr2->version = hdr->version;
631 hdr2->data_pre_padding_len = 0;
632 hdr2->data_off = hdr->data_off;
633 hdr2->ack_seq = cpu_to_le64(ack_seq);
634 hdr2->flags = 0;
635 hdr2->compat_version = hdr->compat_version;
636 hdr2->reserved = 0;
637}
638
639static int verify_control_crc(struct ceph_connection *con)
640{
641 int ctrl_len = con->v2.in_desc.fd_lens[0];
642 u32 crc, expected_crc;
643
644 WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len);
645 WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN);
646
647 crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len);
648 expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base);
649 if (crc != expected_crc) {
650 pr_err("bad control crc, calculated %u, expected %u\n",
651 crc, expected_crc);
652 return -EBADMSG;
653 }
654
655 return 0;
656}
657
658static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc,
659 u32 middle_crc, u32 data_crc)
660{
661 if (front_len(con->in_msg)) {
662 con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base,
663 front_len(con->in_msg));
664 } else {
665 WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg));
666 con->in_front_crc = -1;
667 }
668
669 if (middle_len(con->in_msg))
670 con->in_middle_crc = crc32c(-1,
671 con->in_msg->middle->vec.iov_base,
672 middle_len(con->in_msg));
673 else if (data_len(con->in_msg))
674 con->in_middle_crc = -1;
675 else
676 con->in_middle_crc = 0;
677
678 if (!data_len(con->in_msg))
679 con->in_data_crc = 0;
680
681 dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg,
682 con->in_front_crc, con->in_middle_crc, con->in_data_crc);
683
684 if (con->in_front_crc != front_crc) {
685 pr_err("bad front crc, calculated %u, expected %u\n",
686 con->in_front_crc, front_crc);
687 return -EBADMSG;
688 }
689 if (con->in_middle_crc != middle_crc) {
690 pr_err("bad middle crc, calculated %u, expected %u\n",
691 con->in_middle_crc, middle_crc);
692 return -EBADMSG;
693 }
694 if (con->in_data_crc != data_crc) {
695 pr_err("bad data crc, calculated %u, expected %u\n",
696 con->in_data_crc, data_crc);
697 return -EBADMSG;
698 }
699
700 return 0;
701}
702
703static int setup_crypto(struct ceph_connection *con,
10f42b3e
ID		 704 const u8 *session_key, int session_key_len,
705 const u8 *con_secret, int con_secret_len)
cd1a677c
ID		 706{
707 unsigned int noio_flag;
cd1a677c
ID		 708 int ret;
709
710 dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n",
711 __func__, con, con->v2.con_mode, session_key_len, con_secret_len);
712 WARN_ON(con->v2.hmac_tfm || con->v2.gcm_tfm || con->v2.gcm_req);
713
714 if (con->v2.con_mode != CEPH_CON_MODE_CRC &&
715 con->v2.con_mode != CEPH_CON_MODE_SECURE) {
716 pr_err("bad con_mode %d\n", con->v2.con_mode);
717 return -EINVAL;
718 }
719
720 if (!session_key_len) {
721 WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC);
722 WARN_ON(con_secret_len);
723 return 0; /* auth_none */
724 }
725
726 noio_flag = memalloc_noio_save();
727 con->v2.hmac_tfm = crypto_alloc_shash("hmac(sha256)", 0, 0);
728 memalloc_noio_restore(noio_flag);
729 if (IS_ERR(con->v2.hmac_tfm)) {
730 ret = PTR_ERR(con->v2.hmac_tfm);
731 con->v2.hmac_tfm = NULL;
732 pr_err("failed to allocate hmac tfm context: %d\n", ret);
733 return ret;
734 }
735
736 WARN_ON((unsigned long)session_key &
737 crypto_shash_alignmask(con->v2.hmac_tfm));
738 ret = crypto_shash_setkey(con->v2.hmac_tfm, session_key,
739 session_key_len);
740 if (ret) {
741 pr_err("failed to set hmac key: %d\n", ret);
742 return ret;
743 }
744
745 if (con->v2.con_mode == CEPH_CON_MODE_CRC) {
746 WARN_ON(con_secret_len);
747 return 0; /* auth_x, plain mode */
748 }
749
750 if (con_secret_len < CEPH_GCM_KEY_LEN + 2 * CEPH_GCM_IV_LEN) {
751 pr_err("con_secret too small %d\n", con_secret_len);
752 return -EINVAL;
753 }
754
755 noio_flag = memalloc_noio_save();
756 con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0);
757 memalloc_noio_restore(noio_flag);
758 if (IS_ERR(con->v2.gcm_tfm)) {
759 ret = PTR_ERR(con->v2.gcm_tfm);
760 con->v2.gcm_tfm = NULL;
761 pr_err("failed to allocate gcm tfm context: %d\n", ret);
762 return ret;
763 }
764
10f42b3e
ID		 765 WARN_ON((unsigned long)con_secret &
766 crypto_aead_alignmask(con->v2.gcm_tfm));
767 ret = crypto_aead_setkey(con->v2.gcm_tfm, con_secret, CEPH_GCM_KEY_LEN);
cd1a677c
ID		 768 if (ret) {
769 pr_err("failed to set gcm key: %d\n", ret);
770 return ret;
771 }
772
cd1a677c
ID		 773 WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN);
774 ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN);
775 if (ret) {
776 pr_err("failed to set gcm tag size: %d\n", ret);
777 return ret;
778 }
779
780 con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO);
781 if (!con->v2.gcm_req) {
782 pr_err("failed to allocate gcm request\n");
783 return -ENOMEM;
784 }
785
786 crypto_init_wait(&con->v2.gcm_wait);
787 aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
788 crypto_req_done, &con->v2.gcm_wait);
789
10f42b3e
ID		 790 memcpy(&con->v2.in_gcm_nonce, con_secret + CEPH_GCM_KEY_LEN,
791 CEPH_GCM_IV_LEN);
792 memcpy(&con->v2.out_gcm_nonce,
793 con_secret + CEPH_GCM_KEY_LEN + CEPH_GCM_IV_LEN,
794 CEPH_GCM_IV_LEN);
cd1a677c
ID		 795 return 0; /* auth_x, secure mode */
796}
797
798static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs,
799 int kvec_cnt, u8 *hmac)
800{
801 SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm); /* tfm arg is ignored */
802 int ret;
803 int i;
804
805 dout("%s con %p hmac_tfm %p kvec_cnt %d\n", __func__, con,
806 con->v2.hmac_tfm, kvec_cnt);
807
808 if (!con->v2.hmac_tfm) {
809 memset(hmac, 0, SHA256_DIGEST_SIZE);
810 return 0; /* auth_none */
811 }
812
813 desc->tfm = con->v2.hmac_tfm;
814 ret = crypto_shash_init(desc);
815 if (ret)
10f42b3e 816 goto out;
cd1a677c
ID		 817
818 for (i = 0; i < kvec_cnt; i++) {
819 WARN_ON((unsigned long)kvecs[i].iov_base &
820 crypto_shash_alignmask(con->v2.hmac_tfm));
821 ret = crypto_shash_update(desc, kvecs[i].iov_base,
822 kvecs[i].iov_len);
823 if (ret)
10f42b3e 824 goto out;
cd1a677c
ID		 825 }
826
827 ret = crypto_shash_final(desc, hmac);
cd1a677c 828
10f42b3e 829out:
cd1a677c 830 shash_desc_zero(desc);
10f42b3e 831 return ret; /* auth_x, both plain and secure modes */
cd1a677c
ID		 832}
833
834static void gcm_inc_nonce(struct ceph_gcm_nonce *nonce)
835{
836 u64 counter;
837
838 counter = le64_to_cpu(nonce->counter);
839 nonce->counter = cpu_to_le64(counter + 1);
840}
841
842static int gcm_crypt(struct ceph_connection *con, bool encrypt,
843 struct scatterlist *src, struct scatterlist *dst,
844 int src_len)
845{
846 struct ceph_gcm_nonce *nonce;
847 int ret;
848
849 nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce;
850
851 aead_request_set_ad(con->v2.gcm_req, 0); /* no AAD */
852 aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce);
853 ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) :
854 crypto_aead_decrypt(con->v2.gcm_req),
855 &con->v2.gcm_wait);
856 if (ret)
857 return ret;
858
859 gcm_inc_nonce(nonce);
860 return 0;
861}
862
863static void get_bvec_at(struct ceph_msg_data_cursor *cursor,
864 struct bio_vec *bv)
865{
866 struct page *page;
867 size_t off, len;
868
869 WARN_ON(!cursor->total_resid);
870
871 /* skip zero-length data items */
872 while (!cursor->resid)
873 ceph_msg_data_advance(cursor, 0);
874
875 /* get a piece of data, cursor isn't advanced */
da4ab869 876 page = ceph_msg_data_next(cursor, &off, &len);
1eb9cd15 877 bvec_set_page(bv, page, len, off);
cd1a677c
ID		 878}
879
880static int calc_sg_cnt(void *buf, int buf_len)
881{
882 int sg_cnt;
883
884 if (!buf_len)
885 return 0;
886
887 sg_cnt = need_padding(buf_len) ? 1 : 0;
888 if (is_vmalloc_addr(buf)) {
889 WARN_ON(offset_in_page(buf));
890 sg_cnt += PAGE_ALIGN(buf_len) >> PAGE_SHIFT;
891 } else {
892 sg_cnt++;
893 }
894
895 return sg_cnt;
896}
897
898static int calc_sg_cnt_cursor(struct ceph_msg_data_cursor *cursor)
899{
900 int data_len = cursor->total_resid;
901 struct bio_vec bv;
902 int sg_cnt;
903
904 if (!data_len)
905 return 0;
906
907 sg_cnt = need_padding(data_len) ? 1 : 0;
908 do {
909 get_bvec_at(cursor, &bv);
910 sg_cnt++;
911
912 ceph_msg_data_advance(cursor, bv.bv_len);
913 } while (cursor->total_resid);
914
915 return sg_cnt;
916}
917
918static void init_sgs(struct scatterlist **sg, void *buf, int buf_len, u8 *pad)
919{
920 void *end = buf + buf_len;
921 struct page *page;
922 int len;
923 void *p;
924
925 if (!buf_len)
926 return;
927
928 if (is_vmalloc_addr(buf)) {
929 p = buf;
930 do {
931 page = vmalloc_to_page(p);
932 len = min_t(int, end - p, PAGE_SIZE);
933 WARN_ON(!page || !len || offset_in_page(p));
934 sg_set_page(*sg, page, len, 0);
935 *sg = sg_next(*sg);
936 p += len;
937 } while (p != end);
938 } else {
939 sg_set_buf(*sg, buf, buf_len);
940 *sg = sg_next(*sg);
941 }
942
943 if (need_padding(buf_len)) {
944 sg_set_buf(*sg, pad, padding_len(buf_len));
945 *sg = sg_next(*sg);
946 }
947}
948
949static void init_sgs_cursor(struct scatterlist **sg,
950 struct ceph_msg_data_cursor *cursor, u8 *pad)
951{
952 int data_len = cursor->total_resid;
953 struct bio_vec bv;
954
955 if (!data_len)
956 return;
957
958 do {
959 get_bvec_at(cursor, &bv);
960 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
961 *sg = sg_next(*sg);
962
963 ceph_msg_data_advance(cursor, bv.bv_len);
964 } while (cursor->total_resid);
965
966 if (need_padding(data_len)) {
967 sg_set_buf(*sg, pad, padding_len(data_len));
968 *sg = sg_next(*sg);
969 }
970}
971
972static int setup_message_sgs(struct sg_table *sgt, struct ceph_msg *msg,
973 u8 *front_pad, u8 *middle_pad, u8 *data_pad,
974 void *epilogue, bool add_tag)
975{
976 struct ceph_msg_data_cursor cursor;
977 struct scatterlist *cur_sg;
978 int sg_cnt;
979 int ret;
980
981 if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
982 return 0;
983
984 sg_cnt = 1; /* epilogue + [auth tag] */
985 if (front_len(msg))
986 sg_cnt += calc_sg_cnt(msg->front.iov_base,
987 front_len(msg));
988 if (middle_len(msg))
989 sg_cnt += calc_sg_cnt(msg->middle->vec.iov_base,
990 middle_len(msg));
991 if (data_len(msg)) {
992 ceph_msg_data_cursor_init(&cursor, msg, data_len(msg));
993 sg_cnt += calc_sg_cnt_cursor(&cursor);
994 }
995
996 ret = sg_alloc_table(sgt, sg_cnt, GFP_NOIO);
997 if (ret)
998 return ret;
999
1000 cur_sg = sgt->sgl;
1001 if (front_len(msg))
1002 init_sgs(&cur_sg, msg->front.iov_base, front_len(msg),
1003 front_pad);
1004 if (middle_len(msg))
1005 init_sgs(&cur_sg, msg->middle->vec.iov_base, middle_len(msg),
1006 middle_pad);
1007 if (data_len(msg)) {
1008 ceph_msg_data_cursor_init(&cursor, msg, data_len(msg));
1009 init_sgs_cursor(&cur_sg, &cursor, data_pad);
1010 }
1011
1012 WARN_ON(!sg_is_last(cur_sg));
1013 sg_set_buf(cur_sg, epilogue,
1014 CEPH_GCM_BLOCK_LEN + (add_tag ? CEPH_GCM_TAG_LEN : 0));
1015 return 0;
1016}
1017
1018static int decrypt_preamble(struct ceph_connection *con)
1019{
1020 struct scatterlist sg;
1021
1022 sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN);
1023 return gcm_crypt(con, false, &sg, &sg, CEPH_PREAMBLE_SECURE_LEN);
1024}
1025
1026static int decrypt_control_remainder(struct ceph_connection *con)
1027{
1028 int ctrl_len = con->v2.in_desc.fd_lens[0];
1029 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1030 int pt_len = padding_len(rem_len) + CEPH_GCM_TAG_LEN;
1031 struct scatterlist sgs[2];
1032
1033 WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len);
1034 WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len);
1035
1036 sg_init_table(sgs, 2);
1037 sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len);
1038 sg_set_buf(&sgs[1], con->v2.in_buf, pt_len);
1039
1040 return gcm_crypt(con, false, sgs, sgs,
1041 padded_len(rem_len) + CEPH_GCM_TAG_LEN);
1042}
1043
2ea88716 1044static int decrypt_tail(struct ceph_connection *con)
cd1a677c 1045{
2ea88716 1046 struct sg_table enc_sgt = {};
cd1a677c 1047 struct sg_table sgt = {};
2ea88716 1048 int tail_len;
cd1a677c
ID		 1049 int ret;
1050
2ea88716
ID		 1051 tail_len = tail_onwire_len(con->in_msg, true);
1052 ret = sg_alloc_table_from_pages(&enc_sgt, con->v2.in_enc_pages,
1053 con->v2.in_enc_page_cnt, 0, tail_len,
1054 GFP_NOIO);
1055 if (ret)
1056 goto out;
1057
cd1a677c
ID		 1058 ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf),
1059 MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf),
1060 con->v2.in_buf, true);
1061 if (ret)
1062 goto out;
1063
2ea88716
ID		 1064 dout("%s con %p msg %p enc_page_cnt %d sg_cnt %d\n", __func__, con,
1065 con->in_msg, con->v2.in_enc_page_cnt, sgt.orig_nents);
1066 ret = gcm_crypt(con, false, enc_sgt.sgl, sgt.sgl, tail_len);
1067 if (ret)
1068 goto out;
1069
1070 WARN_ON(!con->v2.in_enc_page_cnt);
1071 ceph_release_page_vector(con->v2.in_enc_pages,
1072 con->v2.in_enc_page_cnt);
1073 con->v2.in_enc_pages = NULL;
1074 con->v2.in_enc_page_cnt = 0;
cd1a677c
ID		 1075
1076out:
1077 sg_free_table(&sgt);
2ea88716 1078 sg_free_table(&enc_sgt);
cd1a677c
ID		 1079 return ret;
1080}
1081
1082static int prepare_banner(struct ceph_connection *con)
1083{
1084 int buf_len = CEPH_BANNER_V2_LEN + 2 + 8 + 8;
1085 void *buf, *p;
1086
1087 buf = alloc_conn_buf(con, buf_len);
1088 if (!buf)
1089 return -ENOMEM;
1090
1091 p = buf;
1092 ceph_encode_copy(&p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN);
1093 ceph_encode_16(&p, sizeof(u64) + sizeof(u64));
1094 ceph_encode_64(&p, CEPH_MSGR2_SUPPORTED_FEATURES);
1095 ceph_encode_64(&p, CEPH_MSGR2_REQUIRED_FEATURES);
1096 WARN_ON(p != buf + buf_len);
1097
1098 add_out_kvec(con, buf, buf_len);
1099 add_out_sign_kvec(con, buf, buf_len);
1100 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1101 return 0;
1102}
1103
1104/*
1105 * base:
1106 * preamble
1107 * control body (ctrl_len bytes)
1108 * space for control crc
1109 *
1110 * extdata (optional):
1111 * control body (extdata_len bytes)
1112 *
1113 * Compute control crc and gather base and extdata into:
1114 *
1115 * preamble
1116 * control body (ctrl_len + extdata_len bytes)
1117 * control crc
1118 *
1119 * Preamble should already be encoded at the start of base.
1120 */
1121static void prepare_head_plain(struct ceph_connection *con, void *base,
1122 int ctrl_len, void *extdata, int extdata_len,
1123 bool to_be_signed)
1124{
1125 int base_len = CEPH_PREAMBLE_LEN + ctrl_len + CEPH_CRC_LEN;
1126 void *crcp = base + base_len - CEPH_CRC_LEN;
1127 u32 crc;
1128
1129 crc = crc32c(-1, CTRL_BODY(base), ctrl_len);
1130 if (extdata_len)
1131 crc = crc32c(crc, extdata, extdata_len);
1132 put_unaligned_le32(crc, crcp);
1133
1134 if (!extdata_len) {
1135 add_out_kvec(con, base, base_len);
1136 if (to_be_signed)
1137 add_out_sign_kvec(con, base, base_len);
1138 return;
1139 }
1140
1141 add_out_kvec(con, base, crcp - base);
1142 add_out_kvec(con, extdata, extdata_len);
1143 add_out_kvec(con, crcp, CEPH_CRC_LEN);
1144 if (to_be_signed) {
1145 add_out_sign_kvec(con, base, crcp - base);
1146 add_out_sign_kvec(con, extdata, extdata_len);
1147 add_out_sign_kvec(con, crcp, CEPH_CRC_LEN);
1148 }
1149}
1150
1151static int prepare_head_secure_small(struct ceph_connection *con,
1152 void *base, int ctrl_len)
1153{
1154 struct scatterlist sg;
1155 int ret;
1156
1157 /* inline buffer padding? */
1158 if (ctrl_len < CEPH_PREAMBLE_INLINE_LEN)
1159 memset(CTRL_BODY(base) + ctrl_len, 0,
1160 CEPH_PREAMBLE_INLINE_LEN - ctrl_len);
1161
1162 sg_init_one(&sg, base, CEPH_PREAMBLE_SECURE_LEN);
1163 ret = gcm_crypt(con, true, &sg, &sg,
1164 CEPH_PREAMBLE_SECURE_LEN - CEPH_GCM_TAG_LEN);
1165 if (ret)
1166 return ret;
1167
1168 add_out_kvec(con, base, CEPH_PREAMBLE_SECURE_LEN);
1169 return 0;
1170}
1171
1172/*
1173 * base:
1174 * preamble
1175 * control body (ctrl_len bytes)
1176 * space for padding, if needed
1177 * space for control remainder auth tag
1178 * space for preamble auth tag
1179 *
1180 * Encrypt preamble and the inline portion, then encrypt the remainder
1181 * and gather into:
1182 *
1183 * preamble
1184 * control body (48 bytes)
1185 * preamble auth tag
1186 * control body (ctrl_len - 48 bytes)
1187 * zero padding, if needed
1188 * control remainder auth tag
1189 *
1190 * Preamble should already be encoded at the start of base.
1191 */
1192static int prepare_head_secure_big(struct ceph_connection *con,
1193 void *base, int ctrl_len)
1194{
1195 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1196 void *rem = CTRL_BODY(base) + CEPH_PREAMBLE_INLINE_LEN;
1197 void *rem_tag = rem + padded_len(rem_len);
1198 void *pmbl_tag = rem_tag + CEPH_GCM_TAG_LEN;
1199 struct scatterlist sgs[2];
1200 int ret;
1201
1202 sg_init_table(sgs, 2);
1203 sg_set_buf(&sgs[0], base, rem - base);
1204 sg_set_buf(&sgs[1], pmbl_tag, CEPH_GCM_TAG_LEN);
1205 ret = gcm_crypt(con, true, sgs, sgs, rem - base);
1206 if (ret)
1207 return ret;
1208
1209 /* control remainder padding? */
1210 if (need_padding(rem_len))
1211 memset(rem + rem_len, 0, padding_len(rem_len));
1212
1213 sg_init_one(&sgs[0], rem, pmbl_tag - rem);
1214 ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem);
1215 if (ret)
1216 return ret;
1217
1218 add_out_kvec(con, base, rem - base);
1219 add_out_kvec(con, pmbl_tag, CEPH_GCM_TAG_LEN);
1220 add_out_kvec(con, rem, pmbl_tag - rem);
1221 return 0;
1222}
1223
1224static int __prepare_control(struct ceph_connection *con, int tag,
1225 void *base, int ctrl_len, void *extdata,
1226 int extdata_len, bool to_be_signed)
1227{
1228 int total_len = ctrl_len + extdata_len;
1229 struct ceph_frame_desc desc;
1230 int ret;
1231
1232 dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag,
1233 total_len, ctrl_len, extdata_len);
1234
1235 /* extdata may be vmalloc'ed but not base */
1236 if (WARN_ON(is_vmalloc_addr(base) || !ctrl_len))
1237 return -EINVAL;
1238
1239 init_frame_desc(&desc, tag, &total_len, 1);
1240 encode_preamble(&desc, base);
1241
1242 if (con_secure(con)) {
1243 if (WARN_ON(extdata_len || to_be_signed))
1244 return -EINVAL;
1245
1246 if (ctrl_len <= CEPH_PREAMBLE_INLINE_LEN)
1247 /* fully inlined, inline buffer may need padding */
1248 ret = prepare_head_secure_small(con, base, ctrl_len);
1249 else
1250 /* partially inlined, inline buffer is full */
1251 ret = prepare_head_secure_big(con, base, ctrl_len);
1252 if (ret)
1253 return ret;
1254 } else {
1255 prepare_head_plain(con, base, ctrl_len, extdata, extdata_len,
1256 to_be_signed);
1257 }
1258
1259 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1260 return 0;
1261}
1262
1263static int prepare_control(struct ceph_connection *con, int tag,
1264 void *base, int ctrl_len)
1265{
1266 return __prepare_control(con, tag, base, ctrl_len, NULL, 0, false);
1267}
1268
1269static int prepare_hello(struct ceph_connection *con)
1270{
1271 void *buf, *p;
1272 int ctrl_len;
1273
1274 ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr);
1275 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1276 if (!buf)
1277 return -ENOMEM;
1278
1279 p = CTRL_BODY(buf);
1280 ceph_encode_8(&p, CEPH_ENTITY_TYPE_CLIENT);
1281 ceph_encode_entity_addr(&p, &con->peer_addr);
1282 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1283
1284 return __prepare_control(con, FRAME_TAG_HELLO, buf, ctrl_len,
1285 NULL, 0, true);
1286}
1287
1288/* so that head_onwire_len(AUTH_BUF_LEN, false) is 512 */
1289#define AUTH_BUF_LEN (512 - CEPH_CRC_LEN - CEPH_PREAMBLE_PLAIN_LEN)
1290
1291static int prepare_auth_request(struct ceph_connection *con)
1292{
1293 void *authorizer, *authorizer_copy;
1294 int ctrl_len, authorizer_len;
1295 void *buf;
1296 int ret;
1297
1298 ctrl_len = AUTH_BUF_LEN;
1299 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1300 if (!buf)
1301 return -ENOMEM;
1302
1303 mutex_unlock(&con->mutex);
1304 ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len,
1305 &authorizer, &authorizer_len);
1306 mutex_lock(&con->mutex);
1307 if (con->state != CEPH_CON_S_V2_HELLO) {
1308 dout("%s con %p state changed to %d\n", __func__, con,
1309 con->state);
1310 return -EAGAIN;
1311 }
1312
1313 dout("%s con %p get_auth_request ret %d\n", __func__, con, ret);
1314 if (ret)
1315 return ret;
1316
1317 authorizer_copy = alloc_conn_buf(con, authorizer_len);
1318 if (!authorizer_copy)
1319 return -ENOMEM;
1320
1321 memcpy(authorizer_copy, authorizer, authorizer_len);
1322
1323 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, buf, ctrl_len,
1324 authorizer_copy, authorizer_len, true);
1325}
1326
1327static int prepare_auth_request_more(struct ceph_connection *con,
1328 void *reply, int reply_len)
1329{
1330 int ctrl_len, authorizer_len;
1331 void *authorizer;
1332 void *buf;
1333 int ret;
1334
1335 ctrl_len = AUTH_BUF_LEN;
1336 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1337 if (!buf)
1338 return -ENOMEM;
1339
1340 mutex_unlock(&con->mutex);
1341 ret = con->ops->handle_auth_reply_more(con, reply, reply_len,
1342 CTRL_BODY(buf), &ctrl_len,
1343 &authorizer, &authorizer_len);
1344 mutex_lock(&con->mutex);
1345 if (con->state != CEPH_CON_S_V2_AUTH) {
1346 dout("%s con %p state changed to %d\n", __func__, con,
1347 con->state);
1348 return -EAGAIN;
1349 }
1350
1351 dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret);
1352 if (ret)
1353 return ret;
1354
1355 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, buf,
1356 ctrl_len, authorizer, authorizer_len, true);
1357}
1358
1359static int prepare_auth_signature(struct ceph_connection *con)
1360{
1361 void *buf;
1362 int ret;
1363
ad32fe88
ID		 1364 buf = alloc_conn_buf(con, head_onwire_len(SHA256_DIGEST_SIZE,
1365 con_secure(con)));
cd1a677c
ID		 1366 if (!buf)
1367 return -ENOMEM;
1368
1369 ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt,
1370 CTRL_BODY(buf));
1371 if (ret)
1372 return ret;
1373
1374 return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf,
1375 SHA256_DIGEST_SIZE);
1376}
1377
1378static int prepare_client_ident(struct ceph_connection *con)
1379{
1380 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1381 struct ceph_client *client = from_msgr(con->msgr);
1382 u64 global_id = ceph_client_gid(client);
1383 void *buf, *p;
1384 int ctrl_len;
1385
1386 WARN_ON(con->v2.server_cookie);
1387 WARN_ON(con->v2.connect_seq);
1388 WARN_ON(con->v2.peer_global_seq);
1389
1390 if (!con->v2.client_cookie) {
1391 do {
1392 get_random_bytes(&con->v2.client_cookie,
1393 sizeof(con->v2.client_cookie));
1394 } while (!con->v2.client_cookie);
1395 dout("%s con %p generated cookie 0x%llx\n", __func__, con,
1396 con->v2.client_cookie);
1397 } else {
1398 dout("%s con %p cookie already set 0x%llx\n", __func__, con,
1399 con->v2.client_cookie);
1400 }
1401
1402 dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx cookie 0x%llx\n",
1403 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1404 ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce),
1405 global_id, con->v2.global_seq, client->supported_features,
1406 client->required_features, con->v2.client_cookie);
1407
1408 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) +
1409 ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8;
1410 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1411 if (!buf)
1412 return -ENOMEM;
1413
1414 p = CTRL_BODY(buf);
1415 ceph_encode_8(&p, 2); /* addrvec marker */
1416 ceph_encode_32(&p, 1); /* addr_cnt */
1417 ceph_encode_entity_addr(&p, my_addr);
1418 ceph_encode_entity_addr(&p, &con->peer_addr);
1419 ceph_encode_64(&p, global_id);
1420 ceph_encode_64(&p, con->v2.global_seq);
1421 ceph_encode_64(&p, client->supported_features);
1422 ceph_encode_64(&p, client->required_features);
1423 ceph_encode_64(&p, 0); /* flags */
1424 ceph_encode_64(&p, con->v2.client_cookie);
1425 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1426
1427 return prepare_control(con, FRAME_TAG_CLIENT_IDENT, buf, ctrl_len);
1428}
1429
1430static int prepare_session_reconnect(struct ceph_connection *con)
1431{
1432 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1433 void *buf, *p;
1434 int ctrl_len;
1435
1436 WARN_ON(!con->v2.client_cookie);
1437 WARN_ON(!con->v2.server_cookie);
1438 WARN_ON(!con->v2.connect_seq);
1439 WARN_ON(!con->v2.peer_global_seq);
1440
1441 dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_seq %llu in_seq %llu\n",
1442 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1443 con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq,
1444 con->v2.connect_seq, con->in_seq);
1445
1446 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + 5 * 8;
1447 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1448 if (!buf)
1449 return -ENOMEM;
1450
1451 p = CTRL_BODY(buf);
1452 ceph_encode_8(&p, 2); /* entity_addrvec_t marker */
1453 ceph_encode_32(&p, 1); /* my_addrs len */
1454 ceph_encode_entity_addr(&p, my_addr);
1455 ceph_encode_64(&p, con->v2.client_cookie);
1456 ceph_encode_64(&p, con->v2.server_cookie);
1457 ceph_encode_64(&p, con->v2.global_seq);
1458 ceph_encode_64(&p, con->v2.connect_seq);
1459 ceph_encode_64(&p, con->in_seq);
1460 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1461
1462 return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, buf, ctrl_len);
1463}
1464
1465static int prepare_keepalive2(struct ceph_connection *con)
1466{
1467 struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf);
1468 struct timespec64 now;
1469
1470 ktime_get_real_ts64(&now);
1471 dout("%s con %p timestamp %lld.%09ld\n", __func__, con, now.tv_sec,
1472 now.tv_nsec);
1473
1474 ceph_encode_timespec64(ts, &now);
1475
1476 reset_out_kvecs(con);
1477 return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf,
1478 sizeof(struct ceph_timespec));
1479}
1480
1481static int prepare_ack(struct ceph_connection *con)
1482{
1483 void *p;
1484
1485 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1486 con->in_seq_acked, con->in_seq);
1487 con->in_seq_acked = con->in_seq;
1488
1489 p = CTRL_BODY(con->v2.out_buf);
1490 ceph_encode_64(&p, con->in_seq_acked);
1491
1492 reset_out_kvecs(con);
1493 return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8);
1494}
1495
1496static void prepare_epilogue_plain(struct ceph_connection *con, bool aborted)
1497{
1498 dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con,
1499 con->out_msg, aborted, con->v2.out_epil.front_crc,
1500 con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc);
1501
1502 encode_epilogue_plain(con, aborted);
1503 add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN);
1504}
1505
1506/*
1507 * For "used" empty segments, crc is -1. For unused (trailing)
1508 * segments, crc is 0.
1509 */
1510static void prepare_message_plain(struct ceph_connection *con)
1511{
1512 struct ceph_msg *msg = con->out_msg;
1513
1514 prepare_head_plain(con, con->v2.out_buf,
1515 sizeof(struct ceph_msg_header2), NULL, 0, false);
1516
1517 if (!front_len(msg) && !middle_len(msg)) {
1518 if (!data_len(msg)) {
1519 /*
1520 * Empty message: once the head is written,
1521 * we are done -- there is no epilogue.
1522 */
1523 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1524 return;
1525 }
1526
1527 con->v2.out_epil.front_crc = -1;
1528 con->v2.out_epil.middle_crc = -1;
1529 con->v2.out_state = OUT_S_QUEUE_DATA;
1530 return;
1531 }
1532
1533 if (front_len(msg)) {
1534 con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base,
1535 front_len(msg));
1536 add_out_kvec(con, msg->front.iov_base, front_len(msg));
1537 } else {
1538 /* middle (at least) is there, checked above */
1539 con->v2.out_epil.front_crc = -1;
1540 }
1541
1542 if (middle_len(msg)) {
1543 con->v2.out_epil.middle_crc =
1544 crc32c(-1, msg->middle->vec.iov_base, middle_len(msg));
1545 add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
1546 } else {
1547 con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0;
1548 }
1549
1550 if (data_len(msg)) {
1551 con->v2.out_state = OUT_S_QUEUE_DATA;
1552 } else {
1553 con->v2.out_epil.data_crc = 0;
1554 prepare_epilogue_plain(con, false);
1555 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1556 }
1557}
1558
1559/*
1560 * Unfortunately the kernel crypto API doesn't support streaming
1561 * (piecewise) operation for AEAD algorithms, so we can't get away
1562 * with a fixed size buffer and a couple sgs. Instead, we have to
1563 * allocate pages for the entire tail of the message (currently up
1564 * to ~32M) and two sgs arrays (up to ~256K each)...
1565 */
1566static int prepare_message_secure(struct ceph_connection *con)
1567{
1568 void *zerop = page_address(ceph_zero_page);
1569 struct sg_table enc_sgt = {};
1570 struct sg_table sgt = {};
1571 struct page **enc_pages;
1572 int enc_page_cnt;
1573 int tail_len;
1574 int ret;
1575
1576 ret = prepare_head_secure_small(con, con->v2.out_buf,
1577 sizeof(struct ceph_msg_header2));
1578 if (ret)
1579 return ret;
1580
1581 tail_len = tail_onwire_len(con->out_msg, true);
1582 if (!tail_len) {
1583 /*
1584 * Empty message: once the head is written,
1585 * we are done -- there is no epilogue.
1586 */
1587 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1588 return 0;
1589 }
1590
1591 encode_epilogue_secure(con, false);
1592 ret = setup_message_sgs(&sgt, con->out_msg, zerop, zerop, zerop,
1593 &con->v2.out_epil, false);
1594 if (ret)
1595 goto out;
1596
1597 enc_page_cnt = calc_pages_for(0, tail_len);
1598 enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
1599 if (IS_ERR(enc_pages)) {
1600 ret = PTR_ERR(enc_pages);
1601 goto out;
1602 }
1603
1604 WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt);
1605 con->v2.out_enc_pages = enc_pages;
1606 con->v2.out_enc_page_cnt = enc_page_cnt;
1607 con->v2.out_enc_resid = tail_len;
1608 con->v2.out_enc_i = 0;
1609
1610 ret = sg_alloc_table_from_pages(&enc_sgt, enc_pages, enc_page_cnt,
1611 0, tail_len, GFP_NOIO);
1612 if (ret)
1613 goto out;
1614
1615 ret = gcm_crypt(con, true, sgt.sgl, enc_sgt.sgl,
1616 tail_len - CEPH_GCM_TAG_LEN);
1617 if (ret)
1618 goto out;
1619
1620 dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con,
1621 con->out_msg, sgt.orig_nents, enc_page_cnt);
1622 con->v2.out_state = OUT_S_QUEUE_ENC_PAGE;
1623
1624out:
1625 sg_free_table(&sgt);
1626 sg_free_table(&enc_sgt);
1627 return ret;
1628}
1629
1630static int prepare_message(struct ceph_connection *con)
1631{
1632 int lens[] = {
1633 sizeof(struct ceph_msg_header2),
1634 front_len(con->out_msg),
1635 middle_len(con->out_msg),
1636 data_len(con->out_msg)
1637 };
1638 struct ceph_frame_desc desc;
1639 int ret;
1640
1641 dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con,
1642 con->out_msg, lens[0], lens[1], lens[2], lens[3]);
1643
1644 if (con->in_seq > con->in_seq_acked) {
1645 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1646 con->in_seq_acked, con->in_seq);
1647 con->in_seq_acked = con->in_seq;
1648 }
1649
1650 reset_out_kvecs(con);
1651 init_frame_desc(&desc, FRAME_TAG_MESSAGE, lens, 4);
1652 encode_preamble(&desc, con->v2.out_buf);
1653 fill_header2(CTRL_BODY(con->v2.out_buf), &con->out_msg->hdr,
1654 con->in_seq_acked);
1655
1656 if (con_secure(con)) {
1657 ret = prepare_message_secure(con);
1658 if (ret)
1659 return ret;
1660 } else {
1661 prepare_message_plain(con);
1662 }
1663
1664 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1665 return 0;
1666}
1667
1668static int prepare_read_banner_prefix(struct ceph_connection *con)
1669{
1670 void *buf;
1671
1672 buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN);
1673 if (!buf)
1674 return -ENOMEM;
1675
1676 reset_in_kvecs(con);
1677 add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1678 add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1679 con->state = CEPH_CON_S_V2_BANNER_PREFIX;
1680 return 0;
1681}
1682
1683static int prepare_read_banner_payload(struct ceph_connection *con,
1684 int payload_len)
1685{
1686 void *buf;
1687
1688 buf = alloc_conn_buf(con, payload_len);
1689 if (!buf)
1690 return -ENOMEM;
1691
1692 reset_in_kvecs(con);
1693 add_in_kvec(con, buf, payload_len);
1694 add_in_sign_kvec(con, buf, payload_len);
1695 con->state = CEPH_CON_S_V2_BANNER_PAYLOAD;
1696 return 0;
1697}
1698
1699static void prepare_read_preamble(struct ceph_connection *con)
1700{
1701 reset_in_kvecs(con);
1702 add_in_kvec(con, con->v2.in_buf,
1703 con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN :
1704 CEPH_PREAMBLE_PLAIN_LEN);
1705 con->v2.in_state = IN_S_HANDLE_PREAMBLE;
1706}
1707
1708static int prepare_read_control(struct ceph_connection *con)
1709{
1710 int ctrl_len = con->v2.in_desc.fd_lens[0];
1711 int head_len;
1712 void *buf;
1713
1714 reset_in_kvecs(con);
1715 if (con->state == CEPH_CON_S_V2_HELLO ||
1716 con->state == CEPH_CON_S_V2_AUTH) {
1717 head_len = head_onwire_len(ctrl_len, false);
1718 buf = alloc_conn_buf(con, head_len);
1719 if (!buf)
1720 return -ENOMEM;
1721
1722 /* preserve preamble */
1723 memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN);
1724
1725 add_in_kvec(con, CTRL_BODY(buf), ctrl_len);
1726 add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN);
1727 add_in_sign_kvec(con, buf, head_len);
1728 } else {
1729 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
1730 buf = alloc_conn_buf(con, ctrl_len);
1731 if (!buf)
1732 return -ENOMEM;
1733
1734 add_in_kvec(con, buf, ctrl_len);
1735 } else {
1736 add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len);
1737 }
1738 add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN);
1739 }
1740 con->v2.in_state = IN_S_HANDLE_CONTROL;
1741 return 0;
1742}
1743
1744static int prepare_read_control_remainder(struct ceph_connection *con)
1745{
1746 int ctrl_len = con->v2.in_desc.fd_lens[0];
1747 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1748 void *buf;
1749
1750 buf = alloc_conn_buf(con, ctrl_len);
1751 if (!buf)
1752 return -ENOMEM;
1753
1754 memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN);
1755
1756 reset_in_kvecs(con);
1757 add_in_kvec(con, buf + CEPH_PREAMBLE_INLINE_LEN, rem_len);
1758 add_in_kvec(con, con->v2.in_buf,
1759 padding_len(rem_len) + CEPH_GCM_TAG_LEN);
1760 con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER;
1761 return 0;
1762}
1763
038b8d1d 1764static int prepare_read_data(struct ceph_connection *con)
cd1a677c
ID		 1765{
1766 struct bio_vec bv;
1767
2ea88716 1768 con->in_data_crc = -1;
cd1a677c
ID		 1769 ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg,
1770 data_len(con->in_msg));
1771
1772 get_bvec_at(&con->v2.in_cursor, &bv);
038b8d1d
ID		 1773 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1774 if (unlikely(!con->bounce_page)) {
1775 con->bounce_page = alloc_page(GFP_NOIO);
1776 if (!con->bounce_page) {
1777 pr_err("failed to allocate bounce page\n");
1778 return -ENOMEM;
1779 }
1780 }
1781
1782 bv.bv_page = con->bounce_page;
1783 bv.bv_offset = 0;
038b8d1d 1784 }
27884f4b 1785 set_in_bvec(con, &bv);
cd1a677c 1786 con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT;
038b8d1d 1787 return 0;
cd1a677c
ID		 1788}
1789
1790static void prepare_read_data_cont(struct ceph_connection *con)
1791{
1792 struct bio_vec bv;
1793
038b8d1d
ID		 1794 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1795 con->in_data_crc = crc32c(con->in_data_crc,
1796 page_address(con->bounce_page),
1797 con->v2.in_bvec.bv_len);
1798
1799 get_bvec_at(&con->v2.in_cursor, &bv);
1800 memcpy_to_page(bv.bv_page, bv.bv_offset,
1801 page_address(con->bounce_page),
1802 con->v2.in_bvec.bv_len);
1803 } else {
1804 con->in_data_crc = ceph_crc32c_page(con->in_data_crc,
1805 con->v2.in_bvec.bv_page,
1806 con->v2.in_bvec.bv_offset,
1807 con->v2.in_bvec.bv_len);
1808 }
cd1a677c
ID		 1809
1810 ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len);
1811 if (con->v2.in_cursor.total_resid) {
1812 get_bvec_at(&con->v2.in_cursor, &bv);
038b8d1d
ID		 1813 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1814 bv.bv_page = con->bounce_page;
1815 bv.bv_offset = 0;
038b8d1d 1816 }
27884f4b 1817 set_in_bvec(con, &bv);
cd1a677c
ID		 1818 WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT);
1819 return;
1820 }
1821
1822 /*
2ea88716 1823 * We've read all data. Prepare to read epilogue.
cd1a677c
ID		 1824 */
1825 reset_in_kvecs(con);
2ea88716
ID		 1826 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1827 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1828}
1829
ec3bc567
JL		 1830static int prepare_sparse_read_cont(struct ceph_connection *con)
1831{
1832 int ret;
1833 struct bio_vec bv;
1834 char *buf = NULL;
1835 struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor;
1836
1837 WARN_ON(con->v2.in_state != IN_S_PREPARE_SPARSE_DATA_CONT);
1838
1839 if (iov_iter_is_bvec(&con->v2.in_iter)) {
1840 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1841 con->in_data_crc = crc32c(con->in_data_crc,
1842 page_address(con->bounce_page),
1843 con->v2.in_bvec.bv_len);
1844 get_bvec_at(cursor, &bv);
1845 memcpy_to_page(bv.bv_page, bv.bv_offset,
1846 page_address(con->bounce_page),
1847 con->v2.in_bvec.bv_len);
1848 } else {
1849 con->in_data_crc = ceph_crc32c_page(con->in_data_crc,
1850 con->v2.in_bvec.bv_page,
1851 con->v2.in_bvec.bv_offset,
1852 con->v2.in_bvec.bv_len);
1853 }
1854
1855 ceph_msg_data_advance(cursor, con->v2.in_bvec.bv_len);
1856 cursor->sr_resid -= con->v2.in_bvec.bv_len;
1857 dout("%s: advance by 0x%x sr_resid 0x%x\n", __func__,
1858 con->v2.in_bvec.bv_len, cursor->sr_resid);
1859 WARN_ON_ONCE(cursor->sr_resid > cursor->total_resid);
1860 if (cursor->sr_resid) {
1861 get_bvec_at(cursor, &bv);
1862 if (bv.bv_len > cursor->sr_resid)
1863 bv.bv_len = cursor->sr_resid;
1864 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1865 bv.bv_page = con->bounce_page;
1866 bv.bv_offset = 0;
1867 }
1868 set_in_bvec(con, &bv);
1869 con->v2.data_len_remain -= bv.bv_len;
1870 return 0;
1871 }
1872 } else if (iov_iter_is_kvec(&con->v2.in_iter)) {
1873 /* On first call, we have no kvec so don't compute crc */
1874 if (con->v2.in_kvec_cnt) {
1875 WARN_ON_ONCE(con->v2.in_kvec_cnt > 1);
1876 con->in_data_crc = crc32c(con->in_data_crc,
1877 con->v2.in_kvecs[0].iov_base,
1878 con->v2.in_kvecs[0].iov_len);
1879 }
1880 } else {
1881 return -EIO;
1882 }
1883
1884 /* get next extent */
1885 ret = con->ops->sparse_read(con, cursor, &buf);
1886 if (ret <= 0) {
1887 if (ret < 0)
1888 return ret;
1889
1890 reset_in_kvecs(con);
1891 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1892 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1893 return 0;
1894 }
1895
1896 if (buf) {
1897 /* receive into buffer */
1898 reset_in_kvecs(con);
1899 add_in_kvec(con, buf, ret);
1900 con->v2.data_len_remain -= ret;
1901 return 0;
1902 }
1903
1904 if (ret > cursor->total_resid) {
1905 pr_warn("%s: ret 0x%x total_resid 0x%zx resid 0x%zx\n",
1906 __func__, ret, cursor->total_resid, cursor->resid);
1907 return -EIO;
1908 }
1909 get_bvec_at(cursor, &bv);
1910 if (bv.bv_len > cursor->sr_resid)
1911 bv.bv_len = cursor->sr_resid;
1912 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1913 if (unlikely(!con->bounce_page)) {
1914 con->bounce_page = alloc_page(GFP_NOIO);
1915 if (!con->bounce_page) {
1916 pr_err("failed to allocate bounce page\n");
1917 return -ENOMEM;
1918 }
1919 }
1920
1921 bv.bv_page = con->bounce_page;
1922 bv.bv_offset = 0;
1923 }
1924 set_in_bvec(con, &bv);
1925 con->v2.data_len_remain -= ret;
1926 return ret;
1927}
1928
1929static int prepare_sparse_read_data(struct ceph_connection *con)
1930{
1931 struct ceph_msg *msg = con->in_msg;
1932
1933 dout("%s: starting sparse read\n", __func__);
1934
1935 if (WARN_ON_ONCE(!con->ops->sparse_read))
1936 return -EOPNOTSUPP;
1937
1938 if (!con_secure(con))
1939 con->in_data_crc = -1;
1940
1941 reset_in_kvecs(con);
1942 con->v2.in_state = IN_S_PREPARE_SPARSE_DATA_CONT;
1943 con->v2.data_len_remain = data_len(msg);
1944 return prepare_sparse_read_cont(con);
1945}
1946
038b8d1d 1947static int prepare_read_tail_plain(struct ceph_connection *con)
2ea88716
ID		 1948{
1949 struct ceph_msg *msg = con->in_msg;
1950
1951 if (!front_len(msg) && !middle_len(msg)) {
1952 WARN_ON(!data_len(msg));
038b8d1d 1953 return prepare_read_data(con);
2ea88716
ID		 1954 }
1955
1956 reset_in_kvecs(con);
1957 if (front_len(msg)) {
1958 add_in_kvec(con, msg->front.iov_base, front_len(msg));
1959 WARN_ON(msg->front.iov_len != front_len(msg));
1960 }
1961 if (middle_len(msg)) {
1962 add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
1963 WARN_ON(msg->middle->vec.iov_len != middle_len(msg));
1964 }
1965
1966 if (data_len(msg)) {
ec3bc567
JL		 1967 if (msg->sparse_read)
1968 con->v2.in_state = IN_S_PREPARE_SPARSE_DATA;
1969 else
1970 con->v2.in_state = IN_S_PREPARE_READ_DATA;
cd1a677c
ID		 1971 } else {
1972 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
2ea88716
ID		 1973 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1974 }
038b8d1d 1975 return 0;
2ea88716
ID		 1976}
1977
1978static void prepare_read_enc_page(struct ceph_connection *con)
1979{
1980 struct bio_vec bv;
1981
1982 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.in_enc_i,
1983 con->v2.in_enc_resid);
1984 WARN_ON(!con->v2.in_enc_resid);
1985
1eb9cd15
CH		 1986 bvec_set_page(&bv, con->v2.in_enc_pages[con->v2.in_enc_i],
1987 min(con->v2.in_enc_resid, (int)PAGE_SIZE), 0);
2ea88716
ID		 1988
1989 set_in_bvec(con, &bv);
1990 con->v2.in_enc_i++;
1991 con->v2.in_enc_resid -= bv.bv_len;
1992
1993 if (con->v2.in_enc_resid) {
1994 con->v2.in_state = IN_S_PREPARE_READ_ENC_PAGE;
1995 return;
cd1a677c 1996 }
2ea88716
ID		 1997
1998 /*
1999 * We are set to read the last piece of ciphertext (ending
2000 * with epilogue) + auth tag.
2001 */
2002 WARN_ON(con->v2.in_enc_i != con->v2.in_enc_page_cnt);
cd1a677c
ID		 2003 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
2004}
2005
2ea88716
ID		 2006static int prepare_read_tail_secure(struct ceph_connection *con)
2007{
2008 struct page **enc_pages;
2009 int enc_page_cnt;
2010 int tail_len;
2011
2012 tail_len = tail_onwire_len(con->in_msg, true);
2013 WARN_ON(!tail_len);
2014
2015 enc_page_cnt = calc_pages_for(0, tail_len);
2016 enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
2017 if (IS_ERR(enc_pages))
2018 return PTR_ERR(enc_pages);
2019
2020 WARN_ON(con->v2.in_enc_pages || con->v2.in_enc_page_cnt);
2021 con->v2.in_enc_pages = enc_pages;
2022 con->v2.in_enc_page_cnt = enc_page_cnt;
2023 con->v2.in_enc_resid = tail_len;
2024 con->v2.in_enc_i = 0;
2025
2026 prepare_read_enc_page(con);
2027 return 0;
2028}
2029
cd1a677c
ID		 2030static void __finish_skip(struct ceph_connection *con)
2031{
2032 con->in_seq++;
2033 prepare_read_preamble(con);
2034}
2035
2036static void prepare_skip_message(struct ceph_connection *con)
2037{
2038 struct ceph_frame_desc *desc = &con->v2.in_desc;
2039 int tail_len;
2040
2041 dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1],
2042 desc->fd_lens[2], desc->fd_lens[3]);
2043
2044 tail_len = __tail_onwire_len(desc->fd_lens[1], desc->fd_lens[2],
2045 desc->fd_lens[3], con_secure(con));
2046 if (!tail_len) {
2047 __finish_skip(con);
2048 } else {
2049 set_in_skip(con, tail_len);
2050 con->v2.in_state = IN_S_FINISH_SKIP;
2051 }
2052}
2053
2054static int process_banner_prefix(struct ceph_connection *con)
2055{
2056 int payload_len;
2057 void *p;
2058
2059 WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN);
2060
2061 p = con->v2.in_kvecs[0].iov_base;
2062 if (memcmp(p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN)) {
2063 if (!memcmp(p, CEPH_BANNER, CEPH_BANNER_LEN))
2064 con->error_msg = "server is speaking msgr1 protocol";
2065 else
2066 con->error_msg = "protocol error, bad banner";
2067 return -EINVAL;
2068 }
2069
2070 p += CEPH_BANNER_V2_LEN;
2071 payload_len = ceph_decode_16(&p);
2072 dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2073
2074 return prepare_read_banner_payload(con, payload_len);
2075}
2076
2077static int process_banner_payload(struct ceph_connection *con)
2078{
2079 void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len;
2080 u64 feat = CEPH_MSGR2_SUPPORTED_FEATURES;
2081 u64 req_feat = CEPH_MSGR2_REQUIRED_FEATURES;
2082 u64 server_feat, server_req_feat;
2083 void *p;
2084 int ret;
2085
2086 p = con->v2.in_kvecs[0].iov_base;
2087 ceph_decode_64_safe(&p, end, server_feat, bad);
2088 ceph_decode_64_safe(&p, end, server_req_feat, bad);
2089
2090 dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n",
2091 __func__, con, server_feat, server_req_feat);
2092
2093 if (req_feat & ~server_feat) {
2094 pr_err("msgr2 feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2095 server_feat, req_feat & ~server_feat);
2096 con->error_msg = "missing required protocol features";
2097 return -EINVAL;
2098 }
2099 if (server_req_feat & ~feat) {
2100 pr_err("msgr2 feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2101 feat, server_req_feat & ~feat);
2102 con->error_msg = "missing required protocol features";
2103 return -EINVAL;
2104 }
2105
2106 /* no reset_out_kvecs() as our banner may still be pending */
2107 ret = prepare_hello(con);
2108 if (ret) {
2109 pr_err("prepare_hello failed: %d\n", ret);
2110 return ret;
2111 }
2112
2113 con->state = CEPH_CON_S_V2_HELLO;
2114 prepare_read_preamble(con);
2115 return 0;
2116
2117bad:
2118 pr_err("failed to decode banner payload\n");
2119 return -EINVAL;
2120}
2121
2122static int process_hello(struct ceph_connection *con, void *p, void *end)
2123{
2124 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
2125 struct ceph_entity_addr addr_for_me;
2126 u8 entity_type;
2127 int ret;
2128
2129 if (con->state != CEPH_CON_S_V2_HELLO) {
2130 con->error_msg = "protocol error, unexpected hello";
2131 return -EINVAL;
2132 }
2133
2134 ceph_decode_8_safe(&p, end, entity_type, bad);
2135 ret = ceph_decode_entity_addr(&p, end, &addr_for_me);
2136 if (ret) {
2137 pr_err("failed to decode addr_for_me: %d\n", ret);
2138 return ret;
2139 }
2140
2141 dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con,
2142 entity_type, ceph_pr_addr(&addr_for_me));
2143
2144 if (entity_type != con->peer_name.type) {
2145 pr_err("bad peer type, want %d, got %d\n",
2146 con->peer_name.type, entity_type);
2147 con->error_msg = "wrong peer at address";
2148 return -EINVAL;
2149 }
2150
2151 /*
2152 * Set our address to the address our first peer (i.e. monitor)
2153 * sees that we are connecting from. If we are behind some sort
2154 * of NAT and want to be identified by some private (not NATed)
2155 * address, ip option should be used.
2156 */
2157 if (ceph_addr_is_blank(my_addr)) {
2158 memcpy(&my_addr->in_addr, &addr_for_me.in_addr,
2159 sizeof(my_addr->in_addr));
2160 ceph_addr_set_port(my_addr, 0);
2161 dout("%s con %p set my addr %s, as seen by peer %s\n",
2162 __func__, con, ceph_pr_addr(my_addr),
2163 ceph_pr_addr(&con->peer_addr));
2164 } else {
2165 dout("%s con %p my addr already set %s\n",
2166 __func__, con, ceph_pr_addr(my_addr));
2167 }
2168
2169 WARN_ON(ceph_addr_is_blank(my_addr) || ceph_addr_port(my_addr));
2170 WARN_ON(my_addr->type != CEPH_ENTITY_ADDR_TYPE_ANY);
2171 WARN_ON(!my_addr->nonce);
2172
2173 /* no reset_out_kvecs() as our hello may still be pending */
2174 ret = prepare_auth_request(con);
2175 if (ret) {
2176 if (ret != -EAGAIN)
2177 pr_err("prepare_auth_request failed: %d\n", ret);
2178 return ret;
2179 }
2180
2181 con->state = CEPH_CON_S_V2_AUTH;
2182 return 0;
2183
2184bad:
2185 pr_err("failed to decode hello\n");
2186 return -EINVAL;
2187}
2188
2189static int process_auth_bad_method(struct ceph_connection *con,
2190 void *p, void *end)
2191{
2192 int allowed_protos[8], allowed_modes[8];
2193 int allowed_proto_cnt, allowed_mode_cnt;
2194 int used_proto, result;
2195 int ret;
2196 int i;
2197
2198 if (con->state != CEPH_CON_S_V2_AUTH) {
2199 con->error_msg = "protocol error, unexpected auth_bad_method";
2200 return -EINVAL;
2201 }
2202
2203 ceph_decode_32_safe(&p, end, used_proto, bad);
2204 ceph_decode_32_safe(&p, end, result, bad);
2205 dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto,
2206 result);
2207
2208 ceph_decode_32_safe(&p, end, allowed_proto_cnt, bad);
2209 if (allowed_proto_cnt > ARRAY_SIZE(allowed_protos)) {
2210 pr_err("allowed_protos too big %d\n", allowed_proto_cnt);
2211 return -EINVAL;
2212 }
2213 for (i = 0; i < allowed_proto_cnt; i++) {
2214 ceph_decode_32_safe(&p, end, allowed_protos[i], bad);
2215 dout("%s con %p allowed_protos[%d] %d\n", __func__, con,
2216 i, allowed_protos[i]);
2217 }
2218
2219 ceph_decode_32_safe(&p, end, allowed_mode_cnt, bad);
2220 if (allowed_mode_cnt > ARRAY_SIZE(allowed_modes)) {
2221 pr_err("allowed_modes too big %d\n", allowed_mode_cnt);
2222 return -EINVAL;
2223 }
2224 for (i = 0; i < allowed_mode_cnt; i++) {
2225 ceph_decode_32_safe(&p, end, allowed_modes[i], bad);
2226 dout("%s con %p allowed_modes[%d] %d\n", __func__, con,
2227 i, allowed_modes[i]);
2228 }
2229
2230 mutex_unlock(&con->mutex);
2231 ret = con->ops->handle_auth_bad_method(con, used_proto, result,
2232 allowed_protos,
2233 allowed_proto_cnt,
2234 allowed_modes,
2235 allowed_mode_cnt);
2236 mutex_lock(&con->mutex);
2237 if (con->state != CEPH_CON_S_V2_AUTH) {
2238 dout("%s con %p state changed to %d\n", __func__, con,
2239 con->state);
2240 return -EAGAIN;
2241 }
2242
2243 dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret);
2244 return ret;
2245
2246bad:
2247 pr_err("failed to decode auth_bad_method\n");
2248 return -EINVAL;
2249}
2250
2251static int process_auth_reply_more(struct ceph_connection *con,
2252 void *p, void *end)
2253{
2254 int payload_len;
2255 int ret;
2256
2257 if (con->state != CEPH_CON_S_V2_AUTH) {
2258 con->error_msg = "protocol error, unexpected auth_reply_more";
2259 return -EINVAL;
2260 }
2261
2262 ceph_decode_32_safe(&p, end, payload_len, bad);
2263 ceph_decode_need(&p, end, payload_len, bad);
2264
2265 dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2266
2267 reset_out_kvecs(con);
2268 ret = prepare_auth_request_more(con, p, payload_len);
2269 if (ret) {
2270 if (ret != -EAGAIN)
2271 pr_err("prepare_auth_request_more failed: %d\n", ret);
2272 return ret;
2273 }
2274
2275 return 0;
2276
2277bad:
2278 pr_err("failed to decode auth_reply_more\n");
2279 return -EINVAL;
2280}
2281
f5f2c9a0
ID		 2282/*
2283 * Align session_key and con_secret to avoid GFP_ATOMIC allocation
2284 * inside crypto_shash_setkey() and crypto_aead_setkey() called from
2285 * setup_crypto(). __aligned(16) isn't guaranteed to work for stack
2286 * objects, so do it by hand.
2287 */
cd1a677c
ID		 2288static int process_auth_done(struct ceph_connection *con, void *p, void *end)
2289{
f5f2c9a0
ID		 2290 u8 session_key_buf[CEPH_KEY_LEN + 16];
2291 u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16];
2292 u8 *session_key = PTR_ALIGN(&session_key_buf[0], 16);
2293 u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16);
cd1a677c
ID		 2294 int session_key_len, con_secret_len;
2295 int payload_len;
2296 u64 global_id;
2297 int ret;
2298
2299 if (con->state != CEPH_CON_S_V2_AUTH) {
2300 con->error_msg = "protocol error, unexpected auth_done";
2301 return -EINVAL;
2302 }
2303
2304 ceph_decode_64_safe(&p, end, global_id, bad);
2305 ceph_decode_32_safe(&p, end, con->v2.con_mode, bad);
2306 ceph_decode_32_safe(&p, end, payload_len, bad);
2307
2308 dout("%s con %p global_id %llu con_mode %d payload_len %d\n",
2309 __func__, con, global_id, con->v2.con_mode, payload_len);
2310
2311 mutex_unlock(&con->mutex);
2312 session_key_len = 0;
2313 con_secret_len = 0;
2314 ret = con->ops->handle_auth_done(con, global_id, p, payload_len,
2315 session_key, &session_key_len,
2316 con_secret, &con_secret_len);
2317 mutex_lock(&con->mutex);
2318 if (con->state != CEPH_CON_S_V2_AUTH) {
2319 dout("%s con %p state changed to %d\n", __func__, con,
2320 con->state);
10f42b3e
ID		 2321 ret = -EAGAIN;
2322 goto out;
cd1a677c
ID		 2323 }
2324
2325 dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret);
2326 if (ret)
10f42b3e 2327 goto out;
cd1a677c
ID		 2328
2329 ret = setup_crypto(con, session_key, session_key_len, con_secret,
2330 con_secret_len);
2331 if (ret)
10f42b3e 2332 goto out;
cd1a677c
ID		 2333
2334 reset_out_kvecs(con);
2335 ret = prepare_auth_signature(con);
2336 if (ret) {
2337 pr_err("prepare_auth_signature failed: %d\n", ret);
10f42b3e 2338 goto out;
cd1a677c
ID		 2339 }
2340
2341 con->state = CEPH_CON_S_V2_AUTH_SIGNATURE;
10f42b3e
ID		 2342
2343out:
2344 memzero_explicit(session_key_buf, sizeof(session_key_buf));
2345 memzero_explicit(con_secret_buf, sizeof(con_secret_buf));
2346 return ret;
cd1a677c
ID		 2347
2348bad:
2349 pr_err("failed to decode auth_done\n");
2350 return -EINVAL;
2351}
2352
2353static int process_auth_signature(struct ceph_connection *con,
2354 void *p, void *end)
2355{
2356 u8 hmac[SHA256_DIGEST_SIZE];
2357 int ret;
2358
2359 if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) {
2360 con->error_msg = "protocol error, unexpected auth_signature";
2361 return -EINVAL;
2362 }
2363
2364 ret = hmac_sha256(con, con->v2.out_sign_kvecs,
2365 con->v2.out_sign_kvec_cnt, hmac);
2366 if (ret)
2367 return ret;
2368
2369 ceph_decode_need(&p, end, SHA256_DIGEST_SIZE, bad);
2370 if (crypto_memneq(p, hmac, SHA256_DIGEST_SIZE)) {
2371 con->error_msg = "integrity error, bad auth signature";
2372 return -EBADMSG;
2373 }
2374
2375 dout("%s con %p auth signature ok\n", __func__, con);
2376
2377 /* no reset_out_kvecs() as our auth_signature may still be pending */
2378 if (!con->v2.server_cookie) {
2379 ret = prepare_client_ident(con);
2380 if (ret) {
2381 pr_err("prepare_client_ident failed: %d\n", ret);
2382 return ret;
2383 }
2384
2385 con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2386 } else {
2387 ret = prepare_session_reconnect(con);
2388 if (ret) {
2389 pr_err("prepare_session_reconnect failed: %d\n", ret);
2390 return ret;
2391 }
2392
2393 con->state = CEPH_CON_S_V2_SESSION_RECONNECT;
2394 }
2395
2396 return 0;
2397
2398bad:
2399 pr_err("failed to decode auth_signature\n");
2400 return -EINVAL;
2401}
2402
2403static int process_server_ident(struct ceph_connection *con,
2404 void *p, void *end)
2405{
2406 struct ceph_client *client = from_msgr(con->msgr);
2407 u64 features, required_features;
2408 struct ceph_entity_addr addr;
2409 u64 global_seq;
2410 u64 global_id;
2411 u64 cookie;
2412 u64 flags;
2413 int ret;
2414
2415 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2416 con->error_msg = "protocol error, unexpected server_ident";
2417 return -EINVAL;
2418 }
2419
2420 ret = ceph_decode_entity_addrvec(&p, end, true, &addr);
2421 if (ret) {
2422 pr_err("failed to decode server addrs: %d\n", ret);
2423 return ret;
2424 }
2425
2426 ceph_decode_64_safe(&p, end, global_id, bad);
2427 ceph_decode_64_safe(&p, end, global_seq, bad);
2428 ceph_decode_64_safe(&p, end, features, bad);
2429 ceph_decode_64_safe(&p, end, required_features, bad);
2430 ceph_decode_64_safe(&p, end, flags, bad);
2431 ceph_decode_64_safe(&p, end, cookie, bad);
2432
2433 dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx flags 0x%llx cookie 0x%llx\n",
2434 __func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce),
2435 global_id, global_seq, features, required_features, flags, cookie);
2436
2437 /* is this who we intended to talk to? */
2438 if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) {
2439 pr_err("bad peer addr/nonce, want %s/%u, got %s/%u\n",
2440 ceph_pr_addr(&con->peer_addr),
2441 le32_to_cpu(con->peer_addr.nonce),
2442 ceph_pr_addr(&addr), le32_to_cpu(addr.nonce));
2443 con->error_msg = "wrong peer at address";
2444 return -EINVAL;
2445 }
2446
2447 if (client->required_features & ~features) {
2448 pr_err("RADOS feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2449 features, client->required_features & ~features);
2450 con->error_msg = "missing required protocol features";
2451 return -EINVAL;
2452 }
2453
2454 /*
2455 * Both name->type and name->num are set in ceph_con_open() but
2456 * name->num may be bogus in the initial monmap. name->type is
2457 * verified in handle_hello().
2458 */
2459 WARN_ON(!con->peer_name.type);
2460 con->peer_name.num = cpu_to_le64(global_id);
2461 con->v2.peer_global_seq = global_seq;
2462 con->peer_features = features;
2463 WARN_ON(required_features & ~client->supported_features);
2464 con->v2.server_cookie = cookie;
2465
2466 if (flags & CEPH_MSG_CONNECT_LOSSY) {
2467 ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX);
2468 WARN_ON(con->v2.server_cookie);
2469 } else {
2470 WARN_ON(!con->v2.server_cookie);
2471 }
2472
2473 clear_in_sign_kvecs(con);
2474 clear_out_sign_kvecs(con);
2475 free_conn_bufs(con);
2476 con->delay = 0; /* reset backoff memory */
2477
2478 con->state = CEPH_CON_S_OPEN;
2479 con->v2.out_state = OUT_S_GET_NEXT;
2480 return 0;
2481
2482bad:
2483 pr_err("failed to decode server_ident\n");
2484 return -EINVAL;
2485}
2486
2487static int process_ident_missing_features(struct ceph_connection *con,
2488 void *p, void *end)
2489{
2490 struct ceph_client *client = from_msgr(con->msgr);
2491 u64 missing_features;
2492
2493 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2494 con->error_msg = "protocol error, unexpected ident_missing_features";
2495 return -EINVAL;
2496 }
2497
2498 ceph_decode_64_safe(&p, end, missing_features, bad);
2499 pr_err("RADOS feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2500 client->supported_features, missing_features);
2501 con->error_msg = "missing required protocol features";
2502 return -EINVAL;
2503
2504bad:
2505 pr_err("failed to decode ident_missing_features\n");
2506 return -EINVAL;
2507}
2508
2509static int process_session_reconnect_ok(struct ceph_connection *con,
2510 void *p, void *end)
2511{
2512 u64 seq;
2513
2514 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2515 con->error_msg = "protocol error, unexpected session_reconnect_ok";
2516 return -EINVAL;
2517 }
2518
2519 ceph_decode_64_safe(&p, end, seq, bad);
2520
2521 dout("%s con %p seq %llu\n", __func__, con, seq);
2522 ceph_con_discard_requeued(con, seq);
2523
2524 clear_in_sign_kvecs(con);
2525 clear_out_sign_kvecs(con);
2526 free_conn_bufs(con);
2527 con->delay = 0; /* reset backoff memory */
2528
2529 con->state = CEPH_CON_S_OPEN;
2530 con->v2.out_state = OUT_S_GET_NEXT;
2531 return 0;
2532
2533bad:
2534 pr_err("failed to decode session_reconnect_ok\n");
2535 return -EINVAL;
2536}
2537
2538static int process_session_retry(struct ceph_connection *con,
2539 void *p, void *end)
2540{
2541 u64 connect_seq;
2542 int ret;
2543
2544 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2545 con->error_msg = "protocol error, unexpected session_retry";
2546 return -EINVAL;
2547 }
2548
2549 ceph_decode_64_safe(&p, end, connect_seq, bad);
2550
2551 dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq);
2552 WARN_ON(connect_seq <= con->v2.connect_seq);
2553 con->v2.connect_seq = connect_seq + 1;
2554
2555 free_conn_bufs(con);
2556
2557 reset_out_kvecs(con);
2558 ret = prepare_session_reconnect(con);
2559 if (ret) {
2560 pr_err("prepare_session_reconnect (cseq) failed: %d\n", ret);
2561 return ret;
2562 }
2563
2564 return 0;
2565
2566bad:
2567 pr_err("failed to decode session_retry\n");
2568 return -EINVAL;
2569}
2570
2571static int process_session_retry_global(struct ceph_connection *con,
2572 void *p, void *end)
2573{
2574 u64 global_seq;
2575 int ret;
2576
2577 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2578 con->error_msg = "protocol error, unexpected session_retry_global";
2579 return -EINVAL;
2580 }
2581
2582 ceph_decode_64_safe(&p, end, global_seq, bad);
2583
2584 dout("%s con %p global_seq %llu\n", __func__, con, global_seq);
2585 WARN_ON(global_seq <= con->v2.global_seq);
2586 con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq);
2587
2588 free_conn_bufs(con);
2589
2590 reset_out_kvecs(con);
2591 ret = prepare_session_reconnect(con);
2592 if (ret) {
2593 pr_err("prepare_session_reconnect (gseq) failed: %d\n", ret);
2594 return ret;
2595 }
2596
2597 return 0;
2598
2599bad:
2600 pr_err("failed to decode session_retry_global\n");
2601 return -EINVAL;
2602}
2603
2604static int process_session_reset(struct ceph_connection *con,
2605 void *p, void *end)
2606{
2607 bool full;
2608 int ret;
2609
2610 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2611 con->error_msg = "protocol error, unexpected session_reset";
2612 return -EINVAL;
2613 }
2614
2615 ceph_decode_8_safe(&p, end, full, bad);
2616 if (!full) {
2617 con->error_msg = "protocol error, bad session_reset";
2618 return -EINVAL;
2619 }
2620
2621 pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name),
2622 ceph_pr_addr(&con->peer_addr));
2623 ceph_con_reset_session(con);
2624
2625 mutex_unlock(&con->mutex);
2626 if (con->ops->peer_reset)
2627 con->ops->peer_reset(con);
2628 mutex_lock(&con->mutex);
2629 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2630 dout("%s con %p state changed to %d\n", __func__, con,
2631 con->state);
2632 return -EAGAIN;
2633 }
2634
2635 free_conn_bufs(con);
2636
2637 reset_out_kvecs(con);
2638 ret = prepare_client_ident(con);
2639 if (ret) {
2640 pr_err("prepare_client_ident (rst) failed: %d\n", ret);
2641 return ret;
2642 }
2643
2644 con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2645 return 0;
2646
2647bad:
2648 pr_err("failed to decode session_reset\n");
2649 return -EINVAL;
2650}
2651
2652static int process_keepalive2_ack(struct ceph_connection *con,
2653 void *p, void *end)
2654{
2655 if (con->state != CEPH_CON_S_OPEN) {
2656 con->error_msg = "protocol error, unexpected keepalive2_ack";
2657 return -EINVAL;
2658 }
2659
2660 ceph_decode_need(&p, end, sizeof(struct ceph_timespec), bad);
2661 ceph_decode_timespec64(&con->last_keepalive_ack, p);
2662
2663 dout("%s con %p timestamp %lld.%09ld\n", __func__, con,
2664 con->last_keepalive_ack.tv_sec, con->last_keepalive_ack.tv_nsec);
2665
2666 return 0;
2667
2668bad:
2669 pr_err("failed to decode keepalive2_ack\n");
2670 return -EINVAL;
2671}
2672
2673static int process_ack(struct ceph_connection *con, void *p, void *end)
2674{
2675 u64 seq;
2676
2677 if (con->state != CEPH_CON_S_OPEN) {
2678 con->error_msg = "protocol error, unexpected ack";
2679 return -EINVAL;
2680 }
2681
2682 ceph_decode_64_safe(&p, end, seq, bad);
2683
2684 dout("%s con %p seq %llu\n", __func__, con, seq);
2685 ceph_con_discard_sent(con, seq);
2686 return 0;
2687
2688bad:
2689 pr_err("failed to decode ack\n");
2690 return -EINVAL;
2691}
2692
2693static int process_control(struct ceph_connection *con, void *p, void *end)
2694{
2695 int tag = con->v2.in_desc.fd_tag;
2696 int ret;
2697
2698 dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p));
2699
2700 switch (tag) {
2701 case FRAME_TAG_HELLO:
2702 ret = process_hello(con, p, end);
2703 break;
2704 case FRAME_TAG_AUTH_BAD_METHOD:
2705 ret = process_auth_bad_method(con, p, end);
2706 break;
2707 case FRAME_TAG_AUTH_REPLY_MORE:
2708 ret = process_auth_reply_more(con, p, end);
2709 break;
2710 case FRAME_TAG_AUTH_DONE:
2711 ret = process_auth_done(con, p, end);
2712 break;
2713 case FRAME_TAG_AUTH_SIGNATURE:
2714 ret = process_auth_signature(con, p, end);
2715 break;
2716 case FRAME_TAG_SERVER_IDENT:
2717 ret = process_server_ident(con, p, end);
2718 break;
2719 case FRAME_TAG_IDENT_MISSING_FEATURES:
2720 ret = process_ident_missing_features(con, p, end);
2721 break;
2722 case FRAME_TAG_SESSION_RECONNECT_OK:
2723 ret = process_session_reconnect_ok(con, p, end);
2724 break;
2725 case FRAME_TAG_SESSION_RETRY:
2726 ret = process_session_retry(con, p, end);
2727 break;
2728 case FRAME_TAG_SESSION_RETRY_GLOBAL:
2729 ret = process_session_retry_global(con, p, end);
2730 break;
2731 case FRAME_TAG_SESSION_RESET:
2732 ret = process_session_reset(con, p, end);
2733 break;
2734 case FRAME_TAG_KEEPALIVE2_ACK:
2735 ret = process_keepalive2_ack(con, p, end);
2736 break;
2737 case FRAME_TAG_ACK:
2738 ret = process_ack(con, p, end);
2739 break;
2740 default:
2741 pr_err("bad tag %d\n", tag);
2742 con->error_msg = "protocol error, bad tag";
2743 return -EINVAL;
2744 }
2745 if (ret) {
2746 dout("%s con %p error %d\n", __func__, con, ret);
2747 return ret;
2748 }
2749
2750 prepare_read_preamble(con);
2751 return 0;
2752}
2753
2754/*
2755 * Return:
2756 * 1 - con->in_msg set, read message
2757 * 0 - skip message
2758 * <0 - error
2759 */
2760static int process_message_header(struct ceph_connection *con,
2761 void *p, void *end)
2762{
2763 struct ceph_frame_desc *desc = &con->v2.in_desc;
2764 struct ceph_msg_header2 *hdr2 = p;
2765 struct ceph_msg_header hdr;
2766 int skip;
2767 int ret;
2768 u64 seq;
2769
2770 /* verify seq# */
2771 seq = le64_to_cpu(hdr2->seq);
2772 if ((s64)seq - (s64)con->in_seq < 1) {
2773 pr_info("%s%lld %s skipping old message: seq %llu, expected %llu\n",
2774 ENTITY_NAME(con->peer_name),
2775 ceph_pr_addr(&con->peer_addr),
2776 seq, con->in_seq + 1);
2777 return 0;
2778 }
2779 if ((s64)seq - (s64)con->in_seq > 1) {
2780 pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1);
2781 con->error_msg = "bad message sequence # for incoming message";
2782 return -EBADE;
2783 }
2784
2785 ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq));
2786
2787 fill_header(&hdr, hdr2, desc->fd_lens[1], desc->fd_lens[2],
2788 desc->fd_lens[3], &con->peer_name);
2789 ret = ceph_con_in_msg_alloc(con, &hdr, &skip);
2790 if (ret)
2791 return ret;
2792
2793 WARN_ON(!con->in_msg ^ skip);
2794 if (skip)
2795 return 0;
2796
2797 WARN_ON(!con->in_msg);
2798 WARN_ON(con->in_msg->con != con);
2799 return 1;
2800}
2801
2802static int process_message(struct ceph_connection *con)
2803{
2804 ceph_con_process_message(con);
2805
2806 /*
2807 * We could have been closed by ceph_con_close() because
2808 * ceph_con_process_message() temporarily drops con->mutex.
2809 */
2810 if (con->state != CEPH_CON_S_OPEN) {
2811 dout("%s con %p state changed to %d\n", __func__, con,
2812 con->state);
2813 return -EAGAIN;
2814 }
2815
2816 prepare_read_preamble(con);
2817 return 0;
2818}
2819
2820static int __handle_control(struct ceph_connection *con, void *p)
2821{
2822 void *end = p + con->v2.in_desc.fd_lens[0];
2823 struct ceph_msg *msg;
2824 int ret;
2825
2826 if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE)
2827 return process_control(con, p, end);
2828
2829 ret = process_message_header(con, p, end);
2830 if (ret < 0)
2831 return ret;
2832 if (ret == 0) {
2833 prepare_skip_message(con);
2834 return 0;
2835 }
2836
2837 msg = con->in_msg; /* set in process_message_header() */
cd1a677c
ID		 2838 if (front_len(msg)) {
2839 WARN_ON(front_len(msg) > msg->front_alloc_len);
cd1a677c 2840 msg->front.iov_len = front_len(msg);
cd1a677c
ID		 2841 } else {
2842 msg->front.iov_len = 0;
2843 }
2844 if (middle_len(msg)) {
2845 WARN_ON(middle_len(msg) > msg->middle->alloc_len);
cd1a677c 2846 msg->middle->vec.iov_len = middle_len(msg);
cd1a677c
ID		 2847 } else if (msg->middle) {
2848 msg->middle->vec.iov_len = 0;
2849 }
2850
2ea88716
ID		 2851 if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
2852 return process_message(con);
2853
2854 if (con_secure(con))
2855 return prepare_read_tail_secure(con);
2856
038b8d1d 2857 return prepare_read_tail_plain(con);
cd1a677c
ID		 2858}
2859
2860static int handle_preamble(struct ceph_connection *con)
2861{
2862 struct ceph_frame_desc *desc = &con->v2.in_desc;
2863 int ret;
2864
2865 if (con_secure(con)) {
2866 ret = decrypt_preamble(con);
2867 if (ret) {
2868 if (ret == -EBADMSG)
2869 con->error_msg = "integrity error, bad preamble auth tag";
2870 return ret;
2871 }
2872 }
2873
2874 ret = decode_preamble(con->v2.in_buf, desc);
2875 if (ret) {
2876 if (ret == -EBADMSG)
2877 con->error_msg = "integrity error, bad crc";
2878 else
2879 con->error_msg = "protocol error, bad preamble";
2880 return ret;
2881 }
2882
2883 dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__,
2884 con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0],
2885 desc->fd_lens[1], desc->fd_lens[2], desc->fd_lens[3]);
2886
2887 if (!con_secure(con))
2888 return prepare_read_control(con);
2889
2890 if (desc->fd_lens[0] > CEPH_PREAMBLE_INLINE_LEN)
2891 return prepare_read_control_remainder(con);
2892
2893 return __handle_control(con, CTRL_BODY(con->v2.in_buf));
2894}
2895
2896static int handle_control(struct ceph_connection *con)
2897{
2898 int ctrl_len = con->v2.in_desc.fd_lens[0];
2899 void *buf;
2900 int ret;
2901
2902 WARN_ON(con_secure(con));
2903
2904 ret = verify_control_crc(con);
2905 if (ret) {
2906 con->error_msg = "integrity error, bad crc";
2907 return ret;
2908 }
2909
2910 if (con->state == CEPH_CON_S_V2_AUTH) {
2911 buf = alloc_conn_buf(con, ctrl_len);
2912 if (!buf)
2913 return -ENOMEM;
2914
2915 memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len);
2916 return __handle_control(con, buf);
2917 }
2918
2919 return __handle_control(con, con->v2.in_kvecs[0].iov_base);
2920}
2921
2922static int handle_control_remainder(struct ceph_connection *con)
2923{
2924 int ret;
2925
2926 WARN_ON(!con_secure(con));
2927
2928 ret = decrypt_control_remainder(con);
2929 if (ret) {
2930 if (ret == -EBADMSG)
2931 con->error_msg = "integrity error, bad control remainder auth tag";
2932 return ret;
2933 }
2934
2935 return __handle_control(con, con->v2.in_kvecs[0].iov_base -
2936 CEPH_PREAMBLE_INLINE_LEN);
2937}
2938
2939static int handle_epilogue(struct ceph_connection *con)
2940{
2941 u32 front_crc, middle_crc, data_crc;
2942 int ret;
2943
2944 if (con_secure(con)) {
2ea88716 2945 ret = decrypt_tail(con);
cd1a677c
ID		 2946 if (ret) {
2947 if (ret == -EBADMSG)
2948 con->error_msg = "integrity error, bad epilogue auth tag";
2949 return ret;
2950 }
2951
2952 /* just late_status */
2953 ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL);
2954 if (ret) {
2955 con->error_msg = "protocol error, bad epilogue";
2956 return ret;
2957 }
2958 } else {
2959 ret = decode_epilogue(con->v2.in_buf, &front_crc,
2960 &middle_crc, &data_crc);
2961 if (ret) {
2962 con->error_msg = "protocol error, bad epilogue";
2963 return ret;
2964 }
2965
2966 ret = verify_epilogue_crcs(con, front_crc, middle_crc,
2967 data_crc);
2968 if (ret) {
2969 con->error_msg = "integrity error, bad crc";
2970 return ret;
2971 }
2972 }
2973
2974 return process_message(con);
2975}
2976
2977static void finish_skip(struct ceph_connection *con)
2978{
2979 dout("%s con %p\n", __func__, con);
2980
2981 if (con_secure(con))
2982 gcm_inc_nonce(&con->v2.in_gcm_nonce);
2983
2984 __finish_skip(con);
2985}
2986
2987static int populate_in_iter(struct ceph_connection *con)
2988{
2989 int ret;
2990
2991 dout("%s con %p state %d in_state %d\n", __func__, con, con->state,
2992 con->v2.in_state);
2993 WARN_ON(iov_iter_count(&con->v2.in_iter));
2994
2995 if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) {
2996 ret = process_banner_prefix(con);
2997 } else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) {
2998 ret = process_banner_payload(con);
2999 } else if ((con->state >= CEPH_CON_S_V2_HELLO &&
3000 con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) ||
3001 con->state == CEPH_CON_S_OPEN) {
3002 switch (con->v2.in_state) {
3003 case IN_S_HANDLE_PREAMBLE:
3004 ret = handle_preamble(con);
3005 break;
3006 case IN_S_HANDLE_CONTROL:
3007 ret = handle_control(con);
3008 break;
3009 case IN_S_HANDLE_CONTROL_REMAINDER:
3010 ret = handle_control_remainder(con);
3011 break;
3012 case IN_S_PREPARE_READ_DATA:
038b8d1d 3013 ret = prepare_read_data(con);
cd1a677c
ID		 3014 break;
3015 case IN_S_PREPARE_READ_DATA_CONT:
3016 prepare_read_data_cont(con);
3017 ret = 0;
3018 break;
2ea88716
ID		 3019 case IN_S_PREPARE_READ_ENC_PAGE:
3020 prepare_read_enc_page(con);
3021 ret = 0;
3022 break;
ec3bc567
JL		 3023 case IN_S_PREPARE_SPARSE_DATA:
3024 ret = prepare_sparse_read_data(con);
3025 break;
3026 case IN_S_PREPARE_SPARSE_DATA_CONT:
3027 ret = prepare_sparse_read_cont(con);
3028 break;
cd1a677c
ID		 3029 case IN_S_HANDLE_EPILOGUE:
3030 ret = handle_epilogue(con);
3031 break;
3032 case IN_S_FINISH_SKIP:
3033 finish_skip(con);
3034 ret = 0;
3035 break;
3036 default:
3037 WARN(1, "bad in_state %d", con->v2.in_state);
3038 return -EINVAL;
3039 }
3040 } else {
3041 WARN(1, "bad state %d", con->state);
3042 return -EINVAL;
3043 }
3044 if (ret) {
3045 dout("%s con %p error %d\n", __func__, con, ret);
3046 return ret;
3047 }
3048
3049 if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
3050 return -ENODATA;
3051 dout("%s con %p populated %zu\n", __func__, con,
3052 iov_iter_count(&con->v2.in_iter));
3053 return 1;
3054}
3055
3056int ceph_con_v2_try_read(struct ceph_connection *con)
3057{
3058 int ret;
3059
3060 dout("%s con %p state %d need %zu\n", __func__, con, con->state,
3061 iov_iter_count(&con->v2.in_iter));
3062
3063 if (con->state == CEPH_CON_S_PREOPEN)
3064 return 0;
3065
3066 /*
3067 * We should always have something pending here. If not,
3068 * avoid calling populate_in_iter() as if we read something
3069 * (ceph_tcp_recv() would immediately return 1).
3070 */
3071 if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
3072 return -ENODATA;
3073
3074 for (;;) {
3075 ret = ceph_tcp_recv(con);
3076 if (ret <= 0)
3077 return ret;
3078
3079 ret = populate_in_iter(con);
3080 if (ret <= 0) {
3081 if (ret && ret != -EAGAIN && !con->error_msg)
3082 con->error_msg = "read processing error";
3083 return ret;
3084 }
3085 }
3086}
3087
3088static void queue_data(struct ceph_connection *con)
3089{
3090 struct bio_vec bv;
3091
3092 con->v2.out_epil.data_crc = -1;
3093 ceph_msg_data_cursor_init(&con->v2.out_cursor, con->out_msg,
3094 data_len(con->out_msg));
3095
3096 get_bvec_at(&con->v2.out_cursor, &bv);
3097 set_out_bvec(con, &bv, true);
3098 con->v2.out_state = OUT_S_QUEUE_DATA_CONT;
3099}
3100
3101static void queue_data_cont(struct ceph_connection *con)
3102{
3103 struct bio_vec bv;
3104
3105 con->v2.out_epil.data_crc = ceph_crc32c_page(
3106 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
3107 con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len);
3108
3109 ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len);
3110 if (con->v2.out_cursor.total_resid) {
3111 get_bvec_at(&con->v2.out_cursor, &bv);
3112 set_out_bvec(con, &bv, true);
3113 WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT);
3114 return;
3115 }
3116
3117 /*
3118 * We've written all data. Queue epilogue. Once it's written,
3119 * we are done.
3120 */
3121 reset_out_kvecs(con);
3122 prepare_epilogue_plain(con, false);
3123 con->v2.out_state = OUT_S_FINISH_MESSAGE;
3124}
3125
3126static void queue_enc_page(struct ceph_connection *con)
3127{
3128 struct bio_vec bv;
3129
3130 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i,
3131 con->v2.out_enc_resid);
3132 WARN_ON(!con->v2.out_enc_resid);
3133
1eb9cd15
CH		 3134 bvec_set_page(&bv, con->v2.out_enc_pages[con->v2.out_enc_i],
3135 min(con->v2.out_enc_resid, (int)PAGE_SIZE), 0);
cd1a677c
ID		 3136
3137 set_out_bvec(con, &bv, false);
3138 con->v2.out_enc_i++;
3139 con->v2.out_enc_resid -= bv.bv_len;
3140
3141 if (con->v2.out_enc_resid) {
3142 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE);
3143 return;
3144 }
3145
3146 /*
3147 * We've queued the last piece of ciphertext (ending with
3148 * epilogue) + auth tag. Once it's written, we are done.
3149 */
3150 WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt);
3151 con->v2.out_state = OUT_S_FINISH_MESSAGE;
3152}
3153
3154static void queue_zeros(struct ceph_connection *con)
3155{
3156 dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero);
3157
3158 if (con->v2.out_zero) {
3159 set_out_bvec_zero(con);
3160 con->v2.out_zero -= con->v2.out_bvec.bv_len;
3161 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3162 return;
3163 }
3164
3165 /*
3166 * We've zero-filled everything up to epilogue. Queue epilogue
3167 * with late_status set to ABORTED and crcs adjusted for zeros.
3168 * Once it's written, we are done patching up for the revoke.
3169 */
3170 reset_out_kvecs(con);
3171 prepare_epilogue_plain(con, true);
3172 con->v2.out_state = OUT_S_FINISH_MESSAGE;
3173}
3174
3175static void finish_message(struct ceph_connection *con)
3176{
3177 dout("%s con %p msg %p\n", __func__, con, con->out_msg);
3178
3179 /* we end up here both plain and secure modes */
3180 if (con->v2.out_enc_pages) {
3181 WARN_ON(!con->v2.out_enc_page_cnt);
3182 ceph_release_page_vector(con->v2.out_enc_pages,
3183 con->v2.out_enc_page_cnt);
3184 con->v2.out_enc_pages = NULL;
3185 con->v2.out_enc_page_cnt = 0;
3186 }
3187 /* message may have been revoked */
3188 if (con->out_msg) {
3189 ceph_msg_put(con->out_msg);
3190 con->out_msg = NULL;
3191 }
3192
3193 con->v2.out_state = OUT_S_GET_NEXT;
3194}
3195
3196static int populate_out_iter(struct ceph_connection *con)
3197{
3198 int ret;
3199
3200 dout("%s con %p state %d out_state %d\n", __func__, con, con->state,
3201 con->v2.out_state);
3202 WARN_ON(iov_iter_count(&con->v2.out_iter));
3203
3204 if (con->state != CEPH_CON_S_OPEN) {
3205 WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX ||
3206 con->state > CEPH_CON_S_V2_SESSION_RECONNECT);
3207 goto nothing_pending;
3208 }
3209
3210 switch (con->v2.out_state) {
3211 case OUT_S_QUEUE_DATA:
3212 WARN_ON(!con->out_msg);
3213 queue_data(con);
3214 goto populated;
3215 case OUT_S_QUEUE_DATA_CONT:
3216 WARN_ON(!con->out_msg);
3217 queue_data_cont(con);
3218 goto populated;
3219 case OUT_S_QUEUE_ENC_PAGE:
3220 queue_enc_page(con);
3221 goto populated;
3222 case OUT_S_QUEUE_ZEROS:
3223 WARN_ON(con->out_msg); /* revoked */
3224 queue_zeros(con);
3225 goto populated;
3226 case OUT_S_FINISH_MESSAGE:
3227 finish_message(con);
3228 break;
3229 case OUT_S_GET_NEXT:
3230 break;
3231 default:
3232 WARN(1, "bad out_state %d", con->v2.out_state);
3233 return -EINVAL;
3234 }
3235
3236 WARN_ON(con->v2.out_state != OUT_S_GET_NEXT);
3237 if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) {
3238 ret = prepare_keepalive2(con);
3239 if (ret) {
3240 pr_err("prepare_keepalive2 failed: %d\n", ret);
3241 return ret;
3242 }
3243 } else if (!list_empty(&con->out_queue)) {
3244 ceph_con_get_out_msg(con);
3245 ret = prepare_message(con);
3246 if (ret) {
3247 pr_err("prepare_message failed: %d\n", ret);
3248 return ret;
3249 }
3250 } else if (con->in_seq > con->in_seq_acked) {
3251 ret = prepare_ack(con);
3252 if (ret) {
3253 pr_err("prepare_ack failed: %d\n", ret);
3254 return ret;
3255 }
3256 } else {
3257 goto nothing_pending;
3258 }
3259
3260populated:
3261 if (WARN_ON(!iov_iter_count(&con->v2.out_iter)))
3262 return -ENODATA;
3263 dout("%s con %p populated %zu\n", __func__, con,
3264 iov_iter_count(&con->v2.out_iter));
3265 return 1;
3266
3267nothing_pending:
3268 WARN_ON(iov_iter_count(&con->v2.out_iter));
3269 dout("%s con %p nothing pending\n", __func__, con);
3270 ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING);
3271 return 0;
3272}
3273
3274int ceph_con_v2_try_write(struct ceph_connection *con)
3275{
3276 int ret;
3277
3278 dout("%s con %p state %d have %zu\n", __func__, con, con->state,
3279 iov_iter_count(&con->v2.out_iter));
3280
3281 /* open the socket first? */
3282 if (con->state == CEPH_CON_S_PREOPEN) {
3283 WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2);
3284
3285 /*
3286 * Always bump global_seq. Bump connect_seq only if
3287 * there is a session (i.e. we are reconnecting and will
3288 * send session_reconnect instead of client_ident).
3289 */
3290 con->v2.global_seq = ceph_get_global_seq(con->msgr, 0);
3291 if (con->v2.server_cookie)
3292 con->v2.connect_seq++;
3293
3294 ret = prepare_read_banner_prefix(con);
3295 if (ret) {
3296 pr_err("prepare_read_banner_prefix failed: %d\n", ret);
3297 con->error_msg = "connect error";
3298 return ret;
3299 }
3300
3301 reset_out_kvecs(con);
3302 ret = prepare_banner(con);
3303 if (ret) {
3304 pr_err("prepare_banner failed: %d\n", ret);
3305 con->error_msg = "connect error";
3306 return ret;
3307 }
3308
3309 ret = ceph_tcp_connect(con);
3310 if (ret) {
3311 pr_err("ceph_tcp_connect failed: %d\n", ret);
3312 con->error_msg = "connect error";
3313 return ret;
3314 }
3315 }
3316
3317 if (!iov_iter_count(&con->v2.out_iter)) {
3318 ret = populate_out_iter(con);
3319 if (ret <= 0) {
3320 if (ret && ret != -EAGAIN && !con->error_msg)
3321 con->error_msg = "write processing error";
3322 return ret;
3323 }
3324 }
3325
3326 tcp_sock_set_cork(con->sock->sk, true);
3327 for (;;) {
3328 ret = ceph_tcp_send(con);
3329 if (ret <= 0)
3330 break;
3331
3332 ret = populate_out_iter(con);
3333 if (ret <= 0) {
3334 if (ret && ret != -EAGAIN && !con->error_msg)
3335 con->error_msg = "write processing error";
3336 break;
3337 }
3338 }
3339
3340 tcp_sock_set_cork(con->sock->sk, false);
3341 return ret;
3342}
3343
3344static u32 crc32c_zeros(u32 crc, int zero_len)
3345{
3346 int len;
3347
3348 while (zero_len) {
3349 len = min(zero_len, (int)PAGE_SIZE);
3350 crc = crc32c(crc, page_address(ceph_zero_page), len);
3351 zero_len -= len;
3352 }
3353
3354 return crc;
3355}
3356
3357static void prepare_zero_front(struct ceph_connection *con, int resid)
3358{
3359 int sent;
3360
3361 WARN_ON(!resid || resid > front_len(con->out_msg));
3362 sent = front_len(con->out_msg) - resid;
3363 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3364
3365 if (sent) {
3366 con->v2.out_epil.front_crc =
3367 crc32c(-1, con->out_msg->front.iov_base, sent);
3368 con->v2.out_epil.front_crc =
3369 crc32c_zeros(con->v2.out_epil.front_crc, resid);
3370 } else {
3371 con->v2.out_epil.front_crc = crc32c_zeros(-1, resid);
3372 }
3373
3374 con->v2.out_iter.count -= resid;
3375 out_zero_add(con, resid);
3376}
3377
3378static void prepare_zero_middle(struct ceph_connection *con, int resid)
3379{
3380 int sent;
3381
3382 WARN_ON(!resid || resid > middle_len(con->out_msg));
3383 sent = middle_len(con->out_msg) - resid;
3384 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3385
3386 if (sent) {
3387 con->v2.out_epil.middle_crc =
3388 crc32c(-1, con->out_msg->middle->vec.iov_base, sent);
3389 con->v2.out_epil.middle_crc =
3390 crc32c_zeros(con->v2.out_epil.middle_crc, resid);
3391 } else {
3392 con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid);
3393 }
3394
3395 con->v2.out_iter.count -= resid;
3396 out_zero_add(con, resid);
3397}
3398
3399static void prepare_zero_data(struct ceph_connection *con)
3400{
3401 dout("%s con %p\n", __func__, con);
3402 con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(con->out_msg));
3403 out_zero_add(con, data_len(con->out_msg));
3404}
3405
3406static void revoke_at_queue_data(struct ceph_connection *con)
3407{
3408 int boundary;
3409 int resid;
3410
3411 WARN_ON(!data_len(con->out_msg));
3412 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3413 resid = iov_iter_count(&con->v2.out_iter);
3414
3415 boundary = front_len(con->out_msg) + middle_len(con->out_msg);
3416 if (resid > boundary) {
3417 resid -= boundary;
3418 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3419 dout("%s con %p was sending head\n", __func__, con);
3420 if (front_len(con->out_msg))
3421 prepare_zero_front(con, front_len(con->out_msg));
3422 if (middle_len(con->out_msg))
3423 prepare_zero_middle(con, middle_len(con->out_msg));
3424 prepare_zero_data(con);
3425 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3426 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3427 return;
3428 }
3429
3430 boundary = middle_len(con->out_msg);
3431 if (resid > boundary) {
3432 resid -= boundary;
3433 dout("%s con %p was sending front\n", __func__, con);
3434 prepare_zero_front(con, resid);
3435 if (middle_len(con->out_msg))
3436 prepare_zero_middle(con, middle_len(con->out_msg));
3437 prepare_zero_data(con);
3438 queue_zeros(con);
3439 return;
3440 }
3441
3442 WARN_ON(!resid);
3443 dout("%s con %p was sending middle\n", __func__, con);
3444 prepare_zero_middle(con, resid);
3445 prepare_zero_data(con);
3446 queue_zeros(con);
3447}
3448
3449static void revoke_at_queue_data_cont(struct ceph_connection *con)
3450{
3451 int sent, resid; /* current piece of data */
3452
3453 WARN_ON(!data_len(con->out_msg));
3454 WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter));
3455 resid = iov_iter_count(&con->v2.out_iter);
3456 WARN_ON(!resid || resid > con->v2.out_bvec.bv_len);
3457 sent = con->v2.out_bvec.bv_len - resid;
3458 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3459
3460 if (sent) {
3461 con->v2.out_epil.data_crc = ceph_crc32c_page(
3462 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
3463 con->v2.out_bvec.bv_offset, sent);
3464 ceph_msg_data_advance(&con->v2.out_cursor, sent);
3465 }
3466 WARN_ON(resid > con->v2.out_cursor.total_resid);
3467 con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc,
3468 con->v2.out_cursor.total_resid);
3469
3470 con->v2.out_iter.count -= resid;
3471 out_zero_add(con, con->v2.out_cursor.total_resid);
3472 queue_zeros(con);
3473}
3474
3475static void revoke_at_finish_message(struct ceph_connection *con)
3476{
3477 int boundary;
3478 int resid;
3479
3480 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3481 resid = iov_iter_count(&con->v2.out_iter);
3482
3483 if (!front_len(con->out_msg) && !middle_len(con->out_msg) &&
3484 !data_len(con->out_msg)) {
3485 WARN_ON(!resid || resid > MESSAGE_HEAD_PLAIN_LEN);
3486 dout("%s con %p was sending head (empty message) - noop\n",
3487 __func__, con);
3488 return;
3489 }
3490
3491 boundary = front_len(con->out_msg) + middle_len(con->out_msg) +
3492 CEPH_EPILOGUE_PLAIN_LEN;
3493 if (resid > boundary) {
3494 resid -= boundary;
3495 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3496 dout("%s con %p was sending head\n", __func__, con);
3497 if (front_len(con->out_msg))
3498 prepare_zero_front(con, front_len(con->out_msg));
3499 if (middle_len(con->out_msg))
3500 prepare_zero_middle(con, middle_len(con->out_msg));
3501 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3502 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3503 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3504 return;
3505 }
3506
3507 boundary = middle_len(con->out_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3508 if (resid > boundary) {
3509 resid -= boundary;
3510 dout("%s con %p was sending front\n", __func__, con);
3511 prepare_zero_front(con, resid);
3512 if (middle_len(con->out_msg))
3513 prepare_zero_middle(con, middle_len(con->out_msg));
3514 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3515 queue_zeros(con);
3516 return;
3517 }
3518
3519 boundary = CEPH_EPILOGUE_PLAIN_LEN;
3520 if (resid > boundary) {
3521 resid -= boundary;
3522 dout("%s con %p was sending middle\n", __func__, con);
3523 prepare_zero_middle(con, resid);
3524 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3525 queue_zeros(con);
3526 return;
3527 }
3528
3529 WARN_ON(!resid);
3530 dout("%s con %p was sending epilogue - noop\n", __func__, con);
3531}
3532
3533void ceph_con_v2_revoke(struct ceph_connection *con)
3534{
3535 WARN_ON(con->v2.out_zero);
3536
3537 if (con_secure(con)) {
3538 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE &&
3539 con->v2.out_state != OUT_S_FINISH_MESSAGE);
3540 dout("%s con %p secure - noop\n", __func__, con);
3541 return;
3542 }
3543
3544 switch (con->v2.out_state) {
3545 case OUT_S_QUEUE_DATA:
3546 revoke_at_queue_data(con);
3547 break;
3548 case OUT_S_QUEUE_DATA_CONT:
3549 revoke_at_queue_data_cont(con);
3550 break;
3551 case OUT_S_FINISH_MESSAGE:
3552 revoke_at_finish_message(con);
3553 break;
3554 default:
3555 WARN(1, "bad out_state %d", con->v2.out_state);
3556 break;
3557 }
3558}
3559
3560static void revoke_at_prepare_read_data(struct ceph_connection *con)
3561{
2ea88716 3562 int remaining;
cd1a677c
ID		 3563 int resid;
3564
2ea88716 3565 WARN_ON(con_secure(con));
cd1a677c
ID		 3566 WARN_ON(!data_len(con->in_msg));
3567 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
3568 resid = iov_iter_count(&con->v2.in_iter);
3569 WARN_ON(!resid);
3570
2ea88716 3571 remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN;
cd1a677c
ID		 3572 dout("%s con %p resid %d remaining %d\n", __func__, con, resid,
3573 remaining);
3574 con->v2.in_iter.count -= resid;
3575 set_in_skip(con, resid + remaining);
3576 con->v2.in_state = IN_S_FINISH_SKIP;
3577}
3578
3579static void revoke_at_prepare_read_data_cont(struct ceph_connection *con)
3580{
3581 int recved, resid; /* current piece of data */
2ea88716 3582 int remaining;
cd1a677c 3583
2ea88716 3584 WARN_ON(con_secure(con));
cd1a677c
ID		 3585 WARN_ON(!data_len(con->in_msg));
3586 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3587 resid = iov_iter_count(&con->v2.in_iter);
3588 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3589 recved = con->v2.in_bvec.bv_len - resid;
3590 dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid);
3591
3592 if (recved)
3593 ceph_msg_data_advance(&con->v2.in_cursor, recved);
3594 WARN_ON(resid > con->v2.in_cursor.total_resid);
3595
2ea88716 3596 remaining = CEPH_EPILOGUE_PLAIN_LEN;
cd1a677c
ID		 3597 dout("%s con %p total_resid %zu remaining %d\n", __func__, con,
3598 con->v2.in_cursor.total_resid, remaining);
3599 con->v2.in_iter.count -= resid;
3600 set_in_skip(con, con->v2.in_cursor.total_resid + remaining);
3601 con->v2.in_state = IN_S_FINISH_SKIP;
3602}
3603
2ea88716
ID		 3604static void revoke_at_prepare_read_enc_page(struct ceph_connection *con)
3605{
3606 int resid; /* current enc page (not necessarily data) */
3607
3608 WARN_ON(!con_secure(con));
3609 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3610 resid = iov_iter_count(&con->v2.in_iter);
3611 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3612
3613 dout("%s con %p resid %d enc_resid %d\n", __func__, con, resid,
3614 con->v2.in_enc_resid);
3615 con->v2.in_iter.count -= resid;
3616 set_in_skip(con, resid + con->v2.in_enc_resid);
3617 con->v2.in_state = IN_S_FINISH_SKIP;
3618}
3619
ec3bc567
JL		 3620static void revoke_at_prepare_sparse_data(struct ceph_connection *con)
3621{
3622 int resid; /* current piece of data */
3623 int remaining;
3624
3625 WARN_ON(con_secure(con));
3626 WARN_ON(!data_len(con->in_msg));
3627 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3628 resid = iov_iter_count(&con->v2.in_iter);
3629 dout("%s con %p resid %d\n", __func__, con, resid);
3630
3631 remaining = CEPH_EPILOGUE_PLAIN_LEN + con->v2.data_len_remain;
3632 con->v2.in_iter.count -= resid;
3633 set_in_skip(con, resid + remaining);
3634 con->v2.in_state = IN_S_FINISH_SKIP;
3635}
3636
cd1a677c
ID		 3637static void revoke_at_handle_epilogue(struct ceph_connection *con)
3638{
3639 int resid;
3640
cd1a677c
ID		 3641 resid = iov_iter_count(&con->v2.in_iter);
3642 WARN_ON(!resid);
3643
3644 dout("%s con %p resid %d\n", __func__, con, resid);
3645 con->v2.in_iter.count -= resid;
3646 set_in_skip(con, resid);
3647 con->v2.in_state = IN_S_FINISH_SKIP;
3648}
3649
3650void ceph_con_v2_revoke_incoming(struct ceph_connection *con)
3651{
3652 switch (con->v2.in_state) {
ec3bc567 3653 case IN_S_PREPARE_SPARSE_DATA:
cd1a677c
ID		 3654 case IN_S_PREPARE_READ_DATA:
3655 revoke_at_prepare_read_data(con);
3656 break;
3657 case IN_S_PREPARE_READ_DATA_CONT:
3658 revoke_at_prepare_read_data_cont(con);
3659 break;
2ea88716
ID		 3660 case IN_S_PREPARE_READ_ENC_PAGE:
3661 revoke_at_prepare_read_enc_page(con);
3662 break;
ec3bc567
JL		 3663 case IN_S_PREPARE_SPARSE_DATA_CONT:
3664 revoke_at_prepare_sparse_data(con);
3665 break;
cd1a677c
ID		 3666 case IN_S_HANDLE_EPILOGUE:
3667 revoke_at_handle_epilogue(con);
3668 break;
3669 default:
3670 WARN(1, "bad in_state %d", con->v2.in_state);
3671 break;
3672 }
3673}
3674
3675bool ceph_con_v2_opened(struct ceph_connection *con)
3676{
3677 return con->v2.peer_global_seq;
3678}
3679
3680void ceph_con_v2_reset_session(struct ceph_connection *con)
3681{
3682 con->v2.client_cookie = 0;
3683 con->v2.server_cookie = 0;
3684 con->v2.global_seq = 0;
3685 con->v2.connect_seq = 0;
3686 con->v2.peer_global_seq = 0;
3687}
3688
3689void ceph_con_v2_reset_protocol(struct ceph_connection *con)
3690{
3691 iov_iter_truncate(&con->v2.in_iter, 0);
3692 iov_iter_truncate(&con->v2.out_iter, 0);
3693 con->v2.out_zero = 0;
3694
3695 clear_in_sign_kvecs(con);
3696 clear_out_sign_kvecs(con);
3697 free_conn_bufs(con);
3698
2ea88716
ID		 3699 if (con->v2.in_enc_pages) {
3700 WARN_ON(!con->v2.in_enc_page_cnt);
3701 ceph_release_page_vector(con->v2.in_enc_pages,
3702 con->v2.in_enc_page_cnt);
3703 con->v2.in_enc_pages = NULL;
3704 con->v2.in_enc_page_cnt = 0;
3705 }
cd1a677c
ID		 3706 if (con->v2.out_enc_pages) {
3707 WARN_ON(!con->v2.out_enc_page_cnt);
3708 ceph_release_page_vector(con->v2.out_enc_pages,
3709 con->v2.out_enc_page_cnt);
3710 con->v2.out_enc_pages = NULL;
3711 con->v2.out_enc_page_cnt = 0;
3712 }
3713
3714 con->v2.con_mode = CEPH_CON_MODE_UNKNOWN;
10f42b3e
ID		 3715 memzero_explicit(&con->v2.in_gcm_nonce, CEPH_GCM_IV_LEN);
3716 memzero_explicit(&con->v2.out_gcm_nonce, CEPH_GCM_IV_LEN);
cd1a677c
ID		 3717
3718 if (con->v2.hmac_tfm) {
3719 crypto_free_shash(con->v2.hmac_tfm);
3720 con->v2.hmac_tfm = NULL;
3721 }
3722 if (con->v2.gcm_req) {
3723 aead_request_free(con->v2.gcm_req);
3724 con->v2.gcm_req = NULL;
3725 }
3726 if (con->v2.gcm_tfm) {
3727 crypto_free_aead(con->v2.gcm_tfm);
3728 con->v2.gcm_tfm = NULL;
3729 }
3730}
Linux 6.x block layer and io_uring tree(s)