Commit | Line | Data |
---|---|---|
d7e09d03 PT |
1 | /* |
2 | * GPL HEADER START | |
3 | * | |
4 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 only, | |
8 | * as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | * General Public License version 2 for more details (a copy is included | |
14 | * in the LICENSE file that accompanied this code). | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * version 2 along with this program; If not, see | |
18 | * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf | |
19 | * | |
20 | * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, | |
21 | * CA 95054 USA or visit www.sun.com if you need additional information or | |
22 | * have any questions. | |
23 | * | |
24 | * GPL HEADER END | |
25 | */ | |
26 | /* | |
27 | * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. | |
28 | * Use is subject to license terms. | |
29 | * | |
1dc563a6 | 30 | * Copyright (c) 2012, 2015, Intel Corporation. |
d7e09d03 PT |
31 | */ |
32 | /* | |
33 | * This file is part of Lustre, http://www.lustre.org/ | |
34 | * Lustre is a trademark of Sun Microsystems, Inc. | |
35 | * | |
36 | * lnet/selftest/rpc.c | |
37 | * | |
38 | * Author: Isaac Huang <isaac@clusterfs.com> | |
39 | * | |
40 | * 2012-05-13: Liang Zhen <liang@whamcloud.com> | |
41 | * - percpt data for service to improve smp performance | |
42 | * - code cleanup | |
43 | */ | |
44 | ||
45 | #define DEBUG_SUBSYSTEM S_LNET | |
46 | ||
47 | #include "selftest.h" | |
48 | ||
49 | typedef enum { | |
50 | SRPC_STATE_NONE, | |
51 | SRPC_STATE_NI_INIT, | |
52 | SRPC_STATE_EQ_INIT, | |
53 | SRPC_STATE_RUNNING, | |
54 | SRPC_STATE_STOPPING, | |
55 | } srpc_state_t; | |
56 | ||
62366bf1 | 57 | static struct smoketest_rpc { |
d7e09d03 PT |
58 | spinlock_t rpc_glock; /* global lock */ |
59 | srpc_service_t *rpc_services[SRPC_SERVICE_MAX_ID + 1]; | |
60 | lnet_handle_eq_t rpc_lnet_eq; /* _the_ LNet event queue */ | |
61 | srpc_state_t rpc_state; | |
62 | srpc_counters_t rpc_counters; | |
63 | __u64 rpc_matchbits; /* matchbits counter */ | |
64 | } srpc_data; | |
65 | ||
66 | static inline int | |
67 | srpc_serv_portal(int svc_id) | |
68 | { | |
69 | return svc_id < SRPC_FRAMEWORK_SERVICE_MAX_ID ? | |
70 | SRPC_FRAMEWORK_REQUEST_PORTAL : SRPC_REQUEST_PORTAL; | |
71 | } | |
72 | ||
73 | /* forward ref's */ | |
a659df62 | 74 | int srpc_handle_rpc(swi_workitem_t *wi); |
d7e09d03 | 75 | |
a659df62 | 76 | void srpc_get_counters(srpc_counters_t *cnt) |
d7e09d03 PT |
77 | { |
78 | spin_lock(&srpc_data.rpc_glock); | |
79 | *cnt = srpc_data.rpc_counters; | |
80 | spin_unlock(&srpc_data.rpc_glock); | |
81 | } | |
82 | ||
a659df62 | 83 | void srpc_set_counters(const srpc_counters_t *cnt) |
d7e09d03 PT |
84 | { |
85 | spin_lock(&srpc_data.rpc_glock); | |
86 | srpc_data.rpc_counters = *cnt; | |
87 | spin_unlock(&srpc_data.rpc_glock); | |
88 | } | |
89 | ||
8d94b6d2 | 90 | static int |
d7e09d03 PT |
91 | srpc_add_bulk_page(srpc_bulk_t *bk, struct page *pg, int i, int nob) |
92 | { | |
09cbfeaf | 93 | nob = min_t(int, nob, PAGE_SIZE); |
d7e09d03 PT |
94 | |
95 | LASSERT(nob > 0); | |
96 | LASSERT(i >= 0 && i < bk->bk_niov); | |
97 | ||
98 | bk->bk_iovs[i].kiov_offset = 0; | |
ec436b9a JS |
99 | bk->bk_iovs[i].kiov_page = pg; |
100 | bk->bk_iovs[i].kiov_len = nob; | |
d7e09d03 PT |
101 | return nob; |
102 | } | |
103 | ||
104 | void | |
a659df62 | 105 | srpc_free_bulk(srpc_bulk_t *bk) |
d7e09d03 | 106 | { |
74d68011 | 107 | int i; |
d7e09d03 PT |
108 | struct page *pg; |
109 | ||
06ace26e | 110 | LASSERT(bk); |
d7e09d03 PT |
111 | |
112 | for (i = 0; i < bk->bk_niov; i++) { | |
113 | pg = bk->bk_iovs[i].kiov_page; | |
06ace26e | 114 | if (!pg) |
f64a6f3c | 115 | break; |
d7e09d03 PT |
116 | |
117 | __free_page(pg); | |
118 | } | |
119 | ||
120 | LIBCFS_FREE(bk, offsetof(srpc_bulk_t, bk_iovs[bk->bk_niov])); | |
d7e09d03 PT |
121 | } |
122 | ||
123 | srpc_bulk_t * | |
124 | srpc_alloc_bulk(int cpt, unsigned bulk_npg, unsigned bulk_len, int sink) | |
125 | { | |
74d68011 MS |
126 | srpc_bulk_t *bk; |
127 | int i; | |
d7e09d03 PT |
128 | |
129 | LASSERT(bulk_npg > 0 && bulk_npg <= LNET_MAX_IOV); | |
130 | ||
131 | LIBCFS_CPT_ALLOC(bk, lnet_cpt_table(), cpt, | |
132 | offsetof(srpc_bulk_t, bk_iovs[bulk_npg])); | |
06ace26e | 133 | if (!bk) { |
d7e09d03 PT |
134 | CERROR("Can't allocate descriptor for %d pages\n", bulk_npg); |
135 | return NULL; | |
136 | } | |
137 | ||
138 | memset(bk, 0, offsetof(srpc_bulk_t, bk_iovs[bulk_npg])); | |
ec436b9a JS |
139 | bk->bk_sink = sink; |
140 | bk->bk_len = bulk_len; | |
141 | bk->bk_niov = bulk_npg; | |
d7e09d03 PT |
142 | |
143 | for (i = 0; i < bulk_npg; i++) { | |
144 | struct page *pg; | |
74d68011 | 145 | int nob; |
d7e09d03 | 146 | |
49c02a75 | 147 | pg = alloc_pages_node(cfs_cpt_spread_node(lnet_cpt_table(), cpt), |
40113370 | 148 | GFP_KERNEL, 0); |
06ace26e | 149 | if (!pg) { |
d7e09d03 PT |
150 | CERROR("Can't allocate page %d of %d\n", i, bulk_npg); |
151 | srpc_free_bulk(bk); | |
152 | return NULL; | |
153 | } | |
154 | ||
155 | nob = srpc_add_bulk_page(bk, pg, i, bulk_len); | |
156 | bulk_len -= nob; | |
157 | } | |
158 | ||
159 | return bk; | |
160 | } | |
161 | ||
162 | static inline __u64 | |
a659df62 | 163 | srpc_next_id(void) |
d7e09d03 PT |
164 | { |
165 | __u64 id; | |
166 | ||
167 | spin_lock(&srpc_data.rpc_glock); | |
168 | id = srpc_data.rpc_matchbits++; | |
169 | spin_unlock(&srpc_data.rpc_glock); | |
170 | return id; | |
171 | } | |
172 | ||
8d94b6d2 | 173 | static void |
d7e09d03 PT |
174 | srpc_init_server_rpc(struct srpc_server_rpc *rpc, |
175 | struct srpc_service_cd *scd, | |
176 | struct srpc_buffer *buffer) | |
177 | { | |
178 | memset(rpc, 0, sizeof(*rpc)); | |
179 | swi_init_workitem(&rpc->srpc_wi, rpc, srpc_handle_rpc, | |
180 | srpc_serv_is_framework(scd->scd_svc) ? | |
181 | lst_sched_serial : lst_sched_test[scd->scd_cpt]); | |
182 | ||
183 | rpc->srpc_ev.ev_fired = 1; /* no event expected now */ | |
184 | ||
ec436b9a | 185 | rpc->srpc_scd = scd; |
d7e09d03 | 186 | rpc->srpc_reqstbuf = buffer; |
ec436b9a JS |
187 | rpc->srpc_peer = buffer->buf_peer; |
188 | rpc->srpc_self = buffer->buf_self; | |
d7e09d03 PT |
189 | LNetInvalidateHandle(&rpc->srpc_replymdh); |
190 | } | |
191 | ||
192 | static void | |
193 | srpc_service_fini(struct srpc_service *svc) | |
194 | { | |
74d68011 MS |
195 | struct srpc_service_cd *scd; |
196 | struct srpc_server_rpc *rpc; | |
197 | struct srpc_buffer *buf; | |
198 | struct list_head *q; | |
199 | int i; | |
d7e09d03 | 200 | |
06ace26e | 201 | if (!svc->sv_cpt_data) |
d7e09d03 PT |
202 | return; |
203 | ||
204 | cfs_percpt_for_each(scd, i, svc->sv_cpt_data) { | |
205 | while (1) { | |
206 | if (!list_empty(&scd->scd_buf_posted)) | |
207 | q = &scd->scd_buf_posted; | |
208 | else if (!list_empty(&scd->scd_buf_blocked)) | |
209 | q = &scd->scd_buf_blocked; | |
210 | else | |
211 | break; | |
212 | ||
213 | while (!list_empty(q)) { | |
c314c319 JS |
214 | buf = list_entry(q->next, struct srpc_buffer, |
215 | buf_list); | |
d7e09d03 PT |
216 | list_del(&buf->buf_list); |
217 | LIBCFS_FREE(buf, sizeof(*buf)); | |
218 | } | |
219 | } | |
220 | ||
221 | LASSERT(list_empty(&scd->scd_rpc_active)); | |
222 | ||
223 | while (!list_empty(&scd->scd_rpc_free)) { | |
224 | rpc = list_entry(scd->scd_rpc_free.next, | |
c314c319 JS |
225 | struct srpc_server_rpc, |
226 | srpc_list); | |
d7e09d03 PT |
227 | list_del(&rpc->srpc_list); |
228 | LIBCFS_FREE(rpc, sizeof(*rpc)); | |
229 | } | |
230 | } | |
231 | ||
232 | cfs_percpt_free(svc->sv_cpt_data); | |
233 | svc->sv_cpt_data = NULL; | |
234 | } | |
235 | ||
236 | static int | |
237 | srpc_service_nrpcs(struct srpc_service *svc) | |
238 | { | |
239 | int nrpcs = svc->sv_wi_total / svc->sv_ncpts; | |
240 | ||
241 | return srpc_serv_is_framework(svc) ? | |
242 | max(nrpcs, SFW_FRWK_WI_MIN) : max(nrpcs, SFW_TEST_WI_MIN); | |
243 | } | |
244 | ||
245 | int srpc_add_buffer(struct swi_workitem *wi); | |
246 | ||
247 | static int | |
248 | srpc_service_init(struct srpc_service *svc) | |
249 | { | |
74d68011 MS |
250 | struct srpc_service_cd *scd; |
251 | struct srpc_server_rpc *rpc; | |
252 | int nrpcs; | |
253 | int i; | |
254 | int j; | |
d7e09d03 PT |
255 | |
256 | svc->sv_shuttingdown = 0; | |
257 | ||
258 | svc->sv_cpt_data = cfs_percpt_alloc(lnet_cpt_table(), | |
259 | sizeof(struct srpc_service_cd)); | |
06ace26e | 260 | if (!svc->sv_cpt_data) |
d7e09d03 PT |
261 | return -ENOMEM; |
262 | ||
263 | svc->sv_ncpts = srpc_serv_is_framework(svc) ? | |
264 | 1 : cfs_cpt_number(lnet_cpt_table()); | |
265 | nrpcs = srpc_service_nrpcs(svc); | |
266 | ||
267 | cfs_percpt_for_each(scd, i, svc->sv_cpt_data) { | |
268 | scd->scd_cpt = i; | |
269 | scd->scd_svc = svc; | |
270 | spin_lock_init(&scd->scd_lock); | |
271 | INIT_LIST_HEAD(&scd->scd_rpc_free); | |
272 | INIT_LIST_HEAD(&scd->scd_rpc_active); | |
273 | INIT_LIST_HEAD(&scd->scd_buf_posted); | |
274 | INIT_LIST_HEAD(&scd->scd_buf_blocked); | |
275 | ||
276 | scd->scd_ev.ev_data = scd; | |
277 | scd->scd_ev.ev_type = SRPC_REQUEST_RCVD; | |
278 | ||
4420cfd3 JS |
279 | /* |
280 | * NB: don't use lst_sched_serial for adding buffer, | |
281 | * see details in srpc_service_add_buffers() | |
282 | */ | |
d7e09d03 PT |
283 | swi_init_workitem(&scd->scd_buf_wi, scd, |
284 | srpc_add_buffer, lst_sched_test[i]); | |
285 | ||
5fd88337 | 286 | if (i && srpc_serv_is_framework(svc)) { |
4420cfd3 JS |
287 | /* |
288 | * NB: framework service only needs srpc_service_cd for | |
d7e09d03 PT |
289 | * one partition, but we allocate for all to make |
290 | * it easier to implement, it will waste a little | |
4420cfd3 JS |
291 | * memory but nobody should care about this |
292 | */ | |
d7e09d03 PT |
293 | continue; |
294 | } | |
295 | ||
296 | for (j = 0; j < nrpcs; j++) { | |
297 | LIBCFS_CPT_ALLOC(rpc, lnet_cpt_table(), | |
298 | i, sizeof(*rpc)); | |
06ace26e | 299 | if (!rpc) { |
d7e09d03 PT |
300 | srpc_service_fini(svc); |
301 | return -ENOMEM; | |
302 | } | |
303 | list_add(&rpc->srpc_list, &scd->scd_rpc_free); | |
304 | } | |
305 | } | |
306 | ||
307 | return 0; | |
308 | } | |
309 | ||
310 | int | |
311 | srpc_add_service(struct srpc_service *sv) | |
312 | { | |
313 | int id = sv->sv_id; | |
314 | ||
315 | LASSERT(0 <= id && id <= SRPC_SERVICE_MAX_ID); | |
316 | ||
5fd88337 | 317 | if (srpc_service_init(sv)) |
d7e09d03 PT |
318 | return -ENOMEM; |
319 | ||
320 | spin_lock(&srpc_data.rpc_glock); | |
321 | ||
322 | LASSERT(srpc_data.rpc_state == SRPC_STATE_RUNNING); | |
323 | ||
06ace26e | 324 | if (srpc_data.rpc_services[id]) { |
d7e09d03 PT |
325 | spin_unlock(&srpc_data.rpc_glock); |
326 | goto failed; | |
327 | } | |
328 | ||
329 | srpc_data.rpc_services[id] = sv; | |
330 | spin_unlock(&srpc_data.rpc_glock); | |
331 | ||
332 | CDEBUG(D_NET, "Adding service: id %d, name %s\n", id, sv->sv_name); | |
333 | return 0; | |
334 | ||
335 | failed: | |
336 | srpc_service_fini(sv); | |
337 | return -EBUSY; | |
338 | } | |
339 | ||
340 | int | |
a659df62 | 341 | srpc_remove_service(srpc_service_t *sv) |
d7e09d03 PT |
342 | { |
343 | int id = sv->sv_id; | |
344 | ||
345 | spin_lock(&srpc_data.rpc_glock); | |
346 | ||
347 | if (srpc_data.rpc_services[id] != sv) { | |
348 | spin_unlock(&srpc_data.rpc_glock); | |
349 | return -ENOENT; | |
350 | } | |
351 | ||
352 | srpc_data.rpc_services[id] = NULL; | |
353 | spin_unlock(&srpc_data.rpc_glock); | |
354 | return 0; | |
355 | } | |
356 | ||
8d94b6d2 | 357 | static int |
d7e09d03 PT |
358 | srpc_post_passive_rdma(int portal, int local, __u64 matchbits, void *buf, |
359 | int len, int options, lnet_process_id_t peer, | |
360 | lnet_handle_md_t *mdh, srpc_event_t *ev) | |
361 | { | |
74d68011 MS |
362 | int rc; |
363 | lnet_md_t md; | |
d7e09d03 PT |
364 | lnet_handle_me_t meh; |
365 | ||
366 | rc = LNetMEAttach(portal, peer, matchbits, 0, LNET_UNLINK, | |
367 | local ? LNET_INS_LOCAL : LNET_INS_AFTER, &meh); | |
5fd88337 | 368 | if (rc) { |
a659df62 HE |
369 | CERROR("LNetMEAttach failed: %d\n", rc); |
370 | LASSERT(rc == -ENOMEM); | |
d7e09d03 PT |
371 | return -ENOMEM; |
372 | } | |
373 | ||
374 | md.threshold = 1; | |
ec436b9a JS |
375 | md.user_ptr = ev; |
376 | md.start = buf; | |
377 | md.length = len; | |
378 | md.options = options; | |
d7e09d03 PT |
379 | md.eq_handle = srpc_data.rpc_lnet_eq; |
380 | ||
381 | rc = LNetMDAttach(meh, md, LNET_UNLINK, mdh); | |
5fd88337 | 382 | if (rc) { |
a659df62 HE |
383 | CERROR("LNetMDAttach failed: %d\n", rc); |
384 | LASSERT(rc == -ENOMEM); | |
d7e09d03 PT |
385 | |
386 | rc = LNetMEUnlink(meh); | |
5fd88337 | 387 | LASSERT(!rc); |
d7e09d03 PT |
388 | return -ENOMEM; |
389 | } | |
390 | ||
c314c319 JS |
391 | CDEBUG(D_NET, "Posted passive RDMA: peer %s, portal %d, matchbits %#llx\n", |
392 | libcfs_id2str(peer), portal, matchbits); | |
d7e09d03 PT |
393 | return 0; |
394 | } | |
395 | ||
8d94b6d2 | 396 | static int |
d7e09d03 PT |
397 | srpc_post_active_rdma(int portal, __u64 matchbits, void *buf, int len, |
398 | int options, lnet_process_id_t peer, lnet_nid_t self, | |
399 | lnet_handle_md_t *mdh, srpc_event_t *ev) | |
400 | { | |
74d68011 | 401 | int rc; |
d7e09d03 PT |
402 | lnet_md_t md; |
403 | ||
ec436b9a JS |
404 | md.user_ptr = ev; |
405 | md.start = buf; | |
406 | md.length = len; | |
d7e09d03 | 407 | md.eq_handle = srpc_data.rpc_lnet_eq; |
5fd88337 | 408 | md.threshold = options & LNET_MD_OP_GET ? 2 : 1; |
ec436b9a | 409 | md.options = options & ~(LNET_MD_OP_PUT | LNET_MD_OP_GET); |
d7e09d03 PT |
410 | |
411 | rc = LNetMDBind(md, LNET_UNLINK, mdh); | |
5fd88337 | 412 | if (rc) { |
a659df62 HE |
413 | CERROR("LNetMDBind failed: %d\n", rc); |
414 | LASSERT(rc == -ENOMEM); | |
d7e09d03 PT |
415 | return -ENOMEM; |
416 | } | |
417 | ||
4420cfd3 JS |
418 | /* |
419 | * this is kind of an abuse of the LNET_MD_OP_{PUT,GET} options. | |
d7e09d03 | 420 | * they're only meaningful for MDs attached to an ME (i.e. passive |
4420cfd3 JS |
421 | * buffers... |
422 | */ | |
5fd88337 | 423 | if (options & LNET_MD_OP_PUT) { |
d7e09d03 PT |
424 | rc = LNetPut(self, *mdh, LNET_NOACK_REQ, peer, |
425 | portal, matchbits, 0, 0); | |
426 | } else { | |
5fd88337 | 427 | LASSERT(options & LNET_MD_OP_GET); |
d7e09d03 PT |
428 | |
429 | rc = LNetGet(self, *mdh, peer, portal, matchbits, 0); | |
430 | } | |
431 | ||
5fd88337 | 432 | if (rc) { |
a659df62 | 433 | CERROR("LNet%s(%s, %d, %lld) failed: %d\n", |
5fd88337 | 434 | options & LNET_MD_OP_PUT ? "Put" : "Get", |
c314c319 | 435 | libcfs_id2str(peer), portal, matchbits, rc); |
d7e09d03 | 436 | |
4420cfd3 JS |
437 | /* |
438 | * The forthcoming unlink event will complete this operation | |
d7e09d03 PT |
439 | * with failure, so fall through and return success here. |
440 | */ | |
441 | rc = LNetMDUnlink(*mdh); | |
5fd88337 | 442 | LASSERT(!rc); |
d7e09d03 | 443 | } else { |
c314c319 JS |
444 | CDEBUG(D_NET, "Posted active RDMA: peer %s, portal %u, matchbits %#llx\n", |
445 | libcfs_id2str(peer), portal, matchbits); | |
d7e09d03 PT |
446 | } |
447 | return 0; | |
448 | } | |
449 | ||
8d94b6d2 | 450 | static int |
d7e09d03 PT |
451 | srpc_post_passive_rqtbuf(int service, int local, void *buf, int len, |
452 | lnet_handle_md_t *mdh, srpc_event_t *ev) | |
453 | { | |
00b5fdb7 | 454 | lnet_process_id_t any = { 0 }; |
d7e09d03 PT |
455 | |
456 | any.nid = LNET_NID_ANY; | |
457 | any.pid = LNET_PID_ANY; | |
458 | ||
459 | return srpc_post_passive_rdma(srpc_serv_portal(service), | |
460 | local, service, buf, len, | |
461 | LNET_MD_OP_PUT, any, mdh, ev); | |
462 | } | |
463 | ||
8d94b6d2 | 464 | static int |
d7e09d03 | 465 | srpc_service_post_buffer(struct srpc_service_cd *scd, struct srpc_buffer *buf) |
b794d796 | 466 | __must_hold(&scd->scd_lock) |
d7e09d03 | 467 | { |
74d68011 | 468 | struct srpc_service *sv = scd->scd_svc; |
00b5fdb7 | 469 | struct srpc_msg *msg = &buf->buf_msg; |
74d68011 | 470 | int rc; |
d7e09d03 PT |
471 | |
472 | LNetInvalidateHandle(&buf->buf_mdh); | |
473 | list_add(&buf->buf_list, &scd->scd_buf_posted); | |
474 | scd->scd_buf_nposted++; | |
475 | spin_unlock(&scd->scd_lock); | |
476 | ||
477 | rc = srpc_post_passive_rqtbuf(sv->sv_id, | |
478 | !srpc_serv_is_framework(sv), | |
479 | msg, sizeof(*msg), &buf->buf_mdh, | |
480 | &scd->scd_ev); | |
481 | ||
4420cfd3 JS |
482 | /* |
483 | * At this point, a RPC (new or delayed) may have arrived in | |
d7e09d03 | 484 | * msg and its event handler has been called. So we must add |
4420cfd3 JS |
485 | * buf to scd_buf_posted _before_ dropping scd_lock |
486 | */ | |
d7e09d03 PT |
487 | spin_lock(&scd->scd_lock); |
488 | ||
5fd88337 | 489 | if (!rc) { |
d7e09d03 PT |
490 | if (!sv->sv_shuttingdown) |
491 | return 0; | |
492 | ||
493 | spin_unlock(&scd->scd_lock); | |
4420cfd3 JS |
494 | /* |
495 | * srpc_shutdown_service might have tried to unlink me | |
496 | * when my buf_mdh was still invalid | |
497 | */ | |
d7e09d03 PT |
498 | LNetMDUnlink(buf->buf_mdh); |
499 | spin_lock(&scd->scd_lock); | |
500 | return 0; | |
501 | } | |
502 | ||
503 | scd->scd_buf_nposted--; | |
504 | if (sv->sv_shuttingdown) | |
505 | return rc; /* don't allow to change scd_buf_posted */ | |
506 | ||
507 | list_del(&buf->buf_list); | |
508 | spin_unlock(&scd->scd_lock); | |
509 | ||
510 | LIBCFS_FREE(buf, sizeof(*buf)); | |
511 | ||
512 | spin_lock(&scd->scd_lock); | |
513 | return rc; | |
514 | } | |
515 | ||
516 | int | |
517 | srpc_add_buffer(struct swi_workitem *wi) | |
518 | { | |
74d68011 MS |
519 | struct srpc_service_cd *scd = wi->swi_workitem.wi_data; |
520 | struct srpc_buffer *buf; | |
521 | int rc = 0; | |
d7e09d03 | 522 | |
4420cfd3 JS |
523 | /* |
524 | * it's called by workitem scheduler threads, these threads | |
d7e09d03 | 525 | * should have been set CPT affinity, so buffers will be posted |
4420cfd3 JS |
526 | * on CPT local list of Portal |
527 | */ | |
d7e09d03 PT |
528 | spin_lock(&scd->scd_lock); |
529 | ||
530 | while (scd->scd_buf_adjust > 0 && | |
531 | !scd->scd_svc->sv_shuttingdown) { | |
532 | scd->scd_buf_adjust--; /* consume it */ | |
533 | scd->scd_buf_posting++; | |
534 | ||
535 | spin_unlock(&scd->scd_lock); | |
536 | ||
537 | LIBCFS_ALLOC(buf, sizeof(*buf)); | |
06ace26e | 538 | if (!buf) { |
d7e09d03 PT |
539 | CERROR("Failed to add new buf to service: %s\n", |
540 | scd->scd_svc->sv_name); | |
541 | spin_lock(&scd->scd_lock); | |
542 | rc = -ENOMEM; | |
543 | break; | |
544 | } | |
545 | ||
546 | spin_lock(&scd->scd_lock); | |
547 | if (scd->scd_svc->sv_shuttingdown) { | |
548 | spin_unlock(&scd->scd_lock); | |
549 | LIBCFS_FREE(buf, sizeof(*buf)); | |
550 | ||
551 | spin_lock(&scd->scd_lock); | |
552 | rc = -ESHUTDOWN; | |
553 | break; | |
554 | } | |
555 | ||
556 | rc = srpc_service_post_buffer(scd, buf); | |
5fd88337 | 557 | if (rc) |
d7e09d03 PT |
558 | break; /* buf has been freed inside */ |
559 | ||
560 | LASSERT(scd->scd_buf_posting > 0); | |
561 | scd->scd_buf_posting--; | |
562 | scd->scd_buf_total++; | |
0c575417 | 563 | scd->scd_buf_low = max(2, scd->scd_buf_total / 4); |
d7e09d03 PT |
564 | } |
565 | ||
5fd88337 | 566 | if (rc) { |
d9f79e6b | 567 | scd->scd_buf_err_stamp = ktime_get_real_seconds(); |
d7e09d03 PT |
568 | scd->scd_buf_err = rc; |
569 | ||
570 | LASSERT(scd->scd_buf_posting > 0); | |
571 | scd->scd_buf_posting--; | |
572 | } | |
573 | ||
574 | spin_unlock(&scd->scd_lock); | |
575 | return 0; | |
576 | } | |
577 | ||
578 | int | |
579 | srpc_service_add_buffers(struct srpc_service *sv, int nbuffer) | |
580 | { | |
74d68011 MS |
581 | struct srpc_service_cd *scd; |
582 | int rc = 0; | |
583 | int i; | |
d7e09d03 PT |
584 | |
585 | LASSERTF(nbuffer > 0, "nbuffer must be positive: %d\n", nbuffer); | |
586 | ||
587 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) { | |
588 | spin_lock(&scd->scd_lock); | |
589 | ||
590 | scd->scd_buf_err = 0; | |
591 | scd->scd_buf_err_stamp = 0; | |
592 | scd->scd_buf_posting = 0; | |
593 | scd->scd_buf_adjust = nbuffer; | |
594 | /* start to post buffers */ | |
595 | swi_schedule_workitem(&scd->scd_buf_wi); | |
596 | spin_unlock(&scd->scd_lock); | |
597 | ||
598 | /* framework service only post buffer for one partition */ | |
599 | if (srpc_serv_is_framework(sv)) | |
600 | break; | |
601 | } | |
602 | ||
603 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) { | |
604 | spin_lock(&scd->scd_lock); | |
605 | /* | |
606 | * NB: srpc_service_add_buffers() can be called inside | |
607 | * thread context of lst_sched_serial, and we don't normally | |
608 | * allow to sleep inside thread context of WI scheduler | |
609 | * because it will block current scheduler thread from doing | |
610 | * anything else, even worse, it could deadlock if it's | |
611 | * waiting on result from another WI of the same scheduler. | |
612 | * However, it's safe at here because scd_buf_wi is scheduled | |
613 | * by thread in a different WI scheduler (lst_sched_test), | |
614 | * so we don't have any risk of deadlock, though this could | |
615 | * block all WIs pending on lst_sched_serial for a moment | |
616 | * which is not good but not fatal. | |
617 | */ | |
5fd88337 JS |
618 | lst_wait_until(scd->scd_buf_err || |
619 | (!scd->scd_buf_adjust && | |
620 | !scd->scd_buf_posting), | |
d7e09d03 PT |
621 | scd->scd_lock, "waiting for adding buffer\n"); |
622 | ||
5fd88337 | 623 | if (scd->scd_buf_err && !rc) |
d7e09d03 PT |
624 | rc = scd->scd_buf_err; |
625 | ||
626 | spin_unlock(&scd->scd_lock); | |
627 | } | |
628 | ||
629 | return rc; | |
630 | } | |
631 | ||
632 | void | |
633 | srpc_service_remove_buffers(struct srpc_service *sv, int nbuffer) | |
634 | { | |
74d68011 MS |
635 | struct srpc_service_cd *scd; |
636 | int num; | |
637 | int i; | |
d7e09d03 PT |
638 | |
639 | LASSERT(!sv->sv_shuttingdown); | |
640 | ||
641 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) { | |
642 | spin_lock(&scd->scd_lock); | |
643 | ||
644 | num = scd->scd_buf_total + scd->scd_buf_posting; | |
645 | scd->scd_buf_adjust -= min(nbuffer, num); | |
646 | ||
647 | spin_unlock(&scd->scd_lock); | |
648 | } | |
649 | } | |
650 | ||
651 | /* returns 1 if sv has finished, otherwise 0 */ | |
652 | int | |
653 | srpc_finish_service(struct srpc_service *sv) | |
654 | { | |
74d68011 MS |
655 | struct srpc_service_cd *scd; |
656 | struct srpc_server_rpc *rpc; | |
657 | int i; | |
d7e09d03 PT |
658 | |
659 | LASSERT(sv->sv_shuttingdown); /* srpc_shutdown_service called */ | |
660 | ||
661 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) { | |
662 | spin_lock(&scd->scd_lock); | |
f8c47be4 SB |
663 | if (!swi_deschedule_workitem(&scd->scd_buf_wi)) { |
664 | spin_unlock(&scd->scd_lock); | |
d7e09d03 | 665 | return 0; |
f8c47be4 | 666 | } |
d7e09d03 PT |
667 | |
668 | if (scd->scd_buf_nposted > 0) { | |
19b2056f | 669 | CDEBUG(D_NET, "waiting for %d posted buffers to unlink\n", |
d7e09d03 PT |
670 | scd->scd_buf_nposted); |
671 | spin_unlock(&scd->scd_lock); | |
672 | return 0; | |
673 | } | |
674 | ||
675 | if (list_empty(&scd->scd_rpc_active)) { | |
676 | spin_unlock(&scd->scd_lock); | |
677 | continue; | |
678 | } | |
679 | ||
680 | rpc = list_entry(scd->scd_rpc_active.next, | |
c314c319 | 681 | struct srpc_server_rpc, srpc_list); |
2d00bd17 | 682 | CNETERR("Active RPC %p on shutdown: sv %s, peer %s, wi %s scheduled %d running %d, ev fired %d type %d status %d lnet %d\n", |
d7e09d03 PT |
683 | rpc, sv->sv_name, libcfs_id2str(rpc->srpc_peer), |
684 | swi_state2str(rpc->srpc_wi.swi_state), | |
685 | rpc->srpc_wi.swi_workitem.wi_scheduled, | |
686 | rpc->srpc_wi.swi_workitem.wi_running, | |
687 | rpc->srpc_ev.ev_fired, rpc->srpc_ev.ev_type, | |
688 | rpc->srpc_ev.ev_status, rpc->srpc_ev.ev_lnet); | |
689 | spin_unlock(&scd->scd_lock); | |
690 | return 0; | |
691 | } | |
692 | ||
693 | /* no lock needed from now on */ | |
694 | srpc_service_fini(sv); | |
695 | return 1; | |
696 | } | |
697 | ||
698 | /* called with sv->sv_lock held */ | |
8d94b6d2 | 699 | static void |
d7e09d03 | 700 | srpc_service_recycle_buffer(struct srpc_service_cd *scd, srpc_buffer_t *buf) |
b794d796 | 701 | __must_hold(&scd->scd_lock) |
d7e09d03 PT |
702 | { |
703 | if (!scd->scd_svc->sv_shuttingdown && scd->scd_buf_adjust >= 0) { | |
5fd88337 | 704 | if (srpc_service_post_buffer(scd, buf)) { |
d7e09d03 PT |
705 | CWARN("Failed to post %s buffer\n", |
706 | scd->scd_svc->sv_name); | |
707 | } | |
708 | return; | |
709 | } | |
710 | ||
711 | /* service is shutting down, or we want to recycle some buffers */ | |
712 | scd->scd_buf_total--; | |
713 | ||
714 | if (scd->scd_buf_adjust < 0) { | |
715 | scd->scd_buf_adjust++; | |
716 | if (scd->scd_buf_adjust < 0 && | |
5fd88337 | 717 | !scd->scd_buf_total && !scd->scd_buf_posting) { |
d7e09d03 | 718 | CDEBUG(D_INFO, |
23ebb3fd | 719 | "Try to recycle %d buffers but nothing left\n", |
d7e09d03 PT |
720 | scd->scd_buf_adjust); |
721 | scd->scd_buf_adjust = 0; | |
722 | } | |
723 | } | |
724 | ||
725 | spin_unlock(&scd->scd_lock); | |
726 | LIBCFS_FREE(buf, sizeof(*buf)); | |
727 | spin_lock(&scd->scd_lock); | |
728 | } | |
729 | ||
730 | void | |
731 | srpc_abort_service(struct srpc_service *sv) | |
732 | { | |
74d68011 MS |
733 | struct srpc_service_cd *scd; |
734 | struct srpc_server_rpc *rpc; | |
735 | int i; | |
d7e09d03 PT |
736 | |
737 | CDEBUG(D_NET, "Aborting service: id %d, name %s\n", | |
738 | sv->sv_id, sv->sv_name); | |
739 | ||
740 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) { | |
741 | spin_lock(&scd->scd_lock); | |
742 | ||
4420cfd3 JS |
743 | /* |
744 | * schedule in-flight RPCs to notice the abort, NB: | |
d7e09d03 | 745 | * racing with incoming RPCs; complete fix should make test |
4420cfd3 JS |
746 | * RPCs carry session ID in its headers |
747 | */ | |
d7e09d03 PT |
748 | list_for_each_entry(rpc, &scd->scd_rpc_active, srpc_list) { |
749 | rpc->srpc_aborted = 1; | |
750 | swi_schedule_workitem(&rpc->srpc_wi); | |
751 | } | |
752 | ||
753 | spin_unlock(&scd->scd_lock); | |
754 | } | |
755 | } | |
756 | ||
757 | void | |
758 | srpc_shutdown_service(srpc_service_t *sv) | |
759 | { | |
74d68011 MS |
760 | struct srpc_service_cd *scd; |
761 | struct srpc_server_rpc *rpc; | |
762 | srpc_buffer_t *buf; | |
763 | int i; | |
d7e09d03 PT |
764 | |
765 | CDEBUG(D_NET, "Shutting down service: id %d, name %s\n", | |
766 | sv->sv_id, sv->sv_name); | |
767 | ||
768 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) | |
769 | spin_lock(&scd->scd_lock); | |
770 | ||
771 | sv->sv_shuttingdown = 1; /* i.e. no new active RPC */ | |
772 | ||
773 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) | |
774 | spin_unlock(&scd->scd_lock); | |
775 | ||
776 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) { | |
777 | spin_lock(&scd->scd_lock); | |
778 | ||
779 | /* schedule in-flight RPCs to notice the shutdown */ | |
780 | list_for_each_entry(rpc, &scd->scd_rpc_active, srpc_list) | |
781 | swi_schedule_workitem(&rpc->srpc_wi); | |
782 | ||
783 | spin_unlock(&scd->scd_lock); | |
784 | ||
4420cfd3 JS |
785 | /* |
786 | * OK to traverse scd_buf_posted without lock, since no one | |
787 | * touches scd_buf_posted now | |
788 | */ | |
d7e09d03 PT |
789 | list_for_each_entry(buf, &scd->scd_buf_posted, buf_list) |
790 | LNetMDUnlink(buf->buf_mdh); | |
791 | } | |
792 | } | |
793 | ||
8d94b6d2 | 794 | static int |
a659df62 | 795 | srpc_send_request(srpc_client_rpc_t *rpc) |
d7e09d03 PT |
796 | { |
797 | srpc_event_t *ev = &rpc->crpc_reqstev; | |
74d68011 | 798 | int rc; |
d7e09d03 PT |
799 | |
800 | ev->ev_fired = 0; | |
ec436b9a JS |
801 | ev->ev_data = rpc; |
802 | ev->ev_type = SRPC_REQUEST_SENT; | |
d7e09d03 | 803 | |
ab419903 SB |
804 | rc = srpc_post_active_rdma(srpc_serv_portal(rpc->crpc_service), |
805 | rpc->crpc_service, &rpc->crpc_reqstmsg, | |
806 | sizeof(srpc_msg_t), LNET_MD_OP_PUT, | |
807 | rpc->crpc_dest, LNET_NID_ANY, | |
808 | &rpc->crpc_reqstmdh, ev); | |
5fd88337 | 809 | if (rc) { |
a659df62 | 810 | LASSERT(rc == -ENOMEM); |
d7e09d03 PT |
811 | ev->ev_fired = 1; /* no more event expected */ |
812 | } | |
813 | return rc; | |
814 | } | |
815 | ||
8d94b6d2 | 816 | static int |
a659df62 | 817 | srpc_prepare_reply(srpc_client_rpc_t *rpc) |
d7e09d03 PT |
818 | { |
819 | srpc_event_t *ev = &rpc->crpc_replyev; | |
74d68011 MS |
820 | __u64 *id = &rpc->crpc_reqstmsg.msg_body.reqst.rpyid; |
821 | int rc; | |
d7e09d03 PT |
822 | |
823 | ev->ev_fired = 0; | |
ec436b9a JS |
824 | ev->ev_data = rpc; |
825 | ev->ev_type = SRPC_REPLY_RCVD; | |
d7e09d03 PT |
826 | |
827 | *id = srpc_next_id(); | |
828 | ||
829 | rc = srpc_post_passive_rdma(SRPC_RDMA_PORTAL, 0, *id, | |
830 | &rpc->crpc_replymsg, sizeof(srpc_msg_t), | |
831 | LNET_MD_OP_PUT, rpc->crpc_dest, | |
832 | &rpc->crpc_replymdh, ev); | |
5fd88337 | 833 | if (rc) { |
a659df62 | 834 | LASSERT(rc == -ENOMEM); |
d7e09d03 PT |
835 | ev->ev_fired = 1; /* no more event expected */ |
836 | } | |
837 | return rc; | |
838 | } | |
839 | ||
8d94b6d2 | 840 | static int |
a659df62 | 841 | srpc_prepare_bulk(srpc_client_rpc_t *rpc) |
d7e09d03 | 842 | { |
74d68011 | 843 | srpc_bulk_t *bk = &rpc->crpc_bulk; |
d7e09d03 | 844 | srpc_event_t *ev = &rpc->crpc_bulkev; |
00b5fdb7 | 845 | __u64 *id = &rpc->crpc_reqstmsg.msg_body.reqst.bulkid; |
74d68011 MS |
846 | int rc; |
847 | int opt; | |
d7e09d03 | 848 | |
a659df62 | 849 | LASSERT(bk->bk_niov <= LNET_MAX_IOV); |
d7e09d03 | 850 | |
5fd88337 | 851 | if (!bk->bk_niov) |
f64a6f3c | 852 | return 0; /* nothing to do */ |
d7e09d03 PT |
853 | |
854 | opt = bk->bk_sink ? LNET_MD_OP_PUT : LNET_MD_OP_GET; | |
855 | opt |= LNET_MD_KIOV; | |
856 | ||
857 | ev->ev_fired = 0; | |
ec436b9a JS |
858 | ev->ev_data = rpc; |
859 | ev->ev_type = SRPC_BULK_REQ_RCVD; | |
d7e09d03 PT |
860 | |
861 | *id = srpc_next_id(); | |
862 | ||
863 | rc = srpc_post_passive_rdma(SRPC_RDMA_PORTAL, 0, *id, | |
864 | &bk->bk_iovs[0], bk->bk_niov, opt, | |
865 | rpc->crpc_dest, &bk->bk_mdh, ev); | |
5fd88337 | 866 | if (rc) { |
a659df62 | 867 | LASSERT(rc == -ENOMEM); |
d7e09d03 PT |
868 | ev->ev_fired = 1; /* no more event expected */ |
869 | } | |
870 | return rc; | |
871 | } | |
872 | ||
8d94b6d2 | 873 | static int |
168c7a13 | 874 | srpc_do_bulk(struct srpc_server_rpc *rpc) |
d7e09d03 | 875 | { |
74d68011 MS |
876 | srpc_event_t *ev = &rpc->srpc_ev; |
877 | srpc_bulk_t *bk = rpc->srpc_bulk; | |
878 | __u64 id = rpc->srpc_reqstbuf->buf_msg.msg_body.reqst.bulkid; | |
879 | int rc; | |
880 | int opt; | |
d7e09d03 | 881 | |
06ace26e | 882 | LASSERT(bk); |
d7e09d03 PT |
883 | |
884 | opt = bk->bk_sink ? LNET_MD_OP_GET : LNET_MD_OP_PUT; | |
885 | opt |= LNET_MD_KIOV; | |
886 | ||
887 | ev->ev_fired = 0; | |
ec436b9a JS |
888 | ev->ev_data = rpc; |
889 | ev->ev_type = bk->bk_sink ? SRPC_BULK_GET_RPLD : SRPC_BULK_PUT_SENT; | |
d7e09d03 PT |
890 | |
891 | rc = srpc_post_active_rdma(SRPC_RDMA_PORTAL, id, | |
892 | &bk->bk_iovs[0], bk->bk_niov, opt, | |
893 | rpc->srpc_peer, rpc->srpc_self, | |
894 | &bk->bk_mdh, ev); | |
5fd88337 | 895 | if (rc) |
d7e09d03 PT |
896 | ev->ev_fired = 1; /* no more event expected */ |
897 | return rc; | |
898 | } | |
899 | ||
900 | /* only called from srpc_handle_rpc */ | |
8d94b6d2 | 901 | static void |
168c7a13 | 902 | srpc_server_rpc_done(struct srpc_server_rpc *rpc, int status) |
d7e09d03 | 903 | { |
74d68011 | 904 | struct srpc_service_cd *scd = rpc->srpc_scd; |
00b5fdb7 | 905 | struct srpc_service *sv = scd->scd_svc; |
74d68011 | 906 | srpc_buffer_t *buffer; |
d7e09d03 | 907 | |
5fd88337 | 908 | LASSERT(status || rpc->srpc_wi.swi_state == SWI_STATE_DONE); |
d7e09d03 PT |
909 | |
910 | rpc->srpc_status = status; | |
911 | ||
5fd88337 | 912 | CDEBUG_LIMIT(!status ? D_NET : D_NETERROR, |
c314c319 JS |
913 | "Server RPC %p done: service %s, peer %s, status %s:%d\n", |
914 | rpc, sv->sv_name, libcfs_id2str(rpc->srpc_peer), | |
915 | swi_state2str(rpc->srpc_wi.swi_state), status); | |
d7e09d03 | 916 | |
5fd88337 | 917 | if (status) { |
d7e09d03 PT |
918 | spin_lock(&srpc_data.rpc_glock); |
919 | srpc_data.rpc_counters.rpcs_dropped++; | |
920 | spin_unlock(&srpc_data.rpc_glock); | |
921 | } | |
922 | ||
06ace26e | 923 | if (rpc->srpc_done) |
d7e09d03 | 924 | (*rpc->srpc_done) (rpc); |
06ace26e | 925 | LASSERT(!rpc->srpc_bulk); |
d7e09d03 PT |
926 | |
927 | spin_lock(&scd->scd_lock); | |
928 | ||
06ace26e | 929 | if (rpc->srpc_reqstbuf) { |
4420cfd3 JS |
930 | /* |
931 | * NB might drop sv_lock in srpc_service_recycle_buffer, but | |
932 | * sv won't go away for scd_rpc_active must not be empty | |
933 | */ | |
d7e09d03 PT |
934 | srpc_service_recycle_buffer(scd, rpc->srpc_reqstbuf); |
935 | rpc->srpc_reqstbuf = NULL; | |
936 | } | |
937 | ||
938 | list_del(&rpc->srpc_list); /* from scd->scd_rpc_active */ | |
939 | ||
940 | /* | |
941 | * No one can schedule me now since: | |
942 | * - I'm not on scd_rpc_active. | |
943 | * - all LNet events have been fired. | |
944 | * Cancel pending schedules and prevent future schedule attempts: | |
945 | */ | |
946 | LASSERT(rpc->srpc_ev.ev_fired); | |
947 | swi_exit_workitem(&rpc->srpc_wi); | |
948 | ||
949 | if (!sv->sv_shuttingdown && !list_empty(&scd->scd_buf_blocked)) { | |
950 | buffer = list_entry(scd->scd_buf_blocked.next, | |
c314c319 | 951 | srpc_buffer_t, buf_list); |
d7e09d03 PT |
952 | list_del(&buffer->buf_list); |
953 | ||
954 | srpc_init_server_rpc(rpc, scd, buffer); | |
955 | list_add_tail(&rpc->srpc_list, &scd->scd_rpc_active); | |
956 | swi_schedule_workitem(&rpc->srpc_wi); | |
957 | } else { | |
958 | list_add(&rpc->srpc_list, &scd->scd_rpc_free); | |
959 | } | |
960 | ||
961 | spin_unlock(&scd->scd_lock); | |
d7e09d03 PT |
962 | } |
963 | ||
964 | /* handles an incoming RPC */ | |
965 | int | |
966 | srpc_handle_rpc(swi_workitem_t *wi) | |
967 | { | |
74d68011 MS |
968 | struct srpc_server_rpc *rpc = wi->swi_workitem.wi_data; |
969 | struct srpc_service_cd *scd = rpc->srpc_scd; | |
970 | struct srpc_service *sv = scd->scd_svc; | |
971 | srpc_event_t *ev = &rpc->srpc_ev; | |
972 | int rc = 0; | |
d7e09d03 PT |
973 | |
974 | LASSERT(wi == &rpc->srpc_wi); | |
975 | ||
976 | spin_lock(&scd->scd_lock); | |
977 | ||
978 | if (sv->sv_shuttingdown || rpc->srpc_aborted) { | |
979 | spin_unlock(&scd->scd_lock); | |
980 | ||
06ace26e | 981 | if (rpc->srpc_bulk) |
d7e09d03 PT |
982 | LNetMDUnlink(rpc->srpc_bulk->bk_mdh); |
983 | LNetMDUnlink(rpc->srpc_replymdh); | |
984 | ||
985 | if (ev->ev_fired) { /* no more event, OK to finish */ | |
986 | srpc_server_rpc_done(rpc, -ESHUTDOWN); | |
987 | return 1; | |
988 | } | |
989 | return 0; | |
990 | } | |
991 | ||
992 | spin_unlock(&scd->scd_lock); | |
993 | ||
994 | switch (wi->swi_state) { | |
995 | default: | |
a659df62 | 996 | LBUG(); |
d7e09d03 | 997 | case SWI_STATE_NEWBORN: { |
74d68011 | 998 | srpc_msg_t *msg; |
d7e09d03 PT |
999 | srpc_generic_reply_t *reply; |
1000 | ||
1001 | msg = &rpc->srpc_reqstbuf->buf_msg; | |
1002 | reply = &rpc->srpc_replymsg.msg_body.reply; | |
1003 | ||
5fd88337 | 1004 | if (!msg->msg_magic) { |
d7e09d03 PT |
1005 | /* moaned already in srpc_lnet_ev_handler */ |
1006 | srpc_server_rpc_done(rpc, EBADMSG); | |
1007 | return 1; | |
1008 | } | |
1009 | ||
1010 | srpc_unpack_msg_hdr(msg); | |
1011 | if (msg->msg_version != SRPC_MSG_VERSION) { | |
1012 | CWARN("Version mismatch: %u, %u expected, from %s\n", | |
1013 | msg->msg_version, SRPC_MSG_VERSION, | |
1014 | libcfs_id2str(rpc->srpc_peer)); | |
1015 | reply->status = EPROTO; | |
1016 | /* drop through and send reply */ | |
1017 | } else { | |
1018 | reply->status = 0; | |
1019 | rc = (*sv->sv_handler)(rpc); | |
5fd88337 JS |
1020 | LASSERT(!reply->status || !rpc->srpc_bulk); |
1021 | if (rc) { | |
d7e09d03 PT |
1022 | srpc_server_rpc_done(rpc, rc); |
1023 | return 1; | |
1024 | } | |
1025 | } | |
1026 | ||
1027 | wi->swi_state = SWI_STATE_BULK_STARTED; | |
1028 | ||
06ace26e | 1029 | if (rpc->srpc_bulk) { |
d7e09d03 | 1030 | rc = srpc_do_bulk(rpc); |
5fd88337 | 1031 | if (!rc) |
d7e09d03 PT |
1032 | return 0; /* wait for bulk */ |
1033 | ||
a659df62 | 1034 | LASSERT(ev->ev_fired); |
d7e09d03 PT |
1035 | ev->ev_status = rc; |
1036 | } | |
1037 | } | |
1038 | case SWI_STATE_BULK_STARTED: | |
06ace26e | 1039 | LASSERT(!rpc->srpc_bulk || ev->ev_fired); |
d7e09d03 | 1040 | |
06ace26e | 1041 | if (rpc->srpc_bulk) { |
d7e09d03 PT |
1042 | rc = ev->ev_status; |
1043 | ||
06ace26e | 1044 | if (sv->sv_bulk_ready) |
d7e09d03 PT |
1045 | rc = (*sv->sv_bulk_ready) (rpc, rc); |
1046 | ||
5fd88337 | 1047 | if (rc) { |
d7e09d03 PT |
1048 | srpc_server_rpc_done(rpc, rc); |
1049 | return 1; | |
1050 | } | |
1051 | } | |
1052 | ||
1053 | wi->swi_state = SWI_STATE_REPLY_SUBMITTED; | |
1054 | rc = srpc_send_reply(rpc); | |
5fd88337 | 1055 | if (!rc) |
d7e09d03 PT |
1056 | return 0; /* wait for reply */ |
1057 | srpc_server_rpc_done(rpc, rc); | |
1058 | return 1; | |
1059 | ||
1060 | case SWI_STATE_REPLY_SUBMITTED: | |
1061 | if (!ev->ev_fired) { | |
1062 | CERROR("RPC %p: bulk %p, service %d\n", | |
1063 | rpc, rpc->srpc_bulk, sv->sv_id); | |
1064 | CERROR("Event: status %d, type %d, lnet %d\n", | |
1065 | ev->ev_status, ev->ev_type, ev->ev_lnet); | |
a659df62 | 1066 | LASSERT(ev->ev_fired); |
d7e09d03 PT |
1067 | } |
1068 | ||
1069 | wi->swi_state = SWI_STATE_DONE; | |
1070 | srpc_server_rpc_done(rpc, ev->ev_status); | |
1071 | return 1; | |
1072 | } | |
1073 | ||
1074 | return 0; | |
1075 | } | |
1076 | ||
8d94b6d2 | 1077 | static void |
a659df62 | 1078 | srpc_client_rpc_expired(void *data) |
d7e09d03 PT |
1079 | { |
1080 | srpc_client_rpc_t *rpc = data; | |
1081 | ||
a659df62 | 1082 | CWARN("Client RPC expired: service %d, peer %s, timeout %d.\n", |
c314c319 JS |
1083 | rpc->crpc_service, libcfs_id2str(rpc->crpc_dest), |
1084 | rpc->crpc_timeout); | |
d7e09d03 PT |
1085 | |
1086 | spin_lock(&rpc->crpc_lock); | |
1087 | ||
1088 | rpc->crpc_timeout = 0; | |
1089 | srpc_abort_rpc(rpc, -ETIMEDOUT); | |
1090 | ||
1091 | spin_unlock(&rpc->crpc_lock); | |
1092 | ||
1093 | spin_lock(&srpc_data.rpc_glock); | |
1094 | srpc_data.rpc_counters.rpcs_expired++; | |
1095 | spin_unlock(&srpc_data.rpc_glock); | |
1096 | } | |
1097 | ||
505510fd | 1098 | static void |
a659df62 | 1099 | srpc_add_client_rpc_timer(srpc_client_rpc_t *rpc) |
d7e09d03 | 1100 | { |
27f9aea3 | 1101 | struct stt_timer *timer = &rpc->crpc_timer; |
d7e09d03 | 1102 | |
5fd88337 | 1103 | if (!rpc->crpc_timeout) |
f64a6f3c | 1104 | return; |
d7e09d03 PT |
1105 | |
1106 | INIT_LIST_HEAD(&timer->stt_list); | |
b794d796 JS |
1107 | timer->stt_data = rpc; |
1108 | timer->stt_func = srpc_client_rpc_expired; | |
d9f79e6b | 1109 | timer->stt_expires = ktime_get_real_seconds() + rpc->crpc_timeout; |
d7e09d03 | 1110 | stt_add_timer(timer); |
d7e09d03 PT |
1111 | } |
1112 | ||
1113 | /* | |
1114 | * Called with rpc->crpc_lock held. | |
1115 | * | |
1116 | * Upon exit the RPC expiry timer is not queued and the handler is not | |
4420cfd3 JS |
1117 | * running on any CPU. |
1118 | */ | |
8d94b6d2 | 1119 | static void |
a659df62 | 1120 | srpc_del_client_rpc_timer(srpc_client_rpc_t *rpc) |
d7e09d03 PT |
1121 | { |
1122 | /* timer not planted or already exploded */ | |
5fd88337 | 1123 | if (!rpc->crpc_timeout) |
d7e09d03 PT |
1124 | return; |
1125 | ||
2b284326 | 1126 | /* timer successfully defused */ |
d7e09d03 PT |
1127 | if (stt_del_timer(&rpc->crpc_timer)) |
1128 | return; | |
1129 | ||
1130 | /* timer detonated, wait for it to explode */ | |
5fd88337 | 1131 | while (rpc->crpc_timeout) { |
d7e09d03 PT |
1132 | spin_unlock(&rpc->crpc_lock); |
1133 | ||
1134 | schedule(); | |
1135 | ||
1136 | spin_lock(&rpc->crpc_lock); | |
1137 | } | |
1138 | } | |
1139 | ||
8d94b6d2 | 1140 | static void |
a659df62 | 1141 | srpc_client_rpc_done(srpc_client_rpc_t *rpc, int status) |
d7e09d03 PT |
1142 | { |
1143 | swi_workitem_t *wi = &rpc->crpc_wi; | |
1144 | ||
5fd88337 | 1145 | LASSERT(status || wi->swi_state == SWI_STATE_DONE); |
d7e09d03 PT |
1146 | |
1147 | spin_lock(&rpc->crpc_lock); | |
1148 | ||
1149 | rpc->crpc_closed = 1; | |
5fd88337 | 1150 | if (!rpc->crpc_status) |
d7e09d03 PT |
1151 | rpc->crpc_status = status; |
1152 | ||
1153 | srpc_del_client_rpc_timer(rpc); | |
1154 | ||
5fd88337 | 1155 | CDEBUG_LIMIT(!status ? D_NET : D_NETERROR, |
c314c319 JS |
1156 | "Client RPC done: service %d, peer %s, status %s:%d:%d\n", |
1157 | rpc->crpc_service, libcfs_id2str(rpc->crpc_dest), | |
1158 | swi_state2str(wi->swi_state), rpc->crpc_aborted, status); | |
d7e09d03 PT |
1159 | |
1160 | /* | |
1161 | * No one can schedule me now since: | |
1162 | * - RPC timer has been defused. | |
1163 | * - all LNet events have been fired. | |
1164 | * - crpc_closed has been set, preventing srpc_abort_rpc from | |
1165 | * scheduling me. | |
1166 | * Cancel pending schedules and prevent future schedule attempts: | |
1167 | */ | |
a659df62 | 1168 | LASSERT(!srpc_event_pending(rpc)); |
d7e09d03 PT |
1169 | swi_exit_workitem(wi); |
1170 | ||
1171 | spin_unlock(&rpc->crpc_lock); | |
1172 | ||
1173 | (*rpc->crpc_done)(rpc); | |
d7e09d03 PT |
1174 | } |
1175 | ||
1176 | /* sends an outgoing RPC */ | |
1177 | int | |
a659df62 | 1178 | srpc_send_rpc(swi_workitem_t *wi) |
d7e09d03 | 1179 | { |
74d68011 | 1180 | int rc = 0; |
d7e09d03 | 1181 | srpc_client_rpc_t *rpc; |
74d68011 MS |
1182 | srpc_msg_t *reply; |
1183 | int do_bulk; | |
d7e09d03 | 1184 | |
06ace26e | 1185 | LASSERT(wi); |
d7e09d03 PT |
1186 | |
1187 | rpc = wi->swi_workitem.wi_data; | |
1188 | ||
06ace26e | 1189 | LASSERT(rpc); |
a659df62 | 1190 | LASSERT(wi == &rpc->crpc_wi); |
d7e09d03 PT |
1191 | |
1192 | reply = &rpc->crpc_replymsg; | |
1193 | do_bulk = rpc->crpc_bulk.bk_niov > 0; | |
1194 | ||
1195 | spin_lock(&rpc->crpc_lock); | |
1196 | ||
1197 | if (rpc->crpc_aborted) { | |
1198 | spin_unlock(&rpc->crpc_lock); | |
1199 | goto abort; | |
1200 | } | |
1201 | ||
1202 | spin_unlock(&rpc->crpc_lock); | |
1203 | ||
1204 | switch (wi->swi_state) { | |
1205 | default: | |
a659df62 | 1206 | LBUG(); |
d7e09d03 | 1207 | case SWI_STATE_NEWBORN: |
a659df62 | 1208 | LASSERT(!srpc_event_pending(rpc)); |
d7e09d03 PT |
1209 | |
1210 | rc = srpc_prepare_reply(rpc); | |
5fd88337 | 1211 | if (rc) { |
d7e09d03 PT |
1212 | srpc_client_rpc_done(rpc, rc); |
1213 | return 1; | |
1214 | } | |
1215 | ||
1216 | rc = srpc_prepare_bulk(rpc); | |
5fd88337 | 1217 | if (rc) |
f64a6f3c | 1218 | break; |
d7e09d03 PT |
1219 | |
1220 | wi->swi_state = SWI_STATE_REQUEST_SUBMITTED; | |
1221 | rc = srpc_send_request(rpc); | |
1222 | break; | |
1223 | ||
1224 | case SWI_STATE_REQUEST_SUBMITTED: | |
4420cfd3 JS |
1225 | /* |
1226 | * CAVEAT EMPTOR: rqtev, rpyev, and bulkev may come in any | |
d7e09d03 | 1227 | * order; however, they're processed in a strict order: |
4420cfd3 JS |
1228 | * rqt, rpy, and bulk. |
1229 | */ | |
f64a6f3c HE |
1230 | if (!rpc->crpc_reqstev.ev_fired) |
1231 | break; | |
d7e09d03 PT |
1232 | |
1233 | rc = rpc->crpc_reqstev.ev_status; | |
5fd88337 | 1234 | if (rc) |
f64a6f3c | 1235 | break; |
d7e09d03 PT |
1236 | |
1237 | wi->swi_state = SWI_STATE_REQUEST_SENT; | |
1238 | /* perhaps more events, fall thru */ | |
1239 | case SWI_STATE_REQUEST_SENT: { | |
1240 | srpc_msg_type_t type = srpc_service2reply(rpc->crpc_service); | |
1241 | ||
f64a6f3c HE |
1242 | if (!rpc->crpc_replyev.ev_fired) |
1243 | break; | |
d7e09d03 PT |
1244 | |
1245 | rc = rpc->crpc_replyev.ev_status; | |
5fd88337 | 1246 | if (rc) |
f64a6f3c | 1247 | break; |
d7e09d03 PT |
1248 | |
1249 | srpc_unpack_msg_hdr(reply); | |
1250 | if (reply->msg_type != type || | |
1251 | (reply->msg_magic != SRPC_MSG_MAGIC && | |
1252 | reply->msg_magic != __swab32(SRPC_MSG_MAGIC))) { | |
2d00bd17 JP |
1253 | CWARN("Bad message from %s: type %u (%d expected), magic %u (%d expected).\n", |
1254 | libcfs_id2str(rpc->crpc_dest), | |
1255 | reply->msg_type, type, | |
1256 | reply->msg_magic, SRPC_MSG_MAGIC); | |
d7e09d03 PT |
1257 | rc = -EBADMSG; |
1258 | break; | |
1259 | } | |
1260 | ||
5fd88337 | 1261 | if (do_bulk && reply->msg_body.reply.status) { |
2d00bd17 JP |
1262 | CWARN("Remote error %d at %s, unlink bulk buffer in case peer didn't initiate bulk transfer\n", |
1263 | reply->msg_body.reply.status, | |
1264 | libcfs_id2str(rpc->crpc_dest)); | |
d7e09d03 PT |
1265 | LNetMDUnlink(rpc->crpc_bulk.bk_mdh); |
1266 | } | |
1267 | ||
1268 | wi->swi_state = SWI_STATE_REPLY_RECEIVED; | |
1269 | } | |
1270 | case SWI_STATE_REPLY_RECEIVED: | |
f64a6f3c HE |
1271 | if (do_bulk && !rpc->crpc_bulkev.ev_fired) |
1272 | break; | |
d7e09d03 PT |
1273 | |
1274 | rc = do_bulk ? rpc->crpc_bulkev.ev_status : 0; | |
1275 | ||
4420cfd3 JS |
1276 | /* |
1277 | * Bulk buffer was unlinked due to remote error. Clear error | |
d7e09d03 PT |
1278 | * since reply buffer still contains valid data. |
1279 | * NB rpc->crpc_done shouldn't look into bulk data in case of | |
4420cfd3 JS |
1280 | * remote error. |
1281 | */ | |
d7e09d03 | 1282 | if (do_bulk && rpc->crpc_bulkev.ev_lnet == LNET_EVENT_UNLINK && |
5fd88337 | 1283 | !rpc->crpc_status && reply->msg_body.reply.status) |
d7e09d03 PT |
1284 | rc = 0; |
1285 | ||
1286 | wi->swi_state = SWI_STATE_DONE; | |
1287 | srpc_client_rpc_done(rpc, rc); | |
1288 | return 1; | |
1289 | } | |
1290 | ||
5fd88337 | 1291 | if (rc) { |
d7e09d03 PT |
1292 | spin_lock(&rpc->crpc_lock); |
1293 | srpc_abort_rpc(rpc, rc); | |
1294 | spin_unlock(&rpc->crpc_lock); | |
1295 | } | |
1296 | ||
1297 | abort: | |
1298 | if (rpc->crpc_aborted) { | |
1299 | LNetMDUnlink(rpc->crpc_reqstmdh); | |
1300 | LNetMDUnlink(rpc->crpc_replymdh); | |
1301 | LNetMDUnlink(rpc->crpc_bulk.bk_mdh); | |
1302 | ||
1303 | if (!srpc_event_pending(rpc)) { | |
1304 | srpc_client_rpc_done(rpc, -EINTR); | |
1305 | return 1; | |
1306 | } | |
1307 | } | |
1308 | return 0; | |
1309 | } | |
1310 | ||
1311 | srpc_client_rpc_t * | |
a659df62 | 1312 | srpc_create_client_rpc(lnet_process_id_t peer, int service, |
c314c319 JS |
1313 | int nbulkiov, int bulklen, |
1314 | void (*rpc_done)(srpc_client_rpc_t *), | |
1315 | void (*rpc_fini)(srpc_client_rpc_t *), void *priv) | |
d7e09d03 PT |
1316 | { |
1317 | srpc_client_rpc_t *rpc; | |
1318 | ||
1319 | LIBCFS_ALLOC(rpc, offsetof(srpc_client_rpc_t, | |
1320 | crpc_bulk.bk_iovs[nbulkiov])); | |
06ace26e | 1321 | if (!rpc) |
d7e09d03 PT |
1322 | return NULL; |
1323 | ||
1324 | srpc_init_client_rpc(rpc, peer, service, nbulkiov, | |
1325 | bulklen, rpc_done, rpc_fini, priv); | |
1326 | return rpc; | |
1327 | } | |
1328 | ||
1329 | /* called with rpc->crpc_lock held */ | |
1330 | void | |
a659df62 | 1331 | srpc_abort_rpc(srpc_client_rpc_t *rpc, int why) |
d7e09d03 | 1332 | { |
5fd88337 | 1333 | LASSERT(why); |
d7e09d03 PT |
1334 | |
1335 | if (rpc->crpc_aborted || /* already aborted */ | |
b794d796 | 1336 | rpc->crpc_closed) /* callback imminent */ |
d7e09d03 PT |
1337 | return; |
1338 | ||
c314c319 JS |
1339 | CDEBUG(D_NET, "Aborting RPC: service %d, peer %s, state %s, why %d\n", |
1340 | rpc->crpc_service, libcfs_id2str(rpc->crpc_dest), | |
1341 | swi_state2str(rpc->crpc_wi.swi_state), why); | |
d7e09d03 PT |
1342 | |
1343 | rpc->crpc_aborted = 1; | |
ec436b9a | 1344 | rpc->crpc_status = why; |
d7e09d03 | 1345 | swi_schedule_workitem(&rpc->crpc_wi); |
d7e09d03 PT |
1346 | } |
1347 | ||
1348 | /* called with rpc->crpc_lock held */ | |
1349 | void | |
a659df62 | 1350 | srpc_post_rpc(srpc_client_rpc_t *rpc) |
d7e09d03 | 1351 | { |
a659df62 HE |
1352 | LASSERT(!rpc->crpc_aborted); |
1353 | LASSERT(srpc_data.rpc_state == SRPC_STATE_RUNNING); | |
d7e09d03 | 1354 | |
a659df62 | 1355 | CDEBUG(D_NET, "Posting RPC: peer %s, service %d, timeout %d\n", |
c314c319 JS |
1356 | libcfs_id2str(rpc->crpc_dest), rpc->crpc_service, |
1357 | rpc->crpc_timeout); | |
d7e09d03 PT |
1358 | |
1359 | srpc_add_client_rpc_timer(rpc); | |
1360 | swi_schedule_workitem(&rpc->crpc_wi); | |
d7e09d03 PT |
1361 | } |
1362 | ||
d7e09d03 PT |
1363 | int |
1364 | srpc_send_reply(struct srpc_server_rpc *rpc) | |
1365 | { | |
74d68011 MS |
1366 | srpc_event_t *ev = &rpc->srpc_ev; |
1367 | struct srpc_msg *msg = &rpc->srpc_replymsg; | |
1368 | struct srpc_buffer *buffer = rpc->srpc_reqstbuf; | |
1369 | struct srpc_service_cd *scd = rpc->srpc_scd; | |
1370 | struct srpc_service *sv = scd->scd_svc; | |
1371 | __u64 rpyid; | |
1372 | int rc; | |
d7e09d03 | 1373 | |
06ace26e | 1374 | LASSERT(buffer); |
d7e09d03 PT |
1375 | rpyid = buffer->buf_msg.msg_body.reqst.rpyid; |
1376 | ||
1377 | spin_lock(&scd->scd_lock); | |
1378 | ||
1379 | if (!sv->sv_shuttingdown && !srpc_serv_is_framework(sv)) { | |
4420cfd3 JS |
1380 | /* |
1381 | * Repost buffer before replying since test client | |
1382 | * might send me another RPC once it gets the reply | |
1383 | */ | |
5fd88337 | 1384 | if (srpc_service_post_buffer(scd, buffer)) |
d7e09d03 PT |
1385 | CWARN("Failed to repost %s buffer\n", sv->sv_name); |
1386 | rpc->srpc_reqstbuf = NULL; | |
1387 | } | |
1388 | ||
1389 | spin_unlock(&scd->scd_lock); | |
1390 | ||
1391 | ev->ev_fired = 0; | |
ec436b9a JS |
1392 | ev->ev_data = rpc; |
1393 | ev->ev_type = SRPC_REPLY_SENT; | |
d7e09d03 | 1394 | |
ec436b9a | 1395 | msg->msg_magic = SRPC_MSG_MAGIC; |
d7e09d03 | 1396 | msg->msg_version = SRPC_MSG_VERSION; |
ec436b9a | 1397 | msg->msg_type = srpc_service2reply(sv->sv_id); |
d7e09d03 PT |
1398 | |
1399 | rc = srpc_post_active_rdma(SRPC_RDMA_PORTAL, rpyid, msg, | |
1400 | sizeof(*msg), LNET_MD_OP_PUT, | |
1401 | rpc->srpc_peer, rpc->srpc_self, | |
1402 | &rpc->srpc_replymdh, ev); | |
5fd88337 | 1403 | if (rc) |
d7e09d03 PT |
1404 | ev->ev_fired = 1; /* no more event expected */ |
1405 | return rc; | |
1406 | } | |
1407 | ||
1408 | /* when in kernel always called with LNET_LOCK() held, and in thread context */ | |
8d94b6d2 | 1409 | static void |
d7e09d03 PT |
1410 | srpc_lnet_ev_handler(lnet_event_t *ev) |
1411 | { | |
74d68011 MS |
1412 | struct srpc_service_cd *scd; |
1413 | srpc_event_t *rpcev = ev->md.user_ptr; | |
d7e09d03 | 1414 | srpc_client_rpc_t *crpc; |
168c7a13 | 1415 | struct srpc_server_rpc *srpc; |
74d68011 MS |
1416 | srpc_buffer_t *buffer; |
1417 | srpc_service_t *sv; | |
1418 | srpc_msg_t *msg; | |
1419 | srpc_msg_type_t type; | |
d7e09d03 | 1420 | |
a659df62 | 1421 | LASSERT(!in_interrupt()); |
d7e09d03 | 1422 | |
5fd88337 | 1423 | if (ev->status) { |
c27d0a08 IH |
1424 | __u32 errors; |
1425 | ||
d7e09d03 | 1426 | spin_lock(&srpc_data.rpc_glock); |
c27d0a08 IH |
1427 | if (ev->status != -ECANCELED) /* cancellation is not error */ |
1428 | srpc_data.rpc_counters.errors++; | |
1429 | errors = srpc_data.rpc_counters.errors; | |
d7e09d03 | 1430 | spin_unlock(&srpc_data.rpc_glock); |
c27d0a08 IH |
1431 | |
1432 | CNETERR("LNet event status %d type %d, RPC errors %u\n", | |
1433 | ev->status, ev->type, errors); | |
d7e09d03 PT |
1434 | } |
1435 | ||
1436 | rpcev->ev_lnet = ev->type; | |
1437 | ||
1438 | switch (rpcev->ev_type) { | |
1439 | default: | |
1440 | CERROR("Unknown event: status %d, type %d, lnet %d\n", | |
1441 | rpcev->ev_status, rpcev->ev_type, rpcev->ev_lnet); | |
a659df62 | 1442 | LBUG(); |
d7e09d03 | 1443 | case SRPC_REQUEST_SENT: |
5fd88337 | 1444 | if (!ev->status && ev->type != LNET_EVENT_UNLINK) { |
d7e09d03 PT |
1445 | spin_lock(&srpc_data.rpc_glock); |
1446 | srpc_data.rpc_counters.rpcs_sent++; | |
1447 | spin_unlock(&srpc_data.rpc_glock); | |
1448 | } | |
1449 | case SRPC_REPLY_RCVD: | |
1450 | case SRPC_BULK_REQ_RCVD: | |
1451 | crpc = rpcev->ev_data; | |
1452 | ||
1453 | if (rpcev != &crpc->crpc_reqstev && | |
1454 | rpcev != &crpc->crpc_replyev && | |
1455 | rpcev != &crpc->crpc_bulkev) { | |
1456 | CERROR("rpcev %p, crpc %p, reqstev %p, replyev %p, bulkev %p\n", | |
1457 | rpcev, crpc, &crpc->crpc_reqstev, | |
1458 | &crpc->crpc_replyev, &crpc->crpc_bulkev); | |
1459 | CERROR("Bad event: status %d, type %d, lnet %d\n", | |
1460 | rpcev->ev_status, rpcev->ev_type, rpcev->ev_lnet); | |
a659df62 | 1461 | LBUG(); |
d7e09d03 PT |
1462 | } |
1463 | ||
1464 | spin_lock(&crpc->crpc_lock); | |
1465 | ||
5fd88337 | 1466 | LASSERT(!rpcev->ev_fired); |
ec436b9a | 1467 | rpcev->ev_fired = 1; |
d7e09d03 PT |
1468 | rpcev->ev_status = (ev->type == LNET_EVENT_UNLINK) ? |
1469 | -EINTR : ev->status; | |
1470 | swi_schedule_workitem(&crpc->crpc_wi); | |
1471 | ||
1472 | spin_unlock(&crpc->crpc_lock); | |
1473 | break; | |
1474 | ||
1475 | case SRPC_REQUEST_RCVD: | |
1476 | scd = rpcev->ev_data; | |
1477 | sv = scd->scd_svc; | |
1478 | ||
1479 | LASSERT(rpcev == &scd->scd_ev); | |
1480 | ||
1481 | spin_lock(&scd->scd_lock); | |
1482 | ||
a659df62 HE |
1483 | LASSERT(ev->unlinked); |
1484 | LASSERT(ev->type == LNET_EVENT_PUT || | |
c314c319 | 1485 | ev->type == LNET_EVENT_UNLINK); |
a659df62 | 1486 | LASSERT(ev->type != LNET_EVENT_UNLINK || |
c314c319 | 1487 | sv->sv_shuttingdown); |
d7e09d03 PT |
1488 | |
1489 | buffer = container_of(ev->md.start, srpc_buffer_t, buf_msg); | |
1490 | buffer->buf_peer = ev->initiator; | |
1491 | buffer->buf_self = ev->target.nid; | |
1492 | ||
1493 | LASSERT(scd->scd_buf_nposted > 0); | |
1494 | scd->scd_buf_nposted--; | |
1495 | ||
1496 | if (sv->sv_shuttingdown) { | |
4420cfd3 JS |
1497 | /* |
1498 | * Leave buffer on scd->scd_buf_nposted since | |
1499 | * srpc_finish_service needs to traverse it. | |
1500 | */ | |
d7e09d03 PT |
1501 | spin_unlock(&scd->scd_lock); |
1502 | break; | |
1503 | } | |
1504 | ||
5fd88337 | 1505 | if (scd->scd_buf_err_stamp && |
d9f79e6b | 1506 | scd->scd_buf_err_stamp < ktime_get_real_seconds()) { |
d7e09d03 PT |
1507 | /* re-enable adding buffer */ |
1508 | scd->scd_buf_err_stamp = 0; | |
1509 | scd->scd_buf_err = 0; | |
1510 | } | |
1511 | ||
5fd88337 JS |
1512 | if (!scd->scd_buf_err && /* adding buffer is enabled */ |
1513 | !scd->scd_buf_adjust && | |
d7e09d03 | 1514 | scd->scd_buf_nposted < scd->scd_buf_low) { |
0c575417 | 1515 | scd->scd_buf_adjust = max(scd->scd_buf_total / 2, |
d7e09d03 PT |
1516 | SFW_TEST_WI_MIN); |
1517 | swi_schedule_workitem(&scd->scd_buf_wi); | |
1518 | } | |
1519 | ||
1520 | list_del(&buffer->buf_list); /* from scd->scd_buf_posted */ | |
1521 | msg = &buffer->buf_msg; | |
1522 | type = srpc_service2request(sv->sv_id); | |
1523 | ||
5fd88337 | 1524 | if (ev->status || ev->mlength != sizeof(*msg) || |
d7e09d03 PT |
1525 | (msg->msg_type != type && |
1526 | msg->msg_type != __swab32(type)) || | |
1527 | (msg->msg_magic != SRPC_MSG_MAGIC && | |
1528 | msg->msg_magic != __swab32(SRPC_MSG_MAGIC))) { | |
2d00bd17 JP |
1529 | CERROR("Dropping RPC (%s) from %s: status %d mlength %d type %u magic %u.\n", |
1530 | sv->sv_name, libcfs_id2str(ev->initiator), | |
1531 | ev->status, ev->mlength, | |
1532 | msg->msg_type, msg->msg_magic); | |
d7e09d03 | 1533 | |
4420cfd3 JS |
1534 | /* |
1535 | * NB can't call srpc_service_recycle_buffer here since | |
d7e09d03 | 1536 | * it may call LNetM[DE]Attach. The invalid magic tells |
4420cfd3 JS |
1537 | * srpc_handle_rpc to drop this RPC |
1538 | */ | |
d7e09d03 PT |
1539 | msg->msg_magic = 0; |
1540 | } | |
1541 | ||
1542 | if (!list_empty(&scd->scd_rpc_free)) { | |
1543 | srpc = list_entry(scd->scd_rpc_free.next, | |
c314c319 JS |
1544 | struct srpc_server_rpc, |
1545 | srpc_list); | |
d7e09d03 PT |
1546 | list_del(&srpc->srpc_list); |
1547 | ||
1548 | srpc_init_server_rpc(srpc, scd, buffer); | |
1549 | list_add_tail(&srpc->srpc_list, | |
c314c319 | 1550 | &scd->scd_rpc_active); |
d7e09d03 PT |
1551 | swi_schedule_workitem(&srpc->srpc_wi); |
1552 | } else { | |
1553 | list_add_tail(&buffer->buf_list, | |
c314c319 | 1554 | &scd->scd_buf_blocked); |
d7e09d03 PT |
1555 | } |
1556 | ||
1557 | spin_unlock(&scd->scd_lock); | |
1558 | ||
1559 | spin_lock(&srpc_data.rpc_glock); | |
1560 | srpc_data.rpc_counters.rpcs_rcvd++; | |
1561 | spin_unlock(&srpc_data.rpc_glock); | |
1562 | break; | |
1563 | ||
1564 | case SRPC_BULK_GET_RPLD: | |
a659df62 | 1565 | LASSERT(ev->type == LNET_EVENT_SEND || |
c314c319 JS |
1566 | ev->type == LNET_EVENT_REPLY || |
1567 | ev->type == LNET_EVENT_UNLINK); | |
d7e09d03 PT |
1568 | |
1569 | if (!ev->unlinked) | |
1570 | break; /* wait for final event */ | |
1571 | ||
1572 | case SRPC_BULK_PUT_SENT: | |
5fd88337 | 1573 | if (!ev->status && ev->type != LNET_EVENT_UNLINK) { |
d7e09d03 PT |
1574 | spin_lock(&srpc_data.rpc_glock); |
1575 | ||
1576 | if (rpcev->ev_type == SRPC_BULK_GET_RPLD) | |
1577 | srpc_data.rpc_counters.bulk_get += ev->mlength; | |
1578 | else | |
1579 | srpc_data.rpc_counters.bulk_put += ev->mlength; | |
1580 | ||
1581 | spin_unlock(&srpc_data.rpc_glock); | |
1582 | } | |
1583 | case SRPC_REPLY_SENT: | |
1584 | srpc = rpcev->ev_data; | |
ec436b9a | 1585 | scd = srpc->srpc_scd; |
d7e09d03 PT |
1586 | |
1587 | LASSERT(rpcev == &srpc->srpc_ev); | |
1588 | ||
1589 | spin_lock(&scd->scd_lock); | |
1590 | ||
ec436b9a | 1591 | rpcev->ev_fired = 1; |
d7e09d03 PT |
1592 | rpcev->ev_status = (ev->type == LNET_EVENT_UNLINK) ? |
1593 | -EINTR : ev->status; | |
1594 | swi_schedule_workitem(&srpc->srpc_wi); | |
1595 | ||
1596 | spin_unlock(&scd->scd_lock); | |
1597 | break; | |
1598 | } | |
1599 | } | |
1600 | ||
d7e09d03 | 1601 | int |
a659df62 | 1602 | srpc_startup(void) |
d7e09d03 PT |
1603 | { |
1604 | int rc; | |
1605 | ||
1606 | memset(&srpc_data, 0, sizeof(struct smoketest_rpc)); | |
1607 | spin_lock_init(&srpc_data.rpc_glock); | |
1608 | ||
1609 | /* 1 second pause to avoid timestamp reuse */ | |
d3caf4d5 PT |
1610 | set_current_state(TASK_UNINTERRUPTIBLE); |
1611 | schedule_timeout(cfs_time_seconds(1)); | |
d9f79e6b | 1612 | srpc_data.rpc_matchbits = ((__u64)ktime_get_real_seconds()) << 48; |
d7e09d03 PT |
1613 | |
1614 | srpc_data.rpc_state = SRPC_STATE_NONE; | |
1615 | ||
fe7cb65d | 1616 | rc = LNetNIInit(LNET_PID_LUSTRE); |
d7e09d03 | 1617 | if (rc < 0) { |
a659df62 | 1618 | CERROR("LNetNIInit() has failed: %d\n", rc); |
d7e09d03 PT |
1619 | return rc; |
1620 | } | |
1621 | ||
1622 | srpc_data.rpc_state = SRPC_STATE_NI_INIT; | |
1623 | ||
1624 | LNetInvalidateHandle(&srpc_data.rpc_lnet_eq); | |
1625 | rc = LNetEQAlloc(0, srpc_lnet_ev_handler, &srpc_data.rpc_lnet_eq); | |
5fd88337 | 1626 | if (rc) { |
d7e09d03 PT |
1627 | CERROR("LNetEQAlloc() has failed: %d\n", rc); |
1628 | goto bail; | |
1629 | } | |
1630 | ||
1631 | rc = LNetSetLazyPortal(SRPC_FRAMEWORK_REQUEST_PORTAL); | |
5fd88337 | 1632 | LASSERT(!rc); |
d7e09d03 | 1633 | rc = LNetSetLazyPortal(SRPC_REQUEST_PORTAL); |
5fd88337 | 1634 | LASSERT(!rc); |
d7e09d03 PT |
1635 | |
1636 | srpc_data.rpc_state = SRPC_STATE_EQ_INIT; | |
1637 | ||
1638 | rc = stt_startup(); | |
1639 | ||
1640 | bail: | |
5fd88337 | 1641 | if (rc) |
d7e09d03 PT |
1642 | srpc_shutdown(); |
1643 | else | |
1644 | srpc_data.rpc_state = SRPC_STATE_RUNNING; | |
1645 | ||
1646 | return rc; | |
1647 | } | |
1648 | ||
1649 | void | |
a659df62 | 1650 | srpc_shutdown(void) |
d7e09d03 PT |
1651 | { |
1652 | int i; | |
1653 | int rc; | |
1654 | int state; | |
1655 | ||
1656 | state = srpc_data.rpc_state; | |
1657 | srpc_data.rpc_state = SRPC_STATE_STOPPING; | |
1658 | ||
1659 | switch (state) { | |
1660 | default: | |
a659df62 | 1661 | LBUG(); |
d7e09d03 PT |
1662 | case SRPC_STATE_RUNNING: |
1663 | spin_lock(&srpc_data.rpc_glock); | |
1664 | ||
1665 | for (i = 0; i <= SRPC_SERVICE_MAX_ID; i++) { | |
1666 | srpc_service_t *sv = srpc_data.rpc_services[i]; | |
1667 | ||
06ace26e | 1668 | LASSERTF(!sv, "service not empty: id %d, name %s\n", |
c314c319 | 1669 | i, sv->sv_name); |
d7e09d03 PT |
1670 | } |
1671 | ||
1672 | spin_unlock(&srpc_data.rpc_glock); | |
1673 | ||
1674 | stt_shutdown(); | |
1675 | ||
1676 | case SRPC_STATE_EQ_INIT: | |
1677 | rc = LNetClearLazyPortal(SRPC_FRAMEWORK_REQUEST_PORTAL); | |
1678 | rc = LNetClearLazyPortal(SRPC_REQUEST_PORTAL); | |
5fd88337 | 1679 | LASSERT(!rc); |
d7e09d03 | 1680 | rc = LNetEQFree(srpc_data.rpc_lnet_eq); |
5fd88337 | 1681 | LASSERT(!rc); /* the EQ should have no user by now */ |
d7e09d03 PT |
1682 | |
1683 | case SRPC_STATE_NI_INIT: | |
1684 | LNetNIFini(); | |
1685 | } | |
d7e09d03 | 1686 | } |