Commit | Line | Data |
---|---|---|
d7e09d03 PT |
1 | /* |
2 | * GPL HEADER START | |
3 | * | |
4 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 only, | |
8 | * as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | * General Public License version 2 for more details (a copy is included | |
14 | * in the LICENSE file that accompanied this code). | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * version 2 along with this program; If not, see | |
18 | * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf | |
19 | * | |
20 | * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, | |
21 | * CA 95054 USA or visit www.sun.com if you need additional information or | |
22 | * have any questions. | |
23 | * | |
24 | * GPL HEADER END | |
25 | */ | |
26 | /* | |
27 | * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. | |
28 | * Use is subject to license terms. | |
29 | * | |
30 | * Copyright (c) 2012, Intel Corporation. | |
31 | */ | |
32 | /* | |
33 | * This file is part of Lustre, http://www.lustre.org/ | |
34 | * Lustre is a trademark of Sun Microsystems, Inc. | |
35 | * | |
36 | * lnet/lnet/lib-md.c | |
37 | * | |
38 | * Memory Descriptor management routines | |
39 | */ | |
40 | ||
41 | #define DEBUG_SUBSYSTEM S_LNET | |
42 | ||
9fdaf8c0 | 43 | #include "../../include/linux/lnet/lib-lnet.h" |
d7e09d03 PT |
44 | |
45 | /* must be called with lnet_res_lock held */ | |
46 | void | |
47 | lnet_md_unlink(lnet_libmd_t *md) | |
48 | { | |
49 | if ((md->md_flags & LNET_MD_FLAG_ZOMBIE) == 0) { | |
50 | /* first unlink attempt... */ | |
51 | lnet_me_t *me = md->md_me; | |
52 | ||
53 | md->md_flags |= LNET_MD_FLAG_ZOMBIE; | |
54 | ||
55 | /* Disassociate from ME (if any), and unlink it if it was created | |
56 | * with LNET_UNLINK */ | |
57 | if (me != NULL) { | |
58 | /* detach MD from portal */ | |
59 | lnet_ptl_detach_md(me, md); | |
60 | if (me->me_unlink == LNET_UNLINK) | |
61 | lnet_me_unlink(me); | |
62 | } | |
63 | ||
64 | /* ensure all future handle lookups fail */ | |
65 | lnet_res_lh_invalidate(&md->md_lh); | |
66 | } | |
67 | ||
68 | if (md->md_refcount != 0) { | |
69 | CDEBUG(D_NET, "Queueing unlink of md %p\n", md); | |
70 | return; | |
71 | } | |
72 | ||
73 | CDEBUG(D_NET, "Unlinking md %p\n", md); | |
74 | ||
75 | if (md->md_eq != NULL) { | |
76 | int cpt = lnet_cpt_of_cookie(md->md_lh.lh_cookie); | |
77 | ||
78 | LASSERT(*md->md_eq->eq_refs[cpt] > 0); | |
79 | (*md->md_eq->eq_refs[cpt])--; | |
80 | } | |
81 | ||
82 | LASSERT(!list_empty(&md->md_list)); | |
83 | list_del_init(&md->md_list); | |
84 | lnet_md_free_locked(md); | |
85 | } | |
86 | ||
87 | static int | |
88 | lnet_md_build(lnet_libmd_t *lmd, lnet_md_t *umd, int unlink) | |
89 | { | |
90 | int i; | |
91 | unsigned int niov; | |
92 | int total_length = 0; | |
93 | ||
94 | lmd->md_me = NULL; | |
95 | lmd->md_start = umd->start; | |
96 | lmd->md_offset = 0; | |
97 | lmd->md_max_size = umd->max_size; | |
98 | lmd->md_options = umd->options; | |
99 | lmd->md_user_ptr = umd->user_ptr; | |
100 | lmd->md_eq = NULL; | |
101 | lmd->md_threshold = umd->threshold; | |
102 | lmd->md_refcount = 0; | |
103 | lmd->md_flags = (unlink == LNET_UNLINK) ? LNET_MD_FLAG_AUTO_UNLINK : 0; | |
104 | ||
105 | if ((umd->options & LNET_MD_IOVEC) != 0) { | |
106 | ||
107 | if ((umd->options & LNET_MD_KIOV) != 0) /* Can't specify both */ | |
108 | return -EINVAL; | |
109 | ||
110 | lmd->md_niov = niov = umd->length; | |
111 | memcpy(lmd->md_iov.iov, umd->start, | |
112 | niov * sizeof (lmd->md_iov.iov[0])); | |
113 | ||
114 | for (i = 0; i < (int)niov; i++) { | |
115 | /* We take the base address on trust */ | |
116 | if (lmd->md_iov.iov[i].iov_len <= 0) /* invalid length */ | |
117 | return -EINVAL; | |
118 | ||
119 | total_length += lmd->md_iov.iov[i].iov_len; | |
120 | } | |
121 | ||
122 | lmd->md_length = total_length; | |
123 | ||
124 | if ((umd->options & LNET_MD_MAX_SIZE) != 0 && /* max size used */ | |
125 | (umd->max_size < 0 || | |
126 | umd->max_size > total_length)) // illegal max_size | |
127 | return -EINVAL; | |
128 | ||
129 | } else if ((umd->options & LNET_MD_KIOV) != 0) { | |
130 | lmd->md_niov = niov = umd->length; | |
131 | memcpy(lmd->md_iov.kiov, umd->start, | |
132 | niov * sizeof (lmd->md_iov.kiov[0])); | |
133 | ||
134 | for (i = 0; i < (int)niov; i++) { | |
135 | /* We take the page pointer on trust */ | |
136 | if (lmd->md_iov.kiov[i].kiov_offset + | |
137 | lmd->md_iov.kiov[i].kiov_len > PAGE_CACHE_SIZE ) | |
138 | return -EINVAL; /* invalid length */ | |
139 | ||
140 | total_length += lmd->md_iov.kiov[i].kiov_len; | |
141 | } | |
142 | ||
143 | lmd->md_length = total_length; | |
144 | ||
145 | if ((umd->options & LNET_MD_MAX_SIZE) != 0 && /* max size used */ | |
146 | (umd->max_size < 0 || | |
147 | umd->max_size > total_length)) // illegal max_size | |
148 | return -EINVAL; | |
149 | } else { /* contiguous */ | |
150 | lmd->md_length = umd->length; | |
151 | lmd->md_niov = niov = 1; | |
152 | lmd->md_iov.iov[0].iov_base = umd->start; | |
153 | lmd->md_iov.iov[0].iov_len = umd->length; | |
154 | ||
155 | if ((umd->options & LNET_MD_MAX_SIZE) != 0 && /* max size used */ | |
156 | (umd->max_size < 0 || | |
157 | umd->max_size > (int)umd->length)) // illegal max_size | |
158 | return -EINVAL; | |
159 | } | |
160 | ||
161 | return 0; | |
162 | } | |
163 | ||
164 | /* must be called with resource lock held */ | |
165 | static int | |
166 | lnet_md_link(lnet_libmd_t *md, lnet_handle_eq_t eq_handle, int cpt) | |
167 | { | |
168 | struct lnet_res_container *container = the_lnet.ln_md_containers[cpt]; | |
169 | ||
170 | /* NB we are passed an allocated, but inactive md. | |
171 | * if we return success, caller may lnet_md_unlink() it. | |
172 | * otherwise caller may only lnet_md_free() it. | |
173 | */ | |
174 | /* This implementation doesn't know how to create START events or | |
175 | * disable END events. Best to LASSERT our caller is compliant so | |
176 | * we find out quickly... */ | |
177 | /* TODO - reevaluate what should be here in light of | |
178 | * the removal of the start and end events | |
179 | * maybe there we shouldn't even allow LNET_EQ_NONE!) | |
180 | * LASSERT (eq == NULL); | |
181 | */ | |
182 | if (!LNetHandleIsInvalid(eq_handle)) { | |
183 | md->md_eq = lnet_handle2eq(&eq_handle); | |
184 | ||
185 | if (md->md_eq == NULL) | |
186 | return -ENOENT; | |
187 | ||
188 | (*md->md_eq->eq_refs[cpt])++; | |
189 | } | |
190 | ||
191 | lnet_res_lh_initialize(container, &md->md_lh); | |
192 | ||
193 | LASSERT(list_empty(&md->md_list)); | |
194 | list_add(&md->md_list, &container->rec_active); | |
195 | ||
196 | return 0; | |
197 | } | |
198 | ||
199 | /* must be called with lnet_res_lock held */ | |
200 | void | |
201 | lnet_md_deconstruct(lnet_libmd_t *lmd, lnet_md_t *umd) | |
202 | { | |
203 | /* NB this doesn't copy out all the iov entries so when a | |
204 | * discontiguous MD is copied out, the target gets to know the | |
205 | * original iov pointer (in start) and the number of entries it had | |
206 | * and that's all. | |
207 | */ | |
208 | umd->start = lmd->md_start; | |
209 | umd->length = ((lmd->md_options & (LNET_MD_IOVEC | LNET_MD_KIOV)) == 0) ? | |
210 | lmd->md_length : lmd->md_niov; | |
211 | umd->threshold = lmd->md_threshold; | |
212 | umd->max_size = lmd->md_max_size; | |
213 | umd->options = lmd->md_options; | |
214 | umd->user_ptr = lmd->md_user_ptr; | |
215 | lnet_eq2handle(&umd->eq_handle, lmd->md_eq); | |
216 | } | |
217 | ||
218 | int | |
219 | lnet_md_validate(lnet_md_t *umd) | |
220 | { | |
221 | if (umd->start == NULL && umd->length != 0) { | |
222 | CERROR("MD start pointer can not be NULL with length %u\n", | |
223 | umd->length); | |
224 | return -EINVAL; | |
225 | } | |
226 | ||
227 | if ((umd->options & (LNET_MD_KIOV | LNET_MD_IOVEC)) != 0 && | |
228 | umd->length > LNET_MAX_IOV) { | |
229 | CERROR("Invalid option: too many fragments %u, %d max\n", | |
230 | umd->length, LNET_MAX_IOV); | |
231 | return -EINVAL; | |
232 | } | |
233 | ||
234 | return 0; | |
235 | } | |
236 | ||
237 | /** | |
238 | * Create a memory descriptor and attach it to a ME | |
239 | * | |
240 | * \param meh A handle for a ME to associate the new MD with. | |
241 | * \param umd Provides initial values for the user-visible parts of a MD. | |
242 | * Other than its use for initialization, there is no linkage between this | |
243 | * structure and the MD maintained by the LNet. | |
244 | * \param unlink A flag to indicate whether the MD is automatically unlinked | |
245 | * when it becomes inactive, either because the operation threshold drops to | |
246 | * zero or because the available memory becomes less than \a umd.max_size. | |
247 | * (Note that the check for unlinking a MD only occurs after the completion | |
248 | * of a successful operation on the MD.) The value LNET_UNLINK enables auto | |
249 | * unlinking; the value LNET_RETAIN disables it. | |
250 | * \param handle On successful returns, a handle to the newly created MD is | |
251 | * saved here. This handle can be used later in LNetMDUnlink(). | |
252 | * | |
253 | * \retval 0 On success. | |
254 | * \retval -EINVAL If \a umd is not valid. | |
255 | * \retval -ENOMEM If new MD cannot be allocated. | |
256 | * \retval -ENOENT Either \a meh or \a umd.eq_handle does not point to a | |
257 | * valid object. Note that it's OK to supply a NULL \a umd.eq_handle by | |
258 | * calling LNetInvalidateHandle() on it. | |
259 | * \retval -EBUSY If the ME pointed to by \a meh is already associated with | |
260 | * a MD. | |
261 | */ | |
262 | int | |
263 | LNetMDAttach(lnet_handle_me_t meh, lnet_md_t umd, | |
264 | lnet_unlink_t unlink, lnet_handle_md_t *handle) | |
265 | { | |
266 | LIST_HEAD (matches); | |
267 | LIST_HEAD (drops); | |
268 | struct lnet_me *me; | |
269 | struct lnet_libmd *md; | |
270 | int cpt; | |
271 | int rc; | |
272 | ||
273 | LASSERT (the_lnet.ln_init); | |
274 | LASSERT (the_lnet.ln_refcount > 0); | |
275 | ||
276 | if (lnet_md_validate(&umd) != 0) | |
277 | return -EINVAL; | |
278 | ||
279 | if ((umd.options & (LNET_MD_OP_GET | LNET_MD_OP_PUT)) == 0) { | |
280 | CERROR("Invalid option: no MD_OP set\n"); | |
281 | return -EINVAL; | |
282 | } | |
283 | ||
284 | md = lnet_md_alloc(&umd); | |
285 | if (md == NULL) | |
286 | return -ENOMEM; | |
287 | ||
288 | rc = lnet_md_build(md, &umd, unlink); | |
289 | cpt = lnet_cpt_of_cookie(meh.cookie); | |
290 | ||
291 | lnet_res_lock(cpt); | |
292 | if (rc != 0) | |
293 | goto failed; | |
294 | ||
295 | me = lnet_handle2me(&meh); | |
296 | if (me == NULL) | |
297 | rc = -ENOENT; | |
298 | else if (me->me_md != NULL) | |
299 | rc = -EBUSY; | |
300 | else | |
301 | rc = lnet_md_link(md, umd.eq_handle, cpt); | |
302 | ||
303 | if (rc != 0) | |
304 | goto failed; | |
305 | ||
306 | /* attach this MD to portal of ME and check if it matches any | |
307 | * blocked msgs on this portal */ | |
308 | lnet_ptl_attach_md(me, md, &matches, &drops); | |
309 | ||
310 | lnet_md2handle(handle, md); | |
311 | ||
312 | lnet_res_unlock(cpt); | |
313 | ||
314 | lnet_drop_delayed_msg_list(&drops, "Bad match"); | |
315 | lnet_recv_delayed_msg_list(&matches); | |
316 | ||
317 | return 0; | |
318 | ||
319 | failed: | |
320 | lnet_md_free_locked(md); | |
321 | ||
322 | lnet_res_unlock(cpt); | |
323 | return rc; | |
324 | } | |
325 | EXPORT_SYMBOL(LNetMDAttach); | |
326 | ||
327 | /** | |
328 | * Create a "free floating" memory descriptor - a MD that is not associated | |
329 | * with a ME. Such MDs are usually used in LNetPut() and LNetGet() operations. | |
330 | * | |
331 | * \param umd,unlink See the discussion for LNetMDAttach(). | |
332 | * \param handle On successful returns, a handle to the newly created MD is | |
333 | * saved here. This handle can be used later in LNetMDUnlink(), LNetPut(), | |
334 | * and LNetGet() operations. | |
335 | * | |
336 | * \retval 0 On success. | |
337 | * \retval -EINVAL If \a umd is not valid. | |
338 | * \retval -ENOMEM If new MD cannot be allocated. | |
339 | * \retval -ENOENT \a umd.eq_handle does not point to a valid EQ. Note that | |
340 | * it's OK to supply a NULL \a umd.eq_handle by calling | |
341 | * LNetInvalidateHandle() on it. | |
342 | */ | |
343 | int | |
344 | LNetMDBind(lnet_md_t umd, lnet_unlink_t unlink, lnet_handle_md_t *handle) | |
345 | { | |
346 | lnet_libmd_t *md; | |
347 | int cpt; | |
348 | int rc; | |
349 | ||
350 | LASSERT (the_lnet.ln_init); | |
351 | LASSERT (the_lnet.ln_refcount > 0); | |
352 | ||
353 | if (lnet_md_validate(&umd) != 0) | |
354 | return -EINVAL; | |
355 | ||
356 | if ((umd.options & (LNET_MD_OP_GET | LNET_MD_OP_PUT)) != 0) { | |
357 | CERROR("Invalid option: GET|PUT illegal on active MDs\n"); | |
358 | return -EINVAL; | |
359 | } | |
360 | ||
361 | md = lnet_md_alloc(&umd); | |
362 | if (md == NULL) | |
363 | return -ENOMEM; | |
364 | ||
365 | rc = lnet_md_build(md, &umd, unlink); | |
366 | ||
367 | cpt = lnet_res_lock_current(); | |
368 | if (rc != 0) | |
369 | goto failed; | |
370 | ||
371 | rc = lnet_md_link(md, umd.eq_handle, cpt); | |
372 | if (rc != 0) | |
373 | goto failed; | |
374 | ||
375 | lnet_md2handle(handle, md); | |
376 | ||
377 | lnet_res_unlock(cpt); | |
378 | return 0; | |
379 | ||
380 | failed: | |
381 | lnet_md_free_locked(md); | |
382 | ||
383 | lnet_res_unlock(cpt); | |
384 | return rc; | |
385 | } | |
386 | EXPORT_SYMBOL(LNetMDBind); | |
387 | ||
388 | /** | |
389 | * Unlink the memory descriptor from any ME it may be linked to and release | |
dee2857e IH |
390 | * the internal resources associated with it. As a result, active messages |
391 | * associated with the MD may get aborted. | |
d7e09d03 PT |
392 | * |
393 | * This function does not free the memory region associated with the MD; | |
394 | * i.e., the memory the user allocated for this MD. If the ME associated with | |
395 | * this MD is not NULL and was created with auto unlink enabled, the ME is | |
396 | * unlinked as well (see LNetMEAttach()). | |
397 | * | |
398 | * Explicitly unlinking a MD via this function call has the same behavior as | |
399 | * a MD that has been automatically unlinked, except that no LNET_EVENT_UNLINK | |
400 | * is generated in the latter case. | |
401 | * | |
402 | * An unlinked event can be reported in two ways: | |
403 | * - If there's no pending operations on the MD, it's unlinked immediately | |
404 | * and an LNET_EVENT_UNLINK event is logged before this function returns. | |
405 | * - Otherwise, the MD is only marked for deletion when this function | |
406 | * returns, and the unlinked event will be piggybacked on the event of | |
407 | * the completion of the last operation by setting the unlinked field of | |
408 | * the event. No dedicated LNET_EVENT_UNLINK event is generated. | |
409 | * | |
410 | * Note that in both cases the unlinked field of the event is always set; no | |
411 | * more event will happen on the MD after such an event is logged. | |
412 | * | |
413 | * \param mdh A handle for the MD to be unlinked. | |
414 | * | |
415 | * \retval 0 On success. | |
416 | * \retval -ENOENT If \a mdh does not point to a valid MD object. | |
417 | */ | |
418 | int | |
419 | LNetMDUnlink (lnet_handle_md_t mdh) | |
420 | { | |
421 | lnet_event_t ev; | |
422 | lnet_libmd_t *md; | |
423 | int cpt; | |
424 | ||
425 | LASSERT(the_lnet.ln_init); | |
426 | LASSERT(the_lnet.ln_refcount > 0); | |
427 | ||
428 | cpt = lnet_cpt_of_cookie(mdh.cookie); | |
429 | lnet_res_lock(cpt); | |
430 | ||
431 | md = lnet_handle2md(&mdh); | |
432 | if (md == NULL) { | |
433 | lnet_res_unlock(cpt); | |
434 | return -ENOENT; | |
435 | } | |
436 | ||
dee2857e | 437 | md->md_flags |= LNET_MD_FLAG_ABORTED; |
d7e09d03 | 438 | /* If the MD is busy, lnet_md_unlink just marks it for deletion, and |
dee2857e | 439 | * when the LND is done, the completion event flags that the MD was |
d7e09d03 | 440 | * unlinked. Otherwise, we enqueue an event now... */ |
dee2857e | 441 | if (md->md_eq != NULL && md->md_refcount == 0) { |
d7e09d03 PT |
442 | lnet_build_unlink_event(md, &ev); |
443 | lnet_eq_enqueue_event(md->md_eq, &ev); | |
444 | } | |
445 | ||
446 | lnet_md_unlink(md); | |
447 | ||
448 | lnet_res_unlock(cpt); | |
449 | return 0; | |
450 | } | |
451 | EXPORT_SYMBOL(LNetMDUnlink); |