Commit | Line | Data |
---|---|---|
d7e09d03 PT |
1 | /* |
2 | * GPL HEADER START | |
3 | * | |
4 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 only, | |
8 | * as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | * General Public License version 2 for more details (a copy is included | |
14 | * in the LICENSE file that accompanied this code). | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * version 2 along with this program; If not, see | |
18 | * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf | |
19 | * | |
20 | * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, | |
21 | * CA 95054 USA or visit www.sun.com if you need additional information or | |
22 | * have any questions. | |
23 | * | |
24 | * GPL HEADER END | |
25 | */ | |
26 | /* | |
27 | * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. | |
28 | * Use is subject to license terms. | |
29 | * | |
30 | * Copyright (c) 2011, 2012, Intel Corporation. | |
31 | */ | |
32 | /* | |
33 | * This file is part of Lustre, http://www.lustre.org/ | |
34 | * Lustre is a trademark of Sun Microsystems, Inc. | |
35 | * | |
36 | * Client Extent Lock. | |
37 | * | |
38 | * Author: Nikita Danilov <nikita.danilov@sun.com> | |
39 | */ | |
40 | ||
41 | #define DEBUG_SUBSYSTEM S_CLASS | |
42 | ||
610f7377 GKH |
43 | #include "../include/obd_class.h" |
44 | #include "../include/obd_support.h" | |
45 | #include "../include/lustre_fid.h" | |
d7e09d03 | 46 | #include <linux/list.h> |
610f7377 | 47 | #include "../include/cl_object.h" |
d7e09d03 PT |
48 | #include "cl_internal.h" |
49 | ||
50 | /** Lock class of cl_lock::cll_guard */ | |
51 | static struct lock_class_key cl_lock_guard_class; | |
52 | static struct kmem_cache *cl_lock_kmem; | |
53 | ||
54 | static struct lu_kmem_descr cl_lock_caches[] = { | |
55 | { | |
56 | .ckd_cache = &cl_lock_kmem, | |
57 | .ckd_name = "cl_lock_kmem", | |
58 | .ckd_size = sizeof (struct cl_lock) | |
59 | }, | |
60 | { | |
61 | .ckd_cache = NULL | |
62 | } | |
63 | }; | |
64 | ||
65 | #define CS_LOCK_INC(o, item) | |
66 | #define CS_LOCK_DEC(o, item) | |
67 | #define CS_LOCKSTATE_INC(o, state) | |
68 | #define CS_LOCKSTATE_DEC(o, state) | |
69 | ||
70 | /** | |
71 | * Basic lock invariant that is maintained at all times. Caller either has a | |
72 | * reference to \a lock, or somehow assures that \a lock cannot be freed. | |
73 | * | |
74 | * \see cl_lock_invariant() | |
75 | */ | |
76 | static int cl_lock_invariant_trusted(const struct lu_env *env, | |
77 | const struct cl_lock *lock) | |
78 | { | |
79 | return ergo(lock->cll_state == CLS_FREEING, lock->cll_holds == 0) && | |
80 | atomic_read(&lock->cll_ref) >= lock->cll_holds && | |
81 | lock->cll_holds >= lock->cll_users && | |
82 | lock->cll_holds >= 0 && | |
83 | lock->cll_users >= 0 && | |
84 | lock->cll_depth >= 0; | |
85 | } | |
86 | ||
87 | /** | |
88 | * Stronger lock invariant, checking that caller has a reference on a lock. | |
89 | * | |
90 | * \see cl_lock_invariant_trusted() | |
91 | */ | |
92 | static int cl_lock_invariant(const struct lu_env *env, | |
93 | const struct cl_lock *lock) | |
94 | { | |
95 | int result; | |
96 | ||
97 | result = atomic_read(&lock->cll_ref) > 0 && | |
98 | cl_lock_invariant_trusted(env, lock); | |
cce3c2da | 99 | if (!result && env) |
d7e09d03 PT |
100 | CL_LOCK_DEBUG(D_ERROR, env, lock, "invariant broken"); |
101 | return result; | |
102 | } | |
103 | ||
104 | /** | |
105 | * Returns lock "nesting": 0 for a top-lock and 1 for a sub-lock. | |
106 | */ | |
107 | static enum clt_nesting_level cl_lock_nesting(const struct cl_lock *lock) | |
108 | { | |
109 | return cl_object_header(lock->cll_descr.cld_obj)->coh_nesting; | |
110 | } | |
111 | ||
112 | /** | |
113 | * Returns a set of counters for this lock, depending on a lock nesting. | |
114 | */ | |
115 | static struct cl_thread_counters *cl_lock_counters(const struct lu_env *env, | |
116 | const struct cl_lock *lock) | |
117 | { | |
118 | struct cl_thread_info *info; | |
119 | enum clt_nesting_level nesting; | |
120 | ||
121 | info = cl_env_info(env); | |
122 | nesting = cl_lock_nesting(lock); | |
123 | LASSERT(nesting < ARRAY_SIZE(info->clt_counters)); | |
124 | return &info->clt_counters[nesting]; | |
125 | } | |
126 | ||
127 | static void cl_lock_trace0(int level, const struct lu_env *env, | |
128 | const char *prefix, const struct cl_lock *lock, | |
129 | const char *func, const int line) | |
130 | { | |
131 | struct cl_object_header *h = cl_object_header(lock->cll_descr.cld_obj); | |
50ffcb7e | 132 | |
2d00bd17 | 133 | CDEBUG(level, "%s: %p@(%d %p %d %d %d %d %d %lx)(%p/%d/%d) at %s():%d\n", |
d7e09d03 PT |
134 | prefix, lock, atomic_read(&lock->cll_ref), |
135 | lock->cll_guarder, lock->cll_depth, | |
136 | lock->cll_state, lock->cll_error, lock->cll_holds, | |
137 | lock->cll_users, lock->cll_flags, | |
138 | env, h->coh_nesting, cl_lock_nr_mutexed(env), | |
139 | func, line); | |
140 | } | |
c9f6bb96 | 141 | |
d7e09d03 | 142 | #define cl_lock_trace(level, env, prefix, lock) \ |
f9bd9c1a | 143 | cl_lock_trace0(level, env, prefix, lock, __func__, __LINE__) |
d7e09d03 PT |
144 | |
145 | #define RETIP ((unsigned long)__builtin_return_address(0)) | |
146 | ||
147 | #ifdef CONFIG_LOCKDEP | |
148 | static struct lock_class_key cl_lock_key; | |
149 | ||
150 | static void cl_lock_lockdep_init(struct cl_lock *lock) | |
151 | { | |
152 | lockdep_set_class_and_name(lock, &cl_lock_key, "EXT"); | |
153 | } | |
154 | ||
155 | static void cl_lock_lockdep_acquire(const struct lu_env *env, | |
156 | struct cl_lock *lock, __u32 enqflags) | |
157 | { | |
158 | cl_lock_counters(env, lock)->ctc_nr_locks_acquired++; | |
159 | lock_map_acquire(&lock->dep_map); | |
160 | } | |
161 | ||
162 | static void cl_lock_lockdep_release(const struct lu_env *env, | |
163 | struct cl_lock *lock) | |
164 | { | |
165 | cl_lock_counters(env, lock)->ctc_nr_locks_acquired--; | |
166 | lock_release(&lock->dep_map, 0, RETIP); | |
167 | } | |
168 | ||
169 | #else /* !CONFIG_LOCKDEP */ | |
170 | ||
171 | static void cl_lock_lockdep_init(struct cl_lock *lock) | |
172 | {} | |
173 | static void cl_lock_lockdep_acquire(const struct lu_env *env, | |
174 | struct cl_lock *lock, __u32 enqflags) | |
175 | {} | |
176 | static void cl_lock_lockdep_release(const struct lu_env *env, | |
177 | struct cl_lock *lock) | |
178 | {} | |
179 | ||
180 | #endif /* !CONFIG_LOCKDEP */ | |
181 | ||
182 | /** | |
183 | * Adds lock slice to the compound lock. | |
184 | * | |
185 | * This is called by cl_object_operations::coo_lock_init() methods to add a | |
186 | * per-layer state to the lock. New state is added at the end of | |
187 | * cl_lock::cll_layers list, that is, it is at the bottom of the stack. | |
188 | * | |
189 | * \see cl_req_slice_add(), cl_page_slice_add(), cl_io_slice_add() | |
190 | */ | |
191 | void cl_lock_slice_add(struct cl_lock *lock, struct cl_lock_slice *slice, | |
192 | struct cl_object *obj, | |
193 | const struct cl_lock_operations *ops) | |
194 | { | |
d7e09d03 PT |
195 | slice->cls_lock = lock; |
196 | list_add_tail(&slice->cls_linkage, &lock->cll_layers); | |
197 | slice->cls_obj = obj; | |
198 | slice->cls_ops = ops; | |
d7e09d03 PT |
199 | } |
200 | EXPORT_SYMBOL(cl_lock_slice_add); | |
201 | ||
202 | /** | |
203 | * Returns true iff a lock with the mode \a has provides at least the same | |
204 | * guarantees as a lock with the mode \a need. | |
205 | */ | |
206 | int cl_lock_mode_match(enum cl_lock_mode has, enum cl_lock_mode need) | |
207 | { | |
208 | LINVRNT(need == CLM_READ || need == CLM_WRITE || | |
209 | need == CLM_PHANTOM || need == CLM_GROUP); | |
210 | LINVRNT(has == CLM_READ || has == CLM_WRITE || | |
211 | has == CLM_PHANTOM || has == CLM_GROUP); | |
212 | CLASSERT(CLM_PHANTOM < CLM_READ); | |
213 | CLASSERT(CLM_READ < CLM_WRITE); | |
214 | CLASSERT(CLM_WRITE < CLM_GROUP); | |
215 | ||
216 | if (has != CLM_GROUP) | |
217 | return need <= has; | |
218 | else | |
219 | return need == has; | |
220 | } | |
221 | EXPORT_SYMBOL(cl_lock_mode_match); | |
222 | ||
223 | /** | |
224 | * Returns true iff extent portions of lock descriptions match. | |
225 | */ | |
226 | int cl_lock_ext_match(const struct cl_lock_descr *has, | |
227 | const struct cl_lock_descr *need) | |
228 | { | |
229 | return | |
230 | has->cld_start <= need->cld_start && | |
231 | has->cld_end >= need->cld_end && | |
232 | cl_lock_mode_match(has->cld_mode, need->cld_mode) && | |
233 | (has->cld_mode != CLM_GROUP || has->cld_gid == need->cld_gid); | |
234 | } | |
235 | EXPORT_SYMBOL(cl_lock_ext_match); | |
236 | ||
237 | /** | |
238 | * Returns true iff a lock with the description \a has provides at least the | |
239 | * same guarantees as a lock with the description \a need. | |
240 | */ | |
241 | int cl_lock_descr_match(const struct cl_lock_descr *has, | |
242 | const struct cl_lock_descr *need) | |
243 | { | |
244 | return | |
245 | cl_object_same(has->cld_obj, need->cld_obj) && | |
246 | cl_lock_ext_match(has, need); | |
247 | } | |
248 | EXPORT_SYMBOL(cl_lock_descr_match); | |
249 | ||
250 | static void cl_lock_free(const struct lu_env *env, struct cl_lock *lock) | |
251 | { | |
252 | struct cl_object *obj = lock->cll_descr.cld_obj; | |
253 | ||
254 | LINVRNT(!cl_lock_is_mutexed(lock)); | |
255 | ||
d7e09d03 PT |
256 | cl_lock_trace(D_DLMTRACE, env, "free lock", lock); |
257 | might_sleep(); | |
258 | while (!list_empty(&lock->cll_layers)) { | |
259 | struct cl_lock_slice *slice; | |
260 | ||
261 | slice = list_entry(lock->cll_layers.next, | |
262 | struct cl_lock_slice, cls_linkage); | |
263 | list_del_init(lock->cll_layers.next); | |
264 | slice->cls_ops->clo_fini(env, slice); | |
265 | } | |
266 | CS_LOCK_DEC(obj, total); | |
267 | CS_LOCKSTATE_DEC(obj, lock->cll_state); | |
631abc6e | 268 | lu_object_ref_del_at(&obj->co_lu, &lock->cll_obj_ref, "cl_lock", lock); |
d7e09d03 PT |
269 | cl_object_put(env, obj); |
270 | lu_ref_fini(&lock->cll_reference); | |
271 | lu_ref_fini(&lock->cll_holders); | |
272 | mutex_destroy(&lock->cll_guard); | |
50d30362 | 273 | kmem_cache_free(cl_lock_kmem, lock); |
d7e09d03 PT |
274 | } |
275 | ||
276 | /** | |
277 | * Releases a reference on a lock. | |
278 | * | |
279 | * When last reference is released, lock is returned to the cache, unless it | |
280 | * is in cl_lock_state::CLS_FREEING state, in which case it is destroyed | |
281 | * immediately. | |
282 | * | |
283 | * \see cl_object_put(), cl_page_put() | |
284 | */ | |
285 | void cl_lock_put(const struct lu_env *env, struct cl_lock *lock) | |
286 | { | |
287 | struct cl_object *obj; | |
288 | ||
289 | LINVRNT(cl_lock_invariant(env, lock)); | |
d7e09d03 | 290 | obj = lock->cll_descr.cld_obj; |
cce3c2da | 291 | LINVRNT(obj); |
d7e09d03 PT |
292 | |
293 | CDEBUG(D_TRACE, "releasing reference: %d %p %lu\n", | |
294 | atomic_read(&lock->cll_ref), lock, RETIP); | |
295 | ||
296 | if (atomic_dec_and_test(&lock->cll_ref)) { | |
297 | if (lock->cll_state == CLS_FREEING) { | |
298 | LASSERT(list_empty(&lock->cll_linkage)); | |
299 | cl_lock_free(env, lock); | |
300 | } | |
301 | CS_LOCK_DEC(obj, busy); | |
302 | } | |
d7e09d03 PT |
303 | } |
304 | EXPORT_SYMBOL(cl_lock_put); | |
305 | ||
306 | /** | |
307 | * Acquires an additional reference to a lock. | |
308 | * | |
309 | * This can be called only by caller already possessing a reference to \a | |
310 | * lock. | |
311 | * | |
312 | * \see cl_object_get(), cl_page_get() | |
313 | */ | |
314 | void cl_lock_get(struct cl_lock *lock) | |
315 | { | |
316 | LINVRNT(cl_lock_invariant(NULL, lock)); | |
317 | CDEBUG(D_TRACE, "acquiring reference: %d %p %lu\n", | |
318 | atomic_read(&lock->cll_ref), lock, RETIP); | |
319 | atomic_inc(&lock->cll_ref); | |
320 | } | |
321 | EXPORT_SYMBOL(cl_lock_get); | |
322 | ||
323 | /** | |
324 | * Acquires a reference to a lock. | |
325 | * | |
326 | * This is much like cl_lock_get(), except that this function can be used to | |
327 | * acquire initial reference to the cached lock. Caller has to deal with all | |
328 | * possible races. Use with care! | |
329 | * | |
330 | * \see cl_page_get_trust() | |
331 | */ | |
332 | void cl_lock_get_trust(struct cl_lock *lock) | |
333 | { | |
334 | CDEBUG(D_TRACE, "acquiring trusted reference: %d %p %lu\n", | |
335 | atomic_read(&lock->cll_ref), lock, RETIP); | |
336 | if (atomic_inc_return(&lock->cll_ref) == 1) | |
337 | CS_LOCK_INC(lock->cll_descr.cld_obj, busy); | |
338 | } | |
339 | EXPORT_SYMBOL(cl_lock_get_trust); | |
340 | ||
341 | /** | |
342 | * Helper function destroying the lock that wasn't completely initialized. | |
343 | * | |
344 | * Other threads can acquire references to the top-lock through its | |
345 | * sub-locks. Hence, it cannot be cl_lock_free()-ed immediately. | |
346 | */ | |
347 | static void cl_lock_finish(const struct lu_env *env, struct cl_lock *lock) | |
348 | { | |
349 | cl_lock_mutex_get(env, lock); | |
350 | cl_lock_cancel(env, lock); | |
351 | cl_lock_delete(env, lock); | |
352 | cl_lock_mutex_put(env, lock); | |
353 | cl_lock_put(env, lock); | |
354 | } | |
355 | ||
356 | static struct cl_lock *cl_lock_alloc(const struct lu_env *env, | |
357 | struct cl_object *obj, | |
358 | const struct cl_io *io, | |
359 | const struct cl_lock_descr *descr) | |
360 | { | |
361 | struct cl_lock *lock; | |
362 | struct lu_object_header *head; | |
363 | ||
ccaabce1 | 364 | lock = kmem_cache_alloc(cl_lock_kmem, GFP_NOFS | __GFP_ZERO); |
cce3c2da | 365 | if (lock) { |
d7e09d03 PT |
366 | atomic_set(&lock->cll_ref, 1); |
367 | lock->cll_descr = *descr; | |
368 | lock->cll_state = CLS_NEW; | |
369 | cl_object_get(obj); | |
631abc6e JH |
370 | lu_object_ref_add_at(&obj->co_lu, &lock->cll_obj_ref, "cl_lock", |
371 | lock); | |
d7e09d03 PT |
372 | INIT_LIST_HEAD(&lock->cll_layers); |
373 | INIT_LIST_HEAD(&lock->cll_linkage); | |
374 | INIT_LIST_HEAD(&lock->cll_inclosure); | |
375 | lu_ref_init(&lock->cll_reference); | |
376 | lu_ref_init(&lock->cll_holders); | |
377 | mutex_init(&lock->cll_guard); | |
378 | lockdep_set_class(&lock->cll_guard, &cl_lock_guard_class); | |
379 | init_waitqueue_head(&lock->cll_wq); | |
380 | head = obj->co_lu.lo_header; | |
381 | CS_LOCKSTATE_INC(obj, CLS_NEW); | |
382 | CS_LOCK_INC(obj, total); | |
383 | CS_LOCK_INC(obj, create); | |
384 | cl_lock_lockdep_init(lock); | |
385 | list_for_each_entry(obj, &head->loh_layers, | |
386 | co_lu.lo_linkage) { | |
387 | int err; | |
388 | ||
389 | err = obj->co_ops->coo_lock_init(env, obj, lock, io); | |
390 | if (err != 0) { | |
391 | cl_lock_finish(env, lock); | |
392 | lock = ERR_PTR(err); | |
393 | break; | |
394 | } | |
395 | } | |
396 | } else | |
397 | lock = ERR_PTR(-ENOMEM); | |
0a3bdb00 | 398 | return lock; |
d7e09d03 PT |
399 | } |
400 | ||
401 | /** | |
402 | * Transfer the lock into INTRANSIT state and return the original state. | |
403 | * | |
404 | * \pre state: CLS_CACHED, CLS_HELD or CLS_ENQUEUED | |
405 | * \post state: CLS_INTRANSIT | |
406 | * \see CLS_INTRANSIT | |
407 | */ | |
a90a2917 SB |
408 | static enum cl_lock_state cl_lock_intransit(const struct lu_env *env, |
409 | struct cl_lock *lock) | |
d7e09d03 PT |
410 | { |
411 | enum cl_lock_state state = lock->cll_state; | |
412 | ||
413 | LASSERT(cl_lock_is_mutexed(lock)); | |
414 | LASSERT(state != CLS_INTRANSIT); | |
415 | LASSERTF(state >= CLS_ENQUEUED && state <= CLS_CACHED, | |
416 | "Malformed lock state %d.\n", state); | |
417 | ||
418 | cl_lock_state_set(env, lock, CLS_INTRANSIT); | |
419 | lock->cll_intransit_owner = current; | |
420 | cl_lock_hold_add(env, lock, "intransit", current); | |
421 | return state; | |
422 | } | |
d7e09d03 PT |
423 | |
424 | /** | |
425 | * Exit the intransit state and restore the lock state to the original state | |
426 | */ | |
a90a2917 SB |
427 | static void cl_lock_extransit(const struct lu_env *env, struct cl_lock *lock, |
428 | enum cl_lock_state state) | |
d7e09d03 PT |
429 | { |
430 | LASSERT(cl_lock_is_mutexed(lock)); | |
431 | LASSERT(lock->cll_state == CLS_INTRANSIT); | |
432 | LASSERT(state != CLS_INTRANSIT); | |
433 | LASSERT(lock->cll_intransit_owner == current); | |
434 | ||
435 | lock->cll_intransit_owner = NULL; | |
436 | cl_lock_state_set(env, lock, state); | |
437 | cl_lock_unhold(env, lock, "intransit", current); | |
438 | } | |
d7e09d03 PT |
439 | |
440 | /** | |
441 | * Checking whether the lock is intransit state | |
442 | */ | |
443 | int cl_lock_is_intransit(struct cl_lock *lock) | |
444 | { | |
445 | LASSERT(cl_lock_is_mutexed(lock)); | |
446 | return lock->cll_state == CLS_INTRANSIT && | |
447 | lock->cll_intransit_owner != current; | |
448 | } | |
449 | EXPORT_SYMBOL(cl_lock_is_intransit); | |
450 | /** | |
451 | * Returns true iff lock is "suitable" for given io. E.g., locks acquired by | |
452 | * truncate and O_APPEND cannot be reused for read/non-append-write, as they | |
453 | * cover multiple stripes and can trigger cascading timeouts. | |
454 | */ | |
455 | static int cl_lock_fits_into(const struct lu_env *env, | |
456 | const struct cl_lock *lock, | |
457 | const struct cl_lock_descr *need, | |
458 | const struct cl_io *io) | |
459 | { | |
460 | const struct cl_lock_slice *slice; | |
461 | ||
462 | LINVRNT(cl_lock_invariant_trusted(env, lock)); | |
d7e09d03 | 463 | list_for_each_entry(slice, &lock->cll_layers, cls_linkage) { |
cce3c2da | 464 | if (slice->cls_ops->clo_fits_into && |
d7e09d03 | 465 | !slice->cls_ops->clo_fits_into(env, slice, need, io)) |
0a3bdb00 | 466 | return 0; |
d7e09d03 | 467 | } |
0a3bdb00 | 468 | return 1; |
d7e09d03 PT |
469 | } |
470 | ||
471 | static struct cl_lock *cl_lock_lookup(const struct lu_env *env, | |
472 | struct cl_object *obj, | |
473 | const struct cl_io *io, | |
474 | const struct cl_lock_descr *need) | |
475 | { | |
476 | struct cl_lock *lock; | |
477 | struct cl_object_header *head; | |
478 | ||
d7e09d03 | 479 | head = cl_object_header(obj); |
5e42bc9d | 480 | assert_spin_locked(&head->coh_lock_guard); |
d7e09d03 PT |
481 | CS_LOCK_INC(obj, lookup); |
482 | list_for_each_entry(lock, &head->coh_locks, cll_linkage) { | |
483 | int matched; | |
484 | ||
485 | matched = cl_lock_ext_match(&lock->cll_descr, need) && | |
486 | lock->cll_state < CLS_FREEING && | |
487 | lock->cll_error == 0 && | |
488 | !(lock->cll_flags & CLF_CANCELLED) && | |
489 | cl_lock_fits_into(env, lock, need, io); | |
490 | CDEBUG(D_DLMTRACE, "has: "DDESCR"(%d) need: "DDESCR": %d\n", | |
491 | PDESCR(&lock->cll_descr), lock->cll_state, PDESCR(need), | |
492 | matched); | |
493 | if (matched) { | |
494 | cl_lock_get_trust(lock); | |
495 | CS_LOCK_INC(obj, hit); | |
0a3bdb00 | 496 | return lock; |
d7e09d03 PT |
497 | } |
498 | } | |
0a3bdb00 | 499 | return NULL; |
d7e09d03 PT |
500 | } |
501 | ||
502 | /** | |
503 | * Returns a lock matching description \a need. | |
504 | * | |
505 | * This is the main entry point into the cl_lock caching interface. First, a | |
506 | * cache (implemented as a per-object linked list) is consulted. If lock is | |
507 | * found there, it is returned immediately. Otherwise new lock is allocated | |
508 | * and returned. In any case, additional reference to lock is acquired. | |
509 | * | |
510 | * \see cl_object_find(), cl_page_find() | |
511 | */ | |
512 | static struct cl_lock *cl_lock_find(const struct lu_env *env, | |
513 | const struct cl_io *io, | |
514 | const struct cl_lock_descr *need) | |
515 | { | |
516 | struct cl_object_header *head; | |
517 | struct cl_object *obj; | |
518 | struct cl_lock *lock; | |
519 | ||
d7e09d03 PT |
520 | obj = need->cld_obj; |
521 | head = cl_object_header(obj); | |
522 | ||
523 | spin_lock(&head->coh_lock_guard); | |
524 | lock = cl_lock_lookup(env, obj, io, need); | |
525 | spin_unlock(&head->coh_lock_guard); | |
526 | ||
cce3c2da | 527 | if (!lock) { |
d7e09d03 PT |
528 | lock = cl_lock_alloc(env, obj, io, need); |
529 | if (!IS_ERR(lock)) { | |
530 | struct cl_lock *ghost; | |
531 | ||
532 | spin_lock(&head->coh_lock_guard); | |
533 | ghost = cl_lock_lookup(env, obj, io, need); | |
cce3c2da | 534 | if (!ghost) { |
8d67c821 | 535 | cl_lock_get_trust(lock); |
d7e09d03 PT |
536 | list_add_tail(&lock->cll_linkage, |
537 | &head->coh_locks); | |
538 | spin_unlock(&head->coh_lock_guard); | |
539 | CS_LOCK_INC(obj, busy); | |
540 | } else { | |
541 | spin_unlock(&head->coh_lock_guard); | |
542 | /* | |
543 | * Other threads can acquire references to the | |
544 | * top-lock through its sub-locks. Hence, it | |
545 | * cannot be cl_lock_free()-ed immediately. | |
546 | */ | |
547 | cl_lock_finish(env, lock); | |
548 | lock = ghost; | |
549 | } | |
550 | } | |
551 | } | |
0a3bdb00 | 552 | return lock; |
d7e09d03 PT |
553 | } |
554 | ||
555 | /** | |
556 | * Returns existing lock matching given description. This is similar to | |
557 | * cl_lock_find() except that no new lock is created, and returned lock is | |
558 | * guaranteed to be in enum cl_lock_state::CLS_HELD state. | |
559 | */ | |
560 | struct cl_lock *cl_lock_peek(const struct lu_env *env, const struct cl_io *io, | |
561 | const struct cl_lock_descr *need, | |
562 | const char *scope, const void *source) | |
563 | { | |
564 | struct cl_object_header *head; | |
565 | struct cl_object *obj; | |
566 | struct cl_lock *lock; | |
567 | ||
568 | obj = need->cld_obj; | |
569 | head = cl_object_header(obj); | |
570 | ||
571 | do { | |
572 | spin_lock(&head->coh_lock_guard); | |
573 | lock = cl_lock_lookup(env, obj, io, need); | |
574 | spin_unlock(&head->coh_lock_guard); | |
cce3c2da | 575 | if (!lock) |
d7e09d03 PT |
576 | return NULL; |
577 | ||
578 | cl_lock_mutex_get(env, lock); | |
579 | if (lock->cll_state == CLS_INTRANSIT) | |
580 | /* Don't care return value. */ | |
581 | cl_lock_state_wait(env, lock); | |
582 | if (lock->cll_state == CLS_FREEING) { | |
583 | cl_lock_mutex_put(env, lock); | |
584 | cl_lock_put(env, lock); | |
585 | lock = NULL; | |
586 | } | |
cce3c2da | 587 | } while (!lock); |
d7e09d03 PT |
588 | |
589 | cl_lock_hold_add(env, lock, scope, source); | |
590 | cl_lock_user_add(env, lock); | |
591 | if (lock->cll_state == CLS_CACHED) | |
592 | cl_use_try(env, lock, 1); | |
593 | if (lock->cll_state == CLS_HELD) { | |
594 | cl_lock_mutex_put(env, lock); | |
595 | cl_lock_lockdep_acquire(env, lock, 0); | |
596 | cl_lock_put(env, lock); | |
597 | } else { | |
598 | cl_unuse_try(env, lock); | |
599 | cl_lock_unhold(env, lock, scope, source); | |
600 | cl_lock_mutex_put(env, lock); | |
601 | cl_lock_put(env, lock); | |
602 | lock = NULL; | |
603 | } | |
604 | ||
605 | return lock; | |
606 | } | |
607 | EXPORT_SYMBOL(cl_lock_peek); | |
608 | ||
609 | /** | |
610 | * Returns a slice within a lock, corresponding to the given layer in the | |
611 | * device stack. | |
612 | * | |
613 | * \see cl_page_at() | |
614 | */ | |
615 | const struct cl_lock_slice *cl_lock_at(const struct cl_lock *lock, | |
616 | const struct lu_device_type *dtype) | |
617 | { | |
618 | const struct cl_lock_slice *slice; | |
619 | ||
620 | LINVRNT(cl_lock_invariant_trusted(NULL, lock)); | |
d7e09d03 PT |
621 | |
622 | list_for_each_entry(slice, &lock->cll_layers, cls_linkage) { | |
623 | if (slice->cls_obj->co_lu.lo_dev->ld_type == dtype) | |
0a3bdb00 | 624 | return slice; |
d7e09d03 | 625 | } |
0a3bdb00 | 626 | return NULL; |
d7e09d03 PT |
627 | } |
628 | EXPORT_SYMBOL(cl_lock_at); | |
629 | ||
630 | static void cl_lock_mutex_tail(const struct lu_env *env, struct cl_lock *lock) | |
631 | { | |
632 | struct cl_thread_counters *counters; | |
633 | ||
634 | counters = cl_lock_counters(env, lock); | |
635 | lock->cll_depth++; | |
636 | counters->ctc_nr_locks_locked++; | |
637 | lu_ref_add(&counters->ctc_locks_locked, "cll_guard", lock); | |
638 | cl_lock_trace(D_TRACE, env, "got mutex", lock); | |
639 | } | |
640 | ||
641 | /** | |
642 | * Locks cl_lock object. | |
643 | * | |
644 | * This is used to manipulate cl_lock fields, and to serialize state | |
645 | * transitions in the lock state machine. | |
646 | * | |
647 | * \post cl_lock_is_mutexed(lock) | |
648 | * | |
649 | * \see cl_lock_mutex_put() | |
650 | */ | |
651 | void cl_lock_mutex_get(const struct lu_env *env, struct cl_lock *lock) | |
652 | { | |
653 | LINVRNT(cl_lock_invariant(env, lock)); | |
654 | ||
655 | if (lock->cll_guarder == current) { | |
656 | LINVRNT(cl_lock_is_mutexed(lock)); | |
657 | LINVRNT(lock->cll_depth > 0); | |
658 | } else { | |
659 | struct cl_object_header *hdr; | |
660 | struct cl_thread_info *info; | |
661 | int i; | |
662 | ||
663 | LINVRNT(lock->cll_guarder != current); | |
664 | hdr = cl_object_header(lock->cll_descr.cld_obj); | |
665 | /* | |
666 | * Check that mutices are taken in the bottom-to-top order. | |
667 | */ | |
668 | info = cl_env_info(env); | |
669 | for (i = 0; i < hdr->coh_nesting; ++i) | |
670 | LASSERT(info->clt_counters[i].ctc_nr_locks_locked == 0); | |
671 | mutex_lock_nested(&lock->cll_guard, hdr->coh_nesting); | |
672 | lock->cll_guarder = current; | |
673 | LINVRNT(lock->cll_depth == 0); | |
674 | } | |
675 | cl_lock_mutex_tail(env, lock); | |
676 | } | |
677 | EXPORT_SYMBOL(cl_lock_mutex_get); | |
678 | ||
679 | /** | |
680 | * Try-locks cl_lock object. | |
681 | * | |
682 | * \retval 0 \a lock was successfully locked | |
683 | * | |
684 | * \retval -EBUSY \a lock cannot be locked right now | |
685 | * | |
686 | * \post ergo(result == 0, cl_lock_is_mutexed(lock)) | |
687 | * | |
688 | * \see cl_lock_mutex_get() | |
689 | */ | |
5e6f5901 | 690 | static int cl_lock_mutex_try(const struct lu_env *env, struct cl_lock *lock) |
d7e09d03 PT |
691 | { |
692 | int result; | |
693 | ||
694 | LINVRNT(cl_lock_invariant_trusted(env, lock)); | |
d7e09d03 PT |
695 | |
696 | result = 0; | |
697 | if (lock->cll_guarder == current) { | |
698 | LINVRNT(lock->cll_depth > 0); | |
699 | cl_lock_mutex_tail(env, lock); | |
700 | } else if (mutex_trylock(&lock->cll_guard)) { | |
701 | LINVRNT(lock->cll_depth == 0); | |
702 | lock->cll_guarder = current; | |
703 | cl_lock_mutex_tail(env, lock); | |
704 | } else | |
705 | result = -EBUSY; | |
0a3bdb00 | 706 | return result; |
d7e09d03 | 707 | } |
d7e09d03 PT |
708 | |
709 | /** | |
710 | {* Unlocks cl_lock object. | |
711 | * | |
712 | * \pre cl_lock_is_mutexed(lock) | |
713 | * | |
714 | * \see cl_lock_mutex_get() | |
715 | */ | |
716 | void cl_lock_mutex_put(const struct lu_env *env, struct cl_lock *lock) | |
717 | { | |
718 | struct cl_thread_counters *counters; | |
719 | ||
720 | LINVRNT(cl_lock_invariant(env, lock)); | |
721 | LINVRNT(cl_lock_is_mutexed(lock)); | |
722 | LINVRNT(lock->cll_guarder == current); | |
723 | LINVRNT(lock->cll_depth > 0); | |
724 | ||
725 | counters = cl_lock_counters(env, lock); | |
726 | LINVRNT(counters->ctc_nr_locks_locked > 0); | |
727 | ||
728 | cl_lock_trace(D_TRACE, env, "put mutex", lock); | |
729 | lu_ref_del(&counters->ctc_locks_locked, "cll_guard", lock); | |
730 | counters->ctc_nr_locks_locked--; | |
731 | if (--lock->cll_depth == 0) { | |
732 | lock->cll_guarder = NULL; | |
733 | mutex_unlock(&lock->cll_guard); | |
734 | } | |
735 | } | |
736 | EXPORT_SYMBOL(cl_lock_mutex_put); | |
737 | ||
738 | /** | |
739 | * Returns true iff lock's mutex is owned by the current thread. | |
740 | */ | |
741 | int cl_lock_is_mutexed(struct cl_lock *lock) | |
742 | { | |
743 | return lock->cll_guarder == current; | |
744 | } | |
745 | EXPORT_SYMBOL(cl_lock_is_mutexed); | |
746 | ||
747 | /** | |
748 | * Returns number of cl_lock mutices held by the current thread (environment). | |
749 | */ | |
750 | int cl_lock_nr_mutexed(const struct lu_env *env) | |
751 | { | |
752 | struct cl_thread_info *info; | |
753 | int i; | |
754 | int locked; | |
755 | ||
756 | /* | |
757 | * NOTE: if summation across all nesting levels (currently 2) proves | |
758 | * too expensive, a summary counter can be added to | |
759 | * struct cl_thread_info. | |
760 | */ | |
761 | info = cl_env_info(env); | |
762 | for (i = 0, locked = 0; i < ARRAY_SIZE(info->clt_counters); ++i) | |
763 | locked += info->clt_counters[i].ctc_nr_locks_locked; | |
764 | return locked; | |
765 | } | |
766 | EXPORT_SYMBOL(cl_lock_nr_mutexed); | |
767 | ||
768 | static void cl_lock_cancel0(const struct lu_env *env, struct cl_lock *lock) | |
769 | { | |
770 | LINVRNT(cl_lock_is_mutexed(lock)); | |
771 | LINVRNT(cl_lock_invariant(env, lock)); | |
d7e09d03 PT |
772 | if (!(lock->cll_flags & CLF_CANCELLED)) { |
773 | const struct cl_lock_slice *slice; | |
774 | ||
775 | lock->cll_flags |= CLF_CANCELLED; | |
776 | list_for_each_entry_reverse(slice, &lock->cll_layers, | |
777 | cls_linkage) { | |
cce3c2da | 778 | if (slice->cls_ops->clo_cancel) |
d7e09d03 PT |
779 | slice->cls_ops->clo_cancel(env, slice); |
780 | } | |
781 | } | |
d7e09d03 PT |
782 | } |
783 | ||
784 | static void cl_lock_delete0(const struct lu_env *env, struct cl_lock *lock) | |
785 | { | |
786 | struct cl_object_header *head; | |
787 | const struct cl_lock_slice *slice; | |
788 | ||
789 | LINVRNT(cl_lock_is_mutexed(lock)); | |
790 | LINVRNT(cl_lock_invariant(env, lock)); | |
791 | ||
d7e09d03 | 792 | if (lock->cll_state < CLS_FREEING) { |
8d67c821 JX |
793 | bool in_cache; |
794 | ||
d7e09d03 PT |
795 | LASSERT(lock->cll_state != CLS_INTRANSIT); |
796 | cl_lock_state_set(env, lock, CLS_FREEING); | |
797 | ||
798 | head = cl_object_header(lock->cll_descr.cld_obj); | |
799 | ||
800 | spin_lock(&head->coh_lock_guard); | |
8d67c821 JX |
801 | in_cache = !list_empty(&lock->cll_linkage); |
802 | if (in_cache) | |
803 | list_del_init(&lock->cll_linkage); | |
d7e09d03 PT |
804 | spin_unlock(&head->coh_lock_guard); |
805 | ||
8d67c821 JX |
806 | if (in_cache) /* coh_locks cache holds a refcount. */ |
807 | cl_lock_put(env, lock); | |
808 | ||
d7e09d03 PT |
809 | /* |
810 | * From now on, no new references to this lock can be acquired | |
811 | * by cl_lock_lookup(). | |
812 | */ | |
813 | list_for_each_entry_reverse(slice, &lock->cll_layers, | |
814 | cls_linkage) { | |
cce3c2da | 815 | if (slice->cls_ops->clo_delete) |
d7e09d03 PT |
816 | slice->cls_ops->clo_delete(env, slice); |
817 | } | |
818 | /* | |
819 | * From now on, no new references to this lock can be acquired | |
820 | * by layer-specific means (like a pointer from struct | |
821 | * ldlm_lock in osc, or a pointer from top-lock to sub-lock in | |
822 | * lov). | |
823 | * | |
824 | * Lock will be finally freed in cl_lock_put() when last of | |
825 | * existing references goes away. | |
826 | */ | |
827 | } | |
d7e09d03 PT |
828 | } |
829 | ||
830 | /** | |
831 | * Mod(ifie)s cl_lock::cll_holds counter for a given lock. Also, for a | |
832 | * top-lock (nesting == 0) accounts for this modification in the per-thread | |
833 | * debugging counters. Sub-lock holds can be released by a thread different | |
834 | * from one that acquired it. | |
835 | */ | |
836 | static void cl_lock_hold_mod(const struct lu_env *env, struct cl_lock *lock, | |
837 | int delta) | |
838 | { | |
839 | struct cl_thread_counters *counters; | |
840 | enum clt_nesting_level nesting; | |
841 | ||
842 | lock->cll_holds += delta; | |
843 | nesting = cl_lock_nesting(lock); | |
844 | if (nesting == CNL_TOP) { | |
845 | counters = &cl_env_info(env)->clt_counters[CNL_TOP]; | |
846 | counters->ctc_nr_held += delta; | |
847 | LASSERT(counters->ctc_nr_held >= 0); | |
848 | } | |
849 | } | |
850 | ||
851 | /** | |
852 | * Mod(ifie)s cl_lock::cll_users counter for a given lock. See | |
853 | * cl_lock_hold_mod() for the explanation of the debugging code. | |
854 | */ | |
855 | static void cl_lock_used_mod(const struct lu_env *env, struct cl_lock *lock, | |
856 | int delta) | |
857 | { | |
858 | struct cl_thread_counters *counters; | |
859 | enum clt_nesting_level nesting; | |
860 | ||
861 | lock->cll_users += delta; | |
862 | nesting = cl_lock_nesting(lock); | |
863 | if (nesting == CNL_TOP) { | |
864 | counters = &cl_env_info(env)->clt_counters[CNL_TOP]; | |
865 | counters->ctc_nr_used += delta; | |
866 | LASSERT(counters->ctc_nr_used >= 0); | |
867 | } | |
868 | } | |
869 | ||
870 | void cl_lock_hold_release(const struct lu_env *env, struct cl_lock *lock, | |
871 | const char *scope, const void *source) | |
872 | { | |
873 | LINVRNT(cl_lock_is_mutexed(lock)); | |
874 | LINVRNT(cl_lock_invariant(env, lock)); | |
875 | LASSERT(lock->cll_holds > 0); | |
876 | ||
d7e09d03 PT |
877 | cl_lock_trace(D_DLMTRACE, env, "hold release lock", lock); |
878 | lu_ref_del(&lock->cll_holders, scope, source); | |
879 | cl_lock_hold_mod(env, lock, -1); | |
880 | if (lock->cll_holds == 0) { | |
881 | CL_LOCK_ASSERT(lock->cll_state != CLS_HELD, env, lock); | |
882 | if (lock->cll_descr.cld_mode == CLM_PHANTOM || | |
883 | lock->cll_descr.cld_mode == CLM_GROUP || | |
884 | lock->cll_state != CLS_CACHED) | |
885 | /* | |
886 | * If lock is still phantom or grouplock when user is | |
887 | * done with it---destroy the lock. | |
888 | */ | |
889 | lock->cll_flags |= CLF_CANCELPEND|CLF_DOOMED; | |
890 | if (lock->cll_flags & CLF_CANCELPEND) { | |
891 | lock->cll_flags &= ~CLF_CANCELPEND; | |
892 | cl_lock_cancel0(env, lock); | |
893 | } | |
894 | if (lock->cll_flags & CLF_DOOMED) { | |
895 | /* no longer doomed: it's dead... Jim. */ | |
896 | lock->cll_flags &= ~CLF_DOOMED; | |
897 | cl_lock_delete0(env, lock); | |
898 | } | |
899 | } | |
d7e09d03 PT |
900 | } |
901 | EXPORT_SYMBOL(cl_lock_hold_release); | |
902 | ||
903 | /** | |
904 | * Waits until lock state is changed. | |
905 | * | |
906 | * This function is called with cl_lock mutex locked, atomically releases | |
907 | * mutex and goes to sleep, waiting for a lock state change (signaled by | |
908 | * cl_lock_signal()), and re-acquires the mutex before return. | |
909 | * | |
910 | * This function is used to wait until lock state machine makes some progress | |
911 | * and to emulate synchronous operations on top of asynchronous lock | |
912 | * interface. | |
913 | * | |
914 | * \retval -EINTR wait was interrupted | |
915 | * | |
916 | * \retval 0 wait wasn't interrupted | |
917 | * | |
918 | * \pre cl_lock_is_mutexed(lock) | |
919 | * | |
920 | * \see cl_lock_signal() | |
921 | */ | |
922 | int cl_lock_state_wait(const struct lu_env *env, struct cl_lock *lock) | |
923 | { | |
924 | wait_queue_t waiter; | |
925 | sigset_t blocked; | |
926 | int result; | |
927 | ||
d7e09d03 PT |
928 | LINVRNT(cl_lock_is_mutexed(lock)); |
929 | LINVRNT(cl_lock_invariant(env, lock)); | |
930 | LASSERT(lock->cll_depth == 1); | |
931 | LASSERT(lock->cll_state != CLS_FREEING); /* too late to wait */ | |
932 | ||
933 | cl_lock_trace(D_DLMTRACE, env, "state wait lock", lock); | |
934 | result = lock->cll_error; | |
935 | if (result == 0) { | |
936 | /* To avoid being interrupted by the 'non-fatal' signals | |
937 | * (SIGCHLD, for instance), we'd block them temporarily. | |
6ba59179 OD |
938 | * LU-305 |
939 | */ | |
d7e09d03 PT |
940 | blocked = cfs_block_sigsinv(LUSTRE_FATAL_SIGS); |
941 | ||
9e795d35 | 942 | init_waitqueue_entry(&waiter, current); |
d7e09d03 PT |
943 | add_wait_queue(&lock->cll_wq, &waiter); |
944 | set_current_state(TASK_INTERRUPTIBLE); | |
945 | cl_lock_mutex_put(env, lock); | |
946 | ||
947 | LASSERT(cl_lock_nr_mutexed(env) == 0); | |
948 | ||
949 | /* Returning ERESTARTSYS instead of EINTR so syscalls | |
6ba59179 OD |
950 | * can be restarted if signals are pending here |
951 | */ | |
d7e09d03 PT |
952 | result = -ERESTARTSYS; |
953 | if (likely(!OBD_FAIL_CHECK(OBD_FAIL_LOCK_STATE_WAIT_INTR))) { | |
b3669a7f | 954 | schedule(); |
d7e09d03 PT |
955 | if (!cfs_signal_pending()) |
956 | result = 0; | |
957 | } | |
958 | ||
959 | cl_lock_mutex_get(env, lock); | |
960 | set_current_state(TASK_RUNNING); | |
961 | remove_wait_queue(&lock->cll_wq, &waiter); | |
962 | ||
963 | /* Restore old blocked signals */ | |
964 | cfs_restore_sigs(blocked); | |
965 | } | |
0a3bdb00 | 966 | return result; |
d7e09d03 PT |
967 | } |
968 | EXPORT_SYMBOL(cl_lock_state_wait); | |
969 | ||
970 | static void cl_lock_state_signal(const struct lu_env *env, struct cl_lock *lock, | |
971 | enum cl_lock_state state) | |
972 | { | |
973 | const struct cl_lock_slice *slice; | |
974 | ||
d7e09d03 PT |
975 | LINVRNT(cl_lock_is_mutexed(lock)); |
976 | LINVRNT(cl_lock_invariant(env, lock)); | |
977 | ||
978 | list_for_each_entry(slice, &lock->cll_layers, cls_linkage) | |
cce3c2da | 979 | if (slice->cls_ops->clo_state) |
d7e09d03 PT |
980 | slice->cls_ops->clo_state(env, slice, state); |
981 | wake_up_all(&lock->cll_wq); | |
d7e09d03 PT |
982 | } |
983 | ||
984 | /** | |
985 | * Notifies waiters that lock state changed. | |
986 | * | |
987 | * Wakes up all waiters sleeping in cl_lock_state_wait(), also notifies all | |
988 | * layers about state change by calling cl_lock_operations::clo_state() | |
989 | * top-to-bottom. | |
990 | */ | |
991 | void cl_lock_signal(const struct lu_env *env, struct cl_lock *lock) | |
992 | { | |
d7e09d03 PT |
993 | cl_lock_trace(D_DLMTRACE, env, "state signal lock", lock); |
994 | cl_lock_state_signal(env, lock, lock->cll_state); | |
d7e09d03 PT |
995 | } |
996 | EXPORT_SYMBOL(cl_lock_signal); | |
997 | ||
998 | /** | |
999 | * Changes lock state. | |
1000 | * | |
1001 | * This function is invoked to notify layers that lock state changed, possible | |
1002 | * as a result of an asynchronous event such as call-back reception. | |
1003 | * | |
1004 | * \post lock->cll_state == state | |
1005 | * | |
1006 | * \see cl_lock_operations::clo_state() | |
1007 | */ | |
1008 | void cl_lock_state_set(const struct lu_env *env, struct cl_lock *lock, | |
1009 | enum cl_lock_state state) | |
1010 | { | |
d7e09d03 PT |
1011 | LASSERT(lock->cll_state <= state || |
1012 | (lock->cll_state == CLS_CACHED && | |
1013 | (state == CLS_HELD || /* lock found in cache */ | |
1014 | state == CLS_NEW || /* sub-lock canceled */ | |
1015 | state == CLS_INTRANSIT)) || | |
1016 | /* lock is in transit state */ | |
1017 | lock->cll_state == CLS_INTRANSIT); | |
1018 | ||
1019 | if (lock->cll_state != state) { | |
1020 | CS_LOCKSTATE_DEC(lock->cll_descr.cld_obj, lock->cll_state); | |
1021 | CS_LOCKSTATE_INC(lock->cll_descr.cld_obj, state); | |
1022 | ||
1023 | cl_lock_state_signal(env, lock, state); | |
1024 | lock->cll_state = state; | |
1025 | } | |
d7e09d03 PT |
1026 | } |
1027 | EXPORT_SYMBOL(cl_lock_state_set); | |
1028 | ||
1029 | static int cl_unuse_try_internal(const struct lu_env *env, struct cl_lock *lock) | |
1030 | { | |
1031 | const struct cl_lock_slice *slice; | |
1032 | int result; | |
1033 | ||
1034 | do { | |
1035 | result = 0; | |
1036 | ||
1037 | LINVRNT(cl_lock_is_mutexed(lock)); | |
1038 | LINVRNT(cl_lock_invariant(env, lock)); | |
1039 | LASSERT(lock->cll_state == CLS_INTRANSIT); | |
1040 | ||
1041 | result = -ENOSYS; | |
1042 | list_for_each_entry_reverse(slice, &lock->cll_layers, | |
1043 | cls_linkage) { | |
cce3c2da | 1044 | if (slice->cls_ops->clo_unuse) { |
d7e09d03 PT |
1045 | result = slice->cls_ops->clo_unuse(env, slice); |
1046 | if (result != 0) | |
1047 | break; | |
1048 | } | |
1049 | } | |
1050 | LASSERT(result != -ENOSYS); | |
1051 | } while (result == CLO_REPEAT); | |
1052 | ||
1053 | return result; | |
1054 | } | |
1055 | ||
1056 | /** | |
1057 | * Yanks lock from the cache (cl_lock_state::CLS_CACHED state) by calling | |
1058 | * cl_lock_operations::clo_use() top-to-bottom to notify layers. | |
1059 | * @atomic = 1, it must unuse the lock to recovery the lock to keep the | |
1060 | * use process atomic | |
1061 | */ | |
1062 | int cl_use_try(const struct lu_env *env, struct cl_lock *lock, int atomic) | |
1063 | { | |
1064 | const struct cl_lock_slice *slice; | |
1065 | int result; | |
1066 | enum cl_lock_state state; | |
1067 | ||
d7e09d03 PT |
1068 | cl_lock_trace(D_DLMTRACE, env, "use lock", lock); |
1069 | ||
1070 | LASSERT(lock->cll_state == CLS_CACHED); | |
1071 | if (lock->cll_error) | |
0a3bdb00 | 1072 | return lock->cll_error; |
d7e09d03 PT |
1073 | |
1074 | result = -ENOSYS; | |
1075 | state = cl_lock_intransit(env, lock); | |
1076 | list_for_each_entry(slice, &lock->cll_layers, cls_linkage) { | |
cce3c2da | 1077 | if (slice->cls_ops->clo_use) { |
d7e09d03 PT |
1078 | result = slice->cls_ops->clo_use(env, slice); |
1079 | if (result != 0) | |
1080 | break; | |
1081 | } | |
1082 | } | |
1083 | LASSERT(result != -ENOSYS); | |
1084 | ||
1085 | LASSERTF(lock->cll_state == CLS_INTRANSIT, "Wrong state %d.\n", | |
1086 | lock->cll_state); | |
1087 | ||
1088 | if (result == 0) { | |
1089 | state = CLS_HELD; | |
1090 | } else { | |
1091 | if (result == -ESTALE) { | |
1092 | /* | |
1093 | * ESTALE means sublock being cancelled | |
1094 | * at this time, and set lock state to | |
1095 | * be NEW here and ask the caller to repeat. | |
1096 | */ | |
1097 | state = CLS_NEW; | |
1098 | result = CLO_REPEAT; | |
1099 | } | |
1100 | ||
1101 | /* @atomic means back-off-on-failure. */ | |
1102 | if (atomic) { | |
1103 | int rc; | |
50ffcb7e | 1104 | |
d7e09d03 PT |
1105 | rc = cl_unuse_try_internal(env, lock); |
1106 | /* Vet the results. */ | |
1107 | if (rc < 0 && result > 0) | |
1108 | result = rc; | |
1109 | } | |
1110 | ||
1111 | } | |
1112 | cl_lock_extransit(env, lock, state); | |
0a3bdb00 | 1113 | return result; |
d7e09d03 PT |
1114 | } |
1115 | EXPORT_SYMBOL(cl_use_try); | |
1116 | ||
1117 | /** | |
1118 | * Helper for cl_enqueue_try() that calls ->clo_enqueue() across all layers | |
1119 | * top-to-bottom. | |
1120 | */ | |
1121 | static int cl_enqueue_kick(const struct lu_env *env, | |
1122 | struct cl_lock *lock, | |
1123 | struct cl_io *io, __u32 flags) | |
1124 | { | |
1125 | int result; | |
1126 | const struct cl_lock_slice *slice; | |
1127 | ||
d7e09d03 PT |
1128 | result = -ENOSYS; |
1129 | list_for_each_entry(slice, &lock->cll_layers, cls_linkage) { | |
cce3c2da | 1130 | if (slice->cls_ops->clo_enqueue) { |
d7e09d03 PT |
1131 | result = slice->cls_ops->clo_enqueue(env, |
1132 | slice, io, flags); | |
1133 | if (result != 0) | |
1134 | break; | |
1135 | } | |
1136 | } | |
1137 | LASSERT(result != -ENOSYS); | |
0a3bdb00 | 1138 | return result; |
d7e09d03 PT |
1139 | } |
1140 | ||
1141 | /** | |
1142 | * Tries to enqueue a lock. | |
1143 | * | |
1144 | * This function is called repeatedly by cl_enqueue() until either lock is | |
1145 | * enqueued, or error occurs. This function does not block waiting for | |
1146 | * networking communication to complete. | |
1147 | * | |
1148 | * \post ergo(result == 0, lock->cll_state == CLS_ENQUEUED || | |
1149 | * lock->cll_state == CLS_HELD) | |
1150 | * | |
1151 | * \see cl_enqueue() cl_lock_operations::clo_enqueue() | |
1152 | * \see cl_lock_state::CLS_ENQUEUED | |
1153 | */ | |
1154 | int cl_enqueue_try(const struct lu_env *env, struct cl_lock *lock, | |
1155 | struct cl_io *io, __u32 flags) | |
1156 | { | |
1157 | int result; | |
1158 | ||
d7e09d03 PT |
1159 | cl_lock_trace(D_DLMTRACE, env, "enqueue lock", lock); |
1160 | do { | |
1161 | LINVRNT(cl_lock_is_mutexed(lock)); | |
1162 | ||
1163 | result = lock->cll_error; | |
1164 | if (result != 0) | |
1165 | break; | |
1166 | ||
1167 | switch (lock->cll_state) { | |
1168 | case CLS_NEW: | |
1169 | cl_lock_state_set(env, lock, CLS_QUEUING); | |
1170 | /* fall-through */ | |
1171 | case CLS_QUEUING: | |
1172 | /* kick layers. */ | |
1173 | result = cl_enqueue_kick(env, lock, io, flags); | |
1174 | /* For AGL case, the cl_lock::cll_state may | |
6ba59179 OD |
1175 | * become CLS_HELD already. |
1176 | */ | |
d7e09d03 PT |
1177 | if (result == 0 && lock->cll_state == CLS_QUEUING) |
1178 | cl_lock_state_set(env, lock, CLS_ENQUEUED); | |
1179 | break; | |
1180 | case CLS_INTRANSIT: | |
1181 | LASSERT(cl_lock_is_intransit(lock)); | |
1182 | result = CLO_WAIT; | |
1183 | break; | |
1184 | case CLS_CACHED: | |
1185 | /* yank lock from the cache. */ | |
1186 | result = cl_use_try(env, lock, 0); | |
1187 | break; | |
1188 | case CLS_ENQUEUED: | |
1189 | case CLS_HELD: | |
1190 | result = 0; | |
1191 | break; | |
1192 | default: | |
1193 | case CLS_FREEING: | |
1194 | /* | |
1195 | * impossible, only held locks with increased | |
1196 | * ->cll_holds can be enqueued, and they cannot be | |
1197 | * freed. | |
1198 | */ | |
1199 | LBUG(); | |
1200 | } | |
1201 | } while (result == CLO_REPEAT); | |
0a3bdb00 | 1202 | return result; |
d7e09d03 PT |
1203 | } |
1204 | EXPORT_SYMBOL(cl_enqueue_try); | |
1205 | ||
1206 | /** | |
1207 | * Cancel the conflicting lock found during previous enqueue. | |
1208 | * | |
1209 | * \retval 0 conflicting lock has been canceled. | |
1210 | * \retval -ve error code. | |
1211 | */ | |
1212 | int cl_lock_enqueue_wait(const struct lu_env *env, | |
1213 | struct cl_lock *lock, | |
1214 | int keep_mutex) | |
1215 | { | |
1216 | struct cl_lock *conflict; | |
1217 | int rc = 0; | |
d7e09d03 PT |
1218 | |
1219 | LASSERT(cl_lock_is_mutexed(lock)); | |
1220 | LASSERT(lock->cll_state == CLS_QUEUING); | |
cce3c2da | 1221 | LASSERT(lock->cll_conflict); |
d7e09d03 PT |
1222 | |
1223 | conflict = lock->cll_conflict; | |
1224 | lock->cll_conflict = NULL; | |
1225 | ||
1226 | cl_lock_mutex_put(env, lock); | |
1227 | LASSERT(cl_lock_nr_mutexed(env) == 0); | |
1228 | ||
1229 | cl_lock_mutex_get(env, conflict); | |
1230 | cl_lock_trace(D_DLMTRACE, env, "enqueue wait", conflict); | |
1231 | cl_lock_cancel(env, conflict); | |
1232 | cl_lock_delete(env, conflict); | |
1233 | ||
1234 | while (conflict->cll_state != CLS_FREEING) { | |
1235 | rc = cl_lock_state_wait(env, conflict); | |
1236 | if (rc != 0) | |
1237 | break; | |
1238 | } | |
1239 | cl_lock_mutex_put(env, conflict); | |
1240 | lu_ref_del(&conflict->cll_reference, "cancel-wait", lock); | |
1241 | cl_lock_put(env, conflict); | |
1242 | ||
1243 | if (keep_mutex) | |
1244 | cl_lock_mutex_get(env, lock); | |
1245 | ||
1246 | LASSERT(rc <= 0); | |
0a3bdb00 | 1247 | return rc; |
d7e09d03 PT |
1248 | } |
1249 | EXPORT_SYMBOL(cl_lock_enqueue_wait); | |
1250 | ||
1251 | static int cl_enqueue_locked(const struct lu_env *env, struct cl_lock *lock, | |
1252 | struct cl_io *io, __u32 enqflags) | |
1253 | { | |
1254 | int result; | |
1255 | ||
d7e09d03 PT |
1256 | LINVRNT(cl_lock_is_mutexed(lock)); |
1257 | LINVRNT(cl_lock_invariant(env, lock)); | |
1258 | LASSERT(lock->cll_holds > 0); | |
1259 | ||
1260 | cl_lock_user_add(env, lock); | |
1261 | do { | |
1262 | result = cl_enqueue_try(env, lock, io, enqflags); | |
1263 | if (result == CLO_WAIT) { | |
cce3c2da | 1264 | if (lock->cll_conflict) |
d7e09d03 PT |
1265 | result = cl_lock_enqueue_wait(env, lock, 1); |
1266 | else | |
1267 | result = cl_lock_state_wait(env, lock); | |
1268 | if (result == 0) | |
1269 | continue; | |
1270 | } | |
1271 | break; | |
1272 | } while (1); | |
1273 | if (result != 0) | |
1274 | cl_unuse_try(env, lock); | |
1275 | LASSERT(ergo(result == 0 && !(enqflags & CEF_AGL), | |
1276 | lock->cll_state == CLS_ENQUEUED || | |
1277 | lock->cll_state == CLS_HELD)); | |
0a3bdb00 | 1278 | return result; |
d7e09d03 PT |
1279 | } |
1280 | ||
d7e09d03 PT |
1281 | /** |
1282 | * Tries to unlock a lock. | |
1283 | * | |
1284 | * This function is called to release underlying resource: | |
1285 | * 1. for top lock, the resource is sublocks it held; | |
1286 | * 2. for sublock, the resource is the reference to dlmlock. | |
1287 | * | |
1288 | * cl_unuse_try is a one-shot operation, so it must NOT return CLO_WAIT. | |
1289 | * | |
1290 | * \see cl_unuse() cl_lock_operations::clo_unuse() | |
1291 | * \see cl_lock_state::CLS_CACHED | |
1292 | */ | |
1293 | int cl_unuse_try(const struct lu_env *env, struct cl_lock *lock) | |
1294 | { | |
1295 | int result; | |
1296 | enum cl_lock_state state = CLS_NEW; | |
1297 | ||
d7e09d03 PT |
1298 | cl_lock_trace(D_DLMTRACE, env, "unuse lock", lock); |
1299 | ||
1300 | if (lock->cll_users > 1) { | |
1301 | cl_lock_user_del(env, lock); | |
0a3bdb00 | 1302 | return 0; |
d7e09d03 PT |
1303 | } |
1304 | ||
1305 | /* Only if the lock is in CLS_HELD or CLS_ENQUEUED state, it can hold | |
6ba59179 OD |
1306 | * underlying resources. |
1307 | */ | |
d7e09d03 PT |
1308 | if (!(lock->cll_state == CLS_HELD || lock->cll_state == CLS_ENQUEUED)) { |
1309 | cl_lock_user_del(env, lock); | |
0a3bdb00 | 1310 | return 0; |
d7e09d03 PT |
1311 | } |
1312 | ||
1313 | /* | |
1314 | * New lock users (->cll_users) are not protecting unlocking | |
1315 | * from proceeding. From this point, lock eventually reaches | |
1316 | * CLS_CACHED, is reinitialized to CLS_NEW or fails into | |
1317 | * CLS_FREEING. | |
1318 | */ | |
1319 | state = cl_lock_intransit(env, lock); | |
1320 | ||
1321 | result = cl_unuse_try_internal(env, lock); | |
1322 | LASSERT(lock->cll_state == CLS_INTRANSIT); | |
1323 | LASSERT(result != CLO_WAIT); | |
1324 | cl_lock_user_del(env, lock); | |
1325 | if (result == 0 || result == -ESTALE) { | |
1326 | /* | |
1327 | * Return lock back to the cache. This is the only | |
1328 | * place where lock is moved into CLS_CACHED state. | |
1329 | * | |
1330 | * If one of ->clo_unuse() methods returned -ESTALE, lock | |
1331 | * cannot be placed into cache and has to be | |
1332 | * re-initialized. This happens e.g., when a sub-lock was | |
1333 | * canceled while unlocking was in progress. | |
1334 | */ | |
1335 | if (state == CLS_HELD && result == 0) | |
1336 | state = CLS_CACHED; | |
1337 | else | |
1338 | state = CLS_NEW; | |
1339 | cl_lock_extransit(env, lock, state); | |
1340 | ||
1341 | /* | |
1342 | * Hide -ESTALE error. | |
1343 | * If the lock is a glimpse lock, and it has multiple | |
1344 | * stripes. Assuming that one of its sublock returned -ENAVAIL, | |
1345 | * and other sublocks are matched write locks. In this case, | |
1346 | * we can't set this lock to error because otherwise some of | |
1347 | * its sublocks may not be canceled. This causes some dirty | |
1348 | * pages won't be written to OSTs. -jay | |
1349 | */ | |
1350 | result = 0; | |
1351 | } else { | |
1352 | CERROR("result = %d, this is unlikely!\n", result); | |
1353 | state = CLS_NEW; | |
1354 | cl_lock_extransit(env, lock, state); | |
1355 | } | |
0a3bdb00 | 1356 | return result ?: lock->cll_error; |
d7e09d03 PT |
1357 | } |
1358 | EXPORT_SYMBOL(cl_unuse_try); | |
1359 | ||
1360 | static void cl_unuse_locked(const struct lu_env *env, struct cl_lock *lock) | |
1361 | { | |
1362 | int result; | |
d7e09d03 PT |
1363 | |
1364 | result = cl_unuse_try(env, lock); | |
1365 | if (result) | |
1366 | CL_LOCK_DEBUG(D_ERROR, env, lock, "unuse return %d\n", result); | |
d7e09d03 PT |
1367 | } |
1368 | ||
1369 | /** | |
1370 | * Unlocks a lock. | |
1371 | */ | |
1372 | void cl_unuse(const struct lu_env *env, struct cl_lock *lock) | |
1373 | { | |
d7e09d03 PT |
1374 | cl_lock_mutex_get(env, lock); |
1375 | cl_unuse_locked(env, lock); | |
1376 | cl_lock_mutex_put(env, lock); | |
1377 | cl_lock_lockdep_release(env, lock); | |
d7e09d03 PT |
1378 | } |
1379 | EXPORT_SYMBOL(cl_unuse); | |
1380 | ||
1381 | /** | |
1382 | * Tries to wait for a lock. | |
1383 | * | |
1384 | * This function is called repeatedly by cl_wait() until either lock is | |
1385 | * granted, or error occurs. This function does not block waiting for network | |
1386 | * communication to complete. | |
1387 | * | |
1388 | * \see cl_wait() cl_lock_operations::clo_wait() | |
1389 | * \see cl_lock_state::CLS_HELD | |
1390 | */ | |
1391 | int cl_wait_try(const struct lu_env *env, struct cl_lock *lock) | |
1392 | { | |
1393 | const struct cl_lock_slice *slice; | |
1394 | int result; | |
1395 | ||
d7e09d03 PT |
1396 | cl_lock_trace(D_DLMTRACE, env, "wait lock try", lock); |
1397 | do { | |
1398 | LINVRNT(cl_lock_is_mutexed(lock)); | |
1399 | LINVRNT(cl_lock_invariant(env, lock)); | |
1400 | LASSERTF(lock->cll_state == CLS_QUEUING || | |
1401 | lock->cll_state == CLS_ENQUEUED || | |
1402 | lock->cll_state == CLS_HELD || | |
1403 | lock->cll_state == CLS_INTRANSIT, | |
1404 | "lock state: %d\n", lock->cll_state); | |
1405 | LASSERT(lock->cll_users > 0); | |
1406 | LASSERT(lock->cll_holds > 0); | |
1407 | ||
1408 | result = lock->cll_error; | |
1409 | if (result != 0) | |
1410 | break; | |
1411 | ||
1412 | if (cl_lock_is_intransit(lock)) { | |
1413 | result = CLO_WAIT; | |
1414 | break; | |
1415 | } | |
1416 | ||
1417 | if (lock->cll_state == CLS_HELD) | |
1418 | /* nothing to do */ | |
1419 | break; | |
1420 | ||
1421 | result = -ENOSYS; | |
1422 | list_for_each_entry(slice, &lock->cll_layers, cls_linkage) { | |
cce3c2da | 1423 | if (slice->cls_ops->clo_wait) { |
d7e09d03 PT |
1424 | result = slice->cls_ops->clo_wait(env, slice); |
1425 | if (result != 0) | |
1426 | break; | |
1427 | } | |
1428 | } | |
1429 | LASSERT(result != -ENOSYS); | |
1430 | if (result == 0) { | |
1431 | LASSERT(lock->cll_state != CLS_INTRANSIT); | |
1432 | cl_lock_state_set(env, lock, CLS_HELD); | |
1433 | } | |
1434 | } while (result == CLO_REPEAT); | |
0a3bdb00 | 1435 | return result; |
d7e09d03 PT |
1436 | } |
1437 | EXPORT_SYMBOL(cl_wait_try); | |
1438 | ||
1439 | /** | |
1440 | * Waits until enqueued lock is granted. | |
1441 | * | |
1442 | * \pre current thread or io owns a hold on the lock | |
1443 | * \pre ergo(result == 0, lock->cll_state == CLS_ENQUEUED || | |
1444 | * lock->cll_state == CLS_HELD) | |
1445 | * | |
1446 | * \post ergo(result == 0, lock->cll_state == CLS_HELD) | |
1447 | */ | |
1448 | int cl_wait(const struct lu_env *env, struct cl_lock *lock) | |
1449 | { | |
1450 | int result; | |
1451 | ||
d7e09d03 PT |
1452 | cl_lock_mutex_get(env, lock); |
1453 | ||
1454 | LINVRNT(cl_lock_invariant(env, lock)); | |
1455 | LASSERTF(lock->cll_state == CLS_ENQUEUED || lock->cll_state == CLS_HELD, | |
72a87fca | 1456 | "Wrong state %d\n", lock->cll_state); |
d7e09d03 PT |
1457 | LASSERT(lock->cll_holds > 0); |
1458 | ||
1459 | do { | |
1460 | result = cl_wait_try(env, lock); | |
1461 | if (result == CLO_WAIT) { | |
1462 | result = cl_lock_state_wait(env, lock); | |
1463 | if (result == 0) | |
1464 | continue; | |
1465 | } | |
1466 | break; | |
1467 | } while (1); | |
1468 | if (result < 0) { | |
1469 | cl_unuse_try(env, lock); | |
1470 | cl_lock_lockdep_release(env, lock); | |
1471 | } | |
1472 | cl_lock_trace(D_DLMTRACE, env, "wait lock", lock); | |
1473 | cl_lock_mutex_put(env, lock); | |
1474 | LASSERT(ergo(result == 0, lock->cll_state == CLS_HELD)); | |
0a3bdb00 | 1475 | return result; |
d7e09d03 PT |
1476 | } |
1477 | EXPORT_SYMBOL(cl_wait); | |
1478 | ||
1479 | /** | |
1480 | * Executes cl_lock_operations::clo_weigh(), and sums results to estimate lock | |
1481 | * value. | |
1482 | */ | |
1483 | unsigned long cl_lock_weigh(const struct lu_env *env, struct cl_lock *lock) | |
1484 | { | |
1485 | const struct cl_lock_slice *slice; | |
1486 | unsigned long pound; | |
1487 | unsigned long ounce; | |
1488 | ||
d7e09d03 PT |
1489 | LINVRNT(cl_lock_is_mutexed(lock)); |
1490 | LINVRNT(cl_lock_invariant(env, lock)); | |
1491 | ||
1492 | pound = 0; | |
1493 | list_for_each_entry_reverse(slice, &lock->cll_layers, cls_linkage) { | |
cce3c2da | 1494 | if (slice->cls_ops->clo_weigh) { |
d7e09d03 PT |
1495 | ounce = slice->cls_ops->clo_weigh(env, slice); |
1496 | pound += ounce; | |
1497 | if (pound < ounce) /* over-weight^Wflow */ | |
1498 | pound = ~0UL; | |
1499 | } | |
1500 | } | |
0a3bdb00 | 1501 | return pound; |
d7e09d03 PT |
1502 | } |
1503 | EXPORT_SYMBOL(cl_lock_weigh); | |
1504 | ||
1505 | /** | |
1506 | * Notifies layers that lock description changed. | |
1507 | * | |
1508 | * The server can grant client a lock different from one that was requested | |
1509 | * (e.g., larger in extent). This method is called when actually granted lock | |
1510 | * description becomes known to let layers to accommodate for changed lock | |
1511 | * description. | |
1512 | * | |
1513 | * \see cl_lock_operations::clo_modify() | |
1514 | */ | |
1515 | int cl_lock_modify(const struct lu_env *env, struct cl_lock *lock, | |
1516 | const struct cl_lock_descr *desc) | |
1517 | { | |
1518 | const struct cl_lock_slice *slice; | |
1519 | struct cl_object *obj = lock->cll_descr.cld_obj; | |
1520 | struct cl_object_header *hdr = cl_object_header(obj); | |
1521 | int result; | |
1522 | ||
d7e09d03 PT |
1523 | cl_lock_trace(D_DLMTRACE, env, "modify lock", lock); |
1524 | /* don't allow object to change */ | |
1525 | LASSERT(obj == desc->cld_obj); | |
1526 | LINVRNT(cl_lock_is_mutexed(lock)); | |
1527 | LINVRNT(cl_lock_invariant(env, lock)); | |
1528 | ||
1529 | list_for_each_entry_reverse(slice, &lock->cll_layers, cls_linkage) { | |
cce3c2da | 1530 | if (slice->cls_ops->clo_modify) { |
d7e09d03 PT |
1531 | result = slice->cls_ops->clo_modify(env, slice, desc); |
1532 | if (result != 0) | |
0a3bdb00 | 1533 | return result; |
d7e09d03 PT |
1534 | } |
1535 | } | |
1536 | CL_LOCK_DEBUG(D_DLMTRACE, env, lock, " -> "DDESCR"@"DFID"\n", | |
1537 | PDESCR(desc), PFID(lu_object_fid(&desc->cld_obj->co_lu))); | |
1538 | /* | |
1539 | * Just replace description in place. Nothing more is needed for | |
1540 | * now. If locks were indexed according to their extent and/or mode, | |
1541 | * that index would have to be updated here. | |
1542 | */ | |
1543 | spin_lock(&hdr->coh_lock_guard); | |
1544 | lock->cll_descr = *desc; | |
1545 | spin_unlock(&hdr->coh_lock_guard); | |
0a3bdb00 | 1546 | return 0; |
d7e09d03 PT |
1547 | } |
1548 | EXPORT_SYMBOL(cl_lock_modify); | |
1549 | ||
1550 | /** | |
1551 | * Initializes lock closure with a given origin. | |
1552 | * | |
1553 | * \see cl_lock_closure | |
1554 | */ | |
1555 | void cl_lock_closure_init(const struct lu_env *env, | |
1556 | struct cl_lock_closure *closure, | |
1557 | struct cl_lock *origin, int wait) | |
1558 | { | |
1559 | LINVRNT(cl_lock_is_mutexed(origin)); | |
1560 | LINVRNT(cl_lock_invariant(env, origin)); | |
1561 | ||
1562 | INIT_LIST_HEAD(&closure->clc_list); | |
1563 | closure->clc_origin = origin; | |
1564 | closure->clc_wait = wait; | |
1565 | closure->clc_nr = 0; | |
1566 | } | |
1567 | EXPORT_SYMBOL(cl_lock_closure_init); | |
1568 | ||
1569 | /** | |
1570 | * Builds a closure of \a lock. | |
1571 | * | |
1572 | * Building of a closure consists of adding initial lock (\a lock) into it, | |
1573 | * and calling cl_lock_operations::clo_closure() methods of \a lock. These | |
1574 | * methods might call cl_lock_closure_build() recursively again, adding more | |
1575 | * locks to the closure, etc. | |
1576 | * | |
1577 | * \see cl_lock_closure | |
1578 | */ | |
1579 | int cl_lock_closure_build(const struct lu_env *env, struct cl_lock *lock, | |
1580 | struct cl_lock_closure *closure) | |
1581 | { | |
1582 | const struct cl_lock_slice *slice; | |
1583 | int result; | |
1584 | ||
d7e09d03 PT |
1585 | LINVRNT(cl_lock_is_mutexed(closure->clc_origin)); |
1586 | LINVRNT(cl_lock_invariant(env, closure->clc_origin)); | |
1587 | ||
1588 | result = cl_lock_enclosure(env, lock, closure); | |
1589 | if (result == 0) { | |
1590 | list_for_each_entry(slice, &lock->cll_layers, cls_linkage) { | |
cce3c2da | 1591 | if (slice->cls_ops->clo_closure) { |
d7e09d03 PT |
1592 | result = slice->cls_ops->clo_closure(env, slice, |
1593 | closure); | |
1594 | if (result != 0) | |
1595 | break; | |
1596 | } | |
1597 | } | |
1598 | } | |
1599 | if (result != 0) | |
1600 | cl_lock_disclosure(env, closure); | |
0a3bdb00 | 1601 | return result; |
d7e09d03 PT |
1602 | } |
1603 | EXPORT_SYMBOL(cl_lock_closure_build); | |
1604 | ||
1605 | /** | |
1606 | * Adds new lock to a closure. | |
1607 | * | |
1608 | * Try-locks \a lock and if succeeded, adds it to the closure (never more than | |
1609 | * once). If try-lock failed, returns CLO_REPEAT, after optionally waiting | |
1610 | * until next try-lock is likely to succeed. | |
1611 | */ | |
1612 | int cl_lock_enclosure(const struct lu_env *env, struct cl_lock *lock, | |
1613 | struct cl_lock_closure *closure) | |
1614 | { | |
1615 | int result = 0; | |
29aaf496 | 1616 | |
d7e09d03 PT |
1617 | cl_lock_trace(D_DLMTRACE, env, "enclosure lock", lock); |
1618 | if (!cl_lock_mutex_try(env, lock)) { | |
1619 | /* | |
1620 | * If lock->cll_inclosure is not empty, lock is already in | |
1621 | * this closure. | |
1622 | */ | |
1623 | if (list_empty(&lock->cll_inclosure)) { | |
1624 | cl_lock_get_trust(lock); | |
1625 | lu_ref_add(&lock->cll_reference, "closure", closure); | |
1626 | list_add(&lock->cll_inclosure, &closure->clc_list); | |
1627 | closure->clc_nr++; | |
1628 | } else | |
1629 | cl_lock_mutex_put(env, lock); | |
1630 | result = 0; | |
1631 | } else { | |
1632 | cl_lock_disclosure(env, closure); | |
1633 | if (closure->clc_wait) { | |
1634 | cl_lock_get_trust(lock); | |
1635 | lu_ref_add(&lock->cll_reference, "closure-w", closure); | |
1636 | cl_lock_mutex_put(env, closure->clc_origin); | |
1637 | ||
1638 | LASSERT(cl_lock_nr_mutexed(env) == 0); | |
1639 | cl_lock_mutex_get(env, lock); | |
1640 | cl_lock_mutex_put(env, lock); | |
1641 | ||
1642 | cl_lock_mutex_get(env, closure->clc_origin); | |
1643 | lu_ref_del(&lock->cll_reference, "closure-w", closure); | |
1644 | cl_lock_put(env, lock); | |
1645 | } | |
1646 | result = CLO_REPEAT; | |
1647 | } | |
0a3bdb00 | 1648 | return result; |
d7e09d03 PT |
1649 | } |
1650 | EXPORT_SYMBOL(cl_lock_enclosure); | |
1651 | ||
1652 | /** Releases mutices of enclosed locks. */ | |
1653 | void cl_lock_disclosure(const struct lu_env *env, | |
1654 | struct cl_lock_closure *closure) | |
1655 | { | |
1656 | struct cl_lock *scan; | |
1657 | struct cl_lock *temp; | |
1658 | ||
1659 | cl_lock_trace(D_DLMTRACE, env, "disclosure lock", closure->clc_origin); | |
1660 | list_for_each_entry_safe(scan, temp, &closure->clc_list, | |
1661 | cll_inclosure){ | |
1662 | list_del_init(&scan->cll_inclosure); | |
1663 | cl_lock_mutex_put(env, scan); | |
1664 | lu_ref_del(&scan->cll_reference, "closure", closure); | |
1665 | cl_lock_put(env, scan); | |
1666 | closure->clc_nr--; | |
1667 | } | |
1668 | LASSERT(closure->clc_nr == 0); | |
1669 | } | |
1670 | EXPORT_SYMBOL(cl_lock_disclosure); | |
1671 | ||
1672 | /** Finalizes a closure. */ | |
1673 | void cl_lock_closure_fini(struct cl_lock_closure *closure) | |
1674 | { | |
1675 | LASSERT(closure->clc_nr == 0); | |
1676 | LASSERT(list_empty(&closure->clc_list)); | |
1677 | } | |
1678 | EXPORT_SYMBOL(cl_lock_closure_fini); | |
1679 | ||
1680 | /** | |
1681 | * Destroys this lock. Notifies layers (bottom-to-top) that lock is being | |
1682 | * destroyed, then destroy the lock. If there are holds on the lock, postpone | |
1683 | * destruction until all holds are released. This is called when a decision is | |
1684 | * made to destroy the lock in the future. E.g., when a blocking AST is | |
1685 | * received on it, or fatal communication error happens. | |
1686 | * | |
1687 | * Caller must have a reference on this lock to prevent a situation, when | |
1688 | * deleted lock lingers in memory for indefinite time, because nobody calls | |
1689 | * cl_lock_put() to finish it. | |
1690 | * | |
1691 | * \pre atomic_read(&lock->cll_ref) > 0 | |
1692 | * \pre ergo(cl_lock_nesting(lock) == CNL_TOP, | |
1693 | * cl_lock_nr_mutexed(env) == 1) | |
1694 | * [i.e., if a top-lock is deleted, mutices of no other locks can be | |
1695 | * held, as deletion of sub-locks might require releasing a top-lock | |
1696 | * mutex] | |
1697 | * | |
1698 | * \see cl_lock_operations::clo_delete() | |
1699 | * \see cl_lock::cll_holds | |
1700 | */ | |
1701 | void cl_lock_delete(const struct lu_env *env, struct cl_lock *lock) | |
1702 | { | |
1703 | LINVRNT(cl_lock_is_mutexed(lock)); | |
1704 | LINVRNT(cl_lock_invariant(env, lock)); | |
1705 | LASSERT(ergo(cl_lock_nesting(lock) == CNL_TOP, | |
1706 | cl_lock_nr_mutexed(env) == 1)); | |
1707 | ||
d7e09d03 PT |
1708 | cl_lock_trace(D_DLMTRACE, env, "delete lock", lock); |
1709 | if (lock->cll_holds == 0) | |
1710 | cl_lock_delete0(env, lock); | |
1711 | else | |
1712 | lock->cll_flags |= CLF_DOOMED; | |
d7e09d03 PT |
1713 | } |
1714 | EXPORT_SYMBOL(cl_lock_delete); | |
1715 | ||
1716 | /** | |
1717 | * Mark lock as irrecoverably failed, and mark it for destruction. This | |
1718 | * happens when, e.g., server fails to grant a lock to us, or networking | |
1719 | * time-out happens. | |
1720 | * | |
1721 | * \pre atomic_read(&lock->cll_ref) > 0 | |
1722 | * | |
1723 | * \see clo_lock_delete() | |
1724 | * \see cl_lock::cll_holds | |
1725 | */ | |
1726 | void cl_lock_error(const struct lu_env *env, struct cl_lock *lock, int error) | |
1727 | { | |
1728 | LINVRNT(cl_lock_is_mutexed(lock)); | |
1729 | LINVRNT(cl_lock_invariant(env, lock)); | |
1730 | ||
d7e09d03 PT |
1731 | if (lock->cll_error == 0 && error != 0) { |
1732 | cl_lock_trace(D_DLMTRACE, env, "set lock error", lock); | |
1733 | lock->cll_error = error; | |
1734 | cl_lock_signal(env, lock); | |
1735 | cl_lock_cancel(env, lock); | |
1736 | cl_lock_delete(env, lock); | |
1737 | } | |
d7e09d03 PT |
1738 | } |
1739 | EXPORT_SYMBOL(cl_lock_error); | |
1740 | ||
1741 | /** | |
1742 | * Cancels this lock. Notifies layers | |
1743 | * (bottom-to-top) that lock is being cancelled, then destroy the lock. If | |
1744 | * there are holds on the lock, postpone cancellation until | |
1745 | * all holds are released. | |
1746 | * | |
1747 | * Cancellation notification is delivered to layers at most once. | |
1748 | * | |
1749 | * \see cl_lock_operations::clo_cancel() | |
1750 | * \see cl_lock::cll_holds | |
1751 | */ | |
1752 | void cl_lock_cancel(const struct lu_env *env, struct cl_lock *lock) | |
1753 | { | |
1754 | LINVRNT(cl_lock_is_mutexed(lock)); | |
1755 | LINVRNT(cl_lock_invariant(env, lock)); | |
1756 | ||
d7e09d03 PT |
1757 | cl_lock_trace(D_DLMTRACE, env, "cancel lock", lock); |
1758 | if (lock->cll_holds == 0) | |
1759 | cl_lock_cancel0(env, lock); | |
1760 | else | |
1761 | lock->cll_flags |= CLF_CANCELPEND; | |
d7e09d03 PT |
1762 | } |
1763 | EXPORT_SYMBOL(cl_lock_cancel); | |
1764 | ||
1765 | /** | |
1766 | * Finds an existing lock covering given index and optionally different from a | |
1767 | * given \a except lock. | |
1768 | */ | |
1769 | struct cl_lock *cl_lock_at_pgoff(const struct lu_env *env, | |
1770 | struct cl_object *obj, pgoff_t index, | |
1771 | struct cl_lock *except, | |
1772 | int pending, int canceld) | |
1773 | { | |
1774 | struct cl_object_header *head; | |
1775 | struct cl_lock *scan; | |
1776 | struct cl_lock *lock; | |
1777 | struct cl_lock_descr *need; | |
1778 | ||
d7e09d03 PT |
1779 | head = cl_object_header(obj); |
1780 | need = &cl_env_info(env)->clt_descr; | |
1781 | lock = NULL; | |
1782 | ||
1783 | need->cld_mode = CLM_READ; /* CLM_READ matches both READ & WRITE, but | |
6ba59179 OD |
1784 | * not PHANTOM |
1785 | */ | |
d7e09d03 PT |
1786 | need->cld_start = need->cld_end = index; |
1787 | need->cld_enq_flags = 0; | |
1788 | ||
1789 | spin_lock(&head->coh_lock_guard); | |
1790 | /* It is fine to match any group lock since there could be only one | |
6ba59179 OD |
1791 | * with a uniq gid and it conflicts with all other lock modes too |
1792 | */ | |
d7e09d03 PT |
1793 | list_for_each_entry(scan, &head->coh_locks, cll_linkage) { |
1794 | if (scan != except && | |
1795 | (scan->cll_descr.cld_mode == CLM_GROUP || | |
1796 | cl_lock_ext_match(&scan->cll_descr, need)) && | |
1797 | scan->cll_state >= CLS_HELD && | |
1798 | scan->cll_state < CLS_FREEING && | |
1799 | /* | |
1800 | * This check is racy as the lock can be canceled right | |
1801 | * after it is done, but this is fine, because page exists | |
1802 | * already. | |
1803 | */ | |
1804 | (canceld || !(scan->cll_flags & CLF_CANCELLED)) && | |
1805 | (pending || !(scan->cll_flags & CLF_CANCELPEND))) { | |
1806 | /* Don't increase cs_hit here since this | |
6ba59179 OD |
1807 | * is just a helper function. |
1808 | */ | |
d7e09d03 PT |
1809 | cl_lock_get_trust(scan); |
1810 | lock = scan; | |
1811 | break; | |
1812 | } | |
1813 | } | |
1814 | spin_unlock(&head->coh_lock_guard); | |
0a3bdb00 | 1815 | return lock; |
d7e09d03 PT |
1816 | } |
1817 | EXPORT_SYMBOL(cl_lock_at_pgoff); | |
1818 | ||
1819 | /** | |
1820 | * Calculate the page offset at the layer of @lock. | |
1821 | * At the time of this writing, @page is top page and @lock is sub lock. | |
1822 | */ | |
1823 | static pgoff_t pgoff_at_lock(struct cl_page *page, struct cl_lock *lock) | |
1824 | { | |
1825 | struct lu_device_type *dtype; | |
1826 | const struct cl_page_slice *slice; | |
1827 | ||
1828 | dtype = lock->cll_descr.cld_obj->co_lu.lo_dev->ld_type; | |
1829 | slice = cl_page_at(page, dtype); | |
d7e09d03 PT |
1830 | return slice->cpl_page->cp_index; |
1831 | } | |
1832 | ||
1833 | /** | |
1834 | * Check if page @page is covered by an extra lock or discard it. | |
1835 | */ | |
1836 | static int check_and_discard_cb(const struct lu_env *env, struct cl_io *io, | |
1837 | struct cl_page *page, void *cbdata) | |
1838 | { | |
1839 | struct cl_thread_info *info = cl_env_info(env); | |
1840 | struct cl_lock *lock = cbdata; | |
1841 | pgoff_t index = pgoff_at_lock(page, lock); | |
1842 | ||
1843 | if (index >= info->clt_fn_index) { | |
1844 | struct cl_lock *tmp; | |
1845 | ||
1846 | /* refresh non-overlapped index */ | |
1847 | tmp = cl_lock_at_pgoff(env, lock->cll_descr.cld_obj, index, | |
1848 | lock, 1, 0); | |
cce3c2da | 1849 | if (tmp) { |
d7e09d03 PT |
1850 | /* Cache the first-non-overlapped index so as to skip |
1851 | * all pages within [index, clt_fn_index). This | |
1852 | * is safe because if tmp lock is canceled, it will | |
6ba59179 OD |
1853 | * discard these pages. |
1854 | */ | |
d7e09d03 PT |
1855 | info->clt_fn_index = tmp->cll_descr.cld_end + 1; |
1856 | if (tmp->cll_descr.cld_end == CL_PAGE_EOF) | |
1857 | info->clt_fn_index = CL_PAGE_EOF; | |
1858 | cl_lock_put(env, tmp); | |
1859 | } else if (cl_page_own(env, io, page) == 0) { | |
1860 | /* discard the page */ | |
1861 | cl_page_unmap(env, io, page); | |
1862 | cl_page_discard(env, io, page); | |
1863 | cl_page_disown(env, io, page); | |
1864 | } else { | |
1865 | LASSERT(page->cp_state == CPS_FREEING); | |
1866 | } | |
1867 | } | |
1868 | ||
1869 | info->clt_next_index = index + 1; | |
1870 | return CLP_GANG_OKAY; | |
1871 | } | |
1872 | ||
1873 | static int discard_cb(const struct lu_env *env, struct cl_io *io, | |
1874 | struct cl_page *page, void *cbdata) | |
1875 | { | |
1876 | struct cl_thread_info *info = cl_env_info(env); | |
1877 | struct cl_lock *lock = cbdata; | |
1878 | ||
1879 | LASSERT(lock->cll_descr.cld_mode >= CLM_WRITE); | |
1880 | KLASSERT(ergo(page->cp_type == CPT_CACHEABLE, | |
1881 | !PageWriteback(cl_page_vmpage(env, page)))); | |
1882 | KLASSERT(ergo(page->cp_type == CPT_CACHEABLE, | |
1883 | !PageDirty(cl_page_vmpage(env, page)))); | |
1884 | ||
1885 | info->clt_next_index = pgoff_at_lock(page, lock) + 1; | |
1886 | if (cl_page_own(env, io, page) == 0) { | |
1887 | /* discard the page */ | |
1888 | cl_page_unmap(env, io, page); | |
1889 | cl_page_discard(env, io, page); | |
1890 | cl_page_disown(env, io, page); | |
1891 | } else { | |
1892 | LASSERT(page->cp_state == CPS_FREEING); | |
1893 | } | |
1894 | ||
1895 | return CLP_GANG_OKAY; | |
1896 | } | |
1897 | ||
1898 | /** | |
1899 | * Discard pages protected by the given lock. This function traverses radix | |
1900 | * tree to find all covering pages and discard them. If a page is being covered | |
1901 | * by other locks, it should remain in cache. | |
1902 | * | |
1903 | * If error happens on any step, the process continues anyway (the reasoning | |
1904 | * behind this being that lock cancellation cannot be delayed indefinitely). | |
1905 | */ | |
1906 | int cl_lock_discard_pages(const struct lu_env *env, struct cl_lock *lock) | |
1907 | { | |
1908 | struct cl_thread_info *info = cl_env_info(env); | |
1909 | struct cl_io *io = &info->clt_io; | |
1910 | struct cl_lock_descr *descr = &lock->cll_descr; | |
1911 | cl_page_gang_cb_t cb; | |
1912 | int res; | |
1913 | int result; | |
1914 | ||
1915 | LINVRNT(cl_lock_invariant(env, lock)); | |
d7e09d03 PT |
1916 | |
1917 | io->ci_obj = cl_object_top(descr->cld_obj); | |
1918 | io->ci_ignore_layout = 1; | |
1919 | result = cl_io_init(env, io, CIT_MISC, io->ci_obj); | |
1920 | if (result != 0) | |
d212afd9 | 1921 | goto out; |
d7e09d03 PT |
1922 | |
1923 | cb = descr->cld_mode == CLM_READ ? check_and_discard_cb : discard_cb; | |
1924 | info->clt_fn_index = info->clt_next_index = descr->cld_start; | |
1925 | do { | |
1926 | res = cl_page_gang_lookup(env, descr->cld_obj, io, | |
1927 | info->clt_next_index, descr->cld_end, | |
1928 | cb, (void *)lock); | |
1929 | if (info->clt_next_index > descr->cld_end) | |
1930 | break; | |
1931 | ||
1932 | if (res == CLP_GANG_RESCHED) | |
1933 | cond_resched(); | |
1934 | } while (res != CLP_GANG_OKAY); | |
1935 | out: | |
1936 | cl_io_fini(env, io); | |
0a3bdb00 | 1937 | return result; |
d7e09d03 PT |
1938 | } |
1939 | EXPORT_SYMBOL(cl_lock_discard_pages); | |
1940 | ||
1941 | /** | |
1942 | * Eliminate all locks for a given object. | |
1943 | * | |
1944 | * Caller has to guarantee that no lock is in active use. | |
1945 | * | |
1946 | * \param cancel when this is set, cl_locks_prune() cancels locks before | |
1947 | * destroying. | |
1948 | */ | |
1949 | void cl_locks_prune(const struct lu_env *env, struct cl_object *obj, int cancel) | |
1950 | { | |
1951 | struct cl_object_header *head; | |
1952 | struct cl_lock *lock; | |
1953 | ||
d7e09d03 PT |
1954 | head = cl_object_header(obj); |
1955 | /* | |
1956 | * If locks are destroyed without cancellation, all pages must be | |
1957 | * already destroyed (as otherwise they will be left unprotected). | |
1958 | */ | |
1959 | LASSERT(ergo(!cancel, | |
cce3c2da | 1960 | !head->coh_tree.rnode && head->coh_pages == 0)); |
d7e09d03 PT |
1961 | |
1962 | spin_lock(&head->coh_lock_guard); | |
1963 | while (!list_empty(&head->coh_locks)) { | |
1964 | lock = container_of(head->coh_locks.next, | |
1965 | struct cl_lock, cll_linkage); | |
1966 | cl_lock_get_trust(lock); | |
1967 | spin_unlock(&head->coh_lock_guard); | |
1968 | lu_ref_add(&lock->cll_reference, "prune", current); | |
1969 | ||
1970 | again: | |
1971 | cl_lock_mutex_get(env, lock); | |
1972 | if (lock->cll_state < CLS_FREEING) { | |
1973 | LASSERT(lock->cll_users <= 1); | |
1974 | if (unlikely(lock->cll_users == 1)) { | |
1975 | struct l_wait_info lwi = { 0 }; | |
1976 | ||
1977 | cl_lock_mutex_put(env, lock); | |
1978 | l_wait_event(lock->cll_wq, | |
1979 | lock->cll_users == 0, | |
1980 | &lwi); | |
1981 | goto again; | |
1982 | } | |
1983 | ||
1984 | if (cancel) | |
1985 | cl_lock_cancel(env, lock); | |
1986 | cl_lock_delete(env, lock); | |
1987 | } | |
1988 | cl_lock_mutex_put(env, lock); | |
1989 | lu_ref_del(&lock->cll_reference, "prune", current); | |
1990 | cl_lock_put(env, lock); | |
1991 | spin_lock(&head->coh_lock_guard); | |
1992 | } | |
1993 | spin_unlock(&head->coh_lock_guard); | |
d7e09d03 PT |
1994 | } |
1995 | EXPORT_SYMBOL(cl_locks_prune); | |
1996 | ||
1997 | static struct cl_lock *cl_lock_hold_mutex(const struct lu_env *env, | |
1998 | const struct cl_io *io, | |
1999 | const struct cl_lock_descr *need, | |
2000 | const char *scope, const void *source) | |
2001 | { | |
2002 | struct cl_lock *lock; | |
2003 | ||
d7e09d03 PT |
2004 | while (1) { |
2005 | lock = cl_lock_find(env, io, need); | |
2006 | if (IS_ERR(lock)) | |
2007 | break; | |
2008 | cl_lock_mutex_get(env, lock); | |
2009 | if (lock->cll_state < CLS_FREEING && | |
2010 | !(lock->cll_flags & CLF_CANCELLED)) { | |
b2952d62 | 2011 | cl_lock_hold_mod(env, lock, 1); |
d7e09d03 PT |
2012 | lu_ref_add(&lock->cll_holders, scope, source); |
2013 | lu_ref_add(&lock->cll_reference, scope, source); | |
2014 | break; | |
2015 | } | |
2016 | cl_lock_mutex_put(env, lock); | |
2017 | cl_lock_put(env, lock); | |
2018 | } | |
0a3bdb00 | 2019 | return lock; |
d7e09d03 PT |
2020 | } |
2021 | ||
2022 | /** | |
2023 | * Returns a lock matching \a need description with a reference and a hold on | |
2024 | * it. | |
2025 | * | |
2026 | * This is much like cl_lock_find(), except that cl_lock_hold() additionally | |
2027 | * guarantees that lock is not in the CLS_FREEING state on return. | |
2028 | */ | |
2029 | struct cl_lock *cl_lock_hold(const struct lu_env *env, const struct cl_io *io, | |
2030 | const struct cl_lock_descr *need, | |
2031 | const char *scope, const void *source) | |
2032 | { | |
2033 | struct cl_lock *lock; | |
2034 | ||
d7e09d03 PT |
2035 | lock = cl_lock_hold_mutex(env, io, need, scope, source); |
2036 | if (!IS_ERR(lock)) | |
2037 | cl_lock_mutex_put(env, lock); | |
0a3bdb00 | 2038 | return lock; |
d7e09d03 PT |
2039 | } |
2040 | EXPORT_SYMBOL(cl_lock_hold); | |
2041 | ||
2042 | /** | |
2043 | * Main high-level entry point of cl_lock interface that finds existing or | |
2044 | * enqueues new lock matching given description. | |
2045 | */ | |
2046 | struct cl_lock *cl_lock_request(const struct lu_env *env, struct cl_io *io, | |
2047 | const struct cl_lock_descr *need, | |
2048 | const char *scope, const void *source) | |
2049 | { | |
2050 | struct cl_lock *lock; | |
2051 | int rc; | |
2052 | __u32 enqflags = need->cld_enq_flags; | |
2053 | ||
d7e09d03 PT |
2054 | do { |
2055 | lock = cl_lock_hold_mutex(env, io, need, scope, source); | |
2056 | if (IS_ERR(lock)) | |
2057 | break; | |
2058 | ||
2059 | rc = cl_enqueue_locked(env, lock, io, enqflags); | |
2060 | if (rc == 0) { | |
2061 | if (cl_lock_fits_into(env, lock, need, io)) { | |
2062 | if (!(enqflags & CEF_AGL)) { | |
2063 | cl_lock_mutex_put(env, lock); | |
2064 | cl_lock_lockdep_acquire(env, lock, | |
2065 | enqflags); | |
2066 | break; | |
2067 | } | |
2068 | rc = 1; | |
2069 | } | |
2070 | cl_unuse_locked(env, lock); | |
2071 | } | |
2072 | cl_lock_trace(D_DLMTRACE, env, | |
2073 | rc <= 0 ? "enqueue failed" : "agl succeed", lock); | |
2074 | cl_lock_hold_release(env, lock, scope, source); | |
2075 | cl_lock_mutex_put(env, lock); | |
2076 | lu_ref_del(&lock->cll_reference, scope, source); | |
2077 | cl_lock_put(env, lock); | |
2078 | if (rc > 0) { | |
2079 | LASSERT(enqflags & CEF_AGL); | |
2080 | lock = NULL; | |
2081 | } else if (rc != 0) { | |
2082 | lock = ERR_PTR(rc); | |
2083 | } | |
2084 | } while (rc == 0); | |
0a3bdb00 | 2085 | return lock; |
d7e09d03 PT |
2086 | } |
2087 | EXPORT_SYMBOL(cl_lock_request); | |
2088 | ||
2089 | /** | |
2090 | * Adds a hold to a known lock. | |
2091 | */ | |
2092 | void cl_lock_hold_add(const struct lu_env *env, struct cl_lock *lock, | |
2093 | const char *scope, const void *source) | |
2094 | { | |
2095 | LINVRNT(cl_lock_is_mutexed(lock)); | |
2096 | LINVRNT(cl_lock_invariant(env, lock)); | |
2097 | LASSERT(lock->cll_state != CLS_FREEING); | |
2098 | ||
b2952d62 | 2099 | cl_lock_hold_mod(env, lock, 1); |
d7e09d03 PT |
2100 | cl_lock_get(lock); |
2101 | lu_ref_add(&lock->cll_holders, scope, source); | |
2102 | lu_ref_add(&lock->cll_reference, scope, source); | |
d7e09d03 PT |
2103 | } |
2104 | EXPORT_SYMBOL(cl_lock_hold_add); | |
2105 | ||
2106 | /** | |
2107 | * Releases a hold and a reference on a lock, on which caller acquired a | |
2108 | * mutex. | |
2109 | */ | |
2110 | void cl_lock_unhold(const struct lu_env *env, struct cl_lock *lock, | |
2111 | const char *scope, const void *source) | |
2112 | { | |
2113 | LINVRNT(cl_lock_invariant(env, lock)); | |
d7e09d03 PT |
2114 | cl_lock_hold_release(env, lock, scope, source); |
2115 | lu_ref_del(&lock->cll_reference, scope, source); | |
2116 | cl_lock_put(env, lock); | |
d7e09d03 PT |
2117 | } |
2118 | EXPORT_SYMBOL(cl_lock_unhold); | |
2119 | ||
2120 | /** | |
2121 | * Releases a hold and a reference on a lock, obtained by cl_lock_hold(). | |
2122 | */ | |
2123 | void cl_lock_release(const struct lu_env *env, struct cl_lock *lock, | |
2124 | const char *scope, const void *source) | |
2125 | { | |
2126 | LINVRNT(cl_lock_invariant(env, lock)); | |
d7e09d03 PT |
2127 | cl_lock_trace(D_DLMTRACE, env, "release lock", lock); |
2128 | cl_lock_mutex_get(env, lock); | |
2129 | cl_lock_hold_release(env, lock, scope, source); | |
2130 | cl_lock_mutex_put(env, lock); | |
2131 | lu_ref_del(&lock->cll_reference, scope, source); | |
2132 | cl_lock_put(env, lock); | |
d7e09d03 PT |
2133 | } |
2134 | EXPORT_SYMBOL(cl_lock_release); | |
2135 | ||
2136 | void cl_lock_user_add(const struct lu_env *env, struct cl_lock *lock) | |
2137 | { | |
2138 | LINVRNT(cl_lock_is_mutexed(lock)); | |
2139 | LINVRNT(cl_lock_invariant(env, lock)); | |
2140 | ||
b2952d62 | 2141 | cl_lock_used_mod(env, lock, 1); |
d7e09d03 PT |
2142 | } |
2143 | EXPORT_SYMBOL(cl_lock_user_add); | |
2144 | ||
2145 | void cl_lock_user_del(const struct lu_env *env, struct cl_lock *lock) | |
2146 | { | |
2147 | LINVRNT(cl_lock_is_mutexed(lock)); | |
2148 | LINVRNT(cl_lock_invariant(env, lock)); | |
2149 | LASSERT(lock->cll_users > 0); | |
2150 | ||
d7e09d03 PT |
2151 | cl_lock_used_mod(env, lock, -1); |
2152 | if (lock->cll_users == 0) | |
2153 | wake_up_all(&lock->cll_wq); | |
d7e09d03 PT |
2154 | } |
2155 | EXPORT_SYMBOL(cl_lock_user_del); | |
2156 | ||
2157 | const char *cl_lock_mode_name(const enum cl_lock_mode mode) | |
2158 | { | |
2159 | static const char *names[] = { | |
2160 | [CLM_PHANTOM] = "P", | |
2161 | [CLM_READ] = "R", | |
2162 | [CLM_WRITE] = "W", | |
2163 | [CLM_GROUP] = "G" | |
2164 | }; | |
2165 | if (0 <= mode && mode < ARRAY_SIZE(names)) | |
2166 | return names[mode]; | |
2167 | else | |
2168 | return "U"; | |
2169 | } | |
2170 | EXPORT_SYMBOL(cl_lock_mode_name); | |
2171 | ||
2172 | /** | |
2173 | * Prints human readable representation of a lock description. | |
2174 | */ | |
2175 | void cl_lock_descr_print(const struct lu_env *env, void *cookie, | |
2176 | lu_printer_t printer, | |
2177 | const struct cl_lock_descr *descr) | |
2178 | { | |
2179 | const struct lu_fid *fid; | |
2180 | ||
2181 | fid = lu_object_fid(&descr->cld_obj->co_lu); | |
2182 | (*printer)(env, cookie, DDESCR"@"DFID, PDESCR(descr), PFID(fid)); | |
2183 | } | |
2184 | EXPORT_SYMBOL(cl_lock_descr_print); | |
2185 | ||
2186 | /** | |
2187 | * Prints human readable representation of \a lock to the \a f. | |
2188 | */ | |
2189 | void cl_lock_print(const struct lu_env *env, void *cookie, | |
2190 | lu_printer_t printer, const struct cl_lock *lock) | |
2191 | { | |
2192 | const struct cl_lock_slice *slice; | |
2193 | (*printer)(env, cookie, "lock@%p[%d %d %d %d %d %08lx] ", | |
2194 | lock, atomic_read(&lock->cll_ref), | |
2195 | lock->cll_state, lock->cll_error, lock->cll_holds, | |
2196 | lock->cll_users, lock->cll_flags); | |
2197 | cl_lock_descr_print(env, cookie, printer, &lock->cll_descr); | |
2198 | (*printer)(env, cookie, " {\n"); | |
2199 | ||
2200 | list_for_each_entry(slice, &lock->cll_layers, cls_linkage) { | |
2201 | (*printer)(env, cookie, " %s@%p: ", | |
2202 | slice->cls_obj->co_lu.lo_dev->ld_type->ldt_name, | |
2203 | slice); | |
cce3c2da | 2204 | if (slice->cls_ops->clo_print) |
d7e09d03 PT |
2205 | slice->cls_ops->clo_print(env, cookie, printer, slice); |
2206 | (*printer)(env, cookie, "\n"); | |
2207 | } | |
2208 | (*printer)(env, cookie, "} lock@%p\n", lock); | |
2209 | } | |
2210 | EXPORT_SYMBOL(cl_lock_print); | |
2211 | ||
2212 | int cl_lock_init(void) | |
2213 | { | |
2214 | return lu_kmem_init(cl_lock_caches); | |
2215 | } | |
2216 | ||
2217 | void cl_lock_fini(void) | |
2218 | { | |
2219 | lu_kmem_fini(cl_lock_caches); | |
2220 | } |