2 * Copyright (c) 2009 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 #ifndef _SYS_MUTEX2_H_
36 #define _SYS_MUTEX2_H_
39 #include <sys/mutex.h>
41 #ifndef _SYS_THREAD2_H_
42 #include <sys/thread2.h>
44 #ifndef _SYS_GLOBALDATA_H_
45 #include <sys/globaldata.h>
47 #include <machine/atomic.h>
50 * Initialize a new mutex, placing it in an unlocked state with no refs.
53 mtx_init(mtx_t *mtx, const char *ident)
56 mtx->mtx_owner = NULL;
57 mtx->mtx_exlink = NULL;
58 mtx->mtx_shlink = NULL;
59 mtx->mtx_ident = ident;
63 * Initialize a mtx link structure for deeper control over the mutex
67 mtx_link_init(mtx_link_t *link)
69 link->state = MTX_LINK_IDLE;
70 link->callback = NULL;
75 * A link structure initialized this way causes mutex operations to not block,
76 * caller must specify a callback. Caller may still abort the mutex via
80 mtx_link_init_async(mtx_link_t *link,
81 void (*callback)(mtx_link_t *link, void *arg, int error),
84 link->state = MTX_LINK_IDLE;
85 link->callback = callback;
90 * Deinitialize a mutex
93 mtx_uninit(mtx_t *mtx)
99 * Exclusive-lock a mutex, block until acquired or aborted. Recursion
102 * This version of the function allows the mtx_link to be passed in, thus
103 * giving the caller visibility for the link structure which is required
104 * when calling mtx_abort_ex_link() or when requesting an asynchronous lock.
106 * The mutex may be aborted at any time while the passed link structure
110 mtx_lock_ex_link(mtx_t *mtx, mtx_link_t *link, int flags, int to)
112 if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0)
113 return(_mtx_lock_ex_link(mtx, link, flags, to));
114 mtx->mtx_owner = curthread;
115 link->state = MTX_LINK_ACQUIRED;
121 * Short-form exclusive-lock a mutex, block until acquired. Recursion is
122 * allowed. This is equivalent to mtx_lock_ex(mtx, "mtxex", 0, 0).
127 if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0) {
128 _mtx_lock_ex(mtx, 0, 0);
131 mtx->mtx_owner = curthread;
135 * Exclusive-lock a mutex, block until acquired. Recursion is allowed.
137 * Returns 0 on success, or the tsleep() return code on failure.
138 * An error can only be returned if PCATCH is specified in the flags.
141 mtx_lock_ex(mtx_t *mtx, int flags, int to)
143 if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0)
144 return(_mtx_lock_ex(mtx, flags, to));
145 mtx->mtx_owner = curthread;
150 mtx_lock_ex_quick(mtx_t *mtx)
152 if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0)
153 return(_mtx_lock_ex_quick(mtx));
154 mtx->mtx_owner = curthread;
159 mtx_lock_sh_link(mtx_t *mtx, mtx_link_t *link, int flags, int to)
161 if (atomic_cmpset_int(&mtx->mtx_lock, 0, 1) == 0)
162 return(_mtx_lock_sh_link(mtx, link, flags, to));
163 link->state = MTX_LINK_ACQUIRED;
168 * Share-lock a mutex, block until acquired. Recursion is allowed.
170 * Returns 0 on success, or the tsleep() return code on failure.
171 * An error can only be returned if PCATCH is specified in the flags.
174 mtx_lock_sh(mtx_t *mtx, int flags, int to)
176 if (atomic_cmpset_int(&mtx->mtx_lock, 0, 1) == 0)
177 return(_mtx_lock_sh(mtx, flags, to));
182 mtx_lock_sh_quick(mtx_t *mtx)
184 if (atomic_cmpset_int(&mtx->mtx_lock, 0, 1) == 0)
185 return(_mtx_lock_sh_quick(mtx));
190 * Adds a shared lock reference to a lock already locked shared,
191 * does not block on pending exclusive request.
194 mtx_lock_sh_again(mtx_t *mtx)
196 KKASSERT((mtx->mtx_lock & MTX_EXCLUSIVE) == 0 &&
197 (mtx->mtx_lock & MTX_MASK) > 0);
198 atomic_add_int(&mtx->mtx_lock, 1);
202 * Short-form exclusive spinlock a mutex. Must be paired with
206 mtx_spinlock(mtx_t *mtx)
208 globaldata_t gd = mycpu;
211 * Predispose a hard critical section
213 ++gd->gd_curthread->td_critcount;
218 * If we cannot get it trivially get it the hard way.
220 * Note that mtx_owner will be set twice if we fail to get it
221 * trivially, but there's no point conditionalizing it as a
222 * conditional will be slower.
224 if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0)
226 mtx->mtx_owner = gd->gd_curthread;
230 mtx_spinlock_try(mtx_t *mtx)
232 globaldata_t gd = mycpu;
235 * Predispose a hard critical section
237 ++gd->gd_curthread->td_critcount;
242 * If we cannot get it trivially call _mtx_spinlock_try(). This
243 * function will clean up the hard critical section if it fails.
245 if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0)
246 return(_mtx_spinlock_try(mtx));
247 mtx->mtx_owner = gd->gd_curthread;
252 * Short-form exclusive-lock a mutex, spin until acquired. Recursion is
253 * allowed. This form is identical to mtx_spinlock_ex().
255 * Attempt to exclusive-lock a mutex, return 0 on success and
259 mtx_lock_ex_try(mtx_t *mtx)
261 if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0)
262 return (_mtx_lock_ex_try(mtx));
263 mtx->mtx_owner = curthread;
268 * Attempt to share-lock a mutex, return 0 on success and
272 mtx_lock_sh_try(mtx_t *mtx)
274 if (atomic_cmpset_int(&mtx->mtx_lock, 0, 1) == 0)
275 return (_mtx_lock_sh_try(mtx));
280 * If the lock is held exclusively it must be owned by the caller. If the
281 * lock is already a shared lock this operation is a NOP. A panic will
282 * occur if the lock is not held either shared or exclusive.
284 * The exclusive count is converted to a shared count.
287 mtx_downgrade(mtx_t *mtx)
289 mtx->mtx_owner = NULL;
290 if (atomic_cmpset_int(&mtx->mtx_lock, MTX_EXCLUSIVE | 1, 1) == 0)
295 * Upgrade a shared lock to an exclusive lock. The upgrade will fail if
296 * the shared lock has a count other then 1. Optimize the most likely case
297 * but note that a single cmpset can fail due to WANTED races.
299 * If the lock is held exclusively it must be owned by the caller and
300 * this function will simply return without doing anything. A panic will
301 * occur if the lock is held exclusively by someone other then the caller.
303 * Returns 0 on success, EDEADLK on failure.
306 mtx_upgrade_try(mtx_t *mtx)
308 if (atomic_cmpset_int(&mtx->mtx_lock, 1, MTX_EXCLUSIVE | 1))
310 return (_mtx_upgrade_try(mtx));
314 * Optimized unlock cases.
316 * NOTE: mtx_unlock() handles any type of mutex: exclusive, shared, and
317 * both blocking and spin methods.
319 * The mtx_unlock_ex/sh() forms are optimized for exclusive or shared
320 * mutexes and produce less code, but it is ok for code to just use
321 * mtx_unlock() and, in fact, if code uses the short-form mtx_lock()
322 * or mtx_spinlock() to lock it should also use mtx_unlock() to unlock.
325 mtx_unlock(mtx_t *mtx)
327 u_int lock = mtx->mtx_lock;
329 if (lock == (MTX_EXCLUSIVE | 1)) {
330 mtx->mtx_owner = NULL;
331 if (atomic_cmpset_int(&mtx->mtx_lock, lock, 0) == 0)
333 } else if (lock == 1) {
334 if (atomic_cmpset_int(&mtx->mtx_lock, lock, 0) == 0)
342 mtx_unlock_ex(mtx_t *mtx)
344 u_int lock = mtx->mtx_lock;
346 if (lock == (MTX_EXCLUSIVE | 1)) {
347 mtx->mtx_owner = NULL;
348 if (atomic_cmpset_int(&mtx->mtx_lock, lock, 0) == 0)
356 mtx_unlock_sh(mtx_t *mtx)
358 if (atomic_cmpset_int(&mtx->mtx_lock, 1, 0) == 0)
363 * NOTE: spinlocks are exclusive-only
366 mtx_spinunlock(mtx_t *mtx)
368 globaldata_t gd = mycpu;
374 --gd->gd_curthread->td_critcount;
378 * Return TRUE (non-zero) if the mutex is locked shared or exclusive by
379 * anyone, including the owner.
382 mtx_islocked(mtx_t *mtx)
384 return(mtx->mtx_lock != 0);
388 * Return TRUE (non-zero) if the mutex is locked exclusively by anyone,
389 * including the owner. Returns FALSE (0) if the mutex is unlocked or
390 * if it is locked shared by one or more entities.
392 * A caller wishing to check whether a lock is owned exclusively by it
393 * should use mtx_owned().
396 mtx_islocked_ex(mtx_t *mtx)
398 return((mtx->mtx_lock & MTX_EXCLUSIVE) != 0);
402 * Return TRUE (non-zero) if the mutex is not locked.
405 mtx_notlocked(mtx_t *mtx)
407 return(mtx->mtx_lock == 0);
411 * Return TRUE (non-zero) if the mutex is not locked exclusively.
412 * The mutex may in an unlocked or shared lock state.
415 mtx_notlocked_ex(mtx_t *mtx)
417 return((mtx->mtx_lock & MTX_EXCLUSIVE) != 0);
421 * Return TRUE (non-zero) if the mutex is exclusively locked by
425 mtx_owned(mtx_t *mtx)
427 return((mtx->mtx_lock & MTX_EXCLUSIVE) && mtx->mtx_owner == curthread);
431 * Return TRUE (non-zero) if the mutex is not exclusively locked by
435 mtx_notowned(mtx_t *mtx)
437 return((mtx->mtx_lock & MTX_EXCLUSIVE) == 0 ||
438 mtx->mtx_owner != curthread);
442 * Return the shared or exclusive lock count. A return value of 0
443 * indicate that the mutex is not locked.
445 * NOTE: If the mutex is held exclusively by someone other then the
446 * caller the lock count for the other owner is still returned.
450 mtx_lockrefs(mtx_t *mtx)
452 return(mtx->mtx_lock & MTX_MASK);
456 * Lock must held and will be released on return. Returns state
457 * which can be passed to mtx_lock_temp_restore() to return the
458 * lock to its previous state.
462 mtx_lock_temp_release(mtx_t *mtx)
466 state = (mtx->mtx_lock & MTX_EXCLUSIVE);
473 * Restore the previous state of a lock released with
474 * mtx_lock_temp_release() or mtx_lock_upgrade().
478 mtx_lock_temp_restore(mtx_t *mtx, mtx_state_t state)
480 if (state & MTX_EXCLUSIVE)
481 mtx_lock_ex_quick(mtx);
483 mtx_lock_sh_quick(mtx);