From d666840ad3312b8c37863f61c8ee3139682230ed Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Sun, 21 May 2006 20:23:29 +0000 Subject: [PATCH] Implement a much faster spinlock. * Spinlocks can't conflict with FAST interrupts without deadlocking anyway, so instead of using a critical section simply do not allow an interrupt thread to preempt the current thread if it is holding a spinlock. This cuts spinlock overhead in half. * Implement shared spinlocks in addition to exclusive spinlocks. Shared spinlocks would be used, e.g. for file descriptor table lookups. * Cache a shared spinlock by using the spinlock's lock field as a bitfield, one for each cpu (bit 31 for exclusive locks). A shared spinlock sets its cpu's shared bit and does not bother clearing it on unlock. This means that multiple, parallel shared spinlock accessors do NOT incur a cache conflict on the spinlock. ALL parallel shared accessors operate at full speed (~10ns vs ~40-100ns in overhead). 90% of the 10ns in overhead is due to a necessary MFENCE to interlock against exclusive spinlocks on the mutex. However, this MFENCE only has to play with pending cpu-local memory writes so it will always run at near full speed. * Exclusive spinlocks in the face of previously cached shared spinlocks are now slightly more expensive because they have to clear the cached shared spinlock bits by checking the globaldata structure for each conflicting cpu to see if it is still holding a shared spinlock. However, only the initial (unavoidable) atomic swap involves potential cache conflicts. The shared bit checks involve only memory reads and the situation should be self-correcting from a performance standpoint since the shared bits then get cleared. * Add sysctl's for basic spinlock performance testing. Setting debug.spin_lock_test issues a test. Tests #2 and #3 loop debug.spin_test_count times. p.s. these tests will stall the whole machine. 1 Test the indefinite wait code 2 Time the best-case exclusive lock overhead 3 Time the best-case shared lock overhead * TODO: A shared->exclusive spinlock upgrade inline with positive feedback, and an exclusive->shared spinlock downgrade inline. --- sys/kern/kern_ktr.c | 16 +- sys/kern/kern_lock.c | 32 ++-- sys/kern/kern_spinlock.c | 281 ++++++++++++++++++++++++++++-------- sys/kern/lwkt_rwlock.c | 30 ++-- sys/kern/lwkt_thread.c | 42 +++--- sys/kern/lwkt_token.c | 14 +- sys/netproto/smb/smb_subr.h | 6 +- sys/sys/globaldata.h | 6 +- sys/sys/spinlock.h | 4 +- sys/sys/spinlock2.h | 214 ++++++++++++++++----------- sys/sys/thread.h | 4 +- sys/vfs/ntfs/ntfs_subr.c | 18 +-- sys/vm/vm_zone.c | 16 +- 13 files changed, 454 insertions(+), 229 deletions(-) diff --git a/sys/kern/kern_ktr.c b/sys/kern/kern_ktr.c index 0508c9f475..5d80e0a4b8 100644 --- a/sys/kern/kern_ktr.c +++ b/sys/kern/kern_ktr.c @@ -62,7 +62,7 @@ * SUCH DAMAGE. */ /* - * $DragonFly: src/sys/kern/kern_ktr.c,v 1.13 2006/03/25 21:28:07 swildner Exp $ + * $DragonFly: src/sys/kern/kern_ktr.c,v 1.14 2006/05/21 20:23:25 dillon Exp $ */ /* * Kernel tracepoint facility. @@ -280,12 +280,18 @@ ktr_resync_callback(void *dummy __unused) struct spinlock spin; spin_init(&spin); - spin_lock_quick(&spin); - spin_unlock_quick(&spin); + spin_lock_wr(&spin); + spin_unlock_wr(&spin); logtest_noargs(spin_beg); for (count = ktr_testspincnt; count; --count) { - spin_lock_quick(&spin); - spin_unlock_quick(&spin); + spin_lock_wr(&spin); + spin_unlock_wr(&spin); + } + logtest_noargs(spin_end); + logtest_noargs(spin_beg); + for (count = ktr_testspincnt; count; --count) { + spin_lock_rd(&spin); + spin_unlock_rd(&spin); } logtest_noargs(spin_end); ktr_testspincnt = 0; diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c index ab6e805e33..53561d93f7 100644 --- a/sys/kern/kern_lock.c +++ b/sys/kern/kern_lock.c @@ -39,7 +39,7 @@ * * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 * $FreeBSD: src/sys/kern/kern_lock.c,v 1.31.2.3 2001/12/25 01:44:44 dillon Exp $ - * $DragonFly: src/sys/kern/kern_lock.c,v 1.20 2006/05/07 00:51:11 dillon Exp $ + * $DragonFly: src/sys/kern/kern_lock.c,v 1.21 2006/05/21 20:23:25 dillon Exp $ */ #include "opt_lint.h" @@ -135,12 +135,12 @@ acquire(struct lock *lkp, int extflags, int wanted) * happening here. */ tsleep_interlock(lkp); - spin_unlock_quick(&lkp->lk_spinlock); + spin_unlock_wr(&lkp->lk_spinlock); error = tsleep(lkp, ((extflags & LK_PCATCH) ? PCATCH : 0), lkp->lk_wmesg, ((extflags & LK_TIMELOCK) ? lkp->lk_timo : 0)); - spin_lock_quick(&lkp->lk_spinlock); + spin_lock_wr(&lkp->lk_spinlock); if (lkp->lk_waitcount == 1) { lkp->lk_flags &= ~LK_WAIT_NONZERO; lkp->lk_waitcount = 0; @@ -210,7 +210,7 @@ debuglockmgr(struct lock *lkp, u_int flags, #endif } - spin_lock(&lkp->lk_spinlock); + spin_lock_wr(&lkp->lk_spinlock); extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; td = curthread; @@ -257,7 +257,7 @@ debuglockmgr(struct lock *lkp, u_int flags, case LK_DOWNGRADE: if (lkp->lk_lockholder != td || lkp->lk_exclusivecount == 0) { - spin_unlock(&lkp->lk_spinlock); + spin_unlock_wr(&lkp->lk_spinlock); panic("lockmgr: not holding exclusive lock"); } sharelock(lkp, lkp->lk_exclusivecount); @@ -292,7 +292,7 @@ debuglockmgr(struct lock *lkp, u_int flags, * will always be unlocked. */ if ((lkp->lk_lockholder == td) || (lkp->lk_sharecount <= 0)) { - spin_unlock(&lkp->lk_spinlock); + spin_unlock_wr(&lkp->lk_spinlock); panic("lockmgr: upgrade exclusive lock"); } shareunlock(lkp, 1); @@ -321,7 +321,7 @@ debuglockmgr(struct lock *lkp, u_int flags, lkp->lk_flags |= LK_HAVE_EXCL; lkp->lk_lockholder = td; if (lkp->lk_exclusivecount != 0) { - spin_unlock(&lkp->lk_spinlock); + spin_unlock_wr(&lkp->lk_spinlock); panic("lockmgr: non-zero exclusive count"); } lkp->lk_exclusivecount = 1; @@ -349,7 +349,7 @@ debuglockmgr(struct lock *lkp, u_int flags, * Recursive lock. */ if ((extflags & (LK_NOWAIT | LK_CANRECURSE)) == 0) { - spin_unlock(&lkp->lk_spinlock); + spin_unlock_wr(&lkp->lk_spinlock); panic("lockmgr: locking against myself"); } if ((extflags & LK_CANRECURSE) != 0) { @@ -383,7 +383,7 @@ debuglockmgr(struct lock *lkp, u_int flags, lkp->lk_flags |= LK_HAVE_EXCL; lkp->lk_lockholder = td; if (lkp->lk_exclusivecount != 0) { - spin_unlock(&lkp->lk_spinlock); + spin_unlock_wr(&lkp->lk_spinlock); panic("lockmgr: non-zero exclusive count"); } lkp->lk_exclusivecount = 1; @@ -399,7 +399,7 @@ debuglockmgr(struct lock *lkp, u_int flags, if (lkp->lk_exclusivecount != 0) { if (lkp->lk_lockholder != td && lkp->lk_lockholder != LK_KERNTHREAD) { - spin_unlock(&lkp->lk_spinlock); + spin_unlock_wr(&lkp->lk_spinlock); panic("lockmgr: pid %d, not %s thr %p unlocking", (td->td_proc ? td->td_proc->p_pid : -99), "exclusive lock holder", @@ -424,12 +424,12 @@ debuglockmgr(struct lock *lkp, u_int flags, break; default: - spin_unlock(&lkp->lk_spinlock); + spin_unlock_wr(&lkp->lk_spinlock); panic("lockmgr: unknown locktype request %d", flags & LK_TYPE_MASK); /* NOTREACHED */ } - spin_unlock(&lkp->lk_spinlock); + spin_unlock_wr(&lkp->lk_spinlock); return (error); } @@ -484,7 +484,7 @@ lockstatus(struct lock *lkp, struct thread *td) { int lock_type = 0; - spin_lock(&lkp->lk_spinlock); + spin_lock_wr(&lkp->lk_spinlock); if (lkp->lk_exclusivecount != 0) { if (td == NULL || lkp->lk_lockholder == td) lock_type = LK_EXCLUSIVE; @@ -493,7 +493,7 @@ lockstatus(struct lock *lkp, struct thread *td) } else if (lkp->lk_sharecount != 0) { lock_type = LK_SHARED; } - spin_unlock(&lkp->lk_spinlock); + spin_unlock_wr(&lkp->lk_spinlock); return (lock_type); } @@ -507,9 +507,9 @@ lockcount(struct lock *lkp) { int count; - spin_lock(&lkp->lk_spinlock); + spin_lock_wr(&lkp->lk_spinlock); count = lkp->lk_exclusivecount + lkp->lk_sharecount; - spin_unlock(&lkp->lk_spinlock); + spin_unlock_wr(&lkp->lk_spinlock); return (count); } diff --git a/sys/kern/kern_spinlock.c b/sys/kern/kern_spinlock.c index 9ae3964433..fc80fe51f4 100644 --- a/sys/kern/kern_spinlock.c +++ b/sys/kern/kern_spinlock.c @@ -2,7 +2,7 @@ * Copyright (c) 2005 Jeffrey M. Hsu. All rights reserved. * * This code is derived from software contributed to The DragonFly Project - * by Jeffrey M. Hsu. + * by Jeffrey M. Hsu. and Matthew Dillon * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -29,7 +29,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/kern/kern_spinlock.c,v 1.3 2006/05/18 17:53:45 dillon Exp $ + * $DragonFly: src/sys/kern/kern_spinlock.c,v 1.4 2006/05/21 20:23:25 dillon Exp $ */ #include @@ -52,8 +52,6 @@ #ifdef SMP -static void spin_lock_contested_long(struct spinlock *mtx); - #ifdef INVARIANTS static int spin_lock_test_mode; #endif @@ -63,102 +61,271 @@ SYSCTL_QUAD(_debug, OID_AUTO, spinlocks_contested1, CTLFLAG_RD, &spinlocks_conte static int64_t spinlocks_contested2; SYSCTL_QUAD(_debug, OID_AUTO, spinlocks_contested2, CTLFLAG_RD, &spinlocks_contested2, 0, ""); +struct exponential_backoff { + int backoff; + int nsec; + struct spinlock *mtx; + sysclock_t base; +}; +static int exponential_backoff(struct exponential_backoff *bo); + +static __inline void -spin_lock_contested(struct spinlock *mtx) +exponential_init(struct exponential_backoff *bo, struct spinlock *mtx) { - int i; - int backoff = BACKOFF_INITIAL; + bo->backoff = BACKOFF_INITIAL; + bo->nsec = 0; + bo->mtx = mtx; +} + +/* + * We were either contested due to another exclusive lock holder, + * or due to the presence of shared locks. We have to undo the mess + * we created by returning the shared locks. + * + * If there was another exclusive lock holder only the exclusive bit + * in value will be the only bit set. We don't have to do anything since + * restoration does not involve any work. + * + * Otherwise we successfully obtained the exclusive bit. Attempt to + * clear the shared bits. If we are able to clear the shared bits + * we win. Otherwise we lose and we have to restore the shared bits + * we couldn't clear (and also clear our exclusive bit). + */ +int +spin_trylock_wr_contested(struct spinlock *mtx, int value) +{ + int bit; ++spinlocks_contested1; - do { - /* exponential backoff to reduce contention */ - for (i = 0; i < backoff; i++) - cpu_nop(); - if (backoff < BACKOFF_LIMIT) { - backoff <<= 1; - } else { - spin_lock_contested_long(mtx); - return; + if ((value & SPINLOCK_EXCLUSIVE) == 0) { + while (value) { + bit = bsfl(value); + if (globaldata_find(bit)->gd_spinlocks_rd != 0) { + atomic_swap_int(&mtx->lock, value); + return (FALSE); + } + value &= ~(1 << bit); } - /* do non-bus-locked check first */ - } while (mtx->lock != 0 || atomic_swap_int(&mtx->lock, 1) != 0); + return (TRUE); + } + return (FALSE); } +/* + * We were either contested due to another exclusive lock holder, + * or due to the presence of shared locks + * + * NOTE: If value indicates an exclusively held mutex, no shared bits + * would have been set and we can throw away value. + */ +void +spin_lock_wr_contested(struct spinlock *mtx, int value) +{ + struct exponential_backoff backoff; + globaldata_t gd = mycpu; + int bit; + int mask; + + /* + * Wait until we can gain exclusive access vs another exclusive + * holder. + */ + exponential_init(&backoff, mtx); + ++spinlocks_contested1; + + while (value & SPINLOCK_EXCLUSIVE) { + value = atomic_swap_int(&mtx->lock, 0x80000000); + if (exponential_backoff(&backoff)) { + value &= ~SPINLOCK_EXCLUSIVE; + break; + } + } + + /* + * Kill the cached shared bit for our own cpu. This is the most + * common case and there's no sense wasting cpu on it. Since + * spinlocks aren't recursive, we can't own a shared ref on the + * spinlock while trying to get an exclusive one. + * + * If multiple bits are set do not stall on any single cpu. Check + * all cpus that have the cache bit set, then loop and check again, + * until we've cleaned all the bits. + */ + value &= ~gd->gd_cpumask; + + while ((mask = value) != 0) { + while (mask) { + bit = bsfl(value); + if (globaldata_find(bit)->gd_spinlocks_rd == 0) { + value &= ~(1 << bit); + } else if (exponential_backoff(&backoff)) { + value = 0; + break; + } + mask &= ~(1 << bit); + } + } +} /* - * This code deals with indefinite waits. If the system is handling a - * panic we hand the spinlock over to the caller after 1 second. After - * 10 seconds we attempt to print a debugger backtrace. We also run - * pending interrupts in order to allow a console break into DDB. + * The cache bit wasn't set for our cpu. Loop until we can set the bit. + * As with the spin_lock_rd() inline we need a memory fence after incrementing + * gd_spinlocks_rd to interlock against exclusive spinlocks waiting for + * that field to clear. */ -static void -spin_lock_contested_long(struct spinlock *mtx) +void +spin_lock_rd_contested(struct spinlock *mtx) +{ + struct exponential_backoff backoff; + globaldata_t gd = mycpu; + int value = mtx->lock; + + exponential_init(&backoff, mtx); + ++spinlocks_contested1; + + while ((value & gd->gd_cpumask) == 0) { + if (value & SPINLOCK_EXCLUSIVE) { + --gd->gd_spinlocks_rd; + if (exponential_backoff(&backoff)) { + ++gd->gd_spinlocks_rd; + break; + } + ++gd->gd_spinlocks_rd; + cpu_mfence(); + } else { + if (atomic_cmpset_int(&mtx->lock, value, value|gd->gd_cpumask)) + break; + } + value = mtx->lock; + } +} + +/* + * Handle exponential backoff and indefinite waits. + * + * If the system is handling a panic we hand the spinlock over to the caller + * after 1 second. After 10 seconds we attempt to print a debugger + * backtrace. We also run pending interrupts in order to allow a console + * break into DDB. + */ +static +int +exponential_backoff(struct exponential_backoff *bo) { - sysclock_t base; sysclock_t count; - int nsec; + int i; + + /* + * Quick backoff + */ + for (i = 0; i < bo->backoff; ++i) + cpu_nop(); + if (bo->backoff < BACKOFF_LIMIT) { + bo->backoff <<= 1; + return (FALSE); + } + /* + * Indefinite + */ ++spinlocks_contested2; - base = sys_cputimer->count(); - nsec = 0; - - for (;;) { - if (mtx->lock == 0 && atomic_swap_int(&mtx->lock, 1) == 0) - return; - count = sys_cputimer->count(); - if (count - base > sys_cputimer->freq) { - printf("spin_lock: %p, indefinite wait!\n", mtx); - if (panicstr) - return; + if (bo->nsec == 0) { + bo->base = sys_cputimer->count(); + bo->nsec = 1; + } + + count = sys_cputimer->count(); + if (count - bo->base > sys_cputimer->freq) { + printf("spin_lock: %p, indefinite wait!\n", bo->mtx); + if (panicstr) + return (TRUE); #ifdef INVARIANTS - if (spin_lock_test_mode) { - db_print_backtrace(); - return; - } -#endif - if (++nsec == 10) { - nsec = 0; - db_print_backtrace(); - } - splz(); - base = count; + if (spin_lock_test_mode) { + db_print_backtrace(); + return (TRUE); } +#endif + if (++bo->nsec == 11) + db_print_backtrace(); + if (bo->nsec == 60) + panic("spin_lock: %p, indefinite wait!\n", bo->mtx); + splz(); + bo->base = count; } + return (FALSE); } /* - * If INVARIANTS is enabled an indefinite wait spinlock test can be - * executed with 'sysctl debug.spin_lock_test=1' + * If INVARIANTS is enabled various spinlock timing tests can be run + * by setting debug.spin_lock_test: + * + * 1 Test the indefinite wait code + * 2 Time the best-case exclusive lock overhead (spin_test_count) + * 3 Time the best-case shared lock overhead (spin_test_count) */ #ifdef INVARIANTS +static int spin_test_count = 10000000; +SYSCTL_INT(_debug, OID_AUTO, spin_test_count, CTLFLAG_RW, &spin_test_count, 0, ""); + static int sysctl_spin_lock_test(SYSCTL_HANDLER_ARGS) { struct spinlock mtx; int error; int value = 0; + int i; if ((error = suser(curthread)) != 0) return (error); if ((error = SYSCTL_IN(req, &value, sizeof(value))) != 0) return (error); + /* + * Indefinite wait test + */ if (value == 1) { - mtx.lock = 1; + spin_init(&mtx); + spin_lock_wr(&mtx); /* force an indefinite wait */ spin_lock_test_mode = 1; - spin_lock(&mtx); - spin_unlock(&mtx); + spin_lock_wr(&mtx); + spin_unlock_wr(&mtx); /* Clean up the spinlock count */ + spin_unlock_wr(&mtx); spin_lock_test_mode = 0; } + + /* + * Time best-case exclusive spinlocks + */ + if (value == 2) { + globaldata_t gd = mycpu; + + spin_init(&mtx); + for (i = spin_test_count; i > 0; --i) { + spin_lock_wr_quick(gd, &mtx); + spin_unlock_wr_quick(gd, &mtx); + } + } + + /* + * Time best-case shared spinlocks + */ + if (value == 3) { + globaldata_t gd = mycpu; + + spin_init(&mtx); + for (i = spin_test_count; i > 0; --i) { + spin_lock_rd_quick(gd, &mtx); + spin_unlock_rd_quick(gd, &mtx); + } + } return (0); } SYSCTL_PROC(_debug, KERN_PROC_ALL, spin_lock_test, CTLFLAG_RW|CTLTYPE_INT, 0, 0, sysctl_spin_lock_test, "I", "Test spinlock wait code"); - -#endif - -#endif +#endif /* INVARIANTS */ +#endif /* SMP */ diff --git a/sys/kern/lwkt_rwlock.c b/sys/kern/lwkt_rwlock.c index 364ccf027d..61fd3f68a6 100644 --- a/sys/kern/lwkt_rwlock.c +++ b/sys/kern/lwkt_rwlock.c @@ -35,7 +35,7 @@ * * Implements simple shared/exclusive locks using LWKT. * - * $DragonFly: src/sys/kern/Attic/lwkt_rwlock.c,v 1.10 2006/05/19 05:15:34 dillon Exp $ + * $DragonFly: src/sys/kern/Attic/lwkt_rwlock.c,v 1.11 2006/05/21 20:23:25 dillon Exp $ */ #include @@ -86,7 +86,7 @@ lwkt_exlock(lwkt_rwlock_t lock, const char *wmesg) { int gen; - spin_lock(&lock->rw_spinlock); + spin_lock_wr(&lock->rw_spinlock); gen = lock->rw_wait.wa_gen; while (lock->rw_owner != curthread) { if (lock->rw_owner == NULL && lock->rw_count == 0) { @@ -94,13 +94,13 @@ lwkt_exlock(lwkt_rwlock_t lock, const char *wmesg) break; } ++lock->rw_requests; - spin_unlock(&lock->rw_spinlock); + spin_unlock_wr(&lock->rw_spinlock); lwkt_block(&lock->rw_wait, wmesg, &gen); - spin_lock(&lock->rw_spinlock); + spin_lock_wr(&lock->rw_spinlock); --lock->rw_requests; } ++lock->rw_count; - spin_unlock(&lock->rw_spinlock); + spin_unlock_wr(&lock->rw_spinlock); } /* @@ -111,17 +111,17 @@ lwkt_shlock(lwkt_rwlock_t lock, const char *wmesg) { int gen; - spin_lock(&lock->rw_spinlock); + spin_lock_wr(&lock->rw_spinlock); gen = lock->rw_wait.wa_gen; while (lock->rw_owner != NULL) { ++lock->rw_requests; - spin_unlock(&lock->rw_spinlock); + spin_unlock_wr(&lock->rw_spinlock); lwkt_block(&lock->rw_wait, wmesg, &gen); - spin_lock(&lock->rw_spinlock); + spin_lock_wr(&lock->rw_spinlock); --lock->rw_requests; } ++lock->rw_count; - spin_unlock(&lock->rw_spinlock); + spin_unlock_wr(&lock->rw_spinlock); } /* @@ -132,16 +132,16 @@ lwkt_exunlock(lwkt_rwlock_t lock) { KASSERT(lock->rw_owner != NULL, ("lwkt_exunlock: shared lock")); KASSERT(lock->rw_owner == curthread, ("lwkt_exunlock: not owner")); - spin_lock(&lock->rw_spinlock); + spin_lock_wr(&lock->rw_spinlock); if (--lock->rw_count == 0) { lock->rw_owner = NULL; if (lock->rw_requests) { - spin_unlock(&lock->rw_spinlock); + spin_unlock_wr(&lock->rw_spinlock); lwkt_signal(&lock->rw_wait, 1); return; } } - spin_unlock(&lock->rw_spinlock); + spin_unlock_wr(&lock->rw_spinlock); } /* @@ -151,14 +151,14 @@ void lwkt_shunlock(lwkt_rwlock_t lock) { KASSERT(lock->rw_owner == NULL, ("lwkt_shunlock: exclusive lock")); - spin_lock(&lock->rw_spinlock); + spin_lock_wr(&lock->rw_spinlock); if (--lock->rw_count == 0) { if (lock->rw_requests) { - spin_unlock(&lock->rw_spinlock); + spin_unlock_wr(&lock->rw_spinlock); lwkt_signal(&lock->rw_wait, 1); return; } } - spin_unlock(&lock->rw_spinlock); + spin_unlock_wr(&lock->rw_spinlock); } diff --git a/sys/kern/lwkt_thread.c b/sys/kern/lwkt_thread.c index b9a76bc698..e9db34b45c 100644 --- a/sys/kern/lwkt_thread.c +++ b/sys/kern/lwkt_thread.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/kern/lwkt_thread.c,v 1.95 2006/05/19 18:26:28 dillon Exp $ + * $DragonFly: src/sys/kern/lwkt_thread.c,v 1.96 2006/05/21 20:23:25 dillon Exp $ */ /* @@ -520,8 +520,12 @@ lwkt_switch(void) * We had better not be holding any spin locks, but don't get into an * endless panic loop. */ - KASSERT(td->td_spinlocks == 0 || panicstr != NULL, - ("lwkt_switch: still holding %d spinlocks!", td->td_spinlocks)); + KASSERT(gd->gd_spinlocks_rd == 0 || panicstr != NULL, + ("lwkt_switch: still holding %d shared spinlocks!", + gd->gd_spinlocks_rd)); + KASSERT(gd->gd_spinlocks_wr == 0 || panicstr != NULL, + ("lwkt_switch: still holding %d exclusive spinlocks!", + gd->gd_spinlocks_wr)); #ifdef SMP @@ -780,15 +784,13 @@ lwkt_preempt(thread_t ntd, int critpri) /* * The caller has put us in a critical section. We can only preempt * if the caller of the caller was not in a critical section (basically - * a local interrupt), as determined by the 'critpri' parameter. + * a local interrupt), as determined by the 'critpri' parameter. We + * also acn't preempt if the caller is holding any spinlocks (even if + * he isn't in a critical section). This also handles the tokens test. * * YYY The target thread must be in a critical section (else it must * inherit our critical section? I dunno yet). * - * Any tokens held by the target may not be held by thread(s) being - * preempted. We take the easy way out and do not preempt if - * the target is holding tokens. - * * Set need_lwkt_resched() unconditionally for now YYY. */ KASSERT(ntd->td_pri >= TDPRI_CRIT, ("BADCRIT0 %d", ntd->td_pri)); @@ -812,12 +814,14 @@ lwkt_preempt(thread_t ntd, int critpri) #endif /* * Take the easy way out and do not preempt if the target is holding - * one or more tokens. We could test whether the thread(s) being + * any spinlocks. We could test whether the thread(s) being * preempted interlock against the target thread's tokens and whether * we can get all the target thread's tokens, but this situation * should not occur very often so its easier to simply not preempt. + * Also, plain spinlocks are impossible to figure out at this point so + * just don't preempt. */ - if (ntd->td_toks != NULL) { + if (gd->gd_spinlocks_rd + gd->gd_spinlocks_wr != 0) { ++preempt_miss; need_lwkt_resched(); return; @@ -1016,11 +1020,11 @@ lwkt_schedule(thread_t td) * section if the token is owned by our cpu. */ if ((w = td->td_wait) != NULL) { - spin_lock_quick(&w->wa_spinlock); + spin_lock_wr(&w->wa_spinlock); TAILQ_REMOVE(&w->wa_waitq, td, td_threadq); --w->wa_count; td->td_wait = NULL; - spin_unlock_quick(&w->wa_spinlock); + spin_unlock_wr(&w->wa_spinlock); #ifdef SMP if (td->td_gd == mygd) { _lwkt_enqueue(td); @@ -1274,7 +1278,7 @@ lwkt_block(lwkt_wait_t w, const char *wmesg, int *gen) { thread_t td = curthread; - spin_lock(&w->wa_spinlock); + spin_lock_wr(&w->wa_spinlock); if (w->wa_gen == *gen) { _lwkt_dequeue(td); td->td_flags |= TDF_BLOCKQ; @@ -1282,14 +1286,14 @@ lwkt_block(lwkt_wait_t w, const char *wmesg, int *gen) ++w->wa_count; td->td_wait = w; td->td_wmesg = wmesg; - spin_unlock(&w->wa_spinlock); + spin_unlock_wr(&w->wa_spinlock); lwkt_switch(); KKASSERT((td->td_flags & TDF_BLOCKQ) == 0); td->td_wmesg = NULL; *gen = w->wa_gen; } else { *gen = w->wa_gen; - spin_unlock(&w->wa_spinlock); + spin_unlock_wr(&w->wa_spinlock); } } @@ -1306,7 +1310,7 @@ lwkt_signal(lwkt_wait_t w, int count) { thread_t td; - spin_lock(&w->wa_spinlock); + spin_lock_wr(&w->wa_spinlock); ++w->wa_gen; if (count < 0) count = w->wa_count; @@ -1317,7 +1321,7 @@ lwkt_signal(lwkt_wait_t w, int count) TAILQ_REMOVE(&w->wa_waitq, td, td_threadq); td->td_flags &= ~TDF_BLOCKQ; td->td_wait = NULL; - spin_unlock(&w->wa_spinlock); + spin_unlock_wr(&w->wa_spinlock); KKASSERT(td->td_proc == NULL || (td->td_proc->p_flag & P_ONRUNQ) == 0); #ifdef SMP if (td->td_gd == mycpu) { @@ -1328,9 +1332,9 @@ lwkt_signal(lwkt_wait_t w, int count) #else _lwkt_enqueue(td); #endif - spin_lock(&w->wa_spinlock); + spin_lock_wr(&w->wa_spinlock); } - spin_unlock(&w->wa_spinlock); + spin_unlock_wr(&w->wa_spinlock); } /* diff --git a/sys/kern/lwkt_token.c b/sys/kern/lwkt_token.c index fadc15f8d6..2d3a2e79e3 100644 --- a/sys/kern/lwkt_token.c +++ b/sys/kern/lwkt_token.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/kern/lwkt_token.c,v 1.25 2006/05/19 18:26:28 dillon Exp $ + * $DragonFly: src/sys/kern/lwkt_token.c,v 1.26 2006/05/21 20:23:25 dillon Exp $ */ #ifdef _KERNEL @@ -152,7 +152,7 @@ lwkt_getalltokens(thread_t td) KKASSERT(refs->tr_state == 0); tok = refs->tr_tok; if (tok->t_owner != td) { - if (spin_trylock(td, &tok->t_spinlock) == 0) { + if (spin_trylock_wr(&tok->t_spinlock) == 0) { /* * Release the partial list of tokens obtained and return * failure. @@ -162,7 +162,7 @@ lwkt_getalltokens(thread_t td) undo->tr_state = 0; if (--tok->t_count == 0) { tok->t_owner = NULL; - spin_tryunlock(td, &tok->t_spinlock); + spin_unlock_wr(&tok->t_spinlock); } } return (FALSE); @@ -192,7 +192,7 @@ lwkt_relalltokens(thread_t td) KKASSERT(tok->t_owner == td && tok->t_count > 0); if (--tok->t_count == 0) { tok->t_owner = NULL; - spin_unlock_quick(&tok->t_spinlock); + spin_unlock_wr(&tok->t_spinlock); } } } @@ -235,7 +235,7 @@ _lwkt_gettokref(lwkt_tokref_t ref) * Gain ownership of the token's spinlock, SMP version. */ if (tok->t_owner != td) { - if (spin_trylock(td, &tok->t_spinlock) == 0) { + if (spin_trylock_wr(&tok->t_spinlock) == 0) { lwkt_yield(); return; } @@ -290,7 +290,7 @@ _lwkt_trytokref(lwkt_tokref_t ref) * Gain ownership of the token's spinlock, SMP version. */ if (tok->t_owner != td) { - if (spin_trylock(td, &tok->t_spinlock) == 0) { + if (spin_trylock_wr(&tok->t_spinlock) == 0) { td->td_toks = ref->tr_next; return (FALSE); } @@ -377,7 +377,7 @@ lwkt_reltoken(lwkt_tokref *ref) #ifdef SMP if (--tok->t_count == 0) { tok->t_owner = NULL; - spin_unlock_quick(&tok->t_spinlock); + spin_unlock_wr(&tok->t_spinlock); } #else --tok->t_globalcount; diff --git a/sys/netproto/smb/smb_subr.h b/sys/netproto/smb/smb_subr.h index 5047f5478f..eea5710db9 100644 --- a/sys/netproto/smb/smb_subr.h +++ b/sys/netproto/smb/smb_subr.h @@ -30,7 +30,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/netsmb/smb_subr.h,v 1.1.2.1 2001/05/22 08:32:34 bp Exp $ - * $DragonFly: src/sys/netproto/smb/smb_subr.h,v 1.12 2005/12/06 04:03:56 dillon Exp $ + * $DragonFly: src/sys/netproto/smb/smb_subr.h,v 1.13 2006/05/21 20:23:28 dillon Exp $ */ #ifndef _NETSMB_SMB_SUBR_H_ #define _NETSMB_SMB_SUBR_H_ @@ -84,8 +84,8 @@ void m_dumpm(struct mbuf *m); #define smb_slock spinlock #define smb_sl_init(sl, desc) spin_init(sl) #define smb_sl_destroy(sl) -#define smb_sl_lock(sl) spin_lock(sl) -#define smb_sl_unlock(sl) spin_unlock(sl) +#define smb_sl_lock(sl) spin_lock_wr(sl) +#define smb_sl_unlock(sl) spin_unlock_wr(sl) #define SMB_STRFREE(p) do { if (p) smb_strfree(p); } while(0) diff --git a/sys/sys/globaldata.h b/sys/sys/globaldata.h index a46bb5246a..9f9f3df71c 100644 --- a/sys/sys/globaldata.h +++ b/sys/sys/globaldata.h @@ -55,7 +55,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/i386/include/globaldata.h,v 1.11.2.1 2000/05/16 06:58:10 dillon Exp $ - * $DragonFly: src/sys/sys/globaldata.h,v 1.41 2006/05/18 16:25:20 dillon Exp $ + * $DragonFly: src/sys/sys/globaldata.h,v 1.42 2006/05/21 20:23:27 dillon Exp $ */ #ifndef _SYS_GLOBALDATA_H_ @@ -163,7 +163,9 @@ struct globaldata { struct lwp *gd_uschedcp; /* userland scheduler */ struct tslpque *gd_tsleep_hash; /* tsleep/wakeup support */ - void *gd_reserved[15]; /* future fields */ + int gd_spinlocks_rd; /* Shared spinlocks held */ + int gd_spinlocks_wr; /* Exclusive spinlocks held */ + void *gd_reserved[13]; /* future fields */ /* extended by */ }; diff --git a/sys/sys/spinlock.h b/sys/sys/spinlock.h index 426d6c2c3d..71f911fbd1 100644 --- a/sys/sys/spinlock.h +++ b/sys/sys/spinlock.h @@ -29,7 +29,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/sys/spinlock.h,v 1.3 2005/09/16 21:50:13 dillon Exp $ + * $DragonFly: src/sys/sys/spinlock.h,v 1.4 2006/05/21 20:23:27 dillon Exp $ */ #ifndef _SYS_SPINLOCK_H_ @@ -44,5 +44,7 @@ struct spinlock { volatile int lock; /* 0 = unlocked, 1 = locked */ }; +#define SPINLOCK_EXCLUSIVE 0x80000000 + #endif diff --git a/sys/sys/spinlock2.h b/sys/sys/spinlock2.h index 29db53c42d..5a4f30d307 100644 --- a/sys/sys/spinlock2.h +++ b/sys/sys/spinlock2.h @@ -29,7 +29,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/sys/spinlock2.h,v 1.8 2006/05/21 03:43:47 dillon Exp $ + * $DragonFly: src/sys/sys/spinlock2.h,v 1.9 2006/05/21 20:23:27 dillon Exp $ */ #ifndef _SYS_SPINLOCK2_H_ @@ -44,6 +44,9 @@ #ifndef _SYS_THREAD2_H_ #include #endif +#ifndef _SYS_GLOBALDATA_H_ +#include +#endif #ifndef _MACHINE_ATOMIC_H_ #include #endif @@ -51,144 +54,185 @@ #include #endif +/* + * SPECIAL NOTE! Obtaining a spinlock does not enter a critical section + * or protect against FAST interrupts but it will prevent thread preemption. + * Because the spinlock code path is ultra critical, we do not check for + * LWKT reschedule requests (due to an interrupt thread not being able to + * preempt). + */ + #ifdef SMP -#ifdef INVARIANTS +extern int spin_trylock_wr_contested(struct spinlock *mtx, int value); +extern void spin_lock_wr_contested(struct spinlock *mtx, int value); +extern void spin_lock_rd_contested(struct spinlock *mtx); -static __inline void -spin_lock_debug(thread_t td, int count) +#endif + +#ifdef SMP + +/* + * Attempt to obtain an exclusive spinlock. Returns FALSE on failure, + * TRUE on success. Since the caller assumes that spinlocks must actually + * work when using this function, it is only made available to SMP builds. + */ +static __inline boolean_t +spin_trylock_wr(struct spinlock *mtx) { - td->td_spinlocks += count; + globaldata_t gd = mycpu; + int value; + + ++gd->gd_spinlocks_wr; + if ((value = atomic_swap_int(&mtx->lock, SPINLOCK_EXCLUSIVE)) != 0) + return (spin_trylock_wr_contested(mtx, value)); + return (TRUE); } #endif /* - * Attempt to obtain a spinlock on behalf of the specified thread. Returns - * FALSE on failure, TRUE on success. + * Obtain an exclusive spinlock and return. */ -static __inline boolean_t -spin_trylock(thread_t td, struct spinlock *mtx) +static __inline void +spin_lock_wr_quick(globaldata_t gd, struct spinlock *mtx) { - if (atomic_swap_int(&mtx->lock, 1) == 0) { -#ifdef INVARIANTS - spin_lock_debug(td, 1); +#ifdef SMP + int value; #endif - return (TRUE); - } - return (FALSE); + + ++gd->gd_spinlocks_wr; +#ifdef SMP + if ((value = atomic_swap_int(&mtx->lock, SPINLOCK_EXCLUSIVE)) != 0) + spin_lock_wr_contested(mtx, value); +#endif +} + +static __inline void +spin_lock_wr(struct spinlock *mtx) +{ + spin_lock_wr_quick(mycpu, mtx); } +#if 0 + /* - * Relase a spinlock obtained via spin_trylock() on behalf of the specified - * thread. This function always succeeds. It exists because the other - * standard release functions only operate on the current thread. + * Upgrade a shared spinlock to exclusive. Return TRUE if we were + * able to upgrade without another exclusive holder getting in before + * us, FALSE otherwise. */ -static __inline void -spin_tryunlock(thread_t td, struct spinlock *mtx) +static __inline int +spin_lock_upgrade(struct spinlock *mtx) { -#ifdef INVARIANTS - if (td->td_spinlocks <= 0) - panic("spin_tryunlock: wasn't locked!"); - spin_lock_debug(td, -1); + globaldata_t gd = mycpu; +#ifdef SMP + int value; #endif + + ++gd->gd_spinlocks_wr; +#ifdef SMP + value = atomic_swap_int(&mtx->lock, SPINLOCK_EXCLUSIVE); cpu_sfence(); - mtx->lock = 0; /* non-bus-locked lock release */ +#endif + --gd->gd_spinlocks_rd; +#ifdef SMP + value &= ~gd->gd_cpumask; + if (value) { + spin_lock_wr_contested(mtx, value); + if (value & SPINLOCK_EXCLUSIVE) + return (FALSE); + XXX regain original shared lock? + } + return (TRUE); +#endif } -extern void spin_lock_contested(struct spinlock *mtx); +#endif /* - * The quick versions should be used only if you are already - * in a critical section or you know the spinlock will never - * be used by an hard interrupt, IPI, or soft interrupt. + * Obtain a shared spinlock and return. This is a critical code path. + * + * The vast majority of the overhead is in the cpu_mfence() (5ns vs 1ns for + * the entire rest of the procedure). Unfortunately we have to ensure that + * spinlock count is written out before we check the cpumask to interlock + * against an exclusive spinlock that clears the cpumask and then checks + * the spinlock count. + * + * But what is EXTREMELY important here is that we do not have to perform + * a locked bus cycle on the spinlock itself if the shared bit for our cpu + * is already found to be set. We only need the mfence, and the mfence is + * local to the cpu and never conflicts with other cpu's. * - * Obtain a spinlock and return. + * This means that multiple parallel shared acessors (e.g. filedescriptor + * table lookups, namecache lookups) run at full speed and incur NO cache + * contention at all. Its the difference between 10ns and 40-100ns. */ static __inline void -spin_lock_quick(struct spinlock *mtx) +spin_lock_rd_quick(globaldata_t gd, struct spinlock *mtx) { -#ifdef INVARIANTS - spin_lock_debug(curthread, 1); + ++gd->gd_spinlocks_rd; +#ifdef SMP + cpu_mfence(); + if ((mtx->lock & gd->gd_cpumask) == 0) + spin_lock_rd_contested(mtx); #endif - if (atomic_swap_int(&mtx->lock, 1) != 0) - spin_lock_contested(mtx); /* slow path */ } -/* - * Release a spinlock previously obtained by the current thread. - */ static __inline void -spin_unlock_quick(struct spinlock *mtx) +spin_lock_rd(struct spinlock *mtx) { -#ifdef INVARIANTS - if (curthread->td_spinlocks <= 0) - panic("spin_unlock_quick: wasn't locked!"); - spin_lock_debug(curthread, -1); -#endif - cpu_sfence(); - mtx->lock = 0; /* non-bus-locked lock release */ + spin_lock_rd_quick(mycpu,mtx); } /* - * Returns whether a spinlock is locked or not. 0 indicates not locked, - * non-zero indicates locked (by any thread, not necessarily the current - * thread). + * Release an exclusive spinlock. We can just do this passively, only + * ensuring that our spinlock count is left intact until the mutex is + * cleared. */ -static __inline boolean_t -spin_is_locked(struct spinlock *mtx) -{ - return (mtx->lock); -} - static __inline void -spin_init(struct spinlock *mtx) +spin_unlock_wr_quick(globaldata_t gd, struct spinlock *mtx) { - mtx->lock = 0; +#ifdef SMP + mtx->lock = 0; +#endif + --gd->gd_spinlocks_wr; } static __inline void -spin_uninit(struct spinlock *mtx) +spin_unlock_wr(struct spinlock *mtx) { - /* unused */ + spin_unlock_wr_quick(mycpu, mtx); } -#else /* SMP */ - /* - * There is no spin_trylock(), spin_tryunlock(), or spin_is_locked() - * for UP builds. These functions are used by the kernel only in - * situations where the spinlock actually has to work. - * - * We provide the rest of the calls for UP as degenerate inlines (note - * that the non-quick versions still obtain/release a critical section!). - * This way we don't have to have a billion #ifdef's floating around - * the rest of the kernel. + * Release a shared spinlock. We leave the shared bit set in the spinlock + * as a cache and simply decrement the spinlock count for the cpu. This + * fast-paths another shared lock later at the cost of an exclusive lock + * having to check per-cpu spinlock counts to determine when there are no + * shared holders remaining. */ +static __inline void +spin_unlock_rd_quick(globaldata_t gd, struct spinlock *mtx) +{ + --gd->gd_spinlocks_rd; +} -static __inline void spin_lock_quick(struct spinlock *mtx) { } -static __inline void spin_unlock_quick(struct spinlock *mtx) { } -static __inline void spin_init(struct spinlock *mtx) { } - -#endif /* SMP */ +static __inline void +spin_unlock_rd(struct spinlock *mtx) +{ + spin_unlock_rd_quick(mycpu, mtx); +} -/* - * The normal spin_lock() API automatically enters and exits a - * critical section, preventing deadlocks from interrupt preemption - * if the interrupt thread accesses the same spinlock. - */ static __inline void -spin_lock(struct spinlock *mtx) +spin_init(struct spinlock *mtx) { - crit_enter_id("spin"); - spin_lock_quick(mtx); + mtx->lock = 0; } static __inline void -spin_unlock(struct spinlock *mtx) +spin_uninit(struct spinlock *mtx) { - spin_unlock_quick(mtx); - crit_exit_id("spin"); + /* unused */ } #endif /* _KERNEL */ diff --git a/sys/sys/thread.h b/sys/sys/thread.h index c68fc7086a..e9033a1646 100644 --- a/sys/sys/thread.h +++ b/sys/sys/thread.h @@ -7,7 +7,7 @@ * Types which must already be defined when this header is included by * userland: struct md_thread * - * $DragonFly: src/sys/sys/thread.h,v 1.80 2006/05/20 02:42:13 dillon Exp $ + * $DragonFly: src/sys/sys/thread.h,v 1.81 2006/05/21 20:23:27 dillon Exp $ */ #ifndef _SYS_THREAD_H_ @@ -246,7 +246,7 @@ struct thread { __uint64_t td_sticks; /* Statclock hits in system mode (uS) */ __uint64_t td_iticks; /* Statclock hits processing intr (uS) */ int td_locks; /* lockmgr lock debugging */ - int td_spinlocks; /* spinlock debugging */ + int td_unused01; int td_refs; /* hold position in gd_tdallq / hold free */ int td_nest_count; /* prevent splz nesting */ #ifdef SMP diff --git a/sys/vfs/ntfs/ntfs_subr.c b/sys/vfs/ntfs/ntfs_subr.c index cc1400041d..c3fe76401e 100644 --- a/sys/vfs/ntfs/ntfs_subr.c +++ b/sys/vfs/ntfs/ntfs_subr.c @@ -26,7 +26,7 @@ * SUCH DAMAGE. * * $FreeBSD: src/sys/ntfs/ntfs_subr.c,v 1.7.2.4 2001/10/12 22:08:49 semenu Exp $ - * $DragonFly: src/sys/vfs/ntfs/ntfs_subr.c,v 1.23 2006/04/23 03:08:04 dillon Exp $ + * $DragonFly: src/sys/vfs/ntfs/ntfs_subr.c,v 1.24 2006/05/21 20:23:28 dillon Exp $ */ #include @@ -424,19 +424,19 @@ ntfs_ntput(struct ntnode *ip) dprintf(("ntfs_ntput: rele ntnode %"PRId64": %p, usecount: %d\n", ip->i_number, ip, ip->i_usecount)); - spin_lock(&ip->i_interlock); + spin_lock_wr(&ip->i_interlock); ip->i_usecount--; #ifdef DIAGNOSTIC if (ip->i_usecount < 0) { - spin_unlock(&ip->i_interlock); + spin_unlock_wr(&ip->i_interlock); panic("ntfs_ntput: ino: %"PRId64" usecount: %d \n", ip->i_number,ip->i_usecount); } #endif if (ip->i_usecount > 0) { - spin_unlock(&ip->i_interlock); + spin_unlock_wr(&ip->i_interlock); LOCKMGR(&ip->i_lock, LK_RELEASE); return; } @@ -444,7 +444,7 @@ ntfs_ntput(struct ntnode *ip) dprintf(("ntfs_ntput: deallocating ntnode: %"PRId64"\n", ip->i_number)); if (ip->i_fnlist.lh_first) { - spin_unlock(&ip->i_interlock); + spin_unlock_wr(&ip->i_interlock); panic("ntfs_ntput: ntnode has fnodes\n"); } @@ -458,7 +458,7 @@ ntfs_ntput(struct ntnode *ip) LIST_REMOVE(vap,va_list); ntfs_freentvattr(vap); } - spin_unlock(&ip->i_interlock); + spin_unlock_wr(&ip->i_interlock); vrele(ip->i_devvp); FREE(ip, M_NTFSNTNODE); } @@ -485,15 +485,15 @@ ntfs_ntrele(struct ntnode *ip) dprintf(("ntfs_ntrele: rele ntnode %"PRId64": %p, usecount: %d\n", ip->i_number, ip, ip->i_usecount)); - spin_lock(&ip->i_interlock); + spin_lock_wr(&ip->i_interlock); ip->i_usecount--; if (ip->i_usecount < 0) { - spin_unlock(&ip->i_interlock); + spin_unlock_wr(&ip->i_interlock); panic("ntfs_ntrele: ino: %"PRId64" usecount: %d \n", ip->i_number,ip->i_usecount); } - spin_unlock(&ip->i_interlock); + spin_unlock_wr(&ip->i_interlock); } /* diff --git a/sys/vm/vm_zone.c b/sys/vm/vm_zone.c index 8ff47e37d2..80129c73dc 100644 --- a/sys/vm/vm_zone.c +++ b/sys/vm/vm_zone.c @@ -12,7 +12,7 @@ * John S. Dyson. * * $FreeBSD: src/sys/vm/vm_zone.c,v 1.30.2.6 2002/10/10 19:50:16 dillon Exp $ - * $DragonFly: src/sys/vm/vm_zone.c,v 1.19 2005/11/08 22:40:01 dillon Exp $ + * $DragonFly: src/sys/vm/vm_zone.c,v 1.20 2006/05/21 20:23:29 dillon Exp $ */ #include @@ -57,7 +57,7 @@ zalloc(vm_zone_t z) if (z == NULL) zerror(ZONE_ERROR_INVALID); #endif - spin_lock(&z->zlock); + spin_lock_wr(&z->zlock); if (z->zfreecnt > z->zfreemin) { item = z->zitems; #ifdef INVARIANTS @@ -69,9 +69,9 @@ zalloc(vm_zone_t z) z->zitems = ((void **) item)[0]; z->zfreecnt--; z->znalloc++; - spin_unlock(&z->zlock); + spin_unlock_wr(&z->zlock); } else { - spin_unlock(&z->zlock); + spin_unlock_wr(&z->zlock); item = zget(z); /* * PANICFAIL allows the caller to assume that the zalloc() @@ -91,7 +91,7 @@ void zfree(vm_zone_t z, void *item) { - spin_lock(&z->zlock); + spin_lock_wr(&z->zlock); ((void **) item)[0] = z->zitems; #ifdef INVARIANTS if (((void **) item)[1] == (void *) ZENTRY_FREE) @@ -100,7 +100,7 @@ zfree(vm_zone_t z, void *item) #endif z->zitems = item; z->zfreecnt++; - spin_unlock(&z->zlock); + spin_unlock_wr(&z->zlock); } /* @@ -381,7 +381,7 @@ zget(vm_zone_t z) nitems = nbytes / z->zsize; } - spin_lock(&z->zlock); + spin_lock_wr(&z->zlock); z->ztotal += nitems; /* * Save one for immediate allocation @@ -411,7 +411,7 @@ zget(vm_zone_t z) } else { item = NULL; } - spin_unlock(&z->zlock); + spin_unlock_wr(&z->zlock); /* * A special zone may have used a kernel-reserved vm_map_entry. If -- 2.41.0