From b5d16701e255c342d21e69a6c80b8711c028dc65 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Sat, 11 Dec 2010 12:59:46 -0800 Subject: [PATCH] kernel - Change the discrete mplock into mp_token * Use a lwkt_token for the mp_lock. This consolidates our longer-term spinnable locks (the mplock and tokens) into just tokens, making it easier to solve performance issues. * Some refactoring of the token code was needed to guarantee the ordering when acquiring and releasing the mp_token vs other tokens. * The thread switch code, lwkt_switch(), is simplified by this change though not necessarily faster. * Remove td_mpcount, mp_lock, and other related fields. * Remove assertions related to td_mpcount and friends, generally folding them into similar assertions for tokens. --- sys/bus/cam/cam_sim.c | 2 +- sys/conf/files | 1 - sys/ddb/db_ps.c | 14 +- sys/kern/init_main.c | 3 - sys/kern/kern_intr.c | 33 +-- sys/kern/kern_kinfo.c | 5 +- sys/kern/kern_mplock.c | 316 --------------------- sys/kern/kern_shutdown.c | 1 - sys/kern/lwkt_thread.c | 153 +--------- sys/kern/lwkt_token.c | 255 ++++++++++++----- sys/kern/tty_cons.c | 3 - sys/kern/usched_bsd4.c | 6 - sys/kern/usched_dummy.c | 6 - sys/platform/pc32/apic/apic_vector.s | 3 +- sys/platform/pc32/i386/exception.s | 12 - sys/platform/pc32/i386/genassym.c | 3 - sys/platform/pc32/i386/locore.s | 5 - sys/platform/pc32/i386/machdep.c | 7 +- sys/platform/pc32/i386/mp_machdep.c | 22 +- sys/platform/pc32/i386/trap.c | 19 +- sys/platform/pc32/i386/vm86.c | 4 +- sys/platform/pc32/i386/vm_machdep.c | 4 - sys/platform/pc32/isa/ipl.s | 3 +- sys/platform/pc64/apic/apic_vector.s | 3 +- sys/platform/pc64/x86_64/exception.S | 14 - sys/platform/pc64/x86_64/genassym.c | 3 - sys/platform/pc64/x86_64/ipl.s | 3 +- sys/platform/pc64/x86_64/machdep.c | 7 +- sys/platform/pc64/x86_64/mp_machdep.c | 22 +- sys/platform/pc64/x86_64/trap.c | 14 - sys/platform/vkernel/i386/cpu_regs.c | 9 +- sys/platform/vkernel/i386/fork_tramp.s | 12 - sys/platform/vkernel/i386/genassym.c | 3 - sys/platform/vkernel/i386/mp.c | 8 +- sys/platform/vkernel/i386/trap.c | 12 - sys/platform/vkernel64/x86_64/fork_tramp.s | 14 - sys/platform/vkernel64/x86_64/genassym.c | 3 - sys/platform/vkernel64/x86_64/mp.c | 8 +- sys/platform/vkernel64/x86_64/trap.c | 12 - sys/sys/mplock2.h | 176 +----------- sys/sys/thread.h | 19 +- sys/sys/thread2.h | 13 +- sys/vfs/procfs/procfs_ctl.c | 5 +- sys/vfs/tmpfs/tmpfs_vnops.c | 16 +- 44 files changed, 268 insertions(+), 988 deletions(-) delete mode 100644 sys/kern/kern_mplock.c diff --git a/sys/bus/cam/cam_sim.c b/sys/bus/cam/cam_sim.c index 05754c60a4..f0404cae24 100644 --- a/sys/bus/cam/cam_sim.c +++ b/sys/bus/cam/cam_sim.c @@ -103,7 +103,7 @@ sim_lock_assert_owned(sim_lock *lock) { if (lock) { if (lock == &sim_mplock) - ASSERT_MP_LOCK_HELD(curthread); + ASSERT_MP_LOCK_HELD(); else KKASSERT(lockstatus(lock, curthread) != 0); } diff --git a/sys/conf/files b/sys/conf/files index 9e4d3734e0..9fe977a797 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -923,7 +923,6 @@ kern/kern_memio.c standard kern/kern_udev.c standard kern/kern_upcall.c standard kern/kern_sfbuf.c standard -kern/kern_mplock.c standard kern/kern_subr.c standard kern/kern_iosched.c standard kern/kern_usched.c standard diff --git a/sys/ddb/db_ps.c b/sys/ddb/db_ps.c index ddc849838b..dae025d2fe 100644 --- a/sys/ddb/db_ps.c +++ b/sys/ddb/db_ps.c @@ -139,17 +139,12 @@ db_ps(db_expr_t dummy1, boolean_t dummy2, db_expr_t dummy3, char *dummy4) TAILQ_FOREACH(td, &gd->gd_tdrunq, td_threadq) { if (db_more(&nl) < 0) return; - db_printf(" %p %3d %08x %2d/%02d/%02d %p %8.8s %s\n", + db_printf(" %p %3d %08x %2d/%02d %p %8.8s %s\n", td, (td->td_proc ? td->td_proc->p_pid : -1), td->td_flags, td->td_pri, td->td_critcount, -#ifdef SMP - td->td_mpcount, -#else - 0, -#endif td->td_sp, td->td_wmesg ? td->td_wmesg : "-", td->td_proc ? td->td_proc->p_comm : td->td_comm); @@ -166,17 +161,12 @@ db_ps(db_expr_t dummy1, boolean_t dummy2, db_expr_t dummy3, char *dummy4) TAILQ_FOREACH(td, &gd->gd_tdallq, td_allq) { if (db_more(&nl) < 0) return; - db_printf(" %3d %p %3d %08x %2d/%02d/%02d %p %8.8s %s\n", + db_printf(" %3d %p %3d %08x %2d/%02d %p %8.8s %s\n", np, td, (td->td_proc ? td->td_proc->p_pid : -1), td->td_flags, td->td_pri, td->td_critcount, -#ifdef SMP - td->td_mpcount, -#else - 0, -#endif td->td_sp, td->td_wmesg ? td->td_wmesg : "-", td->td_proc ? td->td_proc->p_comm : td->td_comm); diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index 68e3248406..6d59388300 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -165,9 +165,6 @@ mi_proc0init(struct globaldata *gd, struct user *proc0paddr) { lwkt_init_thread(&thread0, proc0paddr, LWKT_THREAD_STACK, 0, gd); lwkt_set_comm(&thread0, "thread0"); -#ifdef SMP - thread0.td_mpcount = 1; /* will hold mplock initially */ -#endif RB_INIT(&proc0.p_lwp_tree); spin_init(&proc0.p_spin); proc0.p_lasttid = 0; /* +1 = next TID */ diff --git a/sys/kern/kern_intr.c b/sys/kern/kern_intr.c index 8b62aec3cb..0cc9193b9d 100644 --- a/sys/kern/kern_intr.c +++ b/sys/kern/kern_intr.c @@ -85,38 +85,13 @@ int max_installed_soft_intr; * dangling tokens, spinlocks, or mp locks. */ #ifdef INVARIANTS -# ifdef SMP -/* INVARIANTS & SMP */ -# define SMP_INVARIANTS_DECLARE \ - int mpcount; - -# define SMP_INVARIANTS_GET(td) \ - mpcount = (td)->td_mpcount - -# define SMP_INVARIANTS_TEST(td, name) \ - KASSERT(mpcount == (td)->td_mpcount, \ - ("mpcount mismatch after interrupt handler %s", \ - name)) - -# define SMP_INVARIANTS_ADJMP(count) \ - mpcount += (count) - -# else -/* INVARIANTS & !SMP */ -# define SMP_INVARIANTS_DECLARE -# define SMP_INVARIANTS_GET(td) -# define SMP_INVARIANTS_TEST(td, name) - -# endif /* ndef SMP */ #define TD_INVARIANTS_DECLARE \ - SMP_INVARIANTS_DECLARE \ int spincount; \ lwkt_tokref_t curstop #define TD_INVARIANTS_GET(td) \ do { \ - SMP_INVARIANTS_GET(td); \ spincount = (td)->td_gd->gd_spinlocks_wr; \ curstop = (td)->td_toks_stop; \ } while(0) @@ -129,16 +104,11 @@ int max_installed_soft_intr; KASSERT(curstop == (td)->td_toks_stop, \ ("token count mismatch after interrupt handler %s", \ name)); \ - SMP_INVARIANTS_TEST(td, name); \ } while(0) #else -/* !INVARIANTS */ -# ifdef SMP -/* !INVARIANTS & SMP */ -# define SMP_INVARIANTS_ADJMP(count) -# endif +/* !INVARIANTS */ #define TD_INVARIANTS_DECLARE #define TD_INVARIANTS_GET(td) @@ -735,7 +705,6 @@ ithread_fast_handler(struct intrframe *frame) break; } got_mplock = 1; - SMP_INVARIANTS_ADJMP(1); } #endif if (rec->serializer) { diff --git a/sys/kern/kern_kinfo.c b/sys/kern/kern_kinfo.c index ef3fa3a87b..58095a6f8f 100644 --- a/sys/kern/kern_kinfo.c +++ b/sys/kern/kern_kinfo.c @@ -49,6 +49,7 @@ #include #include #include +#include #include #ifdef _KERNEL #include @@ -191,7 +192,7 @@ fill_kinfo_lwp(struct lwp *lwp, struct kinfo_lwp *kl) } } #ifdef SMP - kl->kl_mpcount = lwp->lwp_thread->td_mpcount; + kl->kl_mpcount = get_mplock_count(lwp->lwp_thread); #else kl->kl_mpcount = 0; #endif @@ -244,7 +245,7 @@ fill_kinfo_proc_kthread(struct thread *td, struct kinfo_proc *kp) kp->kp_lwp.kl_tid = -1; kp->kp_lwp.kl_tdflags = td->td_flags; #ifdef SMP - kp->kp_lwp.kl_mpcount = td->td_mpcount; + kp->kp_lwp.kl_mpcount = get_mplock_count(td); #else /* !SMP */ kp->kp_lwp.kl_mpcount = 0; #endif /* SMP */ diff --git a/sys/kern/kern_mplock.c b/sys/kern/kern_mplock.c deleted file mode 100644 index 20cecc99b9..0000000000 --- a/sys/kern/kern_mplock.c +++ /dev/null @@ -1,316 +0,0 @@ -/* - * Copyright (c) 2009 The DragonFly Project. All rights reserved. - * - * This code is derived from software contributed to The DragonFly Project - * by Matthew Dillon - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * 3. Neither the name of The DragonFly Project nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific, prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * Helper functions for MP lock acquisition and release. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#ifdef SMP -static int chain_mplock = 0; -static int mplock_yield = 10; -static int mplock_backtrace; -static __int64_t mplock_contention_count = 0; - -SYSCTL_INT(_lwkt, OID_AUTO, chain_mplock, CTLFLAG_RW, &chain_mplock, 0, - "Chain IPI's to other CPU's potentially needing the MP lock when it is yielded"); -SYSCTL_INT(_lwkt, OID_AUTO, mplock_yield_delay, CTLFLAG_RW, &mplock_yield, 0, - "Duration of delay when MP lock is temporarily yielded"); -SYSCTL_INT(_lwkt, OID_AUTO, mplock_backtrace, CTLFLAG_RW, &mplock_backtrace, 0, - "Output backplane when mplock contention occurs"); -SYSCTL_QUAD(_lwkt, OID_AUTO, mplock_contention_count, CTLFLAG_RW, - &mplock_contention_count, 0, "spinning due to MPLOCK contention"); - -/* - * Kernel Trace - */ -#if !defined(KTR_GIANT_CONTENTION) -#define KTR_GIANT_CONTENTION KTR_ALL -#endif - -KTR_INFO_MASTER(giant); -KTR_INFO(KTR_GIANT_CONTENTION, giant, beg, 0, - "thread=%p held %s:%-5d want %s:%-5d", - sizeof(void *) * 3 + sizeof(int) * 2); -KTR_INFO(KTR_GIANT_CONTENTION, giant, end, 1, - "thread=%p held %s:%-5d want %s:%-5d", - sizeof(void *) * 3 + sizeof(int) * 2); - -#define loggiant(name) \ - KTR_LOG(giant_ ## name, curthread, \ - mp_lock_holder_file, mp_lock_holder_line, \ - file, line) - -int mp_lock; -cpumask_t cpu_contention_mask; -const char *mp_lock_holder_file; /* debugging */ -int mp_lock_holder_line; /* debugging */ - -/* - * Sets up the initial MP lock state near the start of the kernel boot - */ -void -cpu_get_initial_mplock(void) -{ - mp_lock = 0; /* cpu 0 */ - curthread->td_mpcount = 1; -} - -/* - * This code is called from the get_mplock() inline when the mplock - * is not already held. td_mpcount has already been predisposed - * (incremented). - */ -void -_get_mplock_predisposed(const char *file, int line) -{ - globaldata_t gd = mycpu; - - if (gd->gd_intr_nesting_level) { - panic("Attempt to acquire mplock not already held " - "in hard section, ipi or interrupt %s:%d", - file, line); - } - if (atomic_cmpset_int(&mp_lock, -1, gd->gd_cpuid) == 0) - _get_mplock_contested(file, line); -#ifdef INVARIANTS - mp_lock_holder_file = file; - mp_lock_holder_line = line; -#endif -} - -/* - * Called when the MP lock could not be trvially acquired. The caller - * has already bumped td_mpcount. - */ -void -_get_mplock_contested(const char *file, int line) -{ - globaldata_t gd = mycpu; - int ov; - int nv; - const void **stkframe = (const void **)&file; - - if (mplock_backtrace > 0) { - --mplock_backtrace; - print_backtrace(-1); - } - ++mplock_contention_count; - for (;;) { - ov = mp_lock; - nv = gd->gd_cpuid; - if (ov == gd->gd_cpuid) - break; - if (ov == -1) { - if (atomic_cmpset_int(&mp_lock, ov, gd->gd_cpuid)) - break; - } else { - gd->gd_curthread->td_mplock_stallpc = stkframe[-1]; - loggiant(beg); - lwkt_switch(); - loggiant(end); - KKASSERT(gd->gd_cpuid == mp_lock); - break; - } - } -} - -/* - * Called if td_mpcount went negative or if td_mpcount + td_xpcount is 0 - * and we were unable to release the MP lock. Handles sanity checks - * and conflicts. - * - * It is possible for the inline release to have raced an interrupt which - * get/rel'd the MP lock, causing the inline's cmpset to fail. If this - * case occurs mp_lock will either already be in a released state or it - * will have already been acquired by another cpu. - */ -void -_rel_mplock_contested(void) -{ - globaldata_t gd = mycpu; - thread_t td = gd->gd_curthread; - int ov; - - KKASSERT(td->td_mpcount >= 0); - if (td->td_mpcount + td->td_xpcount == 0) { - for (;;) { - ov = mp_lock; - if (ov != gd->gd_cpuid) - break; - if (atomic_cmpset_int(&mp_lock, ov, -1)) - break; - } - } -} - -/* - * Called when try_mplock() fails. - * - * The inline bumped td_mpcount so we have to undo it. - * - * It is possible to race an interrupt which acquired and released the - * MP lock. When combined with the td_mpcount decrement we do the MP lock - * can wind up in any state and possibly not even owned by us. - * - * It is also possible for this function to be called even if td_mpcount > 1 - * if someone bumped it and raced an interrupt which then called try_mpock(). - */ -void -_try_mplock_contested(const char *file, int line) -{ - globaldata_t gd = mycpu; - thread_t td = gd->gd_curthread; - int ov; - - --td->td_mpcount; - KKASSERT(td->td_mpcount >= 0); - ++mplock_contention_count; - - if (td->td_mpcount + td->td_xpcount == 0) { - for (;;) { - ov = mp_lock; - if (ov != gd->gd_cpuid) - break; - if (atomic_cmpset_int(&mp_lock, ov, -1)) - break; - } - } -} - -/* - * Called when cpu_try_mplock() fails. - * - * The inline did not touch td_mpcount so we do not either. - */ -void -_cpu_try_mplock_contested(const char *file, int line) -{ - ++mplock_contention_count; -} - -/* - * Temporarily yield the MP lock. This is part of lwkt_user_yield() - * which is kinda hackish. The MP lock cannot be yielded if inherited - * due to a preemption. - */ -void -yield_mplock(thread_t td) -{ - int savecnt; - - if (td->td_xpcount == 0 && mplock_yield > 0) { - savecnt = td->td_mpcount; - td->td_mpcount = 1; - rel_mplock(); - DELAY(mplock_yield); - get_mplock(); - td->td_mpcount = savecnt; - } -} - -#if 0 - -/* - * The rel_mplock() code will call this function after releasing the - * last reference on the MP lock if cpu_contention_mask is non-zero. - * - * We then chain an IPI to a single other cpu potentially needing the - * lock. This is a bit heuristical and we can wind up with IPIs flying - * all over the place. - */ -static void lwkt_mp_lock_uncontested_remote(void *arg __unused); - -void -lwkt_mp_lock_uncontested(void) -{ - globaldata_t gd; - globaldata_t dgd; - cpumask_t mask; - cpumask_t tmpmask; - int cpuid; - - if (chain_mplock) { - gd = mycpu; - clr_mplock_contention_mask(gd); - mask = cpu_contention_mask; - tmpmask = ~(CPUMASK(gd->gd_cpuid) - 1); - - if (mask) { - if (mask & tmpmask) - cpuid = BSFCPUMASK(mask & tmpmask); - else - cpuid = BSFCPUMASK(mask); - atomic_clear_cpumask(&cpu_contention_mask, CPUMASK(cpuid)); - dgd = globaldata_find(cpuid); - lwkt_send_ipiq(dgd, lwkt_mp_lock_uncontested_remote, NULL); - } - } -} - -/* - * The idea is for this IPI to interrupt a potentially lower priority - * thread, such as a user thread, to allow the scheduler to reschedule - * a higher priority kernel thread that needs the MP lock. - * - * For now we set the LWKT reschedule flag which generates an AST in - * doreti, though theoretically it is also possible to possibly preempt - * here if the underlying thread was operating in user mode. Nah. - */ -static void -lwkt_mp_lock_uncontested_remote(void *arg __unused) -{ - need_lwkt_resched(); -} - -#endif - -#endif /* SMP */ diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c index 3ecd38d947..320ff74d8b 100644 --- a/sys/kern/kern_shutdown.c +++ b/sys/kern/kern_shutdown.c @@ -767,7 +767,6 @@ panic(const char *fmt, ...) kprintf("panic: %s\n", buf); #ifdef SMP /* two separate prints in case of an unmapped page and trap */ - kprintf("mp_lock = %08x; ", mp_lock); kprintf("cpuid = %d\n", mycpu->gd_cpuid); #endif diff --git a/sys/kern/lwkt_thread.c b/sys/kern/lwkt_thread.c index 49f5cedeb9..c8b28b2340 100644 --- a/sys/kern/lwkt_thread.c +++ b/sys/kern/lwkt_thread.c @@ -472,9 +472,6 @@ lwkt_free_thread(thread_t td) * different beast and LWKT priorities should not be confused with * user process priorities. * - * The MP lock may be out of sync with the thread's td_mpcount + td_xpcount. - * lwkt_switch() cleans it up. - * * Note that the td_switch() function cannot do anything that requires * the MP lock since the MP lock will have already been setup for * the target thread (not the current thread). It's nice to have a scheduler @@ -494,9 +491,6 @@ lwkt_switch(void) thread_t xtd; thread_t nlast; int nquserok; -#ifdef SMP - int mpheld; -#endif int didaccumulate; /* @@ -560,15 +554,6 @@ lwkt_switch(void) #ifdef SMP - /* - * td_mpcount + td_xpcount cannot be used to determine if we currently - * hold the MP lock because get_mplock() will increment it prior to - * attempting to get the lock, and switch out if it can't. Our - * ownership of the actual lock will remain stable while we are - * in a critical section, and once we actually acquire the underlying - * lock as long as the count is greater than 0. - */ - mpheld = MP_LOCK_HELD(gd); #ifdef INVARIANTS if (td->td_cscount) { kprintf("Diagnostic: attempt to switch while mastering cpusync: %p\n", @@ -591,13 +576,6 @@ lwkt_switch(void) */ if ((ntd = td->td_preempted) != NULL) { KKASSERT(ntd->td_flags & TDF_PREEMPT_LOCK); -#ifdef SMP - if (ntd->td_mpcount + ntd->td_xpcount && mpheld == 0) { - panic("MPLOCK NOT HELD ON RETURN: %p %p %d %d", - td, ntd, td->td_mpcount, ntd->td_mpcount + ntd->td_xpcount); - } - td->td_xpcount = 0; -#endif ntd->td_flags |= TDF_PREEMPT_DONE; /* @@ -641,19 +619,8 @@ lwkt_switch(void) if (gd->gd_reqflags & RQF_IDLECHECK_MASK) ntd->td_flags |= TDF_IDLE_NOHLT; #ifdef SMP - KKASSERT(ntd->td_xpcount == 0); - if (ntd->td_mpcount) { - if (gd->gd_trap_nesting_level == 0 && panicstr == NULL) - panic("Idle thread %p was holding the BGL!", ntd); - if (mpheld == 0) { - set_cpu_contention_mask(gd); - handle_cpu_contention_mask(); - cpu_try_mplock(); - mpheld = MP_LOCK_HELD(gd); - cpu_pause(); - continue; - } - } + if (gd->gd_trap_nesting_level == 0 && panicstr == NULL) + ASSERT_NO_TOKENS_HELD(ntd); clr_cpu_contention_mask(gd); #endif cpu_time.cp_msg[0] = 0; @@ -668,12 +635,7 @@ lwkt_switch(void) * always succeeds. */ if (ntd->td_fairq_accum >= 0 && -#ifdef SMP - (ntd->td_mpcount + ntd->td_xpcount == 0 || - mpheld || cpu_try_mplock_msg(&ntd->td_wmesg)) && -#endif - (!TD_TOKS_HELD(ntd) || - lwkt_getalltokens(ntd)) + (TD_TOKS_NOT_HELD(ntd) || lwkt_getalltokens(ntd)) ) { #ifdef SMP clr_cpu_contention_mask(gd); @@ -684,8 +646,6 @@ lwkt_switch(void) #ifdef SMP if (ntd->td_fairq_accum >= 0) set_cpu_contention_mask(gd); - /* Reload mpheld (it become stale after mplock/token ops) */ - mpheld = MP_LOCK_HELD(gd); #endif /* @@ -747,20 +707,9 @@ lwkt_switch(void) ntd = &gd->gd_idlethread; ntd->td_flags |= TDF_IDLE_NOHLT; #ifdef SMP - KKASSERT(ntd->td_xpcount == 0); - if (ntd->td_mpcount) { - mpheld = MP_LOCK_HELD(gd); - if (gd->gd_trap_nesting_level == 0 && panicstr == NULL) - panic("Idle thread %p was holding the BGL!", ntd); - if (mpheld == 0) { - set_cpu_contention_mask(gd); - handle_cpu_contention_mask(); - cpu_try_mplock(); - mpheld = MP_LOCK_HELD(gd); - cpu_pause(); - break; /* try again from the top, almost */ - } - } + if (gd->gd_trap_nesting_level == 0 && panicstr == NULL) + ASSERT_NO_TOKENS_HELD(ntd); + /* contention case, do not clear contention mask */ #endif /* @@ -781,11 +730,7 @@ lwkt_switch(void) */ if ((ntd->td_pri >= TDPRI_KERN_LPSCHED || nquserok || user_pri_sched) && ntd->td_fairq_accum >= 0 && -#ifdef SMP - (ntd->td_mpcount + ntd->td_xpcount == 0 || - mpheld || cpu_try_mplock_msg(&ntd->td_wmesg)) && -#endif - (!TD_TOKS_HELD(ntd) || lwkt_getalltokens(ntd)) + (TD_TOKS_NOT_HELD(ntd) || lwkt_getalltokens(ntd)) ) { #ifdef SMP clr_cpu_contention_mask(gd); @@ -800,10 +745,6 @@ lwkt_switch(void) #ifdef SMP if (ntd->td_fairq_accum >= 0) set_cpu_contention_mask(gd); - /* - * Reload mpheld (it become stale after mplock/token ops). - */ - mpheld = MP_LOCK_HELD(gd); if (ntd->td_pri >= TDPRI_KERN_LPSCHED && ntd->td_fairq_accum >= 0) nquserok = 0; #endif @@ -816,26 +757,11 @@ lwkt_switch(void) * While we are looping in the scheduler be sure to service * any interrupts which were made pending due to our critical * section, otherwise we could livelock (e.g.) IPIs. - * - * NOTE: splz can enter and exit the mplock so mpheld is - * stale after this call. */ splz_check(); - -#ifdef SMP - /* - * Our mplock can be cached and cause other cpus to livelock - * if we loop due to e.g. not being able to acquire tokens. - */ - if (MP_LOCK_HELD(gd)) - cpu_rel_mplock(gd->gd_cpuid); - mpheld = 0; -#endif } /* - * Do the actual switch. WARNING: mpheld is stale here. - * * We must always decrement td_fairq_accum on non-idle threads just * in case a thread never gets a tick due to being in a continuous * critical section. The page-zeroing code does that. @@ -859,20 +785,12 @@ havethread_preempted: * If the new target does not need the MP lock and we are holding it, * release the MP lock. If the new target requires the MP lock we have * already acquired it for the target. - * - * WARNING: mpheld is stale here. */ haveidle: KASSERT(ntd->td_critcount, - ("priority problem in lwkt_switch %d %d", td->td_pri, ntd->td_pri)); -#ifdef SMP - if (ntd->td_mpcount + ntd->td_xpcount == 0 ) { - if (MP_LOCK_HELD(gd)) - cpu_rel_mplock(gd->gd_cpuid); - } else { - ASSERT_MP_LOCK_HELD(ntd); - } -#endif + ("priority problem in lwkt_switch %d %d", + td->td_critcount, ntd->td_critcount)); + if (td != ntd) { ++switch_count; KTR_LOG(ctxsw_sw, gd->gd_cpuid, ntd); @@ -907,24 +825,12 @@ haveidle: * preempted source thread will be resumed the instant the target blocks * whether or not the source is scheduled (i.e. preemption is supposed to * be as transparent as possible). - * - * The target thread inherits our MP count (added to its own) for the - * duration of the preemption in order to preserve the atomicy of the - * MP lock during the preemption. Therefore, any preempting targets must be - * careful in regards to MP assertions. Note that the MP count may be - * out of sync with the physical mp_lock, but we do not have to preserve - * the original ownership of the lock if it was out of synch (that is, we - * can leave it synchronized on return). */ void lwkt_preempt(thread_t ntd, int critcount) { struct globaldata *gd = mycpu; thread_t td; -#ifdef SMP - int mpheld; - int savecnt; -#endif int save_gd_intr_nesting_level; /* @@ -988,24 +894,6 @@ lwkt_preempt(thread_t ntd, int critcount) need_lwkt_resched(); return; } -#ifdef SMP - /* - * NOTE: An interrupt might have occured just as we were transitioning - * to or from the MP lock. In this case td_mpcount will be pre-disposed - * (non-zero) but not actually synchronized with the mp_lock itself. - * We can use it to imply an MP lock requirement for the preemption but - * we cannot use it to test whether we hold the MP lock or not. - */ - savecnt = td->td_mpcount; - mpheld = MP_LOCK_HELD(gd); - ntd->td_xpcount = td->td_mpcount + td->td_xpcount; - if (mpheld == 0 && ntd->td_mpcount + ntd->td_xpcount && !cpu_try_mplock()) { - ntd->td_xpcount = 0; - ++preempt_miss; - need_lwkt_resched(); - return; - } -#endif /* * Since we are able to preempt the current thread, there is no need to @@ -1025,14 +913,6 @@ lwkt_preempt(thread_t ntd, int critcount) gd->gd_intr_nesting_level = save_gd_intr_nesting_level; KKASSERT(ntd->td_preempted && (td->td_flags & TDF_PREEMPT_DONE)); -#ifdef SMP - KKASSERT(savecnt == td->td_mpcount); - mpheld = MP_LOCK_HELD(gd); - if (mpheld && td->td_mpcount == 0) - cpu_rel_mplock(gd->gd_cpuid); - else if (mpheld == 0 && td->td_mpcount + td->td_xpcount) - panic("lwkt_preempt(): MP lock was not held through"); -#endif ntd->td_preempted = NULL; td->td_flags &= ~(TDF_PREEMPT_LOCK|TDF_PREEMPT_DONE); } @@ -1148,19 +1028,6 @@ lwkt_user_yield(void) if ((gd->gd_reqflags & RQF_IDLECHECK_MASK) && td->td_nest_count < 2) splz(); -#ifdef SMP - /* - * XXX SEVERE TEMPORARY HACK. A cpu-bound operation running in the - * kernel can prevent other cpus from servicing interrupt threads - * which still require the MP lock (which is a lot of them). This - * has a chaining effect since if the interrupt is blocked, so is - * the event, so normal scheduling will not pick up on the problem. - */ - if (cpu_contention_mask && td->td_mpcount + td->td_xpcount) { - yield_mplock(td); - } -#endif - /* * Switch (which forces a release) if another kernel thread needs * the cpu, if userland wants us to resched, or if our kernel diff --git a/sys/kern/lwkt_token.c b/sys/kern/lwkt_token.c index 5d6fdb8ad8..a298645919 100644 --- a/sys/kern/lwkt_token.c +++ b/sys/kern/lwkt_token.c @@ -106,6 +106,11 @@ KTR_INFO(KTR_TOKENS, tokens, contention_stop, 7, UNCONTENDED_STRING, sizeof(void #define logtoken(name, ref) \ KTR_LOG(tokens_ ## name, ref, ref->tr_tok, curthread) +/* + * Cpu contention mask for directed wakeups. + */ +cpumask_t cpu_contention_mask; + /* * Global tokens. These replace the MP lock for major subsystem locking. * These tokens are initially used to lockup both global and individual @@ -120,6 +125,7 @@ KTR_INFO(KTR_TOKENS, tokens, contention_stop, 7, UNCONTENDED_STRING, sizeof(void * any time, the MP state is copied to the tokref when the token is acquired * and will not race against sysctl changes. */ +struct lwkt_token mp_token = LWKT_TOKEN_MP_INITIALIZER(mp_token); struct lwkt_token pmap_token = LWKT_TOKEN_UP_INITIALIZER(pmap_token); struct lwkt_token dev_token = LWKT_TOKEN_UP_INITIALIZER(dev_token); struct lwkt_token vm_token = LWKT_TOKEN_UP_INITIALIZER(vm_token); @@ -154,6 +160,8 @@ SYSCTL_INT(_lwkt, OID_AUTO, vmobj_mpsafe, CTLFLAG_RW, * to acquire needed tokens in addition to a normal lwkt_gettoken() * stall. */ +SYSCTL_LONG(_lwkt, OID_AUTO, mp_collisions, CTLFLAG_RW, + &mp_token.t_collisions, 0, "Collision counter of mp_token"); SYSCTL_LONG(_lwkt, OID_AUTO, pmap_collisions, CTLFLAG_RW, &pmap_token.t_collisions, 0, "Collision counter of pmap_token"); SYSCTL_LONG(_lwkt, OID_AUTO, dev_collisions, CTLFLAG_RW, @@ -171,6 +179,21 @@ SYSCTL_LONG(_lwkt, OID_AUTO, tty_collisions, CTLFLAG_RW, SYSCTL_LONG(_lwkt, OID_AUTO, vnode_collisions, CTLFLAG_RW, &vnode_token.t_collisions, 0, "Collision counter of vnode_token"); +#ifdef SMP +/* + * Acquire the initial mplock + * + * (low level boot only) + */ +void +cpu_get_initial_mplock(void) +{ + KKASSERT(mp_token.t_ref == NULL); + if (lwkt_trytoken(&mp_token) == FALSE) + panic("cpu_get_initial_mplock"); +} +#endif + /* * Return a pool token given an address */ @@ -189,25 +212,42 @@ _lwkt_token_pool_lookup(void *ptr) * token array. * * As an optimization we set the MPSAFE flag if the thread is already - * holding the MP lock. This bypasses unncessary calls to get_mplock() and + * holding the mp_token. This bypasses unncessary calls to get_mplock() and * rel_mplock() on tokens which are not normally MPSAFE when the thread * is already holding the MP lock. - * - * WARNING: The inherited td_xpcount does not count here because a switch - * could schedule the preempted thread and blow away the inherited - * mplock. */ +static __inline +intptr_t +_lwkt_tok_flags(lwkt_token_t tok, thread_t td) +{ + intptr_t flags; + + /* + * tok->t_flags can change out from under us, make sure we have + * a local copy. + */ + flags = tok->t_flags; + cpu_ccfence(); +#ifdef SMP + if ((flags & LWKT_TOKEN_MPSAFE) == 0 && + _lwkt_token_held(&mp_token, td)) { + return (flags | LWKT_TOKEN_MPSAFE); + } else { + return (flags); + } +#else + return (flags | LWKT_TOKEN_MPSAFE); +#endif +} + static __inline void -_lwkt_tokref_init(lwkt_tokref_t ref, lwkt_token_t tok, thread_t td) +_lwkt_tokref_init(lwkt_tokref_t ref, lwkt_token_t tok, thread_t td, + intptr_t flags) { ref->tr_tok = tok; ref->tr_owner = td; - ref->tr_flags = tok->t_flags; -#ifdef SMP - if (td->td_mpcount) -#endif - ref->tr_flags |= LWKT_TOKEN_MPSAFE; + ref->tr_flags = flags; } /* @@ -378,39 +418,25 @@ _lwkt_trytokref2(lwkt_tokref_t nref, thread_t td, int blocking) } /* - * Acquire a serializing token. This routine does not block. + * Get a serializing token. This routine can block. */ -static __inline -int -_lwkt_trytokref(lwkt_tokref_t ref, thread_t td) -{ - if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0) { - if (try_mplock() == 0) { - --td->td_toks_stop; - return (FALSE); - } - } - if (_lwkt_trytokref2(ref, td, 0) == FALSE) { - /* - * Cleanup, deactivate the failed token. - */ - if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0) - rel_mplock(); - --td->td_toks_stop; - return (FALSE); - } - return (TRUE); -} - -/* - * Acquire a serializing token. This routine can block. - */ -static __inline void -_lwkt_gettokref(lwkt_tokref_t ref, thread_t td, const void **stkframe) +lwkt_gettoken(lwkt_token_t tok) { - if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0) + thread_t td = curthread; + lwkt_tokref_t ref; + intptr_t flags; + + flags = _lwkt_tok_flags(tok, td); + if ((flags & LWKT_TOKEN_MPSAFE) == 0) get_mplock(); + + ref = td->td_toks_stop; + KKASSERT(ref < &td->td_toks_end); + ++td->td_toks_stop; + cpu_ccfence(); + _lwkt_tokref_init(ref, tok, td, flags); + if (_lwkt_trytokref2(ref, td, 1) == FALSE) { /* * Give up running if we can't acquire the token right now. @@ -423,7 +449,6 @@ _lwkt_gettokref(lwkt_tokref_t ref, thread_t td, const void **stkframe) * return tr_tok->t_ref should be assigned to this specific * ref. */ - ref->tr_stallpc = stkframe[-1]; atomic_add_long(&ref->tr_tok->t_collisions, 1); logtoken(fail, ref); lwkt_switch(); @@ -432,32 +457,41 @@ _lwkt_gettokref(lwkt_tokref_t ref, thread_t td, const void **stkframe) } } -void -lwkt_gettoken(lwkt_token_t tok) -{ - thread_t td = curthread; - lwkt_tokref_t ref; - - ref = td->td_toks_stop; - KKASSERT(ref < &td->td_toks_end); - ++td->td_toks_stop; - cpu_ccfence(); - _lwkt_tokref_init(ref, tok, td); - _lwkt_gettokref(ref, td, (const void **)&tok); -} - void lwkt_gettoken_hard(lwkt_token_t tok) { thread_t td = curthread; lwkt_tokref_t ref; + intptr_t flags; + + flags = _lwkt_tok_flags(tok, td); + if ((flags & LWKT_TOKEN_MPSAFE) == 0) + get_mplock(); ref = td->td_toks_stop; KKASSERT(ref < &td->td_toks_end); ++td->td_toks_stop; cpu_ccfence(); - _lwkt_tokref_init(ref, tok, td); - _lwkt_gettokref(ref, td, (const void **)&tok); + _lwkt_tokref_init(ref, tok, td, flags); + + if (_lwkt_trytokref2(ref, td, 1) == FALSE) { + /* + * Give up running if we can't acquire the token right now. + * + * Since the tokref is already active the scheduler now + * takes care of acquisition, so we need only call + * lwkt_switch(). + * + * Since we failed this was not a recursive token so upon + * return tr_tok->t_ref should be assigned to this specific + * ref. + */ + atomic_add_long(&ref->tr_tok->t_collisions, 1); + logtoken(fail, ref); + lwkt_switch(); + logtoken(succ, ref); + KKASSERT(ref->tr_tok->t_ref == ref); + } crit_enter_hard_gd(td->td_gd); } @@ -467,14 +501,37 @@ lwkt_getpooltoken(void *ptr) thread_t td = curthread; lwkt_token_t tok; lwkt_tokref_t ref; + intptr_t flags; + + tok = _lwkt_token_pool_lookup(ptr); + flags = _lwkt_tok_flags(tok, td); + if ((flags & LWKT_TOKEN_MPSAFE) == 0) + get_mplock(); ref = td->td_toks_stop; KKASSERT(ref < &td->td_toks_end); ++td->td_toks_stop; cpu_ccfence(); - tok = _lwkt_token_pool_lookup(ptr); - _lwkt_tokref_init(ref, tok, td); - _lwkt_gettokref(ref, td, (const void **)&ptr); + _lwkt_tokref_init(ref, tok, td, flags); + + if (_lwkt_trytokref2(ref, td, 1) == FALSE) { + /* + * Give up running if we can't acquire the token right now. + * + * Since the tokref is already active the scheduler now + * takes care of acquisition, so we need only call + * lwkt_switch(). + * + * Since we failed this was not a recursive token so upon + * return tr_tok->t_ref should be assigned to this specific + * ref. + */ + atomic_add_long(&ref->tr_tok->t_collisions, 1); + logtoken(fail, ref); + lwkt_switch(); + logtoken(succ, ref); + KKASSERT(ref->tr_tok->t_ref == ref); + } return(tok); } @@ -486,13 +543,36 @@ lwkt_trytoken(lwkt_token_t tok) { thread_t td = curthread; lwkt_tokref_t ref; + intptr_t flags; + + flags = _lwkt_tok_flags(tok, td); + if ((flags & LWKT_TOKEN_MPSAFE) == 0) { + if (try_mplock() == 0) + return (FALSE); + } ref = td->td_toks_stop; KKASSERT(ref < &td->td_toks_end); ++td->td_toks_stop; cpu_ccfence(); - _lwkt_tokref_init(ref, tok, td); - return(_lwkt_trytokref(ref, td)); + _lwkt_tokref_init(ref, tok, td, flags); + + if (_lwkt_trytokref2(ref, td, 0) == FALSE) { + /* + * Cleanup, deactivate the failed token. + */ + if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0) { + cpu_ccfence(); + --td->td_toks_stop; + cpu_ccfence(); + rel_mplock(); + } else { + cpu_ccfence(); + --td->td_toks_stop; + } + return (FALSE); + } + return (TRUE); } /* @@ -516,27 +596,31 @@ lwkt_reltoken(lwkt_token_t tok) /* * Only clear the token if it matches ref. If ref was a recursively - * acquired token it may not match. + * acquired token it may not match. Then adjust td_toks_stop. * - * If the token was not MPSAFE release the MP lock. + * Some comparisons must be run prior to adjusting td_toks_stop + * to avoid racing against a fast interrupt/ ipi which tries to + * acquire a token. * - * NOTE: We have to do this before adjust td_toks_stop, otherwise - * a fast interrupt can come along and reuse our ref while - * tok is still attached to it. + * We must also be absolutely sure that the compiler does not + * reorder the clearing of t_ref and the adjustment of td_toks_stop, + * or reorder the adjustment of td_toks_stop against the conditional. + * + * NOTE: The mplock is a token also so sequencing is a bit complex. */ if (tok->t_ref == ref) tok->t_ref = NULL; - cpu_ccfence(); - if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0) + cpu_sfence(); + if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0) { + cpu_ccfence(); + td->td_toks_stop = ref; + cpu_ccfence(); rel_mplock(); - - /* - * Finally adjust td_toks_stop, be very sure that the compiler - * does not reorder the clearing of tok->t_ref with the - * decrementing of td->td_toks_stop. - */ - cpu_ccfence(); - td->td_toks_stop = ref; + } else { + cpu_ccfence(); + td->td_toks_stop = ref; + cpu_ccfence(); + } KKASSERT(tok->t_ref != ref); } @@ -559,6 +643,23 @@ lwkt_relpooltoken(void *ptr) lwkt_reltoken(tok); } +/* + * Return a count of the number of token refs the thread has to the + * specified token, whether it currently owns the token or not. + */ +int +lwkt_cnttoken(lwkt_token_t tok, thread_t td) +{ + lwkt_tokref_t scan; + int count = 0; + + for (scan = &td->td_toks_base; scan < td->td_toks_stop; ++scan) { + if (scan->tr_tok == tok) + ++count; + } + return(count); +} + /* * Pool tokens are used to provide a type-stable serializing token diff --git a/sys/kern/tty_cons.c b/sys/kern/tty_cons.c index 73e0eeaa2c..cb9e2de5b4 100644 --- a/sys/kern/tty_cons.c +++ b/sys/kern/tty_cons.c @@ -189,9 +189,6 @@ done: * the SMP/AP boot will blow up on us. */ lwkt_reltoken(&tty_token); -#ifdef SMP - KKASSERT(curthread->td_mpcount == 1); -#endif } diff --git a/sys/kern/usched_bsd4.c b/sys/kern/usched_bsd4.c index e72170a44c..29dd464152 100644 --- a/sys/kern/usched_bsd4.c +++ b/sys/kern/usched_bsd4.c @@ -335,12 +335,6 @@ bsd4_acquire_curproc(struct lwp *lp) * Additionally, note that we may already be on a run queue if releasing * via the lwkt_switch() in bsd4_setrunqueue(). * - * WARNING! The MP lock may be in an unsynchronized state due to the - * way get_mplock() works and the fact that this function may be called - * from a passive release during a lwkt_switch(). try_mplock() will deal - * with this for us but you should be aware that td_mpcount may not be - * useable. - * * MPSAFE */ static void diff --git a/sys/kern/usched_dummy.c b/sys/kern/usched_dummy.c index 301de531fb..cf9383400e 100644 --- a/sys/kern/usched_dummy.c +++ b/sys/kern/usched_dummy.c @@ -197,12 +197,6 @@ dummy_acquire_curproc(struct lwp *lp) * This routine is also responsible for selecting a new thread to * make the current thread. * - * WARNING! The MP lock may be in an unsynchronized state due to the - * way get_mplock() works and the fact that this function may be called - * from a passive release during a lwkt_switch(). try_mplock() will deal - * with this for us but you should be aware that td_mpcount may not be - * useable. - * * MPSAFE */ static void diff --git a/sys/platform/pc32/apic/apic_vector.s b/sys/platform/pc32/apic/apic_vector.s index 92328b1261..3134e55cd9 100644 --- a/sys/platform/pc32/apic/apic_vector.s +++ b/sys/platform/pc32/apic/apic_vector.s @@ -124,8 +124,7 @@ * - Push the trap frame required by doreti * - Mask the interrupt and reenable its source * - If we cannot take the interrupt set its fpending bit and - * doreti. Note that we cannot mess with mp_lock at all - * if we entered from a critical section! + * doreti. * - If we can take the interrupt clear its fpending bit, * call the handler, then unmask and doreti. * diff --git a/sys/platform/pc32/i386/exception.s b/sys/platform/pc32/i386/exception.s index c2f813515f..da9b70fe30 100644 --- a/sys/platform/pc32/i386/exception.s +++ b/sys/platform/pc32/i386/exception.s @@ -918,18 +918,6 @@ ENTRY(fork_trampoline) sti call splz -#if defined(INVARIANTS) && defined(SMP) - movl PCPU(curthread),%eax - cmpl $0,TD_MPCOUNT(%eax) - je 1f - pushl %esi - pushl TD_MPCOUNT(%eax) - pushl $pmsg4 - call panic -pmsg4: .asciz "fork_trampoline mpcount %d after calling %p" - .p2align 2 -1: -#endif /* * Return via doreti to handle ASTs. */ diff --git a/sys/platform/pc32/i386/genassym.c b/sys/platform/pc32/i386/genassym.c index 4cbb97ce27..f3813d47ec 100644 --- a/sys/platform/pc32/i386/genassym.c +++ b/sys/platform/pc32/i386/genassym.c @@ -89,9 +89,6 @@ ASSYM(TD_PRI, offsetof(struct thread, td_pri)); ASSYM(TD_MACH, offsetof(struct thread, td_mach)); ASSYM(TD_WCHAN, offsetof(struct thread, td_wchan)); ASSYM(TD_NEST_COUNT, offsetof(struct thread, td_nest_count)); -#ifdef SMP -ASSYM(TD_MPCOUNT, offsetof(struct thread, td_mpcount)); -#endif ASSYM(TD_CRITCOUNT, offsetof(struct thread, td_critcount)); ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); ASSYM(TDF_RUNNING, TDF_RUNNING); diff --git a/sys/platform/pc32/i386/locore.s b/sys/platform/pc32/i386/locore.s index d550de1f94..37d28fbf20 100644 --- a/sys/platform/pc32/i386/locore.s +++ b/sys/platform/pc32/i386/locore.s @@ -816,11 +816,6 @@ map_read_write: movl $1, %ecx /* one private pt coming right up */ fillkpt(R(SMPptpa), $PG_RW) -#ifdef SMP -/* Initialize mp lock to allow early traps */ - movl $0, R(mp_lock) -#endif /* SMP */ - /* install a pde for temporary double map of bottom of VA */ movl R(KPTphys), %eax xorl %ebx, %ebx diff --git a/sys/platform/pc32/i386/machdep.c b/sys/platform/pc32/i386/machdep.c index 5af7336b07..dd9bad568c 100644 --- a/sys/platform/pc32/i386/machdep.c +++ b/sys/platform/pc32/i386/machdep.c @@ -2621,14 +2621,11 @@ struct spinlock_deprecated smp_rv_spinlock; static void init_locks(void) { +#ifdef SMP /* - * mp_lock = 0; BSP already owns the MP lock - */ - /* - * Get the initial mp_lock with a count of 1 for the BSP. + * Get the initial mplock with a count of 1 for the BSP. * This uses a LOGICAL cpu ID, ie BSP == 0. */ -#ifdef SMP cpu_get_initial_mplock(); #endif /* DEPRECATED */ diff --git a/sys/platform/pc32/i386/mp_machdep.c b/sys/platform/pc32/i386/mp_machdep.c index 46d7abbd9d..fa1538cbe0 100644 --- a/sys/platform/pc32/i386/mp_machdep.c +++ b/sys/platform/pc32/i386/mp_machdep.c @@ -2761,9 +2761,6 @@ ap_init(void) * * Note: We are in a critical section. * - * Note: We have to synchronize td_mpcount to our desired MP state - * before calling cpu_try_mplock(). - * * Note: we are the idle thread, we can only spin. * * Note: The load fence is memory volatile and prevents the compiler @@ -2771,17 +2768,16 @@ ap_init(void) * caching it. */ while (mp_finish == 0) - cpu_lfence(); - ++curthread->td_mpcount; - while (cpu_try_mplock() == 0) - ; + cpu_lfence(); + while (try_mplock() == 0) + ; if (cpu_feature & CPUID_TSC) { - /* - * The BSP is constantly updating tsc0_offset, figure out the - * relative difference to synchronize ktrdump. - */ - tsc_offsets[mycpu->gd_cpuid] = rdtsc() - tsc0_offset; + /* + * The BSP is constantly updating tsc0_offset, figure out + * the relative difference to synchronize ktrdump. + */ + tsc_offsets[mycpu->gd_cpuid] = rdtsc() - tsc0_offset; } /* BSP may have changed PTD while we're waiting for the lock */ @@ -2825,7 +2821,7 @@ ap_init(void) * The idle thread is never placed on the runq, make sure * nothing we've done put it there. */ - KKASSERT(curthread->td_mpcount == 1); + KKASSERT(get_mplock_count(curthread) == 1); smp_active_mask |= CPUMASK(mycpu->gd_cpuid); /* diff --git a/sys/platform/pc32/i386/trap.c b/sys/platform/pc32/i386/trap.c index c4c144a795..c89928424b 100644 --- a/sys/platform/pc32/i386/trap.c +++ b/sys/platform/pc32/i386/trap.c @@ -476,11 +476,11 @@ restart: if (frame->tf_eflags & PSL_VM && (type == T_PROTFLT || type == T_STKFLT)) { #ifdef SMP - KKASSERT(td->td_mpcount > 0); + KKASSERT(get_mplock_count(curthread) > 0); #endif i = vm86_emulate((struct vm86frame *)frame); #ifdef SMP - KKASSERT(td->td_mpcount > 0); + KKASSERT(get_mplock_count(curthread) > 0); #endif if (i != 0) { /* @@ -909,12 +909,6 @@ kernel_trap: #endif out: -#ifdef SMP - if (ISPL(frame->tf_cs) == SEL_UPL) { - KASSERT(td->td_mpcount == have_mplock, - ("badmpcount trap/end from %p", (void *)frame->tf_eip)); - } -#endif userret(lp, frame, sticks); userexit(lp); out2: ; @@ -1051,7 +1045,6 @@ trap_fatal(struct trapframe *frame, vm_offset_t eva) ISPL(frame->tf_cs) == SEL_UPL ? "user" : "kernel"); #ifdef SMP /* three separate prints in case of a trap on an unmapped page */ - kprintf("mp_lock = %08x; ", mp_lock); kprintf("cpuid = %d; ", mycpu->gd_cpuid); kprintf("lapic.id = %08x\n", lapic.id); #endif @@ -1168,7 +1161,6 @@ dblfault_handler(void) kprintf("ebp = 0x%x\n", gd->gd_common_tss.tss_ebp); #ifdef SMP /* three separate prints in case of a trap on an unmapped page */ - kprintf("mp_lock = %08x; ", mp_lock); kprintf("cpuid = %d; ", gd->mi.gd_cpuid); kprintf("lapic.id = %08x\n", lapic.id); #endif @@ -1216,10 +1208,6 @@ syscall2(struct trapframe *frame) KTR_LOG(kernentry_syscall, p->p_pid, lp->lwp_tid, frame->tf_eax); -#ifdef SMP - KASSERT(td->td_mpcount == 0, - ("badmpcount syscall2 from %p", (void *)frame->tf_eip)); -#endif userenter(td, p); /* lazy raise our priority */ /* @@ -1408,9 +1396,6 @@ bad: /* * Release the MP lock if we had to get it */ - KASSERT(td->td_mpcount == have_mplock, - ("badmpcount syscall2/end from %p callp %p", - (void *)frame->tf_eip, callp)); if (have_mplock) rel_mplock(); #endif diff --git a/sys/platform/pc32/i386/vm86.c b/sys/platform/pc32/i386/vm86.c index be479d0cc4..31913e679f 100644 --- a/sys/platform/pc32/i386/vm86.c +++ b/sys/platform/pc32/i386/vm86.c @@ -630,7 +630,7 @@ vm86_intcall(int intnum, struct vm86frame *vmf) return (EINVAL); crit_enter(); - ASSERT_MP_LOCK_HELD(curthread); + ASSERT_MP_LOCK_HELD(); vm86_setup_timer_fault(); vmf->vmf_trapno = intnum; @@ -666,7 +666,7 @@ vm86_datacall(int intnum, struct vm86frame *vmf, struct vm86context *vmc) int i, entry, retval; crit_enter(); - ASSERT_MP_LOCK_HELD(curthread); + ASSERT_MP_LOCK_HELD(); for (i = 0; i < vmc->npages; i++) { page = vtophys(vmc->pmap[i].kva & PG_FRAME); diff --git a/sys/platform/pc32/i386/vm_machdep.c b/sys/platform/pc32/i386/vm_machdep.c index fcc81dfb7f..a2a4ab4c44 100644 --- a/sys/platform/pc32/i386/vm_machdep.c +++ b/sys/platform/pc32/i386/vm_machdep.c @@ -374,13 +374,9 @@ kvtop(void *addr) static void cpu_reset_proxy(void) { - u_int saved_mp_lock; - cpu_reset_proxy_active = 1; while (cpu_reset_proxy_active == 1) ; /* Wait for other cpu to disable interupts */ - saved_mp_lock = mp_lock; - mp_lock = 0; /* BSP */ kprintf("cpu_reset_proxy: Grabbed mp lock for BSP\n"); cpu_reset_proxy_active = 3; while (cpu_reset_proxy_active == 3) diff --git a/sys/platform/pc32/isa/ipl.s b/sys/platform/pc32/isa/ipl.s index 2feb2376e4..97c2a32c4b 100644 --- a/sys/platform/pc32/isa/ipl.s +++ b/sys/platform/pc32/isa/ipl.s @@ -92,8 +92,7 @@ fastunpend_count: .long 0 * checks the cpl for unmasked pending interrupts (fast, normal, or * soft) and schedules them if appropriate, then irets. * - * If we are in a critical section we cannot run any pending ints - * nor can be play with mp_lock. + * If we are in a critical section we cannot run any pending ints. * * NOTE: Since SPLs no longer exist, all callers of this function * push $0 for the CPL. HOWEVER, we *STILL* use the cpl mask within diff --git a/sys/platform/pc64/apic/apic_vector.s b/sys/platform/pc64/apic/apic_vector.s index 21d9c9110b..7fdb1f1bad 100644 --- a/sys/platform/pc64/apic/apic_vector.s +++ b/sys/platform/pc64/apic/apic_vector.s @@ -109,8 +109,7 @@ * - Push the trap frame required by doreti * - Mask the interrupt and reenable its source * - If we cannot take the interrupt set its fpending bit and - * doreti. Note that we cannot mess with mp_lock at all - * if we entered from a critical section! + * doreti. * - If we can take the interrupt clear its fpending bit, * call the handler, then unmask and doreti. * diff --git a/sys/platform/pc64/x86_64/exception.S b/sys/platform/pc64/x86_64/exception.S index 15ae309ef3..10e3debde0 100644 --- a/sys/platform/pc64/x86_64/exception.S +++ b/sys/platform/pc64/x86_64/exception.S @@ -440,20 +440,6 @@ ENTRY(fork_trampoline) sti call splz -#if defined(INVARIANTS) && defined(SMP) - movq PCPU(curthread),%rax - cmpl $0,TD_MPCOUNT(%rax) - je 1f - movq $pmsg4, %rdi - movl TD_MPCOUNT(%rax), %esi - movq %rbx, %rdx - xorl %eax, %eax - call panic -pmsg4: .asciz "fork_trampoline mpcount %d after calling %p" - /* JG what's the purpose of this alignment and is it enough on x86_64? */ - .p2align 2 -1: -#endif /* * Return via doreti to handle ASTs. * diff --git a/sys/platform/pc64/x86_64/genassym.c b/sys/platform/pc64/x86_64/genassym.c index 6dd09f466e..8ce794f083 100644 --- a/sys/platform/pc64/x86_64/genassym.c +++ b/sys/platform/pc64/x86_64/genassym.c @@ -182,9 +182,6 @@ ASSYM(TD_CRITCOUNT, offsetof(struct thread, td_critcount)); ASSYM(TD_MACH, offsetof(struct thread, td_mach)); ASSYM(TD_WCHAN, offsetof(struct thread, td_wchan)); ASSYM(TD_NEST_COUNT, offsetof(struct thread, td_nest_count)); -#ifdef SMP -ASSYM(TD_MPCOUNT, offsetof(struct thread, td_mpcount)); -#endif ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); ASSYM(TD_SAVEFPU, offsetof(struct thread, td_savefpu)); ASSYM(TDF_RUNNING, TDF_RUNNING); diff --git a/sys/platform/pc64/x86_64/ipl.s b/sys/platform/pc64/x86_64/ipl.s index 1f7ce0ebf5..c6af9f8230 100644 --- a/sys/platform/pc64/x86_64/ipl.s +++ b/sys/platform/pc64/x86_64/ipl.s @@ -130,8 +130,7 @@ fastunpend_count: .long 0 * checks the cpl for unmasked pending interrupts (fast, normal, or * soft) and schedules them if appropriate, then irets. * - * If we are in a critical section we cannot run any pending ints - * nor can be play with mp_lock. + * If we are in a critical section we cannot run any pending ints. * * The stack contains a trapframe at the start of doreti. */ diff --git a/sys/platform/pc64/x86_64/machdep.c b/sys/platform/pc64/x86_64/machdep.c index da5d5e21d1..5d36728cd6 100644 --- a/sys/platform/pc64/x86_64/machdep.c +++ b/sys/platform/pc64/x86_64/machdep.c @@ -2356,14 +2356,11 @@ struct spinlock_deprecated clock_spinlock; static void init_locks(void) { +#ifdef SMP /* - * mp_lock = 0; BSP already owns the MP lock - */ - /* - * Get the initial mp_lock with a count of 1 for the BSP. + * Get the initial mplock with a count of 1 for the BSP. * This uses a LOGICAL cpu ID, ie BSP == 0. */ -#ifdef SMP cpu_get_initial_mplock(); #endif /* DEPRECATED */ diff --git a/sys/platform/pc64/x86_64/mp_machdep.c b/sys/platform/pc64/x86_64/mp_machdep.c index 0966a83a3b..19c39dc030 100644 --- a/sys/platform/pc64/x86_64/mp_machdep.c +++ b/sys/platform/pc64/x86_64/mp_machdep.c @@ -2756,9 +2756,6 @@ ap_init(void) * * Note: We are in a critical section. * - * Note: We have to synchronize td_mpcount to our desired MP state - * before calling cpu_try_mplock(). - * * Note: we are the idle thread, we can only spin. * * Note: The load fence is memory volatile and prevents the compiler @@ -2766,17 +2763,16 @@ ap_init(void) * caching it. */ while (mp_finish == 0) - cpu_lfence(); - ++curthread->td_mpcount; - while (cpu_try_mplock() == 0) - ; + cpu_lfence(); + while (try_mplock() == 0) + ; if (cpu_feature & CPUID_TSC) { - /* - * The BSP is constantly updating tsc0_offset, figure out the - * relative difference to synchronize ktrdump. - */ - tsc_offsets[mycpu->gd_cpuid] = rdtsc() - tsc0_offset; + /* + * The BSP is constantly updating tsc0_offset, figure out + * the relative difference to synchronize ktrdump. + */ + tsc_offsets[mycpu->gd_cpuid] = rdtsc() - tsc0_offset; } /* BSP may have changed PTD while we're waiting for the lock */ @@ -2823,7 +2819,7 @@ ap_init(void) * The idle thread is never placed on the runq, make sure * nothing we've done put it there. */ - KKASSERT(curthread->td_mpcount == 1); + KKASSERT(get_mplock_count(curthread) == 1); smp_active_mask |= CPUMASK(mycpu->gd_cpuid); /* diff --git a/sys/platform/pc64/x86_64/trap.c b/sys/platform/pc64/x86_64/trap.c index e4e8214532..8d912c043a 100644 --- a/sys/platform/pc64/x86_64/trap.c +++ b/sys/platform/pc64/x86_64/trap.c @@ -756,12 +756,6 @@ trap(struct trapframe *frame) #endif out: -#ifdef SMP - if (ISPL(frame->tf_cs) == SEL_UPL) { - KASSERT(td->td_mpcount == have_mplock, - ("badmpcount trap/end from %p", (void *)frame->tf_rip)); - } -#endif userret(lp, frame, sticks); userexit(lp); out2: ; @@ -921,7 +915,6 @@ trap_fatal(struct trapframe *frame, vm_offset_t eva) ISPL(frame->tf_cs) == SEL_UPL ? "user" : "kernel"); #ifdef SMP /* three separate prints in case of a trap on an unmapped page */ - kprintf("mp_lock = %08x; ", mp_lock); kprintf("cpuid = %d; ", mycpu->gd_cpuid); kprintf("lapic->id = %08x\n", lapic->id); #endif @@ -1021,7 +1014,6 @@ dblfault_handler(struct trapframe *frame) kprintf("rbp = 0x%lx\n", frame->tf_rbp); #ifdef SMP /* three separate prints in case of a trap on an unmapped page */ - kprintf("mp_lock = %08x; ", mp_lock); kprintf("cpuid = %d; ", mycpu->gd_cpuid); kprintf("lapic->id = %08x\n", lapic->id); #endif @@ -1075,10 +1067,6 @@ syscall2(struct trapframe *frame) KTR_LOG(kernentry_syscall, p->p_pid, lp->lwp_tid, frame->tf_rax); -#ifdef SMP - KASSERT(td->td_mpcount == 0, - ("badmpcount syscall2 from %p", (void *)frame->tf_rip)); -#endif userenter(td, p); /* lazy raise our priority */ reg = 0; @@ -1271,8 +1259,6 @@ bad: /* * Release the MP lock if we had to get it */ - KASSERT(td->td_mpcount == have_mplock, - ("badmpcount syscall2/end from %p", (void *)frame->tf_rip)); if (have_mplock) rel_mplock(); #endif diff --git a/sys/platform/vkernel/i386/cpu_regs.c b/sys/platform/vkernel/i386/cpu_regs.c index a849c18a29..8338461570 100644 --- a/sys/platform/vkernel/i386/cpu_regs.c +++ b/sys/platform/vkernel/i386/cpu_regs.c @@ -704,9 +704,6 @@ cpu_idle(void) crit_exit(); KKASSERT(td->td_critcount == 0); -#ifdef SMP - KKASSERT(td->td_mpcount == 0); -#endif cpu_enable_intr(); for (;;) { /* @@ -714,9 +711,6 @@ cpu_idle(void) */ lwkt_switch(); -#ifdef SMP - KKASSERT(td->td_mpcount == 0); -#endif /* * The idle loop halts only if no threads are scheduleable * and no signals have occured. @@ -725,8 +719,7 @@ cpu_idle(void) (td->td_flags & TDF_IDLE_NOHLT) == 0) { splz(); #ifdef SMP - KKASSERT(td->td_mpcount == 0); - KKASSERT(MP_LOCK_HELD(td->td_gd) == 0); + KKASSERT(MP_LOCK_HELD() == 0); #endif if (!lwkt_runnable()) { #ifdef DEBUGIDLE diff --git a/sys/platform/vkernel/i386/fork_tramp.s b/sys/platform/vkernel/i386/fork_tramp.s index 9a454d2261..4b34122ebf 100644 --- a/sys/platform/vkernel/i386/fork_tramp.s +++ b/sys/platform/vkernel/i386/fork_tramp.s @@ -78,18 +78,6 @@ ENTRY(fork_trampoline) call splz -#if defined(INVARIANTS) && defined(SMP) - movl PCPU(curthread),%eax - cmpl $0,TD_MPCOUNT(%eax) - je 1f - pushl %esi - pushl TD_MPCOUNT(%eax) - pushl $pmsg4 - call panic -pmsg4: .asciz "fork_trampoline mpcount %d after calling %p" - .p2align 2 -1: -#endif /* * Return via doreti to handle ASTs. */ diff --git a/sys/platform/vkernel/i386/genassym.c b/sys/platform/vkernel/i386/genassym.c index 87108ef71c..31ca30de14 100644 --- a/sys/platform/vkernel/i386/genassym.c +++ b/sys/platform/vkernel/i386/genassym.c @@ -87,9 +87,6 @@ ASSYM(TD_CRITCOUNT, offsetof(struct thread, td_critcount)); ASSYM(TD_MACH, offsetof(struct thread, td_mach)); ASSYM(TD_WCHAN, offsetof(struct thread, td_wchan)); ASSYM(TD_NEST_COUNT, offsetof(struct thread, td_nest_count)); -#ifdef SMP -ASSYM(TD_MPCOUNT, offsetof(struct thread, td_mpcount)); -#endif ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); ASSYM(TDF_RUNNING, TDF_RUNNING); diff --git a/sys/platform/vkernel/i386/mp.c b/sys/platform/vkernel/i386/mp.c index c01f31033d..cb15ba077a 100644 --- a/sys/platform/vkernel/i386/mp.c +++ b/sys/platform/vkernel/i386/mp.c @@ -294,9 +294,6 @@ ap_init(void) * * Note: We are in a critical section. * - * Note: We have to synchronize td_mpcount to our desired MP state - * before calling cpu_try_mplock(). - * * Note: we are the idle thread, we can only spin. * * Note: The load fence is memory volatile and prevents the compiler @@ -308,8 +305,7 @@ ap_init(void) cpu_lfence(); DELAY(500000); } - ++curthread->td_mpcount; - while (cpu_try_mplock() == 0) + while (try_mplock() == 0) DELAY(100000); /* BSP may have changed PTD while we're waiting for the lock */ @@ -337,7 +333,7 @@ ap_init(void) * The idle thread is never placed on the runq, make sure * nothing we've done put it there. */ - KKASSERT(curthread->td_mpcount == 1); + KKASSERT(get_mplock_count(curthread) == 1); smp_active_mask |= CPUMASK(mycpu->gd_cpuid); mdcpu->gd_fpending = 0; diff --git a/sys/platform/vkernel/i386/trap.c b/sys/platform/vkernel/i386/trap.c index 5006e84d25..e2f28deb57 100644 --- a/sys/platform/vkernel/i386/trap.c +++ b/sys/platform/vkernel/i386/trap.c @@ -632,10 +632,6 @@ restart: #endif out: -#ifdef SMP - KASSERT(td->td_mpcount == have_mplock, - ("badmpcount trap/end from %p", (void *)frame->tf_eip)); -#endif userret(lp, frame, sticks); userexit(lp); out2: ; @@ -957,7 +953,6 @@ trap_fatal(struct trapframe *frame, int usermode, vm_offset_t eva) } #ifdef SMP /* two separate prints in case of a trap on an unmapped page */ - kprintf("mp_lock = %08x; ", mp_lock); kprintf("cpuid = %d\n", mycpu->gd_cpuid); #endif if (type == T_PAGEFLT) { @@ -1052,7 +1047,6 @@ dblfault_handler(void) kprintf("ebp = 0x%x\n", gd->gd_common_tss.tss_ebp); #ifdef SMP /* two separate prints in case of a trap on an unmapped page */ - kprintf("mp_lock = %08x; ", mp_lock); kprintf("cpuid = %d\n", mycpu->gd_cpuid); #endif panic("double fault"); @@ -1092,10 +1086,6 @@ syscall2(struct trapframe *frame) KTR_LOG(kernentry_syscall, lp->lwp_proc->p_pid, lp->lwp_tid, frame->tf_eax); -#ifdef SMP - KASSERT(td->td_mpcount == 0, - ("badmpcount syscall2 from %p", (void *)frame->tf_eip)); -#endif userenter(td, p); /* lazy raise our priority */ /* @@ -1281,8 +1271,6 @@ bad: /* * Release the MP lock if we had to get it */ - KASSERT(td->td_mpcount == have_mplock, - ("badmpcount syscall2/end from %p", (void *)frame->tf_eip)); if (have_mplock) rel_mplock(); #endif diff --git a/sys/platform/vkernel64/x86_64/fork_tramp.s b/sys/platform/vkernel64/x86_64/fork_tramp.s index 86fbd4cfbe..72cca283e6 100644 --- a/sys/platform/vkernel64/x86_64/fork_tramp.s +++ b/sys/platform/vkernel64/x86_64/fork_tramp.s @@ -79,20 +79,6 @@ ENTRY(fork_trampoline) call splz -#if defined(INVARIANTS) && defined(SMP) - movq PCPU(curthread),%rax - cmpl $0,TD_MPCOUNT(%rax) - je 1f - movq $pmsg4, %rdi - movl TD_MPCOUNT(%rax), %esi - movq %rbx, %rdx - xorl %eax, %eax - call panic -pmsg4: .asciz "fork_trampoline mpcount %d after calling %p" - /* JG what's the purpose of this alignment and is it enough on x86_64? */ - .p2align 2 -1: -#endif /* * Return via doreti to handle ASTs. * diff --git a/sys/platform/vkernel64/x86_64/genassym.c b/sys/platform/vkernel64/x86_64/genassym.c index ee7725a0cd..8b10f2868a 100644 --- a/sys/platform/vkernel64/x86_64/genassym.c +++ b/sys/platform/vkernel64/x86_64/genassym.c @@ -110,9 +110,6 @@ ASSYM(TD_PCB, offsetof(struct thread, td_pcb)); ASSYM(TD_SP, offsetof(struct thread, td_sp)); ASSYM(TD_PRI, offsetof(struct thread, td_pri)); ASSYM(TD_CRITCOUNT, offsetof(struct thread, td_critcount)); -#ifdef SMP -ASSYM(TD_MPCOUNT, offsetof(struct thread, td_mpcount)); -#endif ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); ASSYM(TD_SAVEFPU, offsetof(struct thread, td_savefpu)); ASSYM(TDF_RUNNING, TDF_RUNNING); diff --git a/sys/platform/vkernel64/x86_64/mp.c b/sys/platform/vkernel64/x86_64/mp.c index ce726f9ba1..03378c9e05 100644 --- a/sys/platform/vkernel64/x86_64/mp.c +++ b/sys/platform/vkernel64/x86_64/mp.c @@ -294,9 +294,6 @@ ap_init(void) * * Note: We are in a critical section. * - * Note: We have to synchronize td_mpcount to our desired MP state - * before calling cpu_try_mplock(). - * * Note: we are the idle thread, we can only spin. * * Note: The load fence is memory volatile and prevents the compiler @@ -308,8 +305,7 @@ ap_init(void) cpu_lfence(); DELAY(500000); } - ++curthread->td_mpcount; - while (cpu_try_mplock() == 0) + while (try_mplock() == 0) DELAY(100000); /* BSP may have changed PTD while we're waiting for the lock */ @@ -337,7 +333,7 @@ ap_init(void) * The idle thread is never placed on the runq, make sure * nothing we've done put it there. */ - KKASSERT(curthread->td_mpcount == 1); + KKASSERT(get_mplock_count(curthread) == 1); smp_active_mask |= CPUMASK(mycpu->gd_cpuid); mdcpu->gd_fpending = 0; diff --git a/sys/platform/vkernel64/x86_64/trap.c b/sys/platform/vkernel64/x86_64/trap.c index 5c46b35ffe..2214924a27 100644 --- a/sys/platform/vkernel64/x86_64/trap.c +++ b/sys/platform/vkernel64/x86_64/trap.c @@ -608,10 +608,6 @@ restart: #endif out: -#ifdef SMP - KASSERT(td->td_mpcount == have_mplock, - ("badmpcount trap/end from %p", (void *)frame->tf_rip)); -#endif userret(lp, frame, sticks); userexit(lp); out2: ; @@ -951,7 +947,6 @@ trap_fatal(struct trapframe *frame, int usermode, vm_offset_t eva) } #ifdef SMP /* two separate prints in case of a trap on an unmapped page */ - kprintf("mp_lock = %08x; ", mp_lock); kprintf("cpuid = %d\n", mycpu->gd_cpuid); #endif if (type == T_PAGEFLT) { @@ -1050,7 +1045,6 @@ dblfault_handler(void) #endif #ifdef SMP /* two separate prints in case of a trap on an unmapped page */ - kprintf("mp_lock = %08x; ", mp_lock); kprintf("cpuid = %d\n", mycpu->gd_cpuid); #endif panic("double fault"); @@ -1144,10 +1138,6 @@ syscall2(struct trapframe *frame) KTR_LOG(kernentry_syscall, lp->lwp_proc->p_pid, lp->lwp_tid, frame->tf_eax); -#ifdef SMP - KASSERT(td->td_mpcount == 0, - ("badmpcount syscall2 from %p", (void *)frame->tf_rip)); -#endif userenter(td, p); /* lazy raise our priority */ reg = 0; @@ -1342,8 +1332,6 @@ bad: /* * Release the MP lock if we had to get it */ - KASSERT(td->td_mpcount == have_mplock, - ("badmpcount syscall2/end from %p", (void *)frame->tf_rip)); if (have_mplock) rel_mplock(); #endif diff --git a/sys/sys/mplock2.h b/sys/sys/mplock2.h index 110899367e..160617d67d 100644 --- a/sys/sys/mplock2.h +++ b/sys/sys/mplock2.h @@ -18,151 +18,18 @@ #ifdef SMP -#define get_mplock() get_mplock_debug(__FILE__, __LINE__) -#define try_mplock() try_mplock_debug(__FILE__, __LINE__) -#define cpu_try_mplock() cpu_try_mplock_debug(__FILE__, __LINE__) -#define cpu_try_mplock_msg(lmsg) cpu_try_mplock_msg_debug(lmsg, __FILE__, __LINE__) +/* + * NOTE: try_mplock()/lwkt_trytoken() return non-zero on success. + */ +#define get_mplock() lwkt_gettoken(&mp_token) +#define try_mplock() lwkt_trytoken(&mp_token) +#define rel_mplock() lwkt_reltoken(&mp_token) +#define get_mplock_count(td) lwkt_cnttoken(&mp_token, td) -void _get_mplock_predisposed(const char *file, int line); -void _get_mplock_contested(const char *file, int line); -void _try_mplock_contested(const char *file, int line); -void _cpu_try_mplock_contested(const char *file, int line); -void _rel_mplock_contested(void); void cpu_get_initial_mplock(void); void handle_cpu_contention_mask(void); -void yield_mplock(struct thread *td); -extern int mp_lock; extern cpumask_t cpu_contention_mask; -extern const char *mp_lock_holder_file; -extern int mp_lock_holder_line; - -/* - * Acquire the MP lock, block until we get it. - * - * In order to acquire the MP lock we must first pre-dispose td_mpcount - * for the acquisition and then get the actual lock. - * - * The mplock must check a number of conditions and it is better to - * leave it to a procedure if we cannot get it trivially. - * - * WARNING: The mp_lock and td_mpcount are not necessarily synchronized. - * We must synchronize them here. They can be unsynchronized - * for a variety of reasons including predisposition, td_xpcount, - * and so forth. - */ -static __inline -void -get_mplock_debug(const char *file, int line) -{ - globaldata_t gd = mycpu; - thread_t td = gd->gd_curthread; - - ++td->td_mpcount; - if (mp_lock != gd->gd_cpuid) - _get_mplock_predisposed(file, line); -} - -/* - * Release the MP lock - * - * In order to release the MP lock we must first pre-dispose td_mpcount - * for the release and then, if it is 0 and td_xpcount is also zero, - * release the actual lock. - * - * The contested function is called only if we are unable to release the - * Actual lock. This can occur if we raced an interrupt after decrementing - * td_mpcount to 0 and the interrupt acquired and released the lock. - * - * The function also catches the td_mpcount underflow case because the - * lock will be in a released state and thus fail the subsequent release. - * - * WARNING: The mp_lock and td_mpcount are not necessarily synchronized. - * We must synchronize them here. They can be unsynchronized - * for a variety of reasons including predisposition, td_xpcount, - * and so forth. - */ -static __inline -void -rel_mplock(void) -{ - globaldata_t gd = mycpu; - thread_t td = gd->gd_curthread; - int n; - - n = --td->td_mpcount; - if (n < 0 || ((n + td->td_xpcount) == 0 && - atomic_cmpset_int(&mp_lock, gd->gd_cpuid, -1) == 0)) { - _rel_mplock_contested(); - } -} - -/* - * Attempt to acquire the MP lock, returning 0 on failure and 1 on success. - * - * The contested function is called on failure and typically serves simply - * to log the attempt (if debugging enabled). - */ -static __inline -int -try_mplock_debug(const char *file, int line) -{ - globaldata_t gd = mycpu; - thread_t td = gd->gd_curthread; - - ++td->td_mpcount; - if (mp_lock != gd->gd_cpuid && - atomic_cmpset_int(&mp_lock, -1, gd->gd_cpuid) == 0) { - _try_mplock_contested(file, line); - return(0); - } -#ifdef INVARIANTS - mp_lock_holder_file = file; - mp_lock_holder_line = line; -#endif - return(1); -} - -/* - * Low level acquisition of the MP lock ignoring curthred->td_mpcount - * - * This version of try_mplock() is used when the caller has already - * predisposed td->td_mpcount. - * - * Returns non-zero on success, 0 on failure. - * - * WARNING: Must be called from within a critical section if td_mpcount is - * zero, otherwise an itnerrupt race can cause the lock to be lost. - */ -static __inline -int -cpu_try_mplock_debug(const char *file, int line) -{ - globaldata_t gd = mycpu; - - if (mp_lock != gd->gd_cpuid && - atomic_cmpset_int(&mp_lock, -1, gd->gd_cpuid) == 0) { - _cpu_try_mplock_contested(file, line); - return(0); - } -#ifdef INVARIANTS - mp_lock_holder_file = file; - mp_lock_holder_line = line; -#endif - return(1); -} - -static __inline -int -cpu_try_mplock_msg_debug(const char **lmsg, const char *file, int line) -{ - if (cpu_try_mplock_debug(file, line)) { - return(1); - } else { - *lmsg = "mplock"; - return(0); - } -} /* * A cpu wanted the MP lock but could not get it. This function is also @@ -192,32 +59,9 @@ clr_cpu_contention_mask(globaldata_t gd) atomic_clear_cpumask(&cpu_contention_mask, gd->gd_cpumask); } -static __inline -int -owner_mplock(void) -{ - return (mp_lock); -} - -/* - * Low level release of the MP lock ignoring curthread->td_mpcount - * - * WARNING: Caller must be in a critical section, otherwise the - * mp_lock can be lost from an interrupt race and we would - * end up clearing someone else's lock. - */ -static __inline void -cpu_rel_mplock(int cpu) -{ - (void)atomic_cmpset_int(&mp_lock, cpu, -1); -} - -#define MP_LOCK_HELD(gd) \ - (mp_lock == gd->gd_cpuid) +#define MP_LOCK_HELD() LWKT_TOKEN_HELD(&mp_token) +#define ASSERT_MP_LOCK_HELD() ASSERT_LWKT_TOKEN_HELD(&mp_token) -#define ASSERT_MP_LOCK_HELD(td) \ - KASSERT(MP_LOCK_HELD(td->td_gd), \ - ("MP_LOCK_HELD: Not held thread %p", td)) #else @@ -228,7 +72,7 @@ cpu_rel_mplock(int cpu) #define rel_mplock() #define try_mplock() 1 #define owner_mplock() 0 -#define MP_LOCK_HELD(gd) (!0) +#define MP_LOCK_HELD(gd) 1 #define ASSERT_MP_LOCK_HELD(td) #endif diff --git a/sys/sys/thread.h b/sys/sys/thread.h index 84eea7a15a..9f98cc2b03 100644 --- a/sys/sys/thread.h +++ b/sys/sys/thread.h @@ -133,10 +133,13 @@ typedef struct lwkt_token { /* * Assert that a particular token is held */ +#define LWKT_TOKEN_HELD(tok) _lwkt_token_held(tok, curthread) + #define ASSERT_LWKT_TOKEN_HELD(tok) \ - KKASSERT((tok)->t_ref && (tok)->t_ref->tr_owner == curthread) + KKASSERT(LWKT_TOKEN_HELD(tok)) + #define ASSERT_NO_TOKENS_HELD(td) \ - KKASSERT((td)->td_toks_stop == &td->toks_array[0]) + KKASSERT((td)->td_toks_stop == &td->td_toks_array[0]) /* * Assert that a particular token is held and we are in a hard @@ -269,18 +272,14 @@ struct thread { void *td_dsched_priv1; /* priv data for I/O schedulers */ int td_refs; /* hold position in gd_tdallq / hold free */ int td_nest_count; /* prevent splz nesting */ + int td_unused01[2]; /* for future fields */ #ifdef SMP - int td_mpcount; /* MP lock held (count) */ - int td_xpcount; /* MP lock held inherited (count) */ int td_cscount; /* cpu synchronization master */ - int td_unused02[4]; /* for future fields */ #else - int td_mpcount_unused; /* filler so size matches */ - int td_xpcount_unused; int td_cscount_unused; - int td_unused02[4]; #endif - int td_unused03[4]; /* for future fields */ + int td_unused02[4]; /* for future fields */ + int td_unused03[4]; /* for future fields */ struct iosched_data td_iosdata; /* Dynamic I/O scheduling data */ struct timeval td_start; /* start time for a thread/process */ char td_comm[MAXCOMLEN+1]; /* typ 16+1 bytes */ @@ -403,6 +402,7 @@ struct thread { /* * Global tokens */ +extern struct lwkt_token mp_token; extern struct lwkt_token pmap_token; extern struct lwkt_token dev_token; extern struct lwkt_token vm_token; @@ -444,6 +444,7 @@ extern void lwkt_gettoken_hard(lwkt_token_t); extern int lwkt_trytoken(lwkt_token_t); extern void lwkt_reltoken(lwkt_token_t); extern void lwkt_reltoken_hard(lwkt_token_t); +extern int lwkt_cnttoken(lwkt_token_t, thread_t); extern int lwkt_getalltokens(thread_t); extern void lwkt_relalltokens(thread_t); extern void lwkt_drain_token_requests(void); diff --git a/sys/sys/thread2.h b/sys/sys/thread2.h index 4cd7b12fc1..20bdc1d67c 100644 --- a/sys/sys/thread2.h +++ b/sys/sys/thread2.h @@ -33,6 +33,16 @@ #include #endif +/* + * Is a token held by the specified thread? + */ +static __inline int +_lwkt_token_held(lwkt_token_t tok, thread_t td) +{ + return (tok->t_ref >= &td->td_toks_base && + tok->t_ref < td->td_toks_stop); +} + /* * Critical section debugging */ @@ -204,8 +214,7 @@ crit_test(thread_t td) } /* - * Return whether any threads are runnable, whether they meet mp_lock - * requirements or not. + * Return whether any threads are runnable. */ static __inline int lwkt_runnable(void) diff --git a/sys/vfs/procfs/procfs_ctl.c b/sys/vfs/procfs/procfs_ctl.c index 4fdb164cdb..4a343291ea 100644 --- a/sys/vfs/procfs/procfs_ctl.c +++ b/sys/vfs/procfs/procfs_ctl.c @@ -48,9 +48,12 @@ #include #include #include -#include #include +#include +#include +#include + #include #ifndef FIX_SSTEP diff --git a/sys/vfs/tmpfs/tmpfs_vnops.c b/sys/vfs/tmpfs/tmpfs_vnops.c index 2584f99188..1563245b4c 100644 --- a/sys/vfs/tmpfs/tmpfs_vnops.c +++ b/sys/vfs/tmpfs/tmpfs_vnops.c @@ -497,7 +497,6 @@ tmpfs_write (struct vop_write_args *ap) size_t offset; size_t len; struct rlimit limit; - int got_mplock; int trivial = 0; int kflags = 0; @@ -538,16 +537,8 @@ tmpfs_write (struct vop_write_args *ap) */ extended = ((uio->uio_offset + uio->uio_resid) > node->tn_size); -#ifdef SMP - if (curthread->td_mpcount) { - got_mplock = -1; - } else { - got_mplock = 1; - get_mplock(); - } -#else - got_mplock = -1; -#endif + get_mplock(); + while (uio->uio_resid > 0) { /* * Use buffer cache I/O (via tmpfs_strategy) @@ -628,8 +619,7 @@ tmpfs_write (struct vop_write_args *ap) } } - if (got_mplock > 0) - rel_mplock(); + rel_mplock(); if (error) { if (extended) { -- 2.41.0