{
if (lock) {
if (lock == &sim_mplock)
- ASSERT_MP_LOCK_HELD(curthread);
+ ASSERT_MP_LOCK_HELD();
else
KKASSERT(lockstatus(lock, curthread) != 0);
}
kern/kern_udev.c standard
kern/kern_upcall.c standard
kern/kern_sfbuf.c standard
-kern/kern_mplock.c standard
kern/kern_subr.c standard
kern/kern_iosched.c standard
kern/kern_usched.c standard
TAILQ_FOREACH(td, &gd->gd_tdrunq, td_threadq) {
if (db_more(&nl) < 0)
return;
- db_printf(" %p %3d %08x %2d/%02d/%02d %p %8.8s %s\n",
+ db_printf(" %p %3d %08x %2d/%02d %p %8.8s %s\n",
td,
(td->td_proc ? td->td_proc->p_pid : -1),
td->td_flags,
td->td_pri,
td->td_critcount,
-#ifdef SMP
- td->td_mpcount,
-#else
- 0,
-#endif
td->td_sp,
td->td_wmesg ? td->td_wmesg : "-",
td->td_proc ? td->td_proc->p_comm : td->td_comm);
TAILQ_FOREACH(td, &gd->gd_tdallq, td_allq) {
if (db_more(&nl) < 0)
return;
- db_printf(" %3d %p %3d %08x %2d/%02d/%02d %p %8.8s %s\n",
+ db_printf(" %3d %p %3d %08x %2d/%02d %p %8.8s %s\n",
np, td,
(td->td_proc ? td->td_proc->p_pid : -1),
td->td_flags,
td->td_pri,
td->td_critcount,
-#ifdef SMP
- td->td_mpcount,
-#else
- 0,
-#endif
td->td_sp,
td->td_wmesg ? td->td_wmesg : "-",
td->td_proc ? td->td_proc->p_comm : td->td_comm);
{
lwkt_init_thread(&thread0, proc0paddr, LWKT_THREAD_STACK, 0, gd);
lwkt_set_comm(&thread0, "thread0");
-#ifdef SMP
- thread0.td_mpcount = 1; /* will hold mplock initially */
-#endif
RB_INIT(&proc0.p_lwp_tree);
spin_init(&proc0.p_spin);
proc0.p_lasttid = 0; /* +1 = next TID */
* dangling tokens, spinlocks, or mp locks.
*/
#ifdef INVARIANTS
-# ifdef SMP
-/* INVARIANTS & SMP */
-# define SMP_INVARIANTS_DECLARE \
- int mpcount;
-
-# define SMP_INVARIANTS_GET(td) \
- mpcount = (td)->td_mpcount
-
-# define SMP_INVARIANTS_TEST(td, name) \
- KASSERT(mpcount == (td)->td_mpcount, \
- ("mpcount mismatch after interrupt handler %s", \
- name))
-
-# define SMP_INVARIANTS_ADJMP(count) \
- mpcount += (count)
-
-# else
-/* INVARIANTS & !SMP */
-# define SMP_INVARIANTS_DECLARE
-# define SMP_INVARIANTS_GET(td)
-# define SMP_INVARIANTS_TEST(td, name)
-
-# endif /* ndef SMP */
#define TD_INVARIANTS_DECLARE \
- SMP_INVARIANTS_DECLARE \
int spincount; \
lwkt_tokref_t curstop
#define TD_INVARIANTS_GET(td) \
do { \
- SMP_INVARIANTS_GET(td); \
spincount = (td)->td_gd->gd_spinlocks_wr; \
curstop = (td)->td_toks_stop; \
} while(0)
KASSERT(curstop == (td)->td_toks_stop, \
("token count mismatch after interrupt handler %s", \
name)); \
- SMP_INVARIANTS_TEST(td, name); \
} while(0)
#else
-/* !INVARIANTS */
-# ifdef SMP
-/* !INVARIANTS & SMP */
-# define SMP_INVARIANTS_ADJMP(count)
-# endif
+/* !INVARIANTS */
#define TD_INVARIANTS_DECLARE
#define TD_INVARIANTS_GET(td)
break;
}
got_mplock = 1;
- SMP_INVARIANTS_ADJMP(1);
}
#endif
if (rec->serializer) {
#include <sys/tty.h>
#include <sys/conf.h>
#include <sys/jail.h>
+#include <sys/mplock2.h>
#include <sys/globaldata.h>
#ifdef _KERNEL
#include <sys/systm.h>
}
}
#ifdef SMP
- kl->kl_mpcount = lwp->lwp_thread->td_mpcount;
+ kl->kl_mpcount = get_mplock_count(lwp->lwp_thread);
#else
kl->kl_mpcount = 0;
#endif
kp->kp_lwp.kl_tid = -1;
kp->kp_lwp.kl_tdflags = td->td_flags;
#ifdef SMP
- kp->kp_lwp.kl_mpcount = td->td_mpcount;
+ kp->kp_lwp.kl_mpcount = get_mplock_count(td);
#else /* !SMP */
kp->kp_lwp.kl_mpcount = 0;
#endif /* SMP */
+++ /dev/null
-/*
- * Copyright (c) 2009 The DragonFly Project. All rights reserved.
- *
- * This code is derived from software contributed to The DragonFly Project
- * by Matthew Dillon <dillon@backplane.com>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * 3. Neither the name of The DragonFly Project nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific, prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-/*
- * Helper functions for MP lock acquisition and release.
- */
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/kernel.h>
-#include <sys/proc.h>
-#include <sys/rtprio.h>
-#include <sys/queue.h>
-#include <sys/sysctl.h>
-#include <sys/kthread.h>
-#include <machine/cpu.h>
-#include <sys/lock.h>
-#include <sys/caps.h>
-#include <sys/spinlock.h>
-#include <sys/ktr.h>
-
-#include <sys/thread2.h>
-#include <sys/mplock2.h>
-#include <sys/spinlock2.h>
-
-#ifdef SMP
-static int chain_mplock = 0;
-static int mplock_yield = 10;
-static int mplock_backtrace;
-static __int64_t mplock_contention_count = 0;
-
-SYSCTL_INT(_lwkt, OID_AUTO, chain_mplock, CTLFLAG_RW, &chain_mplock, 0,
- "Chain IPI's to other CPU's potentially needing the MP lock when it is yielded");
-SYSCTL_INT(_lwkt, OID_AUTO, mplock_yield_delay, CTLFLAG_RW, &mplock_yield, 0,
- "Duration of delay when MP lock is temporarily yielded");
-SYSCTL_INT(_lwkt, OID_AUTO, mplock_backtrace, CTLFLAG_RW, &mplock_backtrace, 0,
- "Output backplane when mplock contention occurs");
-SYSCTL_QUAD(_lwkt, OID_AUTO, mplock_contention_count, CTLFLAG_RW,
- &mplock_contention_count, 0, "spinning due to MPLOCK contention");
-
-/*
- * Kernel Trace
- */
-#if !defined(KTR_GIANT_CONTENTION)
-#define KTR_GIANT_CONTENTION KTR_ALL
-#endif
-
-KTR_INFO_MASTER(giant);
-KTR_INFO(KTR_GIANT_CONTENTION, giant, beg, 0,
- "thread=%p held %s:%-5d want %s:%-5d",
- sizeof(void *) * 3 + sizeof(int) * 2);
-KTR_INFO(KTR_GIANT_CONTENTION, giant, end, 1,
- "thread=%p held %s:%-5d want %s:%-5d",
- sizeof(void *) * 3 + sizeof(int) * 2);
-
-#define loggiant(name) \
- KTR_LOG(giant_ ## name, curthread, \
- mp_lock_holder_file, mp_lock_holder_line, \
- file, line)
-
-int mp_lock;
-cpumask_t cpu_contention_mask;
-const char *mp_lock_holder_file; /* debugging */
-int mp_lock_holder_line; /* debugging */
-
-/*
- * Sets up the initial MP lock state near the start of the kernel boot
- */
-void
-cpu_get_initial_mplock(void)
-{
- mp_lock = 0; /* cpu 0 */
- curthread->td_mpcount = 1;
-}
-
-/*
- * This code is called from the get_mplock() inline when the mplock
- * is not already held. td_mpcount has already been predisposed
- * (incremented).
- */
-void
-_get_mplock_predisposed(const char *file, int line)
-{
- globaldata_t gd = mycpu;
-
- if (gd->gd_intr_nesting_level) {
- panic("Attempt to acquire mplock not already held "
- "in hard section, ipi or interrupt %s:%d",
- file, line);
- }
- if (atomic_cmpset_int(&mp_lock, -1, gd->gd_cpuid) == 0)
- _get_mplock_contested(file, line);
-#ifdef INVARIANTS
- mp_lock_holder_file = file;
- mp_lock_holder_line = line;
-#endif
-}
-
-/*
- * Called when the MP lock could not be trvially acquired. The caller
- * has already bumped td_mpcount.
- */
-void
-_get_mplock_contested(const char *file, int line)
-{
- globaldata_t gd = mycpu;
- int ov;
- int nv;
- const void **stkframe = (const void **)&file;
-
- if (mplock_backtrace > 0) {
- --mplock_backtrace;
- print_backtrace(-1);
- }
- ++mplock_contention_count;
- for (;;) {
- ov = mp_lock;
- nv = gd->gd_cpuid;
- if (ov == gd->gd_cpuid)
- break;
- if (ov == -1) {
- if (atomic_cmpset_int(&mp_lock, ov, gd->gd_cpuid))
- break;
- } else {
- gd->gd_curthread->td_mplock_stallpc = stkframe[-1];
- loggiant(beg);
- lwkt_switch();
- loggiant(end);
- KKASSERT(gd->gd_cpuid == mp_lock);
- break;
- }
- }
-}
-
-/*
- * Called if td_mpcount went negative or if td_mpcount + td_xpcount is 0
- * and we were unable to release the MP lock. Handles sanity checks
- * and conflicts.
- *
- * It is possible for the inline release to have raced an interrupt which
- * get/rel'd the MP lock, causing the inline's cmpset to fail. If this
- * case occurs mp_lock will either already be in a released state or it
- * will have already been acquired by another cpu.
- */
-void
-_rel_mplock_contested(void)
-{
- globaldata_t gd = mycpu;
- thread_t td = gd->gd_curthread;
- int ov;
-
- KKASSERT(td->td_mpcount >= 0);
- if (td->td_mpcount + td->td_xpcount == 0) {
- for (;;) {
- ov = mp_lock;
- if (ov != gd->gd_cpuid)
- break;
- if (atomic_cmpset_int(&mp_lock, ov, -1))
- break;
- }
- }
-}
-
-/*
- * Called when try_mplock() fails.
- *
- * The inline bumped td_mpcount so we have to undo it.
- *
- * It is possible to race an interrupt which acquired and released the
- * MP lock. When combined with the td_mpcount decrement we do the MP lock
- * can wind up in any state and possibly not even owned by us.
- *
- * It is also possible for this function to be called even if td_mpcount > 1
- * if someone bumped it and raced an interrupt which then called try_mpock().
- */
-void
-_try_mplock_contested(const char *file, int line)
-{
- globaldata_t gd = mycpu;
- thread_t td = gd->gd_curthread;
- int ov;
-
- --td->td_mpcount;
- KKASSERT(td->td_mpcount >= 0);
- ++mplock_contention_count;
-
- if (td->td_mpcount + td->td_xpcount == 0) {
- for (;;) {
- ov = mp_lock;
- if (ov != gd->gd_cpuid)
- break;
- if (atomic_cmpset_int(&mp_lock, ov, -1))
- break;
- }
- }
-}
-
-/*
- * Called when cpu_try_mplock() fails.
- *
- * The inline did not touch td_mpcount so we do not either.
- */
-void
-_cpu_try_mplock_contested(const char *file, int line)
-{
- ++mplock_contention_count;
-}
-
-/*
- * Temporarily yield the MP lock. This is part of lwkt_user_yield()
- * which is kinda hackish. The MP lock cannot be yielded if inherited
- * due to a preemption.
- */
-void
-yield_mplock(thread_t td)
-{
- int savecnt;
-
- if (td->td_xpcount == 0 && mplock_yield > 0) {
- savecnt = td->td_mpcount;
- td->td_mpcount = 1;
- rel_mplock();
- DELAY(mplock_yield);
- get_mplock();
- td->td_mpcount = savecnt;
- }
-}
-
-#if 0
-
-/*
- * The rel_mplock() code will call this function after releasing the
- * last reference on the MP lock if cpu_contention_mask is non-zero.
- *
- * We then chain an IPI to a single other cpu potentially needing the
- * lock. This is a bit heuristical and we can wind up with IPIs flying
- * all over the place.
- */
-static void lwkt_mp_lock_uncontested_remote(void *arg __unused);
-
-void
-lwkt_mp_lock_uncontested(void)
-{
- globaldata_t gd;
- globaldata_t dgd;
- cpumask_t mask;
- cpumask_t tmpmask;
- int cpuid;
-
- if (chain_mplock) {
- gd = mycpu;
- clr_mplock_contention_mask(gd);
- mask = cpu_contention_mask;
- tmpmask = ~(CPUMASK(gd->gd_cpuid) - 1);
-
- if (mask) {
- if (mask & tmpmask)
- cpuid = BSFCPUMASK(mask & tmpmask);
- else
- cpuid = BSFCPUMASK(mask);
- atomic_clear_cpumask(&cpu_contention_mask, CPUMASK(cpuid));
- dgd = globaldata_find(cpuid);
- lwkt_send_ipiq(dgd, lwkt_mp_lock_uncontested_remote, NULL);
- }
- }
-}
-
-/*
- * The idea is for this IPI to interrupt a potentially lower priority
- * thread, such as a user thread, to allow the scheduler to reschedule
- * a higher priority kernel thread that needs the MP lock.
- *
- * For now we set the LWKT reschedule flag which generates an AST in
- * doreti, though theoretically it is also possible to possibly preempt
- * here if the underlying thread was operating in user mode. Nah.
- */
-static void
-lwkt_mp_lock_uncontested_remote(void *arg __unused)
-{
- need_lwkt_resched();
-}
-
-#endif
-
-#endif /* SMP */
kprintf("panic: %s\n", buf);
#ifdef SMP
/* two separate prints in case of an unmapped page and trap */
- kprintf("mp_lock = %08x; ", mp_lock);
kprintf("cpuid = %d\n", mycpu->gd_cpuid);
#endif
* different beast and LWKT priorities should not be confused with
* user process priorities.
*
- * The MP lock may be out of sync with the thread's td_mpcount + td_xpcount.
- * lwkt_switch() cleans it up.
- *
* Note that the td_switch() function cannot do anything that requires
* the MP lock since the MP lock will have already been setup for
* the target thread (not the current thread). It's nice to have a scheduler
thread_t xtd;
thread_t nlast;
int nquserok;
-#ifdef SMP
- int mpheld;
-#endif
int didaccumulate;
/*
#ifdef SMP
- /*
- * td_mpcount + td_xpcount cannot be used to determine if we currently
- * hold the MP lock because get_mplock() will increment it prior to
- * attempting to get the lock, and switch out if it can't. Our
- * ownership of the actual lock will remain stable while we are
- * in a critical section, and once we actually acquire the underlying
- * lock as long as the count is greater than 0.
- */
- mpheld = MP_LOCK_HELD(gd);
#ifdef INVARIANTS
if (td->td_cscount) {
kprintf("Diagnostic: attempt to switch while mastering cpusync: %p\n",
*/
if ((ntd = td->td_preempted) != NULL) {
KKASSERT(ntd->td_flags & TDF_PREEMPT_LOCK);
-#ifdef SMP
- if (ntd->td_mpcount + ntd->td_xpcount && mpheld == 0) {
- panic("MPLOCK NOT HELD ON RETURN: %p %p %d %d",
- td, ntd, td->td_mpcount, ntd->td_mpcount + ntd->td_xpcount);
- }
- td->td_xpcount = 0;
-#endif
ntd->td_flags |= TDF_PREEMPT_DONE;
/*
if (gd->gd_reqflags & RQF_IDLECHECK_MASK)
ntd->td_flags |= TDF_IDLE_NOHLT;
#ifdef SMP
- KKASSERT(ntd->td_xpcount == 0);
- if (ntd->td_mpcount) {
- if (gd->gd_trap_nesting_level == 0 && panicstr == NULL)
- panic("Idle thread %p was holding the BGL!", ntd);
- if (mpheld == 0) {
- set_cpu_contention_mask(gd);
- handle_cpu_contention_mask();
- cpu_try_mplock();
- mpheld = MP_LOCK_HELD(gd);
- cpu_pause();
- continue;
- }
- }
+ if (gd->gd_trap_nesting_level == 0 && panicstr == NULL)
+ ASSERT_NO_TOKENS_HELD(ntd);
clr_cpu_contention_mask(gd);
#endif
cpu_time.cp_msg[0] = 0;
* always succeeds.
*/
if (ntd->td_fairq_accum >= 0 &&
-#ifdef SMP
- (ntd->td_mpcount + ntd->td_xpcount == 0 ||
- mpheld || cpu_try_mplock_msg(&ntd->td_wmesg)) &&
-#endif
- (!TD_TOKS_HELD(ntd) ||
- lwkt_getalltokens(ntd))
+ (TD_TOKS_NOT_HELD(ntd) || lwkt_getalltokens(ntd))
) {
#ifdef SMP
clr_cpu_contention_mask(gd);
#ifdef SMP
if (ntd->td_fairq_accum >= 0)
set_cpu_contention_mask(gd);
- /* Reload mpheld (it become stale after mplock/token ops) */
- mpheld = MP_LOCK_HELD(gd);
#endif
/*
ntd = &gd->gd_idlethread;
ntd->td_flags |= TDF_IDLE_NOHLT;
#ifdef SMP
- KKASSERT(ntd->td_xpcount == 0);
- if (ntd->td_mpcount) {
- mpheld = MP_LOCK_HELD(gd);
- if (gd->gd_trap_nesting_level == 0 && panicstr == NULL)
- panic("Idle thread %p was holding the BGL!", ntd);
- if (mpheld == 0) {
- set_cpu_contention_mask(gd);
- handle_cpu_contention_mask();
- cpu_try_mplock();
- mpheld = MP_LOCK_HELD(gd);
- cpu_pause();
- break; /* try again from the top, almost */
- }
- }
+ if (gd->gd_trap_nesting_level == 0 && panicstr == NULL)
+ ASSERT_NO_TOKENS_HELD(ntd);
+ /* contention case, do not clear contention mask */
#endif
/*
*/
if ((ntd->td_pri >= TDPRI_KERN_LPSCHED || nquserok ||
user_pri_sched) && ntd->td_fairq_accum >= 0 &&
-#ifdef SMP
- (ntd->td_mpcount + ntd->td_xpcount == 0 ||
- mpheld || cpu_try_mplock_msg(&ntd->td_wmesg)) &&
-#endif
- (!TD_TOKS_HELD(ntd) || lwkt_getalltokens(ntd))
+ (TD_TOKS_NOT_HELD(ntd) || lwkt_getalltokens(ntd))
) {
#ifdef SMP
clr_cpu_contention_mask(gd);
#ifdef SMP
if (ntd->td_fairq_accum >= 0)
set_cpu_contention_mask(gd);
- /*
- * Reload mpheld (it become stale after mplock/token ops).
- */
- mpheld = MP_LOCK_HELD(gd);
if (ntd->td_pri >= TDPRI_KERN_LPSCHED && ntd->td_fairq_accum >= 0)
nquserok = 0;
#endif
* While we are looping in the scheduler be sure to service
* any interrupts which were made pending due to our critical
* section, otherwise we could livelock (e.g.) IPIs.
- *
- * NOTE: splz can enter and exit the mplock so mpheld is
- * stale after this call.
*/
splz_check();
-
-#ifdef SMP
- /*
- * Our mplock can be cached and cause other cpus to livelock
- * if we loop due to e.g. not being able to acquire tokens.
- */
- if (MP_LOCK_HELD(gd))
- cpu_rel_mplock(gd->gd_cpuid);
- mpheld = 0;
-#endif
}
/*
- * Do the actual switch. WARNING: mpheld is stale here.
- *
* We must always decrement td_fairq_accum on non-idle threads just
* in case a thread never gets a tick due to being in a continuous
* critical section. The page-zeroing code does that.
* If the new target does not need the MP lock and we are holding it,
* release the MP lock. If the new target requires the MP lock we have
* already acquired it for the target.
- *
- * WARNING: mpheld is stale here.
*/
haveidle:
KASSERT(ntd->td_critcount,
- ("priority problem in lwkt_switch %d %d", td->td_pri, ntd->td_pri));
-#ifdef SMP
- if (ntd->td_mpcount + ntd->td_xpcount == 0 ) {
- if (MP_LOCK_HELD(gd))
- cpu_rel_mplock(gd->gd_cpuid);
- } else {
- ASSERT_MP_LOCK_HELD(ntd);
- }
-#endif
+ ("priority problem in lwkt_switch %d %d",
+ td->td_critcount, ntd->td_critcount));
+
if (td != ntd) {
++switch_count;
KTR_LOG(ctxsw_sw, gd->gd_cpuid, ntd);
* preempted source thread will be resumed the instant the target blocks
* whether or not the source is scheduled (i.e. preemption is supposed to
* be as transparent as possible).
- *
- * The target thread inherits our MP count (added to its own) for the
- * duration of the preemption in order to preserve the atomicy of the
- * MP lock during the preemption. Therefore, any preempting targets must be
- * careful in regards to MP assertions. Note that the MP count may be
- * out of sync with the physical mp_lock, but we do not have to preserve
- * the original ownership of the lock if it was out of synch (that is, we
- * can leave it synchronized on return).
*/
void
lwkt_preempt(thread_t ntd, int critcount)
{
struct globaldata *gd = mycpu;
thread_t td;
-#ifdef SMP
- int mpheld;
- int savecnt;
-#endif
int save_gd_intr_nesting_level;
/*
need_lwkt_resched();
return;
}
-#ifdef SMP
- /*
- * NOTE: An interrupt might have occured just as we were transitioning
- * to or from the MP lock. In this case td_mpcount will be pre-disposed
- * (non-zero) but not actually synchronized with the mp_lock itself.
- * We can use it to imply an MP lock requirement for the preemption but
- * we cannot use it to test whether we hold the MP lock or not.
- */
- savecnt = td->td_mpcount;
- mpheld = MP_LOCK_HELD(gd);
- ntd->td_xpcount = td->td_mpcount + td->td_xpcount;
- if (mpheld == 0 && ntd->td_mpcount + ntd->td_xpcount && !cpu_try_mplock()) {
- ntd->td_xpcount = 0;
- ++preempt_miss;
- need_lwkt_resched();
- return;
- }
-#endif
/*
* Since we are able to preempt the current thread, there is no need to
gd->gd_intr_nesting_level = save_gd_intr_nesting_level;
KKASSERT(ntd->td_preempted && (td->td_flags & TDF_PREEMPT_DONE));
-#ifdef SMP
- KKASSERT(savecnt == td->td_mpcount);
- mpheld = MP_LOCK_HELD(gd);
- if (mpheld && td->td_mpcount == 0)
- cpu_rel_mplock(gd->gd_cpuid);
- else if (mpheld == 0 && td->td_mpcount + td->td_xpcount)
- panic("lwkt_preempt(): MP lock was not held through");
-#endif
ntd->td_preempted = NULL;
td->td_flags &= ~(TDF_PREEMPT_LOCK|TDF_PREEMPT_DONE);
}
if ((gd->gd_reqflags & RQF_IDLECHECK_MASK) && td->td_nest_count < 2)
splz();
-#ifdef SMP
- /*
- * XXX SEVERE TEMPORARY HACK. A cpu-bound operation running in the
- * kernel can prevent other cpus from servicing interrupt threads
- * which still require the MP lock (which is a lot of them). This
- * has a chaining effect since if the interrupt is blocked, so is
- * the event, so normal scheduling will not pick up on the problem.
- */
- if (cpu_contention_mask && td->td_mpcount + td->td_xpcount) {
- yield_mplock(td);
- }
-#endif
-
/*
* Switch (which forces a release) if another kernel thread needs
* the cpu, if userland wants us to resched, or if our kernel
KTR_LOG(tokens_ ## name, ref, ref->tr_tok, curthread)
/*
+ * Cpu contention mask for directed wakeups.
+ */
+cpumask_t cpu_contention_mask;
+
+/*
* Global tokens. These replace the MP lock for major subsystem locking.
* These tokens are initially used to lockup both global and individual
* operations.
* any time, the MP state is copied to the tokref when the token is acquired
* and will not race against sysctl changes.
*/
+struct lwkt_token mp_token = LWKT_TOKEN_MP_INITIALIZER(mp_token);
struct lwkt_token pmap_token = LWKT_TOKEN_UP_INITIALIZER(pmap_token);
struct lwkt_token dev_token = LWKT_TOKEN_UP_INITIALIZER(dev_token);
struct lwkt_token vm_token = LWKT_TOKEN_UP_INITIALIZER(vm_token);
* to acquire needed tokens in addition to a normal lwkt_gettoken()
* stall.
*/
+SYSCTL_LONG(_lwkt, OID_AUTO, mp_collisions, CTLFLAG_RW,
+ &mp_token.t_collisions, 0, "Collision counter of mp_token");
SYSCTL_LONG(_lwkt, OID_AUTO, pmap_collisions, CTLFLAG_RW,
&pmap_token.t_collisions, 0, "Collision counter of pmap_token");
SYSCTL_LONG(_lwkt, OID_AUTO, dev_collisions, CTLFLAG_RW,
SYSCTL_LONG(_lwkt, OID_AUTO, vnode_collisions, CTLFLAG_RW,
&vnode_token.t_collisions, 0, "Collision counter of vnode_token");
+#ifdef SMP
+/*
+ * Acquire the initial mplock
+ *
+ * (low level boot only)
+ */
+void
+cpu_get_initial_mplock(void)
+{
+ KKASSERT(mp_token.t_ref == NULL);
+ if (lwkt_trytoken(&mp_token) == FALSE)
+ panic("cpu_get_initial_mplock");
+}
+#endif
+
/*
* Return a pool token given an address
*/
* token array.
*
* As an optimization we set the MPSAFE flag if the thread is already
- * holding the MP lock. This bypasses unncessary calls to get_mplock() and
+ * holding the mp_token. This bypasses unncessary calls to get_mplock() and
* rel_mplock() on tokens which are not normally MPSAFE when the thread
* is already holding the MP lock.
- *
- * WARNING: The inherited td_xpcount does not count here because a switch
- * could schedule the preempted thread and blow away the inherited
- * mplock.
*/
static __inline
+intptr_t
+_lwkt_tok_flags(lwkt_token_t tok, thread_t td)
+{
+ intptr_t flags;
+
+ /*
+ * tok->t_flags can change out from under us, make sure we have
+ * a local copy.
+ */
+ flags = tok->t_flags;
+ cpu_ccfence();
+#ifdef SMP
+ if ((flags & LWKT_TOKEN_MPSAFE) == 0 &&
+ _lwkt_token_held(&mp_token, td)) {
+ return (flags | LWKT_TOKEN_MPSAFE);
+ } else {
+ return (flags);
+ }
+#else
+ return (flags | LWKT_TOKEN_MPSAFE);
+#endif
+}
+
+static __inline
void
-_lwkt_tokref_init(lwkt_tokref_t ref, lwkt_token_t tok, thread_t td)
+_lwkt_tokref_init(lwkt_tokref_t ref, lwkt_token_t tok, thread_t td,
+ intptr_t flags)
{
ref->tr_tok = tok;
ref->tr_owner = td;
- ref->tr_flags = tok->t_flags;
-#ifdef SMP
- if (td->td_mpcount)
-#endif
- ref->tr_flags |= LWKT_TOKEN_MPSAFE;
+ ref->tr_flags = flags;
}
/*
}
/*
- * Acquire a serializing token. This routine does not block.
+ * Get a serializing token. This routine can block.
*/
-static __inline
-int
-_lwkt_trytokref(lwkt_tokref_t ref, thread_t td)
-{
- if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0) {
- if (try_mplock() == 0) {
- --td->td_toks_stop;
- return (FALSE);
- }
- }
- if (_lwkt_trytokref2(ref, td, 0) == FALSE) {
- /*
- * Cleanup, deactivate the failed token.
- */
- if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0)
- rel_mplock();
- --td->td_toks_stop;
- return (FALSE);
- }
- return (TRUE);
-}
-
-/*
- * Acquire a serializing token. This routine can block.
- */
-static __inline
void
-_lwkt_gettokref(lwkt_tokref_t ref, thread_t td, const void **stkframe)
+lwkt_gettoken(lwkt_token_t tok)
{
- if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0)
+ thread_t td = curthread;
+ lwkt_tokref_t ref;
+ intptr_t flags;
+
+ flags = _lwkt_tok_flags(tok, td);
+ if ((flags & LWKT_TOKEN_MPSAFE) == 0)
get_mplock();
+
+ ref = td->td_toks_stop;
+ KKASSERT(ref < &td->td_toks_end);
+ ++td->td_toks_stop;
+ cpu_ccfence();
+ _lwkt_tokref_init(ref, tok, td, flags);
+
if (_lwkt_trytokref2(ref, td, 1) == FALSE) {
/*
* Give up running if we can't acquire the token right now.
* return tr_tok->t_ref should be assigned to this specific
* ref.
*/
- ref->tr_stallpc = stkframe[-1];
atomic_add_long(&ref->tr_tok->t_collisions, 1);
logtoken(fail, ref);
lwkt_switch();
}
void
-lwkt_gettoken(lwkt_token_t tok)
-{
- thread_t td = curthread;
- lwkt_tokref_t ref;
-
- ref = td->td_toks_stop;
- KKASSERT(ref < &td->td_toks_end);
- ++td->td_toks_stop;
- cpu_ccfence();
- _lwkt_tokref_init(ref, tok, td);
- _lwkt_gettokref(ref, td, (const void **)&tok);
-}
-
-void
lwkt_gettoken_hard(lwkt_token_t tok)
{
thread_t td = curthread;
lwkt_tokref_t ref;
+ intptr_t flags;
+
+ flags = _lwkt_tok_flags(tok, td);
+ if ((flags & LWKT_TOKEN_MPSAFE) == 0)
+ get_mplock();
ref = td->td_toks_stop;
KKASSERT(ref < &td->td_toks_end);
++td->td_toks_stop;
cpu_ccfence();
- _lwkt_tokref_init(ref, tok, td);
- _lwkt_gettokref(ref, td, (const void **)&tok);
+ _lwkt_tokref_init(ref, tok, td, flags);
+
+ if (_lwkt_trytokref2(ref, td, 1) == FALSE) {
+ /*
+ * Give up running if we can't acquire the token right now.
+ *
+ * Since the tokref is already active the scheduler now
+ * takes care of acquisition, so we need only call
+ * lwkt_switch().
+ *
+ * Since we failed this was not a recursive token so upon
+ * return tr_tok->t_ref should be assigned to this specific
+ * ref.
+ */
+ atomic_add_long(&ref->tr_tok->t_collisions, 1);
+ logtoken(fail, ref);
+ lwkt_switch();
+ logtoken(succ, ref);
+ KKASSERT(ref->tr_tok->t_ref == ref);
+ }
crit_enter_hard_gd(td->td_gd);
}
thread_t td = curthread;
lwkt_token_t tok;
lwkt_tokref_t ref;
+ intptr_t flags;
+
+ tok = _lwkt_token_pool_lookup(ptr);
+ flags = _lwkt_tok_flags(tok, td);
+ if ((flags & LWKT_TOKEN_MPSAFE) == 0)
+ get_mplock();
ref = td->td_toks_stop;
KKASSERT(ref < &td->td_toks_end);
++td->td_toks_stop;
cpu_ccfence();
- tok = _lwkt_token_pool_lookup(ptr);
- _lwkt_tokref_init(ref, tok, td);
- _lwkt_gettokref(ref, td, (const void **)&ptr);
+ _lwkt_tokref_init(ref, tok, td, flags);
+
+ if (_lwkt_trytokref2(ref, td, 1) == FALSE) {
+ /*
+ * Give up running if we can't acquire the token right now.
+ *
+ * Since the tokref is already active the scheduler now
+ * takes care of acquisition, so we need only call
+ * lwkt_switch().
+ *
+ * Since we failed this was not a recursive token so upon
+ * return tr_tok->t_ref should be assigned to this specific
+ * ref.
+ */
+ atomic_add_long(&ref->tr_tok->t_collisions, 1);
+ logtoken(fail, ref);
+ lwkt_switch();
+ logtoken(succ, ref);
+ KKASSERT(ref->tr_tok->t_ref == ref);
+ }
return(tok);
}
{
thread_t td = curthread;
lwkt_tokref_t ref;
+ intptr_t flags;
+
+ flags = _lwkt_tok_flags(tok, td);
+ if ((flags & LWKT_TOKEN_MPSAFE) == 0) {
+ if (try_mplock() == 0)
+ return (FALSE);
+ }
ref = td->td_toks_stop;
KKASSERT(ref < &td->td_toks_end);
++td->td_toks_stop;
cpu_ccfence();
- _lwkt_tokref_init(ref, tok, td);
- return(_lwkt_trytokref(ref, td));
+ _lwkt_tokref_init(ref, tok, td, flags);
+
+ if (_lwkt_trytokref2(ref, td, 0) == FALSE) {
+ /*
+ * Cleanup, deactivate the failed token.
+ */
+ if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0) {
+ cpu_ccfence();
+ --td->td_toks_stop;
+ cpu_ccfence();
+ rel_mplock();
+ } else {
+ cpu_ccfence();
+ --td->td_toks_stop;
+ }
+ return (FALSE);
+ }
+ return (TRUE);
}
/*
/*
* Only clear the token if it matches ref. If ref was a recursively
- * acquired token it may not match.
+ * acquired token it may not match. Then adjust td_toks_stop.
*
- * If the token was not MPSAFE release the MP lock.
+ * Some comparisons must be run prior to adjusting td_toks_stop
+ * to avoid racing against a fast interrupt/ ipi which tries to
+ * acquire a token.
*
- * NOTE: We have to do this before adjust td_toks_stop, otherwise
- * a fast interrupt can come along and reuse our ref while
- * tok is still attached to it.
+ * We must also be absolutely sure that the compiler does not
+ * reorder the clearing of t_ref and the adjustment of td_toks_stop,
+ * or reorder the adjustment of td_toks_stop against the conditional.
+ *
+ * NOTE: The mplock is a token also so sequencing is a bit complex.
*/
if (tok->t_ref == ref)
tok->t_ref = NULL;
- cpu_ccfence();
- if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0)
+ cpu_sfence();
+ if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0) {
+ cpu_ccfence();
+ td->td_toks_stop = ref;
+ cpu_ccfence();
rel_mplock();
-
- /*
- * Finally adjust td_toks_stop, be very sure that the compiler
- * does not reorder the clearing of tok->t_ref with the
- * decrementing of td->td_toks_stop.
- */
- cpu_ccfence();
- td->td_toks_stop = ref;
+ } else {
+ cpu_ccfence();
+ td->td_toks_stop = ref;
+ cpu_ccfence();
+ }
KKASSERT(tok->t_ref != ref);
}
lwkt_reltoken(tok);
}
+/*
+ * Return a count of the number of token refs the thread has to the
+ * specified token, whether it currently owns the token or not.
+ */
+int
+lwkt_cnttoken(lwkt_token_t tok, thread_t td)
+{
+ lwkt_tokref_t scan;
+ int count = 0;
+
+ for (scan = &td->td_toks_base; scan < td->td_toks_stop; ++scan) {
+ if (scan->tr_tok == tok)
+ ++count;
+ }
+ return(count);
+}
+
/*
* Pool tokens are used to provide a type-stable serializing token
* the SMP/AP boot will blow up on us.
*/
lwkt_reltoken(&tty_token);
-#ifdef SMP
- KKASSERT(curthread->td_mpcount == 1);
-#endif
}
* Additionally, note that we may already be on a run queue if releasing
* via the lwkt_switch() in bsd4_setrunqueue().
*
- * WARNING! The MP lock may be in an unsynchronized state due to the
- * way get_mplock() works and the fact that this function may be called
- * from a passive release during a lwkt_switch(). try_mplock() will deal
- * with this for us but you should be aware that td_mpcount may not be
- * useable.
- *
* MPSAFE
*/
static void
* This routine is also responsible for selecting a new thread to
* make the current thread.
*
- * WARNING! The MP lock may be in an unsynchronized state due to the
- * way get_mplock() works and the fact that this function may be called
- * from a passive release during a lwkt_switch(). try_mplock() will deal
- * with this for us but you should be aware that td_mpcount may not be
- * useable.
- *
* MPSAFE
*/
static void
* - Push the trap frame required by doreti
* - Mask the interrupt and reenable its source
* - If we cannot take the interrupt set its fpending bit and
- * doreti. Note that we cannot mess with mp_lock at all
- * if we entered from a critical section!
+ * doreti.
* - If we can take the interrupt clear its fpending bit,
* call the handler, then unmask and doreti.
*
sti
call splz
-#if defined(INVARIANTS) && defined(SMP)
- movl PCPU(curthread),%eax
- cmpl $0,TD_MPCOUNT(%eax)
- je 1f
- pushl %esi
- pushl TD_MPCOUNT(%eax)
- pushl $pmsg4
- call panic
-pmsg4: .asciz "fork_trampoline mpcount %d after calling %p"
- .p2align 2
-1:
-#endif
/*
* Return via doreti to handle ASTs.
*/
ASSYM(TD_MACH, offsetof(struct thread, td_mach));
ASSYM(TD_WCHAN, offsetof(struct thread, td_wchan));
ASSYM(TD_NEST_COUNT, offsetof(struct thread, td_nest_count));
-#ifdef SMP
-ASSYM(TD_MPCOUNT, offsetof(struct thread, td_mpcount));
-#endif
ASSYM(TD_CRITCOUNT, offsetof(struct thread, td_critcount));
ASSYM(TD_FLAGS, offsetof(struct thread, td_flags));
ASSYM(TDF_RUNNING, TDF_RUNNING);
movl $1, %ecx /* one private pt coming right up */
fillkpt(R(SMPptpa), $PG_RW)
-#ifdef SMP
-/* Initialize mp lock to allow early traps */
- movl $0, R(mp_lock)
-#endif /* SMP */
-
/* install a pde for temporary double map of bottom of VA */
movl R(KPTphys), %eax
xorl %ebx, %ebx
static void
init_locks(void)
{
+#ifdef SMP
/*
- * mp_lock = 0; BSP already owns the MP lock
- */
- /*
- * Get the initial mp_lock with a count of 1 for the BSP.
+ * Get the initial mplock with a count of 1 for the BSP.
* This uses a LOGICAL cpu ID, ie BSP == 0.
*/
-#ifdef SMP
cpu_get_initial_mplock();
#endif
/* DEPRECATED */
*
* Note: We are in a critical section.
*
- * Note: We have to synchronize td_mpcount to our desired MP state
- * before calling cpu_try_mplock().
- *
* Note: we are the idle thread, we can only spin.
*
* Note: The load fence is memory volatile and prevents the compiler
* caching it.
*/
while (mp_finish == 0)
- cpu_lfence();
- ++curthread->td_mpcount;
- while (cpu_try_mplock() == 0)
- ;
+ cpu_lfence();
+ while (try_mplock() == 0)
+ ;
if (cpu_feature & CPUID_TSC) {
- /*
- * The BSP is constantly updating tsc0_offset, figure out the
- * relative difference to synchronize ktrdump.
- */
- tsc_offsets[mycpu->gd_cpuid] = rdtsc() - tsc0_offset;
+ /*
+ * The BSP is constantly updating tsc0_offset, figure out
+ * the relative difference to synchronize ktrdump.
+ */
+ tsc_offsets[mycpu->gd_cpuid] = rdtsc() - tsc0_offset;
}
/* BSP may have changed PTD while we're waiting for the lock */
* The idle thread is never placed on the runq, make sure
* nothing we've done put it there.
*/
- KKASSERT(curthread->td_mpcount == 1);
+ KKASSERT(get_mplock_count(curthread) == 1);
smp_active_mask |= CPUMASK(mycpu->gd_cpuid);
/*
if (frame->tf_eflags & PSL_VM &&
(type == T_PROTFLT || type == T_STKFLT)) {
#ifdef SMP
- KKASSERT(td->td_mpcount > 0);
+ KKASSERT(get_mplock_count(curthread) > 0);
#endif
i = vm86_emulate((struct vm86frame *)frame);
#ifdef SMP
- KKASSERT(td->td_mpcount > 0);
+ KKASSERT(get_mplock_count(curthread) > 0);
#endif
if (i != 0) {
/*
#endif
out:
-#ifdef SMP
- if (ISPL(frame->tf_cs) == SEL_UPL) {
- KASSERT(td->td_mpcount == have_mplock,
- ("badmpcount trap/end from %p", (void *)frame->tf_eip));
- }
-#endif
userret(lp, frame, sticks);
userexit(lp);
out2: ;
ISPL(frame->tf_cs) == SEL_UPL ? "user" : "kernel");
#ifdef SMP
/* three separate prints in case of a trap on an unmapped page */
- kprintf("mp_lock = %08x; ", mp_lock);
kprintf("cpuid = %d; ", mycpu->gd_cpuid);
kprintf("lapic.id = %08x\n", lapic.id);
#endif
kprintf("ebp = 0x%x\n", gd->gd_common_tss.tss_ebp);
#ifdef SMP
/* three separate prints in case of a trap on an unmapped page */
- kprintf("mp_lock = %08x; ", mp_lock);
kprintf("cpuid = %d; ", gd->mi.gd_cpuid);
kprintf("lapic.id = %08x\n", lapic.id);
#endif
KTR_LOG(kernentry_syscall, p->p_pid, lp->lwp_tid,
frame->tf_eax);
-#ifdef SMP
- KASSERT(td->td_mpcount == 0,
- ("badmpcount syscall2 from %p", (void *)frame->tf_eip));
-#endif
userenter(td, p); /* lazy raise our priority */
/*
/*
* Release the MP lock if we had to get it
*/
- KASSERT(td->td_mpcount == have_mplock,
- ("badmpcount syscall2/end from %p callp %p",
- (void *)frame->tf_eip, callp));
if (have_mplock)
rel_mplock();
#endif
return (EINVAL);
crit_enter();
- ASSERT_MP_LOCK_HELD(curthread);
+ ASSERT_MP_LOCK_HELD();
vm86_setup_timer_fault();
vmf->vmf_trapno = intnum;
int i, entry, retval;
crit_enter();
- ASSERT_MP_LOCK_HELD(curthread);
+ ASSERT_MP_LOCK_HELD();
for (i = 0; i < vmc->npages; i++) {
page = vtophys(vmc->pmap[i].kva & PG_FRAME);
static void
cpu_reset_proxy(void)
{
- u_int saved_mp_lock;
-
cpu_reset_proxy_active = 1;
while (cpu_reset_proxy_active == 1)
; /* Wait for other cpu to disable interupts */
- saved_mp_lock = mp_lock;
- mp_lock = 0; /* BSP */
kprintf("cpu_reset_proxy: Grabbed mp lock for BSP\n");
cpu_reset_proxy_active = 3;
while (cpu_reset_proxy_active == 3)
* checks the cpl for unmasked pending interrupts (fast, normal, or
* soft) and schedules them if appropriate, then irets.
*
- * If we are in a critical section we cannot run any pending ints
- * nor can be play with mp_lock.
+ * If we are in a critical section we cannot run any pending ints.
*
* NOTE: Since SPLs no longer exist, all callers of this function
* push $0 for the CPL. HOWEVER, we *STILL* use the cpl mask within
* - Push the trap frame required by doreti
* - Mask the interrupt and reenable its source
* - If we cannot take the interrupt set its fpending bit and
- * doreti. Note that we cannot mess with mp_lock at all
- * if we entered from a critical section!
+ * doreti.
* - If we can take the interrupt clear its fpending bit,
* call the handler, then unmask and doreti.
*
sti
call splz
-#if defined(INVARIANTS) && defined(SMP)
- movq PCPU(curthread),%rax
- cmpl $0,TD_MPCOUNT(%rax)
- je 1f
- movq $pmsg4, %rdi
- movl TD_MPCOUNT(%rax), %esi
- movq %rbx, %rdx
- xorl %eax, %eax
- call panic
-pmsg4: .asciz "fork_trampoline mpcount %d after calling %p"
- /* JG what's the purpose of this alignment and is it enough on x86_64? */
- .p2align 2
-1:
-#endif
/*
* Return via doreti to handle ASTs.
*
ASSYM(TD_MACH, offsetof(struct thread, td_mach));
ASSYM(TD_WCHAN, offsetof(struct thread, td_wchan));
ASSYM(TD_NEST_COUNT, offsetof(struct thread, td_nest_count));
-#ifdef SMP
-ASSYM(TD_MPCOUNT, offsetof(struct thread, td_mpcount));
-#endif
ASSYM(TD_FLAGS, offsetof(struct thread, td_flags));
ASSYM(TD_SAVEFPU, offsetof(struct thread, td_savefpu));
ASSYM(TDF_RUNNING, TDF_RUNNING);
* checks the cpl for unmasked pending interrupts (fast, normal, or
* soft) and schedules them if appropriate, then irets.
*
- * If we are in a critical section we cannot run any pending ints
- * nor can be play with mp_lock.
+ * If we are in a critical section we cannot run any pending ints.
*
* The stack contains a trapframe at the start of doreti.
*/
static void
init_locks(void)
{
+#ifdef SMP
/*
- * mp_lock = 0; BSP already owns the MP lock
- */
- /*
- * Get the initial mp_lock with a count of 1 for the BSP.
+ * Get the initial mplock with a count of 1 for the BSP.
* This uses a LOGICAL cpu ID, ie BSP == 0.
*/
-#ifdef SMP
cpu_get_initial_mplock();
#endif
/* DEPRECATED */
*
* Note: We are in a critical section.
*
- * Note: We have to synchronize td_mpcount to our desired MP state
- * before calling cpu_try_mplock().
- *
* Note: we are the idle thread, we can only spin.
*
* Note: The load fence is memory volatile and prevents the compiler
* caching it.
*/
while (mp_finish == 0)
- cpu_lfence();
- ++curthread->td_mpcount;
- while (cpu_try_mplock() == 0)
- ;
+ cpu_lfence();
+ while (try_mplock() == 0)
+ ;
if (cpu_feature & CPUID_TSC) {
- /*
- * The BSP is constantly updating tsc0_offset, figure out the
- * relative difference to synchronize ktrdump.
- */
- tsc_offsets[mycpu->gd_cpuid] = rdtsc() - tsc0_offset;
+ /*
+ * The BSP is constantly updating tsc0_offset, figure out
+ * the relative difference to synchronize ktrdump.
+ */
+ tsc_offsets[mycpu->gd_cpuid] = rdtsc() - tsc0_offset;
}
/* BSP may have changed PTD while we're waiting for the lock */
* The idle thread is never placed on the runq, make sure
* nothing we've done put it there.
*/
- KKASSERT(curthread->td_mpcount == 1);
+ KKASSERT(get_mplock_count(curthread) == 1);
smp_active_mask |= CPUMASK(mycpu->gd_cpuid);
/*
#endif
out:
-#ifdef SMP
- if (ISPL(frame->tf_cs) == SEL_UPL) {
- KASSERT(td->td_mpcount == have_mplock,
- ("badmpcount trap/end from %p", (void *)frame->tf_rip));
- }
-#endif
userret(lp, frame, sticks);
userexit(lp);
out2: ;
ISPL(frame->tf_cs) == SEL_UPL ? "user" : "kernel");
#ifdef SMP
/* three separate prints in case of a trap on an unmapped page */
- kprintf("mp_lock = %08x; ", mp_lock);
kprintf("cpuid = %d; ", mycpu->gd_cpuid);
kprintf("lapic->id = %08x\n", lapic->id);
#endif
kprintf("rbp = 0x%lx\n", frame->tf_rbp);
#ifdef SMP
/* three separate prints in case of a trap on an unmapped page */
- kprintf("mp_lock = %08x; ", mp_lock);
kprintf("cpuid = %d; ", mycpu->gd_cpuid);
kprintf("lapic->id = %08x\n", lapic->id);
#endif
KTR_LOG(kernentry_syscall, p->p_pid, lp->lwp_tid,
frame->tf_rax);
-#ifdef SMP
- KASSERT(td->td_mpcount == 0,
- ("badmpcount syscall2 from %p", (void *)frame->tf_rip));
-#endif
userenter(td, p); /* lazy raise our priority */
reg = 0;
/*
* Release the MP lock if we had to get it
*/
- KASSERT(td->td_mpcount == have_mplock,
- ("badmpcount syscall2/end from %p", (void *)frame->tf_rip));
if (have_mplock)
rel_mplock();
#endif
crit_exit();
KKASSERT(td->td_critcount == 0);
-#ifdef SMP
- KKASSERT(td->td_mpcount == 0);
-#endif
cpu_enable_intr();
for (;;) {
/*
*/
lwkt_switch();
-#ifdef SMP
- KKASSERT(td->td_mpcount == 0);
-#endif
/*
* The idle loop halts only if no threads are scheduleable
* and no signals have occured.
(td->td_flags & TDF_IDLE_NOHLT) == 0) {
splz();
#ifdef SMP
- KKASSERT(td->td_mpcount == 0);
- KKASSERT(MP_LOCK_HELD(td->td_gd) == 0);
+ KKASSERT(MP_LOCK_HELD() == 0);
#endif
if (!lwkt_runnable()) {
#ifdef DEBUGIDLE
call splz
-#if defined(INVARIANTS) && defined(SMP)
- movl PCPU(curthread),%eax
- cmpl $0,TD_MPCOUNT(%eax)
- je 1f
- pushl %esi
- pushl TD_MPCOUNT(%eax)
- pushl $pmsg4
- call panic
-pmsg4: .asciz "fork_trampoline mpcount %d after calling %p"
- .p2align 2
-1:
-#endif
/*
* Return via doreti to handle ASTs.
*/
ASSYM(TD_MACH, offsetof(struct thread, td_mach));
ASSYM(TD_WCHAN, offsetof(struct thread, td_wchan));
ASSYM(TD_NEST_COUNT, offsetof(struct thread, td_nest_count));
-#ifdef SMP
-ASSYM(TD_MPCOUNT, offsetof(struct thread, td_mpcount));
-#endif
ASSYM(TD_FLAGS, offsetof(struct thread, td_flags));
ASSYM(TDF_RUNNING, TDF_RUNNING);
*
* Note: We are in a critical section.
*
- * Note: We have to synchronize td_mpcount to our desired MP state
- * before calling cpu_try_mplock().
- *
* Note: we are the idle thread, we can only spin.
*
* Note: The load fence is memory volatile and prevents the compiler
cpu_lfence();
DELAY(500000);
}
- ++curthread->td_mpcount;
- while (cpu_try_mplock() == 0)
+ while (try_mplock() == 0)
DELAY(100000);
/* BSP may have changed PTD while we're waiting for the lock */
* The idle thread is never placed on the runq, make sure
* nothing we've done put it there.
*/
- KKASSERT(curthread->td_mpcount == 1);
+ KKASSERT(get_mplock_count(curthread) == 1);
smp_active_mask |= CPUMASK(mycpu->gd_cpuid);
mdcpu->gd_fpending = 0;
#endif
out:
-#ifdef SMP
- KASSERT(td->td_mpcount == have_mplock,
- ("badmpcount trap/end from %p", (void *)frame->tf_eip));
-#endif
userret(lp, frame, sticks);
userexit(lp);
out2: ;
}
#ifdef SMP
/* two separate prints in case of a trap on an unmapped page */
- kprintf("mp_lock = %08x; ", mp_lock);
kprintf("cpuid = %d\n", mycpu->gd_cpuid);
#endif
if (type == T_PAGEFLT) {
kprintf("ebp = 0x%x\n", gd->gd_common_tss.tss_ebp);
#ifdef SMP
/* two separate prints in case of a trap on an unmapped page */
- kprintf("mp_lock = %08x; ", mp_lock);
kprintf("cpuid = %d\n", mycpu->gd_cpuid);
#endif
panic("double fault");
KTR_LOG(kernentry_syscall, lp->lwp_proc->p_pid, lp->lwp_tid,
frame->tf_eax);
-#ifdef SMP
- KASSERT(td->td_mpcount == 0,
- ("badmpcount syscall2 from %p", (void *)frame->tf_eip));
-#endif
userenter(td, p); /* lazy raise our priority */
/*
/*
* Release the MP lock if we had to get it
*/
- KASSERT(td->td_mpcount == have_mplock,
- ("badmpcount syscall2/end from %p", (void *)frame->tf_eip));
if (have_mplock)
rel_mplock();
#endif
call splz
-#if defined(INVARIANTS) && defined(SMP)
- movq PCPU(curthread),%rax
- cmpl $0,TD_MPCOUNT(%rax)
- je 1f
- movq $pmsg4, %rdi
- movl TD_MPCOUNT(%rax), %esi
- movq %rbx, %rdx
- xorl %eax, %eax
- call panic
-pmsg4: .asciz "fork_trampoline mpcount %d after calling %p"
- /* JG what's the purpose of this alignment and is it enough on x86_64? */
- .p2align 2
-1:
-#endif
/*
* Return via doreti to handle ASTs.
*
ASSYM(TD_SP, offsetof(struct thread, td_sp));
ASSYM(TD_PRI, offsetof(struct thread, td_pri));
ASSYM(TD_CRITCOUNT, offsetof(struct thread, td_critcount));
-#ifdef SMP
-ASSYM(TD_MPCOUNT, offsetof(struct thread, td_mpcount));
-#endif
ASSYM(TD_FLAGS, offsetof(struct thread, td_flags));
ASSYM(TD_SAVEFPU, offsetof(struct thread, td_savefpu));
ASSYM(TDF_RUNNING, TDF_RUNNING);
*
* Note: We are in a critical section.
*
- * Note: We have to synchronize td_mpcount to our desired MP state
- * before calling cpu_try_mplock().
- *
* Note: we are the idle thread, we can only spin.
*
* Note: The load fence is memory volatile and prevents the compiler
cpu_lfence();
DELAY(500000);
}
- ++curthread->td_mpcount;
- while (cpu_try_mplock() == 0)
+ while (try_mplock() == 0)
DELAY(100000);
/* BSP may have changed PTD while we're waiting for the lock */
* The idle thread is never placed on the runq, make sure
* nothing we've done put it there.
*/
- KKASSERT(curthread->td_mpcount == 1);
+ KKASSERT(get_mplock_count(curthread) == 1);
smp_active_mask |= CPUMASK(mycpu->gd_cpuid);
mdcpu->gd_fpending = 0;
#endif
out:
-#ifdef SMP
- KASSERT(td->td_mpcount == have_mplock,
- ("badmpcount trap/end from %p", (void *)frame->tf_rip));
-#endif
userret(lp, frame, sticks);
userexit(lp);
out2: ;
}
#ifdef SMP
/* two separate prints in case of a trap on an unmapped page */
- kprintf("mp_lock = %08x; ", mp_lock);
kprintf("cpuid = %d\n", mycpu->gd_cpuid);
#endif
if (type == T_PAGEFLT) {
#endif
#ifdef SMP
/* two separate prints in case of a trap on an unmapped page */
- kprintf("mp_lock = %08x; ", mp_lock);
kprintf("cpuid = %d\n", mycpu->gd_cpuid);
#endif
panic("double fault");
KTR_LOG(kernentry_syscall, lp->lwp_proc->p_pid, lp->lwp_tid,
frame->tf_eax);
-#ifdef SMP
- KASSERT(td->td_mpcount == 0,
- ("badmpcount syscall2 from %p", (void *)frame->tf_rip));
-#endif
userenter(td, p); /* lazy raise our priority */
reg = 0;
/*
* Release the MP lock if we had to get it
*/
- KASSERT(td->td_mpcount == have_mplock,
- ("badmpcount syscall2/end from %p", (void *)frame->tf_rip));
if (have_mplock)
rel_mplock();
#endif
#ifdef SMP
-#define get_mplock() get_mplock_debug(__FILE__, __LINE__)
-#define try_mplock() try_mplock_debug(__FILE__, __LINE__)
-#define cpu_try_mplock() cpu_try_mplock_debug(__FILE__, __LINE__)
-#define cpu_try_mplock_msg(lmsg) cpu_try_mplock_msg_debug(lmsg, __FILE__, __LINE__)
+/*
+ * NOTE: try_mplock()/lwkt_trytoken() return non-zero on success.
+ */
+#define get_mplock() lwkt_gettoken(&mp_token)
+#define try_mplock() lwkt_trytoken(&mp_token)
+#define rel_mplock() lwkt_reltoken(&mp_token)
+#define get_mplock_count(td) lwkt_cnttoken(&mp_token, td)
-void _get_mplock_predisposed(const char *file, int line);
-void _get_mplock_contested(const char *file, int line);
-void _try_mplock_contested(const char *file, int line);
-void _cpu_try_mplock_contested(const char *file, int line);
-void _rel_mplock_contested(void);
void cpu_get_initial_mplock(void);
void handle_cpu_contention_mask(void);
-void yield_mplock(struct thread *td);
-extern int mp_lock;
extern cpumask_t cpu_contention_mask;
-extern const char *mp_lock_holder_file;
-extern int mp_lock_holder_line;
-
-/*
- * Acquire the MP lock, block until we get it.
- *
- * In order to acquire the MP lock we must first pre-dispose td_mpcount
- * for the acquisition and then get the actual lock.
- *
- * The mplock must check a number of conditions and it is better to
- * leave it to a procedure if we cannot get it trivially.
- *
- * WARNING: The mp_lock and td_mpcount are not necessarily synchronized.
- * We must synchronize them here. They can be unsynchronized
- * for a variety of reasons including predisposition, td_xpcount,
- * and so forth.
- */
-static __inline
-void
-get_mplock_debug(const char *file, int line)
-{
- globaldata_t gd = mycpu;
- thread_t td = gd->gd_curthread;
-
- ++td->td_mpcount;
- if (mp_lock != gd->gd_cpuid)
- _get_mplock_predisposed(file, line);
-}
-
-/*
- * Release the MP lock
- *
- * In order to release the MP lock we must first pre-dispose td_mpcount
- * for the release and then, if it is 0 and td_xpcount is also zero,
- * release the actual lock.
- *
- * The contested function is called only if we are unable to release the
- * Actual lock. This can occur if we raced an interrupt after decrementing
- * td_mpcount to 0 and the interrupt acquired and released the lock.
- *
- * The function also catches the td_mpcount underflow case because the
- * lock will be in a released state and thus fail the subsequent release.
- *
- * WARNING: The mp_lock and td_mpcount are not necessarily synchronized.
- * We must synchronize them here. They can be unsynchronized
- * for a variety of reasons including predisposition, td_xpcount,
- * and so forth.
- */
-static __inline
-void
-rel_mplock(void)
-{
- globaldata_t gd = mycpu;
- thread_t td = gd->gd_curthread;
- int n;
-
- n = --td->td_mpcount;
- if (n < 0 || ((n + td->td_xpcount) == 0 &&
- atomic_cmpset_int(&mp_lock, gd->gd_cpuid, -1) == 0)) {
- _rel_mplock_contested();
- }
-}
-
-/*
- * Attempt to acquire the MP lock, returning 0 on failure and 1 on success.
- *
- * The contested function is called on failure and typically serves simply
- * to log the attempt (if debugging enabled).
- */
-static __inline
-int
-try_mplock_debug(const char *file, int line)
-{
- globaldata_t gd = mycpu;
- thread_t td = gd->gd_curthread;
-
- ++td->td_mpcount;
- if (mp_lock != gd->gd_cpuid &&
- atomic_cmpset_int(&mp_lock, -1, gd->gd_cpuid) == 0) {
- _try_mplock_contested(file, line);
- return(0);
- }
-#ifdef INVARIANTS
- mp_lock_holder_file = file;
- mp_lock_holder_line = line;
-#endif
- return(1);
-}
-
-/*
- * Low level acquisition of the MP lock ignoring curthred->td_mpcount
- *
- * This version of try_mplock() is used when the caller has already
- * predisposed td->td_mpcount.
- *
- * Returns non-zero on success, 0 on failure.
- *
- * WARNING: Must be called from within a critical section if td_mpcount is
- * zero, otherwise an itnerrupt race can cause the lock to be lost.
- */
-static __inline
-int
-cpu_try_mplock_debug(const char *file, int line)
-{
- globaldata_t gd = mycpu;
-
- if (mp_lock != gd->gd_cpuid &&
- atomic_cmpset_int(&mp_lock, -1, gd->gd_cpuid) == 0) {
- _cpu_try_mplock_contested(file, line);
- return(0);
- }
-#ifdef INVARIANTS
- mp_lock_holder_file = file;
- mp_lock_holder_line = line;
-#endif
- return(1);
-}
-
-static __inline
-int
-cpu_try_mplock_msg_debug(const char **lmsg, const char *file, int line)
-{
- if (cpu_try_mplock_debug(file, line)) {
- return(1);
- } else {
- *lmsg = "mplock";
- return(0);
- }
-}
/*
* A cpu wanted the MP lock but could not get it. This function is also
atomic_clear_cpumask(&cpu_contention_mask, gd->gd_cpumask);
}
-static __inline
-int
-owner_mplock(void)
-{
- return (mp_lock);
-}
-
-/*
- * Low level release of the MP lock ignoring curthread->td_mpcount
- *
- * WARNING: Caller must be in a critical section, otherwise the
- * mp_lock can be lost from an interrupt race and we would
- * end up clearing someone else's lock.
- */
-static __inline void
-cpu_rel_mplock(int cpu)
-{
- (void)atomic_cmpset_int(&mp_lock, cpu, -1);
-}
-
-#define MP_LOCK_HELD(gd) \
- (mp_lock == gd->gd_cpuid)
+#define MP_LOCK_HELD() LWKT_TOKEN_HELD(&mp_token)
+#define ASSERT_MP_LOCK_HELD() ASSERT_LWKT_TOKEN_HELD(&mp_token)
-#define ASSERT_MP_LOCK_HELD(td) \
- KASSERT(MP_LOCK_HELD(td->td_gd), \
- ("MP_LOCK_HELD: Not held thread %p", td))
#else
#define rel_mplock()
#define try_mplock() 1
#define owner_mplock() 0
-#define MP_LOCK_HELD(gd) (!0)
+#define MP_LOCK_HELD(gd) 1
#define ASSERT_MP_LOCK_HELD(td)
#endif
/*
* Assert that a particular token is held
*/
+#define LWKT_TOKEN_HELD(tok) _lwkt_token_held(tok, curthread)
+
#define ASSERT_LWKT_TOKEN_HELD(tok) \
- KKASSERT((tok)->t_ref && (tok)->t_ref->tr_owner == curthread)
+ KKASSERT(LWKT_TOKEN_HELD(tok))
+
#define ASSERT_NO_TOKENS_HELD(td) \
- KKASSERT((td)->td_toks_stop == &td->toks_array[0])
+ KKASSERT((td)->td_toks_stop == &td->td_toks_array[0])
/*
* Assert that a particular token is held and we are in a hard
void *td_dsched_priv1; /* priv data for I/O schedulers */
int td_refs; /* hold position in gd_tdallq / hold free */
int td_nest_count; /* prevent splz nesting */
+ int td_unused01[2]; /* for future fields */
#ifdef SMP
- int td_mpcount; /* MP lock held (count) */
- int td_xpcount; /* MP lock held inherited (count) */
int td_cscount; /* cpu synchronization master */
- int td_unused02[4]; /* for future fields */
#else
- int td_mpcount_unused; /* filler so size matches */
- int td_xpcount_unused;
int td_cscount_unused;
- int td_unused02[4];
#endif
- int td_unused03[4]; /* for future fields */
+ int td_unused02[4]; /* for future fields */
+ int td_unused03[4]; /* for future fields */
struct iosched_data td_iosdata; /* Dynamic I/O scheduling data */
struct timeval td_start; /* start time for a thread/process */
char td_comm[MAXCOMLEN+1]; /* typ 16+1 bytes */
/*
* Global tokens
*/
+extern struct lwkt_token mp_token;
extern struct lwkt_token pmap_token;
extern struct lwkt_token dev_token;
extern struct lwkt_token vm_token;
extern int lwkt_trytoken(lwkt_token_t);
extern void lwkt_reltoken(lwkt_token_t);
extern void lwkt_reltoken_hard(lwkt_token_t);
+extern int lwkt_cnttoken(lwkt_token_t, thread_t);
extern int lwkt_getalltokens(thread_t);
extern void lwkt_relalltokens(thread_t);
extern void lwkt_drain_token_requests(void);
#endif
/*
+ * Is a token held by the specified thread?
+ */
+static __inline int
+_lwkt_token_held(lwkt_token_t tok, thread_t td)
+{
+ return (tok->t_ref >= &td->td_toks_base &&
+ tok->t_ref < td->td_toks_stop);
+}
+
+/*
* Critical section debugging
*/
#ifdef DEBUG_CRIT_SECTIONS
}
/*
- * Return whether any threads are runnable, whether they meet mp_lock
- * requirements or not.
+ * Return whether any threads are runnable.
*/
static __inline int
lwkt_runnable(void)
#include <sys/vnode.h>
#include <sys/ptrace.h>
#include <sys/signalvar.h>
-#include <sys/signal2.h>
#include <vfs/procfs/procfs.h>
+#include <sys/signal2.h>
+#include <sys/thread2.h>
+#include <sys/mplock2.h>
+
#include <vm/vm.h>
#ifndef FIX_SSTEP
size_t offset;
size_t len;
struct rlimit limit;
- int got_mplock;
int trivial = 0;
int kflags = 0;
*/
extended = ((uio->uio_offset + uio->uio_resid) > node->tn_size);
-#ifdef SMP
- if (curthread->td_mpcount) {
- got_mplock = -1;
- } else {
- got_mplock = 1;
- get_mplock();
- }
-#else
- got_mplock = -1;
-#endif
+ get_mplock();
+
while (uio->uio_resid > 0) {
/*
* Use buffer cache I/O (via tmpfs_strategy)
}
}
- if (got_mplock > 0)
- rel_mplock();
+ rel_mplock();
if (error) {
if (extended) {
alexh's projects / dragonfly.git / commitdiff