X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/blobdiff_plain/70ccbb6b59534e9e261f8831dd0dccd133701ee9..3b998fa96afe52828957ea4f65d15320eb0fe240:/sys/kern/lwkt_thread.c diff --git a/sys/kern/lwkt_thread.c b/sys/kern/lwkt_thread.c index 959549cc89..9f6ba58ec9 100644 --- a/sys/kern/lwkt_thread.c +++ b/sys/kern/lwkt_thread.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003,2004 The DragonFly Project. All rights reserved. + * Copyright (c) 2003-2010 The DragonFly Project. All rights reserved. * * This code is derived from software contributed to The DragonFly Project * by Matthew Dillon @@ -55,6 +55,9 @@ #include #include +#include + +#include #include #include @@ -72,14 +75,16 @@ #define KTR_CTXSW KTR_ALL #endif KTR_INFO_MASTER(ctxsw); -KTR_INFO(KTR_CTXSW, ctxsw, sw, 0, "sw %p > %p", 2 * sizeof(struct thread *)); -KTR_INFO(KTR_CTXSW, ctxsw, pre, 1, "pre %p > %p", 2 * sizeof(struct thread *)); +KTR_INFO(KTR_CTXSW, ctxsw, sw, 0, "#cpu[%d].td = %p", + sizeof(int) + sizeof(struct thread *)); +KTR_INFO(KTR_CTXSW, ctxsw, pre, 1, "#cpu[%d].td = %p", + sizeof(int) + sizeof(struct thread *)); +KTR_INFO(KTR_CTXSW, ctxsw, newtd, 2, "#threads[%p].name = %s", + sizeof (struct thread *) + sizeof(char *)); +KTR_INFO(KTR_CTXSW, ctxsw, deadtd, 3, "#threads[%p].name = ", sizeof (struct thread *)); static MALLOC_DEFINE(M_THREAD, "thread", "lwkt threads"); -#ifdef SMP -static int mplock_countx = 0; -#endif #ifdef INVARIANTS static int panic_on_cscount = 0; #endif @@ -87,17 +92,10 @@ static __int64_t switch_count = 0; static __int64_t preempt_hit = 0; static __int64_t preempt_miss = 0; static __int64_t preempt_weird = 0; -static __int64_t token_contention_count = 0; -static __int64_t mplock_contention_count = 0; +static __int64_t token_contention_count __debugvar = 0; static int lwkt_use_spin_port; -#ifdef SMP -static int chain_mplock = 0; -static int bgl_yield = 10; -#endif static struct objcache *thread_cache; -volatile cpumask_t mp_lock_contention_mask; - #ifdef SMP static void lwkt_schedule_remote(void *arg, int arg2, struct intrframe *frame); #endif @@ -139,10 +137,6 @@ TUNABLE_INT("lwkt.use_spin_port", &lwkt_use_spin_port); #ifdef INVARIANTS SYSCTL_INT(_lwkt, OID_AUTO, panic_on_cscount, CTLFLAG_RW, &panic_on_cscount, 0, ""); #endif -#ifdef SMP -SYSCTL_INT(_lwkt, OID_AUTO, chain_mplock, CTLFLAG_RW, &chain_mplock, 0, ""); -SYSCTL_INT(_lwkt, OID_AUTO, bgl_yield_delay, CTLFLAG_RW, &bgl_yield, 0, ""); -#endif SYSCTL_QUAD(_lwkt, OID_AUTO, switch_count, CTLFLAG_RW, &switch_count, 0, ""); SYSCTL_QUAD(_lwkt, OID_AUTO, preempt_hit, CTLFLAG_RW, &preempt_hit, 0, ""); SYSCTL_QUAD(_lwkt, OID_AUTO, preempt_miss, CTLFLAG_RW, &preempt_miss, 0, ""); @@ -150,23 +144,8 @@ SYSCTL_QUAD(_lwkt, OID_AUTO, preempt_weird, CTLFLAG_RW, &preempt_weird, 0, ""); #ifdef INVARIANTS SYSCTL_QUAD(_lwkt, OID_AUTO, token_contention_count, CTLFLAG_RW, &token_contention_count, 0, "spinning due to token contention"); -SYSCTL_QUAD(_lwkt, OID_AUTO, mplock_contention_count, CTLFLAG_RW, - &mplock_contention_count, 0, "spinning due to MPLOCK contention"); #endif -/* - * Kernel Trace - */ -#if !defined(KTR_GIANT_CONTENTION) -#define KTR_GIANT_CONTENTION KTR_ALL -#endif - -KTR_INFO_MASTER(giant); -KTR_INFO(KTR_GIANT_CONTENTION, giant, beg, 0, "thread=%p", sizeof(void *)); -KTR_INFO(KTR_GIANT_CONTENTION, giant, end, 1, "thread=%p", sizeof(void *)); - -#define loggiant(name) KTR_LOG(giant_ ## name, curthread) - /* * These helper procedures handle the runq, they can only be called from * within a critical section. @@ -377,6 +356,7 @@ lwkt_init_thread(thread_t td, void *stack, int stksize, int flags, td->td_flags = flags; td->td_gd = gd; td->td_pri = TDPRI_KERN_DAEMON + TDPRI_CRIT; + td->td_toks_stop = &td->td_toks_base; #ifdef SMP if ((flags & TDF_MPSAFE) == 0) td->td_mpcount = 1; @@ -405,6 +385,8 @@ lwkt_init_thread(thread_t td, void *stack, int stksize, int flags, TAILQ_INSERT_TAIL(&gd->gd_tdallq, td, td_allq); crit_exit_gd(mygd); #endif + + dsched_new_thread(td); } void @@ -415,6 +397,7 @@ lwkt_set_comm(thread_t td, const char *ctl, ...) __va_start(va, ctl); kvsnprintf(td->td_comm, sizeof(td->td_comm), ctl, va); __va_end(va); + KTR_LOG(ctxsw_newtd, td, &td->td_comm[0]); } void @@ -453,6 +436,7 @@ lwkt_free_thread(thread_t td) td->td_kstack = NULL; td->td_kstack_size = 0; } + KTR_LOG(ctxsw_deadtd, td); } @@ -515,7 +499,7 @@ lwkt_switch(void) td->td_flags |= TDF_PANICWARN; kprintf("Warning: thread switch from interrupt or IPI, " "thread %p (%s)\n", td, td->td_comm); - print_backtrace(); + print_backtrace(-1); } lwkt_switch(); gd->gd_intr_nesting_level = savegdnest; @@ -537,7 +521,7 @@ lwkt_switch(void) td->td_release(td); crit_enter_gd(gd); - if (td->td_toks) + if (TD_TOKS_HELD(td)) lwkt_relalltokens(td); /* @@ -652,7 +636,7 @@ again: * cause the core MP lock to be released. */ if ((ntd->td_mpcount && mpheld == 0 && !cpu_try_mplock()) || - (ntd->td_toks && lwkt_getalltokens(ntd) == 0) + (TD_TOKS_HELD(ntd) && lwkt_getalltokens(ntd) == 0) ) { u_int32_t rqmask = gd->gd_runqmask; @@ -662,10 +646,6 @@ again: TAILQ_FOREACH(ntd, &gd->gd_tdrunq[nq], td_threadq) { if (ntd->td_mpcount && !mpheld && !cpu_try_mplock()) { /* spinning due to MP lock being held */ -#ifdef INVARIANTS - ++mplock_contention_count; -#endif - /* mplock still not held, 'mpheld' still valid */ continue; } @@ -674,7 +654,7 @@ again: * failure, but the variable is only needed for * the loop. */ - if (ntd->td_toks && !lwkt_getalltokens(ntd)) { + if (TD_TOKS_HELD(ntd) && !lwkt_getalltokens(ntd)) { /* spinning due to token contention */ #ifdef INVARIANTS ++token_contention_count; @@ -697,26 +677,35 @@ again: * reschedule when the MP lock might become available. */ if (nq < TDPRI_KERN_LPSCHED) { + break; /* for now refuse to run */ +#if 0 if (chain_mplock == 0) break; - atomic_set_int(&mp_lock_contention_mask, - gd->gd_cpumask); /* continue loop, allow user threads to be scheduled */ +#endif } } + + /* + * Case where a (kernel) thread needed the MP lock and could + * not get one, and we may or may not have found another + * thread which does not need the MP lock to run while + * we wait (ntd). + */ if (ntd == NULL) { - cpu_mplock_contested(); ntd = &gd->gd_idlethread; ntd->td_flags |= TDF_IDLE_NOHLT; + set_mplock_contention_mask(gd); + cpu_mplock_contested(); goto using_idle_thread; } else { + clr_mplock_contention_mask(gd); ++gd->gd_cnt.v_swtch; TAILQ_REMOVE(&gd->gd_tdrunq[nq], ntd, td_threadq); TAILQ_INSERT_TAIL(&gd->gd_tdrunq[nq], ntd, td_threadq); } } else { - if (ntd->td_mpcount) - ++mplock_countx; + clr_mplock_contention_mask(gd); ++gd->gd_cnt.v_swtch; TAILQ_REMOVE(&gd->gd_tdrunq[nq], ntd, td_threadq); TAILQ_INSERT_TAIL(&gd->gd_tdrunq[nq], ntd, td_threadq); @@ -752,12 +741,10 @@ using_idle_thread: */ if (ntd->td_mpcount) { mpheld = MP_LOCK_HELD(); - if (gd->gd_trap_nesting_level == 0 && panicstr == NULL) { + if (gd->gd_trap_nesting_level == 0 && panicstr == NULL) panic("Idle thread %p was holding the BGL!", ntd); - } else if (mpheld == 0) { - cpu_mplock_contested(); + if (mpheld == 0) goto again; - } } #endif } @@ -781,9 +768,12 @@ using_idle_thread: if (td != ntd) { ++switch_count; #ifdef __x86_64__ - KKASSERT(jg_tos_ok(ntd)); + { + int tos_ok __debugvar = jg_tos_ok(ntd); + KKASSERT(tos_ok); + } #endif - KTR_LOG(ctxsw_sw, td, ntd); + KTR_LOG(ctxsw_sw, gd->gd_cpuid, ntd); td->td_switch(ntd); } /* NOTE: current cpu may have changed after switch */ @@ -883,7 +873,7 @@ lwkt_preempt(thread_t ntd, int critpri) need_lwkt_resched(); return; } - if (ntd->td_toks) { + if (TD_TOKS_HELD(ntd)) { ++preempt_miss; need_lwkt_resched(); return; @@ -925,7 +915,7 @@ lwkt_preempt(thread_t ntd, int critpri) ++preempt_hit; ntd->td_preempted = td; td->td_flags |= TDF_PREEMPT_LOCK; - KTR_LOG(ctxsw_pre, td, ntd); + KTR_LOG(ctxsw_pre, gd->gd_cpuid, ntd); td->td_switch(ntd); KKASSERT(ntd->td_preempted && (td->td_flags & TDF_PREEMPT_DONE)); @@ -1013,15 +1003,8 @@ lwkt_user_yield(void) * has a chaining effect since if the interrupt is blocked, so is * the event, so normal scheduling will not pick up on the problem. */ - if (mplock_countx && td->td_mpcount) { - int savecnt = td->td_mpcount; - - td->td_mpcount = 1; - mplock_countx = 0; - rel_mplock(); - DELAY(bgl_yield); - get_mplock(); - td->td_mpcount = savecnt; + if (mp_lock_contention_mask && td->td_mpcount) { + yield_mplock(td); } #endif @@ -1496,6 +1479,7 @@ lwkt_exit(void) if (td->td_flags & TDF_TSLEEPQ) tsleep_remove(td); biosched_done(td); + dsched_exit_thread(td); lwkt_deschedule_self(td); lwkt_remove_tdallq(td); if (td->td_flags & TDF_ALLOCATED_THREAD) @@ -1546,73 +1530,4 @@ lwkt_smp_stopped(void) crit_exit_gd(gd); } -/* - * get_mplock() calls this routine if it is unable to obtain the MP lock. - * get_mplock() has already incremented td_mpcount. We must block and - * not return until giant is held. - * - * All we have to do is lwkt_switch() away. The LWKT scheduler will not - * reschedule the thread until it can obtain the giant lock for it. - */ -void -lwkt_mp_lock_contested(void) -{ - ++mplock_countx; - loggiant(beg); - lwkt_switch(); - loggiant(end); -} - -/* - * The rel_mplock() code will call this function after releasing the - * last reference on the MP lock if mp_lock_contention_mask is non-zero. - * - * We then chain an IPI to a single other cpu potentially needing the - * lock. This is a bit heuristical and we can wind up with IPIs flying - * all over the place. - */ -static void lwkt_mp_lock_uncontested_remote(void *arg __unused); - -void -lwkt_mp_lock_uncontested(void) -{ - globaldata_t gd; - globaldata_t dgd; - cpumask_t mask; - cpumask_t tmpmask; - int cpuid; - - if (chain_mplock) { - gd = mycpu; - atomic_clear_int(&mp_lock_contention_mask, gd->gd_cpumask); - mask = mp_lock_contention_mask; - tmpmask = ~((1 << gd->gd_cpuid) - 1); - - if (mask) { - if (mask & tmpmask) - cpuid = bsfl(mask & tmpmask); - else - cpuid = bsfl(mask); - atomic_clear_int(&mp_lock_contention_mask, 1 << cpuid); - dgd = globaldata_find(cpuid); - lwkt_send_ipiq(dgd, lwkt_mp_lock_uncontested_remote, NULL); - } - } -} - -/* - * The idea is for this IPI to interrupt a potentially lower priority - * thread, such as a user thread, to allow the scheduler to reschedule - * a higher priority kernel thread that needs the MP lock. - * - * For now we set the LWKT reschedule flag which generates an AST in - * doreti, though theoretically it is also possible to possibly preempt - * here if the underlying thread was operating in user mode. Nah. - */ -static void -lwkt_mp_lock_uncontested_remote(void *arg __unused) -{ - need_lwkt_resched(); -} - #endif