void
linprocfs_init(void)
{
- lwkt_token_init(&pfs_token, 1);
+ lwkt_token_init(&pfs_token, 1, "linprocfs");
}
void
struct ndis_req *r;
int i, error = 0;
- lwkt_token_init(&ndis_thr_token, 1);
+ lwkt_token_init(&ndis_thr_token, 1, "ndis");
STAILQ_INIT(&ndis_ttodo);
STAILQ_INIT(&ndis_itodo);
int
ntoskrnl_libinit(void)
{
- lwkt_token_init(&ntoskrnl_dispatchtoken, 1);
+ lwkt_token_init(&ntoskrnl_dispatchtoken, 1, "ndiskrnl");
ntoskrnl_init_lock(&ntoskrnl_global);
TAILQ_INIT(&ntoskrnl_reflist);
return(0);
globaldata_t gd = mycpu;
int ov;
int nv;
+ const void **stkframe = (const void **)&file;
++mplock_contention_count;
for (;;) {
if (atomic_cmpset_int(&mp_lock, ov, gd->gd_cpuid))
break;
} else {
+ gd->gd_curthread->td_mplock_stallpc = stkframe[-1];
loggiant(beg);
lwkt_switch();
loggiant(end);
#include <sys/kernel.h>
#include <sys/proc.h>
#include <sys/rtprio.h>
+#include <sys/kinfo.h>
#include <sys/queue.h>
#include <sys/sysctl.h>
#include <sys/kthread.h>
int mpheld;
#endif
int didaccumulate;
+ const char *lmsg; /* diagnostic - 'systat -pv 1' */
+ const void *laddr;
/*
* Switching from within a 'fast' (non thread switched) interrupt or IPI
continue;
}
}
+ cpu_time.cp_msg[0] = 0;
+ cpu_time.cp_stallpc = 0;
goto haveidle;
}
#ifdef SMP
(ntd->td_mpcount == 0 || mpheld || cpu_try_mplock()) &&
#endif
- (!TD_TOKS_HELD(ntd) || lwkt_getalltokens(ntd))
+ (!TD_TOKS_HELD(ntd) || lwkt_getalltokens(ntd, &lmsg, &laddr))
) {
#ifdef SMP
clr_mplock_contention_mask(gd);
goto havethread;
}
+ lmsg = NULL;
+ laddr = NULL;
+
#ifdef SMP
/* Reload mpheld (it become stale after mplock/token ops) */
mpheld = MP_LOCK_HELD();
+ if (ntd->td_mpcount && mpheld == 0) {
+ lmsg = "mplock";
+ laddr = ntd->td_mplock_stallpc;
+ }
#endif
/*
* the (almost) top.
*/
if (didaccumulate)
- break;
+ break; /* try again from the top, almost */
+ if (lmsg)
+ strlcpy(cpu_time.cp_msg, lmsg, sizeof(cpu_time.cp_msg));
+ cpu_time.cp_stallpc = (uintptr_t)laddr;
goto haveidle;
}
#ifdef SMP
(ntd->td_mpcount == 0 || mpheld || cpu_try_mplock()) &&
#endif
- (!TD_TOKS_HELD(ntd) || lwkt_getalltokens(ntd))
+ (!TD_TOKS_HELD(ntd) || lwkt_getalltokens(ntd, &lmsg, &laddr))
) {
#ifdef SMP
clr_mplock_contention_mask(gd);
#ifdef SMP
/* Reload mpheld (it become stale after mplock/token ops) */
mpheld = MP_LOCK_HELD();
+ if (ntd->td_mpcount && mpheld == 0) {
+ lmsg = "mplock";
+ laddr = ntd->td_mplock_stallpc;
+ }
+
if (ntd->td_pri >= TDPRI_KERN_LPSCHED && ntd->td_fairq_accum >= 0)
nquserok = 0;
#endif
* Called from a critical section.
*/
int
-lwkt_getalltokens(thread_t td)
+lwkt_getalltokens(thread_t td, const char **msgp, const void **addrp)
{
lwkt_tokref_t scan;
lwkt_tokref_t ref;
* Otherwise we failed to acquire all the tokens.
* Undo and return.
*/
+ *msgp = tok->t_desc;
+ *addrp = scan->tr_stallpc;
atomic_add_long(&tok->t_collisions, 1);
lwkt_relalltokens(td);
return(FALSE);
*/
static __inline
void
-_lwkt_gettokref(lwkt_tokref_t ref, thread_t td)
+_lwkt_gettokref(lwkt_tokref_t ref, thread_t td, const void **stkframe)
{
if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0)
get_mplock();
* return tr_tok->t_ref should be assigned to this specific
* ref.
*/
+ ref->tr_stallpc = stkframe[-1];
atomic_add_long(&ref->tr_tok->t_collisions, 1);
logtoken(fail, ref);
lwkt_switch();
KKASSERT(ref < &td->td_toks_end);
_lwkt_tokref_init(ref, tok, td);
++td->td_toks_stop;
- _lwkt_gettokref(ref, td);
+ _lwkt_gettokref(ref, td, (const void **)&tok);
}
lwkt_token_t
tok = _lwkt_token_pool_lookup(ptr);
_lwkt_tokref_init(ref, tok, td);
++td->td_toks_stop;
- _lwkt_gettokref(ref, td);
+ _lwkt_gettokref(ref, td, (const void **)&ptr);
return(tok);
}
int i;
for (i = 0; i < LWKT_NUM_POOL_TOKENS; ++i)
- lwkt_token_init(&pool_tokens[i], 1);
+ lwkt_token_init(&pool_tokens[i], 1, "pool");
}
lwkt_token_t
* acquiring the token and released after releasing the token.
*/
void
-lwkt_token_init(lwkt_token_t tok, int mpsafe)
+lwkt_token_init(lwkt_token_t tok, int mpsafe, const char *desc)
{
tok->t_ref = NULL;
tok->t_flags = mpsafe ? LWKT_TOKEN_MPSAFE : 0;
objcache_malloc_free,
&disk_msg_malloc_args);
- lwkt_token_init(&disklist_token, 1);
+ lwkt_token_init(&disklist_token, 1, "disks");
/*
* Initialize the reply-only port which acts as a message drain
static void
kobj_init_token(void *arg)
{
- lwkt_token_init(&kobj_token, 1);
+ lwkt_token_init(&kobj_token, 1, "kobj");
}
SYSINIT(kobj, SI_BOOT1_LOCK, SI_ORDER_ANY, kobj_init_token, NULL);
if (once == 0) {
once = 1;
TAILQ_INIT(&rman_head);
- lwkt_token_init(&rman_tok, 1);
+ lwkt_token_init(&rman_tok, 1, "rman");
}
if (rm->rm_type == RMAN_UNINIT)
rm->rm_slock = kmalloc(sizeof *rm->rm_slock, M_RMAN, M_NOWAIT);
if (rm->rm_slock == NULL)
return ENOMEM;
- lwkt_token_init(rm->rm_slock, 1);
+ lwkt_token_init(rm->rm_slock, 1, "rmanslock");
lwkt_gettoken(&rman_tok);
TAILQ_INSERT_TAIL(&rman_head, rm, rm_link);
vfs_timestamp(&cpipe->pipe_ctime);
cpipe->pipe_atime = cpipe->pipe_ctime;
cpipe->pipe_mtime = cpipe->pipe_ctime;
- lwkt_token_init(&cpipe->pipe_rlock, 1);
- lwkt_token_init(&cpipe->pipe_wlock, 1);
+ lwkt_token_init(&cpipe->pipe_rlock, 1, "piper");
+ lwkt_token_init(&cpipe->pipe_wlock, 1, "pipew");
return (0);
}
{
struct vnode *vp = obj;
- lwkt_token_init(&vp->v_token, 1);
+ lwkt_token_init(&vp->v_token, 1, "vnode");
lockinit(&vp->v_lock, "vnode", 0, 0);
ccms_dataspace_init(&vp->v_ccms);
TAILQ_INIT(&vp->v_namecache);
void
vfs_mount_init(void)
{
- lwkt_token_init(&mountlist_token, 1);
- lwkt_token_init(&mntvnode_token, 1);
- lwkt_token_init(&mntid_token, 1);
+ lwkt_token_init(&mountlist_token, 1, "mntlist");
+ lwkt_token_init(&mntvnode_token, 1, "mntvnode");
+ lwkt_token_init(&mntid_token, 1, "mntid");
TAILQ_INIT(&mountscan_list);
TAILQ_INIT(&mntvnodescan_list);
mount_init(&dummymount);
mount_init(struct mount *mp)
{
lockinit(&mp->mnt_lock, "vfslock", 0, 0);
- lwkt_token_init(&mp->mnt_token, 1);
+ lwkt_token_init(&mp->mnt_token, 1, "permnt");
TAILQ_INIT(&mp->mnt_nvnodelist);
TAILQ_INIT(&mp->mnt_reservedvnlist);
KvaSize / factor2);
desiredvnodes = imax(desiredvnodes, maxproc * 8);
- lwkt_token_init(&spechash_token, 1);
+ lwkt_token_init(&spechash_token, 1, "spechash");
}
/*
syncer_workitem_pending = hashinit(syncer_maxdelay, M_DEVBUF,
&syncer_mask);
syncer_maxdelay = syncer_mask + 1;
- lwkt_token_init(&syncer_token, 1);
+ lwkt_token_init(&syncer_token, 1, "syncer");
}
/*
uint64_t cp_sys;
uint64_t cp_intr;
uint64_t cp_idle;
+ uint64_t cp_unused01;
+ uint64_t cp_unused02;
+ uint64_t cp_unused03;
+ uint64_t cp_stallpc; /* code stall address */
+ char cp_msg[32]; /* code stall token or mplock */
};
/*
struct lwkt_tokref *t_ref; /* Owning ref or NULL */
intptr_t t_flags; /* MP lock required */
long t_collisions; /* Collision counter */
+ const char *t_desc; /* Descriptive name */
} lwkt_token;
#define LWKT_TOKEN_MPSAFE 0x0001
lwkt_token_t tr_tok; /* token in question */
struct thread *tr_owner; /* me */
intptr_t tr_flags; /* copy of t_flags */
+ const void *tr_stallpc; /* stalled at pc */
};
#define MAXCPUFIFO 16 /* power of 2 */
__uint64_t td_sticks; /* Statclock hits in system mode (uS) */
__uint64_t td_iticks; /* Statclock hits processing intr (uS) */
int td_locks; /* lockmgr lock debugging */
- int td_fairq_lticks; /* fairq wakeup accumulator reset */
void *td_dsched_priv1; /* priv data for I/O schedulers */
int td_refs; /* hold position in gd_tdallq / hold free */
int td_nest_count; /* prevent splz nesting */
struct caps_kinfo *td_caps; /* list of client and server registrations */
lwkt_tokref_t td_toks_stop;
struct lwkt_tokref td_toks_array[LWKT_MAXTOKENS];
+ int td_fairq_lticks; /* fairq wakeup accumulator reset */
+ int td_fairq_accum; /* fairq priority accumulator */
+ const void *td_mplock_stallpc; /* last mplock stall address */
#ifdef DEBUG_CRIT_SECTIONS
#define CRIT_DEBUG_ARRAY_SIZE 32
#define CRIT_DEBUG_ARRAY_MASK (CRIT_DEBUG_ARRAY_SIZE - 1)
int td_crit_debug_index;
int td_in_crit_report;
#endif
- int td_fairq_accum; /* fairq priority accumulator */
struct md_thread td_mach;
};
extern void lwkt_gettoken(lwkt_token_t);
extern int lwkt_trytoken(lwkt_token_t);
extern void lwkt_reltoken(lwkt_token_t);
-extern int lwkt_getalltokens(thread_t);
+extern int lwkt_getalltokens(thread_t, const char **, const void **);
extern void lwkt_relalltokens(thread_t);
extern void lwkt_drain_token_requests(void);
-extern void lwkt_token_init(lwkt_token_t, int);
+extern void lwkt_token_init(lwkt_token_t, int, const char *);
extern void lwkt_token_uninit(lwkt_token_t);
extern void lwkt_token_pool_init(void);
ext2_ihash <<= 1;
ext2_ihashtbl = kmalloc(sizeof(void *) * ext2_ihash, M_EXT2IHASH, M_WAITOK|M_ZERO);
--ext2_ihash;
- lwkt_token_init(&ext2_ihash_token, 1);
+ lwkt_token_init(&ext2_ihash_token, 1, "ext2ihash");
}
int
lockinit (&hpfs_hphash_lock, "hpfs_hphashlock", 0, 0);
hpfs_hphashtbl = HASHINIT(desiredvnodes, M_HPFSHASH, M_WAITOK,
&hpfs_hphash);
- lwkt_token_init(&hpfs_hphash_token, 1);
+ lwkt_token_init(&hpfs_hphash_token, 1, "hpfsihash");
}
/*
if (ino == (ino_t)hpmp->hpm_su.su_rootfno)
vsetflags(vp, VROOT);
- lwkt_token_init(&hp->h_interlock, 1);
+ lwkt_token_init(&hp->h_interlock, 1, "hpfsilock");
hp->h_flag = H_INVAL;
hp->h_vp = vp;
isohashtbl = kmalloc(sizeof(void *) * isohash,
M_ISOFSMNT, M_WAITOK|M_ZERO);
--isohash;
- lwkt_token_init(&cd9660_ihash_token, 1);
+ lwkt_token_init(&cd9660_ihash_token, 1, "cd9660ihash");
return (0);
}
dehashtbl = kmalloc(sizeof(void *) * dehash, M_MSDOSFSMNT,
M_WAITOK|M_ZERO);
--dehash;
- lwkt_token_init(&dehash_token, 1);
+ lwkt_token_init(&dehash_token, 1, "msdosihash");
return (0);
}
lockinit(&ntfs_hashlock, "ntfs_nthashlock", 0, 0);
ntfs_nthashtbl = HASHINIT(desiredvnodes, M_NTFSNTHASH, M_WAITOK,
&ntfs_nthash);
- lwkt_token_init(&ntfs_nthash_slock, 1);
+ lwkt_token_init(&ntfs_nthash_slock, 1, "ntfsihash");
}
/*
brelse(bp);
bp = NULL;
- lwkt_token_init(&udfmp->hash_token, 1);
+ lwkt_token_init(&udfmp->hash_token, 1, "udfihash");
udfmp->hashtbl = phashinit(UDF_HASHTBLSIZE, M_UDFMOUNT, &udfmp->hashsz);
return(0);
ihash <<= 1;
ihashtbl = kmalloc(sizeof(void *) * ihash, M_UFSIHASH, M_WAITOK|M_ZERO);
--ihash;
- lwkt_token_init(&ufs_ihash_token, 1);
+ lwkt_token_init(&ufs_ihash_token, 1, "ufsihash");
}
int
lwkt_gettoken(&proc_token);
if (p->p_vkernel == NULL) {
vkp->refs = 1;
- lwkt_token_init(&vkp->token, 1);
+ lwkt_token_init(&vkp->token, 1, "vkernel");
RB_INIT(&vkp->root);
p->p_vkernel = vkp;
} else {
* resched has been requested.
*/
while (i < PAGE_SIZE) {
- lwkt_yield();
if (idlezero_nocache == 1)
bzeront(&pg[i], IDLEZERO_RUN);
else