struct ifnet *ifp = &iwl->iwl_ic.ic_if;
struct netmsg *nmsg;
+ get_mplock();
lwkt_serialize_enter(ifp->if_serializer);
while ((nmsg = lwkt_waitport(&iwl->iwl_thread_port, 0))) {
nmsg->nm_dispatch(nmsg);
break;
}
lwkt_serialize_exit(ifp->if_serializer);
-
- lwkt_exit();
+ rel_mplock();
}
void
iwl->iwl_thread_port.mp_putport = iwl_put_port;
lwkt_create(iwl_service_loop, iwl, NULL, &iwl->iwl_thread,
- 0, unit % ncpus, "iwl%d", unit);
+ TDF_MPSAFE, unit % ncpus, "iwl%d", unit);
}
static void
void
mi_proc0init(struct globaldata *gd, struct user *proc0paddr)
{
- lwkt_init_thread(&thread0, proc0paddr, LWKT_THREAD_STACK, 0, gd);
+ lwkt_init_thread(&thread0, proc0paddr, LWKT_THREAD_STACK,
+ TDF_MPSAFE, gd);
lwkt_set_comm(&thread0, "thread0");
+#ifdef SMP
+ thread0.td_mpcount = 1; /* will hold mplock initially */
+#endif
RB_INIT(&proc0.p_lwp_tree);
spin_init(&proc0.p_spin);
proc0.p_lasttid = 0; /* +1 = next TID */
static void report_stray_interrupt(int intr, struct intr_info *info);
static void int_moveto_destcpu(int *, int *, int);
static void int_moveto_origcpu(int, int);
-#ifdef SMP
-static void intr_get_mplock(void);
-#endif
int intr_info_size = sizeof(intr_info_ary) / sizeof(intr_info_ary[0]);
#define ISTATE_NORMAL 1
#define ISTATE_LIVELOCKED 2
-#ifdef SMP
-static int intr_mpsafe = 1;
-static int intr_migrate = 0;
-static int intr_migrate_count;
-TUNABLE_INT("kern.intr_mpsafe", &intr_mpsafe);
-SYSCTL_INT(_kern, OID_AUTO, intr_mpsafe,
- CTLFLAG_RW, &intr_mpsafe, 0, "Run INTR_MPSAFE handlers without the BGL");
-SYSCTL_INT(_kern, OID_AUTO, intr_migrate,
- CTLFLAG_RW, &intr_migrate, 0, "Migrate to cpu holding BGL");
-SYSCTL_INT(_kern, OID_AUTO, intr_migrate_count,
- CTLFLAG_RW, &intr_migrate_count, 0, "");
-#endif
static int livelock_limit = 40000;
static int livelock_lowater = 20000;
static int livelock_debug = -1;
*/
if (emergency_intr_thread.td_kstack == NULL) {
lwkt_create(ithread_emergency, NULL, NULL,
- &emergency_intr_thread, TDF_STOPREQ|TDF_INTTHREAD, -1,
- "ithread emerg");
+ &emergency_intr_thread,
+ TDF_STOPREQ|TDF_INTTHREAD|TDF_MPSAFE,
+ -1, "ithread emerg");
systimer_init_periodic_nq(&emergency_intr_timer,
emergency_intr_timer_callback, &emergency_intr_thread,
(emergency_intr_enable ? emergency_intr_freq : 1));
* always operate with the BGL.
*/
#ifdef SMP
- if (intr_mpsafe == 0) {
- if (mpheld == 0) {
- intr_get_mplock();
- mpheld = 1;
- }
- } else if (info->i_mplock_required != mpheld) {
+ if (info->i_mplock_required != mpheld) {
if (info->i_mplock_required) {
KKASSERT(mpheld == 0);
- intr_get_mplock();
+ get_mplock();
mpheld = 1;
} else {
KKASSERT(mpheld != 0);
mpheld = 0;
}
}
-
- /*
- * scheduled cpu may have changed, see intr_get_mplock()
- */
- gd = mycpu;
#endif
/*
*/
if (ill_count <= livelock_limit) {
if (info->i_running == 0) {
-#ifdef SMP
- if (mpheld && intr_migrate) {
- rel_mplock();
- mpheld = 0;
- }
-#endif
lwkt_deschedule_self(gd->gd_curthread);
lwkt_switch();
}
/* not reached */
}
-#ifdef SMP
-
-/*
- * An interrupt thread is trying to get the MP lock. To avoid cpu-bound
- * code in the kernel on cpu X from interfering we chase the MP lock.
- */
-static void
-intr_get_mplock(void)
-{
- int owner;
-
- if (intr_migrate == 0) {
- get_mplock();
- return;
- }
- while (try_mplock() == 0) {
- owner = owner_mplock();
- if (owner >= 0 && owner != mycpu->gd_cpuid) {
- lwkt_migratecpu(owner);
- ++intr_migrate_count;
- } else {
- lwkt_switch();
- }
- }
-}
-
-#endif
-
/*
* Emergency interrupt polling thread. The thread begins execution
* outside a critical section with the BGL held.
intrec_t rec, nrec;
int intr;
+ get_mplock();
+
for (;;) {
for (intr = 0; intr < max_installed_hard_intr; ++intr) {
info = &intr_info_ary[intr];
#include <sys/thread2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
/*
* Portability note: The u_char/unsigned char type is used where
/*
* Random number generator helper thread. This limits code overhead from
* high frequency events by delaying the clearing of rand_thread_signal.
+ *
+ * MPSAFE thread
*/
static
void
{
int count;
+ get_mplock();
+
for (;;) {
NANOUP_EVENT ();
spin_lock_wr(&rand_spin);
- count = (int)(L15_Byte() * hz / (256 * 10) + hz / 10);
+ count = (int)(L15_Byte() * hz / (256 * 10) + hz / 10 + 1);
spin_unlock_wr(&rand_spin);
tsleep(rand_td, 0, "rwait", count);
crit_enter();
void
rand_thread_init(void)
{
- lwkt_create(rand_thread_loop, NULL, &rand_td, NULL, 0, 0, "random");
+ lwkt_create(rand_thread_loop, NULL, &rand_td, NULL,
+ TDF_MPSAFE, 0, "random");
}
SYSINIT(rand, SI_SUB_HELPER_THREADS, SI_ORDER_ANY, rand_thread_init, 0);
* the cpu they were scheduled on.
*/
lwkt_create(softclock_handler, sc, NULL,
- &sc->thread, TDF_STOPREQ|TDF_INTTHREAD, cpu,
- "softclock %d", cpu);
+ &sc->thread, TDF_STOPREQ|TDF_INTTHREAD|TDF_MPSAFE,
+ cpu, "softclock %d", cpu);
}
}
* a critical section is sufficient to interlock sc->curticks and protect
* us from remote IPI's / list removal.
*
- * The thread starts with the MP lock held and not in a critical section.
- * The loop itself is MP safe while individual callbacks may or may not
- * be, so we obtain or release the MP lock as appropriate.
+ * The thread starts with the MP lock released and not in a critical
+ * section. The loop itself is MP safe while individual callbacks
+ * may or may not be, so we obtain or release the MP lock as appropriate.
*/
static void
softclock_handler(void *arg)
void (*c_func)(void *);
void *c_arg;
#ifdef SMP
- int mpsafe = 0;
+ int mpsafe = 1;
#endif
lwkt_setpri_self(TDPRI_SOFT_NORM);
{
globaldata_t mygd = mycpu;
+ /* all threads start mpsafe now */
+ KKASSERT(flags & TDF_MPSAFE);
+
bzero(td, sizeof(struct thread));
td->td_kstack = stack;
td->td_kstack_size = stksize;
* rel_mplock() at the start of the new thread.
*/
int
-lwkt_create(void (*func)(void *), void *arg,
- struct thread **tdp, thread_t template, int tdflags, int cpu,
- const char *fmt, ...)
+lwkt_create(void (*func)(void *), void *arg, struct thread **tdp,
+ thread_t template, int tdflags, int cpu, const char *fmt, ...)
{
thread_t td;
__va_list ap;
/*
* Device is known to be alive, do the attach asynchronously.
- *
- * The MP lock is held by all threads.
+ * However, serialize the attaches with the mp lock.
*/
static void
device_attach_async(device_t dev)
atomic_add_int(&numasyncthreads, 1);
lwkt_create(device_attach_thread, dev, &td, NULL,
- 0, 0, (dev->desc ? dev->desc : "devattach"));
+ TDF_MPSAFE, 0,
+ (dev->desc ? dev->desc : "devattach"));
}
static void
{
device_t dev = arg;
+ get_mplock(); /* XXX replace with devattach_token later */
(void)device_doattach(dev);
atomic_subtract_int(&numasyncthreads, 1);
wakeup(&numasyncthreads);
+ rel_mplock(); /* XXX replace with devattach_token later */
}
/*
disk_msg_t msg;
int run;
+ lwkt_gettoken(&disklist_token);
lwkt_initport_thread(&disk_msg_port, curthread);
- wakeup(curthread);
+ wakeup(curthread); /* synchronous startup */
+ lwkt_reltoken(&disklist_token);
+
+ get_mplock(); /* not mpsafe yet? */
run = 1;
while (run) {
*/
lwkt_initport_replyonly(&disk_dispose_port, disk_msg_autofree_reply);
+ lwkt_gettoken(&disklist_token);
lwkt_create(disk_msg_core, /*args*/NULL, &td_core, NULL,
- 0, 0, "disk_msg_core");
-
+ TDF_MPSAFE, 0, "disk_msg_core");
tsleep(td_core, 0, "diskcore", 0);
+ lwkt_reltoken(&disklist_token);
}
static void
/*
* For SMP systems a user scheduler helper thread is created for each
* cpu and is used to allow one cpu to wakeup another for the purposes of
- * scheduling userland threads from setrunqueue(). UP systems do not
- * need the helper since there is only one cpu. We can't use the idle
- * thread for this because we need to hold the MP lock. Additionally,
- * doing things this way allows us to HLT idle cpus on MP systems.
+ * scheduling userland threads from setrunqueue().
+ *
+ * UP systems do not need the helper since there is only one cpu.
+ *
+ * We can't use the idle thread for this because we might block.
+ * Additionally, doing things this way allows us to HLT idle cpus
+ * on MP systems.
*
* MPSAFE
*/
dd = &bsd4_pcpu[cpuid];
/*
- * The scheduler thread does not need to hold the MP lock. Since we
- * are woken up only when no user processes are scheduled on a cpu, we
- * can run at an ultra low priority.
+ * Since we are woken up only when no user processes are scheduled
+ * on a cpu, we can run at an ultra low priority.
*/
- rel_mplock();
lwkt_setpri_self(TDPRI_USER_SCHEDULER);
for (;;) {
kprintf(" %d", i);
lwkt_create(sched_thread, NULL, NULL, &dd->helper_thread,
- TDF_STOPREQ, i, "usched %d", i);
+ TDF_STOPREQ | TDF_MPSAFE, i, "usched %d", i);
/*
* Allow user scheduling on the target cpu. cpu #0 has already
* is possible to deschedule an LWKT thread and then do some work before
* switching away. The thread can be rescheduled at any time, even before
* we switch away.
+ *
+ * MPSAFE
*/
#ifdef SMP
dd = &dummy_pcpu[cpuid];
cpumask = 1 << cpuid;
- /*
- * Our Scheduler helper thread does not need to hold the MP lock
- */
- rel_mplock();
-
for (;;) {
lwkt_deschedule_self(gd->gd_curthread); /* interlock */
atomic_set_int(&dummy_rdyprocmask, cpumask);
kprintf(" %d", i);
lwkt_create(dummy_sched_thread, NULL, NULL, &dd->helper_thread,
- TDF_STOPREQ, i, "dsched %d", i);
+ TDF_STOPREQ | TDF_MPSAFE, i, "dsched %d", i);
/*
* Allow user scheduling on the target cpu. cpu #0 has already
#include <sys/file2.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <sys/spinlock2.h>
static void journal_wthread(void *info);
jo->flags &= ~(MC_JOURNAL_STOP_REQ | MC_JOURNAL_STOP_IMM);
jo->flags |= MC_JOURNAL_WACTIVE;
lwkt_create(journal_wthread, jo, NULL, &jo->wthread,
- TDF_STOPREQ, -1, "journal w:%.*s", JIDMAX, jo->id);
+ TDF_STOPREQ | TDF_MPSAFE,
+ -1, "journal w:%.*s", JIDMAX, jo->id);
lwkt_setpri(&jo->wthread, TDPRI_KERN_DAEMON);
lwkt_schedule(&jo->wthread);
if (jo->flags & MC_JOURNAL_WANT_FULLDUPLEX) {
jo->flags |= MC_JOURNAL_RACTIVE;
lwkt_create(journal_rthread, jo, NULL, &jo->rthread,
- TDF_STOPREQ, -1, "journal r:%.*s", JIDMAX, jo->id);
+ TDF_STOPREQ | TDF_MPSAFE,
+ -1, "journal r:%.*s", JIDMAX, jo->id);
lwkt_setpri(&jo->rthread, TDPRI_KERN_DAEMON);
lwkt_schedule(&jo->rthread);
}
size_t bytes;
size_t res;
+ /* not MPSAFE yet */
+ get_mplock();
+
for (;;) {
/*
* Calculate the number of bytes available to write. This buffer
jo->flags &= ~MC_JOURNAL_WACTIVE;
wakeup(jo);
wakeup(&jo->fifo.windex);
+ rel_mplock();
}
/*
transid = 0;
error = 0;
+ /* not MPSAFE yet */
+ get_mplock();
+
for (;;) {
/*
* We have been asked to stop
jo->flags &= ~MC_JOURNAL_RACTIVE;
wakeup(jo);
wakeup(&jo->fifo.windex);
+ rel_mplock();
}
/*
#include <sys/thread2.h>
#include <sys/msgport2.h>
+#include <sys/mplock2.h>
#include <net/netmsg2.h>
#ifdef MPLS
for (cpu = 0; cpu < ncpus; cpu++) {
lwkt_create(rtable_service_loop, NULL, &rtd, NULL,
- 0, cpu, "rtable_cpu %d", cpu);
+ TDF_MPSAFE, cpu, "rtable_cpu %d", cpu);
rt_ports[cpu] = &rtd->td_msgport;
}
}
struct netmsg *netmsg;
thread_t td = curthread;
+ get_mplock(); /* XXX is this mpsafe yet? */
+
while ((netmsg = lwkt_waitport(&td->td_msgport, 0)) != NULL) {
netmsg->nm_dispatch(netmsg);
}
#include <sys/types.h>
#include <sys/lock.h>
#include <sys/msgport.h>
-#include <sys/msgport2.h>
-#include <sys/spinlock2.h>
#include <sys/sysctl.h>
#include <sys/ucred.h>
#include <sys/param.h>
-#include <sys/sysref2.h>
#include <sys/systm.h>
#include <sys/devfs.h>
#include <sys/devfs_rules.h>
#include <sys/hotplug.h>
#include <sys/udev.h>
+#include <sys/msgport2.h>
+#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
+#include <sys/sysref2.h>
+
MALLOC_DEFINE(M_DEVFS, "devfs", "Device File System (devfs) allocations");
DEVFS_DECLARE_CLONE_BITMAP(ops_id);
/*
{
devfs_msg_t msg;
- devfs_run = 1;
lwkt_initport_thread(&devfs_msg_port, curthread);
+
+ lockmgr(&devfs_lock, LK_EXCLUSIVE);
+ devfs_run = 1;
wakeup(td_core);
+ lockmgr(&devfs_lock, LK_RELEASE);
+
+ get_mplock(); /* mpsafe yet? */
while (devfs_run) {
msg = (devfs_msg_t)lwkt_waitport(&devfs_msg_port, 0);
devfs_msg_exec(msg);
lwkt_replymsg(&msg->hdr, 0);
}
+
+ rel_mplock();
wakeup(td_core);
+
lwkt_exit();
}
/* Initialize *THE* devfs lock */
lockinit(&devfs_lock, "devfs_core lock", 0, 0);
-
+ lockmgr(&devfs_lock, LK_EXCLUSIVE);
lwkt_create(devfs_msg_core, /*args*/NULL, &td_core, NULL,
- 0, 0, "devfs_msg_core");
-
+ TDF_MPSAFE, 0, "devfs_msg_core");
while (devfs_run == 0)
- tsleep(td_core, 0, "devfsc", 0);
+ lksleep(td_core, &devfs_lock, 0, "devfsc", 0);
+ lockmgr(&devfs_lock, LK_RELEASE);
devfs_debug(DEVFS_DEBUG_DEBUG, "devfs_init finished\n");
}
#include <sys/mutex.h>
#include <sys/signal2.h>
+#include <sys/thread2.h>
#include <sys/mutex2.h>
+#include <sys/mplock2.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
-#include <sys/thread2.h>
#include "rpcv2.h"
#include "nfsproto.h"
#include "nfsnode.h"
#include "nfsrtt.h"
+/*
+ * nfs service connection reader thread
+ */
void
nfssvc_iod_reader(void *arg)
{
struct nfsreq *req;
int error;
+ get_mplock();
+
if (nmp->nm_rxstate == NFSSVC_INIT)
nmp->nm_rxstate = NFSSVC_PENDING;
crit_enter();
}
/*
+ * nfs service connection writer thread
+ *
* The writer sits on the send side of the client's socket and
* does both the initial processing of BIOs and also transmission
* and retransmission of nfsreq's.
struct vnode *vp;
nfsm_info_t info;
+ get_mplock();
+
if (nmp->nm_txstate == NFSSVC_INIT)
nmp->nm_txstate = NFSSVC_PENDING;
+
crit_enter();
for (;;) {
if (nmp->nm_txstate == NFSSVC_WAITING) {
* Start the reader and writer threads.
*/
lwkt_create(nfssvc_iod_reader, nmp, &nmp->nm_rxthread,
- NULL, 0, rxcpu, "nfsiod_rx");
+ NULL, TDF_MPSAFE, rxcpu, "nfsiod_rx");
lwkt_create(nfssvc_iod_writer, nmp, &nmp->nm_txthread,
- NULL, 0, txcpu, "nfsiod_tx");
+ NULL, TDF_MPSAFE, txcpu, "nfsiod_tx");
return (0);
bad:
projects / dragonfly.git / commitdiff