From cd8ab232f5149d9b74e3bda36d4e0ba83e5d417c Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Sat, 28 Aug 2010 14:32:41 -0700 Subject: [PATCH] kernel - unwind kthread_create() mplock * All kthread_create*() calls and kproc_start() calls now create threads which do not hold the mplock at startup. * Add get_mplock()/rel_mplock() to threads which are not yet mpsafe. * Remove rel_mplock() calls from thread startups which were making themselves mpsafe by releasing the mplock. * Kernel eventhandler API is now MPSAFE * Kernel kproc API is now MPSAFE * Rename a few thread procedures to make their function more obvious. --- sys/bus/cam/cam_xpt.c | 4 ++ sys/bus/usb/usb.c | 7 +++- sys/dev/acpica5/Osd/OsdSchedule.c | 4 +- sys/dev/acpica5/acpi_thermal.c | 7 +++- sys/dev/disk/ahci/ahci_dragonfly.c | 4 +- sys/dev/disk/aic7xxx/aic79xx_osm.h | 2 + sys/dev/disk/aic7xxx/aic7xxx_osm.h | 2 + sys/dev/disk/aic7xxx/aic_osm_lib.c | 6 ++- sys/dev/disk/iscsi/initiator/isc_sm.c | 6 ++- sys/dev/disk/iscsi/initiator/isc_soc.c | 8 +++- sys/dev/disk/isp/isp_freebsd.c | 2 + sys/dev/disk/isp/isp_freebsd.h | 2 + sys/dev/disk/mmcsd/mmcsd.c | 12 ++++-- sys/dev/disk/mpt/mpt.h | 15 +++----- sys/dev/disk/mpt/mpt_cam.c | 5 ++- sys/dev/disk/mpt/mpt_raid.c | 5 ++- sys/dev/disk/sili/sili_dragonfly.c | 4 +- sys/dev/pccard/pccbb/pccbb.c | 5 ++- sys/dev/raid/aac/aac.c | 10 +++-- sys/dev/raid/ciss/ciss.c | 9 +++-- sys/dev/video/bktr/msp34xx.c | 14 ++++--- sys/emulation/ndis/kern_ndis.c | 5 ++- sys/emulation/ndis/subr_ntoskrnl.c | 11 ++++-- sys/kern/kern_kthread.c | 33 ++++++++-------- sys/kern/kern_sensors.c | 6 ++- sys/kern/subr_eventhandler.c | 53 +++++++++++++++++--------- sys/kern/vfs_bio.c | 10 +++-- sys/kern/vfs_lock.c | 2 +- sys/kern/vfs_mount.c | 7 +++- sys/kern/vfs_sync.c | 25 +++++++----- sys/net/pf/pf.c | 4 ++ sys/net/pf/pf_ioctl.c | 16 +++----- sys/netproto/smb/smb_iod.c | 7 +++- sys/netproto/smb/smb_subr.c | 10 +++-- sys/netproto/smb/smb_subr.h | 6 +-- sys/opencrypto/crypto.c | 8 +++- sys/vm/vm_pageout.c | 20 +++++----- sys/vm/vm_swapcache.c | 20 +++++----- sys/vm/vm_zeroidle.c | 11 +++--- 39 files changed, 249 insertions(+), 138 deletions(-) diff --git a/sys/bus/cam/cam_xpt.c b/sys/bus/cam/cam_xpt.c index 982140d76d..56bf705fa2 100644 --- a/sys/bus/cam/cam_xpt.c +++ b/sys/bus/cam/cam_xpt.c @@ -1399,6 +1399,8 @@ xpt_scanner_thread(void *dummy) union ccb *ccb; struct cam_sim *sim; + get_mplock(); + for (;;) { xpt_lock_buses(); xsoftc.ccb_scanq_running = 1; @@ -1419,6 +1421,8 @@ xpt_scanner_thread(void *dummy) xpt_unlock_buses(); tsleep(&xsoftc.ccb_scanq, PINTERLOCKED, "ccb_scanq", 0); } + + rel_mplock(); /* not reached */ } /* diff --git a/sys/bus/usb/usb.c b/sys/bus/usb/usb.c index 185375b79f..2e98bde164 100644 --- a/sys/bus/usb/usb.c +++ b/sys/bus/usb/usb.c @@ -73,7 +73,9 @@ #include #include #include + #include +#include #include #include @@ -443,6 +445,7 @@ usb_event_thread(void *arg) */ usb_delay_ms(sc->sc_bus, 500); + get_mplock(); crit_enter(); /* Make sure first discover does something. */ @@ -465,12 +468,12 @@ usb_event_thread(void *arg) sc->sc_event_thread = NULL; crit_exit(); + rel_mplock(); /* In case parent is waiting for us to exit. */ wakeup(sc); DPRINTF(("usb_event_thread: exit\n")); - kthread_exit(); } void @@ -479,6 +482,7 @@ usb_task_thread(void *arg) struct usb_task *task; struct usb_taskq *taskq; + get_mplock(); crit_enter(); taskq = arg; @@ -502,6 +506,7 @@ usb_task_thread(void *arg) } crit_exit(); + rel_mplock(); taskq->taskcreated = 0; wakeup(&taskq->taskcreated); diff --git a/sys/dev/acpica5/Osd/OsdSchedule.c b/sys/dev/acpica5/Osd/OsdSchedule.c index f0a32f327f..3a43c79efa 100644 --- a/sys/dev/acpica5/Osd/OsdSchedule.c +++ b/sys/dev/acpica5/Osd/OsdSchedule.c @@ -47,6 +47,7 @@ #include #include +#include #include "acpi.h" #include "accommon.h" @@ -96,13 +97,14 @@ acpi_task_thread(void *arg) ACPI_OSD_EXEC_CALLBACK func; struct acpi_task *at; + get_mplock(); for (;;) { at = (void *)lwkt_waitport(&curthread->td_msgport, 0); func = (ACPI_OSD_EXEC_CALLBACK)at->at_function; func((void *)at->at_context); lwkt_replymsg(&at->at_msg, 0); } - kthread_exit(); + rel_mplock(); } /* diff --git a/sys/dev/acpica5/acpi_thermal.c b/sys/dev/acpica5/acpi_thermal.c index 4fd68934f8..3e2ab7fcbd 100644 --- a/sys/dev/acpica5/acpi_thermal.c +++ b/sys/dev/acpica5/acpi_thermal.c @@ -42,6 +42,8 @@ #include #include +#include + #include "acpi.h" #include "accommon.h" @@ -949,6 +951,7 @@ acpi_tz_thread(void *arg) devs = NULL; devcount = 0; sc = NULL; + get_mplock(); for (;;) { /* If the number of devices has changed, re-evaluate. */ @@ -991,6 +994,7 @@ acpi_tz_thread(void *arg) ACPI_UNLOCK(thermal); } } + rel_mplock(); } #ifdef __FreeBSD__ @@ -1132,6 +1136,7 @@ acpi_tz_cooling_thread(void *arg) ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__); sc = (struct acpi_tz_softc *)arg; + get_mplock(); prev_temp = sc->tz_temperature; while (sc->tz_cooling_enabled) { @@ -1175,7 +1180,7 @@ acpi_tz_cooling_thread(void *arg) ACPI_LOCK(thermal); sc->tz_cooling_proc_running = FALSE; ACPI_UNLOCK(thermal); - kthread_exit(); + rel_mplock(); } /* diff --git a/sys/dev/disk/ahci/ahci_dragonfly.c b/sys/dev/disk/ahci/ahci_dragonfly.c index fc8b6fc3ed..c6e087a7c0 100644 --- a/sys/dev/disk/ahci/ahci_dragonfly.c +++ b/sys/dev/disk/ahci/ahci_dragonfly.c @@ -347,6 +347,8 @@ ahci_os_unlock_port(struct ahci_port *ap) * Per-port thread helper. This helper thread is responsible for * atomically retrieving and clearing the signal mask and calling * the machine-independant driver core. + * + * MPSAFE */ static void @@ -355,8 +357,6 @@ ahci_port_thread(void *arg) struct ahci_port *ap = arg; int mask; - rel_mplock(); - /* * The helper thread is responsible for the initial port init, * so all the ports can be inited in parallel. diff --git a/sys/dev/disk/aic7xxx/aic79xx_osm.h b/sys/dev/disk/aic7xxx/aic79xx_osm.h index 73541879c6..52ed6fd8eb 100644 --- a/sys/dev/disk/aic7xxx/aic79xx_osm.h +++ b/sys/dev/disk/aic7xxx/aic79xx_osm.h @@ -51,7 +51,9 @@ #include #include #include + #include +#include #define AIC_PCI_CONFIG 1 #include diff --git a/sys/dev/disk/aic7xxx/aic7xxx_osm.h b/sys/dev/disk/aic7xxx/aic7xxx_osm.h index 77d6cba6fa..fa0f00f4e5 100644 --- a/sys/dev/disk/aic7xxx/aic7xxx_osm.h +++ b/sys/dev/disk/aic7xxx/aic7xxx_osm.h @@ -51,7 +51,9 @@ #include #include #include + #include +#include #if defined(__DragonFly__) || __FreeBSD_version < 500000 #include diff --git a/sys/dev/disk/aic7xxx/aic_osm_lib.c b/sys/dev/disk/aic7xxx/aic_osm_lib.c index d3268d52ac..e32a3138ed 100644 --- a/sys/dev/disk/aic7xxx/aic_osm_lib.c +++ b/sys/dev/disk/aic7xxx/aic_osm_lib.c @@ -112,8 +112,11 @@ aic_recovery_thread(void *arg) { struct aic_softc *aic; + get_mplock(); + aic = (struct aic_softc *)arg; aic_lock(aic); + for (;;) { if (LIST_EMPTY(&aic->timedout_scbs) != 0 @@ -132,5 +135,6 @@ aic_recovery_thread(void *arg) aic->platform_data->recovery_thread = NULL; wakeup(aic->platform_data); aic_unlock(aic); - kthread_exit(); + + rel_mplock(); } diff --git a/sys/dev/disk/iscsi/initiator/isc_sm.c b/sys/dev/disk/iscsi/initiator/isc_sm.c index 4ab7bebfa1..a819f66fa9 100644 --- a/sys/dev/disk/iscsi/initiator/isc_sm.c +++ b/sys/dev/disk/iscsi/initiator/isc_sm.c @@ -54,7 +54,10 @@ #include #include #include + +#include #include +#include #include #include @@ -570,6 +573,7 @@ ism_proc(void *vp) isc_session_t *sp = (isc_session_t *)vp; int error; + get_mplock(); debug_called(8); sdebug(3, "started sp->flags=%x", sp->flags); @@ -604,7 +608,7 @@ ism_proc(void *vp) debug(3, "terminated sp=%p sp->sid=%d", sp, sp->sid); - kthread_exit(); + rel_mplock(); } #if 0 diff --git a/sys/dev/disk/iscsi/initiator/isc_soc.c b/sys/dev/disk/iscsi/initiator/isc_soc.c index df117d9a56..448d41fafa 100644 --- a/sys/dev/disk/iscsi/initiator/isc_soc.c +++ b/sys/dev/disk/iscsi/initiator/isc_soc.c @@ -55,9 +55,12 @@ #include #include #include -#include #include +#include +#include +#include + #include #include @@ -578,6 +581,7 @@ isc_soc(void *vp) struct socket *so = sp->soc; int error; + get_mplock(); debug_called(8); sp->td = curthread; @@ -631,7 +635,7 @@ isc_soc(void *vp) sdebug(2, "dropped ISC_CON_RUNNING"); - kthread_exit(); + rel_mplock(); } void diff --git a/sys/dev/disk/isp/isp_freebsd.c b/sys/dev/disk/isp/isp_freebsd.c index fd3f7dbe2e..5271808707 100644 --- a/sys/dev/disk/isp/isp_freebsd.c +++ b/sys/dev/disk/isp/isp_freebsd.c @@ -1917,6 +1917,7 @@ isp_kthread(void *arg) { struct ispsoftc *isp = arg; + get_mplock(); crit_enter(); isp->isp_osinfo.intsok = 1; @@ -1958,6 +1959,7 @@ isp_kthread(void *arg) tsleep(&isp->isp_osinfo.kthread, 0, "isp_fc_worker", 0); isp_prt(isp, ISP_LOGDEBUG0, "kthread: waiting until called"); } + rel_mplock(); } static void diff --git a/sys/dev/disk/isp/isp_freebsd.h b/sys/dev/disk/isp/isp_freebsd.h index f68afd5462..21031e5464 100644 --- a/sys/dev/disk/isp/isp_freebsd.h +++ b/sys/dev/disk/isp/isp_freebsd.h @@ -38,7 +38,9 @@ #include #include #include + #include +#include #include #include diff --git a/sys/dev/disk/mmcsd/mmcsd.c b/sys/dev/disk/mmcsd/mmcsd.c index b3f02e1839..48bc6d3609 100644 --- a/sys/dev/disk/mmcsd/mmcsd.c +++ b/sys/dev/disk/mmcsd/mmcsd.c @@ -55,7 +55,6 @@ #include #include #include -#include #include #include #include @@ -68,6 +67,9 @@ #include #include +#include +#include + #include #include @@ -184,7 +186,7 @@ mmcsd_attach(device_t dev) sc->running = 1; sc->suspend = 0; sc->eblock = sc->eend = 0; - kthread_create(&mmcsd_task, sc, &sc->td, "mmc/sd card task"); + kthread_create(mmcsd_task, sc, &sc->td, "mmc/sd card task"); return (0); } @@ -259,7 +261,7 @@ mmcsd_resume(device_t dev) if (sc->running <= 0) { sc->running = 1; MMCSD_UNLOCK(sc); - kthread_create(&mmcsd_task, sc, &sc->td, "mmc/sd card task"); + kthread_create(mmcsd_task, sc, &sc->td, "mmc/sd card task"); } else MMCSD_UNLOCK(sc); return (0); @@ -495,7 +497,9 @@ mmcsd_task(void *arg) daddr_t block, end; device_t dev; + get_mplock(); dev = sc->dev; + while (1) { MMCSD_LOCK(sc); do { @@ -543,7 +547,7 @@ out: MMCSD_UNLOCK(sc); wakeup(sc); - kthread_exit(); + rel_mplock(); } static const char * diff --git a/sys/dev/disk/mpt/mpt.h b/sys/dev/disk/mpt/mpt.h index 3a22e7e368..1df6a7170e 100644 --- a/sys/dev/disk/mpt/mpt.h +++ b/sys/dev/disk/mpt/mpt.h @@ -105,24 +105,19 @@ #include #include #include -#if __FreeBSD_version < 500000 + #include #include #include #include -#include -#else -#include -#include -#include -#include -#include -#include -#endif + #include #include #include +#include +#include + #include #if __FreeBSD_version < 500000 diff --git a/sys/dev/disk/mpt/mpt_cam.c b/sys/dev/disk/mpt/mpt_cam.c index b7983774dc..92f78bb22d 100644 --- a/sys/dev/disk/mpt/mpt_cam.c +++ b/sys/dev/disk/mpt/mpt_cam.c @@ -3985,7 +3985,10 @@ mpt_recovery_thread(void *arg) struct mpt_softc *mpt; mpt = (struct mpt_softc *)arg; + + get_mplock(); MPT_LOCK(mpt); + for (;;) { if (TAILQ_EMPTY(&mpt->request_timeout_list) != 0) { if (mpt->shutdwn_recovery == 0) { @@ -4000,7 +4003,7 @@ mpt_recovery_thread(void *arg) mpt->recovery_thread = NULL; wakeup(&mpt->recovery_thread); MPT_UNLOCK(mpt); - mpt_kthread_exit(0); + rel_mplock(); } static int diff --git a/sys/dev/disk/mpt/mpt_raid.c b/sys/dev/disk/mpt/mpt_raid.c index 08940071d2..407ec44371 100644 --- a/sys/dev/disk/mpt/mpt_raid.c +++ b/sys/dev/disk/mpt/mpt_raid.c @@ -670,7 +670,10 @@ mpt_raid_thread(void *arg) mpt = (struct mpt_softc *)arg; firstrun = 1; + + get_mplock(); MPT_LOCK(mpt); + while (mpt->shutdwn_raid == 0) { if (mpt->raid_wakeup == 0) { @@ -727,7 +730,7 @@ mpt_raid_thread(void *arg) mpt->raid_thread = NULL; wakeup(&mpt->raid_thread); MPT_UNLOCK(mpt); - mpt_kthread_exit(0); + rel_mplock(); } #if 0 diff --git a/sys/dev/disk/sili/sili_dragonfly.c b/sys/dev/disk/sili/sili_dragonfly.c index e2641b69a5..f437265028 100644 --- a/sys/dev/disk/sili/sili_dragonfly.c +++ b/sys/dev/disk/sili/sili_dragonfly.c @@ -269,6 +269,8 @@ sili_os_unlock_port(struct sili_port *ap) * Per-port thread helper. This helper thread is responsible for * atomically retrieving and clearing the signal mask and calling * the machine-independant driver core. + * + * MPSAFE */ static void @@ -277,8 +279,6 @@ sili_port_thread(void *arg) struct sili_port *ap = arg; int mask; - rel_mplock(); - /* * The helper thread is responsible for the initial port init, * so all the ports can be inited in parallel. diff --git a/sys/dev/pccard/pccbb/pccbb.c b/sys/dev/pccard/pccbb/pccbb.c index c5e5d3b442..cb0d2d4e15 100644 --- a/sys/dev/pccard/pccbb/pccbb.c +++ b/sys/dev/pccard/pccbb/pccbb.c @@ -89,7 +89,9 @@ #include #include #include + #include +#include #include #include @@ -437,6 +439,7 @@ cbb_event_thread(void *arg) int err; int not_a_card = 0; + get_mplock(); sc->flags |= CBB_KTHREAD_RUNNING; while ((sc->flags & CBB_KTHREAD_DONE) == 0) { /* @@ -494,7 +497,7 @@ cbb_event_thread(void *arg) } sc->flags &= ~CBB_KTHREAD_RUNNING; wakeup(sc->event_thread); - kthread_exit(); + rel_mplock(); } /************************************************************************/ diff --git a/sys/dev/raid/aac/aac.c b/sys/dev/raid/aac/aac.c index b8205d82fc..cd940b39ec 100644 --- a/sys/dev/raid/aac/aac.c +++ b/sys/dev/raid/aac/aac.c @@ -79,7 +79,7 @@ static void aac_complete(void *context, int pending); static int aac_bio_command(struct aac_softc *sc, struct aac_command **cmp); static void aac_bio_complete(struct aac_command *cm); static int aac_wait_command(struct aac_command *cm); -static void aac_command_thread(struct aac_softc *sc); +static void aac_command_thread(void *arg); /* Command Buffer Management */ static void aac_map_command_sg(void *arg, bus_dma_segment_t *segs, @@ -360,7 +360,7 @@ aac_attach(struct aac_softc *sc) reference_dev(sc->aac_dev_t); /* Create the AIF thread */ - if (kthread_create((void(*)(void *))aac_command_thread, sc, + if (kthread_create(aac_command_thread, sc, &sc->aifthread, "aac%daif", unit)) panic("Could not create AIF thread\n"); @@ -909,12 +909,14 @@ aac_map_command(struct aac_command *cm) * Handle notification of one or more FIBs coming from the controller. */ static void -aac_command_thread(struct aac_softc *sc) +aac_command_thread(void *arg) { + struct aac_softc *sc = arg; struct aac_fib *fib; u_int32_t fib_size; int size, retval; + get_mplock(); debug_called(2); AAC_LOCK_ACQUIRE(&sc->aac_io_lock); @@ -1005,7 +1007,7 @@ aac_command_thread(struct aac_softc *sc) AAC_LOCK_RELEASE(&sc->aac_io_lock); wakeup(sc->aac_dev); - kthread_exit(); + rel_mplock(); } /* diff --git a/sys/dev/raid/ciss/ciss.c b/sys/dev/raid/ciss/ciss.c index e42b0fe262..9afc43423c 100644 --- a/sys/dev/raid/ciss/ciss.c +++ b/sys/dev/raid/ciss/ciss.c @@ -88,6 +88,8 @@ #include #include +#include + #include #include #include @@ -3541,7 +3543,9 @@ ciss_notify_thread(void *arg) sc = (struct ciss_softc *)arg; + get_mplock(); crit_enter(); + for (;;) { if (TAILQ_EMPTY(&sc->ciss_notify) != 0 && (sc->ciss_flags & CISS_FLAG_THREAD_SHUT) == 0) { @@ -3577,8 +3581,7 @@ ciss_notify_thread(void *arg) sc->ciss_notify_thread = NULL; wakeup(&sc->ciss_notify_thread); crit_exit(); - - kthread_exit(); + rel_mplock(); } /************************************************************************ @@ -3587,7 +3590,7 @@ ciss_notify_thread(void *arg) static void ciss_spawn_notify_thread(struct ciss_softc *sc) { - if (kthread_create((void(*)(void *))ciss_notify_thread, sc, + if (kthread_create(ciss_notify_thread, sc, &sc->ciss_notify_thread, "ciss_notify%d", device_get_unit(sc->ciss_dev))) panic("Could not create notify thread\n"); diff --git a/sys/dev/video/bktr/msp34xx.c b/sys/dev/video/bktr/msp34xx.c index da3b410844..d294e33a60 100644 --- a/sys/dev/video/bktr/msp34xx.c +++ b/sys/dev/video/bktr/msp34xx.c @@ -86,6 +86,8 @@ #include #include +#include + #include #include /* extensions to ioctl_meteor.h */ #include @@ -692,13 +694,15 @@ static void watch_stereo(bktr_ptr_t client) } -static void msp3400c_thread(void *data) +static void +msp3400c_thread(void *data) { bktr_ptr_t client = data; struct msp3400c *msp = (struct msp3400c*)client->msp3400c_info; - struct CARRIER_DETECT *cd; int count, max1,max2,val1,val2, val,this; + + get_mplock(); dprintk(("msp3400: thread started\n")); @@ -894,8 +898,7 @@ done: msp->kthread = NULL; wakeup(&msp->kthread); - - kthread_exit(); + rel_mplock(); } /* ----------------------------------------------------------------------- */ @@ -936,6 +939,7 @@ static void msp3410d_thread(void *data) int mode,val,i,std; int timo = 0; + get_mplock(); dprintk(("msp3410: thread started\n")); for (;;) { @@ -1121,7 +1125,7 @@ done: msp->kthread = NULL; wakeup(&msp->kthread); - kthread_exit(); + rel_mplock(); } int msp_attach(bktr_ptr_t bktr) diff --git a/sys/emulation/ndis/kern_ndis.c b/sys/emulation/ndis/kern_ndis.c index 777812549d..88da46330e 100644 --- a/sys/emulation/ndis/kern_ndis.c +++ b/sys/emulation/ndis/kern_ndis.c @@ -53,6 +53,8 @@ #include #include +#include + #include #include #include @@ -199,6 +201,7 @@ ndis_runq(void *arg) struct ndisproc *p; p = arg; + get_mplock(); while (1) { @@ -237,7 +240,7 @@ ndis_runq(void *arg) } wakeup(die); - kthread_exit(); + rel_mplock(); } static int diff --git a/sys/emulation/ndis/subr_ntoskrnl.c b/sys/emulation/ndis/subr_ntoskrnl.c index 8c69c14e89..6795b70979 100644 --- a/sys/emulation/ndis/subr_ntoskrnl.c +++ b/sys/emulation/ndis/subr_ntoskrnl.c @@ -52,6 +52,8 @@ #include #include +#include + #include #include #include @@ -1382,15 +1384,16 @@ ntoskrnl_thrfunc(void *arg) void *tctx; uint32_t rval; + get_mplock(); + thrctx = arg; tfunc = thrctx->tc_thrfunc; tctx = thrctx->tc_thrctx; kfree(thrctx, M_TEMP); rval = tfunc(tctx); - ntoskrnl_thread_exit(rval); - return; /* notreached */ + /* not reached */ } __stdcall static ndis_status @@ -1440,7 +1443,9 @@ ntoskrnl_thread_exit(ndis_status status) ntoskrnl_kth--; - kthread_exit(); + rel_mplock(); + kthread_exit(); /* call explicitly */ + return(0); /* notreached */ } diff --git a/sys/kern/kern_kthread.c b/sys/kern/kern_kthread.c index a3aed7fcba..f9f39983f0 100644 --- a/sys/kern/kern_kthread.c +++ b/sys/kern/kern_kthread.c @@ -37,15 +37,18 @@ #include #include +#include + #include +static struct lwkt_token kpsus_token = LWKT_TOKEN_MP_INITIALIZER(kpsus_token); + + /* * Create a kernel process/thread/whatever. It shares it's address space * with proc0 - ie: kernel only. 5.x compatible. * - * NOTE! By default kthreads are created with the MP lock held. A - * thread which does not require the MP lock should release it by calling - * rel_mplock() at the start of the new thread. + * All kthreads are created as MPSAFE threads. */ int kthread_create(void (*func)(void *), void *arg, @@ -54,13 +57,11 @@ kthread_create(void (*func)(void *), void *arg, thread_t td; __va_list ap; - td = lwkt_alloc_thread(NULL, LWKT_THREAD_STACK, -1, TDF_VERBOSE); + td = lwkt_alloc_thread(NULL, LWKT_THREAD_STACK, -1, + TDF_VERBOSE | TDF_MPSAFE); if (tdp) *tdp = td; cpu_set_thread_handler(td, kthread_exit, func, arg); -#ifdef SMP - KKASSERT(td->td_mpcount == 1); -#endif /* * Set up arg0 for 'ps' etc @@ -85,13 +86,11 @@ kthread_create_cpu(void (*func)(void *), void *arg, thread_t td; __va_list ap; - td = lwkt_alloc_thread(NULL, LWKT_THREAD_STACK, cpu, TDF_VERBOSE); + td = lwkt_alloc_thread(NULL, LWKT_THREAD_STACK, cpu, + TDF_VERBOSE | TDF_MPSAFE); if (tdp) *tdp = td; cpu_set_thread_handler(td, kthread_exit, func, arg); -#ifdef SMP - KKASSERT(td->td_mpcount == 1); -#endif /* * Set up arg0 for 'ps' etc @@ -119,13 +118,11 @@ kthread_create_stk(void (*func)(void *), void *arg, thread_t td; __va_list ap; - td = lwkt_alloc_thread(NULL, stksize, -1, TDF_VERBOSE); + td = lwkt_alloc_thread(NULL, stksize, -1, TDF_VERBOSE | TDF_MPSAFE); if (tdp) *tdp = td; cpu_set_thread_handler(td, kthread_exit, func, arg); -#ifdef SMP - KKASSERT(td->td_mpcount == 1); -#endif + __va_start(ap, fmt); kvsnprintf(td->td_comm, sizeof(td->td_comm), fmt, ap); __va_end(ap); @@ -153,6 +150,8 @@ kthread_exit(void) * * This function is used to start "internal" daemons and intended * to be called from SYSINIT(). + * + * These threads are created MPSAFE. */ void kproc_start(const void *udata) @@ -175,6 +174,7 @@ int suspend_kproc(struct thread *td, int timo) { if (td->td_proc == NULL) { + lwkt_gettoken(&kpsus_token); td->td_flags |= TDF_STOPREQ; /* request thread pause */ wakeup(td); while (td->td_flags & TDF_STOPREQ) { @@ -183,6 +183,7 @@ suspend_kproc(struct thread *td, int timo) break; } td->td_flags &= ~TDF_STOPREQ; + lwkt_reltoken(&kpsus_token); return(0); } else { return(EINVAL); /* not a kernel thread */ @@ -195,6 +196,7 @@ kproc_suspend_loop(void) struct thread *td = curthread; if (td->td_flags & TDF_STOPREQ) { + lwkt_gettoken(&kpsus_token); td->td_flags &= ~TDF_STOPREQ; while ((td->td_flags & TDF_WAKEREQ) == 0) { wakeup(td); @@ -202,6 +204,7 @@ kproc_suspend_loop(void) } td->td_flags &= ~TDF_WAKEREQ; wakeup(td); + lwkt_reltoken(&kpsus_token); } } diff --git a/sys/kern/kern_sensors.c b/sys/kern/kern_sensors.c index f151370a11..a9e49ecafb 100644 --- a/sys/kern/kern_sensors.c +++ b/sys/kern/kern_sensors.c @@ -30,6 +30,8 @@ #include #include +#include + int sensordev_count = 0; SLIST_HEAD(, ksensordev) sensordev_list = SLIST_HEAD_INITIALIZER(sensordev_list); @@ -221,6 +223,8 @@ sensor_task_thread(void *arg) struct sensor_task *st, *nst; time_t now; + get_mplock(); + while (!TAILQ_EMPTY(&tasklist)) { while ((nst = TAILQ_FIRST(&tasklist))->nextrun > (now = time_second)) @@ -248,7 +252,7 @@ sensor_task_thread(void *arg) } } - kthread_exit(); + rel_mplock(); } void diff --git a/sys/kern/subr_eventhandler.c b/sys/kern/subr_eventhandler.c index fa9c4849fa..f58ebe7f60 100644 --- a/sys/kern/subr_eventhandler.c +++ b/sys/kern/subr_eventhandler.c @@ -1,4 +1,6 @@ -/*- +/* + * (MPSAFE) + * * Copyright (c) 1999 Michael Smith * All rights reserved. * @@ -33,11 +35,13 @@ #include #include +#include + MALLOC_DEFINE(M_EVENTHANDLER, "eventhandler", "Event handler records"); /* List of 'slow' lists */ -static TAILQ_HEAD(, eventhandler_list) eventhandler_lists; -static int eventhandler_lists_initted = 0; +static TAILQ_HEAD(, eventhandler_list) eventhandler_lists = TAILQ_HEAD_INITIALIZER(eventhandler_lists); +static struct lwkt_token evlist_token = LWKT_TOKEN_MP_INITIALIZER(evlist_token); struct eventhandler_entry_generic { @@ -48,6 +52,8 @@ struct eventhandler_entry_generic /* * Insertion is O(n) due to the priority scan, but optimises to O(1) * if all priorities are identical. + * + * MPSAFE */ eventhandler_tag eventhandler_register(struct eventhandler_list *list, char *name, @@ -56,27 +62,28 @@ eventhandler_register(struct eventhandler_list *list, char *name, struct eventhandler_entry_generic *eg; struct eventhandler_entry *ep; - /* avoid the need for a SYSINIT just to init the list */ - if (!eventhandler_lists_initted) { - TAILQ_INIT(&eventhandler_lists); - eventhandler_lists_initted = 1; - } + lwkt_gettoken(&evlist_token); - /* Do we need to find/create the (slow) list? */ - if (list == NULL) { - /* look for a matching, existing list */ + /* + * find/create the list as needed + */ + while (list == NULL) { list = eventhandler_find_list(name); - - /* Do we need to create the list? */ - if (list == NULL) { - list = kmalloc(sizeof(struct eventhandler_list) + strlen(name) + 1, - M_EVENTHANDLER, M_INTWAIT); + if (list) + break; + list = kmalloc(sizeof(struct eventhandler_list) + strlen(name) + 1, + M_EVENTHANDLER, M_INTWAIT); + if (eventhandler_find_list(name)) { + kfree(list, M_EVENTHANDLER); + list = NULL; + } else { list->el_flags = 0; list->el_name = (char *)list + sizeof(struct eventhandler_list); strcpy(list->el_name, name); TAILQ_INSERT_HEAD(&eventhandler_lists, list, el_link); } } + if (!(list->el_flags & EHE_INITTED)) { TAILQ_INIT(&list->el_entries); list->el_flags = EHE_INITTED; @@ -100,15 +107,22 @@ eventhandler_register(struct eventhandler_list *list, char *name, } if (ep == NULL) TAILQ_INSERT_TAIL(&list->el_entries, &eg->ee, ee_link); + lwkt_reltoken(&evlist_token); + return(&eg->ee); } +/* + * MPSAFE + */ void eventhandler_deregister(struct eventhandler_list *list, eventhandler_tag tag) { struct eventhandler_entry *ep = tag; + lwkt_gettoken(&evlist_token); /* XXX insert diagnostic check here? */ + if (ep != NULL) { /* remove just this entry */ TAILQ_REMOVE(&list->el_entries, ep, ee_link); @@ -121,19 +135,24 @@ eventhandler_deregister(struct eventhandler_list *list, eventhandler_tag tag) kfree(ep, M_EVENTHANDLER); } } + lwkt_reltoken(&evlist_token); } +/* + * Locate the requested list + */ struct eventhandler_list * eventhandler_find_list(char *name) { struct eventhandler_list *list; - /* scan looking for the requested list */ + lwkt_gettoken(&evlist_token); for (list = TAILQ_FIRST(&eventhandler_lists); list != NULL; list = TAILQ_NEXT(list, el_link)) { if (!strcmp(name, list->el_name)) break; } + lwkt_reltoken(&evlist_token); return(list); } diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 2ad6754a6e..fa6bc2d371 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -2413,6 +2413,9 @@ static struct kproc_desc bufhw_kp = { SYSINIT(bufdaemon_hw, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &bufhw_kp) +/* + * MPSAFE thread + */ static void buf_daemon(void) { @@ -2425,8 +2428,6 @@ buf_daemon(void) bufdaemon_td, SHUTDOWN_PRI_LAST); curthread->td_flags |= TDF_SYSTHREAD; - rel_mplock(); - /* * This process is allowed to take the buffer cache to the limit */ @@ -2471,6 +2472,9 @@ buf_daemon(void) } } +/* + * MPSAFE thread + */ static void buf_daemon_hw(void) { @@ -2483,8 +2487,6 @@ buf_daemon_hw(void) bufdaemonhw_td, SHUTDOWN_PRI_LAST); curthread->td_flags |= TDF_SYSTHREAD; - rel_mplock(); - /* * This process is allowed to take the buffer cache to the limit */ diff --git a/sys/kern/vfs_lock.c b/sys/kern/vfs_lock.c index 6982ef50b8..44c1aaca5e 100644 --- a/sys/kern/vfs_lock.c +++ b/sys/kern/vfs_lock.c @@ -57,6 +57,7 @@ #include #include #include +#include static void vnode_terminate(struct vnode *vp); static boolean_t vnode_ctor(void *obj, void *private, int ocflags); @@ -957,4 +958,3 @@ freesomevnodes(int n) } return(count); } - diff --git a/sys/kern/vfs_mount.c b/sys/kern/vfs_mount.c index 7595a63bba..c444503b38 100644 --- a/sys/kern/vfs_mount.c +++ b/sys/kern/vfs_mount.c @@ -92,6 +92,7 @@ #include #include #include +#include #include #include @@ -693,9 +694,11 @@ vnlru_proc(void) int done; EVENTHANDLER_REGISTER(shutdown_pre_sync, shutdown_kproc, td, - SHUTDOWN_PRI_FIRST); + SHUTDOWN_PRI_FIRST); + get_mplock(); crit_enter(); + for (;;) { kproc_suspend_loop(); @@ -757,7 +760,9 @@ vnlru_proc(void) vnlru_nowhere = 0; } } + crit_exit(); + rel_mplock(); } /* diff --git a/sys/kern/vfs_sync.c b/sys/kern/vfs_sync.c index c094e9ace4..3cae967785 100644 --- a/sys/kern/vfs_sync.c +++ b/sys/kern/vfs_sync.c @@ -82,6 +82,7 @@ #include #include +#include /* * The workitem queue. @@ -197,19 +198,14 @@ vn_syncer_remove(struct vnode *vp) } struct thread *updatethread; -static void sched_sync (void); -static struct kproc_desc up_kp = { - "syncer", - sched_sync, - &updatethread -}; -SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp) /* * System filesystem synchronizer daemon. + * + * NOTE: Started MPSAFE but is not yet mpsafe. */ -void -sched_sync(void) +static void +syncer_thread(void) { struct thread *td = curthread; struct synclist *slp; @@ -217,7 +213,8 @@ sched_sync(void) long starttime; EVENTHANDLER_REGISTER(shutdown_pre_sync, shutdown_kproc, td, - SHUTDOWN_PRI_LAST); + SHUTDOWN_PRI_LAST); + get_mplock(); for (;;) { kproc_suspend_loop(); @@ -296,8 +293,16 @@ sched_sync(void) if (time_second == starttime) tsleep(&lbolt_syncer, 0, "syncer", 0); } + rel_mplock(); } +static struct kproc_desc up_kp = { + "syncer", + syncer_thread, + &updatethread +}; +SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp) + /* * Request the syncer daemon to speed up its work. * We never push it to speed up more than half of its diff --git a/sys/net/pf/pf.c b/sys/net/pf/pf.c index d600b47edb..4c78941b48 100644 --- a/sys/net/pf/pf.c +++ b/sys/net/pf/pf.c @@ -60,6 +60,8 @@ #include #include +#include + #include #include @@ -889,6 +891,7 @@ pf_purge_thread(void *v) int nloops = 0; int locked = 0; + get_mplock(); for (;;) { tsleep(pf_purge_thread, PWAIT, "pftm", 1 * hz); @@ -925,6 +928,7 @@ pf_purge_thread(void *v) crit_exit(); lockmgr(&pf_consistency_lock, LK_RELEASE); } + rel_mplock(); } u_int32_t diff --git a/sys/net/pf/pf_ioctl.c b/sys/net/pf/pf_ioctl.c index 9a6d5e5cc3..8bfcd6e789 100644 --- a/sys/net/pf/pf_ioctl.c +++ b/sys/net/pf/pf_ioctl.c @@ -54,7 +54,6 @@ #include #include #include -#include #include #include #include @@ -62,6 +61,9 @@ #include #include +#include +#include + #include #include #include @@ -103,7 +105,6 @@ u_int rt_numfibs = RT_NUMFIBS; void init_zone_var(void); void cleanup_pf_zone(void); int pfattach(void); -void pf_thread_create(void *); struct pf_pool *pf_get_pool(char *, u_int32_t, u_int8_t, u_int32_t, u_int8_t, u_int8_t, u_int8_t); @@ -315,17 +316,10 @@ pfattach(void) /* XXX do our best to avoid a conflict */ pf_status.hostid = karc4random(); - /* require process context to purge states, so perform in a thread */ - kthread_create(pf_thread_create, NULL, NULL, "dummy"); - - return (error); -} - -void -pf_thread_create(void *v) -{ if (kthread_create(pf_purge_thread, NULL, NULL, "pfpurge")) panic("pfpurge thread"); + + return (error); } int diff --git a/sys/netproto/smb/smb_iod.c b/sys/netproto/smb/smb_iod.c index 6a509b305e..1e73ee5da7 100644 --- a/sys/netproto/smb/smb_iod.c +++ b/sys/netproto/smb/smb_iod.c @@ -42,6 +42,8 @@ #include #include +#include + #include "smb.h" #include "smb_conn.h" #include "smb_rq.h" @@ -637,14 +639,15 @@ smb_iod_main(struct smbiod *iod) return; } -#define kthread_create_compat kthread_create2 -#define kthread_exit_compat kthread_exit2 +#define kthread_create_compat smb_kthread_create +#define kthread_exit_compat smb_kthread_exit void smb_iod_thread(void *arg) { struct smbiod *iod = arg; + ASSERT_MP_LOCK_HELD(curthread); smb_makescred(&iod->iod_scred, iod->iod_td, NULL); while ((iod->iod_flags & SMBIOD_SHUTDOWN) == 0) { smb_iod_main(iod); diff --git a/sys/netproto/smb/smb_subr.c b/sys/netproto/smb/smb_subr.c index 5170a74b1a..f36c7b82cc 100644 --- a/sys/netproto/smb/smb_subr.c +++ b/sys/netproto/smb/smb_subr.c @@ -44,10 +44,12 @@ #include #include #include -#include #include #include +#include +#include + #include #include @@ -367,8 +369,8 @@ smb_put_asunistring(struct smb_rq *rqp, const char *src) * pure thread when possible. */ int -kthread_create2(void (*func)(void *), void *arg, - struct proc **newpp, int flags, const char *fmt, ...) +smb_kthread_create(void (*func)(void *), void *arg, + struct proc **newpp, int flags, const char *fmt, ...) { int error; __va_list ap; @@ -405,7 +407,7 @@ kthread_create2(void (*func)(void *), void *arg, } void -kthread_exit2(void) +smb_kthread_exit(void) { exit1(0); } diff --git a/sys/netproto/smb/smb_subr.h b/sys/netproto/smb/smb_subr.h index c79833c84c..9a13c86443 100644 --- a/sys/netproto/smb/smb_subr.h +++ b/sys/netproto/smb/smb_subr.h @@ -168,9 +168,9 @@ int smb_put_asunistring(struct smb_rq *rqp, const char *src); /* * Compatibilty with 5.x functions */ -int kthread_create2(void (*func)(void *), void *arg, - struct proc **newpp, int flags, const char *fmt, ...); -void kthread_exit2(void); +int smb_kthread_create(void (*func)(void *), void *arg, + struct proc **newpp, int flags, const char *fmt, ...); +void smb_kthread_exit(void); int smb_sleep(void *chan, struct smb_slock *sl, int slpflags, const char *wmesg, int timo); diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c index 7e34357c80..79fe3dbb98 100644 --- a/sys/opencrypto/crypto.c +++ b/sys/opencrypto/crypto.c @@ -66,6 +66,7 @@ #include #include #include + #include #include @@ -1292,6 +1293,8 @@ crypto_finis(void *chan) /* * Crypto thread, dispatches crypto requests. + * + * MPSAFE */ static void crypto_proc(void *arg) @@ -1303,8 +1306,6 @@ crypto_proc(void *arg) u_int32_t hid; int result, hint; - rel_mplock(); /* release the mplock held on startup */ - CRYPTO_Q_LOCK(tdinfo); curthread->td_flags |= TDF_CRYPTO; @@ -1458,6 +1459,8 @@ crypto_proc(void *arg) * Crypto returns thread, does callbacks for processed crypto requests. * Callbacks are done here, rather than in the crypto drivers, because * callbacks typically are expensive and would slow interrupt handling. + * + * MPSAFE */ static void crypto_ret_proc(void *dummy __unused) @@ -1465,6 +1468,7 @@ crypto_ret_proc(void *dummy __unused) struct cryptop *crpt; struct cryptkop *krpt; + get_mplock(); CRYPTO_RETQ_LOCK(); for (;;) { /* Harvest return q's for completed ops */ diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 32a57e1f61..c68ef8a907 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -99,6 +99,7 @@ #include #include +#include #include /* @@ -106,19 +107,11 @@ */ /* the kernel process "vm_pageout"*/ -static void vm_pageout (void); static int vm_pageout_clean (vm_page_t); static int vm_pageout_scan (int pass); static int vm_pageout_free_page_calc (vm_size_t count); struct thread *pagethread; -static struct kproc_desc page_kp = { - "pagedaemon", - vm_pageout, - &pagethread -}; -SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, kproc_start, &page_kp) - #if !defined(NO_SWAPPING) /* the kernel process "vm_daemon"*/ static void vm_daemon (void); @@ -1482,7 +1475,7 @@ vm_pageout_free_page_calc(vm_size_t count) * No requirements. */ static void -vm_pageout(void) +vm_pageout_thread(void) { int pass; int inactive_shortage; @@ -1490,6 +1483,7 @@ vm_pageout(void) /* * Permanently hold vm_token. */ + get_mplock(); lwkt_gettoken(&vm_token); /* @@ -1657,6 +1651,14 @@ vm_pageout(void) } } +static struct kproc_desc page_kp = { + "pagedaemon", + vm_pageout_thread, + &pagethread +}; +SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, kproc_start, &page_kp) + + /* * Called after allocating a page out of the cache or free queue * to possibly wake the pagedaemon up to replentish our supply. diff --git a/sys/vm/vm_swapcache.c b/sys/vm/vm_swapcache.c index 0b492538c5..235b40b74f 100644 --- a/sys/vm/vm_swapcache.c +++ b/sys/vm/vm_swapcache.c @@ -75,25 +75,18 @@ #include #include +#include #include #define INACTIVE_LIST (&vm_page_queues[PQ_INACTIVE].pl) /* the kernel process "vm_pageout"*/ -static void vm_swapcached (void); static int vm_swapcached_flush (vm_page_t m, int isblkdev); static int vm_swapcache_test(vm_page_t m); static void vm_swapcache_writing(vm_page_t marker); static void vm_swapcache_cleaning(vm_object_t marker); struct thread *swapcached_thread; -static struct kproc_desc swpc_kp = { - "swapcached", - vm_swapcached, - &swapcached_thread -}; -SYSINIT(swapcached, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start, &swpc_kp) - SYSCTL_NODE(_vm, OID_AUTO, swapcache, CTLFLAG_RW, NULL, NULL); int vm_swapcache_read_enable; @@ -150,7 +143,7 @@ SYSCTL_QUAD(_vm_swapcache, OID_AUTO, write_count, * No requirements. */ static void -vm_swapcached(void) +vm_swapcached_thread(void) { enum { SWAPC_WRITING, SWAPC_CLEANING } state = SWAPC_WRITING; enum { SWAPB_BURSTING, SWAPB_RECOVERING } burst = SWAPB_BURSTING; @@ -161,6 +154,8 @@ vm_swapcached(void) * Thread setup */ curthread->td_flags |= TDF_SYSTHREAD; + + get_mplock(); crit_enter(); lwkt_gettoken(&vm_token); @@ -254,6 +249,13 @@ vm_swapcached(void) crit_exit(); } +static struct kproc_desc swpc_kp = { + "swapcached", + vm_swapcached_thread, + &swapcached_thread +}; +SYSINIT(swapcached, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start, &swpc_kp) + /* * The caller must hold vm_token. */ diff --git a/sys/vm/vm_zeroidle.c b/sys/vm/vm_zeroidle.c index f3442c80c3..3d181a3a96 100644 --- a/sys/vm/vm_zeroidle.c +++ b/sys/vm/vm_zeroidle.c @@ -47,14 +47,15 @@ #include #include #include -#include #include -#include #include #include #include #include +#include +#include + /* * Implement the pre-zeroed page mechanism. */ @@ -142,6 +143,9 @@ vm_page_zero_time(void) return (DEFAULT_SLEEP_TIME); } +/* + * MPSAFE thread + */ static void vm_pagezero(void __unused *arg) { @@ -153,9 +157,6 @@ vm_pagezero(void __unused *arg) int sleep_time; int i = 0; - /* MPSAFE thread */ - rel_mplock(); - /* * Adjust thread parameters before entering our loop. The thread * is started with the MP lock held and with normal kernel thread -- 2.41.0