#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/spinlock.h>
+
#include <sys/thread2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
#include "cam.h"
#include "cam_ccb.h"
#include <sys/thread.h>
#include <sys/lock.h>
#include <sys/spinlock.h>
+
#include <sys/thread2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
#include <machine/clock.h>
#include <machine/stdarg.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <sys/proc.h>
-#include <sys/buf2.h>
#include <sys/camlib.h>
+
+#include <sys/buf2.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include "../cam.h"
#include "../cam_ccb.h"
#include <sys/bootmaj.h>
#ifdef _KERNEL
+
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/buf.h>
#include <sys/malloc.h>
#include <sys/cons.h>
#include <sys/proc.h>
+
#include <sys/buf2.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
+
#endif /* _KERNEL */
#ifdef _KERNEL
kern/kern_memio.c standard
kern/kern_upcall.c standard
kern/kern_sfbuf.c standard
+kern/kern_mplock.c standard
kern/kern_msfbuf.c standard
kern/kern_subr.c standard
kern/kern_iosched.c standard
#include <bus/pci/pcidevs.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#define AHCI_CDEV_MAJOR 188
#include <sys/nata.h>
#include <sys/objcache.h>
#include <sys/queue.h>
-#include <sys/spinlock2.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
+#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
+
#include "ata-all.h"
#include "ata_if.h"
#include <sys/queue.h>
#include <sys/rman.h>
#include <sys/spinlock.h>
-#include <sys/spinlock2.h>
#include <sys/systm.h>
#include <sys/taskqueue.h>
+#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
+
#include <machine/bus_dma.h>
#include <bus/pci/pcireg.h>
#include <sys/module.h>
#include <sys/nata.h>
#include <sys/spinlock.h>
-#include <sys/spinlock2.h>
#include <sys/queue.h>
#include <sys/systm.h>
+
#include <sys/thread2.h>
+#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
#include <bus/cam/cam.h>
#include <bus/cam/cam_ccb.h>
#include <bus/pci/pcidevs.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#define SILI_CDEV_MAJOR 188
#include <sys/eventhandler.h>
#include <sys/rman.h>
+#include <sys/mplock2.h>
+
#include <bus/pci/pcireg.h>
#include <bus/pci/pcivar.h>
#include <machine/md_var.h>
+#include <sys/mplock2.h>
+
#include <vm/vm.h>
#include <vm/pmap.h>
#include <sys/nlookup.h>
#include <sys/vnode.h>
+#include <sys/mplock2.h>
+
#include <vfs/union/union.h>
/*
#include <sys/priv.h>
#include <sys/socket.h>
#include <sys/sysctl.h>
+
#include <vm/vm_param.h>
+#include <sys/mplock2.h>
+
/*
* MPALMOSTSAFE
*/
#include <sys/proc.h>
#include <sys/signal.h>
#include <sys/signalvar.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#define ONSIG 32 /* NSIG for osig* syscalls. XXX. */
#include <sys/socketvar.h>
#include <sys/uio.h>
+#include <sys/mplock2.h>
+
#include "43bsd_socket.h"
/*
#include <sys/namei.h>
#include <sys/nlookup.h>
+#include <sys/mplock2.h>
+
#include <emulation/43bsd/stat.h>
static int
#include <sys/kern_syscall.h>
#include <sys/mman.h>
#include <sys/proc.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
/*
* MPSAFE
#include <sys/sysproto.h>
#include <sys/uio.h>
+#include <sys/mplock2.h>
+
#define PADDED_SIZE(x) \
((sizeof(struct dfbsd12_dirent) + (x) + 1 + 3) & ~3)
#define MAX_NAMELEN 255
#include <sys/vnode.h>
#include <emulation/dragonfly12/stat.h>
+#include <sys/mplock2.h>
+
static void
cvtstat(struct dfbsd12_stat *oldstat, struct stat *newstat)
{
#include <vm/pmap.h>
#include <vm/vm_map.h>
+#include <sys/mplock2.h>
+
#include "linux.h"
#include "linux_proto.h"
#include "../linux_ipc.h"
sys_linux_clone(struct linux_clone_args *args)
{
int error, ff = RFPROC;
- struct proc *p2;
+ struct proc *p2 = NULL;
int exit_signal;
vm_offset_t start;
struct rfork_args rf_args;
#include <sys/user.h>
#include <sys/reg.h>
+#include <sys/mplock2.h>
+
#include <machine/md_var.h>
#include <machine/pcb.h>
#include <vfs/ufs/ufsmount.h>
#include <sys/file2.h>
+#include <sys/mplock2.h>
#include <arch_linux/linux.h>
#include <arch_linux/linux_proto.h>
#include <sys/kern_syscall.h>
#include <vfs/ufs/dir.h> /* XXX only for DIRBLKSIZ */
+#include <sys/mplock2.h>
+
#include <arch_linux/linux.h>
#include <arch_linux/linux_proto.h>
#include "linux_util.h"
#include <net/if.h>
#include <net/if_dl.h>
#include <net/if_types.h>
+
#include <sys/file2.h>
+#include <sys/mplock2.h>
#include <arch_linux/linux.h>
#include <arch_linux/linux_proto.h>
#include <sys/reboot.h>
#include <sys/resourcevar.h>
#include <sys/signalvar.h>
-#include <sys/signal2.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/vmmeter.h>
#include <sys/vnode.h>
#include <sys/wait.h>
+
+#include <sys/signal2.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <sys/sysproto.h>
#include <sys/kern_syscall.h>
#include <sys/thread.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
+
#include <arch_linux/linux.h>
#include <arch_linux/linux_proto.h>
#include "linux_signal.h"
#include <sys/malloc.h>
#include <sys/mbuf.h>
+#include <sys/mplock2.h>
+
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <sys/unistd.h>
#include <sys/vnode.h>
#include <sys/device.h>
-#include <sys/file2.h>
#include <sys/kern_syscall.h>
+#include <sys/file2.h>
+#include <sys/mplock2.h>
+
#include <arch_linux/linux.h>
#include <arch_linux/linux_proto.h>
#include "linux_util.h"
#include <sys/proc.h>
#include <sys/sysproto.h>
+#include <sys/mplock2.h>
+
#include <arch_linux/linux.h>
#include <arch_linux/linux_proto.h>
#include "linux_util.h"
#include <sys/sysproto.h>
#include <sys/thread.h>
+#include <sys/mplock2.h>
+
#include <arch_linux/linux.h>
#include <arch_linux/linux_proto.h>
#include "linux_util.h"
#include <vm/vm_extern.h>
#include <sys/sysref2.h>
+#include <sys/mplock2.h>
static int exec_res_id = 0;
#include <sys/unistd.h>
#include <sys/malloc.h>
#include <sys/machintr.h>
+
#include <sys/file2.h>
#include <sys/thread2.h>
#include <sys/sysref2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
#include <machine/cpu.h>
#include <vm/vm_zone.h>
+#include <sys/mplock2.h>
+
/*
* The routines implemented in this file are described in:
* Leffler, et al.: The Design and Implementation of the 4.3BSD
#include <sys/stat.h>
#include <sys/acl.h>
+#include <sys/mplock2.h>
+
static int vacl_set_acl(struct vnode *vp, acl_type_t type, struct acl *aclp);
static int vacl_get_acl(struct vnode *vp, acl_type_t type, struct acl *aclp);
static int vacl_aclcheck(struct vnode *vp, acl_type_t type, struct acl *aclp);
#include <sys/nlookup.h>
#include <sys/file.h>
-/* only on dragonfly */
-#include <sys/file2.h>
#include <sys/fcntl.h>
#include <sys/signal.h>
#include <vm/vm_param.h>
#include <sys/mount.h>
#include <sys/ckpt.h>
+#include <sys/mplock2.h>
+#include <sys/file2.h>
+
static int elf_loadphdrs(struct file *fp, Elf_Phdr *phdr, int numsegs);
static int elf_getnotes(struct lwp *lp, struct file *fp, size_t notesz);
static int elf_demarshalnotes(void *src, prpsinfo_t *psinfo,
#include <vm/vm_map.h>
#include <vm/vm_extern.h>
#include <sys/sysctl.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <machine/cpu.h>
#include <machine/limits.h>
#include <sys/thread2.h>
#include <sys/file2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
static void fsetfd_locked(struct filedesc *fdp, struct file *fp, int fd);
static void fdreserve_locked (struct filedesc *fdp, int fd0, int incr);
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/uio.h>
-#include <sys/thread2.h>
#include <sys/signalvar.h>
#include <sys/filio.h>
+
+#include <sys/thread2.h>
#include <sys/file2.h>
+#include <sys/mplock2.h>
#include <vm/vm_zone.h>
#include <sys/reg.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
MALLOC_DEFINE(M_PARGS, "proc-args", "Process arguments");
MALLOC_DEFINE(M_EXECARGS, "exec-args", "Exec arguments");
#include <sys/thread2.h>
#include <sys/sysref2.h>
+#include <sys/mplock2.h>
static void reaplwps(void *context, int dummy);
static void reaplwp(struct lwp *lp);
#include <sys/thread2.h>
#include <sys/signal2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
static MALLOC_DEFINE(M_ATFORK, "atfork", "atfork callback");
#include <sys/sysctl.h>
#include <sys/thread.h>
#include <sys/proc.h>
-#include <sys/thread2.h>
#include <sys/random.h>
#include <sys/serialize.h>
#include <sys/interrupt.h>
#include <sys/interrupt.h>
+#include <sys/thread2.h>
+#include <sys/mplock2.h>
+
struct info_info;
typedef struct intrec {
#include <netinet/in.h>
#include <netinet6/in6_var.h>
+#include <sys/mplock2.h>
+
static struct prison *prison_find(int);
static void prison_ipcache_init(struct prison *);
#include <sys/sysent.h>
#include <vm/vm_zone.h>
+
+#include <sys/mplock2.h>
+
static MALLOC_DEFINE(M_KTRACE, "KTRACE", "KTRACE");
#ifdef KTRACE
#include <vm/vm_zone.h>
+#include <sys/mplock2.h>
+
#ifdef _KERNEL_VIRTUAL
#include <dlfcn.h>
#endif
#include <sys/linker.h>
#include <sys/proc.h>
+#include <sys/mplock2.h>
+
MALLOC_DEFINE(M_MODULE, "module", "module data structures");
typedef TAILQ_HEAD(, module) modulelist_t;
--- /dev/null
+/*
+ * Copyright (c) 2009 The DragonFly Project. All rights reserved.
+ *
+ * This code is derived from software contributed to The DragonFly Project
+ * by Matthew Dillon <dillon@backplane.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of The DragonFly Project nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific, prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Helper functions for MP lock acquisition and release.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/proc.h>
+#include <sys/rtprio.h>
+#include <sys/queue.h>
+#include <sys/sysctl.h>
+#include <sys/kthread.h>
+#include <machine/cpu.h>
+#include <sys/lock.h>
+#include <sys/caps.h>
+#include <sys/spinlock.h>
+#include <sys/ktr.h>
+
+#include <sys/thread2.h>
+#include <sys/mplock2.h>
+#include <sys/spinlock2.h>
+
+#ifdef SMP
+static int chain_mplock = 0;
+static int bgl_yield = 10;
+static __int64_t mplock_contention_count = 0;
+
+SYSCTL_INT(_lwkt, OID_AUTO, chain_mplock, CTLFLAG_RW, &chain_mplock, 0, "");
+SYSCTL_INT(_lwkt, OID_AUTO, bgl_yield_delay, CTLFLAG_RW, &bgl_yield, 0, "");
+SYSCTL_QUAD(_lwkt, OID_AUTO, mplock_contention_count, CTLFLAG_RW,
+ &mplock_contention_count, 0, "spinning due to MPLOCK contention");
+
+/*
+ * Kernel Trace
+ */
+#if !defined(KTR_GIANT_CONTENTION)
+#define KTR_GIANT_CONTENTION KTR_ALL
+#endif
+
+KTR_INFO_MASTER(giant);
+KTR_INFO(KTR_GIANT_CONTENTION, giant, beg, 0,
+ "thread=%p held %s:%-5d want %s:%-5d",
+ sizeof(void *) * 3 + sizeof(int) * 2);
+KTR_INFO(KTR_GIANT_CONTENTION, giant, end, 1,
+ "thread=%p held %s:%-5d want %s:%-5d",
+ sizeof(void *) * 3 + sizeof(int) * 2);
+
+#define loggiant(name) \
+ KTR_LOG(giant_ ## name, curthread, \
+ mp_lock_holder_file, mp_lock_holder_line, \
+ file, line)
+
+int mp_lock;
+int mp_lock_contention_mask;
+const char *mp_lock_holder_file; /* debugging */
+int mp_lock_holder_line; /* debugging */
+
+/*
+ * Sets up the initial MP lock state near the start of the kernel boot
+ */
+void
+cpu_get_initial_mplock(void)
+{
+ mp_lock = 0; /* cpu 0 */
+ curthread->td_mpcount = 1;
+}
+
+/*
+ * Called when the MP lock could not be trvially acquired. The caller
+ * has already bumped td_mpcount.
+ */
+void
+_get_mplock_contested(const char *file, int line)
+{
+ globaldata_t gd = mycpu;
+ int ov;
+ int nv;
+
+ ++mplock_contention_count;
+ for (;;) {
+ ov = mp_lock;
+ nv = gd->gd_cpuid;
+ if (ov == gd->gd_cpuid)
+ break;
+ if (ov == -1) {
+ if (atomic_cmpset_int(&mp_lock, ov, gd->gd_cpuid))
+ break;
+ } else {
+ loggiant(beg);
+ lwkt_switch();
+ loggiant(end);
+ KKASSERT(gd->gd_cpuid == mp_lock);
+ break;
+ }
+ }
+}
+
+/*
+ * Called if td_mpcount went negative or if td_mpcount is 0 and we were
+ * unable to release the MP lock. Handles sanity checks and conflicts.
+ *
+ * It is possible for the inline release to have raced an interrupt which
+ * get/rel'd the MP lock, causing the inline's cmpset to fail. If this
+ * case occurs mp_lock will either already be in a released state or it
+ * will have already been acquired by another cpu.
+ */
+void
+_rel_mplock_contested(void)
+{
+ globaldata_t gd = mycpu;
+ int ov;
+
+ KKASSERT(gd->gd_curthread->td_mpcount >= 0);
+ for (;;) {
+ ov = mp_lock;
+ if (ov != gd->gd_cpuid)
+ break;
+ if (atomic_cmpset_int(&mp_lock, ov, -1))
+ break;
+ }
+}
+
+/*
+ * Called when try_mplock() fails.
+ *
+ * The inline bumped td_mpcount so we have to undo it.
+ *
+ * It is possible to race an interrupt which acquired and released the
+ * MP lock. When combined with the td_mpcount decrement we do the MP lock
+ * can wind up in any state and possibly not even owned by us.
+ *
+ * It is also possible for this function to be called even if td_mpcount > 1
+ * if someone bumped it and raced an interrupt which then called try_mpock().
+ */
+void
+_try_mplock_contested(const char *file, int line)
+{
+ globaldata_t gd = mycpu;
+ thread_t td = gd->gd_curthread;
+ int ov;
+
+ --td->td_mpcount;
+ KKASSERT(td->td_mpcount >= 0);
+ ++mplock_contention_count;
+
+ for (;;) {
+ ov = mp_lock;
+ if (ov != gd->gd_cpuid)
+ break;
+ if (atomic_cmpset_int(&mp_lock, ov, -1))
+ break;
+ }
+}
+
+/*
+ * Called when cpu_try_mplock() fails.
+ *
+ * The inline did not touch td_mpcount so we do not either.
+ */
+void
+_cpu_try_mplock_contested(const char *file, int line)
+{
+ ++mplock_contention_count;
+}
+
+/*
+ * Temporarily yield the MP lock. This is part of lwkt_user_yield()
+ * which is kinda hackish.
+ */
+void
+yield_mplock(thread_t td)
+{
+ int savecnt;
+
+ savecnt = td->td_mpcount;
+ td->td_mpcount = 1;
+ rel_mplock();
+ DELAY(bgl_yield);
+ get_mplock();
+ td->td_mpcount = savecnt;
+}
+
+#if 0
+
+/*
+ * The rel_mplock() code will call this function after releasing the
+ * last reference on the MP lock if mp_lock_contention_mask is non-zero.
+ *
+ * We then chain an IPI to a single other cpu potentially needing the
+ * lock. This is a bit heuristical and we can wind up with IPIs flying
+ * all over the place.
+ */
+static void lwkt_mp_lock_uncontested_remote(void *arg __unused);
+
+void
+lwkt_mp_lock_uncontested(void)
+{
+ globaldata_t gd;
+ globaldata_t dgd;
+ cpumask_t mask;
+ cpumask_t tmpmask;
+ int cpuid;
+
+ if (chain_mplock) {
+ gd = mycpu;
+ clr_mplock_contention_mask(gd);
+ mask = mp_lock_contention_mask;
+ tmpmask = ~((1 << gd->gd_cpuid) - 1);
+
+ if (mask) {
+ if (mask & tmpmask)
+ cpuid = bsfl(mask & tmpmask);
+ else
+ cpuid = bsfl(mask);
+ atomic_clear_int(&mp_lock_contention_mask, 1 << cpuid);
+ dgd = globaldata_find(cpuid);
+ lwkt_send_ipiq(dgd, lwkt_mp_lock_uncontested_remote, NULL);
+ }
+ }
+}
+
+/*
+ * The idea is for this IPI to interrupt a potentially lower priority
+ * thread, such as a user thread, to allow the scheduler to reschedule
+ * a higher priority kernel thread that needs the MP lock.
+ *
+ * For now we set the LWKT reschedule flag which generates an AST in
+ * doreti, though theoretically it is also possible to possibly preempt
+ * here if the underlying thread was operating in user mode. Nah.
+ */
+static void
+lwkt_mp_lock_uncontested_remote(void *arg __unused)
+{
+ need_lwkt_resched();
+}
+
+#endif
+
+#endif /* SMP */
#include <sys/timex.h>
#include <sys/timepps.h>
#include <sys/sysctl.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
/*
* Single-precision macros for 64-bit machines
#include <sys/sysctl.h>
#include <sys/unistd.h>
+#include <sys/mplock2.h>
+
MALLOC_DEFINE(M_P31B, "p1003.1b", "Posix 1003.1B");
/* p31b_proc: Return a proc struct corresponding to a pid to operate on.
#include <machine/smp.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
static MALLOC_DEFINE(M_PGRP, "pgrp", "process group header");
MALLOC_DEFINE(M_SESSION, "session", "session header");
#include <sys/thread2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
static MALLOC_DEFINE(M_CRED, "cred", "credentials");
#include <sys/thread2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
static int donice (struct proc *chgp, int n);
#include <sys/thread2.h>
#include <sys/buf2.h>
+#include <sys/mplock2.h>
#include <machine/clock.h>
#include <machine/md_var.h>
#include <sys/kernel.h>
#include <sys/sysproto.h>
#include <sys/signalvar.h>
-#include <sys/signal2.h>
#include <sys/resourcevar.h>
#include <sys/vnode.h>
#include <sys/event.h>
#include <sys/unistd.h>
#include <sys/kern_syscall.h>
#include <sys/vkernel.h>
+
+#include <sys/signal2.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <machine/cpu.h>
#include <machine/smp.h>
#include <machine/cpu.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#define arysize(ary) (sizeof(ary)/sizeof((ary)[0]))
#include <sys/proc.h>
#include <sys/kernel.h>
#include <sys/signalvar.h>
-#include <sys/signal2.h>
#include <sys/resourcevar.h>
#include <sys/vmmeter.h>
#include <sys/sysctl.h>
#endif
#include <sys/xwait.h>
#include <sys/ktr.h>
+#include <sys/serialize.h>
+#include <sys/signal2.h>
#include <sys/thread2.h>
#include <sys/spinlock2.h>
#include <sys/mutex2.h>
-#include <sys/serialize.h>
+#include <sys/mplock2.h>
#include <machine/cpu.h>
#include <machine/smp.h>
#include <sys/priv.h>
#include <sys/sysproto.h>
#include <sys/lock.h>
+
+#include <sys/mplock2.h>
+
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <sys/thread2.h>
#include <sys/spinlock2.h>
#include <sys/buf2.h>
+#include <sys/mplock2.h>
#include "opt_syslink.h"
#include <sys/kern_syscall.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
+
#include <sys/msgport2.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
struct timezone tz;
#include <sys/kernel.h>
#include <sys/interrupt.h>
#include <sys/thread.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#ifndef MAX_SOFTCLOCK_STEPS
#define MAX_SOFTCLOCK_STEPS 100 /* Maximum allowed value of steps. */
#include <vm/vm_kern.h>
#include <vm/vm_page2.h>
+#include <sys/mplock2.h>
static void umtx_sleep_page_action_cow(vm_page_t m, vm_page_action_t action);
#include <sys/lock.h>
#include <sys/signalvar.h>
+#include <sys/mplock2.h>
+
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_kern.h>
#include <sys/sysproto.h> /* struct usched_set_args */
#include <sys/systm.h> /* strcmp() */
#include <sys/usched.h>
+
+#include <sys/mplock2.h>
+
#include <machine/smp.h>
static TAILQ_HEAD(, usched) usched_list = TAILQ_HEAD_INITIALIZER(usched_list);
#include <sys/gpt.h>
#include <net/if_var.h>
+#include <sys/mplock2.h>
+
/*
* See also:
* http://www.opengroup.org/dce/info/draft-leach-uuids-guids-01.txt
#include <sys/varsym.h>
#include <sys/sysproto.h>
+#include <sys/mplock2.h>
+
MALLOC_DEFINE(M_VARSYM, "varsym", "variable sets for variant symlinks");
struct varsymset varsymset_sys;
#include <sys/sysctl.h>
#include <sys/utsname.h>
+#include <sys/mplock2.h>
+
/*
* MPALMOSTSAFE
*/
#include <sys/ucred.h>
#include <sys/caps.h>
#include <sys/sysctl.h>
+
+#include <sys/mplock2.h>
+
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <sys/thread2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
static MALLOC_DEFINE(M_THREAD, "thread", "lwkt threads");
-#ifdef SMP
-static int mplock_countx = 0;
-#endif
#ifdef INVARIANTS
static int panic_on_cscount = 0;
#endif
static __int64_t preempt_miss = 0;
static __int64_t preempt_weird = 0;
static __int64_t token_contention_count __debugvar = 0;
-static __int64_t mplock_contention_count __debugvar = 0;
static int lwkt_use_spin_port;
-#ifdef SMP
-static int chain_mplock = 0;
-static int bgl_yield = 10;
-#endif
static struct objcache *thread_cache;
-volatile cpumask_t mp_lock_contention_mask;
-
#ifdef SMP
static void lwkt_schedule_remote(void *arg, int arg2, struct intrframe *frame);
#endif
#ifdef INVARIANTS
SYSCTL_INT(_lwkt, OID_AUTO, panic_on_cscount, CTLFLAG_RW, &panic_on_cscount, 0, "");
#endif
-#ifdef SMP
-SYSCTL_INT(_lwkt, OID_AUTO, chain_mplock, CTLFLAG_RW, &chain_mplock, 0, "");
-SYSCTL_INT(_lwkt, OID_AUTO, bgl_yield_delay, CTLFLAG_RW, &bgl_yield, 0, "");
-#endif
SYSCTL_QUAD(_lwkt, OID_AUTO, switch_count, CTLFLAG_RW, &switch_count, 0, "");
SYSCTL_QUAD(_lwkt, OID_AUTO, preempt_hit, CTLFLAG_RW, &preempt_hit, 0, "");
SYSCTL_QUAD(_lwkt, OID_AUTO, preempt_miss, CTLFLAG_RW, &preempt_miss, 0, "");
#ifdef INVARIANTS
SYSCTL_QUAD(_lwkt, OID_AUTO, token_contention_count, CTLFLAG_RW,
&token_contention_count, 0, "spinning due to token contention");
-SYSCTL_QUAD(_lwkt, OID_AUTO, mplock_contention_count, CTLFLAG_RW,
- &mplock_contention_count, 0, "spinning due to MPLOCK contention");
#endif
/*
- * Kernel Trace
- */
-#if !defined(KTR_GIANT_CONTENTION)
-#define KTR_GIANT_CONTENTION KTR_ALL
-#endif
-
-KTR_INFO_MASTER(giant);
-KTR_INFO(KTR_GIANT_CONTENTION, giant, beg, 0, "thread=%p", sizeof(void *));
-KTR_INFO(KTR_GIANT_CONTENTION, giant, end, 1, "thread=%p", sizeof(void *));
-
-#define loggiant(name) KTR_LOG(giant_ ## name, curthread)
-
-/*
* These helper procedures handle the runq, they can only be called from
* within a critical section.
*
TAILQ_FOREACH(ntd, &gd->gd_tdrunq[nq], td_threadq) {
if (ntd->td_mpcount && !mpheld && !cpu_try_mplock()) {
/* spinning due to MP lock being held */
-#ifdef INVARIANTS
- ++mplock_contention_count;
-#endif
- /* mplock still not held, 'mpheld' still valid */
continue;
}
* reschedule when the MP lock might become available.
*/
if (nq < TDPRI_KERN_LPSCHED) {
+ break; /* for now refuse to run */
+#if 0
if (chain_mplock == 0)
break;
- atomic_set_int(&mp_lock_contention_mask,
- gd->gd_cpumask);
/* continue loop, allow user threads to be scheduled */
+#endif
}
}
+
+ /*
+ * Case where a (kernel) thread needed the MP lock and could
+ * not get one, and we may or may not have found another
+ * thread which does not need the MP lock to run while
+ * we wait (ntd).
+ */
if (ntd == NULL) {
- cpu_mplock_contested();
ntd = &gd->gd_idlethread;
ntd->td_flags |= TDF_IDLE_NOHLT;
+ set_mplock_contention_mask(gd);
+ cpu_mplock_contested();
goto using_idle_thread;
} else {
+ clr_mplock_contention_mask(gd);
++gd->gd_cnt.v_swtch;
TAILQ_REMOVE(&gd->gd_tdrunq[nq], ntd, td_threadq);
TAILQ_INSERT_TAIL(&gd->gd_tdrunq[nq], ntd, td_threadq);
}
} else {
- if (ntd->td_mpcount)
- ++mplock_countx;
+ clr_mplock_contention_mask(gd);
++gd->gd_cnt.v_swtch;
TAILQ_REMOVE(&gd->gd_tdrunq[nq], ntd, td_threadq);
TAILQ_INSERT_TAIL(&gd->gd_tdrunq[nq], ntd, td_threadq);
*/
if (ntd->td_mpcount) {
mpheld = MP_LOCK_HELD();
- if (gd->gd_trap_nesting_level == 0 && panicstr == NULL) {
+ if (gd->gd_trap_nesting_level == 0 && panicstr == NULL)
panic("Idle thread %p was holding the BGL!", ntd);
- } else if (mpheld == 0) {
- cpu_mplock_contested();
+ if (mpheld == 0)
goto again;
- }
}
#endif
}
* has a chaining effect since if the interrupt is blocked, so is
* the event, so normal scheduling will not pick up on the problem.
*/
- if (mplock_countx && td->td_mpcount) {
- int savecnt = td->td_mpcount;
-
- td->td_mpcount = 1;
- mplock_countx = 0;
- rel_mplock();
- DELAY(bgl_yield);
- get_mplock();
- td->td_mpcount = savecnt;
+ if (mp_lock_contention_mask && td->td_mpcount) {
+ yield_mplock(td);
}
#endif
crit_exit_gd(gd);
}
-/*
- * get_mplock() calls this routine if it is unable to obtain the MP lock.
- * get_mplock() has already incremented td_mpcount. We must block and
- * not return until giant is held.
- *
- * All we have to do is lwkt_switch() away. The LWKT scheduler will not
- * reschedule the thread until it can obtain the giant lock for it.
- */
-void
-lwkt_mp_lock_contested(void)
-{
- ++mplock_countx;
- loggiant(beg);
- lwkt_switch();
- loggiant(end);
-}
-
-/*
- * The rel_mplock() code will call this function after releasing the
- * last reference on the MP lock if mp_lock_contention_mask is non-zero.
- *
- * We then chain an IPI to a single other cpu potentially needing the
- * lock. This is a bit heuristical and we can wind up with IPIs flying
- * all over the place.
- */
-static void lwkt_mp_lock_uncontested_remote(void *arg __unused);
-
-void
-lwkt_mp_lock_uncontested(void)
-{
- globaldata_t gd;
- globaldata_t dgd;
- cpumask_t mask;
- cpumask_t tmpmask;
- int cpuid;
-
- if (chain_mplock) {
- gd = mycpu;
- atomic_clear_int(&mp_lock_contention_mask, gd->gd_cpumask);
- mask = mp_lock_contention_mask;
- tmpmask = ~((1 << gd->gd_cpuid) - 1);
-
- if (mask) {
- if (mask & tmpmask)
- cpuid = bsfl(mask & tmpmask);
- else
- cpuid = bsfl(mask);
- atomic_clear_int(&mp_lock_contention_mask, 1 << cpuid);
- dgd = globaldata_find(cpuid);
- lwkt_send_ipiq(dgd, lwkt_mp_lock_uncontested_remote, NULL);
- }
- }
-}
-
-/*
- * The idea is for this IPI to interrupt a potentially lower priority
- * thread, such as a user thread, to allow the scheduler to reschedule
- * a higher priority kernel thread that needs the MP lock.
- *
- * For now we set the LWKT reschedule flag which generates an AST in
- * doreti, though theoretically it is also possible to possibly preempt
- * here if the underlying thread was operating in user mode. Nah.
- */
-static void
-lwkt_mp_lock_uncontested_remote(void *arg __unused)
-{
- need_lwkt_resched();
-}
-
#endif
#include <machine/stdarg.h> /* for device_printf() */
#include <sys/thread2.h>
+#include <sys/mplock2.h>
SYSCTL_NODE(_hw, OID_AUTO, bus, CTLFLAG_RW, NULL, NULL);
#include <sys/proc.h>
#include <sys/resourcevar.h>
#include <sys/sysctl.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <machine/cpu.h>
#endif
#include <vm/vm.h>
#include <vm/vm_page.h>
+
#include <sys/file2.h>
+#include <sys/mplock2.h>
#include <machine/limits.h>
#include <sys/file2.h>
#include <sys/signal2.h>
+#include <sys/mplock2.h>
#include <machine/cpufunc.h>
#include <sys/user.h>
#include <vfs/procfs/procfs.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
/* use the equivalent procfs code */
#if 0
#include <sys/filedesc.h>
#include <sys/ucred.h>
+#include <sys/mplock2.h>
+
#include <net/if.h>
#include <net/route.h>
#include <sys/malloc.h>
#include <sys/jail.h>
+#include <sys/mplock2.h>
+
static MALLOC_DEFINE(M_MSG, "msg", "SVID compatible message queues");
static void msginit (void *);
#include <sys/malloc.h>
#include <sys/jail.h>
+#include <sys/mplock2.h>
+
static MALLOC_DEFINE(M_SEM, "sem", "SVID compatible semaphores");
static void seminit (void *);
#include <sys/sysent.h>
#include <sys/jail.h>
+#include <sys/mplock2.h>
+
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <sys/lock.h>
#include <sys/thread2.h>
#include <sys/msgport2.h>
#include <sys/socketvar2.h>
+#include <sys/mplock2.h>
#include <net/netmsg2.h>
#ifdef SCTP
#include <sys/thread2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
/*
* Priorities. Note that with 32 run queues per scheduler each queue
#include <sys/thread2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
#define MAXPRI 128
#define PRIBASE_REALTIME 0
#include <vm/vm_map.h>
#include <vm/vm_zone.h>
#include <sys/aio.h>
+
#include <sys/file2.h>
#include <sys/buf2.h>
#include <sys/sysref2.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <machine/limits.h>
#include "opt_vfs_aio.h"
#include <sys/buf2.h>
#include <sys/thread2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
#include <vm/vm_page2.h>
#include "opt_ddb.h"
#include <ddb/ddb.h>
#include <sys/sysref2.h>
+#include <sys/mplock2.h>
#define MAX_RECURSION_DEPTH 64
#include <sys/buf2.h>
#include <sys/file2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
#include <vm/vm.h>
#include <vm/vm_object.h>
#include <sys/syslog.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
static int vn_closefile (struct file *fp);
static int vn_ioctl (struct file *fp, u_long com, caddr_t data,
#include <sys/vnode.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <net/if.h>
#include <net/bpf.h>
#include <sys/sockio.h>
#include <sys/sysctl.h>
#include <sys/thread.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <net/if.h>
#include <net/netisr.h>
#include <sys/socket.h>
#include <sys/sockio.h>
+#include <sys/mplock2.h>
+
#include <net/if.h>
#include <net/if_types.h>
#include <net/ifq_var.h>
#include <sys/socketvar.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
-#include <sys/thread2.h>
#include <sys/ucred.h>
#include <sys/in_cksum.h>
#include <sys/lock.h>
#include <net/if.h>
#include <net/route.h>
-#include <net/netmsg2.h>
#include <net/pfil.h>
#include <net/dummynet/ip_dummynet.h>
+#include <sys/thread2.h>
+#include <sys/mplock2.h>
+#include <net/netmsg2.h>
+
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/in_var.h>
#include <sys/thread2.h>
#include <sys/msgport2.h>
#include <net/netmsg2.h>
+#include <sys/mplock2.h>
#define NETISR_GET_MPLOCK(ni) \
do { \
#include <net/if.h>
#include <net/pfil.h>
#include <net/netmsg2.h>
+#include <sys/mplock2.h>
#define PFIL_CFGPORT cpu_portfn(0)
#include <sys/sockio.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
-#include <sys/thread2.h>
#include <sys/ttycom.h>
#include <sys/uio.h>
#include <sys/vnode.h>
#include <sys/serialize.h>
+#include <sys/thread2.h>
+#include <sys/mplock2.h>
+
#include <net/bpf.h>
#include <net/ethernet.h>
#include <net/if.h>
#include <sys/vnode.h>
#include <sys/malloc.h>
+#include <sys/mplock2.h>
+
#include <net/if.h>
#include <net/if_types.h>
#include <net/ifq_var.h>
#include <sys/thread2.h>
#include <sys/msgport2.h>
#include <net/netmsg2.h>
+#include <sys/mplock2.h>
#ifdef CARP
#include <netinet/ip_carp.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/priv.h>
-#include <sys/thread2.h>
#include <sys/in_cksum.h>
#include <sys/lock.h>
#ifdef SMP
#include <net/if.h>
#include <net/route.h>
+
#ifdef SMP
#include <net/netmsg2.h>
#endif
+#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <sys/in_cksum.h>
#include <sys/lock.h>
+#include <sys/mplock2.h>
+
#include <machine/stdarg.h>
#include <net/if.h>
#include <sys/proc.h>
#include <sys/priv.h>
#include <sys/sysctl.h>
-#include <sys/thread2.h>
#include <sys/in_cksum.h>
#include <sys/lock.h>
+#include <sys/thread2.h>
+#include <sys/mplock2.h>
+
#include <net/if.h>
#include <net/netisr.h>
#include <net/pfil.h>
#endif
#include <sys/md5.h>
-#include <sys/msgport2.h>
#include <machine/smp.h>
+#include <sys/msgport2.h>
+#include <sys/mplock2.h>
#include <net/netmsg2.h>
#if !defined(KTR_TCP)
#include <sys/uio.h>
#include <sys/msgport.h>
+#include <sys/mplock2.h>
+
#include "ncp.h"
#include "ncp_conn.h"
#include "ncp_subr.h"
#include <sys/kernel.h>
#include <sys/fcntl.h>
#include <sys/proc.h>
+
#include <sys/file2.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <opencrypto/cryptodev.h>
#include <opencrypto/xform.h>
#arch/i386/i386/io_apic.c optional smp
#arch/i386/i386/local_apic.c optional smp
platform/pc32/i386/mpboot.s optional smp
-platform/pc32/i386/mplock.s optional smp
platform/pc32/i386/mp_clock.c optional smp
platform/pc32/i386/mp_machdep.c optional smp
platform/pc32/i386/mp_madt.c optional smp
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/uio.h>
-#include <sys/thread2.h>
#include <sys/bus_dma.h>
#include <sys/kernel.h>
#include <sys/sysctl.h>
#include <sys/lock.h>
+
+#include <sys/thread2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
ASSYM(TDF_RUNNING, TDF_RUNNING);
ASSYM(TDF_USINGFP, TDF_USINGFP);
ASSYM(TDF_KERNELFP, TDF_KERNELFP);
-#ifdef SMP
-ASSYM(MP_FREE_LOCK, MP_FREE_LOCK);
-#endif
ASSYM(MACHINTR_INTREN, offsetof(struct machintr_abi, intren));
ASSYM(TD_SAVEFPU, offsetof(struct thread, td_mach) + offsetof(struct md_thread, mtd_savefpu));
#include <vm/vm_extern.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <sys/user.h>
#include <sys/exec.h>
#include <sys/gmon.h>
#endif
+#include <sys/mplock2.h>
+
#include <machine/smp.h>
#include <machine_base/apic/apicreg.h>
#include <machine/atomic.h>
+++ /dev/null
-/*
- * $FreeBSD: src/sys/i386/i386/mplock.s,v 1.29.2.2 2000/05/16 06:58:06 dillon Exp $
- * $DragonFly: src/sys/platform/pc32/i386/mplock.s,v 1.21 2006/11/07 06:43:24 dillon Exp $
- *
- * Copyright (c) 2003,2004 The DragonFly Project. All rights reserved.
- *
- * This code is derived from software contributed to The DragonFly Project
- * by Matthew Dillon <dillon@backplane.com>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * 3. Neither the name of The DragonFly Project nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific, prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * DragonFly MPLOCK operation
- *
- * Each thread has an MP lock count, td_mpcount, and there is a shared
- * global called mp_lock. mp_lock is the physical MP lock and contains either
- * -1 or the cpuid of the cpu owning the lock. The count is *NOT* integrated
- * into mp_lock but instead resides in each thread td_mpcount.
- *
- * When obtaining or releasing the MP lock the td_mpcount is PREDISPOSED
- * to the desired count *PRIOR* to operating on the mp_lock itself. MP
- * lock operations can occur outside a critical section with interrupts
- * enabled with the provisio (which the routines below handle) that an
- * interrupt may come along and preempt us, racing our cmpxchgl instruction
- * to perform the operation we have requested by pre-disposing td_mpcount.
- *
- * Additionally, the LWKT threading system manages the MP lock and
- * lwkt_switch(), in particular, may be called after pre-disposing td_mpcount
- * to handle 'blocking' on the MP lock.
- *
- *
- * Recoded from the FreeBSD original:
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file. As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <machine/asmacros.h>
-#include <machine_base/apic/apicreg.h>
-
-#include "assym.s"
-
-/*
- * YYY Debugging only. Define this to be paranoid about invalidating the
- * TLB when we get giant.
- */
-#undef PARANOID_INVLTLB
-
- .data
- ALIGN_DATA
-#ifdef SMP
- .globl mp_lock
-mp_lock:
- .long -1 /* initialized to not held */
-#endif
-
- .text
- SUPERALIGN_TEXT
-
- /*
- * Note on cmpxchgl... exchanges ecx with mem if mem matches eax.
- * Z=1 (jz) on success. A lock prefix is required for MP.
- */
-NON_GPROF_ENTRY(cpu_get_initial_mplock)
- movl PCPU(curthread),%ecx
- movl $1,TD_MPCOUNT(%ecx) /* curthread has mpcount of 1 */
- movl $0,mp_lock /* owned by cpu 0 */
- NON_GPROF_RET
-
- /*
- * cpu_try_mplock() returns non-zero on success, 0 on failure. It
- * only adjusts mp_lock, it does not touch td_mpcount. Callers
- * should always increment td_mpcount *before* trying to acquire
- * the actual lock, predisposing td_mpcount to the desired state of
- * the lock.
- *
- * NOTE! Only call cpu_try_mplock() inside a critical section. If
- * you don't an interrupt can come along and get and release
- * the lock before our cmpxchgl instruction, causing us to fail
- * but resulting in the lock being held by our cpu.
- */
-NON_GPROF_ENTRY(cpu_try_mplock)
- movl PCPU(cpuid),%ecx
- movl $-1,%eax
- lock cmpxchgl %ecx,mp_lock /* ecx<->mem if eax matches */
- jnz 1f
-#ifdef PARANOID_INVLTLB
- movl %cr3,%eax; movl %eax,%cr3 /* YYY check and remove */
-#endif
- movl $1,%eax
- NON_GPROF_RET
-1:
- subl %eax,%eax
- NON_GPROF_RET
-
- /*
- * get_mplock() Obtains the MP lock and may switch away if it cannot
- * get it. This routine may be called WITHOUT a critical section
- * and with cpu interrupts enabled.
- *
- * To handle races in a sane fashion we predispose TD_MPCOUNT,
- * which prevents us from losing the lock in a race if we already
- * have it or happen to get it. It also means that we might get
- * the lock in an interrupt race before we have a chance to execute
- * our cmpxchgl instruction, so we have to handle that case.
- * Fortunately simply calling lwkt_switch() handles the situation
- * for us and also 'blocks' us until the MP lock can be obtained.
- */
-NON_GPROF_ENTRY(get_mplock)
- movl PCPU(cpuid),%ecx
- movl PCPU(curthread),%edx
- incl TD_MPCOUNT(%edx) /* predispose */
- cmpl %ecx,mp_lock
- jne 1f
- NON_GPROF_RET /* success! */
-
- /*
- * We don't already own the mp_lock, use cmpxchgl to try to get
- * it.
- */
-1:
- movl $-1,%eax
- lock cmpxchgl %ecx,mp_lock
- jnz 2f
-#ifdef PARANOID_INVLTLB
- movl %cr3,%eax; movl %eax,%cr3 /* YYY check and remove */
-#endif
- NON_GPROF_RET /* success */
-
- /*
- * Failure, but we could end up owning mp_lock anyway due to
- * an interrupt race. lwkt_switch() will clean up the mess
- * and 'block' until the mp_lock is obtained.
- *
- * Create a stack frame for the call so KTR logs the stack
- * backtrace properly.
- */
-2:
- pushl %ebp
- movl %esp,%ebp
- call lwkt_mp_lock_contested
- popl %ebp
-#ifdef INVARIANTS
- movl PCPU(cpuid),%eax /* failure */
- cmpl %eax,mp_lock
- jne 4f
-#endif
- NON_GPROF_RET
-#ifdef INVARIANTS
-4:
- cmpl $0,panicstr /* don't double panic */
- je badmp_get2
- NON_GPROF_RET
-#endif
-
- /*
- * try_mplock() attempts to obtain the MP lock. 1 is returned on
- * success, 0 on failure. We do not have to be in a critical section
- * and interrupts are almost certainly enabled.
- *
- * We must pre-dispose TD_MPCOUNT in order to deal with races in
- * a reasonable way.
- *
- */
-NON_GPROF_ENTRY(try_mplock)
- movl PCPU(cpuid),%ecx
- movl PCPU(curthread),%edx
- incl TD_MPCOUNT(%edx) /* pre-dispose for race */
- cmpl %ecx,mp_lock
- je 1f /* trivial success */
- movl $-1,%eax
- lock cmpxchgl %ecx,mp_lock
- jnz 2f
- /*
- * Success
- */
-#ifdef PARANOID_INVLTLB
- movl %cr3,%eax; movl %eax,%cr3 /* YYY check and remove */
-#endif
-1:
- movl $1,%eax /* success (cmpxchgl good!) */
- NON_GPROF_RET
-
- /*
- * The cmpxchgl failed but we might have raced. Undo the mess by
- * predispoing TD_MPCOUNT and then checking. If TD_MPCOUNT is
- * still non-zero we don't care what state the lock is in (since
- * we obviously didn't own it above), just return failure even if
- * we won the lock in an interrupt race. If TD_MPCOUNT is zero
- * make sure we don't own the lock in case we did win it in a race.
- */
-2:
- decl TD_MPCOUNT(%edx)
- cmpl $0,TD_MPCOUNT(%edx)
- jne 3f
- movl PCPU(cpuid),%eax
- movl $-1,%ecx
- lock cmpxchgl %ecx,mp_lock
-3:
- subl %eax,%eax
- NON_GPROF_RET
-
- /*
- * rel_mplock() releases a previously obtained MP lock.
- *
- * In order to release the MP lock we pre-dispose TD_MPCOUNT for
- * the release and basically repeat the release portion of try_mplock
- * above.
- */
-NON_GPROF_ENTRY(rel_mplock)
- movl PCPU(curthread),%edx
- movl TD_MPCOUNT(%edx),%eax
-#ifdef INVARIANTS
- cmpl $0,%eax
- je badmp_rel
-#endif
- subl $1,%eax
- movl %eax,TD_MPCOUNT(%edx)
- cmpl $0,%eax
- jne 3f
- movl PCPU(cpuid),%eax
- movl $-1,%ecx
- lock cmpxchgl %ecx,mp_lock
- movl mp_lock_contention_mask,%eax
- cmpl $0,%eax
- je 3f
- call lwkt_mp_lock_uncontested
-3:
- NON_GPROF_RET
-
-#ifdef INVARIANTS
-
-badmp_get:
- pushl $bmpsw1
- call panic
-badmp_get2:
- pushl $bmpsw1a
- call panic
-badmp_rel:
- pushl $bmpsw2
- call panic
-
- .data
-
-bmpsw1:
- .asciz "try/get_mplock(): already have lock! %d %p"
-
-bmpsw1a:
- .asciz "try/get_mplock(): failed on count or switch %d %p"
-
-bmpsw2:
- .asciz "rel_mplock(): mpcount already 0 @ %p %p %p %p %p %p %p %p!"
-
-#endif
-
#include <machine/globaldata.h> /* mdcpu */
#include <vm/vm_kern.h> /* for kernel_map */
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#define MAX_LD 8192
#define LD_PER_PAGE 512
#include <machine/vm86.h>
#include <ddb/ddb.h>
+
#include <sys/msgport2.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#ifdef SMP
#include <vm/vm_page.h>
#include <sys/user.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <machine/md_var.h>
#include <machine/pcb_ext.h> /* pcb.h included via sys/user.h */
#include <vm/vm_extern.h>
#include <sys/user.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <bus/isa/isa.h>
#include <machine/psl.h>
#endif
-/*
- * MP_FREE_LOCK is used by both assembly and C under SMP.
- */
-#ifdef SMP
-#define MP_FREE_LOCK 0xffffffff /* value of lock when free */
-#endif
-
#ifdef LOCORE
/*
#endif /* _KERNEL */
-#if defined(_KERNEL) || defined(_UTHREAD)
-
-/*
- * MP LOCK functions for SMP and UP. Under UP the MP lock does not exist
- * but we leave a few functions intact as macros for convenience.
- */
-#ifdef SMP
-
-void get_mplock(void);
-int try_mplock(void);
-void rel_mplock(void);
-int cpu_try_mplock(void);
-void cpu_get_initial_mplock(void);
-
-extern u_int mp_lock;
-
-#define MP_LOCK_HELD() (mp_lock == mycpu->gd_cpuid)
-#define ASSERT_MP_LOCK_HELD(td) KASSERT(MP_LOCK_HELD(), ("MP_LOCK_HELD(): not held thread %p", td))
-
-static __inline void
-cpu_rel_mplock(void)
-{
- mp_lock = MP_FREE_LOCK;
-}
-
-static __inline int
-owner_mplock(void)
-{
- return (mp_lock);
-}
-
-#else
-
-#define get_mplock()
-#define try_mplock() 1
-#define rel_mplock()
-#define owner_mplock() 0 /* always cpu 0 */
-#define MP_LOCK_HELD() (!0)
-#define ASSERT_MP_LOCK_HELD(td)
-
-#endif /* SMP */
-#endif /* _KERNEL || _UTHREAD */
#endif /* LOCORE */
#endif /* !_MACHINE_LOCK_H_ */
#include <sys/syslog.h>
#endif
#include <sys/signalvar.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#ifndef SMP
#include <machine/asmacros.h>
compile-with "${CC} -c ${CFLAGS} ${WERROR} ${DEFINED_PROF:S/^$/-fomit-frame-pointer/} ${.IMPSRC}"
platform/pc64/x86_64/autoconf.c standard
platform/pc64/x86_64/mpboot.S optional smp
-platform/pc64/x86_64/mplock.s optional smp
# DDB XXX
cpu/x86_64/misc/x86_64-gdbstub.c optional ddb
#include <machine/psl.h>
#endif
-/*
- * MP_FREE_LOCK is used by both assembly and C under SMP.
- */
-#ifdef SMP
-#define MP_FREE_LOCK 0xffffffff /* value of lock when free */
-#endif
-
#ifdef LOCORE
/*
#endif /* _KERNEL */
-#if defined(_KERNEL) || defined(_UTHREAD)
-
-/*
- * MP LOCK functions for SMP and UP. Under UP the MP lock does not exist
- * but we leave a few functions intact as macros for convenience.
- */
-#ifdef SMP
-
-void get_mplock(void);
-int try_mplock(void);
-void rel_mplock(void);
-int cpu_try_mplock(void);
-void cpu_get_initial_mplock(void);
-
-extern u_int mp_lock;
-
-#define MP_LOCK_HELD() (mp_lock == mycpu->gd_cpuid)
-#define ASSERT_MP_LOCK_HELD(td) KASSERT(MP_LOCK_HELD(), ("MP_LOCK_HELD(): not held thread %p", td))
-
-static __inline void
-cpu_rel_mplock(void)
-{
- mp_lock = MP_FREE_LOCK;
-}
-
-static __inline int
-owner_mplock(void)
-{
- return (mp_lock);
-}
-
-#else /* !SMP */
-
-#define get_mplock()
-#define try_mplock() 1
-#define rel_mplock()
-#define owner_mplock() 0 /* always cpu 0 */
-#define MP_LOCK_HELD() (!0)
-#define ASSERT_MP_LOCK_HELD(td)
-
-#endif /* SMP */
-#endif /* _KERNEL || _UTHREAD */
#endif /* LOCORE */
#endif /* !_MACHINE_LOCK_H_ */
#include <sys/syslog.h>
#endif
#include <sys/signalvar.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#ifndef SMP
#include <machine/asmacros.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/uio.h>
-#include <sys/thread2.h>
#include <sys/bus_dma.h>
#include <sys/kernel.h>
#include <sys/sysctl.h>
#include <sys/lock.h>
+
+#include <sys/thread2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <vm/vm_extern.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <sys/user.h>
#include <sys/exec.h>
#include <sys/cons.h> /* cngetc() */
#include <sys/machintr.h>
+#include <sys/mplock2.h>
+
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/pmap.h>
+++ /dev/null
-/*
- * $FreeBSD: src/sys/i386/i386/mplock.s,v 1.29.2.2 2000/05/16 06:58:06 dillon Exp $
- * $DragonFly: src/sys/platform/pc32/i386/mplock.s,v 1.21 2006/11/07 06:43:24 dillon Exp $
- *
- * Copyright (c) 2003,2004 The DragonFly Project. All rights reserved.
- *
- * This code is derived from software contributed to The DragonFly Project
- * by Matthew Dillon <dillon@backplane.com>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * 3. Neither the name of The DragonFly Project nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific, prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * DragonFly MPLOCK operation
- *
- * Each thread has an MP lock count, td_mpcount, and there is a shared
- * global called mp_lock. mp_lock is the physical MP lock and contains either
- * -1 or the cpuid of the cpu owning the lock. The count is *NOT* integrated
- * into mp_lock but instead resides in each thread td_mpcount.
- *
- * When obtaining or releasing the MP lock the td_mpcount is PREDISPOSED
- * to the desired count *PRIOR* to operating on the mp_lock itself. MP
- * lock operations can occur outside a critical section with interrupts
- * enabled with the provisio (which the routines below handle) that an
- * interrupt may come along and preempt us, racing our cmpxchgl instruction
- * to perform the operation we have requested by pre-disposing td_mpcount.
- *
- * Additionally, the LWKT threading system manages the MP lock and
- * lwkt_switch(), in particular, may be called after pre-disposing td_mpcount
- * to handle 'blocking' on the MP lock.
- *
- *
- * Recoded from the FreeBSD original:
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file. As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <machine/asmacros.h>
-#include <machine_base/apic/apicreg.h>
-
-#include "assym.s"
-
-/*
- * YYY Debugging only. Define this to be paranoid about invalidating the
- * TLB when we get giant.
- */
-#undef PARANOID_INVLTLB
-
- .data
- ALIGN_DATA
-#ifdef SMP
- .globl mp_lock
-mp_lock:
- .long -1 /* initialized to not held */
-#endif
-
- .text
- SUPERALIGN_TEXT
-
- /*
- * Note on cmpxchgl... exchanges ecx with mem if mem matches eax.
- * Z=1 (jz) on success. A lock prefix is required for MP.
- */
-NON_GPROF_ENTRY(cpu_get_initial_mplock)
- movq PCPU(curthread),%rcx
- movl $1,TD_MPCOUNT(%rcx) /* curthread has mpcount of 1 */
- movl $0,mp_lock /* owned by cpu 0 */
- NON_GPROF_RET
-
- /*
- * cpu_try_mplock() returns non-zero on success, 0 on failure. It
- * only adjusts mp_lock, it does not touch td_mpcount. Callers
- * should always increment td_mpcount *before* trying to acquire
- * the actual lock, predisposing td_mpcount to the desired state of
- * the lock.
- *
- * NOTE! Only call cpu_try_mplock() inside a critical section. If
- * you don't an interrupt can come along and get and release
- * the lock before our cmpxchgl instruction, causing us to fail
- * but resulting in the lock being held by our cpu.
- */
-NON_GPROF_ENTRY(cpu_try_mplock)
- movl PCPU(cpuid),%ecx
- movl $-1,%eax
- lock cmpxchgl %ecx,mp_lock /* ecx<->mem if eax matches */
- jnz 1f
-#ifdef PARANOID_INVLTLB
- movq %cr3,%rax; movq %rax,%cr3 /* YYY check and remove */
-#endif
- movl $1,%eax
- NON_GPROF_RET
-1:
- subl %eax,%eax
- NON_GPROF_RET
-
- /*
- * get_mplock() Obtains the MP lock and may switch away if it cannot
- * get it. This routine may be called WITHOUT a critical section
- * and with cpu interrupts enabled.
- *
- * To handle races in a sane fashion we predispose TD_MPCOUNT,
- * which prevents us from losing the lock in a race if we already
- * have it or happen to get it. It also means that we might get
- * the lock in an interrupt race before we have a chance to execute
- * our cmpxchgl instruction, so we have to handle that case.
- * Fortunately simply calling lwkt_switch() handles the situation
- * for us and also 'blocks' us until the MP lock can be obtained.
- */
-NON_GPROF_ENTRY(get_mplock)
- movl PCPU(cpuid),%ecx
- movq PCPU(curthread),%rdx
- incl TD_MPCOUNT(%rdx) /* predispose */
- cmpl %ecx,mp_lock
- jne 1f
- NON_GPROF_RET /* success! */
-
- /*
- * We don't already own the mp_lock, use cmpxchgl to try to get
- * it.
- */
-1:
- movl $-1,%eax
- lock cmpxchgl %ecx,mp_lock
- jnz 2f
-#ifdef PARANOID_INVLTLB
- movq %cr3,%rax; movq %rax,%cr3 /* YYY check and remove */
-#endif
- NON_GPROF_RET /* success */
-
- /*
- * Failure, but we could end up owning mp_lock anyway due to
- * an interrupt race. lwkt_switch() will clean up the mess
- * and 'block' until the mp_lock is obtained.
- *
- * Create a stack frame for the call so KTR logs the stack
- * backtrace properly.
- */
-2:
- pushq %rbp
- movq %rsp,%rbp
- call lwkt_mp_lock_contested
- popq %rbp
-#ifdef INVARIANTS
- movl PCPU(cpuid),%eax /* failure */
- cmpl %eax,mp_lock
- jne 4f
-#endif
- NON_GPROF_RET
-#ifdef INVARIANTS
-4:
- cmpl $0,panicstr /* don't double panic */
- je badmp_get2
- NON_GPROF_RET
-#endif
-
- /*
- * try_mplock() attempts to obtain the MP lock. 1 is returned on
- * success, 0 on failure. We do not have to be in a critical section
- * and interrupts are almost certainly enabled.
- *
- * We must pre-dispose TD_MPCOUNT in order to deal with races in
- * a reasonable way.
- *
- */
-NON_GPROF_ENTRY(try_mplock)
- movl PCPU(cpuid),%ecx
- movq PCPU(curthread),%rdx
- incl TD_MPCOUNT(%rdx) /* pre-dispose for race */
- cmpl %ecx,mp_lock
- je 1f /* trivial success */
- movl $-1,%eax
- lock cmpxchgl %ecx,mp_lock
- jnz 2f
- /*
- * Success
- */
-#ifdef PARANOID_INVLTLB
- movq %cr3,%rax; movq %rax,%cr3 /* YYY check and remove */
-#endif
-1:
- movl $1,%eax /* success (cmpxchgl good!) */
- NON_GPROF_RET
-
- /*
- * The cmpxchgl failed but we might have raced. Undo the mess by
- * predispoing TD_MPCOUNT and then checking. If TD_MPCOUNT is
- * still non-zero we don't care what state the lock is in (since
- * we obviously didn't own it above), just return failure even if
- * we won the lock in an interrupt race. If TD_MPCOUNT is zero
- * make sure we don't own the lock in case we did win it in a race.
- */
-2:
- decl TD_MPCOUNT(%rdx)
- cmpl $0,TD_MPCOUNT(%rdx)
- jne 3f
- movl PCPU(cpuid),%eax
- movl $-1,%ecx
- lock cmpxchgl %ecx,mp_lock
-3:
- subl %eax,%eax
- NON_GPROF_RET
-
- /*
- * rel_mplock() releases a previously obtained MP lock.
- *
- * In order to release the MP lock we pre-dispose TD_MPCOUNT for
- * the release and basically repeat the release portion of try_mplock
- * above.
- */
-NON_GPROF_ENTRY(rel_mplock)
- movq PCPU(curthread),%rdx
- movl TD_MPCOUNT(%rdx),%eax
-#ifdef INVARIANTS
- cmpl $0,%eax
- je badmp_rel
-#endif
- subl $1,%eax
- movl %eax,TD_MPCOUNT(%rdx)
- cmpl $0,%eax
- jne 3f
- movl PCPU(cpuid),%eax
- movl $-1,%ecx
- lock cmpxchgl %ecx,mp_lock
- movl mp_lock_contention_mask,%eax
- cmpl $0,%eax
- je 3f
- call lwkt_mp_lock_uncontested
-3:
- NON_GPROF_RET
-
-#ifdef INVARIANTS
-
-badmp_get:
- movq $bmpsw1,%rdi
- movl $0,%eax
- call panic
-badmp_get2:
- movq $bmpsw1a,%rdi
- movl $0,%eax
- call panic
-badmp_rel:
- movq $bmpsw2,%rdi
- movl $0,%eax
- call panic
-
- .data
-
-bmpsw1:
- .asciz "try/get_mplock(): already have lock! %d %p"
-
-bmpsw1a:
- .asciz "try/get_mplock(): failed on count or switch %d %p"
-
-bmpsw2:
- .asciz "rel_mplock(): mpcount already 0 @ %p %p %p %p %p %p %p %p!"
-
-#endif
-
#include <sys/syslog.h>
#endif
#include <sys/signalvar.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#ifndef SMP
#include <machine/asmacros.h>
#include <machine_base/isa/intr_machdep.h>
#include <ddb/ddb.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#ifdef SMP
#include <vm/vm_extern.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <bus/isa/isa.h>
platform/vkernel/i386/autoconf.c standard
platform/vkernel/i386/mp.c optional smp \
compile-with "${CC} -c -pthread ${CFLAGS} ${WERROR} -I/usr/include ${.IMPSRC}"
-platform/vkernel/i386/mplock.s optional smp
#
# DDB XXX
cpu/i386/misc/elf_machdep.c standard
#include <vm/vm_extern.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <sys/user.h>
#include <sys/exec.h>
#endif
ASSYM(TD_FLAGS, offsetof(struct thread, td_flags));
ASSYM(TDF_RUNNING, TDF_RUNNING);
-#ifdef SMP
-ASSYM(MP_FREE_LOCK, MP_FREE_LOCK);
-#endif
ASSYM(TD_SAVEFPU, offsetof(struct thread, td_mach) + offsetof(struct md_thread, mtd_savefpu));
#include <vm/vm_object.h>
#include <vm/vm_page.h>
+#include <sys/mplock2.h>
+
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/globaldata.h>
+++ /dev/null
-/*
- * $FreeBSD: src/sys/i386/i386/mplock.s,v 1.29.2.2 2000/05/16 06:58:06 dillon Exp $
- * $DragonFly: src/sys/platform/vkernel/i386/mplock.s,v 1.2 2007/07/01 02:51:43 dillon Exp $
- *
- * Copyright (c) 2003,2004 The DragonFly Project. All rights reserved.
- *
- * This code is derived from software contributed to The DragonFly Project
- * by Matthew Dillon <dillon@backplane.com>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * 3. Neither the name of The DragonFly Project nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific, prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * DragonFly MPLOCK operation
- *
- * Each thread has an MP lock count, td_mpcount, and there is a shared
- * global called mp_lock. mp_lock is the physical MP lock and contains either
- * -1 or the cpuid of the cpu owning the lock. The count is *NOT* integrated
- * into mp_lock but instead resides in each thread td_mpcount.
- *
- * When obtaining or releasing the MP lock the td_mpcount is PREDISPOSED
- * to the desired count *PRIOR* to operating on the mp_lock itself. MP
- * lock operations can occur outside a critical section with interrupts
- * enabled with the provisio (which the routines below handle) that an
- * interrupt may come along and preempt us, racing our cmpxchgl instruction
- * to perform the operation we have requested by pre-disposing td_mpcount.
- *
- * Additionally, the LWKT threading system manages the MP lock and
- * lwkt_switch(), in particular, may be called after pre-disposing td_mpcount
- * to handle 'blocking' on the MP lock.
- *
- *
- * Recoded from the FreeBSD original:
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file. As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <machine/asmacros.h>
-#if 0
-#include <machine_base/apic/apicreg.h>
-#endif
-
-#include "assym.s"
-
-/*
- * YYY Debugging only. Define this to be paranoid about invalidating the
- * TLB when we get giant.
- */
-#undef PARANOID_INVLTLB
-
- .data
- ALIGN_DATA
-#ifdef SMP
- .globl mp_lock
-mp_lock:
- .long -1 /* initialized to not held */
-#endif
-
- .text
- SUPERALIGN_TEXT
-
- /*
- * Note on cmpxchgl... exchanges ecx with mem if mem matches eax.
- * Z=1 (jz) on success. A lock prefix is required for MP.
- */
-NON_GPROF_ENTRY(cpu_get_initial_mplock)
- movl PCPU(curthread),%ecx
- movl $1,TD_MPCOUNT(%ecx) /* curthread has mpcount of 1 */
- movl $0,mp_lock /* owned by cpu 0 */
- NON_GPROF_RET
-
- /*
- * cpu_try_mplock() returns non-zero on success, 0 on failure. It
- * only adjusts mp_lock, it does not touch td_mpcount. Callers
- * should always increment td_mpcount *before* trying to acquire
- * the actual lock, predisposing td_mpcount to the desired state of
- * the lock.
- *
- * NOTE! Only call cpu_try_mplock() inside a critical section. If
- * you don't an interrupt can come along and get and release
- * the lock before our cmpxchgl instruction, causing us to fail
- * but resulting in the lock being held by our cpu.
- */
-NON_GPROF_ENTRY(cpu_try_mplock)
- movl PCPU(cpuid),%ecx
- movl $-1,%eax
- lock cmpxchgl %ecx,mp_lock /* ecx<->mem if eax matches */
- jnz 1f
-#ifdef PARANOID_INVLTLB
- movl %cr3,%eax; movl %eax,%cr3 /* YYY check and remove */
-#endif
- movl $1,%eax
- NON_GPROF_RET
-1:
- subl %eax,%eax
- NON_GPROF_RET
-
- /*
- * get_mplock() Obtains the MP lock and may switch away if it cannot
- * get it. This routine may be called WITHOUT a critical section
- * and with cpu interrupts enabled.
- *
- * To handle races in a sane fashion we predispose TD_MPCOUNT,
- * which prevents us from losing the lock in a race if we already
- * have it or happen to get it. It also means that we might get
- * the lock in an interrupt race before we have a chance to execute
- * our cmpxchgl instruction, so we have to handle that case.
- * Fortunately simply calling lwkt_switch() handles the situation
- * for us and also 'blocks' us until the MP lock can be obtained.
- */
-NON_GPROF_ENTRY(get_mplock)
- movl PCPU(cpuid),%ecx
- movl PCPU(curthread),%edx
- incl TD_MPCOUNT(%edx) /* predispose */
- cmpl %ecx,mp_lock
- jne 1f
- NON_GPROF_RET /* success! */
-
- /*
- * We don't already own the mp_lock, use cmpxchgl to try to get
- * it.
- */
-1:
- movl $-1,%eax
- lock cmpxchgl %ecx,mp_lock
- jnz 2f
- NON_GPROF_RET /* success */
-
- /*
- * Failure, but we could end up owning mp_lock anyway due to
- * an interrupt race. lwkt_switch() will clean up the mess
- * and 'block' until the mp_lock is obtained.
- *
- * Create a stack frame for the call so KTR logs the stack
- * backtrace properly.
- */
-2:
- pushl %ebp
- movl %esp,%ebp
- call lwkt_mp_lock_contested
- popl %ebp
-#ifdef INVARIANTS
- movl PCPU(cpuid),%eax /* failure */
- cmpl %eax,mp_lock
- jne 4f
-#endif
- NON_GPROF_RET
-#ifdef INVARIANTS
-4:
- cmpl $0,panicstr /* don't double panic */
- je badmp_get2
- NON_GPROF_RET
-#endif
-
- /*
- * try_mplock() attempts to obtain the MP lock. 1 is returned on
- * success, 0 on failure. We do not have to be in a critical section
- * and interrupts are almost certainly enabled.
- *
- * We must pre-dispose TD_MPCOUNT in order to deal with races in
- * a reasonable way.
- *
- */
-NON_GPROF_ENTRY(try_mplock)
- movl PCPU(cpuid),%ecx
- movl PCPU(curthread),%edx
- incl TD_MPCOUNT(%edx) /* pre-dispose for race */
- cmpl %ecx,mp_lock
- je 1f /* trivial success */
- movl $-1,%eax
- lock cmpxchgl %ecx,mp_lock
- jnz 2f
- /*
- * Success
- */
-#ifdef PARANOID_INVLTLB
- movl %cr3,%eax; movl %eax,%cr3 /* YYY check and remove */
-#endif
-1:
- movl $1,%eax /* success (cmpxchgl good!) */
- NON_GPROF_RET
-
- /*
- * The cmpxchgl failed but we might have raced. Undo the mess by
- * predispoing TD_MPCOUNT and then checking. If TD_MPCOUNT is
- * still non-zero we don't care what state the lock is in (since
- * we obviously didn't own it above), just return failure even if
- * we won the lock in an interrupt race. If TD_MPCOUNT is zero
- * make sure we don't own the lock in case we did win it in a race.
- */
-2:
- decl TD_MPCOUNT(%edx)
- cmpl $0,TD_MPCOUNT(%edx)
- jne 3f
- movl PCPU(cpuid),%eax
- movl $-1,%ecx
- lock cmpxchgl %ecx,mp_lock
-3:
- subl %eax,%eax
- NON_GPROF_RET
-
- /*
- * rel_mplock() releases a previously obtained MP lock.
- *
- * In order to release the MP lock we pre-dispose TD_MPCOUNT for
- * the release and basically repeat the release portion of try_mplock
- * above.
- */
-NON_GPROF_ENTRY(rel_mplock)
- movl PCPU(curthread),%edx
- movl TD_MPCOUNT(%edx),%eax
-#ifdef INVARIANTS
- cmpl $0,%eax
- je badmp_rel
-#endif
- subl $1,%eax
- movl %eax,TD_MPCOUNT(%edx)
- cmpl $0,%eax
- jne 3f
- movl PCPU(cpuid),%eax
- movl $-1,%ecx
- lock cmpxchgl %ecx,mp_lock
- movl mp_lock_contention_mask,%eax
- cmpl $0,%eax
- je 3f
- call lwkt_mp_lock_uncontested
-3:
- NON_GPROF_RET
-
-#ifdef INVARIANTS
-
-badmp_get:
- pushl $bmpsw1
- call panic
-badmp_get2:
- pushl $bmpsw1a
- call panic
-badmp_rel:
- pushl $bmpsw2
- call panic
-
- .data
-
-bmpsw1:
- .asciz "try/get_mplock(): already have lock! %d %p"
-
-bmpsw1a:
- .asciz "try/get_mplock(): failed on count or switch %d %p"
-
-bmpsw2:
- .asciz "rel_mplock(): mpcount already 0 @ %p %p %p %p %p %p %p %p!"
-
-#endif
-
#include <sys/syslog.h>
#endif
#include <sys/signalvar.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#ifndef SMP
#include <machine/asmacros.h>
#include <machine/vm86.h>
#include <ddb/ddb.h>
+
#include <sys/msgport2.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#ifdef SMP
#include <vm/vm_extern.h>
#include <sys/user.h>
+
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <bus/isa/isa.h>
#include <machine/psl.h>
#endif
-/*
- * MP_FREE_LOCK is used by both assembly and C under SMP.
- */
-#ifdef SMP
-#define MP_FREE_LOCK 0xffffffff /* value of lock when free */
-#endif
-
#ifndef LOCORE
-#if defined(_KERNEL) || defined(_UTHREAD)
-
-/*
- * MP LOCK functions for SMP and UP. Under UP the MP lock does not exist
- * but we leave a few functions intact as macros for convenience.
- */
-#ifdef SMP
-
-void get_mplock(void);
-int try_mplock(void);
-void rel_mplock(void);
-int cpu_try_mplock(void);
-void cpu_get_initial_mplock(void);
-
-extern u_int mp_lock;
-
-#define MP_LOCK_HELD() (mp_lock == mycpu->gd_cpuid)
-#define ASSERT_MP_LOCK_HELD(td) KASSERT(MP_LOCK_HELD(), ("MP_LOCK_HELD(): not held thread %p", td))
-
-static __inline void
-cpu_rel_mplock(void)
-{
- mp_lock = MP_FREE_LOCK;
-}
-
-static __inline int
-owner_mplock(void)
-{
- return (mp_lock);
-}
-
-#else
-
-#define get_mplock()
-#define try_mplock() 1
-#define rel_mplock()
-#define owner_mplock() 0 /* always cpu 0 */
-#define MP_LOCK_HELD() (!0)
-#define ASSERT_MP_LOCK_HELD(td)
-
-#endif /* SMP */
-#endif /* _KERNEL || _UTHREAD */
#endif /* LOCORE */
#endif /* !_MACHINE_LOCK_H_ */
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/uio.h>
-#include <sys/thread2.h>
#include <sys/bus_dma.h>
#include <sys/kernel.h>
#include <sys/sysctl.h>
#include <sys/lock.h>
+
+#include <sys/thread2.h>
#include <sys/spinlock2.h>
+#include <sys/mplock2.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <sys/stat.h>
#include <sys/mman.h>
+#include <sys/mplock2.h>
+
/*
* A bcopy that works dring low level boot, before FP is working
*/
--- /dev/null
+/*
+ * SYS/MPLOCK2.H
+ *
+ * Implement the MP lock. Note that debug operations
+ */
+#ifndef _SYS_MPLOCK2_H_
+#define _SYS_MPLOCK2_H_
+
+#ifndef _MACHINE_ATOMIC_H_
+#include <machine/atomic.h>
+#endif
+#ifndef _SYS_THREAD_H_
+#include <sys/thread.h>
+#endif
+#ifndef _SYS_GLOBALDATA_H_
+#include <sys/globaldata.h>
+#endif
+
+#ifdef SMP
+
+#define get_mplock() get_mplock_debug(__FILE__, __LINE__)
+#define try_mplock() try_mplock_debug(__FILE__, __LINE__)
+#define cpu_try_mplock() cpu_try_mplock_debug(__FILE__, __LINE__)
+
+void _get_mplock_contested(const char *file, int line);
+void _try_mplock_contested(const char *file, int line);
+void _cpu_try_mplock_contested(const char *file, int line);
+void _rel_mplock_contested(void);
+void cpu_get_initial_mplock(void);
+void cpu_mplock_contested(void);
+void yield_mplock(struct thread *td);
+
+extern int mp_lock;
+extern int mp_lock_contention_mask;
+extern const char *mp_lock_holder_file;
+extern int mp_lock_holder_line;
+
+/*
+ * Acquire the MP lock, block until we get it.
+ *
+ * In order to acquire the MP lock we must first pre-dispose td_mpcount
+ * for the acquisition and then get the actual lock.
+ *
+ * The contested function is called only if we do not have or are unable
+ * to acquire the actual lock. It will not return until the lock has
+ * been acquired.
+ */
+static __inline
+void
+get_mplock_debug(const char *file, int line)
+{
+ globaldata_t gd = mycpu;
+ thread_t td = gd->gd_curthread;
+
+ ++td->td_mpcount;
+ if (mp_lock != gd->gd_cpuid) {
+ if (atomic_cmpset_int(&mp_lock, -1, gd->gd_cpuid) == 0)
+ _get_mplock_contested(file, line);
+#ifdef INVARIANTS
+ mp_lock_holder_file = file;
+ mp_lock_holder_line = line;
+#endif
+ }
+}
+
+/*
+ * Release the MP lock
+ *
+ * In order to release the MP lock we must first pre-dispose td_mpcount
+ * for the release and then, if it is 0, release the actual lock.
+ *
+ * The contested function is called only if we are unable to release the
+ * Actual lock. This can occur if we raced an interrupt after decrementing
+ * td_mpcount to 0 and the interrupt acquired and released the lock.
+ *
+ * The function also catches the td_mpcount underflow case because the
+ * lock will be in a released state and thus fail the subsequent release.
+ */
+static __inline
+void
+rel_mplock(void)
+{
+ globaldata_t gd = mycpu;
+ thread_t td = gd->gd_curthread;
+ int n;
+
+ n = --td->td_mpcount;
+ if (n <= 0 && atomic_cmpset_int(&mp_lock, gd->gd_cpuid, -1) == 0)
+ _rel_mplock_contested();
+}
+
+/*
+ * Attempt to acquire the MP lock, returning 0 on failure and 1 on success.
+ *
+ * The contested function is called on failure and typically serves simply
+ * to log the attempt (if debugging enabled).
+ */
+static __inline
+int
+try_mplock_debug(const char *file, int line)
+{
+ globaldata_t gd = mycpu;
+ thread_t td = gd->gd_curthread;
+
+ ++td->td_mpcount;
+ if (mp_lock != gd->gd_cpuid &&
+ atomic_cmpset_int(&mp_lock, -1, gd->gd_cpuid) == 0) {
+ _try_mplock_contested(file, line);
+ return(0);
+ }
+#ifdef INVARIANTS
+ mp_lock_holder_file = file;
+ mp_lock_holder_line = line;
+#endif
+ return(1);
+}
+
+/*
+ * Low level acquisition of the MP lock ignoring curthred->td_mpcount
+ *
+ * This version of try_mplock() is used when the caller has already
+ * predisposed td->td_mpcount.
+ *
+ * Returns non-zero on success, 0 on failure.
+ *
+ * WARNING: Must be called from within a critical section if td_mpcount is
+ * zero, otherwise an itnerrupt race can cause the lock to be lost.
+ */
+static __inline
+int
+cpu_try_mplock_debug(const char *file, int line)
+{
+ globaldata_t gd = mycpu;
+
+ if (mp_lock != gd->gd_cpuid &&
+ atomic_cmpset_int(&mp_lock, -1, gd->gd_cpuid) == 0) {
+ _cpu_try_mplock_contested(file, line);
+ return(0);
+ }
+#ifdef INVARIANTS
+ mp_lock_holder_file = file;
+ mp_lock_holder_line = line;
+#endif
+ return(1);
+}
+
+/*
+ * A cpu wanted the MP lock but could not get it. This function is also
+ * called directly from the LWKT scheduler.
+ *
+ * Reentrant, may be called even if the cpu is already contending the MP
+ * lock.
+ */
+static __inline
+void
+set_mplock_contention_mask(globaldata_t gd)
+{
+ atomic_set_int(&mp_lock_contention_mask, gd->gd_cpumask);
+}
+
+/*
+ * A cpu is no longer contending for the MP lock after previously contending
+ * for it.
+ *
+ * Reentrant, may be called even if the cpu was not previously contending
+ * the MP lock.
+ */
+static __inline
+void
+clr_mplock_contention_mask(globaldata_t gd)
+{
+ atomic_clear_int(&mp_lock_contention_mask, gd->gd_cpumask);
+}
+
+static __inline
+int
+owner_mplock(void)
+{
+ return (mp_lock);
+}
+
+/*
+ * Low level release of the MP lock ignoring curthread->td_mpcount
+ *
+ * WARNING: Caller must be in a critical section, otherwise the
+ * mp_lock can be lost from an interrupt race and we would
+ * end up clearing someone else's lock.
+ */
+static __inline void
+cpu_rel_mplock(void)
+{
+ mp_lock = -1;
+}
+
+#define MP_LOCK_HELD() \
+ (mp_lock == mycpu->gd_cpuid)
+#define ASSERT_MP_LOCK_HELD(td) \
+ KASSERT(MP_LOCK_HELD(), ("MP_LOCK_HELD: Not held thread %p", td))
+
+#else
+
+/*
+ * UNI-PROCESSOR BUILD - Degenerate case macros
+ */
+#define get_mplock()
+#define rel_mplock()
+#define try_mplock() 1
+#define owner_mplock() 0
+#define MP_LOCK_HELD() (!0)
+#define ASSERT_MP_LOCK_HELD(td)
+
+#endif
+
+#endif
int cpu_sanitize_frame (struct trapframe *);
int cpu_sanitize_tls (struct savetls *);
-void cpu_mplock_contested(void);
void cpu_spinlock_contested(void);
void cpu_halt (void);
void cpu_reset (void);
struct thread *, int, int, const char *, ...);
extern void lwkt_exit (void) __dead2;
extern void lwkt_remove_tdallq (struct thread *);
-extern void lwkt_mp_lock_contested(void);
-extern void lwkt_mp_lock_uncontested(void);
#endif
#include <sys/pioctl.h>
#include <machine/limits.h>
-#include <vm/vm_page2.h>
+
#include <sys/buf2.h>
#include <sys/sysref2.h>
+#include <sys/mplock2.h>
+#include <vm/vm_page2.h>
MALLOC_DECLARE(M_DEVFS);
#define DEVFS_BADOP (void *)devfs_badop
#include <sys/file.h>
#include <vm/vm_extern.h>
#include <vfs/fifofs/fifo.h>
+
+#include <sys/mplock2.h>
+
#include "hammer.h"
/*
#include <vm/vm_zone.h>
#include <sys/mutex2.h>
+#include <sys/mplock2.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <sys/file2.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
static int max_proc_mmap;
SYSCTL_INT(_vm, OID_AUTO, max_proc_mmap, CTLFLAG_RW, &max_proc_mmap, 0, "");
#include <sys/lock.h>
#include <sys/conf.h>
#include <sys/stat.h>
-#include <sys/thread2.h>
+
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/swap_pager.h>
#include <vm/vm_zone.h>
+#include <sys/thread2.h>
+#include <sys/mplock2.h>
+
/*
* Indirect driver for multi-controller paging.
*/
#include <vm/pmap.h>
#include <vm/vm_map.h>
+#include <sys/mplock2.h>
+
/*
* obreak_args(char *nsize)
*
#include <sys/spinlock2.h>
#include <sys/sysref2.h>
+#include <sys/mplock2.h>
static struct vmspace_entry *vkernel_find_vmspace(struct vkernel_proc *vkp,
void *id);
projects / dragonfly.git / commitdiff