* SUCH DAMAGE.
*
* $FreeBSD: src/sys/i386/include/cpufunc.h,v 1.96.2.3 2002/04/28 22:50:54 dwmalone Exp $
- * $DragonFly: src/sys/cpu/i386/include/cpufunc.h,v 1.12 2005/06/03 20:20:44 dillon Exp $
+ * $DragonFly: src/sys/cpu/i386/include/cpufunc.h,v 1.13 2005/06/03 23:57:31 dillon Exp $
*/
/*
__asm __volatile("sti");
}
+/*
+ * Cpu and compiler memory ordering fence. mfence ensures strong read and
+ * write ordering.
+ *
+ * A serializing or fence instruction is required here. A locked bus
+ * cycle on data for which we already own cache mastership is the most
+ * portable.
+ */
+static __inline void
+cpu_mfence(void)
+{
+#ifdef SMP
+ __asm __volatile("lock; addl $0,(%%esp)" : : : "memory");
+#else
+ __asm __volatile("" : : : "memory");
+#endif
+}
+
+/*
+ * cpu_lfence() ensures strong read ordering for reads issued prior
+ * to the instruction verses reads issued afterwords.
+ *
+ * A serializing or fence instruction is required here. A locked bus
+ * cycle on data for which we already own cache mastership is the most
+ * portable.
+ */
static __inline void
-cpu_mb1(void)
+cpu_lfence(void)
+{
+#ifdef SMP
+ __asm __volatile("lock; addl $0,(%%esp)" : : : "memory");
+#else
+ __asm __volatile("" : : : "memory");
+#endif
+}
+
+/*
+ * cpu_lfence() ensures strong write ordering for writes issued prior
+ * to the instruction verses writes issued afterwords. Writes are
+ * ordered on intel cpus so we do not actually have to do anything.
+ */
+static __inline void
+cpu_sfence(void)
{
__asm __volatile("" : : : "memory");
}
+/*
+ * cpu_ccfence() prevents the compiler from reordering instructions, in
+ * particular stores, relative to the current cpu. Use cpu_sfence() if
+ * you need to guarentee ordering by both the compiler and by the cpu.
+ *
+ * This also prevents the compiler from caching memory loads into local
+ * variables across the routine.
+ */
static __inline void
-cpu_mb2(void)
+cpu_ccfence(void)
{
- __asm __volatile("subl %%eax,%%eax; cpuid" : : : "ax", "bx", "cx", "dx", "memory");
+ __asm __volatile("" : : : "memory");
}
#ifdef _KERNEL
* SUCH DAMAGE.
*
* $FreeBSD: src/sys/i386/i386/mp_machdep.c,v 1.115.2.15 2003/03/14 21:22:35 jhb Exp $
- * $DragonFly: src/sys/i386/i386/Attic/mp_machdep.c,v 1.35 2005/04/26 00:31:05 dillon Exp $
+ * $DragonFly: src/sys/i386/i386/Attic/mp_machdep.c,v 1.36 2005/06/03 23:57:30 dillon Exp $
*/
#include "opt_cpu.h"
* trying to send us an IPI.
*/
smp_startup_mask |= 1 << mycpu->gd_cpuid;
- cpu_mb1();
+ cpu_mfence();
/*
* Interlock for finalization. Wait until mp_finish is non-zero,
*
* Note: we are the idle thread, we can only spin.
*
- * Note: cpu_mb1() is memory volatile and prevents mp_finish from
- * being cached.
+ * Note: The load fence is memory volatile and prevents the compiler
+ * from improperly caching mp_finish, and the cpu from improperly
+ * caching it.
*/
while (mp_finish == 0)
- cpu_mb1();
+ cpu_lfence();
++curthread->td_mpcount;
while (cpu_try_mplock() == 0)
;
printf("Finish MP startup");
rel_mplock();
while (smp_active_mask != smp_startup_mask)
- cpu_mb1();
+ cpu_lfence();
while (try_mplock() == 0)
;
if (bootverbose)
* SUCH DAMAGE.
*
* $FreeBSD: src/sys/i386/include/cpufunc.h,v 1.96.2.3 2002/04/28 22:50:54 dwmalone Exp $
- * $DragonFly: src/sys/i386/include/Attic/cpufunc.h,v 1.12 2005/06/03 20:20:44 dillon Exp $
+ * $DragonFly: src/sys/i386/include/Attic/cpufunc.h,v 1.13 2005/06/03 23:57:31 dillon Exp $
*/
/*
__asm __volatile("sti");
}
+/*
+ * Cpu and compiler memory ordering fence. mfence ensures strong read and
+ * write ordering.
+ *
+ * A serializing or fence instruction is required here. A locked bus
+ * cycle on data for which we already own cache mastership is the most
+ * portable.
+ */
+static __inline void
+cpu_mfence(void)
+{
+#ifdef SMP
+ __asm __volatile("lock; addl $0,(%%esp)" : : : "memory");
+#else
+ __asm __volatile("" : : : "memory");
+#endif
+}
+
+/*
+ * cpu_lfence() ensures strong read ordering for reads issued prior
+ * to the instruction verses reads issued afterwords.
+ *
+ * A serializing or fence instruction is required here. A locked bus
+ * cycle on data for which we already own cache mastership is the most
+ * portable.
+ */
static __inline void
-cpu_mb1(void)
+cpu_lfence(void)
+{
+#ifdef SMP
+ __asm __volatile("lock; addl $0,(%%esp)" : : : "memory");
+#else
+ __asm __volatile("" : : : "memory");
+#endif
+}
+
+/*
+ * cpu_lfence() ensures strong write ordering for writes issued prior
+ * to the instruction verses writes issued afterwords. Writes are
+ * ordered on intel cpus so we do not actually have to do anything.
+ */
+static __inline void
+cpu_sfence(void)
{
__asm __volatile("" : : : "memory");
}
+/*
+ * cpu_ccfence() prevents the compiler from reordering instructions, in
+ * particular stores, relative to the current cpu. Use cpu_sfence() if
+ * you need to guarentee ordering by both the compiler and by the cpu.
+ *
+ * This also prevents the compiler from caching memory loads into local
+ * variables across the routine.
+ */
static __inline void
-cpu_mb2(void)
+cpu_ccfence(void)
{
- __asm __volatile("subl %%eax,%%eax; cpuid" : : : "ax", "bx", "cx", "dx", "memory");
+ __asm __volatile("" : : : "memory");
}
#ifdef _KERNEL
*
* @(#)kern_clock.c 8.5 (Berkeley) 1/21/94
* $FreeBSD: src/sys/kern/kern_clock.c,v 1.105.2.10 2002/10/17 13:19:40 maxim Exp $
- * $DragonFly: src/sys/kern/kern_clock.c,v 1.41 2005/06/01 17:43:42 dillon Exp $
+ * $DragonFly: src/sys/kern/kern_clock.c,v 1.42 2005/06/03 23:57:32 dillon Exp $
*/
#include "opt_ntp.h"
{
struct timespec *bt;
int error;
+ int index;
- bt = &basetime[basetime_index];
+ /*
+ * Because basetime data and index may be updated by another cpu,
+ * a load fence is required to ensure that the data we read has
+ * not been speculatively read relative to a possibly updated index.
+ */
+ index = basetime_index;
+ cpu_lfence();
+ bt = &basetime[index];
error = SYSCTL_OUT(req, bt, sizeof(*bt));
return (error);
}
ntp_delta = 0;
/*
- * We now have a new basetime, update the index.
+ * We now have a new basetime, make sure all other cpus have it,
+ * then update the index.
*/
- cpu_mb1();
+ cpu_sfence();
basetime_index = ni;
crit_exit();
/*
* Finally, our new basetime is ready to go live!
*/
- cpu_mb1();
+ cpu_sfence();
basetime_index = ni;
}
*
* From: @(#)kern_clock.c 8.5 (Berkeley) 1/21/94
* $FreeBSD: src/sys/kern/kern_timeout.c,v 1.59.2.1 2001/11/13 18:24:52 archie Exp $
- * $DragonFly: src/sys/kern/kern_timeout.c,v 1.14 2004/09/19 02:52:26 dillon Exp $
+ * $DragonFly: src/sys/kern/kern_timeout.c,v 1.15 2005/06/03 23:57:32 dillon Exp $
*/
/*
* DRAGONFLY BGL STATUS
*/
int seq;
- cpu_mb1(); /* don't let tgd alias c_gd */
+ cpu_ccfence(); /* don't let tgd alias c_gd */
seq = lwkt_send_ipiq(tgd, (void *)callout_stop, c);
lwkt_wait_ipiq(tgd, seq);
} else
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $DragonFly: src/sys/kern/kern_umtx.c,v 1.3 2005/03/22 23:42:53 davidxu Exp $
+ * $DragonFly: src/sys/kern/kern_umtx.c,v 1.4 2005/06/03 23:57:32 dillon Exp $
*/
/*
vm_paddr_t pa;
void *waddr;
- cpu_mb2();
+ cpu_mfence();
if (vm_fault_quick((caddr_t)__DEQUALIFY(int *, uap->ptr), VM_PROT_READ) < 0)
return (EFAULT);
if ((pa = pmap_kextract((vm_offset_t)uap->ptr)) == 0)
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $DragonFly: src/sys/kern/lwkt_ipiq.c,v 1.10 2005/04/18 01:02:58 dillon Exp $
+ * $DragonFly: src/sys/kern/lwkt_ipiq.c,v 1.11 2005/06/03 23:57:32 dillon Exp $
*/
/*
windex = ip->ip_windex & MAXCPUFIFO_MASK;
ip->ip_func[windex] = (ipifunc2_t)func;
ip->ip_arg[windex] = arg;
- cpu_mb1();
+ cpu_sfence();
++ip->ip_windex;
--gd->gd_intr_nesting_level;
windex = ip->ip_windex & MAXCPUFIFO_MASK;
ip->ip_func[windex] = (ipifunc2_t)func;
ip->ip_arg[windex] = arg;
- cpu_mb1();
+ cpu_sfence();
++ip->ip_windex;
--gd->gd_intr_nesting_level;
windex = ip->ip_windex & MAXCPUFIFO_MASK;
ip->ip_func[windex] = (ipifunc2_t)func;
ip->ip_arg[windex] = arg;
- cpu_mb1();
+ cpu_sfence();
++ip->ip_windex;
/*
printf("LWKT_WAIT_IPIQ WARNING! %d wait %d (%d)\n", mycpu->gd_cpuid, target->gd_cpuid, ip->ip_xindex - seq);
if (maxc < -1000000)
panic("LWKT_WAIT_IPIQ");
+ /*
+ * xindex may be modified by another cpu, use a load fence
+ * to ensure that the loop does not use a speculative value
+ * (which may improve performance).
+ */
+ cpu_lfence();
}
write_eflags(eflags);
}
lwkt_process_ipiq1(lwkt_ipiq_t ip, struct intrframe *frame)
{
int ri;
- int wi = ip->ip_windex;
+ int wi;
+
+ /*
+ * Obtain the current write index, which is modified by a remote cpu.
+ * Issue a load fence to prevent speculative reads of e.g. data written
+ * by the other cpu prior to it updating the index.
+ */
+ wi = ip->ip_windex;
+ cpu_lfence();
+
/*
* Note: xindex is only updated after we are sure the function has
* finished execution. Beware lwkt_process_ipiq() reentrancy! The
ip->ip_rindex = ri + 1;
ri &= MAXCPUFIFO_MASK;
ip->ip_func[ri](ip->ip_arg[ri], frame);
- /* YYY memory barrier */
+ cpu_sfence();
ip->ip_xindex = ip->ip_rindex;
}
wi = ip->ip_windex & MAXCPUFIFO_MASK;
ip->ip_func[wi] = (ipifunc2_t)lwkt_cpusync_remote2;
ip->ip_arg[wi] = poll;
- cpu_mb1();
+ cpu_sfence();
++ip->ip_windex;
}
}
* NOTE! This file may be compiled for userland libraries as well as for
* the kernel.
*
- * $DragonFly: src/sys/kern/lwkt_msgport.c,v 1.31 2005/01/19 17:41:20 dillon Exp $
+ * $DragonFly: src/sys/kern/lwkt_msgport.c,v 1.32 2005/06/03 23:57:32 dillon Exp $
*/
#ifdef _KERNEL
port = msg->ms_reply_port;
else
port = msg->ms_target_port;
- cpu_mb1();
+
+ cpu_ccfence(); /* don't let the compiler reload ms_*_port */
/*
* The chase call must run on the cpu owning the port. Fully
port = msg->ms_reply_port;
else
port = msg->ms_target_port;
- cpu_mb1();
+ cpu_ccfence(); /* don't let the compiler reload ms_*_port */
td = port->mp_td;
if (td->td_gd != mycpu) {
lwkt_send_ipiq(td->td_gd, (ipifunc_t)lwkt_abortmsg_remote, msg);
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $DragonFly: src/sys/kern/lwkt_thread.c,v 1.73 2005/06/03 22:57:27 dillon Exp $
+ * $DragonFly: src/sys/kern/lwkt_thread.c,v 1.74 2005/06/03 23:57:32 dillon Exp $
*/
/*
gd = td->td_gd;
mygd = mycpu;
+ cpu_lfence();
KKASSERT((td->td_flags & TDF_RUNQ) == 0);
while (td->td_flags & TDF_RUNNING) /* XXX spin */
- cpu_mb1();
+ cpu_lfence();
if (gd != mygd) {
crit_enter_gd(mygd);
TAILQ_REMOVE(&gd->gd_tdallq, td, td_allq); /* protected by BGL */
lwkt_switch();
/* we are now on the target cpu */
crit_exit_quick(td);
- cpu_mb1();
}
#endif
}
globaldata_t gd = mycpu;
while (td->td_flags & TDF_RUNNING)
- cpu_mb1();
+ cpu_lfence();
td->td_gd = gd;
- cpu_mb2();
+ cpu_sfence();
td->td_flags &= ~TDF_MIGRATING;
_lwkt_enqueue(td);
}
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $DragonFly: src/sys/kern/lwkt_token.c,v 1.14 2005/06/02 21:55:22 dillon Exp $
+ * $DragonFly: src/sys/kern/lwkt_token.c,v 1.15 2005/06/03 23:57:32 dillon Exp $
*/
#ifdef _KERNEL
for (refs = td->td_toks; refs; refs = refs->tr_next) {
tok = refs->tr_tok;
if ((dgd = tok->t_cpu) != gd) {
- cpu_mb1();
+ cpu_ccfence(); /* don't let the compiler reload tok->t_cpu */
r = 0;
/*
* avoid using a critical section.
*/
ref->tr_next = td->td_toks;
- cpu_mb1(); /* order memory / we can be interrupted */
+ cpu_ccfence(); /* prevent compiler reordering */
td->td_toks = ref;
/*
* avoid using a critical section.
*/
ref->tr_next = td->td_toks;
- cpu_mb1(); /* order memory / we can be interrupted */
+ cpu_ccfence(); /* prevent compiler reordering */
td->td_toks = ref;
/*
if (lwkt_oktogiveaway_token(tok)) {
if (tok->t_cpu == gd)
tok->t_cpu = ref->tr_reqgd;
- cpu_mb1();
+ cpu_ccfence(); /* prevent compiler reordering */
ref->tr_magic = LWKT_TOKREF_MAGIC1;
} else {
ref->tr_gdreqnext = gd->gd_tokreqbase;
KKASSERT(ref->tr_magic == LWKT_TOKREF_MAGIC2);
if (ref->tr_tok->t_cpu == gd)
ref->tr_tok->t_cpu = ref->tr_reqgd;
- cpu_mb1();
+ cpu_ccfence(); /* prevent compiler reordering */
ref->tr_magic = LWKT_TOKREF_MAGIC1;
}
}
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $DragonFly: src/sys/kern/vfs_jops.c,v 1.12 2005/03/22 22:13:28 dillon Exp $
+ * $DragonFly: src/sys/kern/vfs_jops.c,v 1.13 2005/06/03 23:57:32 dillon Exp $
*/
/*
* Each mount point may have zero or more independantly configured journals
rawp->transid = transid;
/*
* WARNING, rendp may overlap rawp->seqno. This is necessary to
- * allow PAD records to fit in 16 bytes. Use cpu_mb1() to
+ * allow PAD records to fit in 16 bytes. Use cpu_ccfence() to
* hopefully cause the compiler to not make any assumptions.
*/
rendp = (void *)((char *)rawp + rawp->recsize - sizeof(*rendp));
/*
* Set the begin magic last. This is what will allow the journal
- * thread to write the record out.
+ * thread to write the record out. Use a store fence to prevent
+ * compiler and cpu reordering of the writes.
*/
- cpu_mb1();
+ cpu_sfence();
rawp->begmagic = JREC_BEGMAGIC;
}
*
* Note that stream records are always 16-byte aligned.
*/
- cpu_mb1();
+ cpu_sfence();
jo->fifo.windex += (req + 15) & ~15;
*rawpp = rawp;
return(rawp + 1);
*/
if (closeout)
rawp->streamid |= JREC_STREAMCTL_END;
- cpu_mb1(); /* memory barrier */
+ cpu_sfence(); /* memory and compiler barrier */
rawp->begmagic = JREC_BEGMAGIC;
journal_commit_wakeup(jo);
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $DragonFly: src/sys/kern/vfs_journal.c,v 1.12 2005/03/22 22:13:28 dillon Exp $
+ * $DragonFly: src/sys/kern/vfs_journal.c,v 1.13 2005/06/03 23:57:32 dillon Exp $
*/
/*
* Each mount point may have zero or more independantly configured journals
rawp->transid = transid;
/*
* WARNING, rendp may overlap rawp->seqno. This is necessary to
- * allow PAD records to fit in 16 bytes. Use cpu_mb1() to
+ * allow PAD records to fit in 16 bytes. Use cpu_ccfence() to
* hopefully cause the compiler to not make any assumptions.
*/
rendp = (void *)((char *)rawp + rawp->recsize - sizeof(*rendp));
/*
* Set the begin magic last. This is what will allow the journal
- * thread to write the record out.
+ * thread to write the record out. Use a store fence to prevent
+ * compiler and cpu reordering of the writes.
*/
- cpu_mb1();
+ cpu_sfence();
rawp->begmagic = JREC_BEGMAGIC;
}
*
* Note that stream records are always 16-byte aligned.
*/
- cpu_mb1();
+ cpu_sfence();
jo->fifo.windex += (req + 15) & ~15;
*rawpp = rawp;
return(rawp + 1);
*/
if (closeout)
rawp->streamid |= JREC_STREAMCTL_END;
- cpu_mb1(); /* memory barrier */
+ cpu_sfence(); /* memory and compiler barrier */
rawp->begmagic = JREC_BEGMAGIC;
journal_commit_wakeup(jo);
* SUCH DAMAGE.
*
* $FreeBSD: src/sys/i386/i386/mp_machdep.c,v 1.115.2.15 2003/03/14 21:22:35 jhb Exp $
- * $DragonFly: src/sys/platform/pc32/i386/mp_machdep.c,v 1.35 2005/04/26 00:31:05 dillon Exp $
+ * $DragonFly: src/sys/platform/pc32/i386/mp_machdep.c,v 1.36 2005/06/03 23:57:30 dillon Exp $
*/
#include "opt_cpu.h"
* trying to send us an IPI.
*/
smp_startup_mask |= 1 << mycpu->gd_cpuid;
- cpu_mb1();
+ cpu_mfence();
/*
* Interlock for finalization. Wait until mp_finish is non-zero,
*
* Note: we are the idle thread, we can only spin.
*
- * Note: cpu_mb1() is memory volatile and prevents mp_finish from
- * being cached.
+ * Note: The load fence is memory volatile and prevents the compiler
+ * from improperly caching mp_finish, and the cpu from improperly
+ * caching it.
*/
while (mp_finish == 0)
- cpu_mb1();
+ cpu_lfence();
++curthread->td_mpcount;
while (cpu_try_mplock() == 0)
;
printf("Finish MP startup");
rel_mplock();
while (smp_active_mask != smp_startup_mask)
- cpu_mb1();
+ cpu_lfence();
while (try_mplock() == 0)
;
if (bootverbose)
* on a different cpu will not be immediately scheduled by a yield() on
* this cpu.
*
- * $DragonFly: src/sys/sys/thread2.h,v 1.17 2004/10/12 19:29:29 dillon Exp $
+ * $DragonFly: src/sys/sys/thread2.h,v 1.18 2005/06/03 23:57:34 dillon Exp $
*/
#ifndef _SYS_THREAD2_H_
if (td->td_pri < 0)
crit_panic();
#endif
- cpu_mb1(); /* must flush td_pri before checking gd_reqflags */
+ cpu_ccfence(); /* prevent compiler reordering */
if (td->td_gd->gd_reqflags && td->td_pri < TDPRI_CRIT)
lwkt_yield_quick();
}
globaldata_t gd = curtd->td_gd;
curtd->td_pri -= TDPRI_CRIT;
- cpu_mb1(); /* must flush td_pri before checking gd_reqflags */
+ cpu_ccfence(); /* prevent compiler reordering */
if (gd->gd_reqflags && curtd->td_pri < TDPRI_CRIT)
lwkt_yield_quick();
}
projects / dragonfly.git / commitdiff