From 35238fa584e1b78855badc10357c79b8126bcf0f Mon Sep 17 00:00:00 2001
From: Matthew Dillon <dillon@dragonflybsd.org>
Date: Fri, 3 Jun 2005 23:57:34 +0000
Subject: [PATCH] Replace cpu_mb1() and cpu_mb2() with cpu_mfence(),
 cpu_lfence(), cpu_sfence(), and cpu_ccfence().  These provide memory and
 compiler fences to guarentee read/write ordering of memory and to prevent the
 compiler itself from generating reordered code in particular cases.

Most of the cases where care must be taken are tail-chasing FIFO cases,
especially in the IPI messaging code.

At the moment on SMP boxes we use a locked bus cycle on (%%esp) for the
lfence and mfence.  The only other choices are to use a cpuid instruction
or one of the {S,L,M}FENCE instructions.  cpuid is usually horrible, and
the *FENCE instructions do not exist on older cpus.

Linux seems to use the locked bus cycle or *FENCE instruction method.

Although I have no conclusive evidence, a number of crash dumps provided
by David Rhodus has led me to believe that speculative reads by modern cpus,
in particular in HTT situations with Intel cpus, can survive many more
instructions then previously believed.  The only safe solution is to use
an instruction sequence which guarentees proper operation.

I would prefer to avoid the use of a locked bus cycle but at least in the
IPIQ case (the most common case we have to worry about), only one locked
bus cycle is required and then the entire IPIQ can be processed without
further locked cycles.
---
 sys/cpu/i386/include/cpufunc.h      | 57 +++++++++++++++++++++++++++--
 sys/i386/i386/mp_machdep.c          | 13 ++++---
 sys/i386/include/cpufunc.h          | 57 +++++++++++++++++++++++++++--
 sys/kern/kern_clock.c               | 19 +++++++---
 sys/kern/kern_timeout.c             |  4 +-
 sys/kern/kern_umtx.c                |  4 +-
 sys/kern/lwkt_ipiq.c                | 29 +++++++++++----
 sys/kern/lwkt_msgport.c             |  7 ++--
 sys/kern/lwkt_thread.c              | 10 ++---
 sys/kern/lwkt_token.c               | 12 +++---
 sys/kern/vfs_jops.c                 | 13 ++++---
 sys/kern/vfs_journal.c              | 13 ++++---
 sys/platform/pc32/i386/mp_machdep.c | 13 ++++---
 sys/sys/thread2.h                   |  6 +--
 14 files changed, 192 insertions(+), 65 deletions(-)

diff --git a/sys/cpu/i386/include/cpufunc.h b/sys/cpu/i386/include/cpufunc.h
index 71497d511d..b39a5d47d9 100644
--- a/sys/cpu/i386/include/cpufunc.h
+++ b/sys/cpu/i386/include/cpufunc.h
@@ -31,7 +31,7 @@
  * SUCH DAMAGE.
  *
  * $FreeBSD: src/sys/i386/include/cpufunc.h,v 1.96.2.3 2002/04/28 22:50:54 dwmalone Exp $
- * $DragonFly: src/sys/cpu/i386/include/cpufunc.h,v 1.12 2005/06/03 20:20:44 dillon Exp $
+ * $DragonFly: src/sys/cpu/i386/include/cpufunc.h,v 1.13 2005/06/03 23:57:31 dillon Exp $
  */
 
 /*
@@ -142,16 +142,65 @@ cpu_enable_intr(void)
 	__asm __volatile("sti");
 }
 
+/*
+ * Cpu and compiler memory ordering fence.  mfence ensures strong read and
+ * write ordering.
+ *
+ * A serializing or fence instruction is required here.  A locked bus
+ * cycle on data for which we already own cache mastership is the most
+ * portable.
+ */
+static __inline void
+cpu_mfence(void)
+{
+#ifdef SMP
+	__asm __volatile("lock; addl $0,(%%esp)" : : : "memory");
+#else
+	__asm __volatile("" : : : "memory");
+#endif
+}
+
+/*
+ * cpu_lfence() ensures strong read ordering for reads issued prior
+ * to the instruction verses reads issued afterwords.
+ *
+ * A serializing or fence instruction is required here.  A locked bus
+ * cycle on data for which we already own cache mastership is the most
+ * portable.
+ */
 static __inline void
-cpu_mb1(void)
+cpu_lfence(void)
+{
+#ifdef SMP
+	__asm __volatile("lock; addl $0,(%%esp)" : : : "memory");
+#else
+	__asm __volatile("" : : : "memory");
+#endif
+}
+
+/*
+ * cpu_lfence() ensures strong write ordering for writes issued prior
+ * to the instruction verses writes issued afterwords.  Writes are
+ * ordered on intel cpus so we do not actually have to do anything.
+ */
+static __inline void
+cpu_sfence(void)
 {
 	__asm __volatile("" : : : "memory");
 }
 
+/*
+ * cpu_ccfence() prevents the compiler from reordering instructions, in
+ * particular stores, relative to the current cpu.  Use cpu_sfence() if
+ * you need to guarentee ordering by both the compiler and by the cpu.
+ *
+ * This also prevents the compiler from caching memory loads into local
+ * variables across the routine.
+ */
 static __inline void
-cpu_mb2(void)
+cpu_ccfence(void)
 {
-	__asm __volatile("subl %%eax,%%eax; cpuid" : : : "ax", "bx", "cx", "dx", "memory");
+	__asm __volatile("" : : : "memory");
 }
 
 #ifdef _KERNEL
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 8778250f84..f4812b48a2 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -23,7 +23,7 @@
  * SUCH DAMAGE.
  *
  * $FreeBSD: src/sys/i386/i386/mp_machdep.c,v 1.115.2.15 2003/03/14 21:22:35 jhb Exp $
- * $DragonFly: src/sys/i386/i386/Attic/mp_machdep.c,v 1.35 2005/04/26 00:31:05 dillon Exp $
+ * $DragonFly: src/sys/i386/i386/Attic/mp_machdep.c,v 1.36 2005/06/03 23:57:30 dillon Exp $
  */
 
 #include "opt_cpu.h"
@@ -2348,7 +2348,7 @@ ap_init(void)
 	 * trying to send us an IPI.
 	 */
 	smp_startup_mask |= 1 << mycpu->gd_cpuid;
-	cpu_mb1();
+	cpu_mfence();
 
 	/*
 	 * Interlock for finalization.  Wait until mp_finish is non-zero,
@@ -2361,11 +2361,12 @@ ap_init(void)
 	 *
 	 * Note: we are the idle thread, we can only spin.
 	 *
-	 * Note: cpu_mb1() is memory volatile and prevents mp_finish from
-	 * 	 being cached.
+	 * Note: The load fence is memory volatile and prevents the compiler
+	 * from improperly caching mp_finish, and the cpu from improperly
+	 * caching it.
 	 */
 	while (mp_finish == 0)
-	    cpu_mb1();
+	    cpu_lfence();
 	++curthread->td_mpcount;
 	while (cpu_try_mplock() == 0)
 	    ;
@@ -2431,7 +2432,7 @@ ap_finish(void)
 		printf("Finish MP startup");
 	rel_mplock();
 	while (smp_active_mask != smp_startup_mask)
-		cpu_mb1();
+		cpu_lfence();
 	while (try_mplock() == 0)
 		;
 	if (bootverbose)
diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h
index 4627e3cefa..292f1218e1 100644
--- a/sys/i386/include/cpufunc.h
+++ b/sys/i386/include/cpufunc.h
@@ -31,7 +31,7 @@
  * SUCH DAMAGE.
  *
  * $FreeBSD: src/sys/i386/include/cpufunc.h,v 1.96.2.3 2002/04/28 22:50:54 dwmalone Exp $
- * $DragonFly: src/sys/i386/include/Attic/cpufunc.h,v 1.12 2005/06/03 20:20:44 dillon Exp $
+ * $DragonFly: src/sys/i386/include/Attic/cpufunc.h,v 1.13 2005/06/03 23:57:31 dillon Exp $
  */
 
 /*
@@ -142,16 +142,65 @@ cpu_enable_intr(void)
 	__asm __volatile("sti");
 }
 
+/*
+ * Cpu and compiler memory ordering fence.  mfence ensures strong read and
+ * write ordering.
+ *
+ * A serializing or fence instruction is required here.  A locked bus
+ * cycle on data for which we already own cache mastership is the most
+ * portable.
+ */
+static __inline void
+cpu_mfence(void)
+{
+#ifdef SMP
+	__asm __volatile("lock; addl $0,(%%esp)" : : : "memory");
+#else
+	__asm __volatile("" : : : "memory");
+#endif
+}
+
+/*
+ * cpu_lfence() ensures strong read ordering for reads issued prior
+ * to the instruction verses reads issued afterwords.
+ *
+ * A serializing or fence instruction is required here.  A locked bus
+ * cycle on data for which we already own cache mastership is the most
+ * portable.
+ */
 static __inline void
-cpu_mb1(void)
+cpu_lfence(void)
+{
+#ifdef SMP
+	__asm __volatile("lock; addl $0,(%%esp)" : : : "memory");
+#else
+	__asm __volatile("" : : : "memory");
+#endif
+}
+
+/*
+ * cpu_lfence() ensures strong write ordering for writes issued prior
+ * to the instruction verses writes issued afterwords.  Writes are
+ * ordered on intel cpus so we do not actually have to do anything.
+ */
+static __inline void
+cpu_sfence(void)
 {
 	__asm __volatile("" : : : "memory");
 }
 
+/*
+ * cpu_ccfence() prevents the compiler from reordering instructions, in
+ * particular stores, relative to the current cpu.  Use cpu_sfence() if
+ * you need to guarentee ordering by both the compiler and by the cpu.
+ *
+ * This also prevents the compiler from caching memory loads into local
+ * variables across the routine.
+ */
 static __inline void
-cpu_mb2(void)
+cpu_ccfence(void)
 {
-	__asm __volatile("subl %%eax,%%eax; cpuid" : : : "ax", "bx", "cx", "dx", "memory");
+	__asm __volatile("" : : : "memory");
 }
 
 #ifdef _KERNEL
diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index 8878c37111..742325f747 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -70,7 +70,7 @@
  *
  *	@(#)kern_clock.c	8.5 (Berkeley) 1/21/94
  * $FreeBSD: src/sys/kern/kern_clock.c,v 1.105.2.10 2002/10/17 13:19:40 maxim Exp $
- * $DragonFly: src/sys/kern/kern_clock.c,v 1.41 2005/06/01 17:43:42 dillon Exp $
+ * $DragonFly: src/sys/kern/kern_clock.c,v 1.42 2005/06/03 23:57:32 dillon Exp $
  */
 
 #include "opt_ntp.h"
@@ -172,8 +172,16 @@ sysctl_get_basetime(SYSCTL_HANDLER_ARGS)
 {
 	struct timespec *bt;
 	int error;
+	int index;
 
-	bt = &basetime[basetime_index];
+	/*
+	 * Because basetime data and index may be updated by another cpu,
+	 * a load fence is required to ensure that the data we read has
+	 * not been speculatively read relative to a possibly updated index.
+	 */
+	index = basetime_index;
+	cpu_lfence();
+	bt = &basetime[index];
 	error = SYSCTL_OUT(req, bt, sizeof(*bt));
 	return (error);
 }
@@ -297,9 +305,10 @@ set_timeofday(struct timespec *ts)
 	ntp_delta = 0;
 
 	/*
-	 * We now have a new basetime, update the index.
+	 * We now have a new basetime, make sure all other cpus have it,
+	 * then update the index.
 	 */
-	cpu_mb1();
+	cpu_sfence();
 	basetime_index = ni;
 
 	crit_exit();
@@ -478,7 +487,7 @@ hardclock(systimer_t info, struct intrframe *frame)
 	    /*
 	     * Finally, our new basetime is ready to go live!
 	     */
-	    cpu_mb1();
+	    cpu_sfence();
 	    basetime_index = ni;
 	}
 
diff --git a/sys/kern/kern_timeout.c b/sys/kern/kern_timeout.c
index 0bf71a77cc..8a2eba12f2 100644
--- a/sys/kern/kern_timeout.c
+++ b/sys/kern/kern_timeout.c
@@ -70,7 +70,7 @@
  *
  *	From: @(#)kern_clock.c	8.5 (Berkeley) 1/21/94
  * $FreeBSD: src/sys/kern/kern_timeout.c,v 1.59.2.1 2001/11/13 18:24:52 archie Exp $
- * $DragonFly: src/sys/kern/kern_timeout.c,v 1.14 2004/09/19 02:52:26 dillon Exp $
+ * $DragonFly: src/sys/kern/kern_timeout.c,v 1.15 2005/06/03 23:57:32 dillon Exp $
  */
 /*
  * DRAGONFLY BGL STATUS
@@ -494,7 +494,7 @@ callout_stop(struct callout *c)
 		 */
 		int seq;
 
-		cpu_mb1();	/* don't let tgd alias c_gd */
+		cpu_ccfence();	/* don't let tgd alias c_gd */
 		seq = lwkt_send_ipiq(tgd, (void *)callout_stop, c);
 		lwkt_wait_ipiq(tgd, seq);
 	} else 
diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index 971502bfbb..9be9ec17a9 100644
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -31,7 +31,7 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  * 
- * $DragonFly: src/sys/kern/kern_umtx.c,v 1.3 2005/03/22 23:42:53 davidxu Exp $
+ * $DragonFly: src/sys/kern/kern_umtx.c,v 1.4 2005/06/03 23:57:32 dillon Exp $
  */
 
 /*
@@ -135,7 +135,7 @@ umtx_wakeup(struct umtx_wakeup_args *uap)
     vm_paddr_t pa;
     void *waddr;
 
-    cpu_mb2();
+    cpu_mfence();
     if (vm_fault_quick((caddr_t)__DEQUALIFY(int *, uap->ptr), VM_PROT_READ) < 0)
 	return (EFAULT);
     if ((pa = pmap_kextract((vm_offset_t)uap->ptr)) == 0)
diff --git a/sys/kern/lwkt_ipiq.c b/sys/kern/lwkt_ipiq.c
index 5f451a8168..d651ed7c53 100644
--- a/sys/kern/lwkt_ipiq.c
+++ b/sys/kern/lwkt_ipiq.c
@@ -31,7 +31,7 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  * 
- * $DragonFly: src/sys/kern/lwkt_ipiq.c,v 1.10 2005/04/18 01:02:58 dillon Exp $
+ * $DragonFly: src/sys/kern/lwkt_ipiq.c,v 1.11 2005/06/03 23:57:32 dillon Exp $
  */
 
 /*
@@ -182,7 +182,7 @@ lwkt_send_ipiq(globaldata_t target, ipifunc_t func, void *arg)
     windex = ip->ip_windex & MAXCPUFIFO_MASK;
     ip->ip_func[windex] = (ipifunc2_t)func;
     ip->ip_arg[windex] = arg;
-    cpu_mb1();
+    cpu_sfence();
     ++ip->ip_windex;
     --gd->gd_intr_nesting_level;
 
@@ -254,7 +254,7 @@ lwkt_send_ipiq_passive(globaldata_t target, ipifunc_t func, void *arg)
     windex = ip->ip_windex & MAXCPUFIFO_MASK;
     ip->ip_func[windex] = (ipifunc2_t)func;
     ip->ip_arg[windex] = arg;
-    cpu_mb1();
+    cpu_sfence();
     ++ip->ip_windex;
     --gd->gd_intr_nesting_level;
 
@@ -292,7 +292,7 @@ lwkt_send_ipiq_nowait(globaldata_t target, ipifunc_t func, void *arg)
     windex = ip->ip_windex & MAXCPUFIFO_MASK;
     ip->ip_func[windex] = (ipifunc2_t)func;
     ip->ip_arg[windex] = arg;
-    cpu_mb1();
+    cpu_sfence();
     ++ip->ip_windex;
 
     /*
@@ -367,6 +367,12 @@ lwkt_wait_ipiq(globaldata_t target, int seq)
 			printf("LWKT_WAIT_IPIQ WARNING! %d wait %d (%d)\n", mycpu->gd_cpuid, target->gd_cpuid, ip->ip_xindex - seq);
 		if (maxc < -1000000)
 			panic("LWKT_WAIT_IPIQ");
+		/*
+		 * xindex may be modified by another cpu, use a load fence
+		 * to ensure that the loop does not use a speculative value
+		 * (which may improve performance).
+		 */
+		cpu_lfence();
 	    }
 	    write_eflags(eflags);
 	}
@@ -450,7 +456,16 @@ static int
 lwkt_process_ipiq1(lwkt_ipiq_t ip, struct intrframe *frame)
 {
     int ri;
-    int wi = ip->ip_windex;
+    int wi;
+
+    /*
+     * Obtain the current write index, which is modified by a remote cpu.
+     * Issue a load fence to prevent speculative reads of e.g. data written
+     * by the other cpu prior to it updating the index.
+     */
+    wi = ip->ip_windex;
+    cpu_lfence();
+
     /*
      * Note: xindex is only updated after we are sure the function has
      * finished execution.  Beware lwkt_process_ipiq() reentrancy!  The
@@ -460,7 +475,7 @@ lwkt_process_ipiq1(lwkt_ipiq_t ip, struct intrframe *frame)
 	ip->ip_rindex = ri + 1;
 	ri &= MAXCPUFIFO_MASK;
 	ip->ip_func[ri](ip->ip_arg[ri], frame);
-	/* YYY memory barrier */
+	cpu_sfence();
 	ip->ip_xindex = ip->ip_rindex;
     }
 
@@ -709,7 +724,7 @@ lwkt_cpusync_remote2(lwkt_cpusync_t poll)
 	wi = ip->ip_windex & MAXCPUFIFO_MASK;
 	ip->ip_func[wi] = (ipifunc2_t)lwkt_cpusync_remote2;
 	ip->ip_arg[wi] = poll;
-	cpu_mb1();
+	cpu_sfence();
 	++ip->ip_windex;
     }
 }
diff --git a/sys/kern/lwkt_msgport.c b/sys/kern/lwkt_msgport.c
index 85cf350395..27bad55141 100644
--- a/sys/kern/lwkt_msgport.c
+++ b/sys/kern/lwkt_msgport.c
@@ -34,7 +34,7 @@
  * NOTE! This file may be compiled for userland libraries as well as for
  * the kernel.
  *
- * $DragonFly: src/sys/kern/lwkt_msgport.c,v 1.31 2005/01/19 17:41:20 dillon Exp $
+ * $DragonFly: src/sys/kern/lwkt_msgport.c,v 1.32 2005/06/03 23:57:32 dillon Exp $
  */
 
 #ifdef _KERNEL
@@ -502,7 +502,8 @@ lwkt_abortmsg(lwkt_msg_t msg)
 	    port = msg->ms_reply_port;
 	else
 	    port = msg->ms_target_port;
-	cpu_mb1();
+
+	cpu_ccfence();	/* don't let the compiler reload ms_*_port */
 
 	/*
 	 * The chase call must run on the cpu owning the port.  Fully
@@ -529,7 +530,7 @@ lwkt_abortmsg_remote(lwkt_msg_t msg)
 	port = msg->ms_reply_port;
     else
 	port = msg->ms_target_port;
-    cpu_mb1();
+    cpu_ccfence();	/* don't let the compiler reload ms_*_port */
     td = port->mp_td;
     if (td->td_gd != mycpu) {
 	lwkt_send_ipiq(td->td_gd, (ipifunc_t)lwkt_abortmsg_remote, msg);
diff --git a/sys/kern/lwkt_thread.c b/sys/kern/lwkt_thread.c
index 1f2a052e5c..9a589113ce 100644
--- a/sys/kern/lwkt_thread.c
+++ b/sys/kern/lwkt_thread.c
@@ -31,7 +31,7 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  * 
- * $DragonFly: src/sys/kern/lwkt_thread.c,v 1.73 2005/06/03 22:57:27 dillon Exp $
+ * $DragonFly: src/sys/kern/lwkt_thread.c,v 1.74 2005/06/03 23:57:32 dillon Exp $
  */
 
 /*
@@ -976,9 +976,10 @@ lwkt_acquire(thread_t td)
 
     gd = td->td_gd;
     mygd = mycpu;
+    cpu_lfence();
     KKASSERT((td->td_flags & TDF_RUNQ) == 0);
     while (td->td_flags & TDF_RUNNING)	/* XXX spin */
-	cpu_mb1();
+	cpu_lfence();
     if (gd != mygd) {
 	crit_enter_gd(mygd);
 	TAILQ_REMOVE(&gd->gd_tdallq, td, td_allq);	/* protected by BGL */
@@ -1110,7 +1111,6 @@ lwkt_setcpu_self(globaldata_t rgd)
 	lwkt_switch();
 	/* we are now on the target cpu */
 	crit_exit_quick(td);
-	cpu_mb1();
     }
 #endif
 }
@@ -1135,9 +1135,9 @@ lwkt_setcpu_remote(void *arg)
     globaldata_t gd = mycpu;
 
     while (td->td_flags & TDF_RUNNING)
-	cpu_mb1();
+	cpu_lfence();
     td->td_gd = gd;
-    cpu_mb2();
+    cpu_sfence();
     td->td_flags &= ~TDF_MIGRATING;
     _lwkt_enqueue(td);
 }
diff --git a/sys/kern/lwkt_token.c b/sys/kern/lwkt_token.c
index 2def4544ff..f4f061ab0c 100644
--- a/sys/kern/lwkt_token.c
+++ b/sys/kern/lwkt_token.c
@@ -31,7 +31,7 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  * 
- * $DragonFly: src/sys/kern/lwkt_token.c,v 1.14 2005/06/02 21:55:22 dillon Exp $
+ * $DragonFly: src/sys/kern/lwkt_token.c,v 1.15 2005/06/03 23:57:32 dillon Exp $
  */
 
 #ifdef _KERNEL
@@ -134,7 +134,7 @@ lwkt_chktokens(thread_t td)
     for (refs = td->td_toks; refs; refs = refs->tr_next) {
 	tok = refs->tr_tok;
 	if ((dgd = tok->t_cpu) != gd) {
-	    cpu_mb1();
+	    cpu_ccfence();	/* don't let the compiler reload tok->t_cpu */
 	    r = 0;
 
 	    /*
@@ -238,7 +238,7 @@ _lwkt_gettokref(lwkt_tokref_t ref)
      * avoid using a critical section.
      */
     ref->tr_next = td->td_toks;
-    cpu_mb1();		/* order memory / we can be interrupted */
+    cpu_ccfence();	/* prevent compiler reordering */
     td->td_toks = ref;
 
     /*
@@ -322,7 +322,7 @@ _lwkt_trytokref(lwkt_tokref_t ref)
      * avoid using a critical section.
      */
     ref->tr_next = td->td_toks;
-    cpu_mb1();		/* order memory / we can be interrupted */
+    cpu_ccfence();	/* prevent compiler reordering */
     td->td_toks = ref;
 
     /*
@@ -490,7 +490,7 @@ lwkt_reqtoken_remote(void *data)
     if (lwkt_oktogiveaway_token(tok)) {
 	if (tok->t_cpu == gd)
 	    tok->t_cpu = ref->tr_reqgd;
-	cpu_mb1();
+	cpu_ccfence();	/* prevent compiler reordering */
 	ref->tr_magic = LWKT_TOKREF_MAGIC1;
     } else {
 	ref->tr_gdreqnext = gd->gd_tokreqbase;
@@ -519,7 +519,7 @@ lwkt_drain_token_requests(void)
 	KKASSERT(ref->tr_magic == LWKT_TOKREF_MAGIC2);
 	if (ref->tr_tok->t_cpu == gd)
 	    ref->tr_tok->t_cpu = ref->tr_reqgd;
-	cpu_mb1();
+	cpu_ccfence();	/* prevent compiler reordering */
 	ref->tr_magic = LWKT_TOKREF_MAGIC1;
     }
 }
diff --git a/sys/kern/vfs_jops.c b/sys/kern/vfs_jops.c
index 930e83013b..2436d10277 100644
--- a/sys/kern/vfs_jops.c
+++ b/sys/kern/vfs_jops.c
@@ -31,7 +31,7 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $DragonFly: src/sys/kern/vfs_jops.c,v 1.12 2005/03/22 22:13:28 dillon Exp $
+ * $DragonFly: src/sys/kern/vfs_jops.c,v 1.13 2005/06/03 23:57:32 dillon Exp $
  */
 /*
  * Each mount point may have zero or more independantly configured journals
@@ -712,7 +712,7 @@ journal_build_pad(struct journal_rawrecbeg *rawp, int recsize, int64_t transid)
     rawp->transid = transid;
     /*
      * WARNING, rendp may overlap rawp->seqno.  This is necessary to
-     * allow PAD records to fit in 16 bytes.  Use cpu_mb1() to
+     * allow PAD records to fit in 16 bytes.  Use cpu_ccfence() to
      * hopefully cause the compiler to not make any assumptions.
      */
     rendp = (void *)((char *)rawp + rawp->recsize - sizeof(*rendp));
@@ -722,9 +722,10 @@ journal_build_pad(struct journal_rawrecbeg *rawp, int recsize, int64_t transid)
 
     /*
      * Set the begin magic last.  This is what will allow the journal
-     * thread to write the record out.
+     * thread to write the record out.  Use a store fence to prevent
+     * compiler and cpu reordering of the writes.
      */
-    cpu_mb1();
+    cpu_sfence();
     rawp->begmagic = JREC_BEGMAGIC;
 }
 
@@ -857,7 +858,7 @@ journal_reserve(struct journal *jo, struct journal_rawrecbeg **rawpp,
 	 *
 	 * Note that stream records are always 16-byte aligned.
 	 */
-	cpu_mb1();
+	cpu_sfence();
 	jo->fifo.windex += (req + 15) & ~15;
 	*rawpp = rawp;
 	return(rawp + 1);
@@ -1049,7 +1050,7 @@ journal_commit(struct journal *jo, struct journal_rawrecbeg **rawpp,
      */
     if (closeout)
 	rawp->streamid |= JREC_STREAMCTL_END;
-    cpu_mb1();			/* memory barrier */
+    cpu_sfence();		/* memory and compiler barrier */
     rawp->begmagic = JREC_BEGMAGIC;
 
     journal_commit_wakeup(jo);
diff --git a/sys/kern/vfs_journal.c b/sys/kern/vfs_journal.c
index 4351d69164..0d8cff52c8 100644
--- a/sys/kern/vfs_journal.c
+++ b/sys/kern/vfs_journal.c
@@ -31,7 +31,7 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $DragonFly: src/sys/kern/vfs_journal.c,v 1.12 2005/03/22 22:13:28 dillon Exp $
+ * $DragonFly: src/sys/kern/vfs_journal.c,v 1.13 2005/06/03 23:57:32 dillon Exp $
  */
 /*
  * Each mount point may have zero or more independantly configured journals
@@ -712,7 +712,7 @@ journal_build_pad(struct journal_rawrecbeg *rawp, int recsize, int64_t transid)
     rawp->transid = transid;
     /*
      * WARNING, rendp may overlap rawp->seqno.  This is necessary to
-     * allow PAD records to fit in 16 bytes.  Use cpu_mb1() to
+     * allow PAD records to fit in 16 bytes.  Use cpu_ccfence() to
      * hopefully cause the compiler to not make any assumptions.
      */
     rendp = (void *)((char *)rawp + rawp->recsize - sizeof(*rendp));
@@ -722,9 +722,10 @@ journal_build_pad(struct journal_rawrecbeg *rawp, int recsize, int64_t transid)
 
     /*
      * Set the begin magic last.  This is what will allow the journal
-     * thread to write the record out.
+     * thread to write the record out.  Use a store fence to prevent
+     * compiler and cpu reordering of the writes.
      */
-    cpu_mb1();
+    cpu_sfence();
     rawp->begmagic = JREC_BEGMAGIC;
 }
 
@@ -857,7 +858,7 @@ journal_reserve(struct journal *jo, struct journal_rawrecbeg **rawpp,
 	 *
 	 * Note that stream records are always 16-byte aligned.
 	 */
-	cpu_mb1();
+	cpu_sfence();
 	jo->fifo.windex += (req + 15) & ~15;
 	*rawpp = rawp;
 	return(rawp + 1);
@@ -1049,7 +1050,7 @@ journal_commit(struct journal *jo, struct journal_rawrecbeg **rawpp,
      */
     if (closeout)
 	rawp->streamid |= JREC_STREAMCTL_END;
-    cpu_mb1();			/* memory barrier */
+    cpu_sfence();		/* memory and compiler barrier */
     rawp->begmagic = JREC_BEGMAGIC;
 
     journal_commit_wakeup(jo);
diff --git a/sys/platform/pc32/i386/mp_machdep.c b/sys/platform/pc32/i386/mp_machdep.c
index 96074235ee..299dab1f5b 100644
--- a/sys/platform/pc32/i386/mp_machdep.c
+++ b/sys/platform/pc32/i386/mp_machdep.c
@@ -23,7 +23,7 @@
  * SUCH DAMAGE.
  *
  * $FreeBSD: src/sys/i386/i386/mp_machdep.c,v 1.115.2.15 2003/03/14 21:22:35 jhb Exp $
- * $DragonFly: src/sys/platform/pc32/i386/mp_machdep.c,v 1.35 2005/04/26 00:31:05 dillon Exp $
+ * $DragonFly: src/sys/platform/pc32/i386/mp_machdep.c,v 1.36 2005/06/03 23:57:30 dillon Exp $
  */
 
 #include "opt_cpu.h"
@@ -2348,7 +2348,7 @@ ap_init(void)
 	 * trying to send us an IPI.
 	 */
 	smp_startup_mask |= 1 << mycpu->gd_cpuid;
-	cpu_mb1();
+	cpu_mfence();
 
 	/*
 	 * Interlock for finalization.  Wait until mp_finish is non-zero,
@@ -2361,11 +2361,12 @@ ap_init(void)
 	 *
 	 * Note: we are the idle thread, we can only spin.
 	 *
-	 * Note: cpu_mb1() is memory volatile and prevents mp_finish from
-	 * 	 being cached.
+	 * Note: The load fence is memory volatile and prevents the compiler
+	 * from improperly caching mp_finish, and the cpu from improperly
+	 * caching it.
 	 */
 	while (mp_finish == 0)
-	    cpu_mb1();
+	    cpu_lfence();
 	++curthread->td_mpcount;
 	while (cpu_try_mplock() == 0)
 	    ;
@@ -2431,7 +2432,7 @@ ap_finish(void)
 		printf("Finish MP startup");
 	rel_mplock();
 	while (smp_active_mask != smp_startup_mask)
-		cpu_mb1();
+		cpu_lfence();
 	while (try_mplock() == 0)
 		;
 	if (bootverbose)
diff --git a/sys/sys/thread2.h b/sys/sys/thread2.h
index 04447895e0..b04eba9889 100644
--- a/sys/sys/thread2.h
+++ b/sys/sys/thread2.h
@@ -8,7 +8,7 @@
  *	on a different cpu will not be immediately scheduled by a yield() on
  *	this cpu.
  *
- * $DragonFly: src/sys/sys/thread2.h,v 1.17 2004/10/12 19:29:29 dillon Exp $
+ * $DragonFly: src/sys/sys/thread2.h,v 1.18 2005/06/03 23:57:34 dillon Exp $
  */
 
 #ifndef _SYS_THREAD2_H_
@@ -84,7 +84,7 @@ crit_exit(void)
     if (td->td_pri < 0)
 	crit_panic();
 #endif
-    cpu_mb1();	/* must flush td_pri before checking gd_reqflags */
+    cpu_ccfence();	/* prevent compiler reordering */
     if (td->td_gd->gd_reqflags && td->td_pri < TDPRI_CRIT)
 	lwkt_yield_quick();
 }
@@ -95,7 +95,7 @@ crit_exit_quick(struct thread *curtd)
     globaldata_t gd = curtd->td_gd;
 
     curtd->td_pri -= TDPRI_CRIT;
-    cpu_mb1();	/* must flush td_pri before checking gd_reqflags */
+    cpu_ccfence();	/* prevent compiler reordering */
     if (gd->gd_reqflags && curtd->td_pri < TDPRI_CRIT)
 	lwkt_yield_quick();
 }
-- 
2.41.0