From: Matthew Dillon <dillon@apollo.backplane.com>
Date: Fri, 4 Nov 2011 05:25:31 +0000 (-0700)
Subject: kernel - Fix localhost packet misordering
X-Git-Tag: v3.0.0~740
X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/commitdiff_plain/392cd266cf0fd6bc128fba0cd2b0cf1f757f5628?hp=919eb2192b0e38503be55613615e752194429107

kernel - Fix localhost packet misordering

* netisr thread ports are based on IPIs, but when we enable asynch socket
  writes a user thread which gets moved between cpus sending async netmsgs
  while doing so can result in the netisr receiving those messages out
  of order, corrupting the data stream.

* Add TDF_FORCE_SPINPORT to allow the netisr threads to implement their
  message ports as spinports instead of threadports, which guarantees
  message ordering.
---

diff --git a/sys/kern/lwkt_thread.c b/sys/kern/lwkt_thread.c
index c55ebdd827..d91187664e 100644
--- a/sys/kern/lwkt_thread.c
+++ b/sys/kern/lwkt_thread.c
@@ -474,7 +474,7 @@ lwkt_init_thread(thread_t td, void *stack, int stksize, int flags,
     td->td_pri = TDPRI_KERN_DAEMON;
     td->td_critcount = 1;
     td->td_toks_stop = &td->td_toks_base;
-    if (lwkt_use_spin_port)
+    if (lwkt_use_spin_port || (flags & TDF_FORCE_SPINPORT))
 	lwkt_initport_spin(&td->td_msgport);
     else
 	lwkt_initport_thread(&td->td_msgport, td);
diff --git a/sys/net/if.c b/sys/net/if.c
index 7ad51ab935..9619eb557a 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -2753,7 +2753,8 @@ ifnetinit(void *dummy __unused)
 		struct thread *thr = &ifnet_threads[i];
 
 		lwkt_create(ifnet_service_loop, NULL, NULL,
-			    thr, TDF_STOPREQ, i, "ifnet %d", i);
+			    thr, TDF_STOPREQ|TDF_FORCE_SPINPORT,
+			    i, "ifnet %d", i);
 		netmsg_service_port_init(&thr->td_msgport);
 		lwkt_schedule(thr);
 	}
diff --git a/sys/net/netisr.c b/sys/net/netisr.c
index a62c555a74..945ebef679 100644
--- a/sys/net/netisr.c
+++ b/sys/net/netisr.c
@@ -181,8 +181,8 @@ netisr_init(void)
 	 */
 	for (i = 0; i < ncpus; ++i) {
 		lwkt_create(netmsg_service_loop, NULL, NULL,
-			    &netisr_cpu[i], TDF_STOPREQ, i,
-			    "netisr_cpu %d", i);
+			    &netisr_cpu[i], TDF_STOPREQ|TDF_FORCE_SPINPORT,
+			    i, "netisr_cpu %d", i);
 		netmsg_service_port_init(&netisr_cpu[i].td_msgport);
 		lwkt_schedule(&netisr_cpu[i]);
 	}
diff --git a/sys/sys/thread.h b/sys/sys/thread.h
index 0f2259516a..7d22ac4cdf 100644
--- a/sys/sys/thread.h
+++ b/sys/sys/thread.h
@@ -344,7 +344,7 @@ struct thread {
 #define TDF_BLOCKED		0x00040000	/* Thread is blocked */
 #define TDF_PANICWARN		0x00080000	/* panic warning in switch */
 #define TDF_BLOCKQ		0x00100000	/* on block queue */
-#define TDF_UNUSED00200000	0x00200000
+#define TDF_FORCE_SPINPORT	0x00200000
 #define TDF_EXITING		0x00400000	/* thread exiting */
 #define TDF_USINGFP		0x00800000	/* thread using fp coproc */
 #define TDF_KERNELFP		0x01000000	/* kernel using fp coproc */