kernel - Add MONITOR/MWAIT support to the LWKT scheduler

* Adjust the FIFO contention resequencer (which deals with spinning
  on tokens) to use MONITOR/MWAIT when available instead of DELAY(1)
  when waiting to become the head of the queue.

* Adjust the x86-64 idle loop to use MONITOR/MWAIT when available when
  the idle halt mode (machdep.cpu_idle_hlt) is set to 1.  This
  significantly improves performance for event-oriented programs, including
  compile pipelines.

  NOTE: On the 48-core monster setting machdep.cpu_idle_hlt to 1 improves
  performance but at the cost of an additional 200W of power at idle vs
  the default value of 2 (ACPI idle halt).  Look for a hybrid approach in
  a future commit.

diff --git a/sys/kern/lwkt_thread.c b/sys/kern/lwkt_thread.c
index 370b5c9..e448da3 100644 (file)
--- a/sys/kern/lwkt_thread.c
+++ b/sys/kern/lwkt_thread.c
@@ -500,6 +500,7 @@ lwkt_switch(void)
     int spinning = lwkt_spin_loops;    /* loops before HLTing */
     int reqflags;
     int cseq;
+    int oseq;
 
     /*
      * Switching from within a 'fast' (non thread switched) interrupt or IPI
@@ -847,9 +848,14 @@ skip:
         *           it could cause a deadlock.
         */
        cseq = atomic_fetchadd_int(&lwkt_cseq_windex, 1);
-       while (lwkt_cseq_rindex != cseq) {
-           DELAY(1);
-           cpu_lfence();
+       while ((oseq = lwkt_cseq_rindex) != cseq) {
+           cpu_ccfence();
+           if (cpu_mi_feature & CPU_MI_MONITOR) {
+               cpu_mmw_pause_int(&lwkt_cseq_rindex, oseq);
+           } else {
+               DELAY(1);
+               cpu_lfence();
+           }
        }
        cseq = lwkt_spin_delay; /* don't trust the system operator */
        cpu_ccfence();

diff --git a/sys/platform/pc64/x86_64/machdep.c b/sys/platform/pc64/x86_64/machdep.c
index cef1c16..6de4928 100644 (file)
--- a/sys/platform/pc64/x86_64/machdep.c
+++ b/sys/platform/pc64/x86_64/machdep.c
@@ -929,7 +929,9 @@ void (*cpu_idle_hook)(void) = cpu_idle_default_hook;
 void
 cpu_idle(void)
 {
-       struct thread *td = curthread;
+       globaldata_t gd = mycpu;
+       struct thread *td = gd->gd_curthread;
+       int reqflags;
 
        crit_exit();
        KKASSERT(td->td_critcount == 0);
@@ -944,11 +946,15 @@ cpu_idle(void)
                 * CLIing to catch any interrupt races.  Note that we are
                 * at SPL0 and interrupts are enabled.
                 */
-               if (cpu_idle_hlt &&
-                   (td->td_gd->gd_reqflags & RQF_IDLECHECK_WK_MASK) == 0) {
+               reqflags = gd->gd_reqflags;
+               if (cpu_idle_hlt == 1 &&
+                   (cpu_mi_feature & CPU_MI_MONITOR) &&
+                   (reqflags & RQF_IDLECHECK_WK_MASK) == 0) {
+                       cpu_mmw_pause_int(&gd->gd_reqflags, reqflags);
+               } else if (cpu_idle_hlt) {
                        __asm __volatile("cli");
                        splz();
-                       if ((td->td_gd->gd_reqflags & RQF_IDLECHECK_WK_MASK) == 0) {
+                       if ((gd->gd_reqflags & RQF_IDLECHECK_WK_MASK) == 0) {
                                if (cpu_idle_hlt == 1)
                                        cpu_idle_default_hook();
                                else