x86_64: Implement x2apic support.
authorSepherosa Ziehau <sephe@dragonflybsd.org>
Tue, 5 Jun 2018 15:03:46 +0000 (23:03 +0800)
committerSepherosa Ziehau <sephe@dragonflybsd.org>
Fri, 8 Jun 2018 14:48:55 +0000 (22:48 +0800)
Now LAPIC registers are accessed through MSR at fixed location, instead
of going through MMIO region.

Most noticeable is that ICR operation is greatly simplified, i.e. IPI
sending operation:
- Reserved bits are read as 0; there is no need to read ICR first for
  OR with the new values.
- No more pending bit, i.e. ICR write is synchronized; there is no need
  to read ICR to test pending bit.
- ICR is 64 bits in x2apic mode, i.e. two 32 bits writes to ICR-low and
  ICR-high become one write to ICR.

NOTE:
Though Intel SDM says that wrmsr to LAPIC registers are relaxed, we
don't need to put mfence or sfence before them, especially for sending
IPIs, since the generic IPIQ and the machdep code already uses atomic
operation before doing ICR operation.  For the rest of the code, there
really are no needs to add mfence/sfence before rdmsr/wrmsr to LAPIC
registers.

As of this commit, x2apic mode is _not_ enabled by default.  It can be
enabled through hw.x2apic_enable tuneable, and a read-only sysctl node
with the same name is available for debugging purpose.

Based on work by ivadasz@.

sys/cpu/x86_64/include/specialreg.h
sys/platform/pc64/acpica/acpi_madt.c
sys/platform/pc64/apic/apicreg.h
sys/platform/pc64/apic/lapic.c
sys/platform/pc64/apic/lapic.h
sys/platform/pc64/x86_64/mp_machdep.c

index d13a867..fd738f3 100644 (file)
  */
 #define        APICBASE_RESERVED       0x000006ff
 #define        APICBASE_BSP            0x00000100
+#define        APICBASE_X2APIC         0x00000400
 #define        APICBASE_ENABLED        0x00000800
 #define        APICBASE_ADDRESS        0xfffff000
 
index 8c7d0ad..05e18c0 100644 (file)
@@ -42,6 +42,7 @@
 #include <machine_base/apic/ioapic.h>
 #include <machine_base/apic/apicvar.h>
 #include <machine_base/acpica/acpi_md_cpu.h>
+#include <machine/specialreg.h>
 
 #include <contrib/dev/acpica/source/include/acpi.h>
 
@@ -462,10 +463,12 @@ madt_lapic_probe(struct lapic_enumerator *e)
 
                if (arg.lapic_addr == 0) {
                        /*
-                        * XXX x2apic mode.
+                        * The LAPIC address does not matter in X2APIC mode.
                         */
-                       kprintf("madt_lapic_probe: zero LAPIC address\n");
-                       error = EOPNOTSUPP;
+                       if ((cpu_feature2 & CPUID2_X2APIC) == 0) {
+                               kprintf("madt_lapic_probe: zero LAPIC address\n");
+                               error = EOPNOTSUPP;
+                       }
                }
        }
 
@@ -476,16 +479,41 @@ madt_lapic_probe(struct lapic_enumerator *e)
 static int
 madt_lapic_enumerate(struct lapic_enumerator *e)
 {
-       vm_paddr_t lapic_addr;
+       vm_paddr_t lapic_addr = 0;
        int bsp_apic_id;
 
        KKASSERT(madt_phyaddr != 0);
 
-       lapic_addr = madt_lapic_pass1();
-       if (lapic_addr == 0)
-               panic("madt_lapic_enumerate: no local apic");
+       if (!x2apic_enable) {
+               lapic_addr = madt_lapic_pass1();
+               if (lapic_addr == 0) {
+                       /*
+                        * No LAPIC address.
+                        */
+                       if (cpu_feature2 & CPUID2_X2APIC) {
+                               /*
+                                * X2APIC mode is not enabled, but the CPU supports
+                                * it.  Forcefully enable X2APIC mode, which nullifies
+                                * the requirement of the LAPIC address.
+                                */
+                               kprintf("MADT: no LAPIC address, force X2APIC mode\n");
+                               KKASSERT(!x2apic_enable);
+                               x2apic_enable = 1;
+                               lapic_x2apic_enter(TRUE);
+                       } else {
+                               /*
+                                * We should not reach here, madt_lapic_probe() must
+                                * have failed.
+                                */
+                               panic("madt_lapic_enumerate: no local apic");
+                       }
+               }
+       }
 
-       lapic_map(lapic_addr);
+       if (!x2apic_enable) {
+               KASSERT(lapic_addr != 0, ("madt_lapic_enumerate: zero LAPIC address"));
+               lapic_map(lapic_addr);
+       }
 
        bsp_apic_id = LAPIC_READID;
        if (bsp_apic_id == APICID_MAX) {
index 0afa4a3..29c9519 100644 (file)
@@ -456,6 +456,34 @@ typedef struct IOAPIC ioapic_t;
 #undef PAD4
 #undef PAD3
 
+/* X2APIC */
+#define MSR_X2APIC_BASE                0x00000800
+#define MSR_X2APIC_ID          0x00000802
+#define MSR_X2APIC_EOI         0x0000080b
+#define MSR_X2APIC_DFR_RSVD    0x0000080e
+#define MSR_X2APIC_ICR         0x00000830
+#define MSR_X2APIC_ICR_RSVD    0x00000831
+#define MSR_X2APIC_ICR_TIMER   0x00000838
+#define MSR_X2APIC_CCR_TIMER   0x00000839
+#define MSR_X2APIC_SELFIPI     0x0000083f
+
+#define MSR_X2APIC_MIN         MSR_X2APIC_ID
+#define MSR_X2APIC_MAX         MSR_X2APIC_SELFIPI
+
+#define LAPIC2MSR(field) __extension__ ({              \
+       uint32_t __msr;                                 \
+                                                       \
+       __msr = MSR_X2APIC_BASE +                       \
+               (__offsetof(struct LAPIC, field) >> 4); \
+       KASSERT(__msr >= MSR_X2APIC_MIN &&              \
+               __msr <= MSR_X2APIC_MAX &&              \
+               __msr != MSR_X2APIC_DFR_RSVD &&         \
+               __msr != MSR_X2APIC_ICR &&              \
+               __msr != MSR_X2APIC_ICR_RSVD,           \
+               ("invalid MSR 0x%08x", __msr));         \
+       __msr;                                          \
+})
+
 #endif  /* !LOCORE */
 
 
index 8f2d005..1433443 100644 (file)
@@ -31,6 +31,7 @@
 #include <sys/ktr.h>
 #include <sys/bus.h>
 #include <sys/machintr.h>
+#include <sys/sysctl.h>
 #include <machine/globaldata.h>
 #include <machine/clock.h>
 #include <machine/limits.h>
@@ -55,6 +56,7 @@
 #endif
 KTR_INFO_MASTER(lapic);
 KTR_INFO(KTR_LAPIC, lapic, mem_eoi, 0, "mem_eoi");
+KTR_INFO(KTR_LAPIC, lapic, msr_eoi, 0, "msr_eoi");
 #define log_lapic(name)     KTR_LOG(lapic_ ## name)
 
 extern int naps;
@@ -81,6 +83,7 @@ TUNABLE_INT("hw.lapic_calibrate_fast", &lapic_calibrate_fast);
 
 static void    lapic_timer_tscdlt_reload(struct cputimer_intr *, sysclock_t);
 static void    lapic_mem_timer_intr_reload(struct cputimer_intr *, sysclock_t);
+static void    lapic_msr_timer_intr_reload(struct cputimer_intr *, sysclock_t);
 static void    lapic_timer_intr_enable(struct cputimer_intr *);
 static void    lapic_timer_intr_restart(struct cputimer_intr *);
 static void    lapic_timer_intr_pmfixup(struct cputimer_intr *);
@@ -120,6 +123,9 @@ int cpu_id_to_apic_id[NAPICID];
 int    apic_id_to_cpu_id[NAPICID];
 int    lapic_enable = 1;
 int    lapic_usable = 0;
+int    x2apic_enable = 0;
+
+SYSCTL_INT(_hw, OID_AUTO, x2apic_enable, CTLFLAG_RD, &x2apic_enable, 0, "");
 
 /* Separate cachelines for each cpu's info. */
 struct deadlines {
@@ -133,6 +139,10 @@ static void        lapic_mem_eoi(void);
 static int     lapic_mem_ipi(int dest_type, int vector, int delivery_mode);
 static void    lapic_mem_single_ipi(int cpu, int vector, int delivery_mode);
 
+static void    lapic_msr_eoi(void);
+static int     lapic_msr_ipi(int dest_type, int vector, int delivery_mode);
+static void    lapic_msr_single_ipi(int cpu, int vector, int delivery_mode);
+
 void           (*lapic_eoi)(void);
 int            (*apic_ipi)(int dest_type, int vector, int delivery_mode);
 void           (*single_apic_ipi)(int cpu, int vector, int delivery_mode);
@@ -150,6 +160,13 @@ lapic_mem_icr_set(uint32_t apic_id, uint32_t icr_lo_val)
        LAPIC_MEM_WRITE(icr_lo, icr_lo);
 }
 
+static __inline void
+lapic_msr_icr_set(uint32_t apic_id, uint32_t icr_lo_val)
+{
+       LAPIC_MSR_WRITE(MSR_X2APIC_ICR,
+           ((uint64_t)apic_id << 32) | ((uint64_t)icr_lo_val));
+}
+
 /*
  * Enable LAPIC, configure interrupts.
  */
@@ -690,6 +707,24 @@ lapic_mem_timer_intr_reload(struct cputimer_intr *cti, sysclock_t reload)
        }
 }
 
+static void
+lapic_msr_timer_intr_reload(struct cputimer_intr *cti, sysclock_t reload)
+{
+       struct globaldata *gd = mycpu;
+
+       reload = (int64_t)reload * cti->freq / sys_cputimer->freq;
+       if (reload < 2)
+               reload = 2;
+
+       if (gd->gd_timer_running) {
+               if (reload < LAPIC_MSR_READ(MSR_X2APIC_CCR_TIMER))
+                       LAPIC_MSR_WRITE(MSR_X2APIC_ICR_TIMER, reload);
+       } else {
+               gd->gd_timer_running = 1;
+               LAPIC_MSR_WRITE(MSR_X2APIC_ICR_TIMER, reload);
+       }
+}
+
 static void
 lapic_timer_intr_enable(struct cputimer_intr *cti __unused)
 {
@@ -873,6 +908,14 @@ lapic_mem_ipi(int dest_type, int vector, int delivery_mode)
        return 0;
 }
 
+static int
+lapic_msr_ipi(int dest_type, int vector, int delivery_mode)
+{
+       lapic_msr_icr_set(0,
+           dest_type | APIC_LEVEL_ASSERT | delivery_mode | vector);
+       return 0;
+}
+
 /*
  * Interrupts must be hard-disabled by caller
  */
@@ -884,6 +927,13 @@ lapic_mem_single_ipi(int cpu, int vector, int delivery_mode)
            APIC_DEST_DESTFLD | APIC_LEVEL_ASSERT | delivery_mode | vector);
 }
 
+static void
+lapic_msr_single_ipi(int cpu, int vector, int delivery_mode)
+{
+       lapic_msr_icr_set(CPUID_TO_APICID(cpu),
+           APIC_DEST_DESTFLD | APIC_LEVEL_ASSERT | delivery_mode | vector);
+}
+
 /*
  * Send APIC IPI 'vector' to 'target's via 'delivery_mode'.
  *
@@ -1004,6 +1054,30 @@ lapic_map(vm_paddr_t lapic_addr)
        lapic_mem = pmap_mapdev_uncacheable(lapic_addr, sizeof(struct LAPIC));
 }
 
+void
+lapic_x2apic_enter(boolean_t bsp)
+{
+       uint64_t apic_base;
+
+       KASSERT(x2apic_enable, ("X2APIC mode is not enabled"));
+
+       /*
+        * X2APIC mode is requested, if it has not been enabled by the BIOS,
+        * enable it now.
+        */
+       apic_base = rdmsr(MSR_APICBASE);
+       if ((apic_base & APICBASE_X2APIC) == 0) {
+               wrmsr(MSR_APICBASE,
+                   apic_base | APICBASE_X2APIC | APICBASE_ENABLED);
+       }
+       if (bsp) {
+               lapic_eoi = lapic_msr_eoi;
+               apic_ipi = lapic_msr_ipi;
+               single_apic_ipi = lapic_msr_single_ipi;
+               lapic_cputimer_intr.reload = lapic_msr_timer_intr_reload;
+       }
+}
+
 static TAILQ_HEAD(, lapic_enumerator) lapic_enumerators =
        TAILQ_HEAD_INITIALIZER(lapic_enumerators);
 
@@ -1011,10 +1085,42 @@ int
 lapic_config(void)
 {
        struct lapic_enumerator *e;
+       uint64_t apic_base;
        int error, i, ap_max;
 
        KKASSERT(lapic_enable);
 
+       lapic_eoi = lapic_mem_eoi;
+       apic_ipi = lapic_mem_ipi;
+       single_apic_ipi = lapic_mem_single_ipi;
+
+       TUNABLE_INT_FETCH("hw.x2apic_enable", &x2apic_enable);
+       if (x2apic_enable < 0)
+               x2apic_enable = 1;
+
+       if ((cpu_feature2 & CPUID2_X2APIC) == 0) {
+               /* X2APIC is not supported. */
+               x2apic_enable = 0;
+       } else if (!x2apic_enable) {
+               /*
+                * If the BIOS enabled the X2APIC mode, then we would stick
+                * with the X2APIC mode.
+                */
+               apic_base = rdmsr(MSR_APICBASE);
+               if (apic_base & APICBASE_X2APIC) {
+                       kprintf("LAPIC: BIOS enabled X2APIC mode\n");
+                       x2apic_enable = 1;
+               }
+       }
+
+       if (x2apic_enable) {
+               /*
+                * Enter X2APIC mode.
+                */
+               kprintf("LAPIC: enter X2APIC mode\n");
+               lapic_x2apic_enter(TRUE);
+       }
+
        for (i = 0; i < NAPICID; ++i)
                APICID_TO_CPUID(i) = -1;
 
@@ -1098,6 +1204,13 @@ lapic_mem_eoi(void)
        LAPIC_MEM_WRITE(eoi, 0);
 }
 
+static void
+lapic_msr_eoi(void)
+{
+       log_lapic(msr_eoi);
+       LAPIC_MSR_WRITE(MSR_X2APIC_EOI, 0);
+}
+
 static void
 lapic_mem_seticr_sync(uint32_t apic_id, uint32_t icr_lo_val)
 {
@@ -1109,8 +1222,10 @@ lapic_mem_seticr_sync(uint32_t apic_id, uint32_t icr_lo_val)
 void
 lapic_seticr_sync(uint32_t apic_id, uint32_t icr_lo_val)
 {
-       /* TODO: x2apic */
-       lapic_mem_seticr_sync(apic_id, icr_lo_val);
+       if (x2apic_enable)
+               lapic_msr_icr_set(apic_id, icr_lo_val);
+       else
+               lapic_mem_seticr_sync(apic_id, icr_lo_val);
 }
 
 static void
@@ -1119,14 +1234,12 @@ lapic_sysinit(void *dummy __unused)
        if (lapic_enable) {
                int error;
 
-               lapic_eoi = lapic_mem_eoi;
-               apic_ipi = lapic_mem_ipi;
-               single_apic_ipi = lapic_mem_single_ipi;
-
                error = lapic_config();
                if (error)
                        lapic_enable = 0;
        }
+       if (!lapic_enable)
+               x2apic_enable = 0;
 
        if (lapic_enable) {
                /* Initialize BSP's local APIC */
index 771d738..d6a5afd 100644 (file)
@@ -56,6 +56,7 @@ extern int                    cpu_id_to_apic_id[];
 extern int                     apic_id_to_cpu_id[];
 extern int                     lapic_enable;
 extern int                     lapic_usable;
+extern int                     x2apic_enable;
 extern void                    (*lapic_eoi)(void);
 extern int                     (*apic_ipi)(int, int, int);
 extern void                    (*single_apic_ipi)(int, int, int);
@@ -73,6 +74,7 @@ void  lapic_map(vm_paddr_t);
 int    lapic_unused_apic_id(int);
 void   lapic_fixup_noioapic(void);
 void   lapic_seticr_sync(uint32_t, uint32_t);
+void   lapic_x2apic_enter(boolean_t);
 
 #ifndef _MACHINE_SMP_H_
 #include <machine/smp.h>
@@ -101,11 +103,23 @@ do {                                      \
        lapic_mem->field = (val);       \
 } while (0)
 
-/* TODO: x2apic */
-#define LAPIC_READ(field)              LAPIC_MEM_READ(field)
-#define LAPIC_WRITE(field, val)                LAPIC_MEM_WRITE(field, (val))
+#define LAPIC_MSR_READ(msr)            rdmsr((msr))
+#define LAPIC_MSR_WRITE(msr, val)      wrmsr((msr), (val))
 
-/* TODO: x2apic */
-#define LAPIC_READID                   APIC_ID(LAPIC_MEM_READ(id))
+#define LAPIC_READ(field)                                              \
+       (x2apic_enable ? (uint32_t)LAPIC_MSR_READ(LAPIC2MSR(field))     \
+                      : LAPIC_MEM_READ(field))
+
+#define LAPIC_WRITE(field, val)                                                \
+do {                                                                   \
+       if (x2apic_enable)                                              \
+               LAPIC_MSR_WRITE(LAPIC2MSR(field), (val));               \
+       else                                                            \
+               LAPIC_MEM_WRITE(field, (val));                          \
+} while (0)
+
+#define LAPIC_READID                                                   \
+       (x2apic_enable ? (uint32_t)LAPIC_MSR_READ(MSR_X2APIC_ID)        \
+                      : APIC_ID(LAPIC_MEM_READ(id)))
 
 #endif /* _ARCH_APIC_LAPIC_H_ */
index 4b0fcc1..22fe47e 100644 (file)
@@ -347,6 +347,10 @@ init_secondary(void)
        /* set up FPU state on the AP */
        npxinit();
 
+       /* If BSP is in the X2APIC mode, put the AP into the X2APIC mode. */
+       if (x2apic_enable)
+               lapic_x2apic_enter(FALSE);
+
        /* disable the APIC, just to be SURE */
        LAPIC_WRITE(svr, (LAPIC_READ(svr) & ~APIC_SVR_ENABLE));
 }