sys/platform/pc64/x86_64/vm_machdep.c

   1 /*-
   2  * Copyright (c) 1982, 1986 The Regents of the University of California.
   3  * Copyright (c) 1989, 1990 William Jolitz
   4  * Copyright (c) 1994 John Dyson
   5  * Copyright (c) 2008-2018 The DragonFly Project.
   6  * All rights reserved.
   7  *
   8  * This code is derived from software contributed to Berkeley by
   9  * the Systems Programming Group of the University of Utah Computer
  10  * Science Department, and William Jolitz.
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice, this list of conditions and the following disclaimer.
  17  * 2. Redistributions in binary form must reproduce the above copyright
  18  *    notice, this list of conditions and the following disclaimer in the
  19  *    documentation and/or other materials provided with the distribution.
  20  * 3. All advertising materials mentioning features or use of this software
  21  *    must display the following acknowledgement:
  22  *      This product includes software developed by the University of
  23  *      California, Berkeley and its contributors.
  24  * 4. Neither the name of the University nor the names of its contributors
  25  *    may be used to endorse or promote products derived from this software
  26  *    without specific prior written permission.
  27  *
  28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  38  * SUCH DAMAGE.
  39  *
  40  *      from: @(#)vm_machdep.c  7.3 (Berkeley) 5/13/91
  41  *      Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$
  42  * $FreeBSD: src/sys/i386/i386/vm_machdep.c,v 1.132.2.9 2003/01/25 19:02:23 dillon Exp $
  43  */
  44
  45 #include <sys/param.h>
  46 #include <sys/systm.h>
  47 #include <sys/malloc.h>
  48 #include <sys/proc.h>
  49 #include <sys/buf.h>
  50 #include <sys/interrupt.h>
  51 #include <sys/vnode.h>
  52 #include <sys/vmmeter.h>
  53 #include <sys/kernel.h>
  54 #include <sys/sysctl.h>
  55 #include <sys/unistd.h>
  56 #include <sys/lwp.h>
  57
  58 #include <machine/clock.h>
  59 #include <machine/cpu.h>
  60 #include <machine/md_var.h>
  61 #include <machine/smp.h>
  62 #include <machine/pcb.h>
  63 #include <machine/pcb_ext.h>
  64 #include <machine/segments.h>
  65 #include <machine/globaldata.h> /* npxthread */
  66 #include <machine/specialreg.h>
  67 #include <machine/vmm.h>
  68
  69 #include <vm/vm.h>
  70 #include <vm/vm_param.h>
  71 #include <sys/lock.h>
  72 #include <vm/vm_kern.h>
  73 #include <vm/vm_page.h>
  74 #include <vm/vm_map.h>
  75 #include <vm/vm_extern.h>
  76
  77 #include <sys/thread2.h>
  78
  79 #include <bus/isa/isa.h>
  80
  81 static void     cpu_reset_real (void);
  82
  83 static int spectre_mitigation = -1;
  84 static int spectre_support = 0;
  85 static int spectre_mode = 0;
  86 SYSCTL_INT(_machdep, OID_AUTO, spectre_mode, CTLFLAG_RD,
  87         &spectre_mode, 0, "current Spectre enablements");
  88
  89 static int mds_mitigation = -1;
  90 static int mds_support = 0;
  91 static int mds_mode = 0;
  92 SYSCTL_INT(_machdep, OID_AUTO, mds_mode, CTLFLAG_RD,
  93         &mds_mode, 0, "current MDS enablements");
  94
  95 /*
  96  * Finish a fork operation, with lwp lp2 nearly set up.
  97  * Copy and update the pcb, set up the stack so that the child
  98  * ready to run and return to user mode.
  99  */
 100 void
 101 cpu_fork(struct lwp *lp1, struct lwp *lp2, int flags)
 102 {
 103         struct pcb *pcb2;
 104         struct pmap *pmap2;
 105
 106         if ((flags & RFPROC) == 0) {
 107                 if ((flags & RFMEM) == 0) {
 108                         /*
 109                          * Unshare user LDT.  > 1 test is MPSAFE.  While
 110                          * it can potentially race a 2->1 transition, the
 111                          * worst that happens is that we do an unnecessary
 112                          * ldt replacement.
 113                          */
 114                         struct pcb *pcb1 = lp1->lwp_thread->td_pcb;
 115                         struct pcb_ldt *pcb_ldt = pcb1->pcb_ldt;
 116
 117                         if (pcb_ldt && pcb_ldt->ldt_refcnt > 1) {
 118                                 pcb_ldt = user_ldt_alloc(pcb1,pcb_ldt->ldt_len);
 119                                 user_ldt_free(pcb1);
 120                                 pcb1->pcb_ldt = pcb_ldt;
 121                                 set_user_ldt(pcb1);
 122                         }
 123                 }
 124                 return;
 125         }
 126
 127         /* Ensure that lp1's pcb is up to date. */
 128         if (mdcpu->gd_npxthread == lp1->lwp_thread)
 129                 npxsave(lp1->lwp_thread->td_savefpu);
 130
 131         /*
 132          * Copy lp1's PCB.  This really only applies to the
 133          * debug registers and FP state, but its faster to just copy the
 134          * whole thing.  Because we only save the PCB at switchout time,
 135          * the register state may not be current.
 136          */
 137         pcb2 = lp2->lwp_thread->td_pcb;
 138         *pcb2 = *lp1->lwp_thread->td_pcb;
 139
 140         /*
 141          * Create a new fresh stack for the new process.
 142          * Copy the trap frame for the return to user mode as if from a
 143          * syscall.  This copies the user mode register values.
 144          *
 145          * pcb_rsp must allocate an additional call-return pointer below
 146          * the trap frame which will be restored by cpu_heavy_restore from
 147          * PCB_RIP, and the thread's td_sp pointer must allocate an
 148          * additonal two quadwords below the pcb_rsp call-return pointer to
 149          * hold the LWKT restore function pointer and rflags.
 150          *
 151          * The LWKT restore function pointer must be set to cpu_heavy_restore,
 152          * which is our standard heavy-weight process switch-in function.
 153          * YYY eventually we should shortcut fork_return and fork_trampoline
 154          * to use the LWKT restore function directly so we can get rid of
 155          * all the extra crap we are setting up.
 156          */
 157         lp2->lwp_md.md_regs = (struct trapframe *)pcb2 - 1;
 158         bcopy(lp1->lwp_md.md_regs, lp2->lwp_md.md_regs, sizeof(*lp2->lwp_md.md_regs));
 159
 160         /*
 161          * Set registers for trampoline to user mode.  Leave space for the
 162          * return address on stack.  These are the kernel mode register values.
 163          *
 164          * Set the new pmap CR3.  If the new process uses isolated VM spaces,
 165          * also set the isolated CR3.
 166          */
 167         pmap2 = vmspace_pmap(lp2->lwp_proc->p_vmspace);
 168         pcb2->pcb_cr3 = vtophys(pmap2->pm_pml4);
 169         if ((pcb2->pcb_flags & PCB_ISOMMU) && pmap2->pm_pmlpv_iso) {
 170                 pcb2->pcb_cr3_iso = vtophys(pmap2->pm_pml4_iso);
 171         } else {
 172                 pcb2->pcb_flags &= ~PCB_ISOMMU;
 173                 pcb2->pcb_cr3_iso = 0;
 174         }
 175
 176 #if 0
 177         /*
 178          * Per-process spectre mitigation (future)
 179          */
 180         pcb2->pcb_flags &= ~(PCB_IBRS1 | PCB_IBRS2);
 181         switch (spectre_mitigation) {
 182         case 1:
 183                 pcb2->pcb_flags |= PCB_IBRS1;
 184                 break;
 185         case 2:
 186                 pcb2->pcb_flags |= PCB_IBRS2;
 187                 break;
 188         default:
 189                 break;
 190         }
 191 #endif
 192
 193         pcb2->pcb_rbx = (unsigned long)fork_return;     /* fork_trampoline argument */
 194         pcb2->pcb_rbp = 0;
 195         pcb2->pcb_rsp = (unsigned long)lp2->lwp_md.md_regs - sizeof(void *);
 196         pcb2->pcb_r12 = (unsigned long)lp2;             /* fork_trampoline argument */
 197         pcb2->pcb_r13 = 0;
 198         pcb2->pcb_r14 = 0;
 199         pcb2->pcb_r15 = 0;
 200         pcb2->pcb_rip = (unsigned long)fork_trampoline;
 201         lp2->lwp_thread->td_sp = (char *)(pcb2->pcb_rsp - sizeof(void *));
 202         *(u_int64_t *)lp2->lwp_thread->td_sp = PSL_USER;
 203         lp2->lwp_thread->td_sp -= sizeof(void *);
 204         *(void **)lp2->lwp_thread->td_sp = (void *)cpu_heavy_restore;
 205
 206         /*
 207          * pcb2->pcb_ldt:       duplicated below, if necessary.
 208          * pcb2->pcb_savefpu:   cloned above.
 209          * pcb2->pcb_flags:     cloned above
 210          * pcb2->pcb_onfault:   cloned above (always NULL here).
 211          * pcb2->pcb_onfault_sp:cloned above (dont care)
 212          */
 213
 214         /*
 215          * XXX don't copy the i/o pages.  this should probably be fixed.
 216          */
 217         pcb2->pcb_ext = NULL;
 218
 219         /* Copy the LDT, if necessary. */
 220         if (pcb2->pcb_ldt != NULL) {
 221                 if (flags & RFMEM) {
 222                         atomic_add_int(&pcb2->pcb_ldt->ldt_refcnt, 1);
 223                 } else {
 224                         pcb2->pcb_ldt = user_ldt_alloc(pcb2,
 225                                                        pcb2->pcb_ldt->ldt_len);
 226                 }
 227         }
 228         bcopy(&lp1->lwp_thread->td_tls, &lp2->lwp_thread->td_tls,
 229               sizeof(lp2->lwp_thread->td_tls));
 230         /*
 231          * Now, cpu_switch() can schedule the new lwp.
 232          * pcb_rsp is loaded pointing to the cpu_switch() stack frame
 233          * containing the return address when exiting cpu_switch.
 234          * This will normally be to fork_trampoline(), which will have
 235          * %rbx loaded with the new lwp's pointer.  fork_trampoline()
 236          * will set up a stack to call fork_return(lp, frame); to complete
 237          * the return to user-mode.
 238          */
 239 }
 240
 241 /*
 242  * Prepare new lwp to return to the address specified in params.
 243  */
 244 int
 245 cpu_prepare_lwp(struct lwp *lp, struct lwp_params *params)
 246 {
 247         struct trapframe *regs = lp->lwp_md.md_regs;
 248         void *bad_return = NULL;
 249         int error;
 250
 251         regs->tf_rip = (long)params->lwp_func;
 252         regs->tf_rsp = (long)params->lwp_stack;
 253         /* Set up argument for function call */
 254         regs->tf_rdi = (long)params->lwp_arg;
 255
 256         /*
 257          * Set up fake return address.  As the lwp function may never return,
 258          * we simply copy out a NULL pointer and force the lwp to receive
 259          * a SIGSEGV if it returns anyways.
 260          */
 261         regs->tf_rsp -= sizeof(void *);
 262         error = copyout(&bad_return, (void *)regs->tf_rsp, sizeof(bad_return));
 263         if (error)
 264                 return (error);
 265
 266         if (lp->lwp_proc->p_vmm) {
 267                 lp->lwp_thread->td_pcb->pcb_cr3 = KPML4phys;
 268                 cpu_set_fork_handler(lp,
 269                     (void (*)(void *, struct trapframe *))vmm_lwp_return, lp);
 270         } else {
 271                 cpu_set_fork_handler(lp,
 272                     (void (*)(void *, struct trapframe *))generic_lwp_return, lp);
 273         }
 274         return (0);
 275 }
 276
 277 /*
 278  * Intercept the return address from a freshly forked process that has NOT
 279  * been scheduled yet.
 280  *
 281  * This is needed to make kernel threads stay in kernel mode.
 282  */
 283 void
 284 cpu_set_fork_handler(struct lwp *lp, void (*func)(void *, struct trapframe *),
 285                      void *arg)
 286 {
 287         /*
 288          * Note that the trap frame follows the args, so the function
 289          * is really called like this:  func(arg, frame);
 290          */
 291         lp->lwp_thread->td_pcb->pcb_rbx = (long)func;   /* function */
 292         lp->lwp_thread->td_pcb->pcb_r12 = (long)arg;    /* first arg */
 293 }
 294
 295 void
 296 cpu_set_thread_handler(thread_t td, void (*rfunc)(void), void *func, void *arg)
 297 {
 298         td->td_pcb->pcb_rbx = (long)func;
 299         td->td_pcb->pcb_r12 = (long)arg;
 300         td->td_switch = cpu_lwkt_switch;
 301         td->td_sp -= sizeof(void *);
 302         *(void **)td->td_sp = rfunc;    /* exit function on return */
 303         td->td_sp -= sizeof(void *);
 304         *(void **)td->td_sp = cpu_kthread_restore;
 305 }
 306
 307 void
 308 cpu_lwp_exit(void)
 309 {
 310         struct thread *td = curthread;
 311         struct pcb *pcb;
 312
 313         pcb = td->td_pcb;
 314
 315         /* Some x86 functionality was dropped */
 316         KKASSERT(pcb->pcb_ext == NULL);
 317
 318         /*
 319          * disable all hardware breakpoints
 320          */
 321         if (pcb->pcb_flags & PCB_DBREGS) {
 322                 reset_dbregs();
 323                 pcb->pcb_flags &= ~PCB_DBREGS;
 324         }
 325         td->td_gd->gd_cnt.v_swtch++;
 326
 327         crit_enter_quick(td);
 328         if (td->td_flags & TDF_TSLEEPQ)
 329                 tsleep_remove(td);
 330         lwkt_deschedule_self(td);
 331         lwkt_remove_tdallq(td);
 332         cpu_thread_exit();
 333 }
 334
 335 /*
 336  * Terminate the current thread.  The caller must have already acquired
 337  * the thread's rwlock and placed it on a reap list or otherwise notified
 338  * a reaper of its existance.  We set a special assembly switch function which
 339  * releases td_rwlock after it has cleaned up the MMU state and switched
 340  * out the stack.
 341  *
 342  * Must be caller from a critical section and with the thread descheduled.
 343  */
 344 void
 345 cpu_thread_exit(void)
 346 {
 347         npxexit();
 348         curthread->td_switch = cpu_exit_switch;
 349         curthread->td_flags |= TDF_EXITING;
 350         lwkt_switch();
 351         panic("cpu_thread_exit: lwkt_switch() unexpectedly returned");
 352 }
 353
 354 void
 355 cpu_reset(void)
 356 {
 357         cpu_reset_real();
 358 }
 359
 360 static void
 361 cpu_reset_real(void)
 362 {
 363         /*
 364          * Attempt to do a CPU reset via the keyboard controller,
 365          * do not turn off the GateA20, as any machine that fails
 366          * to do the reset here would then end up in no man's land.
 367          */
 368
 369 #if !defined(BROKEN_KEYBOARD_RESET)
 370         outb(IO_KBD + 4, 0xFE);
 371         DELAY(500000);  /* wait 0.5 sec to see if that did it */
 372         kprintf("Keyboard reset did not work, attempting CPU shutdown\n");
 373         DELAY(1000000); /* wait 1 sec for kprintf to complete */
 374 #endif
 375 #if 0 /* JG */
 376         /* force a shutdown by unmapping entire address space ! */
 377         bzero((caddr_t) PTD, PAGE_SIZE);
 378 #endif
 379
 380         /* "good night, sweet prince .... <THUNK!>" */
 381         cpu_invltlb();
 382         /* NOTREACHED */
 383         while(1);
 384 }
 385
 386 static void
 387 swi_vm(void *arg, void *frame)
 388 {
 389         if (busdma_swi_pending != 0)
 390                 busdma_swi();
 391 }
 392
 393 static void
 394 swi_vm_setup(void *arg)
 395 {
 396         register_swi_mp(SWI_VM, swi_vm, NULL, "swi_vm", NULL, 0);
 397 }
 398
 399 SYSINIT(swi_vm_setup, SI_BOOT2_MACHDEP, SI_ORDER_ANY, swi_vm_setup, NULL);
 400
 401 /*
 402  * NOTE: This routine is also called after a successful microcode
 403  *       reload on cpu 0.
 404  */
 405 void mitigation_vm_setup(void *arg);
 406
 407 /*
 408  * Check for IBPB and IBRS support
 409  *
 410  * This bits also specify desired modes in the spectre_mitigation sysctl.
 411  */
 412 #define IBRS_SUPPORTED          0x0001
 413 #define STIBP_SUPPORTED         0x0002
 414 #define IBPB_SUPPORTED          0x0004
 415 #define IBRS_AUTO_SUPPORTED     0x0008
 416 #define STIBP_AUTO_SUPPORTED    0x0010
 417 #define IBRS_PREFERRED_REQUEST  0x0020
 418
 419 static
 420 int
 421 spectre_check_support(void)
 422 {
 423         uint32_t p[4];
 424         int rv = 0;
 425
 426         /*
 427          * Spectre mitigation hw bits
 428          *
 429          * IBRS         Indirect Branch Restricted Speculation   (isolation)
 430          * STIBP        Single Thread Indirect Branch Prediction (isolation)
 431          * IBPB         Branch Prediction Barrier                (barrier)
 432          *
 433          * IBRS and STIBP must be toggled (enabled on entry to kernel,
 434          * disabled on exit, as well as disabled during any MWAIT/HLT).
 435          * When *_AUTO bits are available, IBRS and STIBP may be left
 436          * turned on and do not have to be toggled on kernel entry/exit.
 437          *
 438          * All this shit has enormous overhead.  IBPB in particular, and
 439          * non-auto modes are disabled by default.
 440          */
 441         if (cpu_vendor_id == CPU_VENDOR_INTEL) {
 442                 p[0] = 0;
 443                 p[1] = 0;
 444                 p[2] = 0;
 445                 p[3] = 0;
 446                 cpuid_count(7, 0, p);
 447                 if (p[3] & CPUID_7_0_I3_SPEC_CTRL)
 448                         rv |= IBRS_SUPPORTED | IBPB_SUPPORTED;
 449                 if (p[3] & CPUID_7_0_I3_STIBP)
 450                         rv |= STIBP_SUPPORTED;
 451
 452                 /*
 453                  * 0x80000008 p[1] bit 12 indicates IBPB support
 454                  *
 455                  * This bit might be set even though SPEC_CTRL is not set.
 456                  */
 457                 p[0] = 0;
 458                 p[1] = 0;
 459                 p[2] = 0;
 460                 p[3] = 0;
 461                 do_cpuid(0x80000008U, p);
 462                 if (p[1] & CPUID_INTEL_80000008_I1_IBPB_SUPPORT)
 463                         rv |= IBPB_SUPPORTED;
 464         } else if (cpu_vendor_id == CPU_VENDOR_AMD) {
 465                 /*
 466                  * 0x80000008 p[1] bit 12 indicates IBPB support
 467                  *            p[1] bit 14 indicates IBRS support
 468                  *            p[1] bit 15 indicates STIBP support
 469                  *
 470                  *            p[1] bit 16 indicates IBRS auto support
 471                  *            p[1] bit 17 indicates STIBP auto support
 472                  *            p[1] bit 18 indicates processor prefers using
 473                  *              IBRS instead of retpoline.
 474                  */
 475                 p[0] = 0;
 476                 p[1] = 0;
 477                 p[2] = 0;
 478                 p[3] = 0;
 479                 do_cpuid(0x80000008U, p);
 480                 if (p[1] & CPUID_AMD_80000008_I1_IBPB_SUPPORT)
 481                         rv |= IBPB_SUPPORTED;
 482                 if (p[1] & CPUID_AMD_80000008_I1_IBRS_SUPPORT)
 483                         rv |= IBRS_SUPPORTED;
 484                 if (p[1] & CPUID_AMD_80000008_I1_STIBP_SUPPORT)
 485                         rv |= STIBP_SUPPORTED;
 486
 487                 if (p[1] & CPUID_AMD_80000008_I1_IBRS_AUTO)
 488                         rv |= IBRS_AUTO_SUPPORTED;
 489                 if (p[1] & CPUID_AMD_80000008_I1_STIBP_AUTO)
 490                         rv |= STIBP_AUTO_SUPPORTED;
 491                 if (p[1] & CPUID_AMD_80000008_I1_IBRS_REQUESTED)
 492                         rv |= IBRS_PREFERRED_REQUEST;
 493         }
 494
 495         return rv;
 496 }
 497
 498 /*
 499  * Iterate CPUs and adjust MSR for global operations, since
 500  * the KMMU* code won't do it if spectre_mitigation is 0 or 2.
 501  */
 502 #define CHECK(flag)     (spectre_mitigation & spectre_support & (flag))
 503
 504 static
 505 void
 506 spectre_sysctl_changed(void)
 507 {
 508         globaldata_t save_gd;
 509         struct trampframe *tr;
 510         int spec_ctrl;
 511         int spec_mask;
 512         int mode;
 513         int n;
 514
 515
 516         spec_mask = SPEC_CTRL_IBRS | SPEC_CTRL_STIBP |
 517                     SPEC_CTRL_DUMMY_ENABLE | SPEC_CTRL_DUMMY_IBPB;
 518
 519         /*
 520          * Fixup state
 521          */
 522         mode = 0;
 523         save_gd = mycpu;
 524         for (n = 0; n < ncpus; ++n) {
 525                 lwkt_setcpu_self(globaldata_find(n));
 526                 cpu_ccfence();
 527                 tr = &pscpu->trampoline;
 528
 529                 /*
 530                  * Make sure we are cleaned out.
 531                  *
 532                  * XXX cleanup, reusing globals inside the loop (they get
 533                  * set to the same thing each loop)
 534                  *
 535                  * [0] kernel entry (idle exit)
 536                  * [1] kernel exit  (idle entry)
 537                  */
 538                 tr->tr_pcb_spec_ctrl[0] &= ~spec_mask;
 539                 tr->tr_pcb_spec_ctrl[1] &= ~spec_mask;
 540
 541                 /*
 542                  * Don't try to parse if not available
 543                  */
 544                 if (spectre_mitigation < 0)
 545                         continue;
 546
 547                 /*
 548                  * IBRS mode.  Auto overrides toggling.
 549                  *
 550                  * Only set the ENABLE flag if we have to toggle something
 551                  * on entry and exit.
 552                  */
 553                 spec_ctrl = 0;
 554                 if (CHECK(IBRS_AUTO_SUPPORTED)) {
 555                         spec_ctrl |= SPEC_CTRL_IBRS;
 556                         mode |= IBRS_AUTO_SUPPORTED;
 557                 } else if (CHECK(IBRS_SUPPORTED)) {
 558                         spec_ctrl |= SPEC_CTRL_IBRS | SPEC_CTRL_DUMMY_ENABLE;
 559                         mode |= IBRS_SUPPORTED;
 560                 }
 561                 if (CHECK(STIBP_AUTO_SUPPORTED)) {
 562                         spec_ctrl |= SPEC_CTRL_STIBP;
 563                         mode |= STIBP_AUTO_SUPPORTED;
 564                 } else if (CHECK(STIBP_SUPPORTED)) {
 565                         spec_ctrl |= SPEC_CTRL_STIBP | SPEC_CTRL_DUMMY_ENABLE;
 566                         mode |= STIBP_SUPPORTED;
 567                 }
 568
 569                 /*
 570                  * IBPB requested and supported.
 571                  */
 572                 if (CHECK(IBPB_SUPPORTED)) {
 573                         spec_ctrl |= SPEC_CTRL_DUMMY_IBPB;
 574                         mode |= IBPB_SUPPORTED;
 575                 }
 576
 577                 /*
 578                  * Update the MSR if the cpu supports the modes to ensure
 579                  * proper disablement if the user disabled the mode.
 580                  */
 581                 if (spectre_support & (IBRS_SUPPORTED | IBRS_AUTO_SUPPORTED |
 582                                     STIBP_SUPPORTED | STIBP_AUTO_SUPPORTED)) {
 583                         wrmsr(MSR_SPEC_CTRL,
 584                               spec_ctrl & (SPEC_CTRL_IBRS|SPEC_CTRL_STIBP));
 585                 }
 586
 587                 /*
 588                  * Update spec_ctrl fields in the trampoline.
 589                  *
 590                  * [0] on-kernel-entry (on-idle-exit)
 591                  * [1] on-kernel-exit  (on-idle-entry)
 592                  *
 593                  * When auto mode is supported we leave the bit set, otherwise
 594                  * we clear the bits.
 595                  */
 596                 tr->tr_pcb_spec_ctrl[0] |= spec_ctrl;
 597                 if (CHECK(IBRS_AUTO_SUPPORTED) == 0)
 598                         spec_ctrl &= ~SPEC_CTRL_IBRS;
 599                 if (CHECK(STIBP_AUTO_SUPPORTED) == 0)
 600                         spec_ctrl &= ~SPEC_CTRL_STIBP;
 601                 tr->tr_pcb_spec_ctrl[1] |= spec_ctrl;
 602
 603                 /*
 604                  * Make sure we set this on the first loop.  It will be
 605                  * the same value on remaining loops.
 606                  */
 607                 spectre_mode = mode;
 608         }
 609         lwkt_setcpu_self(save_gd);
 610         cpu_ccfence();
 611
 612         /*
 613          * Console message on mitigation mode change
 614          */
 615         kprintf("Spectre: support=(");
 616         if (spectre_support == 0) {
 617                 kprintf(" none");
 618         } else {
 619                 if (spectre_support & IBRS_SUPPORTED)
 620                         kprintf(" IBRS");
 621                 if (spectre_support & STIBP_SUPPORTED)
 622                         kprintf(" STIBP");
 623                 if (spectre_support & IBPB_SUPPORTED)
 624                         kprintf(" IBPB");
 625                 if (spectre_support & IBRS_AUTO_SUPPORTED)
 626                         kprintf(" IBRS_AUTO");
 627                 if (spectre_support & STIBP_AUTO_SUPPORTED)
 628                         kprintf(" STIBP_AUTO");
 629                 if (spectre_support & IBRS_PREFERRED_REQUEST)
 630                         kprintf(" IBRS_REQUESTED");
 631         }
 632         kprintf(" ) req=%04x operating=(", (uint16_t)spectre_mitigation);
 633         if (spectre_mode == 0) {
 634                 kprintf(" none");
 635         } else {
 636                 if (spectre_mode & IBRS_SUPPORTED)
 637                         kprintf(" IBRS");
 638                 if (spectre_mode & STIBP_SUPPORTED)
 639                         kprintf(" STIBP");
 640                 if (spectre_mode & IBPB_SUPPORTED)
 641                         kprintf(" IBPB");
 642                 if (spectre_mode & IBRS_AUTO_SUPPORTED)
 643                         kprintf(" IBRS_AUTO");
 644                 if (spectre_mode & STIBP_AUTO_SUPPORTED)
 645                         kprintf(" STIBP_AUTO");
 646                 if (spectre_mode & IBRS_PREFERRED_REQUEST)
 647                         kprintf(" IBRS_REQUESTED");
 648         }
 649         kprintf(" )\n");
 650 }
 651
 652 #undef CHECK
 653
 654 /*
 655  * User changes sysctl value
 656  */
 657 static int
 658 sysctl_spectre_mitigation(SYSCTL_HANDLER_ARGS)
 659 {
 660         char buf[128];
 661         char *ptr;
 662         char *iter;
 663         size_t len;
 664         int spectre;
 665         int error = 0;
 666         int loop = 0;
 667
 668         /*
 669          * Return current operating mode or support.
 670          */
 671         if (oidp->oid_kind & CTLFLAG_WR)
 672                 spectre = spectre_mode;
 673         else
 674                 spectre = spectre_support;
 675
 676         spectre &= (IBRS_SUPPORTED | IBRS_AUTO_SUPPORTED |
 677                     STIBP_SUPPORTED | STIBP_AUTO_SUPPORTED |
 678                     IBPB_SUPPORTED);
 679         while (spectre) {
 680                 if (error)
 681                         break;
 682                 if (loop++) {
 683                         error = SYSCTL_OUT(req, " ", 1);
 684                         if (error)
 685                                 break;
 686                 }
 687                 if (spectre & IBRS_SUPPORTED) {
 688                         spectre &= ~IBRS_SUPPORTED;
 689                         error = SYSCTL_OUT(req, "IBRS", 4);
 690                 } else
 691                 if (spectre & IBRS_AUTO_SUPPORTED) {
 692                         spectre &= ~IBRS_AUTO_SUPPORTED;
 693                         error = SYSCTL_OUT(req, "IBRS_AUTO", 9);
 694                 } else
 695                 if (spectre & STIBP_SUPPORTED) {
 696                         spectre &= ~STIBP_SUPPORTED;
 697                         error = SYSCTL_OUT(req, "STIBP", 5);
 698                 } else
 699                 if (spectre & STIBP_AUTO_SUPPORTED) {
 700                         spectre &= ~STIBP_AUTO_SUPPORTED;
 701                         error = SYSCTL_OUT(req, "STIBP_AUTO", 10);
 702                 } else
 703                 if (spectre & IBPB_SUPPORTED) {
 704                         spectre &= ~IBPB_SUPPORTED;
 705                         error = SYSCTL_OUT(req, "IBPB", 4);
 706                 }
 707         }
 708         if (loop == 0) {
 709                 error = SYSCTL_OUT(req, "NONE", 4);
 710         }
 711
 712         if (error || req->newptr == NULL)
 713                 return error;
 714         if ((oidp->oid_kind & CTLFLAG_WR) == 0)
 715                 return error;
 716
 717         /*
 718          * Change current operating mode
 719          */
 720         len = req->newlen - req->newidx;
 721         if (len >= sizeof(buf)) {
 722                 error = EINVAL;
 723                 len = 0;
 724         } else {
 725                 error = SYSCTL_IN(req, buf, len);
 726         }
 727         buf[len] = 0;
 728         iter = &buf[0];
 729         spectre = 0;
 730
 731         while (error == 0 && iter) {
 732                 ptr = strsep(&iter, " ,\t\r\n");
 733                 if (*ptr == 0)
 734                         continue;
 735                 if (strcasecmp(ptr, "NONE") == 0)
 736                         spectre |= 0;
 737                 else if (strcasecmp(ptr, "IBRS") == 0)
 738                         spectre |= IBRS_SUPPORTED;
 739                 else if (strcasecmp(ptr, "IBRS_AUTO") == 0)
 740                         spectre |= IBRS_AUTO_SUPPORTED;
 741                 else if (strcasecmp(ptr, "STIBP") == 0)
 742                         spectre |= STIBP_SUPPORTED;
 743                 else if (strcasecmp(ptr, "STIBP_AUTO") == 0)
 744                         spectre |= STIBP_AUTO_SUPPORTED;
 745                 else if (strcasecmp(ptr, "IBPB") == 0)
 746                         spectre |= IBPB_SUPPORTED;
 747                 else
 748                         error = ENOENT;
 749         }
 750         if (error == 0) {
 751                 spectre_mitigation = spectre;
 752                 spectre_sysctl_changed();
 753         }
 754         return error;
 755 }
 756
 757 SYSCTL_PROC(_machdep, OID_AUTO, spectre_mitigation,
 758         CTLTYPE_STRING | CTLFLAG_RW,
 759         0, 0, sysctl_spectre_mitigation, "A", "Spectre exploit mitigation");
 760 SYSCTL_PROC(_machdep, OID_AUTO, spectre_support,
 761         CTLTYPE_STRING | CTLFLAG_RD,
 762         0, 0, sysctl_spectre_mitigation, "A", "Spectre supported features");
 763
 764 /*
 765  * NOTE: Called at SI_BOOT2_MACHDEP and also when the microcode is
 766  *       updated.  Microcode updates must be applied to all cpus
 767  *       for support to be recognized.
 768  */
 769 static void
 770 spectre_vm_setup(void *arg)
 771 {
 772         int inconsistent = 0;
 773         int supmask;
 774
 775         /*
 776          * Fetch tunable in auto mode
 777          */
 778         if (spectre_mitigation < 0) {
 779                 TUNABLE_INT_FETCH("machdep.spectre_mitigation",
 780                                   &spectre_mitigation);
 781         }
 782
 783         if ((supmask = spectre_check_support()) != 0) {
 784                 /*
 785                  * Must be supported on all cpus before we
 786                  * can enable it.  Returns silently if it
 787                  * isn't.
 788                  *
 789                  * NOTE! arg != NULL indicates we were called
 790                  *       from cpuctl after a successful microcode
 791                  *       update.
 792                  */
 793                 if (arg != NULL) {
 794                         globaldata_t save_gd;
 795                         int n;
 796
 797                         save_gd = mycpu;
 798                         for (n = 0; n < ncpus; ++n) {
 799                                 lwkt_setcpu_self(globaldata_find(n));
 800                                 cpu_ccfence();
 801                                 if (spectre_check_support() !=
 802                                     supmask) {
 803                                         inconsistent = 1;
 804                                         break;
 805                                 }
 806                         }
 807                         lwkt_setcpu_self(save_gd);
 808                         cpu_ccfence();
 809                 }
 810         }
 811
 812         /*
 813          * Be silent while microcode is being loaded on various CPUs,
 814          * until all done.
 815          */
 816         if (inconsistent) {
 817                 spectre_mitigation = -1;
 818                 spectre_support = 0;
 819                 return;
 820         }
 821
 822         /*
 823          * IBRS support
 824          */
 825         spectre_support = supmask;
 826
 827         /*
 828          * Enable spectre_mitigation, set defaults if -1, adjust
 829          * tuned value according to support if not.
 830          *
 831          * NOTE!  We do not enable IBPB for user->kernel transitions
 832          *        by default, so this code is commented out for now.
 833          */
 834         if (spectre_support) {
 835                 if (spectre_mitigation < 0) {
 836                         spectre_mitigation = 0;
 837
 838                         /*
 839                          * IBRS toggling not currently recommended as a
 840                          * default.
 841                          */
 842                         if (spectre_support & IBRS_AUTO_SUPPORTED)
 843                                 spectre_mitigation |= IBRS_AUTO_SUPPORTED;
 844                         else if (spectre_support & IBRS_SUPPORTED)
 845                                 spectre_mitigation |= 0;
 846
 847                         /*
 848                          * STIBP toggling not currently recommended as a
 849                          * default.
 850                          */
 851                         if (spectre_support & STIBP_AUTO_SUPPORTED)
 852                                 spectre_mitigation |= STIBP_AUTO_SUPPORTED;
 853                         else if (spectre_support & STIBP_SUPPORTED)
 854                                 spectre_mitigation |= 0;
 855
 856                         /*
 857                          * IBPB adds enormous (~2uS) overhead to system
 858                          * calls etc, we do not enable it by default.
 859                          */
 860                         if (spectre_support & IBPB_SUPPORTED)
 861                                 spectre_mitigation |= 0;
 862                 }
 863         } else {
 864                 spectre_mitigation = -1;
 865         }
 866
 867         /*
 868          * Disallow sysctl changes when there is no support (otherwise
 869          * the wrmsr will cause a protection fault).
 870          */
 871         if (spectre_mitigation < 0)
 872                 sysctl___machdep_spectre_mitigation.oid_kind &= ~CTLFLAG_WR;
 873         else
 874                 sysctl___machdep_spectre_mitigation.oid_kind |= CTLFLAG_WR;
 875
 876         spectre_sysctl_changed();
 877 }
 878
 879 #define MDS_AVX512_4VNNIW_SUPPORTED     0x0001
 880 #define MDS_AVX512_4FMAPS_SUPPORTED     0x0002
 881 #define MDS_MD_CLEAR_SUPPORTED          0x0004
 882 #define MDS_TSX_FORCE_ABORT_SUPPORTED   0x0008
 883 #define MDS_NOT_REQUIRED                0x8000
 884
 885 static
 886 int
 887 mds_check_support(void)
 888 {
 889         uint64_t msr;
 890         uint32_t p[4];
 891         int rv = 0;
 892
 893         /*
 894          * MDS mitigation hw bits
 895          *
 896          * MD_CLEAR     Use microcode-supported verf insn.  This is the
 897          *              only mode we really support.
 898          */
 899         if (cpu_vendor_id == CPU_VENDOR_INTEL) {
 900                 p[0] = 0;
 901                 p[1] = 0;
 902                 p[2] = 0;
 903                 p[3] = 0;
 904                 cpuid_count(7, 0, p);
 905                 if (p[3] & CPUID_SEF_ARCH_CAP) {
 906                         msr = rdmsr(MSR_IA32_ARCH_CAPABILITIES);
 907                         if (msr & IA32_ARCH_MDS_NO)
 908                                 rv = MDS_NOT_REQUIRED;
 909                 }
 910                 if (p[3] & CPUID_SEF_AVX512_4VNNIW)
 911                         rv |= MDS_AVX512_4VNNIW_SUPPORTED;
 912                 if (p[3] & CPUID_SEF_AVX512_4FMAPS)
 913                         rv |= MDS_AVX512_4FMAPS_SUPPORTED;
 914                 if (p[3] & CPUID_SEF_MD_CLEAR)
 915                         rv |= MDS_MD_CLEAR_SUPPORTED;
 916                 if (p[3] & CPUID_SEF_TSX_FORCE_ABORT)
 917                         rv |= MDS_TSX_FORCE_ABORT_SUPPORTED;
 918         } else {
 919                 rv = MDS_NOT_REQUIRED;
 920         }
 921
 922         return rv;
 923 }
 924
 925 /*
 926  * Iterate CPUs and adjust MSR for global operations, since
 927  * the KMMU* code won't do it if spectre_mitigation is 0 or 2.
 928  */
 929 #define CHECK(flag)     (mds_mitigation & mds_support & (flag))
 930
 931 static
 932 void
 933 mds_sysctl_changed(void)
 934 {
 935         globaldata_t save_gd;
 936         struct trampframe *tr;
 937         int spec_ctrl;
 938         int spec_mask;
 939         int mode;
 940         int n;
 941
 942         spec_mask = SPEC_CTRL_MDS_ENABLE;
 943
 944         /*
 945          * Fixup state
 946          */
 947         mode = 0;
 948         save_gd = mycpu;
 949         for (n = 0; n < ncpus; ++n) {
 950                 lwkt_setcpu_self(globaldata_find(n));
 951                 cpu_ccfence();
 952                 tr = &pscpu->trampoline;
 953
 954                 /*
 955                  * Make sure we are cleaned out.
 956                  *
 957                  * XXX cleanup, reusing globals inside the loop (they get
 958                  * set to the same thing each loop)
 959                  *
 960                  * [0] kernel entry (idle exit)
 961                  * [1] kernel exit  (idle entry)
 962                  */
 963                 tr->tr_pcb_spec_ctrl[0] &= ~spec_mask;
 964                 tr->tr_pcb_spec_ctrl[1] &= ~spec_mask;
 965
 966                 /*
 967                  * Don't try to parse if not available
 968                  */
 969                 if (mds_mitigation < 0)
 970                         continue;
 971
 972                 spec_ctrl = 0;
 973                 if (CHECK(MDS_MD_CLEAR_SUPPORTED)) {
 974                         spec_ctrl |= SPEC_CTRL_MDS_ENABLE;
 975                         mode |= MDS_MD_CLEAR_SUPPORTED;
 976                 }
 977
 978                 /*
 979                  * Update spec_ctrl fields in the trampoline.
 980                  *
 981                  * [0] on-kernel-entry (on-idle-exit)
 982                  * [1] on-kernel-exit  (on-idle-entry)
 983                  *
 984                  * The MDS stuff is only needed on kernel-exit or idle-entry
 985                  */
 986                 /* tr->tr_pcb_spec_ctrl[0] |= spec_ctrl; */
 987                 tr->tr_pcb_spec_ctrl[1] |= spec_ctrl;
 988
 989                 /*
 990                  * Make sure we set this on the first loop.  It will be
 991                  * the same value on remaining loops.
 992                  */
 993                 mds_mode = mode;
 994         }
 995         lwkt_setcpu_self(save_gd);
 996         cpu_ccfence();
 997
 998         /*
 999          * Console message on mitigation mode change
1000          */
1001         kprintf("MDS: support=(");
1002         if (mds_support == 0) {
1003                 kprintf(" none");
1004         } else {
1005                 if (mds_support & MDS_AVX512_4VNNIW_SUPPORTED)
1006                         kprintf(" AVX512_4VNNIW");
1007                 if (mds_support & MDS_AVX512_4FMAPS_SUPPORTED)
1008                         kprintf(" AVX512_4FMAPS");
1009                 if (mds_support & MDS_MD_CLEAR_SUPPORTED)
1010                         kprintf(" MD_CLEAR");
1011                 if (mds_support & MDS_TSX_FORCE_ABORT_SUPPORTED)
1012                         kprintf(" TSX_FORCE_ABORT");
1013                 if (mds_support & MDS_NOT_REQUIRED)
1014                         kprintf(" MDS_NOT_REQUIRED");
1015         }
1016         kprintf(" ) req=%04x operating=(", (uint16_t)mds_mitigation);
1017         if (mds_mode == 0) {
1018                 kprintf(" none");
1019         } else {
1020                 if (mds_mode & MDS_AVX512_4VNNIW_SUPPORTED)
1021                         kprintf(" AVX512_4VNNIW");
1022                 if (mds_mode & MDS_AVX512_4FMAPS_SUPPORTED)
1023                         kprintf(" AVX512_4FMAPS");
1024                 if (mds_mode & MDS_MD_CLEAR_SUPPORTED)
1025                         kprintf(" MD_CLEAR");
1026                 if (mds_mode & MDS_TSX_FORCE_ABORT_SUPPORTED)
1027                         kprintf(" TSX_FORCE_ABORT");
1028                 if (mds_mode & MDS_NOT_REQUIRED)
1029                         kprintf(" MDS_NOT_REQUIRED");
1030         }
1031         kprintf(" )\n");
1032 }
1033
1034 #undef CHECK
1035
1036 /*
1037  * User changes sysctl value
1038  */
1039 static int
1040 sysctl_mds_mitigation(SYSCTL_HANDLER_ARGS)
1041 {
1042         char buf[128];
1043         char *ptr;
1044         char *iter;
1045         size_t len;
1046         int mds;
1047         int error = 0;
1048         int loop = 0;
1049
1050         /*
1051          * Return current operating mode or support.
1052          */
1053         if (oidp->oid_kind & CTLFLAG_WR)
1054                 mds = mds_mode;
1055         else
1056                 mds = mds_support;
1057
1058         mds &= MDS_AVX512_4VNNIW_SUPPORTED |
1059                MDS_AVX512_4FMAPS_SUPPORTED |
1060                MDS_MD_CLEAR_SUPPORTED |
1061                MDS_TSX_FORCE_ABORT_SUPPORTED |
1062                MDS_NOT_REQUIRED;
1063
1064         while (mds) {
1065                 if (error)
1066                         break;
1067                 if (loop++) {
1068                         error = SYSCTL_OUT(req, " ", 1);
1069                         if (error)
1070                                 break;
1071                 }
1072                 if (mds & MDS_AVX512_4VNNIW_SUPPORTED) {
1073                         mds &= ~MDS_AVX512_4VNNIW_SUPPORTED;
1074                         error = SYSCTL_OUT(req, "AVX512_4VNNIW", 13);
1075                 } else
1076                 if (mds & MDS_AVX512_4FMAPS_SUPPORTED) {
1077                         mds &= ~MDS_AVX512_4FMAPS_SUPPORTED;
1078                         error = SYSCTL_OUT(req, "AVX512_4FMAPS", 13);
1079                 } else
1080                 if (mds & MDS_MD_CLEAR_SUPPORTED) {
1081                         mds &= ~MDS_MD_CLEAR_SUPPORTED;
1082                         error = SYSCTL_OUT(req, "MD_CLEAR", 8);
1083                 } else
1084                 if (mds & MDS_TSX_FORCE_ABORT_SUPPORTED) {
1085                         mds &= ~MDS_TSX_FORCE_ABORT_SUPPORTED;
1086                         error = SYSCTL_OUT(req, "TSX_FORCE_ABORT", 15);
1087                 } else
1088                 if (mds & MDS_NOT_REQUIRED) {
1089                         mds &= ~MDS_NOT_REQUIRED;
1090                         error = SYSCTL_OUT(req, "MDS_NOT_REQUIRED", 16);
1091                 }
1092         }
1093         if (loop == 0) {
1094                 error = SYSCTL_OUT(req, "NONE", 4);
1095         }
1096
1097         if (error || req->newptr == NULL)
1098                 return error;
1099         if ((oidp->oid_kind & CTLFLAG_WR) == 0)
1100                 return error;
1101
1102         /*
1103          * Change current operating mode
1104          */
1105         len = req->newlen - req->newidx;
1106         if (len >= sizeof(buf)) {
1107                 error = EINVAL;
1108                 len = 0;
1109         } else {
1110                 error = SYSCTL_IN(req, buf, len);
1111         }
1112         buf[len] = 0;
1113         iter = &buf[0];
1114         mds = 0;
1115
1116         while (error == 0 && iter) {
1117                 ptr = strsep(&iter, " ,\t\r\n");
1118                 if (*ptr == 0)
1119                         continue;
1120                 if (strcasecmp(ptr, "NONE") == 0)
1121                         mds |= 0;
1122                 else if (strcasecmp(ptr, "AVX512_4VNNIW") == 0)
1123                         mds |= MDS_AVX512_4VNNIW_SUPPORTED;
1124                 else if (strcasecmp(ptr, "AVX512_4FMAPS") == 0)
1125                         mds |= MDS_AVX512_4FMAPS_SUPPORTED;
1126                 else if (strcasecmp(ptr, "MD_CLEAR") == 0)
1127                         mds |= MDS_MD_CLEAR_SUPPORTED;
1128                 else if (strcasecmp(ptr, "TSX_FORCE_ABORT") == 0)
1129                         mds |= MDS_TSX_FORCE_ABORT_SUPPORTED;
1130                 else if (strcasecmp(ptr, "MDS_NOT_REQUIRED") == 0)
1131                         mds |= MDS_NOT_REQUIRED;
1132                 else
1133                         error = ENOENT;
1134         }
1135         if (error == 0) {
1136                 mds_mitigation = mds;
1137                 mds_sysctl_changed();
1138         }
1139         return error;
1140 }
1141
1142 SYSCTL_PROC(_machdep, OID_AUTO, mds_mitigation,
1143         CTLTYPE_STRING | CTLFLAG_RW,
1144         0, 0, sysctl_mds_mitigation, "A", "MDS exploit mitigation");
1145 SYSCTL_PROC(_machdep, OID_AUTO, mds_support,
1146         CTLTYPE_STRING | CTLFLAG_RD,
1147         0, 0, sysctl_mds_mitigation, "A", "MDS supported features");
1148
1149 /*
1150  * NOTE: Called at SI_BOOT2_MACHDEP and also when the microcode is
1151  *       updated.  Microcode updates must be applied to all cpus
1152  *       for support to be recognized.
1153  */
1154 static void
1155 mds_vm_setup(void *arg)
1156 {
1157         int inconsistent = 0;
1158         int supmask;
1159
1160         /*
1161          * Fetch tunable in auto mode
1162          */
1163         if (mds_mitigation < 0) {
1164                 TUNABLE_INT_FETCH("machdep.mds_mitigation", &mds_mitigation);
1165         }
1166
1167         if ((supmask = mds_check_support()) != 0) {
1168                 /*
1169                  * Must be supported on all cpus before we
1170                  * can enable it.  Returns silently if it
1171                  * isn't.
1172                  *
1173                  * NOTE! arg != NULL indicates we were called
1174                  *       from cpuctl after a successful microcode
1175                  *       update.
1176                  */
1177                 if (arg != NULL) {
1178                         globaldata_t save_gd;
1179                         int n;
1180
1181                         save_gd = mycpu;
1182                         for (n = 0; n < ncpus; ++n) {
1183                                 lwkt_setcpu_self(globaldata_find(n));
1184                                 cpu_ccfence();
1185                                 if (mds_check_support() != supmask) {
1186                                         inconsistent = 1;
1187                                         break;
1188                                 }
1189                         }
1190                         lwkt_setcpu_self(save_gd);
1191                         cpu_ccfence();
1192                 }
1193         }
1194
1195         /*
1196          * Be silent while microcode is being loaded on various CPUs,
1197          * until all done.
1198          */
1199         if (inconsistent) {
1200                 mds_mitigation = -1;
1201                 mds_support = 0;
1202                 return;
1203         }
1204
1205         /*
1206          * IBRS support
1207          */
1208         mds_support = supmask;
1209
1210         /*
1211          * Enable mds_mitigation, set defaults if -1, adjust
1212          * tuned value according to support if not.
1213          *
1214          * NOTE!  MDS is not enabled by default.
1215          */
1216         if (mds_support) {
1217                 if (mds_mitigation < 0) {
1218                         mds_mitigation = 0;
1219
1220                         if ((mds_support & MDS_NOT_REQUIRED) == 0 &&
1221                             (mds_support & MDS_MD_CLEAR_SUPPORTED)) {
1222                                 /* mds_mitigation |= MDS_MD_CLEAR_SUPPORTED; */
1223                         }
1224                 }
1225         } else {
1226                 mds_mitigation = -1;
1227         }
1228
1229         /*
1230          * Disallow sysctl changes when there is no support (otherwise
1231          * the wrmsr will cause a protection fault).
1232          */
1233         if (mds_mitigation < 0)
1234                 sysctl___machdep_mds_mitigation.oid_kind &= ~CTLFLAG_WR;
1235         else
1236                 sysctl___machdep_mds_mitigation.oid_kind |= CTLFLAG_WR;
1237
1238         mds_sysctl_changed();
1239 }
1240
1241 /*
1242  * NOTE: Called at SI_BOOT2_MACHDEP and also when the microcode is
1243  *       updated.  Microcode updates must be applied to all cpus
1244  *       for support to be recognized.
1245  */
1246 void
1247 mitigation_vm_setup(void *arg)
1248 {
1249         spectre_vm_setup(arg);
1250         mds_vm_setup(arg);
1251 }
1252
1253 SYSINIT(mitigation_vm_setup, SI_BOOT2_MACHDEP, SI_ORDER_ANY,
1254         mitigation_vm_setup, NULL);
1255
1256 /*
1257  * platform-specific vmspace initialization (nothing for x86_64)
1258  */
1259 void
1260 cpu_vmspace_alloc(struct vmspace *vm __unused)
1261 {
1262 }
1263
1264 void
1265 cpu_vmspace_free(struct vmspace *vm __unused)
1266 {
1267 }
1268
1269 int
1270 kvm_access_check(vm_offset_t saddr, vm_offset_t eaddr, int prot)
1271 {
1272         vm_offset_t addr;
1273
1274         if (saddr < KvaStart)
1275                 return EFAULT;
1276         if (eaddr >= KvaEnd)
1277                 return EFAULT;
1278         for (addr = saddr; addr < eaddr; addr += PAGE_SIZE)  {
1279                 if (pmap_kextract(addr) == 0)
1280                         return EFAULT;
1281         }
1282         if (!kernacc((caddr_t)saddr, eaddr - saddr, prot))
1283                 return EFAULT;
1284         return 0;
1285 }
1286
1287 #if 0
1288
1289 void _test_frame_enter(struct trapframe *frame);
1290 void _test_frame_exit(struct trapframe *frame);
1291
1292 void
1293 _test_frame_enter(struct trapframe *frame)
1294 {
1295         thread_t td = curthread;
1296
1297         if (ISPL(frame->tf_cs) == SEL_UPL) {
1298                 KKASSERT(td->td_lwp);
1299                 KASSERT(td->td_lwp->lwp_md.md_regs == frame,
1300                         ("_test_frame_exit: Frame mismatch %p %p",
1301                         td->td_lwp->lwp_md.md_regs, frame));
1302             td->td_lwp->lwp_saveusp = (void *)frame->tf_rsp;
1303             td->td_lwp->lwp_saveupc = (void *)frame->tf_rip;
1304         }
1305         if ((char *)frame < td->td_kstack ||
1306             (char *)frame > td->td_kstack + td->td_kstack_size) {
1307                 panic("_test_frame_exit: frame not on kstack %p kstack=%p",
1308                         frame, td->td_kstack);
1309         }
1310 }
1311
1312 void
1313 _test_frame_exit(struct trapframe *frame)
1314 {
1315         thread_t td = curthread;
1316
1317         if (ISPL(frame->tf_cs) == SEL_UPL) {
1318                 KKASSERT(td->td_lwp);
1319                 KASSERT(td->td_lwp->lwp_md.md_regs == frame,
1320                         ("_test_frame_exit: Frame mismatch %p %p",
1321                         td->td_lwp->lwp_md.md_regs, frame));
1322                 if (td->td_lwp->lwp_saveusp != (void *)frame->tf_rsp) {
1323                         kprintf("_test_frame_exit: %s:%d usp mismatch %p/%p\n",
1324                                 td->td_comm, td->td_proc->p_pid,
1325                                 td->td_lwp->lwp_saveusp,
1326                                 (void *)frame->tf_rsp);
1327                 }
1328                 if (td->td_lwp->lwp_saveupc != (void *)frame->tf_rip) {
1329                         kprintf("_test_frame_exit: %s:%d upc mismatch %p/%p\n",
1330                                 td->td_comm, td->td_proc->p_pid,
1331                                 td->td_lwp->lwp_saveupc,
1332                                 (void *)frame->tf_rip);
1333                 }
1334
1335                 /*
1336                  * adulterate the fields to catch entries that
1337                  * don't run through test_frame_enter
1338                  */
1339                 td->td_lwp->lwp_saveusp =
1340                         (void *)~(intptr_t)td->td_lwp->lwp_saveusp;
1341                 td->td_lwp->lwp_saveupc =
1342                         (void *)~(intptr_t)td->td_lwp->lwp_saveupc;
1343         }
1344         if ((char *)frame < td->td_kstack ||
1345             (char *)frame > td->td_kstack + td->td_kstack_size) {
1346                 panic("_test_frame_exit: frame not on kstack %p kstack=%p",
1347                         frame, td->td_kstack);
1348         }
1349 }
1350
1351 #endif