sys/platform/pc64/x86_64/vm_machdep.c

   1 /*-
   2  * Copyright (c) 1982, 1986 The Regents of the University of California.
   3  * Copyright (c) 1989, 1990 William Jolitz
   4  * Copyright (c) 1994 John Dyson
   5  * Copyright (c) 2008-2018 The DragonFly Project.
   6  * All rights reserved.
   7  *
   8  * This code is derived from software contributed to Berkeley by
   9  * the Systems Programming Group of the University of Utah Computer
  10  * Science Department, and William Jolitz.
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice, this list of conditions and the following disclaimer.
  17  * 2. Redistributions in binary form must reproduce the above copyright
  18  *    notice, this list of conditions and the following disclaimer in the
  19  *    documentation and/or other materials provided with the distribution.
  20  * 3. All advertising materials mentioning features or use of this software
  21  *    must display the following acknowledgement:
  22  *      This product includes software developed by the University of
  23  *      California, Berkeley and its contributors.
  24  * 4. Neither the name of the University nor the names of its contributors
  25  *    may be used to endorse or promote products derived from this software
  26  *    without specific prior written permission.
  27  *
  28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  38  * SUCH DAMAGE.
  39  *
  40  *      from: @(#)vm_machdep.c  7.3 (Berkeley) 5/13/91
  41  *      Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$
  42  * $FreeBSD: src/sys/i386/i386/vm_machdep.c,v 1.132.2.9 2003/01/25 19:02:23 dillon Exp $
  43  */
  44
  45 #include "opt_reset.h"
  46
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/malloc.h>
  50 #include <sys/proc.h>
  51 #include <sys/buf.h>
  52 #include <sys/interrupt.h>
  53 #include <sys/vnode.h>
  54 #include <sys/vmmeter.h>
  55 #include <sys/kernel.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/unistd.h>
  58 #include <sys/lwp.h>
  59
  60 #include <machine/clock.h>
  61 #include <machine/cpu.h>
  62 #include <machine/md_var.h>
  63 #include <machine/smp.h>
  64 #include <machine/pcb.h>
  65 #include <machine/pcb_ext.h>
  66 #include <machine/segments.h>
  67 #include <machine/globaldata.h> /* npxthread */
  68 #include <machine/specialreg.h>
  69
  70 #include <vm/vm.h>
  71 #include <vm/vm_param.h>
  72 #include <sys/lock.h>
  73 #include <vm/vm_kern.h>
  74 #include <vm/vm_page.h>
  75 #include <vm/vm_map.h>
  76 #include <vm/vm_extern.h>
  77
  78 #include <sys/thread2.h>
  79
  80 #include <bus/isa/isa.h>
  81
  82 static void     cpu_reset_real (void);
  83
  84 static int spectre_mitigation = -1;
  85 static int spectre_support = 0;
  86 static int spectre_mode = 0;
  87 SYSCTL_INT(_machdep, OID_AUTO, spectre_mode, CTLFLAG_RD,
  88         &spectre_mode, 0, "current Spectre enablements");
  89
  90 static int mds_mitigation = -1;
  91 static int mds_support = 0;
  92 static int mds_mode = 0;
  93 SYSCTL_INT(_machdep, OID_AUTO, mds_mode, CTLFLAG_RD,
  94         &mds_mode, 0, "current MDS enablements");
  95
  96 /*
  97  * Finish a fork operation, with lwp lp2 nearly set up.
  98  * Copy and update the pcb, set up the stack so that the child
  99  * ready to run and return to user mode.
 100  */
 101 void
 102 cpu_fork(struct lwp *lp1, struct lwp *lp2, int flags)
 103 {
 104         struct pcb *pcb2;
 105         struct pmap *pmap2;
 106
 107         if ((flags & RFPROC) == 0) {
 108                 if ((flags & RFMEM) == 0) {
 109                         /*
 110                          * Unshare user LDT.  > 1 test is MPSAFE.  While
 111                          * it can potentially race a 2->1 transition, the
 112                          * worst that happens is that we do an unnecessary
 113                          * ldt replacement.
 114                          */
 115                         struct pcb *pcb1 = lp1->lwp_thread->td_pcb;
 116                         struct pcb_ldt *pcb_ldt = pcb1->pcb_ldt;
 117
 118                         if (pcb_ldt && pcb_ldt->ldt_refcnt > 1) {
 119                                 pcb_ldt = user_ldt_alloc(pcb1,pcb_ldt->ldt_len);
 120                                 user_ldt_free(pcb1);
 121                                 pcb1->pcb_ldt = pcb_ldt;
 122                                 set_user_ldt(pcb1);
 123                         }
 124                 }
 125                 return;
 126         }
 127
 128         /* Ensure that lp1's pcb is up to date. */
 129         if (mdcpu->gd_npxthread == lp1->lwp_thread)
 130                 npxsave(lp1->lwp_thread->td_savefpu);
 131
 132         /*
 133          * Copy lp1's PCB.  This really only applies to the
 134          * debug registers and FP state, but its faster to just copy the
 135          * whole thing.  Because we only save the PCB at switchout time,
 136          * the register state may not be current.
 137          */
 138         pcb2 = lp2->lwp_thread->td_pcb;
 139         *pcb2 = *lp1->lwp_thread->td_pcb;
 140
 141         /*
 142          * Create a new fresh stack for the new process.
 143          * Copy the trap frame for the return to user mode as if from a
 144          * syscall.  This copies the user mode register values.
 145          *
 146          * pcb_rsp must allocate an additional call-return pointer below
 147          * the trap frame which will be restored by cpu_heavy_restore from
 148          * PCB_RIP, and the thread's td_sp pointer must allocate an
 149          * additonal two quadwords below the pcb_rsp call-return pointer to
 150          * hold the LWKT restore function pointer and rflags.
 151          *
 152          * The LWKT restore function pointer must be set to cpu_heavy_restore,
 153          * which is our standard heavy-weight process switch-in function.
 154          * YYY eventually we should shortcut fork_return and fork_trampoline
 155          * to use the LWKT restore function directly so we can get rid of
 156          * all the extra crap we are setting up.
 157          */
 158         lp2->lwp_md.md_regs = (struct trapframe *)pcb2 - 1;
 159         bcopy(lp1->lwp_md.md_regs, lp2->lwp_md.md_regs, sizeof(*lp2->lwp_md.md_regs));
 160
 161         /*
 162          * Set registers for trampoline to user mode.  Leave space for the
 163          * return address on stack.  These are the kernel mode register values.
 164          *
 165          * Set the new pmap CR3.  If the new process uses isolated VM spaces,
 166          * also set the isolated CR3.
 167          */
 168         pmap2 = vmspace_pmap(lp2->lwp_proc->p_vmspace);
 169         pcb2->pcb_cr3 = vtophys(pmap2->pm_pml4);
 170         if ((pcb2->pcb_flags & PCB_ISOMMU) && pmap2->pm_pmlpv_iso) {
 171                 pcb2->pcb_cr3_iso = vtophys(pmap2->pm_pml4_iso);
 172         } else {
 173                 pcb2->pcb_flags &= ~PCB_ISOMMU;
 174                 pcb2->pcb_cr3_iso = 0;
 175         }
 176
 177 #if 0
 178         /*
 179          * Per-process spectre mitigation (future)
 180          */
 181         pcb2->pcb_flags &= ~(PCB_IBRS1 | PCB_IBRS2);
 182         switch (spectre_mitigation) {
 183         case 1:
 184                 pcb2->pcb_flags |= PCB_IBRS1;
 185                 break;
 186         case 2:
 187                 pcb2->pcb_flags |= PCB_IBRS2;
 188                 break;
 189         default:
 190                 break;
 191         }
 192 #endif
 193
 194         pcb2->pcb_rbx = (unsigned long)fork_return;     /* fork_trampoline argument */
 195         pcb2->pcb_rbp = 0;
 196         pcb2->pcb_rsp = (unsigned long)lp2->lwp_md.md_regs - sizeof(void *);
 197         pcb2->pcb_r12 = (unsigned long)lp2;             /* fork_trampoline argument */
 198         pcb2->pcb_r13 = 0;
 199         pcb2->pcb_r14 = 0;
 200         pcb2->pcb_r15 = 0;
 201         pcb2->pcb_rip = (unsigned long)fork_trampoline;
 202         lp2->lwp_thread->td_sp = (char *)(pcb2->pcb_rsp - sizeof(void *));
 203         *(u_int64_t *)lp2->lwp_thread->td_sp = PSL_USER;
 204         lp2->lwp_thread->td_sp -= sizeof(void *);
 205         *(void **)lp2->lwp_thread->td_sp = (void *)cpu_heavy_restore;
 206
 207         /*
 208          * pcb2->pcb_ldt:       duplicated below, if necessary.
 209          * pcb2->pcb_savefpu:   cloned above.
 210          * pcb2->pcb_flags:     cloned above
 211          * pcb2->pcb_onfault:   cloned above (always NULL here).
 212          * pcb2->pcb_onfault_sp:cloned above (dont care)
 213          */
 214
 215         /*
 216          * XXX don't copy the i/o pages.  this should probably be fixed.
 217          */
 218         pcb2->pcb_ext = NULL;
 219
 220         /* Copy the LDT, if necessary. */
 221         if (pcb2->pcb_ldt != NULL) {
 222                 if (flags & RFMEM) {
 223                         atomic_add_int(&pcb2->pcb_ldt->ldt_refcnt, 1);
 224                 } else {
 225                         pcb2->pcb_ldt = user_ldt_alloc(pcb2,
 226                                                        pcb2->pcb_ldt->ldt_len);
 227                 }
 228         }
 229         bcopy(&lp1->lwp_thread->td_tls, &lp2->lwp_thread->td_tls,
 230               sizeof(lp2->lwp_thread->td_tls));
 231         /*
 232          * Now, cpu_switch() can schedule the new lwp.
 233          * pcb_rsp is loaded pointing to the cpu_switch() stack frame
 234          * containing the return address when exiting cpu_switch.
 235          * This will normally be to fork_trampoline(), which will have
 236          * %rbx loaded with the new lwp's pointer.  fork_trampoline()
 237          * will set up a stack to call fork_return(lp, frame); to complete
 238          * the return to user-mode.
 239          */
 240 }
 241
 242 /*
 243  * Prepare new lwp to return to the address specified in params.
 244  */
 245 int
 246 cpu_prepare_lwp(struct lwp *lp, struct lwp_params *params)
 247 {
 248         struct trapframe *regs = lp->lwp_md.md_regs;
 249         void *bad_return = NULL;
 250         int error;
 251
 252         regs->tf_rip = (long)params->lwp_func;
 253         regs->tf_rsp = (long)params->lwp_stack;
 254         /* Set up argument for function call */
 255         regs->tf_rdi = (long)params->lwp_arg;
 256
 257         /*
 258          * Set up fake return address.  As the lwp function may never return,
 259          * we simply copy out a NULL pointer and force the lwp to receive
 260          * a SIGSEGV if it returns anyways.
 261          */
 262         regs->tf_rsp -= sizeof(void *);
 263         error = copyout(&bad_return, (void *)regs->tf_rsp, sizeof(bad_return));
 264         if (error)
 265                 return (error);
 266
 267         cpu_set_fork_handler(lp,
 268             (void (*)(void *, struct trapframe *))generic_lwp_return, lp);
 269         return (0);
 270 }
 271
 272 /*
 273  * Intercept the return address from a freshly forked process that has NOT
 274  * been scheduled yet.
 275  *
 276  * This is needed to make kernel threads stay in kernel mode.
 277  */
 278 void
 279 cpu_set_fork_handler(struct lwp *lp, void (*func)(void *, struct trapframe *),
 280                      void *arg)
 281 {
 282         /*
 283          * Note that the trap frame follows the args, so the function
 284          * is really called like this:  func(arg, frame);
 285          */
 286         lp->lwp_thread->td_pcb->pcb_rbx = (long)func;   /* function */
 287         lp->lwp_thread->td_pcb->pcb_r12 = (long)arg;    /* first arg */
 288 }
 289
 290 void
 291 cpu_set_thread_handler(thread_t td, void (*rfunc)(void), void *func, void *arg)
 292 {
 293         td->td_pcb->pcb_rbx = (long)func;
 294         td->td_pcb->pcb_r12 = (long)arg;
 295         td->td_switch = cpu_lwkt_switch;
 296         td->td_sp -= sizeof(void *);
 297         *(void **)td->td_sp = rfunc;    /* exit function on return */
 298         td->td_sp -= sizeof(void *);
 299         *(void **)td->td_sp = cpu_kthread_restore;
 300 }
 301
 302 void
 303 cpu_lwp_exit(void)
 304 {
 305         struct thread *td = curthread;
 306         struct pcb *pcb;
 307
 308         pcb = td->td_pcb;
 309
 310         /* Some x86 functionality was dropped */
 311         KKASSERT(pcb->pcb_ext == NULL);
 312
 313         /*
 314          * disable all hardware breakpoints
 315          */
 316         if (pcb->pcb_flags & PCB_DBREGS) {
 317                 reset_dbregs();
 318                 pcb->pcb_flags &= ~PCB_DBREGS;
 319         }
 320         td->td_gd->gd_cnt.v_swtch++;
 321
 322         crit_enter_quick(td);
 323         if (td->td_flags & TDF_TSLEEPQ)
 324                 tsleep_remove(td);
 325         lwkt_deschedule_self(td);
 326         lwkt_remove_tdallq(td);
 327         cpu_thread_exit();
 328 }
 329
 330 /*
 331  * Terminate the current thread.  The caller must have already acquired
 332  * the thread's rwlock and placed it on a reap list or otherwise notified
 333  * a reaper of its existance.  We set a special assembly switch function which
 334  * releases td_rwlock after it has cleaned up the MMU state and switched
 335  * out the stack.
 336  *
 337  * Must be caller from a critical section and with the thread descheduled.
 338  */
 339 void
 340 cpu_thread_exit(void)
 341 {
 342         npxexit();
 343         curthread->td_switch = cpu_exit_switch;
 344         curthread->td_flags |= TDF_EXITING;
 345         lwkt_switch();
 346         panic("cpu_thread_exit: lwkt_switch() unexpectedly returned");
 347 }
 348
 349 void
 350 cpu_reset(void)
 351 {
 352         cpu_reset_real();
 353 }
 354
 355 static void
 356 cpu_reset_real(void)
 357 {
 358         /*
 359          * Attempt to do a CPU reset via the keyboard controller,
 360          * do not turn off the GateA20, as any machine that fails
 361          * to do the reset here would then end up in no man's land.
 362          */
 363
 364 #if !defined(BROKEN_KEYBOARD_RESET)
 365         outb(IO_KBD + 4, 0xFE);
 366         DELAY(500000);  /* wait 0.5 sec to see if that did it */
 367         kprintf("Keyboard reset did not work, attempting CPU shutdown\n");
 368         DELAY(1000000); /* wait 1 sec for kprintf to complete */
 369 #endif
 370 #if 0 /* JG */
 371         /* force a shutdown by unmapping entire address space ! */
 372         bzero((caddr_t) PTD, PAGE_SIZE);
 373 #endif
 374
 375         /* "good night, sweet prince .... <THUNK!>" */
 376         cpu_invltlb();
 377         /* NOTREACHED */
 378         while(1);
 379 }
 380
 381 static void
 382 swi_vm(void *arg, void *frame)
 383 {
 384         if (busdma_swi_pending != 0)
 385                 busdma_swi();
 386 }
 387
 388 static void
 389 swi_vm_setup(void *arg)
 390 {
 391         register_swi_mp(SWI_VM, swi_vm, NULL, "swi_vm", NULL, 0);
 392 }
 393
 394 SYSINIT(swi_vm_setup, SI_BOOT2_MACHDEP, SI_ORDER_ANY, swi_vm_setup, NULL);
 395
 396 /*
 397  * NOTE: This routine is also called after a successful microcode
 398  *       reload on cpu 0.
 399  */
 400 void mitigation_vm_setup(void *arg);
 401
 402 /*
 403  * Check for IBPB and IBRS support
 404  *
 405  * This bits also specify desired modes in the spectre_mitigation sysctl.
 406  */
 407 #define IBRS_SUPPORTED          0x0001
 408 #define STIBP_SUPPORTED         0x0002
 409 #define IBPB_SUPPORTED          0x0004
 410 #define IBRS_AUTO_SUPPORTED     0x0008
 411 #define STIBP_AUTO_SUPPORTED    0x0010
 412 #define IBRS_PREFERRED_REQUEST  0x0020
 413
 414 static
 415 int
 416 spectre_check_support(void)
 417 {
 418         uint32_t p[4];
 419         int rv = 0;
 420
 421         /*
 422          * Spectre mitigation hw bits
 423          *
 424          * IBRS         Indirect Branch Restricted Speculation   (isolation)
 425          * STIBP        Single Thread Indirect Branch Prediction (isolation)
 426          * IBPB         Branch Prediction Barrier                (barrier)
 427          *
 428          * IBRS and STIBP must be toggled (enabled on entry to kernel,
 429          * disabled on exit, as well as disabled during any MWAIT/HLT).
 430          * When *_AUTO bits are available, IBRS and STIBP may be left
 431          * turned on and do not have to be toggled on kernel entry/exit.
 432          * Be sure to clear before going idle (else hyperthread performance
 433          * will drop).
 434          *
 435          * All this shit has enormous overhead.  IBPB in particular, and
 436          * non-auto modes are disabled by default.
 437          */
 438         if (cpu_vendor_id == CPU_VENDOR_INTEL) {
 439                 p[0] = 0;
 440                 p[1] = 0;
 441                 p[2] = 0;
 442                 p[3] = 0;
 443                 cpuid_count(7, 0, p);
 444                 if (p[3] & CPUID_STDEXT3_IBPB)
 445                         rv |= IBRS_SUPPORTED | IBPB_SUPPORTED;
 446                 if (p[3] & CPUID_STDEXT3_STIBP)
 447                         rv |= STIBP_SUPPORTED;
 448
 449                 /*
 450                  * 0x80000008 p[1] bit 12 indicates IBPB support
 451                  *
 452                  * This bit might be set even though STDEXT3_IBPB is not set.
 453                  */
 454                 p[0] = 0;
 455                 p[1] = 0;
 456                 p[2] = 0;
 457                 p[3] = 0;
 458                 do_cpuid(0x80000008U, p);
 459                 if (p[1] & CPUID_CAPEX_IBPB)
 460                         rv |= IBPB_SUPPORTED;
 461         } else if (cpu_vendor_id == CPU_VENDOR_AMD) {
 462                 /*
 463                  * 0x80000008
 464                  *      p[1] bit 12 indicates IBPB support
 465                  *      p[1] bit 14 indicates IBRS support
 466                  *      p[1] bit 15 indicates STIBP support
 467                  *
 468                  *      p[1] bit 16 indicates IBRS auto support
 469                  *      p[1] bit 17 indicates STIBP auto support
 470                  *      p[1] bit 18 indicates processor prefers using
 471                  *              IBRS instead of retpoline.
 472                  */
 473                 p[0] = 0;
 474                 p[1] = 0;
 475                 p[2] = 0;
 476                 p[3] = 0;
 477                 do_cpuid(0x80000008U, p);
 478                 if (p[1] & CPUID_CAPEX_IBPB)
 479                         rv |= IBPB_SUPPORTED;
 480                 if (p[1] & CPUID_CAPEX_IBRS)
 481                         rv |= IBRS_SUPPORTED;
 482                 if (p[1] & CPUID_CAPEX_STIBP)
 483                         rv |= STIBP_SUPPORTED;
 484
 485                 if (p[1] & CPUID_CAPEX_IBRS_ALWAYSON)
 486                         rv |= IBRS_AUTO_SUPPORTED;
 487                 if (p[1] & CPUID_CAPEX_STIBP_ALWAYSON)
 488                         rv |= STIBP_AUTO_SUPPORTED;
 489                 if (p[1] & CPUID_CAPEX_PREFER_IBRS)
 490                         rv |= IBRS_PREFERRED_REQUEST;
 491         }
 492
 493         return rv;
 494 }
 495
 496 /*
 497  * Iterate CPUs and adjust MSR for global operations, since
 498  * the KMMU* code won't do it if spectre_mitigation is 0 or 2.
 499  */
 500 #define CHECK(flag)     (spectre_mitigation & spectre_support & (flag))
 501
 502 static
 503 void
 504 spectre_sysctl_changed(void)
 505 {
 506         globaldata_t save_gd;
 507         struct trampframe *tr;
 508         int spec_ctrl;
 509         int spec_mask;
 510         int mode;
 511         int n;
 512
 513         spec_mask = SPEC_CTRL_IBRS | SPEC_CTRL_STIBP |
 514                     SPEC_CTRL_DUMMY_ENABLE | SPEC_CTRL_DUMMY_IBPB;
 515
 516         /*
 517          * Fixup state
 518          */
 519         mode = 0;
 520         save_gd = mycpu;
 521         for (n = 0; n < ncpus; ++n) {
 522                 lwkt_setcpu_self(globaldata_find(n));
 523                 cpu_ccfence();
 524                 tr = &pscpu->trampoline;
 525
 526                 /*
 527                  * Make sure we are cleaned out.
 528                  *
 529                  * XXX cleanup, reusing globals inside the loop (they get
 530                  * set to the same thing each loop)
 531                  *
 532                  * [0] kernel entry (idle exit)
 533                  * [1] kernel exit  (idle entry)
 534                  */
 535                 tr->tr_pcb_spec_ctrl[0] &= ~spec_mask;
 536                 tr->tr_pcb_spec_ctrl[1] &= ~spec_mask;
 537
 538                 /*
 539                  * Don't try to parse if not available
 540                  */
 541                 if (spectre_mitigation < 0)
 542                         continue;
 543
 544                 /*
 545                  * IBRS mode.  Auto overrides toggling.
 546                  *
 547                  * Only set the ENABLE flag if we have to toggle something
 548                  * on entry and exit.
 549                  */
 550                 spec_ctrl = 0;
 551                 if (CHECK(IBRS_AUTO_SUPPORTED)) {
 552                         spec_ctrl |= SPEC_CTRL_IBRS;
 553                         mode |= IBRS_AUTO_SUPPORTED;
 554                 } else if (CHECK(IBRS_SUPPORTED)) {
 555                         spec_ctrl |= SPEC_CTRL_IBRS | SPEC_CTRL_DUMMY_ENABLE;
 556                         mode |= IBRS_SUPPORTED;
 557                 }
 558                 if (CHECK(STIBP_AUTO_SUPPORTED)) {
 559                         spec_ctrl |= SPEC_CTRL_STIBP;
 560                         mode |= STIBP_AUTO_SUPPORTED;
 561                 } else if (CHECK(STIBP_SUPPORTED)) {
 562                         spec_ctrl |= SPEC_CTRL_STIBP | SPEC_CTRL_DUMMY_ENABLE;
 563                         mode |= STIBP_SUPPORTED;
 564                 }
 565
 566                 /*
 567                  * IBPB requested and supported.
 568                  */
 569                 if (CHECK(IBPB_SUPPORTED)) {
 570                         spec_ctrl |= SPEC_CTRL_DUMMY_IBPB;
 571                         mode |= IBPB_SUPPORTED;
 572                 }
 573
 574                 /*
 575                  * Update the MSR if the cpu supports the modes to ensure
 576                  * proper disablement if the user disabled the mode.
 577                  */
 578                 if (spectre_support & (IBRS_SUPPORTED | IBRS_AUTO_SUPPORTED |
 579                                     STIBP_SUPPORTED | STIBP_AUTO_SUPPORTED)) {
 580                         wrmsr(MSR_SPEC_CTRL,
 581                               spec_ctrl & (SPEC_CTRL_IBRS|SPEC_CTRL_STIBP));
 582                 }
 583
 584                 /*
 585                  * Update spec_ctrl fields in the trampoline.
 586                  *
 587                  * [0] on-kernel-entry (on-idle-exit)
 588                  * [1] on-kernel-exit  (on-idle-entry)
 589                  *
 590                  * When auto mode is supported we leave the bit set, otherwise
 591                  * we clear the bits.
 592                  */
 593                 tr->tr_pcb_spec_ctrl[0] |= spec_ctrl;
 594                 if (CHECK(IBRS_AUTO_SUPPORTED) == 0)
 595                         spec_ctrl &= ~SPEC_CTRL_IBRS;
 596                 if (CHECK(STIBP_AUTO_SUPPORTED) == 0)
 597                         spec_ctrl &= ~SPEC_CTRL_STIBP;
 598                 tr->tr_pcb_spec_ctrl[1] |= spec_ctrl;
 599
 600                 /*
 601                  * Make sure we set this on the first loop.  It will be
 602                  * the same value on remaining loops.
 603                  */
 604                 spectre_mode = mode;
 605         }
 606         lwkt_setcpu_self(save_gd);
 607         cpu_ccfence();
 608
 609         /*
 610          * Console message on mitigation mode change
 611          */
 612         kprintf("Spectre: support=(");
 613         if (spectre_support == 0) {
 614                 kprintf(" none");
 615         } else {
 616                 if (spectre_support & IBRS_SUPPORTED)
 617                         kprintf(" IBRS");
 618                 if (spectre_support & STIBP_SUPPORTED)
 619                         kprintf(" STIBP");
 620                 if (spectre_support & IBPB_SUPPORTED)
 621                         kprintf(" IBPB");
 622                 if (spectre_support & IBRS_AUTO_SUPPORTED)
 623                         kprintf(" IBRS_AUTO");
 624                 if (spectre_support & STIBP_AUTO_SUPPORTED)
 625                         kprintf(" STIBP_AUTO");
 626                 if (spectre_support & IBRS_PREFERRED_REQUEST)
 627                         kprintf(" IBRS_REQUESTED");
 628         }
 629         kprintf(" ) req=%04x operating=(", (uint16_t)spectre_mitigation);
 630         if (spectre_mode == 0) {
 631                 kprintf(" none");
 632         } else {
 633                 if (spectre_mode & IBRS_SUPPORTED)
 634                         kprintf(" IBRS");
 635                 if (spectre_mode & STIBP_SUPPORTED)
 636                         kprintf(" STIBP");
 637                 if (spectre_mode & IBPB_SUPPORTED)
 638                         kprintf(" IBPB");
 639                 if (spectre_mode & IBRS_AUTO_SUPPORTED)
 640                         kprintf(" IBRS_AUTO");
 641                 if (spectre_mode & STIBP_AUTO_SUPPORTED)
 642                         kprintf(" STIBP_AUTO");
 643                 if (spectre_mode & IBRS_PREFERRED_REQUEST)
 644                         kprintf(" IBRS_REQUESTED");
 645         }
 646         kprintf(" )\n");
 647 }
 648
 649 #undef CHECK
 650
 651 /*
 652  * User changes sysctl value
 653  */
 654 static int
 655 sysctl_spectre_mitigation(SYSCTL_HANDLER_ARGS)
 656 {
 657         char buf[128];
 658         char *ptr;
 659         char *iter;
 660         size_t len;
 661         int spectre;
 662         int error = 0;
 663         int loop = 0;
 664
 665         /*
 666          * Return current operating mode or support.
 667          */
 668         if (oidp->oid_kind & CTLFLAG_WR)
 669                 spectre = spectre_mode;
 670         else
 671                 spectre = spectre_support;
 672
 673         spectre &= (IBRS_SUPPORTED | IBRS_AUTO_SUPPORTED |
 674                     STIBP_SUPPORTED | STIBP_AUTO_SUPPORTED |
 675                     IBPB_SUPPORTED);
 676         while (spectre) {
 677                 if (error)
 678                         break;
 679                 if (loop++) {
 680                         error = SYSCTL_OUT(req, " ", 1);
 681                         if (error)
 682                                 break;
 683                 }
 684                 if (spectre & IBRS_SUPPORTED) {
 685                         spectre &= ~IBRS_SUPPORTED;
 686                         error = SYSCTL_OUT(req, "IBRS", 4);
 687                 } else
 688                 if (spectre & IBRS_AUTO_SUPPORTED) {
 689                         spectre &= ~IBRS_AUTO_SUPPORTED;
 690                         error = SYSCTL_OUT(req, "IBRS_AUTO", 9);
 691                 } else
 692                 if (spectre & STIBP_SUPPORTED) {
 693                         spectre &= ~STIBP_SUPPORTED;
 694                         error = SYSCTL_OUT(req, "STIBP", 5);
 695                 } else
 696                 if (spectre & STIBP_AUTO_SUPPORTED) {
 697                         spectre &= ~STIBP_AUTO_SUPPORTED;
 698                         error = SYSCTL_OUT(req, "STIBP_AUTO", 10);
 699                 } else
 700                 if (spectre & IBPB_SUPPORTED) {
 701                         spectre &= ~IBPB_SUPPORTED;
 702                         error = SYSCTL_OUT(req, "IBPB", 4);
 703                 }
 704         }
 705         if (loop == 0) {
 706                 error = SYSCTL_OUT(req, "NONE", 4);
 707         }
 708
 709         if (error || req->newptr == NULL)
 710                 return error;
 711         if ((oidp->oid_kind & CTLFLAG_WR) == 0)
 712                 return error;
 713
 714         /*
 715          * Change current operating mode
 716          */
 717         len = req->newlen - req->newidx;
 718         if (len >= sizeof(buf)) {
 719                 error = EINVAL;
 720                 len = 0;
 721         } else {
 722                 error = SYSCTL_IN(req, buf, len);
 723         }
 724         buf[len] = 0;
 725         iter = &buf[0];
 726         spectre = 0;
 727
 728         while (error == 0 && iter) {
 729                 ptr = strsep(&iter, " ,\t\r\n");
 730                 if (*ptr == 0)
 731                         continue;
 732                 if (strcasecmp(ptr, "NONE") == 0)
 733                         spectre |= 0;
 734                 else if (strcasecmp(ptr, "IBRS") == 0)
 735                         spectre |= IBRS_SUPPORTED;
 736                 else if (strcasecmp(ptr, "IBRS_AUTO") == 0)
 737                         spectre |= IBRS_AUTO_SUPPORTED;
 738                 else if (strcasecmp(ptr, "STIBP") == 0)
 739                         spectre |= STIBP_SUPPORTED;
 740                 else if (strcasecmp(ptr, "STIBP_AUTO") == 0)
 741                         spectre |= STIBP_AUTO_SUPPORTED;
 742                 else if (strcasecmp(ptr, "IBPB") == 0)
 743                         spectre |= IBPB_SUPPORTED;
 744                 else
 745                         error = ENOENT;
 746         }
 747         if (error == 0) {
 748                 spectre_mitigation = spectre;
 749                 spectre_sysctl_changed();
 750         }
 751         return error;
 752 }
 753
 754 SYSCTL_PROC(_machdep, OID_AUTO, spectre_mitigation,
 755         CTLTYPE_STRING | CTLFLAG_RW,
 756         0, 0, sysctl_spectre_mitigation, "A", "Spectre exploit mitigation");
 757 SYSCTL_PROC(_machdep, OID_AUTO, spectre_support,
 758         CTLTYPE_STRING | CTLFLAG_RD,
 759         0, 0, sysctl_spectre_mitigation, "A", "Spectre supported features");
 760
 761 /*
 762  * NOTE: Called at SI_BOOT2_MACHDEP and also when the microcode is
 763  *       updated.  Microcode updates must be applied to all cpus
 764  *       for support to be recognized.
 765  */
 766 static void
 767 spectre_vm_setup(void *arg)
 768 {
 769         int inconsistent = 0;
 770         int supmask;
 771
 772         /*
 773          * Fetch tunable in auto mode
 774          */
 775         if (spectre_mitigation < 0) {
 776                 TUNABLE_INT_FETCH("machdep.spectre_mitigation",
 777                                   &spectre_mitigation);
 778         }
 779
 780         if ((supmask = spectre_check_support()) != 0) {
 781                 /*
 782                  * Must be supported on all cpus before we
 783                  * can enable it.  Returns silently if it
 784                  * isn't.
 785                  *
 786                  * NOTE! arg != NULL indicates we were called
 787                  *       from cpuctl after a successful microcode
 788                  *       update.
 789                  */
 790                 if (arg != NULL) {
 791                         globaldata_t save_gd;
 792                         int n;
 793
 794                         save_gd = mycpu;
 795                         for (n = 0; n < ncpus; ++n) {
 796                                 lwkt_setcpu_self(globaldata_find(n));
 797                                 cpu_ccfence();
 798                                 if (spectre_check_support() !=
 799                                     supmask) {
 800                                         inconsistent = 1;
 801                                         break;
 802                                 }
 803                         }
 804                         lwkt_setcpu_self(save_gd);
 805                         cpu_ccfence();
 806                 }
 807         }
 808
 809         /*
 810          * Be silent while microcode is being loaded on various CPUs,
 811          * until all done.
 812          */
 813         if (inconsistent) {
 814                 spectre_mitigation = -1;
 815                 spectre_support = 0;
 816                 return;
 817         }
 818
 819         /*
 820          * IBRS support
 821          */
 822         spectre_support = supmask;
 823
 824         /*
 825          * Enable spectre_mitigation, set defaults if -1, adjust
 826          * tuned value according to support if not.
 827          *
 828          * NOTE!  We do not enable IBPB for user->kernel transitions
 829          *        by default, so this code is commented out for now.
 830          */
 831         if (spectre_support) {
 832                 if (spectre_mitigation < 0) {
 833                         spectre_mitigation = 0;
 834
 835                         /*
 836                          * IBRS toggling not currently recommended as a
 837                          * default.
 838                          */
 839                         if (spectre_support & IBRS_AUTO_SUPPORTED)
 840                                 spectre_mitigation |= IBRS_AUTO_SUPPORTED;
 841                         else if (spectre_support & IBRS_SUPPORTED)
 842                                 spectre_mitigation |= 0;
 843
 844                         /*
 845                          * STIBP toggling not currently recommended as a
 846                          * default.
 847                          */
 848                         if (spectre_support & STIBP_AUTO_SUPPORTED)
 849                                 spectre_mitigation |= STIBP_AUTO_SUPPORTED;
 850                         else if (spectre_support & STIBP_SUPPORTED)
 851                                 spectre_mitigation |= 0;
 852
 853                         /*
 854                          * IBPB adds enormous (~2uS) overhead to system
 855                          * calls etc, we do not enable it by default.
 856                          */
 857                         if (spectre_support & IBPB_SUPPORTED)
 858                                 spectre_mitigation |= 0;
 859                 }
 860         } else {
 861                 spectre_mitigation = -1;
 862         }
 863
 864         /*
 865          * Disallow sysctl changes when there is no support (otherwise
 866          * the wrmsr will cause a protection fault).
 867          */
 868         if (spectre_mitigation < 0)
 869                 sysctl___machdep_spectre_mitigation.oid_kind &= ~CTLFLAG_WR;
 870         else
 871                 sysctl___machdep_spectre_mitigation.oid_kind |= CTLFLAG_WR;
 872
 873         spectre_sysctl_changed();
 874 }
 875
 876 #define MDS_AVX512_4VNNIW_SUPPORTED     0x0001
 877 #define MDS_AVX512_4FMAPS_SUPPORTED     0x0002
 878 #define MDS_MD_CLEAR_SUPPORTED          0x0004
 879 #define MDS_TSX_FORCE_ABORT_SUPPORTED   0x0008
 880 #define MDS_NOT_REQUIRED                0x8000
 881
 882 static
 883 int
 884 mds_check_support(void)
 885 {
 886         uint64_t msr;
 887         uint32_t p[4];
 888         int rv = 0;
 889
 890         /*
 891          * MDS mitigation hw bits
 892          *
 893          * MD_CLEAR     Use microcode-supported verf insn.  This is the
 894          *              only mode we really support.
 895          */
 896         if (cpu_vendor_id == CPU_VENDOR_INTEL) {
 897                 p[0] = 0;
 898                 p[1] = 0;
 899                 p[2] = 0;
 900                 p[3] = 0;
 901                 cpuid_count(7, 0, p);
 902
 903                 /*
 904                  * Some hypervisors fail to implement
 905                  * MSR_IA32_ARCH_CAPABILITIES.
 906                  */
 907                 if (p[3] & CPUID_STDEXT3_ARCH_CAP) {
 908                         msr = 0;
 909                         if (rdmsr_safe(MSR_IA32_ARCH_CAPABILITIES, &msr)) {
 910                                 kprintf("Warning: MSR_IA32_ARCH_CAPABILITIES "
 911                                         "cannot be accessed\n");
 912                         }
 913                         if (msr & IA32_ARCH_CAP_MDS_NO)
 914                                 rv = MDS_NOT_REQUIRED;
 915                 }
 916                 if (p[3] & CPUID_STDEXT3_AVX5124VNNIW)
 917                         rv |= MDS_AVX512_4VNNIW_SUPPORTED;
 918                 if (p[3] & CPUID_STDEXT3_AVX5124FMAPS)
 919                         rv |= MDS_AVX512_4FMAPS_SUPPORTED;
 920                 if (p[3] & CPUID_STDEXT3_MD_CLEAR)
 921                         rv |= MDS_MD_CLEAR_SUPPORTED;
 922                 if (p[3] & CPUID_STDEXT3_TSXFA)
 923                         rv |= MDS_TSX_FORCE_ABORT_SUPPORTED;
 924         } else {
 925                 rv = MDS_NOT_REQUIRED;
 926         }
 927
 928         return rv;
 929 }
 930
 931 /*
 932  * Iterate CPUs and adjust MSR for global operations, since
 933  * the KMMU* code won't do it if spectre_mitigation is 0 or 2.
 934  */
 935 #define CHECK(flag)     (mds_mitigation & mds_support & (flag))
 936
 937 static
 938 void
 939 mds_sysctl_changed(void)
 940 {
 941         globaldata_t save_gd;
 942         struct trampframe *tr;
 943         int spec_ctrl;
 944         int spec_mask;
 945         int mode;
 946         int n;
 947
 948         spec_mask = SPEC_CTRL_MDS_ENABLE;
 949
 950         /*
 951          * Fixup state
 952          */
 953         mode = 0;
 954         save_gd = mycpu;
 955         for (n = 0; n < ncpus; ++n) {
 956                 lwkt_setcpu_self(globaldata_find(n));
 957                 cpu_ccfence();
 958                 tr = &pscpu->trampoline;
 959
 960                 /*
 961                  * Make sure we are cleaned out.
 962                  *
 963                  * XXX cleanup, reusing globals inside the loop (they get
 964                  * set to the same thing each loop)
 965                  *
 966                  * [0] kernel entry (idle exit)
 967                  * [1] kernel exit  (idle entry)
 968                  */
 969                 tr->tr_pcb_spec_ctrl[0] &= ~spec_mask;
 970                 tr->tr_pcb_spec_ctrl[1] &= ~spec_mask;
 971
 972                 /*
 973                  * Don't try to parse if not available
 974                  */
 975                 if (mds_mitigation < 0)
 976                         continue;
 977
 978                 spec_ctrl = 0;
 979                 if (CHECK(MDS_MD_CLEAR_SUPPORTED)) {
 980                         spec_ctrl |= SPEC_CTRL_MDS_ENABLE;
 981                         mode |= MDS_MD_CLEAR_SUPPORTED;
 982                 }
 983
 984                 /*
 985                  * Update spec_ctrl fields in the trampoline.
 986                  *
 987                  * [0] on-kernel-entry (on-idle-exit)
 988                  * [1] on-kernel-exit  (on-idle-entry)
 989                  *
 990                  * The MDS stuff is only needed on kernel-exit or idle-entry
 991                  */
 992                 /* tr->tr_pcb_spec_ctrl[0] |= spec_ctrl; */
 993                 tr->tr_pcb_spec_ctrl[1] |= spec_ctrl;
 994
 995                 /*
 996                  * Make sure we set this on the first loop.  It will be
 997                  * the same value on remaining loops.
 998                  */
 999                 mds_mode = mode;
1000         }
1001         lwkt_setcpu_self(save_gd);
1002         cpu_ccfence();
1003
1004         /*
1005          * Console message on mitigation mode change
1006          */
1007         kprintf("MDS: support=(");
1008         if (mds_support == 0) {
1009                 kprintf(" none");
1010         } else {
1011                 if (mds_support & MDS_AVX512_4VNNIW_SUPPORTED)
1012                         kprintf(" AVX512_4VNNIW");
1013                 if (mds_support & MDS_AVX512_4FMAPS_SUPPORTED)
1014                         kprintf(" AVX512_4FMAPS");
1015                 if (mds_support & MDS_MD_CLEAR_SUPPORTED)
1016                         kprintf(" MD_CLEAR");
1017                 if (mds_support & MDS_TSX_FORCE_ABORT_SUPPORTED)
1018                         kprintf(" TSX_FORCE_ABORT");
1019                 if (mds_support & MDS_NOT_REQUIRED)
1020                         kprintf(" MDS_NOT_REQUIRED");
1021         }
1022         kprintf(" ) req=%04x operating=(", (uint16_t)mds_mitigation);
1023         if (mds_mode == 0) {
1024                 kprintf(" none");
1025         } else {
1026                 if (mds_mode & MDS_AVX512_4VNNIW_SUPPORTED)
1027                         kprintf(" AVX512_4VNNIW");
1028                 if (mds_mode & MDS_AVX512_4FMAPS_SUPPORTED)
1029                         kprintf(" AVX512_4FMAPS");
1030                 if (mds_mode & MDS_MD_CLEAR_SUPPORTED)
1031                         kprintf(" MD_CLEAR");
1032                 if (mds_mode & MDS_TSX_FORCE_ABORT_SUPPORTED)
1033                         kprintf(" TSX_FORCE_ABORT");
1034                 if (mds_mode & MDS_NOT_REQUIRED)
1035                         kprintf(" MDS_NOT_REQUIRED");
1036         }
1037         kprintf(" )\n");
1038 }
1039
1040 #undef CHECK
1041
1042 /*
1043  * User changes sysctl value
1044  */
1045 static int
1046 sysctl_mds_mitigation(SYSCTL_HANDLER_ARGS)
1047 {
1048         char buf[128];
1049         char *ptr;
1050         char *iter;
1051         size_t len;
1052         int mds;
1053         int error = 0;
1054         int loop = 0;
1055
1056         /*
1057          * Return current operating mode or support.
1058          */
1059         if (oidp->oid_kind & CTLFLAG_WR)
1060                 mds = mds_mode;
1061         else
1062                 mds = mds_support;
1063
1064         mds &= MDS_AVX512_4VNNIW_SUPPORTED |
1065                MDS_AVX512_4FMAPS_SUPPORTED |
1066                MDS_MD_CLEAR_SUPPORTED |
1067                MDS_TSX_FORCE_ABORT_SUPPORTED |
1068                MDS_NOT_REQUIRED;
1069
1070         while (mds) {
1071                 if (error)
1072                         break;
1073                 if (loop++) {
1074                         error = SYSCTL_OUT(req, " ", 1);
1075                         if (error)
1076                                 break;
1077                 }
1078                 if (mds & MDS_AVX512_4VNNIW_SUPPORTED) {
1079                         mds &= ~MDS_AVX512_4VNNIW_SUPPORTED;
1080                         error = SYSCTL_OUT(req, "AVX512_4VNNIW", 13);
1081                 } else
1082                 if (mds & MDS_AVX512_4FMAPS_SUPPORTED) {
1083                         mds &= ~MDS_AVX512_4FMAPS_SUPPORTED;
1084                         error = SYSCTL_OUT(req, "AVX512_4FMAPS", 13);
1085                 } else
1086                 if (mds & MDS_MD_CLEAR_SUPPORTED) {
1087                         mds &= ~MDS_MD_CLEAR_SUPPORTED;
1088                         error = SYSCTL_OUT(req, "MD_CLEAR", 8);
1089                 } else
1090                 if (mds & MDS_TSX_FORCE_ABORT_SUPPORTED) {
1091                         mds &= ~MDS_TSX_FORCE_ABORT_SUPPORTED;
1092                         error = SYSCTL_OUT(req, "TSX_FORCE_ABORT", 15);
1093                 } else
1094                 if (mds & MDS_NOT_REQUIRED) {
1095                         mds &= ~MDS_NOT_REQUIRED;
1096                         error = SYSCTL_OUT(req, "MDS_NOT_REQUIRED", 16);
1097                 }
1098         }
1099         if (loop == 0) {
1100                 error = SYSCTL_OUT(req, "NONE", 4);
1101         }
1102
1103         if (error || req->newptr == NULL)
1104                 return error;
1105         if ((oidp->oid_kind & CTLFLAG_WR) == 0)
1106                 return error;
1107
1108         /*
1109          * Change current operating mode
1110          */
1111         len = req->newlen - req->newidx;
1112         if (len >= sizeof(buf)) {
1113                 error = EINVAL;
1114                 len = 0;
1115         } else {
1116                 error = SYSCTL_IN(req, buf, len);
1117         }
1118         buf[len] = 0;
1119         iter = &buf[0];
1120         mds = 0;
1121
1122         while (error == 0 && iter) {
1123                 ptr = strsep(&iter, " ,\t\r\n");
1124                 if (*ptr == 0)
1125                         continue;
1126                 if (strcasecmp(ptr, "NONE") == 0)
1127                         mds |= 0;
1128                 else if (strcasecmp(ptr, "AVX512_4VNNIW") == 0)
1129                         mds |= MDS_AVX512_4VNNIW_SUPPORTED;
1130                 else if (strcasecmp(ptr, "AVX512_4FMAPS") == 0)
1131                         mds |= MDS_AVX512_4FMAPS_SUPPORTED;
1132                 else if (strcasecmp(ptr, "MD_CLEAR") == 0)
1133                         mds |= MDS_MD_CLEAR_SUPPORTED;
1134                 else if (strcasecmp(ptr, "TSX_FORCE_ABORT") == 0)
1135                         mds |= MDS_TSX_FORCE_ABORT_SUPPORTED;
1136                 else if (strcasecmp(ptr, "MDS_NOT_REQUIRED") == 0)
1137                         mds |= MDS_NOT_REQUIRED;
1138                 else
1139                         error = ENOENT;
1140         }
1141         if (error == 0) {
1142                 mds_mitigation = mds;
1143                 mds_sysctl_changed();
1144         }
1145         return error;
1146 }
1147
1148 SYSCTL_PROC(_machdep, OID_AUTO, mds_mitigation,
1149         CTLTYPE_STRING | CTLFLAG_RW,
1150         0, 0, sysctl_mds_mitigation, "A", "MDS exploit mitigation");
1151 SYSCTL_PROC(_machdep, OID_AUTO, mds_support,
1152         CTLTYPE_STRING | CTLFLAG_RD,
1153         0, 0, sysctl_mds_mitigation, "A", "MDS supported features");
1154
1155 /*
1156  * NOTE: Called at SI_BOOT2_MACHDEP and also when the microcode is
1157  *       updated.  Microcode updates must be applied to all cpus
1158  *       for support to be recognized.
1159  */
1160 static void
1161 mds_vm_setup(void *arg)
1162 {
1163         int inconsistent = 0;
1164         int supmask;
1165
1166         /*
1167          * Fetch tunable in auto mode
1168          */
1169         if (mds_mitigation < 0) {
1170                 TUNABLE_INT_FETCH("machdep.mds_mitigation", &mds_mitigation);
1171         }
1172
1173         if ((supmask = mds_check_support()) != 0) {
1174                 /*
1175                  * Must be supported on all cpus before we
1176                  * can enable it.  Returns silently if it
1177                  * isn't.
1178                  *
1179                  * NOTE! arg != NULL indicates we were called
1180                  *       from cpuctl after a successful microcode
1181                  *       update.
1182                  */
1183                 if (arg != NULL) {
1184                         globaldata_t save_gd;
1185                         int n;
1186
1187                         save_gd = mycpu;
1188                         for (n = 0; n < ncpus; ++n) {
1189                                 lwkt_setcpu_self(globaldata_find(n));
1190                                 cpu_ccfence();
1191                                 if (mds_check_support() != supmask) {
1192                                         inconsistent = 1;
1193                                         break;
1194                                 }
1195                         }
1196                         lwkt_setcpu_self(save_gd);
1197                         cpu_ccfence();
1198                 }
1199         }
1200
1201         /*
1202          * Be silent while microcode is being loaded on various CPUs,
1203          * until all done.
1204          */
1205         if (inconsistent) {
1206                 mds_mitigation = -1;
1207                 mds_support = 0;
1208                 return;
1209         }
1210
1211         /*
1212          * IBRS support
1213          */
1214         mds_support = supmask;
1215
1216         /*
1217          * Enable mds_mitigation, set defaults if -1, adjust
1218          * tuned value according to support if not.
1219          *
1220          * NOTE!  MDS is not enabled by default.
1221          */
1222         if (mds_support) {
1223                 if (mds_mitigation < 0) {
1224                         mds_mitigation = 0;
1225
1226                         if ((mds_support & MDS_NOT_REQUIRED) == 0 &&
1227                             (mds_support & MDS_MD_CLEAR_SUPPORTED)) {
1228                                 /* mds_mitigation |= MDS_MD_CLEAR_SUPPORTED; */
1229                         }
1230                 }
1231         } else {
1232                 mds_mitigation = -1;
1233         }
1234
1235         /*
1236          * Disallow sysctl changes when there is no support (otherwise
1237          * the wrmsr will cause a protection fault).
1238          */
1239         if (mds_mitigation < 0)
1240                 sysctl___machdep_mds_mitigation.oid_kind &= ~CTLFLAG_WR;
1241         else
1242                 sysctl___machdep_mds_mitigation.oid_kind |= CTLFLAG_WR;
1243
1244         mds_sysctl_changed();
1245 }
1246
1247 /*
1248  * NOTE: Called at SI_BOOT2_MACHDEP and also when the microcode is
1249  *       updated.  Microcode updates must be applied to all cpus
1250  *       for support to be recognized.
1251  */
1252 void
1253 mitigation_vm_setup(void *arg)
1254 {
1255         spectre_vm_setup(arg);
1256         mds_vm_setup(arg);
1257 }
1258
1259 SYSINIT(mitigation_vm_setup, SI_BOOT2_MACHDEP, SI_ORDER_ANY,
1260         mitigation_vm_setup, NULL);
1261
1262 /*
1263  * platform-specific vmspace initialization (nothing for x86_64)
1264  */
1265 void
1266 cpu_vmspace_alloc(struct vmspace *vm __unused)
1267 {
1268 }
1269
1270 void
1271 cpu_vmspace_free(struct vmspace *vm __unused)
1272 {
1273 }
1274
1275 int
1276 kvm_access_check(vm_offset_t saddr, vm_offset_t eaddr, int prot)
1277 {
1278         vm_offset_t addr;
1279
1280         if (saddr < KvaStart)
1281                 return EFAULT;
1282         if (eaddr >= KvaEnd)
1283                 return EFAULT;
1284         for (addr = saddr; addr < eaddr; addr += PAGE_SIZE)  {
1285                 if (pmap_kextract(addr) == 0)
1286                         return EFAULT;
1287         }
1288         if (!kernacc((caddr_t)saddr, eaddr - saddr, prot))
1289                 return EFAULT;
1290         return 0;
1291 }
1292
1293 #if 0
1294
1295 void _test_frame_enter(struct trapframe *frame);
1296 void _test_frame_exit(struct trapframe *frame);
1297
1298 void
1299 _test_frame_enter(struct trapframe *frame)
1300 {
1301         thread_t td = curthread;
1302
1303         if (ISPL(frame->tf_cs) == SEL_UPL) {
1304                 KKASSERT(td->td_lwp);
1305                 KASSERT(td->td_lwp->lwp_md.md_regs == frame,
1306                         ("_test_frame_exit: Frame mismatch %p %p",
1307                         td->td_lwp->lwp_md.md_regs, frame));
1308             td->td_lwp->lwp_saveusp = (void *)frame->tf_rsp;
1309             td->td_lwp->lwp_saveupc = (void *)frame->tf_rip;
1310         }
1311         if ((char *)frame < td->td_kstack ||
1312             (char *)frame > td->td_kstack + td->td_kstack_size) {
1313                 panic("_test_frame_exit: frame not on kstack %p kstack=%p",
1314                         frame, td->td_kstack);
1315         }
1316 }
1317
1318 void
1319 _test_frame_exit(struct trapframe *frame)
1320 {
1321         thread_t td = curthread;
1322
1323         if (ISPL(frame->tf_cs) == SEL_UPL) {
1324                 KKASSERT(td->td_lwp);
1325                 KASSERT(td->td_lwp->lwp_md.md_regs == frame,
1326                         ("_test_frame_exit: Frame mismatch %p %p",
1327                         td->td_lwp->lwp_md.md_regs, frame));
1328                 if (td->td_lwp->lwp_saveusp != (void *)frame->tf_rsp) {
1329                         kprintf("_test_frame_exit: %s:%d usp mismatch %p/%p\n",
1330                                 td->td_comm, td->td_proc->p_pid,
1331                                 td->td_lwp->lwp_saveusp,
1332                                 (void *)frame->tf_rsp);
1333                 }
1334                 if (td->td_lwp->lwp_saveupc != (void *)frame->tf_rip) {
1335                         kprintf("_test_frame_exit: %s:%d upc mismatch %p/%p\n",
1336                                 td->td_comm, td->td_proc->p_pid,
1337                                 td->td_lwp->lwp_saveupc,
1338                                 (void *)frame->tf_rip);
1339                 }
1340
1341                 /*
1342                  * adulterate the fields to catch entries that
1343                  * don't run through test_frame_enter
1344                  */
1345                 td->td_lwp->lwp_saveusp =
1346                         (void *)~(intptr_t)td->td_lwp->lwp_saveusp;
1347                 td->td_lwp->lwp_saveupc =
1348                         (void *)~(intptr_t)td->td_lwp->lwp_saveupc;
1349         }
1350         if ((char *)frame < td->td_kstack ||
1351             (char *)frame > td->td_kstack + td->td_kstack_size) {
1352                 panic("_test_frame_exit: frame not on kstack %p kstack=%p",
1353                         frame, td->td_kstack);
1354         }
1355 }
1356
1357 #endif