| 1 | /* |
| 2 | * Copyright (c) 2006 The DragonFly Project. All rights reserved. |
| 3 | * Copyright (c) 1990 William Jolitz. |
| 4 | * Copyright (c) 1991 The Regents of the University of California. |
| 5 | * All rights reserved. |
| 6 | * |
| 7 | * This code is derived from software contributed to The DragonFly Project |
| 8 | * by Matthew Dillon <dillon@backplane.com> |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or without |
| 11 | * modification, are permitted provided that the following conditions |
| 12 | * are met: |
| 13 | * |
| 14 | * 1. Redistributions of source code must retain the above copyright |
| 15 | * notice, this list of conditions and the following disclaimer. |
| 16 | * 2. Redistributions in binary form must reproduce the above copyright |
| 17 | * notice, this list of conditions and the following disclaimer in |
| 18 | * the documentation and/or other materials provided with the |
| 19 | * distribution. |
| 20 | * 3. Neither the name of The DragonFly Project nor the names of its |
| 21 | * contributors may be used to endorse or promote products derived |
| 22 | * from this software without specific, prior written permission. |
| 23 | * |
| 24 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 25 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 26 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS |
| 27 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE |
| 28 | * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, |
| 29 | * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, |
| 30 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| 31 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
| 32 | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
| 33 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT |
| 34 | * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
| 35 | * SUCH DAMAGE. |
| 36 | * |
| 37 | * from: @(#)npx.c 7.2 (Berkeley) 5/12/91 |
| 38 | * $FreeBSD: src/sys/i386/isa/npx.c,v 1.80.2.3 2001/10/20 19:04:38 tegge Exp $ |
| 39 | */ |
| 40 | |
| 41 | #include "opt_debug_npx.h" |
| 42 | |
| 43 | #include <sys/param.h> |
| 44 | #include <sys/systm.h> |
| 45 | #include <sys/bus.h> |
| 46 | #include <sys/kernel.h> |
| 47 | #include <sys/malloc.h> |
| 48 | #include <sys/module.h> |
| 49 | #include <sys/sysctl.h> |
| 50 | #include <sys/proc.h> |
| 51 | #include <sys/rman.h> |
| 52 | #ifdef NPX_DEBUG |
| 53 | #include <sys/syslog.h> |
| 54 | #endif |
| 55 | #include <sys/signalvar.h> |
| 56 | |
| 57 | #include <sys/thread2.h> |
| 58 | #include <sys/mplock2.h> |
| 59 | |
| 60 | #include <machine/cputypes.h> |
| 61 | #include <machine/frame.h> |
| 62 | #include <machine/md_var.h> |
| 63 | #include <machine/pcb.h> |
| 64 | #include <machine/psl.h> |
| 65 | #include <machine/specialreg.h> |
| 66 | #include <machine/segments.h> |
| 67 | #include <machine/globaldata.h> |
| 68 | |
| 69 | #define fldcw(addr) __asm("fldcw %0" : : "m" (*(addr))) |
| 70 | #define fnclex() __asm("fnclex") |
| 71 | #define fninit() __asm("fninit") |
| 72 | #define fnop() __asm("fnop") |
| 73 | #define fnsave(addr) __asm __volatile("fnsave %0" : "=m" (*(addr))) |
| 74 | #define fnstcw(addr) __asm __volatile("fnstcw %0" : "=m" (*(addr))) |
| 75 | #define fnstsw(addr) __asm __volatile("fnstsw %0" : "=m" (*(addr))) |
| 76 | #define frstor(addr) __asm("frstor %0" : : "m" (*(addr))) |
| 77 | #ifndef CPU_DISABLE_SSE |
| 78 | #define fxrstor(addr) __asm("fxrstor %0" : : "m" (*(addr))) |
| 79 | #define fxsave(addr) __asm __volatile("fxsave %0" : "=m" (*(addr))) |
| 80 | #endif |
| 81 | |
| 82 | typedef u_char bool_t; |
| 83 | #ifndef CPU_DISABLE_SSE |
| 84 | static void fpu_clean_state(void); |
| 85 | #endif |
| 86 | |
| 87 | int cpu_fxsr = 0; |
| 88 | |
| 89 | static struct krate badfprate = { 1 }; |
| 90 | |
| 91 | /*static int npx_attach (device_t dev);*/ |
| 92 | static void fpusave (union savefpu *); |
| 93 | static void fpurstor (union savefpu *); |
| 94 | |
| 95 | #if (defined(I586_CPU) || defined(I686_CPU)) && !defined(CPU_DISABLE_SSE) |
| 96 | int mmxopt = 1; |
| 97 | SYSCTL_INT(_kern, OID_AUTO, mmxopt, CTLFLAG_RD, &mmxopt, 0, |
| 98 | "MMX/XMM optimized bcopy/copyin/copyout support"); |
| 99 | #endif |
| 100 | |
| 101 | static int hw_instruction_sse; |
| 102 | SYSCTL_INT(_hw, OID_AUTO, instruction_sse, CTLFLAG_RD, |
| 103 | &hw_instruction_sse, 0, "SIMD/MMX2 instructions available in CPU"); |
| 104 | |
| 105 | #if 0 |
| 106 | /* |
| 107 | * Attach routine - announce which it is, and wire into system |
| 108 | */ |
| 109 | int |
| 110 | npx_attach(device_t dev) |
| 111 | { |
| 112 | npxinit(__INITIAL_NPXCW__); |
| 113 | return (0); |
| 114 | } |
| 115 | #endif |
| 116 | |
| 117 | void |
| 118 | init_fpu(int supports_sse) |
| 119 | { |
| 120 | cpu_fxsr = hw_instruction_sse = supports_sse; |
| 121 | } |
| 122 | |
| 123 | /* |
| 124 | * Initialize the floating point unit. |
| 125 | */ |
| 126 | void |
| 127 | npxinit(u_short control) |
| 128 | { |
| 129 | static union savefpu dummy __aligned(16); |
| 130 | |
| 131 | /* |
| 132 | * fninit has the same h/w bugs as fnsave. Use the detoxified |
| 133 | * fnsave to throw away any junk in the fpu. npxsave() initializes |
| 134 | * the fpu and sets npxthread = NULL as important side effects. |
| 135 | */ |
| 136 | npxsave(&dummy); |
| 137 | crit_enter(); |
| 138 | /*stop_emulating();*/ |
| 139 | fldcw(&control); |
| 140 | fpusave(curthread->td_savefpu); |
| 141 | mdcpu->gd_npxthread = NULL; |
| 142 | /*start_emulating();*/ |
| 143 | crit_exit(); |
| 144 | } |
| 145 | |
| 146 | /* |
| 147 | * Free coprocessor (if we have it). |
| 148 | */ |
| 149 | void |
| 150 | npxexit(void) |
| 151 | { |
| 152 | if (curthread == mdcpu->gd_npxthread) |
| 153 | npxsave(curthread->td_savefpu); |
| 154 | } |
| 155 | |
| 156 | #if 0 |
| 157 | /* |
| 158 | * The following mechanism is used to ensure that the FPE_... value |
| 159 | * that is passed as a trapcode to the signal handler of the user |
| 160 | * process does not have more than one bit set. |
| 161 | * |
| 162 | * Multiple bits may be set if the user process modifies the control |
| 163 | * word while a status word bit is already set. While this is a sign |
| 164 | * of bad coding, we have no choise than to narrow them down to one |
| 165 | * bit, since we must not send a trapcode that is not exactly one of |
| 166 | * the FPE_ macros. |
| 167 | * |
| 168 | * The mechanism has a static table with 127 entries. Each combination |
| 169 | * of the 7 FPU status word exception bits directly translates to a |
| 170 | * position in this table, where a single FPE_... value is stored. |
| 171 | * This FPE_... value stored there is considered the "most important" |
| 172 | * of the exception bits and will be sent as the signal code. The |
| 173 | * precedence of the bits is based upon Intel Document "Numerical |
| 174 | * Applications", Chapter "Special Computational Situations". |
| 175 | * |
| 176 | * The macro to choose one of these values does these steps: 1) Throw |
| 177 | * away status word bits that cannot be masked. 2) Throw away the bits |
| 178 | * currently masked in the control word, assuming the user isn't |
| 179 | * interested in them anymore. 3) Reinsert status word bit 7 (stack |
| 180 | * fault) if it is set, which cannot be masked but must be presered. |
| 181 | * 4) Use the remaining bits to point into the trapcode table. |
| 182 | * |
| 183 | * The 6 maskable bits in order of their preference, as stated in the |
| 184 | * above referenced Intel manual: |
| 185 | * 1 Invalid operation (FP_X_INV) |
| 186 | * 1a Stack underflow |
| 187 | * 1b Stack overflow |
| 188 | * 1c Operand of unsupported format |
| 189 | * 1d SNaN operand. |
| 190 | * 2 QNaN operand (not an exception, irrelavant here) |
| 191 | * 3 Any other invalid-operation not mentioned above or zero divide |
| 192 | * (FP_X_INV, FP_X_DZ) |
| 193 | * 4 Denormal operand (FP_X_DNML) |
| 194 | * 5 Numeric over/underflow (FP_X_OFL, FP_X_UFL) |
| 195 | * 6 Inexact result (FP_X_IMP) |
| 196 | */ |
| 197 | static char fpetable[128] = { |
| 198 | 0, |
| 199 | FPE_FLTINV, /* 1 - INV */ |
| 200 | FPE_FLTUND, /* 2 - DNML */ |
| 201 | FPE_FLTINV, /* 3 - INV | DNML */ |
| 202 | FPE_FLTDIV, /* 4 - DZ */ |
| 203 | FPE_FLTINV, /* 5 - INV | DZ */ |
| 204 | FPE_FLTDIV, /* 6 - DNML | DZ */ |
| 205 | FPE_FLTINV, /* 7 - INV | DNML | DZ */ |
| 206 | FPE_FLTOVF, /* 8 - OFL */ |
| 207 | FPE_FLTINV, /* 9 - INV | OFL */ |
| 208 | FPE_FLTUND, /* A - DNML | OFL */ |
| 209 | FPE_FLTINV, /* B - INV | DNML | OFL */ |
| 210 | FPE_FLTDIV, /* C - DZ | OFL */ |
| 211 | FPE_FLTINV, /* D - INV | DZ | OFL */ |
| 212 | FPE_FLTDIV, /* E - DNML | DZ | OFL */ |
| 213 | FPE_FLTINV, /* F - INV | DNML | DZ | OFL */ |
| 214 | FPE_FLTUND, /* 10 - UFL */ |
| 215 | FPE_FLTINV, /* 11 - INV | UFL */ |
| 216 | FPE_FLTUND, /* 12 - DNML | UFL */ |
| 217 | FPE_FLTINV, /* 13 - INV | DNML | UFL */ |
| 218 | FPE_FLTDIV, /* 14 - DZ | UFL */ |
| 219 | FPE_FLTINV, /* 15 - INV | DZ | UFL */ |
| 220 | FPE_FLTDIV, /* 16 - DNML | DZ | UFL */ |
| 221 | FPE_FLTINV, /* 17 - INV | DNML | DZ | UFL */ |
| 222 | FPE_FLTOVF, /* 18 - OFL | UFL */ |
| 223 | FPE_FLTINV, /* 19 - INV | OFL | UFL */ |
| 224 | FPE_FLTUND, /* 1A - DNML | OFL | UFL */ |
| 225 | FPE_FLTINV, /* 1B - INV | DNML | OFL | UFL */ |
| 226 | FPE_FLTDIV, /* 1C - DZ | OFL | UFL */ |
| 227 | FPE_FLTINV, /* 1D - INV | DZ | OFL | UFL */ |
| 228 | FPE_FLTDIV, /* 1E - DNML | DZ | OFL | UFL */ |
| 229 | FPE_FLTINV, /* 1F - INV | DNML | DZ | OFL | UFL */ |
| 230 | FPE_FLTRES, /* 20 - IMP */ |
| 231 | FPE_FLTINV, /* 21 - INV | IMP */ |
| 232 | FPE_FLTUND, /* 22 - DNML | IMP */ |
| 233 | FPE_FLTINV, /* 23 - INV | DNML | IMP */ |
| 234 | FPE_FLTDIV, /* 24 - DZ | IMP */ |
| 235 | FPE_FLTINV, /* 25 - INV | DZ | IMP */ |
| 236 | FPE_FLTDIV, /* 26 - DNML | DZ | IMP */ |
| 237 | FPE_FLTINV, /* 27 - INV | DNML | DZ | IMP */ |
| 238 | FPE_FLTOVF, /* 28 - OFL | IMP */ |
| 239 | FPE_FLTINV, /* 29 - INV | OFL | IMP */ |
| 240 | FPE_FLTUND, /* 2A - DNML | OFL | IMP */ |
| 241 | FPE_FLTINV, /* 2B - INV | DNML | OFL | IMP */ |
| 242 | FPE_FLTDIV, /* 2C - DZ | OFL | IMP */ |
| 243 | FPE_FLTINV, /* 2D - INV | DZ | OFL | IMP */ |
| 244 | FPE_FLTDIV, /* 2E - DNML | DZ | OFL | IMP */ |
| 245 | FPE_FLTINV, /* 2F - INV | DNML | DZ | OFL | IMP */ |
| 246 | FPE_FLTUND, /* 30 - UFL | IMP */ |
| 247 | FPE_FLTINV, /* 31 - INV | UFL | IMP */ |
| 248 | FPE_FLTUND, /* 32 - DNML | UFL | IMP */ |
| 249 | FPE_FLTINV, /* 33 - INV | DNML | UFL | IMP */ |
| 250 | FPE_FLTDIV, /* 34 - DZ | UFL | IMP */ |
| 251 | FPE_FLTINV, /* 35 - INV | DZ | UFL | IMP */ |
| 252 | FPE_FLTDIV, /* 36 - DNML | DZ | UFL | IMP */ |
| 253 | FPE_FLTINV, /* 37 - INV | DNML | DZ | UFL | IMP */ |
| 254 | FPE_FLTOVF, /* 38 - OFL | UFL | IMP */ |
| 255 | FPE_FLTINV, /* 39 - INV | OFL | UFL | IMP */ |
| 256 | FPE_FLTUND, /* 3A - DNML | OFL | UFL | IMP */ |
| 257 | FPE_FLTINV, /* 3B - INV | DNML | OFL | UFL | IMP */ |
| 258 | FPE_FLTDIV, /* 3C - DZ | OFL | UFL | IMP */ |
| 259 | FPE_FLTINV, /* 3D - INV | DZ | OFL | UFL | IMP */ |
| 260 | FPE_FLTDIV, /* 3E - DNML | DZ | OFL | UFL | IMP */ |
| 261 | FPE_FLTINV, /* 3F - INV | DNML | DZ | OFL | UFL | IMP */ |
| 262 | FPE_FLTSUB, /* 40 - STK */ |
| 263 | FPE_FLTSUB, /* 41 - INV | STK */ |
| 264 | FPE_FLTUND, /* 42 - DNML | STK */ |
| 265 | FPE_FLTSUB, /* 43 - INV | DNML | STK */ |
| 266 | FPE_FLTDIV, /* 44 - DZ | STK */ |
| 267 | FPE_FLTSUB, /* 45 - INV | DZ | STK */ |
| 268 | FPE_FLTDIV, /* 46 - DNML | DZ | STK */ |
| 269 | FPE_FLTSUB, /* 47 - INV | DNML | DZ | STK */ |
| 270 | FPE_FLTOVF, /* 48 - OFL | STK */ |
| 271 | FPE_FLTSUB, /* 49 - INV | OFL | STK */ |
| 272 | FPE_FLTUND, /* 4A - DNML | OFL | STK */ |
| 273 | FPE_FLTSUB, /* 4B - INV | DNML | OFL | STK */ |
| 274 | FPE_FLTDIV, /* 4C - DZ | OFL | STK */ |
| 275 | FPE_FLTSUB, /* 4D - INV | DZ | OFL | STK */ |
| 276 | FPE_FLTDIV, /* 4E - DNML | DZ | OFL | STK */ |
| 277 | FPE_FLTSUB, /* 4F - INV | DNML | DZ | OFL | STK */ |
| 278 | FPE_FLTUND, /* 50 - UFL | STK */ |
| 279 | FPE_FLTSUB, /* 51 - INV | UFL | STK */ |
| 280 | FPE_FLTUND, /* 52 - DNML | UFL | STK */ |
| 281 | FPE_FLTSUB, /* 53 - INV | DNML | UFL | STK */ |
| 282 | FPE_FLTDIV, /* 54 - DZ | UFL | STK */ |
| 283 | FPE_FLTSUB, /* 55 - INV | DZ | UFL | STK */ |
| 284 | FPE_FLTDIV, /* 56 - DNML | DZ | UFL | STK */ |
| 285 | FPE_FLTSUB, /* 57 - INV | DNML | DZ | UFL | STK */ |
| 286 | FPE_FLTOVF, /* 58 - OFL | UFL | STK */ |
| 287 | FPE_FLTSUB, /* 59 - INV | OFL | UFL | STK */ |
| 288 | FPE_FLTUND, /* 5A - DNML | OFL | UFL | STK */ |
| 289 | FPE_FLTSUB, /* 5B - INV | DNML | OFL | UFL | STK */ |
| 290 | FPE_FLTDIV, /* 5C - DZ | OFL | UFL | STK */ |
| 291 | FPE_FLTSUB, /* 5D - INV | DZ | OFL | UFL | STK */ |
| 292 | FPE_FLTDIV, /* 5E - DNML | DZ | OFL | UFL | STK */ |
| 293 | FPE_FLTSUB, /* 5F - INV | DNML | DZ | OFL | UFL | STK */ |
| 294 | FPE_FLTRES, /* 60 - IMP | STK */ |
| 295 | FPE_FLTSUB, /* 61 - INV | IMP | STK */ |
| 296 | FPE_FLTUND, /* 62 - DNML | IMP | STK */ |
| 297 | FPE_FLTSUB, /* 63 - INV | DNML | IMP | STK */ |
| 298 | FPE_FLTDIV, /* 64 - DZ | IMP | STK */ |
| 299 | FPE_FLTSUB, /* 65 - INV | DZ | IMP | STK */ |
| 300 | FPE_FLTDIV, /* 66 - DNML | DZ | IMP | STK */ |
| 301 | FPE_FLTSUB, /* 67 - INV | DNML | DZ | IMP | STK */ |
| 302 | FPE_FLTOVF, /* 68 - OFL | IMP | STK */ |
| 303 | FPE_FLTSUB, /* 69 - INV | OFL | IMP | STK */ |
| 304 | FPE_FLTUND, /* 6A - DNML | OFL | IMP | STK */ |
| 305 | FPE_FLTSUB, /* 6B - INV | DNML | OFL | IMP | STK */ |
| 306 | FPE_FLTDIV, /* 6C - DZ | OFL | IMP | STK */ |
| 307 | FPE_FLTSUB, /* 6D - INV | DZ | OFL | IMP | STK */ |
| 308 | FPE_FLTDIV, /* 6E - DNML | DZ | OFL | IMP | STK */ |
| 309 | FPE_FLTSUB, /* 6F - INV | DNML | DZ | OFL | IMP | STK */ |
| 310 | FPE_FLTUND, /* 70 - UFL | IMP | STK */ |
| 311 | FPE_FLTSUB, /* 71 - INV | UFL | IMP | STK */ |
| 312 | FPE_FLTUND, /* 72 - DNML | UFL | IMP | STK */ |
| 313 | FPE_FLTSUB, /* 73 - INV | DNML | UFL | IMP | STK */ |
| 314 | FPE_FLTDIV, /* 74 - DZ | UFL | IMP | STK */ |
| 315 | FPE_FLTSUB, /* 75 - INV | DZ | UFL | IMP | STK */ |
| 316 | FPE_FLTDIV, /* 76 - DNML | DZ | UFL | IMP | STK */ |
| 317 | FPE_FLTSUB, /* 77 - INV | DNML | DZ | UFL | IMP | STK */ |
| 318 | FPE_FLTOVF, /* 78 - OFL | UFL | IMP | STK */ |
| 319 | FPE_FLTSUB, /* 79 - INV | OFL | UFL | IMP | STK */ |
| 320 | FPE_FLTUND, /* 7A - DNML | OFL | UFL | IMP | STK */ |
| 321 | FPE_FLTSUB, /* 7B - INV | DNML | OFL | UFL | IMP | STK */ |
| 322 | FPE_FLTDIV, /* 7C - DZ | OFL | UFL | IMP | STK */ |
| 323 | FPE_FLTSUB, /* 7D - INV | DZ | OFL | UFL | IMP | STK */ |
| 324 | FPE_FLTDIV, /* 7E - DNML | DZ | OFL | UFL | IMP | STK */ |
| 325 | FPE_FLTSUB, /* 7F - INV | DNML | DZ | OFL | UFL | IMP | STK */ |
| 326 | }; |
| 327 | #endif |
| 328 | |
| 329 | #if 0 |
| 330 | |
| 331 | /* |
| 332 | * Preserve the FP status word, clear FP exceptions, then generate a SIGFPE. |
| 333 | * |
| 334 | * Clearing exceptions is necessary mainly to avoid IRQ13 bugs. We now |
| 335 | * depend on longjmp() restoring a usable state. Restoring the state |
| 336 | * or examining it might fail if we didn't clear exceptions. |
| 337 | * |
| 338 | * The error code chosen will be one of the FPE_... macros. It will be |
| 339 | * sent as the second argument to old BSD-style signal handlers and as |
| 340 | * "siginfo_t->si_code" (second argument) to SA_SIGINFO signal handlers. |
| 341 | * |
| 342 | * XXX the FP state is not preserved across signal handlers. So signal |
| 343 | * handlers cannot afford to do FP unless they preserve the state or |
| 344 | * longjmp() out. Both preserving the state and longjmp()ing may be |
| 345 | * destroyed by IRQ13 bugs. Clearing FP exceptions is not an acceptable |
| 346 | * solution for signals other than SIGFPE. |
| 347 | * |
| 348 | * The MP lock is not held on entry (see i386/i386/exception.s) and |
| 349 | * should not be held on exit. Interrupts are enabled. We must enter |
| 350 | * a critical section to stabilize the FP system and prevent an interrupt |
| 351 | * or preemption from changing the FP state out from under us. |
| 352 | */ |
| 353 | void |
| 354 | npx_intr(void *dummy) |
| 355 | { |
| 356 | int code; |
| 357 | u_short control; |
| 358 | u_short status; |
| 359 | struct intrframe *frame; |
| 360 | |
| 361 | crit_enter(); |
| 362 | |
| 363 | /* |
| 364 | * This exception can only occur with CR0_TS clear, otherwise we |
| 365 | * would get a DNA exception. However, since interrupts were |
| 366 | * enabled a preemption could have sneaked in and used the FP system |
| 367 | * before we entered our critical section. If that occured, the |
| 368 | * TS bit will be set and npxthread will be NULL. |
| 369 | */ |
| 370 | panic("npx_intr: not coded"); |
| 371 | /* XXX FP STATE FLAG MUST BE PART OF CONTEXT SUPPLIED BY REAL KERNEL */ |
| 372 | #if 0 |
| 373 | if (rcr0() & CR0_TS) { |
| 374 | KASSERT(mdcpu->gd_npxthread == NULL, ("gd_npxthread was %p with TS set!", mdcpu->gd_npxthread)); |
| 375 | npxdna(); |
| 376 | crit_exit(); |
| 377 | return; |
| 378 | } |
| 379 | #endif |
| 380 | if (mdcpu->gd_npxthread == NULL) { |
| 381 | get_mplock(); |
| 382 | kprintf("npxintr: npxthread = %p, curthread = %p\n", |
| 383 | mdcpu->gd_npxthread, curthread); |
| 384 | panic("npxintr from nowhere"); |
| 385 | } |
| 386 | if (mdcpu->gd_npxthread != curthread) { |
| 387 | get_mplock(); |
| 388 | kprintf("npxintr: npxthread = %p, curthread = %p\n", |
| 389 | mdcpu->gd_npxthread, curthread); |
| 390 | panic("npxintr from non-current process"); |
| 391 | } |
| 392 | |
| 393 | outb(0xf0, 0); |
| 394 | fnstsw(&status); |
| 395 | fnstcw(&control); |
| 396 | fnclex(); |
| 397 | |
| 398 | get_mplock(); |
| 399 | |
| 400 | /* |
| 401 | * Pass exception to process. |
| 402 | */ |
| 403 | frame = (struct intrframe *)&dummy; /* XXX */ |
| 404 | if ((ISPL(frame->if_cs) == SEL_UPL) /*||(frame->if_eflags&PSL_VM)*/) { |
| 405 | /* |
| 406 | * Interrupt is essentially a trap, so we can afford to call |
| 407 | * the SIGFPE handler (if any) as soon as the interrupt |
| 408 | * returns. |
| 409 | * |
| 410 | * XXX little or nothing is gained from this, and plenty is |
| 411 | * lost - the interrupt frame has to contain the trap frame |
| 412 | * (this is otherwise only necessary for the rescheduling trap |
| 413 | * in doreti, and the frame for that could easily be set up |
| 414 | * just before it is used). |
| 415 | */ |
| 416 | curthread->td_lwp->lwp_md.md_regs = INTR_TO_TRAPFRAME(frame); |
| 417 | /* |
| 418 | * Encode the appropriate code for detailed information on |
| 419 | * this exception. |
| 420 | */ |
| 421 | code = |
| 422 | fpetable[(status & ~control & 0x3f) | (status & 0x40)]; |
| 423 | trapsignal(curthread->td_lwp, SIGFPE, code); |
| 424 | } else { |
| 425 | /* |
| 426 | * Nested interrupt. These losers occur when: |
| 427 | * o an IRQ13 is bogusly generated at a bogus time, e.g.: |
| 428 | * o immediately after an fnsave or frstor of an |
| 429 | * error state. |
| 430 | * o a couple of 386 instructions after |
| 431 | * "fstpl _memvar" causes a stack overflow. |
| 432 | * These are especially nasty when combined with a |
| 433 | * trace trap. |
| 434 | * o an IRQ13 occurs at the same time as another higher- |
| 435 | * priority interrupt. |
| 436 | * |
| 437 | * Treat them like a true async interrupt. |
| 438 | */ |
| 439 | lwpsignal(curproc, curthread->td_lwp, SIGFPE); |
| 440 | } |
| 441 | rel_mplock(); |
| 442 | crit_exit(); |
| 443 | } |
| 444 | |
| 445 | #endif |
| 446 | |
| 447 | /* |
| 448 | * Implement the device not available (DNA) exception. gd_npxthread had |
| 449 | * better be NULL. Restore the current thread's FP state and set gd_npxthread |
| 450 | * to curthread. |
| 451 | * |
| 452 | * Interrupts are enabled and preemption can occur. Enter a critical |
| 453 | * section to stabilize the FP state. |
| 454 | */ |
| 455 | int |
| 456 | npxdna(struct trapframe *frame) |
| 457 | { |
| 458 | thread_t td = curthread; |
| 459 | int didinit = 0; |
| 460 | |
| 461 | if (mdcpu->gd_npxthread != NULL) { |
| 462 | kprintf("npxdna: npxthread = %p, curthread = %p\n", |
| 463 | mdcpu->gd_npxthread, td); |
| 464 | panic("npxdna"); |
| 465 | } |
| 466 | |
| 467 | /* |
| 468 | * Setup the initial saved state if the thread has never before |
| 469 | * used the FP unit. This also occurs when a thread pushes a |
| 470 | * signal handler and uses FP in the handler. |
| 471 | */ |
| 472 | if ((curthread->td_flags & TDF_USINGFP) == 0) { |
| 473 | curthread->td_flags |= TDF_USINGFP; |
| 474 | npxinit(__INITIAL_NPXCW__); |
| 475 | didinit = 1; |
| 476 | } |
| 477 | |
| 478 | /* |
| 479 | * The setting of gd_npxthread and the call to fpurstor() must not |
| 480 | * be preempted by an interrupt thread or we will take an npxdna |
| 481 | * trap and potentially save our current fpstate (which is garbage) |
| 482 | * and then restore the garbage rather then the originally saved |
| 483 | * fpstate. |
| 484 | */ |
| 485 | crit_enter(); |
| 486 | /*stop_emulating();*/ |
| 487 | /* |
| 488 | * Record new context early in case frstor causes an IRQ13. |
| 489 | */ |
| 490 | mdcpu->gd_npxthread = td; |
| 491 | /* |
| 492 | * The following frstor may cause an IRQ13 when the state being |
| 493 | * restored has a pending error. The error will appear to have been |
| 494 | * triggered by the current (npx) user instruction even when that |
| 495 | * instruction is a no-wait instruction that should not trigger an |
| 496 | * error (e.g., fnclex). On at least one 486 system all of the |
| 497 | * no-wait instructions are broken the same as frstor, so our |
| 498 | * treatment does not amplify the breakage. On at least one |
| 499 | * 386/Cyrix 387 system, fnclex works correctly while frstor and |
| 500 | * fnsave are broken, so our treatment breaks fnclex if it is the |
| 501 | * first FPU instruction after a context switch. |
| 502 | */ |
| 503 | if ((td->td_savefpu->sv_xmm.sv_env.en_mxcsr & ~0xFFBF) && cpu_fxsr) { |
| 504 | krateprintf(&badfprate, |
| 505 | "FXRSTR: illegal FP MXCSR %08x didinit = %d\n", |
| 506 | td->td_savefpu->sv_xmm.sv_env.en_mxcsr, didinit); |
| 507 | td->td_savefpu->sv_xmm.sv_env.en_mxcsr &= 0xFFBF; |
| 508 | lwpsignal(curproc, curthread->td_lwp, SIGFPE); |
| 509 | } |
| 510 | fpurstor(curthread->td_savefpu); |
| 511 | crit_exit(); |
| 512 | |
| 513 | return (1); |
| 514 | } |
| 515 | |
| 516 | /* |
| 517 | * Wrapper for the fnsave instruction to handle h/w bugs. If there is an error |
| 518 | * pending, then fnsave generates a bogus IRQ13 on some systems. Force |
| 519 | * any IRQ13 to be handled immediately, and then ignore it. This routine is |
| 520 | * often called at splhigh so it must not use many system services. In |
| 521 | * particular, it's much easier to install a special handler than to |
| 522 | * guarantee that it's safe to use npxintr() and its supporting code. |
| 523 | * |
| 524 | * WARNING! This call is made during a switch and the MP lock will be |
| 525 | * setup for the new target thread rather then the current thread, so we |
| 526 | * cannot do anything here that depends on the *_mplock() functions as |
| 527 | * we may trip over their assertions. |
| 528 | * |
| 529 | * WARNING! When using fxsave we MUST fninit after saving the FP state. The |
| 530 | * kernel will always assume that the FP state is 'safe' (will not cause |
| 531 | * exceptions) for mmx/xmm use if npxthread is NULL. The kernel must still |
| 532 | * setup a custom save area before actually using the FP unit, but it will |
| 533 | * not bother calling fninit. This greatly improves kernel performance when |
| 534 | * it wishes to use the FP unit. |
| 535 | */ |
| 536 | void |
| 537 | npxsave(union savefpu *addr) |
| 538 | { |
| 539 | crit_enter(); |
| 540 | /*stop_emulating();*/ |
| 541 | fpusave(addr); |
| 542 | mdcpu->gd_npxthread = NULL; |
| 543 | fninit(); |
| 544 | /*start_emulating();*/ |
| 545 | crit_exit(); |
| 546 | } |
| 547 | |
| 548 | static void |
| 549 | fpusave(union savefpu *addr) |
| 550 | { |
| 551 | if (cpu_fxsr) |
| 552 | fxsave(addr); |
| 553 | else |
| 554 | fnsave(addr); |
| 555 | } |
| 556 | |
| 557 | /* |
| 558 | * Save the FP state to the mcontext structure. |
| 559 | * |
| 560 | * WARNING: If you want to try to npxsave() directly to mctx->mc_fpregs, |
| 561 | * then it MUST be 16-byte aligned. Currently this is not guarenteed. |
| 562 | */ |
| 563 | void |
| 564 | npxpush(mcontext_t *mctx) |
| 565 | { |
| 566 | thread_t td = curthread; |
| 567 | |
| 568 | if (td->td_flags & TDF_USINGFP) { |
| 569 | if (mdcpu->gd_npxthread == td) { |
| 570 | /* |
| 571 | * XXX Note: This is a bit inefficient if the signal |
| 572 | * handler uses floating point, extra faults will |
| 573 | * occur. |
| 574 | */ |
| 575 | mctx->mc_ownedfp = _MC_FPOWNED_FPU; |
| 576 | npxsave(td->td_savefpu); |
| 577 | } else { |
| 578 | mctx->mc_ownedfp = _MC_FPOWNED_PCB; |
| 579 | } |
| 580 | bcopy(td->td_savefpu, mctx->mc_fpregs, sizeof(mctx->mc_fpregs)); |
| 581 | td->td_flags &= ~TDF_USINGFP; |
| 582 | mctx->mc_fpformat = |
| 583 | #ifndef CPU_DISABLE_SSE |
| 584 | (cpu_fxsr) ? _MC_FPFMT_XMM : |
| 585 | #endif |
| 586 | _MC_FPFMT_387; |
| 587 | } else { |
| 588 | mctx->mc_ownedfp = _MC_FPOWNED_NONE; |
| 589 | mctx->mc_fpformat = _MC_FPFMT_NODEV; |
| 590 | } |
| 591 | } |
| 592 | |
| 593 | /* |
| 594 | * Restore the FP state from the mcontext structure. |
| 595 | */ |
| 596 | void |
| 597 | npxpop(mcontext_t *mctx) |
| 598 | { |
| 599 | thread_t td = curthread; |
| 600 | |
| 601 | switch(mctx->mc_ownedfp) { |
| 602 | case _MC_FPOWNED_NONE: |
| 603 | /* |
| 604 | * If the signal handler used the FP unit but the interrupted |
| 605 | * code did not, release the FP unit. Clear TDF_USINGFP will |
| 606 | * force the FP unit to reinit so the interrupted code sees |
| 607 | * a clean slate. |
| 608 | */ |
| 609 | if (td->td_flags & TDF_USINGFP) { |
| 610 | if (td == mdcpu->gd_npxthread) |
| 611 | npxsave(td->td_savefpu); |
| 612 | td->td_flags &= ~TDF_USINGFP; |
| 613 | } |
| 614 | break; |
| 615 | case _MC_FPOWNED_FPU: |
| 616 | case _MC_FPOWNED_PCB: |
| 617 | /* |
| 618 | * Clear ownership of the FP unit and restore our saved state. |
| 619 | * |
| 620 | * NOTE: The signal handler may have set-up some FP state and |
| 621 | * enabled the FP unit, so we have to restore no matter what. |
| 622 | * |
| 623 | * XXX: This is bit inefficient, if the code being returned |
| 624 | * to is actively using the FP this results in multiple |
| 625 | * kernel faults. |
| 626 | * |
| 627 | * WARNING: The saved state was exposed to userland and may |
| 628 | * have to be sanitized to avoid a GP fault in the kernel. |
| 629 | */ |
| 630 | if (td == mdcpu->gd_npxthread) |
| 631 | npxsave(td->td_savefpu); |
| 632 | bcopy(mctx->mc_fpregs, td->td_savefpu, sizeof(*td->td_savefpu)); |
| 633 | if ((td->td_savefpu->sv_xmm.sv_env.en_mxcsr & ~0xFFBF) && |
| 634 | cpu_fxsr) { |
| 635 | krateprintf(&badfprate, |
| 636 | "pid %d (%s) signal return from user: " |
| 637 | "illegal FP MXCSR %08x\n", |
| 638 | td->td_proc->p_pid, |
| 639 | td->td_proc->p_comm, |
| 640 | td->td_savefpu->sv_xmm.sv_env.en_mxcsr); |
| 641 | td->td_savefpu->sv_xmm.sv_env.en_mxcsr &= 0xFFBF; |
| 642 | } |
| 643 | td->td_flags |= TDF_USINGFP; |
| 644 | break; |
| 645 | } |
| 646 | } |
| 647 | |
| 648 | |
| 649 | #ifndef CPU_DISABLE_SSE |
| 650 | /* |
| 651 | * On AuthenticAMD processors, the fxrstor instruction does not restore |
| 652 | * the x87's stored last instruction pointer, last data pointer, and last |
| 653 | * opcode values, except in the rare case in which the exception summary |
| 654 | * (ES) bit in the x87 status word is set to 1. |
| 655 | * |
| 656 | * In order to avoid leaking this information across processes, we clean |
| 657 | * these values by performing a dummy load before executing fxrstor(). |
| 658 | */ |
| 659 | static double dummy_variable = 0.0; |
| 660 | static void |
| 661 | fpu_clean_state(void) |
| 662 | { |
| 663 | u_short status; |
| 664 | |
| 665 | /* |
| 666 | * Clear the ES bit in the x87 status word if it is currently |
| 667 | * set, in order to avoid causing a fault in the upcoming load. |
| 668 | */ |
| 669 | fnstsw(&status); |
| 670 | if (status & 0x80) |
| 671 | fnclex(); |
| 672 | |
| 673 | /* |
| 674 | * Load the dummy variable into the x87 stack. This mangles |
| 675 | * the x87 stack, but we don't care since we're about to call |
| 676 | * fxrstor() anyway. |
| 677 | */ |
| 678 | __asm __volatile("ffree %%st(7); fld %0" : : "m" (dummy_variable)); |
| 679 | } |
| 680 | #endif /* CPU_DISABLE_SSE */ |
| 681 | |
| 682 | static void |
| 683 | fpurstor(union savefpu *addr) |
| 684 | { |
| 685 | #ifndef CPU_DISABLE_SSE |
| 686 | if (cpu_fxsr) { |
| 687 | fpu_clean_state(); |
| 688 | fxrstor(addr); |
| 689 | } else { |
| 690 | frstor(addr); |
| 691 | } |
| 692 | #else |
| 693 | frstor(addr); |
| 694 | #endif |
| 695 | } |
| 696 | |