pc32: Split out isa_intr.h and move isa/intr_machdep.h to include/
[dragonfly.git] / sys / platform / pc64 / x86_64 / machdep.c
CommitLineData
e130cdcb
MD
1/*-
2 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California.
3 * Copyright (c) 1992 Terrence R. Lambert.
4 * Copyright (c) 2003 Peter Wemm.
5 * Copyright (c) 2008 The DragonFly Project.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to Berkeley by
9 * William Jolitz.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed by the University of
22 * California, Berkeley and its contributors.
23 * 4. Neither the name of the University nor the names of its contributors
24 * may be used to endorse or promote products derived from this software
25 * without specific prior written permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * SUCH DAMAGE.
38 *
39 * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91
40 * $FreeBSD: src/sys/i386/i386/machdep.c,v 1.385.2.30 2003/05/31 08:48:05 alc Exp $
41 */
42
43//#include "use_npx.h"
44#include "use_isa.h"
45#include "opt_atalk.h"
46#include "opt_compat.h"
47#include "opt_cpu.h"
48#include "opt_ddb.h"
49#include "opt_directio.h"
50#include "opt_inet.h"
51#include "opt_ipx.h"
52#include "opt_msgbuf.h"
53#include "opt_swap.h"
54#include "opt_apic.h"
55
56#include <sys/param.h>
57#include <sys/systm.h>
58#include <sys/sysproto.h>
59#include <sys/signalvar.h>
60#include <sys/kernel.h>
61#include <sys/linker.h>
62#include <sys/malloc.h>
63#include <sys/proc.h>
64#include <sys/priv.h>
65#include <sys/buf.h>
66#include <sys/reboot.h>
67#include <sys/mbuf.h>
68#include <sys/msgbuf.h>
69#include <sys/sysent.h>
70#include <sys/sysctl.h>
71#include <sys/vmmeter.h>
72#include <sys/bus.h>
73#include <sys/upcall.h>
74#include <sys/usched.h>
75#include <sys/reg.h>
76
77#include <vm/vm.h>
78#include <vm/vm_param.h>
79#include <sys/lock.h>
80#include <vm/vm_kern.h>
81#include <vm/vm_object.h>
82#include <vm/vm_page.h>
83#include <vm/vm_map.h>
84#include <vm/vm_pager.h>
85#include <vm/vm_extern.h>
86
87#include <sys/thread2.h>
88#include <sys/mplock2.h>
89
90#include <sys/user.h>
91#include <sys/exec.h>
92#include <sys/cons.h>
93
94#include <ddb/ddb.h>
95
96#include <machine/cpu.h>
97#include <machine/clock.h>
98#include <machine/specialreg.h>
99#if JG
100#include <machine/bootinfo.h>
101#endif
102#include <machine/md_var.h>
103#include <machine/metadata.h>
104#include <machine/pc/bios.h>
105#include <machine/pcb_ext.h> /* pcb.h included via sys/user.h */
106#include <machine/globaldata.h> /* CPU_prvspace */
107#include <machine/smp.h>
108#ifdef PERFMON
109#include <machine/perfmon.h>
110#endif
111#include <machine/cputypes.h>
57a9c56b 112#include <machine/intr_machdep.h>
e130cdcb
MD
113
114#ifdef OLD_BUS_ARCH
115#include <bus/isa/isa_device.h>
116#endif
57a9c56b 117#include <machine_base/isa/isa_intr.h>
e24dd6e0 118#include <machine_base/isa/elcr_var.h>
e130cdcb
MD
119#include <bus/isa/rtc.h>
120#include <sys/random.h>
121#include <sys/ptrace.h>
122#include <machine/sigframe.h>
123
124#include <sys/machintr.h>
125
126#define PHYSMAP_ENTRIES 10
127
128extern void init386(int first);
129extern void dblfault_handler(void);
130extern u_int64_t hammer_time(u_int64_t, u_int64_t);
131
132extern void printcpuinfo(void); /* XXX header file */
133extern void identify_cpu(void);
134#if JG
135extern void finishidentcpu(void);
136#endif
137extern void panicifcpuunsupported(void);
138
139static void cpu_startup(void *);
140#ifndef CPU_DISABLE_SSE
141static void set_fpregs_xmm(struct save87 *, struct savexmm *);
142static void fill_fpregs_xmm(struct savexmm *, struct save87 *);
143#endif /* CPU_DISABLE_SSE */
144#ifdef DIRECTIO
145extern void ffs_rawread_setup(void);
146#endif /* DIRECTIO */
147static void init_locks(void);
148
149SYSINIT(cpu, SI_BOOT2_SMP, SI_ORDER_FIRST, cpu_startup, NULL)
150
151#ifdef DDB
152extern vm_offset_t ksym_start, ksym_end;
153#endif
154
155struct privatespace CPU_prvspace[MAXCPU] __aligned(4096); /* XXX */
156
157int _udatasel, _ucodesel, _ucode32sel;
158u_long atdevbase;
159#ifdef SMP
160int64_t tsc_offsets[MAXCPU];
161#else
162int64_t tsc_offsets[1];
163#endif
164
165#if defined(SWTCH_OPTIM_STATS)
166extern int swtch_optim_stats;
167SYSCTL_INT(_debug, OID_AUTO, swtch_optim_stats,
168 CTLFLAG_RD, &swtch_optim_stats, 0, "");
169SYSCTL_INT(_debug, OID_AUTO, tlb_flush_count,
170 CTLFLAG_RD, &tlb_flush_count, 0, "");
171#endif
172
173int physmem = 0;
174
175u_long ebda_addr = 0;
176
177static int
178sysctl_hw_physmem(SYSCTL_HANDLER_ARGS)
179{
180 int error = sysctl_handle_int(oidp, 0, ctob(physmem), req);
181 return (error);
182}
183
184SYSCTL_PROC(_hw, HW_PHYSMEM, physmem, CTLTYPE_INT|CTLFLAG_RD,
185 0, 0, sysctl_hw_physmem, "IU", "");
186
187static int
188sysctl_hw_usermem(SYSCTL_HANDLER_ARGS)
189{
190 int error = sysctl_handle_int(oidp, 0,
191 ctob(physmem - vmstats.v_wire_count), req);
192 return (error);
193}
194
195SYSCTL_PROC(_hw, HW_USERMEM, usermem, CTLTYPE_INT|CTLFLAG_RD,
196 0, 0, sysctl_hw_usermem, "IU", "");
197
198static int
199sysctl_hw_availpages(SYSCTL_HANDLER_ARGS)
200{
201 int error = sysctl_handle_int(oidp, 0,
202 x86_64_btop(avail_end - avail_start), req);
203 return (error);
204}
205
206SYSCTL_PROC(_hw, OID_AUTO, availpages, CTLTYPE_INT|CTLFLAG_RD,
207 0, 0, sysctl_hw_availpages, "I", "");
208
209vm_paddr_t Maxmem;
210vm_paddr_t Realmem;
211
212/*
213 * The number of PHYSMAP entries must be one less than the number of
214 * PHYSSEG entries because the PHYSMAP entry that spans the largest
215 * physical address that is accessible by ISA DMA is split into two
216 * PHYSSEG entries.
217 */
218#define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1))
219
220vm_paddr_t phys_avail[PHYSMAP_SIZE + 2];
221vm_paddr_t dump_avail[PHYSMAP_SIZE + 2];
222
223/* must be 2 less so 0 0 can signal end of chunks */
c157ff7a
SW
224#define PHYS_AVAIL_ARRAY_END (NELEM(phys_avail) - 2)
225#define DUMP_AVAIL_ARRAY_END (NELEM(dump_avail) - 2)
e130cdcb
MD
226
227static vm_offset_t buffer_sva, buffer_eva;
228vm_offset_t clean_sva, clean_eva;
229static vm_offset_t pager_sva, pager_eva;
230static struct trapframe proc0_tf;
231
232static void
233cpu_startup(void *dummy)
234{
235 caddr_t v;
236 vm_size_t size = 0;
237 vm_offset_t firstaddr;
238
239 if (boothowto & RB_VERBOSE)
240 bootverbose++;
241
242 /*
243 * Good {morning,afternoon,evening,night}.
244 */
245 kprintf("%s", version);
246 startrtclock();
247 printcpuinfo();
248 panicifcpuunsupported();
249#ifdef PERFMON
250 perfmon_init();
251#endif
252 kprintf("real memory = %ju (%ju MB)\n",
253 (intmax_t)Realmem,
254 (intmax_t)Realmem / 1024 / 1024);
255 /*
256 * Display any holes after the first chunk of extended memory.
257 */
258 if (bootverbose) {
259 int indx;
260
261 kprintf("Physical memory chunk(s):\n");
262 for (indx = 0; phys_avail[indx + 1] != 0; indx += 2) {
263 vm_paddr_t size1 = phys_avail[indx + 1] - phys_avail[indx];
264
265 kprintf("0x%08jx - 0x%08jx, %ju bytes (%ju pages)\n",
266 (intmax_t)phys_avail[indx],
267 (intmax_t)phys_avail[indx + 1] - 1,
268 (intmax_t)size1,
269 (intmax_t)(size1 / PAGE_SIZE));
270 }
271 }
272
273 /*
274 * Allocate space for system data structures.
275 * The first available kernel virtual address is in "v".
276 * As pages of kernel virtual memory are allocated, "v" is incremented.
277 * As pages of memory are allocated and cleared,
278 * "firstaddr" is incremented.
279 * An index into the kernel page table corresponding to the
280 * virtual memory address maintained in "v" is kept in "mapaddr".
281 */
282
283 /*
284 * Make two passes. The first pass calculates how much memory is
285 * needed and allocates it. The second pass assigns virtual
286 * addresses to the various data structures.
287 */
288 firstaddr = 0;
289again:
290 v = (caddr_t)firstaddr;
291
292#define valloc(name, type, num) \
293 (name) = (type *)v; v = (caddr_t)((name)+(num))
294#define valloclim(name, type, num, lim) \
295 (name) = (type *)v; v = (caddr_t)((lim) = ((name)+(num)))
296
297 /*
298 * The nominal buffer size (and minimum KVA allocation) is BKVASIZE.
299 * For the first 64MB of ram nominally allocate sufficient buffers to
300 * cover 1/4 of our ram. Beyond the first 64MB allocate additional
301 * buffers to cover 1/20 of our ram over 64MB. When auto-sizing
302 * the buffer cache we limit the eventual kva reservation to
303 * maxbcache bytes.
304 *
305 * factor represents the 1/4 x ram conversion.
306 */
307 if (nbuf == 0) {
308 int factor = 4 * BKVASIZE / 1024;
309 int kbytes = physmem * (PAGE_SIZE / 1024);
310
311 nbuf = 50;
312 if (kbytes > 4096)
313 nbuf += min((kbytes - 4096) / factor, 65536 / factor);
314 if (kbytes > 65536)
315 nbuf += (kbytes - 65536) * 2 / (factor * 5);
316 if (maxbcache && nbuf > maxbcache / BKVASIZE)
317 nbuf = maxbcache / BKVASIZE;
318 }
319
320 /*
321 * Do not allow the buffer_map to be more then 1/2 the size of the
322 * kernel_map.
323 */
324 if (nbuf > (virtual_end - virtual_start) / (BKVASIZE * 2)) {
325 nbuf = (virtual_end - virtual_start) / (BKVASIZE * 2);
326 kprintf("Warning: nbufs capped at %d\n", nbuf);
327 }
328
329 nswbuf = max(min(nbuf/4, 256), 16);
330#ifdef NSWBUF_MIN
331 if (nswbuf < NSWBUF_MIN)
332 nswbuf = NSWBUF_MIN;
333#endif
334#ifdef DIRECTIO
335 ffs_rawread_setup();
336#endif
337
338 valloc(swbuf, struct buf, nswbuf);
339 valloc(buf, struct buf, nbuf);
340
341 /*
342 * End of first pass, size has been calculated so allocate memory
343 */
344 if (firstaddr == 0) {
345 size = (vm_size_t)(v - firstaddr);
346 firstaddr = kmem_alloc(&kernel_map, round_page(size));
347 if (firstaddr == 0)
348 panic("startup: no room for tables");
349 goto again;
350 }
351
352 /*
353 * End of second pass, addresses have been assigned
354 */
355 if ((vm_size_t)(v - firstaddr) != size)
356 panic("startup: table size inconsistency");
357
358 kmem_suballoc(&kernel_map, &clean_map, &clean_sva, &clean_eva,
359 (nbuf*BKVASIZE) + (nswbuf*MAXPHYS) + pager_map_size);
360 kmem_suballoc(&clean_map, &buffer_map, &buffer_sva, &buffer_eva,
361 (nbuf*BKVASIZE));
362 buffer_map.system_map = 1;
363 kmem_suballoc(&clean_map, &pager_map, &pager_sva, &pager_eva,
364 (nswbuf*MAXPHYS) + pager_map_size);
365 pager_map.system_map = 1;
366
367#if defined(USERCONFIG)
368 userconfig();
369 cninit(); /* the preferred console may have changed */
370#endif
371
372 kprintf("avail memory = %ju (%ju MB)\n",
373 (uintmax_t)ptoa(vmstats.v_free_count),
374 (uintmax_t)ptoa(vmstats.v_free_count) / 1024 / 1024);
375
376 /*
377 * Set up buffers, so they can be used to read disk labels.
378 */
379 bufinit();
380 vm_pager_bufferinit();
381
e24dd6e0
SZ
382 /* Log ELCR information */
383 elcr_dump();
384
e130cdcb
MD
385#ifdef SMP
386 /*
387 * OK, enough kmem_alloc/malloc state should be up, lets get on with it!
388 */
389 mp_start(); /* fire up the APs and APICs */
390 mp_announce();
391#endif /* SMP */
392 cpu_setregs();
393}
394
395/*
396 * Send an interrupt to process.
397 *
398 * Stack is set up to allow sigcode stored
399 * at top to call routine, followed by kcall
400 * to sigreturn routine below. After sigreturn
401 * resets the signal mask, the stack, and the
402 * frame pointer, it returns to the user
403 * specified pc, psl.
404 */
405void
406sendsig(sig_t catcher, int sig, sigset_t *mask, u_long code)
407{
408 struct lwp *lp = curthread->td_lwp;
409 struct proc *p = lp->lwp_proc;
410 struct trapframe *regs;
411 struct sigacts *psp = p->p_sigacts;
412 struct sigframe sf, *sfp;
413 int oonstack;
414 char *sp;
415
416 regs = lp->lwp_md.md_regs;
417 oonstack = (lp->lwp_sigstk.ss_flags & SS_ONSTACK) ? 1 : 0;
418
419 /* Save user context */
420 bzero(&sf, sizeof(struct sigframe));
421 sf.sf_uc.uc_sigmask = *mask;
422 sf.sf_uc.uc_stack = lp->lwp_sigstk;
423 sf.sf_uc.uc_mcontext.mc_onstack = oonstack;
424 KKASSERT(__offsetof(struct trapframe, tf_rdi) == 0);
425 bcopy(regs, &sf.sf_uc.uc_mcontext.mc_rdi, sizeof(struct trapframe));
426
427 /* Make the size of the saved context visible to userland */
428 sf.sf_uc.uc_mcontext.mc_len = sizeof(sf.sf_uc.uc_mcontext);
429
430 /* Save mailbox pending state for syscall interlock semantics */
431 if (p->p_flag & P_MAILBOX)
432 sf.sf_uc.uc_mcontext.mc_xflags |= PGEX_MAILBOX;
433
434 /* Allocate and validate space for the signal handler context. */
435 if ((lp->lwp_flag & LWP_ALTSTACK) != 0 && !oonstack &&
436 SIGISMEMBER(psp->ps_sigonstack, sig)) {
437 sp = (char *)(lp->lwp_sigstk.ss_sp + lp->lwp_sigstk.ss_size -
438 sizeof(struct sigframe));
439 lp->lwp_sigstk.ss_flags |= SS_ONSTACK;
440 } else {
441 /* We take red zone into account */
442 sp = (char *)regs->tf_rsp - sizeof(struct sigframe) - 128;
443 }
444
445 /* Align to 16 bytes */
446 sfp = (struct sigframe *)((intptr_t)sp & ~(intptr_t)0xF);
447
448 /* Translate the signal is appropriate */
449 if (p->p_sysent->sv_sigtbl) {
450 if (sig <= p->p_sysent->sv_sigsize)
451 sig = p->p_sysent->sv_sigtbl[_SIG_IDX(sig)];
452 }
453
454 /*
455 * Build the argument list for the signal handler.
456 *
457 * Arguments are in registers (%rdi, %rsi, %rdx, %rcx)
458 */
459 regs->tf_rdi = sig; /* argument 1 */
460 regs->tf_rdx = (register_t)&sfp->sf_uc; /* argument 3 */
461
462 if (SIGISMEMBER(psp->ps_siginfo, sig)) {
463 /*
464 * Signal handler installed with SA_SIGINFO.
465 *
466 * action(signo, siginfo, ucontext)
467 */
468 regs->tf_rsi = (register_t)&sfp->sf_si; /* argument 2 */
469 regs->tf_rcx = (register_t)regs->tf_addr; /* argument 4 */
470 sf.sf_ahu.sf_action = (__siginfohandler_t *)catcher;
471
472 /* fill siginfo structure */
473 sf.sf_si.si_signo = sig;
474 sf.sf_si.si_code = code;
475 sf.sf_si.si_addr = (void *)regs->tf_addr;
476 } else {
477 /*
478 * Old FreeBSD-style arguments.
479 *
480 * handler (signo, code, [uc], addr)
481 */
482 regs->tf_rsi = (register_t)code; /* argument 2 */
483 regs->tf_rcx = (register_t)regs->tf_addr; /* argument 4 */
484 sf.sf_ahu.sf_handler = catcher;
485 }
486
487 /*
488 * If we're a vm86 process, we want to save the segment registers.
489 * We also change eflags to be our emulated eflags, not the actual
490 * eflags.
491 */
492#if JG
493 if (regs->tf_eflags & PSL_VM) {
494 struct trapframe_vm86 *tf = (struct trapframe_vm86 *)regs;
495 struct vm86_kernel *vm86 = &lp->lwp_thread->td_pcb->pcb_ext->ext_vm86;
496
497 sf.sf_uc.uc_mcontext.mc_gs = tf->tf_vm86_gs;
498 sf.sf_uc.uc_mcontext.mc_fs = tf->tf_vm86_fs;
499 sf.sf_uc.uc_mcontext.mc_es = tf->tf_vm86_es;
500 sf.sf_uc.uc_mcontext.mc_ds = tf->tf_vm86_ds;
501
502 if (vm86->vm86_has_vme == 0)
503 sf.sf_uc.uc_mcontext.mc_eflags =
504 (tf->tf_eflags & ~(PSL_VIF | PSL_VIP)) |
505 (vm86->vm86_eflags & (PSL_VIF | PSL_VIP));
506
507 /*
508 * Clear PSL_NT to inhibit T_TSSFLT faults on return from
509 * syscalls made by the signal handler. This just avoids
510 * wasting time for our lazy fixup of such faults. PSL_NT
511 * does nothing in vm86 mode, but vm86 programs can set it
512 * almost legitimately in probes for old cpu types.
513 */
514 tf->tf_eflags &= ~(PSL_VM | PSL_NT | PSL_VIF | PSL_VIP);
515 }
516#endif
517
518 /*
519 * Save the FPU state and reinit the FP unit
520 */
521 npxpush(&sf.sf_uc.uc_mcontext);
522
523 /*
524 * Copy the sigframe out to the user's stack.
525 */
526 if (copyout(&sf, sfp, sizeof(struct sigframe)) != 0) {
527 /*
528 * Something is wrong with the stack pointer.
529 * ...Kill the process.
530 */
531 sigexit(lp, SIGILL);
532 }
533
534 regs->tf_rsp = (register_t)sfp;
535 regs->tf_rip = PS_STRINGS - *(p->p_sysent->sv_szsigcode);
536
537 /*
538 * i386 abi specifies that the direction flag must be cleared
539 * on function entry
540 */
541 regs->tf_rflags &= ~(PSL_T|PSL_D);
542
543 /*
544 * 64 bit mode has a code and stack selector but
545 * no data or extra selector. %fs and %gs are not
546 * stored in-context.
547 */
548 regs->tf_cs = _ucodesel;
549 regs->tf_ss = _udatasel;
550}
551
552/*
553 * Sanitize the trapframe for a virtual kernel passing control to a custom
554 * VM context. Remove any items that would otherwise create a privilage
555 * issue.
556 *
557 * XXX at the moment we allow userland to set the resume flag. Is this a
558 * bad idea?
559 */
560int
561cpu_sanitize_frame(struct trapframe *frame)
562{
563 frame->tf_cs = _ucodesel;
564 frame->tf_ss = _udatasel;
565 /* XXX VM (8086) mode not supported? */
566 frame->tf_rflags &= (PSL_RF | PSL_USERCHANGE | PSL_VM_UNSUPP);
567 frame->tf_rflags |= PSL_RESERVED_DEFAULT | PSL_I;
568
569 return(0);
570}
571
572/*
573 * Sanitize the tls so loading the descriptor does not blow up
574 * on us. For x86_64 we don't have to do anything.
575 */
576int
577cpu_sanitize_tls(struct savetls *tls)
578{
579 return(0);
580}
581
582/*
583 * sigreturn(ucontext_t *sigcntxp)
584 *
585 * System call to cleanup state after a signal
586 * has been taken. Reset signal mask and
587 * stack state from context left by sendsig (above).
588 * Return to previous pc and psl as specified by
589 * context left by sendsig. Check carefully to
590 * make sure that the user has not modified the
591 * state to gain improper privileges.
592 *
593 * MPSAFE
594 */
595#define EFL_SECURE(ef, oef) ((((ef) ^ (oef)) & ~PSL_USERCHANGE) == 0)
596#define CS_SECURE(cs) (ISPL(cs) == SEL_UPL)
597
598int
599sys_sigreturn(struct sigreturn_args *uap)
600{
601 struct lwp *lp = curthread->td_lwp;
602 struct proc *p = lp->lwp_proc;
603 struct trapframe *regs;
604 ucontext_t uc;
605 ucontext_t *ucp;
606 register_t rflags;
607 int cs;
608 int error;
609
610 /*
611 * We have to copy the information into kernel space so userland
612 * can't modify it while we are sniffing it.
613 */
614 regs = lp->lwp_md.md_regs;
615 error = copyin(uap->sigcntxp, &uc, sizeof(uc));
616 if (error)
617 return (error);
618 ucp = &uc;
619 rflags = ucp->uc_mcontext.mc_rflags;
620
621 /* VM (8086) mode not supported */
622 rflags &= ~PSL_VM_UNSUPP;
623
624#if JG
625 if (eflags & PSL_VM) {
626 struct trapframe_vm86 *tf = (struct trapframe_vm86 *)regs;
627 struct vm86_kernel *vm86;
628
629 /*
630 * if pcb_ext == 0 or vm86_inited == 0, the user hasn't
631 * set up the vm86 area, and we can't enter vm86 mode.
632 */
633 if (lp->lwp_thread->td_pcb->pcb_ext == 0)
634 return (EINVAL);
635 vm86 = &lp->lwp_thread->td_pcb->pcb_ext->ext_vm86;
636 if (vm86->vm86_inited == 0)
637 return (EINVAL);
638
639 /* go back to user mode if both flags are set */
640 if ((eflags & PSL_VIP) && (eflags & PSL_VIF))
641 trapsignal(lp, SIGBUS, 0);
642
643 if (vm86->vm86_has_vme) {
644 eflags = (tf->tf_eflags & ~VME_USERCHANGE) |
645 (eflags & VME_USERCHANGE) | PSL_VM;
646 } else {
647 vm86->vm86_eflags = eflags; /* save VIF, VIP */
648 eflags = (tf->tf_eflags & ~VM_USERCHANGE) |
649 (eflags & VM_USERCHANGE) | PSL_VM;
650 }
651 bcopy(&ucp->uc_mcontext.mc_gs, tf, sizeof(struct trapframe));
652 tf->tf_eflags = eflags;
653 tf->tf_vm86_ds = tf->tf_ds;
654 tf->tf_vm86_es = tf->tf_es;
655 tf->tf_vm86_fs = tf->tf_fs;
656 tf->tf_vm86_gs = tf->tf_gs;
657 tf->tf_ds = _udatasel;
658 tf->tf_es = _udatasel;
659 tf->tf_fs = _udatasel;
660 tf->tf_gs = _udatasel;
661 } else
662#endif
663 {
664 /*
665 * Don't allow users to change privileged or reserved flags.
666 */
667 /*
668 * XXX do allow users to change the privileged flag PSL_RF.
669 * The cpu sets PSL_RF in tf_eflags for faults. Debuggers
670 * should sometimes set it there too. tf_eflags is kept in
671 * the signal context during signal handling and there is no
672 * other place to remember it, so the PSL_RF bit may be
673 * corrupted by the signal handler without us knowing.
674 * Corruption of the PSL_RF bit at worst causes one more or
675 * one less debugger trap, so allowing it is fairly harmless.
676 */
677 if (!EFL_SECURE(rflags & ~PSL_RF, regs->tf_rflags & ~PSL_RF)) {
678 kprintf("sigreturn: rflags = 0x%lx\n", (long)rflags);
679 return(EINVAL);
680 }
681
682 /*
683 * Don't allow users to load a valid privileged %cs. Let the
684 * hardware check for invalid selectors, excess privilege in
685 * other selectors, invalid %eip's and invalid %esp's.
686 */
687 cs = ucp->uc_mcontext.mc_cs;
688 if (!CS_SECURE(cs)) {
689 kprintf("sigreturn: cs = 0x%x\n", cs);
690 trapsignal(lp, SIGBUS, T_PROTFLT);
691 return(EINVAL);
692 }
693 bcopy(&ucp->uc_mcontext.mc_rdi, regs, sizeof(struct trapframe));
694 }
695
696 /*
697 * Restore the FPU state from the frame
698 */
699 crit_enter();
700 npxpop(&ucp->uc_mcontext);
701
702 /*
703 * Merge saved signal mailbox pending flag to maintain interlock
704 * semantics against system calls.
705 */
706 if (ucp->uc_mcontext.mc_xflags & PGEX_MAILBOX)
707 p->p_flag |= P_MAILBOX;
708
709 if (ucp->uc_mcontext.mc_onstack & 1)
710 lp->lwp_sigstk.ss_flags |= SS_ONSTACK;
711 else
712 lp->lwp_sigstk.ss_flags &= ~SS_ONSTACK;
713
714 lp->lwp_sigmask = ucp->uc_sigmask;
715 SIG_CANTMASK(lp->lwp_sigmask);
716 crit_exit();
717 return(EJUSTRETURN);
718}
719
720/*
721 * Stack frame on entry to function. %rax will contain the function vector,
722 * %rcx will contain the function data. flags, rcx, and rax will have
723 * already been pushed on the stack.
724 */
725struct upc_frame {
726 register_t rax;
727 register_t rcx;
728 register_t rdx;
729 register_t flags;
730 register_t oldip;
731};
732
733void
734sendupcall(struct vmupcall *vu, int morepending)
735{
736 struct lwp *lp = curthread->td_lwp;
737 struct trapframe *regs;
738 struct upcall upcall;
739 struct upc_frame upc_frame;
740 int crit_count = 0;
741
742 /*
743 * If we are a virtual kernel running an emulated user process
744 * context, switch back to the virtual kernel context before
745 * trying to post the signal.
746 */
747 if (lp->lwp_vkernel && lp->lwp_vkernel->ve) {
748 lp->lwp_md.md_regs->tf_trapno = 0;
749 vkernel_trap(lp, lp->lwp_md.md_regs);
750 }
751
752 /*
753 * Get the upcall data structure
754 */
755 if (copyin(lp->lwp_upcall, &upcall, sizeof(upcall)) ||
756 copyin((char *)upcall.upc_uthread + upcall.upc_critoff, &crit_count, sizeof(int))
757 ) {
758 vu->vu_pending = 0;
759 kprintf("bad upcall address\n");
760 return;
761 }
762
763 /*
764 * If the data structure is already marked pending or has a critical
765 * section count, mark the data structure as pending and return
766 * without doing an upcall. vu_pending is left set.
767 */
768 if (upcall.upc_pending || crit_count >= vu->vu_pending) {
769 if (upcall.upc_pending < vu->vu_pending) {
770 upcall.upc_pending = vu->vu_pending;
771 copyout(&upcall.upc_pending, &lp->lwp_upcall->upc_pending,
772 sizeof(upcall.upc_pending));
773 }
774 return;
775 }
776
777 /*
778 * We can run this upcall now, clear vu_pending.
779 *
780 * Bump our critical section count and set or clear the
781 * user pending flag depending on whether more upcalls are
782 * pending. The user will be responsible for calling
783 * upc_dispatch(-1) to process remaining upcalls.
784 */
785 vu->vu_pending = 0;
786 upcall.upc_pending = morepending;
787 ++crit_count;
788 copyout(&upcall.upc_pending, &lp->lwp_upcall->upc_pending,
789 sizeof(upcall.upc_pending));
790 copyout(&crit_count, (char *)upcall.upc_uthread + upcall.upc_critoff,
791 sizeof(int));
792
793 /*
794 * Construct a stack frame and issue the upcall
795 */
796 regs = lp->lwp_md.md_regs;
797 upc_frame.rax = regs->tf_rax;
798 upc_frame.rcx = regs->tf_rcx;
799 upc_frame.rdx = regs->tf_rdx;
800 upc_frame.flags = regs->tf_rflags;
801 upc_frame.oldip = regs->tf_rip;
802 if (copyout(&upc_frame, (void *)(regs->tf_rsp - sizeof(upc_frame)),
803 sizeof(upc_frame)) != 0) {
804 kprintf("bad stack on upcall\n");
805 } else {
806 regs->tf_rax = (register_t)vu->vu_func;
807 regs->tf_rcx = (register_t)vu->vu_data;
808 regs->tf_rdx = (register_t)lp->lwp_upcall;
809 regs->tf_rip = (register_t)vu->vu_ctx;
810 regs->tf_rsp -= sizeof(upc_frame);
811 }
812}
813
814/*
815 * fetchupcall occurs in the context of a system call, which means that
816 * we have to return EJUSTRETURN in order to prevent eax and edx from
817 * being overwritten by the syscall return value.
818 *
819 * if vu is not NULL we return the new context in %edx, the new data in %ecx,
820 * and the function pointer in %eax.
821 */
822int
823fetchupcall(struct vmupcall *vu, int morepending, void *rsp)
824{
825 struct upc_frame upc_frame;
826 struct lwp *lp = curthread->td_lwp;
827 struct trapframe *regs;
828 int error;
829 struct upcall upcall;
830 int crit_count;
831
832 regs = lp->lwp_md.md_regs;
833
834 error = copyout(&morepending, &lp->lwp_upcall->upc_pending, sizeof(int));
835 if (error == 0) {
836 if (vu) {
837 /*
838 * This jumps us to the next ready context.
839 */
840 vu->vu_pending = 0;
841 error = copyin(lp->lwp_upcall, &upcall, sizeof(upcall));
842 crit_count = 0;
843 if (error == 0)
844 error = copyin((char *)upcall.upc_uthread + upcall.upc_critoff, &crit_count, sizeof(int));
845 ++crit_count;
846 if (error == 0)
847 error = copyout(&crit_count, (char *)upcall.upc_uthread + upcall.upc_critoff, sizeof(int));
848 regs->tf_rax = (register_t)vu->vu_func;
849 regs->tf_rcx = (register_t)vu->vu_data;
850 regs->tf_rdx = (register_t)lp->lwp_upcall;
851 regs->tf_rip = (register_t)vu->vu_ctx;
852 regs->tf_rsp = (register_t)rsp;
853 } else {
854 /*
855 * This returns us to the originally interrupted code.
856 */
857 error = copyin(rsp, &upc_frame, sizeof(upc_frame));
858 regs->tf_rax = upc_frame.rax;
859 regs->tf_rcx = upc_frame.rcx;
860 regs->tf_rdx = upc_frame.rdx;
861 regs->tf_rflags = (regs->tf_rflags & ~PSL_USERCHANGE) |
862 (upc_frame.flags & PSL_USERCHANGE);
863 regs->tf_rip = upc_frame.oldip;
864 regs->tf_rsp = (register_t)((char *)rsp + sizeof(upc_frame));
865 }
866 }
867 if (error == 0)
868 error = EJUSTRETURN;
869 return(error);
870}
871
872/*
873 * Machine dependent boot() routine
874 *
875 * I haven't seen anything to put here yet
876 * Possibly some stuff might be grafted back here from boot()
877 */
878void
879cpu_boot(int howto)
880{
881}
882
883/*
884 * Shutdown the CPU as much as possible
885 */
886void
887cpu_halt(void)
888{
889 for (;;)
890 __asm__ __volatile("hlt");
891}
892
893/*
894 * cpu_idle() represents the idle LWKT. You cannot return from this function
895 * (unless you want to blow things up!). Instead we look for runnable threads
896 * and loop or halt as appropriate. Giant is not held on entry to the thread.
897 *
898 * The main loop is entered with a critical section held, we must release
899 * the critical section before doing anything else. lwkt_switch() will
900 * check for pending interrupts due to entering and exiting its own
901 * critical section.
902 *
903 * NOTE: On an SMP system we rely on a scheduler IPI to wake a HLTed cpu up.
904 * However, there are cases where the idlethread will be entered with
905 * the possibility that no IPI will occur and in such cases
906 * lwkt_switch() sets TDF_IDLE_NOHLT.
907 *
908 * NOTE: cpu_idle_hlt again defaults to 2 (use ACPI sleep states). Set to
909 * 1 to just use hlt and for debugging purposes.
be71787b
MD
910 *
911 * NOTE: cpu_idle_repeat determines how many entries into the idle thread
912 * must occur before it starts using ACPI halt.
e130cdcb
MD
913 */
914static int cpu_idle_hlt = 2;
915static int cpu_idle_hltcnt;
916static int cpu_idle_spincnt;
be71787b 917static u_int cpu_idle_repeat = 4;
e130cdcb
MD
918SYSCTL_INT(_machdep, OID_AUTO, cpu_idle_hlt, CTLFLAG_RW,
919 &cpu_idle_hlt, 0, "Idle loop HLT enable");
920SYSCTL_INT(_machdep, OID_AUTO, cpu_idle_hltcnt, CTLFLAG_RW,
921 &cpu_idle_hltcnt, 0, "Idle loop entry halts");
922SYSCTL_INT(_machdep, OID_AUTO, cpu_idle_spincnt, CTLFLAG_RW,
923 &cpu_idle_spincnt, 0, "Idle loop entry spins");
be71787b
MD
924SYSCTL_INT(_machdep, OID_AUTO, cpu_idle_repeat, CTLFLAG_RW,
925 &cpu_idle_repeat, 0, "Idle entries before acpi hlt");
e130cdcb
MD
926
927static void
928cpu_idle_default_hook(void)
929{
930 /*
931 * We must guarentee that hlt is exactly the instruction
932 * following the sti.
933 */
934 __asm __volatile("sti; hlt");
935}
936
937/* Other subsystems (e.g., ACPI) can hook this later. */
938void (*cpu_idle_hook)(void) = cpu_idle_default_hook;
939
940void
941cpu_idle(void)
942{
0f0466c0 943 globaldata_t gd = mycpu;
86232a57 944 struct thread *td __debugvar = gd->gd_curthread;
0f0466c0 945 int reqflags;
be71787b 946 int quick;
e130cdcb
MD
947
948 crit_exit();
949 KKASSERT(td->td_critcount == 0);
950 for (;;) {
951 /*
952 * See if there are any LWKTs ready to go.
953 */
954 lwkt_switch();
955
956 /*
be71787b
MD
957 * When halting inside a cli we must check for reqflags
958 * races, particularly [re]schedule requests. Running
959 * splz() does the job.
960 *
961 * cpu_idle_hlt:
962 * 0 Never halt, just spin
963 *
964 * 1 Always use HLT (or MONITOR/MWAIT if avail).
965 * This typically eats more power than the
966 * ACPI halt.
967 *
968 * 2 Use HLT/MONITOR/MWAIT up to a point and then
969 * use the ACPI halt (default). This is a hybrid
970 * approach. See machdep.cpu_idle_repeat.
971 *
972 * 3 Always use the ACPI halt. This typically
973 * eats the least amount of power but the cpu
974 * will be slow waking up. Slows down e.g.
975 * compiles and other pipe/event oriented stuff.
976 *
977 * NOTE: Interrupts are enabled and we are not in a critical
978 * section.
979 *
980 * NOTE: Preemptions do not reset gd_idle_repeat. Also we
981 * don't bother capping gd_idle_repeat, it is ok if
982 * it overflows.
e130cdcb 983 */
be71787b 984 ++gd->gd_idle_repeat;
0f0466c0 985 reqflags = gd->gd_reqflags;
be71787b
MD
986 quick = (cpu_idle_hlt == 1) ||
987 (cpu_idle_hlt < 3 &&
988 gd->gd_idle_repeat < cpu_idle_repeat);
989
990 if (quick && (cpu_mi_feature & CPU_MI_MONITOR) &&
0f0466c0
MD
991 (reqflags & RQF_IDLECHECK_WK_MASK) == 0) {
992 cpu_mmw_pause_int(&gd->gd_reqflags, reqflags);
be71787b 993 ++cpu_idle_hltcnt;
0f0466c0 994 } else if (cpu_idle_hlt) {
e130cdcb
MD
995 __asm __volatile("cli");
996 splz();
0f0466c0 997 if ((gd->gd_reqflags & RQF_IDLECHECK_WK_MASK) == 0) {
be71787b 998 if (quick)
e130cdcb
MD
999 cpu_idle_default_hook();
1000 else
1001 cpu_idle_hook();
1002 }
1003 __asm __volatile("sti");
1004 ++cpu_idle_hltcnt;
1005 } else {
1006 splz();
1007 __asm __volatile("sti");
1008 ++cpu_idle_spincnt;
1009 }
1010 }
1011}
1012
1013#ifdef SMP
1014
1015/*
1016 * This routine is called if a spinlock has been held through the
1017 * exponential backoff period and is seriously contested. On a real cpu
1018 * we let it spin.
1019 */
1020void
1021cpu_spinlock_contested(void)
1022{
1023 cpu_pause();
1024}
1025
1026#endif
1027
1028/*
1029 * Clear registers on exec
1030 */
1031void
1032exec_setregs(u_long entry, u_long stack, u_long ps_strings)
1033{
1034 struct thread *td = curthread;
1035 struct lwp *lp = td->td_lwp;
1036 struct pcb *pcb = td->td_pcb;
1037 struct trapframe *regs = lp->lwp_md.md_regs;
1038
1039 /* was i386_user_cleanup() in NetBSD */
1040 user_ldt_free(pcb);
1041
1042 bzero((char *)regs, sizeof(struct trapframe));
1043 regs->tf_rip = entry;
1044 regs->tf_rsp = ((stack - 8) & ~0xFul) + 8; /* align the stack */
1045 regs->tf_rdi = stack; /* argv */
1046 regs->tf_rflags = PSL_USER | (regs->tf_rflags & PSL_T);
1047 regs->tf_ss = _udatasel;
1048 regs->tf_cs = _ucodesel;
1049 regs->tf_rbx = ps_strings;
1050
1051 /*
1052 * Reset the hardware debug registers if they were in use.
1053 * They won't have any meaning for the newly exec'd process.
1054 */
1055 if (pcb->pcb_flags & PCB_DBREGS) {
1056 pcb->pcb_dr0 = 0;
1057 pcb->pcb_dr1 = 0;
1058 pcb->pcb_dr2 = 0;
1059 pcb->pcb_dr3 = 0;
1060 pcb->pcb_dr6 = 0;
1061 pcb->pcb_dr7 = 0; /* JG set bit 10? */
1062 if (pcb == td->td_pcb) {
1063 /*
1064 * Clear the debug registers on the running
1065 * CPU, otherwise they will end up affecting
1066 * the next process we switch to.
1067 */
1068 reset_dbregs();
1069 }
1070 pcb->pcb_flags &= ~PCB_DBREGS;
1071 }
1072
1073 /*
1074 * Initialize the math emulator (if any) for the current process.
1075 * Actually, just clear the bit that says that the emulator has
1076 * been initialized. Initialization is delayed until the process
1077 * traps to the emulator (if it is done at all) mainly because
1078 * emulators don't provide an entry point for initialization.
1079 */
1080 pcb->pcb_flags &= ~FP_SOFTFP;
1081
1082 /*
1083 * NOTE: do not set CR0_TS here. npxinit() must do it after clearing
1084 * gd_npxthread. Otherwise a preemptive interrupt thread
1085 * may panic in npxdna().
1086 */
1087 crit_enter();
1088 load_cr0(rcr0() | CR0_MP);
1089
1090 /*
1091 * NOTE: The MSR values must be correct so we can return to
1092 * userland. gd_user_fs/gs must be correct so the switch
1093 * code knows what the current MSR values are.
1094 */
1095 pcb->pcb_fsbase = 0; /* Values loaded from PCB on switch */
1096 pcb->pcb_gsbase = 0;
1097 mdcpu->gd_user_fs = 0; /* Cache of current MSR values */
1098 mdcpu->gd_user_gs = 0;
1099 wrmsr(MSR_FSBASE, 0); /* Set MSR values for return to userland */
1100 wrmsr(MSR_KGSBASE, 0);
1101
1102 /* Initialize the npx (if any) for the current process. */
1103 npxinit(__INITIAL_NPXCW__);
1104 crit_exit();
1105
1106 pcb->pcb_ds = _udatasel;
1107 pcb->pcb_es = _udatasel;
1108 pcb->pcb_fs = _udatasel;
1109 pcb->pcb_gs = _udatasel;
1110}
1111
1112void
1113cpu_setregs(void)
1114{
1115 register_t cr0;
1116
1117 cr0 = rcr0();
1118 cr0 |= CR0_NE; /* Done by npxinit() */
1119 cr0 |= CR0_MP | CR0_TS; /* Done at every execve() too. */
1120 cr0 |= CR0_WP | CR0_AM;
1121 load_cr0(cr0);
1122 load_gs(_udatasel);
1123}
1124
1125static int
1126sysctl_machdep_adjkerntz(SYSCTL_HANDLER_ARGS)
1127{
1128 int error;
1129 error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2,
1130 req);
1131 if (!error && req->newptr)
1132 resettodr();
1133 return (error);
1134}
1135
1136SYSCTL_PROC(_machdep, CPU_ADJKERNTZ, adjkerntz, CTLTYPE_INT|CTLFLAG_RW,
1137 &adjkerntz, 0, sysctl_machdep_adjkerntz, "I", "");
1138
1139SYSCTL_INT(_machdep, CPU_DISRTCSET, disable_rtc_set,
1140 CTLFLAG_RW, &disable_rtc_set, 0, "");
1141
1142#if JG
1143SYSCTL_STRUCT(_machdep, CPU_BOOTINFO, bootinfo,
1144 CTLFLAG_RD, &bootinfo, bootinfo, "");
1145#endif
1146
1147SYSCTL_INT(_machdep, CPU_WALLCLOCK, wall_cmos_clock,
1148 CTLFLAG_RW, &wall_cmos_clock, 0, "");
1149
1150extern u_long bootdev; /* not a cdev_t - encoding is different */
1151SYSCTL_ULONG(_machdep, OID_AUTO, guessed_bootdev,
1152 CTLFLAG_RD, &bootdev, 0, "Boot device (not in cdev_t format)");
1153
1154/*
1155 * Initialize 386 and configure to run kernel
1156 */
1157
1158/*
1159 * Initialize segments & interrupt table
1160 */
1161
1162int _default_ldt;
1163struct user_segment_descriptor gdt[NGDT * MAXCPU]; /* global descriptor table */
1164static struct gate_descriptor idt0[NIDT];
1165struct gate_descriptor *idt = &idt0[0]; /* interrupt descriptor table */
1166#if JG
1167union descriptor ldt[NLDT]; /* local descriptor table */
1168#endif
1169
1170/* table descriptors - used to load tables by cpu */
1171struct region_descriptor r_gdt, r_idt;
1172
1173#if defined(I586_CPU) && !defined(NO_F00F_HACK)
1174extern int has_f00f_bug;
1175#endif
1176
1177/* JG proc0paddr is a virtual address */
1178void *proc0paddr;
1179/* JG alignment? */
1180char proc0paddr_buff[LWKT_THREAD_STACK];
1181
1182
1183/* software prototypes -- in more palatable form */
1184struct soft_segment_descriptor gdt_segs[] = {
1185/* GNULL_SEL 0 Null Descriptor */
1186{ 0x0, /* segment base address */
1187 0x0, /* length */
1188 0, /* segment type */
1189 0, /* segment descriptor priority level */
1190 0, /* segment descriptor present */
1191 0, /* long */
1192 0, /* default 32 vs 16 bit size */
1193 0 /* limit granularity (byte/page units)*/ },
1194/* GCODE_SEL 1 Code Descriptor for kernel */
1195{ 0x0, /* segment base address */
1196 0xfffff, /* length - all address space */
1197 SDT_MEMERA, /* segment type */
1198 SEL_KPL, /* segment descriptor priority level */
1199 1, /* segment descriptor present */
1200 1, /* long */
1201 0, /* default 32 vs 16 bit size */
1202 1 /* limit granularity (byte/page units)*/ },
1203/* GDATA_SEL 2 Data Descriptor for kernel */
1204{ 0x0, /* segment base address */
1205 0xfffff, /* length - all address space */
1206 SDT_MEMRWA, /* segment type */
1207 SEL_KPL, /* segment descriptor priority level */
1208 1, /* segment descriptor present */
1209 1, /* long */
1210 0, /* default 32 vs 16 bit size */
1211 1 /* limit granularity (byte/page units)*/ },
1212/* GUCODE32_SEL 3 32 bit Code Descriptor for user */
1213{ 0x0, /* segment base address */
1214 0xfffff, /* length - all address space */
1215 SDT_MEMERA, /* segment type */
1216 SEL_UPL, /* segment descriptor priority level */
1217 1, /* segment descriptor present */
1218 0, /* long */
1219 1, /* default 32 vs 16 bit size */
1220 1 /* limit granularity (byte/page units)*/ },
1221/* GUDATA_SEL 4 32/64 bit Data Descriptor for user */
1222{ 0x0, /* segment base address */
1223 0xfffff, /* length - all address space */
1224 SDT_MEMRWA, /* segment type */
1225 SEL_UPL, /* segment descriptor priority level */
1226 1, /* segment descriptor present */
1227 0, /* long */
1228 1, /* default 32 vs 16 bit size */
1229 1 /* limit granularity (byte/page units)*/ },
1230/* GUCODE_SEL 5 64 bit Code Descriptor for user */
1231{ 0x0, /* segment base address */
1232 0xfffff, /* length - all address space */
1233 SDT_MEMERA, /* segment type */
1234 SEL_UPL, /* segment descriptor priority level */
1235 1, /* segment descriptor present */
1236 1, /* long */
1237 0, /* default 32 vs 16 bit size */
1238 1 /* limit granularity (byte/page units)*/ },
1239/* GPROC0_SEL 6 Proc 0 Tss Descriptor */
1240{
1241 0x0, /* segment base address */
1242 sizeof(struct x86_64tss)-1,/* length - all address space */
1243 SDT_SYSTSS, /* segment type */
1244 SEL_KPL, /* segment descriptor priority level */
1245 1, /* segment descriptor present */
1246 0, /* long */
1247 0, /* unused - default 32 vs 16 bit size */
1248 0 /* limit granularity (byte/page units)*/ },
1249/* Actually, the TSS is a system descriptor which is double size */
1250{ 0x0, /* segment base address */
1251 0x0, /* length */
1252 0, /* segment type */
1253 0, /* segment descriptor priority level */
1254 0, /* segment descriptor present */
1255 0, /* long */
1256 0, /* default 32 vs 16 bit size */
1257 0 /* limit granularity (byte/page units)*/ },
1258/* GUGS32_SEL 8 32 bit GS Descriptor for user */
1259{ 0x0, /* segment base address */
1260 0xfffff, /* length - all address space */
1261 SDT_MEMRWA, /* segment type */
1262 SEL_UPL, /* segment descriptor priority level */
1263 1, /* segment descriptor present */
1264 0, /* long */
1265 1, /* default 32 vs 16 bit size */
1266 1 /* limit granularity (byte/page units)*/ },
1267};
1268
1269void
1270setidt(int idx, inthand_t *func, int typ, int dpl, int ist)
1271{
1272 struct gate_descriptor *ip;
1273
1274 ip = idt + idx;
1275 ip->gd_looffset = (uintptr_t)func;
1276 ip->gd_selector = GSEL(GCODE_SEL, SEL_KPL);
1277 ip->gd_ist = ist;
1278 ip->gd_xx = 0;
1279 ip->gd_type = typ;
1280 ip->gd_dpl = dpl;
1281 ip->gd_p = 1;
1282 ip->gd_hioffset = ((uintptr_t)func)>>16 ;
1283}
1284
1285#define IDTVEC(name) __CONCAT(X,name)
1286
1287extern inthand_t
1288 IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl),
1289 IDTVEC(bnd), IDTVEC(ill), IDTVEC(dna), IDTVEC(fpusegm),
1290 IDTVEC(tss), IDTVEC(missing), IDTVEC(stk), IDTVEC(prot),
1291 IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align),
1292 IDTVEC(xmm), IDTVEC(dblfault),
1293 IDTVEC(fast_syscall), IDTVEC(fast_syscall32);
1294
1295#ifdef DEBUG_INTERRUPTS
1296extern inthand_t *Xrsvdary[256];
1297#endif
1298
1299void
1300sdtossd(struct user_segment_descriptor *sd, struct soft_segment_descriptor *ssd)
1301{
1302 ssd->ssd_base = (sd->sd_hibase << 24) | sd->sd_lobase;
1303 ssd->ssd_limit = (sd->sd_hilimit << 16) | sd->sd_lolimit;
1304 ssd->ssd_type = sd->sd_type;
1305 ssd->ssd_dpl = sd->sd_dpl;
1306 ssd->ssd_p = sd->sd_p;
1307 ssd->ssd_def32 = sd->sd_def32;
1308 ssd->ssd_gran = sd->sd_gran;
1309}
1310
1311void
1312ssdtosd(struct soft_segment_descriptor *ssd, struct user_segment_descriptor *sd)
1313{
1314
1315 sd->sd_lobase = (ssd->ssd_base) & 0xffffff;
1316 sd->sd_hibase = (ssd->ssd_base >> 24) & 0xff;
1317 sd->sd_lolimit = (ssd->ssd_limit) & 0xffff;
1318 sd->sd_hilimit = (ssd->ssd_limit >> 16) & 0xf;
1319 sd->sd_type = ssd->ssd_type;
1320 sd->sd_dpl = ssd->ssd_dpl;
1321 sd->sd_p = ssd->ssd_p;
1322 sd->sd_long = ssd->ssd_long;
1323 sd->sd_def32 = ssd->ssd_def32;
1324 sd->sd_gran = ssd->ssd_gran;
1325}
1326
1327void
1328ssdtosyssd(struct soft_segment_descriptor *ssd,
1329 struct system_segment_descriptor *sd)
1330{
1331
1332 sd->sd_lobase = (ssd->ssd_base) & 0xffffff;
1333 sd->sd_hibase = (ssd->ssd_base >> 24) & 0xfffffffffful;
1334 sd->sd_lolimit = (ssd->ssd_limit) & 0xffff;
1335 sd->sd_hilimit = (ssd->ssd_limit >> 16) & 0xf;
1336 sd->sd_type = ssd->ssd_type;
1337 sd->sd_dpl = ssd->ssd_dpl;
1338 sd->sd_p = ssd->ssd_p;
1339 sd->sd_gran = ssd->ssd_gran;
1340}
1341
1342u_int basemem;
1343
1344/*
1345 * Populate the (physmap) array with base/bound pairs describing the
1346 * available physical memory in the system, then test this memory and
1347 * build the phys_avail array describing the actually-available memory.
1348 *
1349 * If we cannot accurately determine the physical memory map, then use
1350 * value from the 0xE801 call, and failing that, the RTC.
1351 *
1352 * Total memory size may be set by the kernel environment variable
1353 * hw.physmem or the compile-time define MAXMEM.
1354 *
1355 * XXX first should be vm_paddr_t.
1356 */
1357static void
1358getmemsize(caddr_t kmdp, u_int64_t first)
1359{
1360 int i, off, physmap_idx, pa_indx, da_indx;
1361 vm_paddr_t pa, physmap[PHYSMAP_SIZE];
1362 u_long physmem_tunable;
1363 pt_entry_t *pte;
1364 struct bios_smap *smapbase, *smap, *smapend;
1365 u_int32_t smapsize;
1366 quad_t dcons_addr, dcons_size;
1367
1368 bzero(physmap, sizeof(physmap));
1369 basemem = 0;
1370 physmap_idx = 0;
1371
1372 /*
1373 * get memory map from INT 15:E820, kindly supplied by the loader.
1374 *
1375 * subr_module.c says:
1376 * "Consumer may safely assume that size value precedes data."
1377 * ie: an int32_t immediately precedes smap.
1378 */
1379 smapbase = (struct bios_smap *)preload_search_info(kmdp,
1380 MODINFO_METADATA | MODINFOMD_SMAP);
1381 if (smapbase == NULL)
1382 panic("No BIOS smap info from loader!");
1383
1384 smapsize = *((u_int32_t *)smapbase - 1);
1385 smapend = (struct bios_smap *)((uintptr_t)smapbase + smapsize);
1386
1387 for (smap = smapbase; smap < smapend; smap++) {
1388 if (boothowto & RB_VERBOSE)
1389 kprintf("SMAP type=%02x base=%016lx len=%016lx\n",
1390 smap->type, smap->base, smap->length);
1391
1392 if (smap->type != SMAP_TYPE_MEMORY)
1393 continue;
1394
1395 if (smap->length == 0)
1396 continue;
1397
1398 for (i = 0; i <= physmap_idx; i += 2) {
1399 if (smap->base < physmap[i + 1]) {
1400 if (boothowto & RB_VERBOSE) {
1401 kprintf("Overlapping or non-monotonic "
1402 "memory region, ignoring "
1403 "second region\n");
1404 }
1405 continue;
1406 }
1407 }
1408 Realmem += smap->length;
1409
1410 if (smap->base == physmap[physmap_idx + 1]) {
1411 physmap[physmap_idx + 1] += smap->length;
1412 continue;
1413 }
1414
1415 physmap_idx += 2;
1416 if (physmap_idx == PHYSMAP_SIZE) {
1417 kprintf("Too many segments in the physical "
1418 "address map, giving up\n");
1419 break;
1420 }
1421 physmap[physmap_idx] = smap->base;
1422 physmap[physmap_idx + 1] = smap->base + smap->length;
1423 }
1424
1425 /*
1426 * Find the 'base memory' segment for SMP
1427 */
1428 basemem = 0;
1429 for (i = 0; i <= physmap_idx; i += 2) {
1430 if (physmap[i] == 0x00000000) {
1431 basemem = physmap[i + 1] / 1024;
1432 break;
1433 }
1434 }
1435 if (basemem == 0)
1436 panic("BIOS smap did not include a basemem segment!");
1437
1438#ifdef SMP
1439 /* make hole for AP bootstrap code */
1440 physmap[1] = mp_bootaddress(physmap[1] / 1024);
1441
1442 /* Save EBDA address, if any */
1443 ebda_addr = (u_long)(*(u_short *)(KERNBASE + 0x40e));
1444 ebda_addr <<= 4;
1445#endif
1446
1447 /*
1448 * Maxmem isn't the "maximum memory", it's one larger than the
1449 * highest page of the physical address space. It should be
1450 * called something like "Maxphyspage". We may adjust this
1451 * based on ``hw.physmem'' and the results of the memory test.
1452 */
1453 Maxmem = atop(physmap[physmap_idx + 1]);
1454
1455#ifdef MAXMEM
1456 Maxmem = MAXMEM / 4;
1457#endif
1458
1459 if (TUNABLE_ULONG_FETCH("hw.physmem", &physmem_tunable))
1460 Maxmem = atop(physmem_tunable);
1461
1462 /*
1463 * Don't allow MAXMEM or hw.physmem to extend the amount of memory
1464 * in the system.
1465 */
1466 if (Maxmem > atop(physmap[physmap_idx + 1]))
1467 Maxmem = atop(physmap[physmap_idx + 1]);
1468
1469 /*
1470 *
1471 */
1472 if (Maxmem > atop(DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS)) {
1473 kprintf("Limiting Maxmem due to DMAP size\n");
1474 Maxmem = atop(DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS);
1475 }
1476
1477 if (atop(physmap[physmap_idx + 1]) != Maxmem &&
1478 (boothowto & RB_VERBOSE))
1479 kprintf("Physical memory use set to %ldK\n", Maxmem * 4);
1480
1481 /* call pmap initialization to make new kernel address space */
1482 pmap_bootstrap(&first);
1483
1484 /*
1485 * Size up each available chunk of physical memory.
1486 */
1487 physmap[0] = PAGE_SIZE; /* mask off page 0 */
1488 pa_indx = 0;
1489 da_indx = 1;
1490 phys_avail[pa_indx++] = physmap[0];
1491 phys_avail[pa_indx] = physmap[0];
1492 dump_avail[da_indx] = physmap[0];
1493 pte = CMAP1;
1494
1495 /*
1496 * Get dcons buffer address
1497 */
1498 if (kgetenv_quad("dcons.addr", &dcons_addr) == 0 ||
1499 kgetenv_quad("dcons.size", &dcons_size) == 0)
1500 dcons_addr = 0;
1501
1502 /*
1503 * physmap is in bytes, so when converting to page boundaries,
1504 * round up the start address and round down the end address.
1505 */
1506 for (i = 0; i <= physmap_idx; i += 2) {
1507 vm_paddr_t end;
1508
1509 end = ptoa((vm_paddr_t)Maxmem);
1510 if (physmap[i + 1] < end)
1511 end = trunc_page(physmap[i + 1]);
1512 for (pa = round_page(physmap[i]); pa < end; pa += PAGE_SIZE) {
1513 int tmp, page_bad, full;
1514 int *ptr = (int *)CADDR1;
1515
1516 full = FALSE;
1517 /*
1518 * block out kernel memory as not available.
1519 */
1520 if (pa >= 0x100000 && pa < first)
1521 goto do_dump_avail;
1522
1523 /*
1524 * block out dcons buffer
1525 */
1526 if (dcons_addr > 0
1527 && pa >= trunc_page(dcons_addr)
1528 && pa < dcons_addr + dcons_size)
1529 goto do_dump_avail;
1530
1531 page_bad = FALSE;
1532
1533 /*
1534 * map page into kernel: valid, read/write,non-cacheable
1535 */
1536 *pte = pa | PG_V | PG_RW | PG_N;
1537 cpu_invltlb();
1538
1539 tmp = *(int *)ptr;
1540 /*
1541 * Test for alternating 1's and 0's
1542 */
1543 *(volatile int *)ptr = 0xaaaaaaaa;
1544 if (*(volatile int *)ptr != 0xaaaaaaaa)
1545 page_bad = TRUE;
1546 /*
1547 * Test for alternating 0's and 1's
1548 */
1549 *(volatile int *)ptr = 0x55555555;
1550 if (*(volatile int *)ptr != 0x55555555)
1551 page_bad = TRUE;
1552 /*
1553 * Test for all 1's
1554 */
1555 *(volatile int *)ptr = 0xffffffff;
1556 if (*(volatile int *)ptr != 0xffffffff)
1557 page_bad = TRUE;
1558 /*
1559 * Test for all 0's
1560 */
1561 *(volatile int *)ptr = 0x0;
1562 if (*(volatile int *)ptr != 0x0)
1563 page_bad = TRUE;
1564 /*
1565 * Restore original value.
1566 */
1567 *(int *)ptr = tmp;
1568
1569 /*
1570 * Adjust array of valid/good pages.
1571 */
1572 if (page_bad == TRUE)
1573 continue;
1574 /*
1575 * If this good page is a continuation of the
1576 * previous set of good pages, then just increase
1577 * the end pointer. Otherwise start a new chunk.
1578 * Note that "end" points one higher than end,
1579 * making the range >= start and < end.
1580 * If we're also doing a speculative memory
1581 * test and we at or past the end, bump up Maxmem
1582 * so that we keep going. The first bad page
1583 * will terminate the loop.
1584 */
1585 if (phys_avail[pa_indx] == pa) {
1586 phys_avail[pa_indx] += PAGE_SIZE;
1587 } else {
1588 pa_indx++;
1589 if (pa_indx == PHYS_AVAIL_ARRAY_END) {
1590 kprintf(
1591 "Too many holes in the physical address space, giving up\n");
1592 pa_indx--;
1593 full = TRUE;
1594 goto do_dump_avail;
1595 }
1596 phys_avail[pa_indx++] = pa; /* start */
1597 phys_avail[pa_indx] = pa + PAGE_SIZE; /* end */
1598 }
1599 physmem++;
1600do_dump_avail:
1601 if (dump_avail[da_indx] == pa) {
1602 dump_avail[da_indx] += PAGE_SIZE;
1603 } else {
1604 da_indx++;
1605 if (da_indx == DUMP_AVAIL_ARRAY_END) {
1606 da_indx--;
1607 goto do_next;
1608 }
1609 dump_avail[da_indx++] = pa; /* start */
1610 dump_avail[da_indx] = pa + PAGE_SIZE; /* end */
1611 }
1612do_next:
1613 if (full)
1614 break;
1615 }
1616 }
1617 *pte = 0;
1618 cpu_invltlb();
1619
1620 /*
1621 * XXX
1622 * The last chunk must contain at least one page plus the message
1623 * buffer to avoid complicating other code (message buffer address
1624 * calculation, etc.).
1625 */
1626 while (phys_avail[pa_indx - 1] + PAGE_SIZE +
1627 round_page(MSGBUF_SIZE) >= phys_avail[pa_indx]) {
1628 physmem -= atop(phys_avail[pa_indx] - phys_avail[pa_indx - 1]);
1629 phys_avail[pa_indx--] = 0;
1630 phys_avail[pa_indx--] = 0;
1631 }
1632
1633 Maxmem = atop(phys_avail[pa_indx]);
1634
1635 /* Trim off space for the message buffer. */
1636 phys_avail[pa_indx] -= round_page(MSGBUF_SIZE);
1637
1638 avail_end = phys_avail[pa_indx];
1639
1640 /* Map the message buffer. */
1641 for (off = 0; off < round_page(MSGBUF_SIZE); off += PAGE_SIZE)
1642 pmap_kenter((vm_offset_t)msgbufp + off, phys_avail[pa_indx] +
1643 off);
1644}
1645
1646#ifdef SMP
1647#ifdef APIC_IO
1648int apic_io_enable = 1; /* Enabled by default for kernels compiled w/APIC_IO */
1649#else
1650int apic_io_enable = 0; /* Disabled by default for kernels compiled without */
1651#endif
1652TUNABLE_INT("hw.apic_io_enable", &apic_io_enable);
1653extern struct machintr_abi MachIntrABI_APIC;
1654#endif
1655
1656extern struct machintr_abi MachIntrABI_ICU;
1657struct machintr_abi MachIntrABI;
1658
1659/*
1660 * IDT VECTORS:
1661 * 0 Divide by zero
1662 * 1 Debug
1663 * 2 NMI
1664 * 3 BreakPoint
1665 * 4 OverFlow
1666 * 5 Bound-Range
1667 * 6 Invalid OpCode
1668 * 7 Device Not Available (x87)
1669 * 8 Double-Fault
1670 * 9 Coprocessor Segment overrun (unsupported, reserved)
1671 * 10 Invalid-TSS
1672 * 11 Segment not present
1673 * 12 Stack
1674 * 13 General Protection
1675 * 14 Page Fault
1676 * 15 Reserved
1677 * 16 x87 FP Exception pending
1678 * 17 Alignment Check
1679 * 18 Machine Check
1680 * 19 SIMD floating point
1681 * 20-31 reserved
1682 * 32-255 INTn/external sources
1683 */
1684u_int64_t
1685hammer_time(u_int64_t modulep, u_int64_t physfree)
1686{
1687 caddr_t kmdp;
1688 int gsel_tss, x;
1689#if JG
1690 int metadata_missing, off;
1691#endif
1692 struct mdglobaldata *gd;
1693 u_int64_t msr;
1694
1695 /*
1696 * Prevent lowering of the ipl if we call tsleep() early.
1697 */
1698 gd = &CPU_prvspace[0].mdglobaldata;
1699 bzero(gd, sizeof(*gd));
1700
1701 /*
1702 * Note: on both UP and SMP curthread must be set non-NULL
1703 * early in the boot sequence because the system assumes
1704 * that 'curthread' is never NULL.
1705 */
1706
1707 gd->mi.gd_curthread = &thread0;
1708 thread0.td_gd = &gd->mi;
1709
1710 atdevbase = ISA_HOLE_START + PTOV_OFFSET;
1711
1712#if JG
1713 metadata_missing = 0;
1714 if (bootinfo.bi_modulep) {
1715 preload_metadata = (caddr_t)bootinfo.bi_modulep + KERNBASE;
1716 preload_bootstrap_relocate(KERNBASE);
1717 } else {
1718 metadata_missing = 1;
1719 }
1720 if (bootinfo.bi_envp)
1721 kern_envp = (caddr_t)bootinfo.bi_envp + KERNBASE;
1722#endif
1723
1724 preload_metadata = (caddr_t)(uintptr_t)(modulep + PTOV_OFFSET);
1725 preload_bootstrap_relocate(PTOV_OFFSET);
1726 kmdp = preload_search_by_type("elf kernel");
1727 if (kmdp == NULL)
1728 kmdp = preload_search_by_type("elf64 kernel");
1729 boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
1730 kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *) + PTOV_OFFSET;
1731#ifdef DDB
1732 ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t);
1733 ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t);
1734#endif
1735
1736 /*
10db3cc6 1737 * Default MachIntrABI to ICU
e130cdcb
MD
1738 */
1739 MachIntrABI = MachIntrABI_ICU;
1740#ifdef SMP
1741 TUNABLE_INT_FETCH("hw.apic_io_enable", &apic_io_enable);
e130cdcb
MD
1742#endif
1743
1744 /*
1745 * start with one cpu. Note: with one cpu, ncpus2_shift, ncpus2_mask,
1746 * and ncpus_fit_mask remain 0.
1747 */
1748 ncpus = 1;
1749 ncpus2 = 1;
1750 ncpus_fit = 1;
1751 /* Init basic tunables, hz etc */
1752 init_param1();
1753
1754 /*
1755 * make gdt memory segments
1756 */
1757 gdt_segs[GPROC0_SEL].ssd_base =
1758 (uintptr_t) &CPU_prvspace[0].mdglobaldata.gd_common_tss;
1759
1760 gd->mi.gd_prvspace = &CPU_prvspace[0];
1761
1762 for (x = 0; x < NGDT; x++) {
1763 if (x != GPROC0_SEL && x != (GPROC0_SEL + 1))
1764 ssdtosd(&gdt_segs[x], &gdt[x]);
1765 }
1766 ssdtosyssd(&gdt_segs[GPROC0_SEL],
1767 (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
1768
1769 r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
1770 r_gdt.rd_base = (long) gdt;
1771 lgdt(&r_gdt);
1772
1773 wrmsr(MSR_FSBASE, 0); /* User value */
1774 wrmsr(MSR_GSBASE, (u_int64_t)&gd->mi);
1775 wrmsr(MSR_KGSBASE, 0); /* User value while in the kernel */
1776
1777 mi_gdinit(&gd->mi, 0);
1778 cpu_gdinit(gd, 0);
1779 proc0paddr = proc0paddr_buff;
1780 mi_proc0init(&gd->mi, proc0paddr);
1781 safepri = TDPRI_MAX;
1782
1783 /* spinlocks and the BGL */
1784 init_locks();
1785
1786 /* exceptions */
1787 for (x = 0; x < NIDT; x++)
1788 setidt(x, &IDTVEC(rsvd), SDT_SYSIGT, SEL_KPL, 0);
1789 setidt(IDT_DE, &IDTVEC(div), SDT_SYSIGT, SEL_KPL, 0);
1790 setidt(IDT_DB, &IDTVEC(dbg), SDT_SYSIGT, SEL_KPL, 0);
1791 setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 1);
1792 setidt(IDT_BP, &IDTVEC(bpt), SDT_SYSIGT, SEL_UPL, 0);
1793 setidt(IDT_OF, &IDTVEC(ofl), SDT_SYSIGT, SEL_KPL, 0);
1794 setidt(IDT_BR, &IDTVEC(bnd), SDT_SYSIGT, SEL_KPL, 0);
1795 setidt(IDT_UD, &IDTVEC(ill), SDT_SYSIGT, SEL_KPL, 0);
1796 setidt(IDT_NM, &IDTVEC(dna), SDT_SYSIGT, SEL_KPL, 0);
1797 setidt(IDT_DF, &IDTVEC(dblfault), SDT_SYSIGT, SEL_KPL, 1);
1798 setidt(IDT_FPUGP, &IDTVEC(fpusegm), SDT_SYSIGT, SEL_KPL, 0);
1799 setidt(IDT_TS, &IDTVEC(tss), SDT_SYSIGT, SEL_KPL, 0);
1800 setidt(IDT_NP, &IDTVEC(missing), SDT_SYSIGT, SEL_KPL, 0);
1801 setidt(IDT_SS, &IDTVEC(stk), SDT_SYSIGT, SEL_KPL, 0);
1802 setidt(IDT_GP, &IDTVEC(prot), SDT_SYSIGT, SEL_KPL, 0);
1803 setidt(IDT_PF, &IDTVEC(page), SDT_SYSIGT, SEL_KPL, 0);
1804 setidt(IDT_MF, &IDTVEC(fpu), SDT_SYSIGT, SEL_KPL, 0);
1805 setidt(IDT_AC, &IDTVEC(align), SDT_SYSIGT, SEL_KPL, 0);
1806 setidt(IDT_MC, &IDTVEC(mchk), SDT_SYSIGT, SEL_KPL, 0);
1807 setidt(IDT_XF, &IDTVEC(xmm), SDT_SYSIGT, SEL_KPL, 0);
1808
1809 r_idt.rd_limit = sizeof(idt0) - 1;
1810 r_idt.rd_base = (long) idt;
1811 lidt(&r_idt);
1812
1813 /*
1814 * Initialize the console before we print anything out.
1815 */
1816 cninit();
1817
1818#if JG
1819 if (metadata_missing)
1820 kprintf("WARNING: loader(8) metadata is missing!\n");
1821#endif
1822
1823#if NISA >0
e24dd6e0 1824 elcr_probe();
e130cdcb
MD
1825 isa_defaultirq();
1826#endif
1827 rand_initialize();
1828
1829#ifdef DDB
1830 kdb_init();
1831 if (boothowto & RB_KDB)
1832 Debugger("Boot flags requested debugger");
1833#endif
1834
1835#if JG
1836 finishidentcpu(); /* Final stage of CPU initialization */
1837 setidt(6, &IDTVEC(ill), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
1838 setidt(13, &IDTVEC(prot), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
1839#endif
1840 identify_cpu(); /* Final stage of CPU initialization */
1841 initializecpu(); /* Initialize CPU registers */
1842
1843 /* make an initial tss so cpu can get interrupt stack on syscall! */
1844 gd->gd_common_tss.tss_rsp0 =
1845 (register_t)(thread0.td_kstack +
1846 KSTACK_PAGES * PAGE_SIZE - sizeof(struct pcb));
1847 /* Ensure the stack is aligned to 16 bytes */
1848 gd->gd_common_tss.tss_rsp0 &= ~(register_t)0xF;
1849
1850 /* double fault stack */
1851 gd->gd_common_tss.tss_ist1 =
1852 (long)&gd->mi.gd_prvspace->idlestack[
1853 sizeof(gd->mi.gd_prvspace->idlestack)];
1854
1855 /* Set the IO permission bitmap (empty due to tss seg limit) */
1856 gd->gd_common_tss.tss_iobase = sizeof(struct x86_64tss);
1857
1858 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
1859 gd->gd_tss_gdt = &gdt[GPROC0_SEL];
1860 gd->gd_common_tssd = *gd->gd_tss_gdt;
1861 ltr(gsel_tss);
1862
1863 /* Set up the fast syscall stuff */
1864 msr = rdmsr(MSR_EFER) | EFER_SCE;
1865 wrmsr(MSR_EFER, msr);
1866 wrmsr(MSR_LSTAR, (u_int64_t)IDTVEC(fast_syscall));
1867 wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
1868 msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
1869 ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
1870 wrmsr(MSR_STAR, msr);
1871 wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D);
1872
1873 getmemsize(kmdp, physfree);
1874 init_param2(physmem);
1875
1876 /* now running on new page tables, configured,and u/iom is accessible */
1877
1878 /* Map the message buffer. */
1879#if JG
1880 for (off = 0; off < round_page(MSGBUF_SIZE); off += PAGE_SIZE)
1881 pmap_kenter((vm_offset_t)msgbufp + off, avail_end + off);
1882#endif
1883
1884 msgbufinit(msgbufp, MSGBUF_SIZE);
1885
1886
1887 /* transfer to user mode */
1888
1889 _ucodesel = GSEL(GUCODE_SEL, SEL_UPL);
1890 _udatasel = GSEL(GUDATA_SEL, SEL_UPL);
1891 _ucode32sel = GSEL(GUCODE32_SEL, SEL_UPL);
1892
1893 load_ds(_udatasel);
1894 load_es(_udatasel);
1895 load_fs(_udatasel);
1896
1897 /* setup proc 0's pcb */
1898 thread0.td_pcb->pcb_flags = 0;
1899 thread0.td_pcb->pcb_cr3 = KPML4phys;
1900 thread0.td_pcb->pcb_ext = 0;
1901 lwp0.lwp_md.md_regs = &proc0_tf; /* XXX needed? */
1902
1903 /* Location of kernel stack for locore */
1904 return ((u_int64_t)thread0.td_pcb);
1905}
1906
1907/*
1908 * Initialize machine-dependant portions of the global data structure.
1909 * Note that the global data area and cpu0's idlestack in the private
1910 * data space were allocated in locore.
1911 *
1912 * Note: the idlethread's cpl is 0
1913 *
1914 * WARNING! Called from early boot, 'mycpu' may not work yet.
1915 */
1916void
1917cpu_gdinit(struct mdglobaldata *gd, int cpu)
1918{
1919 if (cpu)
1920 gd->mi.gd_curthread = &gd->mi.gd_idlethread;
1921
1922 lwkt_init_thread(&gd->mi.gd_idlethread,
1923 gd->mi.gd_prvspace->idlestack,
1924 sizeof(gd->mi.gd_prvspace->idlestack),
1925 0, &gd->mi);
1926 lwkt_set_comm(&gd->mi.gd_idlethread, "idle_%d", cpu);
1927 gd->mi.gd_idlethread.td_switch = cpu_lwkt_switch;
1928 gd->mi.gd_idlethread.td_sp -= sizeof(void *);
1929 *(void **)gd->mi.gd_idlethread.td_sp = cpu_idle_restore;
1930}
1931
1932int
1933is_globaldata_space(vm_offset_t saddr, vm_offset_t eaddr)
1934{
1935 if (saddr >= (vm_offset_t)&CPU_prvspace[0] &&
1936 eaddr <= (vm_offset_t)&CPU_prvspace[MAXCPU]) {
1937 return (TRUE);
1938 }
1939 return (FALSE);
1940}
1941
1942struct globaldata *
1943globaldata_find(int cpu)
1944{
1945 KKASSERT(cpu >= 0 && cpu < ncpus);
1946 return(&CPU_prvspace[cpu].mdglobaldata.mi);
1947}
1948
1949#if defined(I586_CPU) && !defined(NO_F00F_HACK)
1950static void f00f_hack(void *unused);
1951SYSINIT(f00f_hack, SI_BOOT2_BIOS, SI_ORDER_ANY, f00f_hack, NULL);
1952
1953static void
1954f00f_hack(void *unused)
1955{
1956 struct gate_descriptor *new_idt;
1957 vm_offset_t tmp;
1958
1959 if (!has_f00f_bug)
1960 return;
1961
1962 kprintf("Intel Pentium detected, installing workaround for F00F bug\n");
1963
1964 r_idt.rd_limit = sizeof(idt0) - 1;
1965
1966 tmp = kmem_alloc(&kernel_map, PAGE_SIZE * 2);
1967 if (tmp == 0)
1968 panic("kmem_alloc returned 0");
1969 if (((unsigned int)tmp & (PAGE_SIZE-1)) != 0)
1970 panic("kmem_alloc returned non-page-aligned memory");
1971 /* Put the first seven entries in the lower page */
1972 new_idt = (struct gate_descriptor*)(tmp + PAGE_SIZE - (7*8));
1973 bcopy(idt, new_idt, sizeof(idt0));
1974 r_idt.rd_base = (int)new_idt;
1975 lidt(&r_idt);
1976 idt = new_idt;
1977 if (vm_map_protect(&kernel_map, tmp, tmp + PAGE_SIZE,
1978 VM_PROT_READ, FALSE) != KERN_SUCCESS)
1979 panic("vm_map_protect failed");
1980 return;
1981}
1982#endif /* defined(I586_CPU) && !NO_F00F_HACK */
1983
1984int
1985ptrace_set_pc(struct lwp *lp, unsigned long addr)
1986{
1987 lp->lwp_md.md_regs->tf_rip = addr;
1988 return (0);
1989}
1990
1991int
1992ptrace_single_step(struct lwp *lp)
1993{
1994 lp->lwp_md.md_regs->tf_rflags |= PSL_T;
1995 return (0);
1996}
1997
1998int
1999fill_regs(struct lwp *lp, struct reg *regs)
2000{
2001 struct trapframe *tp;
2002
2003 tp = lp->lwp_md.md_regs;
2004 bcopy(&tp->tf_rdi, &regs->r_rdi, sizeof(*regs));
2005 return (0);
2006}
2007
2008int
2009set_regs(struct lwp *lp, struct reg *regs)
2010{
2011 struct trapframe *tp;
2012
2013 tp = lp->lwp_md.md_regs;
2014 if (!EFL_SECURE(regs->r_rflags, tp->tf_rflags) ||
2015 !CS_SECURE(regs->r_cs))
2016 return (EINVAL);
2017 bcopy(&regs->r_rdi, &tp->tf_rdi, sizeof(*regs));
2018 return (0);
2019}
2020
2021#ifndef CPU_DISABLE_SSE
2022static void
2023fill_fpregs_xmm(struct savexmm *sv_xmm, struct save87 *sv_87)
2024{
2025 struct env87 *penv_87 = &sv_87->sv_env;
2026 struct envxmm *penv_xmm = &sv_xmm->sv_env;
2027 int i;
2028
2029 /* FPU control/status */
2030 penv_87->en_cw = penv_xmm->en_cw;
2031 penv_87->en_sw = penv_xmm->en_sw;
2032 penv_87->en_tw = penv_xmm->en_tw;
2033 penv_87->en_fip = penv_xmm->en_fip;
2034 penv_87->en_fcs = penv_xmm->en_fcs;
2035 penv_87->en_opcode = penv_xmm->en_opcode;
2036 penv_87->en_foo = penv_xmm->en_foo;
2037 penv_87->en_fos = penv_xmm->en_fos;
2038
2039 /* FPU registers */
2040 for (i = 0; i < 8; ++i)
2041 sv_87->sv_ac[i] = sv_xmm->sv_fp[i].fp_acc;
2042
2043 sv_87->sv_ex_sw = sv_xmm->sv_ex_sw;
2044}
2045
2046static void
2047set_fpregs_xmm(struct save87 *sv_87, struct savexmm *sv_xmm)
2048{
2049 struct env87 *penv_87 = &sv_87->sv_env;
2050 struct envxmm *penv_xmm = &sv_xmm->sv_env;
2051 int i;
2052
2053 /* FPU control/status */
2054 penv_xmm->en_cw = penv_87->en_cw;
2055 penv_xmm->en_sw = penv_87->en_sw;
2056 penv_xmm->en_tw = penv_87->en_tw;
2057 penv_xmm->en_fip = penv_87->en_fip;
2058 penv_xmm->en_fcs = penv_87->en_fcs;
2059 penv_xmm->en_opcode = penv_87->en_opcode;
2060 penv_xmm->en_foo = penv_87->en_foo;
2061 penv_xmm->en_fos = penv_87->en_fos;
2062
2063 /* FPU registers */
2064 for (i = 0; i < 8; ++i)
2065 sv_xmm->sv_fp[i].fp_acc = sv_87->sv_ac[i];
2066
2067 sv_xmm->sv_ex_sw = sv_87->sv_ex_sw;
2068}
2069#endif /* CPU_DISABLE_SSE */
2070
2071int
2072fill_fpregs(struct lwp *lp, struct fpreg *fpregs)
2073{
2074#ifndef CPU_DISABLE_SSE
2075 if (cpu_fxsr) {
2076 fill_fpregs_xmm(&lp->lwp_thread->td_pcb->pcb_save.sv_xmm,
2077 (struct save87 *)fpregs);
2078 return (0);
2079 }
2080#endif /* CPU_DISABLE_SSE */
2081 bcopy(&lp->lwp_thread->td_pcb->pcb_save.sv_87, fpregs, sizeof *fpregs);
2082 return (0);
2083}
2084
2085int
2086set_fpregs(struct lwp *lp, struct fpreg *fpregs)
2087{
2088#ifndef CPU_DISABLE_SSE
2089 if (cpu_fxsr) {
2090 set_fpregs_xmm((struct save87 *)fpregs,
2091 &lp->lwp_thread->td_pcb->pcb_save.sv_xmm);
2092 return (0);
2093 }
2094#endif /* CPU_DISABLE_SSE */
2095 bcopy(fpregs, &lp->lwp_thread->td_pcb->pcb_save.sv_87, sizeof *fpregs);
2096 return (0);
2097}
2098
2099int
2100fill_dbregs(struct lwp *lp, struct dbreg *dbregs)
2101{
2102 if (lp == NULL) {
2103 dbregs->dr[0] = rdr0();
2104 dbregs->dr[1] = rdr1();
2105 dbregs->dr[2] = rdr2();
2106 dbregs->dr[3] = rdr3();
2107 dbregs->dr[4] = rdr4();
2108 dbregs->dr[5] = rdr5();
2109 dbregs->dr[6] = rdr6();
2110 dbregs->dr[7] = rdr7();
2111 } else {
2112 struct pcb *pcb;
2113
2114 pcb = lp->lwp_thread->td_pcb;
2115 dbregs->dr[0] = pcb->pcb_dr0;
2116 dbregs->dr[1] = pcb->pcb_dr1;
2117 dbregs->dr[2] = pcb->pcb_dr2;
2118 dbregs->dr[3] = pcb->pcb_dr3;
2119 dbregs->dr[4] = 0;
2120 dbregs->dr[5] = 0;
2121 dbregs->dr[6] = pcb->pcb_dr6;
2122 dbregs->dr[7] = pcb->pcb_dr7;
2123 }
2124 return (0);
2125}
2126
2127int
2128set_dbregs(struct lwp *lp, struct dbreg *dbregs)
2129{
2130 if (lp == NULL) {
2131 load_dr0(dbregs->dr[0]);
2132 load_dr1(dbregs->dr[1]);
2133 load_dr2(dbregs->dr[2]);
2134 load_dr3(dbregs->dr[3]);
2135 load_dr4(dbregs->dr[4]);
2136 load_dr5(dbregs->dr[5]);
2137 load_dr6(dbregs->dr[6]);
2138 load_dr7(dbregs->dr[7]);
2139 } else {
2140 struct pcb *pcb;
2141 struct ucred *ucred;
2142 int i;
2143 uint64_t mask1, mask2;
2144
2145 /*
2146 * Don't let an illegal value for dr7 get set. Specifically,
2147 * check for undefined settings. Setting these bit patterns
2148 * result in undefined behaviour and can lead to an unexpected
2149 * TRCTRAP.
2150 */
2151 /* JG this loop looks unreadable */
2152 /* Check 4 2-bit fields for invalid patterns.
2153 * These fields are R/Wi, for i = 0..3
2154 */
2155 /* Is 10 in LENi allowed when running in compatibility mode? */
2156 /* Pattern 10 in R/Wi might be used to indicate
2157 * breakpoint on I/O. Further analysis should be
2158 * carried to decide if it is safe and useful to
2159 * provide access to that capability
2160 */
2161 for (i = 0, mask1 = 0x3<<16, mask2 = 0x2<<16; i < 4;
2162 i++, mask1 <<= 4, mask2 <<= 4)
2163 if ((dbregs->dr[7] & mask1) == mask2)
2164 return (EINVAL);
2165
2166 pcb = lp->lwp_thread->td_pcb;
2167 ucred = lp->lwp_proc->p_ucred;
2168
2169 /*
2170 * Don't let a process set a breakpoint that is not within the
2171 * process's address space. If a process could do this, it
2172 * could halt the system by setting a breakpoint in the kernel
2173 * (if ddb was enabled). Thus, we need to check to make sure
2174 * that no breakpoints are being enabled for addresses outside
2175 * process's address space, unless, perhaps, we were called by
2176 * uid 0.
2177 *
2178 * XXX - what about when the watched area of the user's
2179 * address space is written into from within the kernel
2180 * ... wouldn't that still cause a breakpoint to be generated
2181 * from within kernel mode?
2182 */
2183
2184 if (priv_check_cred(ucred, PRIV_ROOT, 0) != 0) {
2185 if (dbregs->dr[7] & 0x3) {
2186 /* dr0 is enabled */
2187 if (dbregs->dr[0] >= VM_MAX_USER_ADDRESS)
2188 return (EINVAL);
2189 }
2190
2191 if (dbregs->dr[7] & (0x3<<2)) {
2192 /* dr1 is enabled */
2193 if (dbregs->dr[1] >= VM_MAX_USER_ADDRESS)
2194 return (EINVAL);
2195 }
2196
2197 if (dbregs->dr[7] & (0x3<<4)) {
2198 /* dr2 is enabled */
2199 if (dbregs->dr[2] >= VM_MAX_USER_ADDRESS)
2200 return (EINVAL);
2201 }
2202
2203 if (dbregs->dr[7] & (0x3<<6)) {
2204 /* dr3 is enabled */
2205 if (dbregs->dr[3] >= VM_MAX_USER_ADDRESS)
2206 return (EINVAL);
2207 }
2208 }
2209
2210 pcb->pcb_dr0 = dbregs->dr[0];
2211 pcb->pcb_dr1 = dbregs->dr[1];
2212 pcb->pcb_dr2 = dbregs->dr[2];
2213 pcb->pcb_dr3 = dbregs->dr[3];
2214 pcb->pcb_dr6 = dbregs->dr[6];
2215 pcb->pcb_dr7 = dbregs->dr[7];
2216
2217 pcb->pcb_flags |= PCB_DBREGS;
2218 }
2219
2220 return (0);
2221}
2222
2223/*
2224 * Return > 0 if a hardware breakpoint has been hit, and the
2225 * breakpoint was in user space. Return 0, otherwise.
2226 */
2227int
2228user_dbreg_trap(void)
2229{
2230 u_int64_t dr7, dr6; /* debug registers dr6 and dr7 */
2231 u_int64_t bp; /* breakpoint bits extracted from dr6 */
2232 int nbp; /* number of breakpoints that triggered */
2233 caddr_t addr[4]; /* breakpoint addresses */
2234 int i;
2235
2236 dr7 = rdr7();
2237 if ((dr7 & 0xff) == 0) {
2238 /*
2239 * all GE and LE bits in the dr7 register are zero,
2240 * thus the trap couldn't have been caused by the
2241 * hardware debug registers
2242 */
2243 return 0;
2244 }
2245
2246 nbp = 0;
2247 dr6 = rdr6();
2248 bp = dr6 & 0xf;
2249
2250 if (bp == 0) {
2251 /*
2252 * None of the breakpoint bits are set meaning this
2253 * trap was not caused by any of the debug registers
2254 */
2255 return 0;
2256 }
2257
2258 /*
2259 * at least one of the breakpoints were hit, check to see
2260 * which ones and if any of them are user space addresses
2261 */
2262
2263 if (bp & 0x01) {
2264 addr[nbp++] = (caddr_t)rdr0();
2265 }
2266 if (bp & 0x02) {
2267 addr[nbp++] = (caddr_t)rdr1();
2268 }
2269 if (bp & 0x04) {
2270 addr[nbp++] = (caddr_t)rdr2();
2271 }
2272 if (bp & 0x08) {
2273 addr[nbp++] = (caddr_t)rdr3();
2274 }
2275
2276 for (i=0; i<nbp; i++) {
2277 if (addr[i] <
2278 (caddr_t)VM_MAX_USER_ADDRESS) {
2279 /*
2280 * addr[i] is in user space
2281 */
2282 return nbp;
2283 }
2284 }
2285
2286 /*
2287 * None of the breakpoints are in user space.
2288 */
2289 return 0;
2290}
2291
2292
2293#ifndef DDB
2294void
2295Debugger(const char *msg)
2296{
2297 kprintf("Debugger(\"%s\") called.\n", msg);
2298}
2299#endif /* no DDB */
2300
2301#ifdef DDB
2302
2303/*
2304 * Provide inb() and outb() as functions. They are normally only
2305 * available as macros calling inlined functions, thus cannot be
2306 * called inside DDB.
2307 *
2308 * The actual code is stolen from <machine/cpufunc.h>, and de-inlined.
2309 */
2310
2311#undef inb
2312#undef outb
2313
2314/* silence compiler warnings */
2315u_char inb(u_int);
2316void outb(u_int, u_char);
2317
2318u_char
2319inb(u_int port)
2320{
2321 u_char data;
2322 /*
2323 * We use %%dx and not %1 here because i/o is done at %dx and not at
2324 * %edx, while gcc generates inferior code (movw instead of movl)
2325 * if we tell it to load (u_short) port.
2326 */
2327 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port));
2328 return (data);
2329}
2330
2331void
2332outb(u_int port, u_char data)
2333{
2334 u_char al;
2335 /*
2336 * Use an unnecessary assignment to help gcc's register allocator.
2337 * This make a large difference for gcc-1.40 and a tiny difference
2338 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for
2339 * best results. gcc-2.6.0 can't handle this.
2340 */
2341 al = data;
2342 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port));
2343}
2344
2345#endif /* DDB */
2346
2347
2348
2349#include "opt_cpu.h"
2350
2351
2352/*
2353 * initialize all the SMP locks
2354 */
2355
2356/* critical region when masking or unmasking interupts */
2357struct spinlock_deprecated imen_spinlock;
2358
2359/* critical region for old style disable_intr/enable_intr */
2360struct spinlock_deprecated mpintr_spinlock;
2361
2362/* critical region around INTR() routines */
2363struct spinlock_deprecated intr_spinlock;
2364
2365/* lock region used by kernel profiling */
2366struct spinlock_deprecated mcount_spinlock;
2367
2368/* locks com (tty) data/hardware accesses: a FASTINTR() */
2369struct spinlock_deprecated com_spinlock;
2370
2371/* lock regions around the clock hardware */
2372struct spinlock_deprecated clock_spinlock;
2373
2374static void
2375init_locks(void)
2376{
2377#ifdef SMP
2378 /*
2379 * Get the initial mplock with a count of 1 for the BSP.
2380 * This uses a LOGICAL cpu ID, ie BSP == 0.
2381 */
2382 cpu_get_initial_mplock();
2383#endif
2384 /* DEPRECATED */
2385 spin_lock_init(&mcount_spinlock);
2386 spin_lock_init(&intr_spinlock);
2387 spin_lock_init(&mpintr_spinlock);
2388 spin_lock_init(&imen_spinlock);
2389 spin_lock_init(&com_spinlock);
2390 spin_lock_init(&clock_spinlock);
2391
2392 /* our token pool needs to work early */
2393 lwkt_token_pool_init();
2394}
2395