[dragonfly.git] / sys / platform / pc64 / x86_64 / exception.S

/*-
 * Copyright (c) 1989, 1990 William F. Jolitz.
 * Copyright (c) 1990 The Regents of the University of California.
 * Copyright (c) 2007 The FreeBSD Foundation
 * Copyright (c) 2008 The DragonFly Project.
 * Copyright (c) 2008 Jordan Gordeev.
 * All rights reserved.
 *
 * Portions of this software were developed by A. Joseph Koshy under
 * sponsorship from the FreeBSD Foundation and Google, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#if JG
#include "opt_atpic.h"
#endif
#include "opt_compat.h"

#include <machine/asmacros.h>
#include <machine/psl.h>
#include <machine/trap.h>
#include <machine/segments.h>

#include "assym.s"

	.text

/*****************************************************************************/
/* Trap handling                                                             */
/*****************************************************************************/
/*
 * Trap and fault vector routines.
 *
 * All traps are 'interrupt gates', SDT_SYSIGT.  An interrupt gate pushes
 * state on the stack but also disables interrupts.  This is important for
 * us for the use of the swapgs instruction.  We cannot be interrupted
 * until the GS.base value is correct.  For most traps, we automatically
 * then enable interrupts if the interrupted context had them enabled.
 * This is equivalent to the i386 port's use of SDT_SYS386TGT.
 *
 * The cpu will push a certain amount of state onto the kernel stack for
 * the current process.  See x86_64/include/frame.h.  
 * This includes the current RFLAGS (status register, which includes 
 * the interrupt disable state prior to the trap), the code segment register,
 * and the return instruction pointer are pushed by the cpu.  The cpu 
 * will also push an 'error' code for certain traps.  We push a dummy 
 * error code for those traps where the cpu doesn't in order to maintain 
 * a consistent frame.  We also push a contrived 'trap number'.
 *
 * The cpu does not push the general registers, we must do that, and we 
 * must restore them prior to calling 'iret'.  The cpu adjusts the %cs and
 * %ss segment registers, but does not mess with %ds, %es, or %fs.  Thus we
 * must load them with appropriate values for supervisor mode operation.
 */

MCOUNT_LABEL(user)
MCOUNT_LABEL(btrap)

/* Traps that we leave interrupts disabled for.. */
#define	TRAP_NOEN(a)	\
	subq $TF_RIP,%rsp; \
	movq $0,TF_XFLAGS(%rsp) ; \
	movq $(a),TF_TRAPNO(%rsp) ; \
	movq $0,TF_ADDR(%rsp) ; \
	movq $0,TF_ERR(%rsp) ; \
	jmp alltraps_noen
IDTVEC(dbg)
	TRAP_NOEN(T_TRCTRAP)
IDTVEC(bpt)
	TRAP_NOEN(T_BPTFLT)

/* Regular traps; The cpu does not supply tf_err for these. */
#define	TRAP(a)	 \
	subq $TF_RIP,%rsp; \
	movq $0,TF_XFLAGS(%rsp) ; \
	movq $(a),TF_TRAPNO(%rsp) ; \
	movq $0,TF_ADDR(%rsp) ; \
	movq $0,TF_ERR(%rsp) ; \
	jmp alltraps
IDTVEC(div)
	TRAP(T_DIVIDE)
IDTVEC(ofl)
	TRAP(T_OFLOW)
IDTVEC(bnd)
	TRAP(T_BOUND)
IDTVEC(ill)
	TRAP(T_PRIVINFLT)
IDTVEC(dna)
	TRAP(T_DNA)
IDTVEC(fpusegm)
	TRAP(T_FPOPFLT)
IDTVEC(mchk)
	TRAP(T_MCHK)
IDTVEC(rsvd)
	TRAP(T_RESERVED)
IDTVEC(fpu)
	TRAP(T_ARITHTRAP)
IDTVEC(xmm)
	TRAP(T_XMMFLT)

/* This group of traps have tf_err already pushed by the cpu */
#define	TRAP_ERR(a)	\
	subq $TF_ERR,%rsp; \
	movq $(a),TF_TRAPNO(%rsp) ; \
	movq $0,TF_ADDR(%rsp) ; \
	movq $0,TF_XFLAGS(%rsp) ; \
	jmp alltraps
IDTVEC(tss)
	TRAP_ERR(T_TSSFLT)
IDTVEC(missing)
	TRAP_ERR(T_SEGNPFLT)
IDTVEC(stk)
	TRAP_ERR(T_STKFLT)
IDTVEC(align)
	TRAP_ERR(T_ALIGNFLT)

	/*
	 * alltraps entry point.  Use swapgs if this is the first time in the
	 * kernel from userland.  Reenable interrupts if they were enabled
	 * before the trap.  This approximates SDT_SYS386TGT on the i386 port.
	 */

	SUPERALIGN_TEXT
	.globl	alltraps
	.type	alltraps,@function
alltraps:
	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
	jz	alltraps_testi		/* already running with kernel GS.base */
	swapgs
alltraps_testi:
	testq	$PSL_I,TF_RFLAGS(%rsp)
	jz	alltraps_pushregs
	sti
alltraps_pushregs:
	movq	%rdi,TF_RDI(%rsp)
alltraps_pushregs_no_rdi:
	movq	%rsi,TF_RSI(%rsp)
	movq	%rdx,TF_RDX(%rsp)
	movq	%rcx,TF_RCX(%rsp)
	movq	%r8,TF_R8(%rsp)
	movq	%r9,TF_R9(%rsp)
	movq	%rax,TF_RAX(%rsp)
	movq	%rbx,TF_RBX(%rsp)
	movq	%rbp,TF_RBP(%rsp)
	movq	%r10,TF_R10(%rsp)
	movq	%r11,TF_R11(%rsp)
	movq	%r12,TF_R12(%rsp)
	movq	%r13,TF_R13(%rsp)
	movq	%r14,TF_R14(%rsp)
	movq	%r15,TF_R15(%rsp)
	FAKE_MCOUNT(TF_RIP(%rsp))
	.globl	calltrap
	.type	calltrap,@function
calltrap:
	movq	%rsp, %rdi
	call	trap
	MEXITCOUNT
	jmp	doreti			/* Handle any pending ASTs */

	/*
	 * alltraps_noen entry point.  Unlike alltraps above, we want to
	 * leave the interrupts disabled.  This corresponds to
	 * SDT_SYS386IGT on the i386 port.
	 */
	SUPERALIGN_TEXT
	.globl	alltraps_noen
	.type	alltraps_noen,@function
alltraps_noen:
	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
	jz	alltraps_pushregs	/* already running with kernel GS.base */
	swapgs
	jmp	alltraps_pushregs

IDTVEC(dblfault)
	subq	$TF_ERR,%rsp
	movq	$T_DOUBLEFLT,TF_TRAPNO(%rsp)
	movq	$0,TF_ADDR(%rsp)
	movq	$0,TF_ERR(%rsp)
	movq	$0,TF_XFLAGS(%rsp)
	movq	%rdi,TF_RDI(%rsp)
	movq	%rsi,TF_RSI(%rsp)
	movq	%rdx,TF_RDX(%rsp)
	movq	%rcx,TF_RCX(%rsp)
	movq	%r8,TF_R8(%rsp)
	movq	%r9,TF_R9(%rsp)
	movq	%rax,TF_RAX(%rsp)
	movq	%rbx,TF_RBX(%rsp)
	movq	%rbp,TF_RBP(%rsp)
	movq	%r10,TF_R10(%rsp)
	movq	%r11,TF_R11(%rsp)
	movq	%r12,TF_R12(%rsp)
	movq	%r13,TF_R13(%rsp)
	movq	%r14,TF_R14(%rsp)
	movq	%r15,TF_R15(%rsp)
	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
	jz	1f			/* already running with kernel GS.base */
	swapgs
1:	movq	%rsp, %rdi
	call	dblfault_handler
2:	hlt
	jmp	2b

IDTVEC(page)
	subq	$TF_ERR,%rsp
	movq	$T_PAGEFLT,TF_TRAPNO(%rsp)
	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
	jz	1f			/* already running with kernel GS.base */
	swapgs
1:
	movq	%rdi,TF_RDI(%rsp)	/* free up a GP register */
	movq	%cr2,%rdi		/* preserve %cr2 before ..  */
	movq	%rdi,TF_ADDR(%rsp)	/* enabling interrupts. */
	movq	$0,TF_XFLAGS(%rsp)
	testq	$PSL_I,TF_RFLAGS(%rsp)
	jz	alltraps_pushregs_no_rdi
	sti
	jmp	alltraps_pushregs_no_rdi

	/*
	 * We have to special-case this one.  If we get a trap in doreti() at
	 * the iretq stage, we'll reenter with the wrong gs state.  We'll have
	 * to do a special the swapgs in this case even coming from the kernel.
	 * XXX linux has a trap handler for their equivalent of load_gs().
	 */
IDTVEC(prot)
	subq	$TF_ERR,%rsp
	movq	$T_PROTFLT,TF_TRAPNO(%rsp)
	movq	$0,TF_ADDR(%rsp)
	movq	$0,TF_XFLAGS(%rsp)
	movq	%rdi,TF_RDI(%rsp)	/* free up a GP register */
	leaq	doreti_iret(%rip),%rdi
	cmpq	%rdi,TF_RIP(%rsp)
	je	2f			/* kernel but with user gsbase!! */
	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
	jz	1f			/* already running with kernel GS.base */
2:
	swapgs
1:
	testq	$PSL_I,TF_RFLAGS(%rsp)
	jz	alltraps_pushregs_no_rdi
	sti
	jmp	alltraps_pushregs_no_rdi

/*
 * Fast syscall entry point.  We enter here with just our new %cs/%ss set,
 * and the new privilige level.  We are still running on the old user stack
 * pointer.  We have to juggle a few things around to find our stack etc.
 * swapgs gives us access to our PCPU space only.
 */
IDTVEC(fast_syscall)
	swapgs
	movq	%rsp,PCPU(scratch_rsp)
	movq	PCPU(rsp0),%rsp
	/* Now emulate a trapframe. Make the 8 byte alignment odd for call. */
	subq	$TF_SIZE,%rsp
	/* defer TF_RSP till we have a spare register */
	movq	%r11,TF_RFLAGS(%rsp)
	movq	%rcx,TF_RIP(%rsp)	/* %rcx original value is in %r10 */
	movq	PCPU(scratch_rsp),%r11	/* %r11 already saved */
	movq	%r11,TF_RSP(%rsp)	/* user stack pointer */
	sti
	movq	$KUDSEL,TF_SS(%rsp)
	movq	$KUCSEL,TF_CS(%rsp)
	movq	$2,TF_ERR(%rsp)
	movq	$0,TF_XFLAGS(%rsp)	/* note: used in signal frame */
	movq	%rdi,TF_RDI(%rsp)	/* arg 1 */
	movq	%rsi,TF_RSI(%rsp)	/* arg 2 */
	movq	%rdx,TF_RDX(%rsp)	/* arg 3 */
	movq	%r10,TF_RCX(%rsp)	/* arg 4 */
	movq	%r8,TF_R8(%rsp)		/* arg 5 */
	movq	%r9,TF_R9(%rsp)		/* arg 6 */
	movq	%rax,TF_RAX(%rsp)	/* syscall number */
	movq	%rbx,TF_RBX(%rsp)	/* C preserved */
	movq	%rbp,TF_RBP(%rsp)	/* C preserved */
	movq	%r12,TF_R12(%rsp)	/* C preserved */
	movq	%r13,TF_R13(%rsp)	/* C preserved */
	movq	%r14,TF_R14(%rsp)	/* C preserved */
	movq	%r15,TF_R15(%rsp)	/* C preserved */
	FAKE_MCOUNT(TF_RIP(%rsp))
	movq	%rsp, %rdi
	call	syscall2
	MEXITCOUNT
	jmp	doreti

/*
 * Here for CYA insurance, in case a "syscall" instruction gets
 * issued from 32 bit compatability mode. MSR_CSTAR has to point
 * to *something* if EFER_SCE is enabled.
 */
IDTVEC(fast_syscall32)
	sysret

/*
 * NMI handling is special.
 *
 * First, NMIs do not respect the state of the processor's RFLAGS.IF
 * bit and the NMI handler may be invoked at any time, including when
 * the processor is in a critical section with RFLAGS.IF == 0.  In
 * particular, this means that the processor's GS.base values could be
 * inconsistent on entry to the handler, and so we need to read
 * MSR_GSBASE to determine if a 'swapgs' is needed.  We use '%ebx', a
 * C-preserved register, to remember whether to swap GS back on the
 * exit path.
 *
 * Second, the processor treats NMIs specially, blocking further NMIs
 * until an 'iretq' instruction is executed.  We therefore need to
 * execute the NMI handler with interrupts disabled to prevent a
 * nested interrupt from executing an 'iretq' instruction and
 * inadvertently taking the processor out of NMI mode.
 *
 * Third, the NMI handler runs on its own stack (tss_ist1), shared
 * with the double fault handler.
 */

IDTVEC(nmi)
	subq	$TF_RIP,%rsp
	movq	$(T_NMI),TF_TRAPNO(%rsp)
	movq	$0,TF_ADDR(%rsp)
	movq	$0,TF_ERR(%rsp)
	movq	$0,TF_XFLAGS(%rsp)
	movq	%rdi,TF_RDI(%rsp)
	movq	%rsi,TF_RSI(%rsp)
	movq	%rdx,TF_RDX(%rsp)
	movq	%rcx,TF_RCX(%rsp)
	movq	%r8,TF_R8(%rsp)
	movq	%r9,TF_R9(%rsp)
	movq	%rax,TF_RAX(%rsp)
	movq	%rbx,TF_RBX(%rsp)
	movq	%rbp,TF_RBP(%rsp)
	movq	%r10,TF_R10(%rsp)
	movq	%r11,TF_R11(%rsp)
	movq	%r12,TF_R12(%rsp)
	movq	%r13,TF_R13(%rsp)
	movq	%r14,TF_R14(%rsp)
	movq	%r15,TF_R15(%rsp)
	xorl	%ebx,%ebx
	testb	$SEL_RPL_MASK,TF_CS(%rsp)
	jnz	nmi_needswapgs		/* we came from userland */
	movl	$MSR_GSBASE,%ecx
	rdmsr
	cmpl	$VM_MAX_USER_ADDRESS >> 32,%edx
	jae	nmi_calltrap		/* GS.base holds a kernel VA */
nmi_needswapgs:
	incl	%ebx
	swapgs
/* Note: this label is also used by ddb and gdb: */
nmi_calltrap:
	FAKE_MCOUNT(TF_RIP(%rsp))
	movq	%rsp, %rdi
	call	trap
	MEXITCOUNT
	testl	%ebx,%ebx
	jz	nmi_restoreregs
	swapgs
nmi_restoreregs:
	movq	TF_RDI(%rsp),%rdi
	movq	TF_RSI(%rsp),%rsi
	movq	TF_RDX(%rsp),%rdx
	movq	TF_RCX(%rsp),%rcx
	movq	TF_R8(%rsp),%r8
	movq	TF_R9(%rsp),%r9
	movq	TF_RAX(%rsp),%rax
	movq	TF_RBX(%rsp),%rbx
	movq	TF_RBP(%rsp),%rbp
	movq	TF_R10(%rsp),%r10
	movq	TF_R11(%rsp),%r11
	movq	TF_R12(%rsp),%r12
	movq	TF_R13(%rsp),%r13
	movq	TF_R14(%rsp),%r14
	movq	TF_R15(%rsp),%r15
	addq	$TF_RIP,%rsp
	iretq

/*
 * This function is what cpu_heavy_restore jumps to after a new process
 * is created.  The LWKT subsystem switches while holding a critical
 * section and we maintain that abstraction here (e.g. because 
 * cpu_heavy_restore needs it due to PCB_*() manipulation), then get out of
 * it before calling the initial function (typically fork_return()) and/or
 * returning to user mode.
 *
 * The MP lock is held on entry, but for processes fork_return(esi)
 * releases it.  'doreti' always runs without the MP lock.
 */
ENTRY(fork_trampoline)
	movq	PCPU(curthread),%rax
	subl	$TDPRI_CRIT,TD_PRI(%rax)

	/*
	 * cpu_set_fork_handler intercepts this function call to
	 * have this call a non-return function to stay in kernel mode.
	 *
	 * initproc has its own fork handler, start_init(), which DOES
	 * return.
	 *
	 * %rbx - chaining function (typically fork_return)
	 * %r12 -> %rdi (argument)
	 * frame-> %rsi (trap frame)
	 *
	 *   void (func:rbx)(arg:rdi, trapframe:rsi)
	 */
	movq	%rsp, %rsi		/* pass trapframe by reference */
	movq	%r12, %rdi		/* arg1 */
	call	*%rbx			/* function */

	/* cut from syscall */

	sti
	call	splz

#if defined(INVARIANTS) && defined(SMP)
	movq	PCPU(curthread),%rax
	cmpl	$0,TD_MPCOUNT(%rax)
	je	1f
	movq	$pmsg4, %rdi
	movl	TD_MPCOUNT(%rax), %esi
	movq	%rbx, %rdx
	xorl	%eax, %eax
	call	panic
pmsg4:  .asciz	"fork_trampoline mpcount %d after calling %p"
	/* JG what's the purpose of this alignment and is it enough on x86_64? */
	.p2align 2
1:
#endif
	/*
	 * Return via doreti to handle ASTs.
	 *
	 * trapframe is at the top of the stack.
	 */
	MEXITCOUNT
	jmp	doreti

/*
 * To efficiently implement classification of trap and interrupt handlers
 * for profiling, there must be only trap handlers between the labels btrap
 * and bintr, and only interrupt handlers between the labels bintr and
 * eintr.  This is implemented (partly) by including files that contain
 * some of the handlers.  Before including the files, set up a normal asm
 * environment so that the included files doen't need to know that they are
 * included.
 */

#ifdef COMPAT_IA32
	.data
	.p2align 4
	.text
	SUPERALIGN_TEXT

#include <x86_64/ia32/ia32_exception.S>
#endif

	.data
	.p2align 4
	.text
	SUPERALIGN_TEXT
MCOUNT_LABEL(bintr)

#if JG
#include <x86_64/x86_64/apic_vector.S>
#endif

#ifdef DEV_ATPIC
	.data
	.p2align 4
	.text
	SUPERALIGN_TEXT

#include <x86_64/isa/atpic_vector.S>
#endif

	.text
MCOUNT_LABEL(eintr)
Commit	Line	Data
c8fe38ae MD	1	/*-
	2	* Copyright (c) 1989, 1990 William F. Jolitz.
	3	* Copyright (c) 1990 The Regents of the University of California.
	4	* Copyright (c) 2007 The FreeBSD Foundation
	5	* Copyright (c) 2008 The DragonFly Project.
	6	* Copyright (c) 2008 Jordan Gordeev.
	7	* All rights reserved.
	8	*
	9	* Portions of this software were developed by A. Joseph Koshy under
	10	* sponsorship from the FreeBSD Foundation and Google, Inc.
	11	*
	12	* Redistribution and use in source and binary forms, with or without
	13	* modification, are permitted provided that the following conditions
	14	* are met:
	15	* 1. Redistributions of source code must retain the above copyright
	16	* notice, this list of conditions and the following disclaimer.
	17	* 2. Redistributions in binary form must reproduce the above copyright
	18	* notice, this list of conditions and the following disclaimer in the
	19	* documentation and/or other materials provided with the distribution.
	20	* 4. Neither the name of the University nor the names of its contributors
	21	* may be used to endorse or promote products derived from this software
	22	* without specific prior written permission.
	23	*
	24	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	25	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	26	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	27	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	28	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	29	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	30	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	31	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	32	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	33	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	34	* SUCH DAMAGE.
c8fe38ae MD	35	*/
	36
	37	#if JG
	38	#include "opt_atpic.h"
	39	#endif
	40	#include "opt_compat.h"
	41
	42	#include <machine/asmacros.h>
	43	#include <machine/psl.h>
	44	#include <machine/trap.h>
bfc09ba0	45	#include <machine/segments.h>
c8fe38ae MD	46
	47	#include "assym.s"
	48
	49	.text
	50
	51	/*****************************************************************************/
	52	/* Trap handling */
	53	/*****************************************************************************/
	54	/*
	55	* Trap and fault vector routines.
	56	*
	57	* All traps are 'interrupt gates', SDT_SYSIGT. An interrupt gate pushes
	58	* state on the stack but also disables interrupts. This is important for
	59	* us for the use of the swapgs instruction. We cannot be interrupted
	60	* until the GS.base value is correct. For most traps, we automatically
	61	* then enable interrupts if the interrupted context had them enabled.
	62	* This is equivalent to the i386 port's use of SDT_SYS386TGT.
	63	*
	64	* The cpu will push a certain amount of state onto the kernel stack for
c1543a89	65	* the current process. See x86_64/include/frame.h.
c8fe38ae MD	66	* This includes the current RFLAGS (status register, which includes
	67	* the interrupt disable state prior to the trap), the code segment register,
	68	* and the return instruction pointer are pushed by the cpu. The cpu
	69	* will also push an 'error' code for certain traps. We push a dummy
	70	* error code for those traps where the cpu doesn't in order to maintain
	71	* a consistent frame. We also push a contrived 'trap number'.
	72	*
	73	* The cpu does not push the general registers, we must do that, and we
	74	* must restore them prior to calling 'iret'. The cpu adjusts the %cs and
	75	* %ss segment registers, but does not mess with %ds, %es, or %fs. Thus we
	76	* must load them with appropriate values for supervisor mode operation.
	77	*/
	78
	79	MCOUNT_LABEL(user)
	80	MCOUNT_LABEL(btrap)
	81
	82	/* Traps that we leave interrupts disabled for.. */
	83	#define TRAP_NOEN(a) \
	84	subq $TF_RIP,%rsp; \
5b9f6cc4	85	movq $0,TF_XFLAGS(%rsp) ; \
c8fe38ae MD	86	movq $(a),TF_TRAPNO(%rsp) ; \
	87	movq $0,TF_ADDR(%rsp) ; \
	88	movq $0,TF_ERR(%rsp) ; \
	89	jmp alltraps_noen
	90	IDTVEC(dbg)
	91	TRAP_NOEN(T_TRCTRAP)
	92	IDTVEC(bpt)
	93	TRAP_NOEN(T_BPTFLT)
	94
	95	/* Regular traps; The cpu does not supply tf_err for these. */
	96	#define TRAP(a) \
	97	subq $TF_RIP,%rsp; \
5b9f6cc4	98	movq $0,TF_XFLAGS(%rsp) ; \
c8fe38ae MD	99	movq $(a),TF_TRAPNO(%rsp) ; \
	100	movq $0,TF_ADDR(%rsp) ; \
	101	movq $0,TF_ERR(%rsp) ; \
	102	jmp alltraps
	103	IDTVEC(div)
	104	TRAP(T_DIVIDE)
	105	IDTVEC(ofl)
	106	TRAP(T_OFLOW)
	107	IDTVEC(bnd)
	108	TRAP(T_BOUND)
	109	IDTVEC(ill)
	110	TRAP(T_PRIVINFLT)
	111	IDTVEC(dna)
	112	TRAP(T_DNA)
	113	IDTVEC(fpusegm)
	114	TRAP(T_FPOPFLT)
	115	IDTVEC(mchk)
	116	TRAP(T_MCHK)
	117	IDTVEC(rsvd)
	118	TRAP(T_RESERVED)
	119	IDTVEC(fpu)
	120	TRAP(T_ARITHTRAP)
	121	IDTVEC(xmm)
	122	TRAP(T_XMMFLT)
	123
	124	/* This group of traps have tf_err already pushed by the cpu */
	125	#define TRAP_ERR(a) \
	126	subq $TF_ERR,%rsp; \
	127	movq $(a),TF_TRAPNO(%rsp) ; \
	128	movq $0,TF_ADDR(%rsp) ; \
16094a36	129	movq $0,TF_XFLAGS(%rsp) ; \
c8fe38ae MD	130	jmp alltraps
	131	IDTVEC(tss)
	132	TRAP_ERR(T_TSSFLT)
	133	IDTVEC(missing)
	134	TRAP_ERR(T_SEGNPFLT)
	135	IDTVEC(stk)
	136	TRAP_ERR(T_STKFLT)
	137	IDTVEC(align)
	138	TRAP_ERR(T_ALIGNFLT)
	139
	140	/*
	141	* alltraps entry point. Use swapgs if this is the first time in the
	142	* kernel from userland. Reenable interrupts if they were enabled
	143	* before the trap. This approximates SDT_SYS386TGT on the i386 port.
	144	*/
	145
	146	SUPERALIGN_TEXT
	147	.globl alltraps
	148	.type alltraps,@function
	149	alltraps:
	150	testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
	151	jz alltraps_testi /* already running with kernel GS.base */
	152	swapgs
	153	alltraps_testi:
	154	testq $PSL_I,TF_RFLAGS(%rsp)
	155	jz alltraps_pushregs
	156	sti
	157	alltraps_pushregs:
	158	movq %rdi,TF_RDI(%rsp)
	159	alltraps_pushregs_no_rdi:
	160	movq %rsi,TF_RSI(%rsp)
	161	movq %rdx,TF_RDX(%rsp)
	162	movq %rcx,TF_RCX(%rsp)
	163	movq %r8,TF_R8(%rsp)
	164	movq %r9,TF_R9(%rsp)
	165	movq %rax,TF_RAX(%rsp)
	166	movq %rbx,TF_RBX(%rsp)
	167	movq %rbp,TF_RBP(%rsp)
	168	movq %r10,TF_R10(%rsp)
	169	movq %r11,TF_R11(%rsp)
	170	movq %r12,TF_R12(%rsp)
	171	movq %r13,TF_R13(%rsp)
	172	movq %r14,TF_R14(%rsp)
	173	movq %r15,TF_R15(%rsp)
	174	FAKE_MCOUNT(TF_RIP(%rsp))
	175	.globl calltrap
	176	.type calltrap,@function
	177	calltrap:
	178	movq %rsp, %rdi
	179	call trap
	180	MEXITCOUNT
	181	jmp doreti /* Handle any pending ASTs */
	182
	183	/*
	184	* alltraps_noen entry point. Unlike alltraps above, we want to
	185	* leave the interrupts disabled. This corresponds to
	186	* SDT_SYS386IGT on the i386 port.
	187	*/
	188	SUPERALIGN_TEXT
	189	.globl alltraps_noen
	190	.type alltraps_noen,@function
	191	alltraps_noen:
	192	testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
	193	jz alltraps_pushregs /* already running with kernel GS.base */
194	swapgs
195	jmp alltraps_pushregs
196
197	IDTVEC(dblfault)
198	subq $TF_ERR,%rsp
199	movq $T_DOUBLEFLT,TF_TRAPNO(%rsp)
200	movq $0,TF_ADDR(%rsp)
201	movq $0,TF_ERR(%rsp)
16094a36	202	movq $0,TF_XFLAGS(%rsp)
c8fe38ae MD	203	movq %rdi,TF_RDI(%rsp)
	204	movq %rsi,TF_RSI(%rsp)
	205	movq %rdx,TF_RDX(%rsp)
	206	movq %rcx,TF_RCX(%rsp)
	207	movq %r8,TF_R8(%rsp)
	208	movq %r9,TF_R9(%rsp)
	209	movq %rax,TF_RAX(%rsp)
	210	movq %rbx,TF_RBX(%rsp)
	211	movq %rbp,TF_RBP(%rsp)
	212	movq %r10,TF_R10(%rsp)
	213	movq %r11,TF_R11(%rsp)
	214	movq %r12,TF_R12(%rsp)
	215	movq %r13,TF_R13(%rsp)
	216	movq %r14,TF_R14(%rsp)
	217	movq %r15,TF_R15(%rsp)
	218	testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
	219	jz 1f /* already running with kernel GS.base */
	220	swapgs
	221	1: movq %rsp, %rdi
	222	call dblfault_handler
	223	2: hlt
	224	jmp 2b
	225
	226	IDTVEC(page)
	227	subq $TF_ERR,%rsp
	228	movq $T_PAGEFLT,TF_TRAPNO(%rsp)
	229	testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
	230	jz 1f /* already running with kernel GS.base */
	231	swapgs
	232	1:
	233	movq %rdi,TF_RDI(%rsp) /* free up a GP register */
	234	movq %cr2,%rdi /* preserve %cr2 before .. */
	235	movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */
16094a36	236	movq $0,TF_XFLAGS(%rsp)
c8fe38ae MD	237	testq $PSL_I,TF_RFLAGS(%rsp)
	238	jz alltraps_pushregs_no_rdi
	239	sti
	240	jmp alltraps_pushregs_no_rdi
	241
	242	/*
	243	* We have to special-case this one. If we get a trap in doreti() at
	244	* the iretq stage, we'll reenter with the wrong gs state. We'll have
	245	* to do a special the swapgs in this case even coming from the kernel.
	246	* XXX linux has a trap handler for their equivalent of load_gs().
	247	*/
	248	IDTVEC(prot)
	249	subq $TF_ERR,%rsp
	250	movq $T_PROTFLT,TF_TRAPNO(%rsp)
	251	movq $0,TF_ADDR(%rsp)
16094a36	252	movq $0,TF_XFLAGS(%rsp)
c8fe38ae MD	253	movq %rdi,TF_RDI(%rsp) /* free up a GP register */
	254	leaq doreti_iret(%rip),%rdi
	255	cmpq %rdi,TF_RIP(%rsp)
	256	je 2f /* kernel but with user gsbase!! */
	257	testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
	258	jz 1f /* already running with kernel GS.base */
	259	2:
	260	swapgs
	261	1:
	262	testq $PSL_I,TF_RFLAGS(%rsp)
	263	jz alltraps_pushregs_no_rdi
	264	sti
	265	jmp alltraps_pushregs_no_rdi
	266
	267	/*
	268	* Fast syscall entry point. We enter here with just our new %cs/%ss set,
	269	* and the new privilige level. We are still running on the old user stack
	270	* pointer. We have to juggle a few things around to find our stack etc.
	271	* swapgs gives us access to our PCPU space only.
	272	*/
	273	IDTVEC(fast_syscall)
	274	swapgs
	275	movq %rsp,PCPU(scratch_rsp)
	276	movq PCPU(rsp0),%rsp
	277	/* Now emulate a trapframe. Make the 8 byte alignment odd for call. */
	278	subq $TF_SIZE,%rsp
	279	/* defer TF_RSP till we have a spare register */
	280	movq %r11,TF_RFLAGS(%rsp)
	281	movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */
	282	movq PCPU(scratch_rsp),%r11 /* %r11 already saved */
	283	movq %r11,TF_RSP(%rsp) /* user stack pointer */
	284	sti
	285	movq $KUDSEL,TF_SS(%rsp)
	286	movq $KUCSEL,TF_CS(%rsp)
	287	movq $2,TF_ERR(%rsp)
16094a36	288	movq $0,TF_XFLAGS(%rsp) /* note: used in signal frame */
c8fe38ae MD	289	movq %rdi,TF_RDI(%rsp) /* arg 1 */
	290	movq %rsi,TF_RSI(%rsp) /* arg 2 */
	291	movq %rdx,TF_RDX(%rsp) /* arg 3 */
	292	movq %r10,TF_RCX(%rsp) /* arg 4 */
	293	movq %r8,TF_R8(%rsp) /* arg 5 */
	294	movq %r9,TF_R9(%rsp) /* arg 6 */
	295	movq %rax,TF_RAX(%rsp) /* syscall number */
	296	movq %rbx,TF_RBX(%rsp) /* C preserved */
	297	movq %rbp,TF_RBP(%rsp) /* C preserved */
	298	movq %r12,TF_R12(%rsp) /* C preserved */
	299	movq %r13,TF_R13(%rsp) /* C preserved */
	300	movq %r14,TF_R14(%rsp) /* C preserved */
	301	movq %r15,TF_R15(%rsp) /* C preserved */
	302	FAKE_MCOUNT(TF_RIP(%rsp))
	303	movq %rsp, %rdi
	304	call syscall2
c8fe38ae MD	305	MEXITCOUNT
	306	jmp doreti
	307
	308	/*
	309	* Here for CYA insurance, in case a "syscall" instruction gets
	310	* issued from 32 bit compatability mode. MSR_CSTAR has to point
	311	* to something if EFER_SCE is enabled.
	312	*/
	313	IDTVEC(fast_syscall32)
	314	sysret
	315
	316	/*
	317	* NMI handling is special.
	318	*
	319	* First, NMIs do not respect the state of the processor's RFLAGS.IF
	320	* bit and the NMI handler may be invoked at any time, including when
	321	* the processor is in a critical section with RFLAGS.IF == 0. In
	322	* particular, this means that the processor's GS.base values could be
	323	* inconsistent on entry to the handler, and so we need to read
	324	* MSR_GSBASE to determine if a 'swapgs' is needed. We use '%ebx', a
	325	* C-preserved register, to remember whether to swap GS back on the
	326	* exit path.
	327	*
	328	* Second, the processor treats NMIs specially, blocking further NMIs
	329	* until an 'iretq' instruction is executed. We therefore need to
	330	* execute the NMI handler with interrupts disabled to prevent a
	331	* nested interrupt from executing an 'iretq' instruction and
	332	* inadvertently taking the processor out of NMI mode.
	333	*
	334	* Third, the NMI handler runs on its own stack (tss_ist1), shared
	335	* with the double fault handler.
	336	*/
	337
	338	IDTVEC(nmi)
	339	subq $TF_RIP,%rsp
	340	movq $(T_NMI),TF_TRAPNO(%rsp)
	341	movq $0,TF_ADDR(%rsp)
	342	movq $0,TF_ERR(%rsp)
16094a36	343	movq $0,TF_XFLAGS(%rsp)
c8fe38ae MD	344	movq %rdi,TF_RDI(%rsp)
	345	movq %rsi,TF_RSI(%rsp)
	346	movq %rdx,TF_RDX(%rsp)
	347	movq %rcx,TF_RCX(%rsp)
	348	movq %r8,TF_R8(%rsp)
	349	movq %r9,TF_R9(%rsp)
	350	movq %rax,TF_RAX(%rsp)
	351	movq %rbx,TF_RBX(%rsp)
	352	movq %rbp,TF_RBP(%rsp)
	353	movq %r10,TF_R10(%rsp)
	354	movq %r11,TF_R11(%rsp)
	355	movq %r12,TF_R12(%rsp)
	356	movq %r13,TF_R13(%rsp)
	357	movq %r14,TF_R14(%rsp)
	358	movq %r15,TF_R15(%rsp)
	359	xorl %ebx,%ebx
	360	testb $SEL_RPL_MASK,TF_CS(%rsp)
	361	jnz nmi_needswapgs /* we came from userland */
	362	movl $MSR_GSBASE,%ecx
	363	rdmsr
48ffc236	364	cmpl $VM_MAX_USER_ADDRESS >> 32,%edx
c8fe38ae MD	365	jae nmi_calltrap /* GS.base holds a kernel VA */
	366	nmi_needswapgs:
	367	incl %ebx
	368	swapgs
	369	/* Note: this label is also used by ddb and gdb: */
	370	nmi_calltrap:
	371	FAKE_MCOUNT(TF_RIP(%rsp))
	372	movq %rsp, %rdi
	373	call trap
	374	MEXITCOUNT
	375	testl %ebx,%ebx
	376	jz nmi_restoreregs
	377	swapgs
	378	nmi_restoreregs:
	379	movq TF_RDI(%rsp),%rdi
	380	movq TF_RSI(%rsp),%rsi
	381	movq TF_RDX(%rsp),%rdx
	382	movq TF_RCX(%rsp),%rcx
	383	movq TF_R8(%rsp),%r8
	384	movq TF_R9(%rsp),%r9
	385	movq TF_RAX(%rsp),%rax
	386	movq TF_RBX(%rsp),%rbx
	387	movq TF_RBP(%rsp),%rbp
	388	movq TF_R10(%rsp),%r10
	389	movq TF_R11(%rsp),%r11
	390	movq TF_R12(%rsp),%r12
	391	movq TF_R13(%rsp),%r13
	392	movq TF_R14(%rsp),%r14
	393	movq TF_R15(%rsp),%r15
	394	addq $TF_RIP,%rsp
	395	iretq
	396
	397	/*
	398	* This function is what cpu_heavy_restore jumps to after a new process
	399	* is created. The LWKT subsystem switches while holding a critical
	400	* section and we maintain that abstraction here (e.g. because
	401	* cpu_heavy_restore needs it due to PCB_*() manipulation), then get out of
	402	* it before calling the initial function (typically fork_return()) and/or
	403	* returning to user mode.
	404	*
	405	* The MP lock is held on entry, but for processes fork_return(esi)
	406	* releases it. 'doreti' always runs without the MP lock.
	407	*/
	408	ENTRY(fork_trampoline)
	409	movq PCPU(curthread),%rax
	410	subl $TDPRI_CRIT,TD_PRI(%rax)
	411
	412	/*
	413	* cpu_set_fork_handler intercepts this function call to
	414	* have this call a non-return function to stay in kernel mode.
	415	*
	416	* initproc has its own fork handler, start_init(), which DOES
	417	* return.
	418	*
	419	* %rbx - chaining function (typically fork_return)
	420	* %r12 -> %rdi (argument)
	421	* frame-> %rsi (trap frame)
	422	*
	423	* void (func:rbx)(arg:rdi, trapframe:rsi)
	424	*/
	425	movq %rsp, %rsi /* pass trapframe by reference */
	426	movq %r12, %rdi /* arg1 */
	427	call %rbx / function */
	428
429	/* cut from syscall */
430
431	sti
432	call splz
433
434	#if defined(INVARIANTS) && defined(SMP)
435	movq PCPU(curthread),%rax
436	cmpl $0,TD_MPCOUNT(%rax)
437	je 1f
438	movq $pmsg4, %rdi
46d4e165	439	movl TD_MPCOUNT(%rax), %esi
c8fe38ae MD	440	movq %rbx, %rdx
	441	xorl %eax, %eax
	442	call panic
	443	pmsg4: .asciz "fork_trampoline mpcount %d after calling %p"
c1543a89	444	/* JG what's the purpose of this alignment and is it enough on x86_64? */
c8fe38ae MD	445	.p2align 2
	446	1:
	447	#endif
	448	/*
	449	* Return via doreti to handle ASTs.
	450	*
	451	* trapframe is at the top of the stack.
	452	*/
c8fe38ae MD	453	MEXITCOUNT
	454	jmp doreti
	455
	456	/*
	457	* To efficiently implement classification of trap and interrupt handlers
	458	* for profiling, there must be only trap handlers between the labels btrap
	459	* and bintr, and only interrupt handlers between the labels bintr and
	460	* eintr. This is implemented (partly) by including files that contain
	461	* some of the handlers. Before including the files, set up a normal asm
	462	* environment so that the included files doen't need to know that they are
	463	* included.
	464	*/
	465
	466	#ifdef COMPAT_IA32
	467	.data
	468	.p2align 4
	469	.text
	470	SUPERALIGN_TEXT
	471
c1543a89	472	#include <x86_64/ia32/ia32_exception.S>
c8fe38ae MD	473	#endif
	474
	475	.data
	476	.p2align 4
	477	.text
	478	SUPERALIGN_TEXT
	479	MCOUNT_LABEL(bintr)
	480
	481	#if JG
c1543a89	482	#include <x86_64/x86_64/apic_vector.S>
c8fe38ae MD	483	#endif
	484
	485	#ifdef DEV_ATPIC
	486	.data
	487	.p2align 4
	488	.text
	489	SUPERALIGN_TEXT
	490
c1543a89	491	#include <x86_64/isa/atpic_vector.S>
c8fe38ae MD	492	#endif
	493
	494	.text
	495	MCOUNT_LABEL(eintr)
	496