[dragonfly.git] / sys / platform / pc64 / x86_64 / exception.S

/*-
 * Copyright (c) 1989, 1990 William F. Jolitz.
 * Copyright (c) 1990 The Regents of the University of California.
 * Copyright (c) 2007 The FreeBSD Foundation
 * Copyright (c) 2008 The DragonFly Project.
 * Copyright (c) 2008 Jordan Gordeev.
 * All rights reserved.
 *
 * Portions of this software were developed by A. Joseph Koshy under
 * sponsorship from the FreeBSD Foundation and Google, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#if JG
#include "opt_atpic.h"
#endif
#include "opt_compat.h"

#include <machine/asmacros.h>
#include <machine/psl.h>
#include <machine/trap.h>
#include <machine/segments.h>

#include "assym.s"

	.text

	.globl	lwkt_switch_return

/*****************************************************************************/
/* Trap handling                                                             */
/*****************************************************************************/
/*
 * Trap and fault vector routines.
 *
 * All traps are 'interrupt gates', SDT_SYSIGT.  An interrupt gate pushes
 * state on the stack but also disables interrupts.  This is important for
 * us for the use of the swapgs instruction.  We cannot be interrupted
 * until the GS.base value is correct.  For most traps, we automatically
 * then enable interrupts if the interrupted context had them enabled.
 * This is equivalent to the i386 port's use of SDT_SYS386TGT.
 *
 * The cpu will push a certain amount of state onto the kernel stack for
 * the current process.  See x86_64/include/frame.h.  
 * This includes the current RFLAGS (status register, which includes 
 * the interrupt disable state prior to the trap), the code segment register,
 * and the return instruction pointer are pushed by the cpu.  The cpu 
 * will also push an 'error' code for certain traps.  We push a dummy 
 * error code for those traps where the cpu doesn't in order to maintain 
 * a consistent frame.  We also push a contrived 'trap number'.
 *
 * The cpu does not push the general registers, we must do that, and we 
 * must restore them prior to calling 'iret'.  The cpu adjusts the %cs and
 * %ss segment registers, but does not mess with %ds, %es, or %fs.  Thus we
 * must load them with appropriate values for supervisor mode operation.
 */

MCOUNT_LABEL(user)
MCOUNT_LABEL(btrap)

/*
 * Interrupts are enabled for all traps, otherwise horrible livelocks
 * can occur with the smp_invltlb and cpusync ode.
 */
#if 0
#define	TRAP_NOEN(a)	\
	subq $TF_RIP,%rsp; \
	movq $0,TF_XFLAGS(%rsp) ; \
	movq $(a),TF_TRAPNO(%rsp) ; \
	movq $0,TF_ADDR(%rsp) ; \
	movq $0,TF_ERR(%rsp) ; \
	jmp alltraps_noen
#endif

/* Regular traps; The cpu does not supply tf_err for these. */
#define	TRAP(a)	 \
	subq $TF_RIP,%rsp; \
	movq $0,TF_XFLAGS(%rsp) ; \
	movq $(a),TF_TRAPNO(%rsp) ; \
	movq $0,TF_ADDR(%rsp) ; \
	movq $0,TF_ERR(%rsp) ; \
	jmp alltraps
IDTVEC(dbg)
	TRAP(T_TRCTRAP)
IDTVEC(bpt)
	TRAP(T_BPTFLT)
IDTVEC(div)
	TRAP(T_DIVIDE)
IDTVEC(ofl)
	TRAP(T_OFLOW)
IDTVEC(bnd)
	TRAP(T_BOUND)
IDTVEC(ill)
	TRAP(T_PRIVINFLT)
IDTVEC(dna)
	TRAP(T_DNA)
IDTVEC(fpusegm)
	TRAP(T_FPOPFLT)
IDTVEC(mchk)
	TRAP(T_MCHK)
IDTVEC(rsvd)
	TRAP(T_RESERVED)
IDTVEC(fpu)
	TRAP(T_ARITHTRAP)
IDTVEC(xmm)
	TRAP(T_XMMFLT)

/* This group of traps have tf_err already pushed by the cpu */
#define	TRAP_ERR(a)	\
	subq $TF_ERR,%rsp; \
	movq $(a),TF_TRAPNO(%rsp) ; \
	movq $0,TF_ADDR(%rsp) ; \
	movq $0,TF_XFLAGS(%rsp) ; \
	jmp alltraps
IDTVEC(tss)
	TRAP_ERR(T_TSSFLT)
IDTVEC(missing)
	TRAP_ERR(T_SEGNPFLT)
IDTVEC(stk)
	TRAP_ERR(T_STKFLT)
IDTVEC(align)
	TRAP_ERR(T_ALIGNFLT)

	/*
	 * alltraps entry point.  Use swapgs if this is the first time in the
	 * kernel from userland.  Reenable interrupts if they were enabled
	 * before the trap.  This approximates SDT_SYS386TGT on the i386 port.
	 */

	SUPERALIGN_TEXT
	.globl	alltraps
	.type	alltraps,@function
alltraps:
	/* Fixup %gs if coming from userland */
	testb	$SEL_RPL_MASK,TF_CS(%rsp)
	jz	alltraps_testi
	swapgs
alltraps_testi:
	testq	$PSL_I,TF_RFLAGS(%rsp)
	jz	alltraps_pushregs
	sti
alltraps_pushregs:
	movq	%rdi,TF_RDI(%rsp)
alltraps_pushregs_no_rdi:
	movq	%rsi,TF_RSI(%rsp)
	movq	%rdx,TF_RDX(%rsp)
	movq	%rcx,TF_RCX(%rsp)
	movq	%r8,TF_R8(%rsp)
	movq	%r9,TF_R9(%rsp)
	movq	%rax,TF_RAX(%rsp)
	movq	%rbx,TF_RBX(%rsp)
	movq	%rbp,TF_RBP(%rsp)
	movq	%r10,TF_R10(%rsp)
	movq	%r11,TF_R11(%rsp)
	movq	%r12,TF_R12(%rsp)
	movq	%r13,TF_R13(%rsp)
	movq	%r14,TF_R14(%rsp)
	movq	%r15,TF_R15(%rsp)
	FAKE_MCOUNT(TF_RIP(%rsp))
	.globl	calltrap
	.type	calltrap,@function
calltrap:
	cld
	movq	%rsp, %rdi
	call	trap
	MEXITCOUNT
	jmp	doreti			/* Handle any pending ASTs */

	/*
	 * alltraps_noen entry point.  Unlike alltraps above, we want to
	 * leave the interrupts disabled.  This corresponds to
	 * SDT_SYS386IGT on the i386 port.
	 */
	SUPERALIGN_TEXT
	.globl	alltraps_noen
	.type	alltraps_noen,@function
alltraps_noen:
	/* Fixup %gs if coming from userland */
	testb	$SEL_RPL_MASK,TF_CS(%rsp)
	jz	alltraps_pushregs
	swapgs
	jmp	alltraps_pushregs

IDTVEC(dblfault)
	subq	$TF_ERR,%rsp
	movq	$T_DOUBLEFLT,TF_TRAPNO(%rsp)
	movq	$0,TF_ADDR(%rsp)
	movq	$0,TF_ERR(%rsp)
	movq	$0,TF_XFLAGS(%rsp)
	movq	%rdi,TF_RDI(%rsp)
	movq	%rsi,TF_RSI(%rsp)
	movq	%rdx,TF_RDX(%rsp)
	movq	%rcx,TF_RCX(%rsp)
	movq	%r8,TF_R8(%rsp)
	movq	%r9,TF_R9(%rsp)
	movq	%rax,TF_RAX(%rsp)
	movq	%rbx,TF_RBX(%rsp)
	movq	%rbp,TF_RBP(%rsp)
	movq	%r10,TF_R10(%rsp)
	movq	%r11,TF_R11(%rsp)
	movq	%r12,TF_R12(%rsp)
	movq	%r13,TF_R13(%rsp)
	movq	%r14,TF_R14(%rsp)
	movq	%r15,TF_R15(%rsp)
	testb	$SEL_RPL_MASK,TF_CS(%rsp)
	jz	1f
	swapgs
1:	movq	%rsp, %rdi
	cld
	call	dblfault_handler
2:	hlt
	jmp	2b

IDTVEC(page)
	subq	$TF_ERR,%rsp
	movq	$T_PAGEFLT,TF_TRAPNO(%rsp)
	/* Fixup %gs if coming from userland */
	testb	$SEL_RPL_MASK,TF_CS(%rsp)
	jz	1f
	swapgs
1:
	movq	%rdi,TF_RDI(%rsp)	/* free up a GP register */
	movq	%cr2,%rdi		/* preserve %cr2 before ..  */
	movq	%rdi,TF_ADDR(%rsp)	/* enabling interrupts. */
	movq	$0,TF_XFLAGS(%rsp)
	testq	$PSL_I,TF_RFLAGS(%rsp)
	jz	alltraps_pushregs_no_rdi
	sti
	jmp	alltraps_pushregs_no_rdi

	/*
	 * We have to special-case this one.  If we get a trap in doreti() at
	 * the iretq stage, we'll reenter with the wrong gs state.  We'll have
	 * to do a special the swapgs in this case even coming from the kernel.
	 * XXX linux has a trap handler for their equivalent of load_gs().
	 */
IDTVEC(prot)
	subq	$TF_ERR,%rsp
	movq	$T_PROTFLT,TF_TRAPNO(%rsp)
	movq	$0,TF_ADDR(%rsp)
	movq	$0,TF_XFLAGS(%rsp)
	movq	%rdi,TF_RDI(%rsp)	/* free up a GP register */

	/*
	 * Fixup %gs if coming from userland.  Handle the special case where
	 * %fs faults in doreti at the iretq instruction itself.
	 */
	leaq	doreti_iret(%rip),%rdi
	cmpq	%rdi,TF_RIP(%rsp)		/* special iretq fault case */
	je	2f
	testb	$SEL_RPL_MASK,TF_CS(%rsp)	/* check if from userland */
	jz	1f
2:
	swapgs
1:
	testq	$PSL_I,TF_RFLAGS(%rsp)
	jz	alltraps_pushregs_no_rdi
	sti
	jmp	alltraps_pushregs_no_rdi

/*
 * Fast syscall entry point.  We enter here with just our new %cs/%ss set,
 * and the new privilige level.  We are still running on the old user stack
 * pointer.  We have to juggle a few things around to find our stack etc.
 * swapgs gives us access to our PCPU space only.
 */
IDTVEC(fast_syscall)
	swapgs
	movq	%rsp,PCPU(scratch_rsp)
	movq    PCPU(common_tss) + TSS_RSP0, %rsp
	/* Now emulate a trapframe. Make the 8 byte alignment odd for call. */
	subq	$TF_SIZE,%rsp
	/* defer TF_RSP till we have a spare register */
	movq	%r11,TF_RFLAGS(%rsp)
	movq	%rcx,TF_RIP(%rsp)	/* %rcx original value is in %r10 */
	movq	PCPU(scratch_rsp),%r11	/* %r11 already saved */
	movq	%r11,TF_RSP(%rsp)	/* user stack pointer */
	orl	$RQF_QUICKRET,PCPU(reqflags)
	sti
	movq	$KUDSEL,TF_SS(%rsp)
	movq	$KUCSEL,TF_CS(%rsp)
	movq	$2,TF_ERR(%rsp)
	movq	$T_FAST_SYSCALL,TF_TRAPNO(%rsp)	/* for the vkernel */
	movq	$0,TF_XFLAGS(%rsp)	/* note: used in signal frame */
	movq	%rdi,TF_RDI(%rsp)	/* arg 1 */
	movq	%rsi,TF_RSI(%rsp)	/* arg 2 */
	movq	%rdx,TF_RDX(%rsp)	/* arg 3 */
	movq	%r10,TF_RCX(%rsp)	/* arg 4 */
	movq	%r8,TF_R8(%rsp)		/* arg 5 */
	movq	%r9,TF_R9(%rsp)		/* arg 6 */
	movq	%rax,TF_RAX(%rsp)	/* syscall number */
	movq	%rbx,TF_RBX(%rsp)	/* C preserved */
	movq	%rbp,TF_RBP(%rsp)	/* C preserved */
	movq	%r12,TF_R12(%rsp)	/* C preserved */
	movq	%r13,TF_R13(%rsp)	/* C preserved */
	movq	%r14,TF_R14(%rsp)	/* C preserved */
	movq	%r15,TF_R15(%rsp)	/* C preserved */
	FAKE_MCOUNT(TF_RIP(%rsp))
	movq	%rsp, %rdi
	call	syscall2

	/*
	 * Fast return from system call
	 */
	cli
	testl	$RQF_IPIQ|RQF_TIMER|RQF_INTPEND|RQF_AST_MASK,PCPU(reqflags)
	jnz	1f
	testl	$RQF_QUICKRET,PCPU(reqflags)
	jz	1f
	MEXITCOUNT
	movq	TF_RDI(%rsp),%rdi
	movq	TF_RSI(%rsp),%rsi
	movq	TF_RDX(%rsp),%rdx
	movq	TF_RAX(%rsp),%rax
	movq	TF_RFLAGS(%rsp),%r11
	movq	TF_RIP(%rsp),%rcx
	movq	TF_RSP(%rsp),%rsp
	swapgs
	sysretq
	/*
	 * Normal slow / full iret
	 */
1:
	MEXITCOUNT
	jmp	doreti

/*
 * Here for CYA insurance, in case a "syscall" instruction gets
 * issued from 32 bit compatability mode. MSR_CSTAR has to point
 * to *something* if EFER_SCE is enabled.
 */
IDTVEC(fast_syscall32)
	sysret

/*
 * NMI handling is special.
 *
 * First, NMIs do not respect the state of the processor's RFLAGS.IF
 * bit and the NMI handler may be invoked at any time, including when
 * the processor is in a critical section with RFLAGS.IF == 0.  In
 * particular, this means that the processor's GS.base values could be
 * inconsistent on entry to the handler, and so we need to read
 * MSR_GSBASE to determine if a 'swapgs' is needed.  We use '%ebx', a
 * C-preserved register, to remember whether to swap GS back on the
 * exit path.
 *
 * Second, the processor treats NMIs specially, blocking further NMIs
 * until an 'iretq' instruction is executed.  We therefore need to
 * execute the NMI handler with interrupts disabled to prevent a
 * nested interrupt from executing an 'iretq' instruction and
 * inadvertently taking the processor out of NMI mode.
 *
 * Third, the NMI handler runs on its own stack (tss_ist1), shared
 * with the double fault handler.
 */

IDTVEC(nmi)
	subq	$TF_RIP,%rsp
	movq	$(T_NMI),TF_TRAPNO(%rsp)
	movq	$0,TF_ADDR(%rsp)
	movq	$0,TF_ERR(%rsp)
	movq	$0,TF_XFLAGS(%rsp)
	movq	%rdi,TF_RDI(%rsp)
	movq	%rsi,TF_RSI(%rsp)
	movq	%rdx,TF_RDX(%rsp)
	movq	%rcx,TF_RCX(%rsp)
	movq	%r8,TF_R8(%rsp)
	movq	%r9,TF_R9(%rsp)
	movq	%rax,TF_RAX(%rsp)
	movq	%rbx,TF_RBX(%rsp)
	movq	%rbp,TF_RBP(%rsp)
	movq	%r10,TF_R10(%rsp)
	movq	%r11,TF_R11(%rsp)
	movq	%r12,TF_R12(%rsp)
	movq	%r13,TF_R13(%rsp)
	movq	%r14,TF_R14(%rsp)
	movq	%r15,TF_R15(%rsp)
	xorl	%ebx,%ebx
	testb	$SEL_RPL_MASK,TF_CS(%rsp)
	jnz	nmi_needswapgs		/* we came from userland */
	movl	$MSR_GSBASE,%ecx
	rdmsr
	cmpl	$VM_MAX_USER_ADDRESS >> 32,%edx
	jae	nmi_calltrap		/* GS.base holds a kernel VA */
nmi_needswapgs:
	incl	%ebx
	swapgs
/* Note: this label is also used by ddb and gdb: */
nmi_calltrap:
	FAKE_MCOUNT(TF_RIP(%rsp))
	cld
	movq	%rsp, %rdi
	call	trap
	MEXITCOUNT
	testl	%ebx,%ebx
	jz	nmi_restoreregs
	swapgs
nmi_restoreregs:
	movq	TF_RDI(%rsp),%rdi
	movq	TF_RSI(%rsp),%rsi
	movq	TF_RDX(%rsp),%rdx
	movq	TF_RCX(%rsp),%rcx
	movq	TF_R8(%rsp),%r8
	movq	TF_R9(%rsp),%r9
	movq	TF_RAX(%rsp),%rax
	movq	TF_RBX(%rsp),%rbx
	movq	TF_RBP(%rsp),%rbp
	movq	TF_R10(%rsp),%r10
	movq	TF_R11(%rsp),%r11
	movq	TF_R12(%rsp),%r12
	movq	TF_R13(%rsp),%r13
	movq	TF_R14(%rsp),%r14
	movq	TF_R15(%rsp),%r15
	addq	$TF_RIP,%rsp
	iretq

/*
 * This function is what cpu_heavy_restore jumps to after a new process
 * is created.  The LWKT subsystem switches while holding a critical
 * section and we maintain that abstraction here (e.g. because 
 * cpu_heavy_restore needs it due to PCB_*() manipulation), then get out of
 * it before calling the initial function (typically fork_return()) and/or
 * returning to user mode.
 *
 * The MP lock is not held at any point but the critcount is bumped
 * on entry to prevent interruption of the trampoline at a bad point.
 *
 * This is effectively what td->td_switch() returns to.  It 'returns' the
 * old thread in %rax and since this is not returning to a td->td_switch()
 * call from lwkt_switch() we must handle the cleanup for the old thread
 * by calling lwkt_switch_return().
 *
 * fork_trampoline(%rax:otd, %rbx:func, %r12:arg)
 */
ENTRY(fork_trampoline)
	movq	%rax,%rdi
	call	lwkt_switch_return
	movq	PCPU(curthread),%rax
	decl	TD_CRITCOUNT(%rax)

	/*
	 * cpu_set_fork_handler intercepts this function call to
	 * have this call a non-return function to stay in kernel mode.
	 *
	 * initproc has its own fork handler, start_init(), which DOES
	 * return.
	 *
	 * %rbx - chaining function (typically fork_return)
	 * %r12 -> %rdi (argument)
	 * frame-> %rsi (trap frame)
	 *
	 *   void (func:rbx)(arg:rdi, trapframe:rsi)
	 */
	movq	%rsp, %rsi		/* pass trapframe by reference */
	movq	%r12, %rdi		/* arg1 */
	call	*%rbx			/* function */

	/* cut from syscall */

	sti
	call	splz

	/*
	 * Return via doreti to handle ASTs.
	 *
	 * trapframe is at the top of the stack.
	 */
	MEXITCOUNT
	jmp	doreti

/*
 * To efficiently implement classification of trap and interrupt handlers
 * for profiling, there must be only trap handlers between the labels btrap
 * and bintr, and only interrupt handlers between the labels bintr and
 * eintr.  This is implemented (partly) by including files that contain
 * some of the handlers.  Before including the files, set up a normal asm
 * environment so that the included files doen't need to know that they are
 * included.
 */

#ifdef COMPAT_IA32
	.data
	.p2align 4
	.text
	SUPERALIGN_TEXT

#include <x86_64/ia32/ia32_exception.S>
#endif

	.data
	.p2align 4
	.text
	SUPERALIGN_TEXT
MCOUNT_LABEL(bintr)

#if JG
#include <x86_64/x86_64/apic_vector.S>
#endif

#ifdef DEV_ATPIC
	.data
	.p2align 4
	.text
	SUPERALIGN_TEXT

#include <x86_64/isa/atpic_vector.S>
#endif

	.text
MCOUNT_LABEL(eintr)
Commit	Line	Data
c8fe38ae MD	1	/*-
	2	* Copyright (c) 1989, 1990 William F. Jolitz.
	3	* Copyright (c) 1990 The Regents of the University of California.
	4	* Copyright (c) 2007 The FreeBSD Foundation
	5	* Copyright (c) 2008 The DragonFly Project.
	6	* Copyright (c) 2008 Jordan Gordeev.
	7	* All rights reserved.
	8	*
	9	* Portions of this software were developed by A. Joseph Koshy under
	10	* sponsorship from the FreeBSD Foundation and Google, Inc.
	11	*
	12	* Redistribution and use in source and binary forms, with or without
	13	* modification, are permitted provided that the following conditions
	14	* are met:
	15	* 1. Redistributions of source code must retain the above copyright
	16	* notice, this list of conditions and the following disclaimer.
	17	* 2. Redistributions in binary form must reproduce the above copyright
	18	* notice, this list of conditions and the following disclaimer in the
	19	* documentation and/or other materials provided with the distribution.
	20	* 4. Neither the name of the University nor the names of its contributors
	21	* may be used to endorse or promote products derived from this software
	22	* without specific prior written permission.
	23	*
	24	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	25	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	26	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	27	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	28	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	29	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	30	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	31	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	32	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	33	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	34	* SUCH DAMAGE.
c8fe38ae MD	35	*/
	36
	37	#if JG
	38	#include "opt_atpic.h"
	39	#endif
	40	#include "opt_compat.h"
	41
	42	#include <machine/asmacros.h>
	43	#include <machine/psl.h>
	44	#include <machine/trap.h>
bfc09ba0	45	#include <machine/segments.h>
c8fe38ae MD	46
	47	#include "assym.s"
	48
	49	.text
	50
cc9b6223 MD	51	.globl lwkt_switch_return
cc9b6223 MD	52
c8fe38ae MD	53	/*****************************************************************************/
	54	/* Trap handling */
	55	/*****************************************************************************/
	56	/*
	57	* Trap and fault vector routines.
	58	*
	59	* All traps are 'interrupt gates', SDT_SYSIGT. An interrupt gate pushes
	60	* state on the stack but also disables interrupts. This is important for
	61	* us for the use of the swapgs instruction. We cannot be interrupted
	62	* until the GS.base value is correct. For most traps, we automatically
	63	* then enable interrupts if the interrupted context had them enabled.
	64	* This is equivalent to the i386 port's use of SDT_SYS386TGT.
	65	*
	66	* The cpu will push a certain amount of state onto the kernel stack for
b2b3ffcd	67	* the current process. See x86_64/include/frame.h.
c8fe38ae MD	68	* This includes the current RFLAGS (status register, which includes
	69	* the interrupt disable state prior to the trap), the code segment register,
	70	* and the return instruction pointer are pushed by the cpu. The cpu
	71	* will also push an 'error' code for certain traps. We push a dummy
	72	* error code for those traps where the cpu doesn't in order to maintain
	73	* a consistent frame. We also push a contrived 'trap number'.
	74	*
	75	* The cpu does not push the general registers, we must do that, and we
	76	* must restore them prior to calling 'iret'. The cpu adjusts the %cs and
	77	* %ss segment registers, but does not mess with %ds, %es, or %fs. Thus we
	78	* must load them with appropriate values for supervisor mode operation.
	79	*/
	80
	81	MCOUNT_LABEL(user)
	82	MCOUNT_LABEL(btrap)
	83
bd52bedf MD	84	/*
	85	* Interrupts are enabled for all traps, otherwise horrible livelocks
	86	* can occur with the smp_invltlb and cpusync ode.
	87	*/
	88	#if 0
c8fe38ae MD	89	#define TRAP_NOEN(a) \
c8fe38ae MD	90	subq $TF_RIP,%rsp; \
5b9f6cc4	91	movq $0,TF_XFLAGS(%rsp) ; \
c8fe38ae MD	92	movq $(a),TF_TRAPNO(%rsp) ; \
	93	movq $0,TF_ADDR(%rsp) ; \
	94	movq $0,TF_ERR(%rsp) ; \
	95	jmp alltraps_noen
bd52bedf	96	#endif
c8fe38ae MD	97
	98	/* Regular traps; The cpu does not supply tf_err for these. */
	99	#define TRAP(a) \
	100	subq $TF_RIP,%rsp; \
5b9f6cc4	101	movq $0,TF_XFLAGS(%rsp) ; \
c8fe38ae MD	102	movq $(a),TF_TRAPNO(%rsp) ; \
	103	movq $0,TF_ADDR(%rsp) ; \
	104	movq $0,TF_ERR(%rsp) ; \
	105	jmp alltraps
bd52bedf MD	106	IDTVEC(dbg)
	107	TRAP(T_TRCTRAP)
	108	IDTVEC(bpt)
	109	TRAP(T_BPTFLT)
c8fe38ae MD	110	IDTVEC(div)
	111	TRAP(T_DIVIDE)
	112	IDTVEC(ofl)
	113	TRAP(T_OFLOW)
	114	IDTVEC(bnd)
	115	TRAP(T_BOUND)
	116	IDTVEC(ill)
	117	TRAP(T_PRIVINFLT)
	118	IDTVEC(dna)
	119	TRAP(T_DNA)
	120	IDTVEC(fpusegm)
	121	TRAP(T_FPOPFLT)
	122	IDTVEC(mchk)
	123	TRAP(T_MCHK)
	124	IDTVEC(rsvd)
	125	TRAP(T_RESERVED)
	126	IDTVEC(fpu)
	127	TRAP(T_ARITHTRAP)
	128	IDTVEC(xmm)
	129	TRAP(T_XMMFLT)
	130
	131	/* This group of traps have tf_err already pushed by the cpu */
	132	#define TRAP_ERR(a) \
	133	subq $TF_ERR,%rsp; \
	134	movq $(a),TF_TRAPNO(%rsp) ; \
	135	movq $0,TF_ADDR(%rsp) ; \
16094a36	136	movq $0,TF_XFLAGS(%rsp) ; \
c8fe38ae MD	137	jmp alltraps
	138	IDTVEC(tss)
	139	TRAP_ERR(T_TSSFLT)
	140	IDTVEC(missing)
	141	TRAP_ERR(T_SEGNPFLT)
	142	IDTVEC(stk)
	143	TRAP_ERR(T_STKFLT)
	144	IDTVEC(align)
	145	TRAP_ERR(T_ALIGNFLT)
	146
	147	/*
	148	* alltraps entry point. Use swapgs if this is the first time in the
	149	* kernel from userland. Reenable interrupts if they were enabled
	150	* before the trap. This approximates SDT_SYS386TGT on the i386 port.
	151	*/
	152
	153	SUPERALIGN_TEXT
	154	.globl alltraps
	155	.type alltraps,@function
	156	alltraps:
2883d2d8 MD	157	/* Fixup %gs if coming from userland */
	158	testb $SEL_RPL_MASK,TF_CS(%rsp)
	159	jz alltraps_testi
c8fe38ae MD	160	swapgs
	161	alltraps_testi:
	162	testq $PSL_I,TF_RFLAGS(%rsp)
	163	jz alltraps_pushregs
	164	sti
	165	alltraps_pushregs:
	166	movq %rdi,TF_RDI(%rsp)
	167	alltraps_pushregs_no_rdi:
	168	movq %rsi,TF_RSI(%rsp)
	169	movq %rdx,TF_RDX(%rsp)
	170	movq %rcx,TF_RCX(%rsp)
	171	movq %r8,TF_R8(%rsp)
	172	movq %r9,TF_R9(%rsp)
	173	movq %rax,TF_RAX(%rsp)
	174	movq %rbx,TF_RBX(%rsp)
	175	movq %rbp,TF_RBP(%rsp)
	176	movq %r10,TF_R10(%rsp)
	177	movq %r11,TF_R11(%rsp)
	178	movq %r12,TF_R12(%rsp)
	179	movq %r13,TF_R13(%rsp)
	180	movq %r14,TF_R14(%rsp)
	181	movq %r15,TF_R15(%rsp)
	182	FAKE_MCOUNT(TF_RIP(%rsp))
	183	.globl calltrap
	184	.type calltrap,@function
	185	calltrap:
3338cc67	186	cld
c8fe38ae MD	187	movq %rsp, %rdi
	188	call trap
	189	MEXITCOUNT
	190	jmp doreti /* Handle any pending ASTs */
	191
	192	/*
	193	* alltraps_noen entry point. Unlike alltraps above, we want to
	194	* leave the interrupts disabled. This corresponds to
	195	* SDT_SYS386IGT on the i386 port.
	196	*/
	197	SUPERALIGN_TEXT
	198	.globl alltraps_noen
	199	.type alltraps_noen,@function
	200	alltraps_noen:
2883d2d8 MD	201	/* Fixup %gs if coming from userland */
	202	testb $SEL_RPL_MASK,TF_CS(%rsp)
	203	jz alltraps_pushregs
c8fe38ae MD	204	swapgs
	205	jmp alltraps_pushregs
	206
	207	IDTVEC(dblfault)
	208	subq $TF_ERR,%rsp
	209	movq $T_DOUBLEFLT,TF_TRAPNO(%rsp)
	210	movq $0,TF_ADDR(%rsp)
	211	movq $0,TF_ERR(%rsp)
16094a36	212	movq $0,TF_XFLAGS(%rsp)
c8fe38ae MD	213	movq %rdi,TF_RDI(%rsp)
	214	movq %rsi,TF_RSI(%rsp)
	215	movq %rdx,TF_RDX(%rsp)
	216	movq %rcx,TF_RCX(%rsp)
	217	movq %r8,TF_R8(%rsp)
	218	movq %r9,TF_R9(%rsp)
	219	movq %rax,TF_RAX(%rsp)
	220	movq %rbx,TF_RBX(%rsp)
	221	movq %rbp,TF_RBP(%rsp)
	222	movq %r10,TF_R10(%rsp)
	223	movq %r11,TF_R11(%rsp)
	224	movq %r12,TF_R12(%rsp)
	225	movq %r13,TF_R13(%rsp)
	226	movq %r14,TF_R14(%rsp)
	227	movq %r15,TF_R15(%rsp)
2883d2d8 MD	228	testb $SEL_RPL_MASK,TF_CS(%rsp)
2883d2d8 MD	229	jz 1f
c8fe38ae MD	230	swapgs
c8fe38ae MD	231	1: movq %rsp, %rdi
3338cc67	232	cld
c8fe38ae MD	233	call dblfault_handler
	234	2: hlt
	235	jmp 2b
	236
	237	IDTVEC(page)
	238	subq $TF_ERR,%rsp
	239	movq $T_PAGEFLT,TF_TRAPNO(%rsp)
2883d2d8 MD	240	/* Fixup %gs if coming from userland */
	241	testb $SEL_RPL_MASK,TF_CS(%rsp)
	242	jz 1f
c8fe38ae MD	243	swapgs
	244	1:
	245	movq %rdi,TF_RDI(%rsp) /* free up a GP register */
	246	movq %cr2,%rdi /* preserve %cr2 before .. */
	247	movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */
16094a36	248	movq $0,TF_XFLAGS(%rsp)
c8fe38ae MD	249	testq $PSL_I,TF_RFLAGS(%rsp)
	250	jz alltraps_pushregs_no_rdi
	251	sti
	252	jmp alltraps_pushregs_no_rdi
	253
	254	/*
	255	* We have to special-case this one. If we get a trap in doreti() at
	256	* the iretq stage, we'll reenter with the wrong gs state. We'll have
	257	* to do a special the swapgs in this case even coming from the kernel.
	258	* XXX linux has a trap handler for their equivalent of load_gs().
	259	*/
	260	IDTVEC(prot)
	261	subq $TF_ERR,%rsp
	262	movq $T_PROTFLT,TF_TRAPNO(%rsp)
	263	movq $0,TF_ADDR(%rsp)
16094a36	264	movq $0,TF_XFLAGS(%rsp)
c8fe38ae	265	movq %rdi,TF_RDI(%rsp) /* free up a GP register */
2883d2d8 MD	266
	267	/*
	268	* Fixup %gs if coming from userland. Handle the special case where
	269	* %fs faults in doreti at the iretq instruction itself.
	270	*/
c8fe38ae	271	leaq doreti_iret(%rip),%rdi
2883d2d8 MD	272	cmpq %rdi,TF_RIP(%rsp) /* special iretq fault case */
	273	je 2f
	274	testb $SEL_RPL_MASK,TF_CS(%rsp) /* check if from userland */
	275	jz 1f
c8fe38ae MD	276	2:
	277	swapgs
	278	1:
	279	testq $PSL_I,TF_RFLAGS(%rsp)
	280	jz alltraps_pushregs_no_rdi
	281	sti
	282	jmp alltraps_pushregs_no_rdi
	283
	284	/*
	285	* Fast syscall entry point. We enter here with just our new %cs/%ss set,
	286	* and the new privilige level. We are still running on the old user stack
	287	* pointer. We have to juggle a few things around to find our stack etc.
	288	* swapgs gives us access to our PCPU space only.
	289	*/
	290	IDTVEC(fast_syscall)
	291	swapgs
	292	movq %rsp,PCPU(scratch_rsp)
d1368d1a	293	movq PCPU(common_tss) + TSS_RSP0, %rsp
c8fe38ae MD	294	/* Now emulate a trapframe. Make the 8 byte alignment odd for call. */
	295	subq $TF_SIZE,%rsp
	296	/* defer TF_RSP till we have a spare register */
	297	movq %r11,TF_RFLAGS(%rsp)
	298	movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */
	299	movq PCPU(scratch_rsp),%r11 /* %r11 already saved */
	300	movq %r11,TF_RSP(%rsp) /* user stack pointer */
f2081646	301	orl $RQF_QUICKRET,PCPU(reqflags)
c8fe38ae MD	302	sti
	303	movq $KUDSEL,TF_SS(%rsp)
	304	movq $KUCSEL,TF_CS(%rsp)
	305	movq $2,TF_ERR(%rsp)
571cdd83	306	movq $T_FAST_SYSCALL,TF_TRAPNO(%rsp) /* for the vkernel */
16094a36	307	movq $0,TF_XFLAGS(%rsp) /* note: used in signal frame */
c8fe38ae MD	308	movq %rdi,TF_RDI(%rsp) /* arg 1 */
	309	movq %rsi,TF_RSI(%rsp) /* arg 2 */
	310	movq %rdx,TF_RDX(%rsp) /* arg 3 */
	311	movq %r10,TF_RCX(%rsp) /* arg 4 */
	312	movq %r8,TF_R8(%rsp) /* arg 5 */
	313	movq %r9,TF_R9(%rsp) /* arg 6 */
	314	movq %rax,TF_RAX(%rsp) /* syscall number */
	315	movq %rbx,TF_RBX(%rsp) /* C preserved */
	316	movq %rbp,TF_RBP(%rsp) /* C preserved */
	317	movq %r12,TF_R12(%rsp) /* C preserved */
	318	movq %r13,TF_R13(%rsp) /* C preserved */
	319	movq %r14,TF_R14(%rsp) /* C preserved */
	320	movq %r15,TF_R15(%rsp) /* C preserved */
	321	FAKE_MCOUNT(TF_RIP(%rsp))
	322	movq %rsp, %rdi
	323	call syscall2
f2081646 MD	324
	325	/*
	326	* Fast return from system call
	327	*/
	328	cli
	329	testl $RQF_IPIQ\|RQF_TIMER\|RQF_INTPEND\|RQF_AST_MASK,PCPU(reqflags)
	330	jnz 1f
	331	testl $RQF_QUICKRET,PCPU(reqflags)
	332	jz 1f
	333	MEXITCOUNT
	334	movq TF_RDI(%rsp),%rdi
	335	movq TF_RSI(%rsp),%rsi
	336	movq TF_RDX(%rsp),%rdx
	337	movq TF_RAX(%rsp),%rax
	338	movq TF_RFLAGS(%rsp),%r11
	339	movq TF_RIP(%rsp),%rcx
	340	movq TF_RSP(%rsp),%rsp
	341	swapgs
	342	sysretq
	343	/*
	344	* Normal slow / full iret
	345	*/
	346	1:
c8fe38ae MD	347	MEXITCOUNT
	348	jmp doreti
	349
	350	/*
	351	* Here for CYA insurance, in case a "syscall" instruction gets
	352	* issued from 32 bit compatability mode. MSR_CSTAR has to point
	353	* to something if EFER_SCE is enabled.
	354	*/
	355	IDTVEC(fast_syscall32)
	356	sysret
	357
	358	/*
	359	* NMI handling is special.
	360	*
	361	* First, NMIs do not respect the state of the processor's RFLAGS.IF
	362	* bit and the NMI handler may be invoked at any time, including when
	363	* the processor is in a critical section with RFLAGS.IF == 0. In
	364	* particular, this means that the processor's GS.base values could be
	365	* inconsistent on entry to the handler, and so we need to read
	366	* MSR_GSBASE to determine if a 'swapgs' is needed. We use '%ebx', a
	367	* C-preserved register, to remember whether to swap GS back on the
	368	* exit path.
	369	*
	370	* Second, the processor treats NMIs specially, blocking further NMIs
	371	* until an 'iretq' instruction is executed. We therefore need to
	372	* execute the NMI handler with interrupts disabled to prevent a
	373	* nested interrupt from executing an 'iretq' instruction and
	374	* inadvertently taking the processor out of NMI mode.
	375	*
	376	* Third, the NMI handler runs on its own stack (tss_ist1), shared
	377	* with the double fault handler.
	378	*/
	379
	380	IDTVEC(nmi)
	381	subq $TF_RIP,%rsp
	382	movq $(T_NMI),TF_TRAPNO(%rsp)
	383	movq $0,TF_ADDR(%rsp)
	384	movq $0,TF_ERR(%rsp)
16094a36	385	movq $0,TF_XFLAGS(%rsp)
c8fe38ae MD	386	movq %rdi,TF_RDI(%rsp)
	387	movq %rsi,TF_RSI(%rsp)
	388	movq %rdx,TF_RDX(%rsp)
	389	movq %rcx,TF_RCX(%rsp)
	390	movq %r8,TF_R8(%rsp)
	391	movq %r9,TF_R9(%rsp)
	392	movq %rax,TF_RAX(%rsp)
	393	movq %rbx,TF_RBX(%rsp)
	394	movq %rbp,TF_RBP(%rsp)
	395	movq %r10,TF_R10(%rsp)
	396	movq %r11,TF_R11(%rsp)
	397	movq %r12,TF_R12(%rsp)
	398	movq %r13,TF_R13(%rsp)
	399	movq %r14,TF_R14(%rsp)
	400	movq %r15,TF_R15(%rsp)
	401	xorl %ebx,%ebx
	402	testb $SEL_RPL_MASK,TF_CS(%rsp)
	403	jnz nmi_needswapgs /* we came from userland */
	404	movl $MSR_GSBASE,%ecx
	405	rdmsr
48ffc236	406	cmpl $VM_MAX_USER_ADDRESS >> 32,%edx
c8fe38ae MD	407	jae nmi_calltrap /* GS.base holds a kernel VA */
	408	nmi_needswapgs:
	409	incl %ebx
	410	swapgs
	411	/* Note: this label is also used by ddb and gdb: */
	412	nmi_calltrap:
	413	FAKE_MCOUNT(TF_RIP(%rsp))
3338cc67	414	cld
c8fe38ae MD	415	movq %rsp, %rdi
	416	call trap
	417	MEXITCOUNT
	418	testl %ebx,%ebx
	419	jz nmi_restoreregs
	420	swapgs
	421	nmi_restoreregs:
	422	movq TF_RDI(%rsp),%rdi
	423	movq TF_RSI(%rsp),%rsi
	424	movq TF_RDX(%rsp),%rdx
	425	movq TF_RCX(%rsp),%rcx
	426	movq TF_R8(%rsp),%r8
	427	movq TF_R9(%rsp),%r9
	428	movq TF_RAX(%rsp),%rax
	429	movq TF_RBX(%rsp),%rbx
	430	movq TF_RBP(%rsp),%rbp
	431	movq TF_R10(%rsp),%r10
	432	movq TF_R11(%rsp),%r11
	433	movq TF_R12(%rsp),%r12
	434	movq TF_R13(%rsp),%r13
	435	movq TF_R14(%rsp),%r14
	436	movq TF_R15(%rsp),%r15
	437	addq $TF_RIP,%rsp
	438	iretq
	439
	440	/*
	441	* This function is what cpu_heavy_restore jumps to after a new process
	442	* is created. The LWKT subsystem switches while holding a critical
	443	* section and we maintain that abstraction here (e.g. because
	444	* cpu_heavy_restore needs it due to PCB_*() manipulation), then get out of
	445	* it before calling the initial function (typically fork_return()) and/or
	446	* returning to user mode.
	447	*
2b0bd8aa MD	448	* The MP lock is not held at any point but the critcount is bumped
2b0bd8aa MD	449	* on entry to prevent interruption of the trampoline at a bad point.
cc9b6223 MD	450	*
	451	* This is effectively what td->td_switch() returns to. It 'returns' the
	452	* old thread in %rax and since this is not returning to a td->td_switch()
	453	* call from lwkt_switch() we must handle the cleanup for the old thread
	454	* by calling lwkt_switch_return().
	455	*
	456	* fork_trampoline(%rax:otd, %rbx:func, %r12:arg)
c8fe38ae MD	457	*/
c8fe38ae MD	458	ENTRY(fork_trampoline)
cc9b6223 MD	459	movq %rax,%rdi
cc9b6223 MD	460	call lwkt_switch_return
c8fe38ae	461	movq PCPU(curthread),%rax
f9235b6d	462	decl TD_CRITCOUNT(%rax)
c8fe38ae MD	463
	464	/*
	465	* cpu_set_fork_handler intercepts this function call to
	466	* have this call a non-return function to stay in kernel mode.
	467	*
	468	* initproc has its own fork handler, start_init(), which DOES
	469	* return.
	470	*
	471	* %rbx - chaining function (typically fork_return)
	472	* %r12 -> %rdi (argument)
	473	* frame-> %rsi (trap frame)
	474	*
	475	* void (func:rbx)(arg:rdi, trapframe:rsi)
	476	*/
	477	movq %rsp, %rsi /* pass trapframe by reference */
	478	movq %r12, %rdi /* arg1 */
	479	call %rbx / function */
	480
	481	/* cut from syscall */
	482
	483	sti
	484	call splz
	485
c8fe38ae MD	486	/*
	487	* Return via doreti to handle ASTs.
	488	*
	489	* trapframe is at the top of the stack.
	490	*/
c8fe38ae MD	491	MEXITCOUNT
	492	jmp doreti
	493
	494	/*
	495	* To efficiently implement classification of trap and interrupt handlers
	496	* for profiling, there must be only trap handlers between the labels btrap
	497	* and bintr, and only interrupt handlers between the labels bintr and
	498	* eintr. This is implemented (partly) by including files that contain
	499	* some of the handlers. Before including the files, set up a normal asm
	500	* environment so that the included files doen't need to know that they are
	501	* included.
	502	*/
	503
	504	#ifdef COMPAT_IA32
	505	.data
	506	.p2align 4
	507	.text
	508	SUPERALIGN_TEXT
	509
b2b3ffcd	510	#include <x86_64/ia32/ia32_exception.S>
c8fe38ae MD	511	#endif
	512
	513	.data
	514	.p2align 4
	515	.text
	516	SUPERALIGN_TEXT
	517	MCOUNT_LABEL(bintr)
	518
	519	#if JG
b2b3ffcd	520	#include <x86_64/x86_64/apic_vector.S>
c8fe38ae MD	521	#endif
	522
	523	#ifdef DEV_ATPIC
	524	.data
	525	.p2align 4
	526	.text
	527	SUPERALIGN_TEXT
	528
b2b3ffcd	529	#include <x86_64/isa/atpic_vector.S>
c8fe38ae MD	530	#endif
	531
	532	.text
	533	MCOUNT_LABEL(eintr)
	534