[dragonfly.git] / sys / platform / pc64 / x86_64 / ipl.s

/*
 * Copyright (c) 2008 The DragonFly Project.  All rights reserved.
 * 
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * 
 * ---
 *
 * Copyright (c) 1989, 1990 William F. Jolitz.
 * Copyright (c) 1990 The Regents of the University of California.
 * All rights reserved.
 *
 * This code is derived from software contributed to Berkeley by
 * William Jolitz.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by the University of
 *	California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)ipl.s
 *
 * $FreeBSD: src/sys/i386/isa/ipl.s,v 1.32.2.3 2002/05/16 16:03:56 bde Exp $
 */

#include <machine/asmacros.h>
#include <machine/segments.h>
#include <machine/ipl.h>
#include <machine/lock.h>
#include <machine/psl.h>
#include <machine/trap.h>
 
#include "assym.s"

/*
 * AT/386
 * Vector interrupt control section
 *
 *  ipending	- Pending interrupts (set when a masked interrupt occurs)
 *  spending	- Pending software interrupts
 */
	.data
	ALIGN_DATA

	.globl		fastunpend_count
fastunpend_count:	.long	0

	.text
	SUPERALIGN_TEXT

	/*
	 * GENERAL NOTES
	 *
	 *	- interrupts are always called with a critical section held
	 *
	 *	- we release our critical section when scheduling interrupt
	 *	  or softinterrupt threads in order so they can preempt
	 *	  (unless we are called manually from a critical section, in
	 *	  which case there will still be a critical section and
	 *	  they won't preempt anyway).
	 *
	 *	- TD_NEST_COUNT prevents splz from nesting too deeply within
	 *	  itself.  It is *not* actually an interrupt nesting count.
	 *	  PCPU(intr_nesting_level) is an interrupt nesting count.
	 *
	 *	- We have to be careful in regards to local interrupts
	 *	  occuring simultaniously with our doreti and splz 
	 *	  processing.
	 *
	 *	- Interrupts must be enabled when calling higher level
	 *	  functions in order to avoid deadlocking against things
	 *	  like smp_invltlb.
	 */

	/*
	 * DORETI
	 *
	 * Handle return from interrupts, traps and syscalls.  This function
	 * checks the cpl for unmasked pending interrupts (hardware or soft)
	 * and schedules them if appropriate, then irets.
	 *
	 * If we are in a critical section we cannot run any pending ints.
	 *
	 * The stack contains a trapframe at the start of doreti.
	 */
	SUPERALIGN_TEXT
	.globl	doreti
	.type	doreti,@function
doreti:
	FAKE_MCOUNT(bintr)		/* init "from" bintr -> doreti */
	movq	$0,%rax			/* irq mask unavailable due to BGL */
	movq	PCPU(curthread),%rbx
	cli				/* interlock with critical section */
	cmpl	$0,PCPU(reqflags)	/* short cut if nothing to do */
	je	5f
	testl	$-1,TD_CRITCOUNT(%rbx)	/* can't unpend if in critical sec */
	jne	5f
	incl	TD_CRITCOUNT(%rbx)	/* force all ints to pending */
doreti_next:
	cli				/* re-assert cli on loop */
	movq	%rax,%rcx		/* irq mask unavailable due to BGL */
	notq	%rcx
#ifdef SMP
	testl	$RQF_IPIQ,PCPU(reqflags)
	jnz	doreti_ipiq
#endif
	testl	$RQF_TIMER,PCPU(reqflags)
	jnz	doreti_timer
	/*
	 * check for an unmasked int (3 groups)
	 */
	movq	$0,%rdx
	testq	PCPU_E8(ipending,%rdx),%rcx
	jnz	doreti_fast

	movq	$1,%rdx
	testq	PCPU_E8(ipending,%rdx),%rcx
	jnz	doreti_fast

	movq	$2,%rdx
	testq	PCPU_E8(ipending,%rdx),%rcx
	jnz	doreti_fast

	movl	PCPU(spending),%ecx	/* check for a pending software int */
	cmpl	$0,%ecx
	jnz	doreti_soft

	testl	$RQF_AST_MASK,PCPU(reqflags) /* any pending ASTs? */
	jz	2f

	/* ASTs are only applicable when returning to userland */
	testb	$SEL_RPL_MASK,TF_CS(%rsp)
	jnz	doreti_ast
2:
	/*
	 * Nothing left to do, finish up.  Interrupts are still disabled.
	 * %eax contains the mask of IRQ's that are not available due to
	 * BGL requirements.  We can only clear RQF_INTPEND if *ALL* pending
	 * interrupts have been processed.
	 */
	decl	TD_CRITCOUNT(%rbx)	/* interlocked with cli */
	testl	%eax,%eax
	jnz	5f
	andl	$~RQF_INTPEND,PCPU(reqflags)
5:
	MEXITCOUNT

	/*
	 * (interrupts are disabled here)
	 *
	 * Restore register and iret.  iret can fault on %rip (which is
	 * really stupid).  If this occurs we re-fault and vector to
	 * doreti_iret_fault().
	 *
	 * ...
	 * can be set from user mode, this can result in a kernel mode
	 * exception.  The trap code will revector to the *_fault code
	 * which then sets up a T_PROTFLT signal.  If the signal is
	 * sent to userland, sendsig() will automatically clean up all
	 * the segment registers to avoid a loop.
	 */
	.globl	doreti_iret
	.globl	doreti_syscall_ret
doreti_syscall_ret:
	POP_FRAME		/* registers and %gs (+cli) */
	/* WARNING: special global doreti_iret is  also used by exception.S */
doreti_iret:
	iretq

	/*
	 * doreti_iret_fault.  Alternative return code for the case where
	 * we get a fault in the doreti_exit code above.  trap()
	 * (sys/platform/pc64/x86_64/trap.c) catches this specific * case,
	 * sends the process a signal and continues in the corresponding
	 * place in the code below.
	 *
	 * Interrupts are likely disabled due to the above interlock
	 * between cli/iretq.  We must enable them before calling any
	 * high level function.
	 */
	ALIGN_TEXT
	.globl	doreti_iret_fault
doreti_iret_fault:
	PUSH_FRAME_NOSWAP
	sti
	movq	$T_PROTFLT,TF_TRAPNO(%rsp)
	movq	$0,TF_ERR(%rsp)	/* XXX should be the error code */
	movq	$0,TF_ADDR(%rsp)
	FAKE_MCOUNT(TF_RIP(%rsp))
	jmp	calltrap

	/*
	 * Interrupt pending.  NOTE: stack context holds frame structure
	 * for interrupt procedure, do not do random pushes or pops!
	 */
	ALIGN_TEXT
doreti_fast:
	andq	PCPU_E8(ipending,%rdx),%rcx
	sti
	bsfq	%rcx, %rcx		/* locate the next dispatchable int */
	btrq	%rcx, PCPU_E8(ipending,%rdx)
					/* is it really still pending? */
	jnc	doreti_next

	shlq	$6, %rdx
	orq	%rdx, %rcx		/* form intr number */

	pushq	%rax			/* save IRQ mask unavailable for BGL */
					/* NOTE: is also CPL in frame */
	call	dofastunpend		/* unpend intr %rcx */
	popq	%rax
	jmp	doreti_next

	/*
	 *  SOFT interrupt pending
	 *
	 *  Temporarily back-out our critical section to allow an interrupt
	 *  preempt us when we schedule it.  Bump intr_nesting_level to
	 *  prevent the switch code from recursing via splz too deeply.
	 */
	ALIGN_TEXT
doreti_soft:
	sti
	bsfl	%ecx,%ecx		/* locate the next pending softint */
	btrl	%ecx,PCPU(spending)	/* make sure its still pending */
	jnc	doreti_next
	addl	$FIRST_SOFTINT,%ecx	/* actual intr number */
	pushq	%rax
	movl	%ecx,%edi		/* argument to C call */
	incl	TD_NEST_COUNT(%rbx)	/* prevent doreti/splz nesting */
	decl	TD_CRITCOUNT(%rbx)	/* so we can preempt */
	call	sched_ithd		/* YYY must pull in imasks */
	incl	TD_CRITCOUNT(%rbx)
	decl	TD_NEST_COUNT(%rbx)
	popq	%rax
	jmp	doreti_next

	/*
	 * AST pending.  We clear RQF_AST_SIGNAL automatically, the others
	 * are cleared by the trap as they are processed.
	 *
	 * Temporarily back-out our critical section because trap() can be
	 * a long-winded call, and we want to be more syscall-like.  
	 *
	 * YYY theoretically we can call lwkt_switch directly if all we need
	 * to do is a reschedule.
	 */
doreti_ast:
	andl	$~(RQF_AST_SIGNAL|RQF_AST_UPCALL),PCPU(reqflags)
	sti
	movl	%eax,%r12d		/* save cpl (can't use stack) */
	movl	$T_ASTFLT,TF_TRAPNO(%rsp)
	movq	%rsp,%rdi		/* pass frame by ref (%edi = C arg) */
	decl	TD_CRITCOUNT(%rbx)
	call	trap
	incl	TD_CRITCOUNT(%rbx)
	movl	%r12d,%eax		/* restore cpl for loop */
	jmp	doreti_next

#ifdef SMP
	/*
	 * IPIQ message pending.  We clear RQF_IPIQ automatically.
	 */
doreti_ipiq:
	movl	%eax,%r12d		/* save cpl (can't use stack) */
	incl	PCPU(intr_nesting_level)
	andl	$~RQF_IPIQ,PCPU(reqflags)
	sti
	subq	$8,%rsp			/* trapframe->intrframe */
	movq	%rsp,%rdi		/* pass frame by ref (C arg) */
	call	lwkt_process_ipiq_frame
	addq	$8,%rsp			/* intrframe->trapframe */
	decl	PCPU(intr_nesting_level)
	movl	%r12d,%eax		/* restore cpl for loop */
	jmp	doreti_next
#endif

doreti_timer:
	movl	%eax,%r12d		/* save cpl (can't use stack) */
	incl	PCPU(intr_nesting_level)
	andl	$~RQF_TIMER,PCPU(reqflags)
	sti
	subq	$8,%rsp			/* trapframe->intrframe */
	movq	%rsp,%rdi		/* pass frame by ref (C arg) */
	call	lapic_timer_process_frame
	addq	$8,%rsp			/* intrframe->trapframe */
	decl	PCPU(intr_nesting_level)
	movl	%r12d,%eax		/* restore cpl for loop */
	jmp	doreti_next

	/*
	 * SPLZ() a C callable procedure to dispatch any unmasked pending
	 *	  interrupts regardless of critical section nesting.  ASTs
	 *	  are not dispatched.
	 *
	 * 	  Use %eax to track those IRQs that could not be processed
	 *	  due to BGL requirements.
	 */
	SUPERALIGN_TEXT

ENTRY(splz)
	pushfq
	pushq	%rbx
	movq	PCPU(curthread),%rbx
	incl	TD_CRITCOUNT(%rbx)
	movq	$0,%rax

splz_next:
	cli
	movq	%rax,%rcx		/* rcx = ~CPL */
	notq	%rcx
#ifdef SMP
	testl	$RQF_IPIQ,PCPU(reqflags)
	jnz	splz_ipiq
#endif
	testl	$RQF_TIMER,PCPU(reqflags)
	jnz	splz_timer
	/*
	 * check for an unmasked int (3 groups)
	 */
	movq	$0,%rdx
	testq	PCPU_E8(ipending,%rdx),%rcx
	jnz	splz_fast

	movq	$1,%rdx
	testq	PCPU_E8(ipending,%rdx),%rcx
	jnz	splz_fast

	movq	$2,%rdx
	testq	PCPU_E8(ipending,%rdx),%rcx
	jnz	splz_fast

	movl	PCPU(spending),%ecx
	cmpl	$0,%ecx
	jnz	splz_soft

	decl	TD_CRITCOUNT(%rbx)

	/*
	 * Nothing left to do, finish up.  Interrupts are still disabled.
	 * If our mask of IRQs we couldn't process due to BGL requirements
	 * is 0 then there are no pending interrupt sources left and we
	 * can clear RQF_INTPEND.
	 */
	testl	%eax,%eax
	jnz	5f
	andl	$~RQF_INTPEND,PCPU(reqflags)
5:
	popq	%rbx
	popfq
	ret

	/*
	 * Interrupt pending
	 */
	ALIGN_TEXT
splz_fast:
	andq	PCPU_E8(ipending,%rdx),%rcx
	sti
	bsfq	%rcx, %rcx		/* locate the next dispatchable int */
	btrq	%rcx, PCPU_E8(ipending,%rdx)
					/* is it really still pending? */
	jnc	splz_next

	shlq	$6, %rdx
	orq	%rdx, %rcx		/* form intr number */

	pushq	%rax
	call	dofastunpend		/* unpend intr %rcx */
	popq	%rax
	jmp	splz_next

	/*
	 *  SOFT interrupt pending
	 *
	 *  Temporarily back-out our critical section to allow the interrupt
	 *  preempt us.
	 */
	ALIGN_TEXT
splz_soft:
	sti
	bsfl	%ecx,%ecx		/* locate the next pending softint */
	btrl	%ecx,PCPU(spending)	/* make sure its still pending */
	jnc	splz_next
	addl	$FIRST_SOFTINT,%ecx	/* actual intr number */
	sti
	pushq	%rax
	movl	%ecx,%edi		/* C argument */
	incl	TD_NEST_COUNT(%rbx)	/* prevent doreti/splz nesting */
	decl	TD_CRITCOUNT(%rbx)
	call	sched_ithd		/* YYY must pull in imasks */
	incl	TD_CRITCOUNT(%rbx)
	decl	TD_NEST_COUNT(%rbx)	/* prevent doreti/splz nesting */
	popq	%rax
	jmp	splz_next

#ifdef SMP
splz_ipiq:
	andl	$~RQF_IPIQ,PCPU(reqflags)
	sti
	pushq	%rax
	call	lwkt_process_ipiq
	popq	%rax
	jmp	splz_next
#endif

splz_timer:
	andl	$~RQF_TIMER,PCPU(reqflags)
	sti
	pushq	%rax
	call	lapic_timer_process
	popq	%rax
	jmp	splz_next

	/*
	 * dofastunpend(%rcx:intr)
	 *
	 * A interrupt previously made pending can now be run,
	 * execute it by pushing a dummy interrupt frame and 
	 * calling ithread_fast_handler to execute or schedule it.
	 * 
	 * ithread_fast_handler() returns 0 if it wants us to unmask
	 * further interrupts.
	 */
#define PUSH_DUMMY							\
	pushfq ;			/* phys int frame / flags */	\
	xorq	%rax,%rax ;		/* something not SEL_UPL */	\
	pushq	%rax ;			/* phys int frame / cs */	\
	pushq	3*8(%rsp) ;		/* original caller eip */	\
	subq	$TF_RIP,%rsp ;		/* trap frame */		\
	movq	$0,TF_XFLAGS(%rsp) ;	/* extras */			\
	movq	$0,TF_TRAPNO(%rsp) ;	/* extras */			\
	movq	$0,TF_ADDR(%rsp) ;	/* extras */			\
	movq	$0,TF_FLAGS(%rsp) ;	/* extras */			\
	movq	$0,TF_ERR(%rsp) ;	/* extras */			\

#define POP_DUMMY							\
	addq	$TF_RIP+(3*8),%rsp ;					\

dofastunpend:
	pushq	%rbp			/* frame for backtrace */
	movq	%rsp,%rbp
	PUSH_DUMMY
	pushq	%rcx			/* last part of intrframe = intr */
	incl	fastunpend_count
	movq	%rsp,%rdi		/* pass frame by reference C arg */
	call	ithread_fast_handler	/* returns 0 to unmask */
	popq	%rdi			/* intrframe->trapframe */
					/* + also rdi C arg to next call */
	cmpl	$0,%eax
	jnz	1f
	movq	MachIntrABI + MACHINTR_INTREN, %rax
	callq	*%rax			/* MachIntrABI.intren(intr) */
1:
	POP_DUMMY
	popq	%rbp
	ret
Commit	Line	Data
c8fe38ae MD	1	/*
	2	* Copyright (c) 2008 The DragonFly Project. All rights reserved.
	3	*
	4	* This code is derived from software contributed to The DragonFly Project
	5	* by Matthew Dillon <dillon@backplane.com>
	6	*
	7	* Redistribution and use in source and binary forms, with or without
	8	* modification, are permitted provided that the following conditions
	9	* are met:
	10	*
	11	* 1. Redistributions of source code must retain the above copyright
	12	* notice, this list of conditions and the following disclaimer.
	13	* 2. Redistributions in binary form must reproduce the above copyright
	14	* notice, this list of conditions and the following disclaimer in
	15	* the documentation and/or other materials provided with the
	16	* distribution.
	17	* 3. Neither the name of The DragonFly Project nor the names of its
	18	* contributors may be used to endorse or promote products derived
	19	* from this software without specific, prior written permission.
	20	*
	21	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	22	* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	23	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	24	* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
	25	* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
	26	* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
	27	* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	28	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	29	* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	30	* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
	31	* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	32	* SUCH DAMAGE.
	33	*
	34	* ---
	35	*
	36	* Copyright (c) 1989, 1990 William F. Jolitz.
	37	* Copyright (c) 1990 The Regents of the University of California.
	38	* All rights reserved.
	39	*
	40	* This code is derived from software contributed to Berkeley by
	41	* William Jolitz.
	42	*
	43	* Redistribution and use in source and binary forms, with or without
	44	* modification, are permitted provided that the following conditions
	45	* are met:
	46	* 1. Redistributions of source code must retain the above copyright
	47	* notice, this list of conditions and the following disclaimer.
	48	* 2. Redistributions in binary form must reproduce the above copyright
	49	* notice, this list of conditions and the following disclaimer in the
	50	* documentation and/or other materials provided with the distribution.
	51	* 3. All advertising materials mentioning features or use of this software
	52	* must display the following acknowledgement:
	53	* This product includes software developed by the University of
	54	* California, Berkeley and its contributors.
	55	* 4. Neither the name of the University nor the names of its contributors
	56	* may be used to endorse or promote products derived from this software
	57	* without specific prior written permission.
	58	*
	59	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	60	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	61	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	62	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	63	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	64	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
65	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
66	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
67	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
68	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
69	* SUCH DAMAGE.
70	*
71	* @(#)ipl.s
72	*
73	* $FreeBSD: src/sys/i386/isa/ipl.s,v 1.32.2.3 2002/05/16 16:03:56 bde Exp $
c8fe38ae MD	74	*/
	75
	76	#include <machine/asmacros.h>
	77	#include <machine/segments.h>
	78	#include <machine/ipl.h>
	79	#include <machine/lock.h>
	80	#include <machine/psl.h>
	81	#include <machine/trap.h>
	82
	83	#include "assym.s"
	84
	85	/*
	86	* AT/386
	87	* Vector interrupt control section
	88	*
9611ff20	89	* ipending - Pending interrupts (set when a masked interrupt occurs)
c8fe38ae MD	90	* spending - Pending software interrupts
	91	*/
	92	.data
	93	ALIGN_DATA
	94
	95	.globl fastunpend_count
	96	fastunpend_count: .long 0
	97
	98	.text
	99	SUPERALIGN_TEXT
	100
	101	/*
	102	* GENERAL NOTES
	103	*
9611ff20	104	* - interrupts are always called with a critical section held
c8fe38ae MD	105	*
	106	* - we release our critical section when scheduling interrupt
	107	* or softinterrupt threads in order so they can preempt
	108	* (unless we are called manually from a critical section, in
	109	* which case there will still be a critical section and
	110	* they won't preempt anyway).
	111	*
	112	* - TD_NEST_COUNT prevents splz from nesting too deeply within
	113	* itself. It is not actually an interrupt nesting count.
	114	* PCPU(intr_nesting_level) is an interrupt nesting count.
	115	*
	116	* - We have to be careful in regards to local interrupts
	117	* occuring simultaniously with our doreti and splz
	118	* processing.
b4b1a37a MD	119	*
	120	* - Interrupts must be enabled when calling higher level
	121	* functions in order to avoid deadlocking against things
	122	* like smp_invltlb.
c8fe38ae MD	123	*/
	124
	125	/*
	126	* DORETI
	127	*
	128	* Handle return from interrupts, traps and syscalls. This function
9611ff20 SZ	129	* checks the cpl for unmasked pending interrupts (hardware or soft)
9611ff20 SZ	130	* and schedules them if appropriate, then irets.
c8fe38ae	131	*
b5d16701	132	* If we are in a critical section we cannot run any pending ints.
c8fe38ae MD	133	*
	134	* The stack contains a trapframe at the start of doreti.
	135	*/
	136	SUPERALIGN_TEXT
	137	.globl doreti
	138	.type doreti,@function
	139	doreti:
	140	FAKE_MCOUNT(bintr) /* init "from" bintr -> doreti */
	141	movq $0,%rax /* irq mask unavailable due to BGL */
	142	movq PCPU(curthread),%rbx
f9235b6d	143	cli /* interlock with critical section */
c8fe38ae MD	144	cmpl $0,PCPU(reqflags) /* short cut if nothing to do */
c8fe38ae MD	145	je 5f
f9235b6d MD	146	testl $-1,TD_CRITCOUNT(%rbx) /* can't unpend if in critical sec */
	147	jne 5f
	148	incl TD_CRITCOUNT(%rbx) /* force all ints to pending */
c8fe38ae	149	doreti_next:
b4b1a37a	150	cli /* re-assert cli on loop */
9611ff20 SZ	151	movq %rax,%rcx /* irq mask unavailable due to BGL */
9611ff20 SZ	152	notq %rcx
c8fe38ae MD	153	#ifdef SMP
	154	testl $RQF_IPIQ,PCPU(reqflags)
	155	jnz doreti_ipiq
9a4bd8f3	156	#endif
46d4e165 JG	157	testl $RQF_TIMER,PCPU(reqflags)
46d4e165 JG	158	jnz doreti_timer
9611ff20 SZ	159	/*
	160	* check for an unmasked int (3 groups)
	161	*/
	162	movq $0,%rdx
	163	testq PCPU_E8(ipending,%rdx),%rcx
	164	jnz doreti_fast
	165
	166	movq $1,%rdx
	167	testq PCPU_E8(ipending,%rdx),%rcx
	168	jnz doreti_fast
	169
	170	movq $2,%rdx
	171	testq PCPU_E8(ipending,%rdx),%rcx
c8fe38ae MD	172	jnz doreti_fast
c8fe38ae MD	173
c8fe38ae MD	174	movl PCPU(spending),%ecx /* check for a pending software int */
	175	cmpl $0,%ecx
	176	jnz doreti_soft
	177
	178	testl $RQF_AST_MASK,PCPU(reqflags) /* any pending ASTs? */
	179	jz 2f
	180
	181	/* ASTs are only applicable when returning to userland */
	182	testb $SEL_RPL_MASK,TF_CS(%rsp)
	183	jnz doreti_ast
	184	2:
	185	/*
	186	* Nothing left to do, finish up. Interrupts are still disabled.
	187	* %eax contains the mask of IRQ's that are not available due to
	188	* BGL requirements. We can only clear RQF_INTPEND if ALL pending
	189	* interrupts have been processed.
	190	*/
f9235b6d	191	decl TD_CRITCOUNT(%rbx) /* interlocked with cli */
c8fe38ae MD	192	testl %eax,%eax
	193	jnz 5f
	194	andl $~RQF_INTPEND,PCPU(reqflags)
	195	5:
	196	MEXITCOUNT
	197
	198	/*
b4b1a37a MD	199	* (interrupts are disabled here)
b4b1a37a MD	200	*
c8fe38ae MD	201	* Restore register and iret. iret can fault on %rip (which is
	202	* really stupid). If this occurs we re-fault and vector to
	203	* doreti_iret_fault().
	204	*
	205	* ...
	206	* can be set from user mode, this can result in a kernel mode
	207	* exception. The trap code will revector to the *_fault code
	208	* which then sets up a T_PROTFLT signal. If the signal is
	209	* sent to userland, sendsig() will automatically clean up all
	210	* the segment registers to avoid a loop.
	211	*/
	212	.globl doreti_iret
	213	.globl doreti_syscall_ret
	214	doreti_syscall_ret:
	215	POP_FRAME /* registers and %gs (+cli) */
2883d2d8	216	/* WARNING: special global doreti_iret is also used by exception.S */
c8fe38ae MD	217	doreti_iret:
	218	iretq
	219
	220	/*
2883d2d8 MD	221	* doreti_iret_fault. Alternative return code for the case where
	222	* we get a fault in the doreti_exit code above. trap()
	223	* (sys/platform/pc64/x86_64/trap.c) catches this specific * case,
	224	* sends the process a signal and continues in the corresponding
	225	* place in the code below.
b4b1a37a MD	226	*
	227	* Interrupts are likely disabled due to the above interlock
	228	* between cli/iretq. We must enable them before calling any
	229	* high level function.
c8fe38ae MD	230	*/
	231	ALIGN_TEXT
	232	.globl doreti_iret_fault
	233	doreti_iret_fault:
89ffa1cf	234	PUSH_FRAME_NOSWAP
c8fe38ae	235	sti
c8fe38ae MD	236	movq $T_PROTFLT,TF_TRAPNO(%rsp)
	237	movq $0,TF_ERR(%rsp) /* XXX should be the error code */
	238	movq $0,TF_ADDR(%rsp)
	239	FAKE_MCOUNT(TF_RIP(%rsp))
	240	jmp calltrap
	241
	242	/*
9611ff20 SZ	243	* Interrupt pending. NOTE: stack context holds frame structure
9611ff20 SZ	244	* for interrupt procedure, do not do random pushes or pops!
c8fe38ae MD	245	*/
	246	ALIGN_TEXT
	247	doreti_fast:
9611ff20	248	andq PCPU_E8(ipending,%rdx),%rcx
b4b1a37a	249	sti
9611ff20 SZ	250	bsfq %rcx, %rcx /* locate the next dispatchable int */
	251	btrq %rcx, PCPU_E8(ipending,%rdx)
	252	/* is it really still pending? */
c8fe38ae	253	jnc doreti_next
9611ff20 SZ	254
	255	shlq $6, %rdx
	256	orq %rdx, %rcx /* form intr number */
	257
c8fe38ae MD	258	pushq %rax /* save IRQ mask unavailable for BGL */
c8fe38ae MD	259	/* NOTE: is also CPL in frame */
9611ff20	260	call dofastunpend /* unpend intr %rcx */
c8fe38ae MD	261	popq %rax
c8fe38ae MD	262	jmp doreti_next
c8fe38ae MD	263
c8fe38ae MD	264	/*
c8fe38ae MD	265	* SOFT interrupt pending
	266	*
	267	* Temporarily back-out our critical section to allow an interrupt
	268	* preempt us when we schedule it. Bump intr_nesting_level to
	269	* prevent the switch code from recursing via splz too deeply.
	270	*/
	271	ALIGN_TEXT
	272	doreti_soft:
b4b1a37a	273	sti
c8fe38ae MD	274	bsfl %ecx,%ecx /* locate the next pending softint */
	275	btrl %ecx,PCPU(spending) /* make sure its still pending */
	276	jnc doreti_next
	277	addl $FIRST_SOFTINT,%ecx /* actual intr number */
	278	pushq %rax
	279	movl %ecx,%edi /* argument to C call */
	280	incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
f9235b6d	281	decl TD_CRITCOUNT(%rbx) /* so we can preempt */
c8fe38ae	282	call sched_ithd /* YYY must pull in imasks */
f9235b6d	283	incl TD_CRITCOUNT(%rbx)
c8fe38ae MD	284	decl TD_NEST_COUNT(%rbx)
	285	popq %rax
	286	jmp doreti_next
	287
	288	/*
	289	* AST pending. We clear RQF_AST_SIGNAL automatically, the others
	290	* are cleared by the trap as they are processed.
	291	*
	292	* Temporarily back-out our critical section because trap() can be
	293	* a long-winded call, and we want to be more syscall-like.
	294	*
	295	* YYY theoretically we can call lwkt_switch directly if all we need
	296	* to do is a reschedule.
	297	*/
	298	doreti_ast:
	299	andl $~(RQF_AST_SIGNAL\|RQF_AST_UPCALL),PCPU(reqflags)
	300	sti
e58c365d	301	movl %eax,%r12d /* save cpl (can't use stack) */
c8fe38ae MD	302	movl $T_ASTFLT,TF_TRAPNO(%rsp)
c8fe38ae MD	303	movq %rsp,%rdi /* pass frame by ref (%edi = C arg) */
f9235b6d	304	decl TD_CRITCOUNT(%rbx)
c8fe38ae	305	call trap
f9235b6d	306	incl TD_CRITCOUNT(%rbx)
e58c365d	307	movl %r12d,%eax /* restore cpl for loop */
c8fe38ae MD	308	jmp doreti_next
	309
	310	#ifdef SMP
	311	/*
	312	* IPIQ message pending. We clear RQF_IPIQ automatically.
	313	*/
	314	doreti_ipiq:
e58c365d	315	movl %eax,%r12d /* save cpl (can't use stack) */
c8fe38ae MD	316	incl PCPU(intr_nesting_level)
c8fe38ae MD	317	andl $~RQF_IPIQ,PCPU(reqflags)
b4b1a37a	318	sti
061e4845	319	subq $8,%rsp /* trapframe->intrframe */
c8fe38ae MD	320	movq %rsp,%rdi /* pass frame by ref (C arg) */
c8fe38ae MD	321	call lwkt_process_ipiq_frame
a2a636cc	322	addq $8,%rsp /* intrframe->trapframe */
46d4e165	323	decl PCPU(intr_nesting_level)
e58c365d	324	movl %r12d,%eax /* restore cpl for loop */
46d4e165	325	jmp doreti_next
9a4bd8f3	326	#endif
46d4e165 JG	327
46d4e165 JG	328	doreti_timer:
e58c365d	329	movl %eax,%r12d /* save cpl (can't use stack) */
46d4e165 JG	330	incl PCPU(intr_nesting_level)
46d4e165 JG	331	andl $~RQF_TIMER,PCPU(reqflags)
b4b1a37a	332	sti
a2a636cc	333	subq $8,%rsp /* trapframe->intrframe */
b25897b2	334	movq %rsp,%rdi /* pass frame by ref (C arg) */
46d4e165	335	call lapic_timer_process_frame
a2a636cc	336	addq $8,%rsp /* intrframe->trapframe */
c8fe38ae	337	decl PCPU(intr_nesting_level)
e58c365d	338	movl %r12d,%eax /* restore cpl for loop */
c8fe38ae MD	339	jmp doreti_next
c8fe38ae MD	340
c8fe38ae MD	341	/*
	342	* SPLZ() a C callable procedure to dispatch any unmasked pending
	343	* interrupts regardless of critical section nesting. ASTs
	344	* are not dispatched.
	345	*
	346	* Use %eax to track those IRQs that could not be processed
	347	* due to BGL requirements.
	348	*/
	349	SUPERALIGN_TEXT
	350
	351	ENTRY(splz)
	352	pushfq
	353	pushq %rbx
	354	movq PCPU(curthread),%rbx
f9235b6d	355	incl TD_CRITCOUNT(%rbx)
9611ff20	356	movq $0,%rax
c8fe38ae MD	357
	358	splz_next:
	359	cli
9611ff20 SZ	360	movq %rax,%rcx /* rcx = ~CPL */
9611ff20 SZ	361	notq %rcx
c8fe38ae MD	362	#ifdef SMP
	363	testl $RQF_IPIQ,PCPU(reqflags)
	364	jnz splz_ipiq
9a4bd8f3	365	#endif
46d4e165 JG	366	testl $RQF_TIMER,PCPU(reqflags)
46d4e165 JG	367	jnz splz_timer
9611ff20 SZ	368	/*
	369	* check for an unmasked int (3 groups)
	370	*/
	371	movq $0,%rdx
	372	testq PCPU_E8(ipending,%rdx),%rcx
	373	jnz splz_fast
	374
	375	movq $1,%rdx
	376	testq PCPU_E8(ipending,%rdx),%rcx
	377	jnz splz_fast
	378
	379	movq $2,%rdx
	380	testq PCPU_E8(ipending,%rdx),%rcx
c8fe38ae MD	381	jnz splz_fast
c8fe38ae MD	382
c8fe38ae MD	383	movl PCPU(spending),%ecx
	384	cmpl $0,%ecx
	385	jnz splz_soft
	386
f9235b6d	387	decl TD_CRITCOUNT(%rbx)
c8fe38ae MD	388
	389	/*
	390	* Nothing left to do, finish up. Interrupts are still disabled.
	391	* If our mask of IRQs we couldn't process due to BGL requirements
	392	* is 0 then there are no pending interrupt sources left and we
	393	* can clear RQF_INTPEND.
	394	*/
	395	testl %eax,%eax
	396	jnz 5f
	397	andl $~RQF_INTPEND,PCPU(reqflags)
	398	5:
	399	popq %rbx
	400	popfq
	401	ret
	402
	403	/*
9611ff20	404	* Interrupt pending
c8fe38ae MD	405	*/
	406	ALIGN_TEXT
	407	splz_fast:
9611ff20	408	andq PCPU_E8(ipending,%rdx),%rcx
b4b1a37a	409	sti
9611ff20 SZ	410	bsfq %rcx, %rcx /* locate the next dispatchable int */
	411	btrq %rcx, PCPU_E8(ipending,%rdx)
	412	/* is it really still pending? */
c8fe38ae	413	jnc splz_next
9611ff20 SZ	414
	415	shlq $6, %rdx
	416	orq %rdx, %rcx /* form intr number */
	417
c8fe38ae	418	pushq %rax
9611ff20	419	call dofastunpend /* unpend intr %rcx */
c8fe38ae	420	popq %rax
c8fe38ae MD	421	jmp splz_next
	422
	423	/*
c8fe38ae MD	424	* SOFT interrupt pending
	425	*
	426	* Temporarily back-out our critical section to allow the interrupt
	427	* preempt us.
	428	*/
	429	ALIGN_TEXT
	430	splz_soft:
b4b1a37a	431	sti
c8fe38ae MD	432	bsfl %ecx,%ecx /* locate the next pending softint */
	433	btrl %ecx,PCPU(spending) /* make sure its still pending */
	434	jnc splz_next
	435	addl $FIRST_SOFTINT,%ecx /* actual intr number */
	436	sti
	437	pushq %rax
	438	movl %ecx,%edi /* C argument */
c8fe38ae	439	incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
b4b1a37a	440	decl TD_CRITCOUNT(%rbx)
c8fe38ae	441	call sched_ithd /* YYY must pull in imasks */
f9235b6d	442	incl TD_CRITCOUNT(%rbx)
c8fe38ae MD	443	decl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
	444	popq %rax
	445	jmp splz_next
	446
	447	#ifdef SMP
	448	splz_ipiq:
	449	andl $~RQF_IPIQ,PCPU(reqflags)
b4b1a37a	450	sti
c8fe38ae MD	451	pushq %rax
	452	call lwkt_process_ipiq
	453	popq %rax
	454	jmp splz_next
9a4bd8f3	455	#endif
46d4e165 JG	456
	457	splz_timer:
	458	andl $~RQF_TIMER,PCPU(reqflags)
b4b1a37a	459	sti
46d4e165 JG	460	pushq %rax
	461	call lapic_timer_process
	462	popq %rax
	463	jmp splz_next
c8fe38ae MD	464
c8fe38ae MD	465	/*
9611ff20	466	* dofastunpend(%rcx:intr)
c8fe38ae	467	*
9611ff20	468	* A interrupt previously made pending can now be run,
c8fe38ae MD	469	* execute it by pushing a dummy interrupt frame and
	470	* calling ithread_fast_handler to execute or schedule it.
	471	*
	472	* ithread_fast_handler() returns 0 if it wants us to unmask
	473	* further interrupts.
	474	*/
	475	#define PUSH_DUMMY \
	476	pushfq ; /* phys int frame / flags */ \
3338cc67	477	xorq %rax,%rax ; /* something not SEL_UPL */ \
c8fe38ae MD	478	pushq %rax ; /* phys int frame / cs */ \
	479	pushq 38(%rsp) ; / original caller eip */ \
	480	subq $TF_RIP,%rsp ; /* trap frame */ \
5b9f6cc4	481	movq $0,TF_XFLAGS(%rsp) ; /* extras */ \
c8fe38ae MD	482	movq $0,TF_TRAPNO(%rsp) ; /* extras */ \
	483	movq $0,TF_ADDR(%rsp) ; /* extras */ \
	484	movq $0,TF_FLAGS(%rsp) ; /* extras */ \
	485	movq $0,TF_ERR(%rsp) ; /* extras */ \
	486
	487	#define POP_DUMMY \
	488	addq $TF_RIP+(3*8),%rsp ; \
	489
	490	dofastunpend:
	491	pushq %rbp /* frame for backtrace */
	492	movq %rsp,%rbp
	493	PUSH_DUMMY
	494	pushq %rcx /* last part of intrframe = intr */
	495	incl fastunpend_count
	496	movq %rsp,%rdi /* pass frame by reference C arg */
	497	call ithread_fast_handler /* returns 0 to unmask */
	498	popq %rdi /* intrframe->trapframe */
	499	/* + also rdi C arg to next call */
	500	cmpl $0,%eax
	501	jnz 1f
	502	movq MachIntrABI + MACHINTR_INTREN, %rax
	503	callq %rax / MachIntrABI.intren(intr) */
	504	1:
	505	POP_DUMMY
	506	popq %rbp
	507	ret
	508