[dragonfly.git] / sys / platform / pc64 / x86_64 / ipl.s

/*
 * Copyright (c) 2008 The DragonFly Project.  All rights reserved.
 * 
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * 
 * ---
 *
 * Copyright (c) 1989, 1990 William F. Jolitz.
 * Copyright (c) 1990 The Regents of the University of California.
 * All rights reserved.
 *
 * This code is derived from software contributed to Berkeley by
 * William Jolitz.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by the University of
 *	California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)ipl.s
 *
 * $FreeBSD: src/sys/i386/isa/ipl.s,v 1.32.2.3 2002/05/16 16:03:56 bde Exp $
 */

#include <machine/asmacros.h>
#include <machine/segments.h>
#include <machine/ipl.h>
#include <machine/lock.h>
#include <machine/psl.h>
#include <machine/trap.h>
 
#include "assym.s"

/*
 * AT/386
 * Vector interrupt control section
 *
 *  fpending	- Pending interrupts (set when a masked interrupt occurs)
 *  spending	- Pending software interrupts
 */
	.data
	ALIGN_DATA

	.globl		fastunpend_count
fastunpend_count:	.long	0

	.text
	SUPERALIGN_TEXT

	/*
	 * GENERAL NOTES
	 *
	 *	- fast interrupts are always called with a critical section
	 *	  held
	 *
	 *	- we release our critical section when scheduling interrupt
	 *	  or softinterrupt threads in order so they can preempt
	 *	  (unless we are called manually from a critical section, in
	 *	  which case there will still be a critical section and
	 *	  they won't preempt anyway).
	 *
	 *	- TD_NEST_COUNT prevents splz from nesting too deeply within
	 *	  itself.  It is *not* actually an interrupt nesting count.
	 *	  PCPU(intr_nesting_level) is an interrupt nesting count.
	 *
	 *	- We have to be careful in regards to local interrupts
	 *	  occuring simultaniously with our doreti and splz 
	 *	  processing.
	 */

	/*
	 * DORETI
	 *
	 * Handle return from interrupts, traps and syscalls.  This function
	 * checks the cpl for unmasked pending interrupts (fast, normal, or
	 * soft) and schedules them if appropriate, then irets.
	 *
	 * If we are in a critical section we cannot run any pending ints
	 * nor can be play with mp_lock.
	 *
	 * The stack contains a trapframe at the start of doreti.
	 */
	SUPERALIGN_TEXT
	.globl	doreti
	.type	doreti,@function
doreti:
	FAKE_MCOUNT(bintr)		/* init "from" bintr -> doreti */
	movq	$0,%rax			/* irq mask unavailable due to BGL */
	movq	PCPU(curthread),%rbx
	cli				/* interlock with TDPRI_CRIT */
	cmpl	$0,PCPU(reqflags)	/* short cut if nothing to do */
	je	5f
	cmpl	$TDPRI_CRIT,TD_PRI(%rbx) /* can't unpend if in critical sec */
	jge	5f
	addl	$TDPRI_CRIT,TD_PRI(%rbx) /* force all ints to pending */
doreti_next:
	sti				/* allow new interrupts */
	movl	%eax,%ecx		/* irq mask unavailable due to BGL */
	notl	%ecx
	cli				/* disallow YYY remove */
#ifdef SMP
	testl	$RQF_IPIQ,PCPU(reqflags)
	jnz	doreti_ipiq
	testl	$RQF_TIMER,PCPU(reqflags)
	jnz	doreti_timer
#endif
	testl	PCPU(fpending),%ecx	/* check for an unmasked fast int */
	jnz	doreti_fast

	movl	PCPU(spending),%ecx	/* check for a pending software int */
	cmpl	$0,%ecx
	jnz	doreti_soft

	testl	$RQF_AST_MASK,PCPU(reqflags) /* any pending ASTs? */
	jz	2f

	/* ASTs are only applicable when returning to userland */
	testb	$SEL_RPL_MASK,TF_CS(%rsp)
	jnz	doreti_ast
2:
	/*
	 * Nothing left to do, finish up.  Interrupts are still disabled.
	 * %eax contains the mask of IRQ's that are not available due to
	 * BGL requirements.  We can only clear RQF_INTPEND if *ALL* pending
	 * interrupts have been processed.
	 */
	subl	$TDPRI_CRIT,TD_PRI(%rbx)	/* interlocked with cli */
	testl	%eax,%eax
	jnz	5f
	andl	$~RQF_INTPEND,PCPU(reqflags)
5:
	MEXITCOUNT

	/*
	 * Restore register and iret.  iret can fault on %rip (which is
	 * really stupid).  If this occurs we re-fault and vector to
	 * doreti_iret_fault().
	 *
	 * ...
	 * can be set from user mode, this can result in a kernel mode
	 * exception.  The trap code will revector to the *_fault code
	 * which then sets up a T_PROTFLT signal.  If the signal is
	 * sent to userland, sendsig() will automatically clean up all
	 * the segment registers to avoid a loop.
	 */
	.globl	doreti_iret
	.globl	doreti_syscall_ret
doreti_syscall_ret:
	POP_FRAME		/* registers and %gs (+cli) */
	/* special global also used by exception.S */
doreti_iret:
	iretq

	/*
	 * doreti_iret_fault.  Alternative return code for
	 * the case where we get a fault in the doreti_exit code
	 * above.  trap() (sys/platform/pc64/x86_64/trap.c) catches this specific
	 * case, sends the process a signal and continues in the
	 * corresponding place in the code below.
	 */
	ALIGN_TEXT
	.globl	doreti_iret_fault
doreti_iret_fault:
	PUSH_FRAME_NOSWAP
	testq	$PSL_I,TF_RFLAGS(%rsp)
	jz	2f
	sti
2:
	movq	$T_PROTFLT,TF_TRAPNO(%rsp)
	movq	$0,TF_ERR(%rsp)	/* XXX should be the error code */
	movq	$0,TF_ADDR(%rsp)
	FAKE_MCOUNT(TF_RIP(%rsp))
	jmp	calltrap

	/*
	 * FAST interrupt pending.  NOTE: stack context holds frame structure
	 * for fast interrupt procedure, do not do random pushes or pops!
	 */
	ALIGN_TEXT
doreti_fast:
	andl	PCPU(fpending),%ecx	/* only check fast ints */
	bsfl	%ecx, %ecx		/* locate the next dispatchable int */
	btrl	%ecx, PCPU(fpending)	/* is it really still pending? */
	jnc	doreti_next
	pushq	%rax			/* save IRQ mask unavailable for BGL */
					/* NOTE: is also CPL in frame */
#if 0
#ifdef SMP
	pushq	%rcx			/* save ecx */
	call	try_mplock
	popq	%rcx
	testl	%eax,%eax
	jz	1f
	/* MP lock successful */
#endif
#endif
	call	dofastunpend		/* unpend fast intr %ecx */
#if 0
#ifdef SMP
	call	rel_mplock
#endif
#endif
	popq	%rax
	jmp	doreti_next
1:
	btsl	%ecx, PCPU(fpending)	/* oops, couldn't get the MP lock */
	popq	%rax			/* add to temp. cpl mask to ignore */
	orl	PCPU(fpending),%eax
	jmp	doreti_next

	/*
	 *  SOFT interrupt pending
	 *
	 *  Temporarily back-out our critical section to allow an interrupt
	 *  preempt us when we schedule it.  Bump intr_nesting_level to
	 *  prevent the switch code from recursing via splz too deeply.
	 */
	ALIGN_TEXT
doreti_soft:
	bsfl	%ecx,%ecx		/* locate the next pending softint */
	btrl	%ecx,PCPU(spending)	/* make sure its still pending */
	jnc	doreti_next
	addl	$FIRST_SOFTINT,%ecx	/* actual intr number */
	pushq	%rax
	movl	%ecx,%edi		/* argument to C call */
	incl	TD_NEST_COUNT(%rbx)	/* prevent doreti/splz nesting */
	subl	$TDPRI_CRIT,TD_PRI(%rbx) /* so we can preempt */
	call	sched_ithd		/* YYY must pull in imasks */
	addl	$TDPRI_CRIT,TD_PRI(%rbx)
	decl	TD_NEST_COUNT(%rbx)
	popq	%rax
	jmp	doreti_next

	/*
	 * AST pending.  We clear RQF_AST_SIGNAL automatically, the others
	 * are cleared by the trap as they are processed.
	 *
	 * Temporarily back-out our critical section because trap() can be
	 * a long-winded call, and we want to be more syscall-like.  
	 *
	 * YYY theoretically we can call lwkt_switch directly if all we need
	 * to do is a reschedule.
	 */
doreti_ast:
	andl	$~(RQF_AST_SIGNAL|RQF_AST_UPCALL),PCPU(reqflags)
	sti
	movl	%eax,%r12d		/* save cpl (can't use stack) */
	movl	$T_ASTFLT,TF_TRAPNO(%rsp)
	movq	%rsp,%rdi		/* pass frame by ref (%edi = C arg) */
	subl	$TDPRI_CRIT,TD_PRI(%rbx)
	call	trap
	addl	$TDPRI_CRIT,TD_PRI(%rbx)
	movl	%r12d,%eax		/* restore cpl for loop */
	jmp	doreti_next

#ifdef SMP
	/*
	 * IPIQ message pending.  We clear RQF_IPIQ automatically.
	 */
doreti_ipiq:
	movl	%eax,%r12d		/* save cpl (can't use stack) */
	incl	PCPU(intr_nesting_level)
	andl	$~RQF_IPIQ,PCPU(reqflags)
	subq	$8,%rsp			/* trapframe->intrframe */
	movq	%rsp,%rdi		/* pass frame by ref (C arg) */
	call	lwkt_process_ipiq_frame
	addq	$8,%rsp			/* intrframe->trapframe */
	decl	PCPU(intr_nesting_level)
	movl	%r12d,%eax		/* restore cpl for loop */
	jmp	doreti_next

doreti_timer:
	movl	%eax,%r12d		/* save cpl (can't use stack) */
	incl	PCPU(intr_nesting_level)
	andl	$~RQF_TIMER,PCPU(reqflags)
	subq	$8,%rsp			/* trapframe->intrframe */
	movq	%rsp,%rdi			/* pass frame by ref (C arg) */
	call	lapic_timer_process_frame
	addq	$8,%rsp			/* intrframe->trapframe */
	decl	PCPU(intr_nesting_level)
	movl	%r12d,%eax		/* restore cpl for loop */
	jmp	doreti_next

#endif

	/*
	 * SPLZ() a C callable procedure to dispatch any unmasked pending
	 *	  interrupts regardless of critical section nesting.  ASTs
	 *	  are not dispatched.
	 *
	 * 	  Use %eax to track those IRQs that could not be processed
	 *	  due to BGL requirements.
	 */
	SUPERALIGN_TEXT

ENTRY(splz)
	pushfq
	pushq	%rbx
	movq	PCPU(curthread),%rbx
	addl	$TDPRI_CRIT,TD_PRI(%rbx)
	movl	$0,%eax

splz_next:
	cli
	movl	%eax,%ecx		/* ecx = ~CPL */
	notl	%ecx
#ifdef SMP
	testl	$RQF_IPIQ,PCPU(reqflags)
	jnz	splz_ipiq
	testl	$RQF_TIMER,PCPU(reqflags)
	jnz	splz_timer
#endif
	testl	PCPU(fpending),%ecx	/* check for an unmasked fast int */
	jnz	splz_fast

	movl	PCPU(spending),%ecx
	cmpl	$0,%ecx
	jnz	splz_soft

	subl	$TDPRI_CRIT,TD_PRI(%rbx)

	/*
	 * Nothing left to do, finish up.  Interrupts are still disabled.
	 * If our mask of IRQs we couldn't process due to BGL requirements
	 * is 0 then there are no pending interrupt sources left and we
	 * can clear RQF_INTPEND.
	 */
	testl	%eax,%eax
	jnz	5f
	andl	$~RQF_INTPEND,PCPU(reqflags)
5:
	popq	%rbx
	popfq
	ret

	/*
	 * FAST interrupt pending
	 */
	ALIGN_TEXT
splz_fast:
	andl	PCPU(fpending),%ecx	/* only check fast ints */
	bsfl	%ecx, %ecx		/* locate the next dispatchable int */
	btrl	%ecx, PCPU(fpending)	/* is it really still pending? */
	jnc	splz_next
	pushq	%rax
#if 0
#ifdef SMP
	movl	%ecx,%edi		/* argument to try_mplock */
	call	try_mplock
	testl	%eax,%eax
	jz	1f
#endif
#endif
	call	dofastunpend		/* unpend fast intr %ecx */
#if 0
#ifdef SMP
	call	rel_mplock
#endif
#endif
	popq	%rax
	jmp	splz_next
1:
	btsl	%ecx, PCPU(fpending)	/* oops, couldn't get the MP lock */
	popq	%rax
	orl	PCPU(fpending),%eax
	jmp	splz_next

	/*
	 *  SOFT interrupt pending
	 *
	 *  Temporarily back-out our critical section to allow the interrupt
	 *  preempt us.
	 */
	ALIGN_TEXT
splz_soft:
	bsfl	%ecx,%ecx		/* locate the next pending softint */
	btrl	%ecx,PCPU(spending)	/* make sure its still pending */
	jnc	splz_next
	addl	$FIRST_SOFTINT,%ecx	/* actual intr number */
	sti
	pushq	%rax
	movl	%ecx,%edi		/* C argument */
	subl	$TDPRI_CRIT,TD_PRI(%rbx)
	incl	TD_NEST_COUNT(%rbx)	/* prevent doreti/splz nesting */
	call	sched_ithd		/* YYY must pull in imasks */
	addl	$TDPRI_CRIT,TD_PRI(%rbx)
	decl	TD_NEST_COUNT(%rbx)	/* prevent doreti/splz nesting */
	popq	%rax
	jmp	splz_next

#ifdef SMP
splz_ipiq:
	andl	$~RQF_IPIQ,PCPU(reqflags)
	pushq	%rax
	call	lwkt_process_ipiq
	popq	%rax
	jmp	splz_next

splz_timer:
	andl	$~RQF_TIMER,PCPU(reqflags)
	pushq	%rax
	call	lapic_timer_process
	popq	%rax
	jmp	splz_next
#endif

	/*
	 * dofastunpend(%ecx:intr)
	 *
	 * A FAST interrupt previously made pending can now be run,
	 * execute it by pushing a dummy interrupt frame and 
	 * calling ithread_fast_handler to execute or schedule it.
	 * 
	 * ithread_fast_handler() returns 0 if it wants us to unmask
	 * further interrupts.
	 */
#define PUSH_DUMMY							\
	pushfq ;			/* phys int frame / flags */	\
	movl	%cs,%eax ;						\
	pushq	%rax ;			/* phys int frame / cs */	\
	pushq	3*8(%rsp) ;		/* original caller eip */	\
	subq	$TF_RIP,%rsp ;		/* trap frame */		\
	movq	$0,TF_XFLAGS(%rsp) ;	/* extras */			\
	movq	$0,TF_TRAPNO(%rsp) ;	/* extras */			\
	movq	$0,TF_ADDR(%rsp) ;	/* extras */			\
	movq	$0,TF_FLAGS(%rsp) ;	/* extras */			\
	movq	$0,TF_ERR(%rsp) ;	/* extras */			\

#define POP_DUMMY							\
	addq	$TF_RIP+(3*8),%rsp ;					\

dofastunpend:
	pushq	%rbp			/* frame for backtrace */
	movq	%rsp,%rbp
	PUSH_DUMMY
	pushq	%rcx			/* last part of intrframe = intr */
	incl	fastunpend_count
	movq	%rsp,%rdi		/* pass frame by reference C arg */
	call	ithread_fast_handler	/* returns 0 to unmask */
	popq	%rdi			/* intrframe->trapframe */
					/* + also rdi C arg to next call */
	cmpl	$0,%eax
	jnz	1f
	movq	MachIntrABI + MACHINTR_INTREN, %rax
	callq	*%rax			/* MachIntrABI.intren(intr) */
1:
	POP_DUMMY
	popq	%rbp
	ret
Commit	Line	Data
c8fe38ae MD	1	/*
	2	* Copyright (c) 2008 The DragonFly Project. All rights reserved.
	3	*
	4	* This code is derived from software contributed to The DragonFly Project
	5	* by Matthew Dillon <dillon@backplane.com>
	6	*
	7	* Redistribution and use in source and binary forms, with or without
	8	* modification, are permitted provided that the following conditions
	9	* are met:
	10	*
	11	* 1. Redistributions of source code must retain the above copyright
	12	* notice, this list of conditions and the following disclaimer.
	13	* 2. Redistributions in binary form must reproduce the above copyright
	14	* notice, this list of conditions and the following disclaimer in
	15	* the documentation and/or other materials provided with the
	16	* distribution.
	17	* 3. Neither the name of The DragonFly Project nor the names of its
	18	* contributors may be used to endorse or promote products derived
	19	* from this software without specific, prior written permission.
	20	*
	21	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	22	* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	23	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	24	* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
	25	* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
	26	* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
	27	* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	28	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	29	* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	30	* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
	31	* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	32	* SUCH DAMAGE.
	33	*
	34	* ---
	35	*
	36	* Copyright (c) 1989, 1990 William F. Jolitz.
	37	* Copyright (c) 1990 The Regents of the University of California.
	38	* All rights reserved.
	39	*
	40	* This code is derived from software contributed to Berkeley by
	41	* William Jolitz.
	42	*
	43	* Redistribution and use in source and binary forms, with or without
	44	* modification, are permitted provided that the following conditions
	45	* are met:
	46	* 1. Redistributions of source code must retain the above copyright
	47	* notice, this list of conditions and the following disclaimer.
	48	* 2. Redistributions in binary form must reproduce the above copyright
	49	* notice, this list of conditions and the following disclaimer in the
	50	* documentation and/or other materials provided with the distribution.
	51	* 3. All advertising materials mentioning features or use of this software
	52	* must display the following acknowledgement:
	53	* This product includes software developed by the University of
	54	* California, Berkeley and its contributors.
	55	* 4. Neither the name of the University nor the names of its contributors
	56	* may be used to endorse or promote products derived from this software
	57	* without specific prior written permission.
	58	*
	59	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	60	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	61	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	62	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	63	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	64	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
65	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
66	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
67	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
68	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
69	* SUCH DAMAGE.
70	*
71	* @(#)ipl.s
72	*
73	* $FreeBSD: src/sys/i386/isa/ipl.s,v 1.32.2.3 2002/05/16 16:03:56 bde Exp $
c8fe38ae MD	74	*/
	75
	76	#include <machine/asmacros.h>
	77	#include <machine/segments.h>
	78	#include <machine/ipl.h>
	79	#include <machine/lock.h>
	80	#include <machine/psl.h>
	81	#include <machine/trap.h>
	82
	83	#include "assym.s"
	84
	85	/*
	86	* AT/386
	87	* Vector interrupt control section
	88	*
729e15a8	89	* fpending - Pending interrupts (set when a masked interrupt occurs)
c8fe38ae MD	90	* spending - Pending software interrupts
	91	*/
	92	.data
	93	ALIGN_DATA
	94
	95	.globl fastunpend_count
	96	fastunpend_count: .long 0
	97
	98	.text
	99	SUPERALIGN_TEXT
	100
	101	/*
	102	* GENERAL NOTES
	103	*
	104	* - fast interrupts are always called with a critical section
	105	* held
	106	*
	107	* - we release our critical section when scheduling interrupt
	108	* or softinterrupt threads in order so they can preempt
	109	* (unless we are called manually from a critical section, in
	110	* which case there will still be a critical section and
	111	* they won't preempt anyway).
	112	*
	113	* - TD_NEST_COUNT prevents splz from nesting too deeply within
	114	* itself. It is not actually an interrupt nesting count.
	115	* PCPU(intr_nesting_level) is an interrupt nesting count.
	116	*
	117	* - We have to be careful in regards to local interrupts
	118	* occuring simultaniously with our doreti and splz
	119	* processing.
	120	*/
	121
	122	/*
	123	* DORETI
	124	*
	125	* Handle return from interrupts, traps and syscalls. This function
	126	* checks the cpl for unmasked pending interrupts (fast, normal, or
	127	* soft) and schedules them if appropriate, then irets.
	128	*
	129	* If we are in a critical section we cannot run any pending ints
	130	* nor can be play with mp_lock.
	131	*
	132	* The stack contains a trapframe at the start of doreti.
	133	*/
	134	SUPERALIGN_TEXT
	135	.globl doreti
	136	.type doreti,@function
	137	doreti:
	138	FAKE_MCOUNT(bintr) /* init "from" bintr -> doreti */
	139	movq $0,%rax /* irq mask unavailable due to BGL */
	140	movq PCPU(curthread),%rbx
	141	cli /* interlock with TDPRI_CRIT */
	142	cmpl $0,PCPU(reqflags) /* short cut if nothing to do */
	143	je 5f
	144	cmpl $TDPRI_CRIT,TD_PRI(%rbx) /* can't unpend if in critical sec */
	145	jge 5f
	146	addl $TDPRI_CRIT,TD_PRI(%rbx) /* force all ints to pending */
	147	doreti_next:
	148	sti /* allow new interrupts */
	149	movl %eax,%ecx /* irq mask unavailable due to BGL */
	150	notl %ecx
	151	cli /* disallow YYY remove */
	152	#ifdef SMP
	153	testl $RQF_IPIQ,PCPU(reqflags)
154	jnz doreti_ipiq
46d4e165 JG	155	testl $RQF_TIMER,PCPU(reqflags)
46d4e165 JG	156	jnz doreti_timer
c8fe38ae MD	157	#endif
	158	testl PCPU(fpending),%ecx /* check for an unmasked fast int */
	159	jnz doreti_fast
	160
c8fe38ae MD	161	movl PCPU(spending),%ecx /* check for a pending software int */
	162	cmpl $0,%ecx
	163	jnz doreti_soft
	164
	165	testl $RQF_AST_MASK,PCPU(reqflags) /* any pending ASTs? */
	166	jz 2f
	167
	168	/* ASTs are only applicable when returning to userland */
	169	testb $SEL_RPL_MASK,TF_CS(%rsp)
	170	jnz doreti_ast
	171	2:
	172	/*
	173	* Nothing left to do, finish up. Interrupts are still disabled.
	174	* %eax contains the mask of IRQ's that are not available due to
	175	* BGL requirements. We can only clear RQF_INTPEND if ALL pending
	176	* interrupts have been processed.
	177	*/
	178	subl $TDPRI_CRIT,TD_PRI(%rbx) /* interlocked with cli */
	179	testl %eax,%eax
	180	jnz 5f
	181	andl $~RQF_INTPEND,PCPU(reqflags)
	182	5:
	183	MEXITCOUNT
	184
	185	/*
	186	* Restore register and iret. iret can fault on %rip (which is
	187	* really stupid). If this occurs we re-fault and vector to
	188	* doreti_iret_fault().
	189	*
	190	* ...
	191	* can be set from user mode, this can result in a kernel mode
	192	* exception. The trap code will revector to the *_fault code
	193	* which then sets up a T_PROTFLT signal. If the signal is
	194	* sent to userland, sendsig() will automatically clean up all
	195	* the segment registers to avoid a loop.
	196	*/
	197	.globl doreti_iret
	198	.globl doreti_syscall_ret
	199	doreti_syscall_ret:
	200	POP_FRAME /* registers and %gs (+cli) */
	201	/* special global also used by exception.S */
	202	doreti_iret:
	203	iretq
	204
	205	/*
	206	* doreti_iret_fault. Alternative return code for
	207	* the case where we get a fault in the doreti_exit code
c1543a89	208	* above. trap() (sys/platform/pc64/x86_64/trap.c) catches this specific
c8fe38ae MD	209	* case, sends the process a signal and continues in the
	210	* corresponding place in the code below.
	211	*/
	212	ALIGN_TEXT
	213	.globl doreti_iret_fault
	214	doreti_iret_fault:
89ffa1cf	215	PUSH_FRAME_NOSWAP
c8fe38ae MD	216	testq $PSL_I,TF_RFLAGS(%rsp)
	217	jz 2f
	218	sti
	219	2:
	220	movq $T_PROTFLT,TF_TRAPNO(%rsp)
	221	movq $0,TF_ERR(%rsp) /* XXX should be the error code */
	222	movq $0,TF_ADDR(%rsp)
	223	FAKE_MCOUNT(TF_RIP(%rsp))
	224	jmp calltrap
	225
	226	/*
	227	* FAST interrupt pending. NOTE: stack context holds frame structure
	228	* for fast interrupt procedure, do not do random pushes or pops!
	229	*/
	230	ALIGN_TEXT
	231	doreti_fast:
	232	andl PCPU(fpending),%ecx /* only check fast ints */
	233	bsfl %ecx, %ecx /* locate the next dispatchable int */
	234	btrl %ecx, PCPU(fpending) /* is it really still pending? */
	235	jnc doreti_next
	236	pushq %rax /* save IRQ mask unavailable for BGL */
	237	/* NOTE: is also CPL in frame */
	238	#if 0
	239	#ifdef SMP
	240	pushq %rcx /* save ecx */
	241	call try_mplock
	242	popq %rcx
	243	testl %eax,%eax
	244	jz 1f
	245	/* MP lock successful */
	246	#endif
	247	#endif
c8fe38ae	248	call dofastunpend /* unpend fast intr %ecx */
c8fe38ae MD	249	#if 0
	250	#ifdef SMP
	251	call rel_mplock
	252	#endif
	253	#endif
	254	popq %rax
	255	jmp doreti_next
	256	1:
	257	btsl %ecx, PCPU(fpending) /* oops, couldn't get the MP lock */
	258	popq %rax /* add to temp. cpl mask to ignore */
	259	orl PCPU(fpending),%eax
	260	jmp doreti_next
	261
	262	/*
c8fe38ae MD	263	* SOFT interrupt pending
	264	*
	265	* Temporarily back-out our critical section to allow an interrupt
	266	* preempt us when we schedule it. Bump intr_nesting_level to
	267	* prevent the switch code from recursing via splz too deeply.
	268	*/
	269	ALIGN_TEXT
	270	doreti_soft:
	271	bsfl %ecx,%ecx /* locate the next pending softint */
	272	btrl %ecx,PCPU(spending) /* make sure its still pending */
	273	jnc doreti_next
	274	addl $FIRST_SOFTINT,%ecx /* actual intr number */
	275	pushq %rax
	276	movl %ecx,%edi /* argument to C call */
	277	incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
	278	subl $TDPRI_CRIT,TD_PRI(%rbx) /* so we can preempt */
	279	call sched_ithd /* YYY must pull in imasks */
	280	addl $TDPRI_CRIT,TD_PRI(%rbx)
	281	decl TD_NEST_COUNT(%rbx)
	282	popq %rax
	283	jmp doreti_next
	284
	285	/*
	286	* AST pending. We clear RQF_AST_SIGNAL automatically, the others
	287	* are cleared by the trap as they are processed.
	288	*
	289	* Temporarily back-out our critical section because trap() can be
	290	* a long-winded call, and we want to be more syscall-like.
	291	*
	292	* YYY theoretically we can call lwkt_switch directly if all we need
	293	* to do is a reschedule.
	294	*/
	295	doreti_ast:
	296	andl $~(RQF_AST_SIGNAL\|RQF_AST_UPCALL),PCPU(reqflags)
	297	sti
e58c365d	298	movl %eax,%r12d /* save cpl (can't use stack) */
c8fe38ae MD	299	movl $T_ASTFLT,TF_TRAPNO(%rsp)
	300	movq %rsp,%rdi /* pass frame by ref (%edi = C arg) */
	301	subl $TDPRI_CRIT,TD_PRI(%rbx)
	302	call trap
	303	addl $TDPRI_CRIT,TD_PRI(%rbx)
e58c365d	304	movl %r12d,%eax /* restore cpl for loop */
c8fe38ae MD	305	jmp doreti_next
	306
	307	#ifdef SMP
	308	/*
	309	* IPIQ message pending. We clear RQF_IPIQ automatically.
	310	*/
	311	doreti_ipiq:
e58c365d	312	movl %eax,%r12d /* save cpl (can't use stack) */
c8fe38ae MD	313	incl PCPU(intr_nesting_level)
c8fe38ae MD	314	andl $~RQF_IPIQ,PCPU(reqflags)
061e4845	315	subq $8,%rsp /* trapframe->intrframe */
c8fe38ae MD	316	movq %rsp,%rdi /* pass frame by ref (C arg) */
c8fe38ae MD	317	call lwkt_process_ipiq_frame
a2a636cc	318	addq $8,%rsp /* intrframe->trapframe */
46d4e165	319	decl PCPU(intr_nesting_level)
e58c365d	320	movl %r12d,%eax /* restore cpl for loop */
46d4e165 JG	321	jmp doreti_next
	322
	323	doreti_timer:
e58c365d	324	movl %eax,%r12d /* save cpl (can't use stack) */
46d4e165 JG	325	incl PCPU(intr_nesting_level)
46d4e165 JG	326	andl $~RQF_TIMER,PCPU(reqflags)
a2a636cc	327	subq $8,%rsp /* trapframe->intrframe */
46d4e165 JG	328	movq %rsp,%rdi /* pass frame by ref (C arg) */
46d4e165 JG	329	call lapic_timer_process_frame
a2a636cc	330	addq $8,%rsp /* intrframe->trapframe */
c8fe38ae	331	decl PCPU(intr_nesting_level)
e58c365d	332	movl %r12d,%eax /* restore cpl for loop */
c8fe38ae MD	333	jmp doreti_next
	334
	335	#endif
	336
	337	/*
	338	* SPLZ() a C callable procedure to dispatch any unmasked pending
	339	* interrupts regardless of critical section nesting. ASTs
	340	* are not dispatched.
	341	*
	342	* Use %eax to track those IRQs that could not be processed
	343	* due to BGL requirements.
	344	*/
	345	SUPERALIGN_TEXT
	346
	347	ENTRY(splz)
	348	pushfq
	349	pushq %rbx
	350	movq PCPU(curthread),%rbx
	351	addl $TDPRI_CRIT,TD_PRI(%rbx)
	352	movl $0,%eax
	353
	354	splz_next:
	355	cli
	356	movl %eax,%ecx /* ecx = ~CPL */
	357	notl %ecx
	358	#ifdef SMP
	359	testl $RQF_IPIQ,PCPU(reqflags)
	360	jnz splz_ipiq
46d4e165 JG	361	testl $RQF_TIMER,PCPU(reqflags)
46d4e165 JG	362	jnz splz_timer
c8fe38ae MD	363	#endif
	364	testl PCPU(fpending),%ecx /* check for an unmasked fast int */
	365	jnz splz_fast
	366
c8fe38ae MD	367	movl PCPU(spending),%ecx
	368	cmpl $0,%ecx
	369	jnz splz_soft
	370
	371	subl $TDPRI_CRIT,TD_PRI(%rbx)
	372
	373	/*
	374	* Nothing left to do, finish up. Interrupts are still disabled.
	375	* If our mask of IRQs we couldn't process due to BGL requirements
	376	* is 0 then there are no pending interrupt sources left and we
	377	* can clear RQF_INTPEND.
	378	*/
	379	testl %eax,%eax
	380	jnz 5f
	381	andl $~RQF_INTPEND,PCPU(reqflags)
	382	5:
	383	popq %rbx
	384	popfq
	385	ret
	386
	387	/*
	388	* FAST interrupt pending
	389	*/
	390	ALIGN_TEXT
	391	splz_fast:
	392	andl PCPU(fpending),%ecx /* only check fast ints */
	393	bsfl %ecx, %ecx /* locate the next dispatchable int */
	394	btrl %ecx, PCPU(fpending) /* is it really still pending? */
	395	jnc splz_next
	396	pushq %rax
	397	#if 0
	398	#ifdef SMP
	399	movl %ecx,%edi /* argument to try_mplock */
	400	call try_mplock
	401	testl %eax,%eax
	402	jz 1f
	403	#endif
	404	#endif
c8fe38ae	405	call dofastunpend /* unpend fast intr %ecx */
c8fe38ae MD	406	#if 0
	407	#ifdef SMP
	408	call rel_mplock
	409	#endif
	410	#endif
	411	popq %rax
	412	jmp splz_next
	413	1:
	414	btsl %ecx, PCPU(fpending) /* oops, couldn't get the MP lock */
	415	popq %rax
	416	orl PCPU(fpending),%eax
	417	jmp splz_next
	418
	419	/*
c8fe38ae MD	420	* SOFT interrupt pending
	421	*
	422	* Temporarily back-out our critical section to allow the interrupt
	423	* preempt us.
	424	*/
	425	ALIGN_TEXT
	426	splz_soft:
	427	bsfl %ecx,%ecx /* locate the next pending softint */
	428	btrl %ecx,PCPU(spending) /* make sure its still pending */
	429	jnc splz_next
	430	addl $FIRST_SOFTINT,%ecx /* actual intr number */
	431	sti
	432	pushq %rax
	433	movl %ecx,%edi /* C argument */
	434	subl $TDPRI_CRIT,TD_PRI(%rbx)
	435	incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
	436	call sched_ithd /* YYY must pull in imasks */
	437	addl $TDPRI_CRIT,TD_PRI(%rbx)
	438	decl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
	439	popq %rax
	440	jmp splz_next
	441
	442	#ifdef SMP
	443	splz_ipiq:
	444	andl $~RQF_IPIQ,PCPU(reqflags)
	445	pushq %rax
	446	call lwkt_process_ipiq
	447	popq %rax
	448	jmp splz_next
46d4e165 JG	449
	450	splz_timer:
	451	andl $~RQF_TIMER,PCPU(reqflags)
	452	pushq %rax
	453	call lapic_timer_process
	454	popq %rax
	455	jmp splz_next
c8fe38ae MD	456	#endif
	457
	458	/*
	459	* dofastunpend(%ecx:intr)
	460	*
	461	* A FAST interrupt previously made pending can now be run,
	462	* execute it by pushing a dummy interrupt frame and
	463	* calling ithread_fast_handler to execute or schedule it.
	464	*
	465	* ithread_fast_handler() returns 0 if it wants us to unmask
	466	* further interrupts.
	467	*/
	468	#define PUSH_DUMMY \
	469	pushfq ; /* phys int frame / flags */ \
	470	movl %cs,%eax ; \
	471	pushq %rax ; /* phys int frame / cs */ \
	472	pushq 38(%rsp) ; / original caller eip */ \
	473	subq $TF_RIP,%rsp ; /* trap frame */ \
5b9f6cc4	474	movq $0,TF_XFLAGS(%rsp) ; /* extras */ \
c8fe38ae MD	475	movq $0,TF_TRAPNO(%rsp) ; /* extras */ \
	476	movq $0,TF_ADDR(%rsp) ; /* extras */ \
	477	movq $0,TF_FLAGS(%rsp) ; /* extras */ \
	478	movq $0,TF_ERR(%rsp) ; /* extras */ \
	479
	480	#define POP_DUMMY \
	481	addq $TF_RIP+(3*8),%rsp ; \
	482
	483	dofastunpend:
	484	pushq %rbp /* frame for backtrace */
	485	movq %rsp,%rbp
	486	PUSH_DUMMY
	487	pushq %rcx /* last part of intrframe = intr */
	488	incl fastunpend_count
	489	movq %rsp,%rdi /* pass frame by reference C arg */
	490	call ithread_fast_handler /* returns 0 to unmask */
	491	popq %rdi /* intrframe->trapframe */
	492	/* + also rdi C arg to next call */
	493	cmpl $0,%eax
	494	jnz 1f
	495	movq MachIntrABI + MACHINTR_INTREN, %rax
	496	callq %rax / MachIntrABI.intren(intr) */
	497	1:
	498	POP_DUMMY
	499	popq %rbp
	500	ret
	501