[dragonfly.git] / sys / platform / pc32 / apic / apic_vector.s

/*
 *	from: vector.s, 386BSD 0.1 unknown origin
 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
 */

#include "opt_auto_eoi.h"

#include <machine/asmacros.h>
#include <machine/lock.h>
#include <machine/psl.h>
#include <machine/trap.h>

#include <machine_base/icu/icu.h>
#include <bus/isa/isa.h>

#include "assym.s"

#include "apicreg.h"
#include "apic_ipl.h"
#include <machine/smp.h>
#include <machine/intr_machdep.h>

/* convert an absolute IRQ# into bitmask */
#define IRQ_LBIT(irq_num)	(1 << ((irq_num) & 0x1f))

/* convert an absolute IRQ# into ipending index */
#define IRQ_LIDX(irq_num)	((irq_num) >> 5)

#ifdef SMP
#define MPLOCKED     lock ;
#else
#define MPLOCKED
#endif

/*
 * Push an interrupt frame in a format acceptable to doreti, reload
 * the segment registers for the kernel.
 */
#define PUSH_FRAME							\
	pushl	$0 ;		/* dummy error code */			\
	pushl	$0 ;		/* dummy trap type */			\
	pushl	$0 ;		/* dummy xflags type */			\
	pushal ;							\
	pushl	%ds ;		/* save data and extra segments ... */	\
	pushl	%es ;							\
	pushl	%fs ;							\
	pushl	%gs ;							\
	cld ;								\
	mov	$KDSEL,%ax ;						\
	mov	%ax,%ds ;						\
	mov	%ax,%es ;						\
	mov	%ax,%gs ;						\
	mov	$KPSEL,%ax ;						\
	mov	%ax,%fs ;						\

/*
 * Warning: POP_FRAME can only be used if there is no chance of a
 * segment register being changed (e.g. by procfs), which is why syscalls
 * have to use doreti.
 */
#define POP_FRAME							\
	popl	%gs ;							\
	popl	%fs ;							\
	popl	%es ;							\
	popl	%ds ;							\
	popal ;								\
	addl	$3*4,%esp ;	/* dummy xflags, trap & error codes */	\

#define IOAPICADDR(irq_num) \
	CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR
#define REDIRIDX(irq_num) \
	CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX
#define IOAPICFLAGS(irq_num) \
	CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS

#define MASK_IRQ(irq_num)						\
	APIC_IMASK_LOCK ;			/* into critical reg */	\
	testl	$IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;		\
	jne	7f ;			/* masked, don't mask */	\
	orl	$IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;		\
						/* set the mask bit */	\
	movl	IOAPICADDR(irq_num), %ecx ;	/* ioapic addr */	\
	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
	movl	%eax, (%ecx) ;			/* write the index */	\
	orl	$IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* set the mask */	\
7: ;						/* already masked */	\
	APIC_IMASK_UNLOCK ;						\

/*
 * Test to see whether we are handling an edge or level triggered INT.
 *  Level-triggered INTs must still be masked as we don't clear the source,
 *  and the EOI cycle would cause redundant INTs to occur.
 */
#define MASK_LEVEL_IRQ(irq_num)						\
	testl	$IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ;		\
	jz	9f ;				/* edge, don't mask */	\
	MASK_IRQ(irq_num) ;						\
9: ;									\

/*
 * Test to see if the source is currntly masked, clear if so.
 */
#define UNMASK_IRQ(irq_num)						\
	cmpl	$0,%eax ;						\
	jnz	8f ;							\
	APIC_IMASK_LOCK ;			/* into critical reg */	\
	testl	$IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;		\
	je	7f ;			/* bit clear, not masked */	\
	andl	$~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;		\
						/* clear mask bit */	\
	movl	IOAPICADDR(irq_num),%ecx ;	/* ioapic addr */	\
	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
	movl	%eax,(%ecx) ;			/* write the index */	\
	andl	$~IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* clear the mask */ \
7: ;									\
	APIC_IMASK_UNLOCK ;						\
8: ;									\

#ifdef SMP /* APIC-IO */

/*
 * Interrupt call handlers run in the following sequence:
 *
 *	- Push the trap frame required by doreti
 *	- Mask the interrupt and reenable its source
 *	- If we cannot take the interrupt set its ipending bit and
 *	  doreti.
 *	- If we can take the interrupt clear its ipending bit,
 *	  call the handler, then unmask and doreti.
 *
 * YYY can cache gd base opitner instead of using hidden %fs prefixes.
 */

#define	INTR_HANDLER(irq_num)						\
	.text ;								\
	SUPERALIGN_TEXT ;						\
IDTVEC(apic_intr##irq_num) ;						\
	PUSH_FRAME ;							\
	FAKE_MCOUNT(15*4(%esp)) ;					\
	MASK_LEVEL_IRQ(irq_num) ;					\
	movl	$0, lapic_eoi ;						\
	movl	PCPU(curthread),%ebx ;					\
	movl	$0,%eax ;	/* CURRENT CPL IN FRAME (REMOVED) */	\
	pushl	%eax ;							\
	testl	$-1,TD_NEST_COUNT(%ebx) ;				\
	jne	1f ;							\
	testl	$-1,TD_CRITCOUNT(%ebx) ;				\
	je	2f ;							\
1: ;									\
	/* in critical section, make interrupt pending */		\
	/* set the pending bit and return, leave interrupt masked */	\
	movl	$IRQ_LIDX(irq_num),%edx ;				\
	orl	$IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ;		\
	orl	$RQF_INTPEND,PCPU(reqflags) ;				\
	jmp	5f ;							\
2: ;									\
	/* clear pending bit, run handler */				\
	movl	$IRQ_LIDX(irq_num),%edx ;				\
	andl	$~IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ;		\
	pushl	$irq_num ;						\
	pushl	%esp ;			 /* pass frame by reference */	\
	incl	TD_CRITCOUNT(%ebx) ;					\
	sti ;								\
	call	ithread_fast_handler ;	 /* returns 0 to unmask */	\
	decl	TD_CRITCOUNT(%ebx) ;					\
	addl	$8, %esp ;						\
	UNMASK_IRQ(irq_num) ;						\
5: ;									\
	MEXITCOUNT ;							\
	jmp	doreti ;						\

#endif

/*
 * Handle "spurious INTerrupts".
 * Notes:
 *  This is different than the "spurious INTerrupt" generated by an
 *   8259 PIC for missing INTs.  See the APIC documentation for details.
 *  This routine should NOT do an 'EOI' cycle.
 */
	.text
	SUPERALIGN_TEXT
	.globl Xspuriousint
Xspuriousint:

	/* No EOI cycle used here */

	iret


/*
 * Handle TLB shootdowns.
 *
 * NOTE: Interrupts remain disabled.
 */
	.text
	SUPERALIGN_TEXT
	.globl	Xinvltlb
Xinvltlb:
	PUSH_FRAME
	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */
	FAKE_MCOUNT(15*4(%esp))

	subl	$8,%esp			/* make same as interrupt frame */
	pushl	%esp			/* pass frame by reference */
	call	smp_invltlb_intr
	addl	$12,%esp

	MEXITCOUNT
	jmp	doreti_syscall_ret

/*
 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
 *
 *  - Signals its receipt.
 *  - Waits for permission to restart.
 *  - Processing pending IPIQ events while waiting.
 *  - Signals its restart.
 */

	.text
	SUPERALIGN_TEXT
	.globl Xcpustop
Xcpustop:
	pushl	%ebp
	movl	%esp, %ebp
	pushl	%eax
	pushl	%ecx
	pushl	%edx
	pushl	%ds			/* save current data segment */
	pushl	%fs

	movl	$KDSEL, %eax
	mov	%ax, %ds		/* use KERNEL data segment */
	movl	$KPSEL, %eax
	mov	%ax, %fs

	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */

	movl	PCPU(cpuid), %eax
	imull	$PCB_SIZE, %eax
	leal	CNAME(stoppcbs)(%eax), %eax
	pushl	%eax
	call	CNAME(savectx)		/* Save process context */
	addl	$4, %esp
	
		
	movl	PCPU(cpuid), %eax

	/*
	 * Indicate that we have stopped and loop waiting for permission
	 * to start again.  We must still process IPI events while in a
	 * stopped state.
	 *
	 * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
	 * (e.g. Xtimer, Xinvltlb).
	 */
	MPLOCKED
	btsl	%eax, stopped_cpus	/* stopped_cpus |= (1<<id) */
	sti
1:
	andl	$~RQF_IPIQ,PCPU(reqflags)
	pushl	%eax
	call	lwkt_smp_stopped
	popl	%eax
	btl	%eax, started_cpus	/* while (!(started_cpus & (1<<id))) */
	jnc	1b

	MPLOCKED
	btrl	%eax, started_cpus	/* started_cpus &= ~(1<<id) */
	MPLOCKED
	btrl	%eax, stopped_cpus	/* stopped_cpus &= ~(1<<id) */

	test	%eax, %eax
	jnz	2f

	movl	CNAME(cpustop_restartfunc), %eax
	test	%eax, %eax
	jz	2f
	movl	$0, CNAME(cpustop_restartfunc)	/* One-shot */

	call	*%eax
2:
	popl	%fs
	popl	%ds			/* restore previous data segment */
	popl	%edx
	popl	%ecx
	popl	%eax
	movl	%ebp, %esp
	popl	%ebp
	iret

	/*
	 * For now just have one ipiq IPI, but what we really want is
	 * to have one for each source cpu to the APICs don't get stalled
	 * backlogging the requests.
	 */
	.text
	SUPERALIGN_TEXT
	.globl Xipiq
Xipiq:
	PUSH_FRAME
	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */
	FAKE_MCOUNT(15*4(%esp))

	incl    PCPU(cnt) + V_IPI
	movl	PCPU(curthread),%ebx
	testl	$-1,TD_CRITCOUNT(%ebx)
	jne	1f
	subl	$8,%esp			/* make same as interrupt frame */
	pushl	%esp			/* pass frame by reference */
	incl	PCPU(intr_nesting_level)
	incl	TD_CRITCOUNT(%ebx)
	sti
	call	lwkt_process_ipiq_frame
	decl	TD_CRITCOUNT(%ebx)
	decl	PCPU(intr_nesting_level)
	addl	$12,%esp
	pushl	$0			/* CPL for frame (REMOVED) */
	MEXITCOUNT
	jmp	doreti
1:
	orl	$RQF_IPIQ,PCPU(reqflags)
	MEXITCOUNT
	jmp	doreti_syscall_ret

	.text
	SUPERALIGN_TEXT
	.globl Xtimer
Xtimer:
	PUSH_FRAME
	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */
	FAKE_MCOUNT(15*4(%esp))

	incl    PCPU(cnt) + V_TIMER
	movl	PCPU(curthread),%ebx
	testl	$-1,TD_CRITCOUNT(%ebx)
	jne	1f
	testl	$-1,TD_NEST_COUNT(%ebx)
	jne	1f
	subl	$8,%esp			/* make same as interrupt frame */
	pushl	%esp			/* pass frame by reference */
	incl	PCPU(intr_nesting_level)
	incl	TD_CRITCOUNT(%ebx)
	sti
	call	lapic_timer_process_frame
	decl	TD_CRITCOUNT(%ebx)
	decl	PCPU(intr_nesting_level)
	addl	$12,%esp
	pushl	$0			/* CPL for frame (REMOVED) */
	MEXITCOUNT
	jmp	doreti
1:
	orl	$RQF_TIMER,PCPU(reqflags)
	MEXITCOUNT
	jmp	doreti_syscall_ret

#ifdef SMP /* APIC-IO */

MCOUNT_LABEL(bintr)
	INTR_HANDLER(0)
	INTR_HANDLER(1)
	INTR_HANDLER(2)
	INTR_HANDLER(3)
	INTR_HANDLER(4)
	INTR_HANDLER(5)
	INTR_HANDLER(6)
	INTR_HANDLER(7)
	INTR_HANDLER(8)
	INTR_HANDLER(9)
	INTR_HANDLER(10)
	INTR_HANDLER(11)
	INTR_HANDLER(12)
	INTR_HANDLER(13)
	INTR_HANDLER(14)
	INTR_HANDLER(15)
	INTR_HANDLER(16)
	INTR_HANDLER(17)
	INTR_HANDLER(18)
	INTR_HANDLER(19)
	INTR_HANDLER(20)
	INTR_HANDLER(21)
	INTR_HANDLER(22)
	INTR_HANDLER(23)
	INTR_HANDLER(24)
	INTR_HANDLER(25)
	INTR_HANDLER(26)
	INTR_HANDLER(27)
	INTR_HANDLER(28)
	INTR_HANDLER(29)
	INTR_HANDLER(30)
	INTR_HANDLER(31)
	INTR_HANDLER(32)
	INTR_HANDLER(33)
	INTR_HANDLER(34)
	INTR_HANDLER(35)
	INTR_HANDLER(36)
	INTR_HANDLER(37)
	INTR_HANDLER(38)
	INTR_HANDLER(39)
	INTR_HANDLER(40)
	INTR_HANDLER(41)
	INTR_HANDLER(42)
	INTR_HANDLER(43)
	INTR_HANDLER(44)
	INTR_HANDLER(45)
	INTR_HANDLER(46)
	INTR_HANDLER(47)
	INTR_HANDLER(48)
	INTR_HANDLER(49)
	INTR_HANDLER(50)
	INTR_HANDLER(51)
	INTR_HANDLER(52)
	INTR_HANDLER(53)
	INTR_HANDLER(54)
	INTR_HANDLER(55)
	INTR_HANDLER(56)
	INTR_HANDLER(57)
	INTR_HANDLER(58)
	INTR_HANDLER(59)
	INTR_HANDLER(60)
	INTR_HANDLER(61)
	INTR_HANDLER(62)
	INTR_HANDLER(63)
	INTR_HANDLER(64)
	INTR_HANDLER(65)
	INTR_HANDLER(66)
	INTR_HANDLER(67)
	INTR_HANDLER(68)
	INTR_HANDLER(69)
	INTR_HANDLER(70)
	INTR_HANDLER(71)
	INTR_HANDLER(72)
	INTR_HANDLER(73)
	INTR_HANDLER(74)
	INTR_HANDLER(75)
	INTR_HANDLER(76)
	INTR_HANDLER(77)
	INTR_HANDLER(78)
	INTR_HANDLER(79)
	INTR_HANDLER(80)
	INTR_HANDLER(81)
	INTR_HANDLER(82)
	INTR_HANDLER(83)
	INTR_HANDLER(84)
	INTR_HANDLER(85)
	INTR_HANDLER(86)
	INTR_HANDLER(87)
	INTR_HANDLER(88)
	INTR_HANDLER(89)
	INTR_HANDLER(90)
	INTR_HANDLER(91)
	INTR_HANDLER(92)
	INTR_HANDLER(93)
	INTR_HANDLER(94)
	INTR_HANDLER(95)
	INTR_HANDLER(96)
	INTR_HANDLER(97)
	INTR_HANDLER(98)
	INTR_HANDLER(99)
	INTR_HANDLER(100)
	INTR_HANDLER(101)
	INTR_HANDLER(102)
	INTR_HANDLER(103)
	INTR_HANDLER(104)
	INTR_HANDLER(105)
	INTR_HANDLER(106)
	INTR_HANDLER(107)
	INTR_HANDLER(108)
	INTR_HANDLER(109)
	INTR_HANDLER(110)
	INTR_HANDLER(111)
	INTR_HANDLER(112)
	INTR_HANDLER(113)
	INTR_HANDLER(114)
	INTR_HANDLER(115)
	INTR_HANDLER(116)
	INTR_HANDLER(117)
	INTR_HANDLER(118)
	INTR_HANDLER(119)
	INTR_HANDLER(120)
	INTR_HANDLER(121)
	INTR_HANDLER(122)
	INTR_HANDLER(123)
	INTR_HANDLER(124)
	INTR_HANDLER(125)
	INTR_HANDLER(126)
	INTR_HANDLER(127)
	INTR_HANDLER(128)
	INTR_HANDLER(129)
	INTR_HANDLER(130)
	INTR_HANDLER(131)
	INTR_HANDLER(132)
	INTR_HANDLER(133)
	INTR_HANDLER(134)
	INTR_HANDLER(135)
	INTR_HANDLER(136)
	INTR_HANDLER(137)
	INTR_HANDLER(138)
	INTR_HANDLER(139)
	INTR_HANDLER(140)
	INTR_HANDLER(141)
	INTR_HANDLER(142)
	INTR_HANDLER(143)
	INTR_HANDLER(144)
	INTR_HANDLER(145)
	INTR_HANDLER(146)
	INTR_HANDLER(147)
	INTR_HANDLER(148)
	INTR_HANDLER(149)
	INTR_HANDLER(150)
	INTR_HANDLER(151)
	INTR_HANDLER(152)
	INTR_HANDLER(153)
	INTR_HANDLER(154)
	INTR_HANDLER(155)
	INTR_HANDLER(156)
	INTR_HANDLER(157)
	INTR_HANDLER(158)
	INTR_HANDLER(159)
	INTR_HANDLER(160)
	INTR_HANDLER(161)
	INTR_HANDLER(162)
	INTR_HANDLER(163)
	INTR_HANDLER(164)
	INTR_HANDLER(165)
	INTR_HANDLER(166)
	INTR_HANDLER(167)
	INTR_HANDLER(168)
	INTR_HANDLER(169)
	INTR_HANDLER(170)
	INTR_HANDLER(171)
	INTR_HANDLER(172)
	INTR_HANDLER(173)
	INTR_HANDLER(174)
	INTR_HANDLER(175)
	INTR_HANDLER(176)
	INTR_HANDLER(177)
	INTR_HANDLER(178)
	INTR_HANDLER(179)
	INTR_HANDLER(180)
	INTR_HANDLER(181)
	INTR_HANDLER(182)
	INTR_HANDLER(183)
	INTR_HANDLER(184)
	INTR_HANDLER(185)
	INTR_HANDLER(186)
	INTR_HANDLER(187)
	INTR_HANDLER(188)
	INTR_HANDLER(189)
	INTR_HANDLER(190)
	INTR_HANDLER(191)
MCOUNT_LABEL(eintr)

#endif

	.data

/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
	.globl stopped_cpus, started_cpus
stopped_cpus:
	.long	0
started_cpus:
	.long	0

	.globl CNAME(cpustop_restartfunc)
CNAME(cpustop_restartfunc):
	.long 0
		
	.text
Commit	Line	Data
984263bc MD	1	/*
	2	* from: vector.s, 386BSD 0.1 unknown origin
	3	* $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
	4	*/
	5
06f5be02 MD	6	#include "opt_auto_eoi.h"
	7
	8	#include <machine/asmacros.h>
06f5be02 MD	9	#include <machine/lock.h>
	10	#include <machine/psl.h>
	11	#include <machine/trap.h>
06f5be02	12
a9295349	13	#include <machine_base/icu/icu.h>
21ce0dfa	14	#include <bus/isa/isa.h>
06f5be02 MD	15
06f5be02 MD	16	#include "assym.s"
984263bc	17
bdc560a1 MD	18	#include "apicreg.h"
bdc560a1 MD	19	#include "apic_ipl.h"
984263bc	20	#include <machine/smp.h>
87cf6827	21	#include <machine/intr_machdep.h>
984263bc	22
c263294b SZ	23	/* convert an absolute IRQ# into bitmask */
c263294b SZ	24	#define IRQ_LBIT(irq_num) (1 << ((irq_num) & 0x1f))
984263bc	25
c263294b SZ	26	/* convert an absolute IRQ# into ipending index */
c263294b SZ	27	#define IRQ_LIDX(irq_num) ((irq_num) >> 5)
984263bc	28
97359a5b MD	29	#ifdef SMP
	30	#define MPLOCKED lock ;
	31	#else
	32	#define MPLOCKED
	33	#endif
	34
984263bc	35	/*
8a8d5d85 MD	36	* Push an interrupt frame in a format acceptable to doreti, reload
8a8d5d85 MD	37	* the segment registers for the kernel.
984263bc MD	38	*/
	39	#define PUSH_FRAME \
	40	pushl $0 ; /* dummy error code */ \
	41	pushl $0 ; /* dummy trap type */ \
4e7c41c5	42	pushl $0 ; /* dummy xflags type */ \
984263bc MD	43	pushal ; \
	44	pushl %ds ; /* save data and extra segments ... */ \
	45	pushl %es ; \
8a8d5d85	46	pushl %fs ; \
4e7c41c5	47	pushl %gs ; \
c885c20e	48	cld ; \
8a8d5d85 MD	49	mov $KDSEL,%ax ; \
	50	mov %ax,%ds ; \
	51	mov %ax,%es ; \
4e7c41c5	52	mov %ax,%gs ; \
8a8d5d85 MD	53	mov $KPSEL,%ax ; \
8a8d5d85 MD	54	mov %ax,%fs ; \
984263bc	55
8a8d5d85 MD	56	/*
	57	* Warning: POP_FRAME can only be used if there is no chance of a
	58	* segment register being changed (e.g. by procfs), which is why syscalls
	59	* have to use doreti.
	60	*/
984263bc	61	#define POP_FRAME \
4e7c41c5	62	popl %gs ; \
984263bc MD	63	popl %fs ; \
	64	popl %es ; \
	65	popl %ds ; \
	66	popal ; \
4e7c41c5	67	addl $34,%esp ; / dummy xflags, trap & error codes */ \
8a8d5d85	68
3d911e0a SZ	69	#define IOAPICADDR(irq_num) \
	70	CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR
	71	#define REDIRIDX(irq_num) \
	72	CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX
0f546930 SZ	73	#define IOAPICFLAGS(irq_num) \
0f546930 SZ	74	CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS
8a8d5d85	75
984263bc	76	#define MASK_IRQ(irq_num) \
97359a5b	77	APIC_IMASK_LOCK ; /* into critical reg */ \
ea689d1c	78	testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
984263bc	79	jne 7f ; /* masked, don't mask */ \
ea689d1c SZ	80	orl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
ea689d1c SZ	81	/* set the mask bit */ \
984263bc MD	82	movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \
	83	movl REDIRIDX(irq_num), %eax ; /* get the index */ \
	84	movl %eax, (%ecx) ; /* write the index */ \
bda48b43	85	orl $IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* set the mask */ \
984263bc	86	7: ; /* already masked */ \
97359a5b	87	APIC_IMASK_UNLOCK ; \
8a8d5d85	88
984263bc MD	89	/*
	90	* Test to see whether we are handling an edge or level triggered INT.
	91	* Level-triggered INTs must still be masked as we don't clear the source,
	92	* and the EOI cycle would cause redundant INTs to occur.
	93	*/
	94	#define MASK_LEVEL_IRQ(irq_num) \
0f546930	95	testl $IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ; \
984263bc MD	96	jz 9f ; /* edge, don't mask */ \
984263bc MD	97	MASK_IRQ(irq_num) ; \
8a8d5d85	98	9: ; \
984263bc	99
984263bc MD	100	/*
	101	* Test to see if the source is currntly masked, clear if so.
	102	*/
ea689d1c	103	#define UNMASK_IRQ(irq_num) \
477d3c1c MD	104	cmpl $0,%eax ; \
477d3c1c MD	105	jnz 8f ; \
97359a5b	106	APIC_IMASK_LOCK ; /* into critical reg */ \
ea689d1c	107	testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
984263bc	108	je 7f ; /* bit clear, not masked */ \
ea689d1c SZ	109	andl $~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
ea689d1c SZ	110	/* clear mask bit */ \
984263bc MD	111	movl IOAPICADDR(irq_num),%ecx ; /* ioapic addr */ \
	112	movl REDIRIDX(irq_num), %eax ; /* get the index */ \
	113	movl %eax,(%ecx) ; /* write the index */ \
bda48b43	114	andl $~IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* clear the mask */ \
984263bc	115	7: ; \
97359a5b	116	APIC_IMASK_UNLOCK ; \
477d3c1c	117	8: ; \
984263bc	118
30c5f287	119	#ifdef SMP /* APIC-IO */
97359a5b	120
8a8d5d85	121	/*
081be8a5	122	* Interrupt call handlers run in the following sequence:
8a8d5d85 MD	123	*
	124	* - Push the trap frame required by doreti
	125	* - Mask the interrupt and reenable its source
c263294b	126	* - If we cannot take the interrupt set its ipending bit and
b5d16701	127	* doreti.
c263294b	128	* - If we can take the interrupt clear its ipending bit,
8a8d5d85 MD	129	* call the handler, then unmask and doreti.
	130	*
	131	* YYY can cache gd base opitner instead of using hidden %fs prefixes.
	132	*/
	133
20a6b6c7	134	#define INTR_HANDLER(irq_num) \
8a8d5d85 MD	135	.text ; \
8a8d5d85 MD	136	SUPERALIGN_TEXT ; \
20a6b6c7	137	IDTVEC(apic_intr##irq_num) ; \
8a8d5d85	138	PUSH_FRAME ; \
4e7c41c5	139	FAKE_MCOUNT(15*4(%esp)) ; \
8a8d5d85	140	MASK_LEVEL_IRQ(irq_num) ; \
35408d22	141	movl $0, lapic_eoi ; \
8a8d5d85	142	movl PCPU(curthread),%ebx ; \
38787eef	143	movl $0,%eax ; /* CURRENT CPL IN FRAME (REMOVED) */ \
984263bc	144	pushl %eax ; \
1be5027b MD	145	testl $-1,TD_NEST_COUNT(%ebx) ; \
1be5027b MD	146	jne 1f ; \
f9235b6d MD	147	testl $-1,TD_CRITCOUNT(%ebx) ; \
f9235b6d MD	148	je 2f ; \
8a8d5d85	149	1: ; \
545a1cd3	150	/* in critical section, make interrupt pending */ \
8a8d5d85	151	/* set the pending bit and return, leave interrupt masked */ \
c263294b SZ	152	movl $IRQ_LIDX(irq_num),%edx ; \
c263294b SZ	153	orl $IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ; \
235957ed	154	orl $RQF_INTPEND,PCPU(reqflags) ; \
8a8d5d85 MD	155	jmp 5f ; \
	156	2: ; \
	157	/* clear pending bit, run handler */ \
c263294b SZ	158	movl $IRQ_LIDX(irq_num),%edx ; \
c263294b SZ	159	andl $~IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ; \
477d3c1c	160	pushl $irq_num ; \
c7eb0589	161	pushl %esp ; /* pass frame by reference */ \
f9235b6d	162	incl TD_CRITCOUNT(%ebx) ; \
2d910aaf	163	sti ; \
477d3c1c	164	call ithread_fast_handler ; /* returns 0 to unmask */ \
f9235b6d	165	decl TD_CRITCOUNT(%ebx) ; \
c7eb0589	166	addl $8, %esp ; \
8a8d5d85 MD	167	UNMASK_IRQ(irq_num) ; \
	168	5: ; \
	169	MEXITCOUNT ; \
	170	jmp doreti ; \
984263bc	171
97359a5b MD	172	#endif
97359a5b MD	173
984263bc MD	174	/*
	175	* Handle "spurious INTerrupts".
	176	* Notes:
	177	* This is different than the "spurious INTerrupt" generated by an
	178	* 8259 PIC for missing INTs. See the APIC documentation for details.
	179	* This routine should NOT do an 'EOI' cycle.
	180	*/
	181	.text
	182	SUPERALIGN_TEXT
2954c92f MD	183	.globl Xspuriousint
2954c92f MD	184	Xspuriousint:
984263bc MD	185
	186	/* No EOI cycle used here */
	187
	188	iret
	189
	190
	191	/*
	192	* Handle TLB shootdowns.
2d910aaf MD	193	*
2d910aaf MD	194	* NOTE: Interrupts remain disabled.
984263bc MD	195	*/
	196	.text
	197	SUPERALIGN_TEXT
8a8d5d85 MD	198	.globl Xinvltlb
8a8d5d85 MD	199	Xinvltlb:
7d4d6fdb	200	PUSH_FRAME
984263bc	201	movl $0, lapic_eoi /* End Of Interrupt to APIC */
7d4d6fdb	202	FAKE_MCOUNT(15*4(%esp))
984263bc	203
7d4d6fdb MD	204	subl $8,%esp /* make same as interrupt frame */
	205	pushl %esp /* pass frame by reference */
	206	call smp_invltlb_intr
	207	addl $12,%esp
984263bc	208
7d4d6fdb MD	209	MEXITCOUNT
7d4d6fdb MD	210	jmp doreti_syscall_ret
984263bc	211
984263bc	212	/*
984263bc MD	213	* Executed by a CPU when it receives an Xcpustop IPI from another CPU,
	214	*
	215	* - Signals its receipt.
	216	* - Waits for permission to restart.
bd8015ca	217	* - Processing pending IPIQ events while waiting.
984263bc MD	218	* - Signals its restart.
	219	*/
	220
	221	.text
	222	SUPERALIGN_TEXT
2954c92f MD	223	.globl Xcpustop
2954c92f MD	224	Xcpustop:
984263bc MD	225	pushl %ebp
	226	movl %esp, %ebp
	227	pushl %eax
	228	pushl %ecx
	229	pushl %edx
	230	pushl %ds /* save current data segment */
	231	pushl %fs
	232
	233	movl $KDSEL, %eax
	234	mov %ax, %ds /* use KERNEL data segment */
	235	movl $KPSEL, %eax
	236	mov %ax, %fs
	237
	238	movl $0, lapic_eoi /* End Of Interrupt to APIC */
	239
2954c92f	240	movl PCPU(cpuid), %eax
984263bc MD	241	imull $PCB_SIZE, %eax
	242	leal CNAME(stoppcbs)(%eax), %eax
	243	pushl %eax
	244	call CNAME(savectx) /* Save process context */
	245	addl $4, %esp
	246
	247
2954c92f	248	movl PCPU(cpuid), %eax
984263bc	249
bd8015ca MD	250	/*
	251	* Indicate that we have stopped and loop waiting for permission
	252	* to start again. We must still process IPI events while in a
	253	* stopped state.
2d910aaf MD	254	*
	255	* Interrupts must remain enabled for non-IPI'd per-cpu interrupts
	256	* (e.g. Xtimer, Xinvltlb).
bd8015ca	257	*/
97359a5b	258	MPLOCKED
2954c92f	259	btsl %eax, stopped_cpus /* stopped_cpus \|= (1<<id) */
2d910aaf	260	sti
984263bc	261	1:
bd8015ca MD	262	andl $~RQF_IPIQ,PCPU(reqflags)
	263	pushl %eax
	264	call lwkt_smp_stopped
	265	popl %eax
2954c92f	266	btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
984263bc MD	267	jnc 1b
984263bc MD	268
97359a5b	269	MPLOCKED
2954c92f	270	btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
97359a5b	271	MPLOCKED
2954c92f	272	btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
984263bc MD	273
	274	test %eax, %eax
	275	jnz 2f
	276
	277	movl CNAME(cpustop_restartfunc), %eax
	278	test %eax, %eax
	279	jz 2f
	280	movl $0, CNAME(cpustop_restartfunc) /* One-shot */
	281
	282	call *%eax
	283	2:
	284	popl %fs
	285	popl %ds /* restore previous data segment */
	286	popl %edx
	287	popl %ecx
	288	popl %eax
	289	movl %ebp, %esp
	290	popl %ebp
	291	iret
	292
96728c05 MD	293	/*
	294	* For now just have one ipiq IPI, but what we really want is
	295	* to have one for each source cpu to the APICs don't get stalled
	296	* backlogging the requests.
	297	*/
	298	.text
	299	SUPERALIGN_TEXT
	300	.globl Xipiq
	301	Xipiq:
	302	PUSH_FRAME
	303	movl $0, lapic_eoi /* End Of Interrupt to APIC */
4e7c41c5	304	FAKE_MCOUNT(15*4(%esp))
96728c05	305
b785701b	306	incl PCPU(cnt) + V_IPI
96728c05	307	movl PCPU(curthread),%ebx
f9235b6d MD	308	testl $-1,TD_CRITCOUNT(%ebx)
f9235b6d MD	309	jne 1f
88c4d2f6	310	subl $8,%esp /* make same as interrupt frame */
c7eb0589	311	pushl %esp /* pass frame by reference */
03aa8d99	312	incl PCPU(intr_nesting_level)
f9235b6d	313	incl TD_CRITCOUNT(%ebx)
2d910aaf	314	sti
88c4d2f6	315	call lwkt_process_ipiq_frame
f9235b6d	316	decl TD_CRITCOUNT(%ebx)
03aa8d99	317	decl PCPU(intr_nesting_level)
c7eb0589	318	addl $12,%esp
38787eef	319	pushl $0 /* CPL for frame (REMOVED) */
96728c05 MD	320	MEXITCOUNT
	321	jmp doreti
	322	1:
235957ed	323	orl $RQF_IPIQ,PCPU(reqflags)
96728c05	324	MEXITCOUNT
7d4d6fdb	325	jmp doreti_syscall_ret
984263bc	326
78ea5a2a SZ	327	.text
	328	SUPERALIGN_TEXT
	329	.globl Xtimer
	330	Xtimer:
	331	PUSH_FRAME
	332	movl $0, lapic_eoi /* End Of Interrupt to APIC */
	333	FAKE_MCOUNT(15*4(%esp))
	334
b785701b	335	incl PCPU(cnt) + V_TIMER
78ea5a2a	336	movl PCPU(curthread),%ebx
f9235b6d MD	337	testl $-1,TD_CRITCOUNT(%ebx)
f9235b6d MD	338	jne 1f
5119f32b SZ	339	testl $-1,TD_NEST_COUNT(%ebx)
5119f32b SZ	340	jne 1f
78ea5a2a SZ	341	subl $8,%esp /* make same as interrupt frame */
	342	pushl %esp /* pass frame by reference */
	343	incl PCPU(intr_nesting_level)
f9235b6d	344	incl TD_CRITCOUNT(%ebx)
2d910aaf	345	sti
78ea5a2a	346	call lapic_timer_process_frame
f9235b6d	347	decl TD_CRITCOUNT(%ebx)
78ea5a2a SZ	348	decl PCPU(intr_nesting_level)
	349	addl $12,%esp
	350	pushl $0 /* CPL for frame (REMOVED) */
	351	MEXITCOUNT
	352	jmp doreti
	353	1:
	354	orl $RQF_TIMER,PCPU(reqflags)
	355	MEXITCOUNT
7d4d6fdb	356	jmp doreti_syscall_ret
78ea5a2a	357
30c5f287	358	#ifdef SMP /* APIC-IO */
97359a5b	359
984263bc	360	MCOUNT_LABEL(bintr)
20a6b6c7 SZ	361	INTR_HANDLER(0)
	362	INTR_HANDLER(1)
	363	INTR_HANDLER(2)
	364	INTR_HANDLER(3)
	365	INTR_HANDLER(4)
	366	INTR_HANDLER(5)
	367	INTR_HANDLER(6)
	368	INTR_HANDLER(7)
	369	INTR_HANDLER(8)
	370	INTR_HANDLER(9)
	371	INTR_HANDLER(10)
	372	INTR_HANDLER(11)
	373	INTR_HANDLER(12)
	374	INTR_HANDLER(13)
	375	INTR_HANDLER(14)
	376	INTR_HANDLER(15)
	377	INTR_HANDLER(16)
	378	INTR_HANDLER(17)
	379	INTR_HANDLER(18)
	380	INTR_HANDLER(19)
	381	INTR_HANDLER(20)
	382	INTR_HANDLER(21)
	383	INTR_HANDLER(22)
	384	INTR_HANDLER(23)
	385	INTR_HANDLER(24)
	386	INTR_HANDLER(25)
	387	INTR_HANDLER(26)
	388	INTR_HANDLER(27)
	389	INTR_HANDLER(28)
	390	INTR_HANDLER(29)
	391	INTR_HANDLER(30)
	392	INTR_HANDLER(31)
c571da4a SZ	393	INTR_HANDLER(32)
	394	INTR_HANDLER(33)
	395	INTR_HANDLER(34)
	396	INTR_HANDLER(35)
	397	INTR_HANDLER(36)
	398	INTR_HANDLER(37)
	399	INTR_HANDLER(38)
	400	INTR_HANDLER(39)
	401	INTR_HANDLER(40)
	402	INTR_HANDLER(41)
	403	INTR_HANDLER(42)
	404	INTR_HANDLER(43)
	405	INTR_HANDLER(44)
	406	INTR_HANDLER(45)
	407	INTR_HANDLER(46)
	408	INTR_HANDLER(47)
	409	INTR_HANDLER(48)
	410	INTR_HANDLER(49)
	411	INTR_HANDLER(50)
	412	INTR_HANDLER(51)
	413	INTR_HANDLER(52)
	414	INTR_HANDLER(53)
	415	INTR_HANDLER(54)
	416	INTR_HANDLER(55)
	417	INTR_HANDLER(56)
	418	INTR_HANDLER(57)
	419	INTR_HANDLER(58)
	420	INTR_HANDLER(59)
	421	INTR_HANDLER(60)
	422	INTR_HANDLER(61)
	423	INTR_HANDLER(62)
	424	INTR_HANDLER(63)
	425	INTR_HANDLER(64)
	426	INTR_HANDLER(65)
	427	INTR_HANDLER(66)
	428	INTR_HANDLER(67)
	429	INTR_HANDLER(68)
	430	INTR_HANDLER(69)
	431	INTR_HANDLER(70)
	432	INTR_HANDLER(71)
	433	INTR_HANDLER(72)
	434	INTR_HANDLER(73)
	435	INTR_HANDLER(74)
	436	INTR_HANDLER(75)
	437	INTR_HANDLER(76)
	438	INTR_HANDLER(77)
	439	INTR_HANDLER(78)
	440	INTR_HANDLER(79)
	441	INTR_HANDLER(80)
	442	INTR_HANDLER(81)
	443	INTR_HANDLER(82)
	444	INTR_HANDLER(83)
	445	INTR_HANDLER(84)
	446	INTR_HANDLER(85)
	447	INTR_HANDLER(86)
	448	INTR_HANDLER(87)
	449	INTR_HANDLER(88)
	450	INTR_HANDLER(89)
	451	INTR_HANDLER(90)
	452	INTR_HANDLER(91)
	453	INTR_HANDLER(92)
	454	INTR_HANDLER(93)
	455	INTR_HANDLER(94)
	456	INTR_HANDLER(95)
457	INTR_HANDLER(96)
458	INTR_HANDLER(97)
459	INTR_HANDLER(98)
460	INTR_HANDLER(99)
461	INTR_HANDLER(100)
462	INTR_HANDLER(101)
463	INTR_HANDLER(102)
464	INTR_HANDLER(103)
465	INTR_HANDLER(104)
466	INTR_HANDLER(105)
467	INTR_HANDLER(106)
468	INTR_HANDLER(107)
469	INTR_HANDLER(108)
470	INTR_HANDLER(109)
471	INTR_HANDLER(110)
472	INTR_HANDLER(111)
473	INTR_HANDLER(112)
474	INTR_HANDLER(113)
475	INTR_HANDLER(114)
476	INTR_HANDLER(115)
477	INTR_HANDLER(116)
478	INTR_HANDLER(117)
479	INTR_HANDLER(118)
480	INTR_HANDLER(119)
481	INTR_HANDLER(120)
482	INTR_HANDLER(121)
483	INTR_HANDLER(122)
484	INTR_HANDLER(123)
485	INTR_HANDLER(124)
486	INTR_HANDLER(125)
487	INTR_HANDLER(126)
488	INTR_HANDLER(127)
489	INTR_HANDLER(128)
490	INTR_HANDLER(129)
491	INTR_HANDLER(130)
492	INTR_HANDLER(131)
493	INTR_HANDLER(132)
494	INTR_HANDLER(133)
495	INTR_HANDLER(134)
496	INTR_HANDLER(135)
497	INTR_HANDLER(136)
498	INTR_HANDLER(137)
499	INTR_HANDLER(138)
500	INTR_HANDLER(139)
501	INTR_HANDLER(140)
502	INTR_HANDLER(141)
503	INTR_HANDLER(142)
504	INTR_HANDLER(143)
505	INTR_HANDLER(144)
506	INTR_HANDLER(145)
507	INTR_HANDLER(146)
508	INTR_HANDLER(147)
509	INTR_HANDLER(148)
510	INTR_HANDLER(149)
511	INTR_HANDLER(150)
512	INTR_HANDLER(151)
513	INTR_HANDLER(152)
514	INTR_HANDLER(153)
515	INTR_HANDLER(154)
516	INTR_HANDLER(155)
517	INTR_HANDLER(156)
518	INTR_HANDLER(157)
519	INTR_HANDLER(158)
520	INTR_HANDLER(159)
521	INTR_HANDLER(160)
522	INTR_HANDLER(161)
523	INTR_HANDLER(162)
524	INTR_HANDLER(163)
525	INTR_HANDLER(164)
526	INTR_HANDLER(165)
527	INTR_HANDLER(166)
528	INTR_HANDLER(167)
529	INTR_HANDLER(168)
530	INTR_HANDLER(169)
531	INTR_HANDLER(170)
532	INTR_HANDLER(171)
533	INTR_HANDLER(172)
534	INTR_HANDLER(173)
535	INTR_HANDLER(174)
536	INTR_HANDLER(175)
537	INTR_HANDLER(176)
538	INTR_HANDLER(177)
539	INTR_HANDLER(178)
540	INTR_HANDLER(179)
541	INTR_HANDLER(180)
542	INTR_HANDLER(181)
543	INTR_HANDLER(182)
544	INTR_HANDLER(183)
545	INTR_HANDLER(184)
546	INTR_HANDLER(185)
547	INTR_HANDLER(186)
548	INTR_HANDLER(187)
549	INTR_HANDLER(188)
550	INTR_HANDLER(189)
551	INTR_HANDLER(190)
552	INTR_HANDLER(191)
984263bc MD	553	MCOUNT_LABEL(eintr)
984263bc MD	554
97359a5b MD	555	#endif
97359a5b MD	556
984263bc	557	.data
ef0fdad1	558
984263bc	559	/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
2954c92f MD	560	.globl stopped_cpus, started_cpus
2954c92f MD	561	stopped_cpus:
984263bc	562	.long 0
2954c92f	563	started_cpus:
984263bc MD	564	.long 0
984263bc MD	565
984263bc	566	.globl CNAME(cpustop_restartfunc)
984263bc MD	567	CNAME(cpustop_restartfunc):
	568	.long 0
	569
984263bc	570	.text
06f5be02	571