[dragonfly.git] / sys / platform / pc64 / apic / apic_vector.s

/*
 *	from: vector.s, 386BSD 0.1 unknown origin
 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
 */

#if 0
#include "opt_auto_eoi.h"
#endif

#include <machine/asmacros.h>
#include <machine/lock.h>
#include <machine/psl.h>
#include <machine/trap.h>
#include <machine/segments.h>

#include <machine_base/icu/icu.h>
#include <bus/isa/isa.h>

#include "assym.s"

#include "apicreg.h"
#include <machine_base/apic/ioapic_ipl.h>
#include <machine/intr_machdep.h>

#ifdef foo
/* convert an absolute IRQ# into bitmask */
#define IRQ_LBIT(irq_num)	(1UL << (irq_num & 0x3f))
#endif

#define IRQ_SBITS(irq_num)	((irq_num) & 0x3f)

/* convert an absolute IRQ# into gd_ipending index */
#define IRQ_LIDX(irq_num)	((irq_num) >> 6)

#ifdef SMP
#define MPLOCKED     lock ;
#else
#define MPLOCKED
#endif

#define APIC_PUSH_FRAME							\
	PUSH_FRAME ;		/* 15 regs + space for 5 extras */	\
	movq $0,TF_XFLAGS(%rsp) ;					\
	movq $0,TF_TRAPNO(%rsp) ;					\
	movq $0,TF_ADDR(%rsp) ;						\
	movq $0,TF_FLAGS(%rsp) ;					\
	movq $0,TF_ERR(%rsp) ;						\
	cld ;								\

/*
 * JG stale? Warning: POP_FRAME can only be used if there is no chance of a
 * segment register being changed (e.g. by procfs), which is why syscalls
 * have to use doreti.
 */
#define APIC_POP_FRAME							\
	POP_FRAME ;							\

#define IOAPICADDR(irq_num) \
	CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_ADDR
#define REDIRIDX(irq_num) \
	CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_IDX
#define IOAPICFLAGS(irq_num) \
	CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_FLAGS
 
#define MASK_IRQ(irq_num)						\
	IOAPIC_IMASK_LOCK ;			/* into critical reg */	\
	testl	$IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;	\
	jne	7f ;			/* masked, don't mask */	\
	orl	$IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;	\
						/* set the mask bit */	\
	movq	IOAPICADDR(irq_num), %rcx ;	/* ioapic addr */	\
	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
	movl	%eax, (%rcx) ;			/* write the index */	\
	orl	$IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* set the mask */	\
7: ;						/* already masked */	\
	IOAPIC_IMASK_UNLOCK ;						\

/*
 * Test to see whether we are handling an edge or level triggered INT.
 *  Level-triggered INTs must still be masked as we don't clear the source,
 *  and the EOI cycle would cause redundant INTs to occur.
 */
#define MASK_LEVEL_IRQ(irq_num)						\
	testl	$IOAPIC_IRQI_FLAG_LEVEL, IOAPICFLAGS(irq_num) ;		\
	jz	9f ;				/* edge, don't mask */	\
	MASK_IRQ(irq_num) ;						\
9: ;									\

/*
 * Test to see if the source is currntly masked, clear if so.
 */
#define UNMASK_IRQ(irq_num)					\
	cmpl	$0,%eax ;						\
	jnz	8f ;							\
	IOAPIC_IMASK_LOCK ;			/* into critical reg */	\
	testl	$IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;	\
	je	7f ;			/* bit clear, not masked */	\
	andl	$~IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;	\
						/* clear mask bit */	\
	movq	IOAPICADDR(irq_num),%rcx ;	/* ioapic addr */	\
	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
	movl	%eax,(%rcx) ;			/* write the index */	\
	andl	$~IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* clear the mask */ \
7: ;									\
	IOAPIC_IMASK_UNLOCK ;						\
8: ;									\

/*
 * Interrupt call handlers run in the following sequence:
 *
 *	- Push the trap frame required by doreti
 *	- Mask the interrupt and reenable its source
 *	- If we cannot take the interrupt set its ipending bit and
 *	  doreti.
 *	- If we can take the interrupt clear its ipending bit,
 *	  call the handler, then unmask and doreti.
 *
 * YYY can cache gd base opitner instead of using hidden %fs prefixes.
 */

#define	INTR_HANDLER(irq_num)						\
	.text ;								\
	SUPERALIGN_TEXT ;						\
IDTVEC(ioapic_intr##irq_num) ;						\
	APIC_PUSH_FRAME ;						\
	FAKE_MCOUNT(TF_RIP(%rsp)) ;					\
	MASK_LEVEL_IRQ(irq_num) ;					\
	movq	lapic, %rax ;						\
	movl	$0, LA_EOI(%rax) ;					\
	movq	PCPU(curthread),%rbx ;					\
	testl	$-1,TD_NEST_COUNT(%rbx) ;				\
	jne	1f ;							\
	testl	$-1,TD_CRITCOUNT(%rbx) ;				\
	je	2f ;							\
1: ;									\
	/* in critical section, make interrupt pending */		\
	/* set the pending bit and return, leave interrupt masked */	\
	movq	$1,%rcx ;						\
	shlq	$IRQ_SBITS(irq_num),%rcx ;				\
	movq	$IRQ_LIDX(irq_num),%rdx ;				\
	orq	%rcx,PCPU_E8(ipending,%rdx) ;				\
	orl	$RQF_INTPEND,PCPU(reqflags) ;				\
	jmp	5f ;							\
2: ;									\
	/* clear pending bit, run handler */				\
	movq	$1,%rcx ;						\
	shlq	$IRQ_SBITS(irq_num),%rcx ;				\
	notq	%rcx ;							\
	movq	$IRQ_LIDX(irq_num),%rdx ;				\
	andq	%rcx,PCPU_E8(ipending,%rdx) ;				\
	pushq	$irq_num ;		/* trapframe -> intrframe */	\
	movq	%rsp, %rdi ;		/* pass frame by reference */	\
	incl	TD_CRITCOUNT(%rbx) ;					\
	sti ;								\
	call	ithread_fast_handler ;	/* returns 0 to unmask */	\
	decl	TD_CRITCOUNT(%rbx) ;					\
	addq	$8, %rsp ;		/* intrframe -> trapframe */	\
	UNMASK_IRQ(irq_num) ;						\
5: ;									\
	MEXITCOUNT ;							\
	jmp	doreti ;						\

/*
 * Handle "spurious INTerrupts".
 *
 * NOTE: This is different than the "spurious INTerrupt" generated by an
 *	 8259 PIC for missing INTs.  See the APIC documentation for details.
 *	 This routine should NOT do an 'EOI' cycle.
 *
 * NOTE: Even though we don't do anything here we must still swapgs if
 *	 coming from a user frame in case the iretq faults... just use
 *	 the nominal APIC_PUSH_FRAME sequence to get it done.
 */
	.text
	SUPERALIGN_TEXT
	.globl Xspuriousint
Xspuriousint:
	APIC_PUSH_FRAME
	/* No EOI cycle used here */
	FAKE_MCOUNT(TF_RIP(%rsp))
	MEXITCOUNT
	APIC_POP_FRAME
	jmp	doreti_iret

#ifdef SMP

/*
 * Handle TLB shootdowns.
 *
 * NOTE: interrupts are left disabled.
 */
	.text
	SUPERALIGN_TEXT
	.globl	Xinvltlb
Xinvltlb:
	APIC_PUSH_FRAME
	movq	lapic, %rax
	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
	FAKE_MCOUNT(TF_RIP(%rsp))
	subq	$8,%rsp			/* make same as interrupt frame */
	movq	%rsp,%rdi		/* pass frame by reference */
	call	smp_invltlb_intr
	addq	$8,%rsp			/* turn into trapframe */
	MEXITCOUNT
	APIC_POP_FRAME
	jmp	doreti_iret

/*
 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
 *
 *  - We cannot call doreti
 *  - Signals its receipt.
 *  - Waits for permission to restart.
 *  - Processing pending IPIQ events while waiting.
 *  - Signals its restart.
 */

	.text
	SUPERALIGN_TEXT
	.globl Xcpustop
Xcpustop:
	APIC_PUSH_FRAME
	movq	lapic, %rax
	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */

	movl	PCPU(cpuid), %eax
	imull	$PCB_SIZE, %eax
	leaq	CNAME(stoppcbs), %rdi
	addq	%rax, %rdi
	call	CNAME(savectx)		/* Save process context */

	movslq	PCPU(cpuid), %rax

	/*
	 * Indicate that we have stopped and loop waiting for permission
	 * to start again.  We must still process IPI events while in a
	 * stopped state.
	 *
	 * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
	 * (e.g. Xtimer, Xinvltlb).
	 */
	MPLOCKED
	btsq	%rax, stopped_cpus	/* stopped_cpus |= (1<<id) */
	sti
1:
	andl	$~RQF_IPIQ,PCPU(reqflags)
	pushq	%rax
	call	lwkt_smp_stopped
	popq	%rax
	pause
	btq	%rax, started_cpus	/* while (!(started_cpus & (1<<id))) */
	jnc	1b

	MPLOCKED
	btrq	%rax, started_cpus	/* started_cpus &= ~(1<<id) */
	MPLOCKED
	btrq	%rax, stopped_cpus	/* stopped_cpus &= ~(1<<id) */

	test	%eax, %eax
	jnz	2f

	movq	CNAME(cpustop_restartfunc), %rax
	test	%rax, %rax
	jz	2f
	movq	$0, CNAME(cpustop_restartfunc)	/* One-shot */

	call	*%rax
2:
	MEXITCOUNT
	APIC_POP_FRAME
	jmp	doreti_iret

	/*
	 * For now just have one ipiq IPI, but what we really want is
	 * to have one for each source cpu to the APICs don't get stalled
	 * backlogging the requests.
	 */
	.text
	SUPERALIGN_TEXT
	.globl Xipiq
Xipiq:
	APIC_PUSH_FRAME
	movq	lapic, %rax
	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
	FAKE_MCOUNT(TF_RIP(%rsp))

	incl    PCPU(cnt) + V_IPI
	movq	PCPU(curthread),%rbx
	testl	$-1,TD_CRITCOUNT(%rbx)
	jne	1f
	subq	$8,%rsp			/* make same as interrupt frame */
	movq	%rsp,%rdi		/* pass frame by reference */
	incl	PCPU(intr_nesting_level)
	incl	TD_CRITCOUNT(%rbx)
	sti
	call	lwkt_process_ipiq_frame
	decl	TD_CRITCOUNT(%rbx)
	decl	PCPU(intr_nesting_level)
	addq	$8,%rsp			/* turn into trapframe */
	MEXITCOUNT
	jmp	doreti
1:
	orl	$RQF_IPIQ,PCPU(reqflags)
	MEXITCOUNT
	APIC_POP_FRAME
	jmp	doreti_iret

#endif	/* SMP */

	.text
	SUPERALIGN_TEXT
	.globl Xtimer
Xtimer:
	APIC_PUSH_FRAME
	movq	lapic, %rax
	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
	FAKE_MCOUNT(TF_RIP(%rsp))

	subq	$8,%rsp			/* make same as interrupt frame */
	movq	%rsp,%rdi		/* pass frame by reference */
	call	lapic_timer_always
	addq	$8,%rsp			/* turn into trapframe */

	incl    PCPU(cnt) + V_TIMER
	movq	PCPU(curthread),%rbx
	testl	$-1,TD_CRITCOUNT(%rbx)
	jne	1f
	testl	$-1,TD_NEST_COUNT(%rbx)
	jne	1f
	subq	$8,%rsp			/* make same as interrupt frame */
	movq	%rsp,%rdi		/* pass frame by reference */
	incl	PCPU(intr_nesting_level)
	incl	TD_CRITCOUNT(%rbx)
	sti
	call	lapic_timer_process_frame
	decl	TD_CRITCOUNT(%rbx)
	decl	PCPU(intr_nesting_level)
	addq	$8,%rsp			/* turn into trapframe */
	MEXITCOUNT
	jmp	doreti
1:
	orl	$RQF_TIMER,PCPU(reqflags)
	MEXITCOUNT
	APIC_POP_FRAME
	jmp	doreti_iret

MCOUNT_LABEL(bintr)
	INTR_HANDLER(0)
	INTR_HANDLER(1)
	INTR_HANDLER(2)
	INTR_HANDLER(3)
	INTR_HANDLER(4)
	INTR_HANDLER(5)
	INTR_HANDLER(6)
	INTR_HANDLER(7)
	INTR_HANDLER(8)
	INTR_HANDLER(9)
	INTR_HANDLER(10)
	INTR_HANDLER(11)
	INTR_HANDLER(12)
	INTR_HANDLER(13)
	INTR_HANDLER(14)
	INTR_HANDLER(15)
	INTR_HANDLER(16)
	INTR_HANDLER(17)
	INTR_HANDLER(18)
	INTR_HANDLER(19)
	INTR_HANDLER(20)
	INTR_HANDLER(21)
	INTR_HANDLER(22)
	INTR_HANDLER(23)
	INTR_HANDLER(24)
	INTR_HANDLER(25)
	INTR_HANDLER(26)
	INTR_HANDLER(27)
	INTR_HANDLER(28)
	INTR_HANDLER(29)
	INTR_HANDLER(30)
	INTR_HANDLER(31)
	INTR_HANDLER(32)
	INTR_HANDLER(33)
	INTR_HANDLER(34)
	INTR_HANDLER(35)
	INTR_HANDLER(36)
	INTR_HANDLER(37)
	INTR_HANDLER(38)
	INTR_HANDLER(39)
	INTR_HANDLER(40)
	INTR_HANDLER(41)
	INTR_HANDLER(42)
	INTR_HANDLER(43)
	INTR_HANDLER(44)
	INTR_HANDLER(45)
	INTR_HANDLER(46)
	INTR_HANDLER(47)
	INTR_HANDLER(48)
	INTR_HANDLER(49)
	INTR_HANDLER(50)
	INTR_HANDLER(51)
	INTR_HANDLER(52)
	INTR_HANDLER(53)
	INTR_HANDLER(54)
	INTR_HANDLER(55)
	INTR_HANDLER(56)
	INTR_HANDLER(57)
	INTR_HANDLER(58)
	INTR_HANDLER(59)
	INTR_HANDLER(60)
	INTR_HANDLER(61)
	INTR_HANDLER(62)
	INTR_HANDLER(63)
	INTR_HANDLER(64)
	INTR_HANDLER(65)
	INTR_HANDLER(66)
	INTR_HANDLER(67)
	INTR_HANDLER(68)
	INTR_HANDLER(69)
	INTR_HANDLER(70)
	INTR_HANDLER(71)
	INTR_HANDLER(72)
	INTR_HANDLER(73)
	INTR_HANDLER(74)
	INTR_HANDLER(75)
	INTR_HANDLER(76)
	INTR_HANDLER(77)
	INTR_HANDLER(78)
	INTR_HANDLER(79)
	INTR_HANDLER(80)
	INTR_HANDLER(81)
	INTR_HANDLER(82)
	INTR_HANDLER(83)
	INTR_HANDLER(84)
	INTR_HANDLER(85)
	INTR_HANDLER(86)
	INTR_HANDLER(87)
	INTR_HANDLER(88)
	INTR_HANDLER(89)
	INTR_HANDLER(90)
	INTR_HANDLER(91)
	INTR_HANDLER(92)
	INTR_HANDLER(93)
	INTR_HANDLER(94)
	INTR_HANDLER(95)
	INTR_HANDLER(96)
	INTR_HANDLER(97)
	INTR_HANDLER(98)
	INTR_HANDLER(99)
	INTR_HANDLER(100)
	INTR_HANDLER(101)
	INTR_HANDLER(102)
	INTR_HANDLER(103)
	INTR_HANDLER(104)
	INTR_HANDLER(105)
	INTR_HANDLER(106)
	INTR_HANDLER(107)
	INTR_HANDLER(108)
	INTR_HANDLER(109)
	INTR_HANDLER(110)
	INTR_HANDLER(111)
	INTR_HANDLER(112)
	INTR_HANDLER(113)
	INTR_HANDLER(114)
	INTR_HANDLER(115)
	INTR_HANDLER(116)
	INTR_HANDLER(117)
	INTR_HANDLER(118)
	INTR_HANDLER(119)
	INTR_HANDLER(120)
	INTR_HANDLER(121)
	INTR_HANDLER(122)
	INTR_HANDLER(123)
	INTR_HANDLER(124)
	INTR_HANDLER(125)
	INTR_HANDLER(126)
	INTR_HANDLER(127)
	INTR_HANDLER(128)
	INTR_HANDLER(129)
	INTR_HANDLER(130)
	INTR_HANDLER(131)
	INTR_HANDLER(132)
	INTR_HANDLER(133)
	INTR_HANDLER(134)
	INTR_HANDLER(135)
	INTR_HANDLER(136)
	INTR_HANDLER(137)
	INTR_HANDLER(138)
	INTR_HANDLER(139)
	INTR_HANDLER(140)
	INTR_HANDLER(141)
	INTR_HANDLER(142)
	INTR_HANDLER(143)
	INTR_HANDLER(144)
	INTR_HANDLER(145)
	INTR_HANDLER(146)
	INTR_HANDLER(147)
	INTR_HANDLER(148)
	INTR_HANDLER(149)
	INTR_HANDLER(150)
	INTR_HANDLER(151)
	INTR_HANDLER(152)
	INTR_HANDLER(153)
	INTR_HANDLER(154)
	INTR_HANDLER(155)
	INTR_HANDLER(156)
	INTR_HANDLER(157)
	INTR_HANDLER(158)
	INTR_HANDLER(159)
	INTR_HANDLER(160)
	INTR_HANDLER(161)
	INTR_HANDLER(162)
	INTR_HANDLER(163)
	INTR_HANDLER(164)
	INTR_HANDLER(165)
	INTR_HANDLER(166)
	INTR_HANDLER(167)
	INTR_HANDLER(168)
	INTR_HANDLER(169)
	INTR_HANDLER(170)
	INTR_HANDLER(171)
	INTR_HANDLER(172)
	INTR_HANDLER(173)
	INTR_HANDLER(174)
	INTR_HANDLER(175)
	INTR_HANDLER(176)
	INTR_HANDLER(177)
	INTR_HANDLER(178)
	INTR_HANDLER(179)
	INTR_HANDLER(180)
	INTR_HANDLER(181)
	INTR_HANDLER(182)
	INTR_HANDLER(183)
	INTR_HANDLER(184)
	INTR_HANDLER(185)
	INTR_HANDLER(186)
	INTR_HANDLER(187)
	INTR_HANDLER(188)
	INTR_HANDLER(189)
	INTR_HANDLER(190)
	INTR_HANDLER(191)
MCOUNT_LABEL(eintr)

	.data

/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
	.globl stopped_cpus, started_cpus
stopped_cpus:
	.quad	0
started_cpus:
	.quad	0

	.globl CNAME(cpustop_restartfunc)
CNAME(cpustop_restartfunc):
	.quad 0
		
	.text
Commit	Line	Data
c8fe38ae	1	/*
46d4e165	2	* from: vector.s, 386BSD 0.1 unknown origin
c8fe38ae	3	* $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
c8fe38ae MD	4	*/
c8fe38ae MD	5
46d4e165	6	#if 0
c8fe38ae	7	#include "opt_auto_eoi.h"
46d4e165	8	#endif
c8fe38ae MD	9
	10	#include <machine/asmacros.h>
	11	#include <machine/lock.h>
	12	#include <machine/psl.h>
	13	#include <machine/trap.h>
bfc09ba0	14	#include <machine/segments.h>
c8fe38ae MD	15
c8fe38ae MD	16	#include <machine_base/icu/icu.h>
0855a2af	17	#include <bus/isa/isa.h>
c8fe38ae MD	18
	19	#include "assym.s"
	20
	21	#include "apicreg.h"
77f86d14	22	#include <machine_base/apic/ioapic_ipl.h>
57a9c56b	23	#include <machine/intr_machdep.h>
c8fe38ae	24
9611ff20 SZ	25	#ifdef foo
	26	/* convert an absolute IRQ# into bitmask */
	27	#define IRQ_LBIT(irq_num) (1UL << (irq_num & 0x3f))
	28	#endif
	29
	30	#define IRQ_SBITS(irq_num) ((irq_num) & 0x3f)
c8fe38ae	31
9611ff20 SZ	32	/* convert an absolute IRQ# into gd_ipending index */
9611ff20 SZ	33	#define IRQ_LIDX(irq_num) ((irq_num) >> 6)
c8fe38ae MD	34
	35	#ifdef SMP
	36	#define MPLOCKED lock ;
	37	#else
	38	#define MPLOCKED
	39	#endif
	40
46d4e165 JG	41	#define APIC_PUSH_FRAME \
	42	PUSH_FRAME ; /* 15 regs + space for 5 extras */ \
	43	movq $0,TF_XFLAGS(%rsp) ; \
	44	movq $0,TF_TRAPNO(%rsp) ; \
	45	movq $0,TF_ADDR(%rsp) ; \
	46	movq $0,TF_FLAGS(%rsp) ; \
	47	movq $0,TF_ERR(%rsp) ; \
c8fe38ae	48	cld ; \
c8fe38ae MD	49
c8fe38ae MD	50	/*
46d4e165	51	* JG stale? Warning: POP_FRAME can only be used if there is no chance of a
c8fe38ae MD	52	* segment register being changed (e.g. by procfs), which is why syscalls
	53	* have to use doreti.
	54	*/
3338cc67 MD	55	#define APIC_POP_FRAME \
3338cc67 MD	56	POP_FRAME ; \
c8fe38ae	57
29b21b81	58	#define IOAPICADDR(irq_num) \
5ac5ccd2	59	CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_ADDR
29b21b81	60	#define REDIRIDX(irq_num) \
5ac5ccd2	61	CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_IDX
54d63b2f	62	#define IOAPICFLAGS(irq_num) \
5ac5ccd2	63	CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_FLAGS
54d63b2f	64
c8fe38ae	65	#define MASK_IRQ(irq_num) \
9e0e3f85	66	IOAPIC_IMASK_LOCK ; /* into critical reg */ \
5ac5ccd2	67	testl $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
c8fe38ae	68	jne 7f ; /* masked, don't mask */ \
5ac5ccd2	69	orl $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
906448d8	70	/* set the mask bit */ \
46d4e165	71	movq IOAPICADDR(irq_num), %rcx ; /* ioapic addr */ \
c8fe38ae	72	movl REDIRIDX(irq_num), %eax ; /* get the index */ \
46d4e165	73	movl %eax, (%rcx) ; /* write the index */ \
6dfe7381	74	orl $IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* set the mask */ \
c8fe38ae	75	7: ; /* already masked */ \
9e0e3f85	76	IOAPIC_IMASK_UNLOCK ; \
c8fe38ae MD	77
	78	/*
	79	* Test to see whether we are handling an edge or level triggered INT.
	80	* Level-triggered INTs must still be masked as we don't clear the source,
	81	* and the EOI cycle would cause redundant INTs to occur.
	82	*/
	83	#define MASK_LEVEL_IRQ(irq_num) \
5ac5ccd2	84	testl $IOAPIC_IRQI_FLAG_LEVEL, IOAPICFLAGS(irq_num) ; \
c8fe38ae MD	85	jz 9f ; /* edge, don't mask */ \
	86	MASK_IRQ(irq_num) ; \
	87	9: ; \
	88
	89	/*
	90	* Test to see if the source is currntly masked, clear if so.
	91	*/
	92	#define UNMASK_IRQ(irq_num) \
	93	cmpl $0,%eax ; \
	94	jnz 8f ; \
9e0e3f85	95	IOAPIC_IMASK_LOCK ; /* into critical reg */ \
5ac5ccd2	96	testl $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
c8fe38ae	97	je 7f ; /* bit clear, not masked */ \
5ac5ccd2	98	andl $~IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
906448d8	99	/* clear mask bit */ \
46d4e165	100	movq IOAPICADDR(irq_num),%rcx ; /* ioapic addr */ \
c8fe38ae	101	movl REDIRIDX(irq_num), %eax ; /* get the index */ \
46d4e165	102	movl %eax,(%rcx) ; /* write the index */ \
6dfe7381	103	andl $~IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* clear the mask */ \
c8fe38ae	104	7: ; \
9e0e3f85	105	IOAPIC_IMASK_UNLOCK ; \
c8fe38ae MD	106	8: ; \
c8fe38ae MD	107
c8fe38ae	108	/*
35e45e47	109	* Interrupt call handlers run in the following sequence:
c8fe38ae MD	110	*
	111	* - Push the trap frame required by doreti
	112	* - Mask the interrupt and reenable its source
9611ff20	113	* - If we cannot take the interrupt set its ipending bit and
b5d16701	114	* doreti.
9611ff20	115	* - If we can take the interrupt clear its ipending bit,
c8fe38ae MD	116	* call the handler, then unmask and doreti.
	117	*
	118	* YYY can cache gd base opitner instead of using hidden %fs prefixes.
	119	*/
	120
58e8d3d8	121	#define INTR_HANDLER(irq_num) \
c8fe38ae MD	122	.text ; \
c8fe38ae MD	123	SUPERALIGN_TEXT ; \
9e0e3f85	124	IDTVEC(ioapic_intr##irq_num) ; \
46d4e165	125	APIC_PUSH_FRAME ; \
2883d2d8	126	FAKE_MCOUNT(TF_RIP(%rsp)) ; \
c8fe38ae	127	MASK_LEVEL_IRQ(irq_num) ; \
46d4e165 JG	128	movq lapic, %rax ; \
	129	movl $0, LA_EOI(%rax) ; \
	130	movq PCPU(curthread),%rbx ; \
	131	testl $-1,TD_NEST_COUNT(%rbx) ; \
c8fe38ae	132	jne 1f ; \
f9235b6d MD	133	testl $-1,TD_CRITCOUNT(%rbx) ; \
f9235b6d MD	134	je 2f ; \
c8fe38ae MD	135	1: ; \
	136	/* in critical section, make interrupt pending */ \
	137	/* set the pending bit and return, leave interrupt masked */ \
9611ff20 SZ	138	movq $1,%rcx ; \
9611ff20 SZ	139	shlq $IRQ_SBITS(irq_num),%rcx ; \
4fb281af SZ	140	movq $IRQ_LIDX(irq_num),%rdx ; \
4fb281af SZ	141	orq %rcx,PCPU_E8(ipending,%rdx) ; \
c8fe38ae MD	142	orl $RQF_INTPEND,PCPU(reqflags) ; \
	143	jmp 5f ; \
	144	2: ; \
	145	/* clear pending bit, run handler */ \
9611ff20 SZ	146	movq $1,%rcx ; \
	147	shlq $IRQ_SBITS(irq_num),%rcx ; \
	148	notq %rcx ; \
4fb281af SZ	149	movq $IRQ_LIDX(irq_num),%rdx ; \
4fb281af SZ	150	andq %rcx,PCPU_E8(ipending,%rdx) ; \
46d4e165 JG	151	pushq $irq_num ; /* trapframe -> intrframe */ \
46d4e165 JG	152	movq %rsp, %rdi ; /* pass frame by reference */ \
f9235b6d	153	incl TD_CRITCOUNT(%rbx) ; \
2d910aaf	154	sti ; \
46d4e165	155	call ithread_fast_handler ; /* returns 0 to unmask */ \
f9235b6d	156	decl TD_CRITCOUNT(%rbx) ; \
46d4e165	157	addq $8, %rsp ; /* intrframe -> trapframe */ \
c8fe38ae MD	158	UNMASK_IRQ(irq_num) ; \
	159	5: ; \
	160	MEXITCOUNT ; \
	161	jmp doreti ; \
	162
c8fe38ae MD	163	/*
c8fe38ae MD	164	* Handle "spurious INTerrupts".
3338cc67 MD	165	*
	166	* NOTE: This is different than the "spurious INTerrupt" generated by an
	167	* 8259 PIC for missing INTs. See the APIC documentation for details.
	168	* This routine should NOT do an 'EOI' cycle.
	169	*
	170	* NOTE: Even though we don't do anything here we must still swapgs if
	171	* coming from a user frame in case the iretq faults... just use
	172	* the nominal APIC_PUSH_FRAME sequence to get it done.
c8fe38ae MD	173	*/
	174	.text
	175	SUPERALIGN_TEXT
	176	.globl Xspuriousint
	177	Xspuriousint:
3338cc67	178	APIC_PUSH_FRAME
c8fe38ae	179	/* No EOI cycle used here */
3338cc67 MD	180	FAKE_MCOUNT(TF_RIP(%rsp))
	181	MEXITCOUNT
	182	APIC_POP_FRAME
7d4d6fdb	183	jmp doreti_iret
c8fe38ae	184
9a4bd8f3	185	#ifdef SMP
c8fe38ae MD	186
	187	/*
	188	* Handle TLB shootdowns.
2d910aaf MD	189	*
2d910aaf MD	190	* NOTE: interrupts are left disabled.
c8fe38ae MD	191	*/
	192	.text
	193	SUPERALIGN_TEXT
	194	.globl Xinvltlb
	195	Xinvltlb:
7d4d6fdb	196	APIC_PUSH_FRAME
46d4e165 JG	197	movq lapic, %rax
46d4e165 JG	198	movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
7d4d6fdb MD	199	FAKE_MCOUNT(TF_RIP(%rsp))
	200	subq $8,%rsp /* make same as interrupt frame */
	201	movq %rsp,%rdi /* pass frame by reference */
	202	call smp_invltlb_intr
	203	addq $8,%rsp /* turn into trapframe */
	204	MEXITCOUNT
	205	APIC_POP_FRAME
	206	jmp doreti_iret
c8fe38ae MD	207
	208	/*
	209	* Executed by a CPU when it receives an Xcpustop IPI from another CPU,
	210	*
2883d2d8	211	* - We cannot call doreti
c8fe38ae MD	212	* - Signals its receipt.
	213	* - Waits for permission to restart.
	214	* - Processing pending IPIQ events while waiting.
	215	* - Signals its restart.
	216	*/
	217
	218	.text
	219	SUPERALIGN_TEXT
	220	.globl Xcpustop
	221	Xcpustop:
2883d2d8	222	APIC_PUSH_FRAME
46d4e165 JG	223	movq lapic, %rax
46d4e165 JG	224	movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
c8fe38ae MD	225
	226	movl PCPU(cpuid), %eax
	227	imull $PCB_SIZE, %eax
46d4e165 JG	228	leaq CNAME(stoppcbs), %rdi
46d4e165 JG	229	addq %rax, %rdi
c8fe38ae	230	call CNAME(savectx) /* Save process context */
2883d2d8	231
da23a592	232	movslq PCPU(cpuid), %rax
c8fe38ae MD	233
	234	/*
	235	* Indicate that we have stopped and loop waiting for permission
	236	* to start again. We must still process IPI events while in a
	237	* stopped state.
2d910aaf MD	238	*
	239	* Interrupts must remain enabled for non-IPI'd per-cpu interrupts
	240	* (e.g. Xtimer, Xinvltlb).
c8fe38ae MD	241	*/
c8fe38ae MD	242	MPLOCKED
da23a592	243	btsq %rax, stopped_cpus /* stopped_cpus \|= (1<<id) */
2d910aaf	244	sti
c8fe38ae MD	245	1:
c8fe38ae MD	246	andl $~RQF_IPIQ,PCPU(reqflags)
46d4e165	247	pushq %rax
c8fe38ae	248	call lwkt_smp_stopped
46d4e165	249	popq %rax
2883d2d8	250	pause
da23a592	251	btq %rax, started_cpus /* while (!(started_cpus & (1<<id))) */
c8fe38ae MD	252	jnc 1b
	253
	254	MPLOCKED
da23a592	255	btrq %rax, started_cpus /* started_cpus &= ~(1<<id) */
c8fe38ae	256	MPLOCKED
da23a592	257	btrq %rax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
c8fe38ae MD	258
	259	test %eax, %eax
	260	jnz 2f
	261
46d4e165 JG	262	movq CNAME(cpustop_restartfunc), %rax
46d4e165 JG	263	test %rax, %rax
c8fe38ae	264	jz 2f
46d4e165	265	movq $0, CNAME(cpustop_restartfunc) /* One-shot */
c8fe38ae	266
46d4e165	267	call *%rax
c8fe38ae	268	2:
2883d2d8 MD	269	MEXITCOUNT
2883d2d8 MD	270	APIC_POP_FRAME
7d4d6fdb	271	jmp doreti_iret
c8fe38ae MD	272
	273	/*
	274	* For now just have one ipiq IPI, but what we really want is
	275	* to have one for each source cpu to the APICs don't get stalled
	276	* backlogging the requests.
	277	*/
	278	.text
	279	SUPERALIGN_TEXT
	280	.globl Xipiq
	281	Xipiq:
46d4e165 JG	282	APIC_PUSH_FRAME
	283	movq lapic, %rax
	284	movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
2883d2d8	285	FAKE_MCOUNT(TF_RIP(%rsp))
c8fe38ae	286
46d4e165 JG	287	incl PCPU(cnt) + V_IPI
46d4e165 JG	288	movq PCPU(curthread),%rbx
f9235b6d MD	289	testl $-1,TD_CRITCOUNT(%rbx)
f9235b6d MD	290	jne 1f
46d4e165 JG	291	subq $8,%rsp /* make same as interrupt frame */
46d4e165 JG	292	movq %rsp,%rdi /* pass frame by reference */
c8fe38ae	293	incl PCPU(intr_nesting_level)
f9235b6d	294	incl TD_CRITCOUNT(%rbx)
2d910aaf	295	sti
c8fe38ae	296	call lwkt_process_ipiq_frame
f9235b6d	297	decl TD_CRITCOUNT(%rbx)
c8fe38ae	298	decl PCPU(intr_nesting_level)
46d4e165	299	addq $8,%rsp /* turn into trapframe */
c8fe38ae MD	300	MEXITCOUNT
	301	jmp doreti
	302	1:
	303	orl $RQF_IPIQ,PCPU(reqflags)
	304	MEXITCOUNT
46d4e165	305	APIC_POP_FRAME
7d4d6fdb	306	jmp doreti_iret
46d4e165	307
9a4bd8f3 SZ	308	#endif /* SMP */
9a4bd8f3 SZ	309
46d4e165 JG	310	.text
	311	SUPERALIGN_TEXT
	312	.globl Xtimer
	313	Xtimer:
	314	APIC_PUSH_FRAME
	315	movq lapic, %rax
	316	movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
2883d2d8	317	FAKE_MCOUNT(TF_RIP(%rsp))
46d4e165	318
b9f7ba13 MD	319	subq $8,%rsp /* make same as interrupt frame */
	320	movq %rsp,%rdi /* pass frame by reference */
	321	call lapic_timer_always
	322	addq $8,%rsp /* turn into trapframe */
	323
46d4e165 JG	324	incl PCPU(cnt) + V_TIMER
46d4e165 JG	325	movq PCPU(curthread),%rbx
f9235b6d MD	326	testl $-1,TD_CRITCOUNT(%rbx)
f9235b6d MD	327	jne 1f
46d4e165 JG	328	testl $-1,TD_NEST_COUNT(%rbx)
	329	jne 1f
	330	subq $8,%rsp /* make same as interrupt frame */
	331	movq %rsp,%rdi /* pass frame by reference */
	332	incl PCPU(intr_nesting_level)
f9235b6d	333	incl TD_CRITCOUNT(%rbx)
2d910aaf	334	sti
46d4e165	335	call lapic_timer_process_frame
f9235b6d	336	decl TD_CRITCOUNT(%rbx)
46d4e165 JG	337	decl PCPU(intr_nesting_level)
	338	addq $8,%rsp /* turn into trapframe */
	339	MEXITCOUNT
	340	jmp doreti
	341	1:
	342	orl $RQF_TIMER,PCPU(reqflags)
	343	MEXITCOUNT
	344	APIC_POP_FRAME
7d4d6fdb	345	jmp doreti_iret
c8fe38ae	346
c8fe38ae	347	MCOUNT_LABEL(bintr)
58e8d3d8 SZ	348	INTR_HANDLER(0)
	349	INTR_HANDLER(1)
	350	INTR_HANDLER(2)
	351	INTR_HANDLER(3)
	352	INTR_HANDLER(4)
	353	INTR_HANDLER(5)
	354	INTR_HANDLER(6)
	355	INTR_HANDLER(7)
	356	INTR_HANDLER(8)
	357	INTR_HANDLER(9)
	358	INTR_HANDLER(10)
	359	INTR_HANDLER(11)
	360	INTR_HANDLER(12)
	361	INTR_HANDLER(13)
	362	INTR_HANDLER(14)
	363	INTR_HANDLER(15)
	364	INTR_HANDLER(16)
	365	INTR_HANDLER(17)
	366	INTR_HANDLER(18)
	367	INTR_HANDLER(19)
	368	INTR_HANDLER(20)
	369	INTR_HANDLER(21)
	370	INTR_HANDLER(22)
	371	INTR_HANDLER(23)
c571da4a SZ	372	INTR_HANDLER(24)
	373	INTR_HANDLER(25)
	374	INTR_HANDLER(26)
	375	INTR_HANDLER(27)
	376	INTR_HANDLER(28)
	377	INTR_HANDLER(29)
	378	INTR_HANDLER(30)
	379	INTR_HANDLER(31)
	380	INTR_HANDLER(32)
	381	INTR_HANDLER(33)
	382	INTR_HANDLER(34)
	383	INTR_HANDLER(35)
	384	INTR_HANDLER(36)
	385	INTR_HANDLER(37)
	386	INTR_HANDLER(38)
	387	INTR_HANDLER(39)
	388	INTR_HANDLER(40)
	389	INTR_HANDLER(41)
	390	INTR_HANDLER(42)
	391	INTR_HANDLER(43)
	392	INTR_HANDLER(44)
	393	INTR_HANDLER(45)
	394	INTR_HANDLER(46)
	395	INTR_HANDLER(47)
	396	INTR_HANDLER(48)
	397	INTR_HANDLER(49)
	398	INTR_HANDLER(50)
	399	INTR_HANDLER(51)
	400	INTR_HANDLER(52)
	401	INTR_HANDLER(53)
	402	INTR_HANDLER(54)
	403	INTR_HANDLER(55)
	404	INTR_HANDLER(56)
	405	INTR_HANDLER(57)
	406	INTR_HANDLER(58)
	407	INTR_HANDLER(59)
	408	INTR_HANDLER(60)
	409	INTR_HANDLER(61)
	410	INTR_HANDLER(62)
	411	INTR_HANDLER(63)
	412	INTR_HANDLER(64)
	413	INTR_HANDLER(65)
	414	INTR_HANDLER(66)
	415	INTR_HANDLER(67)
	416	INTR_HANDLER(68)
	417	INTR_HANDLER(69)
	418	INTR_HANDLER(70)
	419	INTR_HANDLER(71)
	420	INTR_HANDLER(72)
	421	INTR_HANDLER(73)
	422	INTR_HANDLER(74)
	423	INTR_HANDLER(75)
	424	INTR_HANDLER(76)
	425	INTR_HANDLER(77)
	426	INTR_HANDLER(78)
	427	INTR_HANDLER(79)
	428	INTR_HANDLER(80)
	429	INTR_HANDLER(81)
	430	INTR_HANDLER(82)
	431	INTR_HANDLER(83)
	432	INTR_HANDLER(84)
	433	INTR_HANDLER(85)
	434	INTR_HANDLER(86)
	435	INTR_HANDLER(87)
436	INTR_HANDLER(88)
437	INTR_HANDLER(89)
438	INTR_HANDLER(90)
439	INTR_HANDLER(91)
440	INTR_HANDLER(92)
441	INTR_HANDLER(93)
442	INTR_HANDLER(94)
443	INTR_HANDLER(95)
444	INTR_HANDLER(96)
445	INTR_HANDLER(97)
446	INTR_HANDLER(98)
447	INTR_HANDLER(99)
448	INTR_HANDLER(100)
449	INTR_HANDLER(101)
450	INTR_HANDLER(102)
451	INTR_HANDLER(103)
452	INTR_HANDLER(104)
453	INTR_HANDLER(105)
454	INTR_HANDLER(106)
455	INTR_HANDLER(107)
456	INTR_HANDLER(108)
457	INTR_HANDLER(109)
458	INTR_HANDLER(110)
459	INTR_HANDLER(111)
460	INTR_HANDLER(112)
461	INTR_HANDLER(113)
462	INTR_HANDLER(114)
463	INTR_HANDLER(115)
464	INTR_HANDLER(116)
465	INTR_HANDLER(117)
466	INTR_HANDLER(118)
467	INTR_HANDLER(119)
468	INTR_HANDLER(120)
469	INTR_HANDLER(121)
470	INTR_HANDLER(122)
471	INTR_HANDLER(123)
472	INTR_HANDLER(124)
473	INTR_HANDLER(125)
474	INTR_HANDLER(126)
475	INTR_HANDLER(127)
476	INTR_HANDLER(128)
477	INTR_HANDLER(129)
478	INTR_HANDLER(130)
479	INTR_HANDLER(131)
480	INTR_HANDLER(132)
481	INTR_HANDLER(133)
482	INTR_HANDLER(134)
483	INTR_HANDLER(135)
484	INTR_HANDLER(136)
485	INTR_HANDLER(137)
486	INTR_HANDLER(138)
487	INTR_HANDLER(139)
488	INTR_HANDLER(140)
489	INTR_HANDLER(141)
490	INTR_HANDLER(142)
491	INTR_HANDLER(143)
492	INTR_HANDLER(144)
493	INTR_HANDLER(145)
494	INTR_HANDLER(146)
495	INTR_HANDLER(147)
496	INTR_HANDLER(148)
497	INTR_HANDLER(149)
498	INTR_HANDLER(150)
499	INTR_HANDLER(151)
500	INTR_HANDLER(152)
501	INTR_HANDLER(153)
502	INTR_HANDLER(154)
503	INTR_HANDLER(155)
504	INTR_HANDLER(156)
505	INTR_HANDLER(157)
506	INTR_HANDLER(158)
507	INTR_HANDLER(159)
508	INTR_HANDLER(160)
509	INTR_HANDLER(161)
510	INTR_HANDLER(162)
511	INTR_HANDLER(163)
512	INTR_HANDLER(164)
513	INTR_HANDLER(165)
514	INTR_HANDLER(166)
515	INTR_HANDLER(167)
516	INTR_HANDLER(168)
517	INTR_HANDLER(169)
518	INTR_HANDLER(170)
519	INTR_HANDLER(171)
520	INTR_HANDLER(172)
521	INTR_HANDLER(173)
522	INTR_HANDLER(174)
523	INTR_HANDLER(175)
524	INTR_HANDLER(176)
525	INTR_HANDLER(177)
526	INTR_HANDLER(178)
527	INTR_HANDLER(179)
528	INTR_HANDLER(180)
529	INTR_HANDLER(181)
530	INTR_HANDLER(182)
531	INTR_HANDLER(183)
532	INTR_HANDLER(184)
533	INTR_HANDLER(185)
534	INTR_HANDLER(186)
535	INTR_HANDLER(187)
536	INTR_HANDLER(188)
537	INTR_HANDLER(189)
538	INTR_HANDLER(190)
539	INTR_HANDLER(191)
c8fe38ae MD	540	MCOUNT_LABEL(eintr)
c8fe38ae MD	541
c8fe38ae MD	542	.data
	543
	544	/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
	545	.globl stopped_cpus, started_cpus
	546	stopped_cpus:
da23a592	547	.quad 0
c8fe38ae	548	started_cpus:
da23a592	549	.quad 0
c8fe38ae MD	550
	551	.globl CNAME(cpustop_restartfunc)
	552	CNAME(cpustop_restartfunc):
46d4e165	553	.quad 0
c8fe38ae	554
c8fe38ae MD	555	.text
c8fe38ae MD	556