[dragonfly.git] / sys / platform / pc32 / apic / apic_vector.s

/*
 *	from: vector.s, 386BSD 0.1 unknown origin
 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
 * $DragonFly: src/sys/platform/pc32/apic/apic_vector.s,v 1.39 2008/08/02 01:14:43 dillon Exp $
 */

#include "use_npx.h"
#include "opt_auto_eoi.h"

#include <machine/asmacros.h>
#include <machine/lock.h>
#include <machine/psl.h>
#include <machine/trap.h>

#include <machine_base/icu/icu.h>
#include <bus/isa/isa.h>

#include "assym.s"

#include "apicreg.h"
#include "apic_ipl.h"
#include <machine/smp.h>
#include <machine_base/isa/intr_machdep.h>

/* convert an absolute IRQ# into a bitmask */
#define IRQ_LBIT(irq_num)	(1 << (irq_num))

/* make an index into the IO APIC from the IRQ# */
#define REDTBL_IDX(irq_num)	(0x10 + ((irq_num) * 2))

#ifdef SMP
#define MPLOCKED     lock ;
#else
#define MPLOCKED
#endif

/*
 * Push an interrupt frame in a format acceptable to doreti, reload
 * the segment registers for the kernel.
 */
#define PUSH_FRAME							\
	pushl	$0 ;		/* dummy error code */			\
	pushl	$0 ;		/* dummy trap type */			\
	pushl	$0 ;		/* dummy xflags type */			\
	pushal ;							\
	pushl	%ds ;		/* save data and extra segments ... */	\
	pushl	%es ;							\
	pushl	%fs ;							\
	pushl	%gs ;							\
	cld ;								\
	mov	$KDSEL,%ax ;						\
	mov	%ax,%ds ;						\
	mov	%ax,%es ;						\
	mov	%ax,%gs ;						\
	mov	$KPSEL,%ax ;						\
	mov	%ax,%fs ;						\

/*
 * Warning: POP_FRAME can only be used if there is no chance of a
 * segment register being changed (e.g. by procfs), which is why syscalls
 * have to use doreti.
 */
#define POP_FRAME							\
	popl	%gs ;							\
	popl	%fs ;							\
	popl	%es ;							\
	popl	%ds ;							\
	popal ;								\
	addl	$3*4,%esp ;	/* dummy xflags, trap & error codes */	\

#define IOAPICADDR(irq_num) \
	CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR
#define REDIRIDX(irq_num) \
	CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX
#define IOAPICFLAGS(irq_num) \
	CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS

#define MASK_IRQ(irq_num)						\
	APIC_IMASK_LOCK ;			/* into critical reg */	\
	testl	$IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;		\
	jne	7f ;			/* masked, don't mask */	\
	orl	$IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;		\
						/* set the mask bit */	\
	movl	IOAPICADDR(irq_num), %ecx ;	/* ioapic addr */	\
	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
	movl	%eax, (%ecx) ;			/* write the index */	\
	orl	$IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* set the mask */	\
7: ;						/* already masked */	\
	APIC_IMASK_UNLOCK ;						\

/*
 * Test to see whether we are handling an edge or level triggered INT.
 *  Level-triggered INTs must still be masked as we don't clear the source,
 *  and the EOI cycle would cause redundant INTs to occur.
 */
#define MASK_LEVEL_IRQ(irq_num)						\
	testl	$IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ;		\
	jz	9f ;				/* edge, don't mask */	\
	MASK_IRQ(irq_num) ;						\
9: ;									\

/*
 * Test to see if the source is currntly masked, clear if so.
 */
#define UNMASK_IRQ(irq_num)						\
	cmpl	$0,%eax ;						\
	jnz	8f ;							\
	APIC_IMASK_LOCK ;			/* into critical reg */	\
	testl	$IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;		\
	je	7f ;			/* bit clear, not masked */	\
	andl	$~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;		\
						/* clear mask bit */	\
	movl	IOAPICADDR(irq_num),%ecx ;	/* ioapic addr */	\
	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
	movl	%eax,(%ecx) ;			/* write the index */	\
	andl	$~IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* clear the mask */ \
7: ;									\
	APIC_IMASK_UNLOCK ;						\
8: ;									\

#ifdef SMP /* APIC-IO */

/*
 * Fast interrupt call handlers run in the following sequence:
 *
 *	- Push the trap frame required by doreti
 *	- Mask the interrupt and reenable its source
 *	- If we cannot take the interrupt set its fpending bit and
 *	  doreti.  Note that we cannot mess with mp_lock at all
 *	  if we entered from a critical section!
 *	- If we can take the interrupt clear its fpending bit,
 *	  call the handler, then unmask and doreti.
 *
 * YYY can cache gd base opitner instead of using hidden %fs prefixes.
 */

#define	FAST_INTR(irq_num, vec_name)					\
	.text ;								\
	SUPERALIGN_TEXT ;						\
IDTVEC(vec_name) ;							\
	PUSH_FRAME ;							\
	FAKE_MCOUNT(15*4(%esp)) ;					\
	MASK_LEVEL_IRQ(irq_num) ;					\
	movl	$0, lapic_eoi ;						\
	movl	PCPU(curthread),%ebx ;					\
	movl	$0,%eax ;	/* CURRENT CPL IN FRAME (REMOVED) */	\
	pushl	%eax ;							\
	testl	$-1,TD_NEST_COUNT(%ebx) ;				\
	jne	1f ;							\
	testl	$-1,TD_CRITCOUNT(%ebx) ;				\
	je	2f ;							\
1: ;									\
	/* in critical section, make interrupt pending */		\
	/* set the pending bit and return, leave interrupt masked */	\
	orl	$IRQ_LBIT(irq_num),PCPU(fpending) ;			\
	orl	$RQF_INTPEND,PCPU(reqflags) ;				\
	jmp	5f ;							\
2: ;									\
	/* clear pending bit, run handler */				\
	andl	$~IRQ_LBIT(irq_num),PCPU(fpending) ;			\
	pushl	$irq_num ;						\
	pushl	%esp ;			 /* pass frame by reference */	\
	incl	TD_CRITCOUNT(%ebx) ;					\
	call	ithread_fast_handler ;	 /* returns 0 to unmask */	\
	decl	TD_CRITCOUNT(%ebx) ;					\
	addl	$8, %esp ;						\
	UNMASK_IRQ(irq_num) ;						\
5: ;									\
	MEXITCOUNT ;							\
	jmp	doreti ;						\

#endif

/*
 * Handle "spurious INTerrupts".
 * Notes:
 *  This is different than the "spurious INTerrupt" generated by an
 *   8259 PIC for missing INTs.  See the APIC documentation for details.
 *  This routine should NOT do an 'EOI' cycle.
 */
	.text
	SUPERALIGN_TEXT
	.globl Xspuriousint
Xspuriousint:

	/* No EOI cycle used here */

	iret


/*
 * Handle TLB shootdowns.
 */
	.text
	SUPERALIGN_TEXT
	.globl	Xinvltlb
Xinvltlb:
	pushl	%eax

	movl	%cr3, %eax		/* invalidate the TLB */
	movl	%eax, %cr3

	ss				/* stack segment, avoid %ds load */
	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */

	popl	%eax
	iret


/*
 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
 *
 *  - Signals its receipt.
 *  - Waits for permission to restart.
 *  - Processing pending IPIQ events while waiting.
 *  - Signals its restart.
 */

	.text
	SUPERALIGN_TEXT
	.globl Xcpustop
Xcpustop:
	pushl	%ebp
	movl	%esp, %ebp
	pushl	%eax
	pushl	%ecx
	pushl	%edx
	pushl	%ds			/* save current data segment */
	pushl	%fs

	movl	$KDSEL, %eax
	mov	%ax, %ds		/* use KERNEL data segment */
	movl	$KPSEL, %eax
	mov	%ax, %fs

	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */

	movl	PCPU(cpuid), %eax
	imull	$PCB_SIZE, %eax
	leal	CNAME(stoppcbs)(%eax), %eax
	pushl	%eax
	call	CNAME(savectx)		/* Save process context */
	addl	$4, %esp
	
		
	movl	PCPU(cpuid), %eax

	/*
	 * Indicate that we have stopped and loop waiting for permission
	 * to start again.  We must still process IPI events while in a
	 * stopped state.
	 */
	MPLOCKED
	btsl	%eax, stopped_cpus	/* stopped_cpus |= (1<<id) */
1:
	andl	$~RQF_IPIQ,PCPU(reqflags)
	pushl	%eax
	call	lwkt_smp_stopped
	popl	%eax
	btl	%eax, started_cpus	/* while (!(started_cpus & (1<<id))) */
	jnc	1b

	MPLOCKED
	btrl	%eax, started_cpus	/* started_cpus &= ~(1<<id) */
	MPLOCKED
	btrl	%eax, stopped_cpus	/* stopped_cpus &= ~(1<<id) */

	test	%eax, %eax
	jnz	2f

	movl	CNAME(cpustop_restartfunc), %eax
	test	%eax, %eax
	jz	2f
	movl	$0, CNAME(cpustop_restartfunc)	/* One-shot */

	call	*%eax
2:
	popl	%fs
	popl	%ds			/* restore previous data segment */
	popl	%edx
	popl	%ecx
	popl	%eax
	movl	%ebp, %esp
	popl	%ebp
	iret

	/*
	 * For now just have one ipiq IPI, but what we really want is
	 * to have one for each source cpu to the APICs don't get stalled
	 * backlogging the requests.
	 */
	.text
	SUPERALIGN_TEXT
	.globl Xipiq
Xipiq:
	PUSH_FRAME
	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */
	FAKE_MCOUNT(15*4(%esp))

	incl    PCPU(cnt) + V_IPI
	movl	PCPU(curthread),%ebx
	testl	$-1,TD_CRITCOUNT(%ebx)
	jne	1f
	subl	$8,%esp			/* make same as interrupt frame */
	pushl	%esp			/* pass frame by reference */
	incl	PCPU(intr_nesting_level)
	incl	TD_CRITCOUNT(%ebx)
	call	lwkt_process_ipiq_frame
	decl	TD_CRITCOUNT(%ebx)
	decl	PCPU(intr_nesting_level)
	addl	$12,%esp
	pushl	$0			/* CPL for frame (REMOVED) */
	MEXITCOUNT
	jmp	doreti
1:
	orl	$RQF_IPIQ,PCPU(reqflags)
	MEXITCOUNT
	POP_FRAME
	iret

	.text
	SUPERALIGN_TEXT
	.globl Xtimer
Xtimer:
	PUSH_FRAME
	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */
	FAKE_MCOUNT(15*4(%esp))

	incl    PCPU(cnt) + V_TIMER
	movl	PCPU(curthread),%ebx
	testl	$-1,TD_CRITCOUNT(%ebx)
	jne	1f
	testl	$-1,TD_NEST_COUNT(%ebx)
	jne	1f
	subl	$8,%esp			/* make same as interrupt frame */
	pushl	%esp			/* pass frame by reference */
	incl	PCPU(intr_nesting_level)
	incl	TD_CRITCOUNT(%ebx)
	call	lapic_timer_process_frame
	decl	TD_CRITCOUNT(%ebx)
	decl	PCPU(intr_nesting_level)
	addl	$12,%esp
	pushl	$0			/* CPL for frame (REMOVED) */
	MEXITCOUNT
	jmp	doreti
1:
	orl	$RQF_TIMER,PCPU(reqflags)
	MEXITCOUNT
	POP_FRAME
	iret

#ifdef SMP /* APIC-IO */

MCOUNT_LABEL(bintr)
	FAST_INTR(0,apic_fastintr0)
	FAST_INTR(1,apic_fastintr1)
	FAST_INTR(2,apic_fastintr2)
	FAST_INTR(3,apic_fastintr3)
	FAST_INTR(4,apic_fastintr4)
	FAST_INTR(5,apic_fastintr5)
	FAST_INTR(6,apic_fastintr6)
	FAST_INTR(7,apic_fastintr7)
	FAST_INTR(8,apic_fastintr8)
	FAST_INTR(9,apic_fastintr9)
	FAST_INTR(10,apic_fastintr10)
	FAST_INTR(11,apic_fastintr11)
	FAST_INTR(12,apic_fastintr12)
	FAST_INTR(13,apic_fastintr13)
	FAST_INTR(14,apic_fastintr14)
	FAST_INTR(15,apic_fastintr15)
	FAST_INTR(16,apic_fastintr16)
	FAST_INTR(17,apic_fastintr17)
	FAST_INTR(18,apic_fastintr18)
	FAST_INTR(19,apic_fastintr19)
	FAST_INTR(20,apic_fastintr20)
	FAST_INTR(21,apic_fastintr21)
	FAST_INTR(22,apic_fastintr22)
	FAST_INTR(23,apic_fastintr23)
	FAST_INTR(24,apic_fastintr24)
	FAST_INTR(25,apic_fastintr25)
	FAST_INTR(26,apic_fastintr26)
	FAST_INTR(27,apic_fastintr27)
	FAST_INTR(28,apic_fastintr28)
	FAST_INTR(29,apic_fastintr29)
	FAST_INTR(30,apic_fastintr30)
	FAST_INTR(31,apic_fastintr31)
MCOUNT_LABEL(eintr)

#endif

	.data

/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
	.globl stopped_cpus, started_cpus
stopped_cpus:
	.long	0
started_cpus:
	.long	0

	.globl CNAME(cpustop_restartfunc)
CNAME(cpustop_restartfunc):
	.long 0
		
	.text
Commit	Line	Data
984263bc MD	1	/*
	2	* from: vector.s, 386BSD 0.1 unknown origin
	3	* $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
21ce0dfa	4	* $DragonFly: src/sys/platform/pc32/apic/apic_vector.s,v 1.39 2008/08/02 01:14:43 dillon Exp $
984263bc MD	5	*/
984263bc MD	6
06f5be02 MD	7	#include "use_npx.h"
	8	#include "opt_auto_eoi.h"
	9
	10	#include <machine/asmacros.h>
06f5be02 MD	11	#include <machine/lock.h>
	12	#include <machine/psl.h>
	13	#include <machine/trap.h>
06f5be02	14
a9295349	15	#include <machine_base/icu/icu.h>
21ce0dfa	16	#include <bus/isa/isa.h>
06f5be02 MD	17
06f5be02 MD	18	#include "assym.s"
984263bc	19
bdc560a1 MD	20	#include "apicreg.h"
bdc560a1 MD	21	#include "apic_ipl.h"
984263bc	22	#include <machine/smp.h>
a9295349	23	#include <machine_base/isa/intr_machdep.h>
984263bc MD	24
984263bc MD	25	/* convert an absolute IRQ# into a bitmask */
8a8d5d85	26	#define IRQ_LBIT(irq_num) (1 << (irq_num))
984263bc MD	27
	28	/* make an index into the IO APIC from the IRQ# */
	29	#define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2))
	30
97359a5b MD	31	#ifdef SMP
	32	#define MPLOCKED lock ;
	33	#else
	34	#define MPLOCKED
	35	#endif
	36
984263bc	37	/*
8a8d5d85 MD	38	* Push an interrupt frame in a format acceptable to doreti, reload
8a8d5d85 MD	39	* the segment registers for the kernel.
984263bc MD	40	*/
	41	#define PUSH_FRAME \
	42	pushl $0 ; /* dummy error code */ \
	43	pushl $0 ; /* dummy trap type */ \
4e7c41c5	44	pushl $0 ; /* dummy xflags type */ \
984263bc MD	45	pushal ; \
	46	pushl %ds ; /* save data and extra segments ... */ \
	47	pushl %es ; \
8a8d5d85	48	pushl %fs ; \
4e7c41c5	49	pushl %gs ; \
c885c20e	50	cld ; \
8a8d5d85 MD	51	mov $KDSEL,%ax ; \
	52	mov %ax,%ds ; \
	53	mov %ax,%es ; \
4e7c41c5	54	mov %ax,%gs ; \
8a8d5d85 MD	55	mov $KPSEL,%ax ; \
8a8d5d85 MD	56	mov %ax,%fs ; \
984263bc	57
8a8d5d85 MD	58	/*
	59	* Warning: POP_FRAME can only be used if there is no chance of a
	60	* segment register being changed (e.g. by procfs), which is why syscalls
	61	* have to use doreti.
	62	*/
984263bc	63	#define POP_FRAME \
4e7c41c5	64	popl %gs ; \
984263bc MD	65	popl %fs ; \
	66	popl %es ; \
	67	popl %ds ; \
	68	popal ; \
4e7c41c5	69	addl $34,%esp ; / dummy xflags, trap & error codes */ \
8a8d5d85	70
3d911e0a SZ	71	#define IOAPICADDR(irq_num) \
	72	CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR
	73	#define REDIRIDX(irq_num) \
	74	CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX
0f546930 SZ	75	#define IOAPICFLAGS(irq_num) \
0f546930 SZ	76	CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS
8a8d5d85	77
984263bc	78	#define MASK_IRQ(irq_num) \
97359a5b	79	APIC_IMASK_LOCK ; /* into critical reg */ \
ea689d1c	80	testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
984263bc	81	jne 7f ; /* masked, don't mask */ \
ea689d1c SZ	82	orl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
ea689d1c SZ	83	/* set the mask bit */ \
984263bc MD	84	movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \
	85	movl REDIRIDX(irq_num), %eax ; /* get the index */ \
	86	movl %eax, (%ecx) ; /* write the index */ \
bda48b43	87	orl $IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* set the mask */ \
984263bc	88	7: ; /* already masked */ \
97359a5b	89	APIC_IMASK_UNLOCK ; \
8a8d5d85	90
984263bc MD	91	/*
	92	* Test to see whether we are handling an edge or level triggered INT.
	93	* Level-triggered INTs must still be masked as we don't clear the source,
	94	* and the EOI cycle would cause redundant INTs to occur.
	95	*/
	96	#define MASK_LEVEL_IRQ(irq_num) \
0f546930	97	testl $IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ; \
984263bc MD	98	jz 9f ; /* edge, don't mask */ \
984263bc MD	99	MASK_IRQ(irq_num) ; \
8a8d5d85	100	9: ; \
984263bc	101
984263bc MD	102	/*
	103	* Test to see if the source is currntly masked, clear if so.
	104	*/
ea689d1c	105	#define UNMASK_IRQ(irq_num) \
477d3c1c MD	106	cmpl $0,%eax ; \
477d3c1c MD	107	jnz 8f ; \
97359a5b	108	APIC_IMASK_LOCK ; /* into critical reg */ \
ea689d1c	109	testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
984263bc	110	je 7f ; /* bit clear, not masked */ \
ea689d1c SZ	111	andl $~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
ea689d1c SZ	112	/* clear mask bit */ \
984263bc MD	113	movl IOAPICADDR(irq_num),%ecx ; /* ioapic addr */ \
	114	movl REDIRIDX(irq_num), %eax ; /* get the index */ \
	115	movl %eax,(%ecx) ; /* write the index */ \
bda48b43	116	andl $~IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* clear the mask */ \
984263bc	117	7: ; \
97359a5b	118	APIC_IMASK_UNLOCK ; \
477d3c1c	119	8: ; \
984263bc	120
79b62055	121	#ifdef SMP /* APIC-IO */
97359a5b	122
8a8d5d85 MD	123	/*
	124	* Fast interrupt call handlers run in the following sequence:
	125	*
	126	* - Push the trap frame required by doreti
	127	* - Mask the interrupt and reenable its source
	128	* - If we cannot take the interrupt set its fpending bit and
71ef2f5c MD	129	* doreti. Note that we cannot mess with mp_lock at all
71ef2f5c MD	130	* if we entered from a critical section!
8a8d5d85 MD	131	* - If we can take the interrupt clear its fpending bit,
	132	* call the handler, then unmask and doreti.
	133	*
	134	* YYY can cache gd base opitner instead of using hidden %fs prefixes.
	135	*/
	136
	137	#define FAST_INTR(irq_num, vec_name) \
	138	.text ; \
	139	SUPERALIGN_TEXT ; \
	140	IDTVEC(vec_name) ; \
	141	PUSH_FRAME ; \
4e7c41c5	142	FAKE_MCOUNT(15*4(%esp)) ; \
8a8d5d85	143	MASK_LEVEL_IRQ(irq_num) ; \
35408d22	144	movl $0, lapic_eoi ; \
8a8d5d85	145	movl PCPU(curthread),%ebx ; \
38787eef	146	movl $0,%eax ; /* CURRENT CPL IN FRAME (REMOVED) */ \
984263bc	147	pushl %eax ; \
1be5027b MD	148	testl $-1,TD_NEST_COUNT(%ebx) ; \
1be5027b MD	149	jne 1f ; \
f9235b6d MD	150	testl $-1,TD_CRITCOUNT(%ebx) ; \
f9235b6d MD	151	je 2f ; \
8a8d5d85	152	1: ; \
545a1cd3	153	/* in critical section, make interrupt pending */ \
8a8d5d85 MD	154	/* set the pending bit and return, leave interrupt masked */ \
8a8d5d85 MD	155	orl $IRQ_LBIT(irq_num),PCPU(fpending) ; \
235957ed	156	orl $RQF_INTPEND,PCPU(reqflags) ; \
8a8d5d85 MD	157	jmp 5f ; \
	158	2: ; \
	159	/* clear pending bit, run handler */ \
8a8d5d85	160	andl $~IRQ_LBIT(irq_num),PCPU(fpending) ; \
477d3c1c	161	pushl $irq_num ; \
c7eb0589	162	pushl %esp ; /* pass frame by reference */ \
f9235b6d	163	incl TD_CRITCOUNT(%ebx) ; \
477d3c1c	164	call ithread_fast_handler ; /* returns 0 to unmask */ \
f9235b6d	165	decl TD_CRITCOUNT(%ebx) ; \
c7eb0589	166	addl $8, %esp ; \
8a8d5d85 MD	167	UNMASK_IRQ(irq_num) ; \
	168	5: ; \
	169	MEXITCOUNT ; \
	170	jmp doreti ; \
984263bc	171
97359a5b MD	172	#endif
97359a5b MD	173
984263bc MD	174	/*
	175	* Handle "spurious INTerrupts".
	176	* Notes:
	177	* This is different than the "spurious INTerrupt" generated by an
	178	* 8259 PIC for missing INTs. See the APIC documentation for details.
	179	* This routine should NOT do an 'EOI' cycle.
	180	*/
	181	.text
	182	SUPERALIGN_TEXT
2954c92f MD	183	.globl Xspuriousint
2954c92f MD	184	Xspuriousint:
984263bc MD	185
	186	/* No EOI cycle used here */
	187
	188	iret
	189
	190
	191	/*
	192	* Handle TLB shootdowns.
	193	*/
	194	.text
	195	SUPERALIGN_TEXT
8a8d5d85 MD	196	.globl Xinvltlb
8a8d5d85 MD	197	Xinvltlb:
984263bc MD	198	pushl %eax
984263bc MD	199
984263bc MD	200	movl %cr3, %eax /* invalidate the TLB */
	201	movl %eax, %cr3
	202
	203	ss /* stack segment, avoid %ds load */
	204	movl $0, lapic_eoi /* End Of Interrupt to APIC */
	205
	206	popl %eax
	207	iret
	208
	209
984263bc MD	210	/*
	211	* Executed by a CPU when it receives an Xcpustop IPI from another CPU,
	212	*
	213	* - Signals its receipt.
	214	* - Waits for permission to restart.
bd8015ca	215	* - Processing pending IPIQ events while waiting.
984263bc MD	216	* - Signals its restart.
	217	*/
	218
	219	.text
	220	SUPERALIGN_TEXT
2954c92f MD	221	.globl Xcpustop
2954c92f MD	222	Xcpustop:
984263bc MD	223	pushl %ebp
	224	movl %esp, %ebp
	225	pushl %eax
	226	pushl %ecx
	227	pushl %edx
	228	pushl %ds /* save current data segment */
	229	pushl %fs
	230
	231	movl $KDSEL, %eax
	232	mov %ax, %ds /* use KERNEL data segment */
	233	movl $KPSEL, %eax
	234	mov %ax, %fs
	235
	236	movl $0, lapic_eoi /* End Of Interrupt to APIC */
	237
2954c92f	238	movl PCPU(cpuid), %eax
984263bc MD	239	imull $PCB_SIZE, %eax
	240	leal CNAME(stoppcbs)(%eax), %eax
	241	pushl %eax
	242	call CNAME(savectx) /* Save process context */
	243	addl $4, %esp
	244
	245
2954c92f	246	movl PCPU(cpuid), %eax
984263bc	247
bd8015ca MD	248	/*
	249	* Indicate that we have stopped and loop waiting for permission
	250	* to start again. We must still process IPI events while in a
	251	* stopped state.
	252	*/
97359a5b	253	MPLOCKED
2954c92f	254	btsl %eax, stopped_cpus /* stopped_cpus \|= (1<<id) */
984263bc	255	1:
bd8015ca MD	256	andl $~RQF_IPIQ,PCPU(reqflags)
	257	pushl %eax
	258	call lwkt_smp_stopped
	259	popl %eax
2954c92f	260	btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
984263bc MD	261	jnc 1b
984263bc MD	262
97359a5b	263	MPLOCKED
2954c92f	264	btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
97359a5b	265	MPLOCKED
2954c92f	266	btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
984263bc MD	267
	268	test %eax, %eax
	269	jnz 2f
	270
	271	movl CNAME(cpustop_restartfunc), %eax
	272	test %eax, %eax
	273	jz 2f
	274	movl $0, CNAME(cpustop_restartfunc) /* One-shot */
	275
	276	call *%eax
	277	2:
	278	popl %fs
	279	popl %ds /* restore previous data segment */
	280	popl %edx
	281	popl %ecx
	282	popl %eax
	283	movl %ebp, %esp
	284	popl %ebp
	285	iret
	286
96728c05 MD	287	/*
	288	* For now just have one ipiq IPI, but what we really want is
	289	* to have one for each source cpu to the APICs don't get stalled
	290	* backlogging the requests.
	291	*/
	292	.text
	293	SUPERALIGN_TEXT
	294	.globl Xipiq
	295	Xipiq:
	296	PUSH_FRAME
	297	movl $0, lapic_eoi /* End Of Interrupt to APIC */
4e7c41c5	298	FAKE_MCOUNT(15*4(%esp))
96728c05	299
b785701b	300	incl PCPU(cnt) + V_IPI
96728c05	301	movl PCPU(curthread),%ebx
f9235b6d MD	302	testl $-1,TD_CRITCOUNT(%ebx)
f9235b6d MD	303	jne 1f
88c4d2f6	304	subl $8,%esp /* make same as interrupt frame */
c7eb0589	305	pushl %esp /* pass frame by reference */
03aa8d99	306	incl PCPU(intr_nesting_level)
f9235b6d	307	incl TD_CRITCOUNT(%ebx)
88c4d2f6	308	call lwkt_process_ipiq_frame
f9235b6d	309	decl TD_CRITCOUNT(%ebx)
03aa8d99	310	decl PCPU(intr_nesting_level)
c7eb0589	311	addl $12,%esp
38787eef	312	pushl $0 /* CPL for frame (REMOVED) */
96728c05 MD	313	MEXITCOUNT
	314	jmp doreti
	315	1:
235957ed	316	orl $RQF_IPIQ,PCPU(reqflags)
96728c05 MD	317	MEXITCOUNT
	318	POP_FRAME
	319	iret
984263bc	320
78ea5a2a SZ	321	.text
	322	SUPERALIGN_TEXT
	323	.globl Xtimer
	324	Xtimer:
	325	PUSH_FRAME
	326	movl $0, lapic_eoi /* End Of Interrupt to APIC */
	327	FAKE_MCOUNT(15*4(%esp))
	328
b785701b	329	incl PCPU(cnt) + V_TIMER
78ea5a2a	330	movl PCPU(curthread),%ebx
f9235b6d MD	331	testl $-1,TD_CRITCOUNT(%ebx)
f9235b6d MD	332	jne 1f
5119f32b SZ	333	testl $-1,TD_NEST_COUNT(%ebx)
5119f32b SZ	334	jne 1f
78ea5a2a SZ	335	subl $8,%esp /* make same as interrupt frame */
	336	pushl %esp /* pass frame by reference */
	337	incl PCPU(intr_nesting_level)
f9235b6d	338	incl TD_CRITCOUNT(%ebx)
78ea5a2a	339	call lapic_timer_process_frame
f9235b6d	340	decl TD_CRITCOUNT(%ebx)
78ea5a2a SZ	341	decl PCPU(intr_nesting_level)
	342	addl $12,%esp
	343	pushl $0 /* CPL for frame (REMOVED) */
	344	MEXITCOUNT
	345	jmp doreti
	346	1:
	347	orl $RQF_TIMER,PCPU(reqflags)
	348	MEXITCOUNT
	349	POP_FRAME
	350	iret
	351
79b62055	352	#ifdef SMP /* APIC-IO */
97359a5b	353
984263bc	354	MCOUNT_LABEL(bintr)
10ff1029 MD	355	FAST_INTR(0,apic_fastintr0)
	356	FAST_INTR(1,apic_fastintr1)
	357	FAST_INTR(2,apic_fastintr2)
	358	FAST_INTR(3,apic_fastintr3)
	359	FAST_INTR(4,apic_fastintr4)
	360	FAST_INTR(5,apic_fastintr5)
	361	FAST_INTR(6,apic_fastintr6)
	362	FAST_INTR(7,apic_fastintr7)
	363	FAST_INTR(8,apic_fastintr8)
	364	FAST_INTR(9,apic_fastintr9)
	365	FAST_INTR(10,apic_fastintr10)
	366	FAST_INTR(11,apic_fastintr11)
	367	FAST_INTR(12,apic_fastintr12)
	368	FAST_INTR(13,apic_fastintr13)
	369	FAST_INTR(14,apic_fastintr14)
	370	FAST_INTR(15,apic_fastintr15)
	371	FAST_INTR(16,apic_fastintr16)
	372	FAST_INTR(17,apic_fastintr17)
	373	FAST_INTR(18,apic_fastintr18)
	374	FAST_INTR(19,apic_fastintr19)
	375	FAST_INTR(20,apic_fastintr20)
	376	FAST_INTR(21,apic_fastintr21)
	377	FAST_INTR(22,apic_fastintr22)
	378	FAST_INTR(23,apic_fastintr23)
fe5f755a SZ	379	FAST_INTR(24,apic_fastintr24)
	380	FAST_INTR(25,apic_fastintr25)
	381	FAST_INTR(26,apic_fastintr26)
	382	FAST_INTR(27,apic_fastintr27)
	383	FAST_INTR(28,apic_fastintr28)
	384	FAST_INTR(29,apic_fastintr29)
	385	FAST_INTR(30,apic_fastintr30)
	386	FAST_INTR(31,apic_fastintr31)
984263bc MD	387	MCOUNT_LABEL(eintr)
984263bc MD	388
97359a5b MD	389	#endif
97359a5b MD	390
984263bc	391	.data
ef0fdad1	392
984263bc	393	/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
2954c92f MD	394	.globl stopped_cpus, started_cpus
2954c92f MD	395	stopped_cpus:
984263bc	396	.long 0
2954c92f	397	started_cpus:
984263bc MD	398	.long 0
984263bc MD	399
984263bc	400	.globl CNAME(cpustop_restartfunc)
984263bc MD	401	CNAME(cpustop_restartfunc):
	402	.long 0
	403
984263bc	404	.text
06f5be02	405