Commit | Line | Data |
---|---|---|
984263bc MD |
1 | /* |
2 | * from: vector.s, 386BSD 0.1 unknown origin | |
3 | * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $ | |
21ce0dfa | 4 | * $DragonFly: src/sys/platform/pc32/apic/apic_vector.s,v 1.39 2008/08/02 01:14:43 dillon Exp $ |
984263bc MD |
5 | */ |
6 | ||
06f5be02 MD |
7 | #include "use_npx.h" |
8 | #include "opt_auto_eoi.h" | |
9 | ||
10 | #include <machine/asmacros.h> | |
06f5be02 MD |
11 | #include <machine/lock.h> |
12 | #include <machine/psl.h> | |
13 | #include <machine/trap.h> | |
06f5be02 | 14 | |
a9295349 | 15 | #include <machine_base/icu/icu.h> |
21ce0dfa | 16 | #include <bus/isa/isa.h> |
06f5be02 MD |
17 | |
18 | #include "assym.s" | |
984263bc | 19 | |
bdc560a1 MD |
20 | #include "apicreg.h" |
21 | #include "apic_ipl.h" | |
984263bc | 22 | #include <machine/smp.h> |
a9295349 | 23 | #include <machine_base/isa/intr_machdep.h> |
984263bc MD |
24 | |
25 | /* convert an absolute IRQ# into a bitmask */ | |
8a8d5d85 | 26 | #define IRQ_LBIT(irq_num) (1 << (irq_num)) |
984263bc MD |
27 | |
28 | /* make an index into the IO APIC from the IRQ# */ | |
29 | #define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2)) | |
30 | ||
97359a5b MD |
31 | #ifdef SMP |
32 | #define MPLOCKED lock ; | |
33 | #else | |
34 | #define MPLOCKED | |
35 | #endif | |
36 | ||
984263bc | 37 | /* |
8a8d5d85 MD |
38 | * Push an interrupt frame in a format acceptable to doreti, reload |
39 | * the segment registers for the kernel. | |
984263bc MD |
40 | */ |
41 | #define PUSH_FRAME \ | |
42 | pushl $0 ; /* dummy error code */ \ | |
43 | pushl $0 ; /* dummy trap type */ \ | |
4e7c41c5 | 44 | pushl $0 ; /* dummy xflags type */ \ |
984263bc MD |
45 | pushal ; \ |
46 | pushl %ds ; /* save data and extra segments ... */ \ | |
47 | pushl %es ; \ | |
8a8d5d85 | 48 | pushl %fs ; \ |
4e7c41c5 | 49 | pushl %gs ; \ |
c885c20e | 50 | cld ; \ |
8a8d5d85 MD |
51 | mov $KDSEL,%ax ; \ |
52 | mov %ax,%ds ; \ | |
53 | mov %ax,%es ; \ | |
4e7c41c5 | 54 | mov %ax,%gs ; \ |
8a8d5d85 MD |
55 | mov $KPSEL,%ax ; \ |
56 | mov %ax,%fs ; \ | |
984263bc | 57 | |
8a8d5d85 MD |
58 | /* |
59 | * Warning: POP_FRAME can only be used if there is no chance of a | |
60 | * segment register being changed (e.g. by procfs), which is why syscalls | |
61 | * have to use doreti. | |
62 | */ | |
984263bc | 63 | #define POP_FRAME \ |
4e7c41c5 | 64 | popl %gs ; \ |
984263bc MD |
65 | popl %fs ; \ |
66 | popl %es ; \ | |
67 | popl %ds ; \ | |
68 | popal ; \ | |
4e7c41c5 | 69 | addl $3*4,%esp ; /* dummy xflags, trap & error codes */ \ |
8a8d5d85 | 70 | |
3d911e0a SZ |
71 | #define IOAPICADDR(irq_num) \ |
72 | CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR | |
73 | #define REDIRIDX(irq_num) \ | |
74 | CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX | |
0f546930 SZ |
75 | #define IOAPICFLAGS(irq_num) \ |
76 | CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS | |
8a8d5d85 | 77 | |
984263bc | 78 | #define MASK_IRQ(irq_num) \ |
97359a5b | 79 | APIC_IMASK_LOCK ; /* into critical reg */ \ |
ea689d1c | 80 | testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \ |
984263bc | 81 | jne 7f ; /* masked, don't mask */ \ |
ea689d1c SZ |
82 | orl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \ |
83 | /* set the mask bit */ \ | |
984263bc MD |
84 | movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \ |
85 | movl REDIRIDX(irq_num), %eax ; /* get the index */ \ | |
86 | movl %eax, (%ecx) ; /* write the index */ \ | |
bda48b43 | 87 | orl $IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* set the mask */ \ |
984263bc | 88 | 7: ; /* already masked */ \ |
97359a5b | 89 | APIC_IMASK_UNLOCK ; \ |
8a8d5d85 | 90 | |
984263bc MD |
91 | /* |
92 | * Test to see whether we are handling an edge or level triggered INT. | |
93 | * Level-triggered INTs must still be masked as we don't clear the source, | |
94 | * and the EOI cycle would cause redundant INTs to occur. | |
95 | */ | |
96 | #define MASK_LEVEL_IRQ(irq_num) \ | |
0f546930 | 97 | testl $IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ; \ |
984263bc MD |
98 | jz 9f ; /* edge, don't mask */ \ |
99 | MASK_IRQ(irq_num) ; \ | |
8a8d5d85 | 100 | 9: ; \ |
984263bc | 101 | |
984263bc MD |
102 | /* |
103 | * Test to see if the source is currntly masked, clear if so. | |
104 | */ | |
ea689d1c | 105 | #define UNMASK_IRQ(irq_num) \ |
477d3c1c MD |
106 | cmpl $0,%eax ; \ |
107 | jnz 8f ; \ | |
97359a5b | 108 | APIC_IMASK_LOCK ; /* into critical reg */ \ |
ea689d1c | 109 | testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \ |
984263bc | 110 | je 7f ; /* bit clear, not masked */ \ |
ea689d1c SZ |
111 | andl $~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \ |
112 | /* clear mask bit */ \ | |
984263bc MD |
113 | movl IOAPICADDR(irq_num),%ecx ; /* ioapic addr */ \ |
114 | movl REDIRIDX(irq_num), %eax ; /* get the index */ \ | |
115 | movl %eax,(%ecx) ; /* write the index */ \ | |
bda48b43 | 116 | andl $~IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* clear the mask */ \ |
984263bc | 117 | 7: ; \ |
97359a5b | 118 | APIC_IMASK_UNLOCK ; \ |
477d3c1c | 119 | 8: ; \ |
984263bc | 120 | |
79b62055 | 121 | #ifdef SMP /* APIC-IO */ |
97359a5b | 122 | |
8a8d5d85 MD |
123 | /* |
124 | * Fast interrupt call handlers run in the following sequence: | |
125 | * | |
126 | * - Push the trap frame required by doreti | |
127 | * - Mask the interrupt and reenable its source | |
128 | * - If we cannot take the interrupt set its fpending bit and | |
71ef2f5c MD |
129 | * doreti. Note that we cannot mess with mp_lock at all |
130 | * if we entered from a critical section! | |
8a8d5d85 MD |
131 | * - If we can take the interrupt clear its fpending bit, |
132 | * call the handler, then unmask and doreti. | |
133 | * | |
134 | * YYY can cache gd base opitner instead of using hidden %fs prefixes. | |
135 | */ | |
136 | ||
137 | #define FAST_INTR(irq_num, vec_name) \ | |
138 | .text ; \ | |
139 | SUPERALIGN_TEXT ; \ | |
140 | IDTVEC(vec_name) ; \ | |
141 | PUSH_FRAME ; \ | |
4e7c41c5 | 142 | FAKE_MCOUNT(15*4(%esp)) ; \ |
8a8d5d85 | 143 | MASK_LEVEL_IRQ(irq_num) ; \ |
35408d22 | 144 | movl $0, lapic_eoi ; \ |
8a8d5d85 | 145 | movl PCPU(curthread),%ebx ; \ |
38787eef | 146 | movl $0,%eax ; /* CURRENT CPL IN FRAME (REMOVED) */ \ |
984263bc | 147 | pushl %eax ; \ |
1be5027b MD |
148 | testl $-1,TD_NEST_COUNT(%ebx) ; \ |
149 | jne 1f ; \ | |
f9235b6d MD |
150 | testl $-1,TD_CRITCOUNT(%ebx) ; \ |
151 | je 2f ; \ | |
8a8d5d85 | 152 | 1: ; \ |
545a1cd3 | 153 | /* in critical section, make interrupt pending */ \ |
8a8d5d85 MD |
154 | /* set the pending bit and return, leave interrupt masked */ \ |
155 | orl $IRQ_LBIT(irq_num),PCPU(fpending) ; \ | |
235957ed | 156 | orl $RQF_INTPEND,PCPU(reqflags) ; \ |
8a8d5d85 MD |
157 | jmp 5f ; \ |
158 | 2: ; \ | |
159 | /* clear pending bit, run handler */ \ | |
8a8d5d85 | 160 | andl $~IRQ_LBIT(irq_num),PCPU(fpending) ; \ |
477d3c1c | 161 | pushl $irq_num ; \ |
c7eb0589 | 162 | pushl %esp ; /* pass frame by reference */ \ |
f9235b6d | 163 | incl TD_CRITCOUNT(%ebx) ; \ |
477d3c1c | 164 | call ithread_fast_handler ; /* returns 0 to unmask */ \ |
f9235b6d | 165 | decl TD_CRITCOUNT(%ebx) ; \ |
c7eb0589 | 166 | addl $8, %esp ; \ |
8a8d5d85 MD |
167 | UNMASK_IRQ(irq_num) ; \ |
168 | 5: ; \ | |
169 | MEXITCOUNT ; \ | |
170 | jmp doreti ; \ | |
984263bc | 171 | |
97359a5b MD |
172 | #endif |
173 | ||
984263bc MD |
174 | /* |
175 | * Handle "spurious INTerrupts". | |
176 | * Notes: | |
177 | * This is different than the "spurious INTerrupt" generated by an | |
178 | * 8259 PIC for missing INTs. See the APIC documentation for details. | |
179 | * This routine should NOT do an 'EOI' cycle. | |
180 | */ | |
181 | .text | |
182 | SUPERALIGN_TEXT | |
2954c92f MD |
183 | .globl Xspuriousint |
184 | Xspuriousint: | |
984263bc MD |
185 | |
186 | /* No EOI cycle used here */ | |
187 | ||
188 | iret | |
189 | ||
190 | ||
191 | /* | |
192 | * Handle TLB shootdowns. | |
193 | */ | |
194 | .text | |
195 | SUPERALIGN_TEXT | |
8a8d5d85 MD |
196 | .globl Xinvltlb |
197 | Xinvltlb: | |
984263bc MD |
198 | pushl %eax |
199 | ||
984263bc MD |
200 | movl %cr3, %eax /* invalidate the TLB */ |
201 | movl %eax, %cr3 | |
202 | ||
203 | ss /* stack segment, avoid %ds load */ | |
204 | movl $0, lapic_eoi /* End Of Interrupt to APIC */ | |
205 | ||
206 | popl %eax | |
207 | iret | |
208 | ||
209 | ||
984263bc MD |
210 | /* |
211 | * Executed by a CPU when it receives an Xcpustop IPI from another CPU, | |
212 | * | |
213 | * - Signals its receipt. | |
214 | * - Waits for permission to restart. | |
bd8015ca | 215 | * - Processing pending IPIQ events while waiting. |
984263bc MD |
216 | * - Signals its restart. |
217 | */ | |
218 | ||
219 | .text | |
220 | SUPERALIGN_TEXT | |
2954c92f MD |
221 | .globl Xcpustop |
222 | Xcpustop: | |
984263bc MD |
223 | pushl %ebp |
224 | movl %esp, %ebp | |
225 | pushl %eax | |
226 | pushl %ecx | |
227 | pushl %edx | |
228 | pushl %ds /* save current data segment */ | |
229 | pushl %fs | |
230 | ||
231 | movl $KDSEL, %eax | |
232 | mov %ax, %ds /* use KERNEL data segment */ | |
233 | movl $KPSEL, %eax | |
234 | mov %ax, %fs | |
235 | ||
236 | movl $0, lapic_eoi /* End Of Interrupt to APIC */ | |
237 | ||
2954c92f | 238 | movl PCPU(cpuid), %eax |
984263bc MD |
239 | imull $PCB_SIZE, %eax |
240 | leal CNAME(stoppcbs)(%eax), %eax | |
241 | pushl %eax | |
242 | call CNAME(savectx) /* Save process context */ | |
243 | addl $4, %esp | |
244 | ||
245 | ||
2954c92f | 246 | movl PCPU(cpuid), %eax |
984263bc | 247 | |
bd8015ca MD |
248 | /* |
249 | * Indicate that we have stopped and loop waiting for permission | |
250 | * to start again. We must still process IPI events while in a | |
251 | * stopped state. | |
252 | */ | |
97359a5b | 253 | MPLOCKED |
2954c92f | 254 | btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */ |
984263bc | 255 | 1: |
bd8015ca MD |
256 | andl $~RQF_IPIQ,PCPU(reqflags) |
257 | pushl %eax | |
258 | call lwkt_smp_stopped | |
259 | popl %eax | |
2954c92f | 260 | btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */ |
984263bc MD |
261 | jnc 1b |
262 | ||
97359a5b | 263 | MPLOCKED |
2954c92f | 264 | btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */ |
97359a5b | 265 | MPLOCKED |
2954c92f | 266 | btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */ |
984263bc MD |
267 | |
268 | test %eax, %eax | |
269 | jnz 2f | |
270 | ||
271 | movl CNAME(cpustop_restartfunc), %eax | |
272 | test %eax, %eax | |
273 | jz 2f | |
274 | movl $0, CNAME(cpustop_restartfunc) /* One-shot */ | |
275 | ||
276 | call *%eax | |
277 | 2: | |
278 | popl %fs | |
279 | popl %ds /* restore previous data segment */ | |
280 | popl %edx | |
281 | popl %ecx | |
282 | popl %eax | |
283 | movl %ebp, %esp | |
284 | popl %ebp | |
285 | iret | |
286 | ||
96728c05 MD |
287 | /* |
288 | * For now just have one ipiq IPI, but what we really want is | |
289 | * to have one for each source cpu to the APICs don't get stalled | |
290 | * backlogging the requests. | |
291 | */ | |
292 | .text | |
293 | SUPERALIGN_TEXT | |
294 | .globl Xipiq | |
295 | Xipiq: | |
296 | PUSH_FRAME | |
297 | movl $0, lapic_eoi /* End Of Interrupt to APIC */ | |
4e7c41c5 | 298 | FAKE_MCOUNT(15*4(%esp)) |
96728c05 | 299 | |
b785701b | 300 | incl PCPU(cnt) + V_IPI |
96728c05 | 301 | movl PCPU(curthread),%ebx |
f9235b6d MD |
302 | testl $-1,TD_CRITCOUNT(%ebx) |
303 | jne 1f | |
88c4d2f6 | 304 | subl $8,%esp /* make same as interrupt frame */ |
c7eb0589 | 305 | pushl %esp /* pass frame by reference */ |
03aa8d99 | 306 | incl PCPU(intr_nesting_level) |
f9235b6d | 307 | incl TD_CRITCOUNT(%ebx) |
88c4d2f6 | 308 | call lwkt_process_ipiq_frame |
f9235b6d | 309 | decl TD_CRITCOUNT(%ebx) |
03aa8d99 | 310 | decl PCPU(intr_nesting_level) |
c7eb0589 | 311 | addl $12,%esp |
38787eef | 312 | pushl $0 /* CPL for frame (REMOVED) */ |
96728c05 MD |
313 | MEXITCOUNT |
314 | jmp doreti | |
315 | 1: | |
235957ed | 316 | orl $RQF_IPIQ,PCPU(reqflags) |
96728c05 MD |
317 | MEXITCOUNT |
318 | POP_FRAME | |
319 | iret | |
984263bc | 320 | |
78ea5a2a SZ |
321 | .text |
322 | SUPERALIGN_TEXT | |
323 | .globl Xtimer | |
324 | Xtimer: | |
325 | PUSH_FRAME | |
326 | movl $0, lapic_eoi /* End Of Interrupt to APIC */ | |
327 | FAKE_MCOUNT(15*4(%esp)) | |
328 | ||
b785701b | 329 | incl PCPU(cnt) + V_TIMER |
78ea5a2a | 330 | movl PCPU(curthread),%ebx |
f9235b6d MD |
331 | testl $-1,TD_CRITCOUNT(%ebx) |
332 | jne 1f | |
5119f32b SZ |
333 | testl $-1,TD_NEST_COUNT(%ebx) |
334 | jne 1f | |
78ea5a2a SZ |
335 | subl $8,%esp /* make same as interrupt frame */ |
336 | pushl %esp /* pass frame by reference */ | |
337 | incl PCPU(intr_nesting_level) | |
f9235b6d | 338 | incl TD_CRITCOUNT(%ebx) |
78ea5a2a | 339 | call lapic_timer_process_frame |
f9235b6d | 340 | decl TD_CRITCOUNT(%ebx) |
78ea5a2a SZ |
341 | decl PCPU(intr_nesting_level) |
342 | addl $12,%esp | |
343 | pushl $0 /* CPL for frame (REMOVED) */ | |
344 | MEXITCOUNT | |
345 | jmp doreti | |
346 | 1: | |
347 | orl $RQF_TIMER,PCPU(reqflags) | |
348 | MEXITCOUNT | |
349 | POP_FRAME | |
350 | iret | |
351 | ||
79b62055 | 352 | #ifdef SMP /* APIC-IO */ |
97359a5b | 353 | |
984263bc | 354 | MCOUNT_LABEL(bintr) |
10ff1029 MD |
355 | FAST_INTR(0,apic_fastintr0) |
356 | FAST_INTR(1,apic_fastintr1) | |
357 | FAST_INTR(2,apic_fastintr2) | |
358 | FAST_INTR(3,apic_fastintr3) | |
359 | FAST_INTR(4,apic_fastintr4) | |
360 | FAST_INTR(5,apic_fastintr5) | |
361 | FAST_INTR(6,apic_fastintr6) | |
362 | FAST_INTR(7,apic_fastintr7) | |
363 | FAST_INTR(8,apic_fastintr8) | |
364 | FAST_INTR(9,apic_fastintr9) | |
365 | FAST_INTR(10,apic_fastintr10) | |
366 | FAST_INTR(11,apic_fastintr11) | |
367 | FAST_INTR(12,apic_fastintr12) | |
368 | FAST_INTR(13,apic_fastintr13) | |
369 | FAST_INTR(14,apic_fastintr14) | |
370 | FAST_INTR(15,apic_fastintr15) | |
371 | FAST_INTR(16,apic_fastintr16) | |
372 | FAST_INTR(17,apic_fastintr17) | |
373 | FAST_INTR(18,apic_fastintr18) | |
374 | FAST_INTR(19,apic_fastintr19) | |
375 | FAST_INTR(20,apic_fastintr20) | |
376 | FAST_INTR(21,apic_fastintr21) | |
377 | FAST_INTR(22,apic_fastintr22) | |
378 | FAST_INTR(23,apic_fastintr23) | |
fe5f755a SZ |
379 | FAST_INTR(24,apic_fastintr24) |
380 | FAST_INTR(25,apic_fastintr25) | |
381 | FAST_INTR(26,apic_fastintr26) | |
382 | FAST_INTR(27,apic_fastintr27) | |
383 | FAST_INTR(28,apic_fastintr28) | |
384 | FAST_INTR(29,apic_fastintr29) | |
385 | FAST_INTR(30,apic_fastintr30) | |
386 | FAST_INTR(31,apic_fastintr31) | |
984263bc MD |
387 | MCOUNT_LABEL(eintr) |
388 | ||
97359a5b MD |
389 | #endif |
390 | ||
984263bc | 391 | .data |
ef0fdad1 | 392 | |
984263bc | 393 | /* variables used by stop_cpus()/restart_cpus()/Xcpustop */ |
2954c92f MD |
394 | .globl stopped_cpus, started_cpus |
395 | stopped_cpus: | |
984263bc | 396 | .long 0 |
2954c92f | 397 | started_cpus: |
984263bc MD |
398 | .long 0 |
399 | ||
984263bc | 400 | .globl CNAME(cpustop_restartfunc) |
984263bc MD |
401 | CNAME(cpustop_restartfunc): |
402 | .long 0 | |
403 | ||
984263bc | 404 | .text |
06f5be02 | 405 |