Fix amd64 trap handling.
[dragonfly.git] / sys / platform / pc64 / amd64 / ipl.s
CommitLineData
c8fe38ae
MD
1/*
2 * Copyright (c) 2008 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * ---
35 *
36 * Copyright (c) 1989, 1990 William F. Jolitz.
37 * Copyright (c) 1990 The Regents of the University of California.
38 * All rights reserved.
39 *
40 * This code is derived from software contributed to Berkeley by
41 * William Jolitz.
42 *
43 * Redistribution and use in source and binary forms, with or without
44 * modification, are permitted provided that the following conditions
45 * are met:
46 * 1. Redistributions of source code must retain the above copyright
47 * notice, this list of conditions and the following disclaimer.
48 * 2. Redistributions in binary form must reproduce the above copyright
49 * notice, this list of conditions and the following disclaimer in the
50 * documentation and/or other materials provided with the distribution.
51 * 3. All advertising materials mentioning features or use of this software
52 * must display the following acknowledgement:
53 * This product includes software developed by the University of
54 * California, Berkeley and its contributors.
55 * 4. Neither the name of the University nor the names of its contributors
56 * may be used to endorse or promote products derived from this software
57 * without specific prior written permission.
58 *
59 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
60 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
61 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
62 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
63 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
64 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
65 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
69 * SUCH DAMAGE.
70 *
71 * @(#)ipl.s
72 *
73 * $FreeBSD: src/sys/i386/isa/ipl.s,v 1.32.2.3 2002/05/16 16:03:56 bde Exp $
74 * $DragonFly: src/sys/platform/pc64/amd64/ipl.s,v 1.1 2008/08/29 17:07:10 dillon Exp $
75 */
76
77#include <machine/asmacros.h>
78#include <machine/segments.h>
79#include <machine/ipl.h>
80#include <machine/lock.h>
81#include <machine/psl.h>
82#include <machine/trap.h>
83
84#include "assym.s"
85
86/*
87 * AT/386
88 * Vector interrupt control section
89 *
90 * ipending - Pending interrupts (set when a masked interrupt occurs)
91 * spending - Pending software interrupts
92 */
93 .data
94 ALIGN_DATA
95
96 .globl fastunpend_count
97fastunpend_count: .long 0
98
99 .text
100 SUPERALIGN_TEXT
101
102 /*
103 * GENERAL NOTES
104 *
105 * - fast interrupts are always called with a critical section
106 * held
107 *
108 * - we release our critical section when scheduling interrupt
109 * or softinterrupt threads in order so they can preempt
110 * (unless we are called manually from a critical section, in
111 * which case there will still be a critical section and
112 * they won't preempt anyway).
113 *
114 * - TD_NEST_COUNT prevents splz from nesting too deeply within
115 * itself. It is *not* actually an interrupt nesting count.
116 * PCPU(intr_nesting_level) is an interrupt nesting count.
117 *
118 * - We have to be careful in regards to local interrupts
119 * occuring simultaniously with our doreti and splz
120 * processing.
121 */
122
123 /*
124 * DORETI
125 *
126 * Handle return from interrupts, traps and syscalls. This function
127 * checks the cpl for unmasked pending interrupts (fast, normal, or
128 * soft) and schedules them if appropriate, then irets.
129 *
130 * If we are in a critical section we cannot run any pending ints
131 * nor can be play with mp_lock.
132 *
133 * The stack contains a trapframe at the start of doreti.
134 */
135 SUPERALIGN_TEXT
136 .globl doreti
137 .type doreti,@function
138doreti:
139 FAKE_MCOUNT(bintr) /* init "from" bintr -> doreti */
140 movq $0,%rax /* irq mask unavailable due to BGL */
141 movq PCPU(curthread),%rbx
142 cli /* interlock with TDPRI_CRIT */
143 cmpl $0,PCPU(reqflags) /* short cut if nothing to do */
144 je 5f
145 cmpl $TDPRI_CRIT,TD_PRI(%rbx) /* can't unpend if in critical sec */
146 jge 5f
147 addl $TDPRI_CRIT,TD_PRI(%rbx) /* force all ints to pending */
148doreti_next:
149 sti /* allow new interrupts */
150 movl %eax,%ecx /* irq mask unavailable due to BGL */
151 notl %ecx
152 cli /* disallow YYY remove */
153#ifdef SMP
154 testl $RQF_IPIQ,PCPU(reqflags)
155 jnz doreti_ipiq
156#endif
157 testl PCPU(fpending),%ecx /* check for an unmasked fast int */
158 jnz doreti_fast
159
160 testl PCPU(ipending),%ecx /* check for an unmasked slow int */
161 jnz doreti_intr
162
163 movl PCPU(spending),%ecx /* check for a pending software int */
164 cmpl $0,%ecx
165 jnz doreti_soft
166
167 testl $RQF_AST_MASK,PCPU(reqflags) /* any pending ASTs? */
168 jz 2f
169
170 /* ASTs are only applicable when returning to userland */
171 testb $SEL_RPL_MASK,TF_CS(%rsp)
172 jnz doreti_ast
1732:
174 /*
175 * Nothing left to do, finish up. Interrupts are still disabled.
176 * %eax contains the mask of IRQ's that are not available due to
177 * BGL requirements. We can only clear RQF_INTPEND if *ALL* pending
178 * interrupts have been processed.
179 */
180 subl $TDPRI_CRIT,TD_PRI(%rbx) /* interlocked with cli */
181 testl %eax,%eax
182 jnz 5f
183 andl $~RQF_INTPEND,PCPU(reqflags)
1845:
185 MEXITCOUNT
186
187 /*
188 * Restore register and iret. iret can fault on %rip (which is
189 * really stupid). If this occurs we re-fault and vector to
190 * doreti_iret_fault().
191 *
192 * ...
193 * can be set from user mode, this can result in a kernel mode
194 * exception. The trap code will revector to the *_fault code
195 * which then sets up a T_PROTFLT signal. If the signal is
196 * sent to userland, sendsig() will automatically clean up all
197 * the segment registers to avoid a loop.
198 */
199 .globl doreti_iret
200 .globl doreti_syscall_ret
201doreti_syscall_ret:
202 POP_FRAME /* registers and %gs (+cli) */
203 /* special global also used by exception.S */
204doreti_iret:
205 iretq
206
207 /*
208 * doreti_iret_fault. Alternative return code for
209 * the case where we get a fault in the doreti_exit code
210 * above. trap() (sys/platform/pc64/amd64/trap.c) catches this specific
211 * case, sends the process a signal and continues in the
212 * corresponding place in the code below.
213 */
214 ALIGN_TEXT
215 .globl doreti_iret_fault
216doreti_iret_fault:
89ffa1cf 217 PUSH_FRAME_NOSWAP
c8fe38ae
MD
218 testq $PSL_I,TF_RFLAGS(%rsp)
219 jz 2f
220 sti
2212:
222 movq $T_PROTFLT,TF_TRAPNO(%rsp)
223 movq $0,TF_ERR(%rsp) /* XXX should be the error code */
224 movq $0,TF_ADDR(%rsp)
225 FAKE_MCOUNT(TF_RIP(%rsp))
226 jmp calltrap
227
228 /*
229 * FAST interrupt pending. NOTE: stack context holds frame structure
230 * for fast interrupt procedure, do not do random pushes or pops!
231 */
232 ALIGN_TEXT
233doreti_fast:
234 andl PCPU(fpending),%ecx /* only check fast ints */
235 bsfl %ecx, %ecx /* locate the next dispatchable int */
236 btrl %ecx, PCPU(fpending) /* is it really still pending? */
237 jnc doreti_next
238 pushq %rax /* save IRQ mask unavailable for BGL */
239 /* NOTE: is also CPL in frame */
240#if 0
241#ifdef SMP
242 pushq %rcx /* save ecx */
243 call try_mplock
244 popq %rcx
245 testl %eax,%eax
246 jz 1f
247 /* MP lock successful */
248#endif
249#endif
250 incl PCPU(intr_nesting_level)
251 call dofastunpend /* unpend fast intr %ecx */
252 decl PCPU(intr_nesting_level)
253#if 0
254#ifdef SMP
255 call rel_mplock
256#endif
257#endif
258 popq %rax
259 jmp doreti_next
2601:
261 btsl %ecx, PCPU(fpending) /* oops, couldn't get the MP lock */
262 popq %rax /* add to temp. cpl mask to ignore */
263 orl PCPU(fpending),%eax
264 jmp doreti_next
265
266 /*
267 * INTR interrupt pending
268 *
269 * Temporarily back-out our critical section to allow an interrupt
270 * preempt us when we schedule it. Bump intr_nesting_level to
271 * prevent the switch code from recursing via splz too deeply.
272 */
273 ALIGN_TEXT
274doreti_intr:
275 andl PCPU(ipending),%ecx /* only check normal ints */
276 bsfl %ecx, %ecx /* locate the next dispatchable int */
277 btrl %ecx, PCPU(ipending) /* is it really still pending? */
278 jnc doreti_next
279 pushq %rax
280 movl %ecx,%edi /* argument to C function */
281 incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
282 subl $TDPRI_CRIT,TD_PRI(%rbx) /* so we can preempt */
283 call sched_ithd /* YYY must pull in imasks */
284 addl $TDPRI_CRIT,TD_PRI(%rbx)
285 decl TD_NEST_COUNT(%rbx)
286 popq %rax
287 jmp doreti_next
288
289 /*
290 * SOFT interrupt pending
291 *
292 * Temporarily back-out our critical section to allow an interrupt
293 * preempt us when we schedule it. Bump intr_nesting_level to
294 * prevent the switch code from recursing via splz too deeply.
295 */
296 ALIGN_TEXT
297doreti_soft:
298 bsfl %ecx,%ecx /* locate the next pending softint */
299 btrl %ecx,PCPU(spending) /* make sure its still pending */
300 jnc doreti_next
301 addl $FIRST_SOFTINT,%ecx /* actual intr number */
302 pushq %rax
303 movl %ecx,%edi /* argument to C call */
304 incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
305 subl $TDPRI_CRIT,TD_PRI(%rbx) /* so we can preempt */
306 call sched_ithd /* YYY must pull in imasks */
307 addl $TDPRI_CRIT,TD_PRI(%rbx)
308 decl TD_NEST_COUNT(%rbx)
309 popq %rax
310 jmp doreti_next
311
312 /*
313 * AST pending. We clear RQF_AST_SIGNAL automatically, the others
314 * are cleared by the trap as they are processed.
315 *
316 * Temporarily back-out our critical section because trap() can be
317 * a long-winded call, and we want to be more syscall-like.
318 *
319 * YYY theoretically we can call lwkt_switch directly if all we need
320 * to do is a reschedule.
321 */
322doreti_ast:
323 andl $~(RQF_AST_SIGNAL|RQF_AST_UPCALL),PCPU(reqflags)
324 sti
325 movl %eax,%esi /* save cpl (can't use stack) */
326 movl $T_ASTFLT,TF_TRAPNO(%rsp)
327 movq %rsp,%rdi /* pass frame by ref (%edi = C arg) */
328 subl $TDPRI_CRIT,TD_PRI(%rbx)
329 call trap
330 addl $TDPRI_CRIT,TD_PRI(%rbx)
331 movl %esi,%eax /* restore cpl for loop */
332 jmp doreti_next
333
334#ifdef SMP
335 /*
336 * IPIQ message pending. We clear RQF_IPIQ automatically.
337 */
338doreti_ipiq:
339 movl %eax,%esi /* save cpl (can't use stack) */
340 incl PCPU(intr_nesting_level)
341 andl $~RQF_IPIQ,PCPU(reqflags)
342 subl $16,%rsp /* add dummy vec and ppl */
343 movq %rsp,%rdi /* pass frame by ref (C arg) */
344 call lwkt_process_ipiq_frame
345 addl $16,%rsp
346 decl PCPU(intr_nesting_level)
347 movl %esi,%eax /* restore cpl for loop */
348 jmp doreti_next
349
350#endif
351
352 /*
353 * SPLZ() a C callable procedure to dispatch any unmasked pending
354 * interrupts regardless of critical section nesting. ASTs
355 * are not dispatched.
356 *
357 * Use %eax to track those IRQs that could not be processed
358 * due to BGL requirements.
359 */
360 SUPERALIGN_TEXT
361
362ENTRY(splz)
363 pushfq
364 pushq %rbx
365 movq PCPU(curthread),%rbx
366 addl $TDPRI_CRIT,TD_PRI(%rbx)
367 movl $0,%eax
368
369splz_next:
370 cli
371 movl %eax,%ecx /* ecx = ~CPL */
372 notl %ecx
373#ifdef SMP
374 testl $RQF_IPIQ,PCPU(reqflags)
375 jnz splz_ipiq
376#endif
377 testl PCPU(fpending),%ecx /* check for an unmasked fast int */
378 jnz splz_fast
379
380 testl PCPU(ipending),%ecx
381 jnz splz_intr
382
383 movl PCPU(spending),%ecx
384 cmpl $0,%ecx
385 jnz splz_soft
386
387 subl $TDPRI_CRIT,TD_PRI(%rbx)
388
389 /*
390 * Nothing left to do, finish up. Interrupts are still disabled.
391 * If our mask of IRQs we couldn't process due to BGL requirements
392 * is 0 then there are no pending interrupt sources left and we
393 * can clear RQF_INTPEND.
394 */
395 testl %eax,%eax
396 jnz 5f
397 andl $~RQF_INTPEND,PCPU(reqflags)
3985:
399 popq %rbx
400 popfq
401 ret
402
403 /*
404 * FAST interrupt pending
405 */
406 ALIGN_TEXT
407splz_fast:
408 andl PCPU(fpending),%ecx /* only check fast ints */
409 bsfl %ecx, %ecx /* locate the next dispatchable int */
410 btrl %ecx, PCPU(fpending) /* is it really still pending? */
411 jnc splz_next
412 pushq %rax
413#if 0
414#ifdef SMP
415 movl %ecx,%edi /* argument to try_mplock */
416 call try_mplock
417 testl %eax,%eax
418 jz 1f
419#endif
420#endif
421 incl PCPU(intr_nesting_level)
422 call dofastunpend /* unpend fast intr %ecx */
423 decl PCPU(intr_nesting_level)
424#if 0
425#ifdef SMP
426 call rel_mplock
427#endif
428#endif
429 popq %rax
430 jmp splz_next
4311:
432 btsl %ecx, PCPU(fpending) /* oops, couldn't get the MP lock */
433 popq %rax
434 orl PCPU(fpending),%eax
435 jmp splz_next
436
437 /*
438 * INTR interrupt pending
439 *
440 * Temporarily back-out our critical section to allow the interrupt
441 * preempt us.
442 */
443 ALIGN_TEXT
444splz_intr:
445 andl PCPU(ipending),%ecx /* only check normal ints */
446 bsfl %ecx, %ecx /* locate the next dispatchable int */
447 btrl %ecx, PCPU(ipending) /* is it really still pending? */
448 jnc splz_next
449 sti
450 pushq %rax
451 movl %ecx,%edi /* C argument */
452 subl $TDPRI_CRIT,TD_PRI(%rbx)
453 incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
454 call sched_ithd /* YYY must pull in imasks */
455 addl $TDPRI_CRIT,TD_PRI(%rbx)
456 decl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
457 popq %rax
458 jmp splz_next
459
460 /*
461 * SOFT interrupt pending
462 *
463 * Temporarily back-out our critical section to allow the interrupt
464 * preempt us.
465 */
466 ALIGN_TEXT
467splz_soft:
468 bsfl %ecx,%ecx /* locate the next pending softint */
469 btrl %ecx,PCPU(spending) /* make sure its still pending */
470 jnc splz_next
471 addl $FIRST_SOFTINT,%ecx /* actual intr number */
472 sti
473 pushq %rax
474 movl %ecx,%edi /* C argument */
475 subl $TDPRI_CRIT,TD_PRI(%rbx)
476 incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
477 call sched_ithd /* YYY must pull in imasks */
478 addl $TDPRI_CRIT,TD_PRI(%rbx)
479 decl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
480 popq %rax
481 jmp splz_next
482
483#ifdef SMP
484splz_ipiq:
485 andl $~RQF_IPIQ,PCPU(reqflags)
486 pushq %rax
487 call lwkt_process_ipiq
488 popq %rax
489 jmp splz_next
490#endif
491
492 /*
493 * dofastunpend(%ecx:intr)
494 *
495 * A FAST interrupt previously made pending can now be run,
496 * execute it by pushing a dummy interrupt frame and
497 * calling ithread_fast_handler to execute or schedule it.
498 *
499 * ithread_fast_handler() returns 0 if it wants us to unmask
500 * further interrupts.
501 */
502#define PUSH_DUMMY \
503 pushfq ; /* phys int frame / flags */ \
504 movl %cs,%eax ; \
505 pushq %rax ; /* phys int frame / cs */ \
506 pushq 3*8(%rsp) ; /* original caller eip */ \
507 subq $TF_RIP,%rsp ; /* trap frame */ \
508 movq $0,TF_TRAPNO(%rsp) ; /* extras */ \
509 movq $0,TF_ADDR(%rsp) ; /* extras */ \
510 movq $0,TF_FLAGS(%rsp) ; /* extras */ \
511 movq $0,TF_ERR(%rsp) ; /* extras */ \
512
513#define POP_DUMMY \
514 addq $TF_RIP+(3*8),%rsp ; \
515
516dofastunpend:
517 pushq %rbp /* frame for backtrace */
518 movq %rsp,%rbp
519 PUSH_DUMMY
520 pushq %rcx /* last part of intrframe = intr */
521 incl fastunpend_count
522 movq %rsp,%rdi /* pass frame by reference C arg */
523 call ithread_fast_handler /* returns 0 to unmask */
524 popq %rdi /* intrframe->trapframe */
525 /* + also rdi C arg to next call */
526 cmpl $0,%eax
527 jnz 1f
528 movq MachIntrABI + MACHINTR_INTREN, %rax
529 callq *%rax /* MachIntrABI.intren(intr) */
5301:
531 POP_DUMMY
532 popq %rbp
533 ret
534