kernel - Misc fixes and debugging
[dragonfly.git] / sys / platform / pc64 / x86_64 / ipl.s
CommitLineData
c8fe38ae
MD
1/*
2 * Copyright (c) 2008 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * ---
35 *
36 * Copyright (c) 1989, 1990 William F. Jolitz.
37 * Copyright (c) 1990 The Regents of the University of California.
38 * All rights reserved.
39 *
40 * This code is derived from software contributed to Berkeley by
41 * William Jolitz.
42 *
43 * Redistribution and use in source and binary forms, with or without
44 * modification, are permitted provided that the following conditions
45 * are met:
46 * 1. Redistributions of source code must retain the above copyright
47 * notice, this list of conditions and the following disclaimer.
48 * 2. Redistributions in binary form must reproduce the above copyright
49 * notice, this list of conditions and the following disclaimer in the
50 * documentation and/or other materials provided with the distribution.
51 * 3. All advertising materials mentioning features or use of this software
52 * must display the following acknowledgement:
53 * This product includes software developed by the University of
54 * California, Berkeley and its contributors.
55 * 4. Neither the name of the University nor the names of its contributors
56 * may be used to endorse or promote products derived from this software
57 * without specific prior written permission.
58 *
59 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
60 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
61 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
62 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
63 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
64 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
65 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
69 * SUCH DAMAGE.
70 *
71 * @(#)ipl.s
72 *
73 * $FreeBSD: src/sys/i386/isa/ipl.s,v 1.32.2.3 2002/05/16 16:03:56 bde Exp $
c8fe38ae
MD
74 */
75
76#include <machine/asmacros.h>
77#include <machine/segments.h>
78#include <machine/ipl.h>
79#include <machine/lock.h>
80#include <machine/psl.h>
81#include <machine/trap.h>
82
83#include "assym.s"
84
85/*
86 * AT/386
87 * Vector interrupt control section
88 *
9611ff20 89 * ipending - Pending interrupts (set when a masked interrupt occurs)
c8fe38ae
MD
90 * spending - Pending software interrupts
91 */
92 .data
93 ALIGN_DATA
94
95 .globl fastunpend_count
96fastunpend_count: .long 0
97
98 .text
99 SUPERALIGN_TEXT
100
101 /*
102 * GENERAL NOTES
103 *
9611ff20 104 * - interrupts are always called with a critical section held
c8fe38ae
MD
105 *
106 * - we release our critical section when scheduling interrupt
107 * or softinterrupt threads in order so they can preempt
108 * (unless we are called manually from a critical section, in
109 * which case there will still be a critical section and
110 * they won't preempt anyway).
111 *
112 * - TD_NEST_COUNT prevents splz from nesting too deeply within
113 * itself. It is *not* actually an interrupt nesting count.
114 * PCPU(intr_nesting_level) is an interrupt nesting count.
115 *
116 * - We have to be careful in regards to local interrupts
117 * occuring simultaniously with our doreti and splz
118 * processing.
b4b1a37a
MD
119 *
120 * - Interrupts must be enabled when calling higher level
121 * functions in order to avoid deadlocking against things
122 * like smp_invltlb.
c8fe38ae
MD
123 */
124
125 /*
126 * DORETI
127 *
128 * Handle return from interrupts, traps and syscalls. This function
9611ff20
SZ
129 * checks the cpl for unmasked pending interrupts (hardware or soft)
130 * and schedules them if appropriate, then irets.
c8fe38ae 131 *
b5d16701 132 * If we are in a critical section we cannot run any pending ints.
c8fe38ae
MD
133 *
134 * The stack contains a trapframe at the start of doreti.
135 */
136 SUPERALIGN_TEXT
137 .globl doreti
138 .type doreti,@function
139doreti:
140 FAKE_MCOUNT(bintr) /* init "from" bintr -> doreti */
141 movq $0,%rax /* irq mask unavailable due to BGL */
142 movq PCPU(curthread),%rbx
f9235b6d 143 cli /* interlock with critical section */
c8fe38ae
MD
144 cmpl $0,PCPU(reqflags) /* short cut if nothing to do */
145 je 5f
f9235b6d
MD
146 testl $-1,TD_CRITCOUNT(%rbx) /* can't unpend if in critical sec */
147 jne 5f
148 incl TD_CRITCOUNT(%rbx) /* force all ints to pending */
c8fe38ae 149doreti_next:
b4b1a37a 150 cli /* re-assert cli on loop */
9611ff20
SZ
151 movq %rax,%rcx /* irq mask unavailable due to BGL */
152 notq %rcx
c8fe38ae
MD
153#ifdef SMP
154 testl $RQF_IPIQ,PCPU(reqflags)
155 jnz doreti_ipiq
9a4bd8f3 156#endif
46d4e165
JG
157 testl $RQF_TIMER,PCPU(reqflags)
158 jnz doreti_timer
9611ff20
SZ
159 /*
160 * check for an unmasked int (3 groups)
161 */
162 movq $0,%rdx
163 testq PCPU_E8(ipending,%rdx),%rcx
164 jnz doreti_fast
165
166 movq $1,%rdx
167 testq PCPU_E8(ipending,%rdx),%rcx
168 jnz doreti_fast
169
170 movq $2,%rdx
171 testq PCPU_E8(ipending,%rdx),%rcx
c8fe38ae
MD
172 jnz doreti_fast
173
c8fe38ae
MD
174 movl PCPU(spending),%ecx /* check for a pending software int */
175 cmpl $0,%ecx
176 jnz doreti_soft
177
178 testl $RQF_AST_MASK,PCPU(reqflags) /* any pending ASTs? */
179 jz 2f
180
181 /* ASTs are only applicable when returning to userland */
182 testb $SEL_RPL_MASK,TF_CS(%rsp)
183 jnz doreti_ast
1842:
185 /*
186 * Nothing left to do, finish up. Interrupts are still disabled.
187 * %eax contains the mask of IRQ's that are not available due to
188 * BGL requirements. We can only clear RQF_INTPEND if *ALL* pending
189 * interrupts have been processed.
190 */
f9235b6d 191 decl TD_CRITCOUNT(%rbx) /* interlocked with cli */
c8fe38ae
MD
192 testl %eax,%eax
193 jnz 5f
194 andl $~RQF_INTPEND,PCPU(reqflags)
1955:
196 MEXITCOUNT
197
198 /*
b4b1a37a
MD
199 * (interrupts are disabled here)
200 *
c8fe38ae
MD
201 * Restore register and iret. iret can fault on %rip (which is
202 * really stupid). If this occurs we re-fault and vector to
203 * doreti_iret_fault().
204 *
205 * ...
206 * can be set from user mode, this can result in a kernel mode
207 * exception. The trap code will revector to the *_fault code
208 * which then sets up a T_PROTFLT signal. If the signal is
209 * sent to userland, sendsig() will automatically clean up all
210 * the segment registers to avoid a loop.
211 */
212 .globl doreti_iret
213 .globl doreti_syscall_ret
214doreti_syscall_ret:
215 POP_FRAME /* registers and %gs (+cli) */
2883d2d8 216 /* WARNING: special global doreti_iret is also used by exception.S */
c8fe38ae
MD
217doreti_iret:
218 iretq
219
220 /*
2883d2d8
MD
221 * doreti_iret_fault. Alternative return code for the case where
222 * we get a fault in the doreti_exit code above. trap()
223 * (sys/platform/pc64/x86_64/trap.c) catches this specific * case,
224 * sends the process a signal and continues in the corresponding
225 * place in the code below.
b4b1a37a
MD
226 *
227 * Interrupts are likely disabled due to the above interlock
228 * between cli/iretq. We must enable them before calling any
229 * high level function.
c8fe38ae
MD
230 */
231 ALIGN_TEXT
232 .globl doreti_iret_fault
233doreti_iret_fault:
89ffa1cf 234 PUSH_FRAME_NOSWAP
c8fe38ae 235 sti
c8fe38ae
MD
236 movq $T_PROTFLT,TF_TRAPNO(%rsp)
237 movq $0,TF_ERR(%rsp) /* XXX should be the error code */
238 movq $0,TF_ADDR(%rsp)
239 FAKE_MCOUNT(TF_RIP(%rsp))
240 jmp calltrap
241
242 /*
9611ff20
SZ
243 * Interrupt pending. NOTE: stack context holds frame structure
244 * for interrupt procedure, do not do random pushes or pops!
c8fe38ae
MD
245 */
246 ALIGN_TEXT
247doreti_fast:
9611ff20 248 andq PCPU_E8(ipending,%rdx),%rcx
b4b1a37a 249 sti
9611ff20
SZ
250 bsfq %rcx, %rcx /* locate the next dispatchable int */
251 btrq %rcx, PCPU_E8(ipending,%rdx)
252 /* is it really still pending? */
c8fe38ae 253 jnc doreti_next
9611ff20
SZ
254
255 shlq $6, %rdx
256 orq %rdx, %rcx /* form intr number */
257
c8fe38ae
MD
258 pushq %rax /* save IRQ mask unavailable for BGL */
259 /* NOTE: is also CPL in frame */
9611ff20 260 call dofastunpend /* unpend intr %rcx */
c8fe38ae
MD
261 popq %rax
262 jmp doreti_next
c8fe38ae
MD
263
264 /*
c8fe38ae
MD
265 * SOFT interrupt pending
266 *
267 * Temporarily back-out our critical section to allow an interrupt
268 * preempt us when we schedule it. Bump intr_nesting_level to
269 * prevent the switch code from recursing via splz too deeply.
270 */
271 ALIGN_TEXT
272doreti_soft:
b4b1a37a 273 sti
c8fe38ae
MD
274 bsfl %ecx,%ecx /* locate the next pending softint */
275 btrl %ecx,PCPU(spending) /* make sure its still pending */
276 jnc doreti_next
277 addl $FIRST_SOFTINT,%ecx /* actual intr number */
278 pushq %rax
279 movl %ecx,%edi /* argument to C call */
280 incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
f9235b6d 281 decl TD_CRITCOUNT(%rbx) /* so we can preempt */
c8fe38ae 282 call sched_ithd /* YYY must pull in imasks */
f9235b6d 283 incl TD_CRITCOUNT(%rbx)
c8fe38ae
MD
284 decl TD_NEST_COUNT(%rbx)
285 popq %rax
286 jmp doreti_next
287
288 /*
289 * AST pending. We clear RQF_AST_SIGNAL automatically, the others
290 * are cleared by the trap as they are processed.
291 *
292 * Temporarily back-out our critical section because trap() can be
293 * a long-winded call, and we want to be more syscall-like.
294 *
295 * YYY theoretically we can call lwkt_switch directly if all we need
296 * to do is a reschedule.
297 */
298doreti_ast:
299 andl $~(RQF_AST_SIGNAL|RQF_AST_UPCALL),PCPU(reqflags)
300 sti
e58c365d 301 movl %eax,%r12d /* save cpl (can't use stack) */
c8fe38ae
MD
302 movl $T_ASTFLT,TF_TRAPNO(%rsp)
303 movq %rsp,%rdi /* pass frame by ref (%edi = C arg) */
f9235b6d 304 decl TD_CRITCOUNT(%rbx)
c8fe38ae 305 call trap
f9235b6d 306 incl TD_CRITCOUNT(%rbx)
e58c365d 307 movl %r12d,%eax /* restore cpl for loop */
c8fe38ae
MD
308 jmp doreti_next
309
310#ifdef SMP
311 /*
312 * IPIQ message pending. We clear RQF_IPIQ automatically.
313 */
314doreti_ipiq:
e58c365d 315 movl %eax,%r12d /* save cpl (can't use stack) */
c8fe38ae
MD
316 incl PCPU(intr_nesting_level)
317 andl $~RQF_IPIQ,PCPU(reqflags)
b4b1a37a 318 sti
061e4845 319 subq $8,%rsp /* trapframe->intrframe */
c8fe38ae
MD
320 movq %rsp,%rdi /* pass frame by ref (C arg) */
321 call lwkt_process_ipiq_frame
a2a636cc 322 addq $8,%rsp /* intrframe->trapframe */
46d4e165 323 decl PCPU(intr_nesting_level)
e58c365d 324 movl %r12d,%eax /* restore cpl for loop */
46d4e165 325 jmp doreti_next
9a4bd8f3 326#endif
46d4e165
JG
327
328doreti_timer:
e58c365d 329 movl %eax,%r12d /* save cpl (can't use stack) */
46d4e165
JG
330 incl PCPU(intr_nesting_level)
331 andl $~RQF_TIMER,PCPU(reqflags)
b4b1a37a 332 sti
a2a636cc 333 subq $8,%rsp /* trapframe->intrframe */
b25897b2 334 movq %rsp,%rdi /* pass frame by ref (C arg) */
46d4e165 335 call lapic_timer_process_frame
a2a636cc 336 addq $8,%rsp /* intrframe->trapframe */
c8fe38ae 337 decl PCPU(intr_nesting_level)
e58c365d 338 movl %r12d,%eax /* restore cpl for loop */
c8fe38ae
MD
339 jmp doreti_next
340
c8fe38ae
MD
341 /*
342 * SPLZ() a C callable procedure to dispatch any unmasked pending
343 * interrupts regardless of critical section nesting. ASTs
344 * are not dispatched.
345 *
346 * Use %eax to track those IRQs that could not be processed
347 * due to BGL requirements.
348 */
349 SUPERALIGN_TEXT
350
351ENTRY(splz)
352 pushfq
353 pushq %rbx
354 movq PCPU(curthread),%rbx
f9235b6d 355 incl TD_CRITCOUNT(%rbx)
9611ff20 356 movq $0,%rax
c8fe38ae
MD
357
358splz_next:
359 cli
9611ff20
SZ
360 movq %rax,%rcx /* rcx = ~CPL */
361 notq %rcx
c8fe38ae
MD
362#ifdef SMP
363 testl $RQF_IPIQ,PCPU(reqflags)
364 jnz splz_ipiq
9a4bd8f3 365#endif
46d4e165
JG
366 testl $RQF_TIMER,PCPU(reqflags)
367 jnz splz_timer
9611ff20
SZ
368 /*
369 * check for an unmasked int (3 groups)
370 */
371 movq $0,%rdx
372 testq PCPU_E8(ipending,%rdx),%rcx
373 jnz splz_fast
374
375 movq $1,%rdx
376 testq PCPU_E8(ipending,%rdx),%rcx
377 jnz splz_fast
378
379 movq $2,%rdx
380 testq PCPU_E8(ipending,%rdx),%rcx
c8fe38ae
MD
381 jnz splz_fast
382
c8fe38ae
MD
383 movl PCPU(spending),%ecx
384 cmpl $0,%ecx
385 jnz splz_soft
386
f9235b6d 387 decl TD_CRITCOUNT(%rbx)
c8fe38ae
MD
388
389 /*
390 * Nothing left to do, finish up. Interrupts are still disabled.
391 * If our mask of IRQs we couldn't process due to BGL requirements
392 * is 0 then there are no pending interrupt sources left and we
393 * can clear RQF_INTPEND.
394 */
395 testl %eax,%eax
396 jnz 5f
397 andl $~RQF_INTPEND,PCPU(reqflags)
3985:
399 popq %rbx
400 popfq
401 ret
402
403 /*
9611ff20 404 * Interrupt pending
c8fe38ae
MD
405 */
406 ALIGN_TEXT
407splz_fast:
9611ff20 408 andq PCPU_E8(ipending,%rdx),%rcx
b4b1a37a 409 sti
9611ff20
SZ
410 bsfq %rcx, %rcx /* locate the next dispatchable int */
411 btrq %rcx, PCPU_E8(ipending,%rdx)
412 /* is it really still pending? */
c8fe38ae 413 jnc splz_next
9611ff20
SZ
414
415 shlq $6, %rdx
416 orq %rdx, %rcx /* form intr number */
417
c8fe38ae 418 pushq %rax
9611ff20 419 call dofastunpend /* unpend intr %rcx */
c8fe38ae 420 popq %rax
c8fe38ae
MD
421 jmp splz_next
422
423 /*
c8fe38ae
MD
424 * SOFT interrupt pending
425 *
426 * Temporarily back-out our critical section to allow the interrupt
427 * preempt us.
428 */
429 ALIGN_TEXT
430splz_soft:
b4b1a37a 431 sti
c8fe38ae
MD
432 bsfl %ecx,%ecx /* locate the next pending softint */
433 btrl %ecx,PCPU(spending) /* make sure its still pending */
434 jnc splz_next
435 addl $FIRST_SOFTINT,%ecx /* actual intr number */
436 sti
437 pushq %rax
438 movl %ecx,%edi /* C argument */
c8fe38ae 439 incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
b4b1a37a 440 decl TD_CRITCOUNT(%rbx)
c8fe38ae 441 call sched_ithd /* YYY must pull in imasks */
f9235b6d 442 incl TD_CRITCOUNT(%rbx)
c8fe38ae
MD
443 decl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
444 popq %rax
445 jmp splz_next
446
447#ifdef SMP
448splz_ipiq:
449 andl $~RQF_IPIQ,PCPU(reqflags)
b4b1a37a 450 sti
c8fe38ae
MD
451 pushq %rax
452 call lwkt_process_ipiq
453 popq %rax
454 jmp splz_next
9a4bd8f3 455#endif
46d4e165
JG
456
457splz_timer:
458 andl $~RQF_TIMER,PCPU(reqflags)
b4b1a37a 459 sti
46d4e165
JG
460 pushq %rax
461 call lapic_timer_process
462 popq %rax
463 jmp splz_next
c8fe38ae
MD
464
465 /*
9611ff20 466 * dofastunpend(%rcx:intr)
c8fe38ae 467 *
9611ff20 468 * A interrupt previously made pending can now be run,
c8fe38ae
MD
469 * execute it by pushing a dummy interrupt frame and
470 * calling ithread_fast_handler to execute or schedule it.
471 *
472 * ithread_fast_handler() returns 0 if it wants us to unmask
473 * further interrupts.
474 */
475#define PUSH_DUMMY \
476 pushfq ; /* phys int frame / flags */ \
3338cc67 477 xorq %rax,%rax ; /* something not SEL_UPL */ \
c8fe38ae
MD
478 pushq %rax ; /* phys int frame / cs */ \
479 pushq 3*8(%rsp) ; /* original caller eip */ \
480 subq $TF_RIP,%rsp ; /* trap frame */ \
5b9f6cc4 481 movq $0,TF_XFLAGS(%rsp) ; /* extras */ \
c8fe38ae
MD
482 movq $0,TF_TRAPNO(%rsp) ; /* extras */ \
483 movq $0,TF_ADDR(%rsp) ; /* extras */ \
484 movq $0,TF_FLAGS(%rsp) ; /* extras */ \
485 movq $0,TF_ERR(%rsp) ; /* extras */ \
486
487#define POP_DUMMY \
488 addq $TF_RIP+(3*8),%rsp ; \
489
490dofastunpend:
491 pushq %rbp /* frame for backtrace */
492 movq %rsp,%rbp
493 PUSH_DUMMY
494 pushq %rcx /* last part of intrframe = intr */
495 incl fastunpend_count
496 movq %rsp,%rdi /* pass frame by reference C arg */
497 call ithread_fast_handler /* returns 0 to unmask */
498 popq %rdi /* intrframe->trapframe */
499 /* + also rdi C arg to next call */
500 cmpl $0,%eax
501 jnz 1f
502 movq MachIntrABI + MACHINTR_INTREN, %rax
503 callq *%rax /* MachIntrABI.intren(intr) */
5041:
505 POP_DUMMY
506 popq %rbp
507 ret
508