kernel - Fix serious issue w/ smp_invltlb(), plus other issues (3)
[dragonfly.git] / sys / platform / pc64 / x86_64 / ipl.s
CommitLineData
c8fe38ae
MD
1/*
2 * Copyright (c) 2008 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * ---
35 *
36 * Copyright (c) 1989, 1990 William F. Jolitz.
37 * Copyright (c) 1990 The Regents of the University of California.
38 * All rights reserved.
39 *
40 * This code is derived from software contributed to Berkeley by
41 * William Jolitz.
42 *
43 * Redistribution and use in source and binary forms, with or without
44 * modification, are permitted provided that the following conditions
45 * are met:
46 * 1. Redistributions of source code must retain the above copyright
47 * notice, this list of conditions and the following disclaimer.
48 * 2. Redistributions in binary form must reproduce the above copyright
49 * notice, this list of conditions and the following disclaimer in the
50 * documentation and/or other materials provided with the distribution.
51 * 3. All advertising materials mentioning features or use of this software
52 * must display the following acknowledgement:
53 * This product includes software developed by the University of
54 * California, Berkeley and its contributors.
55 * 4. Neither the name of the University nor the names of its contributors
56 * may be used to endorse or promote products derived from this software
57 * without specific prior written permission.
58 *
59 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
60 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
61 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
62 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
63 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
64 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
65 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
69 * SUCH DAMAGE.
70 *
71 * @(#)ipl.s
72 *
73 * $FreeBSD: src/sys/i386/isa/ipl.s,v 1.32.2.3 2002/05/16 16:03:56 bde Exp $
c8fe38ae
MD
74 */
75
76#include <machine/asmacros.h>
77#include <machine/segments.h>
78#include <machine/ipl.h>
79#include <machine/lock.h>
80#include <machine/psl.h>
81#include <machine/trap.h>
82
83#include "assym.s"
84
85/*
86 * AT/386
87 * Vector interrupt control section
88 *
729e15a8 89 * fpending - Pending interrupts (set when a masked interrupt occurs)
c8fe38ae
MD
90 * spending - Pending software interrupts
91 */
92 .data
93 ALIGN_DATA
94
95 .globl fastunpend_count
96fastunpend_count: .long 0
97
98 .text
99 SUPERALIGN_TEXT
100
101 /*
102 * GENERAL NOTES
103 *
104 * - fast interrupts are always called with a critical section
105 * held
106 *
107 * - we release our critical section when scheduling interrupt
108 * or softinterrupt threads in order so they can preempt
109 * (unless we are called manually from a critical section, in
110 * which case there will still be a critical section and
111 * they won't preempt anyway).
112 *
113 * - TD_NEST_COUNT prevents splz from nesting too deeply within
114 * itself. It is *not* actually an interrupt nesting count.
115 * PCPU(intr_nesting_level) is an interrupt nesting count.
116 *
117 * - We have to be careful in regards to local interrupts
118 * occuring simultaniously with our doreti and splz
119 * processing.
ecf9dec6
MD
120 *
121 * - Interrupts must be enabled when calling higher level
122 * functions in order to avoid deadlocking against things
123 * like smp_invltlb.
c8fe38ae
MD
124 */
125
126 /*
127 * DORETI
128 *
129 * Handle return from interrupts, traps and syscalls. This function
130 * checks the cpl for unmasked pending interrupts (fast, normal, or
131 * soft) and schedules them if appropriate, then irets.
132 *
133 * If we are in a critical section we cannot run any pending ints
134 * nor can be play with mp_lock.
135 *
136 * The stack contains a trapframe at the start of doreti.
137 */
138 SUPERALIGN_TEXT
139 .globl doreti
140 .type doreti,@function
141doreti:
142 FAKE_MCOUNT(bintr) /* init "from" bintr -> doreti */
143 movq $0,%rax /* irq mask unavailable due to BGL */
144 movq PCPU(curthread),%rbx
f9235b6d 145 cli /* interlock with critical section */
c8fe38ae
MD
146 cmpl $0,PCPU(reqflags) /* short cut if nothing to do */
147 je 5f
f9235b6d
MD
148 testl $-1,TD_CRITCOUNT(%rbx) /* can't unpend if in critical sec */
149 jne 5f
150 incl TD_CRITCOUNT(%rbx) /* force all ints to pending */
c8fe38ae 151doreti_next:
ecf9dec6 152 cli /* re-assert cli on loop */
c8fe38ae
MD
153 movl %eax,%ecx /* irq mask unavailable due to BGL */
154 notl %ecx
c8fe38ae
MD
155#ifdef SMP
156 testl $RQF_IPIQ,PCPU(reqflags)
157 jnz doreti_ipiq
46d4e165
JG
158 testl $RQF_TIMER,PCPU(reqflags)
159 jnz doreti_timer
c8fe38ae
MD
160#endif
161 testl PCPU(fpending),%ecx /* check for an unmasked fast int */
162 jnz doreti_fast
163
c8fe38ae
MD
164 movl PCPU(spending),%ecx /* check for a pending software int */
165 cmpl $0,%ecx
166 jnz doreti_soft
167
168 testl $RQF_AST_MASK,PCPU(reqflags) /* any pending ASTs? */
169 jz 2f
170
171 /* ASTs are only applicable when returning to userland */
172 testb $SEL_RPL_MASK,TF_CS(%rsp)
173 jnz doreti_ast
1742:
175 /*
176 * Nothing left to do, finish up. Interrupts are still disabled.
177 * %eax contains the mask of IRQ's that are not available due to
178 * BGL requirements. We can only clear RQF_INTPEND if *ALL* pending
179 * interrupts have been processed.
180 */
f9235b6d 181 decl TD_CRITCOUNT(%rbx) /* interlocked with cli */
c8fe38ae
MD
182 testl %eax,%eax
183 jnz 5f
184 andl $~RQF_INTPEND,PCPU(reqflags)
1855:
186 MEXITCOUNT
187
188 /*
ecf9dec6
MD
189 * (interrupts are disabled here)
190 *
c8fe38ae
MD
191 * Restore register and iret. iret can fault on %rip (which is
192 * really stupid). If this occurs we re-fault and vector to
193 * doreti_iret_fault().
194 *
195 * ...
196 * can be set from user mode, this can result in a kernel mode
197 * exception. The trap code will revector to the *_fault code
198 * which then sets up a T_PROTFLT signal. If the signal is
199 * sent to userland, sendsig() will automatically clean up all
200 * the segment registers to avoid a loop.
201 */
202 .globl doreti_iret
203 .globl doreti_syscall_ret
204doreti_syscall_ret:
205 POP_FRAME /* registers and %gs (+cli) */
16a43447 206 /* WARNING: special global doreti_iret is also used by exception.S */
c8fe38ae
MD
207doreti_iret:
208 iretq
209
210 /*
16a43447
MD
211 * doreti_iret_fault. Alternative return code for the case where
212 * we get a fault in the doreti_exit code above. trap()
213 * (sys/platform/pc64/x86_64/trap.c) catches this specific * case,
214 * sends the process a signal and continues in the corresponding
215 * place in the code below.
ecf9dec6
MD
216 *
217 * Interrupts are likely disabled due to the above interlock
218 * between cli/iretq. We must enable them before calling any
219 * high level function.
c8fe38ae
MD
220 */
221 ALIGN_TEXT
222 .globl doreti_iret_fault
223doreti_iret_fault:
89ffa1cf 224 PUSH_FRAME_NOSWAP
c8fe38ae 225 sti
c8fe38ae
MD
226 movq $T_PROTFLT,TF_TRAPNO(%rsp)
227 movq $0,TF_ERR(%rsp) /* XXX should be the error code */
228 movq $0,TF_ADDR(%rsp)
229 FAKE_MCOUNT(TF_RIP(%rsp))
230 jmp calltrap
231
232 /*
233 * FAST interrupt pending. NOTE: stack context holds frame structure
234 * for fast interrupt procedure, do not do random pushes or pops!
235 */
236 ALIGN_TEXT
237doreti_fast:
238 andl PCPU(fpending),%ecx /* only check fast ints */
ecf9dec6 239 sti
c8fe38ae
MD
240 bsfl %ecx, %ecx /* locate the next dispatchable int */
241 btrl %ecx, PCPU(fpending) /* is it really still pending? */
242 jnc doreti_next
243 pushq %rax /* save IRQ mask unavailable for BGL */
244 /* NOTE: is also CPL in frame */
c8fe38ae 245 call dofastunpend /* unpend fast intr %ecx */
c8fe38ae
MD
246 popq %rax
247 jmp doreti_next
c8fe38ae
MD
248
249 /*
c8fe38ae
MD
250 * SOFT interrupt pending
251 *
252 * Temporarily back-out our critical section to allow an interrupt
253 * preempt us when we schedule it. Bump intr_nesting_level to
254 * prevent the switch code from recursing via splz too deeply.
255 */
256 ALIGN_TEXT
257doreti_soft:
ecf9dec6 258 sti
c8fe38ae
MD
259 bsfl %ecx,%ecx /* locate the next pending softint */
260 btrl %ecx,PCPU(spending) /* make sure its still pending */
261 jnc doreti_next
262 addl $FIRST_SOFTINT,%ecx /* actual intr number */
263 pushq %rax
264 movl %ecx,%edi /* argument to C call */
265 incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
f9235b6d 266 decl TD_CRITCOUNT(%rbx) /* so we can preempt */
c8fe38ae 267 call sched_ithd /* YYY must pull in imasks */
f9235b6d 268 incl TD_CRITCOUNT(%rbx)
c8fe38ae
MD
269 decl TD_NEST_COUNT(%rbx)
270 popq %rax
271 jmp doreti_next
272
273 /*
274 * AST pending. We clear RQF_AST_SIGNAL automatically, the others
275 * are cleared by the trap as they are processed.
276 *
277 * Temporarily back-out our critical section because trap() can be
278 * a long-winded call, and we want to be more syscall-like.
279 *
280 * YYY theoretically we can call lwkt_switch directly if all we need
281 * to do is a reschedule.
282 */
283doreti_ast:
284 andl $~(RQF_AST_SIGNAL|RQF_AST_UPCALL),PCPU(reqflags)
285 sti
e58c365d 286 movl %eax,%r12d /* save cpl (can't use stack) */
c8fe38ae
MD
287 movl $T_ASTFLT,TF_TRAPNO(%rsp)
288 movq %rsp,%rdi /* pass frame by ref (%edi = C arg) */
f9235b6d 289 decl TD_CRITCOUNT(%rbx)
c8fe38ae 290 call trap
f9235b6d 291 incl TD_CRITCOUNT(%rbx)
e58c365d 292 movl %r12d,%eax /* restore cpl for loop */
c8fe38ae
MD
293 jmp doreti_next
294
295#ifdef SMP
296 /*
297 * IPIQ message pending. We clear RQF_IPIQ automatically.
298 */
299doreti_ipiq:
e58c365d 300 movl %eax,%r12d /* save cpl (can't use stack) */
c8fe38ae
MD
301 incl PCPU(intr_nesting_level)
302 andl $~RQF_IPIQ,PCPU(reqflags)
ecf9dec6 303 sti
061e4845 304 subq $8,%rsp /* trapframe->intrframe */
c8fe38ae
MD
305 movq %rsp,%rdi /* pass frame by ref (C arg) */
306 call lwkt_process_ipiq_frame
a2a636cc 307 addq $8,%rsp /* intrframe->trapframe */
46d4e165 308 decl PCPU(intr_nesting_level)
e58c365d 309 movl %r12d,%eax /* restore cpl for loop */
46d4e165
JG
310 jmp doreti_next
311
312doreti_timer:
e58c365d 313 movl %eax,%r12d /* save cpl (can't use stack) */
46d4e165
JG
314 incl PCPU(intr_nesting_level)
315 andl $~RQF_TIMER,PCPU(reqflags)
ecf9dec6 316 sti
a2a636cc 317 subq $8,%rsp /* trapframe->intrframe */
5fe0f3a6 318 movq %rsp,%rdi /* pass frame by ref (C arg) */
46d4e165 319 call lapic_timer_process_frame
a2a636cc 320 addq $8,%rsp /* intrframe->trapframe */
c8fe38ae 321 decl PCPU(intr_nesting_level)
e58c365d 322 movl %r12d,%eax /* restore cpl for loop */
c8fe38ae
MD
323 jmp doreti_next
324
325#endif
326
327 /*
328 * SPLZ() a C callable procedure to dispatch any unmasked pending
329 * interrupts regardless of critical section nesting. ASTs
330 * are not dispatched.
331 *
332 * Use %eax to track those IRQs that could not be processed
333 * due to BGL requirements.
334 */
335 SUPERALIGN_TEXT
336
337ENTRY(splz)
338 pushfq
339 pushq %rbx
340 movq PCPU(curthread),%rbx
f9235b6d 341 incl TD_CRITCOUNT(%rbx)
c8fe38ae
MD
342 movl $0,%eax
343
344splz_next:
345 cli
346 movl %eax,%ecx /* ecx = ~CPL */
347 notl %ecx
348#ifdef SMP
349 testl $RQF_IPIQ,PCPU(reqflags)
350 jnz splz_ipiq
46d4e165
JG
351 testl $RQF_TIMER,PCPU(reqflags)
352 jnz splz_timer
c8fe38ae
MD
353#endif
354 testl PCPU(fpending),%ecx /* check for an unmasked fast int */
355 jnz splz_fast
356
c8fe38ae
MD
357 movl PCPU(spending),%ecx
358 cmpl $0,%ecx
359 jnz splz_soft
360
f9235b6d 361 decl TD_CRITCOUNT(%rbx)
c8fe38ae
MD
362
363 /*
364 * Nothing left to do, finish up. Interrupts are still disabled.
365 * If our mask of IRQs we couldn't process due to BGL requirements
366 * is 0 then there are no pending interrupt sources left and we
367 * can clear RQF_INTPEND.
368 */
369 testl %eax,%eax
370 jnz 5f
371 andl $~RQF_INTPEND,PCPU(reqflags)
3725:
373 popq %rbx
374 popfq
375 ret
376
377 /*
378 * FAST interrupt pending
379 */
380 ALIGN_TEXT
381splz_fast:
382 andl PCPU(fpending),%ecx /* only check fast ints */
ecf9dec6 383 sti
c8fe38ae
MD
384 bsfl %ecx, %ecx /* locate the next dispatchable int */
385 btrl %ecx, PCPU(fpending) /* is it really still pending? */
386 jnc splz_next
387 pushq %rax
c8fe38ae 388 call dofastunpend /* unpend fast intr %ecx */
c8fe38ae
MD
389 popq %rax
390 jmp splz_next
3911:
392 btsl %ecx, PCPU(fpending) /* oops, couldn't get the MP lock */
393 popq %rax
394 orl PCPU(fpending),%eax
395 jmp splz_next
396
397 /*
c8fe38ae
MD
398 * SOFT interrupt pending
399 *
400 * Temporarily back-out our critical section to allow the interrupt
401 * preempt us.
402 */
403 ALIGN_TEXT
404splz_soft:
ecf9dec6 405 sti
c8fe38ae
MD
406 bsfl %ecx,%ecx /* locate the next pending softint */
407 btrl %ecx,PCPU(spending) /* make sure its still pending */
408 jnc splz_next
409 addl $FIRST_SOFTINT,%ecx /* actual intr number */
410 sti
411 pushq %rax
412 movl %ecx,%edi /* C argument */
c8fe38ae 413 incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
ecf9dec6 414 decl TD_CRITCOUNT(%rbx)
c8fe38ae 415 call sched_ithd /* YYY must pull in imasks */
f9235b6d 416 incl TD_CRITCOUNT(%rbx)
c8fe38ae
MD
417 decl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
418 popq %rax
419 jmp splz_next
420
421#ifdef SMP
422splz_ipiq:
423 andl $~RQF_IPIQ,PCPU(reqflags)
ecf9dec6 424 sti
c8fe38ae
MD
425 pushq %rax
426 call lwkt_process_ipiq
427 popq %rax
428 jmp splz_next
46d4e165
JG
429
430splz_timer:
431 andl $~RQF_TIMER,PCPU(reqflags)
ecf9dec6 432 sti
46d4e165
JG
433 pushq %rax
434 call lapic_timer_process
435 popq %rax
436 jmp splz_next
c8fe38ae
MD
437#endif
438
439 /*
440 * dofastunpend(%ecx:intr)
441 *
442 * A FAST interrupt previously made pending can now be run,
443 * execute it by pushing a dummy interrupt frame and
444 * calling ithread_fast_handler to execute or schedule it.
445 *
446 * ithread_fast_handler() returns 0 if it wants us to unmask
447 * further interrupts.
448 */
449#define PUSH_DUMMY \
450 pushfq ; /* phys int frame / flags */ \
451 movl %cs,%eax ; \
452 pushq %rax ; /* phys int frame / cs */ \
453 pushq 3*8(%rsp) ; /* original caller eip */ \
454 subq $TF_RIP,%rsp ; /* trap frame */ \
5b9f6cc4 455 movq $0,TF_XFLAGS(%rsp) ; /* extras */ \
c8fe38ae
MD
456 movq $0,TF_TRAPNO(%rsp) ; /* extras */ \
457 movq $0,TF_ADDR(%rsp) ; /* extras */ \
458 movq $0,TF_FLAGS(%rsp) ; /* extras */ \
459 movq $0,TF_ERR(%rsp) ; /* extras */ \
460
461#define POP_DUMMY \
462 addq $TF_RIP+(3*8),%rsp ; \
463
464dofastunpend:
465 pushq %rbp /* frame for backtrace */
466 movq %rsp,%rbp
467 PUSH_DUMMY
468 pushq %rcx /* last part of intrframe = intr */
469 incl fastunpend_count
470 movq %rsp,%rdi /* pass frame by reference C arg */
471 call ithread_fast_handler /* returns 0 to unmask */
472 popq %rdi /* intrframe->trapframe */
473 /* + also rdi C arg to next call */
474 cmpl $0,%eax
475 jnz 1f
476 movq MachIntrABI + MACHINTR_INTREN, %rax
477 callq *%rax /* MachIntrABI.intren(intr) */
4781:
479 POP_DUMMY
480 popq %rbp
481 ret
482