Initial import from FreeBSD RELENG_4:
[dragonfly.git] / sys / i386 / apic / apic_vector.s
1 /*
2  *      from: vector.s, 386BSD 0.1 unknown origin
3  * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
4  */
5
6
7 #include <machine/apic.h>
8 #include <machine/smp.h>
9
10 #include "i386/isa/intr_machdep.h"
11
12 /* convert an absolute IRQ# into a bitmask */
13 #define IRQ_BIT(irq_num)        (1 << (irq_num))
14
15 /* make an index into the IO APIC from the IRQ# */
16 #define REDTBL_IDX(irq_num)     (0x10 + ((irq_num) * 2))
17
18
19 /*
20  * Macros for interrupt interrupt entry, call to handler, and exit.
21  */
22
23 #define FAST_INTR(irq_num, vec_name)                                    \
24         .text ;                                                         \
25         SUPERALIGN_TEXT ;                                               \
26 IDTVEC(vec_name) ;                                                      \
27         pushl   %eax ;          /* save only call-used registers */     \
28         pushl   %ecx ;                                                  \
29         pushl   %edx ;                                                  \
30         pushl   %ds ;                                                   \
31         MAYBE_PUSHL_ES ;                                                \
32         pushl   %fs ;                                                   \
33         movl    $KDSEL,%eax ;                                           \
34         mov     %ax,%ds ;                                               \
35         MAYBE_MOVW_AX_ES ;                                              \
36         movl    $KPSEL,%eax ;                                           \
37         mov     %ax,%fs ;                                               \
38         FAKE_MCOUNT((5+ACTUALLY_PUSHED)*4(%esp)) ;                      \
39         pushl   _intr_unit + (irq_num) * 4 ;                            \
40         call    *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
41         addl    $4, %esp ;                                              \
42         movl    $0, lapic_eoi ;                                         \
43         lock ;                                                          \
44         incl    _cnt+V_INTR ;   /* book-keeping can wait */             \
45         movl    _intr_countp + (irq_num) * 4, %eax ;                    \
46         lock ;                                                          \
47         incl    (%eax) ;                                                \
48         MEXITCOUNT ;                                                    \
49         popl    %fs ;                                                   \
50         MAYBE_POPL_ES ;                                                 \
51         popl    %ds ;                                                   \
52         popl    %edx ;                                                  \
53         popl    %ecx ;                                                  \
54         popl    %eax ;                                                  \
55         iret
56
57 /*
58  * 
59  */
60 #define PUSH_FRAME                                                      \
61         pushl   $0 ;            /* dummy error code */                  \
62         pushl   $0 ;            /* dummy trap type */                   \
63         pushal ;                                                        \
64         pushl   %ds ;           /* save data and extra segments ... */  \
65         pushl   %es ;                                                   \
66         pushl   %fs
67
68 #define POP_FRAME                                                       \
69         popl    %fs ;                                                   \
70         popl    %es ;                                                   \
71         popl    %ds ;                                                   \
72         popal ;                                                         \
73         addl    $4+4,%esp
74
75 #define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
76 #define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
77         
78 #define MASK_IRQ(irq_num)                                               \
79         IMASK_LOCK ;                            /* into critical reg */ \
80         testl   $IRQ_BIT(irq_num), _apic_imen ;                         \
81         jne     7f ;                    /* masked, don't mask */        \
82         orl     $IRQ_BIT(irq_num), _apic_imen ; /* set the mask bit */  \
83         movl    IOAPICADDR(irq_num), %ecx ;     /* ioapic addr */       \
84         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
85         movl    %eax, (%ecx) ;                  /* write the index */   \
86         movl    IOAPIC_WINDOW(%ecx), %eax ;     /* current value */     \
87         orl     $IOART_INTMASK, %eax ;          /* set the mask */      \
88         movl    %eax, IOAPIC_WINDOW(%ecx) ;     /* new value */         \
89 7: ;                                            /* already masked */    \
90         IMASK_UNLOCK
91 /*
92  * Test to see whether we are handling an edge or level triggered INT.
93  *  Level-triggered INTs must still be masked as we don't clear the source,
94  *  and the EOI cycle would cause redundant INTs to occur.
95  */
96 #define MASK_LEVEL_IRQ(irq_num)                                         \
97         testl   $IRQ_BIT(irq_num), _apic_pin_trigger ;                  \
98         jz      9f ;                            /* edge, don't mask */  \
99         MASK_IRQ(irq_num) ;                                             \
100 9:
101
102
103 #ifdef APIC_INTR_REORDER
104 #define EOI_IRQ(irq_num)                                                \
105         movl    _apic_isrbit_location + 8 * (irq_num), %eax ;           \
106         movl    (%eax), %eax ;                                          \
107         testl   _apic_isrbit_location + 4 + 8 * (irq_num), %eax ;       \
108         jz      9f ;                            /* not active */        \
109         movl    $0, lapic_eoi ;                                         \
110         APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ;        \
111 9:
112
113 #else
114 #define EOI_IRQ(irq_num)                                                \
115         testl   $IRQ_BIT(irq_num), lapic_isr1;                          \
116         jz      9f      ;                       /* not active */        \
117         movl    $0, lapic_eoi;                                          \
118         APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ;        \
119 9:
120 #endif
121         
122         
123 /*
124  * Test to see if the source is currntly masked, clear if so.
125  */
126 #define UNMASK_IRQ(irq_num)                                     \
127         IMASK_LOCK ;                            /* into critical reg */ \
128         testl   $IRQ_BIT(irq_num), _apic_imen ;                         \
129         je      7f ;                    /* bit clear, not masked */     \
130         andl    $~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */    \
131         movl    IOAPICADDR(irq_num),%ecx ;      /* ioapic addr */       \
132         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
133         movl    %eax,(%ecx) ;                   /* write the index */   \
134         movl    IOAPIC_WINDOW(%ecx),%eax ;      /* current value */     \
135         andl    $~IOART_INTMASK,%eax ;          /* clear the mask */    \
136         movl    %eax,IOAPIC_WINDOW(%ecx) ;      /* new value */         \
137 7: ;                                                                    \
138         IMASK_UNLOCK
139
140 #ifdef APIC_INTR_DIAGNOSTIC
141 #ifdef APIC_INTR_DIAGNOSTIC_IRQ
142 log_intr_event:
143         pushf
144         cli
145         pushl   $CNAME(apic_itrace_debuglock)
146         call    CNAME(s_lock_np)
147         addl    $4, %esp
148         movl    CNAME(apic_itrace_debugbuffer_idx), %ecx
149         andl    $32767, %ecx
150         movl    _cpuid, %eax
151         shll    $8,     %eax
152         orl     8(%esp), %eax
153         movw    %ax,    CNAME(apic_itrace_debugbuffer)(,%ecx,2)
154         incl    %ecx
155         andl    $32767, %ecx
156         movl    %ecx,   CNAME(apic_itrace_debugbuffer_idx)
157         pushl   $CNAME(apic_itrace_debuglock)
158         call    CNAME(s_unlock_np)
159         addl    $4, %esp
160         popf
161         ret
162         
163
164 #define APIC_ITRACE(name, irq_num, id)                                  \
165         lock ;                                  /* MP-safe */           \
166         incl    CNAME(name) + (irq_num) * 4 ;                           \
167         pushl   %eax ;                                                  \
168         pushl   %ecx ;                                                  \
169         pushl   %edx ;                                                  \
170         movl    $(irq_num), %eax ;                                      \
171         cmpl    $APIC_INTR_DIAGNOSTIC_IRQ, %eax ;                       \
172         jne     7f ;                                                    \
173         pushl   $id ;                                                   \
174         call    log_intr_event ;                                        \
175         addl    $4, %esp ;                                              \
176 7: ;                                                                    \
177         popl    %edx ;                                                  \
178         popl    %ecx ;                                                  \
179         popl    %eax
180 #else
181 #define APIC_ITRACE(name, irq_num, id)                                  \
182         lock ;                                  /* MP-safe */           \
183         incl    CNAME(name) + (irq_num) * 4
184 #endif
185
186 #define APIC_ITRACE_ENTER 1
187 #define APIC_ITRACE_EOI 2
188 #define APIC_ITRACE_TRYISRLOCK 3
189 #define APIC_ITRACE_GOTISRLOCK 4
190 #define APIC_ITRACE_ENTER2 5
191 #define APIC_ITRACE_LEAVE 6
192 #define APIC_ITRACE_UNMASK 7
193 #define APIC_ITRACE_ACTIVE 8
194 #define APIC_ITRACE_MASKED 9
195 #define APIC_ITRACE_NOISRLOCK 10
196 #define APIC_ITRACE_MASKED2 11
197 #define APIC_ITRACE_SPLZ 12
198 #define APIC_ITRACE_DORETI 13   
199         
200 #else   
201 #define APIC_ITRACE(name, irq_num, id)
202 #endif
203                 
204 #define INTR(irq_num, vec_name, maybe_extra_ipending)                   \
205         .text ;                                                         \
206         SUPERALIGN_TEXT ;                                               \
207 /* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */  \
208 IDTVEC(vec_name) ;                                                      \
209         PUSH_FRAME ;                                                    \
210         movl    $KDSEL, %eax ;  /* reload with kernel's data segment */ \
211         mov     %ax, %ds ;                                              \
212         mov     %ax, %es ;                                              \
213         movl    $KPSEL, %eax ;                                          \
214         mov     %ax, %fs ;                                              \
215 ;                                                                       \
216         maybe_extra_ipending ;                                          \
217 ;                                                                       \
218         APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ;    \
219         lock ;                                  /* MP-safe */           \
220         btsl    $(irq_num), iactive ;           /* lazy masking */      \
221         jc      1f ;                            /* already active */    \
222 ;                                                                       \
223         MASK_LEVEL_IRQ(irq_num) ;                                       \
224         EOI_IRQ(irq_num) ;                                              \
225 0: ;                                                                    \
226         APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
227         MP_TRYLOCK ;            /* XXX this is going away... */         \
228         testl   %eax, %eax ;                    /* did we get it? */    \
229         jz      3f ;                            /* no */                \
230 ;                                                                       \
231         APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
232         testl   $IRQ_BIT(irq_num), _cpl ;                               \
233         jne     2f ;                            /* this INT masked */   \
234 ;                                                                       \
235         incb    _intr_nesting_level ;                                   \
236 ;                                                                       \
237   /* entry point used by doreti_unpend for HWIs. */                     \
238 __CONCAT(Xresume,irq_num): ;                                            \
239         FAKE_MCOUNT(13*4(%esp)) ;               /* XXX avoid dbl cnt */ \
240         lock ;  incl    _cnt+V_INTR ;           /* tally interrupts */  \
241         movl    _intr_countp + (irq_num) * 4, %eax ;                    \
242         lock ;  incl    (%eax) ;                                        \
243 ;                                                                       \
244         movl    _cpl, %eax ;                                            \
245         pushl   %eax ;                                                  \
246         orl     _intr_mask + (irq_num) * 4, %eax ;                      \
247         movl    %eax, _cpl ;                                            \
248         lock ;                                                          \
249         andl    $~IRQ_BIT(irq_num), _ipending ;                         \
250 ;                                                                       \
251         pushl   _intr_unit + (irq_num) * 4 ;                            \
252         APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ;  \
253         sti ;                                                           \
254         call    *_intr_handler + (irq_num) * 4 ;                        \
255         cli ;                                                           \
256         APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ;    \
257 ;                                                                       \
258         lock ;  andl    $~IRQ_BIT(irq_num), iactive ;                   \
259         UNMASK_IRQ(irq_num) ;                                           \
260         APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ;  \
261         sti ;                           /* doreti repeats cli/sti */    \
262         MEXITCOUNT ;                                                    \
263         jmp     _doreti ;                                               \
264 ;                                                                       \
265         ALIGN_TEXT ;                                                    \
266 1: ;                                            /* active  */           \
267         APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ;  \
268         MASK_IRQ(irq_num) ;                                             \
269         EOI_IRQ(irq_num) ;                                              \
270         lock ;                                                          \
271         orl     $IRQ_BIT(irq_num), _ipending ;                          \
272         lock ;                                                          \
273         btsl    $(irq_num), iactive ;           /* still active */      \
274         jnc     0b ;                            /* retry */             \
275         POP_FRAME ;                                                     \
276         iret ;          /* XXX:  iactive bit might be 0 now */          \
277         ALIGN_TEXT ;                                                    \
278 2: ;                            /* masked by cpl, leave iactive set */  \
279         APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ;  \
280         lock ;                                                          \
281         orl     $IRQ_BIT(irq_num), _ipending ;                          \
282         MP_RELLOCK ;                                                    \
283         POP_FRAME ;                                                     \
284         iret ;                                                          \
285         ALIGN_TEXT ;                                                    \
286 3: ;                    /* other cpu has isr lock */                    \
287         APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
288         lock ;                                                          \
289         orl     $IRQ_BIT(irq_num), _ipending ;                          \
290         testl   $IRQ_BIT(irq_num), _cpl ;                               \
291         jne     4f ;                            /* this INT masked */   \
292         call    forward_irq ;    /* forward irq to lock holder */       \
293         POP_FRAME ;                             /* and return */        \
294         iret ;                                                          \
295         ALIGN_TEXT ;                                                    \
296 4: ;                                            /* blocked */           \
297         APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
298         POP_FRAME ;                             /* and return */        \
299         iret
300
301 /*
302  * Handle "spurious INTerrupts".
303  * Notes:
304  *  This is different than the "spurious INTerrupt" generated by an
305  *   8259 PIC for missing INTs.  See the APIC documentation for details.
306  *  This routine should NOT do an 'EOI' cycle.
307  */
308         .text
309         SUPERALIGN_TEXT
310         .globl _Xspuriousint
311 _Xspuriousint:
312
313         /* No EOI cycle used here */
314
315         iret
316
317
318 /*
319  * Handle TLB shootdowns.
320  */
321         .text
322         SUPERALIGN_TEXT
323         .globl  _Xinvltlb
324 _Xinvltlb:
325         pushl   %eax
326
327 #ifdef COUNT_XINVLTLB_HITS
328         pushl   %fs
329         movl    $KPSEL, %eax
330         mov     %ax, %fs
331         movl    _cpuid, %eax
332         popl    %fs
333         ss
334         incl    _xhits(,%eax,4)
335 #endif /* COUNT_XINVLTLB_HITS */
336
337         movl    %cr3, %eax              /* invalidate the TLB */
338         movl    %eax, %cr3
339
340         ss                              /* stack segment, avoid %ds load */
341         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
342
343         popl    %eax
344         iret
345
346
347 #ifdef BETTER_CLOCK
348
349 /*
350  * Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU,
351  *
352  *  - Stores current cpu state in checkstate_cpustate[cpuid]
353  *      0 == user, 1 == sys, 2 == intr
354  *  - Stores current process in checkstate_curproc[cpuid]
355  *
356  *  - Signals its receipt by setting bit cpuid in checkstate_probed_cpus.
357  *
358  * stack: 0->ds, 4->fs, 8->ebx, 12->eax, 16->eip, 20->cs, 24->eflags
359  */
360
361         .text
362         SUPERALIGN_TEXT
363         .globl _Xcpucheckstate
364         .globl _checkstate_cpustate
365         .globl _checkstate_curproc
366         .globl _checkstate_pc
367 _Xcpucheckstate:
368         pushl   %eax
369         pushl   %ebx            
370         pushl   %ds                     /* save current data segment */
371         pushl   %fs
372
373         movl    $KDSEL, %eax
374         mov     %ax, %ds                /* use KERNEL data segment */
375         movl    $KPSEL, %eax
376         mov     %ax, %fs
377
378         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
379
380         movl    $0, %ebx                
381         movl    20(%esp), %eax  
382         andl    $3, %eax
383         cmpl    $3, %eax
384         je      1f
385         testl   $PSL_VM, 24(%esp)
386         jne     1f
387         incl    %ebx                    /* system or interrupt */
388 1:      
389         movl    _cpuid, %eax
390         movl    %ebx, _checkstate_cpustate(,%eax,4)
391         movl    _curproc, %ebx
392         movl    %ebx, _checkstate_curproc(,%eax,4)
393         movl    16(%esp), %ebx
394         movl    %ebx, _checkstate_pc(,%eax,4)
395
396         lock                            /* checkstate_probed_cpus |= (1<<id) */
397         btsl    %eax, _checkstate_probed_cpus
398
399         popl    %fs
400         popl    %ds                     /* restore previous data segment */
401         popl    %ebx
402         popl    %eax
403         iret
404
405 #endif /* BETTER_CLOCK */
406
407 /*
408  * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
409  *
410  *  - Signals its receipt by clearing bit cpuid in checkstate_need_ast.
411  *
412  *  - We need a better method of triggering asts on other cpus.
413  */
414
415         .text
416         SUPERALIGN_TEXT
417         .globl _Xcpuast
418 _Xcpuast:
419         PUSH_FRAME
420         movl    $KDSEL, %eax
421         mov     %ax, %ds                /* use KERNEL data segment */
422         mov     %ax, %es
423         movl    $KPSEL, %eax
424         mov     %ax, %fs
425
426         movl    _cpuid, %eax
427         lock                            /* checkstate_need_ast &= ~(1<<id) */
428         btrl    %eax, _checkstate_need_ast
429         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
430
431         lock
432         btsl    %eax, _checkstate_pending_ast
433         jc      1f
434
435         FAKE_MCOUNT(13*4(%esp))
436
437         /* 
438          * Giant locks do not come cheap.
439          * A lot of cycles are going to be wasted here.
440          */
441         call    _get_mplock
442
443         movl    _cpl, %eax
444         pushl   %eax
445         orl     $AST_PENDING, _astpending       /* XXX */
446         incb    _intr_nesting_level
447         sti
448         
449         pushl   $0
450         
451         movl    _cpuid, %eax
452         lock    
453         btrl    %eax, _checkstate_pending_ast
454         lock    
455         btrl    %eax, CNAME(resched_cpus)
456         jnc     2f
457         orl     $AST_PENDING+AST_RESCHED,_astpending
458         lock
459         incl    CNAME(want_resched_cnt)
460 2:              
461         lock
462         incl    CNAME(cpuast_cnt)
463         MEXITCOUNT
464         jmp     _doreti
465 1:
466         /* We are already in the process of delivering an ast for this CPU */
467         POP_FRAME
468         iret                    
469
470
471 /*
472  *       Executed by a CPU when it receives an XFORWARD_IRQ IPI.
473  */
474
475         .text
476         SUPERALIGN_TEXT
477         .globl _Xforward_irq
478 _Xforward_irq:
479         PUSH_FRAME
480         movl    $KDSEL, %eax
481         mov     %ax, %ds                /* use KERNEL data segment */
482         mov     %ax, %es
483         movl    $KPSEL, %eax
484         mov     %ax, %fs
485
486         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
487
488         FAKE_MCOUNT(13*4(%esp))
489
490         MP_TRYLOCK
491         testl   %eax,%eax               /* Did we get the lock ? */
492         jz  1f                          /* No */
493
494         lock
495         incl    CNAME(forward_irq_hitcnt)
496         cmpb    $4, _intr_nesting_level
497         jae     2f
498         
499         movl    _cpl, %eax
500         pushl   %eax
501         incb    _intr_nesting_level
502         sti
503         
504         pushl   $0
505
506         MEXITCOUNT
507         jmp     _doreti                 /* Handle forwarded interrupt */
508 1:
509         lock
510         incl    CNAME(forward_irq_misscnt)
511         call    forward_irq     /* Oops, we've lost the isr lock */
512         MEXITCOUNT
513         POP_FRAME
514         iret
515 2:
516         lock
517         incl    CNAME(forward_irq_toodeepcnt)
518 3:      
519         MP_RELLOCK
520         MEXITCOUNT
521         POP_FRAME
522         iret
523
524 /*
525  * 
526  */
527 forward_irq:
528         MCOUNT
529         cmpl    $0,_invltlb_ok
530         jz      4f
531
532         cmpl    $0, CNAME(forward_irq_enabled)
533         jz      4f
534
535         movl    _mp_lock,%eax
536         cmpl    $FREE_LOCK,%eax
537         jne     1f
538         movl    $0, %eax                /* Pick CPU #0 if noone has lock */
539 1:
540         shrl    $24,%eax
541         movl    _cpu_num_to_apic_id(,%eax,4),%ecx
542         shll    $24,%ecx
543         movl    lapic_icr_hi, %eax
544         andl    $~APIC_ID_MASK, %eax
545         orl     %ecx, %eax
546         movl    %eax, lapic_icr_hi
547
548 2:
549         movl    lapic_icr_lo, %eax
550         andl    $APIC_DELSTAT_MASK,%eax
551         jnz     2b
552         movl    lapic_icr_lo, %eax
553         andl    $APIC_RESV2_MASK, %eax
554         orl     $(APIC_DEST_DESTFLD|APIC_DELMODE_FIXED|XFORWARD_IRQ_OFFSET), %eax
555         movl    %eax, lapic_icr_lo
556 3:
557         movl    lapic_icr_lo, %eax
558         andl    $APIC_DELSTAT_MASK,%eax
559         jnz     3b
560 4:              
561         ret
562         
563 /*
564  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
565  *
566  *  - Signals its receipt.
567  *  - Waits for permission to restart.
568  *  - Signals its restart.
569  */
570
571         .text
572         SUPERALIGN_TEXT
573         .globl _Xcpustop
574 _Xcpustop:
575         pushl   %ebp
576         movl    %esp, %ebp
577         pushl   %eax
578         pushl   %ecx
579         pushl   %edx
580         pushl   %ds                     /* save current data segment */
581         pushl   %fs
582
583         movl    $KDSEL, %eax
584         mov     %ax, %ds                /* use KERNEL data segment */
585         movl    $KPSEL, %eax
586         mov     %ax, %fs
587
588         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
589
590         movl    _cpuid, %eax
591         imull   $PCB_SIZE, %eax
592         leal    CNAME(stoppcbs)(%eax), %eax
593         pushl   %eax
594         call    CNAME(savectx)          /* Save process context */
595         addl    $4, %esp
596         
597                 
598         movl    _cpuid, %eax
599
600         lock
601         btsl    %eax, _stopped_cpus     /* stopped_cpus |= (1<<id) */
602 1:
603         btl     %eax, _started_cpus     /* while (!(started_cpus & (1<<id))) */
604         jnc     1b
605
606         lock
607         btrl    %eax, _started_cpus     /* started_cpus &= ~(1<<id) */
608         lock
609         btrl    %eax, _stopped_cpus     /* stopped_cpus &= ~(1<<id) */
610
611         test    %eax, %eax
612         jnz     2f
613
614         movl    CNAME(cpustop_restartfunc), %eax
615         test    %eax, %eax
616         jz      2f
617         movl    $0, CNAME(cpustop_restartfunc)  /* One-shot */
618
619         call    *%eax
620 2:
621         popl    %fs
622         popl    %ds                     /* restore previous data segment */
623         popl    %edx
624         popl    %ecx
625         popl    %eax
626         movl    %ebp, %esp
627         popl    %ebp
628         iret
629
630
631 MCOUNT_LABEL(bintr)
632         FAST_INTR(0,fastintr0)
633         FAST_INTR(1,fastintr1)
634         FAST_INTR(2,fastintr2)
635         FAST_INTR(3,fastintr3)
636         FAST_INTR(4,fastintr4)
637         FAST_INTR(5,fastintr5)
638         FAST_INTR(6,fastintr6)
639         FAST_INTR(7,fastintr7)
640         FAST_INTR(8,fastintr8)
641         FAST_INTR(9,fastintr9)
642         FAST_INTR(10,fastintr10)
643         FAST_INTR(11,fastintr11)
644         FAST_INTR(12,fastintr12)
645         FAST_INTR(13,fastintr13)
646         FAST_INTR(14,fastintr14)
647         FAST_INTR(15,fastintr15)
648         FAST_INTR(16,fastintr16)
649         FAST_INTR(17,fastintr17)
650         FAST_INTR(18,fastintr18)
651         FAST_INTR(19,fastintr19)
652         FAST_INTR(20,fastintr20)
653         FAST_INTR(21,fastintr21)
654         FAST_INTR(22,fastintr22)
655         FAST_INTR(23,fastintr23)
656         
657 #define CLKINTR_PENDING                                                 \
658         pushl $clock_lock ;                                             \
659         call s_lock ;                                                   \
660         movl $1,CNAME(clkintr_pending) ;                                \
661         call s_unlock ;                                                 \
662         addl $4, %esp
663
664         INTR(0,intr0, CLKINTR_PENDING)
665         INTR(1,intr1,)
666         INTR(2,intr2,)
667         INTR(3,intr3,)
668         INTR(4,intr4,)
669         INTR(5,intr5,)
670         INTR(6,intr6,)
671         INTR(7,intr7,)
672         INTR(8,intr8,)
673         INTR(9,intr9,)
674         INTR(10,intr10,)
675         INTR(11,intr11,)
676         INTR(12,intr12,)
677         INTR(13,intr13,)
678         INTR(14,intr14,)
679         INTR(15,intr15,)
680         INTR(16,intr16,)
681         INTR(17,intr17,)
682         INTR(18,intr18,)
683         INTR(19,intr19,)
684         INTR(20,intr20,)
685         INTR(21,intr21,)
686         INTR(22,intr22,)
687         INTR(23,intr23,)
688 MCOUNT_LABEL(eintr)
689
690 /*
691  * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
692  *
693  * - Calls the generic rendezvous action function.
694  */
695         .text
696         SUPERALIGN_TEXT
697         .globl  _Xrendezvous
698 _Xrendezvous:
699         PUSH_FRAME
700         movl    $KDSEL, %eax
701         mov     %ax, %ds                /* use KERNEL data segment */
702         mov     %ax, %es
703         movl    $KPSEL, %eax
704         mov     %ax, %fs
705
706         call    _smp_rendezvous_action
707
708         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
709         POP_FRAME
710         iret
711         
712         
713         .data
714 /*
715  * Addresses of interrupt handlers.
716  *  XresumeNN: Resumption addresses for HWIs.
717  */
718         .globl _ihandlers
719 _ihandlers:
720 /*
721  * used by:
722  *  ipl.s:      doreti_unpend
723  */
724         .long   Xresume0,  Xresume1,  Xresume2,  Xresume3 
725         .long   Xresume4,  Xresume5,  Xresume6,  Xresume7
726         .long   Xresume8,  Xresume9,  Xresume10, Xresume11
727         .long   Xresume12, Xresume13, Xresume14, Xresume15 
728         .long   Xresume16, Xresume17, Xresume18, Xresume19
729         .long   Xresume20, Xresume21, Xresume22, Xresume23
730 /*
731  * used by:
732  *  ipl.s:      doreti_unpend
733  *  apic_ipl.s: splz_unpend
734  */
735         .long   _swi_null, swi_net, _swi_null, _swi_null
736         .long   _swi_vm, _swi_null, _softclock
737
738 imasks:                         /* masks for interrupt handlers */
739         .space  NHWI*4          /* padding; HWI masks are elsewhere */
740
741         .long   SWI_TTY_MASK, SWI_NET_MASK, SWI_CAMNET_MASK, SWI_CAMBIO_MASK
742         .long   SWI_VM_MASK, SWI_TQ_MASK, SWI_CLOCK_MASK
743
744 /* active flag for lazy masking */
745 iactive:
746         .long   0
747
748 #ifdef COUNT_XINVLTLB_HITS
749         .globl  _xhits
750 _xhits:
751         .space  (NCPU * 4), 0
752 #endif /* COUNT_XINVLTLB_HITS */
753
754 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
755         .globl _stopped_cpus, _started_cpus
756 _stopped_cpus:
757         .long   0
758 _started_cpus:
759         .long   0
760
761 #ifdef BETTER_CLOCK
762         .globl _checkstate_probed_cpus
763 _checkstate_probed_cpus:
764         .long   0       
765 #endif /* BETTER_CLOCK */
766         .globl _checkstate_need_ast
767 _checkstate_need_ast:
768         .long   0
769 _checkstate_pending_ast:
770         .long   0
771         .globl CNAME(forward_irq_misscnt)
772         .globl CNAME(forward_irq_toodeepcnt)
773         .globl CNAME(forward_irq_hitcnt)
774         .globl CNAME(resched_cpus)
775         .globl CNAME(want_resched_cnt)
776         .globl CNAME(cpuast_cnt)
777         .globl CNAME(cpustop_restartfunc)
778 CNAME(forward_irq_misscnt):     
779         .long 0
780 CNAME(forward_irq_hitcnt):      
781         .long 0
782 CNAME(forward_irq_toodeepcnt):
783         .long 0
784 CNAME(resched_cpus):
785         .long 0
786 CNAME(want_resched_cnt):
787         .long 0
788 CNAME(cpuast_cnt):
789         .long 0
790 CNAME(cpustop_restartfunc):
791         .long 0
792                 
793
794
795         .globl  _apic_pin_trigger
796 _apic_pin_trigger:
797         .long   0
798
799         .text