Finish migrating the cpl into the thread structure.
[dragonfly.git] / sys / platform / pc32 / apic / apic_vector.s
1 /*
2  *      from: vector.s, 386BSD 0.1 unknown origin
3  * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
4  * $DragonFly: src/sys/platform/pc32/apic/apic_vector.s,v 1.5 2003/06/22 08:54:22 dillon Exp $
5  */
6
7
8 #include <machine/apic.h>
9 #include <machine/smp.h>
10
11 #include "i386/isa/intr_machdep.h"
12
13 /* convert an absolute IRQ# into a bitmask */
14 #define IRQ_BIT(irq_num)        (1 << (irq_num))
15
16 /* make an index into the IO APIC from the IRQ# */
17 #define REDTBL_IDX(irq_num)     (0x10 + ((irq_num) * 2))
18
19
20 /*
21  * Macros for interrupt interrupt entry, call to handler, and exit.
22  */
23
24 #define FAST_INTR(irq_num, vec_name)                                    \
25         .text ;                                                         \
26         SUPERALIGN_TEXT ;                                               \
27 IDTVEC(vec_name) ;                                                      \
28         pushl   %eax ;          /* save only call-used registers */     \
29         pushl   %ecx ;                                                  \
30         pushl   %edx ;                                                  \
31         pushl   %ds ;                                                   \
32         MAYBE_PUSHL_ES ;                                                \
33         pushl   %fs ;                                                   \
34         movl    $KDSEL,%eax ;                                           \
35         mov     %ax,%ds ;                                               \
36         MAYBE_MOVW_AX_ES ;                                              \
37         movl    $KPSEL,%eax ;                                           \
38         mov     %ax,%fs ;                                               \
39         FAKE_MCOUNT((5+ACTUALLY_PUSHED)*4(%esp)) ;                      \
40         pushl   _intr_unit + (irq_num) * 4 ;                            \
41         call    *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
42         addl    $4, %esp ;                                              \
43         movl    $0, lapic_eoi ;                                         \
44         lock ;                                                          \
45         incl    _cnt+V_INTR ;   /* book-keeping can wait */             \
46         movl    _intr_countp + (irq_num) * 4, %eax ;                    \
47         lock ;                                                          \
48         incl    (%eax) ;                                                \
49         MEXITCOUNT ;                                                    \
50         popl    %fs ;                                                   \
51         MAYBE_POPL_ES ;                                                 \
52         popl    %ds ;                                                   \
53         popl    %edx ;                                                  \
54         popl    %ecx ;                                                  \
55         popl    %eax ;                                                  \
56         iret
57
58 /*
59  * 
60  */
61 #define PUSH_FRAME                                                      \
62         pushl   $0 ;            /* dummy error code */                  \
63         pushl   $0 ;            /* dummy trap type */                   \
64         pushal ;                                                        \
65         pushl   %ds ;           /* save data and extra segments ... */  \
66         pushl   %es ;                                                   \
67         pushl   %fs
68
69 #define POP_FRAME                                                       \
70         popl    %fs ;                                                   \
71         popl    %es ;                                                   \
72         popl    %ds ;                                                   \
73         popal ;                                                         \
74         addl    $4+4,%esp
75
76 #define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
77 #define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
78         
79 #define MASK_IRQ(irq_num)                                               \
80         IMASK_LOCK ;                            /* into critical reg */ \
81         testl   $IRQ_BIT(irq_num), _apic_imen ;                         \
82         jne     7f ;                    /* masked, don't mask */        \
83         orl     $IRQ_BIT(irq_num), _apic_imen ; /* set the mask bit */  \
84         movl    IOAPICADDR(irq_num), %ecx ;     /* ioapic addr */       \
85         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
86         movl    %eax, (%ecx) ;                  /* write the index */   \
87         movl    IOAPIC_WINDOW(%ecx), %eax ;     /* current value */     \
88         orl     $IOART_INTMASK, %eax ;          /* set the mask */      \
89         movl    %eax, IOAPIC_WINDOW(%ecx) ;     /* new value */         \
90 7: ;                                            /* already masked */    \
91         IMASK_UNLOCK
92 /*
93  * Test to see whether we are handling an edge or level triggered INT.
94  *  Level-triggered INTs must still be masked as we don't clear the source,
95  *  and the EOI cycle would cause redundant INTs to occur.
96  */
97 #define MASK_LEVEL_IRQ(irq_num)                                         \
98         testl   $IRQ_BIT(irq_num), _apic_pin_trigger ;                  \
99         jz      9f ;                            /* edge, don't mask */  \
100         MASK_IRQ(irq_num) ;                                             \
101 9:
102
103
104 #ifdef APIC_INTR_REORDER
105 #define EOI_IRQ(irq_num)                                                \
106         movl    _apic_isrbit_location + 8 * (irq_num), %eax ;           \
107         movl    (%eax), %eax ;                                          \
108         testl   _apic_isrbit_location + 4 + 8 * (irq_num), %eax ;       \
109         jz      9f ;                            /* not active */        \
110         movl    $0, lapic_eoi ;                                         \
111         APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ;        \
112 9:
113
114 #else
115 #define EOI_IRQ(irq_num)                                                \
116         testl   $IRQ_BIT(irq_num), lapic_isr1;                          \
117         jz      9f      ;                       /* not active */        \
118         movl    $0, lapic_eoi;                                          \
119         APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ;        \
120 9:
121 #endif
122         
123         
124 /*
125  * Test to see if the source is currntly masked, clear if so.
126  */
127 #define UNMASK_IRQ(irq_num)                                     \
128         IMASK_LOCK ;                            /* into critical reg */ \
129         testl   $IRQ_BIT(irq_num), _apic_imen ;                         \
130         je      7f ;                    /* bit clear, not masked */     \
131         andl    $~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */    \
132         movl    IOAPICADDR(irq_num),%ecx ;      /* ioapic addr */       \
133         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
134         movl    %eax,(%ecx) ;                   /* write the index */   \
135         movl    IOAPIC_WINDOW(%ecx),%eax ;      /* current value */     \
136         andl    $~IOART_INTMASK,%eax ;          /* clear the mask */    \
137         movl    %eax,IOAPIC_WINDOW(%ecx) ;      /* new value */         \
138 7: ;                                                                    \
139         IMASK_UNLOCK
140
141 #ifdef APIC_INTR_DIAGNOSTIC
142 #ifdef APIC_INTR_DIAGNOSTIC_IRQ
143 log_intr_event:
144         pushf
145         cli
146         pushl   $CNAME(apic_itrace_debuglock)
147         call    CNAME(s_lock_np)
148         addl    $4, %esp
149         movl    CNAME(apic_itrace_debugbuffer_idx), %ecx
150         andl    $32767, %ecx
151         movl    _cpuid, %eax
152         shll    $8,     %eax
153         orl     8(%esp), %eax
154         movw    %ax,    CNAME(apic_itrace_debugbuffer)(,%ecx,2)
155         incl    %ecx
156         andl    $32767, %ecx
157         movl    %ecx,   CNAME(apic_itrace_debugbuffer_idx)
158         pushl   $CNAME(apic_itrace_debuglock)
159         call    CNAME(s_unlock_np)
160         addl    $4, %esp
161         popf
162         ret
163         
164
165 #define APIC_ITRACE(name, irq_num, id)                                  \
166         lock ;                                  /* MP-safe */           \
167         incl    CNAME(name) + (irq_num) * 4 ;                           \
168         pushl   %eax ;                                                  \
169         pushl   %ecx ;                                                  \
170         pushl   %edx ;                                                  \
171         movl    $(irq_num), %eax ;                                      \
172         cmpl    $APIC_INTR_DIAGNOSTIC_IRQ, %eax ;                       \
173         jne     7f ;                                                    \
174         pushl   $id ;                                                   \
175         call    log_intr_event ;                                        \
176         addl    $4, %esp ;                                              \
177 7: ;                                                                    \
178         popl    %edx ;                                                  \
179         popl    %ecx ;                                                  \
180         popl    %eax
181 #else
182 #define APIC_ITRACE(name, irq_num, id)                                  \
183         lock ;                                  /* MP-safe */           \
184         incl    CNAME(name) + (irq_num) * 4
185 #endif
186
187 #define APIC_ITRACE_ENTER 1
188 #define APIC_ITRACE_EOI 2
189 #define APIC_ITRACE_TRYISRLOCK 3
190 #define APIC_ITRACE_GOTISRLOCK 4
191 #define APIC_ITRACE_ENTER2 5
192 #define APIC_ITRACE_LEAVE 6
193 #define APIC_ITRACE_UNMASK 7
194 #define APIC_ITRACE_ACTIVE 8
195 #define APIC_ITRACE_MASKED 9
196 #define APIC_ITRACE_NOISRLOCK 10
197 #define APIC_ITRACE_MASKED2 11
198 #define APIC_ITRACE_SPLZ 12
199 #define APIC_ITRACE_DORETI 13   
200         
201 #else   
202 #define APIC_ITRACE(name, irq_num, id)
203 #endif
204                 
205 #define INTR(irq_num, vec_name, maybe_extra_ipending)                   \
206         .text ;                                                         \
207         SUPERALIGN_TEXT ;                                               \
208 /* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */  \
209 IDTVEC(vec_name) ;                                                      \
210         PUSH_FRAME ;                                                    \
211         movl    $KDSEL, %eax ;  /* reload with kernel's data segment */ \
212         mov     %ax, %ds ;                                              \
213         mov     %ax, %es ;                                              \
214         movl    $KPSEL, %eax ;                                          \
215         mov     %ax, %fs ;                                              \
216 ;                                                                       \
217         maybe_extra_ipending ;                                          \
218 ;                                                                       \
219         APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ;    \
220         lock ;                                  /* MP-safe */           \
221         btsl    $(irq_num), iactive ;           /* lazy masking */      \
222         jc      1f ;                            /* already active */    \
223 ;                                                                       \
224         MASK_LEVEL_IRQ(irq_num) ;                                       \
225         EOI_IRQ(irq_num) ;                                              \
226 0: ;                                                                    \
227         APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
228         MP_TRYLOCK ;            /* XXX this is going away... */         \
229         testl   %eax, %eax ;                    /* did we get it? */    \
230         jz      3f ;                            /* no */                \
231 ;                                                                       \
232         APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
233         movl    _curthread,%ebx ;                                       \
234         testl   $IRQ_BIT(irq_num), TD_MACH+MTD_CPL(%eax) ;              \
235         jne     2f ;                            /* this INT masked */   \
236         cmpl    $TDPRI_CRIT,TD_PRI(%ebx) ;                              \
237         jge     2f ;                            /* in critical sec */   \
238 ;                                                                       \
239         incb    _intr_nesting_level ;                                   \
240 ;                                                                       \
241   /* entry point used by doreti_unpend for HWIs. */                     \
242 __CONCAT(Xresume,irq_num): ;                                            \
243         FAKE_MCOUNT(13*4(%esp)) ;               /* XXX avoid dbl cnt */ \
244         lock ;  incl    _cnt+V_INTR ;           /* tally interrupts */  \
245         movl    _intr_countp + (irq_num) * 4, %eax ;                    \
246         lock ;  incl    (%eax) ;                                        \
247 ;                                                                       \
248         movl    _curthread, %ebx ;                                      \
249         movl    TD_MACH+MTD_CPL(%ebx), %eax ;                           \
250         pushl   %eax ;   /* cpl restored by doreti */                   \
251         orl     _intr_mask + (irq_num) * 4, %eax ;                      \
252         movl    %eax, TD_MACH+MTD_CPL(%ebx) ;                           \
253         lock ;                                                          \
254         andl    $~IRQ_BIT(irq_num), _ipending ;                         \
255 ;                                                                       \
256         pushl   _intr_unit + (irq_num) * 4 ;                            \
257         APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ;  \
258         sti ;                                                           \
259         call    *_intr_handler + (irq_num) * 4 ;                        \
260         cli ;                                                           \
261         APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ;    \
262 ;                                                                       \
263         lock ;  andl    $~IRQ_BIT(irq_num), iactive ;                   \
264         UNMASK_IRQ(irq_num) ;                                           \
265         APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ;  \
266         sti ;                           /* doreti repeats cli/sti */    \
267         MEXITCOUNT ;                                                    \
268         jmp     _doreti ;                                               \
269 ;                                                                       \
270         ALIGN_TEXT ;                                                    \
271 1: ;                                            /* active  */           \
272         APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ;  \
273         MASK_IRQ(irq_num) ;                                             \
274         EOI_IRQ(irq_num) ;                                              \
275         lock ;                                                          \
276         orl     $IRQ_BIT(irq_num), _ipending ;                          \
277         movl    $TDPRI_CRIT,_reqpri ;                                   \
278         lock ;                                                          \
279         btsl    $(irq_num), iactive ;           /* still active */      \
280         jnc     0b ;                            /* retry */             \
281         POP_FRAME ;                                                     \
282         iret ;          /* XXX:  iactive bit might be 0 now */          \
283         ALIGN_TEXT ;                                                    \
284 2: ;                            /* masked by cpl, leave iactive set */  \
285         APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ;  \
286         lock ;                                                          \
287         orl     $IRQ_BIT(irq_num), _ipending ;                          \
288         movl    $TDPRI_CRIT,_reqpri ;                                   \
289         MP_RELLOCK ;                                                    \
290         POP_FRAME ;                                                     \
291         iret ;                                                          \
292         ALIGN_TEXT ;                                                    \
293 3: ;                    /* other cpu has isr lock */                    \
294         APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
295         lock ;                                                          \
296         orl     $IRQ_BIT(irq_num), _ipending ;                          \
297         movl    $TDPRI_CRIT,_reqpri ;                                   \
298         testl   $IRQ_BIT(irq_num), TD_MACH+MTD_CPL(%ebx) ;              \
299         jne     4f ;                            /* this INT masked */   \
300         call    forward_irq ;    /* forward irq to lock holder */       \
301         POP_FRAME ;                             /* and return */        \
302         iret ;                                                          \
303         ALIGN_TEXT ;                                                    \
304 4: ;                                            /* blocked */           \
305         APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
306         POP_FRAME ;                             /* and return */        \
307         iret
308
309 /*
310  * Handle "spurious INTerrupts".
311  * Notes:
312  *  This is different than the "spurious INTerrupt" generated by an
313  *   8259 PIC for missing INTs.  See the APIC documentation for details.
314  *  This routine should NOT do an 'EOI' cycle.
315  */
316         .text
317         SUPERALIGN_TEXT
318         .globl _Xspuriousint
319 _Xspuriousint:
320
321         /* No EOI cycle used here */
322
323         iret
324
325
326 /*
327  * Handle TLB shootdowns.
328  */
329         .text
330         SUPERALIGN_TEXT
331         .globl  _Xinvltlb
332 _Xinvltlb:
333         pushl   %eax
334
335 #ifdef COUNT_XINVLTLB_HITS
336         pushl   %fs
337         movl    $KPSEL, %eax
338         mov     %ax, %fs
339         movl    _cpuid, %eax
340         popl    %fs
341         ss
342         incl    _xhits(,%eax,4)
343 #endif /* COUNT_XINVLTLB_HITS */
344
345         movl    %cr3, %eax              /* invalidate the TLB */
346         movl    %eax, %cr3
347
348         ss                              /* stack segment, avoid %ds load */
349         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
350
351         popl    %eax
352         iret
353
354
355 #ifdef BETTER_CLOCK
356
357 /*
358  * Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU,
359  *
360  *  - Stores current cpu state in checkstate_cpustate[cpuid]
361  *      0 == user, 1 == sys, 2 == intr
362  *  - Stores current process in checkstate_curproc[cpuid]
363  *
364  *  - Signals its receipt by setting bit cpuid in checkstate_probed_cpus.
365  *
366  * stack: 0->ds, 4->fs, 8->ebx, 12->eax, 16->eip, 20->cs, 24->eflags
367  */
368
369         .text
370         SUPERALIGN_TEXT
371         .globl _Xcpucheckstate
372         .globl _checkstate_cpustate
373         .globl _checkstate_curproc
374         .globl _checkstate_pc
375 _Xcpucheckstate:
376         pushl   %eax
377         pushl   %ebx            
378         pushl   %ds                     /* save current data segment */
379         pushl   %fs
380
381         movl    $KDSEL, %eax
382         mov     %ax, %ds                /* use KERNEL data segment */
383         movl    $KPSEL, %eax
384         mov     %ax, %fs
385
386         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
387
388         movl    $0, %ebx                
389         movl    20(%esp), %eax  
390         andl    $3, %eax
391         cmpl    $3, %eax
392         je      1f
393         testl   $PSL_VM, 24(%esp)
394         jne     1f
395         incl    %ebx                    /* system or interrupt */
396 1:      
397         movl    _cpuid, %eax
398         movl    %ebx, _checkstate_cpustate(,%eax,4)
399         movl    _curthread, %ebx
400         movl    TD_PROC(%ebx),%ebx
401         movl    %ebx, _checkstate_curproc(,%eax,4)
402         movl    16(%esp), %ebx
403         movl    %ebx, _checkstate_pc(,%eax,4)
404
405         lock                            /* checkstate_probed_cpus |= (1<<id) */
406         btsl    %eax, _checkstate_probed_cpus
407
408         popl    %fs
409         popl    %ds                     /* restore previous data segment */
410         popl    %ebx
411         popl    %eax
412         iret
413
414 #endif /* BETTER_CLOCK */
415
416 /*
417  * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
418  *
419  *  - Signals its receipt by clearing bit cpuid in checkstate_need_ast.
420  *
421  *  - We need a better method of triggering asts on other cpus.
422  */
423
424         .text
425         SUPERALIGN_TEXT
426         .globl _Xcpuast
427 _Xcpuast:
428         PUSH_FRAME
429         movl    $KDSEL, %eax
430         mov     %ax, %ds                /* use KERNEL data segment */
431         mov     %ax, %es
432         movl    $KPSEL, %eax
433         mov     %ax, %fs
434
435         movl    _cpuid, %eax
436         lock                            /* checkstate_need_ast &= ~(1<<id) */
437         btrl    %eax, _checkstate_need_ast
438         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
439
440         lock
441         btsl    %eax, _checkstate_pending_ast
442         jc      1f
443
444         FAKE_MCOUNT(13*4(%esp))
445
446         /* 
447          * Giant locks do not come cheap.
448          * A lot of cycles are going to be wasted here.
449          */
450         call    _get_mplock
451
452         movl    _curthread, %eax
453         pushl   TD_MACH+MTD_CPL(%eax)           /* cpl restored by doreti */
454
455         orl     $AST_PENDING, _astpending       /* XXX */
456         incb    _intr_nesting_level
457         sti
458         
459         pushl   $0
460         
461         movl    _cpuid, %eax
462         lock    
463         btrl    %eax, _checkstate_pending_ast
464         lock    
465         btrl    %eax, CNAME(resched_cpus)
466         jnc     2f
467         orl     $AST_PENDING+AST_RESCHED,_astpending
468         lock
469         incl    CNAME(want_resched_cnt)
470 2:              
471         lock
472         incl    CNAME(cpuast_cnt)
473         MEXITCOUNT
474         jmp     _doreti
475 1:
476         /* We are already in the process of delivering an ast for this CPU */
477         POP_FRAME
478         iret                    
479
480
481 /*
482  *       Executed by a CPU when it receives an XFORWARD_IRQ IPI.
483  */
484
485         .text
486         SUPERALIGN_TEXT
487         .globl _Xforward_irq
488 _Xforward_irq:
489         PUSH_FRAME
490         movl    $KDSEL, %eax
491         mov     %ax, %ds                /* use KERNEL data segment */
492         mov     %ax, %es
493         movl    $KPSEL, %eax
494         mov     %ax, %fs
495
496         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
497
498         FAKE_MCOUNT(13*4(%esp))
499
500         MP_TRYLOCK
501         testl   %eax,%eax               /* Did we get the lock ? */
502         jz  1f                          /* No */
503
504         lock
505         incl    CNAME(forward_irq_hitcnt)
506         cmpb    $4, _intr_nesting_level
507         jae     2f
508         
509         movl    _curthread, %eax
510         pushl   TD_MACH+MTD_CPL(%eax)           /* cpl restored by doreti */
511
512         incb    _intr_nesting_level
513         sti
514         
515         pushl   $0
516
517         MEXITCOUNT
518         jmp     _doreti                 /* Handle forwarded interrupt */
519 1:
520         lock
521         incl    CNAME(forward_irq_misscnt)
522         call    forward_irq     /* Oops, we've lost the isr lock */
523         MEXITCOUNT
524         POP_FRAME
525         iret
526 2:
527         lock
528         incl    CNAME(forward_irq_toodeepcnt)
529 3:      
530         MP_RELLOCK
531         MEXITCOUNT
532         POP_FRAME
533         iret
534
535 /*
536  * 
537  */
538 forward_irq:
539         MCOUNT
540         cmpl    $0,_invltlb_ok
541         jz      4f
542
543         cmpl    $0, CNAME(forward_irq_enabled)
544         jz      4f
545
546         movl    _mp_lock,%eax
547         cmpl    $FREE_LOCK,%eax
548         jne     1f
549         movl    $0, %eax                /* Pick CPU #0 if noone has lock */
550 1:
551         shrl    $24,%eax
552         movl    _cpu_num_to_apic_id(,%eax,4),%ecx
553         shll    $24,%ecx
554         movl    lapic_icr_hi, %eax
555         andl    $~APIC_ID_MASK, %eax
556         orl     %ecx, %eax
557         movl    %eax, lapic_icr_hi
558
559 2:
560         movl    lapic_icr_lo, %eax
561         andl    $APIC_DELSTAT_MASK,%eax
562         jnz     2b
563         movl    lapic_icr_lo, %eax
564         andl    $APIC_RESV2_MASK, %eax
565         orl     $(APIC_DEST_DESTFLD|APIC_DELMODE_FIXED|XFORWARD_IRQ_OFFSET), %eax
566         movl    %eax, lapic_icr_lo
567 3:
568         movl    lapic_icr_lo, %eax
569         andl    $APIC_DELSTAT_MASK,%eax
570         jnz     3b
571 4:              
572         ret
573         
574 /*
575  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
576  *
577  *  - Signals its receipt.
578  *  - Waits for permission to restart.
579  *  - Signals its restart.
580  */
581
582         .text
583         SUPERALIGN_TEXT
584         .globl _Xcpustop
585 _Xcpustop:
586         pushl   %ebp
587         movl    %esp, %ebp
588         pushl   %eax
589         pushl   %ecx
590         pushl   %edx
591         pushl   %ds                     /* save current data segment */
592         pushl   %fs
593
594         movl    $KDSEL, %eax
595         mov     %ax, %ds                /* use KERNEL data segment */
596         movl    $KPSEL, %eax
597         mov     %ax, %fs
598
599         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
600
601         movl    _cpuid, %eax
602         imull   $PCB_SIZE, %eax
603         leal    CNAME(stoppcbs)(%eax), %eax
604         pushl   %eax
605         call    CNAME(savectx)          /* Save process context */
606         addl    $4, %esp
607         
608                 
609         movl    _cpuid, %eax
610
611         lock
612         btsl    %eax, _stopped_cpus     /* stopped_cpus |= (1<<id) */
613 1:
614         btl     %eax, _started_cpus     /* while (!(started_cpus & (1<<id))) */
615         jnc     1b
616
617         lock
618         btrl    %eax, _started_cpus     /* started_cpus &= ~(1<<id) */
619         lock
620         btrl    %eax, _stopped_cpus     /* stopped_cpus &= ~(1<<id) */
621
622         test    %eax, %eax
623         jnz     2f
624
625         movl    CNAME(cpustop_restartfunc), %eax
626         test    %eax, %eax
627         jz      2f
628         movl    $0, CNAME(cpustop_restartfunc)  /* One-shot */
629
630         call    *%eax
631 2:
632         popl    %fs
633         popl    %ds                     /* restore previous data segment */
634         popl    %edx
635         popl    %ecx
636         popl    %eax
637         movl    %ebp, %esp
638         popl    %ebp
639         iret
640
641
642 MCOUNT_LABEL(bintr)
643         FAST_INTR(0,fastintr0)
644         FAST_INTR(1,fastintr1)
645         FAST_INTR(2,fastintr2)
646         FAST_INTR(3,fastintr3)
647         FAST_INTR(4,fastintr4)
648         FAST_INTR(5,fastintr5)
649         FAST_INTR(6,fastintr6)
650         FAST_INTR(7,fastintr7)
651         FAST_INTR(8,fastintr8)
652         FAST_INTR(9,fastintr9)
653         FAST_INTR(10,fastintr10)
654         FAST_INTR(11,fastintr11)
655         FAST_INTR(12,fastintr12)
656         FAST_INTR(13,fastintr13)
657         FAST_INTR(14,fastintr14)
658         FAST_INTR(15,fastintr15)
659         FAST_INTR(16,fastintr16)
660         FAST_INTR(17,fastintr17)
661         FAST_INTR(18,fastintr18)
662         FAST_INTR(19,fastintr19)
663         FAST_INTR(20,fastintr20)
664         FAST_INTR(21,fastintr21)
665         FAST_INTR(22,fastintr22)
666         FAST_INTR(23,fastintr23)
667         
668 #define CLKINTR_PENDING                                                 \
669         pushl $clock_lock ;                                             \
670         call s_lock ;                                                   \
671         movl $1,CNAME(clkintr_pending) ;                                \
672         call s_unlock ;                                                 \
673         addl $4, %esp
674
675         INTR(0,intr0, CLKINTR_PENDING)
676         INTR(1,intr1,)
677         INTR(2,intr2,)
678         INTR(3,intr3,)
679         INTR(4,intr4,)
680         INTR(5,intr5,)
681         INTR(6,intr6,)
682         INTR(7,intr7,)
683         INTR(8,intr8,)
684         INTR(9,intr9,)
685         INTR(10,intr10,)
686         INTR(11,intr11,)
687         INTR(12,intr12,)
688         INTR(13,intr13,)
689         INTR(14,intr14,)
690         INTR(15,intr15,)
691         INTR(16,intr16,)
692         INTR(17,intr17,)
693         INTR(18,intr18,)
694         INTR(19,intr19,)
695         INTR(20,intr20,)
696         INTR(21,intr21,)
697         INTR(22,intr22,)
698         INTR(23,intr23,)
699 MCOUNT_LABEL(eintr)
700
701 /*
702  * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
703  *
704  * - Calls the generic rendezvous action function.
705  */
706         .text
707         SUPERALIGN_TEXT
708         .globl  _Xrendezvous
709 _Xrendezvous:
710         PUSH_FRAME
711         movl    $KDSEL, %eax
712         mov     %ax, %ds                /* use KERNEL data segment */
713         mov     %ax, %es
714         movl    $KPSEL, %eax
715         mov     %ax, %fs
716
717         call    _smp_rendezvous_action
718
719         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
720         POP_FRAME
721         iret
722         
723         
724         .data
725 /*
726  * Addresses of interrupt handlers.
727  *  XresumeNN: Resumption addresses for HWIs.
728  */
729         .globl _ihandlers
730 _ihandlers:
731 /*
732  * used by:
733  *  ipl.s:      doreti_unpend
734  */
735         .long   Xresume0,  Xresume1,  Xresume2,  Xresume3 
736         .long   Xresume4,  Xresume5,  Xresume6,  Xresume7
737         .long   Xresume8,  Xresume9,  Xresume10, Xresume11
738         .long   Xresume12, Xresume13, Xresume14, Xresume15 
739         .long   Xresume16, Xresume17, Xresume18, Xresume19
740         .long   Xresume20, Xresume21, Xresume22, Xresume23
741 /*
742  * used by:
743  *  ipl.s:      doreti_unpend
744  *  apic_ipl.s: splz_unpend
745  */
746         .long   _swi_null, swi_net, _swi_null, _swi_null
747         .long   _swi_vm, _swi_null, _softclock
748
749 imasks:                         /* masks for interrupt handlers */
750         .space  NHWI*4          /* padding; HWI masks are elsewhere */
751
752         .long   SWI_TTY_MASK, SWI_NET_MASK, SWI_CAMNET_MASK, SWI_CAMBIO_MASK
753         .long   SWI_VM_MASK, SWI_TQ_MASK, SWI_CLOCK_MASK
754
755 /* active flag for lazy masking */
756 iactive:
757         .long   0
758
759 #ifdef COUNT_XINVLTLB_HITS
760         .globl  _xhits
761 _xhits:
762         .space  (NCPU * 4), 0
763 #endif /* COUNT_XINVLTLB_HITS */
764
765 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
766         .globl _stopped_cpus, _started_cpus
767 _stopped_cpus:
768         .long   0
769 _started_cpus:
770         .long   0
771
772 #ifdef BETTER_CLOCK
773         .globl _checkstate_probed_cpus
774 _checkstate_probed_cpus:
775         .long   0       
776 #endif /* BETTER_CLOCK */
777         .globl _checkstate_need_ast
778 _checkstate_need_ast:
779         .long   0
780 _checkstate_pending_ast:
781         .long   0
782         .globl CNAME(forward_irq_misscnt)
783         .globl CNAME(forward_irq_toodeepcnt)
784         .globl CNAME(forward_irq_hitcnt)
785         .globl CNAME(resched_cpus)
786         .globl CNAME(want_resched_cnt)
787         .globl CNAME(cpuast_cnt)
788         .globl CNAME(cpustop_restartfunc)
789 CNAME(forward_irq_misscnt):     
790         .long 0
791 CNAME(forward_irq_hitcnt):      
792         .long 0
793 CNAME(forward_irq_toodeepcnt):
794         .long 0
795 CNAME(resched_cpus):
796         .long 0
797 CNAME(want_resched_cnt):
798         .long 0
799 CNAME(cpuast_cnt):
800         .long 0
801 CNAME(cpustop_restartfunc):
802         .long 0
803                 
804
805
806         .globl  _apic_pin_trigger
807 _apic_pin_trigger:
808         .long   0
809
810         .text