sys/platform/pc32/apic/apic_vector.s

   1 /*
   2  *      from: vector.s, 386BSD 0.1 unknown origin
   3  * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
   4  * $DragonFly: src/sys/platform/pc32/apic/apic_vector.s,v 1.34 2006/11/07 18:50:06 dillon Exp $
   5  */
   6
   7 #include "use_npx.h"
   8 #include "opt_auto_eoi.h"
   9
  10 #include <machine/asmacros.h>
  11 #include <machine/lock.h>
  12 #include <machine/psl.h>
  13 #include <machine/trap.h>
  14
  15 #include <machine_base/icu/icu.h>
  16 #include <bus/isa/i386/isa.h>
  17
  18 #include "assym.s"
  19
  20 #include "apicreg.h"
  21 #include "apic_ipl.h"
  22 #include <machine/smp.h>
  23 #include <machine_base/isa/intr_machdep.h>
  24
  25 /* convert an absolute IRQ# into a bitmask */
  26 #define IRQ_LBIT(irq_num)       (1 << (irq_num))
  27
  28 /* make an index into the IO APIC from the IRQ# */
  29 #define REDTBL_IDX(irq_num)     (0x10 + ((irq_num) * 2))
  30
  31 #ifdef SMP
  32 #define MPLOCKED     lock ;
  33 #else
  34 #define MPLOCKED
  35 #endif
  36
  37 /*
  38  * Push an interrupt frame in a format acceptable to doreti, reload
  39  * the segment registers for the kernel.
  40  */
  41 #define PUSH_FRAME                                                      \
  42         pushl   $0 ;            /* dummy error code */                  \
  43         pushl   $0 ;            /* dummy trap type */                   \
  44         pushal ;                                                        \
  45         pushl   %ds ;           /* save data and extra segments ... */  \
  46         pushl   %es ;                                                   \
  47         pushl   %fs ;                                                   \
  48         mov     $KDSEL,%ax ;                                            \
  49         mov     %ax,%ds ;                                               \
  50         mov     %ax,%es ;                                               \
  51         mov     $KPSEL,%ax ;                                            \
  52         mov     %ax,%fs ;                                               \
  53
  54 #define PUSH_DUMMY                                                      \
  55         pushfl ;                /* phys int frame / flags */            \
  56         pushl %cs ;             /* phys int frame / cs */               \
  57         pushl   12(%esp) ;      /* original caller eip */               \
  58         pushl   $0 ;            /* dummy error code */                  \
  59         pushl   $0 ;            /* dummy trap type */                   \
  60         subl    $12*4,%esp ;    /* pushal + 3 seg regs (dummy) + CPL */ \
  61
  62 /*
  63  * Warning: POP_FRAME can only be used if there is no chance of a
  64  * segment register being changed (e.g. by procfs), which is why syscalls
  65  * have to use doreti.
  66  */
  67 #define POP_FRAME                                                       \
  68         popl    %fs ;                                                   \
  69         popl    %es ;                                                   \
  70         popl    %ds ;                                                   \
  71         popal ;                                                         \
  72         addl    $2*4,%esp ;     /* dummy trap & error codes */          \
  73
  74 #define POP_DUMMY                                                       \
  75         addl    $17*4,%esp ;                                            \
  76
  77 #define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
  78 #define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
  79
  80 #define MASK_IRQ(irq_num)                                               \
  81         APIC_IMASK_LOCK ;                       /* into critical reg */ \
  82         testl   $IRQ_LBIT(irq_num), apic_imen ;                         \
  83         jne     7f ;                    /* masked, don't mask */        \
  84         orl     $IRQ_LBIT(irq_num), apic_imen ; /* set the mask bit */  \
  85         movl    IOAPICADDR(irq_num), %ecx ;     /* ioapic addr */       \
  86         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
  87         movl    %eax, (%ecx) ;                  /* write the index */   \
  88         movl    IOAPIC_WINDOW(%ecx), %eax ;     /* current value */     \
  89         orl     $IOART_INTMASK, %eax ;          /* set the mask */      \
  90         movl    %eax, IOAPIC_WINDOW(%ecx) ;     /* new value */         \
  91 7: ;                                            /* already masked */    \
  92         APIC_IMASK_UNLOCK ;                                             \
  93
  94 /*
  95  * Test to see whether we are handling an edge or level triggered INT.
  96  *  Level-triggered INTs must still be masked as we don't clear the source,
  97  *  and the EOI cycle would cause redundant INTs to occur.
  98  */
  99 #define MASK_LEVEL_IRQ(irq_num)                                         \
 100         testl   $IRQ_LBIT(irq_num), apic_pin_trigger ;                  \
 101         jz      9f ;                            /* edge, don't mask */  \
 102         MASK_IRQ(irq_num) ;                                             \
 103 9: ;                                                                    \
 104
 105 /*
 106  * Test to see if the source is currntly masked, clear if so.
 107  */
 108 #define UNMASK_IRQ(irq_num)                                     \
 109         cmpl    $0,%eax ;                                               \
 110         jnz     8f ;                                                    \
 111         APIC_IMASK_LOCK ;                       /* into critical reg */ \
 112         testl   $IRQ_LBIT(irq_num), apic_imen ;                         \
 113         je      7f ;                    /* bit clear, not masked */     \
 114         andl    $~IRQ_LBIT(irq_num), apic_imen ;/* clear mask bit */    \
 115         movl    IOAPICADDR(irq_num),%ecx ;      /* ioapic addr */       \
 116         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
 117         movl    %eax,(%ecx) ;                   /* write the index */   \
 118         movl    IOAPIC_WINDOW(%ecx),%eax ;      /* current value */     \
 119         andl    $~IOART_INTMASK,%eax ;          /* clear the mask */    \
 120         movl    %eax,IOAPIC_WINDOW(%ecx) ;      /* new value */         \
 121 7: ;                                                                    \
 122         APIC_IMASK_UNLOCK ;                                             \
 123 8: ;                                                                    \
 124
 125 #ifdef APIC_IO
 126
 127 /*
 128  * Fast interrupt call handlers run in the following sequence:
 129  *
 130  *      - Push the trap frame required by doreti
 131  *      - Mask the interrupt and reenable its source
 132  *      - If we cannot take the interrupt set its fpending bit and
 133  *        doreti.  Note that we cannot mess with mp_lock at all
 134  *        if we entered from a critical section!
 135  *      - If we can take the interrupt clear its fpending bit,
 136  *        call the handler, then unmask and doreti.
 137  *
 138  * YYY can cache gd base opitner instead of using hidden %fs prefixes.
 139  */
 140
 141 #define FAST_INTR(irq_num, vec_name)                                    \
 142         .text ;                                                         \
 143         SUPERALIGN_TEXT ;                                               \
 144 IDTVEC(vec_name) ;                                                      \
 145         PUSH_FRAME ;                                                    \
 146         FAKE_MCOUNT(13*4(%esp)) ;                                       \
 147         MASK_LEVEL_IRQ(irq_num) ;                                       \
 148         movl    $0, lapic_eoi ;                                         \
 149         movl    PCPU(curthread),%ebx ;                                  \
 150         movl    $0,%eax ;       /* CURRENT CPL IN FRAME (REMOVED) */    \
 151         pushl   %eax ;                                                  \
 152         cmpl    $TDPRI_CRIT,TD_PRI(%ebx) ;                              \
 153         jl      2f ;                                                    \
 154 1: ;                                                                    \
 155         /* in critical section, make interrupt pending */               \
 156         /* set the pending bit and return, leave interrupt masked */    \
 157         orl     $IRQ_LBIT(irq_num),PCPU(fpending) ;                     \
 158         orl     $RQF_INTPEND,PCPU(reqflags) ;                           \
 159         jmp     5f ;                                                    \
 160 2: ;                                                                    \
 161         /* clear pending bit, run handler */                            \
 162         andl    $~IRQ_LBIT(irq_num),PCPU(fpending) ;                    \
 163         pushl   $irq_num ;                                              \
 164         call    ithread_fast_handler ;   /* returns 0 to unmask */      \
 165         addl    $4, %esp ;                                              \
 166         UNMASK_IRQ(irq_num) ;                                           \
 167 5: ;                                                                    \
 168         MEXITCOUNT ;                                                    \
 169         jmp     doreti ;                                                \
 170
 171 /*
 172  * Slow interrupt call handlers run in the following sequence:
 173  *
 174  *      - Push the trap frame required by doreti.
 175  *      - Mask the interrupt and reenable its source.
 176  *      - If we cannot take the interrupt set its ipending bit and
 177  *        doreti.  In addition to checking for a critical section
 178  *        and cpl mask we also check to see if the thread is still
 179  *        running.  Note that we cannot mess with mp_lock at all
 180  *        if we entered from a critical section!
 181  *      - If we can take the interrupt clear its ipending bit
 182  *        and schedule the thread.  Leave interrupts masked and doreti.
 183  *
 184  *      Note that calls to sched_ithd() are made with interrupts enabled
 185  *      and outside a critical section.  YYY sched_ithd may preempt us
 186  *      synchronously (fix interrupt stacking).
 187  *
 188  *      YYY can cache gd base pointer instead of using hidden %fs
 189  *      prefixes.
 190  */
 191
 192 #define SLOW_INTR(irq_num, vec_name, maybe_extra_ipending)              \
 193         .text ;                                                         \
 194         SUPERALIGN_TEXT ;                                               \
 195 IDTVEC(vec_name) ;                                                      \
 196         PUSH_FRAME ;                                                    \
 197         maybe_extra_ipending ;                                          \
 198 ;                                                                       \
 199         MASK_LEVEL_IRQ(irq_num) ;                                       \
 200         incl    PCPU(cnt) + V_INTR ;                                    \
 201         movl    $0, lapic_eoi ;                                         \
 202         movl    PCPU(curthread),%ebx ;                                  \
 203         movl    $0,%eax ;       /* CURRENT CPL IN FRAME (REMOVED) */    \
 204         pushl   %eax ;          /* cpl do restore */                    \
 205         cmpl    $TDPRI_CRIT,TD_PRI(%ebx) ;                              \
 206         jl      2f ;                                                    \
 207 1: ;                                                                    \
 208         /* set the pending bit and return, leave the interrupt masked */ \
 209         orl     $IRQ_LBIT(irq_num), PCPU(ipending) ;                    \
 210         orl     $RQF_INTPEND,PCPU(reqflags) ;                           \
 211         jmp     5f ;                                                    \
 212 2: ;                                                                    \
 213         /* set running bit, clear pending bit, run handler */           \
 214         andl    $~IRQ_LBIT(irq_num), PCPU(ipending) ;                   \
 215         sti ;                                                           \
 216         pushl   $irq_num ;                                              \
 217         call    sched_ithd ;                                            \
 218         addl    $4,%esp ;                                               \
 219 5: ;                                                                    \
 220         MEXITCOUNT ;                                                    \
 221         jmp     doreti ;                                                \
 222
 223 /*
 224  * Wrong interrupt call handlers.  We program these into APIC vectors
 225  * that should otherwise never occur.  For example, we program the SLOW
 226  * vector for irq N with this when we program the FAST vector with the
 227  * real interrupt.
 228  *
 229  * XXX for now all we can do is EOI it.  We can't call do_wrongintr
 230  * (yet) because we could be in a critical section.
 231  */
 232 #define WRONGINTR(irq_num,vec_name)                                     \
 233         .text ;                                                         \
 234         SUPERALIGN_TEXT  ;                                              \
 235 IDTVEC(vec_name) ;                                                      \
 236         PUSH_FRAME ;                                                    \
 237         movl    $0, lapic_eoi ; /* End Of Interrupt to APIC */          \
 238         /*pushl $irq_num ;*/                                            \
 239         /*call  do_wrongintr ;*/                                        \
 240         /*addl  $4,%esp ;*/                                             \
 241         POP_FRAME ;                                                     \
 242         iret  ;                                                         \
 243
 244 #endif
 245
 246 /*
 247  * Handle "spurious INTerrupts".
 248  * Notes:
 249  *  This is different than the "spurious INTerrupt" generated by an
 250  *   8259 PIC for missing INTs.  See the APIC documentation for details.
 251  *  This routine should NOT do an 'EOI' cycle.
 252  */
 253         .text
 254         SUPERALIGN_TEXT
 255         .globl Xspuriousint
 256 Xspuriousint:
 257
 258         /* No EOI cycle used here */
 259
 260         iret
 261
 262
 263 /*
 264  * Handle TLB shootdowns.
 265  */
 266         .text
 267         SUPERALIGN_TEXT
 268         .globl  Xinvltlb
 269 Xinvltlb:
 270         pushl   %eax
 271
 272         movl    %cr3, %eax              /* invalidate the TLB */
 273         movl    %eax, %cr3
 274
 275         ss                              /* stack segment, avoid %ds load */
 276         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
 277
 278         popl    %eax
 279         iret
 280
 281
 282 /*
 283  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
 284  *
 285  *  - Signals its receipt.
 286  *  - Waits for permission to restart.
 287  *  - Processing pending IPIQ events while waiting.
 288  *  - Signals its restart.
 289  */
 290
 291         .text
 292         SUPERALIGN_TEXT
 293         .globl Xcpustop
 294 Xcpustop:
 295         pushl   %ebp
 296         movl    %esp, %ebp
 297         pushl   %eax
 298         pushl   %ecx
 299         pushl   %edx
 300         pushl   %ds                     /* save current data segment */
 301         pushl   %fs
 302
 303         movl    $KDSEL, %eax
 304         mov     %ax, %ds                /* use KERNEL data segment */
 305         movl    $KPSEL, %eax
 306         mov     %ax, %fs
 307
 308         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
 309
 310         movl    PCPU(cpuid), %eax
 311         imull   $PCB_SIZE, %eax
 312         leal    CNAME(stoppcbs)(%eax), %eax
 313         pushl   %eax
 314         call    CNAME(savectx)          /* Save process context */
 315         addl    $4, %esp
 316
 317
 318         movl    PCPU(cpuid), %eax
 319
 320         /*
 321          * Indicate that we have stopped and loop waiting for permission
 322          * to start again.  We must still process IPI events while in a
 323          * stopped state.
 324          */
 325         MPLOCKED
 326         btsl    %eax, stopped_cpus      /* stopped_cpus |= (1<<id) */
 327 1:
 328         andl    $~RQF_IPIQ,PCPU(reqflags)
 329         pushl   %eax
 330         call    lwkt_smp_stopped
 331         popl    %eax
 332         btl     %eax, started_cpus      /* while (!(started_cpus & (1<<id))) */
 333         jnc     1b
 334
 335         MPLOCKED
 336         btrl    %eax, started_cpus      /* started_cpus &= ~(1<<id) */
 337         MPLOCKED
 338         btrl    %eax, stopped_cpus      /* stopped_cpus &= ~(1<<id) */
 339
 340         test    %eax, %eax
 341         jnz     2f
 342
 343         movl    CNAME(cpustop_restartfunc), %eax
 344         test    %eax, %eax
 345         jz      2f
 346         movl    $0, CNAME(cpustop_restartfunc)  /* One-shot */
 347
 348         call    *%eax
 349 2:
 350         popl    %fs
 351         popl    %ds                     /* restore previous data segment */
 352         popl    %edx
 353         popl    %ecx
 354         popl    %eax
 355         movl    %ebp, %esp
 356         popl    %ebp
 357         iret
 358
 359         /*
 360          * For now just have one ipiq IPI, but what we really want is
 361          * to have one for each source cpu to the APICs don't get stalled
 362          * backlogging the requests.
 363          */
 364         .text
 365         SUPERALIGN_TEXT
 366         .globl Xipiq
 367 Xipiq:
 368         PUSH_FRAME
 369         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
 370         FAKE_MCOUNT(13*4(%esp))
 371
 372         movl    PCPU(curthread),%ebx
 373         cmpl    $TDPRI_CRIT,TD_PRI(%ebx)
 374         jge     1f
 375         subl    $8,%esp                 /* make same as interrupt frame */
 376         incl    PCPU(intr_nesting_level)
 377         addl    $TDPRI_CRIT,TD_PRI(%ebx)
 378         call    lwkt_process_ipiq_frame
 379         subl    $TDPRI_CRIT,TD_PRI(%ebx)
 380         decl    PCPU(intr_nesting_level)
 381         addl    $8,%esp
 382         pushl   $0                      /* CPL for frame (REMOVED) */
 383         MEXITCOUNT
 384         jmp     doreti
 385 1:
 386         orl     $RQF_IPIQ,PCPU(reqflags)
 387         MEXITCOUNT
 388         POP_FRAME
 389         iret
 390
 391 #ifdef APIC_IO
 392
 393 MCOUNT_LABEL(bintr)
 394         FAST_INTR(0,apic_fastintr0)
 395         FAST_INTR(1,apic_fastintr1)
 396         FAST_INTR(2,apic_fastintr2)
 397         FAST_INTR(3,apic_fastintr3)
 398         FAST_INTR(4,apic_fastintr4)
 399         FAST_INTR(5,apic_fastintr5)
 400         FAST_INTR(6,apic_fastintr6)
 401         FAST_INTR(7,apic_fastintr7)
 402         FAST_INTR(8,apic_fastintr8)
 403         FAST_INTR(9,apic_fastintr9)
 404         FAST_INTR(10,apic_fastintr10)
 405         FAST_INTR(11,apic_fastintr11)
 406         FAST_INTR(12,apic_fastintr12)
 407         FAST_INTR(13,apic_fastintr13)
 408         FAST_INTR(14,apic_fastintr14)
 409         FAST_INTR(15,apic_fastintr15)
 410         FAST_INTR(16,apic_fastintr16)
 411         FAST_INTR(17,apic_fastintr17)
 412         FAST_INTR(18,apic_fastintr18)
 413         FAST_INTR(19,apic_fastintr19)
 414         FAST_INTR(20,apic_fastintr20)
 415         FAST_INTR(21,apic_fastintr21)
 416         FAST_INTR(22,apic_fastintr22)
 417         FAST_INTR(23,apic_fastintr23)
 418
 419         /* YYY what is this garbage? */
 420
 421         SLOW_INTR(0,apic_slowintr0,)
 422         SLOW_INTR(1,apic_slowintr1,)
 423         SLOW_INTR(2,apic_slowintr2,)
 424         SLOW_INTR(3,apic_slowintr3,)
 425         SLOW_INTR(4,apic_slowintr4,)
 426         SLOW_INTR(5,apic_slowintr5,)
 427         SLOW_INTR(6,apic_slowintr6,)
 428         SLOW_INTR(7,apic_slowintr7,)
 429         SLOW_INTR(8,apic_slowintr8,)
 430         SLOW_INTR(9,apic_slowintr9,)
 431         SLOW_INTR(10,apic_slowintr10,)
 432         SLOW_INTR(11,apic_slowintr11,)
 433         SLOW_INTR(12,apic_slowintr12,)
 434         SLOW_INTR(13,apic_slowintr13,)
 435         SLOW_INTR(14,apic_slowintr14,)
 436         SLOW_INTR(15,apic_slowintr15,)
 437         SLOW_INTR(16,apic_slowintr16,)
 438         SLOW_INTR(17,apic_slowintr17,)
 439         SLOW_INTR(18,apic_slowintr18,)
 440         SLOW_INTR(19,apic_slowintr19,)
 441         SLOW_INTR(20,apic_slowintr20,)
 442         SLOW_INTR(21,apic_slowintr21,)
 443         SLOW_INTR(22,apic_slowintr22,)
 444         SLOW_INTR(23,apic_slowintr23,)
 445
 446         WRONGINTR(0,apic_wrongintr0)
 447         WRONGINTR(1,apic_wrongintr1)
 448         WRONGINTR(2,apic_wrongintr2)
 449         WRONGINTR(3,apic_wrongintr3)
 450         WRONGINTR(4,apic_wrongintr4)
 451         WRONGINTR(5,apic_wrongintr5)
 452         WRONGINTR(6,apic_wrongintr6)
 453         WRONGINTR(7,apic_wrongintr7)
 454         WRONGINTR(8,apic_wrongintr8)
 455         WRONGINTR(9,apic_wrongintr9)
 456         WRONGINTR(10,apic_wrongintr10)
 457         WRONGINTR(11,apic_wrongintr11)
 458         WRONGINTR(12,apic_wrongintr12)
 459         WRONGINTR(13,apic_wrongintr13)
 460         WRONGINTR(14,apic_wrongintr14)
 461         WRONGINTR(15,apic_wrongintr15)
 462         WRONGINTR(16,apic_wrongintr16)
 463         WRONGINTR(17,apic_wrongintr17)
 464         WRONGINTR(18,apic_wrongintr18)
 465         WRONGINTR(19,apic_wrongintr19)
 466         WRONGINTR(20,apic_wrongintr20)
 467         WRONGINTR(21,apic_wrongintr21)
 468         WRONGINTR(22,apic_wrongintr22)
 469         WRONGINTR(23,apic_wrongintr23)
 470 MCOUNT_LABEL(eintr)
 471
 472 #endif
 473
 474         .data
 475
 476 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
 477         .globl stopped_cpus, started_cpus
 478 stopped_cpus:
 479         .long   0
 480 started_cpus:
 481         .long   0
 482
 483         .globl CNAME(cpustop_restartfunc)
 484 CNAME(cpustop_restartfunc):
 485         .long 0
 486
 487         .globl  apic_pin_trigger
 488 apic_pin_trigger:
 489         .long   0
 490
 491         .text
 492