sys/platform/pc64/apic/apic_vector.s

   1 /*
   2  *      from: vector.s, 386BSD 0.1 unknown origin
   3  * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
   4  */
   5
   6 #if 0
   7 #include "opt_auto_eoi.h"
   8 #endif
   9
  10 #include <machine/asmacros.h>
  11 #include <machine/lock.h>
  12 #include <machine/psl.h>
  13 #include <machine/trap.h>
  14 #include <machine/segments.h>
  15
  16 #include <machine_base/icu/icu.h>
  17 #include <bus/isa/isa.h>
  18
  19 #include "assym.s"
  20
  21 #include "apicreg.h"
  22 #include "apic_ipl.h"
  23 #include <machine/smp.h>
  24 #include <machine_base/isa/intr_machdep.h>
  25
  26 /* convert an absolute IRQ# into a bitmask */
  27 #define IRQ_LBIT(irq_num)       (1 << (irq_num))
  28
  29 /* make an index into the IO APIC from the IRQ# */
  30 #define REDTBL_IDX(irq_num)     (0x10 + ((irq_num) * 2))
  31
  32 #ifdef SMP
  33 #define MPLOCKED     lock ;
  34 #else
  35 #define MPLOCKED
  36 #endif
  37
  38 #define APIC_PUSH_FRAME                                                 \
  39         PUSH_FRAME ;            /* 15 regs + space for 5 extras */      \
  40         movq $0,TF_XFLAGS(%rsp) ;                                       \
  41         movq $0,TF_TRAPNO(%rsp) ;                                       \
  42         movq $0,TF_ADDR(%rsp) ;                                         \
  43         movq $0,TF_FLAGS(%rsp) ;                                        \
  44         movq $0,TF_ERR(%rsp) ;                                          \
  45         cld ;                                                           \
  46
  47 /*
  48  * JG stale? Warning: POP_FRAME can only be used if there is no chance of a
  49  * segment register being changed (e.g. by procfs), which is why syscalls
  50  * have to use doreti.
  51  */
  52 #define APIC_POP_FRAME POP_FRAME
  53
  54 #define IOAPICADDR(irq_num) \
  55         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR
  56 #define REDIRIDX(irq_num) \
  57         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX
  58 #define IOAPICFLAGS(irq_num) \
  59         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS
  60
  61 #define MASK_IRQ(irq_num)                                               \
  62         APIC_IMASK_LOCK ;                       /* into critical reg */ \
  63         testl   $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
  64         jne     7f ;                    /* masked, don't mask */        \
  65         orl     $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
  66                                                 /* set the mask bit */  \
  67         movq    IOAPICADDR(irq_num), %rcx ;     /* ioapic addr */       \
  68         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
  69         movl    %eax, (%rcx) ;                  /* write the index */   \
  70         orl     $IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* set the mask */  \
  71 7: ;                                            /* already masked */    \
  72         APIC_IMASK_UNLOCK ;                                             \
  73
  74 /*
  75  * Test to see whether we are handling an edge or level triggered INT.
  76  *  Level-triggered INTs must still be masked as we don't clear the source,
  77  *  and the EOI cycle would cause redundant INTs to occur.
  78  */
  79 #define MASK_LEVEL_IRQ(irq_num)                                         \
  80         testl   $IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ;           \
  81         jz      9f ;                            /* edge, don't mask */  \
  82         MASK_IRQ(irq_num) ;                                             \
  83 9: ;                                                                    \
  84
  85 /*
  86  * Test to see if the source is currntly masked, clear if so.
  87  */
  88 #define UNMASK_IRQ(irq_num)                                     \
  89         cmpl    $0,%eax ;                                               \
  90         jnz     8f ;                                                    \
  91         APIC_IMASK_LOCK ;                       /* into critical reg */ \
  92         testl   $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
  93         je      7f ;                    /* bit clear, not masked */     \
  94         andl    $~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;         \
  95                                                 /* clear mask bit */    \
  96         movq    IOAPICADDR(irq_num),%rcx ;      /* ioapic addr */       \
  97         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
  98         movl    %eax,(%rcx) ;                   /* write the index */   \
  99         andl    $~IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* clear the mask */ \
 100 7: ;                                                                    \
 101         APIC_IMASK_UNLOCK ;                                             \
 102 8: ;                                                                    \
 103
 104 #ifdef SMP /* APIC-IO */
 105
 106 /*
 107  * Fast interrupt call handlers run in the following sequence:
 108  *
 109  *      - Push the trap frame required by doreti
 110  *      - Mask the interrupt and reenable its source
 111  *      - If we cannot take the interrupt set its fpending bit and
 112  *        doreti.
 113  *      - If we can take the interrupt clear its fpending bit,
 114  *        call the handler, then unmask and doreti.
 115  *
 116  * YYY can cache gd base opitner instead of using hidden %fs prefixes.
 117  */
 118
 119 #define FAST_INTR(irq_num, vec_name)                                    \
 120         .text ;                                                         \
 121         SUPERALIGN_TEXT ;                                               \
 122 IDTVEC(vec_name) ;                                                      \
 123         APIC_PUSH_FRAME ;                                               \
 124         FAKE_MCOUNT(TF_RIP(%rsp)) ;                                     \
 125         MASK_LEVEL_IRQ(irq_num) ;                                       \
 126         movq    lapic, %rax ;                                           \
 127         movl    $0, LA_EOI(%rax) ;                                      \
 128         movq    PCPU(curthread),%rbx ;                                  \
 129         testl   $-1,TD_NEST_COUNT(%rbx) ;                               \
 130         jne     1f ;                                                    \
 131         testl   $-1,TD_CRITCOUNT(%rbx) ;                                \
 132         je      2f ;                                                    \
 133 1: ;                                                                    \
 134         /* in critical section, make interrupt pending */               \
 135         /* set the pending bit and return, leave interrupt masked */    \
 136         orl     $IRQ_LBIT(irq_num),PCPU(fpending) ;                     \
 137         orl     $RQF_INTPEND,PCPU(reqflags) ;                           \
 138         jmp     5f ;                                                    \
 139 2: ;                                                                    \
 140         /* clear pending bit, run handler */                            \
 141         andl    $~IRQ_LBIT(irq_num),PCPU(fpending) ;                    \
 142         pushq   $irq_num ;              /* trapframe -> intrframe */    \
 143         movq    %rsp, %rdi ;            /* pass frame by reference */   \
 144         incl    TD_CRITCOUNT(%rbx) ;                                    \
 145         sti ;                                                           \
 146         call    ithread_fast_handler ;  /* returns 0 to unmask */       \
 147         decl    TD_CRITCOUNT(%rbx) ;                                    \
 148         addq    $8, %rsp ;              /* intrframe -> trapframe */    \
 149         UNMASK_IRQ(irq_num) ;                                           \
 150 5: ;                                                                    \
 151         MEXITCOUNT ;                                                    \
 152         jmp     doreti ;                                                \
 153
 154 #endif
 155
 156 /*
 157  * Handle "spurious INTerrupts".
 158  * Notes:
 159  *  This is different than the "spurious INTerrupt" generated by an
 160  *   8259 PIC for missing INTs.  See the APIC documentation for details.
 161  *  This routine should NOT do an 'EOI' cycle.
 162  */
 163         .text
 164         SUPERALIGN_TEXT
 165         .globl Xspuriousint
 166 Xspuriousint:
 167
 168         /* No EOI cycle used here */
 169
 170         jmp     doreti_iret
 171
 172
 173 /*
 174  * Handle TLB shootdowns.
 175  *
 176  * NOTE: interrupts are left disabled.
 177  */
 178         .text
 179         SUPERALIGN_TEXT
 180         .globl  Xinvltlb
 181 Xinvltlb:
 182         APIC_PUSH_FRAME
 183         movq    lapic, %rax
 184         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
 185         FAKE_MCOUNT(TF_RIP(%rsp))
 186         subq    $8,%rsp                 /* make same as interrupt frame */
 187         movq    %rsp,%rdi               /* pass frame by reference */
 188         call    smp_invltlb_intr
 189         addq    $8,%rsp                 /* turn into trapframe */
 190         MEXITCOUNT
 191         APIC_POP_FRAME
 192         jmp     doreti_iret
 193
 194 /*
 195  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
 196  *
 197  *  - We cannot call doreti
 198  *  - Signals its receipt.
 199  *  - Waits for permission to restart.
 200  *  - Processing pending IPIQ events while waiting.
 201  *  - Signals its restart.
 202  */
 203
 204         .text
 205         SUPERALIGN_TEXT
 206         .globl Xcpustop
 207 Xcpustop:
 208         APIC_PUSH_FRAME
 209         movq    lapic, %rax
 210         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
 211
 212         movl    PCPU(cpuid), %eax
 213         imull   $PCB_SIZE, %eax
 214         leaq    CNAME(stoppcbs), %rdi
 215         addq    %rax, %rdi
 216         call    CNAME(savectx)          /* Save process context */
 217
 218         movslq  PCPU(cpuid), %rax
 219
 220         /*
 221          * Indicate that we have stopped and loop waiting for permission
 222          * to start again.  We must still process IPI events while in a
 223          * stopped state.
 224          *
 225          * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
 226          * (e.g. Xtimer, Xinvltlb).
 227          */
 228         MPLOCKED
 229         btsq    %rax, stopped_cpus      /* stopped_cpus |= (1<<id) */
 230         sti
 231 1:
 232         andl    $~RQF_IPIQ,PCPU(reqflags)
 233         pushq   %rax
 234         call    lwkt_smp_stopped
 235         popq    %rax
 236         pause
 237         btq     %rax, started_cpus      /* while (!(started_cpus & (1<<id))) */
 238         jnc     1b
 239
 240         MPLOCKED
 241         btrq    %rax, started_cpus      /* started_cpus &= ~(1<<id) */
 242         MPLOCKED
 243         btrq    %rax, stopped_cpus      /* stopped_cpus &= ~(1<<id) */
 244
 245         test    %eax, %eax
 246         jnz     2f
 247
 248         movq    CNAME(cpustop_restartfunc), %rax
 249         test    %rax, %rax
 250         jz      2f
 251         movq    $0, CNAME(cpustop_restartfunc)  /* One-shot */
 252
 253         call    *%rax
 254 2:
 255         MEXITCOUNT
 256         APIC_POP_FRAME
 257         jmp     doreti_iret
 258
 259         /*
 260          * For now just have one ipiq IPI, but what we really want is
 261          * to have one for each source cpu to the APICs don't get stalled
 262          * backlogging the requests.
 263          */
 264         .text
 265         SUPERALIGN_TEXT
 266         .globl Xipiq
 267 Xipiq:
 268         APIC_PUSH_FRAME
 269         movq    lapic, %rax
 270         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
 271         FAKE_MCOUNT(TF_RIP(%rsp))
 272
 273         incl    PCPU(cnt) + V_IPI
 274         movq    PCPU(curthread),%rbx
 275         testl   $-1,TD_CRITCOUNT(%rbx)
 276         jne     1f
 277         subq    $8,%rsp                 /* make same as interrupt frame */
 278         movq    %rsp,%rdi               /* pass frame by reference */
 279         incl    PCPU(intr_nesting_level)
 280         incl    TD_CRITCOUNT(%rbx)
 281         sti
 282         call    lwkt_process_ipiq_frame
 283         decl    TD_CRITCOUNT(%rbx)
 284         decl    PCPU(intr_nesting_level)
 285         addq    $8,%rsp                 /* turn into trapframe */
 286         MEXITCOUNT
 287         jmp     doreti
 288 1:
 289         orl     $RQF_IPIQ,PCPU(reqflags)
 290         MEXITCOUNT
 291         APIC_POP_FRAME
 292         jmp     doreti_iret
 293
 294         .text
 295         SUPERALIGN_TEXT
 296         .globl Xtimer
 297 Xtimer:
 298         APIC_PUSH_FRAME
 299         movq    lapic, %rax
 300         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
 301         FAKE_MCOUNT(TF_RIP(%rsp))
 302
 303         subq    $8,%rsp                 /* make same as interrupt frame */
 304         movq    %rsp,%rdi               /* pass frame by reference */
 305         call    lapic_timer_always
 306         addq    $8,%rsp                 /* turn into trapframe */
 307
 308         incl    PCPU(cnt) + V_TIMER
 309         movq    PCPU(curthread),%rbx
 310         testl   $-1,TD_CRITCOUNT(%rbx)
 311         jne     1f
 312         testl   $-1,TD_NEST_COUNT(%rbx)
 313         jne     1f
 314         subq    $8,%rsp                 /* make same as interrupt frame */
 315         movq    %rsp,%rdi               /* pass frame by reference */
 316         incl    PCPU(intr_nesting_level)
 317         incl    TD_CRITCOUNT(%rbx)
 318         sti
 319         call    lapic_timer_process_frame
 320         decl    TD_CRITCOUNT(%rbx)
 321         decl    PCPU(intr_nesting_level)
 322         addq    $8,%rsp                 /* turn into trapframe */
 323         MEXITCOUNT
 324         jmp     doreti
 325 1:
 326         orl     $RQF_TIMER,PCPU(reqflags)
 327         MEXITCOUNT
 328         APIC_POP_FRAME
 329         jmp     doreti_iret
 330
 331 #ifdef SMP /* APIC-IO */
 332
 333 MCOUNT_LABEL(bintr)
 334         FAST_INTR(0,apic_fastintr0)
 335         FAST_INTR(1,apic_fastintr1)
 336         FAST_INTR(2,apic_fastintr2)
 337         FAST_INTR(3,apic_fastintr3)
 338         FAST_INTR(4,apic_fastintr4)
 339         FAST_INTR(5,apic_fastintr5)
 340         FAST_INTR(6,apic_fastintr6)
 341         FAST_INTR(7,apic_fastintr7)
 342         FAST_INTR(8,apic_fastintr8)
 343         FAST_INTR(9,apic_fastintr9)
 344         FAST_INTR(10,apic_fastintr10)
 345         FAST_INTR(11,apic_fastintr11)
 346         FAST_INTR(12,apic_fastintr12)
 347         FAST_INTR(13,apic_fastintr13)
 348         FAST_INTR(14,apic_fastintr14)
 349         FAST_INTR(15,apic_fastintr15)
 350         FAST_INTR(16,apic_fastintr16)
 351         FAST_INTR(17,apic_fastintr17)
 352         FAST_INTR(18,apic_fastintr18)
 353         FAST_INTR(19,apic_fastintr19)
 354         FAST_INTR(20,apic_fastintr20)
 355         FAST_INTR(21,apic_fastintr21)
 356         FAST_INTR(22,apic_fastintr22)
 357         FAST_INTR(23,apic_fastintr23)
 358 MCOUNT_LABEL(eintr)
 359
 360 #endif
 361
 362         .data
 363
 364 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
 365         .globl stopped_cpus, started_cpus
 366 stopped_cpus:
 367         .quad   0
 368 started_cpus:
 369         .quad   0
 370
 371         .globl CNAME(cpustop_restartfunc)
 372 CNAME(cpustop_restartfunc):
 373         .quad 0
 374
 375         .text
 376