sys/platform/pc64/apic/apic_vector.s

   1 /*
   2  *      from: vector.s, 386BSD 0.1 unknown origin
   3  * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
   4  */
   5
   6 #if 0
   7 #include "opt_auto_eoi.h"
   8 #endif
   9
  10 #include <machine/asmacros.h>
  11 #include <machine/lock.h>
  12 #include <machine/psl.h>
  13 #include <machine/trap.h>
  14 #include <machine/segments.h>
  15
  16 #include <machine_base/icu/icu.h>
  17 #include <bus/isa/isa.h>
  18
  19 #include "assym.s"
  20
  21 #include "apicreg.h"
  22 #include <machine_base/apic/ioapic_ipl.h>
  23 #include <machine/smp.h>
  24 #include <machine/intr_machdep.h>
  25
  26 #ifdef foo
  27 /* convert an absolute IRQ# into bitmask */
  28 #define IRQ_LBIT(irq_num)       (1UL << (irq_num & 0x3f))
  29 #endif
  30
  31 #define IRQ_SBITS(irq_num)      ((irq_num) & 0x3f)
  32
  33 /* convert an absolute IRQ# into gd_ipending index */
  34 #define IRQ_LIDX(irq_num)       ((irq_num) >> 6)
  35
  36 #ifdef SMP
  37 #define MPLOCKED     lock ;
  38 #else
  39 #define MPLOCKED
  40 #endif
  41
  42 #define APIC_PUSH_FRAME                                                 \
  43         PUSH_FRAME ;            /* 15 regs + space for 5 extras */      \
  44         movq $0,TF_XFLAGS(%rsp) ;                                       \
  45         movq $0,TF_TRAPNO(%rsp) ;                                       \
  46         movq $0,TF_ADDR(%rsp) ;                                         \
  47         movq $0,TF_FLAGS(%rsp) ;                                        \
  48         movq $0,TF_ERR(%rsp) ;                                          \
  49         cld ;                                                           \
  50
  51 /*
  52  * JG stale? Warning: POP_FRAME can only be used if there is no chance of a
  53  * segment register being changed (e.g. by procfs), which is why syscalls
  54  * have to use doreti.
  55  */
  56 #define APIC_POP_FRAME POP_FRAME
  57
  58 #define IOAPICADDR(irq_num) \
  59         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR
  60 #define REDIRIDX(irq_num) \
  61         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX
  62 #define IOAPICFLAGS(irq_num) \
  63         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS
  64
  65 #define MASK_IRQ(irq_num)                                               \
  66         IOAPIC_IMASK_LOCK ;                     /* into critical reg */ \
  67         testl   $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
  68         jne     7f ;                    /* masked, don't mask */        \
  69         orl     $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
  70                                                 /* set the mask bit */  \
  71         movq    IOAPICADDR(irq_num), %rcx ;     /* ioapic addr */       \
  72         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
  73         movl    %eax, (%rcx) ;                  /* write the index */   \
  74         orl     $IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* set the mask */  \
  75 7: ;                                            /* already masked */    \
  76         IOAPIC_IMASK_UNLOCK ;                                           \
  77
  78 /*
  79  * Test to see whether we are handling an edge or level triggered INT.
  80  *  Level-triggered INTs must still be masked as we don't clear the source,
  81  *  and the EOI cycle would cause redundant INTs to occur.
  82  */
  83 #define MASK_LEVEL_IRQ(irq_num)                                         \
  84         testl   $IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ;           \
  85         jz      9f ;                            /* edge, don't mask */  \
  86         MASK_IRQ(irq_num) ;                                             \
  87 9: ;                                                                    \
  88
  89 /*
  90  * Test to see if the source is currntly masked, clear if so.
  91  */
  92 #define UNMASK_IRQ(irq_num)                                     \
  93         cmpl    $0,%eax ;                                               \
  94         jnz     8f ;                                                    \
  95         IOAPIC_IMASK_LOCK ;                     /* into critical reg */ \
  96         testl   $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
  97         je      7f ;                    /* bit clear, not masked */     \
  98         andl    $~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;         \
  99                                                 /* clear mask bit */    \
 100         movq    IOAPICADDR(irq_num),%rcx ;      /* ioapic addr */       \
 101         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
 102         movl    %eax,(%rcx) ;                   /* write the index */   \
 103         andl    $~IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* clear the mask */ \
 104 7: ;                                                                    \
 105         IOAPIC_IMASK_UNLOCK ;                                           \
 106 8: ;                                                                    \
 107
 108 #ifdef SMP /* APIC-IO */
 109
 110 /*
 111  * Interrupt call handlers run in the following sequence:
 112  *
 113  *      - Push the trap frame required by doreti
 114  *      - Mask the interrupt and reenable its source
 115  *      - If we cannot take the interrupt set its ipending bit and
 116  *        doreti.
 117  *      - If we can take the interrupt clear its ipending bit,
 118  *        call the handler, then unmask and doreti.
 119  *
 120  * YYY can cache gd base opitner instead of using hidden %fs prefixes.
 121  */
 122
 123 #define INTR_HANDLER(irq_num)                                           \
 124         .text ;                                                         \
 125         SUPERALIGN_TEXT ;                                               \
 126 IDTVEC(ioapic_intr##irq_num) ;                                          \
 127         APIC_PUSH_FRAME ;                                               \
 128         FAKE_MCOUNT(TF_RIP(%rsp)) ;                                     \
 129         MASK_LEVEL_IRQ(irq_num) ;                                       \
 130         movq    lapic, %rax ;                                           \
 131         movl    $0, LA_EOI(%rax) ;                                      \
 132         movq    PCPU(curthread),%rbx ;                                  \
 133         testl   $-1,TD_NEST_COUNT(%rbx) ;                               \
 134         jne     1f ;                                                    \
 135         testl   $-1,TD_CRITCOUNT(%rbx) ;                                \
 136         je      2f ;                                                    \
 137 1: ;                                                                    \
 138         /* in critical section, make interrupt pending */               \
 139         /* set the pending bit and return, leave interrupt masked */    \
 140         movq    $1,%rcx ;                                               \
 141         shlq    $IRQ_SBITS(irq_num),%rcx ;                              \
 142         movq    $IRQ_LIDX(irq_num),%rdx ;                               \
 143         orq     %rcx,PCPU_E8(ipending,%rdx) ;                           \
 144         orl     $RQF_INTPEND,PCPU(reqflags) ;                           \
 145         jmp     5f ;                                                    \
 146 2: ;                                                                    \
 147         /* clear pending bit, run handler */                            \
 148         movq    $1,%rcx ;                                               \
 149         shlq    $IRQ_SBITS(irq_num),%rcx ;                              \
 150         notq    %rcx ;                                                  \
 151         movq    $IRQ_LIDX(irq_num),%rdx ;                               \
 152         andq    %rcx,PCPU_E8(ipending,%rdx) ;                           \
 153         pushq   $irq_num ;              /* trapframe -> intrframe */    \
 154         movq    %rsp, %rdi ;            /* pass frame by reference */   \
 155         incl    TD_CRITCOUNT(%rbx) ;                                    \
 156         sti ;                                                           \
 157         call    ithread_fast_handler ;  /* returns 0 to unmask */       \
 158         decl    TD_CRITCOUNT(%rbx) ;                                    \
 159         addq    $8, %rsp ;              /* intrframe -> trapframe */    \
 160         UNMASK_IRQ(irq_num) ;                                           \
 161 5: ;                                                                    \
 162         MEXITCOUNT ;                                                    \
 163         jmp     doreti ;                                                \
 164
 165 #endif
 166
 167 /*
 168  * Handle "spurious INTerrupts".
 169  * Notes:
 170  *  This is different than the "spurious INTerrupt" generated by an
 171  *   8259 PIC for missing INTs.  See the APIC documentation for details.
 172  *  This routine should NOT do an 'EOI' cycle.
 173  */
 174         .text
 175         SUPERALIGN_TEXT
 176         .globl Xspuriousint
 177 Xspuriousint:
 178
 179         /* No EOI cycle used here */
 180
 181         jmp     doreti_iret
 182
 183
 184 /*
 185  * Handle TLB shootdowns.
 186  *
 187  * NOTE: interrupts are left disabled.
 188  */
 189         .text
 190         SUPERALIGN_TEXT
 191         .globl  Xinvltlb
 192 Xinvltlb:
 193         APIC_PUSH_FRAME
 194         movq    lapic, %rax
 195         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
 196         FAKE_MCOUNT(TF_RIP(%rsp))
 197         subq    $8,%rsp                 /* make same as interrupt frame */
 198         movq    %rsp,%rdi               /* pass frame by reference */
 199         call    smp_invltlb_intr
 200         addq    $8,%rsp                 /* turn into trapframe */
 201         MEXITCOUNT
 202         APIC_POP_FRAME
 203         jmp     doreti_iret
 204
 205 /*
 206  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
 207  *
 208  *  - We cannot call doreti
 209  *  - Signals its receipt.
 210  *  - Waits for permission to restart.
 211  *  - Processing pending IPIQ events while waiting.
 212  *  - Signals its restart.
 213  */
 214
 215         .text
 216         SUPERALIGN_TEXT
 217         .globl Xcpustop
 218 Xcpustop:
 219         APIC_PUSH_FRAME
 220         movq    lapic, %rax
 221         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
 222
 223         movl    PCPU(cpuid), %eax
 224         imull   $PCB_SIZE, %eax
 225         leaq    CNAME(stoppcbs), %rdi
 226         addq    %rax, %rdi
 227         call    CNAME(savectx)          /* Save process context */
 228
 229         movslq  PCPU(cpuid), %rax
 230
 231         /*
 232          * Indicate that we have stopped and loop waiting for permission
 233          * to start again.  We must still process IPI events while in a
 234          * stopped state.
 235          *
 236          * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
 237          * (e.g. Xtimer, Xinvltlb).
 238          */
 239         MPLOCKED
 240         btsq    %rax, stopped_cpus      /* stopped_cpus |= (1<<id) */
 241         sti
 242 1:
 243         andl    $~RQF_IPIQ,PCPU(reqflags)
 244         pushq   %rax
 245         call    lwkt_smp_stopped
 246         popq    %rax
 247         pause
 248         btq     %rax, started_cpus      /* while (!(started_cpus & (1<<id))) */
 249         jnc     1b
 250
 251         MPLOCKED
 252         btrq    %rax, started_cpus      /* started_cpus &= ~(1<<id) */
 253         MPLOCKED
 254         btrq    %rax, stopped_cpus      /* stopped_cpus &= ~(1<<id) */
 255
 256         test    %eax, %eax
 257         jnz     2f
 258
 259         movq    CNAME(cpustop_restartfunc), %rax
 260         test    %rax, %rax
 261         jz      2f
 262         movq    $0, CNAME(cpustop_restartfunc)  /* One-shot */
 263
 264         call    *%rax
 265 2:
 266         MEXITCOUNT
 267         APIC_POP_FRAME
 268         jmp     doreti_iret
 269
 270         /*
 271          * For now just have one ipiq IPI, but what we really want is
 272          * to have one for each source cpu to the APICs don't get stalled
 273          * backlogging the requests.
 274          */
 275         .text
 276         SUPERALIGN_TEXT
 277         .globl Xipiq
 278 Xipiq:
 279         APIC_PUSH_FRAME
 280         movq    lapic, %rax
 281         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
 282         FAKE_MCOUNT(TF_RIP(%rsp))
 283
 284         incl    PCPU(cnt) + V_IPI
 285         movq    PCPU(curthread),%rbx
 286         testl   $-1,TD_CRITCOUNT(%rbx)
 287         jne     1f
 288         subq    $8,%rsp                 /* make same as interrupt frame */
 289         movq    %rsp,%rdi               /* pass frame by reference */
 290         incl    PCPU(intr_nesting_level)
 291         incl    TD_CRITCOUNT(%rbx)
 292         sti
 293         call    lwkt_process_ipiq_frame
 294         decl    TD_CRITCOUNT(%rbx)
 295         decl    PCPU(intr_nesting_level)
 296         addq    $8,%rsp                 /* turn into trapframe */
 297         MEXITCOUNT
 298         jmp     doreti
 299 1:
 300         orl     $RQF_IPIQ,PCPU(reqflags)
 301         MEXITCOUNT
 302         APIC_POP_FRAME
 303         jmp     doreti_iret
 304
 305         .text
 306         SUPERALIGN_TEXT
 307         .globl Xtimer
 308 Xtimer:
 309         APIC_PUSH_FRAME
 310         movq    lapic, %rax
 311         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
 312         FAKE_MCOUNT(TF_RIP(%rsp))
 313
 314         subq    $8,%rsp                 /* make same as interrupt frame */
 315         movq    %rsp,%rdi               /* pass frame by reference */
 316         call    lapic_timer_always
 317         addq    $8,%rsp                 /* turn into trapframe */
 318
 319         incl    PCPU(cnt) + V_TIMER
 320         movq    PCPU(curthread),%rbx
 321         testl   $-1,TD_CRITCOUNT(%rbx)
 322         jne     1f
 323         testl   $-1,TD_NEST_COUNT(%rbx)
 324         jne     1f
 325         subq    $8,%rsp                 /* make same as interrupt frame */
 326         movq    %rsp,%rdi               /* pass frame by reference */
 327         incl    PCPU(intr_nesting_level)
 328         incl    TD_CRITCOUNT(%rbx)
 329         sti
 330         call    lapic_timer_process_frame
 331         decl    TD_CRITCOUNT(%rbx)
 332         decl    PCPU(intr_nesting_level)
 333         addq    $8,%rsp                 /* turn into trapframe */
 334         MEXITCOUNT
 335         jmp     doreti
 336 1:
 337         orl     $RQF_TIMER,PCPU(reqflags)
 338         MEXITCOUNT
 339         APIC_POP_FRAME
 340         jmp     doreti_iret
 341
 342 #ifdef SMP /* APIC-IO */
 343
 344 MCOUNT_LABEL(bintr)
 345         INTR_HANDLER(0)
 346         INTR_HANDLER(1)
 347         INTR_HANDLER(2)
 348         INTR_HANDLER(3)
 349         INTR_HANDLER(4)
 350         INTR_HANDLER(5)
 351         INTR_HANDLER(6)
 352         INTR_HANDLER(7)
 353         INTR_HANDLER(8)
 354         INTR_HANDLER(9)
 355         INTR_HANDLER(10)
 356         INTR_HANDLER(11)
 357         INTR_HANDLER(12)
 358         INTR_HANDLER(13)
 359         INTR_HANDLER(14)
 360         INTR_HANDLER(15)
 361         INTR_HANDLER(16)
 362         INTR_HANDLER(17)
 363         INTR_HANDLER(18)
 364         INTR_HANDLER(19)
 365         INTR_HANDLER(20)
 366         INTR_HANDLER(21)
 367         INTR_HANDLER(22)
 368         INTR_HANDLER(23)
 369         INTR_HANDLER(24)
 370         INTR_HANDLER(25)
 371         INTR_HANDLER(26)
 372         INTR_HANDLER(27)
 373         INTR_HANDLER(28)
 374         INTR_HANDLER(29)
 375         INTR_HANDLER(30)
 376         INTR_HANDLER(31)
 377         INTR_HANDLER(32)
 378         INTR_HANDLER(33)
 379         INTR_HANDLER(34)
 380         INTR_HANDLER(35)
 381         INTR_HANDLER(36)
 382         INTR_HANDLER(37)
 383         INTR_HANDLER(38)
 384         INTR_HANDLER(39)
 385         INTR_HANDLER(40)
 386         INTR_HANDLER(41)
 387         INTR_HANDLER(42)
 388         INTR_HANDLER(43)
 389         INTR_HANDLER(44)
 390         INTR_HANDLER(45)
 391         INTR_HANDLER(46)
 392         INTR_HANDLER(47)
 393         INTR_HANDLER(48)
 394         INTR_HANDLER(49)
 395         INTR_HANDLER(50)
 396         INTR_HANDLER(51)
 397         INTR_HANDLER(52)
 398         INTR_HANDLER(53)
 399         INTR_HANDLER(54)
 400         INTR_HANDLER(55)
 401         INTR_HANDLER(56)
 402         INTR_HANDLER(57)
 403         INTR_HANDLER(58)
 404         INTR_HANDLER(59)
 405         INTR_HANDLER(60)
 406         INTR_HANDLER(61)
 407         INTR_HANDLER(62)
 408         INTR_HANDLER(63)
 409         INTR_HANDLER(64)
 410         INTR_HANDLER(65)
 411         INTR_HANDLER(66)
 412         INTR_HANDLER(67)
 413         INTR_HANDLER(68)
 414         INTR_HANDLER(69)
 415         INTR_HANDLER(70)
 416         INTR_HANDLER(71)
 417         INTR_HANDLER(72)
 418         INTR_HANDLER(73)
 419         INTR_HANDLER(74)
 420         INTR_HANDLER(75)
 421         INTR_HANDLER(76)
 422         INTR_HANDLER(77)
 423         INTR_HANDLER(78)
 424         INTR_HANDLER(79)
 425         INTR_HANDLER(80)
 426         INTR_HANDLER(81)
 427         INTR_HANDLER(82)
 428         INTR_HANDLER(83)
 429         INTR_HANDLER(84)
 430         INTR_HANDLER(85)
 431         INTR_HANDLER(86)
 432         INTR_HANDLER(87)
 433         INTR_HANDLER(88)
 434         INTR_HANDLER(89)
 435         INTR_HANDLER(90)
 436         INTR_HANDLER(91)
 437         INTR_HANDLER(92)
 438         INTR_HANDLER(93)
 439         INTR_HANDLER(94)
 440         INTR_HANDLER(95)
 441         INTR_HANDLER(96)
 442         INTR_HANDLER(97)
 443         INTR_HANDLER(98)
 444         INTR_HANDLER(99)
 445         INTR_HANDLER(100)
 446         INTR_HANDLER(101)
 447         INTR_HANDLER(102)
 448         INTR_HANDLER(103)
 449         INTR_HANDLER(104)
 450         INTR_HANDLER(105)
 451         INTR_HANDLER(106)
 452         INTR_HANDLER(107)
 453         INTR_HANDLER(108)
 454         INTR_HANDLER(109)
 455         INTR_HANDLER(110)
 456         INTR_HANDLER(111)
 457         INTR_HANDLER(112)
 458         INTR_HANDLER(113)
 459         INTR_HANDLER(114)
 460         INTR_HANDLER(115)
 461         INTR_HANDLER(116)
 462         INTR_HANDLER(117)
 463         INTR_HANDLER(118)
 464         INTR_HANDLER(119)
 465         INTR_HANDLER(120)
 466         INTR_HANDLER(121)
 467         INTR_HANDLER(122)
 468         INTR_HANDLER(123)
 469         INTR_HANDLER(124)
 470         INTR_HANDLER(125)
 471         INTR_HANDLER(126)
 472         INTR_HANDLER(127)
 473         INTR_HANDLER(128)
 474         INTR_HANDLER(129)
 475         INTR_HANDLER(130)
 476         INTR_HANDLER(131)
 477         INTR_HANDLER(132)
 478         INTR_HANDLER(133)
 479         INTR_HANDLER(134)
 480         INTR_HANDLER(135)
 481         INTR_HANDLER(136)
 482         INTR_HANDLER(137)
 483         INTR_HANDLER(138)
 484         INTR_HANDLER(139)
 485         INTR_HANDLER(140)
 486         INTR_HANDLER(141)
 487         INTR_HANDLER(142)
 488         INTR_HANDLER(143)
 489         INTR_HANDLER(144)
 490         INTR_HANDLER(145)
 491         INTR_HANDLER(146)
 492         INTR_HANDLER(147)
 493         INTR_HANDLER(148)
 494         INTR_HANDLER(149)
 495         INTR_HANDLER(150)
 496         INTR_HANDLER(151)
 497         INTR_HANDLER(152)
 498         INTR_HANDLER(153)
 499         INTR_HANDLER(154)
 500         INTR_HANDLER(155)
 501         INTR_HANDLER(156)
 502         INTR_HANDLER(157)
 503         INTR_HANDLER(158)
 504         INTR_HANDLER(159)
 505         INTR_HANDLER(160)
 506         INTR_HANDLER(161)
 507         INTR_HANDLER(162)
 508         INTR_HANDLER(163)
 509         INTR_HANDLER(164)
 510         INTR_HANDLER(165)
 511         INTR_HANDLER(166)
 512         INTR_HANDLER(167)
 513         INTR_HANDLER(168)
 514         INTR_HANDLER(169)
 515         INTR_HANDLER(170)
 516         INTR_HANDLER(171)
 517         INTR_HANDLER(172)
 518         INTR_HANDLER(173)
 519         INTR_HANDLER(174)
 520         INTR_HANDLER(175)
 521         INTR_HANDLER(176)
 522         INTR_HANDLER(177)
 523         INTR_HANDLER(178)
 524         INTR_HANDLER(179)
 525         INTR_HANDLER(180)
 526         INTR_HANDLER(181)
 527         INTR_HANDLER(182)
 528         INTR_HANDLER(183)
 529         INTR_HANDLER(184)
 530         INTR_HANDLER(185)
 531         INTR_HANDLER(186)
 532         INTR_HANDLER(187)
 533         INTR_HANDLER(188)
 534         INTR_HANDLER(189)
 535         INTR_HANDLER(190)
 536         INTR_HANDLER(191)
 537 MCOUNT_LABEL(eintr)
 538
 539 #endif
 540
 541         .data
 542
 543 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
 544         .globl stopped_cpus, started_cpus
 545 stopped_cpus:
 546         .quad   0
 547 started_cpus:
 548         .quad   0
 549
 550         .globl CNAME(cpustop_restartfunc)
 551 CNAME(cpustop_restartfunc):
 552         .quad 0
 553
 554         .text
 555