sys/platform/pc32/apic/apic_vector.s

   1 /*
   2  *      from: vector.s, 386BSD 0.1 unknown origin
   3  * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
   4  */
   5
   6 #include "opt_auto_eoi.h"
   7
   8 #include <machine/asmacros.h>
   9 #include <machine/lock.h>
  10 #include <machine/psl.h>
  11 #include <machine/trap.h>
  12
  13 #include <machine_base/icu/icu.h>
  14 #include <bus/isa/isa.h>
  15
  16 #include "assym.s"
  17
  18 #include "apicreg.h"
  19 #include <machine_base/apic/ioapic_ipl.h>
  20 #include <machine/intr_machdep.h>
  21
  22 /* convert an absolute IRQ# into bitmask */
  23 #define IRQ_LBIT(irq_num)       (1 << ((irq_num) & 0x1f))
  24
  25 /* convert an absolute IRQ# into ipending index */
  26 #define IRQ_LIDX(irq_num)       ((irq_num) >> 5)
  27
  28 #ifdef SMP
  29 #define MPLOCKED     lock ;
  30 #else
  31 #define MPLOCKED
  32 #endif
  33
  34 /*
  35  * Push an interrupt frame in a format acceptable to doreti, reload
  36  * the segment registers for the kernel.
  37  */
  38 #define PUSH_FRAME                                                      \
  39         pushl   $0 ;            /* dummy error code */                  \
  40         pushl   $0 ;            /* dummy trap type */                   \
  41         pushl   $0 ;            /* dummy xflags type */                 \
  42         pushal ;                                                        \
  43         pushl   %ds ;           /* save data and extra segments ... */  \
  44         pushl   %es ;                                                   \
  45         pushl   %fs ;                                                   \
  46         pushl   %gs ;                                                   \
  47         cld ;                                                           \
  48         mov     $KDSEL,%ax ;                                            \
  49         mov     %ax,%ds ;                                               \
  50         mov     %ax,%es ;                                               \
  51         mov     %ax,%gs ;                                               \
  52         mov     $KPSEL,%ax ;                                            \
  53         mov     %ax,%fs ;                                               \
  54
  55 /*
  56  * Warning: POP_FRAME can only be used if there is no chance of a
  57  * segment register being changed (e.g. by procfs), which is why syscalls
  58  * have to use doreti.
  59  */
  60 #define POP_FRAME                                                       \
  61         popl    %gs ;                                                   \
  62         popl    %fs ;                                                   \
  63         popl    %es ;                                                   \
  64         popl    %ds ;                                                   \
  65         popal ;                                                         \
  66         addl    $3*4,%esp ;     /* dummy xflags, trap & error codes */  \
  67
  68 #define IOAPICADDR(irq_num) \
  69         CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_ADDR
  70 #define REDIRIDX(irq_num) \
  71         CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_IDX
  72 #define IOAPICFLAGS(irq_num) \
  73         CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_FLAGS
  74
  75 #define MASK_IRQ(irq_num)                                               \
  76         IOAPIC_IMASK_LOCK ;                     /* into critical reg */ \
  77         testl   $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;        \
  78         jne     7f ;                    /* masked, don't mask */        \
  79         orl     $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;        \
  80                                                 /* set the mask bit */  \
  81         movl    IOAPICADDR(irq_num), %ecx ;     /* ioapic addr */       \
  82         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
  83         movl    %eax, (%ecx) ;                  /* write the index */   \
  84         orl     $IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* set the mask */  \
  85 7: ;                                            /* already masked */    \
  86         IOAPIC_IMASK_UNLOCK ;                                           \
  87
  88 /*
  89  * Test to see whether we are handling an edge or level triggered INT.
  90  *  Level-triggered INTs must still be masked as we don't clear the source,
  91  *  and the EOI cycle would cause redundant INTs to occur.
  92  */
  93 #define MASK_LEVEL_IRQ(irq_num)                                         \
  94         testl   $IOAPIC_IRQI_FLAG_LEVEL, IOAPICFLAGS(irq_num) ;         \
  95         jz      9f ;                            /* edge, don't mask */  \
  96         MASK_IRQ(irq_num) ;                                             \
  97 9: ;                                                                    \
  98
  99 /*
 100  * Test to see if the source is currntly masked, clear if so.
 101  */
 102 #define UNMASK_IRQ(irq_num)                                             \
 103         cmpl    $0,%eax ;                                               \
 104         jnz     8f ;                                                    \
 105         IOAPIC_IMASK_LOCK ;                     /* into critical reg */ \
 106         testl   $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;        \
 107         je      7f ;                    /* bit clear, not masked */     \
 108         andl    $~IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;       \
 109                                                 /* clear mask bit */    \
 110         movl    IOAPICADDR(irq_num),%ecx ;      /* ioapic addr */       \
 111         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
 112         movl    %eax,(%ecx) ;                   /* write the index */   \
 113         andl    $~IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* clear the mask */ \
 114 7: ;                                                                    \
 115         IOAPIC_IMASK_UNLOCK ;                                           \
 116 8: ;                                                                    \
 117
 118 /*
 119  * Interrupt call handlers run in the following sequence:
 120  *
 121  *      - Push the trap frame required by doreti
 122  *      - Mask the interrupt and reenable its source
 123  *      - If we cannot take the interrupt set its ipending bit and
 124  *        doreti.
 125  *      - If we can take the interrupt clear its ipending bit,
 126  *        call the handler, then unmask and doreti.
 127  *
 128  * YYY can cache gd base opitner instead of using hidden %fs prefixes.
 129  */
 130
 131 #define INTR_HANDLER(irq_num)                                           \
 132         .text ;                                                         \
 133         SUPERALIGN_TEXT ;                                               \
 134 IDTVEC(ioapic_intr##irq_num) ;                                          \
 135         PUSH_FRAME ;                                                    \
 136         FAKE_MCOUNT(15*4(%esp)) ;                                       \
 137         MASK_LEVEL_IRQ(irq_num) ;                                       \
 138         movl    lapic,%eax ;                                            \
 139         movl    $0,LA_EOI(%eax) ;                                       \
 140         movl    PCPU(curthread),%ebx ;                                  \
 141         movl    $0,%eax ;       /* CURRENT CPL IN FRAME (REMOVED) */    \
 142         pushl   %eax ;                                                  \
 143         testl   $-1,TD_NEST_COUNT(%ebx) ;                               \
 144         jne     1f ;                                                    \
 145         testl   $-1,TD_CRITCOUNT(%ebx) ;                                \
 146         je      2f ;                                                    \
 147 1: ;                                                                    \
 148         /* in critical section, make interrupt pending */               \
 149         /* set the pending bit and return, leave interrupt masked */    \
 150         movl    $IRQ_LIDX(irq_num),%edx ;                               \
 151         orl     $IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ;             \
 152         orl     $RQF_INTPEND,PCPU(reqflags) ;                           \
 153         jmp     5f ;                                                    \
 154 2: ;                                                                    \
 155         /* clear pending bit, run handler */                            \
 156         movl    $IRQ_LIDX(irq_num),%edx ;                               \
 157         andl    $~IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ;            \
 158         pushl   $irq_num ;                                              \
 159         pushl   %esp ;                   /* pass frame by reference */  \
 160         incl    TD_CRITCOUNT(%ebx) ;                                    \
 161         sti ;                                                           \
 162         call    ithread_fast_handler ;   /* returns 0 to unmask */      \
 163         decl    TD_CRITCOUNT(%ebx) ;                                    \
 164         addl    $8, %esp ;                                              \
 165         UNMASK_IRQ(irq_num) ;                                           \
 166 5: ;                                                                    \
 167         MEXITCOUNT ;                                                    \
 168         jmp     doreti ;                                                \
 169
 170 /*
 171  * Handle "spurious INTerrupts".
 172  * Notes:
 173  *  This is different than the "spurious INTerrupt" generated by an
 174  *   8259 PIC for missing INTs.  See the APIC documentation for details.
 175  *  This routine should NOT do an 'EOI' cycle.
 176  */
 177         .text
 178         SUPERALIGN_TEXT
 179         .globl Xspuriousint
 180 Xspuriousint:
 181
 182         /* No EOI cycle used here */
 183
 184         iret
 185
 186 #ifdef SMP
 187
 188 /*
 189  * Handle TLB shootdowns.
 190  *
 191  * NOTE: Interrupts remain disabled.
 192  */
 193         .text
 194         SUPERALIGN_TEXT
 195         .globl  Xinvltlb
 196 Xinvltlb:
 197         PUSH_FRAME
 198         movl    lapic,%eax
 199         movl    $0,LA_EOI(%eax)         /* End Of Interrupt to APIC */
 200         FAKE_MCOUNT(15*4(%esp))
 201
 202         subl    $8,%esp                 /* make same as interrupt frame */
 203         pushl   %esp                    /* pass frame by reference */
 204         call    smp_invltlb_intr
 205         addl    $12,%esp
 206
 207         MEXITCOUNT
 208         jmp     doreti_syscall_ret
 209
 210 /*
 211  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
 212  *
 213  *  - Signals its receipt.
 214  *  - Waits for permission to restart.
 215  *  - Processing pending IPIQ events while waiting.
 216  *  - Signals its restart.
 217  */
 218
 219         .text
 220         SUPERALIGN_TEXT
 221         .globl Xcpustop
 222 Xcpustop:
 223         pushl   %ebp
 224         movl    %esp, %ebp
 225         pushl   %eax
 226         pushl   %ecx
 227         pushl   %edx
 228         pushl   %ds                     /* save current data segment */
 229         pushl   %fs
 230
 231         movl    $KDSEL, %eax
 232         mov     %ax, %ds                /* use KERNEL data segment */
 233         movl    $KPSEL, %eax
 234         mov     %ax, %fs
 235
 236         movl    lapic, %eax
 237         movl    $0, LA_EOI(%eax)        /* End Of Interrupt to APIC */
 238
 239         movl    PCPU(cpuid), %eax
 240         imull   $PCB_SIZE, %eax
 241         leal    CNAME(stoppcbs)(%eax), %eax
 242         pushl   %eax
 243         call    CNAME(savectx)          /* Save process context */
 244         addl    $4, %esp
 245
 246
 247         movl    PCPU(cpuid), %eax
 248
 249         /*
 250          * Indicate that we have stopped and loop waiting for permission
 251          * to start again.  We must still process IPI events while in a
 252          * stopped state.
 253          *
 254          * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
 255          * (e.g. Xtimer, Xinvltlb).
 256          */
 257         MPLOCKED
 258         btsl    %eax, stopped_cpus      /* stopped_cpus |= (1<<id) */
 259         sti
 260 1:
 261         andl    $~RQF_IPIQ,PCPU(reqflags)
 262         pushl   %eax
 263         call    lwkt_smp_stopped
 264         popl    %eax
 265         btl     %eax, started_cpus      /* while (!(started_cpus & (1<<id))) */
 266         jnc     1b
 267
 268         MPLOCKED
 269         btrl    %eax, started_cpus      /* started_cpus &= ~(1<<id) */
 270         MPLOCKED
 271         btrl    %eax, stopped_cpus      /* stopped_cpus &= ~(1<<id) */
 272
 273         test    %eax, %eax
 274         jnz     2f
 275
 276         movl    CNAME(cpustop_restartfunc), %eax
 277         test    %eax, %eax
 278         jz      2f
 279         movl    $0, CNAME(cpustop_restartfunc)  /* One-shot */
 280
 281         call    *%eax
 282 2:
 283         popl    %fs
 284         popl    %ds                     /* restore previous data segment */
 285         popl    %edx
 286         popl    %ecx
 287         popl    %eax
 288         movl    %ebp, %esp
 289         popl    %ebp
 290         iret
 291
 292         /*
 293          * For now just have one ipiq IPI, but what we really want is
 294          * to have one for each source cpu to the APICs don't get stalled
 295          * backlogging the requests.
 296          */
 297         .text
 298         SUPERALIGN_TEXT
 299         .globl Xipiq
 300 Xipiq:
 301         PUSH_FRAME
 302         movl    lapic,%eax
 303         movl    $0,LA_EOI(%eax)         /* End Of Interrupt to APIC */
 304         FAKE_MCOUNT(15*4(%esp))
 305
 306         incl    PCPU(cnt) + V_IPI
 307         movl    PCPU(curthread),%ebx
 308         testl   $-1,TD_CRITCOUNT(%ebx)
 309         jne     1f
 310         subl    $8,%esp                 /* make same as interrupt frame */
 311         pushl   %esp                    /* pass frame by reference */
 312         incl    PCPU(intr_nesting_level)
 313         incl    TD_CRITCOUNT(%ebx)
 314         sti
 315         call    lwkt_process_ipiq_frame
 316         decl    TD_CRITCOUNT(%ebx)
 317         decl    PCPU(intr_nesting_level)
 318         addl    $12,%esp
 319         pushl   $0                      /* CPL for frame (REMOVED) */
 320         MEXITCOUNT
 321         jmp     doreti
 322 1:
 323         orl     $RQF_IPIQ,PCPU(reqflags)
 324         MEXITCOUNT
 325         jmp     doreti_syscall_ret
 326
 327 #endif  /* SMP */
 328
 329         .text
 330         SUPERALIGN_TEXT
 331         .globl Xtimer
 332 Xtimer:
 333         PUSH_FRAME
 334         movl    lapic,%eax
 335         movl    $0,LA_EOI(%eax)         /* End Of Interrupt to APIC */
 336         FAKE_MCOUNT(15*4(%esp))
 337
 338         incl    PCPU(cnt) + V_TIMER
 339         movl    PCPU(curthread),%ebx
 340         testl   $-1,TD_CRITCOUNT(%ebx)
 341         jne     1f
 342         testl   $-1,TD_NEST_COUNT(%ebx)
 343         jne     1f
 344         subl    $8,%esp                 /* make same as interrupt frame */
 345         pushl   %esp                    /* pass frame by reference */
 346         incl    PCPU(intr_nesting_level)
 347         incl    TD_CRITCOUNT(%ebx)
 348         sti
 349         call    lapic_timer_process_frame
 350         decl    TD_CRITCOUNT(%ebx)
 351         decl    PCPU(intr_nesting_level)
 352         addl    $12,%esp
 353         pushl   $0                      /* CPL for frame (REMOVED) */
 354         MEXITCOUNT
 355         jmp     doreti
 356 1:
 357         orl     $RQF_TIMER,PCPU(reqflags)
 358         MEXITCOUNT
 359         jmp     doreti_syscall_ret
 360
 361 MCOUNT_LABEL(bintr)
 362         INTR_HANDLER(0)
 363         INTR_HANDLER(1)
 364         INTR_HANDLER(2)
 365         INTR_HANDLER(3)
 366         INTR_HANDLER(4)
 367         INTR_HANDLER(5)
 368         INTR_HANDLER(6)
 369         INTR_HANDLER(7)
 370         INTR_HANDLER(8)
 371         INTR_HANDLER(9)
 372         INTR_HANDLER(10)
 373         INTR_HANDLER(11)
 374         INTR_HANDLER(12)
 375         INTR_HANDLER(13)
 376         INTR_HANDLER(14)
 377         INTR_HANDLER(15)
 378         INTR_HANDLER(16)
 379         INTR_HANDLER(17)
 380         INTR_HANDLER(18)
 381         INTR_HANDLER(19)
 382         INTR_HANDLER(20)
 383         INTR_HANDLER(21)
 384         INTR_HANDLER(22)
 385         INTR_HANDLER(23)
 386         INTR_HANDLER(24)
 387         INTR_HANDLER(25)
 388         INTR_HANDLER(26)
 389         INTR_HANDLER(27)
 390         INTR_HANDLER(28)
 391         INTR_HANDLER(29)
 392         INTR_HANDLER(30)
 393         INTR_HANDLER(31)
 394         INTR_HANDLER(32)
 395         INTR_HANDLER(33)
 396         INTR_HANDLER(34)
 397         INTR_HANDLER(35)
 398         INTR_HANDLER(36)
 399         INTR_HANDLER(37)
 400         INTR_HANDLER(38)
 401         INTR_HANDLER(39)
 402         INTR_HANDLER(40)
 403         INTR_HANDLER(41)
 404         INTR_HANDLER(42)
 405         INTR_HANDLER(43)
 406         INTR_HANDLER(44)
 407         INTR_HANDLER(45)
 408         INTR_HANDLER(46)
 409         INTR_HANDLER(47)
 410         INTR_HANDLER(48)
 411         INTR_HANDLER(49)
 412         INTR_HANDLER(50)
 413         INTR_HANDLER(51)
 414         INTR_HANDLER(52)
 415         INTR_HANDLER(53)
 416         INTR_HANDLER(54)
 417         INTR_HANDLER(55)
 418         INTR_HANDLER(56)
 419         INTR_HANDLER(57)
 420         INTR_HANDLER(58)
 421         INTR_HANDLER(59)
 422         INTR_HANDLER(60)
 423         INTR_HANDLER(61)
 424         INTR_HANDLER(62)
 425         INTR_HANDLER(63)
 426         INTR_HANDLER(64)
 427         INTR_HANDLER(65)
 428         INTR_HANDLER(66)
 429         INTR_HANDLER(67)
 430         INTR_HANDLER(68)
 431         INTR_HANDLER(69)
 432         INTR_HANDLER(70)
 433         INTR_HANDLER(71)
 434         INTR_HANDLER(72)
 435         INTR_HANDLER(73)
 436         INTR_HANDLER(74)
 437         INTR_HANDLER(75)
 438         INTR_HANDLER(76)
 439         INTR_HANDLER(77)
 440         INTR_HANDLER(78)
 441         INTR_HANDLER(79)
 442         INTR_HANDLER(80)
 443         INTR_HANDLER(81)
 444         INTR_HANDLER(82)
 445         INTR_HANDLER(83)
 446         INTR_HANDLER(84)
 447         INTR_HANDLER(85)
 448         INTR_HANDLER(86)
 449         INTR_HANDLER(87)
 450         INTR_HANDLER(88)
 451         INTR_HANDLER(89)
 452         INTR_HANDLER(90)
 453         INTR_HANDLER(91)
 454         INTR_HANDLER(92)
 455         INTR_HANDLER(93)
 456         INTR_HANDLER(94)
 457         INTR_HANDLER(95)
 458         INTR_HANDLER(96)
 459         INTR_HANDLER(97)
 460         INTR_HANDLER(98)
 461         INTR_HANDLER(99)
 462         INTR_HANDLER(100)
 463         INTR_HANDLER(101)
 464         INTR_HANDLER(102)
 465         INTR_HANDLER(103)
 466         INTR_HANDLER(104)
 467         INTR_HANDLER(105)
 468         INTR_HANDLER(106)
 469         INTR_HANDLER(107)
 470         INTR_HANDLER(108)
 471         INTR_HANDLER(109)
 472         INTR_HANDLER(110)
 473         INTR_HANDLER(111)
 474         INTR_HANDLER(112)
 475         INTR_HANDLER(113)
 476         INTR_HANDLER(114)
 477         INTR_HANDLER(115)
 478         INTR_HANDLER(116)
 479         INTR_HANDLER(117)
 480         INTR_HANDLER(118)
 481         INTR_HANDLER(119)
 482         INTR_HANDLER(120)
 483         INTR_HANDLER(121)
 484         INTR_HANDLER(122)
 485         INTR_HANDLER(123)
 486         INTR_HANDLER(124)
 487         INTR_HANDLER(125)
 488         INTR_HANDLER(126)
 489         INTR_HANDLER(127)
 490         INTR_HANDLER(128)
 491         INTR_HANDLER(129)
 492         INTR_HANDLER(130)
 493         INTR_HANDLER(131)
 494         INTR_HANDLER(132)
 495         INTR_HANDLER(133)
 496         INTR_HANDLER(134)
 497         INTR_HANDLER(135)
 498         INTR_HANDLER(136)
 499         INTR_HANDLER(137)
 500         INTR_HANDLER(138)
 501         INTR_HANDLER(139)
 502         INTR_HANDLER(140)
 503         INTR_HANDLER(141)
 504         INTR_HANDLER(142)
 505         INTR_HANDLER(143)
 506         INTR_HANDLER(144)
 507         INTR_HANDLER(145)
 508         INTR_HANDLER(146)
 509         INTR_HANDLER(147)
 510         INTR_HANDLER(148)
 511         INTR_HANDLER(149)
 512         INTR_HANDLER(150)
 513         INTR_HANDLER(151)
 514         INTR_HANDLER(152)
 515         INTR_HANDLER(153)
 516         INTR_HANDLER(154)
 517         INTR_HANDLER(155)
 518         INTR_HANDLER(156)
 519         INTR_HANDLER(157)
 520         INTR_HANDLER(158)
 521         INTR_HANDLER(159)
 522         INTR_HANDLER(160)
 523         INTR_HANDLER(161)
 524         INTR_HANDLER(162)
 525         INTR_HANDLER(163)
 526         INTR_HANDLER(164)
 527         INTR_HANDLER(165)
 528         INTR_HANDLER(166)
 529         INTR_HANDLER(167)
 530         INTR_HANDLER(168)
 531         INTR_HANDLER(169)
 532         INTR_HANDLER(170)
 533         INTR_HANDLER(171)
 534         INTR_HANDLER(172)
 535         INTR_HANDLER(173)
 536         INTR_HANDLER(174)
 537         INTR_HANDLER(175)
 538         INTR_HANDLER(176)
 539         INTR_HANDLER(177)
 540         INTR_HANDLER(178)
 541         INTR_HANDLER(179)
 542         INTR_HANDLER(180)
 543         INTR_HANDLER(181)
 544         INTR_HANDLER(182)
 545         INTR_HANDLER(183)
 546         INTR_HANDLER(184)
 547         INTR_HANDLER(185)
 548         INTR_HANDLER(186)
 549         INTR_HANDLER(187)
 550         INTR_HANDLER(188)
 551         INTR_HANDLER(189)
 552         INTR_HANDLER(190)
 553         INTR_HANDLER(191)
 554 MCOUNT_LABEL(eintr)
 555
 556         .data
 557
 558 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
 559         .globl stopped_cpus, started_cpus
 560 stopped_cpus:
 561         .long   0
 562 started_cpus:
 563         .long   0
 564
 565         .globl CNAME(cpustop_restartfunc)
 566 CNAME(cpustop_restartfunc):
 567         .long 0
 568
 569         .text
 570