projects / dragonfly.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | history | HEAD
pc32: Split out isa_intr.h and move isa/intr_machdep.h to include/
[dragonfly.git] / sys / platform / pc32 / i386 / mp_machdep.c
CommitLineData
984263bc
MD		 1/*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. The name of the developer may NOT be used to endorse or promote products
11 * derived from this software without specific prior written permission.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 * $FreeBSD: src/sys/i386/i386/mp_machdep.c,v 1.115.2.15 2003/03/14 21:22:35 jhb Exp $
c0c5de70 26 * $DragonFly: src/sys/platform/pc32/i386/mp_machdep.c,v 1.60 2008/06/07 12:03:52 mneumann Exp $
984263bc
MD		 27 */
28
29#include "opt_cpu.h"
984263bc 30
984263bc
MD		 31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
984263bc
MD		 34#include <sys/sysctl.h>
35#include <sys/malloc.h>
36#include <sys/memrange.h>
984263bc 37#include <sys/cons.h> /* cngetc() */
37e7efec 38#include <sys/machintr.h>
984263bc
MD		 39
40#include <vm/vm.h>
41#include <vm/vm_param.h>
42#include <vm/pmap.h>
43#include <vm/vm_kern.h>
44#include <vm/vm_extern.h>
984263bc
MD		 45#include <sys/lock.h>
46#include <vm/vm_map.h>
47#include <sys/user.h>
48#ifdef GPROF
49#include <sys/gmon.h>
50#endif
984263bc 51
684a93c4
MD		 52#include <sys/mplock2.h>
53
984263bc 54#include <machine/smp.h>
a9295349 55#include <machine_base/apic/apicreg.h>
984263bc
MD		 56#include <machine/atomic.h>
57#include <machine/cpufunc.h>
90e8a35b 58#include <machine/cputypes.h>
a9295349 59#include <machine_base/apic/mpapic.h>
984263bc
MD		 60#include <machine/psl.h>
61#include <machine/segments.h>
984263bc
MD		 62#include <machine/tss.h>
63#include <machine/specialreg.h>
64#include <machine/globaldata.h>
4117f2fd 65#include <machine/pmap_inval.h>
984263bc 66
984263bc 67#include <machine/md_var.h> /* setidt() */
87cf6827
SZ		 68#include <machine_base/icu/icu.h> /* IPIs */
69#include <machine/intr_machdep.h> /* IPIs */
984263bc 70
1439c090
MD		 71#define FIXUP_EXTRA_APIC_INTS 8 /* additional entries we may create */
72
984263bc
MD		 73#define WARMBOOT_TARGET 0
74#define WARMBOOT_OFF (KERNBASE + 0x0467)
75#define WARMBOOT_SEG (KERNBASE + 0x0469)
76
984263bc 77#define BIOS_BASE (0xf0000)
1df86978 78#define BIOS_BASE2 (0xe0000)
984263bc 79#define BIOS_SIZE (0x10000)
984263bc
MD		 80#define BIOS_COUNT (BIOS_SIZE/4)
81
82#define CMOS_REG (0x70)
83#define CMOS_DATA (0x71)
84#define BIOS_RESET (0x0f)
85#define BIOS_WARM (0x0a)
86
87#define PROCENTRY_FLAG_EN 0x01
88#define PROCENTRY_FLAG_BP 0x02
89#define IOAPICENTRY_FLAG_EN 0x01
90
91
92/* MP Floating Pointer Structure */
93typedef struct MPFPS {
94 char signature[4];
981bebd1 95 u_int32_t pap;
984263bc
MD		 96 u_char length;
97 u_char spec_rev;
98 u_char checksum;
99 u_char mpfb1;
100 u_char mpfb2;
101 u_char mpfb3;
102 u_char mpfb4;
103 u_char mpfb5;
104} *mpfps_t;
105
106/* MP Configuration Table Header */
107typedef struct MPCTH {
108 char signature[4];
109 u_short base_table_length;
110 u_char spec_rev;
111 u_char checksum;
112 u_char oem_id[8];
113 u_char product_id[12];
114 void *oem_table_pointer;
115 u_short oem_table_size;
116 u_short entry_count;
117 void *apic_address;
118 u_short extended_table_length;
119 u_char extended_table_checksum;
120 u_char reserved;
121} *mpcth_t;
122
123
124typedef struct PROCENTRY {
125 u_char type;
126 u_char apic_id;
127 u_char apic_version;
128 u_char cpu_flags;
129 u_long cpu_signature;
130 u_long feature_flags;
131 u_long reserved1;
132 u_long reserved2;
133} *proc_entry_ptr;
134
135typedef struct BUSENTRY {
136 u_char type;
137 u_char bus_id;
138 char bus_type[6];
139} *bus_entry_ptr;
140
141typedef struct IOAPICENTRY {
142 u_char type;
143 u_char apic_id;
144 u_char apic_version;
145 u_char apic_flags;
146 void *apic_address;
147} *io_apic_entry_ptr;
148
149typedef struct INTENTRY {
150 u_char type;
151 u_char int_type;
152 u_short int_flags;
153 u_char src_bus_id;
154 u_char src_bus_irq;
155 u_char dst_apic_id;
156 u_char dst_apic_int;
157} *int_entry_ptr;
158
159/* descriptions of MP basetable entries */
160typedef struct BASETABLE_ENTRY {
161 u_char type;
162 u_char length;
163 char name[16];
164} basetable_entry;
165
981bebd1
SZ		 166struct mptable_pos {
167 mpfps_t mp_fps;
168 mpcth_t mp_cth;
169 vm_size_t mp_cth_mapsz;
170};
171
fa058384
SZ		 172typedef int (*mptable_iter_func)(void *, const void *, int);
173
984263bc
MD		 174/*
175 * this code MUST be enabled here and in mpboot.s.
176 * it follows the very early stages of AP boot by placing values in CMOS ram.
177 * it NORMALLY will never be needed and thus the primitive method for enabling.
178 *
984263bc 179 */
7d34994c 180#if defined(CHECK_POINTS)
984263bc
MD		 181#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA))
182#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
183
184#define CHECK_INIT(D); \
185 CHECK_WRITE(0x34, (D)); \
186 CHECK_WRITE(0x35, (D)); \
187 CHECK_WRITE(0x36, (D)); \
188 CHECK_WRITE(0x37, (D)); \
189 CHECK_WRITE(0x38, (D)); \
190 CHECK_WRITE(0x39, (D));
191
192#define CHECK_PRINT(S); \
26be20a0 193 kprintf("%s: %d, %d, %d, %d, %d, %d\n", \
984263bc
MD		 194 (S), \
195 CHECK_READ(0x34), \
196 CHECK_READ(0x35), \
197 CHECK_READ(0x36), \
198 CHECK_READ(0x37), \
199 CHECK_READ(0x38), \
200 CHECK_READ(0x39));
201
202#else /* CHECK_POINTS */
203
204#define CHECK_INIT(D)
205#define CHECK_PRINT(S)
206
207#endif /* CHECK_POINTS */
208
209/*
210 * Values to send to the POST hardware.
211 */
212#define MP_BOOTADDRESS_POST 0x10
213#define MP_PROBE_POST 0x11
214#define MPTABLE_PASS1_POST 0x12
215
216#define MP_START_POST 0x13
217#define MP_ENABLE_POST 0x14
218#define MPTABLE_PASS2_POST 0x15
219
220#define START_ALL_APS_POST 0x16
221#define INSTALL_AP_TRAMP_POST 0x17
222#define START_AP_POST 0x18
223
224#define MP_ANNOUNCE_POST 0x19
225
984263bc
MD		 226/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
227int current_postcode;
228
229/** XXX FIXME: what system files declare these??? */
230extern struct region_descriptor r_gdt, r_idt;
231
984263bc 232int mp_naps; /* # of Applications processors */
30c5f287 233#ifdef SMP /* APIC-IO */
4f6a8b30 234static int mp_nbusses; /* # of busses */
984263bc 235int mp_napics; /* # of IO APICs */
984263bc 236vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */
97359a5b
MD		 237u_int32_t *io_apic_versions;
238#endif
984263bc
MD		 239extern int nkpt;
240
241u_int32_t cpu_apic_versions[MAXCPU];
374133e3 242int64_t tsc0_offset;
0b698dca 243extern int64_t tsc_offsets[];
984263bc 244
1876681a
SZ		 245extern u_long ebda_addr;
246
30c5f287 247#ifdef SMP /* APIC-IO */
8a8d5d85 248struct apic_intmapinfo int_to_apicintpin[APIC_INTMAPSIZE];
97359a5b 249#endif
984263bc 250
984263bc
MD		 251/*
252 * APIC ID logical/physical mapping structures.
253 * We oversize these to simplify boot-time config.
254 */
255int cpu_num_to_apic_id[NAPICID];
30c5f287 256#ifdef SMP /* APIC-IO */
984263bc 257int io_num_to_apic_id[NAPICID];
97359a5b 258#endif
984263bc
MD		 259int apic_id_to_logical[NAPICID];
260
984263bc
MD		 261/* AP uses this during bootstrap. Do not staticize. */
262char *bootSTK;
263static int bootAP;
264
265/* Hotwire a 0->4MB V==P mapping */
266extern pt_entry_t *KPTphys;
267
f13b5eec
MD		 268/*
269 * SMP page table page. Setup by locore to point to a page table
270 * page from which we allocate per-cpu privatespace areas io_apics,
271 * and so forth.
272 */
273
274#define IO_MAPPING_START_INDEX \
275 (SMP_MAXCPU * sizeof(struct privatespace) / PAGE_SIZE)
276
984263bc 277extern pt_entry_t *SMPpt;
f13b5eec 278static int SMPpt_alloc_index = IO_MAPPING_START_INDEX;
984263bc
MD		 279
280struct pcb stoppcbs[MAXCPU];
281
fa058384
SZ		 282static basetable_entry basetable_entry_types[] =
283{
284 {0, 20, "Processor"},
285 {1, 8, "Bus"},
286 {2, 8, "I/O APIC"},
287 {3, 8, "I/O INT"},
288 {4, 8, "Local INT"}
289};
290
984263bc
MD		 291/*
292 * Local data and functions.
293 */
294
984263bc
MD		 295static u_int boot_address;
296static u_int base_memory;
41a01a4d 297static int mp_finish;
984263bc 298
984263bc
MD		 299static void mp_enable(u_int boot_addr);
300
fa058384
SZ		 301static int mptable_iterate_entries(const mpcth_t,
302 mptable_iter_func, void *);
cb00b5c4 303static int mptable_probe(void);
34e6fa63 304static int mptable_search(void);
fa058384 305static int mptable_check(vm_paddr_t);
3aba8f73 306static int mptable_search_sig(u_int32_t target, int count);
da23a592 307static int mptable_hyperthread_fixup(cpumask_t, int);
30c5f287 308#ifdef SMP /* APIC-IO */
981bebd1 309static void mptable_pass1(struct mptable_pos *);
390b18b0 310static void mptable_pass2(struct mptable_pos *);
3aba8f73
SZ		 311static void mptable_default(int type);
312static void mptable_fix(void);
29bb1d92 313#endif
fa058384 314static int mptable_map(struct mptable_pos *, vm_paddr_t);
981bebd1 315static void mptable_unmap(struct mptable_pos *);
a0eaef71 316static void mptable_imcr(struct mptable_pos *);
3aba8f73 317
281d9482
SZ		 318static int mptable_lapic_probe(struct lapic_enumerator *);
319static void mptable_lapic_enumerate(struct lapic_enumerator *);
320static void mptable_lapic_default(void);
321
30c5f287 322#ifdef SMP /* APIC-IO */
984263bc 323static void setup_apic_irq_mapping(void);
97359a5b
MD		 324static int apic_int_is_bus_type(int intr, int bus_type);
325#endif
984263bc
MD		 326static int start_all_aps(u_int boot_addr);
327static void install_ap_tramp(u_int boot_addr);
bb467734
MD		 328static int start_ap(struct mdglobaldata *gd, u_int boot_addr, int smibest);
329static int smitest(void);
984263bc 330
41a01a4d 331static cpumask_t smp_startup_mask = 1; /* which cpus have been started */
0f7a3396
MD		 332cpumask_t smp_active_mask = 1; /* which cpus are ready for IPIs etc? */
333SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RD, &smp_active_mask, 0, "");
334
984263bc
MD		 335/*
336 * Calculate usable address in base memory for AP trampoline code.
337 */
338u_int
339mp_bootaddress(u_int basemem)
340{
341 POSTCODE(MP_BOOTADDRESS_POST);
342
c0c5de70 343 base_memory = basemem;
984263bc
MD		 344
345 boot_address = base_memory & ~0xfff; /* round down to 4k boundary */
346 if ((base_memory - boot_address) < bootMP_size)
347 boot_address -= 4096; /* not enough, lower by 4k */
348
349 return boot_address;
350}
351
352
34e6fa63
SZ		 353static int
354mptable_probe(void)
355{
356 int mpfps_paddr;
357
358 mpfps_paddr = mptable_search();
359 if (mptable_check(mpfps_paddr))
360 return 0;
361
362 return mpfps_paddr;
363}
364
984263bc
MD		 365/*
366 * Look for an Intel MP spec table (ie, SMP capable hardware).
367 */
cb00b5c4 368static int
34e6fa63 369mptable_search(void)
984263bc
MD		 370{
371 int x;
984263bc 372 u_int32_t target;
f13b5eec
MD		 373
374 /*
375 * Make sure our SMPpt[] page table is big enough to hold all the
376 * mappings we need.
377 */
378 KKASSERT(IO_MAPPING_START_INDEX < NPTEPG - 2);
984263bc
MD		 379
380 POSTCODE(MP_PROBE_POST);
381
382 /* see if EBDA exists */
1876681a 383 if (ebda_addr != 0) {
984263bc 384 /* search first 1K of EBDA */
1876681a 385 target = (u_int32_t)ebda_addr;
3aba8f73 386 if ((x = mptable_search_sig(target, 1024 / 4)) > 0)
aeb48299 387 return x;
984263bc
MD		 388 } else {
389 /* last 1K of base memory, effective 'top of base' passed in */
aeb48299 390 target = (u_int32_t)(base_memory - 0x400);
3aba8f73 391 if ((x = mptable_search_sig(target, 1024 / 4)) > 0)
aeb48299 392 return x;
984263bc
MD		 393 }
394
395 /* search the BIOS */
aeb48299 396 target = (u_int32_t)BIOS_BASE;
3aba8f73 397 if ((x = mptable_search_sig(target, BIOS_COUNT)) > 0)
aeb48299 398 return x;
984263bc 399
1df86978
SZ		 400 /* search the extended BIOS */
401 target = (u_int32_t)BIOS_BASE2;
402 if ((x = mptable_search_sig(target, BIOS_COUNT)) > 0)
403 return x;
404
984263bc 405 /* nothing found */
984263bc 406 return 0;
984263bc
MD		 407}
408
fa058384
SZ		 409struct mptable_check_cbarg {
410 int cpu_count;
411 int found_bsp;
412};
413
414static int
415mptable_check_callback(void *xarg, const void *pos, int type)
416{
417 const struct PROCENTRY *ent;
418 struct mptable_check_cbarg *arg = xarg;
419
420 if (type != 0)
421 return 0;
422 ent = pos;
423
424 if ((ent->cpu_flags & PROCENTRY_FLAG_EN) == 0)
425 return 0;
426 arg->cpu_count++;
427
428 if (ent->cpu_flags & PROCENTRY_FLAG_BP) {
429 if (arg->found_bsp) {
430 kprintf("more than one BSP in base MP table\n");
431 return EINVAL;
432 }
433 arg->found_bsp = 1;
434 }
435 return 0;
436}
437
438static int
439mptable_check(vm_paddr_t mpfps_paddr)
440{
441 struct mptable_pos mpt;
442 struct mptable_check_cbarg arg;
443 mpcth_t cth;
444 int error;
445
446 if (mpfps_paddr == 0)
447 return EOPNOTSUPP;
448
449 error = mptable_map(&mpt, mpfps_paddr);
450 if (error)
451 return error;
452
453 if (mpt.mp_fps->mpfb1 != 0)
454 goto done;
455
456 error = EINVAL;
457
458 cth = mpt.mp_cth;
459 if (cth == NULL)
460 goto done;
461 if (cth->apic_address == 0)
462 goto done;
463
464 bzero(&arg, sizeof(arg));
465 error = mptable_iterate_entries(cth, mptable_check_callback, &arg);
466 if (!error) {
467 if (arg.cpu_count == 0) {
468 kprintf("MP table contains no processor entries\n");
469 error = EINVAL;
470 } else if (!arg.found_bsp) {
471 kprintf("MP table does not contains BSP entry\n");
472 error = EINVAL;
473 }
474 }
475done:
476 mptable_unmap(&mpt);
477 return error;
478}
479
480static int
481mptable_iterate_entries(const mpcth_t cth, mptable_iter_func func, void *arg)
482{
483 int count, total_size;
484 const void *position;
485
486 KKASSERT(cth->base_table_length >= sizeof(struct MPCTH));
487 total_size = cth->base_table_length - sizeof(struct MPCTH);
488 position = (const uint8_t *)cth + sizeof(struct MPCTH);
489 count = cth->entry_count;
490
491 while (count--) {
492 int type, error;
493
494 KKASSERT(total_size >= 0);
495 if (total_size == 0) {
496 kprintf("invalid base MP table, "
497 "entry count and length mismatch\n");
498 return EINVAL;
499 }
500
501 type = *(const uint8_t *)position;
502 switch (type) {
503 case 0: /* processor_entry */
504 case 1: /* bus_entry */
505 case 2: /* io_apic_entry */
506 case 3: /* int_entry */
507 case 4: /* int_entry */
508 break;
509 default:
510 kprintf("unknown base MP table entry type %d\n", type);
511 return EINVAL;
512 }
513
514 if (total_size < basetable_entry_types[type].length) {
515 kprintf("invalid base MP table length, "
516 "does not contain all entries\n");
517 return EINVAL;
518 }
519 total_size -= basetable_entry_types[type].length;
520
521 error = func(arg, position, type);
522 if (error)
523 return error;
524
525 position = (const uint8_t *)position +
526 basetable_entry_types[type].length;
527 }
528 return 0;
529}
530
984263bc
MD		 531
532/*
533 * Startup the SMP processors.
534 */
535void
536mp_start(void)
537{
538 POSTCODE(MP_START_POST);
50bc991e 539 mp_enable(boot_address);
984263bc
MD		 540}
541
542
543/*
544 * Print various information about the SMP system hardware and setup.
545 */
546void
547mp_announce(void)
548{
549 int x;
550
551 POSTCODE(MP_ANNOUNCE_POST);
552
26be20a0
SW		 553 kprintf("DragonFly/MP: Multiprocessor motherboard\n");
554 kprintf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
8629c4ea 555 kprintf(", version: 0x%08x\n", cpu_apic_versions[0]);
984263bc 556 for (x = 1; x <= mp_naps; ++x) {
26be20a0 557 kprintf(" cpu%d (AP): apic id: %2d", x, CPU_TO_ID(x));
8629c4ea 558 kprintf(", version: 0x%08x\n", cpu_apic_versions[x]);
984263bc
MD		 559 }
560
30c5f287 561if (apic_io_enable) {
984263bc 562 for (x = 0; x < mp_napics; ++x) {
26be20a0
SW		 563 kprintf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x));
564 kprintf(", version: 0x%08x", io_apic_versions[x]);
d557216f 565 kprintf(", at 0x%08lx\n", io_apic_address[x]);
984263bc 566 }
30c5f287 567} else {
26be20a0 568 kprintf(" Warning: APIC I/O disabled\n");
30c5f287 569}
984263bc
MD		 570}
571
572/*
573 * AP cpu's call this to sync up protected mode.
7160572f
MD		 574 *
575 * WARNING! We must ensure that the cpu is sufficiently initialized to
576 * be able to use to the FP for our optimized bzero/bcopy code before
577 * we enter more mainstream C code.
a44bdeec
MD		 578 *
579 * WARNING! %fs is not set up on entry. This routine sets up %fs.
984263bc
MD		 580 */
581void
582init_secondary(void)
583{
584 int gsel_tss;
585 int x, myid = bootAP;
586 u_int cr0;
8a8d5d85 587 struct mdglobaldata *md;
0f7a3396 588 struct privatespace *ps;
984263bc 589
0f7a3396
MD		 590 ps = &CPU_prvspace[myid];
591
592 gdt_segs[GPRIV_SEL].ssd_base = (int)ps;
984263bc 593 gdt_segs[GPROC0_SEL].ssd_base =
0f7a3396
MD		 594 (int) &ps->mdglobaldata.gd_common_tss;
595 ps->mdglobaldata.mi.gd_prvspace = ps;
984263bc
MD		 596
597 for (x = 0; x < NGDT; x++) {
598 ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd);
599 }
600
601 r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
602 r_gdt.rd_base = (int) &gdt[myid * NGDT];
603 lgdt(&r_gdt); /* does magic intra-segment return */
604
605 lidt(&r_idt);
606
607 lldt(_default_ldt);
7b95be2a 608 mdcpu->gd_currentldt = _default_ldt;
984263bc
MD		 609
610 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
611 gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS;
8a8d5d85 612
0f7a3396 613 md = mdcpu; /* loaded through %fs:0 (mdglobaldata.mi.gd_prvspace)*/
8a8d5d85
MD		 614
615 md->gd_common_tss.tss_esp0 = 0; /* not used until after switch */
616 md->gd_common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
617 md->gd_common_tss.tss_ioopt = (sizeof md->gd_common_tss) << 16;
618 md->gd_tss_gdt = &gdt[myid * NGDT + GPROC0_SEL].sd;
619 md->gd_common_tssd = *md->gd_tss_gdt;
984263bc
MD		 620 ltr(gsel_tss);
621
622 /*
623 * Set to a known state:
624 * Set by mpboot.s: CR0_PG, CR0_PE
625 * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM
626 */
627 cr0 = rcr0();
628 cr0 &= ~(CR0_CD | CR0_NW | CR0_EM);
629 load_cr0(cr0);
7160572f 630 pmap_set_opt(); /* PSE/4MB pages, etc */
984263bc 631
7160572f
MD		 632 /* set up CPU registers and state */
633 cpu_setregs();
634
635 /* set up FPU state on the AP */
636 npxinit(__INITIAL_NPXCW__);
637
638 /* set up SSE registers */
639 enable_sse();
984263bc
MD		 640}
641
984263bc
MD		 642/*******************************************************************
643 * local functions and data
644 */
645
646/*
647 * start the SMP system
648 */
649static void
650mp_enable(u_int boot_addr)
651{
984263bc
MD		 652 int apic;
653 u_int ux;
981bebd1 654 vm_paddr_t mpfps_paddr;
f65c10b6 655 struct mptable_pos mpt;
984263bc
MD		 656
657 POSTCODE(MP_ENABLE_POST);
658
281d9482 659 lapic_config();
984263bc 660
f65c10b6
SZ		 661 mpfps_paddr = mptable_probe();
662 if (mpfps_paddr) {
663 mptable_map(&mpt, mpfps_paddr);
664 mptable_imcr(&mpt);
665 mptable_unmap(&mpt);
666 }
30c5f287 667if (apic_io_enable) {
984263bc 668
f65c10b6 669 if (!mpfps_paddr)
30c5f287 670 panic("no MP table, disable APIC_IO! (set hw.apic_io_enable=0)\n");
f65c10b6
SZ		 671
672 mptable_map(&mpt, mpfps_paddr);
673
674 /*
675 * Examine the MP table for needed info
676 */
677 mptable_pass1(&mpt);
678 mptable_pass2(&mpt);
679
680 mptable_unmap(&mpt);
681
682 /* Post scan cleanup */
683 mptable_fix();
684
97359a5b
MD		 685 setup_apic_irq_mapping();
686
984263bc
MD		 687 /* fill the LOGICAL io_apic_versions table */
688 for (apic = 0; apic < mp_napics; ++apic) {
689 ux = io_apic_read(apic, IOAPIC_VER);
690 io_apic_versions[apic] = ux;
691 io_apic_set_id(apic, IO_TO_ID(apic));
692 }
693
694 /* program each IO APIC in the system */
695 for (apic = 0; apic < mp_napics; ++apic)
696 if (io_apic_setup(apic) < 0)
697 panic("IO APIC setup failure");
698
30c5f287 699}
97359a5b
MD		 700
701 /*
702 * These are required for SMP operation
703 */
704
984263bc
MD		 705 /* install a 'Spurious INTerrupt' vector */
706 setidt(XSPURIOUSINT_OFFSET, Xspuriousint,
707 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
708
709 /* install an inter-CPU IPI for TLB invalidation */
710 setidt(XINVLTLB_OFFSET, Xinvltlb,
711 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
712
96728c05
MD		 713 /* install an inter-CPU IPI for IPIQ messaging */
714 setidt(XIPIQ_OFFSET, Xipiq,
715 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
78ea5a2a
SZ		 716
717 /* install a timer vector */
718 setidt(XTIMER_OFFSET, Xtimer,
719 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
984263bc 720
984263bc
MD		 721 /* install an inter-CPU IPI for CPU stop/restart */
722 setidt(XCPUSTOP_OFFSET, Xcpustop,
723 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
724
984263bc
MD		 725 /* start each Application Processor */
726 start_all_aps(boot_addr);
727}
728
729
730/*
731 * look for the MP spec signature
732 */
733
734/* string defined by the Intel MP Spec as identifying the MP table */
735#define MP_SIG 0x5f504d5f /* _MP_ */
736#define NEXT(X) ((X) += 4)
737static int
3aba8f73 738mptable_search_sig(u_int32_t target, int count)
984263bc 739{
0f85efa2
SZ		 740 vm_size_t map_size;
741 u_int32_t *addr;
742 int x, ret;
984263bc 743
aeb48299
SZ		 744 KKASSERT(target != 0);
745
0f85efa2
SZ		 746 map_size = count * sizeof(u_int32_t);
747 addr = pmap_mapdev((vm_paddr_t)target, map_size);
984263bc 748
aeb48299 749 ret = 0;
0f85efa2
SZ		 750 for (x = 0; x < count; NEXT(x)) {
751 if (addr[x] == MP_SIG) {
752 /* make array index a byte index */
753 ret = target + (x * sizeof(u_int32_t));
754 break;
755 }
756 }
aeb48299 757
0f85efa2
SZ		 758 pmap_unmapdev((vm_offset_t)addr, map_size);
759 return ret;
984263bc
MD		 760}
761
762
984263bc
MD		 763typedef struct BUSDATA {
764 u_char bus_id;
765 enum busTypes bus_type;
766} bus_datum;
767
768typedef struct INTDATA {
769 u_char int_type;
770 u_short int_flags;
771 u_char src_bus_id;
772 u_char src_bus_irq;
773 u_char dst_apic_id;
774 u_char dst_apic_int;
775 u_char int_vector;
776} io_int, local_int;
777
778typedef struct BUSTYPENAME {
779 u_char type;
780 char name[7];
781} bus_type_name;
782
783static bus_type_name bus_type_table[] =
784{
785 {CBUS, "CBUS"},
786 {CBUSII, "CBUSII"},
787 {EISA, "EISA"},
788 {MCA, "MCA"},
789 {UNKNOWN_BUSTYPE, "---"},
790 {ISA, "ISA"},
791 {MCA, "MCA"},
792 {UNKNOWN_BUSTYPE, "---"},
793 {UNKNOWN_BUSTYPE, "---"},
794 {UNKNOWN_BUSTYPE, "---"},
795 {UNKNOWN_BUSTYPE, "---"},
796 {UNKNOWN_BUSTYPE, "---"},
797 {PCI, "PCI"},
798 {UNKNOWN_BUSTYPE, "---"},
799 {UNKNOWN_BUSTYPE, "---"},
800 {UNKNOWN_BUSTYPE, "---"},
801 {UNKNOWN_BUSTYPE, "---"},
802 {XPRESS, "XPRESS"},
803 {UNKNOWN_BUSTYPE, "---"}
804};
805/* from MP spec v1.4, table 5-1 */
806static int default_data[7][5] =
807{
808/* nbus, id0, type0, id1, type1 */
809 {1, 0, ISA, 255, 255},
810 {1, 0, EISA, 255, 255},
811 {1, 0, EISA, 255, 255},
812 {1, 0, MCA, 255, 255},
813 {2, 0, ISA, 1, PCI},
814 {2, 0, EISA, 1, PCI},
815 {2, 0, MCA, 1, PCI}
816};
817
818
819/* the bus data */
820static bus_datum *bus_data;
821
822/* the IO INT data, one entry per possible APIC INTerrupt */
823static io_int *io_apic_ints;
984263bc 824static int nintrs;
4f6a8b30 825
a0873f07 826static int processor_entry (const struct PROCENTRY *entry, int cpu);
c4717d5c
SZ		 827static int bus_entry (const struct BUSENTRY *entry, int bus);
828static int io_apic_entry (const struct IOAPICENTRY *entry, int apic);
829static int int_entry (const struct INTENTRY *entry, int intr);
3ae0cd58 830static int lookup_bus_type (char *name);
8658b5be
SZ		 831
832static int
833mptable_ioapic_pass1_callback(void *xarg, const void *pos, int type)
834{
835 const struct IOAPICENTRY *ioapic_ent;
836
837 switch (type) {
838 case 1: /* bus_entry */
839 ++mp_nbusses;
840 break;
841
842 case 2: /* io_apic_entry */
843 ioapic_ent = pos;
844 if (ioapic_ent->apic_flags & IOAPICENTRY_FLAG_EN) {
845 io_apic_address[mp_napics++] =
846 (vm_offset_t)ioapic_ent->apic_address;
847 }
848 break;
849
850 case 3: /* int_entry */
851 ++nintrs;
852 break;
853 }
854 return 0;
855}
856
984263bc
MD		 857/*
858 * 1st pass on motherboard's Intel MP specification table.
859 *
984263bc 860 * determines:
984263bc 861 * io_apic_address[N]
984263bc
MD		 862 * mp_nbusses
863 * mp_napics
864 * nintrs
865 */
866static void
981bebd1 867mptable_pass1(struct mptable_pos *mpt)
984263bc 868{
981bebd1 869 mpfps_t fps;
8658b5be 870 int x;
984263bc
MD		 871
872 POSTCODE(MPTABLE_PASS1_POST);
873
981bebd1
SZ		 874 fps = mpt->mp_fps;
875 KKASSERT(fps != NULL);
f9c3b04f 876
984263bc 877 /* clear various tables */
8658b5be 878 for (x = 0; x < NAPICID; ++x)
984263bc 879 io_apic_address[x] = ~0; /* IO APIC address table */
984263bc 880
4f6a8b30 881 mp_nbusses = 0;
984263bc
MD		 882 mp_napics = 0;
883 nintrs = 0;
984263bc
MD		 884
885 /* check for use of 'default' configuration */
981bebd1 886 if (fps->mpfb1 != 0) {
984263bc 887 io_apic_address[0] = DEFAULT_IO_APIC_BASE;
4f6a8b30 888 mp_nbusses = default_data[fps->mpfb1 - 1][0];
984263bc
MD		 889 mp_napics = 1;
890 nintrs = 16;
8658b5be
SZ		 891 } else {
892 int error;
893
894 error = mptable_iterate_entries(mpt->mp_cth,
895 mptable_ioapic_pass1_callback, NULL);
896 if (error)
897 panic("mptable_iterate_entries(ioapic_pass1) failed\n");
984263bc 898 }
984263bc
MD		 899}
900
c4717d5c
SZ		 901struct mptable_ioapic2_cbarg {
902 int bus;
903 int apic;
904 int intr;
905};
906
907static int
908mptable_ioapic_pass2_callback(void *xarg, const void *pos, int type)
909{
910 struct mptable_ioapic2_cbarg *arg = xarg;
911
912 switch (type) {
913 case 1:
914 if (bus_entry(pos, arg->bus))
915 ++arg->bus;
916 break;
917
918 case 2:
919 if (io_apic_entry(pos, arg->apic))
920 ++arg->apic;
921 break;
922
923 case 3:
924 if (int_entry(pos, arg->intr))
925 ++arg->intr;
926 break;
927 }
928 return 0;
929}
930
984263bc
MD		 931/*
932 * 2nd pass on motherboard's Intel MP specification table.
933 *
934 * sets:
984263bc 935 * ID_TO_IO(N), phy APIC ID to log CPU/IO table
984263bc
MD		 936 * IO_TO_ID(N), logical IO to APIC ID table
937 * bus_data[N]
938 * io_apic_ints[N]
939 */
390b18b0 940static void
981bebd1 941mptable_pass2(struct mptable_pos *mpt)
984263bc 942{
c4717d5c 943 struct mptable_ioapic2_cbarg arg;
981bebd1 944 mpfps_t fps;
c4717d5c 945 int error, x;
984263bc
MD		 946
947 POSTCODE(MPTABLE_PASS2_POST);
948
981bebd1
SZ		 949 fps = mpt->mp_fps;
950 KKASSERT(fps != NULL);
951
984263bc
MD		 952 MALLOC(io_apic_versions, u_int32_t *, sizeof(u_int32_t) * mp_napics,
953 M_DEVBUF, M_WAITOK);
954 MALLOC(ioapic, volatile ioapic_t **, sizeof(ioapic_t *) * mp_napics,
e7b4468c 955 M_DEVBUF, M_WAITOK | M_ZERO);
1439c090 956 MALLOC(io_apic_ints, io_int *, sizeof(io_int) * (nintrs + FIXUP_EXTRA_APIC_INTS),
984263bc
MD		 957 M_DEVBUF, M_WAITOK);
958 MALLOC(bus_data, bus_datum *, sizeof(bus_datum) * mp_nbusses,
959 M_DEVBUF, M_WAITOK);
960
c4717d5c
SZ		 961 for (x = 0; x < mp_napics; x++)
962 ioapic[x] = permanent_io_mapping(io_apic_address[x]);
984263bc
MD		 963
964 /* clear various tables */
965 for (x = 0; x < NAPICID; ++x) {
97359a5b 966 ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */
984263bc
MD		 967 IO_TO_ID(x) = -1; /* logical IO to APIC ID table */
968 }
969
970 /* clear bus data table */
971 for (x = 0; x < mp_nbusses; ++x)
972 bus_data[x].bus_id = 0xff;
973
974 /* clear IO APIC INT table */
90a567fb 975 for (x = 0; x < nintrs + FIXUP_EXTRA_APIC_INTS; ++x) {
984263bc
MD		 976 io_apic_ints[x].int_type = 0xff;
977 io_apic_ints[x].int_vector = 0xff;
978 }
979
984263bc 980 /* check for use of 'default' configuration */
390b18b0
SZ		 981 if (fps->mpfb1 != 0) {
982 mptable_default(fps->mpfb1);
983 return;
984 }
984263bc 985
c4717d5c
SZ		 986 bzero(&arg, sizeof(arg));
987 error = mptable_iterate_entries(mpt->mp_cth,
988 mptable_ioapic_pass2_callback, &arg);
989 if (error)
990 panic("mptable_iterate_entries(ioapic_pass2) failed\n");
984263bc
MD		 991}
992
984263bc
MD		 993/*
994 * Check if we should perform a hyperthreading "fix-up" to
995 * enumerate any logical CPU's that aren't already listed
996 * in the table.
997 *
998 * XXX: We assume that all of the physical CPUs in the
999 * system have the same number of logical CPUs.
1000 *
1001 * XXX: We assume that APIC ID's are allocated such that
1002 * the APIC ID's for a physical processor are aligned
1003 * with the number of logical CPU's in the processor.
1004 */
44c36320 1005static int
da23a592 1006mptable_hyperthread_fixup(cpumask_t id_mask, int cpu_count)
984263bc 1007{
44c36320 1008 int i, id, lcpus_max, logical_cpus;
984263bc 1009
984263bc 1010 if ((cpu_feature & CPUID_HTT) == 0)
44c36320 1011 return 0;
7ea07fd2
SZ		 1012
1013 lcpus_max = (cpu_procinfo & CPUID_HTT_CORES) >> 16;
1014 if (lcpus_max <= 1)
44c36320 1015 return 0;
984263bc 1016
90e8a35b 1017 if (cpu_vendor_id == CPU_VENDOR_INTEL) {
7ea07fd2
SZ		 1018 /*
1019 * INSTRUCTION SET REFERENCE, A-M (#253666)
1020 * Page 3-181, Table 3-20
1021 * "The nearest power-of-2 integer that is not smaller
1022 * than EBX[23:16] is the number of unique initial APIC
1023 * IDs reserved for addressing different logical
1024 * processors in a physical package."
1025 */
1026 for (i = 0; ; ++i) {
1027 if ((1 << i) >= lcpus_max) {
1028 lcpus_max = 1 << i;
1029 break;
1030 }
1031 }
1032 }
1033
44c36320
SZ		 1034 KKASSERT(cpu_count != 0);
1035 if (cpu_count == lcpus_max) {
7ea07fd2 1036 /* We have nothing to fix */
44c36320
SZ		 1037 return 0;
1038 } else if (cpu_count == 1) {
7ea07fd2
SZ		 1039 /* XXX this may be incorrect */
1040 logical_cpus = lcpus_max;
1041 } else {
1042 int cur, prev, dist;
1043
1044 /*
1045 * Calculate the distances between two nearest
1046 * APIC IDs. If all such distances are same,
1047 * then it is the number of missing cpus that
1048 * we are going to fill later.
1049 */
1050 dist = cur = prev = -1;
1051 for (id = 0; id < MAXCPU; ++id) {
da23a592 1052 if ((id_mask & CPUMASK(id)) == 0)
7ea07fd2
SZ		 1053 continue;
1054
1055 cur = id;
1056 if (prev >= 0) {
1057 int new_dist = cur - prev;
1058
1059 if (dist < 0)
1060 dist = new_dist;
1061
1062 /*
1063 * Make sure that all distances
1064 * between two nearest APIC IDs
1065 * are same.
1066 */
1067 if (dist != new_dist)
44c36320 1068 return 0;
7ea07fd2
SZ		 1069 }
1070 prev = cur;
1071 }
1072 if (dist == 1)
44c36320 1073 return 0;
7ea07fd2
SZ		 1074
1075 /* Must be power of 2 */
1076 if (dist & (dist - 1))
44c36320 1077 return 0;
7ea07fd2
SZ		 1078
1079 /* Can't exceed CPU package capacity */
1080 if (dist > lcpus_max)
1081 logical_cpus = lcpus_max;
1082 else
1083 logical_cpus = dist;
1084 }
1085
984263bc
MD		 1086 /*
1087 * For each APIC ID of a CPU that is set in the mask,
1088 * scan the other candidate APIC ID's for this
1089 * physical processor. If any of those ID's are
1090 * already in the table, then kill the fixup.
1091 */
7ea07fd2 1092 for (id = 0; id < MAXCPU; id++) {
da23a592 1093 if ((id_mask & CPUMASK(id)) == 0)
984263bc
MD		 1094 continue;
1095 /* First, make sure we are on a logical_cpus boundary. */
1096 if (id % logical_cpus != 0)
44c36320 1097 return 0;
984263bc 1098 for (i = id + 1; i < id + logical_cpus; i++)
da23a592 1099 if ((id_mask & CPUMASK(i)) != 0)
44c36320 1100 return 0;
984263bc 1101 }
44c36320 1102 return logical_cpus;
984263bc 1103}
984263bc 1104
fa058384 1105static int
981bebd1
SZ		 1106mptable_map(struct mptable_pos *mpt, vm_paddr_t mpfps_paddr)
1107{
1108 mpfps_t fps = NULL;
1109 mpcth_t cth = NULL;
1110 vm_size_t cth_mapsz = 0;
1111
fa058384
SZ		 1112 bzero(mpt, sizeof(*mpt));
1113
981bebd1
SZ		 1114 fps = pmap_mapdev(mpfps_paddr, sizeof(*fps));
1115 if (fps->pap != 0) {
1116 /*
1117 * Map configuration table header to get
1118 * the base table size
1119 */
1120 cth = pmap_mapdev(fps->pap, sizeof(*cth));
1121 cth_mapsz = cth->base_table_length;
1122 pmap_unmapdev((vm_offset_t)cth, sizeof(*cth));
1123
fa058384
SZ		 1124 if (cth_mapsz < sizeof(*cth)) {
1125 kprintf("invalid base MP table length %d\n",
1126 (int)cth_mapsz);
1127 pmap_unmapdev((vm_offset_t)fps, sizeof(*fps));
1128 return EINVAL;
1129 }
1130
981bebd1
SZ		 1131 /*
1132 * Map the base table
1133 */
1134 cth = pmap_mapdev(fps->pap, cth_mapsz);
1135 }
1136
1137 mpt->mp_fps = fps;
1138 mpt->mp_cth = cth;
1139 mpt->mp_cth_mapsz = cth_mapsz;
fa058384
SZ		 1140
1141 return 0;
981bebd1
SZ		 1142}
1143
1144static void
1145mptable_unmap(struct mptable_pos *mpt)
1146{
1147 if (mpt->mp_cth != NULL) {
1148 pmap_unmapdev((vm_offset_t)mpt->mp_cth, mpt->mp_cth_mapsz);
1149 mpt->mp_cth = NULL;
1150 mpt->mp_cth_mapsz = 0;
1151 }
1152 if (mpt->mp_fps != NULL) {
1153 pmap_unmapdev((vm_offset_t)mpt->mp_fps, sizeof(*mpt->mp_fps));
1154 mpt->mp_fps = NULL;
1155 }
1156}
1157
984263bc
MD		 1158void
1159assign_apic_irq(int apic, int intpin, int irq)
1160{
1161 int x;
1162
1163 if (int_to_apicintpin[irq].ioapic != -1)
1164 panic("assign_apic_irq: inconsistent table");
1165
1166 int_to_apicintpin[irq].ioapic = apic;
1167 int_to_apicintpin[irq].int_pin = intpin;
1168 int_to_apicintpin[irq].apic_address = ioapic[apic];
1169 int_to_apicintpin[irq].redirindex = IOAPIC_REDTBL + 2 * intpin;
1170
1171 for (x = 0; x < nintrs; x++) {
1172 if ((io_apic_ints[x].int_type == 0 ||
1173 io_apic_ints[x].int_type == 3) &&
1174 io_apic_ints[x].int_vector == 0xff &&
1175 io_apic_ints[x].dst_apic_id == IO_TO_ID(apic) &&
1176 io_apic_ints[x].dst_apic_int == intpin)
1177 io_apic_ints[x].int_vector = irq;
1178 }
1179}
1180
1181void
1182revoke_apic_irq(int irq)
1183{
1184 int x;
1185 int oldapic;
1186 int oldintpin;
1187
1188 if (int_to_apicintpin[irq].ioapic == -1)
1189 panic("revoke_apic_irq: inconsistent table");
1190
1191 oldapic = int_to_apicintpin[irq].ioapic;
1192 oldintpin = int_to_apicintpin[irq].int_pin;
1193
1194 int_to_apicintpin[irq].ioapic = -1;
1195 int_to_apicintpin[irq].int_pin = 0;
1196 int_to_apicintpin[irq].apic_address = NULL;
1197 int_to_apicintpin[irq].redirindex = 0;
1198
1199 for (x = 0; x < nintrs; x++) {
1200 if ((io_apic_ints[x].int_type == 0 ||
1201 io_apic_ints[x].int_type == 3) &&
1202 io_apic_ints[x].int_vector != 0xff &&
1203 io_apic_ints[x].dst_apic_id == IO_TO_ID(oldapic) &&
1204 io_apic_ints[x].dst_apic_int == oldintpin)
1205 io_apic_ints[x].int_vector = 0xff;
1206 }
1207}
1208
1439c090
MD		 1209/*
1210 * Allocate an IRQ
1211 */
984263bc
MD		 1212static void
1213allocate_apic_irq(int intr)
1214{
1215 int apic;
1216 int intpin;
1217 int irq;
1218
1219 if (io_apic_ints[intr].int_vector != 0xff)
1220 return; /* Interrupt handler already assigned */
1221
1222 if (io_apic_ints[intr].int_type != 0 &&
1223 (io_apic_ints[intr].int_type != 3 ||
1224 (io_apic_ints[intr].dst_apic_id == IO_TO_ID(0) &&
1225 io_apic_ints[intr].dst_apic_int == 0)))
1226 return; /* Not INT or ExtInt on != (0, 0) */
1227
1228 irq = 0;
1229 while (irq < APIC_INTMAPSIZE &&
1230 int_to_apicintpin[irq].ioapic != -1)
1231 irq++;
1232
1233 if (irq >= APIC_INTMAPSIZE)
1234 return; /* No free interrupt handlers */
1235
1236 apic = ID_TO_IO(io_apic_ints[intr].dst_apic_id);
1237 intpin = io_apic_ints[intr].dst_apic_int;
1238
1239 assign_apic_irq(apic, intpin, irq);
984263bc
MD		 1240}
1241
1242
1243static void
1244swap_apic_id(int apic, int oldid, int newid)
1245{
1246 int x;
1247 int oapic;
1248
1249
1250 if (oldid == newid)
1251 return; /* Nothing to do */
1252
26be20a0 1253 kprintf("Changing APIC ID for IO APIC #%d from %d to %d in MP table\n",
984263bc
MD		 1254 apic, oldid, newid);
1255
1256 /* Swap physical APIC IDs in interrupt entries */
1257 for (x = 0; x < nintrs; x++) {
1258 if (io_apic_ints[x].dst_apic_id == oldid)
1259 io_apic_ints[x].dst_apic_id = newid;
1260 else if (io_apic_ints[x].dst_apic_id == newid)
1261 io_apic_ints[x].dst_apic_id = oldid;
1262 }
1263
1264 /* Swap physical APIC IDs in IO_TO_ID mappings */
1265 for (oapic = 0; oapic < mp_napics; oapic++)
1266 if (IO_TO_ID(oapic) == newid)
1267 break;
1268
1269 if (oapic < mp_napics) {
26be20a0 1270 kprintf("Changing APIC ID for IO APIC #%d from "
984263bc
MD		 1271 "%d to %d in MP table\n",
1272 oapic, newid, oldid);
1273 IO_TO_ID(oapic) = oldid;
1274 }
1275 IO_TO_ID(apic) = newid;
1276}
1277
1278
1279static void
1280fix_id_to_io_mapping(void)
1281{
1282 int x;
1283
1284 for (x = 0; x < NAPICID; x++)
1285 ID_TO_IO(x) = -1;
1286
1287 for (x = 0; x <= mp_naps; x++)
1288 if (CPU_TO_ID(x) < NAPICID)
1289 ID_TO_IO(CPU_TO_ID(x)) = x;
1290
1291 for (x = 0; x < mp_napics; x++)
1292 if (IO_TO_ID(x) < NAPICID)
1293 ID_TO_IO(IO_TO_ID(x)) = x;
1294}
1295
1296
1297static int
1298first_free_apic_id(void)
1299{
1300 int freeid, x;
1301
1302 for (freeid = 0; freeid < NAPICID; freeid++) {
1303 for (x = 0; x <= mp_naps; x++)
1304 if (CPU_TO_ID(x) == freeid)
1305 break;
1306 if (x <= mp_naps)
1307 continue;
1308 for (x = 0; x < mp_napics; x++)
1309 if (IO_TO_ID(x) == freeid)
1310 break;
1311 if (x < mp_napics)
1312 continue;
1313 return freeid;
1314 }
1315 return freeid;
1316}
1317
1318
1319static int
1320io_apic_id_acceptable(int apic, int id)
1321{
1322 int cpu; /* Logical CPU number */
1323 int oapic; /* Logical IO APIC number for other IO APIC */
1324
1325 if (id >= NAPICID)
1326 return 0; /* Out of range */
1327
1328 for (cpu = 0; cpu <= mp_naps; cpu++)
1329 if (CPU_TO_ID(cpu) == id)
1330 return 0; /* Conflict with CPU */
1331
1332 for (oapic = 0; oapic < mp_napics && oapic < apic; oapic++)
1333 if (IO_TO_ID(oapic) == id)
1334 return 0; /* Conflict with other APIC */
1335
1336 return 1; /* ID is acceptable for IO APIC */
1337}
1338
1439c090
MD		 1339static
1340io_int *
1341io_apic_find_int_entry(int apic, int pin)
1342{
1343 int x;
1344
1345 /* search each of the possible INTerrupt sources */
1346 for (x = 0; x < nintrs; ++x) {
1347 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1348 (pin == io_apic_ints[x].dst_apic_int))
1349 return (&io_apic_ints[x]);
1350 }
1351 return NULL;
1352}
1353
984263bc
MD		 1354/*
1355 * parse an Intel MP specification table
1356 */
1357static void
3aba8f73 1358mptable_fix(void)
984263bc 1359{
4f6a8b30 1360 int x;
984263bc 1361 int id;
984263bc
MD		 1362 int apic; /* IO APIC unit number */
1363 int freeid; /* Free physical APIC ID */
1364 int physid; /* Current physical IO APIC ID */
1439c090 1365 io_int *io14;
97359a5b
MD		 1366 int bus_0 = 0; /* Stop GCC warning */
1367 int bus_pci = 0; /* Stop GCC warning */
1368 int num_pci_bus;
984263bc
MD		 1369
1370 /*
1371 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS
1372 * did it wrong. The MP spec says that when more than 1 PCI bus
1373 * exists the BIOS must begin with bus entries for the PCI bus and use
1374 * actual PCI bus numbering. This implies that when only 1 PCI bus
1375 * exists the BIOS can choose to ignore this ordering, and indeed many
1376 * MP motherboards do ignore it. This causes a problem when the PCI
1377 * sub-system makes requests of the MP sub-system based on PCI bus
1378 * numbers. So here we look for the situation and renumber the
1379 * busses and associated INTs in an effort to "make it right".
1380 */
1381
1382 /* find bus 0, PCI bus, count the number of PCI busses */
1383 for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) {
1384 if (bus_data[x].bus_id == 0) {
1385 bus_0 = x;
1386 }
1387 if (bus_data[x].bus_type == PCI) {
1388 ++num_pci_bus;
1389 bus_pci = x;
1390 }
1391 }
1392 /*
1393 * bus_0 == slot of bus with ID of 0
1394 * bus_pci == slot of last PCI bus encountered
1395 */
1396
1397 /* check the 1 PCI bus case for sanity */
1398 /* if it is number 0 all is well */
1399 if (num_pci_bus == 1 &&
1400 bus_data[bus_pci].bus_id != 0) {
1401
1402 /* mis-numbered, swap with whichever bus uses slot 0 */
1403
1404 /* swap the bus entry types */
1405 bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type;
1406 bus_data[bus_0].bus_type = PCI;
1407
1408 /* swap each relavant INTerrupt entry */
1409 id = bus_data[bus_pci].bus_id;
1410 for (x = 0; x < nintrs; ++x) {
1411 if (io_apic_ints[x].src_bus_id == id) {
1412 io_apic_ints[x].src_bus_id = 0;
1413 }
1414 else if (io_apic_ints[x].src_bus_id == 0) {
1415 io_apic_ints[x].src_bus_id = id;
1416 }
1417 }
1418 }
1419
1420 /* Assign IO APIC IDs.
1421 *
1422 * First try the existing ID. If a conflict is detected, try
1423 * the ID in the MP table. If a conflict is still detected, find
1424 * a free id.
1425 *
1426 * We cannot use the ID_TO_IO table before all conflicts has been
1427 * resolved and the table has been corrected.
1428 */
1429 for (apic = 0; apic < mp_napics; ++apic) { /* For all IO APICs */
1430
1431 /* First try to use the value set by the BIOS */
1432 physid = io_apic_get_id(apic);
1433 if (io_apic_id_acceptable(apic, physid)) {
1434 if (IO_TO_ID(apic) != physid)
1435 swap_apic_id(apic, IO_TO_ID(apic), physid);
1436 continue;
1437 }
1438
1439 /* Then check if the value in the MP table is acceptable */
1440 if (io_apic_id_acceptable(apic, IO_TO_ID(apic)))
1441 continue;
1442
1443 /* Last resort, find a free APIC ID and use it */
1444 freeid = first_free_apic_id();
1445 if (freeid >= NAPICID)
1446 panic("No free physical APIC IDs found");
1447
1448 if (io_apic_id_acceptable(apic, freeid)) {
1449 swap_apic_id(apic, IO_TO_ID(apic), freeid);
1450 continue;
1451 }
1452 panic("Free physical APIC ID not usable");
1453 }
1454 fix_id_to_io_mapping();
1455
1456 /* detect and fix broken Compaq MP table */
1457 if (apic_int_type(0, 0) == -1) {
26be20a0 1458 kprintf("APIC_IO: MP table broken: 8259->APIC entry missing!\n");
984263bc
MD		 1459 io_apic_ints[nintrs].int_type = 3; /* ExtInt */
1460 io_apic_ints[nintrs].int_vector = 0xff; /* Unassigned */
1461 /* XXX fixme, set src bus id etc, but it doesn't seem to hurt */
1462 io_apic_ints[nintrs].dst_apic_id = IO_TO_ID(0);
1463 io_apic_ints[nintrs].dst_apic_int = 0; /* Pin 0 */
1464 nintrs++;
ad12f88c 1465 } else if (apic_int_type(0, 0) == 0) {
26be20a0 1466 kprintf("APIC_IO: MP table broken: ExtINT entry corrupt!\n");
ad12f88c
HP		 1467 for (x = 0; x < nintrs; ++x)
1468 if ((0 == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1469 (0 == io_apic_ints[x].dst_apic_int)) {
1470 io_apic_ints[x].int_type = 3;
1471 io_apic_ints[x].int_vector = 0xff;
1472 break;
1473 }
984263bc 1474 }
1439c090
MD		 1475
1476 /*
1477 * Fix missing IRQ 15 when IRQ 14 is an ISA interrupt. IDE
1478 * controllers universally come in pairs. If IRQ 14 is specified
1479 * as an ISA interrupt, then IRQ 15 had better be too.
1480 *
1481 * [ Shuttle XPC / AMD Athlon X2 ]
1482 * The MPTable is missing an entry for IRQ 15. Note that the
1483 * ACPI table has an entry for both 14 and 15.
1484 */
1485 if (apic_int_type(0, 14) == 0 && apic_int_type(0, 15) == -1) {
26be20a0 1486 kprintf("APIC_IO: MP table broken: IRQ 15 not ISA when IRQ 14 is!\n");
1439c090
MD		 1487 io14 = io_apic_find_int_entry(0, 14);
1488 io_apic_ints[nintrs] = *io14;
1489 io_apic_ints[nintrs].src_bus_irq = 15;
1490 io_apic_ints[nintrs].dst_apic_int = 15;
1491 nintrs++;
1492 }
984263bc
MD		 1493}
1494
984263bc
MD		 1495/* Assign low level interrupt handlers */
1496static void
1497setup_apic_irq_mapping(void)
1498{
1499 int x;
1500 int int_vector;
1501
1502 /* Clear array */
1503 for (x = 0; x < APIC_INTMAPSIZE; x++) {
1504 int_to_apicintpin[x].ioapic = -1;
1505 int_to_apicintpin[x].int_pin = 0;
1506 int_to_apicintpin[x].apic_address = NULL;
1507 int_to_apicintpin[x].redirindex = 0;
ea689d1c
SZ		 1508
1509 /* Default to masked */
1510 int_to_apicintpin[x].flags = IOAPIC_IM_FLAG_MASKED;
984263bc
MD		 1511 }
1512
1513 /* First assign ISA/EISA interrupts */
1514 for (x = 0; x < nintrs; x++) {
1515 int_vector = io_apic_ints[x].src_bus_irq;
1516 if (int_vector < APIC_INTMAPSIZE &&
1517 io_apic_ints[x].int_vector == 0xff &&
1518 int_to_apicintpin[int_vector].ioapic == -1 &&
1519 (apic_int_is_bus_type(x, ISA) ||
1520 apic_int_is_bus_type(x, EISA)) &&
1521 io_apic_ints[x].int_type == 0) {
1522 assign_apic_irq(ID_TO_IO(io_apic_ints[x].dst_apic_id),
1523 io_apic_ints[x].dst_apic_int,
1524 int_vector);
1525 }
1526 }
1527
1528 /* Assign ExtInt entry if no ISA/EISA interrupt 0 entry */
1529 for (x = 0; x < nintrs; x++) {
1530 if (io_apic_ints[x].dst_apic_int == 0 &&
1531 io_apic_ints[x].dst_apic_id == IO_TO_ID(0) &&
1532 io_apic_ints[x].int_vector == 0xff &&
1533 int_to_apicintpin[0].ioapic == -1 &&
1534 io_apic_ints[x].int_type == 3) {
1535 assign_apic_irq(0, 0, 0);
1536 break;
1537 }
1538 }
0956149b
SZ		 1539
1540 /* Assign PCI interrupts */
1541 for (x = 0; x < nintrs; ++x) {
1542 if (io_apic_ints[x].int_type == 0 &&
1543 io_apic_ints[x].int_vector == 0xff &&
1544 apic_int_is_bus_type(x, PCI))
1545 allocate_apic_irq(x);
1546 }
984263bc
MD		 1547}
1548
a9112655
SZ		 1549void
1550mp_set_cpuids(int cpu_id, int apic_id)
1551{
1552 CPU_TO_ID(cpu_id) = apic_id;
1553 ID_TO_CPU(apic_id) = cpu_id;
1554}
1555
984263bc 1556static int
a0873f07 1557processor_entry(const struct PROCENTRY *entry, int cpu)
984263bc 1558{
bd8aa7e2
SZ		 1559 KKASSERT(cpu > 0);
1560
984263bc
MD		 1561 /* check for usability */
1562 if (!(entry->cpu_flags & PROCENTRY_FLAG_EN))
1563 return 0;
1564
984263bc
MD		 1565 /* check for BSP flag */
1566 if (entry->cpu_flags & PROCENTRY_FLAG_BP) {
a9112655 1567 mp_set_cpuids(0, entry->apic_id);
984263bc
MD		 1568 return 0; /* its already been counted */
1569 }
1570
1571 /* add another AP to list, if less than max number of CPUs */
1572 else if (cpu < MAXCPU) {
a9112655 1573 mp_set_cpuids(cpu, entry->apic_id);
984263bc
MD		 1574 return 1;
1575 }
1576
1577 return 0;
1578}
1579
984263bc 1580static int
c4717d5c 1581bus_entry(const struct BUSENTRY *entry, int bus)
984263bc
MD		 1582{
1583 int x;
1584 char c, name[8];
1585
1586 /* encode the name into an index */
1587 for (x = 0; x < 6; ++x) {
1588 if ((c = entry->bus_type[x]) == ' ')
1589 break;
1590 name[x] = c;
1591 }
1592 name[x] = '\0';
1593
1594 if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE)
1595 panic("unknown bus type: '%s'", name);
1596
1597 bus_data[bus].bus_id = entry->bus_id;
1598 bus_data[bus].bus_type = x;
1599
1600 return 1;
1601}
1602
984263bc 1603static int
c4717d5c 1604io_apic_entry(const struct IOAPICENTRY *entry, int apic)
984263bc
MD		 1605{
1606 if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN))
1607 return 0;
1608
1609 IO_TO_ID(apic) = entry->apic_id;
c163176b 1610 ID_TO_IO(entry->apic_id) = apic;
984263bc
MD		 1611
1612 return 1;
1613}
1614
984263bc
MD		 1615static int
1616lookup_bus_type(char *name)
1617{
1618 int x;
1619
1620 for (x = 0; x < MAX_BUSTYPE; ++x)
1621 if (strcmp(bus_type_table[x].name, name) == 0)
1622 return bus_type_table[x].type;
1623
1624 return UNKNOWN_BUSTYPE;
1625}
1626
984263bc 1627static int
c4717d5c 1628int_entry(const struct INTENTRY *entry, int intr)
984263bc
MD		 1629{
1630 int apic;
1631
1632 io_apic_ints[intr].int_type = entry->int_type;
1633 io_apic_ints[intr].int_flags = entry->int_flags;
1634 io_apic_ints[intr].src_bus_id = entry->src_bus_id;
1635 io_apic_ints[intr].src_bus_irq = entry->src_bus_irq;
1636 if (entry->dst_apic_id == 255) {
1637 /* This signal goes to all IO APICS. Select an IO APIC
1638 with sufficient number of interrupt pins */
1639 for (apic = 0; apic < mp_napics; apic++)
1640 if (((io_apic_read(apic, IOAPIC_VER) &
1641 IOART_VER_MAXREDIR) >> MAXREDIRSHIFT) >=
1642 entry->dst_apic_int)
1643 break;
1644 if (apic < mp_napics)
1645 io_apic_ints[intr].dst_apic_id = IO_TO_ID(apic);
1646 else
1647 io_apic_ints[intr].dst_apic_id = entry->dst_apic_id;
1648 } else
1649 io_apic_ints[intr].dst_apic_id = entry->dst_apic_id;
1650 io_apic_ints[intr].dst_apic_int = entry->dst_apic_int;
1651
1652 return 1;
1653}
1654
984263bc
MD		 1655static int
1656apic_int_is_bus_type(int intr, int bus_type)
1657{
1658 int bus;
1659
1660 for (bus = 0; bus < mp_nbusses; ++bus)
1661 if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id)
1662 && ((int) bus_data[bus].bus_type == bus_type))
1663 return 1;
1664
1665 return 0;
1666}
1667
984263bc
MD		 1668/*
1669 * Given a traditional ISA INT mask, return an APIC mask.
1670 */
1671u_int
1672isa_apic_mask(u_int isa_mask)
1673{
1674 int isa_irq;
1675 int apic_pin;
1676
1677#if defined(SKIP_IRQ15_REDIRECT)
1678 if (isa_mask == (1 << 15)) {
26be20a0 1679 kprintf("skipping ISA IRQ15 redirect\n");
984263bc
MD		 1680 return isa_mask;
1681 }
1682#endif /* SKIP_IRQ15_REDIRECT */
1683
1684 isa_irq = ffs(isa_mask); /* find its bit position */
1685 if (isa_irq == 0) /* doesn't exist */
1686 return 0;
1687 --isa_irq; /* make it zero based */
1688
1689 apic_pin = isa_apic_irq(isa_irq); /* look for APIC connection */
1690 if (apic_pin == -1)
1691 return 0;
1692
1693 return (1 << apic_pin); /* convert pin# to a mask */
1694}
1695
984263bc
MD		 1696/*
1697 * Determine which APIC pin an ISA/EISA INT is attached to.
1698 */
1699#define INTTYPE(I) (io_apic_ints[(I)].int_type)
1700#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int)
1701#define INTIRQ(I) (io_apic_ints[(I)].int_vector)
1702#define INTAPIC(I) (ID_TO_IO(io_apic_ints[(I)].dst_apic_id))
1703
1704#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq)
1705int
1706isa_apic_irq(int isa_irq)
1707{
1708 int intr;
1709
1710 for (intr = 0; intr < nintrs; ++intr) { /* check each record */
1711 if (INTTYPE(intr) == 0) { /* standard INT */
1712 if (SRCBUSIRQ(intr) == isa_irq) {
1713 if (apic_int_is_bus_type(intr, ISA) ||
1714 apic_int_is_bus_type(intr, EISA)) {
1715 if (INTIRQ(intr) == 0xff)
1716 return -1; /* unassigned */
1717 return INTIRQ(intr); /* found */
1718 }
1719 }
1720 }
1721 }
1722 return -1; /* NOT found */
1723}
1724
1725
1726/*
1727 * Determine which APIC pin a PCI INT is attached to.
1728 */
1729#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id)
1730#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f)
1731#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03)
1732int
1733pci_apic_irq(int pciBus, int pciDevice, int pciInt)
1734{
1735 int intr;
1736
1737 --pciInt; /* zero based */
1738
1439c090 1739 for (intr = 0; intr < nintrs; ++intr) { /* check each record */
984263bc
MD		 1740 if ((INTTYPE(intr) == 0) /* standard INT */
1741 && (SRCBUSID(intr) == pciBus)
1742 && (SRCBUSDEVICE(intr) == pciDevice)
1439c090 1743 && (SRCBUSLINE(intr) == pciInt)) { /* a candidate IRQ */
984263bc 1744 if (apic_int_is_bus_type(intr, PCI)) {
0956149b
SZ		 1745 if (INTIRQ(intr) == 0xff) {
1746 kprintf("IOAPIC: pci_apic_irq() "
1747 "failed\n");
984263bc 1748 return -1; /* unassigned */
0956149b 1749 }
984263bc
MD		 1750 return INTIRQ(intr); /* exact match */
1751 }
1439c090
MD		 1752 }
1753 }
984263bc
MD		 1754
1755 return -1; /* NOT found */
1756}
1757
1758int
1759next_apic_irq(int irq)
1760{
1761 int intr, ointr;
1762 int bus, bustype;
1763
1764 bus = 0;
1765 bustype = 0;
1766 for (intr = 0; intr < nintrs; intr++) {
1767 if (INTIRQ(intr) != irq || INTTYPE(intr) != 0)
1768 continue;
1769 bus = SRCBUSID(intr);
1770 bustype = apic_bus_type(bus);
1771 if (bustype != ISA &&
1772 bustype != EISA &&
1773 bustype != PCI)
1774 continue;
1775 break;
1776 }
1777 if (intr >= nintrs) {
1778 return -1;
1779 }
1780 for (ointr = intr + 1; ointr < nintrs; ointr++) {
1781 if (INTTYPE(ointr) != 0)
1782 continue;
1783 if (bus != SRCBUSID(ointr))
1784 continue;
1785 if (bustype == PCI) {
1786 if (SRCBUSDEVICE(intr) != SRCBUSDEVICE(ointr))
1787 continue;
1788 if (SRCBUSLINE(intr) != SRCBUSLINE(ointr))
1789 continue;
1790 }
1791 if (bustype == ISA || bustype == EISA) {
1792 if (SRCBUSIRQ(intr) != SRCBUSIRQ(ointr))
1793 continue;
1794 }
1795 if (INTPIN(intr) == INTPIN(ointr))
1796 continue;
1797 break;
1798 }
1799 if (ointr >= nintrs) {
1800 return -1;
1801 }
1802 return INTIRQ(ointr);
1803}
1804#undef SRCBUSLINE
1805#undef SRCBUSDEVICE
1806#undef SRCBUSID
1807#undef SRCBUSIRQ
1808
1809#undef INTPIN
1810#undef INTIRQ
1811#undef INTAPIC
1812#undef INTTYPE
1813
984263bc
MD		 1814/*
1815 * Reprogram the MB chipset to NOT redirect an ISA INTerrupt.
1816 *
1817 * XXX FIXME:
1818 * Exactly what this means is unclear at this point. It is a solution
1819 * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard
1820 * could route any of the ISA INTs to upper (>15) IRQ values. But most would
1821 * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an
1822 * option.
1823 */
1824int
1825undirect_isa_irq(int rirq)
1826{
1827#if defined(READY)
1828 if (bootverbose)
26be20a0 1829 kprintf("Freeing redirected ISA irq %d.\n", rirq);
984263bc 1830 /** FIXME: tickle the MB redirector chip */
c044141b 1831 return /* XXX */;
984263bc
MD		 1832#else
1833 if (bootverbose)
26be20a0 1834 kprintf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq);
984263bc
MD		 1835 return 0;
1836#endif /* READY */
1837}
1838
1839
1840/*
1841 * Reprogram the MB chipset to NOT redirect a PCI INTerrupt
1842 */
1843int
1844undirect_pci_irq(int rirq)
1845{
1846#if defined(READY)
1847 if (bootverbose)
26be20a0 1848 kprintf("Freeing redirected PCI irq %d.\n", rirq);
984263bc
MD		 1849
1850 /** FIXME: tickle the MB redirector chip */
c044141b 1851 return /* XXX */;
984263bc
MD		 1852#else
1853 if (bootverbose)
26be20a0 1854 kprintf("Freeing (NOT implemented) redirected PCI irq %d.\n",
984263bc
MD		 1855 rirq);
1856 return 0;
1857#endif /* READY */
1858}
1859
1860
1861/*
1862 * given a bus ID, return:
1863 * the bus type if found
1864 * -1 if NOT found
1865 */
1866int
1867apic_bus_type(int id)
1868{
1869 int x;
1870
1871 for (x = 0; x < mp_nbusses; ++x)
1872 if (bus_data[x].bus_id == id)
1873 return bus_data[x].bus_type;
1874
1875 return -1;
1876}
1877
984263bc
MD		 1878/*
1879 * given a LOGICAL APIC# and pin#, return:
1880 * the associated src bus ID if found
1881 * -1 if NOT found
1882 */
1883int
1884apic_src_bus_id(int apic, int pin)
1885{
1886 int x;
1887
1888 /* search each of the possible INTerrupt sources */
1889 for (x = 0; x < nintrs; ++x)
1890 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1891 (pin == io_apic_ints[x].dst_apic_int))
1892 return (io_apic_ints[x].src_bus_id);
1893
1894 return -1; /* NOT found */
1895}
1896
984263bc
MD		 1897/*
1898 * given a LOGICAL APIC# and pin#, return:
1899 * the associated src bus IRQ if found
1900 * -1 if NOT found
1901 */
1902int
1903apic_src_bus_irq(int apic, int pin)
1904{
1905 int x;
1906
1907 for (x = 0; x < nintrs; x++)
1908 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1909 (pin == io_apic_ints[x].dst_apic_int))
1910 return (io_apic_ints[x].src_bus_irq);
1911
1912 return -1; /* NOT found */
1913}
1914
1915
1916/*
1917 * given a LOGICAL APIC# and pin#, return:
1918 * the associated INTerrupt type if found
1919 * -1 if NOT found
1920 */
1921int
1922apic_int_type(int apic, int pin)
1923{
1924 int x;
1925
1926 /* search each of the possible INTerrupt sources */
1439c090 1927 for (x = 0; x < nintrs; ++x) {
984263bc
MD		 1928 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1929 (pin == io_apic_ints[x].dst_apic_int))
1930 return (io_apic_ints[x].int_type);
1439c090 1931 }
984263bc
MD		 1932 return -1; /* NOT found */
1933}
1934
1439c090
MD		 1935/*
1936 * Return the IRQ associated with an APIC pin
1937 */
984263bc
MD		 1938int
1939apic_irq(int apic, int pin)
1940{
1941 int x;
1942 int res;
1943
1439c090 1944 for (x = 0; x < nintrs; ++x) {
984263bc
MD		 1945 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1946 (pin == io_apic_ints[x].dst_apic_int)) {
1947 res = io_apic_ints[x].int_vector;
1948 if (res == 0xff)
1949 return -1;
1950 if (apic != int_to_apicintpin[res].ioapic)
1439c090 1951 panic("apic_irq: inconsistent table %d/%d", apic, int_to_apicintpin[res].ioapic);
984263bc
MD		 1952 if (pin != int_to_apicintpin[res].int_pin)
1953 panic("apic_irq inconsistent table (2)");
1954 return res;
1955 }
1439c090 1956 }
984263bc
MD		 1957 return -1;
1958}
1959
1960
1961/*
1962 * given a LOGICAL APIC# and pin#, return:
1963 * the associated trigger mode if found
1964 * -1 if NOT found
1965 */
1966int
1967apic_trigger(int apic, int pin)
1968{
1969 int x;
1970
1971 /* search each of the possible INTerrupt sources */
1972 for (x = 0; x < nintrs; ++x)
1973 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1974 (pin == io_apic_ints[x].dst_apic_int))
1975 return ((io_apic_ints[x].int_flags >> 2) & 0x03);
1976
1977 return -1; /* NOT found */
1978}
1979
1980
1981/*
1982 * given a LOGICAL APIC# and pin#, return:
1983 * the associated 'active' level if found
1984 * -1 if NOT found
1985 */
1986int
1987apic_polarity(int apic, int pin)
1988{
1989 int x;
1990
1991 /* search each of the possible INTerrupt sources */
1992 for (x = 0; x < nintrs; ++x)
1993 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1994 (pin == io_apic_ints[x].dst_apic_int))
1995 return (io_apic_ints[x].int_flags & 0x03);
1996
1997 return -1; /* NOT found */
1998}
1999
984263bc
MD		 2000/*
2001 * set data according to MP defaults
2002 * FIXME: probably not complete yet...
2003 */
2004static void
3aba8f73 2005mptable_default(int type)
984263bc 2006{
984263bc
MD		 2007 int io_apic_id;
2008 int pin;
984263bc
MD		 2009
2010#if 0
26be20a0 2011 kprintf(" MP default config type: %d\n", type);
984263bc
MD		 2012 switch (type) {
2013 case 1:
26be20a0 2014 kprintf(" bus: ISA, APIC: 82489DX\n");
984263bc
MD		 2015 break;
2016 case 2:
26be20a0 2017 kprintf(" bus: EISA, APIC: 82489DX\n");
984263bc
MD		 2018 break;
2019 case 3:
26be20a0 2020 kprintf(" bus: EISA, APIC: 82489DX\n");
984263bc
MD		 2021 break;
2022 case 4:
26be20a0 2023 kprintf(" bus: MCA, APIC: 82489DX\n");
984263bc
MD		 2024 break;
2025 case 5:
26be20a0 2026 kprintf(" bus: ISA+PCI, APIC: Integrated\n");
984263bc
MD		 2027 break;
2028 case 6:
26be20a0 2029 kprintf(" bus: EISA+PCI, APIC: Integrated\n");
984263bc
MD		 2030 break;
2031 case 7:
26be20a0 2032 kprintf(" bus: MCA+PCI, APIC: Integrated\n");
984263bc
MD		 2033 break;
2034 default:
26be20a0 2035 kprintf(" future type\n");
984263bc
MD		 2036 break;
2037 /* NOTREACHED */
2038 }
2039#endif /* 0 */
2040
984263bc
MD		 2041 /* one and only IO APIC */
2042 io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24;
2043
2044 /*
2045 * sanity check, refer to MP spec section 3.6.6, last paragraph
2046 * necessary as some hardware isn't properly setting up the IO APIC
2047 */
2048#if defined(REALLY_ANAL_IOAPICID_VALUE)
2049 if (io_apic_id != 2) {
2050#else
2051 if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
2052#endif /* REALLY_ANAL_IOAPICID_VALUE */
2053 io_apic_set_id(0, 2);
2054 io_apic_id = 2;
2055 }
2056 IO_TO_ID(0) = io_apic_id;
2057 ID_TO_IO(io_apic_id) = 0;
984263bc
MD		 2058
2059 /* fill out bus entries */
2060 switch (type) {
2061 case 1:
2062 case 2:
2063 case 3:
2064 case 4:
2065 case 5:
2066 case 6:
2067 case 7:
2068 bus_data[0].bus_id = default_data[type - 1][1];
2069 bus_data[0].bus_type = default_data[type - 1][2];
2070 bus_data[1].bus_id = default_data[type - 1][3];
2071 bus_data[1].bus_type = default_data[type - 1][4];
2072 break;
2073
2074 /* case 4: case 7: MCA NOT supported */
2075 default: /* illegal/reserved */
2076 panic("BAD default MP config: %d", type);
2077 /* NOTREACHED */
2078 }
2079
984263bc
MD		 2080 /* general cases from MP v1.4, table 5-2 */
2081 for (pin = 0; pin < 16; ++pin) {
2082 io_apic_ints[pin].int_type = 0;
2083 io_apic_ints[pin].int_flags = 0x05; /* edge/active-hi */
2084 io_apic_ints[pin].src_bus_id = 0;
2085 io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 caught below */
2086 io_apic_ints[pin].dst_apic_id = io_apic_id;
2087 io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 */
2088 }
2089
2090 /* special cases from MP v1.4, table 5-2 */
2091 if (type == 2) {
2092 io_apic_ints[2].int_type = 0xff; /* N/C */
2093 io_apic_ints[13].int_type = 0xff; /* N/C */
2094#if !defined(APIC_MIXED_MODE)
2095 /** FIXME: ??? */
2096 panic("sorry, can't support type 2 default yet");
2097#endif /* APIC_MIXED_MODE */
2098 }
2099 else
2100 io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */
2101
2102 if (type == 7)
2103 io_apic_ints[0].int_type = 0xff; /* N/C */
2104 else
2105 io_apic_ints[0].int_type = 3; /* vectored 8259 */
984263bc
MD		 2106}
2107
984263bc 2108/*
f13b5eec
MD		 2109 * Map a physical memory address representing I/O into KVA. The I/O
2110 * block is assumed not to cross a page boundary.
2111 */
2112void *
2113permanent_io_mapping(vm_paddr_t pa)
2114{
2115 vm_offset_t vaddr;
2116 int pgeflag;
2117 int i;
2118
2119 KKASSERT(pa < 0x100000000LL);
2120
2121 pgeflag = 0; /* not used for SMP yet */
2122
2123 /*
2124 * If the requested physical address has already been incidently
2125 * mapped, just use the existing mapping. Otherwise create a new
2126 * mapping.
2127 */
2128 for (i = IO_MAPPING_START_INDEX; i < SMPpt_alloc_index; ++i) {
2129 if (((vm_offset_t)SMPpt[i] & PG_FRAME) ==
2130 ((vm_offset_t)pa & PG_FRAME)) {
2131 break;
2132 }
2133 }
2134 if (i == SMPpt_alloc_index) {
2135 if (i == NPTEPG - 2) {
2136 panic("permanent_io_mapping: We ran out of space"
2137 " in SMPpt[]!");
2138 }
5277b9f6 2139 SMPpt[i] = (pt_entry_t)(PG_V | PG_RW | PG_N | pgeflag |
f13b5eec
MD		 2140 ((vm_offset_t)pa & PG_FRAME));
2141 ++SMPpt_alloc_index;
2142 }
2143 vaddr = (vm_offset_t)CPU_prvspace + (i * PAGE_SIZE) +
2144 ((vm_offset_t)pa & PAGE_MASK);
2145 return ((void *)vaddr);
2146}
2147
2148/*
984263bc
MD		 2149 * start each AP in our list
2150 */
2151static int
2152start_all_aps(u_int boot_addr)
2153{
b45759e1
MD		 2154 int x, i, pg;
2155 int shift;
bb467734
MD		 2156 int smicount;
2157 int smibest;
2158 int smilast;
984263bc
MD		 2159 u_char mpbiosreason;
2160 u_long mpbioswarmvec;
8a8d5d85 2161 struct mdglobaldata *gd;
0f7a3396 2162 struct privatespace *ps;
984263bc
MD		 2163 char *stack;
2164 uintptr_t kptbase;
2165
2166 POSTCODE(START_ALL_APS_POST);
2167
b52c8db0
SZ		 2168 /* Initialize BSP's local APIC */
2169 apic_initialize(TRUE);
984263bc
MD		 2170
2171 /* install the AP 1st level boot code */
2172 install_ap_tramp(boot_addr);
2173
2174
2175 /* save the current value of the warm-start vector */
2176 mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
984263bc
MD		 2177 outb(CMOS_REG, BIOS_RESET);
2178 mpbiosreason = inb(CMOS_DATA);
984263bc 2179
bb467734
MD		 2180 /* setup a vector to our boot code */
2181 *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
2182 *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
2183 outb(CMOS_REG, BIOS_RESET);
2184 outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */
2185
2186 /*
2187 * If we have a TSC we can figure out the SMI interrupt rate.
2188 * The SMI does not necessarily use a constant rate. Spend
2189 * up to 250ms trying to figure it out.
2190 */
2191 smibest = 0;
2192 if (cpu_feature & CPUID_TSC) {
2193 set_apic_timer(275000);
2194 smilast = read_apic_timer();
2195 for (x = 0; x < 20 && read_apic_timer(); ++x) {
2196 smicount = smitest();
2197 if (smibest == 0 || smilast - smicount < smibest)
2198 smibest = smilast - smicount;
2199 smilast = smicount;
2200 }
2201 if (smibest > 250000)
2202 smibest = 0;
2203 if (smibest) {
2204 smibest = smibest * (int64_t)1000000 /
2205 get_apic_timer_frequency();
2206 }
2207 }
2208 if (smibest)
2209 kprintf("SMI Frequency (worst case): %d Hz (%d us)\n",
2210 1000000 / smibest, smibest);
2211
2212
984263bc
MD		 2213 /* set up temporary P==V mapping for AP boot */
2214 /* XXX this is a hack, we should boot the AP on its own stack/PTD */
2215 kptbase = (uintptr_t)(void *)KPTphys;
a44bdeec 2216 for (x = 0; x < NKPT; x++) {
984263bc
MD		 2217 PTD[x] = (pd_entry_t)(PG_V | PG_RW |
2218 ((kptbase + x * PAGE_SIZE) & PG_FRAME));
a44bdeec 2219 }
0f7a3396 2220 cpu_invltlb();
984263bc
MD		 2221
2222 /* start each AP */
2223 for (x = 1; x <= mp_naps; ++x) {
2224
2225 /* This is a bit verbose, it will go away soon. */
2226
2227 /* first page of AP's private space */
2228 pg = x * i386_btop(sizeof(struct privatespace));
2229
81c04d07 2230 /* allocate new private data page(s) */
e4846942 2231 gd = (struct mdglobaldata *)kmem_alloc(&kernel_map,
81c04d07 2232 MDGLOBALDATA_BASEALLOC_SIZE);
984263bc 2233 /* wire it into the private page table page */
81c04d07
MD		 2234 for (i = 0; i < MDGLOBALDATA_BASEALLOC_SIZE; i += PAGE_SIZE) {
2235 SMPpt[pg + i / PAGE_SIZE] = (pt_entry_t)
2236 (PG_V | PG_RW | vtophys_pte((char *)gd + i));
2237 }
2238 pg += MDGLOBALDATA_BASEALLOC_PAGES;
2239
2240 SMPpt[pg + 0] = 0; /* *gd_CMAP1 */
2241 SMPpt[pg + 1] = 0; /* *gd_CMAP2 */
2242 SMPpt[pg + 2] = 0; /* *gd_CMAP3 */
2243 SMPpt[pg + 3] = 0; /* *gd_PMAP1 */
984263bc
MD		 2244
2245 /* allocate and set up an idle stack data page */
e4846942 2246 stack = (char *)kmem_alloc(&kernel_map, UPAGES*PAGE_SIZE);
8a8d5d85 2247 for (i = 0; i < UPAGES; i++) {
81c04d07 2248 SMPpt[pg + 4 + i] = (pt_entry_t)
b5b32410 2249 (PG_V | PG_RW | vtophys_pte(PAGE_SIZE * i + stack));
8a8d5d85 2250 }
984263bc 2251
8a8d5d85
MD		 2252 gd = &CPU_prvspace[x].mdglobaldata; /* official location */
2253 bzero(gd, sizeof(*gd));
0f7a3396 2254 gd->mi.gd_prvspace = ps = &CPU_prvspace[x];
8a8d5d85 2255
984263bc 2256 /* prime data page for it to use */
8a8d5d85 2257 mi_gdinit(&gd->mi, x);
8ad65e08 2258 cpu_gdinit(gd, x);
81c04d07
MD		 2259 gd->gd_CMAP1 = &SMPpt[pg + 0];
2260 gd->gd_CMAP2 = &SMPpt[pg + 1];
2261 gd->gd_CMAP3 = &SMPpt[pg + 2];
2262 gd->gd_PMAP1 = &SMPpt[pg + 3];
0f7a3396
MD		 2263 gd->gd_CADDR1 = ps->CPAGE1;
2264 gd->gd_CADDR2 = ps->CPAGE2;
2265 gd->gd_CADDR3 = ps->CPAGE3;
2266 gd->gd_PADDR1 = (unsigned *)ps->PPAGE1;
9388fcaa
MD		 2267
2268 /*
2269 * Per-cpu pmap for get_ptbase().
2270 */
2271 gd->gd_GDADDR1= (unsigned *)
2272 kmem_alloc_nofault(&kernel_map, SEG_SIZE, SEG_SIZE);
2273 gd->gd_GDMAP1 = &PTD[(vm_offset_t)gd->gd_GDADDR1 >> PDRSHIFT];
2274
e4846942 2275 gd->mi.gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * (mp_naps + 1));
96728c05 2276 bzero(gd->mi.gd_ipiq, sizeof(lwkt_ipiq) * (mp_naps + 1));
984263bc 2277
8a8d5d85
MD		 2278 /*
2279 * Setup the AP boot stack
2280 */
0f7a3396 2281 bootSTK = &ps->idlestack[UPAGES*PAGE_SIZE/2];
984263bc
MD		 2282 bootAP = x;
2283
2284 /* attempt to start the Application Processor */
2285 CHECK_INIT(99); /* setup checkpoints */
bb467734 2286 if (!start_ap(gd, boot_addr, smibest)) {
26be20a0 2287 kprintf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x));
984263bc
MD		 2288 CHECK_PRINT("trace"); /* show checkpoints */
2289 /* better panic as the AP may be running loose */
26be20a0 2290 kprintf("panic y/n? [y] ");
984263bc
MD		 2291 if (cngetc() != 'n')
2292 panic("bye-bye");
2293 }
2294 CHECK_PRINT("trace"); /* show checkpoints */
2295
2296 /* record its version info */
2297 cpu_apic_versions[x] = cpu_apic_versions[0];
984263bc
MD		 2298 }
2299
0f7a3396
MD		 2300 /* set ncpus to 1 + highest logical cpu. Not all may have come up */
2301 ncpus = x;
2302
b45759e1
MD		 2303 /* ncpus2 -- ncpus rounded down to the nearest power of 2 */
2304 for (shift = 0; (1 << shift) <= ncpus; ++shift)
2305 ;
2306 --shift;
2307 ncpus2_shift = shift;
2308 ncpus2 = 1 << shift;
90100055
JH		 2309 ncpus2_mask = ncpus2 - 1;
2310
b45759e1
MD		 2311 /* ncpus_fit -- ncpus rounded up to the nearest power of 2 */
2312 if ((1 << shift) < ncpus)
2313 ++shift;
2314 ncpus_fit = 1 << shift;
2315 ncpus_fit_mask = ncpus_fit - 1;
2316
984263bc 2317 /* build our map of 'other' CPUs */
da23a592 2318 mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
e4846942 2319 mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * ncpus);
96728c05 2320 bzero(mycpu->gd_ipiq, sizeof(lwkt_ipiq) * ncpus);
984263bc
MD		 2321
2322 /* fill in our (BSP) APIC version */
2323 cpu_apic_versions[0] = lapic.version;
2324
2325 /* restore the warmstart vector */
2326 *(u_long *) WARMBOOT_OFF = mpbioswarmvec;
984263bc
MD		 2327 outb(CMOS_REG, BIOS_RESET);
2328 outb(CMOS_DATA, mpbiosreason);
984263bc
MD		 2329
2330 /*
8a8d5d85
MD		 2331 * NOTE! The idlestack for the BSP was setup by locore. Finish
2332 * up, clean out the P==V mapping we did earlier.
984263bc 2333 */
984263bc
MD		 2334 for (x = 0; x < NKPT; x++)
2335 PTD[x] = 0;
2336 pmap_set_opt();
2337
2338 /* number of APs actually started */
8a8d5d85 2339 return ncpus - 1;
984263bc
MD		 2340}
2341
984263bc
MD		 2342/*
2343 * load the 1st level AP boot code into base memory.
2344 */
2345
2346/* targets for relocation */
2347extern void bigJump(void);
2348extern void bootCodeSeg(void);
2349extern void bootDataSeg(void);
2350extern void MPentry(void);
2351extern u_int MP_GDT;
2352extern u_int mp_gdtbase;
2353
2354static void
2355install_ap_tramp(u_int boot_addr)
2356{
2357 int x;
2358 int size = *(int *) ((u_long) & bootMP_size);
2359 u_char *src = (u_char *) ((u_long) bootMP);
2360 u_char *dst = (u_char *) boot_addr + KERNBASE;
2361 u_int boot_base = (u_int) bootMP;
2362 u_int8_t *dst8;
2363 u_int16_t *dst16;
2364 u_int32_t *dst32;
2365
2366 POSTCODE(INSTALL_AP_TRAMP_POST);
2367
2368 for (x = 0; x < size; ++x)
2369 *dst++ = *src++;
2370
2371 /*
2372 * modify addresses in code we just moved to basemem. unfortunately we
2373 * need fairly detailed info about mpboot.s for this to work. changes
2374 * to mpboot.s might require changes here.
2375 */
2376
2377 /* boot code is located in KERNEL space */
2378 dst = (u_char *) boot_addr + KERNBASE;
2379
2380 /* modify the lgdt arg */
2381 dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base));
2382 *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base);
2383
2384 /* modify the ljmp target for MPentry() */
2385 dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1);
2386 *dst32 = ((u_int) MPentry - KERNBASE);
2387
2388 /* modify the target for boot code segment */
2389 dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base));
2390 dst8 = (u_int8_t *) (dst16 + 1);
2391 *dst16 = (u_int) boot_addr & 0xffff;
2392 *dst8 = ((u_int) boot_addr >> 16) & 0xff;
2393
2394 /* modify the target for boot data segment */
2395 dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base));
2396 dst8 = (u_int8_t *) (dst16 + 1);
2397 *dst16 = (u_int) boot_addr & 0xffff;
2398 *dst8 = ((u_int) boot_addr >> 16) & 0xff;
2399}
2400
2401
2402/*
bb467734 2403 * This function starts the AP (application processor) identified
984263bc
MD		 2404 * by the APIC ID 'physicalCpu'. It does quite a "song and dance"
2405 * to accomplish this. This is necessary because of the nuances
2406 * of the different hardware we might encounter. It ain't pretty,
2407 * but it seems to work.
a108bf71
MD		 2408 *
2409 * NOTE: eventually an AP gets to ap_init(), which is called just
2410 * before the AP goes into the LWKT scheduler's idle loop.
984263bc
MD		 2411 */
2412static int
bb467734 2413start_ap(struct mdglobaldata *gd, u_int boot_addr, int smibest)
984263bc
MD		 2414{
2415 int physical_cpu;
2416 int vector;
984263bc
MD		 2417 u_long icr_lo, icr_hi;
2418
2419 POSTCODE(START_AP_POST);
2420
2421 /* get the PHYSICAL APIC ID# */
0f7a3396 2422 physical_cpu = CPU_TO_ID(gd->mi.gd_cpuid);
984263bc
MD		 2423
2424 /* calculate the vector */
2425 vector = (boot_addr >> 12) & 0xff;
2426
bb467734
MD		 2427 /* We don't want anything interfering */
2428 cpu_disable_intr();
2429
8a8d5d85
MD		 2430 /* Make sure the target cpu sees everything */
2431 wbinvd();
984263bc
MD		 2432
2433 /*
bb467734
MD		 2434 * Try to detect when a SMI has occurred, wait up to 200ms.
2435 *
2436 * If a SMI occurs during an AP reset but before we issue
2437 * the STARTUP command, the AP may brick. To work around
2438 * this problem we hold off doing the AP startup until
2439 * after we have detected the SMI. Hopefully another SMI
2440 * will not occur before we finish the AP startup.
2441 *
2442 * Retries don't seem to help. SMIs have a window of opportunity
2443 * and if USB->legacy keyboard emulation is enabled in the BIOS
2444 * the interrupt rate can be quite high.
2445 *
2446 * NOTE: Don't worry about the L1 cache load, it might bloat
2447 * ldelta a little but ndelta will be so huge when the SMI
2448 * occurs the detection logic will still work fine.
2449 */
2450 if (smibest) {
2451 set_apic_timer(200000);
2452 smitest();
2453 }
2454
2455 /*
984263bc
MD		 2456 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
2457 * and running the target CPU. OR this INIT IPI might be latched (P5
2458 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
2459 * ignored.
bb467734
MD		 2460 *
2461 * see apic/apicreg.h for icr bit definitions.
2462 *
2463 * TIME CRITICAL CODE, DO NOT DO ANY KPRINTFS IN THE HOT PATH.
984263bc
MD		 2464 */
2465
bb467734
MD		 2466 /*
2467 * Setup the address for the target AP. We can setup
2468 * icr_hi once and then just trigger operations with
2469 * icr_lo.
2470 */
984263bc
MD		 2471 icr_hi = lapic.icr_hi & ~APIC_ID_MASK;
2472 icr_hi |= (physical_cpu << 24);
bb467734 2473 icr_lo = lapic.icr_lo & 0xfff00000;
984263bc
MD		 2474 lapic.icr_hi = icr_hi;
2475
bb467734
MD		 2476 /*
2477 * Do an INIT IPI: assert RESET
2478 *
2479 * Use edge triggered mode to assert INIT
2480 */
984263bc 2481 lapic.icr_lo = icr_lo | 0x0000c500;
984263bc
MD		 2482 while (lapic.icr_lo & APIC_DELSTAT_MASK)
2483 /* spin */ ;
2484
bb467734
MD		 2485 /*
2486 * The spec calls for a 10ms delay but we may have to use a
2487 * MUCH lower delay to avoid bricking an AP due to a fast SMI
2488 * interrupt. We have other loops here too and dividing by 2
2489 * doesn't seem to be enough even after subtracting 350us,
2490 * so we divide by 4.
2491 *
2492 * Our minimum delay is 150uS, maximum is 10ms. If no SMI
2493 * interrupt was detected we use the full 10ms.
2494 */
2495 if (smibest == 0)
2496 u_sleep(10000);
2497 else if (smibest < 150 * 4 + 350)
2498 u_sleep(150);
2499 else if ((smibest - 350) / 4 < 10000)
2500 u_sleep((smibest - 350) / 4);
2501 else
2502 u_sleep(10000);
984263bc 2503
bb467734
MD		 2504 /*
2505 * Do an INIT IPI: deassert RESET
2506 *
2507 * Use level triggered mode to deassert. It is unclear
2508 * why we need to do this.
2509 */
2510 lapic.icr_lo = icr_lo | 0x00008500;
984263bc
MD		 2511 while (lapic.icr_lo & APIC_DELSTAT_MASK)
2512 /* spin */ ;
bb467734 2513 u_sleep(150); /* wait 150us */
984263bc
MD		 2514
2515 /*
bb467734 2516 * Next we do a STARTUP IPI: the previous INIT IPI might still be
984263bc
MD		 2517 * latched, (P5 bug) this 1st STARTUP would then terminate
2518 * immediately, and the previously started INIT IPI would continue. OR
2519 * the previous INIT IPI has already run. and this STARTUP IPI will
2520 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
2521 * will run.
2522 */
984263bc
MD		 2523 lapic.icr_lo = icr_lo | 0x00000600 | vector;
2524 while (lapic.icr_lo & APIC_DELSTAT_MASK)
2525 /* spin */ ;
2526 u_sleep(200); /* wait ~200uS */
2527
2528 /*
bb467734 2529 * Finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
984263bc
MD		 2530 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
2531 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
2532 * recognized after hardware RESET or INIT IPI.
2533 */
984263bc
MD		 2534 lapic.icr_lo = icr_lo | 0x00000600 | vector;
2535 while (lapic.icr_lo & APIC_DELSTAT_MASK)
2536 /* spin */ ;
bb467734
MD		 2537
2538 /* Resume normal operation */
2539 cpu_enable_intr();
984263bc 2540
8a8d5d85 2541 /* wait for it to start, see ap_init() */
984263bc 2542 set_apic_timer(5000000);/* == 5 seconds */
8a8d5d85 2543 while (read_apic_timer()) {
da23a592 2544 if (smp_startup_mask & CPUMASK(gd->mi.gd_cpuid))
984263bc 2545 return 1; /* return SUCCESS */
8a8d5d85 2546 }
bb467734 2547
984263bc
MD		 2548 return 0; /* return FAILURE */
2549}
2550
bb467734
MD		 2551static
2552int
2553smitest(void)
2554{
2555 int64_t ltsc;
2556 int64_t ntsc;
2557 int64_t ldelta;
2558 int64_t ndelta;
2559 int count;
2560
2561 ldelta = 0;
2562 ndelta = 0;
2563 while (read_apic_timer()) {
2564 ltsc = rdtsc();
2565 for (count = 0; count < 100; ++count)
2566 ntsc = rdtsc(); /* force loop to occur */
2567 if (ldelta) {
2568 ndelta = ntsc - ltsc;
2569 if (ldelta > ndelta)
2570 ldelta = ndelta;
2571 if (ndelta > ldelta * 2)
2572 break;
2573 } else {
2574 ldelta = ntsc - ltsc;
2575 }
2576 }
2577 return(read_apic_timer());
2578}
984263bc
MD		 2579
2580/*
0f7a3396 2581 * Lazy flush the TLB on all other CPU's. DEPRECATED.
984263bc 2582 *
0f7a3396
MD		 2583 * If for some reason we were unable to start all cpus we cannot safely
2584 * use broadcast IPIs.
984263bc 2585 */
7d4d6fdb
MD		 2586
2587static cpumask_t smp_invltlb_req;
b4b1a37a 2588#define SMP_INVLTLB_DEBUG
7d4d6fdb 2589
984263bc
MD		 2590void
2591smp_invltlb(void)
2592{
97359a5b 2593#ifdef SMP
7d4d6fdb 2594 struct mdglobaldata *md = mdcpu;
2d910aaf
MD		 2595#ifdef SMP_INVLTLB_DEBUG
2596 long count = 0;
2597 long xcount = 0;
2598#endif
4117f2fd 2599
7d4d6fdb
MD		 2600 crit_enter_gd(&md->mi);
2601 md->gd_invltlb_ret = 0;
2602 ++md->mi.gd_cnt.v_smpinvltlb;
da23a592 2603 atomic_set_cpumask(&smp_invltlb_req, md->mi.gd_cpumask);
2d910aaf
MD		 2604#ifdef SMP_INVLTLB_DEBUG
2605again:
2606#endif
0f7a3396 2607 if (smp_startup_mask == smp_active_mask) {
984263bc 2608 all_but_self_ipi(XINVLTLB_OFFSET);
0f7a3396 2609 } else {
7d4d6fdb
MD		 2610 selected_apic_ipi(smp_active_mask & ~md->mi.gd_cpumask,
2611 XINVLTLB_OFFSET, APIC_DELMODE_FIXED);
0f7a3396 2612 }
2d910aaf
MD		 2613
2614#ifdef SMP_INVLTLB_DEBUG
2615 if (xcount)
2616 kprintf("smp_invltlb: ipi sent\n");
2617#endif
7d4d6fdb
MD		 2618 while ((md->gd_invltlb_ret & smp_active_mask & ~md->mi.gd_cpumask) !=
2619 (smp_active_mask & ~md->mi.gd_cpumask)) {
2620 cpu_mfence();
2621 cpu_pause();
2d910aaf
MD		 2622#ifdef SMP_INVLTLB_DEBUG
2623 /* DEBUGGING */
2624 if (++count == 400000000) {
2625 print_backtrace(-1);
2626 kprintf("smp_invltlb: endless loop %08lx %08lx, "
2627 "rflags %016lx retry",
2628 (long)md->gd_invltlb_ret,
2629 (long)smp_invltlb_req,
2630 (long)read_eflags());
2631 __asm __volatile ("sti");
2632 ++xcount;
2633 if (xcount > 2)
2634 lwkt_process_ipiq();
2635 if (xcount > 3) {
da23a592
MD		 2636 int bcpu = BSFCPUMASK(~md->gd_invltlb_ret &
2637 ~md->mi.gd_cpumask &
2638 smp_active_mask);
2d910aaf
MD		 2639 globaldata_t xgd;
2640 kprintf("bcpu %d\n", bcpu);
2641 xgd = globaldata_find(bcpu);
2642 kprintf("thread %p %s\n", xgd->gd_curthread, xgd->gd_curthread->td_comm);
2643 }
2644 if (xcount > 5)
2645 panic("giving up");
2646 count = 0;
2647 goto again;
2648 }
2649#endif
7d4d6fdb 2650 }
da23a592 2651 atomic_clear_cpumask(&smp_invltlb_req, md->mi.gd_cpumask);
7d4d6fdb 2652 crit_exit_gd(&md->mi);
4117f2fd 2653#endif
984263bc
MD		 2654}
2655
7d4d6fdb
MD		 2656#ifdef SMP
2657
2658/*
2659 * Called from Xinvltlb assembly with interrupts disabled. We didn't
2660 * bother to bump the critical section count or nested interrupt count
2661 * so only do very low level operations here.
2662 */
2663void
2664smp_invltlb_intr(void)
2665{
2666 struct mdglobaldata *md = mdcpu;
2667 struct mdglobaldata *omd;
2668 cpumask_t mask;
2669 int cpu;
2670
2671 mask = smp_invltlb_req;
2672 cpu_mfence();
2673 cpu_invltlb();
2674 while (mask) {
da23a592
MD		 2675 cpu = BSFCPUMASK(mask);
2676 mask &= ~CPUMASK(cpu);
7d4d6fdb 2677 omd = (struct mdglobaldata *)globaldata_find(cpu);
da23a592 2678 atomic_set_cpumask(&omd->gd_invltlb_ret, md->mi.gd_cpumask);
7d4d6fdb
MD		 2679 }
2680}
2681
2682#endif
2683
984263bc
MD		 2684/*
2685 * When called the executing CPU will send an IPI to all other CPUs
2686 * requesting that they halt execution.
2687 *
2688 * Usually (but not necessarily) called with 'other_cpus' as its arg.
2689 *
2690 * - Signals all CPUs in map to stop.
2691 * - Waits for each to stop.
2692 *
2693 * Returns:
2694 * -1: error
2695 * 0: NA
2696 * 1: ok
2697 *
2698 * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
2699 * from executing at same time.
2700 */
2701int
da23a592 2702stop_cpus(cpumask_t map)
984263bc 2703{
0f7a3396 2704 map &= smp_active_mask;
984263bc
MD		 2705
2706 /* send the Xcpustop IPI to all CPUs in map */
2707 selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
2708
2709 while ((stopped_cpus & map) != map)
2710 /* spin */ ;
2711
2712 return 1;
2713}
2714
2715
2716/*
2717 * Called by a CPU to restart stopped CPUs.
2718 *
2719 * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
2720 *
2721 * - Signals all CPUs in map to restart.
2722 * - Waits for each to restart.
2723 *
2724 * Returns:
2725 * -1: error
2726 * 0: NA
2727 * 1: ok
2728 */
2729int
da23a592 2730restart_cpus(cpumask_t map)
984263bc 2731{
0f7a3396
MD		 2732 /* signal other cpus to restart */
2733 started_cpus = map & smp_active_mask;
984263bc
MD		 2734
2735 while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
2736 /* spin */ ;
2737
2738 return 1;
2739}
2740
984263bc 2741/*
8a8d5d85
MD		 2742 * This is called once the mpboot code has gotten us properly relocated
2743 * and the MMU turned on, etc. ap_init() is actually the idle thread,
2744 * and when it returns the scheduler will call the real cpu_idle() main
2745 * loop for the idlethread. Interrupts are disabled on entry and should
2746 * remain disabled at return.
984263bc 2747 */
984263bc 2748void
8a8d5d85 2749ap_init(void)
984263bc
MD		 2750{
2751 u_int apic_id;
2752
8a8d5d85 2753 /*
0f7a3396
MD		 2754 * Adjust smp_startup_mask to signal the BSP that we have started
2755 * up successfully. Note that we do not yet hold the BGL. The BSP
2756 * is waiting for our signal.
2757 *
2758 * We can't set our bit in smp_active_mask yet because we are holding
2759 * interrupts physically disabled and remote cpus could deadlock
2760 * trying to send us an IPI.
8a8d5d85 2761 */
da23a592 2762 smp_startup_mask |= CPUMASK(mycpu->gd_cpuid);
35238fa5 2763 cpu_mfence();
8a8d5d85
MD		 2764
2765 /*
41a01a4d
MD		 2766 * Interlock for finalization. Wait until mp_finish is non-zero,
2767 * then get the MP lock.
2768 *
2769 * Note: We are in a critical section.
2770 *
41a01a4d
MD		 2771 * Note: we are the idle thread, we can only spin.
2772 *
35238fa5
MD		 2773 * Note: The load fence is memory volatile and prevents the compiler
2774 * from improperly caching mp_finish, and the cpu from improperly
2775 * caching it.
8a8d5d85 2776 */
41a01a4d 2777 while (mp_finish == 0)
b5d16701
MD		 2778 cpu_lfence();
2779 while (try_mplock() == 0)
2780 ;
8a8d5d85 2781
374133e3 2782 if (cpu_feature & CPUID_TSC) {
b5d16701
MD		 2783 /*
2784 * The BSP is constantly updating tsc0_offset, figure out
2785 * the relative difference to synchronize ktrdump.
2786 */
2787 tsc_offsets[mycpu->gd_cpuid] = rdtsc() - tsc0_offset;
374133e3
MD		 2788 }
2789
984263bc
MD		 2790 /* BSP may have changed PTD while we're waiting for the lock */
2791 cpu_invltlb();
2792
984263bc
MD		 2793#if defined(I586_CPU) && !defined(NO_F00F_HACK)
2794 lidt(&r_idt);
2795#endif
2796
2797 /* Build our map of 'other' CPUs. */
da23a592 2798 mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
984263bc 2799
26be20a0 2800 kprintf("SMP: AP CPU #%d Launched!\n", mycpu->gd_cpuid);
984263bc 2801
984263bc 2802 /* A quick check from sanity claus */
d53907dd 2803 apic_id = (apic_id_to_logical[(lapic.id & 0xff000000) >> 24]);
8a8d5d85 2804 if (mycpu->gd_cpuid != apic_id) {
26be20a0
SW		 2805 kprintf("SMP: cpuid = %d\n", mycpu->gd_cpuid);
2806 kprintf("SMP: apic_id = %d\n", apic_id);
2807 kprintf("PTD[MPPTDI] = %p\n", (void *)PTD[MPPTDI]);
984263bc
MD		 2808 panic("cpuid mismatch! boom!!");
2809 }
2810
b52c8db0
SZ		 2811 /* Initialize AP's local APIC for irq's */
2812 apic_initialize(FALSE);
984263bc
MD		 2813
2814 /* Set memory range attributes for this CPU to match the BSP */
2815 mem_range_AP_init();
2816
a2a5ad0d 2817 /*
4c9f5a7f
MD		 2818 * Once we go active we must process any IPIQ messages that may
2819 * have been queued, because no actual IPI will occur until we
2820 * set our bit in the smp_active_mask. If we don't the IPI
2821 * message interlock could be left set which would also prevent
2822 * further IPIs.
2823 *
8a8d5d85
MD		 2824 * The idle loop doesn't expect the BGL to be held and while
2825 * lwkt_switch() normally cleans things up this is a special case
2826 * because we returning almost directly into the idle loop.
41a01a4d
MD		 2827 *
2828 * The idle thread is never placed on the runq, make sure
4c9f5a7f 2829 * nothing we've done put it there.
8a8d5d85 2830 */
b5d16701 2831 KKASSERT(get_mplock_count(curthread) == 1);
da23a592 2832 smp_active_mask |= CPUMASK(mycpu->gd_cpuid);
d19f6edf
MD		 2833
2834 /*
2835 * Enable interrupts here. idle_restore will also do it, but
2836 * doing it here lets us clean up any strays that got posted to
2837 * the CPU during the AP boot while we are still in a critical
2838 * section.
2839 */
2840 __asm __volatile("sti; pause; pause"::);
c263294b 2841 bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending));
d19f6edf 2842
4a19580d 2843 initclocks_pcpu(); /* clock interrupts (via IPIs) */
4c9f5a7f 2844 lwkt_process_ipiq();
d19f6edf
MD		 2845
2846 /*
2847 * Releasing the mp lock lets the BSP finish up the SMP init
2848 */
96728c05 2849 rel_mplock();
41a01a4d 2850 KKASSERT((curthread->td_flags & TDF_RUNQ) == 0);
984263bc
MD		 2851}
2852
41a01a4d
MD		 2853/*
2854 * Get SMP fully working before we start initializing devices.
2855 */
2856static
2857void
2858ap_finish(void)
2859{
2860 mp_finish = 1;
2861 if (bootverbose)
26be20a0 2862 kprintf("Finish MP startup\n");
374133e3
MD		 2863 if (cpu_feature & CPUID_TSC)
2864 tsc0_offset = rdtsc();
2865 tsc_offsets[0] = 0;
41a01a4d 2866 rel_mplock();
374133e3 2867 while (smp_active_mask != smp_startup_mask) {
35238fa5 2868 cpu_lfence();
374133e3
MD		 2869 if (cpu_feature & CPUID_TSC)
2870 tsc0_offset = rdtsc();
2871 }
4da43e1f 2872 while (try_mplock() == 0)
41a01a4d
MD		 2873 ;
2874 if (bootverbose)
26be20a0 2875 kprintf("Active CPU Mask: %08x\n", smp_active_mask);
41a01a4d
MD		 2876}
2877
ba39e2e0 2878SYSINIT(finishsmp, SI_BOOT2_FINISH_SMP, SI_ORDER_FIRST, ap_finish, NULL)
41a01a4d 2879
96728c05
MD		 2880void
2881cpu_send_ipiq(int dcpu)
2882{
da23a592 2883 if (CPUMASK(dcpu) & smp_active_mask)
41a01a4d 2884 single_apic_ipi(dcpu, XIPIQ_OFFSET, APIC_DELMODE_FIXED);
96728c05 2885}
41a01a4d
MD		 2886
2887#if 0 /* single_apic_ipi_passive() not working yet */
2888/*
2889 * Returns 0 on failure, 1 on success
2890 */
2891int
2892cpu_send_ipiq_passive(int dcpu)
2893{
2894 int r = 0;
da23a592 2895 if (CPUMASK(dcpu) & smp_active_mask) {
41a01a4d
MD		 2896 r = single_apic_ipi_passive(dcpu, XIPIQ_OFFSET,
2897 APIC_DELMODE_FIXED);
2898 }
2899 return(r);
2900}
2901#endif
2902
a0873f07
SZ		 2903struct mptable_lapic_cbarg1 {
2904 int cpu_count;
44c36320
SZ		 2905 int ht_fixup;
2906 u_int ht_apicid_mask;
a0873f07
SZ		 2907};
2908
2909static int
2910mptable_lapic_pass1_callback(void *xarg, const void *pos, int type)
2911{
2912 const struct PROCENTRY *ent;
2913 struct mptable_lapic_cbarg1 *arg = xarg;
2914
2915 if (type != 0)
2916 return 0;
2917 ent = pos;
2918
2919 if ((ent->cpu_flags & PROCENTRY_FLAG_EN) == 0)
2920 return 0;
2921
2922 arg->cpu_count++;
44c36320
SZ		 2923 if (ent->apic_id < 32) {
2924 arg->ht_apicid_mask |= 1 << ent->apic_id;
2925 } else if (arg->ht_fixup) {
2926 kprintf("MPTABLE: lapic id > 32, disable HTT fixup\n");
2927 arg->ht_fixup = 0;
2928 }
a0873f07
SZ		 2929 return 0;
2930}
2931
2932struct mptable_lapic_cbarg2 {
2933 int cpu;
44c36320 2934 int logical_cpus;
a0873f07
SZ		 2935 int found_bsp;
2936};
2937
2938static int
2939mptable_lapic_pass2_callback(void *xarg, const void *pos, int type)
2940{
2941 const struct PROCENTRY *ent;
2942 struct mptable_lapic_cbarg2 *arg = xarg;
2943
2944 if (type != 0)
2945 return 0;
2946 ent = pos;
2947
2948 if (ent->cpu_flags & PROCENTRY_FLAG_BP) {
2949 KKASSERT(!arg->found_bsp);
2950 arg->found_bsp = 1;
2951 }
2952
2953 if (processor_entry(ent, arg->cpu))
2954 arg->cpu++;
2955
44c36320 2956 if (arg->logical_cpus) {
a0873f07
SZ		 2957 struct PROCENTRY proc;
2958 int i;
2959
2960 /*
2961 * Create fake mptable processor entries
2962 * and feed them to processor_entry() to
2963 * enumerate the logical CPUs.
2964 */
2965 bzero(&proc, sizeof(proc));
2966 proc.type = 0;
2967 proc.cpu_flags = PROCENTRY_FLAG_EN;
2968 proc.apic_id = ent->apic_id;
2969
44c36320 2970 for (i = 1; i < arg->logical_cpus; i++) {
a0873f07
SZ		 2971 proc.apic_id++;
2972 processor_entry(&proc, arg->cpu);
a0873f07
SZ		 2973 arg->cpu++;
2974 }
2975 }
2976 return 0;
2977}
2978
322abba7 2979static void
281d9482
SZ		 2980mptable_imcr(struct mptable_pos *mpt)
2981{
2982 /* record whether PIC or virtual-wire mode */
2983 machintr_setvar_simple(MACHINTR_VAR_IMCR_PRESENT,
2984 mpt->mp_fps->mpfb2 & 0x80);
2985}
2986
2987struct mptable_lapic_enumerator {
2988 struct lapic_enumerator enumerator;
2989 vm_paddr_t mpfps_paddr;
2990};
2991
2992static void
322abba7
SZ		 2993mptable_lapic_default(void)
2994{
2995 int ap_apicid, bsp_apicid;
2996
2997 mp_naps = 1; /* exclude BSP */
2998
2999 /* Map local apic before the id field is accessed */
84cc808b 3000 lapic_map(DEFAULT_APIC_BASE);
322abba7
SZ		 3001
3002 bsp_apicid = APIC_ID(lapic.id);
3003 ap_apicid = (bsp_apicid == 0) ? 1 : 0;
3004
3005 /* BSP */
3006 mp_set_cpuids(0, bsp_apicid);
3007 /* one and only AP */
3008 mp_set_cpuids(1, ap_apicid);
3009}
3010
a0873f07
SZ		 3011/*
3012 * Configure:
a0873f07 3013 * mp_naps
d787e80c 3014 * ID_TO_CPU(N), APIC ID to logical CPU table
a0873f07
SZ		 3015 * CPU_TO_ID(N), logical CPU to APIC ID table
3016 */
3017static void
281d9482 3018mptable_lapic_enumerate(struct lapic_enumerator *e)
a0873f07 3019{
281d9482 3020 struct mptable_pos mpt;
322abba7
SZ		 3021 struct mptable_lapic_cbarg1 arg1;
3022 struct mptable_lapic_cbarg2 arg2;
3023 mpcth_t cth;
44c36320 3024 int error, logical_cpus = 0;
5a16ccc3 3025 vm_offset_t lapic_addr;
281d9482
SZ		 3026 vm_paddr_t mpfps_paddr;
3027
3028 mpfps_paddr = ((struct mptable_lapic_enumerator *)e)->mpfps_paddr;
3029 KKASSERT(mpfps_paddr != 0);
a0873f07 3030
281d9482
SZ		 3031 error = mptable_map(&mpt, mpfps_paddr);
3032 if (error)
3033 panic("mptable_lapic_enumerate mptable_map failed\n");
3034
3035 KKASSERT(mpt.mp_fps != NULL);
a0873f07 3036
322abba7
SZ		 3037 /*
3038 * Check for use of 'default' configuration
3039 */
281d9482 3040 if (mpt.mp_fps->mpfb1 != 0) {
322abba7 3041 mptable_lapic_default();
281d9482 3042 mptable_unmap(&mpt);
322abba7
SZ		 3043 return;
3044 }
a0873f07 3045
281d9482 3046 cth = mpt.mp_cth;
322abba7 3047 KKASSERT(cth != NULL);
a0873f07 3048
322abba7
SZ		 3049 /* Save local apic address */
3050 lapic_addr = (vm_offset_t)cth->apic_address;
3051 KKASSERT(lapic_addr != 0);
a0873f07 3052
322abba7
SZ		 3053 /*
3054 * Find out how many CPUs do we have
3055 */
3056 bzero(&arg1, sizeof(arg1));
44c36320
SZ		 3057 arg1.ht_fixup = 1; /* Apply ht fixup by default */
3058
322abba7
SZ		 3059 error = mptable_iterate_entries(cth,
3060 mptable_lapic_pass1_callback, &arg1);
3061 if (error)
3062 panic("mptable_iterate_entries(lapic_pass1) failed\n");
322abba7 3063 KKASSERT(arg1.cpu_count != 0);
a0873f07 3064
322abba7 3065 /* See if we need to fixup HT logical CPUs. */
44c36320
SZ		 3066 if (arg1.ht_fixup) {
3067 logical_cpus = mptable_hyperthread_fixup(arg1.ht_apicid_mask,
3068 arg1.cpu_count);
3069 if (logical_cpus != 0)
3070 arg1.cpu_count *= logical_cpus;
3071 }
3072 mp_naps = arg1.cpu_count;
a0873f07 3073
44c36320 3074 /* Qualify the numbers again, after possible HT fixup */
322abba7
SZ		 3075 if (mp_naps > MAXCPU) {
3076 kprintf("Warning: only using %d of %d available CPUs!\n",
3077 MAXCPU, mp_naps);
3078 mp_naps = MAXCPU;
a0873f07
SZ		 3079 }
3080
322abba7 3081 --mp_naps; /* subtract the BSP */
a0873f07 3082
322abba7
SZ		 3083 /*
3084 * Link logical CPU id to local apic id
3085 */
3086 bzero(&arg2, sizeof(arg2));
3087 arg2.cpu = 1;
44c36320 3088 arg2.logical_cpus = logical_cpus;
a0873f07 3089
322abba7
SZ		 3090 error = mptable_iterate_entries(cth,
3091 mptable_lapic_pass2_callback, &arg2);
3092 if (error)
3093 panic("mptable_iterate_entries(lapic_pass2) failed\n");
3094 KKASSERT(arg2.found_bsp);
a0873f07 3095
322abba7 3096 /* Map local apic */
84cc808b 3097 lapic_map(lapic_addr);
281d9482
SZ		 3098
3099 mptable_unmap(&mpt);
3100}
3101
3102static int
3103mptable_lapic_probe(struct lapic_enumerator *e)
3104{
3105 vm_paddr_t mpfps_paddr;
3106
3107 mpfps_paddr = mptable_probe();
3108 if (mpfps_paddr == 0)
3109 return ENXIO;
3110
3111 ((struct mptable_lapic_enumerator *)e)->mpfps_paddr = mpfps_paddr;
3112 return 0;
a0873f07 3113}
5a16ccc3 3114
281d9482
SZ		 3115static struct mptable_lapic_enumerator mptable_lapic_enumerator = {
3116 .enumerator = {
3117 .lapic_prio = LAPIC_ENUM_PRIO_MPTABLE,
3118 .lapic_probe = mptable_lapic_probe,
3119 .lapic_enumerate = mptable_lapic_enumerate
3120 }
3121};
3122
5a16ccc3 3123static void
281d9482 3124mptable_apic_register(void)
a0eaef71 3125{
281d9482 3126 lapic_enumerator_register(&mptable_lapic_enumerator.enumerator);
a0eaef71 3127}
281d9482 3128SYSINIT(madt, SI_BOOT2_PRESMP, SI_ORDER_ANY, mptable_apic_register, 0);
DragonFly Project GIT Source repo