From da23a592849ab6010314c28c571f343b43c9b6d5 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Thu, 9 Dec 2010 14:09:17 -0800 Subject: [PATCH] kernel - Add support for up to 63 cpus & 512G of ram for 64-bit builds. * Increase SMP_MAXCPU to 63 for 64-bit builds. * cpumask_t is 64 bits on 64-bit builds now. It remains 32 bits on 32-bit builds. * Add #define's for atomic_set_cpumask(), atomic_clear_cpumask, and atomic_cmpset_cpumask(). Replace all use cases on cpu masks with these functions. * Add CPUMASK(), BSRCPUMASK(), and BSFCPUMASK() macros. Replace all use cases on cpu masks with these functions. In particular note that (1 << cpu) just doesn't work with a 64-bit cpumask. Numerous bits of assembly also had to be adjusted to use e.g. btq instead of btl, etc. * Change __uint32_t declarations that were meant to be cpu masks to use cpumask_t (most already have). Also change other bits of code which work on cpu masks to be more agnostic. For example, poll_cpumask0 and lwp_cpumask. * 64-bit atomic ops cannot use "iq", they must use "r", because most x86-64 do NOT have 64-bit immediate value support. * Rearrange initial kernel memory allocations to start from KvaStart and not KERNBASE, because only 2GB of KVM is available after KERNBASE. Certain VM allocations with > 32G of ram can exceed 2GB. For example, vm_page_array[]. 2GB was not enough. * Remove numerous mdglobaldata fields that are not used. * Align CPU_prvspace[] for now. Eventually it will be moved into a mapped area. Reserve sufficient space at MPPTDI now, but it is still unused. * When pre-allocating kernel page table PD entries calculate the number of page table pages at KvaStart and at KERNBASE separately, since the KVA space starting at KERNBASE caps out at 2GB. * Change kmem_init() and vm_page_startup() to not take memory range arguments. Instead the globals (virtual_start and virtual_end) are manipualted directly. --- sys/cpu/i386/include/atomic.h | 5 + sys/cpu/i386/include/types.h | 3 + sys/cpu/x86_64/include/atomic.h | 55 +++++----- sys/cpu/x86_64/include/param.h | 2 +- sys/cpu/x86_64/include/types.h | 7 +- sys/dev/acpica5/acpi_cpu.c | 2 +- sys/kern/init_main.c | 4 +- sys/kern/kern_mplock.c | 10 +- sys/kern/kern_poll.c | 14 +-- sys/kern/kern_proc.c | 2 +- sys/kern/kern_synch.c | 10 +- sys/kern/kern_usched.c | 15 +-- sys/kern/lwkt_ipiq.c | 12 +-- sys/kern/usched_bsd4.c | 40 ++++---- sys/kern/usched_dummy.c | 30 +++--- sys/netinet/tcp_subr.c | 2 +- sys/platform/pc32/acpica5/madt.c | 2 +- sys/platform/pc32/apic/mpapic.c | 6 +- sys/platform/pc32/i386/genassym.c | 3 +- sys/platform/pc32/i386/machdep.c | 2 +- sys/platform/pc32/i386/mp_machdep.c | 42 ++++---- sys/platform/pc32/i386/pmap.c | 7 +- sys/platform/pc32/i386/pmap_inval.c | 4 +- sys/platform/pc32/i386/vm_machdep.c | 4 +- sys/platform/pc32/include/globaldata.h | 6 +- sys/platform/pc32/include/pmap.h | 3 +- sys/platform/pc32/include/smp.h | 10 +- sys/platform/pc64/acpica5/madt.c | 2 +- sys/platform/pc64/apic/apic_vector.s | 14 +-- sys/platform/pc64/apic/mpapic.c | 12 +-- sys/platform/pc64/include/globaldata.h | 35 +++---- sys/platform/pc64/include/pmap.h | 21 +++- sys/platform/pc64/include/smp.h | 10 +- sys/platform/pc64/include/vmparam.h | 9 +- sys/platform/pc64/isa/clock.c | 2 +- sys/platform/pc64/x86_64/db_interface.c | 14 +-- sys/platform/pc64/x86_64/genassym.c | 3 +- sys/platform/pc64/x86_64/global.s | 10 -- sys/platform/pc64/x86_64/machdep.c | 8 +- sys/platform/pc64/x86_64/mp_machdep.c | 100 ++++++++----------- sys/platform/pc64/x86_64/pmap.c | 53 +++++----- sys/platform/pc64/x86_64/pmap_inval.c | 4 +- sys/platform/pc64/x86_64/swtch.s | 16 +-- sys/platform/vkernel/i386/cpu_regs.c | 2 +- sys/platform/vkernel/i386/db_interface.c | 8 +- sys/platform/vkernel/i386/genassym.c | 3 +- sys/platform/vkernel/i386/mp.c | 40 ++++---- sys/platform/vkernel/include/globaldata.h | 6 +- sys/platform/vkernel/include/pmap.h | 3 +- sys/platform/vkernel/include/smp.h | 12 +-- sys/platform/vkernel/platform/pmap.c | 34 ++++--- sys/platform/vkernel64/include/globaldata.h | 6 +- sys/platform/vkernel64/include/pmap.h | 3 + sys/platform/vkernel64/include/smp.h | 12 +-- sys/platform/vkernel64/platform/pmap.c | 8 +- sys/platform/vkernel64/x86_64/cpu_regs.c | 2 +- sys/platform/vkernel64/x86_64/db_interface.c | 8 +- sys/platform/vkernel64/x86_64/mp.c | 40 ++++---- sys/platform/vkernel64/x86_64/swtch.s | 12 +-- sys/sys/globaldata.h | 2 +- sys/sys/mplock2.h | 6 +- sys/sys/thread2.h | 4 +- sys/vm/vm_extern.h | 2 +- sys/vm/vm_init.c | 6 +- sys/vm/vm_kern.c | 22 ++-- sys/vm/vm_page.c | 27 +++-- sys/vm/vm_page.h | 2 +- 67 files changed, 448 insertions(+), 437 deletions(-) diff --git a/sys/cpu/i386/include/atomic.h b/sys/cpu/i386/include/atomic.h index 1521b66d4e..8212a031e1 100644 --- a/sys/cpu/i386/include/atomic.h +++ b/sys/cpu/i386/include/atomic.h @@ -526,6 +526,11 @@ ATOMIC_STORE_LOAD(long, "cmpxchgl %0,%1", "xchgl %1,%0"); #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long +/* cpumask_t is 32-bits on i386 */ +#define atomic_set_cpumask atomic_set_int +#define atomic_clear_cpumask atomic_clear_int +#define atomic_cmpset_cpumask atomic_cmpset_int + /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char diff --git a/sys/cpu/i386/include/types.h b/sys/cpu/i386/include/types.h index 1261a03cdd..c3c9618fe0 100644 --- a/sys/cpu/i386/include/types.h +++ b/sys/cpu/i386/include/types.h @@ -66,8 +66,11 @@ typedef __uint32_t pd_entry_t; typedef __uint32_t pt_entry_t; typedef __uint32_t cpumask_t; /* mask representing a set of cpus */ +#if defined(_KERNEL) || defined(_KERNEL_STRUCTURES) #define CPUMASK(cpu) (1U << (cpu)) #define BSRCPUMASK(mask) bsrl(mask) +#define BSFCPUMASK(mask) bsfl(mask) +#endif #define PDESIZE sizeof(pd_entry_t) /* for assembly files */ #define PTESIZE sizeof(pt_entry_t) /* for assembly files */ diff --git a/sys/cpu/x86_64/include/atomic.h b/sys/cpu/x86_64/include/atomic.h index d97b7340ec..4e2fd4085a 100644 --- a/sys/cpu/x86_64/include/atomic.h +++ b/sys/cpu/x86_64/include/atomic.h @@ -69,7 +69,7 @@ * This allows kernel modules to be portable between UP and SMP systems. */ #if defined(KLD_MODULE) -#define ATOMIC_ASM(NAME, TYPE, OP, V) \ +#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ extern void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ extern void atomic_##NAME##_##TYPE##_nonlocked(volatile u_##TYPE *p, u_##TYPE v); #else /* !KLD_MODULE */ @@ -90,47 +90,51 @@ * iq - integer constant or %ax/%bx/%cx/%dx (ir = int constant or any reg) * (Note: byte instructions only work on %ax,%bx,%cx, or %dx). iq * is good enough for our needs so don't get fancy. + * r - any register. + * + * NOTE: 64-bit immediate values are not supported for most x86-64 + * instructions so we have to use "r". */ /* egcs 1.1.2+ version */ -#define ATOMIC_ASM(NAME, TYPE, OP, V) \ +#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(MPLOCKED OP \ : "+m" (*p) \ - : "iq" (V)); \ + : CONS (V)); \ } \ static __inline void \ atomic_##NAME##_##TYPE##_nonlocked(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(OP \ : "+m" (*p) \ - : "iq" (V)); \ + : CONS (V)); \ } #endif /* KLD_MODULE */ /* egcs 1.1.2+ version */ -ATOMIC_ASM(set, char, "orb %b1,%0", v) -ATOMIC_ASM(clear, char, "andb %b1,%0", ~v) -ATOMIC_ASM(add, char, "addb %b1,%0", v) -ATOMIC_ASM(subtract, char, "subb %b1,%0", v) - -ATOMIC_ASM(set, short, "orw %w1,%0", v) -ATOMIC_ASM(clear, short, "andw %w1,%0", ~v) -ATOMIC_ASM(add, short, "addw %w1,%0", v) -ATOMIC_ASM(subtract, short, "subw %w1,%0", v) - -ATOMIC_ASM(set, int, "orl %1,%0", v) -ATOMIC_ASM(clear, int, "andl %1,%0", ~v) -ATOMIC_ASM(add, int, "addl %1,%0", v) -ATOMIC_ASM(subtract, int, "subl %1,%0", v) - -ATOMIC_ASM(set, long, "orq %1,%0", v) -ATOMIC_ASM(clear, long, "andq %1,%0", ~v) -ATOMIC_ASM(add, long, "addq %1,%0", v) -ATOMIC_ASM(subtract, long, "subq %1,%0", v) +ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v) +ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v) +ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v) +ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v) + +ATOMIC_ASM(set, short, "orw %w1,%0", "iq", v) +ATOMIC_ASM(clear, short, "andw %w1,%0", "iq", ~v) +ATOMIC_ASM(add, short, "addw %w1,%0", "iq", v) +ATOMIC_ASM(subtract, short, "subw %w1,%0", "iq", v) + +ATOMIC_ASM(set, int, "orl %1,%0", "iq", v) +ATOMIC_ASM(clear, int, "andl %1,%0", "iq", ~v) +ATOMIC_ASM(add, int, "addl %1,%0", "iq", v) +ATOMIC_ASM(subtract, int, "subl %1,%0", "iq", v) + +ATOMIC_ASM(set, long, "orq %1,%0", "r", v) +ATOMIC_ASM(clear, long, "andq %1,%0", "r", ~v) +ATOMIC_ASM(add, long, "addq %1,%0", "r", v) +ATOMIC_ASM(subtract, long, "subq %1,%0", "r", v) #if defined(KLD_MODULE) @@ -553,6 +557,11 @@ ATOMIC_STORE_LOAD(long, "cmpxchgq %0,%1", "xchgq %1,%0"); #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long +/* cpumask_t is 64-bits on x86-64 */ +#define atomic_set_cpumask atomic_set_long +#define atomic_clear_cpumask atomic_clear_long +#define atomic_cmpset_cpumask atomic_cmpset_long + /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char diff --git a/sys/cpu/x86_64/include/param.h b/sys/cpu/x86_64/include/param.h index de87df1569..d23ae694a7 100644 --- a/sys/cpu/x86_64/include/param.h +++ b/sys/cpu/x86_64/include/param.h @@ -88,7 +88,7 @@ * Use SMP_MAXCPU instead of MAXCPU for structures that are intended to * remain compatible between UP and SMP builds. */ -#define SMP_MAXCPU 31 +#define SMP_MAXCPU 63 #ifdef SMP #define MAXCPU SMP_MAXCPU #else diff --git a/sys/cpu/x86_64/include/types.h b/sys/cpu/x86_64/include/types.h index c6ff8c8b9a..337ff6dd7b 100644 --- a/sys/cpu/x86_64/include/types.h +++ b/sys/cpu/x86_64/include/types.h @@ -75,10 +75,13 @@ typedef __uint64_t pdp_entry_t; typedef __uint64_t pd_entry_t; typedef __uint64_t pt_entry_t; #endif -typedef __uint32_t cpumask_t; /* mask representing a set of cpus */ +typedef __uint64_t cpumask_t; /* mask representing a set of cpus */ -#define CPUMASK(cpu) ((__uint32_t)1 << (cpu)) +#if defined(_KERNEL) || defined(_KERNEL_STRUCTURES) +#define CPUMASK(cpu) ((cpumask_t)1 << (cpu)) #define BSRCPUMASK(mask) bsrq(mask) +#define BSFCPUMASK(mask) bsfq(mask) +#endif #define PML4SIZE sizeof(pml4_entry_t) /* for assembly files */ #define PDPSIZE sizeof(pdp_entry_t) /* for assembly files */ diff --git a/sys/dev/acpica5/acpi_cpu.c b/sys/dev/acpica5/acpi_cpu.c index fbbc00f4cc..fa42912086 100644 --- a/sys/dev/acpica5/acpi_cpu.c +++ b/sys/dev/acpica5/acpi_cpu.c @@ -237,7 +237,7 @@ acpi_cpu_get_id(uint32_t idx, uint32_t *acpi_id, uint32_t *cpu_id) KASSERT(acpi_id != NULL, ("Null acpi_id")); KASSERT(cpu_id != NULL, ("Null cpu_id")); for (i = 0; i < ncpus; i++) { - if ((smp_active_mask & (1 << i)) == 0) + if ((smp_active_mask & CPUMASK(i)) == 0) continue; md = (struct mdglobaldata *)globaldata_find(i); KASSERT(md != NULL, ("no pcpu data for %d", i)); diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index d4e54f6e52..68e3248406 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -175,7 +175,7 @@ mi_proc0init(struct globaldata *gd, struct user *proc0paddr) lwp0.lwp_thread = &thread0; lwp0.lwp_proc = &proc0; proc0.p_usched = usched_init(); - lwp0.lwp_cpumask = 0xFFFFFFFF; + lwp0.lwp_cpumask = (cpumask_t)-1; varsymset_init(&proc0.p_varsymset, NULL); thread0.td_flags |= TDF_RUNNING; thread0.td_proc = &proc0; @@ -705,7 +705,7 @@ mi_gdinit(struct globaldata *gd, int cpuid) TAILQ_INIT(&gd->gd_systimerq); gd->gd_sysid_alloc = cpuid; /* prime low bits for cpu lookup */ gd->gd_cpuid = cpuid; - gd->gd_cpumask = (cpumask_t)1 << cpuid; + gd->gd_cpumask = CPUMASK(cpuid); lwkt_gdinit(gd); vm_map_entry_reserve_cpu_init(gd); sleep_gdinit(gd); diff --git a/sys/kern/kern_mplock.c b/sys/kern/kern_mplock.c index ee6ee9f39a..c2800932a9 100644 --- a/sys/kern/kern_mplock.c +++ b/sys/kern/kern_mplock.c @@ -87,7 +87,7 @@ KTR_INFO(KTR_GIANT_CONTENTION, giant, end, 1, file, line) int mp_lock; -int cpu_contention_mask; +cpumask_t cpu_contention_mask; const char *mp_lock_holder_file; /* debugging */ int mp_lock_holder_line; /* debugging */ @@ -275,14 +275,14 @@ lwkt_mp_lock_uncontested(void) gd = mycpu; clr_mplock_contention_mask(gd); mask = cpu_contention_mask; - tmpmask = ~((1 << gd->gd_cpuid) - 1); + tmpmask = ~(CPUMASK(gd->gd_cpuid) - 1); if (mask) { if (mask & tmpmask) - cpuid = bsfl(mask & tmpmask); + cpuid = BSFCPUMASK(mask & tmpmask); else - cpuid = bsfl(mask); - atomic_clear_int(&cpu_contention_mask, 1 << cpuid); + cpuid = BSFCPUMASK(mask); + atomic_clear_cpumask(&cpu_contention_mask, CPUMASK(cpuid)); dgd = globaldata_find(cpuid); lwkt_send_ipiq(dgd, lwkt_mp_lock_uncontested_remote, NULL); } diff --git a/sys/kern/kern_poll.c b/sys/kern/kern_poll.c index a627bb1658..c2b310c52a 100644 --- a/sys/kern/kern_poll.c +++ b/sys/kern/kern_poll.c @@ -149,11 +149,11 @@ static int poll_defcpu = -1; SYSCTL_INT(_kern_polling, OID_AUTO, defcpu, CTLFLAG_RD, &poll_defcpu, 0, "default CPU to run device polling"); -static uint32_t poll_cpumask0 = 0xffffffff; -TUNABLE_INT("kern.polling.cpumask", (int *)&poll_cpumask0); +static cpumask_t poll_cpumask0 = (cpumask_t)-1; +TUNABLE_ULONG("kern.polling.cpumask", (u_long *)&poll_cpumask0); -static uint32_t poll_cpumask; -SYSCTL_INT(_kern_polling, OID_AUTO, cpumask, CTLFLAG_RD, +static cpumask_t poll_cpumask; +SYSCTL_LONG(_kern_polling, OID_AUTO, cpumask, CTLFLAG_RD, &poll_cpumask, 0, "CPUs that can run device polling"); static int polling_enabled = 1; /* global polling enable */ @@ -231,7 +231,7 @@ init_device_poll_pcpu(int cpuid) if (cpuid >= POLLCTX_MAX) return; - if (((1 << cpuid) & poll_cpumask0) == 0) + if ((CPUMASK(cpuid) & poll_cpumask0) == 0) return; if (poll_burst_max < MIN_POLL_BURST_MAX) @@ -242,7 +242,7 @@ init_device_poll_pcpu(int cpuid) if (poll_each_burst > poll_burst_max) poll_each_burst = poll_burst_max; - poll_cpumask |= (1 << cpuid); + poll_cpumask |= CPUMASK(cpuid); pctx = kmalloc(sizeof(*pctx), M_DEVBUF, M_WAITOK | M_ZERO); @@ -738,7 +738,7 @@ ether_pollcpu_register(struct ifnet *ifp, int cpuid) if (cpuid < 0 || cpuid >= POLLCTX_MAX) return 0; - if (((1 << cpuid) & poll_cpumask) == 0) { + if ((CPUMASK(cpuid) & poll_cpumask) == 0) { /* Polling is not supported on 'cpuid' */ return 0; } diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c index 3bfec9996f..8bf06a78b1 100644 --- a/sys/kern/kern_proc.c +++ b/sys/kern/kern_proc.c @@ -886,7 +886,7 @@ sysctl_kern_proc(SYSCTL_HANDLER_ARGS) int nid; nid = (origcpu + n) % ncpus; - if ((smp_active_mask & (1 << nid)) == 0) + if ((smp_active_mask & CPUMASK(nid)) == 0) continue; rgd = globaldata_find(nid); lwkt_setcpu_self(rgd); diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 9f963f663f..90eb88a743 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -78,8 +78,8 @@ int lbolt; int lbolt_syncer; int sched_quantum; /* Roundrobin scheduling quantum in ticks. */ int ncpus; -int ncpus2, ncpus2_shift, ncpus2_mask; -int ncpus_fit, ncpus_fit_mask; +int ncpus2, ncpus2_shift, ncpus2_mask; /* note: mask not cpumask_t */ +int ncpus_fit, ncpus_fit_mask; /* note: mask not cpumask_t */ int safepri; int tsleep_now_works; int tsleep_crypto_dump = 0; @@ -360,13 +360,13 @@ _tsleep_interlock(globaldata_t gd, const volatile void *ident, int flags) id = LOOKUP(td->td_wchan); TAILQ_REMOVE(&gd->gd_tsleep_hash[id], td, td_sleepq); if (TAILQ_FIRST(&gd->gd_tsleep_hash[id]) == NULL) - atomic_clear_int(&slpque_cpumasks[id], gd->gd_cpumask); + atomic_clear_cpumask(&slpque_cpumasks[id], gd->gd_cpumask); } else { td->td_flags |= TDF_TSLEEPQ; } id = LOOKUP(ident); TAILQ_INSERT_TAIL(&gd->gd_tsleep_hash[id], td, td_sleepq); - atomic_set_int(&slpque_cpumasks[id], gd->gd_cpumask); + atomic_set_cpumask(&slpque_cpumasks[id], gd->gd_cpumask); td->td_wchan = ident; td->td_wdomain = flags & PDOMAIN_MASK; crit_exit_quick(td); @@ -393,7 +393,7 @@ _tsleep_remove(thread_t td) id = LOOKUP(td->td_wchan); TAILQ_REMOVE(&gd->gd_tsleep_hash[id], td, td_sleepq); if (TAILQ_FIRST(&gd->gd_tsleep_hash[id]) == NULL) - atomic_clear_int(&slpque_cpumasks[id], gd->gd_cpumask); + atomic_clear_cpumask(&slpque_cpumasks[id], gd->gd_cpumask); td->td_wchan = NULL; td->td_wdomain = 0; } diff --git a/sys/kern/kern_usched.c b/sys/kern/kern_usched.c index 9ef93fefa4..40641875bd 100644 --- a/sys/kern/kern_usched.c +++ b/sys/kern/kern_usched.c @@ -221,11 +221,11 @@ sys_usched_set(struct usched_set_args *uap) error = EFBIG; break; } - if ((smp_active_mask & (1 << cpuid)) == 0) { + if ((smp_active_mask & CPUMASK(cpuid)) == 0) { error = EINVAL; break; } - lp->lwp_cpumask = 1 << cpuid; + lp->lwp_cpumask = CPUMASK(cpuid); if (cpuid != mycpu->gd_cpuid) lwkt_migratecpu(cpuid); break; @@ -251,11 +251,11 @@ sys_usched_set(struct usched_set_args *uap) error = EFBIG; break; } - if (!(smp_active_mask & (1 << cpuid))) { + if (!(smp_active_mask & CPUMASK(cpuid))) { error = EINVAL; break; } - lp->lwp_cpumask |= 1 << cpuid; + lp->lwp_cpumask |= CPUMASK(cpuid); break; case USCHED_DEL_CPU: /* USCHED_DEL_CPU doesn't require special privileges. */ @@ -271,13 +271,14 @@ sys_usched_set(struct usched_set_args *uap) break; } lp = curthread->td_lwp; - mask = lp->lwp_cpumask & smp_active_mask & ~(1 << cpuid); + mask = lp->lwp_cpumask & smp_active_mask & ~CPUMASK(cpuid); if (mask == 0) error = EPERM; else { - lp->lwp_cpumask &= ~(1 << cpuid); + lp->lwp_cpumask &= ~CPUMASK(cpuid); if ((lp->lwp_cpumask & mycpu->gd_cpumask) == 0) { - cpuid = bsfl(lp->lwp_cpumask & smp_active_mask); + cpuid = BSFCPUMASK(lp->lwp_cpumask & + smp_active_mask); lwkt_migratecpu(cpuid); } } diff --git a/sys/kern/lwkt_ipiq.c b/sys/kern/lwkt_ipiq.c index 5ac07b10ec..59c9600dbc 100644 --- a/sys/kern/lwkt_ipiq.c +++ b/sys/kern/lwkt_ipiq.c @@ -381,16 +381,16 @@ lwkt_send_ipiq3_bycpu(int dcpu, ipifunc3_t func, void *arg1, int arg2) * The message will not be sent to stopped cpus. */ int -lwkt_send_ipiq3_mask(u_int32_t mask, ipifunc3_t func, void *arg1, int arg2) +lwkt_send_ipiq3_mask(cpumask_t mask, ipifunc3_t func, void *arg1, int arg2) { int cpuid; int count = 0; mask &= ~stopped_cpus; while (mask) { - cpuid = bsfl(mask); + cpuid = BSFCPUMASK(mask); lwkt_send_ipiq3(globaldata_find(cpuid), func, arg1, arg2); - mask &= ~(1 << cpuid); + mask &= ~CPUMASK(cpuid); ++count; } return(count); @@ -601,7 +601,7 @@ lwkt_sync_ipiq(void *arg) { cpumask_t *cpumask = arg; - atomic_clear_int(cpumask, mycpu->gd_cpumask); + atomic_clear_cpumask(cpumask, mycpu->gd_cpumask); if (*cpumask == 0) wakeup(cpumask); } @@ -645,7 +645,7 @@ lwkt_cpusync_simple(cpumask_t mask, cpusync_func_t func, void *data) cmd.cs_fin2_func = NULL; cmd.cs_data = data; lwkt_cpusync_start(mask & mycpu->gd_other_cpus, &cmd); - if (mask & (1 << mycpu->gd_cpuid)) + if (mask & CPUMASK(mycpu->gd_cpuid)) func(&cmd); lwkt_cpusync_finish(&cmd); } @@ -675,7 +675,7 @@ lwkt_cpusync_fastdata(cpumask_t mask, cpusync_func2_t func, void *data) cmd.cs_fin2_func = func; cmd.cs_data = NULL; lwkt_cpusync_start(mask & mycpu->gd_other_cpus, &cmd); - if (mask & (1 << mycpu->gd_cpuid)) + if (mask & CPUMASK(mycpu->gd_cpuid)) func(data); lwkt_cpusync_finish(&cmd); } diff --git a/sys/kern/usched_bsd4.c b/sys/kern/usched_bsd4.c index ae1ce84744..23581a9312 100644 --- a/sys/kern/usched_bsd4.c +++ b/sys/kern/usched_bsd4.c @@ -205,7 +205,7 @@ rqinit(void *dummy) TAILQ_INIT(&bsd4_rtqueues[i]); TAILQ_INIT(&bsd4_idqueues[i]); } - atomic_clear_int(&bsd4_curprocmask, 1); + atomic_clear_cpumask(&bsd4_curprocmask, 1); } SYSINIT(runqueue, SI_BOOT2_USCHED, SI_ORDER_FIRST, rqinit, NULL) @@ -267,7 +267,7 @@ bsd4_acquire_curproc(struct lwp *lp) if (dd->uschedcp == lp) { dd->upri = lp->lwp_priority; } else if (dd->uschedcp == NULL) { - atomic_set_int(&bsd4_curprocmask, gd->gd_cpumask); + atomic_set_cpumask(&bsd4_curprocmask, gd->gd_cpumask); dd->uschedcp = lp; dd->upri = lp->lwp_priority; } else if (dd->upri > lp->lwp_priority) { @@ -337,7 +337,7 @@ bsd4_release_curproc(struct lwp *lp) KKASSERT((lp->lwp_flag & LWP_ONRUNQ) == 0); dd->uschedcp = NULL; /* don't let lp be selected */ dd->upri = PRIBASE_NULL; - atomic_clear_int(&bsd4_curprocmask, gd->gd_cpumask); + atomic_clear_cpumask(&bsd4_curprocmask, gd->gd_cpumask); bsd4_select_curproc(gd); crit_exit(); } @@ -370,7 +370,7 @@ bsd4_select_curproc(globaldata_t gd) spin_lock(&bsd4_spin); if ((nlp = chooseproc_locked(dd->uschedcp)) != NULL) { - atomic_set_int(&bsd4_curprocmask, 1 << cpuid); + atomic_set_cpumask(&bsd4_curprocmask, CPUMASK(cpuid)); dd->upri = nlp->lwp_priority; dd->uschedcp = nlp; spin_unlock(&bsd4_spin); @@ -378,8 +378,8 @@ bsd4_select_curproc(globaldata_t gd) lwkt_acquire(nlp->lwp_thread); #endif lwkt_schedule(nlp->lwp_thread); - } else if (bsd4_runqcount && (bsd4_rdyprocmask & (1 << cpuid))) { - atomic_clear_int(&bsd4_rdyprocmask, 1 << cpuid); + } else if (bsd4_runqcount && (bsd4_rdyprocmask & CPUMASK(cpuid))) { + atomic_clear_cpumask(&bsd4_rdyprocmask, CPUMASK(cpuid)); spin_unlock(&bsd4_spin); lwkt_schedule(&dd->helper_thread); } else { @@ -446,7 +446,7 @@ bsd4_setrunqueue(struct lwp *lp) * the kernel. */ if (dd->uschedcp == NULL) { - atomic_set_int(&bsd4_curprocmask, gd->gd_cpumask); + atomic_set_cpumask(&bsd4_curprocmask, gd->gd_cpumask); dd->uschedcp = lp; dd->upri = lp->lwp_priority; lwkt_schedule(lp->lwp_thread); @@ -486,11 +486,11 @@ bsd4_setrunqueue(struct lwp *lp) spin_unlock(&bsd4_spin); while (mask) { - tmpmask = ~((1 << cpuid) - 1); + tmpmask = ~(CPUMASK(cpuid) - 1); if (mask & tmpmask) - cpuid = bsfl(mask & tmpmask); + cpuid = BSFCPUMASK(mask & tmpmask); else - cpuid = bsfl(mask); + cpuid = BSFCPUMASK(mask); gd = globaldata_find(cpuid); dd = &bsd4_pcpu[cpuid]; @@ -501,7 +501,7 @@ bsd4_setrunqueue(struct lwp *lp) lwkt_send_ipiq(gd, need_user_resched_remote, NULL); break; } - mask &= ~(1 << cpuid); + mask &= ~CPUMASK(cpuid); } #else /* @@ -953,7 +953,7 @@ need_user_resched_remote(void *dummy) bsd4_pcpu_t dd = &bsd4_pcpu[gd->gd_cpuid]; if (dd->uschedcp == NULL && (bsd4_rdyprocmask & gd->gd_cpumask)) { - atomic_clear_int(&bsd4_rdyprocmask, gd->gd_cpumask); + atomic_clear_cpumask(&bsd4_rdyprocmask, gd->gd_cpumask); lwkt_schedule(&dd->helper_thread); } else { need_user_resched(); @@ -1114,7 +1114,7 @@ sched_thread(void *dummy) crit_enter_gd(gd); lwkt_deschedule_self(gd->gd_curthread); spin_lock(&bsd4_spin); - atomic_set_int(&bsd4_rdyprocmask, cpumask); + atomic_set_cpumask(&bsd4_rdyprocmask, cpumask); clear_user_resched(); /* This satisfied the reschedule request */ dd->rrcount = 0; /* Reset the round-robin counter */ @@ -1125,7 +1125,7 @@ sched_thread(void *dummy) */ KKASSERT(dd->uschedcp == NULL); if ((nlp = chooseproc_locked(NULL)) != NULL) { - atomic_set_int(&bsd4_curprocmask, cpumask); + atomic_set_cpumask(&bsd4_curprocmask, cpumask); dd->upri = nlp->lwp_priority; dd->uschedcp = nlp; spin_unlock(&bsd4_spin); @@ -1151,11 +1151,11 @@ sched_thread(void *dummy) ~bsd4_curprocmask; if (tmpmask) { if (tmpmask & ~(cpumask - 1)) - tmpid = bsfl(tmpmask & ~(cpumask - 1)); + tmpid = BSFCPUMASK(tmpmask & ~(cpumask - 1)); else - tmpid = bsfl(tmpmask); + tmpid = BSFCPUMASK(tmpmask); bsd4_scancpu = tmpid; - atomic_clear_int(&bsd4_rdyprocmask, 1 << tmpid); + atomic_clear_cpumask(&bsd4_rdyprocmask, CPUMASK(tmpid)); spin_unlock_wr(&bsd4_spin); lwkt_schedule(&bsd4_pcpu[tmpid].helper_thread); } else { @@ -1187,7 +1187,7 @@ sched_thread_cpu_init(void) for (i = 0; i < ncpus; ++i) { bsd4_pcpu_t dd = &bsd4_pcpu[i]; - cpumask_t mask = 1 << i; + cpumask_t mask = CPUMASK(i); if ((mask & smp_active_mask) == 0) continue; @@ -1203,8 +1203,8 @@ sched_thread_cpu_init(void) * been enabled in rqinit(). */ if (i) - atomic_clear_int(&bsd4_curprocmask, mask); - atomic_set_int(&bsd4_rdyprocmask, mask); + atomic_clear_cpumask(&bsd4_curprocmask, mask); + atomic_set_cpumask(&bsd4_rdyprocmask, mask); dd->upri = PRIBASE_NULL; } if (bootverbose) diff --git a/sys/kern/usched_dummy.c b/sys/kern/usched_dummy.c index a718130751..301de531fb 100644 --- a/sys/kern/usched_dummy.c +++ b/sys/kern/usched_dummy.c @@ -119,7 +119,7 @@ dummyinit(void *dummy) { TAILQ_INIT(&dummy_runq); spin_init(&dummy_spin); - atomic_clear_int(&dummy_curprocmask, 1); + atomic_clear_cpumask(&dummy_curprocmask, 1); } SYSINIT(runqueue, SI_BOOT2_USCHED, SI_ORDER_FIRST, dummyinit, NULL) @@ -156,7 +156,7 @@ dummy_acquire_curproc(struct lwp *lp) */ if (dd->uschedcp == lp || (dd->uschedcp == NULL && TAILQ_EMPTY(&dummy_runq))) { - atomic_set_int(&dummy_curprocmask, gd->gd_cpumask); + atomic_set_cpumask(&dummy_curprocmask, gd->gd_cpumask); dd->uschedcp = lp; return; } @@ -241,14 +241,14 @@ dummy_select_curproc(globaldata_t gd) spin_lock(&dummy_spin); if ((lp = TAILQ_FIRST(&dummy_runq)) == NULL) { dd->uschedcp = NULL; - atomic_clear_int(&dummy_curprocmask, gd->gd_cpumask); + atomic_clear_cpumask(&dummy_curprocmask, gd->gd_cpumask); spin_unlock(&dummy_spin); } else { --dummy_runqcount; TAILQ_REMOVE(&dummy_runq, lp, lwp_procq); lp->lwp_flag &= ~LWP_ONRUNQ; dd->uschedcp = lp; - atomic_set_int(&dummy_curprocmask, gd->gd_cpumask); + atomic_set_cpumask(&dummy_curprocmask, gd->gd_cpumask); spin_unlock(&dummy_spin); #ifdef SMP lwkt_acquire(lp->lwp_thread); @@ -280,7 +280,7 @@ dummy_setrunqueue(struct lwp *lp) if (dd->uschedcp == NULL) { dd->uschedcp = lp; - atomic_set_int(&dummy_curprocmask, gd->gd_cpumask); + atomic_set_cpumask(&dummy_curprocmask, gd->gd_cpumask); lwkt_schedule(lp->lwp_thread); } else { /* @@ -309,8 +309,8 @@ dummy_setrunqueue(struct lwp *lp) mask = ~dummy_curprocmask & dummy_rdyprocmask & gd->gd_other_cpus; if (mask) { - cpuid = bsfl(mask); - atomic_clear_int(&dummy_rdyprocmask, 1 << cpuid); + cpuid = BSFCPUMASK(mask); + atomic_clear_cpumask(&dummy_rdyprocmask, CPUMASK(cpuid)); spin_unlock(&dummy_spin); lwkt_schedule(&dummy_pcpu[cpuid].helper_thread); } else { @@ -474,11 +474,11 @@ dummy_sched_thread(void *dummy) gd = mycpu; cpuid = gd->gd_cpuid; dd = &dummy_pcpu[cpuid]; - cpumask = 1 << cpuid; + cpumask = CPUMASK(cpuid); for (;;) { lwkt_deschedule_self(gd->gd_curthread); /* interlock */ - atomic_set_int(&dummy_rdyprocmask, cpumask); + atomic_set_cpumask(&dummy_rdyprocmask, cpumask); spin_lock(&dummy_spin); if (dd->uschedcp) { /* @@ -488,9 +488,9 @@ dummy_sched_thread(void *dummy) tmpmask = ~dummy_curprocmask & dummy_rdyprocmask & gd->gd_other_cpus; if (tmpmask && dummy_runqcount) { - tmpid = bsfl(tmpmask); + tmpid = BSFCPUMASK(tmpmask); KKASSERT(tmpid != cpuid); - atomic_clear_int(&dummy_rdyprocmask, 1 << tmpid); + atomic_clear_cpumask(&dummy_rdyprocmask, CPUMASK(tmpid)); spin_unlock(&dummy_spin); lwkt_schedule(&dummy_pcpu[tmpid].helper_thread); } else { @@ -501,7 +501,7 @@ dummy_sched_thread(void *dummy) TAILQ_REMOVE(&dummy_runq, lp, lwp_procq); lp->lwp_flag &= ~LWP_ONRUNQ; dd->uschedcp = lp; - atomic_set_int(&dummy_curprocmask, cpumask); + atomic_set_cpumask(&dummy_curprocmask, cpumask); spin_unlock(&dummy_spin); #ifdef SMP lwkt_acquire(lp->lwp_thread); @@ -528,7 +528,7 @@ dummy_sched_thread_cpu_init(void) for (i = 0; i < ncpus; ++i) { dummy_pcpu_t dd = &dummy_pcpu[i]; - cpumask_t mask = 1 << i; + cpumask_t mask = CPUMASK(i); if ((mask & smp_active_mask) == 0) continue; @@ -544,8 +544,8 @@ dummy_sched_thread_cpu_init(void) * been enabled in rqinit(). */ if (i) - atomic_clear_int(&dummy_curprocmask, mask); - atomic_set_int(&dummy_rdyprocmask, mask); + atomic_clear_cpumask(&dummy_curprocmask, mask); + atomic_set_cpumask(&dummy_rdyprocmask, mask); } if (bootverbose) kprintf("\n"); diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c index de97ffecb0..62e76f4546 100644 --- a/sys/netinet/tcp_subr.c +++ b/sys/netinet/tcp_subr.c @@ -1174,7 +1174,7 @@ tcp_pcblist(SYSCTL_HANDLER_ARGS) int cpu_id; cpu_id = (origcpu + ccpu) % ncpus; - if ((smp_active_mask & (1 << cpu_id)) == 0) + if ((smp_active_mask & CPUMASK(cpu_id)) == 0) continue; rgd = globaldata_find(cpu_id); lwkt_setcpu_self(rgd); diff --git a/sys/platform/pc32/acpica5/madt.c b/sys/platform/pc32/acpica5/madt.c index bebcf212a8..7a21d0ac6b 100644 --- a/sys/platform/pc32/acpica5/madt.c +++ b/sys/platform/pc32/acpica5/madt.c @@ -761,7 +761,7 @@ madt_set_ids(void *dummy) if (madt == NULL) return; for (i = 0; i < ncpus; i++) { - if ((smp_active_mask & (1 << i)) == 0) + if ((smp_active_mask & CPUMASK(i)) == 0) continue; md = (struct mdglobaldata *)globaldata_find(i); KKASSERT(md != NULL); diff --git a/sys/platform/pc32/apic/mpapic.c b/sys/platform/pc32/apic/mpapic.c index 1443044a18..e4912a96eb 100644 --- a/sys/platform/pc32/apic/mpapic.c +++ b/sys/platform/pc32/apic/mpapic.c @@ -926,12 +926,12 @@ single_apic_ipi_passive(int cpu, int vector, int delivery_mode) * APIC_DELMODE_FIXED or APIC_DELMODE_LOWPRIO. */ void -selected_apic_ipi(u_int target, int vector, int delivery_mode) +selected_apic_ipi(cpumask_t target, int vector, int delivery_mode) { crit_enter(); while (target) { - int n = bsfl(target); - target &= ~(1 << n); + int n = BSFCPUMASK(target); + target &= ~CPUMASK(n); single_apic_ipi(n, vector, delivery_mode); } crit_exit(); diff --git a/sys/platform/pc32/i386/genassym.c b/sys/platform/pc32/i386/genassym.c index 0176996cfd..4cbb97ce27 100644 --- a/sys/platform/pc32/i386/genassym.c +++ b/sys/platform/pc32/i386/genassym.c @@ -104,6 +104,7 @@ ASSYM(TD_SAVEFPU, offsetof(struct thread, td_mach) + offsetof(struct md_thread, ASSYM(TDPRI_INT_SUPPORT, TDPRI_INT_SUPPORT); #ifdef SMP ASSYM(CPUMASK_LOCK, CPUMASK_LOCK); +ASSYM(CPUMASK_BIT, CPUMASK_BIT); #endif ASSYM(V_TRAP, offsetof(struct vmmeter, v_trap)); @@ -213,7 +214,7 @@ ASSYM(GD_TSS_GDT, offsetof(struct mdglobaldata, gd_tss_gdt)); ASSYM(GD_NPXTHREAD, offsetof(struct mdglobaldata, gd_npxthread)); ASSYM(GD_FPU_LOCK, offsetof(struct mdglobaldata, gd_fpu_lock)); ASSYM(GD_SAVEFPU, offsetof(struct mdglobaldata, gd_savefpu)); -ASSYM(GD_OTHER_CPUS, offsetof(struct mdglobaldata, gd_other_cpus)); +ASSYM(GD_OTHER_CPUS, offsetof(struct mdglobaldata, mi.gd_other_cpus)); ASSYM(GD_SS_EFLAGS, offsetof(struct mdglobaldata, gd_ss_eflags)); ASSYM(GD_CMAP1, offsetof(struct mdglobaldata, gd_CMAP1)); ASSYM(GD_CMAP2, offsetof(struct mdglobaldata, gd_CMAP2)); diff --git a/sys/platform/pc32/i386/machdep.c b/sys/platform/pc32/i386/machdep.c index d1372332dc..5af7336b07 100644 --- a/sys/platform/pc32/i386/machdep.c +++ b/sys/platform/pc32/i386/machdep.c @@ -966,7 +966,7 @@ handle_cpu_contention_mask(void) mask = cpu_contention_mask; cpu_ccfence(); - if (mask && bsfl(mask) != mycpu->gd_cpuid) + if (mask && BSFCPUMASK(mask) != mycpu->gd_cpuid) DELAY(2); } diff --git a/sys/platform/pc32/i386/mp_machdep.c b/sys/platform/pc32/i386/mp_machdep.c index 43532e6621..46d7abbd9d 100644 --- a/sys/platform/pc32/i386/mp_machdep.c +++ b/sys/platform/pc32/i386/mp_machdep.c @@ -304,7 +304,7 @@ static int mptable_probe(void); static int mptable_search(void); static int mptable_check(vm_paddr_t); static int mptable_search_sig(u_int32_t target, int count); -static int mptable_hyperthread_fixup(u_int, int); +static int mptable_hyperthread_fixup(cpumask_t, int); #ifdef SMP /* APIC-IO */ static void mptable_pass1(struct mptable_pos *); static void mptable_pass2(struct mptable_pos *); @@ -1003,7 +1003,7 @@ mptable_pass2(struct mptable_pos *mpt) * with the number of logical CPU's in the processor. */ static int -mptable_hyperthread_fixup(u_int id_mask, int cpu_count) +mptable_hyperthread_fixup(cpumask_t id_mask, int cpu_count) { int i, id, lcpus_max, logical_cpus; @@ -1049,7 +1049,7 @@ mptable_hyperthread_fixup(u_int id_mask, int cpu_count) */ dist = cur = prev = -1; for (id = 0; id < MAXCPU; ++id) { - if ((id_mask & 1 << id) == 0) + if ((id_mask & CPUMASK(id)) == 0) continue; cur = id; @@ -1090,13 +1090,13 @@ mptable_hyperthread_fixup(u_int id_mask, int cpu_count) * already in the table, then kill the fixup. */ for (id = 0; id < MAXCPU; id++) { - if ((id_mask & 1 << id) == 0) + if ((id_mask & CPUMASK(id)) == 0) continue; /* First, make sure we are on a logical_cpus boundary. */ if (id % logical_cpus != 0) return 0; for (i = id + 1; i < id + logical_cpus; i++) - if ((id_mask & 1 << i) != 0) + if ((id_mask & CPUMASK(i)) != 0) return 0; } return logical_cpus; @@ -2308,7 +2308,7 @@ start_all_aps(u_int boot_addr) ncpus_fit_mask = ncpus_fit - 1; /* build our map of 'other' CPUs */ - mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * ncpus); bzero(mycpu->gd_ipiq, sizeof(lwkt_ipiq) * ncpus); @@ -2534,7 +2534,7 @@ start_ap(struct mdglobaldata *gd, u_int boot_addr, int smibest) /* wait for it to start, see ap_init() */ set_apic_timer(5000000);/* == 5 seconds */ while (read_apic_timer()) { - if (smp_startup_mask & (1 << gd->mi.gd_cpuid)) + if (smp_startup_mask & CPUMASK(gd->mi.gd_cpuid)) return 1; /* return SUCCESS */ } @@ -2593,7 +2593,7 @@ smp_invltlb(void) crit_enter_gd(&md->mi); md->gd_invltlb_ret = 0; ++md->mi.gd_cnt.v_smpinvltlb; - atomic_set_int(&smp_invltlb_req, md->mi.gd_cpumask); + atomic_set_cpumask(&smp_invltlb_req, md->mi.gd_cpumask); #ifdef SMP_INVLTLB_DEBUG again: #endif @@ -2626,7 +2626,9 @@ again: if (xcount > 2) lwkt_process_ipiq(); if (xcount > 3) { - int bcpu = bsfl(~md->gd_invltlb_ret & ~md->mi.gd_cpumask & smp_active_mask); + int bcpu = BSFCPUMASK(~md->gd_invltlb_ret & + ~md->mi.gd_cpumask & + smp_active_mask); globaldata_t xgd; kprintf("bcpu %d\n", bcpu); xgd = globaldata_find(bcpu); @@ -2639,7 +2641,7 @@ again: } #endif } - atomic_clear_int(&smp_invltlb_req, md->mi.gd_cpumask); + atomic_clear_cpumask(&smp_invltlb_req, md->mi.gd_cpumask); crit_exit_gd(&md->mi); #endif } @@ -2663,10 +2665,10 @@ smp_invltlb_intr(void) cpu_mfence(); cpu_invltlb(); while (mask) { - cpu = bsfl(mask); - mask &= ~(1 << cpu); + cpu = BSFCPUMASK(mask); + mask &= ~CPUMASK(cpu); omd = (struct mdglobaldata *)globaldata_find(cpu); - atomic_set_int(&omd->gd_invltlb_ret, md->mi.gd_cpumask); + atomic_set_cpumask(&omd->gd_invltlb_ret, md->mi.gd_cpumask); } } @@ -2690,7 +2692,7 @@ smp_invltlb_intr(void) * from executing at same time. */ int -stop_cpus(u_int map) +stop_cpus(cpumask_t map) { map &= smp_active_mask; @@ -2718,7 +2720,7 @@ stop_cpus(u_int map) * 1: ok */ int -restart_cpus(u_int map) +restart_cpus(cpumask_t map) { /* signal other cpus to restart */ started_cpus = map & smp_active_mask; @@ -2750,7 +2752,7 @@ ap_init(void) * interrupts physically disabled and remote cpus could deadlock * trying to send us an IPI. */ - smp_startup_mask |= 1 << mycpu->gd_cpuid; + smp_startup_mask |= CPUMASK(mycpu->gd_cpuid); cpu_mfence(); /* @@ -2790,7 +2792,7 @@ ap_init(void) #endif /* Build our map of 'other' CPUs. */ - mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); kprintf("SMP: AP CPU #%d Launched!\n", mycpu->gd_cpuid); @@ -2824,7 +2826,7 @@ ap_init(void) * nothing we've done put it there. */ KKASSERT(curthread->td_mpcount == 1); - smp_active_mask |= 1 << mycpu->gd_cpuid; + smp_active_mask |= CPUMASK(mycpu->gd_cpuid); /* * Enable interrupts here. idle_restore will also do it, but @@ -2875,7 +2877,7 @@ SYSINIT(finishsmp, SI_BOOT2_FINISH_SMP, SI_ORDER_FIRST, ap_finish, NULL) void cpu_send_ipiq(int dcpu) { - if ((1 << dcpu) & smp_active_mask) + if (CPUMASK(dcpu) & smp_active_mask) single_apic_ipi(dcpu, XIPIQ_OFFSET, APIC_DELMODE_FIXED); } @@ -2887,7 +2889,7 @@ int cpu_send_ipiq_passive(int dcpu) { int r = 0; - if ((1 << dcpu) & smp_active_mask) { + if (CPUMASK(dcpu) & smp_active_mask) { r = single_apic_ipi_passive(dcpu, XIPIQ_OFFSET, APIC_DELMODE_FIXED); } diff --git a/sys/platform/pc32/i386/pmap.c b/sys/platform/pc32/i386/pmap.c index 84646f3448..ac49d09f45 100644 --- a/sys/platform/pc32/i386/pmap.c +++ b/sys/platform/pc32/i386/pmap.c @@ -3447,7 +3447,7 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) if (curthread->td_lwp == lp) { pmap = vmspace_pmap(newvm); #if defined(SMP) - atomic_set_int(&pmap->pm_active, mycpu->gd_cpumask); + atomic_set_cpumask(&pmap->pm_active, mycpu->gd_cpumask); if (pmap->pm_active & CPUMASK_LOCK) pmap_interlock_wait(newvm); #else @@ -3460,9 +3460,10 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) load_cr3(curthread->td_pcb->pcb_cr3); pmap = vmspace_pmap(oldvm); #if defined(SMP) - atomic_clear_int(&pmap->pm_active, mycpu->gd_cpumask); + atomic_clear_cpumask(&pmap->pm_active, + mycpu->gd_cpumask); #else - pmap->pm_active &= ~1; + pmap->pm_active &= ~(cpumask_t)1; #endif } } diff --git a/sys/platform/pc32/i386/pmap_inval.c b/sys/platform/pc32/i386/pmap_inval.c index 9b26855e26..29edf8b5fa 100644 --- a/sys/platform/pc32/i386/pmap_inval.c +++ b/sys/platform/pc32/i386/pmap_inval.c @@ -106,7 +106,7 @@ pmap_inval_interlock(pmap_inval_info_t info, pmap_t pmap, vm_offset_t va) for (;;) { oactive = pmap->pm_active & ~CPUMASK_LOCK; nactive = oactive | CPUMASK_LOCK; - if (atomic_cmpset_int(&pmap->pm_active, oactive, nactive)) + if (atomic_cmpset_cpumask(&pmap->pm_active, oactive, nactive)) break; crit_enter(); lwkt_process_ipiq(); @@ -151,7 +151,7 @@ void pmap_inval_deinterlock(pmap_inval_info_t info, pmap_t pmap) { #ifdef SMP - atomic_clear_int(&pmap->pm_active, CPUMASK_LOCK); + atomic_clear_cpumask(&pmap->pm_active, CPUMASK_LOCK); #endif } diff --git a/sys/platform/pc32/i386/vm_machdep.c b/sys/platform/pc32/i386/vm_machdep.c index 5d4a44fd1b..fcc81dfb7f 100644 --- a/sys/platform/pc32/i386/vm_machdep.c +++ b/sys/platform/pc32/i386/vm_machdep.c @@ -385,7 +385,7 @@ cpu_reset_proxy(void) cpu_reset_proxy_active = 3; while (cpu_reset_proxy_active == 3) ; /* Wait for other cpu to enable interrupts */ - stop_cpus((1<gd_cpuid); diff --git a/sys/platform/pc32/include/globaldata.h b/sys/platform/pc32/include/globaldata.h index 2135a1c74b..0733c1bef6 100644 --- a/sys/platform/pc32/include/globaldata.h +++ b/sys/platform/pc32/include/globaldata.h @@ -76,13 +76,13 @@ struct mdglobaldata { union savefpu gd_savefpu; /* fast bcopy/zero temp fpu save area */ int gd_fpu_lock; /* fast bcopy/zero cpu lock */ int gd_fpending; /* fast interrupt pending */ - int unused002; + int unused001; int gd_spending; /* software interrupt pending */ int gd_sdelayed; /* delayed software ints */ int gd_currentldt; int gd_private_tss; - u_int unused001; - u_int gd_other_cpus; + u_int unused002; + u_int unused003; u_int gd_ss_eflags; pt_entry_t *gd_CMAP1; pt_entry_t *gd_CMAP2; diff --git a/sys/platform/pc32/include/pmap.h b/sys/platform/pc32/include/pmap.h index 24b5cb27ac..37d0355c26 100644 --- a/sys/platform/pc32/include/pmap.h +++ b/sys/platform/pc32/include/pmap.h @@ -238,7 +238,8 @@ struct pmap { #define pmap_resident_count(pmap) (pmap)->pm_stats.resident_count -#define CPUMASK_LOCK (1 << SMP_MAXCPU) +#define CPUMASK_LOCK CPUMASK(SMP_MAXCPU) +#define CPUMASK_BIT SMP_MAXCPU /* 1 << SMP_MAXCPU */ typedef struct pmap *pmap_t; diff --git a/sys/platform/pc32/include/smp.h b/sys/platform/pc32/include/smp.h index fd9acb099d..70162d36d1 100644 --- a/sys/platform/pc32/include/smp.h +++ b/sys/platform/pc32/include/smp.h @@ -45,8 +45,8 @@ extern int bootMP_size; void bootMP (void); /* global data in apic_vector.s */ -extern volatile u_int stopped_cpus; -extern volatile u_int started_cpus; +extern volatile cpumask_t stopped_cpus; +extern volatile cpumask_t started_cpus; extern volatile u_int checkstate_probed_cpus; extern void (*cpustop_restartfunc) (void); @@ -109,9 +109,9 @@ int apic_polarity (int, int); void assign_apic_irq (int apic, int intpin, int irq); void revoke_apic_irq (int irq); void init_secondary (void); -int stop_cpus (u_int); +int stop_cpus (cpumask_t); void ap_init (void); -int restart_cpus (u_int); +int restart_cpus (cpumask_t); void forward_signal (struct proc *); #ifndef _SYS_QUEUE_H_ @@ -137,7 +137,7 @@ void apic_dump (char*); void apic_initialize (boolean_t); void imen_dump (void); int apic_ipi (int, int, int); -void selected_apic_ipi (u_int, int, int); +void selected_apic_ipi (cpumask_t, int, int); void single_apic_ipi(int cpu, int vector, int delivery_mode); int single_apic_ipi_passive(int cpu, int vector, int delivery_mode); int io_apic_setup (int); diff --git a/sys/platform/pc64/acpica5/madt.c b/sys/platform/pc64/acpica5/madt.c index 07382715a0..62fa3e8ee4 100644 --- a/sys/platform/pc64/acpica5/madt.c +++ b/sys/platform/pc64/acpica5/madt.c @@ -761,7 +761,7 @@ madt_set_ids(void *dummy) if (madt == NULL) return; for (i = 0; i < ncpus; i++) { - if ((smp_active_mask & (1 << i)) == 0) + if ((smp_active_mask & CPUMASK(i)) == 0) continue; md = (struct mdglobaldata *)globaldata_find(i); KKASSERT(md != NULL); diff --git a/sys/platform/pc64/apic/apic_vector.s b/sys/platform/pc64/apic/apic_vector.s index 9fa158daed..21d9c9110b 100644 --- a/sys/platform/pc64/apic/apic_vector.s +++ b/sys/platform/pc64/apic/apic_vector.s @@ -216,7 +216,7 @@ Xcpustop: addq %rax, %rdi call CNAME(savectx) /* Save process context */ - movl PCPU(cpuid), %eax + movslq PCPU(cpuid), %rax /* * Indicate that we have stopped and loop waiting for permission @@ -227,7 +227,7 @@ Xcpustop: * (e.g. Xtimer, Xinvltlb). */ MPLOCKED - btsl %eax, stopped_cpus /* stopped_cpus |= (1<pm_stats.resident_count diff --git a/sys/platform/pc64/include/smp.h b/sys/platform/pc64/include/smp.h index f348dee992..768c0779dd 100644 --- a/sys/platform/pc64/include/smp.h +++ b/sys/platform/pc64/include/smp.h @@ -54,8 +54,8 @@ extern u_int32_t mptramp_pagetables; void bootMP (void); /* global data in apic_vector.s */ -extern volatile u_int stopped_cpus; -extern volatile u_int started_cpus; +extern volatile cpumask_t stopped_cpus; +extern volatile cpumask_t started_cpus; extern volatile u_int checkstate_probed_cpus; extern void (*cpustop_restartfunc) (void); @@ -111,9 +111,9 @@ int apic_polarity (int, int); void assign_apic_irq (int apic, int intpin, int irq); void revoke_apic_irq (int irq); void init_secondary (void); -int stop_cpus (u_int); +int stop_cpus (cpumask_t); void ap_init (void); -int restart_cpus (u_int); +int restart_cpus (cpumask_t); void forward_signal (struct proc *); #ifndef _SYS_QUEUE_H_ @@ -139,7 +139,7 @@ void apic_dump (char*); void apic_initialize (boolean_t); void imen_dump (void); int apic_ipi (int, int, int); -void selected_apic_ipi (u_int, int, int); +void selected_apic_ipi (cpumask_t, int, int); void single_apic_ipi(int cpu, int vector, int delivery_mode); int single_apic_ipi_passive(int cpu, int vector, int delivery_mode); int io_apic_setup (int); diff --git a/sys/platform/pc64/include/vmparam.h b/sys/platform/pc64/include/vmparam.h index 3025517d3e..35c593874b 100644 --- a/sys/platform/pc64/include/vmparam.h +++ b/sys/platform/pc64/include/vmparam.h @@ -104,10 +104,15 @@ * The kernel address space can be up to (I think) 511 page directory * pages. Each one represents 1G. NKPDPE defines the size of the kernel * address space, curently set to 128G. + * + * The ending address is non-inclusive of the per-cpu data array + * which starts at MPPTDI (-16MB mark). MPPTDI is the page directory + * index in the last PD of the kernel address space and is typically + * set to (NPDEPG - 8) = (512 - 8). */ #define NKPDPE 128 -#define VM_MAX_KERNEL_ADDRESS KVADDR(KPML4I, NPDPEPG-1, NPDEPG-1, NPTEPG-1) -#define VM_MIN_KERNEL_ADDRESS KVADDR(KPML4I, NPDPEPG-NKPDPE, 0, 0) +#define VM_MIN_KERNEL_ADDRESS KVADDR(KPML4I, NPDPEPG - NKPDPE, 0, 0) +#define VM_MAX_KERNEL_ADDRESS KVADDR(KPML4I, NPDPEPG - 1, MPPTDI, NPTEPG - 1) #define DMAP_MIN_ADDRESS KVADDR(DMPML4I, 0, 0, 0) #define DMAP_MAX_ADDRESS KVADDR(DMPML4I+1, 0, 0, 0) diff --git a/sys/platform/pc64/isa/clock.c b/sys/platform/pc64/isa/clock.c index 0399cdc4aa..4685fb4cb6 100644 --- a/sys/platform/pc64/isa/clock.c +++ b/sys/platform/pc64/isa/clock.c @@ -1076,7 +1076,7 @@ i8254_intr_initclock(struct cputimer_intr *cti, boolean_t selected) profhz = RTC_PROFRATE; } - /* Finish initializing 8253 timer 0. */ + /* Finish initializing 8254 timer 0. */ #ifdef SMP /* APIC-IO */ if (apic_io_enable) { apic_8254_intr = isa_apic_irq(0); diff --git a/sys/platform/pc64/x86_64/db_interface.c b/sys/platform/pc64/x86_64/db_interface.c index 249d874036..6a235b8d63 100644 --- a/sys/platform/pc64/x86_64/db_interface.c +++ b/sys/platform/pc64/x86_64/db_interface.c @@ -158,8 +158,8 @@ kdb_trap(int type, int code, struct x86_64_saved_state *regs) crit_enter(); #ifdef SMP - db_printf("\nCPU%d stopping CPUs: 0x%08x\n", - mycpu->gd_cpuid, mycpu->gd_other_cpus); + db_printf("\nCPU%d stopping CPUs: 0x%08jx\n", + mycpu->gd_cpuid, (uintmax_t)mycpu->gd_other_cpus); /* We stop all CPUs except ourselves (obviously) */ stop_cpus(mycpu->gd_other_cpus); @@ -182,13 +182,15 @@ kdb_trap(int type, int code, struct x86_64_saved_state *regs) db_global_jmpbuf_valid = FALSE; #ifdef SMP - db_printf("\nCPU%d restarting CPUs: 0x%08x\n", - mycpu->gd_cpuid, stopped_cpus); + db_printf("\nCPU%d restarting CPUs: 0x%016jx\n", + mycpu->gd_cpuid, (uintmax_t)stopped_cpus); /* Restart all the CPUs we previously stopped */ if (stopped_cpus != mycpu->gd_other_cpus) { - db_printf("whoa, other_cpus: 0x%08x, stopped_cpus: 0x%08x\n", - mycpu->gd_other_cpus, stopped_cpus); + db_printf("whoa, other_cpus: 0x%016jx, " + "stopped_cpus: 0x%016jx\n", + (uintmax_t)mycpu->gd_other_cpus, + (uintmax_t)stopped_cpus); panic("stop_cpus() failed"); } restart_cpus(stopped_cpus); diff --git a/sys/platform/pc64/x86_64/genassym.c b/sys/platform/pc64/x86_64/genassym.c index 2f7ab16ecf..6dd09f466e 100644 --- a/sys/platform/pc64/x86_64/genassym.c +++ b/sys/platform/pc64/x86_64/genassym.c @@ -208,7 +208,7 @@ ASSYM(GD_TSS_GDT, offsetof(struct mdglobaldata, gd_tss_gdt)); ASSYM(GD_NPXTHREAD, offsetof(struct mdglobaldata, gd_npxthread)); ASSYM(GD_FPU_LOCK, offsetof(struct mdglobaldata, gd_fpu_lock)); ASSYM(GD_SAVEFPU, offsetof(struct mdglobaldata, gd_savefpu)); -ASSYM(GD_OTHER_CPUS, offsetof(struct mdglobaldata, gd_other_cpus)); +ASSYM(GD_OTHER_CPUS, offsetof(struct mdglobaldata, mi.gd_other_cpus)); ASSYM(GD_SS_EFLAGS, offsetof(struct mdglobaldata, gd_ss_eflags)); ASSYM(GD_REQFLAGS, offsetof(struct mdglobaldata, mi.gd_reqflags)); @@ -239,6 +239,7 @@ ASSYM(MACHINTR_INTREN, offsetof(struct machintr_abi, intren)); ASSYM(TDPRI_INT_SUPPORT, TDPRI_INT_SUPPORT); #ifdef SMP ASSYM(CPUMASK_LOCK, CPUMASK_LOCK); +ASSYM(CPUMASK_BIT, CPUMASK_BIT); #endif #ifdef SMP diff --git a/sys/platform/pc64/x86_64/global.s b/sys/platform/pc64/x86_64/global.s index 8f154b8bd0..a4b88c3fb5 100644 --- a/sys/platform/pc64/x86_64/global.s +++ b/sys/platform/pc64/x86_64/global.s @@ -78,8 +78,6 @@ */ .globl gd_cpuid, gd_cpumask, gd_other_cpus .globl gd_ss_eflags, gd_intr_nesting_level - .globl gd_CMAP1, gd_CMAP2, gd_CMAP3, gd_PMAP1 - .globl gd_CADDR1, gd_CADDR2, gd_CADDR3, gd_PADDR1 .globl gd_spending, gd_fpending .globl gd_cnt, gd_private_tss .globl gd_scratch_rsp @@ -91,14 +89,6 @@ .set gd_other_cpus,globaldata + GD_OTHER_CPUS .set gd_ss_eflags,globaldata + GD_SS_EFLAGS .set gd_intr_nesting_level,globaldata + GD_INTR_NESTING_LEVEL - .set gd_CMAP1,globaldata + GD_PRV_CMAP1 - .set gd_CMAP2,globaldata + GD_PRV_CMAP2 - .set gd_CMAP3,globaldata + GD_PRV_CMAP3 - .set gd_PMAP1,globaldata + GD_PRV_PMAP1 - .set gd_CADDR1,globaldata + GD_PRV_CADDR1 - .set gd_CADDR2,globaldata + GD_PRV_CADDR2 - .set gd_CADDR3,globaldata + GD_PRV_CADDR3 - .set gd_PADDR1,globaldata + GD_PRV_PADDR1 .set gd_fpending,globaldata + GD_FPENDING .set gd_spending,globaldata + GD_SPENDING .set gd_cnt,globaldata + GD_CNT diff --git a/sys/platform/pc64/x86_64/machdep.c b/sys/platform/pc64/x86_64/machdep.c index d2a77a6390..da5d5e21d1 100644 --- a/sys/platform/pc64/x86_64/machdep.c +++ b/sys/platform/pc64/x86_64/machdep.c @@ -150,11 +150,7 @@ SYSINIT(cpu, SI_BOOT2_SMP, SI_ORDER_FIRST, cpu_startup, NULL) extern vm_offset_t ksym_start, ksym_end; #endif -uint64_t SMPptpa; -pt_entry_t *SMPpt; - - -struct privatespace CPU_prvspace[MAXCPU]; +struct privatespace CPU_prvspace[MAXCPU] __aligned(4096); /* XXX */ int _udatasel, _ucodesel, _ucode32sel; u_long atdevbase; @@ -992,7 +988,7 @@ handle_cpu_contention_mask(void) mask = cpu_contention_mask; cpu_ccfence(); - if (mask && bsfl(mask) != mycpu->gd_cpuid) + if (mask && BSFCPUMASK(mask) != mycpu->gd_cpuid) DELAY(2); } diff --git a/sys/platform/pc64/x86_64/mp_machdep.c b/sys/platform/pc64/x86_64/mp_machdep.c index 8d4c0f825c..8aaebfe3d9 100644 --- a/sys/platform/pc64/x86_64/mp_machdep.c +++ b/sys/platform/pc64/x86_64/mp_machdep.c @@ -261,17 +261,6 @@ int apic_id_to_logical[NAPICID]; char *bootSTK; static int bootAP; -/* - * SMP page table page. Setup by locore to point to a page table - * page from which we allocate per-cpu privatespace areas io_apics, - * and so forth. - */ - -#define IO_MAPPING_START_INDEX \ - (SMP_MAXCPU * sizeof(struct privatespace) / PAGE_SIZE) - -extern pt_entry_t *SMPpt; - struct pcb stoppcbs[MAXCPU]; extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32); @@ -301,7 +290,7 @@ static int mptable_probe(void); static int mptable_search(void); static int mptable_check(vm_paddr_t); static long mptable_search_sig(u_int32_t target, int count); -static int mptable_hyperthread_fixup(u_int, int); +static int mptable_hyperthread_fixup(cpumask_t, int); #ifdef SMP /* APIC-IO */ static void mptable_pass1(struct mptable_pos *); static void mptable_pass2(struct mptable_pos *); @@ -374,12 +363,6 @@ mptable_search(void) long x; u_int32_t target; - /* - * Make sure our SMPpt[] page table is big enough to hold all the - * mappings we need. - */ - KKASSERT(IO_MAPPING_START_INDEX < NPTEPG - 2); - POSTCODE(MP_PROBE_POST); /* see if EBDA exists */ @@ -564,7 +547,6 @@ mp_announce(void) if (apic_io_enable) { for (x = 0; x < mp_napics; ++x) { kprintf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x)); - kprintf("napics %d versions %p address %p\n", mp_napics, io_apic_versions, io_apic_address); kprintf(", version: 0x%08x", io_apic_versions[x]); kprintf(", at 0x%08lx\n", io_apic_address[x]); } @@ -997,8 +979,6 @@ mptable_pass2(struct mptable_pos *mpt) MALLOC(bus_data, bus_datum *, sizeof(bus_datum) * mp_nbusses, M_DEVBUF, M_WAITOK); - kprintf("xapics %d versions %p address %p\n", mp_napics, io_apic_versions, io_apic_address); - for (x = 0; x < mp_napics; x++) ioapic[x] = permanent_io_mapping(io_apic_address[x]); @@ -1044,7 +1024,7 @@ mptable_pass2(struct mptable_pos *mpt) * with the number of logical CPU's in the processor. */ static int -mptable_hyperthread_fixup(u_int id_mask, int cpu_count) +mptable_hyperthread_fixup(cpumask_t id_mask, int cpu_count) { int i, id, lcpus_max, logical_cpus; @@ -1090,7 +1070,7 @@ mptable_hyperthread_fixup(u_int id_mask, int cpu_count) */ dist = cur = prev = -1; for (id = 0; id < MAXCPU; ++id) { - if ((id_mask & 1 << id) == 0) + if ((id_mask & CPUMASK(id)) == 0) continue; cur = id; @@ -1131,13 +1111,13 @@ mptable_hyperthread_fixup(u_int id_mask, int cpu_count) * already in the table, then kill the fixup. */ for (id = 0; id < MAXCPU; id++) { - if ((id_mask & 1 << id) == 0) + if ((id_mask & CPUMASK(id)) == 0) continue; /* First, make sure we are on a logical_cpus boundary. */ if (id % logical_cpus != 0) return 0; for (i = id + 1; i < id + logical_cpus; i++) - if ((id_mask & 1 << i) != 0) + if ((id_mask & CPUMASK(i)) != 0) return 0; } return logical_cpus; @@ -1325,13 +1305,15 @@ fix_id_to_io_mapping(void) for (x = 0; x < NAPICID; x++) ID_TO_IO(x) = -1; - for (x = 0; x <= mp_naps; x++) - if (CPU_TO_ID(x) < NAPICID) + for (x = 0; x <= mp_naps; x++) { + if ((u_int)CPU_TO_ID(x) < NAPICID) ID_TO_IO(CPU_TO_ID(x)) = x; + } - for (x = 0; x < mp_napics; x++) - if (IO_TO_ID(x) < NAPICID) + for (x = 0; x < mp_napics; x++) { + if ((u_int)IO_TO_ID(x) < NAPICID) ID_TO_IO(IO_TO_ID(x)) = x; + } } @@ -1363,16 +1345,18 @@ io_apic_id_acceptable(int apic, int id) int cpu; /* Logical CPU number */ int oapic; /* Logical IO APIC number for other IO APIC */ - if (id >= NAPICID) + if ((u_int)id >= NAPICID) return 0; /* Out of range */ - for (cpu = 0; cpu <= mp_naps; cpu++) + for (cpu = 0; cpu <= mp_naps; cpu++) { if (CPU_TO_ID(cpu) == id) return 0; /* Conflict with CPU */ + } - for (oapic = 0; oapic < mp_napics && oapic < apic; oapic++) + for (oapic = 0; oapic < mp_napics && oapic < apic; oapic++) { if (IO_TO_ID(oapic) == id) return 0; /* Conflict with other APIC */ + } return 1; /* ID is acceptable for IO APIC */ } @@ -1446,7 +1430,7 @@ mptable_fix(void) bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type; bus_data[bus_0].bus_type = PCI; - /* swap each relavant INTerrupt entry */ + /* swap each relevant INTerrupt entry */ id = bus_data[bus_pci].bus_id; for (x = 0; x < nintrs; ++x) { if (io_apic_ints[x].src_bus_id == id) { @@ -1506,8 +1490,8 @@ mptable_fix(void) } else if (apic_int_type(0, 0) == 0) { kprintf("APIC_IO: MP table broken: ExtINT entry corrupt!\n"); for (x = 0; x < nintrs; ++x) - if ((0 == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && - (0 == io_apic_ints[x].dst_apic_int)) { + if ((ID_TO_IO(io_apic_ints[x].dst_apic_id) == 0) && + (io_apic_ints[x].dst_apic_int) == 0) { io_apic_ints[x].int_type = 3; io_apic_ints[x].int_vector = 0xff; break; @@ -2262,14 +2246,6 @@ start_all_aps(u_int boot_addr) /* prime data page for it to use */ mi_gdinit(&gd->mi, x); cpu_gdinit(gd, x); - gd->gd_CMAP1 = &SMPpt[pg + 0]; - gd->gd_CMAP2 = &SMPpt[pg + 1]; - gd->gd_CMAP3 = &SMPpt[pg + 2]; - gd->gd_PMAP1 = &SMPpt[pg + 3]; - gd->gd_CADDR1 = ps->CPAGE1; - gd->gd_CADDR2 = ps->CPAGE2; - gd->gd_CADDR3 = ps->CPAGE3; - gd->gd_PADDR1 = (pt_entry_t *)ps->PPAGE1; gd->mi.gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * (mp_naps + 1)); bzero(gd->mi.gd_ipiq, sizeof(lwkt_ipiq) * (mp_naps + 1)); @@ -2319,7 +2295,7 @@ start_all_aps(u_int boot_addr) ncpus_fit_mask = ncpus_fit - 1; /* build our map of 'other' CPUs */ - mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * ncpus); bzero(mycpu->gd_ipiq, sizeof(lwkt_ipiq) * ncpus); @@ -2547,7 +2523,7 @@ start_ap(struct mdglobaldata *gd, u_int boot_addr, int smibest) /* wait for it to start, see ap_init() */ set_apic_timer(5000000);/* == 5 seconds */ while (read_apic_timer()) { - if (smp_startup_mask & (1 << gd->mi.gd_cpuid)) + if (smp_startup_mask & CPUMASK(gd->mi.gd_cpuid)) return 1; /* return SUCCESS */ } @@ -2609,7 +2585,7 @@ smp_invltlb(void) crit_enter_gd(&md->mi); md->gd_invltlb_ret = 0; ++md->mi.gd_cnt.v_smpinvltlb; - atomic_set_int(&smp_invltlb_req, md->mi.gd_cpumask); + atomic_set_cpumask(&smp_invltlb_req, md->mi.gd_cpumask); #ifdef SMP_INVLTLB_DEBUG again: #endif @@ -2642,7 +2618,9 @@ again: if (xcount > 2) lwkt_process_ipiq(); if (xcount > 3) { - int bcpu = bsfl(~md->gd_invltlb_ret & ~md->mi.gd_cpumask & smp_active_mask); + int bcpu = BSFCPUMASK(~md->gd_invltlb_ret & + ~md->mi.gd_cpumask & + smp_active_mask); globaldata_t xgd; kprintf("bcpu %d\n", bcpu); @@ -2656,7 +2634,7 @@ again: } #endif } - atomic_clear_int(&smp_invltlb_req, md->mi.gd_cpumask); + atomic_clear_cpumask(&smp_invltlb_req, md->mi.gd_cpumask); crit_exit_gd(&md->mi); #endif } @@ -2680,10 +2658,10 @@ smp_invltlb_intr(void) mask = smp_invltlb_req; cpu_invltlb(); while (mask) { - cpu = bsfl(mask); - mask &= ~(1 << cpu); + cpu = BSFCPUMASK(mask); + mask &= ~CPUMASK(cpu); omd = (struct mdglobaldata *)globaldata_find(cpu); - atomic_set_int(&omd->gd_invltlb_ret, md->mi.gd_cpumask); + atomic_set_cpumask(&omd->gd_invltlb_ret, md->mi.gd_cpumask); } } @@ -2707,7 +2685,7 @@ smp_invltlb_intr(void) * from executing at same time. */ int -stop_cpus(u_int map) +stop_cpus(cpumask_t map) { map &= smp_active_mask; @@ -2735,7 +2713,7 @@ stop_cpus(u_int map) * 1: ok */ int -restart_cpus(u_int map) +restart_cpus(cpumask_t map) { /* signal other cpus to restart */ started_cpus = map & smp_active_mask; @@ -2767,7 +2745,7 @@ ap_init(void) * interrupts physically disabled and remote cpus could deadlock * trying to send us an IPI. */ - smp_startup_mask |= 1 << mycpu->gd_cpuid; + smp_startup_mask |= CPUMASK(mycpu->gd_cpuid); cpu_mfence(); /* @@ -2807,7 +2785,7 @@ ap_init(void) #endif /* Build our map of 'other' CPUs. */ - mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); kprintf("SMP: AP CPU #%d Launched!\n", mycpu->gd_cpuid); @@ -2844,7 +2822,7 @@ ap_init(void) * nothing we've done put it there. */ KKASSERT(curthread->td_mpcount == 1); - smp_active_mask |= 1 << mycpu->gd_cpuid; + smp_active_mask |= CPUMASK(mycpu->gd_cpuid); /* * Enable interrupts here. idle_restore will also do it, but @@ -2886,8 +2864,10 @@ ap_finish(void) } while (try_mplock() == 0) ; - if (bootverbose) - kprintf("Active CPU Mask: %08x\n", smp_active_mask); + if (bootverbose) { + kprintf("Active CPU Mask: %016jx\n", + (uintmax_t)smp_active_mask); + } } SYSINIT(finishsmp, SI_BOOT2_FINISH_SMP, SI_ORDER_FIRST, ap_finish, NULL) @@ -2895,7 +2875,7 @@ SYSINIT(finishsmp, SI_BOOT2_FINISH_SMP, SI_ORDER_FIRST, ap_finish, NULL) void cpu_send_ipiq(int dcpu) { - if ((1 << dcpu) & smp_active_mask) + if (CPUMASK(dcpu) & smp_active_mask) single_apic_ipi(dcpu, XIPIQ_OFFSET, APIC_DELMODE_FIXED); } @@ -2907,7 +2887,7 @@ int cpu_send_ipiq_passive(int dcpu) { int r = 0; - if ((1 << dcpu) & smp_active_mask) { + if (CPUMASK(dcpu) & smp_active_mask) { r = single_apic_ipi_passive(dcpu, XIPIQ_OFFSET, APIC_DELMODE_FIXED); } diff --git a/sys/platform/pc64/x86_64/pmap.c b/sys/platform/pc64/x86_64/pmap.c index 65280973dc..f58b1cbb5b 100644 --- a/sys/platform/pc64/x86_64/pmap.c +++ b/sys/platform/pc64/x86_64/pmap.c @@ -203,9 +203,6 @@ struct msgbuf *msgbufp=0; static pt_entry_t *pt_crashdumpmap; static caddr_t crashdumpmap; -extern pt_entry_t *SMPpt; -extern uint64_t SMPptpa; - #define DISABLE_PSE static pv_entry_t get_pv_entry (void); @@ -428,6 +425,8 @@ void create_pagetables(vm_paddr_t *firstaddr) { long i; /* must be 64 bits */ + long nkpt_base; + long nkpt_phys; /* * We are running (mostly) V=P at this point @@ -442,26 +441,35 @@ create_pagetables(vm_paddr_t *firstaddr) if (ndmpdp < 4) /* Minimum 4GB of dirmap */ ndmpdp = 4; - nkpt = (Maxmem * sizeof(struct vm_page) + NBPDR - 1) / NBPDR; - nkpt += (Maxmem * sizeof(struct pv_entry) + NBPDR - 1) / NBPDR; - nkpt += ((nkpt + nkpt + 1 + NKPML4E + NKPDPE + NDMPML4E + ndmpdp) + - 511) / 512; - nkpt += 128; + /* + * Starting at the beginning of kvm (not KERNBASE). + */ + nkpt_phys = (Maxmem * sizeof(struct vm_page) + NBPDR - 1) / NBPDR; + nkpt_phys += (Maxmem * sizeof(struct pv_entry) + NBPDR - 1) / NBPDR; + nkpt_phys += ((nkpt + nkpt + 1 + NKPML4E + NKPDPE + NDMPML4E + ndmpdp) + + 511) / 512; + nkpt_phys += 128; + + /* + * Starting at KERNBASE - map 2G worth of page table pages. + * KERNBASE is offset -2G from the end of kvm. + */ + nkpt_base = (NPDPEPG - KPDPI) * NPTEPG; /* typically 2 x 512 */ /* * Allocate pages */ - KPTbase = allocpages(firstaddr, nkpt); - KPTphys = allocpages(firstaddr, nkpt); + KPTbase = allocpages(firstaddr, nkpt_base); + KPTphys = allocpages(firstaddr, nkpt_phys); KPML4phys = allocpages(firstaddr, 1); KPDPphys = allocpages(firstaddr, NKPML4E); + KPDphys = allocpages(firstaddr, NKPDPE); /* * Calculate the page directory base for KERNBASE, * that is where we start populating the page table pages. * Basically this is the end - 2. */ - KPDphys = allocpages(firstaddr, NKPDPE); KPDbase = KPDphys + ((NKPDPE - (NPDPEPG - KPDPI)) << PAGE_SHIFT); DMPDPphys = allocpages(firstaddr, NDMPML4E); @@ -487,11 +495,11 @@ create_pagetables(vm_paddr_t *firstaddr) * and another block is placed at KERNBASE to map the kernel binary, * data, bss, and initial pre-allocations. */ - for (i = 0; i < nkpt; i++) { + for (i = 0; i < nkpt_base; i++) { ((pd_entry_t *)KPDbase)[i] = KPTbase + (i << PAGE_SHIFT); ((pd_entry_t *)KPDbase)[i] |= PG_RW | PG_V; } - for (i = 0; i < nkpt; i++) { + for (i = 0; i < nkpt_phys; i++) { ((pd_entry_t *)KPDphys)[i] = KPTphys + (i << PAGE_SHIFT); ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V; } @@ -710,14 +718,6 @@ pmap_bootstrap(vm_paddr_t *firstaddr) */ pg = MDGLOBALDATA_BASEALLOC_PAGES; gd = &CPU_prvspace[0].mdglobaldata; - gd->gd_CMAP1 = &SMPpt[pg + 0]; - gd->gd_CMAP2 = &SMPpt[pg + 1]; - gd->gd_CMAP3 = &SMPpt[pg + 2]; - gd->gd_PMAP1 = &SMPpt[pg + 3]; - gd->gd_CADDR1 = CPU_prvspace[0].CPAGE1; - gd->gd_CADDR2 = CPU_prvspace[0].CPAGE2; - gd->gd_CADDR3 = CPU_prvspace[0].CPAGE3; - gd->gd_PADDR1 = (pt_entry_t *)CPU_prvspace[0].PPAGE1; cpu_invltlb(); } @@ -1052,8 +1052,6 @@ pmap_map(vm_offset_t *virtp, vm_paddr_t start, vm_paddr_t end, int prot) va = va_start; while (start < end) { - if ((start / PAGE_SIZE & 15) == 0) - kprintf("%p %p\n", (void *)va, (void *)start); pmap_kenter_quick(va, start); va += PAGE_SIZE; start += PAGE_SIZE; @@ -1809,7 +1807,8 @@ pmap_release(struct pmap *pmap) vm_object_t object = pmap->pm_pteobj; struct rb_vm_page_scan_info info; - KASSERT(pmap->pm_active == 0, ("pmap still active! %08x", pmap->pm_active)); + KASSERT(pmap->pm_active == 0, + ("pmap still active! %016jx", (uintmax_t)pmap->pm_active)); #if defined(DIAGNOSTIC) if (object->ref_count != 1) panic("pmap_release: pteobj reference count != 1"); @@ -3867,7 +3866,7 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) if (curthread->td_lwp == lp) { pmap = vmspace_pmap(newvm); #if defined(SMP) - atomic_set_int(&pmap->pm_active, mycpu->gd_cpumask); + atomic_set_cpumask(&pmap->pm_active, mycpu->gd_cpumask); if (pmap->pm_active & CPUMASK_LOCK) pmap_interlock_wait(newvm); #else @@ -3881,9 +3880,9 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) load_cr3(curthread->td_pcb->pcb_cr3); pmap = vmspace_pmap(oldvm); #if defined(SMP) - atomic_clear_int(&pmap->pm_active, mycpu->gd_cpumask); + atomic_clear_cpumask(&pmap->pm_active, mycpu->gd_cpumask); #else - pmap->pm_active &= ~1; + pmap->pm_active &= ~(cpumask_t)1; #endif } } diff --git a/sys/platform/pc64/x86_64/pmap_inval.c b/sys/platform/pc64/x86_64/pmap_inval.c index 2aafd0bfd8..e03bfda3b9 100644 --- a/sys/platform/pc64/x86_64/pmap_inval.c +++ b/sys/platform/pc64/x86_64/pmap_inval.c @@ -104,7 +104,7 @@ pmap_inval_interlock(pmap_inval_info_t info, pmap_t pmap, vm_offset_t va) for (;;) { oactive = pmap->pm_active & ~CPUMASK_LOCK; nactive = oactive | CPUMASK_LOCK; - if (atomic_cmpset_int(&pmap->pm_active, oactive, nactive)) + if (atomic_cmpset_cpumask(&pmap->pm_active, oactive, nactive)) break; crit_enter(); lwkt_process_ipiq(); @@ -149,7 +149,7 @@ void pmap_inval_deinterlock(pmap_inval_info_t info, pmap_t pmap) { #ifdef SMP - atomic_clear_int(&pmap->pm_active, CPUMASK_LOCK); + atomic_clear_cpumask(&pmap->pm_active, CPUMASK_LOCK); #endif } diff --git a/sys/platform/pc64/x86_64/swtch.s b/sys/platform/pc64/x86_64/swtch.s index da40d8d66b..63bbc31a6b 100644 --- a/sys/platform/pc64/x86_64/swtch.s +++ b/sys/platform/pc64/x86_64/swtch.s @@ -150,8 +150,8 @@ ENTRY(cpu_heavy_switch) cmpq LWP_VMSPACE(%r13),%rcx /* same vmspace? */ je 2f 1: - movl PCPU(cpuid), %eax - MPLOCKED btrl %eax, VM_PMAP+PM_ACTIVE(%rcx) + movslq PCPU(cpuid), %rax + MPLOCKED btrq %rax, VM_PMAP+PM_ACTIVE(%rcx) 2: /* @@ -252,9 +252,9 @@ ENTRY(cpu_exit_switch) movq TD_LWP(%rbx),%rcx testq %rcx,%rcx jz 2f - movl PCPU(cpuid), %eax + movslq PCPU(cpuid), %rax movq LWP_VMSPACE(%rcx), %rcx /* RCX = vmspace */ - MPLOCKED btrl %eax, VM_PMAP+PM_ACTIVE(%rcx) + MPLOCKED btrq %rax, VM_PMAP+PM_ACTIVE(%rcx) 2: /* * Switch to the next thread. RET into the restore function, which @@ -318,11 +318,11 @@ ENTRY(cpu_heavy_restore) * wait for it to complete before we can continue. */ movq LWP_VMSPACE(%rcx), %rcx /* RCX = vmspace */ - movl PCPU(cpumask), %esi - MPLOCKED orl %esi, VM_PMAP+PM_ACTIVE(%rcx) + movslq PCPU(cpumask), %rsi + MPLOCKED orq %rsi, VM_PMAP+PM_ACTIVE(%rcx) #ifdef SMP - testl $CPUMASK_LOCK,VM_PMAP+PM_ACTIVE(%rcx) - jz 1f + btq $CPUMASK_BIT,VM_PMAP+PM_ACTIVE(%rcx) + jnc 1f pushq %rax movq %rcx,%rdi call pmap_interlock_wait /* pmap_interlock_wait(vm) */ diff --git a/sys/platform/vkernel/i386/cpu_regs.c b/sys/platform/vkernel/i386/cpu_regs.c index e800f8fc74..a849c18a29 100644 --- a/sys/platform/vkernel/i386/cpu_regs.c +++ b/sys/platform/vkernel/i386/cpu_regs.c @@ -782,7 +782,7 @@ handle_cpu_contention_mask(void) mask = cpu_contention_mask; cpu_ccfence(); - if (mask && bsfl(mask) != mycpu->gd_cpuid) + if (mask && BSFCPUMASK(mask) != mycpu->gd_cpuid) pthread_yield(); } diff --git a/sys/platform/vkernel/i386/db_interface.c b/sys/platform/vkernel/i386/db_interface.c index 94daaaeea6..627a8a2b11 100644 --- a/sys/platform/vkernel/i386/db_interface.c +++ b/sys/platform/vkernel/i386/db_interface.c @@ -161,13 +161,13 @@ kdb_trap(int type, int code, struct i386_saved_state *regs) db_global_jmpbuf_valid = FALSE; #ifdef SMP - db_printf("\nCPU%d restarting CPUs: 0x%08x\n", - mycpu->gd_cpuid, stopped_cpus); + db_printf("\nCPU%d restarting CPUs: 0x%016jx\n", + mycpu->gd_cpuid, (uintmax_t)stopped_cpus); /* Restart all the CPUs we previously stopped */ if (stopped_cpus != mycpu->gd_other_cpus) { - db_printf("whoa, other_cpus: 0x%08x, stopped_cpus: 0x%08x\n", - mycpu->gd_other_cpus, stopped_cpus); + db_printf("whoa, other_cpus: 0x%08x, stopped_cpus: 0x%016jx\n", + mycpu->gd_other_cpus, (uintmax_t)stopped_cpus); panic("stop_cpus() failed"); } restart_cpus(stopped_cpus); diff --git a/sys/platform/vkernel/i386/genassym.c b/sys/platform/vkernel/i386/genassym.c index 5d332fd143..87108ef71c 100644 --- a/sys/platform/vkernel/i386/genassym.c +++ b/sys/platform/vkernel/i386/genassym.c @@ -98,6 +98,7 @@ ASSYM(TD_SAVEFPU, offsetof(struct thread, td_mach) + offsetof(struct md_thread, ASSYM(TDPRI_INT_SUPPORT, TDPRI_INT_SUPPORT); #ifdef SMP ASSYM(CPUMASK_LOCK, CPUMASK_LOCK); +ASSYM(CPUMASK_BIT, CPUMASK_BIT); #endif ASSYM(V_TRAP, offsetof(struct vmmeter, v_trap)); @@ -203,7 +204,7 @@ ASSYM(GD_TSS_GDT, offsetof(struct mdglobaldata, gd_tss_gdt)); ASSYM(GD_NPXTHREAD, offsetof(struct mdglobaldata, gd_npxthread)); ASSYM(GD_FPU_LOCK, offsetof(struct mdglobaldata, gd_fpu_lock)); ASSYM(GD_SAVEFPU, offsetof(struct mdglobaldata, gd_savefpu)); -ASSYM(GD_OTHER_CPUS, offsetof(struct mdglobaldata, gd_other_cpus)); +ASSYM(GD_OTHER_CPUS, offsetof(struct mdglobaldata, mi.gd_other_cpus)); ASSYM(GD_SS_EFLAGS, offsetof(struct mdglobaldata, gd_ss_eflags)); ASSYM(GD_CMAP1, offsetof(struct mdglobaldata, gd_CMAP1)); diff --git a/sys/platform/vkernel/i386/mp.c b/sys/platform/vkernel/i386/mp.c index b0d195dce0..c01f31033d 100644 --- a/sys/platform/vkernel/i386/mp.c +++ b/sys/platform/vkernel/i386/mp.c @@ -61,7 +61,7 @@ extern pt_entry_t *KPTphys; -volatile u_int stopped_cpus; +volatile cpumask_t stopped_cpus; cpumask_t smp_active_mask = 1; /* which cpus are ready for IPIs etc? */ static int boot_address; static cpumask_t smp_startup_mask = 1; /* which cpus have been started */ @@ -71,7 +71,7 @@ static int mp_finish; /* function prototypes XXX these should go elsewhere */ void bootstrap_idle(void); void single_cpu_ipi(int, int, int); -void selected_cpu_ipi(u_int, int, int); +void selected_cpu_ipi(cpumask_t, int, int); #if 0 void ipi_handler(int); #endif @@ -99,14 +99,14 @@ ap_finish(void) cpumask_t ncpus_mask = 0; for (i = 1; i <= ncpus; i++) - ncpus_mask |= (1 << i); + ncpus_mask |= CPUMASK(i); mp_finish = 1; if (bootverbose) kprintf("Finish MP startup\n"); /* build our map of 'other' CPUs */ - mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); /* * Let the other cpu's finish initializing and build their map @@ -198,7 +198,7 @@ forward_fastint_remote(void *arg) void cpu_send_ipiq(int dcpu) { - if ((1 << dcpu) & smp_active_mask) + if (CPUMASK(dcpu) & smp_active_mask) if (pthread_kill(ap_tids[dcpu], SIGUSR1) != 0) panic("pthread_kill failed in cpu_send_ipiq"); #if 0 @@ -220,27 +220,27 @@ single_cpu_ipi(int cpu, int vector, int delivery_mode) } void -selected_cpu_ipi(u_int target, int vector, int delivery_mode) +selected_cpu_ipi(cpumask_t target, int vector, int delivery_mode) { crit_enter(); while (target) { - int n = bsfl(target); - target &= ~(1 << n); + int n = BSFCPUMASK(target); + target &= ~CPUMASK(n); single_cpu_ipi(n, vector, delivery_mode); } crit_exit(); } int -stop_cpus(u_int map) +stop_cpus(cpumask_t map) { map &= smp_active_mask; crit_enter(); while (map) { - int n = bsfl(map); - map &= ~(1 << n); - stopped_cpus |= 1 << n; + int n = BSFCPUMASK(map); + map &= ~CPUMASK(n); + stopped_cpus |= CPUMASK(n); if (pthread_kill(ap_tids[n], SIGXCPU) != 0) panic("stop_cpus: pthread_kill failed"); } @@ -253,15 +253,15 @@ stop_cpus(u_int map) } int -restart_cpus(u_int map) +restart_cpus(cpumask_t map) { map &= smp_active_mask; crit_enter(); while (map) { - int n = bsfl(map); - map &= ~(1 << n); - stopped_cpus &= ~(1 << n); + int n = BSFCPUMASK(map); + map &= ~CPUMASK(n); + stopped_cpus &= ~CPUMASK(n); if (pthread_kill(ap_tids[n], SIGXCPU) != 0) panic("restart_cpus: pthread_kill failed"); } @@ -285,7 +285,7 @@ ap_init(void) * interrupts physically disabled and remote cpus could deadlock * trying to send us an IPI. */ - smp_startup_mask |= 1 << mycpu->gd_cpuid; + smp_startup_mask |= CPUMASK(mycpu->gd_cpuid); cpu_mfence(); /* @@ -316,7 +316,7 @@ ap_init(void) cpu_invltlb(); /* Build our map of 'other' CPUs. */ - mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); kprintf("SMP: AP CPU #%d Launched!\n", mycpu->gd_cpuid); @@ -338,7 +338,7 @@ ap_init(void) * nothing we've done put it there. */ KKASSERT(curthread->td_mpcount == 1); - smp_active_mask |= 1 << mycpu->gd_cpuid; + smp_active_mask |= CPUMASK(mycpu->gd_cpuid); mdcpu->gd_fpending = 0; mdcpu->gd_ipending = 0; @@ -455,7 +455,7 @@ start_all_aps(u_int boot_addr) pthread_create(&ap_tids[x], NULL, start_ap, NULL); cpu_enable_intr(); - while((smp_startup_mask & (1 << x)) == 0) { + while((smp_startup_mask & CPUMASK(x)) == 0) { cpu_lfence(); /* XXX spin until the AP has started */ DELAY(1000); } diff --git a/sys/platform/vkernel/include/globaldata.h b/sys/platform/vkernel/include/globaldata.h index 336eac90c7..0a0b2a596b 100644 --- a/sys/platform/vkernel/include/globaldata.h +++ b/sys/platform/vkernel/include/globaldata.h @@ -83,10 +83,10 @@ struct mdglobaldata { int gd_spending; /* software interrupt pending */ int gd_sdelayed; /* delayed software ints */ int gd_currentldt; - int unused003; + int unused001; int unused002; - u_int unused001; - u_int gd_other_cpus; + u_int unused003; + u_int unused004; u_int gd_ss_eflags; vpte_t *gd_CMAP1; /* pointer to pte for CADDR1 */ vpte_t *gd_CMAP2; diff --git a/sys/platform/vkernel/include/pmap.h b/sys/platform/vkernel/include/pmap.h index d6d9aa3b4b..51f83403bd 100644 --- a/sys/platform/vkernel/include/pmap.h +++ b/sys/platform/vkernel/include/pmap.h @@ -134,7 +134,8 @@ struct pmap { int pm_generation; /* detect pvlist deletions */ }; -#define CPUMASK_LOCK (1 << SMP_MAXCPU) +#define CPUMASK_LOCK CPUMASK(SMP_MAXCPU) +#define CPUMASK_BIT SMP_MAXCPU /* 1 << SMP_MAXCPU */ #define pmap_resident_count(pmap) (pmap)->pm_stats.resident_count diff --git a/sys/platform/vkernel/include/smp.h b/sys/platform/vkernel/include/smp.h index 1bfc088634..cf342ed235 100644 --- a/sys/platform/vkernel/include/smp.h +++ b/sys/platform/vkernel/include/smp.h @@ -49,13 +49,13 @@ void bootMP (void); #endif /* global data in apic_vector.s */ -extern volatile u_int stopped_cpus; +extern volatile cpumask_t stopped_cpus; extern int optcpus; /* from main() */ #if 0 -extern volatile u_int started_cpus; +extern volatile cpumask_t started_cpus; -extern volatile u_int checkstate_probed_cpus; +extern volatile cpumask_t checkstate_probed_cpus; extern void (*cpustop_restartfunc) (void); /* functions in apic_ipl.s */ @@ -120,9 +120,9 @@ void assign_apic_irq (int apic, int intpin, int irq); void revoke_apic_irq (int irq); void init_secondary (void); #endif -int stop_cpus (u_int); +int stop_cpus (cpumask_t); void ap_init (void); -int restart_cpus (u_int); +int restart_cpus (cpumask_t); #if 0 void forward_signal (struct proc *); @@ -140,7 +140,7 @@ void apic_dump (char*); void apic_initialize (void); void imen_dump (void); int apic_ipi (int, int, int); -void selected_apic_ipi (u_int, int, int); +void selected_apic_ipi (cpumask_t, int, int); void single_apic_ipi(int cpu, int vector, int delivery_mode); int single_apic_ipi_passive(int cpu, int vector, int delivery_mode); int io_apic_setup (int); diff --git a/sys/platform/vkernel/platform/pmap.c b/sys/platform/vkernel/platform/pmap.c index a61cf7c3df..1447805cc4 100644 --- a/sys/platform/vkernel/platform/pmap.c +++ b/sys/platform/vkernel/platform/pmap.c @@ -494,14 +494,16 @@ get_ptbase(struct pmap *pmap, vm_offset_t va) if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) { *gd->gd_PT1pde = pmap->pm_pdirpte; madvise(gd->gd_PT1map, SEG_SIZE, MADV_INVAL); - atomic_set_int(&pmap->pm_cpucachemask, gd->mi.gd_cpumask); + atomic_set_cpumask(&pmap->pm_cpucachemask, + gd->mi.gd_cpumask); } return(gd->gd_PT1map + (va >> PAGE_SHIFT)); } else if (pmap->pm_pdir == gd->gd_PT2pdir) { if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) { *gd->gd_PT2pde = pmap->pm_pdirpte; madvise(gd->gd_PT2map, SEG_SIZE, MADV_INVAL); - atomic_set_int(&pmap->pm_cpucachemask, gd->mi.gd_cpumask); + atomic_set_cpumask(&pmap->pm_cpucachemask, + gd->mi.gd_cpumask); } return(gd->gd_PT2map + (va >> PAGE_SHIFT)); } @@ -520,15 +522,15 @@ get_ptbase(struct pmap *pmap, vm_offset_t va) gd->gd_PT1pdir = pmap->pm_pdir; *gd->gd_PT1pde = pmap->pm_pdirpte; madvise(gd->gd_PT1map, SEG_SIZE, MADV_INVAL); - atomic_set_int(&pmap->pm_cpucachemask, - gd->mi.gd_cpumask); + atomic_set_cpumask(&pmap->pm_cpucachemask, + gd->mi.gd_cpumask); return(gd->gd_PT1map + (va >> PAGE_SHIFT)); } else { gd->gd_PT2pdir = pmap->pm_pdir; *gd->gd_PT2pde = pmap->pm_pdirpte; madvise(gd->gd_PT2map, SEG_SIZE, MADV_INVAL); - atomic_set_int(&pmap->pm_cpucachemask, - gd->mi.gd_cpumask); + atomic_set_cpumask(&pmap->pm_cpucachemask, + gd->mi.gd_cpumask); return(gd->gd_PT2map + (va >> PAGE_SHIFT)); } } @@ -542,15 +544,15 @@ get_ptbase(struct pmap *pmap, vm_offset_t va) if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) { *gd->gd_PT3pde = pmap->pm_pdirpte; madvise(gd->gd_PT3map, SEG_SIZE, MADV_INVAL); - atomic_set_int(&pmap->pm_cpucachemask, - gd->mi.gd_cpumask); + atomic_set_cpumask(&pmap->pm_cpucachemask, + gd->mi.gd_cpumask); } } else { gd->gd_PT3pdir = pmap->pm_pdir; *gd->gd_PT3pde = pmap->pm_pdirpte; madvise(gd->gd_PT3map, SEG_SIZE, MADV_INVAL); - atomic_set_int(&pmap->pm_cpucachemask, - gd->mi.gd_cpumask); + atomic_set_cpumask(&pmap->pm_cpucachemask, + gd->mi.gd_cpumask); } return(gd->gd_PT3map + (va >> PAGE_SHIFT)); } @@ -567,7 +569,8 @@ get_ptbase1(struct pmap *pmap, vm_offset_t va) if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) { *gd->gd_PT1pde = pmap->pm_pdirpte; madvise(gd->gd_PT1map, SEG_SIZE, MADV_INVAL); - atomic_set_int(&pmap->pm_cpucachemask, gd->mi.gd_cpumask); + atomic_set_cpumask(&pmap->pm_cpucachemask, + gd->mi.gd_cpumask); } return(gd->gd_PT1map + (va >> PAGE_SHIFT)); } @@ -591,7 +594,8 @@ get_ptbase2(struct pmap *pmap, vm_offset_t va) if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) { *gd->gd_PT2pde = pmap->pm_pdirpte; madvise(gd->gd_PT2map, SEG_SIZE, MADV_INVAL); - atomic_set_int(&pmap->pm_cpucachemask, gd->mi.gd_cpumask); + atomic_set_cpumask(&pmap->pm_cpucachemask, + gd->mi.gd_cpumask); } return(gd->gd_PT2map + (va >> PAGE_SHIFT)); } @@ -3026,7 +3030,7 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) if (curthread->td_lwp == lp) { pmap = vmspace_pmap(newvm); #if defined(SMP) - atomic_set_int(&pmap->pm_active, mycpu->gd_cpumask); + atomic_set_cpumask(&pmap->pm_active, mycpu->gd_cpumask); #else pmap->pm_active |= 1; #endif @@ -3035,9 +3039,9 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) #endif pmap = vmspace_pmap(oldvm); #if defined(SMP) - atomic_clear_int(&pmap->pm_active, mycpu->gd_cpumask); + atomic_clear_cpumask(&pmap->pm_active, mycpu->gd_cpumask); #else - pmap->pm_active &= ~1; + pmap->pm_active &= ~(cpumask_t)1; #endif } } diff --git a/sys/platform/vkernel64/include/globaldata.h b/sys/platform/vkernel64/include/globaldata.h index 770eed2b4b..a4643d657b 100644 --- a/sys/platform/vkernel64/include/globaldata.h +++ b/sys/platform/vkernel64/include/globaldata.h @@ -83,10 +83,10 @@ struct mdglobaldata { int gd_spending; /* software interrupt pending */ int gd_sdelayed; /* delayed software ints */ int gd_currentldt; - int unused003; + int unused001; int unused002; - u_int unused001; - u_int gd_other_cpus; + u_int unused003; + cpumask_t unused004; u_int gd_ss_eflags; }; diff --git a/sys/platform/vkernel64/include/pmap.h b/sys/platform/vkernel64/include/pmap.h index 244e43c3c4..81fff9ad4f 100644 --- a/sys/platform/vkernel64/include/pmap.h +++ b/sys/platform/vkernel64/include/pmap.h @@ -161,6 +161,9 @@ struct pmap { #define pmap_resident_count(pmap) (pmap)->pm_stats.resident_count +#define CPUMASK_LOCK CPUMASK(SMP_MAXCPU) +#define CPUMASK_BIT SMP_MAXCPU /* 1 << SMP_MAXCPU */ + typedef struct pmap *pmap_t; #ifdef _KERNEL diff --git a/sys/platform/vkernel64/include/smp.h b/sys/platform/vkernel64/include/smp.h index b956d28624..c52bb025aa 100644 --- a/sys/platform/vkernel64/include/smp.h +++ b/sys/platform/vkernel64/include/smp.h @@ -49,13 +49,13 @@ void bootMP (void); #endif /* global data in apic_vector.s */ -extern volatile u_int stopped_cpus; +extern volatile cpumask_t stopped_cpus; extern int optcpus; /* from main() */ #if 0 -extern volatile u_int started_cpus; +extern volatile cpumask_t started_cpus; -extern volatile u_int checkstate_probed_cpus; +extern volatile cpumask_t checkstate_probed_cpus; extern void (*cpustop_restartfunc) (void); /* functions in apic_ipl.s */ @@ -120,9 +120,9 @@ void assign_apic_irq (int apic, int intpin, int irq); void revoke_apic_irq (int irq); void init_secondary (void); #endif -int stop_cpus (u_int); +int stop_cpus (cpumask_t); void ap_init (void); -int restart_cpus (u_int); +int restart_cpus (cpumask_t); #if 0 void forward_signal (struct proc *); @@ -140,7 +140,7 @@ void apic_dump (char*); void apic_initialize (void); void imen_dump (void); int apic_ipi (int, int, int); -void selected_apic_ipi (u_int, int, int); +void selected_apic_ipi (cpumask_t, int, int); void single_apic_ipi(int cpu, int vector, int delivery_mode); int single_apic_ipi_passive(int cpu, int vector, int delivery_mode); int io_apic_setup (int); diff --git a/sys/platform/vkernel64/platform/pmap.c b/sys/platform/vkernel64/platform/pmap.c index a1e32ea355..44ba742641 100644 --- a/sys/platform/vkernel64/platform/pmap.c +++ b/sys/platform/vkernel64/platform/pmap.c @@ -3246,7 +3246,7 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) if (curthread->td_lwp == lp) { pmap = vmspace_pmap(newvm); #if defined(SMP) - atomic_set_int(&pmap->pm_active, 1 << mycpu->gd_cpuid); + atomic_set_cpumask(&pmap->pm_active, CPUMASK(mycpu->gd_cpuid)); #else pmap->pm_active |= 1; #endif @@ -3255,10 +3255,10 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) #endif pmap = vmspace_pmap(oldvm); #if defined(SMP) - atomic_clear_int(&pmap->pm_active, - 1 << mycpu->gd_cpuid); + atomic_clear_cpumask(&pmap->pm_active, + CPUMASK(mycpu->gd_cpuid)); #else - pmap->pm_active &= ~1; + pmap->pm_active &= ~(cpumask_t)1; #endif } } diff --git a/sys/platform/vkernel64/x86_64/cpu_regs.c b/sys/platform/vkernel64/x86_64/cpu_regs.c index 051f1f8241..d46f243667 100644 --- a/sys/platform/vkernel64/x86_64/cpu_regs.c +++ b/sys/platform/vkernel64/x86_64/cpu_regs.c @@ -777,7 +777,7 @@ handle_cpu_contention_mask(void) mask = cpu_contention_mask; cpu_ccfence(); - if (mask && bsfl(mask) != mycpu->gd_cpuid) + if (mask && BSFCPUMASK(mask) != mycpu->gd_cpuid) pthread_yield(); } diff --git a/sys/platform/vkernel64/x86_64/db_interface.c b/sys/platform/vkernel64/x86_64/db_interface.c index cea7d81228..f5ef532b56 100644 --- a/sys/platform/vkernel64/x86_64/db_interface.c +++ b/sys/platform/vkernel64/x86_64/db_interface.c @@ -183,13 +183,13 @@ kdb_trap(int type, int code, struct x86_64_saved_state *regs) db_global_jmpbuf_valid = FALSE; #ifdef SMP - db_printf("\nCPU%d restarting CPUs: 0x%08x\n", - mycpu->gd_cpuid, stopped_cpus); + db_printf("\nCPU%d restarting CPUs: 0x%016jx\n", + mycpu->gd_cpuid, (uintmax_t)stopped_cpus); /* Restart all the CPUs we previously stopped */ if (stopped_cpus != mycpu->gd_other_cpus) { - db_printf("whoa, other_cpus: 0x%08x, stopped_cpus: 0x%08x\n", - mycpu->gd_other_cpus, stopped_cpus); + db_printf("whoa, other_cpus: 0x%08x, stopped_cpus: 0x%016jx\n", + mycpu->gd_other_cpus, (uintmax_t)stopped_cpus); panic("stop_cpus() failed"); } restart_cpus(stopped_cpus); diff --git a/sys/platform/vkernel64/x86_64/mp.c b/sys/platform/vkernel64/x86_64/mp.c index 1c5d0f5d99..ce726f9ba1 100644 --- a/sys/platform/vkernel64/x86_64/mp.c +++ b/sys/platform/vkernel64/x86_64/mp.c @@ -61,7 +61,7 @@ extern pt_entry_t *KPTphys; -volatile u_int stopped_cpus; +volatile cpumask_t stopped_cpus; cpumask_t smp_active_mask = 1; /* which cpus are ready for IPIs etc? */ static int boot_address; static cpumask_t smp_startup_mask = 1; /* which cpus have been started */ @@ -71,7 +71,7 @@ static int mp_finish; /* function prototypes XXX these should go elsewhere */ void bootstrap_idle(void); void single_cpu_ipi(int, int, int); -void selected_cpu_ipi(u_int, int, int); +void selected_cpu_ipi(cpumask_t, int, int); #if 0 void ipi_handler(int); #endif @@ -99,14 +99,14 @@ ap_finish(void) cpumask_t ncpus_mask = 0; for (i = 1; i <= ncpus; i++) - ncpus_mask |= (1 << i); + ncpus_mask |= CPUMASK(i); mp_finish = 1; if (bootverbose) kprintf("Finish MP startup\n"); /* build our map of 'other' CPUs */ - mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); /* * Let the other cpu's finish initializing and build their map @@ -198,7 +198,7 @@ forward_fastint_remote(void *arg) void cpu_send_ipiq(int dcpu) { - if ((1 << dcpu) & smp_active_mask) + if (CPUMASK(dcpu) & smp_active_mask) if (pthread_kill(ap_tids[dcpu], SIGUSR1) != 0) panic("pthread_kill failed in cpu_send_ipiq"); #if 0 @@ -220,27 +220,27 @@ single_cpu_ipi(int cpu, int vector, int delivery_mode) } void -selected_cpu_ipi(u_int target, int vector, int delivery_mode) +selected_cpu_ipi(cpumask_t target, int vector, int delivery_mode) { crit_enter(); while (target) { - int n = bsfl(target); - target &= ~(1 << n); + int n = BSFCPUMASK(target); + target &= ~CPUMASK(n); single_cpu_ipi(n, vector, delivery_mode); } crit_exit(); } int -stop_cpus(u_int map) +stop_cpus(cpumask_t map) { map &= smp_active_mask; crit_enter(); while (map) { - int n = bsfl(map); - map &= ~(1 << n); - stopped_cpus |= 1 << n; + int n = BSFCPUMASK(map); + map &= ~CPUMASK(n); + stopped_cpus |= CPUMASK(n); if (pthread_kill(ap_tids[n], SIGXCPU) != 0) panic("stop_cpus: pthread_kill failed"); } @@ -253,15 +253,15 @@ stop_cpus(u_int map) } int -restart_cpus(u_int map) +restart_cpus(cpumask_t map) { map &= smp_active_mask; crit_enter(); while (map) { - int n = bsfl(map); - map &= ~(1 << n); - stopped_cpus &= ~(1 << n); + int n = BSFCPUMASK(map); + map &= ~CPUMASK(n); + stopped_cpus &= ~CPUMASK(n); if (pthread_kill(ap_tids[n], SIGXCPU) != 0) panic("restart_cpus: pthread_kill failed"); } @@ -285,7 +285,7 @@ ap_init(void) * interrupts physically disabled and remote cpus could deadlock * trying to send us an IPI. */ - smp_startup_mask |= 1 << mycpu->gd_cpuid; + smp_startup_mask |= CPUMASK(mycpu->gd_cpuid); cpu_mfence(); /* @@ -316,7 +316,7 @@ ap_init(void) cpu_invltlb(); /* Build our map of 'other' CPUs. */ - mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); kprintf("SMP: AP CPU #%d Launched!\n", mycpu->gd_cpuid); @@ -338,7 +338,7 @@ ap_init(void) * nothing we've done put it there. */ KKASSERT(curthread->td_mpcount == 1); - smp_active_mask |= 1 << mycpu->gd_cpuid; + smp_active_mask |= CPUMASK(mycpu->gd_cpuid); mdcpu->gd_fpending = 0; mdcpu->gd_ipending = 0; @@ -456,7 +456,7 @@ start_all_aps(u_int boot_addr) pthread_create(&ap_tids[x], NULL, start_ap, NULL); cpu_enable_intr(); - while((smp_startup_mask & (1 << x)) == 0) { + while((smp_startup_mask & CPUMASK(x)) == 0) { cpu_lfence(); /* XXX spin until the AP has started */ DELAY(1000); } diff --git a/sys/platform/vkernel64/x86_64/swtch.s b/sys/platform/vkernel64/x86_64/swtch.s index 65929f39ab..5b5d2f163a 100644 --- a/sys/platform/vkernel64/x86_64/swtch.s +++ b/sys/platform/vkernel64/x86_64/swtch.s @@ -133,9 +133,9 @@ ENTRY(cpu_heavy_switch) movq %rcx,%rbx /* RBX = curthread */ movq TD_LWP(%rcx),%rcx - movl PCPU(cpuid), %eax + movslq PCPU(cpuid), %rax movq LWP_VMSPACE(%rcx), %rcx /* RCX = vmspace */ - MPLOCKED btrl %eax, VM_PMAP+PM_ACTIVE(%rcx) + MPLOCKED btrq %rax, VM_PMAP+PM_ACTIVE(%rcx) /* * Push the LWKT switch restore function, which resumes a heavy @@ -238,9 +238,9 @@ ENTRY(cpu_exit_switch) movq TD_LWP(%rbx),%rcx testq %rcx,%rcx jz 2f - movl PCPU(cpuid), %eax + movslq PCPU(cpuid), %rax movq LWP_VMSPACE(%rcx), %rcx /* RCX = vmspace */ - MPLOCKED btrl %eax, VM_PMAP+PM_ACTIVE(%rcx) + MPLOCKED btrq %rax, VM_PMAP+PM_ACTIVE(%rcx) 2: /* * Switch to the next thread. RET into the restore function, which @@ -297,8 +297,8 @@ ENTRY(cpu_heavy_restore) * pmap (remember, we do not hold the MP lock in the switch code). */ movq LWP_VMSPACE(%rcx), %rcx /* RCX = vmspace */ - movl PCPU(cpuid), %esi - MPLOCKED btsl %esi, VM_PMAP+PM_ACTIVE(%rcx) + movslq PCPU(cpuid), %rsi + MPLOCKED btsq %rsi, VM_PMAP+PM_ACTIVE(%rcx) /* * Restore the MMU address space. If it is the same as the last diff --git a/sys/sys/globaldata.h b/sys/sys/globaldata.h index d78d5d0169..ecc1a7aedb 100644 --- a/sys/sys/globaldata.h +++ b/sys/sys/globaldata.h @@ -131,7 +131,7 @@ struct globaldata { lwkt_queue gd_tdallq; /* all threads */ lwkt_queue gd_tdrunq; /* runnable threads */ __uint32_t gd_cpuid; - cpumask_t gd_cpumask; /* mask = 1<gd_cpumask); + atomic_set_cpumask(&cpu_contention_mask, gd->gd_cpumask); } /* @@ -176,7 +176,7 @@ static __inline void clr_cpu_contention_mask(globaldata_t gd) { - atomic_clear_int(&cpu_contention_mask, gd->gd_cpumask); + atomic_clear_cpumask(&cpu_contention_mask, gd->gd_cpumask); } static __inline diff --git a/sys/sys/thread2.h b/sys/sys/thread2.h index b8673862a5..4cd7b12fc1 100644 --- a/sys/sys/thread2.h +++ b/sys/sys/thread2.h @@ -262,13 +262,13 @@ lwkt_send_ipiq2(globaldata_t target, ipifunc2_t func, void *arg1, int arg2) } static __inline int -lwkt_send_ipiq_mask(u_int32_t mask, ipifunc1_t func, void *arg) +lwkt_send_ipiq_mask(cpumask_t mask, ipifunc1_t func, void *arg) { return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg, 0)); } static __inline int -lwkt_send_ipiq2_mask(u_int32_t mask, ipifunc2_t func, void *arg1, int arg2) +lwkt_send_ipiq2_mask(cpumask_t mask, ipifunc2_t func, void *arg1, int arg2) { return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg1, arg2)); } diff --git a/sys/vm/vm_extern.h b/sys/vm/vm_extern.h index 2ef42ceb1e..f1c3360c70 100644 --- a/sys/vm/vm_extern.h +++ b/sys/vm/vm_extern.h @@ -84,7 +84,7 @@ vm_offset_t kmem_alloc_pageable (vm_map_t, vm_size_t); vm_offset_t kmem_alloc_wait (vm_map_t, vm_size_t); void kmem_free (vm_map_t, vm_offset_t, vm_size_t); void kmem_free_wakeup (vm_map_t, vm_offset_t, vm_size_t); -void kmem_init (vm_offset_t, vm_offset_t); +void kmem_init (void); void kmem_suballoc (vm_map_t, vm_map_t, vm_offset_t *, vm_offset_t *, vm_size_t); void munmapfd (struct proc *, int); int swaponvp (struct thread *, struct vnode *, u_quad_t); diff --git a/sys/vm/vm_init.c b/sys/vm/vm_init.c index 3d72b3d545..7c3d5e8803 100644 --- a/sys/vm/vm_init.c +++ b/sys/vm/vm_init.c @@ -102,14 +102,14 @@ vm_mem_init(void *dummy) * Initializes resident memory structures. From here on, all physical * memory is accounted for, and we use only virtual addresses. */ - vm_set_page_size(); - virtual_start = vm_page_startup(virtual_start); + vm_page_startup(); + /* * Initialize other VM packages */ vm_object_init(); vm_map_startup(); - kmem_init(virtual_start, virtual_end); + kmem_init(); pmap_init(); } diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c index 90e4ef4aee..a464117c5b 100644 --- a/sys/vm/vm_kern.c +++ b/sys/vm/vm_kern.c @@ -362,22 +362,18 @@ kmem_free_wakeup(vm_map_t map, vm_offset_t addr, vm_size_t size) } /* - * Create the kernel_map and insert mappings to cover areas already - * allocated or reserved thus far. That is, the area (KvaStart,start) - * and (end,KvaEnd) must be marked as allocated. + * Create the kernel_ma for (KvaStart,KvaEnd) and insert mappings to + * cover areas already allocated or reserved thus far. * - * virtual2_start/end is a cutout Between KvaStart and start, - * for x86_64 due to the location of KERNBASE (at -2G). - * - * We could use a min_offset of 0 instead of KvaStart, but since the - * min_offset is not used for any calculations other then a bounds check - * it does not effect readability. KvaStart is more appropriate. + * The areas (virtual_start, virtual_end) and (virtual2_start, virtual2_end) + * are available so the cutouts are the areas around these ranges between + * KvaStart and KvaEnd. * * Depend on the zalloc bootstrap cache to get our vm_map_entry_t. * Called from the low level boot code only. */ void -kmem_init(vm_offset_t start, vm_offset_t end) +kmem_init(void) { vm_offset_t addr; vm_map_t m; @@ -399,14 +395,14 @@ kmem_init(vm_offset_t start, vm_offset_t end) } addr = virtual2_end; } - if (addr < start) { + if (addr < virtual_start) { vm_map_insert(m, &count, NULL, (vm_offset_t) 0, - addr, start, + addr, virtual_start, VM_MAPTYPE_NORMAL, VM_PROT_ALL, VM_PROT_ALL, 0); } - addr = end; + addr = virtual_end; if (addr < KvaEnd) { vm_map_insert(m, &count, NULL, (vm_offset_t) 0, addr, KvaEnd, diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 6bf0aafe3c..37af8cc72e 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -201,18 +201,20 @@ vm_add_new_page(vm_paddr_t pa) * * Initializes the resident memory module. * - * Allocates memory for the page cells, and for the object/offset-to-page - * hash table headers. Each page cell is initialized and placed on the - * free list. - * - * starta/enda represents the range of physical memory addresses available - * for use (skipping memory already used by the kernel), subject to - * phys_avail[]. Note that phys_avail[] has already mapped out memory - * already in use by the kernel. + * Preallocates memory for critical VM structures and arrays prior to + * kernel_map becoming available. + * + * Memory is allocated from (virtual2_start, virtual2_end) if available, + * otherwise memory is allocated from (virtual_start, virtual_end). + * + * On x86-64 (virtual_start, virtual_end) is only 2GB and may not be + * large enough to hold vm_page_array & other structures for machines with + * large amounts of ram, so we want to use virtual2* when available. */ -vm_offset_t -vm_page_startup(vm_offset_t vaddr) +void +vm_page_startup(void) { + vm_offset_t vaddr = virtual2_start ? virtual2_start : virtual_start; vm_offset_t mapped; vm_size_t npages; vm_paddr_t page_range; @@ -331,7 +333,10 @@ vm_page_startup(vm_offset_t vaddr) pa += PAGE_SIZE; } } - return (vaddr); + if (virtual2_start) + virtual2_start = vaddr; + else + virtual_start = vaddr; } /* diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index 1f1672dcb8..7a237f7a22 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -496,7 +496,7 @@ void vm_page_insert (vm_page_t, struct vm_object *, vm_pindex_t); vm_page_t vm_page_lookup (struct vm_object *, vm_pindex_t); void vm_page_remove (vm_page_t); void vm_page_rename (vm_page_t, struct vm_object *, vm_pindex_t); -vm_offset_t vm_page_startup (vm_offset_t); +void vm_page_startup (void); vm_page_t vm_add_new_page (vm_paddr_t pa); void vm_page_unmanage (vm_page_t); void vm_page_unwire (vm_page_t, int); -- 2.41.0