* Increase SMP_MAXCPU to 63 for 64-bit builds.
* cpumask_t is 64 bits on 64-bit builds now. It remains 32 bits on 32-bit
builds.
* Add #define's for atomic_set_cpumask(), atomic_clear_cpumask, and
atomic_cmpset_cpumask(). Replace all use cases on cpu masks with
these functions.
* Add CPUMASK(), BSRCPUMASK(), and BSFCPUMASK() macros. Replace all
use cases on cpu masks with these functions.
In particular note that (1 << cpu) just doesn't work with a 64-bit
cpumask.
Numerous bits of assembly also had to be adjusted to use e.g. btq instead
of btl, etc.
* Change __uint32_t declarations that were meant to be cpu masks to use
cpumask_t (most already have).
Also change other bits of code which work on cpu masks to be more agnostic.
For example, poll_cpumask0 and lwp_cpumask.
* 64-bit atomic ops cannot use "iq", they must use "r", because most x86-64
do NOT have 64-bit immediate value support.
* Rearrange initial kernel memory allocations to start from KvaStart and
not KERNBASE, because only 2GB of KVM is available after KERNBASE.
Certain VM allocations with > 32G of ram can exceed 2GB. For example,
vm_page_array[]. 2GB was not enough.
* Remove numerous mdglobaldata fields that are not used.
* Align CPU_prvspace[] for now. Eventually it will be moved into a
mapped area. Reserve sufficient space at MPPTDI now, but it is
still unused.
* When pre-allocating kernel page table PD entries calculate the number
of page table pages at KvaStart and at KERNBASE separately, since
the KVA space starting at KERNBASE caps out at 2GB.
* Change kmem_init() and vm_page_startup() to not take memory range
arguments. Instead the globals (virtual_start and virtual_end) are
manipualted directly.
#define atomic_cmpset_acq_long atomic_cmpset_long
#define atomic_cmpset_rel_long atomic_cmpset_long
+/* cpumask_t is 32-bits on i386 */
+#define atomic_set_cpumask atomic_set_int
+#define atomic_clear_cpumask atomic_clear_int
+#define atomic_cmpset_cpumask atomic_cmpset_int
+
/* Operations on 8-bit bytes. */
#define atomic_set_8 atomic_set_char
#define atomic_set_acq_8 atomic_set_acq_char
typedef __uint32_t pt_entry_t;
typedef __uint32_t cpumask_t; /* mask representing a set of cpus */
+#if defined(_KERNEL) || defined(_KERNEL_STRUCTURES)
#define CPUMASK(cpu) (1U << (cpu))
#define BSRCPUMASK(mask) bsrl(mask)
+#define BSFCPUMASK(mask) bsfl(mask)
+#endif
#define PDESIZE sizeof(pd_entry_t) /* for assembly files */
#define PTESIZE sizeof(pt_entry_t) /* for assembly files */
* This allows kernel modules to be portable between UP and SMP systems.
*/
#if defined(KLD_MODULE)
-#define ATOMIC_ASM(NAME, TYPE, OP, V) \
+#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \
extern void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \
extern void atomic_##NAME##_##TYPE##_nonlocked(volatile u_##TYPE *p, u_##TYPE v);
#else /* !KLD_MODULE */
* iq - integer constant or %ax/%bx/%cx/%dx (ir = int constant or any reg)
* (Note: byte instructions only work on %ax,%bx,%cx, or %dx). iq
* is good enough for our needs so don't get fancy.
+ * r - any register.
+ *
+ * NOTE: 64-bit immediate values are not supported for most x86-64
+ * instructions so we have to use "r".
*/
/* egcs 1.1.2+ version */
-#define ATOMIC_ASM(NAME, TYPE, OP, V) \
+#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \
static __inline void \
atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
{ \
__asm __volatile(MPLOCKED OP \
: "+m" (*p) \
- : "iq" (V)); \
+ : CONS (V)); \
} \
static __inline void \
atomic_##NAME##_##TYPE##_nonlocked(volatile u_##TYPE *p, u_##TYPE v)\
{ \
__asm __volatile(OP \
: "+m" (*p) \
- : "iq" (V)); \
+ : CONS (V)); \
}
#endif /* KLD_MODULE */
/* egcs 1.1.2+ version */
-ATOMIC_ASM(set, char, "orb %b1,%0", v)
-ATOMIC_ASM(clear, char, "andb %b1,%0", ~v)
-ATOMIC_ASM(add, char, "addb %b1,%0", v)
-ATOMIC_ASM(subtract, char, "subb %b1,%0", v)
-
-ATOMIC_ASM(set, short, "orw %w1,%0", v)
-ATOMIC_ASM(clear, short, "andw %w1,%0", ~v)
-ATOMIC_ASM(add, short, "addw %w1,%0", v)
-ATOMIC_ASM(subtract, short, "subw %w1,%0", v)
-
-ATOMIC_ASM(set, int, "orl %1,%0", v)
-ATOMIC_ASM(clear, int, "andl %1,%0", ~v)
-ATOMIC_ASM(add, int, "addl %1,%0", v)
-ATOMIC_ASM(subtract, int, "subl %1,%0", v)
-
-ATOMIC_ASM(set, long, "orq %1,%0", v)
-ATOMIC_ASM(clear, long, "andq %1,%0", ~v)
-ATOMIC_ASM(add, long, "addq %1,%0", v)
-ATOMIC_ASM(subtract, long, "subq %1,%0", v)
+ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v)
+ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v)
+ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v)
+ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v)
+
+ATOMIC_ASM(set, short, "orw %w1,%0", "iq", v)
+ATOMIC_ASM(clear, short, "andw %w1,%0", "iq", ~v)
+ATOMIC_ASM(add, short, "addw %w1,%0", "iq", v)
+ATOMIC_ASM(subtract, short, "subw %w1,%0", "iq", v)
+
+ATOMIC_ASM(set, int, "orl %1,%0", "iq", v)
+ATOMIC_ASM(clear, int, "andl %1,%0", "iq", ~v)
+ATOMIC_ASM(add, int, "addl %1,%0", "iq", v)
+ATOMIC_ASM(subtract, int, "subl %1,%0", "iq", v)
+
+ATOMIC_ASM(set, long, "orq %1,%0", "r", v)
+ATOMIC_ASM(clear, long, "andq %1,%0", "r", ~v)
+ATOMIC_ASM(add, long, "addq %1,%0", "r", v)
+ATOMIC_ASM(subtract, long, "subq %1,%0", "r", v)
#if defined(KLD_MODULE)
#define atomic_cmpset_acq_long atomic_cmpset_long
#define atomic_cmpset_rel_long atomic_cmpset_long
+/* cpumask_t is 64-bits on x86-64 */
+#define atomic_set_cpumask atomic_set_long
+#define atomic_clear_cpumask atomic_clear_long
+#define atomic_cmpset_cpumask atomic_cmpset_long
+
/* Operations on 8-bit bytes. */
#define atomic_set_8 atomic_set_char
#define atomic_set_acq_8 atomic_set_acq_char
* Use SMP_MAXCPU instead of MAXCPU for structures that are intended to
* remain compatible between UP and SMP builds.
*/
-#define SMP_MAXCPU 31
+#define SMP_MAXCPU 63
#ifdef SMP
#define MAXCPU SMP_MAXCPU
#else
typedef __uint64_t pd_entry_t;
typedef __uint64_t pt_entry_t;
#endif
-typedef __uint32_t cpumask_t; /* mask representing a set of cpus */
+typedef __uint64_t cpumask_t; /* mask representing a set of cpus */
-#define CPUMASK(cpu) ((__uint32_t)1 << (cpu))
+#if defined(_KERNEL) || defined(_KERNEL_STRUCTURES)
+#define CPUMASK(cpu) ((cpumask_t)1 << (cpu))
#define BSRCPUMASK(mask) bsrq(mask)
+#define BSFCPUMASK(mask) bsfq(mask)
+#endif
#define PML4SIZE sizeof(pml4_entry_t) /* for assembly files */
#define PDPSIZE sizeof(pdp_entry_t) /* for assembly files */
KASSERT(acpi_id != NULL, ("Null acpi_id"));
KASSERT(cpu_id != NULL, ("Null cpu_id"));
for (i = 0; i < ncpus; i++) {
- if ((smp_active_mask & (1 << i)) == 0)
+ if ((smp_active_mask & CPUMASK(i)) == 0)
continue;
md = (struct mdglobaldata *)globaldata_find(i);
KASSERT(md != NULL, ("no pcpu data for %d", i));
lwp0.lwp_thread = &thread0;
lwp0.lwp_proc = &proc0;
proc0.p_usched = usched_init();
- lwp0.lwp_cpumask = 0xFFFFFFFF;
+ lwp0.lwp_cpumask = (cpumask_t)-1;
varsymset_init(&proc0.p_varsymset, NULL);
thread0.td_flags |= TDF_RUNNING;
thread0.td_proc = &proc0;
TAILQ_INIT(&gd->gd_systimerq);
gd->gd_sysid_alloc = cpuid; /* prime low bits for cpu lookup */
gd->gd_cpuid = cpuid;
- gd->gd_cpumask = (cpumask_t)1 << cpuid;
+ gd->gd_cpumask = CPUMASK(cpuid);
lwkt_gdinit(gd);
vm_map_entry_reserve_cpu_init(gd);
sleep_gdinit(gd);
file, line)
int mp_lock;
-int cpu_contention_mask;
+cpumask_t cpu_contention_mask;
const char *mp_lock_holder_file; /* debugging */
int mp_lock_holder_line; /* debugging */
gd = mycpu;
clr_mplock_contention_mask(gd);
mask = cpu_contention_mask;
- tmpmask = ~((1 << gd->gd_cpuid) - 1);
+ tmpmask = ~(CPUMASK(gd->gd_cpuid) - 1);
if (mask) {
if (mask & tmpmask)
- cpuid = bsfl(mask & tmpmask);
+ cpuid = BSFCPUMASK(mask & tmpmask);
else
- cpuid = bsfl(mask);
- atomic_clear_int(&cpu_contention_mask, 1 << cpuid);
+ cpuid = BSFCPUMASK(mask);
+ atomic_clear_cpumask(&cpu_contention_mask, CPUMASK(cpuid));
dgd = globaldata_find(cpuid);
lwkt_send_ipiq(dgd, lwkt_mp_lock_uncontested_remote, NULL);
}
SYSCTL_INT(_kern_polling, OID_AUTO, defcpu, CTLFLAG_RD,
&poll_defcpu, 0, "default CPU to run device polling");
-static uint32_t poll_cpumask0 = 0xffffffff;
-TUNABLE_INT("kern.polling.cpumask", (int *)&poll_cpumask0);
+static cpumask_t poll_cpumask0 = (cpumask_t)-1;
+TUNABLE_ULONG("kern.polling.cpumask", (u_long *)&poll_cpumask0);
-static uint32_t poll_cpumask;
-SYSCTL_INT(_kern_polling, OID_AUTO, cpumask, CTLFLAG_RD,
+static cpumask_t poll_cpumask;
+SYSCTL_LONG(_kern_polling, OID_AUTO, cpumask, CTLFLAG_RD,
&poll_cpumask, 0, "CPUs that can run device polling");
static int polling_enabled = 1; /* global polling enable */
if (cpuid >= POLLCTX_MAX)
return;
- if (((1 << cpuid) & poll_cpumask0) == 0)
+ if ((CPUMASK(cpuid) & poll_cpumask0) == 0)
return;
if (poll_burst_max < MIN_POLL_BURST_MAX)
if (poll_each_burst > poll_burst_max)
poll_each_burst = poll_burst_max;
- poll_cpumask |= (1 << cpuid);
+ poll_cpumask |= CPUMASK(cpuid);
pctx = kmalloc(sizeof(*pctx), M_DEVBUF, M_WAITOK | M_ZERO);
if (cpuid < 0 || cpuid >= POLLCTX_MAX)
return 0;
- if (((1 << cpuid) & poll_cpumask) == 0) {
+ if ((CPUMASK(cpuid) & poll_cpumask) == 0) {
/* Polling is not supported on 'cpuid' */
return 0;
}
int nid;
nid = (origcpu + n) % ncpus;
- if ((smp_active_mask & (1 << nid)) == 0)
+ if ((smp_active_mask & CPUMASK(nid)) == 0)
continue;
rgd = globaldata_find(nid);
lwkt_setcpu_self(rgd);
int lbolt_syncer;
int sched_quantum; /* Roundrobin scheduling quantum in ticks. */
int ncpus;
-int ncpus2, ncpus2_shift, ncpus2_mask;
-int ncpus_fit, ncpus_fit_mask;
+int ncpus2, ncpus2_shift, ncpus2_mask; /* note: mask not cpumask_t */
+int ncpus_fit, ncpus_fit_mask; /* note: mask not cpumask_t */
int safepri;
int tsleep_now_works;
int tsleep_crypto_dump = 0;
id = LOOKUP(td->td_wchan);
TAILQ_REMOVE(&gd->gd_tsleep_hash[id], td, td_sleepq);
if (TAILQ_FIRST(&gd->gd_tsleep_hash[id]) == NULL)
- atomic_clear_int(&slpque_cpumasks[id], gd->gd_cpumask);
+ atomic_clear_cpumask(&slpque_cpumasks[id], gd->gd_cpumask);
} else {
td->td_flags |= TDF_TSLEEPQ;
}
id = LOOKUP(ident);
TAILQ_INSERT_TAIL(&gd->gd_tsleep_hash[id], td, td_sleepq);
- atomic_set_int(&slpque_cpumasks[id], gd->gd_cpumask);
+ atomic_set_cpumask(&slpque_cpumasks[id], gd->gd_cpumask);
td->td_wchan = ident;
td->td_wdomain = flags & PDOMAIN_MASK;
crit_exit_quick(td);
id = LOOKUP(td->td_wchan);
TAILQ_REMOVE(&gd->gd_tsleep_hash[id], td, td_sleepq);
if (TAILQ_FIRST(&gd->gd_tsleep_hash[id]) == NULL)
- atomic_clear_int(&slpque_cpumasks[id], gd->gd_cpumask);
+ atomic_clear_cpumask(&slpque_cpumasks[id], gd->gd_cpumask);
td->td_wchan = NULL;
td->td_wdomain = 0;
}
error = EFBIG;
break;
}
- if ((smp_active_mask & (1 << cpuid)) == 0) {
+ if ((smp_active_mask & CPUMASK(cpuid)) == 0) {
error = EINVAL;
break;
}
- lp->lwp_cpumask = 1 << cpuid;
+ lp->lwp_cpumask = CPUMASK(cpuid);
if (cpuid != mycpu->gd_cpuid)
lwkt_migratecpu(cpuid);
break;
error = EFBIG;
break;
}
- if (!(smp_active_mask & (1 << cpuid))) {
+ if (!(smp_active_mask & CPUMASK(cpuid))) {
error = EINVAL;
break;
}
- lp->lwp_cpumask |= 1 << cpuid;
+ lp->lwp_cpumask |= CPUMASK(cpuid);
break;
case USCHED_DEL_CPU:
/* USCHED_DEL_CPU doesn't require special privileges. */
break;
}
lp = curthread->td_lwp;
- mask = lp->lwp_cpumask & smp_active_mask & ~(1 << cpuid);
+ mask = lp->lwp_cpumask & smp_active_mask & ~CPUMASK(cpuid);
if (mask == 0)
error = EPERM;
else {
- lp->lwp_cpumask &= ~(1 << cpuid);
+ lp->lwp_cpumask &= ~CPUMASK(cpuid);
if ((lp->lwp_cpumask & mycpu->gd_cpumask) == 0) {
- cpuid = bsfl(lp->lwp_cpumask & smp_active_mask);
+ cpuid = BSFCPUMASK(lp->lwp_cpumask &
+ smp_active_mask);
lwkt_migratecpu(cpuid);
}
}
* The message will not be sent to stopped cpus.
*/
int
-lwkt_send_ipiq3_mask(u_int32_t mask, ipifunc3_t func, void *arg1, int arg2)
+lwkt_send_ipiq3_mask(cpumask_t mask, ipifunc3_t func, void *arg1, int arg2)
{
int cpuid;
int count = 0;
mask &= ~stopped_cpus;
while (mask) {
- cpuid = bsfl(mask);
+ cpuid = BSFCPUMASK(mask);
lwkt_send_ipiq3(globaldata_find(cpuid), func, arg1, arg2);
- mask &= ~(1 << cpuid);
+ mask &= ~CPUMASK(cpuid);
++count;
}
return(count);
{
cpumask_t *cpumask = arg;
- atomic_clear_int(cpumask, mycpu->gd_cpumask);
+ atomic_clear_cpumask(cpumask, mycpu->gd_cpumask);
if (*cpumask == 0)
wakeup(cpumask);
}
cmd.cs_fin2_func = NULL;
cmd.cs_data = data;
lwkt_cpusync_start(mask & mycpu->gd_other_cpus, &cmd);
- if (mask & (1 << mycpu->gd_cpuid))
+ if (mask & CPUMASK(mycpu->gd_cpuid))
func(&cmd);
lwkt_cpusync_finish(&cmd);
}
cmd.cs_fin2_func = func;
cmd.cs_data = NULL;
lwkt_cpusync_start(mask & mycpu->gd_other_cpus, &cmd);
- if (mask & (1 << mycpu->gd_cpuid))
+ if (mask & CPUMASK(mycpu->gd_cpuid))
func(data);
lwkt_cpusync_finish(&cmd);
}
TAILQ_INIT(&bsd4_rtqueues[i]);
TAILQ_INIT(&bsd4_idqueues[i]);
}
- atomic_clear_int(&bsd4_curprocmask, 1);
+ atomic_clear_cpumask(&bsd4_curprocmask, 1);
}
SYSINIT(runqueue, SI_BOOT2_USCHED, SI_ORDER_FIRST, rqinit, NULL)
if (dd->uschedcp == lp) {
dd->upri = lp->lwp_priority;
} else if (dd->uschedcp == NULL) {
- atomic_set_int(&bsd4_curprocmask, gd->gd_cpumask);
+ atomic_set_cpumask(&bsd4_curprocmask, gd->gd_cpumask);
dd->uschedcp = lp;
dd->upri = lp->lwp_priority;
} else if (dd->upri > lp->lwp_priority) {
KKASSERT((lp->lwp_flag & LWP_ONRUNQ) == 0);
dd->uschedcp = NULL; /* don't let lp be selected */
dd->upri = PRIBASE_NULL;
- atomic_clear_int(&bsd4_curprocmask, gd->gd_cpumask);
+ atomic_clear_cpumask(&bsd4_curprocmask, gd->gd_cpumask);
bsd4_select_curproc(gd);
crit_exit();
}
spin_lock(&bsd4_spin);
if ((nlp = chooseproc_locked(dd->uschedcp)) != NULL) {
- atomic_set_int(&bsd4_curprocmask, 1 << cpuid);
+ atomic_set_cpumask(&bsd4_curprocmask, CPUMASK(cpuid));
dd->upri = nlp->lwp_priority;
dd->uschedcp = nlp;
spin_unlock(&bsd4_spin);
lwkt_acquire(nlp->lwp_thread);
#endif
lwkt_schedule(nlp->lwp_thread);
- } else if (bsd4_runqcount && (bsd4_rdyprocmask & (1 << cpuid))) {
- atomic_clear_int(&bsd4_rdyprocmask, 1 << cpuid);
+ } else if (bsd4_runqcount && (bsd4_rdyprocmask & CPUMASK(cpuid))) {
+ atomic_clear_cpumask(&bsd4_rdyprocmask, CPUMASK(cpuid));
spin_unlock(&bsd4_spin);
lwkt_schedule(&dd->helper_thread);
} else {
* the kernel.
*/
if (dd->uschedcp == NULL) {
- atomic_set_int(&bsd4_curprocmask, gd->gd_cpumask);
+ atomic_set_cpumask(&bsd4_curprocmask, gd->gd_cpumask);
dd->uschedcp = lp;
dd->upri = lp->lwp_priority;
lwkt_schedule(lp->lwp_thread);
spin_unlock(&bsd4_spin);
while (mask) {
- tmpmask = ~((1 << cpuid) - 1);
+ tmpmask = ~(CPUMASK(cpuid) - 1);
if (mask & tmpmask)
- cpuid = bsfl(mask & tmpmask);
+ cpuid = BSFCPUMASK(mask & tmpmask);
else
- cpuid = bsfl(mask);
+ cpuid = BSFCPUMASK(mask);
gd = globaldata_find(cpuid);
dd = &bsd4_pcpu[cpuid];
lwkt_send_ipiq(gd, need_user_resched_remote, NULL);
break;
}
- mask &= ~(1 << cpuid);
+ mask &= ~CPUMASK(cpuid);
}
#else
/*
bsd4_pcpu_t dd = &bsd4_pcpu[gd->gd_cpuid];
if (dd->uschedcp == NULL && (bsd4_rdyprocmask & gd->gd_cpumask)) {
- atomic_clear_int(&bsd4_rdyprocmask, gd->gd_cpumask);
+ atomic_clear_cpumask(&bsd4_rdyprocmask, gd->gd_cpumask);
lwkt_schedule(&dd->helper_thread);
} else {
need_user_resched();
crit_enter_gd(gd);
lwkt_deschedule_self(gd->gd_curthread);
spin_lock(&bsd4_spin);
- atomic_set_int(&bsd4_rdyprocmask, cpumask);
+ atomic_set_cpumask(&bsd4_rdyprocmask, cpumask);
clear_user_resched(); /* This satisfied the reschedule request */
dd->rrcount = 0; /* Reset the round-robin counter */
*/
KKASSERT(dd->uschedcp == NULL);
if ((nlp = chooseproc_locked(NULL)) != NULL) {
- atomic_set_int(&bsd4_curprocmask, cpumask);
+ atomic_set_cpumask(&bsd4_curprocmask, cpumask);
dd->upri = nlp->lwp_priority;
dd->uschedcp = nlp;
spin_unlock(&bsd4_spin);
~bsd4_curprocmask;
if (tmpmask) {
if (tmpmask & ~(cpumask - 1))
- tmpid = bsfl(tmpmask & ~(cpumask - 1));
+ tmpid = BSFCPUMASK(tmpmask & ~(cpumask - 1));
else
- tmpid = bsfl(tmpmask);
+ tmpid = BSFCPUMASK(tmpmask);
bsd4_scancpu = tmpid;
- atomic_clear_int(&bsd4_rdyprocmask, 1 << tmpid);
+ atomic_clear_cpumask(&bsd4_rdyprocmask, CPUMASK(tmpid));
spin_unlock_wr(&bsd4_spin);
lwkt_schedule(&bsd4_pcpu[tmpid].helper_thread);
} else {
for (i = 0; i < ncpus; ++i) {
bsd4_pcpu_t dd = &bsd4_pcpu[i];
- cpumask_t mask = 1 << i;
+ cpumask_t mask = CPUMASK(i);
if ((mask & smp_active_mask) == 0)
continue;
* been enabled in rqinit().
*/
if (i)
- atomic_clear_int(&bsd4_curprocmask, mask);
- atomic_set_int(&bsd4_rdyprocmask, mask);
+ atomic_clear_cpumask(&bsd4_curprocmask, mask);
+ atomic_set_cpumask(&bsd4_rdyprocmask, mask);
dd->upri = PRIBASE_NULL;
}
if (bootverbose)
{
TAILQ_INIT(&dummy_runq);
spin_init(&dummy_spin);
- atomic_clear_int(&dummy_curprocmask, 1);
+ atomic_clear_cpumask(&dummy_curprocmask, 1);
}
SYSINIT(runqueue, SI_BOOT2_USCHED, SI_ORDER_FIRST, dummyinit, NULL)
*/
if (dd->uschedcp == lp ||
(dd->uschedcp == NULL && TAILQ_EMPTY(&dummy_runq))) {
- atomic_set_int(&dummy_curprocmask, gd->gd_cpumask);
+ atomic_set_cpumask(&dummy_curprocmask, gd->gd_cpumask);
dd->uschedcp = lp;
return;
}
spin_lock(&dummy_spin);
if ((lp = TAILQ_FIRST(&dummy_runq)) == NULL) {
dd->uschedcp = NULL;
- atomic_clear_int(&dummy_curprocmask, gd->gd_cpumask);
+ atomic_clear_cpumask(&dummy_curprocmask, gd->gd_cpumask);
spin_unlock(&dummy_spin);
} else {
--dummy_runqcount;
TAILQ_REMOVE(&dummy_runq, lp, lwp_procq);
lp->lwp_flag &= ~LWP_ONRUNQ;
dd->uschedcp = lp;
- atomic_set_int(&dummy_curprocmask, gd->gd_cpumask);
+ atomic_set_cpumask(&dummy_curprocmask, gd->gd_cpumask);
spin_unlock(&dummy_spin);
#ifdef SMP
lwkt_acquire(lp->lwp_thread);
if (dd->uschedcp == NULL) {
dd->uschedcp = lp;
- atomic_set_int(&dummy_curprocmask, gd->gd_cpumask);
+ atomic_set_cpumask(&dummy_curprocmask, gd->gd_cpumask);
lwkt_schedule(lp->lwp_thread);
} else {
/*
mask = ~dummy_curprocmask & dummy_rdyprocmask &
gd->gd_other_cpus;
if (mask) {
- cpuid = bsfl(mask);
- atomic_clear_int(&dummy_rdyprocmask, 1 << cpuid);
+ cpuid = BSFCPUMASK(mask);
+ atomic_clear_cpumask(&dummy_rdyprocmask, CPUMASK(cpuid));
spin_unlock(&dummy_spin);
lwkt_schedule(&dummy_pcpu[cpuid].helper_thread);
} else {
gd = mycpu;
cpuid = gd->gd_cpuid;
dd = &dummy_pcpu[cpuid];
- cpumask = 1 << cpuid;
+ cpumask = CPUMASK(cpuid);
for (;;) {
lwkt_deschedule_self(gd->gd_curthread); /* interlock */
- atomic_set_int(&dummy_rdyprocmask, cpumask);
+ atomic_set_cpumask(&dummy_rdyprocmask, cpumask);
spin_lock(&dummy_spin);
if (dd->uschedcp) {
/*
tmpmask = ~dummy_curprocmask & dummy_rdyprocmask &
gd->gd_other_cpus;
if (tmpmask && dummy_runqcount) {
- tmpid = bsfl(tmpmask);
+ tmpid = BSFCPUMASK(tmpmask);
KKASSERT(tmpid != cpuid);
- atomic_clear_int(&dummy_rdyprocmask, 1 << tmpid);
+ atomic_clear_cpumask(&dummy_rdyprocmask, CPUMASK(tmpid));
spin_unlock(&dummy_spin);
lwkt_schedule(&dummy_pcpu[tmpid].helper_thread);
} else {
TAILQ_REMOVE(&dummy_runq, lp, lwp_procq);
lp->lwp_flag &= ~LWP_ONRUNQ;
dd->uschedcp = lp;
- atomic_set_int(&dummy_curprocmask, cpumask);
+ atomic_set_cpumask(&dummy_curprocmask, cpumask);
spin_unlock(&dummy_spin);
#ifdef SMP
lwkt_acquire(lp->lwp_thread);
for (i = 0; i < ncpus; ++i) {
dummy_pcpu_t dd = &dummy_pcpu[i];
- cpumask_t mask = 1 << i;
+ cpumask_t mask = CPUMASK(i);
if ((mask & smp_active_mask) == 0)
continue;
* been enabled in rqinit().
*/
if (i)
- atomic_clear_int(&dummy_curprocmask, mask);
- atomic_set_int(&dummy_rdyprocmask, mask);
+ atomic_clear_cpumask(&dummy_curprocmask, mask);
+ atomic_set_cpumask(&dummy_rdyprocmask, mask);
}
if (bootverbose)
kprintf("\n");
int cpu_id;
cpu_id = (origcpu + ccpu) % ncpus;
- if ((smp_active_mask & (1 << cpu_id)) == 0)
+ if ((smp_active_mask & CPUMASK(cpu_id)) == 0)
continue;
rgd = globaldata_find(cpu_id);
lwkt_setcpu_self(rgd);
if (madt == NULL)
return;
for (i = 0; i < ncpus; i++) {
- if ((smp_active_mask & (1 << i)) == 0)
+ if ((smp_active_mask & CPUMASK(i)) == 0)
continue;
md = (struct mdglobaldata *)globaldata_find(i);
KKASSERT(md != NULL);
* APIC_DELMODE_FIXED or APIC_DELMODE_LOWPRIO.
*/
void
-selected_apic_ipi(u_int target, int vector, int delivery_mode)
+selected_apic_ipi(cpumask_t target, int vector, int delivery_mode)
{
crit_enter();
while (target) {
- int n = bsfl(target);
- target &= ~(1 << n);
+ int n = BSFCPUMASK(target);
+ target &= ~CPUMASK(n);
single_apic_ipi(n, vector, delivery_mode);
}
crit_exit();
ASSYM(TDPRI_INT_SUPPORT, TDPRI_INT_SUPPORT);
#ifdef SMP
ASSYM(CPUMASK_LOCK, CPUMASK_LOCK);
+ASSYM(CPUMASK_BIT, CPUMASK_BIT);
#endif
ASSYM(V_TRAP, offsetof(struct vmmeter, v_trap));
ASSYM(GD_NPXTHREAD, offsetof(struct mdglobaldata, gd_npxthread));
ASSYM(GD_FPU_LOCK, offsetof(struct mdglobaldata, gd_fpu_lock));
ASSYM(GD_SAVEFPU, offsetof(struct mdglobaldata, gd_savefpu));
-ASSYM(GD_OTHER_CPUS, offsetof(struct mdglobaldata, gd_other_cpus));
+ASSYM(GD_OTHER_CPUS, offsetof(struct mdglobaldata, mi.gd_other_cpus));
ASSYM(GD_SS_EFLAGS, offsetof(struct mdglobaldata, gd_ss_eflags));
ASSYM(GD_CMAP1, offsetof(struct mdglobaldata, gd_CMAP1));
ASSYM(GD_CMAP2, offsetof(struct mdglobaldata, gd_CMAP2));
mask = cpu_contention_mask;
cpu_ccfence();
- if (mask && bsfl(mask) != mycpu->gd_cpuid)
+ if (mask && BSFCPUMASK(mask) != mycpu->gd_cpuid)
DELAY(2);
}
static int mptable_search(void);
static int mptable_check(vm_paddr_t);
static int mptable_search_sig(u_int32_t target, int count);
-static int mptable_hyperthread_fixup(u_int, int);
+static int mptable_hyperthread_fixup(cpumask_t, int);
#ifdef SMP /* APIC-IO */
static void mptable_pass1(struct mptable_pos *);
static void mptable_pass2(struct mptable_pos *);
* with the number of logical CPU's in the processor.
*/
static int
-mptable_hyperthread_fixup(u_int id_mask, int cpu_count)
+mptable_hyperthread_fixup(cpumask_t id_mask, int cpu_count)
{
int i, id, lcpus_max, logical_cpus;
*/
dist = cur = prev = -1;
for (id = 0; id < MAXCPU; ++id) {
- if ((id_mask & 1 << id) == 0)
+ if ((id_mask & CPUMASK(id)) == 0)
continue;
cur = id;
* already in the table, then kill the fixup.
*/
for (id = 0; id < MAXCPU; id++) {
- if ((id_mask & 1 << id) == 0)
+ if ((id_mask & CPUMASK(id)) == 0)
continue;
/* First, make sure we are on a logical_cpus boundary. */
if (id % logical_cpus != 0)
return 0;
for (i = id + 1; i < id + logical_cpus; i++)
- if ((id_mask & 1 << i) != 0)
+ if ((id_mask & CPUMASK(i)) != 0)
return 0;
}
return logical_cpus;
ncpus_fit_mask = ncpus_fit - 1;
/* build our map of 'other' CPUs */
- mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid);
+ mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * ncpus);
bzero(mycpu->gd_ipiq, sizeof(lwkt_ipiq) * ncpus);
/* wait for it to start, see ap_init() */
set_apic_timer(5000000);/* == 5 seconds */
while (read_apic_timer()) {
- if (smp_startup_mask & (1 << gd->mi.gd_cpuid))
+ if (smp_startup_mask & CPUMASK(gd->mi.gd_cpuid))
return 1; /* return SUCCESS */
}
crit_enter_gd(&md->mi);
md->gd_invltlb_ret = 0;
++md->mi.gd_cnt.v_smpinvltlb;
- atomic_set_int(&smp_invltlb_req, md->mi.gd_cpumask);
+ atomic_set_cpumask(&smp_invltlb_req, md->mi.gd_cpumask);
#ifdef SMP_INVLTLB_DEBUG
again:
#endif
if (xcount > 2)
lwkt_process_ipiq();
if (xcount > 3) {
- int bcpu = bsfl(~md->gd_invltlb_ret & ~md->mi.gd_cpumask & smp_active_mask);
+ int bcpu = BSFCPUMASK(~md->gd_invltlb_ret &
+ ~md->mi.gd_cpumask &
+ smp_active_mask);
globaldata_t xgd;
kprintf("bcpu %d\n", bcpu);
xgd = globaldata_find(bcpu);
}
#endif
}
- atomic_clear_int(&smp_invltlb_req, md->mi.gd_cpumask);
+ atomic_clear_cpumask(&smp_invltlb_req, md->mi.gd_cpumask);
crit_exit_gd(&md->mi);
#endif
}
cpu_mfence();
cpu_invltlb();
while (mask) {
- cpu = bsfl(mask);
- mask &= ~(1 << cpu);
+ cpu = BSFCPUMASK(mask);
+ mask &= ~CPUMASK(cpu);
omd = (struct mdglobaldata *)globaldata_find(cpu);
- atomic_set_int(&omd->gd_invltlb_ret, md->mi.gd_cpumask);
+ atomic_set_cpumask(&omd->gd_invltlb_ret, md->mi.gd_cpumask);
}
}
* from executing at same time.
*/
int
-stop_cpus(u_int map)
+stop_cpus(cpumask_t map)
{
map &= smp_active_mask;
* 1: ok
*/
int
-restart_cpus(u_int map)
+restart_cpus(cpumask_t map)
{
/* signal other cpus to restart */
started_cpus = map & smp_active_mask;
* interrupts physically disabled and remote cpus could deadlock
* trying to send us an IPI.
*/
- smp_startup_mask |= 1 << mycpu->gd_cpuid;
+ smp_startup_mask |= CPUMASK(mycpu->gd_cpuid);
cpu_mfence();
/*
#endif
/* Build our map of 'other' CPUs. */
- mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid);
+ mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
kprintf("SMP: AP CPU #%d Launched!\n", mycpu->gd_cpuid);
* nothing we've done put it there.
*/
KKASSERT(curthread->td_mpcount == 1);
- smp_active_mask |= 1 << mycpu->gd_cpuid;
+ smp_active_mask |= CPUMASK(mycpu->gd_cpuid);
/*
* Enable interrupts here. idle_restore will also do it, but
void
cpu_send_ipiq(int dcpu)
{
- if ((1 << dcpu) & smp_active_mask)
+ if (CPUMASK(dcpu) & smp_active_mask)
single_apic_ipi(dcpu, XIPIQ_OFFSET, APIC_DELMODE_FIXED);
}
cpu_send_ipiq_passive(int dcpu)
{
int r = 0;
- if ((1 << dcpu) & smp_active_mask) {
+ if (CPUMASK(dcpu) & smp_active_mask) {
r = single_apic_ipi_passive(dcpu, XIPIQ_OFFSET,
APIC_DELMODE_FIXED);
}
if (curthread->td_lwp == lp) {
pmap = vmspace_pmap(newvm);
#if defined(SMP)
- atomic_set_int(&pmap->pm_active, mycpu->gd_cpumask);
+ atomic_set_cpumask(&pmap->pm_active, mycpu->gd_cpumask);
if (pmap->pm_active & CPUMASK_LOCK)
pmap_interlock_wait(newvm);
#else
load_cr3(curthread->td_pcb->pcb_cr3);
pmap = vmspace_pmap(oldvm);
#if defined(SMP)
- atomic_clear_int(&pmap->pm_active, mycpu->gd_cpumask);
+ atomic_clear_cpumask(&pmap->pm_active,
+ mycpu->gd_cpumask);
#else
- pmap->pm_active &= ~1;
+ pmap->pm_active &= ~(cpumask_t)1;
#endif
}
}
for (;;) {
oactive = pmap->pm_active & ~CPUMASK_LOCK;
nactive = oactive | CPUMASK_LOCK;
- if (atomic_cmpset_int(&pmap->pm_active, oactive, nactive))
+ if (atomic_cmpset_cpumask(&pmap->pm_active, oactive, nactive))
break;
crit_enter();
lwkt_process_ipiq();
pmap_inval_deinterlock(pmap_inval_info_t info, pmap_t pmap)
{
#ifdef SMP
- atomic_clear_int(&pmap->pm_active, CPUMASK_LOCK);
+ atomic_clear_cpumask(&pmap->pm_active, CPUMASK_LOCK);
#endif
}
cpu_reset_proxy_active = 3;
while (cpu_reset_proxy_active == 3)
; /* Wait for other cpu to enable interrupts */
- stop_cpus((1<<cpu_reset_proxyid));
+ stop_cpus(CPUMASK(cpu_reset_proxyid));
kprintf("cpu_reset_proxy: Stopped CPU %d\n", cpu_reset_proxyid);
DELAY(1000000);
cpu_reset_real();
cpu_reset_real();
/* NOTREACHED */
} else {
- u_int map;
+ cpumask_t map;
int cnt;
kprintf("cpu_reset called on cpu#%d\n",mycpu->gd_cpuid);
union savefpu gd_savefpu; /* fast bcopy/zero temp fpu save area */
int gd_fpu_lock; /* fast bcopy/zero cpu lock */
int gd_fpending; /* fast interrupt pending */
- int unused002;
+ int unused001;
int gd_spending; /* software interrupt pending */
int gd_sdelayed; /* delayed software ints */
int gd_currentldt;
int gd_private_tss;
- u_int unused001;
- u_int gd_other_cpus;
+ u_int unused002;
+ u_int unused003;
u_int gd_ss_eflags;
pt_entry_t *gd_CMAP1;
pt_entry_t *gd_CMAP2;
#define pmap_resident_count(pmap) (pmap)->pm_stats.resident_count
-#define CPUMASK_LOCK (1 << SMP_MAXCPU)
+#define CPUMASK_LOCK CPUMASK(SMP_MAXCPU)
+#define CPUMASK_BIT SMP_MAXCPU /* 1 << SMP_MAXCPU */
typedef struct pmap *pmap_t;
void bootMP (void);
/* global data in apic_vector.s */
-extern volatile u_int stopped_cpus;
-extern volatile u_int started_cpus;
+extern volatile cpumask_t stopped_cpus;
+extern volatile cpumask_t started_cpus;
extern volatile u_int checkstate_probed_cpus;
extern void (*cpustop_restartfunc) (void);
void assign_apic_irq (int apic, int intpin, int irq);
void revoke_apic_irq (int irq);
void init_secondary (void);
-int stop_cpus (u_int);
+int stop_cpus (cpumask_t);
void ap_init (void);
-int restart_cpus (u_int);
+int restart_cpus (cpumask_t);
void forward_signal (struct proc *);
#ifndef _SYS_QUEUE_H_
void apic_initialize (boolean_t);
void imen_dump (void);
int apic_ipi (int, int, int);
-void selected_apic_ipi (u_int, int, int);
+void selected_apic_ipi (cpumask_t, int, int);
void single_apic_ipi(int cpu, int vector, int delivery_mode);
int single_apic_ipi_passive(int cpu, int vector, int delivery_mode);
int io_apic_setup (int);
if (madt == NULL)
return;
for (i = 0; i < ncpus; i++) {
- if ((smp_active_mask & (1 << i)) == 0)
+ if ((smp_active_mask & CPUMASK(i)) == 0)
continue;
md = (struct mdglobaldata *)globaldata_find(i);
KKASSERT(md != NULL);
addq %rax, %rdi
call CNAME(savectx) /* Save process context */
- movl PCPU(cpuid), %eax
+ movslq PCPU(cpuid), %rax
/*
* Indicate that we have stopped and loop waiting for permission
* (e.g. Xtimer, Xinvltlb).
*/
MPLOCKED
- btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
+ btsq %rax, stopped_cpus /* stopped_cpus |= (1<<id) */
sti
1:
andl $~RQF_IPIQ,PCPU(reqflags)
call lwkt_smp_stopped
popq %rax
pause
- btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
+ btq %rax, started_cpus /* while (!(started_cpus & (1<<id))) */
jnc 1b
MPLOCKED
- btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
+ btrq %rax, started_cpus /* started_cpus &= ~(1<<id) */
MPLOCKED
- btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
+ btrq %rax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
test %eax, %eax
jnz 2f
/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
.globl stopped_cpus, started_cpus
stopped_cpus:
- .long 0
+ .quad 0
started_cpus:
- .long 0
+ .quad 0
.globl CNAME(cpustop_restartfunc)
CNAME(cpustop_restartfunc):
* APIC_DELMODE_FIXED or APIC_DELMODE_LOWPRIO.
*/
void
-selected_apic_ipi(u_int target, int vector, int delivery_mode)
+selected_apic_ipi(cpumask_t target, int vector, int delivery_mode)
{
crit_enter();
while (target) {
- int n = bsfl(target);
- target &= ~(1 << n);
+ int n = BSFCPUMASK(target);
+ target &= ~CPUMASK(n);
single_apic_ipi(n, vector, delivery_mode);
}
crit_exit();
*/
lapic = pmap_mapdev_uncacheable(lapic_addr, sizeof(struct LAPIC));
-#if 0
- /* Local apic is mapped on last page */
- SMPpt[NPTEPG - 1] = (pt_entry_t)(PG_V | PG_RW | PG_N |
- pmap_get_pgeflag() | (lapic_addr & PG_FRAME));
-#endif
-
cpu_apic_addr = lapic_addr;
kprintf("lapic: at 0x%08lx\n", lapic_addr);
int gd_currentldt;
int gd_private_tss;
u_int unused002;
- u_int gd_other_cpus;
+ u_int unused003;
u_int gd_ss_eflags;
- pt_entry_t *gd_CMAP1;
- pt_entry_t *gd_CMAP2;
- pt_entry_t *gd_CMAP3;
- pt_entry_t *gd_PMAP1;
- caddr_t gd_CADDR1;
- caddr_t gd_CADDR2;
- caddr_t gd_CADDR3;
- pt_entry_t *gd_PADDR1;
+ pt_entry_t *gd_cunused0;
+ pt_entry_t *gd_cunused1;
+ pt_entry_t *gd_cunused2;
+ pt_entry_t *gd_cunused3;
+ caddr_t gd_aunused0;
+ caddr_t gd_aunused1;
+ caddr_t gd_aunused2;
+ pt_entry_t *gd_aunused3;
u_int gd_acpi_id;
u_int gd_apic_id;
register_t gd_scratch_rsp;
- register_t unused003;
+ register_t unused004;
register_t gd_user_fs; /* current user fs in MSR */
register_t gd_user_gs; /* current user gs in MSR */
cpumask_t gd_invltlb_ret;
* This is the upper (0xff800000) address space layout that is per-cpu.
* It is setup in locore.s and pmap.c for the BSP and in mp_machdep.c for
* each AP. genassym helps export this to the assembler code.
- *
- * JG WARNING! page-bounded fields are hardwired for SMPpt[] setup in
- * i386/i386/mp_machdep.c and locore.s.
*/
struct privatespace {
/* JG TODO: fix comments describing layout */
struct mdglobaldata mdglobaldata;
char __filler0[MDGLOBALDATA_PAD];
- /* page 1..4 - CPAGE1,CPAGE2,CPAGE3,PPAGE1 */
- char CPAGE1[PAGE_SIZE]; /* SMPpt[1] */
- char CPAGE2[PAGE_SIZE]; /* SMPpt[2] */
- char CPAGE3[PAGE_SIZE]; /* SMPpt[3] */
- char PPAGE1[PAGE_SIZE]; /* SMPpt[4] */
+ /* page 1..4 - CPAGE1,CPAGE2,CPAGE3,PPAGE1 (unused) */
+ char unused1[PAGE_SIZE];
+ char unused2[PAGE_SIZE];
+ char unused3[PAGE_SIZE];
+ char unused4[PAGE_SIZE];
/* page 5..4+UPAGES - idle stack (UPAGES pages) */
- char idlestack[UPAGES * PAGE_SIZE]; /* SMPpt[5..] */
+ char idlestack[UPAGES * PAGE_SIZE];
};
#define mdcpu ((struct mdglobaldata *)_get_mycpu())
#define NDMPML4E 1 /* number of dmap PML4 slots */
/*
- * The *PML4I values control the layout of virtual memory
+ * The *PML4I values control the layout of virtual memory. Each PML4
+ * entry represents 512G.
*/
#define PML4PML4I (NPML4EPG/2) /* Index of recursive pml4 mapping */
#define KPML4I (NPML4EPG-1) /* Top 512GB for KVM */
#define DMPML4I (KPML4I-1) /* Next 512GB down for direct map */
+/*
+ * The location of KERNBASE in the last PD of the kernel's KVM (KPML4I)
+ * space. Each PD represents 1GB. The kernel must be placed here
+ * for the compile/link options to work properly so absolute 32-bit
+ * addressing can be used to access stuff.
+ */
#define KPDPI (NPDPEPG-2) /* kernbase at -2GB */
-/* per-CPU data is at -2MB */
-/* XXX can the kernel decide to use this memory for something else? */
+/*
+ * per-CPU data assume ~64K x SMP_MAXCPU, say up to 256 cpus
+ * in the future or 16MB of space. Each PD represents 2MB so
+ * use NPDEPG-8 to place the per-CPU data.
+ */
#define MPPML4I KPML4I
#define MPPDPI KPDPI
-#define MPPTDI (NPDEPG-1)
+#define MPPTDI (NPDEPG-8)
/*
* XXX doesn't really belong here I guess...
int pm_generation; /* detect pvlist deletions */
};
-#define CPUMASK_LOCK (1 << SMP_MAXCPU)
+#define CPUMASK_LOCK CPUMASK(SMP_MAXCPU)
+#define CPUMASK_BIT SMP_MAXCPU /* for 1LLU << SMP_MAXCPU */
#define pmap_resident_count(pmap) (pmap)->pm_stats.resident_count
void bootMP (void);
/* global data in apic_vector.s */
-extern volatile u_int stopped_cpus;
-extern volatile u_int started_cpus;
+extern volatile cpumask_t stopped_cpus;
+extern volatile cpumask_t started_cpus;
extern volatile u_int checkstate_probed_cpus;
extern void (*cpustop_restartfunc) (void);
void assign_apic_irq (int apic, int intpin, int irq);
void revoke_apic_irq (int irq);
void init_secondary (void);
-int stop_cpus (u_int);
+int stop_cpus (cpumask_t);
void ap_init (void);
-int restart_cpus (u_int);
+int restart_cpus (cpumask_t);
void forward_signal (struct proc *);
#ifndef _SYS_QUEUE_H_
void apic_initialize (boolean_t);
void imen_dump (void);
int apic_ipi (int, int, int);
-void selected_apic_ipi (u_int, int, int);
+void selected_apic_ipi (cpumask_t, int, int);
void single_apic_ipi(int cpu, int vector, int delivery_mode);
int single_apic_ipi_passive(int cpu, int vector, int delivery_mode);
int io_apic_setup (int);
* The kernel address space can be up to (I think) 511 page directory
* pages. Each one represents 1G. NKPDPE defines the size of the kernel
* address space, curently set to 128G.
+ *
+ * The ending address is non-inclusive of the per-cpu data array
+ * which starts at MPPTDI (-16MB mark). MPPTDI is the page directory
+ * index in the last PD of the kernel address space and is typically
+ * set to (NPDEPG - 8) = (512 - 8).
*/
#define NKPDPE 128
-#define VM_MAX_KERNEL_ADDRESS KVADDR(KPML4I, NPDPEPG-1, NPDEPG-1, NPTEPG-1)
-#define VM_MIN_KERNEL_ADDRESS KVADDR(KPML4I, NPDPEPG-NKPDPE, 0, 0)
+#define VM_MIN_KERNEL_ADDRESS KVADDR(KPML4I, NPDPEPG - NKPDPE, 0, 0)
+#define VM_MAX_KERNEL_ADDRESS KVADDR(KPML4I, NPDPEPG - 1, MPPTDI, NPTEPG - 1)
#define DMAP_MIN_ADDRESS KVADDR(DMPML4I, 0, 0, 0)
#define DMAP_MAX_ADDRESS KVADDR(DMPML4I+1, 0, 0, 0)
profhz = RTC_PROFRATE;
}
- /* Finish initializing 8253 timer 0. */
+ /* Finish initializing 8254 timer 0. */
#ifdef SMP /* APIC-IO */
if (apic_io_enable) {
apic_8254_intr = isa_apic_irq(0);
crit_enter();
#ifdef SMP
- db_printf("\nCPU%d stopping CPUs: 0x%08x\n",
- mycpu->gd_cpuid, mycpu->gd_other_cpus);
+ db_printf("\nCPU%d stopping CPUs: 0x%08jx\n",
+ mycpu->gd_cpuid, (uintmax_t)mycpu->gd_other_cpus);
/* We stop all CPUs except ourselves (obviously) */
stop_cpus(mycpu->gd_other_cpus);
db_global_jmpbuf_valid = FALSE;
#ifdef SMP
- db_printf("\nCPU%d restarting CPUs: 0x%08x\n",
- mycpu->gd_cpuid, stopped_cpus);
+ db_printf("\nCPU%d restarting CPUs: 0x%016jx\n",
+ mycpu->gd_cpuid, (uintmax_t)stopped_cpus);
/* Restart all the CPUs we previously stopped */
if (stopped_cpus != mycpu->gd_other_cpus) {
- db_printf("whoa, other_cpus: 0x%08x, stopped_cpus: 0x%08x\n",
- mycpu->gd_other_cpus, stopped_cpus);
+ db_printf("whoa, other_cpus: 0x%016jx, "
+ "stopped_cpus: 0x%016jx\n",
+ (uintmax_t)mycpu->gd_other_cpus,
+ (uintmax_t)stopped_cpus);
panic("stop_cpus() failed");
}
restart_cpus(stopped_cpus);
ASSYM(GD_NPXTHREAD, offsetof(struct mdglobaldata, gd_npxthread));
ASSYM(GD_FPU_LOCK, offsetof(struct mdglobaldata, gd_fpu_lock));
ASSYM(GD_SAVEFPU, offsetof(struct mdglobaldata, gd_savefpu));
-ASSYM(GD_OTHER_CPUS, offsetof(struct mdglobaldata, gd_other_cpus));
+ASSYM(GD_OTHER_CPUS, offsetof(struct mdglobaldata, mi.gd_other_cpus));
ASSYM(GD_SS_EFLAGS, offsetof(struct mdglobaldata, gd_ss_eflags));
ASSYM(GD_REQFLAGS, offsetof(struct mdglobaldata, mi.gd_reqflags));
ASSYM(TDPRI_INT_SUPPORT, TDPRI_INT_SUPPORT);
#ifdef SMP
ASSYM(CPUMASK_LOCK, CPUMASK_LOCK);
+ASSYM(CPUMASK_BIT, CPUMASK_BIT);
#endif
#ifdef SMP
*/
.globl gd_cpuid, gd_cpumask, gd_other_cpus
.globl gd_ss_eflags, gd_intr_nesting_level
- .globl gd_CMAP1, gd_CMAP2, gd_CMAP3, gd_PMAP1
- .globl gd_CADDR1, gd_CADDR2, gd_CADDR3, gd_PADDR1
.globl gd_spending, gd_fpending
.globl gd_cnt, gd_private_tss
.globl gd_scratch_rsp
.set gd_other_cpus,globaldata + GD_OTHER_CPUS
.set gd_ss_eflags,globaldata + GD_SS_EFLAGS
.set gd_intr_nesting_level,globaldata + GD_INTR_NESTING_LEVEL
- .set gd_CMAP1,globaldata + GD_PRV_CMAP1
- .set gd_CMAP2,globaldata + GD_PRV_CMAP2
- .set gd_CMAP3,globaldata + GD_PRV_CMAP3
- .set gd_PMAP1,globaldata + GD_PRV_PMAP1
- .set gd_CADDR1,globaldata + GD_PRV_CADDR1
- .set gd_CADDR2,globaldata + GD_PRV_CADDR2
- .set gd_CADDR3,globaldata + GD_PRV_CADDR3
- .set gd_PADDR1,globaldata + GD_PRV_PADDR1
.set gd_fpending,globaldata + GD_FPENDING
.set gd_spending,globaldata + GD_SPENDING
.set gd_cnt,globaldata + GD_CNT
extern vm_offset_t ksym_start, ksym_end;
#endif
-uint64_t SMPptpa;
-pt_entry_t *SMPpt;
-
-
-struct privatespace CPU_prvspace[MAXCPU];
+struct privatespace CPU_prvspace[MAXCPU] __aligned(4096); /* XXX */
int _udatasel, _ucodesel, _ucode32sel;
u_long atdevbase;
mask = cpu_contention_mask;
cpu_ccfence();
- if (mask && bsfl(mask) != mycpu->gd_cpuid)
+ if (mask && BSFCPUMASK(mask) != mycpu->gd_cpuid)
DELAY(2);
}
char *bootSTK;
static int bootAP;
-/*
- * SMP page table page. Setup by locore to point to a page table
- * page from which we allocate per-cpu privatespace areas io_apics,
- * and so forth.
- */
-
-#define IO_MAPPING_START_INDEX \
- (SMP_MAXCPU * sizeof(struct privatespace) / PAGE_SIZE)
-
-extern pt_entry_t *SMPpt;
-
struct pcb stoppcbs[MAXCPU];
extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32);
static int mptable_search(void);
static int mptable_check(vm_paddr_t);
static long mptable_search_sig(u_int32_t target, int count);
-static int mptable_hyperthread_fixup(u_int, int);
+static int mptable_hyperthread_fixup(cpumask_t, int);
#ifdef SMP /* APIC-IO */
static void mptable_pass1(struct mptable_pos *);
static void mptable_pass2(struct mptable_pos *);
long x;
u_int32_t target;
- /*
- * Make sure our SMPpt[] page table is big enough to hold all the
- * mappings we need.
- */
- KKASSERT(IO_MAPPING_START_INDEX < NPTEPG - 2);
-
POSTCODE(MP_PROBE_POST);
/* see if EBDA exists */
if (apic_io_enable) {
for (x = 0; x < mp_napics; ++x) {
kprintf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x));
- kprintf("napics %d versions %p address %p\n", mp_napics, io_apic_versions, io_apic_address);
kprintf(", version: 0x%08x", io_apic_versions[x]);
kprintf(", at 0x%08lx\n", io_apic_address[x]);
}
MALLOC(bus_data, bus_datum *, sizeof(bus_datum) * mp_nbusses,
M_DEVBUF, M_WAITOK);
- kprintf("xapics %d versions %p address %p\n", mp_napics, io_apic_versions, io_apic_address);
-
for (x = 0; x < mp_napics; x++)
ioapic[x] = permanent_io_mapping(io_apic_address[x]);
* with the number of logical CPU's in the processor.
*/
static int
-mptable_hyperthread_fixup(u_int id_mask, int cpu_count)
+mptable_hyperthread_fixup(cpumask_t id_mask, int cpu_count)
{
int i, id, lcpus_max, logical_cpus;
*/
dist = cur = prev = -1;
for (id = 0; id < MAXCPU; ++id) {
- if ((id_mask & 1 << id) == 0)
+ if ((id_mask & CPUMASK(id)) == 0)
continue;
cur = id;
* already in the table, then kill the fixup.
*/
for (id = 0; id < MAXCPU; id++) {
- if ((id_mask & 1 << id) == 0)
+ if ((id_mask & CPUMASK(id)) == 0)
continue;
/* First, make sure we are on a logical_cpus boundary. */
if (id % logical_cpus != 0)
return 0;
for (i = id + 1; i < id + logical_cpus; i++)
- if ((id_mask & 1 << i) != 0)
+ if ((id_mask & CPUMASK(i)) != 0)
return 0;
}
return logical_cpus;
for (x = 0; x < NAPICID; x++)
ID_TO_IO(x) = -1;
- for (x = 0; x <= mp_naps; x++)
- if (CPU_TO_ID(x) < NAPICID)
+ for (x = 0; x <= mp_naps; x++) {
+ if ((u_int)CPU_TO_ID(x) < NAPICID)
ID_TO_IO(CPU_TO_ID(x)) = x;
+ }
- for (x = 0; x < mp_napics; x++)
- if (IO_TO_ID(x) < NAPICID)
+ for (x = 0; x < mp_napics; x++) {
+ if ((u_int)IO_TO_ID(x) < NAPICID)
ID_TO_IO(IO_TO_ID(x)) = x;
+ }
}
int cpu; /* Logical CPU number */
int oapic; /* Logical IO APIC number for other IO APIC */
- if (id >= NAPICID)
+ if ((u_int)id >= NAPICID)
return 0; /* Out of range */
- for (cpu = 0; cpu <= mp_naps; cpu++)
+ for (cpu = 0; cpu <= mp_naps; cpu++) {
if (CPU_TO_ID(cpu) == id)
return 0; /* Conflict with CPU */
+ }
- for (oapic = 0; oapic < mp_napics && oapic < apic; oapic++)
+ for (oapic = 0; oapic < mp_napics && oapic < apic; oapic++) {
if (IO_TO_ID(oapic) == id)
return 0; /* Conflict with other APIC */
+ }
return 1; /* ID is acceptable for IO APIC */
}
bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type;
bus_data[bus_0].bus_type = PCI;
- /* swap each relavant INTerrupt entry */
+ /* swap each relevant INTerrupt entry */
id = bus_data[bus_pci].bus_id;
for (x = 0; x < nintrs; ++x) {
if (io_apic_ints[x].src_bus_id == id) {
} else if (apic_int_type(0, 0) == 0) {
kprintf("APIC_IO: MP table broken: ExtINT entry corrupt!\n");
for (x = 0; x < nintrs; ++x)
- if ((0 == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
- (0 == io_apic_ints[x].dst_apic_int)) {
+ if ((ID_TO_IO(io_apic_ints[x].dst_apic_id) == 0) &&
+ (io_apic_ints[x].dst_apic_int) == 0) {
io_apic_ints[x].int_type = 3;
io_apic_ints[x].int_vector = 0xff;
break;
/* prime data page for it to use */
mi_gdinit(&gd->mi, x);
cpu_gdinit(gd, x);
- gd->gd_CMAP1 = &SMPpt[pg + 0];
- gd->gd_CMAP2 = &SMPpt[pg + 1];
- gd->gd_CMAP3 = &SMPpt[pg + 2];
- gd->gd_PMAP1 = &SMPpt[pg + 3];
- gd->gd_CADDR1 = ps->CPAGE1;
- gd->gd_CADDR2 = ps->CPAGE2;
- gd->gd_CADDR3 = ps->CPAGE3;
- gd->gd_PADDR1 = (pt_entry_t *)ps->PPAGE1;
gd->mi.gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * (mp_naps + 1));
bzero(gd->mi.gd_ipiq, sizeof(lwkt_ipiq) * (mp_naps + 1));
ncpus_fit_mask = ncpus_fit - 1;
/* build our map of 'other' CPUs */
- mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid);
+ mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * ncpus);
bzero(mycpu->gd_ipiq, sizeof(lwkt_ipiq) * ncpus);
/* wait for it to start, see ap_init() */
set_apic_timer(5000000);/* == 5 seconds */
while (read_apic_timer()) {
- if (smp_startup_mask & (1 << gd->mi.gd_cpuid))
+ if (smp_startup_mask & CPUMASK(gd->mi.gd_cpuid))
return 1; /* return SUCCESS */
}
crit_enter_gd(&md->mi);
md->gd_invltlb_ret = 0;
++md->mi.gd_cnt.v_smpinvltlb;
- atomic_set_int(&smp_invltlb_req, md->mi.gd_cpumask);
+ atomic_set_cpumask(&smp_invltlb_req, md->mi.gd_cpumask);
#ifdef SMP_INVLTLB_DEBUG
again:
#endif
if (xcount > 2)
lwkt_process_ipiq();
if (xcount > 3) {
- int bcpu = bsfl(~md->gd_invltlb_ret & ~md->mi.gd_cpumask & smp_active_mask);
+ int bcpu = BSFCPUMASK(~md->gd_invltlb_ret &
+ ~md->mi.gd_cpumask &
+ smp_active_mask);
globaldata_t xgd;
kprintf("bcpu %d\n", bcpu);
}
#endif
}
- atomic_clear_int(&smp_invltlb_req, md->mi.gd_cpumask);
+ atomic_clear_cpumask(&smp_invltlb_req, md->mi.gd_cpumask);
crit_exit_gd(&md->mi);
#endif
}
mask = smp_invltlb_req;
cpu_invltlb();
while (mask) {
- cpu = bsfl(mask);
- mask &= ~(1 << cpu);
+ cpu = BSFCPUMASK(mask);
+ mask &= ~CPUMASK(cpu);
omd = (struct mdglobaldata *)globaldata_find(cpu);
- atomic_set_int(&omd->gd_invltlb_ret, md->mi.gd_cpumask);
+ atomic_set_cpumask(&omd->gd_invltlb_ret, md->mi.gd_cpumask);
}
}
* from executing at same time.
*/
int
-stop_cpus(u_int map)
+stop_cpus(cpumask_t map)
{
map &= smp_active_mask;
* 1: ok
*/
int
-restart_cpus(u_int map)
+restart_cpus(cpumask_t map)
{
/* signal other cpus to restart */
started_cpus = map & smp_active_mask;
* interrupts physically disabled and remote cpus could deadlock
* trying to send us an IPI.
*/
- smp_startup_mask |= 1 << mycpu->gd_cpuid;
+ smp_startup_mask |= CPUMASK(mycpu->gd_cpuid);
cpu_mfence();
/*
#endif
/* Build our map of 'other' CPUs. */
- mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid);
+ mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
kprintf("SMP: AP CPU #%d Launched!\n", mycpu->gd_cpuid);
* nothing we've done put it there.
*/
KKASSERT(curthread->td_mpcount == 1);
- smp_active_mask |= 1 << mycpu->gd_cpuid;
+ smp_active_mask |= CPUMASK(mycpu->gd_cpuid);
/*
* Enable interrupts here. idle_restore will also do it, but
}
while (try_mplock() == 0)
;
- if (bootverbose)
- kprintf("Active CPU Mask: %08x\n", smp_active_mask);
+ if (bootverbose) {
+ kprintf("Active CPU Mask: %016jx\n",
+ (uintmax_t)smp_active_mask);
+ }
}
SYSINIT(finishsmp, SI_BOOT2_FINISH_SMP, SI_ORDER_FIRST, ap_finish, NULL)
void
cpu_send_ipiq(int dcpu)
{
- if ((1 << dcpu) & smp_active_mask)
+ if (CPUMASK(dcpu) & smp_active_mask)
single_apic_ipi(dcpu, XIPIQ_OFFSET, APIC_DELMODE_FIXED);
}
cpu_send_ipiq_passive(int dcpu)
{
int r = 0;
- if ((1 << dcpu) & smp_active_mask) {
+ if (CPUMASK(dcpu) & smp_active_mask) {
r = single_apic_ipi_passive(dcpu, XIPIQ_OFFSET,
APIC_DELMODE_FIXED);
}
static pt_entry_t *pt_crashdumpmap;
static caddr_t crashdumpmap;
-extern pt_entry_t *SMPpt;
-extern uint64_t SMPptpa;
-
#define DISABLE_PSE
static pv_entry_t get_pv_entry (void);
create_pagetables(vm_paddr_t *firstaddr)
{
long i; /* must be 64 bits */
+ long nkpt_base;
+ long nkpt_phys;
/*
* We are running (mostly) V=P at this point
if (ndmpdp < 4) /* Minimum 4GB of dirmap */
ndmpdp = 4;
- nkpt = (Maxmem * sizeof(struct vm_page) + NBPDR - 1) / NBPDR;
- nkpt += (Maxmem * sizeof(struct pv_entry) + NBPDR - 1) / NBPDR;
- nkpt += ((nkpt + nkpt + 1 + NKPML4E + NKPDPE + NDMPML4E + ndmpdp) +
- 511) / 512;
- nkpt += 128;
+ /*
+ * Starting at the beginning of kvm (not KERNBASE).
+ */
+ nkpt_phys = (Maxmem * sizeof(struct vm_page) + NBPDR - 1) / NBPDR;
+ nkpt_phys += (Maxmem * sizeof(struct pv_entry) + NBPDR - 1) / NBPDR;
+ nkpt_phys += ((nkpt + nkpt + 1 + NKPML4E + NKPDPE + NDMPML4E + ndmpdp) +
+ 511) / 512;
+ nkpt_phys += 128;
+
+ /*
+ * Starting at KERNBASE - map 2G worth of page table pages.
+ * KERNBASE is offset -2G from the end of kvm.
+ */
+ nkpt_base = (NPDPEPG - KPDPI) * NPTEPG; /* typically 2 x 512 */
/*
* Allocate pages
*/
- KPTbase = allocpages(firstaddr, nkpt);
- KPTphys = allocpages(firstaddr, nkpt);
+ KPTbase = allocpages(firstaddr, nkpt_base);
+ KPTphys = allocpages(firstaddr, nkpt_phys);
KPML4phys = allocpages(firstaddr, 1);
KPDPphys = allocpages(firstaddr, NKPML4E);
+ KPDphys = allocpages(firstaddr, NKPDPE);
/*
* Calculate the page directory base for KERNBASE,
* that is where we start populating the page table pages.
* Basically this is the end - 2.
*/
- KPDphys = allocpages(firstaddr, NKPDPE);
KPDbase = KPDphys + ((NKPDPE - (NPDPEPG - KPDPI)) << PAGE_SHIFT);
DMPDPphys = allocpages(firstaddr, NDMPML4E);
* and another block is placed at KERNBASE to map the kernel binary,
* data, bss, and initial pre-allocations.
*/
- for (i = 0; i < nkpt; i++) {
+ for (i = 0; i < nkpt_base; i++) {
((pd_entry_t *)KPDbase)[i] = KPTbase + (i << PAGE_SHIFT);
((pd_entry_t *)KPDbase)[i] |= PG_RW | PG_V;
}
- for (i = 0; i < nkpt; i++) {
+ for (i = 0; i < nkpt_phys; i++) {
((pd_entry_t *)KPDphys)[i] = KPTphys + (i << PAGE_SHIFT);
((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V;
}
*/
pg = MDGLOBALDATA_BASEALLOC_PAGES;
gd = &CPU_prvspace[0].mdglobaldata;
- gd->gd_CMAP1 = &SMPpt[pg + 0];
- gd->gd_CMAP2 = &SMPpt[pg + 1];
- gd->gd_CMAP3 = &SMPpt[pg + 2];
- gd->gd_PMAP1 = &SMPpt[pg + 3];
- gd->gd_CADDR1 = CPU_prvspace[0].CPAGE1;
- gd->gd_CADDR2 = CPU_prvspace[0].CPAGE2;
- gd->gd_CADDR3 = CPU_prvspace[0].CPAGE3;
- gd->gd_PADDR1 = (pt_entry_t *)CPU_prvspace[0].PPAGE1;
cpu_invltlb();
}
va = va_start;
while (start < end) {
- if ((start / PAGE_SIZE & 15) == 0)
- kprintf("%p %p\n", (void *)va, (void *)start);
pmap_kenter_quick(va, start);
va += PAGE_SIZE;
start += PAGE_SIZE;
vm_object_t object = pmap->pm_pteobj;
struct rb_vm_page_scan_info info;
- KASSERT(pmap->pm_active == 0, ("pmap still active! %08x", pmap->pm_active));
+ KASSERT(pmap->pm_active == 0,
+ ("pmap still active! %016jx", (uintmax_t)pmap->pm_active));
#if defined(DIAGNOSTIC)
if (object->ref_count != 1)
panic("pmap_release: pteobj reference count != 1");
if (curthread->td_lwp == lp) {
pmap = vmspace_pmap(newvm);
#if defined(SMP)
- atomic_set_int(&pmap->pm_active, mycpu->gd_cpumask);
+ atomic_set_cpumask(&pmap->pm_active, mycpu->gd_cpumask);
if (pmap->pm_active & CPUMASK_LOCK)
pmap_interlock_wait(newvm);
#else
load_cr3(curthread->td_pcb->pcb_cr3);
pmap = vmspace_pmap(oldvm);
#if defined(SMP)
- atomic_clear_int(&pmap->pm_active, mycpu->gd_cpumask);
+ atomic_clear_cpumask(&pmap->pm_active, mycpu->gd_cpumask);
#else
- pmap->pm_active &= ~1;
+ pmap->pm_active &= ~(cpumask_t)1;
#endif
}
}
for (;;) {
oactive = pmap->pm_active & ~CPUMASK_LOCK;
nactive = oactive | CPUMASK_LOCK;
- if (atomic_cmpset_int(&pmap->pm_active, oactive, nactive))
+ if (atomic_cmpset_cpumask(&pmap->pm_active, oactive, nactive))
break;
crit_enter();
lwkt_process_ipiq();
pmap_inval_deinterlock(pmap_inval_info_t info, pmap_t pmap)
{
#ifdef SMP
- atomic_clear_int(&pmap->pm_active, CPUMASK_LOCK);
+ atomic_clear_cpumask(&pmap->pm_active, CPUMASK_LOCK);
#endif
}
cmpq LWP_VMSPACE(%r13),%rcx /* same vmspace? */
je 2f
1:
- movl PCPU(cpuid), %eax
- MPLOCKED btrl %eax, VM_PMAP+PM_ACTIVE(%rcx)
+ movslq PCPU(cpuid), %rax
+ MPLOCKED btrq %rax, VM_PMAP+PM_ACTIVE(%rcx)
2:
/*
movq TD_LWP(%rbx),%rcx
testq %rcx,%rcx
jz 2f
- movl PCPU(cpuid), %eax
+ movslq PCPU(cpuid), %rax
movq LWP_VMSPACE(%rcx), %rcx /* RCX = vmspace */
- MPLOCKED btrl %eax, VM_PMAP+PM_ACTIVE(%rcx)
+ MPLOCKED btrq %rax, VM_PMAP+PM_ACTIVE(%rcx)
2:
/*
* Switch to the next thread. RET into the restore function, which
* wait for it to complete before we can continue.
*/
movq LWP_VMSPACE(%rcx), %rcx /* RCX = vmspace */
- movl PCPU(cpumask), %esi
- MPLOCKED orl %esi, VM_PMAP+PM_ACTIVE(%rcx)
+ movslq PCPU(cpumask), %rsi
+ MPLOCKED orq %rsi, VM_PMAP+PM_ACTIVE(%rcx)
#ifdef SMP
- testl $CPUMASK_LOCK,VM_PMAP+PM_ACTIVE(%rcx)
- jz 1f
+ btq $CPUMASK_BIT,VM_PMAP+PM_ACTIVE(%rcx)
+ jnc 1f
pushq %rax
movq %rcx,%rdi
call pmap_interlock_wait /* pmap_interlock_wait(vm) */
mask = cpu_contention_mask;
cpu_ccfence();
- if (mask && bsfl(mask) != mycpu->gd_cpuid)
+ if (mask && BSFCPUMASK(mask) != mycpu->gd_cpuid)
pthread_yield();
}
db_global_jmpbuf_valid = FALSE;
#ifdef SMP
- db_printf("\nCPU%d restarting CPUs: 0x%08x\n",
- mycpu->gd_cpuid, stopped_cpus);
+ db_printf("\nCPU%d restarting CPUs: 0x%016jx\n",
+ mycpu->gd_cpuid, (uintmax_t)stopped_cpus);
/* Restart all the CPUs we previously stopped */
if (stopped_cpus != mycpu->gd_other_cpus) {
- db_printf("whoa, other_cpus: 0x%08x, stopped_cpus: 0x%08x\n",
- mycpu->gd_other_cpus, stopped_cpus);
+ db_printf("whoa, other_cpus: 0x%08x, stopped_cpus: 0x%016jx\n",
+ mycpu->gd_other_cpus, (uintmax_t)stopped_cpus);
panic("stop_cpus() failed");
}
restart_cpus(stopped_cpus);
ASSYM(TDPRI_INT_SUPPORT, TDPRI_INT_SUPPORT);
#ifdef SMP
ASSYM(CPUMASK_LOCK, CPUMASK_LOCK);
+ASSYM(CPUMASK_BIT, CPUMASK_BIT);
#endif
ASSYM(V_TRAP, offsetof(struct vmmeter, v_trap));
ASSYM(GD_NPXTHREAD, offsetof(struct mdglobaldata, gd_npxthread));
ASSYM(GD_FPU_LOCK, offsetof(struct mdglobaldata, gd_fpu_lock));
ASSYM(GD_SAVEFPU, offsetof(struct mdglobaldata, gd_savefpu));
-ASSYM(GD_OTHER_CPUS, offsetof(struct mdglobaldata, gd_other_cpus));
+ASSYM(GD_OTHER_CPUS, offsetof(struct mdglobaldata, mi.gd_other_cpus));
ASSYM(GD_SS_EFLAGS, offsetof(struct mdglobaldata, gd_ss_eflags));
ASSYM(GD_CMAP1, offsetof(struct mdglobaldata, gd_CMAP1));
extern pt_entry_t *KPTphys;
-volatile u_int stopped_cpus;
+volatile cpumask_t stopped_cpus;
cpumask_t smp_active_mask = 1; /* which cpus are ready for IPIs etc? */
static int boot_address;
static cpumask_t smp_startup_mask = 1; /* which cpus have been started */
/* function prototypes XXX these should go elsewhere */
void bootstrap_idle(void);
void single_cpu_ipi(int, int, int);
-void selected_cpu_ipi(u_int, int, int);
+void selected_cpu_ipi(cpumask_t, int, int);
#if 0
void ipi_handler(int);
#endif
cpumask_t ncpus_mask = 0;
for (i = 1; i <= ncpus; i++)
- ncpus_mask |= (1 << i);
+ ncpus_mask |= CPUMASK(i);
mp_finish = 1;
if (bootverbose)
kprintf("Finish MP startup\n");
/* build our map of 'other' CPUs */
- mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid);
+ mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
/*
* Let the other cpu's finish initializing and build their map
void
cpu_send_ipiq(int dcpu)
{
- if ((1 << dcpu) & smp_active_mask)
+ if (CPUMASK(dcpu) & smp_active_mask)
if (pthread_kill(ap_tids[dcpu], SIGUSR1) != 0)
panic("pthread_kill failed in cpu_send_ipiq");
#if 0
}
void
-selected_cpu_ipi(u_int target, int vector, int delivery_mode)
+selected_cpu_ipi(cpumask_t target, int vector, int delivery_mode)
{
crit_enter();
while (target) {
- int n = bsfl(target);
- target &= ~(1 << n);
+ int n = BSFCPUMASK(target);
+ target &= ~CPUMASK(n);
single_cpu_ipi(n, vector, delivery_mode);
}
crit_exit();
}
int
-stop_cpus(u_int map)
+stop_cpus(cpumask_t map)
{
map &= smp_active_mask;
crit_enter();
while (map) {
- int n = bsfl(map);
- map &= ~(1 << n);
- stopped_cpus |= 1 << n;
+ int n = BSFCPUMASK(map);
+ map &= ~CPUMASK(n);
+ stopped_cpus |= CPUMASK(n);
if (pthread_kill(ap_tids[n], SIGXCPU) != 0)
panic("stop_cpus: pthread_kill failed");
}
}
int
-restart_cpus(u_int map)
+restart_cpus(cpumask_t map)
{
map &= smp_active_mask;
crit_enter();
while (map) {
- int n = bsfl(map);
- map &= ~(1 << n);
- stopped_cpus &= ~(1 << n);
+ int n = BSFCPUMASK(map);
+ map &= ~CPUMASK(n);
+ stopped_cpus &= ~CPUMASK(n);
if (pthread_kill(ap_tids[n], SIGXCPU) != 0)
panic("restart_cpus: pthread_kill failed");
}
* interrupts physically disabled and remote cpus could deadlock
* trying to send us an IPI.
*/
- smp_startup_mask |= 1 << mycpu->gd_cpuid;
+ smp_startup_mask |= CPUMASK(mycpu->gd_cpuid);
cpu_mfence();
/*
cpu_invltlb();
/* Build our map of 'other' CPUs. */
- mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid);
+ mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
kprintf("SMP: AP CPU #%d Launched!\n", mycpu->gd_cpuid);
* nothing we've done put it there.
*/
KKASSERT(curthread->td_mpcount == 1);
- smp_active_mask |= 1 << mycpu->gd_cpuid;
+ smp_active_mask |= CPUMASK(mycpu->gd_cpuid);
mdcpu->gd_fpending = 0;
mdcpu->gd_ipending = 0;
pthread_create(&ap_tids[x], NULL, start_ap, NULL);
cpu_enable_intr();
- while((smp_startup_mask & (1 << x)) == 0) {
+ while((smp_startup_mask & CPUMASK(x)) == 0) {
cpu_lfence(); /* XXX spin until the AP has started */
DELAY(1000);
}
int gd_spending; /* software interrupt pending */
int gd_sdelayed; /* delayed software ints */
int gd_currentldt;
- int unused003;
+ int unused001;
int unused002;
- u_int unused001;
- u_int gd_other_cpus;
+ u_int unused003;
+ u_int unused004;
u_int gd_ss_eflags;
vpte_t *gd_CMAP1; /* pointer to pte for CADDR1 */
vpte_t *gd_CMAP2;
int pm_generation; /* detect pvlist deletions */
};
-#define CPUMASK_LOCK (1 << SMP_MAXCPU)
+#define CPUMASK_LOCK CPUMASK(SMP_MAXCPU)
+#define CPUMASK_BIT SMP_MAXCPU /* 1 << SMP_MAXCPU */
#define pmap_resident_count(pmap) (pmap)->pm_stats.resident_count
#endif
/* global data in apic_vector.s */
-extern volatile u_int stopped_cpus;
+extern volatile cpumask_t stopped_cpus;
extern int optcpus; /* from main() */
#if 0
-extern volatile u_int started_cpus;
+extern volatile cpumask_t started_cpus;
-extern volatile u_int checkstate_probed_cpus;
+extern volatile cpumask_t checkstate_probed_cpus;
extern void (*cpustop_restartfunc) (void);
/* functions in apic_ipl.s */
void revoke_apic_irq (int irq);
void init_secondary (void);
#endif
-int stop_cpus (u_int);
+int stop_cpus (cpumask_t);
void ap_init (void);
-int restart_cpus (u_int);
+int restart_cpus (cpumask_t);
#if 0
void forward_signal (struct proc *);
void apic_initialize (void);
void imen_dump (void);
int apic_ipi (int, int, int);
-void selected_apic_ipi (u_int, int, int);
+void selected_apic_ipi (cpumask_t, int, int);
void single_apic_ipi(int cpu, int vector, int delivery_mode);
int single_apic_ipi_passive(int cpu, int vector, int delivery_mode);
int io_apic_setup (int);
if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) {
*gd->gd_PT1pde = pmap->pm_pdirpte;
madvise(gd->gd_PT1map, SEG_SIZE, MADV_INVAL);
- atomic_set_int(&pmap->pm_cpucachemask, gd->mi.gd_cpumask);
+ atomic_set_cpumask(&pmap->pm_cpucachemask,
+ gd->mi.gd_cpumask);
}
return(gd->gd_PT1map + (va >> PAGE_SHIFT));
} else if (pmap->pm_pdir == gd->gd_PT2pdir) {
if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) {
*gd->gd_PT2pde = pmap->pm_pdirpte;
madvise(gd->gd_PT2map, SEG_SIZE, MADV_INVAL);
- atomic_set_int(&pmap->pm_cpucachemask, gd->mi.gd_cpumask);
+ atomic_set_cpumask(&pmap->pm_cpucachemask,
+ gd->mi.gd_cpumask);
}
return(gd->gd_PT2map + (va >> PAGE_SHIFT));
}
gd->gd_PT1pdir = pmap->pm_pdir;
*gd->gd_PT1pde = pmap->pm_pdirpte;
madvise(gd->gd_PT1map, SEG_SIZE, MADV_INVAL);
- atomic_set_int(&pmap->pm_cpucachemask,
- gd->mi.gd_cpumask);
+ atomic_set_cpumask(&pmap->pm_cpucachemask,
+ gd->mi.gd_cpumask);
return(gd->gd_PT1map + (va >> PAGE_SHIFT));
} else {
gd->gd_PT2pdir = pmap->pm_pdir;
*gd->gd_PT2pde = pmap->pm_pdirpte;
madvise(gd->gd_PT2map, SEG_SIZE, MADV_INVAL);
- atomic_set_int(&pmap->pm_cpucachemask,
- gd->mi.gd_cpumask);
+ atomic_set_cpumask(&pmap->pm_cpucachemask,
+ gd->mi.gd_cpumask);
return(gd->gd_PT2map + (va >> PAGE_SHIFT));
}
}
if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) {
*gd->gd_PT3pde = pmap->pm_pdirpte;
madvise(gd->gd_PT3map, SEG_SIZE, MADV_INVAL);
- atomic_set_int(&pmap->pm_cpucachemask,
- gd->mi.gd_cpumask);
+ atomic_set_cpumask(&pmap->pm_cpucachemask,
+ gd->mi.gd_cpumask);
}
} else {
gd->gd_PT3pdir = pmap->pm_pdir;
*gd->gd_PT3pde = pmap->pm_pdirpte;
madvise(gd->gd_PT3map, SEG_SIZE, MADV_INVAL);
- atomic_set_int(&pmap->pm_cpucachemask,
- gd->mi.gd_cpumask);
+ atomic_set_cpumask(&pmap->pm_cpucachemask,
+ gd->mi.gd_cpumask);
}
return(gd->gd_PT3map + (va >> PAGE_SHIFT));
}
if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) {
*gd->gd_PT1pde = pmap->pm_pdirpte;
madvise(gd->gd_PT1map, SEG_SIZE, MADV_INVAL);
- atomic_set_int(&pmap->pm_cpucachemask, gd->mi.gd_cpumask);
+ atomic_set_cpumask(&pmap->pm_cpucachemask,
+ gd->mi.gd_cpumask);
}
return(gd->gd_PT1map + (va >> PAGE_SHIFT));
}
if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) {
*gd->gd_PT2pde = pmap->pm_pdirpte;
madvise(gd->gd_PT2map, SEG_SIZE, MADV_INVAL);
- atomic_set_int(&pmap->pm_cpucachemask, gd->mi.gd_cpumask);
+ atomic_set_cpumask(&pmap->pm_cpucachemask,
+ gd->mi.gd_cpumask);
}
return(gd->gd_PT2map + (va >> PAGE_SHIFT));
}
if (curthread->td_lwp == lp) {
pmap = vmspace_pmap(newvm);
#if defined(SMP)
- atomic_set_int(&pmap->pm_active, mycpu->gd_cpumask);
+ atomic_set_cpumask(&pmap->pm_active, mycpu->gd_cpumask);
#else
pmap->pm_active |= 1;
#endif
#endif
pmap = vmspace_pmap(oldvm);
#if defined(SMP)
- atomic_clear_int(&pmap->pm_active, mycpu->gd_cpumask);
+ atomic_clear_cpumask(&pmap->pm_active, mycpu->gd_cpumask);
#else
- pmap->pm_active &= ~1;
+ pmap->pm_active &= ~(cpumask_t)1;
#endif
}
}
int gd_spending; /* software interrupt pending */
int gd_sdelayed; /* delayed software ints */
int gd_currentldt;
- int unused003;
+ int unused001;
int unused002;
- u_int unused001;
- u_int gd_other_cpus;
+ u_int unused003;
+ cpumask_t unused004;
u_int gd_ss_eflags;
};
#define pmap_resident_count(pmap) (pmap)->pm_stats.resident_count
+#define CPUMASK_LOCK CPUMASK(SMP_MAXCPU)
+#define CPUMASK_BIT SMP_MAXCPU /* 1 << SMP_MAXCPU */
+
typedef struct pmap *pmap_t;
#ifdef _KERNEL
#endif
/* global data in apic_vector.s */
-extern volatile u_int stopped_cpus;
+extern volatile cpumask_t stopped_cpus;
extern int optcpus; /* from main() */
#if 0
-extern volatile u_int started_cpus;
+extern volatile cpumask_t started_cpus;
-extern volatile u_int checkstate_probed_cpus;
+extern volatile cpumask_t checkstate_probed_cpus;
extern void (*cpustop_restartfunc) (void);
/* functions in apic_ipl.s */
void revoke_apic_irq (int irq);
void init_secondary (void);
#endif
-int stop_cpus (u_int);
+int stop_cpus (cpumask_t);
void ap_init (void);
-int restart_cpus (u_int);
+int restart_cpus (cpumask_t);
#if 0
void forward_signal (struct proc *);
void apic_initialize (void);
void imen_dump (void);
int apic_ipi (int, int, int);
-void selected_apic_ipi (u_int, int, int);
+void selected_apic_ipi (cpumask_t, int, int);
void single_apic_ipi(int cpu, int vector, int delivery_mode);
int single_apic_ipi_passive(int cpu, int vector, int delivery_mode);
int io_apic_setup (int);
if (curthread->td_lwp == lp) {
pmap = vmspace_pmap(newvm);
#if defined(SMP)
- atomic_set_int(&pmap->pm_active, 1 << mycpu->gd_cpuid);
+ atomic_set_cpumask(&pmap->pm_active, CPUMASK(mycpu->gd_cpuid));
#else
pmap->pm_active |= 1;
#endif
#endif
pmap = vmspace_pmap(oldvm);
#if defined(SMP)
- atomic_clear_int(&pmap->pm_active,
- 1 << mycpu->gd_cpuid);
+ atomic_clear_cpumask(&pmap->pm_active,
+ CPUMASK(mycpu->gd_cpuid));
#else
- pmap->pm_active &= ~1;
+ pmap->pm_active &= ~(cpumask_t)1;
#endif
}
}
mask = cpu_contention_mask;
cpu_ccfence();
- if (mask && bsfl(mask) != mycpu->gd_cpuid)
+ if (mask && BSFCPUMASK(mask) != mycpu->gd_cpuid)
pthread_yield();
}
db_global_jmpbuf_valid = FALSE;
#ifdef SMP
- db_printf("\nCPU%d restarting CPUs: 0x%08x\n",
- mycpu->gd_cpuid, stopped_cpus);
+ db_printf("\nCPU%d restarting CPUs: 0x%016jx\n",
+ mycpu->gd_cpuid, (uintmax_t)stopped_cpus);
/* Restart all the CPUs we previously stopped */
if (stopped_cpus != mycpu->gd_other_cpus) {
- db_printf("whoa, other_cpus: 0x%08x, stopped_cpus: 0x%08x\n",
- mycpu->gd_other_cpus, stopped_cpus);
+ db_printf("whoa, other_cpus: 0x%08x, stopped_cpus: 0x%016jx\n",
+ mycpu->gd_other_cpus, (uintmax_t)stopped_cpus);
panic("stop_cpus() failed");
}
restart_cpus(stopped_cpus);
extern pt_entry_t *KPTphys;
-volatile u_int stopped_cpus;
+volatile cpumask_t stopped_cpus;
cpumask_t smp_active_mask = 1; /* which cpus are ready for IPIs etc? */
static int boot_address;
static cpumask_t smp_startup_mask = 1; /* which cpus have been started */
/* function prototypes XXX these should go elsewhere */
void bootstrap_idle(void);
void single_cpu_ipi(int, int, int);
-void selected_cpu_ipi(u_int, int, int);
+void selected_cpu_ipi(cpumask_t, int, int);
#if 0
void ipi_handler(int);
#endif
cpumask_t ncpus_mask = 0;
for (i = 1; i <= ncpus; i++)
- ncpus_mask |= (1 << i);
+ ncpus_mask |= CPUMASK(i);
mp_finish = 1;
if (bootverbose)
kprintf("Finish MP startup\n");
/* build our map of 'other' CPUs */
- mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid);
+ mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
/*
* Let the other cpu's finish initializing and build their map
void
cpu_send_ipiq(int dcpu)
{
- if ((1 << dcpu) & smp_active_mask)
+ if (CPUMASK(dcpu) & smp_active_mask)
if (pthread_kill(ap_tids[dcpu], SIGUSR1) != 0)
panic("pthread_kill failed in cpu_send_ipiq");
#if 0
}
void
-selected_cpu_ipi(u_int target, int vector, int delivery_mode)
+selected_cpu_ipi(cpumask_t target, int vector, int delivery_mode)
{
crit_enter();
while (target) {
- int n = bsfl(target);
- target &= ~(1 << n);
+ int n = BSFCPUMASK(target);
+ target &= ~CPUMASK(n);
single_cpu_ipi(n, vector, delivery_mode);
}
crit_exit();
}
int
-stop_cpus(u_int map)
+stop_cpus(cpumask_t map)
{
map &= smp_active_mask;
crit_enter();
while (map) {
- int n = bsfl(map);
- map &= ~(1 << n);
- stopped_cpus |= 1 << n;
+ int n = BSFCPUMASK(map);
+ map &= ~CPUMASK(n);
+ stopped_cpus |= CPUMASK(n);
if (pthread_kill(ap_tids[n], SIGXCPU) != 0)
panic("stop_cpus: pthread_kill failed");
}
}
int
-restart_cpus(u_int map)
+restart_cpus(cpumask_t map)
{
map &= smp_active_mask;
crit_enter();
while (map) {
- int n = bsfl(map);
- map &= ~(1 << n);
- stopped_cpus &= ~(1 << n);
+ int n = BSFCPUMASK(map);
+ map &= ~CPUMASK(n);
+ stopped_cpus &= ~CPUMASK(n);
if (pthread_kill(ap_tids[n], SIGXCPU) != 0)
panic("restart_cpus: pthread_kill failed");
}
* interrupts physically disabled and remote cpus could deadlock
* trying to send us an IPI.
*/
- smp_startup_mask |= 1 << mycpu->gd_cpuid;
+ smp_startup_mask |= CPUMASK(mycpu->gd_cpuid);
cpu_mfence();
/*
cpu_invltlb();
/* Build our map of 'other' CPUs. */
- mycpu->gd_other_cpus = smp_startup_mask & ~(1 << mycpu->gd_cpuid);
+ mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
kprintf("SMP: AP CPU #%d Launched!\n", mycpu->gd_cpuid);
* nothing we've done put it there.
*/
KKASSERT(curthread->td_mpcount == 1);
- smp_active_mask |= 1 << mycpu->gd_cpuid;
+ smp_active_mask |= CPUMASK(mycpu->gd_cpuid);
mdcpu->gd_fpending = 0;
mdcpu->gd_ipending = 0;
pthread_create(&ap_tids[x], NULL, start_ap, NULL);
cpu_enable_intr();
- while((smp_startup_mask & (1 << x)) == 0) {
+ while((smp_startup_mask & CPUMASK(x)) == 0) {
cpu_lfence(); /* XXX spin until the AP has started */
DELAY(1000);
}
movq %rcx,%rbx /* RBX = curthread */
movq TD_LWP(%rcx),%rcx
- movl PCPU(cpuid), %eax
+ movslq PCPU(cpuid), %rax
movq LWP_VMSPACE(%rcx), %rcx /* RCX = vmspace */
- MPLOCKED btrl %eax, VM_PMAP+PM_ACTIVE(%rcx)
+ MPLOCKED btrq %rax, VM_PMAP+PM_ACTIVE(%rcx)
/*
* Push the LWKT switch restore function, which resumes a heavy
movq TD_LWP(%rbx),%rcx
testq %rcx,%rcx
jz 2f
- movl PCPU(cpuid), %eax
+ movslq PCPU(cpuid), %rax
movq LWP_VMSPACE(%rcx), %rcx /* RCX = vmspace */
- MPLOCKED btrl %eax, VM_PMAP+PM_ACTIVE(%rcx)
+ MPLOCKED btrq %rax, VM_PMAP+PM_ACTIVE(%rcx)
2:
/*
* Switch to the next thread. RET into the restore function, which
* pmap (remember, we do not hold the MP lock in the switch code).
*/
movq LWP_VMSPACE(%rcx), %rcx /* RCX = vmspace */
- movl PCPU(cpuid), %esi
- MPLOCKED btsl %esi, VM_PMAP+PM_ACTIVE(%rcx)
+ movslq PCPU(cpuid), %rsi
+ MPLOCKED btsq %rsi, VM_PMAP+PM_ACTIVE(%rcx)
/*
* Restore the MMU address space. If it is the same as the last
lwkt_queue gd_tdallq; /* all threads */
lwkt_queue gd_tdrunq; /* runnable threads */
__uint32_t gd_cpuid;
- cpumask_t gd_cpumask; /* mask = 1<<cpuid */
+ cpumask_t gd_cpumask; /* mask = CPUMASK(cpuid) */
cpumask_t gd_other_cpus; /* mask of 'other' cpus */
struct timeval gd_stattv;
int gd_intr_nesting_level; /* hard code, intrs, ipis */
void yield_mplock(struct thread *td);
extern int mp_lock;
-extern int cpu_contention_mask;
+extern cpumask_t cpu_contention_mask;
extern const char *mp_lock_holder_file;
extern int mp_lock_holder_line;
void
set_cpu_contention_mask(globaldata_t gd)
{
- atomic_set_int(&cpu_contention_mask, gd->gd_cpumask);
+ atomic_set_cpumask(&cpu_contention_mask, gd->gd_cpumask);
}
/*
void
clr_cpu_contention_mask(globaldata_t gd)
{
- atomic_clear_int(&cpu_contention_mask, gd->gd_cpumask);
+ atomic_clear_cpumask(&cpu_contention_mask, gd->gd_cpumask);
}
static __inline
}
static __inline int
-lwkt_send_ipiq_mask(u_int32_t mask, ipifunc1_t func, void *arg)
+lwkt_send_ipiq_mask(cpumask_t mask, ipifunc1_t func, void *arg)
{
return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg, 0));
}
static __inline int
-lwkt_send_ipiq2_mask(u_int32_t mask, ipifunc2_t func, void *arg1, int arg2)
+lwkt_send_ipiq2_mask(cpumask_t mask, ipifunc2_t func, void *arg1, int arg2)
{
return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg1, arg2));
}
vm_offset_t kmem_alloc_wait (vm_map_t, vm_size_t);
void kmem_free (vm_map_t, vm_offset_t, vm_size_t);
void kmem_free_wakeup (vm_map_t, vm_offset_t, vm_size_t);
-void kmem_init (vm_offset_t, vm_offset_t);
+void kmem_init (void);
void kmem_suballoc (vm_map_t, vm_map_t, vm_offset_t *, vm_offset_t *, vm_size_t);
void munmapfd (struct proc *, int);
int swaponvp (struct thread *, struct vnode *, u_quad_t);
* Initializes resident memory structures. From here on, all physical
* memory is accounted for, and we use only virtual addresses.
*/
-
vm_set_page_size();
- virtual_start = vm_page_startup(virtual_start);
+ vm_page_startup();
+
/*
* Initialize other VM packages
*/
vm_object_init();
vm_map_startup();
- kmem_init(virtual_start, virtual_end);
+ kmem_init();
pmap_init();
}
}
/*
- * Create the kernel_map and insert mappings to cover areas already
- * allocated or reserved thus far. That is, the area (KvaStart,start)
- * and (end,KvaEnd) must be marked as allocated.
+ * Create the kernel_ma for (KvaStart,KvaEnd) and insert mappings to
+ * cover areas already allocated or reserved thus far.
*
- * virtual2_start/end is a cutout Between KvaStart and start,
- * for x86_64 due to the location of KERNBASE (at -2G).
- *
- * We could use a min_offset of 0 instead of KvaStart, but since the
- * min_offset is not used for any calculations other then a bounds check
- * it does not effect readability. KvaStart is more appropriate.
+ * The areas (virtual_start, virtual_end) and (virtual2_start, virtual2_end)
+ * are available so the cutouts are the areas around these ranges between
+ * KvaStart and KvaEnd.
*
* Depend on the zalloc bootstrap cache to get our vm_map_entry_t.
* Called from the low level boot code only.
*/
void
-kmem_init(vm_offset_t start, vm_offset_t end)
+kmem_init(void)
{
vm_offset_t addr;
vm_map_t m;
}
addr = virtual2_end;
}
- if (addr < start) {
+ if (addr < virtual_start) {
vm_map_insert(m, &count, NULL, (vm_offset_t) 0,
- addr, start,
+ addr, virtual_start,
VM_MAPTYPE_NORMAL,
VM_PROT_ALL, VM_PROT_ALL,
0);
}
- addr = end;
+ addr = virtual_end;
if (addr < KvaEnd) {
vm_map_insert(m, &count, NULL, (vm_offset_t) 0,
addr, KvaEnd,
*
* Initializes the resident memory module.
*
- * Allocates memory for the page cells, and for the object/offset-to-page
- * hash table headers. Each page cell is initialized and placed on the
- * free list.
- *
- * starta/enda represents the range of physical memory addresses available
- * for use (skipping memory already used by the kernel), subject to
- * phys_avail[]. Note that phys_avail[] has already mapped out memory
- * already in use by the kernel.
+ * Preallocates memory for critical VM structures and arrays prior to
+ * kernel_map becoming available.
+ *
+ * Memory is allocated from (virtual2_start, virtual2_end) if available,
+ * otherwise memory is allocated from (virtual_start, virtual_end).
+ *
+ * On x86-64 (virtual_start, virtual_end) is only 2GB and may not be
+ * large enough to hold vm_page_array & other structures for machines with
+ * large amounts of ram, so we want to use virtual2* when available.
*/
-vm_offset_t
-vm_page_startup(vm_offset_t vaddr)
+void
+vm_page_startup(void)
{
+ vm_offset_t vaddr = virtual2_start ? virtual2_start : virtual_start;
vm_offset_t mapped;
vm_size_t npages;
vm_paddr_t page_range;
pa += PAGE_SIZE;
}
}
- return (vaddr);
+ if (virtual2_start)
+ virtual2_start = vaddr;
+ else
+ virtual_start = vaddr;
}
/*
vm_page_t vm_page_lookup (struct vm_object *, vm_pindex_t);
void vm_page_remove (vm_page_t);
void vm_page_rename (vm_page_t, struct vm_object *, vm_pindex_t);
-vm_offset_t vm_page_startup (vm_offset_t);
+void vm_page_startup (void);
vm_page_t vm_add_new_page (vm_paddr_t pa);
void vm_page_unmanage (vm_page_t);
void vm_page_unwire (vm_page_t, int);
projects / dragonfly.git / commitdiff