From c07315c47f12ac7242a5409623dcfbb82f8168ab Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Thu, 3 Jul 2014 20:07:27 -0700 Subject: [PATCH] kernel - Refactor cpumask_t to extend cpus past 64, part 1/2 * 64-bit systems only. 32-bit builds use the macros but cannot be expanded past 32 cpus. * Change cpumask_t from __uint64_t to a structure. This commit implements one 64-bit sub-element (the next one will implement four for 256 cpus). * Create a CPUMASK_*() macro API for non-atomic and atomic cpumask manipulation. These macros generally take lvalues as arguments, allowing for a fairly optimal implementation. * Change all C code operating on cpumask's to use the newly created CPUMASK_*() macro API. * Compile-test 32 and 64-bit. Run-test 64-bit. * Adjust sbin/usched, usr.sbin/powerd. usched currently needs more work. --- sbin/usched/usched.c | 66 ++++--- sys/cpu/i386/include/types.h | 44 ++++- sys/cpu/i386/misc/lwbuf.c | 4 +- sys/cpu/x86_64/include/types.h | 60 +++++- sys/dev/acpica/acpi_cpu.c | 2 +- sys/dev/powermng/clockmod/clockmod.c | 14 +- sys/kern/init_main.c | 6 +- sys/kern/kern_fork.c | 2 +- sys/kern/kern_proc.c | 2 +- sys/kern/kern_shutdown.c | 2 +- sys/kern/kern_synch.c | 23 ++- sys/kern/kern_usched.c | 28 +-- sys/kern/lwkt_ipiq.c | 77 ++++---- sys/kern/subr_cpu_topology.c | 34 ++-- sys/kern/sys_vmm.c | 8 +- sys/kern/usched_bsd4.c | 197 +++++++++++-------- sys/kern/usched_dfly.c | 120 ++++++----- sys/kern/usched_dummy.c | 46 +++-- sys/net/netisr.c | 13 +- sys/netinet/ip_flow.c | 9 +- sys/netinet/ip_input.c | 20 +- sys/platform/pc32/apic/lapic.c | 2 +- sys/platform/pc32/apic/lapic.h | 2 +- sys/platform/pc32/i386/mp_machdep.c | 96 +++++---- sys/platform/pc32/i386/mptable.c | 6 +- sys/platform/pc32/i386/pmap.c | 13 +- sys/platform/pc32/i386/vm_machdep.c | 6 +- sys/platform/pc64/apic/lapic.c | 4 +- sys/platform/pc64/apic/lapic.h | 2 +- sys/platform/pc64/isa/clock.c | 2 +- sys/platform/pc64/vmm/vmx.c | 80 ++++---- sys/platform/pc64/x86_64/db_interface.c | 12 +- sys/platform/pc64/x86_64/mp_machdep.c | 129 +++++++----- sys/platform/pc64/x86_64/mptable.c | 16 +- sys/platform/pc64/x86_64/pmap.c | 22 ++- sys/platform/vkernel/i386/exception.c | 2 +- sys/platform/vkernel/i386/mp.c | 41 ++-- sys/platform/vkernel/platform/pmap.c | 53 ++--- sys/platform/vkernel64/platform/pmap.c | 6 +- sys/platform/vkernel64/platform/pmap_inval.c | 3 +- sys/platform/vkernel64/x86_64/exception.c | 2 +- sys/platform/vkernel64/x86_64/mp.c | 38 ++-- sys/sys/cpu_topology.h | 4 +- sys/sys/globaldata.h | 2 +- sys/sys/param.h | 3 + usr.sbin/powerd/powerd.c | 3 +- 46 files changed, 812 insertions(+), 514 deletions(-) diff --git a/sbin/usched/usched.c b/sbin/usched/usched.c index a2ca74ee30..0bd54a702b 100644 --- a/sbin/usched/usched.c +++ b/sbin/usched/usched.c @@ -33,6 +33,7 @@ * SUCH DAMAGE. */ +#define _KERNEL_STRUCTURES #include #include #include @@ -52,8 +53,10 @@ main(int ac, char **av) char *sched = NULL; char *cpustr = NULL; char *sched_cpustr = NULL; - cpumask_t cpumask = 0; - int cpuid = -1; + cpumask_t cpumask; + int cpuid; + + CPUMASK_ASSZERO(cpumask); while ((ch = getopt(ac, av, "d")) != -1) { switch (ch) { @@ -81,8 +84,19 @@ main(int ac, char **av) usage(); /* NOTREACHED */ } - if (cpustr != NULL) - cpumask = strtoul(cpustr, NULL, 0); + + /* + * XXX needs expanded support for > 64 cpus + */ + if (cpustr != NULL) { + unsigned long v; + + v = strtoul(cpustr, NULL, 0); + for (cpuid = 0; cpuid < (int)sizeof(v) * 8; ++cpuid) { + if (v & (1LU << cpuid)) + CPUMASK_ORBIT(cpumask, cpuid); + } + } if (strlen(sched) != 0) { if (DebugOpt) @@ -93,30 +107,34 @@ main(int ac, char **av) exit(1); } } - if (cpumask != 0) { - while ((cpumask & 1) == 0) { - cpuid++; - cpumask >>= 1; + if (CPUMASK_TESTNZERO(cpumask)) { + for (cpuid = 0; cpuid < (int)sizeof(cpumask) * 8; ++cpuid) { + if (CPUMASK_TESTBIT(cpumask, cpuid)) + break; } - cpuid++; - cpumask >>= 1; - if (DebugOpt) - fprintf(stderr, "DEBUG: USCHED_SET_CPU: cpuid: %d\n", cpuid); - res = usched_set(getpid(), USCHED_SET_CPU, &cpuid, sizeof(int)); + if (DebugOpt) { + fprintf(stderr, "DEBUG: USCHED_SET_CPU: cpuid: %d\n", + cpuid); + } + res = usched_set(getpid(), USCHED_SET_CPU, + &cpuid, sizeof(int)); if (res != 0) { perror("usched_set(,USCHED_SET_CPU,,)"); exit(1); } - while (cpumask != 0) { - while ((cpumask & 1) == 0) { - cpuid++; - cpumask >>= 1; + CPUMASK_NANDBIT(cpumask, cpuid); + while (CPUMASK_TESTNZERO(cpumask)) { + ++cpuid; + if (CPUMASK_TESTBIT(cpumask, cpuid) == 0) + continue; + CPUMASK_NANDBIT(cpumask, cpuid); + if (DebugOpt) { + fprintf(stderr, + "DEBUG: USCHED_ADD_CPU: cpuid: %d\n", + cpuid); } - cpuid++; - cpumask >>= 1; - if (DebugOpt) - fprintf(stderr, "DEBUG: USCHED_ADD_CPU: cpuid: %d\n", cpuid); - res = usched_set(getpid(), USCHED_ADD_CPU, &cpuid, sizeof(int)); + res = usched_set(getpid(), USCHED_ADD_CPU, + &cpuid, sizeof(int)); if (res != 0) { perror("usched_set(,USCHED_ADD_CPU,,)"); exit(1); @@ -131,6 +149,8 @@ static void usage(void) { - fprintf(stderr, "usage: usched [-d] {scheduler[:cpumask] | :cpumask} program [argument ...]\n"); + fprintf(stderr, + "usage: usched [-d] {scheduler[:cpumask] | :cpumask} " + "program [argument ...]\n"); exit(1); } diff --git a/sys/cpu/i386/include/types.h b/sys/cpu/i386/include/types.h index 226d750824..12408522fa 100644 --- a/sys/cpu/i386/include/types.h +++ b/sys/cpu/i386/include/types.h @@ -65,9 +65,51 @@ typedef __uint32_t cpumask_t; /* mask representing a set of cpus */ typedef __uint32_t cpulock_t; /* count and exclusive lock */ #if defined(_KERNEL) || defined(_KERNEL_STRUCTURES) -#define CPUMASK(cpu) (1U << (cpu)) + +/* + * NOTE! CPUMASK_TESTMASK() - caller should only test == 0 or != 0 + */ + +#define CPUMASK_INITIALIZER_ALLONES (cpumask_t)-1 +#define CPUMASK_INITIALIZER_ONLYONE (cpumask_t)1 + +#define CPUMASK_SIMPLE(cpu) (1U << (cpu)) #define BSRCPUMASK(mask) bsrl(mask) #define BSFCPUMASK(mask) bsfl(mask) + +#define CPUMASK_CMPMASKEQ(mask1, mask2) ((mask1) == (mask2)) +#define CPUMASK_CMPMASKNEQ(mask1, mask2) ((mask1) != (mask2)) +#define CPUMASK_ISUP(mask) ((mask) == 1) + +#define CPUMASK_TESTZERO(mask) ((mask) == 0) +#define CPUMASK_TESTNZERO(mask) ((mask) != 0) +#define CPUMASK_TESTBIT(mask, i) ((mask) & CPUMASK_SIMPLE(i)) +#define CPUMASK_TESTMASK(mask1, mask2) ((mask1) & (mask2)) +#define CPUMASK_LOWMASK(mask) (mask) + +#define CPUMASK_ORBIT(mask, i) mask |= CPUMASK_SIMPLE(i) +#define CPUMASK_ANDBIT(mask, i) mask &= CPUMASK_SIMPLE(i) +#define CPUMASK_NANDBIT(mask, i) mask &= ~CPUMASK_SIMPLE(i) + +#define CPUMASK_ASSZERO(mask) mask = 0 +#define CPUMASK_ASSALLONES(mask) mask = (cpumask_t)-1 +#define CPUMASK_ASSBIT(mask, i) mask = CPUMASK_SIMPLE(i) +#define CPUMASK_ASSBMASK(mask, i) mask = (CPUMASK_SIMPLE(i) - 1) +#define CPUMASK_ASSNBMASK(mask, i) mask = ~(CPUMASK_SIMPLE(i) - 1) + +#define CPUMASK_ANDMASK(mask, m1) mask &= (m1) +#define CPUMASK_NANDMASK(mask, m1) mask &= ~(m1) +#define CPUMASK_ORMASK(mask, m1) mask |= (m1) + +#define ATOMIC_CPUMASK_ORBIT(mask, i) \ + atomic_set_cpumask(&(mask), CPUMASK_SIMPLE(i)) +#define ATOMIC_CPUMASK_NANDBIT(mask, i) \ + atomic_clear_cpumask(&(mask), CPUMASK_SIMPLE(i)) +#define ATOMIC_CPUMASK_ORMASK(mask, m1) \ + atomic_set_cpumask(&(mask), m1) +#define ATOMIC_CPUMASK_NANDMASK(mask, m1) \ + atomic_clear_cpumask(&(mask), m1) + #endif #define CPULOCK_EXCLBIT 0 /* exclusive lock bit number */ diff --git a/sys/cpu/i386/misc/lwbuf.c b/sys/cpu/i386/misc/lwbuf.c index c0c1312cab..904ea96ee0 100644 --- a/sys/cpu/i386/misc/lwbuf.c +++ b/sys/cpu/i386/misc/lwbuf.c @@ -159,7 +159,7 @@ _lwbuf_kva(struct lwbuf *lwb, struct mdglobaldata *gd) { pmap_kenter_sync_quick(lwb->kva); - atomic_set_int(&lwb->cpumask, gd->mi.gd_cpumask); + ATOMIC_CPUMASK_ORBIT(lwb->cpumask, gd->mi.gd_cpuid); return (lwb->kva); } @@ -169,7 +169,7 @@ lwbuf_kva(struct lwbuf *lwb) { struct mdglobaldata *gd = mdcpu; - if (lwb->cpumask & gd->mi.gd_cpumask) + if (CPUMASK_TESTBIT(lwb->cpumask, gd->mi.gd_cpuid)) return (lwb->kva); return (_lwbuf_kva(lwb, gd)); diff --git a/sys/cpu/x86_64/include/types.h b/sys/cpu/x86_64/include/types.h index c8128beeab..15cfb86e72 100644 --- a/sys/cpu/x86_64/include/types.h +++ b/sys/cpu/x86_64/include/types.h @@ -70,13 +70,65 @@ typedef __uint64_t pml4_entry_t; typedef __uint64_t pdp_entry_t; typedef __uint64_t pd_entry_t; typedef __uint64_t pt_entry_t; -typedef __uint64_t cpumask_t; /* mask representing a set of cpus */ typedef __uint32_t cpulock_t; /* count and exclusive lock */ +/* + * cpumask_t - a mask representing a set of cpus and supporting routines. + * + * WARNING! It is recommended that this mask NOT be made variably-sized + * because it affects a huge number of system structures. However, + * kernel code (non-module) can be optimized to not operate on the + * whole mask. + */ + +#define CPUMASK_ELEMENTS 1 /* tested by assembly for #error */ + +typedef struct { + __uint64_t m0; +} cpumask_t; + #if defined(_KERNEL) || defined(_KERNEL_STRUCTURES) -#define CPUMASK(cpu) ((cpumask_t)1 << (cpu)) -#define BSRCPUMASK(mask) bsrq(mask) -#define BSFCPUMASK(mask) bsfq(mask) + +#define CPUMASK_INITIALIZER_ALLONES { .m0 = (__uint64_t)-1 } +#define CPUMASK_INITIALIZER_ONLYONE { .m0 = 1 } + +#define CPUMASK_SIMPLE(cpu) ((__uint64_t)1 << (cpu)) +#define BSRCPUMASK(val) bsrq((val).m0) +#define BSFCPUMASK(val) bsfq((val).m0) + +#define CPUMASK_CMPMASKEQ(val1, val2) ((val1).m0 == (val2).m0) +#define CPUMASK_CMPMASKNEQ(val1, val2) ((val1).m0 != (val2).m0) +#define CPUMASK_ISUP(val) ((val).m0 == 1) + +#define CPUMASK_TESTZERO(val) ((val).m0 == 0) +#define CPUMASK_TESTNZERO(val) ((val).m0 != 0) +#define CPUMASK_TESTBIT(val, i) ((val).m0 & CPUMASK_SIMPLE(i)) +#define CPUMASK_TESTMASK(val1, val2) ((val1).m0 & (val2.m0)) +#define CPUMASK_LOWMASK(val) (val).m0 + +#define CPUMASK_ORBIT(mask, i) (mask).m0 |= CPUMASK_SIMPLE(i) +#define CPUMASK_ANDBIT(mask, i) (mask).m0 &= CPUMASK_SIMPLE(i) +#define CPUMASK_NANDBIT(mask, i) (mask).m0 &= ~CPUMASK_SIMPLE(i) + +#define CPUMASK_ASSZERO(mask) (mask).m0 = 0 +#define CPUMASK_ASSALLONES(mask) (mask).m0 = (__uint64_t)-1 +#define CPUMASK_ASSBIT(mask, i) (mask).m0 = CPUMASK_SIMPLE(i) +#define CPUMASK_ASSBMASK(mask, i) (mask).m0 = (CPUMASK_SIMPLE(i) - 1) +#define CPUMASK_ASSNBMASK(mask, i) (mask).m0 = ~(CPUMASK_SIMPLE(i) - 1) + +#define CPUMASK_ANDMASK(mask, val) (mask).m0 &= (val).m0 +#define CPUMASK_NANDMASK(mask, val) (mask).m0 &= ~(val).m0 +#define CPUMASK_ORMASK(mask, val) (mask).m0 |= (val).m0 + +#define ATOMIC_CPUMASK_ORBIT(mask, i) \ + atomic_set_cpumask(&(mask).m0, CPUMASK_SIMPLE(i)) +#define ATOMIC_CPUMASK_NANDBIT(mask, i) \ + atomic_clear_cpumask(&(mask).m0, CPUMASK_SIMPLE(i)) +#define ATOMIC_CPUMASK_ORMASK(mask, val) \ + atomic_set_cpumask(&(mask).m0, val.m0) +#define ATOMIC_CPUMASK_NANDMASK(mask, val) \ + atomic_clear_cpumask(&(mask).m0, val.m0) + #endif #define CPULOCK_EXCLBIT 0 /* exclusive lock bit number */ diff --git a/sys/dev/acpica/acpi_cpu.c b/sys/dev/acpica/acpi_cpu.c index 36d9e9b578..903b0ecd3c 100644 --- a/sys/dev/acpica/acpi_cpu.c +++ b/sys/dev/acpica/acpi_cpu.c @@ -303,7 +303,7 @@ acpi_cpu_get_id(uint32_t idx, uint32_t *acpi_id, uint32_t *cpu_id) KASSERT(acpi_id != NULL, ("Null acpi_id")); KASSERT(cpu_id != NULL, ("Null cpu_id")); for (i = 0; i < ncpus; i++) { - if ((smp_active_mask & CPUMASK(i)) == 0) + if (CPUMASK_TESTBIT(smp_active_mask, i) == 0) continue; md = (struct mdglobaldata *)globaldata_find(i); KASSERT(md != NULL, ("no pcpu data for %d", i)); diff --git a/sys/dev/powermng/clockmod/clockmod.c b/sys/dev/powermng/clockmod/clockmod.c index cf70625eb0..ffe1c8a056 100644 --- a/sys/dev/powermng/clockmod/clockmod.c +++ b/sys/dev/powermng/clockmod/clockmod.c @@ -184,12 +184,14 @@ clockmod_dom_attach(struct clockmod_softc *sc) { struct clockmod_softc *sc1; struct clockmod_dom *dom; - cpumask_t mask, found_mask = 0; + cpumask_t mask, found_mask; int error = 0; + CPUMASK_ASSZERO(found_mask); + mask = get_cpumask_from_level(sc->sc_cpuid, CORE_LEVEL); - if (mask == 0) - mask = CPUMASK(sc->sc_cpuid); + if (CPUMASK_TESTZERO(mask)) + CPUMASK_ASSBIT(mask, sc->sc_cpuid); lwkt_serialize_enter(&clockmod_dom_slize); @@ -206,9 +208,9 @@ clockmod_dom_attach(struct clockmod_softc *sc) TAILQ_INSERT_TAIL(&dom->dom_list, sc, sc_link); TAILQ_FOREACH(sc1, &dom->dom_list, sc_link) - found_mask |= CPUMASK(sc1->sc_cpuid); + CPUMASK_ORBIT(found_mask, sc1->sc_cpuid); - if (found_mask == dom->dom_cpumask) { + if (CPUMASK_CMPMASKEQ(found_mask, dom->dom_cpumask)) { /* All cpus in this domain is found */ dom->dom_flags |= CLOCKMOD_DOM_FLAG_ACTIVE; } @@ -251,7 +253,7 @@ clockmod_dom_find(cpumask_t mask) struct clockmod_dom *dom; TAILQ_FOREACH(dom, &clockmod_dom_list, dom_link) { - if (dom->dom_cpumask == mask) + if (CPUMASK_CMPMASKEQ(dom->dom_cpumask, mask)) return dom; } return NULL; diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index 8ab5170bf9..4ee1176587 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -171,7 +171,7 @@ mi_proc0init(struct globaldata *gd, struct user *proc0paddr) lwp0.lwp_thread = &thread0; lwp0.lwp_proc = &proc0; proc0.p_usched = usched_init(); - lwp0.lwp_cpumask = (cpumask_t)-1; + CPUMASK_ASSALLONES(lwp0.lwp_cpumask); lwkt_token_init(&lwp0.lwp_token, "lwp_token"); spin_init(&lwp0.lwp_spin); varsymset_init(&proc0.p_varsymset, NULL); @@ -724,10 +724,10 @@ mi_gdinit(struct globaldata *gd, int cpuid) TAILQ_INIT(&gd->gd_systimerq); gd->gd_sysid_alloc = cpuid; /* prime low bits for cpu lookup */ gd->gd_cpuid = cpuid; - gd->gd_cpumask = CPUMASK(cpuid); + CPUMASK_ASSBIT(gd->gd_cpumask, cpuid); lwkt_gdinit(gd); vm_map_entry_reserve_cpu_init(gd); sleep_gdinit(gd); - atomic_set_cpumask(&usched_global_cpumask, CPUMASK(cpuid)); + ATOMIC_CPUMASK_ORBIT(usched_global_cpumask, cpuid); } diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index 0d13637a0a..8065ccf496 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -650,7 +650,7 @@ lwp_fork(struct lwp *origlp, struct proc *destproc, int flags) lp->lwp_cpbase = gd->gd_schedclock.time - gd->gd_schedclock.periodic; destproc->p_usched->heuristic_forking(origlp, lp); crit_exit(); - lp->lwp_cpumask &= usched_mastermask; + CPUMASK_ANDMASK(lp->lwp_cpumask, usched_mastermask); lwkt_token_init(&lp->lwp_token, "lwp_token"); spin_init(&lp->lwp_spin); diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c index f4ee251fdc..ea9457ca85 100644 --- a/sys/kern/kern_proc.c +++ b/sys/kern/kern_proc.c @@ -1512,7 +1512,7 @@ sysctl_kern_proc(SYSCTL_HANDLER_ARGS) int nid; nid = (origcpu + n) % ncpus; - if ((smp_active_mask & CPUMASK(nid)) == 0) + if (CPUMASK_TESTBIT(smp_active_mask, nid) == 0) continue; rgd = globaldata_find(nid); lwkt_setcpu_self(rgd); diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c index 6e1d8134cd..e9eb01a588 100644 --- a/sys/kern/kern_shutdown.c +++ b/sys/kern/kern_shutdown.c @@ -261,7 +261,7 @@ boot(int howto) * We really want to shutdown on the BSP. Subsystems such as ACPI * can't power-down the box otherwise. */ - if (smp_active_mask > 1) { + if (!CPUMASK_ISUP(smp_active_mask)) { kprintf("boot() called on cpu#%d\n", mycpu->gd_cpuid); } if (panicstr == NULL && mycpu->gd_cpuid != 0) { diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 570efdac98..59aa363db3 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -366,15 +366,15 @@ _tsleep_interlock(globaldata_t gd, const volatile void *ident, int flags) id = LOOKUP(td->td_wchan); TAILQ_REMOVE(&gd->gd_tsleep_hash[id], td, td_sleepq); if (TAILQ_FIRST(&gd->gd_tsleep_hash[id]) == NULL) { - atomic_clear_cpumask(&slpque_cpumasks[id], - gd->gd_cpumask); + ATOMIC_CPUMASK_NANDBIT(slpque_cpumasks[id], + gd->gd_cpuid); } } else { td->td_flags |= TDF_TSLEEPQ; } id = LOOKUP(ident); TAILQ_INSERT_TAIL(&gd->gd_tsleep_hash[id], td, td_sleepq); - atomic_set_cpumask(&slpque_cpumasks[id], gd->gd_cpumask); + ATOMIC_CPUMASK_ORBIT(slpque_cpumasks[id], gd->gd_cpuid); td->td_wchan = ident; td->td_wdomain = flags & PDOMAIN_MASK; crit_exit_quick(td); @@ -402,8 +402,10 @@ _tsleep_remove(thread_t td) td->td_flags &= ~TDF_TSLEEPQ; id = LOOKUP(td->td_wchan); TAILQ_REMOVE(&gd->gd_tsleep_hash[id], td, td_sleepq); - if (TAILQ_FIRST(&gd->gd_tsleep_hash[id]) == NULL) - atomic_clear_cpumask(&slpque_cpumasks[id], gd->gd_cpumask); + if (TAILQ_FIRST(&gd->gd_tsleep_hash[id]) == NULL) { + ATOMIC_CPUMASK_NANDBIT(slpque_cpumasks[id], + gd->gd_cpuid); + } td->td_wchan = NULL; td->td_wdomain = 0; } @@ -921,10 +923,13 @@ restart: * should be ok since we are passing idents in the IPI rather then * thread pointers. */ - if ((domain & PWAKEUP_MYCPU) == 0 && - (mask = slpque_cpumasks[id] & gd->gd_other_cpus) != 0) { - lwkt_send_ipiq2_mask(mask, _wakeup, ident, - domain | PWAKEUP_MYCPU); + if ((domain & PWAKEUP_MYCPU) == 0) { + mask = slpque_cpumasks[id]; + CPUMASK_ANDMASK(mask, gd->gd_other_cpus); + if (CPUMASK_TESTNZERO(mask)) { + lwkt_send_ipiq2_mask(mask, _wakeup, ident, + domain | PWAKEUP_MYCPU); + } } done: logtsleep1(wakeup_end); diff --git a/sys/kern/kern_usched.c b/sys/kern/kern_usched.c index 0817afbc8c..f361f9d959 100644 --- a/sys/kern/kern_usched.c +++ b/sys/kern/kern_usched.c @@ -48,7 +48,7 @@ static TAILQ_HEAD(, usched) usched_list = TAILQ_HEAD_INITIALIZER(usched_list); -cpumask_t usched_mastermask = -1; +cpumask_t usched_mastermask = CPUMASK_INITIALIZER_ALLONES; /* * Called from very low level boot code, i386/i386/machdep.c/init386(). @@ -243,11 +243,11 @@ sys_usched_set(struct usched_set_args *uap) error = EFBIG; break; } - if ((smp_active_mask & CPUMASK(cpuid)) == 0) { + if (CPUMASK_TESTBIT(smp_active_mask, cpuid) == 0) { error = EINVAL; break; } - lp->lwp_cpumask = CPUMASK(cpuid); + CPUMASK_ASSBIT(lp->lwp_cpumask, cpuid); if (cpuid != mycpu->gd_cpuid) { lwkt_migratecpu(cpuid); p->p_usched->changedcpu(lp); @@ -275,11 +275,11 @@ sys_usched_set(struct usched_set_args *uap) error = EFBIG; break; } - if (!(smp_active_mask & CPUMASK(cpuid))) { + if (CPUMASK_TESTBIT(smp_active_mask, cpuid) == 0) { error = EINVAL; break; } - lp->lwp_cpumask |= CPUMASK(cpuid); + CPUMASK_ORBIT(lp->lwp_cpumask, cpuid); break; case USCHED_DEL_CPU: /* USCHED_DEL_CPU doesn't require special privileges. */ @@ -295,14 +295,18 @@ sys_usched_set(struct usched_set_args *uap) break; } lp = curthread->td_lwp; - mask = lp->lwp_cpumask & smp_active_mask & ~CPUMASK(cpuid); - if (mask == 0) + mask = lp->lwp_cpumask; + CPUMASK_ANDMASK(mask, smp_active_mask); + CPUMASK_NANDBIT(mask, cpuid); + if (CPUMASK_TESTZERO(mask)) { error = EPERM; - else { - lp->lwp_cpumask &= ~CPUMASK(cpuid); - if ((lp->lwp_cpumask & mycpu->gd_cpumask) == 0) { - cpuid = BSFCPUMASK(lp->lwp_cpumask & - smp_active_mask); + } else { + CPUMASK_NANDBIT(lp->lwp_cpumask, cpuid); + if (CPUMASK_TESTMASK(lp->lwp_cpumask, + mycpu->gd_cpumask) == 0) { + mask = lp->lwp_cpumask; + CPUMASK_ANDMASK(mask, smp_active_mask); + cpuid = BSFCPUMASK(mask); lwkt_migratecpu(cpuid); p->p_usched->changedcpu(lp); } diff --git a/sys/kern/lwkt_ipiq.c b/sys/kern/lwkt_ipiq.c index d426b28855..f41ef01036 100644 --- a/sys/kern/lwkt_ipiq.c +++ b/sys/kern/lwkt_ipiq.c @@ -255,7 +255,7 @@ lwkt_send_ipiq3(globaldata_t target, ipifunc3_t func, void *arg1, int arg2) ip->ip_info[windex].arg2 = arg2; cpu_sfence(); ++ip->ip_windex; - atomic_set_cpumask(&target->gd_ipimask, gd->gd_cpumask); + ATOMIC_CPUMASK_ORBIT(target->gd_ipimask, gd->gd_cpuid); /* * signal the target cpu that there is work pending. @@ -352,7 +352,7 @@ lwkt_send_ipiq3_passive(globaldata_t target, ipifunc3_t func, ip->ip_info[windex].arg2 = arg2; cpu_sfence(); ++ip->ip_windex; - atomic_set_cpumask(&target->gd_ipimask, gd->gd_cpumask); + ATOMIC_CPUMASK_ORBIT(target->gd_ipimask, gd->gd_cpuid); --gd->gd_intr_nesting_level; /* @@ -403,7 +403,7 @@ lwkt_send_ipiq3_nowait(globaldata_t target, ipifunc3_t func, ip->ip_info[windex].arg2 = arg2; cpu_sfence(); ++ip->ip_windex; - atomic_set_cpumask(&target->gd_ipimask, gd->gd_cpumask); + ATOMIC_CPUMASK_ORBIT(target->gd_ipimask, gd->gd_cpuid); /* * This isn't a passive IPI, we still have to signal the target cpu. @@ -440,11 +440,11 @@ lwkt_send_ipiq3_mask(cpumask_t mask, ipifunc3_t func, void *arg1, int arg2) int cpuid; int count = 0; - mask &= ~stopped_cpus; - while (mask) { + CPUMASK_NANDMASK(mask, stopped_cpus); + while (CPUMASK_TESTNZERO(mask)) { cpuid = BSFCPUMASK(mask); lwkt_send_ipiq3(globaldata_find(cpuid), func, arg1, arg2); - mask &= ~CPUMASK(cpuid); + CPUMASK_NANDBIT(mask, cpuid); ++count; } return(count); @@ -560,8 +560,8 @@ lwkt_process_ipiq(void) again: cpu_lfence(); mask = gd->gd_ipimask; - atomic_clear_cpumask(&gd->gd_ipimask, mask); - while (mask) { + ATOMIC_CPUMASK_NANDMASK(gd->gd_ipimask, mask); + while (CPUMASK_TESTNZERO(mask)) { n = BSFCPUMASK(mask); if (n != gd->gd_cpuid) { sgd = globaldata_find(n); @@ -571,7 +571,7 @@ again: ; } } - mask &= ~CPUMASK(n); + CPUMASK_NANDBIT(mask, n); } /* @@ -589,11 +589,11 @@ again: * Interlock to allow more IPI interrupts. Recheck ipimask after * releasing gd_npoll. */ - if (gd->gd_ipimask) + if (CPUMASK_TESTNZERO(gd->gd_ipimask)) goto again; atomic_poll_release_int(&gd->gd_npoll); cpu_mfence(); - if (gd->gd_ipimask) + if (CPUMASK_TESTNZERO(gd->gd_ipimask)) goto again; --gd->gd_processing_ipiq; } @@ -610,8 +610,8 @@ lwkt_process_ipiq_frame(struct intrframe *frame) again: cpu_lfence(); mask = gd->gd_ipimask; - atomic_clear_cpumask(&gd->gd_ipimask, mask); - while (mask) { + ATOMIC_CPUMASK_NANDMASK(gd->gd_ipimask, mask); + while (CPUMASK_TESTNZERO(mask)) { n = BSFCPUMASK(mask); if (n != gd->gd_cpuid) { sgd = globaldata_find(n); @@ -621,7 +621,7 @@ again: ; } } - mask &= ~CPUMASK(n); + CPUMASK_NANDBIT(mask, n); } if (gd->gd_cpusyncq.ip_rindex != gd->gd_cpusyncq.ip_windex) { if (lwkt_process_ipiq_core(gd, &gd->gd_cpusyncq, frame)) { @@ -635,11 +635,11 @@ again: * Interlock to allow more IPI interrupts. Recheck ipimask after * releasing gd_npoll. */ - if (gd->gd_ipimask) + if (CPUMASK_TESTNZERO(gd->gd_ipimask)) goto again; atomic_poll_release_int(&gd->gd_npoll); cpu_mfence(); - if (gd->gd_ipimask) + if (CPUMASK_TESTNZERO(gd->gd_ipimask)) goto again; } @@ -774,8 +774,8 @@ lwkt_sync_ipiq(void *arg) { volatile cpumask_t *cpumask = arg; - atomic_clear_cpumask(cpumask, mycpu->gd_cpumask); - if (*cpumask == 0) + ATOMIC_CPUMASK_NANDBIT(*cpumask, mycpu->gd_cpuid); + if (CPUMASK_TESTZERO(*cpumask)) wakeup(cpumask); } @@ -784,13 +784,14 @@ lwkt_synchronize_ipiqs(const char *wmesg) { volatile cpumask_t other_cpumask; - other_cpumask = mycpu->gd_other_cpus & smp_active_mask; + other_cpumask = smp_active_mask; + CPUMASK_ANDMASK(other_cpumask, mycpu->gd_other_cpus); lwkt_send_ipiq_mask(other_cpumask, lwkt_sync_ipiq, - __DEVOLATILE(void *, &other_cpumask)); + __DEVOLATILE(void *, &other_cpumask)); - while (other_cpumask != 0) { + while (CPUMASK_TESTNZERO(other_cpumask)) { tsleep_interlock(&other_cpumask, 0); - if (other_cpumask != 0) + if (CPUMASK_TESTNZERO(other_cpumask)) tsleep(&other_cpumask, PINTERLOCKED, wmesg, 0); } } @@ -830,20 +831,23 @@ lwkt_cpusync_interlock(lwkt_cpusync_t cs) * * mack does not include the current cpu. */ - mask = cs->cs_mask & gd->gd_other_cpus & smp_active_mask; - cs->cs_mack = 0; + mask = cs->cs_mask; + CPUMASK_ANDMASK(mask, gd->gd_other_cpus); + CPUMASK_ANDMASK(mask, smp_active_mask); + CPUMASK_ASSZERO(cs->cs_mack); + crit_enter_id("cpusync"); - if (mask) { + if (CPUMASK_TESTNZERO(mask)) { DEBUG_PUSH_INFO("cpusync_interlock"); ++ipiq_stat(gd).ipiq_cscount; ++gd->gd_curthread->td_cscount; lwkt_send_ipiq_mask(mask, (ipifunc1_t)lwkt_cpusync_remote1, cs); - logipiq2(sync_start, (long)mask); + logipiq2(sync_start, (long)CPUMASK_LOWMASK(mask)); #if 0 if (gd->gd_curthread->td_wmesg == NULL) gd->gd_curthread->td_wmesg = smsg; #endif - while (cs->cs_mack != mask) { + while (CPUMASK_CMPMASKNEQ(cs->cs_mack, mask)) { lwkt_process_ipiq(); cpu_pause(); #ifdef _KERNEL_VIRTUAL @@ -884,17 +888,17 @@ lwkt_cpusync_deinterlock(lwkt_cpusync_t cs) */ mask = cs->cs_mack; cpu_ccfence(); - cs->cs_mack = 0; + CPUMASK_ASSZERO(cs->cs_mack); cpu_ccfence(); - if (cs->cs_func && (cs->cs_mask & gd->gd_cpumask)) + if (cs->cs_func && CPUMASK_TESTBIT(cs->cs_mask, gd->gd_cpuid)) cs->cs_func(cs->cs_data); - if (mask) { + if (CPUMASK_TESTNZERO(mask)) { DEBUG_PUSH_INFO("cpusync_deinterlock"); #if 0 if (gd->gd_curthread->td_wmesg == NULL) gd->gd_curthread->td_wmesg = smsg; #endif - while (cs->cs_mack != mask) { + while (CPUMASK_CMPMASKNEQ(cs->cs_mack, mask)) { lwkt_process_ipiq(); cpu_pause(); #ifdef _KERNEL_VIRTUAL @@ -913,7 +917,7 @@ lwkt_cpusync_deinterlock(lwkt_cpusync_t cs) */ --gd->gd_curthread->td_cscount; lwkt_process_ipiq(); - logipiq2(sync_end, (long)mask); + logipiq2(sync_end, (long)CPUMASK_LOWMASK(mask)); } crit_exit_id("cpusync"); } @@ -930,7 +934,7 @@ lwkt_cpusync_remote1(lwkt_cpusync_t cs) { globaldata_t gd = mycpu; - atomic_set_cpumask(&cs->cs_mack, gd->gd_cpumask); + ATOMIC_CPUMASK_ORBIT(cs->cs_mack, gd->gd_cpuid); lwkt_cpusync_remote2(cs); } @@ -945,10 +949,10 @@ lwkt_cpusync_remote2(lwkt_cpusync_t cs) { globaldata_t gd = mycpu; - if ((cs->cs_mack & gd->gd_cpumask) == 0) { + if (CPUMASK_TESTMASK(cs->cs_mack, gd->gd_cpumask) == 0) { if (cs->cs_func) cs->cs_func(cs->cs_data); - atomic_set_cpumask(&cs->cs_mack, gd->gd_cpumask); + ATOMIC_CPUMASK_ORBIT(cs->cs_mack, gd->gd_cpuid); /* cs can be ripped out at this point */ } else { lwkt_ipiq_t ip; @@ -968,7 +972,8 @@ lwkt_cpusync_remote2(lwkt_cpusync_t cs) if (ipiq_debug && (ip->ip_windex & 0xFFFFFF) == 0) { kprintf("cpu %d cm=%016jx %016jx f=%p\n", gd->gd_cpuid, - (intmax_t)cs->cs_mask, (intmax_t)cs->cs_mack, + (intmax_t)CPUMASK_LOWMASK(cs->cs_mask), + (intmax_t)CPUMASK_LOWMASK(cs->cs_mack), cs->cs_func); } } diff --git a/sys/kern/subr_cpu_topology.c b/sys/kern/subr_cpu_topology.c index e029de1e28..833855162f 100644 --- a/sys/kern/subr_cpu_topology.c +++ b/sys/kern/subr_cpu_topology.c @@ -108,12 +108,12 @@ build_topology_tree(int *children_no_per_level, node->child_no = children_no_per_level[cur_level]; node->type = level_types[cur_level]; - node->members = 0; + CPUMASK_ASSZERO(node->members); node->compute_unit_id = -1; if (node->child_no == 0) { *apicid = get_next_valid_apicid(*apicid); - node->members = CPUMASK(get_cpuid_from_apicid(*apicid)); + CPUMASK_ASSBIT(node->members, get_cpuid_from_apicid(*apicid)); return; } @@ -133,7 +133,7 @@ build_topology_tree(int *children_no_per_level, last_free_node, apicid); - node->members |= node->child_node[i]->members; + CPUMASK_ORMASK(node->members, node->child_node[i]->members); } } @@ -175,10 +175,11 @@ build_cpu_topology(void) * and witin core to build up the topology */ for (i = 0; i < ncpus; i++) { + cpumask_t mask; - cpumask_t mask = CPUMASK(i); + CPUMASK_ASSBIT(mask, i); - if ((mask & smp_active_mask) == 0) + if (CPUMASK_TESTMASK(mask, smp_active_mask) == 0) continue; if (get_chip_ID(BSPID) == get_chip_ID(i)) @@ -293,7 +294,7 @@ build_cpu_topology(void) last_free_node->child_node[last_free_node->child_no] = parent->child_node[j]; last_free_node->child_no++; - last_free_node->members |= parent->child_node[j]->members; + CPUMASK_ORMASK(last_free_node->members, parent->child_node[j]->members); parent->child_node[j]->type = THREAD_LEVEL; parent->child_node[j]->parent_node = last_free_node; @@ -443,9 +444,8 @@ get_cpu_node_by_cpumask(cpu_node_t * node, cpu_node_t * found = NULL; int i; - if (node->members == mask) { + if (CPUMASK_CMPMASKEQ(node->members, mask)) return node; - } for (i = 0; i < node->child_no; i++) { found = get_cpu_node_by_cpumask(node->child_node[i], mask); @@ -458,7 +458,9 @@ get_cpu_node_by_cpumask(cpu_node_t * node, cpu_node_t * get_cpu_node_by_cpuid(int cpuid) { - cpumask_t mask = CPUMASK(cpuid); + cpumask_t mask; + + CPUMASK_ASSBIT(mask, cpuid); KASSERT(cpu_root_node != NULL, ("cpu_root_node isn't initialized")); @@ -471,14 +473,17 @@ get_cpumask_from_level(int cpuid, uint8_t level_type) { cpu_node_t * node; - cpumask_t mask = CPUMASK(cpuid); + cpumask_t mask; + + CPUMASK_ASSBIT(mask, cpuid); KASSERT(cpu_root_node != NULL, ("cpu_root_node isn't initialized")); node = get_cpu_node_by_cpumask(cpu_root_node, mask); if (node == NULL) { - return 0; + CPUMASK_ASSZERO(mask); + return mask; } while (node != NULL) { @@ -487,8 +492,9 @@ get_cpumask_from_level(int cpuid, } node = node->parent_node; } + CPUMASK_ASSZERO(mask); - return 0; + return mask; } /* init pcpu_sysctl structure info */ @@ -510,7 +516,7 @@ init_pcpu_topology_sysctl(void) /* Get physical siblings */ mask = get_cpumask_from_level(i, CHIP_LEVEL); - if (mask == 0) { + if (CPUMASK_TESTZERO(mask)) { pcpu_sysctl[i].physical_id = INVALID_ID; continue; } @@ -527,7 +533,7 @@ init_pcpu_topology_sysctl(void) /* Get core siblings */ mask = get_cpumask_from_level(i, CORE_LEVEL); - if (mask == 0) { + if (CPUMASK_TESTZERO(mask)) { pcpu_sysctl[i].core_id = INVALID_ID; continue; } diff --git a/sys/kern/sys_vmm.c b/sys/kern/sys_vmm.c index b1abc45e26..c58122d092 100644 --- a/sys/kern/sys_vmm.c +++ b/sys/kern/sys_vmm.c @@ -129,6 +129,7 @@ sys_vmm_guest_sync_addr(struct vmm_guest_sync_addr_args *uap) int error = 0; cpulock_t olock; cpulock_t nlock; + cpumask_t mask; long val; struct proc *p = curproc; @@ -140,7 +141,7 @@ sys_vmm_guest_sync_addr(struct vmm_guest_sync_addr_args *uap) /* * Acquire CPULOCK_EXCL, spin while we wait. */ - KKASSERT((p->p_vmm_cpumask & mycpu->gd_cpumask) == 0); + KKASSERT(CPUMASK_TESTMASK(p->p_vmm_cpumask, mycpu->gd_cpumask) == 0); for (;;) { olock = p->p_vmm_cpulock & ~CPULOCK_EXCL; cpu_ccfence(); @@ -163,8 +164,9 @@ sys_vmm_guest_sync_addr(struct vmm_guest_sync_addr_args *uap) * counter. */ if (olock & CPULOCK_CNTMASK) { - lwkt_send_ipiq_mask(p->p_vmm_cpumask & mycpu->gd_other_cpus, - vmm_exit_vmm, NULL); + mask = p->p_vmm_cpumask; + CPUMASK_ANDMASK(mask, mycpu->gd_other_cpus); + lwkt_send_ipiq_mask(mask, vmm_exit_vmm, NULL); while (p->p_vmm_cpulock & CPULOCK_CNTMASK) { lwkt_process_ipiq(); cpu_pause(); diff --git a/sys/kern/usched_bsd4.c b/sys/kern/usched_bsd4.c index 625050bce2..fb9c12cfd3 100644 --- a/sys/kern/usched_bsd4.c +++ b/sys/kern/usched_bsd4.c @@ -158,8 +158,10 @@ static struct rq bsd4_idqueues[NQS]; static u_int32_t bsd4_queuebits; static u_int32_t bsd4_rtqueuebits; static u_int32_t bsd4_idqueuebits; -static cpumask_t bsd4_curprocmask = -1; /* currently running a user process */ -static cpumask_t bsd4_rdyprocmask; /* ready to accept a user process */ +/* currently running a user process */ +static cpumask_t bsd4_curprocmask = CPUMASK_INITIALIZER_ALLONES; +/* ready to accept a user process */ +static cpumask_t bsd4_rdyprocmask; static int bsd4_runqcount; static volatile int bsd4_scancpu; static struct spinlock bsd4_spin; @@ -302,7 +304,7 @@ bsd4_rqinit(void *dummy) TAILQ_INIT(&bsd4_rtqueues[i]); TAILQ_INIT(&bsd4_idqueues[i]); } - atomic_clear_cpumask(&bsd4_curprocmask, 1); + ATOMIC_CPUMASK_NANDBIT(bsd4_curprocmask, 0); } SYSINIT(runqueue, SI_BOOT2_USCHED, SI_ORDER_FIRST, bsd4_rqinit, NULL) @@ -394,7 +396,7 @@ bsd4_acquire_curproc(struct lwp *lp) /* * We can trivially become the current lwp. */ - atomic_set_cpumask(&bsd4_curprocmask, gd->gd_cpumask); + ATOMIC_CPUMASK_ORBIT(bsd4_curprocmask, gd->gd_cpuid); dd->uschedcp = lp; dd->upri = lp->lwp_priority; } else if (dd->upri > lp->lwp_priority) { @@ -489,7 +491,7 @@ bsd4_release_curproc(struct lwp *lp) dd->uschedcp = NULL; /* don't let lp be selected */ dd->upri = PRIBASE_NULL; - atomic_clear_cpumask(&bsd4_curprocmask, gd->gd_cpumask); + ATOMIC_CPUMASK_NANDBIT(bsd4_curprocmask, gd->gd_cpuid); dd->old_uschedcp = lp; /* used only for KTR debug prints */ bsd4_select_curproc(gd); crit_exit(); @@ -537,7 +539,7 @@ bsd4_select_curproc(globaldata_t gd) dd->old_uschedcp->lwp_thread->td_gd->gd_cpuid, gd->gd_cpuid); - atomic_set_cpumask(&bsd4_curprocmask, CPUMASK(cpuid)); + ATOMIC_CPUMASK_ORBIT(bsd4_curprocmask, cpuid); dd->upri = nlp->lwp_priority; dd->uschedcp = nlp; dd->rrcount = 0; /* reset round robin */ @@ -549,8 +551,8 @@ bsd4_select_curproc(globaldata_t gd) } #if 0 - } else if (bsd4_runqcount && (bsd4_rdyprocmask & CPUMASK(cpuid))) { - atomic_clear_cpumask(&bsd4_rdyprocmask, CPUMASK(cpuid)); + } else if (bsd4_runqcount && CPUMASK_TESTBIT(bsd4_rdyprocmask, cpuid)) { + ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, cpuid); spin_unlock(&bsd4_spin); lwkt_schedule(&dd->helper_thread); } else { @@ -572,10 +574,11 @@ bsd4_batchy_looser_pri_test(struct lwp* lp) int cpu; /* Current running processes */ - mask = bsd4_curprocmask & smp_active_mask - & usched_global_cpumask; + mask = bsd4_curprocmask; + CPUMASK_ANDMASK(mask, smp_active_mask); + CPUMASK_ANDMASK(mask, usched_global_cpumask); - while(mask) { + while (CPUMASK_TESTNZERO(mask)) { cpu = BSFCPUMASK(mask); other_dd = &bsd4_pcpu[cpu]; if (other_dd->upri - lp->lwp_priority > usched_bsd4_upri_affinity * PPQ) { @@ -588,7 +591,7 @@ bsd4_batchy_looser_pri_test(struct lwp* lp) return 0; } - mask &= ~CPUMASK(cpu); + CPUMASK_NANDBIT(mask, cpu); } KTR_COND_LOG(usched_batchy_test_true, @@ -687,8 +690,11 @@ bsd4_setrunqueue(struct lwp *lp) int sibling; cpuid = (bsd4_scancpu & 0xFFFF) % ncpus; - mask = ~bsd4_curprocmask & bsd4_rdyprocmask & lp->lwp_cpumask & - smp_active_mask & usched_global_cpumask; + mask = bsd4_rdyprocmask; + CPUMASK_NANDMASK(mask, bsd4_curprocmask); + CPUMASK_ANDMASK(mask, lp->lwp_cpumask); + CPUMASK_ANDMASK(mask, smp_active_mask); + CPUMASK_ANDMASK(mask, usched_global_cpumask); KTR_COND_LOG(usched_bsd4_setrunqueue_fc_smt, lp->lwp_proc->p_pid == usched_bsd4_pid_debug, @@ -697,18 +703,22 @@ bsd4_setrunqueue(struct lwp *lp) (unsigned long)mask, mycpu->gd_cpuid); - while (mask) { - tmpmask = ~(CPUMASK(cpuid) - 1); - if (mask & tmpmask) - cpuid = BSFCPUMASK(mask & tmpmask); - else + while (CPUMASK_TESTNZERO(mask)) { + CPUMASK_ASSNBMASK(tmpmask, cpuid); + if (CPUMASK_TESTMASK(tmpmask, mask)) { + CPUMASK_ANDMASK(tmpmask, mask); + cpuid = BSFCPUMASK(tmpmask); + } else { cpuid = BSFCPUMASK(mask); + } gd = globaldata_find(cpuid); dd = &bsd4_pcpu[cpuid]; if ((dd->upri & ~PPQMASK) >= (lp->lwp_priority & ~PPQMASK)) { - if (dd->cpunode->parent_node->members & ~dd->cpunode->members & mask) { - + tmpmask = dd->cpunode->parent_node->members; + CPUMASK_NANDMASK(tmpmask, dd->cpunode->members); + CPUMASK_ANDMASK(tmpmask, mask); + if (CPUMASK_TESTNZERO(tmpmask)) { KTR_COND_LOG(usched_bsd4_setrunqueue_found, lp->lwp_proc->p_pid == usched_bsd4_pid_debug, lp->lwp_proc->p_pid, @@ -719,15 +729,20 @@ bsd4_setrunqueue(struct lwp *lp) goto found; } else { - sibling = BSFCPUMASK(dd->cpunode->parent_node->members & - ~dd->cpunode->members); - if (min_prio > bsd4_pcpu[sibling].upri) { - min_prio = bsd4_pcpu[sibling].upri; + tmpmask = + dd->cpunode->parent_node->members; + CPUMASK_NANDMASK(tmpmask, + dd->cpunode->members); + sibling = BSFCPUMASK(tmpmask); + if (min_prio > + bsd4_pcpu[sibling].upri) { + min_prio = + bsd4_pcpu[sibling].upri; best_cpuid = cpuid; } } } - mask &= ~CPUMASK(cpuid); + CPUMASK_NANDBIT(mask, cpuid); } if (best_cpuid != -1) { @@ -748,8 +763,11 @@ bsd4_setrunqueue(struct lwp *lp) } else { /* Fallback to the original heuristic */ cpuid = (bsd4_scancpu & 0xFFFF) % ncpus; - mask = ~bsd4_curprocmask & bsd4_rdyprocmask & lp->lwp_cpumask & - smp_active_mask & usched_global_cpumask; + mask = bsd4_rdyprocmask; + CPUMASK_NANDMASK(mask, bsd4_curprocmask); + CPUMASK_ANDMASK(mask, lp->lwp_cpumask); + CPUMASK_ANDMASK(mask, smp_active_mask); + CPUMASK_ANDMASK(mask, usched_global_cpumask); KTR_COND_LOG(usched_bsd4_setrunqueue_fc_non_smt, lp->lwp_proc->p_pid == usched_bsd4_pid_debug, @@ -758,55 +776,61 @@ bsd4_setrunqueue(struct lwp *lp) (unsigned long)mask, mycpu->gd_cpuid); - while (mask) { - tmpmask = ~(CPUMASK(cpuid) - 1); - if (mask & tmpmask) - cpuid = BSFCPUMASK(mask & tmpmask); - else + while (CPUMASK_TESTNZERO(mask)) { + CPUMASK_ASSNBMASK(tmpmask, cpuid); + if (CPUMASK_TESTMASK(tmpmask, mask)) { + CPUMASK_ANDMASK(tmpmask, mask); + cpuid = BSFCPUMASK(tmpmask); + } else { cpuid = BSFCPUMASK(mask); + } gd = globaldata_find(cpuid); dd = &bsd4_pcpu[cpuid]; - if ((dd->upri & ~PPQMASK) >= (lp->lwp_priority & ~PPQMASK)) { - + if ((dd->upri & ~PPQMASK) >= + (lp->lwp_priority & ~PPQMASK)) { KTR_COND_LOG(usched_bsd4_setrunqueue_found, lp->lwp_proc->p_pid == usched_bsd4_pid_debug, lp->lwp_proc->p_pid, lp->lwp_thread->td_gd->gd_cpuid, - (unsigned long)mask, + (unsigned long)CPUMASK_LOWMASK(mask), cpuid, mycpu->gd_cpuid); goto found; } - mask &= ~CPUMASK(cpuid); + CPUMASK_NANDBIT(mask, cpuid); } } /* * Then cpus which might have a currently running lp */ - mask = bsd4_curprocmask & bsd4_rdyprocmask & - lp->lwp_cpumask & smp_active_mask & usched_global_cpumask; + mask = bsd4_curprocmask; + CPUMASK_ANDMASK(mask, bsd4_rdyprocmask); + CPUMASK_ANDMASK(mask, lp->lwp_cpumask); + CPUMASK_ANDMASK(mask, smp_active_mask); + CPUMASK_ANDMASK(mask, usched_global_cpumask); KTR_COND_LOG(usched_bsd4_setrunqueue_rc, lp->lwp_proc->p_pid == usched_bsd4_pid_debug, lp->lwp_proc->p_pid, lp->lwp_thread->td_gd->gd_cpuid, - (unsigned long)mask, + (unsigned long)CPUMASK_LOWMASK(mask), mycpu->gd_cpuid); - while (mask) { - tmpmask = ~(CPUMASK(cpuid) - 1); - if (mask & tmpmask) - cpuid = BSFCPUMASK(mask & tmpmask); - else + while (CPUMASK_TESTNZERO(mask)) { + CPUMASK_ASSNBMASK(tmpmask, cpuid); + if (CPUMASK_TESTMASK(tmpmask, mask)) { + CPUMASK_ANDMASK(tmpmask, mask); + cpuid = BSFCPUMASK(tmpmask); + } else { cpuid = BSFCPUMASK(mask); + } gd = globaldata_find(cpuid); dd = &bsd4_pcpu[cpuid]; if ((dd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) { - KTR_COND_LOG(usched_bsd4_setrunqueue_found, lp->lwp_proc->p_pid == usched_bsd4_pid_debug, lp->lwp_proc->p_pid, @@ -817,7 +841,7 @@ bsd4_setrunqueue(struct lwp *lp) goto found; } - mask &= ~CPUMASK(cpuid); + CPUMASK_NANDBIT(mask, cpuid); } /* @@ -832,9 +856,8 @@ bsd4_setrunqueue(struct lwp *lp) * set the user resched flag because */ cpuid = (bsd4_scancpu & 0xFFFF) % ncpus; - if ((CPUMASK(cpuid) & usched_global_cpumask) == 0) { + if (CPUMASK_TESTBIT(usched_global_cpumask, cpuid) == 0) cpuid = 0; - } gd = globaldata_find(cpuid); dd = &bsd4_pcpu[cpuid]; @@ -856,7 +879,7 @@ found: } } } else { - atomic_clear_cpumask(&bsd4_rdyprocmask, CPUMASK(cpuid)); + ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, cpuid); spin_unlock(&bsd4_spin); if ((dd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) lwkt_send_ipiq(gd, bsd4_need_user_resched_remote, NULL); @@ -1168,7 +1191,7 @@ bsd4_resetpriority(struct lwp *lp) */ if (reschedcpu >= 0) { dd = &bsd4_pcpu[reschedcpu]; - if ((bsd4_rdyprocmask & CPUMASK(reschedcpu)) && + if (CPUMASK_TESTBIT(bsd4_rdyprocmask, reschedcpu) && (checkpri == 0 || (dd->upri & ~PRIMASK) > (lp->lwp_priority & ~PRIMASK))) { if (reschedcpu == mycpu->gd_cpuid) { @@ -1176,8 +1199,8 @@ bsd4_resetpriority(struct lwp *lp) need_user_resched(); } else { spin_unlock(&bsd4_spin); - atomic_clear_cpumask(&bsd4_rdyprocmask, - CPUMASK(reschedcpu)); + ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, + reschedcpu); lwkt_send_ipiq(lp->lwp_thread->td_gd, bsd4_need_user_resched_remote, NULL); @@ -1320,7 +1343,7 @@ again: lp = TAILQ_FIRST(q); KASSERT(lp, ("chooseproc: no lwp on busy queue")); - while ((lp->lwp_cpumask & cpumask) == 0) { + while (CPUMASK_TESTMASK(lp->lwp_cpumask, cpumask) == 0) { lp = TAILQ_NEXT(lp, lwp_procq); if (lp == NULL) { *which2 &= ~(1 << pri); @@ -1448,8 +1471,9 @@ again: * minimize the contention (we are in a locked region */ while (checks < usched_bsd4_queue_checks) { - if ((lp->lwp_cpumask & cpumask) == 0 || - ((siblings & lp->lwp_thread->td_gd->gd_cpumask) == 0 && + if (CPUMASK_TESTMASK(lp->lwp_cpumask, cpumask) == 0 || + (CPUMASK_TESTMASK(siblings, + lp->lwp_thread->td_gd->gd_cpumask) == 0 && (lp->lwp_rebal_ticks == sched_ticks || lp->lwp_rebal_ticks == (int)(sched_ticks - 1)) && bsd4_batchy_looser_pri_test(lp))) { @@ -1457,15 +1481,18 @@ again: KTR_COND_LOG(usched_chooseproc_cc_not_good, lp->lwp_proc->p_pid == usched_bsd4_pid_debug, lp->lwp_proc->p_pid, - (unsigned long)lp->lwp_thread->td_gd->gd_cpumask, - (unsigned long)siblings, - (unsigned long)cpumask); + (unsigned long)CPUMASK_LOWMASK( + lp->lwp_thread->td_gd->gd_cpumask), + (unsigned long)CPUMASK_LOWMASK(siblings), + (unsigned long)CPUMASK_LOWMASK(cpumask)); cpunode = bsd4_pcpu[lp->lwp_thread->td_gd->gd_cpuid].cpunode; level = 0; while (cpunode) { - if (cpunode->members & cpumask) + if (CPUMASK_TESTMASK(cpunode->members, + cpumask)) { break; + } cpunode = cpunode->parent_node; level++; } @@ -1490,9 +1517,10 @@ again: KTR_COND_LOG(usched_chooseproc_cc_elected, lp->lwp_proc->p_pid == usched_bsd4_pid_debug, lp->lwp_proc->p_pid, - (unsigned long)lp->lwp_thread->td_gd->gd_cpumask, - (unsigned long)siblings, - (unsigned long)cpumask); + (unsigned long)CPUMASK_LOWMASK( + lp->lwp_thread->td_gd->gd_cpumask), + (unsigned long)CPUMASK_LOWMASK(siblings), + (unsigned long)CPUMASK_LOWMASK(cpumask)); goto found; } @@ -1553,17 +1581,22 @@ bsd4_kick_helper(struct lwp *lp) { globaldata_t gd; bsd4_pcpu_t dd; + cpumask_t tmpmask; if (lp == NULL) return; gd = lp->lwp_thread->td_gd; dd = &bsd4_pcpu[gd->gd_cpuid]; - if ((smp_active_mask & usched_global_cpumask & - bsd4_rdyprocmask & gd->gd_cpumask) == 0) { + + tmpmask = smp_active_mask; + CPUMASK_ANDMASK(tmpmask, usched_global_cpumask); + CPUMASK_ANDMASK(tmpmask, bsd4_rdyprocmask); + CPUMASK_ANDMASK(tmpmask, gd->gd_cpumask); + if (CPUMASK_TESTZERO(tmpmask)) return; - } + ++usched_bsd4_kicks; - atomic_clear_cpumask(&bsd4_rdyprocmask, gd->gd_cpumask); + ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, gd->gd_cpuid); if ((dd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) { lwkt_send_ipiq(gd, bsd4_need_user_resched_remote, NULL); } else { @@ -1735,12 +1768,12 @@ sched_thread(void *dummy) crit_enter_gd(gd); tsleep_interlock(&dd->helper_thread, 0); spin_lock(&bsd4_spin); - atomic_set_cpumask(&bsd4_rdyprocmask, mask); + ATOMIC_CPUMASK_ORMASK(bsd4_rdyprocmask, mask); clear_user_resched(); /* This satisfied the reschedule request */ dd->rrcount = 0; /* Reset the round-robin counter */ - if ((bsd4_curprocmask & mask) == 0) { + if (CPUMASK_TESTMASK(bsd4_curprocmask, mask) == 0) { /* * No thread is currently scheduled. */ @@ -1752,7 +1785,7 @@ sched_thread(void *dummy) nlp->lwp_proc->p_pid, nlp->lwp_thread->td_gd->gd_cpuid); - atomic_set_cpumask(&bsd4_curprocmask, mask); + ATOMIC_CPUMASK_ORMASK(bsd4_curprocmask, mask); dd->upri = nlp->lwp_priority; dd->uschedcp = nlp; dd->rrcount = 0; /* reset round robin */ @@ -1788,13 +1821,13 @@ sched_thread(void *dummy) * to priority test does not leave other unscheduled * cpus idle when the runqueue is not empty. */ - tmpmask = ~bsd4_curprocmask & - bsd4_rdyprocmask & smp_active_mask; - if (tmpmask) { + tmpmask = bsd4_rdyprocmask; + CPUMASK_NANDMASK(tmpmask, bsd4_curprocmask); + CPUMASK_ANDMASK(tmpmask, smp_active_mask); + if (CPUMASK_TESTNZERO(tmpmask)) { tmpid = BSFCPUMASK(tmpmask); tmpdd = &bsd4_pcpu[tmpid]; - atomic_clear_cpumask(&bsd4_rdyprocmask, - CPUMASK(tmpid)); + ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, tmpid); spin_unlock(&bsd4_spin); wakeup(&tmpdd->helper_thread); } else { @@ -1861,9 +1894,11 @@ sched_thread_cpu_init(void) for (i = 0; i < ncpus; ++i) { bsd4_pcpu_t dd = &bsd4_pcpu[i]; - cpumask_t mask = CPUMASK(i); + cpumask_t mask; - if ((mask & smp_active_mask) == 0) + CPUMASK_ASSBIT(mask, i); + + if (CPUMASK_TESTMASK(mask, smp_active_mask) == 0) continue; dd->cpunode = get_cpu_node_by_cpuid(i); @@ -1914,8 +1949,10 @@ sched_thread_cpu_init(void) if (bootverbose) { if (dd->cpunode->parent_node != NULL) { - CPUSET_FOREACH(cpuid, dd->cpunode->parent_node->members) + CPUSET_FOREACH(cpuid, + dd->cpunode->parent_node->members) { kprintf("cpu%d ", cpuid); + } kprintf("\n"); } else { kprintf(" no siblings\n"); @@ -1931,8 +1968,8 @@ sched_thread_cpu_init(void) * been enabled in rqinit(). */ if (i) - atomic_clear_cpumask(&bsd4_curprocmask, mask); - atomic_set_cpumask(&bsd4_rdyprocmask, mask); + ATOMIC_CPUMASK_NANDMASK(bsd4_curprocmask, mask); + ATOMIC_CPUMASK_ORMASK(bsd4_rdyprocmask, mask); dd->upri = PRIBASE_NULL; } diff --git a/sys/kern/usched_dfly.c b/sys/kern/usched_dfly.c index fc12dae632..9a49defdf6 100644 --- a/sys/kern/usched_dfly.c +++ b/sys/kern/usched_dfly.c @@ -177,7 +177,8 @@ struct usched usched_dfly = { * the state of all 32 queues and then a ffs() to find the first busy * queue. */ -static cpumask_t dfly_curprocmask = -1; /* currently running a user process */ + /* currently running a user process */ +static cpumask_t dfly_curprocmask = CPUMASK_INITIALIZER_ALLONES; static cpumask_t dfly_rdyprocmask; /* ready to accept a user process */ static volatile int dfly_scancpu; static volatile int dfly_ucount; /* total running on whole system */ @@ -366,7 +367,7 @@ dfly_acquire_curproc(struct lwp *lp) if (dd->uschedcp == NULL) { atomic_clear_int(&lp->lwp_thread->td_mpflags, TDF_MP_DIDYIELD); - atomic_set_cpumask(&dfly_curprocmask, gd->gd_cpumask); + ATOMIC_CPUMASK_ORBIT(dfly_curprocmask, gd->gd_cpuid); dd->uschedcp = lp; dd->upri = lp->lwp_priority; KKASSERT(lp->lwp_qcpu == dd->cpuid); @@ -518,7 +519,7 @@ dfly_release_curproc(struct lwp *lp) if (dd->uschedcp == lp) { dd->uschedcp = NULL; /* don't let lp be selected */ dd->upri = PRIBASE_NULL; - atomic_clear_cpumask(&dfly_curprocmask, gd->gd_cpumask); + ATOMIC_CPUMASK_NANDBIT(dfly_curprocmask, gd->gd_cpuid); spin_unlock(&dd->spin); dfly_select_curproc(gd); } else { @@ -554,7 +555,7 @@ dfly_select_curproc(globaldata_t gd) nlp = dfly_chooseproc_locked(dd, dd, dd->uschedcp, 0); if (nlp) { - atomic_set_cpumask(&dfly_curprocmask, CPUMASK(cpuid)); + ATOMIC_CPUMASK_ORBIT(dfly_curprocmask, cpuid); dd->upri = nlp->lwp_priority; dd->uschedcp = nlp; #if 0 @@ -828,7 +829,7 @@ dfly_schedulerclock(struct lwp *lp, sysclock_t period, sysclock_t cpstamp) */ if (nlp && (nlp->lwp_priority & ~PPQMASK) < (dd->upri & ~PPQMASK)) { - atomic_set_cpumask(&dfly_curprocmask, dd->cpumask); + ATOMIC_CPUMASK_ORMASK(dfly_curprocmask, dd->cpumask); dd->upri = nlp->lwp_priority; dd->uschedcp = nlp; #if 0 @@ -1133,7 +1134,7 @@ dfly_resetpriority(struct lwp *lp) * check which will fail in that case. */ if (rcpu >= 0) { - if ((dfly_rdyprocmask & CPUMASK(rcpu)) && + if (CPUMASK_TESTBIT(dfly_rdyprocmask, rcpu) && (checkpri == 0 || (rdd->upri & ~PRIMASK) > (lp->lwp_priority & ~PRIMASK))) { @@ -1455,7 +1456,7 @@ dfly_choose_best_queue(struct lwp *lp) if ((wakecpu = lp->lwp_thread->td_wakefromcpu) >= 0) wakemask = dfly_pcpu[wakecpu].cpumask; else - wakemask = 0; + CPUMASK_ASSZERO(wakemask); /* * When the topology is known choose a cpu whos group has, in @@ -1490,15 +1491,17 @@ dfly_choose_best_queue(struct lwp *lp) * which are members of this node. */ cpun = cpup->child_node[n]; - mask = cpun->members & usched_global_cpumask & - smp_active_mask & lp->lwp_cpumask; - if (mask == 0) + mask = cpun->members; + CPUMASK_ANDMASK(mask, usched_global_cpumask); + CPUMASK_ANDMASK(mask, smp_active_mask); + CPUMASK_ANDMASK(mask, lp->lwp_cpumask); + if (CPUMASK_TESTZERO(mask)) continue; count = 0; load = 0; - while (mask) { + while (CPUMASK_TESTNZERO(mask)) { cpuid = BSFCPUMASK(mask); rdd = &dfly_pcpu[cpuid]; load += rdd->uload; @@ -1515,7 +1518,7 @@ dfly_choose_best_queue(struct lwp *lp) load -= usched_dfly_weight4 / 2; } #endif - mask &= ~CPUMASK(cpuid); + CPUMASK_NANDBIT(mask, cpuid); ++count; } @@ -1526,7 +1529,7 @@ dfly_choose_best_queue(struct lwp *lp) * otherwise the calculation is bogus. */ if ((lp->lwp_mpflags & LWP_MP_ULOAD) && - (dd->cpumask & cpun->members)) { + CPUMASK_TESTMASK(dd->cpumask, cpun->members)) { load -= lp->lwp_uload; load -= usched_dfly_weight3; } @@ -1536,7 +1539,7 @@ dfly_choose_best_queue(struct lwp *lp) /* * Advantage the cpu group (lp) is already on. */ - if (cpun->members & dd->cpumask) + if (CPUMASK_TESTMASK(cpun->members, dd->cpumask)) load -= usched_dfly_weight1; /* @@ -1556,7 +1559,7 @@ dfly_choose_best_queue(struct lwp *lp) * all-but-one by the same amount, so it won't effect * the weight1 factor for the all-but-one nodes. */ - if (cpun->members & wakemask) { + if (CPUMASK_TESTMASK(cpun->members, wakemask)) { if (cpun->child_no != 0) { /* advantage */ load -= usched_dfly_weight2; @@ -1573,7 +1576,7 @@ dfly_choose_best_queue(struct lwp *lp) */ if (cpub == NULL || lowest_load > load || (lowest_load == load && - (cpun->members & dd->cpumask)) + CPUMASK_TESTMASK(cpun->members, dd->cpumask)) ) { lowest_load = load; cpub = cpun; @@ -1657,14 +1660,15 @@ dfly_choose_worst_queue(dfly_pcpu_t dd) * which are members of this node. */ cpun = cpup->child_node[n]; - mask = cpun->members & usched_global_cpumask & - smp_active_mask; - if (mask == 0) + mask = cpun->members; + CPUMASK_ANDMASK(mask, usched_global_cpumask); + CPUMASK_ANDMASK(mask, smp_active_mask); + if (CPUMASK_TESTZERO(mask)) continue; count = 0; load = 0; - while (mask) { + while (CPUMASK_TESTNZERO(mask)) { cpuid = BSFCPUMASK(mask); rdd = &dfly_pcpu[cpuid]; load += rdd->uload; @@ -1680,7 +1684,7 @@ dfly_choose_worst_queue(dfly_pcpu_t dd) load -= usched_dfly_weight4 / 2; } #endif - mask &= ~CPUMASK(cpuid); + CPUMASK_NANDBIT(mask, cpuid); ++count; } load /= count; @@ -1689,7 +1693,7 @@ dfly_choose_worst_queue(dfly_pcpu_t dd) * Prefer candidates which are somewhat closer to * our cpu. */ - if (dd->cpumask & cpun->members) + if (CPUMASK_TESTMASK(dd->cpumask, cpun->members)) load += usched_dfly_weight1; /* @@ -1745,40 +1749,50 @@ dfly_choose_queue_simple(dfly_pcpu_t dd, struct lwp *lp) */ ++dfly_scancpu; cpuid = (dfly_scancpu & 0xFFFF) % ncpus; - mask = ~dfly_curprocmask & dfly_rdyprocmask & lp->lwp_cpumask & - smp_active_mask & usched_global_cpumask; - - while (mask) { - tmpmask = ~(CPUMASK(cpuid) - 1); - if (mask & tmpmask) - cpuid = BSFCPUMASK(mask & tmpmask); - else + mask = dfly_rdyprocmask; + CPUMASK_NANDMASK(mask, dfly_curprocmask); + CPUMASK_ANDMASK(mask, lp->lwp_cpumask); + CPUMASK_ANDMASK(mask, smp_active_mask); + CPUMASK_ANDMASK(mask, usched_global_cpumask); + + while (CPUMASK_TESTNZERO(mask)) { + CPUMASK_ASSNBMASK(tmpmask, cpuid); + if (CPUMASK_TESTMASK(tmpmask, mask)) { + CPUMASK_ANDMASK(tmpmask, mask); + cpuid = BSFCPUMASK(tmpmask); + } else { cpuid = BSFCPUMASK(mask); + } rdd = &dfly_pcpu[cpuid]; if ((rdd->upri & ~PPQMASK) >= (lp->lwp_priority & ~PPQMASK)) goto found; - mask &= ~CPUMASK(cpuid); + CPUMASK_NANDBIT(mask, cpuid); } /* * Then cpus which might have a currently running lp */ cpuid = (dfly_scancpu & 0xFFFF) % ncpus; - mask = dfly_curprocmask & dfly_rdyprocmask & - lp->lwp_cpumask & smp_active_mask & usched_global_cpumask; - - while (mask) { - tmpmask = ~(CPUMASK(cpuid) - 1); - if (mask & tmpmask) - cpuid = BSFCPUMASK(mask & tmpmask); - else + mask = dfly_rdyprocmask; + CPUMASK_ANDMASK(mask, dfly_curprocmask); + CPUMASK_ANDMASK(mask, lp->lwp_cpumask); + CPUMASK_ANDMASK(mask, smp_active_mask); + CPUMASK_ANDMASK(mask, usched_global_cpumask); + + while (CPUMASK_TESTNZERO(mask)) { + CPUMASK_ASSNBMASK(tmpmask, cpuid); + if (CPUMASK_TESTMASK(tmpmask, mask)) { + CPUMASK_ANDMASK(tmpmask, mask); + cpuid = BSFCPUMASK(tmpmask); + } else { cpuid = BSFCPUMASK(mask); + } rdd = &dfly_pcpu[cpuid]; if ((rdd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) goto found; - mask &= ~CPUMASK(cpuid); + CPUMASK_NANDBIT(mask, cpuid); } /* @@ -1793,7 +1807,7 @@ dfly_choose_queue_simple(dfly_pcpu_t dd, struct lwp *lp) * set the user resched flag because */ cpuid = (dfly_scancpu & 0xFFFF) % ncpus; - if ((CPUMASK(cpuid) & usched_global_cpumask) == 0) + if (CPUMASK_TESTBIT(usched_global_cpumask, cpuid) == 0) cpuid = 0; rdd = &dfly_pcpu[cpuid]; found: @@ -1822,8 +1836,9 @@ dfly_need_user_resched_remote(void *dummy) * * Call wakeup_mycpu to avoid sending IPIs to other CPUs */ - if (dd->uschedcp == NULL && (dfly_rdyprocmask & gd->gd_cpumask)) { - atomic_clear_cpumask(&dfly_rdyprocmask, gd->gd_cpumask); + if (dd->uschedcp == NULL && + CPUMASK_TESTBIT(dfly_rdyprocmask, gd->gd_cpuid)) { + ATOMIC_CPUMASK_NANDBIT(dfly_rdyprocmask, gd->gd_cpuid); wakeup_mycpu(&dd->helper_thread); } } @@ -2009,7 +2024,7 @@ dfly_helper_thread(void *dummy) spin_lock(&dd->spin); - atomic_set_cpumask(&dfly_rdyprocmask, mask); + ATOMIC_CPUMASK_ORMASK(dfly_rdyprocmask, mask); clear_user_resched(); /* This satisfied the reschedule request */ #if 0 dd->rrcount = 0; /* Reset the round-robin counter */ @@ -2023,7 +2038,7 @@ dfly_helper_thread(void *dummy) */ nlp = dfly_chooseproc_locked(dd, dd, dd->uschedcp, 0); if (nlp) { - atomic_set_cpumask(&dfly_curprocmask, mask); + ATOMIC_CPUMASK_ORMASK(dfly_curprocmask, mask); dd->upri = nlp->lwp_priority; dd->uschedcp = nlp; #if 0 @@ -2063,7 +2078,7 @@ dfly_helper_thread(void *dummy) nlp = NULL; } if (nlp) { - atomic_set_cpumask(&dfly_curprocmask, mask); + ATOMIC_CPUMASK_ORMASK(dfly_curprocmask, mask); dd->upri = nlp->lwp_priority; dd->uschedcp = nlp; #if 0 @@ -2140,21 +2155,22 @@ usched_dfly_cpu_init(void) for (i = 0; i < ncpus; ++i) { dfly_pcpu_t dd = &dfly_pcpu[i]; - cpumask_t mask = CPUMASK(i); + cpumask_t mask; - if ((mask & smp_active_mask) == 0) + CPUMASK_ASSBIT(mask, i); + if (CPUMASK_TESTMASK(mask, smp_active_mask) == 0) continue; spin_init(&dd->spin); dd->cpunode = get_cpu_node_by_cpuid(i); dd->cpuid = i; - dd->cpumask = CPUMASK(i); + CPUMASK_ASSBIT(dd->cpumask, i); for (j = 0; j < NQS; j++) { TAILQ_INIT(&dd->queues[j]); TAILQ_INIT(&dd->rtqueues[j]); TAILQ_INIT(&dd->idqueues[j]); } - atomic_clear_cpumask(&dfly_curprocmask, 1); + ATOMIC_CPUMASK_NANDBIT(dfly_curprocmask, 0); if (dd->cpunode == NULL) { smt_not_supported = 1; @@ -2219,8 +2235,8 @@ usched_dfly_cpu_init(void) * been enabled in rqinit(). */ if (i) - atomic_clear_cpumask(&dfly_curprocmask, mask); - atomic_set_cpumask(&dfly_rdyprocmask, mask); + ATOMIC_CPUMASK_NANDMASK(dfly_curprocmask, mask); + ATOMIC_CPUMASK_ORMASK(dfly_rdyprocmask, mask); dd->upri = PRIBASE_NULL; } diff --git a/sys/kern/usched_dummy.c b/sys/kern/usched_dummy.c index 919e27a0a9..e747c8f8f1 100644 --- a/sys/kern/usched_dummy.c +++ b/sys/kern/usched_dummy.c @@ -102,7 +102,7 @@ struct usched_dummy_pcpu { typedef struct usched_dummy_pcpu *dummy_pcpu_t; static struct usched_dummy_pcpu dummy_pcpu[MAXCPU]; -static cpumask_t dummy_curprocmask = -1; +static cpumask_t dummy_curprocmask = CPUMASK_INITIALIZER_ALLONES; static cpumask_t dummy_rdyprocmask; static struct spinlock dummy_spin; static TAILQ_HEAD(rq, lwp) dummy_runq; @@ -121,7 +121,7 @@ dummyinit(void *dummy) { TAILQ_INIT(&dummy_runq); spin_init(&dummy_spin); - atomic_clear_cpumask(&dummy_curprocmask, 1); + ATOMIC_CPUMASK_NANDBIT(dummy_curprocmask, 0); } SYSINIT(runqueue, SI_BOOT2_USCHED, SI_ORDER_FIRST, dummyinit, NULL) @@ -164,7 +164,7 @@ dummy_acquire_curproc(struct lwp *lp) */ if (dd->uschedcp == lp || (dd->uschedcp == NULL && TAILQ_EMPTY(&dummy_runq))) { - atomic_set_cpumask(&dummy_curprocmask, gd->gd_cpumask); + ATOMIC_CPUMASK_ORBIT(dummy_curprocmask, gd->gd_cpuid); dd->uschedcp = lp; return; } @@ -243,14 +243,14 @@ dummy_select_curproc(globaldata_t gd) spin_lock(&dummy_spin); if ((lp = TAILQ_FIRST(&dummy_runq)) == NULL) { dd->uschedcp = NULL; - atomic_clear_cpumask(&dummy_curprocmask, gd->gd_cpumask); + ATOMIC_CPUMASK_NANDBIT(dummy_curprocmask, gd->gd_cpuid); spin_unlock(&dummy_spin); } else { --dummy_runqcount; TAILQ_REMOVE(&dummy_runq, lp, lwp_procq); atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ); dd->uschedcp = lp; - atomic_set_cpumask(&dummy_curprocmask, gd->gd_cpumask); + ATOMIC_CPUMASK_ORBIT(dummy_curprocmask, gd->gd_cpuid); spin_unlock(&dummy_spin); lwkt_acquire(lp->lwp_thread); lwkt_schedule(lp->lwp_thread); @@ -280,7 +280,7 @@ dummy_setrunqueue(struct lwp *lp) if (dd->uschedcp == NULL) { dd->uschedcp = lp; - atomic_set_cpumask(&dummy_curprocmask, gd->gd_cpumask); + ATOMIC_CPUMASK_ORBIT(dummy_curprocmask, gd->gd_cpuid); lwkt_schedule(lp->lwp_thread); } else { /* @@ -304,11 +304,12 @@ dummy_setrunqueue(struct lwp *lp) * helper thread cannot find a home for it it will forward * the request to another available cpu. */ - mask = ~dummy_curprocmask & dummy_rdyprocmask & - gd->gd_other_cpus; - if (mask) { + mask = dummy_rdyprocmask; + CPUMASK_NANDMASK(mask, dummy_curprocmask); + CPUMASK_ANDMASK(mask, gd->gd_other_cpus); + if (CPUMASK_TESTNZERO(mask)) { cpuid = BSFCPUMASK(mask); - atomic_clear_cpumask(&dummy_rdyprocmask, CPUMASK(cpuid)); + ATOMIC_CPUMASK_NANDBIT(dummy_rdyprocmask, cpuid); spin_unlock(&dummy_spin); lwkt_schedule(&dummy_pcpu[cpuid].helper_thread); } else { @@ -475,23 +476,24 @@ dummy_sched_thread(void *dummy) gd = mycpu; cpuid = gd->gd_cpuid; dd = &dummy_pcpu[cpuid]; - cpumask = CPUMASK(cpuid); + CPUMASK_ASSBIT(cpumask, cpuid); for (;;) { lwkt_deschedule_self(gd->gd_curthread); /* interlock */ - atomic_set_cpumask(&dummy_rdyprocmask, cpumask); + ATOMIC_CPUMASK_ORBIT(dummy_rdyprocmask, cpuid); spin_lock(&dummy_spin); if (dd->uschedcp) { /* * We raced another cpu trying to schedule a thread onto us. * If the runq isn't empty hit another free cpu. */ - tmpmask = ~dummy_curprocmask & dummy_rdyprocmask & - gd->gd_other_cpus; - if (tmpmask && dummy_runqcount) { + tmpmask = dummy_rdyprocmask; + CPUMASK_NANDMASK(tmpmask, dummy_curprocmask); + CPUMASK_ANDMASK(tmpmask, gd->gd_other_cpus); + if (CPUMASK_TESTNZERO(tmpmask) && dummy_runqcount) { tmpid = BSFCPUMASK(tmpmask); KKASSERT(tmpid != cpuid); - atomic_clear_cpumask(&dummy_rdyprocmask, CPUMASK(tmpid)); + ATOMIC_CPUMASK_NANDBIT(dummy_rdyprocmask, tmpid); spin_unlock(&dummy_spin); lwkt_schedule(&dummy_pcpu[tmpid].helper_thread); } else { @@ -502,7 +504,7 @@ dummy_sched_thread(void *dummy) TAILQ_REMOVE(&dummy_runq, lp, lwp_procq); atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ); dd->uschedcp = lp; - atomic_set_cpumask(&dummy_curprocmask, cpumask); + ATOMIC_CPUMASK_ORBIT(dummy_curprocmask, cpuid); spin_unlock(&dummy_spin); lwkt_acquire(lp->lwp_thread); lwkt_schedule(lp->lwp_thread); @@ -527,9 +529,11 @@ dummy_sched_thread_cpu_init(void) for (i = 0; i < ncpus; ++i) { dummy_pcpu_t dd = &dummy_pcpu[i]; - cpumask_t mask = CPUMASK(i); + cpumask_t mask; + + CPUMASK_ASSBIT(mask, i); - if ((mask & smp_active_mask) == 0) + if (CPUMASK_TESTMASK(mask, smp_active_mask) == 0) continue; if (bootverbose) @@ -543,8 +547,8 @@ dummy_sched_thread_cpu_init(void) * been enabled in rqinit(). */ if (i) - atomic_clear_cpumask(&dummy_curprocmask, mask); - atomic_set_cpumask(&dummy_rdyprocmask, mask); + ATOMIC_CPUMASK_NANDMASK(dummy_curprocmask, mask); + ATOMIC_CPUMASK_ORMASK(dummy_rdyprocmask, mask); } if (bootverbose) kprintf("\n"); diff --git a/sys/net/netisr.c b/sys/net/netisr.c index e428405895..2b90c9aa53 100644 --- a/sys/net/netisr.c +++ b/sys/net/netisr.c @@ -611,8 +611,8 @@ netisr_barrier_dispatch(netmsg_t nmsg) { struct netmsg_barrier *msg = (struct netmsg_barrier *)nmsg; - atomic_clear_cpumask(msg->br_cpumask, mycpu->gd_cpumask); - if (*msg->br_cpumask == 0) + ATOMIC_CPUMASK_NANDBIT(*msg->br_cpumask, mycpu->gd_cpuid); + if (CPUMASK_TESTZERO(*msg->br_cpumask)) wakeup(msg->br_cpumask); for (;;) { @@ -649,7 +649,8 @@ netisr_barrier_set(struct netisr_barrier *br) KKASSERT(&curthread->td_msgport == netisr_cpuport(0)); KKASSERT(!br->br_isset); - other_cpumask = mycpu->gd_other_cpus & smp_active_mask; + other_cpumask = mycpu->gd_other_cpus; + CPUMASK_ANDMASK(other_cpumask, smp_active_mask); cur_cpuid = mycpuid; for (i = 0; i < ncpus; ++i) { @@ -667,7 +668,7 @@ netisr_barrier_set(struct netisr_barrier *br) * the caller. */ netmsg_init(&msg->base, NULL, &netisr_afree_rport, 0, - netisr_barrier_dispatch); + netisr_barrier_dispatch); msg->br_cpumask = &other_cpumask; msg->br_done = NETISR_BR_NOTDONE; @@ -681,9 +682,9 @@ netisr_barrier_set(struct netisr_barrier *br) lwkt_sendmsg(netisr_cpuport(i), &br->br_msgs[i]->base.lmsg); } - while (other_cpumask != 0) { + while (CPUMASK_TESTNZERO(other_cpumask)) { tsleep_interlock(&other_cpumask, 0); - if (other_cpumask != 0) + if (CPUMASK_TESTNZERO(other_cpumask)) tsleep(&other_cpumask, PINTERLOCKED, "nbrset", 0); } br->br_isset = 1; diff --git a/sys/netinet/ip_flow.c b/sys/netinet/ip_flow.c index bd25e4ebbb..039420c7ac 100644 --- a/sys/netinet/ip_flow.c +++ b/sys/netinet/ip_flow.c @@ -431,15 +431,16 @@ ipflow_timo_ipi(void *arg __unused) void ipflow_slowtimo(void) { - cpumask_t mask = 0; + cpumask_t mask; int i; + CPUMASK_ASSZERO(mask); for (i = 0; i < ncpus; ++i) { if (ipflow_inuse_pcpu[i]) - mask |= CPUMASK(i); + CPUMASK_ORBIT(mask, i); } - mask &= smp_active_mask; - if (mask != 0) + CPUMASK_ANDMASK(mask, smp_active_mask); + if (CPUMASK_TESTNZERO(mask)) lwkt_send_ipiq_mask(mask, ipflow_timo_ipi, NULL); } diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c index 5aa234ca79..88c6f81098 100644 --- a/sys/netinet/ip_input.c +++ b/sys/netinet/ip_input.c @@ -1398,13 +1398,11 @@ ipfrag_timeo_ipi(void *arg __unused) static void ipfrag_slowtimo(void) { - cpumask_t mask = 0; - int i; + cpumask_t mask; - for (i = 0; i < ncpus; ++i) - mask |= CPUMASK(i); - mask &= smp_active_mask; - if (mask != 0) + CPUMASK_ASSBMASK(mask, ncpus); + CPUMASK_ANDMASK(mask, smp_active_mask); + if (CPUMASK_TESTNZERO(mask)) lwkt_send_ipiq_mask(mask, ipfrag_timeo_ipi, NULL); } @@ -1456,13 +1454,11 @@ ipfrag_drain_ipi(void *arg __unused) static void ipfrag_drain(void) { - cpumask_t mask = 0; - int i; + cpumask_t mask; - for (i = 0; i < ncpus; ++i) - mask |= CPUMASK(i); - mask &= smp_active_mask; - if (mask != 0) + CPUMASK_ASSBMASK(mask, ncpus); + CPUMASK_ANDMASK(mask, smp_active_mask); + if (CPUMASK_TESTNZERO(mask)) lwkt_send_ipiq_mask(mask, ipfrag_drain_ipi, NULL); } diff --git a/sys/platform/pc32/apic/lapic.c b/sys/platform/pc32/apic/lapic.c index 1bdf72dc98..754db2b25a 100644 --- a/sys/platform/pc32/apic/lapic.c +++ b/sys/platform/pc32/apic/lapic.c @@ -587,7 +587,7 @@ selected_apic_ipi(cpumask_t target, int vector, int delivery_mode) crit_enter(); while (target) { int n = BSFCPUMASK(target); - target &= ~CPUMASK(n); + CPUMASK_NANDBIT(target, n); single_apic_ipi(n, vector, delivery_mode); } crit_exit(); diff --git a/sys/platform/pc32/apic/lapic.h b/sys/platform/pc32/apic/lapic.h index 9a06308920..46249359e9 100644 --- a/sys/platform/pc32/apic/lapic.h +++ b/sys/platform/pc32/apic/lapic.h @@ -84,7 +84,7 @@ int single_apic_ipi_passive(int, int, int); static __inline int all_but_self_ipi(int vector) { - if (smp_active_mask == 1) + if (CPUMASK_ISUP(smp_active_mask)) return 0; return apic_ipi(APIC_DEST_ALLESELF, vector, APIC_DELMODE_FIXED); } diff --git a/sys/platform/pc32/i386/mp_machdep.c b/sys/platform/pc32/i386/mp_machdep.c index 964aeb9ed1..6454027f7a 100644 --- a/sys/platform/pc32/i386/mp_machdep.c +++ b/sys/platform/pc32/i386/mp_machdep.c @@ -173,10 +173,14 @@ static int start_ap(struct mdglobaldata *gd, u_int boot_addr, int smibest); static int smitest(void); static void mp_bsp_simple_setup(void); -static cpumask_t smp_startup_mask = 1; /* which cpus have been started */ -static cpumask_t smp_lapic_mask = 1; /* which cpus have lapic been inited */ -cpumask_t smp_active_mask = 1; /* which cpus are ready for IPIs etc? */ -SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RD, &smp_active_mask, 0, ""); +/* which cpus have been started */ +static cpumask_t smp_startup_mask = CPUMASK_INITIALIZER_ONLYONE; +/* which cpus have lapic been inited */ +static cpumask_t smp_lapic_mask = CPUMASK_INITIALIZER_ONLYONE; +/* which cpus are ready for IPIs etc? */ +cpumask_t smp_active_mask = CPUMASK_INITIALIZER_ONLYONE; +SYSCTL_LONG(_machdep, OID_AUTO, smp_active, CTLFLAG_RD, + &smp_active_mask, 0, ""); /* Local data for detecting CPU TOPOLOGY */ static int core_bits = 0; @@ -473,8 +477,10 @@ start_all_aps(u_int boot_addr) ncpus_fit_mask = ncpus_fit - 1; /* build our map of 'other' CPUs */ - mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); - mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * ncpus); + mycpu->gd_other_cpus = smp_startup_mask; + CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid); + mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, + sizeof(lwkt_ipiq) * ncpus); bzero(mycpu->gd_ipiq, sizeof(lwkt_ipiq) * ncpus); /* restore the warmstart vector */ @@ -500,7 +506,7 @@ start_all_aps(u_int boot_addr) tsc0_offset = rdtsc(); tsc_offsets[0] = 0; rel_mplock(); - while (smp_lapic_mask != smp_startup_mask) { + while (CPUMASK_CMPMASKNEQ(smp_lapic_mask, smp_startup_mask)) { cpu_lfence(); if (cpu_feature & CPUID_TSC) tsc0_offset = rdtsc(); @@ -714,7 +720,7 @@ start_ap(struct mdglobaldata *gd, u_int boot_addr, int smibest) /* wait for it to start, see ap_init() */ set_apic_timer(5000000);/* == 5 seconds */ while (read_apic_timer()) { - if (smp_startup_mask & CPUMASK(gd->mi.gd_cpuid)) + if (CPUMASK_TESTBIT(smp_startup_mask, gd->mi.gd_cpuid)) return 1; /* return SUCCESS */ } @@ -768,27 +774,36 @@ smp_invltlb(void) long count = 0; long xcount = 0; #endif + cpumask_t tmpmask; + cpumask_t tmpmask2; crit_enter_gd(&md->mi); md->gd_invltlb_ret = 0; ++md->mi.gd_cnt.v_smpinvltlb; - atomic_set_cpumask(&smp_invltlb_req, md->mi.gd_cpumask); + ATOMIC_CPUMASK_ORMASK(smp_invltlb_req, md->mi.gd_cpumask); #ifdef SMP_INVLTLB_DEBUG again: #endif - if (smp_startup_mask == smp_active_mask) { + if (CPUMASK_CMPMASKEQ(smp_startup_mask, smp_active_mask)) { all_but_self_ipi(XINVLTLB_OFFSET); } else { - selected_apic_ipi(smp_active_mask & ~md->mi.gd_cpumask, - XINVLTLB_OFFSET, APIC_DELMODE_FIXED); + tmpmask = smp_active_mask; + CPUMASK_NANDMASK(tmpmask, md->mi.gd_cpumask); + selected_apic_ipi(tmpmask, XINVLTLB_OFFSET, APIC_DELMODE_FIXED); } #ifdef SMP_INVLTLB_DEBUG if (xcount) kprintf("smp_invltlb: ipi sent\n"); #endif - while ((md->gd_invltlb_ret & smp_active_mask & ~md->mi.gd_cpumask) != - (smp_active_mask & ~md->mi.gd_cpumask)) { + for (;;) { + tmpmask = smp_active_mask; + tmpmask2 = tmpmask; + CPUMASK_ANDMASK(tmpmask, md->gd_invltlb_ret); + CPUMASK_NANDMASK(tmpmask, md->mi.gd_cpumask); + CPUMASK_NANDMASK(tmpmask2, md->mi.gd_cpumask); + if (CPUMASK_CMPMASKEQ(tmpmask, tmpmask2)) + break; cpu_mfence(); cpu_pause(); #ifdef SMP_INVLTLB_DEBUG @@ -805,13 +820,19 @@ again: if (xcount > 2) lwkt_process_ipiq(); if (xcount > 3) { - int bcpu = BSFCPUMASK(~md->gd_invltlb_ret & - ~md->mi.gd_cpumask & - smp_active_mask); globaldata_t xgd; + int bcpu; + + tmpmask = smp_active_mask; + CPUMASK_NANDMASK(tmpmask, md->gd_invltlb_ret); + CPUMASK_NANDMASK(tmpmask, md->mi.gd_cpumask); + bcpu = BSFCPUMASK(tmpmask); + kprintf("bcpu %d\n", bcpu); xgd = globaldata_find(bcpu); - kprintf("thread %p %s\n", xgd->gd_curthread, xgd->gd_curthread->td_comm); + kprintf("thread %p %s\n", + xgd->gd_curthread, + xgd->gd_curthread->td_comm); } if (xcount > 5) panic("giving up"); @@ -820,7 +841,7 @@ again: } #endif } - atomic_clear_cpumask(&smp_invltlb_req, md->mi.gd_cpumask); + ATOMIC_CPUMASK_NANDMASK(smp_invltlb_req, md->mi.gd_cpumask); crit_exit_gd(&md->mi); } @@ -840,11 +861,11 @@ smp_invltlb_intr(void) mask = smp_invltlb_req; cpu_mfence(); cpu_invltlb(); - while (mask) { + while (CPUMASK_TESTNZERO(mask)) { cpu = BSFCPUMASK(mask); - mask &= ~CPUMASK(cpu); + CPUMASK_NANDBIT(mask, cpu); omd = (struct mdglobaldata *)globaldata_find(cpu); - atomic_set_cpumask(&omd->gd_invltlb_ret, md->mi.gd_cpumask); + ATOMIC_CPUMASK_ORMASK(omd->gd_invltlb_ret, md->mi.gd_cpumask); } } @@ -874,13 +895,18 @@ cpu_wbinvd_on_all_cpus_callback(void *arg) int stop_cpus(cpumask_t map) { + cpumask_t tmpmask; + map &= smp_active_mask; /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); - - while ((stopped_cpus & map) != map) + + do { + tmpmask = map; + CPUMASK_ANDMASK(tmpmask, stopped_cpus); /* spin */ ; + } while (CPUMASK_CMPMASKNEQ(tmpmask, map)); return 1; } @@ -905,7 +931,8 @@ restart_cpus(cpumask_t map) /* signal other cpus to restart */ started_cpus = map & smp_active_mask; - while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */ + /* wait for each to clear its bit */ + while (CPUMASK_TESTMASK(stopped_cpus, map) != 0) /* spin */ ; return 1; @@ -932,7 +959,7 @@ ap_init(void) * interrupts physically disabled and remote cpus could deadlock * trying to send us an IPI. */ - smp_startup_mask |= CPUMASK(mycpu->gd_cpuid); + ATOMIC_CPUMASK_ORBIT(smp_startup_mask, mycpu->gd_cpuid); cpu_mfence(); /* @@ -968,7 +995,8 @@ ap_init(void) #endif /* Build our map of 'other' CPUs. */ - mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask; + CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid); /* A quick check from sanity claus */ cpu_id = APICID_TO_CPUID((lapic->id & 0xff000000) >> 24); @@ -983,7 +1011,7 @@ ap_init(void) lapic_init(FALSE); /* LAPIC initialization is done */ - smp_lapic_mask |= CPUMASK(mycpu->gd_cpuid); + ATOMIC_CPUMASK_ORBIT(smp_lapic_mask, mycpu->gd_cpuid); cpu_mfence(); /* Let BSP move onto the next initialization stage */ @@ -1027,7 +1055,7 @@ ap_init(void) * nothing we've done put it there. */ KKASSERT(get_mplock_count(curthread) == 1); - smp_active_mask |= CPUMASK(mycpu->gd_cpuid); + ATOMIC_CPUMASK_ORBIT(smp_active_mask, mycpu->gd_cpuid); /* * Enable interrupts here. idle_restore will also do it, but @@ -1072,7 +1100,7 @@ SYSINIT(finishsmp, SI_BOOT2_FINISH_SMP, SI_ORDER_FIRST, ap_finish, NULL) void cpu_send_ipiq(int dcpu) { - if (CPUMASK(dcpu) & smp_active_mask) + if (CPUMASK_TESTBIT(smp_active_mask, dcpu)) single_apic_ipi(dcpu, XIPIQ_OFFSET, APIC_DELMODE_FIXED); } @@ -1084,7 +1112,7 @@ int cpu_send_ipiq_passive(int dcpu) { int r = 0; - if (CPUMASK(dcpu) & smp_active_mask) { + if (CPUMASK_TESTBIT(smp_active_mask, dcpu)) { r = single_apic_ipi_passive(dcpu, XIPIQ_OFFSET, APIC_DELMODE_FIXED); } @@ -1096,8 +1124,10 @@ static void mp_bsp_simple_setup(void) { /* build our map of 'other' CPUs */ - mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); - mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * ncpus); + mycpu->gd_other_cpus = smp_startup_mask; + CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid); + mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, + sizeof(lwkt_ipiq) * ncpus); bzero(mycpu->gd_ipiq, sizeof(lwkt_ipiq) * ncpus); pmap_set_opt(); diff --git a/sys/platform/pc32/i386/mptable.c b/sys/platform/pc32/i386/mptable.c index 9da1895777..4d5b163f04 100644 --- a/sys/platform/pc32/i386/mptable.c +++ b/sys/platform/pc32/i386/mptable.c @@ -449,7 +449,7 @@ mptable_hyperthread_fixup(cpumask_t id_mask, int cpu_count) */ dist = cur = prev = -1; for (id = 0; id < MAXCPU; ++id) { - if ((id_mask & CPUMASK(id)) == 0) + if (CPUMASK_TESTBIT(id_mask, id) == 0) continue; cur = id; @@ -490,13 +490,13 @@ mptable_hyperthread_fixup(cpumask_t id_mask, int cpu_count) * already in the table, then kill the fixup. */ for (id = 0; id < MAXCPU; id++) { - if ((id_mask & CPUMASK(id)) == 0) + if (CPUMASK_TESTBIT(id_mask, id) == 0) continue; /* First, make sure we are on a logical_cpus boundary. */ if (id % logical_cpus != 0) return 0; for (i = id + 1; i < id + logical_cpus; i++) - if ((id_mask & CPUMASK(i)) != 0) + if (CPUMASK_TESTBIT(id_mask, i)) return 0; } return logical_cpus; diff --git a/sys/platform/pc32/i386/pmap.c b/sys/platform/pc32/i386/pmap.c index ac609ad8a1..704d54e1d9 100644 --- a/sys/platform/pc32/i386/pmap.c +++ b/sys/platform/pc32/i386/pmap.c @@ -834,10 +834,10 @@ get_ptbase(pmap_t pmap) if ((*gd->gd_GDMAP1 & PG_FRAME) != frame) { *gd->gd_GDMAP1 = frame | PG_RW | PG_V; - pmap->pm_cached |= gd->mi.gd_cpumask; + CPUMASK_ORMASK(pmap->pm_cached, gd->mi.gd_cpumask); cpu_invltlb(); - } else if ((pmap->pm_cached & gd->mi.gd_cpumask) == 0) { - pmap->pm_cached |= gd->mi.gd_cpumask; + } else if (CPUMASK_TESTMASK(pmap->pm_cached, gd->mi.gd_cpumask) == 0) { + CPUMASK_ORMASK(pmap->pm_cached, gd->mi.gd_cpumask); cpu_invltlb(); } else if (dreadful_invltlb) { cpu_invltlb(); @@ -3775,7 +3775,8 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) lp->lwp_vmspace = newvm; if (curthread->td_lwp == lp) { pmap = vmspace_pmap(newvm); - atomic_set_cpumask(&pmap->pm_active, mycpu->gd_cpumask); + ATOMIC_CPUMASK_ORMASK(pmap->pm_active, + mycpu->gd_cpumask); if (pmap->pm_active_lock & CPULOCK_EXCL) pmap_interlock_wait(newvm); #if defined(SWTCH_OPTIM_STATS) @@ -3784,8 +3785,8 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) curthread->td_pcb->pcb_cr3 = vtophys(pmap->pm_pdir); load_cr3(curthread->td_pcb->pcb_cr3); pmap = vmspace_pmap(oldvm); - atomic_clear_cpumask(&pmap->pm_active, - mycpu->gd_cpumask); + ATOMIC_CPUMASK_NANDMASK(pmap->pm_active, + mycpu->gd_cpumask); } } } diff --git a/sys/platform/pc32/i386/vm_machdep.c b/sys/platform/pc32/i386/vm_machdep.c index c16f4fde69..114fa730f9 100644 --- a/sys/platform/pc32/i386/vm_machdep.c +++ b/sys/platform/pc32/i386/vm_machdep.c @@ -382,9 +382,11 @@ cpu_reset(void) int cnt; kprintf("cpu_reset called on cpu#%d\n",mycpu->gd_cpuid); - map = mycpu->gd_other_cpus & ~stopped_cpus & smp_active_mask; + map = mycpu->gd_other_cpus; + CPUMASK_NANDMASK(map, stopped_cpus); + CPUMASK_ANDMASK(map, smp_active_mask); - if (map != 0) { + if (CPUMASK_TESTNZERO(map)) { kprintf("cpu_reset: Stopping other CPUs\n"); stop_cpus(map); /* Stop all other CPUs */ } diff --git a/sys/platform/pc64/apic/lapic.c b/sys/platform/pc64/apic/lapic.c index 50a68487b4..f39396d6f0 100644 --- a/sys/platform/pc64/apic/lapic.c +++ b/sys/platform/pc64/apic/lapic.c @@ -654,9 +654,9 @@ void selected_apic_ipi(cpumask_t target, int vector, int delivery_mode) { crit_enter(); - while (target) { + while (CPUMASK_TESTNZERO(target)) { int n = BSFCPUMASK(target); - target &= ~CPUMASK(n); + CPUMASK_NANDBIT(target, n); single_apic_ipi(n, vector, delivery_mode); } crit_exit(); diff --git a/sys/platform/pc64/apic/lapic.h b/sys/platform/pc64/apic/lapic.h index b308d958bc..bf607c0a63 100644 --- a/sys/platform/pc64/apic/lapic.h +++ b/sys/platform/pc64/apic/lapic.h @@ -85,7 +85,7 @@ int single_apic_ipi_passive(int, int, int); static __inline int all_but_self_ipi(int vector) { - if (smp_active_mask == 1) + if (CPUMASK_ISUP(smp_active_mask)) return 0; return apic_ipi(APIC_DEST_ALLESELF, vector, APIC_DELMODE_FIXED); } diff --git a/sys/platform/pc64/isa/clock.c b/sys/platform/pc64/isa/clock.c index fff4d577e6..5b97da2138 100644 --- a/sys/platform/pc64/isa/clock.c +++ b/sys/platform/pc64/isa/clock.c @@ -1239,7 +1239,7 @@ tsc_mpsync_test(void) crit_enter(); lwkt_cpusync_init(&cs, gd->gd_other_cpus, - tsc_mpsync_test_remote, NULL); + tsc_mpsync_test_remote, NULL); lwkt_cpusync_interlock(&cs); tsc_mpsync_target = rdtsc(); cpu_mfence(); diff --git a/sys/platform/pc64/vmm/vmx.c b/sys/platform/pc64/vmm/vmx.c index f038077bd1..662d1d8009 100644 --- a/sys/platform/pc64/vmm/vmx.c +++ b/sys/platform/pc64/vmm/vmx.c @@ -338,7 +338,8 @@ vmx_init(void) PROCBASED_ACTIVATE_SECONDARY_CONTROLS, ONE); if (err) { - kprintf("VMM: PROCBASED_ACTIVATE_SECONDARY_CONTROLS not supported by this CPU\n"); + kprintf("VMM: PROCBASED_ACTIVATE_SECONDARY_CONTROLS not " + "supported by this CPU\n"); return (ENODEV); } vmx_set_default_settings(&vmx_procbased2); @@ -351,44 +352,43 @@ vmx_init(void) PINBASED_EXTERNAL_INTERRUPT_EXITING, ONE); if (err) { - kprintf("VMM: PINBASED_EXTERNAL_INTERRUPT_EXITING not supported by this CPU\n"); + kprintf("VMM: PINBASED_EXTERNAL_INTERRUPT_EXITING not " + "supported by this CPU\n"); return (ENODEV); } /* Enable non-maskable interrupts exiting */ - err = vmx_set_ctl_setting(&vmx_pinbased, - PINBASED_NMI_EXITING, - ONE); + err = vmx_set_ctl_setting(&vmx_pinbased, PINBASED_NMI_EXITING, ONE); if (err) { - kprintf("VMM: PINBASED_NMI_EXITING not supported by this CPU\n"); + kprintf("VMM: PINBASED_NMI_EXITING not " + "supported by this CPU\n"); return (ENODEV); } /* Set 64bits mode for GUEST */ - err = vmx_set_ctl_setting(&vmx_entry, - VMENTRY_IA32e_MODE_GUEST, - ONE); + err = vmx_set_ctl_setting(&vmx_entry, VMENTRY_IA32e_MODE_GUEST, ONE); if (err) { - kprintf("VMM: VMENTRY_IA32e_MODE_GUEST not supported by this CPU\n"); + kprintf("VMM: VMENTRY_IA32e_MODE_GUEST not " + "supported by this CPU\n"); return (ENODEV); } /* Load MSR EFER on enry */ err = vmx_set_ctl_setting(&vmx_entry, - VMENTRY_LOAD_IA32_EFER, - ONE); + VMENTRY_LOAD_IA32_EFER, ONE); if (err) { - kprintf("VMM: VMENTRY_LOAD_IA32_EFER not supported by this CPU\n"); + kprintf("VMM: VMENTRY_LOAD_IA32_EFER not " + "supported by this CPU\n"); return (ENODEV); } /* Set 64bits mode */ err = vmx_set_ctl_setting(&vmx_exit, - VMEXIT_HOST_ADDRESS_SPACE_SIZE, - ONE); + VMEXIT_HOST_ADDRESS_SPACE_SIZE, ONE); if (err) { - kprintf("VMM: VMEXIT_HOST_ADDRESS_SPACE_SIZE not supported by this CPU\n"); + kprintf("VMM: VMEXIT_HOST_ADDRESS_SPACE_SIZE not " + "supported by this CPU\n"); return (ENODEV); } @@ -397,7 +397,8 @@ vmx_init(void) VMEXIT_SAVE_IA32_EFER, ONE); if (err) { - kprintf("VMM: VMEXIT_SAVE_IA32_EFER not supported by this CPU\n"); + kprintf("VMM: VMEXIT_SAVE_IA32_EFER not " + "supported by this CPU\n"); return (ENODEV); } @@ -406,7 +407,8 @@ vmx_init(void) VMEXIT_LOAD_IA32_EFER, ONE); if (err) { - kprintf("VMM: VMEXIT_LOAD_IA32_EFER not supported by this CPU\n"); + kprintf("VMM: VMEXIT_LOAD_IA32_EFER not " + "supported by this CPU\n"); return (ENODEV); } @@ -415,7 +417,8 @@ vmx_init(void) PROCBASED2_ENABLE_EPT, ONE); if (err) { - kprintf("VMM: PROCBASED2_ENABLE_EPT not supported by this CPU\n"); + kprintf("VMM: PROCBASED2_ENABLE_EPT not " + "supported by this CPU\n"); return (ENODEV); } @@ -430,7 +433,8 @@ vmx_init(void) PROCBASED2_ENABLE_VPID, ONE); if (err) { - kprintf("VMM: PROCBASED2_ENABLE_VPID not supported by this CPU\n"); + kprintf("VMM: PROCBASED2_ENABLE_VPID not " + "supported by this CPU\n"); return (ENODEV); } #endif @@ -578,8 +582,11 @@ vmx_enable(void) alloc_vmxon_regions(); for (cpu = 0; cpu < ncpus; cpu++) { + cpumask_t mask; + err = 0; - lwkt_cpusync_simple(CPUMASK(cpu), execute_vmxon, &err); + CPUMASK_ASSBIT(mask, cpu); + lwkt_cpusync_simple(mask, execute_vmxon, &err); if(err) { kprintf("VMM: vmx_enable error %d on cpu%d\n", err, cpu); return err; @@ -598,8 +605,12 @@ vmx_disable(void) kprintf("VMM: vmx_disable not allowed; vmx wasn't enabled\n"); } - for (cpu = 0; cpu < ncpus; cpu++) - lwkt_cpusync_simple(CPUMASK(cpu), execute_vmxoff, NULL); + for (cpu = 0; cpu < ncpus; cpu++) { + cpumask_t mask; + + CPUMASK_ASSBIT(mask, cpu); + lwkt_cpusync_simple(mask, execute_vmxoff, NULL); + } free_vmxon_regions(); @@ -954,6 +965,7 @@ vmx_check_cpu_migration(void) { struct vmx_thread_info * vti; struct globaldata *gd; + cpumask_t mask; int err; gd = mycpu; @@ -968,11 +980,12 @@ vmx_check_cpu_migration(void) * with. The pcpu_info[] check prevents unecessary extra * cpusyncs. */ - dkprintf("VMM: cpusync from %d to %d\n", gd->gd_cpuid, vti->last_cpu); + dkprintf("VMM: cpusync from %d to %d\n", + gd->gd_cpuid, vti->last_cpu); /* Clear the VMCS area if ran on another CPU */ - lwkt_cpusync_simple(CPUMASK(vti->last_cpu), - execute_vmclear, (void *)vti); + CPUMASK_ASSBIT(mask, vti->last_cpu); + lwkt_cpusync_simple(mask, execute_vmclear, (void *)vti); } return 0; error: @@ -1371,7 +1384,7 @@ restart: * cpu that may desire to IPI us after we have successfully * incremented the cpulock counter. */ - atomic_set_cpumask(&td->td_proc->p_vmm_cpumask, gd->gd_cpumask); + ATOMIC_CPUMASK_ORBIT(td->td_proc->p_vmm_cpumask, gd->gd_cpuid); for (;;) { olock = td->td_proc->p_vmm_cpulock; @@ -1392,8 +1405,8 @@ restart: * More complex. After sleeping we have to re-test * everything. */ - atomic_clear_cpumask(&td->td_proc->p_vmm_cpumask, - gd->gd_cpumask); + ATOMIC_CPUMASK_NANDBIT(td->td_proc->p_vmm_cpumask, + gd->gd_cpuid); cpu_enable_intr(); tsleep_interlock(&td->td_proc->p_vmm_cpulock, 0); if (td->td_proc->p_vmm_cpulock & CPULOCK_EXCL) { @@ -1473,8 +1486,8 @@ restart: if (ret == VM_EXIT) { ERROR_IF(vmx_vmexit_loadinfo()); - atomic_clear_cpumask(&td->td_proc->p_vmm_cpumask, - gd->gd_cpumask); + ATOMIC_CPUMASK_NANDBIT(td->td_proc->p_vmm_cpumask, + gd->gd_cpuid); atomic_add_int(&td->td_proc->p_vmm_cpulock, -CPULOCK_INCR); /* WARNING: don't adjust cpulock twice! */ @@ -1524,13 +1537,14 @@ done: kprintf("VMM: vmx_vmrun: returning with success\n"); return 0; error: - atomic_clear_cpumask(&td->td_proc->p_vmm_cpumask, gd->gd_cpumask); + ATOMIC_CPUMASK_NANDBIT(td->td_proc->p_vmm_cpumask, gd->gd_cpuid); atomic_add_int(&td->td_proc->p_vmm_cpulock, -CPULOCK_INCR); cpu_enable_intr(); error2: trap_handle_userenter(td); td->td_lwp->lwp_md.md_regs = save_frame; - KKASSERT((td->td_proc->p_vmm_cpumask & gd->gd_cpumask) == 0); + KKASSERT(CPUMASK_TESTMASK(td->td_proc->p_vmm_cpumask, + gd->gd_cpumask) == 0); /*atomic_clear_cpumask(&td->td_proc->p_vmm_cpumask, gd->gd_cpumask);*/ crit_exit(); kprintf("VMM: vmx_vmrun failed\n"); diff --git a/sys/platform/pc64/x86_64/db_interface.c b/sys/platform/pc64/x86_64/db_interface.c index 50750f611c..0a9bc19ed0 100644 --- a/sys/platform/pc64/x86_64/db_interface.c +++ b/sys/platform/pc64/x86_64/db_interface.c @@ -158,7 +158,8 @@ kdb_trap(int type, int code, struct x86_64_saved_state *regs) crit_enter(); db_printf("\nCPU%d stopping CPUs: 0x%08jx\n", - mycpu->gd_cpuid, (uintmax_t)mycpu->gd_other_cpus); + mycpu->gd_cpuid, + (uintmax_t)CPUMASK_LOWMASK(mycpu->gd_other_cpus)); /* We stop all CPUs except ourselves (obviously) */ stop_cpus(mycpu->gd_other_cpus); @@ -180,14 +181,15 @@ kdb_trap(int type, int code, struct x86_64_saved_state *regs) db_global_jmpbuf_valid = FALSE; db_printf("\nCPU%d restarting CPUs: 0x%016jx\n", - mycpu->gd_cpuid, (uintmax_t)stopped_cpus); + mycpu->gd_cpuid, + (uintmax_t)CPUMASK_LOWMASK(stopped_cpus)); /* Restart all the CPUs we previously stopped */ - if (stopped_cpus != mycpu->gd_other_cpus) { + if (CPUMASK_CMPMASKNEQ(stopped_cpus, mycpu->gd_other_cpus)) { db_printf("whoa, other_cpus: 0x%016jx, " "stopped_cpus: 0x%016jx\n", - (uintmax_t)mycpu->gd_other_cpus, - (uintmax_t)stopped_cpus); + (uintmax_t)CPUMASK_LOWMASK(mycpu->gd_other_cpus), + (uintmax_t)CPUMASK_LOWMASK(stopped_cpus)); panic("stop_cpus() failed"); } restart_cpus(stopped_cpus); diff --git a/sys/platform/pc64/x86_64/mp_machdep.c b/sys/platform/pc64/x86_64/mp_machdep.c index d73156ce6e..564ccf9516 100644 --- a/sys/platform/pc64/x86_64/mp_machdep.c +++ b/sys/platform/pc64/x86_64/mp_machdep.c @@ -168,9 +168,13 @@ static int start_ap(struct mdglobaldata *gd, u_int boot_addr, int smibest); static int smitest(void); static void mp_bsp_simple_setup(void); -static cpumask_t smp_startup_mask = 1; /* which cpus have been started */ -static cpumask_t smp_lapic_mask = 1; /* which cpus have lapic been inited */ -cpumask_t smp_active_mask = 1; /* which cpus are ready for IPIs etc? */ +/* which cpus have been started */ +static cpumask_t smp_startup_mask = CPUMASK_INITIALIZER_ONLYONE; +/* which cpus have lapic been inited */ +static cpumask_t smp_lapic_mask = CPUMASK_INITIALIZER_ONLYONE; +/* which cpus are ready for IPIs etc? */ +cpumask_t smp_active_mask = CPUMASK_INITIALIZER_ONLYONE; + SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RD, &smp_active_mask, 0, ""); static u_int bootMP_size; @@ -486,8 +490,10 @@ start_all_aps(u_int boot_addr) ncpus_fit_mask = ncpus_fit - 1; /* build our map of 'other' CPUs */ - mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); - mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * ncpus); + mycpu->gd_other_cpus = smp_startup_mask; + CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid); + mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, + sizeof(lwkt_ipiq) * ncpus); bzero(mycpu->gd_ipiq, sizeof(lwkt_ipiq) * ncpus); /* restore the warmstart vector */ @@ -511,7 +517,7 @@ start_all_aps(u_int boot_addr) tsc0_offset = rdtsc(); tsc_offsets[0] = 0; rel_mplock(); - while (smp_lapic_mask != smp_startup_mask) { + while (CPUMASK_CMPMASKNEQ(smp_lapic_mask, smp_startup_mask)) { cpu_lfence(); if (cpu_feature & CPUID_TSC) tsc0_offset = rdtsc(); @@ -729,7 +735,7 @@ start_ap(struct mdglobaldata *gd, u_int boot_addr, int smibest) /* wait for it to start, see ap_init() */ set_apic_timer(5000000);/* == 5 seconds */ while (read_apic_timer()) { - if (smp_startup_mask & CPUMASK(gd->mi.gd_cpuid)) + if (CPUMASK_TESTBIT(smp_startup_mask, gd->mi.gd_cpuid)) return 1; /* return SUCCESS */ } @@ -786,27 +792,37 @@ smp_invltlb(void) long count = 0; long xcount = 0; #endif + cpumask_t tmpmask; + cpumask_t tmpmask2; crit_enter_gd(&md->mi); - md->gd_invltlb_ret = 0; + CPUMASK_ASSZERO(md->gd_invltlb_ret); ++md->mi.gd_cnt.v_smpinvltlb; - atomic_set_cpumask(&smp_invltlb_req, md->mi.gd_cpumask); + ATOMIC_CPUMASK_ORBIT(smp_invltlb_req, md->mi.gd_cpuid); #ifdef SMP_INVLTLB_DEBUG again: #endif - if (smp_startup_mask == smp_active_mask) { + if (CPUMASK_CMPMASKEQ(smp_startup_mask, smp_active_mask)) { all_but_self_ipi(XINVLTLB_OFFSET); } else { - selected_apic_ipi(smp_active_mask & ~md->mi.gd_cpumask, - XINVLTLB_OFFSET, APIC_DELMODE_FIXED); + tmpmask = smp_active_mask; + CPUMASK_NANDMASK(tmpmask, md->mi.gd_cpumask); + selected_apic_ipi(tmpmask, XINVLTLB_OFFSET, APIC_DELMODE_FIXED); } #ifdef SMP_INVLTLB_DEBUG if (xcount) kprintf("smp_invltlb: ipi sent\n"); #endif - while ((md->gd_invltlb_ret & smp_active_mask & ~md->mi.gd_cpumask) != - (smp_active_mask & ~md->mi.gd_cpumask)) { + for (;;) { + tmpmask = smp_active_mask; + tmpmask2 = tmpmask; + CPUMASK_ANDMASK(tmpmask, md->gd_invltlb_ret); + CPUMASK_NANDMASK(tmpmask, md->mi.gd_cpumask); + CPUMASK_NANDMASK(tmpmask2, md->mi.gd_cpumask); + + if (CPUMASK_CMPMASKEQ(tmpmask, tmpmask2)) + break; cpu_mfence(); cpu_pause(); #ifdef SMP_INVLTLB_DEBUG @@ -815,19 +831,22 @@ again: print_backtrace(-1); kprintf("smp_invltlb: endless loop %08lx %08lx, " "rflags %016jx retry", - (long)md->gd_invltlb_ret, - (long)smp_invltlb_req, + (long)CPUMASK_LOWMASK(md->gd_invltlb_ret), + (long)CPUMASK_LOWMASK(smp_invltlb_req), (intmax_t)read_rflags()); __asm __volatile ("sti"); ++xcount; if (xcount > 2) lwkt_process_ipiq(); if (xcount > 3) { - int bcpu = BSFCPUMASK(~md->gd_invltlb_ret & - ~md->mi.gd_cpumask & - smp_active_mask); + int bcpu; globaldata_t xgd; + tmpmask = smp_active_mask; + CPUMASK_NANDMASK(tmpmask, md->gd_invltlb_ret); + CPUMASK_NANDMASK(tmpmask, md->mi.gd_cpumask); + bcpu = BSFCPUMASK(tmpmask); + kprintf("bcpu %d\n", bcpu); xgd = globaldata_find(bcpu); kprintf("thread %p %s\n", xgd->gd_curthread, xgd->gd_curthread->td_comm); @@ -839,7 +858,7 @@ again: } #endif } - atomic_clear_cpumask(&smp_invltlb_req, md->mi.gd_cpumask); + ATOMIC_CPUMASK_NANDBIT(smp_invltlb_req, md->mi.gd_cpuid); crit_exit_gd(&md->mi); } @@ -859,11 +878,11 @@ smp_invltlb_intr(void) cpu_mfence(); mask = smp_invltlb_req; cpu_invltlb(); - while (mask) { + while (CPUMASK_TESTNZERO(mask)) { cpu = BSFCPUMASK(mask); - mask &= ~CPUMASK(cpu); + CPUMASK_NANDBIT(mask, cpu); omd = (struct mdglobaldata *)globaldata_find(cpu); - atomic_set_cpumask(&omd->gd_invltlb_ret, md->mi.gd_cpumask); + ATOMIC_CPUMASK_ORBIT(omd->gd_invltlb_ret, md->mi.gd_cpuid); } } @@ -905,13 +924,18 @@ smp_invlpg_range_cpusync(void *arg) int stop_cpus(cpumask_t map) { - map &= smp_active_mask; + cpumask_t mask; + + CPUMASK_ANDMASK(map, smp_active_mask); /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); - - while ((stopped_cpus & map) != map) - /* spin */ ; + + do { + mask = stopped_cpus; + CPUMASK_ANDMASK(mask, map); + /* spin */ + } while (CPUMASK_CMPMASKNEQ(mask, map)); return 1; } @@ -933,11 +957,18 @@ stop_cpus(cpumask_t map) int restart_cpus(cpumask_t map) { - /* signal other cpus to restart */ - started_cpus = map & smp_active_mask; + cpumask_t mask; - while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */ - /* spin */ ; + /* signal other cpus to restart */ + mask = map; + CPUMASK_ANDMASK(mask, smp_active_mask); + cpu_ccfence(); + started_cpus = mask; + cpu_ccfence(); + + /* wait for each to clear its bit */ + while (CPUMASK_CMPMASKNEQ(stopped_cpus, map)) + cpu_pause(); return 1; } @@ -963,7 +994,7 @@ ap_init(void) * interrupts physically disabled and remote cpus could deadlock * trying to send us an IPI. */ - smp_startup_mask |= CPUMASK(mycpu->gd_cpuid); + ATOMIC_CPUMASK_ORBIT(smp_startup_mask, mycpu->gd_cpuid); cpu_mfence(); /* @@ -995,7 +1026,8 @@ ap_init(void) cpu_invltlb(); /* Build our map of 'other' CPUs. */ - mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask; + CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid); /* A quick check from sanity claus */ cpu_id = APICID_TO_CPUID((lapic->id & 0xff000000) >> 24); @@ -1013,7 +1045,7 @@ ap_init(void) lapic_init(FALSE); /* LAPIC initialization is done */ - smp_lapic_mask |= CPUMASK(mycpu->gd_cpuid); + CPUMASK_ORBIT(smp_lapic_mask, mycpu->gd_cpuid); cpu_mfence(); /* Let BSP move onto the next initialization stage */ @@ -1057,7 +1089,7 @@ ap_init(void) * nothing we've done put it there. */ KKASSERT(get_mplock_count(curthread) == 1); - smp_active_mask |= CPUMASK(mycpu->gd_cpuid); + CPUMASK_ORBIT(smp_active_mask, mycpu->gd_cpuid); /* * Enable interrupts here. idle_restore will also do it, but @@ -1089,13 +1121,15 @@ ap_finish(void) if (bootverbose) kprintf("Finish MP startup\n"); rel_mplock(); - while (smp_active_mask != smp_startup_mask) + while (CPUMASK_CMPMASKNEQ(smp_active_mask, smp_startup_mask)) { cpu_lfence(); + cpu_pause(); + } while (try_mplock() == 0) ; if (bootverbose) { kprintf("Active CPU Mask: %016jx\n", - (uintmax_t)smp_active_mask); + (uintmax_t)CPUMASK_LOWMASK(smp_active_mask)); } } @@ -1104,7 +1138,7 @@ SYSINIT(finishsmp, SI_BOOT2_FINISH_SMP, SI_ORDER_FIRST, ap_finish, NULL) void cpu_send_ipiq(int dcpu) { - if (CPUMASK(dcpu) & smp_active_mask) + if (CPUMASK_TESTBIT(smp_active_mask, dcpu)) single_apic_ipi(dcpu, XIPIQ_OFFSET, APIC_DELMODE_FIXED); } @@ -1116,7 +1150,7 @@ int cpu_send_ipiq_passive(int dcpu) { int r = 0; - if (CPUMASK(dcpu) & smp_active_mask) { + if (CPUMASK_TESTBIT(smp_active_mask, dcpu)) { r = single_apic_ipi_passive(dcpu, XIPIQ_OFFSET, APIC_DELMODE_FIXED); } @@ -1128,8 +1162,10 @@ static void mp_bsp_simple_setup(void) { /* build our map of 'other' CPUs */ - mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); - mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * ncpus); + mycpu->gd_other_cpus = smp_startup_mask; + CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid); + mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, + sizeof(lwkt_ipiq) * ncpus); bzero(mycpu->gd_ipiq, sizeof(lwkt_ipiq) * ncpus); pmap_set_opt(); @@ -1266,17 +1302,22 @@ amd_get_compute_unit_id(void *arg) int fix_amd_topology(void) { + cpumask_t mask; + if (cpu_vendor_id != CPU_VENDOR_AMD) return -1; if ((amd_feature2 & AMDID2_TOPOEXT) == 0) return -1; - lwkt_cpusync_simple(-1, amd_get_compute_unit_id, NULL); + CPUMASK_ASSALLONES(mask); + lwkt_cpusync_simple(mask, amd_get_compute_unit_id, NULL); kprintf("Compute unit iDS:\n"); int i; - for (i = 0; i < ncpus; i++) - kprintf("%d-%d; \n", i, get_cpu_node_by_cpuid(i)->compute_unit_id); + for (i = 0; i < ncpus; i++) { + kprintf("%d-%d; \n", + i, get_cpu_node_by_cpuid(i)->compute_unit_id); + } return 0; } diff --git a/sys/platform/pc64/x86_64/mptable.c b/sys/platform/pc64/x86_64/mptable.c index 94ada937ab..108c745c02 100644 --- a/sys/platform/pc64/x86_64/mptable.c +++ b/sys/platform/pc64/x86_64/mptable.c @@ -450,7 +450,7 @@ mptable_hyperthread_fixup(cpumask_t id_mask, int cpu_count) */ dist = cur = prev = -1; for (id = 0; id < MAXCPU; ++id) { - if ((id_mask & CPUMASK(id)) == 0) + if (CPUMASK_TESTBIT(id_mask, id) == 0) continue; cur = id; @@ -491,13 +491,13 @@ mptable_hyperthread_fixup(cpumask_t id_mask, int cpu_count) * already in the table, then kill the fixup. */ for (id = 0; id < MAXCPU; id++) { - if ((id_mask & CPUMASK(id)) == 0) + if (CPUMASK_TESTBIT(id_mask, id) == 0) continue; /* First, make sure we are on a logical_cpus boundary. */ if (id % logical_cpus != 0) return 0; for (i = id + 1; i < id + logical_cpus; i++) - if ((id_mask & CPUMASK(i)) != 0) + if (CPUMASK_TESTBIT(id_mask, i) != 0) return 0; } return logical_cpus; @@ -780,9 +780,15 @@ mptable_lapic_enumerate(struct lapic_enumerator *e) KKASSERT(arg1.cpu_count != 0); /* See if we need to fixup HT logical CPUs. */ + /* + * XXX fixup for cpus >= 32 ? XXX + */ if (arg1.ht_fixup) { - logical_cpus = mptable_hyperthread_fixup(arg1.ht_apicid_mask, - arg1.cpu_count); + cpumask_t mask; + + CPUMASK_ASSZERO(mask); + mask.m0 = arg1.ht_apicid_mask; + logical_cpus = mptable_hyperthread_fixup(mask, arg1.cpu_count); if (logical_cpus != 0) arg1.cpu_count *= logical_cpus; } diff --git a/sys/platform/pc64/x86_64/pmap.c b/sys/platform/pc64/x86_64/pmap.c index b9654e5901..2dc7aac4e2 100644 --- a/sys/platform/pc64/x86_64/pmap.c +++ b/sys/platform/pc64/x86_64/pmap.c @@ -907,7 +907,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr) */ kernel_pmap.pm_pml4 = (pdp_entry_t *) (PTOV_OFFSET + KPML4phys); kernel_pmap.pm_count = 1; - kernel_pmap.pm_active = (cpumask_t)-1; + CPUMASK_ASSALLONES(kernel_pmap.pm_active); RB_INIT(&kernel_pmap.pm_pvroot); spin_init(&kernel_pmap.pm_spin); lwkt_token_init(&kernel_pmap.pm_token, "kpmap_tok"); @@ -1617,7 +1617,7 @@ pmap_pinit0(struct pmap *pmap) { pmap->pm_pml4 = (pml4_entry_t *)(PTOV_OFFSET + KPML4phys); pmap->pm_count = 1; - pmap->pm_active = 0; + CPUMASK_ASSZERO(pmap->pm_active); pmap->pm_pvhint = NULL; RB_INIT(&pmap->pm_pvroot); spin_init(&pmap->pm_spin); @@ -1637,7 +1637,7 @@ pmap_pinit_simple(struct pmap *pmap) * Misc initialization */ pmap->pm_count = 1; - pmap->pm_active = 0; + CPUMASK_ASSZERO(pmap->pm_active); pmap->pm_pvhint = NULL; pmap->pm_flags = PMAP_FLAG_SIMPLE; @@ -1737,7 +1737,7 @@ pmap_puninit(pmap_t pmap) pv_entry_t pv; vm_page_t p; - KKASSERT(pmap->pm_active == 0); + KKASSERT(CPUMASK_TESTZERO(pmap->pm_active)); if ((pv = pmap->pm_pmlpv) != NULL) { if (pv_hold_try(pv) == 0) pv_lock(pv); @@ -2275,8 +2275,9 @@ pmap_release(struct pmap *pmap) { struct pmap_release_info info; - KASSERT(pmap->pm_active == 0, - ("pmap still active! %016jx", (uintmax_t)pmap->pm_active)); + KASSERT(CPUMASK_TESTZERO(pmap->pm_active), + ("pmap still active! %016jx", + (uintmax_t)CPUMASK_LOWMASK(pmap->pm_active))); spin_lock(&pmap_spin); TAILQ_REMOVE(&pmap_list, pmap, pm_pmnode); @@ -5219,7 +5220,7 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) lp->lwp_vmspace = newvm; if (curthread->td_lwp == lp) { pmap = vmspace_pmap(newvm); - atomic_set_cpumask(&pmap->pm_active, mycpu->gd_cpumask); + ATOMIC_CPUMASK_ORBIT(pmap->pm_active, mycpu->gd_cpuid); if (pmap->pm_active_lock & CPULOCK_EXCL) pmap_interlock_wait(newvm); #if defined(SWTCH_OPTIM_STATS) @@ -5234,7 +5235,8 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) } load_cr3(curthread->td_pcb->pcb_cr3); pmap = vmspace_pmap(oldvm); - atomic_clear_cpumask(&pmap->pm_active, mycpu->gd_cpumask); + ATOMIC_CPUMASK_NANDBIT(pmap->pm_active, + mycpu->gd_cpuid); } crit_exit(); } @@ -5318,7 +5320,7 @@ pmap_object_free(vm_object_t object) object->md.pmap_rw = NULL; pmap_remove_noinval(pmap, VM_MIN_USER_ADDRESS, VM_MAX_USER_ADDRESS); - pmap->pm_active = 0; + CPUMASK_ASSZERO(pmap->pm_active); pmap_release(pmap); pmap_puninit(pmap); kfree(pmap, M_OBJPMAP); @@ -5327,7 +5329,7 @@ pmap_object_free(vm_object_t object) object->md.pmap_ro = NULL; pmap_remove_noinval(pmap, VM_MIN_USER_ADDRESS, VM_MAX_USER_ADDRESS); - pmap->pm_active = 0; + CPUMASK_ASSZERO(pmap->pm_active); pmap_release(pmap); pmap_puninit(pmap); kfree(pmap, M_OBJPMAP); diff --git a/sys/platform/vkernel/i386/exception.c b/sys/platform/vkernel/i386/exception.c index a78c4beb02..54994c0b81 100644 --- a/sys/platform/vkernel/i386/exception.c +++ b/sys/platform/vkernel/i386/exception.c @@ -113,7 +113,7 @@ stopsig(int nada, siginfo_t *info, void *ctxp) ++curthread->td_critcount; ++mycpu->gd_intr_nesting_level; - while (stopped_cpus & mycpu->gd_cpumask) { + while (CPUMASK_TESTMASK(stopped_cpus, mycpu->gd_cpumask)) { sigsuspend(&ss); } --mycpu->gd_intr_nesting_level; diff --git a/sys/platform/vkernel/i386/mp.c b/sys/platform/vkernel/i386/mp.c index 306b0c1313..5959520cb8 100644 --- a/sys/platform/vkernel/i386/mp.c +++ b/sys/platform/vkernel/i386/mp.c @@ -100,24 +100,21 @@ void ap_finish(void) { int i; - cpumask_t ncpus_mask = 0; - - for (i = 1; i <= ncpus; i++) - ncpus_mask |= CPUMASK(i); mp_finish = 1; if (bootverbose) kprintf("Finish MP startup\n"); /* build our map of 'other' CPUs */ - mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask; + CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid); /* * Let the other cpu's finish initializing and build their map * of 'other' CPUs. */ rel_mplock(); - while (smp_active_mask != smp_startup_mask) { + while (CPUMASK_CMPMASKNEQ(smp_active_mask, smp_startup_mask)) { DELAY(100000); cpu_lfence(); } @@ -196,9 +193,10 @@ mp_announce(void) void cpu_send_ipiq(int dcpu) { - if (CPUMASK(dcpu) & smp_active_mask) + if (CPUMASK_TESTBIT(smp_active_mask, dcpu)) { if (pthread_kill(ap_tids[dcpu], SIGUSR1) != 0) panic("pthread_kill failed in cpu_send_ipiq"); + } #if 0 panic("XXX cpu_send_ipiq()"); #endif @@ -219,9 +217,9 @@ void selected_cpu_ipi(cpumask_t target, int vector, int delivery_mode) { crit_enter(); - while (target) { + while (CPUMASK_TESTNZERO(target)) { int n = BSFCPUMASK(target); - target &= ~CPUMASK(n); + CPUMASK_NANDBIT(target, n); single_cpu_ipi(n, vector, delivery_mode); } crit_exit(); @@ -230,13 +228,13 @@ selected_cpu_ipi(cpumask_t target, int vector, int delivery_mode) int stop_cpus(cpumask_t map) { - map &= smp_active_mask; + CPUMASK_ANDMASK(map, smp_active_mask); crit_enter(); - while (map) { + while (CPUMASK_TESTNZERO(map)) { int n = BSFCPUMASK(map); - map &= ~CPUMASK(n); - stopped_cpus |= CPUMASK(n); + CPUMASK_NANDBIT(map, n); + ATOMIC_CPUMASK_ORBIT(stopped_cpus, n); if (pthread_kill(ap_tids[n], SIGXCPU) != 0) panic("stop_cpus: pthread_kill failed"); } @@ -251,13 +249,13 @@ stop_cpus(cpumask_t map) int restart_cpus(cpumask_t map) { - map &= smp_active_mask; + CPUMASK_ANDMASK(map, smp_active_mask); crit_enter(); - while (map) { + while (CPUMASK_TESTNZERO(map)) { int n = BSFCPUMASK(map); - map &= ~CPUMASK(n); - stopped_cpus &= ~CPUMASK(n); + CPUMASK_NANDBIT(map, n); + ATOMIC_CPUMASK_NANDBIT(stopped_cpus, n); if (pthread_kill(ap_tids[n], SIGXCPU) != 0) panic("restart_cpus: pthread_kill failed"); } @@ -281,7 +279,7 @@ ap_init(void) * interrupts physically disabled and remote cpus could deadlock * trying to send us an IPI. */ - smp_startup_mask |= CPUMASK(mycpu->gd_cpuid); + ATOMIC_CPUMASK_ORBIT(smp_startup_mask, mycpu->gd_cpuid); cpu_mfence(); /* @@ -308,7 +306,8 @@ ap_init(void) cpu_invltlb(); /* Build our map of 'other' CPUs. */ - mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask; + CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid); kprintf("SMP: AP CPU #%d Launched!\n", mycpu->gd_cpuid); @@ -330,7 +329,7 @@ ap_init(void) * nothing we've done put it there. */ KKASSERT(get_mplock_count(curthread) == 1); - smp_active_mask |= CPUMASK(mycpu->gd_cpuid); + ATOMIC_CPUMASK_ORBIT(smp_active_mask, mycpu->gd_cpuid); mdcpu->gd_fpending = 0; mdcpu->gd_ipending = 0; @@ -448,7 +447,7 @@ start_all_aps(u_int boot_addr) pthread_create(&ap_tids[x], NULL, start_ap, NULL); cpu_enable_intr(); - while((smp_startup_mask & CPUMASK(x)) == 0) { + while (CPUMASK_TESTBIT(smp_startup_mask, x) == 0) { cpu_lfence(); /* XXX spin until the AP has started */ DELAY(1000); } diff --git a/sys/platform/vkernel/platform/pmap.c b/sys/platform/vkernel/platform/pmap.c index 19be638ba9..956fc47226 100644 --- a/sys/platform/vkernel/platform/pmap.c +++ b/sys/platform/vkernel/platform/pmap.c @@ -491,19 +491,21 @@ get_ptbase(struct pmap *pmap, vm_offset_t va) KKASSERT(va >= KvaStart && va < KvaEnd); return(KernelPTA + (va >> PAGE_SHIFT)); } else if (pmap->pm_pdir == gd->gd_PT1pdir) { - if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) { + if (CPUMASK_TESTMASK(pmap->pm_cpucachemask, + gd->mi.gd_cpumask) == 0) { *gd->gd_PT1pde = pmap->pm_pdirpte; madvise(gd->gd_PT1map, SEG_SIZE, MADV_INVAL); - atomic_set_cpumask(&pmap->pm_cpucachemask, - gd->mi.gd_cpumask); + ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask, + gd->mi.gd_cpumask); } return(gd->gd_PT1map + (va >> PAGE_SHIFT)); } else if (pmap->pm_pdir == gd->gd_PT2pdir) { - if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) { + if (CPUMASK_TESTMASK(pmap->pm_cpucachemask, + gd->mi.gd_cpumask) == 0) { *gd->gd_PT2pde = pmap->pm_pdirpte; madvise(gd->gd_PT2map, SEG_SIZE, MADV_INVAL); - atomic_set_cpumask(&pmap->pm_cpucachemask, - gd->mi.gd_cpumask); + ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask, + gd->mi.gd_cpumask); } return(gd->gd_PT2map + (va >> PAGE_SHIFT)); } @@ -522,15 +524,15 @@ get_ptbase(struct pmap *pmap, vm_offset_t va) gd->gd_PT1pdir = pmap->pm_pdir; *gd->gd_PT1pde = pmap->pm_pdirpte; madvise(gd->gd_PT1map, SEG_SIZE, MADV_INVAL); - atomic_set_cpumask(&pmap->pm_cpucachemask, - gd->mi.gd_cpumask); + ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask, + gd->mi.gd_cpumask); return(gd->gd_PT1map + (va >> PAGE_SHIFT)); } else { gd->gd_PT2pdir = pmap->pm_pdir; *gd->gd_PT2pde = pmap->pm_pdirpte; madvise(gd->gd_PT2map, SEG_SIZE, MADV_INVAL); - atomic_set_cpumask(&pmap->pm_cpucachemask, - gd->mi.gd_cpumask); + ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask, + gd->mi.gd_cpumask); return(gd->gd_PT2map + (va >> PAGE_SHIFT)); } } @@ -541,18 +543,19 @@ get_ptbase(struct pmap *pmap, vm_offset_t va) */ KKASSERT(IN_CRITICAL_SECT(curthread)); if (pmap->pm_pdir == gd->gd_PT3pdir) { - if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) { + if (CPUMASK_TESTMASK(pmap->pm_cpucachemask, + gd->mi.gd_cpumask) == 0) { *gd->gd_PT3pde = pmap->pm_pdirpte; madvise(gd->gd_PT3map, SEG_SIZE, MADV_INVAL); - atomic_set_cpumask(&pmap->pm_cpucachemask, - gd->mi.gd_cpumask); + ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask, + gd->mi.gd_cpumask); } } else { gd->gd_PT3pdir = pmap->pm_pdir; *gd->gd_PT3pde = pmap->pm_pdirpte; madvise(gd->gd_PT3map, SEG_SIZE, MADV_INVAL); - atomic_set_cpumask(&pmap->pm_cpucachemask, - gd->mi.gd_cpumask); + ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask, + gd->mi.gd_cpumask); } return(gd->gd_PT3map + (va >> PAGE_SHIFT)); } @@ -566,11 +569,12 @@ get_ptbase1(struct pmap *pmap, vm_offset_t va) KKASSERT(va >= KvaStart && va < KvaEnd); return(KernelPTA + (va >> PAGE_SHIFT)); } else if (pmap->pm_pdir == gd->gd_PT1pdir) { - if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) { + if (CPUMASK_TESTMASK(pmap->pm_cpucachemask, + gd->mi.gd_cpumask) == 0) { *gd->gd_PT1pde = pmap->pm_pdirpte; madvise(gd->gd_PT1map, SEG_SIZE, MADV_INVAL); - atomic_set_cpumask(&pmap->pm_cpucachemask, - gd->mi.gd_cpumask); + ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask, + gd->mi.gd_cpumask); } return(gd->gd_PT1map + (va >> PAGE_SHIFT)); } @@ -591,11 +595,12 @@ get_ptbase2(struct pmap *pmap, vm_offset_t va) KKASSERT(va >= KvaStart && va < KvaEnd); return(KernelPTA + (va >> PAGE_SHIFT)); } else if (pmap->pm_pdir == gd->gd_PT2pdir) { - if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) { + if (CPUMASK_TESTMASK(pmap->pm_cpucachemask, + gd->mi.gd_cpumask) == 0) { *gd->gd_PT2pde = pmap->pm_pdirpte; madvise(gd->gd_PT2map, SEG_SIZE, MADV_INVAL); - atomic_set_cpumask(&pmap->pm_cpucachemask, - gd->mi.gd_cpumask); + ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask, + gd->mi.gd_cpumask); } return(gd->gd_PT2map + (va >> PAGE_SHIFT)); } @@ -3072,12 +3077,14 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) lp->lwp_vmspace = newvm; if (curthread->td_lwp == lp) { pmap = vmspace_pmap(newvm); - atomic_set_cpumask(&pmap->pm_active, mycpu->gd_cpumask); + ATOMIC_CPUMASK_ORMASK(pmap->pm_active, + gd->mi.gd_cpumask); #if defined(SWTCH_OPTIM_STATS) tlb_flush_count++; #endif pmap = vmspace_pmap(oldvm); - atomic_clear_cpumask(&pmap->pm_active, mycpu->gd_cpumask); + ATOMIC_CPUMASK_NANDMASK(pmap->pm_active, + gd->mi.gd_cpumask); } } crit_exit(); diff --git a/sys/platform/vkernel64/platform/pmap.c b/sys/platform/vkernel64/platform/pmap.c index a31c8acc51..190c267da8 100644 --- a/sys/platform/vkernel64/platform/pmap.c +++ b/sys/platform/vkernel64/platform/pmap.c @@ -564,7 +564,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr, int64_t ptov_offset) kernel_pmap.pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys); kernel_pmap.pm_count = 1; /* don't allow deactivation */ - kernel_pmap.pm_active = (cpumask_t)-1; + CPUMASK_ASSALLONES(kernel_pmap.pm_active); kernel_pmap.pm_pteobj = NULL; /* see pmap_init */ TAILQ_INIT(&kernel_pmap.pm_pvlist); TAILQ_INIT(&kernel_pmap.pm_pvlist_free); @@ -3362,12 +3362,12 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm) */ crit_enter(); pmap = vmspace_pmap(newvm); - atomic_set_cpumask(&pmap->pm_active, CPUMASK(mycpu->gd_cpuid)); + ATOMIC_CPUMASK_ORBIT(pmap->pm_active, mycpu->gd_cpuid); #if defined(SWTCH_OPTIM_STATS) tlb_flush_count++; #endif pmap = vmspace_pmap(oldvm); - atomic_clear_cpumask(&pmap->pm_active, CPUMASK(mycpu->gd_cpuid)); + ATOMIC_CPUMASK_NANDBIT(pmap->pm_active, mycpu->gd_cpuid); crit_exit(); } diff --git a/sys/platform/vkernel64/platform/pmap_inval.c b/sys/platform/vkernel64/platform/pmap_inval.c index 81cf6e2355..e658c924c9 100644 --- a/sys/platform/vkernel64/platform/pmap_inval.c +++ b/sys/platform/vkernel64/platform/pmap_inval.c @@ -220,7 +220,8 @@ pmap_inval_pde(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va) if (vmm_enabled == 0) { *ptep = 0; pmap_inval_cpu(pmap, va, SEG_SIZE); - } else if ((pmap->pm_active & mycpu->gd_other_cpus) == 0) { + } else if (CPUMASK_TESTMASK(pmap->pm_active, + mycpu->gd_other_cpus) == 0) { *ptep = 0; vmm_cpu_invltlb(); } else { diff --git a/sys/platform/vkernel64/x86_64/exception.c b/sys/platform/vkernel64/x86_64/exception.c index 001f0b6ce8..ecb0e888b2 100644 --- a/sys/platform/vkernel64/x86_64/exception.c +++ b/sys/platform/vkernel64/x86_64/exception.c @@ -118,7 +118,7 @@ stopsig(int nada, siginfo_t *info, void *ctxp) ++td->td_critcount; ++gd->gd_intr_nesting_level; - while (stopped_cpus & gd->gd_cpumask) { + while (CPUMASK_TESTMASK(stopped_cpus, gd->gd_cpumask)) { sigsuspend(&ss); } --gd->gd_intr_nesting_level; diff --git a/sys/platform/vkernel64/x86_64/mp.c b/sys/platform/vkernel64/x86_64/mp.c index 5ef2b04308..2b615e0167 100644 --- a/sys/platform/vkernel64/x86_64/mp.c +++ b/sys/platform/vkernel64/x86_64/mp.c @@ -104,17 +104,14 @@ void ap_finish(void) { int i; - cpumask_t ncpus_mask = 0; - - for (i = 1; i <= ncpus; i++) - ncpus_mask |= CPUMASK(i); mp_finish = 1; if (bootverbose) kprintf("Finish MP startup\n"); /* build our map of 'other' CPUs */ - mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask; + CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid); /* * Let the other cpu's finish initializing and build their map @@ -198,7 +195,7 @@ mp_announce(void) void cpu_send_ipiq(int dcpu) { - if (CPUMASK(dcpu) & smp_active_mask) { + if (CPUMASK_TESTBIT(smp_active_mask, dcpu)) { if (pthread_kill(ap_tids[dcpu], SIGUSR1) != 0) panic("pthread_kill failed in cpu_send_ipiq"); } @@ -217,9 +214,9 @@ void selected_cpu_ipi(cpumask_t target, int vector, int delivery_mode) { crit_enter(); - while (target) { + while (CPUMASK_TESTNZERO(target)) { int n = BSFCPUMASK(target); - target &= ~CPUMASK(n); + CPUMASK_NANDBIT(target, n); single_cpu_ipi(n, vector, delivery_mode); } crit_exit(); @@ -228,13 +225,13 @@ selected_cpu_ipi(cpumask_t target, int vector, int delivery_mode) int stop_cpus(cpumask_t map) { - map &= smp_active_mask; + CPUMASK_ANDMASK(map, smp_active_mask); crit_enter(); - while (map) { + while (CPUMASK_TESTNZERO(map)) { int n = BSFCPUMASK(map); - map &= ~CPUMASK(n); - stopped_cpus |= CPUMASK(n); + CPUMASK_NANDBIT(map, n); + ATOMIC_CPUMASK_ORBIT(stopped_cpus, n); if (pthread_kill(ap_tids[n], SIGXCPU) != 0) panic("stop_cpus: pthread_kill failed"); } @@ -249,13 +246,13 @@ stop_cpus(cpumask_t map) int restart_cpus(cpumask_t map) { - map &= smp_active_mask; + CPUMASK_ANDMASK(map, smp_active_mask); crit_enter(); - while (map) { + while (CPUMASK_TESTNZERO(map)) { int n = BSFCPUMASK(map); - map &= ~CPUMASK(n); - stopped_cpus &= ~CPUMASK(n); + CPUMASK_NANDBIT(map, n); + ATOMIC_CPUMASK_NANDBIT(stopped_cpus, n); if (pthread_kill(ap_tids[n], SIGXCPU) != 0) panic("restart_cpus: pthread_kill failed"); } @@ -278,7 +275,7 @@ ap_init(void) * interrupts physically disabled and remote cpus could deadlock * trying to send us an IPI. */ - smp_startup_mask |= CPUMASK(mycpu->gd_cpuid); + ATOMIC_CPUMASK_ORBIT(smp_startup_mask, mycpu->gd_cpuid); cpu_mfence(); /* @@ -305,7 +302,8 @@ ap_init(void) cpu_invltlb(); /* Build our map of 'other' CPUs. */ - mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid); + mycpu->gd_other_cpus = smp_startup_mask; + CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid); kprintf("SMP: AP CPU #%d Launched!\n", mycpu->gd_cpuid); @@ -327,7 +325,7 @@ ap_init(void) * nothing we've done put it there. */ KKASSERT(get_mplock_count(curthread) == 1); - smp_active_mask |= CPUMASK(mycpu->gd_cpuid); + ATOMIC_CPUMASK_ORBIT(smp_active_mask, mycpu->gd_cpuid); mdcpu->gd_fpending = 0; mdcpu->gd_ipending = 0; @@ -460,7 +458,7 @@ start_all_aps(u_int boot_addr) pthread_create(&ap_tids[x], &attr, start_ap, NULL); cpu_enable_intr(); - while((smp_startup_mask & CPUMASK(x)) == 0) { + while (CPUMASK_TESTBIT(smp_startup_mask, x) == 0) { cpu_lfence(); /* XXX spin until the AP has started */ DELAY(1000); } diff --git a/sys/sys/cpu_topology.h b/sys/sys/cpu_topology.h index 213e30dfec..a079eb356a 100644 --- a/sys/sys/cpu_topology.h +++ b/sys/sys/cpu_topology.h @@ -28,11 +28,9 @@ cpu_node_t *get_cpu_node_by_cpuid(int cpuid); #define CORE_LEVEL 3 #define THREAD_LEVEL 4 -#define CPU_ISSET(n, p) ((CPUMASK(n) & p) != 0) - #define CPUSET_FOREACH(cpu, mask) \ for ((cpu) = 0; (cpu) < ncpus; (cpu)++) \ - if (CPU_ISSET(cpu, mask)) + if (CPUMASK_TESTBIT(mask, cpu)) #endif /* _KERNEL */ diff --git a/sys/sys/globaldata.h b/sys/sys/globaldata.h index 3ac989b57f..8c3d8b0916 100644 --- a/sys/sys/globaldata.h +++ b/sys/sys/globaldata.h @@ -128,7 +128,7 @@ struct globaldata { lwkt_queue gd_tdallq; /* all threads */ lwkt_queue gd_tdrunq; /* runnable threads */ __uint32_t gd_cpuid; - cpumask_t gd_cpumask; /* mask = CPUMASK(cpuid) */ + cpumask_t gd_cpumask; /* CPUMASK_ASSBIT(cpuid) */ cpumask_t gd_other_cpus; /* mask of 'other' cpus */ struct timeval gd_stattv; int gd_intr_nesting_level; /* hard code, intrs, ipis */ diff --git a/sys/sys/param.h b/sys/sys/param.h index 32d9b79bd8..e24ebac129 100644 --- a/sys/sys/param.h +++ b/sys/sys/param.h @@ -162,6 +162,9 @@ #include #endif +/* + * WARNING! Max supported cpu's due to PWAKEUP_CPUMASK is 16384. + */ #define PCATCH 0x00000100 /* tsleep checks signals */ #define PUSRFLAG1 0x00000200 /* Subsystem specific flag */ #define PINTERLOCKED 0x00000400 /* Interlocked tsleep */ diff --git a/usr.sbin/powerd/powerd.c b/usr.sbin/powerd/powerd.c index 0a48c3fcb5..43c7948f15 100644 --- a/usr.sbin/powerd/powerd.c +++ b/usr.sbin/powerd/powerd.c @@ -37,6 +37,7 @@ * via hw.acpi.cpu.px_dom*. */ +#define _KERNEL_STRUCTURES #include #include #include @@ -297,7 +298,7 @@ acpi_setcpufreq(int nstate) /* * Set the mask of cpus the userland scheduler is allowed to use. */ - global_cpumask = (1L << nstate) - 1; + CPUMASK_ASSBMASK(global_cpumask, nstate); sysctlbyname("kern.usched_global_cpumask", NULL, 0, &global_cpumask, sizeof(global_cpumask)); -- 2.41.0