kernel - Refactor cpumask_t to extend cpus past 64, part 1/2
authorMatthew Dillon <dillon@apollo.backplane.com>
Fri, 4 Jul 2014 03:07:27 +0000 (20:07 -0700)
committerMatthew Dillon <dillon@apollo.backplane.com>
Fri, 4 Jul 2014 04:01:54 +0000 (21:01 -0700)
* 64-bit systems only.  32-bit builds use the macros but cannot be expanded
  past 32 cpus.

* Change cpumask_t from __uint64_t to a structure.  This commit implements
  one 64-bit sub-element (the next one will implement four for 256 cpus).

* Create a CPUMASK_*() macro API for non-atomic and atomic cpumask
  manipulation.  These macros generally take lvalues as arguments, allowing
  for a fairly optimal implementation.

* Change all C code operating on cpumask's to use the newly created CPUMASK_*()
  macro API.

* Compile-test 32 and 64-bit.  Run-test 64-bit.

* Adjust sbin/usched, usr.sbin/powerd.  usched currently needs more work.

46 files changed:
sbin/usched/usched.c
sys/cpu/i386/include/types.h
sys/cpu/i386/misc/lwbuf.c
sys/cpu/x86_64/include/types.h
sys/dev/acpica/acpi_cpu.c
sys/dev/powermng/clockmod/clockmod.c
sys/kern/init_main.c
sys/kern/kern_fork.c
sys/kern/kern_proc.c
sys/kern/kern_shutdown.c
sys/kern/kern_synch.c
sys/kern/kern_usched.c
sys/kern/lwkt_ipiq.c
sys/kern/subr_cpu_topology.c
sys/kern/sys_vmm.c
sys/kern/usched_bsd4.c
sys/kern/usched_dfly.c
sys/kern/usched_dummy.c
sys/net/netisr.c
sys/netinet/ip_flow.c
sys/netinet/ip_input.c
sys/platform/pc32/apic/lapic.c
sys/platform/pc32/apic/lapic.h
sys/platform/pc32/i386/mp_machdep.c
sys/platform/pc32/i386/mptable.c
sys/platform/pc32/i386/pmap.c
sys/platform/pc32/i386/vm_machdep.c
sys/platform/pc64/apic/lapic.c
sys/platform/pc64/apic/lapic.h
sys/platform/pc64/isa/clock.c
sys/platform/pc64/vmm/vmx.c
sys/platform/pc64/x86_64/db_interface.c
sys/platform/pc64/x86_64/mp_machdep.c
sys/platform/pc64/x86_64/mptable.c
sys/platform/pc64/x86_64/pmap.c
sys/platform/vkernel/i386/exception.c
sys/platform/vkernel/i386/mp.c
sys/platform/vkernel/platform/pmap.c
sys/platform/vkernel64/platform/pmap.c
sys/platform/vkernel64/platform/pmap_inval.c
sys/platform/vkernel64/x86_64/exception.c
sys/platform/vkernel64/x86_64/mp.c
sys/sys/cpu_topology.h
sys/sys/globaldata.h
sys/sys/param.h
usr.sbin/powerd/powerd.c

index a2ca74e..0bd54a7 100644 (file)
@@ -33,6 +33,7 @@
  * SUCH DAMAGE.
  */
 
+#define _KERNEL_STRUCTURES
 #include <sys/types.h>
 #include <sys/usched.h>
 #include <stdio.h>
@@ -52,8 +53,10 @@ main(int ac, char **av)
        char *sched = NULL;
        char *cpustr = NULL;
        char *sched_cpustr = NULL;
-       cpumask_t cpumask = 0;
-       int cpuid = -1;
+       cpumask_t cpumask;
+       int cpuid;
+
+       CPUMASK_ASSZERO(cpumask);
 
        while ((ch = getopt(ac, av, "d")) != -1) {
                switch (ch) {
@@ -81,8 +84,19 @@ main(int ac, char **av)
                usage();
                /* NOTREACHED */
        }
-       if (cpustr != NULL)
-               cpumask = strtoul(cpustr, NULL, 0);
+
+       /*
+        * XXX needs expanded support for > 64 cpus
+        */
+       if (cpustr != NULL) {
+               unsigned long v;
+
+               v = strtoul(cpustr, NULL, 0);
+               for (cpuid = 0; cpuid < (int)sizeof(v) * 8; ++cpuid) {
+                       if (v & (1LU << cpuid))
+                               CPUMASK_ORBIT(cpumask, cpuid);
+               }
+       }
 
        if (strlen(sched) != 0) {
                if (DebugOpt)
@@ -93,30 +107,34 @@ main(int ac, char **av)
                        exit(1);
                }
        }
-       if (cpumask != 0) {
-               while ((cpumask & 1) == 0) {
-                       cpuid++;
-                       cpumask >>= 1;
+       if (CPUMASK_TESTNZERO(cpumask)) {
+               for (cpuid = 0; cpuid < (int)sizeof(cpumask) * 8; ++cpuid) {
+                       if (CPUMASK_TESTBIT(cpumask, cpuid))
+                               break;
                }
-               cpuid++;
-               cpumask >>= 1;
-               if (DebugOpt)
-                       fprintf(stderr, "DEBUG: USCHED_SET_CPU: cpuid: %d\n", cpuid);
-               res = usched_set(getpid(), USCHED_SET_CPU, &cpuid, sizeof(int));
+               if (DebugOpt) {
+                       fprintf(stderr, "DEBUG: USCHED_SET_CPU: cpuid: %d\n",
+                               cpuid);
+               }
+               res = usched_set(getpid(), USCHED_SET_CPU,
+                                &cpuid, sizeof(int));
                if (res != 0) {
                        perror("usched_set(,USCHED_SET_CPU,,)");
                        exit(1);
                }
-               while (cpumask != 0) {
-                       while ((cpumask & 1) == 0) {
-                               cpuid++;
-                               cpumask >>= 1;
+               CPUMASK_NANDBIT(cpumask, cpuid);
+               while (CPUMASK_TESTNZERO(cpumask)) {
+                       ++cpuid;
+                       if (CPUMASK_TESTBIT(cpumask, cpuid) == 0)
+                               continue;
+                       CPUMASK_NANDBIT(cpumask, cpuid);
+                       if (DebugOpt) {
+                               fprintf(stderr,
+                                       "DEBUG: USCHED_ADD_CPU: cpuid: %d\n",
+                                       cpuid);
                        }
-                       cpuid++;
-                       cpumask >>= 1;
-                       if (DebugOpt)
-                               fprintf(stderr, "DEBUG: USCHED_ADD_CPU: cpuid: %d\n", cpuid);
-                       res = usched_set(getpid(), USCHED_ADD_CPU, &cpuid, sizeof(int));
+                       res = usched_set(getpid(), USCHED_ADD_CPU,
+                                        &cpuid, sizeof(int));
                        if (res != 0) {
                                perror("usched_set(,USCHED_ADD_CPU,,)");
                                exit(1);
@@ -131,6 +149,8 @@ static
 void
 usage(void)
 {
-       fprintf(stderr, "usage: usched [-d] {scheduler[:cpumask] | :cpumask} program [argument ...]\n");
+       fprintf(stderr,
+               "usage: usched [-d] {scheduler[:cpumask] | :cpumask} "
+               "program [argument ...]\n");
        exit(1);
 }
index 226d750..1240852 100644 (file)
@@ -65,9 +65,51 @@ typedef __uint32_t   cpumask_t;      /* mask representing a set of cpus */
 typedef __uint32_t     cpulock_t;      /* count and exclusive lock */
 
 #if defined(_KERNEL) || defined(_KERNEL_STRUCTURES)
-#define CPUMASK(cpu)           (1U << (cpu))
+
+/*
+ * NOTE! CPUMASK_TESTMASK() - caller should only test == 0 or != 0
+ */
+
+#define CPUMASK_INITIALIZER_ALLONES    (cpumask_t)-1
+#define CPUMASK_INITIALIZER_ONLYONE    (cpumask_t)1
+
+#define CPUMASK_SIMPLE(cpu)    (1U << (cpu))
 #define BSRCPUMASK(mask)       bsrl(mask)
 #define BSFCPUMASK(mask)       bsfl(mask)
+
+#define CPUMASK_CMPMASKEQ(mask1, mask2)        ((mask1) == (mask2))
+#define CPUMASK_CMPMASKNEQ(mask1, mask2) ((mask1) != (mask2))
+#define CPUMASK_ISUP(mask)              ((mask) == 1)
+
+#define CPUMASK_TESTZERO(mask)         ((mask) == 0)
+#define CPUMASK_TESTNZERO(mask)                ((mask) != 0)
+#define CPUMASK_TESTBIT(mask, i)       ((mask) & CPUMASK_SIMPLE(i))
+#define CPUMASK_TESTMASK(mask1, mask2) ((mask1) & (mask2))
+#define CPUMASK_LOWMASK(mask)          (mask)
+
+#define CPUMASK_ORBIT(mask, i)         mask |= CPUMASK_SIMPLE(i)
+#define CPUMASK_ANDBIT(mask, i)         mask &= CPUMASK_SIMPLE(i)
+#define CPUMASK_NANDBIT(mask, i)        mask &= ~CPUMASK_SIMPLE(i)
+
+#define CPUMASK_ASSZERO(mask)          mask = 0
+#define CPUMASK_ASSALLONES(mask)       mask = (cpumask_t)-1
+#define CPUMASK_ASSBIT(mask, i)                mask = CPUMASK_SIMPLE(i)
+#define CPUMASK_ASSBMASK(mask, i)      mask = (CPUMASK_SIMPLE(i) - 1)
+#define CPUMASK_ASSNBMASK(mask, i)     mask = ~(CPUMASK_SIMPLE(i) - 1)
+
+#define CPUMASK_ANDMASK(mask, m1)      mask &= (m1)
+#define CPUMASK_NANDMASK(mask, m1)     mask &= ~(m1)
+#define CPUMASK_ORMASK(mask, m1)       mask |= (m1)
+
+#define ATOMIC_CPUMASK_ORBIT(mask, i)          \
+                       atomic_set_cpumask(&(mask), CPUMASK_SIMPLE(i))
+#define ATOMIC_CPUMASK_NANDBIT(mask, i)                \
+                       atomic_clear_cpumask(&(mask), CPUMASK_SIMPLE(i))
+#define ATOMIC_CPUMASK_ORMASK(mask, m1)        \
+                       atomic_set_cpumask(&(mask), m1)
+#define ATOMIC_CPUMASK_NANDMASK(mask, m1)      \
+                       atomic_clear_cpumask(&(mask), m1)
+
 #endif
 
 #define CPULOCK_EXCLBIT        0               /* exclusive lock bit number */
index c0c1312..904ea96 100644 (file)
@@ -159,7 +159,7 @@ _lwbuf_kva(struct lwbuf *lwb, struct mdglobaldata *gd)
 {
     pmap_kenter_sync_quick(lwb->kva);
 
-    atomic_set_int(&lwb->cpumask, gd->mi.gd_cpumask);
+    ATOMIC_CPUMASK_ORBIT(lwb->cpumask, gd->mi.gd_cpuid);
 
     return (lwb->kva);
 }
@@ -169,7 +169,7 @@ lwbuf_kva(struct lwbuf *lwb)
 {
     struct mdglobaldata *gd = mdcpu;
 
-    if (lwb->cpumask & gd->mi.gd_cpumask)
+    if (CPUMASK_TESTBIT(lwb->cpumask, gd->mi.gd_cpuid))
         return (lwb->kva);
 
     return (_lwbuf_kva(lwb, gd));
index c8128be..15cfb86 100644 (file)
@@ -70,13 +70,65 @@ typedef __uint64_t  pml4_entry_t;
 typedef __uint64_t     pdp_entry_t;
 typedef __uint64_t     pd_entry_t;
 typedef __uint64_t     pt_entry_t;
-typedef __uint64_t      cpumask_t;      /* mask representing a set of cpus */
 typedef __uint32_t      cpulock_t;      /* count and exclusive lock */
 
+/*
+ * cpumask_t - a mask representing a set of cpus and supporting routines.
+ *
+ * WARNING! It is recommended that this mask NOT be made variably-sized
+ *         because it affects a huge number of system structures.  However,
+ *         kernel code (non-module) can be optimized to not operate on the
+ *         whole mask.
+ */
+
+#define CPUMASK_ELEMENTS       1       /* tested by assembly for #error */
+
+typedef struct {
+       __uint64_t      m0;
+} cpumask_t;
+
 #if defined(_KERNEL) || defined(_KERNEL_STRUCTURES)
-#define CPUMASK(cpu)   ((cpumask_t)1 << (cpu))
-#define BSRCPUMASK(mask)       bsrq(mask)
-#define BSFCPUMASK(mask)       bsfq(mask)
+
+#define CPUMASK_INITIALIZER_ALLONES    { .m0 = (__uint64_t)-1 }
+#define CPUMASK_INITIALIZER_ONLYONE    { .m0 = 1 }
+
+#define CPUMASK_SIMPLE(cpu)    ((__uint64_t)1 << (cpu))
+#define BSRCPUMASK(val)                bsrq((val).m0)
+#define BSFCPUMASK(val)                bsfq((val).m0)
+
+#define CPUMASK_CMPMASKEQ(val1, val2)  ((val1).m0 == (val2).m0)
+#define CPUMASK_CMPMASKNEQ(val1, val2) ((val1).m0 != (val2).m0)
+#define CPUMASK_ISUP(val)              ((val).m0 == 1)
+
+#define CPUMASK_TESTZERO(val)          ((val).m0 == 0)
+#define CPUMASK_TESTNZERO(val)         ((val).m0 != 0)
+#define CPUMASK_TESTBIT(val, i)                ((val).m0 & CPUMASK_SIMPLE(i))
+#define CPUMASK_TESTMASK(val1, val2)   ((val1).m0 & (val2.m0))
+#define CPUMASK_LOWMASK(val)           (val).m0
+
+#define CPUMASK_ORBIT(mask, i)         (mask).m0 |= CPUMASK_SIMPLE(i)
+#define CPUMASK_ANDBIT(mask, i)                (mask).m0 &= CPUMASK_SIMPLE(i)
+#define CPUMASK_NANDBIT(mask, i)       (mask).m0 &= ~CPUMASK_SIMPLE(i)
+
+#define CPUMASK_ASSZERO(mask)          (mask).m0 = 0
+#define CPUMASK_ASSALLONES(mask)       (mask).m0 = (__uint64_t)-1
+#define CPUMASK_ASSBIT(mask, i)                (mask).m0 = CPUMASK_SIMPLE(i)
+#define CPUMASK_ASSBMASK(mask, i)      (mask).m0 = (CPUMASK_SIMPLE(i) - 1)
+#define CPUMASK_ASSNBMASK(mask, i)     (mask).m0 = ~(CPUMASK_SIMPLE(i) - 1)
+
+#define CPUMASK_ANDMASK(mask, val)     (mask).m0 &= (val).m0
+#define CPUMASK_NANDMASK(mask, val)    (mask).m0 &= ~(val).m0
+#define CPUMASK_ORMASK(mask, val)      (mask).m0 |= (val).m0
+
+#define ATOMIC_CPUMASK_ORBIT(mask, i)          \
+                       atomic_set_cpumask(&(mask).m0, CPUMASK_SIMPLE(i))
+#define ATOMIC_CPUMASK_NANDBIT(mask, i)                \
+                        atomic_clear_cpumask(&(mask).m0, CPUMASK_SIMPLE(i))
+#define ATOMIC_CPUMASK_ORMASK(mask, val)               \
+                       atomic_set_cpumask(&(mask).m0, val.m0)
+#define ATOMIC_CPUMASK_NANDMASK(mask, val)     \
+                       atomic_clear_cpumask(&(mask).m0, val.m0)
+
 #endif
 
 #define CPULOCK_EXCLBIT        0               /* exclusive lock bit number */
index 36d9e9b..903b0ec 100644 (file)
@@ -303,7 +303,7 @@ acpi_cpu_get_id(uint32_t idx, uint32_t *acpi_id, uint32_t *cpu_id)
     KASSERT(acpi_id != NULL, ("Null acpi_id"));
     KASSERT(cpu_id != NULL, ("Null cpu_id"));
     for (i = 0; i < ncpus; i++) {
-       if ((smp_active_mask & CPUMASK(i)) == 0)
+       if (CPUMASK_TESTBIT(smp_active_mask, i) == 0)
            continue;
        md = (struct mdglobaldata *)globaldata_find(i);
        KASSERT(md != NULL, ("no pcpu data for %d", i));
index cf70625..ffe1c8a 100644 (file)
@@ -184,12 +184,14 @@ clockmod_dom_attach(struct clockmod_softc *sc)
 {
        struct clockmod_softc *sc1;
        struct clockmod_dom *dom;
-       cpumask_t mask, found_mask = 0;
+       cpumask_t mask, found_mask;
        int error = 0;
 
+       CPUMASK_ASSZERO(found_mask);
+
        mask = get_cpumask_from_level(sc->sc_cpuid, CORE_LEVEL);
-       if (mask == 0)
-               mask = CPUMASK(sc->sc_cpuid);
+       if (CPUMASK_TESTZERO(mask))
+               CPUMASK_ASSBIT(mask, sc->sc_cpuid);
 
        lwkt_serialize_enter(&clockmod_dom_slize);
 
@@ -206,9 +208,9 @@ clockmod_dom_attach(struct clockmod_softc *sc)
        TAILQ_INSERT_TAIL(&dom->dom_list, sc, sc_link);
 
        TAILQ_FOREACH(sc1, &dom->dom_list, sc_link)
-               found_mask |= CPUMASK(sc1->sc_cpuid);
+               CPUMASK_ORBIT(found_mask, sc1->sc_cpuid);
 
-       if (found_mask == dom->dom_cpumask) {
+       if (CPUMASK_CMPMASKEQ(found_mask, dom->dom_cpumask)) {
                /* All cpus in this domain is found */
                dom->dom_flags |= CLOCKMOD_DOM_FLAG_ACTIVE;
        }
@@ -251,7 +253,7 @@ clockmod_dom_find(cpumask_t mask)
        struct clockmod_dom *dom;
 
        TAILQ_FOREACH(dom, &clockmod_dom_list, dom_link) {
-               if (dom->dom_cpumask == mask)
+               if (CPUMASK_CMPMASKEQ(dom->dom_cpumask, mask))
                        return dom;
        }
        return NULL;
index 8ab5170..4ee1176 100644 (file)
@@ -171,7 +171,7 @@ mi_proc0init(struct globaldata *gd, struct user *proc0paddr)
        lwp0.lwp_thread = &thread0;
        lwp0.lwp_proc = &proc0;
        proc0.p_usched = usched_init();
-       lwp0.lwp_cpumask = (cpumask_t)-1;
+       CPUMASK_ASSALLONES(lwp0.lwp_cpumask);
        lwkt_token_init(&lwp0.lwp_token, "lwp_token");
        spin_init(&lwp0.lwp_spin);
        varsymset_init(&proc0.p_varsymset, NULL);
@@ -724,10 +724,10 @@ mi_gdinit(struct globaldata *gd, int cpuid)
        TAILQ_INIT(&gd->gd_systimerq);
        gd->gd_sysid_alloc = cpuid;     /* prime low bits for cpu lookup */
        gd->gd_cpuid = cpuid;
-       gd->gd_cpumask = CPUMASK(cpuid);
+       CPUMASK_ASSBIT(gd->gd_cpumask, cpuid);
        lwkt_gdinit(gd);
        vm_map_entry_reserve_cpu_init(gd);
        sleep_gdinit(gd);
-       atomic_set_cpumask(&usched_global_cpumask, CPUMASK(cpuid));
+       ATOMIC_CPUMASK_ORBIT(usched_global_cpumask, cpuid);
 }
 
index 0d13637..8065ccf 100644 (file)
@@ -650,7 +650,7 @@ lwp_fork(struct lwp *origlp, struct proc *destproc, int flags)
        lp->lwp_cpbase = gd->gd_schedclock.time - gd->gd_schedclock.periodic;
        destproc->p_usched->heuristic_forking(origlp, lp);
        crit_exit();
-       lp->lwp_cpumask &= usched_mastermask;
+       CPUMASK_ANDMASK(lp->lwp_cpumask, usched_mastermask);
        lwkt_token_init(&lp->lwp_token, "lwp_token");
        spin_init(&lp->lwp_spin);
 
index f4ee251..ea9457c 100644 (file)
@@ -1512,7 +1512,7 @@ sysctl_kern_proc(SYSCTL_HANDLER_ARGS)
                int nid;
 
                nid = (origcpu + n) % ncpus;
-               if ((smp_active_mask & CPUMASK(nid)) == 0)
+               if (CPUMASK_TESTBIT(smp_active_mask, nid) == 0)
                        continue;
                rgd = globaldata_find(nid);
                lwkt_setcpu_self(rgd);
index 6e1d813..e9eb01a 100644 (file)
@@ -261,7 +261,7 @@ boot(int howto)
         * We really want to shutdown on the BSP.  Subsystems such as ACPI
         * can't power-down the box otherwise.
         */
-       if (smp_active_mask > 1) {
+       if (!CPUMASK_ISUP(smp_active_mask)) {
                kprintf("boot() called on cpu#%d\n", mycpu->gd_cpuid);
        }
        if (panicstr == NULL && mycpu->gd_cpuid != 0) {
index 570efda..59aa363 100644 (file)
@@ -366,15 +366,15 @@ _tsleep_interlock(globaldata_t gd, const volatile void *ident, int flags)
                id = LOOKUP(td->td_wchan);
                TAILQ_REMOVE(&gd->gd_tsleep_hash[id], td, td_sleepq);
                if (TAILQ_FIRST(&gd->gd_tsleep_hash[id]) == NULL) {
-                       atomic_clear_cpumask(&slpque_cpumasks[id],
-                                            gd->gd_cpumask);
+                       ATOMIC_CPUMASK_NANDBIT(slpque_cpumasks[id],
+                                              gd->gd_cpuid);
                }
        } else {
                td->td_flags |= TDF_TSLEEPQ;
        }
        id = LOOKUP(ident);
        TAILQ_INSERT_TAIL(&gd->gd_tsleep_hash[id], td, td_sleepq);
-       atomic_set_cpumask(&slpque_cpumasks[id], gd->gd_cpumask);
+       ATOMIC_CPUMASK_ORBIT(slpque_cpumasks[id], gd->gd_cpuid);
        td->td_wchan = ident;
        td->td_wdomain = flags & PDOMAIN_MASK;
        crit_exit_quick(td);
@@ -402,8 +402,10 @@ _tsleep_remove(thread_t td)
                td->td_flags &= ~TDF_TSLEEPQ;
                id = LOOKUP(td->td_wchan);
                TAILQ_REMOVE(&gd->gd_tsleep_hash[id], td, td_sleepq);
-               if (TAILQ_FIRST(&gd->gd_tsleep_hash[id]) == NULL)
-                       atomic_clear_cpumask(&slpque_cpumasks[id], gd->gd_cpumask);
+               if (TAILQ_FIRST(&gd->gd_tsleep_hash[id]) == NULL) {
+                       ATOMIC_CPUMASK_NANDBIT(slpque_cpumasks[id],
+                                              gd->gd_cpuid);
+               }
                td->td_wchan = NULL;
                td->td_wdomain = 0;
        }
@@ -921,10 +923,13 @@ restart:
         * should be ok since we are passing idents in the IPI rather then
         * thread pointers.
         */
-       if ((domain & PWAKEUP_MYCPU) == 0 &&
-           (mask = slpque_cpumasks[id] & gd->gd_other_cpus) != 0) {
-               lwkt_send_ipiq2_mask(mask, _wakeup, ident,
-                                    domain | PWAKEUP_MYCPU);
+       if ((domain & PWAKEUP_MYCPU) == 0) {
+               mask = slpque_cpumasks[id];
+               CPUMASK_ANDMASK(mask, gd->gd_other_cpus);
+               if (CPUMASK_TESTNZERO(mask)) {
+                       lwkt_send_ipiq2_mask(mask, _wakeup, ident,
+                                            domain | PWAKEUP_MYCPU);
+               }
        }
 done:
        logtsleep1(wakeup_end);
index 0817afb..f361f9d 100644 (file)
@@ -48,7 +48,7 @@
 
 static TAILQ_HEAD(, usched) usched_list = TAILQ_HEAD_INITIALIZER(usched_list);
 
-cpumask_t usched_mastermask = -1;
+cpumask_t usched_mastermask = CPUMASK_INITIALIZER_ALLONES;
 
 /*
  * Called from very low level boot code, i386/i386/machdep.c/init386().
@@ -243,11 +243,11 @@ sys_usched_set(struct usched_set_args *uap)
                        error = EFBIG;
                        break;
                }
-               if ((smp_active_mask & CPUMASK(cpuid)) == 0) {
+               if (CPUMASK_TESTBIT(smp_active_mask, cpuid) == 0) {
                        error = EINVAL;
                        break;
                }
-               lp->lwp_cpumask = CPUMASK(cpuid);
+               CPUMASK_ASSBIT(lp->lwp_cpumask, cpuid);
                if (cpuid != mycpu->gd_cpuid) {
                        lwkt_migratecpu(cpuid);
                        p->p_usched->changedcpu(lp);
@@ -275,11 +275,11 @@ sys_usched_set(struct usched_set_args *uap)
                        error = EFBIG;
                        break;
                }
-               if (!(smp_active_mask & CPUMASK(cpuid))) {
+               if (CPUMASK_TESTBIT(smp_active_mask, cpuid) == 0) {
                        error = EINVAL;
                        break;
                }
-               lp->lwp_cpumask |= CPUMASK(cpuid);
+               CPUMASK_ORBIT(lp->lwp_cpumask, cpuid);
                break;
        case USCHED_DEL_CPU:
                /* USCHED_DEL_CPU doesn't require special privileges. */
@@ -295,14 +295,18 @@ sys_usched_set(struct usched_set_args *uap)
                        break;
                }
                lp = curthread->td_lwp;
-               mask = lp->lwp_cpumask & smp_active_mask & ~CPUMASK(cpuid);
-               if (mask == 0)
+               mask = lp->lwp_cpumask;
+               CPUMASK_ANDMASK(mask, smp_active_mask);
+               CPUMASK_NANDBIT(mask, cpuid);
+               if (CPUMASK_TESTZERO(mask)) {
                        error = EPERM;
-               else {
-                       lp->lwp_cpumask &= ~CPUMASK(cpuid);
-                       if ((lp->lwp_cpumask & mycpu->gd_cpumask) == 0) {
-                               cpuid = BSFCPUMASK(lp->lwp_cpumask &
-                                                  smp_active_mask);
+               } else {
+                       CPUMASK_NANDBIT(lp->lwp_cpumask, cpuid);
+                       if (CPUMASK_TESTMASK(lp->lwp_cpumask,
+                                           mycpu->gd_cpumask) == 0) {
+                               mask = lp->lwp_cpumask;
+                               CPUMASK_ANDMASK(mask, smp_active_mask);
+                               cpuid = BSFCPUMASK(mask);
                                lwkt_migratecpu(cpuid);
                                p->p_usched->changedcpu(lp);
                        }
index d426b28..f41ef01 100644 (file)
@@ -255,7 +255,7 @@ lwkt_send_ipiq3(globaldata_t target, ipifunc3_t func, void *arg1, int arg2)
     ip->ip_info[windex].arg2 = arg2;
     cpu_sfence();
     ++ip->ip_windex;
-    atomic_set_cpumask(&target->gd_ipimask, gd->gd_cpumask);
+    ATOMIC_CPUMASK_ORBIT(target->gd_ipimask, gd->gd_cpuid);
 
     /*
      * signal the target cpu that there is work pending.
@@ -352,7 +352,7 @@ lwkt_send_ipiq3_passive(globaldata_t target, ipifunc3_t func,
     ip->ip_info[windex].arg2 = arg2;
     cpu_sfence();
     ++ip->ip_windex;
-    atomic_set_cpumask(&target->gd_ipimask, gd->gd_cpumask);
+    ATOMIC_CPUMASK_ORBIT(target->gd_ipimask, gd->gd_cpuid);
     --gd->gd_intr_nesting_level;
 
     /*
@@ -403,7 +403,7 @@ lwkt_send_ipiq3_nowait(globaldata_t target, ipifunc3_t func,
     ip->ip_info[windex].arg2 = arg2;
     cpu_sfence();
     ++ip->ip_windex;
-    atomic_set_cpumask(&target->gd_ipimask, gd->gd_cpumask);
+    ATOMIC_CPUMASK_ORBIT(target->gd_ipimask, gd->gd_cpuid);
 
     /*
      * This isn't a passive IPI, we still have to signal the target cpu.
@@ -440,11 +440,11 @@ lwkt_send_ipiq3_mask(cpumask_t mask, ipifunc3_t func, void *arg1, int arg2)
     int cpuid;
     int count = 0;
 
-    mask &= ~stopped_cpus;
-    while (mask) {
+    CPUMASK_NANDMASK(mask, stopped_cpus);
+    while (CPUMASK_TESTNZERO(mask)) {
        cpuid = BSFCPUMASK(mask);
        lwkt_send_ipiq3(globaldata_find(cpuid), func, arg1, arg2);
-       mask &= ~CPUMASK(cpuid);
+       CPUMASK_NANDBIT(mask, cpuid);
        ++count;
     }
     return(count);
@@ -560,8 +560,8 @@ lwkt_process_ipiq(void)
 again:
     cpu_lfence();
     mask = gd->gd_ipimask;
-    atomic_clear_cpumask(&gd->gd_ipimask, mask);
-    while (mask) {
+    ATOMIC_CPUMASK_NANDMASK(gd->gd_ipimask, mask);
+    while (CPUMASK_TESTNZERO(mask)) {
        n = BSFCPUMASK(mask);
        if (n != gd->gd_cpuid) {
            sgd = globaldata_find(n);
@@ -571,7 +571,7 @@ again:
                    ;
            }
        }
-       mask &= ~CPUMASK(n);
+       CPUMASK_NANDBIT(mask, n);
     }
 
     /*
@@ -589,11 +589,11 @@ again:
      * Interlock to allow more IPI interrupts.  Recheck ipimask after
      * releasing gd_npoll.
      */
-    if (gd->gd_ipimask)
+    if (CPUMASK_TESTNZERO(gd->gd_ipimask))
        goto again;
     atomic_poll_release_int(&gd->gd_npoll);
     cpu_mfence();
-    if (gd->gd_ipimask)
+    if (CPUMASK_TESTNZERO(gd->gd_ipimask))
        goto again;
     --gd->gd_processing_ipiq;
 }
@@ -610,8 +610,8 @@ lwkt_process_ipiq_frame(struct intrframe *frame)
 again:
     cpu_lfence();
     mask = gd->gd_ipimask;
-    atomic_clear_cpumask(&gd->gd_ipimask, mask);
-    while (mask) {
+    ATOMIC_CPUMASK_NANDMASK(gd->gd_ipimask, mask);
+    while (CPUMASK_TESTNZERO(mask)) {
        n = BSFCPUMASK(mask);
        if (n != gd->gd_cpuid) {
            sgd = globaldata_find(n);
@@ -621,7 +621,7 @@ again:
                    ;
            }
        }
-       mask &= ~CPUMASK(n);
+       CPUMASK_NANDBIT(mask, n);
     }
     if (gd->gd_cpusyncq.ip_rindex != gd->gd_cpusyncq.ip_windex) {
        if (lwkt_process_ipiq_core(gd, &gd->gd_cpusyncq, frame)) {
@@ -635,11 +635,11 @@ again:
      * Interlock to allow more IPI interrupts.  Recheck ipimask after
      * releasing gd_npoll.
      */
-    if (gd->gd_ipimask)
+    if (CPUMASK_TESTNZERO(gd->gd_ipimask))
        goto again;
     atomic_poll_release_int(&gd->gd_npoll);
     cpu_mfence();
-    if (gd->gd_ipimask)
+    if (CPUMASK_TESTNZERO(gd->gd_ipimask))
        goto again;
 }
 
@@ -774,8 +774,8 @@ lwkt_sync_ipiq(void *arg)
 {
     volatile cpumask_t *cpumask = arg;
 
-    atomic_clear_cpumask(cpumask, mycpu->gd_cpumask);
-    if (*cpumask == 0)
+    ATOMIC_CPUMASK_NANDBIT(*cpumask, mycpu->gd_cpuid);
+    if (CPUMASK_TESTZERO(*cpumask))
        wakeup(cpumask);
 }
 
@@ -784,13 +784,14 @@ lwkt_synchronize_ipiqs(const char *wmesg)
 {
     volatile cpumask_t other_cpumask;
 
-    other_cpumask = mycpu->gd_other_cpus & smp_active_mask;
+    other_cpumask = smp_active_mask;
+    CPUMASK_ANDMASK(other_cpumask, mycpu->gd_other_cpus);
     lwkt_send_ipiq_mask(other_cpumask, lwkt_sync_ipiq,
-       __DEVOLATILE(void *, &other_cpumask));
+                       __DEVOLATILE(void *, &other_cpumask));
 
-    while (other_cpumask != 0) {
+    while (CPUMASK_TESTNZERO(other_cpumask)) {
        tsleep_interlock(&other_cpumask, 0);
-       if (other_cpumask != 0)
+       if (CPUMASK_TESTNZERO(other_cpumask))
            tsleep(&other_cpumask, PINTERLOCKED, wmesg, 0);
     }
 }
@@ -830,20 +831,23 @@ lwkt_cpusync_interlock(lwkt_cpusync_t cs)
      *
      * mack does not include the current cpu.
      */
-    mask = cs->cs_mask & gd->gd_other_cpus & smp_active_mask;
-    cs->cs_mack = 0;
+    mask = cs->cs_mask;
+    CPUMASK_ANDMASK(mask, gd->gd_other_cpus);
+    CPUMASK_ANDMASK(mask, smp_active_mask);
+    CPUMASK_ASSZERO(cs->cs_mack);
+
     crit_enter_id("cpusync");
-    if (mask) {
+    if (CPUMASK_TESTNZERO(mask)) {
        DEBUG_PUSH_INFO("cpusync_interlock");
        ++ipiq_stat(gd).ipiq_cscount;
        ++gd->gd_curthread->td_cscount;
        lwkt_send_ipiq_mask(mask, (ipifunc1_t)lwkt_cpusync_remote1, cs);
-       logipiq2(sync_start, (long)mask);
+       logipiq2(sync_start, (long)CPUMASK_LOWMASK(mask));
 #if 0
        if (gd->gd_curthread->td_wmesg == NULL)
                gd->gd_curthread->td_wmesg = smsg;
 #endif
-       while (cs->cs_mack != mask) {
+       while (CPUMASK_CMPMASKNEQ(cs->cs_mack, mask)) {
            lwkt_process_ipiq();
            cpu_pause();
 #ifdef _KERNEL_VIRTUAL
@@ -884,17 +888,17 @@ lwkt_cpusync_deinterlock(lwkt_cpusync_t cs)
      */
     mask = cs->cs_mack;
     cpu_ccfence();
-    cs->cs_mack = 0;
+    CPUMASK_ASSZERO(cs->cs_mack);
     cpu_ccfence();
-    if (cs->cs_func && (cs->cs_mask & gd->gd_cpumask))
+    if (cs->cs_func && CPUMASK_TESTBIT(cs->cs_mask, gd->gd_cpuid))
            cs->cs_func(cs->cs_data);
-    if (mask) {
+    if (CPUMASK_TESTNZERO(mask)) {
        DEBUG_PUSH_INFO("cpusync_deinterlock");
 #if 0
        if (gd->gd_curthread->td_wmesg == NULL)
                gd->gd_curthread->td_wmesg = smsg;
 #endif
-       while (cs->cs_mack != mask) {
+       while (CPUMASK_CMPMASKNEQ(cs->cs_mack, mask)) {
            lwkt_process_ipiq();
            cpu_pause();
 #ifdef _KERNEL_VIRTUAL
@@ -913,7 +917,7 @@ lwkt_cpusync_deinterlock(lwkt_cpusync_t cs)
         */
        --gd->gd_curthread->td_cscount;
        lwkt_process_ipiq();
-       logipiq2(sync_end, (long)mask);
+       logipiq2(sync_end, (long)CPUMASK_LOWMASK(mask));
     }
     crit_exit_id("cpusync");
 }
@@ -930,7 +934,7 @@ lwkt_cpusync_remote1(lwkt_cpusync_t cs)
 {
     globaldata_t gd = mycpu;
 
-    atomic_set_cpumask(&cs->cs_mack, gd->gd_cpumask);
+    ATOMIC_CPUMASK_ORBIT(cs->cs_mack, gd->gd_cpuid);
     lwkt_cpusync_remote2(cs);
 }
 
@@ -945,10 +949,10 @@ lwkt_cpusync_remote2(lwkt_cpusync_t cs)
 {
     globaldata_t gd = mycpu;
 
-    if ((cs->cs_mack & gd->gd_cpumask) == 0) {
+    if (CPUMASK_TESTMASK(cs->cs_mack, gd->gd_cpumask) == 0) {
        if (cs->cs_func)
                cs->cs_func(cs->cs_data);
-       atomic_set_cpumask(&cs->cs_mack, gd->gd_cpumask);
+       ATOMIC_CPUMASK_ORBIT(cs->cs_mack, gd->gd_cpuid);
        /* cs can be ripped out at this point */
     } else {
        lwkt_ipiq_t ip;
@@ -968,7 +972,8 @@ lwkt_cpusync_remote2(lwkt_cpusync_t cs)
        if (ipiq_debug && (ip->ip_windex & 0xFFFFFF) == 0) {
                kprintf("cpu %d cm=%016jx %016jx f=%p\n",
                        gd->gd_cpuid,
-                       (intmax_t)cs->cs_mask, (intmax_t)cs->cs_mack,
+                       (intmax_t)CPUMASK_LOWMASK(cs->cs_mask),
+                       (intmax_t)CPUMASK_LOWMASK(cs->cs_mack),
                        cs->cs_func);
        }
     }
index e029de1..8338551 100644 (file)
@@ -108,12 +108,12 @@ build_topology_tree(int *children_no_per_level,
 
        node->child_no = children_no_per_level[cur_level];
        node->type = level_types[cur_level];
-       node->members = 0;
+       CPUMASK_ASSZERO(node->members);
        node->compute_unit_id = -1;
 
        if (node->child_no == 0) {
                *apicid = get_next_valid_apicid(*apicid);
-               node->members = CPUMASK(get_cpuid_from_apicid(*apicid));
+               CPUMASK_ASSBIT(node->members, get_cpuid_from_apicid(*apicid));
                return;
        }
 
@@ -133,7 +133,7 @@ build_topology_tree(int *children_no_per_level,
                    last_free_node,
                    apicid);
 
-               node->members |= node->child_node[i]->members;
+               CPUMASK_ORMASK(node->members, node->child_node[i]->members);
        }
 }
 
@@ -175,10 +175,11 @@ build_cpu_topology(void)
         * and witin core to build up the topology
         */
        for (i = 0; i < ncpus; i++) {
+               cpumask_t mask;
 
-               cpumask_t mask = CPUMASK(i);
+               CPUMASK_ASSBIT(mask, i);
 
-               if ((mask & smp_active_mask) == 0)
+               if (CPUMASK_TESTMASK(mask, smp_active_mask) == 0)
                        continue;
 
                if (get_chip_ID(BSPID) == get_chip_ID(i))
@@ -293,7 +294,7 @@ build_cpu_topology(void)
 
                                                                last_free_node->child_node[last_free_node->child_no] = parent->child_node[j];
                                                                last_free_node->child_no++;
-                                                               last_free_node->members |= parent->child_node[j]->members;
+                                                               CPUMASK_ORMASK(last_free_node->members, parent->child_node[j]->members);
 
                                                                parent->child_node[j]->type = THREAD_LEVEL;
                                                                parent->child_node[j]->parent_node = last_free_node;
@@ -443,9 +444,8 @@ get_cpu_node_by_cpumask(cpu_node_t * node,
        cpu_node_t * found = NULL;
        int i;
 
-       if (node->members == mask) {
+       if (CPUMASK_CMPMASKEQ(node->members, mask))
                return node;
-       }
 
        for (i = 0; i < node->child_no; i++) {
                found = get_cpu_node_by_cpumask(node->child_node[i], mask);
@@ -458,7 +458,9 @@ get_cpu_node_by_cpumask(cpu_node_t * node,
 
 cpu_node_t *
 get_cpu_node_by_cpuid(int cpuid) {
-       cpumask_t mask = CPUMASK(cpuid);
+       cpumask_t mask;
+
+       CPUMASK_ASSBIT(mask, cpuid);
 
        KASSERT(cpu_root_node != NULL, ("cpu_root_node isn't initialized"));
 
@@ -471,14 +473,17 @@ get_cpumask_from_level(int cpuid,
                        uint8_t level_type)
 {
        cpu_node_t * node;
-       cpumask_t mask = CPUMASK(cpuid);
+       cpumask_t mask;
+
+       CPUMASK_ASSBIT(mask, cpuid);
 
        KASSERT(cpu_root_node != NULL, ("cpu_root_node isn't initialized"));
 
        node = get_cpu_node_by_cpumask(cpu_root_node, mask);
 
        if (node == NULL) {
-               return 0;
+               CPUMASK_ASSZERO(mask);
+               return mask;
        }
 
        while (node != NULL) {
@@ -487,8 +492,9 @@ get_cpumask_from_level(int cpuid,
                }
                node = node->parent_node;
        }
+       CPUMASK_ASSZERO(mask);
 
-       return 0;
+       return mask;
 }
 
 /* init pcpu_sysctl structure info */
@@ -510,7 +516,7 @@ init_pcpu_topology_sysctl(void)
 
                /* Get physical siblings */
                mask = get_cpumask_from_level(i, CHIP_LEVEL);
-               if (mask == 0) {
+               if (CPUMASK_TESTZERO(mask)) {
                        pcpu_sysctl[i].physical_id = INVALID_ID;
                        continue;
                }
@@ -527,7 +533,7 @@ init_pcpu_topology_sysctl(void)
 
                /* Get core siblings */
                mask = get_cpumask_from_level(i, CORE_LEVEL);
-               if (mask == 0) {
+               if (CPUMASK_TESTZERO(mask)) {
                        pcpu_sysctl[i].core_id = INVALID_ID;
                        continue;
                }
index b1abc45..c58122d 100644 (file)
@@ -129,6 +129,7 @@ sys_vmm_guest_sync_addr(struct vmm_guest_sync_addr_args *uap)
        int error = 0;
        cpulock_t olock;
        cpulock_t nlock;
+       cpumask_t mask;
        long val;
        struct proc *p = curproc;
 
@@ -140,7 +141,7 @@ sys_vmm_guest_sync_addr(struct vmm_guest_sync_addr_args *uap)
        /*
         * Acquire CPULOCK_EXCL, spin while we wait.
         */
-       KKASSERT((p->p_vmm_cpumask & mycpu->gd_cpumask) == 0);
+       KKASSERT(CPUMASK_TESTMASK(p->p_vmm_cpumask, mycpu->gd_cpumask) == 0);
        for (;;) {
                olock = p->p_vmm_cpulock & ~CPULOCK_EXCL;
                cpu_ccfence();
@@ -163,8 +164,9 @@ sys_vmm_guest_sync_addr(struct vmm_guest_sync_addr_args *uap)
         * counter.
         */
        if (olock & CPULOCK_CNTMASK) {
-               lwkt_send_ipiq_mask(p->p_vmm_cpumask & mycpu->gd_other_cpus,
-                                   vmm_exit_vmm, NULL);
+               mask = p->p_vmm_cpumask;
+               CPUMASK_ANDMASK(mask, mycpu->gd_other_cpus);
+               lwkt_send_ipiq_mask(mask, vmm_exit_vmm, NULL);
                while (p->p_vmm_cpulock & CPULOCK_CNTMASK) {
                        lwkt_process_ipiq();
                        cpu_pause();
index 625050b..fb9c12c 100644 (file)
@@ -158,8 +158,10 @@ static struct rq bsd4_idqueues[NQS];
 static u_int32_t bsd4_queuebits;
 static u_int32_t bsd4_rtqueuebits;
 static u_int32_t bsd4_idqueuebits;
-static cpumask_t bsd4_curprocmask = -1;        /* currently running a user process */
-static cpumask_t bsd4_rdyprocmask;     /* ready to accept a user process */
+/* currently running a user process */
+static cpumask_t bsd4_curprocmask = CPUMASK_INITIALIZER_ALLONES;
+/* ready to accept a user process */
+static cpumask_t bsd4_rdyprocmask;
 static int      bsd4_runqcount;
 static volatile int bsd4_scancpu;
 static struct spinlock bsd4_spin;
@@ -302,7 +304,7 @@ bsd4_rqinit(void *dummy)
                TAILQ_INIT(&bsd4_rtqueues[i]);
                TAILQ_INIT(&bsd4_idqueues[i]);
        }
-       atomic_clear_cpumask(&bsd4_curprocmask, 1);
+       ATOMIC_CPUMASK_NANDBIT(bsd4_curprocmask, 0);
 }
 SYSINIT(runqueue, SI_BOOT2_USCHED, SI_ORDER_FIRST, bsd4_rqinit, NULL)
 
@@ -394,7 +396,7 @@ bsd4_acquire_curproc(struct lwp *lp)
                        /*
                         * We can trivially become the current lwp.
                         */
-                       atomic_set_cpumask(&bsd4_curprocmask, gd->gd_cpumask);
+                       ATOMIC_CPUMASK_ORBIT(bsd4_curprocmask, gd->gd_cpuid);
                        dd->uschedcp = lp;
                        dd->upri = lp->lwp_priority;
                } else if (dd->upri > lp->lwp_priority) {
@@ -489,7 +491,7 @@ bsd4_release_curproc(struct lwp *lp)
 
                dd->uschedcp = NULL;    /* don't let lp be selected */
                dd->upri = PRIBASE_NULL;
-               atomic_clear_cpumask(&bsd4_curprocmask, gd->gd_cpumask);
+               ATOMIC_CPUMASK_NANDBIT(bsd4_curprocmask, gd->gd_cpuid);
                dd->old_uschedcp = lp;  /* used only for KTR debug prints */
                bsd4_select_curproc(gd);
                crit_exit();
@@ -537,7 +539,7 @@ bsd4_select_curproc(globaldata_t gd)
                    dd->old_uschedcp->lwp_thread->td_gd->gd_cpuid,
                    gd->gd_cpuid);
 
-               atomic_set_cpumask(&bsd4_curprocmask, CPUMASK(cpuid));
+               ATOMIC_CPUMASK_ORBIT(bsd4_curprocmask, cpuid);
                dd->upri = nlp->lwp_priority;
                dd->uschedcp = nlp;
                dd->rrcount = 0;                /* reset round robin */
@@ -549,8 +551,8 @@ bsd4_select_curproc(globaldata_t gd)
        }
 
 #if 0
-       } else if (bsd4_runqcount && (bsd4_rdyprocmask & CPUMASK(cpuid))) {
-               atomic_clear_cpumask(&bsd4_rdyprocmask, CPUMASK(cpuid));
+       } else if (bsd4_runqcount && CPUMASK_TESTBIT(bsd4_rdyprocmask, cpuid)) {
+               ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, cpuid);
                spin_unlock(&bsd4_spin);
                lwkt_schedule(&dd->helper_thread);
        } else {
@@ -572,10 +574,11 @@ bsd4_batchy_looser_pri_test(struct lwp* lp)
        int cpu;
 
        /* Current running processes */
-       mask = bsd4_curprocmask & smp_active_mask
-           & usched_global_cpumask;
+       mask = bsd4_curprocmask;
+       CPUMASK_ANDMASK(mask, smp_active_mask);
+       CPUMASK_ANDMASK(mask, usched_global_cpumask);
 
-       while(mask) {
+       while (CPUMASK_TESTNZERO(mask)) {
                cpu = BSFCPUMASK(mask);
                other_dd = &bsd4_pcpu[cpu];
                if (other_dd->upri - lp->lwp_priority > usched_bsd4_upri_affinity * PPQ) {
@@ -588,7 +591,7 @@ bsd4_batchy_looser_pri_test(struct lwp* lp)
 
                        return 0;
                }
-               mask &= ~CPUMASK(cpu);
+               CPUMASK_NANDBIT(mask, cpu);
        }
 
        KTR_COND_LOG(usched_batchy_test_true,
@@ -687,8 +690,11 @@ bsd4_setrunqueue(struct lwp *lp)
                int sibling;
 
                cpuid = (bsd4_scancpu & 0xFFFF) % ncpus;
-               mask = ~bsd4_curprocmask & bsd4_rdyprocmask & lp->lwp_cpumask &
-                   smp_active_mask & usched_global_cpumask;
+               mask = bsd4_rdyprocmask;
+               CPUMASK_NANDMASK(mask, bsd4_curprocmask);
+               CPUMASK_ANDMASK(mask, lp->lwp_cpumask);
+               CPUMASK_ANDMASK(mask, smp_active_mask);
+               CPUMASK_ANDMASK(mask, usched_global_cpumask);
 
                KTR_COND_LOG(usched_bsd4_setrunqueue_fc_smt,
                    lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
@@ -697,18 +703,22 @@ bsd4_setrunqueue(struct lwp *lp)
                    (unsigned long)mask,
                    mycpu->gd_cpuid);
 
-               while (mask) {
-                       tmpmask = ~(CPUMASK(cpuid) - 1);
-                       if (mask & tmpmask)
-                               cpuid = BSFCPUMASK(mask & tmpmask);
-                       else
+               while (CPUMASK_TESTNZERO(mask)) {
+                       CPUMASK_ASSNBMASK(tmpmask, cpuid);
+                       if (CPUMASK_TESTMASK(tmpmask, mask)) {
+                               CPUMASK_ANDMASK(tmpmask, mask);
+                               cpuid = BSFCPUMASK(tmpmask);
+                       } else {
                                cpuid = BSFCPUMASK(mask);
+                       }
                        gd = globaldata_find(cpuid);
                        dd = &bsd4_pcpu[cpuid];
 
                        if ((dd->upri & ~PPQMASK) >= (lp->lwp_priority & ~PPQMASK)) {
-                               if (dd->cpunode->parent_node->members & ~dd->cpunode->members & mask) {
-
+                               tmpmask = dd->cpunode->parent_node->members;
+                               CPUMASK_NANDMASK(tmpmask, dd->cpunode->members);
+                               CPUMASK_ANDMASK(tmpmask, mask);
+                               if (CPUMASK_TESTNZERO(tmpmask)) {
                                        KTR_COND_LOG(usched_bsd4_setrunqueue_found,
                                            lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
                                            lp->lwp_proc->p_pid,
@@ -719,15 +729,20 @@ bsd4_setrunqueue(struct lwp *lp)
 
                                        goto found;
                                } else {
-                                       sibling = BSFCPUMASK(dd->cpunode->parent_node->members &
-                                           ~dd->cpunode->members);
-                                       if (min_prio > bsd4_pcpu[sibling].upri) {
-                                               min_prio = bsd4_pcpu[sibling].upri;
+                                       tmpmask =
+                                           dd->cpunode->parent_node->members;
+                                       CPUMASK_NANDMASK(tmpmask,
+                                           dd->cpunode->members);
+                                       sibling = BSFCPUMASK(tmpmask);
+                                       if (min_prio >
+                                           bsd4_pcpu[sibling].upri) {
+                                               min_prio =
+                                                       bsd4_pcpu[sibling].upri;
                                                best_cpuid = cpuid;
                                        }
                                }
                        }
-                       mask &= ~CPUMASK(cpuid);
+                       CPUMASK_NANDBIT(mask, cpuid);
                }
 
                if (best_cpuid != -1) {
@@ -748,8 +763,11 @@ bsd4_setrunqueue(struct lwp *lp)
        } else {
                /* Fallback to the original heuristic */
                cpuid = (bsd4_scancpu & 0xFFFF) % ncpus;
-               mask = ~bsd4_curprocmask & bsd4_rdyprocmask & lp->lwp_cpumask &
-                      smp_active_mask & usched_global_cpumask;
+               mask = bsd4_rdyprocmask;
+               CPUMASK_NANDMASK(mask, bsd4_curprocmask);
+               CPUMASK_ANDMASK(mask, lp->lwp_cpumask);
+               CPUMASK_ANDMASK(mask, smp_active_mask);
+               CPUMASK_ANDMASK(mask, usched_global_cpumask);
 
                KTR_COND_LOG(usched_bsd4_setrunqueue_fc_non_smt,
                    lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
@@ -758,55 +776,61 @@ bsd4_setrunqueue(struct lwp *lp)
                    (unsigned long)mask,
                    mycpu->gd_cpuid);
 
-               while (mask) {
-                       tmpmask = ~(CPUMASK(cpuid) - 1);
-                       if (mask & tmpmask)
-                               cpuid = BSFCPUMASK(mask & tmpmask);
-                       else
+               while (CPUMASK_TESTNZERO(mask)) {
+                       CPUMASK_ASSNBMASK(tmpmask, cpuid);
+                       if (CPUMASK_TESTMASK(tmpmask, mask)) {
+                               CPUMASK_ANDMASK(tmpmask, mask);
+                               cpuid = BSFCPUMASK(tmpmask);
+                       } else {
                                cpuid = BSFCPUMASK(mask);
+                       }
                        gd = globaldata_find(cpuid);
                        dd = &bsd4_pcpu[cpuid];
 
-                       if ((dd->upri & ~PPQMASK) >= (lp->lwp_priority & ~PPQMASK)) {
-
+                       if ((dd->upri & ~PPQMASK) >=
+                           (lp->lwp_priority & ~PPQMASK)) {
                                KTR_COND_LOG(usched_bsd4_setrunqueue_found,
                                    lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
                                    lp->lwp_proc->p_pid,
                                    lp->lwp_thread->td_gd->gd_cpuid,
-                                   (unsigned long)mask,
+                                   (unsigned long)CPUMASK_LOWMASK(mask),
                                    cpuid,
                                    mycpu->gd_cpuid);
 
                                goto found;
                        }
-                       mask &= ~CPUMASK(cpuid);
+                       CPUMASK_NANDBIT(mask, cpuid);
                }
        }
 
        /*
         * Then cpus which might have a currently running lp
         */
-       mask = bsd4_curprocmask & bsd4_rdyprocmask &
-              lp->lwp_cpumask & smp_active_mask & usched_global_cpumask;
+       mask = bsd4_curprocmask;
+       CPUMASK_ANDMASK(mask, bsd4_rdyprocmask);
+       CPUMASK_ANDMASK(mask, lp->lwp_cpumask);
+       CPUMASK_ANDMASK(mask, smp_active_mask);
+       CPUMASK_ANDMASK(mask, usched_global_cpumask);
 
        KTR_COND_LOG(usched_bsd4_setrunqueue_rc,
            lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
            lp->lwp_proc->p_pid,
            lp->lwp_thread->td_gd->gd_cpuid,
-           (unsigned long)mask,
+           (unsigned long)CPUMASK_LOWMASK(mask),
            mycpu->gd_cpuid);
 
-       while (mask) {
-               tmpmask = ~(CPUMASK(cpuid) - 1);
-               if (mask & tmpmask)
-                       cpuid = BSFCPUMASK(mask & tmpmask);
-               else
+       while (CPUMASK_TESTNZERO(mask)) {
+               CPUMASK_ASSNBMASK(tmpmask, cpuid);
+               if (CPUMASK_TESTMASK(tmpmask, mask)) {
+                       CPUMASK_ANDMASK(tmpmask, mask);
+                       cpuid = BSFCPUMASK(tmpmask);
+               } else {
                        cpuid = BSFCPUMASK(mask);
+               }
                gd = globaldata_find(cpuid);
                dd = &bsd4_pcpu[cpuid];
 
                if ((dd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) {
-
                        KTR_COND_LOG(usched_bsd4_setrunqueue_found,
                            lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
                            lp->lwp_proc->p_pid,
@@ -817,7 +841,7 @@ bsd4_setrunqueue(struct lwp *lp)
 
                        goto found;
                }
-               mask &= ~CPUMASK(cpuid);
+               CPUMASK_NANDBIT(mask, cpuid);
        }
 
        /*
@@ -832,9 +856,8 @@ bsd4_setrunqueue(struct lwp *lp)
         * set the user resched flag because
         */
        cpuid = (bsd4_scancpu & 0xFFFF) % ncpus;
-       if ((CPUMASK(cpuid) & usched_global_cpumask) == 0) {
+       if (CPUMASK_TESTBIT(usched_global_cpumask, cpuid) == 0)
                cpuid = 0;
-       }
        gd = globaldata_find(cpuid);
        dd = &bsd4_pcpu[cpuid];
 
@@ -856,7 +879,7 @@ found:
                        }
                }
        } else {
-               atomic_clear_cpumask(&bsd4_rdyprocmask, CPUMASK(cpuid));
+               ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, cpuid);
                spin_unlock(&bsd4_spin);
                if ((dd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK))
                        lwkt_send_ipiq(gd, bsd4_need_user_resched_remote, NULL);
@@ -1168,7 +1191,7 @@ bsd4_resetpriority(struct lwp *lp)
         */
        if (reschedcpu >= 0) {
                dd = &bsd4_pcpu[reschedcpu];
-               if ((bsd4_rdyprocmask & CPUMASK(reschedcpu)) &&
+               if (CPUMASK_TESTBIT(bsd4_rdyprocmask, reschedcpu) &&
                    (checkpri == 0 ||
                     (dd->upri & ~PRIMASK) > (lp->lwp_priority & ~PRIMASK))) {
                        if (reschedcpu == mycpu->gd_cpuid) {
@@ -1176,8 +1199,8 @@ bsd4_resetpriority(struct lwp *lp)
                                need_user_resched();
                        } else {
                                spin_unlock(&bsd4_spin);
-                               atomic_clear_cpumask(&bsd4_rdyprocmask,
-                                                    CPUMASK(reschedcpu));
+                               ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask,
+                                                      reschedcpu);
                                lwkt_send_ipiq(lp->lwp_thread->td_gd,
                                               bsd4_need_user_resched_remote,
                                               NULL);
@@ -1320,7 +1343,7 @@ again:
        lp = TAILQ_FIRST(q);
        KASSERT(lp, ("chooseproc: no lwp on busy queue"));
 
-       while ((lp->lwp_cpumask & cpumask) == 0) {
+       while (CPUMASK_TESTMASK(lp->lwp_cpumask, cpumask) == 0) {
                lp = TAILQ_NEXT(lp, lwp_procq);
                if (lp == NULL) {
                        *which2 &= ~(1 << pri);
@@ -1448,8 +1471,9 @@ again:
         * minimize the contention (we are in a locked region
         */
        while (checks < usched_bsd4_queue_checks) {
-               if ((lp->lwp_cpumask & cpumask) == 0 ||
-                   ((siblings & lp->lwp_thread->td_gd->gd_cpumask) == 0 &&
+               if (CPUMASK_TESTMASK(lp->lwp_cpumask, cpumask) == 0 ||
+                   (CPUMASK_TESTMASK(siblings,
+                                     lp->lwp_thread->td_gd->gd_cpumask) == 0 &&
                      (lp->lwp_rebal_ticks == sched_ticks ||
                       lp->lwp_rebal_ticks == (int)(sched_ticks - 1)) &&
                      bsd4_batchy_looser_pri_test(lp))) {
@@ -1457,15 +1481,18 @@ again:
                        KTR_COND_LOG(usched_chooseproc_cc_not_good,
                            lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
                            lp->lwp_proc->p_pid,
-                           (unsigned long)lp->lwp_thread->td_gd->gd_cpumask,
-                           (unsigned long)siblings,
-                           (unsigned long)cpumask);
+                           (unsigned long)CPUMASK_LOWMASK(
+                                       lp->lwp_thread->td_gd->gd_cpumask),
+                           (unsigned long)CPUMASK_LOWMASK(siblings),
+                           (unsigned long)CPUMASK_LOWMASK(cpumask));
 
                        cpunode = bsd4_pcpu[lp->lwp_thread->td_gd->gd_cpuid].cpunode;
                        level = 0;
                        while (cpunode) {
-                               if (cpunode->members & cpumask)
+                               if (CPUMASK_TESTMASK(cpunode->members,
+                                                    cpumask)) {
                                        break;
+                               }
                                cpunode = cpunode->parent_node;
                                level++;
                        }
@@ -1490,9 +1517,10 @@ again:
                        KTR_COND_LOG(usched_chooseproc_cc_elected,
                            lp->lwp_proc->p_pid == usched_bsd4_pid_debug,
                            lp->lwp_proc->p_pid,
-                           (unsigned long)lp->lwp_thread->td_gd->gd_cpumask,
-                           (unsigned long)siblings,
-                           (unsigned long)cpumask);
+                           (unsigned long)CPUMASK_LOWMASK(
+                                       lp->lwp_thread->td_gd->gd_cpumask),
+                           (unsigned long)CPUMASK_LOWMASK(siblings),
+                           (unsigned long)CPUMASK_LOWMASK(cpumask));
 
                        goto found;
                }
@@ -1553,17 +1581,22 @@ bsd4_kick_helper(struct lwp *lp)
 {
        globaldata_t gd;
        bsd4_pcpu_t dd;
+       cpumask_t tmpmask;
 
        if (lp == NULL)
                return;
        gd = lp->lwp_thread->td_gd;
        dd = &bsd4_pcpu[gd->gd_cpuid];
-       if ((smp_active_mask & usched_global_cpumask &
-           bsd4_rdyprocmask & gd->gd_cpumask) == 0) {
+
+       tmpmask = smp_active_mask;
+       CPUMASK_ANDMASK(tmpmask, usched_global_cpumask);
+       CPUMASK_ANDMASK(tmpmask, bsd4_rdyprocmask);
+       CPUMASK_ANDMASK(tmpmask, gd->gd_cpumask);
+       if (CPUMASK_TESTZERO(tmpmask))
                return;
-       }
+
        ++usched_bsd4_kicks;
-       atomic_clear_cpumask(&bsd4_rdyprocmask, gd->gd_cpumask);
+       ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, gd->gd_cpuid);
        if ((dd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) {
                lwkt_send_ipiq(gd, bsd4_need_user_resched_remote, NULL);
        } else {
@@ -1735,12 +1768,12 @@ sched_thread(void *dummy)
        crit_enter_gd(gd);
        tsleep_interlock(&dd->helper_thread, 0);
        spin_lock(&bsd4_spin);
-       atomic_set_cpumask(&bsd4_rdyprocmask, mask);
+       ATOMIC_CPUMASK_ORMASK(bsd4_rdyprocmask, mask);
 
        clear_user_resched();   /* This satisfied the reschedule request */
        dd->rrcount = 0;        /* Reset the round-robin counter */
 
-       if ((bsd4_curprocmask & mask) == 0) {
+       if (CPUMASK_TESTMASK(bsd4_curprocmask, mask) == 0) {
                /*
                 * No thread is currently scheduled.
                 */
@@ -1752,7 +1785,7 @@ sched_thread(void *dummy)
                            nlp->lwp_proc->p_pid,
                            nlp->lwp_thread->td_gd->gd_cpuid);
 
-                       atomic_set_cpumask(&bsd4_curprocmask, mask);
+                       ATOMIC_CPUMASK_ORMASK(bsd4_curprocmask, mask);
                        dd->upri = nlp->lwp_priority;
                        dd->uschedcp = nlp;
                        dd->rrcount = 0;        /* reset round robin */
@@ -1788,13 +1821,13 @@ sched_thread(void *dummy)
                         * to priority test does not leave other unscheduled
                         * cpus idle when the runqueue is not empty.
                         */
-                       tmpmask = ~bsd4_curprocmask &
-                                 bsd4_rdyprocmask & smp_active_mask;
-                       if (tmpmask) {
+                       tmpmask = bsd4_rdyprocmask;
+                       CPUMASK_NANDMASK(tmpmask, bsd4_curprocmask);
+                       CPUMASK_ANDMASK(tmpmask, smp_active_mask);
+                       if (CPUMASK_TESTNZERO(tmpmask)) {
                                tmpid = BSFCPUMASK(tmpmask);
                                tmpdd = &bsd4_pcpu[tmpid];
-                               atomic_clear_cpumask(&bsd4_rdyprocmask,
-                                                    CPUMASK(tmpid));
+                               ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, tmpid);
                                spin_unlock(&bsd4_spin);
                                wakeup(&tmpdd->helper_thread);
                        } else {
@@ -1861,9 +1894,11 @@ sched_thread_cpu_init(void)
 
        for (i = 0; i < ncpus; ++i) {
                bsd4_pcpu_t dd = &bsd4_pcpu[i];
-               cpumask_t mask = CPUMASK(i);
+               cpumask_t mask;
 
-               if ((mask & smp_active_mask) == 0)
+               CPUMASK_ASSBIT(mask, i);
+
+               if (CPUMASK_TESTMASK(mask, smp_active_mask) == 0)
                    continue;
 
                dd->cpunode = get_cpu_node_by_cpuid(i);
@@ -1914,8 +1949,10 @@ sched_thread_cpu_init(void)
 
                        if (bootverbose) {
                                if (dd->cpunode->parent_node != NULL) {
-                                       CPUSET_FOREACH(cpuid, dd->cpunode->parent_node->members)
+                                       CPUSET_FOREACH(cpuid,
+                                           dd->cpunode->parent_node->members) {
                                                kprintf("cpu%d ", cpuid);
+                                       }
                                        kprintf("\n");
                                } else {
                                        kprintf(" no siblings\n");
@@ -1931,8 +1968,8 @@ sched_thread_cpu_init(void)
                 * been enabled in rqinit().
                 */
                if (i)
-                   atomic_clear_cpumask(&bsd4_curprocmask, mask);
-               atomic_set_cpumask(&bsd4_rdyprocmask, mask);
+                       ATOMIC_CPUMASK_NANDMASK(bsd4_curprocmask, mask);
+               ATOMIC_CPUMASK_ORMASK(bsd4_rdyprocmask, mask);
                dd->upri = PRIBASE_NULL;
 
        }
index fc12dae..9a49def 100644 (file)
@@ -177,7 +177,8 @@ struct usched usched_dfly = {
  * the state of all 32 queues and then a ffs() to find the first busy
  * queue.
  */
-static cpumask_t dfly_curprocmask = -1;        /* currently running a user process */
+                                       /* currently running a user process */
+static cpumask_t dfly_curprocmask = CPUMASK_INITIALIZER_ALLONES;
 static cpumask_t dfly_rdyprocmask;     /* ready to accept a user process */
 static volatile int dfly_scancpu;
 static volatile int dfly_ucount;       /* total running on whole system */
@@ -366,7 +367,7 @@ dfly_acquire_curproc(struct lwp *lp)
                if (dd->uschedcp == NULL) {
                        atomic_clear_int(&lp->lwp_thread->td_mpflags,
                                         TDF_MP_DIDYIELD);
-                       atomic_set_cpumask(&dfly_curprocmask, gd->gd_cpumask);
+                       ATOMIC_CPUMASK_ORBIT(dfly_curprocmask, gd->gd_cpuid);
                        dd->uschedcp = lp;
                        dd->upri = lp->lwp_priority;
                        KKASSERT(lp->lwp_qcpu == dd->cpuid);
@@ -518,7 +519,7 @@ dfly_release_curproc(struct lwp *lp)
                if (dd->uschedcp == lp) {
                        dd->uschedcp = NULL;    /* don't let lp be selected */
                        dd->upri = PRIBASE_NULL;
-                       atomic_clear_cpumask(&dfly_curprocmask, gd->gd_cpumask);
+                       ATOMIC_CPUMASK_NANDBIT(dfly_curprocmask, gd->gd_cpuid);
                        spin_unlock(&dd->spin);
                        dfly_select_curproc(gd);
                } else {
@@ -554,7 +555,7 @@ dfly_select_curproc(globaldata_t gd)
        nlp = dfly_chooseproc_locked(dd, dd, dd->uschedcp, 0);
 
        if (nlp) {
-               atomic_set_cpumask(&dfly_curprocmask, CPUMASK(cpuid));
+               ATOMIC_CPUMASK_ORBIT(dfly_curprocmask, cpuid);
                dd->upri = nlp->lwp_priority;
                dd->uschedcp = nlp;
 #if 0
@@ -828,7 +829,7 @@ dfly_schedulerclock(struct lwp *lp, sysclock_t period, sysclock_t cpstamp)
                 */
                if (nlp &&
                    (nlp->lwp_priority & ~PPQMASK) < (dd->upri & ~PPQMASK)) {
-                       atomic_set_cpumask(&dfly_curprocmask, dd->cpumask);
+                       ATOMIC_CPUMASK_ORMASK(dfly_curprocmask, dd->cpumask);
                        dd->upri = nlp->lwp_priority;
                        dd->uschedcp = nlp;
 #if 0
@@ -1133,7 +1134,7 @@ dfly_resetpriority(struct lwp *lp)
         * check which will fail in that case.
         */
        if (rcpu >= 0) {
-               if ((dfly_rdyprocmask & CPUMASK(rcpu)) &&
+               if (CPUMASK_TESTBIT(dfly_rdyprocmask, rcpu) &&
                    (checkpri == 0 ||
                     (rdd->upri & ~PRIMASK) >
                     (lp->lwp_priority & ~PRIMASK))) {
@@ -1455,7 +1456,7 @@ dfly_choose_best_queue(struct lwp *lp)
        if ((wakecpu = lp->lwp_thread->td_wakefromcpu) >= 0)
                wakemask = dfly_pcpu[wakecpu].cpumask;
        else
-               wakemask = 0;
+               CPUMASK_ASSZERO(wakemask);
 
        /*
         * When the topology is known choose a cpu whos group has, in
@@ -1490,15 +1491,17 @@ dfly_choose_best_queue(struct lwp *lp)
                         * which are members of this node.
                         */
                        cpun = cpup->child_node[n];
-                       mask = cpun->members & usched_global_cpumask &
-                              smp_active_mask & lp->lwp_cpumask;
-                       if (mask == 0)
+                       mask = cpun->members;
+                       CPUMASK_ANDMASK(mask, usched_global_cpumask);
+                       CPUMASK_ANDMASK(mask, smp_active_mask);
+                       CPUMASK_ANDMASK(mask, lp->lwp_cpumask);
+                       if (CPUMASK_TESTZERO(mask))
                                continue;
 
                        count = 0;
                        load = 0;
 
-                       while (mask) {
+                       while (CPUMASK_TESTNZERO(mask)) {
                                cpuid = BSFCPUMASK(mask);
                                rdd = &dfly_pcpu[cpuid];
                                load += rdd->uload;
@@ -1515,7 +1518,7 @@ dfly_choose_best_queue(struct lwp *lp)
                                        load -= usched_dfly_weight4 / 2;
                                }
 #endif
-                               mask &= ~CPUMASK(cpuid);
+                               CPUMASK_NANDBIT(mask, cpuid);
                                ++count;
                        }
 
@@ -1526,7 +1529,7 @@ dfly_choose_best_queue(struct lwp *lp)
                         * otherwise the calculation is bogus.
                         */
                        if ((lp->lwp_mpflags & LWP_MP_ULOAD) &&
-                           (dd->cpumask & cpun->members)) {
+                           CPUMASK_TESTMASK(dd->cpumask, cpun->members)) {
                                load -= lp->lwp_uload;
                                load -= usched_dfly_weight3;
                        }
@@ -1536,7 +1539,7 @@ dfly_choose_best_queue(struct lwp *lp)
                        /*
                         * Advantage the cpu group (lp) is already on.
                         */
-                       if (cpun->members & dd->cpumask)
+                       if (CPUMASK_TESTMASK(cpun->members, dd->cpumask))
                                load -= usched_dfly_weight1;
 
                        /*
@@ -1556,7 +1559,7 @@ dfly_choose_best_queue(struct lwp *lp)
                         * all-but-one by the same amount, so it won't effect
                         * the weight1 factor for the all-but-one nodes.
                         */
-                       if (cpun->members & wakemask) {
+                       if (CPUMASK_TESTMASK(cpun->members, wakemask)) {
                                if (cpun->child_no != 0) {
                                        /* advantage */
                                        load -= usched_dfly_weight2;
@@ -1573,7 +1576,7 @@ dfly_choose_best_queue(struct lwp *lp)
                         */
                        if (cpub == NULL || lowest_load > load ||
                            (lowest_load == load &&
-                            (cpun->members & dd->cpumask))
+                            CPUMASK_TESTMASK(cpun->members, dd->cpumask))
                        ) {
                                lowest_load = load;
                                cpub = cpun;
@@ -1657,14 +1660,15 @@ dfly_choose_worst_queue(dfly_pcpu_t dd)
                         * which are members of this node.
                         */
                        cpun = cpup->child_node[n];
-                       mask = cpun->members & usched_global_cpumask &
-                              smp_active_mask;
-                       if (mask == 0)
+                       mask = cpun->members;
+                       CPUMASK_ANDMASK(mask, usched_global_cpumask);
+                       CPUMASK_ANDMASK(mask, smp_active_mask);
+                       if (CPUMASK_TESTZERO(mask))
                                continue;
                        count = 0;
                        load = 0;
 
-                       while (mask) {
+                       while (CPUMASK_TESTNZERO(mask)) {
                                cpuid = BSFCPUMASK(mask);
                                rdd = &dfly_pcpu[cpuid];
                                load += rdd->uload;
@@ -1680,7 +1684,7 @@ dfly_choose_worst_queue(dfly_pcpu_t dd)
                                        load -= usched_dfly_weight4 / 2;
                                }
 #endif
-                               mask &= ~CPUMASK(cpuid);
+                               CPUMASK_NANDBIT(mask, cpuid);
                                ++count;
                        }
                        load /= count;
@@ -1689,7 +1693,7 @@ dfly_choose_worst_queue(dfly_pcpu_t dd)
                         * Prefer candidates which are somewhat closer to
                         * our cpu.
                         */
-                       if (dd->cpumask & cpun->members)
+                       if (CPUMASK_TESTMASK(dd->cpumask, cpun->members))
                                load += usched_dfly_weight1;
 
                        /*
@@ -1745,40 +1749,50 @@ dfly_choose_queue_simple(dfly_pcpu_t dd, struct lwp *lp)
         */
        ++dfly_scancpu;
        cpuid = (dfly_scancpu & 0xFFFF) % ncpus;
-       mask = ~dfly_curprocmask & dfly_rdyprocmask & lp->lwp_cpumask &
-              smp_active_mask & usched_global_cpumask;
-
-       while (mask) {
-               tmpmask = ~(CPUMASK(cpuid) - 1);
-               if (mask & tmpmask)
-                       cpuid = BSFCPUMASK(mask & tmpmask);
-               else
+       mask = dfly_rdyprocmask;
+       CPUMASK_NANDMASK(mask, dfly_curprocmask);
+       CPUMASK_ANDMASK(mask, lp->lwp_cpumask);
+       CPUMASK_ANDMASK(mask, smp_active_mask);
+       CPUMASK_ANDMASK(mask, usched_global_cpumask);
+
+       while (CPUMASK_TESTNZERO(mask)) {
+               CPUMASK_ASSNBMASK(tmpmask, cpuid);
+               if (CPUMASK_TESTMASK(tmpmask, mask)) {
+                       CPUMASK_ANDMASK(tmpmask, mask);
+                       cpuid = BSFCPUMASK(tmpmask);
+               } else {
                        cpuid = BSFCPUMASK(mask);
+               }
                rdd = &dfly_pcpu[cpuid];
 
                if ((rdd->upri & ~PPQMASK) >= (lp->lwp_priority & ~PPQMASK))
                        goto found;
-               mask &= ~CPUMASK(cpuid);
+               CPUMASK_NANDBIT(mask, cpuid);
        }
 
        /*
         * Then cpus which might have a currently running lp
         */
        cpuid = (dfly_scancpu & 0xFFFF) % ncpus;
-       mask = dfly_curprocmask & dfly_rdyprocmask &
-              lp->lwp_cpumask & smp_active_mask & usched_global_cpumask;
-
-       while (mask) {
-               tmpmask = ~(CPUMASK(cpuid) - 1);
-               if (mask & tmpmask)
-                       cpuid = BSFCPUMASK(mask & tmpmask);
-               else
+       mask = dfly_rdyprocmask;
+       CPUMASK_ANDMASK(mask, dfly_curprocmask);
+       CPUMASK_ANDMASK(mask, lp->lwp_cpumask);
+       CPUMASK_ANDMASK(mask, smp_active_mask);
+       CPUMASK_ANDMASK(mask, usched_global_cpumask);
+
+       while (CPUMASK_TESTNZERO(mask)) {
+               CPUMASK_ASSNBMASK(tmpmask, cpuid);
+               if (CPUMASK_TESTMASK(tmpmask, mask)) {
+                       CPUMASK_ANDMASK(tmpmask, mask);
+                       cpuid = BSFCPUMASK(tmpmask);
+               } else {
                        cpuid = BSFCPUMASK(mask);
+               }
                rdd = &dfly_pcpu[cpuid];
 
                if ((rdd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK))
                        goto found;
-               mask &= ~CPUMASK(cpuid);
+               CPUMASK_NANDBIT(mask, cpuid);
        }
 
        /*
@@ -1793,7 +1807,7 @@ dfly_choose_queue_simple(dfly_pcpu_t dd, struct lwp *lp)
         * set the user resched flag because
         */
        cpuid = (dfly_scancpu & 0xFFFF) % ncpus;
-       if ((CPUMASK(cpuid) & usched_global_cpumask) == 0)
+       if (CPUMASK_TESTBIT(usched_global_cpumask, cpuid) == 0)
                cpuid = 0;
        rdd = &dfly_pcpu[cpuid];
 found:
@@ -1822,8 +1836,9 @@ dfly_need_user_resched_remote(void *dummy)
         *
         * Call wakeup_mycpu to avoid sending IPIs to other CPUs
         */
-       if (dd->uschedcp == NULL && (dfly_rdyprocmask & gd->gd_cpumask)) {
-               atomic_clear_cpumask(&dfly_rdyprocmask, gd->gd_cpumask);
+       if (dd->uschedcp == NULL &&
+           CPUMASK_TESTBIT(dfly_rdyprocmask, gd->gd_cpuid)) {
+               ATOMIC_CPUMASK_NANDBIT(dfly_rdyprocmask, gd->gd_cpuid);
                wakeup_mycpu(&dd->helper_thread);
        }
 }
@@ -2009,7 +2024,7 @@ dfly_helper_thread(void *dummy)
 
        spin_lock(&dd->spin);
 
-       atomic_set_cpumask(&dfly_rdyprocmask, mask);
+       ATOMIC_CPUMASK_ORMASK(dfly_rdyprocmask, mask);
        clear_user_resched();   /* This satisfied the reschedule request */
 #if 0
        dd->rrcount = 0;        /* Reset the round-robin counter */
@@ -2023,7 +2038,7 @@ dfly_helper_thread(void *dummy)
                 */
                nlp = dfly_chooseproc_locked(dd, dd, dd->uschedcp, 0);
                if (nlp) {
-                       atomic_set_cpumask(&dfly_curprocmask, mask);
+                       ATOMIC_CPUMASK_ORMASK(dfly_curprocmask, mask);
                        dd->upri = nlp->lwp_priority;
                        dd->uschedcp = nlp;
 #if 0
@@ -2063,7 +2078,7 @@ dfly_helper_thread(void *dummy)
                        nlp = NULL;
                }
                if (nlp) {
-                       atomic_set_cpumask(&dfly_curprocmask, mask);
+                       ATOMIC_CPUMASK_ORMASK(dfly_curprocmask, mask);
                        dd->upri = nlp->lwp_priority;
                        dd->uschedcp = nlp;
 #if 0
@@ -2140,21 +2155,22 @@ usched_dfly_cpu_init(void)
 
        for (i = 0; i < ncpus; ++i) {
                dfly_pcpu_t dd = &dfly_pcpu[i];
-               cpumask_t mask = CPUMASK(i);
+               cpumask_t mask;
 
-               if ((mask & smp_active_mask) == 0)
+               CPUMASK_ASSBIT(mask, i);
+               if (CPUMASK_TESTMASK(mask, smp_active_mask) == 0)
                    continue;
 
                spin_init(&dd->spin);
                dd->cpunode = get_cpu_node_by_cpuid(i);
                dd->cpuid = i;
-               dd->cpumask = CPUMASK(i);
+               CPUMASK_ASSBIT(dd->cpumask, i);
                for (j = 0; j < NQS; j++) {
                        TAILQ_INIT(&dd->queues[j]);
                        TAILQ_INIT(&dd->rtqueues[j]);
                        TAILQ_INIT(&dd->idqueues[j]);
                }
-               atomic_clear_cpumask(&dfly_curprocmask, 1);
+               ATOMIC_CPUMASK_NANDBIT(dfly_curprocmask, 0);
 
                if (dd->cpunode == NULL) {
                        smt_not_supported = 1;
@@ -2219,8 +2235,8 @@ usched_dfly_cpu_init(void)
                 * been enabled in rqinit().
                 */
                if (i)
-                   atomic_clear_cpumask(&dfly_curprocmask, mask);
-               atomic_set_cpumask(&dfly_rdyprocmask, mask);
+                       ATOMIC_CPUMASK_NANDMASK(dfly_curprocmask, mask);
+               ATOMIC_CPUMASK_ORMASK(dfly_rdyprocmask, mask);
                dd->upri = PRIBASE_NULL;
 
        }
index 919e27a..e747c8f 100644 (file)
@@ -102,7 +102,7 @@ struct usched_dummy_pcpu {
 typedef struct usched_dummy_pcpu *dummy_pcpu_t;
 
 static struct usched_dummy_pcpu dummy_pcpu[MAXCPU];
-static cpumask_t dummy_curprocmask = -1;
+static cpumask_t dummy_curprocmask = CPUMASK_INITIALIZER_ALLONES;
 static cpumask_t dummy_rdyprocmask;
 static struct spinlock dummy_spin;
 static TAILQ_HEAD(rq, lwp) dummy_runq;
@@ -121,7 +121,7 @@ dummyinit(void *dummy)
 {
        TAILQ_INIT(&dummy_runq);
        spin_init(&dummy_spin);
-       atomic_clear_cpumask(&dummy_curprocmask, 1);
+       ATOMIC_CPUMASK_NANDBIT(dummy_curprocmask, 0);
 }
 SYSINIT(runqueue, SI_BOOT2_USCHED, SI_ORDER_FIRST, dummyinit, NULL)
 
@@ -164,7 +164,7 @@ dummy_acquire_curproc(struct lwp *lp)
         */
        if (dd->uschedcp == lp ||
            (dd->uschedcp == NULL && TAILQ_EMPTY(&dummy_runq))) {
-               atomic_set_cpumask(&dummy_curprocmask, gd->gd_cpumask);
+               ATOMIC_CPUMASK_ORBIT(dummy_curprocmask, gd->gd_cpuid);
                dd->uschedcp = lp;
                return;
        }
@@ -243,14 +243,14 @@ dummy_select_curproc(globaldata_t gd)
        spin_lock(&dummy_spin);
        if ((lp = TAILQ_FIRST(&dummy_runq)) == NULL) {
                dd->uschedcp = NULL;
-               atomic_clear_cpumask(&dummy_curprocmask, gd->gd_cpumask);
+               ATOMIC_CPUMASK_NANDBIT(dummy_curprocmask, gd->gd_cpuid);
                spin_unlock(&dummy_spin);
        } else {
                --dummy_runqcount;
                TAILQ_REMOVE(&dummy_runq, lp, lwp_procq);
                atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ);
                dd->uschedcp = lp;
-               atomic_set_cpumask(&dummy_curprocmask, gd->gd_cpumask);
+               ATOMIC_CPUMASK_ORBIT(dummy_curprocmask, gd->gd_cpuid);
                spin_unlock(&dummy_spin);
                lwkt_acquire(lp->lwp_thread);
                lwkt_schedule(lp->lwp_thread);
@@ -280,7 +280,7 @@ dummy_setrunqueue(struct lwp *lp)
 
        if (dd->uschedcp == NULL) {
                dd->uschedcp = lp;
-               atomic_set_cpumask(&dummy_curprocmask, gd->gd_cpumask);
+               ATOMIC_CPUMASK_ORBIT(dummy_curprocmask, gd->gd_cpuid);
                lwkt_schedule(lp->lwp_thread);
        } else {
                /*
@@ -304,11 +304,12 @@ dummy_setrunqueue(struct lwp *lp)
                 * helper thread cannot find a home for it it will forward
                 * the request to another available cpu.
                 */
-               mask = ~dummy_curprocmask & dummy_rdyprocmask & 
-                      gd->gd_other_cpus;
-               if (mask) {
+               mask = dummy_rdyprocmask;
+               CPUMASK_NANDMASK(mask, dummy_curprocmask);
+               CPUMASK_ANDMASK(mask, gd->gd_other_cpus);
+               if (CPUMASK_TESTNZERO(mask)) {
                        cpuid = BSFCPUMASK(mask);
-                       atomic_clear_cpumask(&dummy_rdyprocmask, CPUMASK(cpuid));
+                       ATOMIC_CPUMASK_NANDBIT(dummy_rdyprocmask, cpuid);
                        spin_unlock(&dummy_spin);
                        lwkt_schedule(&dummy_pcpu[cpuid].helper_thread);
                } else {
@@ -475,23 +476,24 @@ dummy_sched_thread(void *dummy)
     gd = mycpu;
     cpuid = gd->gd_cpuid;
     dd = &dummy_pcpu[cpuid];
-    cpumask = CPUMASK(cpuid);
+    CPUMASK_ASSBIT(cpumask, cpuid);
 
     for (;;) {
        lwkt_deschedule_self(gd->gd_curthread);         /* interlock */
-       atomic_set_cpumask(&dummy_rdyprocmask, cpumask);
+       ATOMIC_CPUMASK_ORBIT(dummy_rdyprocmask, cpuid);
        spin_lock(&dummy_spin);
        if (dd->uschedcp) {
                /*
                 * We raced another cpu trying to schedule a thread onto us.
                 * If the runq isn't empty hit another free cpu.
                 */
-               tmpmask = ~dummy_curprocmask & dummy_rdyprocmask & 
-                         gd->gd_other_cpus;
-               if (tmpmask && dummy_runqcount) {
+               tmpmask = dummy_rdyprocmask;
+               CPUMASK_NANDMASK(tmpmask, dummy_curprocmask);
+               CPUMASK_ANDMASK(tmpmask, gd->gd_other_cpus);
+               if (CPUMASK_TESTNZERO(tmpmask) && dummy_runqcount) {
                        tmpid = BSFCPUMASK(tmpmask);
                        KKASSERT(tmpid != cpuid);
-                       atomic_clear_cpumask(&dummy_rdyprocmask, CPUMASK(tmpid));
+                       ATOMIC_CPUMASK_NANDBIT(dummy_rdyprocmask, tmpid);
                        spin_unlock(&dummy_spin);
                        lwkt_schedule(&dummy_pcpu[tmpid].helper_thread);
                } else {
@@ -502,7 +504,7 @@ dummy_sched_thread(void *dummy)
                TAILQ_REMOVE(&dummy_runq, lp, lwp_procq);
                atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ);
                dd->uschedcp = lp;
-               atomic_set_cpumask(&dummy_curprocmask, cpumask);
+               ATOMIC_CPUMASK_ORBIT(dummy_curprocmask, cpuid);
                spin_unlock(&dummy_spin);
                lwkt_acquire(lp->lwp_thread);
                lwkt_schedule(lp->lwp_thread);
@@ -527,9 +529,11 @@ dummy_sched_thread_cpu_init(void)
 
     for (i = 0; i < ncpus; ++i) {
        dummy_pcpu_t dd = &dummy_pcpu[i];
-       cpumask_t mask = CPUMASK(i);
+       cpumask_t mask;
+
+       CPUMASK_ASSBIT(mask, i);
 
-       if ((mask & smp_active_mask) == 0)
+       if (CPUMASK_TESTMASK(mask, smp_active_mask) == 0)
            continue;
 
        if (bootverbose)
@@ -543,8 +547,8 @@ dummy_sched_thread_cpu_init(void)
         * been enabled in rqinit().
         */
        if (i)
-           atomic_clear_cpumask(&dummy_curprocmask, mask);
-       atomic_set_cpumask(&dummy_rdyprocmask, mask);
+               ATOMIC_CPUMASK_NANDMASK(dummy_curprocmask, mask);
+       ATOMIC_CPUMASK_ORMASK(dummy_rdyprocmask, mask);
     }
     if (bootverbose)
        kprintf("\n");
index e428405..2b90c9a 100644 (file)
@@ -611,8 +611,8 @@ netisr_barrier_dispatch(netmsg_t nmsg)
 {
        struct netmsg_barrier *msg = (struct netmsg_barrier *)nmsg;
 
-       atomic_clear_cpumask(msg->br_cpumask, mycpu->gd_cpumask);
-       if (*msg->br_cpumask == 0)
+       ATOMIC_CPUMASK_NANDBIT(*msg->br_cpumask, mycpu->gd_cpuid);
+       if (CPUMASK_TESTZERO(*msg->br_cpumask))
                wakeup(msg->br_cpumask);
 
        for (;;) {
@@ -649,7 +649,8 @@ netisr_barrier_set(struct netisr_barrier *br)
        KKASSERT(&curthread->td_msgport == netisr_cpuport(0));
        KKASSERT(!br->br_isset);
 
-       other_cpumask = mycpu->gd_other_cpus & smp_active_mask;
+       other_cpumask = mycpu->gd_other_cpus;
+       CPUMASK_ANDMASK(other_cpumask, smp_active_mask);
        cur_cpuid = mycpuid;
 
        for (i = 0; i < ncpus; ++i) {
@@ -667,7 +668,7 @@ netisr_barrier_set(struct netisr_barrier *br)
                 * the caller.
                 */
                netmsg_init(&msg->base, NULL, &netisr_afree_rport, 0,
-                   netisr_barrier_dispatch);
+                           netisr_barrier_dispatch);
                msg->br_cpumask = &other_cpumask;
                msg->br_done = NETISR_BR_NOTDONE;
 
@@ -681,9 +682,9 @@ netisr_barrier_set(struct netisr_barrier *br)
                lwkt_sendmsg(netisr_cpuport(i), &br->br_msgs[i]->base.lmsg);
        }
 
-       while (other_cpumask != 0) {
+       while (CPUMASK_TESTNZERO(other_cpumask)) {
                tsleep_interlock(&other_cpumask, 0);
-               if (other_cpumask != 0)
+               if (CPUMASK_TESTNZERO(other_cpumask))
                        tsleep(&other_cpumask, PINTERLOCKED, "nbrset", 0);
        }
        br->br_isset = 1;
index bd25e4e..039420c 100644 (file)
@@ -431,15 +431,16 @@ ipflow_timo_ipi(void *arg __unused)
 void
 ipflow_slowtimo(void)
 {
-       cpumask_t mask = 0;
+       cpumask_t mask;
        int i;
 
+       CPUMASK_ASSZERO(mask);
        for (i = 0; i < ncpus; ++i) {
                if (ipflow_inuse_pcpu[i])
-                       mask |= CPUMASK(i);
+                       CPUMASK_ORBIT(mask, i);
        }
-       mask &= smp_active_mask;
-       if (mask != 0)
+       CPUMASK_ANDMASK(mask, smp_active_mask);
+       if (CPUMASK_TESTNZERO(mask))
                lwkt_send_ipiq_mask(mask, ipflow_timo_ipi, NULL);
 }
 
index 5aa234c..88c6f81 100644 (file)
@@ -1398,13 +1398,11 @@ ipfrag_timeo_ipi(void *arg __unused)
 static void
 ipfrag_slowtimo(void)
 {
-       cpumask_t mask = 0;
-       int i;
+       cpumask_t mask;
 
-       for (i = 0; i < ncpus; ++i)
-               mask |= CPUMASK(i);
-       mask &= smp_active_mask;
-       if (mask != 0)
+       CPUMASK_ASSBMASK(mask, ncpus);
+       CPUMASK_ANDMASK(mask, smp_active_mask);
+       if (CPUMASK_TESTNZERO(mask))
                lwkt_send_ipiq_mask(mask, ipfrag_timeo_ipi, NULL);
 }
 
@@ -1456,13 +1454,11 @@ ipfrag_drain_ipi(void *arg __unused)
 static void
 ipfrag_drain(void)
 {
-       cpumask_t mask = 0;
-       int i;
+       cpumask_t mask;
 
-       for (i = 0; i < ncpus; ++i)
-               mask |= CPUMASK(i);
-       mask &= smp_active_mask;
-       if (mask != 0)
+       CPUMASK_ASSBMASK(mask, ncpus);
+       CPUMASK_ANDMASK(mask, smp_active_mask);
+       if (CPUMASK_TESTNZERO(mask))
                lwkt_send_ipiq_mask(mask, ipfrag_drain_ipi, NULL);
 }
 
index 1bdf72d..754db2b 100644 (file)
@@ -587,7 +587,7 @@ selected_apic_ipi(cpumask_t target, int vector, int delivery_mode)
        crit_enter();
        while (target) {
                int n = BSFCPUMASK(target);
-               target &= ~CPUMASK(n);
+               CPUMASK_NANDBIT(target, n);
                single_apic_ipi(n, vector, delivery_mode);
        }
        crit_exit();
index 9a06308..4624935 100644 (file)
@@ -84,7 +84,7 @@ int   single_apic_ipi_passive(int, int, int);
 static __inline int
 all_but_self_ipi(int vector)
 {
-       if (smp_active_mask == 1)
+       if (CPUMASK_ISUP(smp_active_mask))
                return 0;
        return apic_ipi(APIC_DEST_ALLESELF, vector, APIC_DELMODE_FIXED);
 }
index 964aeb9..6454027 100644 (file)
@@ -173,10 +173,14 @@ static int        start_ap(struct mdglobaldata *gd, u_int boot_addr, int smibest);
 static int     smitest(void);
 static void    mp_bsp_simple_setup(void);
 
-static cpumask_t smp_startup_mask = 1; /* which cpus have been started */
-static cpumask_t smp_lapic_mask = 1;   /* which cpus have lapic been inited */
-cpumask_t smp_active_mask = 1; /* which cpus are ready for IPIs etc? */
-SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RD, &smp_active_mask, 0, "");
+/* which cpus have been started */
+static cpumask_t smp_startup_mask = CPUMASK_INITIALIZER_ONLYONE;
+/* which cpus have lapic been inited */
+static cpumask_t smp_lapic_mask = CPUMASK_INITIALIZER_ONLYONE;
+/* which cpus are ready for IPIs etc? */
+cpumask_t smp_active_mask = CPUMASK_INITIALIZER_ONLYONE;
+SYSCTL_LONG(_machdep, OID_AUTO, smp_active, CTLFLAG_RD,
+           &smp_active_mask, 0, "");
 
 /* Local data for detecting CPU TOPOLOGY */
 static int core_bits = 0;
@@ -473,8 +477,10 @@ start_all_aps(u_int boot_addr)
        ncpus_fit_mask = ncpus_fit - 1;
 
        /* build our map of 'other' CPUs */
-       mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
-       mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * ncpus);
+       mycpu->gd_other_cpus = smp_startup_mask;
+       CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid);
+       mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map,
+                                           sizeof(lwkt_ipiq) * ncpus);
        bzero(mycpu->gd_ipiq, sizeof(lwkt_ipiq) * ncpus);
 
        /* restore the warmstart vector */
@@ -500,7 +506,7 @@ start_all_aps(u_int boot_addr)
                tsc0_offset = rdtsc();
        tsc_offsets[0] = 0;
        rel_mplock();
-       while (smp_lapic_mask != smp_startup_mask) {
+       while (CPUMASK_CMPMASKNEQ(smp_lapic_mask, smp_startup_mask)) {
                cpu_lfence();
                if (cpu_feature & CPUID_TSC)
                        tsc0_offset = rdtsc();
@@ -714,7 +720,7 @@ start_ap(struct mdglobaldata *gd, u_int boot_addr, int smibest)
        /* wait for it to start, see ap_init() */
        set_apic_timer(5000000);/* == 5 seconds */
        while (read_apic_timer()) {
-               if (smp_startup_mask & CPUMASK(gd->mi.gd_cpuid))
+               if (CPUMASK_TESTBIT(smp_startup_mask, gd->mi.gd_cpuid))
                        return 1;       /* return SUCCESS */
        }
 
@@ -768,27 +774,36 @@ smp_invltlb(void)
        long count = 0;
        long xcount = 0;
 #endif
+       cpumask_t tmpmask;
+       cpumask_t tmpmask2;
 
        crit_enter_gd(&md->mi);
        md->gd_invltlb_ret = 0;
        ++md->mi.gd_cnt.v_smpinvltlb;
-       atomic_set_cpumask(&smp_invltlb_req, md->mi.gd_cpumask);
+       ATOMIC_CPUMASK_ORMASK(smp_invltlb_req, md->mi.gd_cpumask);
 #ifdef SMP_INVLTLB_DEBUG
 again:
 #endif
-       if (smp_startup_mask == smp_active_mask) {
+       if (CPUMASK_CMPMASKEQ(smp_startup_mask, smp_active_mask)) {
                all_but_self_ipi(XINVLTLB_OFFSET);
        } else {
-               selected_apic_ipi(smp_active_mask & ~md->mi.gd_cpumask,
-                                 XINVLTLB_OFFSET, APIC_DELMODE_FIXED);
+               tmpmask = smp_active_mask;
+               CPUMASK_NANDMASK(tmpmask, md->mi.gd_cpumask);
+               selected_apic_ipi(tmpmask, XINVLTLB_OFFSET, APIC_DELMODE_FIXED);
        }
 
 #ifdef SMP_INVLTLB_DEBUG
        if (xcount)
                kprintf("smp_invltlb: ipi sent\n");
 #endif
-       while ((md->gd_invltlb_ret & smp_active_mask & ~md->mi.gd_cpumask) !=
-              (smp_active_mask & ~md->mi.gd_cpumask)) {
+       for (;;) {
+               tmpmask = smp_active_mask;
+               tmpmask2 = tmpmask;
+               CPUMASK_ANDMASK(tmpmask, md->gd_invltlb_ret);
+               CPUMASK_NANDMASK(tmpmask, md->mi.gd_cpumask);
+               CPUMASK_NANDMASK(tmpmask2, md->mi.gd_cpumask);
+               if (CPUMASK_CMPMASKEQ(tmpmask, tmpmask2))
+                       break;
                cpu_mfence();
                cpu_pause();
 #ifdef SMP_INVLTLB_DEBUG
@@ -805,13 +820,19 @@ again:
                        if (xcount > 2)
                                lwkt_process_ipiq();
                        if (xcount > 3) {
-                               int bcpu = BSFCPUMASK(~md->gd_invltlb_ret &
-                                                     ~md->mi.gd_cpumask &
-                                                     smp_active_mask);
                                globaldata_t xgd;
+                               int bcpu;
+
+                               tmpmask = smp_active_mask;
+                               CPUMASK_NANDMASK(tmpmask, md->gd_invltlb_ret);
+                               CPUMASK_NANDMASK(tmpmask, md->mi.gd_cpumask);
+                               bcpu = BSFCPUMASK(tmpmask);
+
                                kprintf("bcpu %d\n", bcpu);
                                xgd = globaldata_find(bcpu);
-                               kprintf("thread %p %s\n", xgd->gd_curthread, xgd->gd_curthread->td_comm);
+                               kprintf("thread %p %s\n",
+                                       xgd->gd_curthread,
+                                       xgd->gd_curthread->td_comm);
                        }
                        if (xcount > 5)
                                panic("giving up");
@@ -820,7 +841,7 @@ again:
                }
 #endif
        }
-       atomic_clear_cpumask(&smp_invltlb_req, md->mi.gd_cpumask);
+       ATOMIC_CPUMASK_NANDMASK(smp_invltlb_req, md->mi.gd_cpumask);
        crit_exit_gd(&md->mi);
 }
 
@@ -840,11 +861,11 @@ smp_invltlb_intr(void)
        mask = smp_invltlb_req;
        cpu_mfence();
        cpu_invltlb();
-       while (mask) {
+       while (CPUMASK_TESTNZERO(mask)) {
                cpu = BSFCPUMASK(mask);
-               mask &= ~CPUMASK(cpu);
+               CPUMASK_NANDBIT(mask, cpu);
                omd = (struct mdglobaldata *)globaldata_find(cpu);
-               atomic_set_cpumask(&omd->gd_invltlb_ret, md->mi.gd_cpumask);
+               ATOMIC_CPUMASK_ORMASK(omd->gd_invltlb_ret, md->mi.gd_cpumask);
        }
 }
 
@@ -874,13 +895,18 @@ cpu_wbinvd_on_all_cpus_callback(void *arg)
 int
 stop_cpus(cpumask_t map)
 {
+       cpumask_t tmpmask;
+
        map &= smp_active_mask;
 
        /* send the Xcpustop IPI to all CPUs in map */
        selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
-       
-       while ((stopped_cpus & map) != map)
+
+       do {
+               tmpmask = map;
+               CPUMASK_ANDMASK(tmpmask, stopped_cpus);
                /* spin */ ;
+       } while (CPUMASK_CMPMASKNEQ(tmpmask, map));
 
        return 1;
 }
@@ -905,7 +931,8 @@ restart_cpus(cpumask_t map)
        /* signal other cpus to restart */
        started_cpus = map & smp_active_mask;
 
-       while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+       /* wait for each to clear its bit */
+       while (CPUMASK_TESTMASK(stopped_cpus, map) != 0)
                /* spin */ ;
 
        return 1;
@@ -932,7 +959,7 @@ ap_init(void)
         * interrupts physically disabled and remote cpus could deadlock
         * trying to send us an IPI.
         */
-       smp_startup_mask |= CPUMASK(mycpu->gd_cpuid);
+       ATOMIC_CPUMASK_ORBIT(smp_startup_mask, mycpu->gd_cpuid);
        cpu_mfence();
 
        /*
@@ -968,7 +995,8 @@ ap_init(void)
 #endif
 
        /* Build our map of 'other' CPUs. */
-       mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
+       mycpu->gd_other_cpus = smp_startup_mask;
+       CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid);
 
        /* A quick check from sanity claus */
        cpu_id = APICID_TO_CPUID((lapic->id & 0xff000000) >> 24);
@@ -983,7 +1011,7 @@ ap_init(void)
        lapic_init(FALSE);
 
        /* LAPIC initialization is done */
-       smp_lapic_mask |= CPUMASK(mycpu->gd_cpuid);
+       ATOMIC_CPUMASK_ORBIT(smp_lapic_mask, mycpu->gd_cpuid);
        cpu_mfence();
 
        /* Let BSP move onto the next initialization stage */
@@ -1027,7 +1055,7 @@ ap_init(void)
         * nothing we've done put it there.
         */
        KKASSERT(get_mplock_count(curthread) == 1);
-       smp_active_mask |= CPUMASK(mycpu->gd_cpuid);
+       ATOMIC_CPUMASK_ORBIT(smp_active_mask, mycpu->gd_cpuid);
 
        /*
         * Enable interrupts here.  idle_restore will also do it, but
@@ -1072,7 +1100,7 @@ SYSINIT(finishsmp, SI_BOOT2_FINISH_SMP, SI_ORDER_FIRST, ap_finish, NULL)
 void
 cpu_send_ipiq(int dcpu)
 {
-        if (CPUMASK(dcpu) & smp_active_mask)
+       if (CPUMASK_TESTBIT(smp_active_mask, dcpu))
                 single_apic_ipi(dcpu, XIPIQ_OFFSET, APIC_DELMODE_FIXED);
 }
 
@@ -1084,7 +1112,7 @@ int
 cpu_send_ipiq_passive(int dcpu)
 {
         int r = 0;
-        if (CPUMASK(dcpu) & smp_active_mask) {
+       if (CPUMASK_TESTBIT(smp_active_mask, dcpu)) {
                 r = single_apic_ipi_passive(dcpu, XIPIQ_OFFSET,
                                         APIC_DELMODE_FIXED);
         }
@@ -1096,8 +1124,10 @@ static void
 mp_bsp_simple_setup(void)
 {
        /* build our map of 'other' CPUs */
-       mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
-       mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * ncpus);
+       mycpu->gd_other_cpus = smp_startup_mask;
+       CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid);
+       mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map,
+                                           sizeof(lwkt_ipiq) * ncpus);
        bzero(mycpu->gd_ipiq, sizeof(lwkt_ipiq) * ncpus);
 
        pmap_set_opt();
index 9da1895..4d5b163 100644 (file)
@@ -449,7 +449,7 @@ mptable_hyperthread_fixup(cpumask_t id_mask, int cpu_count)
                 */
                dist = cur = prev = -1;
                for (id = 0; id < MAXCPU; ++id) {
-                       if ((id_mask & CPUMASK(id)) == 0)
+                       if (CPUMASK_TESTBIT(id_mask, id) == 0)
                                continue;
 
                        cur = id;
@@ -490,13 +490,13 @@ mptable_hyperthread_fixup(cpumask_t id_mask, int cpu_count)
         * already in the table, then kill the fixup.
         */
        for (id = 0; id < MAXCPU; id++) {
-               if ((id_mask & CPUMASK(id)) == 0)
+               if (CPUMASK_TESTBIT(id_mask, id) == 0)
                        continue;
                /* First, make sure we are on a logical_cpus boundary. */
                if (id % logical_cpus != 0)
                        return 0;
                for (i = id + 1; i < id + logical_cpus; i++)
-                       if ((id_mask & CPUMASK(i)) != 0)
+                       if (CPUMASK_TESTBIT(id_mask, i))
                                return 0;
        }
        return logical_cpus;
index ac609ad..704d54e 100644 (file)
@@ -834,10 +834,10 @@ get_ptbase(pmap_t pmap)
 
        if ((*gd->gd_GDMAP1 & PG_FRAME) != frame) {
                *gd->gd_GDMAP1 = frame | PG_RW | PG_V;
-               pmap->pm_cached |= gd->mi.gd_cpumask;
+               CPUMASK_ORMASK(pmap->pm_cached, gd->mi.gd_cpumask);
                cpu_invltlb();
-       } else if ((pmap->pm_cached & gd->mi.gd_cpumask) == 0) {
-               pmap->pm_cached |= gd->mi.gd_cpumask;
+       } else if (CPUMASK_TESTMASK(pmap->pm_cached, gd->mi.gd_cpumask) == 0) {
+               CPUMASK_ORMASK(pmap->pm_cached, gd->mi.gd_cpumask);
                cpu_invltlb();
        } else if (dreadful_invltlb) {
                cpu_invltlb();
@@ -3775,7 +3775,8 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm)
                lp->lwp_vmspace = newvm;
                if (curthread->td_lwp == lp) {
                        pmap = vmspace_pmap(newvm);
-                       atomic_set_cpumask(&pmap->pm_active, mycpu->gd_cpumask);
+                       ATOMIC_CPUMASK_ORMASK(pmap->pm_active,
+                                             mycpu->gd_cpumask);
                        if (pmap->pm_active_lock & CPULOCK_EXCL)
                                pmap_interlock_wait(newvm);
 #if defined(SWTCH_OPTIM_STATS)
@@ -3784,8 +3785,8 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm)
                        curthread->td_pcb->pcb_cr3 = vtophys(pmap->pm_pdir);
                        load_cr3(curthread->td_pcb->pcb_cr3);
                        pmap = vmspace_pmap(oldvm);
-                       atomic_clear_cpumask(&pmap->pm_active,
-                                            mycpu->gd_cpumask);
+                       ATOMIC_CPUMASK_NANDMASK(pmap->pm_active,
+                                               mycpu->gd_cpumask);
                }
        }
 }
index c16f4fd..114fa73 100644 (file)
@@ -382,9 +382,11 @@ cpu_reset(void)
                int cnt;
                kprintf("cpu_reset called on cpu#%d\n",mycpu->gd_cpuid);
 
-               map = mycpu->gd_other_cpus & ~stopped_cpus & smp_active_mask;
+               map = mycpu->gd_other_cpus;
+               CPUMASK_NANDMASK(map, stopped_cpus);
+               CPUMASK_ANDMASK(map, smp_active_mask);
 
-               if (map != 0) {
+               if (CPUMASK_TESTNZERO(map)) {
                        kprintf("cpu_reset: Stopping other CPUs\n");
                        stop_cpus(map);         /* Stop all other CPUs */
                }
index 50a6848..f39396d 100644 (file)
@@ -654,9 +654,9 @@ void
 selected_apic_ipi(cpumask_t target, int vector, int delivery_mode)
 {
        crit_enter();
-       while (target) {
+       while (CPUMASK_TESTNZERO(target)) {
                int n = BSFCPUMASK(target);
-               target &= ~CPUMASK(n);
+               CPUMASK_NANDBIT(target, n);
                single_apic_ipi(n, vector, delivery_mode);
        }
        crit_exit();
index b308d95..bf607c0 100644 (file)
@@ -85,7 +85,7 @@ int   single_apic_ipi_passive(int, int, int);
 static __inline int
 all_but_self_ipi(int vector)
 {
-       if (smp_active_mask == 1)
+       if (CPUMASK_ISUP(smp_active_mask))
                return 0;
        return apic_ipi(APIC_DEST_ALLESELF, vector, APIC_DELMODE_FIXED);
 }
index fff4d57..5b97da2 100644 (file)
@@ -1239,7 +1239,7 @@ tsc_mpsync_test(void)
 
                crit_enter();
                lwkt_cpusync_init(&cs, gd->gd_other_cpus,
-                   tsc_mpsync_test_remote, NULL);
+                                 tsc_mpsync_test_remote, NULL);
                lwkt_cpusync_interlock(&cs);
                tsc_mpsync_target = rdtsc();
                cpu_mfence();
index f038077..662d1d8 100644 (file)
@@ -338,7 +338,8 @@ vmx_init(void)
            PROCBASED_ACTIVATE_SECONDARY_CONTROLS,
            ONE);
        if (err) {
-               kprintf("VMM: PROCBASED_ACTIVATE_SECONDARY_CONTROLS not supported by this CPU\n");
+               kprintf("VMM: PROCBASED_ACTIVATE_SECONDARY_CONTROLS not "
+                       "supported by this CPU\n");
                return (ENODEV);
        }
        vmx_set_default_settings(&vmx_procbased2);
@@ -351,44 +352,43 @@ vmx_init(void)
            PINBASED_EXTERNAL_INTERRUPT_EXITING,
            ONE);
        if (err) {
-               kprintf("VMM: PINBASED_EXTERNAL_INTERRUPT_EXITING not supported by this CPU\n");
+               kprintf("VMM: PINBASED_EXTERNAL_INTERRUPT_EXITING not "
+                       "supported by this CPU\n");
                return (ENODEV);
        }
 
        /* Enable non-maskable interrupts exiting */
-       err = vmx_set_ctl_setting(&vmx_pinbased,
-           PINBASED_NMI_EXITING,
-           ONE);
+       err = vmx_set_ctl_setting(&vmx_pinbased, PINBASED_NMI_EXITING, ONE);
        if (err) {
-               kprintf("VMM: PINBASED_NMI_EXITING not supported by this CPU\n");
+               kprintf("VMM: PINBASED_NMI_EXITING not "
+                       "supported by this CPU\n");
                return (ENODEV);
        }
 
 
        /* Set 64bits mode for GUEST */
-       err = vmx_set_ctl_setting(&vmx_entry,
-           VMENTRY_IA32e_MODE_GUEST,
-           ONE);
+       err = vmx_set_ctl_setting(&vmx_entry, VMENTRY_IA32e_MODE_GUEST, ONE);
        if (err) {
-               kprintf("VMM: VMENTRY_IA32e_MODE_GUEST not supported by this CPU\n");
+               kprintf("VMM: VMENTRY_IA32e_MODE_GUEST not "
+                       "supported by this CPU\n");
                return (ENODEV);
        }
 
        /* Load MSR EFER on enry */
        err = vmx_set_ctl_setting(&vmx_entry,
-           VMENTRY_LOAD_IA32_EFER,
-           ONE);
+                                 VMENTRY_LOAD_IA32_EFER, ONE);
        if (err) {
-               kprintf("VMM: VMENTRY_LOAD_IA32_EFER not supported by this CPU\n");
+               kprintf("VMM: VMENTRY_LOAD_IA32_EFER not "
+                       "supported by this CPU\n");
                return (ENODEV);
        }
 
        /* Set 64bits mode */
        err = vmx_set_ctl_setting(&vmx_exit,
-           VMEXIT_HOST_ADDRESS_SPACE_SIZE,
-           ONE);
+                                 VMEXIT_HOST_ADDRESS_SPACE_SIZE, ONE);
        if (err) {
-               kprintf("VMM: VMEXIT_HOST_ADDRESS_SPACE_SIZE not supported by this CPU\n");
+               kprintf("VMM: VMEXIT_HOST_ADDRESS_SPACE_SIZE not "
+                       "supported by this CPU\n");
                return (ENODEV);
        }
 
@@ -397,7 +397,8 @@ vmx_init(void)
            VMEXIT_SAVE_IA32_EFER,
            ONE);
        if (err) {
-               kprintf("VMM: VMEXIT_SAVE_IA32_EFER not supported by this CPU\n");
+               kprintf("VMM: VMEXIT_SAVE_IA32_EFER not "
+                       "supported by this CPU\n");
                return (ENODEV);
        }
 
@@ -406,7 +407,8 @@ vmx_init(void)
            VMEXIT_LOAD_IA32_EFER,
            ONE);
        if (err) {
-               kprintf("VMM: VMEXIT_LOAD_IA32_EFER not supported by this CPU\n");
+               kprintf("VMM: VMEXIT_LOAD_IA32_EFER not "
+                       "supported by this CPU\n");
                return (ENODEV);
        }
 
@@ -415,7 +417,8 @@ vmx_init(void)
            PROCBASED2_ENABLE_EPT,
            ONE);
        if (err) {
-               kprintf("VMM: PROCBASED2_ENABLE_EPT not supported by this CPU\n");
+               kprintf("VMM: PROCBASED2_ENABLE_EPT not "
+                       "supported by this CPU\n");
                return (ENODEV);
        }
 
@@ -430,7 +433,8 @@ vmx_init(void)
            PROCBASED2_ENABLE_VPID,
            ONE);
        if (err) {
-               kprintf("VMM: PROCBASED2_ENABLE_VPID not supported by this CPU\n");
+               kprintf("VMM: PROCBASED2_ENABLE_VPID not "
+                       "supported by this CPU\n");
                return (ENODEV);
        }
 #endif
@@ -578,8 +582,11 @@ vmx_enable(void)
 
        alloc_vmxon_regions();
        for (cpu = 0; cpu < ncpus; cpu++) {
+               cpumask_t mask;
+
                err = 0;
-               lwkt_cpusync_simple(CPUMASK(cpu), execute_vmxon, &err);
+               CPUMASK_ASSBIT(mask, cpu);
+               lwkt_cpusync_simple(mask, execute_vmxon, &err);
                if(err) {
                        kprintf("VMM: vmx_enable error %d on cpu%d\n", err, cpu);
                        return err;
@@ -598,8 +605,12 @@ vmx_disable(void)
                kprintf("VMM: vmx_disable not allowed; vmx wasn't enabled\n");
        }
 
-       for (cpu = 0; cpu < ncpus; cpu++)
-               lwkt_cpusync_simple(CPUMASK(cpu), execute_vmxoff, NULL);
+       for (cpu = 0; cpu < ncpus; cpu++) {
+               cpumask_t mask;
+
+               CPUMASK_ASSBIT(mask, cpu);
+               lwkt_cpusync_simple(mask, execute_vmxoff, NULL);
+       }
 
        free_vmxon_regions();
 
@@ -954,6 +965,7 @@ vmx_check_cpu_migration(void)
 {
        struct vmx_thread_info * vti;
        struct globaldata *gd;
+       cpumask_t mask;
        int err;
 
        gd = mycpu;
@@ -968,11 +980,12 @@ vmx_check_cpu_migration(void)
                 * with.  The pcpu_info[] check prevents unecessary extra
                 * cpusyncs.
                 */
-               dkprintf("VMM: cpusync from %d to %d\n", gd->gd_cpuid, vti->last_cpu);
+               dkprintf("VMM: cpusync from %d to %d\n",
+                        gd->gd_cpuid, vti->last_cpu);
 
                /* Clear the VMCS area if ran on another CPU */
-               lwkt_cpusync_simple(CPUMASK(vti->last_cpu),
-                                   execute_vmclear, (void *)vti);
+               CPUMASK_ASSBIT(mask, vti->last_cpu);
+               lwkt_cpusync_simple(mask, execute_vmclear, (void *)vti);
        }
        return 0;
 error:
@@ -1371,7 +1384,7 @@ restart:
         * cpu that may desire to IPI us after we have successfully
         * incremented the cpulock counter.
         */
-       atomic_set_cpumask(&td->td_proc->p_vmm_cpumask, gd->gd_cpumask);
+       ATOMIC_CPUMASK_ORBIT(td->td_proc->p_vmm_cpumask, gd->gd_cpuid);
 
         for (;;) {
                olock = td->td_proc->p_vmm_cpulock;
@@ -1392,8 +1405,8 @@ restart:
                 * More complex.  After sleeping we have to re-test
                 * everything.
                 */
-               atomic_clear_cpumask(&td->td_proc->p_vmm_cpumask,
-                                    gd->gd_cpumask);
+               ATOMIC_CPUMASK_NANDBIT(td->td_proc->p_vmm_cpumask,
+                                      gd->gd_cpuid);
                cpu_enable_intr();
                tsleep_interlock(&td->td_proc->p_vmm_cpulock, 0);
                if (td->td_proc->p_vmm_cpulock & CPULOCK_EXCL) {
@@ -1473,8 +1486,8 @@ restart:
        if (ret == VM_EXIT) {
                ERROR_IF(vmx_vmexit_loadinfo());
 
-               atomic_clear_cpumask(&td->td_proc->p_vmm_cpumask,
-                                    gd->gd_cpumask);
+               ATOMIC_CPUMASK_NANDBIT(td->td_proc->p_vmm_cpumask,
+                                      gd->gd_cpuid);
                atomic_add_int(&td->td_proc->p_vmm_cpulock,
                               -CPULOCK_INCR);
                /* WARNING: don't adjust cpulock twice! */
@@ -1524,13 +1537,14 @@ done:
        kprintf("VMM: vmx_vmrun: returning with success\n");
        return 0;
 error:
-       atomic_clear_cpumask(&td->td_proc->p_vmm_cpumask, gd->gd_cpumask);
+       ATOMIC_CPUMASK_NANDBIT(td->td_proc->p_vmm_cpumask, gd->gd_cpuid);
        atomic_add_int(&td->td_proc->p_vmm_cpulock, -CPULOCK_INCR);
        cpu_enable_intr();
 error2:
        trap_handle_userenter(td);
        td->td_lwp->lwp_md.md_regs = save_frame;
-       KKASSERT((td->td_proc->p_vmm_cpumask & gd->gd_cpumask) == 0);
+       KKASSERT(CPUMASK_TESTMASK(td->td_proc->p_vmm_cpumask,
+                                 gd->gd_cpumask) == 0);
        /*atomic_clear_cpumask(&td->td_proc->p_vmm_cpumask, gd->gd_cpumask);*/
        crit_exit();
        kprintf("VMM: vmx_vmrun failed\n");
index 50750f6..0a9bc19 100644 (file)
@@ -158,7 +158,8 @@ kdb_trap(int type, int code, struct x86_64_saved_state *regs)
 
        crit_enter();
        db_printf("\nCPU%d stopping CPUs: 0x%08jx\n",
-           mycpu->gd_cpuid, (uintmax_t)mycpu->gd_other_cpus);
+                 mycpu->gd_cpuid,
+                 (uintmax_t)CPUMASK_LOWMASK(mycpu->gd_other_cpus));
 
        /* We stop all CPUs except ourselves (obviously) */
        stop_cpus(mycpu->gd_other_cpus);
@@ -180,14 +181,15 @@ kdb_trap(int type, int code, struct x86_64_saved_state *regs)
        db_global_jmpbuf_valid = FALSE;
 
        db_printf("\nCPU%d restarting CPUs: 0x%016jx\n",
-           mycpu->gd_cpuid, (uintmax_t)stopped_cpus);
+                 mycpu->gd_cpuid,
+                 (uintmax_t)CPUMASK_LOWMASK(stopped_cpus));
 
        /* Restart all the CPUs we previously stopped */
-       if (stopped_cpus != mycpu->gd_other_cpus) {
+       if (CPUMASK_CMPMASKNEQ(stopped_cpus, mycpu->gd_other_cpus)) {
                db_printf("whoa, other_cpus: 0x%016jx, "
                          "stopped_cpus: 0x%016jx\n",
-                         (uintmax_t)mycpu->gd_other_cpus,
-                         (uintmax_t)stopped_cpus);
+                         (uintmax_t)CPUMASK_LOWMASK(mycpu->gd_other_cpus),
+                         (uintmax_t)CPUMASK_LOWMASK(stopped_cpus));
                panic("stop_cpus() failed");
        }
        restart_cpus(stopped_cpus);
index d73156c..564ccf9 100644 (file)
@@ -168,9 +168,13 @@ static int start_ap(struct mdglobaldata *gd, u_int boot_addr, int smibest);
 static int     smitest(void);
 static void    mp_bsp_simple_setup(void);
 
-static cpumask_t smp_startup_mask = 1; /* which cpus have been started */
-static cpumask_t smp_lapic_mask = 1;   /* which cpus have lapic been inited */
-cpumask_t smp_active_mask = 1; /* which cpus are ready for IPIs etc? */
+/* which cpus have been started */
+static cpumask_t smp_startup_mask = CPUMASK_INITIALIZER_ONLYONE;
+/* which cpus have lapic been inited */
+static cpumask_t smp_lapic_mask = CPUMASK_INITIALIZER_ONLYONE;
+/* which cpus are ready for IPIs etc? */
+cpumask_t smp_active_mask = CPUMASK_INITIALIZER_ONLYONE;
+
 SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RD, &smp_active_mask, 0, "");
 static u_int   bootMP_size;
 
@@ -486,8 +490,10 @@ start_all_aps(u_int boot_addr)
        ncpus_fit_mask = ncpus_fit - 1;
 
        /* build our map of 'other' CPUs */
-       mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
-       mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * ncpus);
+       mycpu->gd_other_cpus = smp_startup_mask;
+       CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid);
+       mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map,
+                                           sizeof(lwkt_ipiq) * ncpus);
        bzero(mycpu->gd_ipiq, sizeof(lwkt_ipiq) * ncpus);
 
        /* restore the warmstart vector */
@@ -511,7 +517,7 @@ start_all_aps(u_int boot_addr)
                tsc0_offset = rdtsc();
        tsc_offsets[0] = 0;
        rel_mplock();
-       while (smp_lapic_mask != smp_startup_mask) {
+       while (CPUMASK_CMPMASKNEQ(smp_lapic_mask, smp_startup_mask)) {
                cpu_lfence();
                if (cpu_feature & CPUID_TSC)
                        tsc0_offset = rdtsc();
@@ -729,7 +735,7 @@ start_ap(struct mdglobaldata *gd, u_int boot_addr, int smibest)
        /* wait for it to start, see ap_init() */
        set_apic_timer(5000000);/* == 5 seconds */
        while (read_apic_timer()) {
-               if (smp_startup_mask & CPUMASK(gd->mi.gd_cpuid))
+               if (CPUMASK_TESTBIT(smp_startup_mask, gd->mi.gd_cpuid))
                        return 1;       /* return SUCCESS */
        }
 
@@ -786,27 +792,37 @@ smp_invltlb(void)
        long count = 0;
        long xcount = 0;
 #endif
+       cpumask_t tmpmask;
+       cpumask_t tmpmask2;
 
        crit_enter_gd(&md->mi);
-       md->gd_invltlb_ret = 0;
+       CPUMASK_ASSZERO(md->gd_invltlb_ret);
        ++md->mi.gd_cnt.v_smpinvltlb;
-       atomic_set_cpumask(&smp_invltlb_req, md->mi.gd_cpumask);
+       ATOMIC_CPUMASK_ORBIT(smp_invltlb_req, md->mi.gd_cpuid);
 #ifdef SMP_INVLTLB_DEBUG
 again:
 #endif
-       if (smp_startup_mask == smp_active_mask) {
+       if (CPUMASK_CMPMASKEQ(smp_startup_mask, smp_active_mask)) {
                all_but_self_ipi(XINVLTLB_OFFSET);
        } else {
-               selected_apic_ipi(smp_active_mask & ~md->mi.gd_cpumask,
-                                 XINVLTLB_OFFSET, APIC_DELMODE_FIXED);
+               tmpmask = smp_active_mask;
+               CPUMASK_NANDMASK(tmpmask, md->mi.gd_cpumask);
+               selected_apic_ipi(tmpmask, XINVLTLB_OFFSET, APIC_DELMODE_FIXED);
        }
 
 #ifdef SMP_INVLTLB_DEBUG
        if (xcount)
                kprintf("smp_invltlb: ipi sent\n");
 #endif
-       while ((md->gd_invltlb_ret & smp_active_mask & ~md->mi.gd_cpumask) !=
-              (smp_active_mask & ~md->mi.gd_cpumask)) {
+       for (;;) {
+               tmpmask = smp_active_mask;
+               tmpmask2 = tmpmask;
+               CPUMASK_ANDMASK(tmpmask, md->gd_invltlb_ret);
+               CPUMASK_NANDMASK(tmpmask, md->mi.gd_cpumask);
+               CPUMASK_NANDMASK(tmpmask2, md->mi.gd_cpumask);
+
+               if (CPUMASK_CMPMASKEQ(tmpmask, tmpmask2))
+                       break;
                cpu_mfence();
                cpu_pause();
 #ifdef SMP_INVLTLB_DEBUG
@@ -815,19 +831,22 @@ again:
                        print_backtrace(-1);
                        kprintf("smp_invltlb: endless loop %08lx %08lx, "
                                "rflags %016jx retry",
-                             (long)md->gd_invltlb_ret,
-                             (long)smp_invltlb_req,
+                             (long)CPUMASK_LOWMASK(md->gd_invltlb_ret),
+                             (long)CPUMASK_LOWMASK(smp_invltlb_req),
                              (intmax_t)read_rflags());
                        __asm __volatile ("sti");
                        ++xcount;
                        if (xcount > 2)
                                lwkt_process_ipiq();
                        if (xcount > 3) {
-                               int bcpu = BSFCPUMASK(~md->gd_invltlb_ret &
-                                                     ~md->mi.gd_cpumask &
-                                                     smp_active_mask);
+                               int bcpu;
                                globaldata_t xgd;
 
+                               tmpmask = smp_active_mask;
+                               CPUMASK_NANDMASK(tmpmask, md->gd_invltlb_ret);
+                               CPUMASK_NANDMASK(tmpmask, md->mi.gd_cpumask);
+                               bcpu = BSFCPUMASK(tmpmask);
+
                                kprintf("bcpu %d\n", bcpu);
                                xgd = globaldata_find(bcpu);
                                kprintf("thread %p %s\n", xgd->gd_curthread, xgd->gd_curthread->td_comm);
@@ -839,7 +858,7 @@ again:
                }
 #endif
        }
-       atomic_clear_cpumask(&smp_invltlb_req, md->mi.gd_cpumask);
+       ATOMIC_CPUMASK_NANDBIT(smp_invltlb_req, md->mi.gd_cpuid);
        crit_exit_gd(&md->mi);
 }
 
@@ -859,11 +878,11 @@ smp_invltlb_intr(void)
        cpu_mfence();
        mask = smp_invltlb_req;
        cpu_invltlb();
-       while (mask) {
+       while (CPUMASK_TESTNZERO(mask)) {
                cpu = BSFCPUMASK(mask);
-               mask &= ~CPUMASK(cpu);
+               CPUMASK_NANDBIT(mask, cpu);
                omd = (struct mdglobaldata *)globaldata_find(cpu);
-               atomic_set_cpumask(&omd->gd_invltlb_ret, md->mi.gd_cpumask);
+               ATOMIC_CPUMASK_ORBIT(omd->gd_invltlb_ret, md->mi.gd_cpuid);
        }
 }
 
@@ -905,13 +924,18 @@ smp_invlpg_range_cpusync(void *arg)
 int
 stop_cpus(cpumask_t map)
 {
-       map &= smp_active_mask;
+       cpumask_t mask;
+
+       CPUMASK_ANDMASK(map, smp_active_mask);
 
        /* send the Xcpustop IPI to all CPUs in map */
        selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
-       
-       while ((stopped_cpus & map) != map)
-               /* spin */ ;
+
+       do {
+               mask = stopped_cpus;
+               CPUMASK_ANDMASK(mask, map);
+               /* spin */
+       } while (CPUMASK_CMPMASKNEQ(mask, map));
 
        return 1;
 }
@@ -933,11 +957,18 @@ stop_cpus(cpumask_t map)
 int
 restart_cpus(cpumask_t map)
 {
-       /* signal other cpus to restart */
-       started_cpus = map & smp_active_mask;
+       cpumask_t mask;
 
-       while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
-               /* spin */ ;
+       /* signal other cpus to restart */
+       mask = map;
+       CPUMASK_ANDMASK(mask, smp_active_mask);
+       cpu_ccfence();
+       started_cpus = mask;
+       cpu_ccfence();
+
+       /* wait for each to clear its bit */
+       while (CPUMASK_CMPMASKNEQ(stopped_cpus, map))
+               cpu_pause();
 
        return 1;
 }
@@ -963,7 +994,7 @@ ap_init(void)
         * interrupts physically disabled and remote cpus could deadlock
         * trying to send us an IPI.
         */
-       smp_startup_mask |= CPUMASK(mycpu->gd_cpuid);
+       ATOMIC_CPUMASK_ORBIT(smp_startup_mask, mycpu->gd_cpuid);
        cpu_mfence();
 
        /*
@@ -995,7 +1026,8 @@ ap_init(void)
        cpu_invltlb();
 
        /* Build our map of 'other' CPUs. */
-       mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
+       mycpu->gd_other_cpus = smp_startup_mask;
+       CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid);
 
        /* A quick check from sanity claus */
        cpu_id = APICID_TO_CPUID((lapic->id & 0xff000000) >> 24);
@@ -1013,7 +1045,7 @@ ap_init(void)
        lapic_init(FALSE);
 
        /* LAPIC initialization is done */
-       smp_lapic_mask |= CPUMASK(mycpu->gd_cpuid);
+       CPUMASK_ORBIT(smp_lapic_mask, mycpu->gd_cpuid);
        cpu_mfence();
 
        /* Let BSP move onto the next initialization stage */
@@ -1057,7 +1089,7 @@ ap_init(void)
         * nothing we've done put it there.
         */
        KKASSERT(get_mplock_count(curthread) == 1);
-       smp_active_mask |= CPUMASK(mycpu->gd_cpuid);
+       CPUMASK_ORBIT(smp_active_mask, mycpu->gd_cpuid);
 
        /*
         * Enable interrupts here.  idle_restore will also do it, but
@@ -1089,13 +1121,15 @@ ap_finish(void)
        if (bootverbose)
                kprintf("Finish MP startup\n");
        rel_mplock();
-       while (smp_active_mask != smp_startup_mask)
+       while (CPUMASK_CMPMASKNEQ(smp_active_mask, smp_startup_mask)) {
                cpu_lfence();
+               cpu_pause();
+       }
        while (try_mplock() == 0)
                ;
        if (bootverbose) {
                kprintf("Active CPU Mask: %016jx\n",
-                       (uintmax_t)smp_active_mask);
+                       (uintmax_t)CPUMASK_LOWMASK(smp_active_mask));
        }
 }
 
@@ -1104,7 +1138,7 @@ SYSINIT(finishsmp, SI_BOOT2_FINISH_SMP, SI_ORDER_FIRST, ap_finish, NULL)
 void
 cpu_send_ipiq(int dcpu)
 {
-        if (CPUMASK(dcpu) & smp_active_mask)
+       if (CPUMASK_TESTBIT(smp_active_mask, dcpu))
                 single_apic_ipi(dcpu, XIPIQ_OFFSET, APIC_DELMODE_FIXED);
 }
 
@@ -1116,7 +1150,7 @@ int
 cpu_send_ipiq_passive(int dcpu)
 {
         int r = 0;
-        if (CPUMASK(dcpu) & smp_active_mask) {
+       if (CPUMASK_TESTBIT(smp_active_mask, dcpu)) {
                 r = single_apic_ipi_passive(dcpu, XIPIQ_OFFSET,
                                         APIC_DELMODE_FIXED);
         }
@@ -1128,8 +1162,10 @@ static void
 mp_bsp_simple_setup(void)
 {
        /* build our map of 'other' CPUs */
-       mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
-       mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map, sizeof(lwkt_ipiq) * ncpus);
+       mycpu->gd_other_cpus = smp_startup_mask;
+       CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid);
+       mycpu->gd_ipiq = (void *)kmem_alloc(&kernel_map,
+                                           sizeof(lwkt_ipiq) * ncpus);
        bzero(mycpu->gd_ipiq, sizeof(lwkt_ipiq) * ncpus);
 
        pmap_set_opt();
@@ -1266,17 +1302,22 @@ amd_get_compute_unit_id(void *arg)
 int
 fix_amd_topology(void)
 {
+       cpumask_t mask;
+
        if (cpu_vendor_id != CPU_VENDOR_AMD)
                return -1;
        if ((amd_feature2 & AMDID2_TOPOEXT) == 0)
                return -1;
 
-       lwkt_cpusync_simple(-1, amd_get_compute_unit_id, NULL);
+       CPUMASK_ASSALLONES(mask);
+       lwkt_cpusync_simple(mask, amd_get_compute_unit_id, NULL);
 
        kprintf("Compute unit iDS:\n");
        int i;
-       for (i = 0; i < ncpus; i++)
-               kprintf("%d-%d; \n", i, get_cpu_node_by_cpuid(i)->compute_unit_id);
+       for (i = 0; i < ncpus; i++) {
+               kprintf("%d-%d; \n",
+                       i, get_cpu_node_by_cpuid(i)->compute_unit_id);
+       }
 
        return 0;
 }
index 94ada93..108c745 100644 (file)
@@ -450,7 +450,7 @@ mptable_hyperthread_fixup(cpumask_t id_mask, int cpu_count)
                 */
                dist = cur = prev = -1;
                for (id = 0; id < MAXCPU; ++id) {
-                       if ((id_mask & CPUMASK(id)) == 0)
+                       if (CPUMASK_TESTBIT(id_mask, id) == 0)
                                continue;
 
                        cur = id;
@@ -491,13 +491,13 @@ mptable_hyperthread_fixup(cpumask_t id_mask, int cpu_count)
         * already in the table, then kill the fixup.
         */
        for (id = 0; id < MAXCPU; id++) {
-               if ((id_mask & CPUMASK(id)) == 0)
+               if (CPUMASK_TESTBIT(id_mask, id) == 0)
                        continue;
                /* First, make sure we are on a logical_cpus boundary. */
                if (id % logical_cpus != 0)
                        return 0;
                for (i = id + 1; i < id + logical_cpus; i++)
-                       if ((id_mask & CPUMASK(i)) != 0)
+                       if (CPUMASK_TESTBIT(id_mask, i) != 0)
                                return 0;
        }
        return logical_cpus;
@@ -780,9 +780,15 @@ mptable_lapic_enumerate(struct lapic_enumerator *e)
        KKASSERT(arg1.cpu_count != 0);
  
        /* See if we need to fixup HT logical CPUs. */
+       /*
+        * XXX fixup for cpus >= 32 ? XXX
+        */
        if (arg1.ht_fixup) {
-               logical_cpus = mptable_hyperthread_fixup(arg1.ht_apicid_mask,
-                                                        arg1.cpu_count);
+               cpumask_t mask;
+
+               CPUMASK_ASSZERO(mask);
+               mask.m0 = arg1.ht_apicid_mask;
+               logical_cpus = mptable_hyperthread_fixup(mask, arg1.cpu_count);
                if (logical_cpus != 0)
                        arg1.cpu_count *= logical_cpus;
        }
index b9654e5..2dc7aac 100644 (file)
@@ -907,7 +907,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
         */
        kernel_pmap.pm_pml4 = (pdp_entry_t *) (PTOV_OFFSET + KPML4phys);
        kernel_pmap.pm_count = 1;
-       kernel_pmap.pm_active = (cpumask_t)-1;
+       CPUMASK_ASSALLONES(kernel_pmap.pm_active);
        RB_INIT(&kernel_pmap.pm_pvroot);
        spin_init(&kernel_pmap.pm_spin);
        lwkt_token_init(&kernel_pmap.pm_token, "kpmap_tok");
@@ -1617,7 +1617,7 @@ pmap_pinit0(struct pmap *pmap)
 {
        pmap->pm_pml4 = (pml4_entry_t *)(PTOV_OFFSET + KPML4phys);
        pmap->pm_count = 1;
-       pmap->pm_active = 0;
+       CPUMASK_ASSZERO(pmap->pm_active);
        pmap->pm_pvhint = NULL;
        RB_INIT(&pmap->pm_pvroot);
        spin_init(&pmap->pm_spin);
@@ -1637,7 +1637,7 @@ pmap_pinit_simple(struct pmap *pmap)
         * Misc initialization
         */
        pmap->pm_count = 1;
-       pmap->pm_active = 0;
+       CPUMASK_ASSZERO(pmap->pm_active);
        pmap->pm_pvhint = NULL;
        pmap->pm_flags = PMAP_FLAG_SIMPLE;
 
@@ -1737,7 +1737,7 @@ pmap_puninit(pmap_t pmap)
        pv_entry_t pv;
        vm_page_t p;
 
-       KKASSERT(pmap->pm_active == 0);
+       KKASSERT(CPUMASK_TESTZERO(pmap->pm_active));
        if ((pv = pmap->pm_pmlpv) != NULL) {
                if (pv_hold_try(pv) == 0)
                        pv_lock(pv);
@@ -2275,8 +2275,9 @@ pmap_release(struct pmap *pmap)
 {
        struct pmap_release_info info;
 
-       KASSERT(pmap->pm_active == 0,
-               ("pmap still active! %016jx", (uintmax_t)pmap->pm_active));
+       KASSERT(CPUMASK_TESTZERO(pmap->pm_active),
+               ("pmap still active! %016jx",
+               (uintmax_t)CPUMASK_LOWMASK(pmap->pm_active)));
 
        spin_lock(&pmap_spin);
        TAILQ_REMOVE(&pmap_list, pmap, pm_pmnode);
@@ -5219,7 +5220,7 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm)
                lp->lwp_vmspace = newvm;
                if (curthread->td_lwp == lp) {
                        pmap = vmspace_pmap(newvm);
-                       atomic_set_cpumask(&pmap->pm_active, mycpu->gd_cpumask);
+                       ATOMIC_CPUMASK_ORBIT(pmap->pm_active, mycpu->gd_cpuid);
                        if (pmap->pm_active_lock & CPULOCK_EXCL)
                                pmap_interlock_wait(newvm);
 #if defined(SWTCH_OPTIM_STATS)
@@ -5234,7 +5235,8 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm)
                        }
                        load_cr3(curthread->td_pcb->pcb_cr3);
                        pmap = vmspace_pmap(oldvm);
-                       atomic_clear_cpumask(&pmap->pm_active, mycpu->gd_cpumask);
+                       ATOMIC_CPUMASK_NANDBIT(pmap->pm_active,
+                                              mycpu->gd_cpuid);
                }
                crit_exit();
        }
@@ -5318,7 +5320,7 @@ pmap_object_free(vm_object_t object)
                object->md.pmap_rw = NULL;
                pmap_remove_noinval(pmap,
                                  VM_MIN_USER_ADDRESS, VM_MAX_USER_ADDRESS);
-               pmap->pm_active = 0;
+               CPUMASK_ASSZERO(pmap->pm_active);
                pmap_release(pmap);
                pmap_puninit(pmap);
                kfree(pmap, M_OBJPMAP);
@@ -5327,7 +5329,7 @@ pmap_object_free(vm_object_t object)
                object->md.pmap_ro = NULL;
                pmap_remove_noinval(pmap,
                                  VM_MIN_USER_ADDRESS, VM_MAX_USER_ADDRESS);
-               pmap->pm_active = 0;
+               CPUMASK_ASSZERO(pmap->pm_active);
                pmap_release(pmap);
                pmap_puninit(pmap);
                kfree(pmap, M_OBJPMAP);
index a78c4be..54994c0 100644 (file)
@@ -113,7 +113,7 @@ stopsig(int nada, siginfo_t *info, void *ctxp)
 
        ++curthread->td_critcount;
        ++mycpu->gd_intr_nesting_level;
-       while (stopped_cpus & mycpu->gd_cpumask) {
+       while (CPUMASK_TESTMASK(stopped_cpus, mycpu->gd_cpumask)) {
                sigsuspend(&ss);
        }
        --mycpu->gd_intr_nesting_level;
index 306b0c1..5959520 100644 (file)
@@ -100,24 +100,21 @@ void
 ap_finish(void)
 {
        int i;
-       cpumask_t ncpus_mask = 0;
-
-       for (i = 1; i <= ncpus; i++)
-               ncpus_mask |= CPUMASK(i);
 
         mp_finish = 1;
         if (bootverbose)
                 kprintf("Finish MP startup\n");
 
        /* build our map of 'other' CPUs */
-       mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
+       mycpu->gd_other_cpus = smp_startup_mask;
+       CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid);
 
        /*
         * Let the other cpu's finish initializing and build their map
         * of 'other' CPUs.
         */
         rel_mplock();
-        while (smp_active_mask != smp_startup_mask) {
+       while (CPUMASK_CMPMASKNEQ(smp_active_mask, smp_startup_mask)) {
                DELAY(100000);
                 cpu_lfence();
        }
@@ -196,9 +193,10 @@ mp_announce(void)
 void
 cpu_send_ipiq(int dcpu)
 {
-       if (CPUMASK(dcpu) & smp_active_mask)
+       if (CPUMASK_TESTBIT(smp_active_mask, dcpu)) {
                if (pthread_kill(ap_tids[dcpu], SIGUSR1) != 0)
                        panic("pthread_kill failed in cpu_send_ipiq");
+       }
 #if 0
        panic("XXX cpu_send_ipiq()");
 #endif
@@ -219,9 +217,9 @@ void
 selected_cpu_ipi(cpumask_t target, int vector, int delivery_mode)
 {
        crit_enter();
-       while (target) {
+       while (CPUMASK_TESTNZERO(target)) {
                int n = BSFCPUMASK(target);
-               target &= ~CPUMASK(n);
+               CPUMASK_NANDBIT(target, n);
                single_cpu_ipi(n, vector, delivery_mode);
        }
        crit_exit();
@@ -230,13 +228,13 @@ selected_cpu_ipi(cpumask_t target, int vector, int delivery_mode)
 int
 stop_cpus(cpumask_t map)
 {
-       map &= smp_active_mask;
+       CPUMASK_ANDMASK(map, smp_active_mask);
 
        crit_enter();
-       while (map) {
+       while (CPUMASK_TESTNZERO(map)) {
                int n = BSFCPUMASK(map);
-               map &= ~CPUMASK(n);
-               stopped_cpus |= CPUMASK(n);
+               CPUMASK_NANDBIT(map, n);
+               ATOMIC_CPUMASK_ORBIT(stopped_cpus, n);
                if (pthread_kill(ap_tids[n], SIGXCPU) != 0)
                        panic("stop_cpus: pthread_kill failed");
        }
@@ -251,13 +249,13 @@ stop_cpus(cpumask_t map)
 int
 restart_cpus(cpumask_t map)
 {
-       map &= smp_active_mask;
+       CPUMASK_ANDMASK(map, smp_active_mask);
 
        crit_enter();
-       while (map) {
+       while (CPUMASK_TESTNZERO(map)) {
                int n = BSFCPUMASK(map);
-               map &= ~CPUMASK(n);
-               stopped_cpus &= ~CPUMASK(n);
+               CPUMASK_NANDBIT(map, n);
+               ATOMIC_CPUMASK_NANDBIT(stopped_cpus, n);
                if (pthread_kill(ap_tids[n], SIGXCPU) != 0)
                        panic("restart_cpus: pthread_kill failed");
        }
@@ -281,7 +279,7 @@ ap_init(void)
          * interrupts physically disabled and remote cpus could deadlock
          * trying to send us an IPI.
          */
-       smp_startup_mask |= CPUMASK(mycpu->gd_cpuid);
+       ATOMIC_CPUMASK_ORBIT(smp_startup_mask, mycpu->gd_cpuid);
        cpu_mfence();
 
         /*
@@ -308,7 +306,8 @@ ap_init(void)
         cpu_invltlb();
 
         /* Build our map of 'other' CPUs. */
-        mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
+        mycpu->gd_other_cpus = smp_startup_mask;
+       CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid);
 
         kprintf("SMP: AP CPU #%d Launched!\n", mycpu->gd_cpuid);
 
@@ -330,7 +329,7 @@ ap_init(void)
          * nothing we've done put it there.
          */
         KKASSERT(get_mplock_count(curthread) == 1);
-        smp_active_mask |= CPUMASK(mycpu->gd_cpuid);
+       ATOMIC_CPUMASK_ORBIT(smp_active_mask, mycpu->gd_cpuid);
 
        mdcpu->gd_fpending = 0;
        mdcpu->gd_ipending = 0;
@@ -448,7 +447,7 @@ start_all_aps(u_int boot_addr)
                pthread_create(&ap_tids[x], NULL, start_ap, NULL);
                cpu_enable_intr();
 
-               while((smp_startup_mask & CPUMASK(x)) == 0) {
+               while (CPUMASK_TESTBIT(smp_startup_mask, x) == 0) {
                        cpu_lfence(); /* XXX spin until the AP has started */
                        DELAY(1000);
                }
index 19be638..956fc47 100644 (file)
@@ -491,19 +491,21 @@ get_ptbase(struct pmap *pmap, vm_offset_t va)
                KKASSERT(va >= KvaStart && va < KvaEnd);
                return(KernelPTA + (va >> PAGE_SHIFT));
        } else if (pmap->pm_pdir == gd->gd_PT1pdir) {
-               if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) {
+               if (CPUMASK_TESTMASK(pmap->pm_cpucachemask,
+                                    gd->mi.gd_cpumask) == 0) {
                        *gd->gd_PT1pde = pmap->pm_pdirpte;
                        madvise(gd->gd_PT1map, SEG_SIZE, MADV_INVAL);
-                       atomic_set_cpumask(&pmap->pm_cpucachemask,
-                                          gd->mi.gd_cpumask);
+                       ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask,
+                                            gd->mi.gd_cpumask);
                }
                return(gd->gd_PT1map + (va >> PAGE_SHIFT));
        } else if (pmap->pm_pdir == gd->gd_PT2pdir) {
-               if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) {
+               if (CPUMASK_TESTMASK(pmap->pm_cpucachemask,
+                                    gd->mi.gd_cpumask) == 0) {
                        *gd->gd_PT2pde = pmap->pm_pdirpte;
                        madvise(gd->gd_PT2map, SEG_SIZE, MADV_INVAL);
-                       atomic_set_cpumask(&pmap->pm_cpucachemask,
-                                          gd->mi.gd_cpumask);
+                       ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask,
+                                            gd->mi.gd_cpumask);
                }
                return(gd->gd_PT2map + (va >> PAGE_SHIFT));
        }
@@ -522,15 +524,15 @@ get_ptbase(struct pmap *pmap, vm_offset_t va)
                        gd->gd_PT1pdir = pmap->pm_pdir;
                        *gd->gd_PT1pde = pmap->pm_pdirpte;
                        madvise(gd->gd_PT1map, SEG_SIZE, MADV_INVAL);
-                       atomic_set_cpumask(&pmap->pm_cpucachemask,
-                                          gd->mi.gd_cpumask);
+                       ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask,
+                                            gd->mi.gd_cpumask);
                        return(gd->gd_PT1map + (va >> PAGE_SHIFT));
                } else {
                        gd->gd_PT2pdir = pmap->pm_pdir;
                        *gd->gd_PT2pde = pmap->pm_pdirpte;
                        madvise(gd->gd_PT2map, SEG_SIZE, MADV_INVAL);
-                       atomic_set_cpumask(&pmap->pm_cpucachemask,
-                                          gd->mi.gd_cpumask);
+                       ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask,
+                                             gd->mi.gd_cpumask);
                        return(gd->gd_PT2map + (va >> PAGE_SHIFT));
                }
        }
@@ -541,18 +543,19 @@ get_ptbase(struct pmap *pmap, vm_offset_t va)
         */
        KKASSERT(IN_CRITICAL_SECT(curthread));
        if (pmap->pm_pdir == gd->gd_PT3pdir) {
-               if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) {
+               if (CPUMASK_TESTMASK(pmap->pm_cpucachemask,
+                                    gd->mi.gd_cpumask) == 0) {
                        *gd->gd_PT3pde = pmap->pm_pdirpte;
                        madvise(gd->gd_PT3map, SEG_SIZE, MADV_INVAL);
-                       atomic_set_cpumask(&pmap->pm_cpucachemask,
-                                          gd->mi.gd_cpumask);
+                       ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask,
+                                             gd->mi.gd_cpumask);
                }
        } else {
                gd->gd_PT3pdir = pmap->pm_pdir;
                *gd->gd_PT3pde = pmap->pm_pdirpte;
                madvise(gd->gd_PT3map, SEG_SIZE, MADV_INVAL);
-               atomic_set_cpumask(&pmap->pm_cpucachemask,
-                                  gd->mi.gd_cpumask);
+               ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask,
+                                     gd->mi.gd_cpumask);
        }
        return(gd->gd_PT3map + (va >> PAGE_SHIFT));
 }
@@ -566,11 +569,12 @@ get_ptbase1(struct pmap *pmap, vm_offset_t va)
                KKASSERT(va >= KvaStart && va < KvaEnd);
                return(KernelPTA + (va >> PAGE_SHIFT));
        } else if (pmap->pm_pdir == gd->gd_PT1pdir) {
-               if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) {
+               if (CPUMASK_TESTMASK(pmap->pm_cpucachemask,
+                                    gd->mi.gd_cpumask) == 0) {
                        *gd->gd_PT1pde = pmap->pm_pdirpte;
                        madvise(gd->gd_PT1map, SEG_SIZE, MADV_INVAL);
-                       atomic_set_cpumask(&pmap->pm_cpucachemask,
-                                          gd->mi.gd_cpumask);
+                       ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask,
+                                             gd->mi.gd_cpumask);
                }
                return(gd->gd_PT1map + (va >> PAGE_SHIFT));
        }
@@ -591,11 +595,12 @@ get_ptbase2(struct pmap *pmap, vm_offset_t va)
                KKASSERT(va >= KvaStart && va < KvaEnd);
                return(KernelPTA + (va >> PAGE_SHIFT));
        } else if (pmap->pm_pdir == gd->gd_PT2pdir) {
-               if ((pmap->pm_cpucachemask & gd->mi.gd_cpumask) == 0) {
+               if (CPUMASK_TESTMASK(pmap->pm_cpucachemask,
+                                    gd->mi.gd_cpumask) == 0) {
                        *gd->gd_PT2pde = pmap->pm_pdirpte;
                        madvise(gd->gd_PT2map, SEG_SIZE, MADV_INVAL);
-                       atomic_set_cpumask(&pmap->pm_cpucachemask,
-                                          gd->mi.gd_cpumask);
+                       ATOMIC_CPUMASK_ORMASK(pmap->pm_cpucachemask,
+                                             gd->mi.gd_cpumask);
                }
                return(gd->gd_PT2map + (va >> PAGE_SHIFT));
        }
@@ -3072,12 +3077,14 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm)
                lp->lwp_vmspace = newvm;
                if (curthread->td_lwp == lp) {
                        pmap = vmspace_pmap(newvm);
-                       atomic_set_cpumask(&pmap->pm_active, mycpu->gd_cpumask);
+                       ATOMIC_CPUMASK_ORMASK(pmap->pm_active,
+                                             gd->mi.gd_cpumask);
 #if defined(SWTCH_OPTIM_STATS)
                        tlb_flush_count++;
 #endif
                        pmap = vmspace_pmap(oldvm);
-                       atomic_clear_cpumask(&pmap->pm_active, mycpu->gd_cpumask);
+                       ATOMIC_CPUMASK_NANDMASK(pmap->pm_active,
+                                               gd->mi.gd_cpumask);
                }
        }
        crit_exit();
index a31c8ac..190c267 100644 (file)
@@ -564,7 +564,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr, int64_t ptov_offset)
        kernel_pmap.pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys);
        kernel_pmap.pm_count = 1;
        /* don't allow deactivation */
-       kernel_pmap.pm_active = (cpumask_t)-1;
+       CPUMASK_ASSALLONES(kernel_pmap.pm_active);
        kernel_pmap.pm_pteobj = NULL;   /* see pmap_init */
        TAILQ_INIT(&kernel_pmap.pm_pvlist);
        TAILQ_INIT(&kernel_pmap.pm_pvlist_free);
@@ -3362,12 +3362,12 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm)
         */
        crit_enter();
        pmap = vmspace_pmap(newvm);
-       atomic_set_cpumask(&pmap->pm_active, CPUMASK(mycpu->gd_cpuid));
+       ATOMIC_CPUMASK_ORBIT(pmap->pm_active, mycpu->gd_cpuid);
 #if defined(SWTCH_OPTIM_STATS)
        tlb_flush_count++;
 #endif
        pmap = vmspace_pmap(oldvm);
-       atomic_clear_cpumask(&pmap->pm_active, CPUMASK(mycpu->gd_cpuid));
+       ATOMIC_CPUMASK_NANDBIT(pmap->pm_active, mycpu->gd_cpuid);
        crit_exit();
 }
 
index 81cf6e2..e658c92 100644 (file)
@@ -220,7 +220,8 @@ pmap_inval_pde(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va)
        if (vmm_enabled == 0) {
                *ptep = 0;
                pmap_inval_cpu(pmap, va, SEG_SIZE);
-       } else if ((pmap->pm_active & mycpu->gd_other_cpus) == 0) {
+       } else if (CPUMASK_TESTMASK(pmap->pm_active,
+                                   mycpu->gd_other_cpus) == 0) {
                *ptep = 0;
                vmm_cpu_invltlb();
        } else {
index 001f0b6..ecb0e88 100644 (file)
@@ -118,7 +118,7 @@ stopsig(int nada, siginfo_t *info, void *ctxp)
 
        ++td->td_critcount;
        ++gd->gd_intr_nesting_level;
-       while (stopped_cpus & gd->gd_cpumask) {
+       while (CPUMASK_TESTMASK(stopped_cpus, gd->gd_cpumask)) {
                sigsuspend(&ss);
        }
        --gd->gd_intr_nesting_level;
index 5ef2b04..2b615e0 100644 (file)
@@ -104,17 +104,14 @@ void
 ap_finish(void)
 {
        int i;
-       cpumask_t ncpus_mask = 0;
-
-       for (i = 1; i <= ncpus; i++)
-               ncpus_mask |= CPUMASK(i);
 
         mp_finish = 1;
         if (bootverbose)
                 kprintf("Finish MP startup\n");
 
        /* build our map of 'other' CPUs */
-       mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
+       mycpu->gd_other_cpus = smp_startup_mask;
+       CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid);
 
        /*
         * Let the other cpu's finish initializing and build their map
@@ -198,7 +195,7 @@ mp_announce(void)
 void
 cpu_send_ipiq(int dcpu)
 {
-       if (CPUMASK(dcpu) & smp_active_mask) {
+       if (CPUMASK_TESTBIT(smp_active_mask, dcpu)) {
                if (pthread_kill(ap_tids[dcpu], SIGUSR1) != 0)
                        panic("pthread_kill failed in cpu_send_ipiq");
        }
@@ -217,9 +214,9 @@ void
 selected_cpu_ipi(cpumask_t target, int vector, int delivery_mode)
 {
        crit_enter();
-       while (target) {
+       while (CPUMASK_TESTNZERO(target)) {
                int n = BSFCPUMASK(target);
-               target &= ~CPUMASK(n);
+               CPUMASK_NANDBIT(target, n);
                single_cpu_ipi(n, vector, delivery_mode);
        }
        crit_exit();
@@ -228,13 +225,13 @@ selected_cpu_ipi(cpumask_t target, int vector, int delivery_mode)
 int
 stop_cpus(cpumask_t map)
 {
-       map &= smp_active_mask;
+       CPUMASK_ANDMASK(map, smp_active_mask);
 
        crit_enter();
-       while (map) {
+       while (CPUMASK_TESTNZERO(map)) {
                int n = BSFCPUMASK(map);
-               map &= ~CPUMASK(n);
-               stopped_cpus |= CPUMASK(n);
+               CPUMASK_NANDBIT(map, n);
+               ATOMIC_CPUMASK_ORBIT(stopped_cpus, n);
                if (pthread_kill(ap_tids[n], SIGXCPU) != 0)
                        panic("stop_cpus: pthread_kill failed");
        }
@@ -249,13 +246,13 @@ stop_cpus(cpumask_t map)
 int
 restart_cpus(cpumask_t map)
 {
-       map &= smp_active_mask;
+       CPUMASK_ANDMASK(map, smp_active_mask);
 
        crit_enter();
-       while (map) {
+       while (CPUMASK_TESTNZERO(map)) {
                int n = BSFCPUMASK(map);
-               map &= ~CPUMASK(n);
-               stopped_cpus &= ~CPUMASK(n);
+               CPUMASK_NANDBIT(map, n);
+               ATOMIC_CPUMASK_NANDBIT(stopped_cpus, n);
                if (pthread_kill(ap_tids[n], SIGXCPU) != 0)
                        panic("restart_cpus: pthread_kill failed");
        }
@@ -278,7 +275,7 @@ ap_init(void)
          * interrupts physically disabled and remote cpus could deadlock
          * trying to send us an IPI.
          */
-       smp_startup_mask |= CPUMASK(mycpu->gd_cpuid);
+       ATOMIC_CPUMASK_ORBIT(smp_startup_mask, mycpu->gd_cpuid);
        cpu_mfence();
 
         /*
@@ -305,7 +302,8 @@ ap_init(void)
         cpu_invltlb();
 
         /* Build our map of 'other' CPUs. */
-        mycpu->gd_other_cpus = smp_startup_mask & ~CPUMASK(mycpu->gd_cpuid);
+        mycpu->gd_other_cpus = smp_startup_mask;
+       CPUMASK_NANDBIT(mycpu->gd_other_cpus, mycpu->gd_cpuid);
 
         kprintf("SMP: AP CPU #%d Launched!\n", mycpu->gd_cpuid);
 
@@ -327,7 +325,7 @@ ap_init(void)
          * nothing we've done put it there.
          */
        KKASSERT(get_mplock_count(curthread) == 1);
-        smp_active_mask |= CPUMASK(mycpu->gd_cpuid);
+       ATOMIC_CPUMASK_ORBIT(smp_active_mask, mycpu->gd_cpuid);
 
        mdcpu->gd_fpending = 0;
        mdcpu->gd_ipending = 0;
@@ -460,7 +458,7 @@ start_all_aps(u_int boot_addr)
                pthread_create(&ap_tids[x], &attr, start_ap, NULL);
                cpu_enable_intr();
 
-               while((smp_startup_mask & CPUMASK(x)) == 0) {
+               while (CPUMASK_TESTBIT(smp_startup_mask, x) == 0) {
                        cpu_lfence(); /* XXX spin until the AP has started */
                        DELAY(1000);
                }
index 213e30d..a079eb3 100644 (file)
@@ -28,11 +28,9 @@ cpu_node_t *get_cpu_node_by_cpuid(int cpuid);
 #define        CORE_LEVEL      3
 #define        THREAD_LEVEL    4
 
-#define        CPU_ISSET(n, p) ((CPUMASK(n) & p) != 0)
-
 #define CPUSET_FOREACH(cpu, mask)                      \
        for ((cpu) = 0; (cpu) < ncpus; (cpu)++)         \
-               if (CPU_ISSET(cpu, mask))
+               if (CPUMASK_TESTBIT(mask, cpu))
 
 
 #endif /* _KERNEL */
index 3ac989b..8c3d8b0 100644 (file)
@@ -128,7 +128,7 @@ struct globaldata {
        lwkt_queue      gd_tdallq;              /* all threads */
        lwkt_queue      gd_tdrunq;              /* runnable threads */
        __uint32_t      gd_cpuid;
-       cpumask_t       gd_cpumask;             /* mask = CPUMASK(cpuid) */
+       cpumask_t       gd_cpumask;             /* CPUMASK_ASSBIT(cpuid) */
        cpumask_t       gd_other_cpus;          /* mask of 'other' cpus */
        struct timeval  gd_stattv;
        int             gd_intr_nesting_level;  /* hard code, intrs, ipis */
index 32d9b79..e24ebac 100644 (file)
 #include <machine/limits.h>
 #endif
 
+/*
+ * WARNING! Max supported cpu's due to PWAKEUP_CPUMASK is 16384.
+ */
 #define PCATCH         0x00000100      /* tsleep checks signals */
 #define PUSRFLAG1      0x00000200      /* Subsystem specific flag */
 #define PINTERLOCKED   0x00000400      /* Interlocked tsleep */
index 0a48c3f..43c7948 100644 (file)
@@ -37,6 +37,7 @@
  * via hw.acpi.cpu.px_dom*.
  */
 
+#define _KERNEL_STRUCTURES
 #include <sys/types.h>
 #include <sys/sysctl.h>
 #include <sys/kinfo.h>
@@ -297,7 +298,7 @@ acpi_setcpufreq(int nstate)
        /*
         * Set the mask of cpus the userland scheduler is allowed to use.
         */
-       global_cpumask = (1L << nstate) - 1;
+       CPUMASK_ASSBMASK(global_cpumask, nstate);
        sysctlbyname("kern.usched_global_cpumask", NULL, 0,
                     &global_cpumask, sizeof(global_cpumask));