sys/kern/usched_dfly.c

   1 /*
   2  * Copyright (c) 2012 The DragonFly Project.  All rights reserved.
   3  * Copyright (c) 1999 Peter Wemm <peter@FreeBSD.org>.  All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Matthew Dillon <dillon@backplane.com>,
   7  * by Mihai Carabas <mihai.carabas@gmail.com>
   8  * and many others.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  *
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in
  18  *    the documentation and/or other materials provided with the
  19  *    distribution.
  20  * 3. Neither the name of The DragonFly Project nor the names of its
  21  *    contributors may be used to endorse or promote products derived
  22  *    from this software without specific, prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  27  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  28  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  29  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  30  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  32  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  33  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  34  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35  * SUCH DAMAGE.
  36  */
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/kernel.h>
  40 #include <sys/lock.h>
  41 #include <sys/queue.h>
  42 #include <sys/proc.h>
  43 #include <sys/rtprio.h>
  44 #include <sys/uio.h>
  45 #include <sys/sysctl.h>
  46 #include <sys/resourcevar.h>
  47 #include <sys/spinlock.h>
  48 #include <sys/cpu_topology.h>
  49 #include <sys/thread2.h>
  50 #include <sys/spinlock2.h>
  51 #include <sys/mplock2.h>
  52
  53 #include <sys/ktr.h>
  54
  55 #include <machine/cpu.h>
  56 #include <machine/smp.h>
  57
  58 /*
  59  * Priorities.  Note that with 32 run queues per scheduler each queue
  60  * represents four priority levels.
  61  */
  62
  63 int dfly_rebalanced;
  64
  65 #define MAXPRI                  128
  66 #define PRIMASK                 (MAXPRI - 1)
  67 #define PRIBASE_REALTIME        0
  68 #define PRIBASE_NORMAL          MAXPRI
  69 #define PRIBASE_IDLE            (MAXPRI * 2)
  70 #define PRIBASE_THREAD          (MAXPRI * 3)
  71 #define PRIBASE_NULL            (MAXPRI * 4)
  72
  73 #define NQS     32                      /* 32 run queues. */
  74 #define PPQ     (MAXPRI / NQS)          /* priorities per queue */
  75 #define PPQMASK (PPQ - 1)
  76
  77 /*
  78  * NICEPPQ      - number of nice units per priority queue
  79  * ESTCPUPPQ    - number of estcpu units per priority queue
  80  * ESTCPUMAX    - number of estcpu units
  81  */
  82 #define NICEPPQ         2
  83 #define ESTCPUPPQ       512
  84 #define ESTCPUMAX       (ESTCPUPPQ * NQS)
  85 #define BATCHMAX        (ESTCPUFREQ * 30)
  86 #define PRIO_RANGE      (PRIO_MAX - PRIO_MIN + 1)
  87
  88 #define ESTCPULIM(v)    min((v), ESTCPUMAX)
  89
  90 TAILQ_HEAD(rq, lwp);
  91
  92 #define lwp_priority    lwp_usdata.dfly.priority
  93 #define lwp_rqindex     lwp_usdata.dfly.rqindex
  94 #define lwp_estcpu      lwp_usdata.dfly.estcpu
  95 #define lwp_batch       lwp_usdata.dfly.batch
  96 #define lwp_rqtype      lwp_usdata.dfly.rqtype
  97 #define lwp_qcpu        lwp_usdata.dfly.qcpu
  98
  99 struct usched_dfly_pcpu {
 100         struct spinlock spin;
 101         struct thread   helper_thread;
 102         short           rrcount;
 103         short           upri;
 104         int             uload;
 105         struct lwp      *uschedcp;
 106         struct rq       queues[NQS];
 107         struct rq       rtqueues[NQS];
 108         struct rq       idqueues[NQS];
 109         u_int32_t       queuebits;
 110         u_int32_t       rtqueuebits;
 111         u_int32_t       idqueuebits;
 112         int             runqcount;
 113         int             cpuid;
 114         cpumask_t       cpumask;
 115 #ifdef SMP
 116         cpu_node_t      *cpunode;
 117 #endif
 118 };
 119
 120 typedef struct usched_dfly_pcpu *dfly_pcpu_t;
 121
 122 static void dfly_acquire_curproc(struct lwp *lp);
 123 static void dfly_release_curproc(struct lwp *lp);
 124 static void dfly_select_curproc(globaldata_t gd);
 125 static void dfly_setrunqueue(struct lwp *lp);
 126 static void dfly_schedulerclock(struct lwp *lp, sysclock_t period,
 127                                 sysclock_t cpstamp);
 128 static void dfly_recalculate_estcpu(struct lwp *lp);
 129 static void dfly_resetpriority(struct lwp *lp);
 130 static void dfly_forking(struct lwp *plp, struct lwp *lp);
 131 static void dfly_exiting(struct lwp *lp, struct proc *);
 132 static void dfly_uload_update(struct lwp *lp);
 133 static void dfly_yield(struct lwp *lp);
 134 #ifdef SMP
 135 static dfly_pcpu_t dfly_choose_best_queue(struct lwp *lp);
 136 static dfly_pcpu_t dfly_choose_worst_queue(dfly_pcpu_t dd);
 137 static dfly_pcpu_t dfly_choose_queue_simple(dfly_pcpu_t dd, struct lwp *lp);
 138 #endif
 139
 140 #ifdef SMP
 141 static void dfly_need_user_resched_remote(void *dummy);
 142 #endif
 143 static struct lwp *dfly_chooseproc_locked(dfly_pcpu_t dd, struct lwp *chklp,
 144                                         int isremote);
 145 static void dfly_remrunqueue_locked(dfly_pcpu_t dd, struct lwp *lp);
 146 static void dfly_setrunqueue_locked(dfly_pcpu_t dd, struct lwp *lp);
 147
 148 struct usched usched_dfly = {
 149         { NULL },
 150         "dfly", "Original DragonFly Scheduler",
 151         NULL,                   /* default registration */
 152         NULL,                   /* default deregistration */
 153         dfly_acquire_curproc,
 154         dfly_release_curproc,
 155         dfly_setrunqueue,
 156         dfly_schedulerclock,
 157         dfly_recalculate_estcpu,
 158         dfly_resetpriority,
 159         dfly_forking,
 160         dfly_exiting,
 161         dfly_uload_update,
 162         NULL,                   /* setcpumask not supported */
 163         dfly_yield
 164 };
 165
 166 /*
 167  * We have NQS (32) run queues per scheduling class.  For the normal
 168  * class, there are 128 priorities scaled onto these 32 queues.  New
 169  * processes are added to the last entry in each queue, and processes
 170  * are selected for running by taking them from the head and maintaining
 171  * a simple FIFO arrangement.  Realtime and Idle priority processes have
 172  * and explicit 0-31 priority which maps directly onto their class queue
 173  * index.  When a queue has something in it, the corresponding bit is
 174  * set in the queuebits variable, allowing a single read to determine
 175  * the state of all 32 queues and then a ffs() to find the first busy
 176  * queue.
 177  */
 178 static cpumask_t dfly_curprocmask = -1; /* currently running a user process */
 179 static cpumask_t dfly_rdyprocmask;      /* ready to accept a user process */
 180 #ifdef SMP
 181 static volatile int dfly_scancpu;
 182 /*static struct spinlock dfly_spin = SPINLOCK_INITIALIZER(dfly_spin);*/
 183 #endif
 184 static struct usched_dfly_pcpu dfly_pcpu[MAXCPU];
 185 static struct sysctl_ctx_list usched_dfly_sysctl_ctx;
 186 static struct sysctl_oid *usched_dfly_sysctl_tree;
 187
 188 /* Debug info exposed through debug.* sysctl */
 189
 190 static int usched_dfly_debug = -1;
 191 SYSCTL_INT(_debug, OID_AUTO, dfly_scdebug, CTLFLAG_RW,
 192            &usched_dfly_debug, 0,
 193            "Print debug information for this pid");
 194
 195 static int usched_dfly_pid_debug = -1;
 196 SYSCTL_INT(_debug, OID_AUTO, dfly_pid_debug, CTLFLAG_RW,
 197            &usched_dfly_pid_debug, 0,
 198            "Print KTR debug information for this pid");
 199
 200 static int usched_dfly_chooser = 0;
 201 SYSCTL_INT(_debug, OID_AUTO, dfly_chooser, CTLFLAG_RW,
 202            &usched_dfly_chooser, 0,
 203            "Print KTR debug information for this pid");
 204
 205 /* Tunning usched_dfly - configurable through kern.usched_dfly.* */
 206 #ifdef SMP
 207 static int usched_dfly_smt = 0;
 208 static int usched_dfly_cache_coherent = 0;
 209 static int usched_dfly_weight1 = 10;
 210 static int usched_dfly_weight2 = 5;
 211 static int usched_dfly_stick_to_level = 0;
 212 #endif
 213 static int usched_dfly_rrinterval = (ESTCPUFREQ + 9) / 10;
 214 static int usched_dfly_decay = 8;
 215 static int usched_dfly_batch_time = 10;
 216
 217 /* KTR debug printings */
 218
 219 KTR_INFO_MASTER(usched);
 220
 221 #if !defined(KTR_USCHED_DFLY)
 222 #define KTR_USCHED_DFLY KTR_ALL
 223 #endif
 224
 225 #if 0
 226 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_acquire_curproc_urw, 0,
 227     "USCHED_DFLY(dfly_acquire_curproc in user_reseched_wanted "
 228     "after release: pid %d, cpuid %d, curr_cpuid %d)",
 229     pid_t pid, int cpuid, int curr);
 230 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_acquire_curproc_before_loop, 0,
 231     "USCHED_DFLY(dfly_acquire_curproc before loop: pid %d, cpuid %d, "
 232     "curr_cpuid %d)",
 233     pid_t pid, int cpuid, int curr);
 234 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_acquire_curproc_not, 0,
 235     "USCHED_DFLY(dfly_acquire_curproc couldn't acquire after "
 236     "dfly_setrunqueue: pid %d, cpuid %d, curr_lp pid %d, curr_cpuid %d)",
 237     pid_t pid, int cpuid, pid_t curr_pid, int curr_cpuid);
 238 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_acquire_curproc_switch, 0,
 239     "USCHED_DFLY(dfly_acquire_curproc after lwkt_switch: pid %d, "
 240     "cpuid %d, curr_cpuid %d)",
 241     pid_t pid, int cpuid, int curr);
 242
 243 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_release_curproc, 0,
 244     "USCHED_DFLY(dfly_release_curproc before select: pid %d, "
 245     "cpuid %d, curr_cpuid %d)",
 246     pid_t pid, int cpuid, int curr);
 247
 248 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_select_curproc, 0,
 249     "USCHED_DFLY(dfly_release_curproc before select: pid %d, "
 250     "cpuid %d, old_pid %d, old_cpuid %d, curr_cpuid %d)",
 251     pid_t pid, int cpuid, pid_t old_pid, int old_cpuid, int curr);
 252
 253 #ifdef SMP
 254 KTR_INFO(KTR_USCHED_DFLY, usched, batchy_test_false, 0,
 255     "USCHED_DFLY(batchy_looser_pri_test false: pid %d, "
 256     "cpuid %d, verify_mask %lu)",
 257     pid_t pid, int cpuid, cpumask_t mask);
 258 KTR_INFO(KTR_USCHED_DFLY, usched, batchy_test_true, 0,
 259     "USCHED_DFLY(batchy_looser_pri_test true: pid %d, "
 260     "cpuid %d, verify_mask %lu)",
 261     pid_t pid, int cpuid, cpumask_t mask);
 262
 263 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_setrunqueue_fc_smt, 0,
 264     "USCHED_DFLY(dfly_setrunqueue free cpus smt: pid %d, cpuid %d, "
 265     "mask %lu, curr_cpuid %d)",
 266     pid_t pid, int cpuid, cpumask_t mask, int curr);
 267 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_setrunqueue_fc_non_smt, 0,
 268     "USCHED_DFLY(dfly_setrunqueue free cpus check non_smt: pid %d, "
 269     "cpuid %d, mask %lu, curr_cpuid %d)",
 270     pid_t pid, int cpuid, cpumask_t mask, int curr);
 271 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_setrunqueue_rc, 0,
 272     "USCHED_DFLY(dfly_setrunqueue running cpus check: pid %d, "
 273     "cpuid %d, mask %lu, curr_cpuid %d)",
 274     pid_t pid, int cpuid, cpumask_t mask, int curr);
 275 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_setrunqueue_found, 0,
 276     "USCHED_DFLY(dfly_setrunqueue found cpu: pid %d, cpuid %d, "
 277     "mask %lu, found_cpuid %d, curr_cpuid %d)",
 278     pid_t pid, int cpuid, cpumask_t mask, int found_cpuid, int curr);
 279 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_setrunqueue_not_found, 0,
 280     "USCHED_DFLY(dfly_setrunqueue not found cpu: pid %d, cpuid %d, "
 281     "try_cpuid %d, curr_cpuid %d)",
 282     pid_t pid, int cpuid, int try_cpuid, int curr);
 283 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_setrunqueue_found_best_cpuid, 0,
 284     "USCHED_DFLY(dfly_setrunqueue found cpu: pid %d, cpuid %d, "
 285     "mask %lu, found_cpuid %d, curr_cpuid %d)",
 286     pid_t pid, int cpuid, cpumask_t mask, int found_cpuid, int curr);
 287 #endif
 288 #endif
 289
 290 KTR_INFO(KTR_USCHED_DFLY, usched, chooseproc, 0,
 291     "USCHED_DFLY(chooseproc: pid %d, old_cpuid %d, curr_cpuid %d)",
 292     pid_t pid, int old_cpuid, int curr);
 293 #ifdef SMP
 294 #if 0
 295 KTR_INFO(KTR_USCHED_DFLY, usched, chooseproc_cc, 0,
 296     "USCHED_DFLY(chooseproc_cc: pid %d, old_cpuid %d, curr_cpuid %d)",
 297     pid_t pid, int old_cpuid, int curr);
 298 KTR_INFO(KTR_USCHED_DFLY, usched, chooseproc_cc_not_good, 0,
 299     "USCHED_DFLY(chooseproc_cc not good: pid %d, old_cpumask %lu, "
 300     "sibling_mask %lu, curr_cpumask %lu)",
 301     pid_t pid, cpumask_t old_cpumask, cpumask_t sibling_mask, cpumask_t curr);
 302 KTR_INFO(KTR_USCHED_DFLY, usched, chooseproc_cc_elected, 0,
 303     "USCHED_DFLY(chooseproc_cc elected: pid %d, old_cpumask %lu, "
 304     "sibling_mask %lu, curr_cpumask: %lu)",
 305     pid_t pid, cpumask_t old_cpumask, cpumask_t sibling_mask, cpumask_t curr);
 306 #endif
 307
 308 KTR_INFO(KTR_USCHED_DFLY, usched, sched_thread_no_process, 0,
 309     "USCHED_DFLY(sched_thread %d no process scheduled: pid %d, old_cpuid %d)",
 310     int id, pid_t pid, int cpuid);
 311 KTR_INFO(KTR_USCHED_DFLY, usched, sched_thread_process, 0,
 312     "USCHED_DFLY(sched_thread %d process scheduled: pid %d, old_cpuid %d)",
 313     int id, pid_t pid, int cpuid);
 314 #if 0
 315 KTR_INFO(KTR_USCHED_DFLY, usched, sched_thread_no_process_found, 0,
 316     "USCHED_DFLY(sched_thread %d no process found; tmpmask %lu)",
 317     int id, cpumask_t tmpmask);
 318 #endif
 319 #endif
 320
 321 /*
 322  * DFLY_ACQUIRE_CURPROC
 323  *
 324  * This function is called when the kernel intends to return to userland.
 325  * It is responsible for making the thread the current designated userland
 326  * thread for this cpu, blocking if necessary.
 327  *
 328  * The kernel has already depressed our LWKT priority so we must not switch
 329  * until we have either assigned or disposed of the thread.
 330  *
 331  * WARNING! THIS FUNCTION IS ALLOWED TO CAUSE THE CURRENT THREAD TO MIGRATE
 332  * TO ANOTHER CPU!  Because most of the kernel assumes that no migration will
 333  * occur, this function is called only under very controlled circumstances.
 334  */
 335 static void
 336 dfly_acquire_curproc(struct lwp *lp)
 337 {
 338         globaldata_t gd;
 339         dfly_pcpu_t dd;
 340         thread_t td;
 341
 342         /*
 343          * Make sure we aren't sitting on a tsleep queue.
 344          */
 345         td = lp->lwp_thread;
 346         crit_enter_quick(td);
 347         if (td->td_flags & TDF_TSLEEPQ)
 348                 tsleep_remove(td);
 349         dfly_recalculate_estcpu(lp);
 350
 351         /*
 352          * If a reschedule was requested give another thread the
 353          * driver's seat.
 354          */
 355         if (user_resched_wanted()) {
 356                 clear_user_resched();
 357                 dfly_release_curproc(lp);
 358         }
 359
 360         /*
 361          * Loop until we are the current user thread
 362          */
 363         gd = mycpu;
 364         dd = &dfly_pcpu[gd->gd_cpuid];
 365
 366         do {
 367                 /*
 368                  * Process any pending events and higher priority threads.
 369                  */
 370                 lwkt_yield();
 371
 372                 /*
 373                  * Become the currently scheduled user thread for this cpu
 374                  * if we can do so trivially.
 375                  *
 376                  * We can steal another thread's current thread designation
 377                  * on this cpu since if we are running that other thread
 378                  * must not be, so we can safely deschedule it.
 379                  */
 380                 if (dd->uschedcp == lp) {
 381                         /*
 382                          * We are already the current lwp (hot path).
 383                          */
 384                         dd->upri = lp->lwp_priority;
 385                 } else if (dd->uschedcp == NULL) {
 386                         /*
 387                          * We can trivially become the current lwp.
 388                          */
 389                         atomic_set_cpumask(&dfly_curprocmask, gd->gd_cpumask);
 390                         dd->uschedcp = lp;
 391                         dd->upri = lp->lwp_priority;
 392                         KKASSERT(lp->lwp_qcpu == dd->cpuid);
 393                 } else if (dd->uschedcp && dd->upri > lp->lwp_priority) {
 394                         /*
 395                          * We can steal the current cpu's lwp designation
 396                          * away simply by replacing it.  The other thread
 397                          * will stall when it tries to return to userland,
 398                          * possibly rescheduling elsewhere when it calls
 399                          * setrunqueue.
 400                          */
 401                         dd->uschedcp = lp;
 402                         dd->upri = lp->lwp_priority;
 403                         KKASSERT(lp->lwp_qcpu == dd->cpuid);
 404                 } else {
 405                         /*
 406                          * We cannot become the current lwp, place the lp
 407                          * on the run-queue of this or another cpu and
 408                          * deschedule ourselves.
 409                          *
 410                          * When we are reactivated we will have another
 411                          * chance.
 412                          */
 413                         lwkt_deschedule(lp->lwp_thread);
 414                         dfly_setrunqueue(lp);
 415
 416                         /*
 417                          * Reload after a switch or setrunqueue/switch possibly
 418                          * moved us to another cpu.
 419                          */
 420                         lwkt_switch();
 421                         gd = mycpu;
 422                         dd = &dfly_pcpu[gd->gd_cpuid];
 423                 }
 424         } while (dd->uschedcp != lp);
 425
 426         crit_exit_quick(td);
 427         KKASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0);
 428 }
 429
 430 /*
 431  * DFLY_RELEASE_CURPROC
 432  *
 433  * This routine detaches the current thread from the userland scheduler,
 434  * usually because the thread needs to run or block in the kernel (at
 435  * kernel priority) for a while.
 436  *
 437  * This routine is also responsible for selecting a new thread to
 438  * make the current thread.
 439  *
 440  * NOTE: This implementation differs from the dummy example in that
 441  * dfly_select_curproc() is able to select the current process, whereas
 442  * dummy_select_curproc() is not able to select the current process.
 443  * This means we have to NULL out uschedcp.
 444  *
 445  * Additionally, note that we may already be on a run queue if releasing
 446  * via the lwkt_switch() in dfly_setrunqueue().
 447  */
 448
 449 static void
 450 dfly_release_curproc(struct lwp *lp)
 451 {
 452         globaldata_t gd = mycpu;
 453         dfly_pcpu_t dd = &dfly_pcpu[gd->gd_cpuid];
 454
 455         /*
 456          * Make sure td_wakefromcpu is defaulted.  This will be overwritten
 457          * by wakeup().
 458          */
 459         lp->lwp_thread->td_wakefromcpu = gd->gd_cpuid;
 460
 461         if (dd->uschedcp == lp) {
 462                 crit_enter();
 463                 KKASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0);
 464
 465                 dd->uschedcp = NULL;    /* don't let lp be selected */
 466                 dd->upri = PRIBASE_NULL;
 467                 atomic_clear_cpumask(&dfly_curprocmask, gd->gd_cpumask);
 468                 dfly_select_curproc(gd);
 469                 crit_exit();
 470         }
 471 }
 472
 473 /*
 474  * DFLY_SELECT_CURPROC
 475  *
 476  * Select a new current process for this cpu and clear any pending user
 477  * reschedule request.  The cpu currently has no current process.
 478  *
 479  * This routine is also responsible for equal-priority round-robining,
 480  * typically triggered from dfly_schedulerclock().  In our dummy example
 481  * all the 'user' threads are LWKT scheduled all at once and we just
 482  * call lwkt_switch().
 483  *
 484  * The calling process is not on the queue and cannot be selected.
 485  */
 486 static
 487 void
 488 dfly_select_curproc(globaldata_t gd)
 489 {
 490         dfly_pcpu_t dd = &dfly_pcpu[gd->gd_cpuid];
 491         struct lwp *nlp;
 492         int cpuid = gd->gd_cpuid;
 493
 494         crit_enter_gd(gd);
 495
 496         /*spin_lock(&dfly_spin);*/
 497         spin_lock(&dd->spin);
 498         nlp = dfly_chooseproc_locked(dd, dd->uschedcp, 0);
 499
 500         if (nlp) {
 501                 atomic_set_cpumask(&dfly_curprocmask, CPUMASK(cpuid));
 502                 dd->upri = nlp->lwp_priority;
 503                 dd->uschedcp = nlp;
 504                 dd->rrcount = 0;                /* reset round robin */
 505                 spin_unlock(&dd->spin);
 506                 /*spin_unlock(&dfly_spin);*/
 507 #ifdef SMP
 508                 lwkt_acquire(nlp->lwp_thread);
 509 #endif
 510                 lwkt_schedule(nlp->lwp_thread);
 511         } else {
 512                 spin_unlock(&dd->spin);
 513                 /*spin_unlock(&dfly_spin);*/
 514         }
 515         crit_exit_gd(gd);
 516 }
 517
 518 /*
 519  * Place the specified lwp on the user scheduler's run queue.  This routine
 520  * must be called with the thread descheduled.  The lwp must be runnable.
 521  * It must not be possible for anyone else to explicitly schedule this thread.
 522  *
 523  * The thread may be the current thread as a special case.
 524  */
 525 static void
 526 dfly_setrunqueue(struct lwp *lp)
 527 {
 528         globaldata_t rgd;
 529         dfly_pcpu_t rdd;
 530
 531         /*
 532          * First validate the process LWKT state.
 533          */
 534         crit_enter();
 535         KASSERT(lp->lwp_stat == LSRUN, ("setrunqueue: lwp not LSRUN"));
 536         KASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0,
 537             ("lwp %d/%d already on runq! flag %08x/%08x", lp->lwp_proc->p_pid,
 538              lp->lwp_tid, lp->lwp_proc->p_flags, lp->lwp_flags));
 539         KKASSERT((lp->lwp_thread->td_flags & TDF_RUNQ) == 0);
 540
 541         /*
 542          * NOTE: rdd does not necessarily represent the current cpu.
 543          *       Instead it represents the cpu the thread was last
 544          *       scheduled on.
 545          */
 546         rdd = &dfly_pcpu[lp->lwp_qcpu];
 547
 548         /*
 549          * This process is not supposed to be scheduled anywhere or assigned
 550          * as the current process anywhere.  Assert the condition.
 551          */
 552         KKASSERT(rdd->uschedcp != lp);
 553
 554 #ifndef SMP
 555         /*
 556          * If we are not SMP we do not have a scheduler helper to kick
 557          * and must directly activate the process if none are scheduled.
 558          *
 559          * This is really only an issue when bootstrapping init since
 560          * the caller in all other cases will be a user process, and
 561          * even if released (rdd->uschedcp == NULL), that process will
 562          * kickstart the scheduler when it returns to user mode from
 563          * the kernel.
 564          *
 565          * NOTE: On SMP we can't just set some other cpu's uschedcp.
 566          */
 567         if (rdd->uschedcp == NULL) {
 568                 spin_lock(&rdd->spin);
 569                 if (rdd->uschedcp == NULL) {
 570                         atomic_set_cpumask(&dfly_curprocmask, 1);
 571                         rdd->uschedcp = lp;
 572                         rdd->upri = lp->lwp_priority;
 573                         spin_unlock(&rdd->spin);
 574                         lwkt_schedule(lp->lwp_thread);
 575                         crit_exit();
 576                         return;
 577                 }
 578                 spin_unlock(&rdd->spin);
 579         }
 580 #endif
 581
 582 #ifdef SMP
 583         /*
 584          * XXX fixme.  Could be part of a remrunqueue/setrunqueue
 585          * operation when the priority is recalculated, so TDF_MIGRATING
 586          * may already be set.
 587          */
 588         if ((lp->lwp_thread->td_flags & TDF_MIGRATING) == 0)
 589                 lwkt_giveaway(lp->lwp_thread);
 590 #endif
 591
 592 #ifdef SMP
 593         /*
 594          * Ok, we have to setrunqueue some target cpu and request a reschedule
 595          * if necessary.
 596          *
 597          * We have to choose the best target cpu.  It might not be the current
 598          * target even if the current cpu has no running user thread (for
 599          * example, because the current cpu might be a hyperthread and its
 600          * sibling has a thread assigned).
 601          */
 602         /*spin_lock(&dfly_spin);*/
 603         rdd = dfly_choose_best_queue(lp);
 604         rgd = globaldata_find(rdd->cpuid);
 605
 606         /*
 607          * We lose control of lp the moment we release the spinlock after
 608          * having placed lp on the queue.  i.e. another cpu could pick it
 609          * up and it could exit, or its priority could be further adjusted,
 610          * or something like that.
 611          *
 612          * WARNING! dd can point to a foreign cpu!
 613          */
 614         spin_lock(&rdd->spin);
 615         dfly_setrunqueue_locked(rdd, lp);
 616         /*spin_unlock(&dfly_spin);*/
 617
 618         if (rgd == mycpu) {
 619                 if ((rdd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) {
 620                         spin_unlock(&rdd->spin);
 621                         if (rdd->uschedcp == NULL) {
 622                                 wakeup_mycpu(&rdd->helper_thread); /* XXX */
 623                                 need_user_resched();
 624                         } else {
 625                                 need_user_resched();
 626                         }
 627                 } else {
 628                         spin_unlock(&rdd->spin);
 629                 }
 630         } else {
 631                 atomic_clear_cpumask(&dfly_rdyprocmask, rgd->gd_cpumask);
 632                 if ((rdd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) {
 633                         spin_unlock(&rdd->spin);
 634                         lwkt_send_ipiq(rgd, dfly_need_user_resched_remote,
 635                                        NULL);
 636                 } else {
 637                         spin_unlock(&rdd->spin);
 638                         wakeup(&rdd->helper_thread);
 639                 }
 640         }
 641 #else
 642         /*
 643          * Request a reschedule if appropriate.
 644          */
 645         spin_lock(&rdd->spin);
 646         dfly_setrunqueue_locked(rdd, lp);
 647         spin_unlock(&rdd->spin);
 648         if ((rdd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) {
 649                 need_user_resched();
 650         }
 651 #endif
 652         crit_exit();
 653 }
 654
 655 /*
 656  * This routine is called from a systimer IPI.  It MUST be MP-safe and
 657  * the BGL IS NOT HELD ON ENTRY.  This routine is called at ESTCPUFREQ on
 658  * each cpu.
 659  */
 660 static
 661 void
 662 dfly_schedulerclock(struct lwp *lp, sysclock_t period, sysclock_t cpstamp)
 663 {
 664         globaldata_t gd = mycpu;
 665         dfly_pcpu_t dd = &dfly_pcpu[gd->gd_cpuid];
 666
 667         /*
 668          * Do we need to round-robin?  We round-robin 10 times a second.
 669          * This should only occur for cpu-bound batch processes.
 670          */
 671         if (++dd->rrcount >= usched_dfly_rrinterval) {
 672                 dd->rrcount = 0;
 673                 need_user_resched();
 674         }
 675
 676         /*
 677          * Adjust estcpu upward using a real time equivalent calculation.
 678          */
 679         lp->lwp_estcpu = ESTCPULIM(lp->lwp_estcpu + ESTCPUMAX / ESTCPUFREQ + 1);
 680
 681         /*
 682          * Spinlocks also hold a critical section so there should not be
 683          * any active.
 684          */
 685         KKASSERT(gd->gd_spinlocks_wr == 0);
 686
 687         dfly_resetpriority(lp);
 688 }
 689
 690 /*
 691  * Called from acquire and from kern_synch's one-second timer (one of the
 692  * callout helper threads) with a critical section held.
 693  *
 694  * Decay p_estcpu based on the number of ticks we haven't been running
 695  * and our p_nice.  As the load increases each process observes a larger
 696  * number of idle ticks (because other processes are running in them).
 697  * This observation leads to a larger correction which tends to make the
 698  * system more 'batchy'.
 699  *
 700  * Note that no recalculation occurs for a process which sleeps and wakes
 701  * up in the same tick.  That is, a system doing thousands of context
 702  * switches per second will still only do serious estcpu calculations
 703  * ESTCPUFREQ times per second.
 704  */
 705 static
 706 void
 707 dfly_recalculate_estcpu(struct lwp *lp)
 708 {
 709         globaldata_t gd = mycpu;
 710         dfly_pcpu_t dd = &dfly_pcpu[gd->gd_cpuid];
 711         sysclock_t cpbase;
 712         sysclock_t ttlticks;
 713         int estcpu;
 714         int decay_factor;
 715
 716         /*
 717          * We have to subtract periodic to get the last schedclock
 718          * timeout time, otherwise we would get the upcoming timeout.
 719          * Keep in mind that a process can migrate between cpus and
 720          * while the scheduler clock should be very close, boundary
 721          * conditions could lead to a small negative delta.
 722          */
 723         cpbase = gd->gd_schedclock.time - gd->gd_schedclock.periodic;
 724
 725         if (lp->lwp_slptime > 1) {
 726                 /*
 727                  * Too much time has passed, do a coarse correction.
 728                  */
 729                 lp->lwp_estcpu = lp->lwp_estcpu >> 1;
 730                 dfly_resetpriority(lp);
 731                 lp->lwp_cpbase = cpbase;
 732                 lp->lwp_cpticks = 0;
 733                 lp->lwp_batch -= ESTCPUFREQ;
 734                 if (lp->lwp_batch < 0)
 735                         lp->lwp_batch = 0;
 736         } else if (lp->lwp_cpbase != cpbase) {
 737                 /*
 738                  * Adjust estcpu if we are in a different tick.  Don't waste
 739                  * time if we are in the same tick.
 740                  *
 741                  * First calculate the number of ticks in the measurement
 742                  * interval.  The ttlticks calculation can wind up 0 due to
 743                  * a bug in the handling of lwp_slptime  (as yet not found),
 744                  * so make sure we do not get a divide by 0 panic.
 745                  */
 746                 ttlticks = (cpbase - lp->lwp_cpbase) /
 747                            gd->gd_schedclock.periodic;
 748                 if (ttlticks < 0) {
 749                         ttlticks = 0;
 750                         lp->lwp_cpbase = cpbase;
 751                 }
 752                 if (ttlticks == 0)
 753                         return;
 754                 updatepcpu(lp, lp->lwp_cpticks, ttlticks);
 755
 756                 /*
 757                  * Calculate the percentage of one cpu used factoring in ncpus
 758                  * and the load and adjust estcpu.  Handle degenerate cases
 759                  * by adding 1 to runqcount.
 760                  *
 761                  * estcpu is scaled by ESTCPUMAX.
 762                  *
 763                  * runqcount is the excess number of user processes
 764                  * that cannot be immediately scheduled to cpus.  We want
 765                  * to count these as running to avoid range compression
 766                  * in the base calculation (which is the actual percentage
 767                  * of one cpu used).
 768                  */
 769                 estcpu = (lp->lwp_cpticks * ESTCPUMAX) *
 770                          (dd->runqcount + ncpus) / (ncpus * ttlticks);
 771
 772                 /*
 773                  * If estcpu is > 50% we become more batch-like
 774                  * If estcpu is <= 50% we become less batch-like
 775                  *
 776                  * It takes 30 cpu seconds to traverse the entire range.
 777                  */
 778                 if (estcpu > ESTCPUMAX / 2) {
 779                         lp->lwp_batch += ttlticks;
 780                         if (lp->lwp_batch > BATCHMAX)
 781                                 lp->lwp_batch = BATCHMAX;
 782                 } else {
 783                         lp->lwp_batch -= ttlticks;
 784                         if (lp->lwp_batch < 0)
 785                                 lp->lwp_batch = 0;
 786                 }
 787
 788                 if (usched_dfly_debug == lp->lwp_proc->p_pid) {
 789                         kprintf("pid %d lwp %p estcpu %3d %3d bat %d cp %d/%d",
 790                                 lp->lwp_proc->p_pid, lp,
 791                                 estcpu, lp->lwp_estcpu,
 792                                 lp->lwp_batch,
 793                                 lp->lwp_cpticks, ttlticks);
 794                 }
 795
 796                 /*
 797                  * Adjust lp->lwp_esetcpu.  The decay factor determines how
 798                  * quickly lwp_estcpu collapses to its realtime calculation.
 799                  * A slower collapse gives us a more accurate number but
 800                  * can cause a cpu hog to eat too much cpu before the
 801                  * scheduler decides to downgrade it.
 802                  *
 803                  * NOTE: p_nice is accounted for in dfly_resetpriority(),
 804                  *       and not here, but we must still ensure that a
 805                  *       cpu-bound nice -20 process does not completely
 806                  *       override a cpu-bound nice +20 process.
 807                  *
 808                  * NOTE: We must use ESTCPULIM() here to deal with any
 809                  *       overshoot.
 810                  */
 811                 decay_factor = usched_dfly_decay;
 812                 if (decay_factor < 1)
 813                         decay_factor = 1;
 814                 if (decay_factor > 1024)
 815                         decay_factor = 1024;
 816
 817                 lp->lwp_estcpu = ESTCPULIM(
 818                         (lp->lwp_estcpu * decay_factor + estcpu) /
 819                         (decay_factor + 1));
 820
 821                 if (usched_dfly_debug == lp->lwp_proc->p_pid)
 822                         kprintf(" finalestcpu %d\n", lp->lwp_estcpu);
 823                 dfly_resetpriority(lp);
 824                 lp->lwp_cpbase += ttlticks * gd->gd_schedclock.periodic;
 825                 lp->lwp_cpticks = 0;
 826         }
 827 }
 828
 829 /*
 830  * Compute the priority of a process when running in user mode.
 831  * Arrange to reschedule if the resulting priority is better
 832  * than that of the current process.
 833  *
 834  * This routine may be called with any process.
 835  *
 836  * This routine is called by fork1() for initial setup with the process
 837  * of the run queue, and also may be called normally with the process on or
 838  * off the run queue.
 839  */
 840 static void
 841 dfly_resetpriority(struct lwp *lp)
 842 {
 843         dfly_pcpu_t rdd;
 844         int newpriority;
 845         u_short newrqtype;
 846         int rcpu;
 847         int checkpri;
 848         int estcpu;
 849
 850         crit_enter();
 851
 852         /*
 853          * Lock the scheduler (lp) belongs to.  This can be on a different
 854          * cpu.  Handle races.  This loop breaks out with the appropriate
 855          * rdd locked.
 856          */
 857         for (;;) {
 858                 rcpu = lp->lwp_qcpu;
 859                 rdd = &dfly_pcpu[rcpu];
 860                 spin_lock(&rdd->spin);
 861                 if (rcpu == lp->lwp_qcpu)
 862                         break;
 863                 spin_unlock(&rdd->spin);
 864         }
 865
 866         /*
 867          * Calculate the new priority and queue type
 868          */
 869         newrqtype = lp->lwp_rtprio.type;
 870
 871         switch(newrqtype) {
 872         case RTP_PRIO_REALTIME:
 873         case RTP_PRIO_FIFO:
 874                 newpriority = PRIBASE_REALTIME +
 875                              (lp->lwp_rtprio.prio & PRIMASK);
 876                 break;
 877         case RTP_PRIO_NORMAL:
 878                 /*
 879                  * Detune estcpu based on batchiness.  lwp_batch ranges
 880                  * from 0 to  BATCHMAX.  Limit estcpu for the sake of
 881                  * the priority calculation to between 50% and 100%.
 882                  */
 883                 estcpu = lp->lwp_estcpu * (lp->lwp_batch + BATCHMAX) /
 884                          (BATCHMAX * 2);
 885
 886                 /*
 887                  * p_nice piece         Adds (0-40) * 2         0-80
 888                  * estcpu               Adds 16384  * 4 / 512   0-128
 889                  */
 890                 newpriority = (lp->lwp_proc->p_nice - PRIO_MIN) * PPQ / NICEPPQ;
 891                 newpriority += estcpu * PPQ / ESTCPUPPQ;
 892                 newpriority = newpriority * MAXPRI / (PRIO_RANGE * PPQ /
 893                               NICEPPQ + ESTCPUMAX * PPQ / ESTCPUPPQ);
 894                 newpriority = PRIBASE_NORMAL + (newpriority & PRIMASK);
 895                 break;
 896         case RTP_PRIO_IDLE:
 897                 newpriority = PRIBASE_IDLE + (lp->lwp_rtprio.prio & PRIMASK);
 898                 break;
 899         case RTP_PRIO_THREAD:
 900                 newpriority = PRIBASE_THREAD + (lp->lwp_rtprio.prio & PRIMASK);
 901                 break;
 902         default:
 903                 panic("Bad RTP_PRIO %d", newrqtype);
 904                 /* NOT REACHED */
 905         }
 906
 907         /*
 908          * The newpriority incorporates the queue type so do a simple masked
 909          * check to determine if the process has moved to another queue.  If
 910          * it has, and it is currently on a run queue, then move it.
 911          *
 912          * Since uload is ~PPQMASK masked, no modifications are necessary if
 913          * we end up in the same run queue.
 914          */
 915         if ((lp->lwp_priority ^ newpriority) & ~PPQMASK) {
 916                 int delta_uload;
 917
 918                 /*
 919                  * uload can change, calculate the adjustment to reduce
 920                  * edge cases since choosers scan the cpu topology without
 921                  * locks.
 922                  */
 923                 if (lp->lwp_mpflags & LWP_MP_ULOAD) {
 924                         delta_uload =
 925                                 -((lp->lwp_priority & ~PPQMASK) & PRIMASK) +
 926                                 ((newpriority & ~PPQMASK) & PRIMASK);
 927                         atomic_add_int(&dfly_pcpu[lp->lwp_qcpu].uload,
 928                                        delta_uload);
 929                 }
 930                 if (lp->lwp_mpflags & LWP_MP_ONRUNQ) {
 931                         dfly_remrunqueue_locked(rdd, lp);
 932                         lp->lwp_priority = newpriority;
 933                         lp->lwp_rqtype = newrqtype;
 934                         lp->lwp_rqindex = (newpriority & PRIMASK) / PPQ;
 935                         dfly_setrunqueue_locked(rdd, lp);
 936                         checkpri = 1;
 937                 } else {
 938                         lp->lwp_priority = newpriority;
 939                         lp->lwp_rqtype = newrqtype;
 940                         lp->lwp_rqindex = (newpriority & PRIMASK) / PPQ;
 941                         checkpri = 0;
 942                 }
 943         } else {
 944                 /*
 945                  * In the same PPQ, uload cannot change.
 946                  */
 947                 lp->lwp_priority = newpriority;
 948                 checkpri = 1;
 949                 rcpu = -1;
 950         }
 951
 952         /*
 953          * Determine if we need to reschedule the target cpu.  This only
 954          * occurs if the LWP is already on a scheduler queue, which means
 955          * that idle cpu notification has already occured.  At most we
 956          * need only issue a need_user_resched() on the appropriate cpu.
 957          *
 958          * The LWP may be owned by a CPU different from the current one,
 959          * in which case dd->uschedcp may be modified without an MP lock
 960          * or a spinlock held.  The worst that happens is that the code
 961          * below causes a spurious need_user_resched() on the target CPU
 962          * and dd->pri to be wrong for a short period of time, both of
 963          * which are harmless.
 964          *
 965          * If checkpri is 0 we are adjusting the priority of the current
 966          * process, possibly higher (less desireable), so ignore the upri
 967          * check which will fail in that case.
 968          */
 969         if (rcpu >= 0) {
 970                 if ((dfly_rdyprocmask & CPUMASK(rcpu)) &&
 971                     (checkpri == 0 ||
 972                      (rdd->upri & ~PRIMASK) > (lp->lwp_priority & ~PRIMASK))) {
 973 #ifdef SMP
 974                         if (rcpu == mycpu->gd_cpuid) {
 975                                 spin_unlock(&rdd->spin);
 976                                 need_user_resched();
 977                         } else {
 978                                 atomic_clear_cpumask(&dfly_rdyprocmask,
 979                                                      CPUMASK(rcpu));
 980                                 spin_unlock(&rdd->spin);
 981                                 lwkt_send_ipiq(globaldata_find(rcpu),
 982                                                dfly_need_user_resched_remote,
 983                                                NULL);
 984                         }
 985 #else
 986                         spin_unlock(&rdd->spin);
 987                         need_user_resched();
 988 #endif
 989                 } else {
 990                         spin_unlock(&rdd->spin);
 991                 }
 992         } else {
 993                 spin_unlock(&rdd->spin);
 994         }
 995         crit_exit();
 996 }
 997
 998 static
 999 void
1000 dfly_yield(struct lwp *lp)
1001 {
1002 #if 0
1003         /* FUTURE (or something similar) */
1004         switch(lp->lwp_rqtype) {
1005         case RTP_PRIO_NORMAL:
1006                 lp->lwp_estcpu = ESTCPULIM(lp->lwp_estcpu + ESTCPUINCR);
1007                 break;
1008         default:
1009                 break;
1010         }
1011 #endif
1012         need_user_resched();
1013 }
1014
1015 /*
1016  * Called from fork1() when a new child process is being created.
1017  *
1018  * Give the child process an initial estcpu that is more batch then
1019  * its parent and dock the parent for the fork (but do not
1020  * reschedule the parent).   This comprises the main part of our batch
1021  * detection heuristic for both parallel forking and sequential execs.
1022  *
1023  * XXX lwp should be "spawning" instead of "forking"
1024  */
1025 static void
1026 dfly_forking(struct lwp *plp, struct lwp *lp)
1027 {
1028         /*
1029          * Put the child 4 queue slots (out of 32) higher than the parent
1030          * (less desireable than the parent).
1031          */
1032         lp->lwp_estcpu = ESTCPULIM(plp->lwp_estcpu + ESTCPUPPQ * 4);
1033
1034         /*
1035          * The batch status of children always starts out centerline
1036          * and will inch-up or inch-down as appropriate.  It takes roughly
1037          * ~15 seconds of >50% cpu to hit the limit.
1038          */
1039         lp->lwp_batch = BATCHMAX / 2;
1040
1041         /*
1042          * Dock the parent a cost for the fork, protecting us from fork
1043          * bombs.  If the parent is forking quickly make the child more
1044          * batchy.
1045          */
1046         plp->lwp_estcpu = ESTCPULIM(plp->lwp_estcpu + ESTCPUPPQ / 16);
1047 }
1048
1049 /*
1050  * Called when a lwp is being removed from this scheduler, typically
1051  * during lwp_exit().
1052  */
1053 static void
1054 dfly_exiting(struct lwp *lp, struct proc *child_proc)
1055 {
1056         dfly_pcpu_t dd = &dfly_pcpu[lp->lwp_qcpu];
1057
1058         if (lp->lwp_mpflags & LWP_MP_ULOAD) {
1059                 atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ULOAD);
1060                 atomic_add_int(&dd->uload,
1061                                -((lp->lwp_priority & ~PPQMASK) & PRIMASK));
1062         }
1063 }
1064
1065 static void
1066 dfly_uload_update(struct lwp *lp)
1067 {
1068         dfly_pcpu_t dd = &dfly_pcpu[lp->lwp_qcpu];
1069
1070         if (lp->lwp_thread->td_flags & TDF_RUNQ) {
1071                 if ((lp->lwp_mpflags & LWP_MP_ULOAD) == 0) {
1072                         atomic_set_int(&lp->lwp_mpflags, LWP_MP_ULOAD);
1073                         atomic_add_int(&dd->uload,
1074                                    ((lp->lwp_priority & ~PPQMASK) & PRIMASK));
1075                 }
1076         } else {
1077                 if (lp->lwp_mpflags & LWP_MP_ULOAD) {
1078                         atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ULOAD);
1079                         atomic_add_int(&dd->uload,
1080                                    -((lp->lwp_priority & ~PPQMASK) & PRIMASK));
1081                 }
1082         }
1083 }
1084
1085 /*
1086  * chooseproc() is called when a cpu needs a user process to LWKT schedule,
1087  * it selects a user process and returns it.  If chklp is non-NULL and chklp
1088  * has a better or equal priority then the process that would otherwise be
1089  * chosen, NULL is returned.
1090  *
1091  * Until we fix the RUNQ code the chklp test has to be strict or we may
1092  * bounce between processes trying to acquire the current process designation.
1093  *
1094  * Must be called with dfly_spin exclusive held.  The spinlock is
1095  * left intact through the entire routine.
1096  *
1097  * if chklp is NULL this function will dive other cpu's queues looking
1098  * for work if the current queue is empty.
1099  */
1100 static
1101 struct lwp *
1102 dfly_chooseproc_locked(dfly_pcpu_t dd, struct lwp *chklp, int isremote)
1103 {
1104 #ifdef SMP
1105         dfly_pcpu_t xdd;
1106 #endif
1107         struct lwp *lp;
1108         struct rq *q;
1109         u_int32_t *which, *which2;
1110         u_int32_t pri;
1111         u_int32_t rtqbits;
1112         u_int32_t tsqbits;
1113         u_int32_t idqbits;
1114
1115         rtqbits = dd->rtqueuebits;
1116         tsqbits = dd->queuebits;
1117         idqbits = dd->idqueuebits;
1118
1119         if (rtqbits) {
1120                 pri = bsfl(rtqbits);
1121                 q = &dd->rtqueues[pri];
1122                 which = &dd->rtqueuebits;
1123                 which2 = &rtqbits;
1124         } else if (tsqbits) {
1125                 pri = bsfl(tsqbits);
1126                 q = &dd->queues[pri];
1127                 which = &dd->queuebits;
1128                 which2 = &tsqbits;
1129         } else if (idqbits) {
1130                 pri = bsfl(idqbits);
1131                 q = &dd->idqueues[pri];
1132                 which = &dd->idqueuebits;
1133                 which2 = &idqbits;
1134         } else
1135 #ifdef SMP
1136         if (isremote) {
1137                 /*
1138                  * Disallow remote->remote recursion
1139                  */
1140                 return (NULL);
1141         } else {
1142                 /*
1143                  * Pull a runnable thread from a remote run queue.  We have
1144                  * to adjust qcpu and uload manually because the lp we return
1145                  * might be assigned directly to uschedcp (setrunqueue might
1146                  * not be called).
1147                  */
1148                 xdd = dfly_choose_worst_queue(dd);
1149                 if (xdd && xdd != dd && spin_trylock(&xdd->spin)) {
1150                         lp = dfly_chooseproc_locked(xdd, NULL, 1);
1151                         if (lp) {
1152                                 if (lp->lwp_mpflags & LWP_MP_ULOAD) {
1153                                         atomic_add_int(&xdd->uload,
1154                                             -((lp->lwp_priority & ~PPQMASK) &
1155                                               PRIMASK));
1156                                 }
1157                                 lp->lwp_qcpu = dd->cpuid;
1158                                 atomic_add_int(&dd->uload,
1159                                     ((lp->lwp_priority & ~PPQMASK) & PRIMASK));
1160                                 atomic_set_int(&lp->lwp_mpflags, LWP_MP_ULOAD);
1161                         }
1162                         spin_unlock(&xdd->spin);
1163                 } else {
1164                         lp = NULL;
1165                 }
1166                 return (lp);
1167         }
1168 #else
1169         {
1170                 return NULL;
1171         }
1172 #endif
1173         lp = TAILQ_FIRST(q);
1174         KASSERT(lp, ("chooseproc: no lwp on busy queue"));
1175
1176         /*
1177          * If the passed lwp <chklp> is reasonably close to the selected
1178          * lwp <lp>, return NULL (indicating that <chklp> should be kept).
1179          *
1180          * Note that we must error on the side of <chklp> to avoid bouncing
1181          * between threads in the acquire code.
1182          */
1183         if (chklp) {
1184                 if (chklp->lwp_priority < lp->lwp_priority + PPQ)
1185                         return(NULL);
1186         }
1187
1188         KTR_COND_LOG(usched_chooseproc,
1189             lp->lwp_proc->p_pid == usched_dfly_pid_debug,
1190             lp->lwp_proc->p_pid,
1191             lp->lwp_thread->td_gd->gd_cpuid,
1192             mycpu->gd_cpuid);
1193
1194         TAILQ_REMOVE(q, lp, lwp_procq);
1195         --dd->runqcount;
1196         if (TAILQ_EMPTY(q))
1197                 *which &= ~(1 << pri);
1198         KASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) != 0, ("not on runq6!"));
1199         atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ);
1200
1201         return lp;
1202 }
1203
1204 #ifdef SMP
1205
1206 /*
1207  * USED TO PUSH RUNNABLE LWPS TO THE LEAST LOADED CPU.
1208  *
1209  * Choose a cpu node to schedule lp on, hopefully nearby its current
1210  * node.  We give the current node a modest advantage for obvious reasons.
1211  *
1212  * We also give the node the thread was woken up FROM a slight advantage
1213  * in order to try to schedule paired threads which synchronize/block waiting
1214  * for each other fairly close to each other.  Similarly in a network setting
1215  * this feature will also attempt to place a user process near the kernel
1216  * protocol thread that is feeding it data.  THIS IS A CRITICAL PART of the
1217  * algorithm as it heuristically groups synchronizing processes for locality
1218  * of reference in multi-socket systems.
1219  *
1220  * The caller will normally dfly_setrunqueue() lp on the returned queue.
1221  *
1222  * When the topology is known choose a cpu whos group has, in aggregate,
1223  * has the lowest weighted load.
1224  */
1225 static
1226 dfly_pcpu_t
1227 dfly_choose_best_queue(struct lwp *lp)
1228 {
1229         cpumask_t mask;
1230         cpu_node_t *cpup;
1231         cpu_node_t *cpun;
1232         cpu_node_t *cpub;
1233         dfly_pcpu_t dd1 = &dfly_pcpu[lp->lwp_qcpu];
1234         dfly_pcpu_t dd2 = &dfly_pcpu[lp->lwp_thread->td_wakefromcpu];
1235         dfly_pcpu_t rdd;
1236         int cpuid;
1237         int n;
1238         int load;
1239         int lowest_load;
1240         int level;
1241
1242         /*
1243          * When the topology is unknown choose a random cpu that is hopefully
1244          * idle.
1245          */
1246         if (dd1->cpunode == NULL)
1247                 return (dfly_choose_queue_simple(dd1, lp));
1248
1249         /*
1250          * When the topology is known choose a cpu whos group has, in
1251          * aggregate, has the lowest weighted load.
1252          */
1253         cpup = root_cpu_node;
1254         rdd = dd1;
1255         level = cpu_topology_levels_number;
1256
1257         while (cpup) {
1258                 /*
1259                  * Degenerate case super-root
1260                  */
1261                 if (cpup->child_node && cpup->child_no == 1) {
1262                         cpup = cpup->child_node;
1263                         --level;
1264                         continue;
1265                 }
1266
1267                 /*
1268                  * Terminal cpunode
1269                  */
1270                 if (cpup->child_node == NULL) {
1271                         rdd = &dfly_pcpu[BSFCPUMASK(cpup->members)];
1272                         break;
1273                 }
1274
1275                 cpub = NULL;
1276                 lowest_load = 0x7FFFFFFF;
1277
1278                 for (n = 0; n < cpup->child_no; ++n) {
1279                         /*
1280                          * Accumulate load information for all cpus
1281                          * which are members of this node.
1282                          */
1283                         cpun = &cpup->child_node[n];
1284                         mask = cpun->members & usched_global_cpumask &
1285                                smp_active_mask & lp->lwp_cpumask;
1286                         if (mask == 0)
1287                                 continue;
1288                         load = 0;
1289                         while (mask) {
1290                                 cpuid = BSFCPUMASK(mask);
1291                                 load += dfly_pcpu[cpuid].uload;
1292                                 mask &= ~CPUMASK(cpuid);
1293                         }
1294
1295                         /*
1296                          * Give a slight advantage to nearby cpus.
1297                          */
1298                         if (cpun->members & dd1->cpumask)
1299                                 load -= PPQ * level * usched_dfly_weight1 / 10;
1300                         else if (cpun->members & dd2->cpumask)
1301                                 load -= PPQ * level * usched_dfly_weight2 / 10;
1302
1303                         /*
1304                          * Calculate the best load
1305                          */
1306                         if (cpub == NULL || lowest_load > load ||
1307                             (lowest_load == load &&
1308                              (cpun->members & dd1->cpumask))
1309                         ) {
1310                                 lowest_load = load;
1311                                 cpub = cpun;
1312                         }
1313                 }
1314                 cpup = cpub;
1315                 --level;
1316         }
1317         if (usched_dfly_chooser)
1318                 kprintf("lp %02d->%02d %s\n",
1319                         lp->lwp_qcpu, rdd->cpuid, lp->lwp_proc->p_comm);
1320         return (rdd);
1321 }
1322
1323 /*
1324  * USED TO PULL RUNNABLE LWPS FROM THE MOST LOADED CPU.
1325  *
1326  * Choose the worst queue close to dd's cpu node with a non-empty runq.
1327  *
1328  * This is used by the thread chooser when the current cpu's queues are
1329  * empty to steal a thread from another cpu's queue.  We want to offload
1330  * the most heavily-loaded queue.
1331  */
1332 static
1333 dfly_pcpu_t
1334 dfly_choose_worst_queue(dfly_pcpu_t dd)
1335 {
1336         cpumask_t mask;
1337         cpu_node_t *cpup;
1338         cpu_node_t *cpun;
1339         cpu_node_t *cpub;
1340         dfly_pcpu_t rdd;
1341         int cpuid;
1342         int n;
1343         int load;
1344         int highest_load;
1345         int uloadok;
1346         int level;
1347
1348         /*
1349          * When the topology is unknown choose a random cpu that is hopefully
1350          * idle.
1351          */
1352         if (dd->cpunode == NULL) {
1353                 return (NULL);
1354         }
1355
1356         /*
1357          * When the topology is known choose a cpu whos group has, in
1358          * aggregate, has the lowest weighted load.
1359          */
1360         cpup = root_cpu_node;
1361         rdd = dd;
1362         level = cpu_topology_levels_number;
1363         while (cpup) {
1364                 /*
1365                  * Degenerate case super-root
1366                  */
1367                 if (cpup->child_node && cpup->child_no == 1) {
1368                         cpup = cpup->child_node;
1369                         --level;
1370                         continue;
1371                 }
1372
1373                 /*
1374                  * Terminal cpunode
1375                  */
1376                 if (cpup->child_node == NULL) {
1377                         rdd = &dfly_pcpu[BSFCPUMASK(cpup->members)];
1378                         break;
1379                 }
1380
1381                 cpub = NULL;
1382                 highest_load = 0;
1383
1384                 for (n = 0; n < cpup->child_no; ++n) {
1385                         /*
1386                          * Accumulate load information for all cpus
1387                          * which are members of this node.
1388                          */
1389                         cpun = &cpup->child_node[n];
1390                         mask = cpun->members & usched_global_cpumask &
1391                                smp_active_mask;
1392                         if (mask == 0)
1393                                 continue;
1394                         load = 0;
1395                         uloadok = 0;
1396                         while (mask) {
1397                                 cpuid = BSFCPUMASK(mask);
1398                                 load += dfly_pcpu[cpuid].uload;
1399                                 if (dfly_pcpu[cpuid].uload)
1400                                         uloadok = 1;
1401                                 mask &= ~CPUMASK(cpuid);
1402                         }
1403
1404                         /*
1405                          * Give a slight advantage to nearby cpus.
1406                          */
1407                         if (cpun->members & dd->cpumask)
1408                                 load += PPQ * level;
1409
1410                         /*
1411                          * The best candidate is the one with the worst
1412                          * (highest) load.  Prefer candiates that are
1413                          * closer to our cpu.
1414                          */
1415                         if (uloadok &&
1416                             (cpub == NULL || highest_load < load ||
1417                              (highest_load == load &&
1418                               (cpun->members & dd->cpumask)))
1419                         ) {
1420                                 highest_load = load;
1421                                 cpub = cpun;
1422                         }
1423                 }
1424                 cpup = cpub;
1425                 --level;
1426         }
1427         return (rdd);
1428 }
1429
1430 static
1431 dfly_pcpu_t
1432 dfly_choose_queue_simple(dfly_pcpu_t dd, struct lwp *lp)
1433 {
1434         dfly_pcpu_t rdd;
1435         cpumask_t tmpmask;
1436         cpumask_t mask;
1437         int cpuid;
1438
1439         /*
1440          * Fallback to the original heuristic, select random cpu,
1441          * first checking cpus not currently running a user thread.
1442          */
1443         cpuid = (dfly_scancpu & 0xFFFF) % ncpus;
1444         mask = ~dfly_curprocmask & dfly_rdyprocmask & lp->lwp_cpumask &
1445                smp_active_mask & usched_global_cpumask;
1446
1447         while (mask) {
1448                 tmpmask = ~(CPUMASK(cpuid) - 1);
1449                 if (mask & tmpmask)
1450                         cpuid = BSFCPUMASK(mask & tmpmask);
1451                 else
1452                         cpuid = BSFCPUMASK(mask);
1453                 rdd = &dfly_pcpu[cpuid];
1454
1455                 if ((rdd->upri & ~PPQMASK) >= (lp->lwp_priority & ~PPQMASK))
1456                         goto found;
1457                 mask &= ~CPUMASK(cpuid);
1458         }
1459
1460         /*
1461          * Then cpus which might have a currently running lp
1462          */
1463         cpuid = (dfly_scancpu & 0xFFFF) % ncpus;
1464         mask = dfly_curprocmask & dfly_rdyprocmask &
1465                lp->lwp_cpumask & smp_active_mask & usched_global_cpumask;
1466
1467         while (mask) {
1468                 tmpmask = ~(CPUMASK(cpuid) - 1);
1469                 if (mask & tmpmask)
1470                         cpuid = BSFCPUMASK(mask & tmpmask);
1471                 else
1472                         cpuid = BSFCPUMASK(mask);
1473                 rdd = &dfly_pcpu[cpuid];
1474
1475                 if ((rdd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK))
1476                         goto found;
1477                 mask &= ~CPUMASK(cpuid);
1478         }
1479
1480         /*
1481          * If we cannot find a suitable cpu we reload from dfly_scancpu
1482          * and round-robin.  Other cpus will pickup as they release their
1483          * current lwps or become ready.
1484          *
1485          * Avoid a degenerate system lockup case if usched_global_cpumask
1486          * is set to 0 or otherwise does not cover lwp_cpumask.
1487          *
1488          * We only kick the target helper thread in this case, we do not
1489          * set the user resched flag because
1490          */
1491         cpuid = (dfly_scancpu & 0xFFFF) % ncpus;
1492         if ((CPUMASK(cpuid) & usched_global_cpumask) == 0)
1493                 cpuid = 0;
1494         rdd = &dfly_pcpu[cpuid];
1495 found:
1496         return (rdd);
1497 }
1498
1499 static
1500 void
1501 dfly_need_user_resched_remote(void *dummy)
1502 {
1503         globaldata_t gd = mycpu;
1504         dfly_pcpu_t  dd = &dfly_pcpu[gd->gd_cpuid];
1505
1506         need_user_resched();
1507
1508         /* Call wakeup_mycpu to avoid sending IPIs to other CPUs */
1509         wakeup_mycpu(&dd->helper_thread);
1510 }
1511
1512 #endif
1513
1514 /*
1515  * dfly_remrunqueue_locked() removes a given process from the run queue
1516  * that it is on, clearing the queue busy bit if it becomes empty.
1517  *
1518  * Note that user process scheduler is different from the LWKT schedule.
1519  * The user process scheduler only manages user processes but it uses LWKT
1520  * underneath, and a user process operating in the kernel will often be
1521  * 'released' from our management.
1522  *
1523  * uload is NOT adjusted here.  It is only adjusted if the lwkt_thread goes
1524  * to sleep or the lwp is moved to a different runq.
1525  */
1526 static void
1527 dfly_remrunqueue_locked(dfly_pcpu_t rdd, struct lwp *lp)
1528 {
1529         struct rq *q;
1530         u_int32_t *which;
1531         u_int8_t pri;
1532
1533         KKASSERT(lp->lwp_mpflags & LWP_MP_ONRUNQ);
1534         atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ);
1535         --rdd->runqcount;
1536         /*rdd->uload -= (lp->lwp_priority & ~PPQMASK) & PRIMASK;*/
1537         KKASSERT(rdd->runqcount >= 0);
1538
1539         pri = lp->lwp_rqindex;
1540         switch(lp->lwp_rqtype) {
1541         case RTP_PRIO_NORMAL:
1542                 q = &rdd->queues[pri];
1543                 which = &rdd->queuebits;
1544                 break;
1545         case RTP_PRIO_REALTIME:
1546         case RTP_PRIO_FIFO:
1547                 q = &rdd->rtqueues[pri];
1548                 which = &rdd->rtqueuebits;
1549                 break;
1550         case RTP_PRIO_IDLE:
1551                 q = &rdd->idqueues[pri];
1552                 which = &rdd->idqueuebits;
1553                 break;
1554         default:
1555                 panic("remrunqueue: invalid rtprio type");
1556                 /* NOT REACHED */
1557         }
1558         TAILQ_REMOVE(q, lp, lwp_procq);
1559         if (TAILQ_EMPTY(q)) {
1560                 KASSERT((*which & (1 << pri)) != 0,
1561                         ("remrunqueue: remove from empty queue"));
1562                 *which &= ~(1 << pri);
1563         }
1564 }
1565
1566 /*
1567  * dfly_setrunqueue_locked()
1568  *
1569  * Add a process whos rqtype and rqindex had previously been calculated
1570  * onto the appropriate run queue.   Determine if the addition requires
1571  * a reschedule on a cpu and return the cpuid or -1.
1572  *
1573  * NOTE:          Lower priorities are better priorities.
1574  *
1575  * NOTE ON ULOAD: This variable specifies the aggregate load on a cpu, the
1576  *                sum of the rough lwp_priority for all running and runnable
1577  *                processes.  Lower priority processes (higher lwp_priority
1578  *                values) actually DO count as more load, not less, because
1579  *                these are the programs which require the most care with
1580  *                regards to cpu selection.
1581  */
1582 static void
1583 dfly_setrunqueue_locked(dfly_pcpu_t rdd, struct lwp *lp)
1584 {
1585         struct rq *q;
1586         u_int32_t *which;
1587         int pri;
1588
1589         if (lp->lwp_qcpu != rdd->cpuid) {
1590                 if (lp->lwp_mpflags & LWP_MP_ULOAD) {
1591                         atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ULOAD);
1592                         atomic_add_int(&dfly_pcpu[lp->lwp_qcpu].uload,
1593                                    -((lp->lwp_priority & ~PPQMASK) & PRIMASK));
1594                 }
1595                 lp->lwp_qcpu = rdd->cpuid;
1596         }
1597
1598         KKASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0);
1599         atomic_set_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ);
1600         ++rdd->runqcount;
1601         if ((lp->lwp_mpflags & LWP_MP_ULOAD) == 0) {
1602                 atomic_set_int(&lp->lwp_mpflags, LWP_MP_ULOAD);
1603                 atomic_add_int(&dfly_pcpu[lp->lwp_qcpu].uload,
1604                                (lp->lwp_priority & ~PPQMASK) & PRIMASK);
1605         }
1606
1607         pri = lp->lwp_rqindex;
1608
1609         switch(lp->lwp_rqtype) {
1610         case RTP_PRIO_NORMAL:
1611                 q = &rdd->queues[pri];
1612                 which = &rdd->queuebits;
1613                 break;
1614         case RTP_PRIO_REALTIME:
1615         case RTP_PRIO_FIFO:
1616                 q = &rdd->rtqueues[pri];
1617                 which = &rdd->rtqueuebits;
1618                 break;
1619         case RTP_PRIO_IDLE:
1620                 q = &rdd->idqueues[pri];
1621                 which = &rdd->idqueuebits;
1622                 break;
1623         default:
1624                 panic("remrunqueue: invalid rtprio type");
1625                 /* NOT REACHED */
1626         }
1627
1628         /*
1629          * Add to the correct queue and set the appropriate bit.  If no
1630          * lower priority (i.e. better) processes are in the queue then
1631          * we want a reschedule, calculate the best cpu for the job.
1632          *
1633          * Always run reschedules on the LWPs original cpu.
1634          */
1635         TAILQ_INSERT_TAIL(q, lp, lwp_procq);
1636         *which |= 1 << pri;
1637 }
1638
1639 #ifdef SMP
1640
1641 /*
1642  * For SMP systems a user scheduler helper thread is created for each
1643  * cpu and is used to allow one cpu to wakeup another for the purposes of
1644  * scheduling userland threads from setrunqueue().
1645  *
1646  * UP systems do not need the helper since there is only one cpu.
1647  *
1648  * We can't use the idle thread for this because we might block.
1649  * Additionally, doing things this way allows us to HLT idle cpus
1650  * on MP systems.
1651  */
1652 static void
1653 dfly_helper_thread(void *dummy)
1654 {
1655     globaldata_t gd;
1656     dfly_pcpu_t  dd;
1657     struct lwp *nlp;
1658     cpumask_t mask;
1659     int cpuid;
1660
1661     gd = mycpu;
1662     cpuid = gd->gd_cpuid;       /* doesn't change */
1663     mask = gd->gd_cpumask;      /* doesn't change */
1664     dd = &dfly_pcpu[cpuid];
1665
1666     /*
1667      * Since we only want to be woken up only when no user processes
1668      * are scheduled on a cpu, run at an ultra low priority.
1669      */
1670     lwkt_setpri_self(TDPRI_USER_SCHEDULER);
1671
1672     tsleep(&dd->helper_thread, 0, "schslp", 0);
1673
1674     for (;;) {
1675         /*
1676          * We use the LWKT deschedule-interlock trick to avoid racing
1677          * dfly_rdyprocmask.  This means we cannot block through to the
1678          * manual lwkt_switch() call we make below.
1679          */
1680         crit_enter_gd(gd);
1681         tsleep_interlock(&dd->helper_thread, 0);
1682
1683         /*spin_lock(&dfly_spin);*/
1684         spin_lock(&dd->spin);
1685
1686         atomic_set_cpumask(&dfly_rdyprocmask, mask);
1687         clear_user_resched();   /* This satisfied the reschedule request */
1688         dd->rrcount = 0;        /* Reset the round-robin counter */
1689
1690         if ((dfly_curprocmask & mask) == 0) {
1691                 /*
1692                  * No thread is currently scheduled.
1693                  */
1694                 KKASSERT(dd->uschedcp == NULL);
1695                 if ((nlp = dfly_chooseproc_locked(dd, NULL, 0)) != NULL) {
1696                         KTR_COND_LOG(usched_sched_thread_no_process,
1697                             nlp->lwp_proc->p_pid == usched_dfly_pid_debug,
1698                             gd->gd_cpuid,
1699                             nlp->lwp_proc->p_pid,
1700                             nlp->lwp_thread->td_gd->gd_cpuid);
1701
1702                         atomic_set_cpumask(&dfly_curprocmask, mask);
1703                         dd->upri = nlp->lwp_priority;
1704                         dd->uschedcp = nlp;
1705                         dd->rrcount = 0;        /* reset round robin */
1706                         spin_unlock(&dd->spin);
1707                         /*spin_unlock(&dfly_spin);*/
1708                         lwkt_acquire(nlp->lwp_thread);
1709                         lwkt_schedule(nlp->lwp_thread);
1710                 } else {
1711                         spin_unlock(&dd->spin);
1712                         /*spin_unlock(&dfly_spin);*/
1713                 }
1714         } else if (dd->runqcount) {
1715                 /*
1716                  * Possibly find a better process to schedule.
1717                  */
1718                 nlp = dfly_chooseproc_locked(dd, dd->uschedcp, 0);
1719                 if (nlp) {
1720                         KTR_COND_LOG(usched_sched_thread_process,
1721                             nlp->lwp_proc->p_pid == usched_dfly_pid_debug,
1722                             gd->gd_cpuid,
1723                             nlp->lwp_proc->p_pid,
1724                             nlp->lwp_thread->td_gd->gd_cpuid);
1725
1726                         dd->upri = nlp->lwp_priority;
1727                         dd->uschedcp = nlp;
1728                         dd->rrcount = 0;        /* reset round robin */
1729                         spin_unlock(&dd->spin);
1730                         /*spin_unlock(&dfly_spin);*/
1731                         lwkt_acquire(nlp->lwp_thread);
1732                         lwkt_schedule(nlp->lwp_thread);
1733                 } else {
1734                         /*
1735                          * Leave the thread on our run queue.  Another
1736                          * scheduler will try to pull it later.
1737                          */
1738                         spin_unlock(&dd->spin);
1739                         /*spin_unlock(&dfly_spin);*/
1740                 }
1741         } else {
1742                 /*
1743                  * The runq is empty.
1744                  */
1745                 spin_unlock(&dd->spin);
1746                 /*spin_unlock(&dfly_spin);*/
1747         }
1748
1749         /*
1750          * We're descheduled unless someone scheduled us.  Switch away.
1751          * Exiting the critical section will cause splz() to be called
1752          * for us if interrupts and such are pending.
1753          */
1754         crit_exit_gd(gd);
1755         tsleep(&dd->helper_thread, PINTERLOCKED, "schslp", 0);
1756     }
1757 }
1758
1759 /* sysctl stick_to_level parameter */
1760 static int
1761 sysctl_usched_dfly_stick_to_level(SYSCTL_HANDLER_ARGS)
1762 {
1763         int error, new_val;
1764
1765         new_val = usched_dfly_stick_to_level;
1766
1767         error = sysctl_handle_int(oidp, &new_val, 0, req);
1768         if (error != 0 || req->newptr == NULL)
1769                 return (error);
1770         if (new_val > cpu_topology_levels_number - 1 || new_val < 0)
1771                 return (EINVAL);
1772         usched_dfly_stick_to_level = new_val;
1773         return (0);
1774 }
1775
1776 /*
1777  * Setup our scheduler helpers.  Note that curprocmask bit 0 has already
1778  * been cleared by rqinit() and we should not mess with it further.
1779  */
1780 static void
1781 dfly_helper_thread_cpu_init(void)
1782 {
1783         int i;
1784         int j;
1785         int cpuid;
1786         int smt_not_supported = 0;
1787         int cache_coherent_not_supported = 0;
1788
1789         if (bootverbose)
1790                 kprintf("Start scheduler helpers on cpus:\n");
1791
1792         sysctl_ctx_init(&usched_dfly_sysctl_ctx);
1793         usched_dfly_sysctl_tree =
1794                 SYSCTL_ADD_NODE(&usched_dfly_sysctl_ctx,
1795                                 SYSCTL_STATIC_CHILDREN(_kern), OID_AUTO,
1796                                 "usched_dfly", CTLFLAG_RD, 0, "");
1797
1798         for (i = 0; i < ncpus; ++i) {
1799                 dfly_pcpu_t dd = &dfly_pcpu[i];
1800                 cpumask_t mask = CPUMASK(i);
1801
1802                 if ((mask & smp_active_mask) == 0)
1803                     continue;
1804
1805                 spin_init(&dd->spin);
1806                 dd->cpunode = get_cpu_node_by_cpuid(i);
1807                 dd->cpuid = i;
1808                 dd->cpumask = CPUMASK(i);
1809                 for (j = 0; j < NQS; j++) {
1810                         TAILQ_INIT(&dd->queues[j]);
1811                         TAILQ_INIT(&dd->rtqueues[j]);
1812                         TAILQ_INIT(&dd->idqueues[j]);
1813                 }
1814                 atomic_clear_cpumask(&dfly_curprocmask, 1);
1815
1816                 if (dd->cpunode == NULL) {
1817                         smt_not_supported = 1;
1818                         cache_coherent_not_supported = 1;
1819                         if (bootverbose)
1820                                 kprintf ("\tcpu%d - WARNING: No CPU NODE "
1821                                          "found for cpu\n", i);
1822                 } else {
1823                         switch (dd->cpunode->type) {
1824                         case THREAD_LEVEL:
1825                                 if (bootverbose)
1826                                         kprintf ("\tcpu%d - HyperThreading "
1827                                                  "available. Core siblings: ",
1828                                                  i);
1829                                 break;
1830                         case CORE_LEVEL:
1831                                 smt_not_supported = 1;
1832
1833                                 if (bootverbose)
1834                                         kprintf ("\tcpu%d - No HT available, "
1835                                                  "multi-core/physical "
1836                                                  "cpu. Physical siblings: ",
1837                                                  i);
1838                                 break;
1839                         case CHIP_LEVEL:
1840                                 smt_not_supported = 1;
1841
1842                                 if (bootverbose)
1843                                         kprintf ("\tcpu%d - No HT available, "
1844                                                  "single-core/physical cpu. "
1845                                                  "Package Siblings: ",
1846                                                  i);
1847                                 break;
1848                         default:
1849                                 /* Let's go for safe defaults here */
1850                                 smt_not_supported = 1;
1851                                 cache_coherent_not_supported = 1;
1852                                 if (bootverbose)
1853                                         kprintf ("\tcpu%d - Unknown cpunode->"
1854                                                  "type=%u. Siblings: ",
1855                                                  i,
1856                                                  (u_int)dd->cpunode->type);
1857                                 break;
1858                         }
1859
1860                         if (bootverbose) {
1861                                 if (dd->cpunode->parent_node != NULL) {
1862                                         CPUSET_FOREACH(cpuid, dd->cpunode->parent_node->members)
1863                                                 kprintf("cpu%d ", cpuid);
1864                                         kprintf("\n");
1865                                 } else {
1866                                         kprintf(" no siblings\n");
1867                                 }
1868                         }
1869                 }
1870
1871                 lwkt_create(dfly_helper_thread, NULL, NULL, &dd->helper_thread,
1872                             0, i, "usched %d", i);
1873
1874                 /*
1875                  * Allow user scheduling on the target cpu.  cpu #0 has already
1876                  * been enabled in rqinit().
1877                  */
1878                 if (i)
1879                     atomic_clear_cpumask(&dfly_curprocmask, mask);
1880                 atomic_set_cpumask(&dfly_rdyprocmask, mask);
1881                 dd->upri = PRIBASE_NULL;
1882
1883         }
1884
1885         /* usched_dfly sysctl configurable parameters */
1886
1887         SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1888                        SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1889                        OID_AUTO, "rrinterval", CTLFLAG_RW,
1890                        &usched_dfly_rrinterval, 0, "");
1891         SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1892                        SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1893                        OID_AUTO, "decay", CTLFLAG_RW,
1894                        &usched_dfly_decay, 0, "Extra decay when not running");
1895         SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1896                        SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1897                        OID_AUTO, "batch_time", CTLFLAG_RW,
1898                        &usched_dfly_batch_time, 0, "Min batch counter value");
1899
1900         /* Add enable/disable option for SMT scheduling if supported */
1901         if (smt_not_supported) {
1902                 usched_dfly_smt = 0;
1903                 SYSCTL_ADD_STRING(&usched_dfly_sysctl_ctx,
1904                                   SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1905                                   OID_AUTO, "smt", CTLFLAG_RD,
1906                                   "NOT SUPPORTED", 0, "SMT NOT SUPPORTED");
1907         } else {
1908                 usched_dfly_smt = 1;
1909                 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1910                                SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1911                                OID_AUTO, "smt", CTLFLAG_RW,
1912                                &usched_dfly_smt, 0, "Enable SMT scheduling");
1913         }
1914
1915         /*
1916          * Add enable/disable option for cache coherent scheduling
1917          * if supported
1918          */
1919         if (cache_coherent_not_supported) {
1920                 usched_dfly_cache_coherent = 0;
1921                 SYSCTL_ADD_STRING(&usched_dfly_sysctl_ctx,
1922                                   SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1923                                   OID_AUTO, "cache_coherent", CTLFLAG_RD,
1924                                   "NOT SUPPORTED", 0,
1925                                   "Cache coherence NOT SUPPORTED");
1926         } else {
1927                 usched_dfly_cache_coherent = 1;
1928                 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1929                                SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1930                                OID_AUTO, "cache_coherent", CTLFLAG_RW,
1931                                &usched_dfly_cache_coherent, 0,
1932                                "Enable/Disable cache coherent scheduling");
1933
1934                 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1935                                SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1936                                OID_AUTO, "weight1", CTLFLAG_RW,
1937                                &usched_dfly_weight1, 10,
1938                                "Weight selection for current cpu");
1939
1940                 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1941                                SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1942                                OID_AUTO, "weight2", CTLFLAG_RW,
1943                                &usched_dfly_weight2, 5,
1944                                "Weight selection for wakefrom cpu");
1945
1946                 SYSCTL_ADD_PROC(&usched_dfly_sysctl_ctx,
1947                                 SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1948                                 OID_AUTO, "stick_to_level",
1949                                 CTLTYPE_INT | CTLFLAG_RW,
1950                                 NULL, sizeof usched_dfly_stick_to_level,
1951                                 sysctl_usched_dfly_stick_to_level, "I",
1952                                 "Stick a process to this level. See sysctl"
1953                                 "paremter hw.cpu_topology.level_description");
1954         }
1955 }
1956 SYSINIT(uschedtd, SI_BOOT2_USCHED, SI_ORDER_SECOND,
1957         dfly_helper_thread_cpu_init, NULL)
1958
1959 #else /* No SMP options - just add the configurable parameters to sysctl */
1960
1961 static void
1962 sched_sysctl_tree_init(void)
1963 {
1964         sysctl_ctx_init(&usched_dfly_sysctl_ctx);
1965         usched_dfly_sysctl_tree =
1966                 SYSCTL_ADD_NODE(&usched_dfly_sysctl_ctx,
1967                                 SYSCTL_STATIC_CHILDREN(_kern), OID_AUTO,
1968                                 "usched_dfly", CTLFLAG_RD, 0, "");
1969
1970         /* usched_dfly sysctl configurable parameters */
1971         SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1972                        SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1973                        OID_AUTO, "rrinterval", CTLFLAG_RW,
1974                        &usched_dfly_rrinterval, 0, "");
1975         SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1976                        SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1977                        OID_AUTO, "decay", CTLFLAG_RW,
1978                        &usched_dfly_decay, 0, "Extra decay when not running");
1979         SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1980                        SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1981                        OID_AUTO, "batch_time", CTLFLAG_RW,
1982                        &usched_dfly_batch_time, 0, "Min batch counter value");
1983 }
1984 SYSINIT(uschedtd, SI_BOOT2_USCHED, SI_ORDER_SECOND,
1985         sched_sysctl_tree_init, NULL)
1986 #endif