sys/kern/usched_dfly.c

   1 /*
   2  * Copyright (c) 2012 The DragonFly Project.  All rights reserved.
   3  * Copyright (c) 1999 Peter Wemm <peter@FreeBSD.org>.  All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Matthew Dillon <dillon@backplane.com>,
   7  * by Mihai Carabas <mihai.carabas@gmail.com>
   8  * and many others.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  *
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in
  18  *    the documentation and/or other materials provided with the
  19  *    distribution.
  20  * 3. Neither the name of The DragonFly Project nor the names of its
  21  *    contributors may be used to endorse or promote products derived
  22  *    from this software without specific, prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  27  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  28  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  29  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  30  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  32  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  33  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  34  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35  * SUCH DAMAGE.
  36  */
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/kernel.h>
  40 #include <sys/lock.h>
  41 #include <sys/queue.h>
  42 #include <sys/proc.h>
  43 #include <sys/rtprio.h>
  44 #include <sys/uio.h>
  45 #include <sys/sysctl.h>
  46 #include <sys/resourcevar.h>
  47 #include <sys/spinlock.h>
  48 #include <sys/cpu_topology.h>
  49 #include <sys/thread2.h>
  50 #include <sys/spinlock2.h>
  51 #include <sys/mplock2.h>
  52
  53 #include <sys/ktr.h>
  54
  55 #include <machine/cpu.h>
  56 #include <machine/smp.h>
  57
  58 /*
  59  * Priorities.  Note that with 32 run queues per scheduler each queue
  60  * represents four priority levels.
  61  */
  62
  63 int dfly_rebalanced;
  64
  65 #define MAXPRI                  128
  66 #define PRIMASK                 (MAXPRI - 1)
  67 #define PRIBASE_REALTIME        0
  68 #define PRIBASE_NORMAL          MAXPRI
  69 #define PRIBASE_IDLE            (MAXPRI * 2)
  70 #define PRIBASE_THREAD          (MAXPRI * 3)
  71 #define PRIBASE_NULL            (MAXPRI * 4)
  72
  73 #define NQS     32                      /* 32 run queues. */
  74 #define PPQ     (MAXPRI / NQS)          /* priorities per queue */
  75 #define PPQMASK (PPQ - 1)
  76
  77 /*
  78  * NICEPPQ      - number of nice units per priority queue
  79  * ESTCPUPPQ    - number of estcpu units per priority queue
  80  * ESTCPUMAX    - number of estcpu units
  81  */
  82 #define NICEPPQ         2
  83 #define ESTCPUPPQ       512
  84 #define ESTCPUMAX       (ESTCPUPPQ * NQS)
  85 #define BATCHMAX        (ESTCPUFREQ * 30)
  86 #define PRIO_RANGE      (PRIO_MAX - PRIO_MIN + 1)
  87
  88 #define ESTCPULIM(v)    min((v), ESTCPUMAX)
  89
  90 TAILQ_HEAD(rq, lwp);
  91
  92 #define lwp_priority    lwp_usdata.dfly.priority
  93 #define lwp_rqindex     lwp_usdata.dfly.rqindex
  94 #define lwp_estcpu      lwp_usdata.dfly.estcpu
  95 #define lwp_batch       lwp_usdata.dfly.batch
  96 #define lwp_rqtype      lwp_usdata.dfly.rqtype
  97 #define lwp_qcpu        lwp_usdata.dfly.qcpu
  98
  99 struct usched_dfly_pcpu {
 100         struct spinlock spin;
 101         struct thread   helper_thread;
 102         short           rrcount;
 103         short           upri;
 104         int             uload;
 105         struct lwp      *uschedcp;
 106         struct rq       queues[NQS];
 107         struct rq       rtqueues[NQS];
 108         struct rq       idqueues[NQS];
 109         u_int32_t       queuebits;
 110         u_int32_t       rtqueuebits;
 111         u_int32_t       idqueuebits;
 112         int             runqcount;
 113         int             cpuid;
 114         cpumask_t       cpumask;
 115 #ifdef SMP
 116         cpu_node_t      *cpunode;
 117 #endif
 118 };
 119
 120 typedef struct usched_dfly_pcpu *dfly_pcpu_t;
 121
 122 static void dfly_acquire_curproc(struct lwp *lp);
 123 static void dfly_release_curproc(struct lwp *lp);
 124 static void dfly_select_curproc(globaldata_t gd);
 125 static void dfly_setrunqueue(struct lwp *lp);
 126 static void dfly_schedulerclock(struct lwp *lp, sysclock_t period,
 127                                 sysclock_t cpstamp);
 128 static void dfly_recalculate_estcpu(struct lwp *lp);
 129 static void dfly_resetpriority(struct lwp *lp);
 130 static void dfly_forking(struct lwp *plp, struct lwp *lp);
 131 static void dfly_exiting(struct lwp *lp, struct proc *);
 132 static void dfly_uload_update(struct lwp *lp);
 133 static void dfly_yield(struct lwp *lp);
 134 #ifdef SMP
 135 static dfly_pcpu_t dfly_choose_best_queue(dfly_pcpu_t dd, struct lwp *lp);
 136 static dfly_pcpu_t dfly_choose_worst_queue(dfly_pcpu_t dd);
 137 static dfly_pcpu_t dfly_choose_queue_simple(dfly_pcpu_t dd, struct lwp *lp);
 138 #endif
 139
 140 #ifdef SMP
 141 static void dfly_need_user_resched_remote(void *dummy);
 142 #endif
 143 static struct lwp *dfly_chooseproc_locked(dfly_pcpu_t dd, struct lwp *chklp,
 144                                         int isremote);
 145 static void dfly_remrunqueue_locked(dfly_pcpu_t dd, struct lwp *lp);
 146 static void dfly_setrunqueue_locked(dfly_pcpu_t dd, struct lwp *lp);
 147
 148 struct usched usched_dfly = {
 149         { NULL },
 150         "dfly", "Original DragonFly Scheduler",
 151         NULL,                   /* default registration */
 152         NULL,                   /* default deregistration */
 153         dfly_acquire_curproc,
 154         dfly_release_curproc,
 155         dfly_setrunqueue,
 156         dfly_schedulerclock,
 157         dfly_recalculate_estcpu,
 158         dfly_resetpriority,
 159         dfly_forking,
 160         dfly_exiting,
 161         dfly_uload_update,
 162         NULL,                   /* setcpumask not supported */
 163         dfly_yield
 164 };
 165
 166 /*
 167  * We have NQS (32) run queues per scheduling class.  For the normal
 168  * class, there are 128 priorities scaled onto these 32 queues.  New
 169  * processes are added to the last entry in each queue, and processes
 170  * are selected for running by taking them from the head and maintaining
 171  * a simple FIFO arrangement.  Realtime and Idle priority processes have
 172  * and explicit 0-31 priority which maps directly onto their class queue
 173  * index.  When a queue has something in it, the corresponding bit is
 174  * set in the queuebits variable, allowing a single read to determine
 175  * the state of all 32 queues and then a ffs() to find the first busy
 176  * queue.
 177  */
 178 static cpumask_t dfly_curprocmask = -1; /* currently running a user process */
 179 static cpumask_t dfly_rdyprocmask;      /* ready to accept a user process */
 180 #ifdef SMP
 181 static volatile int dfly_scancpu;
 182 /*static struct spinlock dfly_spin = SPINLOCK_INITIALIZER(dfly_spin);*/
 183 #endif
 184 static struct usched_dfly_pcpu dfly_pcpu[MAXCPU];
 185 static struct sysctl_ctx_list usched_dfly_sysctl_ctx;
 186 static struct sysctl_oid *usched_dfly_sysctl_tree;
 187
 188 /* Debug info exposed through debug.* sysctl */
 189
 190 static int usched_dfly_debug = -1;
 191 SYSCTL_INT(_debug, OID_AUTO, dfly_scdebug, CTLFLAG_RW,
 192            &usched_dfly_debug, 0,
 193            "Print debug information for this pid");
 194
 195 static int usched_dfly_pid_debug = -1;
 196 SYSCTL_INT(_debug, OID_AUTO, dfly_pid_debug, CTLFLAG_RW,
 197            &usched_dfly_pid_debug, 0,
 198            "Print KTR debug information for this pid");
 199
 200 static int usched_dfly_chooser = 0;
 201 SYSCTL_INT(_debug, OID_AUTO, dfly_chooser, CTLFLAG_RW,
 202            &usched_dfly_chooser, 0,
 203            "Print KTR debug information for this pid");
 204
 205 /* Tunning usched_dfly - configurable through kern.usched_dfly.* */
 206 #ifdef SMP
 207 static int usched_dfly_smt = 0;
 208 static int usched_dfly_cache_coherent = 0;
 209 static int usched_dfly_upri_affinity = 16; /* 32 queues - half-way */
 210 static int usched_dfly_queue_checks = 5;
 211 static int usched_dfly_stick_to_level = 0;
 212 #endif
 213 static int usched_dfly_rrinterval = (ESTCPUFREQ + 9) / 10;
 214 static int usched_dfly_decay = 8;
 215 static int usched_dfly_batch_time = 10;
 216
 217 /* KTR debug printings */
 218
 219 KTR_INFO_MASTER(usched);
 220
 221 #if !defined(KTR_USCHED_DFLY)
 222 #define KTR_USCHED_DFLY KTR_ALL
 223 #endif
 224
 225 #if 0
 226 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_acquire_curproc_urw, 0,
 227     "USCHED_DFLY(dfly_acquire_curproc in user_reseched_wanted "
 228     "after release: pid %d, cpuid %d, curr_cpuid %d)",
 229     pid_t pid, int cpuid, int curr);
 230 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_acquire_curproc_before_loop, 0,
 231     "USCHED_DFLY(dfly_acquire_curproc before loop: pid %d, cpuid %d, "
 232     "curr_cpuid %d)",
 233     pid_t pid, int cpuid, int curr);
 234 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_acquire_curproc_not, 0,
 235     "USCHED_DFLY(dfly_acquire_curproc couldn't acquire after "
 236     "dfly_setrunqueue: pid %d, cpuid %d, curr_lp pid %d, curr_cpuid %d)",
 237     pid_t pid, int cpuid, pid_t curr_pid, int curr_cpuid);
 238 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_acquire_curproc_switch, 0,
 239     "USCHED_DFLY(dfly_acquire_curproc after lwkt_switch: pid %d, "
 240     "cpuid %d, curr_cpuid %d)",
 241     pid_t pid, int cpuid, int curr);
 242
 243 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_release_curproc, 0,
 244     "USCHED_DFLY(dfly_release_curproc before select: pid %d, "
 245     "cpuid %d, curr_cpuid %d)",
 246     pid_t pid, int cpuid, int curr);
 247
 248 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_select_curproc, 0,
 249     "USCHED_DFLY(dfly_release_curproc before select: pid %d, "
 250     "cpuid %d, old_pid %d, old_cpuid %d, curr_cpuid %d)",
 251     pid_t pid, int cpuid, pid_t old_pid, int old_cpuid, int curr);
 252
 253 #ifdef SMP
 254 KTR_INFO(KTR_USCHED_DFLY, usched, batchy_test_false, 0,
 255     "USCHED_DFLY(batchy_looser_pri_test false: pid %d, "
 256     "cpuid %d, verify_mask %lu)",
 257     pid_t pid, int cpuid, cpumask_t mask);
 258 KTR_INFO(KTR_USCHED_DFLY, usched, batchy_test_true, 0,
 259     "USCHED_DFLY(batchy_looser_pri_test true: pid %d, "
 260     "cpuid %d, verify_mask %lu)",
 261     pid_t pid, int cpuid, cpumask_t mask);
 262
 263 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_setrunqueue_fc_smt, 0,
 264     "USCHED_DFLY(dfly_setrunqueue free cpus smt: pid %d, cpuid %d, "
 265     "mask %lu, curr_cpuid %d)",
 266     pid_t pid, int cpuid, cpumask_t mask, int curr);
 267 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_setrunqueue_fc_non_smt, 0,
 268     "USCHED_DFLY(dfly_setrunqueue free cpus check non_smt: pid %d, "
 269     "cpuid %d, mask %lu, curr_cpuid %d)",
 270     pid_t pid, int cpuid, cpumask_t mask, int curr);
 271 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_setrunqueue_rc, 0,
 272     "USCHED_DFLY(dfly_setrunqueue running cpus check: pid %d, "
 273     "cpuid %d, mask %lu, curr_cpuid %d)",
 274     pid_t pid, int cpuid, cpumask_t mask, int curr);
 275 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_setrunqueue_found, 0,
 276     "USCHED_DFLY(dfly_setrunqueue found cpu: pid %d, cpuid %d, "
 277     "mask %lu, found_cpuid %d, curr_cpuid %d)",
 278     pid_t pid, int cpuid, cpumask_t mask, int found_cpuid, int curr);
 279 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_setrunqueue_not_found, 0,
 280     "USCHED_DFLY(dfly_setrunqueue not found cpu: pid %d, cpuid %d, "
 281     "try_cpuid %d, curr_cpuid %d)",
 282     pid_t pid, int cpuid, int try_cpuid, int curr);
 283 KTR_INFO(KTR_USCHED_DFLY, usched, dfly_setrunqueue_found_best_cpuid, 0,
 284     "USCHED_DFLY(dfly_setrunqueue found cpu: pid %d, cpuid %d, "
 285     "mask %lu, found_cpuid %d, curr_cpuid %d)",
 286     pid_t pid, int cpuid, cpumask_t mask, int found_cpuid, int curr);
 287 #endif
 288 #endif
 289
 290 KTR_INFO(KTR_USCHED_DFLY, usched, chooseproc, 0,
 291     "USCHED_DFLY(chooseproc: pid %d, old_cpuid %d, curr_cpuid %d)",
 292     pid_t pid, int old_cpuid, int curr);
 293 #ifdef SMP
 294 #if 0
 295 KTR_INFO(KTR_USCHED_DFLY, usched, chooseproc_cc, 0,
 296     "USCHED_DFLY(chooseproc_cc: pid %d, old_cpuid %d, curr_cpuid %d)",
 297     pid_t pid, int old_cpuid, int curr);
 298 KTR_INFO(KTR_USCHED_DFLY, usched, chooseproc_cc_not_good, 0,
 299     "USCHED_DFLY(chooseproc_cc not good: pid %d, old_cpumask %lu, "
 300     "sibling_mask %lu, curr_cpumask %lu)",
 301     pid_t pid, cpumask_t old_cpumask, cpumask_t sibling_mask, cpumask_t curr);
 302 KTR_INFO(KTR_USCHED_DFLY, usched, chooseproc_cc_elected, 0,
 303     "USCHED_DFLY(chooseproc_cc elected: pid %d, old_cpumask %lu, "
 304     "sibling_mask %lu, curr_cpumask: %lu)",
 305     pid_t pid, cpumask_t old_cpumask, cpumask_t sibling_mask, cpumask_t curr);
 306 #endif
 307
 308 KTR_INFO(KTR_USCHED_DFLY, usched, sched_thread_no_process, 0,
 309     "USCHED_DFLY(sched_thread %d no process scheduled: pid %d, old_cpuid %d)",
 310     int id, pid_t pid, int cpuid);
 311 KTR_INFO(KTR_USCHED_DFLY, usched, sched_thread_process, 0,
 312     "USCHED_DFLY(sched_thread %d process scheduled: pid %d, old_cpuid %d)",
 313     int id, pid_t pid, int cpuid);
 314 #if 0
 315 KTR_INFO(KTR_USCHED_DFLY, usched, sched_thread_no_process_found, 0,
 316     "USCHED_DFLY(sched_thread %d no process found; tmpmask %lu)",
 317     int id, cpumask_t tmpmask);
 318 #endif
 319 #endif
 320
 321 /*
 322  * DFLY_ACQUIRE_CURPROC
 323  *
 324  * This function is called when the kernel intends to return to userland.
 325  * It is responsible for making the thread the current designated userland
 326  * thread for this cpu, blocking if necessary.
 327  *
 328  * The kernel has already depressed our LWKT priority so we must not switch
 329  * until we have either assigned or disposed of the thread.
 330  *
 331  * WARNING! THIS FUNCTION IS ALLOWED TO CAUSE THE CURRENT THREAD TO MIGRATE
 332  * TO ANOTHER CPU!  Because most of the kernel assumes that no migration will
 333  * occur, this function is called only under very controlled circumstances.
 334  */
 335 static void
 336 dfly_acquire_curproc(struct lwp *lp)
 337 {
 338         globaldata_t gd;
 339         dfly_pcpu_t dd;
 340         thread_t td;
 341
 342         /*
 343          * Make sure we aren't sitting on a tsleep queue.
 344          */
 345         td = lp->lwp_thread;
 346         crit_enter_quick(td);
 347         if (td->td_flags & TDF_TSLEEPQ)
 348                 tsleep_remove(td);
 349         dfly_recalculate_estcpu(lp);
 350
 351         /*
 352          * If a reschedule was requested give another thread the
 353          * driver's seat.
 354          */
 355         if (user_resched_wanted()) {
 356                 clear_user_resched();
 357                 dfly_release_curproc(lp);
 358         }
 359
 360         /*
 361          * Loop until we are the current user thread
 362          */
 363         gd = mycpu;
 364         dd = &dfly_pcpu[gd->gd_cpuid];
 365
 366         do {
 367                 /*
 368                  * Process any pending events and higher priority threads.
 369                  */
 370                 lwkt_yield();
 371
 372                 /*
 373                  * Become the currently scheduled user thread for this cpu
 374                  * if we can do so trivially.
 375                  *
 376                  * We can steal another thread's current thread designation
 377                  * on this cpu since if we are running that other thread
 378                  * must not be, so we can safely deschedule it.
 379                  */
 380                 if (dd->uschedcp == lp) {
 381                         /*
 382                          * We are already the current lwp (hot path).
 383                          */
 384                         dd->upri = lp->lwp_priority;
 385                 } else if (dd->uschedcp == NULL) {
 386                         /*
 387                          * We can trivially become the current lwp.
 388                          */
 389                         atomic_set_cpumask(&dfly_curprocmask, gd->gd_cpumask);
 390                         dd->uschedcp = lp;
 391                         dd->upri = lp->lwp_priority;
 392                         KKASSERT(lp->lwp_qcpu == dd->cpuid);
 393                 } else if (dd->uschedcp && dd->upri > lp->lwp_priority) {
 394                         /*
 395                          * We can steal the current cpu's lwp designation
 396                          * away simply by replacing it.  The other thread
 397                          * will stall when it tries to return to userland,
 398                          * possibly rescheduling elsewhere when it calls
 399                          * setrunqueue.
 400                          */
 401                         dd->uschedcp = lp;
 402                         dd->upri = lp->lwp_priority;
 403                         KKASSERT(lp->lwp_qcpu == dd->cpuid);
 404                 } else {
 405                         /*
 406                          * We cannot become the current lwp, place the lp
 407                          * on the run-queue of this or another cpu and
 408                          * deschedule ourselves.
 409                          *
 410                          * When we are reactivated we will have another
 411                          * chance.
 412                          */
 413                         lwkt_deschedule(lp->lwp_thread);
 414                         dfly_setrunqueue(lp);
 415
 416                         /*
 417                          * Reload after a switch or setrunqueue/switch possibly
 418                          * moved us to another cpu.
 419                          */
 420                         lwkt_switch();
 421                         gd = mycpu;
 422                         dd = &dfly_pcpu[gd->gd_cpuid];
 423                 }
 424         } while (dd->uschedcp != lp);
 425
 426         crit_exit_quick(td);
 427         KKASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0);
 428 }
 429
 430 /*
 431  * DFLY_RELEASE_CURPROC
 432  *
 433  * This routine detaches the current thread from the userland scheduler,
 434  * usually because the thread needs to run or block in the kernel (at
 435  * kernel priority) for a while.
 436  *
 437  * This routine is also responsible for selecting a new thread to
 438  * make the current thread.
 439  *
 440  * NOTE: This implementation differs from the dummy example in that
 441  * dfly_select_curproc() is able to select the current process, whereas
 442  * dummy_select_curproc() is not able to select the current process.
 443  * This means we have to NULL out uschedcp.
 444  *
 445  * Additionally, note that we may already be on a run queue if releasing
 446  * via the lwkt_switch() in dfly_setrunqueue().
 447  */
 448
 449 static void
 450 dfly_release_curproc(struct lwp *lp)
 451 {
 452         globaldata_t gd = mycpu;
 453         dfly_pcpu_t dd = &dfly_pcpu[gd->gd_cpuid];
 454
 455         if (dd->uschedcp == lp) {
 456                 crit_enter();
 457                 KKASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0);
 458
 459                 dd->uschedcp = NULL;    /* don't let lp be selected */
 460                 dd->upri = PRIBASE_NULL;
 461                 atomic_clear_cpumask(&dfly_curprocmask, gd->gd_cpumask);
 462                 dfly_select_curproc(gd);
 463                 crit_exit();
 464         }
 465 }
 466
 467 /*
 468  * DFLY_SELECT_CURPROC
 469  *
 470  * Select a new current process for this cpu and clear any pending user
 471  * reschedule request.  The cpu currently has no current process.
 472  *
 473  * This routine is also responsible for equal-priority round-robining,
 474  * typically triggered from dfly_schedulerclock().  In our dummy example
 475  * all the 'user' threads are LWKT scheduled all at once and we just
 476  * call lwkt_switch().
 477  *
 478  * The calling process is not on the queue and cannot be selected.
 479  */
 480 static
 481 void
 482 dfly_select_curproc(globaldata_t gd)
 483 {
 484         dfly_pcpu_t dd = &dfly_pcpu[gd->gd_cpuid];
 485         struct lwp *nlp;
 486         int cpuid = gd->gd_cpuid;
 487
 488         crit_enter_gd(gd);
 489
 490         /*spin_lock(&dfly_spin);*/
 491         spin_lock(&dd->spin);
 492         nlp = dfly_chooseproc_locked(dd, dd->uschedcp, 0);
 493
 494         if (nlp) {
 495                 atomic_set_cpumask(&dfly_curprocmask, CPUMASK(cpuid));
 496                 dd->upri = nlp->lwp_priority;
 497                 dd->uschedcp = nlp;
 498                 dd->rrcount = 0;                /* reset round robin */
 499                 spin_unlock(&dd->spin);
 500                 /*spin_unlock(&dfly_spin);*/
 501 #ifdef SMP
 502                 lwkt_acquire(nlp->lwp_thread);
 503 #endif
 504                 lwkt_schedule(nlp->lwp_thread);
 505         } else {
 506                 spin_unlock(&dd->spin);
 507                 /*spin_unlock(&dfly_spin);*/
 508         }
 509         crit_exit_gd(gd);
 510 }
 511
 512 /*
 513  * Place the specified lwp on the user scheduler's run queue.  This routine
 514  * must be called with the thread descheduled.  The lwp must be runnable.
 515  * It must not be possible for anyone else to explicitly schedule this thread.
 516  *
 517  * The thread may be the current thread as a special case.
 518  */
 519 static void
 520 dfly_setrunqueue(struct lwp *lp)
 521 {
 522         globaldata_t rgd;
 523         dfly_pcpu_t rdd;
 524         int cpuid;
 525
 526         /*
 527          * First validate the process LWKT state.
 528          */
 529         crit_enter();
 530         KASSERT(lp->lwp_stat == LSRUN, ("setrunqueue: lwp not LSRUN"));
 531         KASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0,
 532             ("lwp %d/%d already on runq! flag %08x/%08x", lp->lwp_proc->p_pid,
 533              lp->lwp_tid, lp->lwp_proc->p_flags, lp->lwp_flags));
 534         KKASSERT((lp->lwp_thread->td_flags & TDF_RUNQ) == 0);
 535
 536         /*
 537          * NOTE: gd and dd are relative to the target thread's last cpu,
 538          *       NOT our current cpu.
 539          */
 540         rgd = globaldata_find(lp->lwp_qcpu);
 541         rdd = &dfly_pcpu[lp->lwp_qcpu];
 542         cpuid = rdd->cpuid;
 543
 544         /*
 545          * This process is not supposed to be scheduled anywhere or assigned
 546          * as the current process anywhere.  Assert the condition.
 547          */
 548         KKASSERT(rdd->uschedcp != lp);
 549
 550 #ifndef SMP
 551         /*
 552          * If we are not SMP we do not have a scheduler helper to kick
 553          * and must directly activate the process if none are scheduled.
 554          *
 555          * This is really only an issue when bootstrapping init since
 556          * the caller in all other cases will be a user process, and
 557          * even if released (rdd->uschedcp == NULL), that process will
 558          * kickstart the scheduler when it returns to user mode from
 559          * the kernel.
 560          *
 561          * NOTE: On SMP we can't just set some other cpu's uschedcp.
 562          */
 563         if (rdd->uschedcp == NULL) {
 564                 spin_lock(&rdd->spin);
 565                 if (rdd->uschedcp == NULL) {
 566                         atomic_set_cpumask(&dfly_curprocmask, rgd->gd_cpumask);
 567                         rdd->uschedcp = lp;
 568                         rdd->upri = lp->lwp_priority;
 569                         spin_unlock(&rdd->spin);
 570                         lwkt_schedule(lp->lwp_thread);
 571                         crit_exit();
 572                         return;
 573                 }
 574                 spin_unlock(&rdd->spin);
 575         }
 576 #endif
 577
 578 #ifdef SMP
 579         /*
 580          * XXX fixme.  Could be part of a remrunqueue/setrunqueue
 581          * operation when the priority is recalculated, so TDF_MIGRATING
 582          * may already be set.
 583          */
 584         if ((lp->lwp_thread->td_flags & TDF_MIGRATING) == 0)
 585                 lwkt_giveaway(lp->lwp_thread);
 586 #endif
 587
 588 #ifdef SMP
 589         /*
 590          * Ok, we have to setrunqueue some target cpu and request a reschedule
 591          * if necessary.
 592          *
 593          * We have to choose the best target cpu.  It might not be the current
 594          * target even if the current cpu has no running user thread (for
 595          * example, because the current cpu might be a hyperthread and its
 596          * sibling has a thread assigned).
 597          */
 598         /*spin_lock(&dfly_spin);*/
 599         rdd = dfly_choose_best_queue(rdd, lp);
 600         rgd = globaldata_find(rdd->cpuid);
 601
 602         /*
 603          * We lose control of lp the moment we release the spinlock after
 604          * having placed lp on the queue.  i.e. another cpu could pick it
 605          * up and it could exit, or its priority could be further adjusted,
 606          * or something like that.
 607          *
 608          * WARNING! dd can point to a foreign cpu!
 609          */
 610         spin_lock(&rdd->spin);
 611         dfly_setrunqueue_locked(rdd, lp);
 612         /*spin_unlock(&dfly_spin);*/
 613
 614         if (rgd == mycpu) {
 615                 if ((rdd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) {
 616                         spin_unlock(&rdd->spin);
 617                         if (rdd->uschedcp == NULL) {
 618                                 wakeup_mycpu(&rdd->helper_thread); /* XXX */
 619                                 need_user_resched();
 620                         } else {
 621                                 need_user_resched();
 622                         }
 623                 } else {
 624                         spin_unlock(&rdd->spin);
 625                 }
 626         } else {
 627                 atomic_clear_cpumask(&dfly_rdyprocmask, CPUMASK(cpuid));
 628                 if ((rdd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) {
 629                         spin_unlock(&rdd->spin);
 630                         lwkt_send_ipiq(rgd, dfly_need_user_resched_remote,
 631                                        NULL);
 632                 } else {
 633                         spin_unlock(&rdd->spin);
 634                         wakeup(&rdd->helper_thread);
 635                 }
 636         }
 637 #else
 638         /*
 639          * Request a reschedule if appropriate.
 640          */
 641         spin_lock(&rdd->spin);
 642         dfly_setrunqueue_locked(rdd, lp);
 643         spin_unlock(&rdd->spin);
 644         if ((rdd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) {
 645                 need_user_resched();
 646         }
 647 #endif
 648         crit_exit();
 649 }
 650
 651 /*
 652  * This routine is called from a systimer IPI.  It MUST be MP-safe and
 653  * the BGL IS NOT HELD ON ENTRY.  This routine is called at ESTCPUFREQ on
 654  * each cpu.
 655  */
 656 static
 657 void
 658 dfly_schedulerclock(struct lwp *lp, sysclock_t period, sysclock_t cpstamp)
 659 {
 660         globaldata_t gd = mycpu;
 661         dfly_pcpu_t dd = &dfly_pcpu[gd->gd_cpuid];
 662
 663         /*
 664          * Do we need to round-robin?  We round-robin 10 times a second.
 665          * This should only occur for cpu-bound batch processes.
 666          */
 667         if (++dd->rrcount >= usched_dfly_rrinterval) {
 668                 dd->rrcount = 0;
 669                 need_user_resched();
 670         }
 671
 672         /*
 673          * Adjust estcpu upward using a real time equivalent calculation.
 674          */
 675         lp->lwp_estcpu = ESTCPULIM(lp->lwp_estcpu + ESTCPUMAX / ESTCPUFREQ + 1);
 676
 677         /*
 678          * Spinlocks also hold a critical section so there should not be
 679          * any active.
 680          */
 681         KKASSERT(gd->gd_spinlocks_wr == 0);
 682
 683         dfly_resetpriority(lp);
 684 }
 685
 686 /*
 687  * Called from acquire and from kern_synch's one-second timer (one of the
 688  * callout helper threads) with a critical section held.
 689  *
 690  * Decay p_estcpu based on the number of ticks we haven't been running
 691  * and our p_nice.  As the load increases each process observes a larger
 692  * number of idle ticks (because other processes are running in them).
 693  * This observation leads to a larger correction which tends to make the
 694  * system more 'batchy'.
 695  *
 696  * Note that no recalculation occurs for a process which sleeps and wakes
 697  * up in the same tick.  That is, a system doing thousands of context
 698  * switches per second will still only do serious estcpu calculations
 699  * ESTCPUFREQ times per second.
 700  */
 701 static
 702 void
 703 dfly_recalculate_estcpu(struct lwp *lp)
 704 {
 705         globaldata_t gd = mycpu;
 706         dfly_pcpu_t dd = &dfly_pcpu[gd->gd_cpuid];
 707         sysclock_t cpbase;
 708         sysclock_t ttlticks;
 709         int estcpu;
 710         int decay_factor;
 711
 712         /*
 713          * We have to subtract periodic to get the last schedclock
 714          * timeout time, otherwise we would get the upcoming timeout.
 715          * Keep in mind that a process can migrate between cpus and
 716          * while the scheduler clock should be very close, boundary
 717          * conditions could lead to a small negative delta.
 718          */
 719         cpbase = gd->gd_schedclock.time - gd->gd_schedclock.periodic;
 720
 721         if (lp->lwp_slptime > 1) {
 722                 /*
 723                  * Too much time has passed, do a coarse correction.
 724                  */
 725                 lp->lwp_estcpu = lp->lwp_estcpu >> 1;
 726                 dfly_resetpriority(lp);
 727                 lp->lwp_cpbase = cpbase;
 728                 lp->lwp_cpticks = 0;
 729                 lp->lwp_batch -= ESTCPUFREQ;
 730                 if (lp->lwp_batch < 0)
 731                         lp->lwp_batch = 0;
 732         } else if (lp->lwp_cpbase != cpbase) {
 733                 /*
 734                  * Adjust estcpu if we are in a different tick.  Don't waste
 735                  * time if we are in the same tick.
 736                  *
 737                  * First calculate the number of ticks in the measurement
 738                  * interval.  The ttlticks calculation can wind up 0 due to
 739                  * a bug in the handling of lwp_slptime  (as yet not found),
 740                  * so make sure we do not get a divide by 0 panic.
 741                  */
 742                 ttlticks = (cpbase - lp->lwp_cpbase) /
 743                            gd->gd_schedclock.periodic;
 744                 if (ttlticks < 0) {
 745                         ttlticks = 0;
 746                         lp->lwp_cpbase = cpbase;
 747                 }
 748                 if (ttlticks == 0)
 749                         return;
 750                 updatepcpu(lp, lp->lwp_cpticks, ttlticks);
 751
 752                 /*
 753                  * Calculate the percentage of one cpu used factoring in ncpus
 754                  * and the load and adjust estcpu.  Handle degenerate cases
 755                  * by adding 1 to runqcount.
 756                  *
 757                  * estcpu is scaled by ESTCPUMAX.
 758                  *
 759                  * runqcount is the excess number of user processes
 760                  * that cannot be immediately scheduled to cpus.  We want
 761                  * to count these as running to avoid range compression
 762                  * in the base calculation (which is the actual percentage
 763                  * of one cpu used).
 764                  */
 765                 estcpu = (lp->lwp_cpticks * ESTCPUMAX) *
 766                          (dd->runqcount + ncpus) / (ncpus * ttlticks);
 767
 768                 /*
 769                  * If estcpu is > 50% we become more batch-like
 770                  * If estcpu is <= 50% we become less batch-like
 771                  *
 772                  * It takes 30 cpu seconds to traverse the entire range.
 773                  */
 774                 if (estcpu > ESTCPUMAX / 2) {
 775                         lp->lwp_batch += ttlticks;
 776                         if (lp->lwp_batch > BATCHMAX)
 777                                 lp->lwp_batch = BATCHMAX;
 778                 } else {
 779                         lp->lwp_batch -= ttlticks;
 780                         if (lp->lwp_batch < 0)
 781                                 lp->lwp_batch = 0;
 782                 }
 783
 784                 if (usched_dfly_debug == lp->lwp_proc->p_pid) {
 785                         kprintf("pid %d lwp %p estcpu %3d %3d bat %d cp %d/%d",
 786                                 lp->lwp_proc->p_pid, lp,
 787                                 estcpu, lp->lwp_estcpu,
 788                                 lp->lwp_batch,
 789                                 lp->lwp_cpticks, ttlticks);
 790                 }
 791
 792                 /*
 793                  * Adjust lp->lwp_esetcpu.  The decay factor determines how
 794                  * quickly lwp_estcpu collapses to its realtime calculation.
 795                  * A slower collapse gives us a more accurate number but
 796                  * can cause a cpu hog to eat too much cpu before the
 797                  * scheduler decides to downgrade it.
 798                  *
 799                  * NOTE: p_nice is accounted for in dfly_resetpriority(),
 800                  *       and not here, but we must still ensure that a
 801                  *       cpu-bound nice -20 process does not completely
 802                  *       override a cpu-bound nice +20 process.
 803                  *
 804                  * NOTE: We must use ESTCPULIM() here to deal with any
 805                  *       overshoot.
 806                  */
 807                 decay_factor = usched_dfly_decay;
 808                 if (decay_factor < 1)
 809                         decay_factor = 1;
 810                 if (decay_factor > 1024)
 811                         decay_factor = 1024;
 812
 813                 lp->lwp_estcpu = ESTCPULIM(
 814                         (lp->lwp_estcpu * decay_factor + estcpu) /
 815                         (decay_factor + 1));
 816
 817                 if (usched_dfly_debug == lp->lwp_proc->p_pid)
 818                         kprintf(" finalestcpu %d\n", lp->lwp_estcpu);
 819                 dfly_resetpriority(lp);
 820                 lp->lwp_cpbase += ttlticks * gd->gd_schedclock.periodic;
 821                 lp->lwp_cpticks = 0;
 822         }
 823 }
 824
 825 /*
 826  * Compute the priority of a process when running in user mode.
 827  * Arrange to reschedule if the resulting priority is better
 828  * than that of the current process.
 829  *
 830  * This routine may be called with any process.
 831  *
 832  * This routine is called by fork1() for initial setup with the process
 833  * of the run queue, and also may be called normally with the process on or
 834  * off the run queue.
 835  */
 836 static void
 837 dfly_resetpriority(struct lwp *lp)
 838 {
 839         dfly_pcpu_t rdd;
 840         int newpriority;
 841         u_short newrqtype;
 842         int rcpu;
 843         int checkpri;
 844         int estcpu;
 845
 846         crit_enter();
 847
 848         /*
 849          * Lock the scheduler (lp) belongs to.  This can be on a different
 850          * cpu.  Handle races.  This loop breaks out with the appropriate
 851          * rdd locked.
 852          */
 853         for (;;) {
 854                 rcpu = lp->lwp_qcpu;
 855                 rdd = &dfly_pcpu[rcpu];
 856                 spin_lock(&rdd->spin);
 857                 if (rcpu == lp->lwp_qcpu)
 858                         break;
 859                 spin_unlock(&rdd->spin);
 860         }
 861
 862         /*
 863          * Calculate the new priority and queue type
 864          */
 865         newrqtype = lp->lwp_rtprio.type;
 866
 867         switch(newrqtype) {
 868         case RTP_PRIO_REALTIME:
 869         case RTP_PRIO_FIFO:
 870                 newpriority = PRIBASE_REALTIME +
 871                              (lp->lwp_rtprio.prio & PRIMASK);
 872                 break;
 873         case RTP_PRIO_NORMAL:
 874                 /*
 875                  * Detune estcpu based on batchiness.  lwp_batch ranges
 876                  * from 0 to  BATCHMAX.  Limit estcpu for the sake of
 877                  * the priority calculation to between 50% and 100%.
 878                  */
 879                 estcpu = lp->lwp_estcpu * (lp->lwp_batch + BATCHMAX) /
 880                          (BATCHMAX * 2);
 881
 882                 /*
 883                  * p_nice piece         Adds (0-40) * 2         0-80
 884                  * estcpu               Adds 16384  * 4 / 512   0-128
 885                  */
 886                 newpriority = (lp->lwp_proc->p_nice - PRIO_MIN) * PPQ / NICEPPQ;
 887                 newpriority += estcpu * PPQ / ESTCPUPPQ;
 888                 newpriority = newpriority * MAXPRI / (PRIO_RANGE * PPQ /
 889                               NICEPPQ + ESTCPUMAX * PPQ / ESTCPUPPQ);
 890                 newpriority = PRIBASE_NORMAL + (newpriority & PRIMASK);
 891                 break;
 892         case RTP_PRIO_IDLE:
 893                 newpriority = PRIBASE_IDLE + (lp->lwp_rtprio.prio & PRIMASK);
 894                 break;
 895         case RTP_PRIO_THREAD:
 896                 newpriority = PRIBASE_THREAD + (lp->lwp_rtprio.prio & PRIMASK);
 897                 break;
 898         default:
 899                 panic("Bad RTP_PRIO %d", newrqtype);
 900                 /* NOT REACHED */
 901         }
 902
 903         /*
 904          * The newpriority incorporates the queue type so do a simple masked
 905          * check to determine if the process has moved to another queue.  If
 906          * it has, and it is currently on a run queue, then move it.
 907          *
 908          * Since uload is ~PPQMASK masked, no modifications are necessary if
 909          * we end up in the same run queue.
 910          */
 911         if ((lp->lwp_priority ^ newpriority) & ~PPQMASK) {
 912                 int delta_uload;
 913
 914                 /*
 915                  * uload can change, calculate the adjustment to reduce
 916                  * edge cases since choosers scan the cpu topology without
 917                  * locks.
 918                  */
 919                 if (lp->lwp_mpflags & LWP_MP_ULOAD) {
 920                         delta_uload =
 921                                 -((lp->lwp_priority & ~PPQMASK) & PRIMASK) +
 922                                 ((newpriority & ~PPQMASK) & PRIMASK);
 923                         atomic_add_int(&dfly_pcpu[lp->lwp_qcpu].uload,
 924                                        delta_uload);
 925                 }
 926                 if (lp->lwp_mpflags & LWP_MP_ONRUNQ) {
 927                         dfly_remrunqueue_locked(rdd, lp);
 928                         lp->lwp_priority = newpriority;
 929                         lp->lwp_rqtype = newrqtype;
 930                         lp->lwp_rqindex = (newpriority & PRIMASK) / PPQ;
 931                         dfly_setrunqueue_locked(rdd, lp);
 932                         checkpri = 1;
 933                 } else {
 934                         lp->lwp_priority = newpriority;
 935                         lp->lwp_rqtype = newrqtype;
 936                         lp->lwp_rqindex = (newpriority & PRIMASK) / PPQ;
 937                         checkpri = 0;
 938                 }
 939         } else {
 940                 /*
 941                  * In the same PPQ, uload cannot change.
 942                  */
 943                 lp->lwp_priority = newpriority;
 944                 checkpri = 1;
 945                 rcpu = -1;
 946         }
 947
 948         /*
 949          * Determine if we need to reschedule the target cpu.  This only
 950          * occurs if the LWP is already on a scheduler queue, which means
 951          * that idle cpu notification has already occured.  At most we
 952          * need only issue a need_user_resched() on the appropriate cpu.
 953          *
 954          * The LWP may be owned by a CPU different from the current one,
 955          * in which case dd->uschedcp may be modified without an MP lock
 956          * or a spinlock held.  The worst that happens is that the code
 957          * below causes a spurious need_user_resched() on the target CPU
 958          * and dd->pri to be wrong for a short period of time, both of
 959          * which are harmless.
 960          *
 961          * If checkpri is 0 we are adjusting the priority of the current
 962          * process, possibly higher (less desireable), so ignore the upri
 963          * check which will fail in that case.
 964          */
 965         if (rcpu >= 0) {
 966                 if ((dfly_rdyprocmask & CPUMASK(rcpu)) &&
 967                     (checkpri == 0 ||
 968                      (rdd->upri & ~PRIMASK) > (lp->lwp_priority & ~PRIMASK))) {
 969 #ifdef SMP
 970                         if (rcpu == mycpu->gd_cpuid) {
 971                                 spin_unlock(&rdd->spin);
 972                                 need_user_resched();
 973                         } else {
 974                                 atomic_clear_cpumask(&dfly_rdyprocmask,
 975                                                      CPUMASK(rcpu));
 976                                 spin_unlock(&rdd->spin);
 977                                 lwkt_send_ipiq(globaldata_find(rcpu),
 978                                                dfly_need_user_resched_remote,
 979                                                NULL);
 980                         }
 981 #else
 982                         spin_unlock(&rdd->spin);
 983                         need_user_resched();
 984 #endif
 985                 } else {
 986                         spin_unlock(&rdd->spin);
 987                 }
 988         } else {
 989                 spin_unlock(&rdd->spin);
 990         }
 991         crit_exit();
 992 }
 993
 994 static
 995 void
 996 dfly_yield(struct lwp *lp)
 997 {
 998 #if 0
 999         /* FUTURE (or something similar) */
1000         switch(lp->lwp_rqtype) {
1001         case RTP_PRIO_NORMAL:
1002                 lp->lwp_estcpu = ESTCPULIM(lp->lwp_estcpu + ESTCPUINCR);
1003                 break;
1004         default:
1005                 break;
1006         }
1007 #endif
1008         need_user_resched();
1009 }
1010
1011 /*
1012  * Called from fork1() when a new child process is being created.
1013  *
1014  * Give the child process an initial estcpu that is more batch then
1015  * its parent and dock the parent for the fork (but do not
1016  * reschedule the parent).   This comprises the main part of our batch
1017  * detection heuristic for both parallel forking and sequential execs.
1018  *
1019  * XXX lwp should be "spawning" instead of "forking"
1020  */
1021 static void
1022 dfly_forking(struct lwp *plp, struct lwp *lp)
1023 {
1024         /*
1025          * Put the child 4 queue slots (out of 32) higher than the parent
1026          * (less desireable than the parent).
1027          */
1028         lp->lwp_estcpu = ESTCPULIM(plp->lwp_estcpu + ESTCPUPPQ * 4);
1029
1030         /*
1031          * The batch status of children always starts out centerline
1032          * and will inch-up or inch-down as appropriate.  It takes roughly
1033          * ~15 seconds of >50% cpu to hit the limit.
1034          */
1035         lp->lwp_batch = BATCHMAX / 2;
1036
1037         /*
1038          * Dock the parent a cost for the fork, protecting us from fork
1039          * bombs.  If the parent is forking quickly make the child more
1040          * batchy.
1041          */
1042         plp->lwp_estcpu = ESTCPULIM(plp->lwp_estcpu + ESTCPUPPQ / 16);
1043 }
1044
1045 /*
1046  * Called when a lwp is being removed from this scheduler, typically
1047  * during lwp_exit().
1048  */
1049 static void
1050 dfly_exiting(struct lwp *lp, struct proc *child_proc)
1051 {
1052         dfly_pcpu_t dd = &dfly_pcpu[lp->lwp_qcpu];
1053
1054         if (lp->lwp_mpflags & LWP_MP_ULOAD) {
1055                 atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ULOAD);
1056                 atomic_add_int(&dd->uload,
1057                                -((lp->lwp_priority & ~PPQMASK) & PRIMASK));
1058         }
1059 }
1060
1061 static void
1062 dfly_uload_update(struct lwp *lp)
1063 {
1064         dfly_pcpu_t dd = &dfly_pcpu[lp->lwp_qcpu];
1065
1066         if (lp->lwp_thread->td_flags & TDF_RUNQ) {
1067                 if ((lp->lwp_mpflags & LWP_MP_ULOAD) == 0) {
1068                         atomic_set_int(&lp->lwp_mpflags, LWP_MP_ULOAD);
1069                         atomic_add_int(&dd->uload,
1070                                    ((lp->lwp_priority & ~PPQMASK) & PRIMASK));
1071                 }
1072         } else {
1073                 if (lp->lwp_mpflags & LWP_MP_ULOAD) {
1074                         atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ULOAD);
1075                         atomic_add_int(&dd->uload,
1076                                    -((lp->lwp_priority & ~PPQMASK) & PRIMASK));
1077                 }
1078         }
1079 }
1080
1081 /*
1082  * chooseproc() is called when a cpu needs a user process to LWKT schedule,
1083  * it selects a user process and returns it.  If chklp is non-NULL and chklp
1084  * has a better or equal priority then the process that would otherwise be
1085  * chosen, NULL is returned.
1086  *
1087  * Until we fix the RUNQ code the chklp test has to be strict or we may
1088  * bounce between processes trying to acquire the current process designation.
1089  *
1090  * Must be called with dfly_spin exclusive held.  The spinlock is
1091  * left intact through the entire routine.
1092  *
1093  * if chklp is NULL this function will dive other cpu's queues looking
1094  * for work if the current queue is empty.
1095  */
1096 static
1097 struct lwp *
1098 dfly_chooseproc_locked(dfly_pcpu_t dd, struct lwp *chklp, int isremote)
1099 {
1100 #ifdef SMP
1101         dfly_pcpu_t xdd;
1102 #endif
1103         struct lwp *lp;
1104         struct rq *q;
1105         u_int32_t *which, *which2;
1106         u_int32_t pri;
1107         u_int32_t rtqbits;
1108         u_int32_t tsqbits;
1109         u_int32_t idqbits;
1110         /*usched_dfly_queue_checks*/
1111
1112         rtqbits = dd->rtqueuebits;
1113         tsqbits = dd->queuebits;
1114         idqbits = dd->idqueuebits;
1115
1116         if (rtqbits) {
1117                 pri = bsfl(rtqbits);
1118                 q = &dd->rtqueues[pri];
1119                 which = &dd->rtqueuebits;
1120                 which2 = &rtqbits;
1121         } else if (tsqbits) {
1122                 pri = bsfl(tsqbits);
1123                 q = &dd->queues[pri];
1124                 which = &dd->queuebits;
1125                 which2 = &tsqbits;
1126         } else if (idqbits) {
1127                 pri = bsfl(idqbits);
1128                 q = &dd->idqueues[pri];
1129                 which = &dd->idqueuebits;
1130                 which2 = &idqbits;
1131         } else
1132 #ifdef SMP
1133         if (isremote) {
1134                 /*
1135                  * Disallow remote->remote recursion
1136                  */
1137                 return (NULL);
1138         } else {
1139                 /*
1140                  * Pull a runnable thread from a remote run queue.  We have
1141                  * to adjust qcpu and uload manually because the lp we return
1142                  * might be assigned directly to uschedcp (setrunqueue might
1143                  * not be called).
1144                  */
1145                 xdd = dfly_choose_worst_queue(dd);
1146                 if (xdd && xdd != dd && spin_trylock(&xdd->spin)) {
1147                         lp = dfly_chooseproc_locked(xdd, NULL, 1);
1148                         if (lp) {
1149                                 if (lp->lwp_mpflags & LWP_MP_ULOAD) {
1150                                         atomic_add_int(&xdd->uload,
1151                                             -((lp->lwp_priority & ~PPQMASK) &
1152                                               PRIMASK));
1153                                 }
1154                                 lp->lwp_qcpu = dd->cpuid;
1155                                 atomic_add_int(&dd->uload,
1156                                     ((lp->lwp_priority & ~PPQMASK) & PRIMASK));
1157                                 atomic_set_int(&lp->lwp_mpflags, LWP_MP_ULOAD);
1158                         }
1159                         spin_unlock(&xdd->spin);
1160                 } else {
1161                         lp = NULL;
1162                 }
1163                 return (lp);
1164         }
1165 #else
1166         {
1167                 return NULL;
1168         }
1169 #endif
1170         lp = TAILQ_FIRST(q);
1171         KASSERT(lp, ("chooseproc: no lwp on busy queue"));
1172
1173         /*
1174          * If the passed lwp <chklp> is reasonably close to the selected
1175          * lwp <lp>, return NULL (indicating that <chklp> should be kept).
1176          *
1177          * Note that we must error on the side of <chklp> to avoid bouncing
1178          * between threads in the acquire code.
1179          */
1180         if (chklp) {
1181                 if (chklp->lwp_priority < lp->lwp_priority + PPQ)
1182                         return(NULL);
1183         }
1184
1185         KTR_COND_LOG(usched_chooseproc,
1186             lp->lwp_proc->p_pid == usched_dfly_pid_debug,
1187             lp->lwp_proc->p_pid,
1188             lp->lwp_thread->td_gd->gd_cpuid,
1189             mycpu->gd_cpuid);
1190
1191         TAILQ_REMOVE(q, lp, lwp_procq);
1192         --dd->runqcount;
1193         if (TAILQ_EMPTY(q))
1194                 *which &= ~(1 << pri);
1195         KASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) != 0, ("not on runq6!"));
1196         atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ);
1197
1198         return lp;
1199 }
1200
1201 #ifdef SMP
1202
1203 /*
1204  * USED TO PUSH RUNNABLE LWPS TO THE LEAST LOADED CPU.
1205  *
1206  * Choose a cpu node to schedule lp on, hopefully nearby its current
1207  * node.  The current node is passed in (dd) (though it can also be obtained
1208  * from lp->lwp_qcpu).  The caller will dfly_setrunqueue() lp on the queue
1209  * we return.
1210  *
1211  * When the topology is known choose a cpu whos group has, in aggregate,
1212  * has the lowest weighted load.
1213  */
1214 static
1215 dfly_pcpu_t
1216 dfly_choose_best_queue(dfly_pcpu_t dd, struct lwp *lp)
1217 {
1218         cpumask_t mask;
1219         cpu_node_t *cpup;
1220         cpu_node_t *cpun;
1221         cpu_node_t *cpub;
1222         dfly_pcpu_t rdd;
1223         int cpuid;
1224         int n;
1225         int load;
1226         int lowest_load;
1227         int level;
1228
1229         /*
1230          * When the topology is unknown choose a random cpu that is hopefully
1231          * idle.
1232          */
1233         if (dd->cpunode == NULL)
1234                 return (dfly_choose_queue_simple(dd, lp));
1235
1236         /*
1237          * When the topology is known choose a cpu whos group has, in
1238          * aggregate, has the lowest weighted load.
1239          */
1240         cpup = root_cpu_node;
1241         rdd = dd;
1242         level = cpu_topology_levels_number;
1243
1244         while (cpup) {
1245                 /*
1246                  * Degenerate case super-root
1247                  */
1248                 if (cpup->child_node && cpup->child_no == 1) {
1249                         cpup = cpup->child_node;
1250                         --level;
1251                         continue;
1252                 }
1253
1254                 /*
1255                  * Terminal cpunode
1256                  */
1257                 if (cpup->child_node == NULL) {
1258                         rdd = &dfly_pcpu[BSFCPUMASK(cpup->members)];
1259                         break;
1260                 }
1261
1262                 cpub = NULL;
1263                 lowest_load = 0x7FFFFFFF;
1264
1265                 for (n = 0; n < cpup->child_no; ++n) {
1266                         /*
1267                          * Accumulate load information for all cpus
1268                          * which are members of this node.
1269                          */
1270                         cpun = &cpup->child_node[n];
1271                         mask = cpun->members & usched_global_cpumask &
1272                                smp_active_mask & lp->lwp_cpumask;
1273                         if (mask == 0)
1274                                 continue;
1275                         load = 0;
1276                         while (mask) {
1277                                 cpuid = BSFCPUMASK(mask);
1278                                 load += dfly_pcpu[cpuid].uload;
1279                                 mask &= ~CPUMASK(cpuid);
1280                         }
1281
1282                         /*
1283                          * Give a slight advantage to nearby cpus.
1284                          */
1285                         if (cpun->members & dd->cpumask)
1286                                 load -= PPQ * level;
1287
1288                         /*
1289                          * Calculate the best load
1290                          */
1291                         if (cpub == NULL || lowest_load > load ||
1292                             (lowest_load == load &&
1293                              (cpun->members & dd->cpumask))
1294                         ) {
1295                                 lowest_load = load;
1296                                 cpub = cpun;
1297                         }
1298                 }
1299                 cpup = cpub;
1300                 --level;
1301         }
1302         if (usched_dfly_chooser)
1303                 kprintf("lp %02d->%02d %s\n",
1304                         lp->lwp_qcpu, rdd->cpuid, lp->lwp_proc->p_comm);
1305         return (rdd);
1306 }
1307
1308 /*
1309  * USED TO PULL RUNNABLE LWPS FROM THE MOST LOADED CPU.
1310  *
1311  * Choose the worst queue close to dd's cpu node with a non-empty runq.
1312  *
1313  * This is used by the thread chooser when the current cpu's queues are
1314  * empty to steal a thread from another cpu's queue.  We want to offload
1315  * the most heavily-loaded queue.
1316  */
1317 static
1318 dfly_pcpu_t
1319 dfly_choose_worst_queue(dfly_pcpu_t dd)
1320 {
1321         cpumask_t mask;
1322         cpu_node_t *cpup;
1323         cpu_node_t *cpun;
1324         cpu_node_t *cpub;
1325         dfly_pcpu_t rdd;
1326         int cpuid;
1327         int n;
1328         int load;
1329         int highest_load;
1330         int uloadok;
1331         int level;
1332
1333         /*
1334          * When the topology is unknown choose a random cpu that is hopefully
1335          * idle.
1336          */
1337         if (dd->cpunode == NULL) {
1338                 return (NULL);
1339         }
1340
1341         /*
1342          * When the topology is known choose a cpu whos group has, in
1343          * aggregate, has the lowest weighted load.
1344          */
1345         cpup = root_cpu_node;
1346         rdd = dd;
1347         level = cpu_topology_levels_number;
1348         while (cpup) {
1349                 /*
1350                  * Degenerate case super-root
1351                  */
1352                 if (cpup->child_node && cpup->child_no == 1) {
1353                         cpup = cpup->child_node;
1354                         --level;
1355                         continue;
1356                 }
1357
1358                 /*
1359                  * Terminal cpunode
1360                  */
1361                 if (cpup->child_node == NULL) {
1362                         rdd = &dfly_pcpu[BSFCPUMASK(cpup->members)];
1363                         break;
1364                 }
1365
1366                 cpub = NULL;
1367                 highest_load = 0;
1368
1369                 for (n = 0; n < cpup->child_no; ++n) {
1370                         /*
1371                          * Accumulate load information for all cpus
1372                          * which are members of this node.
1373                          */
1374                         cpun = &cpup->child_node[n];
1375                         mask = cpun->members & usched_global_cpumask &
1376                                smp_active_mask;
1377                         if (mask == 0)
1378                                 continue;
1379                         load = 0;
1380                         uloadok = 0;
1381                         while (mask) {
1382                                 cpuid = BSFCPUMASK(mask);
1383                                 load += dfly_pcpu[cpuid].uload;
1384                                 if (dfly_pcpu[cpuid].uload)
1385                                         uloadok = 1;
1386                                 mask &= ~CPUMASK(cpuid);
1387                         }
1388
1389                         /*
1390                          * Give a slight advantage to nearby cpus.
1391                          */
1392                         if (cpun->members & dd->cpumask)
1393                                 load += PPQ * level;
1394
1395                         /*
1396                          * The best candidate is the one with the worst
1397                          * (highest) load.  Prefer candiates that are
1398                          * closer to our cpu.
1399                          */
1400                         if (uloadok &&
1401                             (cpub == NULL || highest_load < load ||
1402                              (highest_load == load &&
1403                               (cpun->members & dd->cpumask)))
1404                         ) {
1405                                 highest_load = load;
1406                                 cpub = cpun;
1407                         }
1408                 }
1409                 cpup = cpub;
1410                 --level;
1411         }
1412         return (rdd);
1413 }
1414
1415 static
1416 dfly_pcpu_t
1417 dfly_choose_queue_simple(dfly_pcpu_t dd, struct lwp *lp)
1418 {
1419         dfly_pcpu_t rdd;
1420         cpumask_t tmpmask;
1421         cpumask_t mask;
1422         int cpuid;
1423
1424         /*
1425          * Fallback to the original heuristic, select random cpu,
1426          * first checking cpus not currently running a user thread.
1427          */
1428         cpuid = (dfly_scancpu & 0xFFFF) % ncpus;
1429         mask = ~dfly_curprocmask & dfly_rdyprocmask & lp->lwp_cpumask &
1430                smp_active_mask & usched_global_cpumask;
1431
1432         while (mask) {
1433                 tmpmask = ~(CPUMASK(cpuid) - 1);
1434                 if (mask & tmpmask)
1435                         cpuid = BSFCPUMASK(mask & tmpmask);
1436                 else
1437                         cpuid = BSFCPUMASK(mask);
1438                 rdd = &dfly_pcpu[cpuid];
1439
1440                 if ((rdd->upri & ~PPQMASK) >= (lp->lwp_priority & ~PPQMASK))
1441                         goto found;
1442                 mask &= ~CPUMASK(cpuid);
1443         }
1444
1445         /*
1446          * Then cpus which might have a currently running lp
1447          */
1448         cpuid = (dfly_scancpu & 0xFFFF) % ncpus;
1449         mask = dfly_curprocmask & dfly_rdyprocmask &
1450                lp->lwp_cpumask & smp_active_mask & usched_global_cpumask;
1451
1452         while (mask) {
1453                 tmpmask = ~(CPUMASK(cpuid) - 1);
1454                 if (mask & tmpmask)
1455                         cpuid = BSFCPUMASK(mask & tmpmask);
1456                 else
1457                         cpuid = BSFCPUMASK(mask);
1458                 rdd = &dfly_pcpu[cpuid];
1459
1460                 if ((rdd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK))
1461                         goto found;
1462                 mask &= ~CPUMASK(cpuid);
1463         }
1464
1465         /*
1466          * If we cannot find a suitable cpu we reload from dfly_scancpu
1467          * and round-robin.  Other cpus will pickup as they release their
1468          * current lwps or become ready.
1469          *
1470          * Avoid a degenerate system lockup case if usched_global_cpumask
1471          * is set to 0 or otherwise does not cover lwp_cpumask.
1472          *
1473          * We only kick the target helper thread in this case, we do not
1474          * set the user resched flag because
1475          */
1476         cpuid = (dfly_scancpu & 0xFFFF) % ncpus;
1477         if ((CPUMASK(cpuid) & usched_global_cpumask) == 0)
1478                 cpuid = 0;
1479         rdd = &dfly_pcpu[cpuid];
1480 found:
1481         return (rdd);
1482 }
1483
1484 static
1485 void
1486 dfly_need_user_resched_remote(void *dummy)
1487 {
1488         globaldata_t gd = mycpu;
1489         dfly_pcpu_t  dd = &dfly_pcpu[gd->gd_cpuid];
1490
1491         need_user_resched();
1492
1493         /* Call wakeup_mycpu to avoid sending IPIs to other CPUs */
1494         wakeup_mycpu(&dd->helper_thread);
1495 }
1496
1497 #endif
1498
1499 /*
1500  * dfly_remrunqueue_locked() removes a given process from the run queue
1501  * that it is on, clearing the queue busy bit if it becomes empty.
1502  *
1503  * Note that user process scheduler is different from the LWKT schedule.
1504  * The user process scheduler only manages user processes but it uses LWKT
1505  * underneath, and a user process operating in the kernel will often be
1506  * 'released' from our management.
1507  *
1508  * uload is NOT adjusted here.  It is only adjusted if the lwkt_thread goes
1509  * to sleep or the lwp is moved to a different runq.
1510  */
1511 static void
1512 dfly_remrunqueue_locked(dfly_pcpu_t rdd, struct lwp *lp)
1513 {
1514         struct rq *q;
1515         u_int32_t *which;
1516         u_int8_t pri;
1517
1518         KKASSERT(lp->lwp_mpflags & LWP_MP_ONRUNQ);
1519         atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ);
1520         --rdd->runqcount;
1521         /*rdd->uload -= (lp->lwp_priority & ~PPQMASK) & PRIMASK;*/
1522         KKASSERT(rdd->runqcount >= 0);
1523
1524         pri = lp->lwp_rqindex;
1525         switch(lp->lwp_rqtype) {
1526         case RTP_PRIO_NORMAL:
1527                 q = &rdd->queues[pri];
1528                 which = &rdd->queuebits;
1529                 break;
1530         case RTP_PRIO_REALTIME:
1531         case RTP_PRIO_FIFO:
1532                 q = &rdd->rtqueues[pri];
1533                 which = &rdd->rtqueuebits;
1534                 break;
1535         case RTP_PRIO_IDLE:
1536                 q = &rdd->idqueues[pri];
1537                 which = &rdd->idqueuebits;
1538                 break;
1539         default:
1540                 panic("remrunqueue: invalid rtprio type");
1541                 /* NOT REACHED */
1542         }
1543         TAILQ_REMOVE(q, lp, lwp_procq);
1544         if (TAILQ_EMPTY(q)) {
1545                 KASSERT((*which & (1 << pri)) != 0,
1546                         ("remrunqueue: remove from empty queue"));
1547                 *which &= ~(1 << pri);
1548         }
1549 }
1550
1551 /*
1552  * dfly_setrunqueue_locked()
1553  *
1554  * Add a process whos rqtype and rqindex had previously been calculated
1555  * onto the appropriate run queue.   Determine if the addition requires
1556  * a reschedule on a cpu and return the cpuid or -1.
1557  *
1558  * NOTE:          Lower priorities are better priorities.
1559  *
1560  * NOTE ON ULOAD: This variable specifies the aggregate load on a cpu, the
1561  *                sum of the rough lwp_priority for all running and runnable
1562  *                processes.  Lower priority processes (higher lwp_priority
1563  *                values) actually DO count as more load, not less, because
1564  *                these are the programs which require the most care with
1565  *                regards to cpu selection.
1566  */
1567 static void
1568 dfly_setrunqueue_locked(dfly_pcpu_t rdd, struct lwp *lp)
1569 {
1570         struct rq *q;
1571         u_int32_t *which;
1572         int pri;
1573
1574         if (lp->lwp_qcpu != rdd->cpuid) {
1575                 if (lp->lwp_mpflags & LWP_MP_ULOAD) {
1576                         atomic_clear_int(&lp->lwp_mpflags, LWP_MP_ULOAD);
1577                         atomic_add_int(&dfly_pcpu[lp->lwp_qcpu].uload,
1578                                    -((lp->lwp_priority & ~PPQMASK) & PRIMASK));
1579                 }
1580                 lp->lwp_qcpu = rdd->cpuid;
1581         }
1582
1583         KKASSERT((lp->lwp_mpflags & LWP_MP_ONRUNQ) == 0);
1584         atomic_set_int(&lp->lwp_mpflags, LWP_MP_ONRUNQ);
1585         ++rdd->runqcount;
1586         if ((lp->lwp_mpflags & LWP_MP_ULOAD) == 0) {
1587                 atomic_set_int(&lp->lwp_mpflags, LWP_MP_ULOAD);
1588                 atomic_add_int(&dfly_pcpu[lp->lwp_qcpu].uload,
1589                                (lp->lwp_priority & ~PPQMASK) & PRIMASK);
1590         }
1591
1592         pri = lp->lwp_rqindex;
1593
1594         switch(lp->lwp_rqtype) {
1595         case RTP_PRIO_NORMAL:
1596                 q = &rdd->queues[pri];
1597                 which = &rdd->queuebits;
1598                 break;
1599         case RTP_PRIO_REALTIME:
1600         case RTP_PRIO_FIFO:
1601                 q = &rdd->rtqueues[pri];
1602                 which = &rdd->rtqueuebits;
1603                 break;
1604         case RTP_PRIO_IDLE:
1605                 q = &rdd->idqueues[pri];
1606                 which = &rdd->idqueuebits;
1607                 break;
1608         default:
1609                 panic("remrunqueue: invalid rtprio type");
1610                 /* NOT REACHED */
1611         }
1612
1613         /*
1614          * Add to the correct queue and set the appropriate bit.  If no
1615          * lower priority (i.e. better) processes are in the queue then
1616          * we want a reschedule, calculate the best cpu for the job.
1617          *
1618          * Always run reschedules on the LWPs original cpu.
1619          */
1620         TAILQ_INSERT_TAIL(q, lp, lwp_procq);
1621         *which |= 1 << pri;
1622 }
1623
1624 #ifdef SMP
1625
1626 /*
1627  * For SMP systems a user scheduler helper thread is created for each
1628  * cpu and is used to allow one cpu to wakeup another for the purposes of
1629  * scheduling userland threads from setrunqueue().
1630  *
1631  * UP systems do not need the helper since there is only one cpu.
1632  *
1633  * We can't use the idle thread for this because we might block.
1634  * Additionally, doing things this way allows us to HLT idle cpus
1635  * on MP systems.
1636  */
1637 static void
1638 dfly_helper_thread(void *dummy)
1639 {
1640     globaldata_t gd;
1641     dfly_pcpu_t  dd;
1642     struct lwp *nlp;
1643     cpumask_t mask;
1644     int cpuid;
1645
1646     gd = mycpu;
1647     cpuid = gd->gd_cpuid;       /* doesn't change */
1648     mask = gd->gd_cpumask;      /* doesn't change */
1649     dd = &dfly_pcpu[cpuid];
1650
1651     /*
1652      * Since we only want to be woken up only when no user processes
1653      * are scheduled on a cpu, run at an ultra low priority.
1654      */
1655     lwkt_setpri_self(TDPRI_USER_SCHEDULER);
1656
1657     tsleep(&dd->helper_thread, 0, "schslp", 0);
1658
1659     for (;;) {
1660         /*
1661          * We use the LWKT deschedule-interlock trick to avoid racing
1662          * dfly_rdyprocmask.  This means we cannot block through to the
1663          * manual lwkt_switch() call we make below.
1664          */
1665         crit_enter_gd(gd);
1666         tsleep_interlock(&dd->helper_thread, 0);
1667
1668         /*spin_lock(&dfly_spin);*/
1669         spin_lock(&dd->spin);
1670
1671         atomic_set_cpumask(&dfly_rdyprocmask, mask);
1672         clear_user_resched();   /* This satisfied the reschedule request */
1673         dd->rrcount = 0;        /* Reset the round-robin counter */
1674
1675         if ((dfly_curprocmask & mask) == 0) {
1676                 /*
1677                  * No thread is currently scheduled.
1678                  */
1679                 KKASSERT(dd->uschedcp == NULL);
1680                 if ((nlp = dfly_chooseproc_locked(dd, NULL, 0)) != NULL) {
1681                         KTR_COND_LOG(usched_sched_thread_no_process,
1682                             nlp->lwp_proc->p_pid == usched_dfly_pid_debug,
1683                             gd->gd_cpuid,
1684                             nlp->lwp_proc->p_pid,
1685                             nlp->lwp_thread->td_gd->gd_cpuid);
1686
1687                         atomic_set_cpumask(&dfly_curprocmask, mask);
1688                         dd->upri = nlp->lwp_priority;
1689                         dd->uschedcp = nlp;
1690                         dd->rrcount = 0;        /* reset round robin */
1691                         spin_unlock(&dd->spin);
1692                         /*spin_unlock(&dfly_spin);*/
1693                         lwkt_acquire(nlp->lwp_thread);
1694                         lwkt_schedule(nlp->lwp_thread);
1695                 } else {
1696                         spin_unlock(&dd->spin);
1697                         /*spin_unlock(&dfly_spin);*/
1698                 }
1699         } else if (dd->runqcount) {
1700                 /*
1701                  * Possibly find a better process to schedule.
1702                  */
1703                 nlp = dfly_chooseproc_locked(dd, dd->uschedcp, 0);
1704                 if (nlp) {
1705                         KTR_COND_LOG(usched_sched_thread_process,
1706                             nlp->lwp_proc->p_pid == usched_dfly_pid_debug,
1707                             gd->gd_cpuid,
1708                             nlp->lwp_proc->p_pid,
1709                             nlp->lwp_thread->td_gd->gd_cpuid);
1710
1711                         dd->upri = nlp->lwp_priority;
1712                         dd->uschedcp = nlp;
1713                         dd->rrcount = 0;        /* reset round robin */
1714                         spin_unlock(&dd->spin);
1715                         /*spin_unlock(&dfly_spin);*/
1716                         lwkt_acquire(nlp->lwp_thread);
1717                         lwkt_schedule(nlp->lwp_thread);
1718                 } else {
1719                         /*
1720                          * Leave the thread on our run queue.  Another
1721                          * scheduler will try to pull it later.
1722                          */
1723                         spin_unlock(&dd->spin);
1724                         /*spin_unlock(&dfly_spin);*/
1725                 }
1726         } else {
1727                 /*
1728                  * The runq is empty.
1729                  */
1730                 spin_unlock(&dd->spin);
1731                 /*spin_unlock(&dfly_spin);*/
1732         }
1733
1734         /*
1735          * We're descheduled unless someone scheduled us.  Switch away.
1736          * Exiting the critical section will cause splz() to be called
1737          * for us if interrupts and such are pending.
1738          */
1739         crit_exit_gd(gd);
1740         tsleep(&dd->helper_thread, PINTERLOCKED, "schslp", 0);
1741     }
1742 }
1743
1744 /* sysctl stick_to_level parameter */
1745 static int
1746 sysctl_usched_dfly_stick_to_level(SYSCTL_HANDLER_ARGS)
1747 {
1748         int error, new_val;
1749
1750         new_val = usched_dfly_stick_to_level;
1751
1752         error = sysctl_handle_int(oidp, &new_val, 0, req);
1753         if (error != 0 || req->newptr == NULL)
1754                 return (error);
1755         if (new_val > cpu_topology_levels_number - 1 || new_val < 0)
1756                 return (EINVAL);
1757         usched_dfly_stick_to_level = new_val;
1758         return (0);
1759 }
1760
1761 /*
1762  * Setup our scheduler helpers.  Note that curprocmask bit 0 has already
1763  * been cleared by rqinit() and we should not mess with it further.
1764  */
1765 static void
1766 dfly_helper_thread_cpu_init(void)
1767 {
1768         int i;
1769         int j;
1770         int cpuid;
1771         int smt_not_supported = 0;
1772         int cache_coherent_not_supported = 0;
1773
1774         if (bootverbose)
1775                 kprintf("Start scheduler helpers on cpus:\n");
1776
1777         sysctl_ctx_init(&usched_dfly_sysctl_ctx);
1778         usched_dfly_sysctl_tree =
1779                 SYSCTL_ADD_NODE(&usched_dfly_sysctl_ctx,
1780                                 SYSCTL_STATIC_CHILDREN(_kern), OID_AUTO,
1781                                 "usched_dfly", CTLFLAG_RD, 0, "");
1782
1783         for (i = 0; i < ncpus; ++i) {
1784                 dfly_pcpu_t dd = &dfly_pcpu[i];
1785                 cpumask_t mask = CPUMASK(i);
1786
1787                 if ((mask & smp_active_mask) == 0)
1788                     continue;
1789
1790                 spin_init(&dd->spin);
1791                 dd->cpunode = get_cpu_node_by_cpuid(i);
1792                 dd->cpuid = i;
1793                 dd->cpumask = CPUMASK(i);
1794                 for (j = 0; j < NQS; j++) {
1795                         TAILQ_INIT(&dd->queues[j]);
1796                         TAILQ_INIT(&dd->rtqueues[j]);
1797                         TAILQ_INIT(&dd->idqueues[j]);
1798                 }
1799                 atomic_clear_cpumask(&dfly_curprocmask, 1);
1800
1801                 if (dd->cpunode == NULL) {
1802                         smt_not_supported = 1;
1803                         cache_coherent_not_supported = 1;
1804                         if (bootverbose)
1805                                 kprintf ("\tcpu%d - WARNING: No CPU NODE "
1806                                          "found for cpu\n", i);
1807                 } else {
1808                         switch (dd->cpunode->type) {
1809                         case THREAD_LEVEL:
1810                                 if (bootverbose)
1811                                         kprintf ("\tcpu%d - HyperThreading "
1812                                                  "available. Core siblings: ",
1813                                                  i);
1814                                 break;
1815                         case CORE_LEVEL:
1816                                 smt_not_supported = 1;
1817
1818                                 if (bootverbose)
1819                                         kprintf ("\tcpu%d - No HT available, "
1820                                                  "multi-core/physical "
1821                                                  "cpu. Physical siblings: ",
1822                                                  i);
1823                                 break;
1824                         case CHIP_LEVEL:
1825                                 smt_not_supported = 1;
1826
1827                                 if (bootverbose)
1828                                         kprintf ("\tcpu%d - No HT available, "
1829                                                  "single-core/physical cpu. "
1830                                                  "Package Siblings: ",
1831                                                  i);
1832                                 break;
1833                         default:
1834                                 /* Let's go for safe defaults here */
1835                                 smt_not_supported = 1;
1836                                 cache_coherent_not_supported = 1;
1837                                 if (bootverbose)
1838                                         kprintf ("\tcpu%d - Unknown cpunode->"
1839                                                  "type=%u. Siblings: ",
1840                                                  i,
1841                                                  (u_int)dd->cpunode->type);
1842                                 break;
1843                         }
1844
1845                         if (bootverbose) {
1846                                 if (dd->cpunode->parent_node != NULL) {
1847                                         CPUSET_FOREACH(cpuid, dd->cpunode->parent_node->members)
1848                                                 kprintf("cpu%d ", cpuid);
1849                                         kprintf("\n");
1850                                 } else {
1851                                         kprintf(" no siblings\n");
1852                                 }
1853                         }
1854                 }
1855
1856                 lwkt_create(dfly_helper_thread, NULL, NULL, &dd->helper_thread,
1857                             0, i, "usched %d", i);
1858
1859                 /*
1860                  * Allow user scheduling on the target cpu.  cpu #0 has already
1861                  * been enabled in rqinit().
1862                  */
1863                 if (i)
1864                     atomic_clear_cpumask(&dfly_curprocmask, mask);
1865                 atomic_set_cpumask(&dfly_rdyprocmask, mask);
1866                 dd->upri = PRIBASE_NULL;
1867
1868         }
1869
1870         /* usched_dfly sysctl configurable parameters */
1871
1872         SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1873                        SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1874                        OID_AUTO, "rrinterval", CTLFLAG_RW,
1875                        &usched_dfly_rrinterval, 0, "");
1876         SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1877                        SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1878                        OID_AUTO, "decay", CTLFLAG_RW,
1879                        &usched_dfly_decay, 0, "Extra decay when not running");
1880         SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1881                        SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1882                        OID_AUTO, "batch_time", CTLFLAG_RW,
1883                        &usched_dfly_batch_time, 0, "Min batch counter value");
1884
1885         /* Add enable/disable option for SMT scheduling if supported */
1886         if (smt_not_supported) {
1887                 usched_dfly_smt = 0;
1888                 SYSCTL_ADD_STRING(&usched_dfly_sysctl_ctx,
1889                                   SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1890                                   OID_AUTO, "smt", CTLFLAG_RD,
1891                                   "NOT SUPPORTED", 0, "SMT NOT SUPPORTED");
1892         } else {
1893                 usched_dfly_smt = 1;
1894                 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1895                                SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1896                                OID_AUTO, "smt", CTLFLAG_RW,
1897                                &usched_dfly_smt, 0, "Enable SMT scheduling");
1898         }
1899
1900         /*
1901          * Add enable/disable option for cache coherent scheduling
1902          * if supported
1903          */
1904         if (cache_coherent_not_supported) {
1905                 usched_dfly_cache_coherent = 0;
1906                 SYSCTL_ADD_STRING(&usched_dfly_sysctl_ctx,
1907                                   SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1908                                   OID_AUTO, "cache_coherent", CTLFLAG_RD,
1909                                   "NOT SUPPORTED", 0,
1910                                   "Cache coherence NOT SUPPORTED");
1911         } else {
1912                 usched_dfly_cache_coherent = 1;
1913                 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1914                                SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1915                                OID_AUTO, "cache_coherent", CTLFLAG_RW,
1916                                &usched_dfly_cache_coherent, 0,
1917                                "Enable/Disable cache coherent scheduling");
1918
1919                 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1920                                SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1921                                OID_AUTO, "upri_affinity", CTLFLAG_RW,
1922                                &usched_dfly_upri_affinity, 1,
1923                                "Number of PPQs in user priority check");
1924
1925                 SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1926                                SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1927                                OID_AUTO, "queue_checks", CTLFLAG_RW,
1928                                &usched_dfly_queue_checks, 5,
1929                                "LWPs to check from a queue before giving up");
1930
1931                 SYSCTL_ADD_PROC(&usched_dfly_sysctl_ctx,
1932                                 SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1933                                 OID_AUTO, "stick_to_level",
1934                                 CTLTYPE_INT | CTLFLAG_RW,
1935                                 NULL, sizeof usched_dfly_stick_to_level,
1936                                 sysctl_usched_dfly_stick_to_level, "I",
1937                                 "Stick a process to this level. See sysctl"
1938                                 "paremter hw.cpu_topology.level_description");
1939         }
1940 }
1941 SYSINIT(uschedtd, SI_BOOT2_USCHED, SI_ORDER_SECOND,
1942         dfly_helper_thread_cpu_init, NULL)
1943
1944 #else /* No SMP options - just add the configurable parameters to sysctl */
1945
1946 static void
1947 sched_sysctl_tree_init(void)
1948 {
1949         sysctl_ctx_init(&usched_dfly_sysctl_ctx);
1950         usched_dfly_sysctl_tree =
1951                 SYSCTL_ADD_NODE(&usched_dfly_sysctl_ctx,
1952                                 SYSCTL_STATIC_CHILDREN(_kern), OID_AUTO,
1953                                 "usched_dfly", CTLFLAG_RD, 0, "");
1954
1955         /* usched_dfly sysctl configurable parameters */
1956         SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1957                        SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1958                        OID_AUTO, "rrinterval", CTLFLAG_RW,
1959                        &usched_dfly_rrinterval, 0, "");
1960         SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1961                        SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1962                        OID_AUTO, "decay", CTLFLAG_RW,
1963                        &usched_dfly_decay, 0, "Extra decay when not running");
1964         SYSCTL_ADD_INT(&usched_dfly_sysctl_ctx,
1965                        SYSCTL_CHILDREN(usched_dfly_sysctl_tree),
1966                        OID_AUTO, "batch_time", CTLFLAG_RW,
1967                        &usched_dfly_batch_time, 0, "Min batch counter value");
1968 }
1969 SYSINIT(uschedtd, SI_BOOT2_USCHED, SI_ORDER_SECOND,
1970         sched_sysctl_tree_init, NULL)
1971 #endif