sys/dsched/fq/dsched_fq_core.c

   1 /*
   2  * Copyright (c) 2009, 2010 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Alex Hornung <ahornung@gmail.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/kernel.h>
  37 #include <sys/proc.h>
  38 #include <sys/sysctl.h>
  39 #include <sys/buf.h>
  40 #include <sys/conf.h>
  41 #include <sys/diskslice.h>
  42 #include <sys/disk.h>
  43 #include <machine/atomic.h>
  44 #include <sys/malloc.h>
  45 #include <sys/thread.h>
  46 #include <sys/thread2.h>
  47 #include <sys/sysctl.h>
  48 #include <sys/spinlock2.h>
  49 #include <machine/md_var.h>
  50 #include <sys/ctype.h>
  51 #include <sys/syslog.h>
  52 #include <sys/device.h>
  53 #include <sys/msgport.h>
  54 #include <sys/msgport2.h>
  55 #include <sys/buf2.h>
  56 #include <sys/dsched.h>
  57 #include <machine/varargs.h>
  58 #include <machine/param.h>
  59
  60 #include <dsched/fq/dsched_fq.h>
  61
  62 MALLOC_DECLARE(M_DSCHEDFQ);
  63
  64 static int      dsched_fq_version_maj = 0;
  65 static int      dsched_fq_version_min = 8;
  66
  67 struct dsched_fq_stats  fq_stats;
  68
  69 struct objcache_malloc_args dsched_fq_dpriv_malloc_args = {
  70         sizeof(struct dsched_fq_dpriv), M_DSCHEDFQ };
  71 struct objcache_malloc_args dsched_fq_priv_malloc_args = {
  72         sizeof(struct dsched_fq_priv), M_DSCHEDFQ };
  73 struct objcache_malloc_args dsched_fq_mpriv_malloc_args = {
  74         sizeof(struct dsched_fq_mpriv), M_DSCHEDFQ };
  75
  76 static struct objcache  *fq_dpriv_cache;
  77 static struct objcache  *fq_mpriv_cache;
  78 static struct objcache  *fq_priv_cache;
  79
  80 TAILQ_HEAD(, dsched_fq_mpriv)   dsched_fqmp_list =
  81                 TAILQ_HEAD_INITIALIZER(dsched_fqmp_list);
  82
  83 struct lock     fq_fqmp_lock;
  84 struct callout  fq_callout;
  85
  86 extern struct dsched_ops dsched_fq_ops;
  87
  88 void
  89 fq_reference_dpriv(struct dsched_fq_dpriv *dpriv)
  90 {
  91         int refcount;
  92
  93         refcount = atomic_fetchadd_int(&dpriv->refcount, 1);
  94
  95         KKASSERT(refcount >= 0);
  96 }
  97
  98 void
  99 fq_reference_priv(struct dsched_fq_priv *fqp)
 100 {
 101         int refcount;
 102
 103         refcount = atomic_fetchadd_int(&fqp->refcount, 1);
 104
 105         KKASSERT(refcount >= 0);
 106 }
 107
 108 void
 109 fq_reference_mpriv(struct dsched_fq_mpriv *fqmp)
 110 {
 111         int refcount;
 112
 113         refcount = atomic_fetchadd_int(&fqmp->refcount, 1);
 114
 115         KKASSERT(refcount >= 0);
 116 }
 117
 118 void
 119 fq_dereference_dpriv(struct dsched_fq_dpriv *dpriv)
 120 {
 121         struct dsched_fq_priv   *fqp, *fqp2;
 122         int refcount;
 123
 124         refcount = atomic_fetchadd_int(&dpriv->refcount, -1);
 125
 126
 127         KKASSERT(refcount >= 0 || refcount <= -0x400);
 128
 129         if (refcount == 1) {
 130                 atomic_subtract_int(&dpriv->refcount, 0x400); /* mark as: in destruction */
 131 #if 1
 132                 kprintf("dpriv (%p) destruction started, trace:\n", dpriv);
 133                 print_backtrace(4);
 134 #endif
 135                 lockmgr(&dpriv->lock, LK_EXCLUSIVE);
 136                 TAILQ_FOREACH_MUTABLE(fqp, &dpriv->fq_priv_list, dlink, fqp2) {
 137                         TAILQ_REMOVE(&dpriv->fq_priv_list, fqp, dlink);
 138                         fqp->flags &= ~FQP_LINKED_DPRIV;
 139                         fq_dereference_priv(fqp);
 140                 }
 141                 lockmgr(&dpriv->lock, LK_RELEASE);
 142
 143                 objcache_put(fq_dpriv_cache, dpriv);
 144                 atomic_subtract_int(&fq_stats.dpriv_allocations, 1);
 145         }
 146 }
 147
 148 void
 149 fq_dereference_priv(struct dsched_fq_priv *fqp)
 150 {
 151         struct dsched_fq_mpriv  *fqmp;
 152         struct dsched_fq_dpriv  *dpriv;
 153         int refcount;
 154
 155         refcount = atomic_fetchadd_int(&fqp->refcount, -1);
 156
 157         KKASSERT(refcount >= 0 || refcount <= -0x400);
 158
 159         if (refcount == 1) {
 160                 atomic_subtract_int(&fqp->refcount, 0x400); /* mark as: in destruction */
 161 #if 0
 162                 kprintf("fqp (%p) destruction started, trace:\n", fqp);
 163                 print_backtrace(8);
 164 #endif
 165                 dpriv = fqp->dpriv;
 166                 KKASSERT(dpriv != NULL);
 167                 KKASSERT(fqp->qlength == 0);
 168
 169                 if (fqp->flags & FQP_LINKED_DPRIV) {
 170                         lockmgr(&dpriv->lock, LK_EXCLUSIVE);
 171
 172                         TAILQ_REMOVE(&dpriv->fq_priv_list, fqp, dlink);
 173                         fqp->flags &= ~FQP_LINKED_DPRIV;
 174
 175                         lockmgr(&dpriv->lock, LK_RELEASE);
 176                 }
 177
 178                 if (fqp->flags & FQP_LINKED_FQMP) {
 179                         fqmp = fqp->fqmp;
 180                         KKASSERT(fqmp != NULL);
 181
 182                         spin_lock_wr(&fqmp->lock);
 183
 184                         TAILQ_REMOVE(&fqmp->fq_priv_list, fqp, link);
 185                         fqp->flags &= ~FQP_LINKED_FQMP;
 186
 187                         spin_unlock_wr(&fqmp->lock);
 188                 }
 189
 190                 objcache_put(fq_priv_cache, fqp);
 191                 atomic_subtract_int(&fq_stats.fqp_allocations, 1);
 192 #if 0
 193                 fq_dereference_dpriv(dpriv);
 194 #endif
 195         }
 196 }
 197
 198 void
 199 fq_dereference_mpriv(struct dsched_fq_mpriv *fqmp)
 200 {
 201         struct dsched_fq_priv   *fqp, *fqp2;
 202         int refcount;
 203
 204         refcount = atomic_fetchadd_int(&fqmp->refcount, -1);
 205
 206         KKASSERT(refcount >= 0 || refcount <= -0x400);
 207
 208         if (refcount == 1) {
 209                 atomic_subtract_int(&fqmp->refcount, 0x400); /* mark as: in destruction */
 210 #if 0
 211                 kprintf("fqmp (%p) destruction started, trace:\n", fqmp);
 212                 print_backtrace(8);
 213 #endif
 214                 FQ_GLOBAL_FQMP_LOCK();
 215
 216                 TAILQ_FOREACH_MUTABLE(fqp, &fqmp->fq_priv_list, link, fqp2) {
 217                         TAILQ_REMOVE(&fqmp->fq_priv_list, fqp, link);
 218                         fqp->flags &= ~FQP_LINKED_FQMP;
 219                         fq_dereference_priv(fqp);
 220                 }
 221                 TAILQ_REMOVE(&dsched_fqmp_list, fqmp, link);
 222
 223                 FQ_GLOBAL_FQMP_UNLOCK();
 224
 225                 objcache_put(fq_mpriv_cache, fqmp);
 226                 atomic_subtract_int(&fq_stats.fqmp_allocations, 1);
 227         }
 228 }
 229
 230
 231 struct dsched_fq_priv *
 232 fq_alloc_priv(struct disk *dp, struct dsched_fq_mpriv *fqmp)
 233 {
 234         struct dsched_fq_priv   *fqp;
 235 #if 0
 236         fq_reference_dpriv(dsched_get_disk_priv(dp));
 237 #endif
 238         fqp = objcache_get(fq_priv_cache, M_WAITOK);
 239         bzero(fqp, sizeof(struct dsched_fq_priv));
 240
 241         /* XXX: maybe we do need another ref for the disk list for fqp */
 242         fq_reference_priv(fqp);
 243
 244         FQ_FQP_LOCKINIT(fqp);
 245         fqp->dp = dp;
 246
 247         fqp->dpriv = dsched_get_disk_priv(dp);
 248         TAILQ_INIT(&fqp->queue);
 249
 250         TAILQ_INSERT_TAIL(&fqp->dpriv->fq_priv_list, fqp, dlink);
 251         fqp->flags |= FQP_LINKED_DPRIV;
 252
 253         if (fqmp) {
 254                 fqp->fqmp = fqmp;
 255                 fqp->p = fqmp->p;
 256
 257                 /* Put the fqp in the fqmp list */
 258                 FQ_FQMP_LOCK(fqmp);
 259                 TAILQ_INSERT_TAIL(&fqmp->fq_priv_list, fqp, link);
 260                 FQ_FQMP_UNLOCK(fqmp);
 261                 fqp->flags |= FQP_LINKED_FQMP;
 262         }
 263
 264         atomic_add_int(&fq_stats.fqp_allocations, 1);
 265         return fqp;
 266 }
 267
 268
 269 struct dsched_fq_dpriv *
 270 fq_alloc_dpriv(struct disk *dp)
 271 {
 272         struct dsched_fq_dpriv *dpriv;
 273
 274         dpriv = objcache_get(fq_dpriv_cache, M_WAITOK);
 275         bzero(dpriv, sizeof(struct dsched_fq_dpriv));
 276         fq_reference_dpriv(dpriv);
 277         dpriv->dp = dp;
 278         dpriv->avg_rq_time = 0;
 279         dpriv->incomplete_tp = 0;
 280         FQ_DPRIV_LOCKINIT(dpriv);
 281         TAILQ_INIT(&dpriv->fq_priv_list);
 282
 283         atomic_add_int(&fq_stats.dpriv_allocations, 1);
 284         return dpriv;
 285 }
 286
 287
 288 struct dsched_fq_mpriv *
 289 fq_alloc_mpriv(struct proc *p)
 290 {
 291         struct dsched_fq_mpriv  *fqmp;
 292         struct dsched_fq_priv   *fqp;
 293         struct disk     *dp = NULL;
 294
 295         fqmp = objcache_get(fq_mpriv_cache, M_WAITOK);
 296         bzero(fqmp, sizeof(struct dsched_fq_mpriv));
 297         fq_reference_mpriv(fqmp);
 298 #if 0
 299         kprintf("fq_alloc_mpriv, new fqmp = %p\n", fqmp);
 300 #endif
 301         FQ_FQMP_LOCKINIT(fqmp);
 302         TAILQ_INIT(&fqmp->fq_priv_list);
 303         fqmp->p = p;
 304
 305         while ((dp = dsched_disk_enumerate(dp, &dsched_fq_ops))) {
 306                 fqp = fq_alloc_priv(dp, fqmp);
 307 #if 0
 308                 fq_reference_priv(fqp);
 309 #endif
 310         }
 311
 312         FQ_GLOBAL_FQMP_LOCK();
 313         TAILQ_INSERT_TAIL(&dsched_fqmp_list, fqmp, link);
 314         FQ_GLOBAL_FQMP_UNLOCK();
 315
 316         atomic_add_int(&fq_stats.fqmp_allocations, 1);
 317         return fqmp;
 318 }
 319
 320
 321 void
 322 fq_dispatcher(struct dsched_fq_dpriv *dpriv)
 323 {
 324         struct dsched_fq_mpriv  *fqmp;
 325         struct dsched_fq_priv   *fqp, *fqp2;
 326         struct bio *bio, *bio2;
 327         int idle;
 328
 329         /*
 330          * We need to manually assign an fqp to the fqmp of this thread
 331          * since it isn't assigned one during fq_prepare, as the disk
 332          * is not set up yet.
 333          */
 334         fqmp = dsched_get_thread_priv(curthread);
 335         KKASSERT(fqmp != NULL);
 336
 337         fqp = fq_alloc_priv(dpriv->dp, fqmp);
 338 #if 0
 339         fq_reference_priv(fqp);
 340 #endif
 341
 342         FQ_DPRIV_LOCK(dpriv);
 343         for(;;) {
 344                 idle = 0;
 345                 /* sleep ~60 ms */
 346                 if ((lksleep(dpriv, &dpriv->lock, 0, "fq_dispatcher", hz/15) == 0)) {
 347                         /*
 348                          * We've been woken up; this either means that we are
 349                          * supposed to die away nicely or that the disk is idle.
 350                          */
 351
 352                         if (__predict_false(dpriv->die == 1)) {
 353                                 /* If we are supposed to die, drain all queues */
 354                                 fq_drain(dpriv, FQ_DRAIN_FLUSH);
 355
 356                                 /* Now we can safely unlock and exit */
 357                                 FQ_DPRIV_UNLOCK(dpriv);
 358                                 kprintf("fq_dispatcher is peacefully dying\n");
 359                                 lwkt_exit();
 360                                 /* NOTREACHED */
 361                         }
 362
 363                         /*
 364                          * We have been awakened because the disk is idle.
 365                          * So let's get ready to dispatch some extra bios.
 366                          */
 367                         idle = 1;
 368                 }
 369
 370                 /* Maybe the disk is idle and we just didn't get the wakeup */
 371                 if (idle == 0)
 372                         idle = dpriv->idle;
 373
 374                 /*
 375                  * XXX: further room for improvements here. It would be better
 376                  *      to dispatch a few requests from each fqp as to ensure
 377                  *      real fairness.
 378                  */
 379                 TAILQ_FOREACH_MUTABLE(fqp, &dpriv->fq_priv_list, dlink, fqp2) {
 380                         if (fqp->qlength == 0)
 381                                 continue;
 382
 383                         FQ_FQP_LOCK(fqp);
 384                         if (atomic_cmpset_int(&fqp->rebalance, 1, 0))
 385                                 fq_balance_self(fqp);
 386                         /*
 387                          * XXX: why 5 extra? should probably be dynamic,
 388                          *      relying on information on latency.
 389                          */
 390                         if ((fqp->max_tp > 0) && idle &&
 391                             (fqp->issued >= fqp->max_tp)) {
 392                                 fqp->max_tp += 5;
 393                         }
 394
 395                         TAILQ_FOREACH_MUTABLE(bio, &fqp->queue, link, bio2) {
 396                                 if (atomic_cmpset_int(&fqp->rebalance, 1, 0))
 397                                         fq_balance_self(fqp);
 398                                 if ((fqp->max_tp > 0) &&
 399                                     ((fqp->issued >= fqp->max_tp)))
 400                                         break;
 401
 402                                 TAILQ_REMOVE(&fqp->queue, bio, link);
 403                                 --fqp->qlength;
 404
 405                                 /*
 406                                  * beware that we do have an fqp reference
 407                                  * from the queueing
 408                                  */
 409                                 fq_dispatch(dpriv, bio, fqp);
 410                         }
 411                         FQ_FQP_UNLOCK(fqp);
 412
 413                 }
 414         }
 415 }
 416
 417 void
 418 fq_balance_thread(struct dsched_fq_dpriv *dpriv)
 419 {
 420         struct  dsched_fq_priv  *fqp, *fqp2;
 421         static struct timeval old_tv;
 422         struct timeval tv;
 423         int64_t total_budget, product;
 424         int64_t budget[FQ_PRIO_MAX+1];
 425         int     n, i, sum, total_disk_time;
 426         int     lost_bits;
 427
 428         getmicrotime(&old_tv);
 429
 430         FQ_DPRIV_LOCK(dpriv);
 431         for (;;) {
 432                 /* sleep ~1s */
 433                 if ((lksleep(curthread, &dpriv->lock, 0, "fq_balancer", hz/2) == 0)) {
 434                         if (__predict_false(dpriv->die)) {
 435                                 FQ_DPRIV_UNLOCK(dpriv);
 436                                 lwkt_exit();
 437                         }
 438                 }
 439
 440                 bzero(budget, sizeof(budget));
 441                 total_budget = 0;
 442                 n = 0;
 443
 444                 getmicrotime(&tv);
 445
 446                 total_disk_time = (int)(1000000*((tv.tv_sec - old_tv.tv_sec)) +
 447                     (tv.tv_usec - old_tv.tv_usec));
 448
 449                 if (total_disk_time == 0)
 450                         total_disk_time = 1;
 451
 452                 dsched_debug(LOG_INFO, "total_disk_time = %d\n", total_disk_time);
 453
 454                 old_tv = tv;
 455
 456                 dpriv->disk_busy = (100*(total_disk_time - dpriv->idle_time)) / total_disk_time;
 457                 if (dpriv->disk_busy < 0)
 458                         dpriv->disk_busy = 0;
 459
 460                 dpriv->idle_time = 0;
 461                 lost_bits = 0;
 462
 463                 TAILQ_FOREACH_MUTABLE(fqp, &dpriv->fq_priv_list, dlink, fqp2) {
 464                         fqp->s_avg_latency = fqp->avg_latency;
 465                         fqp->s_transactions = fqp->transactions;
 466                         if (fqp->s_transactions > 0 /* 30 */) {
 467                                 product = fqp->s_avg_latency * fqp->s_transactions;
 468                                 product >>= lost_bits;
 469                                 while(total_budget >= INT64_MAX - product) {
 470                                         ++lost_bits;
 471                                         product >>= 1;
 472                                         total_budget >>= 1;
 473                                 }
 474                                 total_budget += product;
 475                                 ++budget[(fqp->p) ? fqp->p->p_ionice : 0];
 476                                 KKASSERT(total_budget >= 0);
 477                                 dsched_debug(LOG_INFO,
 478                                     "%d) avg_latency = %d, transactions = %d, ioprio = %d\n",
 479                                     n, fqp->s_avg_latency, fqp->s_transactions,
 480                                     (fqp->p) ? fqp->p->p_ionice : 0);
 481                                 ++n;
 482                         } else {
 483                                 fqp->max_tp = 0;
 484                         }
 485                         fqp->rebalance = 0;
 486                         fqp->transactions = 0;
 487                         fqp->avg_latency = 0;
 488                         fqp->issued = 0;
 489                 }
 490
 491                 dsched_debug(LOG_INFO, "%d procs competing for disk\n"
 492                     "total_budget = %jd (lost bits = %d)\n"
 493                     "incomplete tp = %d\n", n, (intmax_t)total_budget,
 494                     lost_bits, dpriv->incomplete_tp);
 495
 496                 if (n == 0)
 497                         continue;
 498
 499                 sum = 0;
 500
 501                 for (i = 0; i < FQ_PRIO_MAX+1; i++) {
 502                         if (budget[i] == 0)
 503                                 continue;
 504                         sum += (FQ_PRIO_BIAS+i)*budget[i];
 505                 }
 506
 507                 if (sum == 0)
 508                         sum = 1;
 509
 510                 dsched_debug(LOG_INFO, "sum = %d\n", sum);
 511
 512                 for (i = 0; i < FQ_PRIO_MAX+1; i++) {
 513                         if (budget[i] == 0)
 514                                 continue;
 515
 516                         /*
 517                          * XXX: if we still overflow here, we really need to switch to
 518                          *      some more advanced mechanism such as compound int128 or
 519                          *      storing the lost bits so they can be used in the
 520                          *      fq_balance_self.
 521                          */
 522                         dpriv->budgetpb[i] = ((FQ_PRIO_BIAS+i)*total_budget/sum) << lost_bits;
 523                         KKASSERT(dpriv->budgetpb[i] >= 0);
 524                 }
 525
 526                 if (total_budget > dpriv->max_budget)
 527                         dpriv->max_budget = total_budget;
 528
 529                 dsched_debug(4, "disk is %d%% busy\n", dpriv->disk_busy);
 530                 TAILQ_FOREACH(fqp, &dpriv->fq_priv_list, dlink) {
 531                         fqp->rebalance = 1;
 532                 }
 533
 534                 dpriv->prev_full = dpriv->last_full;
 535                 dpriv->last_full = (dpriv->disk_busy >= 90)?1:0;
 536         }
 537 }
 538
 539
 540 /*
 541  * fq_balance_self should be called from all sorts of dispatchers. It basically
 542  * offloads some of the heavier calculations on throttling onto the process that
 543  * wants to do I/O instead of doing it in the fq_balance thread.
 544  * - should be called with dpriv lock held
 545  */
 546 void
 547 fq_balance_self(struct dsched_fq_priv *fqp) {
 548         struct dsched_fq_dpriv *dpriv;
 549
 550         int64_t budget, used_budget;
 551         int64_t avg_latency;
 552         int64_t transactions;
 553
 554         transactions = (int64_t)fqp->s_transactions;
 555         avg_latency = (int64_t)fqp->s_avg_latency;
 556         dpriv = fqp->dpriv;
 557
 558         used_budget = ((int64_t)avg_latency * transactions);
 559         budget = dpriv->budgetpb[(fqp->p) ? fqp->p->p_ionice : 0];
 560
 561         if (used_budget > 0) {
 562                 dsched_debug(LOG_INFO,
 563                     "info: used_budget = %jd, budget = %jd\n",
 564                     (intmax_t)used_budget, budget);
 565         }
 566
 567         if ((used_budget > budget) && (dpriv->disk_busy >= 90)) {
 568                 KKASSERT(avg_latency != 0);
 569
 570                 fqp->max_tp = budget/(avg_latency);
 571                 atomic_add_int(&fq_stats.procs_limited, 1);
 572
 573                 dsched_debug(LOG_INFO,
 574                     "rate limited to %d transactions\n", fqp->max_tp);
 575
 576         } else if (((used_budget*2 < budget) || (dpriv->disk_busy < 80)) &&
 577             (!dpriv->prev_full && !dpriv->last_full)) {
 578                 fqp->max_tp = 0;
 579         }
 580 }
 581
 582
 583 static int
 584 do_fqstats(SYSCTL_HANDLER_ARGS)
 585 {
 586         return (sysctl_handle_opaque(oidp, &fq_stats, sizeof(struct dsched_fq_stats), req));
 587 }
 588
 589
 590 SYSCTL_PROC(_kern, OID_AUTO, fq_stats, CTLTYPE_OPAQUE|CTLFLAG_RD,
 591     0, sizeof(struct dsched_fq_stats), do_fqstats, "fq_stats",
 592     "dsched_fq statistics");
 593
 594
 595 static void
 596 fq_init(void)
 597 {
 598
 599 }
 600
 601 static void
 602 fq_uninit(void)
 603 {
 604
 605 }
 606
 607 static void
 608 fq_earlyinit(void)
 609 {
 610         fq_priv_cache = objcache_create("fq-priv-cache", 0, 0,
 611                                            NULL, NULL, NULL,
 612                                            objcache_malloc_alloc,
 613                                            objcache_malloc_free,
 614                                            &dsched_fq_priv_malloc_args );
 615
 616         fq_mpriv_cache = objcache_create("fq-mpriv-cache", 0, 0,
 617                                            NULL, NULL, NULL,
 618                                            objcache_malloc_alloc,
 619                                            objcache_malloc_free,
 620                                            &dsched_fq_mpriv_malloc_args );
 621
 622         FQ_GLOBAL_FQMP_LOCKINIT();
 623
 624         fq_dpriv_cache = objcache_create("fq-dpriv-cache", 0, 0,
 625                                            NULL, NULL, NULL,
 626                                            objcache_malloc_alloc,
 627                                            objcache_malloc_free,
 628                                            &dsched_fq_dpriv_malloc_args );
 629
 630         bzero(&fq_stats, sizeof(struct dsched_fq_stats));
 631
 632         dsched_register(&dsched_fq_ops);
 633         callout_init_mp(&fq_callout);
 634
 635         kprintf("FQ scheduler policy version %d.%d loaded\n",
 636             dsched_fq_version_maj, dsched_fq_version_min);
 637 }
 638
 639 static void
 640 fq_earlyuninit(void)
 641 {
 642         callout_stop(&fq_callout);
 643         callout_deactivate(&fq_callout);
 644         return;
 645 }
 646
 647 SYSINIT(fq_register, SI_SUB_PRE_DRIVERS, SI_ORDER_ANY, fq_init, NULL);
 648 SYSUNINIT(fq_register, SI_SUB_PRE_DRIVERS, SI_ORDER_FIRST, fq_uninit, NULL);
 649
 650 SYSINIT(fq_early, SI_SUB_CREATE_INIT-1, SI_ORDER_FIRST, fq_earlyinit, NULL);
 651 SYSUNINIT(fq_early, SI_SUB_CREATE_INIT-1, SI_ORDER_ANY, fq_earlyuninit, NULL);