We need to restore unallocated_objects if objcache alloc fails.
[dragonfly.git] / sys / kern / kern_objcache.c
CommitLineData
8a268428
JH
1/*
2 * Copyright (c) 2005 Jeffrey M. Hsu. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Jeffrey M. Hsu.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of The DragonFly Project nor the names of its
16 * contributors may be used to endorse or promote products derived
17 * from this software without specific, prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
22 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
23 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
27 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
29 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 *
0aa16b5d 32 * $DragonFly: src/sys/kern/kern_objcache.c,v 1.23 2008/10/26 04:29:19 sephe Exp $
8a268428
JH
33 */
34
35#include <sys/param.h>
36#include <sys/kernel.h>
37#include <sys/systm.h>
38#include <sys/callout.h>
39#include <sys/globaldata.h>
40#include <sys/malloc.h>
41#include <sys/queue.h>
42#include <sys/objcache.h>
54d017fd 43#include <sys/spinlock.h>
8a268428
JH
44#include <sys/thread.h>
45#include <sys/thread2.h>
54d017fd 46#include <sys/spinlock2.h>
8a268428
JH
47
48static MALLOC_DEFINE(M_OBJCACHE, "objcache", "Object Cache");
49static MALLOC_DEFINE(M_OBJMAG, "objcache magazine", "Object Cache Magazine");
50
a1f6a1fe 51#define INITIAL_MAG_CAPACITY 64
8a268428
JH
52
53struct magazine {
54 int rounds;
55 int capacity;
54d017fd 56 int cleaning;
8a268428
JH
57 SLIST_ENTRY(magazine) nextmagazine;
58 void *objects[];
59};
60
61SLIST_HEAD(magazinelist, magazine);
62
63/*
64 * per-cluster cache of magazines
54d017fd
MD
65 *
66 * All fields in this structure are protected by the spinlock.
8a268428
JH
67 */
68struct magazinedepot {
69 /*
70 * The per-cpu object caches only exchanges completely full or
71 * completely empty magazines with the depot layer, so only have
72 * to cache these two types of magazines.
73 */
74 struct magazinelist fullmagazines;
75 struct magazinelist emptymagazines;
76 int magcapacity;
77
77e294a1 78 /* protect this structure */
54d017fd 79 struct spinlock spin;
8a268428 80
77e294a1
MD
81 /* magazines not yet allocated towards limit */
82 int unallocated_objects;
8a268428
JH
83
84 /* infrequently used fields */
85 int waiting; /* waiting for another cpu to
86 * return a full magazine to
87 * the depot */
88 int contested; /* depot contention count */
89};
90
91/*
92 * per-cpu object cache
93 * All fields in this structure are protected by crit_enter().
94 */
95struct percpu_objcache {
96 struct magazine *loaded_magazine; /* active magazine */
97 struct magazine *previous_magazine; /* backup magazine */
98
99 /* statistics */
100 int gets_cumulative; /* total calls to get */
101 int gets_null; /* objcache_get returned NULL */
102 int puts_cumulative; /* total calls to put */
103 int puts_othercluster; /* returned to other cluster */
104
105 /* infrequently used fields */
106 int waiting; /* waiting for a thread on this cpu to
107 * return an obj to the per-cpu cache */
108};
109
110/* only until we have NUMA cluster topology information XXX */
111#define MAXCLUSTERS 1
112#define myclusterid 0
113#define CLUSTER_OF(obj) 0
114
115/*
116 * Two-level object cache consisting of NUMA cluster-level depots of
117 * fully loaded or completely empty magazines and cpu-level caches of
118 * individual objects.
119 */
120struct objcache {
121 char *name;
122
123 /* object constructor and destructor from blank storage */
124 objcache_ctor_fn *ctor;
125 objcache_dtor_fn *dtor;
698331b0 126 void *privdata;
8a268428
JH
127
128 /* interface to underlying allocator */
129 objcache_alloc_fn *alloc;
130 objcache_free_fn *free;
131 void *allocator_args;
132
00de24de 133 LIST_ENTRY(objcache) oc_next;
a1f6a1fe 134 int exhausted; /* oops */
8a268428
JH
135
136 /* NUMA-cluster level caches */
137 struct magazinedepot depot[MAXCLUSTERS];
138
139 struct percpu_objcache cache_percpu[]; /* per-cpu caches */
140};
141
54d017fd 142static struct spinlock objcachelist_spin;
00de24de 143static LIST_HEAD(objcachelist, objcache) allobjcaches;
8a268428
JH
144
145static struct magazine *
146mag_alloc(int capacity)
147{
148 struct magazine *mag;
149
77652cad 150 mag = kmalloc(__offsetof(struct magazine, objects[capacity]),
77e294a1 151 M_OBJMAG, M_INTWAIT | M_ZERO);
8a268428
JH
152 mag->capacity = capacity;
153 mag->rounds = 0;
54d017fd 154 mag->cleaning = 0;
8a268428
JH
155 return (mag);
156}
157
5b7da64a
MD
158/*
159 * Utility routine for objects that don't require any de-construction.
160 */
161
162static void
698331b0 163null_dtor(void *obj, void *privdata)
5b7da64a
MD
164{
165 /* do nothing */
166}
167
168static boolean_t
698331b0 169null_ctor(void *obj, void *privdata, int ocflags)
5b7da64a
MD
170{
171 return TRUE;
172}
173
8a268428
JH
174/*
175 * Create an object cache.
176 */
177struct objcache *
0aa16b5d 178objcache_create(const char *name, int *cluster_limit0, int mag_capacity,
698331b0 179 objcache_ctor_fn *ctor, objcache_dtor_fn *dtor, void *privdata,
8a268428
JH
180 objcache_alloc_fn *alloc, objcache_free_fn *free,
181 void *allocator_args)
182{
183 struct objcache *oc;
184 struct magazinedepot *depot;
8a268428 185 int cpuid;
a1f6a1fe
MD
186 int need;
187 int factor;
1a21fef5
NT
188 int nmagdepot;
189 int i;
0aa16b5d
SZ
190 int cluster_limit;
191
192 if (cluster_limit0 == NULL)
193 cluster_limit = 0;
194 else
195 cluster_limit = *cluster_limit0;
8a268428
JH
196
197 /* allocate object cache structure */
77652cad 198 oc = kmalloc(__offsetof(struct objcache, cache_percpu[ncpus]),
77e294a1 199 M_OBJCACHE, M_WAITOK | M_ZERO);
59302080 200 oc->name = kstrdup(name, M_TEMP);
5b7da64a
MD
201 oc->ctor = ctor ? ctor : null_ctor;
202 oc->dtor = dtor ? dtor : null_dtor;
698331b0 203 oc->privdata = privdata;
8a268428
JH
204 oc->free = free;
205 oc->allocator_args = allocator_args;
206
207 /* initialize depots */
208 depot = &oc->depot[0];
77e294a1 209
54d017fd 210 spin_init(&depot->spin);
8a268428
JH
211 SLIST_INIT(&depot->fullmagazines);
212 SLIST_INIT(&depot->emptymagazines);
77e294a1 213
8a268428
JH
214 if (mag_capacity == 0)
215 mag_capacity = INITIAL_MAG_CAPACITY;
77e294a1
MD
216
217 /*
218 * The cluster_limit must be sufficient to have three magazines per
a1f6a1fe
MD
219 * cpu. If we have a lot of cpus the mag_capacity might just be
220 * too big, reduce it if necessary.
221 *
222 * Each cpu can hold up to two magazines, with the remainder in the
223 * depot. If many objects are allocated fewer magazines are
224 * available. We have to make sure that each cpu has access to
225 * free objects until the object cache hits 75% of its limit.
77e294a1
MD
226 */
227 if (cluster_limit == 0) {
228 depot->unallocated_objects = -1;
229 } else {
a1f6a1fe
MD
230 factor = 8;
231 need = mag_capacity * ncpus * factor;
232 if (cluster_limit < need && mag_capacity > 16) {
233 kprintf("objcache(%s): too small for ncpus"
234 ", adjusting mag_capacity %d->",
235 name, mag_capacity);
236 while (need > cluster_limit && mag_capacity > 16) {
237 mag_capacity >>= 1;
238 need = mag_capacity * ncpus * factor;
239 }
240 kprintf("%d\n", mag_capacity);
241 }
242 if (cluster_limit < need) {
243 kprintf("objcache(%s): too small for ncpus"
244 ", adjusting cluster_limit %d->%d\n",
245 name, cluster_limit, need);
246 cluster_limit = need;
247 }
77e294a1
MD
248 depot->unallocated_objects = cluster_limit;
249 }
a1f6a1fe 250 depot->magcapacity = mag_capacity;
8a268428 251 oc->alloc = alloc;
8a268428
JH
252
253 /* initialize per-cpu caches */
254 for (cpuid = 0; cpuid < ncpus; cpuid++) {
255 struct percpu_objcache *cache_percpu = &oc->cache_percpu[cpuid];
256
257 cache_percpu->loaded_magazine = mag_alloc(mag_capacity);
258 cache_percpu->previous_magazine = mag_alloc(mag_capacity);
8a268428 259 }
1a21fef5
NT
260
261 /* compute initial number of empty magazines in depot */
262 nmagdepot = 0;
263 if (cluster_limit > 0) {
264 /* max number of magazines in depot */
265 nmagdepot = (cluster_limit - ncpus * 2 * mag_capacity) /
266 mag_capacity;
267
268 /* retain at most 50% of the limit */
269 nmagdepot /= 2;
270 }
271 /* bound result to acceptable range */
272 if (nmagdepot < 2)
273 nmagdepot = 2;
274 if (nmagdepot > 10)
275 nmagdepot = 10;
276
277 /* put empty magazines in depot */
278 for (i = 0; i < nmagdepot; i++) {
279 struct magazine *mag = mag_alloc(mag_capacity);
280 SLIST_INSERT_HEAD(&depot->emptymagazines, mag, nextmagazine);
281 }
282
54d017fd 283 spin_lock_wr(&objcachelist_spin);
00de24de 284 LIST_INSERT_HEAD(&allobjcaches, oc, oc_next);
54d017fd 285 spin_unlock_wr(&objcachelist_spin);
8a268428 286
0aa16b5d
SZ
287 if (cluster_limit0 != NULL)
288 *cluster_limit0 = cluster_limit;
8a268428
JH
289 return (oc);
290}
291
70aac194
MD
292struct objcache *
293objcache_create_simple(malloc_type_t mtype, size_t objsize)
294{
295 struct objcache_malloc_args *margs;
296 struct objcache *oc;
297
efda3bd0 298 margs = kmalloc(sizeof(*margs), M_OBJCACHE, M_WAITOK|M_ZERO);
70aac194
MD
299 margs->objsize = objsize;
300 margs->mtype = mtype;
0aa16b5d 301 oc = objcache_create(mtype->ks_shortdesc, NULL, 0,
5b7da64a 302 NULL, NULL, NULL,
70aac194
MD
303 objcache_malloc_alloc, objcache_malloc_free,
304 margs);
305 return (oc);
306}
307
b1641984
MD
308struct objcache *
309objcache_create_mbacked(malloc_type_t mtype, size_t objsize,
0aa16b5d 310 int *cluster_limit, int mag_capacity,
b1641984 311 objcache_ctor_fn *ctor, objcache_dtor_fn *dtor,
698331b0 312 void *privdata)
b1641984
MD
313{
314 struct objcache_malloc_args *margs;
315 struct objcache *oc;
316
317 margs = kmalloc(sizeof(*margs), M_OBJCACHE, M_WAITOK|M_ZERO);
318 margs->objsize = objsize;
319 margs->mtype = mtype;
320 oc = objcache_create(mtype->ks_shortdesc,
321 cluster_limit, mag_capacity,
698331b0 322 ctor, dtor, privdata,
b1641984
MD
323 objcache_malloc_alloc, objcache_malloc_free,
324 margs);
325 return(oc);
326}
327
328
8a268428 329#define MAGAZINE_EMPTY(mag) (mag->rounds == 0)
77e294a1 330#define MAGAZINE_NOTEMPTY(mag) (mag->rounds != 0)
8a268428
JH
331#define MAGAZINE_FULL(mag) (mag->rounds == mag->capacity)
332
333#define swap(x, y) ({ struct magazine *t = x; x = y; y = t; })
334
335/*
336 * Get an object from the object cache.
9bfc4d6d
MD
337 *
338 * WARNING! ocflags are only used when we have to go to the underlying
339 * allocator, so we cannot depend on flags such as M_ZERO.
8a268428
JH
340 */
341void *
342objcache_get(struct objcache *oc, int ocflags)
343{
344 struct percpu_objcache *cpucache = &oc->cache_percpu[mycpuid];
345 struct magazine *loadedmag;
77e294a1 346 struct magazine *emptymag;
8a268428
JH
347 void *obj;
348 struct magazinedepot *depot;
8a268428 349
5b7da64a 350 KKASSERT((ocflags & M_ZERO) == 0);
8a268428
JH
351 crit_enter();
352 ++cpucache->gets_cumulative;
353
354retry:
355 /*
356 * Loaded magazine has an object. This is the hot path.
357 * It is lock-free and uses a critical section to block
358 * out interrupt handlers on the same processor.
359 */
360 loadedmag = cpucache->loaded_magazine;
77e294a1
MD
361 if (MAGAZINE_NOTEMPTY(loadedmag)) {
362 obj = loadedmag->objects[--loadedmag->rounds];
8a268428
JH
363 crit_exit();
364 return (obj);
365 }
366
367 /* Previous magazine has an object. */
77e294a1 368 if (MAGAZINE_NOTEMPTY(cpucache->previous_magazine)) {
54d017fd
MD
369 KKASSERT(cpucache->previous_magazine->cleaning +
370 cpucache->loaded_magazine->cleaning == 0);
8a268428
JH
371 swap(cpucache->loaded_magazine, cpucache->previous_magazine);
372 loadedmag = cpucache->loaded_magazine;
77e294a1 373 obj = loadedmag->objects[--loadedmag->rounds];
5b7da64a
MD
374 crit_exit();
375 return (obj);
8a268428
JH
376 }
377
378 /*
77e294a1 379 * Both magazines empty. Get a full magazine from the depot and
208c0e5b 380 * move one of the empty ones to the depot.
77e294a1 381 *
54d017fd 382 * Obtain the depot spinlock.
b2560af1 383 *
5b7da64a 384 * NOTE: Beyond this point, M_* flags are handled via oc->alloc()
8a268428 385 */
8a268428 386 depot = &oc->depot[myclusterid];
54d017fd 387 spin_lock_wr(&depot->spin);
208c0e5b
MD
388
389 /*
54d017fd
MD
390 * Recheck the cpucache after obtaining the depot spinlock. This
391 * shouldn't be necessary now but don't take any chances.
208c0e5b
MD
392 */
393 if (MAGAZINE_NOTEMPTY(cpucache->loaded_magazine) ||
394 MAGAZINE_NOTEMPTY(cpucache->previous_magazine)
395 ) {
54d017fd 396 spin_unlock_wr(&depot->spin);
208c0e5b
MD
397 goto retry;
398 }
8a268428
JH
399
400 /* Check if depot has a full magazine. */
401 if (!SLIST_EMPTY(&depot->fullmagazines)) {
77e294a1 402 emptymag = cpucache->previous_magazine;
8a268428
JH
403 cpucache->previous_magazine = cpucache->loaded_magazine;
404 cpucache->loaded_magazine = SLIST_FIRST(&depot->fullmagazines);
8a268428 405 SLIST_REMOVE_HEAD(&depot->fullmagazines, nextmagazine);
77e294a1
MD
406
407 /*
208c0e5b 408 * Return emptymag to the depot.
77e294a1 409 */
208c0e5b
MD
410 KKASSERT(MAGAZINE_EMPTY(emptymag));
411 SLIST_INSERT_HEAD(&depot->emptymagazines,
412 emptymag, nextmagazine);
54d017fd 413 spin_unlock_wr(&depot->spin);
77e294a1 414 goto retry;
8a268428
JH
415 }
416
417 /*
77e294a1
MD
418 * The depot does not have any non-empty magazines. If we have
419 * not hit our object limit we can allocate a new object using
420 * the back-end allocator.
421 *
422 * note: unallocated_objects can be initialized to -1, which has
423 * the effect of removing any allocation limits.
8a268428 424 */
77e294a1
MD
425 if (depot->unallocated_objects) {
426 --depot->unallocated_objects;
54d017fd 427 spin_unlock_wr(&depot->spin);
77e294a1 428 crit_exit();
8a268428 429
77e294a1
MD
430 obj = oc->alloc(oc->allocator_args, ocflags);
431 if (obj) {
698331b0 432 if (oc->ctor(obj, oc->privdata, ocflags))
77e294a1
MD
433 return (obj);
434 oc->free(obj, oc->allocator_args);
8d968f1d
SZ
435 obj = NULL;
436 }
437 if (obj == NULL) {
54d017fd 438 spin_lock_wr(&depot->spin);
77e294a1 439 ++depot->unallocated_objects;
54d017fd 440 spin_unlock_wr(&depot->spin);
77e294a1
MD
441 if (depot->waiting)
442 wakeup(depot);
8d968f1d 443
77e294a1 444 crit_enter();
69982f91
MD
445 /*
446 * makes debugging easier when gets_cumulative does
447 * not include gets_null.
448 */
77e294a1 449 ++cpucache->gets_null;
69982f91 450 --cpucache->gets_cumulative;
77e294a1
MD
451 crit_exit();
452 }
453 return(obj);
8a268428 454 }
a1f6a1fe
MD
455 if (oc->exhausted == 0) {
456 kprintf("Warning, objcache(%s): Exhausted!\n", oc->name);
457 oc->exhausted = 1;
458 }
8a268428 459
77e294a1
MD
460 /*
461 * Otherwise block if allowed to.
462 */
463 if ((ocflags & (M_WAITOK|M_NULLOK)) == M_WAITOK) {
464 ++cpucache->waiting;
465 ++depot->waiting;
54d017fd 466 msleep(depot, &depot->spin, 0, "objcache_get", 0);
77e294a1
MD
467 --cpucache->waiting;
468 --depot->waiting;
54d017fd 469 spin_unlock_wr(&depot->spin);
77e294a1 470 goto retry;
8a268428 471 }
69982f91
MD
472
473 /*
474 * Otherwise fail
475 */
8a268428 476 ++cpucache->gets_null;
69982f91 477 --cpucache->gets_cumulative;
8a268428 478 crit_exit();
54d017fd 479 spin_unlock_wr(&depot->spin);
8a268428
JH
480 return (NULL);
481}
482
483/*
484 * Wrapper for malloc allocation routines.
485 */
486void *
487objcache_malloc_alloc(void *allocator_args, int ocflags)
488{
489 struct objcache_malloc_args *alloc_args = allocator_args;
490
efda3bd0 491 return (kmalloc(alloc_args->objsize, alloc_args->mtype,
8a268428
JH
492 ocflags & OC_MFLAGS));
493}
494
495void
496objcache_malloc_free(void *obj, void *allocator_args)
497{
498 struct objcache_malloc_args *alloc_args = allocator_args;
499
efda3bd0 500 kfree(obj, alloc_args->mtype);
8a268428
JH
501}
502
503/*
504 * Wrapper for allocation policies that pre-allocate at initialization time
505 * and don't do run-time allocation.
506 */
507void *
508objcache_nop_alloc(void *allocator_args, int ocflags)
509{
510 return (NULL);
511}
512
513void
514objcache_nop_free(void *obj, void *allocator_args)
515{
8a268428
JH
516}
517
518/*
519 * Return an object to the object cache.
520 */
521void
522objcache_put(struct objcache *oc, void *obj)
523{
524 struct percpu_objcache *cpucache = &oc->cache_percpu[mycpuid];
525 struct magazine *loadedmag;
526 struct magazinedepot *depot;
8a268428
JH
527
528 crit_enter();
529 ++cpucache->puts_cumulative;
530
531 if (CLUSTER_OF(obj) != myclusterid) {
532#ifdef notyet
533 /* use lazy IPI to send object to owning cluster XXX todo */
534 ++cpucache->puts_othercluster;
69982f91 535 crit_exit();
8a268428
JH
536 return;
537#endif
538 }
539
540retry:
541 /*
542 * Free slot available in loaded magazine. This is the hot path.
543 * It is lock-free and uses a critical section to block out interrupt
544 * handlers on the same processor.
545 */
546 loadedmag = cpucache->loaded_magazine;
547 if (!MAGAZINE_FULL(loadedmag)) {
77e294a1 548 loadedmag->objects[loadedmag->rounds++] = obj;
8a268428 549 if (cpucache->waiting)
6f266dc1 550 wakeup_mycpu(&oc->depot[myclusterid]);
8a268428
JH
551 crit_exit();
552 return;
553 }
554
77e294a1
MD
555 /*
556 * Current magazine full, but previous magazine has room. XXX
557 */
8a268428 558 if (!MAGAZINE_FULL(cpucache->previous_magazine)) {
54d017fd
MD
559 KKASSERT(cpucache->previous_magazine->cleaning +
560 cpucache->loaded_magazine->cleaning == 0);
8a268428
JH
561 swap(cpucache->loaded_magazine, cpucache->previous_magazine);
562 loadedmag = cpucache->loaded_magazine;
77e294a1
MD
563 loadedmag->objects[loadedmag->rounds++] = obj;
564 if (cpucache->waiting)
6f266dc1 565 wakeup_mycpu(&oc->depot[myclusterid]);
77e294a1
MD
566 crit_exit();
567 return;
8a268428
JH
568 }
569
570 /*
77e294a1
MD
571 * Both magazines full. Get an empty magazine from the depot and
572 * move a full loaded magazine to the depot. Even though the
573 * magazine may wind up with space available after we block on
54d017fd 574 * the spinlock, we still cycle it through to avoid the non-optimal
77e294a1
MD
575 * corner-case.
576 *
54d017fd 577 * Obtain the depot spinlock.
8a268428 578 */
8a268428 579 depot = &oc->depot[myclusterid];
54d017fd 580 spin_lock_wr(&depot->spin);
8a268428 581
77e294a1
MD
582 /*
583 * If an empty magazine is available in the depot, cycle it
584 * through and retry.
585 */
8a268428 586 if (!SLIST_EMPTY(&depot->emptymagazines)) {
54d017fd
MD
587 KKASSERT(cpucache->previous_magazine->cleaning +
588 cpucache->loaded_magazine->cleaning == 0);
77e294a1
MD
589 loadedmag = cpucache->previous_magazine;
590 cpucache->previous_magazine = cpucache->loaded_magazine;
591 cpucache->loaded_magazine = SLIST_FIRST(&depot->emptymagazines);
8a268428 592 SLIST_REMOVE_HEAD(&depot->emptymagazines, nextmagazine);
77e294a1
MD
593
594 /*
595 * Return loadedmag to the depot. Due to blocking it may
596 * not be entirely full and could even be empty.
597 */
598 if (MAGAZINE_EMPTY(loadedmag)) {
599 SLIST_INSERT_HEAD(&depot->emptymagazines,
600 loadedmag, nextmagazine);
54d017fd 601 spin_unlock_wr(&depot->spin);
77e294a1 602 } else {
8a268428 603 SLIST_INSERT_HEAD(&depot->fullmagazines,
77e294a1 604 loadedmag, nextmagazine);
54d017fd 605 spin_unlock_wr(&depot->spin);
77e294a1
MD
606 if (depot->waiting)
607 wakeup(depot);
608 }
77e294a1 609 goto retry;
8a268428
JH
610 }
611
77e294a1
MD
612 /*
613 * An empty mag is not available. This is a corner case which can
614 * occur due to cpus holding partially full magazines. Do not try
615 * to allocate a mag, just free the object.
616 */
617 ++depot->unallocated_objects;
54d017fd 618 spin_unlock_wr(&depot->spin);
8a268428
JH
619 if (depot->waiting)
620 wakeup(depot);
8a268428 621 crit_exit();
698331b0 622 oc->dtor(obj, oc->privdata);
77e294a1
MD
623 oc->free(obj, oc->allocator_args);
624}
625
626/*
627 * The object is being put back into the cache, but the caller has
628 * indicated that the object is not in any shape to be reused and should
629 * be dtor'd immediately.
630 */
631void
632objcache_dtor(struct objcache *oc, void *obj)
633{
634 struct magazinedepot *depot;
8a268428 635
77e294a1 636 depot = &oc->depot[myclusterid];
54d017fd 637 spin_lock_wr(&depot->spin);
77e294a1 638 ++depot->unallocated_objects;
54d017fd 639 spin_unlock_wr(&depot->spin);
77e294a1
MD
640 if (depot->waiting)
641 wakeup(depot);
698331b0 642 oc->dtor(obj, oc->privdata);
8a268428 643 oc->free(obj, oc->allocator_args);
8a268428
JH
644}
645
8a268428 646/*
54d017fd
MD
647 * Deallocate all objects in a magazine and free the magazine if requested.
648 * The magazine must already be disassociated from the depot.
649 *
650 * Must be called with a critical section held when called with a per-cpu
651 * magazine. The magazine may be indirectly modified during the loop.
652 *
653 * The number of objects freed is returned.
8a268428
JH
654 */
655static int
54d017fd 656mag_purge(struct objcache *oc, struct magazine *mag, int freeit)
8a268428 657{
54d017fd 658 int count;
8a268428 659 void *obj;
8a268428 660
54d017fd
MD
661 count = 0;
662 ++mag->cleaning;
77e294a1
MD
663 while (mag->rounds) {
664 obj = mag->objects[--mag->rounds];
698331b0 665 oc->dtor(obj, oc->privdata); /* MAY BLOCK */
54d017fd
MD
666 oc->free(obj, oc->allocator_args); /* MAY BLOCK */
667 ++count;
668
669 /*
670 * Cycle for interrupts
671 */
672 if ((count & 15) == 0) {
673 crit_exit();
674 crit_enter();
675 }
8a268428 676 }
54d017fd
MD
677 --mag->cleaning;
678 if (freeit)
679 kfree(mag, M_OBJMAG);
680 return(count);
8a268428
JH
681}
682
683/*
54d017fd
MD
684 * Disassociate zero or more magazines from a magazine list associated with
685 * the depot, update the depot, and move the magazines to a temporary
686 * list.
687 *
688 * The caller must check the depot for waiters and wake it up, typically
689 * after disposing of the magazines this function loads onto the temporary
690 * list.
691 */
692static void
693maglist_disassociate(struct magazinedepot *depot, struct magazinelist *maglist,
694 struct magazinelist *tmplist, boolean_t purgeall)
695{
696 struct magazine *mag;
697
698 while ((mag = SLIST_FIRST(maglist)) != NULL) {
699 SLIST_REMOVE_HEAD(maglist, nextmagazine);
700 SLIST_INSERT_HEAD(tmplist, mag, nextmagazine);
701 depot->unallocated_objects += mag->rounds;
702 }
703}
704
705/*
706 * Deallocate all magazines and their contents from the passed temporary
707 * list. The magazines have already been accounted for by their depots.
708 *
709 * The total number of rounds freed is returned. This number is typically
710 * only used to determine whether a wakeup on the depot is needed or not.
8a268428
JH
711 */
712static int
54d017fd 713maglist_purge(struct objcache *oc, struct magazinelist *maglist)
8a268428
JH
714{
715 struct magazine *mag;
54d017fd 716 int count = 0;
8a268428 717
54d017fd
MD
718 /*
719 * can't use SLIST_FOREACH because blocking releases the depot
720 * spinlock
721 */
722 while ((mag = SLIST_FIRST(maglist)) != NULL) {
8a268428 723 SLIST_REMOVE_HEAD(maglist, nextmagazine);
54d017fd 724 count += mag_purge(oc, mag, TRUE);
8a268428 725 }
54d017fd 726 return(count);
8a268428
JH
727}
728
729/*
730 * De-allocates all magazines on the full and empty magazine lists.
54d017fd
MD
731 *
732 * Because this routine is called with a spinlock held, the magazines
733 * can only be disassociated and moved to a temporary list, not freed.
734 *
735 * The caller is responsible for freeing the magazines.
8a268428
JH
736 */
737static void
54d017fd 738depot_disassociate(struct magazinedepot *depot, struct magazinelist *tmplist)
8a268428 739{
54d017fd
MD
740 maglist_disassociate(depot, &depot->fullmagazines, tmplist, TRUE);
741 maglist_disassociate(depot, &depot->emptymagazines, tmplist, TRUE);
8a268428
JH
742}
743
744#ifdef notneeded
745void
746objcache_reclaim(struct objcache *oc)
747{
748 struct percpu_objcache *cache_percpu = &oc->cache_percpu[myclusterid];
749 struct magazinedepot *depot = &oc->depot[myclusterid];
54d017fd
MD
750 struct magazinelist tmplist;
751 int count;
8a268428 752
54d017fd
MD
753 SLIST_INIT(&tmplist);
754 crit_enter();
755 count = mag_purge(oc, cache_percpu->loaded_magazine, FALSE);
756 count += mag_purge(oc, cache_percpu->previous_magazine, FALSE);
757 crit_exit();
8a268428 758
54d017fd
MD
759 spin_lock_wr(&depot->spin);
760 depot->unallocated_objects += count;
761 depot_disassociate(depot, &tmplist);
762 spin_unlock_wr(&depot->spin);
763 count += maglist_purge(oc, &tmplist);
764 if (count && depot->waiting)
765 wakeup(depot);
8a268428
JH
766}
767#endif
768
769/*
54d017fd 770 * Try to free up some memory. Return as soon as some free memory is found.
8a268428
JH
771 * For each object cache on the reclaim list, first try the current per-cpu
772 * cache, then the full magazine depot.
773 */
774boolean_t
775objcache_reclaimlist(struct objcache *oclist[], int nlist, int ocflags)
776{
777 struct objcache *oc;
778 struct percpu_objcache *cpucache;
779 struct magazinedepot *depot;
54d017fd
MD
780 struct magazinelist tmplist;
781 int i, count;
782
783 SLIST_INIT(&tmplist);
8a268428
JH
784
785 for (i = 0; i < nlist; i++) {
786 oc = oclist[i];
787 cpucache = &oc->cache_percpu[mycpuid];
788 depot = &oc->depot[myclusterid];
789
790 crit_enter();
54d017fd
MD
791 count = mag_purge(oc, cpucache->loaded_magazine, FALSE);
792 if (count == 0)
793 count += mag_purge(oc, cpucache->previous_magazine, FALSE);
794 crit_exit();
795 if (count > 0) {
796 spin_lock_wr(&depot->spin);
797 depot->unallocated_objects += count;
798 spin_unlock_wr(&depot->spin);
799 if (depot->waiting)
77e294a1 800 wakeup(depot);
8a268428
JH
801 return (TRUE);
802 }
54d017fd
MD
803 spin_lock_wr(&depot->spin);
804 maglist_disassociate(depot, &depot->fullmagazines,
805 &tmplist, FALSE);
806 spin_unlock_wr(&depot->spin);
807 count = maglist_purge(oc, &tmplist);
808 if (count > 0) {
809 if (depot->waiting)
77e294a1 810 wakeup(depot);
8a268428
JH
811 return (TRUE);
812 }
8a268428
JH
813 }
814 return (FALSE);
815}
816
817/*
818 * Destroy an object cache. Must have no existing references.
8a268428
JH
819 */
820void
821objcache_destroy(struct objcache *oc)
822{
823 struct percpu_objcache *cache_percpu;
54d017fd 824 struct magazinedepot *depot;
8a268428 825 int clusterid, cpuid;
54d017fd
MD
826 struct magazinelist tmplist;
827
00de24de
NT
828 spin_lock_wr(&objcachelist_spin);
829 LIST_REMOVE(oc, oc_next);
830 spin_unlock_wr(&objcachelist_spin);
831
54d017fd
MD
832 SLIST_INIT(&tmplist);
833 for (clusterid = 0; clusterid < MAXCLUSTERS; clusterid++) {
834 depot = &oc->depot[clusterid];
835 spin_lock_wr(&depot->spin);
836 depot_disassociate(depot, &tmplist);
837 spin_unlock_wr(&depot->spin);
838 }
839 maglist_purge(oc, &tmplist);
8a268428
JH
840
841 for (cpuid = 0; cpuid < ncpus; cpuid++) {
842 cache_percpu = &oc->cache_percpu[cpuid];
843
54d017fd
MD
844 mag_purge(oc, cache_percpu->loaded_magazine, TRUE);
845 mag_purge(oc, cache_percpu->previous_magazine, TRUE);
846 cache_percpu->loaded_magazine = NULL;
847 cache_percpu->previous_magazine = NULL;
848 /* don't bother adjusting depot->unallocated_objects */
8a268428
JH
849 }
850
efda3bd0
MD
851 kfree(oc->name, M_TEMP);
852 kfree(oc, M_OBJCACHE);
8a268428
JH
853}
854
77e294a1 855#if 0
8a268428
JH
856/*
857 * Populate the per-cluster depot with elements from a linear block
858 * of memory. Must be called for individually for each cluster.
859 * Populated depots should not be destroyed.
860 */
861void
862objcache_populate_linear(struct objcache *oc, void *base, int nelts, int size)
863{
864 char *p = base;
865 char *end = (char *)base + (nelts * size);
866 struct magazinedepot *depot = &oc->depot[myclusterid];
54d017fd 867 struct magazine *emptymag = mag_alloc(depot->magcapcity);
8a268428 868
8a268428 869 while (p < end) {
54d017fd 870 emptymag->objects[emptymag->rounds++] = p;
8a268428 871 if (MAGAZINE_FULL(emptymag)) {
54d017fd 872 spin_lock_wr(&depot->spin);
8a268428
JH
873 SLIST_INSERT_HEAD(&depot->fullmagazines, emptymag,
874 nextmagazine);
54d017fd
MD
875 depot->unallocated_objects += emptymag->rounds;
876 spin_unlock_wr(&depot->spin);
877 if (depot->waiting)
878 wakeup(depot);
879 emptymag = mag_alloc(depot->magcapacity);
8a268428 880 }
8a268428
JH
881 p += size;
882 }
54d017fd
MD
883 if (MAGAZINE_EMPTY(emptymag)) {
884 mag_purge(oc, emptymag, TRUE);
885 } else {
886 spin_lock_wr(&depot->spin);
887 SLIST_INSERT_HEAD(&depot->fullmagazines, emptymag,
888 nextmagazine);
889 depot->unallocated_objects += emptymag->rounds;
890 spin_unlock_wr(&depot->spin);
891 if (depot->waiting)
892 wakeup(depot);
893 emptymag = mag_alloc(depot->magcapacity);
894 }
8a268428 895}
77e294a1 896#endif
8a268428 897
77e294a1 898#if 0
8a268428
JH
899/*
900 * Check depot contention once a minute.
901 * 2 contested locks per second allowed.
902 */
903static int objcache_rebalance_period;
904static const int objcache_contention_rate = 120;
905static struct callout objcache_callout;
906
907#define MAXMAGSIZE 512
908
909/*
910 * Check depot contention and increase magazine size if necessary.
911 */
912static void
913objcache_timer(void *dummy)
914{
915 struct objcache *oc;
916 struct magazinedepot *depot;
54d017fd
MD
917 struct magazinelist tmplist;
918
919 XXX we need to detect when an objcache is destroyed out from under
920 us XXX
921
922 SLIST_INIT(&tmplist);
8a268428 923
54d017fd 924 spin_lock_wr(&objcachelist_spin);
00de24de 925 LIST_FOREACH(oc, &allobjcaches, oc_next) {
8a268428
JH
926 depot = &oc->depot[myclusterid];
927 if (depot->magcapacity < MAXMAGSIZE) {
928 if (depot->contested > objcache_contention_rate) {
54d017fd
MD
929 spin_lock_wr(&depot->spin);
930 depot_disassociate(depot, &tmplist);
8a268428 931 depot->magcapacity *= 2;
54d017fd 932 spin_unlock_wr(&depot->spin);
6ea70f76 933 kprintf("objcache_timer: increasing cache %s"
8a268428
JH
934 " magsize to %d, contested %d times\n",
935 oc->name, depot->magcapacity,
936 depot->contested);
937 }
938 depot->contested = 0;
939 }
54d017fd
MD
940 spin_unlock_wr(&objcachelist_spin);
941 if (maglist_purge(oc, &tmplist) > 0 && depot->waiting)
942 wakeup(depot);
943 spin_lock_wr(&objcachelist_spin);
8a268428 944 }
54d017fd 945 spin_unlock_wr(&objcachelist_spin);
8a268428
JH
946
947 callout_reset(&objcache_callout, objcache_rebalance_period,
948 objcache_timer, NULL);
949}
950
77e294a1
MD
951#endif
952
8a268428
JH
953static void
954objcache_init(void)
955{
54d017fd 956 spin_init(&objcachelist_spin);
77e294a1 957#if 0
8a268428 958 callout_init(&objcache_callout);
77e294a1 959 objcache_rebalance_period = 60 * hz;
8a268428
JH
960 callout_reset(&objcache_callout, objcache_rebalance_period,
961 objcache_timer, NULL);
77e294a1 962#endif
8a268428 963}
ba39e2e0 964SYSINIT(objcache, SI_BOOT2_OBJCACHE, SI_ORDER_FIRST, objcache_init, 0);