2 * Copyright (c) 2005 Jeffrey M. Hsu. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of The DragonFly Project nor the names of its
16 * contributors may be used to endorse or promote products derived
17 * from this software without specific, prior written permission.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
22 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
23 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
27 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
29 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * $DragonFly: src/sys/kern/kern_objcache.c,v 1.23 2008/10/26 04:29:19 sephe Exp $
35 #include <sys/param.h>
36 #include <sys/kernel.h>
37 #include <sys/systm.h>
38 #include <sys/callout.h>
39 #include <sys/globaldata.h>
40 #include <sys/malloc.h>
41 #include <sys/queue.h>
42 #include <sys/objcache.h>
43 #include <sys/spinlock.h>
44 #include <sys/thread.h>
45 #include <sys/thread2.h>
46 #include <sys/spinlock2.h>
48 static MALLOC_DEFINE(M_OBJCACHE, "objcache", "Object Cache");
49 static MALLOC_DEFINE(M_OBJMAG, "objcache magazine", "Object Cache Magazine");
51 #define INITIAL_MAG_CAPACITY 64
56 SLIST_ENTRY(magazine) nextmagazine;
60 SLIST_HEAD(magazinelist, magazine);
63 * per-cluster cache of magazines
65 * All fields in this structure are protected by the spinlock.
67 struct magazinedepot {
69 * The per-cpu object caches only exchanges completely full or
70 * completely empty magazines with the depot layer, so only have
71 * to cache these two types of magazines.
73 struct magazinelist fullmagazines;
74 struct magazinelist emptymagazines;
77 /* protect this structure */
80 /* magazines not yet allocated towards limit */
81 int unallocated_objects;
83 /* infrequently used fields */
84 int waiting; /* waiting for another cpu to
85 * return a full magazine to
87 int contested; /* depot contention count */
91 * per-cpu object cache
92 * All fields in this structure are protected by crit_enter().
94 struct percpu_objcache {
95 struct magazine *loaded_magazine; /* active magazine */
96 struct magazine *previous_magazine; /* backup magazine */
99 int gets_cumulative; /* total calls to get */
100 int gets_null; /* objcache_get returned NULL */
101 int puts_cumulative; /* total calls to put */
102 int puts_othercluster; /* returned to other cluster */
104 /* infrequently used fields */
105 int waiting; /* waiting for a thread on this cpu to
106 * return an obj to the per-cpu cache */
109 /* only until we have NUMA cluster topology information XXX */
110 #define MAXCLUSTERS 1
111 #define myclusterid 0
112 #define CLUSTER_OF(obj) 0
115 * Two-level object cache consisting of NUMA cluster-level depots of
116 * fully loaded or completely empty magazines and cpu-level caches of
117 * individual objects.
122 /* object constructor and destructor from blank storage */
123 objcache_ctor_fn *ctor;
124 objcache_dtor_fn *dtor;
127 /* interface to underlying allocator */
128 objcache_alloc_fn *alloc;
129 objcache_free_fn *free;
130 void *allocator_args;
132 LIST_ENTRY(objcache) oc_next;
133 int exhausted; /* oops */
135 /* NUMA-cluster level caches */
136 struct magazinedepot depot[MAXCLUSTERS];
138 struct percpu_objcache cache_percpu[]; /* per-cpu caches */
141 static struct spinlock objcachelist_spin;
142 static LIST_HEAD(objcachelist, objcache) allobjcaches;
144 static struct magazine *
145 mag_alloc(int capacity)
147 struct magazine *mag;
149 mag = kmalloc(__offsetof(struct magazine, objects[capacity]),
150 M_OBJMAG, M_INTWAIT | M_ZERO);
151 mag->capacity = capacity;
157 * Utility routine for objects that don't require any de-construction.
161 null_dtor(void *obj, void *privdata)
167 null_ctor(void *obj, void *privdata, int ocflags)
173 * Create an object cache.
176 objcache_create(const char *name, int *cluster_limit0, int mag_capacity,
177 objcache_ctor_fn *ctor, objcache_dtor_fn *dtor, void *privdata,
178 objcache_alloc_fn *alloc, objcache_free_fn *free,
179 void *allocator_args)
182 struct magazinedepot *depot;
190 if (cluster_limit0 == NULL)
193 cluster_limit = *cluster_limit0;
195 /* allocate object cache structure */
196 oc = kmalloc(__offsetof(struct objcache, cache_percpu[ncpus]),
197 M_OBJCACHE, M_WAITOK | M_ZERO);
198 oc->name = kstrdup(name, M_TEMP);
199 oc->ctor = ctor ? ctor : null_ctor;
200 oc->dtor = dtor ? dtor : null_dtor;
201 oc->privdata = privdata;
203 oc->allocator_args = allocator_args;
205 /* initialize depots */
206 depot = &oc->depot[0];
208 spin_init(&depot->spin);
209 SLIST_INIT(&depot->fullmagazines);
210 SLIST_INIT(&depot->emptymagazines);
212 if (mag_capacity == 0)
213 mag_capacity = INITIAL_MAG_CAPACITY;
216 * The cluster_limit must be sufficient to have three magazines per
217 * cpu. If we have a lot of cpus the mag_capacity might just be
218 * too big, reduce it if necessary.
220 * Each cpu can hold up to two magazines, with the remainder in the
221 * depot. If many objects are allocated fewer magazines are
222 * available. We have to make sure that each cpu has access to
223 * free objects until the object cache hits 75% of its limit.
225 if (cluster_limit == 0) {
226 depot->unallocated_objects = -1;
229 need = mag_capacity * ncpus * factor;
230 if (cluster_limit < need && mag_capacity > 16) {
231 kprintf("objcache(%s): too small for ncpus"
232 ", adjusting mag_capacity %d->",
234 while (need > cluster_limit && mag_capacity > 16) {
236 need = mag_capacity * ncpus * factor;
238 kprintf("%d\n", mag_capacity);
240 if (cluster_limit < need) {
241 kprintf("objcache(%s): too small for ncpus"
242 ", adjusting cluster_limit %d->%d\n",
243 name, cluster_limit, need);
244 cluster_limit = need;
246 depot->unallocated_objects = cluster_limit;
248 depot->magcapacity = mag_capacity;
251 /* initialize per-cpu caches */
252 for (cpuid = 0; cpuid < ncpus; cpuid++) {
253 struct percpu_objcache *cache_percpu = &oc->cache_percpu[cpuid];
255 cache_percpu->loaded_magazine = mag_alloc(mag_capacity);
256 cache_percpu->previous_magazine = mag_alloc(mag_capacity);
259 /* compute initial number of empty magazines in depot */
261 if (cluster_limit > 0) {
262 /* max number of magazines in depot */
263 nmagdepot = (cluster_limit - ncpus * 2 * mag_capacity) /
266 /* retain at most 50% of the limit */
269 /* bound result to acceptable range */
272 if (nmagdepot > 1000)
275 /* put empty magazines in depot */
276 for (i = 0; i < nmagdepot; i++) {
277 struct magazine *mag = mag_alloc(mag_capacity);
278 SLIST_INSERT_HEAD(&depot->emptymagazines, mag, nextmagazine);
281 spin_lock(&objcachelist_spin);
282 LIST_INSERT_HEAD(&allobjcaches, oc, oc_next);
283 spin_unlock(&objcachelist_spin);
285 if (cluster_limit0 != NULL)
286 *cluster_limit0 = cluster_limit;
291 objcache_create_simple(malloc_type_t mtype, size_t objsize)
293 struct objcache_malloc_args *margs;
296 margs = kmalloc(sizeof(*margs), M_OBJCACHE, M_WAITOK|M_ZERO);
297 margs->objsize = objsize;
298 margs->mtype = mtype;
299 oc = objcache_create(mtype->ks_shortdesc, NULL, 0,
301 objcache_malloc_alloc, objcache_malloc_free,
307 objcache_create_mbacked(malloc_type_t mtype, size_t objsize,
308 int *cluster_limit, int mag_capacity,
309 objcache_ctor_fn *ctor, objcache_dtor_fn *dtor,
312 struct objcache_malloc_args *margs;
315 margs = kmalloc(sizeof(*margs), M_OBJCACHE, M_WAITOK|M_ZERO);
316 margs->objsize = objsize;
317 margs->mtype = mtype;
318 oc = objcache_create(mtype->ks_shortdesc,
319 cluster_limit, mag_capacity,
320 ctor, dtor, privdata,
321 objcache_malloc_alloc, objcache_malloc_free,
327 #define MAGAZINE_EMPTY(mag) (mag->rounds == 0)
328 #define MAGAZINE_NOTEMPTY(mag) (mag->rounds != 0)
329 #define MAGAZINE_FULL(mag) (mag->rounds == mag->capacity)
331 #define swap(x, y) ({ struct magazine *t = x; x = y; y = t; })
334 * Get an object from the object cache.
336 * WARNING! ocflags are only used when we have to go to the underlying
337 * allocator, so we cannot depend on flags such as M_ZERO.
340 objcache_get(struct objcache *oc, int ocflags)
342 struct percpu_objcache *cpucache = &oc->cache_percpu[mycpuid];
343 struct magazine *loadedmag;
344 struct magazine *emptymag;
346 struct magazinedepot *depot;
348 KKASSERT((ocflags & M_ZERO) == 0);
350 ++cpucache->gets_cumulative;
354 * Loaded magazine has an object. This is the hot path.
355 * It is lock-free and uses a critical section to block
356 * out interrupt handlers on the same processor.
358 loadedmag = cpucache->loaded_magazine;
359 if (MAGAZINE_NOTEMPTY(loadedmag)) {
360 obj = loadedmag->objects[--loadedmag->rounds];
365 /* Previous magazine has an object. */
366 if (MAGAZINE_NOTEMPTY(cpucache->previous_magazine)) {
367 swap(cpucache->loaded_magazine, cpucache->previous_magazine);
368 loadedmag = cpucache->loaded_magazine;
369 obj = loadedmag->objects[--loadedmag->rounds];
375 * Both magazines empty. Get a full magazine from the depot and
376 * move one of the empty ones to the depot.
378 * Obtain the depot spinlock.
380 * NOTE: Beyond this point, M_* flags are handled via oc->alloc()
382 depot = &oc->depot[myclusterid];
383 spin_lock(&depot->spin);
386 * Recheck the cpucache after obtaining the depot spinlock. This
387 * shouldn't be necessary now but don't take any chances.
389 if (MAGAZINE_NOTEMPTY(cpucache->loaded_magazine) ||
390 MAGAZINE_NOTEMPTY(cpucache->previous_magazine)
392 spin_unlock(&depot->spin);
396 /* Check if depot has a full magazine. */
397 if (!SLIST_EMPTY(&depot->fullmagazines)) {
398 emptymag = cpucache->previous_magazine;
399 cpucache->previous_magazine = cpucache->loaded_magazine;
400 cpucache->loaded_magazine = SLIST_FIRST(&depot->fullmagazines);
401 SLIST_REMOVE_HEAD(&depot->fullmagazines, nextmagazine);
404 * Return emptymag to the depot.
406 KKASSERT(MAGAZINE_EMPTY(emptymag));
407 SLIST_INSERT_HEAD(&depot->emptymagazines,
408 emptymag, nextmagazine);
409 spin_unlock(&depot->spin);
414 * The depot does not have any non-empty magazines. If we have
415 * not hit our object limit we can allocate a new object using
416 * the back-end allocator.
418 * note: unallocated_objects can be initialized to -1, which has
419 * the effect of removing any allocation limits.
421 if (depot->unallocated_objects) {
422 --depot->unallocated_objects;
423 spin_unlock(&depot->spin);
426 obj = oc->alloc(oc->allocator_args, ocflags);
428 if (oc->ctor(obj, oc->privdata, ocflags))
430 oc->free(obj, oc->allocator_args);
434 spin_lock(&depot->spin);
435 ++depot->unallocated_objects;
436 spin_unlock(&depot->spin);
442 * makes debugging easier when gets_cumulative does
443 * not include gets_null.
445 ++cpucache->gets_null;
446 --cpucache->gets_cumulative;
451 if (oc->exhausted == 0) {
452 kprintf("Warning, objcache(%s): Exhausted!\n", oc->name);
457 * Otherwise block if allowed to.
459 if ((ocflags & (M_WAITOK|M_NULLOK)) == M_WAITOK) {
462 ssleep(depot, &depot->spin, 0, "objcache_get", 0);
465 spin_unlock(&depot->spin);
472 ++cpucache->gets_null;
473 --cpucache->gets_cumulative;
475 spin_unlock(&depot->spin);
480 * Wrapper for malloc allocation routines.
483 objcache_malloc_alloc(void *allocator_args, int ocflags)
485 struct objcache_malloc_args *alloc_args = allocator_args;
487 return (kmalloc(alloc_args->objsize, alloc_args->mtype,
488 ocflags & OC_MFLAGS));
492 objcache_malloc_free(void *obj, void *allocator_args)
494 struct objcache_malloc_args *alloc_args = allocator_args;
496 kfree(obj, alloc_args->mtype);
500 * Wrapper for allocation policies that pre-allocate at initialization time
501 * and don't do run-time allocation.
504 objcache_nop_alloc(void *allocator_args, int ocflags)
510 objcache_nop_free(void *obj, void *allocator_args)
515 * Return an object to the object cache.
518 objcache_put(struct objcache *oc, void *obj)
520 struct percpu_objcache *cpucache = &oc->cache_percpu[mycpuid];
521 struct magazine *loadedmag;
522 struct magazinedepot *depot;
525 ++cpucache->puts_cumulative;
527 if (CLUSTER_OF(obj) != myclusterid) {
529 /* use lazy IPI to send object to owning cluster XXX todo */
530 ++cpucache->puts_othercluster;
538 * Free slot available in loaded magazine. This is the hot path.
539 * It is lock-free and uses a critical section to block out interrupt
540 * handlers on the same processor.
542 loadedmag = cpucache->loaded_magazine;
543 if (!MAGAZINE_FULL(loadedmag)) {
544 loadedmag->objects[loadedmag->rounds++] = obj;
545 if (cpucache->waiting)
546 wakeup_mycpu(&oc->depot[myclusterid]);
552 * Current magazine full, but previous magazine has room. XXX
554 if (!MAGAZINE_FULL(cpucache->previous_magazine)) {
555 swap(cpucache->loaded_magazine, cpucache->previous_magazine);
556 loadedmag = cpucache->loaded_magazine;
557 loadedmag->objects[loadedmag->rounds++] = obj;
558 if (cpucache->waiting)
559 wakeup_mycpu(&oc->depot[myclusterid]);
565 * Both magazines full. Get an empty magazine from the depot and
566 * move a full loaded magazine to the depot. Even though the
567 * magazine may wind up with space available after we block on
568 * the spinlock, we still cycle it through to avoid the non-optimal
571 * Obtain the depot spinlock.
573 depot = &oc->depot[myclusterid];
574 spin_lock(&depot->spin);
577 * If an empty magazine is available in the depot, cycle it
580 if (!SLIST_EMPTY(&depot->emptymagazines)) {
581 loadedmag = cpucache->previous_magazine;
582 cpucache->previous_magazine = cpucache->loaded_magazine;
583 cpucache->loaded_magazine = SLIST_FIRST(&depot->emptymagazines);
584 SLIST_REMOVE_HEAD(&depot->emptymagazines, nextmagazine);
587 * Return loadedmag to the depot. Due to blocking it may
588 * not be entirely full and could even be empty.
590 if (MAGAZINE_EMPTY(loadedmag)) {
591 SLIST_INSERT_HEAD(&depot->emptymagazines,
592 loadedmag, nextmagazine);
593 spin_unlock(&depot->spin);
595 SLIST_INSERT_HEAD(&depot->fullmagazines,
596 loadedmag, nextmagazine);
597 spin_unlock(&depot->spin);
605 * An empty mag is not available. This is a corner case which can
606 * occur due to cpus holding partially full magazines. Do not try
607 * to allocate a mag, just free the object.
609 ++depot->unallocated_objects;
610 spin_unlock(&depot->spin);
614 oc->dtor(obj, oc->privdata);
615 oc->free(obj, oc->allocator_args);
619 * The object is being put back into the cache, but the caller has
620 * indicated that the object is not in any shape to be reused and should
621 * be dtor'd immediately.
624 objcache_dtor(struct objcache *oc, void *obj)
626 struct magazinedepot *depot;
628 depot = &oc->depot[myclusterid];
629 spin_lock(&depot->spin);
630 ++depot->unallocated_objects;
631 spin_unlock(&depot->spin);
634 oc->dtor(obj, oc->privdata);
635 oc->free(obj, oc->allocator_args);
639 * Deallocate all objects in a magazine and free the magazine if requested.
640 * When freeit is TRUE the magazine must already be disassociated from the
643 * Must be called with a critical section held when called with a per-cpu
644 * magazine. The magazine may be indirectly modified during the loop.
646 * If the magazine moves during a dtor the operation is aborted. This is
647 * only allowed when freeit is FALSE.
649 * The number of objects freed is returned.
652 mag_purge(struct objcache *oc, struct magazine **magp, int freeit)
654 struct magazine *mag = *magp;
659 while (mag->rounds) {
660 obj = mag->objects[--mag->rounds];
661 oc->dtor(obj, oc->privdata); /* MAY BLOCK */
662 oc->free(obj, oc->allocator_args); /* MAY BLOCK */
666 * Cycle for interrupts.
668 if ((count & 15) == 0) {
674 * mag may have become invalid either due to dtor/free
675 * blocking or interrupt cycling, do not derefernce it
679 kprintf("mag_purge: mag ripped out\n");
684 KKASSERT(*magp == mag);
686 kfree(mag, M_OBJMAG);
692 * Disassociate zero or more magazines from a magazine list associated with
693 * the depot, update the depot, and move the magazines to a temporary
696 * The caller must check the depot for waiters and wake it up, typically
697 * after disposing of the magazines this function loads onto the temporary
701 maglist_disassociate(struct magazinedepot *depot, struct magazinelist *maglist,
702 struct magazinelist *tmplist, boolean_t purgeall)
704 struct magazine *mag;
706 while ((mag = SLIST_FIRST(maglist)) != NULL) {
707 SLIST_REMOVE_HEAD(maglist, nextmagazine);
708 SLIST_INSERT_HEAD(tmplist, mag, nextmagazine);
709 depot->unallocated_objects += mag->rounds;
714 * Deallocate all magazines and their contents from the passed temporary
715 * list. The magazines have already been accounted for by their depots.
717 * The total number of rounds freed is returned. This number is typically
718 * only used to determine whether a wakeup on the depot is needed or not.
721 maglist_purge(struct objcache *oc, struct magazinelist *maglist)
723 struct magazine *mag;
727 * can't use SLIST_FOREACH because blocking releases the depot
731 while ((mag = SLIST_FIRST(maglist)) != NULL) {
732 SLIST_REMOVE_HEAD(maglist, nextmagazine);
733 count += mag_purge(oc, &mag, TRUE);
740 * De-allocates all magazines on the full and empty magazine lists.
742 * Because this routine is called with a spinlock held, the magazines
743 * can only be disassociated and moved to a temporary list, not freed.
745 * The caller is responsible for freeing the magazines.
748 depot_disassociate(struct magazinedepot *depot, struct magazinelist *tmplist)
750 maglist_disassociate(depot, &depot->fullmagazines, tmplist, TRUE);
751 maglist_disassociate(depot, &depot->emptymagazines, tmplist, TRUE);
756 objcache_reclaim(struct objcache *oc)
758 struct percpu_objcache *cache_percpu = &oc->cache_percpu[myclusterid];
759 struct magazinedepot *depot = &oc->depot[myclusterid];
760 struct magazinelist tmplist;
763 SLIST_INIT(&tmplist);
765 count = mag_purge(oc, &cache_percpu->loaded_magazine, FALSE);
766 count += mag_purge(oc, &cache_percpu->previous_magazine, FALSE);
769 spin_lock(&depot->spin);
770 depot->unallocated_objects += count;
771 depot_disassociate(depot, &tmplist);
772 spin_unlock(&depot->spin);
773 count += maglist_purge(oc, &tmplist);
774 if (count && depot->waiting)
780 * Try to free up some memory. Return as soon as some free memory is found.
781 * For each object cache on the reclaim list, first try the current per-cpu
782 * cache, then the full magazine depot.
785 objcache_reclaimlist(struct objcache *oclist[], int nlist, int ocflags)
788 struct percpu_objcache *cpucache;
789 struct magazinedepot *depot;
790 struct magazinelist tmplist;
793 kprintf("objcache_reclaimlist\n");
795 SLIST_INIT(&tmplist);
797 for (i = 0; i < nlist; i++) {
799 cpucache = &oc->cache_percpu[mycpuid];
800 depot = &oc->depot[myclusterid];
803 count = mag_purge(oc, &cpucache->loaded_magazine, FALSE);
805 count += mag_purge(oc, &cpucache->previous_magazine, FALSE);
808 spin_lock(&depot->spin);
809 depot->unallocated_objects += count;
810 spin_unlock(&depot->spin);
815 spin_lock(&depot->spin);
816 maglist_disassociate(depot, &depot->fullmagazines,
818 spin_unlock(&depot->spin);
819 count = maglist_purge(oc, &tmplist);
830 * Destroy an object cache. Must have no existing references.
833 objcache_destroy(struct objcache *oc)
835 struct percpu_objcache *cache_percpu;
836 struct magazinedepot *depot;
837 int clusterid, cpuid;
838 struct magazinelist tmplist;
840 spin_lock(&objcachelist_spin);
841 LIST_REMOVE(oc, oc_next);
842 spin_unlock(&objcachelist_spin);
844 SLIST_INIT(&tmplist);
845 for (clusterid = 0; clusterid < MAXCLUSTERS; clusterid++) {
846 depot = &oc->depot[clusterid];
847 spin_lock(&depot->spin);
848 depot_disassociate(depot, &tmplist);
849 spin_unlock(&depot->spin);
851 maglist_purge(oc, &tmplist);
853 for (cpuid = 0; cpuid < ncpus; cpuid++) {
854 cache_percpu = &oc->cache_percpu[cpuid];
857 mag_purge(oc, &cache_percpu->loaded_magazine, TRUE);
858 mag_purge(oc, &cache_percpu->previous_magazine, TRUE);
860 cache_percpu->loaded_magazine = NULL;
861 cache_percpu->previous_magazine = NULL;
862 /* don't bother adjusting depot->unallocated_objects */
865 kfree(oc->name, M_TEMP);
866 kfree(oc, M_OBJCACHE);
871 * Populate the per-cluster depot with elements from a linear block
872 * of memory. Must be called for individually for each cluster.
873 * Populated depots should not be destroyed.
876 objcache_populate_linear(struct objcache *oc, void *base, int nelts, int size)
879 char *end = (char *)base + (nelts * size);
880 struct magazinedepot *depot = &oc->depot[myclusterid];
881 struct magazine *emptymag = mag_alloc(depot->magcapcity);
884 emptymag->objects[emptymag->rounds++] = p;
885 if (MAGAZINE_FULL(emptymag)) {
886 spin_lock_wr(&depot->spin);
887 SLIST_INSERT_HEAD(&depot->fullmagazines, emptymag,
889 depot->unallocated_objects += emptymag->rounds;
890 spin_unlock_wr(&depot->spin);
893 emptymag = mag_alloc(depot->magcapacity);
897 if (MAGAZINE_EMPTY(emptymag)) {
899 mag_purge(oc, &emptymag, TRUE);
902 spin_lock_wr(&depot->spin);
903 SLIST_INSERT_HEAD(&depot->fullmagazines, emptymag,
905 depot->unallocated_objects += emptymag->rounds;
906 spin_unlock_wr(&depot->spin);
909 emptymag = mag_alloc(depot->magcapacity);
916 * Check depot contention once a minute.
917 * 2 contested locks per second allowed.
919 static int objcache_rebalance_period;
920 static const int objcache_contention_rate = 120;
921 static struct callout objcache_callout;
923 #define MAXMAGSIZE 512
926 * Check depot contention and increase magazine size if necessary.
929 objcache_timer(void *dummy)
932 struct magazinedepot *depot;
933 struct magazinelist tmplist;
935 XXX we need to detect when an objcache is destroyed out from under
938 SLIST_INIT(&tmplist);
940 spin_lock_wr(&objcachelist_spin);
941 LIST_FOREACH(oc, &allobjcaches, oc_next) {
942 depot = &oc->depot[myclusterid];
943 if (depot->magcapacity < MAXMAGSIZE) {
944 if (depot->contested > objcache_contention_rate) {
945 spin_lock_wr(&depot->spin);
946 depot_disassociate(depot, &tmplist);
947 depot->magcapacity *= 2;
948 spin_unlock_wr(&depot->spin);
949 kprintf("objcache_timer: increasing cache %s"
950 " magsize to %d, contested %d times\n",
951 oc->name, depot->magcapacity,
954 depot->contested = 0;
956 spin_unlock_wr(&objcachelist_spin);
957 if (maglist_purge(oc, &tmplist) > 0 && depot->waiting)
959 spin_lock_wr(&objcachelist_spin);
961 spin_unlock_wr(&objcachelist_spin);
963 callout_reset(&objcache_callout, objcache_rebalance_period,
964 objcache_timer, NULL);
972 spin_init(&objcachelist_spin);
974 callout_init(&objcache_callout);
975 objcache_rebalance_period = 60 * hz;
976 callout_reset(&objcache_callout, objcache_rebalance_period,
977 objcache_timer, NULL);
980 SYSINIT(objcache, SI_BOOT2_OBJCACHE, SI_ORDER_FIRST, objcache_init, 0);