kernel - MPSAFE stabilization
[dragonfly.git] / sys / kern / kern_sysref.c
CommitLineData
10aa77c0
MD
1/*
2 * Copyright (c) 2007 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
0aa16b5d 34 * $DragonFly: src/sys/kern/kern_sysref.c,v 1.7 2008/10/26 04:29:19 sephe Exp $
10aa77c0
MD
35 */
36/*
37 * System resource control module for all cluster-addressable system resource
38 * structures.
39 *
40 * This module implements the core ref counting, sysid registration, and
41 * objcache-backed allocation mechanism for all major system resource
42 * structures.
43 *
44 * sysid registrations operate via the objcache ctor/dtor mechanism and
45 * sysids will be reused if the resource is not explicitly accessed via
46 * its sysid. This removes all RB tree handling overhead from the critical
47 * path for locally used resources.
48 */
49
50#include <sys/param.h>
51#include <sys/systm.h>
52#include <sys/kernel.h>
53#include <sys/tree.h>
10aa77c0
MD
54#include <sys/spinlock.h>
55#include <machine/atomic.h>
56#include <machine/cpufunc.h>
57
58#include <sys/spinlock2.h>
59#include <sys/sysref2.h>
60
698331b0
MD
61static boolean_t sysref_ctor(void *data, void *privdata, int ocflags);
62static void sysref_dtor(void *data, void *privdata);
10aa77c0
MD
63
64/*
65 * Red-Black tree support
66 */
67static int rb_sysref_compare(struct sysref *sr1, struct sysref *sr2);
68RB_GENERATE2(sysref_rb_tree, sysref, rbnode, rb_sysref_compare, sysid_t, sysid);
69
70static struct srpercpu {
71 struct sysref_rb_tree rbtree;
72 struct spinlock spin;
73} sysref_array[MAXCPU];
74
75static void
76sysrefbootinit(void *dummy __unused)
77{
78 struct srpercpu *sa;
79 int i;
80
81 for (i = 0; i < ncpus; ++i) {
82 sa = &sysref_array[i];
83 spin_init(&sa->spin);
84 RB_INIT(&sa->rbtree);
85 }
86}
87
ba39e2e0 88SYSINIT(sysref, SI_BOOT2_MACHDEP, SI_ORDER_ANY, sysrefbootinit, NULL);
10aa77c0
MD
89
90static
91int
92rb_sysref_compare(struct sysref *sr1, struct sysref *sr2)
93{
94 if (sr1->sysid < sr2->sysid)
95 return(-1);
96 if (sr1->sysid > sr2->sysid)
97 return(1);
98 return(0);
99}
100
101/*
102 * Manual initialization of a resource structure's sysref, only used during
103 * booting to set up certain statically declared resources which cannot
104 * be deallocated.
105 */
106void
698331b0 107sysref_init(struct sysref *sr, struct sysref_class *srclass)
10aa77c0
MD
108{
109 struct srpercpu *sa;
110 globaldata_t gd;
111
112 gd = mycpu;
113 crit_enter_gd(gd);
114 gd->gd_sysid_alloc += ncpus_fit; /* next unique sysid */
115 sr->sysid = gd->gd_sysid_alloc;
116 KKASSERT(((int)sr->sysid & ncpus_fit_mask) == gd->gd_cpuid);
117 sr->refcnt = -0x40000000;
118 sr->flags = 0;
698331b0 119 sr->srclass = srclass;
10aa77c0
MD
120
121 sa = &sysref_array[gd->gd_cpuid];
122 spin_lock_wr(&sa->spin);
123 sysref_rb_tree_RB_INSERT(&sa->rbtree, sr);
124 spin_unlock_wr(&sa->spin);
125 crit_exit_gd(gd);
126}
127
128/*
129 * Allocate a resource structure of the specified class, initialize a
130 * sysid and add the resource to the RB tree. The caller must complete
131 * initialization of the resource and call sysref_activate() to activate it.
132 */
133void *
698331b0 134sysref_alloc(struct sysref_class *srclass)
10aa77c0
MD
135{
136 struct sysref *sr;
137 char *data;
138 int n;
139
140 /*
141 * Create the object cache backing store.
142 */
698331b0
MD
143 if (srclass->oc == NULL) {
144 KKASSERT(srclass->mtype != NULL);
145 srclass->oc = objcache_create_mbacked(
146 srclass->mtype, srclass->objsize,
0aa16b5d 147 NULL, srclass->mag_capacity,
698331b0 148 sysref_ctor, sysref_dtor, srclass);
10aa77c0
MD
149 }
150
151 /*
152 * Allocate the resource.
153 */
698331b0
MD
154 data = objcache_get(srclass->oc, M_WAITOK);
155 sr = (struct sysref *)(data + srclass->offset);
10aa77c0
MD
156
157 /*
158 * Refcnt isn't touched while it is zero. The objcache ctor
159 * function has already allocated a sysid and emplaced the
160 * structure in the RB tree.
161 */
162 KKASSERT(sr->refcnt == 0);
163 sr->refcnt = -0x40000000;
164
165 /*
166 * Clean out the structure unless the caller wants to deal with
167 * it (e.g. like the vmspace code).
168 */
698331b0
MD
169 if ((srclass->flags & SRC_MANAGEDINIT) == 0) {
170 if (srclass->offset != 0)
171 bzero(data, srclass->offset);
172 n = srclass->offset + sizeof(struct sysref);
173 KKASSERT(n <= srclass->objsize);
174 if (n != srclass->objsize)
175 bzero(data + n, srclass->objsize - n);
10aa77c0
MD
176 }
177 return(data);
178}
179
180/*
181 * Object cache backing store ctor function.
182 *
183 * This allocates the sysid and associates the structure with the
184 * red-black tree, allowing it to be looked up. The actual resource
185 * structure has NOT yet been allocated so it is marked free.
186 *
187 * If the sysid is not used to access the resource, we will just
188 * allow the sysid to be reused when the resource structure is reused,
189 * allowing the RB tree operation to be 'cached'. This results in
190 * virtually no performance penalty for using the sysref facility.
191 */
192static
193boolean_t
698331b0 194sysref_ctor(void *data, void *privdata, int ocflags)
10aa77c0
MD
195{
196 globaldata_t gd;
197 struct srpercpu *sa;
698331b0
MD
198 struct sysref_class *srclass = privdata;
199 struct sysref *sr = (void *)((char *)data + srclass->offset);
10aa77c0
MD
200
201 /*
202 * Resource structures need to be cleared when allocating from
203 * malloc backing store. This is different from the zeroing
204 * that we do in sysref_alloc().
205 */
698331b0 206 bzero(data, srclass->objsize);
10aa77c0
MD
207
208 /*
209 * Resources managed by our objcache do the sysid and RB tree
210 * handling in the objcache ctor/dtor, so we can reuse the
211 * structure without re-treeing it over and over again.
212 */
213 gd = mycpu;
214 crit_enter_gd(gd);
215 gd->gd_sysid_alloc += ncpus_fit; /* next unique sysid */
216 sr->sysid = gd->gd_sysid_alloc;
217 KKASSERT(((int)sr->sysid & ncpus_fit_mask) == gd->gd_cpuid);
218 /* sr->refcnt= 0; already zero */
219 sr->flags = SRF_ALLOCATED;
698331b0 220 sr->srclass = srclass;
10aa77c0
MD
221
222 sa = &sysref_array[gd->gd_cpuid];
223 spin_lock_wr(&sa->spin);
224 sysref_rb_tree_RB_INSERT(&sa->rbtree, sr);
225 spin_unlock_wr(&sa->spin);
226 crit_exit_gd(gd);
227
228 /*
229 * Execute the class's ctor function, if any. NOTE: The class
230 * should not try to zero out the structure, we've already handled
231 * that and preinitialized the sysref.
232 *
233 * XXX ignores return value for now
234 */
698331b0
MD
235 if (srclass->ctor)
236 srclass->ctor(data, privdata, ocflags);
10aa77c0
MD
237 return TRUE;
238}
239
240/*
241 * Object cache destructor, allowing the structure to be returned
242 * to the system memory pool. The resource structure must be
243 * removed from the RB tree. All other references have already
244 * been destroyed and the RB tree will not create any new references
245 * to the structure in its current state.
246 */
247static
248void
698331b0 249sysref_dtor(void *data, void *privdata)
10aa77c0
MD
250{
251 struct srpercpu *sa;
698331b0
MD
252 struct sysref_class *srclass = privdata;
253 struct sysref *sr = (void *)((char *)data + srclass->offset);
10aa77c0
MD
254
255 KKASSERT(sr->refcnt == 0);
256 sa = &sysref_array[(int)sr->sysid & ncpus_fit_mask];
257 spin_lock_wr(&sa->spin);
258 sysref_rb_tree_RB_REMOVE(&sa->rbtree, sr);
259 spin_unlock_wr(&sa->spin);
698331b0
MD
260 if (srclass->dtor)
261 srclass->dtor(data, privdata);
10aa77c0
MD
262}
263
264/*
265 * Activate or reactivate a resource. 0x40000001 is added to the ref count
266 * so -0x40000000 (during initialization) will translate to a ref count of 1.
267 * Any references made during initialization will translate to additional
268 * positive ref counts.
b0911300
MD
269 *
270 * MPSAFE
10aa77c0
MD
271 */
272void
273sysref_activate(struct sysref *sr)
274{
275 int count;
276
277 for (;;) {
278 count = sr->refcnt;
3c37c940
MD
279 KASSERT(count < 0 && count + 0x40000001 > 0,
280 ("sysref_activate: bad count %08x", count));
10aa77c0
MD
281 if (atomic_cmpset_int(&sr->refcnt, count, count + 0x40000001))
282 break;
283 cpu_pause();
284 }
285}
286
287/*
288 * Release a reference under special circumstances. This call is made
289 * from the sysref_put() inline from sys/sysref2.h for any 1->0 transitions,
290 * negative->negative 'termination in progress' transitions, and when the
291 * cmpset instruction fails during a normal transition.
292 *
293 * This function is called from the sysref_put() inline in sys/sysref2.h,
294 * but handles all cases regardless.
295 */
296void
297_sysref_put(struct sysref *sr)
298{
299 int count;
300 void *data;
301
302 for (;;) {
303 count = sr->refcnt;
304 if (count > 1) {
305 /*
306 * release 1 count, nominal case, active resource
307 * structure, no other action required.
308 */
309 if (atomic_cmpset_int(&sr->refcnt, count, count - 1))
310 break;
311 } else if (count == 1) {
312 /*
313 * 1->0 transitions transition to -0x40000000 instead,
314 * placing the resource structure into a termination-
315 * in-progress state. The termination function is
316 * then called.
317 */
318 if (atomic_cmpset_int(&sr->refcnt, count, -0x40000000)) {
698331b0
MD
319 data = (char *)sr - sr->srclass->offset;
320 sr->srclass->ops.terminate(data);
10aa77c0
MD
321 break;
322 }
323 } else if (count > -0x40000000) {
324 /*
325 * release 1 count, nominal case, resource undergoing
326 * termination. The Resource can be ref'd and
327 * deref'd while undergoing termination.
328 */
329 if (atomic_cmpset_int(&sr->refcnt, count, count - 1))
330 break;
331 } else {
332 /*
333 * Final release, set refcnt to 0.
334 * Resource must have been allocated.
335 *
336 * If SRF_SYSIDUSED is not set just objcache_put() the
337 * resource, otherwise objcache_dtor() the resource.
338 */
63f45d59 339 KKASSERT(count == -0x40000000);
10aa77c0 340 if (atomic_cmpset_int(&sr->refcnt, count, 0)) {
63f45d59 341 KKASSERT(sr->flags & SRF_ALLOCATED);
698331b0 342 data = (char *)sr - sr->srclass->offset;
10aa77c0 343 if (sr->flags & SRF_SYSIDUSED)
698331b0 344 objcache_dtor(sr->srclass->oc, data);
10aa77c0 345 else
698331b0 346 objcache_put(sr->srclass->oc, data);
10aa77c0
MD
347 break;
348 }
349 }
350 /* loop until the cmpset succeeds */
351 cpu_pause();
352 }
353}
354
3551ce6b
MD
355sysid_t
356allocsysid(void)
357{
358 globaldata_t gd = mycpu;
359 sysid_t sysid;
360
361 crit_enter_gd(gd);
362 gd->gd_sysid_alloc += ncpus_fit;
363 sysid = gd->gd_sysid_alloc;
364 crit_exit_gd(gd);
365 return(sysid);
366}
367