[dragonfly.git] / sys / kern / kern_sysref.c

/*
 * Copyright (c) 2007 The DragonFly Project.  All rights reserved.
 * 
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * 
 * $DragonFly: src/sys/kern/kern_sysref.c,v 1.7 2008/10/26 04:29:19 sephe Exp $
 */
/*
 * System resource control module for all cluster-addressable system resource
 * structures.
 *
 * This module implements the core ref counting, sysid registration, and
 * objcache-backed allocation mechanism for all major system resource
 * structures.
 *
 * sysid registrations operate via the objcache ctor/dtor mechanism and
 * sysids will be reused if the resource is not explicitly accessed via
 * its sysid.  This removes all RB tree handling overhead from the critical
 * path for locally used resources.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/tree.h>
#include <sys/spinlock.h>
#include <machine/atomic.h>
#include <machine/cpufunc.h>

#include <sys/spinlock2.h>
#include <sys/sysref2.h>

static boolean_t sysref_ctor(void *data, void *privdata, int ocflags);
static void sysref_dtor(void *data, void *privdata);

/*
 * Red-Black tree support
 */
static int rb_sysref_compare(struct sysref *sr1, struct sysref *sr2);
RB_GENERATE2(sysref_rb_tree, sysref, rbnode, rb_sysref_compare, sysid_t, sysid);

static struct srpercpu {
	struct sysref_rb_tree rbtree;
	struct spinlock spin;
} sysref_array[MAXCPU];

static void
sysrefbootinit(void *dummy __unused)
{
	struct srpercpu *sa;
	int i;

	for (i = 0; i < ncpus; ++i) {
		sa = &sysref_array[i];
		spin_init(&sa->spin);
		RB_INIT(&sa->rbtree);
	}
}

SYSINIT(sysref, SI_BOOT2_MACHDEP, SI_ORDER_ANY, sysrefbootinit, NULL);

static
int
rb_sysref_compare(struct sysref *sr1, struct sysref *sr2)
{
	if (sr1->sysid < sr2->sysid)
		return(-1);
	if (sr1->sysid > sr2->sysid)
		return(1);
	return(0);
}

/*
 * Manual initialization of a resource structure's sysref, only used during
 * booting to set up certain statically declared resources which cannot
 * be deallocated.
 */
void
sysref_init(struct sysref *sr, struct sysref_class *srclass)
{
	struct srpercpu *sa;
	globaldata_t gd;

	gd = mycpu;
	crit_enter_gd(gd);
	gd->gd_sysid_alloc += ncpus_fit; /* next unique sysid */
	sr->sysid = gd->gd_sysid_alloc;
	KKASSERT(((int)sr->sysid & ncpus_fit_mask) == gd->gd_cpuid);
	sr->refcnt = -0x40000000;
	sr->flags = 0;
	sr->srclass = srclass;

	sa = &sysref_array[gd->gd_cpuid];
	spin_lock_wr(&sa->spin);
	sysref_rb_tree_RB_INSERT(&sa->rbtree, sr);
	spin_unlock_wr(&sa->spin);
	crit_exit_gd(gd);
}

/*
 * Allocate a resource structure of the specified class, initialize a
 * sysid and add the resource to the RB tree.  The caller must complete
 * initialization of the resource and call sysref_activate() to activate it.
 */
void *
sysref_alloc(struct sysref_class *srclass)
{
	struct sysref *sr;
	char *data;
	int n;

	/*
	 * Create the object cache backing store.
	 */
	if (srclass->oc == NULL) {
		KKASSERT(srclass->mtype != NULL);
		srclass->oc = objcache_create_mbacked(
				srclass->mtype, srclass->objsize, 
				NULL, srclass->mag_capacity,
				sysref_ctor, sysref_dtor, srclass);
	}

	/*
	 * Allocate the resource.
	 */
	data = objcache_get(srclass->oc, M_WAITOK);
	sr = (struct sysref *)(data + srclass->offset);

	/*
	 * Refcnt isn't touched while it is zero.  The objcache ctor
	 * function has already allocated a sysid and emplaced the
	 * structure in the RB tree.
	 */
	KKASSERT(sr->refcnt == 0);
	sr->refcnt = -0x40000000;

	/*
	 * Clean out the structure unless the caller wants to deal with
	 * it (e.g. like the vmspace code).
	 */
	if ((srclass->flags & SRC_MANAGEDINIT) == 0) {
		if (srclass->offset != 0)
			bzero(data, srclass->offset);
		n = srclass->offset + sizeof(struct sysref);
		KKASSERT(n <= srclass->objsize);
		if (n != srclass->objsize)
			bzero(data + n, srclass->objsize - n);
	}
	return(data);
}

/*
 * Object cache backing store ctor function.
 *
 * This allocates the sysid and associates the structure with the
 * red-black tree, allowing it to be looked up.  The actual resource
 * structure has NOT yet been allocated so it is marked free.
 *
 * If the sysid is not used to access the resource, we will just
 * allow the sysid to be reused when the resource structure is reused,
 * allowing the RB tree operation to be 'cached'.  This results in
 * virtually no performance penalty for using the sysref facility.
 */
static
boolean_t
sysref_ctor(void *data, void *privdata, int ocflags)
{
	globaldata_t gd;
	struct srpercpu *sa;
	struct sysref_class *srclass = privdata;
	struct sysref *sr = (void *)((char *)data + srclass->offset);

	/*
	 * Resource structures need to be cleared when allocating from
	 * malloc backing store.  This is different from the zeroing
	 * that we do in sysref_alloc().
	 */
	bzero(data, srclass->objsize);

	/*
	 * Resources managed by our objcache do the sysid and RB tree
	 * handling in the objcache ctor/dtor, so we can reuse the
	 * structure without re-treeing it over and over again.
	 */
	gd = mycpu;
	crit_enter_gd(gd);
	gd->gd_sysid_alloc += ncpus_fit; /* next unique sysid */
	sr->sysid = gd->gd_sysid_alloc;
	KKASSERT(((int)sr->sysid & ncpus_fit_mask) == gd->gd_cpuid);
	/* sr->refcnt= 0; already zero */
	sr->flags = SRF_ALLOCATED;
	sr->srclass = srclass;

	sa = &sysref_array[gd->gd_cpuid];
	spin_lock_wr(&sa->spin);
	sysref_rb_tree_RB_INSERT(&sa->rbtree, sr);
	spin_unlock_wr(&sa->spin);
	crit_exit_gd(gd);

	/*
	 * Execute the class's ctor function, if any.  NOTE: The class
	 * should not try to zero out the structure, we've already handled
	 * that and preinitialized the sysref.
	 *
	 * XXX ignores return value for now
	 */
	if (srclass->ctor)
		srclass->ctor(data, privdata, ocflags);
	return TRUE;
}

/*
 * Object cache destructor, allowing the structure to be returned
 * to the system memory pool.  The resource structure must be
 * removed from the RB tree.  All other references have already
 * been destroyed and the RB tree will not create any new references
 * to the structure in its current state.
 */
static
void
sysref_dtor(void *data, void *privdata)
{
	struct srpercpu *sa;
	struct sysref_class *srclass = privdata;
	struct sysref *sr = (void *)((char *)data + srclass->offset);

	KKASSERT(sr->refcnt == 0);
	sa = &sysref_array[(int)sr->sysid & ncpus_fit_mask];
	spin_lock_wr(&sa->spin);
	sysref_rb_tree_RB_REMOVE(&sa->rbtree, sr);
	spin_unlock_wr(&sa->spin);
	if (srclass->dtor)
		srclass->dtor(data, privdata);
}

/*
 * Activate or reactivate a resource. 0x40000001 is added to the ref count
 * so -0x40000000 (during initialization) will translate to a ref count of 1.
 * Any references made during initialization will translate to additional
 * positive ref counts.
 *
 * MPSAFE
 */
void
sysref_activate(struct sysref *sr)
{
	int count;

	for (;;) {
		count = sr->refcnt;
		KASSERT(count < 0 && count + 0x40000001 > 0,
			("sysref_activate: bad count %08x", count));
		if (atomic_cmpset_int(&sr->refcnt, count, count + 0x40000001))
			break;
		cpu_pause();
	}
}

/*
 * Release a reference under special circumstances.  This call is made
 * from the sysref_put() inline from sys/sysref2.h for any 1->0 transitions,
 * negative->negative 'termination in progress' transitions, and when the
 * cmpset instruction fails during a normal transition.
 *
 * This function is called from the sysref_put() inline in sys/sysref2.h,
 * but handles all cases regardless.
 */
void
_sysref_put(struct sysref *sr)
{
	int count;
	void *data;

	for (;;) {
		count = sr->refcnt;
		if (count > 1) {
			/*
			 * release 1 count, nominal case, active resource
			 * structure, no other action required.
			 */
			if (atomic_cmpset_int(&sr->refcnt, count, count - 1))
				break;
		} else if (count == 1) {
			/*
			 * 1->0 transitions transition to -0x40000000 instead,
			 * placing the resource structure into a termination-
			 * in-progress state.  The termination function is
			 * then called.
			 */
			if (atomic_cmpset_int(&sr->refcnt, count, -0x40000000)) {
				data = (char *)sr - sr->srclass->offset;
				sr->srclass->ops.terminate(data);
				break;
			}
		} else if (count > -0x40000000) {
			/*
			 * release 1 count, nominal case, resource undergoing
			 * termination.  The Resource can be ref'd and
			 * deref'd while undergoing termination.
			 */
			if (atomic_cmpset_int(&sr->refcnt, count, count - 1))
				break;
		} else {
			/*
			 * Final release, set refcnt to 0.
			 * Resource must have been allocated.
			 *
			 * If SRF_SYSIDUSED is not set just objcache_put() the
			 * resource, otherwise objcache_dtor() the resource.
			 */
			KKASSERT(count == -0x40000000);
			if (atomic_cmpset_int(&sr->refcnt, count, 0)) {
				KKASSERT(sr->flags & SRF_ALLOCATED);
				data = (char *)sr - sr->srclass->offset;
				if (sr->flags & SRF_SYSIDUSED)
					objcache_dtor(sr->srclass->oc, data);
				else
					objcache_put(sr->srclass->oc, data);
				break;
			}
		}
		/* loop until the cmpset succeeds */
		cpu_pause();
	}
}

sysid_t
allocsysid(void)
{
	globaldata_t gd = mycpu;
	sysid_t sysid;

	crit_enter_gd(gd);
	gd->gd_sysid_alloc += ncpus_fit;
	sysid = gd->gd_sysid_alloc;
	crit_exit_gd(gd);
	return(sysid);
}
Commit	Line	Data
10aa77c0 MD	1	/*
	2	* Copyright (c) 2007 The DragonFly Project. All rights reserved.
	3	*
	4	* This code is derived from software contributed to The DragonFly Project
	5	* by Matthew Dillon <dillon@backplane.com>
	6	*
	7	* Redistribution and use in source and binary forms, with or without
	8	* modification, are permitted provided that the following conditions
	9	* are met:
	10	*
	11	* 1. Redistributions of source code must retain the above copyright
	12	* notice, this list of conditions and the following disclaimer.
	13	* 2. Redistributions in binary form must reproduce the above copyright
	14	* notice, this list of conditions and the following disclaimer in
	15	* the documentation and/or other materials provided with the
	16	* distribution.
	17	* 3. Neither the name of The DragonFly Project nor the names of its
	18	* contributors may be used to endorse or promote products derived
	19	* from this software without specific, prior written permission.
	20	*
	21	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	22	* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	23	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	24	* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
	25	* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
	26	* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
	27	* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	28	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	29	* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	30	* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
	31	* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	32	* SUCH DAMAGE.
	33	*
0aa16b5d	34	* $DragonFly: src/sys/kern/kern_sysref.c,v 1.7 2008/10/26 04:29:19 sephe Exp $
10aa77c0 MD	35	*/
	36	/*
	37	* System resource control module for all cluster-addressable system resource
	38	* structures.
	39	*
	40	* This module implements the core ref counting, sysid registration, and
	41	* objcache-backed allocation mechanism for all major system resource
	42	* structures.
	43	*
	44	* sysid registrations operate via the objcache ctor/dtor mechanism and
	45	* sysids will be reused if the resource is not explicitly accessed via
	46	* its sysid. This removes all RB tree handling overhead from the critical
	47	* path for locally used resources.
	48	*/
	49
	50	#include <sys/param.h>
	51	#include <sys/systm.h>
	52	#include <sys/kernel.h>
	53	#include <sys/tree.h>
10aa77c0 MD	54	#include <sys/spinlock.h>
	55	#include <machine/atomic.h>
	56	#include <machine/cpufunc.h>
	57
	58	#include <sys/spinlock2.h>
	59	#include <sys/sysref2.h>
	60
698331b0 MD	61	static boolean_t sysref_ctor(void data, void privdata, int ocflags);
698331b0 MD	62	static void sysref_dtor(void data, void privdata);
10aa77c0 MD	63
	64	/*
	65	* Red-Black tree support
	66	*/
	67	static int rb_sysref_compare(struct sysref sr1, struct sysref sr2);
	68	RB_GENERATE2(sysref_rb_tree, sysref, rbnode, rb_sysref_compare, sysid_t, sysid);
	69
	70	static struct srpercpu {
	71	struct sysref_rb_tree rbtree;
	72	struct spinlock spin;
	73	} sysref_array[MAXCPU];
	74
	75	static void
	76	sysrefbootinit(void *dummy __unused)
	77	{
	78	struct srpercpu *sa;
	79	int i;
	80
	81	for (i = 0; i < ncpus; ++i) {
	82	sa = &sysref_array[i];
	83	spin_init(&sa->spin);
	84	RB_INIT(&sa->rbtree);
	85	}
	86	}
	87
ba39e2e0	88	SYSINIT(sysref, SI_BOOT2_MACHDEP, SI_ORDER_ANY, sysrefbootinit, NULL);
10aa77c0 MD	89
	90	static
	91	int
	92	rb_sysref_compare(struct sysref sr1, struct sysref sr2)
	93	{
	94	if (sr1->sysid < sr2->sysid)
	95	return(-1);
	96	if (sr1->sysid > sr2->sysid)
	97	return(1);
	98	return(0);
	99	}
	100
	101	/*
	102	* Manual initialization of a resource structure's sysref, only used during
	103	* booting to set up certain statically declared resources which cannot
	104	* be deallocated.
	105	*/
	106	void
698331b0	107	sysref_init(struct sysref sr, struct sysref_class srclass)
10aa77c0 MD	108	{
	109	struct srpercpu *sa;
	110	globaldata_t gd;
	111
	112	gd = mycpu;
	113	crit_enter_gd(gd);
	114	gd->gd_sysid_alloc += ncpus_fit; /* next unique sysid */
	115	sr->sysid = gd->gd_sysid_alloc;
	116	KKASSERT(((int)sr->sysid & ncpus_fit_mask) == gd->gd_cpuid);
	117	sr->refcnt = -0x40000000;
	118	sr->flags = 0;
698331b0	119	sr->srclass = srclass;
10aa77c0 MD	120
	121	sa = &sysref_array[gd->gd_cpuid];
	122	spin_lock_wr(&sa->spin);
	123	sysref_rb_tree_RB_INSERT(&sa->rbtree, sr);
	124	spin_unlock_wr(&sa->spin);
	125	crit_exit_gd(gd);
	126	}
	127
	128	/*
	129	* Allocate a resource structure of the specified class, initialize a
	130	* sysid and add the resource to the RB tree. The caller must complete
	131	* initialization of the resource and call sysref_activate() to activate it.
	132	*/
	133	void *
698331b0	134	sysref_alloc(struct sysref_class *srclass)
10aa77c0 MD	135	{
	136	struct sysref *sr;
	137	char *data;
	138	int n;
	139
	140	/*
	141	* Create the object cache backing store.
	142	*/
698331b0 MD	143	if (srclass->oc == NULL) {
	144	KKASSERT(srclass->mtype != NULL);
	145	srclass->oc = objcache_create_mbacked(
	146	srclass->mtype, srclass->objsize,
0aa16b5d	147	NULL, srclass->mag_capacity,
698331b0	148	sysref_ctor, sysref_dtor, srclass);
10aa77c0 MD	149	}
	150
	151	/*
	152	* Allocate the resource.
	153	*/
698331b0 MD	154	data = objcache_get(srclass->oc, M_WAITOK);
698331b0 MD	155	sr = (struct sysref *)(data + srclass->offset);
10aa77c0 MD	156
	157	/*
	158	* Refcnt isn't touched while it is zero. The objcache ctor
	159	* function has already allocated a sysid and emplaced the
	160	* structure in the RB tree.
	161	*/
	162	KKASSERT(sr->refcnt == 0);
	163	sr->refcnt = -0x40000000;
	164
	165	/*
	166	* Clean out the structure unless the caller wants to deal with
	167	* it (e.g. like the vmspace code).
	168	*/
698331b0 MD	169	if ((srclass->flags & SRC_MANAGEDINIT) == 0) {
	170	if (srclass->offset != 0)
	171	bzero(data, srclass->offset);
	172	n = srclass->offset + sizeof(struct sysref);
	173	KKASSERT(n <= srclass->objsize);
	174	if (n != srclass->objsize)
	175	bzero(data + n, srclass->objsize - n);
10aa77c0 MD	176	}
	177	return(data);
	178	}
	179
	180	/*
	181	* Object cache backing store ctor function.
	182	*
	183	* This allocates the sysid and associates the structure with the
	184	* red-black tree, allowing it to be looked up. The actual resource
	185	* structure has NOT yet been allocated so it is marked free.
	186	*
	187	* If the sysid is not used to access the resource, we will just
	188	* allow the sysid to be reused when the resource structure is reused,
	189	* allowing the RB tree operation to be 'cached'. This results in
	190	* virtually no performance penalty for using the sysref facility.
	191	*/
	192	static
	193	boolean_t
698331b0	194	sysref_ctor(void data, void privdata, int ocflags)
10aa77c0 MD	195	{
	196	globaldata_t gd;
	197	struct srpercpu *sa;
698331b0 MD	198	struct sysref_class *srclass = privdata;
698331b0 MD	199	struct sysref sr = (void )((char *)data + srclass->offset);
10aa77c0 MD	200
	201	/*
	202	* Resource structures need to be cleared when allocating from
	203	* malloc backing store. This is different from the zeroing
	204	* that we do in sysref_alloc().
	205	*/
698331b0	206	bzero(data, srclass->objsize);
10aa77c0 MD	207
	208	/*
	209	* Resources managed by our objcache do the sysid and RB tree
	210	* handling in the objcache ctor/dtor, so we can reuse the
	211	* structure without re-treeing it over and over again.
	212	*/
	213	gd = mycpu;
	214	crit_enter_gd(gd);
	215	gd->gd_sysid_alloc += ncpus_fit; /* next unique sysid */
	216	sr->sysid = gd->gd_sysid_alloc;
	217	KKASSERT(((int)sr->sysid & ncpus_fit_mask) == gd->gd_cpuid);
	218	/* sr->refcnt= 0; already zero */
	219	sr->flags = SRF_ALLOCATED;
698331b0	220	sr->srclass = srclass;
10aa77c0 MD	221
	222	sa = &sysref_array[gd->gd_cpuid];
	223	spin_lock_wr(&sa->spin);
	224	sysref_rb_tree_RB_INSERT(&sa->rbtree, sr);
	225	spin_unlock_wr(&sa->spin);
	226	crit_exit_gd(gd);
	227
	228	/*
	229	* Execute the class's ctor function, if any. NOTE: The class
	230	* should not try to zero out the structure, we've already handled
	231	* that and preinitialized the sysref.
	232	*
	233	* XXX ignores return value for now
	234	*/
698331b0 MD	235	if (srclass->ctor)
698331b0 MD	236	srclass->ctor(data, privdata, ocflags);
10aa77c0 MD	237	return TRUE;
	238	}
	239
	240	/*
	241	* Object cache destructor, allowing the structure to be returned
	242	* to the system memory pool. The resource structure must be
	243	* removed from the RB tree. All other references have already
	244	* been destroyed and the RB tree will not create any new references
	245	* to the structure in its current state.
	246	*/
	247	static
	248	void
698331b0	249	sysref_dtor(void data, void privdata)
10aa77c0 MD	250	{
10aa77c0 MD	251	struct srpercpu *sa;
698331b0 MD	252	struct sysref_class *srclass = privdata;
698331b0 MD	253	struct sysref sr = (void )((char *)data + srclass->offset);
10aa77c0 MD	254
	255	KKASSERT(sr->refcnt == 0);
	256	sa = &sysref_array[(int)sr->sysid & ncpus_fit_mask];
	257	spin_lock_wr(&sa->spin);
	258	sysref_rb_tree_RB_REMOVE(&sa->rbtree, sr);
	259	spin_unlock_wr(&sa->spin);
698331b0 MD	260	if (srclass->dtor)
698331b0 MD	261	srclass->dtor(data, privdata);
10aa77c0 MD	262	}
	263
	264	/*
	265	* Activate or reactivate a resource. 0x40000001 is added to the ref count
	266	* so -0x40000000 (during initialization) will translate to a ref count of 1.
	267	* Any references made during initialization will translate to additional
	268	* positive ref counts.
b0911300 MD	269	*
b0911300 MD	270	* MPSAFE
10aa77c0 MD	271	*/
	272	void
	273	sysref_activate(struct sysref *sr)
	274	{
	275	int count;
	276
	277	for (;;) {
	278	count = sr->refcnt;
3c37c940 MD	279	KASSERT(count < 0 && count + 0x40000001 > 0,
3c37c940 MD	280	("sysref_activate: bad count %08x", count));
10aa77c0 MD	281	if (atomic_cmpset_int(&sr->refcnt, count, count + 0x40000001))
	282	break;
	283	cpu_pause();
	284	}
	285	}
	286
	287	/*
	288	* Release a reference under special circumstances. This call is made
	289	* from the sysref_put() inline from sys/sysref2.h for any 1->0 transitions,
	290	* negative->negative 'termination in progress' transitions, and when the
	291	* cmpset instruction fails during a normal transition.
	292	*
	293	* This function is called from the sysref_put() inline in sys/sysref2.h,
	294	* but handles all cases regardless.
	295	*/
	296	void
	297	_sysref_put(struct sysref *sr)
	298	{
	299	int count;
	300	void *data;
	301
	302	for (;;) {
	303	count = sr->refcnt;
	304	if (count > 1) {
	305	/*
	306	* release 1 count, nominal case, active resource
	307	* structure, no other action required.
	308	*/
	309	if (atomic_cmpset_int(&sr->refcnt, count, count - 1))
	310	break;
	311	} else if (count == 1) {
	312	/*
	313	* 1->0 transitions transition to -0x40000000 instead,
	314	* placing the resource structure into a termination-
	315	* in-progress state. The termination function is
	316	* then called.
	317	*/
	318	if (atomic_cmpset_int(&sr->refcnt, count, -0x40000000)) {
698331b0 MD	319	data = (char *)sr - sr->srclass->offset;
698331b0 MD	320	sr->srclass->ops.terminate(data);
10aa77c0 MD	321	break;
	322	}
	323	} else if (count > -0x40000000) {
	324	/*
	325	* release 1 count, nominal case, resource undergoing
	326	* termination. The Resource can be ref'd and
	327	* deref'd while undergoing termination.
	328	*/
	329	if (atomic_cmpset_int(&sr->refcnt, count, count - 1))
	330	break;
	331	} else {
	332	/*
	333	* Final release, set refcnt to 0.
	334	* Resource must have been allocated.
	335	*
	336	* If SRF_SYSIDUSED is not set just objcache_put() the
	337	* resource, otherwise objcache_dtor() the resource.
	338	*/
63f45d59	339	KKASSERT(count == -0x40000000);
10aa77c0	340	if (atomic_cmpset_int(&sr->refcnt, count, 0)) {
63f45d59	341	KKASSERT(sr->flags & SRF_ALLOCATED);
698331b0	342	data = (char *)sr - sr->srclass->offset;
10aa77c0	343	if (sr->flags & SRF_SYSIDUSED)
698331b0	344	objcache_dtor(sr->srclass->oc, data);
10aa77c0	345	else
698331b0	346	objcache_put(sr->srclass->oc, data);
10aa77c0 MD	347	break;
	348	}
	349	}
	350	/* loop until the cmpset succeeds */
	351	cpu_pause();
	352	}
	353	}
	354
3551ce6b MD	355	sysid_t
	356	allocsysid(void)
	357	{
	358	globaldata_t gd = mycpu;
	359	sysid_t sysid;
	360
	361	crit_enter_gd(gd);
	362	gd->gd_sysid_alloc += ncpus_fit;
	363	sysid = gd->gd_sysid_alloc;
	364	crit_exit_gd(gd);
	365	return(sysid);
	366	}
	367