[dragonfly.git] / sys / kern / lwkt_serialize.c

/*
 * Copyright (c) 2005 The DragonFly Project.  All rights reserved.
 * 
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * 
 * $DragonFly: src/sys/kern/lwkt_serialize.c,v 1.18 2008/10/04 14:22:44 swildner Exp $
 */
/*
 * This API provides a fast locked-bus-cycle-based serializer.  It's
 * basically a low level NON-RECURSIVE exclusive lock that can be held across
 * a blocking condition.  It is NOT a mutex.
 *
 * This serializer is primarily designed for low level situations and
 * interrupt/device interaction.  There are two primary facilities.  First,
 * the serializer facility itself.  Second, an integrated interrupt handler 
 * disablement facility.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/proc.h>
#include <sys/rtprio.h>
#include <sys/queue.h>
#include <sys/thread2.h>
#include <sys/serialize.h>
#include <sys/sysctl.h>
#include <sys/ktr.h>
#include <sys/kthread.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/specialreg.h>
#include <sys/lock.h>
#include <sys/caps.h>

struct exp_backoff {
	int backoff;
	int round;
	lwkt_serialize_t s;
};

#define SLZ_KTR_STRING		"slz=%p"
#define SLZ_KTR_ARGS		lwkt_serialize_t slz

#ifndef KTR_SERIALIZER
#define KTR_SERIALIZER	KTR_ALL
#endif

KTR_INFO_MASTER(slz);
KTR_INFO(KTR_SERIALIZER, slz, enter_beg, 0, SLZ_KTR_STRING, SLZ_KTR_ARGS);
KTR_INFO(KTR_SERIALIZER, slz, sleep_beg, 1, SLZ_KTR_STRING, SLZ_KTR_ARGS);
KTR_INFO(KTR_SERIALIZER, slz, sleep_end, 2, SLZ_KTR_STRING, SLZ_KTR_ARGS);
KTR_INFO(KTR_SERIALIZER, slz, exit_end, 3, SLZ_KTR_STRING, SLZ_KTR_ARGS);
KTR_INFO(KTR_SERIALIZER, slz, wakeup_beg, 4, SLZ_KTR_STRING, SLZ_KTR_ARGS);
KTR_INFO(KTR_SERIALIZER, slz, wakeup_end, 5, SLZ_KTR_STRING, SLZ_KTR_ARGS);
KTR_INFO(KTR_SERIALIZER, slz, try, 6, SLZ_KTR_STRING, SLZ_KTR_ARGS);
KTR_INFO(KTR_SERIALIZER, slz, tryfail, 7, SLZ_KTR_STRING, SLZ_KTR_ARGS);
KTR_INFO(KTR_SERIALIZER, slz, tryok, 8, SLZ_KTR_STRING, SLZ_KTR_ARGS);
#ifdef SMP
KTR_INFO(KTR_SERIALIZER, slz, spinbo, 9,
	 "slz=%p bo1=%d bo=%d", lwkt_serialize_t slz, int backoff1, int backoff);
#endif
KTR_INFO(KTR_SERIALIZER, slz, enter_end, 10, SLZ_KTR_STRING, SLZ_KTR_ARGS);
KTR_INFO(KTR_SERIALIZER, slz, exit_beg, 11, SLZ_KTR_STRING, SLZ_KTR_ARGS);

#define logslz(name, slz)		KTR_LOG(slz_ ## name, slz)
#ifdef SMP
#define logslz_spinbo(slz, bo1, bo)	KTR_LOG(slz_spinbo, slz, bo1, bo)
#endif

static void lwkt_serialize_sleep(void *info);
static void lwkt_serialize_wakeup(void *info);

#ifdef SMP
static void lwkt_serialize_adaptive_sleep(void *bo);

static int slz_backoff_limit = 128;
SYSCTL_INT(_debug, OID_AUTO, serialize_bolimit, CTLFLAG_RW,
    &slz_backoff_limit, 0, "Backoff limit");

static int slz_backoff_shift = 1;
SYSCTL_INT(_debug, OID_AUTO, serialize_boshift, CTLFLAG_RW,
    &slz_backoff_shift, 0, "Backoff shift");

static int slz_backoff_round;
TUNABLE_INT("debug.serialize_boround", &slz_backoff_round);
SYSCTL_INT(_debug, OID_AUTO, serialize_boround, CTLFLAG_RW,
    &slz_backoff_round, 0,
    "Backoff rounding");
#endif	/* SMP */

void
lwkt_serialize_init(lwkt_serialize_t s)
{
    atomic_intr_init(&s->interlock);
#ifdef INVARIANTS
    s->last_td = (void *)-4;
#endif
}

#ifdef SMP
void
lwkt_serialize_adaptive_enter(lwkt_serialize_t s)
{
    struct exp_backoff bo;

    bo.backoff = 1;
    bo.round = 0;
    bo.s = s;

    ASSERT_NOT_SERIALIZED(s);

    logslz(enter_beg, s);
    atomic_intr_cond_enter(&s->interlock, lwkt_serialize_adaptive_sleep, &bo);
    logslz(enter_end, s);
#ifdef INVARIANTS
    s->last_td = curthread;
#endif
}
#endif	/* SMP */

void
lwkt_serialize_enter(lwkt_serialize_t s)
{
    ASSERT_NOT_SERIALIZED(s);

    logslz(enter_beg, s);
    atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
    logslz(enter_end, s);
#ifdef INVARIANTS
    s->last_td = curthread;
#endif
}

/*
 * Returns non-zero on success
 */
int
lwkt_serialize_try(lwkt_serialize_t s)
{
    int error;

    ASSERT_NOT_SERIALIZED(s);

    logslz(try, s);
    if ((error = atomic_intr_cond_try(&s->interlock)) == 0) {
#ifdef INVARIANTS
	s->last_td = curthread;
#endif
	logslz(tryok, s);
	return(1);
    }
    logslz(tryfail, s);
    return (0);
}

void
lwkt_serialize_exit(lwkt_serialize_t s)
{
    ASSERT_SERIALIZED(s);
#ifdef INVARIANTS
    s->last_td = (void *)-2;
#endif
    logslz(exit_beg, s);
    atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
    logslz(exit_end, s);
}

/*
 * Interrupt handler disablement support, used by drivers.  Non-stackable
 * (uses bit 30).
 */
void
lwkt_serialize_handler_disable(lwkt_serialize_t s)
{
    atomic_intr_handler_disable(&s->interlock);
}

void
lwkt_serialize_handler_enable(lwkt_serialize_t s)
{
    atomic_intr_handler_enable(&s->interlock);
}

void
lwkt_serialize_handler_call(lwkt_serialize_t s, void (*func)(void *, void *), 
			    void *arg, void *frame)
{
    /*
     * note: a return value of 0 indicates that the interrupt handler is 
     * enabled.
     */
    if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
	logslz(enter_beg, s);
	atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
	logslz(enter_end, s);
#ifdef INVARIANTS
	s->last_td = curthread;
#endif
	if (atomic_intr_handler_is_enabled(&s->interlock) == 0)
	    func(arg, frame);

	ASSERT_SERIALIZED(s);
#ifdef INVARIANTS
	s->last_td = (void *)-2;
#endif
	logslz(exit_beg, s);
	atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
	logslz(exit_end, s);
    }
}

/*
 * Similar to handler_call but does not block.  Returns 0 on success, 
 * and 1 on failure.
 */
int
lwkt_serialize_handler_try(lwkt_serialize_t s, void (*func)(void *, void *),
			   void *arg, void *frame)
{
    /*
     * note: a return value of 0 indicates that the interrupt handler is 
     * enabled.
     */
    if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
	logslz(try, s);
	if (atomic_intr_cond_try(&s->interlock) == 0) {
#ifdef INVARIANTS
	    s->last_td = curthread;
#endif
	    logslz(tryok, s);

	    func(arg, frame);

	    ASSERT_SERIALIZED(s);
#ifdef INVARIANTS
	    s->last_td = (void *)-2;
#endif
	    logslz(exit_beg, s);
	    atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
	    logslz(exit_end, s);
	    return(0);
	}
    }
    logslz(tryfail, s);
    return(1);
}


/*
 * Helper functions
 *
 * It is possible to race an interrupt which acquires and releases the
 * bit, then calls wakeup before we actually go to sleep, so we
 * need to check that the interlock is still acquired from within
 * a critical section prior to sleeping.
 */
static void
lwkt_serialize_sleep(void *info)
{
    lwkt_serialize_t s = info;

    tsleep_interlock(s, 0);
    if (atomic_intr_cond_test(&s->interlock) != 0) {
	logslz(sleep_beg, s);
	tsleep(s, PINTERLOCKED, "slize", 0);
	logslz(sleep_end, s);
    }
}

#ifdef SMP

static void
lwkt_serialize_adaptive_sleep(void *arg)
{
    struct exp_backoff *bo = arg;
    lwkt_serialize_t s = bo->s;
    int backoff;

    /*
     * Randomize backoff value
     */
#ifdef _RDTSC_SUPPORTED_
    if (cpu_feature & CPUID_TSC) {
	backoff =
	(((u_long)rdtsc() ^ (((u_long)curthread) >> 5)) &
	 (bo->backoff - 1)) + 1;
    } else
#endif
	backoff = bo->backoff;

    logslz_spinbo(s, bo->backoff, backoff);

    /*
     * Quick backoff
     */
    for (; backoff; --backoff)
	cpu_pause();
    if (bo->backoff < slz_backoff_limit) {
	bo->backoff <<= slz_backoff_shift;
	return;
    } else {
	bo->backoff = 1;
	bo->round++;
	if (bo->round >= slz_backoff_round)
	    bo->round = 0;
    	else
	    return;
    }

    tsleep_interlock(s, 0);
    if (atomic_intr_cond_test(&s->interlock) != 0) {
	logslz(sleep_beg, s);
	tsleep(s, PINTERLOCKED, "slize", 0);
	logslz(sleep_end, s);
    }
}

#endif	/* SMP */

static void
lwkt_serialize_wakeup(void *info)
{
    logslz(wakeup_beg, info);
    wakeup(info);
    logslz(wakeup_end, info);
}

#ifdef SMP
static void
lwkt_serialize_sysinit(void *dummy __unused)
{
	if (slz_backoff_round <= 0)
		slz_backoff_round = ncpus * 2;
}
SYSINIT(lwkt_serialize, SI_SUB_PRE_DRIVERS, SI_ORDER_SECOND,
	lwkt_serialize_sysinit, NULL);
#endif
Commit	Line	Data
ed03f3a3 MD	1	/*
	2	* Copyright (c) 2005 The DragonFly Project. All rights reserved.
	3	*
	4	* This code is derived from software contributed to The DragonFly Project
	5	* by Matthew Dillon <dillon@backplane.com>
	6	*
	7	* Redistribution and use in source and binary forms, with or without
	8	* modification, are permitted provided that the following conditions
	9	* are met:
	10	*
	11	* 1. Redistributions of source code must retain the above copyright
	12	* notice, this list of conditions and the following disclaimer.
	13	* 2. Redistributions in binary form must reproduce the above copyright
	14	* notice, this list of conditions and the following disclaimer in
	15	* the documentation and/or other materials provided with the
	16	* distribution.
	17	* 3. Neither the name of The DragonFly Project nor the names of its
	18	* contributors may be used to endorse or promote products derived
	19	* from this software without specific, prior written permission.
	20	*
	21	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	22	* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	23	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	24	* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
	25	* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
	26	* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
	27	* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	28	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	29	* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	30	* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
	31	* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	32	* SUCH DAMAGE.
	33	*
7e0d629c	34	* $DragonFly: src/sys/kern/lwkt_serialize.c,v 1.18 2008/10/04 14:22:44 swildner Exp $
ed03f3a3 MD	35	*/
	36	/*
	37	* This API provides a fast locked-bus-cycle-based serializer. It's
	38	* basically a low level NON-RECURSIVE exclusive lock that can be held across
	39	* a blocking condition. It is NOT a mutex.
	40	*
	41	* This serializer is primarily designed for low level situations and
	42	* interrupt/device interaction. There are two primary facilities. First,
	43	* the serializer facility itself. Second, an integrated interrupt handler
	44	* disablement facility.
	45	*/
	46
	47	#include <sys/param.h>
	48	#include <sys/systm.h>
	49	#include <sys/kernel.h>
	50	#include <sys/proc.h>
	51	#include <sys/rtprio.h>
	52	#include <sys/queue.h>
	53	#include <sys/thread2.h>
	54	#include <sys/serialize.h>
	55	#include <sys/sysctl.h>
027f7bf2	56	#include <sys/ktr.h>
ed03f3a3 MD	57	#include <sys/kthread.h>
ed03f3a3 MD	58	#include <machine/cpu.h>
d1d48a34 SZ	59	#include <machine/cpufunc.h>
d1d48a34 SZ	60	#include <machine/specialreg.h>
ed03f3a3 MD	61	#include <sys/lock.h>
	62	#include <sys/caps.h>
	63
d1d48a34 SZ	64	struct exp_backoff {
	65	int backoff;
	66	int round;
	67	lwkt_serialize_t s;
	68	};
	69
027f7bf2	70	#define SLZ_KTR_STRING "slz=%p"
5bf48697	71	#define SLZ_KTR_ARGS lwkt_serialize_t slz
027f7bf2 SZ	72
	73	#ifndef KTR_SERIALIZER
	74	#define KTR_SERIALIZER KTR_ALL
	75	#endif
	76
	77	KTR_INFO_MASTER(slz);
5bf48697 AE	78	KTR_INFO(KTR_SERIALIZER, slz, enter_beg, 0, SLZ_KTR_STRING, SLZ_KTR_ARGS);
	79	KTR_INFO(KTR_SERIALIZER, slz, sleep_beg, 1, SLZ_KTR_STRING, SLZ_KTR_ARGS);
	80	KTR_INFO(KTR_SERIALIZER, slz, sleep_end, 2, SLZ_KTR_STRING, SLZ_KTR_ARGS);
	81	KTR_INFO(KTR_SERIALIZER, slz, exit_end, 3, SLZ_KTR_STRING, SLZ_KTR_ARGS);
	82	KTR_INFO(KTR_SERIALIZER, slz, wakeup_beg, 4, SLZ_KTR_STRING, SLZ_KTR_ARGS);
	83	KTR_INFO(KTR_SERIALIZER, slz, wakeup_end, 5, SLZ_KTR_STRING, SLZ_KTR_ARGS);
	84	KTR_INFO(KTR_SERIALIZER, slz, try, 6, SLZ_KTR_STRING, SLZ_KTR_ARGS);
	85	KTR_INFO(KTR_SERIALIZER, slz, tryfail, 7, SLZ_KTR_STRING, SLZ_KTR_ARGS);
	86	KTR_INFO(KTR_SERIALIZER, slz, tryok, 8, SLZ_KTR_STRING, SLZ_KTR_ARGS);
7e0d629c	87	#ifdef SMP
d1d48a34	88	KTR_INFO(KTR_SERIALIZER, slz, spinbo, 9,
5bf48697	89	"slz=%p bo1=%d bo=%d", lwkt_serialize_t slz, int backoff1, int backoff);
7e0d629c	90	#endif
5bf48697 AE	91	KTR_INFO(KTR_SERIALIZER, slz, enter_end, 10, SLZ_KTR_STRING, SLZ_KTR_ARGS);
5bf48697 AE	92	KTR_INFO(KTR_SERIALIZER, slz, exit_beg, 11, SLZ_KTR_STRING, SLZ_KTR_ARGS);
027f7bf2	93
d1d48a34	94	#define logslz(name, slz) KTR_LOG(slz_ ## name, slz)
7e0d629c	95	#ifdef SMP
d1d48a34	96	#define logslz_spinbo(slz, bo1, bo) KTR_LOG(slz_spinbo, slz, bo1, bo)
7e0d629c	97	#endif
027f7bf2	98
ed03f3a3 MD	99	static void lwkt_serialize_sleep(void *info);
	100	static void lwkt_serialize_wakeup(void *info);
	101
d1d48a34	102	#ifdef SMP
9db4b353 SZ	103	static void lwkt_serialize_adaptive_sleep(void *bo);
9db4b353 SZ	104
d1d48a34 SZ	105	static int slz_backoff_limit = 128;
d1d48a34 SZ	106	SYSCTL_INT(_debug, OID_AUTO, serialize_bolimit, CTLFLAG_RW,
0c52fa62	107	&slz_backoff_limit, 0, "Backoff limit");
d1d48a34 SZ	108
	109	static int slz_backoff_shift = 1;
	110	SYSCTL_INT(_debug, OID_AUTO, serialize_boshift, CTLFLAG_RW,
0c52fa62	111	&slz_backoff_shift, 0, "Backoff shift");
d1d48a34 SZ	112
	113	static int slz_backoff_round;
	114	TUNABLE_INT("debug.serialize_boround", &slz_backoff_round);
	115	SYSCTL_INT(_debug, OID_AUTO, serialize_boround, CTLFLAG_RW,
0c52fa62 SG	116	&slz_backoff_round, 0,
0c52fa62 SG	117	"Backoff rounding");
d1d48a34 SZ	118	#endif /* SMP */
d1d48a34 SZ	119
ed03f3a3 MD	120	void
	121	lwkt_serialize_init(lwkt_serialize_t s)
	122	{
	123	atomic_intr_init(&s->interlock);
dcad4d19	124	#ifdef INVARIANTS
f0ca723e	125	s->last_td = (void *)-4;
dcad4d19	126	#endif
ed03f3a3 MD	127	}
ed03f3a3 MD	128
d1d48a34	129	#ifdef SMP
d1d48a34 SZ	130	void
	131	lwkt_serialize_adaptive_enter(lwkt_serialize_t s)
	132	{
	133	struct exp_backoff bo;
	134
	135	bo.backoff = 1;
	136	bo.round = 0;
	137	bo.s = s;
	138
9db4b353 SZ	139	ASSERT_NOT_SERIALIZED(s);
9db4b353 SZ	140
d1d48a34 SZ	141	logslz(enter_beg, s);
	142	atomic_intr_cond_enter(&s->interlock, lwkt_serialize_adaptive_sleep, &bo);
	143	logslz(enter_end, s);
dcad4d19	144	#ifdef INVARIANTS
d1d48a34	145	s->last_td = curthread;
dcad4d19	146	#endif
d1d48a34	147	}
d1d48a34 SZ	148	#endif /* SMP */
d1d48a34 SZ	149
ed03f3a3 MD	150	void
	151	lwkt_serialize_enter(lwkt_serialize_t s)
	152	{
9db4b353 SZ	153	ASSERT_NOT_SERIALIZED(s);
9db4b353 SZ	154
d1d48a34	155	logslz(enter_beg, s);
ed03f3a3	156	atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
d1d48a34	157	logslz(enter_end, s);
dcad4d19	158	#ifdef INVARIANTS
ed03f3a3	159	s->last_td = curthread;
dcad4d19	160	#endif
ed03f3a3 MD	161	}
ed03f3a3 MD	162
3e086307 MD	163	/*
	164	* Returns non-zero on success
	165	*/
	166	int
	167	lwkt_serialize_try(lwkt_serialize_t s)
	168	{
	169	int error;
	170
9db4b353 SZ	171	ASSERT_NOT_SERIALIZED(s);
9db4b353 SZ	172
027f7bf2	173	logslz(try, s);
3e086307	174	if ((error = atomic_intr_cond_try(&s->interlock)) == 0) {
dcad4d19	175	#ifdef INVARIANTS
3e086307	176	s->last_td = curthread;
dcad4d19	177	#endif
027f7bf2	178	logslz(tryok, s);
3e086307 MD	179	return(1);
3e086307 MD	180	}
027f7bf2	181	logslz(tryfail, s);
3e086307 MD	182	return (0);
	183	}
	184
ed03f3a3 MD	185	void
	186	lwkt_serialize_exit(lwkt_serialize_t s)
	187	{
9db4b353	188	ASSERT_SERIALIZED(s);
dcad4d19	189	#ifdef INVARIANTS
f0ca723e	190	s->last_td = (void *)-2;
dcad4d19	191	#endif
d1d48a34	192	logslz(exit_beg, s);
ed03f3a3	193	atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
d1d48a34	194	logslz(exit_end, s);
ed03f3a3 MD	195	}
	196
	197	/*
	198	* Interrupt handler disablement support, used by drivers. Non-stackable
	199	* (uses bit 30).
	200	*/
	201	void
	202	lwkt_serialize_handler_disable(lwkt_serialize_t s)
	203	{
	204	atomic_intr_handler_disable(&s->interlock);
	205	}
	206
	207	void
	208	lwkt_serialize_handler_enable(lwkt_serialize_t s)
	209	{
	210	atomic_intr_handler_enable(&s->interlock);
	211	}
	212
	213	void
477d3c1c MD	214	lwkt_serialize_handler_call(lwkt_serialize_t s, void (func)(void , void *),
477d3c1c MD	215	void arg, void frame)
ed03f3a3	216	{
9c095379 MD	217	/*
	218	* note: a return value of 0 indicates that the interrupt handler is
	219	* enabled.
	220	*/
	221	if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
d1d48a34	222	logslz(enter_beg, s);
ed03f3a3	223	atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
d1d48a34	224	logslz(enter_end, s);
dcad4d19	225	#ifdef INVARIANTS
92a4dd35	226	s->last_td = curthread;
dcad4d19	227	#endif
ed03f3a3	228	if (atomic_intr_handler_is_enabled(&s->interlock) == 0)
477d3c1c	229	func(arg, frame);
9db4b353 SZ	230
9db4b353 SZ	231	ASSERT_SERIALIZED(s);
dcad4d19	232	#ifdef INVARIANTS
92a4dd35	233	s->last_td = (void *)-2;
dcad4d19	234	#endif
d1d48a34	235	logslz(exit_beg, s);
e139cf18	236	atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
d1d48a34	237	logslz(exit_end, s);
ed03f3a3 MD	238	}
	239	}
	240
	241	/*
477d3c1c MD	242	* Similar to handler_call but does not block. Returns 0 on success,
	243	* and 1 on failure.
	244	*/
	245	int
	246	lwkt_serialize_handler_try(lwkt_serialize_t s, void (func)(void , void *),
	247	void arg, void frame)
	248	{
	249	/*
	250	* note: a return value of 0 indicates that the interrupt handler is
	251	* enabled.
	252	*/
	253	if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
027f7bf2	254	logslz(try, s);
477d3c1c	255	if (atomic_intr_cond_try(&s->interlock) == 0) {
dcad4d19	256	#ifdef INVARIANTS
92a4dd35	257	s->last_td = curthread;
dcad4d19	258	#endif
027f7bf2	259	logslz(tryok, s);
9db4b353	260
477d3c1c	261	func(arg, frame);
9db4b353 SZ	262
9db4b353 SZ	263	ASSERT_SERIALIZED(s);
dcad4d19	264	#ifdef INVARIANTS
92a4dd35	265	s->last_td = (void *)-2;
dcad4d19	266	#endif
d1d48a34	267	logslz(exit_beg, s);
92a4dd35	268	atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
d1d48a34	269	logslz(exit_end, s);
477d3c1c MD	270	return(0);
	271	}
	272	}
027f7bf2	273	logslz(tryfail, s);
477d3c1c MD	274	return(1);
	275	}
	276
	277
	278	/*
ed03f3a3	279	* Helper functions
d02b46cd MD	280	*
	281	* It is possible to race an interrupt which acquires and releases the
	282	* bit, then calls wakeup before we actually go to sleep, so we
f0ca723e MD	283	* need to check that the interlock is still acquired from within
f0ca723e MD	284	* a critical section prior to sleeping.
ed03f3a3 MD	285	*/
	286	static void
	287	lwkt_serialize_sleep(void *info)
	288	{
	289	lwkt_serialize_t s = info;
ae8e83e6 MD	290
ae8e83e6 MD	291	tsleep_interlock(s, 0);
21fa6062	292	if (atomic_intr_cond_test(&s->interlock) != 0) {
3246ae89	293	logslz(sleep_beg, s);
d9345d3a	294	tsleep(s, PINTERLOCKED, "slize", 0);
3246ae89	295	logslz(sleep_end, s);
21fa6062	296	}
ed03f3a3 MD	297	}
ed03f3a3 MD	298
d1d48a34 SZ	299	#ifdef SMP
	300
	301	static void
	302	lwkt_serialize_adaptive_sleep(void *arg)
	303	{
	304	struct exp_backoff *bo = arg;
	305	lwkt_serialize_t s = bo->s;
	306	int backoff;
	307
	308	/*
	309	* Randomize backoff value
	310	*/
	311	#ifdef _RDTSC_SUPPORTED_
	312	if (cpu_feature & CPUID_TSC) {
	313	backoff =
	314	(((u_long)rdtsc() ^ (((u_long)curthread) >> 5)) &
	315	(bo->backoff - 1)) + 1;
	316	} else
	317	#endif
	318	backoff = bo->backoff;
	319
	320	logslz_spinbo(s, bo->backoff, backoff);
	321
	322	/*
	323	* Quick backoff
	324	*/
	325	for (; backoff; --backoff)
14dd663d	326	cpu_pause();
d1d48a34 SZ	327	if (bo->backoff < slz_backoff_limit) {
	328	bo->backoff <<= slz_backoff_shift;
	329	return;
	330	} else {
	331	bo->backoff = 1;
	332	bo->round++;
	333	if (bo->round >= slz_backoff_round)
	334	bo->round = 0;
	335	else
	336	return;
	337	}
	338
ae8e83e6	339	tsleep_interlock(s, 0);
d1d48a34	340	if (atomic_intr_cond_test(&s->interlock) != 0) {
d1d48a34	341	logslz(sleep_beg, s);
d9345d3a	342	tsleep(s, PINTERLOCKED, "slize", 0);
d1d48a34 SZ	343	logslz(sleep_end, s);
d1d48a34 SZ	344	}
d1d48a34 SZ	345	}
	346
	347	#endif /* SMP */
	348
ed03f3a3 MD	349	static void
	350	lwkt_serialize_wakeup(void *info)
	351	{
3246ae89	352	logslz(wakeup_beg, info);
ed03f3a3	353	wakeup(info);
3246ae89	354	logslz(wakeup_end, info);
ed03f3a3	355	}
d1d48a34 SZ	356
	357	#ifdef SMP
	358	static void
	359	lwkt_serialize_sysinit(void *dummy __unused)
	360	{
	361	if (slz_backoff_round <= 0)
	362	slz_backoff_round = ncpus * 2;
	363	}
	364	SYSINIT(lwkt_serialize, SI_SUB_PRE_DRIVERS, SI_ORDER_SECOND,
	365	lwkt_serialize_sysinit, NULL);
	366	#endif