2 * Copyright (c) 2005 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * $DragonFly: src/sys/kern/lwkt_serialize.c,v 1.18 2008/10/04 14:22:44 swildner Exp $
37 * This API provides a fast locked-bus-cycle-based serializer. It's
38 * basically a low level NON-RECURSIVE exclusive lock that can be held across
39 * a blocking condition. It is NOT a mutex.
41 * This serializer is primarily designed for low level situations and
42 * interrupt/device interaction. There are two primary facilities. First,
43 * the serializer facility itself. Second, an integrated interrupt handler
44 * disablement facility.
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/kernel.h>
51 #include <sys/rtprio.h>
52 #include <sys/queue.h>
53 #include <sys/thread2.h>
54 #include <sys/serialize.h>
55 #include <sys/sysctl.h>
57 #include <sys/kthread.h>
58 #include <machine/cpu.h>
59 #include <machine/cpufunc.h>
60 #include <machine/specialreg.h>
70 #define SLZ_KTR_STRING "slz=%p"
71 #define SLZ_KTR_ARG_SIZE (sizeof(void *))
73 #ifndef KTR_SERIALIZER
74 #define KTR_SERIALIZER KTR_ALL
78 KTR_INFO(KTR_SERIALIZER, slz, enter_beg, 0, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
79 KTR_INFO(KTR_SERIALIZER, slz, sleep_beg, 1, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
80 KTR_INFO(KTR_SERIALIZER, slz, sleep_end, 2, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
81 KTR_INFO(KTR_SERIALIZER, slz, exit_end, 3, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
82 KTR_INFO(KTR_SERIALIZER, slz, wakeup_beg, 4, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
83 KTR_INFO(KTR_SERIALIZER, slz, wakeup_end, 5, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
84 KTR_INFO(KTR_SERIALIZER, slz, try, 6, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
85 KTR_INFO(KTR_SERIALIZER, slz, tryfail, 7, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
86 KTR_INFO(KTR_SERIALIZER, slz, tryok, 8, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
88 KTR_INFO(KTR_SERIALIZER, slz, spinbo, 9,
89 "slz=%p bo1=%d bo=%d", (sizeof(void *) + (2 * sizeof(int))));
91 KTR_INFO(KTR_SERIALIZER, slz, enter_end, 10, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
92 KTR_INFO(KTR_SERIALIZER, slz, exit_beg, 11, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
94 #define logslz(name, slz) KTR_LOG(slz_ ## name, slz)
96 #define logslz_spinbo(slz, bo1, bo) KTR_LOG(slz_spinbo, slz, bo1, bo)
99 static void lwkt_serialize_sleep(void *info);
100 static void lwkt_serialize_wakeup(void *info);
103 static void lwkt_serialize_adaptive_sleep(void *bo);
105 static int slz_backoff_limit = 128;
106 SYSCTL_INT(_debug, OID_AUTO, serialize_bolimit, CTLFLAG_RW,
107 &slz_backoff_limit, 0, "");
109 static int slz_backoff_shift = 1;
110 SYSCTL_INT(_debug, OID_AUTO, serialize_boshift, CTLFLAG_RW,
111 &slz_backoff_shift, 0, "");
113 static int slz_backoff_round;
114 TUNABLE_INT("debug.serialize_boround", &slz_backoff_round);
115 SYSCTL_INT(_debug, OID_AUTO, serialize_boround, CTLFLAG_RW,
116 &slz_backoff_round, 0, "");
120 lwkt_serialize_init(lwkt_serialize_t s)
122 atomic_intr_init(&s->interlock);
124 s->last_td = (void *)-4;
130 lwkt_serialize_adaptive_enter(lwkt_serialize_t s)
132 struct exp_backoff bo;
138 ASSERT_NOT_SERIALIZED(s);
140 logslz(enter_beg, s);
141 atomic_intr_cond_enter(&s->interlock, lwkt_serialize_adaptive_sleep, &bo);
142 logslz(enter_end, s);
144 s->last_td = curthread;
150 lwkt_serialize_enter(lwkt_serialize_t s)
152 ASSERT_NOT_SERIALIZED(s);
154 logslz(enter_beg, s);
155 atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
156 logslz(enter_end, s);
158 s->last_td = curthread;
163 * Returns non-zero on success
166 lwkt_serialize_try(lwkt_serialize_t s)
170 ASSERT_NOT_SERIALIZED(s);
173 if ((error = atomic_intr_cond_try(&s->interlock)) == 0) {
175 s->last_td = curthread;
185 lwkt_serialize_exit(lwkt_serialize_t s)
187 ASSERT_SERIALIZED(s);
189 s->last_td = (void *)-2;
192 atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
197 * Interrupt handler disablement support, used by drivers. Non-stackable
201 lwkt_serialize_handler_disable(lwkt_serialize_t s)
203 atomic_intr_handler_disable(&s->interlock);
207 lwkt_serialize_handler_enable(lwkt_serialize_t s)
209 atomic_intr_handler_enable(&s->interlock);
213 lwkt_serialize_handler_call(lwkt_serialize_t s, void (*func)(void *, void *),
214 void *arg, void *frame)
217 * note: a return value of 0 indicates that the interrupt handler is
220 if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
221 logslz(enter_beg, s);
222 atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
223 logslz(enter_end, s);
225 s->last_td = curthread;
227 if (atomic_intr_handler_is_enabled(&s->interlock) == 0)
230 ASSERT_SERIALIZED(s);
232 s->last_td = (void *)-2;
235 atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
241 * Similar to handler_call but does not block. Returns 0 on success,
245 lwkt_serialize_handler_try(lwkt_serialize_t s, void (*func)(void *, void *),
246 void *arg, void *frame)
249 * note: a return value of 0 indicates that the interrupt handler is
252 if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
254 if (atomic_intr_cond_try(&s->interlock) == 0) {
256 s->last_td = curthread;
262 ASSERT_SERIALIZED(s);
264 s->last_td = (void *)-2;
267 atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
280 * It is possible to race an interrupt which acquires and releases the
281 * bit, then calls wakeup before we actually go to sleep, so we
282 * need to check that the interlock is still acquired from within
283 * a critical section prior to sleeping.
286 lwkt_serialize_sleep(void *info)
288 lwkt_serialize_t s = info;
290 tsleep_interlock(s, 0);
291 if (atomic_intr_cond_test(&s->interlock) != 0) {
292 logslz(sleep_beg, s);
293 tsleep(s, PINTERLOCKED, "slize", 0);
294 logslz(sleep_end, s);
301 lwkt_serialize_adaptive_sleep(void *arg)
303 struct exp_backoff *bo = arg;
304 lwkt_serialize_t s = bo->s;
308 * Randomize backoff value
310 #ifdef _RDTSC_SUPPORTED_
311 if (cpu_feature & CPUID_TSC) {
313 (((u_long)rdtsc() ^ (((u_long)curthread) >> 5)) &
314 (bo->backoff - 1)) + 1;
317 backoff = bo->backoff;
319 logslz_spinbo(s, bo->backoff, backoff);
324 for (; backoff; --backoff)
326 if (bo->backoff < slz_backoff_limit) {
327 bo->backoff <<= slz_backoff_shift;
332 if (bo->round >= slz_backoff_round)
338 tsleep_interlock(s, 0);
339 if (atomic_intr_cond_test(&s->interlock) != 0) {
340 logslz(sleep_beg, s);
341 tsleep(s, PINTERLOCKED, "slize", 0);
342 logslz(sleep_end, s);
349 lwkt_serialize_wakeup(void *info)
351 logslz(wakeup_beg, info);
353 logslz(wakeup_end, info);
358 lwkt_serialize_sysinit(void *dummy __unused)
360 if (slz_backoff_round <= 0)
361 slz_backoff_round = ncpus * 2;
363 SYSINIT(lwkt_serialize, SI_SUB_PRE_DRIVERS, SI_ORDER_SECOND,
364 lwkt_serialize_sysinit, NULL);