kernel - Provide descriptions for lwkt.* and debug.* sysctl's
[dragonfly.git] / sys / kern / lwkt_serialize.c
CommitLineData
ed03f3a3
MD
1/*
2 * Copyright (c) 2005 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
7e0d629c 34 * $DragonFly: src/sys/kern/lwkt_serialize.c,v 1.18 2008/10/04 14:22:44 swildner Exp $
ed03f3a3
MD
35 */
36/*
37 * This API provides a fast locked-bus-cycle-based serializer. It's
38 * basically a low level NON-RECURSIVE exclusive lock that can be held across
39 * a blocking condition. It is NOT a mutex.
40 *
41 * This serializer is primarily designed for low level situations and
42 * interrupt/device interaction. There are two primary facilities. First,
43 * the serializer facility itself. Second, an integrated interrupt handler
44 * disablement facility.
45 */
46
47#include <sys/param.h>
48#include <sys/systm.h>
49#include <sys/kernel.h>
50#include <sys/proc.h>
51#include <sys/rtprio.h>
52#include <sys/queue.h>
53#include <sys/thread2.h>
54#include <sys/serialize.h>
55#include <sys/sysctl.h>
027f7bf2 56#include <sys/ktr.h>
ed03f3a3
MD
57#include <sys/kthread.h>
58#include <machine/cpu.h>
d1d48a34
SZ
59#include <machine/cpufunc.h>
60#include <machine/specialreg.h>
ed03f3a3
MD
61#include <sys/lock.h>
62#include <sys/caps.h>
63
d1d48a34
SZ
64struct exp_backoff {
65 int backoff;
66 int round;
67 lwkt_serialize_t s;
68};
69
027f7bf2
SZ
70#define SLZ_KTR_STRING "slz=%p"
71#define SLZ_KTR_ARG_SIZE (sizeof(void *))
72
73#ifndef KTR_SERIALIZER
74#define KTR_SERIALIZER KTR_ALL
75#endif
76
77KTR_INFO_MASTER(slz);
d1d48a34 78KTR_INFO(KTR_SERIALIZER, slz, enter_beg, 0, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
3246ae89
SZ
79KTR_INFO(KTR_SERIALIZER, slz, sleep_beg, 1, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
80KTR_INFO(KTR_SERIALIZER, slz, sleep_end, 2, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
d1d48a34 81KTR_INFO(KTR_SERIALIZER, slz, exit_end, 3, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
3246ae89
SZ
82KTR_INFO(KTR_SERIALIZER, slz, wakeup_beg, 4, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
83KTR_INFO(KTR_SERIALIZER, slz, wakeup_end, 5, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
84KTR_INFO(KTR_SERIALIZER, slz, try, 6, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
85KTR_INFO(KTR_SERIALIZER, slz, tryfail, 7, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
86KTR_INFO(KTR_SERIALIZER, slz, tryok, 8, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
7e0d629c 87#ifdef SMP
d1d48a34
SZ
88KTR_INFO(KTR_SERIALIZER, slz, spinbo, 9,
89 "slz=%p bo1=%d bo=%d", (sizeof(void *) + (2 * sizeof(int))));
7e0d629c 90#endif
d1d48a34
SZ
91KTR_INFO(KTR_SERIALIZER, slz, enter_end, 10, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
92KTR_INFO(KTR_SERIALIZER, slz, exit_beg, 11, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
027f7bf2 93
d1d48a34 94#define logslz(name, slz) KTR_LOG(slz_ ## name, slz)
7e0d629c 95#ifdef SMP
d1d48a34 96#define logslz_spinbo(slz, bo1, bo) KTR_LOG(slz_spinbo, slz, bo1, bo)
7e0d629c 97#endif
027f7bf2 98
ed03f3a3
MD
99static void lwkt_serialize_sleep(void *info);
100static void lwkt_serialize_wakeup(void *info);
101
d1d48a34 102#ifdef SMP
9db4b353
SZ
103static void lwkt_serialize_adaptive_sleep(void *bo);
104
d1d48a34
SZ
105static int slz_backoff_limit = 128;
106SYSCTL_INT(_debug, OID_AUTO, serialize_bolimit, CTLFLAG_RW,
0c52fa62 107 &slz_backoff_limit, 0, "Backoff limit");
d1d48a34
SZ
108
109static int slz_backoff_shift = 1;
110SYSCTL_INT(_debug, OID_AUTO, serialize_boshift, CTLFLAG_RW,
0c52fa62 111 &slz_backoff_shift, 0, "Backoff shift");
d1d48a34
SZ
112
113static int slz_backoff_round;
114TUNABLE_INT("debug.serialize_boround", &slz_backoff_round);
115SYSCTL_INT(_debug, OID_AUTO, serialize_boround, CTLFLAG_RW,
0c52fa62
SG
116 &slz_backoff_round, 0,
117 "Backoff rounding");
d1d48a34
SZ
118#endif /* SMP */
119
ed03f3a3
MD
120void
121lwkt_serialize_init(lwkt_serialize_t s)
122{
123 atomic_intr_init(&s->interlock);
dcad4d19 124#ifdef INVARIANTS
f0ca723e 125 s->last_td = (void *)-4;
dcad4d19 126#endif
ed03f3a3
MD
127}
128
d1d48a34 129#ifdef SMP
d1d48a34
SZ
130void
131lwkt_serialize_adaptive_enter(lwkt_serialize_t s)
132{
133 struct exp_backoff bo;
134
135 bo.backoff = 1;
136 bo.round = 0;
137 bo.s = s;
138
9db4b353
SZ
139 ASSERT_NOT_SERIALIZED(s);
140
d1d48a34
SZ
141 logslz(enter_beg, s);
142 atomic_intr_cond_enter(&s->interlock, lwkt_serialize_adaptive_sleep, &bo);
143 logslz(enter_end, s);
dcad4d19 144#ifdef INVARIANTS
d1d48a34 145 s->last_td = curthread;
dcad4d19 146#endif
d1d48a34 147}
d1d48a34
SZ
148#endif /* SMP */
149
ed03f3a3
MD
150void
151lwkt_serialize_enter(lwkt_serialize_t s)
152{
9db4b353
SZ
153 ASSERT_NOT_SERIALIZED(s);
154
d1d48a34 155 logslz(enter_beg, s);
ed03f3a3 156 atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
d1d48a34 157 logslz(enter_end, s);
dcad4d19 158#ifdef INVARIANTS
ed03f3a3 159 s->last_td = curthread;
dcad4d19 160#endif
ed03f3a3
MD
161}
162
3e086307
MD
163/*
164 * Returns non-zero on success
165 */
166int
167lwkt_serialize_try(lwkt_serialize_t s)
168{
169 int error;
170
9db4b353
SZ
171 ASSERT_NOT_SERIALIZED(s);
172
027f7bf2 173 logslz(try, s);
3e086307 174 if ((error = atomic_intr_cond_try(&s->interlock)) == 0) {
dcad4d19 175#ifdef INVARIANTS
3e086307 176 s->last_td = curthread;
dcad4d19 177#endif
027f7bf2 178 logslz(tryok, s);
3e086307
MD
179 return(1);
180 }
027f7bf2 181 logslz(tryfail, s);
3e086307
MD
182 return (0);
183}
184
ed03f3a3
MD
185void
186lwkt_serialize_exit(lwkt_serialize_t s)
187{
9db4b353 188 ASSERT_SERIALIZED(s);
dcad4d19 189#ifdef INVARIANTS
f0ca723e 190 s->last_td = (void *)-2;
dcad4d19 191#endif
d1d48a34 192 logslz(exit_beg, s);
ed03f3a3 193 atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
d1d48a34 194 logslz(exit_end, s);
ed03f3a3
MD
195}
196
197/*
198 * Interrupt handler disablement support, used by drivers. Non-stackable
199 * (uses bit 30).
200 */
201void
202lwkt_serialize_handler_disable(lwkt_serialize_t s)
203{
204 atomic_intr_handler_disable(&s->interlock);
205}
206
207void
208lwkt_serialize_handler_enable(lwkt_serialize_t s)
209{
210 atomic_intr_handler_enable(&s->interlock);
211}
212
213void
477d3c1c
MD
214lwkt_serialize_handler_call(lwkt_serialize_t s, void (*func)(void *, void *),
215 void *arg, void *frame)
ed03f3a3 216{
9c095379
MD
217 /*
218 * note: a return value of 0 indicates that the interrupt handler is
219 * enabled.
220 */
221 if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
d1d48a34 222 logslz(enter_beg, s);
ed03f3a3 223 atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
d1d48a34 224 logslz(enter_end, s);
dcad4d19 225#ifdef INVARIANTS
92a4dd35 226 s->last_td = curthread;
dcad4d19 227#endif
ed03f3a3 228 if (atomic_intr_handler_is_enabled(&s->interlock) == 0)
477d3c1c 229 func(arg, frame);
9db4b353
SZ
230
231 ASSERT_SERIALIZED(s);
dcad4d19 232#ifdef INVARIANTS
92a4dd35 233 s->last_td = (void *)-2;
dcad4d19 234#endif
d1d48a34 235 logslz(exit_beg, s);
e139cf18 236 atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
d1d48a34 237 logslz(exit_end, s);
ed03f3a3
MD
238 }
239}
240
241/*
477d3c1c
MD
242 * Similar to handler_call but does not block. Returns 0 on success,
243 * and 1 on failure.
244 */
245int
246lwkt_serialize_handler_try(lwkt_serialize_t s, void (*func)(void *, void *),
247 void *arg, void *frame)
248{
249 /*
250 * note: a return value of 0 indicates that the interrupt handler is
251 * enabled.
252 */
253 if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
027f7bf2 254 logslz(try, s);
477d3c1c 255 if (atomic_intr_cond_try(&s->interlock) == 0) {
dcad4d19 256#ifdef INVARIANTS
92a4dd35 257 s->last_td = curthread;
dcad4d19 258#endif
027f7bf2 259 logslz(tryok, s);
9db4b353 260
477d3c1c 261 func(arg, frame);
9db4b353
SZ
262
263 ASSERT_SERIALIZED(s);
dcad4d19 264#ifdef INVARIANTS
92a4dd35 265 s->last_td = (void *)-2;
dcad4d19 266#endif
d1d48a34 267 logslz(exit_beg, s);
92a4dd35 268 atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
d1d48a34 269 logslz(exit_end, s);
477d3c1c
MD
270 return(0);
271 }
272 }
027f7bf2 273 logslz(tryfail, s);
477d3c1c
MD
274 return(1);
275}
276
277
278/*
ed03f3a3 279 * Helper functions
d02b46cd
MD
280 *
281 * It is possible to race an interrupt which acquires and releases the
282 * bit, then calls wakeup before we actually go to sleep, so we
f0ca723e
MD
283 * need to check that the interlock is still acquired from within
284 * a critical section prior to sleeping.
ed03f3a3
MD
285 */
286static void
287lwkt_serialize_sleep(void *info)
288{
289 lwkt_serialize_t s = info;
ae8e83e6
MD
290
291 tsleep_interlock(s, 0);
21fa6062 292 if (atomic_intr_cond_test(&s->interlock) != 0) {
3246ae89 293 logslz(sleep_beg, s);
d9345d3a 294 tsleep(s, PINTERLOCKED, "slize", 0);
3246ae89 295 logslz(sleep_end, s);
21fa6062 296 }
ed03f3a3
MD
297}
298
d1d48a34
SZ
299#ifdef SMP
300
301static void
302lwkt_serialize_adaptive_sleep(void *arg)
303{
304 struct exp_backoff *bo = arg;
305 lwkt_serialize_t s = bo->s;
306 int backoff;
307
308 /*
309 * Randomize backoff value
310 */
311#ifdef _RDTSC_SUPPORTED_
312 if (cpu_feature & CPUID_TSC) {
313 backoff =
314 (((u_long)rdtsc() ^ (((u_long)curthread) >> 5)) &
315 (bo->backoff - 1)) + 1;
316 } else
317#endif
318 backoff = bo->backoff;
319
320 logslz_spinbo(s, bo->backoff, backoff);
321
322 /*
323 * Quick backoff
324 */
325 for (; backoff; --backoff)
14dd663d 326 cpu_pause();
d1d48a34
SZ
327 if (bo->backoff < slz_backoff_limit) {
328 bo->backoff <<= slz_backoff_shift;
329 return;
330 } else {
331 bo->backoff = 1;
332 bo->round++;
333 if (bo->round >= slz_backoff_round)
334 bo->round = 0;
335 else
336 return;
337 }
338
ae8e83e6 339 tsleep_interlock(s, 0);
d1d48a34 340 if (atomic_intr_cond_test(&s->interlock) != 0) {
d1d48a34 341 logslz(sleep_beg, s);
d9345d3a 342 tsleep(s, PINTERLOCKED, "slize", 0);
d1d48a34
SZ
343 logslz(sleep_end, s);
344 }
d1d48a34
SZ
345}
346
347#endif /* SMP */
348
ed03f3a3
MD
349static void
350lwkt_serialize_wakeup(void *info)
351{
3246ae89 352 logslz(wakeup_beg, info);
ed03f3a3 353 wakeup(info);
3246ae89 354 logslz(wakeup_end, info);
ed03f3a3 355}
d1d48a34
SZ
356
357#ifdef SMP
358static void
359lwkt_serialize_sysinit(void *dummy __unused)
360{
361 if (slz_backoff_round <= 0)
362 slz_backoff_round = ncpus * 2;
363}
364SYSINIT(lwkt_serialize, SI_SUB_PRE_DRIVERS, SI_ORDER_SECOND,
365 lwkt_serialize_sysinit, NULL);
366#endif