kernel: Make SMP support default (and non-optional).
[dragonfly.git] / sys / kern / lwkt_serialize.c
CommitLineData
ed03f3a3
MD
1/*
2 * Copyright (c) 2005 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
ed03f3a3
MD
33 */
34/*
35 * This API provides a fast locked-bus-cycle-based serializer. It's
36 * basically a low level NON-RECURSIVE exclusive lock that can be held across
37 * a blocking condition. It is NOT a mutex.
38 *
39 * This serializer is primarily designed for low level situations and
40 * interrupt/device interaction. There are two primary facilities. First,
41 * the serializer facility itself. Second, an integrated interrupt handler
42 * disablement facility.
43 */
44
45#include <sys/param.h>
46#include <sys/systm.h>
47#include <sys/kernel.h>
48#include <sys/proc.h>
49#include <sys/rtprio.h>
50#include <sys/queue.h>
51#include <sys/thread2.h>
52#include <sys/serialize.h>
53#include <sys/sysctl.h>
027f7bf2 54#include <sys/ktr.h>
ed03f3a3
MD
55#include <sys/kthread.h>
56#include <machine/cpu.h>
d1d48a34
SZ
57#include <machine/cpufunc.h>
58#include <machine/specialreg.h>
ed03f3a3 59#include <sys/lock.h>
ed03f3a3 60
d1d48a34
SZ
61struct exp_backoff {
62 int backoff;
63 int round;
64 lwkt_serialize_t s;
65};
66
027f7bf2 67#define SLZ_KTR_STRING "slz=%p"
5bf48697 68#define SLZ_KTR_ARGS lwkt_serialize_t slz
027f7bf2
SZ
69
70#ifndef KTR_SERIALIZER
71#define KTR_SERIALIZER KTR_ALL
72#endif
73
74KTR_INFO_MASTER(slz);
5bf48697
AE
75KTR_INFO(KTR_SERIALIZER, slz, enter_beg, 0, SLZ_KTR_STRING, SLZ_KTR_ARGS);
76KTR_INFO(KTR_SERIALIZER, slz, sleep_beg, 1, SLZ_KTR_STRING, SLZ_KTR_ARGS);
77KTR_INFO(KTR_SERIALIZER, slz, sleep_end, 2, SLZ_KTR_STRING, SLZ_KTR_ARGS);
78KTR_INFO(KTR_SERIALIZER, slz, exit_end, 3, SLZ_KTR_STRING, SLZ_KTR_ARGS);
79KTR_INFO(KTR_SERIALIZER, slz, wakeup_beg, 4, SLZ_KTR_STRING, SLZ_KTR_ARGS);
80KTR_INFO(KTR_SERIALIZER, slz, wakeup_end, 5, SLZ_KTR_STRING, SLZ_KTR_ARGS);
81KTR_INFO(KTR_SERIALIZER, slz, try, 6, SLZ_KTR_STRING, SLZ_KTR_ARGS);
82KTR_INFO(KTR_SERIALIZER, slz, tryfail, 7, SLZ_KTR_STRING, SLZ_KTR_ARGS);
83KTR_INFO(KTR_SERIALIZER, slz, tryok, 8, SLZ_KTR_STRING, SLZ_KTR_ARGS);
d1d48a34 84KTR_INFO(KTR_SERIALIZER, slz, spinbo, 9,
5bf48697 85 "slz=%p bo1=%d bo=%d", lwkt_serialize_t slz, int backoff1, int backoff);
5bf48697
AE
86KTR_INFO(KTR_SERIALIZER, slz, enter_end, 10, SLZ_KTR_STRING, SLZ_KTR_ARGS);
87KTR_INFO(KTR_SERIALIZER, slz, exit_beg, 11, SLZ_KTR_STRING, SLZ_KTR_ARGS);
027f7bf2 88
d1d48a34
SZ
89#define logslz(name, slz) KTR_LOG(slz_ ## name, slz)
90#define logslz_spinbo(slz, bo1, bo) KTR_LOG(slz_spinbo, slz, bo1, bo)
027f7bf2 91
ed03f3a3
MD
92static void lwkt_serialize_sleep(void *info);
93static void lwkt_serialize_wakeup(void *info);
9db4b353
SZ
94static void lwkt_serialize_adaptive_sleep(void *bo);
95
d1d48a34
SZ
96static int slz_backoff_limit = 128;
97SYSCTL_INT(_debug, OID_AUTO, serialize_bolimit, CTLFLAG_RW,
0c52fa62 98 &slz_backoff_limit, 0, "Backoff limit");
d1d48a34
SZ
99
100static int slz_backoff_shift = 1;
101SYSCTL_INT(_debug, OID_AUTO, serialize_boshift, CTLFLAG_RW,
0c52fa62 102 &slz_backoff_shift, 0, "Backoff shift");
d1d48a34
SZ
103
104static int slz_backoff_round;
105TUNABLE_INT("debug.serialize_boround", &slz_backoff_round);
106SYSCTL_INT(_debug, OID_AUTO, serialize_boround, CTLFLAG_RW,
0c52fa62
SG
107 &slz_backoff_round, 0,
108 "Backoff rounding");
d1d48a34 109
ed03f3a3
MD
110void
111lwkt_serialize_init(lwkt_serialize_t s)
112{
113 atomic_intr_init(&s->interlock);
dcad4d19 114#ifdef INVARIANTS
f0ca723e 115 s->last_td = (void *)-4;
dcad4d19 116#endif
ed03f3a3
MD
117}
118
d1d48a34
SZ
119void
120lwkt_serialize_adaptive_enter(lwkt_serialize_t s)
121{
122 struct exp_backoff bo;
123
124 bo.backoff = 1;
125 bo.round = 0;
126 bo.s = s;
127
9db4b353
SZ
128 ASSERT_NOT_SERIALIZED(s);
129
d1d48a34
SZ
130 logslz(enter_beg, s);
131 atomic_intr_cond_enter(&s->interlock, lwkt_serialize_adaptive_sleep, &bo);
132 logslz(enter_end, s);
dcad4d19 133#ifdef INVARIANTS
d1d48a34 134 s->last_td = curthread;
dcad4d19 135#endif
d1d48a34 136}
d1d48a34 137
ed03f3a3
MD
138void
139lwkt_serialize_enter(lwkt_serialize_t s)
140{
9db4b353
SZ
141 ASSERT_NOT_SERIALIZED(s);
142
d1d48a34 143 logslz(enter_beg, s);
ed03f3a3 144 atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
d1d48a34 145 logslz(enter_end, s);
dcad4d19 146#ifdef INVARIANTS
ed03f3a3 147 s->last_td = curthread;
dcad4d19 148#endif
ed03f3a3
MD
149}
150
3e086307
MD
151/*
152 * Returns non-zero on success
153 */
154int
155lwkt_serialize_try(lwkt_serialize_t s)
156{
157 int error;
158
9db4b353
SZ
159 ASSERT_NOT_SERIALIZED(s);
160
027f7bf2 161 logslz(try, s);
3e086307 162 if ((error = atomic_intr_cond_try(&s->interlock)) == 0) {
dcad4d19 163#ifdef INVARIANTS
3e086307 164 s->last_td = curthread;
dcad4d19 165#endif
027f7bf2 166 logslz(tryok, s);
3e086307
MD
167 return(1);
168 }
027f7bf2 169 logslz(tryfail, s);
3e086307
MD
170 return (0);
171}
172
ed03f3a3
MD
173void
174lwkt_serialize_exit(lwkt_serialize_t s)
175{
9db4b353 176 ASSERT_SERIALIZED(s);
dcad4d19 177#ifdef INVARIANTS
f0ca723e 178 s->last_td = (void *)-2;
dcad4d19 179#endif
d1d48a34 180 logslz(exit_beg, s);
ed03f3a3 181 atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
d1d48a34 182 logslz(exit_end, s);
ed03f3a3
MD
183}
184
185/*
186 * Interrupt handler disablement support, used by drivers. Non-stackable
187 * (uses bit 30).
188 */
189void
190lwkt_serialize_handler_disable(lwkt_serialize_t s)
191{
192 atomic_intr_handler_disable(&s->interlock);
193}
194
195void
196lwkt_serialize_handler_enable(lwkt_serialize_t s)
197{
198 atomic_intr_handler_enable(&s->interlock);
199}
200
201void
477d3c1c
MD
202lwkt_serialize_handler_call(lwkt_serialize_t s, void (*func)(void *, void *),
203 void *arg, void *frame)
ed03f3a3 204{
9c095379
MD
205 /*
206 * note: a return value of 0 indicates that the interrupt handler is
207 * enabled.
208 */
209 if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
d1d48a34 210 logslz(enter_beg, s);
ed03f3a3 211 atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
d1d48a34 212 logslz(enter_end, s);
dcad4d19 213#ifdef INVARIANTS
92a4dd35
MD
214 s->last_td = curthread;
215#endif
ed03f3a3 216 if (atomic_intr_handler_is_enabled(&s->interlock) == 0)
477d3c1c 217 func(arg, frame);
9db4b353
SZ
218
219 ASSERT_SERIALIZED(s);
dcad4d19 220#ifdef INVARIANTS
92a4dd35 221 s->last_td = (void *)-2;
dcad4d19 222#endif
d1d48a34 223 logslz(exit_beg, s);
e139cf18 224 atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
d1d48a34 225 logslz(exit_end, s);
ed03f3a3
MD
226 }
227}
228
477d3c1c
MD
229/*
230 * Similar to handler_call but does not block. Returns 0 on success,
231 * and 1 on failure.
232 */
233int
234lwkt_serialize_handler_try(lwkt_serialize_t s, void (*func)(void *, void *),
235 void *arg, void *frame)
236{
237 /*
238 * note: a return value of 0 indicates that the interrupt handler is
239 * enabled.
240 */
241 if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
027f7bf2 242 logslz(try, s);
477d3c1c 243 if (atomic_intr_cond_try(&s->interlock) == 0) {
dcad4d19 244#ifdef INVARIANTS
92a4dd35 245 s->last_td = curthread;
dcad4d19 246#endif
027f7bf2 247 logslz(tryok, s);
9db4b353 248
477d3c1c 249 func(arg, frame);
9db4b353
SZ
250
251 ASSERT_SERIALIZED(s);
dcad4d19 252#ifdef INVARIANTS
92a4dd35 253 s->last_td = (void *)-2;
dcad4d19 254#endif
d1d48a34 255 logslz(exit_beg, s);
92a4dd35 256 atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
d1d48a34 257 logslz(exit_end, s);
477d3c1c
MD
258 return(0);
259 }
260 }
027f7bf2 261 logslz(tryfail, s);
477d3c1c
MD
262 return(1);
263}
264
265
ed03f3a3
MD
266/*
267 * Helper functions
d02b46cd
MD
268 *
269 * It is possible to race an interrupt which acquires and releases the
270 * bit, then calls wakeup before we actually go to sleep, so we
f0ca723e
MD
271 * need to check that the interlock is still acquired from within
272 * a critical section prior to sleeping.
ed03f3a3
MD
273 */
274static void
275lwkt_serialize_sleep(void *info)
276{
277 lwkt_serialize_t s = info;
ae8e83e6
MD
278
279 tsleep_interlock(s, 0);
21fa6062 280 if (atomic_intr_cond_test(&s->interlock) != 0) {
3246ae89 281 logslz(sleep_beg, s);
d9345d3a 282 tsleep(s, PINTERLOCKED, "slize", 0);
3246ae89 283 logslz(sleep_end, s);
21fa6062 284 }
ed03f3a3
MD
285}
286
d1d48a34
SZ
287static void
288lwkt_serialize_adaptive_sleep(void *arg)
289{
290 struct exp_backoff *bo = arg;
291 lwkt_serialize_t s = bo->s;
292 int backoff;
293
294 /*
295 * Randomize backoff value
296 */
297#ifdef _RDTSC_SUPPORTED_
298 if (cpu_feature & CPUID_TSC) {
299 backoff =
300 (((u_long)rdtsc() ^ (((u_long)curthread) >> 5)) &
301 (bo->backoff - 1)) + 1;
302 } else
303#endif
304 backoff = bo->backoff;
305
306 logslz_spinbo(s, bo->backoff, backoff);
307
308 /*
309 * Quick backoff
310 */
311 for (; backoff; --backoff)
14dd663d 312 cpu_pause();
d1d48a34
SZ
313 if (bo->backoff < slz_backoff_limit) {
314 bo->backoff <<= slz_backoff_shift;
315 return;
316 } else {
317 bo->backoff = 1;
318 bo->round++;
319 if (bo->round >= slz_backoff_round)
320 bo->round = 0;
321 else
322 return;
323 }
324
ae8e83e6 325 tsleep_interlock(s, 0);
d1d48a34 326 if (atomic_intr_cond_test(&s->interlock) != 0) {
d1d48a34 327 logslz(sleep_beg, s);
d9345d3a 328 tsleep(s, PINTERLOCKED, "slize", 0);
d1d48a34
SZ
329 logslz(sleep_end, s);
330 }
d1d48a34
SZ
331}
332
ed03f3a3
MD
333static void
334lwkt_serialize_wakeup(void *info)
335{
3246ae89 336 logslz(wakeup_beg, info);
ed03f3a3 337 wakeup(info);
3246ae89 338 logslz(wakeup_end, info);
ed03f3a3 339}
d1d48a34 340
d1d48a34
SZ
341static void
342lwkt_serialize_sysinit(void *dummy __unused)
343{
344 if (slz_backoff_round <= 0)
345 slz_backoff_round = ncpus * 2;
346}
347SYSINIT(lwkt_serialize, SI_SUB_PRE_DRIVERS, SI_ORDER_SECOND,
348 lwkt_serialize_sysinit, NULL);