kernel - pipe locks are not needed in the kqueue event code
[dragonfly.git] / sys / sys / spinlock2.h
1 /*
2  * Copyright (c) 2005 Jeffrey M. Hsu.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Jeffrey M. Hsu.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  * 3. Neither the name of The DragonFly Project nor the names of its
16  *    contributors may be used to endorse or promote products derived
17  *    from this software without specific, prior written permission.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
22  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
23  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
24  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
25  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
26  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
27  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
29  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30  * SUCH DAMAGE.
31  */
32
33 #ifndef _SYS_SPINLOCK2_H_
34 #define _SYS_SPINLOCK2_H_
35
36 #ifndef _KERNEL
37
38 #error "This file should not be included by userland programs."
39
40 #else
41
42 #ifndef _SYS_SYSTM_H_
43 #include <sys/systm.h>
44 #endif
45 #ifndef _SYS_THREAD2_H_
46 #include <sys/thread2.h>
47 #endif
48 #ifndef _SYS_GLOBALDATA_H_
49 #include <sys/globaldata.h>
50 #endif
51 #include <machine/atomic.h>
52 #include <machine/cpufunc.h>
53
54 extern struct spinlock pmap_spin;
55
56 int spin_trylock_contested(struct spinlock *spin);
57 void _spin_lock_contested(struct spinlock *spin, const char *ident, int count);
58 void _spin_lock_shared_contested(struct spinlock *spin, const char *ident);
59
60 #define spin_lock(spin)                 _spin_lock(spin, __func__)
61 #define spin_lock_quick(spin)           _spin_lock_quick(spin, __func__)
62 #define spin_lock_shared(spin)          _spin_lock_shared(spin, __func__)
63 #define spin_lock_shared_quick(spin)    _spin_lock_shared_quick(spin, __func__)
64
65 /*
66  * Attempt to obtain an exclusive spinlock.  Returns FALSE on failure,
67  * TRUE on success.
68  */
69 static __inline boolean_t
70 spin_trylock(struct spinlock *spin)
71 {
72         globaldata_t gd = mycpu;
73
74         crit_enter_raw(gd->gd_curthread);
75         ++gd->gd_spinlocks;
76         cpu_ccfence();
77         if (atomic_cmpset_int(&spin->counta, 0, 1) == 0)
78                 return (spin_trylock_contested(spin));
79 #ifdef DEBUG_LOCKS
80         int i;
81         for (i = 0; i < SPINLOCK_DEBUG_ARRAY_SIZE; i++) {
82                 if (gd->gd_curthread->td_spinlock_stack_id[i] == 0) {
83                         gd->gd_curthread->td_spinlock_stack_id[i] = 1;
84                         gd->gd_curthread->td_spinlock_stack[i] = spin;
85                         gd->gd_curthread->td_spinlock_caller_pc[i] =
86                                 __builtin_return_address(0);
87                         break;
88                 }
89         }
90 #endif
91         return (TRUE);
92 }
93
94 /*
95  * Return TRUE if the spinlock is held (we can't tell by whom, though)
96  */
97 static __inline int
98 spin_held(struct spinlock *spin)
99 {
100         return((spin->counta & ~SPINLOCK_SHARED) != 0);
101 }
102
103 /*
104  * Obtain an exclusive spinlock and return.  It is possible for the
105  * SPINLOCK_SHARED bit to already be set, in which case the contested
106  * code is called to fix it up.
107  */
108 static __inline void
109 _spin_lock_quick(globaldata_t gd, struct spinlock *spin, const char *ident)
110 {
111         int count;
112
113         crit_enter_raw(gd->gd_curthread);
114         ++gd->gd_spinlocks;
115         cpu_ccfence();
116
117         count = atomic_fetchadd_int(&spin->counta, 1);
118         if (__predict_false(count != 0)) {
119                 _spin_lock_contested(spin, ident, count);
120         }
121 #ifdef DEBUG_LOCKS
122         int i;
123         for (i = 0; i < SPINLOCK_DEBUG_ARRAY_SIZE; i++) {
124                 if (gd->gd_curthread->td_spinlock_stack_id[i] == 0) {
125                         gd->gd_curthread->td_spinlock_stack_id[i] = 1;
126                         gd->gd_curthread->td_spinlock_stack[i] = spin;
127                         gd->gd_curthread->td_spinlock_caller_pc[i] =
128                                 __builtin_return_address(0);
129                         break;
130                 }
131         }
132 #endif
133 }
134
135 static __inline void
136 _spin_lock(struct spinlock *spin, const char *ident)
137 {
138         _spin_lock_quick(mycpu, spin, ident);
139 }
140
141 /*
142  * Release an exclusive spinlock.  We can just do this passively, only
143  * ensuring that our spinlock count is left intact until the mutex is
144  * cleared.
145  */
146 static __inline void
147 spin_unlock_quick(globaldata_t gd, struct spinlock *spin)
148 {
149 #ifdef DEBUG_LOCKS
150         int i;
151         for (i = 0; i < SPINLOCK_DEBUG_ARRAY_SIZE; i++) {
152                 if ((gd->gd_curthread->td_spinlock_stack_id[i] == 1) &&
153                     (gd->gd_curthread->td_spinlock_stack[i] == spin)) {
154                         gd->gd_curthread->td_spinlock_stack_id[i] = 0;
155                         gd->gd_curthread->td_spinlock_stack[i] = NULL;
156                         gd->gd_curthread->td_spinlock_caller_pc[i] = NULL;
157                         break;
158                 }
159         }
160 #endif
161         /*
162          * Don't use a locked instruction here.  To reduce latency we avoid
163          * reading spin->counta prior to writing to it.
164          */
165 #ifdef DEBUG_LOCKS
166         KKASSERT(spin->counta != 0);
167 #endif
168         cpu_sfence();
169         atomic_add_int(&spin->counta, -1);
170         cpu_sfence();
171 #ifdef DEBUG_LOCKS
172         KKASSERT(gd->gd_spinlocks > 0);
173 #endif
174         cpu_ccfence();
175         --gd->gd_spinlocks;
176         crit_exit_raw(gd->gd_curthread);
177 }
178
179 static __inline void
180 spin_unlock(struct spinlock *spin)
181 {
182         spin_unlock_quick(mycpu, spin);
183 }
184
185 /*
186  * Shared spinlock.  Acquire a count, if SPINLOCK_SHARED is not already
187  * set then try a trivial conversion and drop into the contested code if
188  * the trivial cocnversion fails.  The SHARED bit is 'cached' when lock
189  * counts go to 0 so the critical path is typically just the fetchadd.
190  *
191  * WARNING!  Due to the way exclusive conflict resolution works, we cannot
192  *           just unconditionally set the SHARED bit on previous-count == 0.
193  *           Doing so will interfere with the exclusive contended code.
194  */
195 static __inline void
196 _spin_lock_shared_quick(globaldata_t gd, struct spinlock *spin,
197                         const char *ident)
198 {
199         int counta;
200
201         crit_enter_raw(gd->gd_curthread);
202         ++gd->gd_spinlocks;
203         cpu_ccfence();
204
205         counta = atomic_fetchadd_int(&spin->counta, 1);
206         if (__predict_false((counta & SPINLOCK_SHARED) == 0)) {
207                 if (counta != 0 ||
208                     !atomic_cmpset_int(&spin->counta, 1, SPINLOCK_SHARED | 1)) {
209                         _spin_lock_shared_contested(spin, ident);
210                 }
211         }
212 #ifdef DEBUG_LOCKS
213         int i;
214         for (i = 0; i < SPINLOCK_DEBUG_ARRAY_SIZE; i++) {
215                 if (gd->gd_curthread->td_spinlock_stack_id[i] == 0) {
216                         gd->gd_curthread->td_spinlock_stack_id[i] = 1;
217                         gd->gd_curthread->td_spinlock_stack[i] = spin;
218                         gd->gd_curthread->td_spinlock_caller_pc[i] =
219                                 __builtin_return_address(0);
220                         break;
221                 }
222         }
223 #endif
224 }
225
226 /*
227  * Unlock a shared lock.  For convenience we allow the last transition
228  * to be to (SPINLOCK_SHARED|0), leaving the SPINLOCK_SHARED bit set
229  * with a count to 0 which will optimize the next shared lock obtained.
230  *
231  * WARNING! In order to implement shared and exclusive spinlocks, an
232  *          exclusive request will convert a multiply-held shared lock
233  *          to exclusive and wait for shared holders to unlock.  So keep
234  *          in mind that as of now the spinlock could actually be in an
235  *          exclusive state.
236  */
237 static __inline void
238 spin_unlock_shared_quick(globaldata_t gd, struct spinlock *spin)
239 {
240 #ifdef DEBUG_LOCKS
241         int i;
242         for (i = 0; i < SPINLOCK_DEBUG_ARRAY_SIZE; i++) {
243                 if ((gd->gd_curthread->td_spinlock_stack_id[i] == 1) &&
244                     (gd->gd_curthread->td_spinlock_stack[i] == spin)) {
245                         gd->gd_curthread->td_spinlock_stack_id[i] = 0;
246                         gd->gd_curthread->td_spinlock_stack[i] = NULL;
247                         gd->gd_curthread->td_spinlock_caller_pc[i] = NULL;
248                         break;
249                 }
250         }
251 #endif
252 #ifdef DEBUG_LOCKS
253         KKASSERT(spin->counta != 0);
254 #endif
255         cpu_sfence();
256         atomic_add_int(&spin->counta, -1);
257
258 #ifdef DEBUG_LOCKS
259         KKASSERT(gd->gd_spinlocks > 0);
260 #endif
261         cpu_ccfence();
262         --gd->gd_spinlocks;
263         crit_exit_raw(gd->gd_curthread);
264 }
265
266 static __inline void
267 _spin_lock_shared(struct spinlock *spin, const char *ident)
268 {
269         _spin_lock_shared_quick(mycpu, spin, ident);
270 }
271
272 static __inline void
273 spin_unlock_shared(struct spinlock *spin)
274 {
275         spin_unlock_shared_quick(mycpu, spin);
276 }
277
278 /*
279  * Attempt to upgrade a shared spinlock to exclusive.  Return non-zero
280  * on success, 0 on failure.
281  */
282 static __inline int
283 spin_lock_upgrade_try(struct spinlock *spin)
284 {
285         if (atomic_cmpset_int(&spin->counta, SPINLOCK_SHARED|1, 1))
286                 return 1;
287         else
288                 return 0;
289 }
290
291 static __inline void
292 spin_init(struct spinlock *spin, const char *descr __unused)
293 {
294         spin->counta = 0;
295         spin->countb = 0;
296 #if 0
297         spin->descr  = descr;
298 #endif
299 }
300
301 static __inline void
302 spin_uninit(struct spinlock *spin)
303 {
304         /* unused */
305 }
306
307 #endif  /* _KERNEL */
308 #endif  /* _SYS_SPINLOCK2_H_ */
309