kernel - revamp mtx_spinlock()
[dragonfly.git] / sys / kern / kern_mutex.c
CommitLineData
33b0b87c
MD
1/*
2 * Copyright (c) 2009 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34/*
35 * Implement fast persistent locks based on atomic_cmpset_int() with
36 * semantics similar to lockmgr locks but faster and taking up much less
37 * space. Taken from HAMMER's lock implementation.
38 *
39 * These are meant to complement our LWKT tokens. Tokens are only held
40 * while the thread is running. Mutexes can be held across blocking
41 * conditions.
42 *
43 * Most of the support is in sys/mutex[2].h. We mostly provide backoff
44 * functions here.
45 */
46
47#include <sys/param.h>
48#include <sys/systm.h>
49#include <sys/kernel.h>
50#include <sys/sysctl.h>
51#include <sys/thread.h>
33b0b87c
MD
52
53#include <machine/cpufunc.h>
54
55#include <sys/thread2.h>
56#include <sys/mutex2.h>
57
58static __int64_t mtx_contention_count;
59static __int64_t mtx_collision_count;
60static __int64_t mtx_wakeup_count;
61
62SYSCTL_QUAD(_kern, OID_AUTO, mtx_contention_count, CTLFLAG_RW,
63 &mtx_contention_count, 0, "");
64SYSCTL_QUAD(_kern, OID_AUTO, mtx_collision_count, CTLFLAG_RW,
65 &mtx_collision_count, 0, "");
66SYSCTL_QUAD(_kern, OID_AUTO, mtx_wakeup_count, CTLFLAG_RW,
67 &mtx_wakeup_count, 0, "");
68
685ebdab
MD
69static void mtx_chain_link(mtx_t mtx);
70static void mtx_delete_link(mtx_t mtx, mtx_link_t link);
71
33b0b87c
MD
72/*
73 * Exclusive-lock a mutex, block until acquired. Recursion is allowed.
7355baa5
MD
74 *
75 * Returns 0 on success, or the tsleep() return code on failure.
76 * An error can only be returned if PCATCH is specified in the flags.
33b0b87c 77 */
7355baa5 78static __inline int
685ebdab 79__mtx_lock_ex(mtx_t mtx, mtx_link_t link, const char *ident, int flags, int to)
33b0b87c
MD
80{
81 u_int lock;
82 u_int nlock;
7355baa5 83 int error;
33b0b87c
MD
84
85 for (;;) {
86 lock = mtx->mtx_lock;
87 if (lock == 0) {
88 nlock = MTX_EXCLUSIVE | 1;
89 if (atomic_cmpset_int(&mtx->mtx_lock, 0, nlock)) {
7355baa5
MD
90 mtx->mtx_owner = curthread;
91 error = 0;
92 break;
33b0b87c
MD
93 }
94 } else if ((lock & MTX_EXCLUSIVE) &&
95 mtx->mtx_owner == curthread) {
96 KKASSERT((lock & MTX_MASK) != MTX_MASK);
685ebdab
MD
97 nlock = lock + 1;
98 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock)) {
7355baa5
MD
99 error = 0;
100 break;
101 }
33b0b87c 102 } else {
685ebdab
MD
103 /*
104 * Clearing MTX_EXLINK in lock causes us to loop until
105 * MTX_EXLINK is available. However, to avoid
106 * unnecessary cpu cache traffic we poll instead.
107 *
108 * Setting MTX_EXLINK in nlock causes us to loop until
109 * we can acquire MTX_EXLINK.
110 *
111 * Also set MTX_EXWANTED coincident with EXLINK, if
112 * not already set.
113 */
77912481
MD
114 thread_t td;
115
685ebdab
MD
116 if (lock & MTX_EXLINK) {
117 cpu_pause();
118 ++mtx_collision_count;
119 continue;
120 }
77912481 121 td = curthread;
685ebdab
MD
122 /*lock &= ~MTX_EXLINK;*/
123 nlock = lock | MTX_EXWANTED | MTX_EXLINK;
77912481 124 ++td->td_critcount;
685ebdab
MD
125 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock)) {
126 /*
127 * Check for early abort
128 */
129 if (link->state == MTX_LINK_ABORTED) {
130 atomic_clear_int(&mtx->mtx_lock,
131 MTX_EXLINK);
77912481 132 --td->td_critcount;
685ebdab
MD
133 error = ENOLCK;
134 if (mtx->mtx_link == NULL) {
135 atomic_clear_int(&mtx->mtx_lock,
136 MTX_EXWANTED);
137 }
138 break;
139 }
140
141 /*
142 * Success. Link in our structure then
143 * release EXLINK and sleep.
144 */
77912481 145 link->owner = td;
685ebdab
MD
146 link->state = MTX_LINK_LINKED;
147 if (mtx->mtx_link) {
148 link->next = mtx->mtx_link;
149 link->prev = link->next->prev;
150 link->next->prev = link;
151 link->prev->next = link;
152 } else {
153 link->next = link;
154 link->prev = link;
155 mtx->mtx_link = link;
156 }
157 tsleep_interlock(link, 0);
158 atomic_clear_int(&mtx->mtx_lock, MTX_EXLINK);
77912481 159 --td->td_critcount;
685ebdab
MD
160
161 error = tsleep(link, flags, ident, to);
33b0b87c 162 ++mtx_contention_count;
685ebdab
MD
163
164 /*
165 * Normal unlink, we should own the exclusive
166 * lock now.
167 */
168 if (link->state == MTX_LINK_LINKED)
169 mtx_delete_link(mtx, link);
170 if (link->state == MTX_LINK_ACQUIRED) {
171 KKASSERT(mtx->mtx_owner == link->owner);
172 error = 0;
7355baa5
MD
173 break;
174 }
685ebdab
MD
175
176 /*
177 * Aborted lock (mtx_abort_ex called).
178 */
179 if (link->state == MTX_LINK_ABORTED) {
180 error = ENOLCK;
181 break;
182 }
183
184 /*
185 * tsleep error, else retry.
186 */
187 if (error)
188 break;
54d51ad1 189 } else {
77912481 190 --td->td_critcount;
33b0b87c
MD
191 }
192 }
193 ++mtx_collision_count;
194 }
7355baa5
MD
195 return (error);
196}
197
198int
685ebdab
MD
199_mtx_lock_ex_link(mtx_t mtx, mtx_link_t link,
200 const char *ident, int flags, int to)
201{
202 return(__mtx_lock_ex(mtx, link, ident, flags, to));
203}
204
205int
7355baa5
MD
206_mtx_lock_ex(mtx_t mtx, const char *ident, int flags, int to)
207{
685ebdab
MD
208 struct mtx_link link;
209
210 mtx_link_init(&link);
211 return(__mtx_lock_ex(mtx, &link, ident, flags, to));
7355baa5
MD
212}
213
214int
215_mtx_lock_ex_quick(mtx_t mtx, const char *ident)
216{
685ebdab
MD
217 struct mtx_link link;
218
219 mtx_link_init(&link);
220 return(__mtx_lock_ex(mtx, &link, ident, 0, 0));
33b0b87c
MD
221}
222
223/*
224 * Share-lock a mutex, block until acquired. Recursion is allowed.
7355baa5
MD
225 *
226 * Returns 0 on success, or the tsleep() return code on failure.
227 * An error can only be returned if PCATCH is specified in the flags.
685ebdab
MD
228 *
229 * NOTE: Shared locks get a mass-wakeup so if the tsleep fails we
230 * do not have to chain the wakeup().
33b0b87c 231 */
7355baa5
MD
232static __inline int
233__mtx_lock_sh(mtx_t mtx, const char *ident, int flags, int to)
33b0b87c
MD
234{
235 u_int lock;
236 u_int nlock;
7355baa5 237 int error;
33b0b87c
MD
238
239 for (;;) {
240 lock = mtx->mtx_lock;
241 if ((lock & MTX_EXCLUSIVE) == 0) {
242 KKASSERT((lock & MTX_MASK) != MTX_MASK);
243 nlock = lock + 1;
7355baa5
MD
244 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock)) {
245 error = 0;
246 break;
247 }
33b0b87c
MD
248 } else {
249 nlock = lock | MTX_SHWANTED;
250 tsleep_interlock(mtx, 0);
251 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock)) {
7355baa5
MD
252 error = tsleep(mtx, flags, ident, to);
253 if (error)
254 break;
33b0b87c 255 ++mtx_contention_count;
685ebdab 256 /* retry */
54d51ad1
MD
257 } else {
258 tsleep_remove(curthread);
33b0b87c
MD
259 }
260 }
261 ++mtx_collision_count;
262 }
7355baa5
MD
263 return (error);
264}
265
266int
267_mtx_lock_sh(mtx_t mtx, const char *ident, int flags, int to)
268{
269 return (__mtx_lock_sh(mtx, ident, flags, to));
270}
271
272int
273_mtx_lock_sh_quick(mtx_t mtx, const char *ident)
274{
275 return (__mtx_lock_sh(mtx, ident, 0, 0));
33b0b87c
MD
276}
277
c40689e9
MD
278/*
279 * Get an exclusive spinlock the hard way.
280 */
33b0b87c 281void
c40689e9 282_mtx_spinlock(mtx_t mtx)
33b0b87c
MD
283{
284 u_int lock;
285 u_int nlock;
286 int bb = 1;
287 int bo;
288
289 for (;;) {
290 lock = mtx->mtx_lock;
291 if (lock == 0) {
292 nlock = MTX_EXCLUSIVE | 1;
293 if (atomic_cmpset_int(&mtx->mtx_lock, 0, nlock)) {
685ebdab
MD
294 mtx->mtx_owner = curthread;
295 break;
33b0b87c
MD
296 }
297 } else if ((lock & MTX_EXCLUSIVE) &&
298 mtx->mtx_owner == curthread) {
299 KKASSERT((lock & MTX_MASK) != MTX_MASK);
685ebdab
MD
300 nlock = lock + 1;
301 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock))
302 break;
33b0b87c
MD
303 } else {
304 /* MWAIT here */
305 if (bb < 1000)
306 ++bb;
307 cpu_pause();
308 for (bo = 0; bo < bb; ++bo)
309 ;
310 ++mtx_contention_count;
311 }
685ebdab 312 cpu_pause();
33b0b87c
MD
313 ++mtx_collision_count;
314 }
315}
316
c40689e9
MD
317/*
318 * Attempt to acquire a spinlock, if we fail we must undo the
319 * gd->gd_spinlocks_wr/gd->gd_curthead->td_critcount predisposition.
320 *
321 * Returns 0 on success, EAGAIN on failure.
322 */
323int
324_mtx_spinlock_try(mtx_t mtx)
325{
326 globaldata_t gd = mycpu;
327 u_int lock;
328 u_int nlock;
329 int res = 0;
330
331 for (;;) {
332 lock = mtx->mtx_lock;
333 if (lock == 0) {
334 nlock = MTX_EXCLUSIVE | 1;
335 if (atomic_cmpset_int(&mtx->mtx_lock, 0, nlock)) {
336 mtx->mtx_owner = gd->gd_curthread;
337 break;
338 }
339 } else if ((lock & MTX_EXCLUSIVE) &&
340 mtx->mtx_owner == gd->gd_curthread) {
341 KKASSERT((lock & MTX_MASK) != MTX_MASK);
342 nlock = lock + 1;
343 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock))
344 break;
345 } else {
346 --gd->gd_spinlocks_wr;
347 cpu_ccfence();
348 --gd->gd_curthread->td_critcount;
349 res = EAGAIN;
350 break;
351 }
352 cpu_pause();
353 ++mtx_collision_count;
354 }
355 return res;
356}
357
358#if 0
359
33b0b87c
MD
360void
361_mtx_spinlock_sh(mtx_t mtx)
362{
363 u_int lock;
364 u_int nlock;
365 int bb = 1;
366 int bo;
367
368 for (;;) {
369 lock = mtx->mtx_lock;
370 if ((lock & MTX_EXCLUSIVE) == 0) {
371 KKASSERT((lock & MTX_MASK) != MTX_MASK);
372 nlock = lock + 1;
373 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock))
685ebdab 374 break;
33b0b87c
MD
375 } else {
376 /* MWAIT here */
377 if (bb < 1000)
378 ++bb;
379 cpu_pause();
380 for (bo = 0; bo < bb; ++bo)
381 ;
382 ++mtx_contention_count;
383 }
685ebdab 384 cpu_pause();
33b0b87c
MD
385 ++mtx_collision_count;
386 }
387}
388
c40689e9
MD
389#endif
390
33b0b87c
MD
391int
392_mtx_lock_ex_try(mtx_t mtx)
393{
394 u_int lock;
395 u_int nlock;
396 int error = 0;
397
398 for (;;) {
399 lock = mtx->mtx_lock;
400 if (lock == 0) {
401 nlock = MTX_EXCLUSIVE | 1;
402 if (atomic_cmpset_int(&mtx->mtx_lock, 0, nlock)) {
685ebdab 403 mtx->mtx_owner = curthread;
33b0b87c
MD
404 break;
405 }
406 } else if ((lock & MTX_EXCLUSIVE) &&
407 mtx->mtx_owner == curthread) {
408 KKASSERT((lock & MTX_MASK) != MTX_MASK);
685ebdab
MD
409 nlock = lock + 1;
410 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock))
33b0b87c
MD
411 break;
412 } else {
413 error = EAGAIN;
414 break;
415 }
685ebdab 416 cpu_pause();
33b0b87c
MD
417 ++mtx_collision_count;
418 }
419 return (error);
420}
421
422int
423_mtx_lock_sh_try(mtx_t mtx)
424{
425 u_int lock;
426 u_int nlock;
427 int error = 0;
428
429 for (;;) {
430 lock = mtx->mtx_lock;
431 if ((lock & MTX_EXCLUSIVE) == 0) {
432 KKASSERT((lock & MTX_MASK) != MTX_MASK);
433 nlock = lock + 1;
434 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock))
435 break;
436 } else {
437 error = EAGAIN;
438 break;
439 }
685ebdab 440 cpu_pause();
33b0b87c
MD
441 ++mtx_collision_count;
442 }
443 return (error);
444}
445
446/*
447 * If the lock is held exclusively it must be owned by the caller. If the
448 * lock is already a shared lock this operation is a NOP. A panic will
449 * occur if the lock is not held either shared or exclusive.
450 *
451 * The exclusive count is converted to a shared count.
452 */
453void
454_mtx_downgrade(mtx_t mtx)
455{
456 u_int lock;
457 u_int nlock;
458
459 for (;;) {
460 lock = mtx->mtx_lock;
461 if ((lock & MTX_EXCLUSIVE) == 0) {
462 KKASSERT((lock & MTX_MASK) > 0);
463 break;
464 }
465 KKASSERT(mtx->mtx_owner == curthread);
466 nlock = lock & ~(MTX_EXCLUSIVE | MTX_SHWANTED);
467 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock)) {
468 if (lock & MTX_SHWANTED) {
33b0b87c 469 wakeup(mtx);
7355baa5 470 ++mtx_wakeup_count;
33b0b87c
MD
471 }
472 break;
473 }
685ebdab 474 cpu_pause();
33b0b87c
MD
475 ++mtx_collision_count;
476 }
477}
478
479/*
480 * Upgrade a shared lock to an exclusive lock. The upgrade will fail if
481 * the shared lock has a count other then 1. Optimize the most likely case
482 * but note that a single cmpset can fail due to WANTED races.
483 *
484 * If the lock is held exclusively it must be owned by the caller and
485 * this function will simply return without doing anything. A panic will
486 * occur if the lock is held exclusively by someone other then the caller.
487 *
488 * Returns 0 on success, EDEADLK on failure.
489 */
490int
491_mtx_upgrade_try(mtx_t mtx)
492{
493 u_int lock;
494 u_int nlock;
495 int error = 0;
496
497 for (;;) {
498 lock = mtx->mtx_lock;
499
500 if ((lock & ~MTX_EXWANTED) == 1) {
501 nlock = lock | MTX_EXCLUSIVE;
502 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock)) {
503 mtx->mtx_owner = curthread;
504 break;
505 }
506 } else if (lock & MTX_EXCLUSIVE) {
507 KKASSERT(mtx->mtx_owner == curthread);
508 break;
509 } else {
510 error = EDEADLK;
511 break;
512 }
685ebdab 513 cpu_pause();
33b0b87c
MD
514 ++mtx_collision_count;
515 }
516 return (error);
517}
518
519/*
520 * Unlock a lock. The caller must hold the lock either shared or exclusive.
685ebdab
MD
521 *
522 * Any release which makes the lock available when others want an exclusive
523 * lock causes us to chain the owner to the next exclusive lock instead of
524 * releasing the lock.
33b0b87c
MD
525 */
526void
527_mtx_unlock(mtx_t mtx)
528{
529 u_int lock;
530 u_int nlock;
531
532 for (;;) {
533 lock = mtx->mtx_lock;
685ebdab
MD
534 nlock = lock & ~(MTX_SHWANTED | MTX_EXLINK);
535
536 if (nlock == 1) {
537 /*
538 * Last release, shared lock, no exclusive waiters.
539 */
540 nlock = lock & MTX_EXLINK;
541 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock))
542 break;
543 } else if (nlock == (MTX_EXCLUSIVE | 1)) {
544 /*
545 * Last release, exclusive lock, no exclusive waiters.
546 * Wake up any shared waiters.
547 */
33b0b87c 548 mtx->mtx_owner = NULL;
685ebdab
MD
549 nlock = lock & MTX_EXLINK;
550 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock)) {
54d51ad1 551 if (lock & MTX_SHWANTED) {
33b0b87c 552 wakeup(mtx);
7355baa5 553 ++mtx_wakeup_count;
33b0b87c
MD
554 }
555 break;
556 }
685ebdab
MD
557 } else if (nlock == (MTX_EXWANTED | 1)) {
558 /*
559 * Last release, shared lock, with exclusive
560 * waiters.
561 *
562 * Wait for EXLINK to clear, then acquire it.
563 * We could use the cmpset for this but polling
564 * is better on the cpu caches.
565 *
566 * Acquire an exclusive lock leaving the lockcount
567 * set to 1, and get EXLINK for access to mtx_link.
568 */
77912481
MD
569 thread_t td;
570
685ebdab
MD
571 if (lock & MTX_EXLINK) {
572 cpu_pause();
573 ++mtx_collision_count;
574 continue;
575 }
77912481 576 td = curthread;
685ebdab
MD
577 /*lock &= ~MTX_EXLINK;*/
578 nlock |= MTX_EXLINK | MTX_EXCLUSIVE;
579 nlock |= (lock & MTX_SHWANTED);
77912481 580 ++td->td_critcount;
685ebdab
MD
581 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock)) {
582 mtx_chain_link(mtx);
77912481 583 --td->td_critcount;
685ebdab
MD
584 break;
585 }
77912481 586 --td->td_critcount;
685ebdab
MD
587 } else if (nlock == (MTX_EXCLUSIVE | MTX_EXWANTED | 1)) {
588 /*
589 * Last release, exclusive lock, with exclusive
590 * waiters.
591 *
592 * leave the exclusive lock intact and the lockcount
593 * set to 1, and get EXLINK for access to mtx_link.
594 */
77912481
MD
595 thread_t td;
596
685ebdab
MD
597 if (lock & MTX_EXLINK) {
598 cpu_pause();
599 ++mtx_collision_count;
600 continue;
601 }
77912481 602 td = curthread;
685ebdab
MD
603 /*lock &= ~MTX_EXLINK;*/
604 nlock |= MTX_EXLINK;
605 nlock |= (lock & MTX_SHWANTED);
77912481 606 ++td->td_critcount;
685ebdab
MD
607 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock)) {
608 mtx_chain_link(mtx);
77912481 609 --td->td_critcount;
685ebdab
MD
610 break;
611 }
77912481 612 --td->td_critcount;
33b0b87c 613 } else {
685ebdab
MD
614 /*
615 * Not the last release (shared or exclusive)
616 */
33b0b87c
MD
617 nlock = lock - 1;
618 KKASSERT((nlock & MTX_MASK) != MTX_MASK);
619 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock))
620 break;
621 }
685ebdab
MD
622 cpu_pause();
623 ++mtx_collision_count;
624 }
625}
626
627/*
628 * Chain mtx_chain_link. Called with the lock held exclusively with a
629 * single ref count, and also with MTX_EXLINK held.
630 */
631static void
632mtx_chain_link(mtx_t mtx)
633{
634 mtx_link_t link;
635 u_int lock;
636 u_int nlock;
637 u_int clock; /* bits we own and want to clear */
638
639 /*
640 * Chain the exclusive lock to the next link. The caller cleared
641 * SHWANTED so if there is no link we have to wake up any shared
642 * waiters.
643 */
644 clock = MTX_EXLINK;
645 if ((link = mtx->mtx_link) != NULL) {
646 KKASSERT(link->state == MTX_LINK_LINKED);
647 if (link->next == link) {
648 mtx->mtx_link = NULL;
649 clock |= MTX_EXWANTED;
650 } else {
651 mtx->mtx_link = link->next;
652 link->next->prev = link->prev;
653 link->prev->next = link->next;
654 }
655 link->state = MTX_LINK_ACQUIRED;
656 mtx->mtx_owner = link->owner;
657 } else {
658 /*
659 * Chain was empty, release the exclusive lock's last count
660 * as well the bits shown.
661 */
662 clock |= MTX_EXCLUSIVE | MTX_EXWANTED | MTX_SHWANTED | 1;
663 }
664
665 /*
666 * We have to uset cmpset here to deal with MTX_SHWANTED. If
667 * we just clear the bits we can miss a wakeup or, worse,
668 * leave mtx_lock unlocked with MTX_SHWANTED still set.
669 */
670 for (;;) {
671 lock = mtx->mtx_lock;
672 nlock = lock & ~clock;
673
674 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock)) {
675 if (link) {
676 /*
677 * Wakeup new exclusive holder. Leave
678 * SHWANTED intact.
679 */
680 wakeup(link);
681 } else if (lock & MTX_SHWANTED) {
682 /*
683 * Signal any shared waiters (and we also
684 * clear SHWANTED).
685 */
686 mtx->mtx_owner = NULL;
687 wakeup(mtx);
688 ++mtx_wakeup_count;
689 }
690 break;
691 }
692 cpu_pause();
33b0b87c
MD
693 ++mtx_collision_count;
694 }
695}
685ebdab
MD
696
697/*
698 * Delete a link structure after tsleep has failed. This code is not
699 * in the critical path as most exclusive waits are chained.
700 */
701static
702void
703mtx_delete_link(mtx_t mtx, mtx_link_t link)
704{
77912481 705 thread_t td = curthread;
685ebdab
MD
706 u_int lock;
707 u_int nlock;
708
709 /*
710 * Acquire MTX_EXLINK.
711 *
712 * Do not use cmpxchg to wait for EXLINK to clear as this might
713 * result in too much cpu cache traffic.
714 */
77912481 715 ++td->td_critcount;
685ebdab
MD
716 for (;;) {
717 lock = mtx->mtx_lock;
718 if (lock & MTX_EXLINK) {
719 cpu_pause();
720 ++mtx_collision_count;
721 continue;
722 }
723 /* lock &= ~MTX_EXLINK; */
724 nlock = lock | MTX_EXLINK;
725 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock))
726 break;
727 cpu_pause();
728 ++mtx_collision_count;
729 }
730
731 /*
732 * Delete the link and release EXLINK.
733 */
734 if (link->state == MTX_LINK_LINKED) {
735 if (link->next == link) {
736 mtx->mtx_link = NULL;
737 } else {
738 mtx->mtx_link = link->next;
739 link->next->prev = link->prev;
740 link->prev->next = link->next;
741 }
742 link->state = MTX_LINK_IDLE;
743 }
744 atomic_clear_int(&mtx->mtx_lock, MTX_EXLINK);
77912481 745 --td->td_critcount;
685ebdab
MD
746}
747
748/*
749 * Abort a mutex locking operation, causing mtx_lock_ex_link() to
750 * return ENOLCK. This may be called at any time after the
751 * mtx_link is initialized, including both before and after the call
752 * to mtx_lock_ex_link().
753 */
754void
755mtx_abort_ex_link(mtx_t mtx, mtx_link_t link)
756{
77912481 757 thread_t td = curthread;
685ebdab
MD
758 u_int lock;
759 u_int nlock;
760
761 /*
762 * Acquire MTX_EXLINK
763 */
77912481 764 ++td->td_critcount;
685ebdab
MD
765 for (;;) {
766 lock = mtx->mtx_lock;
767 if (lock & MTX_EXLINK) {
768 cpu_pause();
769 ++mtx_collision_count;
770 continue;
771 }
772 /* lock &= ~MTX_EXLINK; */
773 nlock = lock | MTX_EXLINK;
774 if (atomic_cmpset_int(&mtx->mtx_lock, lock, nlock))
775 break;
776 cpu_pause();
777 ++mtx_collision_count;
778 }
779
780 /*
781 * Do the abort
782 */
783 switch(link->state) {
784 case MTX_LINK_IDLE:
785 /*
786 * Link not started yet
787 */
788 link->state = MTX_LINK_ABORTED;
789 break;
790 case MTX_LINK_LINKED:
791 /*
792 * de-link, mark aborted, and wakeup the thread.
793 */
794 if (link->next == link) {
795 mtx->mtx_link = NULL;
796 } else {
797 mtx->mtx_link = link->next;
798 link->next->prev = link->prev;
799 link->prev->next = link->next;
800 }
801 link->state = MTX_LINK_ABORTED;
802 wakeup(link);
803 break;
804 case MTX_LINK_ACQUIRED:
805 /*
806 * Too late, the lock was acquired. Let it complete.
807 */
808 break;
809 default:
810 /*
811 * link already aborted, do nothing.
812 */
813 break;
814 }
815 atomic_clear_int(&mtx->mtx_lock, MTX_EXLINK);
77912481 816 --td->td_critcount;
685ebdab 817}