2 * Copyright (c) 2009 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * Helper functions for MP lock acquisition and release.
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/kernel.h>
43 #include <sys/rtprio.h>
44 #include <sys/queue.h>
45 #include <sys/sysctl.h>
46 #include <sys/kthread.h>
47 #include <machine/cpu.h>
50 #include <sys/spinlock.h>
53 #include <sys/thread2.h>
54 #include <sys/mplock2.h>
55 #include <sys/spinlock2.h>
58 static int chain_mplock = 0;
59 static int mplock_yield = 10;
60 static int mplock_backtrace;
61 static __int64_t mplock_contention_count = 0;
63 SYSCTL_INT(_lwkt, OID_AUTO, chain_mplock, CTLFLAG_RW, &chain_mplock, 0,
64 "Chain IPI's to other CPU's potentially needing the MP lock when it is yielded");
65 SYSCTL_INT(_lwkt, OID_AUTO, mplock_yield_delay, CTLFLAG_RW, &mplock_yield, 0,
66 "Duration of delay when MP lock is temporarily yielded");
67 SYSCTL_INT(_lwkt, OID_AUTO, mplock_backtrace, CTLFLAG_RW, &mplock_backtrace, 0,
68 "Output backplane when mplock contention occurs");
69 SYSCTL_QUAD(_lwkt, OID_AUTO, mplock_contention_count, CTLFLAG_RW,
70 &mplock_contention_count, 0, "spinning due to MPLOCK contention");
75 #if !defined(KTR_GIANT_CONTENTION)
76 #define KTR_GIANT_CONTENTION KTR_ALL
79 KTR_INFO_MASTER(giant);
80 KTR_INFO(KTR_GIANT_CONTENTION, giant, beg, 0,
81 "thread=%p held %s:%-5d want %s:%-5d",
82 sizeof(void *) * 3 + sizeof(int) * 2);
83 KTR_INFO(KTR_GIANT_CONTENTION, giant, end, 1,
84 "thread=%p held %s:%-5d want %s:%-5d",
85 sizeof(void *) * 3 + sizeof(int) * 2);
87 #define loggiant(name) \
88 KTR_LOG(giant_ ## name, curthread, \
89 mp_lock_holder_file, mp_lock_holder_line, \
93 cpumask_t cpu_contention_mask;
94 const char *mp_lock_holder_file; /* debugging */
95 int mp_lock_holder_line; /* debugging */
98 * Sets up the initial MP lock state near the start of the kernel boot
101 cpu_get_initial_mplock(void)
103 mp_lock = 0; /* cpu 0 */
104 curthread->td_mpcount = 1;
108 * This code is called from the get_mplock() inline when the mplock
109 * is not already held. td_mpcount has already been predisposed
113 _get_mplock_predisposed(const char *file, int line)
115 globaldata_t gd = mycpu;
117 if (gd->gd_intr_nesting_level) {
118 panic("Attempt to acquire mplock not already held "
119 "in hard section, ipi or interrupt %s:%d",
122 if (atomic_cmpset_int(&mp_lock, -1, gd->gd_cpuid) == 0)
123 _get_mplock_contested(file, line);
125 mp_lock_holder_file = file;
126 mp_lock_holder_line = line;
131 * Called when the MP lock could not be trvially acquired. The caller
132 * has already bumped td_mpcount.
135 _get_mplock_contested(const char *file, int line)
137 globaldata_t gd = mycpu;
140 const void **stkframe = (const void **)&file;
142 if (mplock_backtrace > 0) {
146 ++mplock_contention_count;
150 if (ov == gd->gd_cpuid)
153 if (atomic_cmpset_int(&mp_lock, ov, gd->gd_cpuid))
156 gd->gd_curthread->td_mplock_stallpc = stkframe[-1];
160 KKASSERT(gd->gd_cpuid == mp_lock);
167 * Called if td_mpcount went negative or if td_mpcount + td_xpcount is 0
168 * and we were unable to release the MP lock. Handles sanity checks
171 * It is possible for the inline release to have raced an interrupt which
172 * get/rel'd the MP lock, causing the inline's cmpset to fail. If this
173 * case occurs mp_lock will either already be in a released state or it
174 * will have already been acquired by another cpu.
177 _rel_mplock_contested(void)
179 globaldata_t gd = mycpu;
180 thread_t td = gd->gd_curthread;
183 KKASSERT(td->td_mpcount >= 0);
184 if (td->td_mpcount + td->td_xpcount == 0) {
187 if (ov != gd->gd_cpuid)
189 if (atomic_cmpset_int(&mp_lock, ov, -1))
196 * Called when try_mplock() fails.
198 * The inline bumped td_mpcount so we have to undo it.
200 * It is possible to race an interrupt which acquired and released the
201 * MP lock. When combined with the td_mpcount decrement we do the MP lock
202 * can wind up in any state and possibly not even owned by us.
204 * It is also possible for this function to be called even if td_mpcount > 1
205 * if someone bumped it and raced an interrupt which then called try_mpock().
208 _try_mplock_contested(const char *file, int line)
210 globaldata_t gd = mycpu;
211 thread_t td = gd->gd_curthread;
215 KKASSERT(td->td_mpcount >= 0);
216 ++mplock_contention_count;
218 if (td->td_mpcount + td->td_xpcount == 0) {
221 if (ov != gd->gd_cpuid)
223 if (atomic_cmpset_int(&mp_lock, ov, -1))
230 * Called when cpu_try_mplock() fails.
232 * The inline did not touch td_mpcount so we do not either.
235 _cpu_try_mplock_contested(const char *file, int line)
237 ++mplock_contention_count;
241 * Temporarily yield the MP lock. This is part of lwkt_user_yield()
242 * which is kinda hackish. The MP lock cannot be yielded if inherited
243 * due to a preemption.
246 yield_mplock(thread_t td)
250 if (td->td_xpcount == 0 && mplock_yield > 0) {
251 savecnt = td->td_mpcount;
256 td->td_mpcount = savecnt;
263 * The rel_mplock() code will call this function after releasing the
264 * last reference on the MP lock if cpu_contention_mask is non-zero.
266 * We then chain an IPI to a single other cpu potentially needing the
267 * lock. This is a bit heuristical and we can wind up with IPIs flying
268 * all over the place.
270 static void lwkt_mp_lock_uncontested_remote(void *arg __unused);
273 lwkt_mp_lock_uncontested(void)
283 clr_mplock_contention_mask(gd);
284 mask = cpu_contention_mask;
285 tmpmask = ~(CPUMASK(gd->gd_cpuid) - 1);
289 cpuid = BSFCPUMASK(mask & tmpmask);
291 cpuid = BSFCPUMASK(mask);
292 atomic_clear_cpumask(&cpu_contention_mask, CPUMASK(cpuid));
293 dgd = globaldata_find(cpuid);
294 lwkt_send_ipiq(dgd, lwkt_mp_lock_uncontested_remote, NULL);
300 * The idea is for this IPI to interrupt a potentially lower priority
301 * thread, such as a user thread, to allow the scheduler to reschedule
302 * a higher priority kernel thread that needs the MP lock.
304 * For now we set the LWKT reschedule flag which generates an AST in
305 * doreti, though theoretically it is also possible to possibly preempt
306 * here if the underlying thread was operating in user mode. Nah.
309 lwkt_mp_lock_uncontested_remote(void *arg __unused)