- Promote em(4) polling begin/end ktr into polling(4)
[dragonfly.git] / sys / kern / lwkt_ipiq.c
CommitLineData
3b6b7bd1 1/*
8c10bfcf
MD
2 * Copyright (c) 2003,2004 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
3b6b7bd1
MD
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
8c10bfcf 10 *
3b6b7bd1
MD
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
8c10bfcf
MD
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
3b6b7bd1 32 * SUCH DAMAGE.
8c10bfcf 33 *
5118bbc4 34 * $DragonFly: src/sys/kern/lwkt_ipiq.c,v 1.24 2008/04/29 14:26:09 sephe Exp $
3b6b7bd1
MD
35 */
36
37/*
38 * This module implements IPI message queueing and the MI portion of IPI
39 * message processing.
40 */
41
42#ifdef _KERNEL
43
e8f15168
MD
44#include "opt_ddb.h"
45
3b6b7bd1
MD
46#include <sys/param.h>
47#include <sys/systm.h>
48#include <sys/kernel.h>
49#include <sys/proc.h>
50#include <sys/rtprio.h>
51#include <sys/queue.h>
52#include <sys/thread2.h>
53#include <sys/sysctl.h>
ac72c7f4 54#include <sys/ktr.h>
3b6b7bd1
MD
55#include <sys/kthread.h>
56#include <machine/cpu.h>
57#include <sys/lock.h>
58#include <sys/caps.h>
59
60#include <vm/vm.h>
61#include <vm/vm_param.h>
62#include <vm/vm_kern.h>
63#include <vm/vm_object.h>
64#include <vm/vm_page.h>
65#include <vm/vm_map.h>
66#include <vm/vm_pager.h>
67#include <vm/vm_extern.h>
68#include <vm/vm_zone.h>
69
70#include <machine/stdarg.h>
3b6b7bd1
MD
71#include <machine/smp.h>
72#include <machine/atomic.h>
73
3b6b7bd1
MD
74#else
75
76#include <sys/stdint.h>
77#include <libcaps/thread.h>
78#include <sys/thread.h>
79#include <sys/msgport.h>
80#include <sys/errno.h>
81#include <libcaps/globaldata.h>
7e8303ad 82#include <machine/cpufunc.h>
3b6b7bd1
MD
83#include <sys/thread2.h>
84#include <sys/msgport2.h>
85#include <stdio.h>
86#include <stdlib.h>
87#include <string.h>
3b6b7bd1 88#include <machine/lock.h>
5e940450
MD
89#include <machine/cpu.h>
90#include <machine/atomic.h>
3b6b7bd1
MD
91
92#endif
93
94#ifdef SMP
4c9f5a7f
MD
95static __int64_t ipiq_count; /* total calls to lwkt_send_ipiq*() */
96static __int64_t ipiq_fifofull; /* number of fifo full conditions detected */
97static __int64_t ipiq_avoided; /* interlock with target avoids cpu ipi */
98static __int64_t ipiq_passive; /* passive IPI messages */
99static __int64_t ipiq_cscount; /* number of cpu synchronizations */
100static int ipiq_optimized = 1; /* XXX temporary sysctl */
e8f15168
MD
101#ifdef PANIC_DEBUG
102static int panic_ipiq_cpu = -1;
103static int panic_ipiq_count = 100;
104#endif
3b6b7bd1
MD
105#endif
106
107#ifdef _KERNEL
108
109#ifdef SMP
110SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_count, CTLFLAG_RW, &ipiq_count, 0, "");
111SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_fifofull, CTLFLAG_RW, &ipiq_fifofull, 0, "");
4c9f5a7f
MD
112SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_avoided, CTLFLAG_RW, &ipiq_avoided, 0, "");
113SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_passive, CTLFLAG_RW, &ipiq_passive, 0, "");
0f7a3396 114SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_cscount, CTLFLAG_RW, &ipiq_cscount, 0, "");
4c9f5a7f 115SYSCTL_INT(_lwkt, OID_AUTO, ipiq_optimized, CTLFLAG_RW, &ipiq_optimized, 0, "");
e8f15168
MD
116#ifdef PANIC_DEBUG
117SYSCTL_INT(_lwkt, OID_AUTO, panic_ipiq_cpu, CTLFLAG_RW, &panic_ipiq_cpu, 0, "");
118SYSCTL_INT(_lwkt, OID_AUTO, panic_ipiq_count, CTLFLAG_RW, &panic_ipiq_count, 0, "");
119#endif
3b6b7bd1 120
a7adb95a 121#define IPIQ_STRING "func=%p arg1=%p arg2=%d scpu=%d dcpu=%d"
5118bbc4 122#define IPIQ_ARG_SIZE (sizeof(void *) * 2 + sizeof(int) * 3)
ac72c7f4
MD
123
124#if !defined(KTR_IPIQ)
125#define KTR_IPIQ KTR_ALL
3b6b7bd1 126#endif
ac72c7f4
MD
127KTR_INFO_MASTER(ipiq);
128KTR_INFO(KTR_IPIQ, ipiq, send_norm, 0, IPIQ_STRING, IPIQ_ARG_SIZE);
129KTR_INFO(KTR_IPIQ, ipiq, send_pasv, 1, IPIQ_STRING, IPIQ_ARG_SIZE);
130KTR_INFO(KTR_IPIQ, ipiq, send_nbio, 2, IPIQ_STRING, IPIQ_ARG_SIZE);
131KTR_INFO(KTR_IPIQ, ipiq, send_fail, 3, IPIQ_STRING, IPIQ_ARG_SIZE);
132KTR_INFO(KTR_IPIQ, ipiq, receive, 4, IPIQ_STRING, IPIQ_ARG_SIZE);
d7ed9e5e
MD
133KTR_INFO(KTR_IPIQ, ipiq, sync_start, 5, "cpumask=%08x", sizeof(cpumask_t));
134KTR_INFO(KTR_IPIQ, ipiq, sync_add, 6, "cpumask=%08x", sizeof(cpumask_t));
ac72c7f4 135
a7adb95a
SZ
136#define logipiq(name, func, arg1, arg2, sgd, dgd) \
137 KTR_LOG(ipiq_ ## name, func, arg1, arg2, sgd->gd_cpuid, dgd->gd_cpuid)
d7ed9e5e
MD
138#define logipiq2(name, arg) \
139 KTR_LOG(ipiq_ ## name, arg)
ac72c7f4
MD
140
141#endif /* SMP */
142#endif /* KERNEL */
3b6b7bd1
MD
143
144#ifdef SMP
145
b8a98473
MD
146static int lwkt_process_ipiq_core(globaldata_t sgd, lwkt_ipiq_t ip,
147 struct intrframe *frame);
3b6b7bd1
MD
148static void lwkt_cpusync_remote1(lwkt_cpusync_t poll);
149static void lwkt_cpusync_remote2(lwkt_cpusync_t poll);
150
151/*
152 * Send a function execution request to another cpu. The request is queued
153 * on the cpu<->cpu ipiq matrix. Each cpu owns a unique ipiq FIFO for every
154 * possible target cpu. The FIFO can be written.
155 *
4c9f5a7f
MD
156 * If the FIFO fills up we have to enable interrupts to avoid an APIC
157 * deadlock and process pending IPIQs while waiting for it to empty.
158 * Otherwise we may soft-deadlock with another cpu whos FIFO is also full.
3b6b7bd1
MD
159 *
160 * We can safely bump gd_intr_nesting_level because our crit_exit() at the
161 * end will take care of any pending interrupts.
162 *
4c9f5a7f
MD
163 * The actual hardware IPI is avoided if the target cpu is already processing
164 * the queue from a prior IPI. It is possible to pipeline IPI messages
165 * very quickly between cpus due to the FIFO hysteresis.
166 *
167 * Need not be called from a critical section.
3b6b7bd1
MD
168 */
169int
b8a98473 170lwkt_send_ipiq3(globaldata_t target, ipifunc3_t func, void *arg1, int arg2)
3b6b7bd1
MD
171{
172 lwkt_ipiq_t ip;
173 int windex;
174 struct globaldata *gd = mycpu;
175
a7adb95a 176 logipiq(send_norm, func, arg1, arg2, gd, target);
ac72c7f4 177
3b6b7bd1 178 if (target == gd) {
b8a98473 179 func(arg1, arg2, NULL);
3b6b7bd1
MD
180 return(0);
181 }
182 crit_enter();
183 ++gd->gd_intr_nesting_level;
184#ifdef INVARIANTS
185 if (gd->gd_intr_nesting_level > 20)
186 panic("lwkt_send_ipiq: TOO HEAVILY NESTED!");
187#endif
188 KKASSERT(curthread->td_pri >= TDPRI_CRIT);
189 ++ipiq_count;
190 ip = &gd->gd_ipiq[target->gd_cpuid];
191
192 /*
4c9f5a7f
MD
193 * Do not allow the FIFO to become full. Interrupts must be physically
194 * enabled while we liveloop to avoid deadlocking the APIC.
195 */
196 if (ip->ip_windex - ip->ip_rindex > MAXCPUFIFO / 2) {
197 unsigned int eflags = read_eflags();
198
199 if (atomic_poll_acquire_int(&ip->ip_npoll) || ipiq_optimized == 0)
200 cpu_send_ipiq(target->gd_cpuid);
201 cpu_enable_intr();
202 ++ipiq_fifofull;
203 while (ip->ip_windex - ip->ip_rindex > MAXCPUFIFO / 4) {
204 KKASSERT(ip->ip_windex - ip->ip_rindex != MAXCPUFIFO - 1);
205 lwkt_process_ipiq();
206 }
207 write_eflags(eflags);
208 }
209
210 /*
211 * Queue the new message
3b6b7bd1 212 */
3b6b7bd1 213 windex = ip->ip_windex & MAXCPUFIFO_MASK;
b8a98473
MD
214 ip->ip_func[windex] = func;
215 ip->ip_arg1[windex] = arg1;
216 ip->ip_arg2[windex] = arg2;
35238fa5 217 cpu_sfence();
3b6b7bd1 218 ++ip->ip_windex;
4c9f5a7f
MD
219 --gd->gd_intr_nesting_level;
220
221 /*
222 * signal the target cpu that there is work pending.
223 */
224 if (atomic_poll_acquire_int(&ip->ip_npoll)) {
225 cpu_send_ipiq(target->gd_cpuid);
226 } else {
227 if (ipiq_optimized == 0)
228 cpu_send_ipiq(target->gd_cpuid);
229 ++ipiq_avoided;
230 }
231 crit_exit();
232 return(ip->ip_windex);
233}
234
235/*
236 * Similar to lwkt_send_ipiq() but this function does not actually initiate
237 * the IPI to the target cpu unless the FIFO has become too full, so it is
238 * very fast.
239 *
240 * This function is used for non-critical IPI messages, such as memory
241 * deallocations. The queue will typically be flushed by the target cpu at
242 * the next clock interrupt.
243 *
244 * Need not be called from a critical section.
245 */
246int
b8a98473
MD
247lwkt_send_ipiq3_passive(globaldata_t target, ipifunc3_t func,
248 void *arg1, int arg2)
4c9f5a7f
MD
249{
250 lwkt_ipiq_t ip;
251 int windex;
252 struct globaldata *gd = mycpu;
253
254 KKASSERT(target != gd);
255 crit_enter();
a7adb95a 256 logipiq(send_pasv, func, arg1, arg2, gd, target);
4c9f5a7f
MD
257 ++gd->gd_intr_nesting_level;
258#ifdef INVARIANTS
259 if (gd->gd_intr_nesting_level > 20)
260 panic("lwkt_send_ipiq: TOO HEAVILY NESTED!");
261#endif
262 KKASSERT(curthread->td_pri >= TDPRI_CRIT);
263 ++ipiq_count;
264 ++ipiq_passive;
265 ip = &gd->gd_ipiq[target->gd_cpuid];
266
267 /*
268 * Do not allow the FIFO to become full. Interrupts must be physically
269 * enabled while we liveloop to avoid deadlocking the APIC.
270 */
3b6b7bd1
MD
271 if (ip->ip_windex - ip->ip_rindex > MAXCPUFIFO / 2) {
272 unsigned int eflags = read_eflags();
4c9f5a7f
MD
273
274 if (atomic_poll_acquire_int(&ip->ip_npoll) || ipiq_optimized == 0)
275 cpu_send_ipiq(target->gd_cpuid);
3b6b7bd1
MD
276 cpu_enable_intr();
277 ++ipiq_fifofull;
278 while (ip->ip_windex - ip->ip_rindex > MAXCPUFIFO / 4) {
279 KKASSERT(ip->ip_windex - ip->ip_rindex != MAXCPUFIFO - 1);
280 lwkt_process_ipiq();
281 }
282 write_eflags(eflags);
283 }
4c9f5a7f
MD
284
285 /*
286 * Queue the new message
287 */
288 windex = ip->ip_windex & MAXCPUFIFO_MASK;
b8a98473
MD
289 ip->ip_func[windex] = func;
290 ip->ip_arg1[windex] = arg1;
291 ip->ip_arg2[windex] = arg2;
35238fa5 292 cpu_sfence();
4c9f5a7f 293 ++ip->ip_windex;
3b6b7bd1 294 --gd->gd_intr_nesting_level;
4c9f5a7f
MD
295
296 /*
297 * Do not signal the target cpu, it will pick up the IPI when it next
298 * polls (typically on the next tick).
299 */
3b6b7bd1
MD
300 crit_exit();
301 return(ip->ip_windex);
302}
303
41a01a4d 304/*
4c9f5a7f
MD
305 * Send an IPI request without blocking, return 0 on success, ENOENT on
306 * failure. The actual queueing of the hardware IPI may still force us
307 * to spin and process incoming IPIs but that will eventually go away
308 * when we've gotten rid of the other general IPIs.
41a01a4d
MD
309 */
310int
b8a98473
MD
311lwkt_send_ipiq3_nowait(globaldata_t target, ipifunc3_t func,
312 void *arg1, int arg2)
41a01a4d
MD
313{
314 lwkt_ipiq_t ip;
315 int windex;
316 struct globaldata *gd = mycpu;
317
a7adb95a 318 logipiq(send_nbio, func, arg1, arg2, gd, target);
41a01a4d
MD
319 KKASSERT(curthread->td_pri >= TDPRI_CRIT);
320 if (target == gd) {
b8a98473 321 func(arg1, arg2, NULL);
41a01a4d
MD
322 return(0);
323 }
324 ++ipiq_count;
325 ip = &gd->gd_ipiq[target->gd_cpuid];
326
ac72c7f4 327 if (ip->ip_windex - ip->ip_rindex >= MAXCPUFIFO * 2 / 3) {
a7adb95a 328 logipiq(send_fail, func, arg1, arg2, gd, target);
41a01a4d 329 return(ENOENT);
ac72c7f4 330 }
41a01a4d 331 windex = ip->ip_windex & MAXCPUFIFO_MASK;
b8a98473
MD
332 ip->ip_func[windex] = func;
333 ip->ip_arg1[windex] = arg1;
334 ip->ip_arg2[windex] = arg2;
35238fa5 335 cpu_sfence();
41a01a4d 336 ++ip->ip_windex;
4c9f5a7f 337
41a01a4d 338 /*
4c9f5a7f 339 * This isn't a passive IPI, we still have to signal the target cpu.
41a01a4d 340 */
4c9f5a7f
MD
341 if (atomic_poll_acquire_int(&ip->ip_npoll)) {
342 cpu_send_ipiq(target->gd_cpuid);
343 } else {
344 if (ipiq_optimized == 0)
345 cpu_send_ipiq(target->gd_cpuid);
728f6208
MD
346 else
347 ++ipiq_avoided;
4c9f5a7f 348 }
41a01a4d
MD
349 return(0);
350}
351
3b6b7bd1
MD
352/*
353 * deprecated, used only by fast int forwarding.
354 */
355int
b8a98473 356lwkt_send_ipiq3_bycpu(int dcpu, ipifunc3_t func, void *arg1, int arg2)
3b6b7bd1 357{
b8a98473 358 return(lwkt_send_ipiq3(globaldata_find(dcpu), func, arg1, arg2));
3b6b7bd1
MD
359}
360
361/*
362 * Send a message to several target cpus. Typically used for scheduling.
363 * The message will not be sent to stopped cpus.
364 */
365int
b8a98473 366lwkt_send_ipiq3_mask(u_int32_t mask, ipifunc3_t func, void *arg1, int arg2)
3b6b7bd1
MD
367{
368 int cpuid;
369 int count = 0;
370
371 mask &= ~stopped_cpus;
372 while (mask) {
373 cpuid = bsfl(mask);
b8a98473 374 lwkt_send_ipiq3(globaldata_find(cpuid), func, arg1, arg2);
3b6b7bd1
MD
375 mask &= ~(1 << cpuid);
376 ++count;
377 }
378 return(count);
379}
380
381/*
382 * Wait for the remote cpu to finish processing a function.
383 *
384 * YYY we have to enable interrupts and process the IPIQ while waiting
385 * for it to empty or we may deadlock with another cpu. Create a CPU_*()
386 * function to do this! YYY we really should 'block' here.
387 *
388 * MUST be called from a critical section. This routine may be called
389 * from an interrupt (for example, if an interrupt wakes a foreign thread
390 * up).
391 */
392void
393lwkt_wait_ipiq(globaldata_t target, int seq)
394{
395 lwkt_ipiq_t ip;
396 int maxc = 100000000;
397
398 if (target != mycpu) {
399 ip = &mycpu->gd_ipiq[target->gd_cpuid];
400 if ((int)(ip->ip_xindex - seq) < 0) {
401 unsigned int eflags = read_eflags();
402 cpu_enable_intr();
403 while ((int)(ip->ip_xindex - seq) < 0) {
41a01a4d 404 crit_enter();
3b6b7bd1 405 lwkt_process_ipiq();
41a01a4d 406 crit_exit();
3b6b7bd1 407 if (--maxc == 0)
6ea70f76 408 kprintf("LWKT_WAIT_IPIQ WARNING! %d wait %d (%d)\n", mycpu->gd_cpuid, target->gd_cpuid, ip->ip_xindex - seq);
3b6b7bd1
MD
409 if (maxc < -1000000)
410 panic("LWKT_WAIT_IPIQ");
35238fa5
MD
411 /*
412 * xindex may be modified by another cpu, use a load fence
413 * to ensure that the loop does not use a speculative value
414 * (which may improve performance).
415 */
416 cpu_lfence();
3b6b7bd1
MD
417 }
418 write_eflags(eflags);
419 }
420 }
421}
422
41a01a4d
MD
423int
424lwkt_seq_ipiq(globaldata_t target)
425{
426 lwkt_ipiq_t ip;
427
428 ip = &mycpu->gd_ipiq[target->gd_cpuid];
429 return(ip->ip_windex);
430}
431
3b6b7bd1
MD
432/*
433 * Called from IPI interrupt (like a fast interrupt), which has placed
434 * us in a critical section. The MP lock may or may not be held.
435 * May also be called from doreti or splz, or be reentrantly called
436 * indirectly through the ip_func[] we run.
437 *
438 * There are two versions, one where no interrupt frame is available (when
439 * called from the send code and from splz, and one where an interrupt
440 * frame is available.
441 */
442void
443lwkt_process_ipiq(void)
444{
445 globaldata_t gd = mycpu;
ac72c7f4 446 globaldata_t sgd;
3b6b7bd1
MD
447 lwkt_ipiq_t ip;
448 int n;
449
450again:
451 for (n = 0; n < ncpus; ++n) {
452 if (n != gd->gd_cpuid) {
ac72c7f4
MD
453 sgd = globaldata_find(n);
454 ip = sgd->gd_ipiq;
3b6b7bd1 455 if (ip != NULL) {
b8a98473 456 while (lwkt_process_ipiq_core(sgd, &ip[gd->gd_cpuid], NULL))
3b6b7bd1
MD
457 ;
458 }
459 }
460 }
461 if (gd->gd_cpusyncq.ip_rindex != gd->gd_cpusyncq.ip_windex) {
b8a98473 462 if (lwkt_process_ipiq_core(gd, &gd->gd_cpusyncq, NULL)) {
0f7a3396
MD
463 if (gd->gd_curthread->td_cscount == 0)
464 goto again;
465 need_ipiq();
466 }
3b6b7bd1
MD
467 }
468}
469
470#ifdef _KERNEL
471void
c7eb0589 472lwkt_process_ipiq_frame(struct intrframe *frame)
3b6b7bd1
MD
473{
474 globaldata_t gd = mycpu;
ac72c7f4 475 globaldata_t sgd;
3b6b7bd1
MD
476 lwkt_ipiq_t ip;
477 int n;
478
479again:
480 for (n = 0; n < ncpus; ++n) {
481 if (n != gd->gd_cpuid) {
ac72c7f4
MD
482 sgd = globaldata_find(n);
483 ip = sgd->gd_ipiq;
3b6b7bd1 484 if (ip != NULL) {
c7eb0589 485 while (lwkt_process_ipiq_core(sgd, &ip[gd->gd_cpuid], frame))
3b6b7bd1
MD
486 ;
487 }
488 }
489 }
490 if (gd->gd_cpusyncq.ip_rindex != gd->gd_cpusyncq.ip_windex) {
c7eb0589 491 if (lwkt_process_ipiq_core(gd, &gd->gd_cpusyncq, frame)) {
0f7a3396
MD
492 if (gd->gd_curthread->td_cscount == 0)
493 goto again;
494 need_ipiq();
495 }
3b6b7bd1
MD
496 }
497}
498#endif
499
500static int
b8a98473
MD
501lwkt_process_ipiq_core(globaldata_t sgd, lwkt_ipiq_t ip,
502 struct intrframe *frame)
3b6b7bd1
MD
503{
504 int ri;
35238fa5 505 int wi;
b8a98473
MD
506 ipifunc3_t copy_func;
507 void *copy_arg1;
508 int copy_arg2;
35238fa5
MD
509
510 /*
511 * Obtain the current write index, which is modified by a remote cpu.
512 * Issue a load fence to prevent speculative reads of e.g. data written
513 * by the other cpu prior to it updating the index.
514 */
728f6208 515 KKASSERT(curthread->td_pri >= TDPRI_CRIT);
35238fa5
MD
516 wi = ip->ip_windex;
517 cpu_lfence();
518
3b6b7bd1
MD
519 /*
520 * Note: xindex is only updated after we are sure the function has
521 * finished execution. Beware lwkt_process_ipiq() reentrancy! The
522 * function may send an IPI which may block/drain.
d64a7617
MD
523 *
524 * Note: due to additional IPI operations that the callback function
525 * may make, it is possible for both rindex and windex to advance and
526 * thus for rindex to advance passed our cached windex.
3b6b7bd1 527 */
d64a7617 528 while (wi - (ri = ip->ip_rindex) > 0) {
3b6b7bd1 529 ri &= MAXCPUFIFO_MASK;
728f6208 530 copy_func = ip->ip_func[ri];
b8a98473
MD
531 copy_arg1 = ip->ip_arg1[ri];
532 copy_arg2 = ip->ip_arg2[ri];
728f6208
MD
533 cpu_mfence();
534 ++ip->ip_rindex;
535 KKASSERT((ip->ip_rindex & MAXCPUFIFO_MASK) == ((ri + 1) & MAXCPUFIFO_MASK));
a7adb95a 536 logipiq(receive, copy_func, copy_arg1, copy_arg2, sgd, mycpu);
b8a98473 537 copy_func(copy_arg1, copy_arg2, frame);
35238fa5 538 cpu_sfence();
3b6b7bd1 539 ip->ip_xindex = ip->ip_rindex;
e8f15168
MD
540
541#ifdef PANIC_DEBUG
542 /*
543 * Simulate panics during the processing of an IPI
544 */
545 if (mycpu->gd_cpuid == panic_ipiq_cpu && panic_ipiq_count) {
546 if (--panic_ipiq_count == 0) {
547#ifdef DDB
548 Debugger("PANIC_DEBUG");
549#else
550 panic("PANIC_DEBUG");
551#endif
552 }
553 }
554#endif
3b6b7bd1 555 }
4c9f5a7f
MD
556
557 /*
558 * Return non-zero if there are more IPI messages pending on this
559 * ipiq. ip_npoll is left set as long as possible to reduce the
560 * number of IPIs queued by the originating cpu, but must be cleared
561 * *BEFORE* checking windex.
562 */
563 atomic_poll_release_int(&ip->ip_npoll);
3b6b7bd1
MD
564 return(wi != ip->ip_windex);
565}
566
6c92c1f2
SZ
567static void
568lwkt_sync_ipiq(void *arg)
569{
570 cpumask_t *cpumask = arg;
571
572 atomic_clear_int(cpumask, mycpu->gd_cpumask);
573 if (*cpumask == 0)
574 wakeup(cpumask);
575}
576
577void
578lwkt_synchronize_ipiqs(const char *wmesg)
579{
580 cpumask_t other_cpumask;
581
582 other_cpumask = mycpu->gd_other_cpus & smp_active_mask;
583 lwkt_send_ipiq_mask(other_cpumask, lwkt_sync_ipiq, &other_cpumask);
584
585 crit_enter();
586 while (other_cpumask != 0) {
587 tsleep_interlock(&other_cpumask);
588 if (other_cpumask != 0)
589 tsleep(&other_cpumask, 0, wmesg, 0);
590 }
591 crit_exit();
592}
593
0f7a3396
MD
594#endif
595
3b6b7bd1
MD
596/*
597 * CPU Synchronization Support
5c71a36a
MD
598 *
599 * lwkt_cpusync_simple()
600 *
601 * The function is executed synchronously before return on remote cpus.
602 * A lwkt_cpusync_t pointer is passed as an argument. The data can
603 * be accessed via arg->cs_data.
604 *
605 * XXX should I just pass the data as an argument to be consistent?
3b6b7bd1
MD
606 */
607
608void
5c71a36a
MD
609lwkt_cpusync_simple(cpumask_t mask, cpusync_func_t func, void *data)
610{
611 struct lwkt_cpusync cmd;
612
613 cmd.cs_run_func = NULL;
614 cmd.cs_fin1_func = func;
615 cmd.cs_fin2_func = NULL;
616 cmd.cs_data = data;
617 lwkt_cpusync_start(mask & mycpu->gd_other_cpus, &cmd);
618 if (mask & (1 << mycpu->gd_cpuid))
619 func(&cmd);
620 lwkt_cpusync_finish(&cmd);
621}
622
623/*
624 * lwkt_cpusync_fastdata()
625 *
626 * The function is executed in tandem with return on remote cpus.
627 * The data is directly passed as an argument. Do not pass pointers to
628 * temporary storage as the storage might have
629 * gone poof by the time the target cpu executes
630 * the function.
631 *
632 * At the moment lwkt_cpusync is declared on the stack and we must wait
633 * for all remote cpus to ack in lwkt_cpusync_finish(), but as a future
634 * optimization we should be able to put a counter in the globaldata
635 * structure (if it is not otherwise being used) and just poke it and
636 * return without waiting. XXX
637 */
638void
639lwkt_cpusync_fastdata(cpumask_t mask, cpusync_func2_t func, void *data)
3b6b7bd1
MD
640{
641 struct lwkt_cpusync cmd;
3b6b7bd1
MD
642
643 cmd.cs_run_func = NULL;
644 cmd.cs_fin1_func = NULL;
645 cmd.cs_fin2_func = func;
5c71a36a
MD
646 cmd.cs_data = NULL;
647 lwkt_cpusync_start(mask & mycpu->gd_other_cpus, &cmd);
3b6b7bd1
MD
648 if (mask & (1 << mycpu->gd_cpuid))
649 func(data);
5c71a36a 650 lwkt_cpusync_finish(&cmd);
3b6b7bd1
MD
651}
652
653/*
5c71a36a
MD
654 * lwkt_cpusync_start()
655 *
656 * Start synchronization with a set of target cpus, return once they are
657 * known to be in a synchronization loop. The target cpus will execute
658 * poll->cs_run_func() IN TANDEM WITH THE RETURN.
659 *
660 * XXX future: add lwkt_cpusync_start_quick() and require a call to
661 * lwkt_cpusync_add() or lwkt_cpusync_wait(), allowing the caller to
662 * potentially absorb the IPI latency doing something useful.
3b6b7bd1 663 */
5c71a36a 664void
3b6b7bd1
MD
665lwkt_cpusync_start(cpumask_t mask, lwkt_cpusync_t poll)
666{
0f7a3396
MD
667 globaldata_t gd = mycpu;
668
3b6b7bd1 669 poll->cs_count = 0;
5c71a36a 670 poll->cs_mask = mask;
0f7a3396 671#ifdef SMP
d7ed9e5e 672 logipiq2(sync_start, mask & gd->gd_other_cpus);
0f7a3396
MD
673 poll->cs_maxcount = lwkt_send_ipiq_mask(
674 mask & gd->gd_other_cpus & smp_active_mask,
b8a98473 675 (ipifunc1_t)lwkt_cpusync_remote1, poll);
0f7a3396 676#endif
fda1ad89 677 if (mask & gd->gd_cpumask) {
5c71a36a
MD
678 if (poll->cs_run_func)
679 poll->cs_run_func(poll);
680 }
0f7a3396
MD
681#ifdef SMP
682 if (poll->cs_maxcount) {
683 ++ipiq_cscount;
684 ++gd->gd_curthread->td_cscount;
685 while (poll->cs_count != poll->cs_maxcount) {
686 crit_enter();
687 lwkt_process_ipiq();
688 crit_exit();
689 }
5c71a36a 690 }
0f7a3396 691#endif
5c71a36a
MD
692}
693
694void
695lwkt_cpusync_add(cpumask_t mask, lwkt_cpusync_t poll)
696{
0f7a3396 697 globaldata_t gd = mycpu;
41a01a4d 698#ifdef SMP
0f7a3396 699 int count;
41a01a4d 700#endif
0f7a3396 701
5c71a36a
MD
702 mask &= ~poll->cs_mask;
703 poll->cs_mask |= mask;
0f7a3396 704#ifdef SMP
d7ed9e5e 705 logipiq2(sync_add, mask & gd->gd_other_cpus);
0f7a3396
MD
706 count = lwkt_send_ipiq_mask(
707 mask & gd->gd_other_cpus & smp_active_mask,
b8a98473 708 (ipifunc1_t)lwkt_cpusync_remote1, poll);
0f7a3396 709#endif
fda1ad89 710 if (mask & gd->gd_cpumask) {
5c71a36a
MD
711 if (poll->cs_run_func)
712 poll->cs_run_func(poll);
713 }
0f7a3396
MD
714#ifdef SMP
715 poll->cs_maxcount += count;
716 if (poll->cs_maxcount) {
717 if (poll->cs_maxcount == count)
718 ++gd->gd_curthread->td_cscount;
719 while (poll->cs_count != poll->cs_maxcount) {
720 crit_enter();
721 lwkt_process_ipiq();
722 crit_exit();
723 }
3b6b7bd1 724 }
0f7a3396 725#endif
3b6b7bd1
MD
726}
727
728/*
729 * Finish synchronization with a set of target cpus. The target cpus will
730 * execute cs_fin1_func(poll) prior to this function returning, and will
731 * execute cs_fin2_func(data) IN TANDEM WITH THIS FUNCTION'S RETURN.
0f7a3396
MD
732 *
733 * If cs_maxcount is non-zero then we are mastering a cpusync with one or
734 * more remote cpus and must account for it in our thread structure.
3b6b7bd1
MD
735 */
736void
5c71a36a 737lwkt_cpusync_finish(lwkt_cpusync_t poll)
3b6b7bd1 738{
0f7a3396 739 globaldata_t gd = mycpu;
5c71a36a 740
3b6b7bd1 741 poll->cs_count = -1;
fda1ad89 742 if (poll->cs_mask & gd->gd_cpumask) {
5c71a36a
MD
743 if (poll->cs_fin1_func)
744 poll->cs_fin1_func(poll);
745 if (poll->cs_fin2_func)
746 poll->cs_fin2_func(poll->cs_data);
747 }
0f7a3396
MD
748#ifdef SMP
749 if (poll->cs_maxcount) {
750 while (poll->cs_count != -(poll->cs_maxcount + 1)) {
751 crit_enter();
752 lwkt_process_ipiq();
753 crit_exit();
754 }
755 --gd->gd_curthread->td_cscount;
3b6b7bd1 756 }
0f7a3396 757#endif
3b6b7bd1
MD
758}
759
0f7a3396
MD
760#ifdef SMP
761
3b6b7bd1
MD
762/*
763 * helper IPI remote messaging function.
764 *
765 * Called on remote cpu when a new cpu synchronization request has been
766 * sent to us. Execute the run function and adjust cs_count, then requeue
767 * the request so we spin on it.
768 */
769static void
770lwkt_cpusync_remote1(lwkt_cpusync_t poll)
771{
772 atomic_add_int(&poll->cs_count, 1);
773 if (poll->cs_run_func)
774 poll->cs_run_func(poll);
775 lwkt_cpusync_remote2(poll);
776}
777
778/*
779 * helper IPI remote messaging function.
780 *
781 * Poll for the originator telling us to finish. If it hasn't, requeue
782 * our request so we spin on it. When the originator requests that we
783 * finish we execute cs_fin1_func(poll) synchronously and cs_fin2_func(data)
784 * in tandem with the release.
785 */
786static void
787lwkt_cpusync_remote2(lwkt_cpusync_t poll)
788{
789 if (poll->cs_count < 0) {
790 cpusync_func2_t savef;
791 void *saved;
792
793 if (poll->cs_fin1_func)
794 poll->cs_fin1_func(poll);
795 if (poll->cs_fin2_func) {
796 savef = poll->cs_fin2_func;
797 saved = poll->cs_data;
798 atomic_add_int(&poll->cs_count, -1);
799 savef(saved);
800 } else {
801 atomic_add_int(&poll->cs_count, -1);
802 }
803 } else {
804 globaldata_t gd = mycpu;
805 lwkt_ipiq_t ip;
806 int wi;
807
808 ip = &gd->gd_cpusyncq;
809 wi = ip->ip_windex & MAXCPUFIFO_MASK;
b8a98473
MD
810 ip->ip_func[wi] = (ipifunc3_t)(ipifunc1_t)lwkt_cpusync_remote2;
811 ip->ip_arg1[wi] = poll;
812 ip->ip_arg2[wi] = 0;
35238fa5 813 cpu_sfence();
3b6b7bd1
MD
814 ++ip->ip_windex;
815 }
816}
817
3b6b7bd1 818#endif