Fix inquiry length detection for the ses(4) driver.
[dragonfly.git] / sys / kern / lwkt_ipiq.c
CommitLineData
3b6b7bd1 1/*
8c10bfcf
MD
2 * Copyright (c) 2003,2004 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
3b6b7bd1
MD
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
8c10bfcf 10 *
3b6b7bd1
MD
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
8c10bfcf
MD
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
3b6b7bd1 32 * SUCH DAMAGE.
8c10bfcf 33 *
d7ed9e5e 34 * $DragonFly: src/sys/kern/lwkt_ipiq.c,v 1.22 2007/06/07 20:35:54 dillon Exp $
3b6b7bd1
MD
35 */
36
37/*
38 * This module implements IPI message queueing and the MI portion of IPI
39 * message processing.
40 */
41
42#ifdef _KERNEL
43
e8f15168
MD
44#include "opt_ddb.h"
45
3b6b7bd1
MD
46#include <sys/param.h>
47#include <sys/systm.h>
48#include <sys/kernel.h>
49#include <sys/proc.h>
50#include <sys/rtprio.h>
51#include <sys/queue.h>
52#include <sys/thread2.h>
53#include <sys/sysctl.h>
ac72c7f4 54#include <sys/ktr.h>
3b6b7bd1
MD
55#include <sys/kthread.h>
56#include <machine/cpu.h>
57#include <sys/lock.h>
58#include <sys/caps.h>
59
60#include <vm/vm.h>
61#include <vm/vm_param.h>
62#include <vm/vm_kern.h>
63#include <vm/vm_object.h>
64#include <vm/vm_page.h>
65#include <vm/vm_map.h>
66#include <vm/vm_pager.h>
67#include <vm/vm_extern.h>
68#include <vm/vm_zone.h>
69
70#include <machine/stdarg.h>
3b6b7bd1
MD
71#include <machine/smp.h>
72#include <machine/atomic.h>
73
3b6b7bd1
MD
74#else
75
76#include <sys/stdint.h>
77#include <libcaps/thread.h>
78#include <sys/thread.h>
79#include <sys/msgport.h>
80#include <sys/errno.h>
81#include <libcaps/globaldata.h>
7e8303ad 82#include <machine/cpufunc.h>
3b6b7bd1
MD
83#include <sys/thread2.h>
84#include <sys/msgport2.h>
85#include <stdio.h>
86#include <stdlib.h>
87#include <string.h>
3b6b7bd1 88#include <machine/lock.h>
5e940450
MD
89#include <machine/cpu.h>
90#include <machine/atomic.h>
3b6b7bd1
MD
91
92#endif
93
94#ifdef SMP
4c9f5a7f
MD
95static __int64_t ipiq_count; /* total calls to lwkt_send_ipiq*() */
96static __int64_t ipiq_fifofull; /* number of fifo full conditions detected */
97static __int64_t ipiq_avoided; /* interlock with target avoids cpu ipi */
98static __int64_t ipiq_passive; /* passive IPI messages */
99static __int64_t ipiq_cscount; /* number of cpu synchronizations */
100static int ipiq_optimized = 1; /* XXX temporary sysctl */
e8f15168
MD
101#ifdef PANIC_DEBUG
102static int panic_ipiq_cpu = -1;
103static int panic_ipiq_count = 100;
104#endif
3b6b7bd1
MD
105#endif
106
107#ifdef _KERNEL
108
109#ifdef SMP
110SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_count, CTLFLAG_RW, &ipiq_count, 0, "");
111SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_fifofull, CTLFLAG_RW, &ipiq_fifofull, 0, "");
4c9f5a7f
MD
112SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_avoided, CTLFLAG_RW, &ipiq_avoided, 0, "");
113SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_passive, CTLFLAG_RW, &ipiq_passive, 0, "");
0f7a3396 114SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_cscount, CTLFLAG_RW, &ipiq_cscount, 0, "");
4c9f5a7f 115SYSCTL_INT(_lwkt, OID_AUTO, ipiq_optimized, CTLFLAG_RW, &ipiq_optimized, 0, "");
e8f15168
MD
116#ifdef PANIC_DEBUG
117SYSCTL_INT(_lwkt, OID_AUTO, panic_ipiq_cpu, CTLFLAG_RW, &panic_ipiq_cpu, 0, "");
118SYSCTL_INT(_lwkt, OID_AUTO, panic_ipiq_count, CTLFLAG_RW, &panic_ipiq_count, 0, "");
119#endif
3b6b7bd1 120
a7adb95a 121#define IPIQ_STRING "func=%p arg1=%p arg2=%d scpu=%d dcpu=%d"
ac72c7f4
MD
122#define IPIQ_ARG_SIZE (sizeof(void *) * 2 + sizeof(int) * 2)
123
124#if !defined(KTR_IPIQ)
125#define KTR_IPIQ KTR_ALL
3b6b7bd1 126#endif
ac72c7f4
MD
127KTR_INFO_MASTER(ipiq);
128KTR_INFO(KTR_IPIQ, ipiq, send_norm, 0, IPIQ_STRING, IPIQ_ARG_SIZE);
129KTR_INFO(KTR_IPIQ, ipiq, send_pasv, 1, IPIQ_STRING, IPIQ_ARG_SIZE);
130KTR_INFO(KTR_IPIQ, ipiq, send_nbio, 2, IPIQ_STRING, IPIQ_ARG_SIZE);
131KTR_INFO(KTR_IPIQ, ipiq, send_fail, 3, IPIQ_STRING, IPIQ_ARG_SIZE);
132KTR_INFO(KTR_IPIQ, ipiq, receive, 4, IPIQ_STRING, IPIQ_ARG_SIZE);
d7ed9e5e
MD
133KTR_INFO(KTR_IPIQ, ipiq, sync_start, 5, "cpumask=%08x", sizeof(cpumask_t));
134KTR_INFO(KTR_IPIQ, ipiq, sync_add, 6, "cpumask=%08x", sizeof(cpumask_t));
ac72c7f4 135
a7adb95a
SZ
136#define logipiq(name, func, arg1, arg2, sgd, dgd) \
137 KTR_LOG(ipiq_ ## name, func, arg1, arg2, sgd->gd_cpuid, dgd->gd_cpuid)
d7ed9e5e
MD
138#define logipiq2(name, arg) \
139 KTR_LOG(ipiq_ ## name, arg)
ac72c7f4
MD
140
141#endif /* SMP */
142#endif /* KERNEL */
3b6b7bd1
MD
143
144#ifdef SMP
145
b8a98473
MD
146static int lwkt_process_ipiq_core(globaldata_t sgd, lwkt_ipiq_t ip,
147 struct intrframe *frame);
3b6b7bd1
MD
148static void lwkt_cpusync_remote1(lwkt_cpusync_t poll);
149static void lwkt_cpusync_remote2(lwkt_cpusync_t poll);
150
151/*
152 * Send a function execution request to another cpu. The request is queued
153 * on the cpu<->cpu ipiq matrix. Each cpu owns a unique ipiq FIFO for every
154 * possible target cpu. The FIFO can be written.
155 *
4c9f5a7f
MD
156 * If the FIFO fills up we have to enable interrupts to avoid an APIC
157 * deadlock and process pending IPIQs while waiting for it to empty.
158 * Otherwise we may soft-deadlock with another cpu whos FIFO is also full.
3b6b7bd1
MD
159 *
160 * We can safely bump gd_intr_nesting_level because our crit_exit() at the
161 * end will take care of any pending interrupts.
162 *
4c9f5a7f
MD
163 * The actual hardware IPI is avoided if the target cpu is already processing
164 * the queue from a prior IPI. It is possible to pipeline IPI messages
165 * very quickly between cpus due to the FIFO hysteresis.
166 *
167 * Need not be called from a critical section.
3b6b7bd1
MD
168 */
169int
b8a98473 170lwkt_send_ipiq3(globaldata_t target, ipifunc3_t func, void *arg1, int arg2)
3b6b7bd1
MD
171{
172 lwkt_ipiq_t ip;
173 int windex;
174 struct globaldata *gd = mycpu;
175
a7adb95a 176 logipiq(send_norm, func, arg1, arg2, gd, target);
ac72c7f4 177
3b6b7bd1 178 if (target == gd) {
b8a98473 179 func(arg1, arg2, NULL);
3b6b7bd1
MD
180 return(0);
181 }
182 crit_enter();
183 ++gd->gd_intr_nesting_level;
184#ifdef INVARIANTS
185 if (gd->gd_intr_nesting_level > 20)
186 panic("lwkt_send_ipiq: TOO HEAVILY NESTED!");
187#endif
188 KKASSERT(curthread->td_pri >= TDPRI_CRIT);
189 ++ipiq_count;
190 ip = &gd->gd_ipiq[target->gd_cpuid];
191
192 /*
4c9f5a7f
MD
193 * Do not allow the FIFO to become full. Interrupts must be physically
194 * enabled while we liveloop to avoid deadlocking the APIC.
195 */
196 if (ip->ip_windex - ip->ip_rindex > MAXCPUFIFO / 2) {
197 unsigned int eflags = read_eflags();
198
199 if (atomic_poll_acquire_int(&ip->ip_npoll) || ipiq_optimized == 0)
200 cpu_send_ipiq(target->gd_cpuid);
201 cpu_enable_intr();
202 ++ipiq_fifofull;
203 while (ip->ip_windex - ip->ip_rindex > MAXCPUFIFO / 4) {
204 KKASSERT(ip->ip_windex - ip->ip_rindex != MAXCPUFIFO - 1);
205 lwkt_process_ipiq();
206 }
207 write_eflags(eflags);
208 }
209
210 /*
211 * Queue the new message
3b6b7bd1 212 */
3b6b7bd1 213 windex = ip->ip_windex & MAXCPUFIFO_MASK;
b8a98473
MD
214 ip->ip_func[windex] = func;
215 ip->ip_arg1[windex] = arg1;
216 ip->ip_arg2[windex] = arg2;
35238fa5 217 cpu_sfence();
3b6b7bd1 218 ++ip->ip_windex;
4c9f5a7f
MD
219 --gd->gd_intr_nesting_level;
220
221 /*
222 * signal the target cpu that there is work pending.
223 */
224 if (atomic_poll_acquire_int(&ip->ip_npoll)) {
225 cpu_send_ipiq(target->gd_cpuid);
226 } else {
227 if (ipiq_optimized == 0)
228 cpu_send_ipiq(target->gd_cpuid);
229 ++ipiq_avoided;
230 }
231 crit_exit();
232 return(ip->ip_windex);
233}
234
235/*
236 * Similar to lwkt_send_ipiq() but this function does not actually initiate
237 * the IPI to the target cpu unless the FIFO has become too full, so it is
238 * very fast.
239 *
240 * This function is used for non-critical IPI messages, such as memory
241 * deallocations. The queue will typically be flushed by the target cpu at
242 * the next clock interrupt.
243 *
244 * Need not be called from a critical section.
245 */
246int
b8a98473
MD
247lwkt_send_ipiq3_passive(globaldata_t target, ipifunc3_t func,
248 void *arg1, int arg2)
4c9f5a7f
MD
249{
250 lwkt_ipiq_t ip;
251 int windex;
252 struct globaldata *gd = mycpu;
253
254 KKASSERT(target != gd);
255 crit_enter();
a7adb95a 256 logipiq(send_pasv, func, arg1, arg2, gd, target);
4c9f5a7f
MD
257 ++gd->gd_intr_nesting_level;
258#ifdef INVARIANTS
259 if (gd->gd_intr_nesting_level > 20)
260 panic("lwkt_send_ipiq: TOO HEAVILY NESTED!");
261#endif
262 KKASSERT(curthread->td_pri >= TDPRI_CRIT);
263 ++ipiq_count;
264 ++ipiq_passive;
265 ip = &gd->gd_ipiq[target->gd_cpuid];
266
267 /*
268 * Do not allow the FIFO to become full. Interrupts must be physically
269 * enabled while we liveloop to avoid deadlocking the APIC.
270 */
3b6b7bd1
MD
271 if (ip->ip_windex - ip->ip_rindex > MAXCPUFIFO / 2) {
272 unsigned int eflags = read_eflags();
4c9f5a7f
MD
273
274 if (atomic_poll_acquire_int(&ip->ip_npoll) || ipiq_optimized == 0)
275 cpu_send_ipiq(target->gd_cpuid);
3b6b7bd1
MD
276 cpu_enable_intr();
277 ++ipiq_fifofull;
278 while (ip->ip_windex - ip->ip_rindex > MAXCPUFIFO / 4) {
279 KKASSERT(ip->ip_windex - ip->ip_rindex != MAXCPUFIFO - 1);
280 lwkt_process_ipiq();
281 }
282 write_eflags(eflags);
283 }
4c9f5a7f
MD
284
285 /*
286 * Queue the new message
287 */
288 windex = ip->ip_windex & MAXCPUFIFO_MASK;
b8a98473
MD
289 ip->ip_func[windex] = func;
290 ip->ip_arg1[windex] = arg1;
291 ip->ip_arg2[windex] = arg2;
35238fa5 292 cpu_sfence();
4c9f5a7f 293 ++ip->ip_windex;
3b6b7bd1 294 --gd->gd_intr_nesting_level;
4c9f5a7f
MD
295
296 /*
297 * Do not signal the target cpu, it will pick up the IPI when it next
298 * polls (typically on the next tick).
299 */
3b6b7bd1
MD
300 crit_exit();
301 return(ip->ip_windex);
302}
303
41a01a4d 304/*
4c9f5a7f
MD
305 * Send an IPI request without blocking, return 0 on success, ENOENT on
306 * failure. The actual queueing of the hardware IPI may still force us
307 * to spin and process incoming IPIs but that will eventually go away
308 * when we've gotten rid of the other general IPIs.
41a01a4d
MD
309 */
310int
b8a98473
MD
311lwkt_send_ipiq3_nowait(globaldata_t target, ipifunc3_t func,
312 void *arg1, int arg2)
41a01a4d
MD
313{
314 lwkt_ipiq_t ip;
315 int windex;
316 struct globaldata *gd = mycpu;
317
a7adb95a 318 logipiq(send_nbio, func, arg1, arg2, gd, target);
41a01a4d
MD
319 KKASSERT(curthread->td_pri >= TDPRI_CRIT);
320 if (target == gd) {
b8a98473 321 func(arg1, arg2, NULL);
41a01a4d
MD
322 return(0);
323 }
324 ++ipiq_count;
325 ip = &gd->gd_ipiq[target->gd_cpuid];
326
ac72c7f4 327 if (ip->ip_windex - ip->ip_rindex >= MAXCPUFIFO * 2 / 3) {
a7adb95a 328 logipiq(send_fail, func, arg1, arg2, gd, target);
41a01a4d 329 return(ENOENT);
ac72c7f4 330 }
41a01a4d 331 windex = ip->ip_windex & MAXCPUFIFO_MASK;
b8a98473
MD
332 ip->ip_func[windex] = func;
333 ip->ip_arg1[windex] = arg1;
334 ip->ip_arg2[windex] = arg2;
35238fa5 335 cpu_sfence();
41a01a4d 336 ++ip->ip_windex;
4c9f5a7f 337
41a01a4d 338 /*
4c9f5a7f 339 * This isn't a passive IPI, we still have to signal the target cpu.
41a01a4d 340 */
4c9f5a7f
MD
341 if (atomic_poll_acquire_int(&ip->ip_npoll)) {
342 cpu_send_ipiq(target->gd_cpuid);
343 } else {
344 if (ipiq_optimized == 0)
345 cpu_send_ipiq(target->gd_cpuid);
728f6208
MD
346 else
347 ++ipiq_avoided;
4c9f5a7f 348 }
41a01a4d
MD
349 return(0);
350}
351
3b6b7bd1
MD
352/*
353 * deprecated, used only by fast int forwarding.
354 */
355int
b8a98473 356lwkt_send_ipiq3_bycpu(int dcpu, ipifunc3_t func, void *arg1, int arg2)
3b6b7bd1 357{
b8a98473 358 return(lwkt_send_ipiq3(globaldata_find(dcpu), func, arg1, arg2));
3b6b7bd1
MD
359}
360
361/*
362 * Send a message to several target cpus. Typically used for scheduling.
363 * The message will not be sent to stopped cpus.
364 */
365int
b8a98473 366lwkt_send_ipiq3_mask(u_int32_t mask, ipifunc3_t func, void *arg1, int arg2)
3b6b7bd1
MD
367{
368 int cpuid;
369 int count = 0;
370
371 mask &= ~stopped_cpus;
372 while (mask) {
373 cpuid = bsfl(mask);
b8a98473 374 lwkt_send_ipiq3(globaldata_find(cpuid), func, arg1, arg2);
3b6b7bd1
MD
375 mask &= ~(1 << cpuid);
376 ++count;
377 }
378 return(count);
379}
380
381/*
382 * Wait for the remote cpu to finish processing a function.
383 *
384 * YYY we have to enable interrupts and process the IPIQ while waiting
385 * for it to empty or we may deadlock with another cpu. Create a CPU_*()
386 * function to do this! YYY we really should 'block' here.
387 *
388 * MUST be called from a critical section. This routine may be called
389 * from an interrupt (for example, if an interrupt wakes a foreign thread
390 * up).
391 */
392void
393lwkt_wait_ipiq(globaldata_t target, int seq)
394{
395 lwkt_ipiq_t ip;
396 int maxc = 100000000;
397
398 if (target != mycpu) {
399 ip = &mycpu->gd_ipiq[target->gd_cpuid];
400 if ((int)(ip->ip_xindex - seq) < 0) {
401 unsigned int eflags = read_eflags();
402 cpu_enable_intr();
403 while ((int)(ip->ip_xindex - seq) < 0) {
41a01a4d 404 crit_enter();
3b6b7bd1 405 lwkt_process_ipiq();
41a01a4d 406 crit_exit();
3b6b7bd1 407 if (--maxc == 0)
6ea70f76 408 kprintf("LWKT_WAIT_IPIQ WARNING! %d wait %d (%d)\n", mycpu->gd_cpuid, target->gd_cpuid, ip->ip_xindex - seq);
3b6b7bd1
MD
409 if (maxc < -1000000)
410 panic("LWKT_WAIT_IPIQ");
35238fa5
MD
411 /*
412 * xindex may be modified by another cpu, use a load fence
413 * to ensure that the loop does not use a speculative value
414 * (which may improve performance).
415 */
416 cpu_lfence();
3b6b7bd1
MD
417 }
418 write_eflags(eflags);
419 }
420 }
421}
422
41a01a4d
MD
423int
424lwkt_seq_ipiq(globaldata_t target)
425{
426 lwkt_ipiq_t ip;
427
428 ip = &mycpu->gd_ipiq[target->gd_cpuid];
429 return(ip->ip_windex);
430}
431
3b6b7bd1
MD
432/*
433 * Called from IPI interrupt (like a fast interrupt), which has placed
434 * us in a critical section. The MP lock may or may not be held.
435 * May also be called from doreti or splz, or be reentrantly called
436 * indirectly through the ip_func[] we run.
437 *
438 * There are two versions, one where no interrupt frame is available (when
439 * called from the send code and from splz, and one where an interrupt
440 * frame is available.
441 */
442void
443lwkt_process_ipiq(void)
444{
445 globaldata_t gd = mycpu;
ac72c7f4 446 globaldata_t sgd;
3b6b7bd1
MD
447 lwkt_ipiq_t ip;
448 int n;
449
450again:
451 for (n = 0; n < ncpus; ++n) {
452 if (n != gd->gd_cpuid) {
ac72c7f4
MD
453 sgd = globaldata_find(n);
454 ip = sgd->gd_ipiq;
3b6b7bd1 455 if (ip != NULL) {
b8a98473 456 while (lwkt_process_ipiq_core(sgd, &ip[gd->gd_cpuid], NULL))
3b6b7bd1
MD
457 ;
458 }
459 }
460 }
461 if (gd->gd_cpusyncq.ip_rindex != gd->gd_cpusyncq.ip_windex) {
b8a98473 462 if (lwkt_process_ipiq_core(gd, &gd->gd_cpusyncq, NULL)) {
0f7a3396
MD
463 if (gd->gd_curthread->td_cscount == 0)
464 goto again;
465 need_ipiq();
466 }
3b6b7bd1
MD
467 }
468}
469
470#ifdef _KERNEL
471void
c7eb0589 472lwkt_process_ipiq_frame(struct intrframe *frame)
3b6b7bd1
MD
473{
474 globaldata_t gd = mycpu;
ac72c7f4 475 globaldata_t sgd;
3b6b7bd1
MD
476 lwkt_ipiq_t ip;
477 int n;
478
479again:
480 for (n = 0; n < ncpus; ++n) {
481 if (n != gd->gd_cpuid) {
ac72c7f4
MD
482 sgd = globaldata_find(n);
483 ip = sgd->gd_ipiq;
3b6b7bd1 484 if (ip != NULL) {
c7eb0589 485 while (lwkt_process_ipiq_core(sgd, &ip[gd->gd_cpuid], frame))
3b6b7bd1
MD
486 ;
487 }
488 }
489 }
490 if (gd->gd_cpusyncq.ip_rindex != gd->gd_cpusyncq.ip_windex) {
c7eb0589 491 if (lwkt_process_ipiq_core(gd, &gd->gd_cpusyncq, frame)) {
0f7a3396
MD
492 if (gd->gd_curthread->td_cscount == 0)
493 goto again;
494 need_ipiq();
495 }
3b6b7bd1
MD
496 }
497}
498#endif
499
500static int
b8a98473
MD
501lwkt_process_ipiq_core(globaldata_t sgd, lwkt_ipiq_t ip,
502 struct intrframe *frame)
3b6b7bd1
MD
503{
504 int ri;
35238fa5 505 int wi;
b8a98473
MD
506 ipifunc3_t copy_func;
507 void *copy_arg1;
508 int copy_arg2;
35238fa5
MD
509
510 /*
511 * Obtain the current write index, which is modified by a remote cpu.
512 * Issue a load fence to prevent speculative reads of e.g. data written
513 * by the other cpu prior to it updating the index.
514 */
728f6208 515 KKASSERT(curthread->td_pri >= TDPRI_CRIT);
35238fa5
MD
516 wi = ip->ip_windex;
517 cpu_lfence();
518
3b6b7bd1
MD
519 /*
520 * Note: xindex is only updated after we are sure the function has
521 * finished execution. Beware lwkt_process_ipiq() reentrancy! The
522 * function may send an IPI which may block/drain.
d64a7617
MD
523 *
524 * Note: due to additional IPI operations that the callback function
525 * may make, it is possible for both rindex and windex to advance and
526 * thus for rindex to advance passed our cached windex.
3b6b7bd1 527 */
d64a7617 528 while (wi - (ri = ip->ip_rindex) > 0) {
3b6b7bd1 529 ri &= MAXCPUFIFO_MASK;
728f6208 530 copy_func = ip->ip_func[ri];
b8a98473
MD
531 copy_arg1 = ip->ip_arg1[ri];
532 copy_arg2 = ip->ip_arg2[ri];
728f6208
MD
533 cpu_mfence();
534 ++ip->ip_rindex;
535 KKASSERT((ip->ip_rindex & MAXCPUFIFO_MASK) == ((ri + 1) & MAXCPUFIFO_MASK));
a7adb95a 536 logipiq(receive, copy_func, copy_arg1, copy_arg2, sgd, mycpu);
b8a98473 537 copy_func(copy_arg1, copy_arg2, frame);
35238fa5 538 cpu_sfence();
3b6b7bd1 539 ip->ip_xindex = ip->ip_rindex;
e8f15168
MD
540
541#ifdef PANIC_DEBUG
542 /*
543 * Simulate panics during the processing of an IPI
544 */
545 if (mycpu->gd_cpuid == panic_ipiq_cpu && panic_ipiq_count) {
546 if (--panic_ipiq_count == 0) {
547#ifdef DDB
548 Debugger("PANIC_DEBUG");
549#else
550 panic("PANIC_DEBUG");
551#endif
552 }
553 }
554#endif
3b6b7bd1 555 }
4c9f5a7f
MD
556
557 /*
558 * Return non-zero if there are more IPI messages pending on this
559 * ipiq. ip_npoll is left set as long as possible to reduce the
560 * number of IPIs queued by the originating cpu, but must be cleared
561 * *BEFORE* checking windex.
562 */
563 atomic_poll_release_int(&ip->ip_npoll);
3b6b7bd1
MD
564 return(wi != ip->ip_windex);
565}
566
0f7a3396
MD
567#endif
568
3b6b7bd1
MD
569/*
570 * CPU Synchronization Support
5c71a36a
MD
571 *
572 * lwkt_cpusync_simple()
573 *
574 * The function is executed synchronously before return on remote cpus.
575 * A lwkt_cpusync_t pointer is passed as an argument. The data can
576 * be accessed via arg->cs_data.
577 *
578 * XXX should I just pass the data as an argument to be consistent?
3b6b7bd1
MD
579 */
580
581void
5c71a36a
MD
582lwkt_cpusync_simple(cpumask_t mask, cpusync_func_t func, void *data)
583{
584 struct lwkt_cpusync cmd;
585
586 cmd.cs_run_func = NULL;
587 cmd.cs_fin1_func = func;
588 cmd.cs_fin2_func = NULL;
589 cmd.cs_data = data;
590 lwkt_cpusync_start(mask & mycpu->gd_other_cpus, &cmd);
591 if (mask & (1 << mycpu->gd_cpuid))
592 func(&cmd);
593 lwkt_cpusync_finish(&cmd);
594}
595
596/*
597 * lwkt_cpusync_fastdata()
598 *
599 * The function is executed in tandem with return on remote cpus.
600 * The data is directly passed as an argument. Do not pass pointers to
601 * temporary storage as the storage might have
602 * gone poof by the time the target cpu executes
603 * the function.
604 *
605 * At the moment lwkt_cpusync is declared on the stack and we must wait
606 * for all remote cpus to ack in lwkt_cpusync_finish(), but as a future
607 * optimization we should be able to put a counter in the globaldata
608 * structure (if it is not otherwise being used) and just poke it and
609 * return without waiting. XXX
610 */
611void
612lwkt_cpusync_fastdata(cpumask_t mask, cpusync_func2_t func, void *data)
3b6b7bd1
MD
613{
614 struct lwkt_cpusync cmd;
3b6b7bd1
MD
615
616 cmd.cs_run_func = NULL;
617 cmd.cs_fin1_func = NULL;
618 cmd.cs_fin2_func = func;
5c71a36a
MD
619 cmd.cs_data = NULL;
620 lwkt_cpusync_start(mask & mycpu->gd_other_cpus, &cmd);
3b6b7bd1
MD
621 if (mask & (1 << mycpu->gd_cpuid))
622 func(data);
5c71a36a 623 lwkt_cpusync_finish(&cmd);
3b6b7bd1
MD
624}
625
626/*
5c71a36a
MD
627 * lwkt_cpusync_start()
628 *
629 * Start synchronization with a set of target cpus, return once they are
630 * known to be in a synchronization loop. The target cpus will execute
631 * poll->cs_run_func() IN TANDEM WITH THE RETURN.
632 *
633 * XXX future: add lwkt_cpusync_start_quick() and require a call to
634 * lwkt_cpusync_add() or lwkt_cpusync_wait(), allowing the caller to
635 * potentially absorb the IPI latency doing something useful.
3b6b7bd1 636 */
5c71a36a 637void
3b6b7bd1
MD
638lwkt_cpusync_start(cpumask_t mask, lwkt_cpusync_t poll)
639{
0f7a3396
MD
640 globaldata_t gd = mycpu;
641
3b6b7bd1 642 poll->cs_count = 0;
5c71a36a 643 poll->cs_mask = mask;
0f7a3396 644#ifdef SMP
d7ed9e5e 645 logipiq2(sync_start, mask & gd->gd_other_cpus);
0f7a3396
MD
646 poll->cs_maxcount = lwkt_send_ipiq_mask(
647 mask & gd->gd_other_cpus & smp_active_mask,
b8a98473 648 (ipifunc1_t)lwkt_cpusync_remote1, poll);
0f7a3396 649#endif
fda1ad89 650 if (mask & gd->gd_cpumask) {
5c71a36a
MD
651 if (poll->cs_run_func)
652 poll->cs_run_func(poll);
653 }
0f7a3396
MD
654#ifdef SMP
655 if (poll->cs_maxcount) {
656 ++ipiq_cscount;
657 ++gd->gd_curthread->td_cscount;
658 while (poll->cs_count != poll->cs_maxcount) {
659 crit_enter();
660 lwkt_process_ipiq();
661 crit_exit();
662 }
5c71a36a 663 }
0f7a3396 664#endif
5c71a36a
MD
665}
666
667void
668lwkt_cpusync_add(cpumask_t mask, lwkt_cpusync_t poll)
669{
0f7a3396 670 globaldata_t gd = mycpu;
41a01a4d 671#ifdef SMP
0f7a3396 672 int count;
41a01a4d 673#endif
0f7a3396 674
5c71a36a
MD
675 mask &= ~poll->cs_mask;
676 poll->cs_mask |= mask;
0f7a3396 677#ifdef SMP
d7ed9e5e 678 logipiq2(sync_add, mask & gd->gd_other_cpus);
0f7a3396
MD
679 count = lwkt_send_ipiq_mask(
680 mask & gd->gd_other_cpus & smp_active_mask,
b8a98473 681 (ipifunc1_t)lwkt_cpusync_remote1, poll);
0f7a3396 682#endif
fda1ad89 683 if (mask & gd->gd_cpumask) {
5c71a36a
MD
684 if (poll->cs_run_func)
685 poll->cs_run_func(poll);
686 }
0f7a3396
MD
687#ifdef SMP
688 poll->cs_maxcount += count;
689 if (poll->cs_maxcount) {
690 if (poll->cs_maxcount == count)
691 ++gd->gd_curthread->td_cscount;
692 while (poll->cs_count != poll->cs_maxcount) {
693 crit_enter();
694 lwkt_process_ipiq();
695 crit_exit();
696 }
3b6b7bd1 697 }
0f7a3396 698#endif
3b6b7bd1
MD
699}
700
701/*
702 * Finish synchronization with a set of target cpus. The target cpus will
703 * execute cs_fin1_func(poll) prior to this function returning, and will
704 * execute cs_fin2_func(data) IN TANDEM WITH THIS FUNCTION'S RETURN.
0f7a3396
MD
705 *
706 * If cs_maxcount is non-zero then we are mastering a cpusync with one or
707 * more remote cpus and must account for it in our thread structure.
3b6b7bd1
MD
708 */
709void
5c71a36a 710lwkt_cpusync_finish(lwkt_cpusync_t poll)
3b6b7bd1 711{
0f7a3396 712 globaldata_t gd = mycpu;
5c71a36a 713
3b6b7bd1 714 poll->cs_count = -1;
fda1ad89 715 if (poll->cs_mask & gd->gd_cpumask) {
5c71a36a
MD
716 if (poll->cs_fin1_func)
717 poll->cs_fin1_func(poll);
718 if (poll->cs_fin2_func)
719 poll->cs_fin2_func(poll->cs_data);
720 }
0f7a3396
MD
721#ifdef SMP
722 if (poll->cs_maxcount) {
723 while (poll->cs_count != -(poll->cs_maxcount + 1)) {
724 crit_enter();
725 lwkt_process_ipiq();
726 crit_exit();
727 }
728 --gd->gd_curthread->td_cscount;
3b6b7bd1 729 }
0f7a3396 730#endif
3b6b7bd1
MD
731}
732
0f7a3396
MD
733#ifdef SMP
734
3b6b7bd1
MD
735/*
736 * helper IPI remote messaging function.
737 *
738 * Called on remote cpu when a new cpu synchronization request has been
739 * sent to us. Execute the run function and adjust cs_count, then requeue
740 * the request so we spin on it.
741 */
742static void
743lwkt_cpusync_remote1(lwkt_cpusync_t poll)
744{
745 atomic_add_int(&poll->cs_count, 1);
746 if (poll->cs_run_func)
747 poll->cs_run_func(poll);
748 lwkt_cpusync_remote2(poll);
749}
750
751/*
752 * helper IPI remote messaging function.
753 *
754 * Poll for the originator telling us to finish. If it hasn't, requeue
755 * our request so we spin on it. When the originator requests that we
756 * finish we execute cs_fin1_func(poll) synchronously and cs_fin2_func(data)
757 * in tandem with the release.
758 */
759static void
760lwkt_cpusync_remote2(lwkt_cpusync_t poll)
761{
762 if (poll->cs_count < 0) {
763 cpusync_func2_t savef;
764 void *saved;
765
766 if (poll->cs_fin1_func)
767 poll->cs_fin1_func(poll);
768 if (poll->cs_fin2_func) {
769 savef = poll->cs_fin2_func;
770 saved = poll->cs_data;
771 atomic_add_int(&poll->cs_count, -1);
772 savef(saved);
773 } else {
774 atomic_add_int(&poll->cs_count, -1);
775 }
776 } else {
777 globaldata_t gd = mycpu;
778 lwkt_ipiq_t ip;
779 int wi;
780
781 ip = &gd->gd_cpusyncq;
782 wi = ip->ip_windex & MAXCPUFIFO_MASK;
b8a98473
MD
783 ip->ip_func[wi] = (ipifunc3_t)(ipifunc1_t)lwkt_cpusync_remote2;
784 ip->ip_arg1[wi] = poll;
785 ip->ip_arg2[wi] = 0;
35238fa5 786 cpu_sfence();
3b6b7bd1
MD
787 ++ip->ip_windex;
788 }
789}
790
3b6b7bd1 791#endif