[dragonfly.git] / sys / kern / kern_ktr.c

/*
 * Copyright (c) 2005 The DragonFly Project.  All rights reserved.
 * 
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
/*
 * The following copyright applies to the DDB command code:
 *
 * Copyright (c) 2000 John Baldwin <jhb@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the author nor the names of any co-contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*
 * Kernel tracepoint facility.
 */

#include "opt_ddb.h"
#include "opt_ktr.h"

#include <sys/param.h>
#include <sys/cons.h>
#include <sys/kernel.h>
#include <sys/libkern.h>
#include <sys/proc.h>
#include <sys/sysctl.h>
#include <sys/ktr.h>
#include <sys/systm.h>
#include <sys/time.h>
#include <sys/malloc.h>
#include <sys/spinlock.h>
#include <sys/thread2.h>
#include <sys/spinlock2.h>
#include <sys/ctype.h>

#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/specialreg.h>
#include <machine/md_var.h>

#include <ddb/ddb.h>

#ifndef KTR_ENTRIES
#define	KTR_ENTRIES		2048
#elif (KTR_ENTRIES & KTR_ENTRIES - 1)
#error KTR_ENTRIES must be a power of two
#endif
#define KTR_ENTRIES_MASK	(KTR_ENTRIES - 1)

/*
 * test logging support.  When ktr_testlogcnt is non-zero each synchronization
 * interrupt will issue six back-to-back ktr logging messages on cpu 0
 * so the user can determine KTR logging overheads.
 */
#if !defined(KTR_TESTLOG)
#define KTR_TESTLOG	KTR_ALL
#endif
KTR_INFO_MASTER(testlog);
#if KTR_TESTLOG
KTR_INFO(KTR_TESTLOG, testlog, test1, 0, "test1 %d %d %d %d", int dummy1, int dummy2, int dummy3, int dummy4);
KTR_INFO(KTR_TESTLOG, testlog, test2, 1, "test2 %d %d %d %d", int dummy1, int dummy2, int dummy3, int dummy4);
KTR_INFO(KTR_TESTLOG, testlog, test3, 2, "test3 %d %d %d %d", int dummy1, int dummy2, int dummy3, int dummy4);
KTR_INFO(KTR_TESTLOG, testlog, test4, 3, "test4");
KTR_INFO(KTR_TESTLOG, testlog, test5, 4, "test5");
KTR_INFO(KTR_TESTLOG, testlog, test6, 5, "test6");
#ifdef SMP
KTR_INFO(KTR_TESTLOG, testlog, pingpong, 6, "pingpong");
KTR_INFO(KTR_TESTLOG, testlog, pipeline, 7, "pipeline");
KTR_INFO(KTR_TESTLOG, testlog, crit_beg, 8, "crit_beg");
KTR_INFO(KTR_TESTLOG, testlog, crit_end, 9, "crit_end");
KTR_INFO(KTR_TESTLOG, testlog, spin_beg, 10, "spin_beg");
KTR_INFO(KTR_TESTLOG, testlog, spin_end, 11, "spin_end");
#endif
#define logtest(name)	KTR_LOG(testlog_ ## name, 0, 0, 0, 0)
#define logtest_noargs(name)	KTR_LOG(testlog_ ## name)
#endif

MALLOC_DEFINE(M_KTR, "ktr", "ktr buffers");

SYSCTL_NODE(_debug, OID_AUTO, ktr, CTLFLAG_RW, 0, "ktr");

int		ktr_entries = KTR_ENTRIES;
SYSCTL_INT(_debug_ktr, OID_AUTO, entries, CTLFLAG_RD, &ktr_entries, 0,
    "Size of the event buffer");

int		ktr_version = KTR_VERSION;
SYSCTL_INT(_debug_ktr, OID_AUTO, version, CTLFLAG_RD, &ktr_version, 0, "");

static int	ktr_stacktrace = 1;
SYSCTL_INT(_debug_ktr, OID_AUTO, stacktrace, CTLFLAG_RD, &ktr_stacktrace, 0, "");

static int	ktr_resynchronize = 0;
SYSCTL_INT(_debug_ktr, OID_AUTO, resynchronize, CTLFLAG_RW,
    &ktr_resynchronize, 0, "Resynchronize TSC 10 times a second");

#if KTR_TESTLOG
static int	ktr_testlogcnt = 0;
SYSCTL_INT(_debug_ktr, OID_AUTO, testlogcnt, CTLFLAG_RW, &ktr_testlogcnt, 0, "");
static int	ktr_testipicnt = 0;
#ifdef SMP
static int	ktr_testipicnt_remainder;
#endif
SYSCTL_INT(_debug_ktr, OID_AUTO, testipicnt, CTLFLAG_RW, &ktr_testipicnt, 0, "");
static int	ktr_testcritcnt = 0;
SYSCTL_INT(_debug_ktr, OID_AUTO, testcritcnt, CTLFLAG_RW, &ktr_testcritcnt, 0, "");
static int	ktr_testspincnt = 0;
SYSCTL_INT(_debug_ktr, OID_AUTO, testspincnt, CTLFLAG_RW, &ktr_testspincnt, 0, "");
#endif

/*
 * Give cpu0 a static buffer so the tracepoint facility can be used during
 * early boot (note however that we still use a critical section, XXX).
 */
static struct	ktr_entry ktr_buf0[KTR_ENTRIES];

struct ktr_cpu ktr_cpu[MAXCPU] = {
	{ .core.ktr_buf = &ktr_buf0[0] }
};

#ifdef SMP
static int64_t	ktr_sync_tsc;
#endif
struct callout	ktr_resync_callout;

#ifdef KTR_VERBOSE
int	ktr_verbose = KTR_VERBOSE;
TUNABLE_INT("debug.ktr.verbose", &ktr_verbose);
SYSCTL_INT(_debug_ktr, OID_AUTO, verbose, CTLFLAG_RW, &ktr_verbose, 0,
    "Log events to the console as well");
#endif

static void ktr_resync_callback(void *dummy __unused);

extern int64_t tsc_offsets[];

static void
ktr_sysinit(void *dummy)
{
	struct ktr_cpu_core *kcpu;
	int i;

	for(i = 1; i < ncpus; ++i) {
		kcpu = &ktr_cpu[i].core;
		kcpu->ktr_buf = kmalloc(KTR_ENTRIES * sizeof(struct ktr_entry),
					M_KTR, M_WAITOK | M_ZERO);
	}
	callout_init_mp(&ktr_resync_callout);
	callout_reset(&ktr_resync_callout, hz / 10, ktr_resync_callback, NULL);
}
SYSINIT(ktr_sysinit, SI_BOOT2_KLD, SI_ORDER_ANY, ktr_sysinit, NULL);

/*
 * Try to resynchronize the TSC's for all cpus.  This is really, really nasty.
 * We have to send an IPIQ message to all remote cpus, wait until they 
 * get into their IPIQ processing code loop, then do an even stricter hard
 * loop to get the cpus as close to synchronized as we can to get the most
 * accurate reading.
 *
 * This callback occurs on cpu0.
 */
#if KTR_TESTLOG
#ifdef SMP
static void ktr_pingpong_remote(void *dummy);
static void ktr_pipeline_remote(void *dummy);
#endif
#endif

#if defined(SMP) && defined(_RDTSC_SUPPORTED_)

static void ktr_resync_remote(void *dummy);
extern cpumask_t smp_active_mask;

/*
 * We use a callout callback instead of a systimer because we cannot afford
 * to preempt anyone to do this, or we might deadlock a spin-lock or 
 * serializer between two cpus.
 */
static
void 
ktr_resync_callback(void *dummy __unused)
{
	struct lwkt_cpusync cs;
#if KTR_TESTLOG
	int count;
#endif

	KKASSERT(mycpu->gd_cpuid == 0);

#if KTR_TESTLOG
	/*
	 * Test logging
	 */
	if (ktr_testlogcnt) {
		--ktr_testlogcnt;
		cpu_disable_intr();
		logtest(test1);
		logtest(test2);
		logtest(test3);
		logtest_noargs(test4);
		logtest_noargs(test5);
		logtest_noargs(test6);
		cpu_enable_intr();
	}

	/*
	 * Test IPI messaging
	 */
	if (ktr_testipicnt && ktr_testipicnt_remainder == 0 && ncpus > 1) {
		ktr_testipicnt_remainder = ktr_testipicnt;
		ktr_testipicnt = 0;
		lwkt_send_ipiq_bycpu(1, ktr_pingpong_remote, NULL);
	}

	/*
	 * Test critical sections
	 */
	if (ktr_testcritcnt) {
		crit_enter();
		crit_exit();
		logtest_noargs(crit_beg);
		for (count = ktr_testcritcnt; count; --count) {
			crit_enter();
			crit_exit();
		}
		logtest_noargs(crit_end);
		ktr_testcritcnt = 0;
	}

	/*
	 * Test spinlock sections
	 */
	if (ktr_testspincnt) {
		struct spinlock spin;

		spin_init(&spin);
		spin_lock(&spin);
		spin_unlock(&spin);
		logtest_noargs(spin_beg);
		for (count = ktr_testspincnt; count; --count) {
			spin_lock(&spin);
			spin_unlock(&spin);
		}
		logtest_noargs(spin_end);
		ktr_testspincnt = 0;
	}
#endif

	/*
	 * Resynchronize the TSC
	 */
	if (ktr_resynchronize == 0)
		goto done;
	if ((cpu_feature & CPUID_TSC) == 0)
		return;

	crit_enter();
	lwkt_cpusync_init(&cs, smp_active_mask, ktr_resync_remote,
			  (void *)(intptr_t)mycpu->gd_cpuid);
	lwkt_cpusync_interlock(&cs);
	ktr_sync_tsc = rdtsc();
	lwkt_cpusync_deinterlock(&cs);
	crit_exit();
done:
	callout_reset(&ktr_resync_callout, hz / 10, ktr_resync_callback, NULL);
}

/*
 * The remote-end of the KTR synchronization protocol runs on all cpus.
 * The one we run on the controlling cpu updates its tsc continuously
 * until the others have finished syncing (theoretically), but we don't
 * loop forever.
 *
 * This is a bit ad-hoc but we need to avoid livelocking inside an IPI
 * callback.  rdtsc() is a synchronizing instruction (I think).
 */
static void
ktr_resync_remote(void *arg)
{
	globaldata_t gd = mycpu;
	int64_t delta;
	int i;

	if (gd->gd_cpuid == (int)(intptr_t)arg) {
		for (i = 0; i < 2000; ++i)
			ktr_sync_tsc = rdtsc();
	} else {
		delta = rdtsc() - ktr_sync_tsc;
		if (tsc_offsets[gd->gd_cpuid] == 0)
			tsc_offsets[gd->gd_cpuid] = delta;
		tsc_offsets[gd->gd_cpuid] =
			(tsc_offsets[gd->gd_cpuid] * 7 + delta) / 8;
	}
}

#if KTR_TESTLOG

static
void
ktr_pingpong_remote(void *dummy __unused)
{
	int other_cpu;

	logtest_noargs(pingpong);
	other_cpu = 1 - mycpu->gd_cpuid;
	if (ktr_testipicnt_remainder) {
		--ktr_testipicnt_remainder;
		lwkt_send_ipiq_bycpu(other_cpu, ktr_pingpong_remote, NULL);
	} else {
		lwkt_send_ipiq_bycpu(other_cpu, ktr_pipeline_remote, NULL);
		lwkt_send_ipiq_bycpu(other_cpu, ktr_pipeline_remote, NULL);
		lwkt_send_ipiq_bycpu(other_cpu, ktr_pipeline_remote, NULL);
		lwkt_send_ipiq_bycpu(other_cpu, ktr_pipeline_remote, NULL);
		lwkt_send_ipiq_bycpu(other_cpu, ktr_pipeline_remote, NULL);
	}
}

static
void
ktr_pipeline_remote(void *dummy __unused)
{
	logtest_noargs(pipeline);
}

#endif

#else	/* !SMP */

/*
 * The resync callback for UP doesn't do anything other then run the test
 * log messages.  If test logging is not enabled, don't bother resetting
 * the callout.
 */
static
void 
ktr_resync_callback(void *dummy __unused)
{
#if KTR_TESTLOG
	/*
	 * Test logging
	 */
	if (ktr_testlogcnt) {
		--ktr_testlogcnt;
		cpu_disable_intr();
		logtest(test1);
		logtest(test2);
		logtest(test3);
		logtest_noargs(test4);
		logtest_noargs(test5);
		logtest_noargs(test6);
		cpu_enable_intr();
	}
	callout_reset(&ktr_resync_callout, hz / 10, ktr_resync_callback, NULL);
#endif
}

#endif

/*
 * Setup the next empty slot and return it to the caller to store the data
 * directly.
 */
struct ktr_entry *
ktr_begin_write_entry(struct ktr_info *info, const char *file, int line)
{
	struct ktr_cpu_core *kcpu;
	struct ktr_entry *entry;
	int cpu;

	cpu = mycpu->gd_cpuid;
	kcpu = &ktr_cpu[cpu].core;
	if (panicstr)			/* stop logging during panic */
		return NULL;
	if (kcpu->ktr_buf == NULL)	/* too early in boot */
		return NULL;

	crit_enter();
	entry = kcpu->ktr_buf + (kcpu->ktr_idx & KTR_ENTRIES_MASK);
	++kcpu->ktr_idx;
#ifdef _RDTSC_SUPPORTED_
	if (cpu_feature & CPUID_TSC) {
#ifdef SMP
		entry->ktr_timestamp = rdtsc() - tsc_offsets[cpu];
#else
		entry->ktr_timestamp = rdtsc();
#endif
	} else
#endif
	{
		entry->ktr_timestamp = get_approximate_time_t();
	}
	entry->ktr_info = info;
	entry->ktr_file = file;
	entry->ktr_line = line;
	crit_exit();
	return entry;
}

int
ktr_finish_write_entry(struct ktr_info *info, struct ktr_entry *entry)
{
	if (ktr_stacktrace)
		cpu_ktr_caller(entry);
#ifdef KTR_VERBOSE
	if (ktr_verbose && info->kf_format) {
#ifdef SMP
		kprintf("cpu%d ", mycpu->gd_cpuid);
#endif
		if (ktr_verbose > 1) {
			kprintf("%s.%d\t", entry->ktr_file, entry->ktr_line);
		}
		return !0;
	}
#endif
	return 0;
}

#ifdef DDB

#define	NUM_LINES_PER_PAGE	19

struct tstate {
	int	cur;
	int	first;
};

static	int db_ktr_verbose;
static	int db_mach_vtrace(int cpu, struct ktr_entry *kp, int idx);

DB_SHOW_COMMAND(ktr, db_ktr_all)
{
	struct ktr_cpu_core *kcpu;
	int a_flag = 0;
	int c;
	int nl = 0;
	int i;
	struct tstate tstate[MAXCPU];
	int printcpu = -1;

	for(i = 0; i < ncpus; i++) {
		kcpu = &ktr_cpu[i].core;
		tstate[i].first = -1;
		tstate[i].cur = (kcpu->ktr_idx - 1) & KTR_ENTRIES_MASK;
	}
	db_ktr_verbose = 0;
	while ((c = *(modif++)) != '\0') {
		if (c == 'v') {
			db_ktr_verbose = 1;
		}
		else if (c == 'a') {
			a_flag = 1;
		}
		else if (c == 'c') {
			printcpu = 0;
			while ((c = *(modif++)) != '\0') {
				if (isdigit(c)) {
					printcpu *= 10;
					printcpu += c - '0';
				}
				else {
					modif++;
					break;
				}
			}
			modif--;
		}
	}
	if (printcpu > ncpus - 1) {
		db_printf("Invalid cpu number\n");
		return;
	}
	/*
	 * Lopp throug all the buffers and print the content of them, sorted
	 * by the timestamp.
	 */
	while (1) {
		int counter;
		u_int64_t highest_ts;
		int highest_cpu;
		struct ktr_entry *kp;

		if (a_flag == 1 && cncheckc() != -1)
			return;
		highest_ts = 0;
		highest_cpu = -1;
		/*
		 * Find the lowest timestamp
		 */
		for (i = 0, counter = 0; i < ncpus; i++) {
			kcpu = &ktr_cpu[i].core;
			if (kcpu->ktr_buf == NULL)
				continue;
			if (printcpu != -1 && printcpu != i)
				continue;
			if (tstate[i].cur == -1) {
				counter++;
				if (counter == ncpus) {
					db_printf("--- End of trace buffer ---\n");
					return;
				}
				continue;
			}
			if (kcpu->ktr_buf[tstate[i].cur].ktr_timestamp > highest_ts) {
				highest_ts = kcpu->ktr_buf[tstate[i].cur].ktr_timestamp;
				highest_cpu = i;
			}
		}
		if (highest_cpu < 0) {
			db_printf("no KTR data available\n");
			break;
		}
		i = highest_cpu;
		kcpu = &ktr_cpu[i].core;
		kp = &kcpu->ktr_buf[tstate[i].cur];
		if (tstate[i].first == -1)
			tstate[i].first = tstate[i].cur;
		if (--tstate[i].cur < 0)
			tstate[i].cur = KTR_ENTRIES - 1;
		if (tstate[i].first == tstate[i].cur) {
			db_mach_vtrace(i, kp, tstate[i].cur + 1);
			tstate[i].cur = -1;
			continue;
		}
		if (kcpu->ktr_buf[tstate[i].cur].ktr_info == NULL)
			tstate[i].cur = -1;
		if (db_more(&nl) == -1)
			break;
		if (db_mach_vtrace(i, kp, tstate[i].cur + 1) == 0)
			tstate[i].cur = -1;
	}
}

static int
db_mach_vtrace(int cpu, struct ktr_entry *kp, int idx)
{
	if (kp->ktr_info == NULL)
		return(0);
#ifdef SMP
	db_printf("cpu%d ", cpu);
#endif
	db_printf("%d: ", idx);
	if (db_ktr_verbose) {
		db_printf("%10.10lld %s.%d\t", (long long)kp->ktr_timestamp,
		    kp->ktr_file, kp->ktr_line);
	}
	db_printf("%s\t", kp->ktr_info->kf_name);
	db_printf("from(%p,%p) ", kp->ktr_caller1, kp->ktr_caller2);
#ifdef __i386__
	if (kp->ktr_info->kf_format)
		db_vprintf(kp->ktr_info->kf_format, (__va_list)kp->ktr_data);
#endif
	db_printf("\n");

	return(1);
}

#endif	/* DDB */
Commit	Line	Data
d3776285 MD	1	/*
	2	* Copyright (c) 2005 The DragonFly Project. All rights reserved.
	3	*
	4	* This code is derived from software contributed to The DragonFly Project
	5	* by Matthew Dillon <dillon@backplane.com>
	6	*
	7	* Redistribution and use in source and binary forms, with or without
	8	* modification, are permitted provided that the following conditions
	9	* are met:
	10	*
	11	* 1. Redistributions of source code must retain the above copyright
	12	* notice, this list of conditions and the following disclaimer.
	13	* 2. Redistributions in binary form must reproduce the above copyright
	14	* notice, this list of conditions and the following disclaimer in
	15	* the documentation and/or other materials provided with the
	16	* distribution.
	17	* 3. Neither the name of The DragonFly Project nor the names of its
	18	* contributors may be used to endorse or promote products derived
	19	* from this software without specific, prior written permission.
	20	*
	21	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	22	* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	23	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	24	* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
	25	* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
	26	* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
	27	* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	28	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	29	* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	30	* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
	31	* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	32	* SUCH DAMAGE.
	33	*/
	34	/*
	35	* The following copyright applies to the DDB command code:
	36	*
81540c2d EN	37	* Copyright (c) 2000 John Baldwin <jhb@FreeBSD.org>
	38	* All rights reserved.
	39	*
	40	* Redistribution and use in source and binary forms, with or without
	41	* modification, are permitted provided that the following conditions
	42	* are met:
	43	* 1. Redistributions of source code must retain the above copyright
	44	* notice, this list of conditions and the following disclaimer.
	45	* 2. Redistributions in binary form must reproduce the above copyright
	46	* notice, this list of conditions and the following disclaimer in the
	47	* documentation and/or other materials provided with the distribution.
	48	* 3. Neither the name of the author nor the names of any co-contributors
	49	* may be used to endorse or promote products derived from this software
	50	* without specific prior written permission.
	51	*
	52	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	53	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	54	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	55	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	56	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	57	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	58	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	59	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	60	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	61	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	62	* SUCH DAMAGE.
	63	*/
6d22945a	64
81540c2d	65	/*
d3776285	66	* Kernel tracepoint facility.
81540c2d EN	67	*/
	68
	69	#include "opt_ddb.h"
	70	#include "opt_ktr.h"
	71
	72	#include <sys/param.h>
	73	#include <sys/cons.h>
	74	#include <sys/kernel.h>
81540c2d EN	75	#include <sys/libkern.h>
	76	#include <sys/proc.h>
	77	#include <sys/sysctl.h>
d3776285	78	#include <sys/ktr.h>
81540c2d EN	79	#include <sys/systm.h>
81540c2d EN	80	#include <sys/time.h>
8e273a1d	81	#include <sys/malloc.h>
c2751817	82	#include <sys/spinlock.h>
8e273a1d	83	#include <sys/thread2.h>
c2751817	84	#include <sys/spinlock2.h>
8e273a1d	85	#include <sys/ctype.h>
81540c2d EN	86
	87	#include <machine/cpu.h>
	88	#include <machine/cpufunc.h>
	89	#include <machine/specialreg.h>
	90	#include <machine/md_var.h>
	91
	92	#include <ddb/ddb.h>
	93
	94	#ifndef KTR_ENTRIES
d3776285	95	#define KTR_ENTRIES 2048
6d22945a SW	96	#elif (KTR_ENTRIES & KTR_ENTRIES - 1)
6d22945a SW	97	#error KTR_ENTRIES must be a power of two
81540c2d	98	#endif
d3776285	99	#define KTR_ENTRIES_MASK (KTR_ENTRIES - 1)
81540c2d	100
96328d40 MD	101	/*
96328d40 MD	102	* test logging support. When ktr_testlogcnt is non-zero each synchronization
0b698dca	103	* interrupt will issue six back-to-back ktr logging messages on cpu 0
96328d40 MD	104	* so the user can determine KTR logging overheads.
	105	*/
	106	#if !defined(KTR_TESTLOG)
	107	#define KTR_TESTLOG KTR_ALL
	108	#endif
	109	KTR_INFO_MASTER(testlog);
8cb417fa	110	#if KTR_TESTLOG
5bf48697 AE	111	KTR_INFO(KTR_TESTLOG, testlog, test1, 0, "test1 %d %d %d %d", int dummy1, int dummy2, int dummy3, int dummy4);
	112	KTR_INFO(KTR_TESTLOG, testlog, test2, 1, "test2 %d %d %d %d", int dummy1, int dummy2, int dummy3, int dummy4);
	113	KTR_INFO(KTR_TESTLOG, testlog, test3, 2, "test3 %d %d %d %d", int dummy1, int dummy2, int dummy3, int dummy4);
	114	KTR_INFO(KTR_TESTLOG, testlog, test4, 3, "test4");
	115	KTR_INFO(KTR_TESTLOG, testlog, test5, 4, "test5");
	116	KTR_INFO(KTR_TESTLOG, testlog, test6, 5, "test6");
c2751817	117	#ifdef SMP
5bf48697 AE	118	KTR_INFO(KTR_TESTLOG, testlog, pingpong, 6, "pingpong");
	119	KTR_INFO(KTR_TESTLOG, testlog, pipeline, 7, "pipeline");
	120	KTR_INFO(KTR_TESTLOG, testlog, crit_beg, 8, "crit_beg");
	121	KTR_INFO(KTR_TESTLOG, testlog, crit_end, 9, "crit_end");
	122	KTR_INFO(KTR_TESTLOG, testlog, spin_beg, 10, "spin_beg");
	123	KTR_INFO(KTR_TESTLOG, testlog, spin_end, 11, "spin_end");
0d1d845c	124	#endif
96328d40	125	#define logtest(name) KTR_LOG(testlog_ ## name, 0, 0, 0, 0)
0b698dca	126	#define logtest_noargs(name) KTR_LOG(testlog_ ## name)
8cb417fa	127	#endif
96328d40	128
8e273a1d EN	129	MALLOC_DEFINE(M_KTR, "ktr", "ktr buffers");
8e273a1d EN	130
d3776285	131	SYSCTL_NODE(_debug, OID_AUTO, ktr, CTLFLAG_RW, 0, "ktr");
81540c2d	132
ffc26552	133	int ktr_entries = KTR_ENTRIES;
f22af1e9 SW	134	SYSCTL_INT(_debug_ktr, OID_AUTO, entries, CTLFLAG_RD, &ktr_entries, 0,
f22af1e9 SW	135	"Size of the event buffer");
81540c2d	136
ffc26552	137	int ktr_version = KTR_VERSION;
81540c2d EN	138	SYSCTL_INT(_debug_ktr, OID_AUTO, version, CTLFLAG_RD, &ktr_version, 0, "");
81540c2d EN	139
af6ff89e MD	140	static int ktr_stacktrace = 1;
	141	SYSCTL_INT(_debug_ktr, OID_AUTO, stacktrace, CTLFLAG_RD, &ktr_stacktrace, 0, "");
	142
374133e3	143	static int ktr_resynchronize = 0;
f22af1e9 SW	144	SYSCTL_INT(_debug_ktr, OID_AUTO, resynchronize, CTLFLAG_RW,
f22af1e9 SW	145	&ktr_resynchronize, 0, "Resynchronize TSC 10 times a second");
374133e3	146
96328d40 MD	147	#if KTR_TESTLOG
	148	static int ktr_testlogcnt = 0;
	149	SYSCTL_INT(_debug_ktr, OID_AUTO, testlogcnt, CTLFLAG_RW, &ktr_testlogcnt, 0, "");
72d379df	150	static int ktr_testipicnt = 0;
0d1d845c	151	#ifdef SMP
72d379df	152	static int ktr_testipicnt_remainder;
0d1d845c	153	#endif
72d379df	154	SYSCTL_INT(_debug_ktr, OID_AUTO, testipicnt, CTLFLAG_RW, &ktr_testipicnt, 0, "");
c2751817 MD	155	static int ktr_testcritcnt = 0;
	156	SYSCTL_INT(_debug_ktr, OID_AUTO, testcritcnt, CTLFLAG_RW, &ktr_testcritcnt, 0, "");
	157	static int ktr_testspincnt = 0;
	158	SYSCTL_INT(_debug_ktr, OID_AUTO, testspincnt, CTLFLAG_RW, &ktr_testspincnt, 0, "");
96328d40 MD	159	#endif
96328d40 MD	160
d3776285 MD	161	/*
	162	* Give cpu0 a static buffer so the tracepoint facility can be used during
	163	* early boot (note however that we still use a critical section, XXX).
	164	*/
	165	static struct ktr_entry ktr_buf0[KTR_ENTRIES];
ddca1582	166
02289741	167	struct ktr_cpu ktr_cpu[MAXCPU] = {
ddca1582 MD	168	{ .core.ktr_buf = &ktr_buf0[0] }
	169	};
	170
0b698dca	171	#ifdef SMP
374133e3	172	static int64_t ktr_sync_tsc;
0b698dca	173	#endif
374133e3 MD	174	struct callout ktr_resync_callout;
374133e3 MD	175
81540c2d EN	176	#ifdef KTR_VERBOSE
	177	int ktr_verbose = KTR_VERBOSE;
	178	TUNABLE_INT("debug.ktr.verbose", &ktr_verbose);
f22af1e9 SW	179	SYSCTL_INT(_debug_ktr, OID_AUTO, verbose, CTLFLAG_RW, &ktr_verbose, 0,
f22af1e9 SW	180	"Log events to the console as well");
81540c2d EN	181	#endif
81540c2d EN	182
ba39e2e0 MD	183	static void ktr_resync_callback(void *dummy __unused);
ba39e2e0 MD	184
c2751817	185	extern int64_t tsc_offsets[];
0b698dca	186
81540c2d EN	187	static void
	188	ktr_sysinit(void *dummy)
	189	{
ddca1582	190	struct ktr_cpu_core *kcpu;
8e273a1d EN	191	int i;
8e273a1d EN	192
d3776285	193	for(i = 1; i < ncpus; ++i) {
ddca1582 MD	194	kcpu = &ktr_cpu[i].core;
	195	kcpu->ktr_buf = kmalloc(KTR_ENTRIES * sizeof(struct ktr_entry),
	196	M_KTR, M_WAITOK \| M_ZERO);
8e273a1d	197	}
bf0ecf68	198	callout_init_mp(&ktr_resync_callout);
ba39e2e0	199	callout_reset(&ktr_resync_callout, hz / 10, ktr_resync_callback, NULL);
81540c2d	200	}
ba39e2e0	201	SYSINIT(ktr_sysinit, SI_BOOT2_KLD, SI_ORDER_ANY, ktr_sysinit, NULL);
374133e3	202
374133e3 MD	203	/*
	204	* Try to resynchronize the TSC's for all cpus. This is really, really nasty.
	205	* We have to send an IPIQ message to all remote cpus, wait until they
	206	* get into their IPIQ processing code loop, then do an even stricter hard
	207	* loop to get the cpus as close to synchronized as we can to get the most
	208	* accurate reading.
	209	*
	210	* This callback occurs on cpu0.
	211	*/
8cb417fa	212	#if KTR_TESTLOG
0d1d845c	213	#ifdef SMP
72d379df	214	static void ktr_pingpong_remote(void *dummy);
c2751817	215	static void ktr_pipeline_remote(void *dummy);
8cb417fa	216	#endif
0d1d845c	217	#endif
374133e3	218
527fddf7	219	#if defined(SMP) && defined(_RDTSC_SUPPORTED_)
b4c3db6f MD	220
b4c3db6f MD	221	static void ktr_resync_remote(void *dummy);
374133e3 MD	222	extern cpumask_t smp_active_mask;
	223
	224	/*
	225	* We use a callout callback instead of a systimer because we cannot afford
	226	* to preempt anyone to do this, or we might deadlock a spin-lock or
	227	* serializer between two cpus.
	228	*/
	229	static
	230	void
	231	ktr_resync_callback(void *dummy __unused)
	232	{
f697b97d	233	struct lwkt_cpusync cs;
ef4b87ee	234	#if KTR_TESTLOG
374133e3	235	int count;
ef4b87ee	236	#endif
374133e3 MD	237
374133e3 MD	238	KKASSERT(mycpu->gd_cpuid == 0);
96328d40 MD	239
	240	#if KTR_TESTLOG
	241	/*
	242	* Test logging
	243	*/
	244	if (ktr_testlogcnt) {
	245	--ktr_testlogcnt;
	246	cpu_disable_intr();
	247	logtest(test1);
	248	logtest(test2);
	249	logtest(test3);
0b698dca MD	250	logtest_noargs(test4);
	251	logtest_noargs(test5);
	252	logtest_noargs(test6);
96328d40 MD	253	cpu_enable_intr();
96328d40 MD	254	}
72d379df MD	255
	256	/*
	257	* Test IPI messaging
	258	*/
	259	if (ktr_testipicnt && ktr_testipicnt_remainder == 0 && ncpus > 1) {
	260	ktr_testipicnt_remainder = ktr_testipicnt;
	261	ktr_testipicnt = 0;
	262	lwkt_send_ipiq_bycpu(1, ktr_pingpong_remote, NULL);
	263	}
c2751817 MD	264
	265	/*
	266	* Test critical sections
	267	*/
	268	if (ktr_testcritcnt) {
	269	crit_enter();
	270	crit_exit();
	271	logtest_noargs(crit_beg);
	272	for (count = ktr_testcritcnt; count; --count) {
	273	crit_enter();
	274	crit_exit();
	275	}
	276	logtest_noargs(crit_end);
	277	ktr_testcritcnt = 0;
	278	}
	279
	280	/*
	281	* Test spinlock sections
	282	*/
	283	if (ktr_testspincnt) {
	284	struct spinlock spin;
	285
	286	spin_init(&spin);
287a8577 AH	287	spin_lock(&spin);
287a8577 AH	288	spin_unlock(&spin);
c2751817 MD	289	logtest_noargs(spin_beg);
c2751817 MD	290	for (count = ktr_testspincnt; count; --count) {
287a8577 AH	291	spin_lock(&spin);
287a8577 AH	292	spin_unlock(&spin);
d666840a MD	293	}
d666840a MD	294	logtest_noargs(spin_end);
c2751817 MD	295	ktr_testspincnt = 0;
c2751817 MD	296	}
96328d40 MD	297	#endif
	298
	299	/*
	300	* Resynchronize the TSC
	301	*/
374133e3 MD	302	if (ktr_resynchronize == 0)
	303	goto done;
	304	if ((cpu_feature & CPUID_TSC) == 0)
	305	return;
0b698dca	306
374133e3	307	crit_enter();
f697b97d MD	308	lwkt_cpusync_init(&cs, smp_active_mask, ktr_resync_remote,
	309	(void *)(intptr_t)mycpu->gd_cpuid);
	310	lwkt_cpusync_interlock(&cs);
374133e3	311	ktr_sync_tsc = rdtsc();
f697b97d	312	lwkt_cpusync_deinterlock(&cs);
374133e3	313	crit_exit();
374133e3 MD	314	done:
	315	callout_reset(&ktr_resync_callout, hz / 10, ktr_resync_callback, NULL);
	316	}
	317
0b698dca	318	/*
f697b97d MD	319	* The remote-end of the KTR synchronization protocol runs on all cpus.
	320	* The one we run on the controlling cpu updates its tsc continuously
	321	* until the others have finished syncing (theoretically), but we don't
	322	* loop forever.
	323	*
	324	* This is a bit ad-hoc but we need to avoid livelocking inside an IPI
	325	* callback. rdtsc() is a synchronizing instruction (I think).
0b698dca	326	*/
374133e3	327	static void
f697b97d	328	ktr_resync_remote(void *arg)
374133e3	329	{
f697b97d MD	330	globaldata_t gd = mycpu;
	331	int64_t delta;
	332	int i;
374133e3	333
f697b97d MD	334	if (gd->gd_cpuid == (int)(intptr_t)arg) {
	335	for (i = 0; i < 2000; ++i)
	336	ktr_sync_tsc = rdtsc();
	337	} else {
	338	delta = rdtsc() - ktr_sync_tsc;
	339	if (tsc_offsets[gd->gd_cpuid] == 0)
	340	tsc_offsets[gd->gd_cpuid] = delta;
	341	tsc_offsets[gd->gd_cpuid] =
	342	(tsc_offsets[gd->gd_cpuid] * 7 + delta) / 8;
374133e3	343	}
374133e3	344	}
81540c2d	345
8cb417fa MD	346	#if KTR_TESTLOG
8cb417fa MD	347
72d379df MD	348	static
	349	void
	350	ktr_pingpong_remote(void *dummy __unused)
	351	{
c2751817 MD	352	int other_cpu;
c2751817 MD	353
72d379df	354	logtest_noargs(pingpong);
c2751817	355	other_cpu = 1 - mycpu->gd_cpuid;
72d379df MD	356	if (ktr_testipicnt_remainder) {
72d379df MD	357	--ktr_testipicnt_remainder;
c2751817 MD	358	lwkt_send_ipiq_bycpu(other_cpu, ktr_pingpong_remote, NULL);
	359	} else {
	360	lwkt_send_ipiq_bycpu(other_cpu, ktr_pipeline_remote, NULL);
	361	lwkt_send_ipiq_bycpu(other_cpu, ktr_pipeline_remote, NULL);
	362	lwkt_send_ipiq_bycpu(other_cpu, ktr_pipeline_remote, NULL);
	363	lwkt_send_ipiq_bycpu(other_cpu, ktr_pipeline_remote, NULL);
	364	lwkt_send_ipiq_bycpu(other_cpu, ktr_pipeline_remote, NULL);
72d379df MD	365	}
	366	}
	367
c2751817 MD	368	static
	369	void
	370	ktr_pipeline_remote(void *dummy __unused)
	371	{
	372	logtest_noargs(pipeline);
	373	}
	374
8cb417fa MD	375	#endif
8cb417fa MD	376
0b698dca MD	377	#else /* !SMP */
	378
	379	/*
	380	* The resync callback for UP doesn't do anything other then run the test
	381	* log messages. If test logging is not enabled, don't bother resetting
	382	* the callout.
	383	*/
	384	static
	385	void
	386	ktr_resync_callback(void *dummy __unused)
	387	{
	388	#if KTR_TESTLOG
	389	/*
	390	* Test logging
	391	*/
	392	if (ktr_testlogcnt) {
	393	--ktr_testlogcnt;
	394	cpu_disable_intr();
	395	logtest(test1);
	396	logtest(test2);
	397	logtest(test3);
	398	logtest_noargs(test4);
	399	logtest_noargs(test5);
	400	logtest_noargs(test6);
	401	cpu_enable_intr();
	402	}
	403	callout_reset(&ktr_resync_callout, hz / 10, ktr_resync_callback, NULL);
	404	#endif
	405	}
	406
374133e3	407	#endif
81540c2d	408
0b698dca	409	/*
5bf48697 AE	410	* Setup the next empty slot and return it to the caller to store the data
5bf48697 AE	411	* directly.
0b698dca	412	*/
5bf48697 AE	413	struct ktr_entry *
5bf48697 AE	414	ktr_begin_write_entry(struct ktr_info info, const char file, int line)
81540c2d	415	{
ddca1582	416	struct ktr_cpu_core *kcpu;
81540c2d	417	struct ktr_entry *entry;
d3776285 MD	418	int cpu;
	419
	420	cpu = mycpu->gd_cpuid;
ddca1582	421	kcpu = &ktr_cpu[cpu].core;
ab1b4385 MD	422	if (panicstr) /* stop logging during panic */
	423	return NULL;
	424	if (kcpu->ktr_buf == NULL) /* too early in boot */
5bf48697	425	return NULL;
b4cb74b5 SW	426
b4cb74b5 SW	427	crit_enter();
ddca1582 MD	428	entry = kcpu->ktr_buf + (kcpu->ktr_idx & KTR_ENTRIES_MASK);
ddca1582 MD	429	++kcpu->ktr_idx;
527fddf7	430	#ifdef _RDTSC_SUPPORTED_
b4cb74b5	431	if (cpu_feature & CPUID_TSC) {
0b698dca	432	#ifdef SMP
b4cb74b5	433	entry->ktr_timestamp = rdtsc() - tsc_offsets[cpu];
0b698dca	434	#else
b4cb74b5	435	entry->ktr_timestamp = rdtsc();
0b698dca	436	#endif
527fddf7 MD	437	} else
	438	#endif
	439	{
b4cb74b5	440	entry->ktr_timestamp = get_approximate_time_t();
8e273a1d	441	}
b4cb74b5 SW	442	entry->ktr_info = info;
	443	entry->ktr_file = file;
	444	entry->ktr_line = line;
	445	crit_exit();
5bf48697 AE	446	return entry;
	447	}
	448
	449	int
	450	ktr_finish_write_entry(struct ktr_info info, struct ktr_entry entry)
	451	{
b4cb74b5 SW	452	if (ktr_stacktrace)
b4cb74b5 SW	453	cpu_ktr_caller(entry);
81540c2d	454	#ifdef KTR_VERBOSE
d3776285	455	if (ktr_verbose && info->kf_format) {
81540c2d	456	#ifdef SMP
5bf48697	457	kprintf("cpu%d ", mycpu->gd_cpuid);
81540c2d EN	458	#endif
81540c2d EN	459	if (ktr_verbose > 1) {
6ea70f76	460	kprintf("%s.%d\t", entry->ktr_file, entry->ktr_line);
81540c2d	461	}
5bf48697	462	return !0;
81540c2d EN	463	}
81540c2d EN	464	#endif
5bf48697	465	return 0;
d3776285 MD	466	}
d3776285 MD	467
81540c2d EN	468	#ifdef DDB
81540c2d EN	469
8e273a1d	470	#define NUM_LINES_PER_PAGE 19
81540c2d EN	471
	472	struct tstate {
	473	int cur;
	474	int first;
	475	};
d3776285	476
81540c2d	477	static int db_ktr_verbose;
d3776285	478	static int db_mach_vtrace(int cpu, struct ktr_entry *kp, int idx);
81540c2d EN	479
	480	DB_SHOW_COMMAND(ktr, db_ktr_all)
	481	{
ddca1582	482	struct ktr_cpu_core *kcpu;
8e273a1d EN	483	int a_flag = 0;
8e273a1d EN	484	int c;
81540c2d	485	int nl = 0;
8e273a1d EN	486	int i;
	487	struct tstate tstate[MAXCPU];
	488	int printcpu = -1;
	489
	490	for(i = 0; i < ncpus; i++) {
ddca1582	491	kcpu = &ktr_cpu[i].core;
8e273a1d	492	tstate[i].first = -1;
ddca1582	493	tstate[i].cur = (kcpu->ktr_idx - 1) & KTR_ENTRIES_MASK;
8e273a1d EN	494	}
	495	db_ktr_verbose = 0;
	496	while ((c = *(modif++)) != '\0') {
	497	if (c == 'v') {
	498	db_ktr_verbose = 1;
	499	}
	500	else if (c == 'a') {
	501	a_flag = 1;
81540c2d	502	}
8e273a1d EN	503	else if (c == 'c') {
	504	printcpu = 0;
	505	while ((c = *(modif++)) != '\0') {
	506	if (isdigit(c)) {
	507	printcpu *= 10;
	508	printcpu += c - '0';
	509	}
	510	else {
	511	modif++;
	512	break;
	513	}
	514	}
	515	modif--;
81540c2d EN	516	}
81540c2d EN	517	}
8e273a1d EN	518	if (printcpu > ncpus - 1) {
	519	db_printf("Invalid cpu number\n");
	520	return;
	521	}
	522	/*
	523	* Lopp throug all the buffers and print the content of them, sorted
	524	* by the timestamp.
	525	*/
	526	while (1) {
	527	int counter;
	528	u_int64_t highest_ts;
	529	int highest_cpu;
	530	struct ktr_entry *kp;
	531
	532	if (a_flag == 1 && cncheckc() != -1)
	533	return;
	534	highest_ts = 0;
	535	highest_cpu = -1;
	536	/*
	537	* Find the lowest timestamp
	538	*/
	539	for (i = 0, counter = 0; i < ncpus; i++) {
ddca1582 MD	540	kcpu = &ktr_cpu[i].core;
ddca1582 MD	541	if (kcpu->ktr_buf == NULL)
d3776285	542	continue;
8e273a1d EN	543	if (printcpu != -1 && printcpu != i)
	544	continue;
	545	if (tstate[i].cur == -1) {
	546	counter++;
	547	if (counter == ncpus) {
	548	db_printf("--- End of trace buffer ---\n");
	549	return;
	550	}
	551	continue;
	552	}
ddca1582 MD	553	if (kcpu->ktr_buf[tstate[i].cur].ktr_timestamp > highest_ts) {
ddca1582 MD	554	highest_ts = kcpu->ktr_buf[tstate[i].cur].ktr_timestamp;
8e273a1d EN	555	highest_cpu = i;
	556	}
	557	}
438c526f MD	558	if (highest_cpu < 0) {
	559	db_printf("no KTR data available\n");
	560	break;
	561	}
8e273a1d	562	i = highest_cpu;
ddca1582 MD	563	kcpu = &ktr_cpu[i].core;
ddca1582 MD	564	kp = &kcpu->ktr_buf[tstate[i].cur];
8e273a1d EN	565	if (tstate[i].first == -1)
	566	tstate[i].first = tstate[i].cur;
	567	if (--tstate[i].cur < 0)
	568	tstate[i].cur = KTR_ENTRIES - 1;
	569	if (tstate[i].first == tstate[i].cur) {
d3776285	570	db_mach_vtrace(i, kp, tstate[i].cur + 1);
8e273a1d EN	571	tstate[i].cur = -1;
	572	continue;
	573	}
ddca1582	574	if (kcpu->ktr_buf[tstate[i].cur].ktr_info == NULL)
8e273a1d EN	575	tstate[i].cur = -1;
	576	if (db_more(&nl) == -1)
	577	break;
d3776285	578	if (db_mach_vtrace(i, kp, tstate[i].cur + 1) == 0)
8e273a1d EN	579	tstate[i].cur = -1;
8e273a1d EN	580	}
81540c2d EN	581	}
	582
	583	static int
d3776285	584	db_mach_vtrace(int cpu, struct ktr_entry *kp, int idx)
81540c2d	585	{
d3776285	586	if (kp->ktr_info == NULL)
8e273a1d	587	return(0);
81540c2d	588	#ifdef SMP
d3776285	589	db_printf("cpu%d ", cpu);
81540c2d	590	#endif
8e273a1d	591	db_printf("%d: ", idx);
81540c2d EN	592	if (db_ktr_verbose) {
	593	db_printf("%10.10lld %s.%d\t", (long long)kp->ktr_timestamp,
	594	kp->ktr_file, kp->ktr_line);
	595	}
d3776285	596	db_printf("%s\t", kp->ktr_info->kf_name);
af6ff89e	597	db_printf("from(%p,%p) ", kp->ktr_caller1, kp->ktr_caller2);
4afe74da	598	#ifdef __i386__
00828b33 MD	599	if (kp->ktr_info->kf_format)
00828b33 MD	600	db_vprintf(kp->ktr_info->kf_format, (__va_list)kp->ktr_data);
4afe74da	601	#endif
81540c2d EN	602	db_printf("\n");
81540c2d EN	603
8e273a1d	604	return(1);
81540c2d EN	605	}
	606
	607	#endif /* DDB */