usr.bin/ktrdump/ktrdump.c

   1 /*-
   2  * Copyright (c) 2002 Jake Burkholder
   3  * Copyright (c) 2004 Robert Watson
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  *
  27  * $FreeBSD: src/usr.bin/ktrdump/ktrdump.c,v 1.10 2005/05/21 09:55:06 ru Exp $
  28  * $DragonFly: src/usr.bin/ktrdump/ktrdump.c,v 1.13 2008/11/10 02:05:31 swildner Exp $
  29  */
  30
  31 #include <sys/cdefs.h>
  32
  33 #include <sys/types.h>
  34 #include <sys/ktr.h>
  35 #include <sys/mman.h>
  36 #include <sys/stat.h>
  37 #include <sys/queue.h>
  38
  39 #include <err.h>
  40 #include <fcntl.h>
  41 #include <kvm.h>
  42 #include <limits.h>
  43 #include <nlist.h>
  44 #include <stdint.h>
  45 #include <stdio.h>
  46 #include <stdlib.h>
  47 #include <string.h>
  48 #include <unistd.h>
  49
  50 #define SBUFLEN 128
  51
  52 struct ktr_buffer {
  53         struct ktr_entry *ents;
  54         int modified;
  55         int reset;
  56         int beg_idx;            /* Beginning index */
  57         int end_idx;            /* Ending index */
  58 };
  59
  60 static struct nlist nl1[] = {
  61         { .n_name = "_ktr_version" },
  62         { .n_name = "_ktr_entries" },
  63         { .n_name = "_ncpus" },
  64         { .n_name = NULL }
  65 };
  66
  67 static struct nlist nl2[] = {
  68         { .n_name = "_tsc_frequency" },
  69         { .n_name = NULL }
  70 };
  71
  72 static struct nlist nl_version_ktr_idx[] = {
  73         { .n_name = "_ktr_idx" },
  74         { .n_name = "_ktr_buf" },
  75         { .n_name = NULL }
  76 };
  77
  78 static struct nlist nl_version_ktr_cpu[] = {
  79         { .n_name = "_ktr_cpu" },
  80         { .n_name = NULL }
  81 };
  82
  83 static int cflag;
  84 static int fflag;
  85 static int iflag;
  86 static int lflag;
  87 static int nflag;
  88 static int qflag;
  89 static int rflag;
  90 static int sflag;
  91 static int tflag;
  92 static int xflag;
  93 static int pflag;
  94 static int Mflag;
  95 static int Nflag;
  96 static double tsc_frequency;
  97 static double correction_factor = 0.0;
  98
  99 static char corefile[PATH_MAX];
 100 static char execfile[PATH_MAX];
 101
 102 static char errbuf[_POSIX2_LINE_MAX];
 103 static int ncpus;
 104 static kvm_t *kd;
 105 static int entries_per_buf;
 106 static int fifo_mask;
 107 static int ktr_version;
 108
 109 static void usage(void);
 110 static int earliest_ts(struct ktr_buffer *);
 111 static void print_header(FILE *, int);
 112 static void print_entry(FILE *, int, int, struct ktr_entry *, u_int64_t *);
 113 static struct ktr_info *kvm_ktrinfo(void *);
 114 static const char *kvm_string(const char *);
 115 static const char *trunc_path(const char *, int);
 116 static void read_symbols(const char *);
 117 static const char *address_to_symbol(void *);
 118 static struct ktr_buffer *ktr_bufs_init(void);
 119 static void get_indices(struct ktr_entry **, int *);
 120 static void load_bufs(struct ktr_buffer *, struct ktr_entry **, int *);
 121 static void print_buf(FILE *, struct ktr_buffer *, int, u_int64_t *);
 122 static void print_bufs_timesorted(FILE *, struct ktr_buffer *, u_int64_t *);
 123
 124
 125 /*
 126  * Reads the ktr trace buffer from kernel memory and prints the trace entries.
 127  */
 128 int
 129 main(int ac, char **av)
 130 {
 131         struct ktr_buffer *ktr_bufs;
 132         struct ktr_entry **ktr_kbuf;
 133         int *ktr_idx;
 134         FILE *fo;
 135         int64_t tts;
 136         int *ktr_start_index;
 137         int c;
 138         int n;
 139
 140         /*
 141          * Parse commandline arguments.
 142          */
 143         fo = stdout;
 144         while ((c = getopt(ac, av, "acfinqrtxpslA:N:M:o:")) != -1) {
 145                 switch (c) {
 146                 case 'a':
 147                         cflag = 1;
 148                         iflag = 1;
 149                         rflag = 1;
 150                         xflag = 1;
 151                         pflag = 1;
 152                         rflag = 1;
 153                         sflag = 1;
 154                         break;
 155                 case 'c':
 156                         cflag = 1;
 157                         break;
 158                 case 'N':
 159                         if (strlcpy(execfile, optarg, sizeof(execfile))
 160                             >= sizeof(execfile))
 161                                 errx(1, "%s: File name too long", optarg);
 162                         Nflag = 1;
 163                         break;
 164                 case 'f':
 165                         fflag = 1;
 166                         break;
 167                 case 'l':
 168                         lflag = 1;
 169                         break;
 170                 case 'i':
 171                         iflag = 1;
 172                         break;
 173                 case 'A':
 174                         correction_factor = strtod(optarg, NULL);
 175                         break;
 176                 case 'M':
 177                         if (strlcpy(corefile, optarg, sizeof(corefile))
 178                             >= sizeof(corefile))
 179                                 errx(1, "%s: File name too long", optarg);
 180                         Mflag = 1;
 181                         break;
 182                 case 'n':
 183                         nflag = 1;
 184                         break;
 185                 case 'o':
 186                         if ((fo = fopen(optarg, "w")) == NULL)
 187                                 err(1, "%s", optarg);
 188                         break;
 189                 case 'p':
 190                         pflag++;
 191                         break;
 192                 case 'q':
 193                         qflag++;
 194                         break;
 195                 case 'r':
 196                         rflag = 1;
 197                         break;
 198                 case 's':
 199                         sflag = 1;      /* sort across the cpus */
 200                         break;
 201                 case 't':
 202                         tflag = 1;
 203                         break;
 204                 case 'x':
 205                         xflag = 1;
 206                         break;
 207                 case '?':
 208                 default:
 209                         usage();
 210                 }
 211         }
 212         if (cflag + iflag + tflag + xflag + fflag + pflag == 0) {
 213                 cflag = 1;
 214                 iflag = 1;
 215                 tflag = 1;
 216                 pflag = 1;
 217         }
 218         if (correction_factor != 0.0 && (rflag == 0 || nflag)) {
 219                 fprintf(stderr, "Correction factor can only be applied with -r and without -n\n");
 220                 exit(1);
 221         }
 222         ac -= optind;
 223         av += optind;
 224         if (ac != 0)
 225                 usage();
 226
 227         /*
 228          * Open our execfile and corefile, resolve needed symbols and read in
 229          * the trace buffer.
 230          */
 231         if ((kd = kvm_openfiles(Nflag ? execfile : NULL,
 232             Mflag ? corefile : NULL, NULL, O_RDONLY, errbuf)) == NULL)
 233                 errx(1, "%s", errbuf);
 234         if (kvm_nlist(kd, nl1) != 0)
 235                 errx(1, "%s", kvm_geterr(kd));
 236         if (kvm_read(kd, nl1[0].n_value, &ktr_version, sizeof(ktr_version)) == -1)
 237                 errx(1, "%s", kvm_geterr(kd));
 238         if (kvm_read(kd, nl1[2].n_value, &ncpus, sizeof(ncpus)) == -1)
 239                 errx(1, "%s", kvm_geterr(kd));
 240         ktr_start_index = malloc(sizeof(*ktr_start_index) * ncpus);
 241         if (ktr_version >= KTR_VERSION_WITH_FREQ && kvm_nlist(kd, nl2) == 0) {
 242                 if (kvm_read(kd, nl2[0].n_value, &tts, sizeof(tts)) == -1)
 243                         errx(1, "%s", kvm_geterr(kd));
 244                 tsc_frequency = (double)tts;
 245         }
 246         if (ktr_version > KTR_VERSION)
 247                 errx(1, "ktr version too high for us to handle");
 248         if (kvm_read(kd, nl1[1].n_value, &entries_per_buf,
 249                                 sizeof(entries_per_buf)) == -1)
 250                 errx(1, "%s", kvm_geterr(kd));
 251         fifo_mask = entries_per_buf - 1;
 252
 253         printf("TSC frequency is %6.3f MHz\n", tsc_frequency / 1000000.0);
 254
 255         ktr_kbuf = calloc(ncpus, sizeof(*ktr_kbuf));
 256         ktr_idx = calloc(ncpus, sizeof(*ktr_idx));
 257
 258         if (nflag == 0)
 259                 read_symbols(Nflag ? execfile : NULL);
 260
 261         if (ktr_version < KTR_VERSION_KTR_CPU) {
 262                 if (kvm_nlist(kd, nl_version_ktr_idx))
 263                         errx(1, "%s", kvm_geterr(kd));
 264         } else {
 265                 if (kvm_nlist(kd, nl_version_ktr_cpu))
 266                         errx(1, "%s", kvm_geterr(kd));
 267         }
 268
 269         get_indices(ktr_kbuf, ktr_idx);
 270
 271         ktr_bufs = ktr_bufs_init();
 272
 273         if (sflag) {
 274                 u_int64_t last_timestamp = 0;
 275                 do {
 276                         load_bufs(ktr_bufs, ktr_kbuf, ktr_idx);
 277                         print_bufs_timesorted(fo, ktr_bufs, &last_timestamp);
 278                         if (lflag)
 279                                 usleep(1000000 / 10);
 280                 } while (lflag);
 281         } else {
 282                 u_int64_t *last_timestamp = calloc(sizeof(u_int64_t), ncpus);
 283                 do {
 284                         load_bufs(ktr_bufs, ktr_kbuf, ktr_idx);
 285                         for (n = 0; n < ncpus; ++n)
 286                                 print_buf(fo, ktr_bufs, n, &last_timestamp[n]);
 287                         if (lflag)
 288                                 usleep(1000000 / 10);
 289                 } while (lflag);
 290         }
 291         return (0);
 292 }
 293
 294 static void
 295 print_header(FILE *fo, int row)
 296 {
 297         if (qflag == 0 && (u_int32_t)row % 20 == 0) {
 298                 fprintf(fo, "%-6s ", "index");
 299                 if (cflag)
 300                         fprintf(fo, "%-3s ", "cpu");
 301                 if (tflag || rflag)
 302                         fprintf(fo, "%-16s ", "timestamp");
 303                 if (xflag) {
 304                         if (nflag)
 305                             fprintf(fo, "%-10s %-10s", "caller2", "caller1");
 306                         else
 307                             fprintf(fo, "%-20s %-20s", "caller2", "caller1");
 308                 }
 309                 if (iflag)
 310                         fprintf(fo, "%-20s ", "ID");
 311                 if (fflag)
 312                         fprintf(fo, "%10s%-30s ", "", "file and line");
 313                 if (pflag)
 314                         fprintf(fo, "%s", "trace");
 315                 fprintf(fo, "\n");
 316         }
 317 }
 318
 319 static void
 320 print_entry(FILE *fo, int n, int row, struct ktr_entry *entry,
 321             u_int64_t *last_timestamp)
 322 {
 323         struct ktr_info *info = NULL;
 324
 325         fprintf(fo, " %06x ", row & 0x00FFFFFF);
 326         if (cflag)
 327                 fprintf(fo, "%-3d ", n);
 328         if (tflag || rflag) {
 329                 if (rflag && !nflag && tsc_frequency != 0.0) {
 330                         fprintf(fo, "%13.3f uS ",
 331                                 (double)(entry->ktr_timestamp - *last_timestamp) * 1000000.0 / tsc_frequency - correction_factor);
 332                 } else if (rflag) {
 333                         fprintf(fo, "%-16ju ",
 334                             (uintmax_t)(entry->ktr_timestamp - *last_timestamp));
 335                 } else {
 336                         fprintf(fo, "%-16ju ",
 337                             (uintmax_t)entry->ktr_timestamp);
 338                 }
 339         }
 340         if (xflag) {
 341                 if (nflag) {
 342                     fprintf(fo, "%p %p ",
 343                             entry->ktr_caller2, entry->ktr_caller1);
 344                 } else {
 345                     fprintf(fo, "%-20s ",
 346                             address_to_symbol(entry->ktr_caller2));
 347                     fprintf(fo, "%-20s ",
 348                             address_to_symbol(entry->ktr_caller1));
 349                 }
 350         }
 351         if (iflag) {
 352                 info = kvm_ktrinfo(entry->ktr_info);
 353                 if (info)
 354                         fprintf(fo, "%-20s ", kvm_string(info->kf_name));
 355                 else
 356                         fprintf(fo, "%-20s ", "<empty>");
 357         }
 358         if (fflag)
 359                 fprintf(fo, "%34s:%-4d ", trunc_path(kvm_string(entry->ktr_file), 34), entry->ktr_line);
 360         if (pflag) {
 361                 if (info == NULL)
 362                         info = kvm_ktrinfo(entry->ktr_info);
 363                 if (info)
 364                         vfprintf(fo, kvm_string(info->kf_format), (void *)&entry->ktr_data);
 365         }
 366         fprintf(fo, "\n");
 367         *last_timestamp = entry->ktr_timestamp;
 368 }
 369
 370 static
 371 struct ktr_info *
 372 kvm_ktrinfo(void *kptr)
 373 {
 374         static struct ktr_info save_info;
 375         static void *save_kptr;
 376
 377         if (kptr == NULL)
 378                 return(NULL);
 379         if (save_kptr != kptr) {
 380                 if (kvm_read(kd, (uintptr_t)kptr, &save_info, sizeof(save_info)) == -1) {
 381                         bzero(&save_info, sizeof(save_info));
 382                 } else {
 383                         save_kptr = kptr;
 384                 }
 385         }
 386         return(&save_info);
 387 }
 388
 389 static
 390 const char *
 391 kvm_string(const char *kptr)
 392 {
 393         static char save_str[128];
 394         static const char *save_kptr;
 395         u_int l;
 396         u_int n;
 397
 398         if (kptr == NULL)
 399                 return("?");
 400         if (save_kptr != kptr) {
 401                 save_kptr = kptr;
 402                 l = 0;
 403                 while (l < sizeof(save_str) - 1) {
 404                         n = 256 - ((intptr_t)(kptr + l) & 255);
 405                         if (n > sizeof(save_str) - l - 1)
 406                                 n = sizeof(save_str) - l - 1;
 407                         if (kvm_read(kd, (uintptr_t)(kptr + l), save_str + l, n) < 0)
 408                                 break;
 409                         while (l < sizeof(save_str) && n) {
 410                             if (save_str[l] == 0)
 411                                     break;
 412                             --n;
 413                             ++l;
 414                         }
 415                         if (n)
 416                             break;
 417                 }
 418                 save_str[l] = 0;
 419         }
 420         return(save_str);
 421 }
 422
 423 static
 424 const char *
 425 trunc_path(const char *str, int maxlen)
 426 {
 427         int len = strlen(str);
 428
 429         if (len > maxlen)
 430                 return(str + len - maxlen);
 431         else
 432                 return(str);
 433 }
 434
 435 struct symdata {
 436         TAILQ_ENTRY(symdata) link;
 437         const char *symname;
 438         char *symaddr;
 439         char symtype;
 440 };
 441
 442 static TAILQ_HEAD(symlist, symdata) symlist;
 443 static struct symdata *symcache;
 444 static char *symbegin;
 445 static char *symend;
 446
 447 static
 448 void
 449 read_symbols(const char *file)
 450 {
 451         char buf[256];
 452         char cmd[256];
 453         size_t buflen = sizeof(buf);
 454         FILE *fp;
 455         struct symdata *sym;
 456         char *s1;
 457         char *s2;
 458         char *s3;
 459
 460         TAILQ_INIT(&symlist);
 461
 462         if (file == NULL) {
 463                 if (sysctlbyname("kern.bootfile", buf, &buflen, NULL, 0) < 0)
 464                         file = "/boot/kernel";
 465                 else
 466                         file = buf;
 467         }
 468         snprintf(cmd, sizeof(cmd), "nm -n %s", file);
 469         if ((fp = popen(cmd, "r")) != NULL) {
 470                 while (fgets(buf, sizeof(buf), fp) != NULL) {
 471                     s1 = strtok(buf, " \t\n");
 472                     s2 = strtok(NULL, " \t\n");
 473                     s3 = strtok(NULL, " \t\n");
 474                     if (s1 && s2 && s3) {
 475                         sym = malloc(sizeof(struct symdata));
 476                         sym->symaddr = (char *)strtoul(s1, NULL, 16);
 477                         sym->symtype = s2[0];
 478                         sym->symname = strdup(s3);
 479                         if (strcmp(s3, "kernbase") == 0)
 480                                 symbegin = sym->symaddr;
 481                         if (strcmp(s3, "end") == 0)
 482                                 symend = sym->symaddr;
 483                         TAILQ_INSERT_TAIL(&symlist, sym, link);
 484                     }
 485                 }
 486                 pclose(fp);
 487         }
 488         symcache = TAILQ_FIRST(&symlist);
 489 }
 490
 491 static
 492 const char *
 493 address_to_symbol(void *kptr)
 494 {
 495         static char buf[64];
 496
 497         if (symcache == NULL ||
 498            (char *)kptr < symbegin || (char *)kptr >= symend
 499         ) {
 500                 snprintf(buf, sizeof(buf), "%p", kptr);
 501                 return(buf);
 502         }
 503         while ((char *)symcache->symaddr < (char *)kptr) {
 504                 if (TAILQ_NEXT(symcache, link) == NULL)
 505                         break;
 506                 symcache = TAILQ_NEXT(symcache, link);
 507         }
 508         while ((char *)symcache->symaddr > (char *)kptr) {
 509                 if (symcache != TAILQ_FIRST(&symlist))
 510                         symcache = TAILQ_PREV(symcache, symlist, link);
 511         }
 512         snprintf(buf, sizeof(buf), "%s+%d", symcache->symname,
 513                 (int)((char *)kptr - symcache->symaddr));
 514         return(buf);
 515 }
 516
 517 static
 518 struct ktr_buffer *
 519 ktr_bufs_init(void)
 520 {
 521         struct ktr_buffer *ktr_bufs, *it;
 522         int i;
 523
 524         ktr_bufs = malloc(sizeof(*ktr_bufs) * ncpus);
 525         if (!ktr_bufs)
 526                 err(1, "can't allocate data structures\n");
 527         for (i = 0; i < ncpus; ++i) {
 528                 it = ktr_bufs + i;
 529                 it->ents = malloc(sizeof(struct ktr_entry) * entries_per_buf);
 530                 if (it->ents == NULL)
 531                         err(1, "can't allocate data structures\n");
 532                 it->reset = 1;
 533                 it->beg_idx = -1;
 534                 it->end_idx = -1;
 535         }
 536         return ktr_bufs;
 537 }
 538
 539 static
 540 void
 541 get_indices(struct ktr_entry **ktr_kbuf, int *ktr_idx)
 542 {
 543         static struct ktr_cpu *ktr_cpus;
 544         int i;
 545
 546         if (ktr_cpus == NULL)
 547                 ktr_cpus = malloc(sizeof(*ktr_cpus) * ncpus);
 548
 549         if (ktr_version < KTR_VERSION_KTR_CPU) {
 550                 if (kvm_read(kd, nl_version_ktr_idx[0].n_value, ktr_idx,
 551                     sizeof(*ktr_idx) * ncpus) == -1) {
 552                         errx(1, "%s", kvm_geterr(kd));
 553                 }
 554                 if (ktr_kbuf[0] == NULL) {
 555                         if (kvm_read(kd, nl_version_ktr_idx[1].n_value,
 556                             ktr_kbuf, sizeof(*ktr_kbuf) * ncpus) == -1) {
 557                                 errx(1, "%s", kvm_geterr(kd));
 558                         }
 559                 }
 560         } else {
 561                 if (kvm_read(kd, nl_version_ktr_cpu[0].n_value,
 562                              ktr_cpus, sizeof(*ktr_cpus) * ncpus) == -1) {
 563                                 errx(1, "%s", kvm_geterr(kd));
 564                 }
 565                 for (i = 0; i < ncpus; ++i) {
 566                         ktr_idx[i] = ktr_cpus[i].core.ktr_idx;
 567                         ktr_kbuf[i] = ktr_cpus[i].core.ktr_buf;
 568                 }
 569         }
 570 }
 571
 572 /*
 573  * Get the trace buffer data from the kernel
 574  */
 575 static
 576 void
 577 load_bufs(struct ktr_buffer *ktr_bufs, struct ktr_entry **kbufs, int *ktr_idx)
 578 {
 579         struct ktr_buffer *kbuf;
 580         int i;
 581
 582         get_indices(kbufs, ktr_idx);
 583         for (i = 0; i < ncpus; ++i) {
 584                 kbuf = &ktr_bufs[i];
 585                 if (ktr_idx[i] == kbuf->end_idx)
 586                         continue;
 587                 kbuf->end_idx = ktr_idx[i];
 588
 589                 /*
 590                  * If we do not have a notion of the beginning index, assume
 591                  * it is entries_per_buf before the ending index.  Don't
 592                  * worry about underflows/negative numbers, the indices will
 593                  * be masked.
 594                  */
 595                 if (kbuf->reset) {
 596                         kbuf->beg_idx = kbuf->end_idx - entries_per_buf + 1;
 597                         kbuf->reset = 0;
 598                 }
 599                 if (kvm_read(kd, (uintptr_t)kbufs[i], ktr_bufs[i].ents,
 600                                 sizeof(struct ktr_entry) * entries_per_buf)
 601                                                                         == -1)
 602                         errx(1, "%s", kvm_geterr(kd));
 603                 kbuf->modified = 1;
 604                 kbuf->beg_idx = earliest_ts(kbuf);
 605         }
 606
 607 }
 608
 609 /*
 610  * Locate the earliest timestamp iterating backwards from end_idx, but
 611  * not going further back then beg_idx.  We have to do this because
 612  * the kernel uses a circulating buffer.
 613  */
 614 static
 615 int
 616 earliest_ts(struct ktr_buffer *buf)
 617 {
 618         struct ktr_entry *save;
 619         int count, scan, i, earliest;
 620
 621         count = 0;
 622         earliest = buf->end_idx - 1;
 623         save = &buf->ents[earliest & fifo_mask];
 624         for (scan = buf->end_idx - 1; scan != buf->beg_idx -1; --scan) {
 625                 i = scan & fifo_mask;
 626                 if (buf->ents[i].ktr_timestamp <= save->ktr_timestamp &&
 627                     buf->ents[i].ktr_timestamp > 0)
 628                         earliest = scan;
 629                 /*
 630                  * We may have gotten so far behind that beg_idx wrapped
 631                  * more then once around the buffer.  Just stop
 632                  */
 633                 if (++count == entries_per_buf)
 634                         break;
 635         }
 636         return earliest;
 637 }
 638
 639 static
 640 void
 641 print_buf(FILE *fo, struct ktr_buffer *ktr_bufs, int cpu,
 642           u_int64_t *last_timestamp)
 643 {
 644         struct ktr_buffer *buf = ktr_bufs + cpu;
 645
 646         if (buf->modified == 0)
 647                 return;
 648         if (*last_timestamp == 0) {
 649                 *last_timestamp =
 650                         buf->ents[buf->beg_idx & fifo_mask].ktr_timestamp;
 651         }
 652         while (buf->beg_idx != buf->end_idx) {
 653                 print_header(fo, buf->beg_idx);
 654                 print_entry(fo, cpu, buf->beg_idx,
 655                             &buf->ents[buf->beg_idx & fifo_mask],
 656                             last_timestamp);
 657                 ++buf->beg_idx;
 658         }
 659         buf->modified = 0;
 660 }
 661
 662 static
 663 void
 664 print_bufs_timesorted(FILE *fo, struct ktr_buffer *ktr_bufs,
 665                       u_int64_t *last_timestamp)
 666 {
 667         struct ktr_entry *ent;
 668         struct ktr_buffer *buf;
 669         int n, bestn;
 670         u_int64_t ts;
 671         static int row = 0;
 672
 673         for (;;) {
 674                 ts = 0;
 675                 bestn = -1;
 676                 for (n = 0; n < ncpus; ++n) {
 677                         buf = ktr_bufs + n;
 678                         if (buf->beg_idx == buf->end_idx)
 679                                 continue;
 680                         ent = &buf->ents[buf->beg_idx & fifo_mask];
 681                         if (ts == 0 || (ts >= ent->ktr_timestamp)) {
 682                                 ts = ent->ktr_timestamp;
 683                                 bestn = n;
 684                         }
 685                 }
 686                 if ((bestn < 0) || (ts < *last_timestamp))
 687                         break;
 688                 buf = ktr_bufs + bestn;
 689                 print_header(fo, row);
 690                 print_entry(fo, bestn, row,
 691                             &buf->ents[buf->beg_idx & fifo_mask],
 692                             last_timestamp);
 693                 ++buf->beg_idx;
 694                 *last_timestamp = ts;
 695                 ++row;
 696         }
 697 }
 698
 699 static void
 700 usage(void)
 701 {
 702         fprintf(stderr, "usage: ktrdump [-acfilnpqrstx] [-A factor] [-N execfile] "
 703                         "[-M corefile] [-o outfile]\n");
 704         exit(1);
 705 }