kcollect - Implement gnuplot output feature
authorMatthew Dillon <dillon@apollo.backplane.com>
Sat, 29 Jul 2017 23:24:41 +0000 (16:24 -0700)
committerMatthew Dillon <dillon@apollo.backplane.com>
Sun, 30 Jul 2017 00:24:10 +0000 (17:24 -0700)
* Implement the gunplot output feature.  This feature currently
  hard-selects a set of fields (fields cannot be specified).

  Generates two graphs.  The first collects memory statistics
  and machine load.  The second collects cpu utilization and
  fault, syscall, and nlookup (file path resolution) rate.

* In gnuplot output mode, -f will cause the entire dataset to be
  regenerated every 60 seconds (I don't see any way to avoid this
  to update an existing gnuplot window).

* Finish implementing -o fields

usr.bin/kcollect/Makefile
usr.bin/kcollect/gnuplot.c [new file with mode: 0644]
usr.bin/kcollect/kcollect.8
usr.bin/kcollect/kcollect.c
usr.bin/kcollect/kcollect.h [new file with mode: 0644]

index cde0d13..c1e3fb2 100644 (file)
@@ -1,6 +1,7 @@
 
 PROG=  kcollect
-MAN= kcollect.8
+SRCS=  kcollect.c gnuplot.c
+MAN=   kcollect.8
 
 DPADD= ${LIBUTIL}
 LDADD= -lutil
diff --git a/usr.bin/kcollect/gnuplot.c b/usr.bin/kcollect/gnuplot.c
new file mode 100644 (file)
index 0000000..62c0ed8
--- /dev/null
@@ -0,0 +1,205 @@
+/*
+ * Copyright (c) 2017 The DragonFly Project.  All rights reserved.
+ *
+ * This code is derived from software contributed to The DragonFly Project
+ * by Matthew Dillon <dillon@backplane.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "kcollect.h"
+
+void
+start_gnuplot(int ac __unused, char **av __unused)
+{
+       OutFP = popen("gnuplot", "w");
+       if (OutFP == NULL) {
+               fprintf(stderr, "can't find gnuplot\n");
+               exit(1);
+       }
+}
+
+void
+dump_gnuplot(kcollect_t *ary, size_t count, const char *plotfile)
+{
+       int plot1[] = { KCOLLECT_MEMWIR, KCOLLECT_MEMACT,
+                       KCOLLECT_MEMINA, KCOLLECT_MEMCAC,
+                       KCOLLECT_MEMFRE, KCOLLECT_LOAD };
+       int plot2[] = { KCOLLECT_USERPCT, KCOLLECT_SYSTPCT,
+                       KCOLLECT_INTRPCT, KCOLLECT_IDLEPCT,
+                       KCOLLECT_VMFAULT, KCOLLECT_SYSCALLS, KCOLLECT_NLOOKUP };
+       struct tm *tmv;
+       char buf[64];
+       uint64_t value;
+       time_t t;
+       double dv;
+       double smoothed_dv;
+       int i;
+       int j;
+       int jj;
+       int k;
+
+       /*
+        * If plotfile is specified allow .jpg or .JPG or .png or .PNG
+        */
+       if (plotfile) {
+               const char *ext;
+
+               if ((ext = strrchr(plotfile, '.')) == NULL) {
+                       ext = "";
+               } else {
+                       ++ext;
+               }
+               if (strcmp(ext, "jpg") == 0 ||
+                   strcmp(ext, "JPG") == 0) {
+                       fprintf(OutFP, "set terminal jpeg size %d,%d\n",
+                               OutputWidth, OutputHeight);
+               } else if (strcmp(ext, "png") == 0 ||
+                          strcmp(ext, "PNG") == 0) {
+                       fprintf(OutFP, "set terminal png size %d,%d\n",
+                               OutputWidth, OutputHeight);
+               } else {
+                       fprintf(stderr, "plotfile must be .jpg or .png\n");
+                       exit(1);
+               }
+               fprintf(OutFP, "set output \"%s\"\n", plotfile);
+       } else {
+               fprintf(OutFP, "set terminal x11 persist size %d,%d\n",
+                       OutputWidth, OutputHeight);
+       }
+
+       /*
+        * NOTE: be sure to reset any fields adjusted by the second plot,
+        *       in case we are streaming plots with -f.
+        */
+       fprintf(OutFP, "set xdata time\n");
+       fprintf(OutFP, "set timefmt \"%%d-%%b-%%Y %%H:%%M:%%S\"\n");
+       fprintf(OutFP, "set style fill solid 1.0\n");
+       fprintf(OutFP, "set multiplot layout 2,1\n");
+       fprintf(OutFP, "set key outside\n");
+       fprintf(OutFP, "set lmargin 10\n");
+       fprintf(OutFP, "set rmargin 25\n");
+       fprintf(OutFP, "set xtics rotate\n");
+       fprintf(OutFP, "set format x '%%H:%%M'\n");
+
+       fprintf(OutFP, "set ylabel \"GB\"\n");
+
+       fprintf(OutFP, "set yrange [0:%d]\n",
+               (int)((KCOLLECT_GETSCALE(ary[0].data[KCOLLECT_MEMFRE]) +
+                      999999) / 1000000000));
+       fprintf(OutFP, "set autoscale y2\n");
+       fprintf(OutFP, "set y2label \"Load\"\n");
+       fprintf(OutFP, "set ytics nomirror\n");
+       fprintf(OutFP, "set y2tics nomirror\n");
+
+       fprintf(OutFP,
+               "plot "
+               "\"-\" using 1:3 title \"wired\" with boxes lw 1, "
+               "\"-\" using 1:3 title \"active\" with boxes lw 1, "
+               "\"-\" using 1:3 title \"inact\" with boxes lw 1, "
+               "\"-\" using 1:3 title \"cache\" with boxes lw 1, "
+               "\"-\" using 1:3 title \"free\" with boxes lw 1, "
+               "\"-\" using 1:3 axes x1y2 title \"load\" with lines lw 1\n");
+
+       for (jj = 0; jj < (int)(sizeof(plot1) / sizeof(plot1[0])); ++jj) {
+               j = plot1[jj];
+
+               for (i = count - 1; i >= 2; --i) {
+                       /*
+                        * Timestamp
+                        */
+                       t = ary[i].realtime.tv_sec;
+                       if (t < 1000)
+                               continue;
+                       if (UseGMT)
+                               tmv = gmtime(&t);
+                       else
+                               tmv = localtime(&t);
+                       strftime(buf, sizeof(buf), "%d-%b-%Y %H:%M:%S", tmv);
+                       value = ary[i].data[j];
+                       if (jj <= 4) {
+                               for (k = jj + 1; k <= 4; ++k)
+                                       value += ary[i].data[plot1[k]];
+                               dv = (double)value / 1e9;
+                       } else {
+                               dv = (double)value / 100.0;
+                       }
+                       fprintf(OutFP, "%s %6.2f\n", buf, dv);
+               }
+               fprintf(OutFP, "e\n");
+       }
+
+       fprintf(OutFP, "set ylabel \"Cpu Utilization\"\n");
+       fprintf(OutFP, "set y2label \"MOps/sec (smoothed)\"\n");
+       fprintf(OutFP, "set ytics nomirror\n");
+       fprintf(OutFP, "set y2tics nomirror\n");
+       fprintf(OutFP, "set yrange [0:105]\n");
+       fprintf(OutFP, "set y2range [0:1.0]\n");
+
+       fprintf(OutFP,
+               "plot "
+               "\"-\" using 1:3 title \"user\" with boxes lw 1, "
+               "\"-\" using 1:3 title \"system\" with boxes lw 1, "
+               "\"-\" using 1:3 title \"intr\" with boxes lw 1, "
+               "\"-\" using 1:3 title \"idle\" with boxes lw 1, "
+               "\"-\" using 1:3 axes x1y2 title \"faults\" with lines lw 1, "
+               "\"-\" using 1:3 axes x1y2 title \"syscalls\" with lines lw 1, "
+               "\"-\" using 1:3 axes x1y2 title \"nlookup\" with lines lw 1\n"
+       );
+
+       for (jj = 0; jj < (int)(sizeof(plot2) / sizeof(plot2[0])); ++jj) {
+               j = plot2[jj];
+
+               smoothed_dv = 0.0;
+               for (i = count - 1; i >= 2; --i) {
+                       /*
+                        * Timestamp
+                        */
+                       t = ary[i].realtime.tv_sec;
+                       if (t < 1000)
+                               continue;
+                       if (UseGMT)
+                               tmv = gmtime(&t);
+                       else
+                               tmv = localtime(&t);
+                       strftime(buf, sizeof(buf), "%d-%b-%Y %H:%M:%S", tmv);
+                       value = ary[i].data[j];
+
+                       if (jj <= 3) {
+                               for (k = jj + 1; k <= 3; ++k)
+                                       value += ary[i].data[plot2[k]];
+                               dv = (double)value / 100.0;
+                       } else {
+                               dv = (double)value / KCOLLECT_INTERVAL;
+                               dv = dv / 1e6;
+                               smoothed_dv = (smoothed_dv * 9.0 + dv) / 10.0;
+                               dv = smoothed_dv;
+                       }
+                       fprintf(OutFP, "%s %6.2f\n", buf, dv);
+               }
+               fprintf(OutFP, "e\n");
+       }
+       fflush(OutFP);
+}
index 7df3d40..b449af5 100644 (file)
@@ -41,6 +41,9 @@
 .Op Fl l
 .Op Fl g
 .Op Fl G
+.Op Fl W Ar width
+.Op Fl H Ar height
+.Op Fl w Ar plotfile
 .Op Fl x Ar gnuplot_options
 .Sh DESCRIPTION
 The
@@ -55,31 +58,45 @@ The following options are available:
 .Bl -tag -width indent
 .It Fl o Ar fields
 Indicate which fields to extract.  If not specified, all available
-fields are extracted.  Unknown fields are ignored.
+fields are extracted.  Unknown fields are ignored.  Note that not
+all commands will filter based on the field list.
+.Pp
+You may specify space or comma-separated fieldnames.  Whitespace is
+ignored.  Use the
+.Fl l
+option to get a list of available fields.
 .It Fl b Ar file
 Dump the data to a dbm database, creating the database if necessary.
 Data is indexed by gmt time, which will be calculated from the ticks.
-Any existing data records for the specified time will be overwritten,
-but
+Duplicate tuples are ignored.  The database will retain data from prior
+runs.
+.Pp
+THIS IS A TODO OPTION, NOT YET IMPLEMENTED.
 .Nm
 is smart and will not overwrite the record if it exactly matches what
 would be written.
 .It Fl g
 Generate gnuplot output for the data.
-NOT YET - UNDER CONSTRUCTION.
 .It Fl x
 Generate gnuplot output for the data and pipe it to gnuplot for display
 to X11.
-Any options specified after this option will be passed into gnuplot.
-NOT YET - UNDER CONSTRUCTION.
 .It Fl l
 List all available fields and the field format
 .It Fl f
 Dump available output then enter a 60-second sleep/collection loop
 to incrementally collect and output more data.
-If outputting to gnuplot, the plot will be updated regularly.
+If outputting to gnuplot, the plot will be updated regularly.  However,
+please note that this is fairly expensive since the plot data has to
+be completely re-sent to gunplot on each update.
+.It Fl w Ar plotfile
+Generate gnuplot output to a .png or .jpg file, depending on the extension
+of the filename you supply.
 .It Fl G
 Timestamps for text output will be in GMT instead of localtime.
+.It Fl W Ar width
+.It Fl H Ar height
+Set the width and height of the output plot for the plotfile or for X.
+The default is 512x512.
 .El
 .Pp
 .Sh SEE ALSO
index d3f5c36..9730cb2 100644 (file)
  * SUCH DAMAGE.
  */
 
-#include <sys/types.h>
-#include <sys/sysctl.h>
-#include <sys/kcollect.h>
-#include <time.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <time.h>
-#include <unistd.h>
-#include <string.h>
-#include <libutil.h>
+#include "kcollect.h"
 
 #define SLEEP_INTERVAL 60      /* minimum is KCOLLECT_INTERVAL */
 
 static void dump_text(kcollect_t *ary, size_t count, size_t total_count);
-static void dump_gnuplot(kcollect_t *ary, size_t count);
 static void dump_dbm(kcollect_t *ary, size_t count, const char *datafile);
 static void dump_fields(kcollect_t *ary);
-static void start_gnuplot(int ac, char **av);
+static void adjust_fields(kcollect_t *ent, const char *fields);
 
-static const char *Fields = NULL;
-static int UseGmt = 0;
+FILE *OutFP;
+int UseGMT;
+int OutputWidth = 1024;
+int OutputHeight = 1024;
 
 int
 main(int ac, char **av)
@@ -59,22 +51,25 @@ main(int ac, char **av)
        size_t count;
        size_t total_count;
        const char *datafile = NULL;
+       const char *fields = NULL;
        int cmd = 't';
        int ch;
        int keepalive = 0;
        int last_ticks;
        int loops = 0;
 
+       OutFP = stdout;
+
        sysctlbyname("kern.collect_data", NULL, &bytes, NULL, 0);
        if (bytes == 0) {
                fprintf(stderr, "kern.collect_data not available\n");
                exit(1);
        }
 
-       while ((ch = getopt(ac, av, "o:b:flgx")) != -1) {
+       while ((ch = getopt(ac, av, "o:b:flgxw:GW:H:")) != -1) {
                switch(ch) {
                case 'o':
-                       Fields = optarg;
+                       fields = optarg;
                        break;
                case 'b':
                        datafile = optarg;
@@ -86,6 +81,10 @@ main(int ac, char **av)
                case 'l':
                        cmd = 'l';
                        break;
+               case 'w':
+                       datafile = optarg;
+                       cmd = 'w';
+                       break;
                case 'g':
                        cmd = 'g';
                        break;
@@ -93,7 +92,13 @@ main(int ac, char **av)
                        cmd = 'x';
                        break;
                case 'G':
-                       UseGmt = 1;
+                       UseGMT = 1;
+                       break;
+               case 'W':
+                       OutputWidth = strtol(optarg, NULL, 0);
+                       break;
+               case 'H':
+                       OutputHeight = strtol(optarg, NULL, 0);
                        break;
                default:
                        fprintf(stderr, "Unknown option %c\n", ch);
@@ -108,9 +113,11 @@ main(int ac, char **av)
        }
 
        total_count = 0;
+       last_ticks = 0;
 
-       if (cmd == 'x')
+       if (cmd == 'x' || cmd == 'w')
                start_gnuplot(ac - optind, av + optind);
+
        do {
                /*
                 * Snarf as much data as we can.  If we are looping,
@@ -133,6 +140,9 @@ main(int ac, char **av)
                ary = malloc(bytes);
                sysctlbyname("kern.collect_data", ary, &bytes, NULL, 0);
 
+               if (fields)
+                       adjust_fields(&ary[1], fields);
+
                count = bytes / sizeof(kcollect_t);
                if (loops) {
                        while (count > 2) {
@@ -157,26 +167,51 @@ main(int ac, char **av)
                        break;          /* NOT REACHED */
                case 'g':
                        if (count > 2)
-                               dump_gnuplot(ary, count);
+                               dump_gnuplot(ary, count, NULL);
+                       break;
+               case 'w':
+                       if (count >= 2)
+                               dump_gnuplot(ary, count, datafile);
                        break;
                case 'x':
                        if (count > 2)
-                               dump_gnuplot(ary, count);
+                               dump_gnuplot(ary, count, NULL);
                        break;
                }
                if (keepalive) {
+                       fflush(OutFP);
                        fflush(stdout);
-                       sleep(1);
+                       switch(cmd) {
+                       case 't':
+                               sleep(1);
+                               break;
+                       case 'x':
+                       case 'g':
+                       case 'w':
+                               sleep(60);
+                               break;
+                       default:
+                               sleep(10);
+                               break;
+                       }
                }
                last_ticks = ary[2].ticks;
                if (count >= 2)
                        total_count += count - 2;
-               ++loops;
+
+               /*
+                * Loop for incremental aquisition.  When outputting to
+                * gunplot, we have to send the whole data-set again so
+                * do not increment loops in that case.
+                */
+               if (cmd != 'g' && cmd != 'x' && cmd != 'w')
+                       ++loops;
+
                free(ary);
        } while (keepalive);
 
        if (cmd == 'x')
-               pclose(stdout);
+               pclose(OutFP);
 }
 
 static
@@ -208,7 +243,7 @@ dump_text(kcollect_t *ary, size_t count, size_t total_count)
                 * Timestamp
                 */
                t = ary[i].realtime.tv_sec;
-               if (UseGmt)
+               if (UseGMT)
                        tmv = gmtime(&t);
                else
                        tmv = localtime(&t);
@@ -299,21 +334,11 @@ dump_text(kcollect_t *ary, size_t count, size_t total_count)
        }
 }
 
-static void
-dump_gnuplot(kcollect_t *ary __unused, size_t count __unused)
-{
-}
-
 static void
 dump_dbm(kcollect_t *ary __unused, size_t count __unused, const char *datafile __unused)
 {
 }
 
-static void
-start_gnuplot(int ac __unused, char **av __unused)
-{
-}
-
 static void
 dump_fields(kcollect_t *ary)
 {
@@ -327,3 +352,30 @@ dump_fields(kcollect_t *ary)
                       KCOLLECT_GETFMT(ary[0].data[j]));
        }
 }
+
+static void
+adjust_fields(kcollect_t *ent, const char *fields)
+{
+       char *copy = strdup(fields);
+       char *word;
+       int selected[KCOLLECT_ENTRIES];
+       int j;
+
+       bzero(selected, sizeof(selected));
+
+       word = strtok(copy, ", \t");
+       while (word) {
+               for (j = 0; j < KCOLLECT_ENTRIES; ++j) {
+                       if (strncmp(word, (char *)&ent->data[j], 8) == 0) {
+                               selected[j] = 1;
+                               break;
+                       }
+               }
+               word = strtok(NULL, ", \t");
+       }
+       free(copy);
+       for (j = 0; j < KCOLLECT_ENTRIES; ++j) {
+               if (!selected[j])
+                       ent->data[j] = 0;
+       }
+}
diff --git a/usr.bin/kcollect/kcollect.h b/usr.bin/kcollect/kcollect.h
new file mode 100644 (file)
index 0000000..c0faeba
--- /dev/null
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2017 The DragonFly Project.  All rights reserved.
+ *
+ * This code is derived from software contributed to The DragonFly Project
+ * by Matthew Dillon <dillon@backplane.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <sys/kcollect.h>
+#include <time.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <unistd.h>
+#include <string.h>
+#include <libutil.h>
+
+extern FILE *OutFP;
+extern int UseGMT;
+extern int OutputWidth;
+extern int OutputHeight;
+
+void start_gnuplot(int ac, char **av);
+void dump_gnuplot(kcollect_t *ary, size_t count, const char *plotfile);