From cbeb73b99d7bc3acc24b167c7dcee1be03977db2 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Tue, 6 Sep 2005 22:33:00 +0000 Subject: [PATCH] Rework and expand the algorithms in JSCAN, part 3/?. * Have the debug dump print out raw record transaction id's. * Fix a number of bugs in the prefix set code (the files were being strtoul'd in base 10 rather then base 16). * Make multiple commands on the command line work (e.g. -w and -m together). * Fix a few jseek() bugs. * Add '-f' which operates like tail -f and allows a batch operation on e.g. a prefix file set to poll for more data. Still TODO: ACK protocol, raw output (-o/-O) protocol, temporary mode for record files (-W), and many other things. --- sbin/jscan/dump_debug.c | 5 +- sbin/jscan/jfile.c | 101 ++++++++++++++++++++++++++++++++++------ sbin/jscan/jscan.8 | 12 ++++- sbin/jscan/jscan.c | 29 ++++++++++-- sbin/jscan/jscan.h | 3 +- sbin/jscan/jstream.c | 4 +- 6 files changed, 132 insertions(+), 22 deletions(-) diff --git a/sbin/jscan/dump_debug.c b/sbin/jscan/dump_debug.c index 1719dcdbf4..a1a65fd9ff 100644 --- a/sbin/jscan/dump_debug.c +++ b/sbin/jscan/dump_debug.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/jscan/dump_debug.c,v 1.5 2005/09/06 18:43:52 dillon Exp $ + * $DragonFly: src/sbin/jscan/dump_debug.c,v 1.6 2005/09/06 22:33:00 dillon Exp $ */ #include "jscan.h" @@ -65,7 +65,8 @@ dump_debug_stream(struct jstream *js) jsread(js, 0, &head, sizeof(head)); sid = head.streamid & JREC_STREAMID_MASK; - printf("STREAM %04x {\n", (int)(u_int16_t)head.streamid); + printf("STREAM %04x %016llx {\n", (int)(u_int16_t)head.streamid, + head.transid); if (sid >= JREC_STREAMID_JMIN && sid < JREC_STREAMID_JMAX) { off_t off = sizeof(head); diff --git a/sbin/jscan/jfile.c b/sbin/jscan/jfile.c index 0dc6173cad..fdfde018eb 100644 --- a/sbin/jscan/jfile.c +++ b/sbin/jscan/jfile.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/jscan/jfile.c,v 1.7 2005/09/06 18:43:52 dillon Exp $ + * $DragonFly: src/sbin/jscan/jfile.c,v 1.8 2005/09/06 22:33:00 dillon Exp $ */ #include "jscan.h" @@ -101,7 +101,7 @@ jopen_prefix(const char *prefix, enum jdirection direction, int rw) if (strncmp(den->d_name, basename, baselen) == 0 && den->d_name[baselen] == '.' ) { - seq = strtoul(den->d_name + baselen + 1, &ptr, 10); + seq = strtoul(den->d_name + baselen + 1, &ptr, 16); if (*ptr == 0 && seq != ULONG_MAX) { if (seq_beg == (unsigned int)-1 || seq_beg > seq) seq_beg = seq; @@ -140,7 +140,8 @@ jopen_prefix(const char *prefix, enum jdirection direction, int rw) jf->jf_seq_end = seq_end; jf->jf_open_flags = rw ? (O_RDWR|O_CREAT) : O_RDONLY; jreset(jf, seq_end, JD_BACKWARDS); - fprintf(stderr, "Open prefix set %u-%u\n", seq_beg, seq_end); + if (verbose_opt) + fprintf(stderr, "Open prefix set %08x-%08x\n", seq_beg, seq_end); if (jread(jf, &jd, JD_BACKWARDS) == 0) { jf->jf_last_transid = jd->jd_transid; jfree(jf, jd); @@ -241,10 +242,12 @@ jread(struct jfile *jf, struct jdata **jdp, enum jdirection direction) struct journal_rawrecend tail; struct journal_rawrecend *tailp; struct jdata *jd; + struct stat st; char *filename; int allocsize; int recsize; int search; + int error; int n; /* @@ -280,7 +283,8 @@ top: jf->jf_pos = lseek(jf->jf_fd, 0L, SEEK_END); search = 0; } - fprintf(stderr, "Open %s fd %d\n", filename, jf->jf_fd); + if (verbose_opt > 1) + fprintf(stderr, "Open %s fd %d\n", filename, jf->jf_fd); free(filename); } @@ -296,12 +300,13 @@ top: search = 0; } + error = 0; if (jf->jf_direction == JD_FORWARDS) { /* * Scan the journal forwards. Note that the file pointer might not * be seekable. */ - while (jreadbuf(jf, &head, sizeof(head)) == sizeof(head)) { + while ((error = jreadbuf(jf, &head, sizeof(head))) == sizeof(head)) { if (head.begmagic != JREC_BEGMAGIC) { if (search == 0) jf_warn(jf, "bad beginmagic, searching for new record"); @@ -364,7 +369,7 @@ top: * Scan the journal backwards. Note that jread()'s reverse-seek and * read. The data read will be forward ordered, however. */ - while (jreadbuf(jf, &tail, sizeof(tail)) == sizeof(tail)) { + while ((error = jreadbuf(jf, &tail, sizeof(tail))) == sizeof(tail)) { if (tail.endmagic != JREC_ENDMAGIC) { if (search == 0) jf_warn(jf, "bad endmagic, searching for new record"); @@ -426,23 +431,67 @@ top: /* * If reading in prefix mode and there is no more data, close the * current descriptor, adjust the sequence number, and loop. + * + * If we hit the end of the sequence space and were asked to loop, + * check for the next sequence number and adjust jf_seq_end. Leave + * the current descriptor open so we do not loose track of its seek + * position, and also to catch a race where another jscan may have + * written more data to the current sequence number before rolling + * the next sequence number. */ - if (jf->jf_prefix) { - close(jf->jf_fd); - jf->jf_fd = -1; + if (error == 0 && jf->jf_prefix) { if (jf->jf_direction == JD_FORWARDS) { if (jf->jf_seq < jf->jf_seq_end) { ++jf->jf_seq; + if (verbose_opt) + fprintf(stderr, "jread: roll to seq %08x\n", jf->jf_seq); + if (jf->jf_fd >= 0) { + close(jf->jf_fd); + jf->jf_fd = -1; + } + goto top; + } + if (jmodes & JMODEF_LOOP_FOREVER) { + asprintf(&filename, "%s.%08x", jf->jf_prefix, jf->jf_seq + 1); + if (stat(filename, &st) == 0) { + ++jf->jf_seq_end; + if (verbose_opt) + fprintf(stderr, "jread: roll seq_end to %08x\n", + jf->jf_seq_end); + } else { + sleep(5); + } goto top; } } else { if (jf->jf_seq > jf->jf_seq_beg) { --jf->jf_seq; + if (verbose_opt) + fprintf(stderr, "jread: roll to seq %08x\n", jf->jf_seq); + if (jf->jf_fd >= 0) { + close(jf->jf_fd); + jf->jf_fd = -1; + } goto top; } } } + /* + * If we hit EOF and were asked to loop forever on the input, leave + * the current descriptor open, sleep, and loop. + * + * We have already handled the prefix case. This feature only works + * when doing forward scans and the input is not a pipe. + */ + if (error == 0 && (jmodes & JMODEF_LOOP_FOREVER) && + !(jmodes & JMODEF_INPUT_PIPE) && jf->jf_direction == JD_FORWARDS && + jf->jf_prefix == NULL + ) { + sleep(5); + goto top; + } + /* * Otherwise there are no more records and we are done. */ @@ -557,10 +606,18 @@ jseek(struct jfile *jf, int64_t transid, enum jdirection direction) * we find the file most likely to contain the transaction id. */ if (jf->jf_prefix) { - for (seq = jf->jf_seq_end; seq >= jf->jf_seq_beg; --seq) { + if (verbose_opt > 2) { + fprintf(stderr, "jseek prefix set %s %08x-%08x\n", jf->jf_prefix, + jf->jf_seq_beg, jf->jf_seq_end); + } + for (seq = jf->jf_seq_end; seq != jf->jf_seq_beg - 1; --seq) { jreset(jf, seq, JD_FORWARDS); + if (verbose_opt > 2) + fprintf(stderr, "try seq %08x\n", seq); if (jread(jf, &jd, JD_FORWARDS) == 0) { transid_beg = jd->jd_transid; + if (verbose_opt > 2) + fprintf(stderr, "transid %016llx\n", jd->jd_transid); jfree(jf, jd); if (transid_beg == transid) { jreset(jf, seq, JD_FORWARDS); @@ -570,6 +627,11 @@ jseek(struct jfile *jf, int64_t transid, enum jdirection direction) break; } } + if (seq == jf->jf_seq_beg - 1) { + seq = jf->jf_seq_beg; + } + if (verbose_opt > 1) + fprintf(stderr, "jseek input prefix set to seq %08x\n", seq); } /* @@ -601,8 +663,13 @@ jseek(struct jfile *jf, int64_t transid, enum jdirection direction) if (transid_end < transid) { if (jread(jf, &jd, JD_FORWARDS) == 0) jfree(jf, jd); + break; } } + if (verbose_opt) { + fprintf(stderr, "jseek %s to seq %08x offset 0x%08llx\n", + jf->jf_prefix, jf->jf_seq, jf->jf_pos); + } } /* @@ -633,11 +700,13 @@ jalign(struct jfile *jf) { char dummy[16]; int bytes; + int n; if ((int)jf->jf_pos & 15) { if (jf->jf_direction == JD_FORWARDS) { bytes = 16 - ((int)jf->jf_pos & 15); - jf->jf_pos += jreadbuf(jf, dummy, bytes); + if ((n = jreadbuf(jf, dummy, bytes)) > 0) + jf->jf_pos += n; } else { jf->jf_pos = jf->jf_pos & ~(off_t)15; } @@ -661,8 +730,11 @@ jreadbuf(struct jfile *jf, void *buf, int bytes) if (jf->jf_direction == JD_FORWARDS) { while (ttl != bytes) { n = read(jf->jf_fd, (char *)buf + ttl, bytes - ttl); - if (n <= 0) + if (n <= 0) { + if (n < 0 && ttl == 0) + ttl = -errno; break; + } ttl += n; } } else { @@ -671,8 +743,11 @@ jreadbuf(struct jfile *jf, void *buf, int bytes) lseek(jf->jf_fd, jf->jf_pos, 0); while (ttl != bytes) { n = read(jf->jf_fd, (char *)buf + ttl, bytes - ttl); - if (n <= 0) + if (n <= 0) { + if (n < 0 && ttl == 0) + ttl = -errno; break; + } ttl += n; } } diff --git a/sbin/jscan/jscan.8 b/sbin/jscan/jscan.8 index d669ce5322..f1f889b0c6 100644 --- a/sbin/jscan/jscan.8 +++ b/sbin/jscan/jscan.8 @@ -31,7 +31,7 @@ .\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.\" $DragonFly: src/sbin/jscan/jscan.8,v 1.7 2005/09/06 18:43:52 dillon Exp $ +.\" $DragonFly: src/sbin/jscan/jscan.8,v 1.8 2005/09/06 22:33:00 dillon Exp $ .\" .Dd March 6, 2005 .Dt JSCAN 8 @@ -41,7 +41,7 @@ .Nd journal file processing program .Sh SYNOPSIS .Nm -.Fl 2duvF +.Fl 2dfuvF .Op Fl c Ar count[k,m,g,t] .Op Fl D Ar directory .Op Fl m Ar mirror_transid_file/none @@ -72,6 +72,14 @@ can be. Display the contents of the journaling file or stream in a human readable format on stderr. Note that stdout is used only for .Fl o . +.It Fl f +.Nm +will sleep for 5 seconds and loop when it hits EOF on file or prefix +set input rather then exit. This option is typically used when running +on an input file or prefix set which is live (being written to by +another +.Nm +instance). .It Fl D Ar directory Specify the base directory for the mirroring option. .It Fl m Ar mirror_transid_file/none diff --git a/sbin/jscan/jscan.c b/sbin/jscan/jscan.c index da74a2c432..e27ffa49e2 100644 --- a/sbin/jscan/jscan.c +++ b/sbin/jscan/jscan.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/jscan/jscan.c,v 1.6 2005/09/06 18:43:52 dillon Exp $ + * $DragonFly: src/sbin/jscan/jscan.c,v 1.7 2005/09/06 22:33:00 dillon Exp $ */ #include "jscan.h" @@ -83,7 +83,7 @@ main(int ac, char **av) int error; int ch; - while ((ch = getopt(ac, av, "2dm:o:s:uvw:D:O:W:F")) != -1) { + while ((ch = getopt(ac, av, "2dfm:o:s:uvw:D:O:W:F")) != -1) { switch(ch) { case '2': jmodes |= JMODEF_INPUT_FULL; @@ -113,6 +113,9 @@ main(int ac, char **av) case 'd': jmodes |= JMODEF_DEBUG; break; + case 'f': + jmodes |= JMODEF_LOOP_FOREVER; + break; case 'v': ++verbose_opt; break; @@ -251,6 +254,20 @@ main(int ac, char **av) transid = record_transid; if ((jmodes & JMODEF_TRANSID_GOOD_MASK) == 0) transid = 0; + if (verbose_opt) { + if (jmodes & JMODEF_OUTPUT) { + fprintf(stderr, "Starting transid for OUTPUT: %016llx\n", + output_transid); + } + if (jmodes & JMODEF_MIRROR) { + fprintf(stderr, "Starting transid for MIRROR: %016llx\n", + mirror_transid); + } + if (jmodes & JMODEF_RECORD) { + fprintf(stderr, "Starting transid for RECORD: %016llx\n", + record_transid); + } + } /* * Now it gets more difficult. If we are recording then the input @@ -344,6 +361,11 @@ main(int ac, char **av) exit(error ? 1 : 0); } +/* + * When we have multiple commands and are writing to a prefix set, we can + * 'background' the output and/or mirroring command and have the background + * processes feed off the prefix set the foreground process is writing to. + */ static void fork_subprocess(struct jfile *jftoclose, @@ -355,7 +377,8 @@ fork_subprocess(struct jfile *jftoclose, struct jfile *jf; if ((pid = fork()) == 0) { - jmodes &= ~JMODEF_DEBUG; + jmodes &= ~(JMODEF_DEBUG | JMODEF_INPUT_PIPE); + jmodes |= JMODEF_LOOP_FOREVER; /* keep checking for new input */ jclose(jftoclose); jf = jopen_prefix(input_prefix, jdirection, 0); jmodes |= JMODEF_INPUT_PREFIX; diff --git a/sbin/jscan/jscan.h b/sbin/jscan/jscan.h index 540eb30895..9b0e84ace8 100644 --- a/sbin/jscan/jscan.h +++ b/sbin/jscan/jscan.h @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/jscan/jscan.h,v 1.7 2005/09/06 18:43:52 dillon Exp $ + * $DragonFly: src/sbin/jscan/jscan.h,v 1.8 2005/09/06 22:33:00 dillon Exp $ */ #include @@ -139,6 +139,7 @@ struct jhash { #define JMODEF_OUTPUT 0x00000100 #define JMODEF_OUTPUT_FULL 0x00000200 #define JMODEF_MEMORY_TRACKING 0x00000400 +#define JMODEF_LOOP_FOREVER 0x00000800 #define JMODEF_OUTPUT_TRANSID_GOOD 0x00010000 #define JMODEF_RECORD_TRANSID_GOOD 0x00020000 diff --git a/sbin/jscan/jstream.c b/sbin/jscan/jstream.c index eb4c3bbeab..824982bb0b 100644 --- a/sbin/jscan/jstream.c +++ b/sbin/jscan/jstream.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/jscan/jstream.c,v 1.6 2005/09/06 18:43:52 dillon Exp $ + * $DragonFly: src/sbin/jscan/jstream.c,v 1.7 2005/09/06 22:33:00 dillon Exp $ */ #include "jscan.h" @@ -221,6 +221,8 @@ jsreadp(struct jstream *js, off_t off, const void **bufp, free(js->js_alloc_buf); js->js_alloc_buf = malloc(bytes); js->js_alloc_size = bytes; + if (js->js_alloc_buf == NULL) + fprintf(stderr, "attempt to allocate %d bytes failed\n", bytes); assert(js->js_alloc_buf != NULL); } error = jsread(js, off, js->js_alloc_buf, bytes); -- 2.41.0