From c2b2044a8a1cfb35ebaa5e7e2a9decc93fc24f75 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Tue, 6 Sep 2005 06:42:44 +0000 Subject: [PATCH] Rework and expand the algorithms in JSCAN, part 1/2. Implement a new infrastructure in the userland jscan program to create a framework for supporting the following: * Pipe-throughs (without storing data in a buffering file). * File-based buffering * Streamed replication of journaled data. * Sequenced output files limited to a specific size (to match against backup media) * Transaction id tracking to support journal restarts and resynchronization. This is needed to make off-site and networked backups reliable. * Decoupled journal operations. e.g. (once finished) we will be able to run a jscan in realtime to record journal data, and another one in batch to take that data and use it to maintain a mirror or to send it to another machine in a restartable and resynchronizable manner. * Infrastructure to be able to scan a journal forwards or backwards by an arbitrary number of raw records and maintain a mirror in both the forwards (redo) and backwards (undo) direction based on that ability. * Infrastructure that will aid in the preferred backup methodology of maintaining a real time mirror and UNDO/REDO journal on a 100% full backup partition such that one is able to free space on the partition simply by removing the oldest journaling file. * Infrastructure that will make it easier to solve the overlapping meta-transaction problem illustrated below: current synchronization point V [ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ] <= Raw records A <= meta transaction A B B B B <= meta transaction B C C C <= meta transaction C D D <= meta transaction D E E <= meta transaction E As you can see, the point at which a journal is 'broken' could be right smack in the middle of several meta transactions at once, due to the fact that meta transactions are laid out as their records can be generated and multiple processes could be modifying the filesystem at the same time. If we are attempting to mirror this data, only transactions A and E will have been completed and issued to the mirror. If the journal is broken at the sychnronization point and then restarted, in order to pick up where we left off we have to scan existing journal records all the way back to the first 'B' in order to recover transactions B and C, then continue recording the journal from the synchronization point until those transactions are complete and can be issued to the mirror. The difficulty here is that we have to carefully track exactly what needs to be synchronized. In recovering transaction 'B' and 'C' we do not want to reissue transaction E, for example, which had already been completed prior to the break. The problem exists in both the forwards and reverse direction, and we need to be able to go in both directions to control the 'snapshot' view the mirror is presenting to us. NOTE: this is just part 1/2. This code and the new options are NON WORKING at the moment. --- sbin/jscan/Makefile | 5 +- sbin/jscan/dump_debug.c | 10 +- sbin/jscan/dump_mirror.c | 72 +---- sbin/jscan/dump_output.c | 43 +++ sbin/jscan/dump_record.c | 43 +++ sbin/jscan/jfile.c | 614 ++++++++++++++++++++++++++++++++++----- sbin/jscan/jscan.8 | 237 ++++++++++++++- sbin/jscan/jscan.c | 377 ++++++++++++++++++++---- sbin/jscan/jscan.h | 93 ++++-- sbin/jscan/jstream.c | 195 ++----------- sbin/jscan/subs.c | 20 +- sys/sys/journal.h | 4 +- 12 files changed, 1294 insertions(+), 419 deletions(-) create mode 100644 sbin/jscan/dump_output.c create mode 100644 sbin/jscan/dump_record.c diff --git a/sbin/jscan/Makefile b/sbin/jscan/Makefile index 404dcf4c90..66d11b3f83 100644 --- a/sbin/jscan/Makefile +++ b/sbin/jscan/Makefile @@ -1,8 +1,9 @@ # -# $DragonFly: src/sbin/jscan/Makefile,v 1.2 2005/07/05 00:26:03 dillon Exp $ +# $DragonFly: src/sbin/jscan/Makefile,v 1.3 2005/09/06 06:42:44 dillon Exp $ PROG= jscan -SRCS= jscan.c jfile.c jstream.c subs.c dump_debug.c dump_mirror.c +SRCS= jscan.c jfile.c jstream.c subs.c dump_debug.c dump_mirror.c \ + dump_record.c dump_output.c MAN= jscan.8 WARNS?= 6 diff --git a/sbin/jscan/dump_debug.c b/sbin/jscan/dump_debug.c index 4cfca45794..81370a0401 100644 --- a/sbin/jscan/dump_debug.c +++ b/sbin/jscan/dump_debug.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/jscan/dump_debug.c,v 1.3 2005/07/05 00:26:03 dillon Exp $ + * $DragonFly: src/sbin/jscan/dump_debug.c,v 1.4 2005/09/06 06:42:44 dillon Exp $ */ #include "jscan.h" @@ -39,13 +39,15 @@ static void dump_debug_stream(struct jstream *js); static int dump_debug_subrecord(struct jstream *js, off_t *off, off_t recsize, int level); +static int dump_debug_payload(int16_t rectype, struct jstream *js, off_t off, + int recsize, int level); void -dump_debug(struct jfile *jf) +dump_debug(struct jfile *jf, struct jdata *jd, int64_t transid) { struct jstream *js; - while ((js = jscan_stream(jf)) != NULL) { + if ((js = jaddrecord(jf, jd)) != NULL) { dump_debug_stream(js); jscan_dispose(js); } @@ -152,7 +154,7 @@ dump_debug_subrecord(struct jstream *js, off_t *off, off_t recsize, int level) return(error); } -int +static int dump_debug_payload(int16_t rectype, struct jstream *js, off_t off, int recsize, int level) { diff --git a/sbin/jscan/dump_mirror.c b/sbin/jscan/dump_mirror.c index 59f1a1885a..d3afca9a6a 100644 --- a/sbin/jscan/dump_mirror.c +++ b/sbin/jscan/dump_mirror.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/jscan/dump_mirror.c,v 1.4 2005/07/05 06:20:07 dillon Exp $ + * $DragonFly: src/sbin/jscan/dump_mirror.c,v 1.5 2005/09/06 06:42:44 dillon Exp $ */ #include "jscan.h" @@ -46,11 +46,11 @@ static int dump_mirror_payload(int16_t rectype, struct jstream *js, off_t off, static int dump_mirror_rebuild(u_int16_t rectype, struct jstream *js, struct jattr *jattr); void -dump_mirror(struct jfile *jf) +dump_mirror(struct jfile *jf, struct jdata *jd, int64_t transid) { struct jstream *js; - while ((js = jscan_stream(jf)) != NULL) { + if ((js = jaddrecord(jf, jd)) != NULL) { dump_mirror_stream(js); jscan_dispose(js); } @@ -67,10 +67,6 @@ dump_mirror_stream(struct jstream *js) jsread(js, 0, &head, sizeof(head)); sid = head.streamid & JREC_STREAMID_MASK; - if (debug_opt) { - printf("STREAM %04x DATA (%lld) {\n", - (int)(u_int16_t)head.streamid, js->js_normalized_total); - } if (sid >= JREC_STREAMID_JMIN && sid < JREC_STREAMID_JMAX) { off_t off = sizeof(head); dump_mirror_toprecord(js, &off, js->js_normalized_total - @@ -79,32 +75,18 @@ dump_mirror_stream(struct jstream *js) } else { switch(head.streamid & JREC_STREAMID_MASK) { case JREC_STREAMID_SYNCPT & JREC_STREAMID_MASK: - if (debug_opt) - printf(" SYNCPT\n"); break; case JREC_STREAMID_PAD & JREC_STREAMID_MASK: - if (debug_opt) - printf(" PAD\n"); break; case JREC_STREAMID_DISCONT & JREC_STREAMID_MASK: - if (debug_opt) - printf(" DISCONT\n"); break; case JREC_STREAMID_ANNOTATE & JREC_STREAMID_MASK: - if (debug_opt) - printf(" ANNOTATION\n"); break; default: - if (debug_opt) - printf(" UNKNOWN\n"); break; } } umask(save_umask); - if (debug_opt) { - printf("}\n"); - fflush(stdout); - } } static int @@ -124,12 +106,6 @@ dump_mirror_toprecord(struct jstream *js, off_t *off, off_t recsize, int level) while (recsize > 0) { if ((error = jsread(js, base, &sub, sizeof(sub))) != 0) break; - if (debug_opt) { - printf("%*.*s", level * 4, level * 4, ""); - printf("@%lld ", base); - printf("RECORD %s [%04x/%d]", type_to_name(sub.rectype), - (int)(u_int16_t)sub.rectype, sub.recsize); - } if (sub.recsize == -1) { if ((sub.rectype & JMASK_NESTED) == 0) { printf("Record size of -1 only works for nested records\n"); @@ -143,30 +119,13 @@ dump_mirror_toprecord(struct jstream *js, off_t *off, off_t recsize, int level) subsize = (sub.recsize + 7) & ~7; } if (sub.rectype & JMASK_NESTED) { - if (debug_opt) - printf(" {\n"); *off = base + sizeof(sub); error = dump_mirror_subrecord(js, off, payload, level + 1, &jattr); - if (debug_opt) - printf("%*.*s}\n", level * 4, level * 4, ""); } else if (sub.rectype & JMASK_SUBRECORD) { - if (debug_opt) { - printf(" DATA (%d)", payload); - error = dump_debug_payload(sub.rectype, js, base + sizeof(sub), payload, level); - } *off = base + sizeof(sub) + payload; - if (debug_opt) - printf("\n"); } else if ((sub.rectype & JTYPE_MASK) == JLEAF_PAD) { - if (debug_opt) { - if (payload) - printf(" DATA (%d)", payload); - printf("\n"); - } } else { - if (debug_opt) - printf("[%d bytes of unknown content]\n", payload); } dump_mirror_rebuild(sub.rectype, js, &jattr); jattr_reset(&jattr); @@ -209,12 +168,6 @@ dump_mirror_subrecord(struct jstream *js, off_t *off, off_t recsize, int level, if ((error = jsread(js, base, &sub, sizeof(sub))) != 0) break; rectype = sub.rectype & JTYPE_MASK; - if (debug_opt) { - printf("%*.*s", level * 4, level * 4, ""); - printf("@%lld ", base); - printf("SRECORD %s [%04x/%d]", type_to_name(sub.rectype), - (int)(u_int16_t)sub.rectype, sub.recsize); - } if (sub.recsize == -1) { payload = 0x7FFFFFFF; subsize = 0x7FFFFFFF; @@ -223,9 +176,6 @@ dump_mirror_subrecord(struct jstream *js, off_t *off, off_t recsize, int level, subsize = (sub.recsize + 7) & ~7; } if (sub.rectype & JMASK_NESTED) { - if (debug_opt) - printf(" {\n"); - /* * Only recurse through vattr records. XXX currently assuming * only on VATTR subrecord. @@ -238,28 +188,12 @@ dump_mirror_subrecord(struct jstream *js, off_t *off, off_t recsize, int level, error = dump_mirror_subrecord(js, off, payload, level + 1, NULL); } - if (debug_opt) - printf("%*.*s}\n", level * 4, level * 4, ""); } else if (sub.rectype & JMASK_SUBRECORD) { - if (debug_opt) { - printf(" DATA (%d)", payload); - dump_debug_payload(sub.rectype, js, base + sizeof(sub), - payload, level); - } error = dump_mirror_payload(sub.rectype, js, base + sizeof(sub), payload, level, jattr); *off = base + sizeof(sub) + payload; - if (debug_opt) - printf("\n"); } else if ((sub.rectype & JTYPE_MASK) == JLEAF_PAD) { - if (debug_opt) { - if (payload) - printf(" DATA (%d)", payload); - printf("\n"); - } } else { - if (debug_opt) - printf("[%d bytes of unknown content]\n", sub.recsize); } if (error) break; diff --git a/sbin/jscan/dump_output.c b/sbin/jscan/dump_output.c new file mode 100644 index 0000000000..76eb35974d --- /dev/null +++ b/sbin/jscan/dump_output.c @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2005 The DragonFly Project. All rights reserved. + * + * This code is derived from software contributed to The DragonFly Project + * by Matthew Dillon + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of The DragonFly Project nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific, prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $DragonFly: src/sbin/jscan/dump_output.c,v 1.1 2005/09/06 06:42:44 dillon Exp $ + */ + +#include "jscan.h" + +void +dump_output(struct jfile *jf, struct jdata *jd, int64_t transid) +{ + fprintf(stderr, "OUTPUT_DUMP\n"); +} diff --git a/sbin/jscan/dump_record.c b/sbin/jscan/dump_record.c new file mode 100644 index 0000000000..d35645bc66 --- /dev/null +++ b/sbin/jscan/dump_record.c @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2005 The DragonFly Project. All rights reserved. + * + * This code is derived from software contributed to The DragonFly Project + * by Matthew Dillon + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of The DragonFly Project nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific, prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $DragonFly: src/sbin/jscan/dump_record.c,v 1.1 2005/09/06 06:42:44 dillon Exp $ + */ + +#include "jscan.h" + +void +dump_record(struct jfile *jf, struct jdata *jd, int64_t transid) +{ + fprintf(stderr, "JRECORD_DUMP\n"); +} diff --git a/sbin/jscan/jfile.c b/sbin/jscan/jfile.c index 9154c27515..58cebe2d99 100644 --- a/sbin/jscan/jfile.c +++ b/sbin/jscan/jfile.c @@ -31,144 +31,606 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/jscan/jfile.c,v 1.5 2005/07/06 06:06:44 dillon Exp $ + * $DragonFly: src/sbin/jscan/jfile.c,v 1.6 2005/09/06 06:42:44 dillon Exp $ */ #include "jscan.h" +#include + +static void jalign(struct jfile *jf); +static int jreadbuf(struct jfile *jf, void *buf, int bytes); /* - * Open a journal for directional scanning + * Open a file descriptor for journal record access. + * + * NOTE: only seekable descriptors are supported for backwards scans. */ struct jfile * -jopen_stream(const char *path, enum jdirection jdir, int flags) +jopen_fd(int fd, enum jdirection direction) { - FILE *fp; struct jfile *jf; - if ((fp = fopen(path, "r")) == NULL) - return (NULL); - if ((jf = jopen_fp(fp, jdir, flags)) == NULL) - fclose (fp); + jf = malloc(sizeof(struct jfile)); + bzero(jf, sizeof(struct jfile)); + jf->jf_fd = fd; + jf->jf_open_flags = O_RDONLY; + if (direction == JD_BACKWARDS) { + jf->jf_pos = lseek(jf->jf_fd, 0L, SEEK_END); + } + jf->jf_direction = direction; return(jf); } +/* + * Open a prefix set. .nnnnnnnnn files or a .transid file + * must exist to succeed. No file descriptor is actually opened but + * the sequence number is initialized to the beginning or end of the set. + */ struct jfile * -jopen_fp(FILE *fp, enum jdirection jdir, int flags) +jopen_prefix(const char *prefix, enum jdirection direction, int rw) { struct jfile *jf; + unsigned int seq_beg = -1; + unsigned int seq_end = -1; + unsigned int seq; + struct stat st; + const char *dirname; + struct dirent *den; + DIR *dir; + char *basename; + char *data; + char *ptr; + int hastransid; + int baselen; + int fd; - jf = malloc(sizeof(struct jfile)); - bzero(jf, sizeof(struct jfile)); - jf->jf_fp = fp; - jf->jf_direction = jdir; - jf->jf_setpt = -1; - jf->jf_flags = flags; - if (jdir == JF_BACKWARDS) { - fseeko(jf->jf_fp, 0L, SEEK_END); - jf->jf_pos = ftello(jf->jf_fp); + dirname = data = strdup(prefix); + if ((basename = strrchr(dirname, '/')) != NULL) { + *basename++ = 0; + } else { + basename = data; + dirname = "./"; + } + baselen = strlen(basename); + if ((dir = opendir(dirname)) != NULL) { + while ((den = readdir(dir)) != NULL) { + if (strncmp(den->d_name, basename, baselen) == 0 && + den->d_name[baselen] == '.' + ) { + seq = strtoul(den->d_name + baselen + 1, &ptr, 10); + if (*ptr == 0 && seq > 0) { + if (seq_beg == (unsigned int)-1 || seq_beg > seq) + seq_beg = seq; + if (seq_end == (unsigned int)-1 || seq_end < seq) + seq_end = seq; + } + } + } + closedir(dir); + } + free(data); + + hastransid = 0; + asprintf(&data, "%s.transid", prefix); + if (stat(data, &st) == 0) + hastransid = 1; + free(data); + + if (seq_beg != (unsigned int)-1 || hastransid) { + if (seq_beg == (unsigned int)-1) { + seq_beg = 0; + seq_end = 0; + if (rw) { + asprintf(&data, "%s.%08x", prefix, 0); + if ((fd = open(data, O_RDWR|O_CREAT, 0666)) >= 0) + close(fd); + free(data); + } + } + jf = malloc(sizeof(struct jfile)); + bzero(jf, sizeof(struct jfile)); + jf->jf_fd = -1; + jf->jf_prefix = strdup(prefix); + jf->jf_seq_beg = seq_beg; + jf->jf_seq_end = seq_end; + jf->jf_pos = -1; + if (direction == JD_BACKWARDS) { + jf->jf_seq = jf->jf_seq_end; + } else { + jf->jf_seq = jf->jf_seq_beg; + } + jf->jf_direction = direction; + jf->jf_open_flags = rw ? (O_RDWR|O_CREAT) : O_RDONLY; + } else { + jf = NULL; } return(jf); } /* - * Close a previously opened journal, clean up any side allocations. + * Get a prefix set ready for append. */ -void -jclose_stream(struct jfile *jf) +int +jrecord_init(const char *prefix) { - struct jdata *jd; + struct jfile *jf; + struct stat st; + char *data; + int hasseqspace; + int fd; - fclose(jf->jf_fp); - jf->jf_fp = NULL; - while ((jd = jf->jf_saved) != NULL) { - jf->jf_saved = jd->jd_next; - free(jd); + /* + * Determine whether we already have a prefix set or whether we need + * to create one. + */ + jf = jopen_prefix(prefix, 0, 0); + hasseqspace = 0; + if (jf) { + if (jf->jf_seq_beg != (unsigned int)-1) + hasseqspace = 1; + jclose(jf); } - free(jf); + asprintf(&data, "%s.transid", prefix); + + /* + * If the sequence exists the transid file must ALREADY exist for us + * to be able to safely 'append' to the space. Locked-down sequence + * spaces do not have a transid file. + */ + if (hasseqspace) { + fd = open(data, O_RDWR, 0666); + } else { + fd = open(data, O_RDWR|O_CREAT, 0666); + } + free(data); + if (fd < 0) + return(-1); + if (fstat(fd, &st) == 0 && st.st_size == 0) + write(fd, "0000000000000000\n", 17); /* starting transid in hex */ + close(fd); + return(0); } /* - * Align us to the next 16 byte boundary. If scanning forwards we align - * forwards if not already aligned. If scanning backwards we align - * backwards if not already aligned. + * Close a previously opened journal, clean up any side allocations. */ void -jalign(struct jfile *jf) +jclose(struct jfile *jf) { - if (jf->jf_direction == JF_FORWARDS) { - jf->jf_pos = (jf->jf_pos + 15) & ~15; - fseeko(jf->jf_fp, jf->jf_pos, SEEK_SET); - } else { - jf->jf_pos = jf->jf_pos & ~15; - } + close(jf->jf_fd); + jf->jf_fd = -1; + free(jf); } /* - * Read data from a journal forwards or backwards. Note that the file - * pointer's actual seek position does not match jf_pos in the reverse - * scan case. Callers should never access jf_fp directly. + * Locate the next (or previous) complete virtual stream transaction given a + * file descriptor and direction. Keep track of partial stream records as + * a side effect. + * + * Note that a transaction might represent a huge I/O operation, resulting + * in an overall node structure that spans gigabytes, but individual + * subrecord leaf nodes are limited in size and we depend on this to simplify + * the handling of leaf records. + * + * A transaction may cover several raw records. The jstream collection for + * a transaction is only returned when the entire transaction has been + * successfully scanned. Due to the interleaving of transactions the ordering + * of returned JS's may be different (not exactly reversed) when scanning a + * journal backwards verses forwards. Since parallel operations are + * theoretically non-conflicting, this should not present a problem. */ int -jread(struct jfile *jf, void *buf, int bytes) +jread(struct jfile *jf, struct jdata **jdp, enum jdirection direction) { + struct journal_rawrecbeg head; + struct journal_rawrecbeg *headp; + struct journal_rawrecend tail; + struct journal_rawrecend *tailp; + struct jdata *jd; + char *filename; + int allocsize; + int recsize; + int search; int n; - - if (jf->jf_direction == JF_FORWARDS) { - while (bytes) { - n = fread(buf, 1, bytes, jf->jf_fp); - if (n <= 0) - break; - assert(n <= bytes); - jf->jf_pos += n; - buf = (char *)buf + n; - bytes -= n; + + /* + * If changing direction on an open descriptor we have to fixup jf_pos. + * When reading backwards the actual file seek position does not match + * jf_pos. + * + * If you read forwards then read backwards, or read backwords then + * read forwards, you will get the same record. + */ + if (jf->jf_direction != direction) { + if (jf->jf_fd >= 0) { + if (direction == JD_FORWARDS) { + lseek(jf->jf_fd, jf->jf_pos, 0); + } } - if (bytes == 0) { - return (0); - } else { - fseeko(jf->jf_fp, jf->jf_pos, SEEK_SET); - return (errno ? errno : ENOENT); + jf->jf_direction = direction; + } + +top: + /* + * If reading in prefix mode and we have no descriptor, open + * a new descriptor based on the current sequence number. If + * this fails we will fall all the way through to the end which will + * setup the next sequence number and loop. + */ + if (jf->jf_fd == -1 && jf->jf_prefix) { + asprintf(&filename, "%s.%08x", jf->jf_prefix, jf->jf_seq); + if ((jf->jf_fd = open(filename, O_RDONLY)) >= 0) { + if (jf->jf_direction == JD_FORWARDS) + jf->jf_pos = 0; + else + jf->jf_pos = lseek(jf->jf_fd, 0L, SEEK_END); + search = 0; + } + fprintf(stderr, "Open %s fd %d\n", filename, jf->jf_fd); + free(filename); + } + + /* + * Get the current offset and make sure it is 16-byte aligned. If it + * isn't, align it and enter search mode. + */ + if (jf->jf_pos & 15) { + jf_warn(jf, "realigning bad offset and entering search mode"); + jalign(jf); + search = 1; + } else { + search = 0; + } + + if (jf->jf_direction == JD_FORWARDS) { + /* + * Scan the journal forwards. Note that the file pointer might not + * be seekable. + */ + while (jreadbuf(jf, &head, sizeof(head)) == sizeof(head)) { + if (head.begmagic != JREC_BEGMAGIC) { + if (search == 0) + jf_warn(jf, "bad beginmagic, searching for new record"); + search = 1; + jalign(jf); + continue; + } + + /* + * The actual record is 16-byte aligned. head.recsize contains + * the unaligned record size. + */ + recsize = (head.recsize + 15) & ~15; + if (recsize < JREC_MINRECSIZE || recsize > JREC_MAXRECSIZE) { + if (search == 0) + jf_warn(jf, "bad recordsize: %d\n", recsize); + search = 1; + jalign(jf); + continue; + } + allocsize = offsetof(struct jdata, jd_data[recsize]); + allocsize = (allocsize + 255) & ~255; + jd = malloc(allocsize); + bzero(jd, offsetof(struct jdata, jd_data[0])); + bcopy(&head, jd->jd_data, sizeof(head)); + n = jreadbuf(jf, jd->jd_data + sizeof(head), + recsize - sizeof(head)); + if (n != (int)(recsize - sizeof(head))) { + if (search == 0) + jf_warn(jf, "Incomplete stream record\n"); + search = 1; + jalign(jf); + free(jd); + continue; + } + + tailp = (void *)(jd->jd_data + recsize - sizeof(*tailp)); + if (tailp->endmagic != JREC_ENDMAGIC) { + if (search == 0) + jf_warn(jf, "bad endmagic, searching for new record"); + search = 1; + jalign(jf); + free(jd); + continue; + } + + /* + * note: recsize is aligned (the actual record size), + * head.recsize is unaligned (the actual payload size). + */ + jd->jd_transid = head.transid; + jd->jd_alloc = allocsize; + jd->jd_size = recsize; + jd->jd_refs = 1; + jd->jd_next = jf->jf_data; + jf->jf_data = jd; + *jdp = jd; + return(0); } } else { - if (bytes > jf->jf_pos) - return (ENOENT); - jf->jf_pos -= bytes; - fseeko(jf->jf_fp, jf->jf_pos, SEEK_SET); - if (fread(buf, bytes, 1, jf->jf_fp) == 1) { - return (0); + /* + * Scan the journal backwards. Note that jread()'s reverse-seek and + * read. The data read will be forward ordered, however. + */ + while (jreadbuf(jf, &tail, sizeof(tail)) == sizeof(tail)) { + if (tail.endmagic != JREC_ENDMAGIC) { + if (search == 0) + jf_warn(jf, "bad endmagic, searching for new record"); + search = 1; + jalign(jf); + continue; + } + + /* + * The actual record is 16-byte aligned. head.recsize contains + * the unaligned record size. + */ + recsize = (tail.recsize + 15) & ~15; + if (recsize < JREC_MINRECSIZE || recsize > JREC_MAXRECSIZE) { + if (search == 0) + jf_warn(jf, "bad recordsize: %d\n", recsize); + search = 1; + jalign(jf); + continue; + } + allocsize = offsetof(struct jdata, jd_data[recsize]); + allocsize = (allocsize + 255) & ~255; + jd = malloc(allocsize); + bzero(jd, offsetof(struct jdata, jd_data[0])); + bcopy(&tail, jd->jd_data + recsize - sizeof(tail), sizeof(tail)); + n = jreadbuf(jf, jd->jd_data, recsize - sizeof(tail)); + if (n != (int)(recsize - sizeof(tail))) { + if (search == 0) + jf_warn(jf, "Incomplete stream record\n"); + search = 1; + jalign(jf); + free(jd); + continue; + } + + headp = (void *)jd->jd_data; + if (headp->begmagic != JREC_BEGMAGIC) { + if (search == 0) + jf_warn(jf, "bad begmagic, searching for new record"); + search = 1; + jalign(jf); + free(jd); + continue; + } + + /* + * note: recsize is aligned (the actual record size), + * head.recsize is unaligned (the actual payload size). + */ + jd->jd_transid = headp->transid; + jd->jd_alloc = allocsize; + jd->jd_size = recsize; + jd->jd_refs = 1; + jd->jd_next = jf->jf_data; + jf->jf_data = jd; + *jdp = jd; + return(0); + } + } + + /* + * If reading in prefix mode and there is no more data, close the + * current descriptor, adjust the sequence number, and loop. + */ + if (jf->jf_prefix) { + close(jf->jf_fd); + jf->jf_fd = -1; + if (jf->jf_direction == JD_FORWARDS) { + if (jf->jf_seq < jf->jf_seq_end) { + ++jf->jf_seq; + goto top; + } } else { - jf->jf_pos += bytes; - return (errno); + if (jf->jf_seq > jf->jf_seq_beg) { + --jf->jf_seq; + goto top; + } } } + + /* + * Otherwise there are no more records and we are done. + */ + *jdp = NULL; + return(-1); } +/* + * Write a record out. If this is a prefix set and the file would + * exceed record_size, we rotate into a new sequence number. + */ int -jwrite(struct jfile *jf, void *buf, int bytes) +jwrite(struct jfile *jf, struct jdata *jd) { int n; - n = write(fileno(jf->jf_fp), buf, bytes); + n = write(jf->jf_fd, jd->jd_data, jd->jd_size); return(n); } -void -jset(struct jfile *jf) +/* + * Reset the direction and seek us to the beginning or end + * of the currenet file. In prefix mode we might as well + * just let jsread() do it since it might have to do it + * anyway. + */ +static void +jreset(struct jfile *jf, unsigned int seq, enum jdirection direction) { - jf->jf_setpt = jf->jf_pos; + if (jf->jf_prefix) { + if (jf->jf_fd >= 0) { + close(jf->jf_fd); + jf->jf_fd = -1; + } + jf->jf_pos = -1; + jf->jf_seq = seq; + } else { + if (direction) { + jf->jf_pos = lseek(jf->jf_fd, 0L, 0); + } else { + jf->jf_pos = lseek(jf->jf_fd, 0L, SEEK_END); + } + } + jf->jf_direction = direction; } +/* + * Position the file such that the next jread() in the specified + * direction will read the record for the specified transaction id. + * If the transaction id does not exist the jseek will position the + * file at the next higher (if reading forwards) or lower (if reading + * backwards) transaction id. + * + * jseek is not required to be exact. It is allowed to position the + * file at any point <= the transid (forwards) or >= the transid + * (backwards). However, the more off jseek is, the more scanning + * the code will have to do to position itself properly. + */ void -jreturn(struct jfile *jf) +jseek(struct jfile *jf, int64_t transid, enum jdirection direction) { - jf->jf_pos = jf->jf_setpt; - jf->jf_setpt = -1; - fseeko(jf->jf_fp, jf->jf_pos, SEEK_SET); + int64_t transid_beg; + int64_t transid_end; + unsigned int seq; + struct jdata *jd; + + /* + * If we have a prefix set search the sequence space backwards until + * we find the file most likely to contain the transaction id. + */ + if (jf->jf_prefix) { + for (seq = jf->jf_seq_end; seq >= jf->jf_seq_beg; --seq) { + jreset(jf, seq, JD_FORWARDS); + if (jread(jf, &jd, JD_FORWARDS) == 0) { + transid_beg = jd->jd_transid; + jfree(jf, jd); + if (transid_beg == transid) { + jreset(jf, seq, JD_FORWARDS); + break; + } + if (transid_beg < transid) + break; + } + } + } + + /* + * Position us within the current file. + */ + jreset(jf, seq, JD_BACKWARDS); + while (jread(jf, &jd, JD_BACKWARDS) == 0) { + transid_end = jd->jd_transid; + jfree(jf, jd); + + /* + * If we are at the sequence number the next forward read + * will re-read the record since we were going backwards. If + * the caller wants to go backwards we have to go forwards one + * record so the caller gets the transid record when it does + * its first backwards read. Confused yet? + * + * If we are at a smaller sequence number we need to read forwards + * by one so the next forwards read gets the first record > transid, + * or the next backwards read gets the first record < transid. + */ + if (transid_end == transid) { + if (direction == JD_BACKWARDS) { + if (jread(jf, &jd, JD_FORWARDS) == 0) + jfree(jf, jd); + } + break; + } + if (transid_end < transid) { + if (jread(jf, &jd, JD_FORWARDS) == 0) + jfree(jf, jd); + } + } +} + +/* + * Data returned by jread() is persistent until released. + */ +struct jdata * +jref(struct jdata *jd) +{ + ++jd->jd_refs; + return(jd); } void -jflush(struct jfile *jf) +jfree(struct jfile *jf, struct jdata *jd) { - jf->jf_setpt = -1; + struct jdata **jdp; + + if (--jd->jd_refs == 0){ + for (jdp = &jf->jf_data; *jdp != jd; jdp = &(*jdp)->jd_next) { + assert(*jdp != NULL); + } + *jdp = jd->jd_next; + free(jd); + } +} + +/* + * Align us to the next 16 byte boundary. If scanning forwards we align + * forwards if not already aligned. If scanning backwards we align + * backwards if not already aligned. We only have to synchronize the + * seek position with the file seek position for forward scans. + */ +static void +jalign(struct jfile *jf) +{ + char dummy[16]; + int bytes; + + if ((int)jf->jf_pos & 15) { + if (jf->jf_direction == JD_FORWARDS) { + bytes = 16 - ((int)jf->jf_pos & 15); + jf->jf_pos += jreadbuf(jf, dummy, bytes); + } else { + jf->jf_pos = jf->jf_pos & ~(off_t)15; + } + } +} + +/* + * Read the next raw journal record forwards or backwards and return a + * pointer to it. Note that the file pointer's actual seek position does + * not match jf_pos in the reverse direction case. + */ +static int +jreadbuf(struct jfile *jf, void *buf, int bytes) +{ + int ttl = 0; + int n; + + if (jf->jf_fd < 0) + return(0); + + if (jf->jf_direction == JD_FORWARDS) { + while (ttl != bytes) { + n = read(jf->jf_fd, (char *)buf + ttl, bytes - ttl); + if (n <= 0) + break; + ttl += n; + } + } else { + if (jf->jf_pos >= bytes) { + jf->jf_pos -= bytes; + lseek(jf->jf_fd, jf->jf_pos, 0); + while (ttl != bytes) { + n = read(jf->jf_fd, (char *)buf + ttl, bytes - ttl); + if (n <= 0) + break; + ttl += n; + } + } + } + return(ttl); } diff --git a/sbin/jscan/jscan.8 b/sbin/jscan/jscan.8 index 905209719e..a8fc8d9dd3 100644 --- a/sbin/jscan/jscan.8 +++ b/sbin/jscan/jscan.8 @@ -31,7 +31,7 @@ .\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.\" $DragonFly: src/sbin/jscan/jscan.8,v 1.5 2005/08/28 05:13:53 asmodai Exp $ +.\" $DragonFly: src/sbin/jscan/jscan.8,v 1.6 2005/09/06 06:42:44 dillon Exp $ .\" .Dd March 6, 2005 .Dt JSCAN 8 @@ -41,28 +41,241 @@ .Nd journal file processing program .Sh SYNOPSIS .Nm -.Fl 2rd -.Op Ar journal_files +.Fl 2duF +.Op Fl c Ar count[k,m,g,t] +.Op Fl D Ar directory +.Op Fl m Ar mirror_transid_file/none +.Op Fl o/O Ar output_transid_file/none +.Op Fl s Ar size[k,m,g,t] +.Op Fl w/W Ar journal_prefix +.Op Ar journal_prefix/file .Pp .Sh DESCRIPTION The .Nm -utility scans journal files for the purposes of debugging dumps, restoration, -undo, mirroring, and other journaling features. +utility scans journal file or input stream for the purposes of debugging +dumps, restoration, undo, mirroring, and other journaling features. .Bl -tag -width indent .It Fl 2 Implement the full-duplex acknowledgement protocol on the input descriptor. Note that shell pipes are full-duplex and can be used with this option. +.It Fl c Ar count +Specify the number of transaction records which should be scanned, then exit. +This option is typically used along with +.Fl m +to limit the amount of work that +.Nm +does, giving you the ability to incrementally run a mirror forwards or +backwards. It is not usually used when piping in a live journal, but it +can be. .It Fl d -dumps the contents of the journaling file in a human readable format. -.It Fl r -will cause the journal be scanned backwards. -Transactions will be dumped in reverse order. -However, note that due to parallelism the order will not be the exact reverse -of a forward scan, since large transactions are not dumped until all stream -segments have been collected for them. +Display the contents of the journaling file or stream in a human readable +format on stderr. Note that stdout is used only for +.Fl o . +.It Fl D Ar directory +Specify the base directory for the mirroring option. +.It Fl m Ar mirror_transid_file/none +Generate a mirror in the directory specified by +.Fl D +or, if not specified, the current directory. +The +.Ar mirror_transid_file +will be used to track the transaction id representing the current +syncnronization point for the mirror. The keyword +.Ar none +may be specified if no tracking file is desired. However, if no tracking +file is specified it will not be possible to roll the mirror forwards or +backwards or restart the journaling stream being used to generate the mirror. +.Pp +It is important to note that journaling streams can contain meta-transactions +representing huge, multi-gigabyte operations. If the journaling data is +not being recorded to regular files via +.Fl w/W +it is possible that +.Nm +could run itself out of memory trying to record the meta-transactions. +In addition, the mirror would not be restartable. If the journaling data +is being recorded via +.Fl w/W +and a mirroring transaction id file is being kept, the mirror can be +restarted. +.Pp +While it is possible to run a journaling stream directly into a mirror, +it is more typical to file the jornaling stream with +.Fl w +and catch the mirror up as a batch job with the journaling file set prefix +specified as the input every so often. This way the system operator can +use other +.Nm +commands to, for example, run a mirror backwards and forwards in time. +.It Fl o/O Ar output_transid_file/none +Generate a journaling stream on stdout using the specified file to track +the transaction id to help with restarts. +The +.Fl o +option indicates a half-duplex output stream while the +.Fl O +option indicates a full-duplex (ACK protocol) output stream. +.Pp +This option is not really designed to output to regular files because it +does NOT necessarily weed out duplicate records. When both the input +stream and output stream are full-duplex and +.Fl w/W +is not specified, +.Nm +acts as a stateless transceiver and the input stream is not acked until +an ack is received from the output stream. +.Pp +This option is most typically used in conjuction with +.Fl w/W . +In this case the ACK protocol is handled independantly for the input side +and the output side uses the journaling data recorded by +.Fl w/W +as a buffer. +.Pp +In half-duplex output mode the output transaction id file is updated +after a raw transaction record has been successfully written to stdout. +In full-duplex output mode the file is only updated with ACK data returned +on the stdout descriptor. +.Pp +As with the +.Fl m +option, you can combine +.Fl o +in a journaling pipe with other options, but if you are trying to use it +as a buffer it may be better to have it separately pull its data off of +a journaling file set generated via +.Fl w . +.It Fl s Ar size +Change the size limit for rotating files created via +.Fl w . +The default is 100M. Values are in bytes or may be suffixed with k, +m, or g. +If a raw transaction causes the file's size limit to be exceeded, a new file +will be created. If a raw transaction is, in-whole, larger the the file's +size limit, the raw transaction will still be fully written to the file before +a new file is created. Raw transactions are typically limited to the size +of the source system's memory FIFO. This option is typically used to size +journaling files to fit onto the appropriate backup media or to provide +bite-sized chunks for other programs to injest. +.Pp +When restarting a journal, a new sequence number will always be chosen for +the resumption of data recording. No existing file will be appended to when +.Nm +is reinvoked. +.It Fl u +Will cause the journal to be scanned backwards (requires seekable media). +Transactions will be dumped in reverse order. If mirroring, the UNDO +data will be executed. If not specified, 1 hour's worth of data will be +undone. Can only be used with a journaling file or journaling prefix +as the input. +.It Fl w Ar prefix +The received journaling stream is recorded in journaling files named +.Ar . +and the current transaction id is tracked in a file named +.Ar .transid . +A journaling file is closed out and a new file with the next sequence +number is created once the file surpasses 100MB. +.Pp +This option is robust across restarts. The current transaction id +will be read and the input stream will be skipped until it is reached. +If the input is a journaling file or prefix set, +.Nm +will be able to quickly seek to the restart point. +.Pp +NOTE: If +you are generating a mirror with the same command via +.Fl m , +and the journaling data input is a stream rather then a file or prefix +set, you must use +.Fl w/W +if you want the mirror to be restartable. This is because while we can +pick up the transaction id where we left off, that raw transaction id may +have cut a larger meta-transaction in half and the mirroring code will +not be able to access the whole of the transaction unless it has a file +or prefix set to work with. +.It Fl W Ar prefix +Similar to +.Fl w +except that the journaling files created are strictly temporary and will +be deleted once they exceed the size limit AND the related meta-transactions +have been completed. +.Pp +If combined with +.Fl m , +the meta-transactions are considered to be completed only when the mirror +finishes executing them. It is possible for several sequence number files +to build if a particularly large meta-transaction is coming down the pipe. +.Pp +If combined with +.Fl o/O , +the meta-transactions are considered to be completed when the data has +been successfully written out to the pipe in half duplex mode, or when +the ACK has been received in full-duplex mode. +.Pp +If both +.Fl m +and +.Fl w/W +is used, the journaling data files are only deleted when both actions +no longer need the data. +.It Fl F +Forces +.Nm +to fsync() after updating a journaling file prior to acknowledging the +data or updating a transaction-id-tracking file. If specified twice, +.Nm +will also fsync() after updating the transaction-id-tracking file. +.It Ar journal_prefix/file +Specify the input to jscan. This can be a journaling file set prefix +or it can be a plain file. If no input file is specified, stdin is +assumed. Note that when generating a mirror from a stdin stream, the +mirror will not be restartable unless +.Fl w/W +is also used. +.Pp .El .Pp +.Sh OPERATIONAL NOTES +It is often important to be able to quickly stage journaled data through +a dedicated backup machine on a LAN. There are several places where data +can be buffered and staged out. +.Pp +The machine generating the journal typically buffers several megabytes of +journal data in the kernel. This local machine can pipe that data to +.Nm +or some other locally run program to add another buffering stage, or you +can directly attach a TCP connection to the kernel's journaling output. +.Pp +The LAN backup box typically buffers gigabytes worth of data by running +multiple jscan's. The jscan on the receiving end of the TCP or pipe (for +example, via ssh) typically records the data via the +.Fl w +option, and then runs other +.Nm +programs from scripts or cron to take that data and copy it to your +off-site backup machine. Other jscan programs may use the same data +set to generate mirrors or other backup streams. +.Pp +It should be noted that if +.Fl w/W +is specified, both mirroring mode and output mode will internally +fork the program once the appropriate synchronization point has been reached, +effectively decoupling their operation, and read all of their data via +the journaling files written out by the master program. In particular, +blockages in the mirroring and output code will not effect our ability +to buffer the journaling input data via +.Fl w/W . +If +.Fl w/W +is not specified then neither the mirroring or output modes will fork. Under +these conditions, if the input is a stream rather then a file +.Nm +will be forced to buffer meta-transactions (for mirroring) entirely in +memory, which could present a serious problem since a single meta-transaction +can exceed a gigabyte (e.g. if someone were to do a single write() system +call writing a gigabyte all in one go). +.Pp .Sh SEE ALSO .Xr mountctl 8 .Sh CAVEATS diff --git a/sbin/jscan/jscan.c b/sbin/jscan/jscan.c index 68088ae456..b1b00cef4b 100644 --- a/sbin/jscan/jscan.c +++ b/sbin/jscan/jscan.c @@ -31,96 +31,375 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/jscan/jscan.c,v 1.4 2005/07/06 06:06:44 dillon Exp $ + * $DragonFly: src/sbin/jscan/jscan.c,v 1.5 2005/09/06 06:42:44 dillon Exp $ */ #include "jscan.h" -enum jmode { JS_NONE, JS_DEBUG, JS_MIRROR }; - static void usage(const char *av0); -int debug_opt; +int jmodes; +int fsync_opt; +off_t record_size = 100 * 1024 * 1024; +off_t trans_count; +static enum jdirection jdirection = JD_FORWARDS; + +static void jscan_do_output(struct jfile *, const char *, int64_t); +static void jscan_do_mirror(struct jfile *, const char *, int64_t); +static void jscan_do_record(struct jfile *, const char *, int64_t); +static void fork_subprocess(struct jfile *, + void (*)(struct jfile *, const char *, int64_t), + const char *, const char *, int64_t); int main(int ac, char **av) { - int ch; - int i; - int jfflags = 0; - enum jdirection direction = JF_FORWARDS; - enum jmode jmode = JS_NONE; + const char *input_prefix = NULL; + char *output_transid_file = NULL; + char *mirror_transid_file = NULL; + const char *mirror_directory = "."; + char *record_prefix = NULL; + char *record_transid_file = NULL; + enum jdirection direction; + char *ptr; + int64_t mirror_transid; + int64_t output_transid; + int64_t record_transid; + int64_t transid; + int input_fd; + struct stat st; struct jfile *jf; + struct jdata *jd; + int bytes; + int error; + int ch; - while ((ch = getopt(ac, av, "2dmrw:W:")) != -1) { + while ((ch = getopt(ac, av, "2dm:o:s:uw:D:O:W:F")) != -1) { switch(ch) { case '2': - jfflags |= JF_FULL_DUPLEX; + jmodes |= JMODEF_INPUT_FULL; + break; + case 'c': + trans_count = strtoll(optarg, &ptr, 0); + switch(*ptr) { + case 't': + record_size *= 1024; + /* fall through */ + case 'g': + record_size *= 1024; + /* fall through */ + case 'm': + record_size *= 1024; + /* fall through */ + case 'k': + record_size *= 1024; + break; + case 0: + break; + default: + fprintf(stderr, "Bad suffix for value specified with -c, use 'k', 'm', 'g', 't', or nothing\n"); + usage(av[0]); + } break; case 'd': - debug_opt = 1; - if (jmode == JS_NONE) - jmode = JS_DEBUG; + jmodes |= JMODEF_DEBUG; break; case 'm': - jmode = JS_MIRROR; + jmodes |= JMODEF_MIRROR; + if (strcmp(optarg, "none") != 0) + mirror_transid_file = optarg; + break; + case 'O': + jmodes |= JMODEF_OUTPUT_FULL; + /* fall through */ + case 'o': + jmodes |= JMODEF_OUTPUT; + if (strcmp(optarg, "none") != 0) + output_transid_file = optarg; + break; + case 's': + record_size = strtoll(optarg, &ptr, 0); + switch(*ptr) { + case 't': + record_size *= 1024; + /* fall through */ + case 'g': + record_size *= 1024; + /* fall through */ + case 'm': + record_size *= 1024; + /* fall through */ + case 'k': + record_size *= 1024; + break; + case 0: + break; + default: + fprintf(stderr, "Bad suffix for value specified with -s, use 'k', 'm', 'g', 't', or nothing\n"); + usage(av[0]); + } break; - case 'r': - direction = JF_BACKWARDS; + case 'u': + jdirection = JD_BACKWARDS; break; case 'W': - /* fallthrough */ + jmodes |= JMODEF_RECORD_TMP; + /* fall through */ case 'w': - /* not implemented yet */ + jmodes |= JMODEF_RECORD; + record_prefix = optarg; + asprintf(&record_transid_file, "%s.transid", record_prefix); + break; + case 'D': + mirror_directory = optarg; + break; + case 'F': + ++fsync_opt; + break; default: fprintf(stderr, "unknown option: -%c\n", optopt); usage(av[0]); } } - if (jmode == JS_NONE) + + /* + * Sanity checks + */ + if ((jmodes & JMODEF_COMMAND_MASK) == 0) usage(av[0]); - if (jmode == JS_MIRROR && direction == JF_BACKWARDS) { - fprintf(stderr, "Cannot mirror in reverse scan mode\n"); + if (optind > ac + 1) { + fprintf(stderr, "Only one input file or prefix may be specified,\n" + "or zero if stdin is to be the input.\n"); usage(av[0]); } + if (jdirection == JD_BACKWARDS && (jmodes & (JMODEF_RECORD|JMODEF_OUTPUT))) { + fprintf(stderr, "Undo mode is only good in mirroring mode and " + "cannot be mixed with other modes.\n"); + exit(1); + } /* - * Using specified input streams. If no files are specified, stdin - * is used. + * STEP1 - OPEN INPUT + * + * The input will either be a pipe, a regular file, or a journaling + * file prefix. */ - if (ac == optind) { - usage(av[0]); - } else { - for (i = optind; i < ac; ++i) { - if (strcmp(av[i], "stdin") == 0) - jf = jopen_fp(stdin, direction, jfflags); - else - jf = jopen_stream(av[i], direction, jfflags); - if (jf != NULL) { - switch(jmode) { - case JS_MIRROR: - dump_mirror(jf); - break; - case JS_DEBUG: - dump_debug(jf); - break; - case JS_NONE: - break; - } - jclose_stream(jf); - } else { - fprintf(stderr, "Unable to open %s: %s\n", - av[i], strerror(errno)); + jf = NULL; + if (optind == ac) { + input_prefix = ""; + input_fd = 0; + if (fstat(0, &st) < 0 || !S_ISREG(st.st_mode)) { + jmodes |= JMODEF_INPUT_PIPE; + if (jdirection == JD_BACKWARDS) { + fprintf(stderr, "Cannot scan journals on pipes backwards\n"); + usage(av[0]); } } + jf = jopen_fd(input_fd, jdirection); + } else if (stat(av[optind], &st) == 0 && S_ISREG(st.st_mode)) { + input_prefix = av[optind]; + if ((input_fd = open(av[optind], O_RDONLY)) != NULL) { + jf = jopen_fd(input_fd, jdirection); + } else { + jf = NULL; + } + } else { + input_prefix = av[optind]; + jf = jopen_prefix(input_prefix, jdirection, 0); + jmodes |= JMODEF_INPUT_PREFIX; + } + if (jf == NULL) { + fprintf(stderr, "Unable to open input %s: %s\n", + input_prefix, strerror(errno)); + exit(1); + } + + /* + * STEP 1 - SYNCHRONIZING THE INPUT STREAM + * + * Figure out the starting point for our various output modes. Figure + * out the earliest transaction id and try to seek to that point, + * otherwise we might have to scan through terrabytes of data. + * + * Invalid transid's will be set to 0, but it should also be noted + * that 0 is also a valid transid. + */ + get_transid_from_file(output_transid_file, &output_transid, + JMODEF_OUTPUT_TRANSID_GOOD); + get_transid_from_file(mirror_transid_file, &mirror_transid, + JMODEF_MIRROR_TRANSID_GOOD); + get_transid_from_file(record_transid_file, &record_transid, + JMODEF_RECORD_TRANSID_GOOD); + transid = LLONG_MAX; + if ((jmodes & JMODEF_OUTPUT_TRANSID_GOOD) && output_transid < transid) + transid = output_transid; + if ((jmodes & JMODEF_MIRROR_TRANSID_GOOD) && mirror_transid < transid) + transid = mirror_transid; + if ((jmodes & JMODEF_RECORD_TRANSID_GOOD) && record_transid < transid) + transid = record_transid; + if ((jmodes & JMODEF_TRANSID_GOOD_MASK) == 0) + transid = 0; + + /* + * Now it gets more difficult. If we are recording then the input + * could be representative of continuing data and not have any + * prior, older data that the output or mirror modes might need. Those + * modes must work off the recording data even as we write to it. + * In that case we fork and have the sub-processes work off the + * record output. + * + * Then we take the input and start recording. + */ + if (jmodes & JMODEF_RECORD) { + if (jrecord_init(record_prefix) < 0) { + fprintf(stderr, "Unable to initialize file set for: %s\n", + record_prefix); + exit(1); + } + if (jmodes & JMODEF_MIRROR) { + fork_subprocess(jf, jscan_do_mirror, record_prefix, + mirror_directory, mirror_transid); + /* XXX ack stream for temporary record file removal */ + } + if (jmodes & JMODEF_OUTPUT) { + fork_subprocess(jf, jscan_do_output, record_prefix, + NULL, output_transid); + /* XXX ack stream for temporary record file removal */ + } + jscan_do_record(jf, record_prefix, record_transid); + exit(0); + } + + /* + * If the input is a prefix set we can just pass it to the appropriate + * jscan_do_*() function. If we are doing both output and mirroring + * we fork the mirror and do the output in the foreground since that + * is going to stdout. + */ + if (jmodes & JMODEF_INPUT_PREFIX) { + if ((jmodes & JMODEF_OUTPUT) && (jmodes & JMODEF_MIRROR)) { + fork_subprocess(jf, jscan_do_mirror, input_prefix, + mirror_directory, mirror_transid); + jscan_do_output(jf, NULL, output_transid); + } else if (jmodes & JMODEF_OUTPUT) { + jscan_do_output(jf, NULL, output_transid); + } else if (jmodes & JMODEF_MIRROR) { + jscan_do_mirror(jf, mirror_directory, mirror_transid); + } + exit(0); + } + + /* + * The input is not a prefix set and we are not recording, which means + * we have to transfer the data on the input pipe to the output and + * mirroring code on the fly. This also means that we must keep track + * of meta-data records in-memory. However, if the input is a regular + * file we *CAN* try to optimize where we start reading. + * + * NOTE: If the mirroring code encounters a transaction record that is + * not marked begin, and it does not have the begin record, it will + * attempt to locate the begin record if the input is not a pipe, then + * seek back. + */ + if ((jmodes & JMODEF_TRANSID_GOOD_MASK) && !(jmodes & JMODEF_INPUT_PIPE)) + jseek(jf, transid, jdirection); + jmodes |= JMODEF_MEMORY_TRACKING; + + while ((error = jread(jf, &jd, jdirection)) == 0) { + if (jmodes & JMODEF_DEBUG) + dump_debug(jf, jd, transid); + if (jmodes & JMODEF_OUTPUT) + dump_output(jf, jd, output_transid); + if (jmodes & JMODEF_MIRROR) + dump_mirror(jf, jd, mirror_transid); + jfree(jf, jd); + } + jclose(jf); + exit(error ? 1 : 0); +} + +static +void +fork_subprocess(struct jfile *jftoclose, + void (*func)(struct jfile *, const char *, int64_t), + const char *input_prefix, const char *info, int64_t transid) +{ + pid_t pid; + struct jfile *jf; + + if ((pid = fork()) == 0) { + jmodes &= ~JMODEF_DEBUG; + jclose(jftoclose); + jf = jopen_prefix(input_prefix, jdirection, 0); + jmodes |= JMODEF_INPUT_PREFIX; + func(jf, info, transid); + jclose(jf); + exit(0); + } else if (pid < 0) { + fprintf(stderr, "fork(): %s\n", strerror(errno)); + exit(1); + } +} + +static +void +jscan_do_output(struct jfile *jf, const char *info, int64_t transid) +{ + struct jdata *jd; + int error; + + if ((jmodes & JMODEF_OUTPUT_TRANSID_GOOD) && !(jmodes & JMODEF_INPUT_PIPE)) + jseek(jf, transid, jdirection); + while ((error = jread(jf, &jd, jdirection)) == 0) { + if (jmodes & JMODEF_DEBUG) + dump_debug(jf, jd, transid); + dump_output(jf, jd, transid); + jfree(jf, jd); + } +} + +static +void +jscan_do_mirror(struct jfile *jf, const char *info, int64_t transid) +{ + struct jdata *jd; + int error; + + if ((jmodes & JMODEF_MIRROR_TRANSID_GOOD) && !(jmodes & JMODEF_INPUT_PIPE)) + jseek(jf, transid, jdirection); + while ((error = jread(jf, &jd, jdirection)) == 0) { + if (jmodes & JMODEF_DEBUG) + dump_debug(jf, jd, transid); + dump_mirror(jf, jd, transid); + jfree(jf, jd); + } +} + +static +void +jscan_do_record(struct jfile *jf, const char *info, int64_t transid) +{ + struct jdata *jd; + int error; + + if ((jmodes & JMODEF_RECORD_TRANSID_GOOD) && !(jmodes & JMODEF_INPUT_PIPE)) + jseek(jf, transid, jdirection); + while ((error = jread(jf, &jd, jdirection)) == 0) { + if (jmodes & JMODEF_DEBUG) + dump_debug(jf, jd, transid); + dump_record(jf, jd, transid); + jfree(jf, jd); } - return(0); } static void usage(const char *av0) { - fprintf(stderr, "%s [-dm] [journal_file/stdin]*\n", av0); + fprintf(stderr, + "%s [-2duF] [-D dir] [-m mirror_transid_file/none]\n" + "\t[-o/O output_trnasid_file/none]\n" + "\t[-s size[kmgt]] -w/W record_prefix] [input_file/input_prefix]\n", + av0); exit(1); } diff --git a/sbin/jscan/jscan.h b/sbin/jscan/jscan.h index 19a0cb8839..c50faa3c49 100644 --- a/sbin/jscan/jscan.h +++ b/sbin/jscan/jscan.h @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/jscan/jscan.h,v 1.5 2005/07/06 06:06:44 dillon Exp $ + * $DragonFly: src/sbin/jscan/jscan.h,v 1.6 2005/09/06 06:42:44 dillon Exp $ */ #include @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -51,30 +52,36 @@ #include #include "jattr.h" -enum jdirection { JF_FORWARDS, JF_BACKWARDS}; +struct jdata; + +enum jdirection { JD_FORWARDS, JD_BACKWARDS }; struct jfile { off_t jf_pos; /* current seek position */ - off_t jf_setpt; /* saved seek position */ - struct jdata *jf_saved; /* saved data */ - FILE *jf_fp; - enum jdirection jf_direction; + int jf_fd; int jf_error; - int jf_flags; + enum jdirection jf_direction; + int jf_open_flags; + struct jdata *jf_data; + char *jf_prefix; /* prefix: name */ + unsigned int jf_seq_beg; /* prefix: sequence space */ + unsigned int jf_seq_end; /* prefix: sequence space */ + unsigned int jf_seq; /* prefix: current sequence number */ }; #define JF_FULL_DUPLEX 0x0001 struct jdata { struct jdata *jd_next; - int jd_size; - off_t jd_pos; + int64_t jd_transid; /* transaction id from header */ + int jd_alloc; /* allocated bytes */ + int jd_size; /* data bytes */ + int jd_refs; /* ref count */ char jd_data[4]; /* must be last field */ }; struct jstream { struct jstream *js_next; /* linked list / same transaction */ - int js_size; /* amount of data, in bytes */ char *js_alloc_buf; int js_alloc_size; @@ -92,8 +99,9 @@ struct jstream { * chain. */ struct jstream *js_cache; - - char js_data[4]; /* variable length (original data) */ + struct jfile *js_jfile; + struct jdata *js_jdata; + struct journal_rawrecbeg *js_head; }; struct jhash { @@ -106,7 +114,31 @@ struct jhash { #define JHASH_SIZE 1024 #define JHASH_MASK (JHASH_SIZE - 1) -extern int debug_opt; +#define JMODEF_DEBUG 0x00000001 +#define JMODEF_MIRROR 0x00000002 +#define JMODEF_UNUSED0004 0x00000004 +#define JMODEF_RECORD 0x00000008 +#define JMODEF_RECORD_TMP 0x00000010 +#define JMODEF_INPUT_FULL 0x00000020 +#define JMODEF_INPUT_PIPE 0x00000040 +#define JMODEF_INPUT_PREFIX 0x00000080 +#define JMODEF_OUTPUT 0x00000100 +#define JMODEF_OUTPUT_FULL 0x00000200 +#define JMODEF_MEMORY_TRACKING 0x00000400 + +#define JMODEF_OUTPUT_TRANSID_GOOD 0x00010000 +#define JMODEF_RECORD_TRANSID_GOOD 0x00020000 +#define JMODEF_MIRROR_TRANSID_GOOD 0x00040000 +#define JMODEF_TRANSID_GOOD_MASK (JMODEF_OUTPUT_TRANSID_GOOD|\ + JMODEF_RECORD_TRANSID_GOOD|\ + JMODEF_MIRROR_TRANSID_GOOD) +#define JMODEF_COMMAND_MASK (JMODEF_RECORD|JMODEF_MIRROR|JMODEF_DEBUG|\ + JMODEF_OUTPUT) + +extern int jmodes; +extern int fsync_opt; +extern off_t record_size; +extern off_t trans_count; const char *type_to_name(int16_t rectype); void stringout(FILE *fp, char c, int exact); @@ -115,29 +147,30 @@ int64_t buf_to_int64(const void *buf, int bytes); void *dupdata(const void *buf, int bytes); char *dupdatastr(const void *buf, int bytes); char *dupdatapath(const void *buf, int bytes); +void get_transid_from_file(const char *path, int64_t *transid, int flags); + +struct jfile *jopen_fd(int fd, enum jdirection direction); +struct jfile *jopen_prefix(const char *prefix, enum jdirection direction, int rw); +void jclose(struct jfile *jf); +int jread(struct jfile *jf, struct jdata **jdp, enum jdirection direction); +int jwrite(struct jfile *jf, struct jdata *jd); +void jseek(struct jfile *jf, int64_t transid, enum jdirection direction); +struct jdata *jref(struct jdata *jd); +void jfree(struct jfile *jf, struct jdata *jd); +void jf_warn(struct jfile *jf, const char *ctl, ...); - -struct jstream *jscan_stream(struct jfile *jf); +struct jstream *jaddrecord(struct jfile *jf, struct jdata *jd); void jscan_dispose(struct jstream *js); -struct jfile *jopen_stream(const char *path, enum jdirection jdir, int flags); -struct jfile *jopen_fp(FILE *fp, enum jdirection jdir, int flags); -void jclose_stream(struct jfile *jf); -void jalign(struct jfile *jf); -int jread(struct jfile *jf, void *buf, int bytes); -int jwrite(struct jfile *jf, void *buf, int bytes); -void jset(struct jfile *jf); -void jreturn(struct jfile *jf); -void jflush(struct jfile *jf); -void jf_warn(struct jfile *jf, const char *ctl, ...); -void dump_debug(struct jfile *jf); -void dump_mirror(struct jfile *jf); -int dump_debug_payload(int16_t rectype, struct jstream *js, off_t off, - int recsize, int level); +void dump_debug(struct jfile *jf, struct jdata *jd, int64_t transid); +void dump_mirror(struct jfile *jf, struct jdata *jd, int64_t transid); +void dump_record(struct jfile *jf, struct jdata *jd, int64_t transid); +void dump_output(struct jfile *jf, struct jdata *jd, int64_t transid); + +int jrecord_init(const char *record_prefix); int jsreadany(struct jstream *js, off_t off, const void **bufp); int jsreadp(struct jstream *js, off_t off, const void **bufp, int bytes); int jsread(struct jstream *js, off_t off, void *buf, int bytes); int jsreadcallback(struct jstream *js, ssize_t (*func)(int, const void *, size_t), int fd, off_t off, int bytes); - diff --git a/sbin/jscan/jstream.c b/sbin/jscan/jstream.c index cffe96a877..3337e2648b 100644 --- a/sbin/jscan/jstream.c +++ b/sbin/jscan/jstream.c @@ -31,182 +31,17 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/jscan/jstream.c,v 1.4 2005/07/06 06:06:44 dillon Exp $ + * $DragonFly: src/sbin/jscan/jstream.c,v 1.5 2005/09/06 06:42:44 dillon Exp $ */ #include "jscan.h" static struct jhash *JHashAry[JHASH_SIZE]; -static struct jstream *jaddrecord(struct jfile *jf, struct jstream *js); static void jnormalize(struct jstream *js); /* - * Locate the next (or previous) complete virtual stream transaction given a - * file descriptor and direction. Keep track of partial stream records as - * a side effect. - * - * Note that a transaction might represent a huge I/O operation, resulting - * in an overall node structure that spans gigabytes, but individual - * subrecord leaf nodes are limited in size and we depend on this to simplify - * the handling of leaf records. - * - * A transaction may cover several raw records. The jstream collection for - * a transaction is only returned when the entire transaction has been - * successfully scanned. Due to the interleaving of transactions the ordering - * of returned JS's may be different (not exactly reversed) when scanning a - * journal backwards verses forwards. Since parallel operations are - * theoretically non-conflicting, this should not present a problem. - */ -struct jstream * -jscan_stream(struct jfile *jf) -{ - struct journal_rawrecbeg head; - struct journal_rawrecend tail; - struct journal_ackrecord ack; - int recsize; - int search; - int error; - struct jstream *js; - - /* - * Get the current offset and make sure it is 16-byte aligned. If it - * isn't, align it and enter search mode. - */ - if (jf->jf_pos & 15) { - jf_warn(jf, "realigning bad offset and entering search mode"); - jalign(jf); - search = 1; - } else { - search = 0; - } - - error = 0; - js = NULL; - - if (jf->jf_direction == JF_FORWARDS) { - /* - * Scan the journal forwards. Note that the file pointer might not - * be seekable. - */ - while ((error = jread(jf, &head, sizeof(head))) == 0) { - if (head.begmagic != JREC_BEGMAGIC) { - if (search == 0) - jf_warn(jf, "bad beginmagic, searching for new record"); - search = 1; - jalign(jf); - continue; - } - recsize = (head.recsize + 15) & ~15; - if (recsize <= 0) { - jf_warn(jf, "bad recordsize: %d\n", recsize); - search = 1; - jalign(jf); - continue; - } - jset(jf); - js = malloc(offsetof(struct jstream, js_data[recsize])); - bzero(js, sizeof(struct jstream)); - bcopy(&head, js->js_data, sizeof(head)); - error = jread(jf, js->js_data + sizeof(head), recsize - sizeof(head)); - if (error) { - jf_warn(jf, "Incomplete stream record\n"); - jreturn(jf); - free(js); - js = NULL; - break; - } - - /* - * XXX if the stream is full duplex send the ack back now. This - * really needs to be delayed until the transaction is committed, - * but there are stalling issues if the transaction being - * collected exceeds to the size of the FIFO. So for now this - * is just for testing. - */ - if (jf->jf_flags & JF_FULL_DUPLEX) { - bzero(&ack, sizeof(ack)); - ack.rbeg.begmagic = JREC_BEGMAGIC; - ack.rbeg.streamid = JREC_STREAMID_ACK; - ack.rbeg.transid = head.transid; - ack.rbeg.recsize = sizeof(ack); - ack.rend.endmagic = JREC_ENDMAGIC; - ack.rend.recsize = sizeof(ack); - jwrite(jf, &ack, sizeof(ack)); - } - - /* - * note: recsize is aligned (the actual record size), - * head.recsize is unaligned (the actual payload size). - */ - js->js_size = head.recsize; - bcopy(js->js_data + recsize - sizeof(tail), &tail, sizeof(tail)); - if (tail.endmagic != JREC_ENDMAGIC) { - jf_warn(jf, "bad endmagic, searching for new record"); - search = 1; - jreturn(jf); - free(js); - js = NULL; - continue; - } - jflush(jf); - if ((js = jaddrecord(jf, js)) != NULL) - break; - } - } else { - /* - * Scan the journal backwards. Note that jread()'s reverse-seek and - * read. The data read will be forward ordered, however. - */ - while ((error = jread(jf, &tail, sizeof(tail))) == 0) { - if (tail.endmagic != JREC_ENDMAGIC) { - if (search == 0) - jf_warn(jf, "bad endmagic, searching for new record"); - search = 1; - jalign(jf); - continue; - } - recsize = (tail.recsize + 15) & ~15; - if (recsize <= 0) { - jf_warn(jf, "bad recordsize: %d\n", recsize); - search = 1; - jalign(jf); - continue; - } - jset(jf); - js = malloc(offsetof(struct jstream, js_data[recsize])); - bzero(js, sizeof(struct jstream)); - bcopy(&tail, js->js_data + recsize - sizeof(tail), sizeof(tail)); - error = jread(jf, js->js_data, recsize - sizeof(tail)); - - if (error) { - jf_warn(jf, "Incomplete stream record\n"); - jreturn(jf); - free(js); - js = NULL; - break; - } - js->js_size = tail.recsize; - bcopy(js->js_data + recsize - sizeof(tail), &tail, sizeof(tail)); - bcopy(js->js_data, &head, sizeof(head)); - if (head.begmagic != JREC_BEGMAGIC) { - jf_warn(jf, "bad begmagic, searching for new record"); - search = 1; - jreturn(jf); - free(js); - continue; - } - jflush(jf); - if ((js = jaddrecord(jf, js)) != NULL) - break; - } - } - jf->jf_error = error; - return(js); -} - -/* - * Integrate a jstream record. Deal with the transaction begin and end flags + * Integrate a raw record. Deal with the transaction begin and end flags * to create a forward-referenced collection of jstream records. If we are * able to complete a transaction, the first js associated with that * transaction is returned. @@ -214,13 +49,21 @@ jscan_stream(struct jfile *jf) * XXX we need to store the data for very large multi-record transactions * separately since it might not fit into memory. */ -static struct jstream * -jaddrecord(struct jfile *jf, struct jstream *js) +struct jstream * +jaddrecord(struct jfile *jf, struct jdata *jd) { - struct journal_rawrecbeg *head = (void *)js->js_data; + struct journal_rawrecbeg *head; + struct jstream *js; struct jhash *jh; struct jhash **jhp; + js = malloc(sizeof(struct jstream)); + bzero(js, sizeof(struct jstream)); + js->js_jdata = jref(jd); + js->js_head = (void *)jd->jd_data; + js->js_jfile = jf; + head = js->js_head; + /* * Check for a completely self-contained transaction, just return the * js if possible. @@ -256,7 +99,7 @@ jaddrecord(struct jfile *jf, struct jstream *js) * Emplace the stream segment */ jh->jh_transid |= head->streamid & JREC_STREAMCTL_MASK; - if (jf->jf_direction == JF_FORWARDS) { + if (jf->jf_direction == JD_FORWARDS) { jh->jh_last->js_next = js; jh->jh_last = js; } else { @@ -294,15 +137,15 @@ jnormalize(struct jstream *js) off_t off; js->js_normalized_off = 0; - js->js_normalized_base = js->js_data; - js->js_normalized_size = ((struct journal_rawrecbeg *)js->js_data)->recsize - sizeof(struct journal_rawrecend); + js->js_normalized_base = (void *)js->js_head; + js->js_normalized_size = js->js_head->recsize - sizeof(struct journal_rawrecend); js->js_normalized_total = js->js_normalized_size; off = js->js_normalized_size; for (jscan = js->js_next; jscan; jscan = jscan->js_next) { jscan->js_normalized_off = off; - jscan->js_normalized_base = jscan->js_data + + jscan->js_normalized_base = (char *)jscan->js_head + sizeof(struct journal_rawrecbeg); - jscan->js_normalized_size = jscan->js_size - + jscan->js_normalized_size = jscan->js_head->recsize - sizeof(struct journal_rawrecbeg) - sizeof(struct journal_rawrecend); off += jscan->js_normalized_size; @@ -323,6 +166,8 @@ jscan_dispose(struct jstream *js) while (js) { jnext = js->js_next; + jfree(js->js_jfile, js->js_jdata); + js->js_jdata = NULL; free(js); js = jnext; } diff --git a/sbin/jscan/subs.c b/sbin/jscan/subs.c index d2385ad5b5..11adc96b4d 100644 --- a/sbin/jscan/subs.c +++ b/sbin/jscan/subs.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sbin/jscan/subs.c,v 1.6 2005/07/06 06:21:05 dillon Exp $ + * $DragonFly: src/sbin/jscan/subs.c,v 1.7 2005/09/06 06:42:44 dillon Exp $ */ #include "jscan.h" @@ -354,3 +354,21 @@ dupdatapath(const void *buf, int bytes) return(res); } +void +get_transid_from_file(const char *path, int64_t *transid, int flags) +{ + int n; + int fd; + char buf[32]; + + *transid = 0; + if ((fd = open(path, O_RDONLY)) >= 0) { + n = read(fd, buf, sizeof(buf) - 1); + if (n >= 0) + buf[n] = 0; + *transid = strtoull(buf, NULL, 16); + jmodes |= flags; + close(fd); + } +} + diff --git a/sys/sys/journal.h b/sys/sys/journal.h index 15f58af5d4..6d314d5c58 100644 --- a/sys/sys/journal.h +++ b/sys/sys/journal.h @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/sys/journal.h,v 1.7 2005/08/24 20:28:33 dillon Exp $ + * $DragonFly: src/sys/sys/journal.h,v 1.8 2005/09/06 06:42:39 dillon Exp $ */ #ifndef _SYS_JOURNAL_H_ @@ -159,6 +159,8 @@ struct journal_ackrecord { #define JREC_STREAMID_JMAX 0x2000 /* (one past the highest allowed id) */ #define JREC_DEFAULTSIZE 64 /* reasonable initial reservation */ +#define JREC_MINRECSIZE 32 /* (after alignment) */ +#define JREC_MAXRECSIZE (128*1024*1024) /* * Each logical journaling stream typically represents a transaction... -- 2.41.0