2 * Copyright (c) 2003,2004 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * $DragonFly: src/sbin/jscan/jscan.c,v 1.9 2005/09/07 19:10:09 dillon Exp $
39 static int donecheck(enum jdirection direction, struct jdata *jd,
41 static void usage(const char *av0);
46 off_t prefix_file_size = 100 * 1024 * 1024;
48 static enum jdirection jdirection = JD_FORWARDS;
50 static void jscan_do_output(struct jfile *, const char *,
51 const char *, int64_t);
52 static void jscan_do_mirror(struct jfile *, const char *,
53 const char *, int64_t);
54 static void jscan_do_record(struct jfile *, const char *,
55 const char *, int64_t);
56 static void jscan_do_debug(struct jfile *, const char *,
57 const char *, int64_t);
58 static void fork_subprocess(struct jfile *,
59 void (*)(struct jfile *, const char *,
60 const char *, int64_t),
62 const char *, const char *, int64_t);
65 main(int ac, char **av)
67 const char *input_prefix = NULL;
68 char *output_transid_file = NULL;
69 char *mirror_transid_file = NULL;
70 const char *mirror_directory = ".";
71 char *record_prefix = NULL;
72 char *record_transid_file = NULL;
73 struct jsession jsdebug;
74 struct jsession jsoutput;
75 struct jsession jsmirror;
77 int64_t mirror_transid;
78 int64_t output_transid;
79 int64_t record_transid;
87 while ((ch = getopt(ac, av, "2c:dfm:o:s:uvw:D:O:W:F")) != -1) {
90 jmodes |= JMODEF_INPUT_FULL;
93 trans_count = strtoll(optarg, &ptr, 0);
110 fprintf(stderr, "Bad suffix for value specified with -c, use 'k', 'm', 'g', 't', or nothing\n");
115 jmodes |= JMODEF_DEBUG;
118 jmodes |= JMODEF_LOOP_FOREVER;
124 jmodes |= JMODEF_MIRROR;
125 if (strcmp(optarg, "none") != 0)
126 mirror_transid_file = optarg;
129 jmodes |= JMODEF_OUTPUT_FULL;
132 jmodes |= JMODEF_OUTPUT;
133 if (strcmp(optarg, "none") != 0)
134 output_transid_file = optarg;
137 prefix_file_size = strtoll(optarg, &ptr, 0);
140 prefix_file_size *= 1024;
143 prefix_file_size *= 1024;
146 prefix_file_size *= 1024;
149 prefix_file_size *= 1024;
154 fprintf(stderr, "Bad suffix for value specified with -s, use 'k', 'm', 'g', 't', or nothing\n");
159 jdirection = JD_BACKWARDS;
162 jmodes |= JMODEF_RECORD_TMP;
165 jmodes |= JMODEF_RECORD;
166 record_prefix = optarg;
167 asprintf(&record_transid_file, "%s.transid", record_prefix);
170 mirror_directory = optarg;
176 fprintf(stderr, "unknown option: -%c\n", optopt);
184 if ((jmodes & JMODEF_COMMAND_MASK) == 0)
186 if (optind > ac + 1) {
187 fprintf(stderr, "Only one input file or prefix may be specified,\n"
188 "or zero if stdin is to be the input.\n");
191 if (jdirection == JD_BACKWARDS && (jmodes & (JMODEF_RECORD|JMODEF_OUTPUT))) {
192 fprintf(stderr, "Undo mode is only good in mirroring mode and "
193 "cannot be mixed with other modes.\n");
200 * The input will either be a pipe, a regular file, or a journaling
205 input_prefix = "<stdin>";
207 if (fstat(0, &st) < 0 || !S_ISREG(st.st_mode)) {
208 jmodes |= JMODEF_INPUT_PIPE;
209 if (jdirection == JD_BACKWARDS) {
210 fprintf(stderr, "Cannot scan journals on pipes backwards\n");
214 jf = jopen_fd(input_fd);
215 } else if (stat(av[optind], &st) == 0 && S_ISREG(st.st_mode)) {
216 input_prefix = av[optind];
217 if ((input_fd = open(av[optind], O_RDONLY)) != NULL) {
218 jf = jopen_fd(input_fd);
223 input_prefix = av[optind];
224 jf = jopen_prefix(input_prefix, 0);
225 jmodes |= JMODEF_INPUT_PREFIX;
228 fprintf(stderr, "Unable to open input %s: %s\n",
229 input_prefix, strerror(errno));
234 * STEP 1 - SYNCHRONIZING THE INPUT STREAM
236 * Figure out the starting point for our various output modes. Figure
237 * out the earliest transaction id and try to seek to that point,
238 * otherwise we might have to scan through terrabytes of data.
240 * Invalid transid's will be set to 0, but it should also be noted
241 * that 0 is also a valid transid.
243 get_transid_from_file(output_transid_file, &output_transid,
244 JMODEF_OUTPUT_TRANSID_GOOD);
245 get_transid_from_file(mirror_transid_file, &mirror_transid,
246 JMODEF_MIRROR_TRANSID_GOOD);
247 get_transid_from_file(record_transid_file, &record_transid,
248 JMODEF_RECORD_TRANSID_GOOD);
250 if ((jmodes & JMODEF_OUTPUT_TRANSID_GOOD) && output_transid < transid)
251 transid = output_transid;
252 if ((jmodes & JMODEF_MIRROR_TRANSID_GOOD) && mirror_transid < transid)
253 transid = mirror_transid;
254 if ((jmodes & JMODEF_RECORD_TRANSID_GOOD) && record_transid < transid)
255 transid = record_transid;
256 if ((jmodes & JMODEF_TRANSID_GOOD_MASK) == 0)
259 if (jmodes & JMODEF_OUTPUT) {
260 fprintf(stderr, "Starting transid for OUTPUT: %016llx\n",
263 if (jmodes & JMODEF_MIRROR) {
264 fprintf(stderr, "Starting transid for MIRROR: %016llx\n",
267 if (jmodes & JMODEF_RECORD) {
268 fprintf(stderr, "Starting transid for RECORD: %016llx\n",
274 * Now it gets more difficult. If we are recording then the input
275 * could be representative of continuing data and not have any
276 * prior, older data that the output or mirror modes might need. Those
277 * modes must work off the recording data even as we write to it.
278 * In that case we fork and have the sub-processes work off the
281 * Then we take the input and start recording.
283 if (jmodes & JMODEF_RECORD) {
284 if (jrecord_init(record_prefix) < 0) {
285 fprintf(stderr, "Unable to initialize file set for: %s\n",
289 if (jmodes & JMODEF_MIRROR) {
290 fork_subprocess(jf, jscan_do_mirror, record_prefix,
292 mirror_directory, mirror_transid);
293 /* XXX ack stream for temporary record file removal */
295 if (jmodes & JMODEF_OUTPUT) {
296 fork_subprocess(jf, jscan_do_output, record_prefix,
298 NULL, output_transid);
299 /* XXX ack stream for temporary record file removal */
301 jscan_do_record(jf, record_transid_file, record_prefix, record_transid);
306 * If the input is a prefix set we can just pass it to the appropriate
307 * jscan_do_*() function. If we are doing both output and mirroring
308 * we fork the mirror and do the output in the foreground since that
309 * is going to stdout.
311 if (jmodes & JMODEF_INPUT_PREFIX) {
312 if ((jmodes & JMODEF_OUTPUT) && (jmodes & JMODEF_MIRROR)) {
313 fork_subprocess(jf, jscan_do_mirror, input_prefix,
315 mirror_directory, mirror_transid);
316 jscan_do_output(jf, output_transid_file, NULL, output_transid);
317 } else if (jmodes & JMODEF_OUTPUT) {
318 jscan_do_output(jf, output_transid_file, NULL, output_transid);
319 } else if (jmodes & JMODEF_MIRROR) {
320 jscan_do_mirror(jf, mirror_transid_file, mirror_directory,
322 } else if (jmodes & JMODEF_DEBUG) {
323 jscan_do_debug(jf, NULL, NULL, 0);
329 * The input is not a prefix set and we are not recording, which means
330 * we have to transfer the data on the input pipe to the output and
331 * mirroring code on the fly. This also means that we must keep track
332 * of meta-data records in-memory. However, if the input is a regular
333 * file we *CAN* try to optimize where we start reading.
335 * NOTE: If the mirroring code encounters a transaction record that is
336 * not marked begin, and it does not have the begin record, it will
337 * attempt to locate the begin record if the input is not a pipe, then
340 if ((jmodes & JMODEF_TRANSID_GOOD_MASK) && !(jmodes & JMODEF_INPUT_PIPE))
341 jd = jseek(jf, transid, jdirection);
343 jd = jread(jf, NULL, jdirection);
344 jmodes |= JMODEF_MEMORY_TRACKING;
346 jsession_init(&jsdebug, jf, jdirection,
348 jsession_init(&jsoutput, jf, jdirection,
349 output_transid_file, output_transid);
350 jsession_init(&jsmirror, jf, jdirection,
351 mirror_transid_file, mirror_transid);
352 jsmirror.ss_mirror_directory = mirror_directory;
355 if ((jmodes & JMODEF_DEBUG) && jsession_check(&jsdebug, jd))
356 dump_debug(&jsdebug, jd);
357 if ((jmodes & JMODEF_OUTPUT) && jsession_check(&jsoutput, jd))
358 dump_output(&jsoutput, jd);
359 if ((jmodes & JMODEF_MIRROR) && jsession_check(&jsmirror, jd))
360 dump_mirror(&jsmirror, jd);
361 if (donecheck(jdirection, jd, transid)) {
365 jd = jread(jf, jd, jdirection);
368 jsession_term(&jsdebug);
369 jsession_term(&jsoutput);
370 jsession_term(&jsmirror);
375 * Returns one if we need to break out of our scanning loop, zero otherwise.
379 donecheck(enum jdirection direction, struct jdata *jd, int64_t transid)
381 if (direction == JD_FORWARDS) {
382 if (jd->jd_transid > transid && trans_count && --trans_count == 0)
385 if (jd->jd_transid <= transid && trans_count && --trans_count == 0)
392 * When we have multiple commands and are writing to a prefix set, we can
393 * 'background' the output and/or mirroring command and have the background
394 * processes feed off the prefix set the foreground process is writing to.
398 fork_subprocess(struct jfile *jftoclose,
399 void (*func)(struct jfile *, const char *, const char *, int64_t),
400 const char *input_prefix, const char *transid_file, const char *info,
406 if ((pid = fork()) == 0) {
407 jmodes &= ~(JMODEF_DEBUG | JMODEF_INPUT_PIPE);
408 jmodes |= JMODEF_LOOP_FOREVER; /* keep checking for new input */
410 jf = jopen_prefix(input_prefix, 0);
411 jmodes |= JMODEF_INPUT_PREFIX;
412 func(jf, transid_file, info, transid);
415 } else if (pid < 0) {
416 fprintf(stderr, "fork(): %s\n", strerror(errno));
423 jscan_do_output(struct jfile *jf, const char *output_transid_file, const char *dummy __unused, int64_t transid)
426 struct jsession jsdebug;
427 struct jsession jsoutput;
429 jsession_init(&jsdebug, jf, jdirection,
431 jsession_init(&jsoutput, jf, jdirection,
432 output_transid_file, transid);
434 if ((jmodes & JMODEF_OUTPUT_TRANSID_GOOD) && !(jmodes & JMODEF_INPUT_PIPE))
435 jd = jseek(jf, transid, jdirection);
437 jd = jread(jf, NULL, jdirection);
439 if ((jmodes & JMODEF_DEBUG) && jsession_check(&jsdebug, jd))
440 dump_debug(&jsdebug, jd);
441 if (jsession_check(&jsoutput, jd))
442 dump_output(&jsoutput, jd);
443 if (donecheck(jdirection, jd, transid)) {
447 jd = jread(jf, jd, jdirection);
449 jsession_term(&jsdebug);
450 jsession_term(&jsoutput);
455 jscan_do_mirror(struct jfile *jf, const char *mirror_transid_file, const char *mirror_directory, int64_t transid)
457 struct jsession jsdebug;
458 struct jsession jsmirror;
461 jsession_init(&jsdebug, jf, jdirection,
463 jsession_init(&jsmirror, jf, jdirection,
464 mirror_transid_file, transid);
465 jsmirror.ss_mirror_directory = mirror_directory;
467 if ((jmodes & JMODEF_MIRROR_TRANSID_GOOD) && !(jmodes & JMODEF_INPUT_PIPE))
468 jd = jseek(jf, transid, jdirection);
470 jd = jread(jf, NULL, jdirection);
472 if ((jmodes & JMODEF_DEBUG) && jsession_check(&jsdebug, jd))
473 dump_debug(&jsdebug, jd);
474 if (jsession_check(&jsmirror, jd))
475 dump_mirror(&jsmirror, jd);
476 if (donecheck(jdirection, jd, transid)) {
480 jd = jread(jf, jd, jdirection);
482 jsession_term(&jsdebug);
483 jsession_term(&jsmirror);
488 jscan_do_record(struct jfile *jfin, const char *record_transid_file, const char *prefix, int64_t transid)
490 struct jsession jsdebug;
491 struct jsession jsrecord;
494 jsession_init(&jsdebug, jfin, jdirection,
496 jsession_init(&jsrecord, jfin, jdirection,
497 record_transid_file, transid);
499 assert(jdirection == JD_FORWARDS);
500 jsrecord.ss_jfout = jopen_prefix(prefix, 1);
501 if (jsrecord.ss_jfout == NULL) {
502 fprintf(stderr, "Unable to open prefix set for writing: %s\n", prefix);
505 if ((jmodes & JMODEF_RECORD_TRANSID_GOOD) && !(jmodes & JMODEF_INPUT_PIPE))
506 jd = jseek(jfin, transid, jdirection);
508 jd = jread(jfin, NULL, jdirection);
510 if ((jmodes & JMODEF_DEBUG) && jsession_check(&jsdebug, jd))
511 dump_debug(&jsdebug, jd);
512 if (jsession_check(&jsrecord, jd))
513 dump_record(&jsrecord, jd);
514 if (donecheck(jdirection, jd, transid)) {
518 jd = jread(jfin, jd, jdirection);
520 jclose(jsrecord.ss_jfout);
521 jsrecord.ss_jfout = NULL;
522 jsession_term(&jsdebug);
523 jsession_term(&jsrecord);
528 jscan_do_debug(struct jfile *jf, const char *dummy1 __unused,
529 const char *dummy __unused, int64_t transid __unused)
531 struct jsession jsdebug;
534 jsession_init(&jsdebug, jf, jdirection,
537 while ((jd = jread(jf, jd, jdirection)) != NULL) {
538 if (jsession_check(&jsdebug, jd))
539 dump_debug(&jsdebug, jd);
540 if (donecheck(jdirection, jd, transid)) {
545 jsession_term(&jsdebug);
549 usage(const char *av0)
552 "%s [-2duF] [-D dir] [-m mirror_transid_file/none]\n"
553 "\t[-o/O output_trnasid_file/none]\n"
554 "\t[-s size[kmgt]] -w/W record_prefix] [input_file/input_prefix]\n",