1 /* $NetBSD: fsort.c,v 1.46 2009/11/06 18:34:22 joerg Exp $ */
4 * Copyright (c) 2000-2003 The NetBSD Foundation, Inc.
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Ben Harris and Jaromir Dolecek.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
34 * The Regents of the University of California. All rights reserved.
36 * This code is derived from software contributed to Berkeley by
39 * Redistribution and use in source and binary forms, with or without
40 * modification, are permitted provided that the following conditions
42 * 1. Redistributions of source code must retain the above copyright
43 * notice, this list of conditions and the following disclaimer.
44 * 2. Redistributions in binary form must reproduce the above copyright
45 * notice, this list of conditions and the following disclaimer in the
46 * documentation and/or other materials provided with the distribution.
47 * 3. Neither the name of the University nor the names of its contributors
48 * may be used to endorse or promote products derived from this software
49 * without specific prior written permission.
51 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
52 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
54 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
65 * Read in a block of records (until 'enough').
66 * sort, write to temp file.
67 * Merge sort temp files into output file
68 * Small files miss out the temp file stage.
69 * Large files might get multiple merges.
77 #define SALIGN(n) roundup2(n, sizeof(length_t))
80 fsort(struct filelist *filelist, int nfiles, FILE *outfp, struct field *ftbl)
83 RECHEADER **keypos, **keyp;
85 size_t bufsize = DEFBUFSIZE;
94 int max_recs = DEBUG('m') ? 16 : MAXNUM;
96 buffer = allocrec(NULL, bufsize);
97 bufend = (u_char *)buffer + bufsize;
98 /* Allocate double length keymap for radix_sort */
99 keylist = malloc(2 * max_recs * sizeof(*keylist));
100 if (buffer == NULL || keylist == NULL)
101 err(2, "failed to malloc initial buffer or keylist");
104 /* Key and data are one! */
107 /* Key (merged key fields) added before data */
111 fp = fopen(filelist->names[0], "r");
113 err(2, "%s", filelist->names[0]);
115 /* Loop through reads of chunk of input files that get sorted
116 * and then merged together. */
121 makeline_copydown(crec);
123 /* Loop reading records */
125 c = get(fp, crec, bufend, ftbl);
126 /* 'c' is 0, EOF or BUFFEND */
128 /* Save start of key in input buffer */
130 if (++nelem == max_recs) {
134 crec = (RECHEADER *)(crec->data + SALIGN(crec->length));
139 if (++file_no >= nfiles)
142 fp = fopen(filelist->names[file_no], "r");
144 err(2, "%s", filelist->names[file_no]);
147 if (nelem >= max_recs
148 || (bufsize >= MAXBUFSIZE && nelem > 8))
149 /* Need to sort and save this lot of data */
152 /* c == BUFFEND, and we can process more data */
153 /* Allocate a larger buffer for this lot of data */
155 nbuffer = allocrec(buffer, bufsize);
157 err(2, "failed to realloc buffer to %zu bytes",
161 /* patch up keylist[] */
162 for (keyp = &keypos[-1]; keyp >= keylist; keyp--)
163 *keyp = nbuffer + (*keyp - buffer);
165 crec = nbuffer + (crec - buffer);
167 bufend = (u_char *)buffer + bufsize;
170 /* Sort this set of records */
171 radix_sort(keylist, keylist + max_recs, nelem);
173 if (c == EOF && mfct == 0) {
174 /* all the data is (sorted) in the buffer */
175 append(keylist, nelem, outfp,
176 DEBUG('k') ? putkeydump : putline);
180 /* Save current data to a temporary file for a later merge */
183 append(keylist, nelem, tmp_fp, putrec);
184 save_for_merge(tmp_fp, geteasy, ftbl);
189 /* merge to output file */
191 DEBUG('k') ? putkeydump : putline, ftbl);