1 /* $Id: preconv.c,v 1.4 2011/05/26 21:13:07 kristaps Exp $ */
3 * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
32 * The read_whole_file() and resize_buf() functions are copied from
33 * read.c, including all dependency code (MAP_FILE, etc.).
41 ENC_UTF_8, /* UTF-8 */
42 ENC_US_ASCII, /* US-ASCII */
43 ENC_LATIN_1, /* Latin-1 */
48 char *buf; /* binary input buffer */
49 size_t sz; /* size of binary buffer */
50 size_t offs; /* starting buffer offset */
55 int (*conv)(const struct buf *);
58 static int cue_enc(const struct buf *, size_t *, enum enc *);
59 static int conv_latin_1(const struct buf *);
60 static int conv_us_ascii(const struct buf *);
61 static int conv_utf_8(const struct buf *);
62 static int read_whole_file(const char *, int,
64 static void resize_buf(struct buf *, size_t);
65 static void usage(void);
67 static const struct encode encs[ENC__MAX] = {
68 { "utf-8", conv_utf_8 }, /* ENC_UTF_8 */
69 { "us-ascii", conv_us_ascii }, /* ENC_US_ASCII */
70 { "latin-1", conv_latin_1 }, /* ENC_LATIN_1 */
73 static const char *progname;
79 fprintf(stderr, "usage: %s "
82 "[file]\n", progname);
86 conv_latin_1(const struct buf *b)
92 cp = b->buf + (int)b->offs;
95 * Latin-1 falls into the first 256 code-points of Unicode, so
96 * there's no need for any sort of translation. Just make the
97 * 8-bit characters use the Unicode escape.
98 * Note that binary values 128 < v < 160 are passed through
99 * unmodified to mandoc.
102 for (i = b->offs; i < b->sz; i++) {
103 cu = (unsigned char)*cp++;
104 cu < 160U ? putchar(cu) : printf("\\[u%.4X]", cu);
111 conv_us_ascii(const struct buf *b)
115 * US-ASCII has no conversion since it falls into the first 128
119 fwrite(b->buf, 1, b->sz, stdout);
124 conv_utf_8(const struct buf *b)
133 cp = b->buf + (int)b->offs;
138 /* Quick test for big-endian value. */
140 if ( ! (*((const char *)(&one))))
143 for (i = b->offs; i < b->sz; i++) {
144 cu = (unsigned char)*cp++;
146 if ( ! (cu & 128) || (cu & 64)) {
147 /* Bad sequence header. */
151 /* Accept only legitimate bit patterns. */
153 if (cu > 191 || cu < 128) {
154 /* Bad in-sequence bits. */
158 accum |= (cu & 63) << --state * 6;
161 * Accum is held in little-endian order as
162 * stipulated by the UTF-8 sequence coding. We
163 * need to convert to a native big-endian if our
164 * architecture requires it.
167 if (0 == state && be)
168 accum = (accum >> 24) |
169 ((accum << 8) & 0x00FF0000) |
170 ((accum >> 8) & 0x0000FF00) |
174 accum < 128U ? putchar(accum) :
175 printf("\\[u%.4X]", accum);
178 } else if (cu & (1 << 7)) {
180 * Entering a UTF-8 state: if we encounter a
181 * UTF-8 bitmask, calculate the expected UTF-8
184 for (state = 0; state < 7; state++)
185 if ( ! (cu & (1 << (7 - state))))
188 /* Accept only legitimate bit patterns. */
192 if (cu <= 244 && cu >= 240) {
193 accum = (cu & 7) << 18;
196 /* Bad 4-sequence start bits. */
199 if (cu <= 239 && cu >= 224) {
200 accum = (cu & 15) << 12;
203 /* Bad 3-sequence start bits. */
206 if (cu <= 223 && cu >= 194) {
207 accum = (cu & 31) << 6;
210 /* Bad 2-sequence start bits. */
213 /* Bad sequence bit mask. */
222 /* Bad trailing bits. */
230 resize_buf(struct buf *buf, size_t initial)
233 buf->sz = buf->sz > initial / 2 ?
234 2 * buf->sz : initial;
236 buf->buf = realloc(buf->buf, buf->sz);
237 if (NULL == buf->buf) {
244 read_whole_file(const char *f, int fd,
245 struct buf *fb, int *with_mmap)
251 if (-1 == fstat(fd, &st)) {
257 * If we're a regular file, try just reading in the whole entry
258 * via mmap(). This is faster than reading it into blocks, and
259 * since each file is only a few bytes to begin with, I'm not
260 * concerned that this is going to tank any machines.
263 if (S_ISREG(st.st_mode) && st.st_size >= (1U << 31)) {
264 fprintf(stderr, "%s: input too large\n", f);
268 if (S_ISREG(st.st_mode)) {
270 fb->sz = (size_t)st.st_size;
271 fb->buf = mmap(NULL, fb->sz, PROT_READ,
272 MAP_FILE|MAP_SHARED, fd, 0);
273 if (fb->buf != MAP_FAILED)
278 * If this isn't a regular file (like, say, stdin), then we must
279 * go the old way and just read things in bit by bit.
287 if (off == fb->sz && fb->sz == (1U << 31)) {
288 fprintf(stderr, "%s: input too large\n", f);
293 resize_buf(fb, 65536);
295 ssz = read(fd, fb->buf + (int)off, fb->sz - off);
313 cue_enc(const struct buf *b, size_t *offs, enum enc *enc)
315 const char *ln, *eoln, *eoph;
316 size_t sz, phsz, nsz;
319 ln = b->buf + (int)*offs;
322 /* Look for the end-of-line. */
324 if (NULL == (eoln = memchr(ln, '\n', sz)))
327 /* Set next-line marker. */
329 *offs = (size_t)((eoln + 1) - b->buf);
331 /* Check if we have the correct header/trailer. */
333 if ((sz = (size_t)(eoln - ln)) < 10 ||
334 memcmp(ln, ".\\\" -*-", 7) ||
335 memcmp(eoln - 3, "-*-", 3))
338 /* Move after the header and adjust for the trailer. */
344 while (sz > 0 && ' ' == *ln) {
351 /* Find the end-of-phrase marker (or eoln). */
353 if (NULL == (eoph = memchr(ln, ';', sz)))
358 /* Only account for the "coding" phrase. */
360 if ((phsz = (size_t)(eoph - ln)) < 7 ||
361 strncasecmp(ln, "coding:", 7)) {
370 while (sz > 0 && ' ' == *ln) {
377 /* Check us against known encodings. */
379 for (i = 0; i < (int)ENC__MAX; i++) {
380 nsz = strlen(encs[i].name);
383 if (strncasecmp(ln, encs[i].name, nsz))
390 /* Unknown encoding. */
400 main(int argc, char *argv[])
402 int i, ch, map, fd, rc;
406 unsigned char bom[3] = { 0xEF, 0xBB, 0xBF };
411 progname = strrchr(argv[0], '/');
412 if (progname == NULL)
420 enc = def = ENC__MAX;
423 memset(&b, 0, sizeof(struct buf));
425 while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
430 for (i = 0; i < (int)ENC__MAX; i++) {
431 if (strcasecmp(optarg, encs[i].name))
435 if (i < (int)ENC__MAX) {
443 fprintf(stderr, "%s: Bad encoding\n", optarg);
444 return(EXIT_FAILURE);
450 /* Compatibility with GNU preconv. */
453 /* Compatibility with GNU preconv. */
457 return(EXIT_FAILURE);
464 * Open and read the first argument on the command-line.
465 * If we don't have one, we default to stdin.
470 fd = open(fn, O_RDONLY, 0);
473 return(EXIT_FAILURE);
477 if ( ! read_whole_file(fn, fd, &b, &map))
480 /* Try to read the UTF-8 BOM. */
483 if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) {
488 /* Try reading from the "-*-" cue. */
490 if (ENC__MAX == enc) {
492 ch = cue_enc(&b, &offs, &enc);
494 ch = cue_enc(&b, &offs, &enc);
498 * No encoding has been detected.
499 * Thus, we either fall into our default encoder, if specified,
500 * or use Latin-1 if all else fails.
504 enc = ENC__MAX == def ? ENC_LATIN_1 : def;
506 if ( ! (*encs[(int)enc].conv)(&b)) {
507 fprintf(stderr, "%s: Bad encoding\n", fn);
518 if (fd > STDIN_FILENO)