dhcpcd: Note update to 9.4.1
[dragonfly.git] / sys / libiconv / iconv_ucs.c
1 /*-
2  * Copyright (c) 2003, 2005 Ryuichiro Imura
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  * $FreeBSD: head/sys/libkern/iconv_ucs.c 267291 2014-06-09 19:27:47Z jhb $
27  */
28
29 #include <sys/param.h>
30 #include <sys/kernel.h>
31 #include <sys/systm.h>
32 #include <sys/malloc.h>
33 #include <sys/iconv.h>
34
35 #include "iconv_converter_if.h"
36
37 /*
38  * "UCS" converter
39  */
40
41 #define KICONV_UCS_COMBINE      0x1
42 #define KICONV_UCS_FROM_UTF8    0x2
43 #define KICONV_UCS_TO_UTF8      0x4
44 #define KICONV_UCS_FROM_LE      0x8
45 #define KICONV_UCS_TO_LE        0x10
46 #define KICONV_UCS_FROM_UTF16   0x20
47 #define KICONV_UCS_TO_UTF16     0x40
48 #define KICONV_UCS_UCS4         0x80
49
50 #define ENCODING_UTF16  "UTF-16BE"
51 #define ENCODING_UTF8   "UTF-8"
52
53 static struct {
54         const char *name;
55         int from_flag, to_flag;
56 } unicode_family[] = {
57         { "UTF-8",      KICONV_UCS_FROM_UTF8,   KICONV_UCS_TO_UTF8 },
58         { "UCS-2LE",    KICONV_UCS_FROM_LE,     KICONV_UCS_TO_LE },
59         { "UTF-16BE",   KICONV_UCS_FROM_UTF16,  KICONV_UCS_TO_UTF16 },
60         { "UTF-16LE",   KICONV_UCS_FROM_UTF16|KICONV_UCS_FROM_LE,
61             KICONV_UCS_TO_UTF16|KICONV_UCS_TO_LE },
62         { NULL,         0,      0 }
63 };
64
65 static uint32_t utf8_to_ucs4(const char *src, size_t *utf8width, size_t srclen);
66 static u_char *ucs4_to_utf8(uint32_t ucs4, char * dst, size_t *utf8width, size_t dstlen);
67 static uint32_t encode_surrogate(uint32_t code);
68 static uint32_t decode_surrogate(const u_char *ucs);
69
70 #ifdef MODULE_DEPEND
71 MODULE_DEPEND(iconv_ucs, libiconv, 2, 2, 2);
72 #endif
73
74 /*
75  * UCS converter instance
76  */
77 struct iconv_ucs {
78         KOBJ_FIELDS;
79         int                     convtype;
80         struct iconv_cspair *   d_csp;
81         struct iconv_cspair *   d_cspf;
82         void *                  f_ctp;
83         void *                  t_ctp;
84         void *                  ctype;
85 };
86
87 static int
88 iconv_ucs_open(struct iconv_converter_class *dcp,
89         struct iconv_cspair *csp, struct iconv_cspair *cspf, void **dpp)
90 {
91         struct iconv_ucs *dp;
92         int i;
93         const char *from, *to;
94
95         dp = (struct iconv_ucs *)kobj_create((struct kobj_class*)dcp, M_ICONV, M_WAITOK);
96         to = csp->cp_to;
97         from = cspf ? cspf->cp_from : csp->cp_from;
98
99         dp->convtype = 0;
100
101         if (cspf)
102                 dp->convtype |= KICONV_UCS_COMBINE;
103         for (i = 0; unicode_family[i].name; i++) {
104                 if (strcasecmp(from, unicode_family[i].name) == 0)
105                         dp->convtype |= unicode_family[i].from_flag;
106                 if (strcasecmp(to, unicode_family[i].name) == 0)
107                         dp->convtype |= unicode_family[i].to_flag;
108         }
109         if (strcmp(ENCODING_UNICODE, ENCODING_UTF16) == 0)
110                 dp->convtype |= KICONV_UCS_UCS4;
111         else
112                 dp->convtype &= ~KICONV_UCS_UCS4;
113
114         dp->f_ctp = dp->t_ctp = NULL;
115         if (dp->convtype & KICONV_UCS_COMBINE) {
116                 if ((dp->convtype & KICONV_UCS_FROM_UTF8) == 0 &&
117                     (dp->convtype & KICONV_UCS_FROM_LE) == 0) {
118                         iconv_open(ENCODING_UNICODE, from, &dp->f_ctp);
119                 }
120                 if ((dp->convtype & KICONV_UCS_TO_UTF8) == 0 &&
121                     (dp->convtype & KICONV_UCS_TO_LE) == 0) {
122                         iconv_open(to, ENCODING_UNICODE, &dp->t_ctp);
123                 }
124         }
125
126         dp->ctype = NULL;
127         if (dp->convtype & (KICONV_UCS_FROM_UTF8 | KICONV_UCS_TO_UTF8))
128                 iconv_open(KICONV_WCTYPE_NAME, ENCODING_UTF8, &dp->ctype);
129
130         dp->d_csp = csp;
131         if (dp->convtype & (KICONV_UCS_FROM_UTF8 | KICONV_UCS_FROM_LE)) {
132                 if (cspf) {
133                         dp->d_cspf = cspf;
134                         cspf->cp_refcount++;
135                 } else
136                         csp->cp_refcount++;
137         }
138         if (dp->convtype & (KICONV_UCS_TO_UTF8 | KICONV_UCS_TO_LE))
139                 csp->cp_refcount++;
140         *dpp = (void*)dp;
141         return 0;
142 }
143
144 static int
145 iconv_ucs_close(void *data)
146 {
147         struct iconv_ucs *dp = data;
148
149         if (dp->f_ctp)
150                 iconv_close(dp->f_ctp);
151         if (dp->t_ctp)
152                 iconv_close(dp->t_ctp);
153         if (dp->ctype)
154                 iconv_close(dp->ctype);
155         if (dp->d_cspf)
156                 dp->d_cspf->cp_refcount--;
157         else if (dp->convtype & (KICONV_UCS_FROM_UTF8 | KICONV_UCS_FROM_LE))
158                 dp->d_csp->cp_refcount--;
159         if (dp->convtype & (KICONV_UCS_TO_UTF8 | KICONV_UCS_TO_LE))
160                 dp->d_csp->cp_refcount--;
161         kobj_delete((struct kobj*)data, M_ICONV);
162         return 0;
163 }
164
165 static int
166 iconv_ucs_conv(void *d2p, const char **inbuf,
167         size_t *inbytesleft, char **outbuf, size_t *outbytesleft,
168         int convchar, int casetype)
169 {
170         struct iconv_ucs *dp = (struct iconv_ucs*)d2p;
171         int ret = 0, i;
172         size_t in, on, ir, or, inlen, outlen, ucslen;
173         const char *src, *p;
174         char *dst;
175         u_char ucs[4], *q;
176         uint32_t code;
177
178         if (inbuf == NULL || *inbuf == NULL || outbuf == NULL || *outbuf == NULL)
179                 return 0;
180         ir = in = *inbytesleft;
181         or = on = *outbytesleft;
182         src = *inbuf;
183         dst = *outbuf;
184
185         while (ir > 0 && or > 0) {
186
187                 /*
188                  * The first half of conversion.
189                  * (convert any code into ENCODING_UNICODE)
190                  */
191                 code = 0;
192                 p = src;
193                 if (dp->convtype & KICONV_UCS_FROM_UTF8) {
194                         /* convert UTF-8 to ENCODING_UNICODE */
195                         inlen = 0;
196                         code = utf8_to_ucs4(p, &inlen, ir);
197                         if (code == 0) {
198                                 ret = -1;
199                                 break;
200                         }
201
202                         if (casetype == KICONV_FROM_LOWER && dp->ctype) {
203                                 code = towlower(code, dp->ctype);
204                         } else if (casetype == KICONV_FROM_UPPER && dp->ctype) {
205                                 code = towupper(code, dp->ctype);
206                         }
207
208                         if ((code >= 0xd800 && code < 0xe000) || code >= 0x110000 ) {
209                                 /* reserved for utf-16 surrogate pair */
210                                 /* invalid unicode */
211                                 ret = -1;
212                                 break;
213                         }
214
215                         if (inlen == 4) {
216                                 if (dp->convtype & KICONV_UCS_UCS4) {
217                                         ucslen = 4;
218                                         code = encode_surrogate(code);
219                                 } else {
220                                         /* can't handle with ucs-2 */
221                                         ret = -1;
222                                         break;
223                                 }
224                         } else {
225                                 ucslen = 2;
226                         }
227
228                         /* save UCS-4 into ucs[] */
229                         for (q = ucs, i = ucslen - 1 ; i >= 0 ; i--)
230                                 *q++ = (code >> (i << 3)) & 0xff;
231
232                 } else if (dp->convtype & KICONV_UCS_COMBINE && dp->f_ctp) {
233                         /* convert local code to ENCODING_UNICODE */
234                         ucslen = 4;
235                         inlen = ir;
236                         q = ucs;
237                         ret = iconv_convchr_case(dp->f_ctp, &p, &inlen, (char **)&q,
238                             &ucslen, casetype & (KICONV_FROM_LOWER | KICONV_FROM_UPPER));
239                         if (ret)
240                                 break;
241                         inlen = ir - inlen;
242                         ucslen = 4 - ucslen;
243
244                 } else {
245                         /* src code is a proper subset of ENCODING_UNICODE */
246                         q = ucs;
247                         if (dp->convtype & KICONV_UCS_FROM_LE) {
248                                 *q = *(p + 1);
249                                 *(q + 1) = *p;
250                                 p += 2;
251                         } else {
252                                 *q = *p++;
253                                 *(q + 1) = *p++;
254                         }
255                         if ((*q & 0xfc) == 0xd8) {
256                                 if (dp->convtype & KICONV_UCS_UCS4 &&
257                                     dp->convtype & KICONV_UCS_FROM_UTF16) {
258                                         inlen = ucslen = 4;
259                                 } else {
260                                         /* invalid unicode */
261                                         ret = -1;
262                                         break;
263                                 }
264                         } else {
265                                 inlen = ucslen = 2;
266                         }
267                         if (ir < inlen) {
268                                 ret = -1;
269                                 break;
270                         }
271                         if (ucslen == 4) {
272                                 q += 2;
273                                 if (dp->convtype & KICONV_UCS_FROM_LE) {
274                                         *q = *(p + 1);
275                                         *(q + 1) = *p;
276                                 } else {
277                                         *q = *p++;
278                                         *(q + 1) = *p;
279                                 }
280                                 if ((*q & 0xfc) != 0xdc) {
281                                         /* invalid unicode */
282                                         ret = -1;
283                                         break;
284                                 }
285                         }
286                 }
287
288                 /*
289                  * The second half of conversion.
290                  * (convert ENCODING_UNICODE into any code)
291                  */
292                 p = ucs;
293                 if (dp->convtype & KICONV_UCS_TO_UTF8) {
294                         q = (u_char *)dst;
295                         if (ucslen == 4 && dp->convtype & KICONV_UCS_UCS4) {
296                                 /* decode surrogate pair */
297                                 code = decode_surrogate(p);
298                         } else {
299                                 code = (ucs[0] << 8) | ucs[1];
300                         }
301
302                         if (casetype == KICONV_LOWER && dp->ctype) {
303                                 code = towlower(code, dp->ctype);
304                         } else if (casetype == KICONV_UPPER && dp->ctype) {
305                                 code = towupper(code, dp->ctype);
306                         }
307
308                         outlen = 0;
309                         if (ucs4_to_utf8(code, q, &outlen, or) == NULL) {
310                                 ret = -1;
311                                 break;
312                         }
313
314                         src += inlen;
315                         ir -= inlen;
316                         dst += outlen;
317                         or -= outlen;
318
319                 } else if (dp->convtype & KICONV_UCS_COMBINE && dp->t_ctp) {
320                         ret = iconv_convchr_case(dp->t_ctp, &p, &ucslen, &dst,
321                             &or, casetype & (KICONV_LOWER | KICONV_UPPER));
322                         if (ret)
323                                 break;
324
325                         src += inlen;
326                         ir -= inlen;
327
328                 } else {
329                         /* dst code is a proper subset of ENCODING_UNICODE */
330                         if (or < ucslen) {
331                                 ret = -1;
332                                 break;
333                         }
334                         src += inlen;
335                         ir -= inlen;
336                         or -= ucslen;
337                         if (dp->convtype & KICONV_UCS_TO_LE) {
338                                 *dst++ = *(p + 1);
339                                 *dst++ = *p;
340                                 p += 2;
341                         } else {
342                                 *dst++ = *p++;
343                                 *dst++ = *p++;
344                         }
345                         if (ucslen == 4) {
346                                 if ((dp->convtype & KICONV_UCS_UCS4) == 0 ||
347                                     (dp->convtype & KICONV_UCS_TO_UTF16) == 0) {
348                                         ret = -1;
349                                         break;
350                                 }
351                                 if (dp->convtype & KICONV_UCS_TO_LE) {
352                                         *dst++ = *(p + 1);
353                                         *dst++ = *p;
354                                 } else {
355                                         *dst++ = *p++;
356                                         *dst++ = *p;
357                                 }
358                         }
359                 }
360
361                 if (convchar == 1)
362                         break;
363         }
364
365         *inbuf += in - ir;
366         *outbuf += on - or;
367         *inbytesleft -= in - ir;
368         *outbytesleft -= on - or;
369         return (ret);
370 }
371
372 static int
373 iconv_ucs_init(struct iconv_converter_class *dcp)
374 {
375         int error;
376
377         error = iconv_add(ENCODING_UNICODE, ENCODING_UNICODE, ENCODING_UTF8);
378         if (error)
379                 return (error);
380         error = iconv_add(ENCODING_UNICODE, ENCODING_UTF8, ENCODING_UNICODE);
381         if (error)
382                 return (error);
383         return (0);
384 }
385
386 static int
387 iconv_ucs_done(struct iconv_converter_class *dcp)
388 {
389         return (0);
390 }
391
392 static const char *
393 iconv_ucs_name(struct iconv_converter_class *dcp)
394 {
395         return (ENCODING_UNICODE);
396 }
397
398 static kobj_method_t iconv_ucs_methods[] = {
399         KOBJMETHOD(iconv_converter_open,        iconv_ucs_open),
400         KOBJMETHOD(iconv_converter_close,       iconv_ucs_close),
401         KOBJMETHOD(iconv_converter_conv,        iconv_ucs_conv),
402         KOBJMETHOD(iconv_converter_init,        iconv_ucs_init),
403         KOBJMETHOD(iconv_converter_done,        iconv_ucs_done),
404         KOBJMETHOD(iconv_converter_name,        iconv_ucs_name),
405         KOBJMETHOD_END
406 };
407
408 KICONV_CONVERTER(ucs, sizeof(struct iconv_ucs));
409
410 static uint32_t
411 utf8_to_ucs4(const char *src, size_t *utf8width, size_t srclen)
412 {
413         size_t i, w = 0;
414         uint32_t ucs4 = 0;
415
416         /*
417          * get leading 1 byte from utf-8
418          */
419         if ((*src & 0x80) == 0) {
420                 /*
421                  * leading 1 bit is "0"
422                  *  utf-8: 0xxxxxxx
423                  *  ucs-4: 00000000 00000000 00000000 0xxxxxxx
424                  */
425                 w = 1;
426                 /* get trailing 7 bits */
427                 ucs4 = *src & 0x7f;
428         } else if ((*src & 0xe0) == 0xc0) {
429                 /*
430                  * leading 3 bits are "110"
431                  *  utf-8: 110xxxxx 10yyyyyy
432                  *  ucs-4: 00000000 00000000 00000xxx xxyyyyyy
433                  */
434                 w = 2;
435                 /* get trailing 5 bits */
436                 ucs4 = *src & 0x1f;
437         } else if ((*src & 0xf0) == 0xe0) {
438                 /*
439                  * leading 4 bits are "1110"
440                  *  utf-8: 1110xxxx 10yyyyyy 10zzzzzz
441                  *  ucs-4: 00000000 00000000 xxxxyyyy yyzzzzzz
442                  */
443                 w = 3;
444                 /* get trailing 4 bits */
445                 ucs4 = *src & 0x0f;
446         } else if ((*src & 0xf8) == 0xf0) {
447                 /*
448                  * leading 5 bits are "11110"
449                  *  utf-8: 11110www 10xxxxxx 10yyyyyy 10zzzzzz
450                  *  ucs-4: 00000000 000wwwxx xxxxyyyy yyzzzzzz
451                  */
452                 w = 4;
453                 /* get trailing 3 bits */
454                 ucs4 = *src & 0x07;
455         } else {
456                 /* out of utf-16 range or having illegal bits */
457                 return (0);
458         }
459
460         if (srclen < w)
461                 return (0);
462
463         /*
464          * get left parts from utf-8
465          */
466         for (i = 1 ; i < w ; i++) {
467                 if ((*(src + i) & 0xc0) != 0x80) {
468                         /* invalid: leading 2 bits are not "10" */
469                         return (0);
470                 }
471                 /* concatenate trailing 6 bits into ucs4 */
472                 ucs4 <<= 6;
473                 ucs4 |= *(src + i) & 0x3f;
474         }
475
476         *utf8width = w;
477         return (ucs4);
478 }
479
480 static u_char *
481 ucs4_to_utf8(uint32_t ucs4, char *dst, size_t *utf8width, size_t dstlen)
482 {
483         u_char lead, *p;
484         size_t i, w;
485
486         /*
487          * determine utf-8 width and leading bits
488          */
489         if (ucs4 < 0x80) {
490                 w = 1;
491                 lead = 0;       /* "0" */
492         } else if (ucs4 < 0x800) {
493                 w = 2;
494                 lead = 0xc0;    /* "11" */
495         } else if (ucs4 < 0x10000) {
496                 w = 3;
497                 lead = 0xe0;    /* "111" */
498         } else if (ucs4 < 0x200000) {
499                 w = 4;
500                 lead = 0xf0;    /* "1111" */
501         } else {
502                 return (NULL);
503         }
504
505         if (dstlen < w)
506                 return (NULL);
507
508         /*
509          * construct utf-8
510          */
511         p = dst;
512         for (i = w - 1 ; i >= 1 ; i--) {
513                 /* get trailing 6 bits and put it with leading bit as "1" */
514                 *(p + i) = (ucs4 & 0x3f) | 0x80;
515                 ucs4 >>= 6;
516         }
517         *p = ucs4 | lead;
518
519         *utf8width = w;
520
521         return (p);
522 }
523
524 static uint32_t
525 encode_surrogate(register uint32_t code)
526 {
527         return ((((code - 0x10000) << 6) & 0x3ff0000) |
528             ((code - 0x10000) & 0x3ff) | 0xd800dc00);
529 }
530
531 static uint32_t
532 decode_surrogate(register const u_char *ucs)
533 {
534         return ((((ucs[0] & 0x3) << 18) | (ucs[1] << 10) |
535             ((ucs[2] & 0x3) << 8) | ucs[3]) + 0x10000);
536 }
537