contrib/less-394/charset.c

   1 /*
   2  * Copyright (C) 1984-2005  Mark Nudelman
   3  *
   4  * You may distribute under the terms of either the GNU General Public
   5  * License or the Less License, as specified in the README file.
   6  *
   7  * For more information about less, or for information on how to
   8  * contact the author, see the README file.
   9  */
  10
  11
  12 /*
  13  * Functions to define the character set
  14  * and do things specific to the character set.
  15  */
  16
  17 #include "less.h"
  18 #if HAVE_LOCALE
  19 #include <locale.h>
  20 #include <ctype.h>
  21 #include <langinfo.h>
  22 #endif
  23
  24 #include "charset.h"
  25
  26 public int utf_mode = 0;
  27
  28 /*
  29  * Predefined character sets,
  30  * selected by the LESSCHARSET environment variable.
  31  */
  32 struct charset {
  33         char *name;
  34         int *p_flag;
  35         char *desc;
  36 } charsets[] = {
  37         { "ascii",              NULL,       "8bcccbcc18b95.b" },
  38         { "utf-8",              &utf_mode,  "8bcccbcc18b95.b126.bb" },
  39         { "iso8859",            NULL,       "8bcccbcc18b95.33b." },
  40         { "latin3",             NULL,       "8bcccbcc18b95.33b5.b8.b15.b4.b12.b18.b12.b." },
  41         { "arabic",             NULL,       "8bcccbcc18b95.33b.3b.7b2.13b.3b.b26.5b19.b" },
  42         { "greek",              NULL,       "8bcccbcc18b95.33b4.2b4.b3.b35.b44.b" },
  43         { "greek2005",          NULL,       "8bcccbcc18b95.33b14.b35.b44.b" },
  44         { "hebrew",             NULL,       "8bcccbcc18b95.33b.b29.32b28.2b2.b" },
  45         { "koi8-r",             NULL,       "8bcccbcc18b95.b." },
  46         { "KOI8-T",             NULL,       "8bcccbcc18b95.b8.b6.b8.b.b.5b7.3b4.b4.b3.b.b.3b." },
  47         { "georgianps",         NULL,       "8bcccbcc18b95.3b11.4b12.2b." },
  48         { "tcvn",               NULL,       "b..b...bcccbccbbb7.8b95.b48.5b." },
  49         { "TIS-620",            NULL,       "8bcccbcc18b95.b.4b.11b7.8b." },
  50         { "next",               NULL,       "8bcccbcc18b95.bb125.bb" },
  51         { "dos",                NULL,       "8bcccbcc12bc5b95.b." },
  52         { "windows-1251",       NULL,       "8bcccbcc12bc5b95.b24.b." },
  53         { "windows-1252",       NULL,       "8bcccbcc12bc5b95.b.b11.b.2b12.b." },
  54         { "windows-1255",       NULL,       "8bcccbcc12bc5b95.b.b8.b.5b9.b.4b." },
  55         { "ebcdic",             NULL,       "5bc6bcc7bcc41b.9b7.9b5.b..8b6.10b6.b9.7b9.8b8.17b3.3b9.7b9.8b8.6b10.b.b.b." },
  56         { "IBM-1047",           NULL,       "4cbcbc3b9cbccbccbb4c6bcc5b3cbbc4bc4bccbc191.b" },
  57         { NULL, NULL, NULL }
  58 };
  59
  60 /*
  61  * Support "locale charmap"/nl_langinfo(CODESET) values, as well as others.
  62  */
  63 struct cs_alias {
  64         char *name;
  65         char *oname;
  66 } cs_aliases[] = {
  67         { "UTF-8",              "utf-8" },
  68         { "ANSI_X3.4-1968",     "ascii" },
  69         { "US-ASCII",           "ascii" },
  70         { "latin1",             "iso8859" },
  71         { "ISO-8859-1",         "iso8859" },
  72         { "latin9",             "iso8859" },
  73         { "ISO-8859-15",        "iso8859" },
  74         { "latin2",             "iso8859" },
  75         { "ISO-8859-2",         "iso8859" },
  76         { "ISO-8859-3",         "latin3" },
  77         { "latin4",             "iso8859" },
  78         { "ISO-8859-4",         "iso8859" },
  79         { "cyrillic",           "iso8859" },
  80         { "ISO-8859-5",         "iso8859" },
  81         { "ISO-8859-6",         "arabic" },
  82         { "ISO-8859-7",         "greek" },
  83         { "IBM9005",            "greek2005" },
  84         { "ISO-8859-8",         "hebrew" },
  85         { "latin5",             "iso8859" },
  86         { "ISO-8859-9",         "iso8859" },
  87         { "latin6",             "iso8859" },
  88         { "ISO-8859-10",        "iso8859" },
  89         { "latin7",             "iso8859" },
  90         { "ISO-8859-13",        "iso8859" },
  91         { "latin8",             "iso8859" },
  92         { "ISO-8859-14",        "iso8859" },
  93         { "latin10",            "iso8859" },
  94         { "ISO-8859-16",        "iso8859" },
  95         { "IBM437",             "dos" },
  96         { "EBCDIC-US",          "ebcdic" },
  97         { "IBM1047",            "IBM-1047" },
  98         { "KOI8-R",             "koi8-r" },
  99         { "KOI8-U",             "koi8-r" },
 100         { "GEORGIAN-PS",        "georgianps" },
 101         { "TCVN5712-1",         "tcvn" },
 102         { "NEXTSTEP",           "next" },
 103         { "windows",            "windows-1252" }, /* backward compatibility */
 104         { "CP1251",             "windows-1251" },
 105         { "CP1252",             "windows-1252" },
 106         { "CP1255",             "windows-1255" },
 107         { NULL, NULL }
 108 };
 109
 110 #define IS_BINARY_CHAR  01
 111 #define IS_CONTROL_CHAR 02
 112
 113 static char chardef[256];
 114 static char *binfmt = NULL;
 115 static char *utfbinfmt = NULL;
 116 public int binattr = AT_STANDOUT;
 117
 118
 119 /*
 120  * Define a charset, given a description string.
 121  * The string consists of 256 letters,
 122  * one for each character in the charset.
 123  * If the string is shorter than 256 letters, missing letters
 124  * are taken to be identical to the last one.
 125  * A decimal number followed by a letter is taken to be a
 126  * repetition of the letter.
 127  *
 128  * Each letter is one of:
 129  *      . normal character
 130  *      b binary character
 131  *      c control character
 132  */
 133         static void
 134 ichardef(s)
 135         char *s;
 136 {
 137         register char *cp;
 138         register int n;
 139         register char v;
 140
 141         n = 0;
 142         v = 0;
 143         cp = chardef;
 144         while (*s != '\0')
 145         {
 146                 switch (*s++)
 147                 {
 148                 case '.':
 149                         v = 0;
 150                         break;
 151                 case 'c':
 152                         v = IS_CONTROL_CHAR;
 153                         break;
 154                 case 'b':
 155                         v = IS_BINARY_CHAR|IS_CONTROL_CHAR;
 156                         break;
 157
 158                 case '0': case '1': case '2': case '3': case '4':
 159                 case '5': case '6': case '7': case '8': case '9':
 160                         n = (10 * n) + (s[-1] - '0');
 161                         continue;
 162
 163                 default:
 164                         error("invalid chardef", NULL_PARG);
 165                         quit(QUIT_ERROR);
 166                         /*NOTREACHED*/
 167                 }
 168
 169                 do
 170                 {
 171                         if (cp >= chardef + sizeof(chardef))
 172                         {
 173                                 error("chardef longer than 256", NULL_PARG);
 174                                 quit(QUIT_ERROR);
 175                                 /*NOTREACHED*/
 176                         }
 177                         *cp++ = v;
 178                 } while (--n > 0);
 179                 n = 0;
 180         }
 181
 182         while (cp < chardef + sizeof(chardef))
 183                 *cp++ = v;
 184 }
 185
 186 /*
 187  * Define a charset, given a charset name.
 188  * The valid charset names are listed in the "charsets" array.
 189  */
 190         static int
 191 icharset(name, no_error)
 192         register char *name;
 193         int no_error;
 194 {
 195         register struct charset *p;
 196         register struct cs_alias *a;
 197
 198         if (name == NULL || *name == '\0')
 199                 return (0);
 200
 201         /* First see if the name is an alias. */
 202         for (a = cs_aliases;  a->name != NULL;  a++)
 203         {
 204                 if (strcmp(name, a->name) == 0)
 205                 {
 206                         name = a->oname;
 207                         break;
 208                 }
 209         }
 210
 211         for (p = charsets;  p->name != NULL;  p++)
 212         {
 213                 if (strcmp(name, p->name) == 0)
 214                 {
 215                         ichardef(p->desc);
 216                         if (p->p_flag != NULL)
 217                                 *(p->p_flag) = 1;
 218                         return (1);
 219                 }
 220         }
 221
 222         if (!no_error) {
 223                 error("invalid charset name", NULL_PARG);
 224                 quit(QUIT_ERROR);
 225         }
 226         return (0);
 227 }
 228
 229 #if HAVE_LOCALE
 230 /*
 231  * Define a charset, given a locale name.
 232  */
 233         static void
 234 ilocale()
 235 {
 236         register int c;
 237
 238         for (c = 0;  c < (int) sizeof(chardef);  c++)
 239         {
 240                 if (isprint(c))
 241                         chardef[c] = 0;
 242                 else if (iscntrl(c))
 243                         chardef[c] = IS_CONTROL_CHAR;
 244                 else
 245                         chardef[c] = IS_BINARY_CHAR|IS_CONTROL_CHAR;
 246         }
 247 }
 248 #endif
 249
 250 /*
 251  * Define the printing format for control (or binary utf) chars.
 252  */
 253         static void
 254 setbinfmt(s, fmtvarptr, default_fmt)
 255         char *s;
 256         char **fmtvarptr;
 257         char *default_fmt;
 258 {
 259         if (s && utf_mode)
 260         {
 261                 /* It would be too hard to account for width otherwise.  */
 262                 char *t = s;
 263                 while (*t)
 264                 {
 265                         if (*t < ' ' || *t > '~')
 266                         {
 267                                 s = default_fmt;
 268                                 goto attr;
 269                         }
 270                         t++;
 271                 }
 272         }
 273
 274         /* %n is evil */
 275         if (s == NULL || *s == '\0' ||
 276             (*s == '*' && (s[1] == '\0' || s[2] == '\0' || strchr(s + 2, 'n'))) ||
 277             (*s != '*' && strchr(s, 'n')))
 278                 s = default_fmt;
 279
 280         /*
 281          * Select the attributes if it starts with "*".
 282          */
 283  attr:
 284         if (*s == '*')
 285         {
 286                 switch (s[1])
 287                 {
 288                 case 'd':  binattr = AT_BOLD;      break;
 289                 case 'k':  binattr = AT_BLINK;     break;
 290                 case 's':  binattr = AT_STANDOUT;  break;
 291                 case 'u':  binattr = AT_UNDERLINE; break;
 292                 default:   binattr = AT_NORMAL;    break;
 293                 }
 294                 s += 2;
 295         }
 296         *fmtvarptr = s;
 297 }
 298
 299 /*
 300  *
 301  */
 302         static void
 303 set_charset()
 304 {
 305         char *s;
 306
 307         /*
 308          * See if environment variable LESSCHARSET is defined.
 309          */
 310         s = lgetenv("LESSCHARSET");
 311         if (icharset(s, 0))
 312                 return;
 313
 314         /*
 315          * LESSCHARSET is not defined: try LESSCHARDEF.
 316          */
 317         s = lgetenv("LESSCHARDEF");
 318         if (s != NULL && *s != '\0')
 319         {
 320                 ichardef(s);
 321                 return;
 322         }
 323
 324 #if HAVE_LOCALE
 325         /*
 326          * Try using the codeset name as the charset name.
 327          */
 328         s = nl_langinfo(CODESET);
 329         if (icharset(s, 1))
 330                 return;
 331 #endif
 332
 333 #if HAVE_STRSTR
 334         /*
 335          * Check whether LC_ALL, LC_CTYPE or LANG look like UTF-8 is used.
 336          */
 337         if ((s = lgetenv("LC_ALL")) != NULL ||
 338             (s = lgetenv("LC_CTYPE")) != NULL ||
 339             (s = lgetenv("LANG")) != NULL)
 340         {
 341                 if (   strstr(s, "UTF-8") != NULL || strstr(s, "utf-8") != NULL
 342                     || strstr(s, "UTF8")  != NULL || strstr(s, "utf8")  != NULL)
 343                         if (icharset("utf-8", 1))
 344                                 return;
 345         }
 346 #endif
 347
 348 #if HAVE_LOCALE
 349         /*
 350          * Get character definitions from locale functions,
 351          * rather than from predefined charset entry.
 352          */
 353         ilocale();
 354 #if MSDOS_COMPILER
 355         /*
 356          * Default to "dos".
 357          */
 358         (void) icharset("dos", 1);
 359 #else
 360         /*
 361          * Default to "latin1".
 362          */
 363         (void) icharset("latin1", 1);
 364 #endif
 365 #endif
 366 }
 367
 368 /*
 369  * Initialize charset data structures.
 370  */
 371         public void
 372 init_charset()
 373 {
 374         char *s;
 375
 376 #if HAVE_LOCALE
 377         setlocale(LC_ALL, "");
 378 #endif
 379
 380         set_charset();
 381
 382         s = lgetenv("LESSBINFMT");
 383         setbinfmt(s, &binfmt, "*s<%02X>");
 384
 385         s = lgetenv("LESSUTFBINFMT");
 386         setbinfmt(s, &utfbinfmt, "<U+%04lX>");
 387 }
 388
 389 /*
 390  * Is a given character a "binary" character?
 391  */
 392         public int
 393 binary_char(c)
 394         unsigned char c;
 395 {
 396         c &= 0377;
 397         return (chardef[c] & IS_BINARY_CHAR);
 398 }
 399
 400 /*
 401  * Is a given character a "control" character?
 402  */
 403         public int
 404 control_char(c)
 405         int c;
 406 {
 407         c &= 0377;
 408         return (chardef[c] & IS_CONTROL_CHAR);
 409 }
 410
 411 /*
 412  * Return the printable form of a character.
 413  * For example, in the "ascii" charset '\3' is printed as "^C".
 414  */
 415         public char *
 416 prchar(c)
 417         int c;
 418 {
 419         /* {{ This buffer can be overrun if LESSBINFMT is a long string. }} */
 420         static char buf[32];
 421
 422         c &= 0377;
 423         if ((c < 128 || !utf_mode) && !control_char(c))
 424                 SNPRINTF1(buf, sizeof(buf), "%c", c);
 425         else if (c == ESC)
 426                 strcpy(buf, "ESC");
 427 #if IS_EBCDIC_HOST
 428         else if (!binary_char(c) && c < 64)
 429                 SNPRINTF1(buf, sizeof(buf), "^%c",
 430                 /*
 431                  * This array roughly inverts CONTROL() #defined in less.h,
 432                  * and should be kept in sync with CONTROL() and IBM-1047.
 433                  */
 434                 "@ABC.I.?...KLMNO"
 435                 "PQRS.JH.XY.."
 436                 "\\]^_"
 437                 "......W[.....EFG"
 438                 "..V....D....TU.Z"[c]);
 439 #else
 440         else if (c < 128 && !control_char(c ^ 0100))
 441                 SNPRINTF1(buf, sizeof(buf), "^%c", c ^ 0100);
 442 #endif
 443         else
 444                 SNPRINTF1(buf, sizeof(buf), binfmt, c);
 445         return (buf);
 446 }
 447
 448 /*
 449  * Return the printable form of a UTF-8 character.
 450  */
 451         public char *
 452 prutfchar(ch)
 453         LWCHAR ch;
 454 {
 455         static char buf[32];
 456
 457         if (ch == ESC)
 458                 strcpy(buf, "ESC");
 459         else if (ch < 128 && control_char(ch))
 460         {
 461                 if (!control_char(ch ^ 0100))
 462                         SNPRINTF1(buf, sizeof(buf), "^%c", ((char) ch) ^ 0100);
 463                 else
 464                         SNPRINTF1(buf, sizeof(buf), binfmt, (char) ch);
 465         } else if (is_ubin_char(ch))
 466                 SNPRINTF1(buf, sizeof(buf), utfbinfmt, ch);
 467         else
 468         {
 469                 int len;
 470                 if (ch >= 0x80000000)
 471                 {
 472                         len = 3;
 473                         ch = 0xFFFD;
 474                 } else
 475                 {
 476                         len =   (ch < 0x80) ? 1
 477                               : (ch < 0x800) ? 2
 478                               : (ch < 0x10000) ? 3
 479                               : (ch < 0x200000) ? 4
 480                               : (ch < 0x4000000) ? 5
 481                               : 6;
 482                 }
 483                 buf[len] = '\0';
 484                 if (len == 1)
 485                         *buf = (char) ch;
 486                 else
 487                 {
 488                         *buf = ((1 << len) - 1) << (8 - len);
 489                         while (--len > 0)
 490                         {
 491                                 buf[len] = (char) (0x80 | (ch & 0x3F));
 492                                 ch >>= 6;
 493                         }
 494                         *buf |= ch;
 495                 }
 496         }
 497         return (buf);
 498 }
 499
 500 /*
 501  * Get the length of a UTF-8 character in bytes.
 502  */
 503         public int
 504 utf_len(ch)
 505         char ch;
 506 {
 507         if ((ch & 0x80) == 0)
 508                 return 1;
 509         if ((ch & 0xE0) == 0xC0)
 510                 return 2;
 511         if ((ch & 0xF0) == 0xE0)
 512                 return 3;
 513         if ((ch & 0xF8) == 0xF0)
 514                 return 4;
 515         if ((ch & 0xFC) == 0xF8)
 516                 return 5;
 517         if ((ch & 0xFE) == 0xFC)
 518                 return 6;
 519         /* Invalid UTF-8 encoding. */
 520         return 1;
 521 }
 522
 523 /*
 524  * Is a UTF-8 character well-formed?
 525  */
 526         public int
 527 is_utf8_well_formed(s)
 528         unsigned char *s;
 529 {
 530         int i;
 531         int len;
 532
 533         if (IS_UTF8_INVALID(s[0]))
 534                 return (0);
 535
 536         len = utf_len((char) s[0]);
 537         if (len == 1)
 538                 return (1);
 539         if (len == 2)
 540         {
 541                 if (s[0] < 0xC2)
 542                     return (0);
 543         } else
 544         {
 545                 unsigned char mask;
 546                 mask = (~((1 << (8-len)) - 1)) & 0xFF;
 547                 if (s[0] == mask && (s[1] & mask) == 0x80)
 548                         return (0);
 549         }
 550
 551         for (i = 1;  i < len;  i++)
 552                 if (!IS_UTF8_TRAIL(s[i]))
 553                         return (0);
 554         return (1);
 555 }
 556
 557 /*
 558  * Get the value of a UTF-8 character.
 559  */
 560         public LWCHAR
 561 get_wchar(p)
 562         char *p;
 563 {
 564         switch (utf_len(p[0]))
 565         {
 566         case 1:
 567         default:
 568                 return (LWCHAR)
 569                         (p[0] & 0xFF);
 570         case 2:
 571                 return (LWCHAR) (
 572                         ((p[0] & 0x1F) << 6) |
 573                         (p[1] & 0x3F));
 574         case 3:
 575                 return (LWCHAR) (
 576                         ((p[0] & 0x0F) << 12) |
 577                         ((p[1] & 0x3F) << 6) |
 578                         (p[2] & 0x3F));
 579         case 4:
 580                 return (LWCHAR) (
 581                         ((p[0] & 0x07) << 18) |
 582                         ((p[1] & 0x3F) << 12) |
 583                         ((p[2] & 0x3F) << 6) |
 584                         (p[3] & 0x3F));
 585         case 5:
 586                 return (LWCHAR) (
 587                         ((p[0] & 0x03) << 24) |
 588                         ((p[1] & 0x3F) << 18) |
 589                         ((p[2] & 0x3F) << 12) |
 590                         ((p[3] & 0x3F) << 6) |
 591                         (p[4] & 0x3F));
 592         case 6:
 593                 return (LWCHAR) (
 594                         ((p[0] & 0x01) << 30) |
 595                         ((p[1] & 0x3F) << 24) |
 596                         ((p[2] & 0x3F) << 18) |
 597                         ((p[3] & 0x3F) << 12) |
 598                         ((p[4] & 0x3F) << 6) |
 599                         (p[5] & 0x3F));
 600         }
 601 }
 602
 603 /*
 604  * Step forward or backward one character in a string.
 605  */
 606         public LWCHAR
 607 step_char(pp, dir, limit)
 608         char **pp;
 609         signed int dir;
 610         char *limit;
 611 {
 612         LWCHAR ch;
 613         char *p = *pp;
 614
 615         if (!utf_mode)
 616         {
 617                 /* It's easy if chars are one byte. */
 618                 if (dir > 0)
 619                         ch = (LWCHAR) ((p < limit) ? *p++ : 0);
 620                 else
 621                         ch = (LWCHAR) ((p > limit) ? *--p : 0);
 622         } else if (dir > 0)
 623         {
 624                 if (p + utf_len(*p) > limit)
 625                         ch = 0;
 626                 else
 627                 {
 628                         ch = get_wchar(p);
 629                         p++;
 630                         while (IS_UTF8_TRAIL(*p))
 631                                 p++;
 632                 }
 633         } else
 634         {
 635                 while (p > limit && IS_UTF8_TRAIL(p[-1]))
 636                         p--;
 637                 if (p > limit)
 638                         ch = get_wchar(--p);
 639                 else
 640                         ch = 0;
 641         }
 642         *pp = p;
 643         return ch;
 644 }
 645
 646 /*
 647  * Unicode characters data
 648  */
 649 struct wchar_range { LWCHAR first, last; };
 650
 651 static struct wchar_range comp_table[] = {
 652         {0x300,0x357}, {0x35d,0x36f}, {0x483,0x486}, {0x488,0x489},
 653         {0x591,0x5a1}, {0x5a3,0x5b9}, {0x5bb,0x5bd}, {0x5bf,0x5bf},
 654         {0x5c1,0x5c2}, {0x5c4,0x5c4}, {0x610,0x615}, {0x64b,0x658},
 655         {0x670,0x670}, {0x6d6,0x6dc}, {0x6de,0x6e4}, {0x6e7,0x6e8},
 656         {0x6ea,0x6ed}, {0x711,0x711}, {0x730,0x74a}, {0x7a6,0x7b0},
 657         {0x901,0x902}, {0x93c,0x93c}, {0x941,0x948}, {0x94d,0x94d},
 658         {0x951,0x954}, {0x962,0x963}, {0x981,0x981}, {0x9bc,0x9bc},
 659         {0x9c1,0x9c4}, {0x9cd,0x9cd}, {0x9e2,0x9e3}, {0xa01,0xa02},
 660         {0xa3c,0xa3c}, {0xa41,0xa42}, {0xa47,0xa48}, {0xa4b,0xa4d},
 661         {0xa70,0xa71}, {0xa81,0xa82}, {0xabc,0xabc}, {0xac1,0xac5},
 662         {0xac7,0xac8}, {0xacd,0xacd}, {0xae2,0xae3}, {0xb01,0xb01},
 663         {0xb3c,0xb3c}, {0xb3f,0xb3f}, {0xb41,0xb43}, {0xb4d,0xb4d},
 664         {0xb56,0xb56}, {0xb82,0xb82}, {0xbc0,0xbc0}, {0xbcd,0xbcd},
 665         {0xc3e,0xc40}, {0xc46,0xc48}, {0xc4a,0xc4d}, {0xc55,0xc56},
 666         {0xcbc,0xcbc}, {0xcbf,0xcbf}, {0xcc6,0xcc6}, {0xccc,0xccd},
 667         {0xd41,0xd43}, {0xd4d,0xd4d}, {0xdca,0xdca}, {0xdd2,0xdd4},
 668         {0xdd6,0xdd6}, {0xe31,0xe31}, {0xe34,0xe3a}, {0xe47,0xe4e},
 669         {0xeb1,0xeb1}, {0xeb4,0xeb9}, {0xebb,0xebc}, {0xec8,0xecd},
 670         {0xf18,0xf19}, {0xf35,0xf35}, {0xf37,0xf37}, {0xf39,0xf39},
 671         {0xf71,0xf7e}, {0xf80,0xf84}, {0xf86,0xf87}, {0xf90,0xf97},
 672         {0xf99,0xfbc}, {0xfc6,0xfc6}, {0x102d,0x1030}, {0x1032,0x1032},
 673         {0x1036,0x1037}, {0x1039,0x1039}, {0x1058,0x1059},
 674         {0x1712,0x1714}, {0x1732,0x1734}, {0x1752,0x1753},
 675         {0x1772,0x1773}, {0x17b7,0x17bd}, {0x17c6,0x17c6},
 676         {0x17c9,0x17d3}, {0x17dd,0x17dd}, {0x180b,0x180d},
 677         {0x18a9,0x18a9}, {0x1920,0x1922}, {0x1927,0x1928},
 678         {0x1932,0x1932}, {0x1939,0x193b}, {0x20d0,0x20ea},
 679         {0x302a,0x302f}, {0x3099,0x309a}, {0xfb1e,0xfb1e},
 680         {0xfe00,0xfe0f}, {0xfe20,0xfe23}, {0x1d167,0x1d169},
 681         {0x1d17b,0x1d182}, {0x1d185,0x1d18b}, {0x1d1aa,0x1d1ad},
 682         {0xe0100,0xe01ef},
 683 };
 684
 685 static struct wchar_range comb_table[] = {
 686         {0x0644,0x0622}, {0x0644,0x0623}, {0x0644,0x0625}, {0x0644,0x0627},
 687 };
 688
 689 /*
 690  * Characters with general category values
 691  *      Cc: Other, Control
 692  *      Cf: Other, Format
 693  *      Cs: Other, Surrogate
 694  *      Co: Other, Private Use
 695  *      Cn: Other, Not Assigned
 696  *      Zl: Separator, Line
 697  *      Zp: Separator, Paragraph
 698  */
 699 static struct wchar_range ubin_table[] = {
 700         {  0x0000,  0x001f} /* Cc */, {  0x007f,  0x009f} /* Cc */,
 701 #if 0
 702         {  0x00ad,  0x00ad} /* Cf */,
 703 #endif
 704         {  0x0237,  0x024f} /* Cn */, {  0x0358,  0x035c} /* Cn */,
 705         {  0x0370,  0x0373} /* Cn */, {  0x0376,  0x0379} /* Cn */,
 706         {  0x037b,  0x037d} /* Cn */, {  0x037f,  0x0383} /* Cn */,
 707         {  0x038b,  0x038b} /* Cn */, {  0x038d,  0x038d} /* Cn */,
 708         {  0x03a2,  0x03a2} /* Cn */, {  0x03cf,  0x03cf} /* Cn */,
 709         {  0x03fc,  0x03ff} /* Cn */, {  0x0487,  0x0487} /* Cn */,
 710         {  0x04cf,  0x04cf} /* Cn */, {  0x04f6,  0x04f7} /* Cn */,
 711         {  0x04fa,  0x04ff} /* Cn */, {  0x0510,  0x0530} /* Cn */,
 712         {  0x0557,  0x0558} /* Cn */, {  0x0560,  0x0560} /* Cn */,
 713         {  0x0588,  0x0588} /* Cn */, {  0x058b,  0x0590} /* Cn */,
 714         {  0x05a2,  0x05a2} /* Cn */, {  0x05ba,  0x05ba} /* Cn */,
 715         {  0x05c5,  0x05cf} /* Cn */, {  0x05eb,  0x05ef} /* Cn */,
 716         {  0x05f5,  0x05ff} /* Cn */,
 717 #if 0
 718         {  0x0600,  0x0603} /* Cf */,
 719 #endif
 720         {  0x0604,  0x060b} /* Cn */, {  0x0616,  0x061a} /* Cn */,
 721         {  0x061c,  0x061e} /* Cn */, {  0x0620,  0x0620} /* Cn */,
 722         {  0x063b,  0x063f} /* Cn */, {  0x0659,  0x065f} /* Cn */,
 723 #if 0
 724         {  0x06dd,  0x06dd} /* Cf */,
 725 #endif
 726         {  0x070e,  0x070e} /* Cn */,
 727 #if 0
 728         {  0x070f,  0x070f} /* Cf */,
 729 #endif
 730         {  0x074b,  0x074c} /* Cn */, {  0x0750,  0x077f} /* Cn */,
 731         {  0x07b2,  0x0900} /* Cn */, {  0x093a,  0x093b} /* Cn */,
 732         {  0x094e,  0x094f} /* Cn */, {  0x0955,  0x0957} /* Cn */,
 733         {  0x0971,  0x0980} /* Cn */, {  0x0984,  0x0984} /* Cn */,
 734         {  0x098d,  0x098e} /* Cn */, {  0x0991,  0x0992} /* Cn */,
 735         {  0x09a9,  0x09a9} /* Cn */, {  0x09b1,  0x09b1} /* Cn */,
 736         {  0x09b3,  0x09b5} /* Cn */, {  0x09ba,  0x09bb} /* Cn */,
 737         {  0x09c5,  0x09c6} /* Cn */, {  0x09c9,  0x09ca} /* Cn */,
 738         {  0x09ce,  0x09d6} /* Cn */, {  0x09d8,  0x09db} /* Cn */,
 739         {  0x09de,  0x09de} /* Cn */, {  0x09e4,  0x09e5} /* Cn */,
 740         {  0x09fb,  0x0a00} /* Cn */, {  0x0a04,  0x0a04} /* Cn */,
 741         {  0x0a0b,  0x0a0e} /* Cn */, {  0x0a11,  0x0a12} /* Cn */,
 742         {  0x0a29,  0x0a29} /* Cn */, {  0x0a31,  0x0a31} /* Cn */,
 743         {  0x0a34,  0x0a34} /* Cn */, {  0x0a37,  0x0a37} /* Cn */,
 744         {  0x0a3a,  0x0a3b} /* Cn */, {  0x0a3d,  0x0a3d} /* Cn */,
 745         {  0x0a43,  0x0a46} /* Cn */, {  0x0a49,  0x0a4a} /* Cn */,
 746         {  0x0a4e,  0x0a58} /* Cn */, {  0x0a5d,  0x0a5d} /* Cn */,
 747         {  0x0a5f,  0x0a65} /* Cn */, {  0x0a75,  0x0a80} /* Cn */,
 748         {  0x0a84,  0x0a84} /* Cn */, {  0x0a8e,  0x0a8e} /* Cn */,
 749         {  0x0a92,  0x0a92} /* Cn */, {  0x0aa9,  0x0aa9} /* Cn */,
 750         {  0x0ab1,  0x0ab1} /* Cn */, {  0x0ab4,  0x0ab4} /* Cn */,
 751         {  0x0aba,  0x0abb} /* Cn */, {  0x0ac6,  0x0ac6} /* Cn */,
 752         {  0x0aca,  0x0aca} /* Cn */, {  0x0ace,  0x0acf} /* Cn */,
 753         {  0x0ad1,  0x0adf} /* Cn */, {  0x0ae4,  0x0ae5} /* Cn */,
 754         {  0x0af0,  0x0af0} /* Cn */, {  0x0af2,  0x0b00} /* Cn */,
 755         {  0x0b04,  0x0b04} /* Cn */, {  0x0b0d,  0x0b0e} /* Cn */,
 756         {  0x0b11,  0x0b12} /* Cn */, {  0x0b29,  0x0b29} /* Cn */,
 757         {  0x0b31,  0x0b31} /* Cn */, {  0x0b34,  0x0b34} /* Cn */,
 758         {  0x0b3a,  0x0b3b} /* Cn */, {  0x0b44,  0x0b46} /* Cn */,
 759         {  0x0b49,  0x0b4a} /* Cn */, {  0x0b4e,  0x0b55} /* Cn */,
 760         {  0x0b58,  0x0b5b} /* Cn */, {  0x0b5e,  0x0b5e} /* Cn */,
 761         {  0x0b62,  0x0b65} /* Cn */, {  0x0b72,  0x0b81} /* Cn */,
 762         {  0x0b84,  0x0b84} /* Cn */, {  0x0b8b,  0x0b8d} /* Cn */,
 763         {  0x0b91,  0x0b91} /* Cn */, {  0x0b96,  0x0b98} /* Cn */,
 764         {  0x0b9b,  0x0b9b} /* Cn */, {  0x0b9d,  0x0b9d} /* Cn */,
 765         {  0x0ba0,  0x0ba2} /* Cn */, {  0x0ba5,  0x0ba7} /* Cn */,
 766         {  0x0bab,  0x0bad} /* Cn */, {  0x0bb6,  0x0bb6} /* Cn */,
 767         {  0x0bba,  0x0bbd} /* Cn */, {  0x0bc3,  0x0bc5} /* Cn */,
 768         {  0x0bc9,  0x0bc9} /* Cn */, {  0x0bce,  0x0bd6} /* Cn */,
 769         {  0x0bd8,  0x0be6} /* Cn */, {  0x0bfb,  0x0c00} /* Cn */,
 770         {  0x0c04,  0x0c04} /* Cn */, {  0x0c0d,  0x0c0d} /* Cn */,
 771         {  0x0c11,  0x0c11} /* Cn */, {  0x0c29,  0x0c29} /* Cn */,
 772         {  0x0c34,  0x0c34} /* Cn */, {  0x0c3a,  0x0c3d} /* Cn */,
 773         {  0x0c45,  0x0c45} /* Cn */, {  0x0c49,  0x0c49} /* Cn */,
 774         {  0x0c4e,  0x0c54} /* Cn */, {  0x0c57,  0x0c5f} /* Cn */,
 775         {  0x0c62,  0x0c65} /* Cn */, {  0x0c70,  0x0c81} /* Cn */,
 776         {  0x0c84,  0x0c84} /* Cn */, {  0x0c8d,  0x0c8d} /* Cn */,
 777         {  0x0c91,  0x0c91} /* Cn */, {  0x0ca9,  0x0ca9} /* Cn */,
 778         {  0x0cb4,  0x0cb4} /* Cn */, {  0x0cba,  0x0cbb} /* Cn */,
 779         {  0x0cc5,  0x0cc5} /* Cn */, {  0x0cc9,  0x0cc9} /* Cn */,
 780         {  0x0cce,  0x0cd4} /* Cn */, {  0x0cd7,  0x0cdd} /* Cn */,
 781         {  0x0cdf,  0x0cdf} /* Cn */, {  0x0ce2,  0x0ce5} /* Cn */,
 782         {  0x0cf0,  0x0d01} /* Cn */, {  0x0d04,  0x0d04} /* Cn */,
 783         {  0x0d0d,  0x0d0d} /* Cn */, {  0x0d11,  0x0d11} /* Cn */,
 784         {  0x0d29,  0x0d29} /* Cn */, {  0x0d3a,  0x0d3d} /* Cn */,
 785         {  0x0d44,  0x0d45} /* Cn */, {  0x0d49,  0x0d49} /* Cn */,
 786         {  0x0d4e,  0x0d56} /* Cn */, {  0x0d58,  0x0d5f} /* Cn */,
 787         {  0x0d62,  0x0d65} /* Cn */, {  0x0d70,  0x0d81} /* Cn */,
 788         {  0x0d84,  0x0d84} /* Cn */, {  0x0d97,  0x0d99} /* Cn */,
 789         {  0x0db2,  0x0db2} /* Cn */, {  0x0dbc,  0x0dbc} /* Cn */,
 790         {  0x0dbe,  0x0dbf} /* Cn */, {  0x0dc7,  0x0dc9} /* Cn */,
 791         {  0x0dcb,  0x0dce} /* Cn */, {  0x0dd5,  0x0dd5} /* Cn */,
 792         {  0x0dd7,  0x0dd7} /* Cn */, {  0x0de0,  0x0df1} /* Cn */,
 793         {  0x0df5,  0x0e00} /* Cn */, {  0x0e3b,  0x0e3e} /* Cn */,
 794         {  0x0e5c,  0x0e80} /* Cn */, {  0x0e83,  0x0e83} /* Cn */,
 795         {  0x0e85,  0x0e86} /* Cn */, {  0x0e89,  0x0e89} /* Cn */,
 796         {  0x0e8b,  0x0e8c} /* Cn */, {  0x0e8e,  0x0e93} /* Cn */,
 797         {  0x0e98,  0x0e98} /* Cn */, {  0x0ea0,  0x0ea0} /* Cn */,
 798         {  0x0ea4,  0x0ea4} /* Cn */, {  0x0ea6,  0x0ea6} /* Cn */,
 799         {  0x0ea8,  0x0ea9} /* Cn */, {  0x0eac,  0x0eac} /* Cn */,
 800         {  0x0eba,  0x0eba} /* Cn */, {  0x0ebe,  0x0ebf} /* Cn */,
 801         {  0x0ec5,  0x0ec5} /* Cn */, {  0x0ec7,  0x0ec7} /* Cn */,
 802         {  0x0ece,  0x0ecf} /* Cn */, {  0x0eda,  0x0edb} /* Cn */,
 803         {  0x0ede,  0x0eff} /* Cn */, {  0x0f48,  0x0f48} /* Cn */,
 804         {  0x0f6b,  0x0f70} /* Cn */, {  0x0f8c,  0x0f8f} /* Cn */,
 805         {  0x0f98,  0x0f98} /* Cn */, {  0x0fbd,  0x0fbd} /* Cn */,
 806         {  0x0fcd,  0x0fce} /* Cn */, {  0x0fd0,  0x0fff} /* Cn */,
 807         {  0x1022,  0x1022} /* Cn */, {  0x1028,  0x1028} /* Cn */,
 808         {  0x102b,  0x102b} /* Cn */, {  0x1033,  0x1035} /* Cn */,
 809         {  0x103a,  0x103f} /* Cn */, {  0x105a,  0x109f} /* Cn */,
 810         {  0x10c6,  0x10cf} /* Cn */, {  0x10f9,  0x10fa} /* Cn */,
 811         {  0x10fc,  0x10ff} /* Cn */, {  0x115a,  0x115e} /* Cn */,
 812         {  0x11a3,  0x11a7} /* Cn */, {  0x11fa,  0x11ff} /* Cn */,
 813         {  0x1207,  0x1207} /* Cn */, {  0x1247,  0x1247} /* Cn */,
 814         {  0x1249,  0x1249} /* Cn */, {  0x124e,  0x124f} /* Cn */,
 815         {  0x1257,  0x1257} /* Cn */, {  0x1259,  0x1259} /* Cn */,
 816         {  0x125e,  0x125f} /* Cn */, {  0x1287,  0x1287} /* Cn */,
 817         {  0x1289,  0x1289} /* Cn */, {  0x128e,  0x128f} /* Cn */,
 818         {  0x12af,  0x12af} /* Cn */, {  0x12b1,  0x12b1} /* Cn */,
 819         {  0x12b6,  0x12b7} /* Cn */, {  0x12bf,  0x12bf} /* Cn */,
 820         {  0x12c1,  0x12c1} /* Cn */, {  0x12c6,  0x12c7} /* Cn */,
 821         {  0x12cf,  0x12cf} /* Cn */, {  0x12d7,  0x12d7} /* Cn */,
 822         {  0x12ef,  0x12ef} /* Cn */, {  0x130f,  0x130f} /* Cn */,
 823         {  0x1311,  0x1311} /* Cn */, {  0x1316,  0x1317} /* Cn */,
 824         {  0x131f,  0x131f} /* Cn */, {  0x1347,  0x1347} /* Cn */,
 825         {  0x135b,  0x1360} /* Cn */, {  0x137d,  0x139f} /* Cn */,
 826         {  0x13f5,  0x1400} /* Cn */, {  0x1677,  0x167f} /* Cn */,
 827         {  0x169d,  0x169f} /* Cn */, {  0x16f1,  0x16ff} /* Cn */,
 828         {  0x170d,  0x170d} /* Cn */, {  0x1715,  0x171f} /* Cn */,
 829         {  0x1737,  0x173f} /* Cn */, {  0x1754,  0x175f} /* Cn */,
 830         {  0x176d,  0x176d} /* Cn */, {  0x1771,  0x1771} /* Cn */,
 831         {  0x1774,  0x177f} /* Cn */,
 832 #if 0
 833         {  0x17b4,  0x17b5} /* Cf */,
 834 #endif
 835         {  0x17de,  0x17df} /* Cn */, {  0x17ea,  0x17ef} /* Cn */,
 836         {  0x17fa,  0x17ff} /* Cn */, {  0x180f,  0x180f} /* Cn */,
 837         {  0x181a,  0x181f} /* Cn */, {  0x1878,  0x187f} /* Cn */,
 838         {  0x18aa,  0x18ff} /* Cn */, {  0x191d,  0x191f} /* Cn */,
 839         {  0x192c,  0x192f} /* Cn */, {  0x193c,  0x193f} /* Cn */,
 840         {  0x1941,  0x1943} /* Cn */, {  0x196e,  0x196f} /* Cn */,
 841         {  0x1975,  0x19df} /* Cn */, {  0x1a00,  0x1cff} /* Cn */,
 842         {  0x1d6c,  0x1dff} /* Cn */, {  0x1e9c,  0x1e9f} /* Cn */,
 843         {  0x1efa,  0x1eff} /* Cn */, {  0x1f16,  0x1f17} /* Cn */,
 844         {  0x1f1e,  0x1f1f} /* Cn */, {  0x1f46,  0x1f47} /* Cn */,
 845         {  0x1f4e,  0x1f4f} /* Cn */, {  0x1f58,  0x1f58} /* Cn */,
 846         {  0x1f5a,  0x1f5a} /* Cn */, {  0x1f5c,  0x1f5c} /* Cn */,
 847         {  0x1f5e,  0x1f5e} /* Cn */, {  0x1f7e,  0x1f7f} /* Cn */,
 848         {  0x1fb5,  0x1fb5} /* Cn */, {  0x1fc5,  0x1fc5} /* Cn */,
 849         {  0x1fd4,  0x1fd5} /* Cn */, {  0x1fdc,  0x1fdc} /* Cn */,
 850         {  0x1ff0,  0x1ff1} /* Cn */, {  0x1ff5,  0x1ff5} /* Cn */,
 851         {  0x1fff,  0x1fff} /* Cn */, {  0x200b,  0x200f} /* Cf */,
 852         {  0x2028,  0x2028} /* Zl */,
 853         {  0x2029,  0x2029} /* Zp */,
 854         {  0x202a,  0x202e} /* Cf */,
 855         {  0x2055,  0x2056} /* Cn */, {  0x2058,  0x205e} /* Cn */,
 856         {  0x2060,  0x2063} /* Cf */,
 857         {  0x2064,  0x2069} /* Cn */,
 858         {  0x206a,  0x206f} /* Cf */,
 859         {  0x2072,  0x2073} /* Cn */, {  0x208f,  0x209f} /* Cn */,
 860         {  0x20b2,  0x20cf} /* Cn */, {  0x20eb,  0x20ff} /* Cn */,
 861         {  0x213c,  0x213c} /* Cn */, {  0x214c,  0x2152} /* Cn */,
 862         {  0x2184,  0x218f} /* Cn */, {  0x23d1,  0x23ff} /* Cn */,
 863         {  0x2427,  0x243f} /* Cn */, {  0x244b,  0x245f} /* Cn */,
 864         {  0x2618,  0x2618} /* Cn */, {  0x267e,  0x267f} /* Cn */,
 865         {  0x2692,  0x269f} /* Cn */, {  0x26a2,  0x2700} /* Cn */,
 866         {  0x2705,  0x2705} /* Cn */, {  0x270a,  0x270b} /* Cn */,
 867         {  0x2728,  0x2728} /* Cn */, {  0x274c,  0x274c} /* Cn */,
 868         {  0x274e,  0x274e} /* Cn */, {  0x2753,  0x2755} /* Cn */,
 869         {  0x2757,  0x2757} /* Cn */, {  0x275f,  0x2760} /* Cn */,
 870         {  0x2795,  0x2797} /* Cn */, {  0x27b0,  0x27b0} /* Cn */,
 871         {  0x27bf,  0x27cf} /* Cn */, {  0x27ec,  0x27ef} /* Cn */,
 872         {  0x2b0e,  0x2e7f} /* Cn */, {  0x2e9a,  0x2e9a} /* Cn */,
 873         {  0x2ef4,  0x2eff} /* Cn */, {  0x2fd6,  0x2fef} /* Cn */,
 874         {  0x2ffc,  0x2fff} /* Cn */, {  0x3040,  0x3040} /* Cn */,
 875         {  0x3097,  0x3098} /* Cn */, {  0x3100,  0x3104} /* Cn */,
 876         {  0x312d,  0x3130} /* Cn */, {  0x318f,  0x318f} /* Cn */,
 877         {  0x31b8,  0x31ef} /* Cn */, {  0x321f,  0x321f} /* Cn */,
 878         {  0x3244,  0x324f} /* Cn */, {  0x327e,  0x327e} /* Cn */,
 879         {  0x32ff,  0x32ff} /* Cn */, {  0x4db6,  0x4dbf} /* Cn */,
 880         {  0x9fa6,  0x9fff} /* Cn */, {  0xa48d,  0xa48f} /* Cn */,
 881         {  0xa4c7,  0xabff} /* Cn */, {  0xd7a4,  0xd7ff} /* Cn */,
 882         {  0xd800,  0xdfff} /* Cs */,
 883         {  0xe000,  0xf8ff} /* Co */,
 884         {  0xfa2e,  0xfa2f} /* Cn */, {  0xfa6b,  0xfaff} /* Cn */,
 885         {  0xfb07,  0xfb12} /* Cn */, {  0xfb18,  0xfb1c} /* Cn */,
 886         {  0xfb37,  0xfb37} /* Cn */, {  0xfb3d,  0xfb3d} /* Cn */,
 887         {  0xfb3f,  0xfb3f} /* Cn */, {  0xfb42,  0xfb42} /* Cn */,
 888         {  0xfb45,  0xfb45} /* Cn */, {  0xfbb2,  0xfbd2} /* Cn */,
 889         {  0xfd40,  0xfd4f} /* Cn */, {  0xfd90,  0xfd91} /* Cn */,
 890         {  0xfdc8,  0xfdef} /* Cn */, {  0xfdfe,  0xfdff} /* Cn */,
 891         {  0xfe10,  0xfe1f} /* Cn */, {  0xfe24,  0xfe2f} /* Cn */,
 892         {  0xfe53,  0xfe53} /* Cn */, {  0xfe67,  0xfe67} /* Cn */,
 893         {  0xfe6c,  0xfe6f} /* Cn */, {  0xfe75,  0xfe75} /* Cn */,
 894         {  0xfefd,  0xfefe} /* Cn */,
 895         {  0xfeff,  0xfeff} /* Cf */,
 896         {  0xff00,  0xff00} /* Cn */, {  0xffbf,  0xffc1} /* Cn */,
 897         {  0xffc8,  0xffc9} /* Cn */, {  0xffd0,  0xffd1} /* Cn */,
 898         {  0xffd8,  0xffd9} /* Cn */, {  0xffdd,  0xffdf} /* Cn */,
 899         {  0xffe7,  0xffe7} /* Cn */, {  0xffef,  0xfff8} /* Cn */,
 900         {  0xfff9,  0xfffb} /* Cf */,
 901         {  0xfffe,  0xffff} /* Cn */, { 0x1000c, 0x1000c} /* Cn */,
 902         { 0x10027, 0x10027} /* Cn */, { 0x1003b, 0x1003b} /* Cn */,
 903         { 0x1003e, 0x1003e} /* Cn */, { 0x1004e, 0x1004f} /* Cn */,
 904         { 0x1005e, 0x1007f} /* Cn */, { 0x100fb, 0x100ff} /* Cn */,
 905         { 0x10103, 0x10106} /* Cn */, { 0x10134, 0x10136} /* Cn */,
 906         { 0x10140, 0x102ff} /* Cn */, { 0x1031f, 0x1031f} /* Cn */,
 907         { 0x10324, 0x1032f} /* Cn */, { 0x1034b, 0x1037f} /* Cn */,
 908         { 0x1039e, 0x1039e} /* Cn */, { 0x103a0, 0x103ff} /* Cn */,
 909         { 0x1049e, 0x1049f} /* Cn */, { 0x104aa, 0x107ff} /* Cn */,
 910         { 0x10806, 0x10807} /* Cn */, { 0x10809, 0x10809} /* Cn */,
 911         { 0x10836, 0x10836} /* Cn */, { 0x10839, 0x1083b} /* Cn */,
 912         { 0x1083d, 0x1083e} /* Cn */, { 0x10840, 0x1cfff} /* Cn */,
 913         { 0x1d0f6, 0x1d0ff} /* Cn */, { 0x1d127, 0x1d129} /* Cn */,
 914         { 0x1d173, 0x1d17a} /* Cf */,
 915         { 0x1d1de, 0x1d2ff} /* Cn */, { 0x1d357, 0x1d3ff} /* Cn */,
 916         { 0x1d455, 0x1d455} /* Cn */, { 0x1d49d, 0x1d49d} /* Cn */,
 917         { 0x1d4a0, 0x1d4a1} /* Cn */, { 0x1d4a3, 0x1d4a4} /* Cn */,
 918         { 0x1d4a7, 0x1d4a8} /* Cn */, { 0x1d4ad, 0x1d4ad} /* Cn */,
 919         { 0x1d4ba, 0x1d4ba} /* Cn */, { 0x1d4bc, 0x1d4bc} /* Cn */,
 920         { 0x1d4c4, 0x1d4c4} /* Cn */, { 0x1d506, 0x1d506} /* Cn */,
 921         { 0x1d50b, 0x1d50c} /* Cn */, { 0x1d515, 0x1d515} /* Cn */,
 922         { 0x1d51d, 0x1d51d} /* Cn */, { 0x1d53a, 0x1d53a} /* Cn */,
 923         { 0x1d53f, 0x1d53f} /* Cn */, { 0x1d545, 0x1d545} /* Cn */,
 924         { 0x1d547, 0x1d549} /* Cn */, { 0x1d551, 0x1d551} /* Cn */,
 925         { 0x1d6a4, 0x1d6a7} /* Cn */, { 0x1d7ca, 0x1d7cd} /* Cn */,
 926         { 0x1d800, 0x1ffff} /* Cn */, { 0x2a6d7, 0x2f7ff} /* Cn */,
 927         { 0x2fa1e, 0xe0000} /* Cn */,
 928         { 0xe0001, 0xe0001} /* Cf */,
 929         { 0xe0002, 0xe001f} /* Cn */,
 930         { 0xe0020, 0xe007f} /* Cf */,
 931         { 0xe0080, 0xe00ff} /* Cn */, { 0xe01f0, 0xeffff} /* Cn */,
 932         { 0xf0000, 0xffffd} /* Co */,
 933         { 0xffffe, 0xfffff} /* Cn */,
 934         {0x100000,0x10fffd} /* Co */,
 935         {0x10fffe,0x10ffff} /* Cn */,
 936         {0x110000,0x7fffffff} /* ISO 10646?? */
 937 };
 938
 939 /*
 940  * Double width characters
 941  *      W: East Asian Wide
 942  *      F: East Asian Full-width
 943  */
 944 static struct wchar_range wide_table[] = {
 945         {  0x1100,  0x115f} /* W */, {  0x2329,  0x232a} /* W */,
 946         {  0x2E80,  0x2FFB} /* W */,
 947         {  0x3000,  0x3000} /* F */,
 948         {  0x3001,  0x303E} /* W */, {  0x3041,  0x4DB5} /* W */,
 949         {  0x4E00,  0x9FA5} /* W */, {  0xA000,  0xA4C6} /* W */,
 950         {  0xAC00,  0xD7A3} /* W */, {  0xF900,  0xFA6A} /* W */,
 951         {  0xFE30,  0xFE6B} /* W */,
 952         {  0xFF01,  0xFF60} /* F */, {  0xFFE0,  0xFFE6} /* F */,
 953         { 0x20000, 0x2FFFD} /* W */, { 0x30000, 0x3FFFD} /* W */,
 954
 955 };
 956
 957         static int
 958 is_in_table(ch, table, tsize)
 959         LWCHAR ch;
 960         struct wchar_range table[];
 961         int tsize;
 962 {
 963         int hi;
 964         int lo;
 965
 966         /* Binary search in the table. */
 967         if (ch < table[0].first)
 968                 return 0;
 969         lo = 0;
 970         hi = tsize - 1;
 971         while (lo <= hi)
 972         {
 973                 int mid = (lo + hi) / 2;
 974                 if (ch > table[mid].last)
 975                         lo = mid + 1;
 976                 else if (ch < table[mid].first)
 977                         hi = mid - 1;
 978                 else
 979                         return 1;
 980         }
 981         return 0;
 982 }
 983
 984 /*
 985  * Is a character a UTF-8 composing character?
 986  * If a composing character follows any char, the two combine into one glyph.
 987  */
 988         public int
 989 is_composing_char(ch)
 990         LWCHAR ch;
 991 {
 992         return is_in_table(ch, comp_table, (sizeof(comp_table) / sizeof(*comp_table)));
 993 }
 994
 995 /*
 996  * Should this UTF-8 character be treated as binary?
 997  */
 998         public int
 999 is_ubin_char(ch)
1000         LWCHAR ch;
1001 {
1002         return is_in_table(ch, ubin_table, (sizeof(ubin_table) / sizeof(*ubin_table)));
1003 }
1004
1005 /*
1006  * Is this a double width UTF-8 character?
1007  */
1008         public int
1009 is_wide_char(ch)
1010         LWCHAR ch;
1011 {
1012         return is_in_table(ch, wide_table, (sizeof(wide_table) / sizeof(*wide_table)));
1013 }
1014
1015 /*
1016  * Is a character a UTF-8 combining character?
1017  * A combining char acts like an ordinary char, but if it follows
1018  * a specific char (not any char), the two combine into one glyph.
1019  */
1020         public int
1021 is_combining_char(ch1, ch2)
1022         LWCHAR ch1;
1023         LWCHAR ch2;
1024 {
1025         /* The table is small; use linear search. */
1026         int i;
1027         for (i = 0;  i < sizeof(comb_table)/sizeof(*comb_table);  i++)
1028         {
1029                 if (ch1 == comb_table[i].first &&
1030                     ch2 == comb_table[i].last)
1031                         return 1;
1032         }
1033         return 0;
1034 }
1035