From 236ac5fcb19a88c43de00e2ab309f57cb3e2d912 Mon Sep 17 00:00:00 2001
From: John Marino <draco@marino.st>
Date: Mon, 17 Aug 2015 00:54:44 +0200
Subject: [PATCH] locale gen tools: Set all UTF-8 to same rollup CTYPE

The CLDR CTYPE definitions are still used for non-UTF8 encodings,
but all UTF-8 locales now share a single "master" CTYPE that knows
all reasonable character sets.
---
 tools/tools/locale/etc/charmaps.xml  |  3 -
 tools/tools/locale/tools/cldr2def.pl | 84 ++++++++++++----------------
 tools/tools/locale/tools/finalize    | 31 +++++-----
 3 files changed, 49 insertions(+), 69 deletions(-)

diff --git a/tools/tools/locale/etc/charmaps.xml b/tools/tools/locale/etc/charmaps.xml
index a4ea28452f..052ccefdf7 100644
--- a/tools/tools/locale/etc/charmaps.xml
+++ b/tools/tools/locale/etc/charmaps.xml
@@ -181,9 +181,6 @@
 	<language name="uk"
 		encoding="CP1251 ISO8859-5 KOI8-U"
 		countries="UA" />
-	<language name="xx"
-		family="Comm"
-		countries="US" />               <!-- Special LC_CTYPE -->
 	<language name="zh"
 		family="Hans"
 		encoding="GB18030 GB2312 GBK eucCN"
diff --git a/tools/tools/locale/tools/cldr2def.pl b/tools/tools/locale/tools/cldr2def.pl
index b70b72d80a..dfc90ba1c2 100644
--- a/tools/tools/locale/tools/cldr2def.pl
+++ b/tools/tools/locale/tools/cldr2def.pl
@@ -348,19 +348,9 @@ sub transform_ctypes {
 		$file .= $c;
 		my $actfile = $file;
 
-		my $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src";
-		$filename = "$ETCDIR/$file.$DEFENCODING.src"
-		    if (! -f $filename);
-		if (! -f $filename
-		 && defined $languages{$l}{$f}{fallback}) {
-			$file = $languages{$l}{$f}{fallback};
-			$filename = "$CLDRDIR/posix/$file.$DEFENCODING.src";
-		}
-		$filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"
-		    if (! -f $filename);
+		my $filename = "$CLDRDIR/posix/xx_Comm_US.UTF-8.src";
 		if (! -f $filename) {
-			print STDERR
-			    "Cannot open $file.$DEFENCODING.src or fallback\n";
+			print STDERR "Cannot open $filename\n";
 			next;
 		}
 		open(FIN, "$filename");
@@ -370,49 +360,45 @@ sub transform_ctypes {
 		my $shex;
 		my $uhex;
 		while (<FIN>) {
-			if ((/^comment_char\s/) || (/^escape_char\s/)){
-				push @lines, $_;
-			}
-			if (/^LC_CTYPE/../^END LC_CTYPE/) {
-				push @lines, $_;
-			}
+			push @lines, $_;
 		}
 		close(FIN);
 		$shex = sha1_hex(join("\n", @lines));
 		$languages{$l}{$f}{data}{$c}{$DEFENCODING} = $shex;
 		$hashtable{$shex}{"${l}_${f}_${c}.$DEFENCODING"} = 1;
 		open(FOUT, ">$TYPE.draft/$actfile.$DEFENCODING.src");
-		print FOUT <<EOF;
-# Warning: Do not edit. This file is automatically extracted from the
-# tools in /usr/src/tools/tools/locale. The data is obtained from the
-# CLDR project, obtained from http://cldr.unicode.org/
-# -----------------------------------------------------------------------------
-EOF
-		my $category = '';
-		foreach my $line (@lines) {
-			if ($actfile eq "xx_Comm_US") {
-				print FOUT $line;
+		print FOUT @lines;
+		close(FOUT);
+		foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) {
+			next if ($enc eq $DEFENCODING);
+			$filename = "$CLDRDIR/posix/$file.$DEFENCODING.src";
+			if (! -f $filename) {
+				print STDERR "Cannot open $filename\n";
 				next;
 			}
-			if ($line =~ /^([a-z]{3,})\s+</) {
-				$category = $1;
-				if ($category eq 'print') {
-					print FOUT "blank\t<NO-BREAK_SPACE>\n";
-					print FOUT "print\t<NO-BREAK_SPACE>\n\n";
+			@lines = ();
+			open(FIN, "$filename");
+			while (<FIN>) {
+				if ((/^comment_char\s/) || (/^escape_char\s/)){
+					push @lines, $_;
+				}
+				if (/^LC_CTYPE/../^END LC_CTYPE/) {
+					push @lines, $_;
 				}
 			}
-			next if ($category eq 'print');
-			print FOUT $line;
-		}
-		close(FOUT);
-
-		foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) {
-			next if ($enc eq $DEFENCODING);
-			copy ("$TYPE.draft/$actfile.$DEFENCODING.src",
-			      "$TYPE.draft/$actfile.$enc.src");
+			close(FIN);
 			$uhex = sha1_hex(join("\n", @lines) . $enc);
 			$languages{$l}{$f}{data}{$c}{$enc} = $uhex;
 			$hashtable{$uhex}{"${l}_${f}_${c}.$enc"} = 1;
+			open(FOUT, ">$TYPE.draft/$actfile.$enc.src");
+			print FOUT <<EOF;
+# Warning: Do not edit. This file is automatically extracted from the
+# tools in /usr/src/tools/tools/locale. The data is obtained from the
+# CLDR project, obtained from http://cldr.unicode.org/
+# -----------------------------------------------------------------------------
+EOF
+			print FOUT @lines;
+			close(FOUT);
 		}
 	}
 	}
@@ -840,17 +826,17 @@ EOF
 				} keys(%{$hashtable{$hash}});
 		} elsif ($TYPE eq "ctypedef") {
 			@files = sort {
-				if ($a =~ /^en_x_US/ ||
-				    $a =~ /^en_x_GB.ISO8859-15/ ||
+				if ($a eq 'en_x_US.UTF-8') { return -1; }
+				elsif ($b eq 'en_x_US.UTF-8') { return 1; }
+				if ($a =~ /^en_x_US/) { return -1; }
+				elsif ($b =~ /^en_x_US/) { return 1; }
+
+				if ($a =~ /^en_x_GB.ISO8859-15/ ||
 				    $a =~ /^ru_x_RU/) { return -1; }
-				elsif ($b =~ /^en_x_US/ ||
-				       $b =~ /^en_x_GB.ISO8859-15/ ||
+				elsif ($b =~ /^en_x_GB.ISO8859-15/ ||
 				       $b =~ /ru_x_RU/) { return 1; }
 				else { return uc($b) cmp uc($a); }
 
-				if ($a eq 'en_x_US.UTF-8') { return -1; }
-				elsif ($b eq 'en_x_US.UTF-8') { return 1; }
-				else { return uc($b) cmp uc($a); }
 				} keys(%{$hashtable{$hash}});
 		} else {
 			@files = sort {
diff --git a/tools/tools/locale/tools/finalize b/tools/tools/locale/tools/finalize
index 3f07f2478b..ff24ee8fb0 100755
--- a/tools/tools/locale/tools/finalize
+++ b/tools/tools/locale/tools/finalize
@@ -31,29 +31,18 @@ AWKCMD="/## PLACEHOLDER/ { \
 
 grep '^LOCALES+' ${old}/Makefile > ${TEMP}
 
-if [ $1 = "ctypedef" ]
-then
-keep=$(cat ${TEMP} | awk '/UTF-8/ { print $2 }')
-else
-keep=$(cat ${TEMP} | awk '{ print $2 }')
-fi
-
-for original in ${keep}
-do
-	cp ${old}/${original}.src ${new}/
-done
-
 if [ $1 = "ctypedef" ]
 then
 keep=$(cat ${TEMP} | awk '{ print $2 ".src" }')
 (cd ${old} && md5 -r ${keep} | sort) > ${TEMP2}
-linx=$(cat ${TEMP2} | awk '!/UTF-8/ { print $2 }')
-for original in ${linx}
+keep=$(awk '{ if ($1 != last1) print $2; last1 = $1; }' ${TEMP2})
+for original in ${keep}
 do
-linkhash=$(fgrep "${original}" ${TEMP2} | awk '{ print $1 }')
-utf8file=$(fgrep "${linkhash}" ${TEMP2} | fgrep 'UTF-8' | awk '{ print $2 }')
-ln -s ${utf8file} ${new}/${original}
+	cp ${old}/${original} ${new}/
 done
+linx=$(awk '{ if ($1 == last1) { print "ln -s " last2 " ${new}/" $2 ";"} \
+else {last1 = $1; last2 = $2}}' ${TEMP2})
+eval $linx
 rm -f ${TEMP2}
 	/usr/bin/sed -E -e 's/[ ]+/ /g' \
 		${CLDRDIR}/posix/UTF-8.cm \
@@ -80,6 +69,14 @@ fi
 	echo map ${map} converted.
 done
 
+else  # below is everything but ctypedef
+
+keep=$(cat ${TEMP} | awk '{ print $2 }')
+for original in ${keep}
+do
+	cp ${old}/${original}.src ${new}/
+done
+
 fi
 
 grep -v '^LOCALES+' ${old}/Makefile | awk "${AWKCMD}" > ${new}/Makefile
-- 
2.41.0