#!/usr/bin/env python3 # Tool to read various files from the Unicode character database and # generate headers containing derived arrays and lookup tables needed # by PuTTY. # # The aim is to have this be a single tool which you can easily re-run # against a new version of Unicode, simply by pointing it at an # appropriate UCD.zip or a directory containing the same files # unpacked. import argparse import collections import io import os import string import sys import zipfile UCDRecord = collections.namedtuple('UCDRecord', [ 'c', 'General_Category', 'Canonical_Combining_Class', 'Bidi_Class', 'Decomposition_Type', 'Decomposition_Mapping', ]) def to_ranges(iterable): """Collect together adjacent ranges in a list of (key, value) pairs. The input iterable should deliver a sequence of (key, value) pairs in which the keys are integers in sorted order. The output is a sequence of tuples with structure ((start, end), value), each indicating that all the keys [start, start+1, ..., end] go with that value. """ start = end = val = None for k, v in iterable: if (k-1, v) == (end, val): end = k else: if start is not None: yield (start, end), val start, end, val = k, k, v if start is not None: yield (start, end), val def map_to_ranges(m): """Convert an integer-keyed map into a list of (range, value) pairs.""" yield from to_ranges(sorted(m.items())) def set_to_ranges(s): """Convert a set into a list of ranges.""" for r, _ in to_ranges((x, None) for x in sorted(s)): yield r def lines(iterable, keep_comments=False): """Deliver the lines of a Unicode data file. The input iterable should yield raw lines of the file: for example, it can be the file handle itself. The output values have their newlines removed, comments and trailing spaces deleted, and blank lines discarded. """ for line in iter(iterable): line = line.rstrip("\r\n") if not keep_comments: line = line.split("#", 1)[0] line = line.rstrip(" \t") if line == "": continue yield line class Main: def run(self): "Parse arguments and generate all the output files." parser = argparse.ArgumentParser( description='Build UCD-derived source files.') parser.add_argument("ucd", help="UCD to work from, either UCD.zip or " "a directory full of unpacked files.") args = parser.parse_args() if os.path.isdir(args.ucd): ucd_dir = args.ucd self.open_ucd_file = lambda filename: ( open(os.path.join(ucd_dir, filename))) else: ucd_zip = zipfile.ZipFile(args.ucd) self.open_ucd_file = lambda filename: ( io.TextIOWrapper(ucd_zip.open(filename))) self.find_unicode_version() with open("version.h", "w") as fh: self.write_version_header(fh) with open("bidi_type.h", "w") as fh: self.write_bidi_type_table(fh) with open("bidi_mirror.h", "w") as fh: self.write_bidi_mirroring_table(fh) with open("bidi_brackets.h", "w") as fh: self.write_bidi_brackets_table(fh) with open("nonspacing_chars.h", "w") as fh: self.write_nonspacing_chars_list(fh) with open("wide_chars.h", "w") as fh: self.write_wide_chars_list(fh) with open("ambiguous_wide_chars.h", "w") as fh: self.write_ambiguous_wide_chars_list(fh) with open("known_chars.h", "w") as fh: self.write_known_chars_table(fh) with open("combining_classes.h", "w") as fh: self.write_combining_class_table(fh) with open("canonical_decomp.h", "w") as fh: self.write_canonical_decomp_table(fh) with open("canonical_comp.h", "w") as fh: self.write_canonical_comp_table(fh) def find_unicode_version(self): """Find out the version of Unicode. This is read from the top of NamesList.txt, which has the closest thing to a machine-readable statement of the version number that I found in the whole collection of files. """ with self.open_ucd_file("NamesList.txt") as fh: for line in lines(fh): if line.startswith("@@@\t"): self.unicode_version_full = line[4:] self.unicode_version_short = " ".join( w for w in self.unicode_version_full.split(" ") if any(c in string.digits for c in w)) return @property def UnicodeData(self): """Records from UnicodeData.txt. Each yielded item is a UCDRecord tuple. """ with self.open_ucd_file("UnicodeData.txt") as fh: for line in lines(fh): # Split up the line into its raw fields. ( codepoint, name, category, cclass, bidiclass, decomp, num6, num7, num8, bidimirrored, obsolete_unicode1_name, obsolete_comment, uppercase, lowercase, titlecase, ) = line.split(";") # By default, we expect that this record describes # just one code point. codepoints = [int(codepoint, 16)] # Spot the special markers where consecutive lines say # and , indicating that the # entire range of code points in between are treated # the same. If so, we replace 'codepoints' with a # range object. if "<" in name: assert name.startswith("<") and name.endswith(">"), ( "Confusing < in character name: {!r}".format(line)) name_pieces = [piece.strip(" \t") for piece in name.lstrip("<").rstrip(">").split(",")] if "First" in name_pieces: assert isinstance(codepoints, list) prev_line_was_first = True prev_codepoint = codepoints[0] continue elif "Last" in name_pieces: assert prev_line_was_first codepoints = range(prev_codepoint, codepoints[0]+1) del prev_codepoint prev_line_was_first = False # Decode some of the raw fields into more cooked # forms. cclass = int(cclass) # Separate the decomposition field into decomposition # type and mapping. if decomp == "": dtype = decomp = None elif "<" not in decomp: dtype = 'canonical' else: assert decomp.startswith("<") dtype, decomp = decomp[1:].split(">", 1) decomp = decomp.lstrip(" ") # And decode the mapping part from hex strings to integers. if decomp is not None: decomp = [int(w, 16) for w in decomp.split(" ")] # And yield a UCDRecord for each code point in our # range. for codepoint in codepoints: yield UCDRecord( c=codepoint, General_Category=category, Canonical_Combining_Class=cclass, Bidi_Class=bidiclass, Decomposition_Type=dtype, Decomposition_Mapping=decomp, ) @property def BidiMirroring(self): """Parsed character pairs from BidiMirroring.txt. Each yielded tuple is a pair of Unicode code points. """ with self.open_ucd_file("BidiMirroring.txt") as fh: for line in lines(fh): cs1, cs2 = line.split(";") c1 = int(cs1, 16) c2 = int(cs2, 16) yield c1, c2 @property def BidiBrackets(self): """Bracket pairs from BidiBrackets.txt. Each yielded tuple is a pair of Unicode code points, followed by either 'o', 'c' or 'n' to indicate whether the first one is an open or closing parenthesis or neither. """ with self.open_ucd_file("BidiBrackets.txt") as fh: for line in lines(fh): cs1, cs2, kind = line.split(";") c1 = int(cs1, 16) c2 = int(cs2, 16) kind = kind.strip(" \t") yield c1, c2, kind @property def EastAsianWidth(self): """East Asian width types from EastAsianWidth.txt. Each yielded tuple is (code point, width type). """ with self.open_ucd_file("EastAsianWidth.txt") as fh: for line in lines(fh): fields = line.split(";") if ".." in fields[0]: start, end = [int(s, 16) for s in fields[0].split("..")] cs = range(start, end+1) else: cs = [int(fields[0], 16)] for c in cs: yield c, fields[1].strip() @property def CompositionExclusions(self): """Composition exclusions from CompositionExclusions.txt. Each yielded item is just a code point. """ with self.open_ucd_file("CompositionExclusions.txt") as fh: for line in lines(fh): yield int(line, 16) def write_file_header_comment(self, fh, description): print("/*", file=fh) print(" * Autogenerated by read_ucd.py from", self.unicode_version_full, file=fh) print(" *", file=fh) for line in description.strip("\n").split("\n"): print(" *" + (" " if line != "" else "") + line, file=fh) print(" */", file=fh) print(file=fh) def write_version_header(self, fh): self.write_file_header_comment(fh, """ String literals giving the currently supported version of Unicode. Useful for error messages and 'about' boxes. """) assert all(0x20 <= ord(c) < 0x7F and c != '"' for c in self.unicode_version_full) print("#define UNICODE_VERSION_FULL \"{}\"".format( self.unicode_version_full), file=fh) print("#define UNICODE_VERSION_SHORT \"{}\"".format( self.unicode_version_short), file=fh) def write_bidi_type_table(self, fh): self.write_file_header_comment(fh, """ Bidirectional type of every Unicode character, excluding those with type ON. Used by terminal/bidi.c, whose associated lookup function returns ON by default for anything not in this list. """) types = {} for rec in self.UnicodeData: if rec.Bidi_Class != "ON": types[rec.c] = rec.Bidi_Class for (start, end), t in map_to_ranges(types): print(f"{{0x{start:04x}, 0x{end:04x}, {t}}},", file=fh) def write_bidi_mirroring_table(self, fh): self.write_file_header_comment(fh, """ Map each Unicode character to its mirrored form when printing right to left. Used by terminal/bidi.c. """) bidi_mirror = {} for c1, c2 in self.BidiMirroring: assert bidi_mirror.get(c1, c2) == c2, f"Clash at {c1:%04X}" bidi_mirror[c1] = c2 assert bidi_mirror.get(c2, c1) == c1, f"Clash at {c2:%04X}" bidi_mirror[c2] = c1 for c1, c2 in sorted(bidi_mirror.items()): print("{{0x{:04x}, 0x{:04x}}},".format(c1, c2), file=fh) def write_bidi_brackets_table(self, fh): self.write_file_header_comment(fh, """ Identify Unicode characters that count as brackets for the purposes of bidirectional text layout. For each one, indicate whether it's an open or closed bracket, and identify up to two characters that can act as its counterpart. Used by terminal/bidi.c. """) bracket_map = {} for c1, c2, kind in self.BidiBrackets: bracket_map[c1] = kind, c2 equivalents = {} for rec in self.UnicodeData: if (rec.Decomposition_Type == 'canonical' and len(rec.Decomposition_Mapping) == 1): c = rec.c c2 = rec.Decomposition_Mapping[0] equivalents[c] = c2 equivalents[c2] = c for src, (kind, dst) in sorted(bracket_map.items()): dsteq = equivalents.get(dst, 0) # UCD claims there's an 'n' kind possible, but as of UCD # 14, no instances of it exist enumval = {'o': 'BT_OPEN', 'c': 'BT_CLOSE'}[kind] print("{{0x{:04x}, {{0x{:04x}, 0x{:04x}, {}}}}},".format( src, dst, dsteq, enumval), file=fh) def write_nonspacing_chars_list(self, fh): self.write_file_header_comment(fh, """ Identify Unicode characters that occupy no character cells of a terminal. Used by utils/wcwidth.c. """) cs = set() for rec in self.UnicodeData: nonspacing = rec.General_Category in {"Me", "Mn", "Cf"} if rec.c == 0xAD: # In typography this is a SOFT HYPHEN and counts as # discardable. But it's also an ISO 8859-1 printing # character, and all of those occupy one character # cell in a terminal. nonspacing = False if 0x1160 <= rec.c <= 0x11FF: # Medial (vowel) and final (consonant) jamo for # decomposed Hangul characters. These are regarded as # non-spacing on the grounds that they compose with # the preceding initial consonant. nonspacing = True if nonspacing: cs.add(rec.c) for start, end in set_to_ranges(cs): print(f"{{0x{start:04x}, 0x{end:04x}}},", file=fh) def write_width_table(self, fh, accept): cs = set() for c, wid in self.EastAsianWidth: if wid in accept: cs.add(c) for start, end in set_to_ranges(cs): print(f"{{0x{start:04x}, 0x{end:04x}}},", file=fh) def write_wide_chars_list(self, fh): self.write_file_header_comment(fh, """ Identify Unicode characters that occupy two adjacent character cells in a terminal. Used by utils/wcwidth.c. """) self.write_width_table(fh, {'W', 'F'}) def write_ambiguous_wide_chars_list(self, fh): self.write_file_header_comment(fh, """ Identify Unicode characters that are width-ambiguous: some regimes regard them as occupying two adjacent character cells in a terminal, and others do not. Used by utils/wcwidth.c. """) self.write_width_table(fh, {'A'}) def write_known_chars_table(self, fh): self.write_file_header_comment(fh, """ List the Unicode code points that are known to this version of the standard at all. Used by utils/unicode-known.c. """) chars = set(rec.c for rec in self.UnicodeData) for start, end in set_to_ranges(chars): print(f"{{0x{start:04x}, 0x{end:04x}}},", file=fh) def write_combining_class_table(self, fh): self.write_file_header_comment(fh, """ List the canonical combining class of each Unicode character, if it is not zero. This controls how combining marks can be reordered by the Unicode normalisation algorithms. Used by utils/unicode-norm.c. """) cclasses = {} for rec in self.UnicodeData: cc = rec.Canonical_Combining_Class if cc != 0: cclasses[rec.c] = cc for (start, end), cclass in map_to_ranges(cclasses): print(f"{{0x{start:04x}, 0x{end:04x}, {cclass:d}}},", file=fh) def write_canonical_decomp_table(self, fh): self.write_file_header_comment(fh, """ List the canonical decomposition of every Unicode character that has one. This consists of up to two characters, but those may need decomposition in turn. Used by utils/unicode-norm.c. """) decomps = {} for rec in self.UnicodeData: if rec.Decomposition_Type != 'canonical': continue # Fill in a zero code point as the second character, if # it's only one character long decomps[rec.c] = (rec.Decomposition_Mapping + [0])[:2] for c, (d1, d2) in sorted(decomps.items()): d2s = f"0x{d2:04x}" if d2 else "0" print(f"{{0x{c:04x}, 0x{d1:04x}, {d2s}}},", file=fh) def write_canonical_comp_table(self, fh): self.write_file_header_comment(fh, """ List the pairs of Unicode characters that canonically recompose to a single character in NFC. Used by utils/unicode-norm.c. """) exclusions = set(self.CompositionExclusions) nonstarters = set(rec.c for rec in self.UnicodeData if rec.Canonical_Combining_Class != 0) decomps = {} for rec in self.UnicodeData: if rec.Decomposition_Type != 'canonical': continue # we don't want compatibility decompositions if len(rec.Decomposition_Mapping) != 2: continue # we don't want singletons either if rec.c in exclusions: continue # we don't want anything explicitly excluded if (rec.c in nonstarters or rec.Decomposition_Mapping[0] in nonstarters): continue # we don't want non-starter decompositions decomps[tuple(rec.Decomposition_Mapping)] = rec.c for (d0, d1), c in sorted(decomps.items()): print(f"{{0x{d0:04x}, 0x{d1:04x}, 0x{c:04x}}},", file=fh) if __name__ == '__main__': Main().run()