878 lines
27 KiB
Python
Executable file
878 lines
27 KiB
Python
Executable file
#!/usr/bin/env python
|
|
|
|
"""
|
|
Regenerate Unicode tables (tables.rs).
|
|
"""
|
|
|
|
# This script uses the Unicode tables as defined
|
|
# in the UnicodeFiles class.
|
|
|
|
# Since this should not require frequent updates, we just store this
|
|
# out-of-line and check the tables.rs file into git.
|
|
|
|
# Note that the "curl" program is required for operation.
|
|
# This script is compatible with Python 2.7 and 3.x.
|
|
|
|
import argparse
|
|
import datetime
|
|
import fileinput
|
|
import itertools
|
|
import os
|
|
import re
|
|
import textwrap
|
|
import subprocess
|
|
|
|
from collections import defaultdict, namedtuple
|
|
|
|
try:
|
|
# Python 3
|
|
from itertools import zip_longest
|
|
from io import StringIO
|
|
except ImportError:
|
|
# Python 2 compatibility
|
|
zip_longest = itertools.izip_longest
|
|
from StringIO import StringIO
|
|
|
|
try:
|
|
# Completely optional type hinting
|
|
# (Python 2 compatible using comments,
|
|
# see: https://mypy.readthedocs.io/en/latest/python2.html)
|
|
# This is very helpful in typing-aware IDE like PyCharm.
|
|
from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple
|
|
except ImportError:
|
|
pass
|
|
|
|
|
|
# We don't use enum.Enum because of Python 2.7 compatibility.
|
|
class UnicodeFiles(object):
|
|
# ReadMe does not contain any Unicode data, we
|
|
# only use it to extract versions.
|
|
README = "ReadMe.txt"
|
|
|
|
DERIVED_CORE_PROPERTIES = "DerivedCoreProperties.txt"
|
|
DERIVED_NORMALIZATION_PROPS = "DerivedNormalizationProps.txt"
|
|
PROPS = "PropList.txt"
|
|
SCRIPTS = "Scripts.txt"
|
|
SPECIAL_CASING = "SpecialCasing.txt"
|
|
UNICODE_DATA = "UnicodeData.txt"
|
|
|
|
|
|
# The order doesn't really matter (Python < 3.6 won't preserve it),
|
|
# we only want to aggregate all the file names.
|
|
ALL_UNICODE_FILES = tuple(
|
|
value for name, value in UnicodeFiles.__dict__.items()
|
|
if not name.startswith("_")
|
|
)
|
|
|
|
assert len(ALL_UNICODE_FILES) == 7, "Unexpected number of unicode files"
|
|
|
|
# The directory this file is located in.
|
|
THIS_DIR = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
# Where to download the Unicode data. The downloaded files
|
|
# will be placed in sub-directories named after Unicode version.
|
|
FETCH_DIR = os.path.join(THIS_DIR, "downloaded")
|
|
|
|
FETCH_URL_LATEST = "ftp://ftp.unicode.org/Public/UNIDATA/{filename}"
|
|
FETCH_URL_VERSION = "ftp://ftp.unicode.org/Public/{version}/ucd/{filename}"
|
|
|
|
PREAMBLE = """\
|
|
// NOTE: The following code was generated by "./unicode.py", do not edit directly
|
|
|
|
#![allow(missing_docs, non_upper_case_globals, non_snake_case, clippy::unreadable_literal)]
|
|
|
|
use crate::unicode::bool_trie::{{BoolTrie, SmallBoolTrie}};
|
|
use crate::unicode::version::UnicodeVersion;
|
|
""".format(year=datetime.datetime.now().year)
|
|
|
|
# Mapping taken from Table 12 from:
|
|
# http://www.unicode.org/reports/tr44/#General_Category_Values
|
|
EXPANDED_CATEGORIES = {
|
|
"Lu": ["LC", "L"], "Ll": ["LC", "L"], "Lt": ["LC", "L"],
|
|
"Lm": ["L"], "Lo": ["L"],
|
|
"Mn": ["M"], "Mc": ["M"], "Me": ["M"],
|
|
"Nd": ["N"], "Nl": ["N"], "No": ["N"],
|
|
"Pc": ["P"], "Pd": ["P"], "Ps": ["P"], "Pe": ["P"],
|
|
"Pi": ["P"], "Pf": ["P"], "Po": ["P"],
|
|
"Sm": ["S"], "Sc": ["S"], "Sk": ["S"], "So": ["S"],
|
|
"Zs": ["Z"], "Zl": ["Z"], "Zp": ["Z"],
|
|
"Cc": ["C"], "Cf": ["C"], "Cs": ["C"], "Co": ["C"], "Cn": ["C"],
|
|
}
|
|
|
|
# This is the (inclusive) range of surrogate codepoints.
|
|
# These are not valid Rust characters.
|
|
SURROGATE_CODEPOINTS_RANGE = (0xd800, 0xdfff)
|
|
|
|
UnicodeData = namedtuple(
|
|
"UnicodeData", (
|
|
# Conversions:
|
|
"to_upper", "to_lower", "to_title",
|
|
|
|
# Decompositions: canonical decompositions, compatibility decomp
|
|
"canon_decomp", "compat_decomp",
|
|
|
|
# Grouped: general categories and combining characters
|
|
"general_categories", "combines",
|
|
)
|
|
)
|
|
|
|
UnicodeVersion = namedtuple(
|
|
"UnicodeVersion", ("major", "minor", "micro", "as_str")
|
|
)
|
|
|
|
|
|
def fetch_files(version=None):
|
|
# type: (str) -> UnicodeVersion
|
|
"""
|
|
Fetch all the Unicode files from unicode.org.
|
|
|
|
This will use cached files (stored in `FETCH_DIR`) if they exist,
|
|
creating them if they don't. In any case, the Unicode version
|
|
is always returned.
|
|
|
|
:param version: The desired Unicode version, as string.
|
|
(If None, defaults to latest final release available,
|
|
querying the unicode.org service).
|
|
"""
|
|
have_version = check_stored_version(version)
|
|
if have_version:
|
|
return have_version
|
|
|
|
if version:
|
|
# Check if the desired version exists on the server.
|
|
get_fetch_url = lambda name: FETCH_URL_VERSION.format(version=version, filename=name)
|
|
else:
|
|
# Extract the latest version.
|
|
get_fetch_url = lambda name: FETCH_URL_LATEST.format(filename=name)
|
|
|
|
readme_url = get_fetch_url(UnicodeFiles.README)
|
|
|
|
print("Fetching: {}".format(readme_url))
|
|
readme_content = subprocess.check_output(("curl", readme_url))
|
|
|
|
unicode_version = parse_readme_unicode_version(
|
|
readme_content.decode("utf8")
|
|
)
|
|
|
|
download_dir = get_unicode_dir(unicode_version)
|
|
if not os.path.exists(download_dir):
|
|
# For 2.7 compat, we don't use `exist_ok=True`.
|
|
os.makedirs(download_dir)
|
|
|
|
for filename in ALL_UNICODE_FILES:
|
|
file_path = get_unicode_file_path(unicode_version, filename)
|
|
|
|
if os.path.exists(file_path):
|
|
# Assume file on the server didn't change if it's been saved before.
|
|
continue
|
|
|
|
if filename == UnicodeFiles.README:
|
|
with open(file_path, "wb") as fd:
|
|
fd.write(readme_content)
|
|
else:
|
|
url = get_fetch_url(filename)
|
|
print("Fetching: {}".format(url))
|
|
subprocess.check_call(("curl", "-o", file_path, url))
|
|
|
|
return unicode_version
|
|
|
|
|
|
def check_stored_version(version):
|
|
# type: (Optional[str]) -> Optional[UnicodeVersion]
|
|
"""
|
|
Given desired Unicode version, return the version
|
|
if stored files are all present, and `None` otherwise.
|
|
"""
|
|
if not version:
|
|
# If no desired version specified, we should check what's the latest
|
|
# version, skipping stored version checks.
|
|
return None
|
|
|
|
fetch_dir = os.path.join(FETCH_DIR, version)
|
|
|
|
for filename in ALL_UNICODE_FILES:
|
|
file_path = os.path.join(fetch_dir, filename)
|
|
|
|
if not os.path.exists(file_path):
|
|
return None
|
|
|
|
with open(os.path.join(fetch_dir, UnicodeFiles.README)) as fd:
|
|
return parse_readme_unicode_version(fd.read())
|
|
|
|
|
|
def parse_readme_unicode_version(readme_content):
|
|
# type: (str) -> UnicodeVersion
|
|
"""
|
|
Parse the Unicode version contained in their `ReadMe.txt` file.
|
|
"""
|
|
# "Raw string" is necessary for \d not being treated as escape char
|
|
# (for the sake of compat with future Python versions).
|
|
# See: https://docs.python.org/3.6/whatsnew/3.6.html#deprecated-python-behavior
|
|
pattern = r"for Version (\d+)\.(\d+)\.(\d+) of the Unicode"
|
|
groups = re.search(pattern, readme_content).groups()
|
|
|
|
return UnicodeVersion(*map(int, groups), as_str=".".join(groups))
|
|
|
|
|
|
def get_unicode_dir(unicode_version):
|
|
# type: (UnicodeVersion) -> str
|
|
"""
|
|
Indicate in which parent dir the Unicode data files should be stored.
|
|
|
|
This returns a full, absolute path.
|
|
"""
|
|
return os.path.join(FETCH_DIR, unicode_version.as_str)
|
|
|
|
|
|
def get_unicode_file_path(unicode_version, filename):
|
|
# type: (UnicodeVersion, str) -> str
|
|
"""
|
|
Indicate where the Unicode data file should be stored.
|
|
"""
|
|
return os.path.join(get_unicode_dir(unicode_version), filename)
|
|
|
|
|
|
def is_surrogate(n):
|
|
# type: (int) -> bool
|
|
"""
|
|
Tell if given codepoint is a surrogate (not a valid Rust character).
|
|
"""
|
|
return SURROGATE_CODEPOINTS_RANGE[0] <= n <= SURROGATE_CODEPOINTS_RANGE[1]
|
|
|
|
|
|
def load_unicode_data(file_path):
|
|
# type: (str) -> UnicodeData
|
|
"""
|
|
Load main Unicode data.
|
|
"""
|
|
# Conversions
|
|
to_lower = {} # type: Dict[int, Tuple[int, int, int]]
|
|
to_upper = {} # type: Dict[int, Tuple[int, int, int]]
|
|
to_title = {} # type: Dict[int, Tuple[int, int, int]]
|
|
|
|
# Decompositions
|
|
compat_decomp = {} # type: Dict[int, List[int]]
|
|
canon_decomp = {} # type: Dict[int, List[int]]
|
|
|
|
# Combining characters
|
|
# FIXME: combines are not used
|
|
combines = defaultdict(set) # type: Dict[str, Set[int]]
|
|
|
|
# Categories
|
|
general_categories = defaultdict(set) # type: Dict[str, Set[int]]
|
|
category_assigned_codepoints = set() # type: Set[int]
|
|
|
|
all_codepoints = {}
|
|
|
|
range_start = -1
|
|
|
|
for line in fileinput.input(file_path):
|
|
data = line.split(";")
|
|
if len(data) != 15:
|
|
continue
|
|
codepoint = int(data[0], 16)
|
|
if is_surrogate(codepoint):
|
|
continue
|
|
if range_start >= 0:
|
|
for i in range(range_start, codepoint):
|
|
all_codepoints[i] = data
|
|
range_start = -1
|
|
if data[1].endswith(", First>"):
|
|
range_start = codepoint
|
|
continue
|
|
all_codepoints[codepoint] = data
|
|
|
|
for code, data in all_codepoints.items():
|
|
(code_org, name, gencat, combine, bidi,
|
|
decomp, deci, digit, num, mirror,
|
|
old, iso, upcase, lowcase, titlecase) = data
|
|
|
|
# Generate char to char direct common and simple conversions:
|
|
|
|
# Uppercase to lowercase
|
|
if lowcase != "" and code_org != lowcase:
|
|
to_lower[code] = (int(lowcase, 16), 0, 0)
|
|
|
|
# Lowercase to uppercase
|
|
if upcase != "" and code_org != upcase:
|
|
to_upper[code] = (int(upcase, 16), 0, 0)
|
|
|
|
# Title case
|
|
if titlecase.strip() != "" and code_org != titlecase:
|
|
to_title[code] = (int(titlecase, 16), 0, 0)
|
|
|
|
# Store decomposition, if given
|
|
if decomp:
|
|
decompositions = decomp.split()[1:]
|
|
decomp_code_points = [int(i, 16) for i in decompositions]
|
|
|
|
if decomp.startswith("<"):
|
|
# Compatibility decomposition
|
|
compat_decomp[code] = decomp_code_points
|
|
else:
|
|
# Canonical decomposition
|
|
canon_decomp[code] = decomp_code_points
|
|
|
|
# Place letter in categories as appropriate.
|
|
for cat in itertools.chain((gencat, ), EXPANDED_CATEGORIES.get(gencat, [])):
|
|
general_categories[cat].add(code)
|
|
category_assigned_codepoints.add(code)
|
|
|
|
# Record combining class, if any.
|
|
if combine != "0":
|
|
combines[combine].add(code)
|
|
|
|
# Generate Not_Assigned from Assigned.
|
|
general_categories["Cn"] = get_unassigned_codepoints(category_assigned_codepoints)
|
|
|
|
# Other contains Not_Assigned
|
|
general_categories["C"].update(general_categories["Cn"])
|
|
|
|
grouped_categories = group_categories(general_categories)
|
|
|
|
# FIXME: combines are not used
|
|
return UnicodeData(
|
|
to_lower=to_lower, to_upper=to_upper, to_title=to_title,
|
|
compat_decomp=compat_decomp, canon_decomp=canon_decomp,
|
|
general_categories=grouped_categories, combines=combines,
|
|
)
|
|
|
|
|
|
def load_special_casing(file_path, unicode_data):
|
|
# type: (str, UnicodeData) -> None
|
|
"""
|
|
Load special casing data and enrich given Unicode data.
|
|
"""
|
|
for line in fileinput.input(file_path):
|
|
data = line.split("#")[0].split(";")
|
|
if len(data) == 5:
|
|
code, lower, title, upper, _comment = data
|
|
elif len(data) == 6:
|
|
code, lower, title, upper, condition, _comment = data
|
|
if condition.strip(): # Only keep unconditional mappins
|
|
continue
|
|
else:
|
|
continue
|
|
code = code.strip()
|
|
lower = lower.strip()
|
|
title = title.strip()
|
|
upper = upper.strip()
|
|
key = int(code, 16)
|
|
for (map_, values) in ((unicode_data.to_lower, lower),
|
|
(unicode_data.to_upper, upper),
|
|
(unicode_data.to_title, title)):
|
|
if values != code:
|
|
split = values.split()
|
|
|
|
codepoints = list(itertools.chain(
|
|
(int(i, 16) for i in split),
|
|
(0 for _ in range(len(split), 3))
|
|
))
|
|
|
|
assert len(codepoints) == 3
|
|
map_[key] = codepoints
|
|
|
|
|
|
def group_categories(mapping):
|
|
# type: (Dict[Any, Iterable[int]]) -> Dict[str, List[Tuple[int, int]]]
|
|
"""
|
|
Group codepoints mapped in "categories".
|
|
"""
|
|
return {category: group_codepoints(codepoints)
|
|
for category, codepoints in mapping.items()}
|
|
|
|
|
|
def group_codepoints(codepoints):
|
|
# type: (Iterable[int]) -> List[Tuple[int, int]]
|
|
"""
|
|
Group integral values into continuous, disjoint value ranges.
|
|
|
|
Performs value deduplication.
|
|
|
|
:return: sorted list of pairs denoting start and end of codepoint
|
|
group values, both ends inclusive.
|
|
|
|
>>> group_codepoints([1, 2, 10, 11, 12, 3, 4])
|
|
[(1, 4), (10, 12)]
|
|
>>> group_codepoints([1])
|
|
[(1, 1)]
|
|
>>> group_codepoints([1, 5, 6])
|
|
[(1, 1), (5, 6)]
|
|
>>> group_codepoints([])
|
|
[]
|
|
"""
|
|
sorted_codes = sorted(set(codepoints))
|
|
result = [] # type: List[Tuple[int, int]]
|
|
|
|
if not sorted_codes:
|
|
return result
|
|
|
|
next_codes = sorted_codes[1:]
|
|
start_code = sorted_codes[0]
|
|
|
|
for code, next_code in zip_longest(sorted_codes, next_codes, fillvalue=None):
|
|
if next_code is None or next_code - code != 1:
|
|
result.append((start_code, code))
|
|
start_code = next_code
|
|
|
|
return result
|
|
|
|
|
|
def ungroup_codepoints(codepoint_pairs):
|
|
# type: (Iterable[Tuple[int, int]]) -> List[int]
|
|
"""
|
|
The inverse of group_codepoints -- produce a flat list of values
|
|
from value range pairs.
|
|
|
|
>>> ungroup_codepoints([(1, 4), (10, 12)])
|
|
[1, 2, 3, 4, 10, 11, 12]
|
|
>>> ungroup_codepoints([(1, 1), (5, 6)])
|
|
[1, 5, 6]
|
|
>>> ungroup_codepoints(group_codepoints([1, 2, 7, 8]))
|
|
[1, 2, 7, 8]
|
|
>>> ungroup_codepoints([])
|
|
[]
|
|
"""
|
|
return list(itertools.chain.from_iterable(
|
|
range(lo, hi + 1) for lo, hi in codepoint_pairs
|
|
))
|
|
|
|
|
|
def get_unassigned_codepoints(assigned_codepoints):
|
|
# type: (Set[int]) -> Set[int]
|
|
"""
|
|
Given a set of "assigned" codepoints, return a set
|
|
of these that are not in assigned and not surrogate.
|
|
"""
|
|
return {i for i in range(0, 0x110000)
|
|
if i not in assigned_codepoints and not is_surrogate(i)}
|
|
|
|
|
|
def generate_table_lines(items, indent, wrap=98):
|
|
# type: (Iterable[str], int, int) -> Iterator[str]
|
|
"""
|
|
Given table items, generate wrapped lines of text with comma-separated items.
|
|
|
|
This is a generator function.
|
|
|
|
:param wrap: soft wrap limit (characters per line), integer.
|
|
"""
|
|
line = " " * indent
|
|
first = True
|
|
for item in items:
|
|
if len(line) + len(item) < wrap:
|
|
if first:
|
|
line += item
|
|
else:
|
|
line += ", " + item
|
|
first = False
|
|
else:
|
|
yield line + ",\n"
|
|
line = " " * indent + item
|
|
|
|
yield line
|
|
|
|
|
|
def load_properties(file_path, interesting_props):
|
|
# type: (str, Iterable[str]) -> Dict[str, List[Tuple[int, int]]]
|
|
"""
|
|
Load properties data and return in grouped form.
|
|
"""
|
|
props = defaultdict(list) # type: Dict[str, List[Tuple[int, int]]]
|
|
# "Raw string" is necessary for `\.` and `\w` not to be treated as escape chars
|
|
# (for the sake of compat with future Python versions).
|
|
# See: https://docs.python.org/3.6/whatsnew/3.6.html#deprecated-python-behavior
|
|
re1 = re.compile(r"^ *([0-9A-F]+) *; *(\w+)")
|
|
re2 = re.compile(r"^ *([0-9A-F]+)\.\.([0-9A-F]+) *; *(\w+)")
|
|
|
|
for line in fileinput.input(file_path):
|
|
match = re1.match(line) or re2.match(line)
|
|
if match:
|
|
groups = match.groups()
|
|
|
|
if len(groups) == 2:
|
|
# `re1` matched (2 groups).
|
|
d_lo, prop = groups
|
|
d_hi = d_lo
|
|
else:
|
|
d_lo, d_hi, prop = groups
|
|
else:
|
|
continue
|
|
|
|
if interesting_props and prop not in interesting_props:
|
|
continue
|
|
|
|
lo_value = int(d_lo, 16)
|
|
hi_value = int(d_hi, 16)
|
|
|
|
props[prop].append((lo_value, hi_value))
|
|
|
|
# Optimize if possible.
|
|
for prop in props:
|
|
props[prop] = group_codepoints(ungroup_codepoints(props[prop]))
|
|
|
|
return props
|
|
|
|
|
|
def escape_char(c):
|
|
# type: (int) -> str
|
|
r"""
|
|
Escape a codepoint for use as Rust char literal.
|
|
|
|
Outputs are OK to use as Rust source code as char literals
|
|
and they also include necessary quotes.
|
|
|
|
>>> escape_char(97)
|
|
"'\\u{61}'"
|
|
>>> escape_char(0)
|
|
"'\\0'"
|
|
"""
|
|
return r"'\u{%x}'" % c if c != 0 else r"'\0'"
|
|
|
|
|
|
def format_char_pair(pair):
|
|
# type: (Tuple[int, int]) -> str
|
|
"""
|
|
Format a pair of two Rust chars.
|
|
"""
|
|
return "(%s,%s)" % (escape_char(pair[0]), escape_char(pair[1]))
|
|
|
|
|
|
def generate_table(
|
|
name, # type: str
|
|
items, # type: List[Tuple[int, int]]
|
|
decl_type="&[(char, char)]", # type: str
|
|
is_pub=True, # type: bool
|
|
format_item=format_char_pair, # type: Callable[[Tuple[int, int]], str]
|
|
):
|
|
# type: (...) -> Iterator[str]
|
|
"""
|
|
Generate a nicely formatted Rust constant "table" array.
|
|
|
|
This generates actual Rust code.
|
|
"""
|
|
pub_string = ""
|
|
if is_pub:
|
|
pub_string = "pub "
|
|
|
|
yield "\n"
|
|
yield " #[rustfmt::skip]\n"
|
|
yield " %sconst %s: %s = &[\n" % (pub_string, name, decl_type)
|
|
|
|
data = []
|
|
first = True
|
|
for item in items:
|
|
if not first:
|
|
data.append(",")
|
|
first = False
|
|
data.extend(format_item(item))
|
|
|
|
for table_line in generate_table_lines("".join(data).split(","), 8):
|
|
yield table_line
|
|
|
|
yield "\n ];\n"
|
|
|
|
|
|
def compute_trie(raw_data, chunk_size):
|
|
# type: (List[int], int) -> Tuple[List[int], List[int]]
|
|
"""
|
|
Compute postfix-compressed trie.
|
|
|
|
See: bool_trie.rs for more details.
|
|
|
|
>>> compute_trie([1, 2, 3, 1, 2, 3, 4, 5, 6], 3)
|
|
([0, 0, 1], [1, 2, 3, 4, 5, 6])
|
|
>>> compute_trie([1, 2, 3, 1, 2, 4, 4, 5, 6], 3)
|
|
([0, 1, 2], [1, 2, 3, 1, 2, 4, 4, 5, 6])
|
|
"""
|
|
root = []
|
|
childmap = {} # type: Dict[Tuple[int, ...], int]
|
|
child_data = []
|
|
|
|
assert len(raw_data) % chunk_size == 0, "Chunks must be equally sized"
|
|
|
|
for i in range(len(raw_data) // chunk_size):
|
|
data = raw_data[i * chunk_size : (i + 1) * chunk_size]
|
|
|
|
# Postfix compression of child nodes (data chunks)
|
|
# (identical child nodes are shared).
|
|
|
|
# Make a tuple out of the list so it's hashable.
|
|
child = tuple(data)
|
|
if child not in childmap:
|
|
childmap[child] = len(childmap)
|
|
child_data.extend(data)
|
|
|
|
root.append(childmap[child])
|
|
|
|
return root, child_data
|
|
|
|
|
|
def generate_bool_trie(name, codepoint_ranges, is_pub=False):
|
|
# type: (str, List[Tuple[int, int]], bool) -> Iterator[str]
|
|
"""
|
|
Generate Rust code for BoolTrie struct.
|
|
|
|
This yields string fragments that should be joined to produce
|
|
the final string.
|
|
|
|
See: `bool_trie.rs`.
|
|
"""
|
|
chunk_size = 64
|
|
rawdata = [False] * 0x110000
|
|
for (lo, hi) in codepoint_ranges:
|
|
for cp in range(lo, hi + 1):
|
|
rawdata[cp] = True
|
|
|
|
# Convert to bitmap chunks of `chunk_size` bits each.
|
|
chunks = []
|
|
for i in range(0x110000 // chunk_size):
|
|
chunk = 0
|
|
for j in range(chunk_size):
|
|
if rawdata[i * chunk_size + j]:
|
|
chunk |= 1 << j
|
|
chunks.append(chunk)
|
|
|
|
pub_string = ""
|
|
if is_pub:
|
|
pub_string = "pub "
|
|
|
|
yield "\n"
|
|
yield " #[rustfmt::skip]\n"
|
|
yield " %sconst %s: &super::BoolTrie = &super::BoolTrie {\n" % (pub_string, name)
|
|
yield " r1: [\n"
|
|
data = ("0x%016x" % chunk for chunk in chunks[:0x800 // chunk_size])
|
|
for fragment in generate_table_lines(data, 12):
|
|
yield fragment
|
|
yield "\n ],\n"
|
|
|
|
# 0x800..0x10000 trie
|
|
(r2, r3) = compute_trie(chunks[0x800 // chunk_size : 0x10000 // chunk_size], 64 // chunk_size)
|
|
yield " r2: [\n"
|
|
data = map(str, r2)
|
|
for fragment in generate_table_lines(data, 12):
|
|
yield fragment
|
|
yield "\n ],\n"
|
|
|
|
yield " r3: &[\n"
|
|
data = ("0x%016x" % node for node in r3)
|
|
for fragment in generate_table_lines(data, 12):
|
|
yield fragment
|
|
yield "\n ],\n"
|
|
|
|
# 0x10000..0x110000 trie
|
|
(mid, r6) = compute_trie(chunks[0x10000 // chunk_size : 0x110000 // chunk_size],
|
|
64 // chunk_size)
|
|
(r4, r5) = compute_trie(mid, 64)
|
|
|
|
yield " r4: [\n"
|
|
data = map(str, r4)
|
|
for fragment in generate_table_lines(data, 12):
|
|
yield fragment
|
|
yield "\n ],\n"
|
|
|
|
yield " r5: &[\n"
|
|
data = map(str, r5)
|
|
for fragment in generate_table_lines(data, 12):
|
|
yield fragment
|
|
yield "\n ],\n"
|
|
|
|
yield " r6: &[\n"
|
|
data = ("0x%016x" % node for node in r6)
|
|
for fragment in generate_table_lines(data, 12):
|
|
yield fragment
|
|
yield "\n ],\n"
|
|
|
|
yield " };\n"
|
|
|
|
|
|
def generate_small_bool_trie(name, codepoint_ranges, is_pub=False):
|
|
# type: (str, List[Tuple[int, int]], bool) -> Iterator[str]
|
|
"""
|
|
Generate Rust code for `SmallBoolTrie` struct.
|
|
|
|
See: `bool_trie.rs`.
|
|
"""
|
|
last_chunk = max(hi // 64 for (lo, hi) in codepoint_ranges)
|
|
n_chunks = last_chunk + 1
|
|
chunks = [0] * n_chunks
|
|
for (lo, hi) in codepoint_ranges:
|
|
for cp in range(lo, hi + 1):
|
|
assert cp // 64 < len(chunks)
|
|
chunks[cp // 64] |= 1 << (cp & 63)
|
|
|
|
pub_string = ""
|
|
if is_pub:
|
|
pub_string = "pub "
|
|
|
|
yield "\n"
|
|
yield " #[rustfmt::skip]\n"
|
|
yield (" %sconst %s: &super::SmallBoolTrie = &super::SmallBoolTrie {\n"
|
|
% (pub_string, name))
|
|
|
|
(r1, r2) = compute_trie(chunks, 1)
|
|
|
|
yield " r1: &[\n"
|
|
data = (str(node) for node in r1)
|
|
for fragment in generate_table_lines(data, 12):
|
|
yield fragment
|
|
yield "\n ],\n"
|
|
|
|
yield " r2: &[\n"
|
|
data = ("0x%016x" % node for node in r2)
|
|
for fragment in generate_table_lines(data, 12):
|
|
yield fragment
|
|
yield "\n ],\n"
|
|
|
|
yield " };\n"
|
|
|
|
|
|
def generate_property_module(mod, grouped_categories, category_subset):
|
|
# type: (str, Dict[str, List[Tuple[int, int]]], Iterable[str]) -> Iterator[str]
|
|
"""
|
|
Generate Rust code for module defining properties.
|
|
"""
|
|
|
|
yield "pub(crate) mod %s {" % mod
|
|
for cat in sorted(category_subset):
|
|
if cat in ("Cc", "White_Space"):
|
|
generator = generate_small_bool_trie("%s_table" % cat, grouped_categories[cat])
|
|
else:
|
|
generator = generate_bool_trie("%s_table" % cat, grouped_categories[cat])
|
|
|
|
for fragment in generator:
|
|
yield fragment
|
|
|
|
yield "\n"
|
|
yield " pub fn %s(c: char) -> bool {\n" % cat
|
|
yield " %s_table.lookup(c)\n" % cat
|
|
yield " }\n"
|
|
|
|
yield "}\n\n"
|
|
|
|
|
|
def generate_conversions_module(unicode_data):
|
|
# type: (UnicodeData) -> Iterator[str]
|
|
"""
|
|
Generate Rust code for module defining conversions.
|
|
"""
|
|
|
|
yield "pub(crate) mod conversions {"
|
|
yield """
|
|
pub fn to_lower(c: char) -> [char; 3] {
|
|
match bsearch_case_table(c, to_lowercase_table) {
|
|
None => [c, '\\0', '\\0'],
|
|
Some(index) => to_lowercase_table[index].1,
|
|
}
|
|
}
|
|
|
|
pub fn to_upper(c: char) -> [char; 3] {
|
|
match bsearch_case_table(c, to_uppercase_table) {
|
|
None => [c, '\\0', '\\0'],
|
|
Some(index) => to_uppercase_table[index].1,
|
|
}
|
|
}
|
|
|
|
fn bsearch_case_table(c: char, table: &[(char, [char; 3])]) -> Option<usize> {
|
|
table.binary_search_by(|&(key, _)| key.cmp(&c)).ok()
|
|
}\n"""
|
|
|
|
decl_type = "&[(char, [char; 3])]"
|
|
format_conversion = lambda x: "({},[{},{},{}])".format(*(
|
|
escape_char(c) for c in (x[0], x[1][0], x[1][1], x[1][2])
|
|
))
|
|
|
|
for fragment in generate_table(
|
|
name="to_lowercase_table",
|
|
items=sorted(unicode_data.to_lower.items(), key=lambda x: x[0]),
|
|
decl_type=decl_type,
|
|
is_pub=False,
|
|
format_item=format_conversion
|
|
):
|
|
yield fragment
|
|
|
|
for fragment in generate_table(
|
|
name="to_uppercase_table",
|
|
items=sorted(unicode_data.to_upper.items(), key=lambda x: x[0]),
|
|
decl_type=decl_type,
|
|
is_pub=False,
|
|
format_item=format_conversion
|
|
):
|
|
yield fragment
|
|
|
|
yield "}\n"
|
|
|
|
|
|
def parse_args():
|
|
# type: () -> argparse.Namespace
|
|
"""
|
|
Parse command line arguments.
|
|
"""
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("-v", "--version", default=None, type=str,
|
|
help="Unicode version to use (if not specified,"
|
|
" defaults to latest release).")
|
|
|
|
return parser.parse_args()
|
|
|
|
|
|
def main():
|
|
# type: () -> None
|
|
"""
|
|
Script entry point.
|
|
"""
|
|
args = parse_args()
|
|
|
|
unicode_version = fetch_files(args.version)
|
|
print("Using Unicode version: {}".format(unicode_version.as_str))
|
|
|
|
# All the writing happens entirely in memory, we only write to file
|
|
# once we have generated the file content (it's not very large, <1 MB).
|
|
buf = StringIO()
|
|
buf.write(PREAMBLE)
|
|
|
|
unicode_version_notice = textwrap.dedent("""
|
|
/// The version of [Unicode](http://www.unicode.org/) that the Unicode parts of
|
|
/// `char` and `str` methods are based on.
|
|
#[unstable(feature = "unicode_version", issue = "49726")]
|
|
pub const UNICODE_VERSION: UnicodeVersion =
|
|
UnicodeVersion {{ major: {v.major}, minor: {v.minor}, micro: {v.micro}, _priv: () }};
|
|
""").format(v=unicode_version)
|
|
buf.write(unicode_version_notice)
|
|
|
|
get_path = lambda f: get_unicode_file_path(unicode_version, f)
|
|
|
|
unicode_data = load_unicode_data(get_path(UnicodeFiles.UNICODE_DATA))
|
|
load_special_casing(get_path(UnicodeFiles.SPECIAL_CASING), unicode_data)
|
|
|
|
want_derived = {"Alphabetic", "Lowercase", "Uppercase",
|
|
"Cased", "Case_Ignorable", "Grapheme_Extend"}
|
|
derived = load_properties(get_path(UnicodeFiles.DERIVED_CORE_PROPERTIES), want_derived)
|
|
|
|
props = load_properties(get_path(UnicodeFiles.PROPS),
|
|
{"White_Space", "Join_Control", "Noncharacter_Code_Point"})
|
|
|
|
# Category tables
|
|
for (name, categories, category_subset) in (
|
|
("general_category", unicode_data.general_categories, ["N", "Cc"]),
|
|
("derived_property", derived, want_derived),
|
|
("property", props, ["White_Space"])
|
|
):
|
|
for fragment in generate_property_module(name, categories, category_subset):
|
|
buf.write(fragment)
|
|
|
|
for fragment in generate_conversions_module(unicode_data):
|
|
buf.write(fragment)
|
|
|
|
tables_rs_path = os.path.join(THIS_DIR, "tables.rs")
|
|
|
|
# Actually write out the file content.
|
|
# Will overwrite the file if it exists.
|
|
with open(tables_rs_path, "w") as fd:
|
|
fd.write(buf.getvalue())
|
|
|
|
print("Regenerated tables.rs.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|