├── .gitignore
├── .travis.yml
├── Cargo.toml
├── LICENSE
├── LICENSE-UCD
├── README.md
├── data
    ├── LICENSE
    ├── blocks.py
    ├── gen_bidi.py
    ├── gen_bool.py
    ├── gen_decomp.py
    ├── gen_func.py
    ├── gen_general.py
    ├── gen_misc.py
    ├── gen_rem.py
    ├── gen_scripts.py
    ├── gen_tests.py
    ├── generate.py
    ├── generate.sh
    ├── misc.py
    ├── process.py
    └── ucd.xml.xz
├── prepare.sh
├── src
    ├── lib.rs
    └── tables
    │   ├── LICENSE-UCD
    │   ├── bidi.rs
    │   ├── bool.rs
    │   ├── decomp.rs
    │   ├── function.rs
    │   ├── general.rs
    │   ├── misc.rs
    │   ├── mod.rs
    │   ├── rem.rs
    │   └── scripts.rs
└── tests
    ├── data.tar.xz
    └── lib.rs


/.gitignore:
--------------------------------------------------------------------------------
1 | target
2 | Cargo.lock
3 | *.swp
4 | __pycache__
5 | .DS_Store
6 | tests/data/*
7 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: rust
2 | script:
3 | - ./prepare.sh
4 | - cargo build --release --verbose
5 | - cargo test --release --verbose
6 | 


--------------------------------------------------------------------------------
/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "ucd"
 3 | version = "0.1.1"
 4 | authors = ["sourtin <sourtin>"]
 5 | 
 6 | homepage = "https://github.com/sourtin/libucd"
 7 | repository = "https://github.com/sourtin/libucd"
 8 | license = "MIT"
 9 | keywords = ["text", "unicode", "ucd", "character"]
10 | readme = "README.md"
11 | description = """
12 | Extends the char type to provide access to most fields of the UCD, Unicode
13 | Character Database, as of version 9.0.0. It aims to be compact, fast, and use
14 | minimal dependencies (only rust's core crate). Not all properties are included,
15 | most notably character names.
16 | """
17 | 
18 | exclude = ["tests/*", "data/*"]
19 | 
20 | [dependencies]
21 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2016 William Earley
 2 | 
 3 | Permission is hereby granted, free of charge, to any
 4 | person obtaining a copy of this software and associated
 5 | documentation files (the "Software"), to deal in the
 6 | Software without restriction, including without
 7 | limitation the rights to use, copy, modify, merge,
 8 | publish, distribute, sublicense, and/or sell copies of
 9 | the Software, and to permit persons to whom the Software
10 | is furnished to do so, subject to the following
11 | conditions:
12 | 
13 | The above copyright notice and this permission notice
14 | shall be included in all copies or substantial portions
15 | of the Software.
16 | 
17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 | DEALINGS IN THE SOFTWARE.


--------------------------------------------------------------------------------
/LICENSE-UCD:
--------------------------------------------------------------------------------
 1 | COPYRIGHT AND PERMISSION NOTICE
 2 | 
 3 | Copyright © 1991-2016 Unicode, Inc. All rights reserved.
 4 | Distributed under the Terms of Use in 
 5 | http://www.unicode.org/copyright.html.
 6 | 
 7 | Permission is hereby granted, free of charge, to any person obtaining
 8 | a copy of the Unicode data files and any associated documentation
 9 | (the "Data Files") or Unicode software and any associated documentation
10 | (the "Software") to deal in the Data Files or Software
11 | without restriction, including without limitation the rights to use,
12 | copy, modify, merge, publish, distribute, and/or sell copies of
13 | the Data Files or Software, and to permit persons to whom the Data Files
14 | or Software are furnished to do so, provided that
15 | (a) this copyright and permission notice appear with all copies 
16 | of the Data Files or Software,
17 | (b) this copyright and permission notice appear in associated 
18 | documentation, and
19 | (c) there is clear notice in each modified Data File or in the Software
20 | as well as in the documentation associated with the Data File(s) or
21 | Software that the data or software has been modified.
22 | 
23 | THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
24 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
25 | WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 | NONINFRINGEMENT OF THIRD PARTY RIGHTS.
27 | IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
28 | NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
29 | DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
30 | DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
31 | TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
32 | PERFORMANCE OF THE DATA FILES OR SOFTWARE.
33 | 
34 | Except as contained in this notice, the name of a copyright holder
35 | shall not be used in advertising or otherwise to promote the sale,
36 | use or other dealings in these Data Files or Software without prior
37 | written authorization of the copyright holder.
38 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # `libucd`
  2 | 
  3 | [![Build Status](https://travis-ci.org/sourtin/libucd.svg?branch=master)](https://travis-ci.org/sourtin/libucd)
  4 | [![crates.io](http://meritbadge.herokuapp.com/ucd)](https://crates.io/crates/ucd)
  5 | 
  6 | This library extends the inbuilt `char` type with the `Codepoint` trait, which implements 100 properties of the UCD (Unicode Character Database). It aims to be fast and compact, and to have minimal dependencies (it does not require the rust standard library so only needs rust's `core` crate).
  7 | 
  8 | ```rust
  9 | extern crate ucd;
 10 | use ucd::Codepoint;
 11 | 
 12 | fn main() {
 13 |     let salawat: char = 'ﷺ';
 14 |     let decomp: String = salawat.decomposition_map().collect();
 15 |     println!("{} -> {}", salawat, decomp);
 16 |     // ﷺ -> صلى الله عليه وسلم
 17 | }
 18 | ```
 19 | 
 20 | Though the library is fairly extensive, it is not complete. The properties it lacks are:
 21 | 
 22 | * Character [names](https://github.com/huonw/unicode_names) and aliases
 23 | * Unihan properties
 24 | * Tangut source data
 25 | * Named sequences
 26 | * Standardised variants
 27 | * CJK Radicals
 28 | * Emoji source data
 29 | 
 30 | Data was compressed as arrays of ranges of codepoints having each value for each property, excluding the most common value. Lookup is then implemented by a binary search which should provide O(1) access. A very unscientific test suggests each lookup takes around 100ns on a core i5 processor.
 31 | 
 32 | ## Disclaimer
 33 | 
 34 | Please note that this data has been derived from the flat XML version of the UCD. Though this is published by Unicode, it is not the official version of the database. As such, it cannot be guaranteed that this library is entirely consistent with the official database. The unit tests do however suggest consistency with the XML database at least.
 35 | 
 36 | ## Reference
 37 | 
 38 | _Note, in most cases enum values are simply the full name of the property value converted to camelcase. Perhaps I will eventually create a rustdoc version that will be more helpful._
 39 | 
 40 | In the tables below, the first column is the name of the property, and the second column is the method name. The methods are implemented via the `ucd::Codepoint` trait on `char`.
 41 | 
 42 | | General | Method | Return Type | Note |
 43 | | ---- | --- | --- | --- |
 44 | | Age | `age` | `Option<(u8,u8)>` | Return `None` if the character is unassigned, else a tuple of major and minor age |
 45 | | Block | `block` | `Option<UnicodeBlock>` |  |
 46 | | General_Category | `category` | `UnicodeCategory` | Return `Unassigned` if unassigned |
 47 | | ISO_Comment (deprecated, stabilized) | `iso_comment` | `&str` | All characters are null, i.e. `""` is returned |
 48 | 
 49 | | Function and Appearance | Method | Return Type | Note |
 50 | | ---- | --- | --- | --- |
 51 | | Alphabetic | `Codepoint::is_alphabetic` or `is_alpha` | `bool` | Rust's `char` already defines a method `is_alphabetic` which shadows this implementation and provides slightly different results (possibly by using an outdated version of the UCD). Use the namespace, or the `is_alpha` alias instead. |
 52 | | ASCII_Hex_Digit | `is_hex_digit_ascii` | `bool` | |
 53 | | Dash | `is_dash` | `bool` | |
 54 | | Default_Ignorable_Code_Point | `is_default_ignorable` | `bool` | |
 55 | | Deprecated | `is_deprecated` | `bool` | |
 56 | | Diacritic | `is_diacritic` | `bool` | |
 57 | | Extender | `is_extender` | `bool` | |
 58 | | Hex_Digit | `is_hex_digit` | `bool` | |
 59 | | Hyphen _(deprecated, stabilized)_ | `is_hyphen` | `bool` | |
 60 | | Logical_Order_Exception | `is_logical_order_exception` | `bool` | |
 61 | | Math | `is_math` | `bool` | |
 62 | | Noncharacter_Code_Point | `is_noncharacter` | `bool` | |
 63 | | Other_Alphabetic | `is_alphabetic_other` | `bool` | |
 64 | | Other_Default_Ignorable_Code_Point | `is_default_ignorable_other` | `bool` | |
 65 | | Other_Math | `is_math_other` | `bool` | |
 66 | | Prepended_Concatenation_Mark | `is_prepended_concatentation_mark` | `bool` | |
 67 | | Quotation_Mark | `is_quotation_mark` | `bool` | |
 68 | | Sentence_Terminal | `is_sentence_terminal` | `bool` | |
 69 | | Soft_Dotted | `is_soft_dotted` | `bool` | |
 70 | | Terminal_Punctuation | `is_terminal_punctuation` | `bool` | |
 71 | | Variation_Selector | `is_variation_selector` | `bool` | |
 72 | | White_Space | `Codepoint::is_whitespace` or `is_white` | `bool` | Again, `char` shadows this with its own implementation so use the namespace or alias. |
 73 | 
 74 | | Numeric | Method | Return Type | Note |
 75 | | ---- | --- | --- | --- |
 76 | | Numeric_Type | `numeric_type` | `Option<NumericType>` |  |
 77 | | Numeric_Value | `numeric_value` | `Option<Number>` | The `Number` type is an enum of `Integer(i64)` and `Rational(i32,u32)`, to cover values as large as 10^12 and as small as 1/160. |
 78 | 
 79 | | Identifiers and Syntax | Method | Return Type | Note |
 80 | | ---- | --- | --- | --- |
 81 | | ID_Continue | `is_id_continue` | `bool` | |
 82 | | ID_Start | `is_id_start` | `bool` | |
 83 | | Other_ID_Continue | `is_id_continue_other` | `bool` | |
 84 | | Other_ID_Start | `is_id_start_other` | `bool` | |
 85 | | Pattern_Syntax | `is_pattern_syntax` | `bool` | |
 86 | | Pattern_White_Space | `is_pattern_whitespace` | `bool` | |
 87 | | XID_Continue | `is_id_continue_nfkc` | `bool` | |
 88 | | XID_Start | `is_id_start_nfkc` | `bool` | |
 89 | 
 90 | | Scripts | Method | Return Type | Note |
 91 | | ---- | --- | --- | --- |
 92 | | East_Asian_Width | `east_asian_width` | `EastAsianWidth` | |
 93 | | Hangul_Syllable_Type | `hangul_syllable_type` | `Option<HangulSyllableType>` | |
 94 | | Ideographic | `is_ideograph` | `bool` | |
 95 | | IDS_Binary_Operator | `is_ideograph_description_sequence_binary_operator` | `bool` | |
 96 | | IDS_Trinary_Operator | `is_ideograph_description_sequence_trinary_operator` | `bool` | |
 97 | | Indic_Positional_Category | `indic_positional_category` | `Option<IndicPositionalCategory>` | |
 98 | | Indic_Syllabic_Category | `indic_syllabic_category` | `Option<IndicSyllabicCategory>` | |
 99 | | Jamo_Short_Name | `jamo_short_name` | `Option<&str>` | |
100 | | Join_Control | `join_control` | `bool` | |
101 | | Joining_Group | `joining_group` | `JoiningGroup` | |
102 | | Joining_Type | `joining_type` | `JoiningType` | |
103 | | Radical | `is_ideograph_description_sequence_radical` | `bool` | |
104 | | Script | `script` | `Option<Script>` | |
105 | | Script_Extensions | `script_extensions` | `Option<&[Script]>` | |
106 | | Unified_Ideograph | `is_ideograph_unified` | `bool` | |
107 | 
108 | | Bidirectionality | Method | Return Type | Note |
109 | | ---- | --- | --- | --- |
110 | | Bidi_Class | `bidi_class` | `BidiClass` | |
111 | | Bidi_Control | `bidi_is_control` | `bool` | |
112 | | Bidi_Mirrored | `bidi_is_mirrored` | `bool` | |
113 | | Bidi_Mirroring_Glyph | `bidi_mirror` | `Option<char>` | |
114 | | Bidi_Paired_Bracket | `bidi_paired_braclet` | `char` | |
115 | | Bidi_Paired_Bracket_Type | `bidi_paired_bracket_type` | `Option<BidiPairedBracketType>` | |
116 | 
117 | | Case | Method | Return Type | Note |
118 | | ---- | --- | --- | --- |
119 | | Case_Folding | `casefold` | `CharIter` | `CharIter` is an iterator over a series of `char`s, and is used because the library makes no use of `std` and thus cannot dynamically allocate memory. |
120 | | Case_Ignorable | `is_case_ignorable` | `bool` | |
121 | | Cased | `is_cased` | `bool` | |
122 | | Changes_When_Casefolded | `changes_when_casefolded` | `bool` | |
123 | | Changes_When_Casemapped | `changes_when_casemapped` | `bool` | |
124 | | Changes_When_Lowercased | `changes_when_lowercased` | `bool` | |
125 | | Changes_When_NFKC_Casefolded | `changes_when_casefolded_nfkc` | `bool` | |
126 | | Changes_When_Titlecased | `changes_when_titlecased` | `bool` | |
127 | | Changes_When_Uppercased | `changes_when_uppercased` | `bool` | |
128 | | FC_NFKC_Closure (deprecated) | `casefold_nfkc_closure` | `CharIter` | |
129 | | Lowercase | `is_lowercase` | `bool` | Again, `char` shadows this with its own implementation so use the namespace or alias. |
130 | | Lowercase_Mapping | `lowercase` | `CharIter` | |
131 | | NFKC_Casefold | `casefold_nfkc` | `CharIter` | |
132 | | Other_Lowercase | `is_lowercase_other` | `bool` |
133 | | Other_Uppercase | `is_uppercase_other` | `bool` |
134 | | Simple_Case_Folding | `casefold_simple` | `char` | |
135 | | Simple_Lowercase_Mapping | `lowercase_simple` | `char` | |
136 | | Simple_Titlecase_Mapping | `titlecase_simple` | `char` | |
137 | | Simple_Uppercase_Mapping | `uppercase_simple` | `char` | |
138 | | Titlecase_Mapping | `titlecase` | `CharIter` | |
139 | | Uppercase | `is_uppercase` | `bool` | Again, `char` shadows this with its own implementation so use the namespace or alias. |
140 | | Uppercase_Mapping | `uppercase` | `CharIter` | |
141 | 
142 | | Normalisation | Method | Return Type | Note |
143 | | ---- | --- | --- | --- |
144 | | Canonical_Combining_Class | `canonical_combining_class` | `u8` | |
145 | | Composition_Exclusion | `excluded_from_composition` | `bool` | |
146 | | Decomposition_Mapping | `decomposition_map` | `CharIter` | |
147 | | Decomposition_Type | `decomposition_type` | `Option<DecompositionType>` | |
148 | | Expands_On_NFC _(deprecated)_ | `expands_on_nfc` | `bool` | |
149 | | Expands_On_NFD _(deprecated)_ | `expands_on_nfd` | `bool` | |
150 | | Expands_On_NFKC _(deprecated)_ | `expands_on_nfkc` | `bool` | |
151 | | Expands_On_NFKD _(deprecated)_ | `expands_on_nfkd` | `bool` |
152 | | Full_Composition_Exclusion | `excluded_from_composition_fully` | `bool` | |
153 | | NFC_Quick_Check | `quick_check_nfc` | `Trilean` | Returns one of `Trilean::True`, `Trilean::Maybe` or `Trilean::False` |
154 | | NFD_Quick_Check | `quick_check_nfd` | `bool` | |
155 | | NFKC_Quick_Check | `quick_check_nfkc` | `Trilean` | |
156 | | NFKD_Quick_Check | `quick_check_nfkd` | `bool` |
157 | 
158 | | Segmentation | Method | Return Type | Note |
159 | | ---- | --- | --- | --- |
160 | | Grapheme_Base | `is_grapheme_base` | `bool` | |
161 | | Grapheme_Cluster_Break | `grapheme_cluster_break` | `GraphemeClusterBreak` | |
162 | | Grapheme_Extend | `is_grapheme_extend` | `bool` | |
163 | | Grapheme_Link _(deprecated)_ | `is_grapheme_link` | `bool` | |
164 | | Line_Break | `linebreak_class` | `Option<LinebreakClass>` | |
165 | | Other_Grapheme_Extend | `is_grapheme_extend_other` | `bool` | |
166 | | Sentence_Break | `sentence_break` | `SentenceBreak` | |
167 | | Word_Break | `word_break` | `WordBreak` |
168 | 


--------------------------------------------------------------------------------
/data/LICENSE:
--------------------------------------------------------------------------------
 1 | COPYRIGHT AND PERMISSION NOTICE
 2 | 
 3 | Copyright © 1991-2016 Unicode, Inc. All rights reserved.
 4 | Distributed under the Terms of Use in 
 5 | http://www.unicode.org/copyright.html.
 6 | 
 7 | Permission is hereby granted, free of charge, to any person obtaining
 8 | a copy of the Unicode data files and any associated documentation
 9 | (the "Data Files") or Unicode software and any associated documentation
10 | (the "Software") to deal in the Data Files or Software
11 | without restriction, including without limitation the rights to use,
12 | copy, modify, merge, publish, distribute, and/or sell copies of
13 | the Data Files or Software, and to permit persons to whom the Data Files
14 | or Software are furnished to do so, provided that
15 | (a) this copyright and permission notice appear with all copies 
16 | of the Data Files or Software,
17 | (b) this copyright and permission notice appear in associated 
18 | documentation, and
19 | (c) there is clear notice in each modified Data File or in the Software
20 | as well as in the documentation associated with the Data File(s) or
21 | Software that the data or software has been modified.
22 | 
23 | THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
24 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
25 | WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 | NONINFRINGEMENT OF THIRD PARTY RIGHTS.
27 | IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
28 | NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
29 | DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
30 | DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
31 | TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
32 | PERFORMANCE OF THE DATA FILES OR SOFTWARE.
33 | 
34 | Except as contained in this notice, the name of a copyright holder
35 | shall not be used in advertising or otherwise to promote the sale,
36 | use or other dealings in these Data Files or Software without prior
37 | written authorization of the copyright holder.
38 | 


--------------------------------------------------------------------------------
/data/blocks.py:
--------------------------------------------------------------------------------
  1 | blocks = [(0x0000, 0x007F, "Basic Latin"), 
  2 |           (0x0080, 0x00FF, "Latin-1 Supplement"), 
  3 |           (0x0100, 0x017F, "Latin Extended-A"), 
  4 |           (0x0180, 0x024F, "Latin Extended-B"), 
  5 |           (0x0250, 0x02AF, "IPA Extensions"), 
  6 |           (0x02B0, 0x02FF, "Spacing Modifier Letters"), 
  7 |           (0x0300, 0x036F, "Combining Diacritical Marks"), 
  8 |           (0x0370, 0x03FF, "Greek and Coptic"), 
  9 |           (0x0400, 0x04FF, "Cyrillic"), 
 10 |           (0x0500, 0x052F, "Cyrillic Supplement"), 
 11 |           (0x0530, 0x058F, "Armenian"), 
 12 |           (0x0590, 0x05FF, "Hebrew"), 
 13 |           (0x0600, 0x06FF, "Arabic"), 
 14 |           (0x0700, 0x074F, "Syriac"), 
 15 |           (0x0750, 0x077F, "Arabic Supplement"), 
 16 |           (0x0780, 0x07BF, "Thaana"), 
 17 |           (0x07C0, 0x07FF, "NKo"), 
 18 |           (0x0800, 0x083F, "Samaritan"), 
 19 |           (0x0840, 0x085F, "Mandaic"), 
 20 |           (0x08A0, 0x08FF, "Arabic Extended-A"), 
 21 |           (0x0900, 0x097F, "Devanagari"), 
 22 |           (0x0980, 0x09FF, "Bengali"), 
 23 |           (0x0A00, 0x0A7F, "Gurmukhi"), 
 24 |           (0x0A80, 0x0AFF, "Gujarati"), 
 25 |           (0x0B00, 0x0B7F, "Oriya"), 
 26 |           (0x0B80, 0x0BFF, "Tamil"), 
 27 |           (0x0C00, 0x0C7F, "Telugu"), 
 28 |           (0x0C80, 0x0CFF, "Kannada"), 
 29 |           (0x0D00, 0x0D7F, "Malayalam"), 
 30 |           (0x0D80, 0x0DFF, "Sinhala"), 
 31 |           (0x0E00, 0x0E7F, "Thai"), 
 32 |           (0x0E80, 0x0EFF, "Lao"), 
 33 |           (0x0F00, 0x0FFF, "Tibetan"), 
 34 |           (0x1000, 0x109F, "Myanmar"), 
 35 |           (0x10A0, 0x10FF, "Georgian"), 
 36 |           (0x1100, 0x11FF, "Hangul Jamo"), 
 37 |           (0x1200, 0x137F, "Ethiopic"), 
 38 |           (0x1380, 0x139F, "Ethiopic Supplement"), 
 39 |           (0x13A0, 0x13FF, "Cherokee"), 
 40 |           (0x1400, 0x167F, "Unified Canadian Aboriginal Syllabics"), 
 41 |           (0x1680, 0x169F, "Ogham"), 
 42 |           (0x16A0, 0x16FF, "Runic"), 
 43 |           (0x1700, 0x171F, "Tagalog"), 
 44 |           (0x1720, 0x173F, "Hanunoo"), 
 45 |           (0x1740, 0x175F, "Buhid"), 
 46 |           (0x1760, 0x177F, "Tagbanwa"), 
 47 |           (0x1780, 0x17FF, "Khmer"), 
 48 |           (0x1800, 0x18AF, "Mongolian"), 
 49 |           (0x18B0, 0x18FF, "Unified Canadian Aboriginal Syllabics Extended"), 
 50 |           (0x1900, 0x194F, "Limbu"), 
 51 |           (0x1950, 0x197F, "Tai Le"), 
 52 |           (0x1980, 0x19DF, "New Tai Lue"), 
 53 |           (0x19E0, 0x19FF, "Khmer Symbols"), 
 54 |           (0x1A00, 0x1A1F, "Buginese"), 
 55 |           (0x1A20, 0x1AAF, "Tai Tham"), 
 56 |           (0x1AB0, 0x1AFF, "Combining Diacritical Marks Extended"), 
 57 |           (0x1B00, 0x1B7F, "Balinese"), 
 58 |           (0x1B80, 0x1BBF, "Sundanese"), 
 59 |           (0x1BC0, 0x1BFF, "Batak"), 
 60 |           (0x1C00, 0x1C4F, "Lepcha"), 
 61 |           (0x1C50, 0x1C7F, "Ol Chiki"), 
 62 |           (0x1C80, 0x1C8F, "Cyrillic Extended-C"), 
 63 |           (0x1CC0, 0x1CCF, "Sundanese Supplement"), 
 64 |           (0x1CD0, 0x1CFF, "Vedic Extensions"), 
 65 |           (0x1D00, 0x1D7F, "Phonetic Extensions"), 
 66 |           (0x1D80, 0x1DBF, "Phonetic Extensions Supplement"), 
 67 |           (0x1DC0, 0x1DFF, "Combining Diacritical Marks Supplement"), 
 68 |           (0x1E00, 0x1EFF, "Latin Extended Additional"), 
 69 |           (0x1F00, 0x1FFF, "Greek Extended"), 
 70 |           (0x2000, 0x206F, "General Punctuation"), 
 71 |           (0x2070, 0x209F, "Superscripts and Subscripts"), 
 72 |           (0x20A0, 0x20CF, "Currency Symbols"), 
 73 |           (0x20D0, 0x20FF, "Combining Diacritical Marks for Symbols"), 
 74 |           (0x2100, 0x214F, "Letterlike Symbols"), 
 75 |           (0x2150, 0x218F, "Number Forms"), 
 76 |           (0x2190, 0x21FF, "Arrows"), 
 77 |           (0x2200, 0x22FF, "Mathematical Operators"), 
 78 |           (0x2300, 0x23FF, "Miscellaneous Technical"), 
 79 |           (0x2400, 0x243F, "Control Pictures"), 
 80 |           (0x2440, 0x245F, "Optical Character Recognition"), 
 81 |           (0x2460, 0x24FF, "Enclosed Alphanumerics"), 
 82 |           (0x2500, 0x257F, "Box Drawing"), 
 83 |           (0x2580, 0x259F, "Block Elements"), 
 84 |           (0x25A0, 0x25FF, "Geometric Shapes"), 
 85 |           (0x2600, 0x26FF, "Miscellaneous Symbols"), 
 86 |           (0x2700, 0x27BF, "Dingbats"), 
 87 |           (0x27C0, 0x27EF, "Miscellaneous Mathematical Symbols-A"), 
 88 |           (0x27F0, 0x27FF, "Supplemental Arrows-A"), 
 89 |           (0x2800, 0x28FF, "Braille Patterns"), 
 90 |           (0x2900, 0x297F, "Supplemental Arrows-B"), 
 91 |           (0x2980, 0x29FF, "Miscellaneous Mathematical Symbols-B"), 
 92 |           (0x2A00, 0x2AFF, "Supplemental Mathematical Operators"), 
 93 |           (0x2B00, 0x2BFF, "Miscellaneous Symbols and Arrows"), 
 94 |           (0x2C00, 0x2C5F, "Glagolitic"), 
 95 |           (0x2C60, 0x2C7F, "Latin Extended-C"), 
 96 |           (0x2C80, 0x2CFF, "Coptic"), 
 97 |           (0x2D00, 0x2D2F, "Georgian Supplement"), 
 98 |           (0x2D30, 0x2D7F, "Tifinagh"), 
 99 |           (0x2D80, 0x2DDF, "Ethiopic Extended"), 
100 |           (0x2DE0, 0x2DFF, "Cyrillic Extended-A"), 
101 |           (0x2E00, 0x2E7F, "Supplemental Punctuation"), 
102 |           (0x2E80, 0x2EFF, "CJK Radicals Supplement"), 
103 |           (0x2F00, 0x2FDF, "Kangxi Radicals"), 
104 |           (0x2FF0, 0x2FFF, "Ideographic Description Characters"), 
105 |           (0x3000, 0x303F, "CJK Symbols and Punctuation"), 
106 |           (0x3040, 0x309F, "Hiragana"), 
107 |           (0x30A0, 0x30FF, "Katakana"), 
108 |           (0x3100, 0x312F, "Bopomofo"), 
109 |           (0x3130, 0x318F, "Hangul Compatibility Jamo"), 
110 |           (0x3190, 0x319F, "Kanbun"), 
111 |           (0x31A0, 0x31BF, "Bopomofo Extended"), 
112 |           (0x31C0, 0x31EF, "CJK Strokes"), 
113 |           (0x31F0, 0x31FF, "Katakana Phonetic Extensions"), 
114 |           (0x3200, 0x32FF, "Enclosed CJK Letters and Months"), 
115 |           (0x3300, 0x33FF, "CJK Compatibility"), 
116 |           (0x3400, 0x4DBF, "CJK Unified Ideographs Extension A"), 
117 |           (0x4DC0, 0x4DFF, "Yijing Hexagram Symbols"), 
118 |           (0x4E00, 0x9FFF, "CJK Unified Ideographs"), 
119 |           (0xA000, 0xA48F, "Yi Syllables"), 
120 |           (0xA490, 0xA4CF, "Yi Radicals"), 
121 |           (0xA4D0, 0xA4FF, "Lisu"), 
122 |           (0xA500, 0xA63F, "Vai"), 
123 |           (0xA640, 0xA69F, "Cyrillic Extended-B"), 
124 |           (0xA6A0, 0xA6FF, "Bamum"), 
125 |           (0xA700, 0xA71F, "Modifier Tone Letters"), 
126 |           (0xA720, 0xA7FF, "Latin Extended-D"), 
127 |           (0xA800, 0xA82F, "Syloti Nagri"), 
128 |           (0xA830, 0xA83F, "Common Indic Number Forms"), 
129 |           (0xA840, 0xA87F, "Phags-pa"), 
130 |           (0xA880, 0xA8DF, "Saurashtra"), 
131 |           (0xA8E0, 0xA8FF, "Devanagari Extended"), 
132 |           (0xA900, 0xA92F, "Kayah Li"), 
133 |           (0xA930, 0xA95F, "Rejang"), 
134 |           (0xA960, 0xA97F, "Hangul Jamo Extended-A"), 
135 |           (0xA980, 0xA9DF, "Javanese"), 
136 |           (0xA9E0, 0xA9FF, "Myanmar Extended-B"), 
137 |           (0xAA00, 0xAA5F, "Cham"), 
138 |           (0xAA60, 0xAA7F, "Myanmar Extended-A"), 
139 |           (0xAA80, 0xAADF, "Tai Viet"), 
140 |           (0xAAE0, 0xAAFF, "Meetei Mayek Extensions"), 
141 |           (0xAB00, 0xAB2F, "Ethiopic Extended-A"), 
142 |           (0xAB30, 0xAB6F, "Latin Extended-E"), 
143 |           (0xAB70, 0xABBF, "Cherokee Supplement"), 
144 |           (0xABC0, 0xABFF, "Meetei Mayek"), 
145 |           (0xAC00, 0xD7AF, "Hangul Syllables"), 
146 |           (0xD7B0, 0xD7FF, "Hangul Jamo Extended-B"), 
147 |           (0xD800, 0xDB7F, "High Surrogates"), 
148 |           (0xDB80, 0xDBFF, "High Private Use Surrogates"), 
149 |           (0xDC00, 0xDFFF, "Low Surrogates"), 
150 |           (0xE000, 0xF8FF, "Private Use Area"), 
151 |           (0xF900, 0xFAFF, "CJK Compatibility Ideographs"), 
152 |           (0xFB00, 0xFB4F, "Alphabetic Presentation Forms"), 
153 |           (0xFB50, 0xFDFF, "Arabic Presentation Forms-A"), 
154 |           (0xFE00, 0xFE0F, "Variation Selectors"), 
155 |           (0xFE10, 0xFE1F, "Vertical Forms"), 
156 |           (0xFE20, 0xFE2F, "Combining Half Marks"), 
157 |           (0xFE30, 0xFE4F, "CJK Compatibility Forms"), 
158 |           (0xFE50, 0xFE6F, "Small Form Variants"), 
159 |           (0xFE70, 0xFEFF, "Arabic Presentation Forms-B"), 
160 |           (0xFF00, 0xFFEF, "Halfwidth and Fullwidth Forms"), 
161 |           (0xFFF0, 0xFFFF, "Specials"), 
162 |           (0x10000, 0x1007F, "Linear B Syllabary"), 
163 |           (0x10080, 0x100FF, "Linear B Ideograms"), 
164 |           (0x10100, 0x1013F, "Aegean Numbers"), 
165 |           (0x10140, 0x1018F, "Ancient Greek Numbers"), 
166 |           (0x10190, 0x101CF, "Ancient Symbols"), 
167 |           (0x101D0, 0x101FF, "Phaistos Disc"), 
168 |           (0x10280, 0x1029F, "Lycian"), 
169 |           (0x102A0, 0x102DF, "Carian"), 
170 |           (0x102E0, 0x102FF, "Coptic Epact Numbers"), 
171 |           (0x10300, 0x1032F, "Old Italic"), 
172 |           (0x10330, 0x1034F, "Gothic"), 
173 |           (0x10350, 0x1037F, "Old Permic"), 
174 |           (0x10380, 0x1039F, "Ugaritic"), 
175 |           (0x103A0, 0x103DF, "Old Persian"), 
176 |           (0x10400, 0x1044F, "Deseret"), 
177 |           (0x10450, 0x1047F, "Shavian"), 
178 |           (0x10480, 0x104AF, "Osmanya"), 
179 |           (0x104B0, 0x104FF, "Osage"), 
180 |           (0x10500, 0x1052F, "Elbasan"), 
181 |           (0x10530, 0x1056F, "Caucasian Albanian"), 
182 |           (0x10600, 0x1077F, "Linear A"), 
183 |           (0x10800, 0x1083F, "Cypriot Syllabary"), 
184 |           (0x10840, 0x1085F, "Imperial Aramaic"), 
185 |           (0x10860, 0x1087F, "Palmyrene"), 
186 |           (0x10880, 0x108AF, "Nabataean"), 
187 |           (0x108E0, 0x108FF, "Hatran"), 
188 |           (0x10900, 0x1091F, "Phoenician"), 
189 |           (0x10920, 0x1093F, "Lydian"), 
190 |           (0x10980, 0x1099F, "Meroitic Hieroglyphs"), 
191 |           (0x109A0, 0x109FF, "Meroitic Cursive"), 
192 |           (0x10A00, 0x10A5F, "Kharoshthi"), 
193 |           (0x10A60, 0x10A7F, "Old South Arabian"), 
194 |           (0x10A80, 0x10A9F, "Old North Arabian"), 
195 |           (0x10AC0, 0x10AFF, "Manichaean"), 
196 |           (0x10B00, 0x10B3F, "Avestan"), 
197 |           (0x10B40, 0x10B5F, "Inscriptional Parthian"), 
198 |           (0x10B60, 0x10B7F, "Inscriptional Pahlavi"), 
199 |           (0x10B80, 0x10BAF, "Psalter Pahlavi"), 
200 |           (0x10C00, 0x10C4F, "Old Turkic"), 
201 |           (0x10C80, 0x10CFF, "Old Hungarian"), 
202 |           (0x10E60, 0x10E7F, "Rumi Numeral Symbols"), 
203 |           (0x11000, 0x1107F, "Brahmi"), 
204 |           (0x11080, 0x110CF, "Kaithi"), 
205 |           (0x110D0, 0x110FF, "Sora Sompeng"), 
206 |           (0x11100, 0x1114F, "Chakma"), 
207 |           (0x11150, 0x1117F, "Mahajani"), 
208 |           (0x11180, 0x111DF, "Sharada"), 
209 |           (0x111E0, 0x111FF, "Sinhala Archaic Numbers"), 
210 |           (0x11200, 0x1124F, "Khojki"), 
211 |           (0x11280, 0x112AF, "Multani"), 
212 |           (0x112B0, 0x112FF, "Khudawadi"), 
213 |           (0x11300, 0x1137F, "Grantha"), 
214 |           (0x11400, 0x1147F, "Newa"), 
215 |           (0x11480, 0x114DF, "Tirhuta"), 
216 |           (0x11580, 0x115FF, "Siddham"), 
217 |           (0x11600, 0x1165F, "Modi"), 
218 |           (0x11660, 0x1167F, "Mongolian Supplement"), 
219 |           (0x11680, 0x116CF, "Takri"), 
220 |           (0x11700, 0x1173F, "Ahom"), 
221 |           (0x118A0, 0x118FF, "Warang Citi"), 
222 |           (0x11AC0, 0x11AFF, "Pau Cin Hau"), 
223 |           (0x11C00, 0x11C6F, "Bhaiksuki"), 
224 |           (0x11C70, 0x11CBF, "Marchen"), 
225 |           (0x12000, 0x123FF, "Cuneiform"), 
226 |           (0x12400, 0x1247F, "Cuneiform Numbers and Punctuation"), 
227 |           (0x12480, 0x1254F, "Early Dynastic Cuneiform"), 
228 |           (0x13000, 0x1342F, "Egyptian Hieroglyphs"), 
229 |           (0x14400, 0x1467F, "Anatolian Hieroglyphs"), 
230 |           (0x16800, 0x16A3F, "Bamum Supplement"), 
231 |           (0x16A40, 0x16A6F, "Mro"), 
232 |           (0x16AD0, 0x16AFF, "Bassa Vah"), 
233 |           (0x16B00, 0x16B8F, "Pahawh Hmong"), 
234 |           (0x16F00, 0x16F9F, "Miao"), 
235 |           (0x16FE0, 0x16FFF, "Ideographic Symbols and Punctuation"), 
236 |           (0x17000, 0x187FF, "Tangut"), 
237 |           (0x18800, 0x18AFF, "Tangut Components"), 
238 |           (0x1B000, 0x1B0FF, "Kana Supplement"), 
239 |           (0x1BC00, 0x1BC9F, "Duployan"), 
240 |           (0x1BCA0, 0x1BCAF, "Shorthand Format Controls"), 
241 |           (0x1D000, 0x1D0FF, "Byzantine Musical Symbols"), 
242 |           (0x1D100, 0x1D1FF, "Musical Symbols"), 
243 |           (0x1D200, 0x1D24F, "Ancient Greek Musical Notation"), 
244 |           (0x1D300, 0x1D35F, "Tai Xuan Jing Symbols"), 
245 |           (0x1D360, 0x1D37F, "Counting Rod Numerals"), 
246 |           (0x1D400, 0x1D7FF, "Mathematical Alphanumeric Symbols"), 
247 |           (0x1D800, 0x1DAAF, "Sutton SignWriting"), 
248 |           (0x1E000, 0x1E02F, "Glagolitic Supplement"), 
249 |           (0x1E800, 0x1E8DF, "Mende Kikakui"), 
250 |           (0x1E900, 0x1E95F, "Adlam"), 
251 |           (0x1EE00, 0x1EEFF, "Arabic Mathematical Alphabetic Symbols"), 
252 |           (0x1F000, 0x1F02F, "Mahjong Tiles"), 
253 |           (0x1F030, 0x1F09F, "Domino Tiles"), 
254 |           (0x1F0A0, 0x1F0FF, "Playing Cards"), 
255 |           (0x1F100, 0x1F1FF, "Enclosed Alphanumeric Supplement"), 
256 |           (0x1F200, 0x1F2FF, "Enclosed Ideographic Supplement"), 
257 |           (0x1F300, 0x1F5FF, "Miscellaneous Symbols and Pictographs"), 
258 |           (0x1F600, 0x1F64F, "Emoticons"), 
259 |           (0x1F650, 0x1F67F, "Ornamental Dingbats"), 
260 |           (0x1F680, 0x1F6FF, "Transport and Map Symbols"), 
261 |           (0x1F700, 0x1F77F, "Alchemical Symbols"), 
262 |           (0x1F780, 0x1F7FF, "Geometric Shapes Extended"), 
263 |           (0x1F800, 0x1F8FF, "Supplemental Arrows-C"), 
264 |           (0x1F900, 0x1F9FF, "Supplemental Symbols and Pictographs"), 
265 |           (0x20000, 0x2A6DF, "CJK Unified Ideographs Extension B"), 
266 |           (0x2A700, 0x2B73F, "CJK Unified Ideographs Extension C"), 
267 |           (0x2B740, 0x2B81F, "CJK Unified Ideographs Extension D"), 
268 |           (0x2B820, 0x2CEAF, "CJK Unified Ideographs Extension E"), 
269 |           (0x2F800, 0x2FA1F, "CJK Compatibility Ideographs Supplement"), 
270 |           (0xE0000, 0xE007F, "Tags"), 
271 |           (0xE0100, 0xE01EF, "Variation Selectors Supplement"), 
272 |           (0xF0000, 0xFFFFF, "Supplementary Private Use Area-A"), 
273 |           (0x100000, 0x10FFFF, "Supplementary Private Use Area-B")]
274 | 
275 | blocks_xml = ["Adlam", "Aegean_Numbers", "Ahom", "Alchemical", "Alphabetic_PF", "Anatolian_Hieroglyphs", "Ancient_Greek_Music", "Ancient_Greek_Numbers", "Ancient_Symbols", "Arabic", "Arabic_Ext_A", "Arabic_Math", "Arabic_PF_A", "Arabic_PF_B", "Arabic_Sup", "Armenian", "Arrows", "ASCII", "Avestan", "Balinese", "Bamum", "Bamum_Sup", "Bassa_Vah", "Batak", "Bengali", "Bhaiksuki", "Block_Elements", "Bopomofo", "Bopomofo_Ext", "Box_Drawing", "Brahmi", "Braille", "Buginese", "Buhid", "Byzantine_Music", "Carian", "Caucasian_Albanian", "Chakma", "Cham", "Cherokee", "Cherokee_Sup", "CJK", "CJK_Compat", "CJK_Compat_Forms", "CJK_Compat_Ideographs", "CJK_Compat_Ideographs_Sup", "CJK_Ext_A", "CJK_Ext_B", "CJK_Ext_C", "CJK_Ext_D", "CJK_Ext_E", "CJK_Radicals_Sup", "CJK_Strokes", "CJK_Symbols", "Compat_Jamo", "Control_Pictures", "Coptic", "Coptic_Epact_Numbers", "Counting_Rod", "Cuneiform", "Cuneiform_Numbers", "Currency_Symbols", "Cypriot_Syllabary", "Cyrillic", "Cyrillic_Ext_A", "Cyrillic_Ext_B", "Cyrillic_Ext_C", "Cyrillic_Sup", "Deseret", "Devanagari", "Devanagari_Ext", "Diacriticals", "Diacriticals_For_Symbols", "Diacriticals_Sup", "Diacriticals_Ext", "Dingbats", "Domino", "Duployan", "Early_Dynastic_Cuneiform", "Egyptian_Hieroglyphs", "Elbasan", "Emoticons", "Enclosed_Alphanum", "Enclosed_Alphanum_Sup", "Enclosed_CJK", "Enclosed_Ideographic_Sup", "Ethiopic", "Ethiopic_Ext", "Ethiopic_Ext_A", "Ethiopic_Sup", "Geometric_Shapes", "Geometric_Shapes_Ext", "Georgian", "Georgian_Sup", "Glagolitic", "Glagolitic_Sup", "Gothic", "Grantha", "Greek", "Greek_Ext", "Gujarati", "Gurmukhi", "Half_And_Full_Forms", "Half_Marks", "Hangul", "Hanunoo", "Hatran", "Hebrew", "High_PU_Surrogates", "High_Surrogates", "Hiragana", "IDC", "Ideographic_Symbols", "Imperial_Aramaic", "Indic_Number_Forms", "Inscriptional_Pahlavi", "Inscriptional_Parthian", "IPA_Ext", "Jamo", "Jamo_Ext_A", "Jamo_Ext_B", "Javanese", "Kaithi", "Kana_Sup", "Kanbun", "Kangxi", "Kannada", "Katakana", "Katakana_Ext", "Kayah_Li", "Kharoshthi", "Khmer", "Khmer_Symbols", "Khojki", "Khudawadi", "Lao", "Latin_1_Sup", "Latin_Ext_A", "Latin_Ext_Additional", "Latin_Ext_B", "Latin_Ext_C", "Latin_Ext_D", "Latin_Ext_E", "Lepcha", "Letterlike_Symbols", "Limbu", "Linear_A", "Linear_B_Ideograms", "Linear_B_Syllabary", "Lisu", "Low_Surrogates", "Lycian", "Lydian", "Mahajani", "Mahjong", "Malayalam", "Mandaic", "Manichaean", "Marchen", "Math_Alphanum", "Math_Operators", "Meetei_Mayek", "Meetei_Mayek_Ext", "Mende_Kikakui", "Meroitic_Cursive", "Meroitic_Hieroglyphs", "Miao", "Misc_Arrows", "Misc_Math_Symbols_A", "Misc_Math_Symbols_B", "Misc_Pictographs", "Misc_Symbols", "Misc_Technical", "Modi", "Modifier_Letters", "Modifier_Tone_Letters", "Mongolian", "Mongolian_Sup", "Mro", "Music", "Multani", "Myanmar", "Myanmar_Ext_A", "Myanmar_Ext_B", "Nabataean", "NB", "New_Tai_Lue", "Newa", "NKo", "Number_Forms", "OCR", "Ogham", "Ol_Chiki", "Old_Hungarian", "Old_Italic", "Old_North_Arabian", "Old_Permic", "Old_Persian", "Old_South_Arabian", "Old_Turkic", "Oriya", "Ornamental_Dingbats", "Osage", "Osmanya", "Pahawh_Hmong", "Palmyrene", "Pau_Cin_Hau", "Phags_Pa", "Phaistos", "Phoenician", "Phonetic_Ext", "Phonetic_Ext_Sup", "Playing_Cards", "Psalter_Pahlavi", "PUA", "Punctuation", "Rejang", "Rumi", "Runic", "Samaritan", "Saurashtra", "Sharada", "Shavian", "Shorthand_Format_Controls", "Siddham", "Sinhala", "Sinhala_Archaic_Numbers", "Small_Forms", "Sora_Sompeng", "Specials", "Sundanese", "Sundanese_Sup", "Sup_Arrows_A", "Sup_Arrows_B", "Sup_Arrows_C", "Sup_Math_Operators", "Sup_PUA_A", "Sup_PUA_B", "Sup_Punctuation", "Sup_Symbols_And_Pictographs", "Super_And_Sub", "Sutton_SignWriting", "Syloti_Nagri", "Syriac", "Tagalog", "Tagbanwa", "Tags", "Tai_Le", "Tai_Tham", "Tai_Viet", "Tai_Xuan_Jing", "Takri", "Tamil", "Tangut", "Tangut_Components", "Telugu", "Thaana", "Thai", "Tibetan", "Tifinagh", "Tirhuta", "Transport_And_Map", "UCAS", "UCAS_Ext", "Ugaritic", "Vai", "Vedic_Ext", "Vertical_Forms", "VS", "VS_Sup", "Warang_Citi", "Yi_Radicals", "Yi_Syllables", "Yijing"]
276 | 
277 | blocks_map = {"Adlam": "Adlam",
278 |               "Aegean_Numbers": "Aegean Numbers",
279 |               "Ahom": "Ahom",
280 |               "Alchemical": "Alchemical Symbols",
281 |               "Alphabetic_PF": "Alphabetic Presentation Forms",
282 |               "Anatolian_Hieroglyphs": "Anatolian Hieroglyphs",
283 |               "Ancient_Greek_Music": "Ancient Greek Musical Notation",
284 |               "Ancient_Greek_Numbers": "Ancient Greek Numbers",
285 |               "Ancient_Symbols": "Ancient Symbols",
286 |               "Arabic": "Arabic",
287 |               "Arabic_Ext_A": "Arabic Extended-A",
288 |               "Arabic_Math": "Arabic Mathematical Alphabetic Symbols",
289 |               "Arabic_PF_A": "Arabic Presentation Forms-A",
290 |               "Arabic_PF_B": "Arabic Presentation Forms-B",
291 |               "Arabic_Sup": "Arabic Supplement",
292 |               "Armenian": "Armenian",
293 |               "Arrows": "Arrows",
294 |               "ASCII": "Basic Latin",
295 |               "Avestan": "Avestan",
296 |               "Balinese": "Balinese",
297 |               "Bamum": "Bamum",
298 |               "Bamum_Sup": "Bamum Supplement",
299 |               "Bassa_Vah": "Bassa Vah",
300 |               "Batak": "Batak",
301 |               "Bengali": "Bengali",
302 |               "Bhaiksuki": "Bhaiksuki",
303 |               "Block_Elements": "Block Elements",
304 |               "Bopomofo": "Bopomofo",
305 |               "Bopomofo_Ext": "Bopomofo Extended",
306 |               "Box_Drawing": "Box Drawing",
307 |               "Brahmi": "Brahmi",
308 |               "Braille": "Braille Patterns",
309 |               "Buginese": "Buginese",
310 |               "Buhid": "Buhid",
311 |               "Byzantine_Music": "Byzantine Musical Symbols",
312 |               "Carian": "Carian",
313 |               "Caucasian_Albanian": "Caucasian Albanian",
314 |               "Chakma": "Chakma",
315 |               "Cham": "Cham",
316 |               "Cherokee": "Cherokee",
317 |               "Cherokee_Sup": "Cherokee Supplement",
318 |               "CJK": "CJK Unified Ideographs",
319 |               "CJK_Compat": "CJK Compatibility",
320 |               "CJK_Compat_Forms": "CJK Compatibility Forms",
321 |               "CJK_Compat_Ideographs": "CJK Compatibility Ideographs",
322 |               "CJK_Compat_Ideographs_Sup": "CJK Compatibility Ideographs Supplement",
323 |               "CJK_Ext_A": "CJK Unified Ideographs Extension A",
324 |               "CJK_Ext_B": "CJK Unified Ideographs Extension B",
325 |               "CJK_Ext_C": "CJK Unified Ideographs Extension C",
326 |               "CJK_Ext_D": "CJK Unified Ideographs Extension D",
327 |               "CJK_Ext_E": "CJK Unified Ideographs Extension E",
328 |               "CJK_Radicals_Sup": "CJK Radicals Supplement",
329 |               "CJK_Strokes": "CJK Strokes",
330 |               "CJK_Symbols": "CJK Symbols and Punctuation",
331 |               "Compat_Jamo": "Hangul Compatibility Jamo",
332 |               "Control_Pictures": "Control Pictures",
333 |               "Coptic": "Coptic",
334 |               "Coptic_Epact_Numbers": "Coptic Epact Numbers",
335 |               "Counting_Rod": "Counting Rod Numerals",
336 |               "Cuneiform": "Cuneiform",
337 |               "Cuneiform_Numbers": "Cuneiform Numbers and Punctuation",
338 |               "Currency_Symbols": "Currency Symbols",
339 |               "Cypriot_Syllabary": "Cypriot Syllabary",
340 |               "Cyrillic": "Cyrillic",
341 |               "Cyrillic_Ext_A": "Cyrillic Extended-A",
342 |               "Cyrillic_Ext_B": "Cyrillic Extended-B",
343 |               "Cyrillic_Ext_C": "Cyrillic Extended-C",
344 |               "Cyrillic_Sup": "Cyrillic Supplement",
345 |               "Deseret": "Deseret",
346 |               "Devanagari": "Devanagari",
347 |               "Devanagari_Ext": "Devanagari Extended",
348 |               "Diacriticals": "Combining Diacritical Marks",
349 |               "Diacriticals_For_Symbols": "Combining Diacritical Marks for Symbols",
350 |               "Diacriticals_Sup": "Combining Diacritical Marks Supplement",
351 |               "Diacriticals_Ext": "Combining Diacritical Marks Extended",
352 |               "Dingbats": "Dingbats",
353 |               "Domino": "Domino Tiles",
354 |               "Duployan": "Duployan",
355 |               "Early_Dynastic_Cuneiform": "Early Dynastic Cuneiform",
356 |               "Egyptian_Hieroglyphs": "Egyptian Hieroglyphs",
357 |               "Elbasan": "Elbasan",
358 |               "Emoticons": "Emoticons",
359 |               "Enclosed_Alphanum": "Enclosed Alphanumerics",
360 |               "Enclosed_Alphanum_Sup": "Enclosed Alphanumeric Supplement",
361 |               "Enclosed_CJK": "Enclosed CJK Letters and Months",
362 |               "Enclosed_Ideographic_Sup": "Enclosed Ideographic Supplement",
363 |               "Ethiopic": "Ethiopic",
364 |               "Ethiopic_Ext": "Ethiopic Extended",
365 |               "Ethiopic_Ext_A": "Ethiopic Extended-A",
366 |               "Ethiopic_Sup": "Ethiopic Supplement",
367 |               "Geometric_Shapes": "Geometric Shapes",
368 |               "Geometric_Shapes_Ext": "Geometric Shapes Extended",
369 |               "Georgian": "Georgian",
370 |               "Georgian_Sup": "Georgian Supplement",
371 |               "Glagolitic": "Glagolitic",
372 |               "Glagolitic_Sup": "Glagolitic Supplement",
373 |               "Gothic": "Gothic",
374 |               "Grantha": "Grantha",
375 |               "Greek": "Greek and Coptic",
376 |               "Greek_Ext": "Greek Extended",
377 |               "Gujarati": "Gujarati",
378 |               "Gurmukhi": "Gurmukhi",
379 |               "Half_And_Full_Forms": "Halfwidth and Fullwidth Forms",
380 |               "Half_Marks": "Combining Half Marks",
381 |               "Hangul": "Hangul Syllables",
382 |               "Hanunoo": "Hanunoo",
383 |               "Hatran": "Hatran",
384 |               "Hebrew": "Hebrew",
385 |               "High_PU_Surrogates": "High Private Use Surrogates",
386 |               "High_Surrogates": "High Surrogates",
387 |               "Hiragana": "Hiragana",
388 |               "IDC": "Ideographic Description Characters",
389 |               "Ideographic_Symbols": "Ideographic Symbols and Punctuation",
390 |               "Imperial_Aramaic": "Imperial Aramaic",
391 |               "Indic_Number_Forms": "Common Indic Number Forms",
392 |               "Inscriptional_Pahlavi": "Inscriptional Pahlavi",
393 |               "Inscriptional_Parthian": "Inscriptional Parthian",
394 |               "IPA_Ext": "IPA Extensions",
395 |               "Jamo": "Hangul Jamo",
396 |               "Jamo_Ext_A": "Hangul Jamo Extended-A",
397 |               "Jamo_Ext_B": "Hangul Jamo Extended-B",
398 |               "Javanese": "Javanese",
399 |               "Kaithi": "Kaithi",
400 |               "Kana_Sup": "Kana Supplement",
401 |               "Kanbun": "Kanbun",
402 |               "Kangxi": "Kangxi Radicals",
403 |               "Kannada": "Kannada",
404 |               "Katakana": "Katakana",
405 |               "Katakana_Ext": "Katakana Phonetic Extensions",
406 |               "Kayah_Li": "Kayah Li",
407 |               "Kharoshthi": "Kharoshthi",
408 |               "Khmer": "Khmer",
409 |               "Khmer_Symbols": "Khmer Symbols",
410 |               "Khojki": "Khojki",
411 |               "Khudawadi": "Khudawadi",
412 |               "Lao": "Lao",
413 |               "Latin_1_Sup": "Latin-1 Supplement",
414 |               "Latin_Ext_A": "Latin Extended-A",
415 |               "Latin_Ext_Additional": "Latin Extended Additional",
416 |               "Latin_Ext_B": "Latin Extended-B",
417 |               "Latin_Ext_C": "Latin Extended-C",
418 |               "Latin_Ext_D": "Latin Extended-D",
419 |               "Latin_Ext_E": "Latin Extended-E",
420 |               "Lepcha": "Lepcha",
421 |               "Letterlike_Symbols": "Letterlike Symbols",
422 |               "Limbu": "Limbu",
423 |               "Linear_A": "Linear A",
424 |               "Linear_B_Ideograms": "Linear B Ideograms",
425 |               "Linear_B_Syllabary": "Linear B Syllabary",
426 |               "Lisu": "Lisu",
427 |               "Low_Surrogates": "Low Surrogates",
428 |               "Lycian": "Lycian",
429 |               "Lydian": "Lydian",
430 |               "Mahajani": "Mahajani",
431 |               "Mahjong": "Mahjong Tiles",
432 |               "Malayalam": "Malayalam",
433 |               "Mandaic": "Mandaic",
434 |               "Manichaean": "Manichaean",
435 |               "Marchen": "Marchen",
436 |               "Math_Alphanum": "Mathematical Alphanumeric Symbols",
437 |               "Math_Operators": "Mathematical Operators",
438 |               "Meetei_Mayek": "Meetei Mayek",
439 |               "Meetei_Mayek_Ext": "Meetei Mayek Extensions",
440 |               "Mende_Kikakui": "Mende Kikakui",
441 |               "Meroitic_Cursive": "Meroitic Cursive",
442 |               "Meroitic_Hieroglyphs": "Meroitic Hieroglyphs",
443 |               "Miao": "Miao",
444 |               "Misc_Arrows": "Miscellaneous Symbols and Arrows",
445 |               "Misc_Math_Symbols_A": "Miscellaneous Mathematical Symbols-A",
446 |               "Misc_Math_Symbols_B": "Miscellaneous Mathematical Symbols-B",
447 |               "Misc_Pictographs": "Miscellaneous Symbols and Pictographs",
448 |               "Misc_Symbols": "Miscellaneous Symbols",
449 |               "Misc_Technical": "Miscellaneous Technical",
450 |               "Modi": "Modi",
451 |               "Modifier_Letters": "Spacing Modifier Letters",
452 |               "Modifier_Tone_Letters": "Modifier Tone Letters",
453 |               "Mongolian": "Mongolian",
454 |               "Mongolian_Sup": "Mongolian Supplement",
455 |               "Mro": "Mro",
456 |               "Music": "Musical Symbols",
457 |               "Multani": "Multani",
458 |               "Myanmar": "Myanmar",
459 |               "Myanmar_Ext_A": "Myanmar Extended-A",
460 |               "Myanmar_Ext_B": "Myanmar Extended-B",
461 |               "Nabataean": "Nabataean",
462 |               "NB": None,
463 |               "New_Tai_Lue": "New Tai Lue",
464 |               "Newa": "Newa",
465 |               "NKo": "NKo",
466 |               "Number_Forms": "Number Forms",
467 |               "OCR": "Optical Character Recognition",
468 |               "Ogham": "Ogham",
469 |               "Ol_Chiki": "Ol Chiki",
470 |               "Old_Hungarian": "Old Hungarian",
471 |               "Old_Italic": "Old Italic",
472 |               "Old_North_Arabian": "Old North Arabian",
473 |               "Old_Permic": "Old Permic",
474 |               "Old_Persian": "Old Persian",
475 |               "Old_South_Arabian": "Old South Arabian",
476 |               "Old_Turkic": "Old Turkic",
477 |               "Oriya": "Oriya",
478 |               "Ornamental_Dingbats": "Ornamental Dingbats",
479 |               "Osage": "Osage",
480 |               "Osmanya": "Osmanya",
481 |               "Pahawh_Hmong": "Pahawh Hmong",
482 |               "Palmyrene": "Palmyrene",
483 |               "Pau_Cin_Hau": "Pau Cin Hau",
484 |               "Phags_Pa": "Phags-pa",
485 |               "Phaistos": "Phaistos Disc",
486 |               "Phoenician": "Phoenician",
487 |               "Phonetic_Ext": "Phonetic Extensions",
488 |               "Phonetic_Ext_Sup": "Phonetic Extensions Supplement",
489 |               "Playing_Cards": "Playing Cards",
490 |               "Psalter_Pahlavi": "Psalter Pahlavi",
491 |               "PUA": "Private Use Area",
492 |               "Punctuation": "General Punctuation",
493 |               "Rejang": "Rejang",
494 |               "Rumi": "Rumi Numeral Symbols",
495 |               "Runic": "Runic",
496 |               "Samaritan": "Samaritan",
497 |               "Saurashtra": "Saurashtra",
498 |               "Sharada": "Sharada",
499 |               "Shavian": "Shavian",
500 |               "Shorthand_Format_Controls": "Shorthand Format Controls",
501 |               "Siddham": "Siddham",
502 |               "Sinhala": "Sinhala",
503 |               "Sinhala_Archaic_Numbers": "Sinhala Archaic Numbers",
504 |               "Small_Forms": "Small Form Variants",
505 |               "Sora_Sompeng": "Sora Sompeng",
506 |               "Specials": "Specials",
507 |               "Sundanese": "Sundanese",
508 |               "Sundanese_Sup": "Sundanese Supplement",
509 |               "Sup_Arrows_A": "Supplemental Arrows-A",
510 |               "Sup_Arrows_B": "Supplemental Arrows-B",
511 |               "Sup_Arrows_C": "Supplemental Arrows-C",
512 |               "Sup_Math_Operators": "Supplemental Mathematical Operators",
513 |               "Sup_PUA_A": "Supplementary Private Use Area-A",
514 |               "Sup_PUA_B": "Supplementary Private Use Area-B",
515 |               "Sup_Punctuation": "Supplemental Punctuation",
516 |               "Sup_Symbols_And_Pictographs": "Supplemental Symbols and Pictographs",
517 |               "Super_And_Sub": "Superscripts and Subscripts",
518 |               "Sutton_SignWriting": "Sutton SignWriting",
519 |               "Syloti_Nagri": "Syloti Nagri",
520 |               "Syriac": "Syriac",
521 |               "Tagalog": "Tagalog",
522 |               "Tagbanwa": "Tagbanwa",
523 |               "Tags": "Tags",
524 |               "Tai_Le": "Tai Le",
525 |               "Tai_Tham": "Tai Tham",
526 |               "Tai_Viet": "Tai Viet",
527 |               "Tai_Xuan_Jing": "Tai Xuan Jing Symbols",
528 |               "Takri": "Takri",
529 |               "Tamil": "Tamil",
530 |               "Tangut": "Tangut",
531 |               "Tangut_Components": "Tangut Components",
532 |               "Telugu": "Telugu",
533 |               "Thaana": "Thaana",
534 |               "Thai": "Thai",
535 |               "Tibetan": "Tibetan",
536 |               "Tifinagh": "Tifinagh",
537 |               "Tirhuta": "Tirhuta",
538 |               "Transport_And_Map": "Transport and Map Symbols",
539 |               "UCAS": "Unified Canadian Aboriginal Syllabics",
540 |               "UCAS_Ext": "Unified Canadian Aboriginal Syllabics Extended",
541 |               "Ugaritic": "Ugaritic",
542 |               "Vai": "Vai",
543 |               "Vedic_Ext": "Vedic Extensions",
544 |               "Vertical_Forms": "Vertical Forms",
545 |               "VS": "Variation Selectors",
546 |               "VS_Sup": "Variation Selectors Supplement",
547 |               "Warang_Citi": "Warang Citi",
548 |               "Yi_Radicals": "Yi Radicals",
549 |               "Yi_Syllables": "Yi Syllables",
550 |               "Yijing": "Yijing Hexagram Symbols"}
551 | 


--------------------------------------------------------------------------------
/data/gen_bidi.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | from generate import *
 3 | from misc import bidicl_alias
 4 | 
 5 | with open(base % "bidi", "w") as f:
 6 |     ranges = cprngs_by('bidi_class', 'bidi_mirrored', 'bidi_bracket_type')
 7 | 
 8 |     enummed("BidiClass", bidicl_alias, "UCD_BIDI_CLASS", ranges['bidi_class'], 'L', f)
 9 |     booled("UCD_BIDI_MIRRORED", ranges['bidi_mirrored'], False, f)
10 |     enummed_single("BidiPairedBracketType", [('(', 'Open'), (')', 'Close')],
11 |                 "UCD_BIDI_BRATYPE", ranges['bidi_bracket_type'], None, f)
12 | 
13 |     d_mirror = []
14 |     d_paired = []
15 |     for cp in cp_iter():
16 |         c = cp.codepoint()
17 | 
18 |         mirror = cp.bidi_mirror()
19 |         if mirror is not None:
20 |             d_mirror.append((c,mirror))
21 | 
22 |         paired = cp.bidi_bracket()
23 |         if paired is not True:
24 |             d_paired.append((c,paired))
25 | 
26 |     mapped16("UCD_BIDI_MIRROR", d_mirror, f)
27 |     mapped16("UCD_BIDI_PAIRED", d_paired, f)
28 | 


--------------------------------------------------------------------------------
/data/gen_bool.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | from generate import *
 3 | # generate the remaining 30 bool properties
 4 | frs = ['quick_nfd', 'quick_nfkd']
 5 | tss = ['graph_link', 'expanding_nfc', 'case_changes_casefold', 'case_changes_lower',
 6 |         'case_changes_upper', 'comp_excl', 'case_changes_title']
 7 | trs = ['case_changes_casemap', 'case_is_lower_other', 'expanding_nfd', 'graph_ext',
 8 |         'cased', 'id_start_nfkc', 'expanding_nfkc', 'comp_excl_full', 'case_is_lower',
 9 |         'case_is_upper', 'ideo_unified', 'patt_syntax', 'case_changes_casefold_nfkc',
10 |         'id_cont', 'case_ignorable', 'graph_base', 'graph_ext_other', 'id_cont_nfkc',
11 |         'expanding_nfkd', 'ideo', 'id_start']
12 | 
13 | with open(base % "bool", 'w') as f:
14 |     ranges = cprngs_by(*(frs + tss + trs))
15 |     for r in frs:
16 |         tn = "UCD_%s" % r.upper()
17 |         booled(tn, ranges[r], True, file=f)
18 |         print("pub fn %s(self) -> bool { !in_ranges(&%s, self.0) }" % (r, tn))
19 |     print()
20 | 
21 |     for s in tss:
22 |         tn = "UCD_%s" % s.upper()
23 |         booled_single(tn, ranges[s], False, file=f)
24 |         print("pub fn %s(self) -> bool { in_table(&%s, self.0) }" % (s, tn))
25 |     print()
26 | 
27 |     for r in trs:
28 |         tn = "UCD_%s" % r.upper()
29 |         booled(tn, ranges[r], False, file=f)
30 |         print("pub fn %s(self) -> bool { in_ranges(&%s, self.0) }" % (r, tn))
31 |     print()
32 | 


--------------------------------------------------------------------------------
/data/gen_decomp.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | from generate import *
 3 | from misc import dt_alias, gcb_alias, wb_alias, sb_alias
 4 | 
 5 | with open(base % "decomp", 'w') as f:
 6 |     ranges = cprngs_by('decomp_type', 'break_graph_cluster', 'break_word', 'break_sentence')
 7 |     gcb = ranges['break_graph_cluster']
 8 |     del gcb['L'], gcb['V'], gcb['T'], gcb['LV'], gcb['LVT']
 9 | 
10 |     enummed("DecompositionType", dt_alias, "UCD_DECOMP_TYPE", ranges['decomp_type'], 'none', file=f)
11 |     enummed("GraphemeClusterBreak", gcb_alias, "UCD_GCB", gcb, 'XX', file=f)
12 |     enummed("WordBreak", wb_alias, "UCD_WBRK", ranges['break_word'], 'XX', file=f)
13 |     enummed("SentenceBreak", sb_alias, "UCD_SBRK", ranges['break_sentence'], 'XX', file=f)
14 | 
15 |     t_dm = []
16 |     for cp in cp_iter():
17 |         cx = cp.codepoint()
18 |         if 44032 <= cx <= 55203: continue
19 |         inserts(t_dm, cx, cp.decomp_map())
20 |     table("UCD_DECOMP_MAP", "[((u8,u8,u8),&'static [(u8,u8,u8)])]", t_dm, file=f)
21 | 


--------------------------------------------------------------------------------
/data/gen_func.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | from generate import *
 3 | 
 4 | ss = ['hex_digit_ascii', 'prepended_concatenation_mark', 'hyphen', 'hex_digit',
 5 |         'white', 'logical_order_exception', 'term_sentence', 'dash', 'quot',
 6 |         'term_punc', 'extender', 'soft_dotted'] 
 7 | rs = ['default_ignorable', 'alpha', 'default_ignorable_other',
 8 |         'math_other', 'diacritic', 'math', 'alpha_other']
 9 | 
10 | with open(base % "function", 'w') as f:
11 |     ranges = cprngs_by(*(ss + rs))
12 |     for s in ss:
13 |         tn = "UCD_%s" % s.upper()
14 |         booled_single(tn, ranges[s], False, file=f)
15 |         print("pub fn %s(self) -> bool { in_table(&%s, self.0) }" % (s, tn))
16 |     print()
17 | 
18 |     for r in rs:
19 |         tn = "UCD_%s" % r.upper()
20 |         booled(tn, ranges[r], False, file=f)
21 |         print("pub fn %s(self) -> bool { in_ranges(&%s, self.0) }" % (r, tn))
22 |     print()
23 | 


--------------------------------------------------------------------------------
/data/gen_general.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | from generate import *
 3 | from misc import cat_alias
 4 | import re
 5 | 
 6 | with open(base % "general", "w") as f:
 7 |     blocks = []
 8 |     t_blk = []
 9 |     for el in db_iter():
10 |         if el.tag == 'block':
11 |             p = el.attrib
12 |             cpb = int(p['first-cp'], 16)
13 |             cpe = int(p['last-cp'], 16)
14 |             name = re.sub('[-\s]+','',p['name'])
15 |             x = xrng(cpb,cpe,"UnicodeBlock::%s"%name)
16 |             blocks.append(name)
17 |             t_blk.append((cpb,cpe,x))
18 |     t_blk.sort()
19 |     enum("UnicodeBlock", blocks, file=f)
20 |     table("UCD_BLOCK", "[((u8,u8,u8), (u8,u8,u8), UnicodeBlock)]",
21 |             map(lambda x:x[2], t_blk), file=f)
22 | 
23 |     ranges = cprngs_by('age', 'category', 'comb_class')
24 |     def age_transform(a):
25 |         maj = int(a[0])
26 |         min = int(a[2])
27 |         return "(%d,%d)" % (maj,min)
28 |     transformed("(u8,u8)", age_transform, "UCD_AGE", ranges['age'], None, f)
29 |     enummed("UnicodeCategory", cat_alias, "UCD_CAT", ranges['category'], 'Cn', f)
30 |     transformed("u8", str, "UCD_COMBCLS", ranges['comb_class'], 0, f)
31 | 


--------------------------------------------------------------------------------
/data/gen_misc.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | from generate import *
 3 | from misc import nt_alias, ea_alias, lb_alias
 4 | 
 5 | with open(base % "misc", "w") as f:
 6 |     ranges = cprngs_by('numeric_type', 'numeric_value', 'linebreak', 'ea_width')
 7 | 
 8 |     """numeric value property
 9 | 
10 |     numeric values held by unicode characters range from 1/160 to 10^12,
11 |     and are all rational. to represent this entire range, we can use a
12 |     tuple of i64 for the numerator (\u0F33 === -1/2 so we need negative
13 |     number support), and a u8 for the denominator (largest is 160). we
14 |     also find however that only 142 values (+NaN) are actually used, so
15 |     the minimal size is to split the table into a codepoint lookup table
16 |     and a second lookup table for the actual numeric value.
17 | 
18 |     note, NaN (most codepoints) is represented as 0/0
19 |     """
20 | 
21 |     numbers = []
22 |     nvs = ranges['numeric_value']
23 |     numbers = [n for n,_ in nvs.items() if n != (0,0)]
24 |     numbers.sort(key=lambda n:(n[0]/n[1],n[0],n[1]))
25 |     assert(len(numbers) < 256)
26 |     table("UCD_NUMS", "[(i64,u8)]", map(str,numbers), file=f)
27 | 
28 |     transformed_single("u8", numbers.index, "UCD_NUMVAL", nvs, (0,0), file=f)
29 |     enummed("NumericType", nt_alias, "UCD_NUMTYPE", ranges['numeric_type'], 'None', file=f)
30 |     enummed("EastAsianWidth", ea_alias, "UCD_EAWIDTH", ranges['ea_width'], 'N', file=f)
31 |     enummed("LinebreakClass", lb_alias, "UCD_LB", ranges['linebreak'], 'XX', file=f)
32 | 


--------------------------------------------------------------------------------
/data/gen_rem.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | from generate import *
 3 | from misc import tri_alias
 4 | 
 5 | with open(base % "rem", 'w') as f:
 6 |     ranges = cprngs_by('quick_nfc', 'quick_nfkc')
 7 |     
 8 |     qa = ranges['quick_nfc']
 9 |     qaf = set(derangify(qa[False]))
10 |     qam = set(derangify(qa[None]))
11 | 
12 |     qb = ranges['quick_nfkc']
13 |     qbf = set(derangify(qb[False]))
14 |     qbm = set(derangify(qb[None]))
15 | 
16 |     assert(qaf < qbf)
17 |     assert(qam == qbm)
18 | 
19 |     qbx = rangify(qbf - qaf)
20 | 
21 |     enummed("Trilean", tri_alias, "UCD_QNFC", qa, True, file=f)
22 |     booled("UCD_QNFKC", {False: qbx, True: []}, True, file=f)
23 | 
24 |     t_csu = []
25 |     t_csl = []
26 |     t_cst = []
27 |     t_csf = []
28 | 
29 |     t_cu = []
30 |     t_cl = []
31 |     t_ct = []
32 |     t_cf = []
33 |     t_cfn = []
34 |     t_cfc = []
35 | 
36 |     for cp in cp_iter():
37 |         cx = cp.codepoint()
38 | 
39 |         insert(t_csu, cx, cp.case_upper_simple())
40 |         insert(t_csl, cx, cp.case_lower_simple())
41 |         insert(t_cst, cx, cp.case_title_simple())
42 |         insert(t_csf, cx, cp.case_fold_simple())
43 | 
44 |         inserts(t_cu, cx, cp.case_upper())
45 |         inserts(t_cl, cx, cp.case_lower())
46 |         inserts(t_ct, cx, cp.case_title())
47 |         inserts(t_cf, cx, cp.case_fold())
48 |         inserts(t_cfn, cx, cp.case_fold_nfkc())
49 |         inserts(t_cfc, cx, cp.casefoldclosure_nfkc())
50 | 
51 |     table("UCD_CASE_SIMP_UP", "[((u8,u8,u8),(u8,u8,u8))]", t_csu, file=f)
52 |     table("UCD_CASE_SIMP_LW", "[((u8,u8,u8),(u8,u8,u8))]", t_csl, file=f)
53 |     table("UCD_CASE_SIMP_TI", "[((u8,u8,u8),(u8,u8,u8))]", t_cst, file=f)
54 |     table("UCD_CASE_SIMP_FD", "[((u8,u8,u8),(u8,u8,u8))]", t_csf, file=f)
55 | 
56 |     table("UCD_CASE_UP", "[((u8,u8,u8),&'static [(u8,u8,u8)])]", t_cu, file=f)
57 |     table("UCD_CASE_LW", "[((u8,u8,u8),&'static [(u8,u8,u8)])]", t_cl, file=f)
58 |     table("UCD_CASE_TI", "[((u8,u8,u8),&'static [(u8,u8,u8)])]", t_ct, file=f)
59 |     table("UCD_CASE_FD", "[((u8,u8,u8),&'static [(u8,u8,u8)])]", t_cf, file=f)
60 |     table("UCD_CASE_FD_NFKC", "[((u8,u8,u8),&'static [(u8,u8,u8)])]", t_cfn, file=f)
61 |     table("UCD_CASE_FD_CLOS", "[((u8,u8,u8),&'static [(u8,u8,u8)])]", t_cfc, file=f)
62 | 


--------------------------------------------------------------------------------
/data/gen_scripts.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | from generate import *
 3 | from misc import jt_alias, jg_alias, sc_alias, hst_alias, insc_alias, inpc_alias
 4 | 
 5 | with open(base % "scripts", "w") as f:
 6 |     ranges = cprngs_by('join_class', 'join_group', 'script',
 7 |                        'hangul_syll_type', 'jamo_short_name',
 8 |                        'indic_category_syll', 'indic_category_pos')
 9 | 
10 |     enummed_single("JoiningGroup", jg_alias, "UCD_JOINGRP",
11 |             ranges['join_group'], 'No_Joining_Group', file=f)
12 |     enummed("JoiningType", jt_alias, "UCD_JOINTYPE",
13 |             ranges['join_class'], 'U', file=f)
14 | 
15 |     enummed("Script", sc_alias, "UCD_SCRIPT", ranges['script'], 'Zzzz', file=f)
16 |     sc_map = dict(sc_alias)
17 |     t_scx = []
18 |     scf = lambda s: "Script::%s" % sc_map[s]
19 |     for cp in cp_iter():
20 |         cdp = cp.codepoint()
21 |         scr = cp.script()
22 |         scx = cp.script_extensions()
23 |         if len(scx) > 1 or scx[0] != scr:
24 |             y = ", ".join(map(scf, scx))
25 |             x = "(%s, &[%s])" % (xcp(cdp), y)
26 |             t_scx.append(x)
27 |     table("UCD_SCRIPT_MAP", "[&'static [Script]]",
28 |             ["&[Script::%s]" % s for _,s in sc_alias], file=f)
29 |     table("UCD_SCRIPTEXT", "[((u8,u8,u8), &'static [Script])]", t_scx, file=f)
30 | 
31 |     enum("HangulSyllableType", [x for _,x in hst_alias], file=f)
32 |     # jamo short name
33 |     # note, one character is indistinguishable in the ucd xml file,
34 |     # \u110B has the value of the empty string, but is not none
35 |     t_jsn = []
36 |     jsns = ranges['jamo_short_name']
37 |     jsns[''] = [(4363,4363)]
38 |     for j,rs in jsns.items():
39 |         for ri,rj in rs:
40 |             for c in range(ri,rj+1):
41 |                 x = '(%s,"%s")' % (xcp(c),j)
42 |                 t_jsn.append((c,x))
43 |     t_jsn.sort()
44 |     table("UCD_JSN", "[((u8,u8,u8), &'static str)]", map(lambda x:x[1], t_jsn), file=f)
45 | 
46 |     enummed("IndicSyllabicCategory", insc_alias, "UCD_INSC",
47 |             ranges['indic_category_syll'], 'Other', file=f)
48 |     enummed_single("IndicPositionalCategory", inpc_alias, "UCD_INPC",
49 |             ranges['indic_category_pos'], 'NA', file=f)
50 | 


--------------------------------------------------------------------------------
/data/gen_tests.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | from generate import *
  3 | from blocks import blocks
  4 | from misc import *
  5 | import re
  6 | 
  7 | def edx(alias, default='~~~~~~~~~~~~~~'):
  8 |     idx = {}
  9 |     for i, (n, _) in enumerate(alias):
 10 |         idx[n] = '%d' % i
 11 |     idx[default] = ''
 12 |     return idx
 13 | b2s = {False: '0', True: '1'}
 14 | 
 15 | def c2s(cp, cx=''):
 16 |     if cp is True:
 17 |         return str(cx)
 18 |     elif cp is None:
 19 |         return ''
 20 |     else:
 21 |         return str(cp)
 22 | 
 23 | def cs2s(cps, cx=''):
 24 |     return ' '.join(c2s(cp, cx) for cp in cps)
 25 | 
 26 | 
 27 | with open(base_test % "age", 'w') as f_age, \
 28 |      open(base_test % "block", 'w') as f_blk, \
 29 |      open(base_test % "cat", 'w') as f_cat, \
 30 |      open(base_test % "ccc", 'w') as f_ccc:
 31 | 
 32 |     bl_idx = {None: '-'}
 33 |     for i, (_,_,block) in enumerate(blocks):
 34 |         bl_idx[block] = '%d' % i
 35 |     cat_idx = edx(cat_alias)
 36 | 
 37 |     for cp in cp_iter():
 38 |         a = cp.age()
 39 |         if a is None: print("-", file=f_age)
 40 |         else: print(a[0]+a[2], file=f_age)
 41 | 
 42 |         # block
 43 |         #  caution, assumes that the enum has
 44 |         #  retained its indexing, otherwise
 45 |         #  test will be corrupted
 46 |         #
 47 |         #  unlike the enums defined in misc.py,
 48 |         #  the block enums are only guaranteed
 49 |         #  valid for v9.0.0, as the actual
 50 |         #  coded enum is recalculated directly
 51 |         #  from the database, whereas the one
 52 |         #  we use here is a hardcoded map from
 53 |         #  the blk property, which can't be
 54 |         #  robustly calculated
 55 |         print(bl_idx[cp.block()], file=f_blk)
 56 | 
 57 |         print(cat_idx[cp.category()], file=f_cat)
 58 |         print('%d' % cp.comb_class(), file=f_ccc)
 59 | 
 60 | 
 61 | with open(base_test % "bidi-control", 'w') as f_bctl, \
 62 |      open(base_test % "bidi-class", 'w') as f_bcls, \
 63 |      open(base_test % "bidi-mirrored", 'w') as f_bmrd, \
 64 |      open(base_test % "bidi-bratype", 'w') as f_bbt, \
 65 |      open(base_test % "bidi-mirror", 'w') as f_bmr, \
 66 |      open(base_test % "bidi-paired", 'w') as f_bpb:
 67 | 
 68 |     bidicl_idx = edx(bidicl_alias)
 69 |     bidipbt = {None: '-', '(': '(', ')': ')'}
 70 | 
 71 |     for cp in cp_iter():
 72 |         print(b2s[cp.bidi_control()], file=f_bctl)
 73 |         print(bidicl_idx[cp.bidi_class()], file=f_bcls)
 74 |         print(b2s[cp.bidi_mirrored()], file=f_bmrd)
 75 |         print(bidipbt[cp.bidi_bracket_type()], file=f_bbt)
 76 |         print(c2s(cp.bidi_mirror()), file=f_bmr)
 77 |         print(c2s(cp.bidi_bracket(), cp.codepoint()), file=f_bpb)
 78 | 
 79 | 
 80 | with open(base_test % "numval", 'w') as f_nv, \
 81 |      open(base_test % "numtype", 'w') as f_nt, \
 82 |      open(base_test % "eawidth", 'w') as f_ea, \
 83 |      open(base_test % "linebreak", 'w') as f_lb, \
 84 |      open(base_test % "deprec", 'w') as f_dep, \
 85 |      open(base_test % "varsel", 'w') as f_vs, \
 86 |      open(base_test % "nonchar", 'w') as f_nc:
 87 | 
 88 |     nt_idx = edx(nt_alias, 'None')
 89 |     ea_idx = edx(ea_alias)
 90 |     lb_idx = edx(lb_alias, 'XX')
 91 | 
 92 |     for cp in cp_iter():
 93 |         print('%d,%d'%cp.numeric_value(), file=f_nv)
 94 |         print(nt_idx[cp.numeric_type()], file=f_nt)
 95 |         print(ea_idx[cp.ea_width()], file=f_ea)
 96 |         print(lb_idx[cp.linebreak()], file=f_lb)
 97 |         print(b2s[cp.deprecated()], file=f_dep)
 98 |         print(b2s[cp.var_sel()], file=f_vs)
 99 |         print(b2s[cp.nonchar()], file=f_nc)
100 | 
101 | 
102 | with open(base_test % "joinctl", 'w') as f_jc, \
103 |      open(base_test % "jointyp", 'w') as f_jt, \
104 |      open(base_test % "joingrp", 'w') as f_jg, \
105 |      open(base_test % "indic-sylcat", 'w') as f_isc, \
106 |      open(base_test % "indic-poscat", 'w') as f_ipc, \
107 |      open(base_test % "hangulst", 'w') as f_hst, \
108 |      open(base_test % "jsn", 'w') as f_jsn:
109 | 
110 |     jt_idx = edx(jt_alias)
111 |     jg_idx = edx(jg_alias)
112 |     isc_idx = edx(insc_alias)
113 |     ipc_idx = edx(inpc_alias, 'NA')
114 |     hst_idx = edx(hst_alias, 'NA')
115 | 
116 |     for cp in cp_iter():
117 |         print(b2s[cp.join_control()], file=f_jc)
118 |         print(jt_idx[cp.join_class()], file=f_jt)
119 |         print(jg_idx[cp.join_group()], file=f_jg)
120 |         print(isc_idx[cp.indic_category_syll()], file=f_isc)
121 |         print(ipc_idx[cp.indic_category_pos()], file=f_ipc)
122 |         print(hst_idx[cp.hangul_syll_type()], file=f_hst)
123 | 
124 |         jsn = cp.jamo_short_name()
125 |         if not jsn and cp.codepoint() != 4363:
126 |             jsn = "-"
127 |         print(jsn, file=f_jsn)
128 | 
129 | with open(base_test % "script", 'w') as f_sc, \
130 |      open(base_test % "scrext", 'w') as f_scx:
131 | 
132 |     sc_idx = edx(sc_alias, 'Zzzz')
133 | 
134 |     for cp in cp_iter():
135 |         scx = cp.script_extensions()
136 |         x = " ".join(sc_idx[s] for s in scx)
137 |         print(sc_idx[cp.script()], file=f_sc)
138 |         print(x, file=f_scx)
139 | 
140 | 
141 | ss = ['hex_digit_ascii', 'prepended_concatenation_mark', 'hyphen', 'hex_digit',
142 |         'white', 'logical_order_exception', 'term_sentence', 'dash', 'quot',
143 |         'term_punc', 'extender', 'soft_dotted', 'default_ignorable', 'alpha',
144 |         'default_ignorable_other', 'math_other', 'diacritic', 'math', 'alpha_other']
145 | fs = {s:open(base_test % re.sub('_', '-', s), 'w') for s in ss}
146 | try:
147 |     for cp in cp_iter():
148 |         for s in ss:
149 |             print(b2s[getattr(cp, s)()], file=fs[s])
150 | finally:
151 |     for f in fs.values(): f.close()
152 | 
153 | 
154 | ss = ['comp_excl', 'comp_excl_full', 'quick_nfd', 'quick_nfkd', 'expanding_nfc',
155 |         'expanding_nfd', 'expanding_nfkc', 'expanding_nfkd', 'case_is_upper',
156 |         'case_is_upper_other', 'case_is_lower', 'case_is_lower_other',
157 |         'case_ignorable', 'cased', 'case_changes_casefold', 'case_changes_casefold_nfkc',
158 |         'case_changes_casemap', 'case_changes_lower', 'case_changes_upper',
159 |         'case_changes_title', 'id_start', 'id_start_other', 'id_start_nfkc', 'id_cont',
160 |         'id_cont_other', 'id_cont_nfkc', 'patt_syntax', 'patt_white', 'graph_base',
161 |         'graph_ext', 'graph_ext_other', 'graph_link', 'ideo', 'ideo_unified',
162 |         'ideo_desc_seq_bin_op', 'ideo_desc_seq_trin_op', 'ideo_desc_seq_radical']
163 | fs = {s:open(base_test % re.sub('_', '-', s), 'w') for s in ss}
164 | try:
165 |     for cp in cp_iter():
166 |         for s in ss:
167 |             print(b2s[getattr(cp, s)()], file=fs[s])
168 | finally:
169 |     for f in fs.values(): f.close()
170 | 
171 | 
172 | with open(base_test % "qnfc", 'w') as f_qnfc, \
173 |      open(base_test % "qnfkc", 'w') as f_qnfkc, \
174 |      open(base_test % "suc", 'w') as f_suc, \
175 |      open(base_test % "slc", 'w') as f_slc, \
176 |      open(base_test % "stc", 'w') as f_stc, \
177 |      open(base_test % "scf", 'w') as f_scf, \
178 |      open(base_test % "uc", 'w') as f_uc, \
179 |      open(base_test % "lc", 'w') as f_lc, \
180 |      open(base_test % "tc", 'w') as f_tc, \
181 |      open(base_test % "cf", 'w') as f_cf, \
182 |      open(base_test % "cf-nfkc", 'w') as f_cfn, \
183 |      open(base_test % "cf-closure", 'w') as f_cfc:
184 | 
185 |     t2s = edx(tri_alias)
186 | 
187 |     for cp in cp_iter():
188 |         cx = cp.codepoint()
189 |         print(t2s[cp.quick_nfc()], file=f_qnfc)
190 |         print(t2s[cp.quick_nfkc()], file=f_qnfkc)
191 | 
192 |         print(c2s(cp.case_upper_simple(), cx), file=f_suc)
193 |         print(c2s(cp.case_lower_simple(), cx), file=f_slc)
194 |         print(c2s(cp.case_title_simple(), cx), file=f_stc)
195 |         print(c2s(cp.case_fold_simple(), cx), file=f_scf)
196 | 
197 |         print(cs2s(cp.case_upper(), cx), file=f_uc)
198 |         print(cs2s(cp.case_lower(), cx), file=f_lc)
199 |         print(cs2s(cp.case_title(), cx), file=f_tc)
200 |         print(cs2s(cp.case_fold(), cx), file=f_cf)
201 |         print(cs2s(cp.case_fold_nfkc(), cx), file=f_cfn)
202 |         print(cs2s(cp.casefoldclosure_nfkc(), cx), file=f_cfc)
203 | 
204 | 
205 | with open(base_test % "dmap", 'w') as f_dm, \
206 |      open(base_test % "dtype", 'w') as f_dt, \
207 |      open(base_test % "wbrk", 'w') as f_wb, \
208 |      open(base_test % "sbrk", 'w') as f_sb, \
209 |      open(base_test % "gcb", 'w') as f_gcb:
210 | 
211 |     dt_idx = edx(dt_alias, 'none')
212 |     wb_idx = edx(wb_alias)
213 |     sb_idx = edx(sb_alias)
214 |     gcb_idx = edx(gcb_alias)
215 | 
216 |     for cp in cp_iter():
217 |         cx = cp.codepoint()
218 |         print(cs2s(cp.decomp_map(), cx), file=f_dm)
219 |         print(dt_idx[cp.decomp_type()], file=f_dt)
220 |         print(wb_idx[cp.break_word()], file=f_wb)
221 |         print(sb_idx[cp.break_sentence()], file=f_sb)
222 |         print(gcb_idx[cp.break_graph_cluster()], file=f_gcb)
223 | 


--------------------------------------------------------------------------------
/data/generate.py:
--------------------------------------------------------------------------------
 1 | from process import *
 2 | import textwrap
 3 | base = "../src/tables/%s.rs"
 4 | base_test = "../tests/data/%s.txt"
 5 | 
 6 | wrapper = textwrap.TextWrapper(
 7 |         width=80,
 8 |         break_long_words=False,
 9 |         break_on_hyphens=False,
10 |         initial_indent='  ',
11 |         subsequent_indent='  ')
12 | 
13 | def enum(name, vals, file=None):
14 |     print("#[derive(Clone,Copy,Eq,PartialEq,Debug)]", file=file)
15 |     print("pub enum %s {" % name, file=file)
16 |     for line in wrapper.wrap(', '.join(vals)):
17 |         print(line, file=file)
18 |     print("}\n", file=file)
19 | 
20 | def table(name, type, vals, file=None):
21 |     print("pub static %s: &'static %s = &[" % (name, type), file=file)
22 |     for line in wrapper.wrap(', '.join(vals)):
23 |         print(line, file=file)
24 |     print("];\n", file=file)
25 | 
26 | def xcp(cp):
27 |     cp3 = cp%256; cp>>=8
28 |     cp2 = cp%256; cp>>=8
29 |     return "(%d,%d,%d)" % (cp,cp2,cp3)
30 | 
31 | def xrng(ri, rj, dat):
32 |     return "(%s,%s,%s)" % (xcp(ri),xcp(rj),dat)
33 | 
34 | def transformed_single(type_sig, transfn, table_name, ranges, default, file=None):
35 |     t_ = []
36 |     del ranges[default]
37 |     for c,rs in ranges.items():
38 |         cn = transfn(c)
39 |         for ri,rj in rs:
40 |             for cp in range(ri,rj+1):
41 |                 x = "(%s,%s)" % (xcp(cp), cn)
42 |                 t_.append((cp,x))
43 |     t_.sort()
44 |     table(table_name, "[((u8,u8,u8), %s)]" % type_sig,
45 |             map(lambda x:x[1], t_), file=file)
46 | def transformed(type_sig, transfn, table_name, ranges, default, file=None):
47 |     t_ = []
48 |     del ranges[default]
49 |     for c,rs in ranges.items():
50 |         cn = transfn(c)
51 |         for ri,rj in rs:
52 |             x = xrng(ri,rj,cn)
53 |             t_.append((ri,rj,x))
54 |     t_.sort()
55 |     table(table_name, "[((u8,u8,u8), (u8,u8,u8), %s)]" % type_sig,
56 |             map(lambda x:x[2], t_), file=file)
57 | 
58 | def enummed(enum_name, aliases, table_name, ranges, default, file=None, tf=transformed):
59 |     enum_map = dict(aliases)
60 |     symbols = [x for _,x in aliases]
61 |     enum(enum_name, symbols, file=file)
62 |     tf(enum_name, lambda c: "%s::%s" % (enum_name, enum_map[c]),
63 |                     table_name, ranges, default, file)
64 | def enummed_single(*args, **kwargs):
65 |     kwargs['tf'] = transformed_single
66 |     enummed(*args, **kwargs)
67 | 
68 | def booled(table_name, ranges, default, file=None):
69 |     # use unit type so works with search and search_range fns unmodified
70 |     transformed("()", lambda b:{not default:"()"}[b],
71 |             table_name, ranges, default, file)
72 | def booled_single(table_name, ranges, default, file=None):
73 |     transformed_single("()", lambda b:{not default:"()"}[b],
74 |             table_name, ranges, default, file)
75 | 
76 | def mapped16(table_name, maps, file=None):
77 |     maps.sort()
78 |     t_ = []
79 |     for x,y in maps:
80 |         assert(x<65536 and y<65536)
81 |         t_.append("(%d,%d)" % (x,y))
82 |     table(table_name, "[(u16,u16)]", t_, file=file)
83 | 
84 | def insert(t_, cx, cm):
85 |     if cm is not True:
86 |         t_.append("(%s,%s)" % (xcp(cx), xcp(cm)))
87 | 
88 | def inserts(t_, cx, cms):
89 |     if not (cms and cms[0] is True):
90 |         x = ", ".join(map(xcp, cms))
91 |         t_.append("(%s, &[%s])" % (xcp(cx), x))
92 | 


--------------------------------------------------------------------------------
/data/generate.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | ./gen_bidi.py
 3 | ./gen_bool.py
 4 | ./gen_decomp.py
 5 | ./gen_func.py
 6 | ./gen_general.py
 7 | ./gen_misc.py
 8 | ./gen_rem.py
 9 | ./gen_scripts.py
10 | ./gen_tests.py
11 | 


--------------------------------------------------------------------------------
/data/misc.py:
--------------------------------------------------------------------------------
  1 | import re
  2 | r2a = lambda raw: [(x, re.sub('_','',x)) for x in raw]
  3 | 
  4 | cat_alias = [('Lu', 'UppercaseLetter'), ('Ll', 'LowercaseLetter'), ('Lt', 'TitlecaseLetter'),
  5 |              ('Lm', 'ModifierLetter'), ('Lo', 'OtherLetter'), ('Mn', 'NonspacingMark'),
  6 |              ('Mc', 'SpacingMark'), ('Me', 'EnclosingMark'), ('Nd', 'DecimalNumber'),
  7 |              ('Nl', 'LetterNumber'), ('No', 'OtherNumber'), ('Pc', 'ConnectorPunctuation'),
  8 |              ('Pd', 'DashPunctuation'), ('Ps', 'OpenPunctuation'), ('Pe', 'ClosePunctuation'),
  9 |              ('Pi', 'InitialPunctuation'), ('Pf', 'FinalPunctuation'), ('Po', 'OtherPunctuation'),
 10 |              ('Sm', 'MathSymbol'), ('Sc', 'CurrencySymbol'), ('Sk', 'ModifierSymbol'),
 11 |              ('So', 'OtherSymbol'), ('Zs', 'SpaceSeparator'), ('Zl', 'LineSeparator'),
 12 |              ('Zp', 'ParagraphSeparator'), ('Cc', 'Control'), ('Cf', 'Format'),
 13 |              ('Cs', 'Surrogate'), ('Co', 'PrivateUse'), ('Cn', 'Unassigned')]
 14 | 
 15 | bidicl_alias = [('L', 'LeftToRight'), ('R', 'RightToLeft'), ('AL', 'ArabicLetter'),
 16 |                 ('EN', 'EuropeanNumber'), ('ES', 'EuropeanSeparator'), ('ET', 'EuropeanTerminator'),
 17 |                 ('AN', 'ArabicNumber'), ('CS', 'CommonSeparator'), ('NSM', 'NonspacingMark'),
 18 |                 ('BN', 'BoundaryNeutral'), ('B', 'ParagraphSeparator'), ('S', 'SegmentSeparator'),
 19 |                 ('WS', 'WhiteSpace'), ('ON', 'OtherNeutral'), ('LRE', 'LeftToRightEmbedding'),
 20 |                 ('LRO', 'LeftToRightOverride'), ('RLE', 'RightToLeftEmbedding'), ('RLO', 'RightToLeftOverride'),
 21 |                 ('PDF', 'PopDirectionalFormat'), ('LRI', 'LeftToRightIsolate'), ('RLI', 'RightToLeftIsolate'),
 22 |                 ('FSI', 'FirstStrongIsolate'), ('PDI', 'PopDirectionalIsolate')]
 23 | 
 24 | nt_alias = [('Nu', 'Numeric'), ('De', 'Decimal'), ('Di', 'Digit')]
 25 | 
 26 | lb_alias = [('AI', 'Ambiguous'), ('AL', 'Alphabetic'), ('B2', 'BreakBoth'), ('BA', 'BreakAfter'),
 27 |             ('BB', 'BreakBefore'), ('BK', 'MandatoryBreak'), ('CB', 'ContingentBreak'),
 28 |             ('CJ', 'ConditionalJapaneseStarter'), ('CL', 'ClosePunctuation'), ('CM', 'CombiningMark'),
 29 |             ('CP', 'CloseParenthesis'), ('CR', 'CarriageReturn'), ('EB', 'EmojiBase'),
 30 |             ('EM', 'EmojiModifier'), ('EX', 'Exclamation'), ('GL', 'NonBreakingGlue'),
 31 |             ('H2', 'HangulLVSyllable'), ('H3', 'HangulLVTSyllable'), ('HL', 'HebrewLetter'),
 32 |             ('HY', 'Hyphen'), ('ID', 'Ideographic'), ('IN', 'Inseparable'), ('IS', 'InfixNumeric'),
 33 |             ('JL', 'HangulJamoL'), ('JT', 'HangulJamoT'), ('JV', 'HangulJamoV'), ('LF', 'LineFeed'),
 34 |             ('NL', 'NextLine'), ('NS', 'Nonstarter'), ('NU', 'Numeric'), ('OP', 'OpenPunctuation'),
 35 |             ('PO', 'PostfixNumeric'), ('PR', 'PrefixNumeric'), ('QU', 'Quotation'),
 36 |             ('RI', 'RegionalIndicator'), ('SA', 'ComplexContext'), ('SG', 'Surrogate'),
 37 |             ('SP', 'Space'), ('SY', 'BreakSymbols'), ('WJ', 'WordJoiner'), ('XX', 'Unknown'),
 38 |             ('ZW', 'ZeroWidthSpace'), ('ZWJ', 'ZeroWidthJoiner')]
 39 | 
 40 | ea_alias = [('Na','Narrow'), ('W','Wide'), ('N','Neutral'), ('A','Ambiguous'), ('F','FullWidth'), ('H','HalfWidth')]
 41 | 
 42 | jt_alias = [('C', 'JoinCausing'), ('D', 'DualJoining'), ('L', 'LeftJoining'),
 43 |             ('R', 'RightJoining'), ('T', 'Transparent'), ('U', 'NonJoining')]
 44 | 
 45 | jg_alias = r2a(['African_Feh', 'African_Noon', 'African_Qaf', 'Ain', 'Alaph', 'Alef', 'Beh',
 46 |                 'Beth', 'Burushaski_Yeh_Barree', 'Dal', 'Dalath_Rish', 'E', 'Farsi_Yeh', 'Fe',
 47 |                 'Feh', 'Final_Semkath', 'Gaf', 'Gamal', 'Hah', 'He', 'Heh', 'Heh_Goal', 'Heth',
 48 |                 'Kaf', 'Kaph', 'Khaph', 'Knotted_Heh', 'Lam', 'Lamadh', 'Manichaean_Aleph',
 49 |                 'Manichaean_Ayin', 'Manichaean_Beth', 'Manichaean_Daleth', 'Manichaean_Dhamedh',
 50 |                 'Manichaean_Five', 'Manichaean_Gimel', 'Manichaean_Heth', 'Manichaean_Hundred',
 51 |                 'Manichaean_Kaph', 'Manichaean_Lamedh', 'Manichaean_Mem', 'Manichaean_Nun',
 52 |                 'Manichaean_One', 'Manichaean_Pe', 'Manichaean_Qoph', 'Manichaean_Resh',
 53 |                 'Manichaean_Sadhe', 'Manichaean_Samekh', 'Manichaean_Taw', 'Manichaean_Ten',
 54 |                 'Manichaean_Teth', 'Manichaean_Thamedh', 'Manichaean_Twenty', 'Manichaean_Waw',
 55 |                 'Manichaean_Yodh', 'Manichaean_Zayin', 'Meem', 'Mim', 'No_Joining_Group', 'Noon',
 56 |                 'Nun', 'Nya', 'Pe', 'Qaf', 'Qaph', 'Reh', 'Reversed_Pe', 'Rohingya_Yeh', 'Sad',
 57 |                 'Sadhe', 'Seen', 'Semkath', 'Shin', 'Straight_Waw', 'Swash_Kaf', 'Syriac_Waw',
 58 |                 'Tah', 'Taw', 'Teh_Marbuta', 'Teh_Marbuta_Goal', 'Teth', 'Waw', 'Yeh',
 59 |                 'Yeh_Barree', 'Yeh_With_Tail', 'Yudh', 'Yudh_He', 'Zain', 'Zhain'])
 60 | 
 61 | sc_alias = [('Adlm', 'Adlam'), ('Aghb', 'CaucasianAlbanian'), ('Ahom', 'Ahom'), ('Arab', 'Arabic'),
 62 |             ('Armi', 'ImperialAramaic'), ('Armn', 'Armenian'), ('Avst', 'Avestan'), ('Bali', 'Balinese'),
 63 |             ('Bamu', 'Bamum'), ('Bass', 'BassaVah'), ('Batk', 'Batak'), ('Beng', 'Bengali'),
 64 |             ('Bhks', 'Bhaiksuki'), ('Bopo', 'Bopomofo'), ('Brah', 'Brahmi'), ('Brai', 'Braille'),
 65 |             ('Bugi', 'Buginese'), ('Buhd', 'Buhid'), ('Cakm', 'Chakma'), ('Cans', 'CanadianAboriginal'),
 66 |             ('Cari', 'Carian'), ('Cham', 'Cham'), ('Cher', 'Cherokee'), ('Copt', 'Coptic'),
 67 |             ('Cprt', 'Cypriot'), ('Cyrl', 'Cyrillic'), ('Deva', 'Devanagari'), ('Dsrt', 'Deseret'),
 68 |             ('Dupl', 'Duployan'), ('Egyp', 'EgyptianHieroglyphs'), ('Elba', 'Elbasan'), ('Ethi', 'Ethiopic'),
 69 |             ('Geor', 'Georgian'), ('Glag', 'Glagolitic'), ('Goth', 'Gothic'), ('Gran', 'Grantha'),
 70 |             ('Grek', 'Greek'), ('Gujr', 'Gujarati'), ('Guru', 'Gurmukhi'), ('Hang', 'Hangul'),
 71 |             ('Hani', 'Han'), ('Hano', 'Hanunoo'), ('Hatr', 'Hatran'), ('Hebr', 'Hebrew'),
 72 |             ('Hira', 'Hiragana'), ('Hluw', 'AnatolianHieroglyphs'), ('Hmng', 'PahawhHmong'),
 73 |             ('Hrkt', 'KatakanaOrHiragana'), ('Hung', 'OldHungarian'), ('Ital', 'OldItalic'),
 74 |             ('Java', 'Javanese'), ('Kali', 'KayahLi'), ('Kana', 'Katakana'), ('Khar', 'Kharoshthi'),
 75 |             ('Khmr', 'Khmer'), ('Khoj', 'Khojki'), ('Knda', 'Kannada'), ('Kthi', 'Kaithi'),
 76 |             ('Lana', 'TaiTham'), ('Laoo', 'Lao'), ('Latn', 'Latin'), ('Lepc', 'Lepcha'), ('Limb', 'Limbu'),
 77 |             ('Lina', 'LinearA'), ('Linb', 'LinearB'), ('Lisu', 'Lisu'), ('Lyci', 'Lycian'),
 78 |             ('Lydi', 'Lydian'), ('Mahj', 'Mahajani'), ('Mand', 'Mandaic'), ('Mani', 'Manichaean'),
 79 |             ('Marc', 'Marchen'), ('Mend', 'MendeKikakui'), ('Merc', 'MeroiticCursive'),
 80 |             ('Mero', 'MeroiticHieroglyphs'), ('Mlym', 'Malayalam'), ('Modi', 'Modi'), ('Mong', 'Mongolian'),
 81 |             ('Mroo', 'Mro'), ('Mtei', 'MeeteiMayek'), ('Mult', 'Multani'), ('Mymr', 'Myanmar'),
 82 |             ('Narb', 'OldNorthArabian'), ('Nbat', 'Nabataean'), ('Newa', 'Newa'), ('Nkoo', 'Nko'),
 83 |             ('Ogam', 'Ogham'), ('Olck', 'OlChiki'), ('Orkh', 'OldTurkic'), ('Orya', 'Oriya'),
 84 |             ('Osge', 'Osage'), ('Osma', 'Osmanya'), ('Palm', 'Palmyrene'), ('Pauc', 'PauCinHau'),
 85 |             ('Perm', 'OldPermic'), ('Phag', 'PhagsPa'), ('Phli', 'InscriptionalPahlavi'),
 86 |             ('Phlp', 'PsalterPahlavi'), ('Phnx', 'Phoenician'), ('Plrd', 'Miao'),
 87 |             ('Prti', 'InscriptionalParthian'), ('Rjng', 'Rejang'), ('Runr', 'Runic'), ('Samr', 'Samaritan'),
 88 |             ('Sarb', 'OldSouthArabian'), ('Saur', 'Saurashtra'), ('Sgnw', 'SignWriting'), ('Shaw', 'Shavian'),
 89 |             ('Shrd', 'Sharada'), ('Sidd', 'Siddham'), ('Sind', 'Khudawadi'), ('Sinh', 'Sinhala'),
 90 |             ('Sora', 'SoraSompeng'), ('Sund', 'Sundanese'), ('Sylo', 'SylotiNagri'), ('Syrc', 'Syriac'),
 91 |             ('Tagb', 'Tagbanwa'), ('Takr', 'Takri'), ('Tale', 'TaiLe'), ('Talu', 'NewTaiLue'),
 92 |             ('Taml', 'Tamil'), ('Tang', 'Tangut'), ('Tavt', 'TaiViet'), ('Telu', 'Telugu'),
 93 |             ('Tfng', 'Tifinagh'), ('Tglg', 'Tagalog'), ('Thaa', 'Thaana'), ('Thai', 'Thai'),
 94 |             ('Tibt', 'Tibetan'), ('Tirh', 'Tirhuta'), ('Ugar', 'Ugaritic'), ('Vaii', 'Vai'),
 95 |             ('Wara', 'WarangCiti'), ('Xpeo', 'OldPersian'), ('Xsux', 'Cuneiform'), ('Yiii', 'Yi'),
 96 |             ('Zinh', 'Inherited'), ('Zyyy', 'Common')]
 97 | 
 98 | hst_alias = [('L', 'LeadingJamo'), ('V', 'VowelJamo'), ('T', 'TrailingJamo'),
 99 |              ('LV', 'LVSyllable'), ('LVT', 'LVTSyllable')]
100 | 
101 | insc_alias = r2a(['Consonant_Dead', 'Tone_Mark', 'Tone_Letter', 'Vowel_Dependent', 'Number_Joiner', 'Virama',
102 |                   'Consonant_Head_Letter', 'Number', 'Consonant_Medial', 'Modifying_Letter',
103 |                   'Consonant_Succeeding_Repha', 'Consonant_Final', 'Avagraha', 'Vowel', 'Vowel_Independent',
104 |                   'Register_Shifter', 'Consonant_Killer', 'Other', 'Consonant_Subjoined', 'Joiner', 'Nukta',
105 |                   'Gemination_Mark', 'Invisible_Stacker', 'Consonant_With_Stacker', 'Consonant_Placeholder',
106 |                   'Consonant_Preceding_Repha', 'Cantillation_Mark', 'Pure_Killer', 'Non_Joiner',
107 |                   'Syllable_Modifier', 'Brahmi_Joining_Number', 'Bindu', 'Visarga', 'Consonant', 'Consonant_Prefixed'])
108 | 
109 | inpc_alias = r2a(['Left', 'Right', 'Top_And_Bottom_And_Right', 'Top_And_Left', 'Top_And_Right', 'Top',
110 |                   'Bottom_And_Right', 'Overstruck', 'Left_And_Right', 'Top_And_Left_And_Right',
111 |                   'Top_And_Bottom', 'Bottom', 'Visual_Order_Left'])
112 | 
113 | tri_alias = [(False, 'False'), (None, 'Maybe'), (True, 'True')]
114 | 
115 | dt_alias = [('can', 'Canonical'), ('com', 'Compat'), ('enc', 'Circle'), ('fin', 'Final'),
116 |             ('font', 'Font'), ('fra', 'Fraction'), ('init', 'Initial'), ('iso', 'Isolated'),
117 |             ('med', 'Medial'), ('nar', 'Narrow'), ('nb', 'Nobreak'), ('sml', 'Small'),
118 |             ('sqr', 'Square'), ('sub', 'Sub'), ('sup', 'Super'), ('vert', 'Vertical'), ('wide', 'Wide')]
119 | 
120 | gcb_alias = [('CN', 'Control'), ('CR', 'CarriageReturn'), ('LF', 'LineFeed'), ('EX', 'Extend'),
121 |              ('PP', 'Prepend'), ('EM', 'EmojiModifier'), ('EB', 'EmojiModifierBase'),
122 |              ('EBG', 'EmojiModiferBaseAfterZWJ'), ('GAZ', 'GlueAfterZWJ'), ('ZWJ', 'ZWJ'),
123 |              ('L', 'LeadingJamo'), ('V', 'VowelJamo'), ('T', 'TrailingJamo'),
124 |              ('LV', 'LVHangulSyllable'), ('LVT', 'LVTHangulSyllable'),
125 |              ('RI', 'RegionalIndicator'), ('SM', 'SpacingMark'), ('XX', 'Other')]
126 | 
127 | wb_alias = [('CR', 'CarriageReturn'), ('DQ', 'DoubleQuote'), ('EM', 'EmojiModifier'),
128 |             ('EB', 'EmojiModifierBase'), ('EBG', 'EmojiModifierBaseAfterZWJ'),
129 |             ('EX', 'ExtendNumLet'), ('Extend', 'Extend'), ('FO', 'Format'),
130 |             ('GAZ', 'GlueAfterZWJ'), ('HL', 'HebrewLetter'), ('KA', 'Katakana'),
131 |             ('LE', 'ALetter'), ('LF', 'LineFeed'), ('MB', 'MidNumericLetter'),
132 |             ('ML', 'MidLetter'), ('MN', 'MidNumeric'), ('NL', 'Newline'),
133 |             ('NU', 'Numeric'), ('RI', 'RegionalIndicator'), ('SQ', 'SingleQuote'),
134 |             ('XX', 'Other'), ('ZWJ', 'ZeroWidthJoiner')]
135 | 
136 | sb_alias = [('AT', 'ATerm'), ('CL', 'Close'), ('CR', 'CarriageReturn'), ('EX', 'Extend'),
137 |             ('FO', 'Format'), ('LE', 'OLetter'), ('LF', 'LineFeed'), ('LO', 'Lowercase'),
138 |             ('NU', 'Numeric'), ('SC', 'SentenceContinue'), ('SE', 'Separator'), ('SP', 'Space'),
139 |             ('ST', 'SentenceTerminal'), ('UP', 'Uppercase'), ('XX', 'Other')]
140 | 


--------------------------------------------------------------------------------
/data/process.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | import xml.etree.ElementTree as et
  3 | from blocks import blocks_map
  4 | import lzma
  5 | 
  6 | def _boolean(name):
  7 |     bool_decode = {
  8 |         'Y': True,
  9 |         'N': False
 10 |     }
 11 | 
 12 |     def method(self):
 13 |         return bool_decode[self._p[name]]
 14 |     return method
 15 | 
 16 | def _booly(name):
 17 |     bool_decode = {
 18 |         'Y': True,
 19 |         'N': False,
 20 |         'M': None
 21 |     }
 22 | 
 23 |     def method(self):
 24 |         return bool_decode[self._p[name]]
 25 |     return method
 26 | 
 27 | def _codepoint(name):
 28 |     def method(self):
 29 |         cp = self._p[name]
 30 |         if not cp: return None
 31 |         elif cp == '#': return True
 32 |         else: return int(cp, 16)
 33 |     return method
 34 | 
 35 | def _codepoints(name):
 36 |     def method(self):
 37 |         cps = self._p[name]
 38 |         if cps == '#':
 39 |             return [True]
 40 |         elif cps:
 41 |             return [int(cp, 16) for cp in cps.split(' ')]
 42 |         else:
 43 |             return []
 44 |     return method
 45 | 
 46 | def _raw(name):
 47 |     def method(self):
 48 |         return self._p[name]
 49 |     return method
 50 | 
 51 | def _raws(name):
 52 |     def method(self):
 53 |         return self._p[name].split(' ')
 54 |     return method
 55 | 
 56 | class Codepoint:
 57 |     def __init__(self, cp, el):
 58 |         self._c = cp
 59 |         self._p = el.attrib.copy()
 60 |         self._n = []
 61 |         for ell in el:
 62 |             if ell.tag == "name-alias":
 63 |                 n = ell.attrib['alias']
 64 |                 t = ell.attrib['type']
 65 |                 self._n.append((t,n))
 66 | 
 67 |     @staticmethod
 68 |     def from_el(el):
 69 |         props = el.attrib
 70 | 
 71 |         if 'cp' in props:
 72 |             cps = (int(props['cp'], 16),) * 2
 73 |         else:
 74 |             cps = (int(props['first-cp'], 16),
 75 |                     int(props['last-cp'], 16))
 76 | 
 77 |         cp0, cpn = cps
 78 |         for cp in range(cp0, cpn+1):
 79 |             yield Codepoint(cp, el)
 80 | 
 81 |     def codepoint(self):
 82 |         return self._c
 83 | 
 84 |     def names(self):
 85 |         names = []
 86 |         if self._p['na']:
 87 |             names.append((True, self._p['na']))
 88 |         if self._p['na1']:
 89 |             names.append((False, self._p['na1']))
 90 |         return self._n + names
 91 | 
 92 |     def age(self):
 93 |         a = self._p['age']
 94 |         if a == "unassigned":
 95 |             return None
 96 |         return a
 97 | 
 98 |     def block(self):
 99 |         return blocks_map[self._p['blk']]
100 | 
101 |     category = _raw('gc')
102 |     def comb_class(self):
103 |         return int(self._p['ccc'])
104 | 
105 |     bidi_class = _raw('bc')
106 |     bidi_mirrored = _boolean('Bidi_M')
107 |     bidi_mirror = _codepoint('bmg')
108 |     bidi_control = _boolean('Bidi_C')
109 |     bidi_bracket = _codepoint('bpb')
110 |     def bidi_bracket_type(self):
111 |         return {
112 |             'n': None,
113 |             'o': '(',
114 |             'c': ')'
115 |         }[self._p['bpt']]
116 | 
117 |     decomp_type = _raw('dt')
118 |     decomp_map = _codepoints('dm')
119 |     comp_excl = _boolean('CE')
120 |     comp_excl_full = _boolean('Comp_Ex')
121 |     quick_nfc = _booly('NFC_QC')
122 |     quick_nfd = _boolean('NFD_QC')
123 |     quick_nfkc = _booly('NFKC_QC')
124 |     quick_nfkd = _boolean('NFKD_QC')
125 | 
126 |     # deprecated
127 |     expanding_nfc = _boolean('XO_NFC')
128 |     expanding_nfd = _boolean('XO_NFD')
129 |     expanding_nfkc = _boolean('XO_NFKC')
130 |     expanding_nfkd = _boolean('XO_NFKD')
131 |     casefoldclosure_nfkc = _codepoints('FC_NFKC')
132 |     iso_comment = _raw('isc')
133 | 
134 |     numeric_type = _raw('nt')
135 |     def numeric_value(self):
136 |         nv = self._p['nv']
137 |         if nv != 'NaN':
138 |             n = list(map(int, nv.split('/')))
139 |             if len(n) == 1:
140 |                 return n[0], 1
141 |             elif len(n) == 2:
142 |                 return n[0], n[1]
143 |         return 0, 0
144 | 
145 |     join_class = _raw('jt')
146 |     join_group = _raw('jg')
147 |     join_control = _boolean('Join_C')
148 |     linebreak = _raw('lb')
149 |     ea_width = _raw('ea')
150 | 
151 |     case_is_upper = _boolean('Upper')
152 |     case_is_upper_other = _boolean('OUpper')
153 |     case_is_lower = _boolean('Lower')
154 |     case_is_lower_other = _boolean('OLower')
155 | 
156 |     case_upper = _codepoints('uc')
157 |     case_upper_simple = _codepoint('suc')
158 |     case_lower = _codepoints('lc')
159 |     case_lower_simple = _codepoint('slc')
160 |     case_title = _codepoints('tc')
161 |     case_title_simple = _codepoint('stc')
162 |     case_fold = _codepoints('cf')
163 |     case_fold_simple = _codepoint('scf')
164 |     case_fold_nfkc = _codepoints('NFKC_CF')
165 | 
166 |     case_ignorable = _boolean('CI')
167 |     cased = _boolean('Cased')
168 |     case_changes_casefold = _boolean('CWCF')
169 |     case_changes_casefold_nfkc = _boolean('CWKCF')
170 |     case_changes_casemap = _boolean('CWCM')
171 |     case_changes_lower = _boolean('CWL')
172 |     case_changes_upper = _boolean('CWU')
173 |     case_changes_title = _boolean('CWT')
174 | 
175 |     script = _raw('sc')
176 |     script_extensions = _raws('scx')
177 |     hangul_syll_type = _raw('hst')
178 |     jamo_short_name = _raw('JSN')
179 |     indic_category_syll = _raw('InSC')
180 |     #indic_category_matra = _raw('InMC') # removed
181 |     indic_category_pos = _raw('InPC')
182 | 
183 |     id_start = _boolean('IDS')
184 |     id_start_other = _boolean('OIDS')
185 |     id_start_nfkc = _boolean('XIDS')
186 |     id_cont = _boolean('IDC')
187 |     id_cont_other = _boolean('OIDC')
188 |     id_cont_nfkc = _boolean('XIDC')
189 |     patt_syntax = _boolean('Pat_Syn')
190 |     patt_white = _boolean('Pat_WS')
191 | 
192 |     dash = _boolean('Dash')
193 |     quot = _boolean('QMark')
194 |     term_punc = _boolean('Term')
195 |     term_sentence = _boolean('STerm')
196 |     diacritic = _boolean('Dia')
197 |     extender = _boolean('Ext')
198 |     prepended_concatenation_mark = _boolean('PCM')
199 |     soft_dotted = _boolean('SD')
200 |     alpha = _boolean('Alpha')
201 |     alpha_other = _boolean('OAlpha')
202 |     math = _boolean('Math')
203 |     math_other = _boolean('OMath')
204 |     hex_digit = _boolean('Hex')
205 |     hex_digit_ascii = _boolean('AHex')
206 |     default_ignorable = _boolean('DI')
207 |     default_ignorable_other = _boolean('ODI')
208 |     logical_order_exception = _boolean('LOE')
209 |     white = _boolean('WSpace')
210 |     # deprecated
211 |     hyphen = _boolean('Hyphen')
212 | 
213 |     graph_base = _boolean('Gr_Base')
214 |     graph_ext = _boolean('Gr_Ext')
215 |     graph_ext_other = _boolean('OGr_Ext')
216 |     # deprecated
217 |     graph_link = _boolean('Gr_Link')
218 | 
219 |     break_graph_cluster = _raw('GCB')
220 |     break_word = _raw('WB')
221 |     break_sentence = _raw('SB')
222 | 
223 |     ideo = _boolean('Ideo')
224 |     ideo_unified = _boolean('UIdeo')
225 |     ideo_desc_seq_bin_op = _boolean('IDSB')
226 |     ideo_desc_seq_trin_op = _boolean('IDST')
227 |     ideo_desc_seq_radical = _boolean('Radical')
228 | 
229 |     deprecated = _boolean('Dep')
230 |     var_sel = _boolean('VS')
231 |     nonchar = _boolean('NChar')
232 | 
233 |     # unihan
234 |     # tangut
235 | 
236 |     def all(self):
237 |         attrs = ['codepoint', 'names', 'age', 'block', 'category', 'comb_class',
238 |                  'bidi_class', 'bidi_mirrored', 'bidi_mirror', 'bidi_control',
239 |                  'bidi_bracket', 'bidi_bracket_type', 'decomp_type', 'decomp_map',
240 |                  'comp_excl', 'comp_excl_full', 'quick_nfc', 'quick_nfd',
241 |                  'quick_nfkc', 'quick_nfkd', 'expanding_nfc', 'expanding_nfd',
242 |                  'expanding_nfkc', 'expanding_nfkd', 'casefoldclosure_nfkc',
243 |                  'iso_comment', 'numeric_type', 'numeric_value', 'join_class',
244 |                  'join_group', 'join_control', 'linebreak', 'ea_width',
245 |                  'case_is_upper', 'case_is_upper_other', 'case_is_lower',
246 |                  'case_is_lower_other', 'case_upper', 'case_upper_simple',
247 |                  'case_lower', 'case_lower_simple', 'case_title',
248 |                  'case_title_simple', 'case_fold', 'case_fold_simple',
249 |                  'case_fold_nfkc', 'case_ignorable', 'cased',
250 |                  'case_changes_casefold', 'case_changes_casefold_nfkc',
251 |                  'case_changes_casemap', 'case_changes_lower',
252 |                  'case_changes_upper', 'case_changes_title', 'script',
253 |                  'script_extensions', 'hangul_syll_type', 'jamo_short_name',
254 |                  'indic_category_syll', 'indic_category_pos', 'id_start',
255 |                  'id_start_other', 'id_start_nfkc', 'id_cont', 'id_cont_other',
256 |                  'id_cont_nfkc', 'patt_syntax', 'patt_white', 'dash', 'quot',
257 |                  'term_punc', 'term_sentence', 'diacritic', 'extender',
258 |                  'prepended_concatenation_mark', 'soft_dotted', 'alpha',
259 |                  'alpha_other', 'math', 'math_other', 'hex_digit',
260 |                  'hex_digit_ascii', 'default_ignorable',
261 |                  'default_ignorable_other', 'logical_order_exception', 'white',
262 |                  'hyphen', 'graph_base', 'graph_ext', 'graph_ext_other',
263 |                  'graph_link', 'break_graph_cluster', 'break_word',
264 |                  'break_sentence', 'ideo', 'ideo_unified',
265 |                  'ideo_desc_seq_bin_op', 'ideo_desc_seq_trin_op',
266 |                  'ideo_desc_seq_radical', 'deprecated', 'var_sel', 'nonchar']
267 | 
268 |         return {attr: getattr(self, attr)() for attr in attrs}
269 | 
270 |         ret = {}
271 |         for attr in attrs:
272 |             m = getattr(self, attr)
273 |             try:
274 |                 ret[attr] = m()
275 |             except:
276 |                 print('%x'%self._c, attr)
277 |                 raise
278 |         return ret
279 | 
280 | # unihan db is sparse wrt fields
281 | unihan = ['kAccountingNumeric', 'kAlternateHanYu', 'kAlternateJEF', 'kAlternateKangXi', 'kAlternateMorohashi', 'kBigFive ', 'kCCCII', 'kCNS1986', 'kCNS1992', 'kCangjie', 'kCantonese', 'kCheungBauer', 'kCheungBauerIndex', 'kCihaiT', 'kCompatibilityVariant', 'kCowles', 'kDaeJaweon', 'kDefinition', 'kEACC', 'kFenn', 'kFennIndex', 'kFourCornerCode', 'kFrequency', 'kGB0', 'kGB1', 'kGB3', 'kGB5', 'kGB7', 'kGB8', 'kGradeLevel', 'kGSR', 'kHangul', 'kHanYu', 'kHanyuPinlu', 'kHanyuPinyin', 'kHDZRadBreak', 'kHKGlyph', 'kHKSCS', 'kIBMJapan', 'kIICore', 'kIRGDaeJaweon', 'kIRGDaiKanwaZiten', 'kIRGHanyuDaZidian', 'kIRGKangXi', 'kIRG_GSource', 'kIRG_HSource', 'kIRG_KPSource', 'kIRG_KSource', 'kIRG_TSource', 'kIRG_USource', 'kJa', 'kJHJ', 'kJIS0213', 'kJapaneseKun', 'kJapaneseOn', 'kJis0', 'kJis1', 'kKPS0', 'kKPS1', 'kKSC0', 'kKSC1', 'kKangXi', 'kKarlgren', 'kKorean', 'kLau', 'kMainlandTelegraph', 'kMandarin', 'kMatthews', 'kMeyerWempe', 'kMorohashi', 'kNelson', 'kOtherNumeric', 'kPhonetic', 'kPrimaryNumeric', 'kPseudoGB1', 'kRSAdobe_Japan1_6', 'kRSJapanese', 'kRSKanWa', 'kRSKangXi', 'kRSKorean', 'kRSMerged', 'kRSUnicode', 'kSBGY', 'kSemanticVariant', 'kSimplifiedVariant', 'kSpecializedSemanticVariant', 'kTaiwanTelegraph', 'kTang', 'kTotalStrokes', 'kTraditionalVariant', 'kVietnamese', 'kXHC1983', 'kWubi', 'kXerox', 'kZVariant']
282 | 
283 | # tangut db is complete wrt fields
284 | tangut = ['kRSTUnicode', 'kTGT_MergedSrc']
285 | 
286 | def db_iter(path='ucd.xml.xz'):
287 |     with lzma.open(path, 'rb') as f:
288 |         for (_, el) in et.iterparse(f):
289 |             if '}' in el.tag:
290 |                 el.tag = el.tag.split('}', 1)[1]
291 |             yield el
292 |             el.clear()
293 | 
294 | def cp_iter(path='ucd.xml.xz', debug=True):
295 |     try:
296 |         with lzma.open(path, 'rb') as f:
297 |             for (_, el) in et.iterparse(f):
298 |                 if '}' in el.tag:
299 |                     el.tag = el.tag.split('}', 1)[1]
300 | 
301 |                 if el.tag in ("char", "reserved", "noncharacter", "surrogate"):
302 |                     for cp in Codepoint.from_el(el):
303 |                         yield cp
304 |                         c = cp.codepoint()
305 |                         if debug and c > 1000:
306 |                             pass#raise StopIteration
307 |                         if debug and c % 8192 == 0:
308 |                             print('.', end='', flush=True)
309 |                     el.clear()
310 |     except StopIteration:
311 |         print('###')
312 |     print()
313 | 
314 | # count codepoints for each value of a property
315 | # for each property in ms
316 | def unicnts(*ms):
317 |     ret = {m:{} for m in ms}
318 |     for cp in cp_iter():
319 |         for m in ms:
320 |             f = getattr(cp, m)
321 |             v = f()
322 |             if v in ret[m]:
323 |                 ret[m][v] += 1
324 |             else:
325 |                 ret[m][v] = 1
326 |     return ret
327 | 
328 | # find the set of contiguous ranges mapped to by m
329 | def cprng_to(m):
330 |     ranges = []
331 |     for cp in cp_iter():
332 |         v = getattr(cp, m)()
333 |         if v is True or v is None:
334 |             continue
335 |         inr = False
336 |         for ab in ranges:
337 |             a,b = ab
338 |             if a <= v <= b:
339 |                 inr = True
340 |                 break
341 |         if inr: continue
342 | 
343 |         ranges.append((v,v))
344 |         ranges.sort()
345 |         r2 = []
346 |         a,b = ranges[0]
347 |         for c,d in ranges[1:]:
348 |             if c <= b + 1:
349 |                 b=d
350 |             else:
351 |                 r2.append((a,b))
352 |                 a,b=c,d
353 |         r2.append((a,b))
354 |         ranges = r2
355 |     return ranges
356 | 
357 | # find the set of contiguous ranges that
358 | # map to a non-default codepoint
359 | def cprng_from(m):
360 |     ranges = []
361 |     for cp in cp_iter():
362 |         v = getattr(cp, m)()
363 |         if v is True or v is None:
364 |             continue
365 | 
366 |         cx = cp.codepoint()
367 |         if ranges:
368 |             a, b = ranges[-1]
369 |             if b + 1 == cx:
370 |                 ranges[-1] = a,cx
371 |                 continue
372 |         ranges.append((cx,cx))
373 |     return ranges
374 | 
375 | # find the set of contiguous ranges that
376 | # map to each value of m, for each
377 | # property m in ms
378 | def cprngs_by(*ms):
379 |     ranges__ = {m:{} for m in ms}
380 |     for cp in cp_iter():
381 |         for m in ms:
382 |             ranges_ = ranges__[m]
383 |             v = getattr(cp, m)()
384 |             if v not in ranges_:
385 |                 ranges_[v] = []
386 |             ranges = ranges_[v]
387 | 
388 |             cx = cp.codepoint()
389 |             if ranges:
390 |                 a, b = ranges[-1]
391 |                 if b + 1 == cx:
392 |                     ranges[-1] = a,cx
393 |                     continue
394 |             ranges.append((cx,cx))
395 |     return ranges__
396 | def cprng_by(m):
397 |     return cprngs_by(m)[m]
398 | 
399 | def rangify(cps):
400 |     cps = list(cps)
401 |     cps.sort()
402 |     rs = []
403 |     a,b = -2,-2
404 |     for c in cps:
405 |         if b+1 == c:
406 |             rs[-1] = a,c
407 |             b = c
408 |         else:
409 |             rs.append((c,c))
410 |             a = b = c
411 |     return rs
412 | 
413 | def derangify(rs):
414 |     return [c for ri,rj in rs for c in range(ri,rj+1)]
415 | 


--------------------------------------------------------------------------------
/data/ucd.xml.xz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sourtin/libucd/fc89190397d21e43ab9b2246ab5b4aad88524875/data/ucd.xml.xz


--------------------------------------------------------------------------------
/prepare.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | cd tests
3 | tar xf data.tar.xz
4 | cd ..
5 | 


--------------------------------------------------------------------------------
/src/lib.rs:
--------------------------------------------------------------------------------
  1 | #![no_std]
  2 | use core::char;
  3 | use core::slice;
  4 | 
  5 | pub mod tables;
  6 | use tables::Search;
  7 | pub use tables::{
  8 |     BidiClass,
  9 |     BidiPairedBracketType,
 10 |     DecompositionType,
 11 |     EastAsianWidth,
 12 |     GraphemeClusterBreak,
 13 |     HangulSyllableType,
 14 |     IndicPositionalCategory,
 15 |     IndicSyllabicCategory,
 16 |     JoiningGroup,
 17 |     JoiningType,
 18 |     LinebreakClass,
 19 |     NumericType,
 20 |     Script,
 21 |     SentenceBreak,
 22 |     Trilean,
 23 |     UnicodeBlock,
 24 |     UnicodeCategory,
 25 |     WordBreak
 26 | };
 27 | 
 28 | // for use with numeric_value
 29 | #[derive(Clone,Copy,Eq,PartialEq,Debug,Ord,PartialOrd)]
 30 | pub enum Number {
 31 |     Integer(i64),
 32 |     Rational(i32,u32)
 33 | }
 34 | 
 35 | fn cp_decode((c1,c2,c3): (u8,u8,u8)) -> char {
 36 |     let c = (c1 as u32)*65536 + (c2 as u32)*256 + (c3 as u32);
 37 |     unsafe { char::from_u32_unchecked(c) }
 38 | }
 39 | 
 40 | enum CharIterInternal {
 41 |     Iterator(slice::Iter<'static, (u8,u8,u8)>),
 42 |     Double(char, char),
 43 |     Single(char),
 44 |     Exhausted
 45 | }
 46 | 
 47 | pub struct CharIter(CharIterInternal);
 48 | 
 49 | impl CharIter {
 50 |     pub fn new(osl: Option<&'static [(u8,u8,u8)]>, cp: char) -> CharIter {
 51 |         CharIter(match osl {
 52 |             Some(sl) => CharIterInternal::Iterator(sl.iter()),
 53 |             None => CharIterInternal::Single(cp)
 54 |         })
 55 |     }
 56 | 
 57 |     pub fn hangul(a: char, b: char) -> CharIter {
 58 |         CharIter(CharIterInternal::Double(a, b))
 59 |     }
 60 | }
 61 | 
 62 | impl Iterator for CharIter {
 63 |     type Item = char;
 64 | 
 65 |     fn next(&mut self) -> Option<char> {
 66 |         match self.0 {
 67 |             CharIterInternal::Iterator(ref mut it) => it.next().map(|c| cp_decode(*c)),
 68 |             CharIterInternal::Double(a, b) => {
 69 |                 self.0 = CharIterInternal::Single(b);
 70 |                 Some(a)
 71 |             },
 72 |             CharIterInternal::Single(c) => {
 73 |                 self.0 = CharIterInternal::Exhausted;
 74 |                 Some(c)
 75 |             },
 76 |             CharIterInternal::Exhausted => None
 77 |         }
 78 |     }
 79 | 
 80 |     fn size_hint(&self) -> (usize, Option<usize>) {
 81 |         match self.0 {
 82 |             CharIterInternal::Iterator(ref it) => it.size_hint(),
 83 |             CharIterInternal::Double(_, _) => (2, Some(2)),
 84 |             CharIterInternal::Single(_) => (1, Some(1)),
 85 |             CharIterInternal::Exhausted => (0, Some(0))
 86 |         }
 87 |     }
 88 | }
 89 | 
 90 | pub trait Codepoint where Self: core::marker::Sized {
 91 |     // general
 92 |     fn age(self) -> Option<(u8,u8)>;
 93 |     fn block(self) -> Option<UnicodeBlock>;
 94 |     fn category(self) -> UnicodeCategory;
 95 |     fn codepoint(self) -> char;
 96 |     fn iso_comment(self) -> &'static str;
 97 | 
 98 |     // function and appearance
 99 |     fn is_alphabetic(self) -> bool;
100 |     fn is_alphabetic_other(self) -> bool;
101 |     fn is_dash(self) -> bool;
102 |     fn is_default_ignorable(self) -> bool;
103 |     fn is_default_ignorable_other(self) -> bool;
104 |     fn is_deprecated(self) -> bool;
105 |     fn is_diacritic(self) -> bool;
106 |     fn is_extender(self) -> bool;
107 |     fn is_hex_digit(self) -> bool;
108 |     fn is_hex_digit_ascii(self) -> bool;
109 |     fn is_hyphen(self) -> bool;
110 |     fn is_logical_order_exception(self) -> bool;
111 |     fn is_math(self) -> bool;
112 |     fn is_math_other(self) -> bool;
113 |     fn is_noncharacter(self) -> bool;
114 |     fn is_preprended_concatenation_mark(self) -> bool;
115 |     fn is_quotation_mark(self) -> bool;
116 |     fn is_sentence_terminal(self) -> bool;
117 |     fn is_soft_dotted(self) -> bool;
118 |     fn is_terminal_punctuation(self) -> bool;
119 |     fn is_variation_selector(self) -> bool;
120 |     fn is_whitespace(self) -> bool;
121 | 
122 |     // numeric
123 |     fn numeric_type(self) -> Option<NumericType>;
124 |     fn numeric_value(self) -> Option<Number>;
125 | 
126 |     // identifiers and syntax
127 |     fn is_id_continue(self) -> bool;
128 |     fn is_id_continue_nfkc(self) -> bool;
129 |     fn is_id_continue_other(self) -> bool;
130 |     fn is_id_start(self) -> bool;
131 |     fn is_id_start_nfkc(self) -> bool;
132 |     fn is_id_start_other(self) -> bool;
133 |     fn is_pattern_syntax(self) -> bool;
134 |     fn is_pattern_whitespace(self) -> bool;
135 | 
136 |     // scripts
137 |     fn east_asian_width(self) -> EastAsianWidth;
138 |     fn hangul_syllable_type(self) -> Option<HangulSyllableType>;
139 |     fn jamo_short_name(self) -> Option<&'static str>;
140 |     fn indic_positional_category(self) -> Option<IndicPositionalCategory>;
141 |     fn indic_syllabic_category(self) -> IndicSyllabicCategory;
142 |     fn is_ideograph(self) -> bool;
143 |     fn is_ideograph_description_sequence_binary_operator(self) -> bool;
144 |     fn is_ideograph_description_sequence_radical(self) -> bool;
145 |     fn is_ideograph_description_sequence_trinary_operator(self) -> bool;
146 |     fn is_ideograph_unified(self) -> bool;
147 |     fn join_control(self) -> bool;
148 |     fn joining_group(self) -> JoiningGroup;
149 |     fn joining_type(self) -> JoiningType;
150 |     fn script(self) -> Option<Script>;
151 |     fn script_extensions(self) -> Option<&'static [Script]>;
152 | 
153 |     // bidirectionality
154 |     fn bidi_class(self) -> BidiClass;
155 |     fn bidi_is_control(self) -> bool;
156 |     fn bidi_is_mirrored(self) -> bool;
157 |     fn bidi_mirror(self) -> Option<char>;
158 |     fn bidi_paired_bracket(self) -> char;
159 |     fn bidi_paired_bracket_type(self) -> Option<BidiPairedBracketType>;
160 | 
161 |     // case
162 |     fn casefold(self) -> CharIter;
163 |     fn casefold_nfkc(self) -> CharIter;
164 |     fn casefold_nfkc_closure(self) -> CharIter;
165 |     fn casefold_simple(self) -> char;
166 |     fn changes_when_casefolded(self) -> bool;
167 |     fn changes_when_casefolded_nfkc(self) -> bool;
168 |     fn changes_when_casemapped(self) -> bool;
169 |     fn changes_when_lowercased(self) -> bool;
170 |     fn changes_when_titlecased(self) -> bool;
171 |     fn changes_when_uppercased(self) -> bool;
172 |     fn is_case_ignorable(self) -> bool;
173 |     fn is_cased(self) -> bool;
174 |     fn is_lowercase(self) -> bool;
175 |     fn is_lowercase_other(self) -> bool;
176 |     fn is_uppercase(self) -> bool;
177 |     fn is_uppercase_other (self) -> bool;
178 |     fn lowercase(self) -> CharIter;
179 |     fn lowercase_simple(self) -> char;
180 |     fn titlecase(self) -> CharIter;
181 |     fn titlecase_simple(self) -> char;
182 |     fn uppercase(self) -> CharIter;
183 |     fn uppercase_simple(self) -> char;
184 | 
185 |     // normalisation
186 |     fn canonical_combining_class(self) -> u8;
187 |     fn decomposition_map(self) -> CharIter;
188 |     fn decomposition_type(self) -> Option<DecompositionType>;
189 |     fn excluded_from_composition(self) -> bool;
190 |     fn excluded_from_composition_fully(self) -> bool;
191 |     fn expands_on_nfc(self) -> bool;
192 |     fn expands_on_nfd(self) -> bool;
193 |     fn expands_on_nfkc(self) -> bool;
194 |     fn expands_on_nfkd(self) -> bool;
195 |     fn quick_check_nfc(self) -> Trilean;
196 |     fn quick_check_nfd(self) -> bool;
197 |     fn quick_check_nfkc(self) -> Trilean;
198 |     fn quick_check_nfkd(self) -> bool;
199 | 
200 |     // segmentation
201 |     fn grapheme_cluster_break(self) -> GraphemeClusterBreak;
202 |     fn is_grapheme_base(self) -> bool;
203 |     fn is_grapheme_extend(self) -> bool;
204 |     fn is_grapheme_extend_other(self) -> bool;
205 |     fn is_grapheme_link(self) -> bool;
206 |     fn linebreak_class(self) -> Option<LinebreakClass>;
207 |     fn sentence_break(self) -> SentenceBreak;
208 |     fn word_break(self) -> WordBreak;
209 | 
210 |     // account for inbuilt char methods, which seem to be for unicode 8.0
211 |     fn is_alpha(self) -> bool { Codepoint::is_alphabetic(self) }
212 |     fn is_lower(self) -> bool { Codepoint::is_lowercase(self) }
213 |     fn is_upper(self) -> bool { Codepoint::is_uppercase(self) }
214 |     fn is_white(self) -> bool { Codepoint::is_whitespace(self) }
215 | }
216 | 
217 | impl Codepoint for char {
218 |     // general
219 |     fn age(self) -> Option<(u8,u8)> {
220 |         tables::UCD_AGE.search(self)
221 |     }
222 | 
223 |     fn block(self) -> Option<UnicodeBlock> {
224 |         tables::UCD_BLOCK.search(self)
225 |     }
226 | 
227 |     fn category(self) -> UnicodeCategory {
228 |         tables::UCD_CAT.search(self)
229 |             .unwrap_or(UnicodeCategory::Unassigned)
230 |     }
231 | 
232 |     fn codepoint(self) -> char {
233 |         self
234 |     }
235 | 
236 |     fn iso_comment(self) -> &'static str {
237 |         ""
238 |     }
239 | 
240 | 
241 | 
242 | 
243 |     // function and appearance
244 |     fn is_alphabetic(self) -> bool {
245 |         tables::UCD_ALPHA.includes(self)
246 |     }
247 | 
248 |     fn is_alphabetic_other(self) -> bool {
249 |         tables::UCD_ALPHA_OTHER.includes(self)
250 |     }
251 | 
252 |     fn is_dash(self) -> bool {
253 |         tables::UCD_DASH.includes(self)
254 |     }
255 | 
256 |     fn is_default_ignorable(self) -> bool {
257 |         tables::UCD_DEFAULT_IGNORABLE.includes(self)
258 |     }
259 | 
260 |     fn is_default_ignorable_other(self) -> bool {
261 |         tables::UCD_DEFAULT_IGNORABLE_OTHER.includes(self)
262 |     }
263 | 
264 |     fn is_deprecated(self) -> bool {
265 |         match self as u32 {
266 |             329 | 1651 | 3959 | 3961 | 6051 | 6052 |
267 |                   8298...8303 | 9001 | 9002 | 917505 => true,
268 |             _ => false
269 |         }
270 |     }
271 |     fn is_diacritic(self) -> bool {
272 |         tables::UCD_DIACRITIC.includes(self)
273 |     }
274 | 
275 |     fn is_extender(self) -> bool {
276 |         tables::UCD_EXTENDER.includes(self)
277 |     }
278 | 
279 |     fn is_hex_digit(self) -> bool {
280 |         tables::UCD_HEX_DIGIT.includes(self)
281 |     }
282 | 
283 |     fn is_hex_digit_ascii(self) -> bool {
284 |         tables::UCD_HEX_DIGIT_ASCII.includes(self)
285 |     }
286 | 
287 |     fn is_hyphen(self) -> bool {
288 |         tables::UCD_HYPHEN.includes(self)
289 |     }
290 | 
291 |     fn is_logical_order_exception(self) -> bool {
292 |         tables::UCD_LOGICAL_ORDER_EXCEPTION.includes(self)
293 |     }
294 | 
295 |     fn is_math(self) -> bool {
296 |         tables::UCD_MATH.includes(self)
297 |     }
298 | 
299 |     fn is_math_other(self) -> bool {
300 |         tables::UCD_MATH_OTHER.includes(self)
301 |     }
302 | 
303 |     fn is_noncharacter(self) -> bool {
304 |         let cp = self as u32;
305 |         (cp >= 0xfdd0 && cp <= 0xfdef)
306 |             || ((cp & 0xffff) >= 0xfffe)
307 |     }
308 | 
309 |     fn is_preprended_concatenation_mark(self) -> bool {
310 |         tables::UCD_PREPENDED_CONCATENATION_MARK.includes(self)
311 |     }
312 | 
313 |     fn is_quotation_mark(self) -> bool {
314 |         tables::UCD_QUOT.includes(self)
315 |     }
316 | 
317 |     fn is_sentence_terminal(self) -> bool {
318 |         tables::UCD_TERM_SENTENCE.includes(self)
319 |     }
320 | 
321 |     fn is_soft_dotted(self) -> bool {
322 |         tables::UCD_SOFT_DOTTED.includes(self)
323 |     }
324 | 
325 |     fn is_terminal_punctuation(self) -> bool {
326 |         tables::UCD_TERM_PUNC.includes(self)
327 |     }
328 | 
329 |     fn is_variation_selector(self) -> bool {
330 |         let cp = self as u32;
331 |         (cp >= 917760 && cp <= 917999)
332 |             || (cp >= 65024 && cp <= 65039)
333 |             || (cp >= 6155 && cp <= 6157)
334 |     }
335 | 
336 |     fn is_whitespace(self) -> bool {
337 |         tables::UCD_WHITE.includes(self)
338 |     }
339 | 
340 | 
341 | 
342 | 
343 |     // numeric
344 |     fn numeric_type(self) -> Option<NumericType> {
345 |         tables::UCD_NUMTYPE.search(self)
346 |     }
347 | 
348 |     fn numeric_value(self) -> Option<Number> {
349 |         tables::UCD_NUMVAL.search(self).map(|i| {
350 |             match tables::UCD_NUMS[i as usize] {
351 |                 (num, 1) => Number::Integer(num),
352 |                 (num, den) => Number::Rational(num as i32, den as u32)
353 |             }
354 |         })
355 |     }
356 | 
357 | 
358 | 
359 |     // identifiers and syntax
360 |     fn is_id_continue(self) -> bool {
361 |         tables::UCD_ID_CONT.includes(self)
362 |     }
363 | 
364 |     fn is_id_continue_nfkc(self) -> bool {
365 |         tables::UCD_ID_CONT_NFKC.includes(self)
366 |     }
367 | 
368 |     fn is_id_continue_other(self) -> bool {
369 |          match self as u32 {
370 |             183 | 903 | 4969...4977 | 6618 => true,
371 |             _ => false
372 |         }
373 |     }
374 | 
375 |     fn is_id_start(self) -> bool {
376 |         tables::UCD_ID_START.includes(self)
377 |     }
378 | 
379 |     fn is_id_start_nfkc(self) -> bool {
380 |         tables::UCD_ID_START_NFKC.includes(self)
381 |     }
382 | 
383 |     fn is_id_start_other(self) -> bool {
384 |         match self as u32 {
385 |             6277 | 6278 | 8472 | 8494| 12443 | 12444 => true,
386 |             _ => false
387 |         }
388 |     }
389 | 
390 |     fn is_pattern_syntax(self) -> bool {
391 |         tables::UCD_PATT_SYNTAX.includes(self)
392 |     }
393 | 
394 |     fn is_pattern_whitespace(self) -> bool {
395 |         match self as u32 {
396 |             9...13 | 32 | 133 | 8206
397 |                    | 8207 | 8232 | 8233 => true,
398 |             _ => false
399 |         }
400 |     }
401 | 
402 | 
403 | 
404 | 
405 |     // scripts
406 |     fn east_asian_width(self) -> EastAsianWidth {
407 |         tables::UCD_EAWIDTH.search(self)
408 |             .unwrap_or(EastAsianWidth::Neutral)
409 |     }
410 | 
411 |     fn hangul_syllable_type(self) -> Option<HangulSyllableType> {
412 |         let cp = self as u32;
413 |         match cp {
414 |             4352...4447 | 43360...43388 => Some(HangulSyllableType::LeadingJamo),
415 |             4448...4519 | 55216...55238 => Some(HangulSyllableType::VowelJamo),
416 |             4520...4607 | 55243...55291 => Some(HangulSyllableType::TrailingJamo),
417 |             44032...55203 => Some({
418 |                 if cp % 28 == 16 { HangulSyllableType::LVSyllable }
419 |                 else { HangulSyllableType::LVTSyllable }
420 |             }),
421 |             _ => None
422 |         }
423 |     }
424 | 
425 |     fn jamo_short_name(self) -> Option<&'static str> {
426 |         tables::UCD_JSN.search(self)
427 |     }
428 | 
429 |     fn indic_positional_category(self) -> Option<IndicPositionalCategory> {
430 |         tables::UCD_INPC.search(self)
431 |     }
432 | 
433 |     fn indic_syllabic_category(self) -> IndicSyllabicCategory {
434 |         tables::UCD_INSC.search(self)
435 |             .unwrap_or(IndicSyllabicCategory::Other)
436 |     }
437 | 
438 |     fn is_ideograph(self) -> bool {
439 |         tables::UCD_IDEO.includes(self)
440 |     }
441 | 
442 |     fn is_ideograph_description_sequence_binary_operator(self) -> bool {
443 |         match self as u32 {
444 |             12272 | 12273 | 12276...12283 => true,
445 |             _ => false
446 |         }
447 |     }
448 | 
449 |     fn is_ideograph_description_sequence_radical(self) -> bool {
450 |         match self as u32 {
451 |             11904...11929 | 11931...12019 | 12032...12245 => true,
452 |             _ => false
453 |         }
454 |     }
455 | 
456 |     fn is_ideograph_description_sequence_trinary_operator(self) -> bool {
457 |         let cp = self as u32;
458 |         cp == 12274 || cp == 12275
459 |     }
460 | 
461 |     fn is_ideograph_unified(self) -> bool {
462 |         tables::UCD_IDEO_UNIFIED.includes(self)
463 |     }
464 | 
465 |     fn join_control(self) -> bool {
466 |         let cp = self as u32;
467 |         cp == 8204 || cp == 8205
468 |     }
469 | 
470 |     fn joining_group(self) -> JoiningGroup {
471 |         tables::UCD_JOINGRP.search(self)
472 |             .unwrap_or(JoiningGroup::NoJoiningGroup)
473 |     }
474 | 
475 |     fn joining_type(self) -> JoiningType {
476 |         tables::UCD_JOINTYPE.search(self)
477 |             .unwrap_or(JoiningType::NonJoining)
478 |     }
479 | 
480 |     fn script(self) -> Option<Script> {
481 |         tables::UCD_SCRIPT.search(self)
482 |     }
483 | 
484 |     fn script_extensions(self) -> Option<&'static [Script]> {
485 |         match tables::UCD_SCRIPTEXT.search(self) {
486 |             None => self.script().map(
487 |                 |s| tables::UCD_SCRIPT_MAP[s as usize]),
488 |             x => x
489 |         }
490 |     }
491 | 
492 | 
493 | 
494 | 
495 |     // bidirectionality
496 |     fn bidi_class(self) -> BidiClass {
497 |         tables::UCD_BIDI_CLASS.search(self)
498 |             .unwrap_or(BidiClass::LeftToRight)
499 |     }
500 | 
501 |     fn bidi_is_control(self) -> bool {
502 |         match self as u32 {
503 |             1564 | 8206 | 8207 | 8234...8238 | 8294...8297 => true,
504 |             _ => false
505 |         }
506 |     }
507 | 
508 |     fn bidi_is_mirrored(self) -> bool {
509 |         tables::UCD_BIDI_MIRRORED.includes(self)
510 |     }
511 | 
512 |     fn bidi_mirror(self) -> Option<char> {
513 |         tables::UCD_BIDI_MIRROR.search(self)
514 |     }
515 | 
516 |     fn bidi_paired_bracket(self) -> char {
517 |         tables::UCD_BIDI_PAIRED.search(self)
518 |             .unwrap_or(self)
519 |     }
520 | 
521 |     fn bidi_paired_bracket_type(self) -> Option<BidiPairedBracketType> {
522 |         tables::UCD_BIDI_BRATYPE.search(self)
523 |     }
524 | 
525 | 
526 | 
527 | 
528 |     // case
529 |     fn casefold(self) -> CharIter {
530 |         CharIter::new(tables::UCD_CASE_FD.search(self), self)
531 |     }
532 | 
533 |     fn casefold_nfkc(self) -> CharIter {
534 |         CharIter::new(tables::UCD_CASE_FD_NFKC.search(self), self)
535 |     }
536 | 
537 |     fn casefold_nfkc_closure(self) -> CharIter {
538 |         CharIter::new(tables::UCD_CASE_FD_CLOS.search(self), self)
539 |     }
540 | 
541 |     fn casefold_simple(self) -> char {
542 |         tables::UCD_CASE_SIMP_FD.search(self)
543 |             .map(cp_decode)
544 |             .unwrap_or(self.codepoint())
545 |     }
546 | 
547 |     fn changes_when_casefolded(self) -> bool {
548 |         tables::UCD_CASE_CHANGES_CASEFOLD.includes(self)
549 |     }
550 | 
551 |     fn changes_when_casefolded_nfkc(self) -> bool {
552 |         tables::UCD_CASE_CHANGES_CASEFOLD_NFKC.includes(self)
553 |     }
554 | 
555 |     fn changes_when_casemapped(self) -> bool {
556 |         tables::UCD_CASE_CHANGES_CASEMAP.includes(self)
557 |     }
558 | 
559 |     fn changes_when_lowercased(self) -> bool {
560 |         tables::UCD_CASE_CHANGES_LOWER.includes(self)
561 |     }
562 | 
563 |     fn changes_when_titlecased(self) -> bool {
564 |         tables::UCD_CASE_CHANGES_TITLE.includes(self)
565 |     }
566 | 
567 |     fn changes_when_uppercased(self) -> bool {
568 |         tables::UCD_CASE_CHANGES_UPPER.includes(self)
569 |     }
570 | 
571 |     fn is_case_ignorable(self) -> bool {
572 |         tables::UCD_CASE_IGNORABLE.includes(self)
573 |     }
574 | 
575 |     fn is_cased(self) -> bool {
576 |         tables::UCD_CASED.includes(self)
577 |     }
578 | 
579 |     fn is_lowercase(self) -> bool {
580 |         tables::UCD_CASE_IS_LOWER.includes(self)
581 |     }
582 | 
583 |     fn is_lowercase_other(self) -> bool {
584 |         tables::UCD_CASE_IS_LOWER_OTHER.includes(self)
585 |     }
586 | 
587 |     fn is_uppercase(self) -> bool {
588 |         tables::UCD_CASE_IS_UPPER.includes(self)
589 |     }
590 | 
591 |     fn is_uppercase_other (self) -> bool {
592 |         match self as u32 {
593 |             8544...8559 | 9398...9423 | 127280...127305
594 |                         | 127312...127337 | 127344...127369 => true,
595 |             _ => false
596 |         }
597 |     }
598 | 
599 |     fn lowercase(self) -> CharIter {
600 |         CharIter::new(tables::UCD_CASE_LW.search(self), self)
601 |     }
602 | 
603 |     fn lowercase_simple(self) -> char {
604 |         tables::UCD_CASE_SIMP_LW.search(self)
605 |             .map(cp_decode)
606 |             .unwrap_or(self.codepoint())
607 |     }
608 | 
609 |     fn titlecase(self) -> CharIter {
610 |         CharIter::new(tables::UCD_CASE_TI.search(self), self)
611 |     }
612 | 
613 |     fn titlecase_simple(self) -> char {
614 |         tables::UCD_CASE_SIMP_TI.search(self)
615 |             .map(cp_decode)
616 |             .unwrap_or(self.codepoint())
617 |     }
618 | 
619 |     fn uppercase(self) -> CharIter {
620 |         CharIter::new(tables::UCD_CASE_UP.search(self), self)
621 |     }
622 | 
623 |     fn uppercase_simple(self) -> char {
624 |         tables::UCD_CASE_SIMP_UP.search(self)
625 |             .map(cp_decode)
626 |             .unwrap_or(self.codepoint())
627 |     }
628 | 
629 | 
630 | 
631 | 
632 |     // normalisation
633 |     fn canonical_combining_class(self) -> u8 {
634 |         tables::UCD_COMBCLS.search(self)
635 |             .unwrap_or(0)
636 |     }
637 | 
638 |     fn decomposition_map(self) -> CharIter {
639 |         // manually handle arithmetic decomposition mapping
640 |         // for hangul syllables, cutting out 11172 entries
641 |         // from the data table; implementation is directly
642 |         // from the unicode standard, chapter 3.12
643 |         const SBASE: u32 = 0xac00;
644 |         const LBASE: u32 = 0x1100;
645 |         const VBASE: u32 = 0x1161;
646 |         const TBASE: u32 = 0x11a7;
647 |         const VCOUNT: u32 = 21;
648 |         const TCOUNT: u32 = 28;
649 |         const NCOUNT: u32 = VCOUNT * TCOUNT;
650 | 
651 |         match self.hangul_syllable_type() {
652 |             Some(HangulSyllableType::LVSyllable) => unsafe {
653 |                 let sindex = (self as u32) - SBASE;
654 |                 let lindex = sindex / NCOUNT;
655 |                 let vindex = (sindex % NCOUNT) / TCOUNT;
656 |                 CharIter::hangul(
657 |                     char::from_u32_unchecked(LBASE + lindex),
658 |                     char::from_u32_unchecked(VBASE + vindex))
659 |             },
660 |             Some(HangulSyllableType::LVTSyllable) => unsafe {
661 |                 let sindex = (self as u32) - SBASE;
662 |                 let tindex = sindex % TCOUNT;
663 |                 let lvindex = sindex - tindex;
664 |                 CharIter::hangul(
665 |                     char::from_u32_unchecked(SBASE + lvindex),
666 |                     char::from_u32_unchecked(TBASE + tindex))
667 |             },
668 |             _ =>
669 |                 CharIter::new(tables::UCD_DECOMP_MAP.search(self), self)
670 |         }
671 |     }
672 | 
673 |     fn decomposition_type(self) -> Option<DecompositionType> {
674 |         tables::UCD_DECOMP_TYPE.search(self)
675 |     }
676 | 
677 |     fn excluded_from_composition(self) -> bool {
678 |         tables::UCD_COMP_EXCL.includes(self)
679 |     }
680 | 
681 |     fn excluded_from_composition_fully(self) -> bool {
682 |         tables::UCD_COMP_EXCL_FULL.includes(self)
683 |     }
684 | 
685 |     fn expands_on_nfc(self) -> bool {
686 |         tables::UCD_EXPANDING_NFC.includes(self)
687 |     }
688 | 
689 |     fn expands_on_nfd(self) -> bool {
690 |         tables::UCD_EXPANDING_NFD.includes(self)
691 |     }
692 | 
693 |     fn expands_on_nfkc(self) -> bool {
694 |         tables::UCD_EXPANDING_NFKC.includes(self)
695 |     }
696 | 
697 |     fn expands_on_nfkd(self) -> bool {
698 |         tables::UCD_EXPANDING_NFKD.includes(self)
699 |     }
700 | 
701 |     fn quick_check_nfc(self) -> Trilean {
702 |         tables::UCD_QNFC.search(self)
703 |             .unwrap_or(Trilean::True)
704 |     }
705 | 
706 |     fn quick_check_nfd(self) -> bool {
707 |         !tables::UCD_QUICK_NFD.includes(self)
708 |     }
709 | 
710 |     fn quick_check_nfkc(self) -> Trilean {
711 |         match tables::UCD_QNFKC.includes(self) {
712 |             true => Trilean::False,
713 |             false => self.quick_check_nfc()
714 |         }
715 |     }
716 | 
717 |     fn quick_check_nfkd(self) -> bool {
718 |         !tables::UCD_QUICK_NFKD.includes(self)
719 |     }
720 | 
721 | 
722 | 
723 | 
724 |     // segmentation
725 |     fn grapheme_cluster_break(self) -> GraphemeClusterBreak {
726 |         let cx = self.clone();
727 |         match self.hangul_syllable_type() {
728 |             Some(HangulSyllableType::LeadingJamo)  => GraphemeClusterBreak::LeadingJamo,
729 |             Some(HangulSyllableType::VowelJamo)    => GraphemeClusterBreak::VowelJamo,
730 |             Some(HangulSyllableType::TrailingJamo) => GraphemeClusterBreak::TrailingJamo,
731 |             Some(HangulSyllableType::LVSyllable)   => GraphemeClusterBreak::LVHangulSyllable,
732 |             Some(HangulSyllableType::LVTSyllable)  => GraphemeClusterBreak::LVTHangulSyllable,
733 |             None => tables::UCD_GCB.search(cx)
734 |                         .unwrap_or(GraphemeClusterBreak::Other)
735 |         }
736 |     }
737 | 
738 |     fn is_grapheme_base(self) -> bool {
739 |         tables::UCD_GRAPH_BASE.includes(self)
740 |     }
741 | 
742 |     fn is_grapheme_extend(self) -> bool {
743 |         tables::UCD_GRAPH_EXT.includes(self)
744 |     }
745 | 
746 |     fn is_grapheme_extend_other(self) -> bool {
747 |         tables::UCD_GRAPH_EXT_OTHER.includes(self)
748 |     }
749 | 
750 |     fn is_grapheme_link(self) -> bool {
751 |         tables::UCD_GRAPH_LINK.includes(self)
752 |     }
753 | 
754 |     fn linebreak_class(self) -> Option<LinebreakClass> {
755 |         tables::UCD_LB.search(self)
756 |     }
757 | 
758 |     fn sentence_break(self) -> SentenceBreak {
759 |         tables::UCD_SBRK.search(self)
760 |             .unwrap_or(SentenceBreak::Other)
761 |     }
762 | 
763 |     fn word_break(self) -> WordBreak {
764 |         tables::UCD_WBRK.search(self)
765 |             .unwrap_or(WordBreak::Other)
766 |     }
767 | }


--------------------------------------------------------------------------------
/src/tables/LICENSE-UCD:
--------------------------------------------------------------------------------
 1 | COPYRIGHT AND PERMISSION NOTICE
 2 | 
 3 | Copyright © 1991-2016 Unicode, Inc. All rights reserved.
 4 | Distributed under the Terms of Use in 
 5 | http://www.unicode.org/copyright.html.
 6 | 
 7 | Permission is hereby granted, free of charge, to any person obtaining
 8 | a copy of the Unicode data files and any associated documentation
 9 | (the "Data Files") or Unicode software and any associated documentation
10 | (the "Software") to deal in the Data Files or Software
11 | without restriction, including without limitation the rights to use,
12 | copy, modify, merge, publish, distribute, and/or sell copies of
13 | the Data Files or Software, and to permit persons to whom the Data Files
14 | or Software are furnished to do so, provided that
15 | (a) this copyright and permission notice appear with all copies 
16 | of the Data Files or Software,
17 | (b) this copyright and permission notice appear in associated 
18 | documentation, and
19 | (c) there is clear notice in each modified Data File or in the Software
20 | as well as in the documentation associated with the Data File(s) or
21 | Software that the data or software has been modified.
22 | 
23 | THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
24 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
25 | WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 | NONINFRINGEMENT OF THIRD PARTY RIGHTS.
27 | IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
28 | NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
29 | DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
30 | DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
31 | TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
32 | PERFORMANCE OF THE DATA FILES OR SOFTWARE.
33 | 
34 | Except as contained in this notice, the name of a copyright holder
35 | shall not be used in advertising or otherwise to promote the sale,
36 | use or other dealings in these Data Files or Software without prior
37 | written authorization of the copyright holder.
38 | 


--------------------------------------------------------------------------------
/src/tables/function.rs:
--------------------------------------------------------------------------------
  1 | pub static UCD_HEX_DIGIT_ASCII: &'static [((u8,u8,u8), ())] = &[
  2 |   ((0,0,48),()), ((0,0,49),()), ((0,0,50),()), ((0,0,51),()), ((0,0,52),()),
  3 |   ((0,0,53),()), ((0,0,54),()), ((0,0,55),()), ((0,0,56),()), ((0,0,57),()),
  4 |   ((0,0,65),()), ((0,0,66),()), ((0,0,67),()), ((0,0,68),()), ((0,0,69),()),
  5 |   ((0,0,70),()), ((0,0,97),()), ((0,0,98),()), ((0,0,99),()), ((0,0,100),()),
  6 |   ((0,0,101),()), ((0,0,102),())
  7 | ];
  8 | 
  9 | pub static UCD_PREPENDED_CONCATENATION_MARK: &'static [((u8,u8,u8), ())] = &[
 10 |   ((0,6,0),()), ((0,6,1),()), ((0,6,2),()), ((0,6,3),()), ((0,6,4),()),
 11 |   ((0,6,5),()), ((0,6,221),()), ((0,7,15),()), ((0,8,226),()), ((1,16,189),())
 12 | ];
 13 | 
 14 | pub static UCD_HYPHEN: &'static [((u8,u8,u8), ())] = &[
 15 |   ((0,0,45),()), ((0,0,173),()), ((0,5,138),()), ((0,24,6),()), ((0,32,16),()),
 16 |   ((0,32,17),()), ((0,46,23),()), ((0,48,251),()), ((0,254,99),()),
 17 |   ((0,255,13),()), ((0,255,101),())
 18 | ];
 19 | 
 20 | pub static UCD_HEX_DIGIT: &'static [((u8,u8,u8), ())] = &[
 21 |   ((0,0,48),()), ((0,0,49),()), ((0,0,50),()), ((0,0,51),()), ((0,0,52),()),
 22 |   ((0,0,53),()), ((0,0,54),()), ((0,0,55),()), ((0,0,56),()), ((0,0,57),()),
 23 |   ((0,0,65),()), ((0,0,66),()), ((0,0,67),()), ((0,0,68),()), ((0,0,69),()),
 24 |   ((0,0,70),()), ((0,0,97),()), ((0,0,98),()), ((0,0,99),()), ((0,0,100),()),
 25 |   ((0,0,101),()), ((0,0,102),()), ((0,255,16),()), ((0,255,17),()),
 26 |   ((0,255,18),()), ((0,255,19),()), ((0,255,20),()), ((0,255,21),()),
 27 |   ((0,255,22),()), ((0,255,23),()), ((0,255,24),()), ((0,255,25),()),
 28 |   ((0,255,33),()), ((0,255,34),()), ((0,255,35),()), ((0,255,36),()),
 29 |   ((0,255,37),()), ((0,255,38),()), ((0,255,65),()), ((0,255,66),()),
 30 |   ((0,255,67),()), ((0,255,68),()), ((0,255,69),()), ((0,255,70),())
 31 | ];
 32 | 
 33 | pub static UCD_WHITE: &'static [((u8,u8,u8), ())] = &[
 34 |   ((0,0,9),()), ((0,0,10),()), ((0,0,11),()), ((0,0,12),()), ((0,0,13),()),
 35 |   ((0,0,32),()), ((0,0,133),()), ((0,0,160),()), ((0,22,128),()), ((0,32,0),()),
 36 |   ((0,32,1),()), ((0,32,2),()), ((0,32,3),()), ((0,32,4),()), ((0,32,5),()),
 37 |   ((0,32,6),()), ((0,32,7),()), ((0,32,8),()), ((0,32,9),()), ((0,32,10),()),
 38 |   ((0,32,40),()), ((0,32,41),()), ((0,32,47),()), ((0,32,95),()), ((0,48,0),())
 39 | ];
 40 | 
 41 | pub static UCD_LOGICAL_ORDER_EXCEPTION: &'static [((u8,u8,u8), ())] = &[
 42 |   ((0,14,64),()), ((0,14,65),()), ((0,14,66),()), ((0,14,67),()),
 43 |   ((0,14,68),()), ((0,14,192),()), ((0,14,193),()), ((0,14,194),()),
 44 |   ((0,14,195),()), ((0,14,196),()), ((0,25,181),()), ((0,25,182),()),
 45 |   ((0,25,183),()), ((0,25,186),()), ((0,170,181),()), ((0,170,182),()),
 46 |   ((0,170,185),()), ((0,170,187),()), ((0,170,188),())
 47 | ];
 48 | 
 49 | pub static UCD_TERM_SENTENCE: &'static [((u8,u8,u8), ())] = &[
 50 |   ((0,0,33),()), ((0,0,46),()), ((0,0,63),()), ((0,5,137),()), ((0,6,31),()),
 51 |   ((0,6,212),()), ((0,7,0),()), ((0,7,1),()), ((0,7,2),()), ((0,7,249),()),
 52 |   ((0,9,100),()), ((0,9,101),()), ((0,16,74),()), ((0,16,75),()),
 53 |   ((0,19,98),()), ((0,19,103),()), ((0,19,104),()), ((0,22,110),()),
 54 |   ((0,23,53),()), ((0,23,54),()), ((0,24,3),()), ((0,24,9),()), ((0,25,68),()),
 55 |   ((0,25,69),()), ((0,26,168),()), ((0,26,169),()), ((0,26,170),()),
 56 |   ((0,26,171),()), ((0,27,90),()), ((0,27,91),()), ((0,27,94),()),
 57 |   ((0,27,95),()), ((0,28,59),()), ((0,28,60),()), ((0,28,126),()),
 58 |   ((0,28,127),()), ((0,32,60),()), ((0,32,61),()), ((0,32,71),()),
 59 |   ((0,32,72),()), ((0,32,73),()), ((0,46,46),()), ((0,46,60),()), ((0,48,2),()),
 60 |   ((0,164,255),()), ((0,166,14),()), ((0,166,15),()), ((0,166,243),()),
 61 |   ((0,166,247),()), ((0,168,118),()), ((0,168,119),()), ((0,168,206),()),
 62 |   ((0,168,207),()), ((0,169,47),()), ((0,169,200),()), ((0,169,201),()),
 63 |   ((0,170,93),()), ((0,170,94),()), ((0,170,95),()), ((0,170,240),()),
 64 |   ((0,170,241),()), ((0,171,235),()), ((0,254,82),()), ((0,254,86),()),
 65 |   ((0,254,87),()), ((0,255,1),()), ((0,255,14),()), ((0,255,31),()),
 66 |   ((0,255,97),()), ((1,10,86),()), ((1,10,87),()), ((1,16,71),()),
 67 |   ((1,16,72),()), ((1,16,190),()), ((1,16,191),()), ((1,16,192),()),
 68 |   ((1,16,193),()), ((1,17,65),()), ((1,17,66),()), ((1,17,67),()),
 69 |   ((1,17,197),()), ((1,17,198),()), ((1,17,205),()), ((1,17,222),()),
 70 |   ((1,17,223),()), ((1,18,56),()), ((1,18,57),()), ((1,18,59),()),
 71 |   ((1,18,60),()), ((1,18,169),()), ((1,20,75),()), ((1,20,76),()),
 72 |   ((1,21,194),()), ((1,21,195),()), ((1,21,201),()), ((1,21,202),()),
 73 |   ((1,21,203),()), ((1,21,204),()), ((1,21,205),()), ((1,21,206),()),
 74 |   ((1,21,207),()), ((1,21,208),()), ((1,21,209),()), ((1,21,210),()),
 75 |   ((1,21,211),()), ((1,21,212),()), ((1,21,213),()), ((1,21,214),()),
 76 |   ((1,21,215),()), ((1,22,65),()), ((1,22,66),()), ((1,23,60),()),
 77 |   ((1,23,61),()), ((1,23,62),()), ((1,28,65),()), ((1,28,66),()),
 78 |   ((1,106,110),()), ((1,106,111),()), ((1,106,245),()), ((1,107,55),()),
 79 |   ((1,107,56),()), ((1,107,68),()), ((1,188,159),()), ((1,218,136),())
 80 | ];
 81 | 
 82 | pub static UCD_DASH: &'static [((u8,u8,u8), ())] = &[
 83 |   ((0,0,45),()), ((0,5,138),()), ((0,5,190),()), ((0,20,0),()), ((0,24,6),()),
 84 |   ((0,32,16),()), ((0,32,17),()), ((0,32,18),()), ((0,32,19),()),
 85 |   ((0,32,20),()), ((0,32,21),()), ((0,32,83),()), ((0,32,123),()),
 86 |   ((0,32,139),()), ((0,34,18),()), ((0,46,23),()), ((0,46,26),()),
 87 |   ((0,46,58),()), ((0,46,59),()), ((0,46,64),()), ((0,48,28),()),
 88 |   ((0,48,48),()), ((0,48,160),()), ((0,254,49),()), ((0,254,50),()),
 89 |   ((0,254,88),()), ((0,254,99),()), ((0,255,13),())
 90 | ];
 91 | 
 92 | pub static UCD_QUOT: &'static [((u8,u8,u8), ())] = &[
 93 |   ((0,0,34),()), ((0,0,39),()), ((0,0,171),()), ((0,0,187),()), ((0,32,24),()),
 94 |   ((0,32,25),()), ((0,32,26),()), ((0,32,27),()), ((0,32,28),()),
 95 |   ((0,32,29),()), ((0,32,30),()), ((0,32,31),()), ((0,32,57),()),
 96 |   ((0,32,58),()), ((0,46,66),()), ((0,48,12),()), ((0,48,13),()),
 97 |   ((0,48,14),()), ((0,48,15),()), ((0,48,29),()), ((0,48,30),()),
 98 |   ((0,48,31),()), ((0,254,65),()), ((0,254,66),()), ((0,254,67),()),
 99 |   ((0,254,68),()), ((0,255,2),()), ((0,255,7),()), ((0,255,98),()),
100 |   ((0,255,99),())
101 | ];
102 | 
103 | pub static UCD_TERM_PUNC: &'static [((u8,u8,u8), ())] = &[
104 |   ((0,0,33),()), ((0,0,44),()), ((0,0,46),()), ((0,0,58),()), ((0,0,59),()),
105 |   ((0,0,63),()), ((0,3,126),()), ((0,3,135),()), ((0,5,137),()), ((0,5,195),()),
106 |   ((0,6,12),()), ((0,6,27),()), ((0,6,31),()), ((0,6,212),()), ((0,7,0),()),
107 |   ((0,7,1),()), ((0,7,2),()), ((0,7,3),()), ((0,7,4),()), ((0,7,5),()),
108 |   ((0,7,6),()), ((0,7,7),()), ((0,7,8),()), ((0,7,9),()), ((0,7,10),()),
109 |   ((0,7,12),()), ((0,7,248),()), ((0,7,249),()), ((0,8,48),()), ((0,8,49),()),
110 |   ((0,8,50),()), ((0,8,51),()), ((0,8,52),()), ((0,8,53),()), ((0,8,54),()),
111 |   ((0,8,55),()), ((0,8,56),()), ((0,8,57),()), ((0,8,58),()), ((0,8,59),()),
112 |   ((0,8,60),()), ((0,8,61),()), ((0,8,62),()), ((0,8,94),()), ((0,9,100),()),
113 |   ((0,9,101),()), ((0,14,90),()), ((0,14,91),()), ((0,15,8),()), ((0,15,13),()),
114 |   ((0,15,14),()), ((0,15,15),()), ((0,15,16),()), ((0,15,17),()),
115 |   ((0,15,18),()), ((0,16,74),()), ((0,16,75),()), ((0,19,97),()),
116 |   ((0,19,98),()), ((0,19,99),()), ((0,19,100),()), ((0,19,101),()),
117 |   ((0,19,102),()), ((0,19,103),()), ((0,19,104),()), ((0,22,109),()),
118 |   ((0,22,110),()), ((0,22,235),()), ((0,22,236),()), ((0,22,237),()),
119 |   ((0,23,53),()), ((0,23,54),()), ((0,23,212),()), ((0,23,213),()),
120 |   ((0,23,214),()), ((0,23,218),()), ((0,24,2),()), ((0,24,3),()), ((0,24,4),()),
121 |   ((0,24,5),()), ((0,24,8),()), ((0,24,9),()), ((0,25,68),()), ((0,25,69),()),
122 |   ((0,26,168),()), ((0,26,169),()), ((0,26,170),()), ((0,26,171),()),
123 |   ((0,27,90),()), ((0,27,91),()), ((0,27,93),()), ((0,27,94),()),
124 |   ((0,27,95),()), ((0,28,59),()), ((0,28,60),()), ((0,28,61),()),
125 |   ((0,28,62),()), ((0,28,63),()), ((0,28,126),()), ((0,28,127),()),
126 |   ((0,32,60),()), ((0,32,61),()), ((0,32,71),()), ((0,32,72),()),
127 |   ((0,32,73),()), ((0,46,46),()), ((0,46,60),()), ((0,46,65),()), ((0,48,1),()),
128 |   ((0,48,2),()), ((0,164,254),()), ((0,164,255),()), ((0,166,13),()),
129 |   ((0,166,14),()), ((0,166,15),()), ((0,166,243),()), ((0,166,244),()),
130 |   ((0,166,245),()), ((0,166,246),()), ((0,166,247),()), ((0,168,118),()),
131 |   ((0,168,119),()), ((0,168,206),()), ((0,168,207),()), ((0,169,47),()),
132 |   ((0,169,199),()), ((0,169,200),()), ((0,169,201),()), ((0,170,93),()),
133 |   ((0,170,94),()), ((0,170,95),()), ((0,170,223),()), ((0,170,240),()),
134 |   ((0,170,241),()), ((0,171,235),()), ((0,254,80),()), ((0,254,81),()),
135 |   ((0,254,82),()), ((0,254,84),()), ((0,254,85),()), ((0,254,86),()),
136 |   ((0,254,87),()), ((0,255,1),()), ((0,255,12),()), ((0,255,14),()),
137 |   ((0,255,26),()), ((0,255,27),()), ((0,255,31),()), ((0,255,97),()),
138 |   ((0,255,100),()), ((1,3,159),()), ((1,3,208),()), ((1,8,87),()),
139 |   ((1,9,31),()), ((1,10,86),()), ((1,10,87),()), ((1,10,240),()),
140 |   ((1,10,241),()), ((1,10,242),()), ((1,10,243),()), ((1,10,244),()),
141 |   ((1,10,245),()), ((1,11,58),()), ((1,11,59),()), ((1,11,60),()),
142 |   ((1,11,61),()), ((1,11,62),()), ((1,11,63),()), ((1,11,153),()),
143 |   ((1,11,154),()), ((1,11,155),()), ((1,11,156),()), ((1,16,71),()),
144 |   ((1,16,72),()), ((1,16,73),()), ((1,16,74),()), ((1,16,75),()),
145 |   ((1,16,76),()), ((1,16,77),()), ((1,16,190),()), ((1,16,191),()),
146 |   ((1,16,192),()), ((1,16,193),()), ((1,17,65),()), ((1,17,66),()),
147 |   ((1,17,67),()), ((1,17,197),()), ((1,17,198),()), ((1,17,205),()),
148 |   ((1,17,222),()), ((1,17,223),()), ((1,18,56),()), ((1,18,57),()),
149 |   ((1,18,58),()), ((1,18,59),()), ((1,18,60),()), ((1,18,169),()),
150 |   ((1,20,75),()), ((1,20,76),()), ((1,20,77),()), ((1,20,91),()),
151 |   ((1,21,194),()), ((1,21,195),()), ((1,21,196),()), ((1,21,197),()),
152 |   ((1,21,201),()), ((1,21,202),()), ((1,21,203),()), ((1,21,204),()),
153 |   ((1,21,205),()), ((1,21,206),()), ((1,21,207),()), ((1,21,208),()),
154 |   ((1,21,209),()), ((1,21,210),()), ((1,21,211),()), ((1,21,212),()),
155 |   ((1,21,213),()), ((1,21,214),()), ((1,21,215),()), ((1,22,65),()),
156 |   ((1,22,66),()), ((1,23,60),()), ((1,23,61),()), ((1,23,62),()),
157 |   ((1,28,65),()), ((1,28,66),()), ((1,28,67),()), ((1,28,113),()),
158 |   ((1,36,112),()), ((1,36,113),()), ((1,36,114),()), ((1,36,115),()),
159 |   ((1,36,116),()), ((1,106,110),()), ((1,106,111),()), ((1,106,245),()),
160 |   ((1,107,55),()), ((1,107,56),()), ((1,107,57),()), ((1,107,68),()),
161 |   ((1,188,159),()), ((1,218,135),()), ((1,218,136),()), ((1,218,137),()),
162 |   ((1,218,138),())
163 | ];
164 | 
165 | pub static UCD_EXTENDER: &'static [((u8,u8,u8), ())] = &[
166 |   ((0,0,183),()), ((0,2,208),()), ((0,2,209),()), ((0,6,64),()), ((0,7,250),()),
167 |   ((0,14,70),()), ((0,14,198),()), ((0,24,10),()), ((0,24,67),()),
168 |   ((0,26,167),()), ((0,28,54),()), ((0,28,123),()), ((0,48,5),()),
169 |   ((0,48,49),()), ((0,48,50),()), ((0,48,51),()), ((0,48,52),()),
170 |   ((0,48,53),()), ((0,48,157),()), ((0,48,158),()), ((0,48,252),()),
171 |   ((0,48,253),()), ((0,48,254),()), ((0,160,21),()), ((0,166,12),()),
172 |   ((0,169,207),()), ((0,169,230),()), ((0,170,112),()), ((0,170,221),()),
173 |   ((0,170,243),()), ((0,170,244),()), ((0,255,112),()), ((1,19,93),()),
174 |   ((1,21,198),()), ((1,21,199),()), ((1,21,200),()), ((1,107,66),()),
175 |   ((1,107,67),()), ((1,111,224),()), ((1,233,68),()), ((1,233,69),()),
176 |   ((1,233,70),())
177 | ];
178 | 
179 | pub static UCD_SOFT_DOTTED: &'static [((u8,u8,u8), ())] = &[
180 |   ((0,0,105),()), ((0,0,106),()), ((0,1,47),()), ((0,2,73),()), ((0,2,104),()),
181 |   ((0,2,157),()), ((0,2,178),()), ((0,3,243),()), ((0,4,86),()), ((0,4,88),()),
182 |   ((0,29,98),()), ((0,29,150),()), ((0,29,164),()), ((0,29,168),()),
183 |   ((0,30,45),()), ((0,30,203),()), ((0,32,113),()), ((0,33,72),()),
184 |   ((0,33,73),()), ((0,44,124),()), ((1,212,34),()), ((1,212,35),()),
185 |   ((1,212,86),()), ((1,212,87),()), ((1,212,138),()), ((1,212,139),()),
186 |   ((1,212,190),()), ((1,212,191),()), ((1,212,242),()), ((1,212,243),()),
187 |   ((1,213,38),()), ((1,213,39),()), ((1,213,90),()), ((1,213,91),()),
188 |   ((1,213,142),()), ((1,213,143),()), ((1,213,194),()), ((1,213,195),()),
189 |   ((1,213,246),()), ((1,213,247),()), ((1,214,42),()), ((1,214,43),()),
190 |   ((1,214,94),()), ((1,214,95),()), ((1,214,146),()), ((1,214,147),())
191 | ];
192 | 
193 | pub static UCD_DEFAULT_IGNORABLE: &'static [((u8,u8,u8), (u8,u8,u8), ())] = &[
194 |   ((0,0,173),(0,0,173),()), ((0,3,79),(0,3,79),()), ((0,6,28),(0,6,28),()),
195 |   ((0,17,95),(0,17,96),()), ((0,23,180),(0,23,181),()),
196 |   ((0,24,11),(0,24,14),()), ((0,32,11),(0,32,15),()), ((0,32,42),(0,32,46),()),
197 |   ((0,32,96),(0,32,111),()), ((0,49,100),(0,49,100),()),
198 |   ((0,254,0),(0,254,15),()), ((0,254,255),(0,254,255),()),
199 |   ((0,255,160),(0,255,160),()), ((0,255,240),(0,255,248),()),
200 |   ((1,188,160),(1,188,163),()), ((1,209,115),(1,209,122),()),
201 |   ((14,0,0),(14,15,255),())
202 | ];
203 | 
204 | pub static UCD_ALPHA: &'static [((u8,u8,u8), (u8,u8,u8), ())] = &[
205 |   ((0,0,65),(0,0,90),()), ((0,0,97),(0,0,122),()), ((0,0,170),(0,0,170),()),
206 |   ((0,0,181),(0,0,181),()), ((0,0,186),(0,0,186),()), ((0,0,192),(0,0,214),()),
207 |   ((0,0,216),(0,0,246),()), ((0,0,248),(0,2,193),()), ((0,2,198),(0,2,209),()),
208 |   ((0,2,224),(0,2,228),()), ((0,2,236),(0,2,236),()), ((0,2,238),(0,2,238),()),
209 |   ((0,3,69),(0,3,69),()), ((0,3,112),(0,3,116),()), ((0,3,118),(0,3,119),()),
210 |   ((0,3,122),(0,3,125),()), ((0,3,127),(0,3,127),()), ((0,3,134),(0,3,134),()),
211 |   ((0,3,136),(0,3,138),()), ((0,3,140),(0,3,140),()), ((0,3,142),(0,3,161),()),
212 |   ((0,3,163),(0,3,245),()), ((0,3,247),(0,4,129),()), ((0,4,138),(0,5,47),()),
213 |   ((0,5,49),(0,5,86),()), ((0,5,89),(0,5,89),()), ((0,5,97),(0,5,135),()),
214 |   ((0,5,176),(0,5,189),()), ((0,5,191),(0,5,191),()), ((0,5,193),(0,5,194),()),
215 |   ((0,5,196),(0,5,197),()), ((0,5,199),(0,5,199),()), ((0,5,208),(0,5,234),()),
216 |   ((0,5,240),(0,5,242),()), ((0,6,16),(0,6,26),()), ((0,6,32),(0,6,87),()),
217 |   ((0,6,89),(0,6,95),()), ((0,6,110),(0,6,211),()), ((0,6,213),(0,6,220),()),
218 |   ((0,6,225),(0,6,232),()), ((0,6,237),(0,6,239),()), ((0,6,250),(0,6,252),()),
219 |   ((0,6,255),(0,6,255),()), ((0,7,16),(0,7,63),()), ((0,7,77),(0,7,177),()),
220 |   ((0,7,202),(0,7,234),()), ((0,7,244),(0,7,245),()), ((0,7,250),(0,7,250),()),
221 |   ((0,8,0),(0,8,23),()), ((0,8,26),(0,8,44),()), ((0,8,64),(0,8,88),()),
222 |   ((0,8,160),(0,8,180),()), ((0,8,182),(0,8,189),()), ((0,8,212),(0,8,223),()),
223 |   ((0,8,227),(0,8,233),()), ((0,8,240),(0,9,59),()), ((0,9,61),(0,9,76),()),
224 |   ((0,9,78),(0,9,80),()), ((0,9,85),(0,9,99),()), ((0,9,113),(0,9,131),()),
225 |   ((0,9,133),(0,9,140),()), ((0,9,143),(0,9,144),()), ((0,9,147),(0,9,168),()),
226 |   ((0,9,170),(0,9,176),()), ((0,9,178),(0,9,178),()), ((0,9,182),(0,9,185),()),
227 |   ((0,9,189),(0,9,196),()), ((0,9,199),(0,9,200),()), ((0,9,203),(0,9,204),()),
228 |   ((0,9,206),(0,9,206),()), ((0,9,215),(0,9,215),()), ((0,9,220),(0,9,221),()),
229 |   ((0,9,223),(0,9,227),()), ((0,9,240),(0,9,241),()), ((0,10,1),(0,10,3),()),
230 |   ((0,10,5),(0,10,10),()), ((0,10,15),(0,10,16),()), ((0,10,19),(0,10,40),()),
231 |   ((0,10,42),(0,10,48),()), ((0,10,50),(0,10,51),()), ((0,10,53),(0,10,54),()),
232 |   ((0,10,56),(0,10,57),()), ((0,10,62),(0,10,66),()), ((0,10,71),(0,10,72),()),
233 |   ((0,10,75),(0,10,76),()), ((0,10,81),(0,10,81),()), ((0,10,89),(0,10,92),()),
234 |   ((0,10,94),(0,10,94),()), ((0,10,112),(0,10,117),()),
235 |   ((0,10,129),(0,10,131),()), ((0,10,133),(0,10,141),()),
236 |   ((0,10,143),(0,10,145),()), ((0,10,147),(0,10,168),()),
237 |   ((0,10,170),(0,10,176),()), ((0,10,178),(0,10,179),()),
238 |   ((0,10,181),(0,10,185),()), ((0,10,189),(0,10,197),()),
239 |   ((0,10,199),(0,10,201),()), ((0,10,203),(0,10,204),()),
240 |   ((0,10,208),(0,10,208),()), ((0,10,224),(0,10,227),()),
241 |   ((0,10,249),(0,10,249),()), ((0,11,1),(0,11,3),()), ((0,11,5),(0,11,12),()),
242 |   ((0,11,15),(0,11,16),()), ((0,11,19),(0,11,40),()), ((0,11,42),(0,11,48),()),
243 |   ((0,11,50),(0,11,51),()), ((0,11,53),(0,11,57),()), ((0,11,61),(0,11,68),()),
244 |   ((0,11,71),(0,11,72),()), ((0,11,75),(0,11,76),()), ((0,11,86),(0,11,87),()),
245 |   ((0,11,92),(0,11,93),()), ((0,11,95),(0,11,99),()),
246 |   ((0,11,113),(0,11,113),()), ((0,11,130),(0,11,131),()),
247 |   ((0,11,133),(0,11,138),()), ((0,11,142),(0,11,144),()),
248 |   ((0,11,146),(0,11,149),()), ((0,11,153),(0,11,154),()),
249 |   ((0,11,156),(0,11,156),()), ((0,11,158),(0,11,159),()),
250 |   ((0,11,163),(0,11,164),()), ((0,11,168),(0,11,170),()),
251 |   ((0,11,174),(0,11,185),()), ((0,11,190),(0,11,194),()),
252 |   ((0,11,198),(0,11,200),()), ((0,11,202),(0,11,204),()),
253 |   ((0,11,208),(0,11,208),()), ((0,11,215),(0,11,215),()),
254 |   ((0,12,0),(0,12,3),()), ((0,12,5),(0,12,12),()), ((0,12,14),(0,12,16),()),
255 |   ((0,12,18),(0,12,40),()), ((0,12,42),(0,12,57),()), ((0,12,61),(0,12,68),()),
256 |   ((0,12,70),(0,12,72),()), ((0,12,74),(0,12,76),()), ((0,12,85),(0,12,86),()),
257 |   ((0,12,88),(0,12,90),()), ((0,12,96),(0,12,99),()),
258 |   ((0,12,128),(0,12,131),()), ((0,12,133),(0,12,140),()),
259 |   ((0,12,142),(0,12,144),()), ((0,12,146),(0,12,168),()),
260 |   ((0,12,170),(0,12,179),()), ((0,12,181),(0,12,185),()),
261 |   ((0,12,189),(0,12,196),()), ((0,12,198),(0,12,200),()),
262 |   ((0,12,202),(0,12,204),()), ((0,12,213),(0,12,214),()),
263 |   ((0,12,222),(0,12,222),()), ((0,12,224),(0,12,227),()),
264 |   ((0,12,241),(0,12,242),()), ((0,13,1),(0,13,3),()), ((0,13,5),(0,13,12),()),
265 |   ((0,13,14),(0,13,16),()), ((0,13,18),(0,13,58),()), ((0,13,61),(0,13,68),()),
266 |   ((0,13,70),(0,13,72),()), ((0,13,74),(0,13,76),()), ((0,13,78),(0,13,78),()),
267 |   ((0,13,84),(0,13,87),()), ((0,13,95),(0,13,99),()),
268 |   ((0,13,122),(0,13,127),()), ((0,13,130),(0,13,131),()),
269 |   ((0,13,133),(0,13,150),()), ((0,13,154),(0,13,177),()),
270 |   ((0,13,179),(0,13,187),()), ((0,13,189),(0,13,189),()),
271 |   ((0,13,192),(0,13,198),()), ((0,13,207),(0,13,212),()),
272 |   ((0,13,214),(0,13,214),()), ((0,13,216),(0,13,223),()),
273 |   ((0,13,242),(0,13,243),()), ((0,14,1),(0,14,58),()), ((0,14,64),(0,14,70),()),
274 |   ((0,14,77),(0,14,77),()), ((0,14,129),(0,14,130),()),
275 |   ((0,14,132),(0,14,132),()), ((0,14,135),(0,14,136),()),
276 |   ((0,14,138),(0,14,138),()), ((0,14,141),(0,14,141),()),
277 |   ((0,14,148),(0,14,151),()), ((0,14,153),(0,14,159),()),
278 |   ((0,14,161),(0,14,163),()), ((0,14,165),(0,14,165),()),
279 |   ((0,14,167),(0,14,167),()), ((0,14,170),(0,14,171),()),
280 |   ((0,14,173),(0,14,185),()), ((0,14,187),(0,14,189),()),
281 |   ((0,14,192),(0,14,196),()), ((0,14,198),(0,14,198),()),
282 |   ((0,14,205),(0,14,205),()), ((0,14,220),(0,14,223),()),
283 |   ((0,15,0),(0,15,0),()), ((0,15,64),(0,15,71),()), ((0,15,73),(0,15,108),()),
284 |   ((0,15,113),(0,15,129),()), ((0,15,136),(0,15,151),()),
285 |   ((0,15,153),(0,15,188),()), ((0,16,0),(0,16,54),()), ((0,16,56),(0,16,56),()),
286 |   ((0,16,59),(0,16,63),()), ((0,16,80),(0,16,98),()),
287 |   ((0,16,101),(0,16,104),()), ((0,16,110),(0,16,134),()),
288 |   ((0,16,142),(0,16,142),()), ((0,16,156),(0,16,157),()),
289 |   ((0,16,160),(0,16,197),()), ((0,16,199),(0,16,199),()),
290 |   ((0,16,205),(0,16,205),()), ((0,16,208),(0,16,250),()),
291 |   ((0,16,252),(0,18,72),()), ((0,18,74),(0,18,77),()), ((0,18,80),(0,18,86),()),
292 |   ((0,18,88),(0,18,88),()), ((0,18,90),(0,18,93),()), ((0,18,96),(0,18,136),()),
293 |   ((0,18,138),(0,18,141),()), ((0,18,144),(0,18,176),()),
294 |   ((0,18,178),(0,18,181),()), ((0,18,184),(0,18,190),()),
295 |   ((0,18,192),(0,18,192),()), ((0,18,194),(0,18,197),()),
296 |   ((0,18,200),(0,18,214),()), ((0,18,216),(0,19,16),()),
297 |   ((0,19,18),(0,19,21),()), ((0,19,24),(0,19,90),()), ((0,19,95),(0,19,95),()),
298 |   ((0,19,128),(0,19,143),()), ((0,19,160),(0,19,245),()),
299 |   ((0,19,248),(0,19,253),()), ((0,20,1),(0,22,108),()),
300 |   ((0,22,111),(0,22,127),()), ((0,22,129),(0,22,154),()),
301 |   ((0,22,160),(0,22,234),()), ((0,22,238),(0,22,248),()),
302 |   ((0,23,0),(0,23,12),()), ((0,23,14),(0,23,19),()), ((0,23,32),(0,23,51),()),
303 |   ((0,23,64),(0,23,83),()), ((0,23,96),(0,23,108),()),
304 |   ((0,23,110),(0,23,112),()), ((0,23,114),(0,23,115),()),
305 |   ((0,23,128),(0,23,179),()), ((0,23,182),(0,23,200),()),
306 |   ((0,23,215),(0,23,215),()), ((0,23,220),(0,23,220),()),
307 |   ((0,24,32),(0,24,119),()), ((0,24,128),(0,24,170),()),
308 |   ((0,24,176),(0,24,245),()), ((0,25,0),(0,25,30),()), ((0,25,32),(0,25,43),()),
309 |   ((0,25,48),(0,25,56),()), ((0,25,80),(0,25,109),()),
310 |   ((0,25,112),(0,25,116),()), ((0,25,128),(0,25,171),()),
311 |   ((0,25,176),(0,25,201),()), ((0,26,0),(0,26,27),()), ((0,26,32),(0,26,94),()),
312 |   ((0,26,97),(0,26,116),()), ((0,26,167),(0,26,167),()),
313 |   ((0,27,0),(0,27,51),()), ((0,27,53),(0,27,67),()), ((0,27,69),(0,27,75),()),
314 |   ((0,27,128),(0,27,169),()), ((0,27,172),(0,27,175),()),
315 |   ((0,27,186),(0,27,229),()), ((0,27,231),(0,27,241),()),
316 |   ((0,28,0),(0,28,53),()), ((0,28,77),(0,28,79),()), ((0,28,90),(0,28,125),()),
317 |   ((0,28,128),(0,28,136),()), ((0,28,233),(0,28,236),()),
318 |   ((0,28,238),(0,28,243),()), ((0,28,245),(0,28,246),()),
319 |   ((0,29,0),(0,29,191),()), ((0,29,231),(0,29,244),()), ((0,30,0),(0,31,21),()),
320 |   ((0,31,24),(0,31,29),()), ((0,31,32),(0,31,69),()), ((0,31,72),(0,31,77),()),
321 |   ((0,31,80),(0,31,87),()), ((0,31,89),(0,31,89),()), ((0,31,91),(0,31,91),()),
322 |   ((0,31,93),(0,31,93),()), ((0,31,95),(0,31,125),()),
323 |   ((0,31,128),(0,31,180),()), ((0,31,182),(0,31,188),()),
324 |   ((0,31,190),(0,31,190),()), ((0,31,194),(0,31,196),()),
325 |   ((0,31,198),(0,31,204),()), ((0,31,208),(0,31,211),()),
326 |   ((0,31,214),(0,31,219),()), ((0,31,224),(0,31,236),()),
327 |   ((0,31,242),(0,31,244),()), ((0,31,246),(0,31,252),()),
328 |   ((0,32,113),(0,32,113),()), ((0,32,127),(0,32,127),()),
329 |   ((0,32,144),(0,32,156),()), ((0,33,2),(0,33,2),()), ((0,33,7),(0,33,7),()),
330 |   ((0,33,10),(0,33,19),()), ((0,33,21),(0,33,21),()), ((0,33,25),(0,33,29),()),
331 |   ((0,33,36),(0,33,36),()), ((0,33,38),(0,33,38),()), ((0,33,40),(0,33,40),()),
332 |   ((0,33,42),(0,33,45),()), ((0,33,47),(0,33,57),()), ((0,33,60),(0,33,63),()),
333 |   ((0,33,69),(0,33,73),()), ((0,33,78),(0,33,78),()), ((0,33,96),(0,33,136),()),
334 |   ((0,36,182),(0,36,233),()), ((0,44,0),(0,44,46),()), ((0,44,48),(0,44,94),()),
335 |   ((0,44,96),(0,44,228),()), ((0,44,235),(0,44,238),()),
336 |   ((0,44,242),(0,44,243),()), ((0,45,0),(0,45,37),()), ((0,45,39),(0,45,39),()),
337 |   ((0,45,45),(0,45,45),()), ((0,45,48),(0,45,103),()),
338 |   ((0,45,111),(0,45,111),()), ((0,45,128),(0,45,150),()),
339 |   ((0,45,160),(0,45,166),()), ((0,45,168),(0,45,174),()),
340 |   ((0,45,176),(0,45,182),()), ((0,45,184),(0,45,190),()),
341 |   ((0,45,192),(0,45,198),()), ((0,45,200),(0,45,206),()),
342 |   ((0,45,208),(0,45,214),()), ((0,45,216),(0,45,222),()),
343 |   ((0,45,224),(0,45,255),()), ((0,46,47),(0,46,47),()), ((0,48,5),(0,48,7),()),
344 |   ((0,48,33),(0,48,41),()), ((0,48,49),(0,48,53),()), ((0,48,56),(0,48,60),()),
345 |   ((0,48,65),(0,48,150),()), ((0,48,157),(0,48,159),()),
346 |   ((0,48,161),(0,48,250),()), ((0,48,252),(0,48,255),()),
347 |   ((0,49,5),(0,49,45),()), ((0,49,49),(0,49,142),()),
348 |   ((0,49,160),(0,49,186),()), ((0,49,240),(0,49,255),()),
349 |   ((0,52,0),(0,77,181),()), ((0,78,0),(0,159,213),()),
350 |   ((0,160,0),(0,164,140),()), ((0,164,208),(0,164,253),()),
351 |   ((0,165,0),(0,166,12),()), ((0,166,16),(0,166,31),()),
352 |   ((0,166,42),(0,166,43),()), ((0,166,64),(0,166,110),()),
353 |   ((0,166,116),(0,166,123),()), ((0,166,127),(0,166,239),()),
354 |   ((0,167,23),(0,167,31),()), ((0,167,34),(0,167,136),()),
355 |   ((0,167,139),(0,167,174),()), ((0,167,176),(0,167,183),()),
356 |   ((0,167,247),(0,168,1),()), ((0,168,3),(0,168,5),()),
357 |   ((0,168,7),(0,168,10),()), ((0,168,12),(0,168,39),()),
358 |   ((0,168,64),(0,168,115),()), ((0,168,128),(0,168,195),()),
359 |   ((0,168,197),(0,168,197),()), ((0,168,242),(0,168,247),()),
360 |   ((0,168,251),(0,168,251),()), ((0,168,253),(0,168,253),()),
361 |   ((0,169,10),(0,169,42),()), ((0,169,48),(0,169,82),()),
362 |   ((0,169,96),(0,169,124),()), ((0,169,128),(0,169,178),()),
363 |   ((0,169,180),(0,169,191),()), ((0,169,207),(0,169,207),()),
364 |   ((0,169,224),(0,169,228),()), ((0,169,230),(0,169,239),()),
365 |   ((0,169,250),(0,169,254),()), ((0,170,0),(0,170,54),()),
366 |   ((0,170,64),(0,170,77),()), ((0,170,96),(0,170,118),()),
367 |   ((0,170,122),(0,170,122),()), ((0,170,126),(0,170,190),()),
368 |   ((0,170,192),(0,170,192),()), ((0,170,194),(0,170,194),()),
369 |   ((0,170,219),(0,170,221),()), ((0,170,224),(0,170,239),()),
370 |   ((0,170,242),(0,170,245),()), ((0,171,1),(0,171,6),()),
371 |   ((0,171,9),(0,171,14),()), ((0,171,17),(0,171,22),()),
372 |   ((0,171,32),(0,171,38),()), ((0,171,40),(0,171,46),()),
373 |   ((0,171,48),(0,171,90),()), ((0,171,92),(0,171,101),()),
374 |   ((0,171,112),(0,171,234),()), ((0,172,0),(0,215,163),()),
375 |   ((0,215,176),(0,215,198),()), ((0,215,203),(0,215,251),()),
376 |   ((0,249,0),(0,250,109),()), ((0,250,112),(0,250,217),()),
377 |   ((0,251,0),(0,251,6),()), ((0,251,19),(0,251,23),()),
378 |   ((0,251,29),(0,251,40),()), ((0,251,42),(0,251,54),()),
379 |   ((0,251,56),(0,251,60),()), ((0,251,62),(0,251,62),()),
380 |   ((0,251,64),(0,251,65),()), ((0,251,67),(0,251,68),()),
381 |   ((0,251,70),(0,251,177),()), ((0,251,211),(0,253,61),()),
382 |   ((0,253,80),(0,253,143),()), ((0,253,146),(0,253,199),()),
383 |   ((0,253,240),(0,253,251),()), ((0,254,112),(0,254,116),()),
384 |   ((0,254,118),(0,254,252),()), ((0,255,33),(0,255,58),()),
385 |   ((0,255,65),(0,255,90),()), ((0,255,102),(0,255,190),()),
386 |   ((0,255,194),(0,255,199),()), ((0,255,202),(0,255,207),()),
387 |   ((0,255,210),(0,255,215),()), ((0,255,218),(0,255,220),()),
388 |   ((1,0,0),(1,0,11),()), ((1,0,13),(1,0,38),()), ((1,0,40),(1,0,58),()),
389 |   ((1,0,60),(1,0,61),()), ((1,0,63),(1,0,77),()), ((1,0,80),(1,0,93),()),
390 |   ((1,0,128),(1,0,250),()), ((1,1,64),(1,1,116),()), ((1,2,128),(1,2,156),()),
391 |   ((1,2,160),(1,2,208),()), ((1,3,0),(1,3,31),()), ((1,3,48),(1,3,74),()),
392 |   ((1,3,80),(1,3,122),()), ((1,3,128),(1,3,157),()), ((1,3,160),(1,3,195),()),
393 |   ((1,3,200),(1,3,207),()), ((1,3,209),(1,3,213),()), ((1,4,0),(1,4,157),()),
394 |   ((1,4,176),(1,4,211),()), ((1,4,216),(1,4,251),()), ((1,5,0),(1,5,39),()),
395 |   ((1,5,48),(1,5,99),()), ((1,6,0),(1,7,54),()), ((1,7,64),(1,7,85),()),
396 |   ((1,7,96),(1,7,103),()), ((1,8,0),(1,8,5),()), ((1,8,8),(1,8,8),()),
397 |   ((1,8,10),(1,8,53),()), ((1,8,55),(1,8,56),()), ((1,8,60),(1,8,60),()),
398 |   ((1,8,63),(1,8,85),()), ((1,8,96),(1,8,118),()), ((1,8,128),(1,8,158),()),
399 |   ((1,8,224),(1,8,242),()), ((1,8,244),(1,8,245),()), ((1,9,0),(1,9,21),()),
400 |   ((1,9,32),(1,9,57),()), ((1,9,128),(1,9,183),()), ((1,9,190),(1,9,191),()),
401 |   ((1,10,0),(1,10,3),()), ((1,10,5),(1,10,6),()), ((1,10,12),(1,10,19),()),
402 |   ((1,10,21),(1,10,23),()), ((1,10,25),(1,10,51),()), ((1,10,96),(1,10,124),()),
403 |   ((1,10,128),(1,10,156),()), ((1,10,192),(1,10,199),()),
404 |   ((1,10,201),(1,10,228),()), ((1,11,0),(1,11,53),()), ((1,11,64),(1,11,85),()),
405 |   ((1,11,96),(1,11,114),()), ((1,11,128),(1,11,145),()),
406 |   ((1,12,0),(1,12,72),()), ((1,12,128),(1,12,178),()),
407 |   ((1,12,192),(1,12,242),()), ((1,16,0),(1,16,69),()),
408 |   ((1,16,130),(1,16,184),()), ((1,16,208),(1,16,232),()),
409 |   ((1,17,0),(1,17,50),()), ((1,17,80),(1,17,114),()),
410 |   ((1,17,118),(1,17,118),()), ((1,17,128),(1,17,191),()),
411 |   ((1,17,193),(1,17,196),()), ((1,17,218),(1,17,218),()),
412 |   ((1,17,220),(1,17,220),()), ((1,18,0),(1,18,17),()), ((1,18,19),(1,18,52),()),
413 |   ((1,18,55),(1,18,55),()), ((1,18,62),(1,18,62),()),
414 |   ((1,18,128),(1,18,134),()), ((1,18,136),(1,18,136),()),
415 |   ((1,18,138),(1,18,141),()), ((1,18,143),(1,18,157),()),
416 |   ((1,18,159),(1,18,168),()), ((1,18,176),(1,18,232),()),
417 |   ((1,19,0),(1,19,3),()), ((1,19,5),(1,19,12),()), ((1,19,15),(1,19,16),()),
418 |   ((1,19,19),(1,19,40),()), ((1,19,42),(1,19,48),()), ((1,19,50),(1,19,51),()),
419 |   ((1,19,53),(1,19,57),()), ((1,19,61),(1,19,68),()), ((1,19,71),(1,19,72),()),
420 |   ((1,19,75),(1,19,76),()), ((1,19,80),(1,19,80),()), ((1,19,87),(1,19,87),()),
421 |   ((1,19,93),(1,19,99),()), ((1,20,0),(1,20,65),()), ((1,20,67),(1,20,69),()),
422 |   ((1,20,71),(1,20,74),()), ((1,20,128),(1,20,193),()),
423 |   ((1,20,196),(1,20,197),()), ((1,20,199),(1,20,199),()),
424 |   ((1,21,128),(1,21,181),()), ((1,21,184),(1,21,190),()),
425 |   ((1,21,216),(1,21,221),()), ((1,22,0),(1,22,62),()), ((1,22,64),(1,22,64),()),
426 |   ((1,22,68),(1,22,68),()), ((1,22,128),(1,22,181),()), ((1,23,0),(1,23,25),()),
427 |   ((1,23,29),(1,23,42),()), ((1,24,160),(1,24,223),()),
428 |   ((1,24,255),(1,24,255),()), ((1,26,192),(1,26,248),()),
429 |   ((1,28,0),(1,28,8),()), ((1,28,10),(1,28,54),()), ((1,28,56),(1,28,62),()),
430 |   ((1,28,64),(1,28,64),()), ((1,28,114),(1,28,143),()),
431 |   ((1,28,146),(1,28,167),()), ((1,28,169),(1,28,182),()),
432 |   ((1,32,0),(1,35,153),()), ((1,36,0),(1,36,110),()), ((1,36,128),(1,37,67),()),
433 |   ((1,48,0),(1,52,46),()), ((1,68,0),(1,70,70),()), ((1,104,0),(1,106,56),()),
434 |   ((1,106,64),(1,106,94),()), ((1,106,208),(1,106,237),()),
435 |   ((1,107,0),(1,107,54),()), ((1,107,64),(1,107,67),()),
436 |   ((1,107,99),(1,107,119),()), ((1,107,125),(1,107,143),()),
437 |   ((1,111,0),(1,111,68),()), ((1,111,80),(1,111,126),()),
438 |   ((1,111,147),(1,111,159),()), ((1,111,224),(1,111,224),()),
439 |   ((1,112,0),(1,135,236),()), ((1,136,0),(1,138,242),()),
440 |   ((1,176,0),(1,176,1),()), ((1,188,0),(1,188,106),()),
441 |   ((1,188,112),(1,188,124),()), ((1,188,128),(1,188,136),()),
442 |   ((1,188,144),(1,188,153),()), ((1,188,158),(1,188,158),()),
443 |   ((1,212,0),(1,212,84),()), ((1,212,86),(1,212,156),()),
444 |   ((1,212,158),(1,212,159),()), ((1,212,162),(1,212,162),()),
445 |   ((1,212,165),(1,212,166),()), ((1,212,169),(1,212,172),()),
446 |   ((1,212,174),(1,212,185),()), ((1,212,187),(1,212,187),()),
447 |   ((1,212,189),(1,212,195),()), ((1,212,197),(1,213,5),()),
448 |   ((1,213,7),(1,213,10),()), ((1,213,13),(1,213,20),()),
449 |   ((1,213,22),(1,213,28),()), ((1,213,30),(1,213,57),()),
450 |   ((1,213,59),(1,213,62),()), ((1,213,64),(1,213,68),()),
451 |   ((1,213,70),(1,213,70),()), ((1,213,74),(1,213,80),()),
452 |   ((1,213,82),(1,214,165),()), ((1,214,168),(1,214,192),()),
453 |   ((1,214,194),(1,214,218),()), ((1,214,220),(1,214,250),()),
454 |   ((1,214,252),(1,215,20),()), ((1,215,22),(1,215,52),()),
455 |   ((1,215,54),(1,215,78),()), ((1,215,80),(1,215,110),()),
456 |   ((1,215,112),(1,215,136),()), ((1,215,138),(1,215,168),()),
457 |   ((1,215,170),(1,215,194),()), ((1,215,196),(1,215,203),()),
458 |   ((1,224,0),(1,224,6),()), ((1,224,8),(1,224,24),()),
459 |   ((1,224,27),(1,224,33),()), ((1,224,35),(1,224,36),()),
460 |   ((1,224,38),(1,224,42),()), ((1,232,0),(1,232,196),()),
461 |   ((1,233,0),(1,233,67),()), ((1,233,71),(1,233,71),()),
462 |   ((1,238,0),(1,238,3),()), ((1,238,5),(1,238,31),()),
463 |   ((1,238,33),(1,238,34),()), ((1,238,36),(1,238,36),()),
464 |   ((1,238,39),(1,238,39),()), ((1,238,41),(1,238,50),()),
465 |   ((1,238,52),(1,238,55),()), ((1,238,57),(1,238,57),()),
466 |   ((1,238,59),(1,238,59),()), ((1,238,66),(1,238,66),()),
467 |   ((1,238,71),(1,238,71),()), ((1,238,73),(1,238,73),()),
468 |   ((1,238,75),(1,238,75),()), ((1,238,77),(1,238,79),()),
469 |   ((1,238,81),(1,238,82),()), ((1,238,84),(1,238,84),()),
470 |   ((1,238,87),(1,238,87),()), ((1,238,89),(1,238,89),()),
471 |   ((1,238,91),(1,238,91),()), ((1,238,93),(1,238,93),()),
472 |   ((1,238,95),(1,238,95),()), ((1,238,97),(1,238,98),()),
473 |   ((1,238,100),(1,238,100),()), ((1,238,103),(1,238,106),()),
474 |   ((1,238,108),(1,238,114),()), ((1,238,116),(1,238,119),()),
475 |   ((1,238,121),(1,238,124),()), ((1,238,126),(1,238,126),()),
476 |   ((1,238,128),(1,238,137),()), ((1,238,139),(1,238,155),()),
477 |   ((1,238,161),(1,238,163),()), ((1,238,165),(1,238,169),()),
478 |   ((1,238,171),(1,238,187),()), ((1,241,48),(1,241,73),()),
479 |   ((1,241,80),(1,241,105),()), ((1,241,112),(1,241,137),()),
480 |   ((2,0,0),(2,166,214),()), ((2,167,0),(2,183,52),()),
481 |   ((2,183,64),(2,184,29),()), ((2,184,32),(2,206,161),()),
482 |   ((2,248,0),(2,250,29),())
483 | ];
484 | 
485 | pub static UCD_DEFAULT_IGNORABLE_OTHER: &'static [((u8,u8,u8), (u8,u8,u8), ())] = &[
486 |   ((0,3,79),(0,3,79),()), ((0,17,95),(0,17,96),()), ((0,23,180),(0,23,181),()),
487 |   ((0,32,101),(0,32,101),()), ((0,49,100),(0,49,100),()),
488 |   ((0,255,160),(0,255,160),()), ((0,255,240),(0,255,248),()),
489 |   ((14,0,0),(14,0,0),()), ((14,0,2),(14,0,31),()), ((14,0,128),(14,0,255),()),
490 |   ((14,1,240),(14,15,255),())
491 | ];
492 | 
493 | pub static UCD_MATH_OTHER: &'static [((u8,u8,u8), (u8,u8,u8), ())] = &[
494 |   ((0,0,94),(0,0,94),()), ((0,3,208),(0,3,210),()), ((0,3,213),(0,3,213),()),
495 |   ((0,3,240),(0,3,241),()), ((0,3,244),(0,3,245),()), ((0,32,22),(0,32,22),()),
496 |   ((0,32,50),(0,32,52),()), ((0,32,64),(0,32,64),()), ((0,32,97),(0,32,100),()),
497 |   ((0,32,125),(0,32,126),()), ((0,32,141),(0,32,142),()),
498 |   ((0,32,208),(0,32,220),()), ((0,32,225),(0,32,225),()),
499 |   ((0,32,229),(0,32,230),()), ((0,32,235),(0,32,239),()),
500 |   ((0,33,2),(0,33,2),()), ((0,33,7),(0,33,7),()), ((0,33,10),(0,33,19),()),
501 |   ((0,33,21),(0,33,21),()), ((0,33,25),(0,33,29),()), ((0,33,36),(0,33,36),()),
502 |   ((0,33,40),(0,33,41),()), ((0,33,44),(0,33,45),()), ((0,33,47),(0,33,49),()),
503 |   ((0,33,51),(0,33,56),()), ((0,33,60),(0,33,63),()), ((0,33,69),(0,33,73),()),
504 |   ((0,33,149),(0,33,153),()), ((0,33,156),(0,33,159),()),
505 |   ((0,33,161),(0,33,162),()), ((0,33,164),(0,33,165),()),
506 |   ((0,33,167),(0,33,167),()), ((0,33,169),(0,33,173),()),
507 |   ((0,33,176),(0,33,177),()), ((0,33,182),(0,33,183),()),
508 |   ((0,33,188),(0,33,205),()), ((0,33,208),(0,33,209),()),
509 |   ((0,33,211),(0,33,211),()), ((0,33,213),(0,33,219),()),
510 |   ((0,33,221),(0,33,221),()), ((0,33,228),(0,33,229),()),
511 |   ((0,35,8),(0,35,11),()), ((0,35,180),(0,35,181),()),
512 |   ((0,35,183),(0,35,183),()), ((0,35,208),(0,35,208),()),
513 |   ((0,35,226),(0,35,226),()), ((0,37,160),(0,37,161),()),
514 |   ((0,37,174),(0,37,182),()), ((0,37,188),(0,37,192),()),
515 |   ((0,37,198),(0,37,199),()), ((0,37,202),(0,37,203),()),
516 |   ((0,37,207),(0,37,211),()), ((0,37,226),(0,37,226),()),
517 |   ((0,37,228),(0,37,228),()), ((0,37,231),(0,37,236),()),
518 |   ((0,38,5),(0,38,6),()), ((0,38,64),(0,38,64),()), ((0,38,66),(0,38,66),()),
519 |   ((0,38,96),(0,38,99),()), ((0,38,109),(0,38,110),()),
520 |   ((0,39,197),(0,39,198),()), ((0,39,230),(0,39,239),()),
521 |   ((0,41,131),(0,41,152),()), ((0,41,216),(0,41,219),()),
522 |   ((0,41,252),(0,41,253),()), ((0,254,97),(0,254,97),()),
523 |   ((0,254,99),(0,254,99),()), ((0,254,104),(0,254,104),()),
524 |   ((0,255,60),(0,255,60),()), ((0,255,62),(0,255,62),()),
525 |   ((1,212,0),(1,212,84),()), ((1,212,86),(1,212,156),()),
526 |   ((1,212,158),(1,212,159),()), ((1,212,162),(1,212,162),()),
527 |   ((1,212,165),(1,212,166),()), ((1,212,169),(1,212,172),()),
528 |   ((1,212,174),(1,212,185),()), ((1,212,187),(1,212,187),()),
529 |   ((1,212,189),(1,212,195),()), ((1,212,197),(1,213,5),()),
530 |   ((1,213,7),(1,213,10),()), ((1,213,13),(1,213,20),()),
531 |   ((1,213,22),(1,213,28),()), ((1,213,30),(1,213,57),()),
532 |   ((1,213,59),(1,213,62),()), ((1,213,64),(1,213,68),()),
533 |   ((1,213,70),(1,213,70),()), ((1,213,74),(1,213,80),()),
534 |   ((1,213,82),(1,214,165),()), ((1,214,168),(1,214,192),()),
535 |   ((1,214,194),(1,214,218),()), ((1,214,220),(1,214,250),()),
536 |   ((1,214,252),(1,215,20),()), ((1,215,22),(1,215,52),()),
537 |   ((1,215,54),(1,215,78),()), ((1,215,80),(1,215,110),()),
538 |   ((1,215,112),(1,215,136),()), ((1,215,138),(1,215,168),()),
539 |   ((1,215,170),(1,215,194),()), ((1,215,196),(1,215,203),()),
540 |   ((1,215,206),(1,215,255),()), ((1,238,0),(1,238,3),()),
541 |   ((1,238,5),(1,238,31),()), ((1,238,33),(1,238,34),()),
542 |   ((1,238,36),(1,238,36),()), ((1,238,39),(1,238,39),()),
543 |   ((1,238,41),(1,238,50),()), ((1,238,52),(1,238,55),()),
544 |   ((1,238,57),(1,238,57),()), ((1,238,59),(1,238,59),()),
545 |   ((1,238,66),(1,238,66),()), ((1,238,71),(1,238,71),()),
546 |   ((1,238,73),(1,238,73),()), ((1,238,75),(1,238,75),()),
547 |   ((1,238,77),(1,238,79),()), ((1,238,81),(1,238,82),()),
548 |   ((1,238,84),(1,238,84),()), ((1,238,87),(1,238,87),()),
549 |   ((1,238,89),(1,238,89),()), ((1,238,91),(1,238,91),()),
550 |   ((1,238,93),(1,238,93),()), ((1,238,95),(1,238,95),()),
551 |   ((1,238,97),(1,238,98),()), ((1,238,100),(1,238,100),()),
552 |   ((1,238,103),(1,238,106),()), ((1,238,108),(1,238,114),()),
553 |   ((1,238,116),(1,238,119),()), ((1,238,121),(1,238,124),()),
554 |   ((1,238,126),(1,238,126),()), ((1,238,128),(1,238,137),()),
555 |   ((1,238,139),(1,238,155),()), ((1,238,161),(1,238,163),()),
556 |   ((1,238,165),(1,238,169),()), ((1,238,171),(1,238,187),())
557 | ];
558 | 
559 | pub static UCD_DIACRITIC: &'static [((u8,u8,u8), (u8,u8,u8), ())] = &[
560 |   ((0,0,94),(0,0,94),()), ((0,0,96),(0,0,96),()), ((0,0,168),(0,0,168),()),
561 |   ((0,0,175),(0,0,175),()), ((0,0,180),(0,0,180),()), ((0,0,183),(0,0,184),()),
562 |   ((0,2,176),(0,3,78),()), ((0,3,80),(0,3,87),()), ((0,3,93),(0,3,98),()),
563 |   ((0,3,116),(0,3,117),()), ((0,3,122),(0,3,122),()), ((0,3,132),(0,3,133),()),
564 |   ((0,4,131),(0,4,135),()), ((0,5,89),(0,5,89),()), ((0,5,145),(0,5,161),()),
565 |   ((0,5,163),(0,5,189),()), ((0,5,191),(0,5,191),()), ((0,5,193),(0,5,194),()),
566 |   ((0,5,196),(0,5,196),()), ((0,6,75),(0,6,82),()), ((0,6,87),(0,6,88),()),
567 |   ((0,6,223),(0,6,224),()), ((0,6,229),(0,6,230),()), ((0,6,234),(0,6,236),()),
568 |   ((0,7,48),(0,7,74),()), ((0,7,166),(0,7,176),()), ((0,7,235),(0,7,245),()),
569 |   ((0,8,24),(0,8,25),()), ((0,8,227),(0,8,254),()), ((0,9,60),(0,9,60),()),
570 |   ((0,9,77),(0,9,77),()), ((0,9,81),(0,9,84),()), ((0,9,113),(0,9,113),()),
571 |   ((0,9,188),(0,9,188),()), ((0,9,205),(0,9,205),()), ((0,10,60),(0,10,60),()),
572 |   ((0,10,77),(0,10,77),()), ((0,10,188),(0,10,188),()),
573 |   ((0,10,205),(0,10,205),()), ((0,11,60),(0,11,60),()),
574 |   ((0,11,77),(0,11,77),()), ((0,11,205),(0,11,205),()),
575 |   ((0,12,77),(0,12,77),()), ((0,12,188),(0,12,188),()),
576 |   ((0,12,205),(0,12,205),()), ((0,13,77),(0,13,77),()),
577 |   ((0,13,202),(0,13,202),()), ((0,14,71),(0,14,76),()),
578 |   ((0,14,78),(0,14,78),()), ((0,14,200),(0,14,204),()),
579 |   ((0,15,24),(0,15,25),()), ((0,15,53),(0,15,53),()), ((0,15,55),(0,15,55),()),
580 |   ((0,15,57),(0,15,57),()), ((0,15,62),(0,15,63),()),
581 |   ((0,15,130),(0,15,132),()), ((0,15,134),(0,15,135),()),
582 |   ((0,15,198),(0,15,198),()), ((0,16,55),(0,16,55),()),
583 |   ((0,16,57),(0,16,58),()), ((0,16,135),(0,16,141),()),
584 |   ((0,16,143),(0,16,143),()), ((0,16,154),(0,16,155),()),
585 |   ((0,23,201),(0,23,211),()), ((0,23,221),(0,23,221),()),
586 |   ((0,25,57),(0,25,59),()), ((0,26,117),(0,26,124),()),
587 |   ((0,26,127),(0,26,127),()), ((0,26,176),(0,26,189),()),
588 |   ((0,27,52),(0,27,52),()), ((0,27,68),(0,27,68),()),
589 |   ((0,27,107),(0,27,115),()), ((0,27,170),(0,27,171),()),
590 |   ((0,28,54),(0,28,55),()), ((0,28,120),(0,28,125),()),
591 |   ((0,28,208),(0,28,232),()), ((0,28,237),(0,28,237),()),
592 |   ((0,28,244),(0,28,244),()), ((0,28,248),(0,28,249),()),
593 |   ((0,29,44),(0,29,106),()), ((0,29,196),(0,29,207),()),
594 |   ((0,29,245),(0,29,245),()), ((0,29,253),(0,29,255),()),
595 |   ((0,31,189),(0,31,189),()), ((0,31,191),(0,31,193),()),
596 |   ((0,31,205),(0,31,207),()), ((0,31,221),(0,31,223),()),
597 |   ((0,31,237),(0,31,239),()), ((0,31,253),(0,31,254),()),
598 |   ((0,44,239),(0,44,241),()), ((0,46,47),(0,46,47),()),
599 |   ((0,48,42),(0,48,47),()), ((0,48,153),(0,48,156),()),
600 |   ((0,48,252),(0,48,252),()), ((0,166,111),(0,166,111),()),
601 |   ((0,166,124),(0,166,125),()), ((0,166,127),(0,166,127),()),
602 |   ((0,166,156),(0,166,157),()), ((0,166,240),(0,166,241),()),
603 |   ((0,167,23),(0,167,33),()), ((0,167,136),(0,167,136),()),
604 |   ((0,167,248),(0,167,249),()), ((0,168,196),(0,168,196),()),
605 |   ((0,168,224),(0,168,241),()), ((0,169,43),(0,169,46),()),
606 |   ((0,169,83),(0,169,83),()), ((0,169,179),(0,169,179),()),
607 |   ((0,169,192),(0,169,192),()), ((0,169,229),(0,169,229),()),
608 |   ((0,170,123),(0,170,125),()), ((0,170,191),(0,170,194),()),
609 |   ((0,170,246),(0,170,246),()), ((0,171,91),(0,171,95),()),
610 |   ((0,171,236),(0,171,237),()), ((0,251,30),(0,251,30),()),
611 |   ((0,254,32),(0,254,47),()), ((0,255,62),(0,255,62),()),
612 |   ((0,255,64),(0,255,64),()), ((0,255,112),(0,255,112),()),
613 |   ((0,255,158),(0,255,159),()), ((0,255,227),(0,255,227),()),
614 |   ((1,2,224),(1,2,224),()), ((1,10,229),(1,10,230),()),
615 |   ((1,16,185),(1,16,186),()), ((1,17,51),(1,17,52),()),
616 |   ((1,17,115),(1,17,115),()), ((1,17,192),(1,17,192),()),
617 |   ((1,17,202),(1,17,204),()), ((1,18,53),(1,18,54),()),
618 |   ((1,18,233),(1,18,234),()), ((1,19,60),(1,19,60),()),
619 |   ((1,19,77),(1,19,77),()), ((1,19,102),(1,19,108),()),
620 |   ((1,19,112),(1,19,116),()), ((1,20,66),(1,20,66),()),
621 |   ((1,20,70),(1,20,70),()), ((1,20,194),(1,20,195),()),
622 |   ((1,21,191),(1,21,192),()), ((1,22,63),(1,22,63),()),
623 |   ((1,22,182),(1,22,183),()), ((1,23,43),(1,23,43),()),
624 |   ((1,28,63),(1,28,63),()), ((1,106,240),(1,106,244),()),
625 |   ((1,111,143),(1,111,159),()), ((1,209,103),(1,209,105),()),
626 |   ((1,209,109),(1,209,114),()), ((1,209,123),(1,209,130),()),
627 |   ((1,209,133),(1,209,139),()), ((1,209,170),(1,209,173),()),
628 |   ((1,232,208),(1,232,214),()), ((1,233,68),(1,233,70),()),
629 |   ((1,233,72),(1,233,74),())
630 | ];
631 | 
632 | pub static UCD_MATH: &'static [((u8,u8,u8), (u8,u8,u8), ())] = &[
633 |   ((0,0,43),(0,0,43),()), ((0,0,60),(0,0,62),()), ((0,0,94),(0,0,94),()),
634 |   ((0,0,124),(0,0,124),()), ((0,0,126),(0,0,126),()), ((0,0,172),(0,0,172),()),
635 |   ((0,0,177),(0,0,177),()), ((0,0,215),(0,0,215),()), ((0,0,247),(0,0,247),()),
636 |   ((0,3,208),(0,3,210),()), ((0,3,213),(0,3,213),()), ((0,3,240),(0,3,241),()),
637 |   ((0,3,244),(0,3,246),()), ((0,6,6),(0,6,8),()), ((0,32,22),(0,32,22),()),
638 |   ((0,32,50),(0,32,52),()), ((0,32,64),(0,32,64),()), ((0,32,68),(0,32,68),()),
639 |   ((0,32,82),(0,32,82),()), ((0,32,97),(0,32,100),()),
640 |   ((0,32,122),(0,32,126),()), ((0,32,138),(0,32,142),()),
641 |   ((0,32,208),(0,32,220),()), ((0,32,225),(0,32,225),()),
642 |   ((0,32,229),(0,32,230),()), ((0,32,235),(0,32,239),()),
643 |   ((0,33,2),(0,33,2),()), ((0,33,7),(0,33,7),()), ((0,33,10),(0,33,19),()),
644 |   ((0,33,21),(0,33,21),()), ((0,33,24),(0,33,29),()), ((0,33,36),(0,33,36),()),
645 |   ((0,33,40),(0,33,41),()), ((0,33,44),(0,33,45),()), ((0,33,47),(0,33,49),()),
646 |   ((0,33,51),(0,33,56),()), ((0,33,60),(0,33,73),()), ((0,33,75),(0,33,75),()),
647 |   ((0,33,144),(0,33,167),()), ((0,33,169),(0,33,174),()),
648 |   ((0,33,176),(0,33,177),()), ((0,33,182),(0,33,183),()),
649 |   ((0,33,188),(0,33,219),()), ((0,33,221),(0,33,221),()),
650 |   ((0,33,228),(0,33,229),()), ((0,33,244),(0,34,255),()),
651 |   ((0,35,8),(0,35,11),()), ((0,35,32),(0,35,33),()), ((0,35,124),(0,35,124),()),
652 |   ((0,35,155),(0,35,181),()), ((0,35,183),(0,35,183),()),
653 |   ((0,35,208),(0,35,208),()), ((0,35,220),(0,35,226),()),
654 |   ((0,37,160),(0,37,161),()), ((0,37,174),(0,37,183),()),
655 |   ((0,37,188),(0,37,193),()), ((0,37,198),(0,37,199),()),
656 |   ((0,37,202),(0,37,203),()), ((0,37,207),(0,37,211),()),
657 |   ((0,37,226),(0,37,226),()), ((0,37,228),(0,37,228),()),
658 |   ((0,37,231),(0,37,236),()), ((0,37,248),(0,37,255),()),
659 |   ((0,38,5),(0,38,6),()), ((0,38,64),(0,38,64),()), ((0,38,66),(0,38,66),()),
660 |   ((0,38,96),(0,38,99),()), ((0,38,109),(0,38,111),()),
661 |   ((0,39,192),(0,39,255),()), ((0,41,0),(0,42,255),()),
662 |   ((0,43,48),(0,43,68),()), ((0,43,71),(0,43,76),()),
663 |   ((0,251,41),(0,251,41),()), ((0,254,97),(0,254,102),()),
664 |   ((0,254,104),(0,254,104),()), ((0,255,11),(0,255,11),()),
665 |   ((0,255,28),(0,255,30),()), ((0,255,60),(0,255,60),()),
666 |   ((0,255,62),(0,255,62),()), ((0,255,92),(0,255,92),()),
667 |   ((0,255,94),(0,255,94),()), ((0,255,226),(0,255,226),()),
668 |   ((0,255,233),(0,255,236),()), ((1,212,0),(1,212,84),()),
669 |   ((1,212,86),(1,212,156),()), ((1,212,158),(1,212,159),()),
670 |   ((1,212,162),(1,212,162),()), ((1,212,165),(1,212,166),()),
671 |   ((1,212,169),(1,212,172),()), ((1,212,174),(1,212,185),()),
672 |   ((1,212,187),(1,212,187),()), ((1,212,189),(1,212,195),()),
673 |   ((1,212,197),(1,213,5),()), ((1,213,7),(1,213,10),()),
674 |   ((1,213,13),(1,213,20),()), ((1,213,22),(1,213,28),()),
675 |   ((1,213,30),(1,213,57),()), ((1,213,59),(1,213,62),()),
676 |   ((1,213,64),(1,213,68),()), ((1,213,70),(1,213,70),()),
677 |   ((1,213,74),(1,213,80),()), ((1,213,82),(1,214,165),()),
678 |   ((1,214,168),(1,215,203),()), ((1,215,206),(1,215,255),()),
679 |   ((1,238,0),(1,238,3),()), ((1,238,5),(1,238,31),()),
680 |   ((1,238,33),(1,238,34),()), ((1,238,36),(1,238,36),()),
681 |   ((1,238,39),(1,238,39),()), ((1,238,41),(1,238,50),()),
682 |   ((1,238,52),(1,238,55),()), ((1,238,57),(1,238,57),()),
683 |   ((1,238,59),(1,238,59),()), ((1,238,66),(1,238,66),()),
684 |   ((1,238,71),(1,238,71),()), ((1,238,73),(1,238,73),()),
685 |   ((1,238,75),(1,238,75),()), ((1,238,77),(1,238,79),()),
686 |   ((1,238,81),(1,238,82),()), ((1,238,84),(1,238,84),()),
687 |   ((1,238,87),(1,238,87),()), ((1,238,89),(1,238,89),()),
688 |   ((1,238,91),(1,238,91),()), ((1,238,93),(1,238,93),()),
689 |   ((1,238,95),(1,238,95),()), ((1,238,97),(1,238,98),()),
690 |   ((1,238,100),(1,238,100),()), ((1,238,103),(1,238,106),()),
691 |   ((1,238,108),(1,238,114),()), ((1,238,116),(1,238,119),()),
692 |   ((1,238,121),(1,238,124),()), ((1,238,126),(1,238,126),()),
693 |   ((1,238,128),(1,238,137),()), ((1,238,139),(1,238,155),()),
694 |   ((1,238,161),(1,238,163),()), ((1,238,165),(1,238,169),()),
695 |   ((1,238,171),(1,238,187),()), ((1,238,240),(1,238,241),())
696 | ];
697 | 
698 | pub static UCD_ALPHA_OTHER: &'static [((u8,u8,u8), (u8,u8,u8), ())] = &[
699 |   ((0,3,69),(0,3,69),()), ((0,5,176),(0,5,189),()), ((0,5,191),(0,5,191),()),
700 |   ((0,5,193),(0,5,194),()), ((0,5,196),(0,5,197),()), ((0,5,199),(0,5,199),()),
701 |   ((0,6,16),(0,6,26),()), ((0,6,75),(0,6,87),()), ((0,6,89),(0,6,95),()),
702 |   ((0,6,112),(0,6,112),()), ((0,6,214),(0,6,220),()), ((0,6,225),(0,6,228),()),
703 |   ((0,6,231),(0,6,232),()), ((0,6,237),(0,6,237),()), ((0,7,17),(0,7,17),()),
704 |   ((0,7,48),(0,7,63),()), ((0,7,166),(0,7,176),()), ((0,8,22),(0,8,23),()),
705 |   ((0,8,27),(0,8,35),()), ((0,8,37),(0,8,39),()), ((0,8,41),(0,8,44),()),
706 |   ((0,8,212),(0,8,223),()), ((0,8,227),(0,8,233),()), ((0,8,240),(0,9,3),()),
707 |   ((0,9,58),(0,9,59),()), ((0,9,62),(0,9,76),()), ((0,9,78),(0,9,79),()),
708 |   ((0,9,85),(0,9,87),()), ((0,9,98),(0,9,99),()), ((0,9,129),(0,9,131),()),
709 |   ((0,9,190),(0,9,196),()), ((0,9,199),(0,9,200),()), ((0,9,203),(0,9,204),()),
710 |   ((0,9,215),(0,9,215),()), ((0,9,226),(0,9,227),()), ((0,10,1),(0,10,3),()),
711 |   ((0,10,62),(0,10,66),()), ((0,10,71),(0,10,72),()), ((0,10,75),(0,10,76),()),
712 |   ((0,10,81),(0,10,81),()), ((0,10,112),(0,10,113),()),
713 |   ((0,10,117),(0,10,117),()), ((0,10,129),(0,10,131),()),
714 |   ((0,10,190),(0,10,197),()), ((0,10,199),(0,10,201),()),
715 |   ((0,10,203),(0,10,204),()), ((0,10,226),(0,10,227),()),
716 |   ((0,11,1),(0,11,3),()), ((0,11,62),(0,11,68),()), ((0,11,71),(0,11,72),()),
717 |   ((0,11,75),(0,11,76),()), ((0,11,86),(0,11,87),()), ((0,11,98),(0,11,99),()),
718 |   ((0,11,130),(0,11,130),()), ((0,11,190),(0,11,194),()),
719 |   ((0,11,198),(0,11,200),()), ((0,11,202),(0,11,204),()),
720 |   ((0,11,215),(0,11,215),()), ((0,12,0),(0,12,3),()), ((0,12,62),(0,12,68),()),
721 |   ((0,12,70),(0,12,72),()), ((0,12,74),(0,12,76),()), ((0,12,85),(0,12,86),()),
722 |   ((0,12,98),(0,12,99),()), ((0,12,129),(0,12,131),()),
723 |   ((0,12,190),(0,12,196),()), ((0,12,198),(0,12,200),()),
724 |   ((0,12,202),(0,12,204),()), ((0,12,213),(0,12,214),()),
725 |   ((0,12,226),(0,12,227),()), ((0,13,1),(0,13,3),()), ((0,13,62),(0,13,68),()),
726 |   ((0,13,70),(0,13,72),()), ((0,13,74),(0,13,76),()), ((0,13,87),(0,13,87),()),
727 |   ((0,13,98),(0,13,99),()), ((0,13,130),(0,13,131),()),
728 |   ((0,13,207),(0,13,212),()), ((0,13,214),(0,13,214),()),
729 |   ((0,13,216),(0,13,223),()), ((0,13,242),(0,13,243),()),
730 |   ((0,14,49),(0,14,49),()), ((0,14,52),(0,14,58),()), ((0,14,77),(0,14,77),()),
731 |   ((0,14,177),(0,14,177),()), ((0,14,180),(0,14,185),()),
732 |   ((0,14,187),(0,14,188),()), ((0,14,205),(0,14,205),()),
733 |   ((0,15,113),(0,15,129),()), ((0,15,141),(0,15,151),()),
734 |   ((0,15,153),(0,15,188),()), ((0,16,43),(0,16,54),()),
735 |   ((0,16,56),(0,16,56),()), ((0,16,59),(0,16,62),()), ((0,16,86),(0,16,89),()),
736 |   ((0,16,94),(0,16,96),()), ((0,16,98),(0,16,98),()),
737 |   ((0,16,103),(0,16,104),()), ((0,16,113),(0,16,116),()),
738 |   ((0,16,130),(0,16,134),()), ((0,16,156),(0,16,157),()),
739 |   ((0,19,95),(0,19,95),()), ((0,23,18),(0,23,19),()), ((0,23,50),(0,23,51),()),
740 |   ((0,23,82),(0,23,83),()), ((0,23,114),(0,23,115),()),
741 |   ((0,23,182),(0,23,200),()), ((0,24,133),(0,24,134),()),
742 |   ((0,24,169),(0,24,169),()), ((0,25,32),(0,25,43),()),
743 |   ((0,25,48),(0,25,56),()), ((0,26,23),(0,26,27),()), ((0,26,85),(0,26,94),()),
744 |   ((0,26,97),(0,26,116),()), ((0,27,0),(0,27,4),()), ((0,27,53),(0,27,67),()),
745 |   ((0,27,128),(0,27,130),()), ((0,27,161),(0,27,169),()),
746 |   ((0,27,172),(0,27,173),()), ((0,27,231),(0,27,241),()),
747 |   ((0,28,36),(0,28,53),()), ((0,28,242),(0,28,243),()),
748 |   ((0,29,231),(0,29,244),()), ((0,36,182),(0,36,233),()),
749 |   ((0,45,224),(0,45,255),()), ((0,166,116),(0,166,123),()),
750 |   ((0,166,158),(0,166,159),()), ((0,168,35),(0,168,39),()),
751 |   ((0,168,128),(0,168,129),()), ((0,168,180),(0,168,195),()),
752 |   ((0,168,197),(0,168,197),()), ((0,169,38),(0,169,42),()),
753 |   ((0,169,71),(0,169,82),()), ((0,169,128),(0,169,131),()),
754 |   ((0,169,180),(0,169,191),()), ((0,170,41),(0,170,54),()),
755 |   ((0,170,67),(0,170,67),()), ((0,170,76),(0,170,77),()),
756 |   ((0,170,176),(0,170,176),()), ((0,170,178),(0,170,180),()),
757 |   ((0,170,183),(0,170,184),()), ((0,170,190),(0,170,190),()),
758 |   ((0,170,235),(0,170,239),()), ((0,170,245),(0,170,245),()),
759 |   ((0,171,227),(0,171,234),()), ((0,251,30),(0,251,30),()),
760 |   ((1,3,118),(1,3,122),()), ((1,10,1),(1,10,3),()), ((1,10,5),(1,10,6),()),
761 |   ((1,10,12),(1,10,15),()), ((1,16,0),(1,16,2),()), ((1,16,56),(1,16,69),()),
762 |   ((1,16,130),(1,16,130),()), ((1,16,176),(1,16,184),()),
763 |   ((1,17,0),(1,17,2),()), ((1,17,39),(1,17,50),()), ((1,17,128),(1,17,130),()),
764 |   ((1,17,179),(1,17,191),()), ((1,18,44),(1,18,52),()),
765 |   ((1,18,55),(1,18,55),()), ((1,18,62),(1,18,62),()),
766 |   ((1,18,223),(1,18,232),()), ((1,19,0),(1,19,3),()), ((1,19,62),(1,19,68),()),
767 |   ((1,19,71),(1,19,72),()), ((1,19,75),(1,19,76),()), ((1,19,87),(1,19,87),()),
768 |   ((1,19,98),(1,19,99),()), ((1,20,53),(1,20,65),()), ((1,20,67),(1,20,69),()),
769 |   ((1,20,176),(1,20,193),()), ((1,21,175),(1,21,181),()),
770 |   ((1,21,184),(1,21,190),()), ((1,21,220),(1,21,221),()),
771 |   ((1,22,48),(1,22,62),()), ((1,22,64),(1,22,64),()),
772 |   ((1,22,171),(1,22,181),()), ((1,23,29),(1,23,42),()),
773 |   ((1,28,47),(1,28,54),()), ((1,28,56),(1,28,62),()),
774 |   ((1,28,146),(1,28,167),()), ((1,28,169),(1,28,182),()),
775 |   ((1,107,48),(1,107,54),()), ((1,111,81),(1,111,126),()),
776 |   ((1,188,158),(1,188,158),()), ((1,224,0),(1,224,6),()),
777 |   ((1,224,8),(1,224,24),()), ((1,224,27),(1,224,33),()),
778 |   ((1,224,35),(1,224,36),()), ((1,224,38),(1,224,42),()),
779 |   ((1,233,71),(1,233,71),()), ((1,241,48),(1,241,73),()),
780 |   ((1,241,80),(1,241,105),()), ((1,241,112),(1,241,137),())
781 | ];
782 | 
783 | 


--------------------------------------------------------------------------------
/src/tables/mod.rs:
--------------------------------------------------------------------------------
 1 | pub mod general;
 2 | pub use self::general::*;
 3 | 
 4 | pub mod bidi;
 5 | pub use self::bidi::*;
 6 | 
 7 | pub mod misc;
 8 | pub use self::misc::*;
 9 | 
10 | pub mod scripts;
11 | pub use self::scripts::*;
12 | 
13 | pub mod function;
14 | pub use self::function::*;
15 | 
16 | pub mod bool;
17 | pub use self::bool::*;
18 | 
19 | pub mod rem;
20 | pub use self::rem::*;
21 | 
22 | pub mod decomp;
23 | pub use self::decomp::*;
24 | 
25 | use core::cmp::Ordering::{Equal, Less, Greater};
26 | use core::char;
27 | 
28 | pub trait Search {
29 |     type T: Clone;
30 |     fn search(&self, cp: char) -> Option<Self::T>;
31 |     fn includes(&self, cp: char) -> bool {
32 |         match self.search(cp) {
33 |             Some(_) => true,
34 |             None => false
35 |         }
36 |     }
37 | }
38 | 
39 | impl<S: Clone> Search for [((u8,u8,u8),(u8,u8,u8),S)] {
40 |     type T = S;
41 |     fn search(&self, cp: char) -> Option<S> {
42 |         let cp = cp as u32;
43 |         match self.binary_search_by(|&((rb1,rb2,rb3), (re1,re2,re3), _)| {
44 |             let rb: u32 = (rb1 as u32)*65536 + (rb2 as u32)*256 + (rb3 as u32);
45 |             let re: u32 = (re1 as u32)*65536 + (re2 as u32)*256 + (re3 as u32);
46 |             if rb <= cp && cp <= re { Equal }
47 |             else if re < cp { Less }
48 |             else { Greater }
49 |         }) {
50 |             Ok(idx) => {
51 |                 let (_, _, ref v) = self[idx];
52 |                 Some(v.clone())
53 |             },
54 |             _ => None
55 |         }
56 |     }
57 | }
58 | 
59 | impl<S: Clone> Search for [((u8,u8,u8),S)] {
60 |     type T = S;
61 |     fn search(&self, cp: char) -> Option<S> {
62 |         let ca = cp as u32;
63 |         match self.binary_search_by(|&((cb1,cb2,cb3), _)| {
64 |             let cb: u32 = (cb1 as u32)*65536 + (cb2 as u32)*256 + (cb3 as u32);
65 |             cb.cmp(&ca)
66 |         }) {
67 |             Ok(idx) => {
68 |                 let (_, ref v) = self[idx];
69 |                 Some(v.clone())
70 |             },
71 |             _ => None
72 |         }
73 |     }
74 | }
75 | 
76 | impl Search for [(u16,u16)] {
77 |     type T = char;
78 |     fn search(&self, cp: char) -> Option<char> {
79 |         let ca = cp as u32;
80 |         let cb = ca as u16;
81 | 
82 |         if ca > 65536 { return None; }
83 | 
84 |         match self.binary_search_by(|&(cc,_)| cc.cmp(&cb)) {
85 |             Ok(idx) => {
86 |                 let (_, v) = self[idx];
87 |                 char::from_u32(v as u32)
88 |             },
89 |             _ => None
90 |         }
91 |     }
92 | }


--------------------------------------------------------------------------------
/tests/data.tar.xz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sourtin/libucd/fc89190397d21e43ab9b2246ab5b4aad88524875/tests/data.tar.xz


--------------------------------------------------------------------------------
/tests/lib.rs:
--------------------------------------------------------------------------------
  1 | extern crate ucd;
  2 | use ucd::Codepoint;
  3 | use std::char;
  4 | use std::io::BufReader;
  5 | use std::io::prelude::*;
  6 | use std::fs::File;
  7 | 
  8 | fn test<F>(path: &str, func: F)
  9 |     where F: Fn(usize, char, &str) -> ()
 10 | {
 11 |     let f = File::open(path).unwrap();
 12 |     let reader = BufReader::new(f);
 13 | 
 14 |     let mut max: usize = 0;
 15 |     for (i, line) in reader.lines().enumerate() {
 16 |         max = i;
 17 |         let line = line.unwrap();
 18 | 
 19 |         assert!(i < 0x110000);
 20 |         let cp = unsafe { char::from_u32_unchecked(i as u32) };
 21 | 
 22 |         func(i, cp, &line);
 23 |     }
 24 | 
 25 |     // make sure the test file covers the full range
 26 |     assert!(max == 0x10ffff);
 27 | }
 28 | 
 29 | fn test_bool<F>(path: &str, func: F)
 30 |     where F: Fn(char) -> bool
 31 | {
 32 |     test(path, |i, cp, line| {
 33 |         let b1 = func(cp);
 34 |         let b2 = match line.chars().next() {
 35 |             Some('0') => false,
 36 |             Some('1') => true,
 37 |             _ => panic!("unexpected test data, line {}", i)
 38 |         };
 39 | 
 40 |         if b1 != b2 { panic!("{}: {:?} {:?}", i, b1, b2); }
 41 |     });
 42 | }
 43 | 
 44 | fn test_oint<F>(path: &str, func: F)
 45 |     where F: Fn(char) -> Option<u32>
 46 | {
 47 |     test(path, |i, cp, line| {
 48 |         let c1 = func(cp);
 49 |         let c2 = line.parse::<u32>().ok();
 50 |         if c1 != c2 { panic!("{}: {:?} {:?}", i, c1, c2); }
 51 |     });
 52 | }
 53 | 
 54 | fn test_int<F>(path: &str, func: F)
 55 |     where F: Fn(char) -> u32
 56 | {
 57 |     test_oint(path, |cp| Some(func(cp)));
 58 | }
 59 | 
 60 | fn test_cps<F>(path: &str, func: F)
 61 |     where F: Fn(char) -> ucd::CharIter
 62 | {
 63 |     test(path, |i, cp, line| {
 64 |         let x1: Vec<u32> = func(cp).map(|c| c as u32).collect();
 65 |         let x2: Vec<u32> = if line.len() > 0 {
 66 |             line.split(' ').map(|v| v.parse::<u32>().unwrap()).collect()
 67 |         } else {
 68 |             Vec::new()
 69 |         };
 70 |     
 71 |         if x1 != x2 { panic!("{}: {:?} {:?}", i, x1, x2); }
 72 |     });
 73 | }
 74 | 
 75 | // ... begin tests ... //
 76 | 
 77 | // general
 78 | 
 79 | #[test]
 80 | fn age() { test("./tests/data/age.txt", |i, cp, line| {
 81 |     let chars = line.chars().collect::<Vec<char>>();
 82 | 
 83 |     let a1 = cp.age();
 84 |     let a2 = if chars[0] == '-' {
 85 |         None
 86 |     } else {
 87 |         Some((chars[0].to_digit(10).unwrap() as u8,
 88 |               chars[1].to_digit(10).unwrap() as u8))
 89 |     };
 90 | 
 91 |     if a1 != a2 { panic!("{}: {:?} {:?}", i, a1, a2); }
 92 | });}
 93 | 
 94 | #[test]
 95 | fn block() {
 96 |     test_oint("./tests/data/block.txt",
 97 |         |cp| cp.block().map(|b| b as u32));
 98 | }
 99 | 
100 | #[test]
101 | fn category() {
102 |     test_int("./tests/data/cat.txt",
103 |         |cp| cp.category() as u32);
104 | }
105 | 
106 | 
107 | 
108 | // function and appearance
109 | 
110 | #[test]
111 | fn is_alphabetic() {
112 |     test_bool("./tests/data/alpha.txt",
113 |         |cp| cp.is_alpha());
114 | }
115 | 
116 | #[test]
117 | fn is_alphabetic_other() {
118 |     test_bool("./tests/data/alpha-other.txt",
119 |         |cp| cp.is_alphabetic_other());
120 | }
121 | 
122 | #[test]
123 | fn is_dash() {
124 |     test_bool("./tests/data/dash.txt",
125 |         |cp| cp.is_dash());
126 | }
127 | 
128 | #[test]
129 | fn is_default_ignorable() {
130 |     test_bool("./tests/data/default-ignorable.txt",
131 |         |cp| cp.is_default_ignorable());
132 | }
133 | 
134 | #[test]
135 | fn is_default_ignorable_other() {
136 |     test_bool("./tests/data/default-ignorable-other.txt",
137 |         |cp| cp.is_default_ignorable_other());
138 | }
139 | 
140 | #[test]
141 | fn is_deprecated() {
142 |     test_bool("./tests/data/deprec.txt",
143 |         |cp| cp.is_deprecated());
144 | }
145 | 
146 | #[test]
147 | fn is_diacritic() {
148 |     test_bool("./tests/data/diacritic.txt",
149 |         |cp| cp.is_diacritic());
150 | }
151 | 
152 | #[test]
153 | fn is_extender() {
154 |     test_bool("./tests/data/extender.txt",
155 |         |cp| cp.is_extender());
156 | }
157 | 
158 | #[test]
159 | fn is_hex_digit() {
160 |     test_bool("./tests/data/hex-digit.txt",
161 |         |cp| cp.is_hex_digit());
162 | }
163 | 
164 | #[test]
165 | fn is_hex_digit_ascii() {
166 |     test_bool("./tests/data/hex-digit-ascii.txt",
167 |         |cp| cp.is_hex_digit_ascii());
168 | }
169 | 
170 | #[test]
171 | fn is_hyphen() {
172 |     test_bool("./tests/data/hyphen.txt",
173 |         |cp| cp.is_hyphen());
174 | }
175 | 
176 | #[test]
177 | fn is_logical_order_exception() {
178 |     test_bool("./tests/data/logical-order-exception.txt",
179 |         |cp| cp.is_logical_order_exception());
180 | }
181 | 
182 | #[test]
183 | fn is_math() {
184 |     test_bool("./tests/data/math.txt",
185 |         |cp| cp.is_math());
186 | }
187 | 
188 | #[test]
189 | fn is_math_other() {
190 |     test_bool("./tests/data/math-other.txt",
191 |         |cp| cp.is_math_other());
192 | }
193 | 
194 | #[test]
195 | fn is_noncharacter() {
196 |     test_bool("./tests/data/nonchar.txt",
197 |         |cp| cp.is_noncharacter());
198 | }
199 | 
200 | #[test]
201 | fn is_preprended_concatenation_mark() {
202 |     test_bool("./tests/data/prepended-concatenation-mark.txt",
203 |         |cp| cp.is_preprended_concatenation_mark());
204 | }
205 | 
206 | #[test]
207 | fn is_quotation_mark() {
208 |     test_bool("./tests/data/quot.txt",
209 |         |cp| cp.is_quotation_mark());
210 | }
211 | 
212 | #[test]
213 | fn is_sentence_terminal() {
214 |     test_bool("./tests/data/term-sentence.txt",
215 |         |cp| cp.is_sentence_terminal());
216 | }
217 | 
218 | #[test]
219 | fn is_soft_dotted() {
220 |     test_bool("./tests/data/soft-dotted.txt",
221 |         |cp| cp.is_soft_dotted());
222 | }
223 | 
224 | #[test]
225 | fn is_terminal_punctuation() {
226 |     test_bool("./tests/data/term-punc.txt",
227 |         |cp| cp.is_terminal_punctuation());
228 | }
229 | 
230 | #[test]
231 | fn is_whitespace() {
232 |     test_bool("./tests/data/white.txt",
233 |         |cp| cp.is_white());
234 | }
235 | 
236 | #[test]
237 | fn is_variation_selector() {
238 |     test_bool("./tests/data/varsel.txt",
239 |         |cp| cp.is_variation_selector());
240 | }
241 | 
242 | 
243 | 
244 | // numeric
245 | 
246 | #[test]
247 | fn numeric_type() {
248 |     test_oint("./tests/data/numtype.txt",
249 |         |cp| cp.numeric_type().map(|t| t as u32));
250 | }
251 | 
252 | #[test]
253 | fn numeric_value() { test("./tests/data/numval.txt", |i, cp, line| {
254 |     let n1 = cp.numeric_value();
255 | 
256 |     let ls: Vec<&str> = line.splitn(2, ',').collect();
257 |     let n2 = match ls[1].parse::<u32>().unwrap() {
258 |         0 => None,
259 |         1 => Some(ucd::Number::Integer(ls[0].parse::<i64>().unwrap())),
260 |         den => Some(ucd::Number::Rational(ls[0].parse::<i32>().unwrap(), den))
261 |     };
262 | 
263 |     if n1 != n2 { panic!("{}: {:?} {:?}", i, n1, n2); }
264 | });}
265 | 
266 | 
267 | 
268 | // identifiers and syntax
269 | 
270 | #[test]
271 | fn is_id_continue() {
272 |     test_bool("./tests/data/id-cont.txt",
273 |         |cp| cp.is_id_continue());
274 | }
275 | 
276 | #[test]
277 | fn is_id_continue_nfkc() {
278 |     test_bool("./tests/data/id-cont-nfkc.txt",
279 |         |cp| cp.is_id_continue_nfkc());
280 | }
281 | 
282 | #[test]
283 | fn is_id_continue_other() {
284 |     test_bool("./tests/data/id-cont-other.txt",
285 |         |cp| cp.is_id_continue_other());
286 | }
287 | 
288 | #[test]
289 | fn is_id_start() {
290 |     test_bool("./tests/data/id-start.txt",
291 |         |cp| cp.is_id_start());
292 | }
293 | 
294 | #[test]
295 | fn is_id_start_nfkc() {
296 |     test_bool("./tests/data/id-start-nfkc.txt",
297 |         |cp| cp.is_id_start_nfkc());
298 | }
299 | 
300 | #[test]
301 | fn is_id_start_other() {
302 |     test_bool("./tests/data/id-start-other.txt",
303 |         |cp| cp.is_id_start_other());
304 | }
305 | 
306 | #[test]
307 | fn is_pattern_syntax() {
308 |     test_bool("./tests/data/patt-syntax.txt",
309 |         |cp| cp.is_pattern_syntax());
310 | }
311 | 
312 | #[test]
313 | fn is_pattern_whitespace() {
314 |     test_bool("./tests/data/patt-white.txt",
315 |         |cp| cp.is_pattern_whitespace());
316 | }
317 | 
318 | 
319 | 
320 | // scripts
321 | 
322 | #[test]
323 | fn east_asian_width() {
324 |     test_int("./tests/data/eawidth.txt",
325 |         |cp| cp.east_asian_width() as u32);
326 | }
327 | 
328 | #[test]
329 | fn hangul_syllable_type() {
330 |     test_oint("./tests/data/hangulst.txt",
331 |         |cp| cp.hangul_syllable_type().map(|c| c as u32));
332 | }
333 | 
334 | #[test]
335 | fn jamo_short_name() { test("./tests/data/jsn.txt", |i, cp, s2| {
336 |     let s1 = cp.jamo_short_name().unwrap_or("-");
337 |     if s1 != s2 { panic!("{}: {:?} {:?}", i, s1, s2); }
338 | });}
339 | 
340 | #[test]
341 | fn indic_positional_category() {
342 |     test_oint("./tests/data/indic-poscat.txt",
343 |         |cp| cp.indic_positional_category().map(|c| c as u32));
344 | }
345 | 
346 | #[test]
347 | fn indic_syllabic_category() {
348 |     test_int("./tests/data/indic-sylcat.txt",
349 |         |cp| cp.indic_syllabic_category() as u32);
350 | }
351 | 
352 | #[test]
353 | fn is_ideograph() {
354 |     test_bool("./tests/data/ideo.txt",
355 |         |cp| cp.is_ideograph());
356 | }
357 | 
358 | #[test]
359 | fn is_ideograph_description_sequence_binary_operator() {
360 |     test_bool("./tests/data/ideo-desc-seq-bin-op.txt",
361 |         |cp| cp.is_ideograph_description_sequence_binary_operator());
362 | }
363 | 
364 | #[test]
365 | fn is_ideograph_description_sequence_radical() {
366 |     test_bool("./tests/data/ideo-desc-seq-radical.txt",
367 |         |cp| cp.is_ideograph_description_sequence_radical());
368 | }
369 | 
370 | #[test]
371 | fn is_ideograph_description_sequence_trinary_operator() {
372 |     test_bool("./tests/data/ideo-desc-seq-trin-op.txt",
373 |         |cp| cp.is_ideograph_description_sequence_trinary_operator());
374 | }
375 | 
376 | #[test]
377 | fn is_ideograph_unified() {
378 |     test_bool("./tests/data/ideo-unified.txt",
379 |         |cp| cp.is_ideograph_unified());
380 | }
381 | 
382 | #[test]
383 | fn join_control() {
384 |     test_bool("./tests/data/joinctl.txt",
385 |         |cp| cp.join_control());
386 | }
387 | 
388 | #[test]
389 | fn joining_group() {
390 |     test_int("./tests/data/joingrp.txt",
391 |         |cp| cp.joining_group() as u32);
392 | }
393 | 
394 | #[test]
395 | fn joining_type() {
396 |     test_int("./tests/data/jointyp.txt",
397 |         |cp| cp.joining_type() as u32);
398 | }
399 | 
400 | #[test]
401 | fn script() {
402 |     test_oint("./tests/data/script.txt",
403 |         |cp| cp.script().map(|c| c as u32));
404 | }
405 | 
406 | #[test]
407 | fn script_extensions() { test("./tests/data/scrext.txt", |i, cp, line| {
408 |     let x1 = cp.script_extensions().map(|s| s.iter().map(|sc| *sc as u32).collect::<Vec<u32>>());
409 |     let x2: Option<Vec<u32>> = if line.len() > 0 {
410 |         Some(line.split(' ').map(|v| v.parse::<u32>().unwrap()).collect())
411 |     } else {
412 |         None
413 |     };
414 | 
415 |     if x1 != x2 { panic!("{}: {:?} {:?}", i, x1, x2); }
416 | });}
417 | 
418 | 
419 | 
420 | // bidirectionality
421 | 
422 | #[test]
423 | fn bidi_class() {
424 |     test_int("./tests/data/bidi-class.txt",
425 |         |cp| cp.bidi_class() as u32);
426 | }
427 | 
428 | #[test]
429 | fn bidi_is_control() {
430 |     test_bool("./tests/data/bidi-control.txt",
431 |         |cp| cp.bidi_is_control());
432 | }
433 | 
434 | #[test]
435 | fn bidi_is_mirrored() {
436 |     test_bool("./tests/data/bidi-mirrored.txt",
437 |         |cp| cp.bidi_is_mirrored());
438 | }
439 | 
440 | #[test]
441 | fn bidi_mirror() {
442 |     test_oint("./tests/data/bidi-mirror.txt",
443 |         |cp| cp.bidi_mirror().map(|c| c as u32));
444 | }
445 | 
446 | #[test]
447 | fn bidi_paired_bracket() {
448 |     test_int("./tests/data/bidi-paired.txt",
449 |         |cp| cp.bidi_paired_bracket() as u32);
450 | }
451 | 
452 | #[test]
453 | fn bidi_paired_bracket_type() { test("./tests/data/bidi-bratype.txt", |i, cp, line| {
454 |     let b1 = cp.bidi_paired_bracket_type();
455 |     let b2 = match line.chars().next() {
456 |         Some('-') => None,
457 |         Some('(') => Some(ucd::BidiPairedBracketType::Open),
458 |         Some(')') => Some(ucd::BidiPairedBracketType::Close),
459 |         _ => panic!("unexpected test data, line {}", i)
460 |     };
461 | 
462 |     if b1 != b2 { panic!("{}: {:?} {:?}", i, b1, b2); }
463 | });}
464 | 
465 | 
466 | 
467 | // case
468 | 
469 | #[test]
470 | fn casefold() {
471 |     test_cps("./tests/data/cf.txt",
472 |         |cp| cp.casefold());
473 | }
474 | 
475 | #[test]
476 | fn casefold_nfkc() {
477 |     test_cps("./tests/data/cf-nfkc.txt",
478 |         |cp| cp.casefold_nfkc());
479 | }
480 | 
481 | #[test]
482 | fn casefold_nfkc_closure() {
483 |     test_cps("./tests/data/cf-closure.txt",
484 |         |cp| cp.casefold_nfkc_closure());
485 | }
486 | 
487 | #[test]
488 | fn casefold_simple() {
489 |     test_int("./tests/data/scf.txt",
490 |         |cp| cp.casefold_simple() as u32);
491 | }
492 | 
493 | #[test]
494 | fn changes_when_casefolded() {
495 |     test_bool("./tests/data/case-changes-casefold.txt",
496 |         |cp| cp.changes_when_casefolded());
497 | }
498 | 
499 | #[test]
500 | fn changes_when_casefolded_nfkc() {
501 |     test_bool("./tests/data/case-changes-casefold-nfkc.txt",
502 |         |cp| cp.changes_when_casefolded_nfkc());
503 | }
504 | 
505 | #[test]
506 | fn changes_when_casemapped() {
507 |     test_bool("./tests/data/case-changes-casemap.txt",
508 |         |cp| cp.changes_when_casemapped());
509 | }
510 | 
511 | #[test]
512 | fn changes_when_lowercased() {
513 |     test_bool("./tests/data/case-changes-lower.txt",
514 |         |cp| cp.changes_when_lowercased());
515 | }
516 | 
517 | #[test]
518 | fn changes_when_titlecased() {
519 |     test_bool("./tests/data/case-changes-title.txt",
520 |         |cp| cp.changes_when_titlecased());
521 | }
522 | 
523 | #[test]
524 | fn changes_when_uppercased() {
525 |     test_bool("./tests/data/case-changes-upper.txt",
526 |         |cp| cp.changes_when_uppercased());
527 | }
528 | 
529 | #[test]
530 | fn is_case_ignorable() {
531 |     test_bool("./tests/data/case-ignorable.txt",
532 |         |cp| cp.is_case_ignorable());
533 | }
534 | 
535 | #[test]
536 | fn is_cased() {
537 |     test_bool("./tests/data/cased.txt",
538 |         |cp| cp.is_cased());
539 | }
540 | 
541 | #[test]
542 | fn is_lowercase() {
543 |     test_bool("./tests/data/case-is-lower.txt",
544 |         |cp| cp.is_lower());
545 | }
546 | 
547 | #[test]
548 | fn is_lowercase_other() {
549 |     test_bool("./tests/data/case-is-lower-other.txt",
550 |         |cp| cp.is_lowercase_other());
551 | }
552 | 
553 | #[test]
554 | fn is_uppercase() {
555 |     test_bool("./tests/data/case-is-upper.txt",
556 |         |cp| cp.is_upper());
557 | }
558 | 
559 | #[test]
560 | fn is_uppercase_other() {
561 |     test_bool("./tests/data/case-is-upper-other.txt",
562 |         |cp| cp.is_uppercase_other());
563 | }
564 | 
565 | #[test]
566 | fn lowercase() {
567 |     test_cps("./tests/data/lc.txt",
568 |         |cp| cp.lowercase());
569 | }
570 | 
571 | #[test]
572 | fn lowercase_simple() {
573 |     test_int("./tests/data/slc.txt",
574 |         |cp| cp.lowercase_simple() as u32);
575 | }
576 | 
577 | #[test]
578 | fn titlecase() {
579 |     test_cps("./tests/data/tc.txt",
580 |         |cp| cp.titlecase());
581 | }
582 | 
583 | #[test]
584 | fn titlecase_simple() {
585 |     test_int("./tests/data/stc.txt",
586 |         |cp| cp.titlecase_simple() as u32);
587 | }
588 | 
589 | #[test]
590 | fn uppercase() {
591 |     test_cps("./tests/data/uc.txt",
592 |         |cp| cp.uppercase());
593 | }
594 | 
595 | #[test]
596 | fn uppercase_simple() {
597 |     test_int("./tests/data/suc.txt",
598 |         |cp| cp.uppercase_simple() as u32);
599 | }
600 | 
601 | 
602 | 
603 | // normalisation
604 | 
605 | #[test]
606 | fn canonical_combining_class() {
607 |     test_int("./tests/data/ccc.txt",
608 |         |cp| cp.canonical_combining_class() as u32);
609 | }
610 | 
611 | #[test]
612 | fn decomposition_map() {
613 |     test_cps("./tests/data/dmap.txt",
614 |         |cp| cp.decomposition_map());
615 | }
616 | 
617 | #[test]
618 | fn decomposition_type() {
619 |     test_oint("./tests/data/dtype.txt",
620 |         |cp| cp.decomposition_type().map(|t| t as u32));
621 | }
622 | 
623 | #[test]
624 | fn excluded_from_composition() {
625 |     test_bool("./tests/data/comp-excl.txt",
626 |         |cp| cp.excluded_from_composition());
627 | }
628 | 
629 | #[test]
630 | fn excluded_from_composition_fully() {
631 |     test_bool("./tests/data/comp-excl-full.txt",
632 |         |cp| cp.excluded_from_composition_fully());
633 | }
634 | 
635 | #[test]
636 | fn expands_on_nfc() {
637 |     test_bool("./tests/data/expanding-nfc.txt",
638 |         |cp| cp.expands_on_nfc());
639 | }
640 | 
641 | #[test]
642 | fn expands_on_nfd() {
643 |     test_bool("./tests/data/expanding-nfd.txt",
644 |         |cp| cp.expands_on_nfd());
645 | }
646 | 
647 | #[test]
648 | fn expands_on_nfkc() {
649 |     test_bool("./tests/data/expanding-nfkc.txt",
650 |         |cp| cp.expands_on_nfkc());
651 | }
652 | 
653 | #[test]
654 | fn expands_on_nfkd() {
655 |     test_bool("./tests/data/expanding-nfkd.txt",
656 |         |cp| cp.expands_on_nfkd());
657 | }
658 | 
659 | #[test]
660 | fn quick_check_nfc() {
661 |     test_int("./tests/data/qnfc.txt",
662 |         |cp| cp.quick_check_nfc() as u32);
663 | }
664 | 
665 | #[test]
666 | fn quick_check_nfd() {
667 |     test_bool("./tests/data/quick-nfd.txt",
668 |         |cp| cp.quick_check_nfd());
669 | }
670 | 
671 | #[test]
672 | fn quick_check_nfkc() {
673 |     test_int("./tests/data/qnfkc.txt",
674 |         |cp| cp.quick_check_nfkc() as u32);
675 | }
676 | 
677 | #[test]
678 | fn quick_check_nfkd() {
679 |     test_bool("./tests/data/quick-nfkd.txt",
680 |         |cp| cp.quick_check_nfkd());
681 | }
682 | 
683 | 
684 | 
685 | // segmentation
686 | 
687 | #[test]
688 | fn grapheme_cluster_break() {
689 |     test_int("./tests/data/gcb.txt",
690 |         |cp| cp.grapheme_cluster_break() as u32);
691 | }
692 | 
693 | #[test]
694 | fn is_grapheme_base() {
695 |     test_bool("./tests/data/graph-base.txt",
696 |         |cp| cp.is_grapheme_base());
697 | }
698 | 
699 | #[test]
700 | fn is_grapheme_extend() {
701 |     test_bool("./tests/data/graph-ext.txt",
702 |         |cp| cp.is_grapheme_extend());
703 | }
704 | 
705 | #[test]
706 | fn is_grapheme_extend_other() {
707 |     test_bool("./tests/data/graph-ext-other.txt",
708 |         |cp| cp.is_grapheme_extend_other());
709 | }
710 | 
711 | #[test]
712 | fn is_grapheme_link() {
713 |     test_bool("./tests/data/graph-link.txt",
714 |         |cp| cp.is_grapheme_link());
715 | }
716 | 
717 | #[test]
718 | fn linebreak() {
719 |     test_oint("./tests/data/linebreak.txt",
720 |         |cp| cp.linebreak_class().map(|c| c as u32));
721 | }
722 | 
723 | #[test]
724 | fn sentence_break() {
725 |     test_int("./tests/data/sbrk.txt",
726 |         |cp| cp.sentence_break() as u32);
727 | }
728 | 
729 | #[test]
730 | fn word_break() {
731 |     test_int("./tests/data/wbrk.txt",
732 |         |cp| cp.word_break() as u32);
733 | }


--------------------------------------------------------------------------------