├── .gitattributes ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── PATENTS ├── README.md ├── cases ├── cases.go ├── context.go ├── context_test.go ├── example_test.go ├── fold.go ├── fold_test.go ├── gen.go ├── gen_trieval.go ├── icu.go ├── icu_test.go ├── info.go ├── map.go ├── map_test.go ├── tables10.0.0.go ├── tables10.0.0_test.go ├── tables11.0.0.go ├── tables11.0.0_test.go ├── tables12.0.0.go ├── tables12.0.0_test.go ├── tables13.0.0.go ├── tables13.0.0_test.go ├── tables15.0.0.go ├── tables15.0.0_test.go ├── tables9.0.0.go ├── tables9.0.0_test.go └── trieval.go ├── cmd └── gotext │ ├── common.go │ ├── doc.go │ ├── examples │ ├── extract │ │ ├── catalog.go │ │ ├── locales │ │ │ ├── de │ │ │ │ ├── messages.gotext.json │ │ │ │ └── out.gotext.json │ │ │ ├── en-US │ │ │ │ ├── messages.gotext.json │ │ │ │ └── out.gotext.json │ │ │ └── zh │ │ │ │ ├── messages.gotext.json │ │ │ │ └── out.gotext.json │ │ └── main.go │ ├── extract_http │ │ ├── catalog_gen.go │ │ ├── locales │ │ │ ├── de │ │ │ │ └── out.gotext.json │ │ │ ├── en-US │ │ │ │ └── out.gotext.json │ │ │ ├── en │ │ │ │ └── out.gotext.json │ │ │ └── zh │ │ │ │ └── out.gotext.json │ │ ├── main.go │ │ └── pkg │ │ │ └── pkg.go │ └── rewrite │ │ ├── main.go │ │ └── printer.go │ ├── extract.go │ ├── generate.go │ ├── main.go │ ├── rewrite.go │ └── update.go ├── codereview.cfg ├── collate ├── build │ ├── builder.go │ ├── builder_test.go │ ├── colelem.go │ ├── colelem_test.go │ ├── contract.go │ ├── contract_test.go │ ├── order.go │ ├── order_test.go │ ├── table.go │ ├── trie.go │ └── trie_test.go ├── collate.go ├── collate_test.go ├── example_sort_test.go ├── examples_test.go ├── export_test.go ├── index.go ├── maketables.go ├── option.go ├── option_test.go ├── reg_test.go ├── sort.go ├── sort_test.go ├── table_test.go ├── tables.go └── tools │ └── colcmp │ ├── Makefile │ ├── chars.go │ ├── col.go │ ├── colcmp.go │ ├── darwin.go │ ├── gen.go │ └── icu.go ├── currency ├── common.go ├── currency.go ├── currency_test.go ├── example_test.go ├── format.go ├── format_test.go ├── gen.go ├── gen_common.go ├── query.go ├── query_test.go ├── tables.go └── tables_test.go ├── date ├── data_test.go ├── gen.go ├── gen_test.go └── tables.go ├── doc.go ├── encoding ├── charmap │ ├── charmap.go │ ├── charmap_test.go │ ├── maketables.go │ └── tables.go ├── encoding.go ├── encoding_test.go ├── example_test.go ├── htmlindex │ ├── gen.go │ ├── htmlindex.go │ ├── htmlindex_test.go │ ├── map.go │ └── tables.go ├── ianaindex │ ├── ascii.go │ ├── ascii_test.go │ ├── example_test.go │ ├── gen.go │ ├── ianaindex.go │ ├── ianaindex_test.go │ └── tables.go ├── internal │ ├── enctest │ │ └── enctest.go │ ├── identifier │ │ ├── gen.go │ │ ├── identifier.go │ │ └── mib.go │ └── internal.go ├── japanese │ ├── all.go │ ├── all_test.go │ ├── eucjp.go │ ├── iso2022jp.go │ ├── maketables.go │ ├── shiftjis.go │ └── tables.go ├── korean │ ├── all_test.go │ ├── euckr.go │ ├── maketables.go │ └── tables.go ├── simplifiedchinese │ ├── all.go │ ├── all_test.go │ ├── gbk.go │ ├── hzgb2312.go │ ├── maketables.go │ └── tables.go ├── testdata │ ├── candide-gb18030.txt │ ├── candide-utf-16le.txt │ ├── candide-utf-32be.txt │ ├── candide-utf-8.txt │ ├── candide-windows-1252.txt │ ├── rashomon-euc-jp.txt │ ├── rashomon-iso-2022-jp.txt │ ├── rashomon-shift-jis.txt │ ├── rashomon-utf-8.txt │ ├── sunzi-bingfa-gb-levels-1-and-2-hz-gb2312.txt │ ├── sunzi-bingfa-gb-levels-1-and-2-utf-8.txt │ ├── sunzi-bingfa-simplified-gbk.txt │ ├── sunzi-bingfa-simplified-utf-8.txt │ ├── sunzi-bingfa-traditional-big5.txt │ ├── sunzi-bingfa-traditional-utf-8.txt │ ├── unsu-joh-eun-nal-euc-kr.txt │ └── unsu-joh-eun-nal-utf-8.txt ├── traditionalchinese │ ├── all_test.go │ ├── big5.go │ ├── maketables.go │ └── tables.go └── unicode │ ├── override.go │ ├── unicode.go │ ├── unicode_test.go │ └── utf32 │ ├── utf32.go │ └── utf32_test.go ├── feature └── plural │ ├── common.go │ ├── data_test.go │ ├── example_test.go │ ├── gen.go │ ├── gen_common.go │ ├── message.go │ ├── message_test.go │ ├── plural.go │ ├── plural_test.go │ └── tables.go ├── gen.go ├── go.mod ├── go.sum ├── internal ├── catmsg │ ├── catmsg.go │ ├── catmsg_test.go │ ├── codec.go │ ├── varint.go │ └── varint_test.go ├── cldrtree │ ├── cldrtree.go │ ├── cldrtree_test.go │ ├── generate.go │ ├── option.go │ ├── testdata │ │ ├── test1 │ │ │ ├── common │ │ │ │ └── main │ │ │ │ │ └── root.xml │ │ │ └── output.go │ │ └── test2 │ │ │ ├── common │ │ │ └── main │ │ │ │ ├── en.xml │ │ │ │ ├── en_001.xml │ │ │ │ ├── en_GB.xml │ │ │ │ └── root.xml │ │ │ └── output.go │ ├── tree.go │ └── type.go ├── colltab │ ├── collate_test.go │ ├── collelem.go │ ├── collelem_test.go │ ├── colltab.go │ ├── colltab_test.go │ ├── contract.go │ ├── contract_test.go │ ├── iter.go │ ├── iter_test.go │ ├── numeric.go │ ├── numeric_test.go │ ├── table.go │ ├── trie.go │ ├── trie_test.go │ ├── weighter.go │ └── weighter_test.go ├── export │ ├── README │ ├── idna │ │ ├── common_test.go │ │ ├── conformance_test.go │ │ ├── example_test.go │ │ ├── gen.go │ │ ├── gen10.0.0_test.go │ │ ├── gen9.0.0_test.go │ │ ├── gen_common.go │ │ ├── gen_trieval.go │ │ ├── go118.go │ │ ├── idna10.0.0.go │ │ ├── idna10.0.0_test.go │ │ ├── idna9.0.0.go │ │ ├── idna9.0.0_test.go │ │ ├── idna_test.go │ │ ├── pre_go118.go │ │ ├── punycode.go │ │ ├── punycode_test.go │ │ ├── tables10.0.0.go │ │ ├── tables11.0.0.go │ │ ├── tables12.0.0.go │ │ ├── tables13.0.0.go │ │ ├── tables15.0.0.go │ │ ├── tables9.0.0.go │ │ ├── trie.go │ │ ├── trie12.0.0.go │ │ ├── trie13.0.0.go │ │ └── trieval.go │ └── unicode │ │ ├── doc.go │ │ ├── gen.go │ │ └── unicode_test.go ├── format │ ├── format.go │ ├── parser.go │ └── parser_test.go ├── gen │ ├── bitfield │ │ ├── bitfield.go │ │ ├── bitfield_test.go │ │ ├── gen1_test.go │ │ └── gen2_test.go │ ├── code.go │ └── gen.go ├── internal.go ├── internal_test.go ├── language │ ├── common.go │ ├── compact.go │ ├── compact │ │ ├── compact.go │ │ ├── gen.go │ │ ├── gen_index.go │ │ ├── gen_parents.go │ │ ├── gen_test.go │ │ ├── language.go │ │ ├── language_test.go │ │ ├── parents.go │ │ ├── parse_test.go │ │ ├── tables.go │ │ └── tags.go │ ├── compose.go │ ├── compose_test.go │ ├── coverage.go │ ├── gen.go │ ├── gen_common.go │ ├── language.go │ ├── language_test.go │ ├── lookup.go │ ├── lookup_test.go │ ├── match.go │ ├── match_test.go │ ├── parse.go │ ├── parse_test.go │ ├── tables.go │ └── tags.go ├── match.go ├── match_test.go ├── number │ ├── common.go │ ├── decimal.go │ ├── decimal_test.go │ ├── format.go │ ├── format_test.go │ ├── gen.go │ ├── gen_common.go │ ├── number.go │ ├── number_test.go │ ├── pattern.go │ ├── pattern_test.go │ ├── roundingmode_string.go │ ├── tables.go │ └── tables_test.go ├── stringset │ ├── set.go │ └── set_test.go ├── tag │ ├── tag.go │ └── tag_test.go ├── testtext │ ├── codesize.go │ ├── flag.go │ ├── gc.go │ ├── gccgo.go │ └── text.go ├── triegen │ ├── compact.go │ ├── data_test.go │ ├── example_compact_test.go │ ├── example_test.go │ ├── gen_test.go │ ├── print.go │ └── triegen.go ├── ucd │ ├── example_test.go │ ├── ucd.go │ └── ucd_test.go └── utf8internal │ └── utf8internal.go ├── language ├── coverage.go ├── coverage_test.go ├── display │ ├── dict.go │ ├── dict_test.go │ ├── display.go │ ├── display_test.go │ ├── examples_test.go │ ├── lookup.go │ ├── maketables.go │ └── tables.go ├── doc.go ├── examples_test.go ├── gen.go ├── httpexample_test.go ├── language.go ├── language_test.go ├── lookup_test.go ├── match.go ├── match_test.go ├── parse.go ├── parse_test.go ├── tables.go ├── tags.go └── testdata │ ├── CLDRLocaleMatcherTest.txt │ └── GoLocaleMatcherTest.txt ├── message ├── catalog.go ├── catalog │ ├── catalog.go │ ├── catalog_test.go │ ├── dict.go │ ├── go19.go │ └── gopre19.go ├── catalog_test.go ├── doc.go ├── examples_test.go ├── fmt_test.go ├── format.go ├── message.go ├── message_test.go ├── pipeline │ ├── extract.go │ ├── generate.go │ ├── go19_test.go │ ├── message.go │ ├── pipeline.go │ ├── pipeline_test.go │ ├── rewrite.go │ └── testdata │ │ ├── ssa │ │ ├── catalog_gen.go │ │ ├── extracted.gotext.json │ │ └── ssa.go │ │ └── test1 │ │ ├── catalog_gen.go.want │ │ ├── catalog_test.go │ │ ├── extracted.gotext.json.want │ │ ├── locales │ │ ├── de │ │ │ ├── messages.gotext.json │ │ │ └── out.gotext.json.want │ │ ├── en-US │ │ │ ├── messages.gotext.json │ │ │ └── out.gotext.json.want │ │ └── zh │ │ │ ├── messages.gotext.json │ │ │ └── out.gotext.json.want │ │ └── test1.go └── print.go ├── number ├── doc.go ├── examples_test.go ├── format.go ├── format_test.go ├── number.go ├── number_test.go └── option.go ├── runes ├── cond.go ├── cond_test.go ├── example_test.go ├── runes.go └── runes_test.go ├── search ├── index.go ├── pattern.go ├── pattern_test.go ├── search.go └── tables.go ├── secure ├── bidirule │ ├── bench_test.go │ ├── bidirule.go │ ├── bidirule10.0.0.go │ ├── bidirule10.0.0_test.go │ ├── bidirule9.0.0.go │ ├── bidirule9.0.0_test.go │ └── bidirule_test.go ├── doc.go └── precis │ ├── benchmark_test.go │ ├── class.go │ ├── class_test.go │ ├── context.go │ ├── doc.go │ ├── enforce10.0.0_test.go │ ├── enforce9.0.0_test.go │ ├── enforce_test.go │ ├── gen.go │ ├── gen_trieval.go │ ├── nickname.go │ ├── options.go │ ├── profile.go │ ├── profile_test.go │ ├── profiles.go │ ├── tables10.0.0.go │ ├── tables11.0.0.go │ ├── tables12.0.0.go │ ├── tables13.0.0.go │ ├── tables15.0.0.go │ ├── tables9.0.0.go │ ├── tables_test.go │ ├── transformer.go │ └── trieval.go ├── transform ├── examples_test.go ├── transform.go └── transform_test.go ├── unicode ├── bidi │ ├── bidi.go │ ├── bidi_test.go │ ├── bracket.go │ ├── core.go │ ├── core_test.go │ ├── gen.go │ ├── gen_ranges.go │ ├── gen_trieval.go │ ├── prop.go │ ├── ranges_test.go │ ├── tables10.0.0.go │ ├── tables11.0.0.go │ ├── tables12.0.0.go │ ├── tables13.0.0.go │ ├── tables15.0.0.go │ ├── tables9.0.0.go │ ├── tables_test.go │ └── trieval.go ├── cldr │ ├── base.go │ ├── cldr.go │ ├── cldr_test.go │ ├── collate.go │ ├── collate_test.go │ ├── data_test.go │ ├── decode.go │ ├── examples_test.go │ ├── makexml.go │ ├── resolve.go │ ├── resolve_test.go │ ├── slice.go │ ├── slice_test.go │ └── xml.go ├── doc.go ├── norm │ ├── composition.go │ ├── composition_test.go │ ├── data10.0.0_test.go │ ├── data11.0.0_test.go │ ├── data12.0.0_test.go │ ├── data13.0.0_test.go │ ├── data15.0.0_test.go │ ├── data9.0.0_test.go │ ├── example_iter_test.go │ ├── example_test.go │ ├── forminfo.go │ ├── forminfo_test.go │ ├── input.go │ ├── iter.go │ ├── iter_test.go │ ├── maketables.go │ ├── normalize.go │ ├── normalize_test.go │ ├── readwriter.go │ ├── readwriter_test.go │ ├── tables10.0.0.go │ ├── tables11.0.0.go │ ├── tables12.0.0.go │ ├── tables13.0.0.go │ ├── tables15.0.0.go │ ├── tables9.0.0.go │ ├── transform.go │ ├── transform_test.go │ ├── trie.go │ ├── triegen.go │ └── ucd_test.go ├── rangetable │ ├── gen.go │ ├── merge.go │ ├── merge_test.go │ ├── rangetable.go │ ├── rangetable_test.go │ ├── tables10.0.0.go │ ├── tables11.0.0.go │ ├── tables12.0.0.go │ ├── tables13.0.0.go │ ├── tables15.0.0.go │ └── tables9.0.0.go └── runenames │ ├── example_test.go │ ├── gen.go │ ├── runenames.go │ ├── runenames_test.go │ ├── tables10.0.0.go │ ├── tables11.0.0.go │ ├── tables12.0.0.go │ ├── tables13.0.0.go │ ├── tables15.0.0.go │ └── tables9.0.0.go └── width ├── common_test.go ├── example_test.go ├── gen.go ├── gen_common.go ├── gen_trieval.go ├── kind_string.go ├── runes_test.go ├── tables10.0.0.go ├── tables11.0.0.go ├── tables12.0.0.go ├── tables13.0.0.go ├── tables15.0.0.go ├── tables9.0.0.go ├── tables_test.go ├── transform.go ├── transform_test.go ├── trieval.go └── width.go /.gitattributes: -------------------------------------------------------------------------------- 1 | # Treat all files in this repo as binary, with no git magic updating 2 | # line endings. Windows users contributing to Go will need to use a 3 | # modern version of git and editors capable of LF line endings. 4 | # 5 | # We'll prevent accidental CRLF line endings from entering the repo 6 | # via the git-review gofmt checks. 7 | # 8 | # See golang.org/issue/9281 9 | 10 | * -text 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Add no patterns to .gitignore except for files generated by the build. 2 | last-change 3 | /DATA 4 | # This file is rather large and the tests really only need to be run 5 | # after generation. 6 | /unicode/norm/data_test.go -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Go 2 | 3 | Go is an open source project. 4 | 5 | It is the work of hundreds of contributors. We appreciate your help! 6 | 7 | ## Filing issues 8 | 9 | When [filing an issue](https://golang.org/issue/new), make sure to answer these five questions: 10 | 11 | 1. What version of Go are you using (`go version`)? 12 | 2. What operating system and processor architecture are you using? 13 | 3. What did you do? 14 | 4. What did you expect to see? 15 | 5. What did you see instead? 16 | 17 | General questions should go to the [golang-nuts mailing list](https://groups.google.com/group/golang-nuts) instead of the issue tracker. 18 | The gophers there will answer or ask you to file an issue if you've tripped over a bug. 19 | 20 | ## Contributing code 21 | 22 | Please read the [Contribution Guidelines](https://golang.org/doc/contribute.html) 23 | before sending patches. 24 | 25 | Unless otherwise noted, the Go source files are distributed under 26 | the BSD-style license found in the LICENSE file. 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2009 The Go Authors. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google LLC nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /PATENTS: -------------------------------------------------------------------------------- 1 | Additional IP Rights Grant (Patents) 2 | 3 | "This implementation" means the copyrightable works distributed by 4 | Google as part of the Go project. 5 | 6 | Google hereby grants to You a perpetual, worldwide, non-exclusive, 7 | no-charge, royalty-free, irrevocable (except as stated in this section) 8 | patent license to make, have made, use, offer to sell, sell, import, 9 | transfer and otherwise run, modify and propagate the contents of this 10 | implementation of Go, where such license applies only to those patent 11 | claims, both currently owned or controlled by Google and acquired in 12 | the future, licensable by Google that are necessarily infringed by this 13 | implementation of Go. This grant does not include claims that would be 14 | infringed only as a consequence of further modification of this 15 | implementation. If you or your agent or exclusive licensee institute or 16 | order or agree to the institution of patent litigation against any 17 | entity (including a cross-claim or counterclaim in a lawsuit) alleging 18 | that this implementation of Go or any code incorporated within this 19 | implementation of Go constitutes direct or contributory patent 20 | infringement, or inducement of patent infringement, then any patent 21 | rights granted to you under this License for this implementation of Go 22 | shall terminate as of the date such litigation is filed. 23 | -------------------------------------------------------------------------------- /cases/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package cases_test 6 | 7 | import ( 8 | "fmt" 9 | 10 | "golang.org/x/text/cases" 11 | "golang.org/x/text/language" 12 | ) 13 | 14 | func Example() { 15 | src := []string{ 16 | "hello world!", 17 | "i with dot", 18 | "'n ijsberg", 19 | "here comes O'Brian", 20 | } 21 | for _, c := range []cases.Caser{ 22 | cases.Lower(language.Und), 23 | cases.Upper(language.Turkish), 24 | cases.Title(language.Dutch), 25 | cases.Title(language.Und, cases.NoLower), 26 | } { 27 | fmt.Println() 28 | for _, s := range src { 29 | fmt.Println(c.String(s)) 30 | } 31 | } 32 | 33 | // Output: 34 | // hello world! 35 | // i with dot 36 | // 'n ijsberg 37 | // here comes o'brian 38 | // 39 | // HELLO WORLD! 40 | // İ WİTH DOT 41 | // 'N İJSBERG 42 | // HERE COMES O'BRİAN 43 | // 44 | // Hello World! 45 | // I With Dot 46 | // 'n IJsberg 47 | // Here Comes O'brian 48 | // 49 | // Hello World! 50 | // I With Dot 51 | // 'N Ijsberg 52 | // Here Comes O'Brian 53 | } 54 | -------------------------------------------------------------------------------- /cases/fold.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package cases 6 | 7 | import "golang.org/x/text/transform" 8 | 9 | type caseFolder struct{ transform.NopResetter } 10 | 11 | // caseFolder implements the Transformer interface for doing case folding. 12 | func (t *caseFolder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { 13 | c := context{dst: dst, src: src, atEOF: atEOF} 14 | for c.next() { 15 | foldFull(&c) 16 | c.checkpoint() 17 | } 18 | return c.ret() 19 | } 20 | 21 | func (t *caseFolder) Span(src []byte, atEOF bool) (n int, err error) { 22 | c := context{src: src, atEOF: atEOF} 23 | for c.next() && isFoldFull(&c) { 24 | c.checkpoint() 25 | } 26 | return c.retSpan() 27 | } 28 | 29 | func makeFold(o options) transform.SpanningTransformer { 30 | // TODO: Special case folding, through option Language, Special/Turkic, or 31 | // both. 32 | // TODO: Implement Compact options. 33 | return &caseFolder{} 34 | } 35 | -------------------------------------------------------------------------------- /cases/fold_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package cases 6 | 7 | import ( 8 | "testing" 9 | 10 | "golang.org/x/text/internal/testtext" 11 | ) 12 | 13 | var foldTestCases = []string{ 14 | "βß\u13f8", // "βssᏰ" 15 | "ab\u13fc\uab7aꭰ", // abᏴᎪᎠ 16 | "affifflast", // affifflast 17 | "Iİiı\u0345", // ii̇iıι 18 | "µµΜΜςσΣΣ", // μμμμσσσσ 19 | } 20 | 21 | func TestFold(t *testing.T) { 22 | for _, tc := range foldTestCases { 23 | testEntry := func(name string, c Caser, m func(r rune) string) { 24 | want := "" 25 | for _, r := range tc { 26 | want += m(r) 27 | } 28 | if got := c.String(tc); got != want { 29 | t.Errorf("%s(%s) = %+q; want %+q", name, tc, got, want) 30 | } 31 | dst := make([]byte, 256) // big enough to hold any result 32 | src := []byte(tc) 33 | v := testtext.AllocsPerRun(20, func() { 34 | c.Transform(dst, src, true) 35 | }) 36 | if v > 0 { 37 | t.Errorf("%s(%s): number of allocs was %f; want 0", name, tc, v) 38 | } 39 | } 40 | testEntry("FullFold", Fold(), func(r rune) string { 41 | return runeFoldData(r).full 42 | }) 43 | // TODO: 44 | // testEntry("SimpleFold", Fold(Compact), func(r rune) string { 45 | // return runeFoldData(r).simple 46 | // }) 47 | // testEntry("SpecialFold", Fold(Turkic), func(r rune) string { 48 | // return runeFoldData(r).special 49 | // }) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /cases/icu.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build icu 6 | 7 | package cases 8 | 9 | // Ideally these functions would be defined in a test file, but go test doesn't 10 | // allow CGO in tests. The build tag should ensure either way that these 11 | // functions will not end up in the package. 12 | 13 | // TODO: Ensure that the correct ICU version is set. 14 | 15 | /* 16 | #cgo LDFLAGS: -licui18n.57 -licuuc.57 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | */ 23 | import "C" 24 | 25 | import "unsafe" 26 | 27 | func doICU(tag, caser, input string) string { 28 | err := C.UErrorCode(0) 29 | loc := C.CString(tag) 30 | cm := C.ucasemap_open(loc, C.uint32_t(0), &err) 31 | 32 | buf := make([]byte, len(input)*4) 33 | dst := (*C.char)(unsafe.Pointer(&buf[0])) 34 | src := C.CString(input) 35 | 36 | cn := C.int32_t(0) 37 | 38 | switch caser { 39 | case "fold": 40 | cn = C.ucasemap_utf8FoldCase(cm, 41 | dst, C.int32_t(len(buf)), 42 | src, C.int32_t(len(input)), 43 | &err) 44 | case "lower": 45 | cn = C.ucasemap_utf8ToLower(cm, 46 | dst, C.int32_t(len(buf)), 47 | src, C.int32_t(len(input)), 48 | &err) 49 | case "upper": 50 | cn = C.ucasemap_utf8ToUpper(cm, 51 | dst, C.int32_t(len(buf)), 52 | src, C.int32_t(len(input)), 53 | &err) 54 | case "title": 55 | cn = C.ucasemap_utf8ToTitle(cm, 56 | dst, C.int32_t(len(buf)), 57 | src, C.int32_t(len(input)), 58 | &err) 59 | } 60 | return string(buf[:cn]) 61 | } 62 | -------------------------------------------------------------------------------- /cmd/gotext/common.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "go/build" 10 | "go/parser" 11 | 12 | "golang.org/x/tools/go/loader" 13 | ) 14 | 15 | const ( 16 | extractFile = "extracted.gotext.json" 17 | outFile = "out.gotext.json" 18 | gotextSuffix = ".gotext.json" 19 | ) 20 | 21 | // NOTE: The command line tool already prefixes with "gotext:". 22 | var ( 23 | wrap = func(err error, msg string) error { 24 | if err == nil { 25 | return nil 26 | } 27 | return fmt.Errorf("%s: %v", msg, err) 28 | } 29 | errorf = fmt.Errorf 30 | ) 31 | 32 | // TODO: still used. Remove when possible. 33 | func loadPackages(conf *loader.Config, args []string) (*loader.Program, error) { 34 | if len(args) == 0 { 35 | args = []string{"."} 36 | } 37 | 38 | conf.Build = &build.Default 39 | conf.ParserMode = parser.ParseComments 40 | 41 | // Use the initial packages from the command line. 42 | args, err := conf.FromArgs(args, false) 43 | if err != nil { 44 | return nil, wrap(err, "loading packages failed") 45 | } 46 | 47 | // Load, parse and type-check the whole program. 48 | return conf.Load() 49 | } 50 | -------------------------------------------------------------------------------- /cmd/gotext/doc.go: -------------------------------------------------------------------------------- 1 | // Code generated by go generate. DO NOT EDIT. 2 | 3 | // gotext is a tool for managing text in Go source code. 4 | // 5 | // Usage: 6 | // 7 | // gotext command [arguments] 8 | // 9 | // The commands are: 10 | // 11 | // update merge translations and generate catalog 12 | // extract extracts strings to be translated from code 13 | // rewrite rewrites fmt functions to use a message Printer 14 | // generate generates code to insert translated messages 15 | // 16 | // Use "gotext help [command]" for more information about a command. 17 | // 18 | // Additional help topics: 19 | // 20 | // Use "gotext help [topic]" for more information about that topic. 21 | // 22 | // # Merge translations and generate catalog 23 | // 24 | // Usage: 25 | // 26 | // gotext update * [-out ] 27 | // 28 | // # Extracts strings to be translated from code 29 | // 30 | // Usage: 31 | // 32 | // gotext extract * 33 | // 34 | // # Rewrites fmt functions to use a message Printer 35 | // 36 | // Usage: 37 | // 38 | // gotext rewrite 39 | // 40 | // rewrite is typically done once for a project. It rewrites all usages of 41 | // fmt to use x/text's message package whenever a message.Printer is in scope. 42 | // It rewrites Print and Println calls with constant strings to the equivalent 43 | // using Printf to allow translators to reorder arguments. 44 | // 45 | // # Generates code to insert translated messages 46 | // 47 | // Usage: 48 | // 49 | // gotext generate [-out ] 50 | package main 51 | -------------------------------------------------------------------------------- /cmd/gotext/examples/extract/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | //go:generate gotext update -out catalog.go 8 | 9 | import ( 10 | "golang.org/x/text/language" 11 | "golang.org/x/text/message" 12 | ) 13 | 14 | func main() { 15 | p := message.NewPrinter(language.English) 16 | 17 | p.Print("Hello world!\n") 18 | 19 | p.Println("Hello", "world!") 20 | 21 | person := "Sheila" 22 | place := "Zürich" 23 | 24 | p.Print("Hello ", person, " in ", place, "!\n") 25 | 26 | // Greet everyone. 27 | p.Printf("Hello world!\n") 28 | 29 | city := "Amsterdam" 30 | // Greet a city. 31 | p.Printf("Hello %s!\n", city) 32 | 33 | town := "Amsterdam" 34 | // Greet a town. 35 | p.Printf("Hello %s!\n", 36 | town, // Town 37 | ) 38 | 39 | // Person visiting a place. 40 | p.Printf("%s is visiting %s!\n", 41 | person, // The person of matter. 42 | place, // Place the person is visiting. 43 | ) 44 | 45 | pp := struct { 46 | Person string // The person of matter. // TODO: get this comment. 47 | Place string 48 | extra int 49 | }{ 50 | person, place, 4, 51 | } 52 | 53 | // extract will drop this comment in favor of the one below. 54 | // argument is added as a placeholder. 55 | p.Printf("%[1]s is visiting %[3]s!\n", // Person visiting a place. 56 | pp.Person, 57 | pp.extra, 58 | pp.Place, // Place the person is visiting. 59 | ) 60 | 61 | // Numeric literal 62 | p.Printf("%d files remaining!", 2) 63 | 64 | const n = 2 65 | 66 | // Numeric var 67 | p.Printf("%d more files remaining!", n) 68 | 69 | // Infer better names from type names. 70 | type referralCode int 71 | 72 | const c = referralCode(5) 73 | p.Printf("Use the following code for your discount: %d\n", c) 74 | 75 | // Using a constant for a message will cause the constant name to be 76 | // added as an identifier, allowing for stable message identifiers. 77 | 78 | // Explain that a device is out of order. 79 | const msgOutOfOrder = "%s is out of order!" // FOO 80 | const device = "Soda machine" 81 | p.Printf(msgOutOfOrder, device) 82 | 83 | // Double arguments. 84 | miles := 1.2345 85 | p.Printf("%.2[1]f miles traveled (%[1]f)", miles) 86 | } 87 | -------------------------------------------------------------------------------- /cmd/gotext/examples/extract_http/catalog_gen.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | package main 4 | 5 | import ( 6 | "golang.org/x/text/language" 7 | "golang.org/x/text/message" 8 | "golang.org/x/text/message/catalog" 9 | ) 10 | 11 | type dictionary struct { 12 | index []uint32 13 | data string 14 | } 15 | 16 | func (d *dictionary) Lookup(key string) (data string, ok bool) { 17 | p, ok := messageKeyToIndex[key] 18 | if !ok { 19 | return "", false 20 | } 21 | start, end := d.index[p], d.index[p+1] 22 | if start == end { 23 | return "", false 24 | } 25 | return d.data[start:end], true 26 | } 27 | 28 | func init() { 29 | dict := map[string]catalog.Dictionary{ 30 | "en": &dictionary{index: enIndex, data: enData}, 31 | "zh": &dictionary{index: zhIndex, data: zhData}, 32 | } 33 | fallback := language.MustParse("en") 34 | cat, err := catalog.NewFromMap(dict, catalog.Fallback(fallback)) 35 | if err != nil { 36 | panic(err) 37 | } 38 | message.DefaultCatalog = cat 39 | } 40 | 41 | var messageKeyToIndex = map[string]int{ 42 | "Do you like your browser (%s)?\n": 1, 43 | "Hello %s!\n": 0, 44 | } 45 | 46 | var enIndex = []uint32{ // 3 elements 47 | 0x00000000, 0x00000012, 0x00000039, 48 | } // Size: 36 bytes 49 | 50 | const enData string = "" + // Size: 57 bytes 51 | "\x04\x00\x01\x0a\x0d\x02Hello %[1]s!\x04\x00\x01\x0a\x22\x02Do you like " + 52 | "your browser (%[1]s)?" 53 | 54 | var zhIndex = []uint32{ // 3 elements 55 | 0x00000000, 0x00000000, 0x00000000, 56 | } // Size: 36 bytes 57 | 58 | const zhData string = "" 59 | 60 | // Total table size 129 bytes (0KiB); checksum: 9C146C82 61 | -------------------------------------------------------------------------------- /cmd/gotext/examples/extract_http/locales/de/out.gotext.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "de", 3 | "messages": [ 4 | { 5 | "id": "Hello {From}!", 6 | "key": "Hello %s!\n", 7 | "message": "Hello {From}!", 8 | "translation": "", 9 | "placeholders": [ 10 | { 11 | "id": "From", 12 | "string": "%[1]s", 13 | "type": "string", 14 | "underlyingType": "string", 15 | "argNum": 1, 16 | "expr": "r.Header.Get(\"From\")" 17 | } 18 | ], 19 | "position": "golang.org/x/text/cmd/gotext/examples/extract_http/pkg/pkg.go:22:11" 20 | }, 21 | { 22 | "id": "Do you like your browser ({User_Agent})?", 23 | "key": "Do you like your browser (%s)?\n", 24 | "message": "Do you like your browser ({User_Agent})?", 25 | "translation": "", 26 | "placeholders": [ 27 | { 28 | "id": "User_Agent", 29 | "string": "%[1]s", 30 | "type": "string", 31 | "underlyingType": "string", 32 | "argNum": 1, 33 | "expr": "r.Header.Get(\"User-Agent\")" 34 | } 35 | ], 36 | "position": "golang.org/x/text/cmd/gotext/examples/extract_http/pkg/pkg.go:24:11" 37 | } 38 | ] 39 | } -------------------------------------------------------------------------------- /cmd/gotext/examples/extract_http/locales/en-US/out.gotext.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "en-US", 3 | "messages": [ 4 | { 5 | "id": "Hello {From}!", 6 | "key": "Hello %s!\n", 7 | "message": "Hello {From}!", 8 | "translation": "", 9 | "placeholders": [ 10 | { 11 | "id": "From", 12 | "string": "%[1]s", 13 | "type": "string", 14 | "underlyingType": "string", 15 | "argNum": 1, 16 | "expr": "r.Header.Get(\"From\")" 17 | } 18 | ], 19 | "position": "golang.org/x/text/cmd/gotext/examples/extract_http/pkg/pkg.go:22:11" 20 | }, 21 | { 22 | "id": "Do you like your browser ({User_Agent})?", 23 | "key": "Do you like your browser (%s)?\n", 24 | "message": "Do you like your browser ({User_Agent})?", 25 | "translation": "", 26 | "placeholders": [ 27 | { 28 | "id": "User_Agent", 29 | "string": "%[1]s", 30 | "type": "string", 31 | "underlyingType": "string", 32 | "argNum": 1, 33 | "expr": "r.Header.Get(\"User-Agent\")" 34 | } 35 | ], 36 | "position": "golang.org/x/text/cmd/gotext/examples/extract_http/pkg/pkg.go:24:11" 37 | } 38 | ] 39 | } -------------------------------------------------------------------------------- /cmd/gotext/examples/extract_http/locales/en/out.gotext.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "en", 3 | "messages": [ 4 | { 5 | "id": "Hello {From}!", 6 | "message": "Hello {From}!", 7 | "translation": "Hello {From}!", 8 | "translatorComment": "Copied from source.", 9 | "placeholders": [ 10 | { 11 | "id": "From", 12 | "string": "%[1]s", 13 | "type": "string", 14 | "underlyingType": "string", 15 | "argNum": 1, 16 | "expr": "r.Header.Get(\"From\")" 17 | } 18 | ], 19 | "fuzzy": true 20 | }, 21 | { 22 | "id": "Do you like your browser ({User_Agent})?", 23 | "message": "Do you like your browser ({User_Agent})?", 24 | "translation": "Do you like your browser ({User_Agent})?", 25 | "translatorComment": "Copied from source.", 26 | "placeholders": [ 27 | { 28 | "id": "User_Agent", 29 | "string": "%[1]s", 30 | "type": "string", 31 | "underlyingType": "string", 32 | "argNum": 1, 33 | "expr": "r.Header.Get(\"User-Agent\")" 34 | } 35 | ], 36 | "fuzzy": true 37 | } 38 | ] 39 | } -------------------------------------------------------------------------------- /cmd/gotext/examples/extract_http/locales/zh/out.gotext.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "zh", 3 | "messages": [ 4 | { 5 | "id": "Hello {From}!", 6 | "message": "Hello {From}!", 7 | "translation": "", 8 | "placeholders": [ 9 | { 10 | "id": "From", 11 | "string": "%[1]s", 12 | "type": "string", 13 | "underlyingType": "string", 14 | "argNum": 1, 15 | "expr": "r.Header.Get(\"From\")" 16 | } 17 | ] 18 | }, 19 | { 20 | "id": "Do you like your browser ({User_Agent})?", 21 | "message": "Do you like your browser ({User_Agent})?", 22 | "translation": "", 23 | "placeholders": [ 24 | { 25 | "id": "User_Agent", 26 | "string": "%[1]s", 27 | "type": "string", 28 | "underlyingType": "string", 29 | "argNum": 1, 30 | "expr": "r.Header.Get(\"User-Agent\")" 31 | } 32 | ] 33 | } 34 | ] 35 | } -------------------------------------------------------------------------------- /cmd/gotext/examples/extract_http/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | //go:generate gotext -srclang=en update -out=catalog_gen.go -lang=en,zh 8 | 9 | import ( 10 | "net/http" 11 | 12 | "golang.org/x/text/cmd/gotext/examples/extract_http/pkg" 13 | ) 14 | 15 | func main() { 16 | http.Handle("/generize", http.HandlerFunc(pkg.Generize)) 17 | } 18 | -------------------------------------------------------------------------------- /cmd/gotext/examples/extract_http/pkg/pkg.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package pkg 6 | 7 | import ( 8 | "net/http" 9 | 10 | "golang.org/x/text/language" 11 | "golang.org/x/text/message" 12 | ) 13 | 14 | var matcher = language.NewMatcher(message.DefaultCatalog.Languages()) 15 | 16 | func Generize(w http.ResponseWriter, r *http.Request) { 17 | lang, _ := r.Cookie("lang") 18 | accept := r.Header.Get("Accept-Language") 19 | tag := message.MatchLanguage(lang.String(), accept) 20 | p := message.NewPrinter(tag) 21 | 22 | p.Fprintf(w, "Hello %s!\n", r.Header.Get("From")) 23 | 24 | p.Fprintf(w, "Do you like your browser (%s)?\n", r.Header.Get("User-Agent")) 25 | } 26 | -------------------------------------------------------------------------------- /cmd/gotext/examples/rewrite/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | 10 | "golang.org/x/text/language" 11 | "golang.org/x/text/message" 12 | ) 13 | 14 | func main() { 15 | var nPizzas = 4 16 | // The following call gets replaced by a call to the globally 17 | // defined printer. 18 | fmt.Println("We ate", nPizzas, "pizzas.") 19 | 20 | p := message.NewPrinter(language.English) 21 | 22 | // Prevent build failure, although it is okay for gotext. 23 | p.Println(1024) 24 | 25 | // Replaced by a call to p. 26 | fmt.Println("Example punctuation:", "$%^&!") 27 | 28 | { 29 | q := message.NewPrinter(language.French) 30 | 31 | const leaveAnIdentBe = "Don't expand me." 32 | fmt.Print(leaveAnIdentBe) 33 | q.Println() // Prevent build failure, although it is okay for gotext. 34 | } 35 | 36 | fmt.Printf("Hello %s\n", "City") 37 | } 38 | -------------------------------------------------------------------------------- /cmd/gotext/examples/rewrite/printer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | package main 8 | 9 | import ( 10 | "golang.org/x/text/language" 11 | "golang.org/x/text/message" 12 | ) 13 | 14 | // The printer defined here will be picked up by the first print statement 15 | // in main.go. 16 | var printer = message.NewPrinter(language.English) 17 | -------------------------------------------------------------------------------- /cmd/gotext/extract.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "golang.org/x/text/message/pipeline" 9 | ) 10 | 11 | // TODO: 12 | // - merge information into existing files 13 | // - handle different file formats (PO, XLIFF) 14 | // - handle features (gender, plural) 15 | // - message rewriting 16 | 17 | var cmdExtract = &Command{ 18 | Init: initExtract, 19 | Run: runExtract, 20 | UsageLine: "extract *", 21 | Short: "extracts strings to be translated from code", 22 | } 23 | 24 | func initExtract(cmd *Command) { 25 | lang = cmd.Flag.String("lang", "en-US", "comma-separated list of languages to process") 26 | } 27 | 28 | func runExtract(cmd *Command, config *pipeline.Config, args []string) error { 29 | config.Packages = args 30 | state, err := pipeline.Extract(config) 31 | if err != nil { 32 | return wrap(err, "extract failed") 33 | } 34 | if err := state.Import(); err != nil { 35 | return wrap(err, "import failed") 36 | } 37 | if err := state.Merge(); err != nil { 38 | return wrap(err, "merge failed") 39 | } 40 | return wrap(state.Export(), "export failed") 41 | } 42 | -------------------------------------------------------------------------------- /cmd/gotext/generate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "golang.org/x/text/message/pipeline" 9 | ) 10 | 11 | var cmdGenerate = &Command{ 12 | Init: initGenerate, 13 | Run: runGenerate, 14 | UsageLine: "generate [-out ]", 15 | Short: "generates code to insert translated messages", 16 | } 17 | 18 | func initGenerate(cmd *Command) { 19 | out = cmd.Flag.String("out", "", "output file to write to") 20 | } 21 | 22 | func runGenerate(cmd *Command, config *pipeline.Config, args []string) error { 23 | config.Packages = args 24 | s, err := pipeline.Extract(config) 25 | if err != nil { 26 | return wrap(err, "extraction failed") 27 | } 28 | if err := s.Import(); err != nil { 29 | return wrap(err, "import failed") 30 | } 31 | if err := s.Merge(); err != nil { 32 | return wrap(err, "merge failed") 33 | } 34 | return wrap(s.Generate(), "generation failed") 35 | } 36 | -------------------------------------------------------------------------------- /cmd/gotext/rewrite.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "io" 9 | "os" 10 | 11 | "golang.org/x/text/message/pipeline" 12 | ) 13 | 14 | const printerType = "golang.org/x/text/message.Printer" 15 | 16 | // TODO: 17 | // - merge information into existing files 18 | // - handle different file formats (PO, XLIFF) 19 | // - handle features (gender, plural) 20 | // - message rewriting 21 | 22 | var cmdRewrite = &Command{ 23 | Init: initRewrite, 24 | Run: runRewrite, 25 | UsageLine: "rewrite ", 26 | Short: "rewrites fmt functions to use a message Printer", 27 | Long: ` 28 | rewrite is typically done once for a project. It rewrites all usages of 29 | fmt to use x/text's message package whenever a message.Printer is in scope. 30 | It rewrites Print and Println calls with constant strings to the equivalent 31 | using Printf to allow translators to reorder arguments. 32 | `, 33 | } 34 | 35 | func initRewrite(cmd *Command) { 36 | overwrite = cmd.Flag.Bool("w", false, "write files in place") 37 | } 38 | 39 | func runRewrite(cmd *Command, _ *pipeline.Config, args []string) error { 40 | var w io.Writer 41 | if !*overwrite { 42 | w = os.Stdout 43 | } 44 | pkg := "." 45 | switch len(args) { 46 | case 0: 47 | case 1: 48 | pkg = args[0] 49 | default: 50 | return errorf("can only specify at most one package") 51 | } 52 | return pipeline.Rewrite(w, pkg) 53 | } 54 | -------------------------------------------------------------------------------- /cmd/gotext/update.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "golang.org/x/text/message/pipeline" 9 | ) 10 | 11 | // TODO: 12 | // - merge information into existing files 13 | // - handle different file formats (PO, XLIFF) 14 | // - handle features (gender, plural) 15 | // - message rewriting 16 | 17 | var cmdUpdate = &Command{ 18 | Init: initUpdate, 19 | Run: runUpdate, 20 | UsageLine: "update * [-out ]", 21 | Short: "merge translations and generate catalog", 22 | } 23 | 24 | func initUpdate(cmd *Command) { 25 | lang = cmd.Flag.String("lang", "en-US", "comma-separated list of languages to process") 26 | out = cmd.Flag.String("out", "", "output file to write to") 27 | } 28 | 29 | func runUpdate(cmd *Command, config *pipeline.Config, args []string) error { 30 | config.Packages = args 31 | state, err := pipeline.Extract(config) 32 | if err != nil { 33 | return wrap(err, "extract failed") 34 | } 35 | if err := state.Import(); err != nil { 36 | return wrap(err, "import failed") 37 | } 38 | if err := state.Merge(); err != nil { 39 | return wrap(err, "merge failed") 40 | } 41 | if err := state.Export(); err != nil { 42 | return wrap(err, "export failed") 43 | } 44 | if *out != "" { 45 | return wrap(state.Generate(), "generation failed") 46 | } 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /codereview.cfg: -------------------------------------------------------------------------------- 1 | issuerepo: golang/go 2 | -------------------------------------------------------------------------------- /collate/build/table.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package build 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | "reflect" 11 | 12 | "golang.org/x/text/internal/colltab" 13 | ) 14 | 15 | // table is an intermediate structure that roughly resembles the table in collate. 16 | type table struct { 17 | colltab.Table 18 | trie trie 19 | root *trieHandle 20 | } 21 | 22 | // print writes the table as Go compilable code to w. It prefixes the 23 | // variable names with name. It returns the number of bytes written 24 | // and the size of the resulting table. 25 | func (t *table) fprint(w io.Writer, name string) (n, size int, err error) { 26 | update := func(nn, sz int, e error) { 27 | n += nn 28 | if err == nil { 29 | err = e 30 | } 31 | size += sz 32 | } 33 | // Write arrays needed for the structure. 34 | update(printColElems(w, t.ExpandElem, name+"ExpandElem")) 35 | update(printColElems(w, t.ContractElem, name+"ContractElem")) 36 | update(t.trie.printArrays(w, name)) 37 | update(printArray(t.ContractTries, w, name)) 38 | 39 | nn, e := fmt.Fprintf(w, "// Total size of %sTable is %d bytes\n", name, size) 40 | update(nn, 0, e) 41 | return 42 | } 43 | 44 | func (t *table) fprintIndex(w io.Writer, h *trieHandle, id string) (n int, err error) { 45 | p := func(f string, a ...interface{}) { 46 | nn, e := fmt.Fprintf(w, f, a...) 47 | n += nn 48 | if err == nil { 49 | err = e 50 | } 51 | } 52 | p("\t{ // %s\n", id) 53 | p("\t\tlookupOffset: 0x%x,\n", h.lookupStart) 54 | p("\t\tvaluesOffset: 0x%x,\n", h.valueStart) 55 | p("\t},\n") 56 | return 57 | } 58 | 59 | func printColElems(w io.Writer, a []uint32, name string) (n, sz int, err error) { 60 | p := func(f string, a ...interface{}) { 61 | nn, e := fmt.Fprintf(w, f, a...) 62 | n += nn 63 | if err == nil { 64 | err = e 65 | } 66 | } 67 | sz = len(a) * int(reflect.TypeOf(uint32(0)).Size()) 68 | p("// %s: %d entries, %d bytes\n", name, len(a), sz) 69 | p("var %s = [%d]uint32 {", name, len(a)) 70 | for i, c := range a { 71 | switch { 72 | case i%64 == 0: 73 | p("\n\t// Block %d, offset 0x%x\n", i/64, i) 74 | case (i%64)%6 == 0: 75 | p("\n\t") 76 | } 77 | p("0x%.8X, ", c) 78 | } 79 | p("\n}\n\n") 80 | return 81 | } 82 | -------------------------------------------------------------------------------- /collate/example_sort_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package collate_test 6 | 7 | import ( 8 | "fmt" 9 | 10 | "golang.org/x/text/collate" 11 | "golang.org/x/text/language" 12 | ) 13 | 14 | type book struct { 15 | title string 16 | } 17 | 18 | type bookcase struct { 19 | books []book 20 | } 21 | 22 | func (bc bookcase) Len() int { 23 | return len(bc.books) 24 | } 25 | 26 | func (bc bookcase) Swap(i, j int) { 27 | temp := bc.books[i] 28 | bc.books[i] = bc.books[j] 29 | bc.books[j] = temp 30 | } 31 | 32 | func (bc bookcase) Bytes(i int) []byte { 33 | // returns the bytes of text at index i 34 | return []byte(bc.books[i].title) 35 | } 36 | 37 | func ExampleCollator_Sort() { 38 | bc := bookcase{ 39 | books: []book{ 40 | {title: "If Cats Disappeared from the World"}, 41 | {title: "The Guest Cat"}, 42 | {title: "Catwings"}, 43 | }, 44 | } 45 | 46 | cc := collate.New(language.English) 47 | cc.Sort(bc) 48 | 49 | for _, b := range bc.books { 50 | fmt.Println(b.title) 51 | } 52 | // Output: 53 | // Catwings 54 | // If Cats Disappeared from the World 55 | // The Guest Cat 56 | } 57 | -------------------------------------------------------------------------------- /collate/examples_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package collate_test 6 | 7 | import ( 8 | "fmt" 9 | 10 | "golang.org/x/text/collate" 11 | "golang.org/x/text/language" 12 | ) 13 | 14 | func ExampleNew() { 15 | letters := []string{"ä", "å", "ö", "o", "a"} 16 | 17 | ec := collate.New(language.English) 18 | ec.SortStrings(letters) 19 | fmt.Printf("English Sorting: %v\n", letters) 20 | 21 | sc := collate.New(language.Swedish) 22 | sc.SortStrings(letters) 23 | fmt.Printf("Swedish Sorting: %v\n", letters) 24 | 25 | numbers := []string{"0", "11", "01", "2", "3", "23"} 26 | 27 | ec.SortStrings(numbers) 28 | fmt.Printf("Alphabetic Sorting: %v\n", numbers) 29 | 30 | nc := collate.New(language.English, collate.Numeric) 31 | nc.SortStrings(numbers) 32 | fmt.Printf("Numeric Sorting: %v\n", numbers) 33 | // Output: 34 | // English Sorting: [a å ä o ö] 35 | // Swedish Sorting: [a o å ä ö] 36 | // Alphabetic Sorting: [0 01 11 2 23 3] 37 | // Numeric Sorting: [0 01 2 3 11 23] 38 | } 39 | 40 | func ExampleCollator_SortStrings() { 41 | c := collate.New(language.English) 42 | words := []string{"meow", "woof", "bark", "moo"} 43 | c.SortStrings(words) 44 | fmt.Println(words) 45 | // Output: 46 | // [bark meow moo woof] 47 | } 48 | 49 | func ExampleCollator_CompareString() { 50 | c := collate.New(language.English) 51 | r := c.CompareString("meow", "woof") 52 | fmt.Println(r) 53 | 54 | r = c.CompareString("woof", "meow") 55 | fmt.Println(r) 56 | 57 | r = c.CompareString("meow", "meow") 58 | fmt.Println(r) 59 | // Output: 60 | // -1 61 | // 1 62 | // 0 63 | } 64 | 65 | func ExampleCollator_Compare() { 66 | c := collate.New(language.English) 67 | r := c.Compare([]byte("meow"), []byte("woof")) 68 | fmt.Println(r) 69 | 70 | r = c.Compare([]byte("woof"), []byte("meow")) 71 | fmt.Println(r) 72 | 73 | r = c.Compare([]byte("meow"), []byte("meow")) 74 | fmt.Println(r) 75 | // Output: 76 | // -1 77 | // 1 78 | // 0 79 | } 80 | -------------------------------------------------------------------------------- /collate/export_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package collate 6 | 7 | // Export for testing. 8 | // TODO: no longer necessary. Remove at some point. 9 | 10 | import ( 11 | "fmt" 12 | 13 | "golang.org/x/text/internal/colltab" 14 | ) 15 | 16 | const ( 17 | defaultSecondary = 0x20 18 | defaultTertiary = 0x2 19 | ) 20 | 21 | type Weights struct { 22 | Primary, Secondary, Tertiary, Quaternary int 23 | } 24 | 25 | func W(ce ...int) Weights { 26 | w := Weights{ce[0], defaultSecondary, defaultTertiary, 0} 27 | if len(ce) > 1 { 28 | w.Secondary = ce[1] 29 | } 30 | if len(ce) > 2 { 31 | w.Tertiary = ce[2] 32 | } 33 | if len(ce) > 3 { 34 | w.Quaternary = ce[3] 35 | } 36 | return w 37 | } 38 | func (w Weights) String() string { 39 | return fmt.Sprintf("[%X.%X.%X.%X]", w.Primary, w.Secondary, w.Tertiary, w.Quaternary) 40 | } 41 | 42 | func convertFromWeights(ws []Weights) []colltab.Elem { 43 | out := make([]colltab.Elem, len(ws)) 44 | for i, w := range ws { 45 | out[i], _ = colltab.MakeElem(w.Primary, w.Secondary, w.Tertiary, 0) 46 | if out[i] == colltab.Ignore && w.Quaternary > 0 { 47 | out[i] = colltab.MakeQuaternary(w.Quaternary) 48 | } 49 | } 50 | return out 51 | } 52 | -------------------------------------------------------------------------------- /collate/index.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package collate 6 | 7 | import "golang.org/x/text/internal/colltab" 8 | 9 | const blockSize = 64 10 | 11 | func getTable(t tableIndex) *colltab.Table { 12 | return &colltab.Table{ 13 | Index: colltab.Trie{ 14 | Index0: mainLookup[:][blockSize*t.lookupOffset:], 15 | Values0: mainValues[:][blockSize*t.valuesOffset:], 16 | Index: mainLookup[:], 17 | Values: mainValues[:], 18 | }, 19 | ExpandElem: mainExpandElem[:], 20 | ContractTries: colltab.ContractTrieSet(mainCTEntries[:]), 21 | ContractElem: mainContractElem[:], 22 | MaxContractLen: 18, 23 | VariableTop: varTop, 24 | } 25 | } 26 | 27 | // tableIndex holds information for constructing a table 28 | // for a certain locale based on the main table. 29 | type tableIndex struct { 30 | lookupOffset uint32 31 | valuesOffset uint32 32 | } 33 | -------------------------------------------------------------------------------- /collate/sort.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package collate 6 | 7 | import ( 8 | "bytes" 9 | "sort" 10 | ) 11 | 12 | const ( 13 | maxSortBuffer = 40960 14 | maxSortEntries = 4096 15 | ) 16 | 17 | type swapper interface { 18 | Swap(i, j int) 19 | } 20 | 21 | type sorter struct { 22 | buf *Buffer 23 | keys [][]byte 24 | src swapper 25 | } 26 | 27 | func (s *sorter) init(n int) { 28 | if s.buf == nil { 29 | s.buf = &Buffer{} 30 | s.buf.init() 31 | } 32 | if cap(s.keys) < n { 33 | s.keys = make([][]byte, n) 34 | } 35 | s.keys = s.keys[0:n] 36 | } 37 | 38 | func (s *sorter) sort(src swapper) { 39 | s.src = src 40 | sort.Sort(s) 41 | } 42 | 43 | func (s sorter) Len() int { 44 | return len(s.keys) 45 | } 46 | 47 | func (s sorter) Less(i, j int) bool { 48 | return bytes.Compare(s.keys[i], s.keys[j]) == -1 49 | } 50 | 51 | func (s sorter) Swap(i, j int) { 52 | s.keys[i], s.keys[j] = s.keys[j], s.keys[i] 53 | s.src.Swap(i, j) 54 | } 55 | 56 | // A Lister can be sorted by Collator's Sort method. 57 | type Lister interface { 58 | Len() int 59 | Swap(i, j int) 60 | // Bytes returns the bytes of the text at index i. 61 | Bytes(i int) []byte 62 | } 63 | 64 | // Sort uses sort.Sort to sort the strings represented by x using the rules of c. 65 | func (c *Collator) Sort(x Lister) { 66 | n := x.Len() 67 | c.sorter.init(n) 68 | for i := 0; i < n; i++ { 69 | c.sorter.keys[i] = c.Key(c.sorter.buf, x.Bytes(i)) 70 | } 71 | c.sorter.sort(x) 72 | } 73 | 74 | // SortStrings uses sort.Sort to sort the strings in x using the rules of c. 75 | func (c *Collator) SortStrings(x []string) { 76 | c.sorter.init(len(x)) 77 | for i, s := range x { 78 | c.sorter.keys[i] = c.KeyFromString(c.sorter.buf, s) 79 | } 80 | c.sorter.sort(sort.StringSlice(x)) 81 | } 82 | -------------------------------------------------------------------------------- /collate/sort_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package collate_test 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | 11 | "golang.org/x/text/collate" 12 | "golang.org/x/text/language" 13 | ) 14 | 15 | func ExampleCollator_strings() { 16 | c := collate.New(language.Und) 17 | strings := []string{ 18 | "ad", 19 | "ab", 20 | "äb", 21 | "ac", 22 | } 23 | c.SortStrings(strings) 24 | fmt.Println(strings) 25 | // Output: [ab äb ac ad] 26 | } 27 | 28 | type sorter []string 29 | 30 | func (s sorter) Len() int { 31 | return len(s) 32 | } 33 | 34 | func (s sorter) Swap(i, j int) { 35 | s[j], s[i] = s[i], s[j] 36 | } 37 | 38 | func (s sorter) Bytes(i int) []byte { 39 | return []byte(s[i]) 40 | } 41 | 42 | func TestSort(t *testing.T) { 43 | c := collate.New(language.English) 44 | strings := []string{ 45 | "bcd", 46 | "abc", 47 | "ddd", 48 | } 49 | c.Sort(sorter(strings)) 50 | res := fmt.Sprint(strings) 51 | want := "[abc bcd ddd]" 52 | if res != want { 53 | t.Errorf("found %s; want %s", res, want) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /collate/tools/colcmp/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2012 The Go Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style 3 | # license that can be found in the LICENSE file. 4 | 5 | chars: 6 | go run ../../maketables.go -tables=chars -package=main > chars.go 7 | gofmt -w -s chars.go 8 | -------------------------------------------------------------------------------- /collate/tools/colcmp/col.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "log" 9 | "unicode/utf16" 10 | 11 | "golang.org/x/text/collate" 12 | "golang.org/x/text/language" 13 | ) 14 | 15 | // Input holds an input string in both UTF-8 and UTF-16 format. 16 | type Input struct { 17 | index int // used for restoring to original random order 18 | UTF8 []byte 19 | UTF16 []uint16 20 | key []byte // used for sorting 21 | } 22 | 23 | func (i Input) String() string { 24 | return string(i.UTF8) 25 | } 26 | 27 | func makeInput(s8 []byte, s16 []uint16) Input { 28 | return Input{UTF8: s8, UTF16: s16} 29 | } 30 | 31 | func makeInputString(s string) Input { 32 | return Input{ 33 | UTF8: []byte(s), 34 | UTF16: utf16.Encode([]rune(s)), 35 | } 36 | } 37 | 38 | // Collator is an interface for architecture-specific implementations of collation. 39 | type Collator interface { 40 | // Key generates a sort key for the given input. Implemenations 41 | // may return nil if a collator does not support sort keys. 42 | Key(s Input) []byte 43 | 44 | // Compare returns -1 if a < b, 1 if a > b and 0 if a == b. 45 | Compare(a, b Input) int 46 | } 47 | 48 | // CollatorFactory creates a Collator for a given language tag. 49 | type CollatorFactory struct { 50 | name string 51 | makeFn func(tag string) (Collator, error) 52 | description string 53 | } 54 | 55 | var collators = []CollatorFactory{} 56 | 57 | // AddFactory registers f as a factory for an implementation of Collator. 58 | func AddFactory(f CollatorFactory) { 59 | collators = append(collators, f) 60 | } 61 | 62 | func getCollator(name, locale string) Collator { 63 | for _, f := range collators { 64 | if f.name == name { 65 | col, err := f.makeFn(locale) 66 | if err != nil { 67 | log.Fatal(err) 68 | } 69 | return col 70 | } 71 | } 72 | log.Fatalf("collator of type %q not found", name) 73 | return nil 74 | } 75 | 76 | // goCollator is an implementation of Collator using go's own collator. 77 | type goCollator struct { 78 | c *collate.Collator 79 | buf collate.Buffer 80 | } 81 | 82 | func init() { 83 | AddFactory(CollatorFactory{"go", newGoCollator, "Go's native collator implementation."}) 84 | } 85 | 86 | func newGoCollator(loc string) (Collator, error) { 87 | c := &goCollator{c: collate.New(language.Make(loc))} 88 | return c, nil 89 | } 90 | 91 | func (c *goCollator) Key(b Input) []byte { 92 | return c.c.Key(&c.buf, b.UTF8) 93 | } 94 | 95 | func (c *goCollator) Compare(a, b Input) int { 96 | return c.c.Compare(a.UTF8, b.UTF8) 97 | } 98 | -------------------------------------------------------------------------------- /currency/common.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | package currency 4 | 5 | import ( 6 | "time" 7 | 8 | "golang.org/x/text/language" 9 | ) 10 | 11 | // This file contains code common to gen.go and the package code. 12 | 13 | const ( 14 | cashShift = 3 15 | roundMask = 0x7 16 | 17 | nonTenderBit = 0x8000 18 | ) 19 | 20 | // currencyInfo contains information about a currency. 21 | // bits 0..2: index into roundings for standard rounding 22 | // bits 3..5: index into roundings for cash rounding 23 | type currencyInfo byte 24 | 25 | // roundingType defines the scale (number of fractional decimals) and increments 26 | // in terms of units of size 10^-scale. For example, for scale == 2 and 27 | // increment == 1, the currency is rounded to units of 0.01. 28 | type roundingType struct { 29 | scale, increment uint8 30 | } 31 | 32 | // roundings contains rounding data for currencies. This struct is 33 | // created by hand as it is very unlikely to change much. 34 | var roundings = [...]roundingType{ 35 | {2, 1}, // default 36 | {0, 1}, 37 | {1, 1}, 38 | {3, 1}, 39 | {4, 1}, 40 | {2, 5}, // cash rounding alternative 41 | {2, 50}, 42 | } 43 | 44 | // regionToCode returns a 16-bit region code. Only two-letter codes are 45 | // supported. (Three-letter codes are not needed.) 46 | func regionToCode(r language.Region) uint16 { 47 | if s := r.String(); len(s) == 2 { 48 | return uint16(s[0])<<8 | uint16(s[1]) 49 | } 50 | return 0 51 | } 52 | 53 | func toDate(t time.Time) uint32 { 54 | y := t.Year() 55 | if y == 1 { 56 | return 0 57 | } 58 | date := uint32(y) << 4 59 | date |= uint32(t.Month()) 60 | date <<= 5 61 | date |= uint32(t.Day()) 62 | return date 63 | } 64 | 65 | func fromDate(date uint32) time.Time { 66 | return time.Date(int(date>>9), time.Month((date>>5)&0xf), int(date&0x1f), 0, 0, 0, 0, time.UTC) 67 | } 68 | -------------------------------------------------------------------------------- /currency/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package currency_test 6 | 7 | import ( 8 | "fmt" 9 | "time" 10 | 11 | "golang.org/x/text/currency" 12 | ) 13 | 14 | func ExampleQuery() { 15 | t1799, _ := time.Parse("2006-01-02", "1799-01-01") 16 | for it := currency.Query(currency.Date(t1799)); it.Next(); { 17 | from := "" 18 | if t, ok := it.From(); ok { 19 | from = t.Format("2006-01-02") 20 | } 21 | fmt.Printf("%v is used in %v since: %v\n", it.Unit(), it.Region(), from) 22 | } 23 | // Output: 24 | // GBP is used in GB since: 1694-07-27 25 | // GIP is used in GI since: 1713-01-01 26 | // USD is used in US since: 1792-01-01 27 | } 28 | -------------------------------------------------------------------------------- /currency/gen_common.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | package main 8 | 9 | import ( 10 | "time" 11 | 12 | "golang.org/x/text/language" 13 | ) 14 | 15 | // This file contains code common to gen.go and the package code. 16 | 17 | const ( 18 | cashShift = 3 19 | roundMask = 0x7 20 | 21 | nonTenderBit = 0x8000 22 | ) 23 | 24 | // currencyInfo contains information about a currency. 25 | // bits 0..2: index into roundings for standard rounding 26 | // bits 3..5: index into roundings for cash rounding 27 | type currencyInfo byte 28 | 29 | // roundingType defines the scale (number of fractional decimals) and increments 30 | // in terms of units of size 10^-scale. For example, for scale == 2 and 31 | // increment == 1, the currency is rounded to units of 0.01. 32 | type roundingType struct { 33 | scale, increment uint8 34 | } 35 | 36 | // roundings contains rounding data for currencies. This struct is 37 | // created by hand as it is very unlikely to change much. 38 | var roundings = [...]roundingType{ 39 | {2, 1}, // default 40 | {0, 1}, 41 | {1, 1}, 42 | {3, 1}, 43 | {4, 1}, 44 | {2, 5}, // cash rounding alternative 45 | {2, 50}, 46 | } 47 | 48 | // regionToCode returns a 16-bit region code. Only two-letter codes are 49 | // supported. (Three-letter codes are not needed.) 50 | func regionToCode(r language.Region) uint16 { 51 | if s := r.String(); len(s) == 2 { 52 | return uint16(s[0])<<8 | uint16(s[1]) 53 | } 54 | return 0 55 | } 56 | 57 | func toDate(t time.Time) uint32 { 58 | y := t.Year() 59 | if y == 1 { 60 | return 0 61 | } 62 | date := uint32(y) << 4 63 | date |= uint32(t.Month()) 64 | date <<= 5 65 | date |= uint32(t.Day()) 66 | return date 67 | } 68 | 69 | func fromDate(date uint32) time.Time { 70 | return time.Date(int(date>>9), time.Month((date>>5)&0xf), int(date&0x1f), 0, 0, 0, 0, time.UTC) 71 | } 72 | -------------------------------------------------------------------------------- /currency/tables_test.go: -------------------------------------------------------------------------------- 1 | package currency 2 | 3 | import ( 4 | "flag" 5 | "strings" 6 | "testing" 7 | "time" 8 | 9 | "golang.org/x/text/internal/gen" 10 | "golang.org/x/text/internal/testtext" 11 | "golang.org/x/text/language" 12 | "golang.org/x/text/message" 13 | "golang.org/x/text/unicode/cldr" 14 | ) 15 | 16 | var draft = flag.String("draft", 17 | "contributed", 18 | `Minimal draft requirements (approved, contributed, provisional, unconfirmed).`) 19 | 20 | func TestTables(t *testing.T) { 21 | testtext.SkipIfNotLong(t) 22 | 23 | // Read the CLDR zip file. 24 | r := gen.OpenCLDRCoreZip() 25 | defer r.Close() 26 | 27 | d := &cldr.Decoder{} 28 | d.SetDirFilter("supplemental", "main") 29 | d.SetSectionFilter("numbers") 30 | data, err := d.DecodeZip(r) 31 | if err != nil { 32 | t.Fatalf("DecodeZip: %v", err) 33 | } 34 | 35 | dr, err := cldr.ParseDraft(*draft) 36 | if err != nil { 37 | t.Fatalf("filter: %v", err) 38 | } 39 | 40 | for _, lang := range data.Locales() { 41 | p := message.NewPrinter(language.MustParse(lang)) 42 | 43 | ldml := data.RawLDML(lang) 44 | if ldml.Numbers == nil || ldml.Numbers.Currencies == nil { 45 | continue 46 | } 47 | for _, c := range ldml.Numbers.Currencies.Currency { 48 | syms := cldr.MakeSlice(&c.Symbol) 49 | syms.SelectDraft(dr) 50 | 51 | for _, sym := range c.Symbol { 52 | cur, err := ParseISO(c.Type) 53 | if err != nil { 54 | continue 55 | } 56 | formatter := Symbol 57 | switch sym.Alt { 58 | case "": 59 | case "narrow": 60 | formatter = NarrowSymbol 61 | default: 62 | continue 63 | } 64 | want := sym.Data() 65 | if got := p.Sprint(formatter(cur)); got != want { 66 | t.Errorf("%s:%sSymbol(%s) = %s; want %s", lang, strings.Title(sym.Alt), c.Type, got, want) 67 | } 68 | } 69 | } 70 | } 71 | 72 | for _, reg := range data.Supplemental().CurrencyData.Region { 73 | i := 0 74 | for ; regionData[i].Region().String() != reg.Iso3166; i++ { 75 | } 76 | it := Query(Historical, NonTender, Region(language.MustParseRegion(reg.Iso3166))) 77 | for _, cur := range reg.Currency { 78 | from, _ := time.Parse("2006-01-02", cur.From) 79 | to, _ := time.Parse("2006-01-02", cur.To) 80 | 81 | it.Next() 82 | for j, r := range []QueryIter{&iter{regionInfo: ®ionData[i]}, it} { 83 | if got, _ := r.From(); from != got { 84 | t.Errorf("%d:%s:%s:from: got %v; want %v", j, reg.Iso3166, cur.Iso4217, got, from) 85 | } 86 | if got, _ := r.To(); to != got { 87 | t.Errorf("%d:%s:%s:to: got %v; want %v", j, reg.Iso3166, cur.Iso4217, got, to) 88 | } 89 | } 90 | i++ 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:generate go run gen.go 6 | 7 | // text is a repository of text-related packages related to internationalization 8 | // (i18n) and localization (l10n), such as character encodings, text 9 | // transformations, and locale-specific text handling. 10 | // 11 | // There is a 30 minute video, recorded on 2017-11-30, on the "State of 12 | // golang.org/x/text" at https://www.youtube.com/watch?v=uYrDrMEGu58 13 | package text 14 | 15 | // TODO: more documentation on general concepts, such as Transformers, use 16 | // of normalization, etc. 17 | -------------------------------------------------------------------------------- /encoding/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package encoding_test 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | "os" 11 | "strings" 12 | 13 | "golang.org/x/text/encoding" 14 | "golang.org/x/text/encoding/charmap" 15 | "golang.org/x/text/encoding/unicode" 16 | "golang.org/x/text/transform" 17 | ) 18 | 19 | func Example_decodeWindows1252() { 20 | sr := strings.NewReader("Gar\xe7on !") 21 | tr := charmap.Windows1252.NewDecoder().Reader(sr) 22 | io.Copy(os.Stdout, tr) 23 | // Output: Garçon ! 24 | } 25 | 26 | func ExampleUTF8Validator() { 27 | for i := 0; i < 2; i++ { 28 | var transformer transform.Transformer 29 | transformer = unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM).NewEncoder() 30 | if i == 1 { 31 | transformer = transform.Chain(encoding.UTF8Validator, transformer) 32 | } 33 | dst := make([]byte, 256) 34 | src := []byte("abc\xffxyz") // src is invalid UTF-8. 35 | nDst, nSrc, err := transformer.Transform(dst, src, true) 36 | fmt.Printf("i=%d: produced %q, consumed %q, error %v\n", 37 | i, dst[:nDst], src[:nSrc], err) 38 | } 39 | // Output: 40 | // i=0: produced "\x00a\x00b\x00c\xff\xfd\x00x\x00y\x00z", consumed "abc\xffxyz", error 41 | // i=1: produced "\x00a\x00b\x00c", consumed "abc", error encoding: invalid UTF-8 42 | } 43 | -------------------------------------------------------------------------------- /encoding/ianaindex/ascii.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package ianaindex 6 | 7 | import ( 8 | "unicode" 9 | "unicode/utf8" 10 | 11 | "golang.org/x/text/encoding" 12 | "golang.org/x/text/encoding/internal" 13 | "golang.org/x/text/encoding/internal/identifier" 14 | "golang.org/x/text/transform" 15 | ) 16 | 17 | type asciiDecoder struct { 18 | transform.NopResetter 19 | } 20 | 21 | func (d asciiDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { 22 | for _, c := range src { 23 | if c > unicode.MaxASCII { 24 | r := unicode.ReplacementChar 25 | if nDst+utf8.RuneLen(r) > len(dst) { 26 | err = transform.ErrShortDst 27 | break 28 | } 29 | nDst += utf8.EncodeRune(dst[nDst:], r) 30 | nSrc++ 31 | continue 32 | } 33 | 34 | if nDst >= len(dst) { 35 | err = transform.ErrShortDst 36 | break 37 | } 38 | dst[nDst] = c 39 | nDst++ 40 | nSrc++ 41 | } 42 | return nDst, nSrc, err 43 | } 44 | 45 | type asciiEncoder struct { 46 | transform.NopResetter 47 | } 48 | 49 | func (d asciiEncoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { 50 | for _, c := range src { 51 | if c > unicode.MaxASCII { 52 | err = internal.RepertoireError(encoding.ASCIISub) 53 | break 54 | } 55 | 56 | if nDst >= len(dst) { 57 | err = transform.ErrShortDst 58 | break 59 | } 60 | dst[nDst] = c 61 | nDst++ 62 | nSrc++ 63 | } 64 | return nDst, nSrc, err 65 | } 66 | 67 | var asciiEnc = &internal.Encoding{ 68 | Encoding: &internal.SimpleEncoding{ 69 | asciiDecoder{}, 70 | asciiEncoder{}, 71 | }, 72 | Name: "US-ASCII", 73 | MIB: identifier.ASCII, 74 | } 75 | -------------------------------------------------------------------------------- /encoding/ianaindex/ascii_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package ianaindex 6 | 7 | import ( 8 | "testing" 9 | "unicode" 10 | 11 | "golang.org/x/text/encoding" 12 | ) 13 | 14 | func TestASCIIDecoder(t *testing.T) { 15 | repl := string(unicode.ReplacementChar) 16 | input := "Comment Candide fut élevé dans un beau château" 17 | want := "Comment Candide fut " + repl + repl + "lev" + repl + repl + " dans un beau ch" + repl + repl + "teau" 18 | got, err := asciiEnc.NewDecoder().String(input) 19 | if err != nil { 20 | t.Fatalf("unexpected error: %v", err) 21 | } 22 | if got != want { 23 | t.Fatalf("asciiEnc.NewDecoder().String() = %q, want %q", got, want) 24 | } 25 | } 26 | 27 | func TestASCIIEncoder(t *testing.T) { 28 | repl := string(encoding.ASCIISub) 29 | input := "Comment Candide fut élevé dans un beau château" 30 | want := "Comment Candide fut " + repl + "lev" + repl + " dans un beau ch" + repl + "teau" 31 | got, err := encoding.ReplaceUnsupported(asciiEnc.NewEncoder()).String(input) 32 | if err != nil { 33 | t.Fatalf("unexpected error: %v", err) 34 | } 35 | if got != want { 36 | t.Fatalf("asciiEnc.NewEncoder().String() = %q, want %q", got, want) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /encoding/ianaindex/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package ianaindex_test 6 | 7 | import ( 8 | "fmt" 9 | 10 | "golang.org/x/text/encoding/charmap" 11 | "golang.org/x/text/encoding/ianaindex" 12 | ) 13 | 14 | func ExampleIndex() { 15 | fmt.Println(ianaindex.MIME.Name(charmap.ISO8859_7)) 16 | fmt.Println(ianaindex.IANA.Name(charmap.ISO8859_7)) 17 | fmt.Println(ianaindex.MIB.Name(charmap.ISO8859_7)) 18 | 19 | e, _ := ianaindex.IANA.Encoding("cp437") 20 | fmt.Println(ianaindex.IANA.Name(e)) 21 | 22 | // Output: 23 | // ISO-8859-7 24 | // ISO_8859-7:1987 25 | // ISOLatinGreek 26 | // IBM437 27 | } 28 | -------------------------------------------------------------------------------- /encoding/internal/internal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package internal contains code that is shared among encoding implementations. 6 | package internal 7 | 8 | import ( 9 | "golang.org/x/text/encoding" 10 | "golang.org/x/text/encoding/internal/identifier" 11 | "golang.org/x/text/transform" 12 | ) 13 | 14 | // Encoding is an implementation of the Encoding interface that adds the String 15 | // and ID methods to an existing encoding. 16 | type Encoding struct { 17 | encoding.Encoding 18 | Name string 19 | MIB identifier.MIB 20 | } 21 | 22 | // _ verifies that Encoding implements identifier.Interface. 23 | var _ identifier.Interface = (*Encoding)(nil) 24 | 25 | func (e *Encoding) String() string { 26 | return e.Name 27 | } 28 | 29 | func (e *Encoding) ID() (mib identifier.MIB, other string) { 30 | return e.MIB, "" 31 | } 32 | 33 | // SimpleEncoding is an Encoding that combines two Transformers. 34 | type SimpleEncoding struct { 35 | Decoder transform.Transformer 36 | Encoder transform.Transformer 37 | } 38 | 39 | func (e *SimpleEncoding) NewDecoder() *encoding.Decoder { 40 | return &encoding.Decoder{Transformer: e.Decoder} 41 | } 42 | 43 | func (e *SimpleEncoding) NewEncoder() *encoding.Encoder { 44 | return &encoding.Encoder{Transformer: e.Encoder} 45 | } 46 | 47 | // FuncEncoding is an Encoding that combines two functions returning a new 48 | // Transformer. 49 | type FuncEncoding struct { 50 | Decoder func() transform.Transformer 51 | Encoder func() transform.Transformer 52 | } 53 | 54 | func (e FuncEncoding) NewDecoder() *encoding.Decoder { 55 | return &encoding.Decoder{Transformer: e.Decoder()} 56 | } 57 | 58 | func (e FuncEncoding) NewEncoder() *encoding.Encoder { 59 | return &encoding.Encoder{Transformer: e.Encoder()} 60 | } 61 | 62 | // A RepertoireError indicates a rune is not in the repertoire of a destination 63 | // encoding. It is associated with an encoding-specific suggested replacement 64 | // byte. 65 | type RepertoireError byte 66 | 67 | // Error implements the error interface. 68 | func (r RepertoireError) Error() string { 69 | return "encoding: rune not supported by encoding." 70 | } 71 | 72 | // Replacement returns the replacement string associated with this error. 73 | func (r RepertoireError) Replacement() byte { return byte(r) } 74 | 75 | var ErrASCIIReplacement = RepertoireError(encoding.ASCIISub) 76 | -------------------------------------------------------------------------------- /encoding/japanese/all.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package japanese 6 | 7 | import ( 8 | "golang.org/x/text/encoding" 9 | ) 10 | 11 | // All is a list of all defined encodings in this package. 12 | var All = []encoding.Encoding{EUCJP, ISO2022JP, ShiftJIS} 13 | -------------------------------------------------------------------------------- /encoding/simplifiedchinese/all.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package simplifiedchinese 6 | 7 | import ( 8 | "golang.org/x/text/encoding" 9 | ) 10 | 11 | // All is a list of all defined encodings in this package. 12 | var All = []encoding.Encoding{GB18030, GBK, HZGB2312} 13 | -------------------------------------------------------------------------------- /encoding/testdata/candide-gb18030.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/text/80721808805f9d846d907c85d73ca6b5b6ecb870/encoding/testdata/candide-gb18030.txt -------------------------------------------------------------------------------- /encoding/testdata/candide-utf-16le.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/text/80721808805f9d846d907c85d73ca6b5b6ecb870/encoding/testdata/candide-utf-16le.txt -------------------------------------------------------------------------------- /encoding/testdata/candide-utf-32be.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/text/80721808805f9d846d907c85d73ca6b5b6ecb870/encoding/testdata/candide-utf-32be.txt -------------------------------------------------------------------------------- /encoding/testdata/candide-windows-1252.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/text/80721808805f9d846d907c85d73ca6b5b6ecb870/encoding/testdata/candide-windows-1252.txt -------------------------------------------------------------------------------- /encoding/testdata/rashomon-euc-jp.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/text/80721808805f9d846d907c85d73ca6b5b6ecb870/encoding/testdata/rashomon-euc-jp.txt -------------------------------------------------------------------------------- /encoding/testdata/rashomon-shift-jis.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/text/80721808805f9d846d907c85d73ca6b5b6ecb870/encoding/testdata/rashomon-shift-jis.txt -------------------------------------------------------------------------------- /encoding/testdata/sunzi-bingfa-simplified-gbk.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/text/80721808805f9d846d907c85d73ca6b5b6ecb870/encoding/testdata/sunzi-bingfa-simplified-gbk.txt -------------------------------------------------------------------------------- /encoding/testdata/sunzi-bingfa-traditional-big5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/text/80721808805f9d846d907c85d73ca6b5b6ecb870/encoding/testdata/sunzi-bingfa-traditional-big5.txt -------------------------------------------------------------------------------- /encoding/testdata/unsu-joh-eun-nal-euc-kr.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/text/80721808805f9d846d907c85d73ca6b5b6ecb870/encoding/testdata/unsu-joh-eun-nal-euc-kr.txt -------------------------------------------------------------------------------- /feature/plural/common.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | package plural 4 | 5 | // Form defines a plural form. 6 | // 7 | // Not all languages support all forms. Also, the meaning of each form varies 8 | // per language. It is important to note that the name of a form does not 9 | // necessarily correspond one-to-one with the set of numbers. For instance, 10 | // for Croation, One matches not only 1, but also 11, 21, etc. 11 | // 12 | // Each language must at least support the form "other". 13 | type Form byte 14 | 15 | const ( 16 | Other Form = iota 17 | Zero 18 | One 19 | Two 20 | Few 21 | Many 22 | ) 23 | 24 | var countMap = map[string]Form{ 25 | "other": Other, 26 | "zero": Zero, 27 | "one": One, 28 | "two": Two, 29 | "few": Few, 30 | "many": Many, 31 | } 32 | 33 | type pluralCheck struct { 34 | // category: 35 | // 3..7: opID 36 | // 0..2: category 37 | cat byte 38 | setID byte 39 | } 40 | 41 | // opID identifies the type of operand in the plural rule, being i, n or f. 42 | // (v, w, and t are treated as filters in our implementation.) 43 | type opID byte 44 | 45 | const ( 46 | opMod opID = 0x1 // is '%' used? 47 | opNotEqual opID = 0x2 // using "!=" to compare 48 | opI opID = 0 << 2 // integers after taking the absolute value 49 | opN opID = 1 << 2 // full number (must be integer) 50 | opF opID = 2 << 2 // fraction 51 | opV opID = 3 << 2 // number of visible digits 52 | opW opID = 4 << 2 // number of visible digits without trailing zeros 53 | opBretonM opID = 5 << 2 // hard-wired rule for Breton 54 | opItalian800 opID = 6 << 2 // hard-wired rule for Italian 55 | opAzerbaijan00s opID = 7 << 2 // hard-wired rule for Azerbaijan 56 | ) 57 | const ( 58 | // Use this plural form to indicate the next rule needs to match as well. 59 | // The last condition in the list will have the correct plural form. 60 | andNext = 0x7 61 | formMask = 0x7 62 | 63 | opShift = 3 64 | 65 | // numN indicates the maximum integer, or maximum mod value, for which we 66 | // have inclusion masks. 67 | numN = 100 68 | // The common denominator of the modulo that is taken. 69 | maxMod = 100 70 | ) 71 | -------------------------------------------------------------------------------- /feature/plural/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package plural_test 6 | 7 | import ( 8 | "golang.org/x/text/feature/plural" 9 | "golang.org/x/text/language" 10 | "golang.org/x/text/message" 11 | ) 12 | 13 | func ExampleSelectf() { 14 | // Manually set some translations. This is typically done programmatically. 15 | message.Set(language.English, "%d files remaining", 16 | plural.Selectf(1, "%d", 17 | "=0", "done!", 18 | plural.One, "one file remaining", 19 | plural.Other, "%[1]d files remaining", 20 | )) 21 | message.Set(language.Dutch, "%d files remaining", 22 | plural.Selectf(1, "%d", 23 | "=0", "klaar!", 24 | // One can also use a string instead of a Kind 25 | "one", "nog één bestand te gaan", 26 | "other", "nog %[1]d bestanden te gaan", 27 | )) 28 | 29 | p := message.NewPrinter(language.English) 30 | p.Printf("%d files remaining", 5) 31 | p.Println() 32 | p.Printf("%d files remaining", 1) 33 | p.Println() 34 | 35 | p = message.NewPrinter(language.Dutch) 36 | p.Printf("%d files remaining", 1) 37 | p.Println() 38 | p.Printf("%d files remaining", 0) 39 | p.Println() 40 | 41 | // Output: 42 | // 5 files remaining 43 | // one file remaining 44 | // nog één bestand te gaan 45 | // klaar! 46 | } 47 | -------------------------------------------------------------------------------- /feature/plural/gen_common.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | package main 8 | 9 | // Form defines a plural form. 10 | // 11 | // Not all languages support all forms. Also, the meaning of each form varies 12 | // per language. It is important to note that the name of a form does not 13 | // necessarily correspond one-to-one with the set of numbers. For instance, 14 | // for Croation, One matches not only 1, but also 11, 21, etc. 15 | // 16 | // Each language must at least support the form "other". 17 | type Form byte 18 | 19 | const ( 20 | Other Form = iota 21 | Zero 22 | One 23 | Two 24 | Few 25 | Many 26 | ) 27 | 28 | var countMap = map[string]Form{ 29 | "other": Other, 30 | "zero": Zero, 31 | "one": One, 32 | "two": Two, 33 | "few": Few, 34 | "many": Many, 35 | } 36 | 37 | type pluralCheck struct { 38 | // category: 39 | // 3..7: opID 40 | // 0..2: category 41 | cat byte 42 | setID byte 43 | } 44 | 45 | // opID identifies the type of operand in the plural rule, being i, n or f. 46 | // (v, w, and t are treated as filters in our implementation.) 47 | type opID byte 48 | 49 | const ( 50 | opMod opID = 0x1 // is '%' used? 51 | opNotEqual opID = 0x2 // using "!=" to compare 52 | opI opID = 0 << 2 // integers after taking the absolute value 53 | opN opID = 1 << 2 // full number (must be integer) 54 | opF opID = 2 << 2 // fraction 55 | opV opID = 3 << 2 // number of visible digits 56 | opW opID = 4 << 2 // number of visible digits without trailing zeros 57 | opBretonM opID = 5 << 2 // hard-wired rule for Breton 58 | opItalian800 opID = 6 << 2 // hard-wired rule for Italian 59 | opAzerbaijan00s opID = 7 << 2 // hard-wired rule for Azerbaijan 60 | ) 61 | const ( 62 | // Use this plural form to indicate the next rule needs to match as well. 63 | // The last condition in the list will have the correct plural form. 64 | andNext = 0x7 65 | formMask = 0x7 66 | 67 | opShift = 3 68 | 69 | // numN indicates the maximum integer, or maximum mod value, for which we 70 | // have inclusion masks. 71 | numN = 100 72 | // The common denominator of the modulo that is taken. 73 | maxMod = 100 74 | ) 75 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module golang.org/x/text 2 | 3 | go 1.23.0 4 | 5 | require golang.org/x/tools v0.33.0 // tagx:ignore 6 | 7 | require ( 8 | golang.org/x/mod v0.25.0 // indirect; tagx:ignore 9 | golang.org/x/sync v0.15.0 // indirect 10 | ) 11 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 2 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 3 | golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= 4 | golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= 5 | golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= 6 | golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 7 | golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= 8 | golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= 9 | -------------------------------------------------------------------------------- /internal/catmsg/varint.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package catmsg 6 | 7 | // This file implements varint encoding analogous to the one in encoding/binary. 8 | // We need a string version of this function, so we add that here and then add 9 | // the rest for consistency. 10 | 11 | import "errors" 12 | 13 | var ( 14 | errIllegalVarint = errors.New("catmsg: illegal varint") 15 | errVarintTooLarge = errors.New("catmsg: varint too large for uint64") 16 | ) 17 | 18 | const maxVarintBytes = 10 // maximum length of a varint 19 | 20 | // encodeUint encodes x as a variable-sized integer into buf and returns the 21 | // number of bytes written. buf must be at least maxVarintBytes long 22 | func encodeUint(buf []byte, x uint64) (n int) { 23 | for ; x > 127; n++ { 24 | buf[n] = 0x80 | uint8(x&0x7F) 25 | x >>= 7 26 | } 27 | buf[n] = uint8(x) 28 | n++ 29 | return n 30 | } 31 | 32 | func decodeUintString(s string) (x uint64, size int, err error) { 33 | i := 0 34 | for shift := uint(0); shift < 64; shift += 7 { 35 | if i >= len(s) { 36 | return 0, i, errIllegalVarint 37 | } 38 | b := uint64(s[i]) 39 | i++ 40 | x |= (b & 0x7F) << shift 41 | if b&0x80 == 0 { 42 | return x, i, nil 43 | } 44 | } 45 | return 0, i, errVarintTooLarge 46 | } 47 | 48 | func decodeUint(b []byte) (x uint64, size int, err error) { 49 | i := 0 50 | for shift := uint(0); shift < 64; shift += 7 { 51 | if i >= len(b) { 52 | return 0, i, errIllegalVarint 53 | } 54 | c := uint64(b[i]) 55 | i++ 56 | x |= (c & 0x7F) << shift 57 | if c&0x80 == 0 { 58 | return x, i, nil 59 | } 60 | } 61 | return 0, i, errVarintTooLarge 62 | } 63 | -------------------------------------------------------------------------------- /internal/cldrtree/option.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package cldrtree 6 | 7 | import ( 8 | "reflect" 9 | 10 | "golang.org/x/text/unicode/cldr" 11 | ) 12 | 13 | // An Option configures an Index. 14 | type Option func(*options) 15 | 16 | type options struct { 17 | parent *Index 18 | 19 | name string 20 | alias *cldr.Common 21 | 22 | sharedType *typeInfo 23 | sharedEnums *enum 24 | } 25 | 26 | func (o *options) fill(opt []Option) { 27 | for _, f := range opt { 28 | f(o) 29 | } 30 | } 31 | 32 | // setAlias sets an alias from the given node, if the node defines one. 33 | func (o *options) setAlias(n Element) { 34 | if n != nil && !reflect.ValueOf(n).IsNil() { 35 | o.alias = n.GetCommon() 36 | } 37 | } 38 | 39 | // Enum defines an enumeration type. The resulting option may be passed for the 40 | // construction of multiple Indexes, which they will share the same enum values. 41 | // Calling Gen on a Builder will generate the Enum for the given name. The 42 | // optional values fix the values for the given identifier to the argument 43 | // position (starting at 0). Other values may still be added and will be 44 | // assigned to subsequent values. 45 | func Enum(name string, value ...string) Option { 46 | return EnumFunc(name, nil, value...) 47 | } 48 | 49 | // EnumFunc is like Enum but also takes a function that allows rewriting keys. 50 | func EnumFunc(name string, rename func(string) string, value ...string) Option { 51 | enum := &enum{name: name, rename: rename, keyMap: map[string]enumIndex{}} 52 | for _, e := range value { 53 | enum.lookup(e) 54 | } 55 | return func(o *options) { 56 | found := false 57 | for _, e := range o.parent.meta.b.enums { 58 | if e.name == enum.name { 59 | found = true 60 | break 61 | } 62 | } 63 | if !found { 64 | o.parent.meta.b.enums = append(o.parent.meta.b.enums, enum) 65 | } 66 | o.sharedEnums = enum 67 | } 68 | } 69 | 70 | // SharedType returns an option which causes all Indexes to which this option is 71 | // passed to have the same type. 72 | func SharedType() Option { 73 | info := &typeInfo{} 74 | return func(o *options) { o.sharedType = info } 75 | } 76 | 77 | func useSharedType() Option { 78 | return func(o *options) { 79 | sub := o.parent.meta.typeInfo.keyTypeInfo 80 | if sub == nil { 81 | sub = &typeInfo{} 82 | o.parent.meta.typeInfo.keyTypeInfo = sub 83 | } 84 | o.sharedType = sub 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /internal/cldrtree/testdata/test2/common/main/en_001.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 001AbbrMo1 14 | 001AbbrMo2 15 | 001AbbrMo3 16 | 001AbbrMo4 17 | 001AbbrMo5 18 | 001AbbrMo6 19 | 001AbbrMo7 20 | 001AbbrMo8 21 | 001AbbrMo9 22 | 001AbbrMo10 23 | 001AbbrMo11 24 | 001AbbrMo12 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | mo 37 | 38 | 001ShortFutMOne 39 | 001ShortFutMOther 40 | 41 | 42 | 43 | 001ShortPastMOther 44 | 45 | 46 | 47 | mo 48 | 49 | 001NarrowFutMOne 50 | 001NarrowFutMTwo 51 | 001NarrowFutMOther 52 | 53 | 54 | 001NarrowPastMOne 55 | 001NarrowPastMOther 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /internal/cldrtree/testdata/test2/common/main/en_GB.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | gbAbbrM1 14 | gbAbbrM2 15 | gbAbbrM3 16 | gbAbbrM4 17 | gbAbbrM5 18 | gbAbbrM6 19 | gbAbbrM7 20 | gbAbbrM8 21 | gbAbbrM9 22 | gbAbbrM10 23 | gbAbbrM11 24 | gbAbbrM12 25 | 26 | 27 | gbWideM1 28 | gbWideM2 29 | gbWideM3 30 | gbWideM4 31 | gbWideM5 32 | gbWideM6 33 | gbWideM7 34 | gbWideM8 35 | gbWideM9 36 | gbWideM10 37 | gbWideM11 38 | gbWideM12 39 | 40 | 41 | 42 | 43 | gbNarrowM1 44 | gbNarrowM2 45 | gbNarrowM3 46 | 47 | gbNarrowM5 48 | gbNarrowM6 49 | 50 | gbNarrowM8 51 | gbNarrowM9 52 | gbNarrowM10 53 | gbNarrowM11 54 | gbNarrowM12 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /internal/colltab/iter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package colltab 6 | 7 | import ( 8 | "testing" 9 | ) 10 | 11 | func TestDoNorm(t *testing.T) { 12 | const div = -1 // The insertion point of the next block. 13 | tests := []struct { 14 | in, out []int 15 | }{{ 16 | in: []int{4, div, 3}, 17 | out: []int{3, 4}, 18 | }, { 19 | in: []int{4, div, 3, 3, 3}, 20 | out: []int{3, 3, 3, 4}, 21 | }, { 22 | in: []int{0, 4, div, 3}, 23 | out: []int{0, 3, 4}, 24 | }, { 25 | in: []int{0, 0, 4, 5, div, 3, 3}, 26 | out: []int{0, 0, 3, 3, 4, 5}, 27 | }, { 28 | in: []int{0, 0, 1, 4, 5, div, 3, 3}, 29 | out: []int{0, 0, 1, 3, 3, 4, 5}, 30 | }, { 31 | in: []int{0, 0, 1, 4, 5, div, 4, 4}, 32 | out: []int{0, 0, 1, 4, 4, 4, 5}, 33 | }, 34 | } 35 | for j, tt := range tests { 36 | i := Iter{} 37 | var w, p int 38 | for k, cc := range tt.in { 39 | 40 | if cc == div { 41 | w = 100 42 | p = k 43 | continue 44 | } 45 | i.Elems = append(i.Elems, makeCE([]int{w, defaultSecondary, 2, cc})) 46 | } 47 | i.doNorm(p, i.Elems[p].CCC()) 48 | if len(i.Elems) != len(tt.out) { 49 | t.Errorf("%d: length was %d; want %d", j, len(i.Elems), len(tt.out)) 50 | } 51 | prevCCC := uint8(0) 52 | for k, ce := range i.Elems { 53 | if int(ce.CCC()) != tt.out[k] { 54 | t.Errorf("%d:%d: unexpected CCC. Was %d; want %d", j, k, ce.CCC(), tt.out[k]) 55 | } 56 | if k > 0 && ce.CCC() == prevCCC && i.Elems[k-1].Primary() > ce.Primary() { 57 | t.Errorf("%d:%d: normalization crossed across CCC boundary.", j, k) 58 | } 59 | } 60 | } 61 | 62 | // Combining rune overflow is tested in search/pattern_test.go. 63 | } 64 | -------------------------------------------------------------------------------- /internal/colltab/weighter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package colltab // import "golang.org/x/text/internal/colltab" 6 | 7 | // A Weighter can be used as a source for Collator and Searcher. 8 | type Weighter interface { 9 | // Start finds the start of the segment that includes position p. 10 | Start(p int, b []byte) int 11 | 12 | // StartString finds the start of the segment that includes position p. 13 | StartString(p int, s string) int 14 | 15 | // AppendNext appends Elems to buf corresponding to the longest match 16 | // of a single character or contraction from the start of s. 17 | // It returns the new buf and the number of bytes consumed. 18 | AppendNext(buf []Elem, s []byte) (ce []Elem, n int) 19 | 20 | // AppendNextString appends Elems to buf corresponding to the longest match 21 | // of a single character or contraction from the start of s. 22 | // It returns the new buf and the number of bytes consumed. 23 | AppendNextString(buf []Elem, s string) (ce []Elem, n int) 24 | 25 | // Domain returns a slice of all single characters and contractions for which 26 | // collation elements are defined in this table. 27 | Domain() []string 28 | 29 | // Top returns the highest variable primary value. 30 | Top() uint32 31 | } 32 | -------------------------------------------------------------------------------- /internal/colltab/weighter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package colltab 6 | 7 | // testWeighter is a simple Weighter that returns weights from a user-defined map. 8 | type testWeighter map[string][]Elem 9 | 10 | func (t testWeighter) Start(int, []byte) int { return 0 } 11 | func (t testWeighter) StartString(int, string) int { return 0 } 12 | func (t testWeighter) Domain() []string { return nil } 13 | func (t testWeighter) Top() uint32 { return 0 } 14 | 15 | // maxContractBytes is the maximum length of any key in the map. 16 | const maxContractBytes = 10 17 | 18 | func (t testWeighter) AppendNext(buf []Elem, s []byte) ([]Elem, int) { 19 | n := len(s) 20 | if n > maxContractBytes { 21 | n = maxContractBytes 22 | } 23 | for i := n; i > 0; i-- { 24 | if e, ok := t[string(s[:i])]; ok { 25 | return append(buf, e...), i 26 | } 27 | } 28 | panic("incomplete testWeighter: could not find " + string(s)) 29 | } 30 | 31 | func (t testWeighter) AppendNextString(buf []Elem, s string) ([]Elem, int) { 32 | n := len(s) 33 | if n > maxContractBytes { 34 | n = maxContractBytes 35 | } 36 | for i := n; i > 0; i-- { 37 | if e, ok := t[s[:i]]; ok { 38 | return append(buf, e...), i 39 | } 40 | } 41 | panic("incomplete testWeighter: could not find " + s) 42 | } 43 | -------------------------------------------------------------------------------- /internal/export/README: -------------------------------------------------------------------------------- 1 | The export directory contains packages that are generated using the x/text 2 | infrastructure, but live elsewhere. 3 | At some point we can expose some of the infrastructure, but for now this 4 | is not done. 5 | -------------------------------------------------------------------------------- /internal/export/idna/common_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | package idna 4 | 5 | // This file contains code that is common between the generation code and the 6 | // package's test code. 7 | 8 | import ( 9 | "log" 10 | 11 | "golang.org/x/text/internal/ucd" 12 | ) 13 | 14 | func catFromEntry(p *ucd.Parser) (cat category) { 15 | r := p.Rune(0) 16 | switch s := p.String(1); s { 17 | case "valid": 18 | cat = valid 19 | case "disallowed": 20 | cat = disallowed 21 | case "disallowed_STD3_valid": 22 | cat = disallowedSTD3Valid 23 | case "disallowed_STD3_mapped": 24 | cat = disallowedSTD3Mapped 25 | case "mapped": 26 | cat = mapped 27 | case "deviation": 28 | cat = deviation 29 | case "ignored": 30 | cat = ignored 31 | default: 32 | log.Fatalf("%U: Unknown category %q", r, s) 33 | } 34 | if s := p.String(3); s != "" { 35 | if cat != valid { 36 | log.Fatalf(`%U: %s defined for %q; want "valid"`, r, s, p.String(1)) 37 | } 38 | switch s { 39 | case "NV8": 40 | cat = validNV8 41 | case "XV8": 42 | cat = validXV8 43 | default: 44 | log.Fatalf("%U: Unexpected exception %q", r, s) 45 | } 46 | } 47 | return cat 48 | } 49 | 50 | var joinType = map[string]info{ 51 | "L": joiningL, 52 | "D": joiningD, 53 | "T": joiningT, 54 | "R": joiningR, 55 | } 56 | -------------------------------------------------------------------------------- /internal/export/idna/conformance_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build go1.10 6 | 7 | package idna 8 | 9 | import ( 10 | "fmt" 11 | "strings" 12 | "testing" 13 | 14 | "golang.org/x/text/internal/gen" 15 | "golang.org/x/text/internal/testtext" 16 | "golang.org/x/text/internal/ucd" 17 | ) 18 | 19 | func TestConformance(t *testing.T) { 20 | testtext.SkipIfNotLong(t) 21 | 22 | r := gen.OpenUnicodeFile("idna", "10.0.0", "IdnaTest.txt") 23 | defer r.Close() 24 | 25 | section := "main" 26 | p := ucd.New(r) 27 | transitional := New(Transitional(true), VerifyDNSLength(true), BidiRule(), MapForLookup()) 28 | nonTransitional := New(VerifyDNSLength(true), BidiRule(), MapForLookup()) 29 | for p.Next() { 30 | // What to test 31 | profiles := []*Profile{} 32 | switch p.String(0) { 33 | case "T": 34 | profiles = append(profiles, transitional) 35 | case "N": 36 | profiles = append(profiles, nonTransitional) 37 | case "B": 38 | profiles = append(profiles, transitional) 39 | profiles = append(profiles, nonTransitional) 40 | } 41 | 42 | src := unescape(p.String(1)) 43 | 44 | wantToUnicode := unescape(p.String(2)) 45 | if wantToUnicode == "" { 46 | wantToUnicode = src 47 | } 48 | wantToASCII := unescape(p.String(3)) 49 | if wantToASCII == "" { 50 | wantToASCII = wantToUnicode 51 | } 52 | wantErrToUnicode := "" 53 | if strings.HasPrefix(wantToUnicode, "[") { 54 | wantErrToUnicode = wantToUnicode 55 | wantToUnicode = "" 56 | } 57 | wantErrToASCII := "" 58 | if strings.HasPrefix(wantToASCII, "[") { 59 | wantErrToASCII = wantToASCII 60 | wantToASCII = "" 61 | } 62 | 63 | // TODO: also do IDNA tests. 64 | // invalidInIDNA2008 := p.String(4) == "NV8" 65 | 66 | for _, p := range profiles { 67 | name := fmt.Sprintf("%s:%s", section, p) 68 | doTest(t, p.ToUnicode, name+":ToUnicode", src, wantToUnicode, wantErrToUnicode) 69 | doTest(t, p.ToASCII, name+":ToASCII", src, wantToASCII, wantErrToASCII) 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /internal/export/idna/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package idna_test 6 | 7 | import ( 8 | "fmt" 9 | 10 | "golang.org/x/text/internal/export/idna" 11 | ) 12 | 13 | func ExampleProfile() { 14 | // Raw Punycode has no restrictions and does no mappings. 15 | fmt.Println(idna.ToASCII("")) 16 | fmt.Println(idna.ToASCII("*.GÖPHER.com")) 17 | fmt.Println(idna.Punycode.ToASCII("*.GÖPHER.com")) 18 | 19 | // Rewrite IDN for lookup. 20 | fmt.Println(idna.Lookup.ToASCII("")) 21 | fmt.Println(idna.Lookup.ToASCII("www.GÖPHER.com")) 22 | 23 | // Convert an IDN to ASCII for registration purposes. 24 | // This reports an error if the input was illformed. 25 | fmt.Println(idna.Registration.ToASCII("www.GÖPHER.com")) 26 | fmt.Println(idna.Registration.ToASCII("www.göpher.com")) 27 | 28 | // Output: 29 | // 30 | // *.xn--GPHER-1oa.com 31 | // *.xn--GPHER-1oa.com 32 | // 33 | // www.xn--gpher-jua.com 34 | // www.xn--GPHER-1oa.com idna: disallowed rune U+0047 35 | // www.xn--gpher-jua.com 36 | } 37 | 38 | func ExampleNew() { 39 | var p *idna.Profile 40 | 41 | // Raw Punycode has no restrictions and does no mappings. 42 | p = idna.New() 43 | fmt.Println(p.ToASCII("*.faß.com")) 44 | 45 | // Do mappings. Note that star is not allowed in a DNS lookup. 46 | p = idna.New( 47 | idna.MapForLookup(), 48 | idna.Transitional(true)) // Map ß -> ss 49 | fmt.Println(p.ToASCII("*.faß.com")) 50 | 51 | // Lookup for registration. Also does not allow '*'. 52 | p = idna.New(idna.ValidateForRegistration()) 53 | fmt.Println(p.ToUnicode("*.faß.com")) 54 | 55 | // Set up a profile maps for lookup, but allows wild cards. 56 | p = idna.New( 57 | idna.MapForLookup(), 58 | idna.Transitional(true), // Map ß -> ss 59 | idna.StrictDomainName(false)) // Set more permissive ASCII rules. 60 | fmt.Println(p.ToASCII("*.faß.com")) 61 | 62 | // Output: 63 | // *.xn--fa-hia.com 64 | // *.fass.com idna: disallowed rune U+002A 65 | // *.faß.com idna: disallowed rune U+002A 66 | // *.fass.com 67 | } 68 | -------------------------------------------------------------------------------- /internal/export/idna/gen10.0.0_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build go1.10 6 | 7 | package idna 8 | 9 | import ( 10 | "testing" 11 | "unicode" 12 | 13 | "golang.org/x/text/internal/gen" 14 | "golang.org/x/text/internal/testtext" 15 | "golang.org/x/text/internal/ucd" 16 | ) 17 | 18 | func TestTables(t *testing.T) { 19 | testtext.SkipIfNotLong(t) 20 | 21 | lookup := func(r rune) info { 22 | v, _ := trie.lookupString(string(r)) 23 | return info(v) 24 | } 25 | 26 | ucd.Parse(gen.OpenUnicodeFile("idna", "", "IdnaMappingTable.txt"), func(p *ucd.Parser) { 27 | r := p.Rune(0) 28 | x := lookup(r) 29 | if got, want := x.category(), catFromEntry(p); got != want { 30 | t.Errorf("%U:category: got %x; want %x", r, got, want) 31 | } 32 | 33 | mapped := false 34 | switch p.String(1) { 35 | case "mapped", "disallowed_STD3_mapped", "deviation": 36 | mapped = true 37 | } 38 | if x.isMapped() != mapped { 39 | t.Errorf("%U:isMapped: got %v; want %v", r, x.isMapped(), mapped) 40 | } 41 | if !mapped { 42 | return 43 | } 44 | want := string(p.Runes(2)) 45 | got := string(x.appendMapping(nil, string(r))) 46 | if got != want { 47 | t.Errorf("%U:mapping: got %+q; want %+q", r, got, want) 48 | } 49 | 50 | if x.isMapped() { 51 | return 52 | } 53 | wantMark := unicode.In(r, unicode.Mark) 54 | gotMark := x.isModifier() 55 | if gotMark != wantMark { 56 | t.Errorf("IsMark(%U) = %v; want %v", r, gotMark, wantMark) 57 | } 58 | }) 59 | 60 | ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) { 61 | r := p.Rune(0) 62 | x := lookup(r) 63 | got := x.isViramaModifier() 64 | 65 | const cccVirama = 9 66 | want := p.Int(ucd.CanonicalCombiningClass) == cccVirama 67 | if got != want { 68 | t.Errorf("IsVirama(%U) = %v; want %v", r, got, want) 69 | } 70 | 71 | rtl := false 72 | switch p.String(ucd.BidiClass) { 73 | case "R", "AL", "AN": 74 | rtl = true 75 | } 76 | if got := x.isBidi("A"); got != rtl && !x.isMapped() { 77 | t.Errorf("IsBidi(%U) = %v; want %v", r, got, rtl) 78 | } 79 | }) 80 | 81 | ucd.Parse(gen.OpenUCDFile("extracted/DerivedJoiningType.txt"), func(p *ucd.Parser) { 82 | r := p.Rune(0) 83 | x := lookup(r) 84 | if x.isMapped() { 85 | return 86 | } 87 | got := x.joinType() 88 | want := joinType[p.String(1)] 89 | if got != want { 90 | t.Errorf("JoinType(%U) = %x; want %x", r, got, want) 91 | } 92 | }) 93 | } 94 | -------------------------------------------------------------------------------- /internal/export/idna/gen9.0.0_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !go1.10 6 | 7 | package idna 8 | 9 | import ( 10 | "testing" 11 | "unicode" 12 | 13 | "golang.org/x/text/internal/gen" 14 | "golang.org/x/text/internal/testtext" 15 | "golang.org/x/text/internal/ucd" 16 | ) 17 | 18 | func TestTables(t *testing.T) { 19 | testtext.SkipIfNotLong(t) 20 | 21 | lookup := func(r rune) info { 22 | v, _ := trie.lookupString(string(r)) 23 | return info(v) 24 | } 25 | 26 | ucd.Parse(gen.OpenUnicodeFile("idna", "", "IdnaMappingTable.txt"), func(p *ucd.Parser) { 27 | r := p.Rune(0) 28 | x := lookup(r) 29 | if got, want := x.category(), catFromEntry(p); got != want { 30 | t.Errorf("%U:category: got %x; want %x", r, got, want) 31 | } 32 | 33 | mapped := false 34 | switch p.String(1) { 35 | case "mapped", "disallowed_STD3_mapped", "deviation": 36 | mapped = true 37 | } 38 | if x.isMapped() != mapped { 39 | t.Errorf("%U:isMapped: got %v; want %v", r, x.isMapped(), mapped) 40 | } 41 | if !mapped { 42 | return 43 | } 44 | want := string(p.Runes(2)) 45 | got := string(x.appendMapping(nil, string(r))) 46 | if got != want { 47 | t.Errorf("%U:mapping: got %+q; want %+q", r, got, want) 48 | } 49 | 50 | if x.isMapped() { 51 | return 52 | } 53 | wantMark := unicode.In(r, unicode.Mark) 54 | gotMark := x.isModifier() 55 | if gotMark != wantMark { 56 | t.Errorf("IsMark(%U) = %v; want %v", r, gotMark, wantMark) 57 | } 58 | }) 59 | 60 | ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) { 61 | r := p.Rune(0) 62 | x := lookup(r) 63 | got := x.isViramaModifier() 64 | 65 | const cccVirama = 9 66 | want := p.Int(ucd.CanonicalCombiningClass) == cccVirama 67 | if got != want { 68 | t.Errorf("IsVirama(%U) = %v; want %v", r, got, want) 69 | } 70 | }) 71 | 72 | ucd.Parse(gen.OpenUCDFile("extracted/DerivedJoiningType.txt"), func(p *ucd.Parser) { 73 | r := p.Rune(0) 74 | x := lookup(r) 75 | if x.isMapped() { 76 | return 77 | } 78 | got := x.joinType() 79 | want := joinType[p.String(1)] 80 | if got != want { 81 | t.Errorf("JoinType(%U) = %x; want %x", r, got, want) 82 | } 83 | }) 84 | } 85 | -------------------------------------------------------------------------------- /internal/export/idna/gen_common.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | package main 8 | 9 | // This file contains code that is common between the generation code and the 10 | // package's test code. 11 | 12 | import ( 13 | "log" 14 | 15 | "golang.org/x/text/internal/ucd" 16 | ) 17 | 18 | func catFromEntry(p *ucd.Parser) (cat category) { 19 | r := p.Rune(0) 20 | switch s := p.String(1); s { 21 | case "valid": 22 | cat = valid 23 | case "disallowed": 24 | cat = disallowed 25 | case "disallowed_STD3_valid": 26 | cat = disallowedSTD3Valid 27 | case "disallowed_STD3_mapped": 28 | cat = disallowedSTD3Mapped 29 | case "mapped": 30 | cat = mapped 31 | case "deviation": 32 | cat = deviation 33 | case "ignored": 34 | cat = ignored 35 | default: 36 | log.Fatalf("%U: Unknown category %q", r, s) 37 | } 38 | if s := p.String(3); s != "" { 39 | if cat != valid { 40 | log.Fatalf(`%U: %s defined for %q; want "valid"`, r, s, p.String(1)) 41 | } 42 | switch s { 43 | case "NV8": 44 | cat = validNV8 45 | case "XV8": 46 | cat = validXV8 47 | default: 48 | log.Fatalf("%U: Unexpected exception %q", r, s) 49 | } 50 | } 51 | return cat 52 | } 53 | 54 | var joinType = map[string]info{ 55 | "L": joiningL, 56 | "D": joiningD, 57 | "T": joiningT, 58 | "R": joiningR, 59 | } 60 | -------------------------------------------------------------------------------- /internal/export/idna/go118.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build go1.18 6 | 7 | package idna 8 | 9 | // Transitional processing is disabled by default in Go 1.18. 10 | // https://golang.org/issue/47510 11 | const transitionalLookup = false 12 | -------------------------------------------------------------------------------- /internal/export/idna/pre_go118.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !go1.18 6 | 7 | package idna 8 | 9 | const transitionalLookup = true 10 | -------------------------------------------------------------------------------- /internal/export/idna/trie.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package idna 6 | 7 | // Sparse block handling code. 8 | 9 | type valueRange struct { 10 | value uint16 // header: value:stride 11 | lo, hi byte // header: lo:n 12 | } 13 | 14 | type sparseBlocks struct { 15 | values []valueRange 16 | offset []uint16 17 | } 18 | 19 | var idnaSparse = sparseBlocks{ 20 | values: idnaSparseValues[:], 21 | offset: idnaSparseOffset[:], 22 | } 23 | 24 | // Don't use newIdnaTrie to avoid unconditional linking in of the table. 25 | var trie = &idnaTrie{} 26 | 27 | // lookup determines the type of block n and looks up the value for b. 28 | // For n < t.cutoff, the block is a simple lookup table. Otherwise, the block 29 | // is a list of ranges with an accompanying value. Given a matching range r, 30 | // the value for b is by r.value + (b - r.lo) * stride. 31 | func (t *sparseBlocks) lookup(n uint32, b byte) uint16 { 32 | offset := t.offset[n] 33 | header := t.values[offset] 34 | lo := offset + 1 35 | hi := lo + uint16(header.lo) 36 | for lo < hi { 37 | m := lo + (hi-lo)/2 38 | r := t.values[m] 39 | if r.lo <= b && b <= r.hi { 40 | return r.value + uint16(b-r.lo)*header.value 41 | } 42 | if b < r.lo { 43 | hi = m 44 | } else { 45 | lo = m + 1 46 | } 47 | } 48 | return 0 49 | } 50 | -------------------------------------------------------------------------------- /internal/export/idna/trie12.0.0.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !go1.16 6 | 7 | package idna 8 | 9 | // appendMapping appends the mapping for the respective rune. isMapped must be 10 | // true. A mapping is a categorization of a rune as defined in UTS #46. 11 | func (c info) appendMapping(b []byte, s string) []byte { 12 | index := int(c >> indexShift) 13 | if c&xorBit == 0 { 14 | s := mappings[index:] 15 | return append(b, s[1:s[0]+1]...) 16 | } 17 | b = append(b, s...) 18 | if c&inlineXOR == inlineXOR { 19 | // TODO: support and handle two-byte inline masks 20 | b[len(b)-1] ^= byte(index) 21 | } else { 22 | for p := len(b) - int(xorData[index]); p < len(b); p++ { 23 | index++ 24 | b[p] ^= xorData[index] 25 | } 26 | } 27 | return b 28 | } 29 | -------------------------------------------------------------------------------- /internal/export/idna/trie13.0.0.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build go1.16 6 | 7 | package idna 8 | 9 | // appendMapping appends the mapping for the respective rune. isMapped must be 10 | // true. A mapping is a categorization of a rune as defined in UTS #46. 11 | func (c info) appendMapping(b []byte, s string) []byte { 12 | index := int(c >> indexShift) 13 | if c&xorBit == 0 { 14 | p := index 15 | return append(b, mappings[mappingIndex[p]:mappingIndex[p+1]]...) 16 | } 17 | b = append(b, s...) 18 | if c&inlineXOR == inlineXOR { 19 | // TODO: support and handle two-byte inline masks 20 | b[len(b)-1] ^= byte(index) 21 | } else { 22 | for p := len(b) - int(xorData[index]); p < len(b); p++ { 23 | index++ 24 | b[p] ^= xorData[index] 25 | } 26 | } 27 | return b 28 | } 29 | -------------------------------------------------------------------------------- /internal/export/unicode/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package unicode generates the Unicode tables in core. 6 | package unicode 7 | 8 | // This package is defined here, instead of core, as Go does not allow any 9 | // standard packages to have non-standard imports, even if imported in files 10 | // with a build ignore tag. 11 | 12 | //go:generate go run gen.go -tables=all 13 | //go:generate mv tables.go $GOROOT/src/unicode 14 | -------------------------------------------------------------------------------- /internal/export/unicode/unicode_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package unicode 6 | 7 | import ( 8 | "testing" 9 | "unicode" 10 | 11 | "golang.org/x/text/internal/gen" 12 | "golang.org/x/text/internal/testtext" 13 | "golang.org/x/text/internal/ucd" 14 | ) 15 | 16 | // TestScripts tests for all runes whether they are included in the correct 17 | // script and, indirectly, whether each script exists. 18 | func TestScripts(t *testing.T) { 19 | testtext.SkipIfNotLong(t) 20 | 21 | ucd.Parse(gen.OpenUCDFile("Scripts.txt"), func(p *ucd.Parser) { 22 | r := p.Rune(0) 23 | script := p.String(1) 24 | if !unicode.Is(unicode.Scripts[script], r) { 25 | t.Errorf("%U: not in script %q", r, script) 26 | } 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /internal/format/format.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package format contains types for defining language-specific formatting of 6 | // values. 7 | // 8 | // This package is internal now, but will eventually be exposed after the API 9 | // settles. 10 | package format // import "golang.org/x/text/internal/format" 11 | 12 | import ( 13 | "fmt" 14 | 15 | "golang.org/x/text/language" 16 | ) 17 | 18 | // State represents the printer state passed to custom formatters. It provides 19 | // access to the fmt.State interface and the sentence and language-related 20 | // context. 21 | type State interface { 22 | fmt.State 23 | 24 | // Language reports the requested language in which to render a message. 25 | Language() language.Tag 26 | 27 | // TODO: consider this and removing rune from the Format method in the 28 | // Formatter interface. 29 | // 30 | // Verb returns the format variant to render, analogous to the types used 31 | // in fmt. Use 'v' for the default or only variant. 32 | // Verb() rune 33 | 34 | // TODO: more info: 35 | // - sentence context such as linguistic features passed by the translator. 36 | } 37 | 38 | // Formatter is analogous to fmt.Formatter. 39 | type Formatter interface { 40 | Format(state State, verb rune) 41 | } 42 | -------------------------------------------------------------------------------- /internal/format/parser_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package format 6 | 7 | import "testing" 8 | 9 | // TODO: most of Parser is tested in x/message. Move some tests here. 10 | 11 | func TestParsenum(t *testing.T) { 12 | testCases := []struct { 13 | s string 14 | start, end int 15 | num int 16 | isnum bool 17 | newi int 18 | }{ 19 | {"a123", 0, 4, 0, false, 0}, 20 | {"1234", 1, 1, 0, false, 1}, 21 | {"123a", 0, 4, 123, true, 3}, 22 | {"12a3", 0, 4, 12, true, 2}, 23 | {"1234", 0, 4, 1234, true, 4}, 24 | {"1a234", 1, 3, 0, false, 1}, 25 | } 26 | for _, tt := range testCases { 27 | num, isnum, newi := parsenum(tt.s, tt.start, tt.end) 28 | if num != tt.num || isnum != tt.isnum || newi != tt.newi { 29 | t.Errorf("parsenum(%q, %d, %d) = %d, %v, %d, want %d, %v, %d", tt.s, tt.start, tt.end, num, isnum, newi, tt.num, tt.isnum, tt.newi) 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /internal/gen/bitfield/gen1_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by golang.org/x/text/internal/gen/bitfield. DO NOT EDIT. 2 | 3 | package bitfield 4 | 5 | type myInt uint32 6 | 7 | func (m myInt) fob() uint16 { 8 | return uint16((m >> 16) & 0xffff) 9 | } 10 | 11 | func (m myInt) baz() int8 { 12 | return int8((m >> 11) & 0x1f) 13 | } 14 | 15 | func (m myInt) bar() myUint8 { 16 | return myUint8((m >> 8) & 0x7) 17 | } 18 | 19 | func (m myInt) Bool() bool { 20 | const bit = 1 << 7 21 | return m&bit == bit 22 | } 23 | 24 | func (m myInt) Baz() int8 { 25 | return int8((m >> 4) & 0x7) 26 | } 27 | -------------------------------------------------------------------------------- /internal/gen/bitfield/gen2_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by golang.org/x/text/internal/gen/bitfield. DO NOT EDIT. 2 | 3 | package bitfield 4 | 5 | type myInt2 uint32 6 | 7 | func (m myInt2) fob() uint16 { 8 | return uint16((m >> 12) & 0xffff) 9 | } 10 | 11 | func (m myInt2) baz() int8 { 12 | return int8((m >> 7) & 0x1f) 13 | } 14 | 15 | func (m myInt2) bar() myUint8 { 16 | return myUint8((m >> 4) & 0x7) 17 | } 18 | 19 | func (m myInt2) Bool() bool { 20 | const bit = 1 << 3 21 | return m&bit == bit 22 | } 23 | 24 | func (m myInt2) Baz() int8 { 25 | return int8((m >> 0) & 0x7) 26 | } 27 | -------------------------------------------------------------------------------- /internal/internal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package internal contains non-exported functionality that are used by 6 | // packages in the text repository. 7 | package internal // import "golang.org/x/text/internal" 8 | 9 | import ( 10 | "sort" 11 | 12 | "golang.org/x/text/language" 13 | ) 14 | 15 | // SortTags sorts tags in place. 16 | func SortTags(tags []language.Tag) { 17 | sort.Sort(sorter(tags)) 18 | } 19 | 20 | type sorter []language.Tag 21 | 22 | func (s sorter) Len() int { 23 | return len(s) 24 | } 25 | 26 | func (s sorter) Swap(i, j int) { 27 | s[i], s[j] = s[j], s[i] 28 | } 29 | 30 | func (s sorter) Less(i, j int) bool { 31 | return s[i].String() < s[j].String() 32 | } 33 | 34 | // UniqueTags sorts and filters duplicate tags in place and returns a slice with 35 | // only unique tags. 36 | func UniqueTags(tags []language.Tag) []language.Tag { 37 | if len(tags) <= 1 { 38 | return tags 39 | } 40 | SortTags(tags) 41 | k := 0 42 | for i := 1; i < len(tags); i++ { 43 | if tags[k].String() < tags[i].String() { 44 | k++ 45 | tags[k] = tags[i] 46 | } 47 | } 48 | return tags[:k+1] 49 | } 50 | -------------------------------------------------------------------------------- /internal/internal_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package internal 6 | 7 | import ( 8 | "fmt" 9 | "strings" 10 | "testing" 11 | 12 | "golang.org/x/text/language" 13 | ) 14 | 15 | func TestUnique(t *testing.T) { 16 | testCases := []struct { 17 | in, want string 18 | }{ 19 | {"", "[]"}, 20 | {"en", "[en]"}, 21 | {"en en", "[en]"}, 22 | {"en en en", "[en]"}, 23 | {"en-u-cu-eur en", "[en en-u-cu-eur]"}, 24 | {"nl en", "[en nl]"}, 25 | {"pt-Pt pt", "[pt pt-PT]"}, 26 | } 27 | for _, tc := range testCases { 28 | tags := []language.Tag{} 29 | for _, s := range strings.Split(tc.in, " ") { 30 | if s != "" { 31 | tags = append(tags, language.MustParse(s)) 32 | } 33 | } 34 | if got := fmt.Sprint(UniqueTags(tags)); got != tc.want { 35 | t.Errorf("Unique(%s) = %s; want %s", tc.in, got, tc.want) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /internal/language/common.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | package language 4 | 5 | // This file contains code common to the maketables.go and the package code. 6 | 7 | // AliasType is the type of an alias in AliasMap. 8 | type AliasType int8 9 | 10 | const ( 11 | Deprecated AliasType = iota 12 | Macro 13 | Legacy 14 | 15 | AliasTypeUnknown AliasType = -1 16 | ) 17 | -------------------------------------------------------------------------------- /internal/language/compact.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package language 6 | 7 | // CompactCoreInfo is a compact integer with the three core tags encoded. 8 | type CompactCoreInfo uint32 9 | 10 | // GetCompactCore generates a uint32 value that is guaranteed to be unique for 11 | // different language, region, and script values. 12 | func GetCompactCore(t Tag) (cci CompactCoreInfo, ok bool) { 13 | if t.LangID > langNoIndexOffset { 14 | return 0, false 15 | } 16 | cci |= CompactCoreInfo(t.LangID) << (8 + 12) 17 | cci |= CompactCoreInfo(t.ScriptID) << 12 18 | cci |= CompactCoreInfo(t.RegionID) 19 | return cci, true 20 | } 21 | 22 | // Tag generates a tag from c. 23 | func (c CompactCoreInfo) Tag() Tag { 24 | return Tag{ 25 | LangID: Language(c >> 20), 26 | RegionID: Region(c & 0x3ff), 27 | ScriptID: Script(c>>12) & 0xff, 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /internal/language/compact/compact.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package compact defines a compact representation of language tags. 6 | // 7 | // Common language tags (at least all for which locale information is defined 8 | // in CLDR) are assigned a unique index. Each Tag is associated with such an 9 | // ID for selecting language-related resources (such as translations) as well 10 | // as one for selecting regional defaults (currency, number formatting, etc.) 11 | // 12 | // It may want to export this functionality at some point, but at this point 13 | // this is only available for use within x/text. 14 | package compact // import "golang.org/x/text/internal/language/compact" 15 | 16 | import ( 17 | "sort" 18 | "strings" 19 | 20 | "golang.org/x/text/internal/language" 21 | ) 22 | 23 | // ID is an integer identifying a single tag. 24 | type ID uint16 25 | 26 | func getCoreIndex(t language.Tag) (id ID, ok bool) { 27 | cci, ok := language.GetCompactCore(t) 28 | if !ok { 29 | return 0, false 30 | } 31 | i := sort.Search(len(coreTags), func(i int) bool { 32 | return cci <= coreTags[i] 33 | }) 34 | if i == len(coreTags) || coreTags[i] != cci { 35 | return 0, false 36 | } 37 | return ID(i), true 38 | } 39 | 40 | // Parent returns the ID of the parent or the root ID if id is already the root. 41 | func (id ID) Parent() ID { 42 | return parents[id] 43 | } 44 | 45 | // Tag converts id to an internal language Tag. 46 | func (id ID) Tag() language.Tag { 47 | if int(id) >= len(coreTags) { 48 | return specialTags[int(id)-len(coreTags)] 49 | } 50 | return coreTags[id].Tag() 51 | } 52 | 53 | var specialTags []language.Tag 54 | 55 | func init() { 56 | tags := strings.Split(specialTagsStr, " ") 57 | specialTags = make([]language.Tag, len(tags)) 58 | for i, t := range tags { 59 | specialTags[i] = language.MustParse(t) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /internal/language/compact/gen.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | // Language tag table generator. 8 | // Data read from the web. 9 | 10 | package main 11 | 12 | import ( 13 | "flag" 14 | "fmt" 15 | "log" 16 | 17 | "golang.org/x/text/internal/gen" 18 | "golang.org/x/text/unicode/cldr" 19 | ) 20 | 21 | var ( 22 | test = flag.Bool("test", 23 | false, 24 | "test existing tables; can be used to compare web data with package data.") 25 | outputFile = flag.String("output", 26 | "tables.go", 27 | "output file for generated tables") 28 | ) 29 | 30 | func main() { 31 | gen.Init() 32 | 33 | w := gen.NewCodeWriter() 34 | defer w.WriteGoFile("tables.go", "compact") 35 | 36 | fmt.Fprintln(w, `import "golang.org/x/text/internal/language"`) 37 | 38 | b := newBuilder(w) 39 | gen.WriteCLDRVersion(w) 40 | 41 | b.writeCompactIndex() 42 | } 43 | 44 | type builder struct { 45 | w *gen.CodeWriter 46 | data *cldr.CLDR 47 | supp *cldr.SupplementalData 48 | } 49 | 50 | func newBuilder(w *gen.CodeWriter) *builder { 51 | r := gen.OpenCLDRCoreZip() 52 | defer r.Close() 53 | d := &cldr.Decoder{} 54 | data, err := d.DecodeZip(r) 55 | if err != nil { 56 | log.Fatal(err) 57 | } 58 | b := builder{ 59 | w: w, 60 | data: data, 61 | supp: data.Supplemental(), 62 | } 63 | return &b 64 | } 65 | -------------------------------------------------------------------------------- /internal/language/compact/gen_parents.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | package main 8 | 9 | import ( 10 | "log" 11 | 12 | "golang.org/x/text/internal/gen" 13 | "golang.org/x/text/internal/language" 14 | "golang.org/x/text/internal/language/compact" 15 | "golang.org/x/text/unicode/cldr" 16 | ) 17 | 18 | func main() { 19 | r := gen.OpenCLDRCoreZip() 20 | defer r.Close() 21 | 22 | d := &cldr.Decoder{} 23 | data, err := d.DecodeZip(r) 24 | if err != nil { 25 | log.Fatalf("DecodeZip: %v", err) 26 | } 27 | 28 | w := gen.NewCodeWriter() 29 | defer w.WriteGoFile("parents.go", "compact") 30 | 31 | // Create parents table. 32 | type ID uint16 33 | parents := make([]ID, compact.NumCompactTags) 34 | for _, loc := range data.Locales() { 35 | tag := language.MustParse(loc) 36 | index, ok := compact.FromTag(tag) 37 | if !ok { 38 | continue 39 | } 40 | parentIndex := compact.ID(0) // und 41 | for p := tag.Parent(); p != language.Und; p = p.Parent() { 42 | if x, ok := compact.FromTag(p); ok { 43 | parentIndex = x 44 | break 45 | } 46 | } 47 | parents[index] = ID(parentIndex) 48 | } 49 | 50 | w.WriteComment(` 51 | parents maps a compact index of a tag to the compact index of the parent of 52 | this tag.`) 53 | w.WriteVar("parents", parents) 54 | } 55 | -------------------------------------------------------------------------------- /internal/language/compact/gen_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package compact 6 | 7 | import ( 8 | "testing" 9 | 10 | "golang.org/x/text/internal/language" 11 | ) 12 | 13 | func TestParents(t *testing.T) { 14 | testCases := []struct { 15 | tag, parent string 16 | }{ 17 | {"af", "und"}, 18 | {"en", "und"}, 19 | {"en-001", "en"}, 20 | {"en-AU", "en-001"}, 21 | {"en-US", "en"}, 22 | {"en-US-u-va-posix", "en-US"}, 23 | {"ca-ES-valencia", "ca-ES"}, 24 | } 25 | for _, tc := range testCases { 26 | tag, ok := LanguageID(Make(language.MustParse(tc.tag))) 27 | if !ok { 28 | t.Fatalf("Could not get index of flag %s", tc.tag) 29 | } 30 | want, ok := LanguageID(Make(language.MustParse(tc.parent))) 31 | if !ok { 32 | t.Fatalf("Could not get index of parent %s of tag %s", tc.parent, tc.tag) 33 | } 34 | if got := parents[tag]; got != want { 35 | t.Errorf("Parent[%s] = %d; want %d (%s)", tc.tag, got, want, tc.parent) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /internal/language/compose_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package language 6 | 7 | import ( 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func parseBase(s string) Language { 13 | if s == "" { 14 | return 0 15 | } 16 | return MustParseBase(s) 17 | } 18 | 19 | func parseScript(s string) Script { 20 | if s == "" { 21 | return 0 22 | } 23 | return MustParseScript(s) 24 | } 25 | 26 | func parseRegion(s string) Region { 27 | if s == "" { 28 | return 0 29 | } 30 | return MustParseRegion(s) 31 | } 32 | 33 | func TestBuilder(t *testing.T) { 34 | partChecks(t, func(t *testing.T, tt *parseTest) (id Tag, skip bool) { 35 | tag := Make(tt.in) 36 | b := Builder{} 37 | b.SetTag(Tag{ 38 | LangID: parseBase(tt.lang), 39 | ScriptID: parseScript(tt.script), 40 | RegionID: parseRegion(tt.region), 41 | }) 42 | if tt.variants != "" { 43 | b.AddVariant(strings.Split(tt.variants, "-")...) 44 | } 45 | for _, e := range tag.Extensions() { 46 | b.AddExt(e) 47 | } 48 | got := b.Make() 49 | if got != tag { 50 | t.Errorf("%s: got %v; want %v", tt.in, got, tag) 51 | } 52 | return got, false 53 | }) 54 | } 55 | 56 | func TestSetTag(t *testing.T) { 57 | partChecks(t, func(t *testing.T, tt *parseTest) (id Tag, skip bool) { 58 | tag := Make(tt.in) 59 | b := Builder{} 60 | b.SetTag(tag) 61 | got := b.Make() 62 | if got != tag { 63 | t.Errorf("%s: got %v; want %v", tt.in, got, tag) 64 | } 65 | return got, false 66 | }) 67 | } 68 | -------------------------------------------------------------------------------- /internal/language/coverage.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package language 6 | 7 | // BaseLanguages returns the list of all supported base languages. It generates 8 | // the list by traversing the internal structures. 9 | func BaseLanguages() []Language { 10 | base := make([]Language, 0, NumLanguages) 11 | for i := 0; i < langNoIndexOffset; i++ { 12 | // We included "und" already for the value 0. 13 | if i != nonCanonicalUnd { 14 | base = append(base, Language(i)) 15 | } 16 | } 17 | i := langNoIndexOffset 18 | for _, v := range langNoIndex { 19 | for k := 0; k < 8; k++ { 20 | if v&1 == 1 { 21 | base = append(base, Language(i)) 22 | } 23 | v >>= 1 24 | i++ 25 | } 26 | } 27 | return base 28 | } 29 | -------------------------------------------------------------------------------- /internal/language/gen_common.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | package main 8 | 9 | // This file contains code common to the maketables.go and the package code. 10 | 11 | // AliasType is the type of an alias in AliasMap. 12 | type AliasType int8 13 | 14 | const ( 15 | Deprecated AliasType = iota 16 | Macro 17 | Legacy 18 | 19 | AliasTypeUnknown AliasType = -1 20 | ) 21 | -------------------------------------------------------------------------------- /internal/language/tags.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package language 6 | 7 | // MustParse is like Parse, but panics if the given BCP 47 tag cannot be parsed. 8 | // It simplifies safe initialization of Tag values. 9 | func MustParse(s string) Tag { 10 | t, err := Parse(s) 11 | if err != nil { 12 | panic(err) 13 | } 14 | return t 15 | } 16 | 17 | // MustParseBase is like ParseBase, but panics if the given base cannot be parsed. 18 | // It simplifies safe initialization of Base values. 19 | func MustParseBase(s string) Language { 20 | b, err := ParseBase(s) 21 | if err != nil { 22 | panic(err) 23 | } 24 | return b 25 | } 26 | 27 | // MustParseScript is like ParseScript, but panics if the given script cannot be 28 | // parsed. It simplifies safe initialization of Script values. 29 | func MustParseScript(s string) Script { 30 | scr, err := ParseScript(s) 31 | if err != nil { 32 | panic(err) 33 | } 34 | return scr 35 | } 36 | 37 | // MustParseRegion is like ParseRegion, but panics if the given region cannot be 38 | // parsed. It simplifies safe initialization of Region values. 39 | func MustParseRegion(s string) Region { 40 | r, err := ParseRegion(s) 41 | if err != nil { 42 | panic(err) 43 | } 44 | return r 45 | } 46 | 47 | // Und is the root language. 48 | var Und Tag 49 | -------------------------------------------------------------------------------- /internal/match.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package internal 6 | 7 | // This file contains matchers that implement CLDR inheritance. 8 | // 9 | // See https://unicode.org/reports/tr35/#Locale_Inheritance. 10 | // 11 | // Some of the inheritance described in this document is already handled by 12 | // the cldr package. 13 | 14 | import ( 15 | "golang.org/x/text/language" 16 | ) 17 | 18 | // TODO: consider if (some of the) matching algorithm needs to be public after 19 | // getting some feel about what is generic and what is specific. 20 | 21 | // NewInheritanceMatcher returns a matcher that matches based on the inheritance 22 | // chain. 23 | // 24 | // The matcher uses canonicalization and the parent relationship to find a 25 | // match. The resulting match will always be either Und or a language with the 26 | // same language and script as the requested language. It will not match 27 | // languages for which there is understood to be mutual or one-directional 28 | // intelligibility. 29 | // 30 | // A Match will indicate an Exact match if the language matches after 31 | // canonicalization and High if the matched tag is a parent. 32 | func NewInheritanceMatcher(t []language.Tag) *InheritanceMatcher { 33 | tags := &InheritanceMatcher{make(map[language.Tag]int)} 34 | for i, tag := range t { 35 | ct, err := language.All.Canonicalize(tag) 36 | if err != nil { 37 | ct = tag 38 | } 39 | tags.index[ct] = i 40 | } 41 | return tags 42 | } 43 | 44 | type InheritanceMatcher struct { 45 | index map[language.Tag]int 46 | } 47 | 48 | func (m InheritanceMatcher) Match(want ...language.Tag) (language.Tag, int, language.Confidence) { 49 | for _, t := range want { 50 | ct, err := language.All.Canonicalize(t) 51 | if err != nil { 52 | ct = t 53 | } 54 | conf := language.Exact 55 | for { 56 | if index, ok := m.index[ct]; ok { 57 | return ct, index, conf 58 | } 59 | if ct == language.Und { 60 | break 61 | } 62 | ct = ct.Parent() 63 | conf = language.High 64 | } 65 | } 66 | return language.Und, 0, language.No 67 | } 68 | -------------------------------------------------------------------------------- /internal/match_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package internal 6 | 7 | import ( 8 | "strings" 9 | "testing" 10 | 11 | "golang.org/x/text/language" 12 | ) 13 | 14 | func TestInheritanceMatcher(t *testing.T) { 15 | for i, tt := range []struct { 16 | haveTags string 17 | wantTags string 18 | match string 19 | conf language.Confidence 20 | }{ 21 | {"und,en,en-US", "en-US", "en-US", language.Exact}, // most specific match 22 | {"zh-Hant,zh", "zh-TW", "zh-Hant", language.High}, // zh-TW implies Hant. 23 | {"und,zh", "zh-TW", "und", language.High}, // zh-TW does not match zh. 24 | {"zh", "zh-TW", "und", language.No}, // zh-TW does not match zh. 25 | {"iw,en,nl", "he", "he", language.Exact}, // matches after canonicalization 26 | {"he,en,nl", "iw", "he", language.Exact}, // matches after canonicalization 27 | // Prefer first match over more specific match for various reasons: 28 | // a) consistency of user interface is more important than an exact match, 29 | // b) _if_ und is specified, it should be considered a correct and useful match, 30 | // Note that a call to this Match will almost always be with a single tag. 31 | {"und,en,en-US", "he,en-US", "und", language.High}, 32 | } { 33 | have := parseTags(tt.haveTags) 34 | m := NewInheritanceMatcher(have) 35 | tag, index, conf := m.Match(parseTags(tt.wantTags)...) 36 | want := language.Raw.Make(tt.match) 37 | if tag != want { 38 | t.Errorf("%d:tag: got %q; want %q", i, tag, want) 39 | } 40 | if conf != language.No { 41 | if got, _ := language.All.Canonicalize(have[index]); got != want { 42 | t.Errorf("%d:index: got %q; want %q ", i, got, want) 43 | } 44 | } 45 | if conf != tt.conf { 46 | t.Errorf("%d:conf: got %v; want %v", i, conf, tt.conf) 47 | } 48 | } 49 | } 50 | 51 | func parseTags(list string) (out []language.Tag) { 52 | for _, s := range strings.Split(list, ",") { 53 | out = append(out, language.Raw.Make(strings.TrimSpace(s))) 54 | } 55 | return out 56 | } 57 | -------------------------------------------------------------------------------- /internal/number/common.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | package number 4 | 5 | import ( 6 | "unicode/utf8" 7 | 8 | "golang.org/x/text/internal/language/compact" 9 | ) 10 | 11 | // A system identifies a CLDR numbering system. 12 | type system byte 13 | 14 | type systemData struct { 15 | id system 16 | digitSize byte // number of UTF-8 bytes per digit 17 | zero [utf8.UTFMax]byte // UTF-8 sequence of zero digit. 18 | } 19 | 20 | // A SymbolType identifies a symbol of a specific kind. 21 | type SymbolType int 22 | 23 | const ( 24 | SymDecimal SymbolType = iota 25 | SymGroup 26 | SymList 27 | SymPercentSign 28 | SymPlusSign 29 | SymMinusSign 30 | SymExponential 31 | SymSuperscriptingExponent 32 | SymPerMille 33 | SymInfinity 34 | SymNan 35 | SymTimeSeparator 36 | 37 | NumSymbolTypes 38 | ) 39 | 40 | const hasNonLatnMask = 0x8000 41 | 42 | // symOffset is an offset into altSymData if the bit indicated by hasNonLatnMask 43 | // is not 0 (with this bit masked out), and an offset into symIndex otherwise. 44 | // 45 | // TODO: this type can be a byte again if we use an indirection into altsymData 46 | // and introduce an alt -> offset slice (the length of this will be number of 47 | // alternatives plus 1). This also allows getting rid of the compactTag field 48 | // in altSymData. In total this will save about 1K. 49 | type symOffset uint16 50 | 51 | type altSymData struct { 52 | compactTag compact.ID 53 | symIndex symOffset 54 | system system 55 | } 56 | -------------------------------------------------------------------------------- /internal/number/gen_common.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | package main 8 | 9 | import ( 10 | "unicode/utf8" 11 | 12 | "golang.org/x/text/internal/language/compact" 13 | ) 14 | 15 | // A system identifies a CLDR numbering system. 16 | type system byte 17 | 18 | type systemData struct { 19 | id system 20 | digitSize byte // number of UTF-8 bytes per digit 21 | zero [utf8.UTFMax]byte // UTF-8 sequence of zero digit. 22 | } 23 | 24 | // A SymbolType identifies a symbol of a specific kind. 25 | type SymbolType int 26 | 27 | const ( 28 | SymDecimal SymbolType = iota 29 | SymGroup 30 | SymList 31 | SymPercentSign 32 | SymPlusSign 33 | SymMinusSign 34 | SymExponential 35 | SymSuperscriptingExponent 36 | SymPerMille 37 | SymInfinity 38 | SymNan 39 | SymTimeSeparator 40 | 41 | NumSymbolTypes 42 | ) 43 | 44 | const hasNonLatnMask = 0x8000 45 | 46 | // symOffset is an offset into altSymData if the bit indicated by hasNonLatnMask 47 | // is not 0 (with this bit masked out), and an offset into symIndex otherwise. 48 | // 49 | // TODO: this type can be a byte again if we use an indirection into altsymData 50 | // and introduce an alt -> offset slice (the length of this will be number of 51 | // alternatives plus 1). This also allows getting rid of the compactTag field 52 | // in altSymData. In total this will save about 1K. 53 | type symOffset uint16 54 | 55 | type altSymData struct { 56 | compactTag compact.ID 57 | symIndex symOffset 58 | system system 59 | } 60 | -------------------------------------------------------------------------------- /internal/number/roundingmode_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type RoundingMode"; DO NOT EDIT. 2 | 3 | package number 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[ToNearestEven-0] 12 | _ = x[ToNearestZero-1] 13 | _ = x[ToNearestAway-2] 14 | _ = x[ToPositiveInf-3] 15 | _ = x[ToNegativeInf-4] 16 | _ = x[ToZero-5] 17 | _ = x[AwayFromZero-6] 18 | _ = x[numModes-7] 19 | } 20 | 21 | const _RoundingMode_name = "ToNearestEvenToNearestZeroToNearestAwayToPositiveInfToNegativeInfToZeroAwayFromZeronumModes" 22 | 23 | var _RoundingMode_index = [...]uint8{0, 13, 26, 39, 52, 65, 71, 83, 91} 24 | 25 | func (i RoundingMode) String() string { 26 | if i >= RoundingMode(len(_RoundingMode_index)-1) { 27 | return "RoundingMode(" + strconv.FormatInt(int64(i), 10) + ")" 28 | } 29 | return _RoundingMode_name[_RoundingMode_index[i]:_RoundingMode_index[i+1]] 30 | } 31 | -------------------------------------------------------------------------------- /internal/stringset/set.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package stringset provides a way to represent a collection of strings 6 | // compactly. 7 | package stringset 8 | 9 | import "sort" 10 | 11 | // A Set holds a collection of strings that can be looked up by an index number. 12 | type Set struct { 13 | // These fields are exported to allow for code generation. 14 | 15 | Data string 16 | Index []uint16 17 | } 18 | 19 | // Elem returns the string with index i. It panics if i is out of range. 20 | func (s *Set) Elem(i int) string { 21 | return s.Data[s.Index[i]:s.Index[i+1]] 22 | } 23 | 24 | // Len returns the number of strings in the set. 25 | func (s *Set) Len() int { 26 | return len(s.Index) - 1 27 | } 28 | 29 | // Search returns the index of the given string or -1 if it is not in the set. 30 | // The Set must have been created with strings in sorted order. 31 | func Search(s *Set, str string) int { 32 | // TODO: optimize this if it gets used a lot. 33 | n := len(s.Index) - 1 34 | p := sort.Search(n, func(i int) bool { 35 | return s.Elem(i) >= str 36 | }) 37 | if p == n || str != s.Elem(p) { 38 | return -1 39 | } 40 | return p 41 | } 42 | 43 | // A Builder constructs Sets. 44 | type Builder struct { 45 | set Set 46 | index map[string]int 47 | } 48 | 49 | // NewBuilder returns a new and initialized Builder. 50 | func NewBuilder() *Builder { 51 | return &Builder{ 52 | set: Set{ 53 | Index: []uint16{0}, 54 | }, 55 | index: map[string]int{}, 56 | } 57 | } 58 | 59 | // Set creates the set created so far. 60 | func (b *Builder) Set() Set { 61 | return b.set 62 | } 63 | 64 | // Index returns the index for the given string, which must have been added 65 | // before. 66 | func (b *Builder) Index(s string) int { 67 | return b.index[s] 68 | } 69 | 70 | // Add adds a string to the index. Strings that are added by a single Add will 71 | // be stored together, unless they match an existing string. 72 | func (b *Builder) Add(ss ...string) { 73 | // First check if the string already exists. 74 | for _, s := range ss { 75 | if _, ok := b.index[s]; ok { 76 | continue 77 | } 78 | b.index[s] = len(b.set.Index) - 1 79 | b.set.Data += s 80 | x := len(b.set.Data) 81 | if x > 0xFFFF { 82 | panic("Index too > 0xFFFF") 83 | } 84 | b.set.Index = append(b.set.Index, uint16(x)) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /internal/stringset/set_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package stringset 6 | 7 | import "testing" 8 | 9 | func TestStringSet(t *testing.T) { 10 | testCases := [][]string{ 11 | {""}, 12 | {"∫"}, 13 | {"a", "b", "c"}, 14 | {"", "a", "bb", "ccc"}, 15 | {" ", "aaa", "bb", "c"}, 16 | } 17 | test := func(tc int, b *Builder) { 18 | set := b.Set() 19 | if set.Len() != len(testCases[tc]) { 20 | t.Errorf("%d:Len() = %d; want %d", tc, set.Len(), len(testCases[tc])) 21 | } 22 | for i, s := range testCases[tc] { 23 | if x := b.Index(s); x != i { 24 | t.Errorf("%d:Index(%q) = %d; want %d", tc, s, x, i) 25 | } 26 | if p := Search(&set, s); p != i { 27 | t.Errorf("%d:Search(%q) = %d; want %d", tc, s, p, i) 28 | } 29 | if set.Elem(i) != s { 30 | t.Errorf("%d:Elem(%d) = %s; want %s", tc, i, set.Elem(i), s) 31 | } 32 | } 33 | if p := Search(&set, "apple"); p != -1 { 34 | t.Errorf(`%d:Search("apple") = %d; want -1`, tc, p) 35 | } 36 | } 37 | for i, tc := range testCases { 38 | b := NewBuilder() 39 | for _, s := range tc { 40 | b.Add(s) 41 | } 42 | b.Add(tc...) 43 | test(i, b) 44 | } 45 | for i, tc := range testCases { 46 | b := NewBuilder() 47 | b.Add(tc...) 48 | for _, s := range tc { 49 | b.Add(s) 50 | } 51 | test(i, b) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /internal/tag/tag_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tag 6 | 7 | import ( 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | var strdata = []string{ 13 | "aa ", 14 | "aaa ", 15 | "aaaa", 16 | "aaab", 17 | "aab ", 18 | "ab ", 19 | "ba ", 20 | "xxxx", 21 | "\xff\xff\xff\xff", 22 | } 23 | 24 | var testCases = map[string]int{ 25 | "a": 0, 26 | "aa": 0, 27 | "aaa": 1, 28 | "aa ": 0, 29 | "aaaa": 2, 30 | "aaab": 3, 31 | "b": 6, 32 | "ba": 6, 33 | " ": -1, 34 | "aaax": -1, 35 | "bbbb": -1, 36 | "zzzz": -1, 37 | } 38 | 39 | func TestIndex(t *testing.T) { 40 | index := Index(strings.Join(strdata, "")) 41 | for k, v := range testCases { 42 | if i := index.Index([]byte(k)); i != v { 43 | t.Errorf("%s: got %d; want %d", k, i, v) 44 | } 45 | } 46 | } 47 | 48 | func TestFixCase(t *testing.T) { 49 | tests := []string{ 50 | "aaaa", "AbCD", "abcd", 51 | "Zzzz", "AbCD", "Abcd", 52 | "Zzzz", "AbC", "", 53 | "XXX", "ab ", "", 54 | "XXX", "usd", "USD", 55 | "cmn", "AB ", "", 56 | "gsw", "CMN", "cmn", 57 | } 58 | for tc := tests; len(tc) > 0; tc = tc[3:] { 59 | b := []byte(tc[1]) 60 | if !FixCase(tc[0], b) { 61 | b = nil 62 | } 63 | if string(b) != tc[2] { 64 | t.Errorf("FixCase(%q, %q) = %q; want %q", tc[0], tc[1], b, tc[2]) 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /internal/testtext/codesize.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package testtext 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "os" 11 | "os/exec" 12 | "path/filepath" 13 | "runtime" 14 | ) 15 | 16 | // CodeSize builds the given code sample and returns the binary size or en error 17 | // if an error occurred. The code sample typically will look like this: 18 | // 19 | // package main 20 | // import "golang.org/x/text/somepackage" 21 | // func main() { 22 | // somepackage.Func() // reference Func to cause it to be linked in. 23 | // } 24 | // 25 | // See dict_test.go in the display package for an example. 26 | func CodeSize(s string) (int, error) { 27 | // Write the file. 28 | tmpdir, err := os.MkdirTemp(os.TempDir(), "testtext") 29 | if err != nil { 30 | return 0, fmt.Errorf("testtext: failed to create tmpdir: %v", err) 31 | } 32 | defer os.RemoveAll(tmpdir) 33 | filename := filepath.Join(tmpdir, "main.go") 34 | if err := os.WriteFile(filename, []byte(s), 0644); err != nil { 35 | return 0, fmt.Errorf("testtext: failed to write main.go: %v", err) 36 | } 37 | 38 | // Build the binary. 39 | w := &bytes.Buffer{} 40 | cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", "main") 41 | cmd.Dir = tmpdir 42 | cmd.Stderr = w 43 | cmd.Stdout = w 44 | if err := cmd.Run(); err != nil { 45 | return 0, fmt.Errorf("testtext: failed to execute command: %v\nmain.go:\n%vErrors:%s", err, s, w) 46 | } 47 | 48 | // Determine the size. 49 | fi, err := os.Stat(filepath.Join(tmpdir, "main")) 50 | if err != nil { 51 | return 0, fmt.Errorf("testtext: failed to get file info: %v", err) 52 | } 53 | return int(fi.Size()), nil 54 | } 55 | -------------------------------------------------------------------------------- /internal/testtext/flag.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package testtext 6 | 7 | import ( 8 | "flag" 9 | "testing" 10 | 11 | "golang.org/x/text/internal/gen" 12 | ) 13 | 14 | var long = flag.Bool("long", false, 15 | "run tests that require fetching data online") 16 | 17 | // SkipIfNotLong returns whether long tests should be performed. 18 | func SkipIfNotLong(t *testing.T) { 19 | if testing.Short() || !(gen.IsLocal() || *long) { 20 | t.Skip("skipping test to prevent downloading; to run use -long or use -local or UNICODE_DIR to specify a local source") 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /internal/testtext/gc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !gccgo 6 | 7 | package testtext 8 | 9 | import "testing" 10 | 11 | // AllocsPerRun wraps testing.AllocsPerRun. 12 | func AllocsPerRun(runs int, f func()) (avg float64) { 13 | return testing.AllocsPerRun(runs, f) 14 | } 15 | -------------------------------------------------------------------------------- /internal/testtext/gccgo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build gccgo 6 | 7 | package testtext 8 | 9 | // AllocsPerRun always returns 0 for gccgo until gccgo implements escape 10 | // analysis equal or better to that of gc. 11 | func AllocsPerRun(runs int, f func()) (avg float64) { return 0 } 12 | -------------------------------------------------------------------------------- /internal/triegen/compact.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package triegen 6 | 7 | // This file defines Compacter and its implementations. 8 | 9 | import "io" 10 | 11 | // A Compacter generates an alternative, more space-efficient way to store a 12 | // trie value block. A trie value block holds all possible values for the last 13 | // byte of a UTF-8 encoded rune. Excluding ASCII characters, a trie value block 14 | // always has 64 values, as a UTF-8 encoding ends with a byte in [0x80, 0xC0). 15 | type Compacter interface { 16 | // Size returns whether the Compacter could encode the given block as well 17 | // as its size in case it can. len(v) is always 64. 18 | Size(v []uint64) (sz int, ok bool) 19 | 20 | // Store stores the block using the Compacter's compression method. 21 | // It returns a handle with which the block can be retrieved. 22 | // len(v) is always 64. 23 | Store(v []uint64) uint32 24 | 25 | // Print writes the data structures associated to the given store to w. 26 | Print(w io.Writer) error 27 | 28 | // Handler returns the name of a function that gets called during trie 29 | // lookup for blocks generated by the Compacter. The function should be of 30 | // the form func (n uint32, b byte) uint64, where n is the index returned by 31 | // the Compacter's Store method and b is the last byte of the UTF-8 32 | // encoding, where 0x80 <= b < 0xC0, for which to do the lookup in the 33 | // block. 34 | Handler() string 35 | } 36 | 37 | // simpleCompacter is the default Compacter used by builder. It implements a 38 | // normal trie block. 39 | type simpleCompacter builder 40 | 41 | func (b *simpleCompacter) Size([]uint64) (sz int, ok bool) { 42 | return blockSize * b.ValueSize, true 43 | } 44 | 45 | func (b *simpleCompacter) Store(v []uint64) uint32 { 46 | h := uint32(len(b.ValueBlocks) - blockOffset) 47 | b.ValueBlocks = append(b.ValueBlocks, v) 48 | return h 49 | } 50 | 51 | func (b *simpleCompacter) Print(io.Writer) error { 52 | // Structures are printed in print.go. 53 | return nil 54 | } 55 | 56 | func (b *simpleCompacter) Handler() string { 57 | panic("Handler should be special-cased for this Compacter") 58 | } 59 | -------------------------------------------------------------------------------- /internal/triegen/example_compact_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package triegen_test 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | 11 | "golang.org/x/text/internal/triegen" 12 | ) 13 | 14 | func ExampleCompacter() { 15 | t := triegen.NewTrie("root") 16 | for r := rune(0); r < 10000; r += 64 { 17 | t.Insert(r, 0x9015BADA55^uint64(r)) 18 | } 19 | sz, _ := t.Gen(io.Discard) 20 | 21 | fmt.Printf("Size normal: %5d\n", sz) 22 | 23 | var c myCompacter 24 | sz, _ = t.Gen(io.Discard, triegen.Compact(&c)) 25 | 26 | fmt.Printf("Size compacted: %5d\n", sz) 27 | 28 | // Output: 29 | // Size normal: 81344 30 | // Size compacted: 3224 31 | } 32 | 33 | // A myCompacter accepts a block if only the first value is given. 34 | type myCompacter []uint64 35 | 36 | func (c *myCompacter) Size(values []uint64) (sz int, ok bool) { 37 | for _, v := range values[1:] { 38 | if v != 0 { 39 | return 0, false 40 | } 41 | } 42 | return 8, true // the size of a uint64 43 | } 44 | 45 | func (c *myCompacter) Store(v []uint64) uint32 { 46 | x := uint32(len(*c)) 47 | *c = append(*c, v[0]) 48 | return x 49 | } 50 | 51 | func (c *myCompacter) Print(w io.Writer) error { 52 | fmt.Fprintln(w, "var firstValue = []uint64{") 53 | for _, v := range *c { 54 | fmt.Fprintf(w, "\t%#x,\n", v) 55 | } 56 | fmt.Fprintln(w, "}") 57 | return nil 58 | } 59 | 60 | func (c *myCompacter) Handler() string { 61 | return "getFirstValue" 62 | 63 | // Where getFirstValue is included along with the generated code: 64 | // func getFirstValue(n uint32, b byte) uint64 { 65 | // if b == 0x80 { // the first continuation byte 66 | // return firstValue[n] 67 | // } 68 | // return 0 69 | // } 70 | } 71 | -------------------------------------------------------------------------------- /internal/triegen/gen_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build generate 6 | 7 | package triegen_test 8 | 9 | // The code in this file generates captures and writes the tries generated in 10 | // the examples to data_test.go. To invoke it, run: 11 | // go test -tags=generate 12 | // 13 | // Making the generation code a "test" allows us to link in the necessary test 14 | // code. 15 | 16 | import ( 17 | "log" 18 | "os" 19 | "os/exec" 20 | ) 21 | 22 | func init() { 23 | const tmpfile = "tmpout" 24 | const dstfile = "data_test.go" 25 | 26 | f, err := os.Create(tmpfile) 27 | if err != nil { 28 | log.Fatalf("Could not create output file: %v", err) 29 | } 30 | defer os.Remove(tmpfile) 31 | defer f.Close() 32 | 33 | // We exit before this function returns, regardless of success or failure, 34 | // so there's no need to save (and later restore) the existing genWriter 35 | // value. 36 | genWriter = f 37 | 38 | f.Write([]byte(header)) 39 | 40 | Example_build() 41 | ExampleGen_build() 42 | 43 | if err := exec.Command("gofmt", "-w", tmpfile).Run(); err != nil { 44 | log.Fatal(err) 45 | } 46 | os.Remove(dstfile) 47 | os.Rename(tmpfile, dstfile) 48 | 49 | os.Exit(0) 50 | } 51 | 52 | const header = `// This file is generated with "go test -tags generate". DO NOT EDIT! 53 | //go:build !generate 54 | 55 | package triegen_test 56 | ` 57 | 58 | // Stubs for generated tries. These are needed as we exclude data_test.go if 59 | // the generate flag is set. This will clearly make the tests fail, but that 60 | // is okay. It allows us to bootstrap. 61 | 62 | type trie struct{} 63 | 64 | func (t *trie) lookupString(string) (uint8, int) { return 0, 1 } 65 | func (t *trie) lookupStringUnsafe(string) uint64 { return 0 } 66 | 67 | func newRandTrie(i int) *trie { return &trie{} } 68 | func newMultiTrie(i int) *trie { return &trie{} } 69 | -------------------------------------------------------------------------------- /language/display/dict_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package display 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | 11 | "golang.org/x/text/internal/testtext" 12 | ) 13 | 14 | func TestLinking(t *testing.T) { 15 | base := getSize(t, `display.Tags(language.English).Name(language.English)`) 16 | compact := getSize(t, `display.English.Languages().Name(language.English)`) 17 | 18 | if d := base - compact; d < 1.5*1024*1024 { 19 | t.Errorf("size(base) - size(compact) = %d - %d = was %d; want > 1.5MB", base, compact, d) 20 | } 21 | } 22 | 23 | func getSize(t *testing.T, main string) int { 24 | size, err := testtext.CodeSize(fmt.Sprintf(body, main)) 25 | if err != nil { 26 | t.Skipf("skipping link size test; binary size could not be determined: %v", err) 27 | } 28 | return size 29 | } 30 | 31 | const body = `package main 32 | import ( 33 | "golang.org/x/text/language" 34 | "golang.org/x/text/language/display" 35 | ) 36 | func main() { 37 | %s 38 | } 39 | ` 40 | -------------------------------------------------------------------------------- /language/httpexample_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package language_test 6 | 7 | import ( 8 | "fmt" 9 | "net/http" 10 | "strings" 11 | 12 | "golang.org/x/text/language" 13 | ) 14 | 15 | // matcher is a language.Matcher configured for all supported languages. 16 | var matcher = language.NewMatcher([]language.Tag{ 17 | language.BritishEnglish, 18 | language.Norwegian, 19 | language.German, 20 | }) 21 | 22 | // handler is an http.HandlerFunc. 23 | func handler(w http.ResponseWriter, r *http.Request) { 24 | t, q, err := language.ParseAcceptLanguage(r.Header.Get("Accept-Language")) 25 | // We ignore the error: the default language will be selected for t == nil. 26 | tag, _, _ := matcher.Match(t...) 27 | fmt.Printf("%17v (t: %6v; q: %3v; err: %v)\n", tag, t, q, err) 28 | } 29 | 30 | func ExampleParseAcceptLanguage() { 31 | for _, al := range []string{ 32 | "nn;q=0.3, en-us;q=0.8, en,", 33 | "gsw, en;q=0.7, en-US;q=0.8", 34 | "gsw, nl, da", 35 | "invalid", 36 | } { 37 | // Create dummy request with Accept-Language set and pass it to handler. 38 | r, _ := http.NewRequest("GET", "example.com", strings.NewReader("Hello")) 39 | r.Header.Set("Accept-Language", al) 40 | handler(nil, r) 41 | } 42 | 43 | // Output: 44 | // en-GB (t: [ en en-US nn]; q: [ 1 0.8 0.3]; err: ) 45 | // en-GB-u-rg-uszzzz (t: [ gsw en-US en]; q: [ 1 0.8 0.7]; err: ) 46 | // de (t: [ gsw nl da]; q: [ 1 1 1]; err: ) 47 | // en-GB (t: []; q: []; err: language: tag is not well-formed) 48 | } 49 | -------------------------------------------------------------------------------- /message/catalog.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package message 6 | 7 | // TODO: some types in this file will need to be made public at some time. 8 | // Documentation and method names will reflect this by using the exported name. 9 | 10 | import ( 11 | "golang.org/x/text/language" 12 | "golang.org/x/text/message/catalog" 13 | ) 14 | 15 | // MatchLanguage reports the matched tag obtained from language.MatchStrings for 16 | // the Matcher of the DefaultCatalog. 17 | func MatchLanguage(preferred ...string) language.Tag { 18 | c := DefaultCatalog 19 | tag, _ := language.MatchStrings(c.Matcher(), preferred...) 20 | return tag 21 | } 22 | 23 | // DefaultCatalog is used by SetString. 24 | var DefaultCatalog catalog.Catalog = defaultCatalog 25 | 26 | var defaultCatalog = catalog.NewBuilder() 27 | 28 | // SetString calls SetString on the initial default Catalog. 29 | func SetString(tag language.Tag, key string, msg string) error { 30 | return defaultCatalog.SetString(tag, key, msg) 31 | } 32 | 33 | // Set calls Set on the initial default Catalog. 34 | func Set(tag language.Tag, key string, msg ...catalog.Message) error { 35 | return defaultCatalog.Set(tag, key, msg...) 36 | } 37 | -------------------------------------------------------------------------------- /message/catalog/go19.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build go1.9 6 | 7 | package catalog 8 | 9 | import "golang.org/x/text/internal/catmsg" 10 | 11 | // A Message holds a collection of translations for the same phrase that may 12 | // vary based on the values of substitution arguments. 13 | type Message = catmsg.Message 14 | 15 | type firstInSequence = catmsg.FirstOf 16 | -------------------------------------------------------------------------------- /message/catalog/gopre19.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !go1.9 6 | 7 | package catalog 8 | 9 | import "golang.org/x/text/internal/catmsg" 10 | 11 | // A Message holds a collection of translations for the same phrase that may 12 | // vary based on the values of substitution arguments. 13 | type Message interface { 14 | catmsg.Message 15 | } 16 | 17 | func firstInSequence(m []Message) catmsg.Message { 18 | a := []catmsg.Message{} 19 | for _, m := range m { 20 | a = append(a, m) 21 | } 22 | return catmsg.FirstOf(a) 23 | } 24 | -------------------------------------------------------------------------------- /message/catalog_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package message 6 | 7 | import ( 8 | "strings" 9 | "testing" 10 | 11 | "golang.org/x/text/language" 12 | "golang.org/x/text/message/catalog" 13 | ) 14 | 15 | func TestMatchLanguage(t *testing.T) { 16 | c := catalog.NewBuilder(catalog.Fallback(language.English)) 17 | c.SetString(language.Bengali, "", "") 18 | c.SetString(language.English, "", "") 19 | c.SetString(language.German, "", "") 20 | 21 | saved := DefaultCatalog 22 | defer func() { DefaultCatalog = saved }() 23 | DefaultCatalog = c 24 | 25 | testCases := []struct { 26 | args string // '|'-separated list 27 | want string 28 | }{{ 29 | args: "de-CH", 30 | want: "de-u-rg-chzzzz", 31 | }, { 32 | args: "bn-u-nu-latn|en-US,en;q=0.9,de;q=0.8,nl;q=0.7", 33 | want: "bn-u-nu-latn", 34 | }, { 35 | args: "gr", 36 | want: "en", 37 | }} 38 | for _, tc := range testCases { 39 | t.Run(tc.args, func(t *testing.T) { 40 | got := MatchLanguage(strings.Split(tc.args, "|")...) 41 | if got != language.Make(tc.want) { 42 | t.Errorf("got %q; want %q", got, tc.want) 43 | } 44 | }) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /message/examples_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package message_test 6 | 7 | import ( 8 | "fmt" 9 | "net/http" 10 | 11 | "golang.org/x/text/language" 12 | "golang.org/x/text/message" 13 | ) 14 | 15 | func Example_http() { 16 | // languages supported by this service: 17 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 18 | lang, _ := r.Cookie("lang") 19 | accept := r.Header.Get("Accept-Language") 20 | fallback := "en" 21 | tag := message.MatchLanguage(lang.String(), accept, fallback) 22 | 23 | p := message.NewPrinter(tag) 24 | 25 | p.Fprintln(w, "User language is", tag) 26 | }) 27 | } 28 | 29 | func ExamplePrinter_numbers() { 30 | for _, lang := range []string{"en", "de", "de-CH", "fr", "bn"} { 31 | p := message.NewPrinter(language.Make(lang)) 32 | p.Printf("%-6s %g\n", lang, 123456.78) 33 | } 34 | 35 | // Output: 36 | // en 123,456.78 37 | // de 123.456,78 38 | // de-CH 123’456.78 39 | // fr 123 456,78 40 | // bn ১,২৩,৪৫৬.৭৮ 41 | } 42 | 43 | func ExamplePrinter_mVerb() { 44 | message.SetString(language.Dutch, "You have chosen to play %m.", "U heeft ervoor gekozen om %m te spelen.") 45 | message.SetString(language.Dutch, "basketball", "basketbal") 46 | message.SetString(language.Dutch, "hockey", "ijshockey") 47 | message.SetString(language.Dutch, "soccer", "voetbal") 48 | message.SetString(language.BritishEnglish, "soccer", "football") 49 | 50 | for _, sport := range []string{"soccer", "basketball", "hockey"} { 51 | for _, lang := range []string{"en", "en-GB", "nl"} { 52 | p := message.NewPrinter(language.Make(lang)) 53 | fmt.Printf("%-6s %s\n", lang, p.Sprintf("You have chosen to play %m.", sport)) 54 | } 55 | fmt.Println() 56 | } 57 | 58 | // Output: 59 | // en You have chosen to play soccer. 60 | // en-GB You have chosen to play football. 61 | // nl U heeft ervoor gekozen om voetbal te spelen. 62 | // 63 | // en You have chosen to play basketball. 64 | // en-GB You have chosen to play basketball. 65 | // nl U heeft ervoor gekozen om basketbal te spelen. 66 | // 67 | // en You have chosen to play hockey. 68 | // en-GB You have chosen to play hockey. 69 | // nl U heeft ervoor gekozen om ijshockey te spelen. 70 | } 71 | -------------------------------------------------------------------------------- /message/pipeline/go19_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build go1.9 6 | 7 | package pipeline 8 | 9 | import "testing" 10 | 11 | func init() { 12 | setHelper = (*testing.T).Helper 13 | } 14 | -------------------------------------------------------------------------------- /message/pipeline/testdata/ssa/catalog_gen.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | package main 4 | 5 | import ( 6 | "golang.org/x/text/language" 7 | "golang.org/x/text/message" 8 | "golang.org/x/text/message/catalog" 9 | ) 10 | 11 | type dictionary struct { 12 | index []uint32 13 | data string 14 | } 15 | 16 | func (d *dictionary) Lookup(key string) (data string, ok bool) { 17 | p, ok := messageKeyToIndex[key] 18 | if !ok { 19 | return "", false 20 | } 21 | start, end := d.index[p], d.index[p+1] 22 | if start == end { 23 | return "", false 24 | } 25 | return d.data[start:end], true 26 | } 27 | 28 | func init() { 29 | dict := map[string]catalog.Dictionary{} 30 | fallback := language.MustParse("en-US") 31 | cat, err := catalog.NewFromMap(dict, catalog.Fallback(fallback)) 32 | if err != nil { 33 | panic(err) 34 | } 35 | message.DefaultCatalog = cat 36 | } 37 | 38 | var messageKeyToIndex = map[string]int{} 39 | -------------------------------------------------------------------------------- /message/pipeline/testdata/test1/catalog_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "path" 9 | "testing" 10 | 11 | "golang.org/x/text/message" 12 | ) 13 | 14 | func TestCatalog(t *testing.T) { 15 | args := func(a ...interface{}) []interface{} { return a } 16 | testCases := []struct { 17 | lang string 18 | key string 19 | args []interface{} 20 | want string 21 | }{{ 22 | lang: "en", 23 | key: "Hello world!\n", 24 | want: "Hello world!\n", 25 | }, { 26 | lang: "en", 27 | key: "non-existing-key\n", 28 | want: "non-existing-key\n", 29 | }, { 30 | lang: "de", 31 | key: "Hello world!\n", 32 | want: "Hallo Welt!\n", 33 | }, { 34 | lang: "en", 35 | key: "%d more files remaining!", 36 | args: args(1), 37 | want: "One file remaining!", 38 | }, { 39 | lang: "en-u-nu-fullwide", 40 | key: "%d more files remaining!", 41 | args: args(5), 42 | want: "There are ５ more files remaining!", 43 | }} 44 | for _, tc := range testCases { 45 | t.Run(path.Join(tc.lang, tc.key), func(t *testing.T) { 46 | p := message.NewPrinter(message.MatchLanguage(tc.lang)) 47 | got := p.Sprintf(tc.key, tc.args...) 48 | if got != tc.want { 49 | t.Errorf("got %q; want %q", got, tc.want) 50 | } 51 | }) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /message/pipeline/testdata/test1/test1.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import "golang.org/x/text/message" 8 | 9 | func main() { 10 | p := message.NewPrinter(message.MatchLanguage("en")) 11 | 12 | // NOT EXTRACTED: strings passed to Println are not extracted. 13 | p.Println("Hello world!") 14 | 15 | // NOT EXTRACTED: strings passed to Print are not extracted. 16 | p.Print("Hello world!\n") 17 | 18 | // Extract and trim whitespace (TODO). 19 | p.Printf("Hello world!\n") 20 | 21 | // NOT EXTRACTED: city is not used as a pattern or passed to %m. 22 | city := "Amsterdam" 23 | // This comment is extracted. 24 | p.Printf("Hello %s!\n", city) 25 | 26 | person := "Sheila" 27 | place := "Zürich" 28 | 29 | // Substitutions replaced by variable names. 30 | p.Printf("%s is visiting %s!\n", 31 | person, // The person of matter. 32 | place, // Place the person is visiting. 33 | ) 34 | 35 | pp := struct { 36 | Person string // The person of matter. // TODO: get this comment. 37 | Place string 38 | extra int 39 | }{ 40 | person, place, 4, 41 | } 42 | 43 | // extract will drop this comment in favor of the one below. 44 | p.Printf("%[1]s is visiting %[3]s!\n", // Field names are placeholders. 45 | pp.Person, 46 | pp.extra, 47 | pp.Place, // Place the person is visiting. 48 | ) 49 | 50 | // Numeric literal becomes placeholder. 51 | p.Printf("%d files remaining!", 2) 52 | 53 | const n = 2 54 | 55 | // Constant identifier becomes placeholder. 56 | p.Printf("%d more files remaining!", n) 57 | 58 | // Infer better names from type names. 59 | type referralCode int 60 | 61 | const c = referralCode(5) 62 | 63 | // Use type name as placeholder. 64 | p.Printf("Use the following code for your discount: %d\n", c) 65 | 66 | // Use constant name as message ID. 67 | const msgOutOfOrder = "%s is out of order!" // This comment wins. 68 | const device = "Soda machine" 69 | // This message has two IDs. 70 | p.Printf(msgOutOfOrder, device) 71 | 72 | // Multiple substitutions for same argument. 73 | miles := 1.2345 74 | p.Printf("%.2[1]f miles traveled (%[1]f)", miles) 75 | } 76 | -------------------------------------------------------------------------------- /number/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package number formats numbers according to the customs of different locales. 6 | // 7 | // The number formats of this package allow for greater formatting flexibility 8 | // than passing values to message.Printf calls as is. It currently supports the 9 | // builtin Go types and anything that implements the Convert interface 10 | // (currently internal). 11 | // 12 | // p := message.NewPrinter(language.English) 13 | // 14 | // p.Printf("%v bottles of beer on the wall.\n", number.Decimal(1234)) 15 | // // Prints: 1,234 bottles of beer on the wall. 16 | // 17 | // p.Printf("%v of gophers lose too much fur.\n", number.Percent(0.12)) 18 | // // Prints: 12% of gophers lose too much fur. 19 | // 20 | // p = message.NewPrinter(language.Dutch) 21 | // p.Printf("There are %v bikes per household.\n", number.Decimal(1.2)) 22 | // // Prints: There are 1,2 bikes per household. 23 | // 24 | // The width and scale specified in the formatting directives override the 25 | // configuration of the formatter. 26 | package number 27 | -------------------------------------------------------------------------------- /number/examples_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package number_test 6 | 7 | import ( 8 | "golang.org/x/text/language" 9 | "golang.org/x/text/message" 10 | "golang.org/x/text/number" 11 | ) 12 | 13 | func ExampleMaxIntegerDigits() { 14 | const year = 1999 15 | p := message.NewPrinter(language.English) 16 | p.Println("Year:", number.Decimal(year, number.MaxIntegerDigits(2))) 17 | 18 | // Output: 19 | // Year: 99 20 | } 21 | 22 | func ExampleIncrementString() { 23 | p := message.NewPrinter(language.English) 24 | 25 | p.Println(number.Decimal(1.33, number.IncrementString("0.50"))) 26 | 27 | // Output: 1.50 28 | } 29 | -------------------------------------------------------------------------------- /runes/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package runes_test 6 | 7 | import ( 8 | "fmt" 9 | "unicode" 10 | 11 | "golang.org/x/text/runes" 12 | "golang.org/x/text/transform" 13 | "golang.org/x/text/unicode/norm" 14 | "golang.org/x/text/width" 15 | ) 16 | 17 | func ExampleRemove() { 18 | t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) 19 | s, _, _ := transform.String(t, "résumé") 20 | fmt.Println(s) 21 | 22 | // Output: 23 | // resume 24 | } 25 | 26 | func ExampleMap() { 27 | replaceHyphens := runes.Map(func(r rune) rune { 28 | if unicode.Is(unicode.Hyphen, r) { 29 | return '|' 30 | } 31 | return r 32 | }) 33 | s, _, _ := transform.String(replaceHyphens, "a-b‐c⸗d﹣e") 34 | fmt.Println(s) 35 | 36 | // Output: 37 | // a|b|c|d|e 38 | } 39 | 40 | func ExampleIn() { 41 | // Convert Latin characters to their canonical form, while keeping other 42 | // width distinctions. 43 | t := runes.If(runes.In(unicode.Latin), width.Fold, nil) 44 | s, _, _ := transform.String(t, "ｱﾙｱﾉﾘｳ tech / アルアノリウ ｔｅｃｈ") 45 | fmt.Println(s) 46 | 47 | // Output: 48 | // ｱﾙｱﾉﾘｳ tech / アルアノリウ tech 49 | } 50 | 51 | func ExampleIf() { 52 | // Widen everything but ASCII. 53 | isASCII := func(r rune) bool { return r <= unicode.MaxASCII } 54 | t := runes.If(runes.Predicate(isASCII), nil, width.Widen) 55 | s, _, _ := transform.String(t, "ｱﾙｱﾉﾘｳ tech / 中國 / 5₩") 56 | fmt.Println(s) 57 | 58 | // Output: 59 | // アルアノリウ tech / 中國 / 5￦ 60 | } 61 | -------------------------------------------------------------------------------- /search/index.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Note: this file is identical to the file text/collate/index.go. Both files 6 | // will be removed when the new colltab package is finished and in use. 7 | 8 | package search 9 | 10 | import "golang.org/x/text/internal/colltab" 11 | 12 | const blockSize = 64 13 | 14 | func getTable(t tableIndex) *colltab.Table { 15 | return &colltab.Table{ 16 | Index: colltab.Trie{ 17 | Index0: mainLookup[:][blockSize*t.lookupOffset:], 18 | Values0: mainValues[:][blockSize*t.valuesOffset:], 19 | Index: mainLookup[:], 20 | Values: mainValues[:], 21 | }, 22 | ExpandElem: mainExpandElem[:], 23 | ContractTries: colltab.ContractTrieSet(mainCTEntries[:]), 24 | ContractElem: mainContractElem[:], 25 | MaxContractLen: 18, 26 | VariableTop: varTop, 27 | } 28 | } 29 | 30 | // tableIndex holds information for constructing a table 31 | // for a certain locale based on the main table. 32 | type tableIndex struct { 33 | lookupOffset uint32 34 | valuesOffset uint32 35 | } 36 | -------------------------------------------------------------------------------- /secure/bidirule/bench_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package bidirule 6 | 7 | import ( 8 | "testing" 9 | ) 10 | 11 | var benchData = []struct{ name, data string }{ 12 | {"ascii", "Scheveningen"}, 13 | {"arabic", "دبي"}, 14 | {"hangul", "다음과"}, 15 | } 16 | 17 | func doBench(b *testing.B, fn func(b *testing.B, data string)) { 18 | for _, d := range benchData { 19 | b.Run(d.name, func(b *testing.B) { fn(b, d.data) }) 20 | } 21 | } 22 | 23 | func BenchmarkSpan(b *testing.B) { 24 | r := New() 25 | doBench(b, func(b *testing.B, str string) { 26 | b.SetBytes(int64(len(str))) 27 | data := []byte(str) 28 | for i := 0; i < b.N; i++ { 29 | r.Reset() 30 | r.Span(data, true) 31 | } 32 | }) 33 | } 34 | 35 | func BenchmarkDirectionASCII(b *testing.B) { 36 | doBench(b, func(b *testing.B, str string) { 37 | b.SetBytes(int64(len(str))) 38 | data := []byte(str) 39 | for i := 0; i < b.N; i++ { 40 | Direction(data) 41 | } 42 | }) 43 | } 44 | 45 | func BenchmarkDirectionStringASCII(b *testing.B) { 46 | doBench(b, func(b *testing.B, str string) { 47 | b.SetBytes(int64(len(str))) 48 | for i := 0; i < b.N; i++ { 49 | DirectionString(str) 50 | } 51 | }) 52 | } 53 | -------------------------------------------------------------------------------- /secure/bidirule/bidirule10.0.0.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build go1.10 6 | 7 | package bidirule 8 | 9 | func (t *Transformer) isFinal() bool { 10 | return t.state == ruleLTRFinal || t.state == ruleRTLFinal || t.state == ruleInitial 11 | } 12 | -------------------------------------------------------------------------------- /secure/bidirule/bidirule9.0.0.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !go1.10 6 | 7 | package bidirule 8 | 9 | func (t *Transformer) isFinal() bool { 10 | if !t.isRTL() { 11 | return true 12 | } 13 | return t.state == ruleLTRFinal || t.state == ruleRTLFinal || t.state == ruleInitial 14 | } 15 | -------------------------------------------------------------------------------- /secure/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // secure is a repository of text security related packages. 6 | package secure // import "golang.org/x/text/secure" 7 | -------------------------------------------------------------------------------- /secure/precis/benchmark_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build go1.7 6 | 7 | package precis 8 | 9 | import ( 10 | "testing" 11 | ) 12 | 13 | var benchData = []struct{ name, str string }{ 14 | {"ASCII", "Malvolio"}, 15 | {"NotNormalized", "abcdefg\u0301\u031f"}, 16 | {"Arabic", "دبي"}, 17 | {"Hangul", "동일조건변경허락"}, 18 | } 19 | 20 | var benchProfiles = []struct { 21 | name string 22 | p *Profile 23 | }{ 24 | {"FreeForm", NewFreeform()}, 25 | {"Nickname", Nickname}, 26 | {"OpaqueString", OpaqueString}, 27 | {"UsernameCaseMapped", UsernameCaseMapped}, 28 | {"UsernameCasePreserved", UsernameCasePreserved}, 29 | } 30 | 31 | func doBench(b *testing.B, f func(b *testing.B, p *Profile, s string)) { 32 | for _, bp := range benchProfiles { 33 | for _, d := range benchData { 34 | b.Run(bp.name+"/"+d.name, func(b *testing.B) { 35 | f(b, bp.p, d.str) 36 | }) 37 | } 38 | } 39 | } 40 | 41 | func BenchmarkString(b *testing.B) { 42 | doBench(b, func(b *testing.B, p *Profile, s string) { 43 | for i := 0; i < b.N; i++ { 44 | p.String(s) 45 | } 46 | }) 47 | } 48 | 49 | func BenchmarkBytes(b *testing.B) { 50 | doBench(b, func(b *testing.B, p *Profile, s string) { 51 | src := []byte(s) 52 | b.ResetTimer() 53 | for i := 0; i < b.N; i++ { 54 | p.Bytes(src) 55 | } 56 | }) 57 | } 58 | 59 | func BenchmarkAppend(b *testing.B) { 60 | doBench(b, func(b *testing.B, p *Profile, s string) { 61 | src := []byte(s) 62 | dst := make([]byte, 0, 4096) 63 | b.ResetTimer() 64 | for i := 0; i < b.N; i++ { 65 | p.Append(dst, src) 66 | } 67 | }) 68 | } 69 | 70 | func BenchmarkTransform(b *testing.B) { 71 | doBench(b, func(b *testing.B, p *Profile, s string) { 72 | src := []byte(s) 73 | dst := make([]byte, 2*len(s)) 74 | t := p.NewTransformer() 75 | b.ResetTimer() 76 | for i := 0; i < b.N; i++ { 77 | t.Transform(dst, src, true) 78 | } 79 | }) 80 | } 81 | -------------------------------------------------------------------------------- /secure/precis/class.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package precis 6 | 7 | import ( 8 | "unicode/utf8" 9 | ) 10 | 11 | // TODO: Add contextual character rules from Appendix A of RFC5892. 12 | 13 | // A class is a set of characters that match certain derived properties. The 14 | // PRECIS framework defines two classes: The Freeform class and the Identifier 15 | // class. The freeform class should be used for profiles where expressiveness is 16 | // prioritized over safety such as nicknames or passwords. The identifier class 17 | // should be used for profiles where safety is the first priority such as 18 | // addressable network labels and usernames. 19 | type class struct { 20 | validFrom property 21 | } 22 | 23 | // Contains satisfies the runes.Set interface and returns whether the given rune 24 | // is a member of the class. 25 | func (c class) Contains(r rune) bool { 26 | b := make([]byte, 4) 27 | n := utf8.EncodeRune(b, r) 28 | 29 | trieval, _ := dpTrie.lookup(b[:n]) 30 | return c.validFrom <= property(trieval) 31 | } 32 | 33 | var ( 34 | identifier = &class{validFrom: pValid} 35 | freeform = &class{validFrom: idDisOrFreePVal} 36 | ) 37 | -------------------------------------------------------------------------------- /secure/precis/class_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package precis 6 | 7 | import ( 8 | "testing" 9 | 10 | "golang.org/x/text/runes" 11 | ) 12 | 13 | // Compile-time regression test to ensure that Class is a Set 14 | var _ runes.Set = (*class)(nil) 15 | 16 | // Ensure that certain characters are (or are not) in the identifier class. 17 | func TestClassContains(t *testing.T) { 18 | tests := []struct { 19 | name string 20 | class *class 21 | allowed []rune 22 | disallowed []rune 23 | }{ 24 | { 25 | name: "Identifier", 26 | class: identifier, 27 | allowed: []rune("Aa0\u0021\u007e\u00df\u3007"), 28 | disallowed: []rune("\u2150\u2100\u2200\u3164\u2190\u2600\u303b\u1e9b"), 29 | }, 30 | { 31 | name: "Freeform", 32 | class: freeform, 33 | allowed: []rune("Aa0\u0021\u007e\u00df\u3007 \u2150\u2100\u2200\u2190\u2600\u1e9b"), 34 | disallowed: []rune("\u3164\u303b"), 35 | }, 36 | } 37 | 38 | for _, rt := range tests { 39 | for _, r := range rt.allowed { 40 | if !rt.class.Contains(r) { 41 | t.Errorf("Class %s should contain %U", rt.name, r) 42 | } 43 | } 44 | for _, r := range rt.disallowed { 45 | if rt.class.Contains(r) { 46 | t.Errorf("Class %s should not contain %U", rt.name, r) 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /secure/precis/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package precis contains types and functions for the preparation, 6 | // enforcement, and comparison of internationalized strings ("PRECIS") as 7 | // defined in RFC 8264. It also contains several pre-defined profiles for 8 | // passwords, nicknames, and usernames as defined in RFC 8265 and RFC 8266. 9 | // 10 | // BE ADVISED: This package is under construction and the API may change in 11 | // backwards incompatible ways and without notice. 12 | package precis // import "golang.org/x/text/secure/precis" 13 | 14 | //go:generate go run gen.go gen_trieval.go 15 | -------------------------------------------------------------------------------- /secure/precis/gen_trieval.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | package main 8 | 9 | // entry is the entry of a trie table 10 | // 7..6 property (unassigned, disallowed, maybe, valid) 11 | // 5..0 category 12 | type entry uint8 13 | 14 | const ( 15 | propShift = 6 16 | propMask = 0xc0 17 | catMask = 0x3f 18 | ) 19 | 20 | func (e entry) property() property { return property(e & propMask) } 21 | func (e entry) category() category { return category(e & catMask) } 22 | 23 | type property uint8 24 | 25 | // The order of these constants matter. A Profile may consider runes to be 26 | // allowed either from pValid or idDisOrFreePVal. 27 | const ( 28 | unassigned property = iota << propShift 29 | disallowed 30 | idDisOrFreePVal // disallowed for Identifier, pValid for FreeForm 31 | pValid 32 | ) 33 | 34 | // compute permutations of all properties and specialCategories. 35 | type category uint8 36 | 37 | const ( 38 | other category = iota 39 | 40 | // Special rune types 41 | joiningL 42 | joiningD 43 | joiningT 44 | joiningR 45 | viramaModifier 46 | viramaJoinT // Virama + JoiningT 47 | latinSmallL // U+006c 48 | greek 49 | greekJoinT // Greek + JoiningT 50 | hebrew 51 | hebrewJoinT // Hebrew + JoiningT 52 | japanese // hirigana, katakana, han 53 | 54 | // Special rune types associated with contextual rules defined in 55 | // https://tools.ietf.org/html/rfc5892#appendix-A. 56 | // ContextO 57 | zeroWidthNonJoiner // rule 1 58 | zeroWidthJoiner // rule 2 59 | // ContextJ 60 | middleDot // rule 3 61 | greekLowerNumeralSign // rule 4 62 | hebrewPreceding // rule 5 and 6 63 | katakanaMiddleDot // rule 7 64 | arabicIndicDigit // rule 8 65 | extendedArabicIndicDigit // rule 9 66 | 67 | numCategories 68 | ) 69 | -------------------------------------------------------------------------------- /secure/precis/nickname.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package precis 6 | 7 | import ( 8 | "unicode" 9 | "unicode/utf8" 10 | 11 | "golang.org/x/text/transform" 12 | ) 13 | 14 | type nickAdditionalMapping struct { 15 | // TODO: This transformer needs to be stateless somehow… 16 | notStart bool 17 | prevSpace bool 18 | } 19 | 20 | func (t *nickAdditionalMapping) Reset() { 21 | t.prevSpace = false 22 | t.notStart = false 23 | } 24 | 25 | func (t *nickAdditionalMapping) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { 26 | // RFC 8266 §2.1. Rules 27 | // 28 | // 2. Additional Mapping Rule: The additional mapping rule consists of 29 | // the following sub-rules. 30 | // 31 | // a. Map any instances of non-ASCII space to SPACE (U+0020); a 32 | // non-ASCII space is any Unicode code point having a general 33 | // category of "Zs", naturally with the exception of SPACE 34 | // (U+0020). (The inclusion of only ASCII space prevents 35 | // confusion with various non-ASCII space code points, many of 36 | // which are difficult to reproduce across different input 37 | // methods.) 38 | // 39 | // b. Remove any instances of the ASCII space character at the 40 | // beginning or end of a nickname (e.g., "stpeter " is mapped to 41 | // "stpeter"). 42 | // 43 | // c. Map interior sequences of more than one ASCII space character 44 | // to a single ASCII space character (e.g., "St Peter" is 45 | // mapped to "St Peter"). 46 | for nSrc < len(src) { 47 | r, size := utf8.DecodeRune(src[nSrc:]) 48 | if size == 0 { // Incomplete UTF-8 encoding 49 | if !atEOF { 50 | return nDst, nSrc, transform.ErrShortSrc 51 | } 52 | size = 1 53 | } 54 | if unicode.Is(unicode.Zs, r) { 55 | t.prevSpace = true 56 | } else { 57 | if t.prevSpace && t.notStart { 58 | dst[nDst] = ' ' 59 | nDst += 1 60 | } 61 | if size != copy(dst[nDst:], src[nSrc:nSrc+size]) { 62 | nDst += size 63 | return nDst, nSrc, transform.ErrShortDst 64 | } 65 | nDst += size 66 | t.prevSpace = false 67 | t.notStart = true 68 | } 69 | nSrc += size 70 | } 71 | return nDst, nSrc, nil 72 | } 73 | -------------------------------------------------------------------------------- /secure/precis/profiles.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package precis 6 | 7 | import ( 8 | "unicode" 9 | 10 | "golang.org/x/text/runes" 11 | "golang.org/x/text/transform" 12 | "golang.org/x/text/unicode/norm" 13 | ) 14 | 15 | var ( 16 | // Implements the Nickname profile specified in RFC 8266. 17 | Nickname *Profile = nickname 18 | 19 | // Implements the UsernameCaseMapped profile specified in RFC 8265. 20 | UsernameCaseMapped *Profile = usernameCaseMap 21 | 22 | // Implements the UsernameCasePreserved profile specified in RFC 8265. 23 | UsernameCasePreserved *Profile = usernameNoCaseMap 24 | 25 | // Implements the OpaqueString profile defined in RFC 8265 for passwords and 26 | // other secure labels. 27 | OpaqueString *Profile = opaquestring 28 | ) 29 | 30 | var ( 31 | nickname = &Profile{ 32 | options: getOpts( 33 | AdditionalMapping(func() transform.Transformer { 34 | return &nickAdditionalMapping{} 35 | }), 36 | IgnoreCase, 37 | Norm(norm.NFKC), 38 | DisallowEmpty, 39 | repeat, 40 | ), 41 | class: freeform, 42 | } 43 | usernameCaseMap = &Profile{ 44 | options: getOpts( 45 | FoldWidth, 46 | LowerCase(), 47 | Norm(norm.NFC), 48 | BidiRule, 49 | ), 50 | class: identifier, 51 | } 52 | usernameNoCaseMap = &Profile{ 53 | options: getOpts( 54 | FoldWidth, 55 | Norm(norm.NFC), 56 | BidiRule, 57 | ), 58 | class: identifier, 59 | } 60 | opaquestring = &Profile{ 61 | options: getOpts( 62 | AdditionalMapping(func() transform.Transformer { 63 | return mapSpaces 64 | }), 65 | Norm(norm.NFC), 66 | DisallowEmpty, 67 | ), 68 | class: freeform, 69 | } 70 | ) 71 | 72 | // mapSpaces is a shared value of a runes.Map transformer. 73 | var mapSpaces transform.Transformer = runes.Map(func(r rune) rune { 74 | if unicode.Is(unicode.Zs, r) { 75 | return ' ' 76 | } 77 | return r 78 | }) 79 | -------------------------------------------------------------------------------- /secure/precis/tables_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package precis 6 | 7 | import ( 8 | "testing" 9 | "unicode" 10 | "unicode/utf8" 11 | 12 | "golang.org/x/text/runes" 13 | "golang.org/x/text/unicode/rangetable" 14 | ) 15 | 16 | type tableTest struct { 17 | rangeTable *unicode.RangeTable 18 | prop property 19 | } 20 | 21 | var exceptions = runes.Predicate(func(r rune) bool { 22 | switch uint32(r) { 23 | case 0x00DF, 0x03C2, 0x06FD, 0x06FE, 0x0F0B, 0x3007, 0x00B7, 0x0375, 0x05F3, 24 | 0x05F4, 0x30FB, 0x0660, 0x0661, 0x0662, 0x0663, 0x0664, 0x0665, 0x0666, 25 | 0x0667, 0x0668, 0x0669, 0x06F0, 0x06F1, 0x06F2, 0x06F3, 0x06F4, 0x06F5, 26 | 0x06F6, 0x06F7, 0x06F8, 0x06F9, 0x0640, 0x07FA, 0x302E, 0x302F, 0x3031, 27 | 0x3032, 0x3033, 0x3034, 0x3035, 0x303B: 28 | return true 29 | default: 30 | return false 31 | } 32 | }) 33 | 34 | // Ensure that certain properties were generated correctly. 35 | func TestTable(t *testing.T) { 36 | tests := []tableTest{ 37 | tableTest{ 38 | rangetable.Merge( 39 | unicode.Lt, unicode.Nl, unicode.No, // Other letter digits 40 | unicode.Me, // Modifiers 41 | unicode.Zs, // Spaces 42 | unicode.So, // Symbols 43 | unicode.Pi, unicode.Pf, // Punctuation 44 | ), 45 | idDisOrFreePVal, 46 | }, 47 | tableTest{ 48 | rangetable.New(0xA0000, 0xA0101, 0xDFFFF), 49 | unassigned, 50 | }, 51 | } 52 | 53 | assigned := rangetable.Assigned(UnicodeVersion) 54 | 55 | for _, test := range tests { 56 | rangetable.Visit(test.rangeTable, func(r rune) { 57 | if !unicode.In(r, assigned) { 58 | return 59 | } 60 | b := make([]byte, 4) 61 | n := utf8.EncodeRune(b, r) 62 | trieval, _ := dpTrie.lookup(b[:n]) 63 | p := entry(trieval).property() 64 | if p != test.prop && !exceptions.Contains(r) { 65 | t.Errorf("%U: got %+x; want %+x", r, test.prop, p) 66 | } 67 | }) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /secure/precis/transformer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package precis 6 | 7 | import "golang.org/x/text/transform" 8 | 9 | // Transformer implements the transform.Transformer interface. 10 | type Transformer struct { 11 | t transform.Transformer 12 | } 13 | 14 | // Reset implements the transform.Transformer interface. 15 | func (t Transformer) Reset() { t.t.Reset() } 16 | 17 | // Transform implements the transform.Transformer interface. 18 | func (t Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { 19 | return t.t.Transform(dst, src, atEOF) 20 | } 21 | 22 | // Bytes returns a new byte slice with the result of applying t to b. 23 | func (t Transformer) Bytes(b []byte) []byte { 24 | b, _, _ = transform.Bytes(t, b) 25 | return b 26 | } 27 | 28 | // String returns a string with the result of applying t to s. 29 | func (t Transformer) String(s string) string { 30 | s, _, _ = transform.String(t, s) 31 | return s 32 | } 33 | -------------------------------------------------------------------------------- /secure/precis/trieval.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | package precis 4 | 5 | // entry is the entry of a trie table 6 | // 7..6 property (unassigned, disallowed, maybe, valid) 7 | // 5..0 category 8 | type entry uint8 9 | 10 | const ( 11 | propShift = 6 12 | propMask = 0xc0 13 | catMask = 0x3f 14 | ) 15 | 16 | func (e entry) property() property { return property(e & propMask) } 17 | func (e entry) category() category { return category(e & catMask) } 18 | 19 | type property uint8 20 | 21 | // The order of these constants matter. A Profile may consider runes to be 22 | // allowed either from pValid or idDisOrFreePVal. 23 | const ( 24 | unassigned property = iota << propShift 25 | disallowed 26 | idDisOrFreePVal // disallowed for Identifier, pValid for FreeForm 27 | pValid 28 | ) 29 | 30 | // compute permutations of all properties and specialCategories. 31 | type category uint8 32 | 33 | const ( 34 | other category = iota 35 | 36 | // Special rune types 37 | joiningL 38 | joiningD 39 | joiningT 40 | joiningR 41 | viramaModifier 42 | viramaJoinT // Virama + JoiningT 43 | latinSmallL // U+006c 44 | greek 45 | greekJoinT // Greek + JoiningT 46 | hebrew 47 | hebrewJoinT // Hebrew + JoiningT 48 | japanese // hirigana, katakana, han 49 | 50 | // Special rune types associated with contextual rules defined in 51 | // https://tools.ietf.org/html/rfc5892#appendix-A. 52 | // ContextO 53 | zeroWidthNonJoiner // rule 1 54 | zeroWidthJoiner // rule 2 55 | // ContextJ 56 | middleDot // rule 3 57 | greekLowerNumeralSign // rule 4 58 | hebrewPreceding // rule 5 and 6 59 | katakanaMiddleDot // rule 7 60 | arabicIndicDigit // rule 8 61 | extendedArabicIndicDigit // rule 9 62 | 63 | numCategories 64 | ) 65 | -------------------------------------------------------------------------------- /transform/examples_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package transform_test 6 | 7 | import ( 8 | "fmt" 9 | "unicode" 10 | 11 | "golang.org/x/text/transform" 12 | "golang.org/x/text/unicode/norm" 13 | ) 14 | 15 | func ExampleRemoveFunc() { 16 | input := []byte(`tschüß; до свидания`) 17 | 18 | b := make([]byte, len(input)) 19 | 20 | t := transform.RemoveFunc(unicode.IsSpace) 21 | n, _, _ := t.Transform(b, input, true) 22 | fmt.Println(string(b[:n])) 23 | 24 | t = transform.RemoveFunc(func(r rune) bool { 25 | return !unicode.Is(unicode.Latin, r) 26 | }) 27 | n, _, _ = t.Transform(b, input, true) 28 | fmt.Println(string(b[:n])) 29 | 30 | n, _, _ = t.Transform(b, norm.NFD.Bytes(input), true) 31 | fmt.Println(string(b[:n])) 32 | 33 | // Output: 34 | // tschüß;досвидания 35 | // tschüß 36 | // tschuß 37 | } 38 | -------------------------------------------------------------------------------- /unicode/bidi/gen_ranges.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | package main 8 | 9 | import ( 10 | "unicode" 11 | 12 | "golang.org/x/text/internal/gen" 13 | "golang.org/x/text/internal/ucd" 14 | "golang.org/x/text/unicode/rangetable" 15 | ) 16 | 17 | // These tables are hand-extracted from: 18 | // https://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedBidiClass.txt 19 | func visitDefaults(fn func(r rune, c Class)) { 20 | // first write default values for ranges listed above. 21 | visitRunes(fn, AL, []rune{ 22 | 0x0600, 0x07BF, // Arabic 23 | 0x08A0, 0x08FF, // Arabic Extended-A 24 | 0xFB50, 0xFDCF, // Arabic Presentation Forms 25 | 0xFDF0, 0xFDFF, 26 | 0xFE70, 0xFEFF, 27 | 0x0001EE00, 0x0001EEFF, // Arabic Mathematical Alpha Symbols 28 | }) 29 | visitRunes(fn, R, []rune{ 30 | 0x0590, 0x05FF, // Hebrew 31 | 0x07C0, 0x089F, // Nko et al. 32 | 0xFB1D, 0xFB4F, 33 | 0x00010800, 0x00010FFF, // Cypriot Syllabary et. al. 34 | 0x0001E800, 0x0001EDFF, 35 | 0x0001EF00, 0x0001EFFF, 36 | }) 37 | visitRunes(fn, ET, []rune{ // European Terminator 38 | 0x20A0, 0x20Cf, // Currency symbols 39 | }) 40 | rangetable.Visit(unicode.Noncharacter_Code_Point, func(r rune) { 41 | fn(r, BN) // Boundary Neutral 42 | }) 43 | ucd.Parse(gen.OpenUCDFile("DerivedCoreProperties.txt"), func(p *ucd.Parser) { 44 | if p.String(1) == "Default_Ignorable_Code_Point" { 45 | fn(p.Rune(0), BN) // Boundary Neutral 46 | } 47 | }) 48 | } 49 | 50 | func visitRunes(fn func(r rune, c Class), c Class, runes []rune) { 51 | for i := 0; i < len(runes); i += 2 { 52 | lo, hi := runes[i], runes[i+1] 53 | for j := lo; j <= hi; j++ { 54 | fn(j, c) 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /unicode/bidi/gen_trieval.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | package main 8 | 9 | // Class is the Unicode BiDi class. Each rune has a single class. 10 | type Class uint 11 | 12 | const ( 13 | L Class = iota // LeftToRight 14 | R // RightToLeft 15 | EN // EuropeanNumber 16 | ES // EuropeanSeparator 17 | ET // EuropeanTerminator 18 | AN // ArabicNumber 19 | CS // CommonSeparator 20 | B // ParagraphSeparator 21 | S // SegmentSeparator 22 | WS // WhiteSpace 23 | ON // OtherNeutral 24 | BN // BoundaryNeutral 25 | NSM // NonspacingMark 26 | AL // ArabicLetter 27 | Control // Control LRO - PDI 28 | 29 | numClass 30 | 31 | LRO // LeftToRightOverride 32 | RLO // RightToLeftOverride 33 | LRE // LeftToRightEmbedding 34 | RLE // RightToLeftEmbedding 35 | PDF // PopDirectionalFormat 36 | LRI // LeftToRightIsolate 37 | RLI // RightToLeftIsolate 38 | FSI // FirstStrongIsolate 39 | PDI // PopDirectionalIsolate 40 | 41 | unknownClass = ^Class(0) 42 | ) 43 | 44 | // A trie entry has the following bits: 45 | // 7..5 XOR mask for brackets 46 | // 4 1: Bracket open, 0: Bracket close 47 | // 3..0 Class type 48 | 49 | const ( 50 | openMask = 0x10 51 | xorMaskShift = 5 52 | ) 53 | -------------------------------------------------------------------------------- /unicode/bidi/ranges_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | package bidi 4 | 5 | import ( 6 | "unicode" 7 | 8 | "golang.org/x/text/internal/gen" 9 | "golang.org/x/text/internal/ucd" 10 | "golang.org/x/text/unicode/rangetable" 11 | ) 12 | 13 | // These tables are hand-extracted from: 14 | // https://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedBidiClass.txt 15 | func visitDefaults(fn func(r rune, c Class)) { 16 | // first write default values for ranges listed above. 17 | visitRunes(fn, AL, []rune{ 18 | 0x0600, 0x07BF, // Arabic 19 | 0x08A0, 0x08FF, // Arabic Extended-A 20 | 0xFB50, 0xFDCF, // Arabic Presentation Forms 21 | 0xFDF0, 0xFDFF, 22 | 0xFE70, 0xFEFF, 23 | 0x0001EE00, 0x0001EEFF, // Arabic Mathematical Alpha Symbols 24 | }) 25 | visitRunes(fn, R, []rune{ 26 | 0x0590, 0x05FF, // Hebrew 27 | 0x07C0, 0x089F, // Nko et al. 28 | 0xFB1D, 0xFB4F, 29 | 0x00010800, 0x00010FFF, // Cypriot Syllabary et. al. 30 | 0x0001E800, 0x0001EDFF, 31 | 0x0001EF00, 0x0001EFFF, 32 | }) 33 | visitRunes(fn, ET, []rune{ // European Terminator 34 | 0x20A0, 0x20Cf, // Currency symbols 35 | }) 36 | rangetable.Visit(unicode.Noncharacter_Code_Point, func(r rune) { 37 | fn(r, BN) // Boundary Neutral 38 | }) 39 | ucd.Parse(gen.OpenUCDFile("DerivedCoreProperties.txt"), func(p *ucd.Parser) { 40 | if p.String(1) == "Default_Ignorable_Code_Point" { 41 | fn(p.Rune(0), BN) // Boundary Neutral 42 | } 43 | }) 44 | } 45 | 46 | func visitRunes(fn func(r rune, c Class), c Class, runes []rune) { 47 | for i := 0; i < len(runes); i += 2 { 48 | lo, hi := runes[i], runes[i+1] 49 | for j := lo; j <= hi; j++ { 50 | fn(j, c) 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /unicode/bidi/tables_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package bidi 6 | 7 | import ( 8 | "testing" 9 | 10 | "golang.org/x/text/internal/gen" 11 | "golang.org/x/text/internal/testtext" 12 | "golang.org/x/text/internal/ucd" 13 | ) 14 | 15 | var labels = []string{ 16 | AL: "AL", 17 | AN: "AN", 18 | B: "B", 19 | BN: "BN", 20 | CS: "CS", 21 | EN: "EN", 22 | ES: "ES", 23 | ET: "ET", 24 | L: "L", 25 | NSM: "NSM", 26 | ON: "ON", 27 | R: "R", 28 | S: "S", 29 | WS: "WS", 30 | 31 | LRO: "LRO", 32 | RLO: "RLO", 33 | LRE: "LRE", 34 | RLE: "RLE", 35 | PDF: "PDF", 36 | LRI: "LRI", 37 | RLI: "RLI", 38 | FSI: "FSI", 39 | PDI: "PDI", 40 | } 41 | 42 | func TestTables(t *testing.T) { 43 | testtext.SkipIfNotLong(t) 44 | 45 | ucd.Parse(gen.OpenUCDFile("BidiBrackets.txt"), func(p *ucd.Parser) { 46 | r1 := p.Rune(0) 47 | want := p.Rune(1) 48 | 49 | e, _ := LookupRune(r1) 50 | if got := e.reverseBracket(r1); got != want { 51 | t.Errorf("Reverse(%U) = %U; want %U", r1, got, want) 52 | } 53 | }) 54 | 55 | done := map[rune]bool{} 56 | test := func(name string, r rune, want string) { 57 | str := string(r) 58 | e, _ := LookupString(str) 59 | if got := labels[e.Class()]; got != want { 60 | t.Errorf("%s:%U: got %s; want %s", name, r, got, want) 61 | } 62 | if e2, sz := LookupRune(r); e != e2 || sz != len(str) { 63 | t.Errorf("LookupRune(%U) = %v, %d; want %v, %d", r, e2, e, sz, len(str)) 64 | } 65 | if e2, sz := Lookup([]byte(str)); e != e2 || sz != len(str) { 66 | t.Errorf("Lookup(%U) = %v, %d; want %v, %d", r, e2, e, sz, len(str)) 67 | } 68 | done[r] = true 69 | } 70 | 71 | // Insert the derived BiDi properties. 72 | ucd.Parse(gen.OpenUCDFile("extracted/DerivedBidiClass.txt"), func(p *ucd.Parser) { 73 | r := p.Rune(0) 74 | test("derived", r, p.String(1)) 75 | }) 76 | visitDefaults(func(r rune, c Class) { 77 | if !done[r] { 78 | test("default", r, labels[c]) 79 | } 80 | }) 81 | 82 | } 83 | -------------------------------------------------------------------------------- /unicode/bidi/trieval.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | package bidi 4 | 5 | // Class is the Unicode BiDi class. Each rune has a single class. 6 | type Class uint 7 | 8 | const ( 9 | L Class = iota // LeftToRight 10 | R // RightToLeft 11 | EN // EuropeanNumber 12 | ES // EuropeanSeparator 13 | ET // EuropeanTerminator 14 | AN // ArabicNumber 15 | CS // CommonSeparator 16 | B // ParagraphSeparator 17 | S // SegmentSeparator 18 | WS // WhiteSpace 19 | ON // OtherNeutral 20 | BN // BoundaryNeutral 21 | NSM // NonspacingMark 22 | AL // ArabicLetter 23 | Control // Control LRO - PDI 24 | 25 | numClass 26 | 27 | LRO // LeftToRightOverride 28 | RLO // RightToLeftOverride 29 | LRE // LeftToRightEmbedding 30 | RLE // RightToLeftEmbedding 31 | PDF // PopDirectionalFormat 32 | LRI // LeftToRightIsolate 33 | RLI // RightToLeftIsolate 34 | FSI // FirstStrongIsolate 35 | PDI // PopDirectionalIsolate 36 | 37 | unknownClass = ^Class(0) 38 | ) 39 | 40 | // A trie entry has the following bits: 41 | // 7..5 XOR mask for brackets 42 | // 4 1: Bracket open, 0: Bracket close 43 | // 3..0 Class type 44 | 45 | const ( 46 | openMask = 0x10 47 | xorMaskShift = 5 48 | ) 49 | -------------------------------------------------------------------------------- /unicode/cldr/cldr_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package cldr 6 | 7 | import "testing" 8 | 9 | func TestParseDraft(t *testing.T) { 10 | tests := []struct { 11 | in string 12 | draft Draft 13 | err bool 14 | }{ 15 | {"unconfirmed", Unconfirmed, false}, 16 | {"provisional", Provisional, false}, 17 | {"contributed", Contributed, false}, 18 | {"approved", Approved, false}, 19 | {"", Approved, false}, 20 | {"foo", Approved, true}, 21 | } 22 | for _, tt := range tests { 23 | if d, err := ParseDraft(tt.in); d != tt.draft || (err != nil) != tt.err { 24 | t.Errorf("%q: was %v, %v; want %v, %v", tt.in, d, err != nil, tt.draft, tt.err) 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /unicode/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // unicode holds packages with implementations of Unicode standards that are 6 | // mostly used as building blocks for other packages in golang.org/x/text, 7 | // layout engines, or are otherwise more low-level in nature. 8 | package unicode 9 | -------------------------------------------------------------------------------- /unicode/norm/example_iter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package norm_test 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "io" 11 | "unicode/utf8" 12 | 13 | "golang.org/x/text/unicode/norm" 14 | ) 15 | 16 | // EqualSimple uses a norm.Iter to compare two non-normalized 17 | // strings for equivalence. 18 | func EqualSimple(a, b string) bool { 19 | var ia, ib norm.Iter 20 | ia.InitString(norm.NFKD, a) 21 | ib.InitString(norm.NFKD, b) 22 | for !ia.Done() && !ib.Done() { 23 | if !bytes.Equal(ia.Next(), ib.Next()) { 24 | return false 25 | } 26 | } 27 | return ia.Done() && ib.Done() 28 | } 29 | 30 | // FindPrefix finds the longest common prefix of ASCII characters 31 | // of a and b. 32 | func FindPrefix(a, b string) int { 33 | i := 0 34 | for ; i < len(a) && i < len(b) && a[i] < utf8.RuneSelf && a[i] == b[i]; i++ { 35 | } 36 | return i 37 | } 38 | 39 | // EqualOpt is like EqualSimple, but optimizes the special 40 | // case for ASCII characters. 41 | func EqualOpt(a, b string) bool { 42 | n := FindPrefix(a, b) 43 | a, b = a[n:], b[n:] 44 | var ia, ib norm.Iter 45 | ia.InitString(norm.NFKD, a) 46 | ib.InitString(norm.NFKD, b) 47 | for !ia.Done() && !ib.Done() { 48 | if !bytes.Equal(ia.Next(), ib.Next()) { 49 | return false 50 | } 51 | if n := int64(FindPrefix(a[ia.Pos():], b[ib.Pos():])); n != 0 { 52 | ia.Seek(n, io.SeekCurrent) 53 | ib.Seek(n, io.SeekCurrent) 54 | } 55 | } 56 | return ia.Done() && ib.Done() 57 | } 58 | 59 | var compareTests = []struct{ a, b string }{ 60 | {"aaa", "aaa"}, 61 | {"aaa", "aab"}, 62 | {"a\u0300a", "\u00E0a"}, 63 | {"a\u0300\u0320b", "a\u0320\u0300b"}, 64 | {"\u1E0A\u0323", "\x44\u0323\u0307"}, 65 | // A character that decomposes into multiple segments 66 | // spans several iterations. 67 | {"\u3304", "\u30A4\u30CB\u30F3\u30AF\u3099"}, 68 | } 69 | 70 | func ExampleIter() { 71 | for i, t := range compareTests { 72 | r0 := EqualSimple(t.a, t.b) 73 | r1 := EqualOpt(t.a, t.b) 74 | fmt.Printf("%d: %v %v\n", i, r0, r1) 75 | } 76 | // Output: 77 | // 0: true true 78 | // 1: false false 79 | // 2: true true 80 | // 3: true true 81 | // 4: true true 82 | // 5: true true 83 | } 84 | -------------------------------------------------------------------------------- /unicode/norm/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package norm_test 6 | 7 | import ( 8 | "fmt" 9 | 10 | "golang.org/x/text/unicode/norm" 11 | ) 12 | 13 | func ExampleForm_NextBoundary() { 14 | s := norm.NFD.String("Mêlée") 15 | 16 | for i := 0; i < len(s); { 17 | d := norm.NFC.NextBoundaryInString(s[i:], true) 18 | fmt.Printf("%[1]s: %+[1]q\n", s[i:i+d]) 19 | i += d 20 | } 21 | // Output: 22 | // M: "M" 23 | // ê: "e\u0302" 24 | // l: "l" 25 | // é: "e\u0301" 26 | // e: "e" 27 | } 28 | -------------------------------------------------------------------------------- /unicode/norm/forminfo_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build test 6 | 7 | package norm 8 | 9 | import "testing" 10 | 11 | func TestProperties(t *testing.T) { 12 | var d runeData 13 | CK := [2]string{"C", "K"} 14 | for k, r := 1, rune(0); r < 0x2ffff; r++ { 15 | if k < len(testData) && r == testData[k].r { 16 | d = testData[k] 17 | k++ 18 | } 19 | s := string(r) 20 | for j, p := range []Properties{NFC.PropertiesString(s), NFKC.PropertiesString(s)} { 21 | f := d.f[j] 22 | if p.CCC() != d.ccc { 23 | t.Errorf("%U: ccc(%s): was %d; want %d %X", r, CK[j], p.CCC(), d.ccc, p.index) 24 | } 25 | if p.isYesC() != (f.qc == Yes) { 26 | t.Errorf("%U: YesC(%s): was %v; want %v", r, CK[j], p.isYesC(), f.qc == Yes) 27 | } 28 | if p.combinesBackward() != (f.qc == Maybe) { 29 | t.Errorf("%U: combines backwards(%s): was %v; want %v", r, CK[j], p.combinesBackward(), f.qc == Maybe) 30 | } 31 | if p.nLeadingNonStarters() != d.nLead { 32 | t.Errorf("%U: nLead(%s): was %d; want %d %#v %#v", r, CK[j], p.nLeadingNonStarters(), d.nLead, p, d) 33 | } 34 | if p.nTrailingNonStarters() != d.nTrail { 35 | t.Errorf("%U: nTrail(%s): was %d; want %d %#v %#v", r, CK[j], p.nTrailingNonStarters(), d.nTrail, p, d) 36 | } 37 | if p.combinesForward() != f.combinesForward { 38 | t.Errorf("%U: combines forward(%s): was %v; want %v %#v", r, CK[j], p.combinesForward(), f.combinesForward, p) 39 | } 40 | // Skip Hangul as it is algorithmically computed. 41 | if r >= hangulBase && r < hangulEnd { 42 | continue 43 | } 44 | if p.hasDecomposition() { 45 | if has := f.decomposition != ""; !has { 46 | t.Errorf("%U: hasDecomposition(%s): was %v; want %v", r, CK[j], p.hasDecomposition(), has) 47 | } 48 | if string(p.Decomposition()) != f.decomposition { 49 | t.Errorf("%U: decomp(%s): was %+q; want %+q", r, CK[j], p.Decomposition(), f.decomposition) 50 | } 51 | } 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /unicode/norm/input.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package norm 6 | 7 | import "unicode/utf8" 8 | 9 | type input struct { 10 | str string 11 | bytes []byte 12 | } 13 | 14 | func inputBytes(str []byte) input { 15 | return input{bytes: str} 16 | } 17 | 18 | func inputString(str string) input { 19 | return input{str: str} 20 | } 21 | 22 | func (in *input) setBytes(str []byte) { 23 | in.str = "" 24 | in.bytes = str 25 | } 26 | 27 | func (in *input) setString(str string) { 28 | in.str = str 29 | in.bytes = nil 30 | } 31 | 32 | func (in *input) _byte(p int) byte { 33 | if in.bytes == nil { 34 | return in.str[p] 35 | } 36 | return in.bytes[p] 37 | } 38 | 39 | func (in *input) skipASCII(p, max int) int { 40 | if in.bytes == nil { 41 | for ; p < max && in.str[p] < utf8.RuneSelf; p++ { 42 | } 43 | } else { 44 | for ; p < max && in.bytes[p] < utf8.RuneSelf; p++ { 45 | } 46 | } 47 | return p 48 | } 49 | 50 | func (in *input) skipContinuationBytes(p int) int { 51 | if in.bytes == nil { 52 | for ; p < len(in.str) && !utf8.RuneStart(in.str[p]); p++ { 53 | } 54 | } else { 55 | for ; p < len(in.bytes) && !utf8.RuneStart(in.bytes[p]); p++ { 56 | } 57 | } 58 | return p 59 | } 60 | 61 | func (in *input) appendSlice(buf []byte, b, e int) []byte { 62 | if in.bytes != nil { 63 | return append(buf, in.bytes[b:e]...) 64 | } 65 | for i := b; i < e; i++ { 66 | buf = append(buf, in.str[i]) 67 | } 68 | return buf 69 | } 70 | 71 | func (in *input) copySlice(buf []byte, b, e int) int { 72 | if in.bytes == nil { 73 | return copy(buf, in.str[b:e]) 74 | } 75 | return copy(buf, in.bytes[b:e]) 76 | } 77 | 78 | func (in *input) charinfoNFC(p int) (uint16, int) { 79 | if in.bytes == nil { 80 | return nfcData.lookupString(in.str[p:]) 81 | } 82 | return nfcData.lookup(in.bytes[p:]) 83 | } 84 | 85 | func (in *input) charinfoNFKC(p int) (uint16, int) { 86 | if in.bytes == nil { 87 | return nfkcData.lookupString(in.str[p:]) 88 | } 89 | return nfkcData.lookup(in.bytes[p:]) 90 | } 91 | 92 | func (in *input) hangul(p int) (r rune) { 93 | var size int 94 | if in.bytes == nil { 95 | if !isHangulString(in.str[p:]) { 96 | return 0 97 | } 98 | r, size = utf8.DecodeRuneInString(in.str[p:]) 99 | } else { 100 | if !isHangul(in.bytes[p:]) { 101 | return 0 102 | } 103 | r, size = utf8.DecodeRune(in.bytes[p:]) 104 | } 105 | if size != hangulUTF8Size { 106 | return 0 107 | } 108 | return r 109 | } 110 | -------------------------------------------------------------------------------- /unicode/norm/readwriter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package norm 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "testing" 11 | ) 12 | 13 | var bufSizes = []int{1, 2, 3, 4, 5, 6, 7, 8, 100, 101, 102, 103, 4000, 4001, 4002, 4003} 14 | 15 | func readFunc(size int) appendFunc { 16 | return func(f Form, out []byte, s string) []byte { 17 | out = append(out, s...) 18 | r := f.Reader(bytes.NewBuffer(out)) 19 | buf := make([]byte, size) 20 | result := []byte{} 21 | for n, err := 0, error(nil); err == nil; { 22 | n, err = r.Read(buf) 23 | result = append(result, buf[:n]...) 24 | } 25 | return result 26 | } 27 | } 28 | 29 | func TestReader(t *testing.T) { 30 | for _, s := range bufSizes { 31 | name := fmt.Sprintf("TestReader%d", s) 32 | runNormTests(t, name, readFunc(s)) 33 | } 34 | } 35 | 36 | func writeFunc(size int) appendFunc { 37 | return func(f Form, out []byte, s string) []byte { 38 | in := append(out, s...) 39 | result := new(bytes.Buffer) 40 | w := f.Writer(result) 41 | buf := make([]byte, size) 42 | for n := 0; len(in) > 0; in = in[n:] { 43 | n = copy(buf, in) 44 | _, _ = w.Write(buf[:n]) 45 | } 46 | w.Close() 47 | return result.Bytes() 48 | } 49 | } 50 | 51 | func TestWriter(t *testing.T) { 52 | for _, s := range bufSizes { 53 | name := fmt.Sprintf("TestWriter%d", s) 54 | runNormTests(t, name, writeFunc(s)) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /unicode/norm/trie.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package norm 6 | 7 | type valueRange struct { 8 | value uint16 // header: value:stride 9 | lo, hi byte // header: lo:n 10 | } 11 | 12 | type sparseBlocks struct { 13 | values []valueRange 14 | offset []uint16 15 | } 16 | 17 | var nfcSparse = sparseBlocks{ 18 | values: nfcSparseValues[:], 19 | offset: nfcSparseOffset[:], 20 | } 21 | 22 | var nfkcSparse = sparseBlocks{ 23 | values: nfkcSparseValues[:], 24 | offset: nfkcSparseOffset[:], 25 | } 26 | 27 | var ( 28 | nfcData = newNfcTrie(0) 29 | nfkcData = newNfkcTrie(0) 30 | ) 31 | 32 | // lookup determines the type of block n and looks up the value for b. 33 | // For n < t.cutoff, the block is a simple lookup table. Otherwise, the block 34 | // is a list of ranges with an accompanying value. Given a matching range r, 35 | // the value for b is by r.value + (b - r.lo) * stride. 36 | func (t *sparseBlocks) lookup(n uint32, b byte) uint16 { 37 | offset := t.offset[n] 38 | header := t.values[offset] 39 | lo := offset + 1 40 | hi := lo + uint16(header.lo) 41 | for lo < hi { 42 | m := lo + (hi-lo)/2 43 | r := t.values[m] 44 | if r.lo <= b && b <= r.hi { 45 | return r.value + uint16(b-r.lo)*header.value 46 | } 47 | if b < r.lo { 48 | hi = m 49 | } else { 50 | lo = m + 1 51 | } 52 | } 53 | return 0 54 | } 55 | -------------------------------------------------------------------------------- /unicode/rangetable/rangetable.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package rangetable provides utilities for creating and inspecting 6 | // unicode.RangeTables. 7 | package rangetable 8 | 9 | import ( 10 | "sort" 11 | "unicode" 12 | ) 13 | 14 | // New creates a RangeTable from the given runes, which may contain duplicates. 15 | func New(r ...rune) *unicode.RangeTable { 16 | if len(r) == 0 { 17 | return &unicode.RangeTable{} 18 | } 19 | 20 | sort.Sort(byRune(r)) 21 | 22 | // Remove duplicates. 23 | k := 1 24 | for i := 1; i < len(r); i++ { 25 | if r[k-1] != r[i] { 26 | r[k] = r[i] 27 | k++ 28 | } 29 | } 30 | 31 | var rt unicode.RangeTable 32 | for _, r := range r[:k] { 33 | if r <= 0xFFFF { 34 | rt.R16 = append(rt.R16, unicode.Range16{Lo: uint16(r), Hi: uint16(r), Stride: 1}) 35 | } else { 36 | rt.R32 = append(rt.R32, unicode.Range32{Lo: uint32(r), Hi: uint32(r), Stride: 1}) 37 | } 38 | } 39 | 40 | // Optimize RangeTable. 41 | return Merge(&rt) 42 | } 43 | 44 | type byRune []rune 45 | 46 | func (r byRune) Len() int { return len(r) } 47 | func (r byRune) Swap(i, j int) { r[i], r[j] = r[j], r[i] } 48 | func (r byRune) Less(i, j int) bool { return r[i] < r[j] } 49 | 50 | // Visit visits all runes in the given RangeTable in order, calling fn for each. 51 | func Visit(rt *unicode.RangeTable, fn func(rune)) { 52 | for _, r16 := range rt.R16 { 53 | for r := rune(r16.Lo); r <= rune(r16.Hi); r += rune(r16.Stride) { 54 | fn(r) 55 | } 56 | } 57 | for _, r32 := range rt.R32 { 58 | for r := rune(r32.Lo); r <= rune(r32.Hi); r += rune(r32.Stride) { 59 | fn(r) 60 | } 61 | } 62 | } 63 | 64 | // Assigned returns a RangeTable with all assigned code points for a given 65 | // Unicode version. This includes graphic, format, control, and private-use 66 | // characters. It returns nil if the data for the given version is not 67 | // available. 68 | func Assigned(version string) *unicode.RangeTable { 69 | return assigned[version] 70 | } 71 | -------------------------------------------------------------------------------- /unicode/rangetable/rangetable_test.go: -------------------------------------------------------------------------------- 1 | package rangetable 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | "unicode" 7 | ) 8 | 9 | var ( 10 | empty = &unicode.RangeTable{} 11 | many = &unicode.RangeTable{ 12 | R16: []unicode.Range16{{0, 0xffff, 5}}, 13 | R32: []unicode.Range32{{0x10004, 0x10009, 5}}, 14 | LatinOffset: 0, 15 | } 16 | ) 17 | 18 | func TestVisit(t *testing.T) { 19 | Visit(empty, func(got rune) { 20 | t.Error("call from empty RangeTable") 21 | }) 22 | 23 | var want rune 24 | Visit(many, func(got rune) { 25 | if got != want { 26 | t.Errorf("got %U; want %U", got, want) 27 | } 28 | want += 5 29 | }) 30 | if want -= 5; want != 0x10009 { 31 | t.Errorf("last run was %U; want U+10009", want) 32 | } 33 | } 34 | 35 | func TestNew(t *testing.T) { 36 | for i, rt := range []*unicode.RangeTable{ 37 | empty, 38 | unicode.Co, 39 | unicode.Letter, 40 | unicode.ASCII_Hex_Digit, 41 | many, 42 | maxRuneTable, 43 | } { 44 | var got, want []rune 45 | Visit(rt, func(r rune) { 46 | want = append(want, r) 47 | }) 48 | Visit(New(want...), func(r rune) { 49 | got = append(got, r) 50 | }) 51 | if !reflect.DeepEqual(got, want) { 52 | t.Errorf("%d:\ngot %v;\nwant %v", i, got, want) 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /unicode/runenames/runenames.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:generate go run gen.go 6 | 7 | // Package runenames provides rune names from the Unicode Character Database. 8 | // For example, the name for '\u0100' is "LATIN CAPITAL LETTER A WITH MACRON". 9 | // 10 | // See https://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt 11 | package runenames 12 | 13 | import ( 14 | "sort" 15 | ) 16 | 17 | // Name returns the name for r. 18 | func Name(r rune) string { 19 | i := sort.Search(len(entries), func(j int) bool { 20 | return entries[j].startRune() > r 21 | }) 22 | if i == 0 { 23 | return "" 24 | } 25 | e := entries[i-1] 26 | 27 | offset := int(r - e.startRune()) 28 | if offset >= e.numRunes() { 29 | return "" 30 | } 31 | 32 | if e.direct() { 33 | o := e.index() 34 | n := e.len() 35 | return directData[o : o+n] 36 | } 37 | 38 | start := int(index[e.index()+offset]) 39 | end := int(index[e.index()+offset+1]) 40 | base1 := e.base() << 16 41 | base2 := base1 42 | if start > end { 43 | base2 += 1 << 16 44 | } 45 | return singleData[start+base1 : end+base2] 46 | } 47 | 48 | func (e entry) len() int { return e.base() } 49 | -------------------------------------------------------------------------------- /unicode/runenames/runenames_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package runenames 6 | 7 | import ( 8 | "strings" 9 | "testing" 10 | "unicode" 11 | 12 | "golang.org/x/text/internal/gen" 13 | "golang.org/x/text/internal/testtext" 14 | "golang.org/x/text/internal/ucd" 15 | ) 16 | 17 | func TestName(t *testing.T) { 18 | testtext.SkipIfNotLong(t) 19 | 20 | wants := make([]string, 1+unicode.MaxRune) 21 | ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) { 22 | wants[p.Rune(0)] = getName(p) 23 | }) 24 | 25 | nErrors := 0 26 | for r, want := range wants { 27 | got := Name(rune(r)) 28 | if got != want { 29 | t.Errorf("r=%#08x: got %q, want %q", r, got, want) 30 | nErrors++ 31 | if nErrors == 100 { 32 | t.Fatal("too many errors") 33 | } 34 | } 35 | } 36 | } 37 | 38 | // Copied from gen.go. 39 | func getName(p *ucd.Parser) string { 40 | s := p.String(ucd.Name) 41 | if s == "" { 42 | return "" 43 | } 44 | if s[0] == '<' { 45 | const first = ", First>" 46 | if i := strings.Index(s, first); i >= 0 { 47 | s = s[:i] + ">" 48 | } 49 | 50 | } 51 | return s 52 | } 53 | -------------------------------------------------------------------------------- /width/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package width_test 6 | 7 | import ( 8 | "fmt" 9 | 10 | "golang.org/x/text/width" 11 | ) 12 | 13 | func ExampleTransformer_fold() { 14 | s := "abｦ￦￮￥Ａ" 15 | f := width.Fold.String(s) 16 | fmt.Printf("%U: %s\n", []rune(s), s) 17 | fmt.Printf("%U: %s\n", []rune(f), f) 18 | 19 | // Output: 20 | // [U+0061 U+0062 U+FF66 U+FFE6 U+FFEE U+FFE5 U+FF21]: abｦ￦￮￥Ａ 21 | // [U+0061 U+0062 U+30F2 U+20A9 U+25CB U+00A5 U+0041]: abヲ₩○¥A 22 | } 23 | 24 | func ExampleTransformer_widen() { 25 | s := "ab¥ｦ₩￮" 26 | w := width.Widen.String(s) 27 | fmt.Printf("%U: %s\n", []rune(s), s) 28 | fmt.Printf("%U: %s\n", []rune(w), w) 29 | 30 | // Output: 31 | // [U+0061 U+0062 U+00A5 U+FF66 U+20A9 U+FFEE]: ab¥ｦ₩￮ 32 | // [U+FF41 U+FF42 U+FFE5 U+30F2 U+FFE6 U+25CB]: ａｂ￥ヲ￦○ 33 | } 34 | 35 | func ExampleTransformer_narrow() { 36 | s := "abヲ￦○￥Ａ" 37 | n := width.Narrow.String(s) 38 | fmt.Printf("%U: %s\n", []rune(s), s) 39 | fmt.Printf("%U: %s\n", []rune(n), n) 40 | 41 | // Ambiguous characters with a halfwidth equivalent get mapped as well. 42 | s = "←" 43 | n = width.Narrow.String(s) 44 | fmt.Printf("%U: %s\n", []rune(s), s) 45 | fmt.Printf("%U: %s\n", []rune(n), n) 46 | 47 | // Output: 48 | // [U+0061 U+0062 U+30F2 U+FFE6 U+25CB U+FFE5 U+FF21]: abヲ￦○￥Ａ 49 | // [U+0061 U+0062 U+FF66 U+20A9 U+FFEE U+00A5 U+0041]: abｦ₩￮¥A 50 | // [U+2190]: ← 51 | // [U+FFE9]: ￩ 52 | } 53 | -------------------------------------------------------------------------------- /width/gen_trieval.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | package main 8 | 9 | // elem is an entry of the width trie. The high byte is used to encode the type 10 | // of the rune. The low byte is used to store the index to a mapping entry in 11 | // the inverseData array. 12 | type elem uint16 13 | 14 | const ( 15 | tagNeutral elem = iota << typeShift 16 | tagAmbiguous 17 | tagWide 18 | tagNarrow 19 | tagFullwidth 20 | tagHalfwidth 21 | ) 22 | 23 | const ( 24 | numTypeBits = 3 25 | typeShift = 16 - numTypeBits 26 | 27 | // tagNeedsFold is true for all fullwidth and halfwidth runes except for 28 | // the Won sign U+20A9. 29 | tagNeedsFold = 0x1000 30 | 31 | // The Korean Won sign is halfwidth, but SHOULD NOT be mapped to a wide 32 | // variant. 33 | wonSign rune = 0x20A9 34 | ) 35 | -------------------------------------------------------------------------------- /width/kind_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=Kind"; DO NOT EDIT. 2 | 3 | package width 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[Neutral-0] 12 | _ = x[EastAsianAmbiguous-1] 13 | _ = x[EastAsianWide-2] 14 | _ = x[EastAsianNarrow-3] 15 | _ = x[EastAsianFullwidth-4] 16 | _ = x[EastAsianHalfwidth-5] 17 | } 18 | 19 | const _Kind_name = "NeutralEastAsianAmbiguousEastAsianWideEastAsianNarrowEastAsianFullwidthEastAsianHalfwidth" 20 | 21 | var _Kind_index = [...]uint8{0, 7, 25, 38, 53, 71, 89} 22 | 23 | func (i Kind) String() string { 24 | if i < 0 || i >= Kind(len(_Kind_index)-1) { 25 | return "Kind(" + strconv.FormatInt(int64(i), 10) + ")" 26 | } 27 | return _Kind_name[_Kind_index[i]:_Kind_index[i+1]] 28 | } 29 | -------------------------------------------------------------------------------- /width/tables_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package width 6 | 7 | import ( 8 | "testing" 9 | 10 | "golang.org/x/text/internal/testtext" 11 | ) 12 | 13 | const ( 14 | loSurrogate = 0xD800 15 | hiSurrogate = 0xDFFF 16 | ) 17 | 18 | func TestTables(t *testing.T) { 19 | testtext.SkipIfNotLong(t) 20 | 21 | runes := map[rune]Kind{} 22 | getWidthData(func(r rune, tag elem, _ rune) { 23 | runes[r] = tag.kind() 24 | }) 25 | for r := rune(0); r < 0x10FFFF; r++ { 26 | if loSurrogate <= r && r <= hiSurrogate { 27 | continue 28 | } 29 | p := LookupRune(r) 30 | if got, want := p.Kind(), runes[r]; got != want { 31 | t.Errorf("Kind of %U was %s; want %s.", r, got, want) 32 | } 33 | want, mapped := foldRune(r) 34 | if got := p.Folded(); (got == 0) == mapped || got != 0 && got != want { 35 | t.Errorf("Folded(%U) = %U; want %U", r, got, want) 36 | } 37 | want, mapped = widenRune(r) 38 | if got := p.Wide(); (got == 0) == mapped || got != 0 && got != want { 39 | t.Errorf("Wide(%U) = %U; want %U", r, got, want) 40 | } 41 | want, mapped = narrowRune(r) 42 | if got := p.Narrow(); (got == 0) == mapped || got != 0 && got != want { 43 | t.Errorf("Narrow(%U) = %U; want %U", r, got, want) 44 | } 45 | } 46 | } 47 | 48 | // TestAmbiguous verifies that ambiguous runes with a mapping always map to 49 | // a halfwidth rune. 50 | func TestAmbiguous(t *testing.T) { 51 | for r, m := range mapRunes { 52 | if m.e != tagAmbiguous { 53 | continue 54 | } 55 | if k := mapRunes[m.r].e.kind(); k != EastAsianHalfwidth { 56 | t.Errorf("Rune %U is ambiguous and maps to a rune of type %v", r, k) 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /width/trieval.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | package width 4 | 5 | // elem is an entry of the width trie. The high byte is used to encode the type 6 | // of the rune. The low byte is used to store the index to a mapping entry in 7 | // the inverseData array. 8 | type elem uint16 9 | 10 | const ( 11 | tagNeutral elem = iota << typeShift 12 | tagAmbiguous 13 | tagWide 14 | tagNarrow 15 | tagFullwidth 16 | tagHalfwidth 17 | ) 18 | 19 | const ( 20 | numTypeBits = 3 21 | typeShift = 16 - numTypeBits 22 | 23 | // tagNeedsFold is true for all fullwidth and halfwidth runes except for 24 | // the Won sign U+20A9. 25 | tagNeedsFold = 0x1000 26 | 27 | // The Korean Won sign is halfwidth, but SHOULD NOT be mapped to a wide 28 | // variant. 29 | wonSign rune = 0x20A9 30 | ) 31 | --------------------------------------------------------------------------------