├── .gitignore ├── COPYING ├── MANIFEST.in ├── README.md ├── covo ├── cvutils ├── __init__.py ├── alphabet.py ├── att.py ├── corpora.py ├── data │ ├── ab │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── af │ │ └── alphabet.txt │ ├── am │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── ar │ │ ├── alphabet.txt │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── as │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── ast │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── az │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── azz │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── ba │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── bas │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── wikipedia.txt │ ├── be │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── bg │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── bn │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── punct.tsv │ │ └── validate.tsv │ ├── br │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── ca │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── ckb │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── ckt │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── cnh │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── co │ │ └── alphabet.txt │ ├── cs │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── ctp │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── cv │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── cy │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── da │ │ ├── alphabet.txt │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── de │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── dv │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── punct.tsv │ │ └── validate.tsv │ ├── dyu │ │ └── alphabet.txt │ ├── ee │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── el │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── en │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── eo │ │ ├── alphabet.txt │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── es │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── et │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── eu │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── fa │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── fi │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── fr │ │ ├── alphabet.txt │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── fy-NL │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ ├── vocab.tsv │ │ └── wikipedia.txt │ ├── ga-IE │ │ ├── alphabet.txt │ │ ├── validate.tsv │ │ └── wikipedia.txt │ ├── gl │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── gn │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── ha │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── he │ │ └── alphabet.txt │ ├── hi │ │ ├── alphabet.txt │ │ ├── phon.att │ │ └── validate.tsv │ ├── hsb │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── punct.tsv │ │ └── validate.tsv │ ├── ht │ │ └── alphabet.txt │ ├── hu │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── hy-AM │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── wikipedia.txt │ ├── ia │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── id │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── ig │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── is │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── it │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── ka │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── kab │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── ki │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── kk │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── kmr │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── kpv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── wikipedia.txt │ ├── ky │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── lg │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── ln │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── lo │ │ └── alphabet.txt │ ├── lt │ │ ├── alphabet.txt │ │ ├── phon.att │ │ ├── phon.lexc │ │ └── validate.tsv │ ├── luo │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── lv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── mdf │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── mhr │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── mk │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── ml │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── mn │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── punct.tsv │ │ └── validate.tsv │ ├── mr │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── mrj │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── mt │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── punct.tsv │ │ └── validate.tsv │ ├── myv │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── ne-NP │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── nl │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── nn-NO │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── nr │ │ ├── alphabet.txt │ │ └── phon.tsv │ ├── nso │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── ny │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── oc │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── or │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── pa-IN │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── pl │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── ps │ │ └── alphabet.txt │ ├── pt │ │ ├── alphabet.txt │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── quc │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── wikipedia.txt │ ├── quy │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── rm-sursilv │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── wikipedia.txt │ ├── rm-vallader │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── wikipedia.txt │ ├── ro │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── ru │ │ ├── alphabet.txt │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── rw │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── sah │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── punct.tsv │ │ └── validate.tsv │ ├── sat │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── sc │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── sk │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── skr │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── sl │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── sq │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── sr │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── ss │ │ ├── alphabet.txt │ │ └── phon.tsv │ ├── st │ │ ├── alphabet.txt │ │ └── phon.tsv │ ├── sv-SE │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ ├── vocab.tsv │ │ └── wikipedia.txt │ ├── sw │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── ta │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── th │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── ti │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── tig │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── tk │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── tn │ │ ├── alphabet.txt │ │ └── phon.tsv │ ├── tos │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── tr │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── ts │ │ ├── alphabet.txt │ │ └── phon.tsv │ ├── tt │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── punct.tsv │ │ ├── validate.tsv │ │ └── vocab.tsv │ ├── tw │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── ug │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── uk │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── ur │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── uz │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── ve │ │ ├── alphabet.txt │ │ └── phon.tsv │ ├── vi │ │ ├── abbr.tsv │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ ├── punct.tsv │ │ └── validate.tsv │ ├── vot │ │ ├── alphabet.txt │ │ ├── validate.tsv │ │ └── wikipedia.txt │ ├── wo │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── xh │ │ ├── alphabet.txt │ │ └── phon.tsv │ ├── xty │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── yi │ │ ├── alphabet.txt │ │ └── validate.tsv │ ├── yo │ │ ├── alphabet.txt │ │ ├── phon.tsv │ │ └── validate.tsv │ ├── zgh │ │ └── alphabet.txt │ └── zu │ │ ├── alphabet.txt │ │ └── phon.tsv ├── exporters │ ├── __init__.py │ └── nemo.py ├── gutenberg.py ├── opus.weights ├── phonemiser.py ├── segmenter.py ├── tagger.py ├── tokeniser.py ├── transliterator.py ├── validator.py └── wikipedia.py ├── doc └── TROUBLESHOOTING.md ├── requirements.txt └── setup.py /cvutils/alphabet.py: -------------------------------------------------------------------------------- 1 | """Alphabet module""" 2 | import sys 3 | import os 4 | 5 | class Alphabet: 6 | """ 7 | >>> p = Alphabet('ab') 8 | >>> p.get_alphabet() 9 | 'абвгдежзийклмнопрстуфхцчшщъыьэюяёӏ' 10 | """ 11 | def __init__(self, lang): 12 | self.lang = lang 13 | self.alphabet = '' 14 | try: 15 | self.load_data() 16 | except FileNotFoundError: 17 | print('[Alphabet] Function not implemented', file=sys.stderr) 18 | 19 | def load_data(self): 20 | data_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') 21 | fd = open(os.path.join(data_dir, self.lang, 'alphabet.txt')) 22 | a = [' '] + [line.strip('\n') for line in fd.readlines()] 23 | a = list(set(''.join(a))) 24 | a.sort() 25 | self.alphabet = ''.join(a) 26 | 27 | def get_alphabet(self): 28 | return self.alphabet 29 | 30 | def write_alphabet(self, fn): 31 | fd = open(fn, 'w') 32 | fd.write(''.join([c + '\n' for c in self.alphabet])) 33 | fd.close() 34 | 35 | 36 | if __name__ == "__main__": 37 | import doctest 38 | doctest.testmod() 39 | 40 | -------------------------------------------------------------------------------- /cvutils/data/ab/alphabet.txt: -------------------------------------------------------------------------------- 1 | abаәбвгӷдежзӡикқҟлмнопԥрстҭуфхҳцҵчҷҽҿџшыьҩ- 2 | -------------------------------------------------------------------------------- /cvutils/data/ab/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | а a 3 | б b 4 | в v 5 | г ɡ 6 | гв ɡʷ 7 | гъ ʁ 8 | гъв ʁʷ 9 | гъь ʁʲ 10 | гь ɡʲ 11 | гӏ ʕ 12 | гӏв ʕʷ 13 | д d 14 | дж d͡ʒ 15 | джв d͡ʒʷ 16 | джь d͡ʑ 17 | дз d͡z 18 | е e 19 | ё jo 20 | ж ʒ 21 | жв ʒʷ 22 | жь ʑ 23 | з z 24 | и i 25 | й j 26 | к k 27 | кв kʷ 28 | къ qʼ 29 | къв qʷʼ 30 | къь qʲʼ 31 | кь kʲ 32 | кӏ kʼ 33 | кӏв kʷʼ 34 | кӏь kʲʼ 35 | л l 36 | ль ɮ 37 | лӏ ɬʼ 38 | м m 39 | н n 40 | о o 41 | п p 42 | пӏ pʼ 43 | р r 44 | с s 45 | т t 46 | тл ɬ 47 | тш t͡ʃ 48 | тӏ tʼ 49 | у w 50 | у u 51 | ф f 52 | фӏ fʼ 53 | х χ 54 | хв χʷ 55 | хъ q 56 | хъв qʷ 57 | хь χʲ 58 | хӏ ħ 59 | хӏв ħʷ 60 | ц t͡s 61 | цӏ t͡sʼ 62 | ч t͡ɕ 63 | чв t͡ʃʷ 64 | чӏ t͡ɕʼ 65 | чӏв t͡ʃʷʼ 66 | ш ʃ 67 | шв ʃʷ 68 | шӏ t͡ʃʼ 69 | щ ɕ 70 | ъ ʔ 71 | ы ə 72 | э e 73 | ю ju 74 | я ja 75 | -------------------------------------------------------------------------------- /cvutils/data/ab/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL ! _ 4 | REPL , _ 5 | REPL . _ 6 | REPL : _ 7 | REPL ; _ 8 | REPL ? _ 9 | REPL :- _ 10 | REPL – _ 11 | ALLOW а _ 12 | ALLOW ә _ 13 | ALLOW б _ 14 | ALLOW в _ 15 | ALLOW г _ 16 | ALLOW ӷ _ 17 | ALLOW д _ 18 | ALLOW е _ 19 | ALLOW ж _ 20 | ALLOW з _ 21 | ALLOW ӡ _ 22 | ALLOW и _ 23 | ALLOW к _ 24 | ALLOW қ _ 25 | ALLOW ҟ _ 26 | ALLOW л _ 27 | ALLOW м _ 28 | ALLOW н _ 29 | ALLOW о _ 30 | ALLOW п _ 31 | ALLOW ԥ _ 32 | ALLOW р _ 33 | ALLOW с _ 34 | ALLOW т _ 35 | ALLOW ҭ _ 36 | ALLOW у _ 37 | ALLOW ф _ 38 | ALLOW х _ 39 | ALLOW ҳ _ 40 | ALLOW ц _ 41 | ALLOW ҵ _ 42 | ALLOW ч _ 43 | ALLOW ҷ _ 44 | ALLOW ҽ _ 45 | ALLOW ҿ _ 46 | ALLOW џ _ 47 | ALLOW ш _ 48 | ALLOW ы _ 49 | ALLOW ь _ 50 | ALLOW ҩ _ 51 | ALLOW - _ 52 | ALLOW _ _ 53 | -------------------------------------------------------------------------------- /cvutils/data/ab/vocab.tsv: -------------------------------------------------------------------------------- 1 | акымзарак 2 | акы 3 | ҩба 4 | хԥа 5 | ԥшьба 6 | хәба 7 | фба 8 | быжьба 9 | ааба 10 | жәба 11 | ааи 12 | мап 13 | -------------------------------------------------------------------------------- /cvutils/data/af/alphabet.txt: -------------------------------------------------------------------------------- 1 | aáäbcdeéèêëfghiíîïjklmnʼnoóôöpqrstuúûüvwxyýz 2 | -------------------------------------------------------------------------------- /cvutils/data/am/alphabet.txt: -------------------------------------------------------------------------------- 1 | ሀሁሂሃሄህሆለሉሊላሌልሎሏሐሑሒሓሔሕሖሗመሙሚማሜምሞሟሠሡሢሣሤሥሦሧረሩሪራሬርሮሯሰሱሲሳሴስሶሷሸሹሺሻሼሽሾሿቀቁቂቃቄቅቆቈቊቋቌቍበቡቢባቤብቦቧቨቩቪቫቬቭቮቯተቱቲታቴትቶቷቸቹቺቻቼችቾቿኀኁኂኃኄኅኆኈኊኋኌኍነኑኒናኔንኖኗኘኙኚኛኜኝኞኟአኡኢኣኤእኦኧከኩኪካኬክኮኰኲኳኴኵኸኹኺኻኼኽኾዀዂዃዄዅወዉዊዋዌውዎዐዑዒዓዔዕዖዘዙዚዛዜዝዞዟዠዡዢዣዤዥዦዧየዩዪያዬይዮደዱዲዳዴድዶዷጀጁጂጃጄጅጆጇገጉጊጋጌግጎጐጒጓጔጕጠጡጢጣጤጥጦጧጨጩጪጫጬጭጮጯጰጱጲጳጴጵጶጷጸጹጺጻጼጽጾጿፀፁፂፃፄፅፆፈፉፊፋፌፍፎፏፐፑፒፓፔፕፖፗ 2 | -------------------------------------------------------------------------------- /cvutils/data/ar/alphabet.txt: -------------------------------------------------------------------------------- 1 | خابتثجحدذرزسشصضطظعغفقكلمنهويیؤةىء 2 | -------------------------------------------------------------------------------- /cvutils/data/ar/vocab.tsv: -------------------------------------------------------------------------------- 1 | صفر 2 | واحد 3 | إثنان 4 | ثلاثة 5 | أربعة 6 | خمسة 7 | ستة 8 | سبعة 9 | ثمانية 10 | تسعة 11 | نعم 12 | -------------------------------------------------------------------------------- /cvutils/data/as/alphabet.txt: -------------------------------------------------------------------------------- 1 | ঁংঃʼঅআইঈউঊঋৠঌৡএঐওঔযািীুূৃৄৢৣেৈোৌ্ৗকখগঘঙচছজঝঞটঠডড়ঢঢ়ণতথদধনপফবভমযয়ৰলৱশষসহৎর -------------------------------------------------------------------------------- /cvutils/data/as/validate.tsv: -------------------------------------------------------------------------------- 1 | NFKC _ _ _ _ 2 | REPL , _ _ _ 3 | REPL ! _ _ _ 4 | REPL - _ _ _ 5 | REPL ” _ _ _ 6 | REPL “ _ _ _ 7 | REPL ? _ _ _ 8 | REPL । _ _ _ 9 | REPL " _ _ _ 10 | REPL ' ʼ _ _ 11 | NORM ’ ʼ 2019 02bc 12 | NORM ‘ ʼ 2018 02bc 13 | ALLOW ঁ _ 0981 _ 14 | ALLOW ং _ 0982 _ 15 | ALLOW ঃ _ 0983 _ 16 | ALLOW ʼ _ 02bc _ 17 | ALLOW অ _ 0985 _ 18 | ALLOW আ _ 0986 _ 19 | ALLOW ই _ 0987 _ 20 | ALLOW ঈ _ 0988 _ 21 | ALLOW উ _ 0989 _ 22 | ALLOW ঊ _ 098a _ 23 | ALLOW ঋ _ 098b _ 24 | ALLOW ৠ _ 09e0 _ 25 | ALLOW ঌ _ 098c _ 26 | ALLOW ৡ _ 09e1 _ 27 | ALLOW এ _ 098f _ 28 | ALLOW ঐ _ 0990 _ 29 | ALLOW ও _ 0993 _ 30 | ALLOW ঔ _ 0994 _ 31 | ALLOW য _ 09af _ 32 | ALLOW া _ 09be _ 33 | ALLOW ি _ 09bf _ 34 | ALLOW ী _ 09c0 _ 35 | ALLOW ু _ 09c1 _ 36 | ALLOW ূ _ 09c2 _ 37 | ALLOW ৃ _ 09c3 _ 38 | ALLOW ৄ _ 09c4 _ 39 | ALLOW ৢ _ 09e2 _ 40 | ALLOW ৣ _ 09e3 _ 41 | ALLOW ে _ 09c7 _ 42 | ALLOW ৈ _ 09c8 _ 43 | ALLOW ো _ 09cb _ 44 | ALLOW ৌ _ 09cc _ 45 | ALLOW ্ _ 09cd _ 46 | ALLOW ৗ _ 09d7 _ 47 | ALLOW ক _ 0995 _ 48 | ALLOW খ _ 0996 _ 49 | ALLOW গ _ 0997 _ 50 | ALLOW ঘ _ 0998 _ 51 | ALLOW ঙ _ 0999 _ 52 | ALLOW চ _ 099a _ 53 | ALLOW ছ _ 099b _ 54 | ALLOW জ _ 099c _ 55 | ALLOW ঝ _ 099d _ 56 | ALLOW ঞ _ 099e _ 57 | ALLOW ট _ 099f _ 58 | ALLOW ঠ _ 09a0 _ 59 | ALLOW ড _ 09a1 _ 60 | ALLOW ড _ 09a1 _ 61 | ALLOW ় _ 09bc _ 62 | ALLOW ঢ _ 09a2 _ 63 | ALLOW ঢ _ 09a2 _ 64 | ALLOW ় _ 09bc _ 65 | ALLOW ণ _ 09a3 _ 66 | ALLOW ত _ 09a4 _ 67 | ALLOW থ _ 09a5 _ 68 | ALLOW দ _ 09a6 _ 69 | ALLOW ধ _ 09a7 _ 70 | ALLOW ন _ 09a8 _ 71 | ALLOW প _ 09aa _ 72 | ALLOW ফ _ 09ab _ 73 | ALLOW ব _ 09ac _ 74 | ALLOW ভ _ 09ad _ 75 | ALLOW ম _ 09ae _ 76 | ALLOW য _ 09af _ 77 | ALLOW য _ 09af _ 78 | ALLOW ় _ 09bc _ 79 | ALLOW ৰ _ 09f0 _ 80 | ALLOW ল _ 09b2 _ 81 | ALLOW ৱ _ 09f1 _ 82 | ALLOW শ _ 09b6 _ 83 | ALLOW ষ _ 09b7 _ 84 | ALLOW স _ 09b8 _ 85 | ALLOW হ _ 09b9 _ 86 | ALLOW ৎ _ 09ce _ 87 | ALLOW র _ 09b0 _ 88 | ALLOW _ _ 0020 _ 89 | -------------------------------------------------------------------------------- /cvutils/data/ast/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghilmnñopqrstuvxyzáéíóúkḷüw'- 2 | -------------------------------------------------------------------------------- /cvutils/data/ast/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL , _ 002c _ 5 | REPL . _ 002e _ 6 | REPL ? _ 003f _ 7 | SKIP ¡ _ 00a1 _ 8 | SKIP ¿ _ 00bf _ 9 | NORM ’ ' 2019 _ 10 | ALLOW ' _ 0027 _ 11 | ALLOW - _ 002d _ 12 | ALLOW a _ 0061 _ 13 | ALLOW á _ 00e1 _ 14 | ALLOW b _ 0062 _ 15 | ALLOW c _ 0063 _ 16 | ALLOW d _ 0064 _ 17 | ALLOW e _ 0065 _ 18 | ALLOW é _ 00e9 _ 19 | ALLOW f _ 0066 _ 20 | ALLOW g _ 0067 _ 21 | ALLOW h _ 0068 _ 22 | ALLOW i _ 0069 _ 23 | ALLOW í _ 00ed _ 24 | ALLOW l _ 006c _ 25 | ALLOW m _ 006d _ 26 | ALLOW n _ 006e _ 27 | ALLOW ñ _ 00f1 _ 28 | ALLOW o _ 006f _ 29 | ALLOW ó _ 00f3 _ 30 | ALLOW p _ 0070 _ 31 | ALLOW q _ 0071 _ 32 | ALLOW r _ 0072 _ 33 | ALLOW s _ 0073 _ 34 | ALLOW t _ 0074 _ 35 | ALLOW u _ 0075 _ 36 | ALLOW ú _ 00fa _ 37 | ALLOW v _ 0076 _ 38 | ALLOW x _ 0078 _ 39 | ALLOW y _ 0079 _ 40 | ALLOW z _ 007a _ 41 | ALLOW k _ 006b _ 42 | ALLOW ḷ _ 1e37 _ 43 | ALLOW ü _ 00fc _ 44 | ALLOW w _ 0077 _ 45 | ALLOW _ _ 0020 _ 46 | -------------------------------------------------------------------------------- /cvutils/data/az/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcçdeəfgğhiıjklmnoöpqrsştuüvxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/az/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a ɑ 3 | b b 4 | c dʒ 5 | ç tʃ 6 | d d 7 | e e 8 | ә æ 9 | f f 10 | g ɟ 11 | ğ ɣ 12 | h h 13 | x x 14 | ı ɯ 15 | i i 16 | j ʒ 17 | k c 18 | k k 19 | q ɡ 20 | l l 21 | m m 22 | n n 23 | o o 24 | ö ø 25 | p p 26 | r r 27 | s s 28 | ş ʃ 29 | t t 30 | u u 31 | ü y 32 | v v 33 | y j 34 | z z 35 | -------------------------------------------------------------------------------- /cvutils/data/az/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL " _ 4 | REPL ' _ 5 | REPL , _ 6 | REPL - _ 7 | REPL . _ 8 | REPL ; _ 9 | REPL ? _ 10 | SKIP 0 _ 11 | SKIP 1 _ 12 | SKIP 2 _ 13 | SKIP 3 _ 14 | SKIP 4 _ 15 | SKIP 5 _ 16 | SKIP 6 _ 17 | SKIP 7 _ 18 | SKIP 8 _ 19 | SKIP 9 _ 20 | NORM i̇ i 21 | ALLOW a _ 22 | ALLOW b _ 23 | ALLOW c _ 24 | ALLOW ç _ 25 | ALLOW d _ 26 | ALLOW e _ 27 | ALLOW ə _ 28 | ALLOW f _ 29 | ALLOW g _ 30 | ALLOW ğ _ 31 | ALLOW h _ 32 | ALLOW i _ 33 | ALLOW ı _ 34 | ALLOW j _ 35 | ALLOW k _ 36 | ALLOW l _ 37 | ALLOW m _ 38 | ALLOW n _ 39 | ALLOW o _ 40 | ALLOW ö _ 41 | ALLOW p _ 42 | ALLOW q _ 43 | ALLOW r _ 44 | ALLOW s _ 45 | ALLOW ş _ 46 | ALLOW t _ 47 | ALLOW u _ 48 | ALLOW ü _ 49 | ALLOW v _ 50 | ALLOW x _ 51 | ALLOW y _ 52 | ALLOW z _ 53 | ALLOW _ _ 54 | -------------------------------------------------------------------------------- /cvutils/data/azz/alphabet.txt: -------------------------------------------------------------------------------- 1 | aábcdeéfghiíjklmnoópqrstuúvwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/azz/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | b b 4 | d d 5 | e e 6 | f f 7 | g g 8 | h h 9 | i i 10 | j h 11 | k k 12 | l l 13 | m m 14 | n n 15 | o o 16 | p p 17 | r r 18 | s s 19 | t t 20 | u u 21 | ua wa 22 | ue we 23 | v v 24 | w w 25 | x ʃ 26 | y j 27 | z s 28 | ca ka 29 | ce se 30 | ci si 31 | co ko 32 | cu ku 33 | qua kwa 34 | que ke 35 | qui ki 36 | quo kwo 37 | á a 38 | é e 39 | í i 40 | ó o 41 | ú u 42 | -------------------------------------------------------------------------------- /cvutils/data/azz/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ' _ _ _ 4 | REPL ¡ _ _ _ 5 | REPL ¿ _ _ _ 6 | REPL * _ _ _ 7 | REPL , _ _ _ 8 | REPL . _ _ _ 9 | REPL ; _ _ _ 10 | REPL ? _ _ _ 11 | REPL [ _ _ _ 12 | REPL ] _ _ _ 13 | REPL { _ _ _ 14 | REPL } _ _ _ 15 | REPL | _ _ _ 16 | REPL “ _ _ _ 17 | REPL ” _ _ _ 18 | REPL ‘ _ _ _ 19 | REPL ’ _ _ _ 20 | SKIP = _ _ _ 21 | SKIP & _ _ _ 22 | SKIP ( _ _ _ 23 | SKIP ) _ _ _ 24 | ALLOW a _ 0061 _ 25 | ALLOW á _ 00e1 _ 26 | ALLOW b _ 0062 _ 27 | ALLOW c _ 0063 _ 28 | ALLOW d _ 0064 _ 29 | ALLOW e _ 0065 _ 30 | ALLOW é _ 00e9 _ 31 | ALLOW f _ 0066 _ 32 | ALLOW g _ 0067 _ 33 | ALLOW h _ 0068 _ 34 | ALLOW i _ 0069 _ 35 | ALLOW í _ 00ed _ 36 | ALLOW j _ 006a _ 37 | ALLOW k _ 006b _ 38 | ALLOW l _ 006c _ 39 | ALLOW m _ 006d _ 40 | ALLOW n _ 006e _ 41 | ALLOW o _ 006f _ 42 | ALLOW ó _ 00f3 _ 43 | ALLOW p _ 0070 _ 44 | ALLOW q _ 0071 _ 45 | ALLOW r _ 0072 _ 46 | ALLOW s _ 0073 _ 47 | ALLOW t _ 0074 _ 48 | ALLOW u _ 0075 _ 49 | ALLOW ú _ 00fa _ 50 | ALLOW v _ 0076 _ 51 | ALLOW w _ 0077 _ 52 | ALLOW x _ 0078 _ 53 | ALLOW y _ 0079 _ 54 | ALLOW z _ 007a _ 55 | ALLOW _ _ _ _ 56 | -------------------------------------------------------------------------------- /cvutils/data/ba/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгғдҙеёжзийкҡлмнңоөпрсҫтуүфхһцчшщъыьэәюя 2 | -------------------------------------------------------------------------------- /cvutils/data/ba/phon.tsv: -------------------------------------------------------------------------------- 1 | а ɑ 2 | а a 3 | б b 4 | в v 5 | в w 6 | ғ ɣ 7 | д d 8 | ҙ ð 9 | е jɪ̞ 10 | е ɪ̞ 11 | е je 12 | е e 13 | ё jo 14 | ж ʒ 15 | з z 16 | и i 17 | й j 18 | к k 19 | ҡ q 20 | л l 21 | м m 22 | н n 23 | ң ŋ 24 | о ʊ̞ 25 | о o 26 | ө ø 27 | п p 28 | р r 29 | с s 30 | ҫ θ 31 | т t 32 | у u 33 | у w 34 | ү y 35 | ү w 36 | ф f 37 | х χ 38 | һ h 39 | ц ts 40 | ч tɕ 41 | ш ʂ 42 | щ ɕɕ 43 | ъ - 44 | ы ɯ 45 | ы ɨ 46 | ь ʲ 47 | э ɪ̞ 48 | э e 49 | ә æ 50 | ю ju 51 | я jɑ 52 | я ja 53 | -------------------------------------------------------------------------------- /cvutils/data/ba/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL " _ _ _ 4 | REPL « _ _ _ 5 | REPL » _ _ _ 6 | REPL : _ _ _ 7 | REPL ; _ _ _ 8 | REPL ! _ _ _ 9 | REPL ? _ _ _ 10 | REPL , _ _ _ 11 | REPL . _ _ _ 12 | REPL - _ _ _ 13 | REPL – _ _ _ 14 | REPL — _ 2014 _ 15 | REPL “ _ _ _ 16 | REPL ” _ _ _ 17 | NORM h һ _ _ 18 | ALLOW а _ 0430 _ 19 | ALLOW б _ 0431 _ 20 | ALLOW в _ 0432 _ 21 | ALLOW г _ 0433 _ 22 | ALLOW ғ _ 0493 _ 23 | ALLOW д _ 0434 _ 24 | ALLOW ҙ _ 0499 _ 25 | ALLOW е _ 0435 _ 26 | ALLOW ё _ 0451 _ 27 | ALLOW ж _ 0436 _ 28 | ALLOW з _ 0437 _ 29 | ALLOW и _ 0438 _ 30 | ALLOW й _ 0439 _ 31 | ALLOW к _ 043a _ 32 | ALLOW ҡ _ 04a1 _ 33 | ALLOW л _ 043b _ 34 | ALLOW м _ 043c _ 35 | ALLOW н _ 043d _ 36 | ALLOW ң _ 04a3 _ 37 | ALLOW о _ 043e _ 38 | ALLOW ө _ 04e9 _ 39 | ALLOW п _ 043f _ 40 | ALLOW р _ 0440 _ 41 | ALLOW с _ 0441 _ 42 | ALLOW ҫ _ 04ab _ 43 | ALLOW т _ 0442 _ 44 | ALLOW у _ 0443 _ 45 | ALLOW ү _ 04af _ 46 | ALLOW ф _ 0444 _ 47 | ALLOW х _ 0445 _ 48 | ALLOW һ _ 04bb _ 49 | ALLOW ц _ 0446 _ 50 | ALLOW ч _ 0447 _ 51 | ALLOW ш _ 0448 _ 52 | ALLOW щ _ 0449 _ 53 | ALLOW ъ _ 044a _ 54 | ALLOW ы _ 044b _ 55 | ALLOW ь _ 044c _ 56 | ALLOW э _ 044d _ 57 | ALLOW ә _ 04d9 _ 58 | ALLOW ю _ 044e _ 59 | ALLOW я _ 044f _ 60 | ALLOW 61 | _ 000a _ 62 | -------------------------------------------------------------------------------- /cvutils/data/ba/vocab.tsv: -------------------------------------------------------------------------------- 1 | ноль 2 | бер 3 | ике 4 | өс 5 | дүрт 6 | биш 7 | алты 8 | ете 9 | һигеҙ 10 | туғыҙ 11 | эйе 12 | юҡ 13 | -------------------------------------------------------------------------------- /cvutils/data/bas/alphabet.txt: -------------------------------------------------------------------------------- 1 | abdeéfghijklmnñoôprstuvwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/bas/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | é e 4 | e ɛ 5 | i i 6 | i ı 7 | ô o 8 | o ɔ 9 | u u 10 | b b 11 | tj c 12 | tj ɟ 13 | d d 14 | f f 15 | g g 16 | h h 17 | j j 18 | k k 19 | l l 20 | p p 21 | q q 22 | r r 23 | s s 24 | t t 25 | v v 26 | w w 27 | x x 28 | y y 29 | z z 30 | b ɓ 31 | m m 32 | n n 33 | ñ ŋ 34 | -------------------------------------------------------------------------------- /cvutils/data/bas/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL : _ _ _ 4 | REPL ; _ _ _ 5 | REPL ! _ _ _ 6 | REPL , _ _ _ 7 | REPL . _ _ _ 8 | REPL ? _ _ _ 9 | SKIP ̈ _ 0308 _ 10 | ALLOW a _ 0061 _ 11 | ALLOW b _ 0062 _ 12 | ALLOW d _ 0064 _ 13 | ALLOW e _ 0065 _ 14 | ALLOW é _ 00e9 _ 15 | ALLOW f _ 0066 _ 16 | ALLOW g _ 0067 _ 17 | ALLOW h _ 0068 _ 18 | ALLOW i _ 0069 _ 19 | ALLOW j _ 006a _ 20 | ALLOW k _ 006b _ 21 | ALLOW l _ 006c _ 22 | ALLOW m _ 006d _ 23 | ALLOW n _ 006e _ 24 | ALLOW ñ _ 00f1 _ 25 | ALLOW o _ 006f _ 26 | ALLOW ô _ 00f4 _ 27 | ALLOW p _ 0070 _ 28 | ALLOW r _ 0072 _ 29 | ALLOW s _ 0073 _ 30 | ALLOW t _ 0074 _ 31 | ALLOW u _ 0075 _ 32 | ALLOW v _ 0076 _ 33 | ALLOW w _ 0077 _ 34 | ALLOW x _ 0078 _ 35 | ALLOW y _ 0079 _ 36 | ALLOW z _ 007a _ 37 | ALLOW _ _ 0020 _ 38 | -------------------------------------------------------------------------------- /cvutils/data/bas/wikipedia.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ftyers/commonvoice-utils/5e3634fe1832aa58602f551156f7871cc142beb5/cvutils/data/bas/wikipedia.txt -------------------------------------------------------------------------------- /cvutils/data/be/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгдееёжзійклмнопрстуўфхцчшыьэюяʼ- -------------------------------------------------------------------------------- /cvutils/data/be/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | а a 3 | б b 4 | в v 5 | г ɣ 6 | д d 7 | е jɛ 8 | е ʲɛ 9 | ё jɔ 10 | ё ʲɔ 11 | ж ʐ 12 | з z 13 | і i 14 | і ʲi 15 | і ji 16 | й j 17 | к k 18 | л l 19 | м m 20 | н n 21 | о ɔ 22 | п p 23 | р r 24 | с s 25 | т t 26 | у u 27 | ў w 28 | ф f 29 | х x 30 | ц t͡s 31 | ч t͡ʂ 32 | ш ʂ 33 | ы ɨ 34 | ь ʲ 35 | э ɛ 36 | ю ju 37 | ю ʲu 38 | я ja 39 | я ʲa 40 | ʼ _ 41 | -------------------------------------------------------------------------------- /cvutils/data/be/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL . _ _ _ 4 | REPL ? _ _ _ 5 | REPL ! _ _ _ 6 | REPL " _ _ _ 7 | REPL ; _ _ _ 8 | REPL — _ _ _ 9 | REPL : _ _ _ 10 | REPL ‒ _ _ _ 11 | REPL « _ _ _ 12 | REPL » _ _ _ 13 | REPL – _ _ _ 14 | REPL , _ _ _ 15 | REPL ' ʼ 0027 02bc 16 | REPL ’ ʼ 2019 02bc 17 | NORM i і 0069 0456 18 | ALLOW а _ 0430 _ 19 | ALLOW б _ 0431 _ 20 | ALLOW в _ 0432 _ 21 | ALLOW г _ 0433 _ 22 | ALLOW д _ 0434 _ 23 | ALLOW е _ 0435 _ 24 | ALLOW е _ 0435 _ 25 | ALLOW ё _ 0451 _ 26 | ALLOW ж _ 0436 _ 27 | ALLOW з _ 0437 _ 28 | ALLOW і _ 0456 _ 29 | ALLOW й _ 0439 _ 30 | ALLOW к _ 043a _ 31 | ALLOW л _ 043b _ 32 | ALLOW м _ 043c _ 33 | ALLOW н _ 043d _ 34 | ALLOW о _ 043e _ 35 | ALLOW п _ 043f _ 36 | ALLOW р _ 0440 _ 37 | ALLOW с _ 0441 _ 38 | ALLOW т _ 0442 _ 39 | ALLOW у _ 0443 _ 40 | ALLOW ў _ 045e _ 41 | ALLOW ф _ 0444 _ 42 | ALLOW х _ 0445 _ 43 | ALLOW ц _ 0446 _ 44 | ALLOW ч _ 0447 _ 45 | ALLOW ш _ 0448 _ 46 | ALLOW ы _ 044b _ 47 | ALLOW ь _ 044c _ 48 | ALLOW э _ 044d _ 49 | ALLOW ю _ 044e _ 50 | ALLOW я _ 044f _ 51 | ALLOW ʼ _ 02bc _ 52 | ALLOW - _ 002d _ 53 | ALLOW _ _ 0020 _ 54 | -------------------------------------------------------------------------------- /cvutils/data/bg/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгдежзийклмнопрстуфхцчшщъьюя 2 | -------------------------------------------------------------------------------- /cvutils/data/bg/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | а a 3 | а ɐ 4 | а ə 5 | б b 6 | б p 7 | в v 8 | в f 9 | г ɡ 10 | г k 11 | д d 12 | д t 13 | е ɛ 14 | ж ʒ 15 | ж ʃ 16 | з z 17 | з s 18 | и i 19 | й j 20 | к k 21 | к ɡ 22 | л l 23 | л ʎ 24 | л ɫ 25 | м m 26 | н n 27 | о ɔ 28 | о o 29 | п p 30 | п b 31 | р r 32 | с s 33 | с z 34 | т t 35 | т d 36 | у u 37 | у o 38 | у ʊ 39 | ф f 40 | ф v 41 | х x 42 | ц t͡s 43 | ч t͡ʃ 44 | ш ʃ 45 | ш ʒ 46 | щ ʃt 47 | ъ ɤ 48 | ъ ɐ 49 | ъ ə 50 | ь j 51 | ю ju 52 | ю jo 53 | ю ʲu 54 | ю ʲo 55 | я ja 56 | я jɐ 57 | я ʲa 58 | я ʲɐ 59 | -------------------------------------------------------------------------------- /cvutils/data/bg/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL , _ 002c _ 6 | REPL . _ 002e _ 7 | REPL : _ 003a _ 8 | REPL ; _ 003b _ 9 | REPL ? _ 003f _ 10 | REPL ` _ 0060 _ 11 | REPL « _ 00ab _ 12 | REPL » _ 00bb _ 13 | REPL — _ 2014 _ 14 | REPL “ _ 201c _ 15 | REPL „ _ 201e _ 16 | REPL … _ 2026 _ 17 | SKIP ( _ 0028 _ 18 | SKIP ) _ 0029 _ 19 | ALLOW - _ 002d _ 20 | ALLOW a _ 0061 _ 21 | ALLOW c _ 0063 _ 22 | ALLOW e _ 0065 _ 23 | ALLOW n _ 006e _ 24 | ALLOW s _ 0073 _ 25 | ALLOW t _ 0074 _ 26 | ALLOW а _ 0430 _ 27 | ALLOW б _ 0431 _ 28 | ALLOW в _ 0432 _ 29 | ALLOW г _ 0433 _ 30 | ALLOW д _ 0434 _ 31 | ALLOW е _ 0435 _ 32 | ALLOW ж _ 0436 _ 33 | ALLOW з _ 0437 _ 34 | ALLOW и _ 0438 _ 35 | ALLOW ѝ _ 045d _ 36 | ALLOW й _ 0439 _ 37 | ALLOW к _ 043a _ 38 | ALLOW л _ 043b _ 39 | ALLOW м _ 043c _ 40 | ALLOW н _ 043d _ 41 | ALLOW о _ 043e _ 42 | ALLOW п _ 043f _ 43 | ALLOW р _ 0440 _ 44 | ALLOW с _ 0441 _ 45 | ALLOW т _ 0442 _ 46 | ALLOW у _ 0443 _ 47 | ALLOW ф _ 0444 _ 48 | ALLOW х _ 0445 _ 49 | ALLOW ц _ 0446 _ 50 | ALLOW ч _ 0447 _ 51 | ALLOW ш _ 0448 _ 52 | ALLOW щ _ 0449 _ 53 | ALLOW ъ _ 044a _ 54 | ALLOW ь _ 044c _ 55 | ALLOW ю _ 044e _ 56 | ALLOW я _ 044f _ 57 | ALLOW _ _ 0020 _ 58 | -------------------------------------------------------------------------------- /cvutils/data/bn/abbr.tsv: -------------------------------------------------------------------------------- 1 | 1 জ. 2 | -------------------------------------------------------------------------------- /cvutils/data/bn/alphabet.txt: -------------------------------------------------------------------------------- 1 | ্ঁখগঘঙচছজঝঞঠডঢণথদধনফবভমযরলশষসঅইউঋএওআঈঊঐঔি ুৃে ো াীূৈৌড়ঢ়য়ংঃঌকটতপহৎৠৡৰৱ 2 | -------------------------------------------------------------------------------- /cvutils/data/bn/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | EOS । 5 | -------------------------------------------------------------------------------- /cvutils/data/br/abbr.tsv: -------------------------------------------------------------------------------- 1 | 274 J. 2 | 228 d.s. 3 | 191 L. 4 | 182 M. 5 | 178 A. 6 | 160 E. 7 | 158 R. 8 | 147 C. 9 | 142 B. 10 | 126 S. 11 | 126 D. 12 | 122 H. 13 | 120 T. 14 | 115 W. 15 | 111 F. 16 | 110 h.a. 17 | 107 G. 18 | 90 P. 19 | 76 K. 20 | 69 d.l.e. 21 | 67 V. 22 | 64 I. 23 | 57 s.o. 24 | 36 d.l. 25 | 35 O. 26 | 30 Q. 27 | 28 X. 28 | 22 Y. 29 | 22 U. 30 | 21 i.e. 31 | 18 N. 32 | 4 É. 33 | 3 Z. 34 | 2 Á. 35 | 1 Y.-P. 36 | 1 v. 37 | 1 St. 38 | 1 s. 39 | 1 p. 40 | 1 Ø. 41 | 1 Ö. 42 | 1 g. 43 | 1 d. 44 | 1 c. 45 | 1 m. 46 | 1 saoz. 47 | -------------------------------------------------------------------------------- /cvutils/data/br/alphabet.txt: -------------------------------------------------------------------------------- 1 | 'aâbcdeêfghiîjklmnñoôpqrstuùûüvwxyz- 2 | -------------------------------------------------------------------------------- /cvutils/data/br/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | i i 3 | u y 4 | ou u 5 | e e 6 | e ɛ 7 | eu ø 8 | eu œ 9 | a a 10 | a ɑ 11 | o o 12 | o ɔ 13 | m m 14 | n n 15 | gn ɲ 16 | b b 17 | d d 18 | g g 19 | gw ɡʷ 20 | gou ɡʷ 21 | p p 22 | t t 23 | k k 24 | kw kʷ 25 | kou kʷ 26 | v v 27 | zh z 28 | j ʒ 29 | c'h ɣ 30 | c'h x 31 | f f 32 | s s 33 | ch ʃ 34 | h h 35 | r r 36 | y j 37 | w w 38 | l l 39 | lh ʎ 40 | ae aj 41 | an ɑ̃n 42 | añ ɑ̃ 43 | ao aw 44 | aou ɔw 45 | c'hw xw 46 | ê ɛː 47 | ei ɛi̯ 48 | eeu eø̯ 49 | eo eː 50 | eu œ 51 | eü ɛɥ 52 | eue ø̯e 53 | gn ɲ 54 | gw ɡw 55 | oa wa 56 | ôa oːa 57 | oe wɛ 58 | on ɔ̃n 59 | oñ ɔ̃ 60 | ou u 61 | oü oy̆ 62 | sh s 63 | sk sk 64 | st st 65 | ui ɥi 66 | vh f 67 | -------------------------------------------------------------------------------- /cvutils/data/br/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/br/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | REPL ! _ 3 | REPL " _ 4 | REPL % _ 5 | REPL ( _ 6 | REPL ) _ 7 | REPL * _ 8 | REPL + _ 9 | REPL , _ 10 | REPL . _ 11 | REPL / _ 12 | REPL : _ 13 | REPL ; _ 14 | REPL = _ 15 | REPL > _ 16 | REPL ? _ 17 | REPL « _ 18 | REPL » _ 19 | REPL – _ 20 | REPL “ _ 21 | REPL ” _ 22 | REPL … _ 23 | SKIP € _ 24 | SKIP 0 _ 25 | SKIP 1 _ 26 | SKIP ½ _ 27 | SKIP 2 _ 28 | SKIP 3 _ 29 | SKIP 4 _ 30 | SKIP 5 _ 31 | SKIP 6 _ 32 | SKIP 7 _ 33 | SKIP 8 _ 34 | SKIP 9 _ 35 | NORM ‘ ' 36 | NORM ’ ' 37 | NORM ʼ ' 38 | NORM é e 39 | NORM è e 40 | NORM ç c 41 | NORM à a 42 | ALLOW ' _ 43 | ALLOW a _ 44 | ALLOW â _ 45 | ALLOW b _ 46 | ALLOW c _ 47 | ALLOW d _ 48 | ALLOW e _ 49 | ALLOW ê _ 50 | ALLOW f _ 51 | ALLOW g _ 52 | ALLOW h _ 53 | ALLOW i _ 54 | ALLOW î _ 55 | ALLOW j _ 56 | ALLOW k _ 57 | ALLOW l _ 58 | ALLOW m _ 59 | ALLOW n _ 60 | ALLOW ñ _ 61 | ALLOW o _ 62 | ALLOW ô _ 63 | ALLOW p _ 64 | ALLOW q _ 65 | ALLOW r _ 66 | ALLOW s _ 67 | ALLOW t _ 68 | ALLOW u _ 69 | ALLOW ü _ 70 | ALLOW û _ 71 | ALLOW ù _ 72 | ALLOW v _ 73 | ALLOW w _ 74 | ALLOW x _ 75 | ALLOW y _ 76 | ALLOW z _ 77 | ALLOW - _ 78 | ALLOW _ _ 79 | -------------------------------------------------------------------------------- /cvutils/data/br/vocab.tsv: -------------------------------------------------------------------------------- 1 | mann 2 | unan 3 | daou 4 | tri 5 | pevar 6 | pemp 7 | c'hwec'h 8 | seizh 9 | eizh 10 | nav 11 | ya 12 | nann 13 | -------------------------------------------------------------------------------- /cvutils/data/ca/alphabet.txt: -------------------------------------------------------------------------------- 1 | aáàbcçdeéèfghiíïjklmnoóòpqrstuúüvwxyz'·- 2 | -------------------------------------------------------------------------------- /cvutils/data/ca/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/ca/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL « " 6 | REPL » " 7 | REPL ‟ " 8 | REPL ‘ ' 9 | REPL ´ ' 10 | REPL ` ' 11 | REPL ʼ ' 12 | REPL , _ 002c _ 13 | REPL . _ 002e _ 14 | REPL / _ 002f _ 15 | REPL : _ 003a _ 16 | REPL ; _ 003b _ 17 | REPL ? _ 003f _ 18 | REPL ~ _ 007e _ 19 | REPL ¡ _ 00a1 _ 20 | REPL « _ 00ab _ 21 | REPL » _ 00bb _ 22 | REPL ¿ _ 00bf _ 23 | REPL – _ 2013 _ 24 | REPL — _ 2014 _ 25 | REPL ― _ 2015 _ 26 | REPL ’ _ 2019 _ 27 | REPL “ _ 201c _ 28 | REPL ” _ 201d _ 29 | REPL … _ 2026 _ 30 | SKIP ( _ 0028 _ 31 | SKIP ) _ 0029 _ 32 | SKIP @ _ 0040 _ 33 | SKIP + _ 002b _ 34 | SKIP * _ 002a _ 35 | SKIP ° _ 00b0 _ 36 | NORM ā a 0101 0061 37 | NORM ō o 014d 006f 38 | NORM ū u 00fc 0075 39 | ALLOW a _ 0061 _ 40 | ALLOW á _ 00e1 _ 41 | ALLOW à _ 00e0 _ 42 | ALLOW b _ 0062 _ 43 | ALLOW c _ 0063 _ 44 | ALLOW ç _ 00e7 _ 45 | ALLOW d _ 0064 _ 46 | ALLOW e _ 0065 _ 47 | ALLOW é _ 00e9 _ 48 | ALLOW è _ 00e8 _ 49 | ALLOW f _ 0066 _ 50 | ALLOW g _ 0067 _ 51 | ALLOW h _ 0068 _ 52 | ALLOW i _ 0069 _ 53 | ALLOW ì _ 00ec _ 54 | ALLOW í _ 00ed _ 55 | ALLOW ï _ 00ef _ 56 | ALLOW j _ 006a _ 57 | ALLOW k _ 006b _ 58 | ALLOW l _ 006c _ 59 | ALLOW m _ 006d _ 60 | ALLOW n _ 006e _ 61 | ALLOW ñ _ 00f1 _ 62 | ALLOW o _ 006f _ 63 | ALLOW ó _ 00f3 _ 64 | ALLOW ò _ 00f2 _ 65 | ALLOW p _ 0070 _ 66 | ALLOW q _ 0071 _ 67 | ALLOW r _ 0072 _ 68 | ALLOW s _ 0073 _ 69 | ALLOW t _ 0074 _ 70 | ALLOW u _ 0075 _ 71 | ALLOW ú _ 00fa _ 72 | ALLOW ü _ 00fc _ 73 | ALLOW v _ 0076 _ 74 | ALLOW w _ 0077 _ 75 | ALLOW x _ 0078 _ 76 | ALLOW y _ 0079 _ 77 | ALLOW z _ 007a _ 78 | ALLOW ' _ 0027 _ 79 | ALLOW · _ 00b7 _ 80 | ALLOW - _ 002d _ 81 | ALLOW _ 0020 _ 82 | -------------------------------------------------------------------------------- /cvutils/data/ca/vocab.tsv: -------------------------------------------------------------------------------- 1 | zero 2 | u 3 | dos 4 | tres 5 | quatre 6 | cinc 7 | sis 8 | set 9 | vuit 10 | nou 11 | sí 12 | no 13 | -------------------------------------------------------------------------------- /cvutils/data/ckb/alphabet.txt: -------------------------------------------------------------------------------- 1 | بپتدکگقافڤسزخعغشژچجحھمنوىرڕلڵى-ێهاههوووۆائئیە 2 | -------------------------------------------------------------------------------- /cvutils/data/ckb/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | ب b 3 | پ p 4 | ت t 5 | د d 6 | ک k 7 | گ g 8 | ق q 9 | ا ʔ 10 | ف f 11 | ڤ v 12 | س s 13 | ز z 14 | خ x 15 | ع ʕ 16 | غ ɣ 17 | ش ʃ 18 | ژ ʒ 19 | چ ʧ 20 | ج ʤ 21 | ح ħ 22 | ھ h 23 | م m 24 | ن n 25 | و w 26 | ى j 27 | ر ɾ 28 | ڕ r 29 | ل l 30 | ڵ ɫ 31 | ى i 32 | - ɪ 33 | ێ e 34 | ه ɛ 35 | ا ه ə 36 | ه æ 37 | وو u 38 | و ʊ 39 | ۆ o 40 | ا ɑ 41 | -------------------------------------------------------------------------------- /cvutils/data/ckb/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | DEL   _ 2006 _ 4 | DEL ‏ _ 200f _ 5 | REPL ـ _ 0640 _ 6 | REPL « _ 00ab _ 7 | REPL » _ 00bb _ 8 | REPL ! _ 0021 _ 9 | REPL , _ 002c _ 10 | REPL . _ 002e _ 11 | REPL : _ 003a _ 12 | REPL ، _ 060c _ 13 | REPL ؛ _ 061b _ 14 | REPL ؟ _ 061f _ 15 | REPL – _ 2013 _ 16 | REPL ‘ _ 2018 _ 17 | REPL ’ _ 2019 _ 18 | REPL “ _ 201c _ 19 | REPL ” _ 201d _ 20 | REPL … _ 2026 _ 21 | SKIP 0 _ 0030 _ 22 | SKIP 1 _ 0031 _ 23 | SKIP 2 _ 0032 _ 24 | SKIP 3 _ 0033 _ 25 | SKIP 4 _ 0034 _ 26 | SKIP 5 _ 0035 _ 27 | SKIP 6 _ 0036 _ 28 | SKIP 7 _ 0037 _ 29 | SKIP 8 _ 0038 _ 30 | SKIP 9 _ 0039 _ 31 | SKIP f _ 0066 _ 32 | SKIP l _ 006c _ 33 | SKIP q _ 0071 _ 34 | SKIP r _ 0072 _ 35 | SKIP t _ 0074 _ 36 | SKIP x _ 0078 _ 37 | SKIP ( _ 0028 _ 38 | SKIP ) _ 0029 _ 39 | SKIP * _ 002a _ 40 | SKIP + _ 002b _ 41 | SKIP \ _ 005c _ 42 | SKIP [ _ 005b _ 43 | SKIP ] _ 005d _ 44 | SKIP ⌞ _ 231e _ 45 | SKIP ⌟ _ 231f _ 46 | ALLOW ‌ _ 200c _ 47 | ALLOW - _ 002d _ 48 | ALLOW ا _ 0627 _ 49 | ALLOW ب _ 0628 _ 50 | ALLOW پ _ 067e _ 51 | ALLOW ت _ 062a _ 52 | ALLOW ج _ 062c _ 53 | ALLOW چ _ 0686 _ 54 | ALLOW ح _ 062d _ 55 | ALLOW خ _ 062e _ 56 | ALLOW د _ 062f _ 57 | ALLOW ر _ 0631 _ 58 | ALLOW ز _ 0632 _ 59 | ALLOW ڕ _ 0695 _ 60 | ALLOW ژ _ 0698 _ 61 | ALLOW س _ 0633 _ 62 | ALLOW ش _ 0634 _ 63 | ALLOW ع _ 0639 _ 64 | ALLOW غ _ 063a _ 65 | ALLOW ف _ 0641 _ 66 | ALLOW ڤ _ 06a4 _ 67 | ALLOW ق _ 0642 _ 68 | ALLOW ک _ 06a9 _ 69 | ALLOW گ _ 06af _ 70 | ALLOW ل _ 0644 _ 71 | ALLOW ڵ _ 06b5 _ 72 | ALLOW م _ 0645 _ 73 | ALLOW ن _ 0646 _ 74 | ALLOW ه _ 0647 _ 75 | ALLOW و _ 0648 _ 76 | ALLOW ۆ _ 06c6 _ 77 | ALLOW ێ _ 06ce _ 78 | ALLOW ئ _ 0626 _ 79 | ALLOW ە _ 06d5 _ 80 | ALLOW ی _ 06cc _ 81 | ALLOW _ _ 0020 _ 82 | -------------------------------------------------------------------------------- /cvutils/data/ckt/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгдеёжзийкӄлԓмнӈопрстуфхчшщъыьэюяʼ 2 | -------------------------------------------------------------------------------- /cvutils/data/ckt/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | а a 3 | в w 4 | г ɣ 5 | е e 6 | ё o 7 | и i 8 | й j 9 | к k 10 | ӄ q 11 | ԓ ɬ 12 | м m 13 | н n 14 | ӈ ŋ 15 | о o 16 | п p 17 | р ɾ 18 | с c 19 | т t 20 | у u 21 | ч c 22 | ъ ʔ 23 | ы ə 24 | ь ʔ 25 | э e 26 | ю u 27 | я a 28 | ʼ ʔ 29 | ш ʃ 30 | щ ʃ 31 | б b 32 | д d 33 | ж ʐ 34 | з z 35 | л l 36 | ф f 37 | х x 38 | -------------------------------------------------------------------------------- /cvutils/data/ckt/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | NORM н' ӈ _ 04c8 4 | NORM к' ӄ _ 04c4 5 | NORM ' ʼ _ _ 6 | SKIP 0 _ _ _ 7 | SKIP 1 _ _ _ 8 | SKIP 2 _ _ _ 9 | SKIP 3 _ _ _ 10 | SKIP 4 _ _ _ 11 | SKIP 5 _ _ _ 12 | SKIP 6 _ _ _ 13 | SKIP 7 _ _ _ 14 | SKIP 8 _ _ _ 15 | SKIP 9 _ _ _ 16 | REPL , _ _ _ 17 | REPL . _ _ _ 18 | ALLOW а _ 0430 _ 19 | ALLOW б _ 0431 _ 20 | ALLOW в _ 0432 _ 21 | ALLOW г _ 0433 _ 22 | ALLOW д _ 0434 _ 23 | ALLOW е _ 0435 _ 24 | ALLOW ё _ 0451 _ 25 | ALLOW ж _ 0436 _ 26 | ALLOW з _ 0437 _ 27 | ALLOW и _ 0438 _ 28 | ALLOW й _ 0439 _ 29 | ALLOW к _ 043a _ 30 | ALLOW ӄ _ 04c4 _ 31 | ALLOW л _ 043b _ 32 | ALLOW ԓ _ 0513 _ 33 | ALLOW м _ 043c _ 34 | ALLOW н _ 043d _ 35 | ALLOW ӈ _ 04c8 _ 36 | ALLOW о _ 043e _ 37 | ALLOW п _ 043f _ 38 | ALLOW р _ 0440 _ 39 | ALLOW с _ 0441 _ 40 | ALLOW т _ 0442 _ 41 | ALLOW у _ 0443 _ 42 | ALLOW ф _ 0444 _ 43 | ALLOW х _ 0445 _ 44 | ALLOW ч _ 0447 _ 45 | ALLOW ш _ 0448 _ 46 | ALLOW щ _ 0449 _ 47 | ALLOW ъ _ 044a _ 48 | ALLOW ы _ 044b _ 49 | ALLOW ь _ 044c _ 50 | ALLOW э _ 044d _ 51 | ALLOW ю _ 044e _ 52 | ALLOW я _ 044f _ 53 | ALLOW ʼ _ 02bc _ 54 | ALLOW _ _ _ _ 55 | -------------------------------------------------------------------------------- /cvutils/data/cnh/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnopqrstṭuvwyz 2 | -------------------------------------------------------------------------------- /cvutils/data/cnh/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL ! _ 4 | REPL " _ 5 | REPL ' _ 6 | REPL , _ 7 | REPL - _ 8 | REPL . _ 9 | REPL ? _ 10 | SKIP / _ 11 | SKIP 1 _ 12 | SKIP 2 _ 13 | SKIP 4 _ 14 | SKIP 5 _ 15 | SKIP 8 _ 16 | SKIP 9 _ 17 | ALLOW a _ 18 | ALLOW b _ 19 | ALLOW c _ 20 | ALLOW d _ 21 | ALLOW e _ 22 | ALLOW f _ 23 | ALLOW g _ 24 | ALLOW h _ 25 | ALLOW i _ 26 | ALLOW j _ 27 | ALLOW k _ 28 | ALLOW l _ 29 | ALLOW m _ 30 | ALLOW n _ 31 | ALLOW o _ 32 | ALLOW p _ 33 | ALLOW q _ 34 | ALLOW r _ 35 | ALLOW s _ 36 | ALLOW t _ 37 | ALLOW ṭ _ 38 | ALLOW u _ 39 | ALLOW v _ 40 | ALLOW w _ 41 | ALLOW y _ 42 | ALLOW z _ 43 | ALLOW _ _ 44 | -------------------------------------------------------------------------------- /cvutils/data/co/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijlmnopqrstuvzàèìòùïü’ 2 | -------------------------------------------------------------------------------- /cvutils/data/cs/alphabet.txt: -------------------------------------------------------------------------------- 1 | aáeéěiíoóuúůyýbcčdďfghchjklmnňprřsštťvxxzž 2 | -------------------------------------------------------------------------------- /cvutils/data/cs/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | á aː 4 | e ɛ 5 | é ɛː 6 | ě ɛ 7 | ě jɛ 8 | i ɪ 9 | í iː 10 | o o 11 | ó oː 12 | u u 13 | ú uː 14 | ů uː 15 | y ɪ 16 | ý iː 17 | b b 18 | c t͡s 19 | č t͡ʃ 20 | d d 21 | ď ɟ 22 | f f 23 | g ɡ 24 | h ɦ 25 | ch x 26 | j j 27 | k k 28 | l l 29 | m m 30 | n n 31 | ň ɲ 32 | p p 33 | r r 34 | ř r̝ 35 | s s 36 | š ʃ 37 | t t 38 | ť c 39 | v v 40 | x ks 41 | x gz 42 | z z 43 | ž ʒ 44 | -------------------------------------------------------------------------------- /cvutils/data/cs/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | REPL ! _ 3 | REPL " _ 4 | REPL ( _ 5 | REPL ) _ 6 | REPL , _ 7 | REPL - _ 8 | REPL . _ 9 | REPL / _ 10 | REPL : _ 11 | REPL ; _ 12 | REPL ? _ 13 | REPL – _ 14 | REPL — _ 15 | REPL “ _ 16 | REPL „ _ 17 | REPL … _ 18 | SKIP 2 _ 19 | NORM ä a 20 | NORM ï i 21 | NORM ö o 22 | NORM ü u 23 | ALLOW a _ 24 | ALLOW á _ 25 | ALLOW e _ 26 | ALLOW é _ 27 | ALLOW ě _ 28 | ALLOW i _ 29 | ALLOW í _ 30 | ALLOW o _ 31 | ALLOW ó _ 32 | ALLOW u _ 33 | ALLOW ú _ 34 | ALLOW ů _ 35 | ALLOW y _ 36 | ALLOW ý _ 37 | ALLOW b _ 38 | ALLOW c _ 39 | ALLOW č _ 40 | ALLOW d _ 41 | ALLOW ď _ 42 | ALLOW f _ 43 | ALLOW g _ 44 | ALLOW h _ 45 | ALLOW c _ 46 | ALLOW h _ 47 | ALLOW j _ 48 | ALLOW k _ 49 | ALLOW l _ 50 | ALLOW m _ 51 | ALLOW n _ 52 | ALLOW ň _ 53 | ALLOW p _ 54 | ALLOW r _ 55 | ALLOW ř _ 56 | ALLOW s _ 57 | ALLOW š _ 58 | ALLOW t _ 59 | ALLOW ť _ 60 | ALLOW v _ 61 | ALLOW x _ 62 | ALLOW x _ 63 | ALLOW z _ 64 | ALLOW ž _ 65 | ALLOW _ _ 66 | -------------------------------------------------------------------------------- /cvutils/data/cs/vocab.tsv: -------------------------------------------------------------------------------- 1 | nula 2 | jedna 3 | dva 4 | tři 5 | čtyři 6 | pět 7 | šest 8 | sedm 9 | osm 10 | devět 11 | ano 12 | ne 13 | -------------------------------------------------------------------------------- /cvutils/data/ctp/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnñoõpqrstuvwxyzʼᴬᴮꟲᴱꟳᴳᴴᴵᴶᴷᴸᴹ 2 | -------------------------------------------------------------------------------- /cvutils/data/ctp/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL , _ 002c _ 5 | REPL - _ 002d _ 6 | REPL . _ 002e _ 7 | REPL : _ 003a _ 8 | REPL ? _ 003f _ 9 | DEL 0 _ _ _ 10 | DEL + _ 002b _ 11 | SKIP ( _ 0028 _ 12 | SKIP ) _ 0029 _ 13 | SKIP < _ 003c _ 14 | SKIP > _ 003e _ 15 | SKIP [ _ 005b _ 16 | SKIP ] _ 005d _ 17 | SKIP ˜ _ 02dc _ 18 | SKIP 6 _ 0036 _ 19 | SKIP 8 _ 0038 _ 20 | SKIP 9 _ 0039 _ 21 | NORM ' ʼ 0027 02bc 22 | NORM 2432 ᴶ _ 1d36 23 | NORM 140 ᴮ _ 1d2e 24 | NORM 24 ᴶ _ 1d36 25 | NORM 42 ᴳ _ 1d33 26 | NORM 20 ᴴ _ 1d34 27 | NORM 32 ᴵ _ 1d35 28 | NORM 14 ᴮ _ 1d2e 29 | NORM 10 ᴷ _ 1d37 30 | NORM 04 ᴹ _ 1d39 31 | NORM 40 ᴸ _ 1d38 32 | NORM 1 ᴱ _ 1d31 33 | NORM 2 ꟲ _ a7f2 34 | NORM 3 ꟳ _ a7f3 35 | NORM 4 ᴬ _ 1d2c 36 | ALLOW ʼ _ 02bc _ 37 | ALLOW ᴬ _ 1d2c _ 38 | ALLOW ᴮ _ 1d2e _ 39 | ALLOW ꟲ _ a7f2 _ 40 | ALLOW ᴱ _ 1d31 _ 41 | ALLOW ꟳ _ a7f3 _ 42 | ALLOW ᴳ _ 1d33 _ 43 | ALLOW ᴴ _ 1d34 _ 44 | ALLOW ᴵ _ 1d35 _ 45 | ALLOW ᴶ _ 1d36 _ 46 | ALLOW ᴷ _ 1d37 _ 47 | ALLOW ᴸ _ 1d38 _ 48 | ALLOW ᴹ _ 1d39 _ 49 | ALLOW a _ 0061 _ 50 | ALLOW b _ 0062 _ 51 | ALLOW c _ 0063 _ 52 | ALLOW d _ 0064 _ 53 | ALLOW e _ 0065 _ 54 | ALLOW f _ 0066 _ 55 | ALLOW g _ 0067 _ 56 | ALLOW h _ 0068 _ 57 | ALLOW i _ 0069 _ 58 | ALLOW j _ 006a _ 59 | ALLOW k _ 006b _ 60 | ALLOW l _ 006c _ 61 | ALLOW m _ 006d _ 62 | ALLOW n _ 006e _ 63 | ALLOW ñ _ 00f1 _ 64 | ALLOW o _ 006f _ 65 | ALLOW õ _ 00f5 _ 66 | ALLOW p _ 0070 _ 67 | ALLOW q _ 0071 _ 68 | ALLOW r _ 0072 _ 69 | ALLOW s _ 0073 _ 70 | ALLOW t _ 0074 _ 71 | ALLOW u _ 0075 _ 72 | ALLOW v _ 0076 _ 73 | ALLOW w _ 0077 _ 74 | ALLOW x _ 0078 _ 75 | ALLOW y _ 0079 _ 76 | ALLOW z _ 007a _ 77 | ALLOW _ _ 0020 _ 78 | -------------------------------------------------------------------------------- /cvutils/data/cv/abbr.tsv: -------------------------------------------------------------------------------- 1 | 139 А. 2 | 118 В. 3 | 111 И. 4 | 84 М. 5 | 66 Н. 6 | 54 С. 7 | 51 П. 8 | 40 Г. 9 | 35 К. 10 | 31 Е. 11 | 27 Ф. 12 | 22 Л. 13 | 19 Я. 14 | 19 Д. 15 | 16 Т. 16 | 14 Б. 17 | 13 ҫ.ҫ. 18 | 12 Р. 19 | 10 Ю. 20 | 8 Х. 21 | 7 Э. 22 | 5 О. 23 | 3 З. 24 | 2 Й. 25 | 1 Ш. 26 | 1 Ч. 27 | 1 У. 28 | 1 Ж. 29 | 1 т.ыт.те. 30 | 1 т.ыт. 31 | 1 куб.см. 32 | -------------------------------------------------------------------------------- /cvutils/data/cv/alphabet.txt: -------------------------------------------------------------------------------- 1 | аӑбвгдеӗёжзийклмнопрсҫтуӳфхцчшщыэюяьъ- 2 | -------------------------------------------------------------------------------- /cvutils/data/cv/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | а a 3 | а ɑ 4 | ӑ ə 5 | ӑ ɒ 6 | б b 7 | в ʋ 8 | в w 9 | в v 10 | г ɡ 11 | д d 12 | е ɛ 13 | ё jo 14 | ё ʲo 15 | ӗ ɘ 16 | ӗ ø 17 | ж ʐ 18 | з z 19 | и i 20 | й j 21 | к k 22 | к kʲ 23 | к c 24 | к gʲ 25 | к ɟ 26 | л l 27 | л ɫ 28 | л lʲ 29 | л ʎ 30 | м m 31 | н n 32 | н nʲ 33 | н ɲ 34 | о o 35 | п p 36 | п b 37 | р r 38 | р ɾ 39 | с s 40 | с s̬ 41 | с z 42 | ҫ ɕ 43 | ҫ ɕ̬ 44 | ҫ ʑ 45 | т t 46 | т d 47 | т tʲ 48 | т dʲ 49 | у u 50 | у o 51 | ӳ y 52 | ф f 53 | ф v 54 | х χ 55 | х χʲ 56 | х ɣ 57 | х ɣʲ 58 | ц ts 59 | ц dz 60 | ч tɕ 61 | ч ʥ 62 | ш ʃ 63 | ш ʒ 64 | щ ɕː 65 | ы ɯ 66 | э e 67 | ю ju 68 | ю ʲu 69 | я ja 70 | я ʲa 71 | -------------------------------------------------------------------------------- /cvutils/data/cv/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/cv/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL " _ 4 | REPL ! _ 5 | REPL ( _ 6 | REPL ) _ 7 | REPL + _ 8 | REPL , _ 9 | REPL . _ 10 | REPL : _ 11 | REPL ; _ 12 | REPL ? _ 13 | REPL « _ 14 | REPL » _ 15 | REPL „ _ 16 | REPL ” _ 17 | REPL “ _ 18 | REPL – _ 19 | REPL — _ 20 | REPL … _ 21 | REPL ́ _ 22 | SKIP % _ 23 | SKIP 0 _ 24 | SKIP 1 _ 25 | SKIP 2 _ 26 | SKIP 3 _ 27 | SKIP 4 _ 28 | SKIP 5 _ 29 | SKIP 6 _ 30 | SKIP 7 _ 31 | SKIP 8 _ 32 | SKIP 9 _ 33 | NORM a а 34 | NORM ă ӑ 35 | NORM c с 36 | NORM ç ҫ 37 | NORM e е 38 | NORM ĕ ӗ 39 | NORM o о 40 | NORM ӱ ӳ 41 | NORM ÿ ӳ 42 | ALLOW а _ 43 | ALLOW ӑ _ 44 | ALLOW б _ 45 | ALLOW в _ 46 | ALLOW г _ 47 | ALLOW д _ 48 | ALLOW е _ 49 | ALLOW ӗ _ 50 | ALLOW ё _ 51 | ALLOW ж _ 52 | ALLOW з _ 53 | ALLOW и _ 54 | ALLOW й _ 55 | ALLOW к _ 56 | ALLOW л _ 57 | ALLOW м _ 58 | ALLOW н _ 59 | ALLOW о _ 60 | ALLOW п _ 61 | ALLOW р _ 62 | ALLOW с _ 63 | ALLOW ҫ _ 64 | ALLOW т _ 65 | ALLOW у _ 66 | ALLOW ӳ _ 67 | ALLOW ф _ 68 | ALLOW х _ 69 | ALLOW ц _ 70 | ALLOW ч _ 71 | ALLOW ш _ 72 | ALLOW щ _ 73 | ALLOW ъ _ 74 | ALLOW ы _ 75 | ALLOW ь _ 76 | ALLOW э _ 77 | ALLOW ю _ 78 | ALLOW я _ 79 | ALLOW - _ 80 | ALLOW _ _ 81 | -------------------------------------------------------------------------------- /cvutils/data/cv/vocab.tsv: -------------------------------------------------------------------------------- 1 | нуль 2 | пӗрре 3 | иккӗ 4 | виҫҫӗ 5 | тӑваттӑ 6 | пиллӗк 7 | улттӑ 8 | ҫиччӗ 9 | саккӑр 10 | тӑххӑр 11 | ҫапла 12 | ҫук 13 | -------------------------------------------------------------------------------- /cvutils/data/cy/alphabet.txt: -------------------------------------------------------------------------------- 1 | 'abcdefghijlmnoprstuwxyzâêîôûŷŵkqv 2 | -------------------------------------------------------------------------------- /cvutils/data/cy/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | â aː 4 | b b 5 | c k 6 | ch χ 7 | d d 8 | dd ð 9 | e ɛ 10 | ê ɛː 11 | f v 12 | ff f 13 | g ɡ 14 | ng ŋ 15 | h h 16 | i ɪ 17 | î ɪː 18 | j d͡ʒ 19 | l l 20 | ll ɬ 21 | m m 22 | n n 23 | o ɔ 24 | ô ɔː 25 | p p 26 | ph f 27 | r r 28 | rh r̥ 29 | s s 30 | t t 31 | th θ 32 | u i 33 | û iː 34 | w ʊ 35 | ŵ ʊː 36 | y ə 37 | ŷ əː 38 | -------------------------------------------------------------------------------- /cvutils/data/cy/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL . _ 002e _ 5 | REPL ? _ 003f _ 6 | REPL , _ 002c _ 7 | REPL - _ 002d _ 8 | REPL " _ 0022 _ 9 | REPL ” _ 201d _ 10 | REPL “ _ 201c _ 11 | REPL : _ 003a _ 12 | REPL ; _ 003b _ 13 | REPL – _ 2013 _ 14 | REPL — _ 2014 _ 15 | REPL ’ ' 2019 _ 16 | REPL ‘ ' 2018 _ 17 | DEL ¬ _ 00ac _ 18 | NORM á a _ _ 19 | NORM à a _ _ 20 | NORM é e _ _ 21 | NORM ò o _ _ 22 | NORM ï i _ _ 23 | NORM ë e _ _ 24 | NORM ä a _ _ 25 | NORM ö o _ _ 26 | NORM ÿ y _ _ 27 | ALLOW a _ 0061 _ 28 | ALLOW b _ 0062 _ 29 | ALLOW c _ 0063 _ 30 | ALLOW d _ 0064 _ 31 | ALLOW e _ 0065 _ 32 | ALLOW f _ 0066 _ 33 | ALLOW g _ 0067 _ 34 | ALLOW h _ 0068 _ 35 | ALLOW i _ 0069 _ 36 | ALLOW j _ 006a _ 37 | ALLOW l _ 006c _ 38 | ALLOW m _ 006d _ 39 | ALLOW n _ 006e _ 40 | ALLOW o _ 006f _ 41 | ALLOW p _ 0070 _ 42 | ALLOW r _ 0072 _ 43 | ALLOW s _ 0073 _ 44 | ALLOW t _ 0074 _ 45 | ALLOW u _ 0075 _ 46 | ALLOW w _ 0077 _ 47 | ALLOW y _ 0079 _ 48 | ALLOW k _ 006b _ 49 | ALLOW q _ 0071 _ 50 | ALLOW v _ 0076 _ 51 | ALLOW x _ 0078 _ 52 | ALLOW z _ 007a _ 53 | ALLOW â _ 00e2 _ 54 | ALLOW ê _ 00ea _ 55 | ALLOW î _ 00ee _ 56 | ALLOW ô _ 00f4 _ 57 | ALLOW û _ 00fb _ 58 | ALLOW ŷ _ 0177 _ 59 | ALLOW ŵ _ 0175 _ 60 | ALLOW ' _ 0027 _ 61 | ALLOW _ _ 0020 _ 62 | -------------------------------------------------------------------------------- /cvutils/data/cy/vocab.tsv: -------------------------------------------------------------------------------- 1 | sero 2 | un 3 | dau 4 | tri 5 | pedwar 6 | pump 7 | chwech 8 | saith 9 | wyth 10 | naw 11 | -------------------------------------------------------------------------------- /cvutils/data/da/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnopqrstuvwxyzæøå 2 | -------------------------------------------------------------------------------- /cvutils/data/da/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL , _ 002c _ 6 | REPL - _ 002d _ 7 | REPL . _ 002e _ 8 | REPL : _ 003a _ 9 | REPL ; _ 003b _ 10 | REPL ? _ 003f _ 11 | REPL « _ 00ab _ 12 | REPL » _ 00bb _ 13 | REPL — _ 2014 _ 14 | REPL ' _ 0027 _ 15 | REPL – _ 2013 _ 16 | NORM é e 00e9 _ 17 | NORM í i 00ed _ 18 | NORM ó o 00f3 _ 19 | ALLOW a _ 0061 _ 20 | ALLOW å _ 00e5 _ 21 | ALLOW æ _ 00e6 _ 22 | ALLOW b _ 0062 _ 23 | ALLOW c _ 0063 _ 24 | ALLOW d _ 0064 _ 25 | ALLOW e _ 0065 _ 26 | ALLOW f _ 0066 _ 27 | ALLOW g _ 0067 _ 28 | ALLOW h _ 0068 _ 29 | ALLOW i _ 0069 _ 30 | ALLOW j _ 006a _ 31 | ALLOW k _ 006b _ 32 | ALLOW l _ 006c _ 33 | ALLOW m _ 006d _ 34 | ALLOW n _ 006e _ 35 | ALLOW o _ 006f _ 36 | ALLOW ø _ 00f8 _ 37 | ALLOW p _ 0070 _ 38 | ALLOW r _ 0072 _ 39 | ALLOW s _ 0073 _ 40 | ALLOW t _ 0074 _ 41 | ALLOW u _ 0075 _ 42 | ALLOW v _ 0076 _ 43 | ALLOW w _ 0077 _ 44 | ALLOW x _ 0078 _ 45 | ALLOW y _ 0079 _ 46 | ALLOW z _ 007a _ 47 | ALLOW _ _ 0020 _ 48 | -------------------------------------------------------------------------------- /cvutils/data/da/vocab.tsv: -------------------------------------------------------------------------------- 1 | nul 2 | en 3 | to 4 | tre 5 | fire 6 | fem 7 | seks 8 | syv 9 | otte 10 | ni 11 | ja 12 | nej 13 | -------------------------------------------------------------------------------- /cvutils/data/de/abbr.tsv: -------------------------------------------------------------------------------- 1 | 1 hl. 2 | 1 no. 3 | 1 Mr. 4 | 1 Ms. 5 | 1 Mrs. 6 | 1 geb. 7 | 1 bspw. 8 | 1 Dr. 9 | 1 Fürstl. 10 | 1 mind. 11 | 1 Gen. 12 | 1 Hll. 13 | 1 evtl. 14 | 1 Ebf. 15 | 1 kath. 16 | 1 Frhr. 17 | 1 St. 18 | 1 Geh. 19 | 1 Vol. 20 | 1 Jr. 21 | 1 Bf. 22 | 1 subsp. 23 | 1 bzgl. 24 | 1 franz. 25 | 1 ahd. 26 | 1 Ch. 27 | -------------------------------------------------------------------------------- /cvutils/data/de/alphabet.txt: -------------------------------------------------------------------------------- 1 | aäbcdefghijklmnoöpqrsßtuüvwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/de/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/de/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | REPL ! _ 3 | REPL " _ 4 | REPL ' _ 5 | REPL ( _ 6 | REPL ) _ 7 | REPL + _ 8 | REPL , _ 9 | REPL - _ 10 | REPL . _ 11 | REPL / _ 12 | REPL : _ 13 | REPL ; _ 14 | REPL = _ 15 | REPL ? _ 16 | REPL @ _ 17 | REPL [ _ 18 | REPL ] _ 19 | REPL ~ _ 20 | REPL § _ 21 | REPL « _ 22 | REPL ° _ 23 | REPL » _ 24 | REPL ‘ _ 25 | REPL ’ _ 26 | REPL ‚ _ 27 | REPL “ _ 28 | REPL ” _ 29 | REPL „ _ 30 | REPL ‟ _ 31 | REPL … _ 32 | REPL – _ 33 | NORM ue ü 34 | NORM oe ö 35 | NORM ae ä 36 | NORM ñ n 37 | NORM á a 38 | NORM é e 39 | NORM í i 40 | NORM ó o 41 | NORM ú u 42 | ALLOW a _ 43 | ALLOW ä _ 44 | ALLOW b _ 45 | ALLOW c _ 46 | ALLOW d _ 47 | ALLOW e _ 48 | ALLOW f _ 49 | ALLOW g _ 50 | ALLOW h _ 51 | ALLOW i _ 52 | ALLOW j _ 53 | ALLOW k _ 54 | ALLOW l _ 55 | ALLOW m _ 56 | ALLOW n _ 57 | ALLOW o _ 58 | ALLOW ö _ 59 | ALLOW p _ 60 | ALLOW q _ 61 | ALLOW r _ 62 | ALLOW s _ 63 | ALLOW ß _ 64 | ALLOW t _ 65 | ALLOW u _ 66 | ALLOW ü _ 67 | ALLOW v _ 68 | ALLOW w _ 69 | ALLOW x _ 70 | ALLOW y _ 71 | ALLOW z _ 72 | ALLOW _ _ 73 | -------------------------------------------------------------------------------- /cvutils/data/de/vocab.tsv: -------------------------------------------------------------------------------- 1 | null 2 | eins 3 | zwei 4 | drei 5 | vier 6 | fünf 7 | sechs 8 | sieben 9 | acht 10 | neun 11 | ja 12 | nein 13 | -------------------------------------------------------------------------------- /cvutils/data/dv/abbr.tsv: -------------------------------------------------------------------------------- 1 | 5 ޓީ. 2 | 5 ޓީ.ވީ. 3 | 5 ޓީ.ވީ. 4 | -------------------------------------------------------------------------------- /cvutils/data/dv/alphabet.txt: -------------------------------------------------------------------------------- 1 | ހ 2 | ށ 3 | ނ 4 | ރ 5 | ބ 6 | ޅ 7 | ކ 8 | އ 9 | ވ 10 | މ 11 | ފ 12 | ދ 13 | ތ 14 | ލ 15 | ގ 16 | ޏ 17 | ސ 18 | ޑ 19 | ޒ 20 | ޓ 21 | ޔ 22 | ޕ 23 | ޖ 24 | ޗ 25 | ޘ 26 | ޙ 27 | ޚ 28 | ޛ 29 | ޜ 30 | ޝ 31 | ޞ 32 | ޟ 33 | ޠ 34 | ޡ 35 | ޢ 36 | ޣ 37 | ޤ 38 | ޥ 39 | ޱ 40 | ަ 41 | ާ 42 | ި 43 | ީ 44 | ު 45 | ޫ 46 | ެ 47 | ޭ 48 | ޮ 49 | ޯ 50 | ް -------------------------------------------------------------------------------- /cvutils/data/dv/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | ހ h 3 | ށ ʂ 4 | ށް ʔ 5 | ނ n̪ 6 | ނބ ᵐb 7 | ނދ ⁿd̪ 8 | ނގ ᵑg 9 | ނޑ ᶯɖ 10 | ން ɳ 11 | ރ ɾ 12 | ބ b 13 | ޅ ɭ 14 | ކ k 15 | އ _ 16 | އޮ o 17 | އޯ oː 18 | އް ʔ 19 | ވ ʋ 20 | މ m 21 | ފ f 22 | ދ d̪ 23 | ތ t̪ 24 | ތް jt̪ 25 | ލ l 26 | ގ ɡ 27 | ޏ ɲ 28 | ސ s̺ 29 | ޑ ɖ 30 | ޒ z̺ 31 | ޓ ʈ 32 | ޔ j 33 | ޕ p 34 | ޖ d͡ʒ 35 | ޗ t͡ʃ 36 | ޘ θ 37 | ޙ ħ 38 | ޚ x 39 | ޛ ð 40 | ޜ ʒ 41 | ޝ ʃ 42 | ޞ sˤ 43 | ޟ dˤ 44 | ޠ tˤ 45 | ޡ ðˤ 46 | ޢ ʕ 47 | ޣ ɣ 48 | ޤ q 49 | ޥ w 50 | ޱ ɳ 51 | ަ a 52 | ާ aː 53 | ި i 54 | ީ iː 55 | ު u 56 | ޫ uː 57 | ެ e 58 | ޭ eː 59 | ޮ ɔ 60 | ޯ ɔː 61 | ް _ 62 | -------------------------------------------------------------------------------- /cvutils/data/dv/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/dv/validate.tsv: -------------------------------------------------------------------------------- 1 | NFKC _ _ _ _ 2 | REPL ! _ 0021 _ 3 | REPL - _ 002d _ 4 | REPL . _ 002e _ 5 | REPL : _ 003a _ 6 | REPL " _ 003a _ 7 | REPL ; _ 003b _ 8 | REPL ، _ 060c _ 9 | REPL ؟ _ 061f _ 10 | REPL ّ _ 0651 _ 11 | SKIP – _ 2013 _ 12 | SKIP ‘ _ 2018 _ 13 | SKIP ’ _ 2019 _ 14 | SKIP d _ 0064 _ 15 | SKIP v _ 0076 _ 16 | SKIP ل _ 0644 _ 17 | SKIP ه _ 0647 _ 18 | ALLOW ހ _ 0780 _ 19 | ALLOW ޙ _ 0799 _ 20 | ALLOW ޚ _ 079a _ 21 | ALLOW ށ _ 0781 _ 22 | ALLOW ނ _ 0782 _ 23 | ALLOW ރ _ 0783 _ 24 | ALLOW ބ _ 0784 _ 25 | ALLOW ޅ _ 0785 _ 26 | ALLOW ކ _ 0786 _ 27 | ALLOW އ _ 0787 _ 28 | ALLOW ޢ _ 07a2 _ 29 | ALLOW ޣ _ 07a3 _ 30 | ALLOW ވ _ 0788 _ 31 | ALLOW ޥ _ 07a5 _ 32 | ALLOW މ _ 0789 _ 33 | ALLOW ފ _ 078a _ 34 | ALLOW ދ _ 078b _ 35 | ALLOW ޛ _ 079b _ 36 | ALLOW ތ _ 078c _ 37 | ALLOW ޘ _ 0798 _ 38 | ALLOW ޠ _ 07a0 _ 39 | ALLOW ޡ _ 07a1 _ 40 | ALLOW ލ _ 078d _ 41 | ALLOW ގ _ 078e _ 42 | ALLOW ޤ _ 07a4 _ 43 | ALLOW ޏ _ 078f _ 44 | ALLOW ސ _ 0790 _ 45 | ALLOW ޝ _ 079d _ 46 | ALLOW ޞ _ 079e _ 47 | ALLOW ޟ _ 079f _ 48 | ALLOW ޑ _ 0791 _ 49 | ALLOW ޒ _ 0792 _ 50 | ALLOW ޓ _ 0793 _ 51 | ALLOW ޔ _ 0794 _ 52 | ALLOW ޕ _ 0795 _ 53 | ALLOW ޖ _ 0796 _ 54 | ALLOW ޗ _ 0797 _ 55 | ALLOW ަ _ 07a6 _ 56 | ALLOW ާ _ 07a7 _ 57 | ALLOW ި _ 07a8 _ 58 | ALLOW ީ _ 07a9 _ 59 | ALLOW ު _ 07aa _ 60 | ALLOW ޫ _ 07ab _ 61 | ALLOW ެ _ 07ac _ 62 | ALLOW ޭ _ 07ad _ 63 | ALLOW ޮ _ 07ae _ 64 | ALLOW ޯ _ 07af _ 65 | ALLOW ް _ 07b0 _ 66 | -------------------------------------------------------------------------------- /cvutils/data/dyu/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdeɛfghijklmnɲŋoɔprstuvwyz 2 | -------------------------------------------------------------------------------- /cvutils/data/ee/alphabet.txt: -------------------------------------------------------------------------------- 1 | abdɖeɛfƒgɣhiklmnŋoɔprstuvʋwxyz̃ 2 | -------------------------------------------------------------------------------- /cvutils/data/ee/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | b b 4 | d d 5 | ɖ ɖ 6 | dz d͡z 7 | e e 8 | e ə 9 | ɛ ɛ 10 | f f 11 | ƒ ɸ 12 | g ɡ 13 | gb ɡ͡b 14 | ɣ ɣ 15 | h h 16 | i i 17 | k k 18 | kp k͡p 19 | l l 20 | m m 21 | n n 22 | ny ɲ 23 | ŋ ŋ 24 | o o 25 | ɔ ɔ 26 | p p 27 | r r 28 | s s 29 | t t 30 | ts t͡s 31 | u u 32 | v v 33 | ʋ β 34 | w w 35 | x x 36 | y j 37 | z z 38 | ̃ ̃ 39 | -------------------------------------------------------------------------------- /cvutils/data/ee/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | DEL   _ 2006 _ 4 | REPL , _ 002c _ 5 | REPL - _ 002d _ 6 | REPL . _ 002e _ 7 | REPL : _ 003a _ 8 | REPL ; _ 003b _ 9 | REPL ? _ 003f _ 10 | REPL ! _ 0021 _ 11 | REPL — _ 2014 _ 12 | REPL ‘ _ 2018 _ 13 | REPL ’ _ 2019 _ 14 | REPL “ _ 201c _ 15 | REPL ” _ 201d _ 16 | REPL … _ 2026 _ 17 | SKIP 0 _ 0030 _ 18 | SKIP 1 _ 0031 _ 19 | SKIP 2 _ 0032 _ 20 | SKIP 3 _ 0033 _ 21 | SKIP 4 _ 0034 _ 22 | SKIP 5 _ 0035 _ 23 | SKIP 6 _ 0036 _ 24 | SKIP 7 _ 0037 _ 25 | SKIP 8 _ 0038 _ 26 | SKIP 9 _ 0039 _ 27 | REPL á a_ 00e1 _ 28 | REPL à a 00e0 _ 29 | REPL ã a 00e3 _ 30 | REPL c c 0063 _ 31 | REPL é e 00e9 _ 32 | REPL è e 00e8 _ 33 | REPL ẽ e 1ebd _ 34 | REPL í i 00ed _ 35 | REPL ì i 00ec _ 36 | REPL ĩ i 0129 _ 37 | REPL ó o 00f3 _ 38 | REPL ò o 00f2 _ 39 | REPL õ o 00f5 _ 40 | REPL ũ u 0169 _ 41 | SKIP ( _ 0028 _ 42 | SKIP ) _ 0029 _ 43 | SKIP * _ 002a _ 44 | SKIP [ _ 005b _ 45 | SKIP \ _ 005c _ 46 | SKIP ] _ 005d _ 47 | ALLOW a _ 0061 _ 48 | ALLOW b _ 0062 _ 49 | ALLOW d _ 0064 _ 50 | ALLOW ɖ _ 0256 _ 51 | ALLOW e _ 0065 _ 52 | ALLOW ɛ _ 025b _ 53 | ALLOW f _ 0066 _ 54 | ALLOW ƒ _ 0192 _ 55 | ALLOW g _ 0067 _ 56 | ALLOW ɣ _ 0263 _ 57 | ALLOW h _ 0068 _ 58 | ALLOW i _ 0069 _ 59 | ALLOW k _ 006b _ 60 | ALLOW l _ 006c _ 61 | ALLOW m _ 006d _ 62 | ALLOW n _ 006e _ 63 | ALLOW ŋ _ 014b _ 64 | ALLOW o _ 006f _ 65 | ALLOW ɔ _ 0254 _ 66 | ALLOW p _ 0070 _ 67 | ALLOW r _ 0072 _ 68 | ALLOW s _ 0073 _ 69 | ALLOW t _ 0074 _ 70 | ALLOW u _ 0075 _ 71 | ALLOW v _ 0076 _ 72 | ALLOW ʋ _ 028b _ 73 | ALLOW w _ 0077 _ 74 | ALLOW x _ 0078 _ 75 | ALLOW y _ 0079 _ 76 | ALLOW z _ 007a _ 77 | ALLOW ̃ _ 0303 _ 78 | ALLOW _ _ 0020 _ 79 | -------------------------------------------------------------------------------- /cvutils/data/el/alphabet.txt: -------------------------------------------------------------------------------- 1 | αάβγδεέζηήθιίϊΐκλμνξοόπρσςτυύϋφχψωώ 2 | -------------------------------------------------------------------------------- /cvutils/data/el/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | α a 3 | β b 4 | γ ɡ 5 | δ d 6 | ε e 7 | ζ zd 8 | η ɛː 9 | θ tʰ 10 | ι i 11 | κ k 12 | λ l 13 | μ m 14 | ν n 15 | ξ ks 16 | ο o 17 | π p 18 | ρ r 19 | σ s 20 | ς s 21 | τ t 22 | υ y 23 | φ pʰ 24 | χ kʰ 25 | ψ ps 26 | ω ɔː 27 | -------------------------------------------------------------------------------- /cvutils/data/el/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ _ _ 4 | REPL " _ _ _ 5 | REPL ' _ _ _ 6 | REPL , _ _ _ 7 | REPL - _ _ _ 8 | REPL . _ _ _ 9 | REPL : _ _ _ 10 | REPL ; _ _ _ 11 | REPL « _ _ _ 12 | REPL \ _ _ _ 13 | REPL ´ _ _ _ 14 | REPL · _ _ _ 15 | REPL » _ _ _ 16 | REPL ‘ _ _ _ 17 | REPL ’ _ _ _ 18 | REPL ́ _ _ _ 19 | SKIP g _ _ _ 20 | ALLOW α _ 03b1 _ 21 | ALLOW ά _ 03ac _ 22 | ALLOW β _ 03b2 _ 23 | ALLOW γ _ 03b3 _ 24 | ALLOW δ _ 03b4 _ 25 | ALLOW ε _ 03b5 _ 26 | ALLOW έ _ 03ad _ 27 | ALLOW ζ _ 03b6 _ 28 | ALLOW η _ 03b7 _ 29 | ALLOW ή _ 03ae _ 30 | ALLOW θ _ 03b8 _ 31 | ALLOW ι _ 03b9 _ 32 | ALLOW ί _ 03af _ 33 | ALLOW ϊ _ 03ca _ 34 | ALLOW ΐ _ 0390 _ 35 | ALLOW κ _ 03ba _ 36 | ALLOW λ _ 03bb _ 37 | ALLOW μ _ 03bc _ 38 | ALLOW ν _ 03bd _ 39 | ALLOW ξ _ 03be _ 40 | ALLOW ο _ 03bf _ 41 | ALLOW ό _ 03cc _ 42 | ALLOW π _ 03c0 _ 43 | ALLOW ρ _ 03c1 _ 44 | ALLOW σ _ 03c3 _ 45 | ALLOW ς _ 03c2 _ 46 | ALLOW τ _ 03c4 _ 47 | ALLOW υ _ 03c5 _ 48 | ALLOW ύ _ 03cd _ 49 | ALLOW ϋ _ 03cb _ 50 | ALLOW φ _ 03c6 _ 51 | ALLOW χ _ 03c7 _ 52 | ALLOW ψ _ 03c8 _ 53 | ALLOW ω _ 03c9 _ 54 | ALLOW ώ _ 03ce _ 55 | ALLOW _ _ _ _ 56 | -------------------------------------------------------------------------------- /cvutils/data/en/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnopqrstuvwxyz' 2 | -------------------------------------------------------------------------------- /cvutils/data/en/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/en/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL . _ 002e _ 4 | REPL , _ 002c _ 5 | REPL ? _ 003f _ 6 | REPL ; _ 003b _ 7 | REPL ! _ 0021 _ 8 | REPL : _ 003a _ 9 | REPL " _ 0022 _ 10 | REPL - _ 002d _ 11 | SKIP [ _ 005b _ 12 | SKIP ] _ 005d _ 13 | SKIP { _ 007b _ 14 | SKIP } _ 007d _ 15 | SKIP & _ 0026 _ 16 | SKIP < _ 003c _ 17 | SKIP * _ 002a _ 18 | SKIP 0 _ 0030 _ 19 | SKIP 1 _ 0031 _ 20 | SKIP 2 _ 0032 _ 21 | SKIP 3 _ 0033 _ 22 | SKIP 4 _ 0034 _ 23 | SKIP 5 _ 0035 _ 24 | SKIP 6 _ 0036 _ 25 | SKIP 7 _ 0037 _ 26 | SKIP 8 _ 0038 _ 27 | SKIP 9 _ 0039 _ 28 | NORM ç c 00e7 0063 29 | NORM ï i 00ef 0069 30 | NORM é e 00e9 0065 31 | ALLOW a _ 0061 _ 32 | ALLOW b _ 0062 _ 33 | ALLOW c _ 0063 _ 34 | ALLOW d _ 0064 _ 35 | ALLOW e _ 0065 _ 36 | ALLOW f _ 0066 _ 37 | ALLOW g _ 0067 _ 38 | ALLOW h _ 0068 _ 39 | ALLOW i _ 0069 _ 40 | ALLOW j _ 006a _ 41 | ALLOW k _ 006b _ 42 | ALLOW l _ 006c _ 43 | ALLOW m _ 006d _ 44 | ALLOW n _ 006e _ 45 | ALLOW o _ 006f _ 46 | ALLOW p _ 0070 _ 47 | ALLOW q _ 0071 _ 48 | ALLOW r _ 0072 _ 49 | ALLOW s _ 0073 _ 50 | ALLOW t _ 0074 _ 51 | ALLOW u _ 0075 _ 52 | ALLOW v _ 0076 _ 53 | ALLOW w _ 0077 _ 54 | ALLOW x _ 0078 _ 55 | ALLOW y _ 0079 _ 56 | ALLOW z _ 007a _ 57 | ALLOW ' _ 0027 _ 58 | ALLOW _ _ 0020 _ 59 | -------------------------------------------------------------------------------- /cvutils/data/en/vocab.tsv: -------------------------------------------------------------------------------- 1 | zero 2 | one 3 | two 4 | three 5 | four 6 | five 7 | six 8 | seven 9 | eight 10 | nine 11 | yes 12 | no 13 | -------------------------------------------------------------------------------- /cvutils/data/eo/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcĉdefgĝhĥijĵklmnoprsŝtuŭvz 2 | -------------------------------------------------------------------------------- /cvutils/data/eo/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL ' _ 0027 _ 6 | REPL ( _ 0028 _ 7 | REPL ) _ 0029 _ 8 | REPL , _ 002c _ 9 | REPL - _ 002d _ 10 | REPL . _ 002e _ 11 | REPL : _ 003a _ 12 | REPL ; _ 003b _ 13 | REPL = _ 003d _ 14 | REPL ? _ 003f _ 15 | REPL ` _ 0060 _ 16 | REPL ¨ _ 00a8 _ 17 | REPL « _ 00ab _ 18 | REPL ¸ _ 00b8 _ 19 | REPL » _ 00bb _ 20 | REPL ‑ _ 2011 _ 21 | REPL – _ 2013 _ 22 | REPL — _ 2014 _ 23 | REPL ‘ _ 2018 _ 24 | REPL ’ _ 2019 _ 25 | REPL “ _ 201c _ 26 | REPL ” _ 201d _ 27 | REPL „ _ 201e _ 28 | REPL … _ 2026 _ 29 | REPL ‹ _ 2039 _ 30 | REPL › _ 203a _ 31 | REPL ？ _ ff1f _ 32 | ALLOW a _ 0061 _ 33 | ALLOW b _ 0062 _ 34 | ALLOW c _ 0063 _ 35 | ALLOW ĉ _ 0109 _ 36 | ALLOW d _ 0064 _ 37 | ALLOW e _ 0065 _ 38 | ALLOW f _ 0066 _ 39 | ALLOW g _ 0067 _ 40 | ALLOW ĝ _ 011d _ 41 | ALLOW h _ 0068 _ 42 | ALLOW ĥ _ 0125 _ 43 | ALLOW i _ 0069 _ 44 | ALLOW j _ 006a _ 45 | ALLOW ĵ _ 0135 _ 46 | ALLOW k _ 006b _ 47 | ALLOW l _ 006c _ 48 | ALLOW m _ 006d _ 49 | ALLOW n _ 006e _ 50 | ALLOW o _ 006f _ 51 | ALLOW p _ 0070 _ 52 | ALLOW r _ 0072 _ 53 | ALLOW s _ 0073 _ 54 | ALLOW ŝ _ 015d _ 55 | ALLOW t _ 0074 _ 56 | ALLOW u _ 0075 _ 57 | ALLOW ŭ _ 016d _ 58 | ALLOW v _ 0076 _ 59 | ALLOW y _ 0079 _ 60 | ALLOW z _ 007a _ 61 | ALLOW _ _ 0020 _ 62 | -------------------------------------------------------------------------------- /cvutils/data/eo/vocab.tsv: -------------------------------------------------------------------------------- 1 | nul 2 | unu 3 | du 4 | tri 5 | kvar 6 | kvin 7 | ses 8 | sep 9 | ok 10 | naŭ 11 | jes 12 | ne 13 | -------------------------------------------------------------------------------- /cvutils/data/es/alphabet.txt: -------------------------------------------------------------------------------- 1 | aábcdeéfghiíjklmnñoópqrstuúüvwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/es/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | á a 4 | b b 5 | c k 6 | ce θe 7 | ci θi 8 | ch tʃ 9 | d d 10 | e e 11 | é e 12 | f f 13 | g ɡ 14 | gua ɡwa 15 | gue ɡe 16 | gui ɡi 17 | gü ɡw 18 | h _ 19 | hu w 20 | hi j 21 | i i 22 | í i 23 | j x 24 | k k 25 | l l 26 | ll ʎ 27 | m m 28 | n n 29 | ñ ɲ 30 | o o 31 | ó o 32 | p p 33 | q k 34 | qu k 35 | r ɾ 36 | rr r 37 | s s 38 | t t 39 | tl tl 40 | tz ts 41 | tx tʃ 42 | u u 43 | ú u 44 | ü u 45 | v b 46 | w w 47 | x ks 48 | x s 49 | y ʝ 50 | z θ 51 | -------------------------------------------------------------------------------- /cvutils/data/es/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/es/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL " _ 4 | REPL “ " 5 | REPL ” " 6 | REPL « " 7 | REPL » " 8 | REPL ‟ " 9 | REPL ‘ ' 10 | REPL ’ ' 11 | REPL ´ ' 12 | REPL ` ' 13 | REPL ʼ ' 14 | REPL ' _ 15 | REPL , _ 16 | REPL - _ 17 | REPL — _ 18 | REPL – _ 19 | REPL . _ 20 | REPL : _ 21 | REPL ; _ 22 | REPL ? _ 23 | REPL ¿ _ 24 | REPL ! _ 25 | REPL ¡ _ 26 | NORM â a 27 | NORM à a 28 | NORM ê e 29 | NORM è e 30 | NORM î i 31 | NORM ì i 32 | NORM ï i 33 | NORM ô o 34 | NORM ò o 35 | NORM û u 36 | NORM ù u 37 | NORM ç c 38 | SKIP 0 _ 39 | SKIP 1 _ 40 | SKIP 2 _ 41 | SKIP 3 _ 42 | SKIP 4 _ 43 | SKIP 5 _ 44 | SKIP 6 _ 45 | SKIP 7 _ 46 | SKIP 8 _ 47 | SKIP 9 _ 48 | ALLOW a _ 49 | ALLOW á _ 50 | ALLOW b _ 51 | ALLOW c _ 52 | ALLOW d _ 53 | ALLOW e _ 54 | ALLOW é _ 55 | ALLOW f _ 56 | ALLOW g _ 57 | ALLOW h _ 58 | ALLOW i _ 59 | ALLOW í _ 60 | ALLOW j _ 61 | ALLOW k _ 62 | ALLOW l _ 63 | ALLOW m _ 64 | ALLOW n _ 65 | ALLOW ñ _ 66 | ALLOW o _ 67 | ALLOW ó _ 68 | ALLOW p _ 69 | ALLOW q _ 70 | ALLOW r _ 71 | ALLOW s _ 72 | ALLOW t _ 73 | ALLOW u _ 74 | ALLOW ú _ 75 | ALLOW ü _ 76 | ALLOW v _ 77 | ALLOW w _ 78 | ALLOW x _ 79 | ALLOW y _ 80 | ALLOW z _ 81 | ALLOW _ _ 82 | -------------------------------------------------------------------------------- /cvutils/data/es/vocab.tsv: -------------------------------------------------------------------------------- 1 | cero 2 | uno 3 | dos 4 | tres 5 | cuatro 6 | cinco 7 | seis 8 | siete 9 | ocho 10 | nueve 11 | sí 12 | no 13 | -------------------------------------------------------------------------------- /cvutils/data/et/alphabet.txt: -------------------------------------------------------------------------------- 1 | aäbcdefghii̇jklmnoöõpqrsštuüvwxyzž 2 | -------------------------------------------------------------------------------- /cvutils/data/et/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a ɑː 3 | b b 4 | c ts 5 | d d 6 | e e 7 | f f 8 | g ɡ 9 | h h 10 | h ɦ 11 | h ç 12 | h x 13 | i i 14 | j j 15 | k k 16 | k kː 17 | l l 18 | m m 19 | n n 20 | o o 21 | p p 22 | p pː 23 | q k 24 | r r 25 | s s 26 | š ʃ 27 | z z 28 | ž ʒ 29 | t t 30 | u u 31 | v ʋ 32 | w ʋ 33 | õ ɤ 34 | ä æ 35 | ö ø 36 | ü y 37 | x ks 38 | y i 39 | aa ɑː 40 | aa ɑːː 41 | ee eː 42 | ee eːː 43 | ii iː 44 | ii iːː 45 | oo oː 46 | oo oːː 47 | uu uː 48 | uu uːː 49 | õõ ɤː 50 | õõ ɤːː 51 | ää æː 52 | ää æːː 53 | öö øː 54 | öö øːː 55 | üü yː 56 | üü yːː 57 | -------------------------------------------------------------------------------- /cvutils/data/et/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL " _ 4 | REPL ! _ 5 | REPL „ _ 6 | REPL “ _ 7 | REPL ’ _ 8 | REPL ' _ 9 | REPL , _ 10 | REPL - _ 11 | REPL . _ 12 | REPL ; _ 13 | REPL ? _ 14 | SKIP 0 _ 15 | SKIP 1 _ 16 | SKIP 2 _ 17 | SKIP 3 _ 18 | SKIP 4 _ 19 | SKIP 5 _ 20 | SKIP 6 _ 21 | SKIP 7 _ 22 | SKIP 8 _ 23 | SKIP 9 _ 24 | NORM i̇ i 25 | ALLOW a _ 26 | ALLOW ä _ 27 | ALLOW b _ 28 | ALLOW c _ 29 | ALLOW d _ 30 | ALLOW e _ 31 | ALLOW f _ 32 | ALLOW g _ 33 | ALLOW h _ 34 | ALLOW i _ 35 | ALLOW j _ 36 | ALLOW k _ 37 | ALLOW l _ 38 | ALLOW m _ 39 | ALLOW n _ 40 | ALLOW o _ 41 | ALLOW ö _ 42 | ALLOW õ _ 43 | ALLOW p _ 44 | ALLOW q _ 45 | ALLOW r _ 46 | ALLOW s _ 47 | ALLOW š _ 48 | ALLOW t _ 49 | ALLOW u _ 50 | ALLOW ü _ 51 | ALLOW v _ 52 | ALLOW w _ 53 | ALLOW x _ 54 | ALLOW y _ 55 | ALLOW z _ 56 | ALLOW ž _ 57 | ALLOW _ _ 58 | -------------------------------------------------------------------------------- /cvutils/data/et/vocab.tsv: -------------------------------------------------------------------------------- 1 | null 2 | üks 3 | kaks 4 | kolm 5 | neli 6 | viis 7 | kuus 8 | seitse 9 | kaheksa 10 | üheksa 11 | jah 12 | ei 13 | -------------------------------------------------------------------------------- /cvutils/data/eu/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnñopqrstuvwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/eu/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | b b 4 | c k 5 | c s 6 | d d̪ 7 | d ð̞ 8 | e e 9 | f f 10 | g ɡ 11 | h _ 12 | i i 13 | i i̭ 14 | j j 15 | j x 16 | j ʝ 17 | j ɟ 18 | k k 19 | l l 20 | m m 21 | n n 22 | ñ ɲ 23 | o o 24 | p p 25 | q k 26 | r ɾ 27 | rr r 28 | s s̺ 29 | t t̪ 30 | u u 31 | u u̯ 32 | v b 33 | v β̞ 34 | w u̯ 35 | x ʃ 36 | y i 37 | y i̭ 38 | y j 39 | z s̻ 40 | dd ɟ 41 | ll ʎ 42 | rr r 43 | ts t͡s̺ 44 | tt c 45 | tx t͡ʃ 46 | tz t͡s̻ 47 | -------------------------------------------------------------------------------- /cvutils/data/eu/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/eu/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | REPL ! _ 3 | REPL " _ 4 | REPL , _ 5 | REPL - _ 6 | REPL . _ 7 | REPL : _ 8 | REPL ; _ 9 | REPL ? _ 10 | REPL ‘ _ 11 | REPL ’ _ 12 | NORM ´ ' 13 | NORM ` ʼ 14 | NORM ' ʼ 15 | NORM ‘ ' 16 | NORM ‘ ʼ 17 | NORM ’ ' 18 | NORM ’ ʼ 19 | NORM “ " 20 | NORM ” " 21 | NORM « " 22 | NORM » " 23 | NORM ‐ - 24 | NORM ‟ " 25 | NORM ʼ ' 26 | NORM á a 27 | NORM â a 28 | NORM à a 29 | NORM é e 30 | NORM ê e 31 | NORM è e 32 | NORM í i 33 | NORM î i 34 | NORM ì i 35 | NORM ï i 36 | NORM ó o 37 | NORM ô o 38 | NORM ò o 39 | NORM ú u 40 | NORM û u 41 | NORM ù u 42 | NORM ü u 43 | NORM ç c 44 | ALLOW a _ 45 | ALLOW b _ 46 | ALLOW c _ 47 | ALLOW d _ 48 | ALLOW e _ 49 | ALLOW f _ 50 | ALLOW g _ 51 | ALLOW h _ 52 | ALLOW i _ 53 | ALLOW j _ 54 | ALLOW k _ 55 | ALLOW l _ 56 | ALLOW m _ 57 | ALLOW n _ 58 | ALLOW ñ _ 59 | ALLOW o _ 60 | ALLOW p _ 61 | ALLOW q _ 62 | ALLOW r _ 63 | ALLOW s _ 64 | ALLOW t _ 65 | ALLOW u _ 66 | ALLOW v _ 67 | ALLOW w _ 68 | ALLOW x _ 69 | ALLOW y _ 70 | ALLOW z _ 71 | ALLOW _ _ 72 | -------------------------------------------------------------------------------- /cvutils/data/eu/vocab.tsv: -------------------------------------------------------------------------------- 1 | zero 2 | bat 3 | bi 4 | hiru 5 | lau 6 | bost 7 | sei 8 | zazpi 9 | zortzi 10 | bederatzi 11 | bai 12 | ez 13 | -------------------------------------------------------------------------------- /cvutils/data/fa/alphabet.txt: -------------------------------------------------------------------------------- 1 | آٓيءأئؤابپتثجچحخدذرزژسشصضطظعغفقکگلمنوهیآ 2 | -------------------------------------------------------------------------------- /cvutils/data/fi/alphabet.txt: -------------------------------------------------------------------------------- 1 | aäbcdefghijklmnoöpqrstuvxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/fi/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | i i 3 | y y 4 | u u 5 | e e 6 | ö ø 7 | o o 8 | ä æ 9 | a ɑ 10 | ii iː 11 | yy yː 12 | uu uː 13 | ee eː 14 | öö øː 15 | oo oː 16 | ää æː 17 | aa ɑː 18 | m m 19 | n n 20 | ng ŋ 21 | p p 22 | b b 23 | t t 24 | d d 25 | k k 26 | g ɡ 27 | f f 28 | s s 29 | š ʃ 30 | h h 31 | v ʋ 32 | l l 33 | j j 34 | r r 35 | -------------------------------------------------------------------------------- /cvutils/data/fi/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | REPL " _ 3 | REPL , _ 4 | REPL - _ 5 | REPL . _ 6 | REPL : _ 7 | REPL ; _ 8 | REPL ? _ 9 | REPL ! _ 10 | REPL – _ 11 | REPL “ _ 12 | REPL ” _ 13 | REPL … _ 14 | DEL ' _ 15 | ALLOW a _ 16 | ALLOW ä _ 17 | ALLOW b _ 18 | ALLOW c _ 19 | ALLOW d _ 20 | ALLOW e _ 21 | ALLOW f _ 22 | ALLOW g _ 23 | ALLOW h _ 24 | ALLOW i _ 25 | ALLOW j _ 26 | ALLOW k _ 27 | ALLOW l _ 28 | ALLOW m _ 29 | ALLOW n _ 30 | ALLOW o _ 31 | ALLOW ö _ 32 | ALLOW p _ 33 | ALLOW q _ 34 | ALLOW r _ 35 | ALLOW s _ 36 | ALLOW t _ 37 | ALLOW u _ 38 | ALLOW v _ 39 | ALLOW x _ 40 | ALLOW y _ 41 | ALLOW z _ 42 | ALLOW _ _ 43 | -------------------------------------------------------------------------------- /cvutils/data/fr/alphabet.txt: -------------------------------------------------------------------------------- 1 | aàâæbcçdeéèêëfghiîïjklmnoôœpqrstuùûüvwxyÿz 2 | -------------------------------------------------------------------------------- /cvutils/data/fr/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL % _ 0025 _ 6 | REPL ( _ 0028 _ 7 | REPL ) _ 0029 _ 8 | REPL * _ 002a _ 9 | REPL + _ 002b _ 10 | REPL , _ 002c _ 11 | REPL . _ 002e _ 12 | REPL / _ 002f _ 13 | REPL : _ 003a _ 14 | REPL ; _ 003b _ 15 | REPL = _ 003d _ 16 | REPL > _ 003e _ 17 | REPL ? _ 003f _ 18 | REPL « _ 00ab _ 19 | REPL » _ 00bb _ 20 | REPL – _ 2013 _ 21 | REPL — _ 2014 _ 22 | REPL “ _ 201c _ 23 | REPL ” _ 201d _ 24 | REPL … _ 2026 _ 25 | NORM ‘ ' 2018 0027 26 | NORM ’ ' 2019 0027 27 | NORM ʼ ' 02bc 0027 28 | ALLOW a _ 0061 _ 29 | ALLOW à _ 00e0 _ 30 | ALLOW â _ 00e2 _ 31 | ALLOW æ _ 00e6 _ 32 | ALLOW b _ 0062 _ 33 | ALLOW c _ 0063 _ 34 | ALLOW ç _ 00e7 _ 35 | ALLOW d _ 0064 _ 36 | ALLOW e _ 0065 _ 37 | ALLOW é _ 00e9 _ 38 | ALLOW è _ 00e8 _ 39 | ALLOW ê _ 00ea _ 40 | ALLOW ë _ 00eb _ 41 | ALLOW f _ 0066 _ 42 | ALLOW g _ 0067 _ 43 | ALLOW h _ 0068 _ 44 | ALLOW i _ 0069 _ 45 | ALLOW î _ 00ee _ 46 | ALLOW ï _ 00ef _ 47 | ALLOW j _ 006a _ 48 | ALLOW k _ 006b _ 49 | ALLOW l _ 006c _ 50 | ALLOW m _ 006d _ 51 | ALLOW n _ 006e _ 52 | ALLOW o _ 006f _ 53 | ALLOW ô _ 00f4 _ 54 | ALLOW œ _ 0153 _ 55 | ALLOW p _ 0070 _ 56 | ALLOW q _ 0071 _ 57 | ALLOW r _ 0072 _ 58 | ALLOW s _ 0073 _ 59 | ALLOW t _ 0074 _ 60 | ALLOW u _ 0075 _ 61 | ALLOW ù _ 00f9 _ 62 | ALLOW û _ 00fb _ 63 | ALLOW ü _ 00fc _ 64 | ALLOW v _ 0076 _ 65 | ALLOW w _ 0077 _ 66 | ALLOW x _ 0078 _ 67 | ALLOW y _ 0079 _ 68 | ALLOW ÿ _ 00ff _ 69 | ALLOW z _ 007a _ 70 | ALLOW - _ 002d _ 71 | ALLOW ' _ 0027 _ 72 | ALLOW _ _ 005f _ 73 | -------------------------------------------------------------------------------- /cvutils/data/fr/vocab.tsv: -------------------------------------------------------------------------------- 1 | zéro 2 | un 3 | deux 4 | trois 5 | quatre 6 | cinq 7 | six 8 | sept 9 | huit 10 | neuf 11 | oui 12 | non 13 | -------------------------------------------------------------------------------- /cvutils/data/fy-NL/alphabet.txt: -------------------------------------------------------------------------------- 1 | aâeêéiyoôuûúbcdfghjklmnpqrstvwxz' 2 | -------------------------------------------------------------------------------- /cvutils/data/fy-NL/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/fy-NL/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL ! _ 4 | REPL " _ 5 | REPL , _ 6 | REPL - _ 7 | REPL . _ 8 | REPL : _ 9 | REPL ; _ 10 | REPL ? _ 11 | NORM ä a 12 | NORM ë e 13 | NORM ï i 14 | NORM ü u 15 | NORM ö o 16 | REPL ‘ ' 17 | ALLOW ' _ 18 | ALLOW a _ 19 | ALLOW â _ 20 | ALLOW e _ 21 | ALLOW ê _ 22 | ALLOW é _ 23 | ALLOW i _ 24 | ALLOW y _ 25 | ALLOW o _ 26 | ALLOW ô _ 27 | ALLOW u _ 28 | ALLOW û _ 29 | ALLOW ú _ 30 | ALLOW b _ 31 | ALLOW c _ 32 | ALLOW d _ 33 | ALLOW f _ 34 | ALLOW g _ 35 | ALLOW h _ 36 | ALLOW j _ 37 | ALLOW k _ 38 | ALLOW l _ 39 | ALLOW m _ 40 | ALLOW n _ 41 | ALLOW p _ 42 | ALLOW q _ 43 | ALLOW r _ 44 | ALLOW s _ 45 | ALLOW t _ 46 | ALLOW v _ 47 | ALLOW w _ 48 | ALLOW x _ 49 | ALLOW z _ 50 | ALLO _ _ 51 | -------------------------------------------------------------------------------- /cvutils/data/fy-NL/vocab.tsv: -------------------------------------------------------------------------------- 1 | nul 2 | ien 3 | twa 4 | trije 5 | fjouwer 6 | fiif 7 | seis 8 | sân 9 | acht 10 | njoggen 11 | ja 12 | nee 13 | -------------------------------------------------------------------------------- /cvutils/data/fy-NL/wikipedia.txt: -------------------------------------------------------------------------------- 1 | fy 2 | -------------------------------------------------------------------------------- /cvutils/data/ga-IE/alphabet.txt: -------------------------------------------------------------------------------- 1 | aábcdeéfghiíjklmnoóprstuúvwxy-' 2 | -------------------------------------------------------------------------------- /cvutils/data/ga-IE/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL ! _ 4 | REPL " _ 5 | REPL ( _ 6 | REPL ) _ 7 | REPL * _ 8 | REPL , _ 9 | REPL . _ 10 | REPL : _ 11 | REPL ; _ 12 | REPL ? _ 13 | REPL – _ 14 | REPL ‘ _ 15 | REPL ’ _ 16 | REPL ' _ 17 | ALLOW - _ 18 | ALLOW a _ 19 | ALLOW á _ 20 | ALLOW b _ 21 | ALLOW c _ 22 | ALLOW d _ 23 | ALLOW e _ 24 | ALLOW é _ 25 | ALLOW f _ 26 | ALLOW g _ 27 | ALLOW h _ 28 | ALLOW i _ 29 | ALLOW í _ 30 | ALLOW j _ 31 | ALLOW k _ 32 | ALLOW l _ 33 | ALLOW m _ 34 | ALLOW n _ 35 | ALLOW o _ 36 | ALLOW ó _ 37 | ALLOW p _ 38 | ALLOW r _ 39 | ALLOW s _ 40 | ALLOW t _ 41 | ALLOW u _ 42 | ALLOW ú _ 43 | ALLOW v _ 44 | ALLOW w _ 45 | ALLOW x _ 46 | ALLOW y _ 47 | ALLOW - _ 48 | ALLOW _ _ 49 | -------------------------------------------------------------------------------- /cvutils/data/ga-IE/wikipedia.txt: -------------------------------------------------------------------------------- 1 | ga 2 | -------------------------------------------------------------------------------- /cvutils/data/gl/alphabet.txt: -------------------------------------------------------------------------------- 1 | aábcdeéfghiílmnñoópqrstuúüvxz 2 | -------------------------------------------------------------------------------- /cvutils/data/gl/phon.tsv: -------------------------------------------------------------------------------- 1 | b b 2 | v b 3 | z θ 4 | c θ 5 | ch tʃ 6 | d d 7 | f f 8 | g ɡ 9 | gu ɡ 10 | c k 11 | qu k 12 | l l 13 | ll ʎ 14 | m m 15 | n n 16 | ñ ɲ 17 | nh ŋ 18 | p p 19 | r ɾ 20 | r r 21 | rr r 22 | s s 23 | t t 24 | x ʃ 25 | i i 26 | e e 27 | a a 28 | o o 29 | u u 30 | í i 31 | é e 32 | á a 33 | ó o 34 | ú u 35 | -------------------------------------------------------------------------------- /cvutils/data/gl/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/gl/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL ' _ 0027 _ 6 | REPL , _ 002c _ 7 | REPL - _ 002d _ 8 | REPL . _ 002e _ 9 | REPL : _ 003a _ 10 | REPL ; _ 003b _ 11 | REPL ? _ 003f _ 12 | REPL _ _ 005f _ 13 | REPL ¡ _ 00a1 _ 14 | REPL « _ 00ab _ 15 | REPL » _ 00bb _ 16 | REPL ¿ _ 00bf _ 17 | REPL – _ 2013 _ 18 | REPL — _ 2014 _ 19 | REPL ‘ _ 2018 _ 20 | REPL ’ _ 2019 _ 21 | REPL “ _ 201c _ 22 | REPL ” _ 201d _ 23 | REPL … _ 2026 _ 24 | REPL − _ 2212 _ 25 | ALLOW a _ 0061 _ 26 | ALLOW á _ 00e1 _ 27 | ALLOW b _ 0062 _ 28 | ALLOW c _ 0063 _ 29 | ALLOW d _ 0064 _ 30 | ALLOW e _ 0065 _ 31 | ALLOW é _ 00e9 _ 32 | ALLOW f _ 0066 _ 33 | ALLOW g _ 0067 _ 34 | ALLOW h _ 0068 _ 35 | ALLOW i _ 0069 _ 36 | ALLOW í _ 00ed _ 37 | ALLOW l _ 006c _ 38 | ALLOW m _ 006d _ 39 | ALLOW n _ 006e _ 40 | ALLOW ñ _ 00f1 _ 41 | ALLOW o _ 006f _ 42 | ALLOW ó _ 00f3 _ 43 | ALLOW p _ 0070 _ 44 | ALLOW q _ 0071 _ 45 | ALLOW r _ 0072 _ 46 | ALLOW s _ 0073 _ 47 | ALLOW t _ 0074 _ 48 | ALLOW u _ 0075 _ 49 | ALLOW ú _ 00fa _ 50 | ALLOW ü _ 00fc _ 51 | ALLOW v _ 0076 _ 52 | ALLOW x _ 0078 _ 53 | ALLOW z _ 007a _ 54 | ALLOW _ _ 0020 _ 55 | NORM â a 56 | NORM à a 57 | NORM ê e 58 | NORM è e 59 | NORM î i 60 | NORM ì i 61 | NORM ï i 62 | NORM ô o 63 | NORM ò o 64 | NORM û u 65 | NORM ù u 66 | NORM ç c 67 | NORM j i 68 | NORM ka ca 69 | NORM ko co 70 | NORM ku cu 71 | NORM ke que 72 | NORM ki qui 73 | NORM w u 74 | NORM y i 75 | -------------------------------------------------------------------------------- /cvutils/data/gl/vocab.tsv: -------------------------------------------------------------------------------- 1 | cero 2 | un 3 | dous 4 | tres 5 | catro 6 | cinco 7 | seis 8 | sete 9 | oito 10 | nove 11 | si 12 | non 13 | -------------------------------------------------------------------------------- /cvutils/data/gn/alphabet.txt: -------------------------------------------------------------------------------- 1 | aãábcdeẽéfgg̃hiĩíjklmnñoõóprstuũúvyỹýʼz 2 | -------------------------------------------------------------------------------- /cvutils/data/gn/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | ã ã 4 | ch ɕ 5 | e e 6 | ẽ ẽ 7 | g ɰ 8 | gu w 9 | g̃u w̃ 10 | g̃ ɰ̃ 11 | h h 12 | h x 13 | i i 14 | ĩ ĩ 15 | j ᵈj 16 | k k 17 | ku kʷ 18 | l l 19 | m m 20 | mb ᵐb 21 | n n 22 | nd ⁿd 23 | ng ᵑɡ 24 | ng ŋ 25 | ngu ᵑɡʷ 26 | ngu ŋʷ 27 | nt nt 28 | ñ ɲ 29 | o o 30 | õ õ 31 | p p 32 | r ɾ 33 | rr r 34 | s s 35 | t t 36 | u u 37 | ũ ũ 38 | v ʋ 39 | y ɨ 40 | ỹ ɨ̃ 41 | ʼ ʔ 42 | -------------------------------------------------------------------------------- /cvutils/data/gn/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL , _ 002c _ 6 | REPL - _ 002d _ 7 | REPL . _ 002e _ 8 | REPL ? _ 003f _ 9 | REPL ¡ _ 00a1 _ 10 | REPL ’ _ 2019 _ 11 | REPL “ _ 201c _ 12 | REPL ” _ 201d _ 13 | NORM ' ʼ 0027 02bc 14 | ALLOW a _ 0061 _ 15 | ALLOW á _ 00e1 _ 16 | ALLOW ã _ 00e3 _ 17 | ALLOW b _ 0062 _ 18 | ALLOW c _ 0063 _ 19 | ALLOW d _ 0064 _ 20 | ALLOW e _ 0065 _ 21 | ALLOW é _ 00e9 _ 22 | ALLOW ẽ _ 1ebd _ 23 | ALLOW f _ 0066 _ 24 | ALLOW g _ 0067 _ 25 | ALLOW ̃ _ 0303 _ 26 | ALLOW h _ 0068 _ 27 | ALLOW i _ 0069 _ 28 | ALLOW í _ 00ed _ 29 | ALLOW ĩ _ 0129 _ 30 | ALLOW j _ 006a _ 31 | ALLOW k _ 006b _ 32 | ALLOW l _ 006c _ 33 | ALLOW m _ 006d _ 34 | ALLOW n _ 006e _ 35 | ALLOW ñ _ 00f1 _ 36 | ALLOW o _ 006f _ 37 | ALLOW ó _ 00f3 _ 38 | ALLOW õ _ 00f5 _ 39 | ALLOW p _ 0070 _ 40 | ALLOW r _ 0072 _ 41 | ALLOW s _ 0073 _ 42 | ALLOW t _ 0074 _ 43 | ALLOW u _ 0075 _ 44 | ALLOW ú _ 00fa _ 45 | ALLOW ũ _ 0169 _ 46 | ALLOW v _ 0076 _ 47 | ALLOW y _ 0079 _ 48 | ALLOW ý _ 00fd _ 49 | ALLOW ỹ _ 1ef9 _ 50 | ALLOW z _ 007a _ 51 | ALLOW ʼ _ 02bc _ 52 | ALLOW _ _ 0020 _ 53 | -------------------------------------------------------------------------------- /cvutils/data/ha/alphabet.txt: -------------------------------------------------------------------------------- 1 | abɓcdɗefghijkƙlmnorstuwyƴzʼāă 2 | -------------------------------------------------------------------------------- /cvutils/data/ha/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | b b 4 | ɓ ɓ 5 | c tʃ 6 | d d 7 | ɗ ɗ 8 | e e 9 | f ɸ 10 | g ɡ 11 | h h 12 | i i 13 | j dʒ 14 | j ʒ 15 | k k 16 | ƙ kʼ 17 | l l 18 | m m 19 | n n 20 | o o 21 | r r 22 | r ɽ 23 | s s 24 | sh ʃ 25 | t t 26 | ts tsʼ 27 | ts sʼ 28 | u u 29 | w w 30 | y j 31 | ƴ ʔʲ 32 | z z 33 | ʼ ʔ 34 | -------------------------------------------------------------------------------- /cvutils/data/ha/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL % _ 0025 _ 6 | REPL , _ 002c _ 7 | REPL - _ 002d _ 8 | REPL . _ 002e _ 9 | REPL / _ 002f _ 10 | REPL : _ 003a _ 11 | REPL ; _ 003b _ 12 | REPL ? _ 003f _ 13 | REPL — _ 2014 _ 14 | REPL “ _ 201c _ 15 | REPL ” _ 201d _ 16 | NORM ' ʼ 0027 02bc 17 | NORM ’ ʼ 2019 02bc 18 | SKIP 0 _ 0030 _ 19 | SKIP 1 _ 0031 _ 20 | SKIP 3 _ 0033 _ 21 | SKIP 5 _ 0035 _ 22 | SKIP ( _ 0028 _ 23 | SKIP ) _ 0029 _ 24 | SKIP [ _ 005b _ 25 | SKIP ] _ 005d _ 26 | ALLOW a _ 0061 _ 27 | ALLOW b _ 0062 _ 28 | ALLOW ɓ _ 0253 _ 29 | ALLOW c _ 0063 _ 30 | ALLOW d _ 0064 _ 31 | ALLOW ɗ _ 0257 _ 32 | ALLOW e _ 0065 _ 33 | ALLOW f _ 0066 _ 34 | ALLOW g _ 0067 _ 35 | ALLOW h _ 0068 _ 36 | ALLOW ʼ _ 02bc _ 37 | ALLOW i _ 0069 _ 38 | ALLOW í _ 00ed _ 39 | ALLOW j _ 006a _ 40 | ALLOW k _ 006b _ 41 | ALLOW ƙ _ 0199 _ 42 | ALLOW l _ 006c _ 43 | ALLOW m _ 006d _ 44 | ALLOW n _ 006e _ 45 | ALLOW o _ 006f _ 46 | ALLOW p _ 0070 _ 47 | ALLOW q _ 0071 _ 48 | ALLOW r _ 0072 _ 49 | ALLOW s _ 0073 _ 50 | ALLOW t _ 0074 _ 51 | ALLOW u _ 0075 _ 52 | ALLOW v _ 0076 _ 53 | ALLOW w _ 0077 _ 54 | ALLOW x _ 0078 _ 55 | ALLOW y _ 0079 _ 56 | ALLOW ƴ _ 01b4 _ 57 | ALLOW z _ 007a _ 58 | ALLOW _ _ 0020 _ 59 | -------------------------------------------------------------------------------- /cvutils/data/he/alphabet.txt: -------------------------------------------------------------------------------- 1 | אבבּגגּדדּהוזחטיככּךךּלמםנןסעפפּףףּצץקרשׂשׁתתּ 2 | -------------------------------------------------------------------------------- /cvutils/data/hi/alphabet.txt: -------------------------------------------------------------------------------- 1 | ऋऐकखगघङचछजझञटठडढणतथदधनपफबभमयरलवशषसहऺऻ़ऽािीुूृॄॅॆेैॉॊोौ्ॎॏंःअआइउएऑओऔँऊई 2 | -------------------------------------------------------------------------------- /cvutils/data/hi/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL , _ _ _ 4 | REPL ‘ _ 2018 _ 5 | REPL ’ _ 2019 _ 6 | REPL . _ _ _ 7 | REPL ! _ 0021 _ 8 | REPL ? _ 003f _ 9 | REPL " _ 0022 _ 10 | REPL : _ 003a _ 11 | REPL ' _ 0027 _ 12 | REPL - _ _ _ 13 | REPL । _ _ _ 14 | REPL | _ _ _ 15 | ALLOW अ _ 0905 _ 16 | ALLOW आ _ 0906 _ 17 | ALLOW इ _ 0907 _ 18 | ALLOW ई _ 0908 _ 19 | ALLOW उ _ 0909 _ 20 | ALLOW ऐ _ 0910 _ 21 | ALLOW ऊ _ 090a _ 22 | ALLOW ऋ _ 090b _ 23 | ALLOW ऌ _ 090c _ 24 | ALLOW क _ 0915 _ 25 | ALLOW ख _ 0916 _ 26 | ALLOW ग _ 0917 _ 27 | ALLOW घ _ 0918 _ 28 | ALLOW ङ _ 0919 _ 29 | ALLOW च _ 091a _ 30 | ALLOW छ _ 091b _ 31 | ALLOW ज _ 091c _ 32 | ALLOW झ _ 091d _ 33 | ALLOW ञ _ 091e _ 34 | ALLOW ट _ 091f _ 35 | ALLOW ठ _ 0920 _ 36 | ALLOW ड _ 0921 _ 37 | ALLOW ढ _ 0922 _ 38 | ALLOW ण _ 0923 _ 39 | ALLOW त _ 0924 _ 40 | ALLOW थ _ 0925 _ 41 | ALLOW द _ 0926 _ 42 | ALLOW ध _ 0927 _ 43 | ALLOW न _ 0928 _ 44 | ALLOW प _ 092a _ 45 | ALLOW फ _ 092b _ 46 | ALLOW ब _ 092c _ 47 | ALLOW भ _ 092d _ 48 | ALLOW म _ 092e _ 49 | ALLOW य _ 092f _ 50 | ALLOW र _ 0930 _ 51 | ALLOW ल _ 0932 _ 52 | ALLOW व _ 0935 _ 53 | ALLOW श _ 0936 _ 54 | ALLOW ष _ 0937 _ 55 | ALLOW स _ 0938 _ 56 | ALLOW ह _ 0939 _ 57 | ALLOW ए _ 090f _ 58 | ALLOW ऑ _ 0911 _ 59 | ALLOW ओ _ 0913 _ 60 | ALLOW औ _ 0914 _ 61 | ALLOW ऺ _ 093a _ 62 | ALLOW ऻ _ 093b _ 63 | ALLOW ़ _ 093c _ 64 | ALLOW ऽ _ 093d _ 65 | ALLOW ा _ 093e _ 66 | ALLOW ि _ 093f _ 67 | ALLOW ी _ 0940 _ 68 | ALLOW ु _ 0941 _ 69 | ALLOW ू _ 0942 _ 70 | ALLOW ृ _ 0943 _ 71 | ALLOW ॄ _ 0944 _ 72 | ALLOW ॅ _ 0945 _ 73 | ALLOW ॆ _ 0946 _ 74 | ALLOW े _ 0947 _ 75 | ALLOW ै _ 0948 _ 76 | ALLOW ॉ _ 0949 _ 77 | ALLOW ॊ _ 094a _ 78 | ALLOW ो _ 094b _ 79 | ALLOW ौ _ 094c _ 80 | ALLOW ् _ 094d _ 81 | ALLOW ॎ _ 094e _ 82 | ALLOW ॏ _ 094f _ 83 | ALLOW ं _ 0902 _ 84 | ALLOW ः _ 0903 _ 85 | ALLOW ँ _ 0901 _ 86 | ALLOW _ _ 0020 _ 87 | -------------------------------------------------------------------------------- /cvutils/data/hsb/abbr.tsv: -------------------------------------------------------------------------------- 1 | 9 z.t. 2 | 5 t.r. 3 | 5 př. 4 | 5 n.př. 5 | 4 t.m. 6 | 4 př.n.l.č. 7 | 3 mj.dr. 8 | 3 h.c. 9 | 2 t.j. 10 | 2 n.l.č. 11 | 2 i.d. 12 | 1 t.z.w.r. 13 | 1 t.mj. 14 | 1 s.str. 15 | 1 s.r.o. 16 | 1 s.l. 17 | 1 n.n. 18 | 1 n.m.hł. 19 | 1 n.m.h. 20 | 1 m.w. 21 | 1 e.tv. 22 | 1 a.t. 23 | 1 a.d. 24 | -------------------------------------------------------------------------------- /cvutils/data/hsb/alphabet.txt: -------------------------------------------------------------------------------- 1 | aábcćčdeěfghijklłmnńňoóöpqrřsśštuüvwxyzźž 2 | -------------------------------------------------------------------------------- /cvutils/data/hsb/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/hsb/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL ! _ 4 | REPL " _ 5 | REPL , _ 6 | REPL - _ 7 | REPL . _ 8 | REPL : _ 9 | REPL ; _ 10 | REPL ? _ 11 | REPL « _ 12 | REPL » _ 13 | REPL – _ 14 | REPL ’ _ 15 | REPL “ _ 16 | REPL „ _ 17 | NORM ň ń 18 | NORM ö o 19 | NORM ü u 20 | NORM á a 21 | ALLOW a _ 22 | ALLOW b _ 23 | ALLOW c _ 24 | ALLOW ć _ 25 | ALLOW č _ 26 | ALLOW d _ 27 | ALLOW e _ 28 | ALLOW ě _ 29 | ALLOW f _ 30 | ALLOW g _ 31 | ALLOW h _ 32 | ALLOW i _ 33 | ALLOW j _ 34 | ALLOW k _ 35 | ALLOW l _ 36 | ALLOW ł _ 37 | ALLOW m _ 38 | ALLOW n _ 39 | ALLOW ń _ 40 | ALLOW o _ 41 | ALLOW ó _ 42 | ALLOW p _ 43 | ALLOW q _ 44 | ALLOW r _ 45 | ALLOW ř _ 46 | ALLOW s _ 47 | ALLOW ś _ 48 | ALLOW š _ 49 | ALLOW t _ 50 | ALLOW u _ 51 | ALLOW v _ 52 | ALLOW w _ 53 | ALLOW x _ 54 | ALLOW y _ 55 | ALLOW z _ 56 | ALLOW ź _ 57 | ALLOW ž _ 58 | ALLOW _ _ 59 | -------------------------------------------------------------------------------- /cvutils/data/ht/alphabet.txt: -------------------------------------------------------------------------------- 1 | bcdfghjklmnprstvzaàeèioòwy 2 | -------------------------------------------------------------------------------- /cvutils/data/hu/alphabet.txt: -------------------------------------------------------------------------------- 1 | aábcdeéfghiíjklmnoóöőpqrstuúüűvwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/hu/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a ɒ 3 | á aː 4 | b b 5 | c ts 6 | cs tʃ 7 | d d 8 | dz dz 9 | dzs dʒ 10 | e ɛ 11 | é eː 12 | f f 13 | g ɡ 14 | gy ɟ 15 | h h 16 | i i 17 | í iː 18 | j j 19 | k k 20 | l l 21 | ly j 22 | m m 23 | n n 24 | ny ɲ 25 | o o 26 | ó oː 27 | ö ø 28 | ő øː 29 | p p 30 | r r 31 | s ʃ 32 | sz s 33 | t t 34 | ty c 35 | u u 36 | ú uː 37 | ü y 38 | ű yː 39 | v v 40 | y i 41 | z z 42 | zs ʒ 43 | -------------------------------------------------------------------------------- /cvutils/data/hu/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL ! _ 4 | REPL " _ 5 | REPL , _ 6 | REPL - _ 7 | REPL . _ 8 | REPL : _ 9 | REPL ; _ 10 | REPL ? _ 11 | REPL – _ 12 | REPL … _ 13 | ALLOW a _ 14 | ALLOW á _ 15 | ALLOW b _ 16 | ALLOW c _ 17 | ALLOW d _ 18 | ALLOW e _ 19 | ALLOW é _ 20 | ALLOW f _ 21 | ALLOW g _ 22 | ALLOW h _ 23 | ALLOW i _ 24 | ALLOW í _ 25 | ALLOW j _ 26 | ALLOW k _ 27 | ALLOW l _ 28 | ALLOW m _ 29 | ALLOW n _ 30 | ALLOW o _ 31 | ALLOW ó _ 32 | ALLOW ö _ 33 | ALLOW ő _ 34 | ALLOW p _ 35 | ALLOW q _ 36 | ALLOW r _ 37 | ALLOW s _ 38 | ALLOW t _ 39 | ALLOW u _ 40 | ALLOW ú _ 41 | ALLOW ü _ 42 | ALLOW ű _ 43 | ALLOW v _ 44 | ALLOW w _ 45 | ALLOW x _ 46 | ALLOW y _ 47 | ALLOW z _ 48 | ALLOW _ _ 49 | -------------------------------------------------------------------------------- /cvutils/data/hy-AM/alphabet.txt: -------------------------------------------------------------------------------- 1 | աբգդեեզէըթժիլխծկհձղճմյյննշոոչպջռսվտրցւփքօֆուևև -------------------------------------------------------------------------------- /cvutils/data/hy-AM/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | ա ɑ 3 | բ b 4 | գ ɡ 5 | դ d 6 | ե ɛ 7 | ե jɛ 8 | զ z 9 | է ɛː 10 | ը ə 11 | թ tʰ 12 | ժ ʒ 13 | ի i 14 | լ l 15 | խ χ 16 | ծ ts 17 | կ k 18 | հ h 19 | ձ dz 20 | ղ ʁ 21 | ճ tʃ 22 | մ m 23 | յ h 24 | յ j 25 | ն n 26 | ն ŋ 27 | շ ʃ 28 | ո ɔ 29 | ո ʋɔ 30 | չ tʃʰ 31 | պ p 32 | ջ dʒ 33 | ռ r 34 | ս s 35 | վ v 36 | տ t 37 | ր ɾ 38 | ց tsʰ 39 | ւ v 40 | փ pʰ 41 | ք kʰ 42 | օ o 43 | ֆ f 44 | ու u 45 | և ɛv 46 | և jɛv 47 | -------------------------------------------------------------------------------- /cvutils/data/hy-AM/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL , _ 002c _ 4 | REPL . _ 002e _ 5 | REPL « _ 00ab _ 6 | REPL » _ 00bb _ 7 | REPL ՛ _ 055b _ 8 | REPL ՝ _ 055d _ 9 | REPL ՞ _ 055e _ 10 | REPL ։ _ 0589 _ 11 | ALLOW ա _ 0531 _ 12 | ALLOW բ _ 0532 _ 13 | ALLOW գ _ 0533 _ 14 | ALLOW դ _ 0534 _ 15 | ALLOW ե _ 0535 _ 16 | ALLOW զ _ 0536 _ 17 | ALLOW է _ 0537 _ 18 | ALLOW ը _ 0538 _ 19 | ALLOW թ _ 0539 _ 20 | ALLOW ժ _ 053a _ 21 | ALLOW ի _ 053b _ 22 | ALLOW լ _ 053c _ 23 | ALLOW խ _ 053d _ 24 | ALLOW ծ _ 053e _ 25 | ALLOW կ _ 053f _ 26 | ALLOW հ _ 0540 _ 27 | ALLOW ձ _ 0541 _ 28 | ALLOW ղ _ 0542 _ 29 | ALLOW ճ _ 0543 _ 30 | ALLOW մ _ 0544 _ 31 | ALLOW յ _ 0545 _ 32 | ALLOW ն _ 0546 _ 33 | ALLOW շ _ 0547 _ 34 | ALLOW ո _ 0548 _ 35 | ALLOW չ _ 0549 _ 36 | ALLOW պ _ 054a _ 37 | ALLOW ջ _ 054b _ 38 | ALLOW ռ _ 054c _ 39 | ALLOW ս _ 054d _ 40 | ALLOW վ _ 054e _ 41 | ALLOW տ _ 054f _ 42 | ALLOW ր _ 0550 _ 43 | ALLOW ց _ 0551 _ 44 | ALLOW ւ _ 0552 _ 45 | ALLOW փ _ 0553 _ 46 | ALLOW ք _ 0554 _ 47 | ALLOW օ _ 0555 _ 48 | ALLOW ֆ _ 0556 _ 49 | ALLOW _ _ 0020 _ 50 | -------------------------------------------------------------------------------- /cvutils/data/hy-AM/wikipedia.txt: -------------------------------------------------------------------------------- 1 | hy 2 | -------------------------------------------------------------------------------- /cvutils/data/ia/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnopqrstuvwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/ia/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | b b 4 | c k 5 | c t͡s 6 | d d 7 | e e 8 | f f 9 | g ɡ 10 | h h 11 | h _ 12 | i i 13 | j ʒ 14 | k k 15 | l l 16 | m m 17 | n n 18 | o o 19 | p p 20 | q k 21 | r r 22 | s s 23 | s z 24 | t t 25 | u u 26 | v v 27 | w w 28 | w v 29 | x ks 30 | y i 31 | z z 32 | -------------------------------------------------------------------------------- /cvutils/data/ia/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL ' _ 0027 _ 6 | REPL , _ 002c _ 7 | REPL - _ 002d _ 8 | REPL . _ 002e _ 9 | REPL : _ 003a _ 10 | REPL ; _ 003b _ 11 | REPL ? _ 003f _ 12 | REPL · _ 00b7 _ 13 | REPL “ _ 201c _ 14 | REPL ” _ 201d _ 15 | NORM é e 00e9 _ 16 | NORM ö o 00f6 _ 17 | ALLOW a _ 0061 _ 18 | ALLOW b _ 0062 _ 19 | ALLOW c _ 0063 _ 20 | ALLOW d _ 0064 _ 21 | ALLOW e _ 0065 _ 22 | ALLOW f _ 0066 _ 23 | ALLOW g _ 0067 _ 24 | ALLOW h _ 0068 _ 25 | ALLOW i _ 0069 _ 26 | ALLOW j _ 006a _ 27 | ALLOW k _ 006b _ 28 | ALLOW l _ 006c _ 29 | ALLOW m _ 006d _ 30 | ALLOW n _ 006e _ 31 | ALLOW o _ 006f _ 32 | ALLOW p _ 0070 _ 33 | ALLOW q _ 0071 _ 34 | ALLOW r _ 0072 _ 35 | ALLOW s _ 0073 _ 36 | ALLOW t _ 0074 _ 37 | ALLOW u _ 0075 _ 38 | ALLOW v _ 0076 _ 39 | ALLOW w _ 0077 _ 40 | ALLOW x _ 0078 _ 41 | ALLOW y _ 0079 _ 42 | ALLOW z _ 007a _ 43 | ALLOW _ _ 0020 _ 44 | -------------------------------------------------------------------------------- /cvutils/data/id/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnoprstuvwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/id/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | b b 4 | c tʃ 5 | d d 6 | e e 7 | f f 8 | g g 9 | h h 10 | i i 11 | j dʒ 12 | k k 13 | l l 14 | m m 15 | n n 16 | o o 17 | p p 18 | q q 19 | q k 20 | r r 21 | s s 22 | t t 23 | u u 24 | v v 25 | w w 26 | x ks 27 | y j 28 | z z 29 | -------------------------------------------------------------------------------- /cvutils/data/id/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL ! _ 4 | REPL " _ 5 | REPL ' _ 6 | REPL , _ 7 | REPL - _ 8 | REPL – _ 9 | REPL — _ 10 | REPL “ _ 11 | REPL ” _ 12 | REPL . _ 13 | REPL : _ 14 | REPL ; _ 15 | REPL ? _ 16 | REPL ！ _ 17 | REPL ， _ 18 | REPL 0 _ 19 | REPL 1 _ 20 | NORM é e 21 | ALLOW a _ 22 | ALLOW b _ 23 | ALLOW c _ 24 | ALLOW d _ 25 | ALLOW e _ 26 | ALLOW f _ 27 | ALLOW g _ 28 | ALLOW h _ 29 | ALLOW i _ 30 | ALLOW j _ 31 | ALLOW k _ 32 | ALLOW l _ 33 | ALLOW m _ 34 | ALLOW n _ 35 | ALLOW o _ 36 | ALLOW p _ 37 | ALLOW r _ 38 | ALLOW s _ 39 | ALLOW t _ 40 | ALLOW u _ 41 | ALLOW v _ 42 | ALLOW w _ 43 | ALLOW x _ 44 | ALLOW y _ 45 | ALLOW z _ 46 | ALLOW _ _ 47 | -------------------------------------------------------------------------------- /cvutils/data/id/vocab.tsv: -------------------------------------------------------------------------------- 1 | nol 2 | satu 3 | dua 4 | tiga 5 | empat 6 | lima 7 | enam 8 | tujuh 9 | delapan 10 | sembilan 11 | ya 12 | tidak 13 | -------------------------------------------------------------------------------- /cvutils/data/ig/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghiịjklmnṅoọprstuụvwyz 2 | -------------------------------------------------------------------------------- /cvutils/data/ig/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | b b 3 | ch tʃ 4 | d d 5 | f f 6 | g ɡ 7 | gb ɡ͡b 8 | gh ɣ 9 | gw ɡʷ 10 | h ɦ 11 | j dʒ 12 | k k 13 | kp k͡p 14 | kw kʷ 15 | l l 16 | m m 17 | n n 18 | ṅ ŋ 19 | nw ŋʷ 20 | ny ɲ 21 | p p 22 | r r 23 | s s 24 | sh ʃ 25 | t t 26 | v v 27 | w w 28 | y j 29 | z z 30 | oo oː 31 | ọọ ɔː 32 | ọọ ɔː 33 | aa aː 34 | ee eː 35 | ii iː 36 | ịị ɪː 37 | ịị ɪː 38 | uu uː 39 | ụụ ʊː 40 | ụụ ʊː 41 | a a 42 | e e 43 | i i 44 | ị ɪ 45 | ị ɪ 46 | o o 47 | ọ ɔ 48 | ọ ɔ 49 | u u 50 | ụ ʊ 51 | ụ ʊ 52 | -------------------------------------------------------------------------------- /cvutils/data/ig/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NKFD _ _ _ _ 3 | REPL " _ 0022 _ 4 | REPL - _ 002d _ 5 | REPL . _ 002e _ 6 | REPL ' _ 0027 _ 7 | NORM á a 00e1 _ 8 | NORM à a 00e0 _ 9 | NORM ì i 00ec _ 10 | NORM ù u 00f9 _ 11 | ALLOW a _ 0061 _ 12 | ALLOW b _ 0062 _ 13 | ALLOW c _ 0063 _ 14 | ALLOW d _ 0064 _ 15 | ALLOW e _ 0065 _ 16 | ALLOW f _ 0066 _ 17 | ALLOW g _ 0067 _ 18 | ALLOW h _ 0068 _ 19 | ALLOW i _ 0069 _ 20 | ALLOW ị _ 1ecb _ 21 | ALLOW j _ 006a _ 22 | ALLOW k _ 006b _ 23 | ALLOW l _ 006c _ 24 | ALLOW m _ 006d _ 25 | ALLOW n _ 006e _ 26 | ALLOW ṅ _ 1e45 _ 27 | ALLOW o _ 006f _ 28 | ALLOW ọ _ 1ecd _ 29 | ALLOW p _ 0070 _ 30 | ALLOW r _ 0072 _ 31 | ALLOW s _ 0073 _ 32 | ALLOW t _ 0074 _ 33 | ALLOW u _ 0075 _ 34 | ALLOW ụ _ 1ee5 _ 35 | ALLOW v _ 0076 _ 36 | ALLOW w _ 0077 _ 37 | ALLOW y _ 0079 _ 38 | ALLOW z _ 007a _ 39 | ALLOW _ _ 0020 _ 40 | -------------------------------------------------------------------------------- /cvutils/data/is/alphabet.txt: -------------------------------------------------------------------------------- 1 | aábdðeéfghiíjklmnoóprstuúvxyýþæö 2 | -------------------------------------------------------------------------------- /cvutils/data/is/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ _ _ 4 | REPL ? _ _ _ 5 | REPL . _ _ _ 6 | REPL , _ _ _ 7 | REPL - _ _ _ 8 | REPL – _ _ _ 9 | NORM đ ð _ _ 10 | NORM ō ð _ _ 11 | NORM ū þ _ _ 12 | NORM ņ n _ _ 13 | NORM ñ n _ _ 14 | NORM ā á 0101 00e1 15 | NORM č é _ _ 16 | NORM ķ ó _ _ 17 | NORM ė í _ _ 18 | NORM ų ú _ _ 19 | NORM ũ ý _ _ 20 | ALLOW a _ 0061 _ 21 | ALLOW á _ 00e1 _ 22 | ALLOW b _ 0062 _ 23 | ALLOW d _ 0064 _ 24 | ALLOW ð _ 00f0 _ 25 | ALLOW e _ 0065 _ 26 | ALLOW é _ 00e9 _ 27 | ALLOW f _ 0066 _ 28 | ALLOW g _ 0067 _ 29 | ALLOW h _ 0068 _ 30 | ALLOW i _ 0069 _ 31 | ALLOW í _ 00ed _ 32 | ALLOW j _ 006a _ 33 | ALLOW k _ 006b _ 34 | ALLOW l _ 006c _ 35 | ALLOW m _ 006d _ 36 | ALLOW n _ 006e _ 37 | ALLOW o _ 006f _ 38 | ALLOW ó _ 00f3 _ 39 | ALLOW p _ 0070 _ 40 | ALLOW r _ 0072 _ 41 | ALLOW s _ 0073 _ 42 | ALLOW t _ 0074 _ 43 | ALLOW u _ 0075 _ 44 | ALLOW ú _ 00fa _ 45 | ALLOW v _ 0076 _ 46 | ALLOW x _ 0078 _ 47 | ALLOW y _ 0079 _ 48 | ALLOW ý _ 00fd _ 49 | ALLOW þ _ 00fe _ 50 | ALLOW æ _ 00e6 _ 51 | ALLOW ö _ 00f6 _ 52 | ALLOW _ _ 0020 _ 53 | -------------------------------------------------------------------------------- /cvutils/data/it/alphabet.txt: -------------------------------------------------------------------------------- 1 | aàbcdeéèfghijkíìlmnoóòpqrstuúùvwxyz' 2 | -------------------------------------------------------------------------------- /cvutils/data/it/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | à a 4 | b b 5 | c k 6 | chi ki 7 | chì ki 8 | che kɛ 9 | chè kɛ 10 | ci tʃi 11 | cì tʃi 12 | ce tʃɛ 13 | cè tʃɛ 14 | d d 15 | e e 16 | é e 17 | è e 18 | e ɛ 19 | é ɛ 20 | è ɛ 21 | f f 22 | g ɡ 23 | ghi gi 24 | gi dʒi 25 | gia dʒa 26 | gì dʒi 27 | ge dʒɛ 28 | gè dʒɛ 29 | h _ 30 | i i 31 | ì i 32 | í i 33 | i j 34 | l l 35 | m m 36 | n n 37 | n ŋ 38 | n ɱ 39 | o o 40 | ó o 41 | ò o 42 | o ɔ 43 | ò ɔ 44 | ó ɔ 45 | p p 46 | q k 47 | qu kw 48 | cqu kkw 49 | qqu kkw 50 | r r 51 | s s 52 | sc ʃ 53 | sca ska 54 | sch sk 55 | sci ʃ 56 | s z 57 | t t 58 | u u 59 | ú u 60 | ù u 61 | u w 62 | v v 63 | z ts 64 | z dz 65 | -------------------------------------------------------------------------------- /cvutils/data/it/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL " _ _ _ 4 | REPL ” _ _ _ 5 | REPL “ _ _ _ 6 | REPL , _ _ _ 7 | REPL - _ _ _ 8 | REPL – _ _ _ 9 | REPL . _ _ _ 10 | REPL : _ _ _ 11 | REPL ; _ _ _ 12 | REPL ? _ _ _ 13 | REPL ! _ _ _ 14 | NORM ’ ' _ _ 15 | ALLOW a _ 0061 _ 16 | ALLOW à _ 00e0 _ 17 | ALLOW b _ 0062 _ 18 | ALLOW c _ 0063 _ 19 | ALLOW d _ 0064 _ 20 | ALLOW e _ 0065 _ 21 | ALLOW é _ 00e9 _ 22 | ALLOW è _ 00e8 _ 23 | ALLOW f _ 0066 _ 24 | ALLOW g _ 0067 _ 25 | ALLOW h _ 0068 _ 26 | ALLOW i _ 0069 _ 27 | ALLOW í _ 00ed _ 28 | ALLOW ì _ 00ec _ 29 | ALLOW j _ 006a _ 30 | ALLOW k _ 006b _ 31 | ALLOW l _ 006c _ 32 | ALLOW m _ 006d _ 33 | ALLOW n _ 006e _ 34 | ALLOW o _ 006f _ 35 | ALLOW ó _ 00f3 _ 36 | ALLOW ò _ 00f2 _ 37 | ALLOW p _ 0070 _ 38 | ALLOW q _ 0071 _ 39 | ALLOW r _ 0072 _ 40 | ALLOW s _ 0073 _ 41 | ALLOW t _ 0074 _ 42 | ALLOW u _ 0075 _ 43 | ALLOW ú _ 00fa _ 44 | ALLOW ù _ 00f9 _ 45 | ALLOW v _ 0076 _ 46 | ALLOW w _ 0077 _ 47 | ALLOW x _ 0078 _ 48 | ALLOW y _ 0079 _ 49 | ALLOW z _ 007a _ 50 | ALLOW ' _ 0027 _ 51 | ALLOW _ _ 0020 _ 52 | -------------------------------------------------------------------------------- /cvutils/data/ka/alphabet.txt: -------------------------------------------------------------------------------- 1 | აბგდევზთიკლმნოპჟრსტუფქღყშჩცძწჭხჯჰ 2 | -------------------------------------------------------------------------------- /cvutils/data/ka/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | ა ɑ 3 | ბ b 4 | გ ɡ 5 | დ d 6 | ე ɛ 7 | ვ v 8 | ზ z 9 | ჱ eɪ 10 | თ t 11 | ი i 12 | კ kʼ 13 | ლ l 14 | მ m 15 | ნ n 16 | ჲ je 17 | ო ɔ 18 | პ pʼ 19 | ჟ ʒ 20 | რ r 21 | ს s 22 | ტ tʼ 23 | ჳ uɪ 24 | უ u 25 | ჷ ə 26 | ფ pʰ 27 | ქ kʰ 28 | ღ ɣ 29 | ყ qʼ 30 | ჸ ʔ 31 | შ ʃ 32 | ჩ tʃʰ 33 | ც tsʰ 34 | ძ dz 35 | წ tsʼ 36 | ჭ tʃʼ 37 | ხ χ 38 | ჴ qʰ 39 | ჯ dʒ 40 | ჰ h 41 | ჵ oː 42 | -------------------------------------------------------------------------------- /cvutils/data/ka/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL ! _ 4 | REPL , _ 5 | REPL - _ 6 | REPL . _ 7 | REPL : _ 8 | REPL ; _ 9 | REPL ? _ 10 | REPL – _ 11 | REPL “ _ 12 | REPL „ _ 13 | SKIP a _ 14 | SKIP k _ 15 | ALLOW ა _ 16 | ALLOW ბ _ 17 | ALLOW გ _ 18 | ALLOW დ _ 19 | ALLOW ე _ 20 | ALLOW ვ _ 21 | ALLOW ზ _ 22 | ALLOW თ _ 23 | ALLOW ი _ 24 | ALLOW კ _ 25 | ALLOW ლ _ 26 | ALLOW მ _ 27 | ALLOW ნ _ 28 | ALLOW ო _ 29 | ALLOW პ _ 30 | ALLOW ჟ _ 31 | ALLOW რ _ 32 | ALLOW ს _ 33 | ALLOW ტ _ 34 | ALLOW უ _ 35 | ALLOW ფ _ 36 | ALLOW ქ _ 37 | ALLOW ღ _ 38 | ALLOW ყ _ 39 | ALLOW შ _ 40 | ALLOW ჩ _ 41 | ALLOW ც _ 42 | ALLOW ძ _ 43 | ALLOW წ _ 44 | ALLOW ჭ _ 45 | ALLOW ხ _ 46 | ALLOW ჯ _ 47 | ALLOW ჰ _ 48 | 49 | -------------------------------------------------------------------------------- /cvutils/data/ka/vocab.tsv: -------------------------------------------------------------------------------- 1 | ნული 2 | ერთი 3 | ორი 4 | სამი 5 | ოთხი 6 | ხუთი 7 | ექვსი 8 | შვიდი 9 | რვა 10 | ცხრა 11 | დიახ 12 | არა 13 | -------------------------------------------------------------------------------- /cvutils/data/kab/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcčdḍeɛfgǧɣhḥijklmnqrřṛsṣtṭuwxyzẓ 2 | -------------------------------------------------------------------------------- /cvutils/data/kab/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a æ 3 | b b 4 | b β 5 | c ʃ 6 | č t͡ʃ 7 | d d 8 | d ð 9 | ḍ ðˤ 10 | e ə 11 | ɛ ʕ 12 | f f 13 | g ɡ 14 | ǧ d͡ʒ 15 | ɣ ɣ 16 | ɣ ʁ 17 | h h 18 | ḥ ħ 19 | i i 20 | j ʒ 21 | k k 22 | l l 23 | l ɫ 24 | m m 25 | n n 26 | q q 27 | q qʷ 28 | q ɢ 29 | r r 30 | r rˤ 31 | ř ɺ 32 | ṛ rˤ 33 | s s 34 | ṣ sˤ 35 | t t 36 | t θ 37 | ṭ tˤ 38 | u ʊ 39 | w w 40 | x x 41 | x χ 42 | y j 43 | z z 44 | ẓ zˤ 45 | -------------------------------------------------------------------------------- /cvutils/data/kab/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | NORM Ğ Ǧ 011e 01e6 4 | NORM ğ ǧ 011f 01e7 5 | NORM Γ Ɣ 0393 0194 6 | NORM γ ɣ 03b3 0263 7 | NORM Σ Ɛ 03a3 0190 8 | NORM Ԑ Ɛ 0510 0190 9 | NORM εσ ɛ 03b5-03c3 025b 10 | NORM ε ɛ 03b5 025b 11 | NORM ԑ ɛ 0511 025b 12 | NORM « " 00ab 0022 13 | NORM » " 00bb 0022 14 | NORM “ " 201c 0022 15 | NORM ‟ " 201f 0022 16 | NORM ” " 201d 0022 17 | NORM ‐ - 2010 002d 18 | REPL " _ 0022 _ 19 | REPL . _ 002e _ 20 | REPL ? _ 003f _ 21 | REPL ! _ 0021 _ 22 | REPL , _ 002c _ 23 | REPL ; _ 003b _ 24 | REPL : _ 003a _ 25 | REPL – _ 2013 _ 26 | SKIP ( _ 0028 _ 27 | SKIP ) _ 0029 _ 28 | ALLOW a _ 0061 _ 29 | ALLOW b _ 0062 _ 30 | ALLOW c _ 0063 _ 31 | ALLOW č _ 010d _ 32 | ALLOW d _ 0064 _ 33 | ALLOW ḍ _ 1e0d _ 34 | ALLOW e _ 0065 _ 35 | ALLOW ɛ _ 025b _ 36 | ALLOW f _ 0066 _ 37 | ALLOW g _ 0067 _ 38 | ALLOW ǧ _ 01e7 _ 39 | ALLOW ɣ _ 0263 _ 40 | ALLOW h _ 0068 _ 41 | ALLOW ḥ _ 1e25 _ 42 | ALLOW i _ 0069 _ 43 | ALLOW j _ 006a _ 44 | ALLOW k _ 006b _ 45 | ALLOW l _ 006c _ 46 | ALLOW m _ 006d _ 47 | ALLOW n _ 006e _ 48 | ALLOW o _ 006f _ 49 | ALLOW p _ 0070 _ 50 | ALLOW q _ 0071 _ 51 | ALLOW r _ 0072 _ 52 | ALLOW ř _ 0159 _ 53 | ALLOW ṛ _ 1e5b _ 54 | ALLOW s _ 0073 _ 55 | ALLOW ṣ _ 1e63 _ 56 | ALLOW t _ 0074 _ 57 | ALLOW ṭ _ 1e6d _ 58 | ALLOW u _ 0075 _ 59 | ALLOW v _ 0076 _ 60 | ALLOW w _ 0077 _ 61 | ALLOW x _ 0078 _ 62 | ALLOW y _ 0079 _ 63 | ALLOW z _ 007a _ 64 | ALLOW ẓ _ 1e93 _ 65 | ALLOW - _ 002d _ 66 | ALLOW _ _ 005f _ 67 | -------------------------------------------------------------------------------- /cvutils/data/kab/vocab.tsv: -------------------------------------------------------------------------------- 1 | ih 2 | ilem 3 | kraḍ 4 | kuẓ 5 | sa 6 | semmus 7 | sin 8 | sḍis 9 | tẓa 10 | uhu 11 | yiwen 12 | ṭam 13 | -------------------------------------------------------------------------------- /cvutils/data/ki/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdeghiĩjkmnortuũwylfpqsvxz 2 | -------------------------------------------------------------------------------- /cvutils/data/ki/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | e ɛ 4 | i i 5 | ĩ e 6 | o ɔ 7 | u u 8 | ũ o 9 | t t 10 | k k 11 | mb ᵐb 12 | nd ⁿd 13 | ng ᵑɡ 14 | nj ᶮdʒ 15 | m m 16 | n n 17 | ny ɲ 18 | ng' ŋ 19 | c ʃ 20 | h h 21 | b β 22 | th ð 23 | g ɣ 24 | r ɾ 25 | y j 26 | w w 27 | z z 28 | f f 29 | p p 30 | s s 31 | v v 32 | x ks 33 | -------------------------------------------------------------------------------- /cvutils/data/ki/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL ʼ _ 02bc _ 5 | REPL , _ 002c _ 6 | REPL - _ 002d _ 7 | REPL . _ 002e _ 8 | REPL : _ 003a _ 9 | REPL ; _ 003b _ 10 | REPL ? _ 003f _ 11 | REPL ‑ _ 2011 _ 12 | REPL – _ 2013 _ 13 | REPL ‘ _ 2018 _ 14 | REPL ’ _ 2019 _ 15 | REPL “ _ 201c _ 16 | REPL ” _ 201d _ 17 | REPL … _ 2026 _ 18 | SKIP 0 _ 0030 _ 19 | SKIP 1 _ 0031 _ 20 | SKIP 2 _ 0032 _ 21 | SKIP 3 _ 0033 _ 22 | SKIP 4 _ 0034 _ 23 | SKIP 5 _ 0035 _ 24 | SKIP 6 _ 0036 _ 25 | SKIP 7 _ 0037 _ 26 | SKIP 8 _ 0038 _ 27 | SKIP 9 _ 0039 _ 28 | SKIP ‎ _ 200e _ 29 | SKIP ( _ 0028 _ 30 | SKIP ) _ 0029 _ 31 | SKIP * _ 002a _ 32 | SKIP + _ 002b _ 33 | SKIP \ _ 005c _ 34 | ALLOW a _ 0061 _ 35 | ALLOW b _ 0062 _ 36 | ALLOW c _ 0063 _ 37 | ALLOW d _ 0064 _ 38 | ALLOW e _ 0065 _ 39 | ALLOW f _ 0066 _ 40 | ALLOW g _ 0067 _ 41 | ALLOW h _ 0068 _ 42 | ALLOW i _ 0069 _ 43 | ALLOW ĩ _ 0129 _ 44 | ALLOW j _ 006a _ 45 | ALLOW k _ 006b _ 46 | ALLOW l _ 006c _ 47 | ALLOW m _ 006d _ 48 | ALLOW n _ 006e _ 49 | ALLOW o _ 006f _ 50 | ALLOW p _ 0070 _ 51 | ALLOW r _ 0072 _ 52 | ALLOW s _ 0073 _ 53 | ALLOW t _ 0074 _ 54 | ALLOW u _ 0075 _ 55 | ALLOW ũ _ 0169 _ 56 | ALLOW v _ 0076 _ 57 | ALLOW w _ 0077 _ 58 | ALLOW x _ 0078 _ 59 | ALLOW y _ 0079 _ 60 | ALLOW z _ 007a _ 61 | ALLOW _ _ 0020 _ 62 | -------------------------------------------------------------------------------- /cvutils/data/kk/alphabet.txt: -------------------------------------------------------------------------------- 1 | аәбвгғдеёжзийкқлмнңоөпрстуұүфхһцчшщъыіьэюя 2 | -------------------------------------------------------------------------------- /cvutils/data/kk/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | м m 3 | п p 4 | б b 5 | ф f 6 | в v 7 | у w 8 | н n 9 | т t 10 | д d 11 | с s 12 | з z 13 | ц t͡s 14 | р ɾ 15 | л l 16 | ш ʃ 17 | ж ʒ 18 | щ ʃʃ 19 | й j 20 | ң ŋ 21 | к k 22 | г ɡ 23 | х χ 24 | қ q 25 | ғ ʁ 26 | һ h 27 | е i̯ɘ 28 | і ɘ 29 | и ɘ 30 | у ɘ 31 | ю ɘ 32 | ы ə 33 | и ə 34 | у ə 35 | ю ə 36 | ә æ 37 | а æ 38 | ө y̯ʉ 39 | ү ʉ 40 | а ɑ 41 | я ɑ 42 | о u̯ʊ 43 | ұ ʊ 44 | х x 45 | э ɛ 46 | о ɔ 47 | ё ɔ 48 | и i 49 | у u 50 | ю u 51 | р r 52 | и ɪ 53 | е e 54 | а a 55 | о a 56 | а aː 57 | -------------------------------------------------------------------------------- /cvutils/data/kk/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL " _ _ _ 4 | REPL » _ _ _ 5 | REPL « _ _ _ 6 | REPL ! _ _ _ 7 | REPL , _ _ _ 8 | REPL : _ _ _ 9 | REPL ; _ _ _ 10 | REPL . _ _ _ 11 | REPL ? _ _ _ 12 | REPL – _ _ _ 13 | REPL — _ _ _ 14 | NORM i і 0069 0456 15 | NORM a а 0061 0430 16 | NORM c с 0063 0441 17 | NORM ӊ ң 04ca 04a3 18 | ALLOW а _ 0430 _ 19 | ALLOW ә _ 04d9 _ 20 | ALLOW б _ 0431 _ 21 | ALLOW в _ 0432 _ 22 | ALLOW г _ 0433 _ 23 | ALLOW ғ _ 0493 _ 24 | ALLOW д _ 0434 _ 25 | ALLOW е _ 0435 _ 26 | ALLOW ё _ 0451 _ 27 | ALLOW ж _ 0436 _ 28 | ALLOW з _ 0437 _ 29 | ALLOW и _ 0438 _ 30 | ALLOW й _ 0439 _ 31 | ALLOW к _ 043a _ 32 | ALLOW қ _ 049b _ 33 | ALLOW л _ 043b _ 34 | ALLOW м _ 043c _ 35 | ALLOW н _ 043d _ 36 | ALLOW ң _ 04a3 _ 37 | ALLOW о _ 043e _ 38 | ALLOW ө _ 04e9 _ 39 | ALLOW п _ 043f _ 40 | ALLOW р _ 0440 _ 41 | ALLOW с _ 0441 _ 42 | ALLOW т _ 0442 _ 43 | ALLOW у _ 0443 _ 44 | ALLOW ұ _ 04b1 _ 45 | ALLOW ү _ 04af _ 46 | ALLOW ф _ 0444 _ 47 | ALLOW х _ 0445 _ 48 | ALLOW һ _ 04bb _ 49 | ALLOW ц _ 0446 _ 50 | ALLOW ч _ 0447 _ 51 | ALLOW ш _ 0448 _ 52 | ALLOW щ _ 0449 _ 53 | ALLOW ъ _ 044a _ 54 | ALLOW ы _ 044b _ 55 | ALLOW і _ 0456 _ 56 | ALLOW ь _ 044c _ 57 | ALLOW э _ 044d _ 58 | ALLOW ю _ 044e _ 59 | ALLOW я _ 044f _ 60 | ALLOW - _ _ _ 61 | ALLOW _ _ _ _ 62 | -------------------------------------------------------------------------------- /cvutils/data/kmr/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcçdeêfghiîjklllmnngopqrrrsştuûvwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/kmr/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | b b 3 | c dʒ 4 | ç tʃ 5 | d d 6 | f f 7 | g ɡ 8 | x ɣ 9 | h h 10 | h ħ 11 | j ʒ 12 | k k 13 | l l 14 | l ɫ 15 | ll ɫ 16 | m m 17 | n n 18 | ng ŋ 19 | p p 20 | q q 21 | r ɾ 22 | r r 23 | rr r 24 | s s 25 | ş ʃ 26 | t t 27 | v v 28 | w w 29 | x x 30 | y j 31 | z z 32 | a ɑː 33 | ê eː 34 | e ɛ 35 | î i 36 | i ɪ 37 | o oː 38 | u ʊ 39 | û uː 40 | -------------------------------------------------------------------------------- /cvutils/data/kmr/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL ' _ 0027 _ 6 | REPL , _ 002c _ 7 | REPL - _ 002d _ 8 | REPL . _ 002e _ 9 | REPL : _ 003a _ 10 | REPL ; _ 003b _ 11 | REPL ? _ 003f _ 12 | REPL ‘ _ 2018 _ 13 | REPL ’ _ 2019 _ 14 | REPL “ _ 201c _ 15 | REPL ” _ 201d _ 16 | REPL … _ 2026 _ 17 | NORM ū û 016b 00fb 18 | NORM ī î 012b 00ee 19 | ALLOW a _ 0061 _ 20 | ALLOW b _ 0062 _ 21 | ALLOW c _ 0063 _ 22 | ALLOW ç _ 00e7 _ 23 | ALLOW d _ 0064 _ 24 | ALLOW e _ 0065 _ 25 | ALLOW ê _ 00ea _ 26 | ALLOW f _ 0066 _ 27 | ALLOW g _ 0067 _ 28 | ALLOW h _ 0068 _ 29 | ALLOW i _ 0069 _ 30 | ALLOW î _ 00ee _ 31 | ALLOW j _ 006a _ 32 | ALLOW k _ 006b _ 33 | ALLOW l _ 006c _ 34 | ALLOW m _ 006d _ 35 | ALLOW n _ 006e _ 36 | ALLOW o _ 006f _ 37 | ALLOW ö _ 00f6 _ 38 | ALLOW p _ 0070 _ 39 | ALLOW q _ 0071 _ 40 | ALLOW r _ 0072 _ 41 | ALLOW s _ 0073 _ 42 | ALLOW ş _ 015f _ 43 | ALLOW t _ 0074 _ 44 | ALLOW u _ 0075 _ 45 | ALLOW û _ 00fb _ 46 | ALLOW v _ 0076 _ 47 | ALLOW w _ 0077 _ 48 | ALLOW x _ 0078 _ 49 | ALLOW y _ 0079 _ 50 | ALLOW z _ 007a _ 51 | ALLOW _ 0020 _ 52 | -------------------------------------------------------------------------------- /cvutils/data/kpv/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгдеёжзиійклмноӧпрстуфхцчшщъыьэюя 2 | -------------------------------------------------------------------------------- /cvutils/data/kpv/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | а ɑ 3 | б b 4 | в v 5 | г g 6 | д d 7 | дж dʒ 8 | дз dʑ 9 | е je 10 | ё jo 11 | ж ʒ 12 | з z 13 | и i 14 | и ʲi 15 | і i 16 | й j 17 | к k 18 | л ɫ 19 | м m 20 | н n 21 | о o 22 | ӧ ə 23 | п p 24 | р r 25 | с s 26 | т t 27 | тш tʃ 28 | у u 29 | ф f 30 | х x 31 | ц ts 32 | ч tɕ 33 | ш ʃ 34 | щ ɕ 35 | щ ɕː 36 | ъ - 37 | ы ɨ 38 | ь ʲ 39 | э e 40 | ю ju 41 | я jɑ 42 | -------------------------------------------------------------------------------- /cvutils/data/kpv/wikipedia.txt: -------------------------------------------------------------------------------- 1 | kv 2 | -------------------------------------------------------------------------------- /cvutils/data/ky/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгдеёжзийклмнңоөпрстуүфхцчшщъыьэюя 2 | -------------------------------------------------------------------------------- /cvutils/data/ky/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | а ɑ 3 | б b 4 | в v 5 | г ɡ 6 | д d 7 | е je 8 | ё jo 9 | ж d͡ʒ 10 | з z 11 | и i 12 | й j 13 | к k 14 | л l 15 | м m 16 | н n 17 | ң ŋ 18 | о o 19 | ө ø 20 | п p 21 | р r 22 | с s 23 | т t 24 | у u 25 | ү y 26 | ф f 27 | х k 28 | ц t͡s 29 | ч t͡ʃ 30 | ш ʃ 31 | щ ʃt͡ʃ 32 | ы ɯ 33 | э e 34 | ю ju 35 | я jɑ 36 | -------------------------------------------------------------------------------- /cvutils/data/ky/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/ky/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | REPL ! _ 3 | REPL , _ 4 | REPL . _ 5 | REPL ? _ 6 | REPL – _ 7 | REPL — _ 8 | REPL ” _ 9 | SKIP ⅛ _ 10 | NORM ӊ ң 11 | ALLOW а _ 12 | ALLOW б _ 13 | ALLOW в _ 14 | ALLOW г _ 15 | ALLOW д _ 16 | ALLOW е _ 17 | ALLOW ё _ 18 | ALLOW ж _ 19 | ALLOW з _ 20 | ALLOW и _ 21 | ALLOW й _ 22 | ALLOW к _ 23 | ALLOW л _ 24 | ALLOW м _ 25 | ALLOW н _ 26 | ALLOW ң _ 27 | ALLOW о _ 28 | ALLOW ө _ 29 | ALLOW п _ 30 | ALLOW р _ 31 | ALLOW с _ 32 | ALLOW т _ 33 | ALLOW у _ 34 | ALLOW ү _ 35 | ALLOW ф _ 36 | ALLOW х _ 37 | ALLOW ц _ 38 | ALLOW ч _ 39 | ALLOW ш _ 40 | ALLOW щ _ 41 | ALLOW ъ _ 42 | ALLOW ы _ 43 | ALLOW ь _ 44 | ALLOW э _ 45 | ALLOW ю _ 46 | ALLOW я _ 47 | ALLOW _ _ 48 | -------------------------------------------------------------------------------- /cvutils/data/ky/vocab.tsv: -------------------------------------------------------------------------------- 1 | нөл 2 | бир 3 | эки 4 | үч 5 | төрт 6 | беш 7 | алты 8 | жети 9 | сегиз 10 | тогуз 11 | ооба 12 | жок 13 | -------------------------------------------------------------------------------- /cvutils/data/lg/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnoprstuvwxyz' 2 | -------------------------------------------------------------------------------- /cvutils/data/lg/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | b b 3 | d d 4 | f f 5 | l l 6 | m m 7 | n n 8 | p p 9 | s s 10 | t t 11 | v v 12 | w w 13 | y j 14 | z z 15 | c c 16 | j ɟ 17 | ny ɲ 18 | ŋ ŋ 19 | a a 20 | e e 21 | i i 22 | o o 23 | u u 24 | k k 25 | ki ci 26 | ky ci 27 | g g 28 | gi ɟi 29 | gy ɟi 30 | nÿ ɲj 31 | -------------------------------------------------------------------------------- /cvutils/data/lg/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | REPL ! _ 3 | REPL " _ 4 | REPL ' _ 5 | REPL , _ 6 | REPL . _ 7 | REPL ? _ 8 | NORM ’ ' 9 | ALLOW a _ 10 | ALLOW b _ 11 | ALLOW c _ 12 | ALLOW d _ 13 | ALLOW e _ 14 | ALLOW f _ 15 | ALLOW g _ 16 | ALLOW h _ 17 | ALLOW i _ 18 | ALLOW j _ 19 | ALLOW k _ 20 | ALLOW l _ 21 | ALLOW m _ 22 | ALLOW n _ 23 | ALLOW o _ 24 | ALLOW p _ 25 | ALLOW r _ 26 | ALLOW s _ 27 | ALLOW t _ 28 | ALLOW u _ 29 | ALLOW v _ 30 | ALLOW w _ 31 | ALLOW x _ 32 | ALLOW y _ 33 | ALLOW z _ 34 | ALLOW ' _ 35 | ALLOW _ _ 36 | -------------------------------------------------------------------------------- /cvutils/data/lg/vocab.tsv: -------------------------------------------------------------------------------- 1 | nooti 2 | emu 3 | bbiri 4 | ssatu 5 | nnya 6 | ttaano 7 | mukaaga 8 | musanvu 9 | munaana 10 | mwenda 11 | ye 12 | nedda 13 | -------------------------------------------------------------------------------- /cvutils/data/ln/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdeɛfghiklmnoɔprstuvwyzjq 2 | -------------------------------------------------------------------------------- /cvutils/data/ln/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | DEL   _ 2006 _ 4 | REPL ! _ 0021 _ 5 | REPL , _ 002c _ 6 | REPL - _ 002d _ 7 | REPL . _ 002e _ 8 | REPL : _ 003a _ 9 | REPL ; _ 003b _ 10 | REPL ? _ 003f _ 11 | REPL « _ 00ab _ 12 | REPL » _ 00bb _ 13 | REPL — _ 2014 _ 14 | REPL ʼ _ 02bc _ 15 | REPL ― _ 2015 _ 16 | REPL ‘ _ 2018 _ 17 | REPL ’ _ 2019 _ 18 | REPL … _ 2026 _ 19 | REPL ‹ _ 2039 _ 20 | REPL › _ 203a _ 21 | NORM é e 00e9 _ 22 | NORM è e 00e8 _ 23 | SKIP ( _ 0028 _ 24 | SKIP ) _ 0029 _ 25 | SKIP * _ 002a _ 26 | SKIP + _ 002b _ 27 | SKIP \ _ 005c _ 28 | SKIP ~ _ 007e _ 29 | SKIP x _ 0078 _ 30 | SKIP ⌞ _ 231e _ 31 | SKIP ⌟ _ 231f _ 32 | SKIP 0 _ 0030 _ 33 | SKIP 1 _ 0031 _ 34 | SKIP 2 _ 0032 _ 35 | SKIP 3 _ 0033 _ 36 | SKIP 4 _ 0034 _ 37 | SKIP 5 _ 0035 _ 38 | SKIP 6 _ 0036 _ 39 | SKIP 7 _ 0037 _ 40 | SKIP 8 _ 0038 _ 41 | SKIP 9 _ 0039 _ 42 | ALLOW a _ 0061 _ 43 | ALLOW b _ 0062 _ 44 | ALLOW c _ 0063 _ 45 | ALLOW d _ 0064 _ 46 | ALLOW e _ 0065 _ 47 | ALLOW f _ 0066 _ 48 | ALLOW g _ 0067 _ 49 | ALLOW h _ 0068 _ 50 | ALLOW i _ 0069 _ 51 | ALLOW j _ 006a _ 52 | ALLOW k _ 006b _ 53 | ALLOW l _ 006c _ 54 | ALLOW m _ 006d _ 55 | ALLOW n _ 006e _ 56 | ALLOW o _ 006f _ 57 | ALLOW p _ 0070 _ 58 | ALLOW q _ 0071 _ 59 | ALLOW r _ 0072 _ 60 | ALLOW s _ 0073 _ 61 | ALLOW t _ 0074 _ 62 | ALLOW u _ 0075 _ 63 | ALLOW v _ 0076 _ 64 | ALLOW w _ 0077 _ 65 | ALLOW y _ 0079 _ 66 | ALLOW z _ 007a _ 67 | ALLOW _ _ 0020 _ 68 | -------------------------------------------------------------------------------- /cvutils/data/lo/alphabet.txt: -------------------------------------------------------------------------------- 1 | ກຂຄງຈຊຍດຕຖທນບປຜຝພຟມຢຣລວສຫອຮຯະັາຳິີຶືຸູົຼຽເແໂໃໄໆ່້໊໋໌ໍໜໝໞໟ 2 | -------------------------------------------------------------------------------- /cvutils/data/lt/alphabet.txt: -------------------------------------------------------------------------------- 1 | aąbcčdeėęfghiįjklmnopqrsštuųūvwxyzž 2 | -------------------------------------------------------------------------------- /cvutils/data/lt/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL " _ 4 | REPL . _ 5 | REPL – _ 6 | REPL “ _ 7 | REPL „ _ 8 | ALLOW a _ 9 | ALLOW ą _ 10 | ALLOW b _ 11 | ALLOW c _ 12 | ALLOW č _ 13 | ALLOW d _ 14 | ALLOW e _ 15 | ALLOW ė _ 16 | ALLOW ę _ 17 | ALLOW f _ 18 | ALLOW g _ 19 | ALLOW h _ 20 | ALLOW i _ 21 | ALLOW į _ 22 | ALLOW j _ 23 | ALLOW k _ 24 | ALLOW l _ 25 | ALLOW m _ 26 | ALLOW n _ 27 | ALLOW o _ 28 | ALLOW p _ 29 | ALLOW q _ 30 | ALLOW r _ 31 | ALLOW s _ 32 | ALLOW š _ 33 | ALLOW t _ 34 | ALLOW u _ 35 | ALLOW ų _ 36 | ALLOW ū _ 37 | ALLOW v _ 38 | ALLOW w _ 39 | ALLOW x _ 40 | ALLOW y _ 41 | ALLOW z _ 42 | ALLOW ž _ 43 | ALLOW _ _ 44 | -------------------------------------------------------------------------------- /cvutils/data/luo/alphabet.txt: -------------------------------------------------------------------------------- 1 | aeioubcdfghjklmnprstwyʼvxzq 2 | -------------------------------------------------------------------------------- /cvutils/data/luo/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | e e 4 | i i 5 | o o 6 | u u 7 | m m 8 | n n 9 | ny ɲ 10 | ngʼ ŋ 11 | mb ᵐb 12 | nd ⁿd 13 | nj ᶮɟ 14 | ng ᵑɡ 15 | p p 16 | th t̪ 17 | t t 18 | ch c 19 | k k 20 | b b 21 | dh d̪ 22 | d d 23 | j ɟ 24 | g ɡ 25 | f f 26 | s s 27 | h h 28 | r r 29 | w w 30 | l l 31 | y j 32 | -------------------------------------------------------------------------------- /cvutils/data/luo/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL , _ 002c _ 5 | REPL - _ 002d _ 6 | REPL . _ 002e _ 7 | REPL : _ 003a _ 8 | REPL ; _ 003b _ 9 | REPL ‘ _ 2018 _ 10 | REPL ’ _ 2019 _ 11 | REPL “ _ 201c _ 12 | REPL ” _ 201d _ 13 | REPL ? _ 003f _ 14 | DEL   _ 2006 _ 15 | SKIP 0 _ 0030 _ 16 | SKIP 1 _ 0031 _ 17 | SKIP 2 _ 0032 _ 18 | SKIP 3 _ 0033 _ 19 | SKIP 4 _ 0034 _ 20 | SKIP 5 _ 0035 _ 21 | SKIP 6 _ 0036 _ 22 | SKIP 7 _ 0037 _ 23 | SKIP 8 _ 0038 _ 24 | SKIP 9 _ 0039 _ 25 | SKIP ( _ 0028 _ 26 | SKIP ) _ 0029 _ 27 | SKIP * _ 002a _ 28 | SKIP + _ 002b _ 29 | SKIP [ _ 005b _ 30 | SKIP \ _ 005c _ 31 | SKIP ] _ 005d _ 32 | SKIP ‑ _ 2011 _ 33 | ALLOW a _ 0061 _ 34 | ALLOW b _ 0062 _ 35 | ALLOW c _ 0063 _ 36 | ALLOW d _ 0064 _ 37 | ALLOW e _ 0065 _ 38 | ALLOW f _ 0066 _ 39 | ALLOW g _ 0067 _ 40 | ALLOW h _ 0068 _ 41 | ALLOW i _ 0069 _ 42 | ALLOW j _ 006a _ 43 | ALLOW k _ 006b _ 44 | ALLOW l _ 006c _ 45 | ALLOW m _ 006d _ 46 | ALLOW n _ 006e _ 47 | ALLOW o _ 006f _ 48 | ALLOW p _ 0070 _ 49 | ALLOW q _ 0071 _ 50 | ALLOW r _ 0072 _ 51 | ALLOW s _ 0073 _ 52 | ALLOW t _ 0074 _ 53 | ALLOW u _ 0075 _ 54 | ALLOW v _ 0076 _ 55 | ALLOW w _ 0077 _ 56 | ALLOW x _ 0078 _ 57 | ALLOW y _ 0079 _ 58 | ALLOW z _ 007a _ 59 | ALLOW ʼ _ 02bc _ 60 | ALLOW _ _ 0020 _ 61 | -------------------------------------------------------------------------------- /cvutils/data/lv/alphabet.txt: -------------------------------------------------------------------------------- 1 | aābcčdeēfgģhiījkķlļmnņoprsštuūvxzž 2 | -------------------------------------------------------------------------------- /cvutils/data/lv/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a ɑ 3 | ā ɑː 4 | e e 5 | e æ 6 | ē eː 7 | ē æː 8 | i i 9 | ī iː 10 | o uɔ̯̯ 11 | o o 12 | o oː 13 | u u 14 | ū uː 15 | b b 16 | c t̪͡s̪ 17 | č t͡ʃ 18 | d d̪ 19 | dz d̪͡z̪ 20 | dž d͡ʒ 21 | f f 22 | g ɡ 23 | ģ ɟ 24 | h x 25 | j j 26 | k k 27 | ķ c 28 | l l 29 | ļ ʎ 30 | m m 31 | n n̪ 32 | n ŋ 33 | ņ ɲ 34 | p p 35 | r r 36 | r rʲ 37 | s s̪ 38 | š ʃ 39 | t t̪ 40 | v v 41 | z z̪ 42 | ž ʒ 43 | -------------------------------------------------------------------------------- /cvutils/data/lv/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL ! _ 4 | REPL " _ 5 | REPL ' _ 6 | REPL , _ 7 | REPL - _ 8 | REPL . _ 9 | REPL : _ 10 | REPL ; _ 11 | REPL ? _ 12 | REPL – _ 13 | REPL … _ 14 | ALLOW a _ 15 | ALLOW ā _ 16 | ALLOW b _ 17 | ALLOW c _ 18 | ALLOW č _ 19 | ALLOW d _ 20 | ALLOW e _ 21 | ALLOW ē _ 22 | ALLOW f _ 23 | ALLOW g _ 24 | ALLOW ģ _ 25 | ALLOW h _ 26 | ALLOW i _ 27 | ALLOW ī _ 28 | ALLOW j _ 29 | ALLOW k _ 30 | ALLOW ķ _ 31 | ALLOW l _ 32 | ALLOW ļ _ 33 | ALLOW m _ 34 | ALLOW n _ 35 | ALLOW ņ _ 36 | ALLOW o _ 37 | ALLOW p _ 38 | ALLOW r _ 39 | ALLOW s _ 40 | ALLOW š _ 41 | ALLOW t _ 42 | ALLOW u _ 43 | ALLOW ū _ 44 | ALLOW v _ 45 | ALLOW x _ 46 | ALLOW z _ 47 | ALLOW ž _ 48 | ALLOW _ _ 49 | -------------------------------------------------------------------------------- /cvutils/data/mdf/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгдеёжзийклмнопрстуфхцчшщъыьэюя 2 | -------------------------------------------------------------------------------- /cvutils/data/mdf/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL , _ 002c _ 5 | REPL - _ 002d _ 6 | REPL . _ 002e _ 7 | REPL : _ 003a _ 8 | REPL ? _ 003f _ 9 | REPL « _ 00ab _ 10 | REPL » _ 00bb _ 11 | REPL — _ 2014 _ 12 | ALLOW а _ 0430 _ 13 | ALLOW б _ 0431 _ 14 | ALLOW в _ 0432 _ 15 | ALLOW г _ 0433 _ 16 | ALLOW д _ 0434 _ 17 | ALLOW е _ 0435 _ 18 | ALLOW ё _ 0451 _ 19 | ALLOW ж _ 0436 _ 20 | ALLOW з _ 0437 _ 21 | ALLOW и _ 0438 _ 22 | ALLOW й _ 0439 _ 23 | ALLOW к _ 043a _ 24 | ALLOW л _ 043b _ 25 | ALLOW м _ 043c _ 26 | ALLOW н _ 043d _ 27 | ALLOW о _ 043e _ 28 | ALLOW п _ 043f _ 29 | ALLOW р _ 0440 _ 30 | ALLOW с _ 0441 _ 31 | ALLOW т _ 0442 _ 32 | ALLOW у _ 0443 _ 33 | ALLOW ф _ 0444 _ 34 | ALLOW х _ 0445 _ 35 | ALLOW ц _ 0446 _ 36 | ALLOW ч _ 0447 _ 37 | ALLOW ш _ 0448 _ 38 | ALLOW щ _ 0449 _ 39 | ALLOW ы _ 044b _ 40 | ALLOW ь _ 044c _ 41 | ALLOW э _ 044d _ 42 | ALLOW ю _ 044e _ 43 | ALLOW я _ 044f _ 44 | ALLOW _ _ 0020 _ 45 | -------------------------------------------------------------------------------- /cvutils/data/mhr/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгдеёжзийклмнҥоӧпрстуӱфхцчшщъыьэюя 2 | -------------------------------------------------------------------------------- /cvutils/data/mk/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгѓдежзѕијкќлљмнњопрстуфхцчџш 2 | -------------------------------------------------------------------------------- /cvutils/data/mk/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL , _ 002c _ 5 | REPL - _ 002d _ 6 | REPL . _ 002e _ 7 | REPL ? _ 003f _ 8 | NORM ѐ е 0450 _ 9 | NORM ѝ и 045d _ 10 | ALLOW а _ 0430 _ 11 | ALLOW б _ 0431 _ 12 | ALLOW в _ 0432 _ 13 | ALLOW г _ 0433 _ 14 | ALLOW ѓ _ 0453 _ 15 | ALLOW д _ 0434 _ 16 | ALLOW е _ 0435 _ 17 | ALLOW ж _ 0436 _ 18 | ALLOW з _ 0437 _ 19 | ALLOW ѕ _ 0455 _ 20 | ALLOW и _ 0438 _ 21 | ALLOW ј _ 0458 _ 22 | ALLOW к _ 043a _ 23 | ALLOW ќ _ 045c _ 24 | ALLOW л _ 043b _ 25 | ALLOW љ _ 0459 _ 26 | ALLOW м _ 043c _ 27 | ALLOW н _ 043d _ 28 | ALLOW њ _ 045a _ 29 | ALLOW о _ 043e _ 30 | ALLOW п _ 043f _ 31 | ALLOW р _ 0440 _ 32 | ALLOW с _ 0441 _ 33 | ALLOW т _ 0442 _ 34 | ALLOW у _ 0443 _ 35 | ALLOW ф _ 0444 _ 36 | ALLOW х _ 0445 _ 37 | ALLOW ц _ 0446 _ 38 | ALLOW ч _ 0447 _ 39 | ALLOW ш _ 0448 _ 40 | ALLOW _ _ 0020 _ 41 | -------------------------------------------------------------------------------- /cvutils/data/ml/alphabet.txt: -------------------------------------------------------------------------------- 1 | കഖഗഘങചചഛഛജജഝഝഞടഠഡഢണതഥദധനനപഫബഭമയരലവശഷസഹളഴററഩഺ -------------------------------------------------------------------------------- /cvutils/data/ml/validate.tsv: -------------------------------------------------------------------------------- 1 | NFKC _ _ _ _ 2 | REPL , _ 002c _ 3 | REPL - _ 002d _ 4 | REPL . _ 002e _ 5 | REPL ? _ 003f _ 6 | REPL ‘ _ 2018 _ 7 | REPL ’ _ 2019 _ 8 | ALLOW ക _ 0d15 _ 9 | ALLOW ഖ _ 0d16 _ 10 | ALLOW ഗ _ 0d17 _ 11 | ALLOW ഘ _ 0d18 _ 12 | ALLOW ങ _ 0d19 _ 13 | ALLOW ച _ 0d1a _ 14 | ALLOW ഛ _ 0d1b _ 15 | ALLOW ജ _ 0d1c _ 16 | ALLOW ഞ _ 0d1e _ 17 | ALLOW ട _ 0d1f _ 18 | ALLOW ഠ _ 0d20 _ 19 | ALLOW ഡ _ 0d21 _ 20 | ALLOW ണ _ 0d23 _ 21 | ALLOW ത _ 0d24 _ 22 | ALLOW ഥ _ 0d25 _ 23 | ALLOW ദ _ 0d26 _ 24 | ALLOW ധ _ 0d27 _ 25 | ALLOW ന _ 0d28 _ 26 | ALLOW പ _ 0d2a _ 27 | ALLOW ഫ _ 0d2b _ 28 | ALLOW ബ _ 0d2c _ 29 | ALLOW ഭ _ 0d2d _ 30 | ALLOW മ _ 0d2e _ 31 | ALLOW യ _ 0d2f _ 32 | ALLOW ര _ 0d30 _ 33 | ALLOW ല _ 0d32 _ 34 | ALLOW വ _ 0d35 _ 35 | ALLOW ശ _ 0d36 _ 36 | ALLOW ഷ _ 0d37 _ 37 | ALLOW സ _ 0d38 _ 38 | ALLOW ഹ _ 0d39 _ 39 | ALLOW ള _ 0d33 _ 40 | ALLOW ഴ _ 0d34 _ 41 | ALLOW റ _ 0d31 _ 42 | ALLOW അ _ 0d05 _ 43 | ALLOW ആ _ 0d06 _ 44 | ALLOW ഇ _ 0d07 _ 45 | ALLOW ഈ _ 0d08 _ 46 | ALLOW ഉ _ 0d09 _ 47 | ALLOW എ _ 0d0e _ 48 | ALLOW ഏ _ 0d0f _ 49 | ALLOW ഒ _ 0d12 _ 50 | ALLOW ഓ _ 0d13 _ 51 | ALLOW ൺ _ 0d7a _ 52 | ALLOW ൻ _ 0d7b _ 53 | ALLOW ർ _ 0d7c _ 54 | ALLOW ൽ _ 0d7d _ 55 | ALLOW ൾ _ 0d7e _ 56 | ALLOW ം _ 0d02 _ 57 | ALLOW ാ _ 0d3e _ 58 | ALLOW ി _ 0d3f _ 59 | ALLOW ീ _ 0d40 _ 60 | ALLOW ു _ 0d41 _ 61 | ALLOW ൂ _ 0d42 _ 62 | ALLOW ൃ _ 0d43 _ 63 | ALLOW െ _ 0d46 _ 64 | ALLOW േ _ 0d47 _ 65 | ALLOW ൈ _ 0d48 _ 66 | ALLOW ൊ _ 0d4a _ 67 | ALLOW ോ _ 0d4b _ 68 | ALLOW ൗ _ 0d57 _ 69 | ALLOW ് _ 0d4d _ 70 | ALLOW _ _ 0020 _ 71 | -------------------------------------------------------------------------------- /cvutils/data/mn/abbr.tsv: -------------------------------------------------------------------------------- 1 | 312 а.х. 2 | 172 г.м. 3 | 49 д.т. 4 | 41 ам. 5 | 35 д.т.д. 6 | 33 а.н. 7 | 21 х.х. 8 | 17 ц.с.ч. 9 | 17 ө.х. 10 | 16 мо.б. 11 | 13 у.ө.и. 12 | 13 о.т. 13 | 11 а.ж.н. 14 | 10 т. 15 | 10 кит. 16 | 9 мян. 17 | 9 монг. 18 | 9 г. 19 | 8 пом. 20 | 8 кв. 21 | 8 ам.доллар. 22 | 7 респ. 23 | 7 прав. 24 | 6 х.с. 25 | 6 общ. 26 | 6 об. 27 | 6 з.у. 28 | 6 дог. 29 | 6 б. 30 | 5 х.ө. 31 | 5 стор. 32 | 5 соглаш. 33 | 5 согл. 34 | 5 ж.нь. 35 | 5 а. 36 | 4 а.е. 37 | 3 тэрбум.ам. 38 | 3 т.д. 39 | 3 м.х. 40 | 3 м.к. 41 | 3 г.т. 42 | 2 х.м. 43 | 2 х.д. 44 | 2 хав.дөр. 45 | 2 с.ч. 46 | 2 мян.км. 47 | 2 м.э. 48 | 2 ж.н. 49 | 2 г.г. 50 | 2 б.м. 51 | 2 а.ха. 52 | 2 а.с. 53 | 2 ам.дөр. 54 | 1 ц.с. 55 | 1 хавт.дөр. 56 | 1 улс.байжээ. 57 | 1 т.цаг. 58 | 1 сам.бөм. 59 | 1 ө.и. 60 | 1 отк.ном. 61 | 1 м.у. 62 | 1 м.а. 63 | 1 км.кв. 64 | 1 кв.км. 65 | 1 г.ж. 66 | 1 ам.км. 67 | 1 ам.долл. 68 | 1 ам.дол. 69 | -------------------------------------------------------------------------------- /cvutils/data/mn/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгдеёжзийклмноөпрстуүфхцчшщыьэюяъ 2 | -------------------------------------------------------------------------------- /cvutils/data/mn/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | а a 3 | б p 4 | б pʲ 5 | в w̜ 6 | в w̜ʲ 7 | г ɡ 8 | г ɡʲ 9 | г ɢ 10 | д t 11 | д tʲ 12 | е jɵ 13 | е ji 14 | ё jɔ 15 | ж tʃ 16 | з ts 17 | и i 18 | й i 19 | к kʰ 20 | к kʲʰ 21 | к x 22 | к xʲ 23 | л ɮ 24 | л ɮʲ 25 | м m 26 | м mʲ 27 | н n 28 | н nʲ 29 | н ŋ 30 | о ɔ 31 | ө ɵ 32 | ө o 33 | п pʰ 34 | п pʰʲ 35 | р r 36 | р rʲ 37 | с s 38 | т tʰ 39 | т tʰʲ 40 | у ʊ 41 | ү u 42 | ф f 43 | ф pʰ 44 | х x 45 | х xʲ 46 | ц tsʰ 47 | ч tʃʰ 48 | ш ʃ 49 | щ ʃ 50 | ы i 51 | ь ʲ 52 | э e 53 | э i 54 | ю jʊ 55 | ю ju 56 | я ja 57 | -------------------------------------------------------------------------------- /cvutils/data/mn/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/mn/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | REPL ! _ 3 | REPL " _ 4 | REPL ' _ 5 | REPL , _ 6 | REPL - _ 7 | REPL . _ 8 | REPL : _ 9 | REPL ? _ 10 | REPL « _ 11 | REPL » _ 12 | REPL ” _ 13 | REPL “ _ 14 | ALLOW а _ 15 | ALLOW б _ 16 | ALLOW в _ 17 | ALLOW г _ 18 | ALLOW д _ 19 | ALLOW е _ 20 | ALLOW ё _ 21 | ALLOW ж _ 22 | ALLOW з _ 23 | ALLOW и _ 24 | ALLOW й _ 25 | ALLOW к _ 26 | ALLOW л _ 27 | ALLOW м _ 28 | ALLOW н _ 29 | ALLOW о _ 30 | ALLOW ө _ 31 | ALLOW п _ 32 | ALLOW р _ 33 | ALLOW с _ 34 | ALLOW т _ 35 | ALLOW у _ 36 | ALLOW ү _ 37 | ALLOW ф _ 38 | ALLOW х _ 39 | ALLOW ц _ 40 | ALLOW ч _ 41 | ALLOW ш _ 42 | ALLOW щ _ 43 | ALLOW ы _ 44 | ALLOW ь _ 45 | ALLOW э _ 46 | ALLOW ю _ 47 | ALLOW я _ 48 | ALLOW ъ _ 49 | ALLOW _ _ 50 | -------------------------------------------------------------------------------- /cvutils/data/mr/alphabet.txt: -------------------------------------------------------------------------------- 1 | ऄअआइईउऊऋऌऍऎएऐऑॲऒओऔकखगघङचछजझञटठडढणतथदधनऩपफबभमयरऱलळऴवशषसहऻऺऽािीुूृॄॅँॆेैॉॊोौ्ॎॐ॓॔क़ख़ग़ज़ड़ढ़फ़य़ॠॡॢॣंः‍ 2 | -------------------------------------------------------------------------------- /cvutils/data/mrj/alphabet.txt: -------------------------------------------------------------------------------- 1 | аӓбвгдеёжзийклмнҥоӧпрстуӱфхцчшщъыӹьэюя 2 | -------------------------------------------------------------------------------- /cvutils/data/mrj/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL , _ 002c _ 5 | REPL - _ 002d _ 6 | REPL . _ 002e _ 7 | REPL : _ 003a _ 8 | REPL ; _ 003b _ 9 | REPL ? _ 003f _ 10 | REPL « _ 00ab _ 11 | REPL » _ 00bb _ 12 | REPL — _ 2014 _ 13 | REPL – _ 2013 _ 14 | REPL … _ 2026 _ 15 | ALLOW а _ 0430 _ 16 | ALLOW б _ 0431 _ 17 | ALLOW в _ 0432 _ 18 | ALLOW г _ 0433 _ 19 | ALLOW д _ 0434 _ 20 | ALLOW е _ 0435 _ 21 | ALLOW ё _ 0451 _ 22 | ALLOW ж _ 0436 _ 23 | ALLOW з _ 0437 _ 24 | ALLOW и _ 0438 _ 25 | ALLOW й _ 0439 _ 26 | ALLOW к _ 043a _ 27 | ALLOW л _ 043b _ 28 | ALLOW м _ 043c _ 29 | ALLOW н _ 043d _ 30 | ALLOW о _ 043e _ 31 | ALLOW ӧ _ 04e7 _ 32 | ALLOW п _ 043f _ 33 | ALLOW р _ 0440 _ 34 | ALLOW с _ 0441 _ 35 | ALLOW т _ 0442 _ 36 | ALLOW у _ 0443 _ 37 | ALLOW ӱ _ 04f1 _ 38 | ALLOW ф _ 0444 _ 39 | ALLOW х _ 0445 _ 40 | ALLOW ц _ 0446 _ 41 | ALLOW ч _ 0447 _ 42 | ALLOW ш _ 0448 _ 43 | ALLOW щ _ 0449 _ 44 | ALLOW ъ _ 044a _ 45 | ALLOW ы _ 044b _ 46 | ALLOW ӹ _ 04f9 _ 47 | ALLOW ь _ 044c _ 48 | ALLOW э _ 044d _ 49 | ALLOW ю _ 044e _ 50 | ALLOW я _ 044f _ 51 | ALLOW ӓ _ 04d3 _ 52 | ALLOW ҥ _ 04a5 _ 53 | ALLOW _ _ 0020 _ 54 | -------------------------------------------------------------------------------- /cvutils/data/mt/abbr.tsv: -------------------------------------------------------------------------------- 1 | 89 A. 2 | 51 J. 3 | 35 P. 4 | 28 B. 5 | 25 L. 6 | 24 C. 7 | 22 K. 8 | 21 S. 9 | 21 F. 10 | 19 M. 11 | 18 W. 12 | 18 H. 13 | 17 G. 14 | 17 E. 15 | 16 V. 16 | 16 R. 17 | 13 D. 18 | 11 I. 19 | 9 p.m. 20 | 9 p. 21 | 8 m. 22 | 8 c. 23 | 7 T. 24 | 7 N. 25 | 6 v. 26 | 6 a.m. 27 | 6 a. 28 | 4 t. 29 | 4 s. 30 | 4 Ċ. 31 | 4 ċ. 32 | 3 X. 33 | 2 Z. 34 | 2 r. 35 | 2 n. 36 | 1 z. 37 | 1 Y. 38 | 1 v.v. 39 | 1 v.k. 40 | 1 U. 41 | 1 R.S. 42 | 1 p.p. 43 | 1 Ph.D. 44 | 1 O. 45 | 1 Nru. 46 | 1 M.A. 47 | 1 J.R.R. 48 | 1 ħ. 49 | 1 Ġ. 50 | 1 g. 51 | 1 f. 52 | 1 d. 53 | -------------------------------------------------------------------------------- /cvutils/data/mt/alphabet.txt: -------------------------------------------------------------------------------- 1 | aàbcċdeèfgġhħiìjklmnoòpqrstuùvwxyzż-' 2 | -------------------------------------------------------------------------------- /cvutils/data/mt/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a ɐ 3 | a aː 4 | a æː 5 | b b 6 | ċ t͡ʃ 7 | d d 8 | e eː 9 | e ɛ 10 | e øː 11 | e ə 12 | f f 13 | ġ d͡ʒ 14 | g ɡ 15 | għ ˤː 16 | għ ħː 17 | h _ 18 | ħ ħ 19 | i i̞ː 20 | i iː 21 | i ɪ 22 | ie ɛː 23 | ie iːᵊ 24 | j j 25 | k k 26 | l l 27 | m m 28 | n n 29 | o o 30 | o ɔ 31 | o ɒ 32 | p p 33 | q ʔ 34 | r r 35 | r ɹ 36 | s s 37 | t t 38 | u u 39 | u ʉ 40 | u ʊ 41 | v v 42 | w w 43 | x ʃ 44 | x ʒ 45 | ż z 46 | z t͡s 47 | z d͡z 48 | -------------------------------------------------------------------------------- /cvutils/data/mt/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/mt/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | REPL ! _ 3 | REPL " _ 4 | REPL , _ 5 | REPL . _ 6 | REPL : _ 7 | REPL ; _ 8 | REPL ? _ 9 | REPL ’ ' 10 | NORM á a 11 | NORM é e 12 | NORM ć ċ 13 | NORM a` à 14 | ALLOW à _ 15 | ALLOW è _ 16 | ALLOW ì _ 17 | ALLOW ò _ 18 | ALLOW ù _ 19 | ALLOW a _ 20 | ALLOW b _ 21 | ALLOW c _ 22 | ALLOW ċ _ 23 | ALLOW d _ 24 | ALLOW e _ 25 | ALLOW f _ 26 | ALLOW g _ 27 | ALLOW ġ _ 28 | ALLOW h _ 29 | ALLOW ħ _ 30 | ALLOW i _ 31 | ALLOW j _ 32 | ALLOW k _ 33 | ALLOW l _ 34 | ALLOW m _ 35 | ALLOW n _ 36 | ALLOW o _ 37 | ALLOW p _ 38 | ALLOW q _ 39 | ALLOW r _ 40 | ALLOW s _ 41 | ALLOW t _ 42 | ALLOW u _ 43 | ALLOW v _ 44 | ALLOW w _ 45 | ALLOW x _ 46 | ALLOW y _ 47 | ALLOW z _ 48 | ALLOW ż _ 49 | ALLOW - _ 50 | ALLOW ' _ 51 | ALLOW _ _ 52 | -------------------------------------------------------------------------------- /cvutils/data/myv/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгдеёжзийклмнопрстуфхцчшщъыьэюя 2 | -------------------------------------------------------------------------------- /cvutils/data/myv/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | REPL ! _ 0021 _ 3 | REPL , _ 002c _ 4 | REPL - _ 002d _ 5 | REPL . _ 002e _ 6 | REPL : _ 003a _ 7 | REPL ; _ 003b _ 8 | REPL ? _ 003f _ 9 | REPL « _ 00ab _ 10 | REPL » _ 00bb _ 11 | REPL — _ 2014 _ 12 | REPL • _ 2022 _ 13 | REPL ─ _ 2500 _ 14 | ALLOW а _ 0430 _ 15 | ALLOW б _ 0431 _ 16 | ALLOW в _ 0432 _ 17 | ALLOW г _ 0433 _ 18 | ALLOW д _ 0434 _ 19 | ALLOW е _ 0435 _ 20 | ALLOW ё _ 0451 _ 21 | ALLOW ж _ 0436 _ 22 | ALLOW з _ 0437 _ 23 | ALLOW и _ 0438 _ 24 | ALLOW й _ 0439 _ 25 | ALLOW к _ 043a _ 26 | ALLOW л _ 043b _ 27 | ALLOW м _ 043c _ 28 | ALLOW н _ 043d _ 29 | ALLOW о _ 043e _ 30 | ALLOW п _ 043f _ 31 | ALLOW р _ 0440 _ 32 | ALLOW с _ 0441 _ 33 | ALLOW т _ 0442 _ 34 | ALLOW у _ 0443 _ 35 | ALLOW ф _ 0444 _ 36 | ALLOW х _ 0445 _ 37 | ALLOW ц _ 0446 _ 38 | ALLOW ч _ 0447 _ 39 | ALLOW ш _ 0448 _ 40 | ALLOW щ _ 0449 _ 41 | ALLOW ъ _ 044a _ 42 | ALLOW ы _ 044b _ 43 | ALLOW ь _ 044c _ 44 | ALLOW э _ 044d _ 45 | ALLOW ю _ 044e _ 46 | ALLOW я _ 044f _ 47 | ALLOW _ _ 0020 _ 48 | -------------------------------------------------------------------------------- /cvutils/data/ne-NP/alphabet.txt: -------------------------------------------------------------------------------- 1 | अआइईउऊएऐओऔकखगघङचछजझञटठडड़ढणतथदधनपफबभमयरलवशषसहािीुूृेैोौ्ँं़ 2 | -------------------------------------------------------------------------------- /cvutils/data/ne-NP/validate.tsv: -------------------------------------------------------------------------------- 1 | NFKC _ _ _ _ 2 | REPL ! _ 0021 _ 3 | REPL " _ 0022 _ 4 | REPL , _ 002c _ 5 | REPL - _ 002d _ 6 | REPL . _ 002e _ 7 | REPL : _ 003a _ 8 | REPL ? _ 003f _ 9 | REPL । _ 0964 _ 10 | REPL – _ 2013 _ 11 | REPL ‘ _ 2018 _ 12 | REPL ’ _ 2019 _ 13 | SKIP ० _ 0966 _ 14 | SKIP १ _ 0967 _ 15 | SKIP २ _ 0968 _ 16 | SKIP ३ _ 0969 _ 17 | SKIP ५ _ 096b _ 18 | SKIP ६ _ 096c _ 19 | SKIP ८ _ 096e _ 20 | SKIP ९ _ 096f _ 21 | SKIP 😂 _ 1f602 _ 22 | ALLOW अ _ 0905 _ 23 | ALLOW आ _ 0906 _ 24 | ALLOW इ _ 0907 _ 25 | ALLOW ई _ 0908 _ 26 | ALLOW उ _ 0909 _ 27 | ALLOW ऊ _ 090a _ 28 | ALLOW ए _ 090f _ 29 | ALLOW ऐ _ 0910 _ 30 | ALLOW ओ _ 0913 _ 31 | ALLOW औ _ 0914 _ 32 | ALLOW क _ 0915 _ 33 | ALLOW ख _ 0916 _ 34 | ALLOW ग _ 0917 _ 35 | ALLOW घ _ 0918 _ 36 | ALLOW ङ _ 0919 _ 37 | ALLOW च _ 091a _ 38 | ALLOW छ _ 091b _ 39 | ALLOW ज _ 091c _ 40 | ALLOW झ _ 091d _ 41 | ALLOW ञ _ 091e _ 42 | ALLOW ट _ 091f _ 43 | ALLOW ठ _ 0920 _ 44 | ALLOW ड _ 0921 _ 45 | ALLOW ड़ _ 095c _ 46 | ALLOW ढ _ 0922 _ 47 | ALLOW ण _ 0923 _ 48 | ALLOW त _ 0924 _ 49 | ALLOW थ _ 0925 _ 50 | ALLOW द _ 0926 _ 51 | ALLOW ध _ 0927 _ 52 | ALLOW न _ 0928 _ 53 | ALLOW प _ 092a _ 54 | ALLOW फ _ 092b _ 55 | ALLOW ब _ 092c _ 56 | ALLOW भ _ 092d _ 57 | ALLOW म _ 092e _ 58 | ALLOW य _ 092f _ 59 | ALLOW र _ 0930 _ 60 | ALLOW ल _ 0932 _ 61 | ALLOW व _ 0935 _ 62 | ALLOW श _ 0936 _ 63 | ALLOW ष _ 0937 _ 64 | ALLOW स _ 0938 _ 65 | ALLOW ह _ 0939 _ 66 | ALLOW ा _ 093e _ 67 | ALLOW ि _ 093f _ 68 | ALLOW ी _ 0940 _ 69 | ALLOW ु _ 0941 _ 70 | ALLOW ू _ 0942 _ 71 | ALLOW ृ _ 0943 _ 72 | ALLOW े _ 0947 _ 73 | ALLOW ै _ 0948 _ 74 | ALLOW ो _ 094b _ 75 | ALLOW ौ _ 094c _ 76 | ALLOW ् _ 094d _ 77 | ALLOW ँ _ 0901 _ 78 | ALLOW ं _ 0902 _ 79 | ALLOW ः _ 0903 _ 80 | ALLOW ‍ _ 200d _ 81 | ALLOW _ _ 0020 _ 82 | -------------------------------------------------------------------------------- /cvutils/data/nl/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnoprstuvwz 2 | -------------------------------------------------------------------------------- /cvutils/data/nl/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | b b 3 | b p 4 | ch x 5 | d d 6 | d t 7 | f f 8 | g ɣ 9 | g x 10 | h ɦ 11 | j j 12 | k k 13 | l l 14 | m m 15 | n n 16 | ng ŋ 17 | p p 18 | r r 19 | s s 20 | sch sx 21 | sch s 22 | t t 23 | th t 24 | v v 25 | w ʋ 26 | z z 27 | a ɑ 28 | a aː 29 | aa aː 30 | aai aːi̯ 31 | ai ɑi̯ 32 | au ɑu̯ 33 | auw ɑu̯ 34 | e ɛ 35 | e ə 36 | e eː 37 | ee eː 38 | eeuw eːu̯ 39 | ei ɛi̯ 40 | eu øː 41 | i ɪ 42 | i ə 43 | i i 44 | i iː 45 | ie iː 46 | ie i 47 | ieuw iu̯ 48 | ij ɛi̯ 49 | ij i 50 | ij iː 51 | ij ə 52 | o ɔ 53 | o oː 54 | oe u 55 | oe uː 56 | oei ui̯ 57 | oi ɔi̯ 58 | oo oː 59 | ooi oːi̯ 60 | ou ɑu̯ 61 | ouw ɑu̯ 62 | u ʏ 63 | u y 64 | u yː 65 | ui œy̯ 66 | uu y 67 | uu yː 68 | uw yu̯ 69 | -------------------------------------------------------------------------------- /cvutils/data/nl/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL & _ 0026 _ 6 | REPL ' _ 0027 _ 7 | REPL , _ 002c _ 8 | REPL - _ 002d _ 9 | REPL . _ 002e _ 10 | REPL : _ 003a _ 11 | REPL ; _ 003b _ 12 | REPL = _ 003d _ 13 | REPL ? _ 003f _ 14 | REPL ´ _ 00b4 _ 15 | REPL – _ 2013 _ 16 | REPL — _ 2014 _ 17 | REPL ’ _ 2019 _ 18 | REPL “ _ 201c _ 19 | REPL ” _ 201d _ 20 | REPL … _ 2026 _ 21 | SKIP ( _ 0028 _ 22 | SKIP ) _ 0029 _ 23 | NORM ç c 00e7 0063 24 | NORM ê e 00ea 0065 25 | NORM è e 00e8 0065 26 | NORM î i 00ee 0069 27 | NORM í i 00ee 0069 28 | NORM ú u 00fa 0075 29 | NORM û u 00fb 0075 30 | ALLOW a _ 0061 _ 31 | ALLOW á _ 00e1 _ 32 | ALLOW à _ 00e0 _ 33 | ALLOW b _ 0062 _ 34 | ALLOW c _ 0063 _ 35 | ALLOW d _ 0064 _ 36 | ALLOW e _ 0065 _ 37 | ALLOW é _ 00e9 _ 38 | ALLOW ë _ 00eb _ 39 | ALLOW f _ 0066 _ 40 | ALLOW g _ 0067 _ 41 | ALLOW h _ 0068 _ 42 | ALLOW i _ 0069 _ 43 | ALLOW ï _ 00ef _ 44 | ALLOW j _ 006a _ 45 | ALLOW k _ 006b _ 46 | ALLOW l _ 006c _ 47 | ALLOW m _ 006d _ 48 | ALLOW n _ 006e _ 49 | ALLOW o _ 006f _ 50 | ALLOW ó _ 00f3 _ 51 | ALLOW ö _ 00f6 _ 52 | ALLOW p _ 0070 _ 53 | ALLOW q _ 0071 _ 54 | ALLOW r _ 0072 _ 55 | ALLOW s _ 0073 _ 56 | ALLOW t _ 0074 _ 57 | ALLOW u _ 0075 _ 58 | ALLOW ü _ 00fc _ 59 | ALLOW v _ 0076 _ 60 | ALLOW w _ 0077 _ 61 | ALLOW x _ 0078 _ 62 | ALLOW y _ 0079 _ 63 | ALLOW z _ 007a _ 64 | ALLOW _ _ 0020 _ 65 | -------------------------------------------------------------------------------- /cvutils/data/nl/vocab.tsv: -------------------------------------------------------------------------------- 1 | nul 2 | één 3 | twee 4 | drie 5 | vier 6 | vijf 7 | zes 8 | zeven 9 | acht 10 | negen 11 | ja 12 | nee 13 | -------------------------------------------------------------------------------- /cvutils/data/nn-NO/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnopqrstuvwxyzæåøéèêóòâô 2 | -------------------------------------------------------------------------------- /cvutils/data/nn-NO/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL . _ _ _ 4 | REPL , _ _ _ 5 | REPL ! _ _ _ 6 | REPL ? _ _ _ 7 | REPL - _ _ _ 8 | REPL â a 00e2 0061 9 | REPL é e 00e9 0065 10 | REPL è e 00e8 0065 11 | REPL ê e 00ea 0065 12 | REPL ó o 00f3 006f 13 | REPL ò o 00f2 006f 14 | REPL ô o 00f4 006f 15 | ALLOW a _ 0061 _ 16 | ALLOW b _ 0062 _ 17 | ALLOW c _ 0063 _ 18 | ALLOW d _ 0064 _ 19 | ALLOW e _ 0065 _ 20 | ALLOW f _ 0066 _ 21 | ALLOW g _ 0067 _ 22 | ALLOW h _ 0068 _ 23 | ALLOW i _ 0069 _ 24 | ALLOW j _ 006a _ 25 | ALLOW k _ 006b _ 26 | ALLOW l _ 006c _ 27 | ALLOW m _ 006d _ 28 | ALLOW n _ 006e _ 29 | ALLOW o _ 006f _ 30 | ALLOW p _ 0070 _ 31 | ALLOW q _ 0071 _ 32 | ALLOW r _ 0072 _ 33 | ALLOW s _ 0073 _ 34 | ALLOW t _ 0074 _ 35 | ALLOW u _ 0075 _ 36 | ALLOW v _ 0076 _ 37 | ALLOW w _ 0077 _ 38 | ALLOW x _ 0078 _ 39 | ALLOW y _ 0079 _ 40 | ALLOW z _ 007a _ 41 | ALLOW æ _ 00e6 _ 42 | ALLOW å _ 00e5 _ 43 | ALLOW ø _ 00f8 _ 44 | ALLOW _ _ 0020 _ 45 | -------------------------------------------------------------------------------- /cvutils/data/nr/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklnopqrstuvwxyz -------------------------------------------------------------------------------- /cvutils/data/nr/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | i i 3 | u u 4 | e e 5 | e ɛ 6 | o o 7 | o ɔ 8 | a a 9 | p pʼ 10 | k kʼ 11 | ph pʰ 12 | h tʰ 13 | kh kʰ 14 | bh b̥ 15 | d d̥ 16 | ɡ ɡ̊ 17 | mp ᵐp 18 | nk ᵑk 19 | mb ᵐb 20 | nd ⁿd 21 | ng ᵑɡ 22 | b ɓ 23 | s tsʼ 24 | l tɬʼ 25 | j tʃʼ 26 | kg kxʼ 27 | sh tsʰ 28 | lh tɬʰ 29 | jh tʃʰ 30 | kgh kxʰ 31 | dz dz 32 | j d̥ʒ 33 | nj ᶮdʒ 34 | f f 35 | s s 36 | hl ɬ 37 | rh x 38 | v v 39 | z z 40 | dl ɮ 41 | h ɦ 42 | mf ᶬf 43 | mv ᶬv 44 | dlh ɮʰ 45 | m m 46 | n n 47 | ny ɲ 48 | ngh ŋ 49 | r r 50 | l l 51 | w w 52 | y j 53 | c ᵏǀ 54 | q ᵏ! 55 | x ᵏǁ 56 | ch ᵏǀʰ 57 | qh ᵏ!ʰ 58 | gc ᶢǀ 59 | gq ᶢ! 60 | nc ᵑǀ 61 | nq ᵑ! 62 | nx ᵑǁ 63 | -------------------------------------------------------------------------------- /cvutils/data/nso/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnoprsštuwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/nso/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NKFC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL , _ 002c _ 5 | REPL - _ 002d _ 6 | REPL . _ 002e _ 7 | REPL ? _ 003f _ 8 | ALLOW a _ 0061 _ 9 | ALLOW b _ 0062 _ 10 | ALLOW c _ 0063 _ 11 | ALLOW d _ 0064 _ 12 | ALLOW e _ 0065 _ 13 | ALLOW f _ 0066 _ 14 | ALLOW g _ 0067 _ 15 | ALLOW h _ 0068 _ 16 | ALLOW i _ 0069 _ 17 | ALLOW j _ 006a _ 18 | ALLOW k _ 006b _ 19 | ALLOW l _ 006c _ 20 | ALLOW m _ 006d _ 21 | ALLOW n _ 006e _ 22 | ALLOW o _ 006f _ 23 | ALLOW p _ 0070 _ 24 | ALLOW r _ 0072 _ 25 | ALLOW s _ 0073 _ 26 | ALLOW š _ 0161 _ 27 | ALLOW t _ 0074 _ 28 | ALLOW u _ 0075 _ 29 | ALLOW w _ 0077 _ 30 | ALLOW x _ 0078 _ 31 | ALLOW y _ 0079 _ 32 | ALLOW z _ 007a _ 33 | ALLOW _ 0020 _ 34 | -------------------------------------------------------------------------------- /cvutils/data/ny/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnoprstuvwŵyz' 2 | -------------------------------------------------------------------------------- /cvutils/data/ny/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | e ɛ 4 | i i 5 | o ɔ 6 | u u 7 | b ɓ 8 | p p 9 | ph pʰ 10 | mb ᵐb 11 | mph ᵐpʰ 12 | m m 13 | ŵa β 14 | bw ɓʷ 15 | pw pʷ 16 | phw pʷʰ 17 | mbw ᵐbʷ 18 | mphw ᵐpʷʰ 19 | mw mʷ 20 | w w 21 | bz bʒʲ 22 | py pʲ 23 | ps pʃʲ 24 | mbz ᵐbzʲ 25 | mps ᵐpsʲ 26 | my mʲ 27 | d ɗ 28 | t t 29 | th tʰ 30 | nd ⁿd 31 | nth ⁿtʰ 32 | n n 33 | l ɽ 34 | r ɽ 35 | dw ɗʷ 36 | tw tʷ 37 | thw tʷʰ 38 | ndw ⁿdʷ 39 | nthw ⁿtʷʰ 40 | lw ɽʷ 41 | rw ɽʷ 42 | dy ɗʲ 43 | ty tʲ 44 | thy tʲʰ 45 | ndy ⁿdʲ 46 | nthy ⁿtʲʰ 47 | ny nʲ 48 | g g 49 | k k 50 | kh kʰ 51 | ng ᵑg 52 | nkh ᵑkʰ 53 | ng' ŋ 54 | gw gʷ 55 | kw kʷ 56 | khw kʷʰ 57 | ngw ᵑgʷ 58 | nkhw ᵑkʷʰ 59 | ng'w ŋʷ 60 | j d͡ʒ 61 | ch t͡ʃ 62 | tch t͡ʃʰ 63 | nj ⁿd͡ʒ 64 | ntch ⁿt͡ʃʰ 65 | y j 66 | v v 67 | f f 68 | mv ᶬv 69 | mf ᶬf 70 | vw vʷ 71 | fw fʷ 72 | vya vʲ 73 | fya fʲ 74 | z z 75 | s s 76 | nz ⁿz 77 | ns ⁿs 78 | zw zʷ 79 | sw sʷ 80 | nzw ⁿzʷ 81 | nsw ⁿsʷ 82 | zya zʲ~ʒ 83 | sh ʃ 84 | dz d͡z 85 | ts t͡s 86 | ndza ⁿd͡z 87 | dzwe d͡zʷ 88 | tsw t͡sʷ 89 | ha h 90 | -------------------------------------------------------------------------------- /cvutils/data/ny/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | DEL   _ 2006 _ 4 | DEL ­ _ 00ad _ 5 | REPL ʼ _ 02bc _ 6 | REPL ! _ 0021 _ 7 | REPL , _ 002c _ 8 | REPL - _ 002d _ 9 | REPL . _ 002e _ 10 | REPL : _ 003a _ 11 | REPL ; _ 003b _ 12 | REPL — _ 2014 _ 13 | REPL ‘ _ 2018 _ 14 | REPL ’ _ 2019 _ 15 | REPL “ _ 201c _ 16 | REPL ” _ 201d _ 17 | REPL ? _ 003f _ 18 | SKIP ( _ 0028 _ 19 | SKIP ) _ 0029 _ 20 | SKIP * _ 002a _ 21 | SKIP [ _ 005b _ 22 | SKIP \ _ 005c _ 23 | SKIP ] _ 005d _ 24 | SKIP 0 _ 0030 _ 25 | SKIP 1 _ 0031 _ 26 | SKIP 2 _ 0032 _ 27 | SKIP 3 _ 0033 _ 28 | SKIP 4 _ 0034 _ 29 | SKIP 5 _ 0035 _ 30 | SKIP 6 _ 0036 _ 31 | SKIP 7 _ 0037 _ 32 | SKIP 8 _ 0038 _ 33 | SKIP 9 _ 0039 _ 34 | ALLOW a _ 0061 _ 35 | ALLOW b _ 0062 _ 36 | ALLOW c _ 0063 _ 37 | ALLOW d _ 0064 _ 38 | ALLOW e _ 0065 _ 39 | ALLOW f _ 0066 _ 40 | ALLOW g _ 0067 _ 41 | ALLOW h _ 0068 _ 42 | ALLOW i _ 0069 _ 43 | ALLOW j _ 006a _ 44 | ALLOW k _ 006b _ 45 | ALLOW l _ 006c _ 46 | ALLOW m _ 006d _ 47 | ALLOW n _ 006e _ 48 | ALLOW o _ 006f _ 49 | ALLOW p _ 0070 _ 50 | ALLOW r _ 0072 _ 51 | ALLOW s _ 0073 _ 52 | ALLOW t _ 0074 _ 53 | ALLOW u _ 0075 _ 54 | ALLOW v _ 0076 _ 55 | ALLOW w _ 0077 _ 56 | ALLOW y _ 0079 _ 57 | ALLOW z _ 007a _ 58 | ALLOW _ _ 0020 _ 59 | -------------------------------------------------------------------------------- /cvutils/data/oc/alphabet.txt: -------------------------------------------------------------------------------- 1 | aàbcdeéèfghiíjklmnoóòpqrstuúvwxyzçïüùá' 2 | -------------------------------------------------------------------------------- /cvutils/data/oc/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL , _ 002c _ 6 | REPL . _ 002e _ 7 | REPL / _ 002f _ 8 | REPL : _ 003a _ 9 | REPL ; _ 003b _ 10 | REPL ? _ 003f _ 11 | REPL ~ _ 007e _ 12 | REPL ¡ _ 00a1 _ 13 | REPL « _ 00ab _ 14 | REPL » _ 00bb _ 15 | REPL ¿ _ 00bf _ 16 | REPL – _ 2013 _ 17 | REPL — _ 2014 _ 18 | REPL ― _ 2015 _ 19 | REPL ’ ' 2019 _ 20 | REPL “ _ 201c _ 21 | REPL ” _ 201d _ 22 | REPL … _ 2026 _ 23 | SKIP ( _ 0028 _ 24 | SKIP ) _ 0029 _ 25 | SKIP @ _ 0040 _ 26 | ALLOW ' _ _ _ 27 | ALLOW a _ 0061 _ 28 | ALLOW à _ 00e0 _ 29 | ALLOW b _ 0062 _ 30 | ALLOW c _ 0063 _ 31 | ALLOW d _ 0064 _ 32 | ALLOW e _ 0065 _ 33 | ALLOW é _ 00e9 _ 34 | ALLOW è _ 00e8 _ 35 | ALLOW f _ 0066 _ 36 | ALLOW g _ 0067 _ 37 | ALLOW h _ 0068 _ 38 | ALLOW i _ 0069 _ 39 | ALLOW í _ 00ed _ 40 | ALLOW j _ 006a _ 41 | ALLOW k _ 006b _ 42 | ALLOW l _ 006c _ 43 | ALLOW m _ 006d _ 44 | ALLOW n _ 006e _ 45 | ALLOW o _ 006f _ 46 | ALLOW ó _ 00f3 _ 47 | ALLOW ò _ 00f2 _ 48 | ALLOW p _ 0070 _ 49 | ALLOW q _ 0071 _ 50 | ALLOW r _ 0072 _ 51 | ALLOW s _ 0073 _ 52 | ALLOW t _ 0074 _ 53 | ALLOW u _ 0075 _ 54 | ALLOW ú _ 00fa _ 55 | ALLOW v _ 0076 _ 56 | ALLOW w _ 0077 _ 57 | ALLOW x _ 0078 _ 58 | ALLOW y _ 0079 _ 59 | ALLOW z _ 007a _ 60 | ALLOW ç _ _ _ 61 | ALLOW ï _ _ _ 62 | ALLOW á _ _ _ 63 | ALLOW ü _ _ _ 64 | ALLOW ù _ _ _ 65 | -------------------------------------------------------------------------------- /cvutils/data/or/abbr.tsv: -------------------------------------------------------------------------------- 1 | 1 _ 2 | -------------------------------------------------------------------------------- /cvutils/data/or/alphabet.txt: -------------------------------------------------------------------------------- 1 | ୕୰୲୳୴୵୶୷଼ଁଂଃଅଆଇଈଉଊଋୠଌୡଏଐଓଔକଖଗଘଙଚଛଜଝଞଟଠଡଡ଼ଢଢ଼ଣତଥଦଧନପଫବଭମଯୟରଲଳଵୱଶଷସହଽାିୀୁୂୃୄୢୣେୈୋୌ୍ୖୗ 2 | -------------------------------------------------------------------------------- /cvutils/data/or/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS । 2 | EOS ! 3 | EOS ? 4 | -------------------------------------------------------------------------------- /cvutils/data/or/vocab.tsv: -------------------------------------------------------------------------------- 1 | ଶୂନ 2 | ଏକ 3 | ଦୁଇ 4 | ତିନି 5 | ଚାରି 6 | ପାଞ୍ଚ 7 | ଛଅ 8 | ସାତ 9 | ଆଠ 10 | ନଅ 11 | ହଁ 12 | ନା 13 | -------------------------------------------------------------------------------- /cvutils/data/pa-IN/alphabet.txt: -------------------------------------------------------------------------------- 1 | ੶਼ਁਂਃੰੱ੦੧੨੩੪੫੬੭੮੯ੴੳਉਊਓਅਆਐਔੲਇਈਏਸਸ਼ਹੑਕਖਖ਼ਗਗ਼ਘਙਚਛਜਜ਼ਝਞਟਠਡਢਣਤਥਦਧਨਪਫਫ਼ਬਭਮਯੵਰਲਲ਼ਵੜਾਿੀੁੂੇੈੋੌ੍ -------------------------------------------------------------------------------- /cvutils/data/pa-IN/validate.tsv: -------------------------------------------------------------------------------- 1 | NFKC _ _ _ _ 2 | REPL " _ 0022 _ 3 | REPL - _ 002d _ 4 | REPL ¦ _ 00a6 _ 5 | REPL – _ 2013 _ 6 | REPL ’ _ 2019 _ 7 | REPL … _ 2026 _ 8 | REPL ¤ _ 00a4 _ 9 | SKIP a _ 0061 _ 10 | SKIP i _ 0069 _ 11 | SKIP n _ 006e _ 12 | SKIP p _ 0070 _ 13 | ALLOW ਼ _ 0a3c _ 14 | ALLOW ਂ _ 0a02 _ 15 | ALLOW ੰ _ 0a70 _ 16 | ALLOW ੱ _ 0a71 _ 17 | ALLOW ੳ _ 0a73 _ 18 | ALLOW ਉ _ 0a09 _ 19 | ALLOW ਊ _ 0a0a _ 20 | ALLOW ਓ _ 0a13 _ 21 | ALLOW ਅ _ 0a05 _ 22 | ALLOW ਆ _ 0a06 _ 23 | ALLOW ਐ _ 0a10 _ 24 | ALLOW ਔ _ 0a14 _ 25 | ALLOW ਇ _ 0a07 _ 26 | ALLOW ਈ _ 0a08 _ 27 | ALLOW ਏ _ 0a0f _ 28 | ALLOW ਸ _ 0a38 _ 29 | ALLOW ਸ਼ _ 0a36 _ 30 | ALLOW ਹ _ 0a39 _ 31 | ALLOW ਕ _ 0a15 _ 32 | ALLOW ਖ _ 0a16 _ 33 | ALLOW ਖ਼ _ 0a59 _ 34 | ALLOW ਗ _ 0a17 _ 35 | ALLOW ਗ਼ _ 0a5a _ 36 | ALLOW ਘ _ 0a18 _ 37 | ALLOW ਚ _ 0a1a _ 38 | ALLOW ਛ _ 0a1b _ 39 | ALLOW ਜ _ 0a1c _ 40 | ALLOW ਜ਼ _ 0a5b _ 41 | ALLOW ਝ _ 0a1d _ 42 | ALLOW ਟ _ 0a1f _ 43 | ALLOW ਠ _ 0a20 _ 44 | ALLOW ਡ _ 0a21 _ 45 | ALLOW ਢ _ 0a22 _ 46 | ALLOW ਣ _ 0a23 _ 47 | ALLOW ਤ _ 0a24 _ 48 | ALLOW ਥ _ 0a25 _ 49 | ALLOW ਦ _ 0a26 _ 50 | ALLOW ਧ _ 0a27 _ 51 | ALLOW ਨ _ 0a28 _ 52 | ALLOW ਪ _ 0a2a _ 53 | ALLOW ਫ _ 0a2b _ 54 | ALLOW ਫ਼ _ 0a5e _ 55 | ALLOW ਬ _ 0a2c _ 56 | ALLOW ਭ _ 0a2d _ 57 | ALLOW ਮ _ 0a2e _ 58 | ALLOW ਯ _ 0a2f _ 59 | ALLOW ਰ _ 0a30 _ 60 | ALLOW ਲ _ 0a32 _ 61 | ALLOW ਲ਼ _ 0a33 _ 62 | ALLOW ਵ _ 0a35 _ 63 | ALLOW ੜ _ 0a5c _ 64 | ALLOW ਾ _ 0a3e _ 65 | ALLOW ਿ _ 0a3f _ 66 | ALLOW ੀ _ 0a40 _ 67 | ALLOW ੁ _ 0a41 _ 68 | ALLOW ੂ _ 0a42 _ 69 | ALLOW ੇ _ 0a47 _ 70 | ALLOW ੈ _ 0a48 _ 71 | ALLOW ੋ _ 0a4b _ 72 | ALLOW ੌ _ 0a4c _ 73 | ALLOW ੍ _ 0a4d _ 74 | ALLOW _ _ _ _ 75 | -------------------------------------------------------------------------------- /cvutils/data/pl/alphabet.txt: -------------------------------------------------------------------------------- 1 | aąbcćdeęfghijklłmnńoópqrsśtuüvwxyzźż 2 | -------------------------------------------------------------------------------- /cvutils/data/pl/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | ą ɔ̃ 4 | b b 5 | b p 6 | c ts 7 | ć tɕ 8 | d d 9 | e ɛ 10 | ę ɛ̃ 11 | f f 12 | g ɡ 13 | g k 14 | h ɣ 15 | i i 16 | i j 17 | j j 18 | k k 19 | l l 20 | ł w 21 | ł ɫ 22 | m m 23 | n n 24 | ń ɲ 25 | o ɔ 26 | ó u 27 | p p 28 | r r 29 | s s 30 | ś ɕ 31 | t t 32 | u u 33 | w v 34 | w f 35 | y ɨ 36 | y ɘ 37 | z z 38 | z s 39 | ź ʑ 40 | ź ɕ 41 | ż ʐ 42 | ż ʂ 43 | ch x 44 | cz tʂ 45 | dz dz 46 | dz ts 47 | dź dʑ 48 | dź tɕ 49 | dż dʐ 50 | dż tʂ 51 | rz ʐ 52 | rz ʂ 53 | sz ʂ 54 | ci tɕ 55 | dzi dʑ 56 | gi ɡʲ 57 | chi xʲ 58 | ki kʲ 59 | ni ɲ 60 | si ɕ 61 | zi ʑ 62 | -------------------------------------------------------------------------------- /cvutils/data/pl/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL . _ 002e _ 4 | REPL ! _ 0021 _ 5 | REPL ? _ 003f _ 6 | REPL " _ 0022 _ 7 | REPL ” _ 201d _ 8 | REPL „ _ 201e _ 9 | REPL , _ 002c _ 10 | REPL : _ 003a _ 11 | REPL ; _ 003b _ 12 | REPL – _ 2013 _ 13 | REPL — _ 2014 _ 14 | REPL - _ 002d _ 15 | REPL ' _ 0027 _ 16 | REPL ’ _ 0027 _ 17 | REPL « _ 00ab _ 18 | REPL » _ 00bb _ 19 | REPL “ _ 201c _ 20 | ALLOW a _ 0061 _ 21 | ALLOW ą _ 0105 _ 22 | ALLOW b _ 0062 _ 23 | ALLOW c _ 0063 _ 24 | ALLOW ć _ 0107 _ 25 | ALLOW d _ 0064 _ 26 | ALLOW e _ 0065 _ 27 | ALLOW ę _ 0119 _ 28 | ALLOW f _ 0066 _ 29 | ALLOW g _ 0067 _ 30 | ALLOW h _ 0068 _ 31 | ALLOW i _ 0069 _ 32 | ALLOW j _ 006a _ 33 | ALLOW k _ 006b _ 34 | ALLOW l _ 006c _ 35 | ALLOW ł _ 0142 _ 36 | ALLOW m _ 006d _ 37 | ALLOW n _ 006e _ 38 | ALLOW ń _ 0144 _ 39 | ALLOW o _ 006f _ 40 | ALLOW ó _ 00f3 _ 41 | ALLOW p _ 0070 _ 42 | ALLOW q _ 0071 _ 43 | ALLOW r _ 0072 _ 44 | ALLOW s _ 0073 _ 45 | ALLOW ś _ 015b _ 46 | ALLOW t _ 0074 _ 47 | ALLOW u _ 0075 _ 48 | ALLOW ü _ 00fc _ 49 | ALLOW v _ 0076 _ 50 | ALLOW w _ 0077 _ 51 | ALLOW x _ 0078 _ 52 | ALLOW y _ 0079 _ 53 | ALLOW z _ 007a _ 54 | ALLOW ź _ 017a _ 55 | ALLOW ż _ 017c _ 56 | ALLOW _ _ 0020 _ 57 | -------------------------------------------------------------------------------- /cvutils/data/pl/vocab.tsv: -------------------------------------------------------------------------------- 1 | zero 2 | jeden 3 | dwa 4 | trzy 5 | cztery 6 | pięć 7 | sześć 8 | siedem 9 | osiem 10 | dziewięć 11 | tak 12 | nie 13 | -------------------------------------------------------------------------------- /cvutils/data/ps/alphabet.txt: -------------------------------------------------------------------------------- 1 | اآبپتټثجچحخڅځدډذرړزژږسشښصضطظعغفقکګلمنڼںوهۀيېیےۍئ -------------------------------------------------------------------------------- /cvutils/data/pt/alphabet.txt: -------------------------------------------------------------------------------- 1 | aáàâãbcçdeéêfghiíjklmnoóôõpqrstuúüvwxyz'- 2 | -------------------------------------------------------------------------------- /cvutils/data/pt/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL " _ 0022 _ 4 | REPL & _ 0026 _ 5 | REPL ! _ 0021 _ 6 | REPL « _ 00ab _ 7 | REPL » _ 00bb _ 8 | REPL , _ 002c _ 9 | REPL . _ 002e _ 10 | REPL : _ 003a _ 11 | REPL ; _ 003b _ 12 | REPL ? _ 003f _ 13 | REPL “ _ 201c _ 14 | REPL ” _ 201d _ 15 | NORM ´ ' 00b4 0027 16 | ALLOW a _ 0061 _ 17 | ALLOW á _ 00e1 _ 18 | ALLOW à _ 00e0 _ 19 | ALLOW â _ 00e2 _ 20 | ALLOW ã _ 00e3 _ 21 | ALLOW b _ 0062 _ 22 | ALLOW c _ 0063 _ 23 | ALLOW ç _ 00e7 _ 24 | ALLOW d _ 0064 _ 25 | ALLOW e _ 0065 _ 26 | ALLOW é _ 00e9 _ 27 | ALLOW ê _ 00ea _ 28 | ALLOW f _ 0066 _ 29 | ALLOW g _ 0067 _ 30 | ALLOW h _ 0068 _ 31 | ALLOW i _ 0069 _ 32 | ALLOW í _ 00ed _ 33 | ALLOW j _ 006a _ 34 | ALLOW k _ 006b _ 35 | ALLOW l _ 006c _ 36 | ALLOW m _ 006d _ 37 | ALLOW n _ 006e _ 38 | ALLOW o _ 006f _ 39 | ALLOW ó _ 00f3 _ 40 | ALLOW ô _ 00f4 _ 41 | ALLOW õ _ 00f5 _ 42 | ALLOW p _ 0070 _ 43 | ALLOW q _ 0071 _ 44 | ALLOW r _ 0072 _ 45 | ALLOW s _ 0073 _ 46 | ALLOW t _ 0074 _ 47 | ALLOW u _ 0075 _ 48 | ALLOW ú _ 00fa _ 49 | ALLOW ü _ 00fc _ 50 | ALLOW v _ 0076 _ 51 | ALLOW w _ 0077 _ 52 | ALLOW x _ 0078 _ 53 | ALLOW y _ 0079 _ 54 | ALLOW z _ 007a _ 55 | ALLOW ' _ 0027 _ 56 | ALLOW - _ 002d _ 57 | ALLOW _ _ 005f _ 58 | -------------------------------------------------------------------------------- /cvutils/data/pt/vocab.tsv: -------------------------------------------------------------------------------- 1 | zero 2 | um 3 | dois 4 | três 5 | quatro 6 | cinco 7 | seis 8 | sete 9 | oito 10 | nove 11 | sim 12 | não 13 | -------------------------------------------------------------------------------- /cvutils/data/quc/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdehijklmnopqrstuwxyzʼ 2 | -------------------------------------------------------------------------------- /cvutils/data/quc/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | m m 3 | bʼ ɓ 4 | p pʰ 5 | w ʋ 6 | n n 7 | tʼ tʼ 8 | t tʰ 9 | tzʼ tsʼ 10 | tz tsʰ 11 | x ʃ 12 | r ɻ 13 | chʼ tʃʼ 14 | ch tʃʰ 15 | j χ 16 | kʼ kʼ 17 | k kʰ 18 | qʼ qʼ 19 | q qʰ 20 | ʼ ʔ 21 | h h 22 | s s 23 | l l 24 | a a 25 | e e 26 | i i 27 | o o 28 | u u 29 | d d 30 | y j 31 | -------------------------------------------------------------------------------- /cvutils/data/quc/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | REPL ! _ 3 | REPL " _ 4 | REPL ( _ 5 | REPL ) _ 6 | REPL * _ 7 | REPL , _ 8 | REPL - _ 9 | REPL . _ 10 | REPL : _ 11 | REPL ; _ 12 | REPL ? _ 13 | REPL ¡ _ 14 | REPL ¿ _ 15 | REPL – _ 16 | REPL “ _ 17 | REPL ” _ 18 | REPL … _ 19 | NORM ' ʼ 20 | NORM ’ ʼ 21 | ALLOW a _ 22 | ALLOW b _ 23 | ALLOW c _ 24 | ALLOW d _ 25 | ALLOW e _ 26 | ALLOW h _ 27 | ALLOW i _ 28 | ALLOW j _ 29 | ALLOW k _ 30 | ALLOW l _ 31 | ALLOW m _ 32 | ALLOW n _ 33 | ALLOW o _ 34 | ALLOW p _ 35 | ALLOW q _ 36 | ALLOW r _ 37 | ALLOW s _ 38 | ALLOW t _ 39 | ALLOW u _ 40 | ALLOW w _ 41 | ALLOW x _ 42 | ALLOW y _ 43 | ALLOW z _ 44 | ALLOW ʼ _ 45 | ALLOW _ _ 46 | -------------------------------------------------------------------------------- /cvutils/data/quc/wikipedia.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ftyers/commonvoice-utils/5e3634fe1832aa58602f551156f7871cc142beb5/cvutils/data/quc/wikipedia.txt -------------------------------------------------------------------------------- /cvutils/data/quy/alphabet.txt: -------------------------------------------------------------------------------- 1 | aiuchklmnñpqrstwyeodgbvfzjx 2 | -------------------------------------------------------------------------------- /cvutils/data/quy/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL " _ 0022 _ 4 | REPL , _ 002c _ 5 | REPL - _ 002d _ 6 | REPL . _ 002e _ 7 | NORM á a _ _ 8 | NORM í i _ _ 9 | NORM ú u _ _ 10 | NORM ó o _ _ 11 | NORM é e _ _ 12 | ALLOW a _ 0061 _ 13 | ALLOW b _ 0062 _ 14 | ALLOW c _ 0063 _ 15 | ALLOW d _ 0064 _ 16 | ALLOW e _ 0065 _ 17 | ALLOW f _ 0066 _ 18 | ALLOW g _ 0067 _ 19 | ALLOW h _ 0068 _ 20 | ALLOW i _ 0069 _ 21 | ALLOW j _ 006a _ 22 | ALLOW k _ 006b _ 23 | ALLOW l _ 006c _ 24 | ALLOW m _ 006d _ 25 | ALLOW n _ 006e _ 26 | ALLOW ñ _ 00f1 _ 27 | ALLOW o _ 006f _ 28 | ALLOW p _ 0070 _ 29 | ALLOW q _ 0071 _ 30 | ALLOW r _ 0072 _ 31 | ALLOW s _ 0073 _ 32 | ALLOW t _ 0074 _ 33 | ALLOW u _ 0075 _ 34 | ALLOW v _ 0076 _ 35 | ALLOW w _ 0077 _ 36 | ALLOW x _ 0078 _ 37 | ALLOW y _ 0079 _ 38 | ALLOW z _ 007a _ 39 | ALLOW _ _ 0020 _ 40 | -------------------------------------------------------------------------------- /cvutils/data/rm-sursilv/abbr.tsv: -------------------------------------------------------------------------------- 1 | 262 e.a. 2 | 211 t.a. 3 | 135 s.m. 4 | 118 m.s.m. 5 | 28 m.s. 6 | 14 q.v.d. 7 | 5 e.u.v. 8 | 5 e.a.v. 9 | 4 v.d. 10 | 4 u.d. 11 | 2 r.a. 12 | 2 q.v. 13 | 2 k.o. 14 | 2 a.s. 15 | 1 t.d.p. 16 | 1 s.u. 17 | 1 s.n.m. 18 | 1 s.l.m. 19 | 1 m.b. 20 | 1 i.a. 21 | 1 h.f. 22 | 1 h.c. 23 | 1 e.u.s. 24 | 1 e.u.i. 25 | 1 d.e.o. 26 | 1 I. 27 | 1 c.f. 28 | 1 cf. 29 | 1 resp. 30 | 1 incl. 31 | 1 m. 32 | 1 s. 33 | 1 etc. 34 | 1 ca. 35 | 1 q. 36 | 1 v. 37 | 1 d. 38 | -------------------------------------------------------------------------------- /cvutils/data/rm-sursilv/alphabet.txt: -------------------------------------------------------------------------------- 1 | aàäbcdeéèfghiîjklmnoòöpqrstuüvwxyz' 2 | -------------------------------------------------------------------------------- /cvutils/data/rm-sursilv/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/rm-sursilv/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL ! _ 4 | REPL " _ 5 | REPL , _ 6 | REPL - _ 7 | REPL . _ 8 | REPL : _ 9 | REPL ; _ 10 | REPL ? _ 11 | REPL « _ 12 | REPL » _ 13 | REPL – _ 14 | REPL ‘ _ 15 | REPL … _ 16 | REPL ‹ _ 17 | REPL › _ 18 | NORM ’ ' 19 | ALLOW a _ 20 | ALLOW à _ 21 | ALLOW ä _ 22 | ALLOW b _ 23 | ALLOW c _ 24 | ALLOW d _ 25 | ALLOW e _ 26 | ALLOW é _ 27 | ALLOW è _ 28 | ALLOW f _ 29 | ALLOW g _ 30 | ALLOW h _ 31 | ALLOW i _ 32 | ALLOW î _ 33 | ALLOW j _ 34 | ALLOW k _ 35 | ALLOW l _ 36 | ALLOW m _ 37 | ALLOW n _ 38 | ALLOW o _ 39 | ALLOW ò _ 40 | ALLOW ö _ 41 | ALLOW p _ 42 | ALLOW q _ 43 | ALLOW r _ 44 | ALLOW s _ 45 | ALLOW t _ 46 | ALLOW u _ 47 | ALLOW ü _ 48 | ALLOW v _ 49 | ALLOW w _ 50 | ALLOW x _ 51 | ALLOW y _ 52 | ALLOW z _ 53 | ALLOW ' _ 54 | ALLOW _ _ 55 | -------------------------------------------------------------------------------- /cvutils/data/rm-sursilv/wikipedia.txt: -------------------------------------------------------------------------------- 1 | rm 2 | -------------------------------------------------------------------------------- /cvutils/data/rm-vallader/abbr.tsv: -------------------------------------------------------------------------------- 1 | 262 e.a. 2 | 211 t.a. 3 | 135 s.m. 4 | 118 m.s.m. 5 | 28 m.s. 6 | 14 q.v.d. 7 | 5 e.u.v. 8 | 5 e.a.v. 9 | 4 v.d. 10 | 4 u.d. 11 | 2 r.a. 12 | 2 q.v. 13 | 2 k.o. 14 | 2 a.s. 15 | 1 t.d.p. 16 | 1 s.u. 17 | 1 s.n.m. 18 | 1 s.l.m. 19 | 1 m.b. 20 | 1 i.a. 21 | 1 h.f. 22 | 1 h.c. 23 | 1 e.u.s. 24 | 1 e.u.i. 25 | 1 d.e.o. 26 | 1 I. 27 | 1 c.f. 28 | 1 cf. 29 | 1 resp. 30 | 1 incl. 31 | 1 m. 32 | 1 s. 33 | 1 etc. 34 | 1 ca. 35 | 1 q. 36 | 1 v. 37 | 1 d. 38 | -------------------------------------------------------------------------------- /cvutils/data/rm-vallader/alphabet.txt: -------------------------------------------------------------------------------- 1 | aàâäbcçdeéèêfghiìïjklmnoòöpqrstuüvwxyz' 2 | -------------------------------------------------------------------------------- /cvutils/data/rm-vallader/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/rm-vallader/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL " _ 0022 _ 4 | REPL , _ 002c _ 5 | REPL ! _ 0021 _ 6 | REPL - _ 002d _ 7 | REPL . _ 002e _ 8 | REPL : _ 003a _ 9 | REPL ; _ 003b _ 10 | REPL ? _ 003f _ 11 | REPL « _ 00ab _ 12 | REPL » _ 00bb _ 13 | REPL – _ 2013 _ 14 | REPL ‘ _ 2018 _ 15 | REPL “ _ 201c _ 16 | REPL „ _ 201e _ 17 | REPL … _ 2026 _ 18 | NORM ’ ' 2019 0027 19 | ALLOW a _ 0061 _ 20 | ALLOW à _ 00e0 _ 21 | ALLOW â _ 00e2 _ 22 | ALLOW ä _ 00e4 _ 23 | ALLOW b _ 0062 _ 24 | ALLOW c _ 0063 _ 25 | ALLOW ç _ 00e7 _ 26 | ALLOW d _ 0064 _ 27 | ALLOW e _ 0065 _ 28 | ALLOW é _ 00e9 _ 29 | ALLOW è _ 00e8 _ 30 | ALLOW ê _ 00ea _ 31 | ALLOW f _ 0066 _ 32 | ALLOW g _ 0067 _ 33 | ALLOW h _ 0068 _ 34 | ALLOW i _ 0069 _ 35 | ALLOW ì _ 00ec _ 36 | ALLOW ï _ 00ef _ 37 | ALLOW j _ 006a _ 38 | ALLOW k _ 006b _ 39 | ALLOW l _ 006c _ 40 | ALLOW m _ 006d _ 41 | ALLOW n _ 006e _ 42 | ALLOW o _ 006f _ 43 | ALLOW ò _ 00f2 _ 44 | ALLOW ö _ 00f6 _ 45 | ALLOW p _ 0070 _ 46 | ALLOW q _ 0071 _ 47 | ALLOW r _ 0072 _ 48 | ALLOW s _ 0073 _ 49 | ALLOW t _ 0074 _ 50 | ALLOW u _ 0075 _ 51 | ALLOW ü _ 00fc _ 52 | ALLOW v _ 0076 _ 53 | ALLOW w _ 0077 _ 54 | ALLOW x _ 0078 _ 55 | ALLOW y _ 0079 _ 56 | ALLOW z _ 007a _ 57 | ALLOW ' _ 0027 _ 58 | ALLOW _ _ 005f _ 59 | -------------------------------------------------------------------------------- /cvutils/data/rm-vallader/wikipedia.txt: -------------------------------------------------------------------------------- 1 | rm 2 | -------------------------------------------------------------------------------- /cvutils/data/ro/alphabet.txt: -------------------------------------------------------------------------------- 1 | aăâbcdefghiîjklmnopqrsștțuvwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/ro/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | ă ə 4 | â ɨ 5 | b b 6 | c k 7 | ca ka 8 | cu ku 9 | câ kɨ 10 | cî kɨ 11 | că kə 12 | ci t͡ʃ 13 | ce t͡ʃ 14 | chi ki 15 | che ke 16 | d d 17 | e e 18 | e e̯ 19 | e je 20 | f f 21 | g ɡ 22 | ga ɡa 23 | gu ɡu 24 | gâ ɡɨ 25 | gî ɡɨ 26 | gă ɡə 27 | gi d͡ʒ 28 | ge d͡ʒ 29 | ghi ɡi 30 | ghe ɡe 31 | h h 32 | h _ 33 | i i 34 | i j 35 | i ʲ 36 | î ɨ 37 | j ʒ 38 | k k 39 | l l 40 | m m 41 | n n 42 | o o 43 | o o̯ 44 | p p 45 | q k 46 | r r 47 | s s 48 | ș ʃ 49 | t t 50 | ț t͡s 51 | u u 52 | u w 53 | v v 54 | w v 55 | x ks 56 | x ɡz 57 | y j 58 | y i 59 | z z 60 | -------------------------------------------------------------------------------- /cvutils/data/ro/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL " _ 0022 _ 4 | REPL , _ 002c _ 5 | REPL - _ 002d _ 6 | REPL . _ 002e _ 7 | REPL : _ 003a _ 8 | REPL ; _ 003b _ 9 | REPL ? _ 003f _ 10 | REPL ! _ 0021 _ 11 | REPL ” _ 201d _ 12 | REPL „ _ 201e _ 13 | SKIP ò _ 00f2 _ 14 | SKIP í _ 00ed _ 15 | NORM á a 00e1 0061 16 | NORM ć c 0107 0063 17 | NORM č c 010d 0063 18 | NORM đ g 0111 0067 19 | NORM é e 00e9 0065 20 | NORM š ș 0161 0219 21 | NORM ü u 00fc 0075 22 | NORM ş ș 015f 0219 23 | NORM ţ ț 0163 021b 24 | ALLOW a _ 0061 _ 25 | ALLOW ă _ 0103 _ 26 | ALLOW â _ 00e2 _ 27 | ALLOW b _ 0062 _ 28 | ALLOW c _ 0063 _ 29 | ALLOW d _ 0064 _ 30 | ALLOW e _ 0065 _ 31 | ALLOW f _ 0066 _ 32 | ALLOW g _ 0067 _ 33 | ALLOW h _ 0068 _ 34 | ALLOW i _ 0069 _ 35 | ALLOW î _ 00ee _ 36 | ALLOW j _ 006a _ 37 | ALLOW k _ 006b _ 38 | ALLOW l _ 006c _ 39 | ALLOW m _ 006d _ 40 | ALLOW n _ 006e _ 41 | ALLOW o _ 006f _ 42 | ALLOW p _ 0070 _ 43 | ALLOW q _ 0071 _ 44 | ALLOW r _ 0072 _ 45 | ALLOW s _ 0073 _ 46 | ALLOW ș _ 0219 _ 47 | ALLOW t _ 0074 _ 48 | ALLOW ț _ 021b _ 49 | ALLOW u _ 0075 _ 50 | ALLOW v _ 0076 _ 51 | ALLOW w _ 0077 _ 52 | ALLOW x _ 0078 _ 53 | ALLOW y _ 0079 _ 54 | ALLOW z _ 007a _ 55 | ALLOW _ _ 0020 _ 56 | -------------------------------------------------------------------------------- /cvutils/data/ru/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгдеёжзийклмнопрстуфхцчшщъыьэюя 2 | -------------------------------------------------------------------------------- /cvutils/data/ru/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL " _ 4 | REPL \ _ 5 | REPL ! _ 6 | REPL ' _ 7 | REPL ( _ 8 | REPL ) _ 9 | REPL , _ 10 | REPL - _ 11 | REPL . _ 12 | REPL : _ 13 | REPL ; _ 14 | REPL ? _ 15 | REPL « _ 16 | REPL » _ 17 | REPL ‑ _ 18 | REPL – _ 19 | REPL — _ 20 | REPL … _ 21 | REPL − _ 22 | SKIP 3 _ 23 | SKIP 4 _ 24 | SKIP a _ 25 | SKIP b _ 26 | SKIP c _ 27 | SKIP e _ 28 | SKIP f _ 29 | SKIP g _ 30 | SKIP h _ 31 | SKIP i _ 32 | SKIP k _ 33 | SKIP l _ 34 | SKIP m _ 35 | SKIP n _ 36 | SKIP o _ 37 | SKIP p _ 38 | SKIP r _ 39 | SKIP s _ 40 | SKIP t _ 41 | SKIP u _ 42 | SKIP x _ 43 | SKIP z _ 44 | ALLOW а _ 45 | ALLOW б _ 46 | ALLOW в _ 47 | ALLOW г _ 48 | ALLOW д _ 49 | ALLOW е _ 50 | ALLOW ё _ 51 | ALLOW ж _ 52 | ALLOW з _ 53 | ALLOW и _ 54 | ALLOW й _ 55 | ALLOW к _ 56 | ALLOW л _ 57 | ALLOW м _ 58 | ALLOW н _ 59 | ALLOW о _ 60 | ALLOW п _ 61 | ALLOW р _ 62 | ALLOW с _ 63 | ALLOW т _ 64 | ALLOW у _ 65 | ALLOW ф _ 66 | ALLOW х _ 67 | ALLOW ц _ 68 | ALLOW ч _ 69 | ALLOW ш _ 70 | ALLOW щ _ 71 | ALLOW ъ _ 72 | ALLOW ы _ 73 | ALLOW ь _ 74 | ALLOW э _ 75 | ALLOW ю _ 76 | ALLOW я _ 77 | ALLOW _ _ 78 | -------------------------------------------------------------------------------- /cvutils/data/ru/vocab.tsv: -------------------------------------------------------------------------------- 1 | ноль 2 | один 3 | два 4 | три 5 | четыре 6 | пять 7 | шесть 8 | семь 9 | восемь 10 | девять 11 | да 12 | нет 13 | -------------------------------------------------------------------------------- /cvutils/data/rw/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnoprstuvwyz 2 | -------------------------------------------------------------------------------- /cvutils/data/rw/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | a aː 4 | b β 5 | b b 6 | c t͡ʃ 7 | cy c 8 | d d 9 | e e 10 | e eː 11 | f f 12 | g ɡ 13 | g ɟ 14 | h h 15 | i i 16 | i iː 17 | j ʒ 18 | jy ɟ 19 | k k 20 | k c 21 | m m 22 | n n 23 | n ŋ 24 | ny ɲ 25 | o o 26 | o oː 27 | p p 28 | pf p͡f 29 | r ɾ 30 | s s 31 | sh ʃ 32 | t t 33 | ts t͡s 34 | u u,uː 35 | v v 36 | w w 37 | y j 38 | z z 39 | pw pk 40 | bw bɡ 41 | tw tkw 42 | dw dɡw 43 | mw mŋ 44 | nw nŋw 45 | nyw ɲŋw 46 | nyw ŋwa 47 | fw fk 48 | vw vɡ 49 | sw skw 50 | zw zɡw 51 | shw ʃkw 52 | jw ʒɡw 53 | pfw p͡fk 54 | tsw t͡skw 55 | cw t͡ʃkw 56 | rw ɾɡw 57 | py pc 58 | by bɟ 59 | ty tc 60 | dy dɟ 61 | my mɲ 62 | sy sc 63 | ry ɾɟ 64 | -------------------------------------------------------------------------------- /cvutils/data/rw/vocab.tsv: -------------------------------------------------------------------------------- 1 | zeru 2 | rimwe 3 | kabiri 4 | gatatu 5 | kane 6 | gatanu 7 | gatandatu 8 | karindwi 9 | umunani 10 | icyenda 11 | yego 12 | oya 13 | -------------------------------------------------------------------------------- /cvutils/data/sah/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгҕдеёжзийклмнҥоөпрстуүфхһцчшщыэюяь 2 | -------------------------------------------------------------------------------- /cvutils/data/sah/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | а a 3 | б b 4 | в v 5 | г ɡ 6 | ҕ ɣ 7 | ҕ ʁ 8 | д d 9 | дь ɟ 10 | е e 11 | е je 12 | ё jo 13 | ж ʒ 14 | з z 15 | и i 16 | й j 17 | й ȷ̃ 18 | к k 19 | л l 20 | м m 21 | н n 22 | ҥ ŋ 23 | нь ɲ 24 | о o 25 | ө ø 26 | п p 27 | р ɾ 28 | с s 29 | һ h 30 | т t 31 | у u 32 | ү y 33 | ф f 34 | х q 35 | х x 36 | ц ts 37 | ч c 38 | ш ʃ 39 | щ ɕː 40 | ы ɯ 41 | э e 42 | ю ju 43 | я ja 44 | -------------------------------------------------------------------------------- /cvutils/data/sah/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/sah/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL , _ 002c _ 4 | REPL - _ 002d _ 5 | REPL . _ 002e _ 6 | REPL : _ 003a _ 7 | REPL ; _ 003b _ 8 | REPL « _ 00ab _ 9 | REPL » _ 00bb _ 10 | NORM a а 0061 0430 11 | NORM o о 006f 043e 12 | NORM h һ 0068 04bb 13 | ALLOW а _ 0430 _ 14 | ALLOW б _ 0431 _ 15 | ALLOW в _ 0432 _ 16 | ALLOW г _ 0433 _ 17 | ALLOW ҕ _ 0495 _ 18 | ALLOW д _ 0434 _ 19 | ALLOW е _ 0435 _ 20 | ALLOW ё _ 0451 _ 21 | ALLOW ж _ 0436 _ 22 | ALLOW з _ 0437 _ 23 | ALLOW и _ 0438 _ 24 | ALLOW й _ 0439 _ 25 | ALLOW к _ 043a _ 26 | ALLOW л _ 043b _ 27 | ALLOW м _ 043c _ 28 | ALLOW н _ 043d _ 29 | ALLOW ҥ _ 04a5 _ 30 | ALLOW о _ 043e _ 31 | ALLOW ө _ 04e9 _ 32 | ALLOW п _ 043f _ 33 | ALLOW р _ 0440 _ 34 | ALLOW с _ 0441 _ 35 | ALLOW т _ 0442 _ 36 | ALLOW у _ 0443 _ 37 | ALLOW ү _ 04af _ 38 | ALLOW ф _ 0444 _ 39 | ALLOW х _ 0445 _ 40 | ALLOW һ _ 04bb _ 41 | ALLOW ц _ 0446 _ 42 | ALLOW ч _ 0447 _ 43 | ALLOW ш _ 0448 _ 44 | ALLOW щ _ 0449 _ 45 | ALLOW ы _ 044b _ 46 | ALLOW э _ 044d _ 47 | ALLOW ю _ 044e _ 48 | ALLOW я _ 044f _ 49 | ALLOW ь _ 044c _ 50 | ALLOW _ _ 005f _ 51 | -------------------------------------------------------------------------------- /cvutils/data/sat/alphabet.txt: -------------------------------------------------------------------------------- 1 | ᱚᱛᱜᱜᱝᱞᱟᱠᱡᱡᱢᱣᱣᱤᱥᱦᱦᱧᱨᱩᱪᱫᱫᱬᱭᱮᱯᱰᱱᱲᱳᱴᱵᱵᱶᱷ 2 | -------------------------------------------------------------------------------- /cvutils/data/sat/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | ᱚ ɔ 3 | ᱛ t 4 | ᱜ k’ 5 | ᱜ g 6 | ᱝ ŋ 7 | ᱞ l 8 | ᱟ a 9 | ᱠ k 10 | ᱡ c’ 11 | ᱡ ɟ 12 | ᱢ m 13 | ᱣ w 14 | ᱣ v 15 | ᱤ i 16 | ᱥ s 17 | ᱦ ʔ 18 | ᱦ h 19 | ᱧ ɲ 20 | ᱨ r 21 | ᱩ u 22 | ᱪ c 23 | ᱫ t’ 24 | ᱫ d 25 | ᱬ ɳ 26 | ᱭ j 27 | ᱮ e 28 | ᱯ p 29 | ᱰ ɖ 30 | ᱱ n 31 | ᱲ ɽ 32 | ᱳ o 33 | ᱴ ʈ 34 | ᱵ p’ 35 | ᱵ b 36 | ᱶ w̃ 37 | ᱷ ʰ 38 | -------------------------------------------------------------------------------- /cvutils/data/sat/validate.tsv: -------------------------------------------------------------------------------- 1 | NFKC _ _ _ _ 2 | REPL ? _ 003f _ 3 | REPL ᱾ _ 1c7e _ 4 | ALLOW ᱽ _ 1c7d _ 5 | ALLOW ᱚ _ 1c5a _ 6 | ALLOW ᱛ _ 1c5b _ 7 | ALLOW ᱜ _ 1c5c _ 8 | ALLOW ᱝ _ 1c5d _ 9 | ALLOW ᱞ _ 1c5e _ 10 | ALLOW ᱟ _ 1c5f _ 11 | ALLOW ᱠ _ 1c60 _ 12 | ALLOW ᱡ _ 1c61 _ 13 | ALLOW ᱢ _ 1c62 _ 14 | ALLOW ᱣ _ 1c63 _ 15 | ALLOW ᱤ _ 1c64 _ 16 | ALLOW ᱥ _ 1c65 _ 17 | ALLOW ᱦ _ 1c66 _ 18 | ALLOW ᱧ _ 1c67 _ 19 | ALLOW ᱨ _ 1c68 _ 20 | ALLOW ᱩ _ 1c69 _ 21 | ALLOW ᱪ _ 1c6a _ 22 | ALLOW ᱫ _ 1c6b _ 23 | ALLOW ᱬ _ 1c6c _ 24 | ALLOW ᱭ _ 1c6d _ 25 | ALLOW ᱮ _ 1c6e _ 26 | ALLOW ᱯ _ 1c6f _ 27 | ALLOW ᱰ _ 1c70 _ 28 | ALLOW ᱱ _ 1c71 _ 29 | ALLOW ᱲ _ 1c72 _ 30 | ALLOW ᱴ _ 1c74 _ 31 | ALLOW ᱵ _ 1c75 _ 32 | ALLOW ᱶ _ 1c76 _ 33 | ALLOW ᱷ _ 1c77 _ 34 | ALLOW ᱸ _ 1c78 _ 35 | ALLOW ᱹ _ 1c79 _ 36 | ALLOW ᱼ _ 1c7c _ 37 | ALLOW _ _ 0020 _ 38 | -------------------------------------------------------------------------------- /cvutils/data/sc/alphabet.txt: -------------------------------------------------------------------------------- 1 | aàbcdeèfghiíìjlmnoòprstuùvwz·' 2 | -------------------------------------------------------------------------------- /cvutils/data/sc/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL , _ 002c _ 6 | REPL - _ 002d _ 7 | REPL . _ 002e _ 8 | REPL : _ 003a _ 9 | REPL ; _ 003b _ 10 | REPL ? _ 003f _ 11 | REPL « _ 00ab _ 12 | REPL » _ 00bb _ 13 | REPL “ _ 201c _ 14 | REPL ” _ 201d _ 15 | NORM ’ ' 2019 _ 16 | ALLOW ' _ 0027 _ 17 | ALLOW · _ 00b7 _ 18 | ALLOW a _ 0061 _ 19 | ALLOW à _ 00e0 _ 20 | ALLOW b _ 0062 _ 21 | ALLOW c _ 0063 _ 22 | ALLOW d _ 0064 _ 23 | ALLOW e _ 0065 _ 24 | ALLOW è _ 00e8 _ 25 | ALLOW f _ 0066 _ 26 | ALLOW g _ 0067 _ 27 | ALLOW h _ 0068 _ 28 | ALLOW i _ 0069 _ 29 | ALLOW í _ 00ed _ 30 | ALLOW ì _ 00ec _ 31 | ALLOW j _ 006a _ 32 | ALLOW l _ 006c _ 33 | ALLOW m _ 006d _ 34 | ALLOW n _ 006e _ 35 | ALLOW o _ 006f _ 36 | ALLOW ò _ 00f2 _ 37 | ALLOW p _ 0070 _ 38 | ALLOW r _ 0072 _ 39 | ALLOW s _ 0073 _ 40 | ALLOW t _ 0074 _ 41 | ALLOW u _ 0075 _ 42 | ALLOW ù _ 00f9 _ 43 | ALLOW v _ 0076 _ 44 | ALLOW w _ 0077 _ 45 | ALLOW z _ 007a _ 46 | ALLOW _ _ 0020 _ 47 | -------------------------------------------------------------------------------- /cvutils/data/sk/alphabet.txt: -------------------------------------------------------------------------------- 1 | aáäbcčchdďdzdžeéfghiíjklĺľmnňoóôpqrŕsštťuúvwxyýzž -------------------------------------------------------------------------------- /cvutils/data/sk/phon.tsv: -------------------------------------------------------------------------------- 1 | a a 2 | á aː 3 | ä æ 4 | ä ɛ 5 | b b 6 | c ts 7 | č tʃ 8 | d d 9 | ď ɟ 10 | dz dz 11 | dž dʒ 12 | e ɛ 13 | é ɛː 14 | f f 15 | g g 16 | h ɦ 17 | ch x 18 | i i 19 | í iː 20 | j j 21 | k k 22 | l l 23 | ľ ʎ 24 | ĺ lː 25 | m m 26 | n n 27 | ň ɲ 28 | o ɔ 29 | ó ɔː 30 | ô ʊ̯ɔ 31 | p p 32 | q kʋ 33 | r ɾ 34 | r r 35 | ŕ r̩ː 36 | s s 37 | š ʃ 38 | t t 39 | ť c 40 | u u 41 | ú uː 42 | v v 43 | w v 44 | x ks 45 | y i 46 | ý iː 47 | z z 48 | ž ʒ 49 | -------------------------------------------------------------------------------- /cvutils/data/sk/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL , _ 002c _ 6 | REPL - _ 002d _ 7 | REPL . _ 002e _ 8 | REPL : _ 003a _ 9 | REPL ; _ 003b _ 10 | REPL ? _ 003f _ 11 | REPL ` _ 0060 _ 12 | REPL – _ 2013 _ 13 | REPL — _ 2014 _ 14 | REPL “ _ 201c _ 15 | REPL „ _ 201e _ 16 | REPL ö o 00f6 006f 17 | SKIP 3 _ 0033 _ 18 | SKIP ³ _ 00b3 _ 19 | ALLOW a _ 0061 _ 20 | ALLOW á _ 00e1 _ 21 | ALLOW ä _ 00e4 _ 22 | ALLOW b _ 0062 _ 23 | ALLOW c _ 0063 _ 24 | ALLOW č _ 010d _ 25 | ALLOW d _ 0064 _ 26 | ALLOW ď _ 010f _ 27 | ALLOW e _ 0065 _ 28 | ALLOW é _ 00e9 _ 29 | ALLOW f _ 0066 _ 30 | ALLOW g _ 0067 _ 31 | ALLOW h _ 0068 _ 32 | ALLOW i _ 0069 _ 33 | ALLOW í _ 00ed _ 34 | ALLOW j _ 006a _ 35 | ALLOW k _ 006b _ 36 | ALLOW l _ 006c _ 37 | ALLOW ĺ _ 013a _ 38 | ALLOW ľ _ 013e _ 39 | ALLOW m _ 006d _ 40 | ALLOW n _ 006e _ 41 | ALLOW ň _ 0148 _ 42 | ALLOW o _ 006f _ 43 | ALLOW ó _ 00f3 _ 44 | ALLOW ô _ 00f4 _ 45 | ALLOW p _ 0070 _ 46 | ALLOW q _ 0071 _ 47 | ALLOW r _ 0072 _ 48 | ALLOW ŕ _ 0155 _ 49 | ALLOW s _ 0073 _ 50 | ALLOW š _ 0161 _ 51 | ALLOW t _ 0074 _ 52 | ALLOW ť _ 0165 _ 53 | ALLOW u _ 0075 _ 54 | ALLOW ú _ 00fa _ 55 | ALLOW v _ 0076 _ 56 | ALLOW w _ 0077 _ 57 | ALLOW x _ 0078 _ 58 | ALLOW y _ 0079 _ 59 | ALLOW ý _ 00fd _ 60 | ALLOW z _ 007a _ 61 | ALLOW ž _ 017e _ 62 | ALLOW _ __ 0020 _ 63 | -------------------------------------------------------------------------------- /cvutils/data/skr/alphabet.txt: -------------------------------------------------------------------------------- 1 | ابٻپتٹثجڄچحخدڈݙذرڑزژسشصضطظعغفقکگڳلمنںݨوہھیےَُُِ 2 | -------------------------------------------------------------------------------- /cvutils/data/skr/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | ا a 3 | ا ə 4 | ا e 5 | ا ɛ 6 | ا o 7 | ا ɪ 8 | ا ʊ 9 | ب b 10 | ٻ ɓ 11 | پ p 12 | ت t 13 | ٹ ʈ 14 | ث s 15 | ج d͡ʒ 16 | ڄ ʄ 17 | چ t͡ʃ 18 | ح ɦ 19 | خ x 20 | د d 21 | ڈ ɖ 22 | ݙ ᶑ 23 | ذ z 24 | ر r 25 | ڑ ɽ 26 | ز z 27 | ژ ʃ 28 | س s 29 | ش ʃ 30 | ص s 31 | ض z 32 | ط t 33 | ظ z 34 | ع ∅ 35 | ع ə 36 | ع e 37 | ع ɛ 38 | ع o 39 | ع ɪ 40 | ع ʊ 41 | غ ɣ 42 | ف f 43 | ق q 44 | ک k 45 | گ g 46 | ڳ ɠ 47 | ل l 48 | م m 49 | ن n 50 | ں ◌̃ 51 | ݨ ɳ 52 | و v 53 | ہ ɦ 54 | ھ ◌ʰ 55 | ھ ◌ʱ 56 | ی j 57 | ی i 58 | ے e 59 | ے ɛ 60 | -------------------------------------------------------------------------------- /cvutils/data/skr/validate.tsv: -------------------------------------------------------------------------------- 1 | NFKC _ _ _ _ 2 | REPL ! _ 0021 _ 3 | REPL " _ 0022 _ 4 | REPL , _ 002c _ 5 | REPL . _ 002e _ 6 | REPL : _ 003a _ 7 | REPL ـ _ 0640 _ 8 | REPL ء _ 0621 _ 9 | REPL آ _ 0622 _ 10 | REPL ، _ 060c _ 11 | REPL ؟ _ 061f _ 12 | REPL ۔ _ 06d4 _ 13 | SKIP ؔ _ 0614 _ 14 | SKIP ً _ 064b _ 15 | SKIP ٌ _ 064c _ 16 | SKIP ْ _ 0652 _ 17 | SKIP ٓ _ 0653 _ 18 | SKIP ٖ _ 0656 _ 19 | SKIP ٗ _ 0657 _ 20 | SKIP ٰ _ 0670 _ 21 | REPL ؤ _ 0624 _ 22 | REPL ئ _ 0626 _ 23 | REPL ۃ _ 06c3 _ 24 | ALLOW ا _ 0627 _ 25 | ALLOW ب _ 0628 _ 26 | ALLOW ٻ _ 067b _ 27 | ALLOW پ _ 067e _ 28 | ALLOW ت _ 062a _ 29 | ALLOW ث _ 062b _ 30 | ALLOW ٹ _ 0679 _ 31 | ALLOW ج _ 062c _ 32 | ALLOW ڄ _ 0684 _ 33 | ALLOW چ _ 0686 _ 34 | ALLOW ح _ 062d _ 35 | ALLOW خ _ 062e _ 36 | ALLOW د _ 062f _ 37 | ALLOW ذ _ 0630 _ 38 | ALLOW ڈ _ 0688 _ 39 | ALLOW ݙ _ 0759 _ 40 | ALLOW ر _ 0631 _ 41 | ALLOW ز _ 0632 _ 42 | ALLOW ڑ _ 0691 _ 43 | ALLOW س _ 0633 _ 44 | ALLOW ش _ 0634 _ 45 | ALLOW ص _ 0635 _ 46 | ALLOW ض _ 0636 _ 47 | ALLOW ط _ 0637 _ 48 | ALLOW ظ _ 0638 _ 49 | ALLOW ع _ 0639 _ 50 | ALLOW غ _ 063a _ 51 | ALLOW ف _ 0641 _ 52 | ALLOW ق _ 0642 _ 53 | ALLOW ک _ 06a9 _ 54 | ALLOW گ _ 06af _ 55 | ALLOW ڳ _ 06b3 _ 56 | ALLOW ل _ 0644 _ 57 | ALLOW م _ 0645 _ 58 | ALLOW ن _ 0646 _ 59 | ALLOW ں _ 06ba _ 60 | ALLOW ݨ _ 0768 _ 61 | ALLOW ھ _ 06be _ 62 | ALLOW ہ _ 06c1 _ 63 | ALLOW و _ 0648 _ 64 | ALLOW ی _ 06cc _ 65 | ALLOW ے _ 06d2 _ 66 | ALLOW ٘ _ 0658 _ 67 | ALLOW ُ _ 064f _ 68 | ALLOW ِ _ 0650 _ 69 | ALLOW َ _ 064e _ 70 | ALLOW ّ _ 0651 _ 71 | ALLOW _ _ 0020 _ 72 | -------------------------------------------------------------------------------- /cvutils/data/sl/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcčdefghijklmnoprsštuvwxyzž 2 | -------------------------------------------------------------------------------- /cvutils/data/sl/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | b b 4 | c ts 5 | č tʃ 6 | d d 7 | e ɛ 8 | f f 9 | g ɡ 10 | h x 11 | i i 12 | j j 13 | k k 14 | l l 15 | m m 16 | n n 17 | o ɔ 18 | p p 19 | r r 20 | s s 21 | š ʃ 22 | t t 23 | u u 24 | v v 25 | z z 26 | ž ʒ 27 | -------------------------------------------------------------------------------- /cvutils/data/sl/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | REPL ! _ 4 | REPL , _ 5 | REPL - _ 6 | REPL . _ 7 | REPL : _ 8 | REPL ; _ 9 | REPL ? _ 10 | REPL « _ 11 | REPL » _ 12 | REPL – _ 13 | SKIP 4 _ 14 | ALLOW a _ 15 | ALLOW b _ 16 | ALLOW c _ 17 | ALLOW č _ 18 | ALLOW d _ 19 | ALLOW e _ 20 | ALLOW f _ 21 | ALLOW g _ 22 | ALLOW h _ 23 | ALLOW i _ 24 | ALLOW j _ 25 | ALLOW k _ 26 | ALLOW l _ 27 | ALLOW m _ 28 | ALLOW n _ 29 | ALLOW o _ 30 | ALLOW p _ 31 | ALLOW r _ 32 | ALLOW s _ 33 | ALLOW š _ 34 | ALLOW t _ 35 | ALLOW u _ 36 | ALLOW v _ 37 | ALLOW w _ 38 | ALLOW x _ 39 | ALLOW y _ 40 | ALLOW z _ 41 | ALLOW ž _ 42 | ALLOW _ _ 43 | -------------------------------------------------------------------------------- /cvutils/data/sl/vocab.tsv: -------------------------------------------------------------------------------- 1 | nìč 2 | êna 3 | dvé 4 | trí 5 | štíri 6 | pét 7 | šést 8 | sédem 9 | ósem 10 | devét 11 | ja 12 | ne 13 | -------------------------------------------------------------------------------- /cvutils/data/sq/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcçdeëfghijklmnopqrstuvxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/sq/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL , _ 002c _ 6 | REPL - _ 002d _ 7 | REPL . _ 002e _ 8 | REPL : _ 003a _ 9 | REPL ; _ 003b _ 10 | REPL ? _ 003f _ 11 | REPL — _ 2014 _ 12 | REPL “ _ 201c _ 13 | REPL ” _ 201d _ 14 | REPL ' _ 0027 _ 15 | REPL ` _ 0060 _ 16 | REPL ‘ _ 2018 _ 17 | REPL ’ _ 2019 _ 18 | REPL … _ 2026 _ 19 | SKIP w _ 0077 _ 20 | NORM á a 00e1 0062 21 | ALLOW _ _ 0020 _ 22 | ALLOW a _ 0061 _ 23 | ALLOW b _ 0062 _ 24 | ALLOW c _ 0063 _ 25 | ALLOW ç _ 00e7 _ 26 | ALLOW d _ 0064 _ 27 | ALLOW e _ 0065 _ 28 | ALLOW ë _ 00eb _ 29 | ALLOW f _ 0066 _ 30 | ALLOW g _ 0067 _ 31 | ALLOW h _ 0068 _ 32 | ALLOW i _ 0069 _ 33 | ALLOW j _ 006a _ 34 | ALLOW k _ 006b _ 35 | ALLOW l _ 006c _ 36 | ALLOW m _ 006d _ 37 | ALLOW n _ 006e _ 38 | ALLOW o _ 006f _ 39 | ALLOW p _ 0070 _ 40 | ALLOW q _ 0071 _ 41 | ALLOW r _ 0072 _ 42 | ALLOW s _ 0073 _ 43 | ALLOW t _ 0074 _ 44 | ALLOW u _ 0075 _ 45 | ALLOW v _ 0076 _ 46 | ALLOW x _ 0078 _ 47 | ALLOW y _ 0079 _ 48 | ALLOW z _ 007a _ 49 | -------------------------------------------------------------------------------- /cvutils/data/sr/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгдђежзијклљмнњопрстћуфхцчџш 2 | -------------------------------------------------------------------------------- /cvutils/data/sr/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | а a 3 | б b 4 | в v 5 | г ɡ 6 | д d 7 | ђ dʑ 8 | е e 9 | ж ʐ 10 | з z 11 | и i 12 | ј j 13 | к k 14 | л l 15 | љ ʎ 16 | м m 17 | н n 18 | њ ɲ 19 | о ɔ 20 | п p 21 | р ɾ 22 | с s 23 | т t 24 | ћ tɕ 25 | у u 26 | ф f 27 | х x 28 | ц ts 29 | ч ʈʂ 30 | џ ɖʐ 31 | ш ʂ 32 | -------------------------------------------------------------------------------- /cvutils/data/sr/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL . _ _ _ 4 | REPL , _ _ _ 5 | REPL ! _ _ _ 6 | REPL ? _ _ _ 7 | ALLOW а _ 0430 _ 8 | ALLOW б _ 0431 _ 9 | ALLOW в _ 0432 _ 10 | ALLOW г _ 0433 _ 11 | ALLOW д _ 0434 _ 12 | ALLOW ђ _ 0452 _ 13 | ALLOW е _ 0435 _ 14 | ALLOW ж _ 0436 _ 15 | ALLOW з _ 0437 _ 16 | ALLOW и _ 0438 _ 17 | ALLOW ј _ 0458 _ 18 | ALLOW к _ 043a _ 19 | ALLOW л _ 043b _ 20 | ALLOW љ _ 0459 _ 21 | ALLOW м _ 043c _ 22 | ALLOW н _ 043d _ 23 | ALLOW њ _ 045a _ 24 | ALLOW о _ 043e _ 25 | ALLOW п _ 043f _ 26 | ALLOW р _ 0440 _ 27 | ALLOW с _ 0441 _ 28 | ALLOW т _ 0442 _ 29 | ALLOW ћ _ 045b _ 30 | ALLOW у _ 0443 _ 31 | ALLOW ф _ 0444 _ 32 | ALLOW х _ 0445 _ 33 | ALLOW ц _ 0446 _ 34 | ALLOW ч _ 0447 _ 35 | ALLOW џ _ 045f _ 36 | ALLOW ш _ 0448 _ 37 | ALLOW _ _ 0020 _ 38 | -------------------------------------------------------------------------------- /cvutils/data/ss/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnopqrstuvwyz 2 | -------------------------------------------------------------------------------- /cvutils/data/ss/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | e ɛ 4 | e e 5 | i i 6 | o ɔ 7 | o o 8 | u u 9 | b ɓ 10 | bh bʱ 11 | c ᵏǀ 12 | ch ᵏǀʰ 13 | d dʱ 14 | dl ɮ 15 | dv dv 16 | dz dz 17 | f f 18 | g gʱ 19 | gc ᶢǀʱ 20 | h h 21 | hh ɦ 22 | hl ɬ 23 | j dʒʱ 24 | k kʼ 25 | k k̬ 26 | kh kʰ 27 | kl kɬ 28 | l l 29 | m m 30 | mb mb 31 | n n 32 | nc ᵑǀ 33 | nch ᵑǀʰ 34 | ndl ⁿɮ 35 | ng ŋ 36 | ng ŋɡ 37 | ngc ᵑǀʱ 38 | nhl ⁿɫ 39 | p pʼ 40 | ph pʰ 41 | q kʼ 42 | q k̬ 43 | s s 44 | sh ʃ 45 | t tʼ 46 | tf tf 47 | th tʰ 48 | tj tʃʼ 49 | ts tsʼ 50 | ts tsʰ 51 | v v 52 | w w 53 | y j 54 | z z 55 | zh ʒ 56 | dvw dvʷ 57 | khw kʰʷ 58 | lw lʷ 59 | nkhw ᵑkʰʷ 60 | ngw ᵑʷ 61 | ngw ᵑgʷ 62 | sw sʷ 63 | vw vʷ 64 | -------------------------------------------------------------------------------- /cvutils/data/st/alphabet.txt: -------------------------------------------------------------------------------- 1 | abdefghijklmnopqrstuwy 2 | -------------------------------------------------------------------------------- /cvutils/data/st/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a ɑ 3 | b b 4 | bj bʒ 5 | d d 6 | e ɨ 7 | e e 8 | e ɛ 9 | f f 10 | fj fʃ 11 | h h 12 | hl ɬ 13 | i i 14 | j ʒ 15 | j d͡ʒ 16 | k kʼ 17 | kh kʰ 18 | kg x 19 | kg k͡xʰ 20 | l l 21 | m m 22 | n n 23 | ng ŋ 24 | nq ᵑǃ 25 | ny ɲ 26 | o ʉ 27 | o o 28 | o ɔ 29 | p pʼ 30 | ph pʰ 31 | pj pʃʼ 32 | pjh pʃʰ 33 | q ǃ 34 | qh ǃʰ 35 | r ʁ 36 | s s 37 | sh ʃ 38 | t tʼ 39 | th tʰ 40 | tj t͡ʃʼ 41 | tjh t͡ʃʰ 42 | tl tɬʼ 43 | tlh t͡ɬʰ 44 | ts tsʼ 45 | tsh tsʰ 46 | u u 47 | w w 48 | y j 49 | -------------------------------------------------------------------------------- /cvutils/data/sv-SE/alphabet.txt: -------------------------------------------------------------------------------- 1 | aåäbcdeéfghijklmnoöpqrstuvwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/sv-SE/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | b b 3 | c k 4 | c s 5 | ch ɧ 6 | ch ɕ 7 | d d 8 | dj j 9 | f f 10 | g ɡ 11 | g j 12 | gj j 13 | gn ɡn 14 | gn ŋn 15 | h h 16 | hj j 17 | j j 18 | k k 19 | k ɕ 20 | kj ɕ 21 | l l 22 | lj j 23 | m m 24 | n n 25 | ng ŋ 26 | ng ŋɡ 27 | p p 28 | r r 29 | s s 30 | sj ɧ 31 | sk sk 32 | sk ɧ 33 | skj ɧ 34 | stj ɧ 35 | t t 36 | tj ɕ 37 | v v 38 | w v 39 | x ks 40 | z s 41 | a ɑː 42 | e eː 43 | a a 44 | e ɛ 45 | i ɪ 46 | u ɵ 47 | y ʏ 48 | å ɔ 49 | ä ɛ 50 | ö œ 51 | i iː 52 | o uː 53 | o oː 54 | o ɔ 55 | o ʊ 56 | u ʉː 57 | y yː 58 | å oː 59 | ä ɛː 60 | ö øː 61 | rd ɖ 62 | rl ɭ 63 | rn ɳ 64 | rs ʂ 65 | rt ʈ 66 | -------------------------------------------------------------------------------- /cvutils/data/sv-SE/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL . _ 002e _ 4 | REPL , _ 002c _ 5 | REPL ? _ 003f _ 6 | REPL ! _ 0021 _ 7 | REPL " _ 0022 _ 8 | REPL - _ 002d _ 9 | REPL : _ 003a _ 10 | REPL ; _ 003b _ 11 | REPL ” _ 201d _ 12 | REPL “ _ 201c _ 13 | REPL – _ 2013 _ 14 | REPL … _ 2026 _ 15 | REPL — _ 2014 _ 16 | NORM ü u 00fc 0075 17 | NORM ô o 00f4 006f 18 | ALLOW a _ 0061 _ 19 | ALLOW å _ 00e5 _ 20 | ALLOW ä _ 00e4 _ 21 | ALLOW b _ 0062 _ 22 | ALLOW c _ 0063 _ 23 | ALLOW d _ 0064 _ 24 | ALLOW e _ 0065 _ 25 | ALLOW é _ 00e9 _ 26 | ALLOW f _ 0066 _ 27 | ALLOW g _ 0067 _ 28 | ALLOW h _ 0068 _ 29 | ALLOW i _ 0069 _ 30 | ALLOW j _ 006a _ 31 | ALLOW k _ 006b _ 32 | ALLOW l _ 006c _ 33 | ALLOW m _ 006d _ 34 | ALLOW n _ 006e _ 35 | ALLOW o _ 006f _ 36 | ALLOW ö _ 00f6 _ 37 | ALLOW p _ 0070 _ 38 | ALLOW q _ 0071 _ 39 | ALLOW r _ 0072 _ 40 | ALLOW s _ 0073 _ 41 | ALLOW t _ 0074 _ 42 | ALLOW u _ 0075 _ 43 | ALLOW v _ 0076 _ 44 | ALLOW w _ 0077 _ 45 | ALLOW x _ 0078 _ 46 | ALLOW y _ 0079 _ 47 | ALLOW z _ 007a _ 48 | ALLOW _ _ 0020 _ 49 | -------------------------------------------------------------------------------- /cvutils/data/sv-SE/vocab.tsv: -------------------------------------------------------------------------------- 1 | noll 2 | ett 3 | två 4 | tre 5 | fyra 6 | fem 7 | sex 8 | sju 9 | åtta 10 | nio 11 | ja 12 | nej 13 | -------------------------------------------------------------------------------- /cvutils/data/sv-SE/wikipedia.txt: -------------------------------------------------------------------------------- 1 | sv 2 | -------------------------------------------------------------------------------- /cvutils/data/sw/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnopqrstuvwxyz' 2 | -------------------------------------------------------------------------------- /cvutils/data/sw/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | m m 3 | n n 4 | ny ɲ 5 | ng' ŋ 6 | mb mb̥ 7 | nd nd̥ 8 | nj nd̥ʒ̊ 9 | ng ŋɡ̊ 10 | b ɓ 11 | b b 12 | d ɗ 13 | d d 14 | j ʄ 15 | j dʒ 16 | g ɠ 17 | g ɡ 18 | p p 19 | t t 20 | ch tʃ 21 | k k 22 | p pʰ 23 | t tʰ 24 | ch tʃʰ 25 | k kʰ 26 | mv mv̥ 27 | nz nz̥ 28 | v v 29 | dh ð 30 | z z 31 | gh ɣ 32 | f f 33 | th θ 34 | s s 35 | sh ʃ 36 | kh x 37 | h h 38 | l l 39 | y j 40 | w w 41 | r r 42 | a ɑ 43 | e ɛ 44 | i i 45 | o ɔ 46 | u u 47 | -------------------------------------------------------------------------------- /cvutils/data/sw/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | REPL - _ 3 | REPL ? _ 4 | REPL . _ 5 | SKIP _ _ 6 | SKIP 0 _ 7 | SKIP 1 _ 8 | SKIP 2 _ 9 | SKIP 3 _ 10 | SKIP 4 _ 11 | SKIP 5 _ 12 | SKIP 6 _ 13 | SKIP 7 _ 14 | SKIP 8 _ 15 | SKIP 9 _ 16 | SKIP < _ 17 | SKIP > _ 18 | ALLOW a _ 19 | ALLOW b _ 20 | ALLOW c _ 21 | ALLOW d _ 22 | ALLOW e _ 23 | ALLOW f _ 24 | ALLOW g _ 25 | ALLOW h _ 26 | ALLOW i _ 27 | ALLOW j _ 28 | ALLOW k _ 29 | ALLOW l _ 30 | ALLOW m _ 31 | ALLOW n _ 32 | ALLOW o _ 33 | ALLOW p _ 34 | ALLOW q _ 35 | ALLOW r _ 36 | ALLOW s _ 37 | ALLOW t _ 38 | ALLOW u _ 39 | ALLOW v _ 40 | ALLOW w _ 41 | ALLOW x _ 42 | ALLOW y _ 43 | ALLOW z _ 44 | ALLOW ' _ 45 | ALLOW _ _ 46 | -------------------------------------------------------------------------------- /cvutils/data/ta/alphabet.txt: -------------------------------------------------------------------------------- 1 | ஃகஙசஞடணதநபமயரலவழளறனஶஜஷஸஹக்ஷஅஆஇஈஉஊஎஏஐஒஓஔாிீுூெேைொோௌ் 2 | -------------------------------------------------------------------------------- /cvutils/data/ta/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | க ka 3 | ங ŋa 4 | ச t͡ʃa 5 | ஞ ɲa 6 | ட ʈa 7 | ண ɳa 8 | த t̪a 9 | ந n̪a 10 | ப pa 11 | ம ma 12 | ய ja 13 | ர ɾa 14 | ல la 15 | வ ʋa 16 | ழ ɻa 17 | ள ɭa 18 | ற ra 19 | ன na 20 | ஶ ʃa 21 | ஜ d͡ʒa 22 | ஷ ʂa 23 | ஸ sa 24 | ஹ ha 25 | க்ஷ kʂa 26 | அ a 27 | ஆ aː 28 | இ i 29 | ஈ iː 30 | உ u 31 | ஊ uː 32 | எ e 33 | ஏ eː 34 | ஐ aj 35 | ஒ o 36 | ஓ oː 37 | ஔ aʋ 38 | ா aː 39 | ி i 40 | ீ iː 41 | ு u 42 | ூ uː 43 | ெ e 44 | ே eː 45 | ை aj 46 | ொ o 47 | ோ oː 48 | ௌ aʋ 49 | ் _ 50 | -------------------------------------------------------------------------------- /cvutils/data/ta/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | REPL ! _ _ _ 3 | REPL " _ _ _ 4 | REPL ' _ _ _ 5 | REPL , _ _ _ 6 | REPL - _ _ _ 7 | REPL . _ _ _ 8 | REPL : _ _ _ 9 | REPL ; _ _ _ 10 | REPL ? _ _ _ 11 | REPL ‘ _ _ _ 12 | REPL ’ _ _ _ 13 | REPL ‘ _ _ _ 14 | REPL “ _ _ _ 15 | REPL ” _ _ _ 16 | REPL — _ _ _ 17 | SKIP ( _ _ _ 18 | SKIP ) _ _ _ 19 | ALLOW அ _ 0b85 _ 20 | ALLOW ஆ _ 0b86 _ 21 | ALLOW இ _ 0b87 _ 22 | ALLOW ஈ _ 0b88 _ 23 | ALLOW உ _ 0b89 _ 24 | ALLOW ஊ _ 0b8a _ 25 | ALLOW எ _ 0b8e _ 26 | ALLOW ஏ _ 0b8f _ 27 | ALLOW ஐ _ 0b90 _ 28 | ALLOW ஒ _ 0b92 _ 29 | ALLOW ஓ _ 0b93 _ 30 | ALLOW ஔ _ 0b94 _ 31 | ALLOW ஃ _ 0b83 _ 32 | ALLOW க _ 0b95 _ 33 | ALLOW ங _ 0b99 _ 34 | ALLOW ச _ 0b9a _ 35 | ALLOW ஞ _ 0b9e _ 36 | ALLOW ட _ 0b9f _ 37 | ALLOW ண _ 0ba3 _ 38 | ALLOW த _ 0ba4 _ 39 | ALLOW ந _ 0ba8 _ 40 | ALLOW ப _ 0baa _ 41 | ALLOW ம _ 0bae _ 42 | ALLOW ய _ 0baf _ 43 | ALLOW ர _ 0bb0 _ 44 | ALLOW ல _ 0bb2 _ 45 | ALLOW வ _ 0bb5 _ 46 | ALLOW ழ _ 0bb4 _ 47 | ALLOW ள _ 0bb3 _ 48 | ALLOW ற _ 0bb1 _ 49 | ALLOW ன _ 0ba9 _ 50 | ALLOW ஜ _ 0b9c _ 51 | ALLOW ஷ _ 0bb7 _ 52 | ALLOW ஸ _ 0bb8 _ 53 | ALLOW ஹ _ 0bb9 _ 54 | ALLOW ா _ 0bbe _ 55 | ALLOW ி _ 0bbf _ 56 | ALLOW ீ _ 0bc0 _ 57 | ALLOW ு _ 0bc1 _ 58 | ALLOW ூ _ 0bc2 _ 59 | ALLOW ெ _ 0bc6 _ 60 | ALLOW ே _ 0bc7 _ 61 | ALLOW ை _ 0bc8 _ 62 | ALLOW ொ _ 0bca _ 63 | ALLOW ோ _ 0bcb _ 64 | ALLOW ௌ _ 0bcc _ 65 | ALLOW ் _ 0bcd _ 66 | ALLOW _ _ _ _ 67 | -------------------------------------------------------------------------------- /cvutils/data/ta/vocab.tsv: -------------------------------------------------------------------------------- 1 | பூஜ்யம் 2 | ஒன்று 3 | இரண்டு 4 | மூன்று 5 | நான்கு 6 | ஐந்து 7 | ஆறு 8 | ஏழு 9 | எட்டு 10 | ஒன்பது 11 | ஆம் 12 | இல்லை 13 | -------------------------------------------------------------------------------- /cvutils/data/th/alphabet.txt: -------------------------------------------------------------------------------- 1 | ํ๎◌์กขคฆงจฉชซญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลวศษสหฬอฮะัาำิีึืุูเแโใไๅ่้๊๋็ 2 | -------------------------------------------------------------------------------- /cvutils/data/th/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | ก kə 3 | ข kʰə 4 | ฃ kʰə 5 | ค kʰə 6 | ฅ kʰə 7 | ฆ kʰə 8 | ง ŋə 9 | จ t͡ɕə 10 | ฉ t͡ɕʰə 11 | ช t͡ɕʰə 12 | ซ sə 13 | ฌ t͡ɕʰə 14 | ญ jə 15 | ฎ də 16 | ฏ tə 17 | ฐ tʰə 18 | ฑ tʰə 19 | ฒ tʰə 20 | ณ nə 21 | ด də 22 | ต tə 23 | ถ tʰə 24 | ท tʰə 25 | ธ tʰə 26 | น nə 27 | บ bə 28 | ป pə 29 | ผ pʰə 30 | ฝ fə 31 | พ pʰə 32 | ฟ fə 33 | ภ pʰə 34 | ม mə 35 | ย jə 36 | ร rə 37 | ล lə 38 | ว wə 39 | ศ sə 40 | ษ sə 41 | ส sə 42 | ห hə 43 | ฬ lə 44 | ฮ hə 45 | า aː 46 | ี iː 47 | ู uː 48 | เ eː 49 | เ็ eː 50 | แ ɛː 51 | ื- ɯː 52 | ื ɯː 53 | เอ ɤː 54 | โ oː 55 | อ ɔː 56 | ะ a 57 | ั a 58 | ิ i 59 | ุ u 60 | เะ e 61 | เิ ɤ 62 | แะ ɛ 63 | ึ ɯ 64 | เอะ ɤ 65 | โะ o 66 | เาะ ɔ 67 | ไ aj 68 | ไย aj 69 | ใ aj 70 | เียะ ia̯ 71 | เีย iːa̯ 72 | เือะ ɯa̯ 73 | เือ ɯːa̯ 74 | ัวะ ua̯ 75 | ัว uːa̯ 76 | -------------------------------------------------------------------------------- /cvutils/data/ti/alphabet.txt: -------------------------------------------------------------------------------- 1 | ሀሁሂሃሄህሆለሉሊላሌልሎሏሐሑሒሓሔሕሖሗመሙሚማሜምሞሟሠሡሢሣሤሥሦሧረሩሪራሬርሮሯሰሱሲሳሴስሶሷሸሹሺሻሼሽሾሿቀቁቂቃቄቅቆቈቊቋቌቍበቡቢባቤብቦቧቨቩቪቫቬቭቮቯተቱቲታቴትቶቷቸቹቺቻቼችቾቿኀኁኂኃኄኅኆኈኊኋኌኍነኑኒናኔንኖኗኘኙኚኛኜኝኞኟአኡኢኣኤእኦኧከኩኪካኬክኮኰኲኳኴኵኸኹኺኻኼኽኾዀዂዃዄዅወዉዊዋዌውዎዐዑዒዓዔዕዖዘዙዚዛዜዝዞዟዠዡዢዣዤዥዦዧየዩዪያዬይዮደዱዲዳዴድዶዷጀጁጂጃጄጅጆጇገጉጊጋጌግጎጐጒጓጔጕጠጡጢጣጤጥጦጧጨጩጪጫጬጭጮጯጰጱጲጳጴጵጶጷጸጹጺጻጼጽጾጿፀፁፂፃፄፅፆፈፉፊፋፌፍፎፏፐፑፒፓፔፕፖፗቒ 2 | -------------------------------------------------------------------------------- /cvutils/data/tig/alphabet.txt: -------------------------------------------------------------------------------- 1 | ሀሁሂሃሄህሆለሉሊላሌልሎሏሐሑሒሓሔሕሖሗመሙሚማሜምሞሟሠሡሢሣሤሥሦሧረሩሪራሬርሮሯሰሱሲሳሴስሶሷሸሹሺሻሼሽሾሿቀቁቂቃቄቅቆቈቊቋቌቍበቡቢባቤብቦቧቨቩቪቫቬቭቮቯተቱቲታቴትቶቷቸቹቺቻቼችቾቿኀኁኂኃኄኅኆኈኊኋኌኍነኑኒናኔንኖኗኘኙኚኛኜኝኞኟአኡኢኣኤእኦኧከኩኪካኬክኮኰኲኳኴኵኸኹኺኻኼኽኾዀዂዃዄዅወዉዊዋዌውዎዐዑዒዓዔዕዖዘዙዚዛዜዝዞዟዠዡዢዣዤዥዦዧየዩዪያዬይዮደዱዲዳዴድዶዷጀጁጂጃጄጅጆጇገጉጊጋጌግጎጐጒጓጔጕጠጡጢጣጤጥጦጧጨጩጪጫጬጭጮጯጰጱጲጳጴጵጶጷጸጹጺጻጼጽጾጿፀፁፂፃፄፅፆፈፉፊፋፌፍፎፏፐፑፒፓፔፕፖፗ 2 | -------------------------------------------------------------------------------- /cvutils/data/tk/alphabet.txt: -------------------------------------------------------------------------------- 1 | abçdeäfghijžklmnňoöprsştuüwyýz 2 | -------------------------------------------------------------------------------- /cvutils/data/tk/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL , _ 002c _ 6 | REPL - _ 002d _ 7 | REPL . _ 002e _ 8 | REPL : _ 003a _ 9 | REPL ? _ 003f _ 10 | REPL « _ 00ab _ 11 | REPL » _ 00bb _ 12 | REPL — _ 2014 _ 13 | REPL “ _ 201c _ 14 | REPL ” _ 201d _ 15 | REPL – _ 2013 _ 16 | REPL „ _ 201e _ 17 | REPL … _ 2026 _ 18 | NORM ñ ň 00f1 _ 19 | NORM е e 0435 _ 20 | NORM о o 043e _ 21 | NORM ì i 00ec _ 22 | SKIP c _ 0063 _ 23 | SKIP x _ 0078 _ 24 | ALLOW a _ 0061 _ 25 | ALLOW ä _ 00e4 _ 26 | ALLOW b _ 0062 _ 27 | ALLOW ç _ 00e7 _ 28 | ALLOW d _ 0064 _ 29 | ALLOW e _ 0065 _ 30 | ALLOW f _ 0066 _ 31 | ALLOW g _ 0067 _ 32 | ALLOW h _ 0068 _ 33 | ALLOW i _ 0069 _ 34 | ALLOW j _ 006a _ 35 | ALLOW k _ 006b _ 36 | ALLOW l _ 006c _ 37 | ALLOW m _ 006d _ 38 | ALLOW n _ 006e _ 39 | ALLOW ň _ 0148 _ 40 | ALLOW o _ 006f _ 41 | ALLOW ö _ 00f6 _ 42 | ALLOW p _ 0070 _ 43 | ALLOW r _ 0072 _ 44 | ALLOW s _ 0073 _ 45 | ALLOW ş _ 015f _ 46 | ALLOW t _ 0074 _ 47 | ALLOW u _ 0075 _ 48 | ALLOW ü _ 00fc _ 49 | ALLOW w _ 0077 _ 50 | ALLOW y _ 0079 _ 51 | ALLOW ý _ 00fd _ 52 | ALLOW z _ 007a _ 53 | ALLOW ž _ 017e _ 54 | ALLOW _ _ 0020 _ 55 | -------------------------------------------------------------------------------- /cvutils/data/tn/alphabet.txt: -------------------------------------------------------------------------------- 1 | abdeêfghijklmnoôprsštuwy -------------------------------------------------------------------------------- /cvutils/data/tn/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | i i 3 | u u 4 | e ɪ 5 | o ʊ 6 | ê ɛ 7 | ô ɔ 8 | a a 9 | m m 10 | n n 11 | ny ɲ 12 | ng ŋ 13 | p p 14 | t t 15 | ts ts 16 | tl tɬ 17 | tš tʃ 18 | k k 19 | b b 20 | d d 21 | j dʒ 22 | ph pʰ 23 | th tʰ 24 | tsh tsʰ 25 | tlh tɬʰ 26 | tšh tʃʰ 27 | kh kʰ 28 | kg kχʰ 29 | f f 30 | s s 31 | š ʃ 32 | g χ 33 | h h 34 | r r 35 | l l 36 | w w 37 | y j 38 | -------------------------------------------------------------------------------- /cvutils/data/tos/alphabet.txt: -------------------------------------------------------------------------------- 1 | aábcdeéfghiíjklmnñoópqrstuúvwxyzʼ 2 | -------------------------------------------------------------------------------- /cvutils/data/tos/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL , _ 002c _ 4 | REPL . _ 002e _ 5 | SKIP ( _ 0028 _ 6 | SKIP ) _ 0029 _ 7 | SKIP * _ 002a _ 8 | SKIP / _ 002f _ 9 | SKIP < _ 003c _ 10 | SKIP > _ 003e _ 11 | SKIP | _ 007c _ 12 | SKIP 1 _ 0031 _ 13 | NORM : ꞉ 003a a789 14 | NORM ' ʼ 0027 02bc 15 | DEL ꞉ _ a789 _ 16 | ALLOW ʼ _ 02bc _ 17 | ALLOW a _ 0061 _ 18 | ALLOW á _ 00e1 _ 19 | ALLOW b _ 0062 _ 20 | ALLOW c _ 0063 _ 21 | ALLOW d _ 0064 _ 22 | ALLOW e _ 0065 _ 23 | ALLOW é _ 00e9 _ 24 | ALLOW f _ 0066 _ 25 | ALLOW g _ 0067 _ 26 | ALLOW h _ 0068 _ 27 | ALLOW i _ 0069 _ 28 | ALLOW í _ 00ed _ 29 | ALLOW j _ 006a _ 30 | ALLOW k _ 006b _ 31 | ALLOW l _ 006c _ 32 | ALLOW m _ 006d _ 33 | ALLOW n _ 006e _ 34 | ALLOW ñ _ 00f1 _ 35 | ALLOW o _ 006f _ 36 | ALLOW ó _ 00f3 _ 37 | ALLOW p _ 0070 _ 38 | ALLOW q _ 0071 _ 39 | ALLOW r _ 0072 _ 40 | ALLOW s _ 0073 _ 41 | ALLOW t _ 0074 _ 42 | ALLOW u _ 0075 _ 43 | ALLOW ú _ 00fa _ 44 | ALLOW v _ 0076 _ 45 | ALLOW w _ 0077 _ 46 | ALLOW x _ 0078 _ 47 | ALLOW y _ 0079 _ 48 | ALLOW z _ 007a _ 49 | ALLOW _ _ 0020 _ 50 | -------------------------------------------------------------------------------- /cvutils/data/tr/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcçdefgğhiıjklmnoöprsştuüvyzâwx' 2 | -------------------------------------------------------------------------------- /cvutils/data/tr/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | i i 3 | ı ɯ 4 | ü y 5 | u u 6 | ö œ 7 | o o 8 | e e 9 | a a 10 | m m 11 | n n 12 | p p 13 | t t 14 | ç t͡ʃ 15 | k k 16 | b b 17 | d d 18 | c d͡ʒ 19 | g ɡ 20 | f f 21 | s s 22 | ş ʃ 23 | h h 24 | v v 25 | z z 26 | j ʒ 27 | l l 28 | y j 29 | r ɾ 30 | -------------------------------------------------------------------------------- /cvutils/data/tr/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | EOS … 5 | -------------------------------------------------------------------------------- /cvutils/data/tr/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL ' _ 0027 _ 6 | REPL . _ 002e _ 7 | REPL , _ 002c _ 8 | REPL - _ 002d _ 9 | REPL — _ 2014 _ 10 | REPL – _ 2013 _ 11 | REPL . _ 002e _ 12 | REPL : _ 003a _ 13 | REPL ; _ 003b _ 14 | REPL ? _ 003f _ 15 | REPL ‘ _ 2018 _ 16 | REPL “ _ 201c _ 17 | REPL ” _ 201d _ 18 | REPL … _ 2026 _ 19 | SKIP 0 _ 0030 _ 20 | SKIP 1 _ 0031 _ 21 | SKIP 2 _ 0032 _ 22 | SKIP 3 _ 0033 _ 23 | SKIP 4 _ 0034 _ 24 | SKIP 5 _ 0035 _ 25 | SKIP 6 _ 0036 _ 26 | SKIP 7 _ 0037 _ 27 | SKIP 8 _ 0038 _ 28 | SKIP 9 _ 0039 _ 29 | SKIP ( _ 0028 _ 30 | SKIP ) _ 0029 _ 31 | NORM ’ ' 2019 0027 32 | NORM û u 00fb 0075 33 | NORM î i 00ee 0069 34 | NORM ı̇ i 0131-0307 0069 35 | NORM İ İ 0049-0307 0130 36 | DEL ̇ _ 0307 _ 37 | ALLOW a _ 0061 _ 38 | ALLOW b _ 0062 _ 39 | ALLOW c _ 0063 _ 40 | ALLOW ç _ 00e7 _ 41 | ALLOW d _ 0064 _ 42 | ALLOW e _ 0065 _ 43 | ALLOW f _ 0066 _ 44 | ALLOW g _ 0067 _ 45 | ALLOW ğ _ 011f _ 46 | ALLOW h _ 0068 _ 47 | ALLOW i _ 0069 _ 48 | ALLOW ı _ 0131 _ 49 | ALLOW j _ 006a _ 50 | ALLOW k _ 006b _ 51 | ALLOW l _ 006c _ 52 | ALLOW m _ 006d _ 53 | ALLOW n _ 006e _ 54 | ALLOW o _ 006f _ 55 | ALLOW ö _ 00f6 _ 56 | ALLOW p _ 0070 _ 57 | ALLOW r _ 0072 _ 58 | ALLOW s _ 0073 _ 59 | ALLOW ş _ 015f _ 60 | ALLOW t _ 0074 _ 61 | ALLOW u _ 0075 _ 62 | ALLOW ü _ 00fc _ 63 | ALLOW v _ 0076 _ 64 | ALLOW y _ 0079 _ 65 | ALLOW z _ 007a _ 66 | ALLOW â _ 00e2 _ 67 | ALLOW w _ 0077 _ 68 | ALLOW x _ 0078 _ 69 | ALLOW ' _ 0027 _ 70 | ALLOW _ _ 005f _ 71 | -------------------------------------------------------------------------------- /cvutils/data/tr/vocab.tsv: -------------------------------------------------------------------------------- 1 | sıfır 2 | bir 3 | iki 4 | üç 5 | dört 6 | beş 7 | altı 8 | yedi 9 | sekiz 10 | dokuz 11 | evet 12 | hayır 13 | -------------------------------------------------------------------------------- /cvutils/data/ts/alphabet.txt: -------------------------------------------------------------------------------- 1 | 'abcdefghijklmnopqrstuvwyz 2 | -------------------------------------------------------------------------------- /cvutils/data/ts/phon.tsv: -------------------------------------------------------------------------------- 1 | a a 2 | aa aː 3 | b b 4 | bh bʱ 5 | bhy bʲʱ 6 | bv b̪v 7 | bvh b̪vʱ 8 | by bʲ 9 | ch tʃʰ 10 | chw tʃʷʰ 11 | c tʃ 12 | cw tʃʷ 13 | d d 14 | dh dʱ 15 | dhl dɮ 16 | dl dˡ 17 | dlw dˡʷ 18 | dw dʷ 19 | dy dʲ 20 | dz dz 21 | dzh dzʱ 22 | dzhw dzʷʱ 23 | dzw dzᵝ 24 | ee ɛː 25 | e ɛ 26 | g ɡ 27 | gh ɡʱ 28 | ghw ɡʷʱ 29 | gq ᶢǀ 30 | gqw ᶢǀʷ 31 | gw ɡʷ 32 | i i 33 | ii iː 34 | j dʒ 35 | jh dʒʱ 36 | jw dʒʷ 37 | kh kʰ 38 | khw kʷʰ 39 | k k 40 | kw kʷ 41 | mh mʱ 42 | m m 43 | my mʲ 44 | nh nʱ 45 | n'h ŋʱ 46 | nhw nʷʱ 47 | n'hw ŋʷʱ 48 | n n 49 | n' ŋ 50 | nw nʷ 51 | n'w ŋʷ 52 | ny ɲ 53 | nyw ɲʷ 54 | oo ɔː 55 | o ɔ 56 | pfh p̪fʰ 57 | pf p̪f 58 | ph pʰ 59 | phy pʲʰ 60 | p p 61 | py pʲ 62 | qh ᵏǀʰ 63 | qhw ᵏǀʷʰ 64 | q ᵏǀ 65 | rh rʱ 66 | rhw rʷʱ 67 | r r 68 | rw rʷ 69 | th tʰ 70 | thw tʷʰ 71 | thy tʲʰ 72 | tlh tˡʰ 73 | tlhw tˡʷʰ 74 | tl tˡ 75 | tlw tˡʷ 76 | tsh tsʰ 77 | tsh tsᶲʰ 78 | ts ts 79 | tsw tsᶲ 80 | tsw tsʷ 81 | t t 82 | tw tʷ 83 | ty tʲ 84 | u u 85 | uu uː 86 | wh wʱ 87 | w w 88 | yh jʱ 89 | y j 90 | yw jʷ 91 | -------------------------------------------------------------------------------- /cvutils/data/tt/alphabet.txt: -------------------------------------------------------------------------------- 1 | аәбвгдеёжҗзийклмнңоөпрстуүфхһцчшщъыьэюя 2 | -------------------------------------------------------------------------------- /cvutils/data/tt/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | а a 3 | ә æ 4 | б b 5 | в w 6 | в v 7 | г ɡ 8 | г ɣ 9 | д d 10 | е je 11 | е jɤ 12 | е e 13 | ё jo 14 | ж ʒ 15 | җ ʑ 16 | з z 17 | и i 18 | й j 19 | к k 20 | к q 21 | л l 22 | м m 23 | н n 24 | ң ŋ 25 | о o 26 | ө ø 27 | п p 28 | р r 29 | с s 30 | т t 31 | у u 32 | у w 33 | ү y 34 | ү w 35 | ф f 36 | х x 37 | һ h 38 | ц t͡s 39 | ч ɕ 40 | ш ʃ 41 | щ ʃɕ 42 | ъ ʔ 43 | ы ɤ 44 | ь ʔ 45 | э e 46 | э ʔ 47 | ю ju 48 | ю jy 49 | я ja 50 | я jæ 51 | -------------------------------------------------------------------------------- /cvutils/data/tt/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/tt/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | REPL ! _ 3 | REPL " _ 4 | REPL ' _ 5 | REPL , _ 6 | REPL - _ 7 | REPL . _ 8 | REPL : _ 9 | REPL ; _ 10 | REPL ? _ 11 | REPL ` _ 12 | REPL « _ 13 | REPL » _ 14 | REPL – _ 15 | REPL — _ 16 | REPL ’ _ 17 | REPL “ _ 18 | REPL ” _ 19 | REPL … _ 20 | ALLOW а _ 21 | ALLOW ә _ 22 | ALLOW б _ 23 | ALLOW в _ 24 | ALLOW г _ 25 | ALLOW д _ 26 | ALLOW е _ 27 | ALLOW ё _ 28 | ALLOW ж _ 29 | ALLOW җ _ 30 | ALLOW з _ 31 | ALLOW и _ 32 | ALLOW й _ 33 | ALLOW к _ 34 | ALLOW л _ 35 | ALLOW м _ 36 | ALLOW н _ 37 | ALLOW ң _ 38 | ALLOW о _ 39 | ALLOW ө _ 40 | ALLOW п _ 41 | ALLOW р _ 42 | ALLOW с _ 43 | ALLOW т _ 44 | ALLOW у _ 45 | ALLOW ү _ 46 | ALLOW ф _ 47 | ALLOW х _ 48 | ALLOW һ _ 49 | ALLOW ц _ 50 | ALLOW ч _ 51 | ALLOW ш _ 52 | ALLOW щ _ 53 | ALLOW ъ _ 54 | ALLOW ы _ 55 | ALLOW ь _ 56 | ALLOW э _ 57 | ALLOW ю _ 58 | ALLOW я _ 59 | ALLOW _ _ 60 | -------------------------------------------------------------------------------- /cvutils/data/tt/vocab.tsv: -------------------------------------------------------------------------------- 1 | ноль 2 | бер 3 | ике 4 | өч 5 | дүрт 6 | биш 7 | алты 8 | җиде 9 | сигез 10 | тугыз 11 | әйе 12 | юк 13 | -------------------------------------------------------------------------------- /cvutils/data/tw/alphabet.txt: -------------------------------------------------------------------------------- 1 | abdeɛfghiklmnoɔprstuwycjvzxq 2 | -------------------------------------------------------------------------------- /cvutils/data/tw/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | e e 4 | i i 5 | o o 6 | u u 7 | ɛ ɛ 8 | ɔ ɔ 9 | m m 10 | n n 11 | ny ɲ 12 | n ɲ 13 | ng ŋ 14 | n ŋ 15 | nw nʷ 16 | b b 17 | d d 18 | dw d͡ʒ 19 | gy ɟ͡ʝ 20 | g g 21 | p pʰ 22 | t tʰ 23 | ky c͡çʰ 24 | k kʰ 25 | tw t͡ɕʷ 26 | kw kʷ 27 | f f 28 | s s 29 | hy ç 30 | h h 31 | hw hʷ 32 | y j 33 | w w 34 | r ɾ 35 | r ɽ 36 | r r 37 | l l 38 | -------------------------------------------------------------------------------- /cvutils/data/tw/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | DEL   _ 2006 _ 4 | REPL ʼ _ 02bc _ 5 | REPL ! _ 0021 _ 6 | REPL – _ 2013 _ 7 | REPL — _ 2014 _ 8 | REPL ‘ _ 2018 _ 9 | REPL ’ _ 2019 _ 10 | REPL “ _ 201c _ 11 | REPL ” _ 201d _ 12 | REPL … _ 2026 _ 13 | REPL , _ 002c _ 14 | REPL - _ 002d _ 15 | REPL . _ 002e _ 16 | REPL : _ 003a _ 17 | REPL ; _ 003b _ 18 | REPL ? _ 003f _ 19 | NORM ã a 00e3 _ 20 | NORM õ o 00f5 _ 21 | SKIP ( _ 0028 _ 22 | SKIP ) _ 0029 _ 23 | SKIP * _ 002a _ 24 | SKIP + _ 002b _ 25 | SKIP \ _ 005c _ 26 | SKIP 0 _ 0030 _ 27 | SKIP 1 _ 0031 _ 28 | SKIP 2 _ 0032 _ 29 | SKIP 3 _ 0033 _ 30 | SKIP 4 _ 0034 _ 31 | SKIP 5 _ 0035 _ 32 | SKIP 6 _ 0036 _ 33 | SKIP 7 _ 0037 _ 34 | SKIP 8 _ 0038 _ 35 | SKIP 9 _ 0039 _ 36 | ALLOW _ _ 0020 _ 37 | ALLOW a _ 0061 _ 38 | ALLOW b _ 0062 _ 39 | ALLOW c _ 0063 _ 40 | ALLOW d _ 0064 _ 41 | ALLOW e _ 0065 _ 42 | ALLOW ɛ _ 025b _ 43 | ALLOW f _ 0066 _ 44 | ALLOW g _ 0067 _ 45 | ALLOW h _ 0068 _ 46 | ALLOW i _ 0069 _ 47 | ALLOW j _ 006a _ 48 | ALLOW k _ 006b _ 49 | ALLOW l _ 006c _ 50 | ALLOW m _ 006d _ 51 | ALLOW n _ 006e _ 52 | ALLOW o _ 006f _ 53 | ALLOW ɔ _ 0254 _ 54 | ALLOW p _ 0070 _ 55 | ALLOW q _ 0071 _ 56 | ALLOW r _ 0072 _ 57 | ALLOW s _ 0073 _ 58 | ALLOW t _ 0074 _ 59 | ALLOW u _ 0075 _ 60 | ALLOW v _ 0076 _ 61 | ALLOW w _ 0077 _ 62 | ALLOW x _ 0078 _ 63 | ALLOW y _ 0079 _ 64 | ALLOW z _ 007a _ 65 | -------------------------------------------------------------------------------- /cvutils/data/ug/alphabet.txt: -------------------------------------------------------------------------------- 1 | ئائەئەبپتجچخدرزژسشغفقكگڭلمنھئوئۇئۆئۈۋۋئېئىئىي -------------------------------------------------------------------------------- /cvutils/data/ug/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | ا ɑ 3 | ە ɛ 4 | ە æ 5 | ب b 6 | پ p 7 | ت t 8 | ج dʒ 9 | چ tʃ 10 | خ χ 11 | د d 12 | ر r 13 | ز z 14 | ژ ʒ 15 | س s 16 | ش ʃ 17 | غ ʁ 18 | ف f 19 | ق q 20 | ك k 21 | گ ɡ 22 | ڭ ŋ 23 | ل l 24 | م m 25 | ن n 26 | ھ h 27 | و o 28 | ۇ u 29 | ۆ ø 30 | ۈ y 31 | ۋ v 32 | ۋ w 33 | ې e 34 | ى ɪ 35 | ى i 36 | ي j 37 | -------------------------------------------------------------------------------- /cvutils/data/ug/validate.tsv: -------------------------------------------------------------------------------- 1 | NFKC _ _ _ _ 2 | REPL ! _ 0021 _ 3 | REPL " _ 0022 _ 4 | REPL , _ 002c _ 5 | REPL - _ 002d _ 6 | REPL . _ 002e _ 7 | REPL : _ 003a _ 8 | REPL « _ 00ab _ 9 | REPL » _ 00bb _ 10 | REPL ، _ 060c _ 11 | REPL ؛ _ 061b _ 12 | REPL ؟ _ 061f _ 13 | REPL — _ 2014 _ 14 | REPL “ _ 201c _ 15 | REPL ” _ 201d _ 16 | REPL ‹ _ 2039 _ 17 | REPL › _ 203a _ 18 | REPL − _ 2212 _ 19 | ALLOW ئ _ 0626 _ 20 | ALLOW ا _ 0627 _ 21 | ALLOW ب _ 0628 _ 22 | ALLOW پ _ 067e _ 23 | ALLOW ت _ 062a _ 24 | ALLOW ج _ 062c _ 25 | ALLOW چ _ 0686 _ 26 | ALLOW خ _ 062e _ 27 | ALLOW د _ 062f _ 28 | ALLOW ر _ 0631 _ 29 | ALLOW ز _ 0632 _ 30 | ALLOW ژ _ 0698 _ 31 | ALLOW س _ 0633 _ 32 | ALLOW ش _ 0634 _ 33 | ALLOW غ _ 063a _ 34 | ALLOW ف _ 0641 _ 35 | ALLOW ق _ 0642 _ 36 | ALLOW ك _ 0643 _ 37 | ALLOW ڭ _ 06ad _ 38 | ALLOW گ _ 06af _ 39 | ALLOW ل _ 0644 _ 40 | ALLOW م _ 0645 _ 41 | ALLOW ن _ 0646 _ 42 | ALLOW ھ _ 06be _ 43 | ALLOW ە _ 06d5 _ 44 | ALLOW و _ 0648 _ 45 | ALLOW ۆ _ 06c6 _ 46 | ALLOW ۇ _ 06c7 _ 47 | ALLOW ۈ _ 06c8 _ 48 | ALLOW ۋ _ 06cb _ 49 | ALLOW ى _ 0649 _ 50 | ALLOW ي _ 064a _ 51 | ALLOW ې _ 06d0 _ 52 | ALLOW _ _ 0020 _ 53 | -------------------------------------------------------------------------------- /cvutils/data/uk/alphabet.txt: -------------------------------------------------------------------------------- 1 | абвгґдеєжзиіїйклмнопрстуфхцчшщьюя-ʼ 2 | -------------------------------------------------------------------------------- /cvutils/data/uk/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | а ɑ 3 | б b 4 | в v 5 | г ɦ 6 | ґ ɡ 7 | д d 8 | е ɛ 9 | є ɛ 10 | ж ʐ 11 | з z 12 | и ɪ 13 | і i 14 | ї ji 15 | й j 16 | к k 17 | л l 18 | м m 19 | н n 20 | о ɔ 21 | п p 22 | р r 23 | с s 24 | т t 25 | у u 26 | ф f 27 | х x 28 | ц t͡s 29 | ч ʈ͡ʂ 30 | ш ʂ 31 | щ ʂʈ͡ʂ 32 | ь ʲ 33 | ю u 34 | я ɑ 35 | -------------------------------------------------------------------------------- /cvutils/data/uk/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | REPL " _ 0022 _ 3 | REPL , _ 002c _ 4 | REPL - _ 002d _ 5 | REPL . _ 002e _ 6 | REPL : _ 003a _ 7 | REPL ; _ 003b _ 8 | REPL ? _ 003f _ 9 | REPL ! _ 0021 _ 10 | REPL « _ 00ab _ 11 | REPL » _ 00bb _ 12 | REPL – _ 2013 _ 13 | REPL — _ 2014 _ 14 | REPL “ _ 201c _ 15 | REPL ” _ 201d _ 16 | REPL … _ 2026 _ 17 | REPL ! _ 0021 _ 18 | NORM ’ ʼ _ _ 19 | NORM ' ʼ _ _ 20 | NORM ` ʼ _ _ 21 | NORM i і 0069 0456 22 | SKIP ы _ 044b _ 23 | ALLOW а _ 0430 _ 24 | ALLOW б _ 0431 _ 25 | ALLOW в _ 0432 _ 26 | ALLOW г _ 0433 _ 27 | ALLOW ґ _ 0491 _ 28 | ALLOW д _ 0434 _ 29 | ALLOW е _ 0435 _ 30 | ALLOW є _ 0454 _ 31 | ALLOW ж _ 0436 _ 32 | ALLOW з _ 0437 _ 33 | ALLOW и _ 0438 _ 34 | ALLOW і _ 0456 _ 35 | ALLOW ї _ 0457 _ 36 | ALLOW й _ 0439 _ 37 | ALLOW к _ 043a _ 38 | ALLOW л _ 043b _ 39 | ALLOW м _ 043c _ 40 | ALLOW н _ 043d _ 41 | ALLOW о _ 043e _ 42 | ALLOW п _ 043f _ 43 | ALLOW р _ 0440 _ 44 | ALLOW с _ 0441 _ 45 | ALLOW т _ 0442 _ 46 | ALLOW у _ 0443 _ 47 | ALLOW ф _ 0444 _ 48 | ALLOW х _ 0445 _ 49 | ALLOW ц _ 0446 _ 50 | ALLOW ч _ 0447 _ 51 | ALLOW ш _ 0448 _ 52 | ALLOW щ _ 0449 _ 53 | ALLOW ы _ 044b _ 54 | ALLOW ь _ 044c _ 55 | ALLOW ю _ 044e _ 56 | ALLOW я _ 044f _ 57 | ALLOW - _ 002d _ 58 | ALLOW ʼ _ 02bc _ 59 | ALLOW _ _ 005f _ 60 | -------------------------------------------------------------------------------- /cvutils/data/ur/alphabet.txt: -------------------------------------------------------------------------------- 1 | أكإںهآيةؤىابپتٹثجچحخدڈذرڑزژسشصضطظعغفقکگلمنےئءوہھی۶ٴۂۃۓ -------------------------------------------------------------------------------- /cvutils/data/ur/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | ا ɑː 3 | ب b 4 | پ p 5 | ت t 6 | ٹ ʈ 7 | ث s 8 | ج d͡ʒ 9 | چ t͡ʃ 10 | ح ɦ 11 | خ x 12 | د d 13 | ڈ ɖ 14 | ذ z 15 | ر r 16 | ڑ ɽ 17 | ز z 18 | ژ ʒ 19 | س s 20 | ش ʃ 21 | ص s 22 | ض z 23 | ط t 24 | ظ z 25 | ع ɑː 26 | غ ɣ 27 | ف f 28 | ق q 29 | ک k 30 | گ ɡ 31 | ل l 32 | م m 33 | ن n 34 | ے ɛː 35 | ئ ʔ 36 | ء ʔ 37 | و ʋ 38 | ہ ɦ 39 | ھ ʰ 40 | ی j 41 | -------------------------------------------------------------------------------- /cvutils/data/ur/validate.tsv: -------------------------------------------------------------------------------- 1 | NFKC _ _ _ _ 2 | REPL ! _ 0021 _ 3 | REPL " _ 0022 _ 4 | REPL ^ _ 005e _ 5 | REPL ` _ 0060 _ 6 | REPL ' _ 0027 _ 7 | REPL , _ 002c _ 8 | REPL - _ 002d _ 9 | REPL . _ 002e _ 10 | REPL : _ 003a _ 11 | REPL ; _ 003b _ 12 | REPL ‘ _ 2018 _ 13 | REPL ? _ 003f _ 14 | REPL _ _ 005f _ 15 | REPL « _ 00ab _ 16 | REPL » _ 00bb _ 17 | REPL ، _ 060c _ 18 | REPL ؛ _ 061b _ 19 | REPL ؟ _ 061f _ 20 | REPL ـ _ 0640 _ 21 | REPL ۔ _ 06d4 _ 22 | REPL ‍ _ 200d _ 23 | REPL ‎ _ 200e _ 24 | REPL ‏ _ 200f _ 25 | REPL  _ feff _ 26 | SKIP ۰ _ 06f0 _ 27 | ALLOW ۶ _ 06f6 _ 28 | ALLOW ٴ _ 0674 _ 29 | ALLOW ۂ _ 06c2 _ 30 | ALLOW ۃ _ 06c3 _ 31 | ALLOW ۓ _ 06d3 _ 32 | ALLOW أ _ 0623 _ 33 | ALLOW ك _ 0643 _ 34 | ALLOW إ _ 0625 _ 35 | ALLOW ں _ 06ba _ 36 | ALLOW ه _ 0647 _ 37 | ALLOW آ _ 0622 _ 38 | ALLOW ي _ 064a _ 39 | ALLOW ة _ 0629 _ 40 | ALLOW ؤ _ 0624 _ 41 | ALLOW ى _ 0649 _ 42 | ALLOW ا _ 0627 _ 43 | ALLOW ب _ 0628 _ 44 | ALLOW پ _ 067e _ 45 | ALLOW ت _ 062a _ 46 | ALLOW ٹ _ 0679 _ 47 | ALLOW ث _ 062b _ 48 | ALLOW ج _ 062c _ 49 | ALLOW چ _ 0686 _ 50 | ALLOW ح _ 062d _ 51 | ALLOW خ _ 062e _ 52 | ALLOW د _ 062f _ 53 | ALLOW ڈ _ 0688 _ 54 | ALLOW ذ _ 0630 _ 55 | ALLOW ر _ 0631 _ 56 | ALLOW ڑ _ 0691 _ 57 | ALLOW ز _ 0632 _ 58 | ALLOW ژ _ 0698 _ 59 | ALLOW س _ 0633 _ 60 | ALLOW ش _ 0634 _ 61 | ALLOW ص _ 0635 _ 62 | ALLOW ض _ 0636 _ 63 | ALLOW ط _ 0637 _ 64 | ALLOW ظ _ 0638 _ 65 | ALLOW ع _ 0639 _ 66 | ALLOW غ _ 063a _ 67 | ALLOW ف _ 0641 _ 68 | ALLOW ق _ 0642 _ 69 | ALLOW ک _ 06a9 _ 70 | ALLOW گ _ 06af _ 71 | ALLOW ل _ 0644 _ 72 | ALLOW م _ 0645 _ 73 | ALLOW ن _ 0646 _ 74 | ALLOW ے _ 06d2 _ 75 | ALLOW ئ _ 0626 _ 76 | ALLOW ء _ 0621 _ 77 | ALLOW و _ 0648 _ 78 | ALLOW ہ _ 06c1 _ 79 | ALLOW ھ _ 06be _ 80 | ALLOW ی _ 06cc _ 81 | ALLOW ۶ _ 06f6 _ 82 | ALLOW ٴ _ 0674 _ 83 | ALLOW ۂ _ 06c2 _ 84 | ALLOW ۃ _ 06c3 _ 85 | ALLOW ۓ _ 06d3 _ 86 | ALLOW _ _ 0020 _ 87 | -------------------------------------------------------------------------------- /cvutils/data/uz/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnopqrstuvxyzʻʼ 2 | -------------------------------------------------------------------------------- /cvutils/data/uz/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | a æ 4 | b b 5 | d d̪ 6 | e e 7 | f ɸ 8 | g ɡ 9 | h h 10 | i i 11 | i ɨ 12 | j dʒ 13 | j ʒ 14 | k k 15 | l l 16 | m m 17 | n n 18 | o ɒ 19 | o o 20 | p p 21 | q q 22 | q x 23 | r r 24 | s s 25 | t t̪ 26 | u u 27 | u ʉ 28 | v v 29 | v w 30 | x x 31 | y j 32 | z z 33 | oʻ o 34 | oʻ ɵ 35 | gʻ ʁ 36 | sh ʃ 37 | ch tʃ 38 | ng ŋ 39 | ts ts 40 | ʼ ʔ 41 | -------------------------------------------------------------------------------- /cvutils/data/uz/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL , _ 002c _ 5 | REPL - _ 002d _ 6 | REPL . _ 002e _ 7 | REPL ? _ 003f _ 8 | NORM а a 0430 0061 9 | NORM е e 0435 0065 10 | NORM о o 043e 006f 11 | NORM ‘ ʻ 2018 02bb 12 | NORM ’ ʼ 2019 02bc 13 | ALLOW a _ 0061 _ 14 | ALLOW b _ 0062 _ 15 | ALLOW c _ 0063 _ 16 | ALLOW d _ 0064 _ 17 | ALLOW e _ 0065 _ 18 | ALLOW f _ 0066 _ 19 | ALLOW g _ 0067 _ 20 | ALLOW h _ 0068 _ 21 | ALLOW i _ 0069 _ 22 | ALLOW j _ 006a _ 23 | ALLOW k _ 006b _ 24 | ALLOW l _ 006c _ 25 | ALLOW m _ 006d _ 26 | ALLOW n _ 006e _ 27 | ALLOW o _ 006f _ 28 | ALLOW p _ 0070 _ 29 | ALLOW q _ 0071 _ 30 | ALLOW r _ 0072 _ 31 | ALLOW s _ 0073 _ 32 | ALLOW t _ 0074 _ 33 | ALLOW u _ 0075 _ 34 | ALLOW v _ 0076 _ 35 | ALLOW x _ 0078 _ 36 | ALLOW y _ 0079 _ 37 | ALLOW z _ 007a _ 38 | ALLOW ʻ _ 02bb _ 39 | ALLOW ʼ _ 02bc _ 40 | ALLOW _ _ 0020 _ 41 | -------------------------------------------------------------------------------- /cvutils/data/ve/alphabet.txt: -------------------------------------------------------------------------------- 1 | abdḓefghijklḽmnṅṋoprstṱuvwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/ve/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | a ɔ 4 | b b 5 | bv b̪v 6 | bw bɣʷ 7 | bw bj 8 | d d 9 | dz d͡z 10 | dzh d͡ʒ 11 | dzw d͡zʷ 12 | ḓ d̪ 13 | e ɛ 14 | e e 15 | f f 16 | fh ɸ 17 | g ɡ 18 | h ɦ 19 | h h 20 | hw ɣʷ 21 | hw hʷ 22 | i i 23 | j j 24 | k kʼ 25 | kh kʰ 26 | khw kʷʰ 27 | l l 28 | ḽ l̪ 29 | m m 30 | m m̩ 31 | n n 32 | n n̩ 33 | ng ŋɡ 34 | ny ɲ 35 | nz nd͡z 36 | ṋ n̪ 37 | ṅ ŋ 38 | ṅw ŋʷ 39 | o ɔ 40 | o o 41 | p pʼ 42 | ph pʰ 43 | pf p̪f 44 | pfh p̪fʰ 45 | r ɾ 46 | s s 47 | sh ʃ 48 | sw ʂ 49 | t tʼ 50 | th tʰ 51 | ths t͡sʰ 52 | thsh t͡ʃʰ 53 | ts t͡sʼ 54 | tsh t͡ʃʼ 55 | tsw t͡sʷ 56 | ty c 57 | ṱ t̪ʼ 58 | ṱh t̪ʰ 59 | u u 60 | v v 61 | vh β 62 | w w 63 | x x 64 | xw xʷ 65 | y j 66 | z z 67 | zh ʒ 68 | zw ʐ 69 | -------------------------------------------------------------------------------- /cvutils/data/vi/abbr.tsv: -------------------------------------------------------------------------------- 1 | 1339 v.v. 2 | 15 l.i. 3 | 13 v.d. 4 | 10 i.e. 5 | 9 a.u.c. 6 | 8 p.m. 7 | 7 e.g. 8 | 6 a.m. 9 | 5 a.k.a. 10 | 4 n.v.v. 11 | 4 n.m. 12 | 4 g.v.v. 13 | 4 f.o. 14 | 3 o.v.v. 15 | 3 i.v.v. 16 | 3 c.v.v. 17 | 3 a.v.v. 18 | 2 x.x.x. 19 | 2 t.v.v. 20 | 2 s.l. 21 | 2 m.v.v. 22 | 1 x.y. 23 | 1 w.x. 24 | 1 v.x. 25 | 1 ử.v.v. 26 | 1 ử.v. 27 | 1 u.a. 28 | 1 t.c. 29 | 1 s.s. 30 | 1 s.r.o. 31 | 1 s.p. 32 | 1 r.v.v. 33 | 1 q.v. 34 | 1 p.v. 35 | 1 p.s.i. 36 | 1 n.v. 37 | 1 n.r. 38 | 1 n.p. 39 | 1 m.p.h. 40 | 1 m.p. 41 | 1 m.h. 42 | 1 l.y. 43 | 1 k.v.v. 44 | 1 k.f. 45 | 1 i.x. 46 | 1 i.i.d. 47 | 1 h.v.v. 48 | 1 g.o.o.g.l. 49 | 1 g.m. 50 | 1 e.d. 51 | 1 e.c. 52 | 1 đ.v.t. 53 | 1 c.v. 54 | 1 b.p. 55 | 1 a.n. 56 | -------------------------------------------------------------------------------- /cvutils/data/vi/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghiklmnopqrstuvxyđ̛̣̀́̂̃̆̉ -------------------------------------------------------------------------------- /cvutils/data/vi/punct.tsv: -------------------------------------------------------------------------------- 1 | EOS ! 2 | EOS ? 3 | EOS . 4 | -------------------------------------------------------------------------------- /cvutils/data/vi/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKD _ _ _ _ 3 | REPL " _ 0022 _ 4 | REPL ' _ 0027 _ 5 | REPL , _ 002c _ 6 | REPL - _ 002d _ 7 | REPL . _ 002e _ 8 | REPL : _ 003a _ 9 | REPL ; _ 003b _ 10 | REPL ? _ 003f _ 11 | REPL ! _ 0021 _ 12 | ALLOW a _ 0061 _ 13 | ALLOW b _ 0062 _ 14 | ALLOW c _ 0063 _ 15 | ALLOW d _ 0064 _ 16 | ALLOW e _ 0065 _ 17 | ALLOW f _ 0066 _ 18 | ALLOW g _ 0067 _ 19 | ALLOW h _ 0068 _ 20 | ALLOW i _ 0069 _ 21 | ALLOW k _ 006b _ 22 | ALLOW l _ 006c _ 23 | ALLOW m _ 006d _ 24 | ALLOW n _ 006e _ 25 | ALLOW o _ 006f _ 26 | ALLOW p _ 0070 _ 27 | ALLOW q _ 0071 _ 28 | ALLOW r _ 0072 _ 29 | ALLOW s _ 0073 _ 30 | ALLOW t _ 0074 _ 31 | ALLOW u _ 0075 _ 32 | ALLOW v _ 0076 _ 33 | ALLOW x _ 0078 _ 34 | ALLOW y _ 0079 _ 35 | ALLOW đ _ 0111 _ 36 | ALLOW z _ 007a _ 37 | ALLOW ̀ _ 0300 _ 38 | ALLOW ́ _ 0301 _ 39 | ALLOW ̂ _ 0302 _ 40 | ALLOW ̃ _ 0303 _ 41 | ALLOW ̆ _ 0306 _ 42 | ALLOW ̉ _ 0309 _ 43 | ALLOW ̛ _ 031b _ 44 | ALLOW ̣ _ 0323 _ 45 | ALLOW _ _ 005f _ 46 | -------------------------------------------------------------------------------- /cvutils/data/vot/alphabet.txt: -------------------------------------------------------------------------------- 1 | aäbčdďefghijklľmnńoöõprsštuüvzž 2 | -------------------------------------------------------------------------------- /cvutils/data/vot/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL , _ 002c _ 4 | REPL . _ 002e _ 5 | ALLOW a _ 0061 _ 6 | ALLOW ä _ 00e4 _ 7 | ALLOW b _ 0062 _ 8 | ALLOW č _ 010d _ 9 | ALLOW d _ 0064 _ 10 | ALLOW ď _ 010f _ 11 | ALLOW e _ 0065 _ 12 | ALLOW f _ 0066 _ 13 | ALLOW g _ 0067 _ 14 | ALLOW h _ 0068 _ 15 | ALLOW i _ 0069 _ 16 | ALLOW j _ 006a _ 17 | ALLOW k _ 006b _ 18 | ALLOW l _ 006c _ 19 | ALLOW ľ _ 013e _ 20 | ALLOW m _ 006d _ 21 | ALLOW n _ 006e _ 22 | ALLOW ń _ 0144 _ 23 | ALLOW o _ 006f _ 24 | ALLOW õ _ 00f5 _ 25 | ALLOW ö _ 00f6 _ 26 | ALLOW p _ 0070 _ 27 | ALLOW r _ 0072 _ 28 | ALLOW s _ 0073 _ 29 | ALLOW š _ 0161 _ 30 | ALLOW t _ 0074 _ 31 | ALLOW u _ 0075 _ 32 | ALLOW ü _ 00fc _ 33 | ALLOW v _ 0076 _ 34 | ALLOW z _ 007a _ 35 | ALLOW ž _ 017e _ 36 | ALLOW _ _ 0020 _ 37 | -------------------------------------------------------------------------------- /cvutils/data/vot/wikipedia.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ftyers/commonvoice-utils/5e3634fe1832aa58602f551156f7871cc142beb5/cvutils/data/vot/wikipedia.txt -------------------------------------------------------------------------------- /cvutils/data/wo/alphabet.txt: -------------------------------------------------------------------------------- 1 | aàbcdeéëfgijklmnñŋoópqrstuwxy 2 | -------------------------------------------------------------------------------- /cvutils/data/wo/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | à aː 4 | e ɛ 5 | ee ɛː 6 | é e 7 | éé eː 8 | ë ə 9 | i i 10 | ii iː 11 | o ɔ 12 | oo ɔː 13 | ó o 14 | óó oː 15 | u u 16 | uu uː 17 | b b 18 | c c 19 | d d 20 | f f 21 | g ɡ 22 | j ɟ 23 | k k 24 | kk kː 25 | l l 26 | m m 27 | mb mb 28 | n n 29 | nd nd 30 | ng ŋɡ 31 | nj ɲɟ 32 | ñ ɲ 33 | ŋ ŋ 34 | p p 35 | q q 36 | r r 37 | s s 38 | t t 39 | w w 40 | x x 41 | x χ 42 | y j 43 | -------------------------------------------------------------------------------- /cvutils/data/wo/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ 2 | NFKC _ _ 3 | ALLOW a _ 4 | ALLOW à _ 5 | ALLOW b _ 6 | ALLOW c _ 7 | ALLOW d _ 8 | ALLOW e _ 9 | ALLOW é _ 10 | ALLOW ë _ 11 | ALLOW f _ 12 | ALLOW g _ 13 | ALLOW i _ 14 | ALLOW j _ 15 | ALLOW k _ 16 | ALLOW l _ 17 | ALLOW m _ 18 | ALLOW n _ 19 | ALLOW ñ _ 20 | ALLOW ŋ _ 21 | ALLOW o _ 22 | ALLOW ó _ 23 | ALLOW p _ 24 | ALLOW q _ 25 | ALLOW r _ 26 | ALLOW s _ 27 | ALLOW t _ 28 | ALLOW u _ 29 | ALLOW w _ 30 | ALLOW x _ 31 | ALLOW y _ 32 | ALLOW _ _ 33 | -------------------------------------------------------------------------------- /cvutils/data/xh/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnopqrstuvwxyzʼ 2 | -------------------------------------------------------------------------------- /cvutils/data/xh/phon.tsv: -------------------------------------------------------------------------------- 1 | a a 2 | aa aː 3 | b ɓ 4 | bh b̥ʱ 5 | ch ᵏǀʰ 6 | c ᵏǀʼ 7 | d d̥ʱ 8 | dl ɮ̈ 9 | dy d̠̥ʲʱ 10 | dz d̥zʱ 11 | ee eː 12 | e ɛ 13 | f f 14 | gc ᶢ̥ǀʱ 15 | g ɡ̊ʱ 16 | gq ᶢ̥ǃʱ 17 | gr ɣ̈ 18 | gx ᶢ̥ǁʱ 19 | h h 20 | hh ɦ 21 | hl ɬ 22 | i i 23 | ii iː 24 | j d̥ʒʱ 25 | kh kʰ 26 | k kʼ 27 | kl kʟ̥ʼ 28 | krh kxʰ 29 | kr kxʼ 30 | lh l̤ 31 | l l 32 | mh m̤ 33 | m m 34 | nc ᵑǀ 35 | ngc ᵑǀʱ 36 | ngh ŋ̈ 37 | ngq ᵑǃʱ 38 | ngx ᵑǁʱ 39 | ngʼ ŋ 40 | nh n̤ 41 | nkc ŋᵏǀʼ 42 | nkq ŋᵏǃʼ 43 | nkx ŋᵏǁʼ 44 | n n 45 | nq ᵑǃ 46 | nx ᵑǁ 47 | nyh n̠̈ʲ 48 | ny n̠ʲ 49 | oo oː 50 | o ɔ 51 | ph pʰ 52 | p pʼ 53 | qh ᵏǃʰ 54 | q ᵏǃʼ 55 | rh x 56 | r r 57 | r r̤ 58 | sh ʃ 59 | s s 60 | thsh tʃʰ 61 | ths tsʰ 62 | th tʰ 63 | tsh tʃʼ 64 | ts tsʼ 65 | t tʼ 66 | tyh t̠ʲʰ 67 | ty t̠ʲʼ 68 | u u 69 | uu uː 70 | v v̤ 71 | wh w̤ 72 | w w 73 | xh ᵏǁʰ 74 | x ᵏǁʼ 75 | yh j̤ 76 | y j 77 | zh ʒ̈ 78 | z z̤ 79 | -------------------------------------------------------------------------------- /cvutils/data/xty/alphabet.txt: -------------------------------------------------------------------------------- 1 | aáãbcdeéẽfghiíĩjklmnñoóõpqrstuúüũvwxyỹzʼ̃ 2 | -------------------------------------------------------------------------------- /cvutils/data/xty/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | REPL ! _ 0021 _ 4 | REPL " _ 0022 _ 5 | REPL , _ 002c _ 6 | REPL - _ 002d _ 7 | REPL . _ 002e _ 8 | REPL : _ 003a _ 9 | REPL ? _ 003f _ 10 | REPL * _ 002a _ 11 | REPL ( _ 0028 _ 12 | REPL ) _ 0029 _ 13 | DEL = _ 003d _ 14 | SKIP š _ 0161 _ 15 | SKIP ý _ 00fd _ 16 | SKIP ÿ _ 00ff _ 17 | SKIP [ _ 005b _ 18 | SKIP \ _ 005c _ 19 | SKIP ] _ 005d _ 20 | SKIP ¡ _ 00a1 _ 21 | SKIP ¿ _ 00bf _ 22 | SKIP ‘ _ 2018 _ 23 | NORM 1 ¹ _ 00b9 24 | NORM 2 ² _ 00b2 25 | NORM 3 ³ _ 00b3 26 | NORM 4 ⁴ _ 2074 27 | NORM ' ʼ 0027 02bc 28 | ALLOW _ 0303 _ 29 | DEL ¹ _ 00b9 _ 30 | DEL ² _ 00b2 _ 31 | DEL ³ _ 00b3 _ 32 | DEL ⁴ _ 2074 _ 33 | ALLOW ʼ _ 02bc _ 34 | ALLOW a _ 0061 _ 35 | ALLOW á _ 00e1 _ 36 | ALLOW ã _ 00e3 _ 37 | ALLOW b _ 0062 _ 38 | ALLOW c _ 0063 _ 39 | ALLOW d _ 0064 _ 40 | ALLOW e _ 0065 _ 41 | ALLOW é _ 00e9 _ 42 | ALLOW ẽ _ 1ebd _ 43 | ALLOW f _ 0066 _ 44 | ALLOW g _ 0067 _ 45 | ALLOW h _ 0068 _ 46 | ALLOW i _ 0069 _ 47 | ALLOW í _ 00ed _ 48 | ALLOW ĩ _ 0129 _ 49 | ALLOW j _ 006a _ 50 | ALLOW k _ 006b _ 51 | ALLOW l _ 006c _ 52 | ALLOW m _ 006d _ 53 | ALLOW n _ 006e _ 54 | ALLOW ñ _ 00f1 _ 55 | ALLOW o _ 006f _ 56 | ALLOW ó _ 00f3 _ 57 | ALLOW õ _ 00f5 _ 58 | ALLOW p _ 0070 _ 59 | ALLOW q _ 0071 _ 60 | ALLOW r _ 0072 _ 61 | ALLOW s _ 0073 _ 62 | ALLOW t _ 0074 _ 63 | ALLOW u _ 0075 _ 64 | ALLOW ú _ 00fa _ 65 | ALLOW ü _ 00fc _ 66 | ALLOW ũ _ 0169 _ 67 | ALLOW v _ 0076 _ 68 | ALLOW w _ 0077 _ 69 | ALLOW x _ 0078 _ 70 | ALLOW y _ 0079 _ 71 | ALLOW ỹ _ 1ef9 _ 72 | ALLOW z _ 007a _ 73 | ALLOW _ _ 0020 _ 74 | -------------------------------------------------------------------------------- /cvutils/data/yi/alphabet.txt: -------------------------------------------------------------------------------- 1 | אבגדהוױזחטיכךלמםנןסעפףצץקרשת'’ַָּֿ -------------------------------------------------------------------------------- /cvutils/data/yi/validate.tsv: -------------------------------------------------------------------------------- 1 | NKFC _ _ _ _ 2 | REPL " _ 0022 _ 3 | REPL , _ 002c _ 4 | REPL - _ 002d _ 5 | REPL . _ 002e _ 6 | REPL ? _ 003f _ 7 | ALLOW א _ 05d0 _ 8 | ALLOW ב _ 05d1 _ 9 | ALLOW ג _ 05d2 _ 10 | ALLOW ד _ 05d3 _ 11 | ALLOW ה _ 05d4 _ 12 | ALLOW ו _ 05d5 _ 13 | ALLOW ױ _ 05f1 _ 14 | ALLOW ז _ 05d6 _ 15 | ALLOW ח _ 05d7 _ 16 | ALLOW ט _ 05d8 _ 17 | ALLOW י _ 05d9 _ 18 | ALLOW כ _ 05db _ 19 | ALLOW ך _ 05da _ 20 | ALLOW ל _ 05dc _ 21 | ALLOW מ _ 05de _ 22 | ALLOW ם _ 05dd _ 23 | ALLOW נ _ 05e0 _ 24 | ALLOW ן _ 05df _ 25 | ALLOW ס _ 05e1 _ 26 | ALLOW ע _ 05e2 _ 27 | ALLOW פ _ 05e4 _ 28 | ALLOW ף _ 05e3 _ 29 | ALLOW צ _ 05e6 _ 30 | ALLOW ץ _ 05e5 _ 31 | ALLOW ק _ 05e7 _ 32 | ALLOW ר _ 05e8 _ 33 | ALLOW ש _ 05e9 _ 34 | ALLOW ת _ 05ea _ 35 | ALLOW ' _ 0027 _ 36 | ALLOW ’ _ 2019 _ 37 | ALLOW ַ _ 05b7 _ 38 | ALLOW ָ _ 05b8 _ 39 | ALLOW ּ _ 05bc _ 40 | ALLOW ֿ _ 05bf _ 41 | ALLOW _ _ 0020 _ 42 | -------------------------------------------------------------------------------- /cvutils/data/yo/alphabet.txt: -------------------------------------------------------------------------------- 1 | 'aeẹioọubdfghjklmnnpprsṣtwỳ́ 2 | -------------------------------------------------------------------------------- /cvutils/data/yo/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | e e 4 | ẹ ɛ 5 | i i 6 | o o 7 | ọ ɔ 8 | u u 9 | aa aː 10 | ee eː 11 | ẹẹ ɛː 12 | ii iː 13 | oo oː 14 | ọọ ɔː 15 | uu uː 16 | b b 17 | d d 18 | f f 19 | g g 20 | gb ɡ͡b 21 | h h 22 | j d͡ʒ 23 | k k 24 | l l 25 | m m 26 | n n 27 | n ŋ̍ 28 | p k͜p 29 | p p 30 | r ɾ 31 | s s 32 | ṣ ʃ 33 | t t 34 | w w 35 | y j 36 | ̀ ˨ 37 | ́ ˦ 38 | -------------------------------------------------------------------------------- /cvutils/data/yo/validate.tsv: -------------------------------------------------------------------------------- 1 | LOWER _ _ _ _ 2 | NFKC _ _ _ _ 3 | NORM á á 00e1 0061-0301 4 | NORM é é 00e9 0065-0301 5 | NORM í í 00ed 0069-0301 6 | NORM ó ó 00f3 006f-0301 7 | NORM ú ú 00fa 0075-0301 8 | NORM à à 00e0 0061-0300 9 | NORM è è 00e8 0065-0300 10 | NORM ì ì 00ec 0069-0300 11 | NORM ò ò 00f2 006f-0300 12 | NORM ù ù 00f9 0075-0300 13 | NORM ǹ ǹ 01f9 006e-0300 14 | NORM ń ń 0144 006e-0301 15 | NORM ‘ ' 2018 0027 16 | NORM ’ ' 2019 0027 17 | REPL ! _ 0021 _ 18 | REPL , _ 002c _ 19 | REPL - _ 002d _ 20 | REPL . _ 002e _ 21 | REPL ; _ 003b _ 22 | REPL ? _ 003f _ 23 | REPL — _ 2014 _ 24 | REPL “ _ 201c _ 25 | REPL ” _ 201d _ 26 | ALLOW ' _ 0027 _ 27 | ALLOW a _ 0061 _ 28 | ALLOW e _ 0065 _ 29 | ALLOW ẹ _ 1eb9 _ 30 | ALLOW i _ 0069 _ 31 | ALLOW o _ 006f _ 32 | ALLOW ọ _ 1ecd _ 33 | ALLOW u _ 0075 _ 34 | ALLOW b _ 0062 _ 35 | ALLOW d _ 0064 _ 36 | ALLOW f _ 0066 _ 37 | ALLOW g _ 0067 _ 38 | ALLOW h _ 0068 _ 39 | ALLOW j _ 006a _ 40 | ALLOW k _ 006b _ 41 | ALLOW l _ 006c _ 42 | ALLOW m _ 006d _ 43 | ALLOW n _ 006e _ 44 | ALLOW p _ 0070 _ 45 | ALLOW r _ 0072 _ 46 | ALLOW s _ 0073 _ 47 | ALLOW ṣ _ 1e63 _ 48 | ALLOW t _ 0074 _ 49 | ALLOW w _ 0077 _ 50 | ALLOW y _ 0079 _ 51 | ALLOW ̀ _ 0300 _ 52 | ALLOW ́ _ 0301 _ 53 | ALLOW _ _ _ _ 54 | -------------------------------------------------------------------------------- /cvutils/data/zgh/alphabet.txt: -------------------------------------------------------------------------------- 1 | ⴰⴱⴲⴳⴴⴵⴶⴷⴸⴹⴺⴻⴼⴽⴾⴿⵀⵁⵂⵃⵄⵅⵆⵇⵈⵉⵊⵋⵌⵎⵏⵐⵑⵒⵓⵔⵕⵖⵗⵘⵙⵚⵛⵜⵝⵞⵟⵠⵡⵢⵣⵤⵥⵦⵧⵯ -------------------------------------------------------------------------------- /cvutils/data/zu/alphabet.txt: -------------------------------------------------------------------------------- 1 | abcdefghijklmnopqrstuvwxyz 2 | -------------------------------------------------------------------------------- /cvutils/data/zu/phon.tsv: -------------------------------------------------------------------------------- 1 | Orth Phon 2 | a a 3 | b ɓ 4 | bh b 5 | c ǀ 6 | ch ǀʰ 7 | d d 8 | dl ɮ 9 | e e 10 | f f 11 | g ɡ 12 | gc ᶢǀʱ 13 | gq ᶢǃʱ 14 | gx ᶢǁʱ 15 | h h 16 | hh ɦ 17 | hl ɬ 18 | i i 19 | j dʒ 20 | k k 21 | k ɠ 22 | kh kʰ 23 | kl kx 24 | l l 25 | m m 26 | m mʱ 27 | mb mb 28 | n n 29 | n nʱ 30 | nc ᵑǀ 31 | ng ŋ(ɡ) 32 | ngc ᵑǀʱ 33 | ngq ᵑǃʱ 34 | ngx ᵑǁʱ 35 | nj ɲdʒ 36 | nk ŋk 37 | nq ᵑǃ 38 | ntsh ɲtʃ 39 | nx ᵑǁ 40 | ny ɲ 41 | o o 42 | p p 43 | ph pʰ 44 | q ǃ 45 | qh ǃʰ 46 | r r 47 | s s 48 | sh ʃ 49 | t t 50 | th tʰ 51 | ts ts 52 | tsh tʃ 53 | u u 54 | v v 55 | w w 56 | w wʱ 57 | x ǁ 58 | xh ǁʰ 59 | y j 60 | y jʱ 61 | z z 62 | -------------------------------------------------------------------------------- /cvutils/exporters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ftyers/commonvoice-utils/5e3634fe1832aa58602f551156f7871cc142beb5/cvutils/exporters/__init__.py -------------------------------------------------------------------------------- /cvutils/opus.weights: -------------------------------------------------------------------------------- 1 | 4 OPUS-OpenSubtitles _ 2 | 3 OPUS-TED2020 _ 3 | 3 OPUS-TED2013 _ 4 | 2 OPUS-Tatoeba _ 5 | 2 OPUS-GlobalVoices _ 6 | 2 OPUS-Europarl _ 7 | 1 OPUS-wikimedia _ 8 | 1 OPUS-WMT-News _ 9 | 1 OPUS-UN _ 10 | 1 OPUS-News-Commentary _ 11 | 1 OPUS-Books _ 12 | 0 OPUS-Wikipedia _ 13 | 0 OPUS-WikiMatrix _ 14 | 0 OPUS-tico-19 _ 15 | 0 OPUS-MultiUN _ 16 | 0 OPUS-MPC1 _ 17 | 0 OPUS-JW300 _ 18 | 0 OPUS-bible-uedin _ 19 | -1 OPUS-QED rm-vallader,rm-sursilv,rm 20 | -1 OPUS-ParaCrawl _ 21 | -1 OPUS-CCAligned _ 22 | -1 OPUS-MultiCCAligned _ 23 | -2 OPUS-ECB _ 24 | -2 OPUS-EUbookshop _ 25 | -2 OPUS-Ubuntu _ 26 | -2 OPUS-PHP _ 27 | -2 OPUS-OpenOffice _ 28 | -2 OPUS-Mozilla-I10n _ 29 | -2 OPUS-KDE4 _ 30 | -2 OPUS-JRC-Acquis _ 31 | -2 OPUS-GNOME _ 32 | -2 OPUS-EuroPat _ 33 | -2 OPUS-EUconst _ 34 | -2 OPUS-EMEA _ 35 | -2 OPUS-DGT _ 36 | -------------------------------------------------------------------------------- /cvutils/tagger.py: -------------------------------------------------------------------------------- 1 | """Grammatical tagging for sentences.""" 2 | import re 3 | 4 | """ 5 | FIXME: 6 | - Put rules in linguistic data files 7 | - Tag other POS 8 | """ 9 | 10 | class Tagger: 11 | 12 | def __init__(self, lang): 13 | self.tag = self._default 14 | 15 | if lang in ["de", "deu"]: 16 | self.tag = self._deu 17 | if lang in ["ja", "jpn"]: 18 | self.tag = self._jpn 19 | 20 | def _deu(self, sentence): 21 | """Convert deu sentences to tags. 22 | 23 | >>> deu(['Ich', 'habe', 'schlechte', 'Nachrichten', 'für', 'ihn', '.']) 24 | ['X', 'X', 'X', 'X', 'X', 'X', 'PUNCT'] 25 | """ 26 | tags = [] 27 | for token in sentence: 28 | if re.match(r"^[^\w+]+$", token): 29 | tags.append("PUNCT") 30 | else: 31 | tags.append("X") 32 | 33 | return tags 34 | 35 | def _jpn(self, sentence): 36 | """Convert jpn sentences to tags. 37 | 38 | >>> _jpn(['切手', 'を', '十', '枚', 'と', 'はがき', 'を', '三', '枚', '買い', 'ます', '。']) 39 | ['X', 'X', 'X', 'X', 'X', 'X', 'X', 'X', 'X', 'X', 'X', 'PUNCT'] 40 | """ 41 | tags = [] 42 | for token in sentence: 43 | if re.match(r"^[^\w+]+$", token): 44 | tags.append("PUNCT") 45 | else: 46 | tags.append("X") 47 | 48 | return tags 49 | 50 | def _default(self, sentence): 51 | """Defaut tags for sentences. 52 | 53 | >>> _default(['This', 'is', 'not', 'Crewe', '.']) 54 | ['X', 'X', 'X', 'PROPN', 'PUNCT'] 55 | """ 56 | tags = [] 57 | first = True 58 | for token in sentence: 59 | if re.match(r"^\W+$", token): 60 | tags.append("PUNCT") 61 | elif token[0] == token[0].upper() and not first: 62 | tags.append("PROPN") 63 | else: 64 | tags.append("X") 65 | first = False 66 | 67 | return tags 68 | 69 | -------------------------------------------------------------------------------- /doc/TROUBLESHOOTING.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting 2 | 3 | ## Encoding issues 4 | 5 | If you get something like: 6 | 7 | ```python 8 | File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/encodings/ascii.py", line 26, in decode 9 | return codecs.ascii_decode(input, self.errors)[0] 10 | UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 1: ordinal not in range(128)``` 11 | ``` 12 | 13 | Look at your locale, you will need a UTF-8 compatible locale. You should check: 14 | 15 | ```bash 16 | $ locale 17 | ``` 18 | 19 | and 20 | 21 | ```python 22 | import sys 23 | sys.stdout.encoding 24 | ``` 25 | 26 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | epitran 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | from setuptools import setup 3 | 4 | # The directory containing this file 5 | HERE = pathlib.Path(__file__).parent 6 | 7 | # The text of the README file 8 | README = (HERE / "README.md").read_text() 9 | 10 | # This call to setup() does all the work 11 | setup( 12 | name="commonvoice-utils", 13 | version="0.2.31", 14 | description="Linguistic processing for languages in Common Voice", 15 | long_description=README, 16 | long_description_content_type="text/markdown", 17 | url="https://github.com/ftyers/commonvoice-utils", 18 | author="Francis M. Tyers", 19 | author_email="ftyers@prompsit.com", 20 | license="AGPL", 21 | classifiers=[ 22 | "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", 23 | "Programming Language :: Python :: 3", 24 | "Programming Language :: Python :: 3.7", 25 | ], 26 | include_package_data=True, 27 | scripts=['covo'], 28 | packages=['cvutils'] 29 | ) 30 | 31 | --------------------------------------------------------------------------------