├── media ├── logo.png ├── logo_hero.png ├── logo_thumb.png └── acquire_characters.png ├── example ├── troika.otf ├── lang │ ├── en.lua │ ├── es.lua │ └── ru.lua ├── example.gui_script ├── example.collection ├── troika.font └── example.gui ├── input └── game.input_binding ├── .github ├── FUNDING.yml └── workflows │ └── ci_workflow.yml ├── test ├── test.ini ├── test.gui ├── test.gui_script ├── test_lang_internal.lua ├── test.collection └── test_lang.lua ├── .gitignore ├── resources └── lang │ ├── es.json │ ├── ru.json │ ├── en.json │ ├── en.lua │ ├── es.lua │ ├── ru.lua │ └── translations.csv ├── game.project ├── settings_deployer ├── .vscode └── settings.json ├── FAQ.md ├── LICENSE ├── lang ├── lang_debug_page.lua ├── editor_script │ ├── lang.editor_script │ ├── json.lua │ └── utf8.lua ├── lang_internal.lua ├── lang.lua └── csv.lua ├── .luacov ├── USE_CASES.md ├── API_REFERENCE.md └── README.md /media/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Insality/defold-lang/HEAD/media/logo.png -------------------------------------------------------------------------------- /example/troika.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Insality/defold-lang/HEAD/example/troika.otf -------------------------------------------------------------------------------- /media/logo_hero.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Insality/defold-lang/HEAD/media/logo_hero.png -------------------------------------------------------------------------------- /media/logo_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Insality/defold-lang/HEAD/media/logo_thumb.png -------------------------------------------------------------------------------- /input/game.input_binding: -------------------------------------------------------------------------------- 1 | mouse_trigger { 2 | input: MOUSE_BUTTON_1 3 | action: "touch" 4 | } 5 | -------------------------------------------------------------------------------- /media/acquire_characters.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Insality/defold-lang/HEAD/media/acquire_characters.png -------------------------------------------------------------------------------- /example/lang/en.lua: -------------------------------------------------------------------------------- 1 | return { 2 | ui_hello = "Hello", 3 | ui_goodbye = "Goodbye", 4 | ui_welcome = "Welcome" 5 | } -------------------------------------------------------------------------------- /example/lang/es.lua: -------------------------------------------------------------------------------- 1 | return { 2 | ui_hello = "Hola", 3 | ui_goodbye = "Adiós", 4 | ui_welcome = "Bienvenido" 5 | } -------------------------------------------------------------------------------- /example/lang/ru.lua: -------------------------------------------------------------------------------- 1 | return { 2 | ui_hello = "Привет", 3 | ui_goodbye = "До свидания", 4 | ui_welcome = "Добро пожаловать" 5 | } -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: insality 4 | ko_fi: insality 5 | buy_me_a_coffee: insality 6 | -------------------------------------------------------------------------------- /test/test.ini: -------------------------------------------------------------------------------- 1 | [bootstrap] 2 | main_collection = /test/test.collectionc 3 | 4 | [display] 5 | height = 256 6 | width = 256 7 | 8 | [test] 9 | report = 1 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.internal 2 | /build 3 | .DS_Store 4 | Thumbs.db 5 | deployer_version_settings.txt 6 | 7 | .deployer_cache 8 | dist 9 | deployer_build_stats.csv 10 | bob*.jar 11 | manifest.*.der 12 | /.editor_settings -------------------------------------------------------------------------------- /resources/lang/es.json: -------------------------------------------------------------------------------- 1 | { 2 | "ui_hello": "¡Hola, Mundo!", 3 | "ui_goodbye": "¡Adiós, Mundo!", 4 | "ui_welcome": "¡Bienvenido!", 5 | "ui_what": "¿Qué es esto?", 6 | "ui_params": "Hola, %s", 7 | "ui_random": "Línea 1\nLínea 2\nLínea 3" 8 | } -------------------------------------------------------------------------------- /resources/lang/ru.json: -------------------------------------------------------------------------------- 1 | { 2 | "ui_hello": "Привет, Мир!", 3 | "ui_goodbye": "До свидания, Мир!", 4 | "ui_welcome": "Добро пожаловать!", 5 | "ui_what": "Что это?", 6 | "ui_params": "Привет, %s", 7 | "ui_random": "Строка 1\nСтрока 2\nСтрока 3" 8 | } 9 | -------------------------------------------------------------------------------- /test/test.gui: -------------------------------------------------------------------------------- 1 | script: "/test/test.gui_script" 2 | background_color { 3 | x: 0.0 4 | y: 0.0 5 | z: 0.0 6 | w: 0.0 7 | } 8 | material: "/builtins/materials/gui.material" 9 | adjust_reference: ADJUST_REFERENCE_PARENT 10 | max_nodes: 512 11 | -------------------------------------------------------------------------------- /resources/lang/en.json: -------------------------------------------------------------------------------- 1 | { 2 | "ui_hello": "Hello, World!", 3 | "ui_goodbye": "Goodbye, World!", 4 | "ui_welcome": "Welcome to the world!", 5 | "ui_what": "What is your name?", 6 | "ui_params": "Hello, %s", 7 | "ui_random": "String 1\nString 2\nString 3" 8 | } 9 | -------------------------------------------------------------------------------- /resources/lang/en.lua: -------------------------------------------------------------------------------- 1 | return { 2 | ui_hello = "Hello, World!", 3 | ui_goodbye = "Goodbye, World!", 4 | ui_welcome = "Welcome to the world!", 5 | ui_what = "What is your name?", 6 | ui_params = "Hello, %s", 7 | ui_random = "String 1\nString 2\nString 3" 8 | } 9 | -------------------------------------------------------------------------------- /resources/lang/es.lua: -------------------------------------------------------------------------------- 1 | return { 2 | ui_hello = "¡Hola, Mundo!", 3 | ui_goodbye = "¡Adiós, Mundo!", 4 | ui_welcome = "¡Bienvenido al mundo!", 5 | ui_what = "¿Cómo te llamas?", 6 | ui_params = "Hola, %s", 7 | ui_random = "Cadena 1\nCadena 2\nCadena 3" 8 | } 9 | -------------------------------------------------------------------------------- /resources/lang/ru.lua: -------------------------------------------------------------------------------- 1 | return { 2 | ui_hello = "Привет, Мир!", 3 | ui_goodbye = "До свидания, Мир!", 4 | ui_welcome = "Добро пожаловать в мир!", 5 | ui_what = "Как тебя зовут?", 6 | ui_params = "Привет, %s", 7 | ui_random = "Строка 1\nСтрока 2\nСтрока 3" 8 | } 9 | -------------------------------------------------------------------------------- /test/test.gui_script: -------------------------------------------------------------------------------- 1 | local deftest = require("deftest.deftest") 2 | 3 | function init(self) 4 | deftest.add(require("test.test_lang")) 5 | deftest.add(require("test.test_lang_internal")) 6 | 7 | local is_report = (sys.get_config("test.report") == "1") 8 | deftest.run({ coverage = { enabled = is_report } }) 9 | end 10 | -------------------------------------------------------------------------------- /resources/lang/translations.csv: -------------------------------------------------------------------------------- 1 | key,en,ru,es 2 | ui_hello,"Hello, World!","Привет, Мир!","¡Hola, Mundo!" 3 | ui_goodbye,"Goodbye, World!","До свидания, Мир!","¡Adiós, Mundo!" 4 | ui_welcome,"Welcome to the world!","Добро пожаловать в мир!","¡Bienvenido al mundo!" 5 | ui_what,"What is your name?","Как тебя зовут?","¿Cómo te llamas?" 6 | ui_params,"Hello, %s","Привет, %s","Hola, %s" 7 | ui_random,"String 1\nString 2\nString 3","Строка 1\nСтрока 2\nСтрока 3","Cadena 1\nCadena 2\nCadena 3" 8 | 9 | -------------------------------------------------------------------------------- /game.project: -------------------------------------------------------------------------------- 1 | [bootstrap] 2 | main_collection = /test/test.collectionc 3 | 4 | [script] 5 | shared_state = 1 6 | 7 | [display] 8 | width = 960 9 | height = 640 10 | 11 | [android] 12 | input_method = HiddenInputField 13 | 14 | [project] 15 | title = Defold Lang 16 | version = 4 17 | publisher = Insality 18 | developer = Maksim Tuprikov, Insality 19 | custom_resources = /resources 20 | dependencies#0 = https://github.com/britzl/deftest/archive/master.zip 21 | 22 | [library] 23 | include_dirs = lang 24 | 25 | -------------------------------------------------------------------------------- /example/example.gui_script: -------------------------------------------------------------------------------- 1 | local lang = require("lang.lang") 2 | 3 | function init(self) 4 | lang.init() 5 | 6 | print(lang.txt("ui_hello")) 7 | print(lang.txt("ui_goodbye")) 8 | print(lang.txp("ui_params", "User")) 9 | print(lang.txr("ui_random")) 10 | 11 | print(lang.is_exist("ui_hello")) 12 | pprint(lang.get_langs()) 13 | 14 | lang.set_lang("ru") 15 | print(lang.txt("ui_hello")) 16 | print(lang.txt("ui_goodbye")) 17 | print(lang.txp("ui_params", "User")) 18 | print(lang.txr("ui_random")) 19 | end 20 | -------------------------------------------------------------------------------- /test/test_lang_internal.lua: -------------------------------------------------------------------------------- 1 | return function() 2 | local lang_internal = {} 3 | 4 | describe("Defold Lang", function() 5 | before(function() 6 | lang_internal = require("lang.lang_internal") 7 | end) 8 | 9 | it("Test index_of", function() 10 | local index = lang_internal.index_of({ "en", "ru", "es" }, "ru") 11 | assert(index == 2) 12 | 13 | index = lang_internal.index_of({ "en", "ru", "es" }, "es") 14 | assert(index == 3) 15 | 16 | index = lang_internal.index_of({ "en", "ru", "es" }, "fr") 17 | assert(index == nil) 18 | end) 19 | end) 20 | end 21 | -------------------------------------------------------------------------------- /settings_deployer: -------------------------------------------------------------------------------- 1 | # Path to bob folder. It will find and save new bob.jar files inside 2 | bob_folder=./ 3 | 4 | # You can point bob version for project in format "filename:sha" 5 | bob_sha="181:fd1ad4c17bfdcd890ea7176f2672c35102384419" 6 | 7 | # Select Defold channel. Values: stable, beta, alpha 8 | bob_channel="stable" 9 | 10 | # If true, it will check and download latest bob version. It will ignore bob_sha param 11 | use_latest_bob=false 12 | 13 | # Select Defold build server 14 | build_server="https://build.defold.com" 15 | 16 | # Is need to build html report 17 | is_build_html_report=true -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Lua.diagnostics.globals": [ 3 | "on_message", 4 | "init", 5 | "final", 6 | "update", 7 | "on_input", 8 | "defos", 9 | "diags", 10 | "filedrop", 11 | "clipboard", 12 | "screenshot", 13 | "share", 14 | "pb", 15 | "uptime", 16 | "lua_script_instance", 17 | "spine", 18 | "on_reload", 19 | "utf8", 20 | "lfs", 21 | "chronos", 22 | "pprint", 23 | "before", 24 | "after", 25 | "it", 26 | "describe", 27 | "json", 28 | "timer", 29 | "socket", 30 | "assert_equal", 31 | "sys", 32 | "editor", 33 | "http" 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /test/test.collection: -------------------------------------------------------------------------------- 1 | name: "default" 2 | scale_along_z: 0 3 | embedded_instances { 4 | id: "test" 5 | data: "components {\n" 6 | " id: \"test\"\n" 7 | " component: \"/test/test.gui\"\n" 8 | " position {\n" 9 | " x: 0.0\n" 10 | " y: 0.0\n" 11 | " z: 0.0\n" 12 | " }\n" 13 | " rotation {\n" 14 | " x: 0.0\n" 15 | " y: 0.0\n" 16 | " z: 0.0\n" 17 | " w: 1.0\n" 18 | " }\n" 19 | " property_decls {\n" 20 | " }\n" 21 | "}\n" 22 | "" 23 | position { 24 | x: 0.0 25 | y: 0.0 26 | z: 0.0 27 | } 28 | rotation { 29 | x: 0.0 30 | y: 0.0 31 | z: 0.0 32 | w: 1.0 33 | } 34 | scale3 { 35 | x: 1.0 36 | y: 1.0 37 | z: 1.0 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /example/example.collection: -------------------------------------------------------------------------------- 1 | name: "example" 2 | scale_along_z: 0 3 | embedded_instances { 4 | id: "go" 5 | data: "components {\n" 6 | " id: \"example\"\n" 7 | " component: \"/example/example.gui\"\n" 8 | " position {\n" 9 | " x: 0.0\n" 10 | " y: 0.0\n" 11 | " z: 0.0\n" 12 | " }\n" 13 | " rotation {\n" 14 | " x: 0.0\n" 15 | " y: 0.0\n" 16 | " z: 0.0\n" 17 | " w: 1.0\n" 18 | " }\n" 19 | " property_decls {\n" 20 | " }\n" 21 | "}\n" 22 | "" 23 | position { 24 | x: 0.0 25 | y: 0.0 26 | z: 0.0 27 | } 28 | rotation { 29 | x: 0.0 30 | y: 0.0 31 | z: 0.0 32 | w: 1.0 33 | } 34 | scale3 { 35 | x: 1.0 36 | y: 1.0 37 | z: 1.0 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /.github/workflows/ci_workflow.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build_and_run: 7 | name: Build and run tests 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v4 11 | with: 12 | lfs: true 13 | 14 | - uses: actions/setup-java@v3 15 | with: 16 | distribution: 'zulu' 17 | java-version: '17' 18 | 19 | - name: Build && Run 20 | run: | 21 | deployer_url="https://raw.githubusercontent.com/Insality/defold-deployer/4/deployer.sh" 22 | curl -s ${deployer_url} | bash -s lbd --headless --settings ./test/test.ini 23 | 24 | - name: Upload coverage reports to Codecov 25 | uses: codecov/codecov-action@v4.0.1 26 | with: 27 | token: ${{ secrets.CODECOV_TOKEN }} 28 | slug: insality/defold-lang -------------------------------------------------------------------------------- /example/troika.font: -------------------------------------------------------------------------------- 1 | font: "/example/troika.otf" 2 | material: "/builtins/fonts/font-df.material" 3 | size: 60 4 | antialias: 1 5 | alpha: 1.0 6 | outline_alpha: 1.0 7 | outline_width: 3.0 8 | shadow_alpha: 1.0 9 | shadow_blur: 1 10 | shadow_x: 0.0 11 | shadow_y: -4.0 12 | extra_characters: "\320\260\320\261\320\262\320\263\320\264\320\265\321\221\320\266\320\267\320\270\320\271\320\272\320\273\320\274\320\275\320\276\320\277\321\200\321\201\321\202\321\203\321\204\321\205\321\206\321\207\321\210\321\211\321\212\321\213\321\214\321\215\321\216\321\217\320\220\320\221\320\222\320\223\320\224\320\225\320\201\320\226\320\227\320\230\320\231\320\232\320\233\320\234\320\235\320\236\320\237\320\240\320\241\320\242\320\243\320\244\320\245\320\246\320\247\320\250\320\251\320\252\320\253\320\254\320\255\320\256\320\257.z\302\247,.``\'\303\227`\'" 13 | output_format: TYPE_DISTANCE_FIELD 14 | all_chars: false 15 | cache_width: 0 16 | cache_height: 0 17 | render_mode: MODE_MULTI_LAYER 18 | -------------------------------------------------------------------------------- /FAQ.md: -------------------------------------------------------------------------------- 1 | # FAQ 2 | 3 | ## How to use several language files in the project? 4 | 5 | This module supports only one language file per language. 6 | 7 | ## Is there any pluralization support? 8 | 9 | No, there is no pluralization support. 10 | 11 | ## Why I should use this module, it looks like a simple table with text? 12 | 13 | It's pretty basic and common module that can be easily implemented by yourself. The advantage of using this module is that it's already implemented and tested. You can use it as is or modify it to fit your needs. I'm sure it's still faster than writing it from scratch. 14 | 15 | # How this library works from tech side? 16 | 17 | - Module selects language base on the `sys.get_sys_info().system_language` value, if not - it uses the default language. 18 | - The module loading the language file from the folder specified in the `game.project` file. 19 | - The module stores the language file in the table. 20 | - The module provides functions to get the text by key and to change the language. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Maksim Tuprikov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /lang/lang_debug_page.lua: -------------------------------------------------------------------------------- 1 | local M = {} 2 | 3 | 4 | ---@param lang lang 5 | ---@param druid table druid instance 6 | ---@param properties_panel table druid properties panel instance 7 | function M.render_properties_panel(lang, druid, properties_panel) 8 | properties_panel:next_scene() 9 | properties_panel:set_header("Lang Panel") 10 | 11 | properties_panel:add_text(function(text) 12 | text:set_text_property("Current Language") 13 | text:set_text_value(lang.get_lang()) 14 | end) 15 | 16 | properties_panel:add_left_right_selector(function(left_right_selector) 17 | left_right_selector:set_text("Langs") 18 | left_right_selector:set_array_type(lang.get_langs()) 19 | left_right_selector:set_value(lang.get_lang()) 20 | left_right_selector.on_change_value:subscribe(function(value) 21 | lang.set_lang(value) 22 | properties_panel.is_dirty = true 23 | end) 24 | end) 25 | 26 | properties_panel:add_button(function(button) 27 | button:set_text_property("Lang Data") 28 | button:set_text_button("Inspect") 29 | button.button.on_click:subscribe(function() 30 | properties_panel:next_scene() 31 | properties_panel:set_header("Lang State") 32 | properties_panel:render_lua_table(lang.get_lang_table()) 33 | end) 34 | end) 35 | end 36 | 37 | 38 | return M 39 | -------------------------------------------------------------------------------- /example/example.gui: -------------------------------------------------------------------------------- 1 | script: "/example/example.gui_script" 2 | fonts { 3 | name: "troika" 4 | font: "/example/troika.font" 5 | } 6 | background_color { 7 | x: 0.0 8 | y: 0.0 9 | z: 0.0 10 | w: 0.0 11 | } 12 | nodes { 13 | position { 14 | x: 480.0 15 | y: 400.0 16 | z: 0.0 17 | w: 1.0 18 | } 19 | rotation { 20 | x: 0.0 21 | y: 0.0 22 | z: 0.0 23 | w: 1.0 24 | } 25 | scale { 26 | x: 1.0 27 | y: 1.0 28 | z: 1.0 29 | w: 1.0 30 | } 31 | size { 32 | x: 330.0 33 | y: 80.0 34 | z: 0.0 35 | w: 1.0 36 | } 37 | color { 38 | x: 1.0 39 | y: 1.0 40 | z: 1.0 41 | w: 1.0 42 | } 43 | type: TYPE_TEXT 44 | blend_mode: BLEND_MODE_ALPHA 45 | text: "Hello!" 46 | font: "troika" 47 | id: "text_ping" 48 | xanchor: XANCHOR_NONE 49 | yanchor: YANCHOR_NONE 50 | pivot: PIVOT_CENTER 51 | outline { 52 | x: 1.0 53 | y: 1.0 54 | z: 1.0 55 | w: 1.0 56 | } 57 | shadow { 58 | x: 1.0 59 | y: 1.0 60 | z: 1.0 61 | w: 1.0 62 | } 63 | adjust_mode: ADJUST_MODE_FIT 64 | line_break: true 65 | layer: "" 66 | inherit_alpha: true 67 | alpha: 1.0 68 | outline_alpha: 0.07 69 | shadow_alpha: 0.29 70 | template_node_child: false 71 | text_leading: 1.0 72 | text_tracking: 0.0 73 | custom_type: 0 74 | enabled: true 75 | visible: true 76 | material: "" 77 | } 78 | material: "/builtins/materials/gui.material" 79 | adjust_reference: ADJUST_REFERENCE_PARENT 80 | max_nodes: 512 81 | -------------------------------------------------------------------------------- /lang/editor_script/lang.editor_script: -------------------------------------------------------------------------------- 1 | local json = require("lang.editor_script.json") 2 | local utf8 = require("lang.editor_script.utf8") 3 | 4 | local M = {} 5 | 6 | local IGNORE_CHARACTERS = { 7 | ["\n"] = true, 8 | ["\r"] = true, 9 | ["\t"] = true, 10 | } 11 | 12 | local function ends_with(str, ending) 13 | return ending == "" or str:sub(-#ending) == ending 14 | end 15 | 16 | 17 | ---Collect all characters used in key-value json string values 18 | local function acquire_user_characters(path) 19 | local is_json = ends_with(path, ".json") 20 | local is_csv = ends_with(path, ".csv") 21 | 22 | local symbols = {} 23 | 24 | if is_json then 25 | local json_content = editor.get(path, "text") 26 | local json_data = json.decode(json_content) 27 | 28 | for key, value in pairs(json_data) do 29 | if type(value) == "string" then 30 | for i = 1, #value do 31 | local char = utf8.sub(value, i, i) 32 | if not symbols[char] and not IGNORE_CHARACTERS[char] then 33 | symbols[char] = true 34 | end 35 | end 36 | end 37 | end 38 | end 39 | 40 | if is_csv then 41 | local csv_content = editor.get(path, "text") 42 | for index = 1, #csv_content do 43 | local char = utf8.sub(csv_content, index, index) 44 | if not symbols[char] and not IGNORE_CHARACTERS[char] then 45 | symbols[char] = true 46 | end 47 | end 48 | end 49 | 50 | return symbols 51 | end 52 | 53 | 54 | function M.get_commands() 55 | return { 56 | { 57 | label = "[Lang] Acquire unique characters", 58 | locations = { "Assets" }, 59 | query = { selection = { type = "resource", cardinality = "many" } }, 60 | active = function(opts) 61 | for index = 1, #opts.selection do 62 | local path = editor.get(opts.selection[index], "path") 63 | if not ends_with(path, ".json") and not ends_with(path, ".csv") then 64 | return false 65 | end 66 | end 67 | 68 | return #opts.selection > 0 69 | end, 70 | 71 | run = function(opts) 72 | local symbols = {} 73 | 74 | for index = 1, #opts.selection do 75 | local path = editor.get(opts.selection[index], "path") 76 | local json_symbols = acquire_user_characters(path) 77 | 78 | for char, _ in pairs(json_symbols) do 79 | symbols[char] = true 80 | end 81 | end 82 | 83 | local unique_chars = {} 84 | for char, _ in pairs(symbols) do 85 | table.insert(unique_chars, char) 86 | end 87 | table.sort(unique_chars) 88 | 89 | print("Unique characters, count", #unique_chars) 90 | print("Copy next line and set it as characters property in your font asset") 91 | print(table.concat(unique_chars)) 92 | print("") 93 | end 94 | } 95 | } 96 | end 97 | 98 | 99 | return M 100 | -------------------------------------------------------------------------------- /.luacov: -------------------------------------------------------------------------------- 1 | local reporter = require("luacov.reporter.defold") 2 | 3 | --- Default values for configuration options. 4 | -- For project specific configuration create '.luacov' file in your project 5 | -- folder. It should be a Lua script setting various options as globals 6 | -- or returning table of options. 7 | -- @class module 8 | -- @name deftest.coverage.configuration 9 | return { 10 | 11 | --- Reporter class to use when creating a report. Default: DefaultReporter from reporter.lua 12 | reporter = reporter, 13 | 14 | --- Filename to store collected stats. Default: "luacov.stats.out". 15 | statsfile = "luacov.stats.out", 16 | 17 | --- Filename to store report. Default: "luacov.report.out". 18 | reportfile = "luacov.report.out", 19 | 20 | --- Enable saving coverage data after every `savestepsize` lines? 21 | -- Setting this flag to `true` in config is equivalent to running LuaCov 22 | -- using `luacov.tick` module. Default: false. 23 | tick = false, 24 | 25 | --- Stats file updating frequency for `luacov.tick`. 26 | -- The lower this value - the more frequently results will be written out to the stats file. 27 | -- You may want to reduce this value (to, for example, 2) to avoid losing coverage data in 28 | -- case your program may terminate without triggering luacov exit hooks that are supposed 29 | -- to save the data. Default: 100. 30 | savestepsize = 100, 31 | 32 | --- Run reporter on completion? Default: true. 33 | runreport = true, 34 | 35 | --- Delete stats file after reporting? Default: false. 36 | deletestats = true, 37 | 38 | --- Process Lua code loaded from raw strings? 39 | -- That is, when the 'source' field in the debug info 40 | -- does not start with '@'. Default: true. 41 | codefromstrings = true, 42 | 43 | --- Lua patterns for files to include when reporting. 44 | -- All will be included if nothing is listed. 45 | -- Do not include the '.lua' extension. Path separator is always '/'. 46 | -- Overruled by `exclude`. 47 | -- @usage 48 | -- include = { 49 | -- "mymodule$", -- the main module 50 | -- "mymodule%/.+$", -- and everything namespaced underneath it 51 | -- } 52 | include = {}, 53 | 54 | --- Lua patterns for files to exclude when reporting. 55 | -- Nothing will be excluded if nothing is listed. 56 | -- Do not include the '.lua' extension. Path separator is always '/'. 57 | -- Overrules `include`. 58 | exclude = { 59 | "^test%/.+$", 60 | }, 61 | 62 | --- Table mapping names of modules to be included to their filenames. 63 | -- Has no effect if empty. 64 | -- Real filenames mentioned here will be used for reporting 65 | -- even if the modules have been installed elsewhere. 66 | -- Module name can contain '*' wildcard to match groups of modules, 67 | -- in this case corresponding path will be used as a prefix directory 68 | -- where modules from the group are located. 69 | -- @usage 70 | -- modules = { 71 | -- ["some_rock"] = "src/some_rock.lua", 72 | -- ["some_rock.*"] = "src" 73 | -- } 74 | modules = {}, 75 | } -------------------------------------------------------------------------------- /USE_CASES.md: -------------------------------------------------------------------------------- 1 | # Use Cases 2 | 3 | This section provides examples of how to use the `lang` module. 4 | 5 | ## Druid Lang Text Integration 6 | 7 | To use lang module with [Druid](https://github.com/Insality/druid), you can use next integration: 8 | 9 | ```lua 10 | local lang = require("lang.lang") 11 | local druid = require("druid.druid") 12 | 13 | local function init_druid(self) 14 | druid.set_text_function(lang.txp) 15 | end 16 | ``` 17 | 18 | Don't forget to call `druid.on_language_change()` when the language changes. 19 | 20 | ```lua 21 | local function next_language(self) 22 | lang.set_next_lang() 23 | druid.on_language_change() 24 | end 25 | ``` 26 | 27 | ## Save current language 28 | 29 | ### Using Defold Saver 30 | 31 | If you want to save the current language, you can use the [Defold-Saver](https://github.com/Insality/defold-saver/) module. 32 | 33 | ```lua 34 | local lang = require("lang.lang") 35 | local saver = require("saver.saver") 36 | 37 | local function init_saver(self) 38 | ---Add save states to annotation 39 | ---@class saver.game_state 40 | ---@field lang lang.state 41 | 42 | saver.init() 43 | -- After saver.init add lang state to the saver 44 | saver.bind_game_state("lang", lang.state) 45 | end 46 | 47 | local function init_lang(self) 48 | -- Init lang after loaded save 49 | lang.init() 50 | end 51 | 52 | function init(self) 53 | --... 54 | init_saver(self) 55 | init_lang(self) 56 | --... 57 | end 58 | ``` 59 | 60 | 61 | ### Using other save system 62 | 63 | If you use another save system, you can save the current language somewhere. Set the current language before calling `lang.init()`. 64 | 65 | ```lua 66 | -- Save current somewhere lang id via `lang.get_lang()` and update it on language change in your game 67 | local current_lang = get_current_language_from_save() 68 | lang.state.lang = current_lang 69 | lang.init() 70 | ``` 71 | 72 | 73 | ## Using GUI text as a text_id 74 | 75 | In some cases, it can be useful or convenient to use the GUI node text as a text_id. The idea is to set the localization ID in your GUI layout and in the initialization step, use it to set the actual text. 76 | 77 | ```lua 78 | local lang = require("lang.lang") 79 | 80 | local function init(self) 81 | --... 82 | local text_node = gui.get_node("text_node") 83 | local text_id = gui.get_text(text_node) 84 | gui.set_text(text_node, lang.txt(text_id)) 85 | --... 86 | end 87 | ``` 88 | 89 | In the Druid's Lang Text, this happens automatically if you don't specify the text ID in the constructor. 90 | 91 | ```lua 92 | local druid = require("druid.druid") 93 | 94 | function init(self) 95 | --... 96 | self.druid = druid.new(self) 97 | -- If text_node has a text "ui_text_id", it will be used as a text_id 98 | self.druid:new_lang_text("text_node") 99 | --... 100 | end 101 | ``` 102 | 103 | 104 | ## Use Editor Script to collect unique characters 105 | 106 | If you want to get the set of characters used in your localization, you can use the Lang Editor Script. To use it, select the JSON files, right-click, and choose 'Acquire Unique Characters'. Then, copy the unique characters to your font settings (in the 'Character' or 'Extra Characters' field). 107 | 108 | ![Unique Characters](media/acquire_characters.png) -------------------------------------------------------------------------------- /lang/lang_internal.lua: -------------------------------------------------------------------------------- 1 | local csv = require("lang.csv") 2 | 3 | ---@class lang.logger 4 | ---@field trace fun(logger: lang.logger, message: string, data: any|nil) 5 | ---@field debug fun(logger: lang.logger, message: string, data: any|nil) 6 | ---@field info fun(logger: lang.logger, message: string, data: any|nil) 7 | ---@field warn fun(logger: lang.logger, message: string, data: any|nil) 8 | ---@field error fun(logger: lang.logger, message: string, data: any|nil) 9 | 10 | 11 | local M = {} 12 | 13 | M.SYSTEM_LANG = sys.get_sys_info().language 14 | 15 | 16 | ---Split string by separator 17 | ---@param s string 18 | ---@param sep string 19 | ---@return table 20 | function M.split(s, sep) 21 | sep = sep or "%s" 22 | local t = {} 23 | local i = 1 24 | for str in string.gmatch(s, "([^" .. sep .. "]+)") do 25 | t[i] = str 26 | i = i + 1 27 | end 28 | return t 29 | end 30 | 31 | 32 | --- Use empty function to save a bit of memory 33 | local EMPTY_FUNCTION = function(_, message, context) end 34 | 35 | ---@type lang.logger 36 | M.empty_logger = { 37 | trace = EMPTY_FUNCTION, 38 | debug = EMPTY_FUNCTION, 39 | info = EMPTY_FUNCTION, 40 | warn = EMPTY_FUNCTION, 41 | error = EMPTY_FUNCTION, 42 | } 43 | 44 | ---@type lang.logger 45 | M.logger = { 46 | trace = function(_, msg, data) print("TRACE:", msg, data) end, 47 | debug = function(_, msg, data) print("DEBUG:", msg, data) end, 48 | info = function(_, msg, data) print("INFO:", msg, data) end, 49 | warn = function(_, msg, data) print("WARN:", msg, data) end, 50 | error = function(_, msg, data) print("ERROR:", msg, data) end 51 | } 52 | 53 | 54 | ---Load JSON file from game resources folder (by relative path to game.project) 55 | ---Return nil if file not found or error 56 | ---@param json_path string 57 | ---@return table|nil 58 | function M.load_json(json_path) 59 | local resource, is_error = sys.load_resource(json_path) 60 | if is_error or not resource then 61 | return nil 62 | end 63 | 64 | return json.decode(resource) 65 | end 66 | 67 | 68 | ---Load CSV file from game resources folder (by relative path to game.project) 69 | ---Return nil if file not found or error 70 | ---@param csv_path string 71 | ---@return table|nil 72 | function M.load_csv(csv_path) 73 | local resource, is_error = sys.load_resource(csv_path) 74 | if is_error or not resource then 75 | return nil 76 | end 77 | 78 | local data = {} 79 | local f = csv.openstring(resource) 80 | local headers = nil 81 | 82 | -- Parse headers, first id is a lang_id to table > 83 | for fields in f:lines() do 84 | if not headers then 85 | -- First row contains language codes 86 | headers = fields 87 | -- Initialize language tables 88 | for i = 2, #headers do 89 | data[headers[i]] = {} 90 | end 91 | else 92 | -- Process data rows 93 | local key = fields[1] -- First column is the translation key 94 | if key then 95 | -- Add translations for each language 96 | for i = 2, #headers do 97 | if fields[i] then 98 | -- Process escape sequences in the field value 99 | local value = fields[i]:gsub("\\n", "\n"):gsub("\\t", "\t"):gsub("\\r", "\r") 100 | data[headers[i]][key] = value 101 | end 102 | end 103 | end 104 | end 105 | end 106 | 107 | return data 108 | end 109 | 110 | 111 | ---Check if a table contains a value 112 | ---@param t table 113 | ---@param value any 114 | ---@return number|nil 115 | function M.index_of(t, value) 116 | for i, v in ipairs(t) do 117 | if v == value then 118 | return i 119 | end 120 | end 121 | return nil 122 | end 123 | 124 | 125 | return M 126 | -------------------------------------------------------------------------------- /API_REFERENCE.md: -------------------------------------------------------------------------------- 1 | # lang API 2 | 3 | > at lang/lang.lua 4 | 5 | ## Functions 6 | 7 | - [reset_state](#reset_state) 8 | - [init](#init) 9 | - [set_logger](#set_logger) 10 | - [set_lang](#set_lang) 11 | - [set_lang_table](#set_lang_table) 12 | - [set_next_lang](#set_next_lang) 13 | - [get_next_lang](#get_next_lang) 14 | - [get_lang](#get_lang) 15 | - [get_default_lang](#get_default_lang) 16 | - [txt](#txt) 17 | - [txr](#txr) 18 | - [txp](#txp) 19 | - [is_exist](#is_exist) 20 | - [get_langs](#get_langs) 21 | - [get_lang_table](#get_lang_table) 22 | - [is_lang_available](#is_lang_available) 23 | - [render_properties_panel](#render_properties_panel) 24 | 25 | ## Fields 26 | 27 | - [state](#state) 28 | - [available_langs](#available_langs) 29 | 30 | 31 | 32 | ### reset_state 33 | 34 | --- 35 | ```lua 36 | lang.reset_state() 37 | ``` 38 | 39 | Reset module lang state 40 | 41 | ### init 42 | 43 | --- 44 | ```lua 45 | lang.init(available_langs, [lang_on_start]) 46 | ``` 47 | 48 | Initialize lang module 49 | 50 | - **Parameters:** 51 | - `available_langs` *(lang.data[])*: List of { id = "en", path = "/locales/en.json" } 52 | - `[lang_on_start]` *(string?)*: Language code to set on start, override saved language 53 | 54 | ### set_logger 55 | 56 | --- 57 | ```lua 58 | lang.set_logger([logger_instance]) 59 | ``` 60 | 61 | Set logger for lang module. Pass nil to use empty logger 62 | 63 | - **Parameters:** 64 | - `[logger_instance]` *(table|lang.logger|nil)*: 65 | 66 | ### set_lang 67 | 68 | --- 69 | ```lua 70 | lang.set_lang(lang_id) 71 | ``` 72 | 73 | Set current language 74 | 75 | - **Parameters:** 76 | - `lang_id` *(string)*: current language code (en, jp, ru, etc.) 77 | 78 | - **Returns:** 79 | - `is` *(boolean)*: language changed 80 | 81 | ### set_lang_table 82 | 83 | --- 84 | ```lua 85 | lang.set_lang_table([lang_table]) 86 | ``` 87 | 88 | - **Parameters:** 89 | - `[lang_table]` *(any)*: 90 | 91 | ### set_next_lang 92 | 93 | --- 94 | ```lua 95 | lang.set_next_lang() 96 | ``` 97 | 98 | Set next language from lang list and return it's code 99 | 100 | - **Returns:** 101 | - `lang_code` *(string)*: The new language code after change 102 | 103 | ### get_next_lang 104 | 105 | --- 106 | ```lua 107 | lang.get_next_lang() 108 | ``` 109 | 110 | Get next language from lang list and return it's code 111 | 112 | - **Returns:** 113 | - `lang_code` *(string)*: next language code 114 | 115 | ### get_lang 116 | 117 | --- 118 | ```lua 119 | lang.get_lang() 120 | ``` 121 | 122 | Get current language 123 | 124 | - **Returns:** 125 | - `Current` *(string)*: language code 126 | 127 | ### get_default_lang 128 | 129 | --- 130 | ```lua 131 | lang.get_default_lang() 132 | ``` 133 | 134 | Get default language 135 | 136 | - **Returns:** 137 | - `Default` *(string)*: language code 138 | 139 | ### txt 140 | 141 | --- 142 | ```lua 143 | lang.txt(text_id) 144 | ``` 145 | 146 | Get translation for text id 147 | 148 | - **Parameters:** 149 | - `text_id` *(string)*: text id from your localization 150 | 151 | - **Returns:** 152 | - `Translated` *(string)*: text 153 | 154 | ### txr 155 | 156 | --- 157 | ```lua 158 | lang.txr(text_id) 159 | ``` 160 | 161 | Get random translation for text id, split by \n symbol 162 | 163 | - **Parameters:** 164 | - `text_id` *(string)*: text id from your localization 165 | 166 | - **Returns:** 167 | - `translated` *(string)*: text 168 | 169 | ### txp 170 | 171 | --- 172 | ```lua 173 | lang.txp(text_id, ...) 174 | ``` 175 | 176 | Get translation for text id with params 177 | 178 | - **Parameters:** 179 | - `text_id` *(string)*: Text id from your localization 180 | - `...` *(...)*: vararg 181 | 182 | - **Returns:** 183 | - `Translated` *(string)*: text 184 | 185 | ### is_exist 186 | 187 | --- 188 | ```lua 189 | lang.is_exist(text_id) 190 | ``` 191 | 192 | Check is translation with text_id exist 193 | 194 | - **Parameters:** 195 | - `text_id` *(string)*: text id from your localization 196 | 197 | - **Returns:** 198 | - `Is` *(boolean)*: translation exist for text_id 199 | 200 | ### get_langs 201 | 202 | --- 203 | ```lua 204 | lang.get_langs() 205 | ``` 206 | 207 | Return list of available languages 208 | 209 | - **Returns:** 210 | - `List` *(string[])*: of available languages 211 | 212 | ### get_lang_table 213 | 214 | --- 215 | ```lua 216 | lang.get_lang_table() 217 | ``` 218 | 219 | Get lang table 220 | 221 | - **Returns:** 222 | - `` *(table)*: 223 | 224 | ### is_lang_available 225 | 226 | --- 227 | ```lua 228 | lang.is_lang_available(lang_id) 229 | ``` 230 | 231 | Check if language is available 232 | 233 | - **Parameters:** 234 | - `lang_id` *(string)*: Language code to check 235 | 236 | - **Returns:** 237 | - `True` *(boolean)*: if language is available 238 | 239 | ### render_properties_panel 240 | 241 | --- 242 | ```lua 243 | lang.render_properties_panel(druid, properties_panel) 244 | ``` 245 | 246 | - **Parameters:** 247 | - `druid` *(table)*: druid instance 248 | - `properties_panel` *(table)*: druid properties panel instance 249 | 250 | 251 | ## Fields 252 | 253 | - **state** (_nil_): Persistent storage 254 | 255 | 256 | - **available_langs** (_nil_): List of available languages 257 | 258 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](media/logo.png) 2 | 3 | [![GitHub release (latest by date)](https://img.shields.io/github/v/tag/insality/defold-lang?style=for-the-badge&label=Release)](https://github.com/Insality/defold-lang/tags) 4 | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/insality/defold-lang/ci_workflow.yml?style=for-the-badge)](https://github.com/Insality/defold-lang/actions) 5 | [![codecov](https://img.shields.io/codecov/c/github/Insality/defold-lang?style=for-the-badge)](https://codecov.io/gh/Insality/defold-lang) 6 | 7 | [![Github-sponsors](https://img.shields.io/badge/sponsor-30363D?style=for-the-badge&logo=GitHub-Sponsors&logoColor=#EA4AAA)](https://github.com/sponsors/insality) [![Ko-Fi](https://img.shields.io/badge/Ko--fi-F16061?style=for-the-badge&logo=ko-fi&logoColor=white)](https://ko-fi.com/insality) [![BuyMeACoffee](https://img.shields.io/badge/Buy%20Me%20a%20Coffee-ffdd00?style=for-the-badge&logo=buy-me-a-coffee&logoColor=black)](https://www.buymeacoffee.com/insality) 8 | 9 | 10 | # Defold Lang 11 | 12 | **Defold Lang** is a simple localization module for **Defold**. It loads language files and manages translations in your project. 13 | 14 | ## Features 15 | 16 | - **Handy API** - Simple and easy to use API 17 | - **Multiple File Formats** - Support for JSON, Lua, and CSV language files 18 | - **Saver Support** - Save current selected language in [Defold-Saver](https://github.com/Insality/defold-saver) 19 | - **Druid Support** - Easy [Druid](https://github.com/Insality/druid) integration 20 | 21 | ## Setup 22 | 23 | ### [Dependency](https://www.defold.com/manuals/libraries/) 24 | 25 | Open your `game.project` file and add the following line to the dependencies field under the project section: 26 | 27 | **[Lang](https://github.com/Insality/defold-lang/archive/refs/tags/4.zip)** 28 | 29 | ``` 30 | https://github.com/Insality/defold-lang/archive/refs/tags/4.zip 31 | ``` 32 | 33 | After that, select `Project ▸ Fetch Libraries` to update [library dependencies]((https://defold.com/manuals/libraries/#setting-up-library-dependencies)). This happens automatically whenever you open a project so you will only need to do this if the dependencies change without re-opening the project. 34 | 35 | ### Library Size 36 | 37 | > **Note:** The library size is calculated based on the build report per platform 38 | 39 | | Platform | Library Size | 40 | | ---------------- | ------------ | 41 | | HTML5 | **7.87 KB** | 42 | | Desktop / Mobile | **12.68 KB** | 43 | 44 | 45 | ### Initialization 46 | 47 | Initialize the **Lang** module by calling `lang.init()` with your language configuration: 48 | 49 | ```lua 50 | local lang = require("lang.lang") 51 | 52 | -- Initialize with language files 53 | lang.init({ 54 | { id = "en", path = "/resources/lang/en.json" }, 55 | { id = "ru", path = "/resources/lang/ru.json" }, 56 | { id = "es", path = "/resources/lang/es.json" }, 57 | }) 58 | ``` 59 | 60 | You can also force a specific language on initialization: 61 | 62 | ```lua 63 | -- Force a specific language on start 64 | lang.init({ 65 | { id = "en", path = "/resources/lang/en.json" }, 66 | { id = "ru", path = "/resources/lang/ru.json" }, 67 | { id = "es", path = "/resources/lang/es.json" }, 68 | }, "es") -- Force Spanish language 69 | ``` 70 | 71 | 72 | ### Default Language 73 | 74 | **Defold Lang** selects the language to use in the following priority order: 75 | 76 | 1. **Force parameter** - If provided as second parameter to `lang.init()` 77 | 2. **Saved language** - From `lang.state.lang` (restored from save system or manually set) 78 | 3. **System language** - Device language from `sys.get_sys_info().language` 79 | 4. **Default language** - First language in the configuration array 80 | 81 | The first language in the configuration array serves as the ultimate fallback. Defold uses the two-character [ISO-639 format](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) for language codes ("en", "ru", "es", etc). 82 | 83 | The module uses `sys.load_resource` to load the files. Place your files inside your [custom resources folder](https://defold.com/manuals/project-settings/#custom-resources) to ensure they are included in the build. 84 | 85 | 86 | ### Localization Files 87 | 88 | **Defold Lang** supports three file formats: **JSON**, **Lua**, and **CSV**. Each format has its own advantages: 89 | 90 | #### JSON Files 91 | JSON files use a simple key-value structure: 92 | 93 | ```json 94 | { 95 | "ui_hello_world": "Hello, World!", 96 | "ui_hello_name": "Hello, %s!", 97 | "ui_settings": "Settings", 98 | "ui_exit": "Exit" 99 | } 100 | ``` 101 | 102 | Initialize with JSON files: 103 | ```lua 104 | lang.init({ 105 | { id = "en", path = "/locales/en.json" }, 106 | { id = "ru", path = "/locales/ru.json" }, 107 | { id = "es", path = "/locales/es.json" }, 108 | }) 109 | ``` 110 | 111 | #### Lua Files 112 | Lua files return a table with translations: 113 | 114 | ```lua 115 | -- en.lua 116 | return { 117 | ui_hello_world = "Hello, World!", 118 | ui_hello_name = "Hello, %s!", 119 | ui_settings = "Settings", 120 | ui_exit = "Exit" 121 | } 122 | ``` 123 | 124 | Initialize with Lua files: 125 | ```lua 126 | lang.init({ 127 | { id = "en", path = require("locales.en") }, 128 | { id = "ru", path = require("locales.ru") }, 129 | { id = "es", path = require("locales.es") }, 130 | }) 131 | ``` 132 | 133 | #### CSV Files 134 | CSV files allow multiple languages in a single file. The first column contains keys, and subsequent columns contain translations: 135 | 136 | ```csv 137 | key,en,ru,es 138 | ui_hello_world,"Hello, World!","Привет, мир!","¡Hola, mundo!" 139 | ui_hello_name,"Hello, %s!","Привет, %s!","¡Hola, %s!" 140 | ui_settings,Settings,Настройки,Configuración 141 | ui_exit,Exit,Выход,Salir 142 | ``` 143 | 144 | Initialize with CSV files (specify column names as language IDs): 145 | ```lua 146 | lang.init({ 147 | { id = "en", path = "/locales/translations.csv" }, 148 | { id = "ru", path = "/locales/translations.csv" }, 149 | { id = "es", path = "/locales/translations.csv" }, 150 | }) 151 | ``` 152 | 153 | #### Mixed Format Example 154 | You can even mix different file formats: 155 | 156 | ```lua 157 | lang.init({ 158 | { id = "en", path = "/resources/lang/en.json" }, 159 | { id = "ru", path = "/resources/lang/ru.lua" }, 160 | { id = "es", path = "/resources/lang/translations.csv" }, 161 | }) 162 | ``` 163 | 164 | 165 | ## API Reference 166 | 167 | ### Quick API Reference 168 | 169 | ```lua 170 | lang.init(available_langs, [lang_on_start]) 171 | lang.set_lang(lang_id) 172 | lang.get_lang() 173 | lang.get_langs() 174 | lang.set_next_lang() 175 | lang.get_next_lang() 176 | lang.txt(text_id) 177 | lang.txp(text_id, ...) 178 | lang.txr(text_id) 179 | lang.is_exist(text_id) 180 | lang.set_logger([logger]) 181 | lang.reset_state() 182 | ``` 183 | 184 | #### Basic Usage Example 185 | 186 | ```lua 187 | local lang = require("lang.lang") 188 | 189 | -- Initialize with language files 190 | lang.init({ 191 | { id = "en", path = "/resources/lang/en.json" }, 192 | { id = "ru", path = "/resources/lang/ru.json" }, 193 | { id = "es", path = "/resources/lang/es.json" }, 194 | }) 195 | 196 | -- Use translations 197 | print(lang.txt("ui_hello_world")) -- "Hello, World!" 198 | print(lang.txp("ui_hello_name", "John")) -- "Hello, John!" 199 | 200 | -- Change language 201 | lang.set_lang("es") 202 | print(lang.txt("ui_hello_world")) -- "¡Hola, mundo!" 203 | ``` 204 | 205 | ### API Reference 206 | 207 | Read the [API Reference](API_REFERENCE.md) file to see the full API documentation for the module. 208 | 209 | 210 | ## Use Cases 211 | 212 | Read the [Use Cases](USE_CASES.md) file to see several examples of how to use the this module in your Defold game development projects. 213 | 214 | 215 | ## FAQ 216 | 217 | Read the [FAQ](FAQ.md) file to see the answers to frequently asked questions about the module. 218 | 219 | 220 | ## License 221 | 222 | This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. 223 | 224 | 225 | ## Issues and Suggestions 226 | 227 | For any issues, questions, or suggestions, please [create an issue](https://github.com/Insality/defold-lang/issues). 228 | 229 | 230 | ## 👏 Contributors 231 | 232 | 233 | 234 | 235 | 236 | ## Changelog 237 | 238 |
239 | 240 | ### **V1** 241 | - Initial release 242 | 243 | ### **V2** 244 | - Add Defold Editor Script to collect unique characters from selected JSON files 245 | 246 | ### **V3** 247 | - Add `lang.get_next_lang()` function 248 | - Better error messages 249 | 250 | ### **V4** 251 | - [Breaking] Lang now use `lang.init()` function to initialize module instead of `game.project` configuration 252 | - Add Lua file support 253 | - Add CSV file support 254 | - Updated editor script to collect unique characters from selected JSON and CSV files 255 | - Add Lang debug properties page for Druid properties panel 256 | 257 |
258 | 259 | 260 | ## ❤️ Support project ❤️ 261 | 262 | Your donation helps me stay engaged in creating valuable projects for **Defold**. If you appreciate what I'm doing, please consider supporting me! 263 | 264 | [![Github-sponsors](https://img.shields.io/badge/sponsor-30363D?style=for-the-badge&logo=GitHub-Sponsors&logoColor=#EA4AAA)](https://github.com/sponsors/insality) [![Ko-Fi](https://img.shields.io/badge/Ko--fi-F16061?style=for-the-badge&logo=ko-fi&logoColor=white)](https://ko-fi.com/insality) [![BuyMeACoffee](https://img.shields.io/badge/Buy%20Me%20a%20Coffee-ffdd00?style=for-the-badge&logo=buy-me-a-coffee&logoColor=black)](https://www.buymeacoffee.com/insality) 265 | -------------------------------------------------------------------------------- /lang/lang.lua: -------------------------------------------------------------------------------- 1 | --- Lang localization helper module 2 | --- Call lang.init() to init module to load last used or default language 3 | --- With saver module use saver.bind_save_state("lang", lang.state) to load lang state to save 4 | --- To load in other way - replace state table before lang.init() 5 | --- Use lang.set_lang("en") to change language 6 | --- Use lang.set_next_lang() to change language to next in list 7 | --- Use lang.txt("key") to get translation 8 | --- Use lang.txr("key") to get random translation, split by \n symbol 9 | --- Use lang.txp("key", "param1", "param2") to get translation with params (Use %s in translation) 10 | --- Use lang.is_exist("key") to check is translation exist 11 | --- Use lang.get_langs() to get list of available languages 12 | 13 | local lang_internal = require("lang.lang_internal") 14 | local lang_debug_page = require("lang.lang_debug_page") 15 | 16 | ---@class lang 17 | local M = {} 18 | 19 | 20 | ---@class lang.state 21 | ---@field lang string current language name (en, jp, ru, etc.) 22 | 23 | ---@class lang.data 24 | ---@field path string|table Lua table, json or csv path, ex: "/resources/lang/en.json", "/resources/lang/en.csv" 25 | ---@field id string Language code, ex: "en". If csv file, it's a header name 26 | 27 | ---Current language translations 28 | ---@type table Contains all current language translations. Key - lang id, Value - translation 29 | local LANG_DICT = nil 30 | 31 | -- Persistent storage 32 | ---@type lang.state 33 | M.state = nil 34 | 35 | ---Order of available languages 36 | ---@type lang.data[] In order 37 | local LANGS_ORDER = nil 38 | 39 | ---Map of available languages for fast lookup 40 | ---@type table Key is language id, value is lang.data 41 | local AVAILABLE_LANGS_MAP = nil 42 | 43 | ---Reset module lang state 44 | function M.reset_state() 45 | M.state = { 46 | lang = lang_internal.SYSTEM_LANG, 47 | } 48 | LANG_DICT = {} 49 | LANGS_ORDER = {} 50 | AVAILABLE_LANGS_MAP = {} 51 | end 52 | M.reset_state() 53 | 54 | 55 | ---Check if language exists in available languages 56 | ---@param lang_id string Language code to check 57 | ---@return boolean True if language exists 58 | local function is_lang_available(lang_id) 59 | return AVAILABLE_LANGS_MAP[lang_id] ~= nil 60 | end 61 | 62 | 63 | ---Get language data by id 64 | ---@param lang_id string Language code 65 | ---@return lang.data|nil Language data or nil if not found 66 | local function get_lang_data(lang_id) 67 | return AVAILABLE_LANGS_MAP[lang_id] 68 | end 69 | 70 | 71 | ---Call this to initialize lang module 72 | ---@param available_langs lang.data[] List of { id = "en", path = "/locales/en.json" } 73 | ---@param lang_on_start string? Language code to set on start, override saved language 74 | function M.init(available_langs, lang_on_start) 75 | if not available_langs or #available_langs == 0 then 76 | lang_internal.logger:error("No available languages provided to init") 77 | return 78 | end 79 | 80 | -- Clear previous language data 81 | LANGS_ORDER = {} 82 | AVAILABLE_LANGS_MAP = {} 83 | LANG_DICT = {} 84 | 85 | -- Build available languages list and map 86 | local default_lang = nil 87 | for index, lang_data in ipairs(available_langs) do 88 | table.insert(LANGS_ORDER, lang_data.id) 89 | AVAILABLE_LANGS_MAP[lang_data.id] = lang_data 90 | default_lang = default_lang or lang_data.id 91 | end 92 | 93 | -- Get system language if no specific language is requested 94 | local system_lang = lang_internal.SYSTEM_LANG 95 | if not is_lang_available(system_lang) then 96 | system_lang = nil 97 | end 98 | 99 | -- Determine target language with validation 100 | local target_lang = lang_on_start or M.state.lang or system_lang or default_lang 101 | 102 | -- Validate the target language exists, fallback to default if not 103 | if not is_lang_available(target_lang) then 104 | lang_internal.logger:warn("Target language not available, falling back to default", { 105 | target_lang = target_lang, 106 | default_lang = default_lang 107 | }) 108 | target_lang = default_lang 109 | end 110 | 111 | M.set_lang(target_lang) 112 | end 113 | 114 | 115 | ---Set logger for lang module. Pass nil to use empty logger 116 | ---@param logger_instance lang.logger|table|nil 117 | function M.set_logger(logger_instance) 118 | lang_internal.logger = logger_instance or lang_internal.empty_logger 119 | end 120 | 121 | 122 | ---Set current language 123 | ---@param lang_id string current language code (en, jp, ru, etc.) 124 | ---@return boolean is language changed 125 | function M.set_lang(lang_id) 126 | if not lang_id then 127 | lang_internal.logger:error("Language id cannot be nil") 128 | return false 129 | end 130 | 131 | local previous_lang = M.state.lang 132 | local previous_loaded_lang = previous_lang or nil 133 | 134 | -- Check if language is available using fast lookup 135 | if not is_lang_available(lang_id) then 136 | lang_internal.logger:error("Lang not found", lang_id) 137 | return false 138 | end 139 | 140 | -- Get language data using fast lookup 141 | local lang_data = get_lang_data(lang_id) 142 | if not lang_data then 143 | lang_internal.logger:error("Lang data not found", lang_id) 144 | return false 145 | end 146 | 147 | local is_lua = type(lang_data.path) == "table" 148 | ---@type string|nil 149 | local path_str = type(lang_data.path) == "string" and lang_data.path --[[@as string]] or nil 150 | local is_csv = not is_lua and path_str and string.find(path_str, ".csv") 151 | local is_json = not is_lua and path_str and string.find(path_str, ".json") 152 | 153 | if is_lua then 154 | M.set_lang_table(lang_data.path) 155 | M.state.lang = lang_id 156 | elseif is_csv and path_str then 157 | M.load_from_csv(path_str, lang_id) 158 | elseif is_json and path_str then 159 | M.load_from_json(path_str, lang_id) 160 | else 161 | lang_internal.logger:error("Lang format not supported", lang_data.path or "unknown") 162 | return false 163 | end 164 | 165 | lang_internal.logger:info("Lang changed", { previous_lang = previous_loaded_lang, lang = lang_id }) 166 | return true 167 | end 168 | 169 | 170 | ---Load lang from json file 171 | ---@private 172 | ---@param lang_path string path to lang file 173 | ---@param locale_id string? locale id 174 | ---@return table? result lang data or false if error 175 | function M.load_from_json(lang_path, locale_id) 176 | locale_id = locale_id or M.state.lang or lang_internal.SYSTEM_LANG 177 | 178 | local is_parsed, lang_data = pcall(lang_internal.load_json, lang_path) 179 | if not is_parsed then 180 | lang_internal.logger:error("Can't load or parse lang file. Check the JSON file is valid", lang_path) 181 | return nil 182 | end 183 | if not lang_data then 184 | lang_internal.logger:error("Lang file not found", lang_path) 185 | return nil 186 | end 187 | 188 | M.set_lang_table(lang_data) 189 | M.state.lang = locale_id 190 | 191 | return lang_data 192 | end 193 | 194 | 195 | ---Load lang from csv file 196 | ---@private 197 | ---@param csv_path string path to csv file 198 | ---@param locale_id string? lang code, default is last used lang 199 | ---@return table? result lang data or false if error 200 | function M.load_from_csv(csv_path, locale_id) 201 | locale_id = locale_id or M.state.lang or lang_internal.SYSTEM_LANG 202 | 203 | local langs_data = lang_internal.load_csv(csv_path) 204 | if not langs_data then 205 | lang_internal.logger:error("Can't load or parse lang file. Check the CSV file is valid", csv_path) 206 | return nil 207 | end 208 | 209 | if not langs_data[locale_id] then 210 | lang_internal.logger:error("Lang code not found", locale_id) 211 | return nil 212 | end 213 | 214 | M.set_lang_table(langs_data[locale_id]) 215 | M.state.lang = locale_id 216 | 217 | return langs_data[locale_id] 218 | end 219 | 220 | 221 | function M.set_lang_table(lang_table) 222 | LANG_DICT = lang_table 223 | end 224 | 225 | 226 | ---Set next language from lang list and return it's code 227 | ---@return string lang_code The new language code after change 228 | function M.set_next_lang() 229 | M.set_lang(M.get_next_lang()) 230 | 231 | return M.get_lang() 232 | end 233 | 234 | 235 | ---Get next language from lang list and return it's code 236 | ---@return string lang_code next language code 237 | function M.get_next_lang() 238 | local current_lang = M.get_lang() 239 | local all_langs = M.get_langs() 240 | local current_index = lang_internal.index_of(all_langs, current_lang) or 1 241 | 242 | local next_index = current_index + 1 243 | if next_index > #all_langs then 244 | next_index = 1 245 | end 246 | 247 | return all_langs[next_index] 248 | end 249 | 250 | 251 | ---Get current language 252 | ---@return string Current language code 253 | function M.get_lang() 254 | return M.state.lang 255 | end 256 | 257 | 258 | ---Get default language 259 | ---@return string Default language code 260 | function M.get_default_lang() 261 | return lang_internal.SYSTEM_LANG 262 | end 263 | 264 | 265 | ---Get translation for text id 266 | ---@param text_id string text id from your localization 267 | ---@return string text ("ui_hello_world") -> "Hello, World!" 268 | function M.txt(text_id) 269 | return LANG_DICT[text_id] or text_id or "" 270 | end 271 | 272 | 273 | ---Get random translation for text id, split by \n symbol 274 | ---@param text_id string text id from your localization 275 | ---@return string text ("ui_hint") -> "Hint 1" or "Hint 2" or ... 276 | function M.txr(text_id) 277 | local texts = lang_internal.split(LANG_DICT[text_id], "\n") 278 | return texts[math.random(1, #texts)] 279 | end 280 | 281 | 282 | ---Get translation for text id with params 283 | ---@param text_id string Text id from your localization 284 | ---@vararg string|number Params for translation 285 | ---@return string text ("ui_hello_name", "John") -> "Hello, John!" 286 | function M.txp(text_id, ...) 287 | return string.format(M.txt(text_id), ...) 288 | end 289 | 290 | 291 | ---Check is translation with text_id exist 292 | ---@param text_id string text id from your localization 293 | ---@return boolean is_exist Is translation exist for text_id 294 | function M.is_exist(text_id) 295 | return (not not LANG_DICT[text_id]) 296 | end 297 | 298 | 299 | ---Return list of available languages 300 | ---@return string[] langs List of available languages 301 | function M.get_langs() 302 | return LANGS_ORDER 303 | end 304 | 305 | 306 | ---Get current lang table { key = "value" } 307 | ---@return table lang_table 308 | function M.get_lang_table() 309 | return LANG_DICT 310 | end 311 | 312 | 313 | ---Check if language is available 314 | ---@param lang_id string Language code to check 315 | ---@return boolean is_available True if language is available 316 | function M.is_lang_available(lang_id) 317 | return is_lang_available(lang_id) 318 | end 319 | 320 | 321 | ---Render properties panel for lang module 322 | ---@param druid table druid instance 323 | ---@param properties_panel table druid properties panel instance 324 | function M.render_properties_panel(druid, properties_panel) 325 | lang_debug_page.render_properties_panel(M, druid, properties_panel) 326 | end 327 | 328 | 329 | return M 330 | -------------------------------------------------------------------------------- /lang/editor_script/json.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- json.lua 3 | -- 4 | -- Copyright (c) 2020 rxi 5 | -- 6 | -- Permission is hereby granted, free of charge, to any person obtaining a copy of 7 | -- this software and associated documentation files (the "Software"), to deal in 8 | -- the Software without restriction, including without limitation the rights to 9 | -- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 10 | -- of the Software, and to permit persons to whom the Software is furnished to do 11 | -- so, subject to the following conditions: 12 | -- 13 | -- The above copyright notice and this permission notice shall be included in all 14 | -- copies or substantial portions of the Software. 15 | -- 16 | -- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | -- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | -- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | -- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | -- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | -- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | -- SOFTWARE. 23 | -- 24 | 25 | local json = { _version = "0.1.2" } 26 | 27 | ------------------------------------------------------------------------------- 28 | -- Encode 29 | ------------------------------------------------------------------------------- 30 | 31 | local encode 32 | 33 | local escape_char_map = { 34 | [ "\\" ] = "\\", 35 | [ "\"" ] = "\"", 36 | [ "\b" ] = "b", 37 | [ "\f" ] = "f", 38 | [ "\n" ] = "n", 39 | [ "\r" ] = "r", 40 | [ "\t" ] = "t", 41 | } 42 | 43 | local escape_char_map_inv = { [ "/" ] = "/" } 44 | for k, v in pairs(escape_char_map) do 45 | escape_char_map_inv[v] = k 46 | end 47 | 48 | 49 | local function escape_char(c) 50 | return "\\" .. (escape_char_map[c] or string.format("u%04x", c:byte())) 51 | end 52 | 53 | 54 | local function encode_nil(val) 55 | return "null" 56 | end 57 | 58 | 59 | local function encode_table(val, stack) 60 | local res = {} 61 | stack = stack or {} 62 | 63 | -- Circular reference? 64 | if stack[val] then error("circular reference") end 65 | 66 | stack[val] = true 67 | 68 | if rawget(val, 1) ~= nil or next(val) == nil then 69 | -- Treat as array -- check keys are valid and it is not sparse 70 | local n = 0 71 | for k in pairs(val) do 72 | if type(k) ~= "number" then 73 | error("invalid table: mixed or invalid key types") 74 | end 75 | n = n + 1 76 | end 77 | if n ~= #val then 78 | error("invalid table: sparse array") 79 | end 80 | -- Encode 81 | for i, v in ipairs(val) do 82 | table.insert(res, encode(v, stack)) 83 | end 84 | stack[val] = nil 85 | return "[" .. table.concat(res, ",") .. "]" 86 | 87 | else 88 | -- Treat as an object 89 | for k, v in pairs(val) do 90 | if type(k) ~= "string" then 91 | error("invalid table: mixed or invalid key types") 92 | end 93 | table.insert(res, encode(k, stack) .. ":" .. encode(v, stack)) 94 | end 95 | stack[val] = nil 96 | return "{" .. table.concat(res, ",") .. "}" 97 | end 98 | end 99 | 100 | 101 | local function encode_string(val) 102 | return '"' .. val:gsub('[%z\1-\31\\"]', escape_char) .. '"' 103 | end 104 | 105 | 106 | local function encode_number(val) 107 | -- Check for NaN, -inf and inf 108 | if val ~= val or val <= -math.huge or val >= math.huge then 109 | error("unexpected number value '" .. tostring(val) .. "'") 110 | end 111 | return string.format("%.14g", val) 112 | end 113 | 114 | 115 | local type_func_map = { 116 | [ "nil" ] = encode_nil, 117 | [ "table" ] = encode_table, 118 | [ "string" ] = encode_string, 119 | [ "number" ] = encode_number, 120 | [ "boolean" ] = tostring, 121 | } 122 | 123 | 124 | encode = function(val, stack) 125 | local t = type(val) 126 | local f = type_func_map[t] 127 | if f then 128 | return f(val, stack) 129 | end 130 | error("unexpected type '" .. t .. "'") 131 | end 132 | 133 | 134 | function json.encode(val) 135 | return ( encode(val) ) 136 | end 137 | 138 | 139 | ------------------------------------------------------------------------------- 140 | -- Decode 141 | ------------------------------------------------------------------------------- 142 | 143 | local parse 144 | 145 | local function create_set(...) 146 | local res = {} 147 | for i = 1, select("#", ...) do 148 | res[ select(i, ...) ] = true 149 | end 150 | return res 151 | end 152 | 153 | local space_chars = create_set(" ", "\t", "\r", "\n") 154 | local delim_chars = create_set(" ", "\t", "\r", "\n", "]", "}", ",") 155 | local escape_chars = create_set("\\", "/", '"', "b", "f", "n", "r", "t", "u") 156 | local literals = create_set("true", "false", "null") 157 | 158 | local literal_map = { 159 | [ "true" ] = true, 160 | [ "false" ] = false, 161 | [ "null" ] = nil, 162 | } 163 | 164 | 165 | local function next_char(str, idx, set, negate) 166 | for i = idx, #str do 167 | if set[str:sub(i, i)] ~= negate then 168 | return i 169 | end 170 | end 171 | return #str + 1 172 | end 173 | 174 | 175 | local function decode_error(str, idx, msg) 176 | local line_count = 1 177 | local col_count = 1 178 | for i = 1, idx - 1 do 179 | col_count = col_count + 1 180 | if str:sub(i, i) == "\n" then 181 | line_count = line_count + 1 182 | col_count = 1 183 | end 184 | end 185 | error( string.format("%s at line %d col %d", msg, line_count, col_count) ) 186 | end 187 | 188 | 189 | local function codepoint_to_utf8(n) 190 | -- http://scripts.sil.org/cms/scripts/page.php?site_id=nrsi&id=iws-appendixa 191 | local f = math.floor 192 | if n <= 0x7f then 193 | return string.char(n) 194 | elseif n <= 0x7ff then 195 | return string.char(f(n / 64) + 192, n % 64 + 128) 196 | elseif n <= 0xffff then 197 | return string.char(f(n / 4096) + 224, f(n % 4096 / 64) + 128, n % 64 + 128) 198 | elseif n <= 0x10ffff then 199 | return string.char(f(n / 262144) + 240, f(n % 262144 / 4096) + 128, 200 | f(n % 4096 / 64) + 128, n % 64 + 128) 201 | end 202 | error( string.format("invalid unicode codepoint '%x'", n) ) 203 | end 204 | 205 | 206 | local function parse_unicode_escape(s) 207 | local n1 = tonumber( s:sub(1, 4), 16 ) 208 | local n2 = tonumber( s:sub(7, 10), 16 ) 209 | -- Surrogate pair? 210 | if n2 then 211 | return codepoint_to_utf8((n1 - 0xd800) * 0x400 + (n2 - 0xdc00) + 0x10000) 212 | else 213 | return codepoint_to_utf8(n1) 214 | end 215 | end 216 | 217 | 218 | local function parse_string(str, i) 219 | local res = "" 220 | local j = i + 1 221 | local k = j 222 | 223 | while j <= #str do 224 | local x = str:byte(j) 225 | 226 | if x < 32 then 227 | decode_error(str, j, "control character in string") 228 | 229 | elseif x == 92 then -- `\`: Escape 230 | res = res .. str:sub(k, j - 1) 231 | j = j + 1 232 | local c = str:sub(j, j) 233 | if c == "u" then 234 | local hex = str:match("^[dD][89aAbB]%x%x\\u%x%x%x%x", j + 1) 235 | or str:match("^%x%x%x%x", j + 1) 236 | or decode_error(str, j - 1, "invalid unicode escape in string") 237 | res = res .. parse_unicode_escape(hex) 238 | j = j + #hex 239 | else 240 | if not escape_chars[c] then 241 | decode_error(str, j - 1, "invalid escape char '" .. c .. "' in string") 242 | end 243 | res = res .. escape_char_map_inv[c] 244 | end 245 | k = j + 1 246 | 247 | elseif x == 34 then -- `"`: End of string 248 | res = res .. str:sub(k, j - 1) 249 | return res, j + 1 250 | end 251 | 252 | j = j + 1 253 | end 254 | 255 | decode_error(str, i, "expected closing quote for string") 256 | end 257 | 258 | 259 | local function parse_number(str, i) 260 | local x = next_char(str, i, delim_chars) 261 | local s = str:sub(i, x - 1) 262 | local n = tonumber(s) 263 | if not n then 264 | decode_error(str, i, "invalid number '" .. s .. "'") 265 | end 266 | return n, x 267 | end 268 | 269 | 270 | local function parse_literal(str, i) 271 | local x = next_char(str, i, delim_chars) 272 | local word = str:sub(i, x - 1) 273 | if not literals[word] then 274 | decode_error(str, i, "invalid literal '" .. word .. "'") 275 | end 276 | return literal_map[word], x 277 | end 278 | 279 | 280 | local function parse_array(str, i) 281 | local res = {} 282 | local n = 1 283 | i = i + 1 284 | while 1 do 285 | local x 286 | i = next_char(str, i, space_chars, true) 287 | -- Empty / end of array? 288 | if str:sub(i, i) == "]" then 289 | i = i + 1 290 | break 291 | end 292 | -- Read token 293 | x, i = parse(str, i) 294 | res[n] = x 295 | n = n + 1 296 | -- Next token 297 | i = next_char(str, i, space_chars, true) 298 | local chr = str:sub(i, i) 299 | i = i + 1 300 | if chr == "]" then break end 301 | if chr ~= "," then decode_error(str, i, "expected ']' or ','") end 302 | end 303 | return res, i 304 | end 305 | 306 | 307 | local function parse_object(str, i) 308 | local res = {} 309 | i = i + 1 310 | while 1 do 311 | local key, val 312 | i = next_char(str, i, space_chars, true) 313 | -- Empty / end of object? 314 | if str:sub(i, i) == "}" then 315 | i = i + 1 316 | break 317 | end 318 | -- Read key 319 | if str:sub(i, i) ~= '"' then 320 | decode_error(str, i, "expected string for key") 321 | end 322 | key, i = parse(str, i) 323 | -- Read ':' delimiter 324 | i = next_char(str, i, space_chars, true) 325 | if str:sub(i, i) ~= ":" then 326 | decode_error(str, i, "expected ':' after key") 327 | end 328 | i = next_char(str, i + 1, space_chars, true) 329 | -- Read value 330 | val, i = parse(str, i) 331 | -- Set 332 | res[key] = val 333 | -- Next token 334 | i = next_char(str, i, space_chars, true) 335 | local chr = str:sub(i, i) 336 | i = i + 1 337 | if chr == "}" then break end 338 | if chr ~= "," then decode_error(str, i, "expected '}' or ','") end 339 | end 340 | return res, i 341 | end 342 | 343 | 344 | local char_func_map = { 345 | [ '"' ] = parse_string, 346 | [ "0" ] = parse_number, 347 | [ "1" ] = parse_number, 348 | [ "2" ] = parse_number, 349 | [ "3" ] = parse_number, 350 | [ "4" ] = parse_number, 351 | [ "5" ] = parse_number, 352 | [ "6" ] = parse_number, 353 | [ "7" ] = parse_number, 354 | [ "8" ] = parse_number, 355 | [ "9" ] = parse_number, 356 | [ "-" ] = parse_number, 357 | [ "t" ] = parse_literal, 358 | [ "f" ] = parse_literal, 359 | [ "n" ] = parse_literal, 360 | [ "[" ] = parse_array, 361 | [ "{" ] = parse_object, 362 | } 363 | 364 | 365 | parse = function(str, idx) 366 | local chr = str:sub(idx, idx) 367 | local f = char_func_map[chr] 368 | if f then 369 | return f(str, idx) 370 | end 371 | decode_error(str, idx, "unexpected character '" .. chr .. "'") 372 | end 373 | 374 | 375 | function json.decode(str) 376 | if type(str) ~= "string" then 377 | error("expected argument of type string, got " .. type(str)) 378 | end 379 | local res, idx = parse(str, next_char(str, 1, space_chars, true)) 380 | idx = next_char(str, idx, space_chars, true) 381 | if idx <= #str then 382 | decode_error(str, idx, "trailing garbage") 383 | end 384 | return res 385 | end 386 | 387 | 388 | return json -------------------------------------------------------------------------------- /test/test_lang.lua: -------------------------------------------------------------------------------- 1 | return function() 2 | local lang = {} --[[@as lang]] 3 | 4 | describe("Defold Lang", function() 5 | before(function() 6 | lang = require("lang.lang") 7 | lang.init({ 8 | { id = "en", path = "/resources/lang/en.json" }, 9 | { id = "ru", path = "/resources/lang/ru.json" }, 10 | { id = "es", path = "/resources/lang/es.json" }, 11 | }, "en") 12 | end) 13 | 14 | after(function() 15 | lang.reset_state() 16 | end) 17 | 18 | it("Should lang.txt return text", function() 19 | local text = lang.txt("ui_hello") 20 | assert_equal(text, "Hello, World!") 21 | 22 | lang.set_lang("ru") 23 | text = lang.txt("ui_hello") 24 | assert_equal(text, "Привет, Мир!") 25 | 26 | lang.set_lang("es") 27 | text = lang.txt("ui_hello") 28 | assert_equal(text, "¡Hola, Mundo!") 29 | end) 30 | 31 | it("Should lang.txt return key if not found", function() 32 | local text = lang.txt("ui_hello_not_found") 33 | assert_equal(text, "ui_hello_not_found") 34 | end) 35 | 36 | it("Should lang.txp return text with params", function() 37 | local text = lang.txp("ui_params", "User") 38 | assert_equal(text, "Hello, User") 39 | end) 40 | 41 | it("Should lang.txp return key if not found", function() 42 | local text = lang.txp("ui_params_not_found", "User") 43 | assert_equal(text, "ui_params_not_found") 44 | end) 45 | 46 | it("Should lang.set_lang change language", function() 47 | lang.set_lang("ru") 48 | local text = lang.txt("ui_hello") 49 | assert_equal(text, "Привет, Мир!") 50 | 51 | lang.set_lang("en") 52 | text = lang.txt("ui_hello") 53 | assert_equal(text, "Hello, World!") 54 | end) 55 | 56 | it("Should lang.get_lang return current language", function() 57 | local lang_name = lang.get_lang() 58 | assert(lang_name == "en") 59 | end) 60 | 61 | it("Should lang.get_langs return all languages", function() 62 | local langs = lang.get_langs() 63 | assert(#langs == 3) 64 | assert(langs[1] == "en") 65 | assert(langs[2] == "ru") 66 | assert(langs[3] == "es") 67 | end) 68 | 69 | it("Should next_lang change language", function() 70 | lang.set_next_lang() 71 | local lang_name = lang.get_lang() 72 | assert(lang_name == "ru") 73 | 74 | lang.set_next_lang() 75 | lang_name = lang.get_lang() 76 | assert(lang_name == "es") 77 | 78 | lang.set_next_lang() 79 | lang_name = lang.get_lang() 80 | assert(lang_name == "en") 81 | end) 82 | 83 | it("Should get next lang before change", function() 84 | local lang_name = lang.get_next_lang() 85 | assert(lang_name == "ru") 86 | 87 | lang.set_lang(lang_name) 88 | lang_name = lang.get_next_lang() 89 | assert(lang_name == "es") 90 | 91 | lang.set_lang(lang_name) 92 | lang_name = lang.get_next_lang() 93 | assert(lang_name == "en") 94 | end) 95 | 96 | it("Should txr return random text", function() 97 | local text = lang.txr("ui_random") 98 | assert_equal(text, "String 1" or text == "String 2" or text == "String 3") 99 | end) 100 | 101 | it("Should not change language if not found", function() 102 | lang.set_lang("fr") 103 | local text = lang.txt("ui_hello") 104 | print(lang.state) 105 | print(lang.get_lang()) 106 | assert_equal(text, "Hello, World!") 107 | end) 108 | 109 | it("Should able to check is text_id exists", function() 110 | local is_exists = lang.is_exist("ui_hello") 111 | assert(is_exists) 112 | 113 | is_exists = lang.is_exist("ui_hello_not_found") 114 | assert(not is_exists) 115 | end) 116 | 117 | it("Should not change language if not found", function() 118 | lang.init({ 119 | { id = "en", path = "/resources/lang/en.json" }, 120 | }, "fr") 121 | 122 | local text = lang.txt("ui_hello") 123 | assert_equal(text, "Hello, World!") 124 | assert_equal(lang.state.lang, "en") 125 | end) 126 | 127 | 128 | -- Lua file format tests 129 | it("Should load Lua files correctly", function() 130 | lang.init({ 131 | { id = "en", path = require("resources.lang.en") }, 132 | { id = "ru", path = require("resources.lang.ru") }, 133 | { id = "es", path = require("resources.lang.es") }, 134 | }, "en") 135 | 136 | local text = lang.txt("ui_hello") 137 | assert_equal(text, "Hello, World!") 138 | 139 | lang.set_lang("ru") 140 | text = lang.txt("ui_hello") 141 | assert_equal(text, "Привет, Мир!") 142 | 143 | lang.set_lang("es") 144 | text = lang.txt("ui_hello") 145 | assert_equal(text, "¡Hola, Mundo!") 146 | end) 147 | 148 | it("Should handle Lua file parameters", function() 149 | lang.init({ 150 | { id = "en", path = require("resources.lang.en") }, 151 | { id = "ru", path = require("resources.lang.ru") }, 152 | }, "en") 153 | 154 | local text = lang.txp("ui_params", "User") 155 | assert_equal(text, "Hello, User") 156 | 157 | lang.set_lang("ru") 158 | text = lang.txp("ui_params", "Пользователь") 159 | assert_equal(text, "Привет, Пользователь") 160 | end) 161 | 162 | it("Should handle Lua file random text", function() 163 | lang.init({ 164 | { id = "en", path = require("resources.lang.en") }, 165 | }, "en") 166 | 167 | local text = lang.txr("ui_random") 168 | assert(text == "String 1" or text == "String 2" or text == "String 3") 169 | end) 170 | 171 | 172 | -- CSV file format tests 173 | it("Should load CSV files correctly", function() 174 | lang.init({ 175 | { id = "en", path = "/resources/lang/translations.csv" }, 176 | { id = "ru", path = "/resources/lang/translations.csv" }, 177 | { id = "es", path = "/resources/lang/translations.csv" }, 178 | }, "en") 179 | 180 | local text = lang.txt("ui_hello") 181 | assert_equal(text, "Hello, World!") 182 | 183 | lang.set_lang("ru") 184 | text = lang.txt("ui_hello") 185 | assert_equal(text, "Привет, Мир!") 186 | 187 | lang.set_lang("es") 188 | text = lang.txt("ui_hello") 189 | assert_equal(text, "¡Hola, Mundo!") 190 | end) 191 | 192 | it("Should handle CSV file parameters", function() 193 | lang.init({ 194 | { id = "en", path = "/resources/lang/translations.csv" }, 195 | { id = "ru", path = "/resources/lang/translations.csv" }, 196 | }, "en") 197 | 198 | local text = lang.txp("ui_params", "User") 199 | assert_equal(text, "Hello, User") 200 | 201 | lang.set_lang("ru") 202 | text = lang.txp("ui_params", "Пользователь") 203 | assert_equal(text, "Привет, Пользователь") 204 | end) 205 | 206 | it("Should handle CSV file random text", function() 207 | lang.init({ 208 | { id = "en", path = "/resources/lang/translations.csv" }, 209 | { id = "ru", path = "/resources/lang/translations.csv" }, 210 | }, "en") 211 | 212 | local text = lang.txr("ui_random") 213 | assert(text == "String 1" or text == "String 2" or text == "String 3") 214 | 215 | lang.set_lang("ru") 216 | text = lang.txr("ui_random") 217 | assert(text == "Строка 1" or text == "Строка 2" or text == "Строка 3") 218 | end) 219 | 220 | 221 | -- Mixed file format tests 222 | it("Should handle mixed file formats", function() 223 | lang.init({ 224 | { id = "en", path = "/resources/lang/en.json" }, 225 | { id = "ru", path = require("resources.lang.ru") }, 226 | { id = "es", path = "/resources/lang/translations.csv" }, 227 | }, "en") 228 | 229 | -- Test JSON (en) 230 | local text = lang.txt("ui_hello") 231 | assert_equal(text, "Hello, World!") 232 | 233 | -- Test Lua (ru) 234 | lang.set_lang("ru") 235 | text = lang.txt("ui_hello") 236 | assert_equal(text, "Привет, Мир!") 237 | 238 | -- Test CSV (es) 239 | lang.set_lang("es") 240 | text = lang.txt("ui_hello") 241 | assert_equal(text, "¡Hola, Mundo!") 242 | end) 243 | 244 | it("Should handle mixed format parameters", function() 245 | lang.init({ 246 | { id = "en", path = "/resources/lang/en.json" }, 247 | { id = "ru", path = require("resources.lang.ru") }, 248 | { id = "es", path = "/resources/lang/translations.csv" }, 249 | }, "en") 250 | 251 | -- JSON format 252 | local text = lang.txp("ui_params", "User") 253 | assert_equal(text, "Hello, User") 254 | 255 | -- Lua format 256 | lang.set_lang("ru") 257 | text = lang.txp("ui_params", "Пользователь") 258 | assert_equal(text, "Привет, Пользователь") 259 | 260 | -- CSV format 261 | lang.set_lang("es") 262 | text = lang.txp("ui_params", "Usuario") 263 | assert_equal(text, "Hola, Usuario") 264 | end) 265 | 266 | it("Should handle language switching between formats", function() 267 | lang.init({ 268 | { id = "en", path = "/resources/lang/en.json" }, 269 | { id = "ru", path = require("resources.lang.ru") }, 270 | { id = "es", path = "/resources/lang/translations.csv" }, 271 | }, "en") 272 | 273 | local langs = lang.get_langs() 274 | assert(#langs == 3) 275 | assert(langs[1] == "en") 276 | assert(langs[2] == "ru") 277 | assert(langs[3] == "es") 278 | 279 | -- Test cycling through mixed formats 280 | lang.set_next_lang() 281 | assert_equal(lang.get_lang(), "ru") 282 | local text = lang.txt("ui_hello") 283 | assert_equal(text, "Привет, Мир!") 284 | 285 | lang.set_next_lang() 286 | assert_equal(lang.get_lang(), "es") 287 | text = lang.txt("ui_hello") 288 | assert_equal(text, "¡Hola, Mundo!") 289 | 290 | lang.set_next_lang() 291 | assert_equal(lang.get_lang(), "en") 292 | text = lang.txt("ui_hello") 293 | assert_equal(text, "Hello, World!") 294 | end) 295 | 296 | 297 | -- Error handling and edge case tests 298 | describe("Error Handling", function() 299 | after(function() 300 | lang.reset_state() 301 | end) 302 | 303 | it("Should handle invalid Lua file gracefully", function() 304 | -- This should not crash, but may not load properly 305 | local success = pcall(function() 306 | lang.init({ 307 | { id = "en", path = "/resources/lang/nonexistent.lua" }, 308 | }, "en") 309 | end) 310 | -- The init should handle the error gracefully 311 | assert(success == true or success == false) -- Just ensure no crash 312 | end) 313 | 314 | it("Should handle invalid CSV file gracefully", function() 315 | -- This should not crash, but may not load properly 316 | local success = pcall(function() 317 | lang.init({ 318 | { id = "en", path = "/resources/lang/nonexistent.csv" }, 319 | }, "en") 320 | end) 321 | -- The init should handle the error gracefully 322 | assert(success == true or success == false) -- Just ensure no crash 323 | end) 324 | 325 | it("Should handle empty language array", function() 326 | local success = pcall(function() 327 | lang.init({}, "en") 328 | end) 329 | -- Should handle empty array gracefully 330 | assert(success == true or success == false) 331 | end) 332 | end) 333 | 334 | 335 | -- State management tests 336 | describe("State Management", function() 337 | before(function() 338 | lang.reset_state() 339 | end) 340 | 341 | after(function() 342 | lang.reset_state() 343 | end) 344 | 345 | it("Should preserve state between inits", function() 346 | -- Set up initial state 347 | lang.state.lang = "ru" 348 | 349 | lang.init({ 350 | { id = "en", path = "/resources/lang/en.json" }, 351 | { id = "ru", path = "/resources/lang/ru.json" }, 352 | { id = "es", path = "/resources/lang/es.json" }, 353 | }) 354 | 355 | -- Should use the state language 356 | assert_equal(lang.get_lang(), "ru") 357 | local text = lang.txt("ui_hello") 358 | assert_equal(text, "Привет, Мир!") 359 | end) 360 | 361 | it("Should override state with force parameter", function() 362 | -- Set up initial state 363 | lang.state.lang = "ru" 364 | 365 | lang.init({ 366 | { id = "en", path = "/resources/lang/en.json" }, 367 | { id = "ru", path = "/resources/lang/ru.json" }, 368 | { id = "es", path = "/resources/lang/es.json" }, 369 | }, "es") -- Force Spanish 370 | 371 | -- Should use forced language, not state 372 | assert_equal(lang.get_lang(), "es") 373 | local text = lang.txt("ui_hello") 374 | assert_equal(text, "¡Hola, Mundo!") 375 | end) 376 | end) 377 | end) 378 | end 379 | -------------------------------------------------------------------------------- /lang/csv.lua: -------------------------------------------------------------------------------- 1 | 2 | --- Read a comma or tab (or other delimiter) separated file. 3 | -- This version of a CSV reader differs from others I've seen in that it 4 | -- 5 | -- + handles embedded newlines in fields (if they're delimited with double 6 | -- quotes) 7 | -- + is line-ending agnostic 8 | -- + reads the file line-by-line, so it can potientially handle large 9 | -- files. 10 | -- 11 | -- Of course, for such a simple format, CSV is horribly complicated, so it 12 | -- likely gets something wrong. 13 | 14 | -- (c) Copyright 2013-2014 Incremental IP Limited. 15 | -- (c) Copyright 2014 Kevin Martin 16 | -- Available under the MIT licence. See LICENSE for more information. 17 | 18 | local DEFAULT_BUFFER_BLOCK_SIZE = 1024 * 1024 19 | 20 | 21 | ------------------------------------------------------------------------------ 22 | 23 | local function trim_space(s) 24 | return s:match("^%s*(.-)%s*$") 25 | end 26 | 27 | 28 | local function fix_quotes(s) 29 | -- the sub(..., -2) is to strip the trailing quote 30 | return string.sub(s:gsub('""', '"'), 1, -2) 31 | end 32 | 33 | 34 | ------------------------------------------------------------------------------ 35 | 36 | local column_map = {} 37 | column_map.__index = column_map 38 | 39 | 40 | local function normalise_string(s) 41 | return (s:lower():gsub("[^%w%d]+", " "):gsub("^ *(.-) *$", "%1")) 42 | end 43 | 44 | 45 | --- Parse a list of columns. 46 | -- The main job here is normalising column names and dealing with columns 47 | -- for which we have more than one possible name in the header. 48 | function column_map:new(columns) 49 | local name_map = {} 50 | for n, v in pairs(columns) do 51 | local names 52 | local t 53 | if type(v) == "table" then 54 | t = { transform = v.transform, default = v.default } 55 | if v.name then 56 | names = { normalise_string(v.name) } 57 | elseif v.names then 58 | names = v.names 59 | for i, n in ipairs(names) do names[i] = normalise_string(n) end 60 | end 61 | else 62 | if type(v) == "function" then 63 | t = { transform = v } 64 | else 65 | t = {} 66 | if type(v) == "string" then 67 | names = { normalise_string(v) } 68 | end 69 | end 70 | end 71 | 72 | if not names then 73 | names = { (n:lower():gsub("[^%w%d]+", " ")) } 74 | end 75 | 76 | t.name = n 77 | for _, n in ipairs(names) do 78 | name_map[n:lower()] = t 79 | end 80 | end 81 | 82 | return setmetatable({ name_map = name_map }, column_map) 83 | end 84 | 85 | 86 | --- Map "virtual" columns to file columns. 87 | -- Once we've read the header, work out which columns we're interested in and 88 | -- what to do with them. Mostly this is about checking we've got the columns 89 | -- we need and writing a nice complaint if we haven't. 90 | function column_map:read_header(header) 91 | local index_map = {} 92 | 93 | -- Match the columns in the file to the columns in the name map 94 | local found = {} 95 | local found_any 96 | for i, word in ipairs(header) do 97 | word = normalise_string(word) 98 | local r = self.name_map[word] 99 | if r then 100 | index_map[i] = r 101 | found[r.name] = true 102 | found_any = true 103 | end 104 | end 105 | 106 | if not found_any then return end 107 | 108 | -- check we found all the columns we need 109 | local not_found = {} 110 | for name, r in pairs(self.name_map) do 111 | if not found[r.name] then 112 | local nf = not_found[r.name] 113 | if nf then 114 | nf[#nf+1] = name 115 | else 116 | not_found[r.name] = { name } 117 | end 118 | end 119 | end 120 | -- If any columns are missing, assemble an error message 121 | if next(not_found) then 122 | local problems = {} 123 | for k, v in pairs(not_found) do 124 | local missing 125 | if #v == 1 then 126 | missing = "'"..v[1].."'" 127 | else 128 | missing = v[1] 129 | for i = 2, #v - 1 do 130 | missing = missing..", '"..v[i].."'" 131 | end 132 | missing = missing.." or '"..v[#v].."'" 133 | end 134 | problems[#problems+1] = "Couldn't find a column named "..missing 135 | end 136 | error(table.concat(problems, "\n"), 0) 137 | end 138 | 139 | self.index_map = index_map 140 | return true 141 | end 142 | 143 | 144 | function column_map:transform(value, index) 145 | local field = self.index_map[index] 146 | if field then 147 | if field.transform then 148 | local ok 149 | ok, value = pcall(field.transform, value) 150 | if not ok then 151 | error(("Error reading field '%s': %s"):format(field.name, value), 0) 152 | end 153 | end 154 | return value or field.default, field.name 155 | end 156 | end 157 | 158 | 159 | ------------------------------------------------------------------------------ 160 | 161 | local file_buffer = {} 162 | file_buffer.__index = file_buffer 163 | 164 | function file_buffer:new(file, buffer_block_size) 165 | return setmetatable({ 166 | file = file, 167 | buffer_block_size = buffer_block_size or DEFAULT_BUFFER_BLOCK_SIZE, 168 | buffer_start = 0, 169 | buffer = "", 170 | }, file_buffer) 171 | end 172 | 173 | 174 | --- Cut the front off the buffer if we've already read it 175 | function file_buffer:truncate(p) 176 | p = p - self.buffer_start 177 | if p > self.buffer_block_size then 178 | local remove = self.buffer_block_size * 179 | math.floor((p-1) / self.buffer_block_size) 180 | self.buffer = self.buffer:sub(remove + 1) 181 | self.buffer_start = self.buffer_start + remove 182 | end 183 | end 184 | 185 | 186 | --- Find something in the buffer, extending it if necessary 187 | function file_buffer:find(pattern, init) 188 | while true do 189 | local first, last, capture = 190 | self.buffer:find(pattern, init - self.buffer_start) 191 | -- if we found nothing, or the last character is at the end of the 192 | -- buffer (and the match could potentially be longer) then read some 193 | -- more. 194 | if not first or last == #self.buffer then 195 | local s = self.file:read(self.buffer_block_size) 196 | if not s then 197 | if not first then 198 | return 199 | else 200 | return first + self.buffer_start, last + self.buffer_start, capture 201 | end 202 | end 203 | self.buffer = self.buffer..s 204 | else 205 | return first + self.buffer_start, last + self.buffer_start, capture 206 | end 207 | end 208 | end 209 | 210 | 211 | --- Extend the buffer so we can see more 212 | function file_buffer:extend(offset) 213 | local extra = offset - #self.buffer - self.buffer_start 214 | if extra > 0 then 215 | local size = self.buffer_block_size * 216 | math.ceil(extra / self.buffer_block_size) 217 | local s = self.file:read(size) 218 | if not s then return end 219 | self.buffer = self.buffer..s 220 | end 221 | end 222 | 223 | 224 | --- Get a substring from the buffer, extending it if necessary 225 | function file_buffer:sub(a, b) 226 | self:extend(b) 227 | b = b == -1 and b or b - self.buffer_start 228 | return self.buffer:sub(a - self.buffer_start, b) 229 | end 230 | 231 | 232 | --- Close a file buffer 233 | function file_buffer:close() 234 | self.file:close() 235 | self.file = nil 236 | end 237 | 238 | 239 | ------------------------------------------------------------------------------ 240 | 241 | local separator_candidates = { ",", "\t", "|" } 242 | local guess_separator_params = { record_limit = 8; } 243 | 244 | 245 | local function try_separator(buffer, sep, f) 246 | guess_separator_params.separator = sep 247 | local min, max = math.huge, 0 248 | local lines, split_lines = 0, 0 249 | local iterator = coroutine.wrap(function() f(buffer, guess_separator_params) end) 250 | for t in iterator do 251 | min = math.min(min, #t) 252 | max = math.max(max, #t) 253 | split_lines = split_lines + (t[2] and 1 or 0) 254 | lines = lines + 1 255 | end 256 | if split_lines / lines > 0.75 then 257 | return max - min 258 | else 259 | return math.huge 260 | end 261 | end 262 | 263 | 264 | --- If the user hasn't specified a separator, try to work out what it is. 265 | function guess_separator(buffer, f) 266 | local best_separator, lowest_diff = "", math.huge 267 | for _, s in ipairs(separator_candidates) do 268 | local ok, diff = pcall(function() return try_separator(buffer, s, f) end) 269 | if ok and diff < lowest_diff then 270 | best_separator = s 271 | lowest_diff = diff 272 | end 273 | end 274 | 275 | return best_separator 276 | end 277 | 278 | 279 | local unicode_BOMS = 280 | { 281 | { 282 | length = 2, 283 | BOMS = 284 | { 285 | ["\254\255"] = true, -- UTF-16 big-endian 286 | ["\255\254"] = true, -- UTF-16 little-endian 287 | } 288 | }, 289 | { 290 | length = 3, 291 | BOMS = 292 | { 293 | ["\239\187\191"] = true, -- UTF-8 294 | } 295 | } 296 | } 297 | 298 | 299 | local function find_unicode_BOM(sub) 300 | for _, x in ipairs(unicode_BOMS) do 301 | local code = sub(1, x.length) 302 | if x.BOMS[code] then 303 | return x.length 304 | end 305 | end 306 | return 0 307 | end 308 | 309 | 310 | --- Iterate through the records in a file 311 | -- Since records might be more than one line (if there's a newline in quotes) 312 | -- and line-endings might not be native, we read the file in chunks of 313 | -- we read the file in chunks using a file_buffer, rather than line-by-line 314 | -- using io.lines. 315 | local function separated_values_iterator(buffer, parameters) 316 | local field_start = 1 317 | 318 | local advance 319 | if buffer.truncate then 320 | advance = function(n) 321 | field_start = field_start + n 322 | buffer:truncate(field_start) 323 | end 324 | else 325 | advance = function(n) 326 | field_start = field_start + n 327 | end 328 | end 329 | 330 | 331 | local function field_sub(a, b) 332 | b = b == -1 and b or b + field_start - 1 333 | return buffer:sub(a + field_start - 1, b) 334 | end 335 | 336 | 337 | local function field_find(pattern, init) 338 | init = init or 1 339 | local f, l, c = buffer:find(pattern, init + field_start - 1) 340 | if not f then return end 341 | return f - field_start + 1, l - field_start + 1, c 342 | end 343 | 344 | 345 | -- Is there some kind of Unicode BOM here? 346 | advance(find_unicode_BOM(field_sub)) 347 | 348 | 349 | -- Start reading the file 350 | local sep = "(["..(parameters.separator or 351 | guess_separator(buffer, separated_values_iterator)).."\n\r])" 352 | local line_start = 1 353 | local line = 1 354 | local field_count, fields, starts, nonblanks = 0, {}, {} 355 | local header, header_read 356 | local field_start_line, field_start_column 357 | local record_count = 0 358 | 359 | 360 | local function problem(message) 361 | error(("%s:%d:%d: %s"): 362 | format(parameters.filename, field_start_line, field_start_column, 363 | message), 0) 364 | end 365 | 366 | 367 | while true do 368 | local field_end, sep_end, this_sep 369 | local tidy 370 | field_start_line = line 371 | field_start_column = field_start - line_start + 1 372 | 373 | -- If the field is quoted, go find the other quote 374 | if field_sub(1, 1) == '"' then 375 | advance(1) 376 | local current_pos = 0 377 | repeat 378 | local a, b, c = field_find('"("?)', current_pos + 1) 379 | current_pos = b 380 | until c ~= '"' 381 | if not current_pos then problem("unmatched quote") end 382 | tidy = fix_quotes 383 | field_end, sep_end, this_sep = field_find(" *([^ ])", current_pos+1) 384 | if this_sep and not this_sep:match(sep) then problem("unmatched quote") end 385 | else 386 | field_end, sep_end, this_sep = field_find(sep, 1) 387 | tidy = trim_space 388 | end 389 | 390 | -- Look for the separator or a newline or the end of the file 391 | field_end = (field_end or 0) - 1 392 | 393 | -- Read the field, then convert all the line endings to \n, and 394 | -- count any embedded line endings 395 | local value = field_sub(1, field_end) 396 | value = value:gsub("\r\n", "\n"):gsub("\r", "\n") 397 | for nl in value:gmatch("\n()") do 398 | line = line + 1 399 | line_start = nl + field_start 400 | end 401 | 402 | value = tidy(value) 403 | if #value > 0 then nonblanks = true end 404 | field_count = field_count + 1 405 | 406 | -- Insert the value into the table for this "line" 407 | local key 408 | if parameters.column_map and header_read then 409 | local ok 410 | ok, value, key = pcall(parameters.column_map.transform, 411 | parameters.column_map, value, field_count) 412 | if not ok then problem(value) end 413 | elseif header then 414 | key = header[field_count] 415 | else 416 | key = field_count 417 | end 418 | if key then 419 | fields[key] = value 420 | starts[key] = { line=field_start_line, column=field_start_column } 421 | end 422 | 423 | -- if we ended on a newline then yield the fields on this line. 424 | if not this_sep or this_sep == "\r" or this_sep == "\n" then 425 | if parameters.column_map and not header_read then 426 | header_read = parameters.column_map:read_header(fields) 427 | elseif parameters.header and not header_read then 428 | if nonblanks or field_count > 1 then -- ignore blank lines 429 | header = fields 430 | header_read = true 431 | end 432 | else 433 | if nonblanks or field_count > 1 then -- ignore blank lines 434 | coroutine.yield(fields, starts) 435 | record_count = record_count + 1 436 | if parameters.record_limit and 437 | record_count >= parameters.record_limit then 438 | break 439 | end 440 | end 441 | end 442 | field_count, fields, starts, nonblanks = 0, {}, {} 443 | end 444 | 445 | -- If we *really* didn't find a separator then we're done. 446 | if not sep_end then break end 447 | 448 | -- If we ended on a newline then count it. 449 | if this_sep == "\r" or this_sep == "\n" then 450 | if this_sep == "\r" and field_sub(sep_end+1, sep_end+1) == "\n" then 451 | sep_end = sep_end + 1 452 | end 453 | line = line + 1 454 | line_start = field_start + sep_end 455 | end 456 | 457 | advance(sep_end) 458 | end 459 | end 460 | 461 | 462 | ------------------------------------------------------------------------------ 463 | 464 | local buffer_mt = 465 | { 466 | lines = function(t) 467 | return coroutine.wrap(function() 468 | separated_values_iterator(t.buffer, t.parameters) 469 | end) 470 | end, 471 | close = function(t) 472 | if t.buffer.close then t.buffer:close() end 473 | end, 474 | name = function(t) 475 | return t.parameters.filename 476 | end, 477 | } 478 | buffer_mt.__index = buffer_mt 479 | 480 | 481 | --- Use an existing file or buffer as a stream to read csv from. 482 | -- (A buffer is just something that looks like a string in that we can do 483 | -- `buffer:sub()` and `buffer:find()`) 484 | -- @return a file object 485 | local function use( 486 | buffer, -- ?string|file|buffer: the buffer to read from. If it's: 487 | -- - a string, read from that; 488 | -- - a file, turn it into a file_buffer; 489 | -- - nil, read from stdin 490 | -- otherwise assume it's already a a buffer. 491 | parameters) -- ?table: parameters controlling reading the file. 492 | -- See README.md 493 | parameters = parameters or {} 494 | parameters.filename = parameters.filename or "" 495 | parameters.column_map = parameters.columns and 496 | column_map:new(parameters.columns) 497 | 498 | if not buffer then 499 | buffer = file_buffer:new(io.stdin) 500 | elseif io.type(buffer) == "file" then 501 | buffer = file_buffer:new(buffer) 502 | end 503 | 504 | local f = { buffer = buffer, parameters = parameters } 505 | return setmetatable(f, buffer_mt) 506 | end 507 | 508 | 509 | ------------------------------------------------------------------------------ 510 | 511 | --- Open a file for reading as a delimited file 512 | -- @return a file object 513 | local function open( 514 | filename, -- string: name of the file to open 515 | parameters) -- ?table: parameters controlling reading the file. 516 | -- See README.md 517 | local file, message = io.open(filename, "r") 518 | if not file then return nil, message end 519 | 520 | parameters = parameters or {} 521 | parameters.filename = filename 522 | return use(file_buffer:new(file), parameters) 523 | end 524 | 525 | 526 | ------------------------------------------------------------------------------ 527 | 528 | local function makename(s) 529 | local t = {} 530 | t[#t+1] = "<(String) " 531 | t[#t+1] = (s:gmatch("[^\n]+")() or ""):sub(1,15) 532 | if #t[#t] > 14 then t[#t+1] = "..." end 533 | t[#t+1] = " >" 534 | return table.concat(t) 535 | end 536 | 537 | 538 | --- Open a string for reading as a delimited file 539 | -- @return a file object 540 | local function openstring( 541 | filecontents, -- string: The contents of the delimited file 542 | parameters) -- ?table: parameters controlling reading the file. 543 | -- See README.md 544 | 545 | parameters = parameters or {} 546 | 547 | 548 | parameters.filename = parameters.filename or makename(filecontents) 549 | parameters.buffer_size = parameters.buffer_size or #filecontents 550 | return use(filecontents, parameters) 551 | end 552 | 553 | 554 | ------------------------------------------------------------------------------ 555 | 556 | return { open = open, openstring = openstring, use = use } 557 | 558 | ------------------------------------------------------------------------------ 559 | -------------------------------------------------------------------------------- /lang/editor_script/utf8.lua: -------------------------------------------------------------------------------- 1 | -- $Id: utf8.lua 179 2009-04-03 18:10:03Z pasta $ 2 | -- 3 | -- Provides UTF-8 aware string functions implemented in pure lua: 4 | -- * utf8len(s) 5 | -- * utf8sub(s, i, j) 6 | -- * utf8reverse(s) 7 | -- * utf8char(unicode) 8 | -- * utf8unicode(s, i, j) 9 | -- * utf8gensub(s, sub_len) 10 | -- * utf8find(str, regex, init, plain) 11 | -- * utf8match(str, regex, init) 12 | -- * utf8gmatch(str, regex, all) 13 | -- * utf8gsub(str, regex, repl, limit) 14 | -- 15 | -- If utf8data.lua (containing the lower<->upper case mappings) is loaded, these 16 | -- additional functions are available: 17 | -- * utf8upper(s) 18 | -- * utf8lower(s) 19 | -- 20 | -- All functions behave as their non UTF-8 aware counterparts with the exception 21 | -- that UTF-8 characters are used instead of bytes for all units. 22 | 23 | --[[ 24 | Copyright (c) 2006-2007, Kyle Smith 25 | All rights reserved. 26 | 27 | Contributors: 28 | Alimov Stepan 29 | 30 | Redistribution and use in source and binary forms, with or without 31 | modification, are permitted provided that the following conditions are met: 32 | 33 | * Redistributions of source code must retain the above copyright notice, 34 | this list of conditions and the following disclaimer. 35 | * Redistributions in binary form must reproduce the above copyright 36 | notice, this list of conditions and the following disclaimer in the 37 | documentation and/or other materials provided with the distribution. 38 | * Neither the name of the author nor the names of its contributors may be 39 | used to endorse or promote products derived from this software without 40 | specific prior written permission. 41 | 42 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 43 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 44 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 45 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 46 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 47 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 48 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 49 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 50 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 51 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 52 | --]] 53 | 54 | -- ABNF from RFC 3629 55 | -- 56 | -- UTF8-octets = *( UTF8-char ) 57 | -- UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4 58 | -- UTF8-1 = %x00-7F 59 | -- UTF8-2 = %xC2-DF UTF8-tail 60 | -- UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / 61 | -- %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) 62 | -- UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / 63 | -- %xF4 %x80-8F 2( UTF8-tail ) 64 | -- UTF8-tail = %x80-BF 65 | -- 66 | 67 | local byte = string.byte 68 | local char = string.char 69 | local dump = string.dump 70 | local find = string.find 71 | local format = string.format 72 | local len = string.len 73 | local lower = string.lower 74 | local rep = string.rep 75 | local sub = string.sub 76 | local upper = string.upper 77 | 78 | -- returns the number of bytes used by the UTF-8 character at byte i in s 79 | -- also doubles as a UTF-8 character validator 80 | local function utf8charbytes (s, i) 81 | -- argument defaults 82 | i = i or 1 83 | 84 | -- argument checking 85 | if type(s) ~= "string" then 86 | error("bad argument #1 to 'utf8charbytes' (string expected, got ".. type(s).. ")") 87 | end 88 | if type(i) ~= "number" then 89 | error("bad argument #2 to 'utf8charbytes' (number expected, got ".. type(i).. ")") 90 | end 91 | 92 | local c = byte(s, i) 93 | 94 | -- determine bytes needed for character, based on RFC 3629 95 | -- validate byte 1 96 | if c > 0 and c <= 127 then 97 | -- UTF8-1 98 | return 1 99 | 100 | elseif c >= 194 and c <= 223 then 101 | -- UTF8-2 102 | local c2 = byte(s, i + 1) 103 | 104 | if not c2 then 105 | error("UTF-8 string terminated early") 106 | end 107 | 108 | -- validate byte 2 109 | if c2 < 128 or c2 > 191 then 110 | error("Invalid UTF-8 character") 111 | end 112 | 113 | return 2 114 | 115 | elseif c >= 224 and c <= 239 then 116 | -- UTF8-3 117 | local c2 = byte(s, i + 1) 118 | local c3 = byte(s, i + 2) 119 | 120 | if not c2 or not c3 then 121 | error("UTF-8 string terminated early") 122 | end 123 | 124 | -- validate byte 2 125 | if c == 224 and (c2 < 160 or c2 > 191) then 126 | error("Invalid UTF-8 character") 127 | elseif c == 237 and (c2 < 128 or c2 > 159) then 128 | error("Invalid UTF-8 character") 129 | elseif c2 < 128 or c2 > 191 then 130 | error("Invalid UTF-8 character") 131 | end 132 | 133 | -- validate byte 3 134 | if c3 < 128 or c3 > 191 then 135 | error("Invalid UTF-8 character") 136 | end 137 | 138 | return 3 139 | 140 | elseif c >= 240 and c <= 244 then 141 | -- UTF8-4 142 | local c2 = byte(s, i + 1) 143 | local c3 = byte(s, i + 2) 144 | local c4 = byte(s, i + 3) 145 | 146 | if not c2 or not c3 or not c4 then 147 | error("UTF-8 string terminated early") 148 | end 149 | 150 | -- validate byte 2 151 | if c == 240 and (c2 < 144 or c2 > 191) then 152 | error("Invalid UTF-8 character") 153 | elseif c == 244 and (c2 < 128 or c2 > 143) then 154 | error("Invalid UTF-8 character") 155 | elseif c2 < 128 or c2 > 191 then 156 | error("Invalid UTF-8 character") 157 | end 158 | 159 | -- validate byte 3 160 | if c3 < 128 or c3 > 191 then 161 | error("Invalid UTF-8 character") 162 | end 163 | 164 | -- validate byte 4 165 | if c4 < 128 or c4 > 191 then 166 | error("Invalid UTF-8 character") 167 | end 168 | 169 | return 4 170 | 171 | else 172 | error("Invalid UTF-8 character") 173 | end 174 | end 175 | 176 | -- returns the number of characters in a UTF-8 string 177 | local function utf8len (s) 178 | -- argument checking 179 | if type(s) ~= "string" then 180 | for k,v in pairs(s) do print('"',tostring(k),'"',tostring(v),'"') end 181 | error("bad argument #1 to 'utf8len' (string expected, got ".. type(s).. ")") 182 | end 183 | 184 | local pos = 1 185 | local bytes = len(s) 186 | local length = 0 187 | 188 | while pos <= bytes do 189 | length = length + 1 190 | pos = pos + utf8charbytes(s, pos) 191 | end 192 | 193 | return length 194 | end 195 | 196 | -- functions identically to string.sub except that i and j are UTF-8 characters 197 | -- instead of bytes 198 | local function utf8sub (s, i, j) 199 | -- argument defaults 200 | j = j or -1 201 | 202 | local pos = 1 203 | local bytes = len(s) 204 | local length = 0 205 | 206 | -- only set l if i or j is negative 207 | local l = (i >= 0 and j >= 0) or utf8len(s) 208 | local startChar = (i >= 0) and i or l + i + 1 209 | local endChar = (j >= 0) and j or l + j + 1 210 | 211 | -- can't have start before end! 212 | if startChar > endChar then 213 | return "" 214 | end 215 | 216 | -- byte offsets to pass to string.sub 217 | local startByte,endByte = 1,bytes 218 | 219 | while pos <= bytes do 220 | length = length + 1 221 | 222 | if length == startChar then 223 | startByte = pos 224 | end 225 | 226 | pos = pos + utf8charbytes(s, pos) 227 | 228 | if length == endChar then 229 | endByte = pos - 1 230 | break 231 | end 232 | end 233 | 234 | if startChar > length then startByte = bytes+1 end 235 | if endChar < 1 then endByte = 0 end 236 | 237 | return sub(s, startByte, endByte) 238 | end 239 | 240 | --[[ 241 | -- replace UTF-8 characters based on a mapping table 242 | local function utf8replace (s, mapping) 243 | -- argument checking 244 | if type(s) ~= "string" then 245 | error("bad argument #1 to 'utf8replace' (string expected, got ".. type(s).. ")") 246 | end 247 | if type(mapping) ~= "table" then 248 | error("bad argument #2 to 'utf8replace' (table expected, got ".. type(mapping).. ")") 249 | end 250 | 251 | local pos = 1 252 | local bytes = len(s) 253 | local charbytes 254 | local newstr = "" 255 | 256 | while pos <= bytes do 257 | charbytes = utf8charbytes(s, pos) 258 | local c = sub(s, pos, pos + charbytes - 1) 259 | 260 | newstr = newstr .. (mapping[c] or c) 261 | 262 | pos = pos + charbytes 263 | end 264 | 265 | return newstr 266 | end 267 | 268 | 269 | -- identical to string.upper except it knows about unicode simple case conversions 270 | local function utf8upper (s) 271 | return utf8replace(s, utf8_lc_uc) 272 | end 273 | 274 | -- identical to string.lower except it knows about unicode simple case conversions 275 | local function utf8lower (s) 276 | return utf8replace(s, utf8_uc_lc) 277 | end 278 | ]] 279 | 280 | -- identical to string.reverse except that it supports UTF-8 281 | local function utf8reverse (s) 282 | -- argument checking 283 | if type(s) ~= "string" then 284 | error("bad argument #1 to 'utf8reverse' (string expected, got ".. type(s).. ")") 285 | end 286 | 287 | local bytes = len(s) 288 | local pos = bytes 289 | local charbytes 290 | local newstr = "" 291 | 292 | while pos > 0 do 293 | local c = byte(s, pos) 294 | while c >= 128 and c <= 191 do 295 | pos = pos - 1 296 | c = byte(s, pos) 297 | end 298 | 299 | charbytes = utf8charbytes(s, pos) 300 | 301 | newstr = newstr .. sub(s, pos, pos + charbytes - 1) 302 | 303 | pos = pos - 1 304 | end 305 | 306 | return newstr 307 | end 308 | 309 | -- http://en.wikipedia.org/wiki/Utf8 310 | -- http://developer.coronalabs.com/code/utf-8-conversion-utility 311 | local function utf8char(unicode) 312 | if unicode <= 0x7F then return char(unicode) end 313 | 314 | if (unicode <= 0x7FF) then 315 | local Byte0 = 0xC0 + math.floor(unicode / 0x40); 316 | local Byte1 = 0x80 + (unicode % 0x40); 317 | return char(Byte0, Byte1); 318 | end; 319 | 320 | if (unicode <= 0xFFFF) then 321 | local Byte0 = 0xE0 + math.floor(unicode / 0x1000); 322 | local Byte1 = 0x80 + (math.floor(unicode / 0x40) % 0x40); 323 | local Byte2 = 0x80 + (unicode % 0x40); 324 | return char(Byte0, Byte1, Byte2); 325 | end; 326 | 327 | if (unicode <= 0x10FFFF) then 328 | local code = unicode 329 | local Byte3= 0x80 + (code % 0x40); 330 | code = math.floor(code / 0x40) 331 | local Byte2= 0x80 + (code % 0x40); 332 | code = math.floor(code / 0x40) 333 | local Byte1= 0x80 + (code % 0x40); 334 | code = math.floor(code / 0x40) 335 | local Byte0= 0xF0 + code; 336 | 337 | return char(Byte0, Byte1, Byte2, Byte3); 338 | end; 339 | 340 | error 'Unicode cannot be greater than U+10FFFF!' 341 | end 342 | 343 | local shift_6 = 2^6 344 | local shift_12 = 2^12 345 | local shift_18 = 2^18 346 | 347 | local utf8unicode 348 | utf8unicode = function(str, i, j, byte_pos) 349 | i = i or 1 350 | j = j or i 351 | 352 | if i > j then return end 353 | 354 | local ch,bytes 355 | 356 | if byte_pos then 357 | bytes = utf8charbytes(str,byte_pos) 358 | ch = sub(str,byte_pos,byte_pos-1+bytes) 359 | else 360 | ch,byte_pos = utf8sub(str,i,i), 0 361 | bytes = #ch 362 | end 363 | 364 | local unicode 365 | 366 | if bytes == 1 then unicode = byte(ch) end 367 | if bytes == 2 then 368 | local byte0,byte1 = byte(ch,1,2) 369 | local code0,code1 = byte0-0xC0,byte1-0x80 370 | unicode = code0*shift_6 + code1 371 | end 372 | if bytes == 3 then 373 | local byte0,byte1,byte2 = byte(ch,1,3) 374 | local code0,code1,code2 = byte0-0xE0,byte1-0x80,byte2-0x80 375 | unicode = code0*shift_12 + code1*shift_6 + code2 376 | end 377 | if bytes == 4 then 378 | local byte0,byte1,byte2,byte3 = byte(ch,1,4) 379 | local code0,code1,code2,code3 = byte0-0xF0,byte1-0x80,byte2-0x80,byte3-0x80 380 | unicode = code0*shift_18 + code1*shift_12 + code2*shift_6 + code3 381 | end 382 | 383 | return unicode,utf8unicode(str, i+1, j, byte_pos+bytes) 384 | end 385 | 386 | -- Returns an iterator which returns the next substring and its byte interval 387 | local function utf8gensub(str, sub_len) 388 | sub_len = sub_len or 1 389 | local byte_pos = 1 390 | local length = #str 391 | return function(skip) 392 | if skip then byte_pos = byte_pos + skip end 393 | local char_count = 0 394 | local start = byte_pos 395 | repeat 396 | if byte_pos > length then return end 397 | char_count = char_count + 1 398 | local bytes = utf8charbytes(str,byte_pos) 399 | byte_pos = byte_pos+bytes 400 | 401 | until char_count == sub_len 402 | 403 | local last = byte_pos-1 404 | local slice = sub(str,start,last) 405 | return slice, start, last 406 | end 407 | end 408 | 409 | local function binsearch(sortedTable, item, comp) 410 | local head, tail = 1, #sortedTable 411 | local mid = math.floor((head + tail)/2) 412 | if not comp then 413 | while (tail - head) > 1 do 414 | if sortedTable[tonumber(mid)] > item then 415 | tail = mid 416 | else 417 | head = mid 418 | end 419 | mid = math.floor((head + tail)/2) 420 | end 421 | end 422 | if sortedTable[tonumber(head)] == item then 423 | return true, tonumber(head) 424 | elseif sortedTable[tonumber(tail)] == item then 425 | return true, tonumber(tail) 426 | else 427 | return false 428 | end 429 | end 430 | local function classMatchGenerator(class, plain) 431 | local codes = {} 432 | local ranges = {} 433 | local ignore = false 434 | local range = false 435 | local firstletter = true 436 | local unmatch = false 437 | 438 | local it = utf8gensub(class) 439 | 440 | local skip 441 | for c, _, be in it do 442 | skip = be 443 | if not ignore and not plain then 444 | if c == "%" then 445 | ignore = true 446 | elseif c == "-" then 447 | table.insert(codes, utf8unicode(c)) 448 | range = true 449 | elseif c == "^" then 450 | if not firstletter then 451 | error('!!!') 452 | else 453 | unmatch = true 454 | end 455 | elseif c == ']' then 456 | break 457 | else 458 | if not range then 459 | table.insert(codes, utf8unicode(c)) 460 | else 461 | table.remove(codes) -- removing '-' 462 | table.insert(ranges, {table.remove(codes), utf8unicode(c)}) 463 | range = false 464 | end 465 | end 466 | elseif ignore and not plain then 467 | if c == 'a' then -- %a: represents all letters. (ONLY ASCII) 468 | table.insert(ranges, {65, 90}) -- A - Z 469 | table.insert(ranges, {97, 122}) -- a - z 470 | elseif c == 'c' then -- %c: represents all control characters. 471 | table.insert(ranges, {0, 31}) 472 | table.insert(codes, 127) 473 | elseif c == 'd' then -- %d: represents all digits. 474 | table.insert(ranges, {48, 57}) -- 0 - 9 475 | elseif c == 'g' then -- %g: represents all printable characters except space. 476 | table.insert(ranges, {1, 8}) 477 | table.insert(ranges, {14, 31}) 478 | table.insert(ranges, {33, 132}) 479 | table.insert(ranges, {134, 159}) 480 | table.insert(ranges, {161, 5759}) 481 | table.insert(ranges, {5761, 8191}) 482 | table.insert(ranges, {8203, 8231}) 483 | table.insert(ranges, {8234, 8238}) 484 | table.insert(ranges, {8240, 8286}) 485 | table.insert(ranges, {8288, 12287}) 486 | elseif c == 'l' then -- %l: represents all lowercase letters. (ONLY ASCII) 487 | table.insert(ranges, {97, 122}) -- a - z 488 | elseif c == 'p' then -- %p: represents all punctuation characters. (ONLY ASCII) 489 | table.insert(ranges, {33, 47}) 490 | table.insert(ranges, {58, 64}) 491 | table.insert(ranges, {91, 96}) 492 | table.insert(ranges, {123, 126}) 493 | elseif c == 's' then -- %s: represents all space characters. 494 | table.insert(ranges, {9, 13}) 495 | table.insert(codes, 32) 496 | table.insert(codes, 133) 497 | table.insert(codes, 160) 498 | table.insert(codes, 5760) 499 | table.insert(ranges, {8192, 8202}) 500 | table.insert(codes, 8232) 501 | table.insert(codes, 8233) 502 | table.insert(codes, 8239) 503 | table.insert(codes, 8287) 504 | table.insert(codes, 12288) 505 | elseif c == 'u' then -- %u: represents all uppercase letters. (ONLY ASCII) 506 | table.insert(ranges, {65, 90}) -- A - Z 507 | elseif c == 'w' then -- %w: represents all alphanumeric characters. (ONLY ASCII) 508 | table.insert(ranges, {48, 57}) -- 0 - 9 509 | table.insert(ranges, {65, 90}) -- A - Z 510 | table.insert(ranges, {97, 122}) -- a - z 511 | elseif c == 'x' then -- %x: represents all hexadecimal digits. 512 | table.insert(ranges, {48, 57}) -- 0 - 9 513 | table.insert(ranges, {65, 70}) -- A - F 514 | table.insert(ranges, {97, 102}) -- a - f 515 | else 516 | if not range then 517 | table.insert(codes, utf8unicode(c)) 518 | else 519 | table.remove(codes) -- removing '-' 520 | table.insert(ranges, {table.remove(codes), utf8unicode(c)}) 521 | range = false 522 | end 523 | end 524 | ignore = false 525 | else 526 | if not range then 527 | table.insert(codes, utf8unicode(c)) 528 | else 529 | table.remove(codes) -- removing '-' 530 | table.insert(ranges, {table.remove(codes), utf8unicode(c)}) 531 | range = false 532 | end 533 | ignore = false 534 | end 535 | 536 | firstletter = false 537 | end 538 | 539 | table.sort(codes) 540 | 541 | local function inRanges(charCode) 542 | for _,r in ipairs(ranges) do 543 | if r[1] <= charCode and charCode <= r[2] then 544 | return true 545 | end 546 | end 547 | return false 548 | end 549 | if not unmatch then 550 | return function(charCode) 551 | return binsearch(codes, charCode) or inRanges(charCode) 552 | end, skip 553 | else 554 | return function(charCode) 555 | return charCode ~= -1 and not (binsearch(codes, charCode) or inRanges(charCode)) 556 | end, skip 557 | end 558 | end 559 | 560 | --[[ 561 | -- utf8sub with extra argument, and extra result value 562 | local function utf8subWithBytes (s, i, j, sb) 563 | -- argument defaults 564 | j = j or -1 565 | 566 | local pos = sb or 1 567 | local bytes = len(s) 568 | local length = 0 569 | 570 | -- only set l if i or j is negative 571 | local l = (i >= 0 and j >= 0) or utf8len(s) 572 | local startChar = (i >= 0) and i or l + i + 1 573 | local endChar = (j >= 0) and j or l + j + 1 574 | 575 | -- can't have start before end! 576 | if startChar > endChar then 577 | return "" 578 | end 579 | 580 | -- byte offsets to pass to string.sub 581 | local startByte,endByte = 1,bytes 582 | 583 | while pos <= bytes do 584 | length = length + 1 585 | 586 | if length == startChar then 587 | startByte = pos 588 | end 589 | 590 | pos = pos + utf8charbytes(s, pos) 591 | 592 | if length == endChar then 593 | endByte = pos - 1 594 | break 595 | end 596 | end 597 | 598 | if startChar > length then startByte = bytes+1 end 599 | if endChar < 1 then endByte = 0 end 600 | 601 | return sub(s, startByte, endByte), endByte + 1 602 | end 603 | ]] 604 | 605 | local cache = setmetatable({},{ 606 | __mode = 'kv' 607 | }) 608 | local cachePlain = setmetatable({},{ 609 | __mode = 'kv' 610 | }) 611 | local function matcherGenerator(regex, plain) 612 | local matcher = { 613 | functions = {}, 614 | captures = {} 615 | } 616 | if not plain then 617 | cache[regex] = matcher 618 | else 619 | cachePlain[regex] = matcher 620 | end 621 | local function simple(func) 622 | return function(cC) 623 | if func(cC) then 624 | matcher:nextFunc() 625 | matcher:nextStr() 626 | else 627 | matcher:reset() 628 | end 629 | end 630 | end 631 | local function star(func) 632 | return function(cC) 633 | if func(cC) then 634 | matcher:fullResetOnNextFunc() 635 | matcher:nextStr() 636 | else 637 | matcher:nextFunc() 638 | end 639 | end 640 | end 641 | local function minus(func) 642 | return function(cC) 643 | if func(cC) then 644 | matcher:fullResetOnNextStr() 645 | end 646 | matcher:nextFunc() 647 | end 648 | end 649 | local function question(func) 650 | return function(cC) 651 | if func(cC) then 652 | matcher:fullResetOnNextFunc() 653 | matcher:nextStr() 654 | end 655 | matcher:nextFunc() 656 | end 657 | end 658 | 659 | local function capture(id) 660 | return function(_) 661 | local l = matcher.captures[id][2] - matcher.captures[id][1] 662 | local captured = utf8sub(matcher.string, matcher.captures[id][1], matcher.captures[id][2]) 663 | local check = utf8sub(matcher.string, matcher.str, matcher.str + l) 664 | if captured == check then 665 | for _ = 0, l do 666 | matcher:nextStr() 667 | end 668 | matcher:nextFunc() 669 | else 670 | matcher:reset() 671 | end 672 | end 673 | end 674 | local function captureStart(id) 675 | return function(_) 676 | matcher.captures[id][1] = matcher.str 677 | matcher:nextFunc() 678 | end 679 | end 680 | local function captureStop(id) 681 | return function(_) 682 | matcher.captures[id][2] = matcher.str - 1 683 | matcher:nextFunc() 684 | end 685 | end 686 | 687 | local function balancer(str) 688 | local sum = 0 689 | local bc, ec = utf8sub(str, 1, 1), utf8sub(str, 2, 2) 690 | local skip = len(bc) + len(ec) 691 | bc, ec = utf8unicode(bc), utf8unicode(ec) 692 | return function(cC) 693 | if cC == ec and sum > 0 then 694 | sum = sum - 1 695 | if sum == 0 then 696 | matcher:nextFunc() 697 | end 698 | matcher:nextStr() 699 | elseif cC == bc then 700 | sum = sum + 1 701 | matcher:nextStr() 702 | else 703 | if sum == 0 or cC == -1 then 704 | sum = 0 705 | matcher:reset() 706 | else 707 | matcher:nextStr() 708 | end 709 | end 710 | end, skip 711 | end 712 | 713 | matcher.functions[1] = function(_) 714 | matcher:fullResetOnNextStr() 715 | matcher.seqStart = matcher.str 716 | matcher:nextFunc() 717 | if (matcher.str > matcher.startStr and matcher.fromStart) or matcher.str >= matcher.stringLen then 718 | matcher.stop = true 719 | matcher.seqStart = nil 720 | end 721 | end 722 | 723 | local lastFunc 724 | local ignore = false 725 | local skip = nil 726 | local it = (function() 727 | local gen = utf8gensub(regex) 728 | return function() 729 | return gen(skip) 730 | end 731 | end)() 732 | local cs = {} 733 | for c, bs, be in it do 734 | skip = nil 735 | if plain then 736 | table.insert(matcher.functions, simple(classMatchGenerator(c, plain))) 737 | else 738 | if ignore then 739 | if find('123456789', c, 1, true) then 740 | if lastFunc then 741 | table.insert(matcher.functions, simple(lastFunc)) 742 | lastFunc = nil 743 | end 744 | table.insert(matcher.functions, capture(tonumber(c))) 745 | elseif c == 'b' then 746 | if lastFunc then 747 | table.insert(matcher.functions, simple(lastFunc)) 748 | lastFunc = nil 749 | end 750 | local b 751 | b, skip = balancer(sub(regex, be + 1, be + 9)) 752 | table.insert(matcher.functions, b) 753 | else 754 | lastFunc = classMatchGenerator('%' .. c) 755 | end 756 | ignore = false 757 | else 758 | if c == '*' then 759 | if lastFunc then 760 | table.insert(matcher.functions, star(lastFunc)) 761 | lastFunc = nil 762 | else 763 | error('invalid regex after ' .. sub(regex, 1, bs)) 764 | end 765 | elseif c == '+' then 766 | if lastFunc then 767 | table.insert(matcher.functions, simple(lastFunc)) 768 | table.insert(matcher.functions, star(lastFunc)) 769 | lastFunc = nil 770 | else 771 | error('invalid regex after ' .. sub(regex, 1, bs)) 772 | end 773 | elseif c == '-' then 774 | if lastFunc then 775 | table.insert(matcher.functions, minus(lastFunc)) 776 | lastFunc = nil 777 | else 778 | error('invalid regex after ' .. sub(regex, 1, bs)) 779 | end 780 | elseif c == '?' then 781 | if lastFunc then 782 | table.insert(matcher.functions, question(lastFunc)) 783 | lastFunc = nil 784 | else 785 | error('invalid regex after ' .. sub(regex, 1, bs)) 786 | end 787 | elseif c == '^' then 788 | if bs == 1 then 789 | matcher.fromStart = true 790 | else 791 | error('invalid regex after ' .. sub(regex, 1, bs)) 792 | end 793 | elseif c == '$' then 794 | if be == len(regex) then 795 | matcher.toEnd = true 796 | else 797 | error('invalid regex after ' .. sub(regex, 1, bs)) 798 | end 799 | elseif c == '[' then 800 | if lastFunc then 801 | table.insert(matcher.functions, simple(lastFunc)) 802 | end 803 | lastFunc, skip = classMatchGenerator(sub(regex, be + 1)) 804 | elseif c == '(' then 805 | if lastFunc then 806 | table.insert(matcher.functions, simple(lastFunc)) 807 | lastFunc = nil 808 | end 809 | table.insert(matcher.captures, {}) 810 | table.insert(cs, #matcher.captures) 811 | table.insert(matcher.functions, captureStart(cs[#cs])) 812 | if sub(regex, be + 1, be + 1) == ')' then matcher.captures[#matcher.captures].empty = true end 813 | elseif c == ')' then 814 | if lastFunc then 815 | table.insert(matcher.functions, simple(lastFunc)) 816 | lastFunc = nil 817 | end 818 | local cap = table.remove(cs) 819 | if not cap then 820 | error('invalid capture: "(" missing') 821 | end 822 | table.insert(matcher.functions, captureStop(cap)) 823 | elseif c == '.' then 824 | if lastFunc then 825 | table.insert(matcher.functions, simple(lastFunc)) 826 | end 827 | lastFunc = function(cC) return cC ~= -1 end 828 | elseif c == '%' then 829 | ignore = true 830 | else 831 | if lastFunc then 832 | table.insert(matcher.functions, simple(lastFunc)) 833 | end 834 | lastFunc = classMatchGenerator(c) 835 | end 836 | end 837 | end 838 | end 839 | if #cs > 0 then 840 | error('invalid capture: ")" missing') 841 | end 842 | if lastFunc then 843 | table.insert(matcher.functions, simple(lastFunc)) 844 | end 845 | 846 | table.insert(matcher.functions, function() 847 | if matcher.toEnd and matcher.str ~= matcher.stringLen then 848 | matcher:reset() 849 | else 850 | matcher.stop = true 851 | end 852 | end) 853 | 854 | matcher.nextFunc = function(self) 855 | self.func = self.func + 1 856 | end 857 | matcher.nextStr = function(self) 858 | self.str = self.str + 1 859 | end 860 | matcher.strReset = function(self) 861 | local oldReset = self.reset 862 | local str = self.str 863 | self.reset = function(s) 864 | s.str = str 865 | s.reset = oldReset 866 | end 867 | end 868 | matcher.fullResetOnNextFunc = function(self) 869 | local oldReset = self.reset 870 | local func = self.func +1 871 | local str = self.str 872 | self.reset = function(s) 873 | s.func = func 874 | s.str = str 875 | s.reset = oldReset 876 | end 877 | end 878 | matcher.fullResetOnNextStr = function(self) 879 | local oldReset = self.reset 880 | local str = self.str + 1 881 | local func = self.func 882 | self.reset = function(s) 883 | s.func = func 884 | s.str = str 885 | s.reset = oldReset 886 | end 887 | end 888 | 889 | matcher.process = function(self, str, start) 890 | 891 | self.func = 1 892 | start = start or 1 893 | self.startStr = (start >= 0) and start or utf8len(str) + start + 1 894 | self.seqStart = self.startStr 895 | self.str = self.startStr 896 | self.stringLen = utf8len(str) + 1 897 | self.string = str 898 | self.stop = false 899 | 900 | self.reset = function(s) 901 | s.func = 1 902 | end 903 | 904 | -- local lastPos = self.str 905 | -- local lastByte 906 | local ch 907 | while not self.stop do 908 | if self.str < self.stringLen then 909 | --[[ if lastPos < self.str then 910 | print('last byte', lastByte) 911 | ch, lastByte = utf8subWithBytes(str, 1, self.str - lastPos - 1, lastByte) 912 | ch, lastByte = utf8subWithBytes(str, 1, 1, lastByte) 913 | lastByte = lastByte - 1 914 | else 915 | ch, lastByte = utf8subWithBytes(str, self.str, self.str) 916 | end 917 | lastPos = self.str ]] 918 | ch = utf8sub(str, self.str,self.str) 919 | --print('char', ch, utf8unicode(ch)) 920 | self.functions[self.func](utf8unicode(ch)) 921 | else 922 | self.functions[self.func](-1) 923 | end 924 | end 925 | 926 | if self.seqStart then 927 | local captures = {} 928 | for _,pair in pairs(self.captures) do 929 | if pair.empty then 930 | table.insert(captures, pair[1]) 931 | else 932 | table.insert(captures, utf8sub(str, pair[1], pair[2])) 933 | end 934 | end 935 | return self.seqStart, self.str - 1, unpack(captures) 936 | end 937 | end 938 | 939 | return matcher 940 | end 941 | 942 | -- string.find 943 | local function utf8find(str, regex, init, plain) 944 | local matcher = cache[regex] or matcherGenerator(regex, plain) 945 | return matcher:process(str, init) 946 | end 947 | 948 | -- string.match 949 | local function utf8match(str, regex, init) 950 | init = init or 1 951 | local found = {utf8find(str, regex, init)} 952 | if found[1] then 953 | if found[3] then 954 | return unpack(found, 3) 955 | end 956 | return utf8sub(str, found[1], found[2]) 957 | end 958 | end 959 | 960 | -- string.gmatch 961 | local function utf8gmatch(str, regex, all) 962 | regex = (utf8sub(regex,1,1) ~= '^') and regex or '%' .. regex 963 | local lastChar = 1 964 | return function() 965 | local found = {utf8find(str, regex, lastChar)} 966 | if found[1] then 967 | lastChar = found[2] + 1 968 | if found[all and 1 or 3] then 969 | return unpack(found, all and 1 or 3) 970 | end 971 | return utf8sub(str, found[1], found[2]) 972 | end 973 | end 974 | end 975 | 976 | local function replace(repl, args) 977 | local ret = '' 978 | if type(repl) == 'string' then 979 | local ignore = false 980 | local num 981 | for c in utf8gensub(repl) do 982 | if not ignore then 983 | if c == '%' then 984 | ignore = true 985 | else 986 | ret = ret .. c 987 | end 988 | else 989 | num = tonumber(c) 990 | if num then 991 | ret = ret .. args[num] 992 | else 993 | ret = ret .. c 994 | end 995 | ignore = false 996 | end 997 | end 998 | elseif type(repl) == 'table' then 999 | ret = repl[args[1] or args[0]] or '' 1000 | elseif type(repl) == 'function' then 1001 | if #args > 0 then 1002 | ret = repl(unpack(args, 1)) or '' 1003 | else 1004 | ret = repl(args[0]) or '' 1005 | end 1006 | end 1007 | return ret 1008 | end 1009 | -- string.gsub 1010 | local function utf8gsub(str, regex, repl, limit) 1011 | limit = limit or -1 1012 | local ret = '' 1013 | local prevEnd = 1 1014 | local it = utf8gmatch(str, regex, true) 1015 | local found = {it()} 1016 | local n = 0 1017 | while #found > 0 and limit ~= n do 1018 | local args = {[0] = utf8sub(str, found[1], found[2]), unpack(found, 3)} 1019 | ret = ret .. utf8sub(str, prevEnd, found[1] - 1) 1020 | .. replace(repl, args) 1021 | prevEnd = found[2] + 1 1022 | n = n + 1 1023 | found = {it()} 1024 | end 1025 | return ret .. utf8sub(str, prevEnd), n 1026 | end 1027 | 1028 | local utf8 = {} 1029 | utf8.len = utf8len 1030 | utf8.sub = utf8sub 1031 | utf8.reverse = utf8reverse 1032 | utf8.char = utf8char 1033 | utf8.unicode = utf8unicode 1034 | utf8.gensub = utf8gensub 1035 | utf8.byte = utf8unicode 1036 | utf8.find = utf8find 1037 | utf8.match = utf8match 1038 | utf8.gmatch = utf8gmatch 1039 | utf8.gsub = utf8gsub 1040 | utf8.dump = dump 1041 | utf8.format = format 1042 | utf8.lower = lower 1043 | utf8.upper = upper 1044 | utf8.rep = rep 1045 | return utf8 1046 | --------------------------------------------------------------------------------